View of /CityKid/citykid/data.py
Parent Directory
| Revision Log
Revision:
2720 -
(
download)
(
as text)
Mon Jun 2 04:24:35 2014 UTC (9 years, 10 months ago) by
pje
File size: 12256 byte(s)
Add dump command, to export raw CityDesk data w/.json sidecars
and/or front matter
from peak.persistence import Persistent
from peak.api import *
from datetime import datetime
from peak.util.imports import importString
import re, os, shutil, json, mimetypes
from pkg_resources import ensure_directory
dt = lambda d: datetime.fromordinal(d.toordinal())
dt.str = lambda d: d.strftime("%Y-%m-%d %H:%M:%S")
FOG = re.compile('PTMFOG(\d+)(\.[A-Za-z]*)?')
BRACKET = re.compile(r'\[\[([^]]+)]]')
def sql_equals(data, clauses=(), sep=' AND ', pre=' WHERE '):
clauses = [c for c in clauses if c]
params = []
for k,v in data.items():
clauses.append(k+'=?')
params.append(v)
sql = sep.join(clauses)
if sql:
sql = pre+sql
return sql, params
def sqlSET(data, clauses=()):
return sql_equals(data, clauses, ', ', ' SET ')
def sqlWHERE(data, clauses=()):
return sql_equals(data, clauses)
try:
sorted = sorted
except NameError:
def sorted(seq,key=None):
if key:
d = [(key(v),v) for v in seq]
d.sort()
if key:
d = [v[1] for v in d]
return d
class SQL_DM(storage.EntityDM):
db = binding.Obtain('db')
table = binding.Require('table name')
key = binding.Require('primary key column')
_select = binding.Make(lambda self: "select * from "+self.table)
_filter = ''
read_key = binding.Obtain('key')
def _load(self, oid, ob):
for row in self._where({self.key:oid}):
return self.stateForRow(row)
else:
raise storage.InvalidKeyError
def _where(self, args):
where, what = sqlWHERE(args, [self._filter])
return self.db(self._select + where, what)
def where(self, **kw):
key = self.read_key
state = self.stateForRow
for row in self._where(kw):
yield self.preloadState(row[key], state(row))
__iter__ = where
def _save(self, ob):
for tbl,values,wheres in self.rowsForOb(ob):
sWhere, what = sqlWHERE(wheres)
sSet, values = sqlSET(values)
self.db('UPDATE '+tbl+sSet+sWhere, values+what)
class Base(Persistent):
class __metaclass__(binding.Activator, Persistent.__class__): pass
class Item(Base):
"""An item that's editable in CityDesk"""
def __setstate__(self, state):
self.__class__ = item_types[state['_itemType']]
self._p_changed = False
super(Item,self).__setstate__(state)
path = binding.Make(
lambda self:
(self.parent and self.parent.path+'/' or '') + self.publishAs
+ self.ext
)
def linkTo(self,tgt,root=None):
if root:
return root[:len(root)-root.endswith('/')]+'/'+tgt.path
src, tgt = self.path.split('/'), tgt.path.split('/')
if src==tgt:
return tgt[-1]
while src and tgt and src[0]==tgt[0]:
del src[0], tgt[0]
dots = len(src) - bool(tgt)
return '/'.join( ['..']*dots + tgt)
# (1) : (2/3/4) -> ../../1 src - 1
# 1/2 : 1/2/(3) -> .. src
# 1/(2) : 1/(3) -> 2 src - 1
# 6/(3/x) : 6/(4/y) -> ../3/x src - 1
parent = None
ext = '' # XXX
active = 1
def save(self, dir):
dir = os.path.dirname(self.filename(dir))
if not os.path.isdir(dir):
os.makedirs(dir)
def filename(self, dir):
return os.path.join(dir, *self.path.split('/'))
def _write_file(self, dir, data, suffix=''):
Item.save(self, dir)
filename = self.filename(dir)+suffix
if os.path.isfile(filename):
old = open(filename,'rb').read()
if old==str(data):
return
open(filename,'wb').write(data)
def _dump(self, dir, metadata, data):
metadata = json.dumps(metadata, sort_keys=True, indent=0)
if is_text(self.path.split('/')[-1]):
self._write_file(dir,
"---\n%s\n---\n%s" % (metadata, data)
)
else:
self._write_file(dir, data)
self._write_file(dir, metadata, '.json')
def is_text(filename):
if filename=='.htaccess':
return True
mt = mimetypes.guess_type(filename, False)[0]
if mt:
return mt.startswith('text/')
class Article(Item):
"""An article"""
def __repr__(self):
return "Article(%s)" % (self.__dict__,)
_article = binding.Make(
lambda self: ~self._p_jar.db(
"select * from tblArticle where ixArticle=?", (self._articleNo,)
)
)
body = binding.Make(lambda self: str(self._article.sArticle).decode('utf16'))
author = binding.Obtain("_article/sAuthor")
teaser = binding.Obtain("_article/sTeaser")
sidebar = binding.Obtain("_article/sSidebar")
about = binding.Obtain("_article/sAboutTheAuthor")
extra1 = binding.Obtain("_article/sExtra1")
extra2 = binding.Obtain("_article/sExtra2")
# get the extension and 'kid' objects from the template
kid = ext = binding.Delegate("template")
def __call__(self, **kw):
text = self.template(my=self, **kw).serialize(output=self.format)
return FOG.sub(self.defog, text)
def defog(self, match, root=None):
item = self._p_jar[int(match.group(1))]
return self.linkTo(item, root)
def active(self):
now = datetime.now()
return (
(self.effective is None or dt(self.effective)<=now) and
(self.retire is None or dt(self.retire) >=now)
)
active = binding.Make(active)
text = binding.Make(lambda self: FOG.sub(self.defog, self.body))
def teaser_for(self, tgt, root=None):
return BRACKET.sub(
lambda m: '<a href="%s#begin-post">%s</a>' %
(self.linkTo(tgt,root), m.group(1)),
tgt.teaser
)
def save(self, dir):
self._write_file(dir, self())
format = binding.Make(lambda self: self.ext=='xml' and 'xml' or 'xhtml')
def abstext(self, root=None):
return FOG.sub(lambda t: self.defog(t,root), self.body)
_dump_map = dict([(k,k) for k in [
"author", "teaser", "sidebar", "about", "extra1", "extra2", "headline",
"audience", "keywords"
]], **{"Fog-ID":"_p_oid"})
del k
def dump(self, dir):
Item.save(self, dir)
#self._write_file(dir, self.body.encode('utf8'))
data = dict([
(k, getattr(self,v)) for k,v in self._dump_map.items()
if getattr(self,v)
])
for k in ["effective", "retire", "filed", "modified"]:
v = getattr(self, k, None)
if v:
data[k] = dt.str(v)
data["template"] = self.template.name
#data = json.dumps(data, sort_keys=True, indent=0)
#data = '---\n%s\n---\n%s' % (data, self.body.encode('utf8'))
self._dump(dir, data, self.body.encode('utf8')) #, '.json')
class Folder(Item):
"""A Folder"""
def delete_orphans(self, dir, items=None, dump=False):
"""Remove files or dirs that aren't in `items`"""
if items is None:
items = self._p_jar.where(ixParent=self._p_oid)
# Build a mapping of known item file/dir names
d = dict.fromkeys(
[item.path.split('/')[-1] for item in items if item.parent is self]
)
if dump:
for key in d.keys():
if not is_text(key):
d[key+'.json'] = None
filename = self.filename(dir)
for f in os.listdir(filename):
if f not in d:
f = os.path.join(filename, f)
if os.path.isdir(f) and not os.path.islink(f):
shutil.rmtree(f)
else:
os.unlink(f)
def save(self, dir):
dir = self.filename(dir)
if not os.path.isdir(dir):
os.makedirs(dir)
dump = save
class File(Item):
"""A File"""
raw_data = binding.Make(
lambda self:
(~self._p_jar.db(
"select oleFile from tblFiles WHERE ixStructure=?", (self._p_oid,)
))[0]
)
def save(self, dir):
Item.save(self, dir)
data = self.raw_data
if self.path.endswith('.html'):
def defog(match, root=None):
item = self._p_jar[int(match.group(1))]
return self.linkTo(item, root)
data = FOG.sub(defog, data)
self._write_file(dir, data)
def dump(self, dir):
Item.save(self, dir)
self._write_file(dir, self.raw_data)
self._dump(dir, {"Fog-ID": self._p_oid}, self.raw_data)
item_types = {
1.0: Folder,
2.0: Article,
3.0: File,
}
class Template(Base):
"""A CityDesk template"""
kid = binding.Make(
lambda self: self.load_template(str(self.body).replace('\000',''))
)
name = binding.Make(
lambda self:(~self._p_jar.db(
"select * from tblTemplateStructure where ixTemplateStructure=?", (self._p_oid,)
)).sName
)
def __call__(self, **kw):
return self.call_template(self.kid, **kw)
call_template = load_template = binding.Delegate('_p_jar')
class TemplateDM(SQL_DM):
resetStatesAfterTxn = False
defaultClass = Template
table = 'tblTemplate'
key = 'ixTemplateStructure'
ixFamily = binding.Obtain('ixFamily')
_filter = binding.Make(
lambda self: 'fDeleted=0 and ixTemplateFamily=%d' % self.ixFamily
)
def stateForRow(self, row):
return dict(ext = str(row.sExt), body = row.utf8Template)
cmd = binding.Obtain('..')
call_template = load_template = binding.Delegate('cmd')
class ItemDM(SQL_DM):
resetStatesAfterTxn = False
defaultClass = Item
table = 'tblStructure'
key = 'i.ixStructure'
read_key = 'ixStructure'
_filter = binding.Make(
lambda self:
'i.fDeleted=0 AND (ixLanguage=%d OR ixLanguage IS NULL)'
% self.ixLanguage
)
_select = """
SELECT iType, ixItem, sName, sPublishAs, sKeywords, ixTemplateStructure,
dtModified, dtFiled, dtEffective, dtRetire, ixParent, sHeadline,
ixArticle, ixAudience, i.ixStructure
FROM (tblStructure i LEFT OUTER JOIN tblArticleSet s ON s.ixSet=i.ixItem)
LEFT OUTER JOIN tblArticle a ON a.ixSet=s.ixSet
"""
ixLanguage = binding.Obtain('ixLanguage')
templateDM = binding.Obtain('templateDM')
def stateForRow(self, row):
state = dict(
_itemType = row.iType, _itemNo = row.ixItem,
name = row.sName, publishAs = str(row.sPublishAs),
keywords = row.sKeywords,
filed = row.dtFiled,
effective = row.dtEffective,
retire = row.dtRetire,
modified = row.dtModified,
headline = row.sHeadline,
_articleNo = row.ixArticle,
audience = self.audiences[row.ixAudience]
)
if row.ixTemplateStructure:
state['template'] = self.templateDM[row.ixTemplateStructure]
if row.ixParent:
state['parent'] = self[row.ixParent]
return state
def rowsForOb(self, ob):
yield (
'tblStructure',
dict(sName=ob.name, sPublishAs=ob.publishAs),
dict(ixStructure=ob._p_oid)
)
if isinstance(ob,Article):
yield (
"tblArticleSet", dict(
dtFiled=ob.filed, dtEffective=ob.effective,
dtRetire=ob.retire
),
dict(ixSet=ob._itemNo)
)
yield (
'tblArticle', dict(
sExtra1=ob.extra1, sExtra2=ob.extra2, sTeaser=ob.teaser,
sHeadline=ob.headline, sSidebar=ob.sidebar,
sAboutTheAuthor=ob.about
),
dict(ixArticle=ob._articleNo)
)
audiences = binding.Make(
lambda self: dict(
list(self.db(
"SELECT ixAudience,sAudience FROM tblAudience WHERE fDeleted=0"
)) + [(0, "(Everyone)"), (None, None)]
)
)