"""Command line generic functionality
This module implements basic functionality of a command line client,
but without the parsing and other stuff. Thus, a client cares only
about interacting with the shell or GUI and can use the API exported
by this module, without caring about cfvers internals.
"""
# Copyright 2003-2005 Iustin Pop
#
# This file is part of cfvers.
#
# cfvers is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# cfvers is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with cfvers; if not, write to the Free Software Foundation,
# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
# $Id: cmd.py 218 2005-10-30 09:26:23Z iusty $
import os, struct, stat, os.path, re, commands, sys
import types
import random
import errno
from mx import DateTime
from ConfigParser import SafeConfigParser
import tarfile
from cStringIO import StringIO
import hmac, sha
import sets
import Pyro.core
import Pyro.naming
import cfvers
import cfvers.main
import cfvers.gateway
from cfvers.main import *
__all__ = [
"AdminCommands", "Commands",
]
class BulkTransport(object):
"""Class used in bulk transfers through the portal"""
def __init__(self, portal, isadditem, revno=None):
self.buffer = []
self.portal = portal
self.currsize = 0
self.results = []
self.isadditem = isadditem
self.revno = revno
def add(self, item):
self.buffer.append(item)
if not self.isadditem and item.filecontents is not None:
self.currsize += len(item.filecontents)
if len(self.buffer) >= 1000 or \
(not self.isadditem and self.currsize >= 1048576):
self.flush()
return
def flush(self):
if self.isadditem:
res = self.portal.bulkAddItem(self.revno, self.buffer)
else:
res = self.portal.bulkAddEntry(self.buffer)
self.results.extend(res)
self.buffer = []
self.currsize = 0
return
class CLIScript(object):
"""Base class for command line handlers"""
def __init__(self, options, prompt_func):
Pyro.core.initClient(0)
self.readconfig(options, prompt_func)
self.options = options
if options.server_type == "remote":
authenticator = (options.username, options.client_password)
factoryURI = "PYROLOC://%s:%s/PortalFactory" % (options.host, options.port)
factory = Pyro.core.getProxyForURI(factoryURI)
factory._setNewConnectionValidator(cfvers.gateway.PortalValidator())
factory._setIdentification(authenticator)
try:
self.portal = factory.getPortal()
except Pyro.errors.ProtocolError, e:
raise cfvers.CommException(*e.args)
self.portal._setNewConnectionValidator(cfvers.gateway.PortalValidator())
self.portal._setIdentification(authenticator)
if not self.authPortal():
raise ValueError("Server failed to authenticate")
else:
self.portal = cfvers.gateway.Portal(local=True,
repo=(options.repo_meth,options.repo_data))
return
def authPortal(self):
token = []
for i in range(20):
token.append(chr(random.randint(32, 127)))
token = "".join(token)
result = self.portal.checkID(token)
if result is None:
return False
preamble, postamble, stuff = result
hm = hmac.new(self.options.server_password, preamble, sha)
hm.update(token)
hm.update(postamble)
mydata = hm.hexdigest()
if mydata != stuff:
return False
return True
def readconfig(options, prompt_func):
def aquire(name):
envval = os.environ.get("CFVERS_%s" % name.upper())
if envval is not None:
setval = envval
elif cp.has_option("server", name):
setval = cp.get("server", name)
else:
setval = prompt_func(name)
return setval
cp = SafeConfigParser()
cp.read(["/etc/cfvers/client.conf", os.path.expanduser("~/.cfvers"),])
st = getattr(options, "server_type", None)
if st is None:
st = aquire("server_type")
options.server_type = st
if st == "local":
attlist = ('repo_meth', 'repo_data', 'area')
elif st == "remote":
attlist = ('host', 'port', 'username', 'client_password', 'server_password', 'area')
else:
raise ValueError("Invalid server type '%s'" % st)
for optname in attlist:
if getattr(options, optname, None) is not None:
continue
setattr(options, optname, aquire(optname))
return
readconfig = staticmethod(readconfig)
def validconfig(self):
st = getattr(self.options, "server_type", None)
if st is None:
return False
if st == "local":
attrlist = ('repo_meth', 'repo_data', 'area')
elif st == "remote":
attrlist = ('host', 'port', 'username', 'client_password', 'server_password', 'area')
else:
return False
for attr in attrlist:
if getattr(self.options, attr, None) is None:
return False
return True
def get_version():
nv = "%d.%d.%d (%s release)" % cfvers.version_info[0:4]
version="""%%prog - cfvers %s
Copyright (C) 2003-2005 Iustin Pop
This is free software; see the source for copying conditions.
There is NO warranty ; not even for MERCHANTABILITY or FITNESS
FOR A PARTICULAR PURPOSE.""" % nv
return version
get_version = staticmethod(get_version)
def parserev(revstring):
"""Tries to parse a revision string of form num[:num]"""
if revstring is None:
return None, None
# match and extract from glob rev[:rev]
m=re.match("^((?P<start>[0-9]+):)?(?P<end>[0-9]+)$", revstring)
if m is None:
raise ValueError, "Invalid syntax in revision specification %s" % revstring
r1 = m.group('start')
r2 = m.group('end')
if r1 is not None:
r1 = int(r1)
if r2 is not None:
r2 = int(r2)
return r1, r2
parserev = staticmethod(parserev)
class AdminCommands(CLIScript):
def __init__(self, options, prompt_func, do_connect=True):
"""Constructor for the Commands class"""
super(AdminCommands, self).__init__(options, prompt_func)
if do_connect:
self.portal.connect()
def open(self):
self.portal.connect()
def close(self):
self.portal.disconnect()
def create_area(self, cmdoptions, name):
if self.portal.getArea(name) is not None:
raise cfvers.OperationError("Area `%s' already exists" % name)
a = Area(name=name, description=cmdoptions.description,
root=cmdoptions.root)
self.portal.addArea(a)
self.portal.commit()
return
def init_repo(self, createopts):
self.portal.create(createopts)
return
class Commands(CLIScript):
def __init__(self, options, prompt_func):
"""Constructor for the Commands class"""
super(Commands, self).__init__(options, prompt_func)
if not self.validconfig():
raise cfvers.ConfigException, "Incomplete configuration"
self.portal.connect()
self.area = self.portal.getArea(options.area)
if self.area is None:
raise cfvers.ConfigException, "can't find area named '%s'" % options.area
return
def close(self):
self.portal.disconnect()
return
def _map_files(self, area, files=None, norecurse=False):
"""Maps a (possibly empty) list of files to a list of targets
Args:
- area: the area in which to work
- files: a list (possibly empty) of which item to process; if
it is none, all the items in the area are processed
- norecurse: don't descend directories
Special care must be taken related to handling deleted items:
- an existing item (in the repo) which was deleted from the
filesystem must pass the lstat test
- a deleted item (in the repo) which is not in the filesystem
should pass completely the stats
- a deleted item (in the repo) which is in the filesystem
must be normally registered
"""
def helper(arg, dirname, names):
mylist, path = arg
# Translate from real root to area root
if path == "/":
realdir = dirname
elif dirname == path:
realdir = "/"
elif dirname.startswith(path):
realdir = dirname[len(path):]
else:
raise ValueError, "Path doesn't lie in its area!"
mylist.extend([os.path.join(realdir, fname) for fname in names])
return
# Step 1. If no files where given, act as if all items in the
# repository where given as arguments
if len(files) == 0:
files = [item.name for item in self.portal.getItems(area.name)]
# Step 2. Now we have a list of files to act on (add,
# store). For each file, try to lstat. If successfull and is
# dir and no-recurse is false, descend it.
files = map(os.path.abspath, files)
files = dict.fromkeys(files).keys()
targets = []
errors = []
for virtf in files:
realf = forcejoin(area.root, virtf)
# Always mark to process
targets.append(virtf)
try:
st = os.lstat(realf)
except EnvironmentError, e:
continue
# If we got here, the file at least exists,
# so check to see if we can descend
if not norecurse and stat.S_ISDIR(st.st_mode):
os.path.walk(realf, helper, (targets, area.root))
targets = dict.fromkeys(targets).keys()
targets.sort()
return (targets, errors)
def add(self, files=[], options=None):
"""Register a set of files in the repository.
Parameters:
- files: a list of filenames
- logmsg: the log message
A new revision won't be stored if no items have been stored
(e.g. all existing, or no items with valid names).
"""
logmsg = options.logmsg
norecurse = options.norecurse
a = self.area
(targets, errors) = self._map_files(a, files=files, norecurse=norecurse)
ar = Revision(area=a, logmsg=logmsg, commiter=options.commiter)
ar = self.portal.putRevision(ar)
newrev = ar.revno
stored = 0
itemdict = dict([(i.name, i) for i in self.portal.getItems(a.name)])
bulk = BulkTransport(self.portal, True, newrev)
for name in targets:
if name in itemdict:
errors.append(Result(Result.ADDED_EXISTING, item=itemdict[name]))
continue
item = Item(area=a.name, name=name,
flags=options.flags)
bulk.add(item)
bulk.flush()
for item, entry in bulk.results:
errors.append(Result(Result.ADDED_OK, item=item, entry=entry))
stored += 1
if stored > 0:
self.portal.commit()
store_done = True
else:
self.portal.rollback()
store_done = False
return errors, store_done, newrev
def addfromdirs(self, options=None):
"""Register new files in tracked directories.
Parameters:
- logmsg: the log message
A new revision won't be stored if no items have been stored
(e.g. all existing, or no items with valid names).
"""
logmsg = options.logmsg
a = self.area
ar = Revision(area=a, logmsg=logmsg, commiter=options.commiter)
ar = self.portal.putRevision(ar)
newrev = ar.revno
items = self.portal.getItems(a.name)
stored = 0
bulk = BulkTransport(self.portal, True, newrev)
errors = []
existing = [item.name for item in items]
worktodo = items
while len(worktodo) > 0:
newitems = []
newnames = []
for item in worktodo:
name = item.name
try:
dirfiles = os.listdir(name)
except OSError, e:
if e.errno in (errno.ENOENT, errno.ENOTDIR):
continue
if e.errno == errno.EACCES:
errors.append(Result(Result.ADDED_EACCES, item=item))
continue
raise
for fname in dirfiles:
absname = os.path.abspath(os.path.join(name, fname))
if absname not in existing and absname not in newnames:
newitem = Item(area=a.name, name=absname,
flags=item.flags)
if not options.norecurse:
newitems.append(newitem)
newnames.append(absname)
bulk.add(newitem)
worktodo = newitems
existing.extend(newnames)
bulk.flush()
for item, entry in bulk.results:
errors.append(Result(Result.ADDED_OK, item=item, entry=entry))
stored += 1
if stored > 0:
self.portal.commit()
store_done = True
else:
self.portal.rollback()
store_done = False
return errors, store_done, newrev
def register(self, name, cmdline, options):
"""Register a virtual item in the repository.
Parameters:
- name: the virtual path; must be a valid path and should not exist
- cmdline: command line, list
A new revision won't be stored if no items have been stored
(e.g. all existing, or no items with valid names).
"""
a = self.area
ar = Revision(area=a, logmsg=options.logmsg,
commiter=options.commiter)
ar = self.portal.putRevision(ar)
newrev = ar.revno
item = self.portal.getItemByName(a.name, name)
if item is not None:
res = Result(Result.ADDED_EXISTING, item=item)
return res, False, newrev
item = Item(area=a.name, name=name, flags=Item.STORE_VIRTUAL,
command=" ".join(cmdline))
i, e = self.portal.addItem(newrev, item)
res = Result(Result.ADDED_OK, item=i, entry=e)
self.portal.commit()
return res, True, newrev
def store(self, files=[], options=None):
"""Store a set of files or all the items already in the repository.
Parameters:
- files: a list of filename, or empty if all the items should be (re)commited
- logmsg: the log message
A new revision won't be stored if no items have been stored (e.g. nothing changed,
or no items could be read).
"""
logmsg = options.logmsg
norecurse = options.norecurse
a = self.area
(targets, errors) = self._map_files(a, files=files, norecurse=norecurse)
ar = Revision(area=a, logmsg=logmsg, commiter=options.commiter)
ar = self.portal.putRevision(ar)
newrev = ar.revno
itemdict = dict([(i.name, i) for i in self.portal.getItems(a.name)])
bulk = BulkTransport(self.portal, False)
for name in targets:
if not name in itemdict:
errors.append(Result(Result.STORED_NOTREG, fname=name))
continue
item = itemdict[name]
if item.flags & (Item.STORE_METADATA | Item.STORE_CHECKSUM | \
Item.STORE_CONTENTS | Item.STORE_VIRTUAL ) == 0:
# Skip over this item which is marked not to be stored
errors.append(Result(Result.STORED_TOSKIP, item=item))
continue
try:
entry = Entry(item=item, revno=newrev, area=a)
except EnvironmentError, e:
errors.append(Result(Result.STORED_IOERROR, item=item, exc=e))
continue
bulk.add(entry)
bulk.flush()
errors.extend(bulk.results)
stored = 0
for r in errors:
if r.code in (Result.STORED_OK, Result.STORED_DELETED):
stored += 1
if stored > 0:
self.portal.commit()
store_done = True
else:
self.portal.rollback()
store_done = False
return errors, store_done, newrev
def _map_dir_diff(self, ao, no, an, nn, ilist):
ilo = self.portal.getItemsByDirname(ao, no)
iln = self.portal.getItemsByDirname(an, nn)
silo = sets.Set([x.name for x in ilo])
siln = sets.Set([x.name for x in iln])
for name in silo - siln:
yield (ao, name, an, name, True, None, [("File %s exists only in `old'" % name,)], None, None)
for name in siln - silo:
yield (ao, name, an, name, True, None, [("File %s exists only in `new'" % name,)], None, None)
for name in silo & siln:
oitem = [x for x in ilo if x.name == name][0]
nitem = [x for x in iln if x.name == name][0]
ilist.append((oitem, nitem))
for r in self._map_dir_diff(ao, name, an, name, ilist):
yield r
return
def diff(self, options=None, files=None):
"""Diff command implementation
Parameters:
- options: revision and checks are used
- files: list of files to compare (by default all are)
Return value: list of
- name (FIXME: when comparing different items this does not
make sense)
- status: True/False, meaning if the diff was successfully computed
- error message when status == False
- diff data when status == True
- old entry
- new entry
If the listonly options is selected, the diff data is an empty
tuple, meaning the `name' item is different. If listonly is
not selected, the diff data is a list of tuples of:
- one element: should be printed as-is, it usually means a
general diff conclusion
- two elements: the first element is the attribute name and
the second element is the already formated diff
- three elements: the first element is the name of attribute
with the old data in second element and new data in the third
element.
"""
ap = re.compile("^([\w]+):(/.*)$")
rev1, rev2 = self.parserev(options.rev)
if files is None or len(files) == 0:
ilist = [(i, i) for i in self.portal.getItems(self.area.name)]
else:
ilist = []
old = None
while len(files) > 0:
fn = files.pop(0)
match = ap.match(fn)
if match is not None:
area, fn = match.groups()
else:
area = self.area
item = self.portal.getItemByName(area.name, os.path.abspath(fn))
if item is None:
yield area, fn, None, None, False, "Item does not exist in the area", (), None, None
if old is None: # Consume also the next item
files.pop(0)
else:
old = None # Clear the already processed item
continue
if old is None:
old = item
else:
ilist.append((old, item))
for r in self._map_dir_diff(old.area, old.name, item.area, item.name, ilist):
yield r
old = None
if old is not None:
ilist.append((old, old))
for r in self._map_dir_diff(old.area, old.name, old.area, old.name, ilist):
yield r
for vo, vn in ilist:
r = self._diff_file(vo, vn, rev1, rev2, options)
if r is not None:
yield (vo.area, vo.name, vn.area, vn.name) + r
return
def _diff_file(self, vo, vn, rev1, rev2, options):
"""Execute a diff between two revisions.
Parameters:
- vo: the `old' item
- vn: the `new' item
- rev1: the `old' revision
- rev2: the `new' revision
Returns:
- status
- message
- diff data
- old entry
- new entry
See docstring for the diff method for more details on the return data.
"""
# rev1 is older (source)
# rev2 is newer (target)
if rev1 is None:
rev1 = rev2
rev2 = None
do_payload = "filecontents" in options.checks
e1 = self.portal.getEntry(vo.id, rev1, do_payload=do_payload)
if e1 is None or e1.status == Entry.STATUS_ADDED:
if rev1 is None:
# We didn't manage to aquire last revision for item,
# it means it has no revisions
return False, "Item doesn't have any revisions containing data", [], None, None
else:
return False, "Item doesn't have revision entry %d or the entry does not contain data" % rev1, [], None, None
if rev2 is None:
# rev2 has not been specified, thus the user wants to check
# against the filesystem; build from filesystem, if
# possible
if vn.flags & Item.STORE_VIRTUAL or cfvers.rexists(vn.name):
try:
e2 = Entry(item=vn, area=self.portal.getArea(vn.area))
except (IOError, OSError), e:
return False, "Can't read current status, error: %s" % e, [], e1, None
else:
return True, None, [("File has been deleted",)], e1, Entry.newDeleted(vn, rev2)
else:
e2 = self.portal.getEntry(vn.id, rev2, do_payload=do_payload)
if e2 is None or e2.status == Entry.STATUS_ADDED:
return False, "Item doesn't have revision entry %d or the entry does not contain data" % rev2, [], e1, e2
if e1.status != Entry.STATUS_MODIFIED or e2.status != Entry.STATUS_MODIFIED:
return True, None, [("Revisions status: %s, %s" % \
(Entry.STATUS_MAP[e1.status],
Entry.STATUS_MAP[e2.status]))], \
e1, e2
if options.list:
if not e2.compare(e1, options.checks):
return True, None, [], None, None
else:
output = e2.diff(e1, options=options)
if len(output) > 0:
return True, None, output, e1, e2
return
def retrieve(self, files=None, options=None):
results = []
if len(files) != 0:
mlist = []
for filename in files:
af = os.path.abspath(filename)
item = self.portal.getItemByName(self.area.name, af)
if item is None:
results.append(Result(Result.RETR_NTRACK, fname=af))
continue
mlist.append(item)
if not options.norecurse:
mlist += self._recur_build_list(mlist, [])
else:
mlist = self.portal.getItems(self.area.name)
self._retrieve_list(mlist, options, results)
return results
def _recur_build_list(self, litems, alsoskip):
nlist = []
for item in litems:
others = self.portal.getItemsByDirname(self.area.name, item.name)
for it in others:
if it not in [x.name for x in litems] and \
it not in [x.name for x in alsoskip] and \
it not in [x.name for x in nlist]:
nlist.append(it)
nlist += self._recur_build_list([it], nlist + litems)
return nlist
def _retrieve_file(self, vi, entry, options=None):
return entry.to_filesys(destdir=options.destdir, use_dirs=options.use_dirs)
def _retrieve_list(self, ilist, options, results):
for item in ilist:
entry = self.portal.getEntry(item.id, options.revno)
if entry is None:
elist = self.portal.getEntryList(item.id)
if len(elist) == 0:
ncod = Result.RETR_NOREVS
else:
ncod = Result.RETR_NOXREV
results.append(Result(ncod, item=item))
else:
results.append(self._retrieve_file(item, entry, options=options))
return
def show(self, filename, rev=None):
vi = self.portal.getItemByName(self.area.name, os.path.abspath(filename))
if vi is None:
raise OperationError("Item '%s' is not in the area!" % filename)
entry = self.portal.getEntry(vi.id, rev)
if entry is None:
if rev is None:
raise OperationError("Item doesn't have any revisions!")
else:
raise OperationError("Can't find revision '%d'!" % rev)
if entry.status != Entry.STATUS_MODIFIED:
raise OperationError("Selected revision does not contain data")
if not entry.isreg():
raise OperationError("File is not regular, cannot display")
if entry.filecontents is None:
raise OperationError("File content has not been stored for this revision")
return entry.filecontents
def stat(self, options, files=None):
"""Stat command implementation
Parameters:
- options: for selecting revision number
- files: list of files to stat (by default all are)
Return value: list of
- name
- error message or None
- stat data when error message is None
"""
if len(files) != 0:
mlist = []
for filename in files:
af = os.path.abspath(filename)
item = self.portal.getItemByName(self.area.name, af)
if item is None:
yield af, "Skipped: file is not being tracked", None
continue
mlist.append(item)
else:
mlist = self.portal.getItems(self.area.name)
for item in mlist:
entry = self.portal.getEntry(item.id, options.rev, do_payload=False)
if entry is None:
yield item.name, "Selected revision not found", None
continue
yield item.name, None, entry.stat()
return
def log(self):
arearevs = self.portal.getRevisions(self.area.name)
return arearevs
def export(self, cmdopts, output):
if cmdopts.format == "tar":
efunc = self._exporttar
doit = lambda entry: (entry is not None and \
entry.status == Entry.STATUS_MODIFIED and \
(entry.filecontents is not None or not entry.isreg()))
elif cmdopts.format == "sha1sum":
efunc = self._exportcksum
doit = lambda entry: (entry is not None and \
entry.status == Entry.STATUS_MODIFIED and \
entry.isreg() and \
entry.sha1sum is not None)
else:
raise OperationError("Unknown export format '%s'!" % cmdopts.format)
cmdopts.area = self.area.name
cmdopts.do_payload = True
cmdopts.match_and = True
elist = self.portal.getEntries(cmdopts)
names = [(x.areaname, x.filename) for x in elist if doit(x)]
multirev = len(dict.fromkeys(names).keys()) != len(names)
names = [x.areaname for x in elist if doit(x)]
multiarea = len(dict.fromkeys(names).keys()) > 1
if multirev:
maxrev = reduce(lambda x, y: max(x,y.revno), elist, 0)
prefix = "/r=%%0%dd%%s" % len(str(maxrev))
for e in elist:
e.filename = prefix % (e.revno, e.filename)
if multiarea:
for e in elist:
e.filename = "/a=%s%s" % (e.areaname, e.filename)
efunc(elist, cmdopts, doit, output)
return
def _exportcksum(self, entries, cmdopts, doit, output):
for entry in entries:
if doit(entry):
output.write("%s %s\n" % (entry.sha1sum, entry.filename[1:]))
return
def _exporttar(self, entries, cmdopts, doit, output):
tarh = tarfile.open(fileobj=output, mode="w|")
for entry in entries:
if entry.isreg():
fdata = StringIO(entry.filecontents)
else:
fdata = None
tarh.addfile(self._genfakefile(entry), fdata)
tarh.close()
return
def _genfakefile(self, entry):
"""Generate a fake TarInfo object from a revision entry.
Parameters:
entry - the revision entry from which to create the archive member.
"""
ti = tarfile.TarInfo()
if entry.filename.startswith("/"):
ti.name = entry.filename[1:]
else:
ti.name = entry.filename
if entry.uname is None:
ti.uname = ""
else:
ti.uname = entry.uname
if entry.gname is None:
ti.gname = ""
else:
ti.gname = entry.gname
ti.uid = entry.uid
ti.gid = entry.gid
ti.mtime = entry.mtime
ti.mode = entry.mode
if entry.isreg():
ti.chksum = tarfile.calc_chksum(entry.filecontents)
ti.size = entry.size
ti.type = tarfile.REGTYPE
elif entry.isdir():
ti.type = tarfile.DIRTYPE
elif entry.islnk():
ti.type = tarfile.SYMTYPE
ti.linkname = entry.filecontents
elif entry.isblk():
ti.type = tarfile.BLKTYPE
elif entry.ischr():
ti.type = tarfile.CHRTYPE
elif entry.ififo():
ti.type = tarfile.FIFOTYPE
elif entry.isock():
return None
return ti
syntax highlighted by Code2HTML, v. 0.9.1