"""Command line generic functionality This module implements basic functionality of a command line client, but without the parsing and other stuff. Thus, a client cares only about interacting with the shell or GUI and can use the API exported by this module, without caring about cfvers internals. """ # Copyright 2003-2005 Iustin Pop # # This file is part of cfvers. # # cfvers is free software; you can redistribute it and/or modify it # under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # cfvers is distributed in the hope that it will be useful, but # WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU # General Public License for more details. # # You should have received a copy of the GNU General Public License # along with cfvers; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA # $Id: cmd.py 218 2005-10-30 09:26:23Z iusty $ import os, struct, stat, os.path, re, commands, sys import types import random import errno from mx import DateTime from ConfigParser import SafeConfigParser import tarfile from cStringIO import StringIO import hmac, sha import sets import Pyro.core import Pyro.naming import cfvers import cfvers.main import cfvers.gateway from cfvers.main import * __all__ = [ "AdminCommands", "Commands", ] class BulkTransport(object): """Class used in bulk transfers through the portal""" def __init__(self, portal, isadditem, revno=None): self.buffer = [] self.portal = portal self.currsize = 0 self.results = [] self.isadditem = isadditem self.revno = revno def add(self, item): self.buffer.append(item) if not self.isadditem and item.filecontents is not None: self.currsize += len(item.filecontents) if len(self.buffer) >= 1000 or \ (not self.isadditem and self.currsize >= 1048576): self.flush() return def flush(self): if self.isadditem: res = self.portal.bulkAddItem(self.revno, self.buffer) else: res = self.portal.bulkAddEntry(self.buffer) self.results.extend(res) self.buffer = [] self.currsize = 0 return class CLIScript(object): """Base class for command line handlers""" def __init__(self, options, prompt_func): Pyro.core.initClient(0) self.readconfig(options, prompt_func) self.options = options if options.server_type == "remote": authenticator = (options.username, options.client_password) factoryURI = "PYROLOC://%s:%s/PortalFactory" % (options.host, options.port) factory = Pyro.core.getProxyForURI(factoryURI) factory._setNewConnectionValidator(cfvers.gateway.PortalValidator()) factory._setIdentification(authenticator) try: self.portal = factory.getPortal() except Pyro.errors.ProtocolError, e: raise cfvers.CommException(*e.args) self.portal._setNewConnectionValidator(cfvers.gateway.PortalValidator()) self.portal._setIdentification(authenticator) if not self.authPortal(): raise ValueError("Server failed to authenticate") else: self.portal = cfvers.gateway.Portal(local=True, repo=(options.repo_meth,options.repo_data)) return def authPortal(self): token = [] for i in range(20): token.append(chr(random.randint(32, 127))) token = "".join(token) result = self.portal.checkID(token) if result is None: return False preamble, postamble, stuff = result hm = hmac.new(self.options.server_password, preamble, sha) hm.update(token) hm.update(postamble) mydata = hm.hexdigest() if mydata != stuff: return False return True def readconfig(options, prompt_func): def aquire(name): envval = os.environ.get("CFVERS_%s" % name.upper()) if envval is not None: setval = envval elif cp.has_option("server", name): setval = cp.get("server", name) else: setval = prompt_func(name) return setval cp = SafeConfigParser() cp.read(["/etc/cfvers/client.conf", os.path.expanduser("~/.cfvers"),]) st = getattr(options, "server_type", None) if st is None: st = aquire("server_type") options.server_type = st if st == "local": attlist = ('repo_meth', 'repo_data', 'area') elif st == "remote": attlist = ('host', 'port', 'username', 'client_password', 'server_password', 'area') else: raise ValueError("Invalid server type '%s'" % st) for optname in attlist: if getattr(options, optname, None) is not None: continue setattr(options, optname, aquire(optname)) return readconfig = staticmethod(readconfig) def validconfig(self): st = getattr(self.options, "server_type", None) if st is None: return False if st == "local": attrlist = ('repo_meth', 'repo_data', 'area') elif st == "remote": attrlist = ('host', 'port', 'username', 'client_password', 'server_password', 'area') else: return False for attr in attrlist: if getattr(self.options, attr, None) is None: return False return True def get_version(): nv = "%d.%d.%d (%s release)" % cfvers.version_info[0:4] version="""%%prog - cfvers %s Copyright (C) 2003-2005 Iustin Pop This is free software; see the source for copying conditions. There is NO warranty ; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.""" % nv return version get_version = staticmethod(get_version) def parserev(revstring): """Tries to parse a revision string of form num[:num]""" if revstring is None: return None, None # match and extract from glob rev[:rev] m=re.match("^((?P[0-9]+):)?(?P[0-9]+)$", revstring) if m is None: raise ValueError, "Invalid syntax in revision specification %s" % revstring r1 = m.group('start') r2 = m.group('end') if r1 is not None: r1 = int(r1) if r2 is not None: r2 = int(r2) return r1, r2 parserev = staticmethod(parserev) class AdminCommands(CLIScript): def __init__(self, options, prompt_func, do_connect=True): """Constructor for the Commands class""" super(AdminCommands, self).__init__(options, prompt_func) if do_connect: self.portal.connect() def open(self): self.portal.connect() def close(self): self.portal.disconnect() def create_area(self, cmdoptions, name): if self.portal.getArea(name) is not None: raise cfvers.OperationError("Area `%s' already exists" % name) a = Area(name=name, description=cmdoptions.description, root=cmdoptions.root) self.portal.addArea(a) self.portal.commit() return def init_repo(self, createopts): self.portal.create(createopts) return class Commands(CLIScript): def __init__(self, options, prompt_func): """Constructor for the Commands class""" super(Commands, self).__init__(options, prompt_func) if not self.validconfig(): raise cfvers.ConfigException, "Incomplete configuration" self.portal.connect() self.area = self.portal.getArea(options.area) if self.area is None: raise cfvers.ConfigException, "can't find area named '%s'" % options.area return def close(self): self.portal.disconnect() return def _map_files(self, area, files=None, norecurse=False): """Maps a (possibly empty) list of files to a list of targets Args: - area: the area in which to work - files: a list (possibly empty) of which item to process; if it is none, all the items in the area are processed - norecurse: don't descend directories Special care must be taken related to handling deleted items: - an existing item (in the repo) which was deleted from the filesystem must pass the lstat test - a deleted item (in the repo) which is not in the filesystem should pass completely the stats - a deleted item (in the repo) which is in the filesystem must be normally registered """ def helper(arg, dirname, names): mylist, path = arg # Translate from real root to area root if path == "/": realdir = dirname elif dirname == path: realdir = "/" elif dirname.startswith(path): realdir = dirname[len(path):] else: raise ValueError, "Path doesn't lie in its area!" mylist.extend([os.path.join(realdir, fname) for fname in names]) return # Step 1. If no files where given, act as if all items in the # repository where given as arguments if len(files) == 0: files = [item.name for item in self.portal.getItems(area.name)] # Step 2. Now we have a list of files to act on (add, # store). For each file, try to lstat. If successfull and is # dir and no-recurse is false, descend it. files = map(os.path.abspath, files) files = dict.fromkeys(files).keys() targets = [] errors = [] for virtf in files: realf = forcejoin(area.root, virtf) # Always mark to process targets.append(virtf) try: st = os.lstat(realf) except EnvironmentError, e: continue # If we got here, the file at least exists, # so check to see if we can descend if not norecurse and stat.S_ISDIR(st.st_mode): os.path.walk(realf, helper, (targets, area.root)) targets = dict.fromkeys(targets).keys() targets.sort() return (targets, errors) def add(self, files=[], options=None): """Register a set of files in the repository. Parameters: - files: a list of filenames - logmsg: the log message A new revision won't be stored if no items have been stored (e.g. all existing, or no items with valid names). """ logmsg = options.logmsg norecurse = options.norecurse a = self.area (targets, errors) = self._map_files(a, files=files, norecurse=norecurse) ar = Revision(area=a, logmsg=logmsg, commiter=options.commiter) ar = self.portal.putRevision(ar) newrev = ar.revno stored = 0 itemdict = dict([(i.name, i) for i in self.portal.getItems(a.name)]) bulk = BulkTransport(self.portal, True, newrev) for name in targets: if name in itemdict: errors.append(Result(Result.ADDED_EXISTING, item=itemdict[name])) continue item = Item(area=a.name, name=name, flags=options.flags) bulk.add(item) bulk.flush() for item, entry in bulk.results: errors.append(Result(Result.ADDED_OK, item=item, entry=entry)) stored += 1 if stored > 0: self.portal.commit() store_done = True else: self.portal.rollback() store_done = False return errors, store_done, newrev def addfromdirs(self, options=None): """Register new files in tracked directories. Parameters: - logmsg: the log message A new revision won't be stored if no items have been stored (e.g. all existing, or no items with valid names). """ logmsg = options.logmsg a = self.area ar = Revision(area=a, logmsg=logmsg, commiter=options.commiter) ar = self.portal.putRevision(ar) newrev = ar.revno items = self.portal.getItems(a.name) stored = 0 bulk = BulkTransport(self.portal, True, newrev) errors = [] existing = [item.name for item in items] worktodo = items while len(worktodo) > 0: newitems = [] newnames = [] for item in worktodo: name = item.name try: dirfiles = os.listdir(name) except OSError, e: if e.errno in (errno.ENOENT, errno.ENOTDIR): continue if e.errno == errno.EACCES: errors.append(Result(Result.ADDED_EACCES, item=item)) continue raise for fname in dirfiles: absname = os.path.abspath(os.path.join(name, fname)) if absname not in existing and absname not in newnames: newitem = Item(area=a.name, name=absname, flags=item.flags) if not options.norecurse: newitems.append(newitem) newnames.append(absname) bulk.add(newitem) worktodo = newitems existing.extend(newnames) bulk.flush() for item, entry in bulk.results: errors.append(Result(Result.ADDED_OK, item=item, entry=entry)) stored += 1 if stored > 0: self.portal.commit() store_done = True else: self.portal.rollback() store_done = False return errors, store_done, newrev def register(self, name, cmdline, options): """Register a virtual item in the repository. Parameters: - name: the virtual path; must be a valid path and should not exist - cmdline: command line, list A new revision won't be stored if no items have been stored (e.g. all existing, or no items with valid names). """ a = self.area ar = Revision(area=a, logmsg=options.logmsg, commiter=options.commiter) ar = self.portal.putRevision(ar) newrev = ar.revno item = self.portal.getItemByName(a.name, name) if item is not None: res = Result(Result.ADDED_EXISTING, item=item) return res, False, newrev item = Item(area=a.name, name=name, flags=Item.STORE_VIRTUAL, command=" ".join(cmdline)) i, e = self.portal.addItem(newrev, item) res = Result(Result.ADDED_OK, item=i, entry=e) self.portal.commit() return res, True, newrev def store(self, files=[], options=None): """Store a set of files or all the items already in the repository. Parameters: - files: a list of filename, or empty if all the items should be (re)commited - logmsg: the log message A new revision won't be stored if no items have been stored (e.g. nothing changed, or no items could be read). """ logmsg = options.logmsg norecurse = options.norecurse a = self.area (targets, errors) = self._map_files(a, files=files, norecurse=norecurse) ar = Revision(area=a, logmsg=logmsg, commiter=options.commiter) ar = self.portal.putRevision(ar) newrev = ar.revno itemdict = dict([(i.name, i) for i in self.portal.getItems(a.name)]) bulk = BulkTransport(self.portal, False) for name in targets: if not name in itemdict: errors.append(Result(Result.STORED_NOTREG, fname=name)) continue item = itemdict[name] if item.flags & (Item.STORE_METADATA | Item.STORE_CHECKSUM | \ Item.STORE_CONTENTS | Item.STORE_VIRTUAL ) == 0: # Skip over this item which is marked not to be stored errors.append(Result(Result.STORED_TOSKIP, item=item)) continue try: entry = Entry(item=item, revno=newrev, area=a) except EnvironmentError, e: errors.append(Result(Result.STORED_IOERROR, item=item, exc=e)) continue bulk.add(entry) bulk.flush() errors.extend(bulk.results) stored = 0 for r in errors: if r.code in (Result.STORED_OK, Result.STORED_DELETED): stored += 1 if stored > 0: self.portal.commit() store_done = True else: self.portal.rollback() store_done = False return errors, store_done, newrev def _map_dir_diff(self, ao, no, an, nn, ilist): ilo = self.portal.getItemsByDirname(ao, no) iln = self.portal.getItemsByDirname(an, nn) silo = sets.Set([x.name for x in ilo]) siln = sets.Set([x.name for x in iln]) for name in silo - siln: yield (ao, name, an, name, True, None, [("File %s exists only in `old'" % name,)], None, None) for name in siln - silo: yield (ao, name, an, name, True, None, [("File %s exists only in `new'" % name,)], None, None) for name in silo & siln: oitem = [x for x in ilo if x.name == name][0] nitem = [x for x in iln if x.name == name][0] ilist.append((oitem, nitem)) for r in self._map_dir_diff(ao, name, an, name, ilist): yield r return def diff(self, options=None, files=None): """Diff command implementation Parameters: - options: revision and checks are used - files: list of files to compare (by default all are) Return value: list of - name (FIXME: when comparing different items this does not make sense) - status: True/False, meaning if the diff was successfully computed - error message when status == False - diff data when status == True - old entry - new entry If the listonly options is selected, the diff data is an empty tuple, meaning the `name' item is different. If listonly is not selected, the diff data is a list of tuples of: - one element: should be printed as-is, it usually means a general diff conclusion - two elements: the first element is the attribute name and the second element is the already formated diff - three elements: the first element is the name of attribute with the old data in second element and new data in the third element. """ ap = re.compile("^([\w]+):(/.*)$") rev1, rev2 = self.parserev(options.rev) if files is None or len(files) == 0: ilist = [(i, i) for i in self.portal.getItems(self.area.name)] else: ilist = [] old = None while len(files) > 0: fn = files.pop(0) match = ap.match(fn) if match is not None: area, fn = match.groups() else: area = self.area item = self.portal.getItemByName(area.name, os.path.abspath(fn)) if item is None: yield area, fn, None, None, False, "Item does not exist in the area", (), None, None if old is None: # Consume also the next item files.pop(0) else: old = None # Clear the already processed item continue if old is None: old = item else: ilist.append((old, item)) for r in self._map_dir_diff(old.area, old.name, item.area, item.name, ilist): yield r old = None if old is not None: ilist.append((old, old)) for r in self._map_dir_diff(old.area, old.name, old.area, old.name, ilist): yield r for vo, vn in ilist: r = self._diff_file(vo, vn, rev1, rev2, options) if r is not None: yield (vo.area, vo.name, vn.area, vn.name) + r return def _diff_file(self, vo, vn, rev1, rev2, options): """Execute a diff between two revisions. Parameters: - vo: the `old' item - vn: the `new' item - rev1: the `old' revision - rev2: the `new' revision Returns: - status - message - diff data - old entry - new entry See docstring for the diff method for more details on the return data. """ # rev1 is older (source) # rev2 is newer (target) if rev1 is None: rev1 = rev2 rev2 = None do_payload = "filecontents" in options.checks e1 = self.portal.getEntry(vo.id, rev1, do_payload=do_payload) if e1 is None or e1.status == Entry.STATUS_ADDED: if rev1 is None: # We didn't manage to aquire last revision for item, # it means it has no revisions return False, "Item doesn't have any revisions containing data", [], None, None else: return False, "Item doesn't have revision entry %d or the entry does not contain data" % rev1, [], None, None if rev2 is None: # rev2 has not been specified, thus the user wants to check # against the filesystem; build from filesystem, if # possible if vn.flags & Item.STORE_VIRTUAL or cfvers.rexists(vn.name): try: e2 = Entry(item=vn, area=self.portal.getArea(vn.area)) except (IOError, OSError), e: return False, "Can't read current status, error: %s" % e, [], e1, None else: return True, None, [("File has been deleted",)], e1, Entry.newDeleted(vn, rev2) else: e2 = self.portal.getEntry(vn.id, rev2, do_payload=do_payload) if e2 is None or e2.status == Entry.STATUS_ADDED: return False, "Item doesn't have revision entry %d or the entry does not contain data" % rev2, [], e1, e2 if e1.status != Entry.STATUS_MODIFIED or e2.status != Entry.STATUS_MODIFIED: return True, None, [("Revisions status: %s, %s" % \ (Entry.STATUS_MAP[e1.status], Entry.STATUS_MAP[e2.status]))], \ e1, e2 if options.list: if not e2.compare(e1, options.checks): return True, None, [], None, None else: output = e2.diff(e1, options=options) if len(output) > 0: return True, None, output, e1, e2 return def retrieve(self, files=None, options=None): results = [] if len(files) != 0: mlist = [] for filename in files: af = os.path.abspath(filename) item = self.portal.getItemByName(self.area.name, af) if item is None: results.append(Result(Result.RETR_NTRACK, fname=af)) continue mlist.append(item) if not options.norecurse: mlist += self._recur_build_list(mlist, []) else: mlist = self.portal.getItems(self.area.name) self._retrieve_list(mlist, options, results) return results def _recur_build_list(self, litems, alsoskip): nlist = [] for item in litems: others = self.portal.getItemsByDirname(self.area.name, item.name) for it in others: if it not in [x.name for x in litems] and \ it not in [x.name for x in alsoskip] and \ it not in [x.name for x in nlist]: nlist.append(it) nlist += self._recur_build_list([it], nlist + litems) return nlist def _retrieve_file(self, vi, entry, options=None): return entry.to_filesys(destdir=options.destdir, use_dirs=options.use_dirs) def _retrieve_list(self, ilist, options, results): for item in ilist: entry = self.portal.getEntry(item.id, options.revno) if entry is None: elist = self.portal.getEntryList(item.id) if len(elist) == 0: ncod = Result.RETR_NOREVS else: ncod = Result.RETR_NOXREV results.append(Result(ncod, item=item)) else: results.append(self._retrieve_file(item, entry, options=options)) return def show(self, filename, rev=None): vi = self.portal.getItemByName(self.area.name, os.path.abspath(filename)) if vi is None: raise OperationError("Item '%s' is not in the area!" % filename) entry = self.portal.getEntry(vi.id, rev) if entry is None: if rev is None: raise OperationError("Item doesn't have any revisions!") else: raise OperationError("Can't find revision '%d'!" % rev) if entry.status != Entry.STATUS_MODIFIED: raise OperationError("Selected revision does not contain data") if not entry.isreg(): raise OperationError("File is not regular, cannot display") if entry.filecontents is None: raise OperationError("File content has not been stored for this revision") return entry.filecontents def stat(self, options, files=None): """Stat command implementation Parameters: - options: for selecting revision number - files: list of files to stat (by default all are) Return value: list of - name - error message or None - stat data when error message is None """ if len(files) != 0: mlist = [] for filename in files: af = os.path.abspath(filename) item = self.portal.getItemByName(self.area.name, af) if item is None: yield af, "Skipped: file is not being tracked", None continue mlist.append(item) else: mlist = self.portal.getItems(self.area.name) for item in mlist: entry = self.portal.getEntry(item.id, options.rev, do_payload=False) if entry is None: yield item.name, "Selected revision not found", None continue yield item.name, None, entry.stat() return def log(self): arearevs = self.portal.getRevisions(self.area.name) return arearevs def export(self, cmdopts, output): if cmdopts.format == "tar": efunc = self._exporttar doit = lambda entry: (entry is not None and \ entry.status == Entry.STATUS_MODIFIED and \ (entry.filecontents is not None or not entry.isreg())) elif cmdopts.format == "sha1sum": efunc = self._exportcksum doit = lambda entry: (entry is not None and \ entry.status == Entry.STATUS_MODIFIED and \ entry.isreg() and \ entry.sha1sum is not None) else: raise OperationError("Unknown export format '%s'!" % cmdopts.format) cmdopts.area = self.area.name cmdopts.do_payload = True cmdopts.match_and = True elist = self.portal.getEntries(cmdopts) names = [(x.areaname, x.filename) for x in elist if doit(x)] multirev = len(dict.fromkeys(names).keys()) != len(names) names = [x.areaname for x in elist if doit(x)] multiarea = len(dict.fromkeys(names).keys()) > 1 if multirev: maxrev = reduce(lambda x, y: max(x,y.revno), elist, 0) prefix = "/r=%%0%dd%%s" % len(str(maxrev)) for e in elist: e.filename = prefix % (e.revno, e.filename) if multiarea: for e in elist: e.filename = "/a=%s%s" % (e.areaname, e.filename) efunc(elist, cmdopts, doit, output) return def _exportcksum(self, entries, cmdopts, doit, output): for entry in entries: if doit(entry): output.write("%s %s\n" % (entry.sha1sum, entry.filename[1:])) return def _exporttar(self, entries, cmdopts, doit, output): tarh = tarfile.open(fileobj=output, mode="w|") for entry in entries: if entry.isreg(): fdata = StringIO(entry.filecontents) else: fdata = None tarh.addfile(self._genfakefile(entry), fdata) tarh.close() return def _genfakefile(self, entry): """Generate a fake TarInfo object from a revision entry. Parameters: entry - the revision entry from which to create the archive member. """ ti = tarfile.TarInfo() if entry.filename.startswith("/"): ti.name = entry.filename[1:] else: ti.name = entry.filename if entry.uname is None: ti.uname = "" else: ti.uname = entry.uname if entry.gname is None: ti.gname = "" else: ti.gname = entry.gname ti.uid = entry.uid ti.gid = entry.gid ti.mtime = entry.mtime ti.mode = entry.mode if entry.isreg(): ti.chksum = tarfile.calc_chksum(entry.filecontents) ti.size = entry.size ti.type = tarfile.REGTYPE elif entry.isdir(): ti.type = tarfile.DIRTYPE elif entry.islnk(): ti.type = tarfile.SYMTYPE ti.linkname = entry.filecontents elif entry.isblk(): ti.type = tarfile.BLKTYPE elif entry.ischr(): ti.type = tarfile.CHRTYPE elif entry.ififo(): ti.type = tarfile.FIFOTYPE elif entry.isock(): return None return ti