Index: trunk/packages/sipb-xen-base/debian/changelog =================================================================== --- trunk/packages/sipb-xen-base/debian/changelog (revision 1205) +++ trunk/packages/sipb-xen-base/debian/changelog (revision 1206) @@ -1,2 +1,8 @@ +sipb-xen-base (8.24) unstable; urgency=low + + * Switch to using a setup.py file with CDBS's Python support + + -- Evan Broder Fri, 24 Oct 2008 05:22:49 -0400 + sipb-xen-base (8.23) unstable; urgency=low Index: trunk/packages/sipb-xen-base/debian/rules =================================================================== --- trunk/packages/sipb-xen-base/debian/rules (revision 1205) +++ trunk/packages/sipb-xen-base/debian/rules (revision 1206) @@ -1,7 +1,9 @@ #!/usr/bin/make -f +DEB_PYTHON_SYSTEM=pysupport + include /usr/share/cdbs/1/rules/debhelper.mk +include /usr/share/cdbs/1/class/python-distutils.mk -# do we, should we, care between pysupport and pycentral? -binary-install/sipb-xen-base:: - dh_pysupport -psipb-xen-base +clean:: + rm -rf invirt.egg-info Index: trunk/packages/sipb-xen-base/files/usr/bin/invirt-getconf =================================================================== --- trunk/packages/sipb-xen-base/files/usr/bin/invirt-getconf (revision 1205) +++ (revision ) @@ -1,92 +1,0 @@ -#!/usr/bin/env python - -""" -invirt-getconf loads an invirt configuration file (either the original YAML -source or the faster-to-load JSON cache) and prints the configuration option -with the given name (key). Keys are dot-separated paths into the YAML -configuration tree. List indexes (0-based) are also treated as path -components. - -(Due to this path language, certain restrictions are placed on the keys used in -the YAML configuration; e.g., they cannot contain dots.) - -Examples: - - invirt-getconf db.uri - invirt-getconf authn.0.type -""" - -from invirt.config import load -from sys import argv, exit, stderr, stdout -from optparse import OptionParser - -class invirt_exception(Exception): pass - -def main(argv): - try: - parser = OptionParser(usage = '%prog [options] key', - description = __doc__.strip().split('\n\n')[0]) - parser.add_option('-s', '--src', - default = '/etc/invirt/master.yaml', - help = 'the source YAML configuration file to read from') - parser.add_option('-c', '--cache', - default = '/var/lib/invirt/invirt.json', - help = 'path to the JSON cache') - parser.add_option('-r', '--refresh', - action = 'store_true', - help = 'force the cache to be regenerated') - parser.add_option('-l', '--ls', - action = 'store_true', - help = 'list node\'s children') - opts, args = parser.parse_args() - - if len(args) > 1: - raise invirt_exception(__doc__.strip()) - elif args and args[0]: - components = args[0].split('.') - else: - components = [] - - conf = load(opts.src, opts.cache, opts.refresh) - for i, component in enumerate(components): - progress = '.'.join(components[:i]) - if type(conf) not in (dict, list): - raise invirt_exception( - '%s: node has no children (atomic datum)' % progress) - if type(conf) == list: - try: component = int(component) - except: raise invirt_exception( - '%s: node a list; integer path component required, ' - 'but got "%s"' % (progress, component)) - try: conf = conf[component] - except KeyError: raise invirt_exception( - '%s: key "%s" not found' % (progress, component)) - except IndexError: raise invirt_exception( - '%s: index %s out of range' % (progress, component)) - - if opts.ls: - if type(conf) not in (dict, list): - raise invirt_exception( - '%s: node has no children (atomic datum)' - % '.'.join(components)) - if type(conf) == list: - for i in xrange(len(conf)): - print i - else: - for k in conf.iterkeys(): - print k - else: - if type(conf) not in (dict, list): - print conf - else: - import yaml - yaml.dump(conf, stdout, - Dumper=yaml.CSafeDumper, default_flow_style=False) - except invirt_exception, ex: - print >> stderr, ex - return 1 - -if __name__ == '__main__': - exit(main(argv)) - -# vim:et:sw=4:ts=4 Index: trunk/packages/sipb-xen-base/files/usr/sbin/invirt-reload =================================================================== --- trunk/packages/sipb-xen-base/files/usr/sbin/invirt-reload (revision 1205) +++ (revision ) @@ -1,8 +1,0 @@ -#!/bin/bash - -for script in $(run-parts --test /etc/init.d); do - if [ "${script#/etc/init.d/sipb-xen-}" != "$script" \ - -o "${script#/etc/init.d/invirt-}" != "$script" ]; then - "$script" reload - fi -done Index: trunk/packages/sipb-xen-base/files/usr/share/python-support/sipb-xen-base/invirt/__init__.py =================================================================== --- trunk/packages/sipb-xen-base/files/usr/share/python-support/sipb-xen-base/invirt/__init__.py (revision 1205) +++ (revision ) @@ -1,27 +1,0 @@ -'''Invirt - a virtualization management system - -Invirt was developed at the Student Information Processing Board of -the Massachusetts Institute of Technology. See http://xvm.mit.edu/. - -Invirt is free software available under the GNU GPL, version 2 or later. -Consult the source files for details. -''' - -# Invirt is free software: you can redistribute it and/or modify it -# under the terms of the GNU General Public License as published by the -# Free Software Foundation, either version 2 of the License, or (at your -# option) any later version. - -# Invirt is distributed in the hope that it will be useful, but WITHOUT -# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License -# for more details. - -# You should have received a copy of the GNU General Public License -# along with Invirt. If not, see . - -__author__ = 'MIT SIPB' -__version__ = '0.1' -__copyright__ = 'Copyright (c) 2008 MIT SIPB' - -# vim:et:sw=4:ts=4 Index: trunk/packages/sipb-xen-base/files/usr/share/python-support/sipb-xen-base/invirt/common.py =================================================================== --- trunk/packages/sipb-xen-base/files/usr/share/python-support/sipb-xen-base/invirt/common.py (revision 1205) +++ (revision ) @@ -1,58 +1,0 @@ -from __future__ import with_statement - -import unittest -from fcntl import flock, LOCK_EX, LOCK_SH, LOCK_UN -import contextlib as clib - -class struct(object): - 'A simple namespace object.' - def __init__(self, d = {}, **kwargs): - 'd is the dictionary to update my __dict__ with.' - self.__dict__.update(d) - self.__dict__.update(kwargs) - -def dicts2struct(x): - """ - Given a tree of lists/dicts, perform a deep traversal to transform all the - dicts to structs. - """ - if type(x) == dict: - return struct((k, dicts2struct(v)) for k,v in x.iteritems()) - elif type(x) == list: - return [dicts2struct(v) for v in x] - else: - return x - -@clib.contextmanager -def lock_file(path, exclusive = True): - with clib.closing(file(path, 'w')) as f: - if exclusive: - locktype = LOCK_EX - else: - locktype = LOCK_SH - flock(f, locktype) - try: - yield - finally: - flock(f, LOCK_UN) - -# -# Tests. -# - -class common_tests(unittest.TestCase): - def test_dicts2structs(self): - dicts = { - 'atom': 0, - 'dict': { 'atom': 'atom', 'list': [1,2,3] }, - 'list': [ 'atom', {'key': 'value'} ] - } - structs = dicts2struct(dicts) - self.assertEqual(structs.atom, dicts['atom']) - self.assertEqual(structs.dict.atom, dicts['dict']['atom']) - self.assertEqual(structs.dict.list, dicts['dict']['list']) - self.assertEqual(structs.list[0], dicts['list'][0]) - self.assertEqual(structs.list[1].key, dicts['list'][1]['key']) - -if __name__ == '__main__': - unittest.main() Index: trunk/packages/sipb-xen-base/files/usr/share/python-support/sipb-xen-base/invirt/config.py =================================================================== --- trunk/packages/sipb-xen-base/files/usr/share/python-support/sipb-xen-base/invirt/config.py (revision 1205) +++ (revision ) @@ -1,91 +1,0 @@ -from __future__ import with_statement - -import json -from invirt.common import * -from os import rename -from os.path import getmtime -from contextlib import closing - -default_src_path = '/etc/invirt/master.yaml' -default_cache_path = '/var/lib/invirt/cache.json' -lock_path = '/var/lib/invirt/cache.lock' - -def load(src_path = default_src_path, - cache_path = default_cache_path, - force_refresh = False): - """ - Try loading the configuration from the faster-to-load JSON cache at - cache_path. If it doesn't exist or is outdated, load the configuration - instead from the original YAML file at src_path and regenerate the cache. - I assume I have the permissions to write to the cache directory. - """ - - # Namespace container for state variables, so that they can be updated by - # closures. - ns = struct() - - if force_refresh: - do_refresh = True - else: - src_mtime = getmtime(src_path) - try: cache_mtime = getmtime(cache_path) - except OSError: do_refresh = True - else: do_refresh = src_mtime + 1 >= cache_mtime - - # We chose not to simply say - # - # do_refresh = src_mtime >= cache_time - # - # because between the getmtime(src_path) and the time the cache is - # rewritten, the master configuration may have been updated, so future - # checks here would find a cache with a newer mtime than the master - # (and thus treat the cache as containing the latest version of the - # master). The +1 means that for at least a full second following the - # update to the master, this function will refresh the cache, giving us - # 1 second to write the cache. Note that if it takes longer than 1 - # second to write the cache, then this situation could still arise. - # - # The getmtime calls should logically be part of the same transaction - # as the rest of this function (cache read + conditional cache - # refresh), but to wrap everything in an flock would cause the - # following cache read to be less streamlined. - - if not do_refresh: - # Try reading from the cache first. This must be transactionally - # isolated from concurrent writes to prevent reading an incomplete - # (changing) version of the data (but the transaction can share the - # lock with other concurrent reads). This isolation is accomplished - # using an atomic filesystem rename in the refreshing stage. - try: - with closing(file(cache_path)) as f: - ns.cfg = json.read(f.read()) - except: do_refresh = True - - if do_refresh: - # Atomically reload the source and regenerate the cache. The read and - # write must be a single transaction, or a stale version may be - # written (if another read/write of a more recent configuration - # is interleaved). The final atomic rename is to keep this - # transactionally isolated from the above cache read. If we fail to - # acquire the lock, just try to load the master configuration. - import yaml - try: loader = yaml.CSafeLoader - except: loader = yaml.SafeLoader - try: - with lock_file(lock_path): - with closing(file(src_path)) as f: - ns.cfg = yaml.load(f, loader) - try: - with closing(file(cache_path + '.tmp', 'w')) as f: - f.write(json.write(ns.cfg)) - except: pass # silent failure - else: rename(cache_path + '.tmp', cache_path) - except IOError: - with closing(file(src_path)) as f: - ns.cfg = yaml.load(f, loader) - return ns.cfg - -dicts = load() -structs = dicts2struct(dicts) - -# vim:et:sw=4:ts=4 Index: trunk/packages/sipb-xen-base/files/usr/share/python-support/sipb-xen-base/invirt/remote.py =================================================================== --- trunk/packages/sipb-xen-base/files/usr/share/python-support/sipb-xen-base/invirt/remote.py (revision 1205) +++ (revision ) @@ -1,19 +1,0 @@ -from subprocess import PIPE, Popen -from invirt.config import structs as config -import yaml - -def bcast(cmd, hosts = [h.hostname for h in config.hosts]): - """ - Given a command and a list of hostnames or IPs, issue the command to all - the nodes and return a list of (host, output) pairs (the order should be - the same as the order of the hosts). - """ - pipes = [(host, - Popen(['remctl', host, 'remote', 'web', cmd], stdout=PIPE)) - for host in hosts] - outputs = [(s, p.communicate()[0]) for (s, p) in pipes] - for (s, p) in pipes: - if p.returncode != 0: - raise RuntimeError("remctl to host %s returned non-zero exit status %d" - % (s, p.returncode)) - return [(s, yaml.load(o, yaml.CSafeLoader)) for (s, o) in outputs] Index: trunk/packages/sipb-xen-base/python/invirt/__init__.py =================================================================== --- trunk/packages/sipb-xen-base/python/invirt/__init__.py (revision 1206) +++ trunk/packages/sipb-xen-base/python/invirt/__init__.py (revision 1206) @@ -0,0 +1,27 @@ +'''Invirt - a virtualization management system + +Invirt was developed at the Student Information Processing Board of +the Massachusetts Institute of Technology. See http://xvm.mit.edu/. + +Invirt is free software available under the GNU GPL, version 2 or later. +Consult the source files for details. +''' + +# Invirt is free software: you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the +# Free Software Foundation, either version 2 of the License, or (at your +# option) any later version. + +# Invirt is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +# for more details. + +# You should have received a copy of the GNU General Public License +# along with Invirt. If not, see . + +__author__ = 'MIT SIPB' +__version__ = '0.1' +__copyright__ = 'Copyright (c) 2008 MIT SIPB' + +# vim:et:sw=4:ts=4 Index: trunk/packages/sipb-xen-base/python/invirt/common.py =================================================================== --- trunk/packages/sipb-xen-base/python/invirt/common.py (revision 1206) +++ trunk/packages/sipb-xen-base/python/invirt/common.py (revision 1206) @@ -0,0 +1,58 @@ +from __future__ import with_statement + +import unittest +from fcntl import flock, LOCK_EX, LOCK_SH, LOCK_UN +import contextlib as clib + +class struct(object): + 'A simple namespace object.' + def __init__(self, d = {}, **kwargs): + 'd is the dictionary to update my __dict__ with.' + self.__dict__.update(d) + self.__dict__.update(kwargs) + +def dicts2struct(x): + """ + Given a tree of lists/dicts, perform a deep traversal to transform all the + dicts to structs. + """ + if type(x) == dict: + return struct((k, dicts2struct(v)) for k,v in x.iteritems()) + elif type(x) == list: + return [dicts2struct(v) for v in x] + else: + return x + +@clib.contextmanager +def lock_file(path, exclusive = True): + with clib.closing(file(path, 'w')) as f: + if exclusive: + locktype = LOCK_EX + else: + locktype = LOCK_SH + flock(f, locktype) + try: + yield + finally: + flock(f, LOCK_UN) + +# +# Tests. +# + +class common_tests(unittest.TestCase): + def test_dicts2structs(self): + dicts = { + 'atom': 0, + 'dict': { 'atom': 'atom', 'list': [1,2,3] }, + 'list': [ 'atom', {'key': 'value'} ] + } + structs = dicts2struct(dicts) + self.assertEqual(structs.atom, dicts['atom']) + self.assertEqual(structs.dict.atom, dicts['dict']['atom']) + self.assertEqual(structs.dict.list, dicts['dict']['list']) + self.assertEqual(structs.list[0], dicts['list'][0]) + self.assertEqual(structs.list[1].key, dicts['list'][1]['key']) + +if __name__ == '__main__': + unittest.main() Index: trunk/packages/sipb-xen-base/python/invirt/config.py =================================================================== --- trunk/packages/sipb-xen-base/python/invirt/config.py (revision 1206) +++ trunk/packages/sipb-xen-base/python/invirt/config.py (revision 1206) @@ -0,0 +1,91 @@ +from __future__ import with_statement + +import json +from invirt.common import * +from os import rename +from os.path import getmtime +from contextlib import closing + +default_src_path = '/etc/invirt/master.yaml' +default_cache_path = '/var/lib/invirt/cache.json' +lock_path = '/var/lib/invirt/cache.lock' + +def load(src_path = default_src_path, + cache_path = default_cache_path, + force_refresh = False): + """ + Try loading the configuration from the faster-to-load JSON cache at + cache_path. If it doesn't exist or is outdated, load the configuration + instead from the original YAML file at src_path and regenerate the cache. + I assume I have the permissions to write to the cache directory. + """ + + # Namespace container for state variables, so that they can be updated by + # closures. + ns = struct() + + if force_refresh: + do_refresh = True + else: + src_mtime = getmtime(src_path) + try: cache_mtime = getmtime(cache_path) + except OSError: do_refresh = True + else: do_refresh = src_mtime + 1 >= cache_mtime + + # We chose not to simply say + # + # do_refresh = src_mtime >= cache_time + # + # because between the getmtime(src_path) and the time the cache is + # rewritten, the master configuration may have been updated, so future + # checks here would find a cache with a newer mtime than the master + # (and thus treat the cache as containing the latest version of the + # master). The +1 means that for at least a full second following the + # update to the master, this function will refresh the cache, giving us + # 1 second to write the cache. Note that if it takes longer than 1 + # second to write the cache, then this situation could still arise. + # + # The getmtime calls should logically be part of the same transaction + # as the rest of this function (cache read + conditional cache + # refresh), but to wrap everything in an flock would cause the + # following cache read to be less streamlined. + + if not do_refresh: + # Try reading from the cache first. This must be transactionally + # isolated from concurrent writes to prevent reading an incomplete + # (changing) version of the data (but the transaction can share the + # lock with other concurrent reads). This isolation is accomplished + # using an atomic filesystem rename in the refreshing stage. + try: + with closing(file(cache_path)) as f: + ns.cfg = json.read(f.read()) + except: do_refresh = True + + if do_refresh: + # Atomically reload the source and regenerate the cache. The read and + # write must be a single transaction, or a stale version may be + # written (if another read/write of a more recent configuration + # is interleaved). The final atomic rename is to keep this + # transactionally isolated from the above cache read. If we fail to + # acquire the lock, just try to load the master configuration. + import yaml + try: loader = yaml.CSafeLoader + except: loader = yaml.SafeLoader + try: + with lock_file(lock_path): + with closing(file(src_path)) as f: + ns.cfg = yaml.load(f, loader) + try: + with closing(file(cache_path + '.tmp', 'w')) as f: + f.write(json.write(ns.cfg)) + except: pass # silent failure + else: rename(cache_path + '.tmp', cache_path) + except IOError: + with closing(file(src_path)) as f: + ns.cfg = yaml.load(f, loader) + return ns.cfg + +dicts = load() +structs = dicts2struct(dicts) + +# vim:et:sw=4:ts=4 Index: trunk/packages/sipb-xen-base/python/invirt/remote.py =================================================================== --- trunk/packages/sipb-xen-base/python/invirt/remote.py (revision 1206) +++ trunk/packages/sipb-xen-base/python/invirt/remote.py (revision 1206) @@ -0,0 +1,19 @@ +from subprocess import PIPE, Popen +from invirt.config import structs as config +import yaml + +def bcast(cmd, hosts = [h.hostname for h in config.hosts]): + """ + Given a command and a list of hostnames or IPs, issue the command to all + the nodes and return a list of (host, output) pairs (the order should be + the same as the order of the hosts). + """ + pipes = [(host, + Popen(['remctl', host, 'remote', 'web', cmd], stdout=PIPE)) + for host in hosts] + outputs = [(s, p.communicate()[0]) for (s, p) in pipes] + for (s, p) in pipes: + if p.returncode != 0: + raise RuntimeError("remctl to host %s returned non-zero exit status %d" + % (s, p.returncode)) + return [(s, yaml.load(o, yaml.CSafeLoader)) for (s, o) in outputs] Index: trunk/packages/sipb-xen-base/scripts/invirt-getconf =================================================================== --- trunk/packages/sipb-xen-base/scripts/invirt-getconf (revision 1206) +++ trunk/packages/sipb-xen-base/scripts/invirt-getconf (revision 1206) @@ -0,0 +1,92 @@ +#!/usr/bin/env python + +""" +invirt-getconf loads an invirt configuration file (either the original YAML +source or the faster-to-load JSON cache) and prints the configuration option +with the given name (key). Keys are dot-separated paths into the YAML +configuration tree. List indexes (0-based) are also treated as path +components. + +(Due to this path language, certain restrictions are placed on the keys used in +the YAML configuration; e.g., they cannot contain dots.) + +Examples: + + invirt-getconf db.uri + invirt-getconf authn.0.type +""" + +from invirt.config import load +from sys import argv, exit, stderr, stdout +from optparse import OptionParser + +class invirt_exception(Exception): pass + +def main(argv): + try: + parser = OptionParser(usage = '%prog [options] key', + description = __doc__.strip().split('\n\n')[0]) + parser.add_option('-s', '--src', + default = '/etc/invirt/master.yaml', + help = 'the source YAML configuration file to read from') + parser.add_option('-c', '--cache', + default = '/var/lib/invirt/invirt.json', + help = 'path to the JSON cache') + parser.add_option('-r', '--refresh', + action = 'store_true', + help = 'force the cache to be regenerated') + parser.add_option('-l', '--ls', + action = 'store_true', + help = 'list node\'s children') + opts, args = parser.parse_args() + + if len(args) > 1: + raise invirt_exception(__doc__.strip()) + elif args and args[0]: + components = args[0].split('.') + else: + components = [] + + conf = load(opts.src, opts.cache, opts.refresh) + for i, component in enumerate(components): + progress = '.'.join(components[:i]) + if type(conf) not in (dict, list): + raise invirt_exception( + '%s: node has no children (atomic datum)' % progress) + if type(conf) == list: + try: component = int(component) + except: raise invirt_exception( + '%s: node a list; integer path component required, ' + 'but got "%s"' % (progress, component)) + try: conf = conf[component] + except KeyError: raise invirt_exception( + '%s: key "%s" not found' % (progress, component)) + except IndexError: raise invirt_exception( + '%s: index %s out of range' % (progress, component)) + + if opts.ls: + if type(conf) not in (dict, list): + raise invirt_exception( + '%s: node has no children (atomic datum)' + % '.'.join(components)) + if type(conf) == list: + for i in xrange(len(conf)): + print i + else: + for k in conf.iterkeys(): + print k + else: + if type(conf) not in (dict, list): + print conf + else: + import yaml + yaml.dump(conf, stdout, + Dumper=yaml.CSafeDumper, default_flow_style=False) + except invirt_exception, ex: + print >> stderr, ex + return 1 + +if __name__ == '__main__': + exit(main(argv)) + +# vim:et:sw=4:ts=4 Index: trunk/packages/sipb-xen-base/scripts/invirt-reload =================================================================== --- trunk/packages/sipb-xen-base/scripts/invirt-reload (revision 1206) +++ trunk/packages/sipb-xen-base/scripts/invirt-reload (revision 1206) @@ -0,0 +1,8 @@ +#!/bin/bash + +for script in $(run-parts --test /etc/init.d); do + if [ "${script#/etc/init.d/sipb-xen-}" != "$script" \ + -o "${script#/etc/init.d/invirt-}" != "$script" ]; then + "$script" reload + fi +done Index: trunk/packages/sipb-xen-base/setup.py =================================================================== --- trunk/packages/sipb-xen-base/setup.py (revision 1206) +++ trunk/packages/sipb-xen-base/setup.py (revision 1206) @@ -0,0 +1,23 @@ +#!/usr/bin/python + +import os +from debian_bundle.changelog import Changelog +from debian_bundle.deb822 import Deb822 +from email.utils import parseaddr +from setuptools import setup + +version = Changelog(open(os.path.join(__file__, 'debian/changelog')).read()).\ + get_version().full_version + +maintainer_full = Deb822(open(os.path.join(__file__, 'debian/control')))['Maintainer'] +maintainer, maintainer_email = parseaddr(maintainer_full) + +setup( + name='invirt', + version=version, + maintainer=maintainer, + maintainer_email=maintainer_full, + + package_dir = {'': 'python'}, + scripts=['scripts/invirt-getconf', 'scripts/invirt-reload'] +)