Index: trunk/packages/invirt-base/config.todo =================================================================== --- trunk/packages/invirt-base/config.todo (revision 1330) +++ trunk/packages/invirt-base/config.todo (revision 1330) @@ -0,0 +1,2 @@ +files/etc/apt/sources.list.d/debathena.list: mit kerberos config +files/etc/apt/sources.list.d/sipb-xen.list: apt repo uri Index: trunk/packages/invirt-base/debian/changelog =================================================================== --- trunk/packages/invirt-base/debian/changelog (revision 1330) +++ trunk/packages/invirt-base/debian/changelog (revision 1330) @@ -0,0 +1,257 @@ +invirt-base (0.0.1) unstable; urgency=low + + * sipb-xen-base -> invirt-base + + -- Greg Price Sun, 25 Oct 2008 00:54:09 -0400 + +sipb-xen-base (8.29) unstable; urgency=low + + * gen-files.sh: degrade gracefully to plain-old sh + + -- Greg Price Sat, 25 Oct 2008 20:06:54 -0400 + +sipb-xen-base (8.28) unstable; urgency=low + + * Factor out more common initscript code: + std-init.sh replaces the usual Debian boilerplate + + -- Greg Price Sat, 25 Oct 2008 15:33:08 -0400 + +sipb-xen-base (8.27) unstable; urgency=low + + * Use invoke-rc.d in invirt-reload so that policy-rc.d is respected + + -- Evan Broder Fri, 24 Oct 2008 13:19:49 -0400 + +sipb-xen-base (8.26) unstable; urgency=low + + * Move invirt-reload to /usr/sbin + + -- Evan Broder Fri, 24 Oct 2008 12:53:18 -0400 + +sipb-xen-base (8.25) unstable; urgency=low + + * Factor out common initscript code, provide it here + + -- Greg Price Fri, 24 Oct 2008 07:06:59 -0400 + +sipb-xen-base (8.24) unstable; urgency=low + + * Switch to using a setup.py file with CDBS's Python support + + -- Evan Broder Fri, 24 Oct 2008 05:22:49 -0400 + +sipb-xen-base (8.23) unstable; urgency=low + + * Now that we're using Python 2.5, we can actually write with statements + + -- Evan Broder Fri, 24 Oct 2008 03:32:00 -0400 + +sipb-xen-base (8.22) unstable; urgency=low + + * Please - anything but etch. Anything! + + -- Evan Broder Thu, 02 Oct 2008 19:50:30 -0400 + +sipb-xen-base (8.21) unstable; urgency=low + + * update sources.list.d/sipb-xen.list for prod and hardy + + -- Greg Price Wed, 01 Oct 2008 20:06:52 -0400 + +sipb-xen-base (8.20) unstable; urgency=low + + * fix distribution + + -- Greg Price Tue, 30 Sep 2008 23:51:11 -0400 + +sipb-xen-base (8.19) hardy; urgency=low + + * depend on invirt-config, which is provided by xvm-devconfig + and xvm-prodconfig. + + -- Greg Price Mon, 29 Sep 2008 06:03:54 +0000 + +sipb-xen-base (8.18) unstable; urgency=low + + * Use production k5login + * Include openafs component + + -- Sam Hartman Fri, 22 Aug 2008 16:37:01 -0400 + +sipb-xen-base (8.17) unstable; urgency=low + + * removed the `mako` program since python-mako already includes a + mako-render + + -- Yang Zhang Wed, 13 Aug 2008 01:45:26 -0400 + +sipb-xen-base (8.16) unstable; urgency=low + + * added `mako` program for processing templates in invirt packages + + -- Yang Zhang Tue, 12 Aug 2008 16:00:43 -0400 + +sipb-xen-base (8.15) unstable; urgency=low + + * put invirt-getconf in /usr/bin + + -- Greg Price Sat, 2 Aug 2008 21:58:36 -0400 + +sipb-xen-base (8.14) unstable; urgency=low + + * using correct default paths in invirt-getconf + * fixed os.rename import bug + + -- Yang Zhang Sat, 2 Aug 2008 20:29:56 -0400 + +sipb-xen-base (8.13) unstable; urgency=low + + * fix an error message + + -- Greg Price Wed, 30 Jul 2008 23:04:07 -0400 + +sipb-xen-base (8.12) unstable; urgency=low + + [ Greg Price ] + * add --ls and some convenience features to invirt-getconf: + formatting composite nodes as yaml, allowing the root + * fix bug in late-import of yaml in invirt.config + * use yaml.CSafeDumper and yaml.CSafeLoader everywhere + + [ Yang Zhang ] + * added shared lock around cache-reading transaction + + -- Greg Price Wed, 30 Jul 2008 22:11:15 -0400 + +sipb-xen-base (8.11) unstable; urgency=low + + * sped up invirt.config module load time by rearranging imports + * no longer removing the lock file + + -- Yang Zhang Wed, 30 Jul 2008 21:55:23 -0400 + +sipb-xen-base (8.10) unstable; urgency=low + + * added file locking around cache + + -- Yang Zhang Tue, 29 Jul 2008 22:35:25 -0400 + +sipb-xen-base (8.9) unstable; urgency=low + + * moved more generic code into `common` package + * silently fail if cache fails + * load the configuration on module load + * produce a struct-based representation of the configuration + * allowing full exception messages for OSErrors (default behavior) + * added some dependencies specs + + -- Yang Zhang Tue, 29 Jul 2008 01:03:16 -0400 + +sipb-xen-base (8.8) unstable; urgency=low + + * added timestamp-based JSON caching of configuration for faster loading + * exposed (more) options to command-line frontend + * improved error messages/handling/help + * removed all python 2.5-isms + * reformatted to fit project style conventions + + -- Yang Zhang Mon, 28 Jul 2008 12:25:03 -0400 + +sipb-xen-base (8.7) unstable; urgency=low + + * back to 2.4 compatibility + + -- Greg Price Mon, 28 Jul 2008 07:44:27 -0400 + +sipb-xen-base (8.6) unstable; urgency=low + + * switching to python 2.5 only + * added y_z to .k5login + + -- Yang Zhang Sun, 27 Jul 2008 20:23:54 -0400 + +sipb-xen-base (8.5) unstable; urgency=low + + * using python 2.5 + * initial working version of invirt-getconf that reads & navigates YAML + + -- Yang Zhang Sun, 27 Jul 2008 20:12:28 -0400 + +sipb-xen-base (8.4) unstable; urgency=low + + [ Greg Price ] + * begin a Python package 'invirt' + + [ Yang Zhang ] + * added invirt-getconf to read configuration files + + -- Yang Zhang Sun, 27 Jul 2008 19:10:10 -0400 + +sipb-xen-base (8.3) unstable; urgency=low + + * add invirt-reload to regenerate and reload configs + + -- Greg Price Mon, 21 Jul 2008 20:35:42 -0400 + +sipb-xen-base (8.2) unstable; urgency=low + + * leave out debathena-system so we get only the packages we ask for + + -- Greg Price Sun, 20 Jul 2008 13:58:52 -0400 + +sipb-xen-base (8.1) unstable; urgency=low + + * include debathena to make stuff like kerberos config easier + + -- Greg Price Sat, 19 Jul 2008 23:16:47 -0400 + +sipb-xen-base (8) unstable; urgency=low + + * update .k5login to match black-mesa + + -- Greg Price Sun, 4 May 2008 20:28:28 -0400 + +sipb-xen-base (7) unstable; urgency=low + + * include backports.org + * sources.list.d doesn't actually need a .sources.list, just .list + + -- Greg Price Thu, 1 May 2008 19:45:50 -0400 + +sipb-xen-base (6) unstable; urgency=low + + * actually use sources.list.d correctly + + -- Greg Price Sat, 26 Apr 2008 21:22:12 -0400 + +sipb-xen-base (5) unstable; urgency=low + + * update sources.list, use sources.list.d + + -- Greg Price Sat, 26 Apr 2008 21:06:05 -0400 + +sipb-xen-base (4) unstable; urgency=low + + * sipb-vm-1 becomes sipb-xen-dev + + -- Sam Hartman Tue, 4 Sep 2007 15:48:50 -0400 + +sipb-xen-base (3) unstable; urgency=low + + * We want security updates too + + -- Sam Hartman Fri, 10 Aug 2007 20:39:14 -0400 + +sipb-xen-base (2) unstable; urgency=low + + * Update sources.list to include our debian mirror + + -- Sam Hartman Sat, 4 Aug 2007 19:11:18 -0400 + +sipb-xen-base (1) unstable; urgency=low + + * New upstream version + + -- Sam Hartman Sat, 4 Aug 2007 18:44:21 -0400 + Index: trunk/packages/invirt-base/debian/compat =================================================================== --- trunk/packages/invirt-base/debian/compat (revision 1330) +++ trunk/packages/invirt-base/debian/compat (revision 1330) @@ -0,0 +1,1 @@ +4 Index: trunk/packages/invirt-base/debian/control =================================================================== --- trunk/packages/invirt-base/debian/control (revision 1330) +++ trunk/packages/invirt-base/debian/control (revision 1330) @@ -0,0 +1,21 @@ +Source: invirt-base +Section: base +Priority: extra +Maintainer: Invirt project +Build-Depends: cdbs (>= 0.4.23-1.1), debhelper (>= 4.1.0), python-all-dev, python-support, python-setuptools, python-debian, python-apt +Standards-Version: 3.8.0 + +Package: invirt-base +Architecture: all +Depends: ${python:Depends}, ${misc:Depends}, python-json (>= 3.4-2), python-yaml (>= 3.05), python-mako (>= 0.2.2), invirt-config +Provides: ${python:Provides} +XB-Python-Version: ${python:Versions} +Description: Base configuration required for all Invirt servers + This package includes common files for the Invirt system. + * apt configuration + * common init-script libraries + * common Python libraries, including the root of the 'invirt' tree + * scripts invirt-getconf and invirt-reload for using + the Invirt configuration system + . + All Invirt hosts and system VMs should have this package. Index: trunk/packages/invirt-base/debian/copyright =================================================================== --- trunk/packages/invirt-base/debian/copyright (revision 1330) +++ trunk/packages/invirt-base/debian/copyright (revision 1330) @@ -0,0 +1,16 @@ +This software was written as part of the Invirt project . + +Copyright : + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + +On Debian systems, the complete text of the GNU General Public License +can be found in the file /usr/share/common-licenses/GPL. Index: trunk/packages/invirt-base/debian/invirt-base.dirs =================================================================== --- trunk/packages/invirt-base/debian/invirt-base.dirs (revision 1330) +++ trunk/packages/invirt-base/debian/invirt-base.dirs (revision 1330) @@ -0,0 +1,1 @@ +usr/sbin Index: trunk/packages/invirt-base/debian/invirt-base.install =================================================================== --- trunk/packages/invirt-base/debian/invirt-base.install (revision 1330) +++ trunk/packages/invirt-base/debian/invirt-base.install (revision 1330) @@ -0,0 +1,1 @@ +files/* . Index: trunk/packages/invirt-base/debian/pycompat =================================================================== --- trunk/packages/invirt-base/debian/pycompat (revision 1330) +++ trunk/packages/invirt-base/debian/pycompat (revision 1330) @@ -0,0 +1,1 @@ +2 Index: trunk/packages/invirt-base/debian/pyversions =================================================================== --- trunk/packages/invirt-base/debian/pyversions (revision 1330) +++ trunk/packages/invirt-base/debian/pyversions (revision 1330) @@ -0,0 +1,1 @@ +2.5- Index: trunk/packages/invirt-base/debian/rules =================================================================== --- trunk/packages/invirt-base/debian/rules (revision 1330) +++ trunk/packages/invirt-base/debian/rules (revision 1330) @@ -0,0 +1,12 @@ +#!/usr/bin/make -f + +DEB_PYTHON_SYSTEM=pysupport + +include /usr/share/cdbs/1/rules/debhelper.mk +include /usr/share/cdbs/1/class/python-distutils.mk + +binary-fixup/invirt-base:: + mv $(DEB_DESTDIR)usr/bin/invirt-reload $(DEB_DESTDIR)usr/sbin/invirt-reload + +clean:: + rm -rf python/invirt.egg-info Index: trunk/packages/invirt-base/files/etc/apt/sources.list.d/debathena.list =================================================================== --- trunk/packages/invirt-base/files/etc/apt/sources.list.d/debathena.list (revision 1330) +++ trunk/packages/invirt-base/files/etc/apt/sources.list.d/debathena.list (revision 1330) @@ -0,0 +1,2 @@ +deb http://debathena.mit.edu/apt hardy debathena debathena-config openafs +deb-src http://debathena.mit.edu/apt hardy debathena debathena-config openafs Index: trunk/packages/invirt-base/files/etc/apt/sources.list.d/invirt.list =================================================================== --- trunk/packages/invirt-base/files/etc/apt/sources.list.d/invirt.list (revision 1330) +++ trunk/packages/invirt-base/files/etc/apt/sources.list.d/invirt.list (revision 1330) @@ -0,0 +1,4 @@ +deb http://xvm-2.mit.edu/sipb-xen stable main +deb-src http://xvm-2.mit.edu/sipb-xen stable main +deb http://mirrors.ccs.neu.edu/ubuntu/ hardy-backports main universe +deb-src http://mirrors.ccs.neu.edu/ubuntu/ hardy-backports main universe Index: trunk/packages/invirt-base/files/lib/init/config-init.sh =================================================================== --- trunk/packages/invirt-base/files/lib/init/config-init.sh (revision 1330) +++ trunk/packages/invirt-base/files/lib/init/config-init.sh (revision 1330) @@ -0,0 +1,33 @@ +# For a package which only configures another, "parent" package. +# +# Global variable PARENTPACKAGE names parent; may be array +# for zero or many parents. +# +# Global variable PACKAGE names this package, for log message. +# +# Requires bash. + +. /lib/init/vars.sh +. /lib/lsb/init-functions +. /lib/init/gen-files.sh + +config_init () { + case "$1" in + start|reload|force-reload|restart) + log_begin_msg "Reloading config for $PACKAGE" + gen_files + log_end_msg $? + for p in "${PARENTPACKAGE[@]}"; do + /etc/init.d/"$p" "$1" + done + ;; + stop) + for p in "${PARENTPACKAGE[@]}"; do + /etc/init.d/"$p" "$1" + done + ;; + *) + log_success_msg "Usage: /etc/init.d/$PACKAGE {start|reload|force-reload|restart|stop}" + ;; + esac +} Index: trunk/packages/invirt-base/files/lib/init/gen-files.sh =================================================================== --- trunk/packages/invirt-base/files/lib/init/gen-files.sh (revision 1330) +++ trunk/packages/invirt-base/files/lib/init/gen-files.sh (revision 1330) @@ -0,0 +1,17 @@ +# Generates files from templates. +# Files should be named in an array variable GEN_FILES. +# If BASH_VERSION is null or unset, accepts only one file. + +if [ $BASH_VERSION ]; then + gen_files() + { + for f in "${GEN_FILES[@]}"; do + mako-render "$f".mako >"$f" + done + } +else + gen_files() + { + mako-render "$GEN_FILES".mako >"$GEN_FILES" + } +fi Index: trunk/packages/invirt-base/files/lib/init/std-init.sh =================================================================== --- trunk/packages/invirt-base/files/lib/init/std-init.sh (revision 1330) +++ trunk/packages/invirt-base/files/lib/init/std-init.sh (revision 1330) @@ -0,0 +1,92 @@ +# Typical Debian initscript, but as a library rather than copy-paste. +# +# Usage: +# NAME=short-name +# DESC="Textual description" +# SCRIPTNAME=/etc/init.d/$NAME # default if omitted +# . /lib/init/std-init.sh +# do_start() { ... } +# do_stop() { ... } +# do_reload() { ... } # optional +# std_init "$1" + +. /lib/init/vars.sh +. /lib/lsb/init-functions + +[ -r /etc/default/"$NAME" ] && . /etc/default/"$NAME" + +SCRIPTNAME="${SCRIPTNAME:-/etc/init.d/$NAME}" + +have_reload() +{ + type do_reload >/dev/null 2>/dev/null +} + +usage_exit() +{ + if have_reload; then + echo "Usage: $SCRIPTNAME {start|stop|restart|force-reload}" >&2 + else + echo "Usage: $SCRIPTNAME {start|stop|restart|reload|force-reload}" >&2 + fi + exit 3 +} + +std_init() +{ + local cmd + case "$1" in + start|stop|restart) + cmd="$1" ;; + reload) + if have_reload; then cmd=reload; else usage_exit; fi ;; + force-reload) + if have_reload; then cmd=reload; else cmd=restart; fi ;; + *) + usage_exit ;; + esac + + case $cmd in + start) + [ "$VERBOSE" != no ] && log_daemon_msg "Starting $DESC" "$NAME" + do_start + case "$?" in + 0|1) [ "$VERBOSE" != no ] && log_end_msg 0 ;; + 2) [ "$VERBOSE" != no ] && log_end_msg 1 ;; + esac + ;; + stop) + [ "$VERBOSE" != no ] && log_daemon_msg "Stopping $DESC" "$NAME" + do_stop + case "$?" in + 0|1) [ "$VERBOSE" != no ] && log_end_msg 0 ;; + 2) [ "$VERBOSE" != no ] && log_end_msg 1 ;; + esac + ;; + reload) + log_daemon_msg "Reloading $DESC" "$NAME" + do_reload + log_end_msg $? + ;; + restart) + log_daemon_msg "Restarting $DESC" "$NAME" + do_stop + case "$?" in + 0|1) + do_start + case "$?" in + 0) log_end_msg 0 ;; + 1) log_end_msg 1 ;; # Old process is still running + *) log_end_msg 1 ;; # Failed to start + esac + ;; + *) + # Failed to stop + log_end_msg 1 + ;; + esac + ;; + esac + + : +} Index: trunk/packages/invirt-base/files/root/.k5login =================================================================== --- trunk/packages/invirt-base/files/root/.k5login (revision 1330) +++ trunk/packages/invirt-base/files/root/.k5login (revision 1330) @@ -0,0 +1,6 @@ +andersk/root@ATHENA.MIT.EDU +broder/root@ATHENA.MIT.EDU +ecprice/root@ATHENA.MIT.EDU +hartmans/root@ATHENA.MIT.EDU +price/root@ATHENA.MIT.EDU +quentin/root@ATHENA.MIT.EDU Index: trunk/packages/invirt-base/python/invirt/__init__.py =================================================================== --- trunk/packages/invirt-base/python/invirt/__init__.py (revision 1330) +++ trunk/packages/invirt-base/python/invirt/__init__.py (revision 1330) @@ -0,0 +1,27 @@ +'''Invirt - a virtualization management system + +Invirt was developed at the Student Information Processing Board of +the Massachusetts Institute of Technology. See http://xvm.mit.edu/. + +Invirt is free software available under the GNU GPL, version 2 or later. +Consult the source files for details. +''' + +# Invirt is free software: you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the +# Free Software Foundation, either version 2 of the License, or (at your +# option) any later version. + +# Invirt is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +# for more details. + +# You should have received a copy of the GNU General Public License +# along with Invirt. If not, see . + +__author__ = 'MIT SIPB' +__version__ = '0.1' +__copyright__ = 'Copyright (c) 2008 MIT SIPB' + +# vim:et:sw=4:ts=4 Index: trunk/packages/invirt-base/python/invirt/common.py =================================================================== --- trunk/packages/invirt-base/python/invirt/common.py (revision 1330) +++ trunk/packages/invirt-base/python/invirt/common.py (revision 1330) @@ -0,0 +1,58 @@ +from __future__ import with_statement + +import unittest +from fcntl import flock, LOCK_EX, LOCK_SH, LOCK_UN +import contextlib as clib + +class struct(object): + 'A simple namespace object.' + def __init__(self, d = {}, **kwargs): + 'd is the dictionary to update my __dict__ with.' + self.__dict__.update(d) + self.__dict__.update(kwargs) + +def dicts2struct(x): + """ + Given a tree of lists/dicts, perform a deep traversal to transform all the + dicts to structs. + """ + if type(x) == dict: + return struct((k, dicts2struct(v)) for k,v in x.iteritems()) + elif type(x) == list: + return [dicts2struct(v) for v in x] + else: + return x + +@clib.contextmanager +def lock_file(path, exclusive = True): + with clib.closing(file(path, 'w')) as f: + if exclusive: + locktype = LOCK_EX + else: + locktype = LOCK_SH + flock(f, locktype) + try: + yield + finally: + flock(f, LOCK_UN) + +# +# Tests. +# + +class common_tests(unittest.TestCase): + def test_dicts2structs(self): + dicts = { + 'atom': 0, + 'dict': { 'atom': 'atom', 'list': [1,2,3] }, + 'list': [ 'atom', {'key': 'value'} ] + } + structs = dicts2struct(dicts) + self.assertEqual(structs.atom, dicts['atom']) + self.assertEqual(structs.dict.atom, dicts['dict']['atom']) + self.assertEqual(structs.dict.list, dicts['dict']['list']) + self.assertEqual(structs.list[0], dicts['list'][0]) + self.assertEqual(structs.list[1].key, dicts['list'][1]['key']) + +if __name__ == '__main__': + unittest.main() Index: trunk/packages/invirt-base/python/invirt/config.py =================================================================== --- trunk/packages/invirt-base/python/invirt/config.py (revision 1330) +++ trunk/packages/invirt-base/python/invirt/config.py (revision 1330) @@ -0,0 +1,91 @@ +from __future__ import with_statement + +import json +from invirt.common import * +from os import rename +from os.path import getmtime +from contextlib import closing + +default_src_path = '/etc/invirt/master.yaml' +default_cache_path = '/var/lib/invirt/cache.json' +lock_path = '/var/lib/invirt/cache.lock' + +def load(src_path = default_src_path, + cache_path = default_cache_path, + force_refresh = False): + """ + Try loading the configuration from the faster-to-load JSON cache at + cache_path. If it doesn't exist or is outdated, load the configuration + instead from the original YAML file at src_path and regenerate the cache. + I assume I have the permissions to write to the cache directory. + """ + + # Namespace container for state variables, so that they can be updated by + # closures. + ns = struct() + + if force_refresh: + do_refresh = True + else: + src_mtime = getmtime(src_path) + try: cache_mtime = getmtime(cache_path) + except OSError: do_refresh = True + else: do_refresh = src_mtime + 1 >= cache_mtime + + # We chose not to simply say + # + # do_refresh = src_mtime >= cache_time + # + # because between the getmtime(src_path) and the time the cache is + # rewritten, the master configuration may have been updated, so future + # checks here would find a cache with a newer mtime than the master + # (and thus treat the cache as containing the latest version of the + # master). The +1 means that for at least a full second following the + # update to the master, this function will refresh the cache, giving us + # 1 second to write the cache. Note that if it takes longer than 1 + # second to write the cache, then this situation could still arise. + # + # The getmtime calls should logically be part of the same transaction + # as the rest of this function (cache read + conditional cache + # refresh), but to wrap everything in an flock would cause the + # following cache read to be less streamlined. + + if not do_refresh: + # Try reading from the cache first. This must be transactionally + # isolated from concurrent writes to prevent reading an incomplete + # (changing) version of the data (but the transaction can share the + # lock with other concurrent reads). This isolation is accomplished + # using an atomic filesystem rename in the refreshing stage. + try: + with closing(file(cache_path)) as f: + ns.cfg = json.read(f.read()) + except: do_refresh = True + + if do_refresh: + # Atomically reload the source and regenerate the cache. The read and + # write must be a single transaction, or a stale version may be + # written (if another read/write of a more recent configuration + # is interleaved). The final atomic rename is to keep this + # transactionally isolated from the above cache read. If we fail to + # acquire the lock, just try to load the master configuration. + import yaml + try: loader = yaml.CSafeLoader + except: loader = yaml.SafeLoader + try: + with lock_file(lock_path): + with closing(file(src_path)) as f: + ns.cfg = yaml.load(f, loader) + try: + with closing(file(cache_path + '.tmp', 'w')) as f: + f.write(json.write(ns.cfg)) + except: pass # silent failure + else: rename(cache_path + '.tmp', cache_path) + except IOError: + with closing(file(src_path)) as f: + ns.cfg = yaml.load(f, loader) + return ns.cfg + +dicts = load() +structs = dicts2struct(dicts) + +# vim:et:sw=4:ts=4 Index: trunk/packages/invirt-base/python/invirt/remote.py =================================================================== --- trunk/packages/invirt-base/python/invirt/remote.py (revision 1330) +++ trunk/packages/invirt-base/python/invirt/remote.py (revision 1330) @@ -0,0 +1,19 @@ +from subprocess import PIPE, Popen +from invirt.config import structs as config +import yaml + +def bcast(cmd, hosts = [h.hostname for h in config.hosts]): + """ + Given a command and a list of hostnames or IPs, issue the command to all + the nodes and return a list of (host, output) pairs (the order should be + the same as the order of the hosts). + """ + pipes = [(host, + Popen(['remctl', host, 'remote', 'web', cmd], stdout=PIPE)) + for host in hosts] + outputs = [(s, p.communicate()[0]) for (s, p) in pipes] + for (s, p) in pipes: + if p.returncode != 0: + raise RuntimeError("remctl to host %s returned non-zero exit status %d" + % (s, p.returncode)) + return [(s, yaml.load(o, yaml.CSafeLoader)) for (s, o) in outputs] Index: trunk/packages/invirt-base/scripts/invirt-getconf =================================================================== --- trunk/packages/invirt-base/scripts/invirt-getconf (revision 1330) +++ trunk/packages/invirt-base/scripts/invirt-getconf (revision 1330) @@ -0,0 +1,92 @@ +#!/usr/bin/env python + +""" +invirt-getconf loads an invirt configuration file (either the original YAML +source or the faster-to-load JSON cache) and prints the configuration option +with the given name (key). Keys are dot-separated paths into the YAML +configuration tree. List indexes (0-based) are also treated as path +components. + +(Due to this path language, certain restrictions are placed on the keys used in +the YAML configuration; e.g., they cannot contain dots.) + +Examples: + + invirt-getconf db.uri + invirt-getconf authn.0.type +""" + +from invirt.config import load +from sys import argv, exit, stderr, stdout +from optparse import OptionParser + +class invirt_exception(Exception): pass + +def main(argv): + try: + parser = OptionParser(usage = '%prog [options] key', + description = __doc__.strip().split('\n\n')[0]) + parser.add_option('-s', '--src', + default = '/etc/invirt/master.yaml', + help = 'the source YAML configuration file to read from') + parser.add_option('-c', '--cache', + default = '/var/lib/invirt/invirt.json', + help = 'path to the JSON cache') + parser.add_option('-r', '--refresh', + action = 'store_true', + help = 'force the cache to be regenerated') + parser.add_option('-l', '--ls', + action = 'store_true', + help = 'list node\'s children') + opts, args = parser.parse_args() + + if len(args) > 1: + raise invirt_exception(__doc__.strip()) + elif args and args[0]: + components = args[0].split('.') + else: + components = [] + + conf = load(opts.src, opts.cache, opts.refresh) + for i, component in enumerate(components): + progress = '.'.join(components[:i]) + if type(conf) not in (dict, list): + raise invirt_exception( + '%s: node has no children (atomic datum)' % progress) + if type(conf) == list: + try: component = int(component) + except: raise invirt_exception( + '%s: node a list; integer path component required, ' + 'but got "%s"' % (progress, component)) + try: conf = conf[component] + except KeyError: raise invirt_exception( + '%s: key "%s" not found' % (progress, component)) + except IndexError: raise invirt_exception( + '%s: index %s out of range' % (progress, component)) + + if opts.ls: + if type(conf) not in (dict, list): + raise invirt_exception( + '%s: node has no children (atomic datum)' + % '.'.join(components)) + if type(conf) == list: + for i in xrange(len(conf)): + print i + else: + for k in conf.iterkeys(): + print k + else: + if type(conf) not in (dict, list): + print conf + else: + import yaml + yaml.dump(conf, stdout, + Dumper=yaml.CSafeDumper, default_flow_style=False) + except invirt_exception, ex: + print >> stderr, ex + return 1 + +if __name__ == '__main__': + exit(main(argv)) + +# vim:et:sw=4:ts=4 Index: trunk/packages/invirt-base/scripts/invirt-reload =================================================================== --- trunk/packages/invirt-base/scripts/invirt-reload (revision 1330) +++ trunk/packages/invirt-base/scripts/invirt-reload (revision 1330) @@ -0,0 +1,8 @@ +#!/bin/bash + +for script in $(run-parts --test /etc/init.d); do + if [ "${script#/etc/init.d/sipb-xen-}" != "$script" \ + -o "${script#/etc/init.d/invirt-}" != "$script" ]; then + invoke-rc.d "${script#/etc/init.d/}" reload + fi +done Index: trunk/packages/invirt-base/setup.py =================================================================== --- trunk/packages/invirt-base/setup.py (revision 1330) +++ trunk/packages/invirt-base/setup.py (revision 1330) @@ -0,0 +1,24 @@ +#!/usr/bin/python + +from os import path +from debian_bundle.changelog import Changelog +from debian_bundle.deb822 import Deb822 +from email.utils import parseaddr +from setuptools import setup + +version = Changelog(open(path.join(path.dirname(__file__), 'debian/changelog')).read()).\ + get_version().full_version + +maintainer_full = Deb822(open(path.join(path.dirname(__file__), 'debian/control')))['Maintainer'] +maintainer, maintainer_email = parseaddr(maintainer_full) + +setup( + name='invirt', + version=version, + maintainer=maintainer, + maintainer_email=maintainer_full, + + packages = ['invirt'], + package_dir = {'': 'python'}, + scripts=['scripts/invirt-getconf', 'scripts/invirt-reload'] +) Index: trunk/packages/sipb-xen-base/config.todo =================================================================== --- trunk/packages/sipb-xen-base/config.todo (revision 1329) +++ (revision ) @@ -1,2 +1,0 @@ -files/etc/apt/sources.list.d/debathena.list: mit kerberos config -files/etc/apt/sources.list.d/sipb-xen.list: apt repo uri Index: trunk/packages/sipb-xen-base/debian/changelog =================================================================== --- trunk/packages/sipb-xen-base/debian/changelog (revision 1329) +++ (revision ) @@ -1,251 +1,0 @@ -sipb-xen-base (8.29) unstable; urgency=low - - * gen-files.sh: degrade gracefully to plain-old sh - - -- Greg Price Sat, 25 Oct 2008 20:06:54 -0400 - -sipb-xen-base (8.28) unstable; urgency=low - - * Factor out more common initscript code: - std-init.sh replaces the usual Debian boilerplate - - -- Greg Price Sat, 25 Oct 2008 15:33:08 -0400 - -sipb-xen-base (8.27) unstable; urgency=low - - * Use invoke-rc.d in invirt-reload so that policy-rc.d is respected - - -- Evan Broder Fri, 24 Oct 2008 13:19:49 -0400 - -sipb-xen-base (8.26) unstable; urgency=low - - * Move invirt-reload to /usr/sbin - - -- Evan Broder Fri, 24 Oct 2008 12:53:18 -0400 - -sipb-xen-base (8.25) unstable; urgency=low - - * Factor out common initscript code, provide it here - - -- Greg Price Fri, 24 Oct 2008 07:06:59 -0400 - -sipb-xen-base (8.24) unstable; urgency=low - - * Switch to using a setup.py file with CDBS's Python support - - -- Evan Broder Fri, 24 Oct 2008 05:22:49 -0400 - -sipb-xen-base (8.23) unstable; urgency=low - - * Now that we're using Python 2.5, we can actually write with statements - - -- Evan Broder Fri, 24 Oct 2008 03:32:00 -0400 - -sipb-xen-base (8.22) unstable; urgency=low - - * Please - anything but etch. Anything! - - -- Evan Broder Thu, 02 Oct 2008 19:50:30 -0400 - -sipb-xen-base (8.21) unstable; urgency=low - - * update sources.list.d/sipb-xen.list for prod and hardy - - -- Greg Price Wed, 01 Oct 2008 20:06:52 -0400 - -sipb-xen-base (8.20) unstable; urgency=low - - * fix distribution - - -- Greg Price Tue, 30 Sep 2008 23:51:11 -0400 - -sipb-xen-base (8.19) hardy; urgency=low - - * depend on invirt-config, which is provided by xvm-devconfig - and xvm-prodconfig. - - -- Greg Price Mon, 29 Sep 2008 06:03:54 +0000 - -sipb-xen-base (8.18) unstable; urgency=low - - * Use production k5login - * Include openafs component - - -- Sam Hartman Fri, 22 Aug 2008 16:37:01 -0400 - -sipb-xen-base (8.17) unstable; urgency=low - - * removed the `mako` program since python-mako already includes a - mako-render - - -- Yang Zhang Wed, 13 Aug 2008 01:45:26 -0400 - -sipb-xen-base (8.16) unstable; urgency=low - - * added `mako` program for processing templates in invirt packages - - -- Yang Zhang Tue, 12 Aug 2008 16:00:43 -0400 - -sipb-xen-base (8.15) unstable; urgency=low - - * put invirt-getconf in /usr/bin - - -- Greg Price Sat, 2 Aug 2008 21:58:36 -0400 - -sipb-xen-base (8.14) unstable; urgency=low - - * using correct default paths in invirt-getconf - * fixed os.rename import bug - - -- Yang Zhang Sat, 2 Aug 2008 20:29:56 -0400 - -sipb-xen-base (8.13) unstable; urgency=low - - * fix an error message - - -- Greg Price Wed, 30 Jul 2008 23:04:07 -0400 - -sipb-xen-base (8.12) unstable; urgency=low - - [ Greg Price ] - * add --ls and some convenience features to invirt-getconf: - formatting composite nodes as yaml, allowing the root - * fix bug in late-import of yaml in invirt.config - * use yaml.CSafeDumper and yaml.CSafeLoader everywhere - - [ Yang Zhang ] - * added shared lock around cache-reading transaction - - -- Greg Price Wed, 30 Jul 2008 22:11:15 -0400 - -sipb-xen-base (8.11) unstable; urgency=low - - * sped up invirt.config module load time by rearranging imports - * no longer removing the lock file - - -- Yang Zhang Wed, 30 Jul 2008 21:55:23 -0400 - -sipb-xen-base (8.10) unstable; urgency=low - - * added file locking around cache - - -- Yang Zhang Tue, 29 Jul 2008 22:35:25 -0400 - -sipb-xen-base (8.9) unstable; urgency=low - - * moved more generic code into `common` package - * silently fail if cache fails - * load the configuration on module load - * produce a struct-based representation of the configuration - * allowing full exception messages for OSErrors (default behavior) - * added some dependencies specs - - -- Yang Zhang Tue, 29 Jul 2008 01:03:16 -0400 - -sipb-xen-base (8.8) unstable; urgency=low - - * added timestamp-based JSON caching of configuration for faster loading - * exposed (more) options to command-line frontend - * improved error messages/handling/help - * removed all python 2.5-isms - * reformatted to fit project style conventions - - -- Yang Zhang Mon, 28 Jul 2008 12:25:03 -0400 - -sipb-xen-base (8.7) unstable; urgency=low - - * back to 2.4 compatibility - - -- Greg Price Mon, 28 Jul 2008 07:44:27 -0400 - -sipb-xen-base (8.6) unstable; urgency=low - - * switching to python 2.5 only - * added y_z to .k5login - - -- Yang Zhang Sun, 27 Jul 2008 20:23:54 -0400 - -sipb-xen-base (8.5) unstable; urgency=low - - * using python 2.5 - * initial working version of invirt-getconf that reads & navigates YAML - - -- Yang Zhang Sun, 27 Jul 2008 20:12:28 -0400 - -sipb-xen-base (8.4) unstable; urgency=low - - [ Greg Price ] - * begin a Python package 'invirt' - - [ Yang Zhang ] - * added invirt-getconf to read configuration files - - -- Yang Zhang Sun, 27 Jul 2008 19:10:10 -0400 - -sipb-xen-base (8.3) unstable; urgency=low - - * add invirt-reload to regenerate and reload configs - - -- Greg Price Mon, 21 Jul 2008 20:35:42 -0400 - -sipb-xen-base (8.2) unstable; urgency=low - - * leave out debathena-system so we get only the packages we ask for - - -- Greg Price Sun, 20 Jul 2008 13:58:52 -0400 - -sipb-xen-base (8.1) unstable; urgency=low - - * include debathena to make stuff like kerberos config easier - - -- Greg Price Sat, 19 Jul 2008 23:16:47 -0400 - -sipb-xen-base (8) unstable; urgency=low - - * update .k5login to match black-mesa - - -- Greg Price Sun, 4 May 2008 20:28:28 -0400 - -sipb-xen-base (7) unstable; urgency=low - - * include backports.org - * sources.list.d doesn't actually need a .sources.list, just .list - - -- Greg Price Thu, 1 May 2008 19:45:50 -0400 - -sipb-xen-base (6) unstable; urgency=low - - * actually use sources.list.d correctly - - -- Greg Price Sat, 26 Apr 2008 21:22:12 -0400 - -sipb-xen-base (5) unstable; urgency=low - - * update sources.list, use sources.list.d - - -- Greg Price Sat, 26 Apr 2008 21:06:05 -0400 - -sipb-xen-base (4) unstable; urgency=low - - * sipb-vm-1 becomes sipb-xen-dev - - -- Sam Hartman Tue, 4 Sep 2007 15:48:50 -0400 - -sipb-xen-base (3) unstable; urgency=low - - * We want security updates too - - -- Sam Hartman Fri, 10 Aug 2007 20:39:14 -0400 - -sipb-xen-base (2) unstable; urgency=low - - * Update sources.list to include our debian mirror - - -- Sam Hartman Sat, 4 Aug 2007 19:11:18 -0400 - -sipb-xen-base (1) unstable; urgency=low - - * New upstream version - - -- Sam Hartman Sat, 4 Aug 2007 18:44:21 -0400 - Index: trunk/packages/sipb-xen-base/debian/compat =================================================================== --- trunk/packages/sipb-xen-base/debian/compat (revision 1329) +++ (revision ) @@ -1,1 +1,0 @@ -4 Index: trunk/packages/sipb-xen-base/debian/control =================================================================== --- trunk/packages/sipb-xen-base/debian/control (revision 1329) +++ (revision ) @@ -1,16 +1,0 @@ -Source: sipb-xen-base -Section: base -Priority: extra -Maintainer: SIPB Xen Project -Build-Depends: cdbs (>= 0.4.23-1.1), debhelper (>= 4.1.0), python-all-dev, python-support, python-setuptools, python-debian, python-apt -Standards-Version: 3.8.0 - -Package: sipb-xen-base -Architecture: all -Depends: ${python:Depends}, ${misc:Depends}, python-json (>= 3.4-2), python-yaml (>= 3.05), python-mako (>= 0.2.2), invirt-config -Provides: ${python:Provides} -XB-Python-Version: ${python:Versions} -Description: Base configuration required for all SIPB xen servers - This package includes apt configuration, .k5login and other files that - should be synchronized among all our servers. - Installing this on a non-sipb-xen machine would be very anti-social. Index: trunk/packages/sipb-xen-base/debian/copyright =================================================================== --- trunk/packages/sipb-xen-base/debian/copyright (revision 1329) +++ (revision ) @@ -1,16 +1,0 @@ -This software was written as part of the Invirt project . - -Copyright : - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 2 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - -On Debian systems, the complete text of the GNU General Public License -can be found in the file /usr/share/common-licenses/GPL. Index: trunk/packages/sipb-xen-base/debian/pycompat =================================================================== --- trunk/packages/sipb-xen-base/debian/pycompat (revision 1329) +++ (revision ) @@ -1,1 +1,0 @@ -2 Index: trunk/packages/sipb-xen-base/debian/pyversions =================================================================== --- trunk/packages/sipb-xen-base/debian/pyversions (revision 1329) +++ (revision ) @@ -1,1 +1,0 @@ -2.5- Index: trunk/packages/sipb-xen-base/debian/rules =================================================================== --- trunk/packages/sipb-xen-base/debian/rules (revision 1329) +++ (revision ) @@ -1,12 +1,0 @@ -#!/usr/bin/make -f - -DEB_PYTHON_SYSTEM=pysupport - -include /usr/share/cdbs/1/rules/debhelper.mk -include /usr/share/cdbs/1/class/python-distutils.mk - -binary-fixup/sipb-xen-base:: - mv $(DEB_DESTDIR)usr/bin/invirt-reload $(DEB_DESTDIR)usr/sbin/invirt-reload - -clean:: - rm -rf python/invirt.egg-info Index: trunk/packages/sipb-xen-base/debian/sipb-xen-base.dirs =================================================================== --- trunk/packages/sipb-xen-base/debian/sipb-xen-base.dirs (revision 1329) +++ (revision ) @@ -1,1 +1,0 @@ -usr/sbin Index: trunk/packages/sipb-xen-base/debian/sipb-xen-base.install =================================================================== --- trunk/packages/sipb-xen-base/debian/sipb-xen-base.install (revision 1329) +++ (revision ) @@ -1,1 +1,0 @@ -files/* . Index: trunk/packages/sipb-xen-base/files/etc/apt/sources.list.d/debathena.list =================================================================== --- trunk/packages/sipb-xen-base/files/etc/apt/sources.list.d/debathena.list (revision 1329) +++ (revision ) @@ -1,2 +1,0 @@ -deb http://debathena.mit.edu/apt hardy debathena debathena-config openafs -deb-src http://debathena.mit.edu/apt hardy debathena debathena-config openafs Index: trunk/packages/sipb-xen-base/files/etc/apt/sources.list.d/sipb-xen.list =================================================================== --- trunk/packages/sipb-xen-base/files/etc/apt/sources.list.d/sipb-xen.list (revision 1329) +++ (revision ) @@ -1,4 +1,0 @@ -deb http://xvm-2.mit.edu/sipb-xen stable main -deb-src http://xvm-2.mit.edu/sipb-xen stable main -deb http://mirrors.ccs.neu.edu/ubuntu/ hardy-backports main universe -deb-src http://mirrors.ccs.neu.edu/ubuntu/ hardy-backports main universe Index: trunk/packages/sipb-xen-base/files/lib/init/config-init.sh =================================================================== --- trunk/packages/sipb-xen-base/files/lib/init/config-init.sh (revision 1329) +++ (revision ) @@ -1,33 +1,0 @@ -# For a package which only configures another, "parent" package. -# -# Global variable PARENTPACKAGE names parent; may be array -# for zero or many parents. -# -# Global variable PACKAGE names this package, for log message. -# -# Requires bash. - -. /lib/init/vars.sh -. /lib/lsb/init-functions -. /lib/init/gen-files.sh - -config_init () { - case "$1" in - start|reload|force-reload|restart) - log_begin_msg "Reloading config for $PACKAGE" - gen_files - log_end_msg $? - for p in "${PARENTPACKAGE[@]}"; do - /etc/init.d/"$p" "$1" - done - ;; - stop) - for p in "${PARENTPACKAGE[@]}"; do - /etc/init.d/"$p" "$1" - done - ;; - *) - log_success_msg "Usage: /etc/init.d/$PACKAGE {start|reload|force-reload|restart|stop}" - ;; - esac -} Index: trunk/packages/sipb-xen-base/files/lib/init/gen-files.sh =================================================================== --- trunk/packages/sipb-xen-base/files/lib/init/gen-files.sh (revision 1329) +++ (revision ) @@ -1,17 +1,0 @@ -# Generates files from templates. -# Files should be named in an array variable GEN_FILES. -# If BASH_VERSION is null or unset, accepts only one file. - -if [ $BASH_VERSION ]; then - gen_files() - { - for f in "${GEN_FILES[@]}"; do - mako-render "$f".mako >"$f" - done - } -else - gen_files() - { - mako-render "$GEN_FILES".mako >"$GEN_FILES" - } -fi Index: trunk/packages/sipb-xen-base/files/lib/init/std-init.sh =================================================================== --- trunk/packages/sipb-xen-base/files/lib/init/std-init.sh (revision 1329) +++ (revision ) @@ -1,92 +1,0 @@ -# Typical Debian initscript, but as a library rather than copy-paste. -# -# Usage: -# NAME=short-name -# DESC="Textual description" -# SCRIPTNAME=/etc/init.d/$NAME # default if omitted -# . /lib/init/std-init.sh -# do_start() { ... } -# do_stop() { ... } -# do_reload() { ... } # optional -# std_init "$1" - -. /lib/init/vars.sh -. /lib/lsb/init-functions - -[ -r /etc/default/"$NAME" ] && . /etc/default/"$NAME" - -SCRIPTNAME="${SCRIPTNAME:-/etc/init.d/$NAME}" - -have_reload() -{ - type do_reload >/dev/null 2>/dev/null -} - -usage_exit() -{ - if have_reload; then - echo "Usage: $SCRIPTNAME {start|stop|restart|force-reload}" >&2 - else - echo "Usage: $SCRIPTNAME {start|stop|restart|reload|force-reload}" >&2 - fi - exit 3 -} - -std_init() -{ - local cmd - case "$1" in - start|stop|restart) - cmd="$1" ;; - reload) - if have_reload; then cmd=reload; else usage_exit; fi ;; - force-reload) - if have_reload; then cmd=reload; else cmd=restart; fi ;; - *) - usage_exit ;; - esac - - case $cmd in - start) - [ "$VERBOSE" != no ] && log_daemon_msg "Starting $DESC" "$NAME" - do_start - case "$?" in - 0|1) [ "$VERBOSE" != no ] && log_end_msg 0 ;; - 2) [ "$VERBOSE" != no ] && log_end_msg 1 ;; - esac - ;; - stop) - [ "$VERBOSE" != no ] && log_daemon_msg "Stopping $DESC" "$NAME" - do_stop - case "$?" in - 0|1) [ "$VERBOSE" != no ] && log_end_msg 0 ;; - 2) [ "$VERBOSE" != no ] && log_end_msg 1 ;; - esac - ;; - reload) - log_daemon_msg "Reloading $DESC" "$NAME" - do_reload - log_end_msg $? - ;; - restart) - log_daemon_msg "Restarting $DESC" "$NAME" - do_stop - case "$?" in - 0|1) - do_start - case "$?" in - 0) log_end_msg 0 ;; - 1) log_end_msg 1 ;; # Old process is still running - *) log_end_msg 1 ;; # Failed to start - esac - ;; - *) - # Failed to stop - log_end_msg 1 - ;; - esac - ;; - esac - - : -} Index: trunk/packages/sipb-xen-base/files/root/.k5login =================================================================== --- trunk/packages/sipb-xen-base/files/root/.k5login (revision 1329) +++ (revision ) @@ -1,6 +1,0 @@ -andersk/root@ATHENA.MIT.EDU -broder/root@ATHENA.MIT.EDU -ecprice/root@ATHENA.MIT.EDU -hartmans/root@ATHENA.MIT.EDU -price/root@ATHENA.MIT.EDU -quentin/root@ATHENA.MIT.EDU Index: trunk/packages/sipb-xen-base/python/invirt/__init__.py =================================================================== --- trunk/packages/sipb-xen-base/python/invirt/__init__.py (revision 1329) +++ (revision ) @@ -1,27 +1,0 @@ -'''Invirt - a virtualization management system - -Invirt was developed at the Student Information Processing Board of -the Massachusetts Institute of Technology. See http://xvm.mit.edu/. - -Invirt is free software available under the GNU GPL, version 2 or later. -Consult the source files for details. -''' - -# Invirt is free software: you can redistribute it and/or modify it -# under the terms of the GNU General Public License as published by the -# Free Software Foundation, either version 2 of the License, or (at your -# option) any later version. - -# Invirt is distributed in the hope that it will be useful, but WITHOUT -# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License -# for more details. - -# You should have received a copy of the GNU General Public License -# along with Invirt. If not, see . - -__author__ = 'MIT SIPB' -__version__ = '0.1' -__copyright__ = 'Copyright (c) 2008 MIT SIPB' - -# vim:et:sw=4:ts=4 Index: trunk/packages/sipb-xen-base/python/invirt/common.py =================================================================== --- trunk/packages/sipb-xen-base/python/invirt/common.py (revision 1329) +++ (revision ) @@ -1,58 +1,0 @@ -from __future__ import with_statement - -import unittest -from fcntl import flock, LOCK_EX, LOCK_SH, LOCK_UN -import contextlib as clib - -class struct(object): - 'A simple namespace object.' - def __init__(self, d = {}, **kwargs): - 'd is the dictionary to update my __dict__ with.' - self.__dict__.update(d) - self.__dict__.update(kwargs) - -def dicts2struct(x): - """ - Given a tree of lists/dicts, perform a deep traversal to transform all the - dicts to structs. - """ - if type(x) == dict: - return struct((k, dicts2struct(v)) for k,v in x.iteritems()) - elif type(x) == list: - return [dicts2struct(v) for v in x] - else: - return x - -@clib.contextmanager -def lock_file(path, exclusive = True): - with clib.closing(file(path, 'w')) as f: - if exclusive: - locktype = LOCK_EX - else: - locktype = LOCK_SH - flock(f, locktype) - try: - yield - finally: - flock(f, LOCK_UN) - -# -# Tests. -# - -class common_tests(unittest.TestCase): - def test_dicts2structs(self): - dicts = { - 'atom': 0, - 'dict': { 'atom': 'atom', 'list': [1,2,3] }, - 'list': [ 'atom', {'key': 'value'} ] - } - structs = dicts2struct(dicts) - self.assertEqual(structs.atom, dicts['atom']) - self.assertEqual(structs.dict.atom, dicts['dict']['atom']) - self.assertEqual(structs.dict.list, dicts['dict']['list']) - self.assertEqual(structs.list[0], dicts['list'][0]) - self.assertEqual(structs.list[1].key, dicts['list'][1]['key']) - -if __name__ == '__main__': - unittest.main() Index: trunk/packages/sipb-xen-base/python/invirt/config.py =================================================================== --- trunk/packages/sipb-xen-base/python/invirt/config.py (revision 1329) +++ (revision ) @@ -1,91 +1,0 @@ -from __future__ import with_statement - -import json -from invirt.common import * -from os import rename -from os.path import getmtime -from contextlib import closing - -default_src_path = '/etc/invirt/master.yaml' -default_cache_path = '/var/lib/invirt/cache.json' -lock_path = '/var/lib/invirt/cache.lock' - -def load(src_path = default_src_path, - cache_path = default_cache_path, - force_refresh = False): - """ - Try loading the configuration from the faster-to-load JSON cache at - cache_path. If it doesn't exist or is outdated, load the configuration - instead from the original YAML file at src_path and regenerate the cache. - I assume I have the permissions to write to the cache directory. - """ - - # Namespace container for state variables, so that they can be updated by - # closures. - ns = struct() - - if force_refresh: - do_refresh = True - else: - src_mtime = getmtime(src_path) - try: cache_mtime = getmtime(cache_path) - except OSError: do_refresh = True - else: do_refresh = src_mtime + 1 >= cache_mtime - - # We chose not to simply say - # - # do_refresh = src_mtime >= cache_time - # - # because between the getmtime(src_path) and the time the cache is - # rewritten, the master configuration may have been updated, so future - # checks here would find a cache with a newer mtime than the master - # (and thus treat the cache as containing the latest version of the - # master). The +1 means that for at least a full second following the - # update to the master, this function will refresh the cache, giving us - # 1 second to write the cache. Note that if it takes longer than 1 - # second to write the cache, then this situation could still arise. - # - # The getmtime calls should logically be part of the same transaction - # as the rest of this function (cache read + conditional cache - # refresh), but to wrap everything in an flock would cause the - # following cache read to be less streamlined. - - if not do_refresh: - # Try reading from the cache first. This must be transactionally - # isolated from concurrent writes to prevent reading an incomplete - # (changing) version of the data (but the transaction can share the - # lock with other concurrent reads). This isolation is accomplished - # using an atomic filesystem rename in the refreshing stage. - try: - with closing(file(cache_path)) as f: - ns.cfg = json.read(f.read()) - except: do_refresh = True - - if do_refresh: - # Atomically reload the source and regenerate the cache. The read and - # write must be a single transaction, or a stale version may be - # written (if another read/write of a more recent configuration - # is interleaved). The final atomic rename is to keep this - # transactionally isolated from the above cache read. If we fail to - # acquire the lock, just try to load the master configuration. - import yaml - try: loader = yaml.CSafeLoader - except: loader = yaml.SafeLoader - try: - with lock_file(lock_path): - with closing(file(src_path)) as f: - ns.cfg = yaml.load(f, loader) - try: - with closing(file(cache_path + '.tmp', 'w')) as f: - f.write(json.write(ns.cfg)) - except: pass # silent failure - else: rename(cache_path + '.tmp', cache_path) - except IOError: - with closing(file(src_path)) as f: - ns.cfg = yaml.load(f, loader) - return ns.cfg - -dicts = load() -structs = dicts2struct(dicts) - -# vim:et:sw=4:ts=4 Index: trunk/packages/sipb-xen-base/python/invirt/remote.py =================================================================== --- trunk/packages/sipb-xen-base/python/invirt/remote.py (revision 1329) +++ (revision ) @@ -1,19 +1,0 @@ -from subprocess import PIPE, Popen -from invirt.config import structs as config -import yaml - -def bcast(cmd, hosts = [h.hostname for h in config.hosts]): - """ - Given a command and a list of hostnames or IPs, issue the command to all - the nodes and return a list of (host, output) pairs (the order should be - the same as the order of the hosts). - """ - pipes = [(host, - Popen(['remctl', host, 'remote', 'web', cmd], stdout=PIPE)) - for host in hosts] - outputs = [(s, p.communicate()[0]) for (s, p) in pipes] - for (s, p) in pipes: - if p.returncode != 0: - raise RuntimeError("remctl to host %s returned non-zero exit status %d" - % (s, p.returncode)) - return [(s, yaml.load(o, yaml.CSafeLoader)) for (s, o) in outputs] Index: trunk/packages/sipb-xen-base/scripts/invirt-getconf =================================================================== --- trunk/packages/sipb-xen-base/scripts/invirt-getconf (revision 1329) +++ (revision ) @@ -1,92 +1,0 @@ -#!/usr/bin/env python - -""" -invirt-getconf loads an invirt configuration file (either the original YAML -source or the faster-to-load JSON cache) and prints the configuration option -with the given name (key). Keys are dot-separated paths into the YAML -configuration tree. List indexes (0-based) are also treated as path -components. - -(Due to this path language, certain restrictions are placed on the keys used in -the YAML configuration; e.g., they cannot contain dots.) - -Examples: - - invirt-getconf db.uri - invirt-getconf authn.0.type -""" - -from invirt.config import load -from sys import argv, exit, stderr, stdout -from optparse import OptionParser - -class invirt_exception(Exception): pass - -def main(argv): - try: - parser = OptionParser(usage = '%prog [options] key', - description = __doc__.strip().split('\n\n')[0]) - parser.add_option('-s', '--src', - default = '/etc/invirt/master.yaml', - help = 'the source YAML configuration file to read from') - parser.add_option('-c', '--cache', - default = '/var/lib/invirt/invirt.json', - help = 'path to the JSON cache') - parser.add_option('-r', '--refresh', - action = 'store_true', - help = 'force the cache to be regenerated') - parser.add_option('-l', '--ls', - action = 'store_true', - help = 'list node\'s children') - opts, args = parser.parse_args() - - if len(args) > 1: - raise invirt_exception(__doc__.strip()) - elif args and args[0]: - components = args[0].split('.') - else: - components = [] - - conf = load(opts.src, opts.cache, opts.refresh) - for i, component in enumerate(components): - progress = '.'.join(components[:i]) - if type(conf) not in (dict, list): - raise invirt_exception( - '%s: node has no children (atomic datum)' % progress) - if type(conf) == list: - try: component = int(component) - except: raise invirt_exception( - '%s: node a list; integer path component required, ' - 'but got "%s"' % (progress, component)) - try: conf = conf[component] - except KeyError: raise invirt_exception( - '%s: key "%s" not found' % (progress, component)) - except IndexError: raise invirt_exception( - '%s: index %s out of range' % (progress, component)) - - if opts.ls: - if type(conf) not in (dict, list): - raise invirt_exception( - '%s: node has no children (atomic datum)' - % '.'.join(components)) - if type(conf) == list: - for i in xrange(len(conf)): - print i - else: - for k in conf.iterkeys(): - print k - else: - if type(conf) not in (dict, list): - print conf - else: - import yaml - yaml.dump(conf, stdout, - Dumper=yaml.CSafeDumper, default_flow_style=False) - except invirt_exception, ex: - print >> stderr, ex - return 1 - -if __name__ == '__main__': - exit(main(argv)) - -# vim:et:sw=4:ts=4 Index: trunk/packages/sipb-xen-base/scripts/invirt-reload =================================================================== --- trunk/packages/sipb-xen-base/scripts/invirt-reload (revision 1329) +++ (revision ) @@ -1,8 +1,0 @@ -#!/bin/bash - -for script in $(run-parts --test /etc/init.d); do - if [ "${script#/etc/init.d/sipb-xen-}" != "$script" \ - -o "${script#/etc/init.d/invirt-}" != "$script" ]; then - invoke-rc.d "${script#/etc/init.d/}" reload - fi -done Index: trunk/packages/sipb-xen-base/setup.py =================================================================== --- trunk/packages/sipb-xen-base/setup.py (revision 1329) +++ (revision ) @@ -1,24 +1,0 @@ -#!/usr/bin/python - -from os import path -from debian_bundle.changelog import Changelog -from debian_bundle.deb822 import Deb822 -from email.utils import parseaddr -from setuptools import setup - -version = Changelog(open(path.join(path.dirname(__file__), 'debian/changelog')).read()).\ - get_version().full_version - -maintainer_full = Deb822(open(path.join(path.dirname(__file__), 'debian/control')))['Maintainer'] -maintainer, maintainer_email = parseaddr(maintainer_full) - -setup( - name='invirt', - version=version, - maintainer=maintainer, - maintainer_email=maintainer_full, - - packages = ['invirt'], - package_dir = {'': 'python'}, - scripts=['scripts/invirt-getconf', 'scripts/invirt-reload'] -)