Index: /package_tags/invirt-base/0.0.4/config.todo =================================================================== --- /package_tags/invirt-base/0.0.4/config.todo (revision 1503) +++ /package_tags/invirt-base/0.0.4/config.todo (revision 1503) @@ -0,0 +1,2 @@ +files/etc/apt/sources.list.d/debathena.list: mit kerberos config +files/etc/apt/sources.list.d/sipb-xen.list: apt repo uri Index: /package_tags/invirt-base/0.0.4/debian/changelog =================================================================== --- /package_tags/invirt-base/0.0.4/debian/changelog (revision 1503) +++ /package_tags/invirt-base/0.0.4/debian/changelog (revision 1503) @@ -0,0 +1,280 @@ +invirt-base (0.0.4) unstable; urgency=low + + * We don't have any more sipb-xen- packages, so punt that from the + invirt-reload script + + -- Evan Broder Sat, 01 Nov 2008 04:52:55 -0400 + +invirt-base (0.0.3) unstable; urgency=low + + * Use invoke-rc.d instead of calling init scripts directly + + -- Evan Broder Fri, 31 Oct 2008 06:30:40 -0400 + +invirt-base (0.0.2) unstable; urgency=low + + * eliminate choice of source and cache files for invirt.config.load and + invirt-getconf, so we can do what follows + * look in /etc/invirt/conf.d/ as well as /etc/invirt/master.yaml + * use run-parts semantics in /etc/invirt/conf.d/ + + -- Greg Price Wed, 29 Oct 2008 00:10:36 -0400 + +invirt-base (0.0.1) unstable; urgency=low + + * sipb-xen-base -> invirt-base + * use /invirt for apt URI rather than /sipb-xen + + -- Greg Price Sun, 25 Oct 2008 00:54:09 -0400 + +sipb-xen-base (8.29) unstable; urgency=low + + * gen-files.sh: degrade gracefully to plain-old sh + + -- Greg Price Sat, 25 Oct 2008 20:06:54 -0400 + +sipb-xen-base (8.28) unstable; urgency=low + + * Factor out more common initscript code: + std-init.sh replaces the usual Debian boilerplate + + -- Greg Price Sat, 25 Oct 2008 15:33:08 -0400 + +sipb-xen-base (8.27) unstable; urgency=low + + * Use invoke-rc.d in invirt-reload so that policy-rc.d is respected + + -- Evan Broder Fri, 24 Oct 2008 13:19:49 -0400 + +sipb-xen-base (8.26) unstable; urgency=low + + * Move invirt-reload to /usr/sbin + + -- Evan Broder Fri, 24 Oct 2008 12:53:18 -0400 + +sipb-xen-base (8.25) unstable; urgency=low + + * Factor out common initscript code, provide it here + + -- Greg Price Fri, 24 Oct 2008 07:06:59 -0400 + +sipb-xen-base (8.24) unstable; urgency=low + + * Switch to using a setup.py file with CDBS's Python support + + -- Evan Broder Fri, 24 Oct 2008 05:22:49 -0400 + +sipb-xen-base (8.23) unstable; urgency=low + + * Now that we're using Python 2.5, we can actually write with statements + + -- Evan Broder Fri, 24 Oct 2008 03:32:00 -0400 + +sipb-xen-base (8.22) unstable; urgency=low + + * Please - anything but etch. Anything! + + -- Evan Broder Thu, 02 Oct 2008 19:50:30 -0400 + +sipb-xen-base (8.21) unstable; urgency=low + + * update sources.list.d/sipb-xen.list for prod and hardy + + -- Greg Price Wed, 01 Oct 2008 20:06:52 -0400 + +sipb-xen-base (8.20) unstable; urgency=low + + * fix distribution + + -- Greg Price Tue, 30 Sep 2008 23:51:11 -0400 + +sipb-xen-base (8.19) hardy; urgency=low + + * depend on invirt-config, which is provided by xvm-devconfig + and xvm-prodconfig. + + -- Greg Price Mon, 29 Sep 2008 06:03:54 +0000 + +sipb-xen-base (8.18) unstable; urgency=low + + * Use production k5login + * Include openafs component + + -- Sam Hartman Fri, 22 Aug 2008 16:37:01 -0400 + +sipb-xen-base (8.17) unstable; urgency=low + + * removed the `mako` program since python-mako already includes a + mako-render + + -- Yang Zhang Wed, 13 Aug 2008 01:45:26 -0400 + +sipb-xen-base (8.16) unstable; urgency=low + + * added `mako` program for processing templates in invirt packages + + -- Yang Zhang Tue, 12 Aug 2008 16:00:43 -0400 + +sipb-xen-base (8.15) unstable; urgency=low + + * put invirt-getconf in /usr/bin + + -- Greg Price Sat, 2 Aug 2008 21:58:36 -0400 + +sipb-xen-base (8.14) unstable; urgency=low + + * using correct default paths in invirt-getconf + * fixed os.rename import bug + + -- Yang Zhang Sat, 2 Aug 2008 20:29:56 -0400 + +sipb-xen-base (8.13) unstable; urgency=low + + * fix an error message + + -- Greg Price Wed, 30 Jul 2008 23:04:07 -0400 + +sipb-xen-base (8.12) unstable; urgency=low + + [ Greg Price ] + * add --ls and some convenience features to invirt-getconf: + formatting composite nodes as yaml, allowing the root + * fix bug in late-import of yaml in invirt.config + * use yaml.CSafeDumper and yaml.CSafeLoader everywhere + + [ Yang Zhang ] + * added shared lock around cache-reading transaction + + -- Greg Price Wed, 30 Jul 2008 22:11:15 -0400 + +sipb-xen-base (8.11) unstable; urgency=low + + * sped up invirt.config module load time by rearranging imports + * no longer removing the lock file + + -- Yang Zhang Wed, 30 Jul 2008 21:55:23 -0400 + +sipb-xen-base (8.10) unstable; urgency=low + + * added file locking around cache + + -- Yang Zhang Tue, 29 Jul 2008 22:35:25 -0400 + +sipb-xen-base (8.9) unstable; urgency=low + + * moved more generic code into `common` package + * silently fail if cache fails + * load the configuration on module load + * produce a struct-based representation of the configuration + * allowing full exception messages for OSErrors (default behavior) + * added some dependencies specs + + -- Yang Zhang Tue, 29 Jul 2008 01:03:16 -0400 + +sipb-xen-base (8.8) unstable; urgency=low + + * added timestamp-based JSON caching of configuration for faster loading + * exposed (more) options to command-line frontend + * improved error messages/handling/help + * removed all python 2.5-isms + * reformatted to fit project style conventions + + -- Yang Zhang Mon, 28 Jul 2008 12:25:03 -0400 + +sipb-xen-base (8.7) unstable; urgency=low + + * back to 2.4 compatibility + + -- Greg Price Mon, 28 Jul 2008 07:44:27 -0400 + +sipb-xen-base (8.6) unstable; urgency=low + + * switching to python 2.5 only + * added y_z to .k5login + + -- Yang Zhang Sun, 27 Jul 2008 20:23:54 -0400 + +sipb-xen-base (8.5) unstable; urgency=low + + * using python 2.5 + * initial working version of invirt-getconf that reads & navigates YAML + + -- Yang Zhang Sun, 27 Jul 2008 20:12:28 -0400 + +sipb-xen-base (8.4) unstable; urgency=low + + [ Greg Price ] + * begin a Python package 'invirt' + + [ Yang Zhang ] + * added invirt-getconf to read configuration files + + -- Yang Zhang Sun, 27 Jul 2008 19:10:10 -0400 + +sipb-xen-base (8.3) unstable; urgency=low + + * add invirt-reload to regenerate and reload configs + + -- Greg Price Mon, 21 Jul 2008 20:35:42 -0400 + +sipb-xen-base (8.2) unstable; urgency=low + + * leave out debathena-system so we get only the packages we ask for + + -- Greg Price Sun, 20 Jul 2008 13:58:52 -0400 + +sipb-xen-base (8.1) unstable; urgency=low + + * include debathena to make stuff like kerberos config easier + + -- Greg Price Sat, 19 Jul 2008 23:16:47 -0400 + +sipb-xen-base (8) unstable; urgency=low + + * update .k5login to match black-mesa + + -- Greg Price Sun, 4 May 2008 20:28:28 -0400 + +sipb-xen-base (7) unstable; urgency=low + + * include backports.org + * sources.list.d doesn't actually need a .sources.list, just .list + + -- Greg Price Thu, 1 May 2008 19:45:50 -0400 + +sipb-xen-base (6) unstable; urgency=low + + * actually use sources.list.d correctly + + -- Greg Price Sat, 26 Apr 2008 21:22:12 -0400 + +sipb-xen-base (5) unstable; urgency=low + + * update sources.list, use sources.list.d + + -- Greg Price Sat, 26 Apr 2008 21:06:05 -0400 + +sipb-xen-base (4) unstable; urgency=low + + * sipb-vm-1 becomes sipb-xen-dev + + -- Sam Hartman Tue, 4 Sep 2007 15:48:50 -0400 + +sipb-xen-base (3) unstable; urgency=low + + * We want security updates too + + -- Sam Hartman Fri, 10 Aug 2007 20:39:14 -0400 + +sipb-xen-base (2) unstable; urgency=low + + * Update sources.list to include our debian mirror + + -- Sam Hartman Sat, 4 Aug 2007 19:11:18 -0400 + +sipb-xen-base (1) unstable; urgency=low + + * New upstream version + + -- Sam Hartman Sat, 4 Aug 2007 18:44:21 -0400 + Index: /package_tags/invirt-base/0.0.4/debian/compat =================================================================== --- /package_tags/invirt-base/0.0.4/debian/compat (revision 1503) +++ /package_tags/invirt-base/0.0.4/debian/compat (revision 1503) @@ -0,0 +1,1 @@ +4 Index: /package_tags/invirt-base/0.0.4/debian/control =================================================================== --- /package_tags/invirt-base/0.0.4/debian/control (revision 1503) +++ /package_tags/invirt-base/0.0.4/debian/control (revision 1503) @@ -0,0 +1,21 @@ +Source: invirt-base +Section: base +Priority: extra +Maintainer: Invirt project +Build-Depends: cdbs (>= 0.4.23-1.1), debhelper (>= 4.1.0), python-all-dev, python-support, python-setuptools, python-debian, python-apt +Standards-Version: 3.8.0 + +Package: invirt-base +Architecture: all +Depends: ${python:Depends}, ${misc:Depends}, python-json (>= 3.4-2), python-yaml (>= 3.05), python-mako (>= 0.2.2), invirt-config +Provides: ${python:Provides} +XB-Python-Version: ${python:Versions} +Description: Base configuration required for all Invirt servers + This package includes common files for the Invirt system. + * apt configuration + * common init-script libraries + * common Python libraries, including the root of the 'invirt' tree + * scripts invirt-getconf and invirt-reload for using + the Invirt configuration system + . + All Invirt hosts and system VMs should have this package. Index: /package_tags/invirt-base/0.0.4/debian/copyright =================================================================== --- /package_tags/invirt-base/0.0.4/debian/copyright (revision 1503) +++ /package_tags/invirt-base/0.0.4/debian/copyright (revision 1503) @@ -0,0 +1,16 @@ +This software was written as part of the Invirt project . + +Copyright : + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + +On Debian systems, the complete text of the GNU General Public License +can be found in the file /usr/share/common-licenses/GPL. Index: /package_tags/invirt-base/0.0.4/debian/invirt-base.dirs =================================================================== --- /package_tags/invirt-base/0.0.4/debian/invirt-base.dirs (revision 1503) +++ /package_tags/invirt-base/0.0.4/debian/invirt-base.dirs (revision 1503) @@ -0,0 +1,2 @@ +usr/sbin +etc/invirt/conf.d Index: /package_tags/invirt-base/0.0.4/debian/invirt-base.install =================================================================== --- /package_tags/invirt-base/0.0.4/debian/invirt-base.install (revision 1503) +++ /package_tags/invirt-base/0.0.4/debian/invirt-base.install (revision 1503) @@ -0,0 +1,1 @@ +files/* . Index: /package_tags/invirt-base/0.0.4/debian/pycompat =================================================================== --- /package_tags/invirt-base/0.0.4/debian/pycompat (revision 1503) +++ /package_tags/invirt-base/0.0.4/debian/pycompat (revision 1503) @@ -0,0 +1,1 @@ +2 Index: /package_tags/invirt-base/0.0.4/debian/pyversions =================================================================== --- /package_tags/invirt-base/0.0.4/debian/pyversions (revision 1503) +++ /package_tags/invirt-base/0.0.4/debian/pyversions (revision 1503) @@ -0,0 +1,1 @@ +2.5- Index: /package_tags/invirt-base/0.0.4/debian/rules =================================================================== --- /package_tags/invirt-base/0.0.4/debian/rules (revision 1503) +++ /package_tags/invirt-base/0.0.4/debian/rules (revision 1503) @@ -0,0 +1,12 @@ +#!/usr/bin/make -f + +DEB_PYTHON_SYSTEM=pysupport + +include /usr/share/cdbs/1/rules/debhelper.mk +include /usr/share/cdbs/1/class/python-distutils.mk + +binary-fixup/invirt-base:: + mv $(DEB_DESTDIR)usr/bin/invirt-reload $(DEB_DESTDIR)usr/sbin/invirt-reload + +clean:: + rm -rf python/invirt.egg-info Index: /package_tags/invirt-base/0.0.4/files/etc/apt/sources.list.d/debathena.list =================================================================== --- /package_tags/invirt-base/0.0.4/files/etc/apt/sources.list.d/debathena.list (revision 1503) +++ /package_tags/invirt-base/0.0.4/files/etc/apt/sources.list.d/debathena.list (revision 1503) @@ -0,0 +1,2 @@ +deb http://debathena.mit.edu/apt hardy debathena debathena-config openafs +deb-src http://debathena.mit.edu/apt hardy debathena debathena-config openafs Index: /package_tags/invirt-base/0.0.4/files/etc/apt/sources.list.d/invirt.list =================================================================== --- /package_tags/invirt-base/0.0.4/files/etc/apt/sources.list.d/invirt.list (revision 1503) +++ /package_tags/invirt-base/0.0.4/files/etc/apt/sources.list.d/invirt.list (revision 1503) @@ -0,0 +1,4 @@ +deb http://xvm-2.mit.edu/invirt stable main +deb-src http://xvm-2.mit.edu/invirt stable main +deb http://mirrors.ccs.neu.edu/ubuntu/ hardy-backports main universe +deb-src http://mirrors.ccs.neu.edu/ubuntu/ hardy-backports main universe Index: /package_tags/invirt-base/0.0.4/files/lib/init/config-init.sh =================================================================== --- /package_tags/invirt-base/0.0.4/files/lib/init/config-init.sh (revision 1503) +++ /package_tags/invirt-base/0.0.4/files/lib/init/config-init.sh (revision 1503) @@ -0,0 +1,33 @@ +# For a package which only configures another, "parent" package. +# +# Global variable PARENTPACKAGE names parent; may be array +# for zero or many parents. +# +# Global variable PACKAGE names this package, for log message. +# +# Requires bash. + +. /lib/init/vars.sh +. /lib/lsb/init-functions +. /lib/init/gen-files.sh + +config_init () { + case "$1" in + start|reload|force-reload|restart) + log_begin_msg "Reloading config for $PACKAGE" + gen_files + log_end_msg $? + for p in "${PARENTPACKAGE[@]}"; do + invoke-rc.d "$p" "$1" + done + ;; + stop) + for p in "${PARENTPACKAGE[@]}"; do + invoke-rc.d "$p" "$1" + done + ;; + *) + log_success_msg "Usage: /etc/init.d/$PACKAGE {start|reload|force-reload|restart|stop}" + ;; + esac +} Index: /package_tags/invirt-base/0.0.4/files/lib/init/gen-files.sh =================================================================== --- /package_tags/invirt-base/0.0.4/files/lib/init/gen-files.sh (revision 1503) +++ /package_tags/invirt-base/0.0.4/files/lib/init/gen-files.sh (revision 1503) @@ -0,0 +1,17 @@ +# Generates files from templates. +# Files should be named in an array variable GEN_FILES. +# If BASH_VERSION is null or unset, accepts only one file. + +if [ $BASH_VERSION ]; then + gen_files() + { + for f in "${GEN_FILES[@]}"; do + mako-render "$f".mako >"$f" + done + } +else + gen_files() + { + mako-render "$GEN_FILES".mako >"$GEN_FILES" + } +fi Index: /package_tags/invirt-base/0.0.4/files/lib/init/std-init.sh =================================================================== --- /package_tags/invirt-base/0.0.4/files/lib/init/std-init.sh (revision 1503) +++ /package_tags/invirt-base/0.0.4/files/lib/init/std-init.sh (revision 1503) @@ -0,0 +1,92 @@ +# Typical Debian initscript, but as a library rather than copy-paste. +# +# Usage: +# NAME=short-name +# DESC="Textual description" +# SCRIPTNAME=/etc/init.d/$NAME # default if omitted +# . /lib/init/std-init.sh +# do_start() { ... } +# do_stop() { ... } +# do_reload() { ... } # optional +# std_init "$1" + +. /lib/init/vars.sh +. /lib/lsb/init-functions + +[ -r /etc/default/"$NAME" ] && . /etc/default/"$NAME" + +SCRIPTNAME="${SCRIPTNAME:-/etc/init.d/$NAME}" + +have_reload() +{ + type do_reload >/dev/null 2>/dev/null +} + +usage_exit() +{ + if have_reload; then + echo "Usage: $SCRIPTNAME {start|stop|restart|force-reload}" >&2 + else + echo "Usage: $SCRIPTNAME {start|stop|restart|reload|force-reload}" >&2 + fi + exit 3 +} + +std_init() +{ + local cmd + case "$1" in + start|stop|restart) + cmd="$1" ;; + reload) + if have_reload; then cmd=reload; else usage_exit; fi ;; + force-reload) + if have_reload; then cmd=reload; else cmd=restart; fi ;; + *) + usage_exit ;; + esac + + case $cmd in + start) + [ "$VERBOSE" != no ] && log_daemon_msg "Starting $DESC" "$NAME" + do_start + case "$?" in + 0|1) [ "$VERBOSE" != no ] && log_end_msg 0 ;; + 2) [ "$VERBOSE" != no ] && log_end_msg 1 ;; + esac + ;; + stop) + [ "$VERBOSE" != no ] && log_daemon_msg "Stopping $DESC" "$NAME" + do_stop + case "$?" in + 0|1) [ "$VERBOSE" != no ] && log_end_msg 0 ;; + 2) [ "$VERBOSE" != no ] && log_end_msg 1 ;; + esac + ;; + reload) + log_daemon_msg "Reloading $DESC" "$NAME" + do_reload + log_end_msg $? + ;; + restart) + log_daemon_msg "Restarting $DESC" "$NAME" + do_stop + case "$?" in + 0|1) + do_start + case "$?" in + 0) log_end_msg 0 ;; + 1) log_end_msg 1 ;; # Old process is still running + *) log_end_msg 1 ;; # Failed to start + esac + ;; + *) + # Failed to stop + log_end_msg 1 + ;; + esac + ;; + esac + + : +} Index: /package_tags/invirt-base/0.0.4/files/root/.k5login =================================================================== --- /package_tags/invirt-base/0.0.4/files/root/.k5login (revision 1503) +++ /package_tags/invirt-base/0.0.4/files/root/.k5login (revision 1503) @@ -0,0 +1,6 @@ +andersk/root@ATHENA.MIT.EDU +broder/root@ATHENA.MIT.EDU +ecprice/root@ATHENA.MIT.EDU +hartmans/root@ATHENA.MIT.EDU +price/root@ATHENA.MIT.EDU +quentin/root@ATHENA.MIT.EDU Index: /package_tags/invirt-base/0.0.4/python/invirt/__init__.py =================================================================== --- /package_tags/invirt-base/0.0.4/python/invirt/__init__.py (revision 1503) +++ /package_tags/invirt-base/0.0.4/python/invirt/__init__.py (revision 1503) @@ -0,0 +1,27 @@ +'''Invirt - a virtualization management system + +Invirt was developed at the Student Information Processing Board of +the Massachusetts Institute of Technology. See http://xvm.mit.edu/. + +Invirt is free software available under the GNU GPL, version 2 or later. +Consult the source files for details. +''' + +# Invirt is free software: you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the +# Free Software Foundation, either version 2 of the License, or (at your +# option) any later version. + +# Invirt is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +# for more details. + +# You should have received a copy of the GNU General Public License +# along with Invirt. If not, see . + +__author__ = 'MIT SIPB' +__version__ = '0.1' +__copyright__ = 'Copyright (c) 2008 MIT SIPB' + +# vim:et:sw=4:ts=4 Index: /package_tags/invirt-base/0.0.4/python/invirt/common.py =================================================================== --- /package_tags/invirt-base/0.0.4/python/invirt/common.py (revision 1503) +++ /package_tags/invirt-base/0.0.4/python/invirt/common.py (revision 1503) @@ -0,0 +1,58 @@ +from __future__ import with_statement + +import unittest +from fcntl import flock, LOCK_EX, LOCK_SH, LOCK_UN +import contextlib as clib + +class struct(object): + 'A simple namespace object.' + def __init__(self, d = {}, **kwargs): + 'd is the dictionary to update my __dict__ with.' + self.__dict__.update(d) + self.__dict__.update(kwargs) + +def dicts2struct(x): + """ + Given a tree of lists/dicts, perform a deep traversal to transform all the + dicts to structs. + """ + if type(x) == dict: + return struct((k, dicts2struct(v)) for k,v in x.iteritems()) + elif type(x) == list: + return [dicts2struct(v) for v in x] + else: + return x + +@clib.contextmanager +def lock_file(path, exclusive = True): + with clib.closing(file(path, 'w')) as f: + if exclusive: + locktype = LOCK_EX + else: + locktype = LOCK_SH + flock(f, locktype) + try: + yield + finally: + flock(f, LOCK_UN) + +# +# Tests. +# + +class common_tests(unittest.TestCase): + def test_dicts2structs(self): + dicts = { + 'atom': 0, + 'dict': { 'atom': 'atom', 'list': [1,2,3] }, + 'list': [ 'atom', {'key': 'value'} ] + } + structs = dicts2struct(dicts) + self.assertEqual(structs.atom, dicts['atom']) + self.assertEqual(structs.dict.atom, dicts['dict']['atom']) + self.assertEqual(structs.dict.list, dicts['dict']['list']) + self.assertEqual(structs.list[0], dicts['list'][0]) + self.assertEqual(structs.list[1].key, dicts['list'][1]['key']) + +if __name__ == '__main__': + unittest.main() Index: /package_tags/invirt-base/0.0.4/python/invirt/config.py =================================================================== --- /package_tags/invirt-base/0.0.4/python/invirt/config.py (revision 1503) +++ /package_tags/invirt-base/0.0.4/python/invirt/config.py (revision 1503) @@ -0,0 +1,140 @@ +from __future__ import with_statement + +import json +from invirt.common import * +import os +from os import rename +from os.path import getmtime +from contextlib import closing +import yaml +import re + +try: loader = yaml.CSafeLoader +except: loader = yaml.SafeLoader + +src_path = '/etc/invirt/master.yaml' +src_dirpath = '/etc/invirt/conf.d' +cache_path = '/var/lib/invirt/cache.json' +lock_path = '/var/lib/invirt/cache.lock' + +def augment(d1, d2): + """Splice dict-tree d2 into d1. Return d1. + + Example: + >>> d = {'a': {'b': 1}, 'c': 2} + >>> augment(d, {'a': {'d': 3}}) + {'a': {'b', 1, 'd': 3}, 'c': 2} + >>> d + {'a': {'b', 1, 'd': 3}, 'c': 2} + """ + for k in d2: + if k in d1 and isinstance(d1[k], dict): + augment(d1[k], d2[k]) + else: + d1[k] = d2[k] + return d1 + +def run_parts_list(dirname): + """Reimplements Debian's run-parts --list. + + One difference from run-parts's behavior: run-parts --list /foo/ + will give output like /foo//bar, because Python code tends to expect this. + + Matches documented behavior of run-parts in debianutils v2.28.2, dated 2007. + """ + # From run-parts(8). + lanana_re = re.compile('^[a-z0-9]+$') + lsb_re = re.compile('^_?([a-z0-9_.]+-)+[a-z0-9]+$') + deb_cron_re = re.compile('^[a-z0-9][a-z0-9-]*$') + for name in os.listdir(dirname): + if lanana_re.match(name) or lsb_re.match(name) or deb_cron_re.match(name): + yield os.path.join(dirname, name) + +def list_files(): + yield src_path + for name in run_parts_list(src_dirpath): + yield name + +def load_master(): + config = dict() + for filename in list_files(): + with closing(file(filename)) as f: + augment(config, yaml.load(f, loader)) + return config + +def get_src_mtime(): + return max(max(getmtime(filename) for filename in list_files()), + getmtime(src_dirpath)) + +def load(force_refresh = False): + """ + Try loading the configuration from the faster-to-load JSON cache at + cache_path. If it doesn't exist or is outdated, load the configuration + instead from the original YAML file at src_path and regenerate the cache. + I assume I have the permissions to write to the cache directory. + """ + + # Namespace container for state variables, so that they can be updated by + # closures. + ns = struct() + + if force_refresh: + do_refresh = True + else: + src_mtime = get_src_mtime() + try: cache_mtime = getmtime(cache_path) + except OSError: do_refresh = True + else: do_refresh = src_mtime + 1 >= cache_mtime + + # We chose not to simply say + # + # do_refresh = src_mtime >= cache_time + # + # because between the getmtime(src_path) and the time the cache is + # rewritten, the master configuration may have been updated, so future + # checks here would find a cache with a newer mtime than the master + # (and thus treat the cache as containing the latest version of the + # master). The +1 means that for at least a full second following the + # update to the master, this function will refresh the cache, giving us + # 1 second to write the cache. Note that if it takes longer than 1 + # second to write the cache, then this situation could still arise. + # + # The getmtime calls should logically be part of the same transaction + # as the rest of this function (cache read + conditional cache + # refresh), but to wrap everything in an flock would cause the + # following cache read to be less streamlined. + + if not do_refresh: + # Try reading from the cache first. This must be transactionally + # isolated from concurrent writes to prevent reading an incomplete + # (changing) version of the data (but the transaction can share the + # lock with other concurrent reads). This isolation is accomplished + # using an atomic filesystem rename in the refreshing stage. + try: + with closing(file(cache_path)) as f: + ns.cfg = json.read(f.read()) + except: do_refresh = True + + if do_refresh: + # Atomically reload the source and regenerate the cache. The read and + # write must be a single transaction, or a stale version may be + # written (if another read/write of a more recent configuration + # is interleaved). The final atomic rename is to keep this + # transactionally isolated from the above cache read. If we fail to + # acquire the lock, just try to load the master configuration. + try: + with lock_file(lock_path): + ns.cfg = load_master() + try: + with closing(file(cache_path + '.tmp', 'w')) as f: + f.write(json.write(ns.cfg)) + except: pass # silent failure + else: rename(cache_path + '.tmp', cache_path) + except IOError: + ns.cfg = load_master() + return ns.cfg + +dicts = load() +structs = dicts2struct(dicts) + +# vim:et:sw=4:ts=4 Index: /package_tags/invirt-base/0.0.4/python/invirt/remote.py =================================================================== --- /package_tags/invirt-base/0.0.4/python/invirt/remote.py (revision 1503) +++ /package_tags/invirt-base/0.0.4/python/invirt/remote.py (revision 1503) @@ -0,0 +1,19 @@ +from subprocess import PIPE, Popen +from invirt.config import structs as config +import yaml + +def bcast(cmd, hosts = [h.hostname for h in config.hosts]): + """ + Given a command and a list of hostnames or IPs, issue the command to all + the nodes and return a list of (host, output) pairs (the order should be + the same as the order of the hosts). + """ + pipes = [(host, + Popen(['remctl', host, 'remote', 'web', cmd], stdout=PIPE)) + for host in hosts] + outputs = [(s, p.communicate()[0]) for (s, p) in pipes] + for (s, p) in pipes: + if p.returncode != 0: + raise RuntimeError("remctl to host %s returned non-zero exit status %d" + % (s, p.returncode)) + return [(s, yaml.load(o, yaml.CSafeLoader)) for (s, o) in outputs] Index: /package_tags/invirt-base/0.0.4/scripts/invirt-getconf =================================================================== --- /package_tags/invirt-base/0.0.4/scripts/invirt-getconf (revision 1503) +++ /package_tags/invirt-base/0.0.4/scripts/invirt-getconf (revision 1503) @@ -0,0 +1,86 @@ +#!/usr/bin/env python + +""" +invirt-getconf loads an invirt configuration file (either the original YAML +source or the faster-to-load JSON cache) and prints the configuration option +with the given name (key). Keys are dot-separated paths into the YAML +configuration tree. List indexes (0-based) are also treated as path +components. + +(Due to this path language, certain restrictions are placed on the keys used in +the YAML configuration; e.g., they cannot contain dots.) + +Examples: + + invirt-getconf db.uri + invirt-getconf authn.0.type +""" + +from invirt import config +from sys import argv, exit, stderr, stdout +from optparse import OptionParser + +class invirt_exception(Exception): pass + +def main(argv): + try: + parser = OptionParser(usage = '%prog [options] key', + description = __doc__.strip().split('\n\n')[0]) + parser.add_option('-r', '--refresh', + action = 'store_true', + help = 'force the cache to be regenerated') + parser.add_option('-l', '--ls', + action = 'store_true', + help = 'list node\'s children') + opts, args = parser.parse_args() + + if len(args) > 1: + raise invirt_exception(__doc__.strip()) + elif args and args[0]: + components = args[0].split('.') + else: + components = [] + + conf = config.load(opts.refresh) + for i, component in enumerate(components): + progress = '.'.join(components[:i]) + if type(conf) not in (dict, list): + raise invirt_exception( + '%s: node has no children (atomic datum)' % progress) + if type(conf) == list: + try: component = int(component) + except: raise invirt_exception( + '%s: node a list; integer path component required, ' + 'but got "%s"' % (progress, component)) + try: conf = conf[component] + except KeyError: raise invirt_exception( + '%s: key "%s" not found' % (progress, component)) + except IndexError: raise invirt_exception( + '%s: index %s out of range' % (progress, component)) + + if opts.ls: + if type(conf) not in (dict, list): + raise invirt_exception( + '%s: node has no children (atomic datum)' + % '.'.join(components)) + if type(conf) == list: + for i in xrange(len(conf)): + print i + else: + for k in conf.iterkeys(): + print k + else: + if type(conf) not in (dict, list): + print conf + else: + import yaml + yaml.dump(conf, stdout, + Dumper=yaml.CSafeDumper, default_flow_style=False) + except invirt_exception, ex: + print >> stderr, ex + return 1 + +if __name__ == '__main__': + exit(main(argv)) + +# vim:et:sw=4:ts=4 Index: /package_tags/invirt-base/0.0.4/scripts/invirt-reload =================================================================== --- /package_tags/invirt-base/0.0.4/scripts/invirt-reload (revision 1503) +++ /package_tags/invirt-base/0.0.4/scripts/invirt-reload (revision 1503) @@ -0,0 +1,7 @@ +#!/bin/bash + +for script in $(run-parts --test /etc/init.d); do + if [ "${script#/etc/init.d/invirt-}" != "$script" ]; then + invoke-rc.d "${script#/etc/init.d/}" reload + fi +done Index: /package_tags/invirt-base/0.0.4/setup.py =================================================================== --- /package_tags/invirt-base/0.0.4/setup.py (revision 1503) +++ /package_tags/invirt-base/0.0.4/setup.py (revision 1503) @@ -0,0 +1,24 @@ +#!/usr/bin/python + +from os import path +from debian_bundle.changelog import Changelog +from debian_bundle.deb822 import Deb822 +from email.utils import parseaddr +from setuptools import setup + +version = Changelog(open(path.join(path.dirname(__file__), 'debian/changelog')).read()).\ + get_version().full_version + +maintainer_full = Deb822(open(path.join(path.dirname(__file__), 'debian/control')))['Maintainer'] +maintainer, maintainer_email = parseaddr(maintainer_full) + +setup( + name='invirt', + version=version, + maintainer=maintainer, + maintainer_email=maintainer_full, + + packages = ['invirt'], + package_dir = {'': 'python'}, + scripts=['scripts/invirt-getconf', 'scripts/invirt-reload'] +)