Index: /package_tags/sipb-xen-base/8.18/config.todo =================================================================== --- /package_tags/sipb-xen-base/8.18/config.todo (revision 905) +++ /package_tags/sipb-xen-base/8.18/config.todo (revision 905) @@ -0,0 +1,2 @@ +files/etc/apt/sources.list.d/debathena.list: mit kerberos config +files/etc/apt/sources.list.d/sipb-xen.list: apt repo uri Index: /package_tags/sipb-xen-base/8.18/debian/changelog =================================================================== --- /package_tags/sipb-xen-base/8.18/debian/changelog (revision 905) +++ /package_tags/sipb-xen-base/8.18/debian/changelog (revision 905) @@ -0,0 +1,183 @@ +sipb-xen-base (8.18) unstable; urgency=low + + * Use production k5login + * Include openafs component + + -- Sam Hartman Fri, 22 Aug 2008 16:37:01 -0400 + +sipb-xen-base (8.17) unstable; urgency=low + + * removed the `mako` program since python-mako already includes a + mako-render + + -- Yang Zhang Wed, 13 Aug 2008 01:45:26 -0400 + +sipb-xen-base (8.16) unstable; urgency=low + + * added `mako` program for processing templates in invirt packages + + -- Yang Zhang Tue, 12 Aug 2008 16:00:43 -0400 + +sipb-xen-base (8.15) unstable; urgency=low + + * put invirt-getconf in /usr/bin + + -- Greg Price Sat, 2 Aug 2008 21:58:36 -0400 + +sipb-xen-base (8.14) unstable; urgency=low + + * using correct default paths in invirt-getconf + * fixed os.rename import bug + + -- Yang Zhang Sat, 2 Aug 2008 20:29:56 -0400 + +sipb-xen-base (8.13) unstable; urgency=low + + * fix an error message + + -- Greg Price Wed, 30 Jul 2008 23:04:07 -0400 + +sipb-xen-base (8.12) unstable; urgency=low + + [ Greg Price ] + * add --ls and some convenience features to invirt-getconf: + formatting composite nodes as yaml, allowing the root + * fix bug in late-import of yaml in invirt.config + * use yaml.CSafeDumper and yaml.CSafeLoader everywhere + + [ Yang Zhang ] + * added shared lock around cache-reading transaction + + -- Greg Price Wed, 30 Jul 2008 22:11:15 -0400 + +sipb-xen-base (8.11) unstable; urgency=low + + * sped up invirt.config module load time by rearranging imports + * no longer removing the lock file + + -- Yang Zhang Wed, 30 Jul 2008 21:55:23 -0400 + +sipb-xen-base (8.10) unstable; urgency=low + + * added file locking around cache + + -- Yang Zhang Tue, 29 Jul 2008 22:35:25 -0400 + +sipb-xen-base (8.9) unstable; urgency=low + + * moved more generic code into `common` package + * silently fail if cache fails + * load the configuration on module load + * produce a struct-based representation of the configuration + * allowing full exception messages for OSErrors (default behavior) + * added some dependencies specs + + -- Yang Zhang Tue, 29 Jul 2008 01:03:16 -0400 + +sipb-xen-base (8.8) unstable; urgency=low + + * added timestamp-based JSON caching of configuration for faster loading + * exposed (more) options to command-line frontend + * improved error messages/handling/help + * removed all python 2.5-isms + * reformatted to fit project style conventions + + -- Yang Zhang Mon, 28 Jul 2008 12:25:03 -0400 + +sipb-xen-base (8.7) unstable; urgency=low + + * back to 2.4 compatibility + + -- Greg Price Mon, 28 Jul 2008 07:44:27 -0400 + +sipb-xen-base (8.6) unstable; urgency=low + + * switching to python 2.5 only + * added y_z to .k5login + + -- Yang Zhang Sun, 27 Jul 2008 20:23:54 -0400 + +sipb-xen-base (8.5) unstable; urgency=low + + * using python 2.5 + * initial working version of invirt-getconf that reads & navigates YAML + + -- Yang Zhang Sun, 27 Jul 2008 20:12:28 -0400 + +sipb-xen-base (8.4) unstable; urgency=low + + [ Greg Price ] + * begin a Python package 'invirt' + + [ Yang Zhang ] + * added invirt-getconf to read configuration files + + -- Yang Zhang Sun, 27 Jul 2008 19:10:10 -0400 + +sipb-xen-base (8.3) unstable; urgency=low + + * add invirt-reload to regenerate and reload configs + + -- Greg Price Mon, 21 Jul 2008 20:35:42 -0400 + +sipb-xen-base (8.2) unstable; urgency=low + + * leave out debathena-system so we get only the packages we ask for + + -- Greg Price Sun, 20 Jul 2008 13:58:52 -0400 + +sipb-xen-base (8.1) unstable; urgency=low + + * include debathena to make stuff like kerberos config easier + + -- Greg Price Sat, 19 Jul 2008 23:16:47 -0400 + +sipb-xen-base (8) unstable; urgency=low + + * update .k5login to match black-mesa + + -- Greg Price Sun, 4 May 2008 20:28:28 -0400 + +sipb-xen-base (7) unstable; urgency=low + + * include backports.org + * sources.list.d doesn't actually need a .sources.list, just .list + + -- Greg Price Thu, 1 May 2008 19:45:50 -0400 + +sipb-xen-base (6) unstable; urgency=low + + * actually use sources.list.d correctly + + -- Greg Price Sat, 26 Apr 2008 21:22:12 -0400 + +sipb-xen-base (5) unstable; urgency=low + + * update sources.list, use sources.list.d + + -- Greg Price Sat, 26 Apr 2008 21:06:05 -0400 + +sipb-xen-base (4) unstable; urgency=low + + * sipb-vm-1 becomes sipb-xen-dev + + -- Sam Hartman Tue, 4 Sep 2007 15:48:50 -0400 + +sipb-xen-base (3) unstable; urgency=low + + * We want security updates too + + -- Sam Hartman Fri, 10 Aug 2007 20:39:14 -0400 + +sipb-xen-base (2) unstable; urgency=low + + * Update sources.list to include our debian mirror + + -- Sam Hartman Sat, 4 Aug 2007 19:11:18 -0400 + +sipb-xen-base (1) unstable; urgency=low + + * New upstream version + + -- Sam Hartman Sat, 4 Aug 2007 18:44:21 -0400 + Index: /package_tags/sipb-xen-base/8.18/debian/compat =================================================================== --- /package_tags/sipb-xen-base/8.18/debian/compat (revision 905) +++ /package_tags/sipb-xen-base/8.18/debian/compat (revision 905) @@ -0,0 +1,1 @@ +4 Index: /package_tags/sipb-xen-base/8.18/debian/control =================================================================== --- /package_tags/sipb-xen-base/8.18/debian/control (revision 905) +++ /package_tags/sipb-xen-base/8.18/debian/control (revision 905) @@ -0,0 +1,14 @@ +Source: sipb-xen-base +Section: base +Priority: extra +Maintainer: SIPB Xen Project +Build-Depends: cdbs (>= 0.4.23-1.1), debhelper (>= 4.1.0), python-support +Standards-Version: 3.7.2 + +Package: sipb-xen-base +Architecture: all +Depends: ${misc:Depends}, python-json (>= 3.4-2), python-yaml (>= 3.05), python-mako (>= 0.2.2) +Description: Base configuration required for all SIPB xen servers + This package includes apt configuration, .k5login and other files that + should be synchronized among all our servers. + Installing this on a non-sipb-xen machine would be very anti-social. Index: /package_tags/sipb-xen-base/8.18/debian/copyright =================================================================== --- /package_tags/sipb-xen-base/8.18/debian/copyright (revision 905) +++ /package_tags/sipb-xen-base/8.18/debian/copyright (revision 905) @@ -0,0 +1,3 @@ +This package was created for internal use of the SIPB Xen Project of +the MIT Student Information Processing Board. Ask tabbott@mit.edu if +you have questions about redistribution. Index: /package_tags/sipb-xen-base/8.18/debian/rules =================================================================== --- /package_tags/sipb-xen-base/8.18/debian/rules (revision 905) +++ /package_tags/sipb-xen-base/8.18/debian/rules (revision 905) @@ -0,0 +1,7 @@ +#!/usr/bin/make -f + +include /usr/share/cdbs/1/rules/debhelper.mk + +# do we, should we, care between pysupport and pycentral? +binary-install/sipb-xen-base:: + dh_pysupport -psipb-xen-base Index: /package_tags/sipb-xen-base/8.18/debian/sipb-xen-base.install =================================================================== --- /package_tags/sipb-xen-base/8.18/debian/sipb-xen-base.install (revision 905) +++ /package_tags/sipb-xen-base/8.18/debian/sipb-xen-base.install (revision 905) @@ -0,0 +1,1 @@ +files/* . Index: /package_tags/sipb-xen-base/8.18/files/etc/apt/sources.list.d/debathena.list =================================================================== --- /package_tags/sipb-xen-base/8.18/files/etc/apt/sources.list.d/debathena.list (revision 905) +++ /package_tags/sipb-xen-base/8.18/files/etc/apt/sources.list.d/debathena.list (revision 905) @@ -0,0 +1,2 @@ +deb http://debathena.mit.edu/apt etch debathena debathena-config openafs +deb-src http://debathena.mit.edu/apt etch debathena debathena-config openafs Index: /package_tags/sipb-xen-base/8.18/files/etc/apt/sources.list.d/sipb-xen.list =================================================================== --- /package_tags/sipb-xen-base/8.18/files/etc/apt/sources.list.d/sipb-xen.list (revision 905) +++ /package_tags/sipb-xen-base/8.18/files/etc/apt/sources.list.d/sipb-xen.list (revision 905) @@ -0,0 +1,4 @@ +deb http://sipb-xen-dev.mit.edu/sipb-xen stable main +deb-src http://sipb-xen-dev.mit.edu/sipb-xen stable main +deb http://www.backports.org/debian etch-backports main +deb-src http://www.backports.org/debian etch-backports main Index: /package_tags/sipb-xen-base/8.18/files/root/.k5login =================================================================== --- /package_tags/sipb-xen-base/8.18/files/root/.k5login (revision 905) +++ /package_tags/sipb-xen-base/8.18/files/root/.k5login (revision 905) @@ -0,0 +1,6 @@ +andersk/root@ATHENA.MIT.EDU +broder/root@ATHENA.MIT.EDU +ecprice/root@ATHENA.MIT.EDU +hartmans/root@ATHENA.MIT.EDU +price/root@ATHENA.MIT.EDU +quentin/root@ATHENA.MIT.EDU Index: /package_tags/sipb-xen-base/8.18/files/usr/bin/invirt-getconf =================================================================== --- /package_tags/sipb-xen-base/8.18/files/usr/bin/invirt-getconf (revision 905) +++ /package_tags/sipb-xen-base/8.18/files/usr/bin/invirt-getconf (revision 905) @@ -0,0 +1,92 @@ +#!/usr/bin/env python + +""" +invirt-getconf loads an invirt configuration file (either the original YAML +source or the faster-to-load JSON cache) and prints the configuration option +with the given name (key). Keys are dot-separated paths into the YAML +configuration tree. List indexes (0-based) are also treated as path +components. + +(Due to this path language, certain restrictions are placed on the keys used in +the YAML configuration; e.g., they cannot contain dots.) + +Examples: + + invirt-getconf db.uri + invirt-getconf authn.0.type +""" + +from invirt.config import load +from sys import argv, exit, stderr, stdout +from optparse import OptionParser + +class invirt_exception(Exception): pass + +def main(argv): + try: + parser = OptionParser(usage = '%prog [options] key', + description = __doc__.strip().split('\n\n')[0]) + parser.add_option('-s', '--src', + default = '/etc/invirt/master.yaml', + help = 'the source YAML configuration file to read from') + parser.add_option('-c', '--cache', + default = '/var/lib/invirt/invirt.json', + help = 'path to the JSON cache') + parser.add_option('-r', '--refresh', + action = 'store_true', + help = 'force the cache to be regenerated') + parser.add_option('-l', '--ls', + action = 'store_true', + help = 'list node\'s children') + opts, args = parser.parse_args() + + if len(args) > 1: + raise invirt_exception(__doc__.strip()) + elif args and args[0]: + components = args[0].split('.') + else: + components = [] + + conf = load(opts.src, opts.cache, opts.refresh) + for i, component in enumerate(components): + progress = '.'.join(components[:i]) + if type(conf) not in (dict, list): + raise invirt_exception( + '%s: node has no children (atomic datum)' % progress) + if type(conf) == list: + try: component = int(component) + except: raise invirt_exception( + '%s: node a list; integer path component required, ' + 'but got "%s"' % (progress, component)) + try: conf = conf[component] + except KeyError: raise invirt_exception( + '%s: key "%s" not found' % (progress, component)) + except IndexError: raise invirt_exception( + '%s: index %s out of range' % (progress, component)) + + if opts.ls: + if type(conf) not in (dict, list): + raise invirt_exception( + '%s: node has no children (atomic datum)' + % '.'.join(components)) + if type(conf) == list: + for i in xrange(len(conf)): + print i + else: + for k in conf.iterkeys(): + print k + else: + if type(conf) not in (dict, list): + print conf + else: + import yaml + yaml.dump(conf, stdout, + Dumper=yaml.CSafeDumper, default_flow_style=False) + except invirt_exception, ex: + print >> stderr, ex + return 1 + +if __name__ == '__main__': + exit(main(argv)) + +# vim:et:sw=4:ts=4 Index: /package_tags/sipb-xen-base/8.18/files/usr/sbin/invirt-reload =================================================================== --- /package_tags/sipb-xen-base/8.18/files/usr/sbin/invirt-reload (revision 905) +++ /package_tags/sipb-xen-base/8.18/files/usr/sbin/invirt-reload (revision 905) @@ -0,0 +1,8 @@ +#!/bin/bash + +for script in $(run-parts --test /etc/init.d); do + if [ "${script#/etc/init.d/sipb-xen-}" != "$script" \ + -o "${script#/etc/init.d/invirt-}" != "$script" ]; then + "$script" reload + fi +done Index: /package_tags/sipb-xen-base/8.18/files/usr/share/python-support/sipb-xen-base/invirt/__init__.py =================================================================== --- /package_tags/sipb-xen-base/8.18/files/usr/share/python-support/sipb-xen-base/invirt/__init__.py (revision 905) +++ /package_tags/sipb-xen-base/8.18/files/usr/share/python-support/sipb-xen-base/invirt/__init__.py (revision 905) @@ -0,0 +1,27 @@ +'''Invirt - a virtualization management system + +Invirt was developed at the Student Information Processing Board of +the Massachusetts Institute of Technology. See http://xvm.mit.edu/. + +Invirt is free software available under the GNU GPL, version 2 or later. +Consult the source files for details. +''' + +# Invirt is free software: you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the +# Free Software Foundation, either version 2 of the License, or (at your +# option) any later version. + +# Invirt is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +# for more details. + +# You should have received a copy of the GNU General Public License +# along with Invirt. If not, see . + +__author__ = 'MIT SIPB' +__version__ = '0.1' +__copyright__ = 'Copyright (c) 2008 MIT SIPB' + +# vim:et:sw=4:ts=4 Index: /package_tags/sipb-xen-base/8.18/files/usr/share/python-support/sipb-xen-base/invirt/common.py =================================================================== --- /package_tags/sipb-xen-base/8.18/files/usr/share/python-support/sipb-xen-base/invirt/common.py (revision 905) +++ /package_tags/sipb-xen-base/8.18/files/usr/share/python-support/sipb-xen-base/invirt/common.py (revision 905) @@ -0,0 +1,83 @@ +import unittest +from fcntl import flock, LOCK_EX, LOCK_SH, LOCK_UN + +class struct(object): + 'A simple namespace object.' + def __init__(self, d = {}, **kwargs): + 'd is the dictionary to update my __dict__ with.' + self.__dict__.update(d) + self.__dict__.update(kwargs) + +def dicts2struct(x): + """ + Given a tree of lists/dicts, perform a deep traversal to transform all the + dicts to structs. + """ + if type(x) == dict: + return struct((k, dicts2struct(v)) for k,v in x.iteritems()) + elif type(x) == list: + return [dicts2struct(v) for v in x] + else: + return x + +# +# Hacks to work around lack of Python 2.5's `with` statement. +# + +def with_closing(rsrc): + """ + Utility to emulate Python 2.5's `with closing(rsrc)` context manager. + + E.g., + @with_closing(file('/tmp/foo')) + def contents(f): + return f.read() + # now 'contents' is the contents of /tmp/foo + """ + def wrapper(func): + try: return func(rsrc) + finally: rsrc.close() + return wrapper + +def with_lock_file(path, exclusive = True): + """ + Context manager for lock files. Example: + + @with_lock_file('/tmp/mylock') + def input(): + print 'locked' + return raw_input() + # prints 'locked' + print input # prints what raw_input() returned + """ + def wrapper(func): + @with_closing(file(path, 'w')) + def g(f): + if exclusive: locktype = LOCK_EX + else: locktype = LOCK_SH + flock(f, locktype) + try: return func() + finally: flock(f, LOCK_UN) + return g + return wrapper + +# +# Tests. +# + +class common_tests(unittest.TestCase): + def test_dicts2structs(self): + dicts = { + 'atom': 0, + 'dict': { 'atom': 'atom', 'list': [1,2,3] }, + 'list': [ 'atom', {'key': 'value'} ] + } + structs = dicts2struct(dicts) + self.assertEqual(structs.atom, dicts['atom']) + self.assertEqual(structs.dict.atom, dicts['dict']['atom']) + self.assertEqual(structs.dict.list, dicts['dict']['list']) + self.assertEqual(structs.list[0], dicts['list'][0]) + self.assertEqual(structs.list[1].key, dicts['list'][1]['key']) + +if __name__ == '__main__': + unittest.main() Index: /package_tags/sipb-xen-base/8.18/files/usr/share/python-support/sipb-xen-base/invirt/config.py =================================================================== --- /package_tags/sipb-xen-base/8.18/files/usr/share/python-support/sipb-xen-base/invirt/config.py (revision 905) +++ /package_tags/sipb-xen-base/8.18/files/usr/share/python-support/sipb-xen-base/invirt/config.py (revision 905) @@ -0,0 +1,87 @@ +import json +from invirt.common import * +from os import rename +from os.path import getmtime + +default_src_path = '/etc/invirt/master.yaml' +default_cache_path = '/var/lib/invirt/cache.json' +lock_file = '/var/lib/invirt/cache.lock' + +def load(src_path = default_src_path, + cache_path = default_cache_path, + force_refresh = False): + """ + Try loading the configuration from the faster-to-load JSON cache at + cache_path. If it doesn't exist or is outdated, load the configuration + instead from the original YAML file at src_path and regenerate the cache. + I assume I have the permissions to write to the cache directory. + """ + + # Namespace container for state variables, so that they can be updated by + # closures. + ns = struct() + + if force_refresh: + do_refresh = True + else: + src_mtime = getmtime(src_path) + try: cache_mtime = getmtime(cache_path) + except OSError: do_refresh = True + else: do_refresh = src_mtime + 1 >= cache_mtime + + # We chose not to simply say + # + # do_refresh = src_mtime >= cache_time + # + # because between the getmtime(src_path) and the time the cache is + # rewritten, the master configuration may have been updated, so future + # checks here would find a cache with a newer mtime than the master + # (and thus treat the cache as containing the latest version of the + # master). The +1 means that for at least a full second following the + # update to the master, this function will refresh the cache, giving us + # 1 second to write the cache. Note that if it takes longer than 1 + # second to write the cache, then this situation could still arise. + # + # The getmtime calls should logically be part of the same transaction + # as the rest of this function (cache read + conditional cache + # refresh), but to wrap everything in an flock would cause the + # following cache read to be less streamlined. + + if not do_refresh: + # Try reading from the cache first. This must be transactionally + # isolated from concurrent writes to prevent reading an incomplete + # (changing) version of the data (but the transaction can share the + # lock with other concurrent reads). This isolation is accomplished + # using an atomic filesystem rename in the refreshing stage. + try: ns.cfg = with_closing(file(cache_path)) ( + lambda f: json.read(f.read())) + except: do_refresh = True + + if do_refresh: + # Atomically reload the source and regenerate the cache. The read and + # write must be a single transaction, or a stale version may be + # written (if another read/write of a more recent configuration + # is interleaved). The final atomic rename is to keep this + # transactionally isolated from the above cache read. If we fail to + # acquire the lock, just try to load the master configuration. + import yaml + try: loader = yaml.CSafeLoader + except: loader = yaml.SafeLoader + try: + @with_lock_file(lock_file) + def refresh_cache(): + ns.cfg = with_closing(file(src_path)) ( + lambda f: yaml.load(f, loader)) + try: with_closing(file(cache_path + '.tmp', 'w')) ( + lambda f: f.write(json.write(ns.cfg))) + except: pass # silent failure + else: rename(cache_path + '.tmp', cache_path) + except IOError: + ns.cfg = with_closing(file(src_path)) ( + lambda f: yaml.load(f, loader)) + return ns.cfg + +dicts = load() +structs = dicts2struct(dicts) + +# vim:et:sw=4:ts=4 Index: /package_tags/sipb-xen-base/8.18/files/usr/share/python-support/sipb-xen-base/invirt/remote.py =================================================================== --- /package_tags/sipb-xen-base/8.18/files/usr/share/python-support/sipb-xen-base/invirt/remote.py (revision 905) +++ /package_tags/sipb-xen-base/8.18/files/usr/share/python-support/sipb-xen-base/invirt/remote.py (revision 905) @@ -0,0 +1,19 @@ +from subprocess import PIPE, Popen +from invirt.config import structs as config +import yaml + +def bcast(cmd, hosts = [h.hostname for h in config.hosts]): + """ + Given a command and a list of hostnames or IPs, issue the command to all + the nodes and return a list of (host, output) pairs (the order should be + the same as the order of the hosts). + """ + pipes = [(host, + Popen(['remctl', host, 'remote', 'web', cmd], stdout=PIPE)) + for host in hosts] + outputs = [(s, p.communicate()[0]) for (s, p) in pipes] + for (s, p) in pipes: + if p.returncode != 0: + raise RuntimeError("remctl to host %s returned non-zero exit status %d" + % (s, p.returncode)) + return [(s, yaml.load(o, yaml.CSafeLoader)) for (s, o) in outputs]