Index: trunk/packages/invirt-dev/README =================================================================== --- trunk/packages/invirt-dev/README (revision 3011) +++ trunk/packages/invirt-dev/README (revision 3028) @@ -6,2 +6,5 @@ The public key should then be in the apt keyring of systems using the packages. + + - Packages should be located at + /srv/git/invirt/packages/${packagename}.git Index: trunk/packages/invirt-dev/debian/changelog =================================================================== --- trunk/packages/invirt-dev/debian/changelog (revision 3011) +++ trunk/packages/invirt-dev/debian/changelog (revision 3028) @@ -1,2 +1,9 @@ +invirt-dev (0.1.1) unstable; urgency=low + + * Added missing imports + * Other minor functional changes + + -- Greg Brockman Mon, 05 Jul 2010 01:43:45 -0400 + invirt-dev (0.1.0) unstable; urgency=low Index: trunk/packages/invirt-dev/debian/control =================================================================== --- trunk/packages/invirt-dev/debian/control (revision 3011) +++ trunk/packages/invirt-dev/debian/control (revision 3028) @@ -8,5 +8,5 @@ Package: invirt-dev Architecture: all -Depends: ${shlibs:Depends}, ${misc:Depends}, dpkg-dev-el, emacs21, reprepro, apache2, postfix, screen, dh-make, fakeroot, quilt, patchutils, config-package-dev, pbuilder, equivs, invirt-base, invirt-database, remctl-server, update-inetd, openbsd-inetd | inet-superserver, python-pyinotify +Depends: ${shlibs:Depends}, ${misc:Depends}, dpkg-dev-el, emacs21, reprepro, apache2, postfix, screen, dh-make, fakeroot, quilt, patchutils, config-package-dev, sbuild, equivs, invirt-base, invirt-database, remctl-server, update-inetd, openbsd-inetd | inet-superserver, python-pyinotify, python-debian, git-core (>= 1.6.4) Description: Invirt build and apt server This packages the build scripts and apt-repository configuration for Invirt. Index: trunk/packages/invirt-dev/invirt-submit-build =================================================================== --- trunk/packages/invirt-dev/invirt-submit-build (revision 3011) +++ trunk/packages/invirt-dev/invirt-submit-build (revision 3028) @@ -19,4 +19,5 @@ import datetime +import optparse import os import sys @@ -28,5 +29,10 @@ def main(): - pocket, package, commit = sys.argv[1:4] + parser = optparse.OptionParser('Usage: %prog pocket package commit') + opts, args = parser.parse_args() + if len(args) != 3: + parser.print_help() + return 1 + pocket, package, commit = args principal = os.environ['REMOTE_USER'] request_time = datetime.datetime.utcnow() @@ -47,4 +53,5 @@ q = os.fdopen(q_fd, 'r+') print >>q, "%s %s %s %s" % (pocket, package, commit, principal) + q.close() os.rename(q_name, q_path) Index: trunk/packages/invirt-dev/invirtibuilder =================================================================== --- trunk/packages/invirt-dev/invirtibuilder (revision 3011) +++ trunk/packages/invirt-dev/invirtibuilder (revision 3028) @@ -30,13 +30,20 @@ import contextlib +import glob import os import re import shutil import subprocess +import tempfile +import traceback import pyinotify +from debian_bundle import deb822 + import invirt.builder as b +import invirt.common as c from invirt import database +from invirt.config import structs as config @@ -52,5 +59,5 @@ """ return deb822.Deb822.iter_paragraphs( - getGitFile(package, ref, 'debian/control').split('\n')) + b.getGitFile(package, ref, 'debian/control').split('\n')) @@ -72,5 +79,5 @@ def getDscName(package, ref): """Return the .dsc file that will be generated for this package.""" - v = getVersion(package, ref) + v = b.getVersion(package, ref) if v.debian_version: v_str = '%s-%s' % (v.upstream_version, @@ -96,11 +103,11 @@ -def aptCopy(packages, dst_pocket, src_pocket): +def aptCopy(package, commit, dst_pocket, src_pocket): """Copy a package from one pocket to another.""" binaries = getBinaries(package, commit) - cpatureOutput(['reprepro-env', 'copy', - b.pocketToApt(dst_pocket), - b.pocketToApt(src_pocket), - package] + binaries) + c.captureOutput(['reprepro-env', 'copy', + b.pocketToApt(dst_pocket), + b.pocketToApt(src_pocket), + package] + binaries) @@ -123,5 +130,5 @@ -def tagSubmodule(pocket, package, ref, principal): +def tagSubmodule(pocket, package, principal, version, env): """Tag a new version of a submodule. @@ -139,10 +146,5 @@ """ if not config.build.pockets[pocket].get('allow_backtracking', False): - env = dict(os.environ) branch = b.pocketToGit(pocket) - version = b.getVersion(package, ref) - - env['GIT_COMMITTER_NAME'] = config.build.tagger.name - env['GIT_COMMITTER_EMAIL'] = config.build.tagger.email tag_msg = ('Tag %s of %s\n\n' 'Requested by %s' % (version.full_version, @@ -160,5 +162,5 @@ branch = b.pocketToGit(pocket) c.captureOutput( - ['git', 'update-ref', 'refs/heads/%s' % branch, ref]) + ['git', 'update-ref', 'refs/heads/%s' % branch, ref], cwd=b.getRepo(package)) @@ -168,11 +170,11 @@ for changes in glob.glob(os.path.join(workdir, '*.changes')): c.captureOutput(['reprepro-env', + '--ignore=wrongdistribution', 'include', - '--ignore=wrongdistribution', apt, changes]) -def updateSuperproject(pocket, package, commit, principal): +def updateSuperproject(pocket, package, commit, principal, version, env): """Update the superproject. @@ -186,14 +188,14 @@ branch = b.pocketToGit(pocket) tree = c.captureOutput(['git', 'ls-tree', branch], - cwd=superproject) + cwd=superproject).strip() new_tree = re.compile( r'^(160000 commit )[0-9a-f]*(\t%s)$' % package, re.M).sub( - r'\1%s\2' % commit, + r'\g<1>%s\g<2>' % commit, tree) - new_tree_id = c.captureOutput(['git', 'mktree'], - cwd=superproject, - stdin_str=new_tree) + new_tree_id = c.captureOutput(['git', 'mktree', '--missing'], + cwd=superproject, + stdin_str=new_tree).strip() commit_msg = ('Update %s to version %s\n\n' @@ -202,8 +204,8 @@ principal)) new_commit = c.captureOutput( - ['git', 'commit-tree', new_tree_hash, '-p', branch], + ['git', 'commit-tree', new_tree_id, '-p', branch], cwd=superproject, env=env, - stdin_str=commit_msg) + stdin_str=commit_msg).strip() c.captureOutput( @@ -211,4 +213,7 @@ cwd=superproject) + +def makeReadable(workdir): + os.chmod(workdir, 0755) @contextlib.contextmanager @@ -228,5 +233,5 @@ ['git', 'archive', '--remote=file://%s' % b.getRepo(package), - '--prefix=%s' % package, + '--prefix=%s/' % package, commit, ], @@ -278,11 +283,15 @@ db.principal = principal database.session.save_or_update(db) - database.commit() - - database.begin() + database.session.commit() + + database.session.begin() try: db.failed_stage = 'validating job' - src = validateBuild(pocket, package, commit) + src = b.validateBuild(pocket, package, commit) + # Don't expand the commit in the DB until we're sure the user + # isn't trying to be tricky. + db.commit = commit = c.captureOutput(['git', 'rev-parse', commit], + cwd=b.getRepo(package)).strip() db.version = str(b.getVersion(package, commit)) @@ -295,5 +304,5 @@ if src != True: db.failed_stage = 'copying package from another pocket' - aptCopy(packages, pocket, src) + aptCopy(package, commit, pocket, src) # If we can't copy the package from somewhere, but # validateBuild didn't raise an exception, then we need to @@ -320,5 +329,5 @@ sbuildAll(package, commit, workdir) finally: - logdir = os.path.join(b._LOG_DIR, db.build_id) + logdir = os.path.join(b._LOG_DIR, str(db.build_id)) if not os.path.exists(logdir): os.makedirs(logdir) @@ -326,10 +335,19 @@ for log in glob.glob(os.path.join(workdir, '*.build')): os.copy2(log, logdir) + + db.failed_stage = 'processing metadata' + env = dict(os.environ) + env['GIT_COMMITTER_NAME'] = config.build.tagger.name + env['GIT_COMMITTER_EMAIL'] = config.build.tagger.email + version = b.getVersion(package, commit) + db.failed_stage = 'tagging submodule' - tagSubmodule(pocket, package, commit, principal) + tagSubmodule(pocket, package, principal, version, env) db.failed_stage = 'updating submodule branches' updateSubmoduleBranch(pocket, package, commit) db.failed_stage = 'updating superproject' - updateSuperproject(pocket, package, commit, principal) + updateSuperproject(pocket, package, commit, principal, version, env) + db.failed_stage = 'relaxing permissions on workdir' + makeReadable(workdir) db.failed_stage = 'uploading packages to apt repo' uploadBuild(pocket, workdir) Index: trunk/packages/invirt-dev/python/invirt/builder.py =================================================================== --- trunk/packages/invirt-dev/python/invirt/builder.py (revision 3011) +++ trunk/packages/invirt-dev/python/invirt/builder.py (revision 3028) @@ -29,4 +29,17 @@ return os.path.join(_REPO_DIR, 'invirt/packages', '%s.git' % package) +def ensureValidRepo(package): + """Perform some basic sanity checks that the requested repo is in a + subdirectory of _REPO_DIR/invirt/packages. This prevents weirdness + such as submitting a package like '../prod/...git'. Also ensures that + the repo exists.""" + # TODO: this might be easier just to regex + repo = os.path.abspath(getRepo(package)) + parent_dir = os.path.dirname(repo) + prefix = os.path.join(_REPO_DIR, 'invirt/packages') + if not parent_dir.startswith(prefix): + raise InvalidBuild('Invalid package name %s' % package) + elif not os.path.exists(repo): + raise InvalidBuild('Nonexisting package %s' % package) def pocketToGit(pocket): @@ -43,5 +56,5 @@ """Return the contents of a path from a git ref in a package.""" return c.captureOutput(['git', 'cat-file', 'blob', '%s:%s' % (ref, path)], - cwd=getRepo(package)) + cwd=getRepo(package)) @@ -86,4 +99,5 @@ it returns True. """ + ensureValidRepo(package) package_repo = getRepo(package) new_version = getVersion(package, commit) @@ -97,5 +111,5 @@ b = pocketToGit(p) current_commit = c.captureOutput(['git', 'rev-parse', b], - cwd=package_repo) + cwd=package_repo).strip() current_version = getVersion(package, b) @@ -104,9 +118,9 @@ ret = p else: - raise InvalidBuild('Version %s of %s already available in ' + raise InvalidBuild('Version %s of %s already available is in ' 'pocket %s from commit %s' % (new_version, package, p, current_commit)) - if config.build.pockets[pocket].get('allow_backtracking', False): + if not config.build.pockets[pocket].get('allow_backtracking', False): branch = pocketToGit(pocket) current_version = getVersion(package, branch)