Changeset 3028


Ignore:
Timestamp:
Jul 5, 2010, 2:01:45 AM (14 years ago)
Author:
gdb
Message:

Make invirtibuilder work

Location:
trunk/packages/invirt-dev
Files:
6 edited

Legend:

Unmodified
Added
Removed

  • trunk/packages/invirt-dev/README

    r1322 r3028  
    66   The public key should then be in the apt keyring of systems using
    77   the packages.
     8
     9 - Packages should be located at
     10   /srv/git/invirt/packages/${packagename}.git

  • trunk/packages/invirt-dev/debian/changelog

    r3011 r3028  
     1invirt-dev (0.1.1) unstable; urgency=low
     2
     3  * Added missing imports
     4  * Other minor functional changes
     5
     6 -- Greg Brockman <gdb@mit.edu>  Mon, 05 Jul 2010 01:43:45 -0400
     7
    18invirt-dev (0.1.0) unstable; urgency=low
    29

  • trunk/packages/invirt-dev/debian/control

    r2819 r3028  
    88Package: invirt-dev
    99Architecture: all
    10 Depends: ${shlibs:Depends}, ${misc:Depends}, dpkg-dev-el, emacs21, reprepro, apache2, postfix, screen, dh-make, fakeroot, quilt, patchutils, config-package-dev, pbuilder, equivs, invirt-base, invirt-database, remctl-server, update-inetd, openbsd-inetd | inet-superserver, python-pyinotify
     10Depends: ${shlibs:Depends}, ${misc:Depends}, dpkg-dev-el, emacs21, reprepro, apache2, postfix, screen, dh-make, fakeroot, quilt, patchutils, config-package-dev, sbuild, equivs, invirt-base, invirt-database, remctl-server, update-inetd, openbsd-inetd | inet-superserver, python-pyinotify, python-debian, git-core (>= 1.6.4)
    1111Description: Invirt build and apt server
    1212 This packages the build scripts and apt-repository configuration for Invirt.

  • trunk/packages/invirt-dev/invirt-submit-build

    r2639 r3028  
    1919
    2020import datetime
     21import optparse
    2122import os
    2223import sys
    … …  
    2829
    2930def main():
    30     pocket, package, commit = sys.argv[1:4]
     31    parser = optparse.OptionParser('Usage: %prog pocket package commit')
     32    opts, args = parser.parse_args()
     33    if len(args) != 3:
     34        parser.print_help()
     35        return 1
     36    pocket, package, commit = args
    3137    principal = os.environ['REMOTE_USER']
    3238    request_time = datetime.datetime.utcnow()
    … …  
    4753    q = os.fdopen(q_fd, 'r+')
    4854    print >>q, "%s %s %s %s" % (pocket, package, commit, principal)
     55    q.close()
    4956    os.rename(q_name, q_path)
    5057

  • trunk/packages/invirt-dev/invirtibuilder

    r2838 r3028  
    3030
    3131import contextlib
     32import glob
    3233import os
    3334import re
    3435import shutil
    3536import subprocess
     37import tempfile
     38import traceback
    3639
    3740import pyinotify
    3841
     42from debian_bundle import deb822
     43
    3944import invirt.builder as b
     45import invirt.common as c
    4046from invirt import database
     47from invirt.config import structs as config
    4148
    4249
    … …  
    5259    """
    5360    return deb822.Deb822.iter_paragraphs(
    54         getGitFile(package, ref, 'debian/control').split('\n'))
     61        b.getGitFile(package, ref, 'debian/control').split('\n'))
    5562
    5663
    … …  
    7279def getDscName(package, ref):
    7380    """Return the .dsc file that will be generated for this package."""
    74     v = getVersion(package, ref)
     81    v = b.getVersion(package, ref)
    7582    if v.debian_version:
    7683        v_str = '%s-%s' % (v.upstream_version,
    … …  
    96103
    97104
    98 def aptCopy(packages, dst_pocket, src_pocket):
     105def aptCopy(package, commit, dst_pocket, src_pocket):
    99106    """Copy a package from one pocket to another."""
    100107    binaries = getBinaries(package, commit)
    101     cpatureOutput(['reprepro-env', 'copy',
    102                    b.pocketToApt(dst_pocket),
    103                    b.pocketToApt(src_pocket),
    104                    package] + binaries)
     108    c.captureOutput(['reprepro-env', 'copy',
     109                     b.pocketToApt(dst_pocket),
     110                     b.pocketToApt(src_pocket),
     111                     package] + binaries)
    105112
    106113
    … …  
    123130
    124131
    125 def tagSubmodule(pocket, package, ref, principal):
     132def tagSubmodule(pocket, package, principal, version, env):
    126133    """Tag a new version of a submodule.
    127134
    … …  
    139146    """
    140147    if not config.build.pockets[pocket].get('allow_backtracking', False):
    141         env = dict(os.environ)
    142148        branch = b.pocketToGit(pocket)
    143         version = b.getVersion(package, ref)
    144 
    145         env['GIT_COMMITTER_NAME'] = config.build.tagger.name
    146         env['GIT_COMMITTER_EMAIL'] = config.build.tagger.email
    147149        tag_msg = ('Tag %s of %s\n\n'
    148150                   'Requested by %s' % (version.full_version,
    … …  
    160162    branch = b.pocketToGit(pocket)
    161163    c.captureOutput(
    162         ['git', 'update-ref', 'refs/heads/%s' % branch, ref])
     164        ['git', 'update-ref', 'refs/heads/%s' % branch, ref], cwd=b.getRepo(package))
    163165
    164166
    … …  
    168170    for changes in glob.glob(os.path.join(workdir, '*.changes')):
    169171        c.captureOutput(['reprepro-env',
     172                       '--ignore=wrongdistribution',
    170173                       'include',
    171                        '--ignore=wrongdistribution',
    172174                       apt,
    173175                       changes])
    174176
    175177
    176 def updateSuperproject(pocket, package, commit, principal):
     178def updateSuperproject(pocket, package, commit, principal, version, env):
    177179    """Update the superproject.
    178180
    … …  
    186188    branch = b.pocketToGit(pocket)
    187189    tree = c.captureOutput(['git', 'ls-tree', branch],
    188                          cwd=superproject)
     190                           cwd=superproject).strip()
    189191
    190192    new_tree = re.compile(
    191193        r'^(160000 commit )[0-9a-f]*(\t%s)$' % package, re.M).sub(
    192         r'\1%s\2' % commit,
     194        r'\g<1>%s\g<2>' % commit,
    193195        tree)
    194196
    195     new_tree_id = c.captureOutput(['git', 'mktree'],
    196                                 cwd=superproject,
    197                                 stdin_str=new_tree)
     197    new_tree_id = c.captureOutput(['git', 'mktree', '--missing'],
     198                                  cwd=superproject,
     199                                  stdin_str=new_tree).strip()
    198200
    199201    commit_msg = ('Update %s to version %s\n\n'
    … …  
    202204                                       principal))
    203205    new_commit = c.captureOutput(
    204         ['git', 'commit-tree', new_tree_hash, '-p', branch],
     206        ['git', 'commit-tree', new_tree_id, '-p', branch],
    205207        cwd=superproject,
    206208        env=env,
    207         stdin_str=commit_msg)
     209        stdin_str=commit_msg).strip()
    208210
    209211    c.captureOutput(
    … …  
    211213        cwd=superproject)
    212214
     215
     216def makeReadable(workdir):
     217    os.chmod(workdir, 0755)
    213218
    214219@contextlib.contextmanager
    … …  
    228233            ['git', 'archive',
    229234             '--remote=file://%s' % b.getRepo(package),
    230              '--prefix=%s' % package,
     235             '--prefix=%s/' % package,
    231236             commit,
    232237             ],
    … …  
    278283        db.principal = principal
    279284        database.session.save_or_update(db)
    280         database.commit()
    281 
    282         database.begin()
     285        database.session.commit()
     286
     287        database.session.begin()
    283288
    284289        try:
    285290            db.failed_stage = 'validating job'
    286             src = validateBuild(pocket, package, commit)
     291            src = b.validateBuild(pocket, package, commit)
     292            # Don't expand the commit in the DB until we're sure the user
     293            # isn't trying to be tricky.
     294            db.commit = commit = c.captureOutput(['git', 'rev-parse', commit],
     295                                                 cwd=b.getRepo(package)).strip()
    287296
    288297            db.version = str(b.getVersion(package, commit))
    … …  
    295304            if src != True:
    296305                db.failed_stage = 'copying package from another pocket'
    297                 aptCopy(packages, pocket, src)
     306                aptCopy(package, commit, pocket, src)
    298307            # If we can't copy the package from somewhere, but
    299308            # validateBuild didn't raise an exception, then we need to
    … …  
    320329                        sbuildAll(package, commit, workdir)
    321330                    finally:
    322                         logdir = os.path.join(b._LOG_DIR, db.build_id)
     331                        logdir = os.path.join(b._LOG_DIR, str(db.build_id))
    323332                        if not os.path.exists(logdir):
    324333                            os.makedirs(logdir)
    … …  
    326335                        for log in glob.glob(os.path.join(workdir, '*.build')):
    327336                            os.copy2(log, logdir)
     337
     338                    db.failed_stage = 'processing metadata'
     339                    env = dict(os.environ)
     340                    env['GIT_COMMITTER_NAME'] = config.build.tagger.name
     341                    env['GIT_COMMITTER_EMAIL'] = config.build.tagger.email
     342                    version = b.getVersion(package, commit)
     343
    328344                    db.failed_stage = 'tagging submodule'
    329                     tagSubmodule(pocket, package, commit, principal)
     345                    tagSubmodule(pocket, package, principal, version, env)
    330346                    db.failed_stage = 'updating submodule branches'
    331347                    updateSubmoduleBranch(pocket, package, commit)
    332348                    db.failed_stage = 'updating superproject'
    333                     updateSuperproject(pocket, package, commit, principal)
     349                    updateSuperproject(pocket, package, commit, principal, version, env)
     350                    db.failed_stage = 'relaxing permissions on workdir'
     351                    makeReadable(workdir)
    334352                    db.failed_stage = 'uploading packages to apt repo'
    335353                    uploadBuild(pocket, workdir)

  • trunk/packages/invirt-dev/python/invirt/builder.py

    r2834 r3028  
    2929    return os.path.join(_REPO_DIR, 'invirt/packages', '%s.git' % package)
    3030
     31def ensureValidRepo(package):
     32    """Perform some basic sanity checks that the requested repo is in a
     33    subdirectory of _REPO_DIR/invirt/packages.  This prevents weirdness
     34    such as submitting a package like '../prod/...git'.  Also ensures that
     35    the repo exists."""
     36    # TODO: this might be easier just to regex
     37    repo = os.path.abspath(getRepo(package))
     38    parent_dir = os.path.dirname(repo)
     39    prefix = os.path.join(_REPO_DIR, 'invirt/packages')
     40    if not parent_dir.startswith(prefix):
     41        raise InvalidBuild('Invalid package name %s' % package)
     42    elif not os.path.exists(repo):
     43        raise InvalidBuild('Nonexisting package %s' % package)
    3144
    3245def pocketToGit(pocket):
    … …  
    4356    """Return the contents of a path from a git ref in a package."""
    4457    return c.captureOutput(['git', 'cat-file', 'blob', '%s:%s' % (ref, path)],
    45                          cwd=getRepo(package))
     58                           cwd=getRepo(package))
    4659
    4760
    … …  
    8699    it returns True.
    87100    """
     101    ensureValidRepo(package)
    88102    package_repo = getRepo(package)
    89103    new_version = getVersion(package, commit)
    … …  
    97111        b = pocketToGit(p)
    98112        current_commit = c.captureOutput(['git', 'rev-parse', b],
    99                                        cwd=package_repo)
     113                                         cwd=package_repo).strip()
    100114        current_version = getVersion(package, b)
    101115
    … …  
    104118                ret = p
    105119            else:
    106                 raise InvalidBuild('Version %s of %s already available in '
     120                raise InvalidBuild('Version %s of %s already available is in '
    107121                                   'pocket %s from commit %s' %
    108122                                   (new_version, package, p, current_commit))
    109123
    110     if config.build.pockets[pocket].get('allow_backtracking', False):
     124    if not config.build.pockets[pocket].get('allow_backtracking', False):
    111125        branch = pocketToGit(pocket)
    112126        current_version = getVersion(package, branch)
Note: See TracChangeset for help on using the changeset viewer.