source: trunk/scripts/git-hooks/builder/invirtibuilder @ 2541

#!/usr/bin/python

"""Process the Invirt build queue.

The Invirtibuilder handles package builds and uploads. On demand, it
attempts to build a particular package.

If the build succeeds, the new version of the package is uploaded to
the apt repository, tagged in its git repository, and the Invirt
superrepo is updated to point at the new version.

If the build fails, the Invirtibuilder sends mail with the build log.

The build queue is tracked via files in /var/lib/invirt-dev/queue. In
order to maintain ordering, all filenames in that directory are the
timestamp of their creation time.

Each queue file contains a file of the form

    pocket package hash principal

where pocket is one of the pockets globally configured in
git.pockets. For instance, the pockets in XVM are "prod" and "dev".

principal is the Kerberos principal that requested the build.
"""


import contextlib
import os
import re
import shutil
import subprocess

import pyinotify

from invirt.config import structs as config
from invirt import database


_QUEUE_DIR = '/var/lib/invirt-dev/queue'
_REPO_DIR = '/srv/git'
_LOG_DIR = '/var/log/invirt/builds'
_HOOKS_DIR = '/usr/share/invirt-dev/build.d'


DISTRIBUTION = 'hardy'


class InvalidBuild(ValueError):
    pass


def captureOutput(popen_args, stdin_str=None, *args, **kwargs):
    """Capture stdout from a command.

    This method will proxy the arguments to subprocess.Popen. It
    returns the output from the command if the call succeeded and
    raises an exception if the process returns a non-0 value.

    This is intended to be a variant on the subprocess.check_call
    function that also allows you access to the output from the
    command.
    """
    if 'stdin' not in kwargs:
        kwargs['stdin'] = subprocess.PIPE
    if 'stdout' not in kwargs:
        kwargs['stdout'] = subprocess.PIPE
    if 'stderr' not in kwargs:
        kwargs['stderr'] = subprocess.STDOUT
    p = subprocess.Popen(popen_args, *args, **kwargs)
    out, _ = p.communicate(stdin_str)
    if p.returncode:
        raise subprocess.CalledProcessError(p.returncode, popen_args, out)
    return out


def getRepo(package):
    """Return the path to the git repo for a given package."""
    return os.path.join(_REPO_DIR, 'packages', '%s.git' % package)


def pocketToGit(pocket):
    """Map a pocket in the configuration to a git branch."""
    return config.git.pockets[pocket].get('git', pocket)


def pocketToApt(pocket):
    """Map a pocket in the configuration to an apt repo pocket."""
    return config.git.pockets[pocket].get('apt', pocket)


def getGitFile(package, ref, path):
    """Return the contents of a path from a git ref in a package."""
    return captureOutput(['git', 'cat-file', 'blob', '%s:%s' % (ref, path)],
                         cwd=getRepo(package))


def getChangelog(package, ref):
    """Get a changelog object for a given ref in a given package.

    This returns a debian_bundle.changelog.Changelog object for a
    given ref of a given package.
    """
    return changelog.Changelog(getGitFile(package, ref, 'debian/changelog'))


def getVersion(package, ref):
    """Get the version of a given package at a particular ref."""
    return getChangelog(package, ref).get_version()


def getControl(package, ref):
    """Get the parsed debian/control file for a given package.

    This returns a list of debian_bundle.deb822.Deb822 objects, one
    for each section of the debian/control file. Each Deb822 object
    acts roughly like a dict.
    """
    return deb822.Deb822.iter_paragraphs(
        getGitFile(package, ref, 'debian/control').split('\n'))


def getBinaries(package, ref):
    """Get a list of binary packages in a package at a given ref."""
    return [p['Package'] for p in getControl(package, ref)
            if 'Package' in p]


def getArches(package, ref):
    """Get the set of all architectures in any binary package."""
    arches = set()
    for section in getControl(package, ref):
        if 'Architecture' in section:
            arches.update(section['Architecture'].split())
    return arches


def getDscName(package, ref):
    """Return the .dsc file that will be generated for this package."""
    v = getVersion(package, ref)
    return '%s_%s-%s.dsc' % (
        package,
        version.upstream_version,
        version.debian_version)


def validateBuild(pocket, package, commit):
    """Given the parameters of a new build, validate that build.

    The checks this function performs vary based on whether or not the
    pocket is configured with allow_backtracking.

    A build of a pocket without allow_backtracking set must be a
    fast-forward of the previous revision, and the most recent version
    in the changelog most be strictly greater than the version
    currently in the repository.

    In all cases, this revision of the package can only have the same
    version number as any other revision currently in the apt
    repository if they have the same commit ID.

    If it's unspecified, it is assumed that pocket do not
    allow_backtracking.

    If this build request fails validation, this function will raise a
    InvalidBuild exception, with information about why the validation
    failed.

    If this build request can be satisfied by copying the package from
    another pocket, then this function returns that pocket. Otherwise,
    it returns True.
    """
    package_repo = getRepo(package)
    new_version = getVersion(package, commit)

    for p in config.git.pockets:
        if p == pocket:
            continue

        b = pocketToGit(p)
        current_commit = captureOutput(['git', 'rev-parse', b],
                                       cwd=package_repo)
        current_version = getVersion(package, b)

        if current_version == new_version:
            if current_commit == commit:
                return p
            else:
                raise InvalidBuild('Version %s of %s already available in '
                                   'pocket %s from commit %s' %
                                   (new_version, package, p, current_commit))

    if config.git.pockets[pocket].get('allow_backtracking', False):
        branch = pocketToGit(pocket)
        current_version = getVersion(package, branch)
        if new_version <= current_version:
            raise InvalidBuild('New version %s of %s is not newer than '
                               'version %s currently in pocket %s' %
                               (new_version, package, current_version, pocket))

        # Almost by definition, A is a fast-forward of B if B..A is
        # empty
        if not captureOutput(['git', 'rev-list', '%s..%s' % (commit, branch)]):
            raise InvalidBuild('New commit %s of %s is not a fast-forward of'
                               'commit currently in pocket %s' %
                               (commit, package, pocket))


def sanitizeVersion(version):
    """Sanitize a Debian package version for use as a git tag.

    This function strips the epoch from the version number and
    replaces any tildes with periods."""
    v = '%s-%s' % (version.upstream_version,
                   version.debian_version)
    return v.replace('~', '.')


def aptCopy(packages, dst_pocket, src_pocket):
    """Copy a package from one pocket to another."""
    binaries = []
    for line in getGitFile(package, commit, 'debian/control').split('\n'):
        m = re.match('Package: (.*)$')
        if m:
            binaries.append(m.group(1))

    cpatureOutput(['reprepro-env', 'copy',
                   pocketToApt(dst_pocket),
                   pocketToApt(src_pocket),
                   package] + binaries)


def sbuild(package, ref, arch, workdir, arch_all=False):
    """Build a package for a particular architecture."""
    args = ['sbuild', '-d', DISTRIBUTION, '--arch', arch]
    if arch_all:
        args.append('-A')
    args.append(getDscName(package, ref))
    captureOutput(args, cwd=workdir, stdout=None)


def sbuildAll(package, ref, workdir):
    """Build a package for all architectures it supports."""
    arches = getArches(package, ref)
    if 'all' in arches or 'any' in arches or 'amd64' in arches:
        sbuild(package, ref, 'amd64', workdir, arch_all=True)
    if 'any' in arches or 'i386' in arches:
        sbuild(package, ref, 'i386', workdir)


def tagSubmodule(pocket, package, ref, principal):
    """Tag a new version of a submodule.

    If this pocket does not allow_backtracking, then this will create
    a new tag of the version at ref.

    This function doesn't need to care about lock
    contention. git-receive-pack updates one ref at a time, and only
    takes out a lock for that ref after it's passed the update
    hook. Because we reject pushes to tags in the update hook, no push
    can ever take out a lock on any tags.

    I'm sure that long description gives you great confidence in teh
    legitimacy of my reasoning.
    """
    if config.git.pockets[pocket].get('allow_backtracking', False):
        env = dict(os.environ)
        branch = pocketToGit(pocket)
        version = getVersion(package, ref)

        env['GIT_COMMITTER_NAME'] = config.git.tagger.name
        env['GIT_COMMITTER_EMAIL'] = config.git.tagger.email
        tag_msg = ('Tag %s of %s\n\n'
                   'Requested by %s' % (version.full_version,
                                        package,
                                        principal))

        captureOutput(
            ['git', 'tag', '-m', tag_msg, commit],
            stdout=None,
            env=env)


def updateSubmoduleBranch(pocket, package, ref):
    """Update the appropriately named branch in the submodule."""
    branch = pocketToGit(pocket)
    captureOutput(
        ['git', 'update-ref', 'refs/heads/%s' % branch, ref])


def uploadBuild(pocket, workdir):
    """Upload all build products in the work directory."""
    apt = pocketToApt(pocket)
    for changes in glob.glob(os.path.join(workdir, '*.changes')):
        captureOutput(['reprepro-env',
                       'include',
                       '--ignore=wrongdistribution',
                       apt,
                       changes])


def updateSuperrepo(pocket, package, commit, principal):
    """Update the superrepo.

    This will create a new commit on the branch for the given pocket
    that sets the commit for the package submodule to commit.

    Note that there's no locking issue here, because we disallow all
    pushes to the superrepo.
    """
    superrepo = os.path.join(_REPO_DIR, 'packages.git')
    branch = pocketToGit(pocket)
    tree = captureOutput(['git', 'ls-tree', branch],
                         cwd=superrepo)

    new_tree = re.compile(
        r'^(160000 commit )[0-9a-f]*(\t%s)$' % package, re.M).sub(
        r'\1%s\2' % commit,
        tree)

    new_tree_id = captureOutput(['git', 'mktree'],
                                cwd=superrepo,
                                stdin_str=new_tree)

    commit_msg = ('Update %s to version %s\n\n'
                  'Requested by %s' % (package,
                                       version.full_version,
                                       principal))
    new_commit = captureOutput(
        ['git', 'commit-tree', new_tree_hash, '-p', branch],
        cwd=superrepo,
        env=env,
        stdin_str=commit_msg)

    captureOutput(
        ['git', 'update-ref', 'refs/heads/%s' % branch, new_commit],
        cwd=superrepo)


@contextlib.contextmanager
def packageWorkdir(package):
    """Checkout the package in a temporary working directory.

    This context manager returns that working directory. The requested
    package is checked out into a subdirectory of the working
    directory with the same name as the package.

    When the context wrapped with this context manager is exited, the
    working directory is automatically deleted.
    """
    workdir = tempfile.mkdtemp()
    try:
        p_archive = subprocess.Popen(
            ['git', 'archive',
             '--remote=file://%s' % getRepo(package),
             '--prefix=%s' % package,
             commit,
             ],
            stdout=subprocess.PIPE,
            )
        p_tar = subprocess.Popen(
            ['tar', '-x'],
            stdin=p_archive.stdout,
            cwd=workdir,
            )
        p_archive.wait()
        p_tar.wait()

        yield workdir
    finally:
        shutil.rmtree(workdir)


def reportBuild(build):
    """Run hooks to report the results of a build attempt."""

    captureOutput(['run-parts',
                   '--arg=%s' % build.build_id,
                   '--',
                   _HOOKS_DIR])


def build():
    """Deal with items in the build queue.

    When triggered, iterate over build queue items one at a time,
    until there are no more pending build jobs.
    """
    while True:
        stage = 'processing incoming job'
        queue = os.listdir(_QUEUE_DIR)
        if not queue:
            break

        build = min(queue)
        job = open(os.path.join(_QUEUE_DIR, build)).read().strip()
        pocket, package, commit, principal = job.split()

        database.session.begin()
        db = database.Build()
        db.package = package
        db.pocket = pocket
        db.commit = commit
        db.principal = principal
        database.session.save_or_update(db)
        database.commit()

        database.begin()

        try:
            db.failed_stage = 'validating job'
            src = validateBuild(pocket, package, commit)

            db.version = str(getVersion(package, commit))

            # If validateBuild returns something other than True, then
            # it means we should copy from that pocket to our pocket.
            #
            # (If the validation failed, validateBuild would have
            # raised an exception)
            if src != True:
                db.failed_stage = 'copying package from another pocket'
                aptCopy(packages, pocket, src)
            # If we can't copy the package from somewhere, but
            # validateBuild didn't raise an exception, then we need to
            # do the build ourselves
            else:
                db.failed_stage = 'checking out package source'
                with packageWorkdir(package) as workdir:
                    db.failed_stage = 'preparing source package'
                    packagedir = os.path.join(workdir, package)

                    # We should be more clever about dealing with
                    # things like non-Debian-native packages than we
                    # are.
                    #
                    # If we were, we could use debuild and get nice
                    # environment scrubbing. Since we're not, debuild
                    # complains about not having an orig.tar.gz
                    captureOutput(['dpkg-buildpackage', '-us', '-uc', '-S'],
                                  cwd=packagedir,
                                  stdout=None)

                    try:
                        db.failed_stage = 'building binary packages'
                        sbuildAll(package, commit, workdir)
                    finally:
                        logdir = os.path.join(_LOG_DIR, db.build_id)
                        if not os.path.exists(logdir):
                            os.makedirs(logdir)

                        for log in glob.glob(os.path.join(workdir, '*.build')):
                            os.copy2(log, logdir)
                    db.failed_stage = 'tagging submodule'
                    tagSubmodule(pocket, package, commit, principal)
                    db.failed_stage = 'updating submodule branches'
                    updateSubmoduleBranch(pocket, package, commit)
                    db.failed_stage = 'updating superrepo'
                    updateSuperrepo(pocket, package, commit, principal)
                    db.failed_stage = 'uploading packages to apt repo'
                    uploadBuild(pocket, workdir)

                    db.failed_stage = 'cleaning up'

                # Finally, now that everything is done, remove the
                # build queue item
                os.unlink(os.path.join(_QUEUE_DIR, build))
        except:
            db.traceback = traceback.format_exc()
        else:
            db.succeeded = True
            db.failed_stage = None
        finally:
            database.session.save_or_update(db)
            database.session.commit()

            reportBuild(db)


class Invirtibuilder(pyinotify.ProcessEvent):
    """Process inotify triggers to build new packages."""
    def process_IN_CREATE(self, event):
        """Handle a created file or directory.

        When an IN_CREATE event comes in, trigger the builder.
        """
        build()


def main():
    """Initialize the inotifications and start the main loop."""
    database.connect()

    watch_manager = pyinotify.WatchManager()
    invirtibuilder = Invirtibuilder()
    notifier = pyinotify.Notifier(watch_manager, invirtibuilder)
    watch_manager.add_watch(_QUEUE_DIR,
                            pyinotify.EventsCodes.ALL_FLAGS['IN_CREATE'])

    # Before inotifying, run any pending builds; otherwise we won't
    # get notified for them.
    build()

    while True:
        notifier.process_events()
        if notifier.check_events():
            notifier.read_events()


if __name__ == '__main__':
    main()