source: trunk/packages/invirt-dev/invirtibuilder @ 3049

Last change on this file since 3049 was 3045, checked in by gdb, 14 years ago

Don't forget to do tagging before an aptCopy

  • Property svn:executable set to *
File size: 14.2 KB
RevLine 
[2538]1#!/usr/bin/python
2
3"""Process the Invirt build queue.
4
5The Invirtibuilder handles package builds and uploads. On demand, it
6attempts to build a particular package.
7
8If the build succeeds, the new version of the package is uploaded to
9the apt repository, tagged in its git repository, and the Invirt
[2801]10superproject is updated to point at the new version.
[2538]11
12If the build fails, the Invirtibuilder sends mail with the build log.
13
14The build queue is tracked via files in /var/lib/invirt-dev/queue. In
15order to maintain ordering, all filenames in that directory are the
16timestamp of their creation time.
17
18Each queue file contains a file of the form
19
20    pocket package hash principal
21
22where pocket is one of the pockets globally configured in
[2593]23build.pockets. For instance, the pockets in XVM are "prod" and "dev".
[2538]24
25principal is the Kerberos principal that requested the build.
26"""
27
28
[2546]29from __future__ import with_statement
30
[2538]31import contextlib
[3028]32import glob
[2538]33import os
34import re
35import shutil
36import subprocess
[3028]37import tempfile
38import traceback
[2538]39
40import pyinotify
41
[3028]42from debian_bundle import deb822
43
[2543]44import invirt.builder as b
[3028]45import invirt.common as c
[2538]46from invirt import database
[3028]47from invirt.config import structs as config
[2538]48
49
50DISTRIBUTION = 'hardy'
51
52
53def getControl(package, ref):
54    """Get the parsed debian/control file for a given package.
55
56    This returns a list of debian_bundle.deb822.Deb822 objects, one
57    for each section of the debian/control file. Each Deb822 object
58    acts roughly like a dict.
59    """
60    return deb822.Deb822.iter_paragraphs(
[3028]61        b.getGitFile(package, ref, 'debian/control').split('\n'))
[2538]62
63
64def getBinaries(package, ref):
65    """Get a list of binary packages in a package at a given ref."""
66    return [p['Package'] for p in getControl(package, ref)
67            if 'Package' in p]
68
69
70def getArches(package, ref):
71    """Get the set of all architectures in any binary package."""
72    arches = set()
73    for section in getControl(package, ref):
74        if 'Architecture' in section:
75            arches.update(section['Architecture'].split())
76    return arches
77
78
79def getDscName(package, ref):
80    """Return the .dsc file that will be generated for this package."""
[3028]81    v = b.getVersion(package, ref)
[2544]82    if v.debian_version:
83        v_str = '%s-%s' % (v.upstream_version,
84                           v.debian_version)
85    else:
86        v_str = v.upstream_version
87    return '%s_%s.dsc' % (
[2538]88        package,
[2544]89        v_str)
[2538]90
91
92def sanitizeVersion(version):
93    """Sanitize a Debian package version for use as a git tag.
94
95    This function strips the epoch from the version number and
96    replaces any tildes with periods."""
[2544]97    if v.debian_version:
98        v = '%s-%s' % (version.upstream_version,
99                       version.debian_version)
100    else:
101        v = version.upstream_version
[2538]102    return v.replace('~', '.')
103
104
[3028]105def aptCopy(package, commit, dst_pocket, src_pocket):
[2538]106    """Copy a package from one pocket to another."""
[2545]107    binaries = getBinaries(package, commit)
[3028]108    c.captureOutput(['reprepro-env', 'copy',
109                     b.pocketToApt(dst_pocket),
110                     b.pocketToApt(src_pocket),
111                     package] + binaries)
[2538]112
113
114def sbuild(package, ref, arch, workdir, arch_all=False):
115    """Build a package for a particular architecture."""
[3032]116    args = ['sbuild', '-v', '-d', DISTRIBUTION, '--arch', arch]
[2538]117    if arch_all:
118        args.append('-A')
119    args.append(getDscName(package, ref))
[3032]120    c.captureOutput(args, cwd=workdir)
[2538]121
122
123def sbuildAll(package, ref, workdir):
124    """Build a package for all architectures it supports."""
125    arches = getArches(package, ref)
126    if 'all' in arches or 'any' in arches or 'amd64' in arches:
127        sbuild(package, ref, 'amd64', workdir, arch_all=True)
128    if 'any' in arches or 'i386' in arches:
129        sbuild(package, ref, 'i386', workdir)
130
131
[3043]132def tagSubmodule(pocket, package, commit, principal, version, env):
[2538]133    """Tag a new version of a submodule.
134
135    If this pocket does not allow_backtracking, then this will create
136    a new tag of the version at ref.
137
138    This function doesn't need to care about lock
139    contention. git-receive-pack updates one ref at a time, and only
140    takes out a lock for that ref after it's passed the update
141    hook. Because we reject pushes to tags in the update hook, no push
142    can ever take out a lock on any tags.
143
[2836]144    I'm sure that long description gives you great confidence in the
[2538]145    legitimacy of my reasoning.
146    """
[2837]147    if not config.build.pockets[pocket].get('allow_backtracking', False):
[2543]148        branch = b.pocketToGit(pocket)
[2538]149        tag_msg = ('Tag %s of %s\n\n'
150                   'Requested by %s' % (version.full_version,
151                                        package,
152                                        principal))
153
[2543]154        c.captureOutput(
[2538]155            ['git', 'tag', '-m', tag_msg, commit],
[3043]156            env=env,
157            cwd=b.getRepo(package))
[2538]158
159
[3043]160def updateSubmoduleBranch(pocket, package, commit):
[2538]161    """Update the appropriately named branch in the submodule."""
[2543]162    branch = b.pocketToGit(pocket)
163    c.captureOutput(
[3043]164        ['git', 'update-ref', 'refs/heads/%s' % branch, commit], cwd=b.getRepo(package))
[2538]165
166
167def uploadBuild(pocket, workdir):
168    """Upload all build products in the work directory."""
[3044]169    force = config.build.pockets[pocket].get('allow_backtracking', False)
[2543]170    apt = b.pocketToApt(pocket)
[2538]171    for changes in glob.glob(os.path.join(workdir, '*.changes')):
[3044]172        upload = ['reprepro-env', '--ignore=wrongdistribution',
173                  'include', apt, changes]
174        try:
175            c.captureOutput(upload)
176        except subprocess.CalledProcessError, e:
177            if not force:
178                raise
179            package = deb822.Changes(open(changes).read())['Binary']
180            c.captureOutput(['reprepro-env', 'remove', apt, package])
181            c.captureOutput(upload)
[2538]182
183
[3028]184def updateSuperproject(pocket, package, commit, principal, version, env):
[2801]185    """Update the superproject.
[2538]186
187    This will create a new commit on the branch for the given pocket
188    that sets the commit for the package submodule to commit.
189
190    Note that there's no locking issue here, because we disallow all
[2801]191    pushes to the superproject.
[2538]192    """
[2801]193    superproject = os.path.join(b._REPO_DIR, 'invirt/packages.git')
[2543]194    branch = b.pocketToGit(pocket)
195    tree = c.captureOutput(['git', 'ls-tree', branch],
[3028]196                           cwd=superproject).strip()
[2538]197
198    new_tree = re.compile(
199        r'^(160000 commit )[0-9a-f]*(\t%s)$' % package, re.M).sub(
[3028]200        r'\g<1>%s\g<2>' % commit,
[2538]201        tree)
202
[3028]203    new_tree_id = c.captureOutput(['git', 'mktree', '--missing'],
204                                  cwd=superproject,
205                                  stdin_str=new_tree).strip()
[2538]206
207    commit_msg = ('Update %s to version %s\n\n'
208                  'Requested by %s' % (package,
209                                       version.full_version,
210                                       principal))
[2543]211    new_commit = c.captureOutput(
[3028]212        ['git', 'commit-tree', new_tree_id, '-p', branch],
[2801]213        cwd=superproject,
[2538]214        env=env,
[3028]215        stdin_str=commit_msg).strip()
[2538]216
[2543]217    c.captureOutput(
[2538]218        ['git', 'update-ref', 'refs/heads/%s' % branch, new_commit],
[2801]219        cwd=superproject)
[2538]220
221
[3028]222def makeReadable(workdir):
223    os.chmod(workdir, 0755)
224
[2538]225@contextlib.contextmanager
[2835]226def packageWorkdir(package, commit):
[2538]227    """Checkout the package in a temporary working directory.
228
229    This context manager returns that working directory. The requested
230    package is checked out into a subdirectory of the working
231    directory with the same name as the package.
232
233    When the context wrapped with this context manager is exited, the
234    working directory is automatically deleted.
235    """
236    workdir = tempfile.mkdtemp()
237    try:
238        p_archive = subprocess.Popen(
239            ['git', 'archive',
[2543]240             '--remote=file://%s' % b.getRepo(package),
[3028]241             '--prefix=%s/' % package,
[2538]242             commit,
243             ],
244            stdout=subprocess.PIPE,
245            )
246        p_tar = subprocess.Popen(
247            ['tar', '-x'],
248            stdin=p_archive.stdout,
249            cwd=workdir,
250            )
251        p_archive.wait()
252        p_tar.wait()
253
254        yield workdir
255    finally:
256        shutil.rmtree(workdir)
257
258def build():
259    """Deal with items in the build queue.
260
261    When triggered, iterate over build queue items one at a time,
262    until there are no more pending build jobs.
263    """
264    while True:
265        stage = 'processing incoming job'
[2543]266        queue = os.listdir(b._QUEUE_DIR)
[2538]267        if not queue:
268            break
269
270        build = min(queue)
[2543]271        job = open(os.path.join(b._QUEUE_DIR, build)).read().strip()
[2538]272        pocket, package, commit, principal = job.split()
273
274        database.session.begin()
275        db = database.Build()
276        db.package = package
277        db.pocket = pocket
278        db.commit = commit
279        db.principal = principal
280        database.session.save_or_update(db)
[3028]281        database.session.commit()
[2538]282
[3028]283        database.session.begin()
[2538]284
285        try:
286            db.failed_stage = 'validating job'
[3028]287            # Don't expand the commit in the DB until we're sure the user
288            # isn't trying to be tricky.
[3035]289            b.ensureValidPackage(package)
290            db.commit = commit = b.canonicalize_commit(package, commit)
291            src = b.validateBuild(pocket, package, commit)
[3045]292            version = b.getVersion(package, commit)
293            db.version = str(version)
[3036]294            b.runHook('pre-build', [str(db.build_id), db.pocket, db.package,
295                                    db.commit, db.principal, db.version, str(db.inserted_at)])
[2538]296
[3045]297            env = dict(os.environ)
298            env['GIT_COMMITTER_NAME'] = config.build.tagger.name
299            env['GIT_COMMITTER_EMAIL'] = config.build.tagger.email
300
[2538]301            # If validateBuild returns something other than True, then
302            # it means we should copy from that pocket to our pocket.
303            #
304            # (If the validation failed, validateBuild would have
305            # raised an exception)
306            if src != True:
[3045]307                # TODO: cut out this code duplication
308                db.failed_stage = 'tagging submodule before copying package'
309                tagSubmodule(pocket, package, commit, principal, version, env)
310                db.failed_stage = 'updating submodule branches before copying package'
311                updateSubmoduleBranch(pocket, package, commit)
312                db.failed_stage = 'updating superproject before copying package'
313                updateSuperproject(pocket, package, commit, principal, version, env)
[2538]314                db.failed_stage = 'copying package from another pocket'
[3028]315                aptCopy(package, commit, pocket, src)
[3045]316               
[2538]317            # If we can't copy the package from somewhere, but
318            # validateBuild didn't raise an exception, then we need to
319            # do the build ourselves
320            else:
321                db.failed_stage = 'checking out package source'
[2835]322                with packageWorkdir(package, commit) as workdir:
[2538]323                    db.failed_stage = 'preparing source package'
324                    packagedir = os.path.join(workdir, package)
325
326                    # We should be more clever about dealing with
327                    # things like non-Debian-native packages than we
328                    # are.
329                    #
330                    # If we were, we could use debuild and get nice
331                    # environment scrubbing. Since we're not, debuild
332                    # complains about not having an orig.tar.gz
[2543]333                    c.captureOutput(['dpkg-buildpackage', '-us', '-uc', '-S'],
[2538]334                                  cwd=packagedir,
335                                  stdout=None)
336
337                    try:
338                        db.failed_stage = 'building binary packages'
339                        sbuildAll(package, commit, workdir)
340                    finally:
[3028]341                        logdir = os.path.join(b._LOG_DIR, str(db.build_id))
[2538]342                        if not os.path.exists(logdir):
343                            os.makedirs(logdir)
344
[3032]345                        for log in glob.glob(os.path.join(workdir, 'build-*.log')):
346                            os.copy(log, logdir)
[3028]347
[2538]348                    db.failed_stage = 'tagging submodule'
[3045]349                    tagSubmodule(pocket, package, commit, principal, version, env)
[2538]350                    db.failed_stage = 'updating submodule branches'
351                    updateSubmoduleBranch(pocket, package, commit)
[2801]352                    db.failed_stage = 'updating superproject'
[3028]353                    updateSuperproject(pocket, package, commit, principal, version, env)
354                    db.failed_stage = 'relaxing permissions on workdir'
355                    makeReadable(workdir)
[2538]356                    db.failed_stage = 'uploading packages to apt repo'
357                    uploadBuild(pocket, workdir)
358
359                    db.failed_stage = 'cleaning up'
360        except:
361            db.traceback = traceback.format_exc()
362        else:
363            db.succeeded = True
364            db.failed_stage = None
365        finally:
366            database.session.save_or_update(db)
367            database.session.commit()
368
[2838]369            # Finally, now that everything is done, remove the
370            # build queue item
371            os.unlink(os.path.join(b._QUEUE_DIR, build))
372
[3036]373            if db.succeeded:
374                b.runHook('post-build', [str(db.build_id)])
375            else:
376                b.runHook('failed-build', [str(db.build_id)])
[2538]377
378class Invirtibuilder(pyinotify.ProcessEvent):
379    """Process inotify triggers to build new packages."""
[2820]380    def process_default(self, event):
381        """Handle an inotify event.
[2538]382
[2820]383        When an inotify event comes in, trigger the builder.
[2538]384        """
385        build()
386
387
388def main():
389    """Initialize the inotifications and start the main loop."""
390    database.connect()
391
392    watch_manager = pyinotify.WatchManager()
393    invirtibuilder = Invirtibuilder()
394    notifier = pyinotify.Notifier(watch_manager, invirtibuilder)
[2543]395    watch_manager.add_watch(b._QUEUE_DIR,
[2820]396                            pyinotify.EventsCodes.ALL_FLAGS['IN_CREATE'] |
397                            pyinotify.EventsCodes.ALL_FLAGS['IN_MOVED_TO'])
[2538]398
399    # Before inotifying, run any pending builds; otherwise we won't
400    # get notified for them.
401    build()
402
403    while True:
404        notifier.process_events()
405        if notifier.check_events():
406            notifier.read_events()
407
408
409if __name__ == '__main__':
410    main()
Note: See TracBrowser for help on using the repository browser.