source: trunk/scripts/git-hooks/builder/invirtibuilder @ 2538

Last change on this file since 2538 was 2538, checked in by broder, 15 years ago

First crack at the invirtibuilder.

No promises of functionality yet.

  • Property svn:executable set to *
File size: 17.0 KB
Line 
1#!/usr/bin/python
2
3"""Process the Invirt build queue.
4
5The Invirtibuilder handles package builds and uploads. On demand, it
6attempts to build a particular package.
7
8If the build succeeds, the new version of the package is uploaded to
9the apt repository, tagged in its git repository, and the Invirt
10superrepo is updated to point at the new version.
11
12If the build fails, the Invirtibuilder sends mail with the build log.
13
14The build queue is tracked via files in /var/lib/invirt-dev/queue. In
15order to maintain ordering, all filenames in that directory are the
16timestamp of their creation time.
17
18Each queue file contains a file of the form
19
20    pocket package hash principal
21
22where pocket is one of the pockets globally configured in
23git.pockets. For instance, the pockets in XVM are "prod" and "dev".
24
25principal is the Kerberos principal that requested the build.
26"""
27
28
29import contextlib
30import os
31import re
32import shutil
33import subprocess
34
35import pyinotify
36
37from invirt.config import structs as config
38from invirt import database
39
40
41_QUEUE_DIR = '/var/lib/invirt-dev/queue'
42_REPO_DIR = '/srv/git'
43_LOG_DIR = '/var/log/invirt/builds'
44_HOOKS_DIR = '/usr/share/invirt-dev/build.d'
45
46
47DISTRIBUTION = 'hardy'
48
49
50class InvalidBuild(ValueError):
51    pass
52
53
54def captureOutput(popen_args, stdin_str=None, *args, **kwargs):
55    """Capture stdout from a command.
56
57    This method will proxy the arguments to subprocess.Popen. It
58    returns the output from the command if the call succeeded and
59    raises an exception if the process returns a non-0 value.
60
61    This is intended to be a variant on the subprocess.check_call
62    function that also allows you access to the output from the
63    command.
64    """
65    if 'stdin' not in kwargs:
66        kwargs['stdin'] = subprocess.PIPE
67    if 'stdout' not in kwargs:
68        kwargs['stdout'] = subprocess.PIPE
69    if 'stderr' not in kwargs:
70        kwargs['stderr'] = subprocess.STDOUT
71    p = subprocess.Popen(popen_args, *args, **kwargs)
72    out, _ = p.communicate(stdin_str)
73    if p.returncode:
74        raise subprocess.CalledProcessError(p.returncode, popen_args, out)
75    return out
76
77
78def getRepo(package):
79    """Return the path to the git repo for a given package."""
80    return os.path.join(_REPO_DIR, 'packages', '%s.git' % package)
81
82
83def pocketToGit(pocket):
84    """Map a pocket in the configuration to a git branch."""
85    return config.git.pockets[pocket].get('git', pocket)
86
87
88def pocketToApt(pocket):
89    """Map a pocket in the configuration to an apt repo pocket."""
90    return config.git.pockets[pocket].get('apt', pocket)
91
92
93def getGitFile(package, ref, path):
94    """Return the contents of a path from a git ref in a package."""
95    return captureOutput(['git', 'cat-file', 'blob', '%s:%s' % (ref, path)],
96                         cwd=getRepo(package))
97
98
99def getChangelog(package, ref):
100    """Get a changelog object for a given ref in a given package.
101
102    This returns a debian_bundle.changelog.Changelog object for a
103    given ref of a given package.
104    """
105    return changelog.Changelog(getGitFile(package, ref, 'debian/changelog'))
106
107
108def getVersion(package, ref):
109    """Get the version of a given package at a particular ref."""
110    return getChangelog(package, ref).get_version()
111
112
113def getControl(package, ref):
114    """Get the parsed debian/control file for a given package.
115
116    This returns a list of debian_bundle.deb822.Deb822 objects, one
117    for each section of the debian/control file. Each Deb822 object
118    acts roughly like a dict.
119    """
120    return deb822.Deb822.iter_paragraphs(
121        getGitFile(package, ref, 'debian/control').split('\n'))
122
123
124def getBinaries(package, ref):
125    """Get a list of binary packages in a package at a given ref."""
126    return [p['Package'] for p in getControl(package, ref)
127            if 'Package' in p]
128
129
130def getArches(package, ref):
131    """Get the set of all architectures in any binary package."""
132    arches = set()
133    for section in getControl(package, ref):
134        if 'Architecture' in section:
135            arches.update(section['Architecture'].split())
136    return arches
137
138
139def getDscName(package, ref):
140    """Return the .dsc file that will be generated for this package."""
141    v = getVersion(package, ref)
142    return '%s_%s-%s.dsc' % (
143        package,
144        version.upstream_version,
145        version.debian_version)
146
147
148def validateBuild(pocket, package, commit):
149    """Given the parameters of a new build, validate that build.
150
151    The checks this function performs vary based on whether or not the
152    pocket is configured with allow_backtracking.
153
154    A build of a pocket without allow_backtracking set must be a
155    fast-forward of the previous revision, and the most recent version
156    in the changelog most be strictly greater than the version
157    currently in the repository.
158
159    In all cases, this revision of the package can only have the same
160    version number as any other revision currently in the apt
161    repository if they have the same commit ID.
162
163    If it's unspecified, it is assumed that pocket do not
164    allow_backtracking.
165
166    If this build request fails validation, this function will raise a
167    InvalidBuild exception, with information about why the validation
168    failed.
169
170    If this build request can be satisfied by copying the package from
171    another pocket, then this function returns that pocket. Otherwise,
172    it returns True.
173    """
174    package_repo = getRepo(package)
175    new_version = getVersion(package, commit)
176
177    for p in config.git.pockets:
178        if p == pocket:
179            continue
180
181        b = pocketToGit(p)
182        current_commit = captureOutput(['git', 'rev-parse', b],
183                                       cwd=package_repo)
184        current_version = getVersion(package, b)
185
186        if current_version == new_version:
187            if current_commit == commit:
188                return p
189            else:
190                raise InvalidBuild('Version %s of %s already available in '
191                                   'pocket %s from commit %s' %
192                                   (new_version, package, p, current_commit))
193
194    if config.git.pockets[pocket].get('allow_backtracking', False):
195        branch = pocketToGit(pocket)
196        current_version = getVersion(package, branch)
197        if new_version <= current_version:
198            raise InvalidBuild('New version %s of %s is not newer than '
199                               'version %s currently in pocket %s' %
200                               (new_version, package, current_version, pocket))
201
202        # Almost by definition, A is a fast-forward of B if B..A is
203        # empty
204        if not captureOutput(['git', 'rev-list', '%s..%s' % (commit, branch)]):
205            raise InvalidBuild('New commit %s of %s is not a fast-forward of'
206                               'commit currently in pocket %s' %
207                               (commit, package, pocket))
208
209
210def sanitizeVersion(version):
211    """Sanitize a Debian package version for use as a git tag.
212
213    This function strips the epoch from the version number and
214    replaces any tildes with periods."""
215    v = '%s-%s' % (version.upstream_version,
216                   version.debian_version)
217    return v.replace('~', '.')
218
219
220def aptCopy(packages, dst_pocket, src_pocket):
221    """Copy a package from one pocket to another."""
222    binaries = []
223    for line in getGitFile(package, commit, 'debian/control').split('\n'):
224        m = re.match('Package: (.*)$')
225        if m:
226            binaries.append(m.group(1))
227
228    cpatureOutput(['reprepro-env', 'copy',
229                   pocketToApt(dst_pocket),
230                   pocketToApt(src_pocket),
231                   package] + binaries)
232
233
234def sbuild(package, ref, arch, workdir, arch_all=False):
235    """Build a package for a particular architecture."""
236    args = ['sbuild', '-d', DISTRIBUTION, '--arch', arch]
237    if arch_all:
238        args.append('-A')
239    args.append(getDscName(package, ref))
240    captureOutput(args, cwd=workdir, stdout=None)
241
242
243def sbuildAll(package, ref, workdir):
244    """Build a package for all architectures it supports."""
245    arches = getArches(package, ref)
246    if 'all' in arches or 'any' in arches or 'amd64' in arches:
247        sbuild(package, ref, 'amd64', workdir, arch_all=True)
248    if 'any' in arches or 'i386' in arches:
249        sbuild(package, ref, 'i386', workdir)
250
251
252def tagSubmodule(pocket, package, ref, principal):
253    """Tag a new version of a submodule.
254
255    If this pocket does not allow_backtracking, then this will create
256    a new tag of the version at ref.
257
258    This function doesn't need to care about lock
259    contention. git-receive-pack updates one ref at a time, and only
260    takes out a lock for that ref after it's passed the update
261    hook. Because we reject pushes to tags in the update hook, no push
262    can ever take out a lock on any tags.
263
264    I'm sure that long description gives you great confidence in teh
265    legitimacy of my reasoning.
266    """
267    if config.git.pockets[pocket].get('allow_backtracking', False):
268        env = dict(os.environ)
269        branch = pocketToGit(pocket)
270        version = getVersion(package, ref)
271
272        env['GIT_COMMITTER_NAME'] = config.git.tagger.name
273        env['GIT_COMMITTER_EMAIL'] = config.git.tagger.email
274        tag_msg = ('Tag %s of %s\n\n'
275                   'Requested by %s' % (version.full_version,
276                                        package,
277                                        principal))
278
279        captureOutput(
280            ['git', 'tag', '-m', tag_msg, commit],
281            stdout=None,
282            env=env)
283
284
285def updateSubmoduleBranch(pocket, package, ref):
286    """Update the appropriately named branch in the submodule."""
287    branch = pocketToGit(pocket)
288    captureOutput(
289        ['git', 'update-ref', 'refs/heads/%s' % branch, ref])
290
291
292def uploadBuild(pocket, workdir):
293    """Upload all build products in the work directory."""
294    apt = pocketToApt(pocket)
295    for changes in glob.glob(os.path.join(workdir, '*.changes')):
296        captureOutput(['reprepro-env',
297                       'include',
298                       '--ignore=wrongdistribution',
299                       apt,
300                       changes])
301
302
303def updateSuperrepo(pocket, package, commit, principal):
304    """Update the superrepo.
305
306    This will create a new commit on the branch for the given pocket
307    that sets the commit for the package submodule to commit.
308
309    Note that there's no locking issue here, because we disallow all
310    pushes to the superrepo.
311    """
312    superrepo = os.path.join(_REPO_DIR, 'packages.git')
313    branch = pocketToGit(pocket)
314    tree = captureOutput(['git', 'ls-tree', branch],
315                         cwd=superrepo)
316
317    new_tree = re.compile(
318        r'^(160000 commit )[0-9a-f]*(\t%s)$' % package, re.M).sub(
319        r'\1%s\2' % commit,
320        tree)
321
322    new_tree_id = captureOutput(['git', 'mktree'],
323                                cwd=superrepo,
324                                stdin_str=new_tree)
325
326    commit_msg = ('Update %s to version %s\n\n'
327                  'Requested by %s' % (package,
328                                       version.full_version,
329                                       principal))
330    new_commit = captureOutput(
331        ['git', 'commit-tree', new_tree_hash, '-p', branch],
332        cwd=superrepo,
333        env=env,
334        stdin_str=commit_msg)
335
336    captureOutput(
337        ['git', 'update-ref', 'refs/heads/%s' % branch, new_commit],
338        cwd=superrepo)
339
340
341@contextlib.contextmanager
342def packageWorkdir(package):
343    """Checkout the package in a temporary working directory.
344
345    This context manager returns that working directory. The requested
346    package is checked out into a subdirectory of the working
347    directory with the same name as the package.
348
349    When the context wrapped with this context manager is exited, the
350    working directory is automatically deleted.
351    """
352    workdir = tempfile.mkdtemp()
353    try:
354        p_archive = subprocess.Popen(
355            ['git', 'archive',
356             '--remote=file://%s' % getRepo(package),
357             '--prefix=%s' % package,
358             commit,
359             ],
360            stdout=subprocess.PIPE,
361            )
362        p_tar = subprocess.Popen(
363            ['tar', '-x'],
364            stdin=p_archive.stdout,
365            cwd=workdir,
366            )
367        p_archive.wait()
368        p_tar.wait()
369
370        yield workdir
371    finally:
372        shutil.rmtree(workdir)
373
374
375def reportBuild(build):
376    """Run hooks to report the results of a build attempt."""
377
378    captureOutput(['run-parts',
379                   '--arg=%s' % build.build_id,
380                   '--',
381                   _HOOKS_DIR])
382
383
384def build():
385    """Deal with items in the build queue.
386
387    When triggered, iterate over build queue items one at a time,
388    until there are no more pending build jobs.
389    """
390    while True:
391        stage = 'processing incoming job'
392        queue = os.listdir(_QUEUE_DIR)
393        if not queue:
394            break
395
396        build = min(queue)
397        job = open(os.path.join(_QUEUE_DIR, build)).read().strip()
398        pocket, package, commit, principal = job.split()
399
400        database.session.begin()
401        db = database.Build()
402        db.package = package
403        db.pocket = pocket
404        db.commit = commit
405        db.principal = principal
406        database.session.save_or_update(db)
407        database.commit()
408
409        database.begin()
410
411        try:
412            db.failed_stage = 'validating job'
413            src = validateBuild(pocket, package, commit)
414
415            db.version = str(getVersion(package, commit))
416
417            # If validateBuild returns something other than True, then
418            # it means we should copy from that pocket to our pocket.
419            #
420            # (If the validation failed, validateBuild would have
421            # raised an exception)
422            if src != True:
423                db.failed_stage = 'copying package from another pocket'
424                aptCopy(packages, pocket, src)
425            # If we can't copy the package from somewhere, but
426            # validateBuild didn't raise an exception, then we need to
427            # do the build ourselves
428            else:
429                db.failed_stage = 'checking out package source'
430                with packageWorkdir(package) as workdir:
431                    db.failed_stage = 'preparing source package'
432                    packagedir = os.path.join(workdir, package)
433
434                    # We should be more clever about dealing with
435                    # things like non-Debian-native packages than we
436                    # are.
437                    #
438                    # If we were, we could use debuild and get nice
439                    # environment scrubbing. Since we're not, debuild
440                    # complains about not having an orig.tar.gz
441                    captureOutput(['dpkg-buildpackage', '-us', '-uc', '-S'],
442                                  cwd=packagedir,
443                                  stdout=None)
444
445                    try:
446                        db.failed_stage = 'building binary packages'
447                        sbuildAll(package, commit, workdir)
448                    finally:
449                        logdir = os.path.join(_LOG_DIR, db.build_id)
450                        if not os.path.exists(logdir):
451                            os.makedirs(logdir)
452
453                        for log in glob.glob(os.path.join(workdir, '*.build')):
454                            os.copy2(log, logdir)
455                    db.failed_stage = 'tagging submodule'
456                    tagSubmodule(pocket, package, commit, principal)
457                    db.failed_stage = 'updating submodule branches'
458                    updateSubmoduleBranch(pocket, package, commit)
459                    db.failed_stage = 'updating superrepo'
460                    updateSuperrepo(pocket, package, commit, principal)
461                    db.failed_stage = 'uploading packages to apt repo'
462                    uploadBuild(pocket, workdir)
463
464                    db.failed_stage = 'cleaning up'
465
466                # Finally, now that everything is done, remove the
467                # build queue item
468                os.unlink(os.path.join(_QUEUE_DIR, build))
469        except:
470            db.traceback = traceback.format_exc()
471        else:
472            db.succeeded = True
473            db.failed_stage = None
474        finally:
475            database.session.save_or_update(db)
476            database.session.commit()
477
478            reportBuild(db)
479
480
481class Invirtibuilder(pyinotify.ProcessEvent):
482    """Process inotify triggers to build new packages."""
483    def process_IN_CREATE(self, event):
484        """Handle a created file or directory.
485
486        When an IN_CREATE event comes in, trigger the builder.
487        """
488        build()
489
490
491def main():
492    """Initialize the inotifications and start the main loop."""
493    database.connect()
494
495    watch_manager = pyinotify.WatchManager()
496    invirtibuilder = Invirtibuilder()
497    notifier = pyinotify.Notifier(watch_manager, invirtibuilder)
498    watch_manager.add_watch(_QUEUE_DIR,
499                            pyinotify.EventsCodes.ALL_FLAGS['IN_CREATE'])
500
501    # Before inotifying, run any pending builds; otherwise we won't
502    # get notified for them.
503    build()
504
505    while True:
506        notifier.process_events()
507        if notifier.check_events():
508            notifier.read_events()
509
510
511if __name__ == '__main__':
512    main()
Note: See TracBrowser for help on using the repository browser.