source: trunk/packages/invirt-dev/invirtibuilder @ 2579

Last change on this file since 2579 was 2577, checked in by broder, 15 years ago

Stuff all of our repos into an /invirt subdirectory, so that we have
room for future expansion.

  • Property svn:executable set to *
File size: 12.5 KB
RevLine 
[2538]1#!/usr/bin/python
2
3"""Process the Invirt build queue.
4
5The Invirtibuilder handles package builds and uploads. On demand, it
6attempts to build a particular package.
7
8If the build succeeds, the new version of the package is uploaded to
9the apt repository, tagged in its git repository, and the Invirt
10superrepo is updated to point at the new version.
11
12If the build fails, the Invirtibuilder sends mail with the build log.
13
14The build queue is tracked via files in /var/lib/invirt-dev/queue. In
15order to maintain ordering, all filenames in that directory are the
16timestamp of their creation time.
17
18Each queue file contains a file of the form
19
20    pocket package hash principal
21
22where pocket is one of the pockets globally configured in
23git.pockets. For instance, the pockets in XVM are "prod" and "dev".
24
25principal is the Kerberos principal that requested the build.
26"""
27
28
[2546]29from __future__ import with_statement
30
[2538]31import contextlib
32import os
33import re
34import shutil
35import subprocess
36
37import pyinotify
38
[2543]39import invirt.builder as b
[2538]40from invirt import database
41
42
43DISTRIBUTION = 'hardy'
44
45
46def getControl(package, ref):
47    """Get the parsed debian/control file for a given package.
48
49    This returns a list of debian_bundle.deb822.Deb822 objects, one
50    for each section of the debian/control file. Each Deb822 object
51    acts roughly like a dict.
52    """
53    return deb822.Deb822.iter_paragraphs(
54        getGitFile(package, ref, 'debian/control').split('\n'))
55
56
57def getBinaries(package, ref):
58    """Get a list of binary packages in a package at a given ref."""
59    return [p['Package'] for p in getControl(package, ref)
60            if 'Package' in p]
61
62
63def getArches(package, ref):
64    """Get the set of all architectures in any binary package."""
65    arches = set()
66    for section in getControl(package, ref):
67        if 'Architecture' in section:
68            arches.update(section['Architecture'].split())
69    return arches
70
71
72def getDscName(package, ref):
73    """Return the .dsc file that will be generated for this package."""
74    v = getVersion(package, ref)
[2544]75    if v.debian_version:
76        v_str = '%s-%s' % (v.upstream_version,
77                           v.debian_version)
78    else:
79        v_str = v.upstream_version
80    return '%s_%s.dsc' % (
[2538]81        package,
[2544]82        v_str)
[2538]83
84
85def sanitizeVersion(version):
86    """Sanitize a Debian package version for use as a git tag.
87
88    This function strips the epoch from the version number and
89    replaces any tildes with periods."""
[2544]90    if v.debian_version:
91        v = '%s-%s' % (version.upstream_version,
92                       version.debian_version)
93    else:
94        v = version.upstream_version
[2538]95    return v.replace('~', '.')
96
97
98def aptCopy(packages, dst_pocket, src_pocket):
99    """Copy a package from one pocket to another."""
[2545]100    binaries = getBinaries(package, commit)
[2538]101    cpatureOutput(['reprepro-env', 'copy',
[2543]102                   b.pocketToApt(dst_pocket),
103                   b.pocketToApt(src_pocket),
[2538]104                   package] + binaries)
105
106
107def sbuild(package, ref, arch, workdir, arch_all=False):
108    """Build a package for a particular architecture."""
109    args = ['sbuild', '-d', DISTRIBUTION, '--arch', arch]
110    if arch_all:
111        args.append('-A')
112    args.append(getDscName(package, ref))
[2543]113    c.captureOutput(args, cwd=workdir, stdout=None)
[2538]114
115
116def sbuildAll(package, ref, workdir):
117    """Build a package for all architectures it supports."""
118    arches = getArches(package, ref)
119    if 'all' in arches or 'any' in arches or 'amd64' in arches:
120        sbuild(package, ref, 'amd64', workdir, arch_all=True)
121    if 'any' in arches or 'i386' in arches:
122        sbuild(package, ref, 'i386', workdir)
123
124
125def tagSubmodule(pocket, package, ref, principal):
126    """Tag a new version of a submodule.
127
128    If this pocket does not allow_backtracking, then this will create
129    a new tag of the version at ref.
130
131    This function doesn't need to care about lock
132    contention. git-receive-pack updates one ref at a time, and only
133    takes out a lock for that ref after it's passed the update
134    hook. Because we reject pushes to tags in the update hook, no push
135    can ever take out a lock on any tags.
136
137    I'm sure that long description gives you great confidence in teh
138    legitimacy of my reasoning.
139    """
140    if config.git.pockets[pocket].get('allow_backtracking', False):
141        env = dict(os.environ)
[2543]142        branch = b.pocketToGit(pocket)
143        version = b.getVersion(package, ref)
[2538]144
145        env['GIT_COMMITTER_NAME'] = config.git.tagger.name
146        env['GIT_COMMITTER_EMAIL'] = config.git.tagger.email
147        tag_msg = ('Tag %s of %s\n\n'
148                   'Requested by %s' % (version.full_version,
149                                        package,
150                                        principal))
151
[2543]152        c.captureOutput(
[2538]153            ['git', 'tag', '-m', tag_msg, commit],
154            stdout=None,
155            env=env)
156
157
158def updateSubmoduleBranch(pocket, package, ref):
159    """Update the appropriately named branch in the submodule."""
[2543]160    branch = b.pocketToGit(pocket)
161    c.captureOutput(
[2538]162        ['git', 'update-ref', 'refs/heads/%s' % branch, ref])
163
164
165def uploadBuild(pocket, workdir):
166    """Upload all build products in the work directory."""
[2543]167    apt = b.pocketToApt(pocket)
[2538]168    for changes in glob.glob(os.path.join(workdir, '*.changes')):
[2543]169        c.captureOutput(['reprepro-env',
[2538]170                       'include',
171                       '--ignore=wrongdistribution',
172                       apt,
173                       changes])
174
175
176def updateSuperrepo(pocket, package, commit, principal):
177    """Update the superrepo.
178
179    This will create a new commit on the branch for the given pocket
180    that sets the commit for the package submodule to commit.
181
182    Note that there's no locking issue here, because we disallow all
183    pushes to the superrepo.
184    """
[2577]185    superrepo = os.path.join(b._REPO_DIR, 'invirt/packages.git')
[2543]186    branch = b.pocketToGit(pocket)
187    tree = c.captureOutput(['git', 'ls-tree', branch],
[2538]188                         cwd=superrepo)
189
190    new_tree = re.compile(
191        r'^(160000 commit )[0-9a-f]*(\t%s)$' % package, re.M).sub(
192        r'\1%s\2' % commit,
193        tree)
194
[2543]195    new_tree_id = c.captureOutput(['git', 'mktree'],
[2538]196                                cwd=superrepo,
197                                stdin_str=new_tree)
198
199    commit_msg = ('Update %s to version %s\n\n'
200                  'Requested by %s' % (package,
201                                       version.full_version,
202                                       principal))
[2543]203    new_commit = c.captureOutput(
[2538]204        ['git', 'commit-tree', new_tree_hash, '-p', branch],
205        cwd=superrepo,
206        env=env,
207        stdin_str=commit_msg)
208
[2543]209    c.captureOutput(
[2538]210        ['git', 'update-ref', 'refs/heads/%s' % branch, new_commit],
211        cwd=superrepo)
212
213
214@contextlib.contextmanager
215def packageWorkdir(package):
216    """Checkout the package in a temporary working directory.
217
218    This context manager returns that working directory. The requested
219    package is checked out into a subdirectory of the working
220    directory with the same name as the package.
221
222    When the context wrapped with this context manager is exited, the
223    working directory is automatically deleted.
224    """
225    workdir = tempfile.mkdtemp()
226    try:
227        p_archive = subprocess.Popen(
228            ['git', 'archive',
[2543]229             '--remote=file://%s' % b.getRepo(package),
[2538]230             '--prefix=%s' % package,
231             commit,
232             ],
233            stdout=subprocess.PIPE,
234            )
235        p_tar = subprocess.Popen(
236            ['tar', '-x'],
237            stdin=p_archive.stdout,
238            cwd=workdir,
239            )
240        p_archive.wait()
241        p_tar.wait()
242
243        yield workdir
244    finally:
245        shutil.rmtree(workdir)
246
247
248def reportBuild(build):
249    """Run hooks to report the results of a build attempt."""
250
[2543]251    c.captureOutput(['run-parts',
[2538]252                   '--arg=%s' % build.build_id,
253                   '--',
[2543]254                   b._HOOKS_DIR])
[2538]255
256
257def build():
258    """Deal with items in the build queue.
259
260    When triggered, iterate over build queue items one at a time,
261    until there are no more pending build jobs.
262    """
263    while True:
264        stage = 'processing incoming job'
[2543]265        queue = os.listdir(b._QUEUE_DIR)
[2538]266        if not queue:
267            break
268
269        build = min(queue)
[2543]270        job = open(os.path.join(b._QUEUE_DIR, build)).read().strip()
[2538]271        pocket, package, commit, principal = job.split()
272
273        database.session.begin()
274        db = database.Build()
275        db.package = package
276        db.pocket = pocket
277        db.commit = commit
278        db.principal = principal
279        database.session.save_or_update(db)
280        database.commit()
281
282        database.begin()
283
284        try:
285            db.failed_stage = 'validating job'
286            src = validateBuild(pocket, package, commit)
287
[2543]288            db.version = str(b.getVersion(package, commit))
[2538]289
290            # If validateBuild returns something other than True, then
291            # it means we should copy from that pocket to our pocket.
292            #
293            # (If the validation failed, validateBuild would have
294            # raised an exception)
295            if src != True:
296                db.failed_stage = 'copying package from another pocket'
297                aptCopy(packages, pocket, src)
298            # If we can't copy the package from somewhere, but
299            # validateBuild didn't raise an exception, then we need to
300            # do the build ourselves
301            else:
302                db.failed_stage = 'checking out package source'
303                with packageWorkdir(package) as workdir:
304                    db.failed_stage = 'preparing source package'
305                    packagedir = os.path.join(workdir, package)
306
307                    # We should be more clever about dealing with
308                    # things like non-Debian-native packages than we
309                    # are.
310                    #
311                    # If we were, we could use debuild and get nice
312                    # environment scrubbing. Since we're not, debuild
313                    # complains about not having an orig.tar.gz
[2543]314                    c.captureOutput(['dpkg-buildpackage', '-us', '-uc', '-S'],
[2538]315                                  cwd=packagedir,
316                                  stdout=None)
317
318                    try:
319                        db.failed_stage = 'building binary packages'
320                        sbuildAll(package, commit, workdir)
321                    finally:
[2543]322                        logdir = os.path.join(b._LOG_DIR, db.build_id)
[2538]323                        if not os.path.exists(logdir):
324                            os.makedirs(logdir)
325
326                        for log in glob.glob(os.path.join(workdir, '*.build')):
327                            os.copy2(log, logdir)
328                    db.failed_stage = 'tagging submodule'
329                    tagSubmodule(pocket, package, commit, principal)
330                    db.failed_stage = 'updating submodule branches'
331                    updateSubmoduleBranch(pocket, package, commit)
332                    db.failed_stage = 'updating superrepo'
333                    updateSuperrepo(pocket, package, commit, principal)
334                    db.failed_stage = 'uploading packages to apt repo'
335                    uploadBuild(pocket, workdir)
336
337                    db.failed_stage = 'cleaning up'
338
339                # Finally, now that everything is done, remove the
340                # build queue item
[2543]341                os.unlink(os.path.join(b._QUEUE_DIR, build))
[2538]342        except:
343            db.traceback = traceback.format_exc()
344        else:
345            db.succeeded = True
346            db.failed_stage = None
347        finally:
348            database.session.save_or_update(db)
349            database.session.commit()
350
351            reportBuild(db)
352
353
354class Invirtibuilder(pyinotify.ProcessEvent):
355    """Process inotify triggers to build new packages."""
356    def process_IN_CREATE(self, event):
357        """Handle a created file or directory.
358
359        When an IN_CREATE event comes in, trigger the builder.
360        """
361        build()
362
363
364def main():
365    """Initialize the inotifications and start the main loop."""
366    database.connect()
367
368    watch_manager = pyinotify.WatchManager()
369    invirtibuilder = Invirtibuilder()
370    notifier = pyinotify.Notifier(watch_manager, invirtibuilder)
[2543]371    watch_manager.add_watch(b._QUEUE_DIR,
[2538]372                            pyinotify.EventsCodes.ALL_FLAGS['IN_CREATE'])
373
374    # Before inotifying, run any pending builds; otherwise we won't
375    # get notified for them.
376    build()
377
378    while True:
379        notifier.process_events()
380        if notifier.check_events():
381            notifier.read_events()
382
383
384if __name__ == '__main__':
385    main()
Note: See TracBrowser for help on using the repository browser.