source: trunk/packages/invirt-dev/invirtibuilder @ 2545

Last change on this file since 2545 was 2545, checked in by broder, 15 years ago

In the invirtibuilder, take advantage of the getBinaries function in
aptCopy.

  • Property svn:executable set to *
File size: 12.4 KB
Line 
1#!/usr/bin/python
2
3"""Process the Invirt build queue.
4
5The Invirtibuilder handles package builds and uploads. On demand, it
6attempts to build a particular package.
7
8If the build succeeds, the new version of the package is uploaded to
9the apt repository, tagged in its git repository, and the Invirt
10superrepo is updated to point at the new version.
11
12If the build fails, the Invirtibuilder sends mail with the build log.
13
14The build queue is tracked via files in /var/lib/invirt-dev/queue. In
15order to maintain ordering, all filenames in that directory are the
16timestamp of their creation time.
17
18Each queue file contains a file of the form
19
20    pocket package hash principal
21
22where pocket is one of the pockets globally configured in
23git.pockets. For instance, the pockets in XVM are "prod" and "dev".
24
25principal is the Kerberos principal that requested the build.
26"""
27
28
29import contextlib
30import os
31import re
32import shutil
33import subprocess
34
35import pyinotify
36
37import invirt.builder as b
38from invirt import database
39
40
41DISTRIBUTION = 'hardy'
42
43
44def getControl(package, ref):
45    """Get the parsed debian/control file for a given package.
46
47    This returns a list of debian_bundle.deb822.Deb822 objects, one
48    for each section of the debian/control file. Each Deb822 object
49    acts roughly like a dict.
50    """
51    return deb822.Deb822.iter_paragraphs(
52        getGitFile(package, ref, 'debian/control').split('\n'))
53
54
55def getBinaries(package, ref):
56    """Get a list of binary packages in a package at a given ref."""
57    return [p['Package'] for p in getControl(package, ref)
58            if 'Package' in p]
59
60
61def getArches(package, ref):
62    """Get the set of all architectures in any binary package."""
63    arches = set()
64    for section in getControl(package, ref):
65        if 'Architecture' in section:
66            arches.update(section['Architecture'].split())
67    return arches
68
69
70def getDscName(package, ref):
71    """Return the .dsc file that will be generated for this package."""
72    v = getVersion(package, ref)
73    if v.debian_version:
74        v_str = '%s-%s' % (v.upstream_version,
75                           v.debian_version)
76    else:
77        v_str = v.upstream_version
78    return '%s_%s.dsc' % (
79        package,
80        v_str)
81
82
83def sanitizeVersion(version):
84    """Sanitize a Debian package version for use as a git tag.
85
86    This function strips the epoch from the version number and
87    replaces any tildes with periods."""
88    if v.debian_version:
89        v = '%s-%s' % (version.upstream_version,
90                       version.debian_version)
91    else:
92        v = version.upstream_version
93    return v.replace('~', '.')
94
95
96def aptCopy(packages, dst_pocket, src_pocket):
97    """Copy a package from one pocket to another."""
98    binaries = getBinaries(package, commit)
99    cpatureOutput(['reprepro-env', 'copy',
100                   b.pocketToApt(dst_pocket),
101                   b.pocketToApt(src_pocket),
102                   package] + binaries)
103
104
105def sbuild(package, ref, arch, workdir, arch_all=False):
106    """Build a package for a particular architecture."""
107    args = ['sbuild', '-d', DISTRIBUTION, '--arch', arch]
108    if arch_all:
109        args.append('-A')
110    args.append(getDscName(package, ref))
111    c.captureOutput(args, cwd=workdir, stdout=None)
112
113
114def sbuildAll(package, ref, workdir):
115    """Build a package for all architectures it supports."""
116    arches = getArches(package, ref)
117    if 'all' in arches or 'any' in arches or 'amd64' in arches:
118        sbuild(package, ref, 'amd64', workdir, arch_all=True)
119    if 'any' in arches or 'i386' in arches:
120        sbuild(package, ref, 'i386', workdir)
121
122
123def tagSubmodule(pocket, package, ref, principal):
124    """Tag a new version of a submodule.
125
126    If this pocket does not allow_backtracking, then this will create
127    a new tag of the version at ref.
128
129    This function doesn't need to care about lock
130    contention. git-receive-pack updates one ref at a time, and only
131    takes out a lock for that ref after it's passed the update
132    hook. Because we reject pushes to tags in the update hook, no push
133    can ever take out a lock on any tags.
134
135    I'm sure that long description gives you great confidence in teh
136    legitimacy of my reasoning.
137    """
138    if config.git.pockets[pocket].get('allow_backtracking', False):
139        env = dict(os.environ)
140        branch = b.pocketToGit(pocket)
141        version = b.getVersion(package, ref)
142
143        env['GIT_COMMITTER_NAME'] = config.git.tagger.name
144        env['GIT_COMMITTER_EMAIL'] = config.git.tagger.email
145        tag_msg = ('Tag %s of %s\n\n'
146                   'Requested by %s' % (version.full_version,
147                                        package,
148                                        principal))
149
150        c.captureOutput(
151            ['git', 'tag', '-m', tag_msg, commit],
152            stdout=None,
153            env=env)
154
155
156def updateSubmoduleBranch(pocket, package, ref):
157    """Update the appropriately named branch in the submodule."""
158    branch = b.pocketToGit(pocket)
159    c.captureOutput(
160        ['git', 'update-ref', 'refs/heads/%s' % branch, ref])
161
162
163def uploadBuild(pocket, workdir):
164    """Upload all build products in the work directory."""
165    apt = b.pocketToApt(pocket)
166    for changes in glob.glob(os.path.join(workdir, '*.changes')):
167        c.captureOutput(['reprepro-env',
168                       'include',
169                       '--ignore=wrongdistribution',
170                       apt,
171                       changes])
172
173
174def updateSuperrepo(pocket, package, commit, principal):
175    """Update the superrepo.
176
177    This will create a new commit on the branch for the given pocket
178    that sets the commit for the package submodule to commit.
179
180    Note that there's no locking issue here, because we disallow all
181    pushes to the superrepo.
182    """
183    superrepo = os.path.join(b._REPO_DIR, 'packages.git')
184    branch = b.pocketToGit(pocket)
185    tree = c.captureOutput(['git', 'ls-tree', branch],
186                         cwd=superrepo)
187
188    new_tree = re.compile(
189        r'^(160000 commit )[0-9a-f]*(\t%s)$' % package, re.M).sub(
190        r'\1%s\2' % commit,
191        tree)
192
193    new_tree_id = c.captureOutput(['git', 'mktree'],
194                                cwd=superrepo,
195                                stdin_str=new_tree)
196
197    commit_msg = ('Update %s to version %s\n\n'
198                  'Requested by %s' % (package,
199                                       version.full_version,
200                                       principal))
201    new_commit = c.captureOutput(
202        ['git', 'commit-tree', new_tree_hash, '-p', branch],
203        cwd=superrepo,
204        env=env,
205        stdin_str=commit_msg)
206
207    c.captureOutput(
208        ['git', 'update-ref', 'refs/heads/%s' % branch, new_commit],
209        cwd=superrepo)
210
211
212@contextlib.contextmanager
213def packageWorkdir(package):
214    """Checkout the package in a temporary working directory.
215
216    This context manager returns that working directory. The requested
217    package is checked out into a subdirectory of the working
218    directory with the same name as the package.
219
220    When the context wrapped with this context manager is exited, the
221    working directory is automatically deleted.
222    """
223    workdir = tempfile.mkdtemp()
224    try:
225        p_archive = subprocess.Popen(
226            ['git', 'archive',
227             '--remote=file://%s' % b.getRepo(package),
228             '--prefix=%s' % package,
229             commit,
230             ],
231            stdout=subprocess.PIPE,
232            )
233        p_tar = subprocess.Popen(
234            ['tar', '-x'],
235            stdin=p_archive.stdout,
236            cwd=workdir,
237            )
238        p_archive.wait()
239        p_tar.wait()
240
241        yield workdir
242    finally:
243        shutil.rmtree(workdir)
244
245
246def reportBuild(build):
247    """Run hooks to report the results of a build attempt."""
248
249    c.captureOutput(['run-parts',
250                   '--arg=%s' % build.build_id,
251                   '--',
252                   b._HOOKS_DIR])
253
254
255def build():
256    """Deal with items in the build queue.
257
258    When triggered, iterate over build queue items one at a time,
259    until there are no more pending build jobs.
260    """
261    while True:
262        stage = 'processing incoming job'
263        queue = os.listdir(b._QUEUE_DIR)
264        if not queue:
265            break
266
267        build = min(queue)
268        job = open(os.path.join(b._QUEUE_DIR, build)).read().strip()
269        pocket, package, commit, principal = job.split()
270
271        database.session.begin()
272        db = database.Build()
273        db.package = package
274        db.pocket = pocket
275        db.commit = commit
276        db.principal = principal
277        database.session.save_or_update(db)
278        database.commit()
279
280        database.begin()
281
282        try:
283            db.failed_stage = 'validating job'
284            src = validateBuild(pocket, package, commit)
285
286            db.version = str(b.getVersion(package, commit))
287
288            # If validateBuild returns something other than True, then
289            # it means we should copy from that pocket to our pocket.
290            #
291            # (If the validation failed, validateBuild would have
292            # raised an exception)
293            if src != True:
294                db.failed_stage = 'copying package from another pocket'
295                aptCopy(packages, pocket, src)
296            # If we can't copy the package from somewhere, but
297            # validateBuild didn't raise an exception, then we need to
298            # do the build ourselves
299            else:
300                db.failed_stage = 'checking out package source'
301                with packageWorkdir(package) as workdir:
302                    db.failed_stage = 'preparing source package'
303                    packagedir = os.path.join(workdir, package)
304
305                    # We should be more clever about dealing with
306                    # things like non-Debian-native packages than we
307                    # are.
308                    #
309                    # If we were, we could use debuild and get nice
310                    # environment scrubbing. Since we're not, debuild
311                    # complains about not having an orig.tar.gz
312                    c.captureOutput(['dpkg-buildpackage', '-us', '-uc', '-S'],
313                                  cwd=packagedir,
314                                  stdout=None)
315
316                    try:
317                        db.failed_stage = 'building binary packages'
318                        sbuildAll(package, commit, workdir)
319                    finally:
320                        logdir = os.path.join(b._LOG_DIR, db.build_id)
321                        if not os.path.exists(logdir):
322                            os.makedirs(logdir)
323
324                        for log in glob.glob(os.path.join(workdir, '*.build')):
325                            os.copy2(log, logdir)
326                    db.failed_stage = 'tagging submodule'
327                    tagSubmodule(pocket, package, commit, principal)
328                    db.failed_stage = 'updating submodule branches'
329                    updateSubmoduleBranch(pocket, package, commit)
330                    db.failed_stage = 'updating superrepo'
331                    updateSuperrepo(pocket, package, commit, principal)
332                    db.failed_stage = 'uploading packages to apt repo'
333                    uploadBuild(pocket, workdir)
334
335                    db.failed_stage = 'cleaning up'
336
337                # Finally, now that everything is done, remove the
338                # build queue item
339                os.unlink(os.path.join(b._QUEUE_DIR, build))
340        except:
341            db.traceback = traceback.format_exc()
342        else:
343            db.succeeded = True
344            db.failed_stage = None
345        finally:
346            database.session.save_or_update(db)
347            database.session.commit()
348
349            reportBuild(db)
350
351
352class Invirtibuilder(pyinotify.ProcessEvent):
353    """Process inotify triggers to build new packages."""
354    def process_IN_CREATE(self, event):
355        """Handle a created file or directory.
356
357        When an IN_CREATE event comes in, trigger the builder.
358        """
359        build()
360
361
362def main():
363    """Initialize the inotifications and start the main loop."""
364    database.connect()
365
366    watch_manager = pyinotify.WatchManager()
367    invirtibuilder = Invirtibuilder()
368    notifier = pyinotify.Notifier(watch_manager, invirtibuilder)
369    watch_manager.add_watch(b._QUEUE_DIR,
370                            pyinotify.EventsCodes.ALL_FLAGS['IN_CREATE'])
371
372    # Before inotifying, run any pending builds; otherwise we won't
373    # get notified for them.
374    build()
375
376    while True:
377        notifier.process_events()
378        if notifier.check_events():
379            notifier.read_events()
380
381
382if __name__ == '__main__':
383    main()
Note: See TracBrowser for help on using the repository browser.