Changeset 2543


Ignore:
Timestamp:
Nov 22, 2009, 4:07:20 PM (15 years ago)
Author:
broder
Message:

Pull functions that are needed for the git remctl scripts out of the
invirtibuilder and into a common module.

Location:
trunk/packages/invirt-dev
Files:
3 added
1 edited

Legend:

Unmodified
Added
Removed

  • trunk/packages/invirt-dev/invirtibuilder

    r2542 r2543  
    3535import pyinotify
    3636
    37 from invirt.config import structs as config
     37import invirt.builder as b
    3838from invirt import database
    3939
    4040
    41 _QUEUE_DIR = '/var/lib/invirt-dev/queue'
    42 _REPO_DIR = '/srv/git'
    43 _LOG_DIR = '/var/log/invirt/builds'
    44 _HOOKS_DIR = '/usr/share/invirt-dev/build.d'
    45 
    46 
    4741DISTRIBUTION = 'hardy'
    48 
    49 
    50 class InvalidBuild(ValueError):
    51     pass
    52 
    53 
    54 def captureOutput(popen_args, stdin_str=None, *args, **kwargs):
    55     """Capture stdout from a command.
    56 
    57     This method will proxy the arguments to subprocess.Popen. It
    58     returns the output from the command if the call succeeded and
    59     raises an exception if the process returns a non-0 value.
    60 
    61     This is intended to be a variant on the subprocess.check_call
    62     function that also allows you access to the output from the
    63     command.
    64     """
    65     if 'stdin' not in kwargs:
    66         kwargs['stdin'] = subprocess.PIPE
    67     if 'stdout' not in kwargs:
    68         kwargs['stdout'] = subprocess.PIPE
    69     if 'stderr' not in kwargs:
    70         kwargs['stderr'] = subprocess.STDOUT
    71     p = subprocess.Popen(popen_args, *args, **kwargs)
    72     out, _ = p.communicate(stdin_str)
    73     if p.returncode:
    74         raise subprocess.CalledProcessError(p.returncode, popen_args, out)
    75     return out
    76 
    77 
    78 def getRepo(package):
    79     """Return the path to the git repo for a given package."""
    80     return os.path.join(_REPO_DIR, 'packages', '%s.git' % package)
    81 
    82 
    83 def pocketToGit(pocket):
    84     """Map a pocket in the configuration to a git branch."""
    85     return config.git.pockets[pocket].get('git', pocket)
    86 
    87 
    88 def pocketToApt(pocket):
    89     """Map a pocket in the configuration to an apt repo pocket."""
    90     return config.git.pockets[pocket].get('apt', pocket)
    91 
    92 
    93 def getGitFile(package, ref, path):
    94     """Return the contents of a path from a git ref in a package."""
    95     return captureOutput(['git', 'cat-file', 'blob', '%s:%s' % (ref, path)],
    96                          cwd=getRepo(package))
    97 
    98 
    99 def getChangelog(package, ref):
    100     """Get a changelog object for a given ref in a given package.
    101 
    102     This returns a debian_bundle.changelog.Changelog object for a
    103     given ref of a given package.
    104     """
    105     return changelog.Changelog(getGitFile(package, ref, 'debian/changelog'))
    106 
    107 
    108 def getVersion(package, ref):
    109     """Get the version of a given package at a particular ref."""
    110     return getChangelog(package, ref).get_version()
    11142
    11243
    … …  
    14677
    14778
    148 def validateBuild(pocket, package, commit):
    149     """Given the parameters of a new build, validate that build.
    150 
    151     The checks this function performs vary based on whether or not the
    152     pocket is configured with allow_backtracking.
    153 
    154     A build of a pocket without allow_backtracking set must be a
    155     fast-forward of the previous revision, and the most recent version
    156     in the changelog most be strictly greater than the version
    157     currently in the repository.
    158 
    159     In all cases, this revision of the package can only have the same
    160     version number as any other revision currently in the apt
    161     repository if they have the same commit ID.
    162 
    163     If it's unspecified, it is assumed that pocket do not
    164     allow_backtracking.
    165 
    166     If this build request fails validation, this function will raise a
    167     InvalidBuild exception, with information about why the validation
    168     failed.
    169 
    170     If this build request can be satisfied by copying the package from
    171     another pocket, then this function returns that pocket. Otherwise,
    172     it returns True.
    173     """
    174     package_repo = getRepo(package)
    175     new_version = getVersion(package, commit)
    176 
    177     for p in config.git.pockets:
    178         if p == pocket:
    179             continue
    180 
    181         b = pocketToGit(p)
    182         current_commit = captureOutput(['git', 'rev-parse', b],
    183                                        cwd=package_repo)
    184         current_version = getVersion(package, b)
    185 
    186         if current_version == new_version:
    187             if current_commit == commit:
    188                 return p
    189             else:
    190                 raise InvalidBuild('Version %s of %s already available in '
    191                                    'pocket %s from commit %s' %
    192                                    (new_version, package, p, current_commit))
    193 
    194     if config.git.pockets[pocket].get('allow_backtracking', False):
    195         branch = pocketToGit(pocket)
    196         current_version = getVersion(package, branch)
    197         if new_version <= current_version:
    198             raise InvalidBuild('New version %s of %s is not newer than '
    199                                'version %s currently in pocket %s' %
    200                                (new_version, package, current_version, pocket))
    201 
    202         # Almost by definition, A is a fast-forward of B if B..A is
    203         # empty
    204         if not captureOutput(['git', 'rev-list', '%s..%s' % (commit, branch)]):
    205             raise InvalidBuild('New commit %s of %s is not a fast-forward of'
    206                                'commit currently in pocket %s' %
    207                                (commit, package, pocket))
    208 
    209 
    21079def sanitizeVersion(version):
    21180    """Sanitize a Debian package version for use as a git tag.
    … …  
    22190    """Copy a package from one pocket to another."""
    22291    binaries = []
    223     for line in getGitFile(package, commit, 'debian/control').split('\n'):
     92    for line in b.getGitFile(package, commit, 'debian/control').split('\n'):
    22493        m = re.match('Package: (.*)$')
    22594        if m:
    … …  
    22796
    22897    cpatureOutput(['reprepro-env', 'copy',
    229                    pocketToApt(dst_pocket),
    230                    pocketToApt(src_pocket),
     98                   b.pocketToApt(dst_pocket),
     99                   b.pocketToApt(src_pocket),
    231100                   package] + binaries)
    232101
    … …  
    238107        args.append('-A')
    239108    args.append(getDscName(package, ref))
    240     captureOutput(args, cwd=workdir, stdout=None)
     109    c.captureOutput(args, cwd=workdir, stdout=None)
    241110
    242111
    … …  
    267136    if config.git.pockets[pocket].get('allow_backtracking', False):
    268137        env = dict(os.environ)
    269         branch = pocketToGit(pocket)
    270         version = getVersion(package, ref)
     138        branch = b.pocketToGit(pocket)
     139        version = b.getVersion(package, ref)
    271140
    272141        env['GIT_COMMITTER_NAME'] = config.git.tagger.name
    … …  
    277146                                        principal))
    278147
    279         captureOutput(
     148        c.captureOutput(
    280149            ['git', 'tag', '-m', tag_msg, commit],
    281150            stdout=None,
    … …  
    285154def updateSubmoduleBranch(pocket, package, ref):
    286155    """Update the appropriately named branch in the submodule."""
    287     branch = pocketToGit(pocket)
    288     captureOutput(
     156    branch = b.pocketToGit(pocket)
     157    c.captureOutput(
    289158        ['git', 'update-ref', 'refs/heads/%s' % branch, ref])
    290159
    … …  
    292161def uploadBuild(pocket, workdir):
    293162    """Upload all build products in the work directory."""
    294     apt = pocketToApt(pocket)
     163    apt = b.pocketToApt(pocket)
    295164    for changes in glob.glob(os.path.join(workdir, '*.changes')):
    296         captureOutput(['reprepro-env',
     165        c.captureOutput(['reprepro-env',
    297166                       'include',
    298167                       '--ignore=wrongdistribution',
    … …  
    310179    pushes to the superrepo.
    311180    """
    312     superrepo = os.path.join(_REPO_DIR, 'packages.git')
    313     branch = pocketToGit(pocket)
    314     tree = captureOutput(['git', 'ls-tree', branch],
     181    superrepo = os.path.join(b._REPO_DIR, 'packages.git')
     182    branch = b.pocketToGit(pocket)
     183    tree = c.captureOutput(['git', 'ls-tree', branch],
    315184                         cwd=superrepo)
    316185
    … …  
    320189        tree)
    321190
    322     new_tree_id = captureOutput(['git', 'mktree'],
     191    new_tree_id = c.captureOutput(['git', 'mktree'],
    323192                                cwd=superrepo,
    324193                                stdin_str=new_tree)
    … …  
    328197                                       version.full_version,
    329198                                       principal))
    330     new_commit = captureOutput(
     199    new_commit = c.captureOutput(
    331200        ['git', 'commit-tree', new_tree_hash, '-p', branch],
    332201        cwd=superrepo,
    … …  
    334203        stdin_str=commit_msg)
    335204
    336     captureOutput(
     205    c.captureOutput(
    337206        ['git', 'update-ref', 'refs/heads/%s' % branch, new_commit],
    338207        cwd=superrepo)
    … …  
    354223        p_archive = subprocess.Popen(
    355224            ['git', 'archive',
    356              '--remote=file://%s' % getRepo(package),
     225             '--remote=file://%s' % b.getRepo(package),
    357226             '--prefix=%s' % package,
    358227             commit,
    … …  
    376245    """Run hooks to report the results of a build attempt."""
    377246
    378     captureOutput(['run-parts',
     247    c.captureOutput(['run-parts',
    379248                   '--arg=%s' % build.build_id,
    380249                   '--',
    381                    _HOOKS_DIR])
     250                   b._HOOKS_DIR])
    382251
    383252
    … …  
    390259    while True:
    391260        stage = 'processing incoming job'
    392         queue = os.listdir(_QUEUE_DIR)
     261        queue = os.listdir(b._QUEUE_DIR)
    393262        if not queue:
    394263            break
    395264
    396265        build = min(queue)
    397         job = open(os.path.join(_QUEUE_DIR, build)).read().strip()
     266        job = open(os.path.join(b._QUEUE_DIR, build)).read().strip()
    398267        pocket, package, commit, principal = job.split()
    399268
    … …  
    413282            src = validateBuild(pocket, package, commit)
    414283
    415             db.version = str(getVersion(package, commit))
     284            db.version = str(b.getVersion(package, commit))
    416285
    417286            # If validateBuild returns something other than True, then
    … …  
    439308                    # environment scrubbing. Since we're not, debuild
    440309                    # complains about not having an orig.tar.gz
    441                     captureOutput(['dpkg-buildpackage', '-us', '-uc', '-S'],
     310                    c.captureOutput(['dpkg-buildpackage', '-us', '-uc', '-S'],
    442311                                  cwd=packagedir,
    443312                                  stdout=None)
    … …  
    447316                        sbuildAll(package, commit, workdir)
    448317                    finally:
    449                         logdir = os.path.join(_LOG_DIR, db.build_id)
     318                        logdir = os.path.join(b._LOG_DIR, db.build_id)
    450319                        if not os.path.exists(logdir):
    451320                            os.makedirs(logdir)
    … …  
    466335                # Finally, now that everything is done, remove the
    467336                # build queue item
    468                 os.unlink(os.path.join(_QUEUE_DIR, build))
     337                os.unlink(os.path.join(b._QUEUE_DIR, build))
    469338        except:
    470339            db.traceback = traceback.format_exc()
    … …  
    496365    invirtibuilder = Invirtibuilder()
    497366    notifier = pyinotify.Notifier(watch_manager, invirtibuilder)
    498     watch_manager.add_watch(_QUEUE_DIR,
     367    watch_manager.add_watch(b._QUEUE_DIR,
    499368                            pyinotify.EventsCodes.ALL_FLAGS['IN_CREATE'])
    500369
Note: See TracChangeset for help on using the changeset viewer.