# Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. # The RdoInfoDriver provides the following: # # 1- A getpackages function based on output provided by rdoinfo # (https://github.com/redhat-openstack/rdoinfo) # # 2- A getinfo function based on a multi-distgit repo paradigm # # 3- A preprocess function that is only used if the distgit directory # containts a .spec.j2 file from dlrn.db import Commit from dlrn.drivers.pkginfo import PkgInfoDriver from dlrn.repositories import getdistrobranch from dlrn.repositories import getsourcebranch from dlrn.repositories import refreshrepo from dlrn.utils import run_external_preprocess import logging import os import sh from distroinfo import query from distroinfo import info logger = logging.getLogger("dlrn-rdoinfo-driver") rdoinfo_repo = ('https://raw.githubusercontent.com/' 'redhat-openstack/rdoinfo/master/') def buildtagsonly(package): return ('tags' in package and package['tags'] is not None and 'build-tags-only' in package['tags'] or 'build-tags-only' in package) class RdoInfoDriver(PkgInfoDriver): DRIVER_CONFIG = { 'rdoinfo_driver': { 'rdoinfo_repo': {'name': 'repo'}, 'rdoinfo_file': {'name': 'info_files', 'type': 'list', 'default': ['rdo.yml']}, 'cache_dir': {}, } } def __init__(self, *args, **kwargs): super(RdoInfoDriver, self).__init__(*args, **kwargs) self.distroinfo_path = None def getpackages(self, **kwargs): """ Valid parameters: :param local_info_repo: local rdoinfo repo to use instead of fetching the default one using rdopkg. :param tags: OpenStack release tags to use (mitaka, newton, etc). """ local_info_repo = kwargs.get('local_info_repo') tags = kwargs.get('tags') inforepo = None info_files = self.config_options.rdoinfo_file if local_info_repo: inforepo = info.DistroInfo( info_files=self.config_options.rdoinfo_file, local_info=local_info_repo, cache_base_path=self.config_options.cache_dir) # NOTE(jpena): in general, info_files will only contain one file, # but it supports multiple... In that case, we will have a comma # separated list of URLs self.distroinfo_path = "%s/%s" % (local_info_repo.rstrip('/'), info_files[0]) for extra_file in info_files[1:]: self.distroinfo_path += ",%s/%s" % ( local_info_repo.rstrip('/')) elif self.config_options.rdoinfo_repo: inforepo = info.DistroInfo( info_files=self.config_options.rdoinfo_file, remote_git_info=self.config_options.rdoinfo_repo, cache_base_path=self.config_options.cache_dir) self.distroinfo_path = "%s/%s" % ( self.config_options.rdoinfo_repo.rstrip('/'), info_files[0]) for extra_file in info_files[1:]: self.distroinfo_path += ",%s/%s" % ( self.config_options.rdoinfo_repo.rstrip('/')) else: # distroinfo will fetch info files from the rdoinfo repo as needed # and store them under ~/.distroinfo/cache inforepo = info.DistroInfo( info_files=self.config_options.rdoinfo_file, remote_info=rdoinfo_repo, cache_base_path=self.config_options.cache_dir) self.distroinfo_path = "%s/%s" % (rdoinfo_repo.rstrip('/'), info_files[0]) for extra_file in info_files[1:]: self.distroinfo_path += ",%s/%s" % ( rdoinfo_repo.rstrip('/')) pkginfo = inforepo.get_info(apply_tag=tags) self.packages = pkginfo["packages"] if tags: # FIXME allow list of tags? self.packages = query.filter_pkgs(self.packages, {'tags': tags}) return self.packages def getinfo(self, **kwargs): project = kwargs.get('project') package = kwargs.get('package') since = kwargs.get('since') local = kwargs.get('local') dev_mode = kwargs.get('dev_mode') datadir = self.config_options.datadir repo = package['upstream'] distro = package['master-distgit'] tags_only = buildtagsonly(package) distro_dir = self._distgit_clone_dir(package['name']) distro_dir_full = self.distgit_dir(package['name']) distro_branch = getdistrobranch(package, default_branch=self.config_options.distro) source_branch = getsourcebranch(package, default_branch=self.config_options.source) if dev_mode is False: try: distro_branch, distro_hash, dt_distro = refreshrepo( distro, distro_dir, self.config_options, distro_branch, local=local, full_path=distro_dir_full) except Exception: # The error was already logged by refreshrepo, and we want # to avoid halting the whole run because this distgit repo # failed, so return an empty list return PkgInfoDriver.Info([], True) else: distro_hash = "dev" dt_distro = 0 # Doesn't get used in dev mode if not os.path.isdir(distro_dir): # We should fail in this case, since we are running # in dev mode, so no try/except refreshrepo(distro, distro_dir, self.config_options, distro_branch, local=local, full_path=distro_dir_full) # repo is usually a string, but if it contains more then one entry we # git clone into a project subdirectory repos = [repo] if isinstance(repo, list): repos = repo project_toprocess = [] for repo in repos: repo_dir = os.path.join(datadir, project) if len(repos) > 1: repo_dir = os.path.join(repo_dir, os.path.split(repo)[1]) try: source_branch, _, _ = refreshrepo(repo, repo_dir, self.config_options, source_branch, local=local) except Exception: # The error was already logged by refreshrepo, and the only # side-effect is that we are not adding this commit to the # list of commits to be processed, so we can ignore it and # move on to the next repo return PkgInfoDriver.Info([], True) git = sh.git.bake(_cwd=repo_dir, _tty_out=False) # Git gives us commits already sorted in the right order if tags_only is True: logger.info('Building tags only for %s' % project) if since == '-1': # we need 2 entries as HEAD will be listed too since = '-2' lines = filter( lambda x: x.find('tag: ') >= 0, git.log('--simplify-by-decoration', "--pretty=format:'%ct %H %d'", since, "--first-parent", "--reverse", "%s" % source_branch)) else: lines = git.log("--pretty=format:'%ct %H'", since, "--first-parent", "--reverse") for line in lines: dt, commit_hash = str(line).strip().strip("'").split(" ")[:2] if self.config_options.use_components and 'component' in package: component = package['component'] else: component = None commit = Commit(dt_commit=float(dt), project_name=project, type="rpm", commit_hash=commit_hash, repo_dir=repo_dir, distro_hash=distro_hash, dt_distro=dt_distro, distgit_dir=self.distgit_dir(package['name']), commit_branch=source_branch, dt_extended=0, extended_hash=None, component=component) project_toprocess.append(commit) # Prepare if needed spec file from jinja file. self._distgit_setup(project) return PkgInfoDriver.Info(project_toprocess, False) def preprocess(self, **kwargs): # Pre-processing is only required if we have a jinja2 spec template package_name = kwargs.get('package_name') commit_hash = kwargs.get('commit_hash') distgit_dir = self.distgit_dir(package_name) source_dir = "%s/%s" % (self.config_options.datadir, package_name) for custom_preprocess in self.config_options.custom_preprocess: if custom_preprocess != '': run_external_preprocess( cmdline=custom_preprocess, pkgname=package_name, distgit=distgit_dir, distroinfo=self.distroinfo_path, source_dir=source_dir, commit_hash=commit_hash) return def _distgit_setup(self, package_name): distgit_dir = self.distgit_dir(package_name) # Now, try to check if we need to run a render job preprocess_needed = False for f in os.listdir(distgit_dir): if f.endswith('.spec.j2'): # We have a template here, so we have to preprocess preprocess_needed = True break if preprocess_needed: logger.info('Pre-processing template at %s' % distgit_dir) renderspec = sh.renderspec.bake(_cwd=distgit_dir, _tty_out=False, _timeout=3600) renderspec('--spec-style', 'fedora', '--epoch', '../../epoch/fedora.yaml') return def distgit_dir(self, package_name): datadir = self.config_options.datadir # Find extra directory inside it, if needed extra_dir = '/' for package in self.packages: if package['name'] == package_name: if 'distgit-path' in package: extra_dir = package['distgit-path'] break return os.path.join(datadir, package_name + "_distro", extra_dir.lstrip('/')) def _distgit_clone_dir(self, package_name): datadir = self.config_options.datadir return os.path.join(datadir, package_name + "_distro")