From df761bd55289800034fbca439f00ef874e5902de Mon Sep 17 00:00:00 2001 From: Daniel Berteaud Date: Mon, 26 Nov 2018 22:40:07 +0100 Subject: [PATCH] Add latest upstream builder to add support for GitLfsBuilder --- fws_tito/builder/__init__.py | 18 + fws_tito/builder/fetch.py | 212 +++++++ fws_tito/builder/main.py | 1374 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 1604 insertions(+) create mode 100644 fws_tito/builder/__init__.py create mode 100644 fws_tito/builder/fetch.py create mode 100644 fws_tito/builder/main.py diff --git a/fws_tito/builder/__init__.py b/fws_tito/builder/__init__.py new file mode 100644 index 0000000..d55f3cc --- /dev/null +++ b/fws_tito/builder/__init__.py @@ -0,0 +1,18 @@ +# Import our builders so they can be referenced in config as tito.builder.Class +# regardless of which submodule they're in. + +# flake8: noqa + +from fws_tito.builder.main import \ + Builder, \ + NoTgzBuilder, \ + GemBuilder, \ + UpstreamBuilder, \ + SatelliteBuilder, \ + MockBuilder, \ + BrewDownloadBuilder, \ + GitAnnexBuilder, \ + GitLfsBuilder, \ + MeadBuilder + +from fws_tito.builder.fetch import FetchBuilder diff --git a/fws_tito/builder/fetch.py b/fws_tito/builder/fetch.py new file mode 100644 index 0000000..b21e08f --- /dev/null +++ b/fws_tito/builder/fetch.py @@ -0,0 +1,212 @@ +# Copyright (c) 2008-2014 Red Hat, Inc. +# +# This software is licensed to you under the GNU General Public License, +# version 2 (GPLv2). There is NO WARRANTY for this software, express or +# implied, including the implied warranties of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2 +# along with this software; if not, see +# http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. +# +# Red Hat trademarks are not licensed under GPLv2. No permission is +# granted to use or replicate Red Hat trademarks that are incorporated +# in this software or its documentation. + +import re +import os.path +import shutil + +from tito.builder.main import BuilderBase +from tito.config_object import ConfigObject +from tito.common import error_out, debug, get_spec_version_and_release, \ + get_class_by_name + + +class FetchBuilder(ConfigObject, BuilderBase): + """ + A separate Builder class for projects whose source is not in git. Source + is fetched via a configurable strategy, which also determines what version + and release to insert into the spec file. + + Cannot build past tags. + """ + # TODO: test only for now, setup a tagger to fetch sources and store in git annex, + # then we can do official builds as well. + REQUIRED_ARGS = [] + + def __init__(self, name=None, tag=None, build_dir=None, + config=None, user_config=None, + args=None, **kwargs): + + BuilderBase.__init__(self, name=name, build_dir=build_dir, + config=config, + user_config=user_config, args=args, **kwargs) + + if tag: + error_out("FetchBuilder does not support building specific tags.") + + if not config.has_option("builder", + "fetch_strategy"): + print("WARNING: no fetch_strategy specified in tito.props" + ", assuming ArgSourceStrategy.") + if not config.has_section("builder"): + config.add_section("builder") + config.set('builder', 'fetch_strategy', + 'tito.builder.fetch.ArgSourceStrategy') + + self.build_tag = '%s-%s' % (self.project_name, + get_spec_version_and_release(self.start_dir, '%s.spec' % self.project_name)) + + def tgz(self): + self.ran_tgz = True + self._create_build_dirs() + + print("Fetching sources...") + source_strat_class = get_class_by_name(self.config.get( + 'builder', 'fetch_strategy')) + source_strat = source_strat_class(self) + source_strat.fetch() + self.sources = source_strat.sources + self.spec_file = source_strat.spec_file + + def _get_rpmbuild_dir_options(self): + return ('--define "_topdir %s" --define "_sourcedir %s" --define "_builddir %s" ' + '--define "_srcrpmdir %s" --define "_rpmdir %s" ' % ( + self.rpmbuild_dir, + self.rpmbuild_sourcedir, self.rpmbuild_builddir, + self.rpmbuild_basedir, self.rpmbuild_basedir)) + + +class SourceStrategy(object): + """ + Base class for source strategies. These are responsible for fetching the + sources the builder will use, and determining what version/release we're + building. + + This is created and run in the tgz step of the builder. It will be passed + a reference to the builder calling it, which will be important for accessing + a lot of required information. + + Ideally sources and the spec file to be used should be copied into + builder.rpmbuild_sourcedir, which will be cleaned up automatically after + the builder runs. + """ + def __init__(self, builder): + """ + Defines fields that should be set when a sub-class runs fetch. + """ + self.builder = builder + + # Full path to the spec file we'll actually use to build, should be a + # copy, never a live spec file in a git repo as sometimes it will be + # modified: + self.spec_file = None + + # Will contain the full path to each source we gather: + self.sources = [] + + # The version we're building: + self.version = None + + # The release we're building: + self.release = None + + def fetch(self): + raise NotImplementedError() + + +class ArgSourceStrategy(SourceStrategy): + """ + Assumes the builder was passed an explicit argument specifying which source + file(s) to use. + """ + def fetch(self): + + # Assuming we're still in the start directory, get the absolute path + # to all sources specified: + # TODO: support passing of multiple sources here. + # TODO: error out if not present + manual_sources = [self.builder.args['source'][0]] + debug("Got sources: %s" % manual_sources) + + # Copy the live spec from our starting location. Unlike most builders, + # we are not using a copy from a past git commit. + self.spec_file = os.path.join(self.builder.rpmbuild_sourcedir, + '%s.spec' % self.builder.project_name) + shutil.copyfile( + os.path.join(self.builder.start_dir, '%s.spec' % + self.builder.project_name), + self.spec_file) + print(" %s.spec" % self.builder.project_name) + + # TODO: Make this a configurable strategy: + i = 0 + replacements = [] + for s in manual_sources: + base_name = os.path.basename(s) + dest_filepath = os.path.join(self.builder.rpmbuild_sourcedir, + base_name) + shutil.copyfile(s, dest_filepath) + self.sources.append(dest_filepath) + + # Add a line to replace in the spec for each source: + source_regex = re.compile("^(source%s:\s*)(.+)$" % i, re.IGNORECASE) + new_line = "Source%s: %s\n" % (i, base_name) + replacements.append((source_regex, new_line)) + + # Replace version and release in spec: + version_regex = re.compile("^(version:\s*)(.+)$", re.IGNORECASE) + release_regex = re.compile("^(release:\s*)(.+)$", re.IGNORECASE) + + (self.version, self.release) = self._get_version_and_release() + print("Building version: %s" % self.version) + print("Building release: %s" % self.release) + replacements.append((version_regex, "Version: %s\n" % self.version)) + replacements.append((release_regex, "Release: %s\n" % self.release)) + + self.replace_in_spec(replacements) + + def _get_version_and_release(self): + """ + Get the version and release from the builder. + Sources are configured at this point. + """ + # Assuming source0 is a tar.gz we can extract a version and possibly + # release from: + base_name = os.path.basename(self.sources[0]) + debug("Extracting version/release from: %s" % base_name) + + # usually a source tarball won't have a release, that is an RPM concept. + # Don't forget dist! + release = "1%{?dist}" + + # Example filename: tito-0.4.18.tar.gz: + simple_version_re = re.compile(".*-(.*).(tar.gz|tgz|zip|bz2)") + match = re.search(simple_version_re, base_name) + if match: + version = match.group(1) + else: + error_out("Unable to determine version from file: %s" % base_name) + + return (version, release) + + def replace_in_spec(self, replacements): + """ + Replace lines in the spec file using the given replacements. + + Replacements are a tuple of a regex to look for, and a new line to + substitute in when the regex matches. + + Replaces all lines with one pass through the file. + """ + in_f = open(self.spec_file, 'r') + out_f = open(self.spec_file + ".new", 'w') + for line in in_f.readlines(): + for line_regex, new_line in replacements: + match = re.match(line_regex, line) + if match: + line = new_line + out_f.write(line) + + in_f.close() + out_f.close() + shutil.move(self.spec_file + ".new", self.spec_file) diff --git a/fws_tito/builder/main.py b/fws_tito/builder/main.py new file mode 100644 index 0000000..daa2d83 --- /dev/null +++ b/fws_tito/builder/main.py @@ -0,0 +1,1374 @@ +# Copyright (c) 2008-2011 Red Hat, Inc. +# +# This software is licensed to you under the GNU General Public License, +# version 2 (GPLv2). There is NO WARRANTY for this software, express or +# implied, including the implied warranties of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2 +# along with this software; if not, see +# http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. +# +# Red Hat trademarks are not licensed under GPLv2. No permission is +# granted to use or replicate Red Hat trademarks that are incorporated +# in this software or its documentation. +""" +Tito builders for a variety of common methods of building sources, srpms, +and rpms. +""" + +import gzip +import os +import sys +import re +import shutil +import rpm +from pkg_resources import require +from distutils.version import LooseVersion as loose_version +from tempfile import mkdtemp + +from tito.common import scl_to_rpm_option, get_latest_tagged_version, \ + find_wrote_in_rpmbuild_output, debug, error_out, run_command_print, \ + find_spec_file, run_command, get_build_commit, get_relative_project_dir, \ + get_relative_project_dir_cwd, get_spec_version_and_release, \ + check_tag_exists, create_tgz, get_latest_commit, \ + get_commit_count, find_gemspec_file, create_builder, compare_version,\ + find_cheetah_template_file, render_cheetah, replace_spec_release, \ + find_spec_like_file, warn_out, get_commit_timestamp, chdir, mkdir_p, \ + find_git_root, info_out, munge_specfile, BUILDCONFIG_SECTION +from tito.compat import getstatusoutput +from tito.exception import RunCommandException +from tito.exception import TitoException +from tito.config_object import ConfigObject +from tito.tar import TarFixer + + +class BuilderBase(object): + """ + A base class for all builders. + + Handles things we will *always* do, primarily handling temporary directories + for rpmbuild. + + This class should *not* assume we're even using git. + """ + REQUIRED_ARGS = [] + + # TODO: merge config into an object and kill the ConfigObject parent class + def __init__(self, name=None, build_dir=None, + config=None, user_config=None, + args=None, **kwargs): + + # Project directory where we started this build: + self.start_dir = os.getcwd() + + self.project_name = name + self.user_config = user_config + self.args = args + self.kwargs = kwargs + self.config = config + + # Optional keyword arguments: + self.dist = self._get_optional_arg(kwargs, 'dist', None) + + self.offline = self._get_optional_arg(kwargs, 'offline', False) + self.auto_install = self._get_optional_arg(kwargs, 'auto_install', + False) + self.escalate_privileges = self._get_optional_arg(kwargs, 'escalate', + True) + self.scl = self._get_optional_arg(args, 'scl', [None])[0] or \ + self._get_optional_arg(kwargs, 'scl', '') + + self.quiet = self._get_optional_arg(kwargs, 'quiet', False) + self.verbose = self._get_optional_arg(kwargs, 'verbose', False) + + rpmbuildopts = self._get_optional_arg(args, 'rpmbuild_options', None) + if rpmbuildopts: + self.rpmbuild_options = ' '.join(rpmbuildopts) + else: + self.rpmbuild_options = self._get_optional_arg(kwargs, 'rpmbuild_options', '') + + self.test = self._get_optional_arg(kwargs, 'test', False) + # Allow a builder arg to override the test setting passed in, used by + # releasers in their config sections. + if args and 'test' in args: + self.test = True + + # Location where we do all tito work and store resulting rpms: + self.rpmbuild_basedir = build_dir + # Location where we do actual rpmbuilds + self.rpmbuild_dir = mkdtemp(dir=self.rpmbuild_basedir, + prefix="rpmbuild-%s" % self.project_name) + debug("Building in temp dir: %s" % self.rpmbuild_dir) + self.rpmbuild_sourcedir = os.path.join(self.rpmbuild_dir, "SOURCES") + self.rpmbuild_builddir = os.path.join(self.rpmbuild_dir, "BUILD") + + self._check_required_args() + + # Set to true once we've created/setup sources: (i.e. tar.gz) + self.ran_tgz = False + + self.no_cleanup = False + + # List of full path to all sources for this package. + self.sources = [] + + # Artifacts we built: + self.artifacts = [] + + # Use most suitable package manager for current OS + self.package_manager = package_manager() + + def _get_optional_arg(self, kwargs, arg, default): + """ + Return the value of an optional keyword argument if it's present, + otherwise the default provided. + """ + if arg in kwargs: + return kwargs[arg] + return default + + def _check_required_args(self): + for arg in self.REQUIRED_ARGS: + if arg not in self.args: + raise TitoException("Builder missing required argument: %s" % + arg) + + def run(self, options): + """ + Perform the actions requested of the builder. + + NOTE: this method may do nothing if the user requested no build actions + be performed. (i.e. only release tagging, etc) + """ + info_out("Building package [%s]" % (self.build_tag)) + self.no_cleanup = options.no_cleanup + + # Reset list of artifacts on each call to run(). + self.artifacts = [] + + try: + try: + if options.tgz: + self.tgz() + if options.srpm: + self.srpm() + if options.rpm: + # TODO: not protected anymore + self.rpm() + self._auto_install() + except KeyboardInterrupt: + print("Interrupted, cleaning up...") + finally: + self.cleanup() + + return self.artifacts + + def cleanup(self): + """ + Remove all temporary files and directories. + """ + if not self.no_cleanup: + debug("Cleaning up %s" % self.rpmbuild_dir) + shutil.rmtree(self.rpmbuild_dir) + else: + warn_out("Leaving rpmbuild files in: %s" % self.rpmbuild_dir) + + def _check_build_dirs_access(self, build_dirs): + """ + Ensure the build directories are writable. + """ + msgs = [] + for d in build_dirs: + if not os.access(d, os.W_OK): + msgs.append("%s is not writable." % d) + if msgs: + error_out(msgs) + + def _create_build_dirs(self): + """ + Create the build directories. Can safely be called multiple times. + """ + build_dirs = [ + self.rpmbuild_basedir, + self.rpmbuild_dir, + self.rpmbuild_sourcedir, + self.rpmbuild_builddir, + ] + for d in build_dirs: + mkdir_p(d) + self._check_build_dirs_access(build_dirs) + + def srpm(self, dist=None): + """ + Build a source RPM. + """ + self._create_build_dirs() + if not self.ran_tgz: + self.tgz() + + if self.test: + self._setup_test_specfile() + + debug("Creating srpm from spec file: %s" % self.spec_file) + define_dist = "" + if self.dist: + debug("using self.dist: %s" % self.dist) + define_dist = "--define 'dist %s'" % self.dist + elif dist: + debug("using dist: %s" % dist) + define_dist = "--define 'dist %s'" % dist + else: + debug("*NOT* using dist at all") + + rpmbuild_options = self.rpmbuild_options + self._scl_to_rpmbuild_option() + + cmd = ('rpmbuild --define "_source_filedigest_algorithm md5" --define' + ' "_binary_filedigest_algorithm md5" %s %s %s --nodeps -bs %s' % ( + rpmbuild_options, self._get_rpmbuild_dir_options(), + define_dist, self.spec_file)) + run_command_func = run_command if self.quiet else run_command_print + output = run_command_func(cmd) + self.srpm_location = find_wrote_in_rpmbuild_output(output)[0] + self.artifacts.append(self.srpm_location) + + # Assume that if tito's --no-cleanup option is set, also disable %clean in rpmbuild: + def _get_clean_option(self): + if self.no_cleanup: + output = run_command('rpmbuild --help') + if '--noclean' in output: + return "--noclean" + else: + return "" + else: + return "--clean" + + def _get_verbosity_option(self): + if self.verbose: + return "--verbose" + else: + return "" + + def rpm(self): + """ Build an RPM. """ + self._create_build_dirs() + if not self.ran_tgz: + self.tgz() + + cmd = 'rpmbuild {0}'.format( + " ".join([ + '--define "_source_filedigest_algorithm md5"', + '--define "_binary_filedigest_algorithm md5"', + self.rpmbuild_options, + self._scl_to_rpmbuild_option(), + self._get_rpmbuild_dir_options(), + "--define 'dist {0}'".format(self.dist) if self.dist else "", + self._get_clean_option(), + self._get_verbosity_option(), + '-ba {0}'.format(self.spec_file), + ]) + ) + try: + if self.quiet: + output = run_command(cmd) + else: + output = run_command_print(cmd) + except (KeyboardInterrupt, SystemExit): + print("") + exit(1) + except RunCommandException: + err = sys.exc_info()[1] + msg = str(err) + if re.search('Failed build dependencies', err.output): + cmd = self.package_manager.builddep(find_spec_file(self.relative_project_dir)) + msg = "Please run '%s' as root." % cmd + error_out('%s' % msg) + except Exception: + err = sys.exc_info()[1] + error_out('%s' % str(err)) + files_written = find_wrote_in_rpmbuild_output(output) + if len(files_written) < 2: + error_out("Error parsing rpmbuild output") + self.srpm_location = files_written[0] + self.artifacts.extend(files_written) + + print + info_out("Successfully built: %s" % '\n\t- '.join(files_written)) + + def _scl_to_rpmbuild_option(self): + """ Returns rpmbuild option which disable or enable SC and print warning if needed """ + return scl_to_rpm_option(self.scl) + + def _auto_install(self): + """ + If requested, auto install the RPMs we just built. + """ + if self.auto_install: + print + print("Auto-installing packages:") + print + + dont_install = [] + if 'NO_AUTO_INSTALL' in self.user_config: + dont_install = self.user_config['NO_AUTO_INSTALL'].split(" ") + debug("Will not auto-install any packages matching: %s" % dont_install) + + do_install = [] + for to_inst in self.artifacts: + # Only install rpms: + if not to_inst.endswith(".rpm") or to_inst.endswith(".src.rpm"): + continue + + install = True + for skip in dont_install: + if skip in to_inst: + install = False + print("Skipping: %s" % to_inst) + break + if install: + do_install.append(to_inst) + + print + reinstall = self.package_manager.is_installed(self.project_name, self.build_version) + cmd = self.package_manager.install(do_install, reinstall=reinstall, auto=True, offline=True, + escalate=self.escalate_privileges) + print("%s" % cmd) + try: + run_command_print(cmd) + print + except KeyboardInterrupt: + pass + + +class Builder(ConfigObject, BuilderBase): + """ + Parent builder class. + + Includes functionality for a standard Spacewalk package build. Packages + which require other unusual behavior can subclass this to inject the + desired behavior. + """ + REQUIRED_ARGS = [] + + # TODO: drop version + def __init__(self, name=None, tag=None, build_dir=None, + config=None, user_config=None, + args=None, **kwargs): + + """ + name - Package name that is being built. + + version - Version and release being built. + + tag - The git tag being built. + + build_dir - Temporary build directory where we can safely work. + + config - Merged configuration. (global plus package specific) + + user_config - User configuration from ~/.titorc. + + args - Optional arguments specific to each builder. Can be passed + in explicitly by user on the CLI, or via a release target config + entry. Only for things which vary on invocations of the builder, + avoid using these if possible. *Given in the format of a dictionary + of lists.* + """ + ConfigObject.__init__(self, config=config) + BuilderBase.__init__(self, name=name, build_dir=build_dir, config=config, + user_config=user_config, args=args, **kwargs) + self.build_tag = tag + + self.build_version = self._get_build_version() + + if kwargs and 'options' in kwargs: + warn_out("'options' no longer a supported builder constructor argument.") + + if self.config.has_section("requirements"): + if self.config.has_option("requirements", "tito"): + if loose_version(self.config.get("requirements", "tito")) > \ + loose_version(require('tito')[0].version): + error_out([ + "tito version %s or later is needed to build this project." % + self.config.get("requirements", "tito"), + "Your version: %s" % require('tito')[0].version + ]) + + self.display_version = self._get_display_version() + + with chdir(find_git_root()): + self.git_commit_id = get_build_commit(tag=self.build_tag, + test=self.test) + + self.relative_project_dir = get_relative_project_dir( + project_name=self.project_name, commit=self.git_commit_id) + if self.relative_project_dir is None and self.test: + warn_out(".tito/packages/%s doesn't exist " + "in git, using current directory" % self.project_name) + self.relative_project_dir = get_relative_project_dir_cwd( + self.git_root) + + tgz_base = self._get_tgz_name_and_ver() + self.tgz_filename = tgz_base + ".tar.gz" + self.tgz_dir = tgz_base + self.artifacts = [] + + # A copy of the git code from commit we're building: + self.rpmbuild_gitcopy = os.path.join(self.rpmbuild_sourcedir, + self.tgz_dir) + + # Used to make sure we only modify the spec file for a test build + # once. The srpm method may be called multiple times during koji + # releases to create the proper disttags, but we only want to modify + # the spec file once. + self.ran_setup_test_specfile = False + + # NOTE: These are defined later when/if we actually dump a copy of the + # project source at the tag we're building. Only then can we search for + # a spec file. + self.spec_file_name = None + self.spec_file = None + + # Set to path to srpm once we build one. + self.srpm_location = None + + def _create_build_dirs(self): + """ + Create the build directories. Can safely be called multiple times. + """ + BuilderBase._create_build_dirs(self) + build_dirs = [ + self.rpmbuild_gitcopy, + ] + for d in build_dirs: + mkdir_p(d) + self._check_build_dirs_access(build_dirs) + + def _get_build_version(self): + """ + Figure out the git tag and version-release we're building. + """ + # Determine which package version we should build: + build_version = None + if self.build_tag: + build_version = self.build_tag[len(self.project_name + "-"):] + else: + build_version = get_latest_tagged_version(self.project_name) + if build_version is None: + if not self.test: + error_out(["Unable to lookup latest package info.", + "Perhaps you need to tag first?"]) + warn_out("unable to lookup latest package " + "tag, building untagged test project") + build_version = get_spec_version_and_release(self.start_dir, + find_spec_like_file(self.start_dir)) + self.build_tag = self._get_tag_for_version(build_version) + + self.spec_version = build_version.split('-')[0] + self.spec_release = build_version.split('-')[-1] + if not self.test: + check_tag_exists(self.build_tag, offline=self.offline) + return build_version + + def _get_tag_for_version(self, version_and_release): + """ + Determine what the tag will look like for a given version. + Can be overridden when custom taggers override counterpart, + tito.Builder._get_tag_for_version(). + """ + version = version_and_release.split('-')[0] + release = version_and_release.split('-')[-1] + if self.config.has_option(BUILDCONFIG_SECTION, "tag_format"): + tag_format = self.config.get(BUILDCONFIG_SECTION, "tag_format") + else: + tag_format = "{component}-{version}-{release}" + kwargs = { + 'component': self.project_name, + 'version': version, + 'release': release + } + # Strip extra dashes if one of the params is empty + return tag_format.format(**kwargs).strip('-') + + def tgz(self): + """ + Create the .tar.gz required to build this package. + + Returns full path to the created tarball. + """ + self._setup_sources() + + run_command("cp %s/%s %s/" % + (self.rpmbuild_sourcedir, self.tgz_filename, + self.rpmbuild_basedir)) + + self.ran_tgz = True + full_path = os.path.join(self.rpmbuild_basedir, self.tgz_filename) + info_out("Wrote: %s" % full_path) + self.sources.append(full_path) + self.artifacts.append(full_path) + return full_path + + def rpm(self): + """ Build an RPM. """ + self._create_build_dirs() + if not self.ran_tgz: + self.tgz() + if self.test: + self._setup_test_specfile() + BuilderBase.rpm(self) + + def _setup_sources(self): + """ + Create a copy of the git source for the project at the point in time + our build tag was created. + + Created in the temporary rpmbuild SOURCES directory. + """ + self._create_build_dirs() + + debug("Creating %s from git tag: %s..." % (self.tgz_filename, + self.git_commit_id)) + create_tgz(self.git_root, self.tgz_dir, self.git_commit_id, + self.relative_project_dir, + os.path.join(self.rpmbuild_sourcedir, self.tgz_filename)) + + # Extract the source so we can get at the spec file, etc. + debug("Copying git source to: %s" % self.rpmbuild_gitcopy) + run_command("cd %s/ && tar xzf %s" % (self.rpmbuild_sourcedir, + self.tgz_filename)) + + # Show contents of the directory structure we just extracted. + debug('', 'ls -lR %s/' % self.rpmbuild_gitcopy) + + # NOTE: The spec file we actually use is the one exported by git + # archive into the temp build directory. This is done so we can + # modify the version/release on the fly when building test rpms + # that use a git SHA1 for their version. + self.spec_file_name = os.path.basename(find_spec_like_file(self.rpmbuild_gitcopy)) + self.spec_file = os.path.join( + self.rpmbuild_gitcopy, self.spec_file_name) + + def _setup_test_specfile(self): + if self.test and not self.ran_setup_test_specfile: + # If making a test rpm we need to get a little crazy with the spec + # file we're building off. (note that this is a temp copy of the + # spec) Swap out the actual release for one that includes the git + # SHA1 we're building for our test package: + sha = self.git_commit_id[:7] + fullname = "%s-%s" % (self.project_name, self.display_version) + munge_specfile( + self.spec_file, + sha, + self.commit_count, + fullname, + self.tgz_filename, + ) + + self.build_version += ".git." + str(self.commit_count) + "." + str(sha) + self.ran_setup_test_specfile = True + + def _get_rpmbuild_dir_options(self): + return ('--define "_topdir %s" --define "_sourcedir %s" --define "_builddir %s" --define ' + '"_srcrpmdir %s" --define "_rpmdir %s" ' % ( + self.rpmbuild_dir, + self.rpmbuild_sourcedir, self.rpmbuild_builddir, + self.rpmbuild_basedir, self.rpmbuild_basedir)) + + def _get_tgz_name_and_ver(self): + """ + Returns the project name for the .tar.gz to build. Normally this is + just the project name, but in the case of Satellite packages it may + be different. + """ + return "%s-%s" % (self.project_name, self.display_version) + + def _get_display_version(self): + """ + Get the package display version to build. + + Normally this is whatever is .tito/packages/. In the case of a --test + build it will be the SHA1 for the HEAD commit of the current git + branch. + """ + if self.test: + # should get latest commit for given directory *NOT* HEAD + latest_commit = get_latest_commit(".") + self.commit_count = get_commit_count(self.build_tag, latest_commit) + version = "git-%s.%s" % (self.commit_count, latest_commit[:7]) + else: + version = self.build_version.split("-")[0] + return version + + +class NoTgzBuilder(Builder): + """ + Builder for packages that do not require the creation of a tarball. + Usually these packages have source tarballs checked directly into git. + """ + + def tgz(self): + """ Override parent behavior, we already have a tgz. """ + # TODO: Does it make sense to allow user to create a tgz for this type + # of project? + self._setup_sources() + self.ran_tgz = True + + debug("Scanning for sources.") + cmd = "/usr/bin/spectool --list-files '%s' | awk '{print $2}' |xargs -l1 --no-run-if-empty basename " % self.spec_file + result = run_command(cmd) + self.sources = [] + for source in result.split("\n"): + self.sources.append(os.path.join(self.rpmbuild_gitcopy, source)) + debug(" Sources: %s" % self.sources) + + def _get_rpmbuild_dir_options(self): + """ + Override parent behavior slightly. + + These packages store tar's, patches, etc, directly in their project + dir, use the git copy we create as the sources directory when + building package so everything can be found: + """ + return ('--define "_topdir %s" --define "_sourcedir %s" --define "_builddir %s" ' + '--define "_srcrpmdir %s" --define "_rpmdir %s" ' % ( + self.rpmbuild_dir, + self.rpmbuild_gitcopy, self.rpmbuild_builddir, + self.rpmbuild_basedir, self.rpmbuild_basedir)) + + def _setup_test_specfile(self): + """ Override parent behavior. """ + if self.test: + # If making a test rpm we need to get a little crazy with the spec + # file we're building off. (note that this is a temp copy of the + # spec) Swap out the actual release for one that includes the git + # SHA1 we're building for our test package: + debug("setup_test_specfile:commit_count = %s" % str(self.commit_count)) + munge_specfile( + self.spec_file, + self.git_commit_id[:7], + self.commit_count + ) + + +class GemBuilder(NoTgzBuilder): + """ + Gem Builder + + Builder for packages whose sources are managed as gem source structures + and the upstream project does not want to store gem files in git. + """ + + def _setup_sources(self): + """ + Create a copy of the git source for the project at the point in time + our build tag was created. + + Created in the temporary rpmbuild SOURCES directory. + """ + self._create_build_dirs() + + debug("Creating %s from git tag: %s..." % (self.tgz_filename, + self.git_commit_id)) + create_tgz(self.git_root, self.tgz_dir, self.git_commit_id, + self.relative_project_dir, + os.path.join(self.rpmbuild_sourcedir, self.tgz_filename)) + + # Extract the source so we can get at the spec file, etc. + debug("Copying git source to: %s" % self.rpmbuild_gitcopy) + run_command("cd %s/ && tar xzf %s" % (self.rpmbuild_sourcedir, + self.tgz_filename)) + + # Find the gemspec + gemspec_filename = find_gemspec_file(self.rpmbuild_gitcopy) + + debug("Building gem: %s in %s" % (gemspec_filename, + self.rpmbuild_gitcopy)) + # FIXME - this is ugly and should probably be handled better + cmd = "gem_name=$(cd %s/ && gem build %s | awk '/File/ {print $2}'); \ + cp %s/$gem_name %s/" % (self.rpmbuild_gitcopy, gemspec_filename, + self.rpmbuild_gitcopy, self.rpmbuild_sourcedir) + + run_command(cmd) + + # NOTE: The spec file we actually use is the one exported by git + # archive into the temp build directory. This is done so we can + # modify the version/release on the fly when building test rpms + # that use a git SHA1 for their version. + self.spec_file_name = find_spec_file(self.rpmbuild_gitcopy) + self.spec_file = os.path.join( + self.rpmbuild_gitcopy, self.spec_file_name) + + +class UpstreamBuilder(NoTgzBuilder): + """ + Builder for packages that are based off an upstream git tag. + Commits applied in downstream git become patches applied to the + upstream tarball. + + i.e. satellite-java-0.4.0-5 built from spacewalk-java-0.4.0-1 and any + patches applied in satellite git. + i.e. spacewalk-setup-0.4.0-20 built from spacewalk-setup-0.4.0-1 and any + patches applied in satellite git. + """ + + def __init__(self, name=None, tag=None, build_dir=None, + config=None, user_config=None, + args=None, **kwargs): + + NoTgzBuilder.__init__(self, name=name, tag=tag, + build_dir=build_dir, config=config, + user_config=user_config, + args=args, **kwargs) + + if not config or not config.has_option("buildconfig", + "upstream_name"): + # No upstream_name defined, assume we're keeping the project name: + self.upstream_name = self.project_name + else: + self.upstream_name = config.get("buildconfig", "upstream_name") + # Need to assign these after we've exported a copy of the spec file: + self.upstream_version = None + self.upstream_tag = None + + def tgz(self): + """ + Override parent behavior, we need a tgz from the upstream spacewalk + project we're based on. + """ + # TODO: Wasteful step here, all we really need is a way to look for a + # spec file at the point in time this release was tagged. + NoTgzBuilder._setup_sources(self) + # If we knew what it was named at that point in time we could just do: + # Export a copy of our spec file at the revision to be built: +# cmd = "git show %s:%s%s > %s" % (self.git_commit_id, +# self.relative_project_dir, self.spec_file_name, +# self.spec_file) +# debug(cmd) + self._create_build_dirs() + + self.upstream_version = self._get_upstream_version() + self.upstream_tag = "%s-%s-1" % (self.upstream_name, + self.upstream_version) + + print("Building upstream tgz for tag [%s]" % (self.upstream_tag)) + if self.upstream_tag != self.build_tag: + check_tag_exists(self.upstream_tag, offline=self.offline) + + self.spec_file = os.path.join(self.rpmbuild_sourcedir, + self.spec_file_name) + command = "cp %s %s" % (os.path.join(self.rpmbuild_gitcopy, + self.spec_file_name), self.spec_file) + run_command(command) + + # Create the upstream tgz: + prefix = "%s-%s" % (self.upstream_name, self.upstream_version) + tgz_filename = "%s.tar.gz" % prefix + commit = get_build_commit(tag=self.upstream_tag) + relative_dir = get_relative_project_dir( + project_name=self.upstream_name, commit=commit) + tgz_fullpath = os.path.join(self.rpmbuild_sourcedir, tgz_filename) + print("Creating %s from git tag: %s..." % (tgz_filename, commit)) + create_tgz(self.git_root, prefix, commit, relative_dir, + tgz_fullpath) + self.ran_tgz = True + self.sources.append(tgz_fullpath) + + # If these are equal then the tag we're building was likely created in + # Spacewalk and thus we don't need to do any patching. + if (self.upstream_tag == self.build_tag and not self.test): + return + + self.patch_upstream() + + def _patch_upstream(self): + """ Insert patches into the spec file we'll be building + returns (patch_number, patch_insert_index, patch_apply_index, lines) + """ + f = open(self.spec_file, 'r') + lines = f.readlines() + f.close() + + patch_pattern = re.compile('^Patch(\d+):') + source_pattern = re.compile('^Source(\d+)?:') + + # Find the largest PatchX: line, or failing that SourceX: + patch_number = 0 # What number should we use for our PatchX line + patch_insert_index = 0 # Where to insert our PatchX line in the list + patch_apply_index = 0 # Where to insert our %patchX line in the list + array_index = 0 # Current index in the array + for line in lines: + match = source_pattern.match(line) + if match: + patch_insert_index = array_index + 1 + + match = patch_pattern.match(line) + if match: + patch_insert_index = array_index + 1 + patch_number = int(match.group(1)) + 1 + + if line.startswith("%prep"): + # We'll apply patch right after prep if there's no %setup line + patch_apply_index = array_index + 2 + elif line.startswith("%setup"): + patch_apply_index = array_index + 2 # already added a line + elif line.startswith("%autosetup"): + patch_apply_index = -1 # autosetup will do this for us + + array_index += 1 + + debug("patch_insert_index = %s" % patch_insert_index) + debug("patch_apply_index = %s" % patch_apply_index) + if patch_insert_index == 0 or patch_apply_index == 0: + error_out("Unable to insert PatchX or %patchX lines in spec file") + return (patch_number, patch_insert_index, patch_apply_index, lines) + + def patch_upstream(self): + """ + Generate patches for any differences between our tag and the + upstream tag, and apply them into an exported copy of the + spec file. + """ + patch_filename = "%s-to-%s-%s.patch" % (self.upstream_tag, + self.project_name, self.build_version) + patch_file = os.path.join(self.rpmbuild_gitcopy, + patch_filename) + patch_dir = self.git_root + if self.relative_project_dir != "/": + patch_dir = os.path.join(self.git_root, + self.relative_project_dir) + os.chdir(patch_dir) + debug("patch dir = %s" % patch_dir) + print("Generating patch [%s]" % patch_filename) + debug("Patch: %s" % patch_file) + patch_command = "git diff --relative %s..%s > %s" % \ + (self.upstream_tag, self.git_commit_id, + patch_file) + debug("Generating patch with: %s" % patch_command) + output = run_command(patch_command) + print(output) + (status, output) = getstatusoutput( + "grep 'Binary files .* differ' %s " % patch_file) + if status == 0 and output != "": + error_out("You are doomed. Diff contains binary files. You can not use this builder") + + # Creating two copies of the patch here in the temp build directories + # just out of laziness. Some builders need sources in SOURCES and + # others need them in the git copy. Being lazy here avoids one-off + # hacks and both copies get cleaned up anyhow. + run_command("cp %s %s" % (patch_file, self.rpmbuild_sourcedir)) + + (patch_number, patch_insert_index, patch_apply_index, lines) = self._patch_upstream() + + lines.insert(patch_insert_index, "Patch%s: %s\n" % (patch_number, + patch_filename)) + if patch_apply_index > 0: + lines.insert(patch_apply_index, "%%patch%s -p1\n" % (patch_number)) + self._write_spec(lines) + + def _write_spec(self, lines): + """ Write 'lines' to self.spec_file """ + # Now write out the modified lines to the spec file copy: + f = open(self.spec_file, 'w') + for line in lines: + f.write(line) + f.close() + + def _get_upstream_version(self): + """ + Get the upstream version. Checks for "upstreamversion" in the spec file + and uses it if found. Otherwise assumes the upstream version is equal + to the version we're building. + + i.e. satellite-java-0.4.15 will be built on spacewalk-java-0.4.15 + with just the package release being incremented on rebuilds. + """ + # Use upstreamversion if defined in the spec file: + (status, output) = getstatusoutput( + "cat %s | grep 'define upstreamversion' | " + "awk '{ print $3 ; exit }'" % self.spec_file) + if status == 0 and output != "": + return output + + if self.test: + return self.build_version.split("-")[0] + # Otherwise, assume we use our version: + else: + return self.display_version + + def _get_rpmbuild_dir_options(self): + """ + Override parent behavior slightly. + + These packages store tar's, patches, etc, directly in their project + dir, use the git copy we create as the sources directory when + building package so everything can be found: + """ + return ('--define "_topdir %s" --define "_sourcedir %s" --define "_builddir %s" ' + '--define "_srcrpmdir %s" --define "_rpmdir %s" ' % ( + self.rpmbuild_dir, + self.rpmbuild_sourcedir, self.rpmbuild_builddir, + self.rpmbuild_basedir, self.rpmbuild_basedir)) + + +# Legacy class name for backward compatability: +class SatelliteBuilder(UpstreamBuilder): + pass + + +class MeadBuilder(Builder): + def __init__(self, name=None, tag=None, build_dir=None, + config=None, user_config=None, args=None, **kwargs): + + Builder.__init__(self, name=name, tag=tag, build_dir=build_dir, + config=config, user_config=user_config, args=args, **kwargs) + + self.ran_maven = False + self.deploy_dir = mkdtemp(dir=self.rpmbuild_basedir, prefix="maven-deploy-%s" % self.project_name) + self.maven_clone_dir = mkdtemp(dir=self.rpmbuild_basedir, prefix="maven-clone-%s" % self.project_name) + + # People calling `tito build` will almost certainly want to do a maven build locally. + # But with a `tito release` we want the Mead stuff to happen on the build system + self.local_build = args.setdefault('local', [True]) + + self.maven_properties = [] + if 'maven_property' in args: + self.maven_properties = args['maven_property'] + else: + # Generally people aren't going to want to run their tests during + # a build. If they do, they can set maven_properties='' + self.maven_properties.append("maven.test.skip=true") + + self.maven_args = ['-B'] + if 'maven_arg' in args: + self.maven_args.extend(args['maven_arg']) + + def _find_tarball(self): + for directory, unused, filenames in os.walk(self.deploy_dir): + for f in filenames: + name, ext = os.path.splitext(f) + if ext == ".gz" and name.startswith("%s-%s" % (self.project_name, self.spec_version)): + return os.path.join(self.deploy_dir, directory, f) + return None + + def cleanup(self): + """ + Remove all temporary files and directories. + """ + if not self.no_cleanup: + for d in [self.rpmbuild_dir, self.deploy_dir, self.maven_clone_dir]: + debug("Cleaning up %s" % d) + shutil.rmtree(d) + else: + warn_out("Leaving rpmbuild files in: %s" % self.rpmbuild_dir) + + def tgz(self): + destination_file = os.path.join(self.rpmbuild_basedir, self.tgz_filename) + formatted_properties = ["-D%s" % x for x in self.maven_properties] + + run_command("git clone --no-hardlinks %s %s" % (find_git_root(), self.maven_clone_dir)) + with chdir(self.maven_clone_dir): + run_command("git checkout %s" % self.git_commit_id) + + try: + info_out("Running Maven build...") + # We always want to deploy to a tito controlled location during local builds + local_properties = formatted_properties + [ + "-DaltDeploymentRepository=local-output::default::file://%s" % self.deploy_dir] + run_command("mvn %s %s deploy" % ( + " ".join(self.maven_args), + " ".join(local_properties))) + except RunCommandException as e: + error_out("Maven build failed! %s" % e.output) + + self._create_build_dirs() + + full_path = self._find_tarball() + if full_path: + fh = gzip.open(full_path, 'rb') + fixed_tar = os.path.join(os.path.splitext(full_path)[0]) + fixed_tar_fh = open(fixed_tar, 'wb') + timestamp = get_commit_timestamp(self.git_commit_id) + try: + tarfixer = TarFixer(fh, fixed_tar_fh, timestamp, self.git_commit_id, maven_built=True) + tarfixer.fix() + finally: + fixed_tar_fh.close() + + # It's a pity we can't use Python's gzip, but it doesn't offer an equivalent of -n + run_command("gzip -n -c < %s > %s" % (fixed_tar, destination_file)) + else: + warn_out([ + "No Maven generated tarball found.", + "Please set up the assembly plugin in your pom.xml to generate a .tar.gz"]) + full_path = os.path.join(self.rpmbuild_sourcedir, self.tgz_filename) + create_tgz(self.git_root, self.tgz_dir, self.git_commit_id, self.relative_project_dir, full_path) + print("Creating %s from git tag: %s..." % (self.tgz_filename, self.build_tag)) + shutil.copy(full_path, destination_file) + + debug("Copying git source to: %s" % self.rpmbuild_gitcopy) + shutil.copy(destination_file, self.rpmbuild_gitcopy) + + # Extract the source so we can get at the spec file, etc. + with chdir(self.rpmbuild_gitcopy): + run_command("tar --strip-components=1 -xvf %s" % os.path.join(self.rpmbuild_gitcopy, self.tgz_filename)) + + if self.local_build: + artifacts = {} + all_artifacts = [] + all_artifacts_with_path = [] + + for directory, unused, filenames in os.walk(self.deploy_dir): + for f in filenames: + artifacts.setdefault(os.path.splitext(f)[1], []).append(f) + dir_artifacts_with_path = [os.path.join(directory, f) for f in filenames] + + # Place the Maven artifacts in the SOURCES directory for rpmbuild to use + for artifact in dir_artifacts_with_path: + shutil.copy(artifact, self.rpmbuild_sourcedir) + + dir_artifacts_with_path = map(lambda x: os.path.relpath(x, self.deploy_dir), dir_artifacts_with_path) + all_artifacts_with_path.extend(dir_artifacts_with_path) + all_artifacts.extend([os.path.basename(f) for f in filenames]) + + cheetah_input = { + 'name': self.project_name, + 'version': self.spec_version, + 'release': self.spec_release, + 'epoch': None, # TODO: May need to support this at some point + 'artifacts': artifacts, + 'all_artifacts': all_artifacts, + 'all_artifacts_with_path': all_artifacts_with_path, + } + debug("Cheetah input: %s" % cheetah_input) + render_cheetah(find_cheetah_template_file(self.start_dir), self.rpmbuild_gitcopy, cheetah_input) + self.spec_file_name = find_spec_file(self.rpmbuild_gitcopy) + else: + self.spec_file_name = find_cheetah_template_file(self.rpmbuild_gitcopy) + + # NOTE: The spec file we actually use is the one exported by git + # archive into the temp build directory. This is done so we can + # modify the version/release on the fly when building test rpms + # that use a git SHA1 for their version. + self.spec_file = os.path.join(self.rpmbuild_gitcopy, self.spec_file_name) + + info_out("Wrote: %s" % destination_file) + self.sources.append(destination_file) + self.artifacts.append(destination_file) + self.ran_tgz = True + + def _setup_test_specfile(self): + if self.test and not self.ran_setup_test_specfile: + # If making a test rpm we need to get a little crazy with the spec + # file we're building off. (note that this is a temp copy of the + # spec) Swap out the actual release for one that includes the git + # SHA1 we're building for our test package: + self.build_version += ".git." + str(self.commit_count) + "." + str(self.git_commit_id[:7]) + replace_spec_release(self.spec_file, self.spec_release) + self.ran_setup_test_specfile = True + + +class MockBuilder(Builder): + """ + Uses the mock tool to create a chroot for building packages for a different + OS version than you may be currently using. + """ + REQUIRED_ARGS = ['mock'] + + def __init__(self, name=None, tag=None, build_dir=None, + config=None, user_config=None, + args=None, **kwargs): + + # Mock builders need to use the packages normally configured builder + # to get at a proper SRPM: + self.normal_builder = create_builder(name, tag, config, + build_dir, user_config, args, **kwargs) + + Builder.__init__(self, name=name, tag=tag, + build_dir=build_dir, config=config, + user_config=user_config, + args=args, **kwargs) + + self.mock_tag = args['mock'][0] + self.mock_cmd_args = "" + if 'mock_config_dir' in args: + mock_config_dir = args['mock_config_dir'][0] + if not mock_config_dir.startswith("/"): + # If not an absolute path, assume below git root: + mock_config_dir = os.path.join(self.git_root, mock_config_dir) + if not os.path.exists(mock_config_dir): + raise TitoException("No such mock config dir: %s" % mock_config_dir) + self.mock_cmd_args = "%s --configdir=%s" % (self.mock_cmd_args, mock_config_dir) + + # Optional argument which will skip mock --init and add --no-clean + # and --no-cleanup-after: + self.speedup = False + if 'speedup' in args: + self.speedup = True + self.mock_cmd_args = "%s --no-clean --no-cleanup-after" % \ + (self.mock_cmd_args) + + if 'mock_args' in args: + self.mock_cmd_args = "%s %s" % (self.mock_cmd_args, args['mock_args'][0]) + + # TODO: error out if mock package is not installed + + # TODO: error out if user does not have mock group + + def srpm(self, dist=None): + """ + Build a source RPM. + + MockBuilder will use an instance of the normal builder for a package + internally just so we can generate a SRPM correctly before we pass it + into mock. + """ + self.normal_builder.srpm(dist) + self.srpm_location = self.normal_builder.srpm_location + self.artifacts.append(self.srpm_location) + + def rpm(self): + """ + Uses the SRPM + Override the base builder rpm method. + """ + + print("Creating rpms for %s-%s in mock: %s" % ( + self.project_name, self.display_version, self.mock_tag)) + if not self.srpm_location: + self.srpm() + print("Using srpm: %s" % self.srpm_location) + self._build_in_mock() + + def cleanup(self): + if self.normal_builder: + self.normal_builder.cleanup() + + def _build_in_mock(self): + run_command_func = run_command if self.quiet else run_command_print + if not self.speedup: + print("Initializing mock...") + run_command_func("mock %s -r %s --init" % (self.mock_cmd_args, self.mock_tag)) + else: + print("Skipping mock --init due to speedup option.") + + print("Building RPMs in mock...") + run_command_func('mock %s -r %s --rebuild %s' % + (self.mock_cmd_args, self.mock_tag, self.srpm_location)) + mock_output_dir = os.path.join(self.rpmbuild_dir, "mockoutput") + run_command_func("mock %s -r %s --copyout /builddir/build/RPMS/ %s" % + (self.mock_cmd_args, self.mock_tag, mock_output_dir)) + + # Copy everything mock wrote out to /tmp/tito: + files = os.listdir(mock_output_dir) + for rpm in files: + shutil.copy2(os.path.join(mock_output_dir, rpm), self.rpmbuild_basedir) + print + info_out("Wrote:") + for rpm in files: + rpm_path = os.path.join(self.rpmbuild_basedir, rpm) + print(" %s" % rpm_path) + self.artifacts.append(rpm_path) + print + + +class BrewDownloadBuilder(Builder): + """ + A special case builder which uses pre-existing Brew builds and + pulls down the resulting rpms locally. Useful in some cases when + generating yum repositories during a release. + """ + REQUIRED_ARGS = ['disttag'] + + def __init__(self, name=None, tag=None, build_dir=None, + config=None, user_config=None, + args=None, **kwargs): + + Builder.__init__(self, name=name, tag=tag, + build_dir=build_dir, config=config, + user_config=user_config, + args=args, **kwargs) + + self.dist_tag = args['disttag'][0] + + def rpm(self): + """ + Uses the SRPM + Override the base builder rpm method. + """ + + print("Fetching rpms for %s.%s from brew:" % ( + self.build_tag, self.dist_tag)) + self._fetch_from_brew() + + def _fetch_from_brew(self): + brew_nvr = "%s.%s" % (self.build_tag, self.dist_tag) + debug("Brew NVR: %s" % brew_nvr) + os.chdir(self.rpmbuild_dir) + run_command("brew download-build %s" % brew_nvr) + + # Wipe out the src rpm for now: + run_command("rm *.src.rpm") + + # Copy everything brew downloaded out to /tmp/tito: + files = os.listdir(self.rpmbuild_dir) + run_command("cp -v %s/*.rpm %s" % + (self.rpmbuild_dir, self.rpmbuild_basedir)) + print + info_out("Wrote:") + for rpm in files: + # Just incase anything slips into the build dir: + if not rpm.endswith(".rpm"): + continue + rpm_path = os.path.join(self.rpmbuild_basedir, rpm) + print(" %s" % rpm_path) + self.artifacts.append(rpm_path) + print + + +class GitAnnexBuilder(NoTgzBuilder): + """ + Builder for packages with existing tarballs checked in using git-annex, + e.g. referencing an external source (web remote). This builder will + "unlock" the source files to get the real contents, include them in the + SRPM, then restore the automatic git-annex symlinks on completion. + """ + + def _setup_sources(self): + super(GitAnnexBuilder, self)._setup_sources() + + self.old_cwd = os.getcwd() + os.chdir(os.path.join(self.old_cwd, self.relative_project_dir)) + + # NOTE: 'which' may not be installed... (docker containers) + (status, output) = getstatusoutput("which git-annex") + if status != 0: + msg = "Please run '%s' as root." % self.package_manager.install(["git-annex"]) + error_out('%s' % msg) + + run_command("git-annex lock") + annexed_files = run_command("git-annex find --include='*'").splitlines() + run_command("git-annex get") + run_command("git-annex unlock") + debug(" Annex files: %s" % annexed_files) + + for annex in annexed_files: + debug("Copying unlocked file %s" % annex) + os.remove(os.path.join(self.rpmbuild_gitcopy, annex)) + shutil.copy(annex, self.rpmbuild_gitcopy) + + self._lock() + os.chdir(self.old_cwd) + + def cleanup(self): + if hasattr(self, 'old_cwd'): + os.chdir(self.old_cwd) + self._lock() + super(GitAnnexBuilder, self).cleanup() + + def _lock(self): + if self._lock_force_supported(self._get_annex_version()): + run_command("git-annex lock --force") + else: + run_command("git-annex lock") + + def _get_annex_version(self): + # git-annex needs to support --force when locking files. + ga_version = run_command('git-annex version').split('\n') + if ga_version[0].startswith('git-annex version'): + return ga_version[0].split()[-1] + else: + return 0 + + def _lock_force_supported(self, version): + return compare_version(version, '5.20131213') >= 0 + + +class GitLfsBuilder(NoTgzBuilder): + """ + Builder class for Git LFS support. + """ + + def _setup_sources(self): + super(GitLfsBuilder, self)._setup_sources() + + self.old_cwd = os.getcwd() + os.chdir(os.path.join(self.old_cwd, self.relative_project_dir)) + debug("learning %s" % self.relative_project_dir) + (status, output) = getstatusoutput("which git-lfs") + if status != 0: + msg = "Please run '%s' as root." % self.package_manager.install(["git-lfs"]) + error_out('%s' % msg) + + gitlfs_files = run_command("git-lfs ls-files").splitlines() + run_command("git-lfs fetch") + debug(" Lfs files: %s" % gitlfs_files) + for gfile in gitlfs_files: + m = re.search('(?<=' + self.relative_project_dir + ')(.*)', gfile) + if m: + filterfile = m.groups(0) + mstr = filterfile[0] + debug("Copying file %s " % mstr) + debug("To %s " % self.rpmbuild_gitcopy) + os.remove(os.path.join(self.rpmbuild_gitcopy, mstr)) + shutil.copy(mstr, self.rpmbuild_gitcopy) + + os.chdir(self.old_cwd) + + def cleanup(self): + if hasattr(self, 'old_cwd'): + os.chdir(self.old_cwd) + super(GitLfsBuilder, self).cleanup() + + +def package_manager(): + if os.path.isfile("/usr/bin/dnf"): + return Dnf() + if os.path.isfile("/usr/bin/yum"): + return Yum() + return Rpm() + + +class Rpm(object): + def install(self, packages, escalate=True, **kwargs): + escalation_cmd = "sudo" if escalate else "" + return "%s rpm -U --force %s" % (escalation_cmd, ' '.join(packages)) + + def builddep(self, spec): + raise NotImplementedError + + def is_installed(self, package, version): + q = self.query(package) + if not q: + return False + + iv = "%s-%s" % (q.version.decode("utf-8"), q.release.decode("utf-8")) + iv_short = ".".join(iv.split(".")[:-1]) + return version == iv_short + + def query(self, package): + ts = rpm.TransactionSet() + results = list(ts.dbMatch("name", package)) + return results[0] if results else None + + +class Dnf(Rpm): + def install(self, packages, reinstall=False, auto=False, offline=False, escalate=True, **kwargs): + action = "reinstall" if reinstall else "install" + args = list(filter(lambda x: x, [ + "-C" if offline else None, + "-y" if auto else None, + ])) + escalation_cmd = "sudo" if escalate else "" + cmd = "%s dnf %s %s" % (escalation_cmd, action, " ".join(args + packages)) + return " ".join(cmd.split()) + + def builddep(self, spec): + return "dnf builddep %s" % spec + + +class Yum(Rpm): + def install(self, packages, **kwargs): + # Not the sexiest implementation, but very short + return Dnf().install(packages, **kwargs).replace("dnf", "yum") + + def builddep(self, spec): + return "yum-builddep %s" % spec