You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
1375 lines
52 KiB
1375 lines
52 KiB
6 years ago
|
# Copyright (c) 2008-2011 Red Hat, Inc.
|
||
|
#
|
||
|
# This software is licensed to you under the GNU General Public License,
|
||
|
# version 2 (GPLv2). There is NO WARRANTY for this software, express or
|
||
|
# implied, including the implied warranties of MERCHANTABILITY or FITNESS
|
||
|
# FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2
|
||
|
# along with this software; if not, see
|
||
|
# http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt.
|
||
|
#
|
||
|
# Red Hat trademarks are not licensed under GPLv2. No permission is
|
||
|
# granted to use or replicate Red Hat trademarks that are incorporated
|
||
|
# in this software or its documentation.
|
||
|
"""
|
||
|
Tito builders for a variety of common methods of building sources, srpms,
|
||
|
and rpms.
|
||
|
"""
|
||
|
|
||
|
import gzip
|
||
|
import os
|
||
|
import sys
|
||
|
import re
|
||
|
import shutil
|
||
|
import rpm
|
||
|
from pkg_resources import require
|
||
|
from distutils.version import LooseVersion as loose_version
|
||
|
from tempfile import mkdtemp
|
||
|
|
||
|
from tito.common import scl_to_rpm_option, get_latest_tagged_version, \
|
||
|
find_wrote_in_rpmbuild_output, debug, error_out, run_command_print, \
|
||
|
find_spec_file, run_command, get_build_commit, get_relative_project_dir, \
|
||
|
get_relative_project_dir_cwd, get_spec_version_and_release, \
|
||
|
check_tag_exists, create_tgz, get_latest_commit, \
|
||
|
get_commit_count, find_gemspec_file, create_builder, compare_version,\
|
||
|
find_cheetah_template_file, render_cheetah, replace_spec_release, \
|
||
|
find_spec_like_file, warn_out, get_commit_timestamp, chdir, mkdir_p, \
|
||
|
find_git_root, info_out, munge_specfile, BUILDCONFIG_SECTION
|
||
|
from tito.compat import getstatusoutput
|
||
|
from tito.exception import RunCommandException
|
||
|
from tito.exception import TitoException
|
||
|
from tito.config_object import ConfigObject
|
||
|
from tito.tar import TarFixer
|
||
|
|
||
|
|
||
|
class BuilderBase(object):
|
||
|
"""
|
||
|
A base class for all builders.
|
||
|
|
||
|
Handles things we will *always* do, primarily handling temporary directories
|
||
|
for rpmbuild.
|
||
|
|
||
|
This class should *not* assume we're even using git.
|
||
|
"""
|
||
|
REQUIRED_ARGS = []
|
||
|
|
||
|
# TODO: merge config into an object and kill the ConfigObject parent class
|
||
|
def __init__(self, name=None, build_dir=None,
|
||
|
config=None, user_config=None,
|
||
|
args=None, **kwargs):
|
||
|
|
||
|
# Project directory where we started this build:
|
||
|
self.start_dir = os.getcwd()
|
||
|
|
||
|
self.project_name = name
|
||
|
self.user_config = user_config
|
||
|
self.args = args
|
||
|
self.kwargs = kwargs
|
||
|
self.config = config
|
||
|
|
||
|
# Optional keyword arguments:
|
||
|
self.dist = self._get_optional_arg(kwargs, 'dist', None)
|
||
|
|
||
|
self.offline = self._get_optional_arg(kwargs, 'offline', False)
|
||
|
self.auto_install = self._get_optional_arg(kwargs, 'auto_install',
|
||
|
False)
|
||
|
self.escalate_privileges = self._get_optional_arg(kwargs, 'escalate',
|
||
|
True)
|
||
|
self.scl = self._get_optional_arg(args, 'scl', [None])[0] or \
|
||
|
self._get_optional_arg(kwargs, 'scl', '')
|
||
|
|
||
|
self.quiet = self._get_optional_arg(kwargs, 'quiet', False)
|
||
|
self.verbose = self._get_optional_arg(kwargs, 'verbose', False)
|
||
|
|
||
|
rpmbuildopts = self._get_optional_arg(args, 'rpmbuild_options', None)
|
||
|
if rpmbuildopts:
|
||
|
self.rpmbuild_options = ' '.join(rpmbuildopts)
|
||
|
else:
|
||
|
self.rpmbuild_options = self._get_optional_arg(kwargs, 'rpmbuild_options', '')
|
||
|
|
||
|
self.test = self._get_optional_arg(kwargs, 'test', False)
|
||
|
# Allow a builder arg to override the test setting passed in, used by
|
||
|
# releasers in their config sections.
|
||
|
if args and 'test' in args:
|
||
|
self.test = True
|
||
|
|
||
|
# Location where we do all tito work and store resulting rpms:
|
||
|
self.rpmbuild_basedir = build_dir
|
||
|
# Location where we do actual rpmbuilds
|
||
|
self.rpmbuild_dir = mkdtemp(dir=self.rpmbuild_basedir,
|
||
|
prefix="rpmbuild-%s" % self.project_name)
|
||
|
debug("Building in temp dir: %s" % self.rpmbuild_dir)
|
||
|
self.rpmbuild_sourcedir = os.path.join(self.rpmbuild_dir, "SOURCES")
|
||
|
self.rpmbuild_builddir = os.path.join(self.rpmbuild_dir, "BUILD")
|
||
|
|
||
|
self._check_required_args()
|
||
|
|
||
|
# Set to true once we've created/setup sources: (i.e. tar.gz)
|
||
|
self.ran_tgz = False
|
||
|
|
||
|
self.no_cleanup = False
|
||
|
|
||
|
# List of full path to all sources for this package.
|
||
|
self.sources = []
|
||
|
|
||
|
# Artifacts we built:
|
||
|
self.artifacts = []
|
||
|
|
||
|
# Use most suitable package manager for current OS
|
||
|
self.package_manager = package_manager()
|
||
|
|
||
|
def _get_optional_arg(self, kwargs, arg, default):
|
||
|
"""
|
||
|
Return the value of an optional keyword argument if it's present,
|
||
|
otherwise the default provided.
|
||
|
"""
|
||
|
if arg in kwargs:
|
||
|
return kwargs[arg]
|
||
|
return default
|
||
|
|
||
|
def _check_required_args(self):
|
||
|
for arg in self.REQUIRED_ARGS:
|
||
|
if arg not in self.args:
|
||
|
raise TitoException("Builder missing required argument: %s" %
|
||
|
arg)
|
||
|
|
||
|
def run(self, options):
|
||
|
"""
|
||
|
Perform the actions requested of the builder.
|
||
|
|
||
|
NOTE: this method may do nothing if the user requested no build actions
|
||
|
be performed. (i.e. only release tagging, etc)
|
||
|
"""
|
||
|
info_out("Building package [%s]" % (self.build_tag))
|
||
|
self.no_cleanup = options.no_cleanup
|
||
|
|
||
|
# Reset list of artifacts on each call to run().
|
||
|
self.artifacts = []
|
||
|
|
||
|
try:
|
||
|
try:
|
||
|
if options.tgz:
|
||
|
self.tgz()
|
||
|
if options.srpm:
|
||
|
self.srpm()
|
||
|
if options.rpm:
|
||
|
# TODO: not protected anymore
|
||
|
self.rpm()
|
||
|
self._auto_install()
|
||
|
except KeyboardInterrupt:
|
||
|
print("Interrupted, cleaning up...")
|
||
|
finally:
|
||
|
self.cleanup()
|
||
|
|
||
|
return self.artifacts
|
||
|
|
||
|
def cleanup(self):
|
||
|
"""
|
||
|
Remove all temporary files and directories.
|
||
|
"""
|
||
|
if not self.no_cleanup:
|
||
|
debug("Cleaning up %s" % self.rpmbuild_dir)
|
||
|
shutil.rmtree(self.rpmbuild_dir)
|
||
|
else:
|
||
|
warn_out("Leaving rpmbuild files in: %s" % self.rpmbuild_dir)
|
||
|
|
||
|
def _check_build_dirs_access(self, build_dirs):
|
||
|
"""
|
||
|
Ensure the build directories are writable.
|
||
|
"""
|
||
|
msgs = []
|
||
|
for d in build_dirs:
|
||
|
if not os.access(d, os.W_OK):
|
||
|
msgs.append("%s is not writable." % d)
|
||
|
if msgs:
|
||
|
error_out(msgs)
|
||
|
|
||
|
def _create_build_dirs(self):
|
||
|
"""
|
||
|
Create the build directories. Can safely be called multiple times.
|
||
|
"""
|
||
|
build_dirs = [
|
||
|
self.rpmbuild_basedir,
|
||
|
self.rpmbuild_dir,
|
||
|
self.rpmbuild_sourcedir,
|
||
|
self.rpmbuild_builddir,
|
||
|
]
|
||
|
for d in build_dirs:
|
||
|
mkdir_p(d)
|
||
|
self._check_build_dirs_access(build_dirs)
|
||
|
|
||
|
def srpm(self, dist=None):
|
||
|
"""
|
||
|
Build a source RPM.
|
||
|
"""
|
||
|
self._create_build_dirs()
|
||
|
if not self.ran_tgz:
|
||
|
self.tgz()
|
||
|
|
||
|
if self.test:
|
||
|
self._setup_test_specfile()
|
||
|
|
||
|
debug("Creating srpm from spec file: %s" % self.spec_file)
|
||
|
define_dist = ""
|
||
|
if self.dist:
|
||
|
debug("using self.dist: %s" % self.dist)
|
||
|
define_dist = "--define 'dist %s'" % self.dist
|
||
|
elif dist:
|
||
|
debug("using dist: %s" % dist)
|
||
|
define_dist = "--define 'dist %s'" % dist
|
||
|
else:
|
||
|
debug("*NOT* using dist at all")
|
||
|
|
||
|
rpmbuild_options = self.rpmbuild_options + self._scl_to_rpmbuild_option()
|
||
|
|
||
|
cmd = ('rpmbuild --define "_source_filedigest_algorithm md5" --define'
|
||
|
' "_binary_filedigest_algorithm md5" %s %s %s --nodeps -bs %s' % (
|
||
|
rpmbuild_options, self._get_rpmbuild_dir_options(),
|
||
|
define_dist, self.spec_file))
|
||
|
run_command_func = run_command if self.quiet else run_command_print
|
||
|
output = run_command_func(cmd)
|
||
|
self.srpm_location = find_wrote_in_rpmbuild_output(output)[0]
|
||
|
self.artifacts.append(self.srpm_location)
|
||
|
|
||
|
# Assume that if tito's --no-cleanup option is set, also disable %clean in rpmbuild:
|
||
|
def _get_clean_option(self):
|
||
|
if self.no_cleanup:
|
||
|
output = run_command('rpmbuild --help')
|
||
|
if '--noclean' in output:
|
||
|
return "--noclean"
|
||
|
else:
|
||
|
return ""
|
||
|
else:
|
||
|
return "--clean"
|
||
|
|
||
|
def _get_verbosity_option(self):
|
||
|
if self.verbose:
|
||
|
return "--verbose"
|
||
|
else:
|
||
|
return ""
|
||
|
|
||
|
def rpm(self):
|
||
|
""" Build an RPM. """
|
||
|
self._create_build_dirs()
|
||
|
if not self.ran_tgz:
|
||
|
self.tgz()
|
||
|
|
||
|
cmd = 'rpmbuild {0}'.format(
|
||
|
" ".join([
|
||
|
'--define "_source_filedigest_algorithm md5"',
|
||
|
'--define "_binary_filedigest_algorithm md5"',
|
||
|
self.rpmbuild_options,
|
||
|
self._scl_to_rpmbuild_option(),
|
||
|
self._get_rpmbuild_dir_options(),
|
||
|
"--define 'dist {0}'".format(self.dist) if self.dist else "",
|
||
|
self._get_clean_option(),
|
||
|
self._get_verbosity_option(),
|
||
|
'-ba {0}'.format(self.spec_file),
|
||
|
])
|
||
|
)
|
||
|
try:
|
||
|
if self.quiet:
|
||
|
output = run_command(cmd)
|
||
|
else:
|
||
|
output = run_command_print(cmd)
|
||
|
except (KeyboardInterrupt, SystemExit):
|
||
|
print("")
|
||
|
exit(1)
|
||
|
except RunCommandException:
|
||
|
err = sys.exc_info()[1]
|
||
|
msg = str(err)
|
||
|
if re.search('Failed build dependencies', err.output):
|
||
|
cmd = self.package_manager.builddep(find_spec_file(self.relative_project_dir))
|
||
|
msg = "Please run '%s' as root." % cmd
|
||
|
error_out('%s' % msg)
|
||
|
except Exception:
|
||
|
err = sys.exc_info()[1]
|
||
|
error_out('%s' % str(err))
|
||
|
files_written = find_wrote_in_rpmbuild_output(output)
|
||
|
if len(files_written) < 2:
|
||
|
error_out("Error parsing rpmbuild output")
|
||
|
self.srpm_location = files_written[0]
|
||
|
self.artifacts.extend(files_written)
|
||
|
|
||
|
print
|
||
|
info_out("Successfully built: %s" % '\n\t- '.join(files_written))
|
||
|
|
||
|
def _scl_to_rpmbuild_option(self):
|
||
|
""" Returns rpmbuild option which disable or enable SC and print warning if needed """
|
||
|
return scl_to_rpm_option(self.scl)
|
||
|
|
||
|
def _auto_install(self):
|
||
|
"""
|
||
|
If requested, auto install the RPMs we just built.
|
||
|
"""
|
||
|
if self.auto_install:
|
||
|
print
|
||
|
print("Auto-installing packages:")
|
||
|
print
|
||
|
|
||
|
dont_install = []
|
||
|
if 'NO_AUTO_INSTALL' in self.user_config:
|
||
|
dont_install = self.user_config['NO_AUTO_INSTALL'].split(" ")
|
||
|
debug("Will not auto-install any packages matching: %s" % dont_install)
|
||
|
|
||
|
do_install = []
|
||
|
for to_inst in self.artifacts:
|
||
|
# Only install rpms:
|
||
|
if not to_inst.endswith(".rpm") or to_inst.endswith(".src.rpm"):
|
||
|
continue
|
||
|
|
||
|
install = True
|
||
|
for skip in dont_install:
|
||
|
if skip in to_inst:
|
||
|
install = False
|
||
|
print("Skipping: %s" % to_inst)
|
||
|
break
|
||
|
if install:
|
||
|
do_install.append(to_inst)
|
||
|
|
||
|
print
|
||
|
reinstall = self.package_manager.is_installed(self.project_name, self.build_version)
|
||
|
cmd = self.package_manager.install(do_install, reinstall=reinstall, auto=True, offline=True,
|
||
|
escalate=self.escalate_privileges)
|
||
|
print("%s" % cmd)
|
||
|
try:
|
||
|
run_command_print(cmd)
|
||
|
print
|
||
|
except KeyboardInterrupt:
|
||
|
pass
|
||
|
|
||
|
|
||
|
class Builder(ConfigObject, BuilderBase):
|
||
|
"""
|
||
|
Parent builder class.
|
||
|
|
||
|
Includes functionality for a standard Spacewalk package build. Packages
|
||
|
which require other unusual behavior can subclass this to inject the
|
||
|
desired behavior.
|
||
|
"""
|
||
|
REQUIRED_ARGS = []
|
||
|
|
||
|
# TODO: drop version
|
||
|
def __init__(self, name=None, tag=None, build_dir=None,
|
||
|
config=None, user_config=None,
|
||
|
args=None, **kwargs):
|
||
|
|
||
|
"""
|
||
|
name - Package name that is being built.
|
||
|
|
||
|
version - Version and release being built.
|
||
|
|
||
|
tag - The git tag being built.
|
||
|
|
||
|
build_dir - Temporary build directory where we can safely work.
|
||
|
|
||
|
config - Merged configuration. (global plus package specific)
|
||
|
|
||
|
user_config - User configuration from ~/.titorc.
|
||
|
|
||
|
args - Optional arguments specific to each builder. Can be passed
|
||
|
in explicitly by user on the CLI, or via a release target config
|
||
|
entry. Only for things which vary on invocations of the builder,
|
||
|
avoid using these if possible. *Given in the format of a dictionary
|
||
|
of lists.*
|
||
|
"""
|
||
|
ConfigObject.__init__(self, config=config)
|
||
|
BuilderBase.__init__(self, name=name, build_dir=build_dir, config=config,
|
||
|
user_config=user_config, args=args, **kwargs)
|
||
|
self.build_tag = tag
|
||
|
|
||
|
self.build_version = self._get_build_version()
|
||
|
|
||
|
if kwargs and 'options' in kwargs:
|
||
|
warn_out("'options' no longer a supported builder constructor argument.")
|
||
|
|
||
|
if self.config.has_section("requirements"):
|
||
|
if self.config.has_option("requirements", "tito"):
|
||
|
if loose_version(self.config.get("requirements", "tito")) > \
|
||
|
loose_version(require('tito')[0].version):
|
||
|
error_out([
|
||
|
"tito version %s or later is needed to build this project." %
|
||
|
self.config.get("requirements", "tito"),
|
||
|
"Your version: %s" % require('tito')[0].version
|
||
|
])
|
||
|
|
||
|
self.display_version = self._get_display_version()
|
||
|
|
||
|
with chdir(find_git_root()):
|
||
|
self.git_commit_id = get_build_commit(tag=self.build_tag,
|
||
|
test=self.test)
|
||
|
|
||
|
self.relative_project_dir = get_relative_project_dir(
|
||
|
project_name=self.project_name, commit=self.git_commit_id)
|
||
|
if self.relative_project_dir is None and self.test:
|
||
|
warn_out(".tito/packages/%s doesn't exist "
|
||
|
"in git, using current directory" % self.project_name)
|
||
|
self.relative_project_dir = get_relative_project_dir_cwd(
|
||
|
self.git_root)
|
||
|
|
||
|
tgz_base = self._get_tgz_name_and_ver()
|
||
|
self.tgz_filename = tgz_base + ".tar.gz"
|
||
|
self.tgz_dir = tgz_base
|
||
|
self.artifacts = []
|
||
|
|
||
|
# A copy of the git code from commit we're building:
|
||
|
self.rpmbuild_gitcopy = os.path.join(self.rpmbuild_sourcedir,
|
||
|
self.tgz_dir)
|
||
|
|
||
|
# Used to make sure we only modify the spec file for a test build
|
||
|
# once. The srpm method may be called multiple times during koji
|
||
|
# releases to create the proper disttags, but we only want to modify
|
||
|
# the spec file once.
|
||
|
self.ran_setup_test_specfile = False
|
||
|
|
||
|
# NOTE: These are defined later when/if we actually dump a copy of the
|
||
|
# project source at the tag we're building. Only then can we search for
|
||
|
# a spec file.
|
||
|
self.spec_file_name = None
|
||
|
self.spec_file = None
|
||
|
|
||
|
# Set to path to srpm once we build one.
|
||
|
self.srpm_location = None
|
||
|
|
||
|
def _create_build_dirs(self):
|
||
|
"""
|
||
|
Create the build directories. Can safely be called multiple times.
|
||
|
"""
|
||
|
BuilderBase._create_build_dirs(self)
|
||
|
build_dirs = [
|
||
|
self.rpmbuild_gitcopy,
|
||
|
]
|
||
|
for d in build_dirs:
|
||
|
mkdir_p(d)
|
||
|
self._check_build_dirs_access(build_dirs)
|
||
|
|
||
|
def _get_build_version(self):
|
||
|
"""
|
||
|
Figure out the git tag and version-release we're building.
|
||
|
"""
|
||
|
# Determine which package version we should build:
|
||
|
build_version = None
|
||
|
if self.build_tag:
|
||
|
build_version = self.build_tag[len(self.project_name + "-"):]
|
||
|
else:
|
||
|
build_version = get_latest_tagged_version(self.project_name)
|
||
|
if build_version is None:
|
||
|
if not self.test:
|
||
|
error_out(["Unable to lookup latest package info.",
|
||
|
"Perhaps you need to tag first?"])
|
||
|
warn_out("unable to lookup latest package "
|
||
|
"tag, building untagged test project")
|
||
|
build_version = get_spec_version_and_release(self.start_dir,
|
||
|
find_spec_like_file(self.start_dir))
|
||
|
self.build_tag = self._get_tag_for_version(build_version)
|
||
|
|
||
|
self.spec_version = build_version.split('-')[0]
|
||
|
self.spec_release = build_version.split('-')[-1]
|
||
|
if not self.test:
|
||
|
check_tag_exists(self.build_tag, offline=self.offline)
|
||
|
return build_version
|
||
|
|
||
|
def _get_tag_for_version(self, version_and_release):
|
||
|
"""
|
||
|
Determine what the tag will look like for a given version.
|
||
|
Can be overridden when custom taggers override counterpart,
|
||
|
tito.Builder._get_tag_for_version().
|
||
|
"""
|
||
|
version = version_and_release.split('-')[0]
|
||
|
release = version_and_release.split('-')[-1]
|
||
|
if self.config.has_option(BUILDCONFIG_SECTION, "tag_format"):
|
||
|
tag_format = self.config.get(BUILDCONFIG_SECTION, "tag_format")
|
||
|
else:
|
||
|
tag_format = "{component}-{version}-{release}"
|
||
|
kwargs = {
|
||
|
'component': self.project_name,
|
||
|
'version': version,
|
||
|
'release': release
|
||
|
}
|
||
|
# Strip extra dashes if one of the params is empty
|
||
|
return tag_format.format(**kwargs).strip('-')
|
||
|
|
||
|
def tgz(self):
|
||
|
"""
|
||
|
Create the .tar.gz required to build this package.
|
||
|
|
||
|
Returns full path to the created tarball.
|
||
|
"""
|
||
|
self._setup_sources()
|
||
|
|
||
|
run_command("cp %s/%s %s/" %
|
||
|
(self.rpmbuild_sourcedir, self.tgz_filename,
|
||
|
self.rpmbuild_basedir))
|
||
|
|
||
|
self.ran_tgz = True
|
||
|
full_path = os.path.join(self.rpmbuild_basedir, self.tgz_filename)
|
||
|
info_out("Wrote: %s" % full_path)
|
||
|
self.sources.append(full_path)
|
||
|
self.artifacts.append(full_path)
|
||
|
return full_path
|
||
|
|
||
|
def rpm(self):
|
||
|
""" Build an RPM. """
|
||
|
self._create_build_dirs()
|
||
|
if not self.ran_tgz:
|
||
|
self.tgz()
|
||
|
if self.test:
|
||
|
self._setup_test_specfile()
|
||
|
BuilderBase.rpm(self)
|
||
|
|
||
|
def _setup_sources(self):
|
||
|
"""
|
||
|
Create a copy of the git source for the project at the point in time
|
||
|
our build tag was created.
|
||
|
|
||
|
Created in the temporary rpmbuild SOURCES directory.
|
||
|
"""
|
||
|
self._create_build_dirs()
|
||
|
|
||
|
debug("Creating %s from git tag: %s..." % (self.tgz_filename,
|
||
|
self.git_commit_id))
|
||
|
create_tgz(self.git_root, self.tgz_dir, self.git_commit_id,
|
||
|
self.relative_project_dir,
|
||
|
os.path.join(self.rpmbuild_sourcedir, self.tgz_filename))
|
||
|
|
||
|
# Extract the source so we can get at the spec file, etc.
|
||
|
debug("Copying git source to: %s" % self.rpmbuild_gitcopy)
|
||
|
run_command("cd %s/ && tar xzf %s" % (self.rpmbuild_sourcedir,
|
||
|
self.tgz_filename))
|
||
|
|
||
|
# Show contents of the directory structure we just extracted.
|
||
|
debug('', 'ls -lR %s/' % self.rpmbuild_gitcopy)
|
||
|
|
||
|
# NOTE: The spec file we actually use is the one exported by git
|
||
|
# archive into the temp build directory. This is done so we can
|
||
|
# modify the version/release on the fly when building test rpms
|
||
|
# that use a git SHA1 for their version.
|
||
|
self.spec_file_name = os.path.basename(find_spec_like_file(self.rpmbuild_gitcopy))
|
||
|
self.spec_file = os.path.join(
|
||
|
self.rpmbuild_gitcopy, self.spec_file_name)
|
||
|
|
||
|
def _setup_test_specfile(self):
|
||
|
if self.test and not self.ran_setup_test_specfile:
|
||
|
# If making a test rpm we need to get a little crazy with the spec
|
||
|
# file we're building off. (note that this is a temp copy of the
|
||
|
# spec) Swap out the actual release for one that includes the git
|
||
|
# SHA1 we're building for our test package:
|
||
|
sha = self.git_commit_id[:7]
|
||
|
fullname = "%s-%s" % (self.project_name, self.display_version)
|
||
|
munge_specfile(
|
||
|
self.spec_file,
|
||
|
sha,
|
||
|
self.commit_count,
|
||
|
fullname,
|
||
|
self.tgz_filename,
|
||
|
)
|
||
|
|
||
|
self.build_version += ".git." + str(self.commit_count) + "." + str(sha)
|
||
|
self.ran_setup_test_specfile = True
|
||
|
|
||
|
def _get_rpmbuild_dir_options(self):
|
||
|
return ('--define "_topdir %s" --define "_sourcedir %s" --define "_builddir %s" --define '
|
||
|
'"_srcrpmdir %s" --define "_rpmdir %s" ' % (
|
||
|
self.rpmbuild_dir,
|
||
|
self.rpmbuild_sourcedir, self.rpmbuild_builddir,
|
||
|
self.rpmbuild_basedir, self.rpmbuild_basedir))
|
||
|
|
||
|
def _get_tgz_name_and_ver(self):
|
||
|
"""
|
||
|
Returns the project name for the .tar.gz to build. Normally this is
|
||
|
just the project name, but in the case of Satellite packages it may
|
||
|
be different.
|
||
|
"""
|
||
|
return "%s-%s" % (self.project_name, self.display_version)
|
||
|
|
||
|
def _get_display_version(self):
|
||
|
"""
|
||
|
Get the package display version to build.
|
||
|
|
||
|
Normally this is whatever is .tito/packages/. In the case of a --test
|
||
|
build it will be the SHA1 for the HEAD commit of the current git
|
||
|
branch.
|
||
|
"""
|
||
|
if self.test:
|
||
|
# should get latest commit for given directory *NOT* HEAD
|
||
|
latest_commit = get_latest_commit(".")
|
||
|
self.commit_count = get_commit_count(self.build_tag, latest_commit)
|
||
|
version = "git-%s.%s" % (self.commit_count, latest_commit[:7])
|
||
|
else:
|
||
|
version = self.build_version.split("-")[0]
|
||
|
return version
|
||
|
|
||
|
|
||
|
class NoTgzBuilder(Builder):
|
||
|
"""
|
||
|
Builder for packages that do not require the creation of a tarball.
|
||
|
Usually these packages have source tarballs checked directly into git.
|
||
|
"""
|
||
|
|
||
|
def tgz(self):
|
||
|
""" Override parent behavior, we already have a tgz. """
|
||
|
# TODO: Does it make sense to allow user to create a tgz for this type
|
||
|
# of project?
|
||
|
self._setup_sources()
|
||
|
self.ran_tgz = True
|
||
|
|
||
|
debug("Scanning for sources.")
|
||
|
cmd = "/usr/bin/spectool --list-files '%s' | awk '{print $2}' |xargs -l1 --no-run-if-empty basename " % self.spec_file
|
||
|
result = run_command(cmd)
|
||
|
self.sources = []
|
||
|
for source in result.split("\n"):
|
||
|
self.sources.append(os.path.join(self.rpmbuild_gitcopy, source))
|
||
|
debug(" Sources: %s" % self.sources)
|
||
|
|
||
|
def _get_rpmbuild_dir_options(self):
|
||
|
"""
|
||
|
Override parent behavior slightly.
|
||
|
|
||
|
These packages store tar's, patches, etc, directly in their project
|
||
|
dir, use the git copy we create as the sources directory when
|
||
|
building package so everything can be found:
|
||
|
"""
|
||
|
return ('--define "_topdir %s" --define "_sourcedir %s" --define "_builddir %s" '
|
||
|
'--define "_srcrpmdir %s" --define "_rpmdir %s" ' % (
|
||
|
self.rpmbuild_dir,
|
||
|
self.rpmbuild_gitcopy, self.rpmbuild_builddir,
|
||
|
self.rpmbuild_basedir, self.rpmbuild_basedir))
|
||
|
|
||
|
def _setup_test_specfile(self):
|
||
|
""" Override parent behavior. """
|
||
|
if self.test:
|
||
|
# If making a test rpm we need to get a little crazy with the spec
|
||
|
# file we're building off. (note that this is a temp copy of the
|
||
|
# spec) Swap out the actual release for one that includes the git
|
||
|
# SHA1 we're building for our test package:
|
||
|
debug("setup_test_specfile:commit_count = %s" % str(self.commit_count))
|
||
|
munge_specfile(
|
||
|
self.spec_file,
|
||
|
self.git_commit_id[:7],
|
||
|
self.commit_count
|
||
|
)
|
||
|
|
||
|
|
||
|
class GemBuilder(NoTgzBuilder):
|
||
|
"""
|
||
|
Gem Builder
|
||
|
|
||
|
Builder for packages whose sources are managed as gem source structures
|
||
|
and the upstream project does not want to store gem files in git.
|
||
|
"""
|
||
|
|
||
|
def _setup_sources(self):
|
||
|
"""
|
||
|
Create a copy of the git source for the project at the point in time
|
||
|
our build tag was created.
|
||
|
|
||
|
Created in the temporary rpmbuild SOURCES directory.
|
||
|
"""
|
||
|
self._create_build_dirs()
|
||
|
|
||
|
debug("Creating %s from git tag: %s..." % (self.tgz_filename,
|
||
|
self.git_commit_id))
|
||
|
create_tgz(self.git_root, self.tgz_dir, self.git_commit_id,
|
||
|
self.relative_project_dir,
|
||
|
os.path.join(self.rpmbuild_sourcedir, self.tgz_filename))
|
||
|
|
||
|
# Extract the source so we can get at the spec file, etc.
|
||
|
debug("Copying git source to: %s" % self.rpmbuild_gitcopy)
|
||
|
run_command("cd %s/ && tar xzf %s" % (self.rpmbuild_sourcedir,
|
||
|
self.tgz_filename))
|
||
|
|
||
|
# Find the gemspec
|
||
|
gemspec_filename = find_gemspec_file(self.rpmbuild_gitcopy)
|
||
|
|
||
|
debug("Building gem: %s in %s" % (gemspec_filename,
|
||
|
self.rpmbuild_gitcopy))
|
||
|
# FIXME - this is ugly and should probably be handled better
|
||
|
cmd = "gem_name=$(cd %s/ && gem build %s | awk '/File/ {print $2}'); \
|
||
|
cp %s/$gem_name %s/" % (self.rpmbuild_gitcopy, gemspec_filename,
|
||
|
self.rpmbuild_gitcopy, self.rpmbuild_sourcedir)
|
||
|
|
||
|
run_command(cmd)
|
||
|
|
||
|
# NOTE: The spec file we actually use is the one exported by git
|
||
|
# archive into the temp build directory. This is done so we can
|
||
|
# modify the version/release on the fly when building test rpms
|
||
|
# that use a git SHA1 for their version.
|
||
|
self.spec_file_name = find_spec_file(self.rpmbuild_gitcopy)
|
||
|
self.spec_file = os.path.join(
|
||
|
self.rpmbuild_gitcopy, self.spec_file_name)
|
||
|
|
||
|
|
||
|
class UpstreamBuilder(NoTgzBuilder):
|
||
|
"""
|
||
|
Builder for packages that are based off an upstream git tag.
|
||
|
Commits applied in downstream git become patches applied to the
|
||
|
upstream tarball.
|
||
|
|
||
|
i.e. satellite-java-0.4.0-5 built from spacewalk-java-0.4.0-1 and any
|
||
|
patches applied in satellite git.
|
||
|
i.e. spacewalk-setup-0.4.0-20 built from spacewalk-setup-0.4.0-1 and any
|
||
|
patches applied in satellite git.
|
||
|
"""
|
||
|
|
||
|
def __init__(self, name=None, tag=None, build_dir=None,
|
||
|
config=None, user_config=None,
|
||
|
args=None, **kwargs):
|
||
|
|
||
|
NoTgzBuilder.__init__(self, name=name, tag=tag,
|
||
|
build_dir=build_dir, config=config,
|
||
|
user_config=user_config,
|
||
|
args=args, **kwargs)
|
||
|
|
||
|
if not config or not config.has_option("buildconfig",
|
||
|
"upstream_name"):
|
||
|
# No upstream_name defined, assume we're keeping the project name:
|
||
|
self.upstream_name = self.project_name
|
||
|
else:
|
||
|
self.upstream_name = config.get("buildconfig", "upstream_name")
|
||
|
# Need to assign these after we've exported a copy of the spec file:
|
||
|
self.upstream_version = None
|
||
|
self.upstream_tag = None
|
||
|
|
||
|
def tgz(self):
|
||
|
"""
|
||
|
Override parent behavior, we need a tgz from the upstream spacewalk
|
||
|
project we're based on.
|
||
|
"""
|
||
|
# TODO: Wasteful step here, all we really need is a way to look for a
|
||
|
# spec file at the point in time this release was tagged.
|
||
|
NoTgzBuilder._setup_sources(self)
|
||
|
# If we knew what it was named at that point in time we could just do:
|
||
|
# Export a copy of our spec file at the revision to be built:
|
||
|
# cmd = "git show %s:%s%s > %s" % (self.git_commit_id,
|
||
|
# self.relative_project_dir, self.spec_file_name,
|
||
|
# self.spec_file)
|
||
|
# debug(cmd)
|
||
|
self._create_build_dirs()
|
||
|
|
||
|
self.upstream_version = self._get_upstream_version()
|
||
|
self.upstream_tag = "%s-%s-1" % (self.upstream_name,
|
||
|
self.upstream_version)
|
||
|
|
||
|
print("Building upstream tgz for tag [%s]" % (self.upstream_tag))
|
||
|
if self.upstream_tag != self.build_tag:
|
||
|
check_tag_exists(self.upstream_tag, offline=self.offline)
|
||
|
|
||
|
self.spec_file = os.path.join(self.rpmbuild_sourcedir,
|
||
|
self.spec_file_name)
|
||
|
command = "cp %s %s" % (os.path.join(self.rpmbuild_gitcopy,
|
||
|
self.spec_file_name), self.spec_file)
|
||
|
run_command(command)
|
||
|
|
||
|
# Create the upstream tgz:
|
||
|
prefix = "%s-%s" % (self.upstream_name, self.upstream_version)
|
||
|
tgz_filename = "%s.tar.gz" % prefix
|
||
|
commit = get_build_commit(tag=self.upstream_tag)
|
||
|
relative_dir = get_relative_project_dir(
|
||
|
project_name=self.upstream_name, commit=commit)
|
||
|
tgz_fullpath = os.path.join(self.rpmbuild_sourcedir, tgz_filename)
|
||
|
print("Creating %s from git tag: %s..." % (tgz_filename, commit))
|
||
|
create_tgz(self.git_root, prefix, commit, relative_dir,
|
||
|
tgz_fullpath)
|
||
|
self.ran_tgz = True
|
||
|
self.sources.append(tgz_fullpath)
|
||
|
|
||
|
# If these are equal then the tag we're building was likely created in
|
||
|
# Spacewalk and thus we don't need to do any patching.
|
||
|
if (self.upstream_tag == self.build_tag and not self.test):
|
||
|
return
|
||
|
|
||
|
self.patch_upstream()
|
||
|
|
||
|
def _patch_upstream(self):
|
||
|
""" Insert patches into the spec file we'll be building
|
||
|
returns (patch_number, patch_insert_index, patch_apply_index, lines)
|
||
|
"""
|
||
|
f = open(self.spec_file, 'r')
|
||
|
lines = f.readlines()
|
||
|
f.close()
|
||
|
|
||
|
patch_pattern = re.compile('^Patch(\d+):')
|
||
|
source_pattern = re.compile('^Source(\d+)?:')
|
||
|
|
||
|
# Find the largest PatchX: line, or failing that SourceX:
|
||
|
patch_number = 0 # What number should we use for our PatchX line
|
||
|
patch_insert_index = 0 # Where to insert our PatchX line in the list
|
||
|
patch_apply_index = 0 # Where to insert our %patchX line in the list
|
||
|
array_index = 0 # Current index in the array
|
||
|
for line in lines:
|
||
|
match = source_pattern.match(line)
|
||
|
if match:
|
||
|
patch_insert_index = array_index + 1
|
||
|
|
||
|
match = patch_pattern.match(line)
|
||
|
if match:
|
||
|
patch_insert_index = array_index + 1
|
||
|
patch_number = int(match.group(1)) + 1
|
||
|
|
||
|
if line.startswith("%prep"):
|
||
|
# We'll apply patch right after prep if there's no %setup line
|
||
|
patch_apply_index = array_index + 2
|
||
|
elif line.startswith("%setup"):
|
||
|
patch_apply_index = array_index + 2 # already added a line
|
||
|
elif line.startswith("%autosetup"):
|
||
|
patch_apply_index = -1 # autosetup will do this for us
|
||
|
|
||
|
array_index += 1
|
||
|
|
||
|
debug("patch_insert_index = %s" % patch_insert_index)
|
||
|
debug("patch_apply_index = %s" % patch_apply_index)
|
||
|
if patch_insert_index == 0 or patch_apply_index == 0:
|
||
|
error_out("Unable to insert PatchX or %patchX lines in spec file")
|
||
|
return (patch_number, patch_insert_index, patch_apply_index, lines)
|
||
|
|
||
|
def patch_upstream(self):
|
||
|
"""
|
||
|
Generate patches for any differences between our tag and the
|
||
|
upstream tag, and apply them into an exported copy of the
|
||
|
spec file.
|
||
|
"""
|
||
|
patch_filename = "%s-to-%s-%s.patch" % (self.upstream_tag,
|
||
|
self.project_name, self.build_version)
|
||
|
patch_file = os.path.join(self.rpmbuild_gitcopy,
|
||
|
patch_filename)
|
||
|
patch_dir = self.git_root
|
||
|
if self.relative_project_dir != "/":
|
||
|
patch_dir = os.path.join(self.git_root,
|
||
|
self.relative_project_dir)
|
||
|
os.chdir(patch_dir)
|
||
|
debug("patch dir = %s" % patch_dir)
|
||
|
print("Generating patch [%s]" % patch_filename)
|
||
|
debug("Patch: %s" % patch_file)
|
||
|
patch_command = "git diff --relative %s..%s > %s" % \
|
||
|
(self.upstream_tag, self.git_commit_id,
|
||
|
patch_file)
|
||
|
debug("Generating patch with: %s" % patch_command)
|
||
|
output = run_command(patch_command)
|
||
|
print(output)
|
||
|
(status, output) = getstatusoutput(
|
||
|
"grep 'Binary files .* differ' %s " % patch_file)
|
||
|
if status == 0 and output != "":
|
||
|
error_out("You are doomed. Diff contains binary files. You can not use this builder")
|
||
|
|
||
|
# Creating two copies of the patch here in the temp build directories
|
||
|
# just out of laziness. Some builders need sources in SOURCES and
|
||
|
# others need them in the git copy. Being lazy here avoids one-off
|
||
|
# hacks and both copies get cleaned up anyhow.
|
||
|
run_command("cp %s %s" % (patch_file, self.rpmbuild_sourcedir))
|
||
|
|
||
|
(patch_number, patch_insert_index, patch_apply_index, lines) = self._patch_upstream()
|
||
|
|
||
|
lines.insert(patch_insert_index, "Patch%s: %s\n" % (patch_number,
|
||
|
patch_filename))
|
||
|
if patch_apply_index > 0:
|
||
|
lines.insert(patch_apply_index, "%%patch%s -p1\n" % (patch_number))
|
||
|
self._write_spec(lines)
|
||
|
|
||
|
def _write_spec(self, lines):
|
||
|
""" Write 'lines' to self.spec_file """
|
||
|
# Now write out the modified lines to the spec file copy:
|
||
|
f = open(self.spec_file, 'w')
|
||
|
for line in lines:
|
||
|
f.write(line)
|
||
|
f.close()
|
||
|
|
||
|
def _get_upstream_version(self):
|
||
|
"""
|
||
|
Get the upstream version. Checks for "upstreamversion" in the spec file
|
||
|
and uses it if found. Otherwise assumes the upstream version is equal
|
||
|
to the version we're building.
|
||
|
|
||
|
i.e. satellite-java-0.4.15 will be built on spacewalk-java-0.4.15
|
||
|
with just the package release being incremented on rebuilds.
|
||
|
"""
|
||
|
# Use upstreamversion if defined in the spec file:
|
||
|
(status, output) = getstatusoutput(
|
||
|
"cat %s | grep 'define upstreamversion' | "
|
||
|
"awk '{ print $3 ; exit }'" % self.spec_file)
|
||
|
if status == 0 and output != "":
|
||
|
return output
|
||
|
|
||
|
if self.test:
|
||
|
return self.build_version.split("-")[0]
|
||
|
# Otherwise, assume we use our version:
|
||
|
else:
|
||
|
return self.display_version
|
||
|
|
||
|
def _get_rpmbuild_dir_options(self):
|
||
|
"""
|
||
|
Override parent behavior slightly.
|
||
|
|
||
|
These packages store tar's, patches, etc, directly in their project
|
||
|
dir, use the git copy we create as the sources directory when
|
||
|
building package so everything can be found:
|
||
|
"""
|
||
|
return ('--define "_topdir %s" --define "_sourcedir %s" --define "_builddir %s" '
|
||
|
'--define "_srcrpmdir %s" --define "_rpmdir %s" ' % (
|
||
|
self.rpmbuild_dir,
|
||
|
self.rpmbuild_sourcedir, self.rpmbuild_builddir,
|
||
|
self.rpmbuild_basedir, self.rpmbuild_basedir))
|
||
|
|
||
|
|
||
|
# Legacy class name for backward compatability:
|
||
|
class SatelliteBuilder(UpstreamBuilder):
|
||
|
pass
|
||
|
|
||
|
|
||
|
class MeadBuilder(Builder):
|
||
|
def __init__(self, name=None, tag=None, build_dir=None,
|
||
|
config=None, user_config=None, args=None, **kwargs):
|
||
|
|
||
|
Builder.__init__(self, name=name, tag=tag, build_dir=build_dir,
|
||
|
config=config, user_config=user_config, args=args, **kwargs)
|
||
|
|
||
|
self.ran_maven = False
|
||
|
self.deploy_dir = mkdtemp(dir=self.rpmbuild_basedir, prefix="maven-deploy-%s" % self.project_name)
|
||
|
self.maven_clone_dir = mkdtemp(dir=self.rpmbuild_basedir, prefix="maven-clone-%s" % self.project_name)
|
||
|
|
||
|
# People calling `tito build` will almost certainly want to do a maven build locally.
|
||
|
# But with a `tito release` we want the Mead stuff to happen on the build system
|
||
|
self.local_build = args.setdefault('local', [True])
|
||
|
|
||
|
self.maven_properties = []
|
||
|
if 'maven_property' in args:
|
||
|
self.maven_properties = args['maven_property']
|
||
|
else:
|
||
|
# Generally people aren't going to want to run their tests during
|
||
|
# a build. If they do, they can set maven_properties=''
|
||
|
self.maven_properties.append("maven.test.skip=true")
|
||
|
|
||
|
self.maven_args = ['-B']
|
||
|
if 'maven_arg' in args:
|
||
|
self.maven_args.extend(args['maven_arg'])
|
||
|
|
||
|
def _find_tarball(self):
|
||
|
for directory, unused, filenames in os.walk(self.deploy_dir):
|
||
|
for f in filenames:
|
||
|
name, ext = os.path.splitext(f)
|
||
|
if ext == ".gz" and name.startswith("%s-%s" % (self.project_name, self.spec_version)):
|
||
|
return os.path.join(self.deploy_dir, directory, f)
|
||
|
return None
|
||
|
|
||
|
def cleanup(self):
|
||
|
"""
|
||
|
Remove all temporary files and directories.
|
||
|
"""
|
||
|
if not self.no_cleanup:
|
||
|
for d in [self.rpmbuild_dir, self.deploy_dir, self.maven_clone_dir]:
|
||
|
debug("Cleaning up %s" % d)
|
||
|
shutil.rmtree(d)
|
||
|
else:
|
||
|
warn_out("Leaving rpmbuild files in: %s" % self.rpmbuild_dir)
|
||
|
|
||
|
def tgz(self):
|
||
|
destination_file = os.path.join(self.rpmbuild_basedir, self.tgz_filename)
|
||
|
formatted_properties = ["-D%s" % x for x in self.maven_properties]
|
||
|
|
||
|
run_command("git clone --no-hardlinks %s %s" % (find_git_root(), self.maven_clone_dir))
|
||
|
with chdir(self.maven_clone_dir):
|
||
|
run_command("git checkout %s" % self.git_commit_id)
|
||
|
|
||
|
try:
|
||
|
info_out("Running Maven build...")
|
||
|
# We always want to deploy to a tito controlled location during local builds
|
||
|
local_properties = formatted_properties + [
|
||
|
"-DaltDeploymentRepository=local-output::default::file://%s" % self.deploy_dir]
|
||
|
run_command("mvn %s %s deploy" % (
|
||
|
" ".join(self.maven_args),
|
||
|
" ".join(local_properties)))
|
||
|
except RunCommandException as e:
|
||
|
error_out("Maven build failed! %s" % e.output)
|
||
|
|
||
|
self._create_build_dirs()
|
||
|
|
||
|
full_path = self._find_tarball()
|
||
|
if full_path:
|
||
|
fh = gzip.open(full_path, 'rb')
|
||
|
fixed_tar = os.path.join(os.path.splitext(full_path)[0])
|
||
|
fixed_tar_fh = open(fixed_tar, 'wb')
|
||
|
timestamp = get_commit_timestamp(self.git_commit_id)
|
||
|
try:
|
||
|
tarfixer = TarFixer(fh, fixed_tar_fh, timestamp, self.git_commit_id, maven_built=True)
|
||
|
tarfixer.fix()
|
||
|
finally:
|
||
|
fixed_tar_fh.close()
|
||
|
|
||
|
# It's a pity we can't use Python's gzip, but it doesn't offer an equivalent of -n
|
||
|
run_command("gzip -n -c < %s > %s" % (fixed_tar, destination_file))
|
||
|
else:
|
||
|
warn_out([
|
||
|
"No Maven generated tarball found.",
|
||
|
"Please set up the assembly plugin in your pom.xml to generate a .tar.gz"])
|
||
|
full_path = os.path.join(self.rpmbuild_sourcedir, self.tgz_filename)
|
||
|
create_tgz(self.git_root, self.tgz_dir, self.git_commit_id, self.relative_project_dir, full_path)
|
||
|
print("Creating %s from git tag: %s..." % (self.tgz_filename, self.build_tag))
|
||
|
shutil.copy(full_path, destination_file)
|
||
|
|
||
|
debug("Copying git source to: %s" % self.rpmbuild_gitcopy)
|
||
|
shutil.copy(destination_file, self.rpmbuild_gitcopy)
|
||
|
|
||
|
# Extract the source so we can get at the spec file, etc.
|
||
|
with chdir(self.rpmbuild_gitcopy):
|
||
|
run_command("tar --strip-components=1 -xvf %s" % os.path.join(self.rpmbuild_gitcopy, self.tgz_filename))
|
||
|
|
||
|
if self.local_build:
|
||
|
artifacts = {}
|
||
|
all_artifacts = []
|
||
|
all_artifacts_with_path = []
|
||
|
|
||
|
for directory, unused, filenames in os.walk(self.deploy_dir):
|
||
|
for f in filenames:
|
||
|
artifacts.setdefault(os.path.splitext(f)[1], []).append(f)
|
||
|
dir_artifacts_with_path = [os.path.join(directory, f) for f in filenames]
|
||
|
|
||
|
# Place the Maven artifacts in the SOURCES directory for rpmbuild to use
|
||
|
for artifact in dir_artifacts_with_path:
|
||
|
shutil.copy(artifact, self.rpmbuild_sourcedir)
|
||
|
|
||
|
dir_artifacts_with_path = map(lambda x: os.path.relpath(x, self.deploy_dir), dir_artifacts_with_path)
|
||
|
all_artifacts_with_path.extend(dir_artifacts_with_path)
|
||
|
all_artifacts.extend([os.path.basename(f) for f in filenames])
|
||
|
|
||
|
cheetah_input = {
|
||
|
'name': self.project_name,
|
||
|
'version': self.spec_version,
|
||
|
'release': self.spec_release,
|
||
|
'epoch': None, # TODO: May need to support this at some point
|
||
|
'artifacts': artifacts,
|
||
|
'all_artifacts': all_artifacts,
|
||
|
'all_artifacts_with_path': all_artifacts_with_path,
|
||
|
}
|
||
|
debug("Cheetah input: %s" % cheetah_input)
|
||
|
render_cheetah(find_cheetah_template_file(self.start_dir), self.rpmbuild_gitcopy, cheetah_input)
|
||
|
self.spec_file_name = find_spec_file(self.rpmbuild_gitcopy)
|
||
|
else:
|
||
|
self.spec_file_name = find_cheetah_template_file(self.rpmbuild_gitcopy)
|
||
|
|
||
|
# NOTE: The spec file we actually use is the one exported by git
|
||
|
# archive into the temp build directory. This is done so we can
|
||
|
# modify the version/release on the fly when building test rpms
|
||
|
# that use a git SHA1 for their version.
|
||
|
self.spec_file = os.path.join(self.rpmbuild_gitcopy, self.spec_file_name)
|
||
|
|
||
|
info_out("Wrote: %s" % destination_file)
|
||
|
self.sources.append(destination_file)
|
||
|
self.artifacts.append(destination_file)
|
||
|
self.ran_tgz = True
|
||
|
|
||
|
def _setup_test_specfile(self):
|
||
|
if self.test and not self.ran_setup_test_specfile:
|
||
|
# If making a test rpm we need to get a little crazy with the spec
|
||
|
# file we're building off. (note that this is a temp copy of the
|
||
|
# spec) Swap out the actual release for one that includes the git
|
||
|
# SHA1 we're building for our test package:
|
||
|
self.build_version += ".git." + str(self.commit_count) + "." + str(self.git_commit_id[:7])
|
||
|
replace_spec_release(self.spec_file, self.spec_release)
|
||
|
self.ran_setup_test_specfile = True
|
||
|
|
||
|
|
||
|
class MockBuilder(Builder):
|
||
|
"""
|
||
|
Uses the mock tool to create a chroot for building packages for a different
|
||
|
OS version than you may be currently using.
|
||
|
"""
|
||
|
REQUIRED_ARGS = ['mock']
|
||
|
|
||
|
def __init__(self, name=None, tag=None, build_dir=None,
|
||
|
config=None, user_config=None,
|
||
|
args=None, **kwargs):
|
||
|
|
||
|
# Mock builders need to use the packages normally configured builder
|
||
|
# to get at a proper SRPM:
|
||
|
self.normal_builder = create_builder(name, tag, config,
|
||
|
build_dir, user_config, args, **kwargs)
|
||
|
|
||
|
Builder.__init__(self, name=name, tag=tag,
|
||
|
build_dir=build_dir, config=config,
|
||
|
user_config=user_config,
|
||
|
args=args, **kwargs)
|
||
|
|
||
|
self.mock_tag = args['mock'][0]
|
||
|
self.mock_cmd_args = ""
|
||
|
if 'mock_config_dir' in args:
|
||
|
mock_config_dir = args['mock_config_dir'][0]
|
||
|
if not mock_config_dir.startswith("/"):
|
||
|
# If not an absolute path, assume below git root:
|
||
|
mock_config_dir = os.path.join(self.git_root, mock_config_dir)
|
||
|
if not os.path.exists(mock_config_dir):
|
||
|
raise TitoException("No such mock config dir: %s" % mock_config_dir)
|
||
|
self.mock_cmd_args = "%s --configdir=%s" % (self.mock_cmd_args, mock_config_dir)
|
||
|
|
||
|
# Optional argument which will skip mock --init and add --no-clean
|
||
|
# and --no-cleanup-after:
|
||
|
self.speedup = False
|
||
|
if 'speedup' in args:
|
||
|
self.speedup = True
|
||
|
self.mock_cmd_args = "%s --no-clean --no-cleanup-after" % \
|
||
|
(self.mock_cmd_args)
|
||
|
|
||
|
if 'mock_args' in args:
|
||
|
self.mock_cmd_args = "%s %s" % (self.mock_cmd_args, args['mock_args'][0])
|
||
|
|
||
|
# TODO: error out if mock package is not installed
|
||
|
|
||
|
# TODO: error out if user does not have mock group
|
||
|
|
||
|
def srpm(self, dist=None):
|
||
|
"""
|
||
|
Build a source RPM.
|
||
|
|
||
|
MockBuilder will use an instance of the normal builder for a package
|
||
|
internally just so we can generate a SRPM correctly before we pass it
|
||
|
into mock.
|
||
|
"""
|
||
|
self.normal_builder.srpm(dist)
|
||
|
self.srpm_location = self.normal_builder.srpm_location
|
||
|
self.artifacts.append(self.srpm_location)
|
||
|
|
||
|
def rpm(self):
|
||
|
"""
|
||
|
Uses the SRPM
|
||
|
Override the base builder rpm method.
|
||
|
"""
|
||
|
|
||
|
print("Creating rpms for %s-%s in mock: %s" % (
|
||
|
self.project_name, self.display_version, self.mock_tag))
|
||
|
if not self.srpm_location:
|
||
|
self.srpm()
|
||
|
print("Using srpm: %s" % self.srpm_location)
|
||
|
self._build_in_mock()
|
||
|
|
||
|
def cleanup(self):
|
||
|
if self.normal_builder:
|
||
|
self.normal_builder.cleanup()
|
||
|
|
||
|
def _build_in_mock(self):
|
||
|
run_command_func = run_command if self.quiet else run_command_print
|
||
|
if not self.speedup:
|
||
|
print("Initializing mock...")
|
||
|
run_command_func("mock %s -r %s --init" % (self.mock_cmd_args, self.mock_tag))
|
||
|
else:
|
||
|
print("Skipping mock --init due to speedup option.")
|
||
|
|
||
|
print("Building RPMs in mock...")
|
||
|
run_command_func('mock %s -r %s --rebuild %s' %
|
||
|
(self.mock_cmd_args, self.mock_tag, self.srpm_location))
|
||
|
mock_output_dir = os.path.join(self.rpmbuild_dir, "mockoutput")
|
||
|
run_command_func("mock %s -r %s --copyout /builddir/build/RPMS/ %s" %
|
||
|
(self.mock_cmd_args, self.mock_tag, mock_output_dir))
|
||
|
|
||
|
# Copy everything mock wrote out to /tmp/tito:
|
||
|
files = os.listdir(mock_output_dir)
|
||
|
for rpm in files:
|
||
|
shutil.copy2(os.path.join(mock_output_dir, rpm), self.rpmbuild_basedir)
|
||
|
print
|
||
|
info_out("Wrote:")
|
||
|
for rpm in files:
|
||
|
rpm_path = os.path.join(self.rpmbuild_basedir, rpm)
|
||
|
print(" %s" % rpm_path)
|
||
|
self.artifacts.append(rpm_path)
|
||
|
print
|
||
|
|
||
|
|
||
|
class BrewDownloadBuilder(Builder):
|
||
|
"""
|
||
|
A special case builder which uses pre-existing Brew builds and
|
||
|
pulls down the resulting rpms locally. Useful in some cases when
|
||
|
generating yum repositories during a release.
|
||
|
"""
|
||
|
REQUIRED_ARGS = ['disttag']
|
||
|
|
||
|
def __init__(self, name=None, tag=None, build_dir=None,
|
||
|
config=None, user_config=None,
|
||
|
args=None, **kwargs):
|
||
|
|
||
|
Builder.__init__(self, name=name, tag=tag,
|
||
|
build_dir=build_dir, config=config,
|
||
|
user_config=user_config,
|
||
|
args=args, **kwargs)
|
||
|
|
||
|
self.dist_tag = args['disttag'][0]
|
||
|
|
||
|
def rpm(self):
|
||
|
"""
|
||
|
Uses the SRPM
|
||
|
Override the base builder rpm method.
|
||
|
"""
|
||
|
|
||
|
print("Fetching rpms for %s.%s from brew:" % (
|
||
|
self.build_tag, self.dist_tag))
|
||
|
self._fetch_from_brew()
|
||
|
|
||
|
def _fetch_from_brew(self):
|
||
|
brew_nvr = "%s.%s" % (self.build_tag, self.dist_tag)
|
||
|
debug("Brew NVR: %s" % brew_nvr)
|
||
|
os.chdir(self.rpmbuild_dir)
|
||
|
run_command("brew download-build %s" % brew_nvr)
|
||
|
|
||
|
# Wipe out the src rpm for now:
|
||
|
run_command("rm *.src.rpm")
|
||
|
|
||
|
# Copy everything brew downloaded out to /tmp/tito:
|
||
|
files = os.listdir(self.rpmbuild_dir)
|
||
|
run_command("cp -v %s/*.rpm %s" %
|
||
|
(self.rpmbuild_dir, self.rpmbuild_basedir))
|
||
|
print
|
||
|
info_out("Wrote:")
|
||
|
for rpm in files:
|
||
|
# Just incase anything slips into the build dir:
|
||
|
if not rpm.endswith(".rpm"):
|
||
|
continue
|
||
|
rpm_path = os.path.join(self.rpmbuild_basedir, rpm)
|
||
|
print(" %s" % rpm_path)
|
||
|
self.artifacts.append(rpm_path)
|
||
|
print
|
||
|
|
||
|
|
||
|
class GitAnnexBuilder(NoTgzBuilder):
|
||
|
"""
|
||
|
Builder for packages with existing tarballs checked in using git-annex,
|
||
|
e.g. referencing an external source (web remote). This builder will
|
||
|
"unlock" the source files to get the real contents, include them in the
|
||
|
SRPM, then restore the automatic git-annex symlinks on completion.
|
||
|
"""
|
||
|
|
||
|
def _setup_sources(self):
|
||
|
super(GitAnnexBuilder, self)._setup_sources()
|
||
|
|
||
|
self.old_cwd = os.getcwd()
|
||
|
os.chdir(os.path.join(self.old_cwd, self.relative_project_dir))
|
||
|
|
||
|
# NOTE: 'which' may not be installed... (docker containers)
|
||
|
(status, output) = getstatusoutput("which git-annex")
|
||
|
if status != 0:
|
||
|
msg = "Please run '%s' as root." % self.package_manager.install(["git-annex"])
|
||
|
error_out('%s' % msg)
|
||
|
|
||
|
run_command("git-annex lock")
|
||
|
annexed_files = run_command("git-annex find --include='*'").splitlines()
|
||
|
run_command("git-annex get")
|
||
|
run_command("git-annex unlock")
|
||
|
debug(" Annex files: %s" % annexed_files)
|
||
|
|
||
|
for annex in annexed_files:
|
||
|
debug("Copying unlocked file %s" % annex)
|
||
|
os.remove(os.path.join(self.rpmbuild_gitcopy, annex))
|
||
|
shutil.copy(annex, self.rpmbuild_gitcopy)
|
||
|
|
||
|
self._lock()
|
||
|
os.chdir(self.old_cwd)
|
||
|
|
||
|
def cleanup(self):
|
||
|
if hasattr(self, 'old_cwd'):
|
||
|
os.chdir(self.old_cwd)
|
||
|
self._lock()
|
||
|
super(GitAnnexBuilder, self).cleanup()
|
||
|
|
||
|
def _lock(self):
|
||
|
if self._lock_force_supported(self._get_annex_version()):
|
||
|
run_command("git-annex lock --force")
|
||
|
else:
|
||
|
run_command("git-annex lock")
|
||
|
|
||
|
def _get_annex_version(self):
|
||
|
# git-annex needs to support --force when locking files.
|
||
|
ga_version = run_command('git-annex version').split('\n')
|
||
|
if ga_version[0].startswith('git-annex version'):
|
||
|
return ga_version[0].split()[-1]
|
||
|
else:
|
||
|
return 0
|
||
|
|
||
|
def _lock_force_supported(self, version):
|
||
|
return compare_version(version, '5.20131213') >= 0
|
||
|
|
||
|
|
||
|
class GitLfsBuilder(NoTgzBuilder):
|
||
|
"""
|
||
|
Builder class for Git LFS support.
|
||
|
"""
|
||
|
|
||
|
def _setup_sources(self):
|
||
|
super(GitLfsBuilder, self)._setup_sources()
|
||
|
|
||
|
self.old_cwd = os.getcwd()
|
||
|
os.chdir(os.path.join(self.old_cwd, self.relative_project_dir))
|
||
|
debug("learning %s" % self.relative_project_dir)
|
||
|
(status, output) = getstatusoutput("which git-lfs")
|
||
|
if status != 0:
|
||
|
msg = "Please run '%s' as root." % self.package_manager.install(["git-lfs"])
|
||
|
error_out('%s' % msg)
|
||
|
|
||
|
gitlfs_files = run_command("git-lfs ls-files").splitlines()
|
||
|
run_command("git-lfs fetch")
|
||
|
debug(" Lfs files: %s" % gitlfs_files)
|
||
|
for gfile in gitlfs_files:
|
||
|
m = re.search('(?<=' + self.relative_project_dir + ')(.*)', gfile)
|
||
|
if m:
|
||
|
filterfile = m.groups(0)
|
||
|
mstr = filterfile[0]
|
||
|
debug("Copying file %s " % mstr)
|
||
|
debug("To %s " % self.rpmbuild_gitcopy)
|
||
|
os.remove(os.path.join(self.rpmbuild_gitcopy, mstr))
|
||
|
shutil.copy(mstr, self.rpmbuild_gitcopy)
|
||
|
|
||
|
os.chdir(self.old_cwd)
|
||
|
|
||
|
def cleanup(self):
|
||
|
if hasattr(self, 'old_cwd'):
|
||
|
os.chdir(self.old_cwd)
|
||
|
super(GitLfsBuilder, self).cleanup()
|
||
|
|
||
|
|
||
|
def package_manager():
|
||
|
if os.path.isfile("/usr/bin/dnf"):
|
||
|
return Dnf()
|
||
|
if os.path.isfile("/usr/bin/yum"):
|
||
|
return Yum()
|
||
|
return Rpm()
|
||
|
|
||
|
|
||
|
class Rpm(object):
|
||
|
def install(self, packages, escalate=True, **kwargs):
|
||
|
escalation_cmd = "sudo" if escalate else ""
|
||
|
return "%s rpm -U --force %s" % (escalation_cmd, ' '.join(packages))
|
||
|
|
||
|
def builddep(self, spec):
|
||
|
raise NotImplementedError
|
||
|
|
||
|
def is_installed(self, package, version):
|
||
|
q = self.query(package)
|
||
|
if not q:
|
||
|
return False
|
||
|
|
||
|
iv = "%s-%s" % (q.version.decode("utf-8"), q.release.decode("utf-8"))
|
||
|
iv_short = ".".join(iv.split(".")[:-1])
|
||
|
return version == iv_short
|
||
|
|
||
|
def query(self, package):
|
||
|
ts = rpm.TransactionSet()
|
||
|
results = list(ts.dbMatch("name", package))
|
||
|
return results[0] if results else None
|
||
|
|
||
|
|
||
|
class Dnf(Rpm):
|
||
|
def install(self, packages, reinstall=False, auto=False, offline=False, escalate=True, **kwargs):
|
||
|
action = "reinstall" if reinstall else "install"
|
||
|
args = list(filter(lambda x: x, [
|
||
|
"-C" if offline else None,
|
||
|
"-y" if auto else None,
|
||
|
]))
|
||
|
escalation_cmd = "sudo" if escalate else ""
|
||
|
cmd = "%s dnf %s %s" % (escalation_cmd, action, " ".join(args + packages))
|
||
|
return " ".join(cmd.split())
|
||
|
|
||
|
def builddep(self, spec):
|
||
|
return "dnf builddep %s" % spec
|
||
|
|
||
|
|
||
|
class Yum(Rpm):
|
||
|
def install(self, packages, **kwargs):
|
||
|
# Not the sexiest implementation, but very short
|
||
|
return Dnf().install(packages, **kwargs).replace("dnf", "yum")
|
||
|
|
||
|
def builddep(self, spec):
|
||
|
return "yum-builddep %s" % spec
|