"""
The Gentoo module for running tests on builds from the portage tree.
This will install a stage3 image of gentoo together with a recent snapshot
of the portage tree. For building / executing arbitrary projects successfully
it is necessary to keep the installed image as close to the host system as
possible.
In order to speed up your experience, you can replace the stage3 image that we
pull from the distfiles mirror with a new image that contains all necessary
dependencies for your experiments. Make sure you update the hash alongside
the gentoo image in benchbuild's source directory.
"""
import logging
import attr
from plumbum import ProcessExecutionError, local
from benchbuild import project
from benchbuild.settings import CFG
from benchbuild.utils import compiler, container, path, run, uchroot
from benchbuild.utils.cmd import cp, ln
LOG = logging.getLogger(__name__)
[docs]@attr.s
class GentooGroup(project.Project):
"""Gentoo ProjectGroup is the base class for every portage build."""
GROUP = 'gentoo'
CONTAINER = container.Gentoo()
SRC_FILE = None
emerge_env = attr.ib(default={}, repr=False, cmp=False)
[docs] def redirect(self):
if not CFG["unionfs"]["enable"]:
container.unpack(self.container, self.builddir)
setup_networking()
setup_benchbuild()
configure_portage()
self.configure_benchbuild(CFG)
path.mkfile_uchroot("/.benchbuild-container")
benchbuild = find_benchbuild()
with local.env(BB_VERBOSITY=str(CFG['verbosity'])):
project_id = "{0}/{1}".format(self.name, self.group)
run.run(benchbuild["run", "-E", self.experiment.name, project_id])
[docs] def compile(self):
package_atom = "{domain}/{name}".format(
domain=self.domain, name=self.name)
LOG.debug('Installing dependencies.')
emerge(package_atom, '--onlydeps', env=self.emerge_env)
c_compiler = local.path(str(compiler.cc(self)))
cxx_compiler = local.path(str(compiler.cxx(self)))
setup_compilers('/etc/portage/make.conf')
ln("-sf", str(c_compiler), local.path('/') / c_compiler.basename)
ln('-sf', str(cxx_compiler), local.path('/') / cxx_compiler.basename)
LOG.debug('Installing %s.', package_atom)
emerge(package_atom, env=self.emerge_env)
[docs]def emerge(package, *args, env=None):
from benchbuild.utils.cmd import emerge as _emerge
with local.env(env):
run.run(_emerge["--autounmask-continue", args, package])
[docs]def setup_networking():
LOG.debug("Setting up networking...")
path.mkfile_uchroot("/etc/resolv.conf")
cp("/etc/resolv.conf", "etc/resolv.conf")
write_wgetrc("etc/wgetrc")
[docs]def write_sandbox_d(_path):
path.mkfile_uchroot(local.path('/') / _path)
with open(_path, 'a') as sandbox_conf:
lines = '''
SANDBOX_WRITE="/clang.stderr:/clang++.stderr:/clang.stdout:/clang++.stdout"
'''
sandbox_conf.write(lines)
[docs]def setup_compilers(_path):
LOG.debug("Arming compiler symlinks.")
with open(_path, 'a') as makeconf:
lines = '''
CC="/clang"
CXX="/clang++"
'''
makeconf.write(lines)
[docs]def write_makeconfig(_path):
"""
Write a valid gentoo make.conf file to :path:.
Args:
path - The output path of the make.conf
"""
http_proxy = str(CFG["gentoo"]["http_proxy"])
ftp_proxy = str(CFG["gentoo"]["ftp_proxy"])
rsync_proxy = str(CFG["gentoo"]["rsync_proxy"])
path.mkfile_uchroot(local.path('/') / _path)
with open(_path, 'w') as makeconf:
lines = '''
PORTAGE_USERNAME=root
PORTAGE_GROUPNAME=root
CFLAGS="-O2 -pipe"
CXXFLAGS="${CFLAGS}"
FEATURES="nostrip -xattr"
CHOST="x86_64-pc-linux-gnu"
USE="bindist mmx sse sse2"
PORTDIR="/usr/portage"
DISTDIR="/mnt/distfiles"
PKGDIR="${PORTDIR}/packages"
'''
makeconf.write(lines)
mounts = CFG["container"]["mounts"].value
tmp_dir = str(CFG["tmp_dir"])
mounts.append({"src": tmp_dir, "tgt": "/mnt/distfiles"})
CFG["container"]["mounts"] = mounts
if http_proxy is not None:
http_s = "http_proxy={0}".format(http_proxy)
https_s = "https_proxy={0}".format(http_proxy)
makeconf.write(http_s + "\n")
makeconf.write(https_s + "\n")
if ftp_proxy is not None:
fp_s = "ftp_proxy={0}".format(ftp_proxy)
makeconf.write(fp_s + "\n")
if rsync_proxy is not None:
rp_s = "RSYNC_PROXY={0}".format(rsync_proxy)
makeconf.write(rp_s + "\n")
[docs]def write_bashrc(_path):
"""
Write a valid gentoo bashrc file to :path:.
Args:
path - The output path of the make.conf
"""
cfg_mounts = CFG["container"]["mounts"].value
cfg_prefix = CFG["container"]["prefixes"].value
path.mkfile_uchroot("/etc/portage/bashrc")
mounts = uchroot.mounts("mnt", cfg_mounts)
p_paths, p_libs = uchroot.env(cfg_prefix)
paths, libs = uchroot.env(mounts)
paths = paths + p_paths
libs = libs + p_libs
with open(_path, 'w') as bashrc:
lines = '''
export PATH="{0}:${{PATH}}"
export LD_LIBRARY_PATH="{1}:${{LD_LIBRARY_PATH}}"
'''.format(path.list_to_path(paths), path.list_to_path(libs))
bashrc.write(lines)
[docs]def write_layout(_path):
"""
Write a valid gentoo layout file to :path:.
Args:
path - The output path of the layout.conf
"""
path.mkdir_uchroot("/etc/portage/metadata")
path.mkfile_uchroot("/etc/portage/metadata/layout.conf")
with open(_path, 'w') as layoutconf:
lines = '''masters = gentoo'''
layoutconf.write(lines)
[docs]def write_wgetrc(_path):
"""
Write a valid gentoo wgetrc file to :path:.
Args:
path - The output path of the wgetrc
"""
http_proxy = str(CFG["gentoo"]["http_proxy"])
ftp_proxy = str(CFG["gentoo"]["ftp_proxy"])
path.mkfile_uchroot("/etc/wgetrc")
with open(_path, 'w') as wgetrc:
if http_proxy is not None:
http_s = "http_proxy = {0}".format(http_proxy)
https_s = "https_proxy = {0}".format(http_proxy)
wgetrc.write("use_proxy = on\n")
wgetrc.write(http_s + "\n")
wgetrc.write(https_s + "\n")
if ftp_proxy is not None:
fp_s = "ftp_proxy={0}".format(ftp_proxy)
wgetrc.write(fp_s + "\n")
[docs]def setup_virtualenv(_path="/benchbuild"):
LOG.debug("Setting up Benchbuild virtualenv...")
env = uchroot.uchroot()["/usr/bin/env"]
env = env['-i', '--']
venv = env["/usr/bin/virtualenv"]
venv = venv("-p", "/usr/bin/python3", _path)
[docs]def find_benchbuild():
try:
uchrt = uchroot.clean_env(uchroot.uchroot(), ['HOME'])
benchbuild_loc = uchrt("which", "benchbuild").strip()
benchbuild = uchrt[benchbuild_loc]
return benchbuild
except ProcessExecutionError as err:
LOG.error("Could not find Benchbuild inside container")
LOG.debug("Reason: %s", str(err))
return None
[docs]def requires_update(benchbuild):
try:
c_version = benchbuild("--version").strip().split()[-1]
h_version = str(CFG["version"])
LOG.debug("container: %s", c_version)
LOG.debug("host: %s", h_version)
if c_version == h_version:
return False
except ProcessExecutionError as err:
LOG.error("Querying Benchbuild inside container failed")
LOG.debug("Reason: %s", str(err))
return True
[docs]def setup_benchbuild():
"""
Setup benchbuild inside a container.
This will query a for an existing installation of benchbuild and
try to upgrade it to the latest version, if possible.
"""
LOG.debug("Setting up Benchbuild...")
venv_dir = local.path("/benchbuild")
prefixes = CFG["container"]["prefixes"].value
prefixes.append(venv_dir)
CFG["container"]["prefixes"] = prefixes
src_dir = str(CFG["source_dir"])
have_src = src_dir is not None
if have_src:
__mount_source(src_dir)
benchbuild = find_benchbuild()
if benchbuild and not requires_update(benchbuild):
if have_src:
__upgrade_from_source(venv_dir, with_deps=False)
return
setup_virtualenv(venv_dir)
if have_src:
__upgrade_from_source(venv_dir)
else:
__upgrade_from_pip(venv_dir)
def __upgrade_from_pip(venv_dir):
LOG.debug("Upgrading from pip")
uchrt_cmd = uchroot.clean_env(uchroot.uchroot(), ['HOME'])
uchroot.uretry(uchrt_cmd[venv_dir / "bin" /
"pip3", "install", "--upgrade", "benchbuild"])
def __mount_source(src_dir):
src_dir = local.path(str(src_dir))
mounts = CFG["container"]["mounts"].value
mount = {"src": src_dir, "tgt": "/mnt/benchbuild"}
mounts.append(mount)
CFG["container"]["mounts"] = mounts
def __upgrade_from_source(venv_dir, with_deps=True):
LOG.debug("Upgrading from source")
uchrt_cmd = uchroot.clean_env(uchroot.uchroot(), ['HOME'])
opts = ["--upgrade"]
if not with_deps:
opts.append("--no-deps")
uchrt_cmd = uchrt_cmd[venv_dir / "bin" / "pip3", "install"]
uchrt_cmd = uchrt_cmd[opts]
uchrt_cmd = uchrt_cmd["/mnt/benchbuild"]
uchroot.uretry(uchrt_cmd)