From 04ba1050b1b3e7b0eb8269f032762f32e91fb32f Mon Sep 17 00:00:00 2001 From: Joshua Watt Date: Thu, 21 Nov 2019 12:36:39 -0600 Subject: [PATCH 01/48] Use explicit run script Instead of running a startup script specified by the container, the container now provides an explicit "run" executable that pyrex executes. This will allow pyrex to run other executables for other purposes (e.g. environment capture) in the future. Signed-off-by: Joshua Watt --- docker/Dockerfile | 6 +++--- docker/{startup.sh => run.sh} | 0 pyrex.py | 3 ++- 3 files changed, 5 insertions(+), 4 deletions(-) rename docker/{startup.sh => run.sh} (100%) diff --git a/docker/Dockerfile b/docker/Dockerfile index 034f42f..87f26bb 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -465,12 +465,12 @@ ENV LANG en_US.UTF-8 ENV LC_ALL en_US.UTF-8 # Add startup scripts -COPY ./startup.sh /usr/libexec/tini/startup.sh +COPY ./run.sh /usr/libexec/pyrex/run COPY ./entry.py /usr/libexec/tini/entry.py COPY ./cleanup.py /usr/libexec/tini/cleanup.py RUN chmod +x /usr/libexec/tini/cleanup.py \ /usr/libexec/tini/entry.py \ - /usr/libexec/tini/startup.sh + /usr/libexec/pyrex/run # Add startup script directory & test script. COPY ./test_startup.sh /usr/libexec/pyrex/startup.d/ @@ -485,7 +485,7 @@ RUN getent group | cut -f1 -d: | grep -v '^root$' | xargs -L 1 groupdel # Use tini as the init process and instruct it to invoke the cleanup script # once the primary command dies -ENTRYPOINT ["/usr/local/bin/tini", "-P", "/usr/libexec/tini/cleanup.py", "{}", ";", "--", "/usr/libexec/tini/entry.py", "/usr/libexec/tini/startup.sh"] +ENTRYPOINT ["/usr/local/bin/tini", "-P", "/usr/libexec/tini/cleanup.py", "{}", ";", "--", "/usr/libexec/tini/entry.py"] # The startup script is expected to chain along to some other # command. By default, we'll use an interactive shell. diff --git a/docker/startup.sh b/docker/run.sh similarity index 100% rename from docker/startup.sh rename to docker/run.sh diff --git a/pyrex.py b/pyrex.py index b81c9dd..4031597 100755 --- a/pyrex.py +++ b/pyrex.py @@ -736,6 +736,7 @@ def run(args): docker_args.append("--") docker_args.append(runid) + docker_args.append("/usr/libexec/pyrex/run") docker_args.extend(args.command) stop_coverage() @@ -746,7 +747,7 @@ def run(args): sys.exit(1) else: startup_args = [ - os.path.join(config["build"]["pyrexroot"], "docker", "startup.sh") + os.path.join(config["build"]["pyrexroot"], "docker", "run.sh") ] startup_args.extend(args.command) From d1d33f128a5ccacae866393c6c7ef40daf5888ef Mon Sep 17 00:00:00 2001 From: Joshua Watt Date: Thu, 21 Nov 2019 13:11:11 -0600 Subject: [PATCH 02/48] Dockerfile: Move scripts to /usr/libexec/pyrex It's a better location than /usr/libexec/tini --- docker/Dockerfile | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 87f26bb..8efd0ac 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -466,17 +466,17 @@ ENV LC_ALL en_US.UTF-8 # Add startup scripts COPY ./run.sh /usr/libexec/pyrex/run -COPY ./entry.py /usr/libexec/tini/entry.py -COPY ./cleanup.py /usr/libexec/tini/cleanup.py -RUN chmod +x /usr/libexec/tini/cleanup.py \ - /usr/libexec/tini/entry.py \ +COPY ./entry.py /usr/libexec/pyrex/entry.py +COPY ./cleanup.py /usr/libexec/pyrex/cleanup.py +RUN chmod +x /usr/libexec/pyrex/cleanup.py \ + /usr/libexec/pyrex/entry.py \ /usr/libexec/pyrex/run # Add startup script directory & test script. COPY ./test_startup.sh /usr/libexec/pyrex/startup.d/ # Precompile python files for improved startup time -RUN python3 -m py_compile /usr/libexec/tini/*.py +RUN python3 -m py_compile /usr/libexec/pyrex/*.py # Remove all non-root users and groups so that there are no conflicts when the # user is added @@ -485,7 +485,7 @@ RUN getent group | cut -f1 -d: | grep -v '^root$' | xargs -L 1 groupdel # Use tini as the init process and instruct it to invoke the cleanup script # once the primary command dies -ENTRYPOINT ["/usr/local/bin/tini", "-P", "/usr/libexec/tini/cleanup.py", "{}", ";", "--", "/usr/libexec/tini/entry.py"] +ENTRYPOINT ["/usr/local/bin/tini", "-P", "/usr/libexec/pyrex/cleanup.py", "{}", ";", "--", "/usr/libexec/pyrex/entry.py"] # The startup script is expected to chain along to some other # command. By default, we'll use an interactive shell. From a5a43c0456b90253669d1b64c2e579bde60aaebd Mon Sep 17 00:00:00 2001 From: Joshua Watt Date: Fri, 22 Nov 2019 10:35:02 -0600 Subject: [PATCH 03/48] Rework capture behavior Completely reworks the mechanism by which pyrex "capture" the build environment. Now, instead of initializing in host environment, the entire initialization and capture is done inside the target container, in a container specific manner. This means that the host no longer needs to have any of the dependencies and setup required for initialization, since all of that is provided by the container. Notable changes: * The only valid mechanism for a user to provide a pyrex configuration is by setting the PYREXCONFFILE environment variable. The previous behaviors of copying template files no longer works. --- README.md | 22 +- ci/test.py | 150 +----- docker/Dockerfile | 8 +- docker/bypass.sh | 27 ++ docker/capture.sh | 63 +++ docker/entry.py | 17 +- docker/run.sh | 13 +- pyrex-init-build-env | 54 +-- pyrex.ini | 27 +- pyrex.py | 1087 +++++++++++++++++++++--------------------- 10 files changed, 687 insertions(+), 781 deletions(-) create mode 100644 docker/bypass.sh create mode 100755 docker/capture.sh diff --git a/README.md b/README.md index d832735..a0b2221 100644 --- a/README.md +++ b/README.md @@ -125,25 +125,9 @@ PYREXCONFTEMPLATE="$(pwd)/pyrex.ini.sample" ``` ### Configuration -Pyrex is configured using a ini-style configuration file and uses the following -precedence to determine where this file is located: -1. The file specified in the environment variable `PYREXCONFFILE`. -2. The `pyrex.ini` file in the bitbake conf directory (e.g. - `${OEROOT}/build/conf/pyrex.ini`, if it exists and has a version number - specified in `${config:confversion}`. For further rules, this file will - be known as `PYREX_USER_CONF` -3. If the file specified in the environment variable `$PYREXCONFTEMPLATE` exists, - is copied to `PYREX_USER_CONF`, then `PYREX_USER_CONF` is used -4. If the file `$TEMPLATECONF/pyrex.ini.sample` exists, it is copied to - `PYREX_USER_CONF`, the `PYREX_USER_CONF` is used. This is the same rules - that bitbake uses for `local.conf.sample`, allowing you to easily put a - `pyrex.conf.sample` file along side the other default config files. See - [TEMPLATECONF][]. -5. The internal default config file provided by Pyrex is coped to - `PYREX_USER_CONF`, then `PYREX_USER_CONF` is used. - -**Note:** The config file is only populated when Pyrex initializes the -environment (e.g. when the init script is sourced). +Pyrex is configured using a ini-style configuration file. The location of this +file is specified by the `PYREXCONFFILE` environment variable. This environment +variable must be set before the environment is initialized. The configuration file is the ini file format supported by Python's [configparser](https://docs.python.org/3/library/configparser.html) class, with diff --git a/ci/test.py b/ci/test.py index 9932913..52f21b4 100755 --- a/ci/test.py +++ b/ci/test.py @@ -64,7 +64,12 @@ def cleanup_build(): else: self.dockerpath = self.docker_provider - self.pyrex_conf = self.prepare_build_dir(self.build_dir) + self.pyrex_conf = os.path.join(self.build_dir, "pyrex.ini") + conf = self.get_config() + conf.write_conf() + + if not os.environ.get(TEST_PREBUILT_TAG_ENV_VAR, ""): + self.prebuild_image() def cleanup_env(): os.environ.clear() @@ -77,6 +82,8 @@ def cleanup_env(): os.symlink("/usr/bin/python2", os.path.join(self.bin_dir, "python")) os.environ["PATH"] = self.bin_dir + ":" + os.environ["PATH"] os.environ["PYREX_DOCKER_BUILD_QUIET"] = "0" + if "SSH_AUTH_SOCK" in os.environ: + del os.environ["SSH_AUTH_SOCK"] self.addCleanup(cleanup_env) self.thread_dir = os.path.join( @@ -84,28 +91,6 @@ def cleanup_env(): ) os.makedirs(self.thread_dir) - def prepare_build_dir(self, build_dir): - def cleanup_build(): - if os.path.isdir(build_dir): - shutil.rmtree(build_dir) - - conf_dir = os.path.join(build_dir, "conf") - try: - os.makedirs(conf_dir) - except FileExistsError: - pass - - pyrex_conf = os.path.join(conf_dir, "pyrex.ini") - - # Write out the default test config - conf = self.get_config(pyrex_conf=pyrex_conf) - conf.write_conf() - - if not os.environ.get(TEST_PREBUILT_TAG_ENV_VAR, ""): - self.prebuild_image() - - return pyrex_conf - def prebuild_image(self): global built_images image = ":".join((self.test_image, self.docker_provider)) @@ -120,21 +105,18 @@ def prebuild_image(self): ) built_images.add(image) - def get_config(self, *, defaults=False, pyrex_conf=None): - if pyrex_conf is None: - pyrex_conf = self.pyrex_conf - + def get_config(self, *, defaults=False): class Config(configparser.RawConfigParser): def write_conf(self): write_config_helper(self) def write_config_helper(conf): - with open(pyrex_conf, "w") as f: + with open(self.pyrex_conf, "w") as f: conf.write(f) config = Config() - if os.path.exists(pyrex_conf) and not defaults: - config.read(pyrex_conf) + if os.path.exists(self.pyrex_conf) and not defaults: + config.read(self.pyrex_conf) else: config.read_string(pyrex.read_default_config(True)) @@ -145,6 +127,7 @@ def write_config_helper(conf): config["config"]["pyrextag"] = ( os.environ.get(TEST_PREBUILT_TAG_ENV_VAR, "") or "ci-test" ) + config["run"]["bind"] = self.build_dir return config @@ -191,6 +174,7 @@ def _write_host_command( cmd_file = os.path.join(self.thread_dir, "command") with open(cmd_file, "w") as f: + f.write("PYREXCONFFILE=%s\n" % self.pyrex_conf) f.write( ". %s/poky/pyrex-init-build-env%s %s && (" % (PYREX_ROOT, " > /dev/null 2>&1" if quiet_init else "", builddir) @@ -320,23 +304,6 @@ def test_quiet_build(self): env["PYREX_DOCKER_BUILD_QUIET"] = "1" self.assertPyrexHostCommand("true", env=env) - def test_no_container_build(self): - # Prevent container from building - os.symlink("/bin/false", os.path.join(self.bin_dir, self.docker_provider)) - - # Container build will fail if invoked here - env = os.environ.copy() - env["PYREX_DOCKER"] = "0" - self.assertPyrexHostCommand("true", env=env) - - # Verify that pyrex won't allow you to try and use the provider later - output = self.assertPyrexHostCommand( - "PYREX_DOCKER=1 bitbake", returncode=1, capture=True, env=env - ) - self.assertIn( - "Container was not enabled when the environment was setup", output - ) - def test_bad_provider(self): # Prevent container build from working os.symlink("/bin/false", os.path.join(self.bin_dir, self.docker_provider)) @@ -454,75 +421,6 @@ def test_conftemplate_ignored(self): self.assertPyrexHostCommand("true") - def test_conf_upgrade(self): - conf = self.get_config() - del conf["config"]["confversion"] - conf.write_conf() - - # Write out a template in an alternate location. It will be respected - temp_dir = tempfile.mkdtemp("-pyrex") - self.addCleanup(shutil.rmtree, temp_dir) - - conftemplate = os.path.join(temp_dir, "pyrex.ini.sample") - - conf = self.get_config(defaults=True) - with open(conftemplate, "w") as f: - conf.write(f) - - env = os.environ.copy() - env["PYREXCONFTEMPLATE"] = conftemplate - - self.assertPyrexHostCommand("true", env=env) - - def test_bad_conf_upgrade(self): - # Write out a template in an alternate location, but it also fails to - # have a confversion - conf = self.get_config() - del conf["config"]["confversion"] - conf.write_conf() - - # Write out a template in an alternate location. It will be respected - temp_dir = tempfile.mkdtemp("-pyrex") - self.addCleanup(shutil.rmtree, temp_dir) - - conftemplate = os.path.join(temp_dir, "pyrex.ini.sample") - - conf = self.get_config(defaults=True) - del conf["config"]["confversion"] - with open(conftemplate, "w") as f: - conf.write(f) - - env = os.environ.copy() - env["PYREXCONFTEMPLATE"] = conftemplate - - self.assertPyrexHostCommand("true", returncode=1, env=env) - - def test_force_conf(self): - # Write out a new config file and set the variable to force it to be - # used - conf = self.get_config() - conf["config"]["test"] = "bar" - force_conf_file = os.path.join(self.thread_dir, "force.ini") - with open(force_conf_file, "w") as f: - conf.write(f) - - # Set the variable to a different value in the standard config file - conf = self.get_config() - conf["config"]["test"] = "foo" - conf.write_conf() - - output = self.assertPyrexHostCommand( - "pyrex-config get config:test", quiet_init=True, capture=True - ) - self.assertEqual(output, "foo") - - env = os.environ.copy() - env["PYREXCONFFILE"] = force_conf_file - output = self.assertPyrexHostCommand( - "pyrex-config get config:test", quiet_init=True, capture=True, env=env - ) - self.assertEqual(output, "bar") - @skipIfPrebuilt def test_local_build(self): conf = self.get_config() @@ -756,16 +654,9 @@ def test_templateconf_abs(self): test_string = "set_by_test.%d" % threading.get_ident() - # Write out a config template that passes along the TEST_ENV variable. - # The variable will only have the correct value in the container if - # the template is used conf = self.get_config() conf["run"]["envvars"] += " TEST_ENV" - with open(os.path.join(template_dir, "pyrex.ini.sample"), "w") as f: - conf.write(f) - # Delete the normal pyrex conf file so a new one will be pulled from - # TEMPLATECONF - os.unlink(self.pyrex_conf) + conf.write_conf() env = os.environ.copy() env["TEMPLATECONF"] = template_dir @@ -793,16 +684,9 @@ def test_templateconf_rel(self): test_string = "set_by_test.%d" % threading.get_ident() - # Write out a config template that passes along the TEST_ENV variable. - # The variable will only have the correct value in the container if - # the template is used conf = self.get_config() conf["run"]["envvars"] += " TEST_ENV" - with open(os.path.join(template_dir, "pyrex.ini.sample"), "w") as f: - conf.write(f) - # Delete the normal pyrex conf file so a new one will be pulled from - # TEMPLATECONF - os.unlink(self.pyrex_conf) + conf.write_conf() env = os.environ.copy() env["TEMPLATECONF"] = os.path.relpath( @@ -826,7 +710,6 @@ def test_top_dir(self): builddir = os.path.join(cwd, "build") - self.prepare_build_dir(builddir) oe_topdir = self.assertSubprocess( [ "/bin/bash", @@ -840,7 +723,6 @@ def test_top_dir(self): shutil.rmtree(builddir) - self.prepare_build_dir(builddir) pyrex_topdir = self.assertPyrexHostCommand( "bitbake -e | grep ^TOPDIR=", quiet_init=True, diff --git a/docker/Dockerfile b/docker/Dockerfile index 8efd0ac..a7138c5 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -466,11 +466,15 @@ ENV LC_ALL en_US.UTF-8 # Add startup scripts COPY ./run.sh /usr/libexec/pyrex/run +COPY ./capture.sh /usr/libexec/pyrex/capture +COPY ./bypass.sh /usr/libexec/pyrex/bypass COPY ./entry.py /usr/libexec/pyrex/entry.py COPY ./cleanup.py /usr/libexec/pyrex/cleanup.py RUN chmod +x /usr/libexec/pyrex/cleanup.py \ /usr/libexec/pyrex/entry.py \ - /usr/libexec/pyrex/run + /usr/libexec/pyrex/run \ + /usr/libexec/pyrex/capture \ + /usr/libexec/pyrex/bypass # Add startup script directory & test script. COPY ./test_startup.sh /usr/libexec/pyrex/startup.d/ @@ -489,7 +493,7 @@ ENTRYPOINT ["/usr/local/bin/tini", "-P", "/usr/libexec/pyrex/cleanup.py", "{}", # The startup script is expected to chain along to some other # command. By default, we'll use an interactive shell. -CMD ["/bin/bash"] +CMD ["/usr/libexec/pyrex/run", "/bin/bash"] # # Yocto compatible target image. diff --git a/docker/bypass.sh b/docker/bypass.sh new file mode 100644 index 0000000..c51d453 --- /dev/null +++ b/docker/bypass.sh @@ -0,0 +1,27 @@ +#! /bin/bash +# +# Copyright 2019 Garmin Ltd. or its subsidiaries +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +declare -a COMMAND=("$@") + +# Consume all arguments before sourcing the environment script +shift $# + +pushd "$PYREX_OEINIT_DIR" > /dev/null +source $PYREX_OEINIT > /dev/null +popd > /dev/null + +exec $PYREX_COMMAND_PREFIX "${COMMAND[@]}" + diff --git a/docker/capture.sh b/docker/capture.sh new file mode 100755 index 0000000..8ef470c --- /dev/null +++ b/docker/capture.sh @@ -0,0 +1,63 @@ +#! /bin/bash +# +# Copyright 2019 Garmin Ltd. or its subsidiaries +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +INIT_PWD=$PWD + +# Consume all arguments before sourcing the environment script +shift $# + +. $PYREX_OEINIT +if [ $? -ne 0 ]; then + exit 1 +fi + +cat > $PYREX_CAPTURE_DEST < /dev/null - source $PYREX_INIT_COMMAND > /dev/null - popd > /dev/null -fi - -exec $PYREX_COMMAND_PREFIX "${COMMAND[@]}" +exec $PYREX_COMMAND_PREFIX "$@" diff --git a/pyrex-init-build-env b/pyrex-init-build-env index b84059a..e2db5ea 100644 --- a/pyrex-init-build-env +++ b/pyrex-init-build-env @@ -40,59 +40,29 @@ if [ -z "$PYREX_OEINIT" ]; then fi unset THIS_SCRIPT -if [ -z "$PYREX_TEMP_FILE" ]; then - PYREX_TEMP_FILE=$(mktemp -t pyrex.XXXXXX) +if [ -z "$PYREX_TEMP_ENV_FILE" ]; then + PYREX_TEMP_ENV_FILE=$(mktemp -t pyrex-env.XXXXXX) fi pyrex_cleanup() { - rm -f "$PYREX_TEMP_FILE" - unset PYREX_OEROOT PYREX_OEINIT PYREX_OEINIT_DIR PYREX_ROOT PYREX_TEMPCONFFILE PYREX_TEMP_FILE pyrex_cleanup + rm -f "$PYREX_TEMP_ENV_FILE" + unset PYREX_OEROOT PYREX_OEINIT PYREX_ROOT PYREX_TEMP_ENV_FILE pyrex_cleanup } -# Capture OE Build environment -( - PYREX_OEINIT_DIR="$PWD" +export PYREXCONFFILE - . $PYREX_OEINIT "$@" - if [ $? -ne 0 ]; then - exit 1 - fi - - if [ -z "${PYREXCONFTEMPLATE}" ]; then - PYREXCONFTEMPLATE="$(cat "$BUILDDIR/conf/templateconf.cfg")/pyrex.ini.sample" - case $PYREXCONFTEMPLATE in - "/"*) ;; - *) PYREXCONFTEMPLATE="$PYREX_OEROOT/$PYREXCONFTEMPLATE" - esac - fi - - export PYREX_OEROOT PYREX_OEINIT PYREX_OEINIT_DIR PYREX_ROOT PYREXCONFTEMPLATE - - # Instruct pyrex.py to output the name of the configuration file to - # file descriptor 9 - $PYREX_ROOT/pyrex.py capture 9 -- "$@" 9> $PYREX_TEMP_FILE -) -if [ $? -ne 0 ]; then - pyrex_cleanup - return 1 -fi -PYREX_TEMPCONFFILE=$(cat $PYREX_TEMP_FILE) - -# Build Pyrex images -$PYREX_ROOT/pyrex.py build "$PYREX_TEMPCONFFILE" -if [ $? -ne 0 ]; then - pyrex_cleanup - return 1 -fi - -# Setup build environment -$PYREX_ROOT/pyrex.py env "$PYREX_TEMPCONFFILE" 9 9> $PYREX_TEMP_FILE +$PYREX_ROOT/pyrex.py capture 9 \ + -a PYREX_OEROOT "$PYREX_OEROOT" \ + -a PYREX_OEINIT "$PYREX_OEINIT $@" \ + --bind $PYREX_OEROOT \ + -e TEMPLATECONF \ + 9> $PYREX_TEMP_ENV_FILE if [ $? -ne 0 ]; then pyrex_cleanup return 1 fi -. $PYREX_TEMP_FILE +. $PYREX_TEMP_ENV_FILE if [ $? -ne 0 ]; then pyrex_cleanup return 1 diff --git a/pyrex.ini b/pyrex.ini index 632b61e..36ac9dd 100644 --- a/pyrex.ini +++ b/pyrex.ini @@ -2,10 +2,7 @@ # # The following variables are made available as variable expansions when # Pyrex is initialized: -# ${build:builddir} The absolute path to the build directory -# ${build:oeroot} The absolute path to the top level Open Embedded -# directory (e.g. $PYREX_OEROOT) -# ${build:pyrexroot} The absolute path to Pyrex (e.g. $PYREX_ROOT) +# ${pyrex:pyrexroot} The absolute path to Pyrex (e.g. $PYREX_ROOT) # # Pyrex build information. Any changes to this section will require @@ -15,21 +12,6 @@ # to be specified in the user config file confversion = @CONFVERSION@ -# The location of the temporary Pyrex shim files and build configuration -tempdir = ${build:builddir}/pyrex - -# A list of globs for commands that should be wrapped by Pyrex -%commands = -% ${build:oeroot}/bitbake/bin/* -% ${build:oeroot}/scripts/* - -# A list of globs that should never be run in pyrex (e.g. not even using the -# shim commands to setup the pyrex environment). These are simply added to -# PATH by symlinks. Any commands listed here will take precedence over -# ${commands} -%commands_nopyrex = -% ${build:oeroot}/scripts/runqemu* - # The docker executable to use %dockerpath = docker @@ -54,9 +36,6 @@ tempdir = ${build:builddir}/pyrex # default %registry = -# The shell command to run for pyrex-shell -%shell = /bin/bash - # A list of environment variables that should be imported as Pyrex # configuration variables in the "env" section, e.g. ${env:HOME}. Note that # environment variables accessed in this way must be set or an error will @@ -68,7 +47,7 @@ tempdir = ${build:builddir}/pyrex [dockerbuild] # The Dockerfile used to build the Pyrex image (if building locally) -%dockerfile = ${build:pyrexroot}/docker/Dockerfile +%dockerfile = ${pyrex:pyrexroot}/docker/Dockerfile # Arguments to pass when building the docker image %args = @@ -89,8 +68,6 @@ tempdir = ${build:builddir}/pyrex # A list of directories that should be bound when running in the Docker image %bind = -% ${build:oeroot} -% ${build:builddir} # A list of environment variables that should be propagated to the Docker # container if set in the parent environment diff --git a/pyrex.py b/pyrex.py index 4031597..9830f42 100755 --- a/pyrex.py +++ b/pyrex.py @@ -28,13 +28,16 @@ import textwrap import stat import hashlib +import json +import tempfile VERSION = "0.0.4" VERSION_REGEX = re.compile(r"^([0-9]+\.){2}[0-9]+(-.*)?$") VERSION_TAG_REGEX = re.compile(r"^v([0-9]+\.){2}[0-9]+(-.*)?$") -THIS_SCRIPT = os.path.basename(__file__) +THIS_SCRIPT = os.path.abspath(__file__) +PYREX_ROOT = os.path.dirname(THIS_SCRIPT) PYREX_CONFVERSION = "1" MINIMUM_DOCKER_VERSION = 17 @@ -59,7 +62,7 @@ def getrawdict(self): def read_default_config(keep_defaults): - with open(os.path.join(os.path.dirname(__file__), "pyrex.ini"), "r") as f: + with open(os.path.join(PYREX_ROOT, "pyrex.ini"), "r") as f: line = f.read().replace("@CONFVERSION@", PYREX_CONFVERSION) if keep_defaults: line = line.replace("%", "") @@ -68,11 +71,11 @@ def read_default_config(keep_defaults): return line -def load_configs(conffile): - # Load the build time config file - build_config = Config() - with open(conffile, "r") as f: - build_config.read_file(f) +def load_config(): + conffile = os.environ.get("PYREXCONFFILE", "") + if not conffile: + sys.stderr.write("Pyrex user config file must be defined in $PYREXCONFFILE!\n") + sys.exit(1) # Load the default config, except the version user_config = Config() @@ -80,12 +83,9 @@ def load_configs(conffile): del user_config["config"]["confversion"] # Load user config file - with open(build_config["build"]["userconfig"], "r") as f: + with open(conffile, "r") as f: user_config.read_file(f) - # Merge build config into user config - user_config.read_dict(build_config.getrawdict()) - # Load environment variables try: user_config.add_section("env") @@ -98,8 +98,21 @@ def load_configs(conffile): user_config.add_section("pyrex") user_config["pyrex"]["version"] = VERSION + user_config["pyrex"]["pyrexroot"] = PYREX_ROOT - return user_config, build_config + try: + confversion = user_config["config"]["confversion"] + if confversion != PYREX_CONFVERSION: + sys.stderr.write( + "Bad pyrex conf version '%s' in %s\n" + % (user_config["config"]["confversion"], conffile) + ) + sys.exit(1) + except KeyError: + sys.stderr.write("Cannot find pyrex conf version in %s!\n" % conffile) + sys.exit(1) + + return user_config def stop_coverage(): @@ -136,30 +149,152 @@ def use_docker(config): return os.environ.get("PYREX_DOCKER", config["run"]["enable"]) == "1" -def copy_templateconf(conffile): - template = os.environ["PYREXCONFTEMPLATE"] - - if os.path.isfile(template): - shutil.copyfile(template, conffile) - else: - with open(conffile, "w") as f: - f.write(read_default_config(False)) +def build_image(config, build_config): + build_config.setdefault("build", {}) + docker_path = config["config"]["dockerpath"] -def check_confversion(user_config, version_required=False): + # Check minimum docker version try: - confversion = user_config["config"]["confversion"] - if confversion != PYREX_CONFVERSION: + (provider, version) = get_docker_info(config) + except (subprocess.CalledProcessError, FileNotFoundError): + print( + textwrap.fill( + ( + "Unable to run '%s' as docker. Please make sure you have it installed." + + "For installation instructions, see the docker website. Commonly, " + + "one of the following is relevant:" + ) + % docker_path + ) + ) + print() + print(" https://docs.docker.com/install/linux/docker-ce/ubuntu/") + print(" https://docs.docker.com/install/linux/docker-ce/fedora/") + print() + print( + textwrap.fill( + "After installing docker, give your login account permission to " + + "docker commands by running:" + ) + ) + print() + print(" sudo usermod -aG docker $USER") + print() + print( + textwrap.fill( + "After adding your user to the 'docker' group, log out and back in " + + "so that the new group membership takes effect." + ) + ) + print() + print( + textwrap.fill( + "To attempt running the build on your native operating system's set " + + "of packages, use:" + ) + ) + print() + print(" export PYREX_DOCKER=0") + print(" . init-build-env ...") + print() + return None + + if provider is None: + sys.stderr.write("Could not get docker version!\n") + return None + + if provider == "docker" and int(version.split(".")[0]) < MINIMUM_DOCKER_VERSION: + sys.stderr.write( + "Docker version is too old (have %s), need >= %d\n" + % (version, MINIMUM_DOCKER_VERSION) + ) + return None + + build_config["docker_provider"] = provider + + tag = config["config"]["tag"] + + if config["config"]["buildlocal"] == "1": + if VERSION_TAG_REGEX.match(tag.split(":")[-1]) is not None: sys.stderr.write( - "Bad pyrex conf version '%s'\n" % user_config["config"]["confversion"] + "Image tag '%s' will overwrite release image tag, which is not what you want\n" + % tag ) - return False - return True - except KeyError: - if version_required: - sys.stderr.write("Cannot find pyrex conf version!\n") - return False - raise + sys.stderr.write("Try changing 'config:pyrextag' to a different value\n") + return None + + print("Getting container image up to date...") + + (_, _, image_type) = config["config"]["dockerimage"].split("-") + + docker_args = [ + docker_path, + "build", + "-t", + tag, + "-f", + config["dockerbuild"]["dockerfile"], + "--network=host", + os.path.join(PYREX_ROOT, "docker"), + "--target", + "pyrex-%s" % image_type, + ] + + if config["config"]["registry"]: + docker_args.extend( + ["--build-arg", "MY_REGISTRY=%s/" % config["config"]["registry"]] + ) + + for e in ("http_proxy", "https_proxy"): + if e in os.environ: + docker_args.extend(["--build-arg", "%s=%s" % (e, os.environ[e])]) + + if config["dockerbuild"].get("args", ""): + docker_args.extend(shlex.split(config["dockerbuild"]["args"])) + + env = os.environ.copy() + for e in shlex.split(config["dockerbuild"]["env"]): + name, val = e.split("=", 1) + env[name] = val + + try: + if os.environ.get("PYREX_DOCKER_BUILD_QUIET", "1") == "1" and config[ + "dockerbuild" + ].getboolean("quiet"): + docker_args.append("-q") + build_config["build"]["buildid"] = ( + subprocess.check_output(docker_args, env=env) + .decode("utf-8") + .rstrip() + ) + else: + subprocess.check_call(docker_args, env=env) + build_config["build"]["buildid"] = get_image_id(config, tag) + + build_config["build"]["runid"] = build_config["build"]["buildid"] + + except subprocess.CalledProcessError: + return None + + build_config["build"]["buildhash"] = get_build_hash(build_config) + else: + try: + # Try to get the image This will fail if the image doesn't + # exist locally + build_config["build"]["buildid"] = get_image_id(config, tag) + except subprocess.CalledProcessError: + try: + docker_args = [docker_path, "pull", tag] + subprocess.check_call(docker_args) + + build_config["build"]["buildid"] = get_image_id(config, tag) + except subprocess.CalledProcessError: + return 1 + + build_config["build"]["runid"] = tag + + return build_config def get_build_hash(config): @@ -171,9 +306,7 @@ def get_build_hash(config): # "docker" folder to determine when it is out of date. h = hashlib.sha256() - for (root, dirs, files) in os.walk( - os.path.join(config["build"]["pyrexroot"], "docker") - ): + for (root, dirs, files) in os.walk(os.path.join(PYREX_ROOT, "docker")): # Process files and directories in alphabetical order so that hashing # is consistent dirs.sort() @@ -217,565 +350,426 @@ def get_subid_length(filename, name): return 0 -def main(): - def capture(args): - builddir = os.environ["BUILDDIR"] - conffile = os.environ.get("PYREXCONFFILE", "") - oeinit = os.environ["PYREX_OEINIT"] +def prep_docker( + config, + build_config, + command, + *, + extra_env={}, + preserve_env=[], + extra_bind=[], + allow_test_config=False +): + runid = build_config["build"]["runid"] + + if not runid: + print( + "Container was not enabled when the environment was setup. Cannot use it now!" + ) + return [] - user_config = Config() + docker_path = config["config"]["dockerpath"] - if not conffile: - conffile = os.path.abspath(os.path.join(builddir, "conf", "pyrex.ini")) + try: + buildid = get_image_id(config, runid) + except subprocess.CalledProcessError as e: + print("Cannot verify docker image: %s\n" % e.output) + return [] + + if buildid != build_config["build"]["buildid"]: + sys.stderr.write("WARNING: buildid for docker image %s has changed\n" % runid) + + if config["config"]["buildlocal"] == "1" and build_config["build"][ + "buildhash" + ] != get_build_hash(config): + sys.stderr.write( + "WARNING: The docker image source has changed and should be rebuilt.\n" + "Try running: 'pyrex-rebuild'\n" + ) - if not os.path.isfile(conffile): - copy_templateconf(conffile) + uid = os.getuid() + gid = os.getgid() + username = pwd.getpwuid(uid).pw_name + groupname = grp.getgrgid(gid).gr_name - user_config.read(conffile) + # These are "hidden" keys in pyrex.ini that aren't publicized, and + # are primarily used for testing. Use they at your own risk, they + # may change + if allow_test_config: + uid = int(config["run"].get("uid", uid)) + gid = int(config["run"].get("gid", gid)) + username = config["run"].get("username") or username + groupname = config["run"].get("groupname") or groupname - try: - if not check_confversion(user_config): - return 1 - except KeyError: - sys.stderr.write( - "Cannot find pyrex conf version! Restoring from template\n" - ) + command_prefix = config["run"].get("commandprefix", "").splitlines() - copy_templateconf(conffile) + docker_args = [ + docker_path, + "run", + "--rm", + "-i", + "--net=host", + "-e", + "PYREX_USER=%s" % username, + "-e", + "PYREX_UID=%d" % uid, + "-e", + "PYREX_GROUP=%s" % groupname, + "-e", + "PYREX_GID=%d" % gid, + "-e", + "PYREX_HOME=%s" % os.environ["HOME"], + "-e", + "PYREX_COMMAND_PREFIX=%s" % " ".join(command_prefix), + "--workdir", + os.getcwd(), + ] - user_config = Config() + docker_envvars = [ + "PYREX_CLEANUP_EXIT_WAIT", + "PYREX_CLEANUP_LOG_FILE", + "PYREX_CLEANUP_LOG_LEVEL", + "TINI_VERBOSITY", + ] - user_config.read(conffile) - if not check_confversion(user_config, True): - return 1 + if build_config["docker_provider"] == "podman": + uid_length = get_subid_length("/etc/subuid", username) + if uid_length < 1: + sys.stderr.write("subuid name space is too small\n") + sys.exit(1) - # Setup the build configuration - build_config = Config() + gid_length = get_subid_length("/etc/subgid", groupname) + if uid_length < 1: + sys.stderr.write("subgid name space is too small\n") + sys.exit(1) - build_config["build"] = {} - build_config["build"]["builddir"] = builddir - build_config["build"]["oeroot"] = os.environ["PYREX_OEROOT"] - build_config["build"]["oeinit"] = oeinit - build_config["build"]["pyrexroot"] = os.environ["PYREX_ROOT"] - build_config["build"]["initcommand"] = " ".join( - shlex.quote(a) for a in [oeinit] + args.init + docker_args.extend( + [ + "--security-opt", + "label=disable", + # Fix up the UID/GID mapping so that the actual UID/GID + # inside the container maps to their actual UID/GID + # outside the container. Note that all offsets outside + # the container are relative to the start of the users + # subuid/subgid range. + # Map UID 0 up the actual user ID inside the container + # to the users subuid + "--uidmap", + "0:1:%d" % uid, + # Map the users actual UID inside the container to 0 in + # the users subuid namespace. The "root" user in the + # subuid namespace is special and maps to the users + # actual UID outside the namespace + "--uidmap", + "%d:0:1" % uid, + # Map the remaining UIDs after the actual user ID to + # continue using the users subuid range + "--uidmap", + "%d:%d:%d" % (uid + 1, uid + 1, uid_length - uid), + # Do the same for the GID + "--gidmap", + "0:1:%d" % gid, + "--gidmap", + "%d:0:1" % gid, + "--gidmap", + "%d:%d:%d" % (gid + 1, gid + 1, gid_length - gid), + ] ) - build_config["build"]["initdir"] = os.environ["PYREX_OEINIT_DIR"] - build_config["build"]["userconfig"] = conffile - # Merge the build config into the user config (so that interpolation works) - user_config.read_dict(build_config.getrawdict()) + # Run the docker image with a TTY if this script was run in a tty + if os.isatty(1): + docker_args.extend(["-t", "-e", "TERM=%s" % os.environ["TERM"]]) + + # Configure binds + binds = config["run"]["bind"].split() + extra_bind + for b in set(binds): + if not os.path.exists(b): + print("Error: bind source path {b} does not exist".format(b=b)) + continue + docker_args.extend(["--mount", "type=bind,src={b},dst={b}".format(b=b)]) + + docker_envvars.extend(config["run"]["envvars"].split()) + + # Special case: Make the user SSH authentication socket available in container + if "SSH_AUTH_SOCK" in os.environ: + socket = os.path.realpath(os.environ["SSH_AUTH_SOCK"]) + if not os.path.exists(socket): + print("Warning: SSH_AUTH_SOCK {} does not exist".format(socket)) + else: + docker_args.extend( + [ + "--mount", + "type=bind,src=%s,dst=/tmp/%s-ssh-agent-sock" % (socket, username), + "-e", + "SSH_AUTH_SOCK=/tmp/%s-ssh-agent-sock" % username, + ] + ) - try: - os.makedirs(user_config["config"]["tempdir"]) - except Exception: - pass + # Pass along BB_ENV_EXTRAWHITE and anything it has whitelisted + if "BB_ENV_EXTRAWHITE" in os.environ: + docker_args.extend(["-e", "BB_ENV_EXTRAWHITE"]) + docker_envvars.extend(os.environ["BB_ENV_EXTRAWHITE"].split()) - build_conffile = os.path.join(user_config["config"]["tempdir"], "build.ini") + # Pass environment variables. If a variable passed with an argument + # "-e VAR" is not set in the parent environment, podman passes an + # empty value, where as docker doesn't pass it at all. For + # consistency, manually check if the variables exist before passing + # them. + for e in docker_envvars + preserve_env: + if e in os.environ: + docker_args.extend(["-e", e]) - with open(build_conffile, "w") as f: - build_config.write(f) + for k, v in extra_env.items(): + docker_args.extend(["-e", "%s=%s" % (k, v)]) - os.write(args.fd, build_conffile.encode("utf-8")) + docker_args.extend(shlex.split(config["run"].get("args", ""))) - return 0 + docker_args.append("--") + docker_args.append(runid) + docker_args.extend(command) + return docker_args - def build(args): - config, build_config = load_configs(args.conffile) - if use_docker(config): - docker_path = config["config"]["dockerpath"] +def create_shims(config, build_config, buildconf): + shimdir = os.path.join(build_config["tempdir"], "bin") - # Check minimum docker version - try: - (provider, version) = get_docker_info(config) - except (subprocess.CalledProcessError, FileNotFoundError): - print( - textwrap.fill( - ( - "Unable to run '%s' as docker. Please make sure you have it installed." - + "For installation instructions, see the docker website. Commonly, " - + "one of the following is relevant:" - ) - % docker_path - ) + try: + shutil.rmtree(shimdir) + except Exception: + pass + os.makedirs(shimdir) + + # Write out run convenience command + runfile = os.path.join(shimdir, "pyrex-run") + with open(runfile, "w") as f: + f.write( + textwrap.dedent( + """\ + #! /bin/sh + exec {this_script} run {conffile} -- "$@" + """.format( + this_script=THIS_SCRIPT, conffile=buildconf ) - print() - print(" https://docs.docker.com/install/linux/docker-ce/ubuntu/") - print(" https://docs.docker.com/install/linux/docker-ce/fedora/") - print() - print( - textwrap.fill( - "After installing docker, give your login account permission to " - + "docker commands by running:" - ) + ) + ) + os.chmod(runfile, stat.S_IRWXU) + + # write out config convenience command + configcmd = os.path.join(shimdir, "pyrex-config") + with open(configcmd, "w") as f: + f.write( + textwrap.dedent( + """\ + #! /bin/sh + exec {this_script} config {conffile} "$@" + """.format( + this_script=THIS_SCRIPT, conffile=buildconf ) - print() - print(" sudo usermod -aG docker $USER") - print() - print( - textwrap.fill( - "After adding your user to the 'docker' group, log out and back in " - + "so that the new group membership takes effect." - ) + ) + ) + os.chmod(configcmd, stat.S_IRWXU) + + # write out the shim file + shimfile = os.path.join(shimdir, "exec-shim-pyrex") + with open(shimfile, "w") as f: + f.write( + textwrap.dedent( + """\ + #! /bin/sh + exec {runfile} "$(basename $0)" "$@" + """.format( + runfile=runfile ) - print() - print( - textwrap.fill( - "To attempt running the build on your native operating system's set " - + "of packages, use:" - ) + ) + ) + os.chmod(shimfile, stat.S_IRWXU) + + # write out the shell convenience command + shellfile = os.path.join(shimdir, "pyrex-shell") + with open(shellfile, "w") as f: + f.write( + textwrap.dedent( + """\ + #! /bin/sh + exec {runfile} {shell} "$@" + """.format( + runfile=runfile, shell=build_config["container"]["shell"] ) - print() - print(" export PYREX_DOCKER=0") - print(" . init-build-env ...") - print() - return 1 + ) + ) + os.chmod(shellfile, stat.S_IRWXU) + + # write out image rebuild command + rebuildfile = os.path.join(shimdir, "pyrex-rebuild") + with open(rebuildfile, "w") as f: + f.write( + textwrap.dedent( + """\ + #! /bin/sh + exec {this_script} rebuild {conffile} + """.format( + this_script=THIS_SCRIPT, conffile=buildconf + ) + ) + ) + os.chmod(rebuildfile, stat.S_IRWXU) - if provider is None: - sys.stderr.write("Could not get docker version!\n") - return 1 + # Create bypass command + bypassfile = os.path.join(shimdir, "pyrex-bypass") + docker_args = [ + config["config"]["dockerpath"], + "run", + "--rm", + "--entrypoint", + "cat", + build_config["build"]["runid"], + "/usr/libexec/pyrex/bypass", + ] + with open(bypassfile, "w") as f: + subprocess.run(docker_args, check=True, stdout=f) + os.chmod(bypassfile, stat.S_IRWXU) - if ( - provider == "docker" - and int(version.split(".")[0]) < MINIMUM_DOCKER_VERSION - ): - sys.stderr.write( - "Docker version is too old (have %s), need >= %d\n" - % (version, MINIMUM_DOCKER_VERSION) - ) - return 1 + # Create shims + command_globs = build_config["container"].get("commands", {}).get("include", {}) + nopyrex_globs = build_config["container"].get("commands", {}).get("exclude", {}) - build_config["build"]["docker_provider"] = provider - - tag = config["config"]["tag"] - - if config["config"]["buildlocal"] == "1": - if VERSION_TAG_REGEX.match(tag.split(":")[-1]) is not None: - sys.stderr.write( - "Image tag '%s' will overwrite release image tag, which is not what you want\n" - % tag - ) - sys.stderr.write( - "Try changing 'config:pyrextag' to a different value\n" - ) - return 1 - - print("Getting container image up to date...") - - (_, _, image_type) = config["config"]["dockerimage"].split("-") - - docker_args = [ - docker_path, - "build", - "-t", - tag, - "-f", - config["dockerbuild"]["dockerfile"], - "--network=host", - os.path.join(config["build"]["pyrexroot"], "docker"), - "--target", - "pyrex-%s" % image_type, - ] + commands = set() - if config["config"]["registry"]: - docker_args.extend( - [ - "--build-arg", - "MY_REGISTRY=%s/" % config["config"]["registry"], - ] - ) - - for e in ("http_proxy", "https_proxy"): - if e in os.environ: - docker_args.extend( - ["--build-arg", "%s=%s" % (e, os.environ[e])] - ) - - if config["dockerbuild"].get("args", ""): - docker_args.extend(shlex.split(config["dockerbuild"]["args"])) - - env = os.environ.copy() - for e in shlex.split(config["dockerbuild"]["env"]): - name, val = e.split("=", 1) - env[name] = val - - try: - if os.environ.get( - "PYREX_DOCKER_BUILD_QUIET", "1" - ) == "1" and config["dockerbuild"].getboolean("quiet"): - docker_args.append("-q") - build_config["build"]["buildid"] = ( - subprocess.check_output(docker_args, env=env) - .decode("utf-8") - .rstrip() - ) - else: - subprocess.check_call(docker_args, env=env) - build_config["build"]["buildid"] = get_image_id(config, tag) - - build_config["build"]["runid"] = build_config["build"]["buildid"] - - except subprocess.CalledProcessError: - return 1 - - build_config["build"]["buildhash"] = get_build_hash(build_config) - else: - try: - # Try to get the image This will fail if the image doesn't - # exist locally - build_config["build"]["buildid"] = get_image_id(config, tag) - except subprocess.CalledProcessError: - try: - docker_args = [docker_path, "pull", tag] - subprocess.check_call(docker_args) - - build_config["build"]["buildid"] = get_image_id(config, tag) - except subprocess.CalledProcessError: - return 1 - - build_config["build"]["runid"] = tag - else: - print( - textwrap.fill( - "Running outside of container. No guarantees are made about your Linux " - + "distribution's compatibility with Yocto." - ) + def add_commands(globs, target): + nonlocal commands + + for g in globs: + for cmd in glob.iglob(g): + norm_cmd = os.path.normpath(cmd) + if ( + norm_cmd not in commands + and os.path.isfile(cmd) + and os.access(cmd, os.X_OK) + ): + commands.add(norm_cmd) + name = os.path.basename(cmd) + + os.symlink(target.format(command=cmd), os.path.join(shimdir, name)) + + add_commands(nopyrex_globs, "{command}") + add_commands(command_globs, "exec-shim-pyrex") + + return shimdir + + +def main(): + def capture(args): + config = load_config() + build_config = build_image(config, {}) + if build_config is None: + return 1 + + with tempfile.NamedTemporaryFile(mode="r") as f: + env_args = {k: v for (k, v) in args.arg} + env_args["PYREX_CAPTURE_DEST"] = f.name + + # Startup script are only supposed to run after the initial capture + env_args["PYREX_SKIP_STARTUP"] = "1" + + docker_args = prep_docker( + config, + build_config, + ["/usr/libexec/pyrex/capture"] + args.init, + extra_env=env_args, + preserve_env=args.env, + extra_bind=[f.name] + args.bind, ) - print() - build_config["build"]["buildid"] = "" - build_config["build"]["runid"] = "" - with open(args.conffile, "w") as f: - build_config.write(f) + if not docker_args: + return 1 + + p = subprocess.run(docker_args) + if p.returncode: + return 1 + + capture = json.load(f) - shimdir = os.path.join(config["config"]["tempdir"], "bin") + build_config["run"] = capture["run"] + build_config["container"] = capture["container"] + build_config["tempdir"] = capture["tempdir"] + build_config["bypass"] = capture["bypass"] try: - shutil.rmtree(shimdir) + os.makedirs(build_config["tempdir"]) except Exception: pass - os.makedirs(shimdir) - - # Write out run convenience command - runfile = os.path.join(shimdir, "pyrex-run") - with open(runfile, "w") as f: - f.write( - textwrap.dedent( - """\ - #! /bin/sh - exec {pyrexroot}/{this_script} run {conffile} -- "$@" - """.format( - pyrexroot=config["build"]["pyrexroot"], - conffile=args.conffile, - this_script=THIS_SCRIPT, - ) - ) - ) - os.chmod(runfile, stat.S_IRWXU) - - # Write out config convenience command - configcmd = os.path.join(shimdir, "pyrex-config") - with open(configcmd, "w") as f: - f.write( - textwrap.dedent( - """\ - #! /bin/sh - exec {pyrexroot}/{this_script} config {conffile} "$@" - """.format( - pyrexroot=config["build"]["pyrexroot"], - conffile=args.conffile, - this_script=THIS_SCRIPT, - ) - ) - ) - os.chmod(configcmd, stat.S_IRWXU) - - # Write out the shim file - shimfile = os.path.join(shimdir, "exec-shim-pyrex") - with open(shimfile, "w") as f: - f.write( - textwrap.dedent( - """\ - #! /bin/sh - exec {runfile} "$(basename $0)" "$@" - """.format( - runfile=runfile - ) - ) - ) - os.chmod(shimfile, stat.S_IRWXU) - - # Write out the shell convenience command - shellfile = os.path.join(shimdir, "pyrex-shell") - with open(shellfile, "w") as f: - f.write( - textwrap.dedent( - """\ - #! /bin/sh - exec {runfile} {shell} "$@" - """.format( - runfile=runfile, shell=config["config"]["shell"] - ) - ) - ) - os.chmod(shellfile, stat.S_IRWXU) - - # Write out image rebuild command - rebuildfile = os.path.join(shimdir, "pyrex-rebuild") - with open(rebuildfile, "w") as f: - f.write( - textwrap.dedent( - """\ - #! /bin/sh - exec {pyrexroot}/{this_script} build {conffile} - """.format( - pyrexroot=config["build"]["pyrexroot"], - conffile=args.conffile, - this_script=THIS_SCRIPT, - ) - ) - ) - os.chmod(rebuildfile, stat.S_IRWXU) - command_globs = [g for g in config["config"]["commands"].split() if g] - nopyrex_globs = [g for g in config["config"]["commands_nopyrex"].split() if g] + buildconf = os.path.join(build_config["tempdir"], "build.json") - commands = set() + build_config["shimdir"] = create_shims(config, build_config, buildconf) - def add_commands(globs, target): - nonlocal commands + with open(buildconf, "w") as f: + json.dump(build_config, f) - for g in globs: - for cmd in glob.iglob(g): - norm_cmd = os.path.normpath(cmd) - if ( - norm_cmd not in commands - and os.path.isfile(cmd) - and os.access(cmd, os.X_OK) - ): - commands.add(norm_cmd) - name = os.path.basename(cmd) - - os.symlink( - target.format(command=cmd), os.path.join(shimdir, name) - ) + def write_cmd(c): + os.write(args.fd, c.encode("utf-8")) + os.write(args.fd, "\n".encode("utf-8")) - add_commands(nopyrex_globs, "{command}") - add_commands(command_globs, "exec-shim-pyrex") + write_cmd("PATH=%s:$PATH" % build_config["shimdir"]) + if capture["user"].get("cwd"): + write_cmd('cd "%s"' % capture["user"]["cwd"]) return 0 - def run(args): - config, _ = load_configs(args.conffile) + def rebuild(args): + config = load_config() + with open(args.buildconf, "r") as f: + build_config = json.load(f) - runid = config["build"]["runid"] + build_config = build_image(config, build_config) - if use_docker(config): - if not runid: - print( - "Container was not enabled when the environment was setup. Cannot use it now!" - ) - return 1 + if build_config is None: + return 1 - docker_path = config["config"]["dockerpath"] + build_config["shimdir"] = create_shims(config, build_config, args.buildconf) - try: - buildid = get_image_id(config, runid) - except subprocess.CalledProcessError as e: - print("Cannot verify docker image: %s\n" % e.output) - return 1 + with open(args.buildconf, "w") as f: + json.dump(build_config, f) - if buildid != config["build"]["buildid"]: - sys.stderr.write( - "WARNING: buildid for docker image %s has changed\n" % runid - ) + return 0 - if config["config"]["buildlocal"] == "1" and config["build"][ - "buildhash" - ] != get_build_hash(config): - sys.stderr.write( - "WARNING: The docker image source has changed and should be rebuilt.\n" - "Try running: 'pyrex-rebuild'\n" - ) + def run(args): + config = load_config() + with open(args.buildconf, "r") as f: + build_config = json.load(f) - # These are "hidden" keys in pyrex.ini that aren't publicized, and - # are primarily used for testing. Use they at your own risk, they - # may change - uid = int(config["run"].get("uid", os.getuid())) - gid = int(config["run"].get("gid", os.getgid())) - username = config["run"].get("username") or pwd.getpwuid(uid).pw_name - groupname = config["run"].get("groupname") or grp.getgrgid(gid).gr_name - init_command = config["run"].get( - "initcommand", config["build"]["initcommand"] + if use_docker(config): + docker_args = prep_docker( + config, + build_config, + ["/usr/libexec/pyrex/run"] + args.command, + extra_bind=build_config.get("run", {}).get("bind", []), + extra_env=build_config.get("run", {}).get("env", {}), + allow_test_config=True, ) - init_dir = config["run"].get("initdir", config["build"]["initdir"]) - - command_prefix = config["run"].get("commandprefix", "").splitlines() - - docker_args = [ - docker_path, - "run", - "--rm", - "-i", - "--net=host", - "-e", - "PYREX_USER=%s" % username, - "-e", - "PYREX_UID=%d" % uid, - "-e", - "PYREX_GROUP=%s" % groupname, - "-e", - "PYREX_GID=%d" % gid, - "-e", - "PYREX_HOME=%s" % os.environ["HOME"], - "-e", - "PYREX_INIT_COMMAND=%s" % init_command, - "-e", - "PYREX_INIT_DIR=%s" % init_dir, - "-e", - "PYREX_COMMAND_PREFIX=%s" % " ".join(command_prefix), - "--workdir", - os.getcwd(), - ] - - docker_envvars = [ - "PYREX_CLEANUP_EXIT_WAIT", - "PYREX_CLEANUP_LOG_FILE", - "PYREX_CLEANUP_LOG_LEVEL", - "TINI_VERBOSITY", - ] - - if config["build"]["docker_provider"] == "podman": - uid_length = get_subid_length("/etc/subuid", username) - if uid_length < 1: - sys.stderr.write("subuid name space is too small\n") - sys.exit(1) - - gid_length = get_subid_length("/etc/subgid", groupname) - if uid_length < 1: - sys.stderr.write("subgid name space is too small\n") - sys.exit(1) - - docker_args.extend( - [ - "--security-opt", - "label=disable", - # Fix up the UID/GID mapping so that the actual UID/GID - # inside the container maps to their actual UID/GID - # outside the container. Note that all offsets outside - # the container are relative to the start of the users - # subuid/subgid range. - # Map UID 0 up the actual user ID inside the container - # to the users subuid - "--uidmap", - "0:1:%d" % uid, - # Map the users actual UID inside the container to 0 in - # the users subuid namespace. The "root" user in the - # subuid namespace is special and maps to the users - # actual UID outside the namespace - "--uidmap", - "%d:0:1" % uid, - # Map the remaining UIDs after the actual user ID to - # continue using the users subuid range - "--uidmap", - "%d:%d:%d" % (uid + 1, uid + 1, uid_length - uid), - # Do the same for the GID - "--gidmap", - "0:1:%d" % gid, - "--gidmap", - "%d:0:1" % gid, - "--gidmap", - "%d:%d:%d" % (gid + 1, gid + 1, gid_length - gid), - ] - ) - # Run the docker image with a TTY if this script was run in a tty - if os.isatty(1): - docker_args.extend(["-t", "-e", "TERM=%s" % os.environ["TERM"]]) - - # Configure binds - for b in set(config["run"]["bind"].split()): - if not os.path.exists(b): - print("Error: bind source path {b} does not exist".format(b=b)) - continue - docker_args.extend(["--mount", "type=bind,src={b},dst={b}".format(b=b)]) - - docker_envvars.extend(config["run"]["envvars"].split()) - - # Special case: Make the user SSH authentication socket available in container - if "SSH_AUTH_SOCK" in os.environ: - socket = os.path.realpath(os.environ["SSH_AUTH_SOCK"]) - if not os.path.exists(socket): - print("Warning: SSH_AUTH_SOCK {} does not exist".format(socket)) - else: - docker_args.extend( - [ - "--mount", - "type=bind,src=%s,dst=/tmp/%s-ssh-agent-sock" - % (socket, username), - "-e", - "SSH_AUTH_SOCK=/tmp/%s-ssh-agent-sock" % username, - ] - ) - - # Pass along BB_ENV_EXTRAWHITE and anything it has whitelisted - if "BB_ENV_EXTRAWHITE" in os.environ: - docker_args.extend(["-e", "BB_ENV_EXTRAWHITE"]) - docker_envvars.extend(os.environ["BB_ENV_EXTRAWHITE"].split()) - - # Pass environment variables. If a variable passed with an argument - # "-e VAR" is not set in the parent environment, podman passes an - # empty value, where as docker doesn't pass it at all. For - # consistency, manually check if the variables exist before passing - # them. - for e in docker_envvars: - if e in os.environ: - docker_args.extend(["-e", e]) - - docker_args.extend(shlex.split(config["run"].get("args", ""))) - - docker_args.append("--") - docker_args.append(runid) - docker_args.append("/usr/libexec/pyrex/run") - docker_args.extend(args.command) + if not docker_args: + sys.exit(1) stop_coverage() - os.execvp(docker_args[0], docker_args) - print("Cannot exec docker!") sys.exit(1) else: - startup_args = [ - os.path.join(config["build"]["pyrexroot"], "docker", "run.sh") - ] - startup_args.extend(args.command) + command = [ + os.path.join(build_config["shimdir"], "pyrex-bypass") + ] + args.command env = os.environ.copy() - env["PYREX_INIT_COMMAND"] = config["build"]["initcommand"] - env["PYREX_INIT_DIR"] = config["build"]["initdir"] + for k, v in build_config.get("bypass", {}).get("env", {}).items(): + env[k] = v stop_coverage() - - os.execve(startup_args[0], startup_args, env) - - print("Cannot exec startup script") + os.execve(command[0], command, env) + print("Cannot exec command!") sys.exit(1) - def env(args): - config, _ = load_configs(args.conffile) - - def write_cmd(c): - os.write(args.fd, c.encode("utf-8")) - os.write(args.fd, "\n".encode("utf-8")) - - write_cmd("PATH=%s:${PATH}" % os.path.join(config["config"]["tempdir"], "bin")) - write_cmd('cd "%s"' % config["build"]["builddir"]) - return 0 - def config_get(args): - config, _ = load_configs(args.conffile) - + config = load_config() try: (section, name) = args.var.split(":") except ValueError: @@ -806,27 +800,41 @@ def config_get(args): "capture", help="Capture OE init environment" ) capture_parser.add_argument("fd", help="Output file descriptor", type=int) + capture_parser.add_argument( + "-a", + "--arg", + nargs=2, + metavar=("NAME", "VALUE"), + action="append", + default=[], + help="Set additional arguments as environment variable when capturing", + ) + capture_parser.add_argument( + "-e", + "--env", + action="append", + default=[], + help="Pass additional environment variables if present in parent shell", + ) + capture_parser.add_argument( + "--bind", action="append", default=[], help="Additional binds when capturing" + ) capture_parser.add_argument( "init", nargs="*", help="Initialization arguments", default=[] ) capture_parser.set_defaults(func=capture) - build_parser = subparsers.add_parser("build", help="Build Pyrex image") - build_parser.add_argument("conffile", help="Pyrex config file") - build_parser.set_defaults(func=build) + rebuild_parser = subparsers.add_parser("rebuild", help="Rebuild Pyrex image") + rebuild_parser.add_argument("buildconf", help="Pyrex build config file") + rebuild_parser.set_defaults(func=rebuild) run_parser = subparsers.add_parser("run", help="Run command in Pyrex") - run_parser.add_argument("conffile", help="Pyrex config file") + run_parser.add_argument("buildconf", help="Pyrex build config file") run_parser.add_argument("command", nargs="*", help="Command to execute", default=[]) run_parser.set_defaults(func=run) - env_parser = subparsers.add_parser("env", help="Setup Pyrex environment") - env_parser.add_argument("conffile", help="Pyrex config file") - env_parser.add_argument("fd", help="Output file descriptor", type=int) - env_parser.set_defaults(func=env) - config_parser = subparsers.add_parser("config", help="Pyrex configuration") - config_parser.add_argument("conffile", help="Pyrex config file") + config_parser.add_argument("buildconf", help="Pyrex build config file") config_subparsers = config_parser.add_subparsers( title="subcommands", description="Config subcommands", @@ -843,6 +851,7 @@ def config_get(args): config_get_parser.set_defaults(func=config_get) args = parser.parse_args() + func = getattr(args, "func", None) if not func: parser.print_usage() From d58772d55f56de90a4663817491cfb0a0232e549 Mon Sep 17 00:00:00 2001 From: Joshua Watt Date: Fri, 22 Nov 2019 16:29:58 -0600 Subject: [PATCH 04/48] podman-helper: Use docker-daemon pull podman can pull directly from the docker daemon, so do that instead of doing 'docker save | podman load' --- ci/podman-helper.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/ci/podman-helper.py b/ci/podman-helper.py index 8affdf4..f4125f4 100755 --- a/ci/podman-helper.py +++ b/ci/podman-helper.py @@ -42,16 +42,11 @@ def main(): try: subprocess.check_call(["docker", "build", "-t", args.tag] + extra_args) + subprocess.check_call(["podman", "pull", "docker-daemon:%s" % args.tag]) except subprocess.CalledProcessError as e: return e.returncode - docker_p = subprocess.Popen(["docker", "save", args.tag], stdout=subprocess.PIPE) - podman_p = subprocess.Popen(["podman", "load", args.tag], stdin=docker_p.stdout) - - docker_p.wait() - podman_p.wait() - - return docker_p.returncode or podman_p.returncode + return 0 if __name__ == "__main__": From 978933533a68b6d2a5ebe3a670c7fe3f4b9dcb64 Mon Sep 17 00:00:00 2001 From: Joshua Watt Date: Sat, 23 Nov 2019 13:50:55 -0600 Subject: [PATCH 05/48] Build using generic build command Encode a generic option for how to build the container image instead of building it up in pyrex. This makes CI testing a lot easier, since the build command can be simply replaced with the helper script. --- ci/test.py | 15 +++++++-------- pyrex.ini | 17 +++++++++++------ pyrex.py | 16 +--------------- 3 files changed, 19 insertions(+), 29 deletions(-) diff --git a/ci/test.py b/ci/test.py index 52f21b4..abadf11 100755 --- a/ci/test.py +++ b/ci/test.py @@ -58,12 +58,6 @@ def cleanup_build(): cleanup_build() os.makedirs(self.build_dir) - helper = os.path.join(PYREX_ROOT, "ci", "%s-helper.py" % self.docker_provider) - if os.path.exists(helper) and os.environ.get("USE_HELPER", "0") == "1": - self.dockerpath = helper - else: - self.dockerpath = self.docker_provider - self.pyrex_conf = os.path.join(self.build_dir, "pyrex.ini") conf = self.get_config() conf.write_conf() @@ -122,12 +116,17 @@ def write_config_helper(conf): # Setup the config suitable for testing config["config"]["dockerimage"] = self.test_image - config["config"]["dockerpath"] = self.dockerpath + config["config"]["dockerpath"] = self.docker_provider config["config"]["buildlocal"] = "0" config["config"]["pyrextag"] = ( os.environ.get(TEST_PREBUILT_TAG_ENV_VAR, "") or "ci-test" ) config["run"]["bind"] = self.build_dir + config["dockerbuild"]["buildcommand"] = "%s --provider=%s %s" % ( + os.path.join(PYREX_ROOT, "ci", "build_image.py"), + self.docker_provider, + self.test_image, + ) return config @@ -567,7 +566,7 @@ def test_guest_image(self): def test_default_ini_image(self): # Tests that the default image specified in pyrex.ini is valid - config = configparser.RawConfigParser() + config = pyrex.Config() config.read_string(pyrex.read_default_config(True)) self.assertIn(config["config"]["dockerimage"], TEST_IMAGES) diff --git a/pyrex.ini b/pyrex.ini index 36ac9dd..5dabedb 100644 --- a/pyrex.ini +++ b/pyrex.ini @@ -15,9 +15,12 @@ confversion = @CONFVERSION@ # The docker executable to use %dockerpath = docker +# The type of image to build +%imagetype = oe + # As a convenience, the name of a Pyrex provided Docker image # can be specified here -%dockerimage = ubuntu-18.04-oe +%dockerimage = ubuntu-18.04-${config:imagetype} # As a convenience, the tag of the Pyrex provided image. Defaults to the # Pyrex version. @@ -46,11 +49,13 @@ confversion = @CONFVERSION@ %envimport = HOME [dockerbuild] -# The Dockerfile used to build the Pyrex image (if building locally) -%dockerfile = ${pyrex:pyrexroot}/docker/Dockerfile - -# Arguments to pass when building the docker image -%args = +# The command used to build container images +%buildcommand = +% ${config:dockerpath} build +% -t ${config:tag} +% --network=host +% -f ${pyrex:pyrexroot}/docker/Dockerfile +% --target=pyrex-${config:imagetype} % --build-arg PYREX_BASE=${config:dockerimage} # Build quietly? diff --git a/pyrex.py b/pyrex.py index 9830f42..47976c3 100755 --- a/pyrex.py +++ b/pyrex.py @@ -228,18 +228,7 @@ def build_image(config, build_config): (_, _, image_type) = config["config"]["dockerimage"].split("-") - docker_args = [ - docker_path, - "build", - "-t", - tag, - "-f", - config["dockerbuild"]["dockerfile"], - "--network=host", - os.path.join(PYREX_ROOT, "docker"), - "--target", - "pyrex-%s" % image_type, - ] + docker_args = shlex.split(config["dockerbuild"]["buildcommand"]) if config["config"]["registry"]: docker_args.extend( @@ -250,9 +239,6 @@ def build_image(config, build_config): if e in os.environ: docker_args.extend(["--build-arg", "%s=%s" % (e, os.environ[e])]) - if config["dockerbuild"].get("args", ""): - docker_args.extend(shlex.split(config["dockerbuild"]["args"])) - env = os.environ.copy() for e in shlex.split(config["dockerbuild"]["env"]): name, val = e.split("=", 1) From ab60d9e7377db44ba113fbef35349fc8c9ac8be8 Mon Sep 17 00:00:00 2001 From: Joshua Watt Date: Sun, 1 Dec 2019 15:10:33 -0600 Subject: [PATCH 06/48] Remove docker references Now that Pyrex supports multiple container engines (docker and podman), remove explicit references to "docker" in the codebase and documentation. --- DEVELOPING.md | 14 +- README.md | 6 +- ci/build_image.py | 12 +- ci/deploy_docker.py | 14 +- ci/test.py | 36 ++--- {docker => image}/Dockerfile | 16 +- {docker => image}/bypass.sh | 0 {docker => image}/capture.sh | 0 {docker => image}/cleanup.py | 0 {docker => image}/entry.py | 6 +- ...0001-Use-pkg-config-to-find-packages.patch | 0 {docker => image}/run.sh | 0 {docker => image}/test_startup.sh | 0 pyrex.ini | 48 +++--- pyrex.py | 137 +++++++++--------- 15 files changed, 147 insertions(+), 142 deletions(-) rename {docker => image}/Dockerfile (98%) rename {docker => image}/bypass.sh (100%) rename {docker => image}/capture.sh (100%) rename {docker => image}/cleanup.py (100%) rename {docker => image}/entry.py (95%) rename {docker => image}/patches/0001-Use-pkg-config-to-find-packages.patch (100%) rename {docker => image}/run.sh (100%) rename {docker => image}/test_startup.sh (100%) diff --git a/DEVELOPING.md b/DEVELOPING.md index d939d24..37374d6 100644 --- a/DEVELOPING.md +++ b/DEVELOPING.md @@ -20,8 +20,8 @@ pass the `--reformat` option: ## Testing Pyrex has a comprehensive test suite that can be used to test all generated -Docker images. Some external test data is required to perform the test. To -download this data and prepare your sandbox for testing, run: +images. Some external test data is required to perform the test. To download +this data and prepare your sandbox for testing, run: ```shell ./ci/prepare.sh @@ -49,10 +49,10 @@ non-word characters replaced by `_`. For example, to test only the ``` ## Building images locally -Pyrex pulls prebuilt Docker containers from DockerHub by default, which should -be sufficient for most users. However, users that are active developing on -Pyrex or wish to build images locally instead of using published images can do -so by making the following changes to `pyrex.ini`: +Pyrex pulls prebuilt images from DockerHub by default, which should be +sufficient for most users. However, users that are active developing on Pyrex +or wish to build images locally instead of using published images can do so by +making the following changes to `pyrex.ini`: 1. Set `config:buildlocal` to `1` 2. Change `config:tag` to an alternate tag suffix instead of @@ -64,7 +64,7 @@ so by making the following changes to `pyrex.ini`: ``` [config] - tag = ${config:dockerimage}:my-image + tag = ${config:image}:my-image ``` 3. Set `config:dockerfile` to the path where the Dockerfile you wish to build diff --git a/README.md b/README.md index a0b2221..5ff3ed1 100644 --- a/README.md +++ b/README.md @@ -194,8 +194,8 @@ wraps locally instead of in the container. This can be done in one of two ways: 1. Set `${run:enable}` to `0` in `pyrex.ini` which will disable using the container engine for all commands -2. Set the environment variable `PYREX_DOCKER` to `0`. Any Pyrex commands run - with this variable will not be run in the container. +2. Set the environment variable `PYREX_USE_CONTAINER` to `0`. Any Pyrex + commands run with this variable will not be run in the container. ## What doesn't work? The following items are either known to not work, or haven't been fully tested: @@ -238,7 +238,7 @@ build the image locally instead. See the [Developer Documentation][]. chosen because it is one of the [Sanity Tested Distros][] that Yocto supports. Pyrex aims to support a vanilla Yocto setup with minimal manual configuration. -* *What's with [cleanup.py](./docker/cleanup.py)?* When a container's main +* *What's with [cleanup.py](./image/cleanup.py)?* When a container's main process exits, any remaining process appears to be sent a `SIGKILL`. This can cause a significant problem with many of the child processes that bitbake spawns, since unceremoniously killing them might result in lost data. The diff --git a/ci/build_image.py b/ci/build_image.py index 718264e..95239f8 100755 --- a/ci/build_image.py +++ b/ci/build_image.py @@ -37,8 +37,8 @@ def main(): (_, _, image_type) = args.image.split("-") this_dir = os.path.abspath(os.path.dirname(__file__)) - docker_dir = os.path.join(this_dir, "..", "docker") - docker_file = os.path.join(docker_dir, "Dockerfile") + image_dir = os.path.join(this_dir, "..", "image") + docker_file = os.path.join(image_dir, "Dockerfile") helper = os.path.join(THIS_DIR, "%s-helper.py" % args.provider) if os.path.exists(helper) and os.environ.get("USE_HELPER", "0") == "1": @@ -47,7 +47,7 @@ def main(): else: provider = args.provider - docker_args = [ + engine_args = [ provider, "build", "-t", @@ -57,14 +57,14 @@ def main(): "--network=host", "--build-arg", "PYREX_BASE=%s" % args.image, - docker_dir, + image_dir, "--target", "pyrex-%s" % image_type, ] if args.quiet: p = subprocess.Popen( - docker_args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT + engine_args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT ) while True: try: @@ -80,7 +80,7 @@ def main(): return p.returncode else: - subprocess.check_call(docker_args) + subprocess.check_call(engine_args) if __name__ == "__main__": diff --git a/ci/deploy_docker.py b/ci/deploy_docker.py index 5298752..6c21db4 100755 --- a/ci/deploy_docker.py +++ b/ci/deploy_docker.py @@ -9,7 +9,7 @@ def main(): - parser = argparse.ArgumentParser(description="Deploy docker images") + parser = argparse.ArgumentParser(description="Deploy container images to Dockerhub") parser.add_argument( "--login", action="store_true", @@ -22,7 +22,7 @@ def main(): args = parser.parse_args() this_dir = os.path.abspath(os.path.dirname(sys.argv[0])) - docker_dir = os.path.join(this_dir, "..", "docker") + image_dir = os.path.join(this_dir, "..", "image") image = args.image if ":" in image: @@ -70,7 +70,7 @@ def main(): print("Deploying %s..." % name) - # Get a login token for the docker registry and download the manifest + # Get a login token for the Docker registry and download the manifest token = requests.get( "https://auth.docker.io/token?service=registry.docker.io&scope=repository:%s:pull" % repo, @@ -91,11 +91,11 @@ def main(): print("Building", name) # Construct the arguments for the build command. - docker_build_args = [ + build_args = [ "-t", name, "-f", - "%s/Dockerfile" % docker_dir, + "%s/Dockerfile" % image_dir, "--build-arg", "PYREX_BASE=%s" % image, "--target", @@ -103,10 +103,10 @@ def main(): ] # Add the build context directory to our arguments. - docker_build_args.extend(["--", docker_dir]) + build_args.extend(["--", image_dir]) try: - subprocess.check_call(["docker", "build"] + docker_build_args) + subprocess.check_call(["docker", "build"] + build_args) except subprocess.CalledProcessError: print("Building failed!") return 1 diff --git a/ci/test.py b/ci/test.py index abadf11..54d723e 100755 --- a/ci/test.py +++ b/ci/test.py @@ -75,7 +75,7 @@ def cleanup_env(): os.makedirs(self.bin_dir) os.symlink("/usr/bin/python2", os.path.join(self.bin_dir, "python")) os.environ["PATH"] = self.bin_dir + ":" + os.environ["PATH"] - os.environ["PYREX_DOCKER_BUILD_QUIET"] = "0" + os.environ["PYREX_BUILD_QUIET"] = "0" if "SSH_AUTH_SOCK" in os.environ: del os.environ["SSH_AUTH_SOCK"] self.addCleanup(cleanup_env) @@ -87,13 +87,13 @@ def cleanup_env(): def prebuild_image(self): global built_images - image = ":".join((self.test_image, self.docker_provider)) + image = ":".join((self.test_image, self.provider)) if image not in built_images: self.assertSubprocess( [ os.path.join(PYREX_ROOT, "ci", "build_image.py"), "--provider", - self.docker_provider, + self.provider, self.test_image, ] ) @@ -115,16 +115,16 @@ def write_config_helper(conf): config.read_string(pyrex.read_default_config(True)) # Setup the config suitable for testing - config["config"]["dockerimage"] = self.test_image - config["config"]["dockerpath"] = self.docker_provider + config["config"]["image"] = self.test_image + config["config"]["engine"] = self.provider config["config"]["buildlocal"] = "0" config["config"]["pyrextag"] = ( os.environ.get(TEST_PREBUILT_TAG_ENV_VAR, "") or "ci-test" ) config["run"]["bind"] = self.build_dir - config["dockerbuild"]["buildcommand"] = "%s --provider=%s %s" % ( + config["imagebuild"]["buildcommand"] = "%s --provider=%s %s" % ( os.path.join(PYREX_ROOT, "ci", "build_image.py"), - self.docker_provider, + self.provider, self.test_image, ) @@ -265,7 +265,7 @@ def test_disable_pyrex(self): def capture_pyrex_state(*args, **kwargs): capture_file = os.path.join(self.thread_dir, "pyrex_capture") - if self.docker_provider == "podman": + if self.provider == "podman": self.assertPyrexContainerShellCommand( "cp --no-preserve=all /proc/1/cmdline %s" % capture_file, *args, @@ -281,7 +281,7 @@ def capture_pyrex_state(*args, **kwargs): return f.read() def capture_local_state(): - if self.docker_provider == "podman": + if self.provider == "podman": with open("/proc/1/cmdline", "rb") as f: return f.read() else: @@ -294,18 +294,18 @@ def capture_local_state(): self.assertNotEqual(local_state, pyrex_state) env = os.environ.copy() - env["PYREX_DOCKER"] = "0" + env["PYREX_USE_CONTAINER"] = "0" pyrex_state = capture_pyrex_state(env=env) self.assertEqual(local_state, pyrex_state) def test_quiet_build(self): env = os.environ.copy() - env["PYREX_DOCKER_BUILD_QUIET"] = "1" + env["PYREX_BUILD_QUIET"] = "1" self.assertPyrexHostCommand("true", env=env) def test_bad_provider(self): # Prevent container build from working - os.symlink("/bin/false", os.path.join(self.bin_dir, self.docker_provider)) + os.symlink("/bin/false", os.path.join(self.bin_dir, self.provider)) # Verify that attempting to run build pyrex without a valid container # provider shows the installation instructions @@ -339,7 +339,7 @@ def test_owner_env(self): # This test is primarily designed to ensure that everything is passed # correctly through 'pyrex run' - if self.docker_provider == "podman": + if self.provider == "podman": self.skipTest("Rootless podman cannot change to another user") conf = self.get_config() @@ -489,7 +489,7 @@ def test_tag_overwrite(self): self.assertPyrexHostCommand("true", returncode=1) output = self.assertSubprocess( - [self.docker_provider, "images", "-q", conf["config"]["tag"]], capture=True + [self.provider, "images", "-q", conf["config"]["tag"]], capture=True ).strip() self.assertEqual(output, "", msg="Tagged image found!") @@ -569,7 +569,7 @@ def test_default_ini_image(self): config = pyrex.Config() config.read_string(pyrex.read_default_config(True)) - self.assertIn(config["config"]["dockerimage"], TEST_IMAGES) + self.assertIn(config["config"]["image"], TEST_IMAGES) def test_envvars(self): conf = self.get_config() @@ -734,7 +734,7 @@ def test_top_dir(self): self.assertEqual(oe_topdir, pyrex_topdir) -DOCKER_PROVIDERS = ("docker", "podman") +PROVIDERS = ("docker", "podman") TEST_IMAGES = ( "ubuntu-14.04-base", @@ -749,7 +749,7 @@ def test_top_dir(self): def add_image_tests(): self = sys.modules[__name__] - for provider in DOCKER_PROVIDERS: + for provider in PROVIDERS: for image in TEST_IMAGES: (_, _, image_type) = image.split("-") @@ -762,7 +762,7 @@ def add_image_tests(): type( name, (parent, unittest.TestCase), - {"test_image": image, "docker_provider": provider}, + {"test_image": image, "provider": provider}, ), ) diff --git a/docker/Dockerfile b/image/Dockerfile similarity index 98% rename from docker/Dockerfile rename to image/Dockerfile index a7138c5..8046b38 100644 --- a/docker/Dockerfile +++ b/image/Dockerfile @@ -301,8 +301,8 @@ RUN set -x && export DEBIAN_FRONTEND=noninteractive && \ g++-multilib \ # Screen to enable devshell screen \ -# Base OS stuff that reasonable workstations have, but which the Docker -# registry image doesn't +# Base OS stuff that reasonable workstations have, but which the registry image +# doesn't tzdata \ && rm -rf /var/lib/apt/lists/* @@ -372,8 +372,8 @@ RUN set -x && export DEBIAN_FRONTEND=noninteractive && \ g++-multilib \ # Screen to enable devshell screen \ -# Base OS stuff that reasonable workstations have, but which the Docker -# registry image doesn't +# Base OS stuff that reasonable workstations have, but which the registry image +# doesn't tzdata \ && rm -rf /var/lib/apt/lists/* @@ -439,8 +439,8 @@ RUN set -x && export DEBIAN_FRONTEND=noninteractive && apt-get -y update && apt- g++-multilib \ # Screen to enable devshell screen \ -# Base OS stuff that reasonable workstations have, but which the Docker -# registry image doesn't +# Base OS stuff that reasonable workstations have, but which the registry image +# doesn't tzdata \ && rm -rf /var/lib/apt/lists/* @@ -509,8 +509,8 @@ ENV MY_REGISTRY none # Setup Icecream distributed compiling client. The client tries several IPC # mechanisms to find the daemon, including connecting to a localhost TCP # socket. Since the local Icecream daemon (iceccd) is not started when the -# docker container starts, the client will not find it and instead connect to -# the host Icecream daemon (as long as the container is run with --net=host). +# container starts, the client will not find it and instead connect to the host +# Icecream daemon (as long as the container is run with --net=host). RUN mkdir -p /usr/share/icecc/toolchain && \ cd /usr/share/icecc/toolchain/ && \ TC_NAME=$(mktemp) && \ diff --git a/docker/bypass.sh b/image/bypass.sh similarity index 100% rename from docker/bypass.sh rename to image/bypass.sh diff --git a/docker/capture.sh b/image/capture.sh similarity index 100% rename from docker/capture.sh rename to image/capture.sh diff --git a/docker/cleanup.py b/image/cleanup.py similarity index 100% rename from docker/cleanup.py rename to image/cleanup.py diff --git a/docker/entry.py b/image/entry.py similarity index 95% rename from docker/entry.py rename to image/entry.py index 5c46161..c348f59 100644 --- a/docker/entry.py +++ b/image/entry.py @@ -34,9 +34,9 @@ def get_var(name): def main(): # Block the SIGTSTP signal. We haven't figured out how to do proper job - # control inside of docker yet, and if the user accidentally presses CTRL+Z - # is will freeze the console without actually stopping the build. To - # prevent this, block SIGTSTP in all child processes. This results in + # control inside of the container yet, and if the user accidentally presses + # CTRL+Z is will freeze the console without actually stopping the build. + # To prevent this, block SIGTSTP in all child processes. This results in # CTRL+Z doing nothing. signal.pthread_sigmask(signal.SIG_BLOCK, [signal.SIGTSTP]) diff --git a/docker/patches/0001-Use-pkg-config-to-find-packages.patch b/image/patches/0001-Use-pkg-config-to-find-packages.patch similarity index 100% rename from docker/patches/0001-Use-pkg-config-to-find-packages.patch rename to image/patches/0001-Use-pkg-config-to-find-packages.patch diff --git a/docker/run.sh b/image/run.sh similarity index 100% rename from docker/run.sh rename to image/run.sh diff --git a/docker/test_startup.sh b/image/test_startup.sh similarity index 100% rename from docker/test_startup.sh rename to image/test_startup.sh diff --git a/pyrex.ini b/pyrex.ini index 5dabedb..ad18cb8 100644 --- a/pyrex.ini +++ b/pyrex.ini @@ -12,27 +12,30 @@ # to be specified in the user config file confversion = @CONFVERSION@ -# The docker executable to use -%dockerpath = docker +# The Container engine executable (e.g. docker, podman) to use. If the path +# does not start with a "/", the $PATH environment variable will be searched +# (i.e. execvp rules) +%engine = docker # The type of image to build %imagetype = oe -# As a convenience, the name of a Pyrex provided Docker image +# As a convenience, the name of a Pyrex provided image # can be specified here -%dockerimage = ubuntu-18.04-${config:imagetype} +%image = ubuntu-18.04-${config:imagetype} # As a convenience, the tag of the Pyrex provided image. Defaults to the # Pyrex version. %pyrextag = v${pyrex:version} -# The name of the tag given to the Docker image. If you want to keep around -# different Pyrex images simultaneously, each should have a unique tag -%tag = garminpyrex/${config:dockerimage}:${config:pyrextag} +# The name of the tag given to the image. If you want to keep around different +# Pyrex images simultaneously, each should have a unique tag +%tag = garminpyrex/${config:image}:${config:pyrextag} -# If set to 1, the Docker image is built up locally every time the environment -# is sourced. If set to 0, building the image will be skipped, which means that -# docker may attempt to download a prebuilt image from a repository +# If set to 1, the image is built up locally every time the environment is +# sourced. If set to 0, building the image will be skipped, which means that +# the container engine may attempt to download a prebuilt image from a +# repository %buildlocal = 0 # The Docker registry from which to fetch the image. Leave empty for the @@ -48,39 +51,40 @@ confversion = @CONFVERSION@ # should be taken %envimport = HOME -[dockerbuild] +[imagebuild] # The command used to build container images %buildcommand = -% ${config:dockerpath} build +% ${config:engine} build % -t ${config:tag} % --network=host -% -f ${pyrex:pyrexroot}/docker/Dockerfile +% -f ${pyrex:pyrexroot}/image/Dockerfile % --target=pyrex-${config:imagetype} -% --build-arg PYREX_BASE=${config:dockerimage} +% --build-arg PYREX_BASE=${config:image} # Build quietly? %quiet = true -# Environment variables to set when building the docker image +# Environment variables to set when building the image %env = % DOCKER_BUILDKIT=1 -# Runtime Docker options. Changes in this section take effect the next time a -# Pyrex command is run +# Runtime options. Changes in this section take effect the next time a Pyrex +# command is run [run] -# Should docker be enabled? Can be used to disable using docker for all commands +# Should the container engine be enabled? Can be used to disable using the +# container engine for all commands %enable = 1 -# A list of directories that should be bound when running in the Docker image +# A list of directories that should be bound when running in the container %bind = -# A list of environment variables that should be propagated to the Docker -# container if set in the parent environment +# A list of environment variables that should be propagated to the container +# if set in the parent environment %envvars = % http_proxy % https_proxy -# Extra arguments to pass to docker run. For example: +# Extra arguments to pass when running the image. For example: # --mount type=bind,src=${env:HOME}/.ssh,dst=${env:HOME}/.ssh,readonly # --device /dev/kvm %args = diff --git a/pyrex.py b/pyrex.py index 47976c3..57ef688 100755 --- a/pyrex.py +++ b/pyrex.py @@ -131,41 +131,40 @@ def stop_coverage(): def get_image_id(config, image): - docker_args = [ - config["config"]["dockerpath"], + engine_args = [ + config["config"]["engine"], "image", "inspect", image, "--format={{.Id}}", ] return ( - subprocess.check_output(docker_args, stderr=subprocess.DEVNULL) + subprocess.check_output(engine_args, stderr=subprocess.DEVNULL) .decode("utf-8") .rstrip() ) -def use_docker(config): - return os.environ.get("PYREX_DOCKER", config["run"]["enable"]) == "1" +def use_container(config): + return os.environ.get("PYREX_USE_CONTAINER", config["run"]["enable"]) == "1" def build_image(config, build_config): build_config.setdefault("build", {}) - docker_path = config["config"]["dockerpath"] + engine = config["config"]["engine"] - # Check minimum docker version try: - (provider, version) = get_docker_info(config) + (provider, version) = get_engine_info(config) except (subprocess.CalledProcessError, FileNotFoundError): print( textwrap.fill( ( - "Unable to run '%s' as docker. Please make sure you have it installed." + "Unable to run '%s'. Please make sure you have it installed." + "For installation instructions, see the docker website. Commonly, " + "one of the following is relevant:" ) - % docker_path + % engine ) ) print() @@ -195,13 +194,13 @@ def build_image(config, build_config): ) ) print() - print(" export PYREX_DOCKER=0") + print(" export PYREX_USE_CONTAINER=0") print(" . init-build-env ...") print() return None if provider is None: - sys.stderr.write("Could not get docker version!\n") + sys.stderr.write("Could not get container engine version!\n") return None if provider == "docker" and int(version.split(".")[0]) < MINIMUM_DOCKER_VERSION: @@ -211,7 +210,7 @@ def build_image(config, build_config): ) return None - build_config["docker_provider"] = provider + build_config["provider"] = provider tag = config["config"]["tag"] @@ -226,36 +225,36 @@ def build_image(config, build_config): print("Getting container image up to date...") - (_, _, image_type) = config["config"]["dockerimage"].split("-") + (_, _, image_type) = config["config"]["image"].split("-") - docker_args = shlex.split(config["dockerbuild"]["buildcommand"]) + engine_args = shlex.split(config["imagebuild"]["buildcommand"]) if config["config"]["registry"]: - docker_args.extend( + engine_args.extend( ["--build-arg", "MY_REGISTRY=%s/" % config["config"]["registry"]] ) for e in ("http_proxy", "https_proxy"): if e in os.environ: - docker_args.extend(["--build-arg", "%s=%s" % (e, os.environ[e])]) + engine_args.extend(["--build-arg", "%s=%s" % (e, os.environ[e])]) env = os.environ.copy() - for e in shlex.split(config["dockerbuild"]["env"]): + for e in shlex.split(config["imagebuild"]["env"]): name, val = e.split("=", 1) env[name] = val try: - if os.environ.get("PYREX_DOCKER_BUILD_QUIET", "1") == "1" and config[ - "dockerbuild" + if os.environ.get("PYREX_BUILD_QUIET", "1") == "1" and config[ + "imagebuild" ].getboolean("quiet"): - docker_args.append("-q") + engine_args.append("-q") build_config["build"]["buildid"] = ( - subprocess.check_output(docker_args, env=env) + subprocess.check_output(engine_args, env=env) .decode("utf-8") .rstrip() ) else: - subprocess.check_call(docker_args, env=env) + subprocess.check_call(engine_args, env=env) build_config["build"]["buildid"] = get_image_id(config, tag) build_config["build"]["runid"] = build_config["build"]["buildid"] @@ -271,8 +270,8 @@ def build_image(config, build_config): build_config["build"]["buildid"] = get_image_id(config, tag) except subprocess.CalledProcessError: try: - docker_args = [docker_path, "pull", tag] - subprocess.check_call(docker_args) + engine_args = [engine, "pull", tag] + subprocess.check_call(engine_args) build_config["build"]["buildid"] = get_image_id(config, tag) except subprocess.CalledProcessError: @@ -285,14 +284,14 @@ def build_image(config, build_config): def get_build_hash(config): # Docker doesn't currently have any sort of "dry-run" mechanism that could - # be used to determine if the dockerfile has changed and needs a rebuild. + # be used to determine if the Dockerfile has changed and needs a rebuild. # (See https://github.com/moby/moby/issues/38101). # # Until one is added, we use a simple hash of the files in the pyrex - # "docker" folder to determine when it is out of date. + # "image" folder to determine when it is out of date. h = hashlib.sha256() - for (root, dirs, files) in os.walk(os.path.join(PYREX_ROOT, "docker")): + for (root, dirs, files) in os.walk(os.path.join(PYREX_ROOT, "image")): # Process files and directories in alphabetical order so that hashing # is consistent dirs.sort() @@ -318,9 +317,9 @@ def get_build_hash(config): return h.hexdigest() -def get_docker_info(config): - docker_path = config["config"]["dockerpath"] - output = subprocess.check_output([docker_path, "--version"]).decode("utf-8") +def get_engine_info(config): + engine = config["config"]["engine"] + output = subprocess.check_output([engine, "--version"]).decode("utf-8") m = re.match(r"(?P\S+) +version +(?P[^\s,]+)", output) if m is not None: return (m.group("provider").lower(), m.group("version")) @@ -336,7 +335,7 @@ def get_subid_length(filename, name): return 0 -def prep_docker( +def prep_container( config, build_config, command, @@ -354,22 +353,24 @@ def prep_docker( ) return [] - docker_path = config["config"]["dockerpath"] + engine = config["config"]["engine"] try: buildid = get_image_id(config, runid) except subprocess.CalledProcessError as e: - print("Cannot verify docker image: %s\n" % e.output) + print("Cannot verify container image: %s\n" % e.output) return [] if buildid != build_config["build"]["buildid"]: - sys.stderr.write("WARNING: buildid for docker image %s has changed\n" % runid) + sys.stderr.write( + "WARNING: buildid for container image %s has changed\n" % runid + ) if config["config"]["buildlocal"] == "1" and build_config["build"][ "buildhash" ] != get_build_hash(config): sys.stderr.write( - "WARNING: The docker image source has changed and should be rebuilt.\n" + "WARNING: The container image source has changed and should be rebuilt.\n" "Try running: 'pyrex-rebuild'\n" ) @@ -389,8 +390,8 @@ def prep_docker( command_prefix = config["run"].get("commandprefix", "").splitlines() - docker_args = [ - docker_path, + engine_args = [ + engine, "run", "--rm", "-i", @@ -411,14 +412,14 @@ def prep_docker( os.getcwd(), ] - docker_envvars = [ + container_envvars = [ "PYREX_CLEANUP_EXIT_WAIT", "PYREX_CLEANUP_LOG_FILE", "PYREX_CLEANUP_LOG_LEVEL", "TINI_VERBOSITY", ] - if build_config["docker_provider"] == "podman": + if build_config["provider"] == "podman": uid_length = get_subid_length("/etc/subuid", username) if uid_length < 1: sys.stderr.write("subuid name space is too small\n") @@ -429,7 +430,7 @@ def prep_docker( sys.stderr.write("subgid name space is too small\n") sys.exit(1) - docker_args.extend( + engine_args.extend( [ "--security-opt", "label=disable", @@ -462,9 +463,9 @@ def prep_docker( ] ) - # Run the docker image with a TTY if this script was run in a tty + # Run the container with a TTY if this script was run in a tty if os.isatty(1): - docker_args.extend(["-t", "-e", "TERM=%s" % os.environ["TERM"]]) + engine_args.extend(["-t", "-e", "TERM=%s" % os.environ["TERM"]]) # Configure binds binds = config["run"]["bind"].split() + extra_bind @@ -472,9 +473,9 @@ def prep_docker( if not os.path.exists(b): print("Error: bind source path {b} does not exist".format(b=b)) continue - docker_args.extend(["--mount", "type=bind,src={b},dst={b}".format(b=b)]) + engine_args.extend(["--mount", "type=bind,src={b},dst={b}".format(b=b)]) - docker_envvars.extend(config["run"]["envvars"].split()) + container_envvars.extend(config["run"]["envvars"].split()) # Special case: Make the user SSH authentication socket available in container if "SSH_AUTH_SOCK" in os.environ: @@ -482,7 +483,7 @@ def prep_docker( if not os.path.exists(socket): print("Warning: SSH_AUTH_SOCK {} does not exist".format(socket)) else: - docker_args.extend( + engine_args.extend( [ "--mount", "type=bind,src=%s,dst=/tmp/%s-ssh-agent-sock" % (socket, username), @@ -493,27 +494,27 @@ def prep_docker( # Pass along BB_ENV_EXTRAWHITE and anything it has whitelisted if "BB_ENV_EXTRAWHITE" in os.environ: - docker_args.extend(["-e", "BB_ENV_EXTRAWHITE"]) - docker_envvars.extend(os.environ["BB_ENV_EXTRAWHITE"].split()) + engine_args.extend(["-e", "BB_ENV_EXTRAWHITE"]) + container_envvars.extend(os.environ["BB_ENV_EXTRAWHITE"].split()) # Pass environment variables. If a variable passed with an argument # "-e VAR" is not set in the parent environment, podman passes an - # empty value, where as docker doesn't pass it at all. For + # empty value, where as Docker doesn't pass it at all. For # consistency, manually check if the variables exist before passing # them. - for e in docker_envvars + preserve_env: + for e in container_envvars + preserve_env: if e in os.environ: - docker_args.extend(["-e", e]) + engine_args.extend(["-e", e]) for k, v in extra_env.items(): - docker_args.extend(["-e", "%s=%s" % (k, v)]) + engine_args.extend(["-e", "%s=%s" % (k, v)]) - docker_args.extend(shlex.split(config["run"].get("args", ""))) + engine_args.extend(shlex.split(config["run"].get("args", ""))) - docker_args.append("--") - docker_args.append(runid) - docker_args.extend(command) - return docker_args + engine_args.append("--") + engine_args.append(runid) + engine_args.extend(command) + return engine_args def create_shims(config, build_config, buildconf): @@ -602,8 +603,8 @@ def create_shims(config, build_config, buildconf): # Create bypass command bypassfile = os.path.join(shimdir, "pyrex-bypass") - docker_args = [ - config["config"]["dockerpath"], + engine_args = [ + config["config"]["engine"], "run", "--rm", "--entrypoint", @@ -612,7 +613,7 @@ def create_shims(config, build_config, buildconf): "/usr/libexec/pyrex/bypass", ] with open(bypassfile, "w") as f: - subprocess.run(docker_args, check=True, stdout=f) + subprocess.run(engine_args, check=True, stdout=f) os.chmod(bypassfile, stat.S_IRWXU) # Create shims @@ -657,7 +658,7 @@ def capture(args): # Startup script are only supposed to run after the initial capture env_args["PYREX_SKIP_STARTUP"] = "1" - docker_args = prep_docker( + engine_args = prep_container( config, build_config, ["/usr/libexec/pyrex/capture"] + args.init, @@ -666,10 +667,10 @@ def capture(args): extra_bind=[f.name] + args.bind, ) - if not docker_args: + if not engine_args: return 1 - p = subprocess.run(docker_args) + p = subprocess.run(engine_args) if p.returncode: return 1 @@ -723,8 +724,8 @@ def run(args): with open(args.buildconf, "r") as f: build_config = json.load(f) - if use_docker(config): - docker_args = prep_docker( + if use_container(config): + engine_args = prep_container( config, build_config, ["/usr/libexec/pyrex/run"] + args.command, @@ -733,12 +734,12 @@ def run(args): allow_test_config=True, ) - if not docker_args: + if not engine_args: sys.exit(1) stop_coverage() - os.execvp(docker_args[0], docker_args) - print("Cannot exec docker!") + os.execvp(engine_args[0], engine_args) + print("Cannot exec container!") sys.exit(1) else: command = [ From fd89be0ceca9a30bcbc06dd18cf174c4bab3cb08 Mon Sep 17 00:00:00 2001 From: Joshua Watt Date: Sun, 1 Dec 2019 15:12:24 -0600 Subject: [PATCH 07/48] Bump config file version All of the changes to the config file for initializing inside the container and removing references to docker have made the old variables in the config file invalid. Bump the config format version to force users to update. --- pyrex.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyrex.py b/pyrex.py index 57ef688..8765954 100755 --- a/pyrex.py +++ b/pyrex.py @@ -38,7 +38,7 @@ THIS_SCRIPT = os.path.abspath(__file__) PYREX_ROOT = os.path.dirname(THIS_SCRIPT) -PYREX_CONFVERSION = "1" +PYREX_CONFVERSION = "2" MINIMUM_DOCKER_VERSION = 17 From 02a24961ea63c471202e2f80dab91cc560e74c5b Mon Sep 17 00:00:00 2001 From: Joshua Watt Date: Mon, 2 Dec 2019 08:46:22 -0600 Subject: [PATCH 08/48] Remove MY_REGISTRY Removes the MY_REGISTRY option when building images. This only existed to work around an improperly configured internal mirror, and is no longer necessary. --- ci/test.py | 19 ------------------- image/Dockerfile | 23 +++++------------------ pyrex.ini | 4 ---- pyrex.py | 5 ----- 4 files changed, 5 insertions(+), 46 deletions(-) diff --git a/ci/test.py b/ci/test.py index 54d723e..dd57388 100755 --- a/ci/test.py +++ b/ci/test.py @@ -427,25 +427,6 @@ def test_local_build(self): conf.write_conf() self.assertPyrexHostCommand("true") - @skipIfPrebuilt - def test_bad_registry(self): - # Run any command to build the images locally - self.assertPyrexHostCommand("true") - - conf = self.get_config() - - # Trying to build with an invalid registry should fail - conf["config"]["registry"] = "does.not.exist.invalid" - conf["config"]["buildlocal"] = "1" - conf.write_conf() - self.assertPyrexHostCommand("true", returncode=1) - - # Disable building locally any try again (from the previously cached build) - conf["config"]["buildlocal"] = "0" - conf.write_conf() - - self.assertPyrexHostCommand("true") - def test_version(self): self.assertRegex( pyrex.VERSION, diff --git a/image/Dockerfile b/image/Dockerfile index 8046b38..1163e50 100644 --- a/image/Dockerfile +++ b/image/Dockerfile @@ -13,14 +13,12 @@ # limitations under the License. ARG PYREX_BASE=ubuntu-18.04-oe -ARG MY_REGISTRY= # # Base image for prebuilt static binaries # -FROM ${MY_REGISTRY}alpine:3.9 AS prebuilt-base +FROM alpine:3.9 AS prebuilt-base ENV PYREX_BASE none -ENV MY_REGISTRY none RUN apk add --update \ acl-dev \ @@ -56,7 +54,6 @@ COPY patches/0001-Use-pkg-config-to-find-packages.patch /usr/src/ # FROM prebuilt-base AS prebuilt-icecream ENV PYREX_BASE none -ENV MY_REGISTRY none # Use a recent version of Icecream, which has many bug fixes ENV ICECREAM_SHA1=1aa08857cb9c2639dbde1c2f6a05212c842581f1 @@ -87,7 +84,6 @@ RUN mkdir -p /usr/src/icecream && \ # FROM prebuilt-base AS prebuilt-setpriv ENV PYREX_BASE none -ENV MY_REGISTRY none RUN mkdir -p /usr/src/util-linux && \ cd /usr/src/util-linux && \ wget https://mirrors.edge.kernel.org/pub/linux/utils/util-linux/v2.33/util-linux-2.33.1.tar.xz && \ @@ -126,9 +122,8 @@ RUN mkdir -p /usr/src/tini && \ # # Ubuntu 14.04 base # -FROM ${MY_REGISTRY}ubuntu:trusty as ubuntu-14.04-base +FROM ubuntu:trusty as ubuntu-14.04-base ENV PYREX_BASE none -ENV MY_REGISTRY none LABEL maintainer="Joshua Watt " # Install software required to run init scripts. @@ -153,9 +148,8 @@ COPY --from=prebuilt-tini /dist/tini / # # Ubuntu 16.04 Base # -FROM ${MY_REGISTRY}ubuntu:xenial as ubuntu-16.04-base +FROM ubuntu:xenial as ubuntu-16.04-base ENV PYREX_BASE none -ENV MY_REGISTRY none LABEL maintainer="Joshua Watt " # Install software required to run init scripts. @@ -180,9 +174,8 @@ COPY --from=prebuilt-tini /dist/tini / # # Ubuntu 18.04 Base # -FROM ${MY_REGISTRY}ubuntu:bionic as ubuntu-18.04-base +FROM ubuntu:bionic as ubuntu-18.04-base ENV PYREX_BASE none -ENV MY_REGISTRY none LABEL maintainer="Joshua Watt " # Install software required to run init scripts. @@ -207,9 +200,8 @@ COPY --from=prebuilt-tini /dist/tini / # # Centos 7 # -FROM ${MY_REGISTRY}centos:7 as centos-7-base +FROM centos:7 as centos-7-base ENV PYREX_BASE none -ENV MY_REGISTRY none LABEL maintainer="James Harris " RUN set -x && \ #Install default components @@ -235,7 +227,6 @@ COPY --from=prebuilt-tini /dist/tini / # FROM ubuntu-14.04-base as ubuntu-14.04-oe ENV PYREX_BASE none -ENV MY_REGISTRY none LABEL maintainer="Joshua Watt " RUN set -x && export DEBIAN_FRONTEND=noninteractive && \ @@ -314,7 +305,6 @@ COPY --from=prebuilt-icecream /dist/icecream / # FROM ubuntu-16.04-base as ubuntu-16.04-oe ENV PYREX_BASE none -ENV MY_REGISTRY none LABEL maintainer="Joshua Watt " RUN set -x && export DEBIAN_FRONTEND=noninteractive && \ @@ -388,7 +378,6 @@ COPY --from=prebuilt-icecream /dist/icecream / # FROM ubuntu-18.04-base as ubuntu-18.04-oe ENV PYREX_BASE none -ENV MY_REGISTRY none LABEL maintainer="Joshua Watt " RUN set -x && export DEBIAN_FRONTEND=noninteractive && apt-get -y update && apt-get -y install \ @@ -458,7 +447,6 @@ COPY --from=prebuilt-icecream /dist/icecream / # FROM ${PYREX_BASE} as pyrex-base ENV PYREX_BASE none -ENV MY_REGISTRY none # Set Locales ENV LANG en_US.UTF-8 @@ -504,7 +492,6 @@ CMD ["/usr/libexec/pyrex/run", "/bin/bash"] # FROM pyrex-base as pyrex-oe ENV PYREX_BASE none -ENV MY_REGISTRY none # Setup Icecream distributed compiling client. The client tries several IPC # mechanisms to find the daemon, including connecting to a localhost TCP diff --git a/pyrex.ini b/pyrex.ini index ad18cb8..ef49374 100644 --- a/pyrex.ini +++ b/pyrex.ini @@ -38,10 +38,6 @@ confversion = @CONFVERSION@ # repository %buildlocal = 0 -# The Docker registry from which to fetch the image. Leave empty for the -# default -%registry = - # A list of environment variables that should be imported as Pyrex # configuration variables in the "env" section, e.g. ${env:HOME}. Note that # environment variables accessed in this way must be set or an error will diff --git a/pyrex.py b/pyrex.py index 8765954..d752d01 100755 --- a/pyrex.py +++ b/pyrex.py @@ -229,11 +229,6 @@ def build_image(config, build_config): engine_args = shlex.split(config["imagebuild"]["buildcommand"]) - if config["config"]["registry"]: - engine_args.extend( - ["--build-arg", "MY_REGISTRY=%s/" % config["config"]["registry"]] - ) - for e in ("http_proxy", "https_proxy"): if e in os.environ: engine_args.extend(["--build-arg", "%s=%s" % (e, os.environ[e])]) From c141cf2fd3e8cf1d9cdaaccee8b02a37fe957a58 Mon Sep 17 00:00:00 2001 From: Joshua Watt Date: Mon, 2 Dec 2019 10:24:39 -0600 Subject: [PATCH 09/48] ci: Test BB_ENV_EXTRAWHITE Tests that BB_ENV_EXTRAWHITE is correctly passed to the container --- ci/test.py | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/ci/test.py b/ci/test.py index dd57388..888d32d 100755 --- a/ci/test.py +++ b/ci/test.py @@ -76,9 +76,9 @@ def cleanup_env(): os.symlink("/usr/bin/python2", os.path.join(self.bin_dir, "python")) os.environ["PATH"] = self.bin_dir + ":" + os.environ["PATH"] os.environ["PYREX_BUILD_QUIET"] = "0" - if "SSH_AUTH_SOCK" in os.environ: - del os.environ["SSH_AUTH_SOCK"] - self.addCleanup(cleanup_env) + for var in ("SSH_AUTH_SOCK", "BB_ENV_EXTRAWHITE"): + if var in os.environ: + del os.environ[var] self.thread_dir = os.path.join( self.build_dir, "%d.%d" % (os.getpid(), threading.get_ident()) @@ -605,6 +605,21 @@ def test_groups(self): ) self.assertEqual(groups, {"root", grp.getgrgid(os.getgid()).gr_name}) + def test_bb_env_extrawhite(self): + env = os.environ.copy() + env["BB_ENV_EXTRAWHITE"] = "TEST_BB_EXTRA" + env["TEST_BB_EXTRA"] = "Hello" + + s = self.assertPyrexContainerShellCommand( + "echo $BB_ENV_EXTRAWHITE", env=env, quiet_init=True, capture=True + ) + self.assertEqual(s, env["BB_ENV_EXTRAWHITE"]) + + s = self.assertPyrexContainerShellCommand( + "echo $TEST_BB_EXTRA", env=env, quiet_init=True, capture=True + ) + self.assertEqual(s, env["TEST_BB_EXTRA"]) + class PyrexImageType_oe(PyrexImageType_base): """ From 7659c3c717b54ad4dda0668478452f9d316c7f4c Mon Sep 17 00:00:00 2001 From: Joshua Watt Date: Mon, 2 Dec 2019 11:30:14 -0600 Subject: [PATCH 10/48] ci: Test SSH_AUTH_SOCK Tests that the SSH_AUTH_SOCK is correctly handled by the container --- ci/test.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/ci/test.py b/ci/test.py index 888d32d..2bd3b62 100755 --- a/ci/test.py +++ b/ci/test.py @@ -620,6 +620,34 @@ def test_bb_env_extrawhite(self): ) self.assertEqual(s, env["TEST_BB_EXTRA"]) + def test_ssh_auth_sock(self): + with tempfile.NamedTemporaryFile() as auth_file: + env = os.environ.copy() + env["SSH_AUTH_SOCK"] = auth_file.name + auth_file_stat = os.stat(auth_file.name) + + s = self.assertPyrexContainerShellCommand( + "stat --format='%d %i' $SSH_AUTH_SOCK", + env=env, + quiet_init=True, + capture=True, + ) + self.assertEqual( + s, "%d %d" % (auth_file_stat.st_dev, auth_file_stat.st_ino) + ) + + auth_sock_path = os.path.join(self.build_dir, "does-not-exist") + env = os.environ.copy() + env["SSH_AUTH_SOCK"] = auth_sock_path + + s = self.assertPyrexContainerShellCommand("true", env=env, capture=True) + self.assertRegex(s, r"Warning: SSH_AUTH_SOCK \S+ does not exist") + + s = self.assertPyrexContainerShellCommand( + "echo $SSH_AUTH_SOCK", env=env, quiet_init=True, capture=True + ) + self.assertEqual(s, "") + class PyrexImageType_oe(PyrexImageType_base): """ From 3e66caf566d00e2c039bfbef4f8eafeebe04f2ed Mon Sep 17 00:00:00 2001 From: Joshua Watt Date: Mon, 2 Dec 2019 12:48:27 -0600 Subject: [PATCH 11/48] ci: Add rebuild test Adds a test to ensure that the 'pyrex-rebuild' command works --- ci/test.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ci/test.py b/ci/test.py index 2bd3b62..c39bd1f 100755 --- a/ci/test.py +++ b/ci/test.py @@ -648,6 +648,10 @@ def test_ssh_auth_sock(self): ) self.assertEqual(s, "") + @skipIfPrebuilt + def test_rebuild(self): + self.assertPyrexHostCommand("pyrex-rebuild") + class PyrexImageType_oe(PyrexImageType_base): """ From 25b136da03fc4317475864d3c3373e3666141a74 Mon Sep 17 00:00:00 2001 From: Joshua Watt Date: Mon, 2 Dec 2019 12:53:13 -0600 Subject: [PATCH 12/48] ci: Test missing binds Checks that missing binds print a warning, but otherwise still allow the container to run. --- ci/test.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/ci/test.py b/ci/test.py index c39bd1f..5d58caa 100755 --- a/ci/test.py +++ b/ci/test.py @@ -397,6 +397,20 @@ def test_duplicate_binds(self): self.assertPyrexContainerShellCommand("true") + def test_missing_bind(self): + temp_dir = tempfile.mkdtemp("-pyrex") + self.addCleanup(shutil.rmtree, temp_dir) + + missing_bind = os.path.join(temp_dir, "does-not-exist") + conf = self.get_config() + conf["run"]["bind"] += " %s" % missing_bind + conf.write_conf() + + s = self.assertPyrexContainerShellCommand( + "test ! -e %s" % missing_bind, capture=True + ) + self.assertRegex(s, r"Error: bind source path \S+ does not exist") + def test_bad_confversion(self): # Verify that a bad config is an error conf = self.get_config() From ed968518ba12e8beea86c47a896dfea328745ce6 Mon Sep 17 00:00:00 2001 From: Joshua Watt Date: Mon, 2 Dec 2019 13:05:01 -0600 Subject: [PATCH 13/48] ci: Test pyrex-config command Tests that the 'pyrex-config' command works as expected --- ci/test.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/ci/test.py b/ci/test.py index 5d58caa..030f125 100755 --- a/ci/test.py +++ b/ci/test.py @@ -666,6 +666,29 @@ def test_ssh_auth_sock(self): def test_rebuild(self): self.assertPyrexHostCommand("pyrex-rebuild") + def test_pyrex_config(self): + conf = self.get_config() + conf.add_section("ci") + conf["ci"]["foo"] = "ABC" + conf["ci"]["bar"] = "${ci:foo}DEF" + conf["ci"]["baz"] = "${bar}GHI" + conf.write_conf() + + s = self.assertPyrexHostCommand( + "pyrex-config get ci:foo", quiet_init=True, capture=True + ) + self.assertEqual(s, "ABC") + + s = self.assertPyrexHostCommand( + "pyrex-config get ci:bar", quiet_init=True, capture=True + ) + self.assertEqual(s, "ABCDEF") + + s = self.assertPyrexHostCommand( + "pyrex-config get ci:baz", quiet_init=True, capture=True + ) + self.assertEqual(s, "ABCDEFGHI") + class PyrexImageType_oe(PyrexImageType_base): """ From ad893e456dedc539a237d450d1457130dbff974b Mon Sep 17 00:00:00 2001 From: Joshua Watt Date: Mon, 2 Dec 2019 13:05:37 -0600 Subject: [PATCH 14/48] Remove dead code The 'getrawdict()' function in the Config class is no longer used and should be removed. --- pyrex.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/pyrex.py b/pyrex.py index d752d01..2820319 100755 --- a/pyrex.py +++ b/pyrex.py @@ -55,11 +55,6 @@ def __init__(self, *args, **kwargs): # All keys are case-sensitive self.optionxform = lambda option: option - def getrawdict(self): - """returns a dictionary that doesn't have any interpolation. Useful for - merging configs together""" - return {section: values for (section, values) in self.items(raw=True)} - def read_default_config(keep_defaults): with open(os.path.join(PYREX_ROOT, "pyrex.ini"), "r") as f: From 4361c0212a767abcb56a96e95aa6afbd25f5390c Mon Sep 17 00:00:00 2001 From: Joshua Watt Date: Fri, 22 Nov 2019 13:23:52 -0600 Subject: [PATCH 15/48] Add command to create a default config Adds a "mkconfig" subcommand and helper to dump the default configuration to a file or to stdout. --- README.md | 8 ++++++++ ci/test.py | 17 +++++++++++++++++ mkconfig | 18 ++++++++++++++++++ pyrex.py | 37 +++++++++++++++++++++++++++++++++++++ 4 files changed, 80 insertions(+) create mode 100755 mkconfig diff --git a/README.md b/README.md index 5ff3ed1..29a443c 100644 --- a/README.md +++ b/README.md @@ -129,6 +129,14 @@ Pyrex is configured using a ini-style configuration file. The location of this file is specified by the `PYREXCONFFILE` environment variable. This environment variable must be set before the environment is initialized. +If you do not yet have a config file, you can use the `mkconfig` command to use +the default one and assign the `PYREXCONFFILE` variable in a single command +like so: + +```shell +$ export PYREXCONFFILE=`./meta-pyrex/mkconfig ./pyrex.ini` +``` + The configuration file is the ini file format supported by Python's [configparser](https://docs.python.org/3/library/configparser.html) class, with the following notes: diff --git a/ci/test.py b/ci/test.py index 030f125..ecf534d 100755 --- a/ci/test.py +++ b/ci/test.py @@ -689,6 +689,23 @@ def test_pyrex_config(self): ) self.assertEqual(s, "ABCDEFGHI") + def test_pyrex_mkconfig(self): + out_file = os.path.join(self.build_dir, "temp-pyrex.ini") + cmd = [os.path.join(PYREX_ROOT, "pyrex.py"), "mkconfig"] + + output = self.assertSubprocess( + cmd + [out_file], capture=True, cwd=self.build_dir + ) + self.assertEqual(output, out_file) + + output = self.assertSubprocess(cmd, capture=True) + self.assertEqual(output, pyrex.read_default_config(False).rstrip()) + + with open(out_file, "r") as f: + self.assertEqual(f.read().rstrip(), output) + + self.assertSubprocess(cmd + [out_file], cwd=self.build_dir, returncode=1) + class PyrexImageType_oe(PyrexImageType_base): """ diff --git a/mkconfig b/mkconfig new file mode 100755 index 0000000..2595605 --- /dev/null +++ b/mkconfig @@ -0,0 +1,18 @@ +#! /bin/sh +# +# Copyright 2019 Garmin Ltd. or its subsidiaries +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +"$(dirname "$0")/pyrex.py" mkconfig "$@" diff --git a/pyrex.py b/pyrex.py index 2820319..1dba3d2 100755 --- a/pyrex.py +++ b/pyrex.py @@ -30,6 +30,7 @@ import hashlib import json import tempfile +import contextlib VERSION = "0.0.4" @@ -761,6 +762,25 @@ def config_get(args): print(val) return 0 + def mkconfig(args): + @contextlib.contextmanager + def get_output_file(): + if args.output == "-": + yield sys.stdout + else: + with open(args.output, "w" if args.force else "x") as f: + yield f + print(os.path.abspath(args.output)) + + try: + with get_output_file() as f: + f.write(read_default_config(False)) + except FileExistsError: + sys.stderr.write("Refusing to overwrite existing file '%s'\n" % args.output) + return 1 + + return 0 + subparser_args = {} if sys.version_info >= (3, 7, 0): subparser_args["required"] = True @@ -827,6 +847,23 @@ def config_get(args): ) config_get_parser.set_defaults(func=config_get) + mkconfig_parser = subparsers.add_parser( + "mkconfig", help="Create a default Pyrex configuration" + ) + mkconfig_parser.add_argument( + "-f", + "--force", + action="store_true", + help="Overwrite destination file if it already exists", + ) + mkconfig_parser.add_argument( + "output", + default="-", + nargs="?", + help="Output file. Use '-' for standard out. Default is %(default)s", + ) + mkconfig_parser.set_defaults(func=mkconfig) + args = parser.parse_args() func = getattr(args, "func", None) From ea3c980e09217b1eb883f7af15da1e20b92d8029 Mon Sep 17 00:00:00 2001 From: Joshua Watt Date: Tue, 3 Dec 2019 08:17:11 -0600 Subject: [PATCH 16/48] Bump version for release Bumps the version in preparation for a beta release --- pyrex.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyrex.py b/pyrex.py index 1dba3d2..fb1d62e 100755 --- a/pyrex.py +++ b/pyrex.py @@ -32,7 +32,7 @@ import tempfile import contextlib -VERSION = "0.0.4" +VERSION = "1.0.0-beta1" VERSION_REGEX = re.compile(r"^([0-9]+\.){2}[0-9]+(-.*)?$") VERSION_TAG_REGEX = re.compile(r"^v([0-9]+\.){2}[0-9]+(-.*)?$") From 7925054fc3253938c33adfc3d826e9b60b8f1d33 Mon Sep 17 00:00:00 2001 From: Joshua Watt Date: Tue, 3 Dec 2019 16:16:59 -0600 Subject: [PATCH 17/48] Ensure PYREXCONFFILE is an absolute path Ensures that the PYREXCONFFILE variable is an absolute path so that it is still valid when the current directory changes. --- pyrex-init-build-env | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pyrex-init-build-env b/pyrex-init-build-env index e2db5ea..840d9b8 100644 --- a/pyrex-init-build-env +++ b/pyrex-init-build-env @@ -51,6 +51,10 @@ pyrex_cleanup() { export PYREXCONFFILE +if [ -n "$PYREXCONFFILE" ] && [ "${PYREXCONFFILE#/}" == "$PYREXCONFFILE" ]; then + PYREXCONFFILE="$(pwd)/$PYREXCONFFILE" +fi + $PYREX_ROOT/pyrex.py capture 9 \ -a PYREX_OEROOT "$PYREX_OEROOT" \ -a PYREX_OEINIT "$PYREX_OEINIT $@" \ From cb381019948b997bf3527ffb117dacc0c02c9d8a Mon Sep 17 00:00:00 2001 From: Joshua Watt Date: Tue, 3 Dec 2019 16:18:21 -0600 Subject: [PATCH 18/48] Pass oe-init-build-env arguments without quoting Changes the environment initialization script to pass arguments to the oe-init-build-env script using $* instead of $@. This fixes a bug where passing more than one argument would be interpreted as passing the argument to pyrex.py instead. --- pyrex-init-build-env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyrex-init-build-env b/pyrex-init-build-env index 840d9b8..22b44d0 100644 --- a/pyrex-init-build-env +++ b/pyrex-init-build-env @@ -57,7 +57,7 @@ fi $PYREX_ROOT/pyrex.py capture 9 \ -a PYREX_OEROOT "$PYREX_OEROOT" \ - -a PYREX_OEINIT "$PYREX_OEINIT $@" \ + -a PYREX_OEINIT "$PYREX_OEINIT $*" \ --bind $PYREX_OEROOT \ -e TEMPLATECONF \ 9> $PYREX_TEMP_ENV_FILE From f70ab591b0cb0df90874414e8112f192ee8b8ce9 Mon Sep 17 00:00:00 2001 From: Joshua Watt Date: Thu, 5 Dec 2019 14:13:54 -0600 Subject: [PATCH 19/48] pyrex.ini: Fix local Docker image builds The context directory is required when building images locally in Docker --- pyrex.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/pyrex.ini b/pyrex.ini index ef49374..53d4106 100644 --- a/pyrex.ini +++ b/pyrex.ini @@ -56,6 +56,7 @@ confversion = @CONFVERSION@ % -f ${pyrex:pyrexroot}/image/Dockerfile % --target=pyrex-${config:imagetype} % --build-arg PYREX_BASE=${config:image} +% ${pyrex:pyrexroot}/image # Build quietly? %quiet = true From 5f2f998eec2f9e78b04d4c7c6d27cd862277d9ac Mon Sep 17 00:00:00 2001 From: Joshua Watt Date: Thu, 5 Dec 2019 15:25:39 -0600 Subject: [PATCH 20/48] Pass all user groups Pass all the groups the user is a member of, and initialize the user to be a member of them --- ci/test.py | 15 +++++++--- image/entry.py | 76 ++++++++++++++++++++++++++++---------------------- pyrex.py | 15 ++++++---- 3 files changed, 64 insertions(+), 42 deletions(-) diff --git a/ci/test.py b/ci/test.py index ecf534d..49ee9ec 100755 --- a/ci/test.py +++ b/ci/test.py @@ -346,9 +346,8 @@ def test_owner_env(self): # Note: These config variables are intended for testing use only conf["run"]["uid"] = "1337" - conf["run"]["gid"] = "7331" conf["run"]["username"] = "theuser" - conf["run"]["groupname"] = "thegroup" + conf["run"]["groups"] = "7331:thegroup 7332:othergroup" conf["run"]["initcommand"] = "" conf.write_conf() @@ -375,7 +374,8 @@ def read_fifo(): thread.start() try: self.assertPyrexContainerShellCommand( - 'echo "$(id -u):$(id -g):$(id -un):$(id -gn):$USER:$GROUP" > %s' % fifo + 'echo "$(id -u):$(id -g):$(id -un):$(id -gn):$USER:$GROUP:$(id -G):$(id -Gn)" > %s' + % fifo ) finally: thread.join() @@ -386,6 +386,8 @@ def read_fifo(): self.assertEqual(output[3], "thegroup") self.assertEqual(output[4], "theuser") self.assertEqual(output[5], "thegroup") + self.assertEqual(output[6], "7331 7332") + self.assertEqual(output[7], "thegroup othergroup") def test_duplicate_binds(self): temp_dir = tempfile.mkdtemp("-pyrex") @@ -617,7 +619,12 @@ def test_groups(self): "getent group | cut -f1 -d:", quiet_init=True, capture=True ).split() ) - self.assertEqual(groups, {"root", grp.getgrgid(os.getgid()).gr_name}) + + my_groups = {"root", grp.getgrgid(os.getgid()).gr_name} + for gid in os.getgroups(): + my_groups.add(grp.getgrgid(gid).gr_name) + + self.assertEqual(groups, my_groups) def test_bb_env_extrawhite(self): env = os.environ.copy() diff --git a/image/entry.py b/image/entry.py index c348f59..0eadcb5 100644 --- a/image/entry.py +++ b/image/entry.py @@ -54,39 +54,48 @@ def main(): ) uid = int(get_var("PYREX_UID")) - gid = int(get_var("PYREX_GID")) user = get_var("PYREX_USER") - group = get_var("PYREX_GROUP") + groups = [] + for s in get_var("PYREX_GROUPS").split(): + gid, name = s.split(":") + groups.append((int(gid), name)) + + primarygid, primarygroup = groups[0] + home = get_var("PYREX_HOME") - check_file = "/var/run/pyrex-%d-%d" % (uid, gid) + check_file = "/var/run/pyrex-%d-%d" % (uid, primarygid) if not os.path.exists(check_file): with open(check_file, "w") as f: - f.write("%d %d %s %s" % (uid, gid, user, group)) - - # Create user and group - subprocess.check_call( - ["groupadd", "--non-unique", "--gid", "%d" % gid, group], - stdout=subprocess.DEVNULL, - ) - - subprocess.check_call( - [ - "useradd", - "--non-unique", - "--uid", - "%d" % uid, - "--gid", - "%d" % gid, - "--home", - home, - "--no-create-home", - "--shell", - "/bin/sh", - user, - ], - stdout=subprocess.DEVNULL, - ) + f.write("%d %d %s %s\n" % (uid, primarygid, user, primarygroup)) + + # Create user and groups + for (gid, group) in groups: + if gid == 0: + continue + subprocess.check_call( + ["groupadd", "--gid", "%d" % gid, group], stdout=f + ) + + subprocess.check_call( + [ + "useradd", + "--non-unique", + "--uid", + "%d" % uid, + "--gid", + "%d" % primarygid, + "--groups", + ",".join(str(g[0]) for g in groups), + "--home", + home, + "--no-create-home", + "--shell", + "/bin/sh", + user, + ], + stdout=f, + ) try: os.makedirs(home, 0o755) @@ -100,7 +109,7 @@ def main(): home_stat = os.stat(home) if home_stat.st_dev == root_stat.st_dev: - os.chown(home, uid, gid) + os.chown(home, uid, primarygid) try: screenrc = os.path.join(home, ".screenrc") @@ -108,7 +117,7 @@ def main(): with open(screenrc, "x") as f: f.write("defbce on\n") - os.chown(screenrc, uid, gid) + os.chown(screenrc, uid, primarygid) except FileExistsError: pass @@ -118,7 +127,7 @@ def main(): # Setup environment os.environ["USER"] = user - os.environ["GROUP"] = group + os.environ["GROUP"] = primarygroup os.environ["HOME"] = home # If a tty is attached, change it over to be owned by the new user. This is @@ -144,11 +153,12 @@ def main(): "setpriv", "setpriv", "--inh-caps=-all", # Drop all root capabilities - "--clear-groups", "--reuid", "%d" % uid, "--regid", - "%d" % gid, + "%d" % primarygid, + "--groups", + ",".join(str(g[0]) for g in groups), *sys.argv[1:] ) diff --git a/pyrex.py b/pyrex.py index fb1d62e..4458d81 100755 --- a/pyrex.py +++ b/pyrex.py @@ -370,14 +370,21 @@ def prep_container( username = pwd.getpwuid(uid).pw_name groupname = grp.getgrgid(gid).gr_name + groups = ["%d:%s" % (gid, groupname)] + + for group in grp.getgrall(): + if group.gr_name == groupname: + continue + if username in group.gr_mem: + groups.append("%d:%s" % (group.gr_gid, group.gr_name)) + # These are "hidden" keys in pyrex.ini that aren't publicized, and # are primarily used for testing. Use they at your own risk, they # may change if allow_test_config: uid = int(config["run"].get("uid", uid)) - gid = int(config["run"].get("gid", gid)) username = config["run"].get("username") or username - groupname = config["run"].get("groupname") or groupname + groups = config["run"].get("groups", "").split() or groups command_prefix = config["run"].get("commandprefix", "").splitlines() @@ -392,9 +399,7 @@ def prep_container( "-e", "PYREX_UID=%d" % uid, "-e", - "PYREX_GROUP=%s" % groupname, - "-e", - "PYREX_GID=%d" % gid, + "PYREX_GROUPS=%s" % " ".join(groups), "-e", "PYREX_HOME=%s" % os.environ["HOME"], "-e", From d7ce226bee76d2767a3f6cb2ac24f004973bc7a6 Mon Sep 17 00:00:00 2001 From: Joshua Watt Date: Fri, 6 Dec 2019 10:38:53 -0600 Subject: [PATCH 21/48] Update README Updates the README to correct some of the instructions --- README.md | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 29a443c..84bf6d7 100644 --- a/README.md +++ b/README.md @@ -110,11 +110,6 @@ PYREX_OEINIT="$(pwd)/oe-init-build-env" # correct) PYREX_ROOT="$(pwd)/meta-pyrex" -# The location of the pyrex.ini template file to use if the user doesn't -# already have one. Defaults to "$TEMPLATECONF/pyrex.ini.sample" (the same -# location that oe-init-build-env will look for local.conf.sample & friends) -PYREXCONFTEMPLATE="$(pwd)/pyrex.ini.sample" - # Alternatively, if it is desired to always use a fixed config file that users # can't change, set the following: #PYREXCONFFILE="$(pwd)/pyrex.ini" @@ -157,10 +152,10 @@ variable called `run:bind` is specified in the config file. Any directory that appears in this variable will be bound into the container image at the same path (e.g. `/foo/bar` in the host will be bound to `/foo/bar` in the container engine. By default, only the Openembedded root directory (a.k.a. -`$PYREX_OEROOT`, `${build:oeroot}`) is bound. This is the minimum that can be -bound, and is generally sufficient for most use cases. If additional -directories need to be accessed by the container image, they can be added to -this list by the user. Common reasons for adding new paths include: +`$PYREX_OEROOT`) is bound. This is the minimum that can be bound, and is +generally sufficient for most use cases. If additional directories need to be +accessed by the container image, they can be added to this list by the user. +Common reasons for adding new paths include: * Alternate (out of tree) locations for sstate and download caches * Alternate (out of tree) build directories * Additional layers that are not under the OEROOT directory @@ -174,10 +169,10 @@ paths might make it difficult to do builds outside of Pyrex if necessary. You should **never** map directories like `/usr/bin`, `/etc/`, `/` as these will probably just break the container. It is probably also unwise to map your entire home directory; although in some cases may be necessary to map -$HOME/.ssh or other directories to access SSH keys and the like. For user +`$HOME/.ssh` or other directories to access SSH keys and the like. For user convenience, the proxy user created in the container image by default has the -same $HOME as the user who created the container, so these types of bind can be -done by simply adding `${env:HOME}/.ssh` to `run:bind` +same `$HOME` as the user who created the container, so these types of bind can +be done by simply adding `${env:HOME}/.ssh` to `run:bind` #### Debugging the container In the event that you need to get a shell into the container to run some @@ -228,6 +223,10 @@ The following items are either known to not work, or haven't been fully tested: from causing bad behaviors. It is still possible to pause the container using the `docker pause` command, but this doesn't integrate with the parent shells job control. +* **Rootless Docker** This in untested, and probably doesn't work. It shouldn't + be too hard since this should be very similar to how `podman` works. + Currently however, it is assumed that if the container engine is `docker` it + is running as root and if it is `podman` it is running rootless. ## Developing on Pyrex If you are doing development on Pyrex itself, please read the [Developer From a0bf45f4d90eb0520390426aea82fadfedd6a506 Mon Sep 17 00:00:00 2001 From: Joshua Watt Date: Fri, 6 Dec 2019 14:19:50 -0600 Subject: [PATCH 22/48] Bump version for release --- pyrex.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyrex.py b/pyrex.py index 4458d81..d6f86d2 100755 --- a/pyrex.py +++ b/pyrex.py @@ -32,7 +32,7 @@ import tempfile import contextlib -VERSION = "1.0.0-beta1" +VERSION = "1.0.0-beta2" VERSION_REGEX = re.compile(r"^([0-9]+\.){2}[0-9]+(-.*)?$") VERSION_TAG_REGEX = re.compile(r"^v([0-9]+\.){2}[0-9]+(-.*)?$") From 5225c7b30bbd58b2ca366afc49e0feb5c351302e Mon Sep 17 00:00:00 2001 From: Joshua Watt Date: Mon, 9 Dec 2019 10:35:19 -0600 Subject: [PATCH 23/48] Add option for the user to override commands Adds (back) an option to the allow the user to override the commands that are run under pyrex, and a test for them. --- ci/test.py | 20 ++++++++++++++++++++ pyrex.ini | 9 +++++++++ pyrex.py | 10 ++++++++-- 3 files changed, 37 insertions(+), 2 deletions(-) diff --git a/ci/test.py b/ci/test.py index 49ee9ec..989ae9b 100755 --- a/ci/test.py +++ b/ci/test.py @@ -713,6 +713,26 @@ def test_pyrex_mkconfig(self): self.assertSubprocess(cmd + [out_file], cwd=self.build_dir, returncode=1) + def test_user_commands(self): + conf = self.get_config() + conf["config"]["commands"] = "/bin/true !/bin/false" + conf.write_conf() + + self.assertPyrexHostCommand("/bin/true") + self.assertPyrexHostCommand("/bin/false", returncode=1) + + true_path = self.assertPyrexHostCommand( + "which true", capture=True, quiet_init=True + ) + true_link_path = os.readlink(true_path) + self.assertEqual(os.path.basename(true_link_path), "exec-shim-pyrex") + + false_path = self.assertPyrexHostCommand( + "which false", capture=True, quiet_init=True + ) + false_link_path = os.readlink(false_path) + self.assertEqual(os.path.basename(false_link_path), "false") + class PyrexImageType_oe(PyrexImageType_base): """ diff --git a/pyrex.ini b/pyrex.ini index 53d4106..785f022 100644 --- a/pyrex.ini +++ b/pyrex.ini @@ -12,6 +12,15 @@ # to be specified in the user config file confversion = @CONFVERSION@ +# A list of globs for commands that should be wrapped by Pyrex. Overrides the +# command set specified by the container itself when it was captured. Any path +# starting with a "!" will be excluded from being wrapped by Pyrex and will +# run directly in the host environment +#commands = +# ${env:PYREX_OEROOT}/bitbake/bin/* +# ${env:PYREX_OEROOT}/scripts/* +# !${env:PYREX_OEROOT}/scripts/runqemu* + # The Container engine executable (e.g. docker, podman) to use. If the path # does not start with a "/", the $PATH environment variable will be searched # (i.e. execvp rules) diff --git a/pyrex.py b/pyrex.py index d6f86d2..cfc6733 100755 --- a/pyrex.py +++ b/pyrex.py @@ -613,8 +613,14 @@ def create_shims(config, build_config, buildconf): os.chmod(bypassfile, stat.S_IRWXU) # Create shims - command_globs = build_config["container"].get("commands", {}).get("include", {}) - nopyrex_globs = build_config["container"].get("commands", {}).get("exclude", {}) + user_commands = config["config"].get("commands") + if user_commands: + user_commands = user_commands.split() + command_globs = [c for c in user_commands if not c.startswith("!")] + nopyrex_globs = [c[1:] for c in user_commands if c.startswith("!")] + else: + command_globs = build_config["container"].get("commands", {}).get("include", {}) + nopyrex_globs = build_config["container"].get("commands", {}).get("exclude", {}) commands = set() From 0ac3e346b848f03872b96928de8b1b84d843810a Mon Sep 17 00:00:00 2001 From: Joshua Watt Date: Mon, 9 Dec 2019 10:43:55 -0600 Subject: [PATCH 24/48] image: Use BITBAKEDIR and OEROOT for commands Use the BITBAKEDIR and OEROOT variables to determine where the commands pyrex should wrap are located. This allows pyrex to correctly wrap commands bitbake when the directory layout is other than the one used by poky. --- image/capture.sh | 32 ++++++++++++++++++++++++++++---- 1 file changed, 28 insertions(+), 4 deletions(-) diff --git a/image/capture.sh b/image/capture.sh index 8ef470c..7d6b697 100755 --- a/image/capture.sh +++ b/image/capture.sh @@ -16,6 +16,19 @@ INIT_PWD=$PWD +# Prevent some variables from being unset so their value can be captured +unset() { + for var in "$@"; do + case "$var" in + BITBAKEDIR) ;; + OEROOT) ;; + *) + builtin unset "$var" + ;; + esac + done +} + # Consume all arguments before sourcing the environment script shift $# @@ -24,6 +37,16 @@ if [ $? -ne 0 ]; then exit 1 fi +if [ -z "$BITBAKEDIR" ]; then + echo "\$BITBAKEDIR not captured!" + exit 1 +fi + +if [ -z "$OEROOT" ]; then + echo "\$OEROOT not captured!" + exit 1 +fi + cat > $PYREX_CAPTURE_DEST < $PYREX_CAPTURE_DEST < $PYREX_CAPTURE_DEST < Date: Mon, 9 Dec 2019 12:18:37 -0600 Subject: [PATCH 25/48] ci: Test alternate bitbake location Adds a test to verify that Pyrex works when bitbake is not a subdirectory of $PYREX_OEROOT. --- ci/test.py | 42 +++++++++++++++++++++++++++++++++++------- 1 file changed, 35 insertions(+), 7 deletions(-) diff --git a/ci/test.py b/ci/test.py index 989ae9b..5c15c77 100755 --- a/ci/test.py +++ b/ci/test.py @@ -166,7 +166,7 @@ def assertSubprocess(self, *args, capture=False, returncode=0, **kwargs): return None def _write_host_command( - self, args, quiet_init=False, cwd=PYREX_ROOT, builddir=None + self, args, *, quiet_init=False, cwd=PYREX_ROOT, builddir=None, bitbakedir="" ): if builddir is None: builddir = self.build_dir @@ -175,8 +175,13 @@ def _write_host_command( with open(cmd_file, "w") as f: f.write("PYREXCONFFILE=%s\n" % self.pyrex_conf) f.write( - ". %s/poky/pyrex-init-build-env%s %s && (" - % (PYREX_ROOT, " > /dev/null 2>&1" if quiet_init else "", builddir) + ". %s/poky/pyrex-init-build-env%s %s %s && (" + % ( + PYREX_ROOT, + " > /dev/null 2>&1" if quiet_init else "", + builddir, + bitbakedir, + ) ) f.write(" && ".join(list(args))) f.write(")") @@ -189,10 +194,20 @@ def _write_container_command(self, args): return cmd_file def assertPyrexHostCommand( - self, *args, quiet_init=False, cwd=PYREX_ROOT, builddir=None, **kwargs + self, + *args, + quiet_init=False, + cwd=PYREX_ROOT, + builddir=None, + bitbakedir="", + **kwargs ): cmd_file = self._write_host_command( - args, quiet_init, cwd=cwd, builddir=builddir + args, + quiet_init=quiet_init, + cwd=cwd, + builddir=builddir, + bitbakedir=bitbakedir, ) return self.assertSubprocess(["/bin/bash", cmd_file], cwd=cwd, **kwargs) @@ -204,11 +219,13 @@ def assertPyrexContainerCommand(self, cmd, **kwargs): return self.assertPyrexHostCommand("pyrex-run %s" % cmd, **kwargs) def assertPyrexContainerShellPTY( - self, *args, returncode=0, env=None, quiet_init=False + self, *args, returncode=0, env=None, quiet_init=False, bitbakedir="" ): container_cmd_file = self._write_container_command(args) host_cmd_file = self._write_host_command( - ["pyrex-shell %s" % container_cmd_file], quiet_init + ["pyrex-shell %s" % container_cmd_file], + quiet_init=quiet_init, + bitbakedir=bitbakedir, ) stdout = [] @@ -742,6 +759,17 @@ class PyrexImageType_oe(PyrexImageType_base): def test_bitbake_parse(self): self.assertPyrexHostCommand("bitbake -p") + def test_bitbake_parse_altpath(self): + bitbakedir = os.path.join(self.build_dir, "bitbake") + shutil.copytree(os.path.join(PYREX_ROOT, "poky/bitbake"), bitbakedir) + + d = self.assertPyrexContainerCommand( + "which bitbake", bitbakedir=bitbakedir, quiet_init=True, capture=True + ) + self.assertEqual(d, os.path.join(bitbakedir, "bin", "bitbake")) + + self.assertPyrexHostCommand("bitbake -p", bitbakedir=bitbakedir) + def test_icecc(self): self.assertPyrexContainerCommand("icecc --version") From 69eb6e51b93a3a7227c5a91139737bc241327975 Mon Sep 17 00:00:00 2001 From: Joshua Watt Date: Mon, 9 Dec 2019 13:40:25 -0600 Subject: [PATCH 26/48] Bump version for release Bumps version for the 3rd beta release --- pyrex.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyrex.py b/pyrex.py index cfc6733..53faa30 100755 --- a/pyrex.py +++ b/pyrex.py @@ -32,7 +32,7 @@ import tempfile import contextlib -VERSION = "1.0.0-beta2" +VERSION = "1.0.0-beta3" VERSION_REGEX = re.compile(r"^([0-9]+\.){2}[0-9]+(-.*)?$") VERSION_TAG_REGEX = re.compile(r"^v([0-9]+\.){2}[0-9]+(-.*)?$") From a141139d7f8d4afc5cff1579b5df9088d58eb760 Mon Sep 17 00:00:00 2001 From: Joshua Watt Date: Fri, 27 Dec 2019 11:03:01 -0600 Subject: [PATCH 27/48] ci: Add pretty command logging Adds support for show a "pretty" command string when reporting errors in subprocesses. This allows the pyrex container commands to show the actual command that failed instead of the shell wrapper --- ci/test.py | 46 ++++++++++++++++++++++++++++------------------ 1 file changed, 28 insertions(+), 18 deletions(-) diff --git a/ci/test.py b/ci/test.py index 5c15c77..e8c547f 100755 --- a/ci/test.py +++ b/ci/test.py @@ -130,7 +130,9 @@ def write_config_helper(conf): return config - def assertSubprocess(self, *args, capture=False, returncode=0, **kwargs): + def assertSubprocess( + self, *args, pretty_command=None, capture=False, returncode=0, **kwargs + ): if capture: try: output = subprocess.check_output( @@ -145,7 +147,8 @@ def assertSubprocess(self, *args, capture=False, returncode=0, **kwargs): self.assertEqual( ret, returncode, - msg="%s: %s" % (" ".join(*args), output.decode("utf-8")), + msg="%s: %s" + % (pretty_command or " ".join(*args), output.decode("utf-8")), ) return output.decode("utf-8").rstrip() else: @@ -171,21 +174,26 @@ def _write_host_command( if builddir is None: builddir = self.build_dir + command = [ + "PYREXCONFFILE=%s\n" % self.pyrex_conf, + ". %s/poky/pyrex-init-build-env%s %s %s && " + % ( + PYREX_ROOT, + " > /dev/null 2>&1" if quiet_init else "", + builddir, + bitbakedir, + ), + "(", + " && ".join(list(args)), + ")", + ] + + command = "".join(command) + cmd_file = os.path.join(self.thread_dir, "command") with open(cmd_file, "w") as f: - f.write("PYREXCONFFILE=%s\n" % self.pyrex_conf) - f.write( - ". %s/poky/pyrex-init-build-env%s %s %s && (" - % ( - PYREX_ROOT, - " > /dev/null 2>&1" if quiet_init else "", - builddir, - bitbakedir, - ) - ) - f.write(" && ".join(list(args))) - f.write(")") - return cmd_file + f.write(command) + return cmd_file, command def _write_container_command(self, args): cmd_file = os.path.join(self.thread_dir, "container_command") @@ -202,14 +210,16 @@ def assertPyrexHostCommand( bitbakedir="", **kwargs ): - cmd_file = self._write_host_command( + cmd_file, command = self._write_host_command( args, quiet_init=quiet_init, cwd=cwd, builddir=builddir, bitbakedir=bitbakedir, ) - return self.assertSubprocess(["/bin/bash", cmd_file], cwd=cwd, **kwargs) + return self.assertSubprocess( + ["/bin/bash", cmd_file], pretty_command=command, cwd=cwd, **kwargs + ) def assertPyrexContainerShellCommand(self, *args, **kwargs): cmd_file = self._write_container_command(args) @@ -222,7 +232,7 @@ def assertPyrexContainerShellPTY( self, *args, returncode=0, env=None, quiet_init=False, bitbakedir="" ): container_cmd_file = self._write_container_command(args) - host_cmd_file = self._write_host_command( + host_cmd_file, _ = self._write_host_command( ["pyrex-shell %s" % container_cmd_file], quiet_init=quiet_init, bitbakedir=bitbakedir, From c12fa75b835141482ac0e134413715947e2a189a Mon Sep 17 00:00:00 2001 From: Joshua Watt Date: Fri, 27 Dec 2019 12:48:20 -0600 Subject: [PATCH 28/48] Dockerfile: Set file descriptor ulimit Sets the ulimit to 1024 when using installing packages. Several install steps (e.g. yum on CentOS7 and pycompile) iterate over all possible file descriptors to set the CLOEXEC flag on them, which significantly slows down the install speed when the maximum is a large value (e.g. when running in Docker). See: https://bugzilla.redhat.com/show_bug.cgi?id=1537564 --- image/Dockerfile | 39 +++++++++++++++++++++++++-------------- 1 file changed, 25 insertions(+), 14 deletions(-) diff --git a/image/Dockerfile b/image/Dockerfile index 1163e50..6ea6a4f 100644 --- a/image/Dockerfile +++ b/image/Dockerfile @@ -127,7 +127,9 @@ ENV PYREX_BASE none LABEL maintainer="Joshua Watt " # Install software required to run init scripts. -RUN set -x && export DEBIAN_FRONTEND=noninteractive && apt-get -y update && apt-get -y install \ +RUN set -x && export DEBIAN_FRONTEND=noninteractive && \ + ulimit -n 1024 && \ + apt-get -y update && apt-get -y install \ locales \ lsb-release \ ncurses-term \ @@ -153,7 +155,9 @@ ENV PYREX_BASE none LABEL maintainer="Joshua Watt " # Install software required to run init scripts. -RUN set -x && export DEBIAN_FRONTEND=noninteractive && apt-get -y update && apt-get -y install \ +RUN set -x && export DEBIAN_FRONTEND=noninteractive && \ + ulimit -n 1024 && \ + apt-get -y update && apt-get -y install \ locales \ lsb-release \ ncurses-term \ @@ -179,7 +183,9 @@ ENV PYREX_BASE none LABEL maintainer="Joshua Watt " # Install software required to run init scripts. -RUN set -x && export DEBIAN_FRONTEND=noninteractive && apt-get -y update && apt-get -y install \ +RUN set -x && export DEBIAN_FRONTEND=noninteractive && \ + ulimit -n 1024 && \ + apt-get -y update && apt-get -y install \ locales \ lsb-release \ ncurses-term \ @@ -204,16 +210,17 @@ FROM centos:7 as centos-7-base ENV PYREX_BASE none LABEL maintainer="James Harris " RUN set -x && \ - #Install default components - yum install -y yum-utils \ - which \ - sudo \ - redhat-lsb-core \ - &&\ - yum install -y https://centos7.iuscommunity.org/ius-release.rpm &&\ - yum install -y python36u &&\ - yum clean all &&\ - localedef -c -f UTF-8 -i en_US en_US.UTF-8 + ulimit -n 1024 && \ + # Install default components + yum install -y yum-utils \ + which \ + sudo \ + redhat-lsb-core \ + &&\ + yum install -y https://centos7.iuscommunity.org/ius-release.rpm &&\ + yum install -y python36u &&\ + yum clean all &&\ + localedef -c -f UTF-8 -i en_US en_US.UTF-8 # Reset the default useradd options for when the primary user is added RUN echo "SHELL=/bin/bash" > /etc/default/useradd @@ -230,6 +237,7 @@ ENV PYREX_BASE none LABEL maintainer="Joshua Watt " RUN set -x && export DEBIAN_FRONTEND=noninteractive && \ + ulimit -n 1024 && \ # Add a non-ancient version of git apt-get -y update && apt-get -y install software-properties-common && \ add-apt-repository -y ppa:git-core/ppa && \ @@ -308,6 +316,7 @@ ENV PYREX_BASE none LABEL maintainer="Joshua Watt " RUN set -x && export DEBIAN_FRONTEND=noninteractive && \ + ulimit -n 1024 && \ # Add a non-ancient version of git apt-get -y update && apt-get -y install software-properties-common && \ add-apt-repository -y ppa:git-core/ppa && \ @@ -380,7 +389,9 @@ FROM ubuntu-18.04-base as ubuntu-18.04-oe ENV PYREX_BASE none LABEL maintainer="Joshua Watt " -RUN set -x && export DEBIAN_FRONTEND=noninteractive && apt-get -y update && apt-get -y install \ +RUN set -x && export DEBIAN_FRONTEND=noninteractive && \ + ulimit -n 1024 && \ + apt-get -y update && apt-get -y install \ # Poky 2.7 build dependencies gawk \ wget \ From 383cfb010ae3048ad6f80622fd702adbd3e850a0 Mon Sep 17 00:00:00 2001 From: Joshua Watt Date: Fri, 27 Dec 2019 13:09:51 -0600 Subject: [PATCH 29/48] Rework bind behavior Reworks how directories are bound into the container to simplify and clarify how it works. Namely: 1) The PYREX_OEROOT variable has been removed. The name of this variable was misleading, since Pyrex doesn't actually care what the OpenEmbedded root directory is anymore. 2) The PYREX_CONFIG_BIND variable was added. This variable can be used by init scripts to specify paths that should be bound into the container. It defaults to the same logic that PYREX_OEROOT used to use, since that is most likely to be correct if the examples are followed. 3) Containers are no longer allowed to specify additional directories they would like to have bound at capture time. This behavior isn't as useful as it first seems, and it was too tricky. It's better to have a more static configuration that can be easily reasoned about by end users. 4) The capture script now verifies that various important files and directories (namely, the bitbake directory and the build directory) were bound into the container and fails if not. This should help users figure out what they need to do sooner if they have incorrectly binds. 5) Testing for detecting invalid binds was improved --- README.md | 56 +++++++++++++++++++++++++++++--------------- ci/test.py | 52 ++++++++++++++++++++++++++++++++++------ image/capture.sh | 44 ++++++++++++++++++++++++++++------ pyrex-init-build-env | 18 +++++++------- pyrex.ini | 3 ++- pyrex.py | 12 +++++----- 6 files changed, 137 insertions(+), 48 deletions(-) diff --git a/README.md b/README.md index 84bf6d7..7ba9e83 100644 --- a/README.md +++ b/README.md @@ -97,12 +97,15 @@ non-standard layout, you can write your own environment init script that tells look like: ```shell -# The top level Yocto/OE directory (usually, poky). This variable *must* be -# specified if writing a custom script. -PYREX_OEROOT="$(pwd)" +# Paths that should be bound into the container. If unspecified, defaults to +# the parent directory of the sourced pyrex-init-build-env script, before +# it is resolved as a symbolic link. You may need to override the default if +# your bitbake directory, build directory, or any of your layer directories are +# not children of the default (and thus, wouldn't be bound into the container). +PYREX_CONFIG_BIND="$(pwd)" # The path to the build init script. If unspecified, defaults to -# "${PYREX_OEROOT}/oe-init-build-env" +# "$(pwd)/oe-init-build-env" PYREX_OEINIT="$(pwd)/oe-init-build-env" # The location of Pyrex itself. If not specified, pyrex-init-build-env will @@ -119,6 +122,11 @@ PYREX_ROOT="$(pwd)/meta-pyrex" . $(pwd)/meta-pyrex/pyrex-init-build-env "$@" ``` +*NOTE: While it might be tempting to combine all of these into a one-liner like +`PYREXCONFFILE="..." . $(pwd)/meta-pyrex/pyrex-init-build-env "$@"`, they must +be specified on separate lines to remain compatible will all shells (i.e. bash +in particular won't keep temporary variables specified in this way)* + ### Configuration Pyrex is configured using a ini-style configuration file. The location of this file is specified by the `PYREXCONFFILE` environment variable. This environment @@ -147,24 +155,34 @@ For more information about specific configuration values, see the default #### Binding directories into the container In order for bitbake running in the container to be able to build, it must have -access to the data and config files from the host system. To make this easy, a -variable called `run:bind` is specified in the config file. Any directory that -appears in this variable will be bound into the container image at the same -path (e.g. `/foo/bar` in the host will be bound to `/foo/bar` in the container -engine. By default, only the Openembedded root directory (a.k.a. -`$PYREX_OEROOT`) is bound. This is the minimum that can be bound, and is -generally sufficient for most use cases. If additional directories need to be -accessed by the container image, they can be added to this list by the user. -Common reasons for adding new paths include: +access to the data and config files from the host system. There are two +variables that can be set to specify what is bound into the container, the +`PYREX_CONFIG_BIND` environment variable and the `run:bind` option specified in +the config file. Both variables are a whitespace separated list of host paths +that should be bound into the container at the same path (e.g. `/foo/bar` in +the host will be bound to `/foo/bar` in the container engine). + +The `PYREX_CONFIG_BIND` environment variable is intended to specify the minimal +set of bound directories required to initialize a default environment, and +should only be set the by the environment initialization script, not by end +users. The default value for this variable if unspecified is the parent of the +sourced Pyrex initialization script. If the sourced script happens to be a +symbolic link, the parent directory is determined before the symbolic link is +resolved. + +The `run:bind` config file option is intended to allow users to specify +additional paths that they want to bind. For convenience, the default value of +this variable allows users to specify binds in the `PYREX_BIND` environment +variable if they wish. + +Common reasons users might need to bind new paths include: * Alternate (out of tree) locations for sstate and download caches * Alternate (out of tree) build directories -* Additional layers that are not under the OEROOT directory +* Additional layers that are not under the default bind directories -It is recommended to use this variable and bind directories in a 1-to-1 fashion -rather than try to remap them to different paths inside the container image. -Bitbake tends to encode file paths into some of its internal state (*Note* -**Not** sstate, which should always be position independent), and remapping the -paths might make it difficult to do builds outside of Pyrex if necessary. +When the container environment is setup some basic sanity checks will be +performed to makes sure that important directories like the bitbake and build +directories are bound into the container. You should **never** map directories like `/usr/bin`, `/etc/`, `/` as these will probably just break the container. It is probably also unwise to map your diff --git a/ci/test.py b/ci/test.py index e8c547f..05c5253 100755 --- a/ci/test.py +++ b/ci/test.py @@ -76,6 +76,10 @@ def cleanup_env(): os.symlink("/usr/bin/python2", os.path.join(self.bin_dir, "python")) os.environ["PATH"] = self.bin_dir + ":" + os.environ["PATH"] os.environ["PYREX_BUILD_QUIET"] = "0" + os.environ["PYREX_OEINIT"] = os.path.join( + PYREX_ROOT, "poky", "oe-init-build-env" + ) + os.environ["PYREX_CONFIG_BIND"] = PYREX_ROOT for var in ("SSH_AUTH_SOCK", "BB_ENV_EXTRAWHITE"): if var in os.environ: del os.environ[var] @@ -770,15 +774,49 @@ def test_bitbake_parse(self): self.assertPyrexHostCommand("bitbake -p") def test_bitbake_parse_altpath(self): - bitbakedir = os.path.join(self.build_dir, "bitbake") - shutil.copytree(os.path.join(PYREX_ROOT, "poky/bitbake"), bitbakedir) + # The new bitbake directory is out of the normally bound tree + with tempfile.TemporaryDirectory() as tmpdir: + bitbakedir = os.path.join(tmpdir, "bitbake") + shutil.copytree(os.path.join(PYREX_ROOT, "poky/bitbake"), bitbakedir) + + # If the bitbake directory is not bound, capture should fail with + # an error + d = self.assertPyrexHostCommand( + "bitbake -p", bitbakedir=bitbakedir, returncode=1, capture=True + ) + self.assertIn("ERROR: %s not bound in container" % bitbakedir, d) - d = self.assertPyrexContainerCommand( - "which bitbake", bitbakedir=bitbakedir, quiet_init=True, capture=True - ) - self.assertEqual(d, os.path.join(bitbakedir, "bin", "bitbake")) + # Binding the build directory in the conf file will allow bitbake + # to be found + conf = self.get_config() + conf["run"]["bind"] = bitbakedir + conf.write_conf() + + d = self.assertPyrexContainerCommand( + "which bitbake", bitbakedir=bitbakedir, quiet_init=True, capture=True + ) + self.assertEqual(d, os.path.join(bitbakedir, "bin", "bitbake")) + + self.assertPyrexHostCommand("bitbake -p", bitbakedir=bitbakedir) + + def test_unbound_builddir(self): + with tempfile.TemporaryDirectory() as tmpdir: + builddir = os.path.join(tmpdir) + + # If the build directory is not bound, capture should fail with an + # error + d = self.assertPyrexHostCommand( + "true", builddir=builddir, returncode=1, capture=True + ) + self.assertIn("ERROR: %s not bound in container" % builddir, d) + + # Binding the build directory in the conf file will allow building + # to continue + conf = self.get_config() + conf["run"]["bind"] = builddir + conf.write_conf() - self.assertPyrexHostCommand("bitbake -p", bitbakedir=bitbakedir) + self.assertPyrexHostCommand("true", builddir=builddir) def test_icecc(self): self.assertPyrexContainerCommand("icecc --version") diff --git a/image/capture.sh b/image/capture.sh index 7d6b697..c9e33fd 100755 --- a/image/capture.sh +++ b/image/capture.sh @@ -29,10 +29,39 @@ unset() { done } +check_bound() { + if [ ! -e "$1" ]; then + echo "ERROR: $1 not bound in container (File doesn't exist). " + echo "Please set either \$PYREX_CONFIG_BIND or 'run:bind' in" \ + "$PYREXCONFFILE to ensure it is bound into the container." + exit 1 + fi + + local FILE_DEV="$(stat --format "%D" "$1" )" + local ROOT_DEV="$(stat --format "%D" "/")" + + if [ "$(findmnt -f -n -o TARGET --target "$1")" == "/" ]; then + echo "ERROR: $1 not bound in container (File mount target is root)" + echo "Please set either \$PYREX_CONFIG_BIND or 'run:bind' in" \ + "$PYREXCONFFILE to ensure it is bound into the container." + exit 1 + fi +} + # Consume all arguments before sourcing the environment script +declare -a PYREX_ARGS=("$@") shift $# -. $PYREX_OEINIT +# Ensure the init script is bound in the container +check_bound $PYREX_OEINIT + +# If the bitbake directory argument is provided, ensure it is bound in the +# container +if [ -n "${PYREX_ARGS[1]}" ]; then + check_bound "${PYREX_ARGS[1]}" +fi + +. $PYREX_OEINIT "${PYREX_ARGS[@]}" if [ $? -ne 0 ]; then exit 1 fi @@ -47,6 +76,11 @@ if [ -z "$OEROOT" ]; then exit 1 fi +# Ensure the build directory is bound into the container. +check_bound "$(pwd)" + +check_bound $PYREX_CAPTURE_DEST + cat > $PYREX_CAPTURE_DEST < $PYREX_CAPTURE_DEST < $PYREX_TEMP_ENV_FILE if [ $? -ne 0 ]; then pyrex_cleanup diff --git a/pyrex.ini b/pyrex.ini index 785f022..c5c5103 100644 --- a/pyrex.ini +++ b/pyrex.ini @@ -83,6 +83,7 @@ confversion = @CONFVERSION@ # A list of directories that should be bound when running in the container %bind = +% ${env:PYREX_BIND} # A list of environment variables that should be propagated to the container # if set in the parent environment @@ -101,4 +102,4 @@ confversion = @CONFVERSION@ # Assign default values for environment variables in this section [env] - +%PYREX_BIND= diff --git a/pyrex.py b/pyrex.py index 53faa30..a3c5ef7 100755 --- a/pyrex.py +++ b/pyrex.py @@ -464,7 +464,11 @@ def prep_container( engine_args.extend(["-t", "-e", "TERM=%s" % os.environ["TERM"]]) # Configure binds - binds = config["run"]["bind"].split() + extra_bind + binds = ( + config["run"]["bind"].split() + + os.environ.get("PYREX_CONFIG_BIND", "").split() + + extra_bind + ) for b in set(binds): if not os.path.exists(b): print("Error: bind source path {b} does not exist".format(b=b)) @@ -666,7 +670,7 @@ def capture(args): ["/usr/libexec/pyrex/capture"] + args.init, extra_env=env_args, preserve_env=args.env, - extra_bind=[f.name] + args.bind, + extra_bind=[f.name], ) if not engine_args: @@ -731,7 +735,6 @@ def run(args): config, build_config, ["/usr/libexec/pyrex/run"] + args.command, - extra_bind=build_config.get("run", {}).get("bind", []), extra_env=build_config.get("run", {}).get("env", {}), allow_test_config=True, ) @@ -824,9 +827,6 @@ def get_output_file(): default=[], help="Pass additional environment variables if present in parent shell", ) - capture_parser.add_argument( - "--bind", action="append", default=[], help="Additional binds when capturing" - ) capture_parser.add_argument( "init", nargs="*", help="Initialization arguments", default=[] ) From b4dc497b9f232dc80757e75c95547ffb57cb7309 Mon Sep 17 00:00:00 2001 From: Joshua Watt Date: Mon, 30 Dec 2019 11:21:38 -0600 Subject: [PATCH 30/48] Dockerfile: Build findmnt for Ubuntu 14.04 Builds newer util-linux for Ubuntu 14.04. This is primarily for the purpose of providing a new enough findmnt command that correctly supports the '--target' option required by Pyrex. util-linux doesn't allow building only a static version of findmnt, so the entire util-linux suite is built including setpriv, which requires libcap-ng to also be built. --- image/Dockerfile | 45 ++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 44 insertions(+), 1 deletion(-) diff --git a/image/Dockerfile b/image/Dockerfile index 6ea6a4f..b4e9610 100644 --- a/image/Dockerfile +++ b/image/Dockerfile @@ -102,6 +102,46 @@ RUN mkdir -p /usr/src/util-linux && \ make -j$(nproc) LDFLAGS="--static" && \ make install-strip DESTDIR=/dist/setpriv +# +# Prebuilt util-linux and libcap-ng for Ubuntu 14.04 +# +FROM ubuntu:trusty AS prebuilt-util-linux-14.04 +ENV PYREX_BASE none +RUN set -x && export DEBIAN_FRONTEND=noninteractive && \ + ulimit -n 1024 && \ + apt-get -y update && apt-get -y install \ + build-essential \ + wget + +RUN set -x && mkdir -p /usr/src/libcap-ng && \ + cd /usr/src/libcap-ng && \ + wget http://people.redhat.com/sgrubb/libcap-ng/libcap-ng-0.7.10.tar.gz && \ + tar -xvf libcap-ng-0.7.10.tar.gz && \ + cd libcap-ng-0.7.10 && \ + mkdir build && \ + cd build && \ + ../configure --prefix=/usr/local && \ + make -j$(nproc) LDFLAGS="-lpthread" && \ + make install-strip + +RUN set -x && mkdir -p /usr/src/util-linux && \ + cd /usr/src/util-linux && \ + wget https://mirrors.edge.kernel.org/pub/linux/utils/util-linux/v2.33/util-linux-2.33.1.tar.xz && \ + tar -xvf util-linux-2.33.1.tar.xz && \ + cd util-linux-2.33.1 && \ + wget https://gist.githubusercontent.com/JoshuaWatt/50642701a7d9c1a3288357bbf94f99ce/raw/0f1cbc4e34286ebd6cd3413983d4687f34b6e2c9/0001-Remove-CAP_LAST_CAP-validation.patch && \ + patch -p1 < 0001-Remove-CAP_LAST_CAP-validation.patch && \ + mkdir build && \ + cd build && \ + ../configure \ + --disable-doc \ + --disable-nls \ + --enable-setpriv \ + --without-bashcompletion \ + --prefix=/usr/local && \ + make -j$(nproc) && \ + make install-strip + # # Prebuilt static tini # @@ -144,9 +184,12 @@ RUN set -x && export DEBIAN_FRONTEND=noninteractive && \ (locale -a | tee /dev/stderr | grep -qx en_US.utf8) # Copy prebuilt items -COPY --from=prebuilt-setpriv /dist/setpriv / +COPY --from=prebuilt-util-linux-14.04 /usr/local/ /usr/local/ COPY --from=prebuilt-tini /dist/tini / +# Rebuild library cache after copying prebuilt utl-linux +RUN ldconfig -v + # # Ubuntu 16.04 Base # From 7491b06035ce818b86d1f5bcae1f40c0ecae9974 Mon Sep 17 00:00:00 2001 From: Joshua Watt Date: Mon, 30 Dec 2019 14:04:43 -0600 Subject: [PATCH 31/48] entry.py: Fix home directory safety check Fixes the check to see if the home directory is safe to modify so that is uses findmnt instead of checking the device node. The device node will be the same for bind mounts originating from the same device so it's not a safe check. Instead, instruct findmnt to find the mounted device for the home directory. If it resolves to "/" it means that the home directory isn't bound into the container and can be modified. --- image/entry.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/image/entry.py b/image/entry.py index 0eadcb5..847246c 100644 --- a/image/entry.py +++ b/image/entry.py @@ -102,13 +102,18 @@ def main(): except OSError: pass - # Be a little paranoid about this. Only coerce the home directory if it - # happens to be on the same device as the root directory (which should - # only be true if it hasn't be bind mounted in the container) - root_stat = os.stat("/") - home_stat = os.stat(home) + # Be a little paranoid about this. Only coerce the home directory if + # it's target mount is is the root directory (which should only be true + # if it hasn't be bind mounted in the container) + target = ( + subprocess.check_output( + ["findmnt", "-f", "-n", "-o", "TARGET", "--target", home] + ) + .decode("utf-8") + .strip() + ) - if home_stat.st_dev == root_stat.st_dev: + if target == "/": os.chown(home, uid, primarygid) try: From c3a88f22b7a809310eb6c8a289552566af3516bc Mon Sep 17 00:00:00 2001 From: Joshua Watt Date: Mon, 30 Dec 2019 16:15:02 -0600 Subject: [PATCH 32/48] Bump version for release Bumps version for the beta4 release --- pyrex.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyrex.py b/pyrex.py index a3c5ef7..f8aeb5f 100755 --- a/pyrex.py +++ b/pyrex.py @@ -32,7 +32,7 @@ import tempfile import contextlib -VERSION = "1.0.0-beta3" +VERSION = "1.0.0-beta4" VERSION_REGEX = re.compile(r"^([0-9]+\.){2}[0-9]+(-.*)?$") VERSION_TAG_REGEX = re.compile(r"^v([0-9]+\.){2}[0-9]+(-.*)?$") From 328238cc6279d64a4bd6bebec910062789798d27 Mon Sep 17 00:00:00 2001 From: Joshua Watt Date: Wed, 1 Jan 2020 21:37:23 -0600 Subject: [PATCH 33/48] image: Exclude wic and oe-run-native Exclude both wic and oe-run-native from executing inside Pyrex. Both of these commands are intended to be run in the host context. --- image/capture.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/image/capture.sh b/image/capture.sh index c9e33fd..1356df6 100755 --- a/image/capture.sh +++ b/image/capture.sh @@ -95,7 +95,10 @@ cat > $PYREX_CAPTURE_DEST < Date: Tue, 7 Jan 2020 09:23:02 -0600 Subject: [PATCH 34/48] Export BB_ENV_EXTRAWHITE and BUILDDIR from capture The environment variables BB_ENV_EXTRAWHITE and BUILDDIR are expected to be present in the host shell --- ci/test.py | 37 +++++++++++++++++++++++++++++++++---- image/capture.sh | 6 +++++- pyrex.ini | 10 ++++++++-- pyrex.py | 46 +++++++++++++++++++++++++++------------------- 4 files changed, 73 insertions(+), 26 deletions(-) diff --git a/ci/test.py b/ci/test.py index 05c5253..32478f0 100755 --- a/ci/test.py +++ b/ci/test.py @@ -1,6 +1,6 @@ #! /usr/bin/env python3 # -# Copyright 2019 Garmin Ltd. or its subsidiaries +# Copyright 2019-2020 Garmin Ltd. or its subsidiaries # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -662,10 +662,12 @@ def test_bb_env_extrawhite(self): env["BB_ENV_EXTRAWHITE"] = "TEST_BB_EXTRA" env["TEST_BB_EXTRA"] = "Hello" - s = self.assertPyrexContainerShellCommand( - "echo $BB_ENV_EXTRAWHITE", env=env, quiet_init=True, capture=True + s = set( + self.assertPyrexContainerShellCommand( + "echo $BB_ENV_EXTRAWHITE", env=env, quiet_init=True, capture=True + ).split() ) - self.assertEqual(s, env["BB_ENV_EXTRAWHITE"]) + self.assertIn(env["BB_ENV_EXTRAWHITE"], s) s = self.assertPyrexContainerShellCommand( "echo $TEST_BB_EXTRA", env=env, quiet_init=True, capture=True @@ -918,6 +920,33 @@ def test_top_dir(self): self.assertEqual(oe_topdir, pyrex_topdir) + def test_env_capture(self): + extra_white = set( + self.assertPyrexHostCommand( + "echo $BB_ENV_EXTRAWHITE", quiet_init=True, capture=True + ).split() + ) + + # The exact values aren't relevant, only that they are correctly + # imported from the capture + self.assertIn("MACHINE", extra_white) + self.assertIn("DISTRO", extra_white) + + builddir = self.assertPyrexHostCommand( + "echo $BUILDDIR", quiet_init=True, capture=True + ) + self.assertEqual(builddir, self.build_dir) + + def test_bb_env_extrawhite_parse(self): + env = os.environ.copy() + env["BB_ENV_EXTRAWHITE"] = "TEST_BB_EXTRA" + env["TEST_BB_EXTRA"] = "foo" + + s = self.assertPyrexHostCommand( + "bitbake -e | grep ^TEST_BB_EXTRA=", env=env, quiet_init=True, capture=True + ) + self.assertEqual(s, 'TEST_BB_EXTRA="foo"') + PROVIDERS = ("docker", "podman") diff --git a/image/capture.sh b/image/capture.sh index 1356df6..b2f4d02 100755 --- a/image/capture.sh +++ b/image/capture.sh @@ -85,7 +85,11 @@ cat > $PYREX_CAPTURE_DEST < Date: Tue, 7 Jan 2020 10:20:16 -0600 Subject: [PATCH 35/48] Bump version for release Bumps the Pyrex version for a beta release --- pyrex.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyrex.py b/pyrex.py index 2a0ccd2..5f7c63c 100755 --- a/pyrex.py +++ b/pyrex.py @@ -32,7 +32,7 @@ import tempfile import contextlib -VERSION = "1.0.0-beta4" +VERSION = "1.0.0-beta5" VERSION_REGEX = re.compile(r"^([0-9]+\.){2}[0-9]+(-.*)?$") VERSION_TAG_REGEX = re.compile(r"^v([0-9]+\.){2}[0-9]+(-.*)?$") From 1bcb918b53526722cd5e5728c7fd6736bdd07609 Mon Sep 17 00:00:00 2001 From: Joshua Watt Date: Thu, 5 Dec 2019 15:24:53 -0600 Subject: [PATCH 36/48] Add OE test image Adds a new image type named "oetest" which contains additional packages such that the OE test functionality can be run. Currently, this includes wine so that the MinGW SDKs can be tested --- .travis.yml | 1 + ci/test.py | 10 ++++++++++ image/Dockerfile | 19 +++++++++++++++++++ 3 files changed, 30 insertions(+) diff --git a/.travis.yml b/.travis.yml index 74d8f11..3df5bc3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,6 +8,7 @@ env: - TEST_IMAGE=ubuntu-18.04-oe DOCKER_PROVIDER=podman SH=bash - TEST_IMAGE=ubuntu-14.04-oe DOCKER_PROVIDER=docker SH=bash - TEST_IMAGE=ubuntu-16.04-oe DOCKER_PROVIDER=docker SH=bash + - TEST_IMAGE=ubuntu-18.04-oetest DOCKER_PROVIDER=docker SH=bash - TEST_IMAGE=ubuntu-18.04-oe DOCKER_PROVIDER=docker SH=zsh - TEST_IMAGE=centos-7-base DOCKER_PROVIDER=docker SH=bash - TEST_IMAGE=ubuntu-14.04-base DOCKER_PROVIDER=docker SH=bash diff --git a/ci/test.py b/ci/test.py index 32478f0..7ec56fc 100755 --- a/ci/test.py +++ b/ci/test.py @@ -948,6 +948,15 @@ def test_bb_env_extrawhite_parse(self): self.assertEqual(s, 'TEST_BB_EXTRA="foo"') +class PyrexImageType_oetest(PyrexImageType_oe): + """ + Tests images designed for building OpenEmbedded Test image + """ + + def test_wine(self): + self.assertPyrexContainerCommand("wine --version") + + PROVIDERS = ("docker", "podman") TEST_IMAGES = ( @@ -958,6 +967,7 @@ def test_bb_env_extrawhite_parse(self): "ubuntu-14.04-oe", "ubuntu-16.04-oe", "ubuntu-18.04-oe", + "ubuntu-18.04-oetest", ) diff --git a/image/Dockerfile b/image/Dockerfile index b4e9610..360a684 100644 --- a/image/Dockerfile +++ b/image/Dockerfile @@ -493,6 +493,19 @@ RUN python3 -m pip install iterfzf # Copy prebuilt items COPY --from=prebuilt-icecream /dist/icecream / +# +# Ubuntu 18.04 OE Test Image +# +FROM ubuntu-18.04-oe as ubuntu-18.04-oetest +ENV PYREX_BASE none + +RUN set -x && export DEBIAN_FRONTEND=noninteractive && \ + sudo dpkg --add-architecture i386 && \ + apt-get -y update && apt-get -y install \ + wine64 \ + wine32 \ +&& rm -rf /var/lib/apt/lists/* + # # Base image target. # @@ -561,3 +574,9 @@ RUN mkdir -p /usr/share/icecc/toolchain && \ ENV ICECC_VERSION=/usr/share/icecc/toolchain/native-gcc.tar.gz +# +# OE build image. Includes many additional packages for testing +# +FROM pyrex-oe as pyrex-oetest +ENV PYREX_BASE none + From ff0dbb04e28c9faf5f38ff84fd52a980e9d9156b Mon Sep 17 00:00:00 2001 From: Joshua Watt Date: Mon, 13 Jan 2020 12:29:56 -0600 Subject: [PATCH 37/48] image: Add testing modules to oetest image Adds the python testtools and subunit packages to the oetest image so that OE tests can run in parallel --- image/Dockerfile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/image/Dockerfile b/image/Dockerfile index 360a684..d4c265d 100644 --- a/image/Dockerfile +++ b/image/Dockerfile @@ -506,6 +506,9 @@ RUN set -x && export DEBIAN_FRONTEND=noninteractive && \ wine32 \ && rm -rf /var/lib/apt/lists/* +# Python modules used for tests +RUN python3 -m pip install testtools python-subunit + # # Base image target. # From 03a5646a5744615acf489918966d86526d3943e9 Mon Sep 17 00:00:00 2001 From: Joshua Watt Date: Mon, 13 Jan 2020 13:21:05 -0600 Subject: [PATCH 38/48] pyrex.py: Fix bad return value build_image() needs to return None on failure, not 1 --- pyrex.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyrex.py b/pyrex.py index 5f7c63c..75e64fc 100755 --- a/pyrex.py +++ b/pyrex.py @@ -266,7 +266,7 @@ def build_image(config, build_config): build_config["build"]["buildid"] = get_image_id(config, tag) except subprocess.CalledProcessError: - return 1 + return None build_config["build"]["runid"] = tag From 775cbcea5c2d17c7a6c24fcb2b636a8983e989f2 Mon Sep 17 00:00:00 2001 From: Joshua Watt Date: Tue, 14 Jan 2020 19:04:02 -0600 Subject: [PATCH 39/48] Pass OE init vars in capture Pass along the environment variables that oe-init-build-env can key off of to configure the build (BDIR, BITBAKEDIR, OEROOT). These variables are only passed during capture, since they are normally unset by oe-init-build-env. Adds some tests to validate the variables are passed correctly. --- ci/test.py | 87 +++++++++++++++++++++++++++++++++++++++--------- image/capture.sh | 20 +++++++++-- pyrex.ini | 8 +++++ pyrex.py | 11 +++--- 4 files changed, 104 insertions(+), 22 deletions(-) diff --git a/ci/test.py b/ci/test.py index 7ec56fc..146d639 100755 --- a/ci/test.py +++ b/ci/test.py @@ -173,24 +173,35 @@ def assertSubprocess( return None def _write_host_command( - self, args, *, quiet_init=False, cwd=PYREX_ROOT, builddir=None, bitbakedir="" + self, + args, + *, + quiet_init=False, + cwd=PYREX_ROOT, + builddir=None, + bitbakedir="", + init_env={} ): if builddir is None: builddir = self.build_dir - command = [ - "PYREXCONFFILE=%s\n" % self.pyrex_conf, - ". %s/poky/pyrex-init-build-env%s %s %s && " - % ( - PYREX_ROOT, - " > /dev/null 2>&1" if quiet_init else "", - builddir, - bitbakedir, - ), - "(", - " && ".join(list(args)), - ")", - ] + command = ['export %s="%s"\n' % (k, v) for k, v in init_env.items()] + + command.extend( + [ + "PYREXCONFFILE=%s\n" % self.pyrex_conf, + ". %s/poky/pyrex-init-build-env%s %s %s && " + % ( + PYREX_ROOT, + " > /dev/null 2>&1" if quiet_init else "", + builddir, + bitbakedir, + ), + "(", + " && ".join(list(args)), + ")", + ] + ) command = "".join(command) @@ -212,6 +223,7 @@ def assertPyrexHostCommand( cwd=PYREX_ROOT, builddir=None, bitbakedir="", + init_env={}, **kwargs ): cmd_file, command = self._write_host_command( @@ -220,6 +232,7 @@ def assertPyrexHostCommand( cwd=cwd, builddir=builddir, bitbakedir=bitbakedir, + init_env=init_env, ) return self.assertSubprocess( ["/bin/bash", cmd_file], pretty_command=command, cwd=cwd, **kwargs @@ -775,8 +788,9 @@ class PyrexImageType_oe(PyrexImageType_base): def test_bitbake_parse(self): self.assertPyrexHostCommand("bitbake -p") - def test_bitbake_parse_altpath(self): - # The new bitbake directory is out of the normally bound tree + def test_bitbake_parse_altpath_arg(self): + # The new bitbake directory is out of the normally bound tree (passed + # as an argument) with tempfile.TemporaryDirectory() as tmpdir: bitbakedir = os.path.join(tmpdir, "bitbake") shutil.copytree(os.path.join(PYREX_ROOT, "poky/bitbake"), bitbakedir) @@ -801,6 +815,47 @@ def test_bitbake_parse_altpath(self): self.assertPyrexHostCommand("bitbake -p", bitbakedir=bitbakedir) + def test_bitbake_parse_altpath_env(self): + # The new bitbake directory is out of the normally bound tree (passed + # as an argument) + with tempfile.TemporaryDirectory() as tmpdir: + bitbakedir = os.path.join(tmpdir, "bitbake") + shutil.copytree(os.path.join(PYREX_ROOT, "poky/bitbake"), bitbakedir) + + env = {"BITBAKEDIR": bitbakedir} + + # If the bitbake directory is not bound, capture should fail with + # an error + d = self.assertPyrexHostCommand( + "bitbake -p", returncode=1, capture=True, init_env=env + ) + self.assertIn("ERROR: %s not bound in container" % bitbakedir, d) + + # Binding the build directory in the conf file will allow bitbake + # to be found + conf = self.get_config() + conf["run"]["bind"] = bitbakedir + conf.write_conf() + + d = self.assertPyrexContainerCommand( + "which bitbake", quiet_init=True, capture=True, init_env=env + ) + self.assertEqual(d, os.path.join(bitbakedir, "bin", "bitbake")) + + self.assertPyrexHostCommand("bitbake -p", init_env=env) + + def test_builddir_alt_env(self): + with tempfile.TemporaryDirectory() as builddir: + # Binding the build directory in the conf file will allow building + # to continue + conf = self.get_config() + conf["run"]["bind"] = builddir + conf.write_conf() + + env = {"BDIR": builddir} + + self.assertPyrexHostCommand("true", builddir="", init_env=env) + def test_unbound_builddir(self): with tempfile.TemporaryDirectory() as tmpdir: builddir = os.path.join(tmpdir) diff --git a/image/capture.sh b/image/capture.sh index b2f4d02..a3eb8ad 100755 --- a/image/capture.sh +++ b/image/capture.sh @@ -55,10 +55,26 @@ shift $# # Ensure the init script is bound in the container check_bound $PYREX_OEINIT -# If the bitbake directory argument is provided, ensure it is bound in the -# container +# If the bitbake directory argument or environment variable is provided, ensure +# it is bound in the container if [ -n "${PYREX_ARGS[1]}" ]; then check_bound "${PYREX_ARGS[1]}" + + if [ -n "${PYREX_ARGS[2]}" ]; then + check_bound "${PYREX_ARGS[2]}" + fi +fi + +if [ -n "$BITBAKEDIR" ]; then + check_bound "$BITBAKEDIR" +fi + +if [ -n "$BDIR" ]; then + check_bound "$BDIR" +fi + +if [ -n "$OEROOT" ]; then + check_bound "$OEROOT" fi . $PYREX_OEINIT "${PYREX_ARGS[@]}" diff --git a/pyrex.ini b/pyrex.ini index b72a31d..c82d101 100644 --- a/pyrex.ini +++ b/pyrex.ini @@ -74,6 +74,14 @@ confversion = @CONFVERSION@ %env = % DOCKER_BUILDKIT=1 +# Capture options. Changes in the section only affect when a Pyrex container is +# initialized +[capture] +%envvars = +% BDIR +% BITBAKEDIR +% OEROOT + # Runtime options. Changes in this section take effect the next time a Pyrex # command is run [run] diff --git a/pyrex.py b/pyrex.py index 75e64fc..2eae947 100755 --- a/pyrex.py +++ b/pyrex.py @@ -50,7 +50,7 @@ def __init__(self, *args, **kwargs): interpolation=configparser.ExtendedInterpolation(), comment_prefixes=["#"], delimiters=["="], - **kwargs + **kwargs, ) # All keys are case-sensitive @@ -334,7 +334,8 @@ def prep_container( extra_env={}, preserve_env=[], extra_bind=[], - allow_test_config=False + allow_test_config=False, + extra_envvars=[], ): runid = build_config["build"]["runid"] @@ -476,6 +477,7 @@ def prep_container( engine_args.extend(["--mount", "type=bind,src={b},dst={b}".format(b=b)]) container_envvars.extend(config["run"]["envvars"].split()) + container_envvars.extend(extra_envvars) # Pass along BB_ENV_EXTRAWHITE and anything it has whitelisted if "BB_ENV_EXTRAWHITE" in os.environ: @@ -676,6 +678,7 @@ def capture(args): extra_env=env_args, preserve_env=args.env, extra_bind=[f.name], + extra_envvars=config["capture"]["envvars"].split(), ) if not engine_args: @@ -812,7 +815,7 @@ def get_output_file(): title="subcommands", description="Setup subcommands", dest="subcommand", - **subparser_args + **subparser_args, ) capture_parser = subparsers.add_parser( @@ -855,7 +858,7 @@ def get_output_file(): title="subcommands", description="Config subcommands", dest="config_subcommand", - **subparser_args + **subparser_args, ) config_get_parser = config_subparsers.add_parser( From 74636d23333adb7d4e6a9e91ec349a7da1806b3e Mon Sep 17 00:00:00 2001 From: Joshua Watt Date: Tue, 14 Jan 2020 20:48:14 -0600 Subject: [PATCH 40/48] ci: Remove useless os.path.join() --- ci/test.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/ci/test.py b/ci/test.py index 146d639..31ab9e3 100755 --- a/ci/test.py +++ b/ci/test.py @@ -857,9 +857,7 @@ def test_builddir_alt_env(self): self.assertPyrexHostCommand("true", builddir="", init_env=env) def test_unbound_builddir(self): - with tempfile.TemporaryDirectory() as tmpdir: - builddir = os.path.join(tmpdir) - + with tempfile.TemporaryDirectory() as builddir: # If the build directory is not bound, capture should fail with an # error d = self.assertPyrexHostCommand( From c191b59e4ee539fcc6a1bbc963aca714f5775a4e Mon Sep 17 00:00:00 2001 From: Joshua Watt Date: Wed, 15 Jan 2020 12:11:48 -0600 Subject: [PATCH 41/48] Allow bind mounts to be optional Adds an "optional" flag so that bind mounts will be ignored if they do not exists. If a bind mount without this flag is specified, it is now an error. --- ci/test.py | 27 ++++++++++++++++++++++++++- pyrex.py | 27 ++++++++++++++++++++++++++- 2 files changed, 52 insertions(+), 2 deletions(-) diff --git a/ci/test.py b/ci/test.py index 31ab9e3..f206e4d 100755 --- a/ci/test.py +++ b/ci/test.py @@ -453,10 +453,35 @@ def test_missing_bind(self): conf.write_conf() s = self.assertPyrexContainerShellCommand( - "test ! -e %s" % missing_bind, capture=True + "test -e %s" % missing_bind, capture=True, returncode=1 ) self.assertRegex(s, r"Error: bind source path \S+ does not exist") + def test_optional_bind(self): + temp_dir = tempfile.mkdtemp("-pyrex") + self.addCleanup(shutil.rmtree, temp_dir) + + missing_bind = os.path.join(temp_dir, "does-not-exist") + conf = self.get_config() + conf["run"]["bind"] += " %s,optional" % missing_bind + conf.write_conf() + + s = self.assertPyrexContainerShellCommand("test ! -e %s" % missing_bind) + + def test_bad_bind_option(self): + temp_dir = tempfile.mkdtemp("-pyrex") + self.addCleanup(shutil.rmtree, temp_dir) + + missing_bind = os.path.join(temp_dir, "does-not-exist") + conf = self.get_config() + conf["run"]["bind"] += " %s,bad-option" % missing_bind + conf.write_conf() + + s = self.assertPyrexContainerShellCommand( + "test ! -e %s" % missing_bind, capture=True, returncode=1 + ) + self.assertIn("Error: bad option(s) 'bad-option' for bind", s) + def test_bad_confversion(self): # Verify that a bad config is an error conf = self.get_config() diff --git a/pyrex.py b/pyrex.py index 2eae947..4bd4b27 100755 --- a/pyrex.py +++ b/pyrex.py @@ -31,6 +31,7 @@ import json import tempfile import contextlib +import types VERSION = "1.0.0-beta5" @@ -326,6 +327,22 @@ def get_subid_length(filename, name): return 0 +def parse_bind_options(bind): + options = types.SimpleNamespace(optional=False) + bad_options = [] + + if "," in bind: + s = bind.split(",") + bind = s[0] + for opt in s[1:]: + if opt == "optional": + options.optional = True + else: + bad_options.append(opt) + + return bind, options, bad_options + + def prep_container( config, build_config, @@ -471,9 +488,17 @@ def prep_container( + extra_bind ) for b in set(binds): + b, options, bad_options = parse_bind_options(b) + if bad_options: + print("Error: bad option(s) '%s' for bind %s" % (" ".join(bad_options), b)) + return [] + if not os.path.exists(b): + if options.optional: + continue print("Error: bind source path {b} does not exist".format(b=b)) - continue + return [] + engine_args.extend(["--mount", "type=bind,src={b},dst={b}".format(b=b)]) container_envvars.extend(config["run"]["envvars"].split()) From af1d1ee8e6a58f3836f863fefa7a7898445d22b5 Mon Sep 17 00:00:00 2001 From: Joshua Watt Date: Wed, 15 Jan 2020 12:26:03 -0600 Subject: [PATCH 42/48] pyrex-init-build-env: Respect OEROOT if set If $OEROOT is set when initializing (and PYREX_OEINIT is not), look for the init script under that path. This allows a more seamless initialization for users not using a poky-like layout. --- README.md | 2 +- pyrex-init-build-env | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 7ba9e83..daddba5 100644 --- a/README.md +++ b/README.md @@ -105,7 +105,7 @@ look like: PYREX_CONFIG_BIND="$(pwd)" # The path to the build init script. If unspecified, defaults to -# "$(pwd)/oe-init-build-env" +# "$OEROOT/oe-init-build-env" or "$(pwd)/oe-init-build-env" PYREX_OEINIT="$(pwd)/oe-init-build-env" # The location of Pyrex itself. If not specified, pyrex-init-build-env will diff --git a/pyrex-init-build-env b/pyrex-init-build-env index faf408e..b4686b3 100644 --- a/pyrex-init-build-env +++ b/pyrex-init-build-env @@ -38,7 +38,11 @@ if [ -z "$PYREX_ROOT" ]; then fi if [ -z "$PYREX_OEINIT" ]; then - PYREX_OEINIT="$(pwd)/oe-init-build-env" + if [ -z "$OEROOT" ]; then + PYREX_OEINIT="$(pwd)/oe-init-build-env" + else + PYREX_OEINIT="$OEROOT/oe-init-build-env" + fi fi unset THIS_SCRIPT From 2e55f3a25b08c4804bca1d9b905ed06b4abf0847 Mon Sep 17 00:00:00 2001 From: Joshua Watt Date: Wed, 15 Jan 2020 12:11:26 -0600 Subject: [PATCH 43/48] README: Add quickstart guides --- README.md | 54 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/README.md b/README.md index daddba5..d21b15d 100644 --- a/README.md +++ b/README.md @@ -4,6 +4,60 @@ Containerize your bitbake [![Build Status](https://travis-ci.com/garmin/pyrex.svg?branch=master)](https://travis-ci.com/garmin/pyrex) [![Coverage Status](https://coveralls.io/repos/github/garmin/pyrex/badge.svg?branch=master)](https://coveralls.io/github/garmin/pyrex?branch=master) +## Quickstart Guide (default layout) +Use this quickstart guide if your project uses the default (poky-style) layout +where bitbake and layers are subdirectories of oe-core: + +```shell +# Clone down Pyrex +git clone https://github.com/garmin/pyrex.git meta-pyrex + +# Create the pyrex environment initialization script symbolic link +ln -s meta-pyrex/pyrex-init-build-env + +# Create a default pyrex.ini config file +meta-pyrex/mkconfig > ./pyrex.ini + +# Set PYREXCONFFILE to the location of the newly created config file +PYREXCONFFILE=./pyrex.ini + +# Initialize the build environment +. pyrex-init-build-env + +# Everything is setup. OpenEmbedded build commands (e.g. `bitbake`) will now +# run in Pyrex +``` + +## Quickstart Guide (alternate layout) +Use this quickstart guide if your project uses a different layout where bitbake +and oe-core are disjointed. In the example, it is assumed that oe-core and +bitbake are both subdirectories of the current working directory, so you will +need to change it to match your actual layout: + +```shell +# Clone down Pyrex +git clone https://github.com/garmin/pyrex.git pyrex + +# Create the pyrex environment initialization script symbolic link +ln -s pyrex/pyrex-init-build-env + +# Create a default pyrex.ini config file +pyrex/mkconfig > ./pyrex.ini + +# Set PYREXCONFFILE to the location of the newly created config file +PYREXCONFFILE=./pyrex.ini + +# Tell Pyrex where bitbake and oe-core live +BITBAKEDIR=$(pwd)/bitbake +OEROOT=$(pwd)/oe-core + +# Initialize the build environment +. pyrex-init-build-env + +# Everything is setup. OpenEmbedded build commands (e.g. `bitbake`) will now +# run in Pyrex +``` + ## What is Pyrex? At its core, Pyrex is an attempt to provided a consistent environment in which developers can run Yocto and bitbake commands. Pyrex is targeted at development From 2c56590b0d2151b940b1dc012ab34285682cb6b0 Mon Sep 17 00:00:00 2001 From: Joshua Watt Date: Wed, 15 Jan 2020 14:41:00 -0600 Subject: [PATCH 44/48] Bump version for release Bumps the version for the 6th beta release --- pyrex.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyrex.py b/pyrex.py index 4bd4b27..b133f4f 100755 --- a/pyrex.py +++ b/pyrex.py @@ -33,7 +33,7 @@ import contextlib import types -VERSION = "1.0.0-beta5" +VERSION = "1.0.0-beta6" VERSION_REGEX = re.compile(r"^([0-9]+\.){2}[0-9]+(-.*)?$") VERSION_TAG_REGEX = re.compile(r"^v([0-9]+\.){2}[0-9]+(-.*)?$") From 087f4180849bf42f1bcc386f31c4bb775e68a12e Mon Sep 17 00:00:00 2001 From: Joshua Watt Date: Wed, 15 Jan 2020 16:01:18 -0600 Subject: [PATCH 45/48] Update developer documentation Corrects several lies in the developer documentation --- DEVELOPING.md | 39 +++++++++++++++++++++------------------ 1 file changed, 21 insertions(+), 18 deletions(-) diff --git a/DEVELOPING.md b/DEVELOPING.md index 37374d6..567e0b8 100644 --- a/DEVELOPING.md +++ b/DEVELOPING.md @@ -2,10 +2,11 @@ Pyrex development information and processes ## Linting -Pyrex conforms to [PEP8](https://pep8.org/) for all Python code, and also make -use of [flake8](https://pypi.org/project/flake8/) as a linter. Please ensure -all code conforms to this. There is helpful tool that will report any places -where the code is non-conformant in the project root: +Pyrex conforms to [black](https://black.readthedocs.io/en/stable/) for all +Python code, and also makes use of [flake8](https://pypi.org/project/flake8/) +as a linter. Please ensure all code conforms to this. There is helpful tool +that will report any places where the code is non-conformant in the project +root: ```shell ./lint @@ -40,12 +41,13 @@ available. The test suite will build all Pyrex image locally and run a set of tests against them. If you would like to limit which images are tested, additional arguments can be passed to filter the tests. Each image is tested using a test -class named `PyrexImage_IMAGE` where `IMAGE` is the test image name will all -non-word characters replaced by `_`. For example, to test only the -`ubuntu-18.04-oe` image, run: +class named `PyrexImage_ENGINE_IMAGE` where `ENGINE` is the container engine +(docker or podman) and `IMAGE` is the test image name will all non-word +characters replaced by `_`. For example, to test only the `ubuntu-18.04-oe` +image under docker, run: ```shell -./ci/test.py -vb PyrexImage_ubuntu_18_04_oe +./ci/test.py -vb PyrexImage_docker_ubuntu_18_04_oe ``` ## Building images locally @@ -55,21 +57,22 @@ or wish to build images locally instead of using published images can do so by making the following changes to `pyrex.ini`: 1. Set `config:buildlocal` to `1` -2. Change `config:tag` to an alternate tag suffix instead of - `:${config:pyrextag}`. While not strictly necessary, this step will help - prevent confusion if you want to switch back to prebuilt images. If you - choose not to change this, realize that your locally built images will - overwrite your local container cache tags for the prebuilt images. As an - example, you might add the following to `pyrex.ini`: +2. Change `config:pyrextag` to an alternate tag instead of referencing the + released version. While not strictly necessary, this step will help prevent + confusion if you want to switch back to prebuilt images. If you choose not + to change this, realize that your locally built images will overwrite your + local container cache tags for the prebuilt images. As an example, you might + add the following to `pyrex.ini`: ``` [config] - tag = ${config:image}:my-image + pyrextag = my-image ``` -3. Set `config:dockerfile` to the path where the Dockerfile you wish to build - is located. Alternatively, you can leave it as the default to build the - standard Pyrex images locally. +3. Set `imagebuild:buildcommand` to adjust any of the build options (e.g. the + path where the Dockerfile you wish to build is located). +4. You may also want to set `imagebuild:quiet` to `false` if you want to see + the images being built for debugging purposes ## Making a release To make a release of Pyrex: From d7fa945d93498cf22cc9938b78b7724923dd75f4 Mon Sep 17 00:00:00 2001 From: Joshua Watt Date: Wed, 15 Jan 2020 16:02:20 -0600 Subject: [PATCH 46/48] Update flake8 support According to the developer documentation, this project adheres to flake8. Update it so that it actually does this and also add tests for it to the CI pipeline. --- .travis.yml | 2 ++ ci/test.py | 2 +- lint | 1 + pyrex.py | 2 +- 4 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 3df5bc3..d28c17c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -36,12 +36,14 @@ install: - pip3 install coveralls - pip3 install requests - pip3 install black +- pip3 install flake8 before_script: - printf '\nimport coverage\ncoverage.current_coverage = coverage.process_startup()\n' >> "/home/travis/virtualenv/python${TRAVIS_PYTHON_VERSION}/lib/python${TRAVIS_PYTHON_VERSION}/sitecustomize.py" - rm -f .coverage-report.* script: - black --check $(git ls-files '*.py') +- flake8 $(git ls-files '*.py') # Pre build the image. This prevents long delays with no output in test cases - ./ci/build_image.py $TEST_IMAGE --provider=$DOCKER_PROVIDER --quiet - COVERAGE_PROCESS_START=${TRAVIS_BUILD_DIR}/.coveragerc $SH -c "ci/test.py -vb PyrexImage_${DOCKER_PROVIDER}_$(echo $TEST_IMAGE | sed 's/\W/_/g')" diff --git a/ci/test.py b/ci/test.py index f206e4d..ff07e7c 100755 --- a/ci/test.py +++ b/ci/test.py @@ -466,7 +466,7 @@ def test_optional_bind(self): conf["run"]["bind"] += " %s,optional" % missing_bind conf.write_conf() - s = self.assertPyrexContainerShellCommand("test ! -e %s" % missing_bind) + self.assertPyrexContainerShellCommand("test ! -e %s" % missing_bind) def test_bad_bind_option(self): temp_dir = tempfile.mkdtemp("-pyrex") diff --git a/lint b/lint index e80a062..d9800f5 100755 --- a/lint +++ b/lint @@ -26,3 +26,4 @@ else black --check $(git ls-files '*.py') fi +flake8 $(git ls-files '*.py') diff --git a/pyrex.py b/pyrex.py index b133f4f..ce68c21 100755 --- a/pyrex.py +++ b/pyrex.py @@ -516,7 +516,7 @@ def prep_container( # them. env_sock_proxy = config["run"]["envsockproxy"].split() for name in set(container_envvars + preserve_env): - if not name in os.environ: + if name not in os.environ: continue val = os.environ[name] From 916ef8dfa993bc1e286b218650826a5a52e588a6 Mon Sep 17 00:00:00 2001 From: Joshua Watt Date: Fri, 17 Jan 2020 16:03:26 -0600 Subject: [PATCH 47/48] travis: Remove encrypted dockerhub credentials Removes the encrypted dockerhub credentials --- .travis.yml | 2 -- ci/travis_secure.sh | 39 --------------------------------------- 2 files changed, 41 deletions(-) delete mode 100755 ci/travis_secure.sh diff --git a/.travis.yml b/.travis.yml index d28c17c..d06178a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -15,8 +15,6 @@ env: - TEST_IMAGE=ubuntu-16.04-base DOCKER_PROVIDER=docker SH=bash - TEST_IMAGE=ubuntu-18.04-base DOCKER_PROVIDER=docker SH=bash global: - - secure: Lr+dsOabZwFuCQbOHL0px61zYRjiUTFkBTxGB5g4OfCUxMq4nA87naRQjSUARR5aZo55pli+ceKVcmIt568L3Z7x5ByHZoCUriwgb+ud2ks6nuWBWpAGleQHzHGbQ0QlQwm1k8IVY4OAiUJz2+yfpcOY3Y89bc/oJJEaonI/a7Qh7PJJ8Hch4P2st6+Pwi49T9WN+SUN1sh76fhzqoD/8M6B5pC7QEbYEnPI4tWS3Q0Tf8m64NEwTIgtKXbpyHObpGh+bth4O2ZmOZK4bZ4AAZAfTGLqBqNmwJ329aZzVpJs6DzpkojFCi0QFYKRgrPIPL0mTNdxogMs5Q8ULSflXkMUnmZSzkqNgaH6OTLbkvuQjbxw6BMNn2VLvQ6ri6iaAoSwgyD0XYvEDGujYW6ZP0sKgAC4JV1Ij71C/p9iiJUVYHEqYTTLSrd413kJQiEaO2F81CPWwOirxsBiknTWjyvRcH/FBiHLcrBG6ToOxm7ELfQ7JD+Ag/Hlbh9jAHk3kkT9WVwcrfeWfd4ulMkXaL2w3UvPkNCB3nmmcqtcFcojs4lrPDPO+lJ+zTBjupty/TwmzbUrza15jzjkFQ1xfMT0yS0Dr8zOH39dYHUw0TmEx0h85JH9FHaOQo4u8J7TeQyK+6OaHuj927bNTSBk23FB0OAOzbaqdTOlM3hgby8= - - secure: olbIFIge5rHlzcsh4AlvQmStOfHkl5YUal5SC/7TMmgG0w6NXTwUl16fIVwV0rLCmJRbceB1XbsoUlETN2whgMBoFzChHtVBVGDgNYuE31zIKTznvEef+HYG6z/bzx+96icf5EsqBsac7qXh8BUcKFuwA7tPSnaK7Wds2i1qDfj2Itrn3d3MlZU/MIeowpVqIP3+p16vPLv4+PZde4gh2pSq4c4QkfQC8c7IDFtIXzB3j75mRumwT2rokEMH/vuP/nX/TUPYYGpRFYTiRXBWcSs70Jd0jg664eQyJRnugIyeSW26h2pxqlp4G6bV96Vl+vzZOVaJo6p1sjzznteo+Rg7U0HZHpfZI1Q21jkPrdqevaGQ/nVGbi/sUkWgSQYIo2zIjxzgbd2PoeFBJ3vm5qg5m1HNXOdVhzwEjhvqX6lsHyMscY5NRdrWfjJrzRpqsPUv6v4Xj/ll4KleylqTubHT5cgsm4B0/fER9B0im5gw5jseh4IeHXT1djeB3ffIjTIywrYt+bgzrkzI4aD0LOLIoN2kFtxIXTM7oem+wKxSwSU/LBI+Jmw2lKXz2r5EkUpCjYUUftPjcT7q8qTpE5hcOvLOUoG4od9JKr5N1Hy4gsGLoYfK5RU19TSduZvQsCOya2G4m8JCGiYNuA14/VbCEzx0z3JCrO79wiUFRYU= - USE_HELPER=1 before_install: - "./ci/prepare.sh" diff --git a/ci/travis_secure.sh b/ci/travis_secure.sh deleted file mode 100755 index 6606564..0000000 --- a/ci/travis_secure.sh +++ /dev/null @@ -1,39 +0,0 @@ -#! /bin/bash -# -# Copyright 2019 Garmin Ltd. or its subsidiaries -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Use this script to setup the secure variables in .travis.yml - -TRAVIS=$(which travis 2> /dev/null) - -if [ -z "$TRAVIS" ]; then - echo "'travis' not found. Please install it with 'gem install travis'" - exit 1 -fi - -echo "NOTE: The Travis command line tool will append the new secure" -echo "environment. If you want to replace the existing one, you must delete it" -echo "from .travis.yml manually before running this script" -echo - -travis encrypt --add env.global < Date: Mon, 27 Jan 2020 16:44:04 -0600 Subject: [PATCH 48/48] Remove bypass support Removes support for bypassing pyrex with an environment variable. This support has some issues now that capture of the environment is done inside of a container and it's easy enough for a user to simply init the environment with a different script. --- README.md | 9 -------- ci/test.py | 7 +----- image/Dockerfile | 4 +--- image/bypass.sh | 27 ---------------------- image/capture.sh | 6 ----- pyrex.ini | 4 ---- pyrex.py | 58 ++++++++++-------------------------------------- 7 files changed, 14 insertions(+), 101 deletions(-) delete mode 100644 image/bypass.sh diff --git a/README.md b/README.md index d21b15d..f46195c 100644 --- a/README.md +++ b/README.md @@ -263,15 +263,6 @@ Pyrex environment setup you created. This will setup up the current shell to run the commands listed in `${config:command}` inside of Pyrex. Once this is done, you can simply run those commands and they will be executed in Pyrex. -### Bypassing Pyrex -In some cases, it may be desirable to bypass Pyrex and run the commands it -wraps locally instead of in the container. This can be done in one of two ways: - -1. Set `${run:enable}` to `0` in `pyrex.ini` which will disable using the - container engine for all commands -2. Set the environment variable `PYREX_USE_CONTAINER` to `0`. Any Pyrex - commands run with this variable will not be run in the container. - ## What doesn't work? The following items are either known to not work, or haven't been fully tested: * **Bitbake Server** Since the container starts and stops each time a command diff --git a/ci/test.py b/ci/test.py index ff07e7c..8174f1b 100755 --- a/ci/test.py +++ b/ci/test.py @@ -305,7 +305,7 @@ def test_pyrex_shell(self): def test_pyrex_run(self): self.assertPyrexContainerCommand("/bin/false", returncode=1) - def test_disable_pyrex(self): + def test_in_container(self): def capture_pyrex_state(*args, **kwargs): capture_file = os.path.join(self.thread_dir, "pyrex_capture") @@ -337,11 +337,6 @@ def capture_local_state(): pyrex_state = capture_pyrex_state() self.assertNotEqual(local_state, pyrex_state) - env = os.environ.copy() - env["PYREX_USE_CONTAINER"] = "0" - pyrex_state = capture_pyrex_state(env=env) - self.assertEqual(local_state, pyrex_state) - def test_quiet_build(self): env = os.environ.copy() env["PYREX_BUILD_QUIET"] = "1" diff --git a/image/Dockerfile b/image/Dockerfile index d4c265d..6afee5f 100644 --- a/image/Dockerfile +++ b/image/Dockerfile @@ -525,14 +525,12 @@ ENV LC_ALL en_US.UTF-8 # Add startup scripts COPY ./run.sh /usr/libexec/pyrex/run COPY ./capture.sh /usr/libexec/pyrex/capture -COPY ./bypass.sh /usr/libexec/pyrex/bypass COPY ./entry.py /usr/libexec/pyrex/entry.py COPY ./cleanup.py /usr/libexec/pyrex/cleanup.py RUN chmod +x /usr/libexec/pyrex/cleanup.py \ /usr/libexec/pyrex/entry.py \ /usr/libexec/pyrex/run \ - /usr/libexec/pyrex/capture \ - /usr/libexec/pyrex/bypass + /usr/libexec/pyrex/capture # Add startup script directory & test script. COPY ./test_startup.sh /usr/libexec/pyrex/startup.d/ diff --git a/image/bypass.sh b/image/bypass.sh deleted file mode 100644 index c51d453..0000000 --- a/image/bypass.sh +++ /dev/null @@ -1,27 +0,0 @@ -#! /bin/bash -# -# Copyright 2019 Garmin Ltd. or its subsidiaries -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -declare -a COMMAND=("$@") - -# Consume all arguments before sourcing the environment script -shift $# - -pushd "$PYREX_OEINIT_DIR" > /dev/null -source $PYREX_OEINIT > /dev/null -popd > /dev/null - -exec $PYREX_COMMAND_PREFIX "${COMMAND[@]}" - diff --git a/image/capture.sh b/image/capture.sh index a3eb8ad..4dcb371 100755 --- a/image/capture.sh +++ b/image/capture.sh @@ -128,12 +128,6 @@ cat > $PYREX_CAPTURE_DEST <