Merge pull request #44 from garmin/next

Merge next branch for 1.0.0 release

.travis.yml

@@ -8,14 +8,13 @@ env:
   - TEST_IMAGE=ubuntu-18.04-oe      DOCKER_PROVIDER=podman SH=bash
   - TEST_IMAGE=ubuntu-14.04-oe      DOCKER_PROVIDER=docker SH=bash
   - TEST_IMAGE=ubuntu-16.04-oe      DOCKER_PROVIDER=docker SH=bash
+  - TEST_IMAGE=ubuntu-18.04-oetest  DOCKER_PROVIDER=docker SH=bash
   - TEST_IMAGE=ubuntu-18.04-oe      DOCKER_PROVIDER=docker SH=zsh
   - TEST_IMAGE=centos-7-base        DOCKER_PROVIDER=docker SH=bash
   - TEST_IMAGE=ubuntu-14.04-base    DOCKER_PROVIDER=docker SH=bash
   - TEST_IMAGE=ubuntu-16.04-base    DOCKER_PROVIDER=docker SH=bash
   - TEST_IMAGE=ubuntu-18.04-base    DOCKER_PROVIDER=docker SH=bash
   global:
-  - secure: Lr+dsOabZwFuCQbOHL0px61zYRjiUTFkBTxGB5g4OfCUxMq4nA87naRQjSUARR5aZo55pli+ceKVcmIt568L3Z7x5ByHZoCUriwgb+ud2ks6nuWBWpAGleQHzHGbQ0QlQwm1k8IVY4OAiUJz2+yfpcOY3Y89bc/oJJEaonI/a7Qh7PJJ8Hch4P2st6+Pwi49T9WN+SUN1sh76fhzqoD/8M6B5pC7QEbYEnPI4tWS3Q0Tf8m64NEwTIgtKXbpyHObpGh+bth4O2ZmOZK4bZ4AAZAfTGLqBqNmwJ329aZzVpJs6DzpkojFCi0QFYKRgrPIPL0mTNdxogMs5Q8ULSflXkMUnmZSzkqNgaH6OTLbkvuQjbxw6BMNn2VLvQ6ri6iaAoSwgyD0XYvEDGujYW6ZP0sKgAC4JV1Ij71C/p9iiJUVYHEqYTTLSrd413kJQiEaO2F81CPWwOirxsBiknTWjyvRcH/FBiHLcrBG6ToOxm7ELfQ7JD+Ag/Hlbh9jAHk3kkT9WVwcrfeWfd4ulMkXaL2w3UvPkNCB3nmmcqtcFcojs4lrPDPO+lJ+zTBjupty/TwmzbUrza15jzjkFQ1xfMT0yS0Dr8zOH39dYHUw0TmEx0h85JH9FHaOQo4u8J7TeQyK+6OaHuj927bNTSBk23FB0OAOzbaqdTOlM3hgby8=
-  - secure: olbIFIge5rHlzcsh4AlvQmStOfHkl5YUal5SC/7TMmgG0w6NXTwUl16fIVwV0rLCmJRbceB1XbsoUlETN2whgMBoFzChHtVBVGDgNYuE31zIKTznvEef+HYG6z/bzx+96icf5EsqBsac7qXh8BUcKFuwA7tPSnaK7Wds2i1qDfj2Itrn3d3MlZU/MIeowpVqIP3+p16vPLv4+PZde4gh2pSq4c4QkfQC8c7IDFtIXzB3j75mRumwT2rokEMH/vuP/nX/TUPYYGpRFYTiRXBWcSs70Jd0jg664eQyJRnugIyeSW26h2pxqlp4G6bV96Vl+vzZOVaJo6p1sjzznteo+Rg7U0HZHpfZI1Q21jkPrdqevaGQ/nVGbi/sUkWgSQYIo2zIjxzgbd2PoeFBJ3vm5qg5m1HNXOdVhzwEjhvqX6lsHyMscY5NRdrWfjJrzRpqsPUv6v4Xj/ll4KleylqTubHT5cgsm4B0/fER9B0im5gw5jseh4IeHXT1djeB3ffIjTIywrYt+bgzrkzI4aD0LOLIoN2kFtxIXTM7oem+wKxSwSU/LBI+Jmw2lKXz2r5EkUpCjYUUftPjcT7q8qTpE5hcOvLOUoG4od9JKr5N1Hy4gsGLoYfK5RU19TSduZvQsCOya2G4m8JCGiYNuA14/VbCEzx0z3JCrO79wiUFRYU=
   - USE_HELPER=1
 before_install:
 - "./ci/prepare.sh"
@@ -35,12 +34,14 @@ install:
 - pip3 install coveralls
 - pip3 install requests
 - pip3 install black
+- pip3 install flake8
 before_script:
 - printf '\nimport coverage\ncoverage.current_coverage = coverage.process_startup()\n'
   >> "/home/travis/virtualenv/python${TRAVIS_PYTHON_VERSION}/lib/python${TRAVIS_PYTHON_VERSION}/sitecustomize.py"
 - rm -f .coverage-report.*
 script:
 - black --check $(git ls-files '*.py')
+- flake8 $(git ls-files '*.py')
 # Pre build the image. This prevents long delays with no output in test cases
 - ./ci/build_image.py $TEST_IMAGE --provider=$DOCKER_PROVIDER --quiet
 - COVERAGE_PROCESS_START=${TRAVIS_BUILD_DIR}/.coveragerc $SH -c "ci/test.py -vb PyrexImage_${DOCKER_PROVIDER}_$(echo $TEST_IMAGE | sed 's/\W/_/g')"

DEVELOPING.md

@@ -2,10 +2,11 @@
 Pyrex development information and processes
 
 ## Linting
-Pyrex conforms to [PEP8](https://pep8.org/) for all Python code, and also make
-use of [flake8](https://pypi.org/project/flake8/) as a linter. Please ensure
-all code conforms to this. There is helpful tool that will report any places
-where the code is non-conformant in the project root:
+Pyrex conforms to [black](https://black.readthedocs.io/en/stable/) for all
+Python code, and also makes use of [flake8](https://pypi.org/project/flake8/)
+as a linter. Please ensure all code conforms to this. There is helpful tool
+that will report any places where the code is non-conformant in the project
+root:
 
 ```shell
 ./lint
@@ -20,8 +21,8 @@ pass the `--reformat` option:
 
 ## Testing
 Pyrex has a comprehensive test suite that can be used to test all generated
-Docker images. Some external test data is required to perform the test. To
-download this data and prepare your sandbox for testing, run:
+images. Some external test data is required to perform the test. To download
+this data and prepare your sandbox for testing, run:
 
 ```shell
 ./ci/prepare.sh
@@ -40,36 +41,38 @@ available.
 The test suite will build all Pyrex image locally and run a set of tests
 against them. If you would like to limit which images are tested, additional
 arguments can be passed to filter the tests. Each image is tested using a test
-class named `PyrexImage_IMAGE` where `IMAGE` is the test image name will all
-non-word characters replaced by `_`.  For example, to test only the
-`ubuntu-18.04-oe` image, run:
+class named `PyrexImage_ENGINE_IMAGE` where `ENGINE` is the container engine
+(docker or podman) and `IMAGE` is the test image name will all non-word
+characters replaced by `_`.  For example, to test only the `ubuntu-18.04-oe`
+image under docker, run:
 
 ```shell
-./ci/test.py -vb PyrexImage_ubuntu_18_04_oe
+./ci/test.py -vb PyrexImage_docker_ubuntu_18_04_oe
 ```
 
 ## Building images locally
-Pyrex pulls prebuilt Docker containers from DockerHub by default, which should
-be sufficient for most users. However, users that are active developing on
-Pyrex or wish to build images locally instead of using published images can do
-so by making the following changes to `pyrex.ini`:
+Pyrex pulls prebuilt images from DockerHub by default, which should be
+sufficient for most users. However, users that are active developing on Pyrex
+or wish to build images locally instead of using published images can do so by
+making the following changes to `pyrex.ini`:
 
 1. Set `config:buildlocal` to `1`
-2. Change `config:tag` to an alternate tag suffix instead of
-   `:${config:pyrextag}`. While not strictly necessary, this step will help
-   prevent confusion if you want to switch back to prebuilt images. If you
-   choose not to change this, realize that your locally built images will
-   overwrite your local container cache tags for the prebuilt images. As an
-   example, you might add the following to `pyrex.ini`:
+2. Change `config:pyrextag` to an alternate tag instead of referencing the
+   released version.  While not strictly necessary, this step will help prevent
+   confusion if you want to switch back to prebuilt images. If you choose not
+   to change this, realize that your locally built images will overwrite your
+   local container cache tags for the prebuilt images. As an example, you might
+   add the following to `pyrex.ini`:
 
     ```
     [config]
-    tag = ${config:dockerimage}:my-image
+    pyrextag = my-image
     ```
 
-3. Set `config:dockerfile` to the path where the Dockerfile you wish to build
-   is located. Alternatively, you can leave it as the default to build the
-   standard Pyrex images locally.
+3. Set `imagebuild:buildcommand` to adjust any of the build options (e.g. the
+   path where the Dockerfile you wish to build is located).
+4. You may also want to set `imagebuild:quiet` to `false` if you want to see
+   the images being built for debugging purposes
 
 ## Making a release
 To make a release of Pyrex:

README.md

@@ -4,6 +4,60 @@ Containerize your bitbake
 [![Build Status](https://travis-ci.com/garmin/pyrex.svg?branch=master)](https://travis-ci.com/garmin/pyrex)
 [![Coverage Status](https://coveralls.io/repos/github/garmin/pyrex/badge.svg?branch=master)](https://coveralls.io/github/garmin/pyrex?branch=master)
 
+## Quickstart Guide (default layout)
+Use this quickstart guide if your project uses the default (poky-style) layout
+where bitbake and layers are subdirectories of oe-core:
+
+```shell
+# Clone down Pyrex
+git clone https://github.com/garmin/pyrex.git meta-pyrex
+
+# Create the pyrex environment initialization script symbolic link
+ln -s meta-pyrex/pyrex-init-build-env
+
+# Create a default pyrex.ini config file
+meta-pyrex/mkconfig > ./pyrex.ini
+
+# Set PYREXCONFFILE to the location of the newly created config file
+PYREXCONFFILE=./pyrex.ini
+
+# Initialize the build environment
+. pyrex-init-build-env
+
+# Everything is setup. OpenEmbedded build commands (e.g. `bitbake`) will now
+# run in Pyrex
+```
+
+## Quickstart Guide (alternate layout)
+Use this quickstart guide if your project uses a different layout where bitbake
+and oe-core are disjointed. In the example, it is assumed that oe-core and
+bitbake are both subdirectories of the current working directory, so you will
+need to change it to match your actual layout:
+
+```shell
+# Clone down Pyrex
+git clone https://github.com/garmin/pyrex.git pyrex
+
+# Create the pyrex environment initialization script symbolic link
+ln -s pyrex/pyrex-init-build-env
+
+# Create a default pyrex.ini config file
+pyrex/mkconfig > ./pyrex.ini
+
+# Set PYREXCONFFILE to the location of the newly created config file
+PYREXCONFFILE=./pyrex.ini
+
+# Tell Pyrex where bitbake and oe-core live
+BITBAKEDIR=$(pwd)/bitbake
+OEROOT=$(pwd)/oe-core
+
+# Initialize the build environment
+. pyrex-init-build-env
+
+# Everything is setup. OpenEmbedded build commands (e.g. `bitbake`) will now
+# run in Pyrex
+```
+
 ## What is Pyrex?
 At its core, Pyrex is an attempt to provided a consistent environment in which
 developers can run Yocto and bitbake commands. Pyrex is targeted at development
@@ -97,24 +151,22 @@ non-standard layout, you can write your own environment init script that tells
 look like:
 
 ```shell
-# The top level Yocto/OE directory (usually, poky). This variable *must* be
-# specified if writing a custom script.
-PYREX_OEROOT="$(pwd)"
+# Paths that should be bound into the container. If unspecified, defaults to
+# the parent directory of the sourced pyrex-init-build-env script, before
+# it is resolved as a symbolic link. You may need to override the default if
+# your bitbake directory, build directory, or any of your layer directories are
+# not children of the default (and thus, wouldn't be bound into the container).
+PYREX_CONFIG_BIND="$(pwd)"
 
 # The path to the build init script. If unspecified, defaults to
-# "${PYREX_OEROOT}/oe-init-build-env"
+# "$OEROOT/oe-init-build-env" or "$(pwd)/oe-init-build-env"
 PYREX_OEINIT="$(pwd)/oe-init-build-env"
 
 # The location of Pyrex itself. If not specified, pyrex-init-build-env will
 # assume it is the directory where it is currently located (which is probably
 # correct)
 PYREX_ROOT="$(pwd)/meta-pyrex"
 
-# The location of the pyrex.ini template file to use if the user doesn't
-# already have one. Defaults to "$TEMPLATECONF/pyrex.ini.sample" (the same
-# location that oe-init-build-env will look for local.conf.sample & friends)
-PYREXCONFTEMPLATE="$(pwd)/pyrex.ini.sample"
-
 # Alternatively, if it is desired to always use a fixed config file that users
 # can't change, set the following:
 #PYREXCONFFILE="$(pwd)/pyrex.ini"
@@ -124,26 +176,23 @@ PYREXCONFTEMPLATE="$(pwd)/pyrex.ini.sample"
 . $(pwd)/meta-pyrex/pyrex-init-build-env "$@"
 ```
 
+*NOTE: While it might be tempting to combine all of these into a one-liner like
+`PYREXCONFFILE="..." . $(pwd)/meta-pyrex/pyrex-init-build-env "$@"`, they must
+be specified on separate lines to remain compatible will all shells (i.e. bash
+in particular won't keep temporary variables specified in this way)*
+
 ### Configuration
-Pyrex is configured using a ini-style configuration file and uses the following
-precedence to determine where this file is located:
-1. The file specified in the environment variable `PYREXCONFFILE`.
-2. The `pyrex.ini` file in the bitbake conf directory (e.g.
-   `${OEROOT}/build/conf/pyrex.ini`, if it exists and has a version number
-   specified in `${config:confversion}`. For further rules, this file will
-   be known as `PYREX_USER_CONF`
-3. If the file specified in the environment variable `$PYREXCONFTEMPLATE` exists,
-   is copied to `PYREX_USER_CONF`, then `PYREX_USER_CONF` is used
-4. If the file `$TEMPLATECONF/pyrex.ini.sample` exists, it is copied to
-   `PYREX_USER_CONF`, the `PYREX_USER_CONF` is used. This is the same rules
-   that bitbake uses for `local.conf.sample`, allowing you to easily put a
-   `pyrex.conf.sample` file along side the other default config files. See
-   [TEMPLATECONF][].
-5. The internal default config file provided by Pyrex is coped to
-   `PYREX_USER_CONF`, then `PYREX_USER_CONF` is used.
-
-**Note:** The config file is only populated when Pyrex initializes the
-environment (e.g. when the init script is sourced).
+Pyrex is configured using a ini-style configuration file. The location of this
+file is specified by the `PYREXCONFFILE` environment variable. This environment
+variable must be set before the environment is initialized.
+
+If you do not yet have a config file, you can use the `mkconfig` command to use
+the default one and assign the `PYREXCONFFILE` variable in a single command
+like so:
+
+```shell
+$ export PYREXCONFFILE=`./meta-pyrex/mkconfig ./pyrex.ini`
+```
 
 The configuration file is the ini file format supported by Python's
 [configparser](https://docs.python.org/3/library/configparser.html) class, with
@@ -160,32 +209,42 @@ For more information about specific configuration values, see the default
 
 #### Binding directories into the container
 In order for bitbake running in the container to be able to build, it must have
-access to the data and config files from the host system. To make this easy, a
-variable called `run:bind` is specified in the config file. Any directory that
-appears in this variable will be bound into the container image at the same
-path (e.g. `/foo/bar` in the host will be bound to `/foo/bar` in the container
-engine. By default, only the Openembedded root directory (a.k.a.
-`$PYREX_OEROOT`, `${build:oeroot}`) is bound. This is the minimum that can be
-bound, and is generally sufficient for most use cases. If additional
-directories need to be accessed by the container image, they can be added to
-this list by the user. Common reasons for adding new paths include:
+access to the data and config files from the host system. There are two
+variables that can be set to specify what is bound into the container, the
+`PYREX_CONFIG_BIND` environment variable and the `run:bind` option specified in
+the config file. Both variables are a whitespace separated list of host paths
+that should be bound into the container at the same path (e.g. `/foo/bar` in
+the host will be bound to `/foo/bar` in the container engine).
+
+The `PYREX_CONFIG_BIND` environment variable is intended to specify the minimal
+set of bound directories required to initialize a default environment, and
+should only be set the by the environment initialization script, not by end
+users. The default value for this variable if unspecified is the parent of the
+sourced Pyrex initialization script. If the sourced script happens to be a
+symbolic link, the parent directory is determined before the symbolic link is
+resolved.
+
+The `run:bind` config file option is intended to allow users to specify
+additional paths that they want to bind. For convenience, the default value of
+this variable allows users to specify binds in the `PYREX_BIND` environment
+variable if they wish.
+
+Common reasons users might need to bind new paths include:
 * Alternate (out of tree) locations for sstate and download caches
 * Alternate (out of tree) build directories
-* Additional layers that are not under the OEROOT directory
+* Additional layers that are not under the default bind directories
 
-It is recommended to use this variable and bind directories in a 1-to-1 fashion
-rather than try to remap them to different paths inside the container image.
-Bitbake tends to encode file paths into some of its internal state (*Note*
-**Not** sstate, which should always be position independent), and remapping the
-paths might make it difficult to do builds outside of Pyrex if necessary.
+When the container environment is setup some basic sanity checks will be
+performed to makes sure that important directories like the bitbake and build
+directories are bound into the container.
 
 You should **never** map directories like `/usr/bin`, `/etc/`, `/` as these
 will probably just break the container. It is probably also unwise to map your
 entire home directory; although in some cases may be necessary to map
-$HOME/.ssh or other directories to access SSH keys and the like. For user
+`$HOME/.ssh` or other directories to access SSH keys and the like. For user
 convenience, the proxy user created in the container image by default has the
-same $HOME as the user who created the container, so these types of bind can be
-done by simply adding `${env:HOME}/.ssh` to `run:bind`
+same `$HOME` as the user who created the container, so these types of bind can
+be done by simply adding `${env:HOME}/.ssh` to `run:bind`
 
 #### Debugging the container
 In the event that you need to get a shell into the container to run some
@@ -204,15 +263,6 @@ Pyrex environment setup you created. This will setup up the current shell to
 run the commands listed in `${config:command}` inside of Pyrex. Once this is
 done, you can simply run those commands and they will be executed in Pyrex.
 
-### Bypassing Pyrex
-In some cases, it may be desirable to bypass Pyrex and run the commands it
-wraps locally instead of in the container. This can be done in one of two ways:
-
-1. Set `${run:enable}` to `0` in `pyrex.ini` which will disable using the
-   container engine for all commands
-2. Set the environment variable `PYREX_DOCKER` to `0`. Any Pyrex commands run
-   with this variable will not be run in the container.
-
 ## What doesn't work?
 The following items are either known to not work, or haven't been fully tested:
 * **Bitbake Server** Since the container starts and stops each time a command
@@ -236,6 +286,10 @@ The following items are either known to not work, or haven't been fully tested:
   from causing bad behaviors. It is still possible to pause the container using
   the `docker pause` command, but this doesn't integrate with the parent shells
   job control.
+* **Rootless Docker** This in untested, and probably doesn't work. It shouldn't
+  be too hard since this should be very similar to how `podman` works.
+  Currently however, it is assumed that if the container engine is `docker` it
+  is running as root and if it is `podman` it is running rootless.
 
 ## Developing on Pyrex
  If you are doing development on Pyrex itself, please read the [Developer
@@ -254,7 +308,7 @@ build the image locally instead. See the [Developer Documentation][].
   chosen because it is one of the [Sanity Tested Distros][] that Yocto
   supports. Pyrex aims to support a vanilla Yocto setup with minimal manual
   configuration.
-* *What's with [cleanup.py](./docker/cleanup.py)?* When a container's main
+* *What's with [cleanup.py](./image/cleanup.py)?* When a container's main
   process exits, any remaining process appears to be sent a `SIGKILL`.  This
   can cause a significant problem with many of the child processes that bitbake
   spawns, since unceremoniously killing them might result in lost data.  The

ci/build_image.py

@@ -37,8 +37,8 @@ def main():
     (_, _, image_type) = args.image.split("-")
 
     this_dir = os.path.abspath(os.path.dirname(__file__))
-    docker_dir = os.path.join(this_dir, "..", "docker")
-    docker_file = os.path.join(docker_dir, "Dockerfile")
+    image_dir = os.path.join(this_dir, "..", "image")
+    docker_file = os.path.join(image_dir, "Dockerfile")
 
     helper = os.path.join(THIS_DIR, "%s-helper.py" % args.provider)
     if os.path.exists(helper) and os.environ.get("USE_HELPER", "0") == "1":
@@ -47,7 +47,7 @@ def main():
     else:
         provider = args.provider
 
-    docker_args = [
+    engine_args = [
         provider,
         "build",
         "-t",
@@ -57,14 +57,14 @@ def main():
         "--network=host",
         "--build-arg",
         "PYREX_BASE=%s" % args.image,
-        docker_dir,
+        image_dir,
         "--target",
         "pyrex-%s" % image_type,
     ]
 
     if args.quiet:
         p = subprocess.Popen(
-            docker_args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT
+            engine_args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT
         )
         while True:
             try:
@@ -80,7 +80,7 @@ def main():
 
         return p.returncode
     else:
-        subprocess.check_call(docker_args)
+        subprocess.check_call(engine_args)
 
 
 if __name__ == "__main__":