From 14ea3c77e41eb18a2ccaf1a7435872f954017dab Mon Sep 17 00:00:00 2001 From: Greg Guthe Date: Thu, 30 Jul 2020 16:24:26 -0400 Subject: [PATCH] Fix 302 containerize (#310) * add Dockerfile * make: add build-image * ci: add circle config to push images * update readme to use docker image --- .circleci/config.yml | 90 +++++++++++++++++++++++++++++++++++ .dockerignore | 109 +++++++++++++++++++++++++++++++++++++++++++ Dockerfile | 29 ++++++++++++ Makefile | 4 ++ README.md | 56 +++------------------- 5 files changed, 239 insertions(+), 49 deletions(-) create mode 100644 .circleci/config.yml create mode 100644 .dockerignore create mode 100644 Dockerfile diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 0000000..ceb0c85 --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,90 @@ +# These environment variables must be set in CircleCI UI +# +# DOCKERHUB_REPO - docker hub repo, format: / +# DOCKER_USER +# DOCKER_PASS +# +version: 2.1 +jobs: + test: + docker: + - image: circleci/python:buster + steps: + - checkout + - run: + name: install python deps in venv + command: | + make install + - run: + name: run tests + command: | + make black check_conftest_imports test coverage + + build-image: + docker: + - image: circleci/python:buster + steps: + - checkout + - setup_remote_docker + - run: + name: build-image + command: make build-image + - run: + name: save built image to cache + command: docker save "localhost/frost:latest" | gzip -c > /tmp/docker.tgz + - save_cache: + key: v1-{{ .Branch }}-{{ epoch }} + paths: + - /tmp/docker.tgz + + deploy: + docker: + - image: circleci/python:buster + steps: + - checkout + - setup_remote_docker + - restore_cache: + key: v1-{{.Branch}} + - run: + name: Restore Docker image cache + command: gunzip -c /tmp/docker.tgz | docker load + - run: + name: deploy to Dockerhub + command: | + # deploy master and scan envs + if [ "${CIRCLE_BRANCH}" == "master" ]; then + docker login -u $DOCKER_USER -p $DOCKER_PASS + docker tag "localhost/frost:latest" "${DOCKERHUB_REPO}:latest" + docker push "${DOCKERHUB_REPO}:latest" + elif [ ! -z "${CIRCLE_TAG}" ]; then + # deploy a release tag + docker login -u $DOCKER_USER -p $DOCKER_PASS + echo "${DOCKERHUB_REPO}:${CIRCLE_TAG}" + docker tag "localhost/frost:latest" "${DOCKERHUB_REPO}:${CIRCLE_TAG}" + docker push "${DOCKERHUB_REPO}:${CIRCLE_TAG}" + fi + +workflows: + version: 2 + check-readme-local-dev: + jobs: + - test: + filters: + tags: + only: /.*/ + + - build-image: + filters: + tags: + only: /.*/ + + - deploy: + requires: + - test + - build-image + filters: + tags: + # only upload the docker container on semver tags + only: /[0-9]\.[0-9]+\.[0-9]+/ + branches: + only: master diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..13fd536 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,109 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + + +# C extensions +*.so + +# Distribution / packaging +.Python +env/ +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +*.egg-info/ +.installed.cfg +*.egg + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +.hypothesis/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# pyenv +.python-version + +# celery beat schedule file +celerybeat-schedule + +# SageMath parsed files +*.sage.py + +# dotenv +.env + +# virtualenv +.venv +venv/ +ENV/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ + +# VCS +.git + +# CI +.circleci +.travis.yml diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..76aeb42 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,29 @@ +# frost + +FROM python:3.8-slim-buster + +ENV PYTHONPATH $PYTHONPATH:/app +ENV PYTHONUNBUFFERED 1 + +RUN groupadd --gid 10001 app && \ + useradd --uid 10001 --gid 10001 --shell /usr/sbin/nologin app +RUN install -o app -g app -d /var/run/depobs /var/log/depobs + +# git for herokuadmintools +RUN DEBIAN_FRONTEND=noninteractive apt-get update && \ + apt-get upgrade -y && \ + apt-get install --no-install-recommends -y \ + ca-certificates \ + curl \ + git \ + jq + +WORKDIR /app + +COPY requirements.txt . +RUN pip install --upgrade --no-cache-dir -r requirements.txt + +COPY * . + +USER app +ENTRYPOINT [ ] diff --git a/Makefile b/Makefile index 3743fc4..26a87b9 100644 --- a/Makefile +++ b/Makefile @@ -78,8 +78,12 @@ venv: python3 -m venv venv ./venv/bin/python -V | grep $(PYTHON_MIN_VERSION) || true; $(PYTHON_VER_WARNING) +build-image: + docker build -t localhost/frost:latest . + .PHONY: all \ + build-image \ check_venv \ check_conftest_imports \ clean \ diff --git a/README.md b/README.md index ba53dba..36c8c62 100644 --- a/README.md +++ b/README.md @@ -15,30 +15,18 @@ want to answer questions whether they are configured properly such as: ### Requirements -* [GNU Make 3.81](https://www.gnu.org/software/make/) -* [Python 3.6.2](https://www.python.org/downloads/) - -Note: other versions may work too these are the versions @g-k used for development +* [docker](https://docs.docker.com/get-docker/) ### Installing -From the project root run: - ```console -make install +docker pull mozilla/frost ``` -This will: - -* create a Python [virtualenv](https://docs.python.org/3/library/venv.html) to isolate it from other Python packages -* install Python requirements in the virtualenv - ### Running -Activate the venv in the project root: - ```console -source venv/bin/activate +docker run --rm mozilla/frost pytest -h ``` To fetch RDS resources from the cache or AWS API and check that @@ -47,7 +35,7 @@ profile](https://docs.aws.amazon.com/cli/latest/userguide/cli-config-files.html) named `default` in the `us-west-2` region we can run: ```console -pytest --ignore aws/s3 --ignore aws/ec2 -k test_rds_db_instance_backup_enabled -s --aws-profiles default --debug-calls +docker run --rm mozilla/frost pytest --ignore gsuite/ --ignore heroku/ --ignore pagerduty/ --ignore gcp/ --ignore aws/s3 --ignore aws/ec2 -k test_rds_db_instance_backup_enabled -s --aws-profiles default --debug-calls ``` The options include pytest options: @@ -67,37 +55,7 @@ and options frost adds: and produces output like the following showing a DB instance with backups disabled: ```console -=========================================================== test session starts =========================================================== -platform darwin -- Python 3.6.2, pytest-3.3.2, py-1.5.2, pluggy-0.6.0 -metadata: {'Python': '3.6.2', 'Platform': 'Darwin-15.6.0-x86_64-i386-64bit', 'Packages': {'pytest': '3.3.2', 'py': '1.5.2', 'pluggy': '0.6. -0'}, 'Plugins': {'metadata': '1.5.1', 'json': '0.4.0', 'html': '1.16.1'}} -rootdir: /Users/gguthe/mozilla/frost, inifile: -plugins: metadata-1.5.1, json-0.4.0, html-1.16.1 -collecting 0 items c -alling AWSAPICall(profile='default', region='us-west-2', service='rds', method='describe_db_instances', args=[], kwargs={}) -collecting 4 items -... -aws/rds/test_rds_db_instance_backup_enabled.py ...F [100%] - -================================================================ FAILURES ================================================================= -_______________________________________ test_rds_db_instance_backup_enabled[test-db] ________________________________________ - -rds_db_instance = {'AllocatedStorage': 50, 'AutoMinorVersionUpgrade': True, 'AvailabilityZone': 'us-west-2c', 'BackupRetentionPeriod': 0, . -..} - - @pytest.mark.rds - @pytest.mark.parametrize('rds_db_instance', - rds_db_instances(), - ids=lambda db_instance: db_instance['DBInstanceIdentifier']) - def test_rds_db_instance_backup_enabled(rds_db_instance): -> assert rds_db_instance['BackupRetentionPeriod'] > 0, \ - 'Backups disabled for {}'.format(rds_db_instance['DBInstanceIdentifier']) -E AssertionError: Backups disabled for test-db -E assert 0 > 0 - -aws/rds/test_rds_db_instance_backup_enabled.py:12: AssertionError -=========================================================== 72 tests deselected =========================================================== -============================================ 1 failed, 3 passed, 72 deselected in 3.12 seconds ============================================ +# TODO: add example output back ``` #### IAM Policy for frost @@ -361,7 +319,7 @@ And results in a severity and severity marker being included in the json metadata: ```console -pytest --ignore aws/s3 --ignore aws/rds --ignore aws/iam -s --aws-profiles stage --aws-require-tags Name Type App Stack -k test_ec2_instance_has_required_tags --config config.yaml.example --json=report.json +docker run --rm mozilla/frost pytest --ignore aws/s3 --ignore aws/rds --ignore aws/iam -s --aws-profiles stage --aws-require-tags Name Type App Stack -k test_ec2_instance_has_required_tags --config config.yaml.example --json=report.json ... ``` @@ -688,7 +646,7 @@ Notes: 1. Running it we see that one of the IPs is an AWS IP: ```console -pytest --ignore aws/ +docker run --rm mozilla/frost pytest --ignore aws/ --ignore gsuite/ --ignore heroku/ --ignore pagerduty/ --ignore gcp/ platform darwin -- Python 3.6.2, pytest-3.3.2, py-1.5.2, pluggy-0.6.0 metadata: {'Python': '3.6.2', 'Platform': 'Darwin-15.6.0-x86_64-i386-64bit', 'Packages': {'pytest': '3.3.2', 'py': '1.5.2', 'pluggy': '0.6.0'}, 'Plugins': {'metadata': '1.5.1', 'json': '0.4.0', 'html': '1.16.1'}} rootdir: /Users/gguthe/mozilla/frost, inifile: