Fix 302 containerize (#310)

* add Dockerfile * make: add build-image * ci: add circle config to push images * update readme to use docker image
mozilla · Jul 30, 2020 · 14ea3c7 · 14ea3c7
1 parent fa8a1ce
commit 14ea3c7
Show file tree

Hide file tree

Showing 5 changed files with 239 additions and 49 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -0,0 +1,90 @@
+# These environment variables must be set in CircleCI UI
+#
+# DOCKERHUB_REPO - docker hub repo, format: <username>/<repo>
+# DOCKER_USER
+# DOCKER_PASS
+#
+version: 2.1
+jobs:
+  test:
+    docker:
+      - image: circleci/python:buster
+    steps:
+      - checkout
+      - run:
+          name: install python deps in venv
+          command: |
+            make install
+      - run:
+          name: run tests
+          command: |
+            make black check_conftest_imports test coverage
+
+  build-image:
+    docker:
+      - image: circleci/python:buster
+    steps:
+      - checkout
+      - setup_remote_docker
+      - run:
+          name: build-image
+          command: make build-image
+      - run:
+          name: save built image to cache
+          command: docker save "localhost/frost:latest" | gzip -c > /tmp/docker.tgz
+      - save_cache:
+          key: v1-{{ .Branch }}-{{ epoch }}
+          paths:
+            - /tmp/docker.tgz
+
+  deploy:
+    docker:
+      - image: circleci/python:buster
+    steps:
+      - checkout
+      - setup_remote_docker
+      - restore_cache:
+          key: v1-{{.Branch}}
+      - run:
+          name: Restore Docker image cache
+          command: gunzip -c /tmp/docker.tgz | docker load
+      - run:
+          name: deploy to Dockerhub
+          command: |
+            # deploy master and scan envs
+            if [ "${CIRCLE_BRANCH}" == "master" ]; then
+              docker login -u $DOCKER_USER -p $DOCKER_PASS
+              docker tag "localhost/frost:latest" "${DOCKERHUB_REPO}:latest"
+              docker push "${DOCKERHUB_REPO}:latest"
+            elif  [ ! -z "${CIRCLE_TAG}" ]; then
+              # deploy a release tag
+              docker login -u $DOCKER_USER -p $DOCKER_PASS
+              echo "${DOCKERHUB_REPO}:${CIRCLE_TAG}"
+              docker tag "localhost/frost:latest" "${DOCKERHUB_REPO}:${CIRCLE_TAG}"
+              docker push "${DOCKERHUB_REPO}:${CIRCLE_TAG}"
+            fi
+
+workflows:
+  version: 2
+  check-readme-local-dev:
+    jobs:
+      - test:
+        filters:
+          tags:
+            only: /.*/
+
+      - build-image:
+          filters:
+            tags:
+              only: /.*/
+
+      - deploy:
+          requires:
+            - test
+            - build-image
+          filters:
+            tags:
+              # only upload the docker container on semver tags
+              only: /[0-9]\.[0-9]+\.[0-9]+/
+            branches:
+              only: master
diff --git a/.dockerignore b/.dockerignore
@@ -0,0 +1,109 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+env/
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+.hypothesis/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# pyenv
+.python-version
+
+# celery beat schedule file
+celerybeat-schedule
+
+# SageMath parsed files
+*.sage.py
+
+# dotenv
+.env
+
+# virtualenv
+.venv
+venv/
+ENV/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+
+# VCS
+.git
+
+# CI
+.circleci
+.travis.yml
diff --git a/Dockerfile b/Dockerfile
@@ -0,0 +1,29 @@
+# frost
+
+FROM python:3.8-slim-buster
+
+ENV PYTHONPATH $PYTHONPATH:/app
+ENV PYTHONUNBUFFERED 1
+
+RUN groupadd --gid 10001 app && \
+    useradd --uid 10001 --gid 10001 --shell /usr/sbin/nologin app
+RUN install -o app -g app -d /var/run/depobs /var/log/depobs
+
+# git for herokuadmintools
+RUN DEBIAN_FRONTEND=noninteractive apt-get update && \
+        apt-get upgrade -y && \
+        apt-get install --no-install-recommends -y \
+            ca-certificates \
+            curl \
+            git \
+            jq
+
+WORKDIR /app
+
+COPY requirements.txt .
+RUN pip install --upgrade --no-cache-dir -r requirements.txt
+
+COPY * .
+
+USER app
+ENTRYPOINT [ ]
diff --git a/Makefile b/Makefile
@@ -78,8 +78,12 @@ venv:
 	python3 -m venv venv
 	./venv/bin/python -V | grep $(PYTHON_MIN_VERSION) || true; $(PYTHON_VER_WARNING)
 
+build-image:
+	docker build -t localhost/frost:latest .
+
 .PHONY:
 	all \
+	build-image \
 	check_venv \
 	check_conftest_imports \
 	clean \

diff --git a/README.md b/README.md
@@ -15,30 +15,18 @@ want to answer questions whether they are configured properly such as:
 
 ### Requirements
 
-* [GNU Make 3.81](https://www.gnu.org/software/make/)
-* [Python 3.6.2](https://www.python.org/downloads/)
-
-Note: other versions may work too these are the versions @g-k used for development
+* [docker](https://docs.docker.com/get-docker/)
 
 ### Installing
 
-From the project root run:
-
 ```console
-make install
+docker pull mozilla/frost
 ```
 
-This will:
-
-* create a Python [virtualenv](https://docs.python.org/3/library/venv.html) to isolate it from other Python packages
-* install Python requirements in the virtualenv
-
 ### Running
 
-Activate the venv in the project root:
-
 ```console
-source venv/bin/activate
+docker run --rm mozilla/frost pytest -h
 ```
 
 To fetch RDS resources from the cache or AWS API and check that
@@ -47,7 +35,7 @@ profile](https://docs.aws.amazon.com/cli/latest/userguide/cli-config-files.html)
 named `default` in the `us-west-2` region we can run:
 
 ```console
-pytest --ignore aws/s3 --ignore aws/ec2 -k test_rds_db_instance_backup_enabled -s --aws-profiles default --debug-calls
+docker run --rm mozilla/frost pytest --ignore gsuite/ --ignore heroku/ --ignore pagerduty/ --ignore gcp/ --ignore aws/s3 --ignore aws/ec2 -k test_rds_db_instance_backup_enabled -s --aws-profiles default --debug-calls
 ```
 
 The options include pytest options:
@@ -67,37 +55,7 @@ and options frost adds:
 and produces output like the following showing a DB instance with backups disabled:
 
 ```console
-=========================================================== test session starts ===========================================================
-platform darwin -- Python 3.6.2, pytest-3.3.2, py-1.5.2, pluggy-0.6.0
-metadata: {'Python': '3.6.2', 'Platform': 'Darwin-15.6.0-x86_64-i386-64bit', 'Packages': {'pytest': '3.3.2', 'py': '1.5.2', 'pluggy': '0.6.
-0'}, 'Plugins': {'metadata': '1.5.1', 'json': '0.4.0', 'html': '1.16.1'}}
-rootdir: /Users/gguthe/mozilla/frost, inifile:
-plugins: metadata-1.5.1, json-0.4.0, html-1.16.1
-collecting 0 items                                                                                                                        c
-alling AWSAPICall(profile='default', region='us-west-2', service='rds', method='describe_db_instances', args=[], kwargs={})
-collecting 4 items
-...
-aws/rds/test_rds_db_instance_backup_enabled.py ...F                                                                                 [100%]
-
-================================================================ FAILURES =================================================================
-_______________________________________ test_rds_db_instance_backup_enabled[test-db] ________________________________________
-
-rds_db_instance = {'AllocatedStorage': 50, 'AutoMinorVersionUpgrade': True, 'AvailabilityZone': 'us-west-2c', 'BackupRetentionPeriod': 0, .
-..}
-
-    @pytest.mark.rds
-    @pytest.mark.parametrize('rds_db_instance',
-                             rds_db_instances(),
-                             ids=lambda db_instance: db_instance['DBInstanceIdentifier'])
-    def test_rds_db_instance_backup_enabled(rds_db_instance):
->       assert rds_db_instance['BackupRetentionPeriod'] > 0, \
-            'Backups disabled for {}'.format(rds_db_instance['DBInstanceIdentifier'])
-E       AssertionError: Backups disabled for test-db
-E       assert 0 > 0
-
-aws/rds/test_rds_db_instance_backup_enabled.py:12: AssertionError
-=========================================================== 72 tests deselected ===========================================================
-============================================ 1 failed, 3 passed, 72 deselected in 3.12 seconds ============================================
+# TODO: add example output back
 ```
 
 #### IAM Policy for frost
@@ -361,7 +319,7 @@ And results in a severity and severity marker being included in the
 json metadata:
 
 ```console
-pytest --ignore aws/s3 --ignore aws/rds --ignore aws/iam -s --aws-profiles stage --aws-require-tags Name Type App Stack -k test_ec2_instance_has_required_tags --config config.yaml.example --json=report.json
+docker run --rm mozilla/frost pytest --ignore aws/s3 --ignore aws/rds --ignore aws/iam -s --aws-profiles stage --aws-require-tags Name Type App Stack -k test_ec2_instance_has_required_tags --config config.yaml.example --json=report.json
 ...
 ```
 
@@ -688,7 +646,7 @@ Notes:
 1. Running it we see that one of the IPs is an AWS IP:
 
 ```console
-pytest --ignore aws/
+docker run --rm mozilla/frost pytest --ignore aws/ --ignore gsuite/ --ignore heroku/ --ignore pagerduty/ --ignore gcp/
 platform darwin -- Python 3.6.2, pytest-3.3.2, py-1.5.2, pluggy-0.6.0
 metadata: {'Python': '3.6.2', 'Platform': 'Darwin-15.6.0-x86_64-i386-64bit', 'Packages': {'pytest': '3.3.2', 'py': '1.5.2', 'pluggy': '0.6.0'}, 'Plugins': {'metadata': '1.5.1', 'json': '0.4.0', 'html': '1.16.1'}}
 rootdir: /Users/gguthe/mozilla/frost, inifile: