Merge pull request #230 from mozilla/support-gcs

Add support for writing to GCS

Author: relud

.circleci/config.yml

@@ -0,0 +1,68 @@
+version: 2.1
+orbs:
+  gcp-gcr: circleci/[email protected]
+jobs:
+  build:
+    docker: &docker
+      - image: cimg/node:10.18.1
+    steps:
+      - checkout
+      - &setup_docker_with_caching
+        setup_remote_docker:
+          docker_layer_caching: true
+      - run:
+          name: Build Image
+          command: |
+            make build
+  test:
+    docker: *docker
+    steps:
+      - checkout
+      - *setup_docker_with_caching
+      - run:
+          name: Run unit tests
+          command: |
+            touch .env
+            make test
+  lint:
+    docker: *docker
+    steps:
+      - checkout
+      - *setup_docker_with_caching
+      - run:
+          name: Check code formatting
+          command: |
+            make lint
+  deploy:
+    docker: *docker
+    steps:
+      - checkout
+      - *setup_docker_with_caching
+      - gcp-gcr/gcr-auth
+      - gcp-gcr/build-image: &private-image
+          image: ensemble-transposer
+          tag: ${CIRCLE_TAG:-latest}
+      - gcp-gcr/push-image: *private-image
+workflows:
+  version: 2
+  ci_workflow:
+    jobs: &build_jobs
+      - build:
+          context: data-eng-circleci-tests
+      - test:
+          requires:
+            - build
+      - lint:
+          requires:
+            - build
+      - deploy:
+          context: data-eng-airflow-gcr
+          filters:
+            branches:
+              only: main
+            tags:
+              only: /.*/
+          requires:
+            - build
+            - test
+            - lint

.gitignore

@@ -8,3 +8,6 @@
 node_modules
 npm-debug.log*
 *.log
+
+# output
+target

README.md

@@ -68,7 +68,9 @@ code, for example.
 
 ## Deployment
 
-This project is meant to be run as a cloud task, like a Lambda function or
+### AWS
+
+This project was originally meant to be run as a cloud task, like a Lambda function or
 Google Cloud Function. The main function is specified as the value of `main` in
 *package.json*. Most services read this value and do the right thing. If not,
 you may need to manually point your service to that function.
@@ -82,6 +84,20 @@ variables:
 * `AWS_ACCESS_KEY_ID`
 * `AWS_SECRET_ACCESS_KEY`
 
+### Google Cloud
+
+This project can be run as a Docker container. The default command is `npm start`, but it may need
+to be explicitly configured in some environments. When running the container in GKE, authentication
+will be automatically detected. Before running, be sure to create a [Google Cloud
+Storage](https://cloud.google.com/storage) bucket and set the following environment variable:
+
+* `GCS_BUCKET_NAME`
+
+### Other
+
+When neither `AWS_BUCKET_NAME` nor `GCS_BUCKET_NAME` are present in the environment, this project
+will write data to `./target`, which can then be copied to otherwise unsupported systems.
+
 ## Notes
 
 ### Versioning