Merge pull request #52 from planetlabs/unify

unify all of the datalake parts here in the one datalake repo to rule them all

Author: bcavagnolo

.gitignore

@@ -14,7 +14,6 @@ dist/
 downloads/
 eggs/
 .eggs/
-lib/
 lib64/
 parts/
 sdist/
@@ -55,7 +54,4 @@ docs/_build/
 
 # PyBuilder
 target/
-
-# Vim
-*.swp
-*.un~
+.vagrant

.travis.yml

@@ -1,30 +1,42 @@
-language: python
-python:
-- '2.7'
-- '3.5'
+services: docker
 
-env:
-  matrix:
-    - EXTRAS='[test]'
-    - EXTRAS='[test,queuable,sentry,krb]'
-  global:
-    # see https://github.com/travis-ci/travis-ci/issues/7940
-    - BOTO_CONFIG=/dev/null
+_python_template: &_python_template
+  language: python
+  stage: test
+  before_install: cd client/
+  install: pip install -e .[test]
+  script: py.test
+  deploy:
+    provider: pypi
+    user: planet
+    password:
+      secure: EgEXpqXu2Iyi7nFuZrFmWBavvOil5r8ctqHGQ6BT2J5+qqE/K1OXblt/l0IcVYryoFTOAyB36Dti4+zgeGmcA+UrvKZyyIT8z0zWyIS/iLWhjiXDJdrDU0zVWXRVC5/AsgsxUkJMVVCke/dxUpIbLNuVV2jwUd3k0VGw+q2XR7Q=
+    distributions: "sdist bdist_wheel"
+    skip_existing: true
+    on:
+      branch: master
+      tags: true
+      repo: planetlabs/datalake
 
-install:
-  - pip install -e .${EXTRAS}
-before_script:
-  - flake8 .
-script:
-- py.test
-deploy:
-  provider: pypi
-  user: planet
-  password:
-    secure: EgEXpqXu2Iyi7nFuZrFmWBavvOil5r8ctqHGQ6BT2J5+qqE/K1OXblt/l0IcVYryoFTOAyB36Dti4+zgeGmcA+UrvKZyyIT8z0zWyIS/iLWhjiXDJdrDU0zVWXRVC5/AsgsxUkJMVVCke/dxUpIbLNuVV2jwUd3k0VGw+q2XR7Q=
-  distributions: "sdist bdist_wheel"
-  skip_existing: true
-  on:
-    branch: master
-    tags: true
-    repo: planetlabs/datalake
+jobs:
+  include:
+  - <<: *_python_template
+    name: py27 bare
+    python: 2.7
+    before_script: flake8 .
+  - <<: *_python_template
+    name: py35 bare
+    python: 3.5
+  - <<: *_python_template
+    name: py27 with extras
+    python: 2.7
+    install: pip install -e .[test,queuable,sentry,krb]
+  - <<: *_python_template
+    name: py35 with extras
+    python: 3.5
+    install: pip install -e .[test,queuable,sentry,krb]
+  - dist: bionic
+    name: docker
+    stage: test
+    script:
+      - make test

Dockerfile

@@ -0,0 +1,58 @@
+FROM python:2.7-slim-buster
+
+MAINTAINER brian <[email protected]>
+
+ENV LANG C.UTF-8
+ENV	LC_ALL C.UTF-8
+
+# NB: gcc is only required to build the pyblake package, which does not ship as
+# a wheel for linux. Once we move to python3 we can eliminate this dependency
+# because hashlib supports blake2.
+RUN apt-get update && \
+    apt-get install --quiet --yes \
+	gcc \
+    && rm -rf /var/lib/apt/lists/* \
+    && apt-get clean
+
+# TODO: keep requirements in one place
+RUN pip install \
+    blinker>=1.4 \
+    boto>=2.38.0 \
+    boto3>=1.1.3 \
+    click>=5.1 \
+    Flask>=0.10.1 \
+    flask-swagger==0.2.8 \
+    memoized_property>=1.0.1 \
+    pyblake2>=0.9.3 \
+    python-dateutil>=2.4.2 \
+    python-dotenv>=0.1.3 \
+    pytz>=2015.4 \
+    raven[flask]>=5.6.0 \
+    requests>=2.5 \
+    simplejson>=3.3.1 \
+    six>=1.10.0 \
+    # test requirements
+    flake8==2.5.0 \
+    freezegun==0.3.9 \
+    moto==0.4.27 \
+    pytest==3.0.2 \
+    responses==0.5.0
+
+RUN mkdir -p /opt/
+COPY . /opt/
+
+# Take care to install clients such that the source code can be mounted into
+# the container and used for development. That is, the python paths and paths
+# to console scripts Just Work (TM)
+ENV PYTHONPATH=/opt/common:/opt/client:/opt/ingester:/opt/api
+RUN cd /opt/client && \
+    python setup.py develop -s /usr/local/bin \
+        --egg-path ../../../../../opt/client/ \
+        -d /usr/local/lib/python2.7/site-packages/ \
+        --no-deps;
+
+ARG VERSION=unspecified
+ENV VERSION=$VERSION
+
+WORKDIR /opt
+ENTRYPOINT ["/usr/local/bin/datalake"]

Makefile

@@ -0,0 +1,21 @@
+VERSION=$(shell git describe --tags --dirty)
+
+.PHONY: docker # build the docker container
+docker:
+	docker build --build-arg VERSION=$(VERSION) -t datalake:$(VERSION) .
+
+.PHONY: devshell  # Open a developer shell in the docker env
+devshell: docker
+	docker run --rm -it -v $$PWD:/opt --entrypoint /bin/bash datalake:$(VERSION)
+
+.PHONY: test  # Run the tests
+
+test: docker
+	echo VERSION=$(VERSION)
+	for p in common client ingester api; do \
+		docker run --rm -it --entrypoint py.test datalake:$(VERSION) $$p; \
+	done
+
+.PHONY: help  # Generate list of targets with descriptions
+help:
+	@grep '^.PHONY: .* #' Makefile | sed 's/\.PHONY: \(.*\) # \(.*\)/  \1: \2/' | expand -t20

api/.gitignore

@@ -0,0 +1,57 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+env/
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib64/
+parts/
+sdist/
+var/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*,cover
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+.vagrant

api/.travis.yml

@@ -0,0 +1,17 @@
+language: python
+python:
+- '2.7'
+install:
+- pip install -e .[test]
+before_script:
+  - flake8 .
+script:
+- py.test
+deploy:
+  provider: pypi
+  user: planet
+  password:
+    secure: moVRXcjStPtmCqW9N0057EY20JhhPAwIh7JbFN8wcx6eYCzN6M8NCErTlEL8fkl5Wx7QreRMHuZo91lf5nDPQfaNieDyRFxRfg/s5oQZePve6etRxxDphSwmlldtRI0Q6osB4muuXsnXx/A72G2PgtgV6f3FdSv3phXMZKEKZfXKU26TYcvjzygK3afXSsv8T4m8LjSnv/nONwYENZYwyZNiT6/gzSQ5TGYpIw0zU4bhMe1ZX54CbRPgzudbe3phV4NpEAbhVWPfZOH0d0iYCjNm/VKPG83Kf2jLT5sD0VzsULfxaxdtPycXjGlPHMzoGFQWH8x8FcnEnTgcD9irATRgQgQQQHDyYjkoAFxymDYglZolWZpswlPK6/D624GzZD19MxWBt2s6UvXDSKupDX/yxNxNwZa3VtXU1/ylfC5IHP9+OMhANKgtw0k4pUNnj1fI5TXQae1EDtLkOxqUfCSIeJr+AZmIFGvqUbQG05G5SGloGv9eGEjoamnUfgU9SM06n0fbVJvXJ5aHwYQ3DUOnfkMLXrKtuRn7qfzTT3hpRbPTQZxM53zMmLfgP7XwzhDT2UE+FaNSXVofFhPq4UP26OycVfYE5y2mt8hAAa/88qmmgrUzakmDJ7lVL/GQx43RynrlaoKAJa+zIChbJP8QWZKR3LFaJZPB9+doKXg=
+  on:
+    tags: true
+    repo: planetlabs/datalake-api