Merge branch 'wb-docker'

* wb-docker: (27 commits)
  fixed celerybeat issues
  included missing dependencies
  no message
  include gunicorn update print for python 3
  no message
  update to using django 3 and
  updated docker files
  no message
  remove django superviser
  removed unnecessary asynchronous task
  correctly install dotenv
  update backend
  inserted notes for docker
  insert get environmental variable method in init
  update settings configurations and added celery to docker
  reinserted backend
  compile front end
  implemented dockerfile for frontend
  updates to docker-compose
  add: editorconfig
  ...

# Conflicts:
#	app/models.py
#	app/tasks.py
#	news/models.py
#	requirements.txt
#	settings/common.py
#	settings/dev.py
#	utils/log.py

.editorconfig

@@ -0,0 +1,19 @@
+# EditorConfig: https://EditorConfig.org
+
+# top-most EditorConfig file
+root = true
+
+# Unix-style newlines with a newline ending every file
+[*]
+end_of_line = lf
+insert_final_newline = true
+
+# Matches multiple files with brace expansion notation
+# Set default charset
+[*.{js,ts}]
+charset = utf-8
+
+# Indentation override for all JS under lib directory
+[**.js]
+indent_style = space
+indent_size = 2

Dockerfile

@@ -0,0 +1,21 @@
+# pull official base image
+FROM python:3.6
+
+# to the terminal with out buffering it first
+ENV PYTHONUNBUFFERED 1
+
+# prevents python from writing pyc files to disk
+ENV PYTHONDONTWRITEBYTECODE 1
+
+# create root directory for our project in the container
+RUN mkdir /app
+
+# Set the working directory to /app
+WORKDIR /app
+COPY . /app
+
+# Install any needed packages specified in requirements.txt
+RUN pip install pathlib
+RUN pip install -r /app/requirements.txt
+RUN apt-get update
+RUN apt-get install -y binutils libproj-dev gdal-bin

app/models.py

@@ -1,3 +1,4 @@
+from __future__ import unicode_literals
 from django.conf import settings
 from datetime import datetime
 import uuid
@@ -9,11 +10,6 @@
 from localflavor.us.us_states import STATE_CHOICES
 from localflavor.us.models import USStateField
 from django_pandas.managers import DataFrameManager
-from rawdata import models as raw_models
-
-from app import choice
-
-from app import const
 
 # Create your models here.
 class location(models.Model):
@@ -28,14 +24,13 @@ class location(models.Model):
 	neighborhood = models.CharField(max_length=100, null=True, blank=True,default='')
 	notes = models.TextField(null=True, blank=True,default='')
 	population_served = models.IntegerField( blank=True, null=True, default = 0)
-	facilities = models.ManyToManyField(raw_models.EpaFacilitySystem)
 	created = models.DateTimeField( auto_now_add=True)
 
 	def __unicode__(self):
 		return '%s, %s' % ( self.major_city, self.state)
 
 class data(models.Model):
-	location = models.ForeignKey(location, on_delete=models.CASCADE)
+	location =  models.ForeignKey(location, models.PROTECT)
 	timestamp = models.DateTimeField( auto_now_add=True)
 	score = models.DecimalField(max_digits=15, decimal_places=3, default=0.0)
 	objects = DataFrameManager()

app/tasks.py

@@ -1,53 +1,10 @@
+from datetime import datetime, timedelta
+
 from waterquality.celery import app
 from app import models
 from annoying.functions import get_object_or_None
 from utils.log import log
-from datetime import datetime, timedelta
-from utils.crawlers.remote_water_quality_monitoring_network import remote_water_quality_monitoring_network
-
-
-from app.const import (
-	_SENSOR_UNITS
-)
 
 @app.task
 def crawl_usgs_waterdata( daysago = 7):
 	pass
-
-
-@app.task
-def remote_water_quality_monitoring_network_task():
-	log('Running Remote Water Quality Monitoring Network Parser','info')
-	crawler = remote_water_quality_monitoring_network()
-	for node in crawler.get():
-		station = get_object_or_None(models.node, name = node['station'])
-		if not station:
-			station = models.node()
-			station.name = node['station']
-			station.sourcetype = 'live'
-			station.devicealias = node['station'].replace(' ','_').lower()
-			station.save()
-			log('Station: %s (%s) created' % (station.name, station.devicealias),'info')
-
-		# insert sensor data
-		for sensor_type, unit in _SENSOR_UNITS.iteritems():
-			if sensor_type in ['orp']:
-				continue
-
-			data = models.data()
-			data.source = station.sourcetype
-			data.node_name = station.name
-			data.node_alias =station.devicealias
-			data.sensor_type = sensor_type
-			data.sensor_units = unit
-			data.lat = station.lat
-			data.long = station.long
-
-			if node[sensor_type]:
-				data.value = round(float(node[sensor_type]), 2)
-			else:
-				data.value = 0.0
-
-			data.save()
-		log('Station: %s data updated' % (station.name),'info')
-	log('Done','success')

docker-compose.yml

@@ -0,0 +1,74 @@
+# REF: https://testdriven.io/blog/dockerizing-django-with-postgres-gunicorn-and-nginx/
+# To build, and deploy
+# docker-compose up -d --build
+# remove from local system
+# docker system prune -a
+version: '3.7'
+
+services:
+  redis:
+    image: "redis:alpine"
+    ports:
+      - 6379:6379
+    volumes:
+      - redis-data:/data
+
+  nginx:
+    build: ./nginx
+    ports:
+      - 1337:80
+
+  frontend:
+    build: ./frontend
+    volumes:
+      - ./frontend:/app
+      - node-modules:/app/node_modules
+    ports:
+      - 80:3000
+    depends_on:
+      - backend
+      - nginx
+
+  backend:
+    build: .
+    command: gunicorn waterquality.wsgi:application --bind 0.0.0.0:8000
+    container_name: app
+    env_file:
+      - ./.env
+    volumes:
+      - static_volume:/app/static
+    expose:
+      - 8000
+    depends_on:
+      - redis
+      - nginx
+
+  celery-worker:
+    build: .
+    command: celery -A waterquality worker -l info
+    env_file:
+      - ./.env
+    depends_on:
+      - redis
+
+  celery-beat:
+    build: .
+    command: celery -A waterquality beat -l info
+    env_file:
+      - ./.env
+    depends_on:
+      - redis
+
+  flower:
+    image: mher/flower
+    env_file:
+      - ./.env
+    ports:
+      - 8888:8888
+    depends_on:
+      - redis
+
+volumes:
+  redis-data:
+  node-modules:
+  static_volume:

frontend/.dockerignore

@@ -0,0 +1,3 @@
+node_modules
+.git
+build

frontend/Dockerfile

@@ -0,0 +1,18 @@
+# REF: https://mherman.org/blog/dockerizing-a-react-app/
+# BUILD Environment
+FROM node:8
+
+WORKDIR  /app
+
+# add `/app/node_modules/.bin` to $PATH
+ENV PATH /app/node_modules/.bin:$PATH
+
+COPY package.json /app/package.json
+
+RUN npm install
+RUN npm install [email protected] -g
+RUN npm rebuild node-sass
+
+COPY .  ./
+
+CMD ["npm", "start"]

manage.py

@@ -5,7 +5,6 @@
 import dotenv
 
 if __name__ == "__main__":
-    dotenv.read_dotenv()
     os.environ.setdefault("DJANGO_SETTINGS_MODULE", "settings.dev")
 
     from django.core.management import execute_from_command_line

news/models.py

@@ -1,4 +1,6 @@
 
+from __future__ import unicode_literals
+
 from django.contrib.gis.db import models
 from .choice import (
     WATER_STATUS,
@@ -56,8 +58,7 @@ class Meta:
         verbose_name = "Advisory Keyword"
 
 class alert(models.Model):
-    location = models.ForeignKey(
-        location, null=True, blank=True, on_delete=models.SET_NULL)
+    location = models.ForeignKey(location, models.PROTECT, null=True, blank=True)
     source = models.CharField(max_length=255, null=True, blank=True,choices=SOURCE, default="")
     sourceId = models.CharField(max_length=255, null=True, blank=True, default="")
     status = models.CharField(max_length=255, null=True, blank=True,choices=WATER_STATUS, default="safe")
@@ -71,8 +72,7 @@ def __unicode__(self):
         return self.sourceId
 
 class url(models.Model):
-    alert = models.ForeignKey(
-        alert, null=True, blank=True, on_delete=models.SET_NULL)
+    alert = models.ForeignKey(alert,models.CASCADE, null=True, blank=True)
     link = models.TextField(null=True, blank=True,default='')
 
 class utility(models.Model):
@@ -94,7 +94,5 @@ class Meta:
         verbose_name_plural = "Utilities"
 
 class county_served(models.Model):
-    utility = models.ForeignKey(
-        utility, null=True, blank=True, on_delete=models.SET_NULL)
-    location = models.ForeignKey(
-        location, null=True, blank=True, on_delete=models.SET_NULL)
+    utility = models.ForeignKey(utility, models.PROTECT, null=True, blank=True)
+    location = models.ForeignKey(location, models.PROTECT, null=True, blank=True)

news/tasks.py

@@ -32,7 +32,7 @@ def save_twitter_data(tweet, location = None, print_test = False ):
     # ref: https://dev.twitter.com/overview/api/tweets
 
     if models.alert.objects.filter(sourceId = tweet.id_str ).exists():
-        print "%s - exists" % ( tweet.full_text )
+        print("%s - exists" % ( tweet.full_text ))
         #log(tweet.full_text, 'success')
         return
 
@@ -58,14 +58,14 @@ def save_twitter_data(tweet, location = None, print_test = False ):
             url.link = item['url']
             url.save()
 
-    print tweet.full_text
+    print(tweet.full_text)
     #log(tweet.full_text, 'success')
 
 def save_feed_data(item, location = None):
 
     sourceId = item['id']
     if models.alert.objects.filter(sourceId = sourceId ).exists():
-        print "%s - exists" % ( sourceId )
+        print("%s - exists" % ( sourceId ))
         #log(sourceId, 'success')
         return
 
@@ -90,7 +90,7 @@ def save_feed_data(item, location = None):
     url.link = link
     url.save()
 
-    print sourceId
+    print(sourceId)
     #log(sourceId, 'success')
 
 @app.task

nginx/Dockerfile

@@ -0,0 +1,4 @@
+FROM nginx:1.17.4-alpine
+
+RUN rm /etc/nginx/conf.d/default.conf
+COPY nginx.conf /etc/nginx/conf.d

nginx/nginx.conf

@@ -0,0 +1,16 @@
+upstream django {
+    server web:8000;
+}
+
+server {
+
+    listen 80;
+
+    location / {
+        proxy_pass http://django;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_set_header Host $host;
+        proxy_redirect off;
+    }
+
+}