Add rawdata epa with facilities (#13)

* add rawdata app and commands to import epa data

* initial cut. not done

* created two classes:
epa_sdw_downloader
epa_sdw_importer
placed both files in /utils/epa
replaced management command guts with classes

* rename class objects

* WIP for merge

* finish up facility importer

* final updates and test again

.gitignore

@@ -88,6 +88,5 @@ ENV/
 # Rope project settings
 .ropeproject
 
-
 #VSCode
 .vscode

README.md

@@ -2,15 +2,15 @@
 
 waterbeacon
 
-# Resources
+## Resources
 
 https://www.waterqualitydata.us/
 
-# NSF Water Quality Index
+## NSF Water Quality Index
 
 http://home.eng.iastate.edu/~dslutz/dmrwqn/water_quality_index_calc.htm
 
-# API Installation Guide
+## API Installation Guide
 
 You'll need to install `python2` along with `pip`.
 
@@ -31,20 +31,28 @@ Make sure you have the folling tools are installed:
 
 Create the waterbeacon database by running `createdb waterbeacon`
 
-When that completes, run `./manage.py migrate --settings=settings.dev` to create a local SQLite db instance for developnent.
+When that completes, run ./manage.py migrate --settings=settings.dev to create a local postgres db instance for development.
+
+(Optional - never necessary if csvs exist) To get all the new facility location data from the EPA, you can run `./manage.py download_epa_facility_data --settings=settings.dev`
+
+(Optional - never necessary if csvs exist) To get all the new SDWA data from the EPA, you can run `./manage.py download_epa_water_data --settings=settings.dev`
+
+(Optional) Populate the rawdata app with EPA data by running the following:
+
+    ./manage.py import_epa_facility_data --settings=settings.dev
+    ./manage.py import_epa_water_data --settings=settings.dev
 
 Finally, run `./manage.py runserver --settings=settings.dev` to kick off a dev server.
 
 The dev server will run on `localhost:8000` by default. If you prefer another port, just pass the desired port number following the previous command. That is `./manage.py migrate --settings=settings.dev ${PORT}`
 
 _Note_: to deactive the virtual env wrapper, just run `deactivate`.
 
-# Frontend Installation Guide
+## Frontend Installation Guide
 
 Navigate to the frontend directory where the react app is located and run install
 
 * `cd frontend`
 * `npm install`
 
-After installation you can run `npm start` which will run the react app on localhost:3000 with "hot reload" enabled.
-
+After installation you can run `npm start` which will run the react app on localhost:3000 with "hot reload" enabled.

app/models.py

@@ -78,7 +78,7 @@ class data(models.Model):
 	objects = DataFrameManager()
 
 	def __unicode__(self):
-		return '%s - %s (%s) ' % (self.node.name, self.timestamp)
+		return '%s - %s' % (self.node.name, self.timestamp)
 
 	class Meta:
 		verbose_name_plural = "Data"

manage.py

@@ -6,7 +6,7 @@
 
 if __name__ == "__main__":
     dotenv.read_dotenv()
-    os.environ.setdefault("DJANGO_SETTINGS_MODULE", "waterquality.settings")
+    os.environ.setdefault("DJANGO_SETTINGS_MODULE", "settings.dev")
 
     from django.core.management import execute_from_command_line
 

rawdata/admin.py

@@ -0,0 +1,6 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.contrib import admin
+
+# Register your models here.

rawdata/apps.py

@@ -0,0 +1,8 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.apps import AppConfig
+
+
+class RawdataConfig(AppConfig):
+    name = 'rawdata'

rawdata/epa/Facilities/DC.csv

@@ -0,0 +1,8 @@
+FacDerivedStctyFIPS,SDWAFormalActionCount,FacDateLastPenalty,SDWISFlag,FacState,FacDerivedHuc,FacTotalPenalties,Score,SDWASystemTypes,SDWAInspections5yr,FacFIPSCode,FacLastPenaltyAmt,FacName,FacZip,FacEPARegion,FacMajorFlag,NC,FacFormalActionCount,ViolFlag,FacReferencePoint,FacCity,FacImpWaterFlg,FacPenaltyCount,SDWAIDs,SDWA3yrComplQtrsHistory,FacPercentMinority,FacDerivedWBD,FacLat,FacCollectionMethod,CurrVioFlag,FacStdCountyName,SDWAInformalCount,FacDerivedZip,RegistryID,FacStreet,FacAccuracyMeters,FacCounty,FacLong
+11001,,01/15/2016,Y,DC,02070010,"$262,000",1,Community water system,,11001,"$250,000",BLUE PLAINS WASTEWATER TREATMENT PLANT,20032,03,Y,Y,2,1,POINT WHERE SUBSTANCE IS RELEASED,WASHINGTON,Y,2,DC0000002,____________U,58.723,020700100301,38.816781,INTERPOLATION-MAP,1,DISTRICT OF COLUMBIA,,20032,110029030144,5000 OVERLOOK AVE SW,50,DISTRICT OF COLUMBIA,-77.032755
+11001,,,Y,DC,02070010,$0,1,Non-Transient non-community system,,11001,,CHILDRENS NATIONAL MEDICAL CENTER,20010,03,Y,N,0,1,CENTER OF A FACILITY OR STATION,WASHINGTON,,,DC0000008,,63.359,020700100204,38.92626,ADDRESS MATCHING-HOUSE NUMBER,0,DISTRICT OF COLUMBIA,,20010,110006180006,111 MICHIGAN AVENUE NW,30,DISTRICT OF COLUMBIA,-77.01341
+11001,,,Y,DC,02070008,$0,1,Community water system,,11001,,DALECARLIA WATER TREATMENT PLANT,20016,03,Y,Y,0,1,CENTER OF A FACILITY OR STATION,WASHINGTON,Y,,DC0000001,____________U,20.561,020700081005,38.93781,INTERPOLATION-MAP,1,DISTRICT OF COLUMBIA,,20016,110000500841,5900 MACARTHUR BOULEVARD NW,50,DISTRICT OF COLUMBIA,-77.113567
+11001,,,Y,DC,02070010,$0,1,Community water system,1,11001,,JOINT BASE ANACOSTIA-BOLLING,20032,03,,Y,1,1,ENTRANCE POINT OF A FACILITY OR STATION,WASHINGTON,,,DC0000004 DC0000007,VVVVVVVVSSSSU,74.954,020700100204,38.85899,ADDRESS MATCHING-HOUSE NUMBER,1,DISTRICT OF COLUMBIA,5,20032,110055217992,"2700 S. CAPITOL STREET, SE, BLDG. 399",50,DISTRICT OF COLUMBIA,-77.0044
+,,,Y,DC,,$0,1,Non-Transient non-community system,,,,MANDARIN ORIENTAL HOTEL,,03,,Y,0,1,,,,,DC0000010,VVVVVVVVVVVVU,,,38.904149,County Centroid,1,,4,,110063333174,,30000,DISTRICT OF COLUMBIA,-77.017094
+,,,Y,DC,,$0,1,Non-Transient non-community system,,,,NAVAL OBSERVATORY,,03,,Y,0,1,,,,,DC0000005,_VVVVVVVVVVVU,,,38.904149,County Centroid,1,,4,,110050015256,,30000,DISTRICT OF COLUMBIA,-77.017094
+11001,,,Y,DC,02070010,$0,1,Community water system,,11001,,NAVAL STATION WASHINGTON - WNY,,03,,Y,0,1,ENTRANCE POINT OF A FACILITY OR STATION,,,,DC0000003,_VVVVVVVVVVVU,78.034,020700100204,38.874611,ADDRESS MATCHING-HOUSE NUMBER,1,DISTRICT OF COLUMBIA,6,20374,110020591689,,50,DISTRICT OF COLUMBIA,-76.992148