run saau

.github/workflows/python.yaml

@@ -28,7 +28,7 @@ jobs:
       - name: Preinstall
         run: |
           pipenv run pip install cython numpy
-          sudo apt install libopenblas-dev libgdal-dev
+          sudo apt install libopenblas-dev libgdal-dev libproj-dev
       - name: Install dependencies
         if: steps.cache-pipenv.outputs.cache-hit != 'true'
         run: |
@@ -52,6 +52,8 @@ jobs:
       - name: Import saau
         run: |
           pipenv run python -c "import saau; print(saau)"
+      - name: Run saau
+        run: pipenv run python -m saau
       # - name: Archive test report
       #   uses: actions/upload-artifact@v1
       #   if: always()

.gitignore

@@ -56,3 +56,7 @@ docs/_build/
 
 # PyBuilder
 target/
+
+stats_data/
+.idea
+*.sqlite

pyproject.toml

@@ -0,0 +1,36 @@
+[tool.poetry]
+name = "statistical-atlas-of-au"
+version = "0.1.0"
+authors = ["Elliana May <[email protected]>"]
+license = "MIT"
+readme = "README.md"
+packages = [{include = "statistical_atlas_of_au"}]
+description = 'Statistical Atlas of Australia'
+
+[tool.poetry.dependencies]
+python = ">=3.10,<3.12"
+
+requests = "*"
+betamax = "*"
+cartopy = "*"
+matplotlib = "*"
+pandas = "~1.5.0"
+seaborn = "*"
+humanize = "*"
+lxml = "*"
+dill = "*"
+tqdm = "*"
+rasterio = "*"
+gn-arcrest = "*"
+cython = "*"
+numpy = "*"
+coloredlogs = "*"
+pandasdmx = { git = "https://github.com/Mause/pandaSDMX", branch = "range-period", extras = ["cache"] }
+fastparquet = "^2023.10.1"
+
+[tool.poetry.group.dev.dependencies]
+pdbpp = "^0.10.3"
+
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"

saau/__main__.py

@@ -4,16 +4,20 @@
 import logging
 import warnings
 import argparse
+import coloredlogs
 from operator import itemgetter
 from concurrent.futures import ThreadPoolExecutor as PoolExecutor
 from os.path import join, dirname, exists, expanduser
+from bdb import BdbQuit
 
+import pandasdmx
 
 STATS_DATA = 'c:\\stats_data'
 CACHE = join(STATS_DATA, 'cache')
 OUTPUT = join(STATS_DATA, 'output')
 HERE = dirname(__file__)
-logging.basicConfig(level=logging.DEBUG)
+coloredlogs.install(level='DEBUG')
+coloredlogs.install(logger=pandasdmx.logger, level='DEBUG')
 sys.path.insert(0, expanduser('~/Dropbox/temp/arcrest'))
 
 from .services import Services
@@ -52,8 +56,10 @@ def ensure_data(prov):
         logging.info('Obtaining data for %s', get_name(prov))
         try:
             val = prov.obtain_data()
+        except BdbQuit:
+            raise
         except Exception as e:
-            logging.exception(e)
+            logging.exception('failed to obtain data')
             return False
 
         if val not in {True, False}:
@@ -160,6 +166,9 @@ def build_image(prov, rerender_all):
         else:
             logging.info('Render unsuccessful')
 
+    except BdbQuit:
+        raise
+
     except NotImplementedError:
         logging.exception("Can't render %s", get_name(prov))
 
@@ -169,6 +178,9 @@ def build_image(prov, rerender_all):
             get_name(prov)
         )
 
+    except Exception:
+        logging.exception("failed to render image")
+
     del prov  # force cleanup
 
 

saau/sections/age/median.py

@@ -21,25 +21,13 @@ def has_required_data(self):
     def obtain_data(self):
         data = get_generic_data(
             DATASETID,
-            and_=[
-                'FREQUENCY.A',
-                'REGIONTYPE.SA2',
-                'MEASURE.MAGE'
-            ],
-            or_=[
-                'STATE.0',
-                'STATE.1',
-                'STATE.2',
-                'STATE.3',
-                'STATE.4',
-                'STATE.5',
-                'STATE.6',
-                'STATE.7',
-                'STATE.8',
-                'STATE.9'
-            ]
+            and_={
+                'FREQUENCY':'A',
+                'REGIONTYPE':'SA2',
+                'MEASURE':'MAGE',
+                'STATE': ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9']
+            }
         )
-        assert data['series']
 
         return self.save_json(FILENAME, data)
 
@@ -49,7 +37,7 @@ def region_lookup(self, sa3):
     def build_image(self):
         colors = get_cmap('Purples')
 
-        age_data = abs_data_to_dataframe(self.load_json(FILENAME))
+        age_data = abs_data_to_dataframe(self.data_dir_join(FILENAME))
         age_data = [
             (
                 self.region_lookup(data_point.REGION),

saau/sections/ancestry/__init__.py

@@ -76,10 +76,7 @@ def obtain_data(self):
         return self.save_json(self.filename, get_data(self.ancestry_name))
 
     def build_image(self):
-        data = abs_data_to_dataframe(
-            self.load_json(self.filename),
-            ['ANCP', 'FREQUENCY']
-        )
+        data = abs_data_to_dataframe(self.data_dir_join(self.filename))
         data = data[data.pop('Time') == 2011]
         del data['REGIONTYPE']
 

saau/sections/image_provider.py

@@ -48,8 +48,13 @@ def data_dir_join(self, name: PathOrStr) -> str:
         return join(self.data_dir, name)
 
     def save_json(self, name: PathOrStr, data: Any) -> bool:
-        with open(self.data_dir_join(name), 'w') as fh:
-            json.dump(data, fh, indent=4)
+        import pandas as pd
+
+        if isinstance(data, (pd.DataFrame, pd.Series)):
+            data.to_json(self.data_dir_join(name))
+        else:
+            with open(self.data_dir_join(name), 'w') as fh:
+                json.dump(data, fh, indent=4)
         return True
 
     def load_json(self, name: PathOrStr) -> Any:

saau/sections/population/density.py

@@ -14,8 +14,7 @@
 
 
 def get_data(data_dir):
-    with open(join(data_dir, filename)) as fh:
-        return abs_data_to_dataframe(json.load(fh))
+    return abs_data_to_dataframe(join(data_dir, filename))
 
 
 def main(services, data_dir):

saau/sections/population/male_vs_female.py

@@ -49,7 +49,7 @@ def obtain_data(self):
         return self.save_json(FILENAME, data)
 
     def load_data(self):
-        df = abs_data_to_dataframe(self.load_json(FILENAME))
+        df = abs_data_to_dataframe(self.data_dir_join(FILENAME))
 
         df.SEX_ABS = (
             df.SEX_ABS

saau/sections/transportation/__init__.py

@@ -45,7 +45,7 @@ def obtain_data(self):
         assert hasattr(self, 'layers')
         return self.save_json(
             self.path,
-            get_paths(self.layers).tolist()
+            get_paths(self.layers)
         )
 
     def has_required_data(self):

saau/sections/transportation/data.py

@@ -1,21 +1,17 @@
 from operator import itemgetter
 from itertools import chain
 from typing import List
+from functools import cache
 
-import cgi
-from urllib.parse import parse_qs
-
-from ...utils.py3_hook import with_hook
-
-with with_hook():
-    from arcrest import Catalog
+from arcrest import Catalog
 import numpy as np
+from tqdm import tqdm
 
 
-cgi.parse_qs = parse_qs  # type: ignore
-
-
-def get_layers(service):
+@cache
+def get_layers():
+    catalog = Catalog('http://services.ga.gov.au/site_7/rest/services')
+    service = catalog['NM_Transport_Infrastructure']
     layers = service.layers
     return {
         layer.name: layer
@@ -29,13 +25,10 @@ def mend_extent(extent):
 
 
 def get_data(requested_layers: List[str]):
-    catalog = Catalog('http://services.ga.gov.au/site_7/rest/services')
-    service = catalog['NM_Transport_Infrastructure']
-    layers = get_layers(service)
-
+    layers = get_layers()
     return chain.from_iterable(
         layers[layer].QueryLayer(Geometry=mend_extent(layers[layer].extent))
-        for layer in requested_layers
+        for layer in tqdm(requested_layers, desc='Fetching requested layers')
     )
 
 
@@ -48,10 +41,12 @@ def get_paths(request_layers: List[str]) -> np.array:
         if hasattr(geometry, 'paths')
     )
 
-    return np.array([
+    result = [
         tuple(
             (part.x, part.y)
             for part in path
         )
         for path in paths
-    ])
+    ]
+
+    return result # np.array(result)

saau/services/aus_map/__init__.py

@@ -10,16 +10,15 @@
 from ...sections.image_provider import RequiresData
 from ...utils.download import get_binary
 
-name = lambda q: q.attributes['NAME_1']
+name = lambda q: q.attributes['Name']
 DummyRecord = namedtuple('DummyRecord', 'attributes,geometry')
 LL = namedtuple('LL', 'lat,lon')
 
 AUS_NW = LL(111.5, -1)
 AUS_SE = LL(155, -42)
 YELLOWED_PAPER = '#F1E4BB'
 
-"http://www.gadm.org/download"
-URL = 'http://biogeo.ucdavis.edu/data/gadm2.8/shp/AUS_adm_shp.zip'
+URL = 'https://www.geoboundaries.org/data/1_3_3/zip/shapefile/AUS/AUS_ADM1.shp.zip'
 FILENAME = basename(URL)
 
 
@@ -35,15 +34,17 @@ def get_states(services):
 
     shpfile = shape_from_zip(
         services.aus_map.data_dir_join(FILENAME),
-        "AUS_adm1"
+        'AUS_ADM1',
     )
 
     val = list(shpfile.records())
 
+    try_simplify = lambda a: a.simplify(0.5) if hasattr(a, 'simplify') else a
+
     val = [
         DummyRecord(
             rec.attributes,
-            rec.geometry.simplify(0.5)
+            try_simplify(rec.geometry)
         )
         for rec in val
     ]
@@ -73,7 +74,7 @@ def get_map(services, show_world=False, zorder=0):
     # ax.background_patch.set_visible(False)
     # ax.outline_patch.set_visible(False)
 
-    ax.background_patch.set_facecolor(YELLOWED_PAPER)
+    # ax.background_patch.set_facecolor(YELLOWED_PAPER)
     ax.patch.set_facecolor(YELLOWED_PAPER)
     ax.set_facecolor(YELLOWED_PAPER)
 

saau/utils/__init__.py

@@ -12,7 +12,6 @@
 import requests
 import sys
 from os.path import expanduser
-sys.path.insert(0, expanduser('~/Dropbox/temp/arcrest'))
 import arcrest.compat
 from betamax import Betamax
 

saau/utils/download/__init__.py

@@ -22,6 +22,7 @@ def get_binary(url, filename):
         )
 
     r = requests.get(url, stream=True)
+    r.raise_for_status()
 
     with open(filename, 'wb') as fh:
         copyfileobj(