cscl - make lionkey unique (#2070)

* guarantee seqnum uniqueness by bfc

* simplify query slightly

* use lionkey instead of lionkey_dev

* add source table pk constraint to int__segments

* reorganize lion ctes

* rename int segments model yml

* keep globalid through lion transformations

* alphabetize

Author: fvankrieken

.github/workflows/build.yml

@@ -16,12 +16,12 @@ on:
         - checkbook
         - colp
         - cpdb
+        - cscl
         - developments
         - facilities
         - factfinder
         - green_fast_track
         - knownprojects
-        - cscl
         - pluto
         - ztl
         - all
@@ -197,6 +197,17 @@ jobs:
       recipe_file: ${{ inputs.recipe_file }}
       plan_command: ${{ needs.health_check.outputs.plan_command }}
       dev_bucket: ${{ inputs.dev_bucket && format('de-dev-{0}', inputs.dev_bucket) || '' }}
+  cscl:
+    needs: health_check
+    if: inputs.dataset_name == 'cscl' || inputs.dataset_name  == 'all'
+    uses: ./.github/workflows/cscl_build.yml
+    secrets: inherit
+    with:
+      image_tag: ${{ needs.health_check.outputs.tag }}
+      recipe_file: ${{ inputs.recipe_file }}
+      build_name: ${{ needs.health_check.outputs.build_name }}
+      plan_command: ${{ needs.health_check.outputs.plan_command }}
+      dev_bucket: ${{ inputs.dev_bucket && format('de-dev-{0}', inputs.dev_bucket) || '' }}
   developments:
     needs: health_check
     if: inputs.dataset_name == 'developments' || inputs.dataset_name  == 'all'
@@ -251,17 +262,6 @@ jobs:
       recipe_file: ${{ inputs.recipe_file }}
       plan_command: ${{ needs.health_check.outputs.plan_command }}
       dev_bucket: ${{ inputs.dev_bucket && format('de-dev-{0}', inputs.dev_bucket) || '' }}
-  cscl:
-    needs: health_check
-    if: inputs.dataset_name == 'cscl' || inputs.dataset_name  == 'all'
-    uses: ./.github/workflows/cscl_build.yml
-    secrets: inherit
-    with:
-      image_tag: ${{ needs.health_check.outputs.tag }}
-      recipe_file: ${{ inputs.recipe_file }}
-      build_name: ${{ needs.health_check.outputs.build_name }}
-      plan_command: ${{ needs.health_check.outputs.plan_command }}
-      dev_bucket: ${{ inputs.dev_bucket && format('de-dev-{0}', inputs.dev_bucket) || '' }}
   pluto:
     needs: health_check
     if: inputs.dataset_name == 'pluto' || inputs.dataset_name  == 'all'

.github/workflows/test_helper.yml

@@ -87,8 +87,8 @@ jobs:
         - cdbg
         - ceqr
         - cpdb
-        - green_fast_track
         - cscl
+        - green_fast_track
         - pluto
         - zoningtaxlots
     steps:

products/cscl/models/intermediate/adjacent_polygons/int__segment_atomicpolygons.sql

@@ -9,7 +9,7 @@ WITH segment_offsets AS (
 )
 
 SELECT
-    so.lionkey_dev,
+    so.lionkey,
     so.segmentid,
     so.boroughcode AS segment_borocode,
     left_poly.atomicid AS left_atomicid,

products/cscl/models/intermediate/adjacent_polygons/int__segment_nypdbeat.sql

@@ -9,7 +9,7 @@ WITH segment_offsets AS (
 )
 
 SELECT
-    co.lionkey_dev,
+    co.lionkey,
     co.segmentid,
     left(left_beat.post, 1) AS left_nypd_service_area,
     left(right_beat.post, 1) AS right_nypd_service_area

products/cscl/models/intermediate/adjacent_polygons/int__segment_offsets.sql

@@ -7,7 +7,7 @@ WITH segments AS (
 )
 
 SELECT
-    lionkey_dev,
+    lionkey,
     segmentid,
     boroughcode,
     feature_type,

products/cscl/models/intermediate/adjacent_polygons/int__segment_zipcodes.sql

@@ -9,7 +9,7 @@ WITH segment_offsets AS (
     SELECT * FROM {{ ref("int__segment_offsets") }}
 )
 SELECT
-    so.lionkey_dev,
+    so.lionkey,
     so.segmentid,
     leftzip.zip_code AS l_zip,
     rightzip.zip_code AS r_zip

products/cscl/models/intermediate/int__lion.sql

@@ -1,73 +1,64 @@
 {{ config(
     materialized = 'table',
     indexes=[
-      {'columns': ['segmentid']},
+      {'columns': ['lionkey']},
       {'columns': ['boroughcode', 'segment_seqnum', 'segmentid']}
     ]
 ) }}
 
+-- backbone
 WITH segments AS (
     SELECT * FROM {{ ref("int__segments") }}
 ),
 
-segment_locational_status AS (
-    SELECT * FROM {{ ref("int__segment_locational_status") }}
-),
-
+-- source data lookup tables
 centerline AS (
     SELECT * FROM {{ ref("stg__centerline") }}
 ),
-
 rail AS (
     SELECT * FROM {{ ref("stg__rail_and_subway" ) }}
 ),
-
 nsf AS (
     SELECT * FROM {{ ref("stg__nonstreetfeatures") }}
 ),
+proto AS (
+    SELECT * FROM {{ ref("stg__altsegmentdata_proto") }}
+),
 
+-- transformation lookup tables
+segment_locational_status AS (
+    SELECT * FROM {{ ref("int__segment_locational_status") }}
+),
 atomic_polygons AS (
     SELECT * FROM {{ ref("int__segment_atomicpolygons") }}
 ),
-
 nodes AS (
     SELECT * FROM {{ ref("int__segments_with_nodes") }}
 ),
-
 saf AS (
     SELECT * FROM {{ ref("int__segment_specialaddress") }}
 ),
-
 centerline_curve AS (
     SELECT * FROM {{ ref("int__centerline_curve") }}
 ),
-
 sedat AS (
     SELECT * FROM {{ ref("int__split_election_district") }}
 ),
-
 nypd_service_areas AS (
     SELECT * FROM {{ ref("int__segment_nypdbeat") }}
 ),
-
 zips AS (
     SELECT * FROM {{ ref("int__segment_zipcodes") }}
 ),
-
 centerline_coincident_subway_or_rail AS (
     SELECT * FROM {{ ref("int__centerline_coincident_subway_or_rail") }}
 ),
-
 noncl_coincident_segment AS (
     SELECT * FROM {{ ref("int__noncenterline_coincident_segment_count") }}
-),
-
-proto AS (
-    SELECT * FROM {{ ref("stg__altsegmentdata_proto") }}
 )
 
 SELECT
-    segments.lionkey_dev,
+    segments.lionkey,
     segments.boroughcode,
     segments.face_code,
     segments.segment_seqnum,
@@ -262,6 +253,7 @@ SELECT
     segments.feature_type_description,
     segments.source_table,
     segments.geom,
+    segments.globalid,
     CASE
         WHEN segments.feature_type = 'centerline' THEN centerline.include_in_geosupport_lion
         WHEN segments.feature_type = 'rail_and_subway' THEN rail.include_in_geosupport_lion
@@ -272,20 +264,20 @@ SELECT
         ELSE TRUE
     END AS include_in_bytes_lion
 FROM segments
-LEFT JOIN nodes ON segments.lionkey_dev = nodes.lionkey_dev
-LEFT JOIN segment_locational_status ON segments.lionkey_dev = segment_locational_status.lionkey_dev
+LEFT JOIN nodes ON segments.lionkey = nodes.lionkey
+LEFT JOIN segment_locational_status ON segments.lionkey = segment_locational_status.lionkey
 LEFT JOIN
     atomic_polygons AS ap_left
     ON
-        segments.lionkey_dev = ap_left.lionkey_dev
+        segments.lionkey = ap_left.lionkey
         AND segment_locational_status.borough_boundary_indicator IS DISTINCT FROM 'L'
 LEFT JOIN
     atomic_polygons AS ap_right
     ON
-        segments.lionkey_dev = ap_right.lionkey_dev
+        segments.lionkey = ap_right.lionkey
         AND segment_locational_status.borough_boundary_indicator IS DISTINCT FROM 'R'
 LEFT JOIN saf ON segments.segmentid = saf.segmentid AND segments.boroughcode = saf.boroughcode
-LEFT JOIN nypd_service_areas ON segments.lionkey_dev = nypd_service_areas.lionkey_dev
+LEFT JOIN nypd_service_areas ON segments.lionkey = nypd_service_areas.lionkey
 LEFT JOIN sedat ON segments.segmentid = sedat.segmentid AND segments.boroughcode = sedat.boroughcode
 -- centerline only
 LEFT JOIN centerline AS primary_centerline ON segments.segmentid = primary_centerline.segmentid
@@ -298,5 +290,5 @@ LEFT JOIN centerline_coincident_subway_or_rail
 LEFT JOIN rail ON segments.segmentid = rail.segmentid
 -- other
 LEFT JOIN noncl_coincident_segment ON segments.segmentid = noncl_coincident_segment.segmentid
-LEFT JOIN zips ON segments.lionkey_dev = zips.lionkey_dev AND segments.feature_type <> 'centerline'
-LEFT JOIN proto ON segments.source_table = 'altsegmentdata' AND segments.ogc_fid = proto.ogc_fid
+LEFT JOIN zips ON segments.lionkey = zips.lionkey AND segments.feature_type <> 'centerline'
+LEFT JOIN proto ON segments.globalid = proto.globalid

products/cscl/models/intermediate/nodes/int__segments_to_nodes.sql

@@ -16,22 +16,22 @@ nodes AS (
 
 segment_endpoints AS (
     SELECT
-        lionkey_dev,
+        lionkey,
         segmentid,
         'from' AS direction,
         start_point AS geom
     FROM segments
     UNION ALL
     SELECT
-        lionkey_dev,
+        lionkey,
         segmentid,
         'to' AS direction,
         end_point AS geom
     FROM segments
 )
 
 SELECT
-    seg.lionkey_dev,
+    seg.lionkey,
     seg.segmentid,
     seg.direction,
     nodes.nodeid

products/cscl/models/intermediate/nodes/int__segments_with_nodes.sql

@@ -7,7 +7,7 @@
 
 WITH segments AS (
     SELECT
-        lionkey_dev,
+        lionkey,
         segmentid,
         st_startpoint(geom) AS from_geom,
         st_endpoint(geom) AS to_geom
@@ -19,7 +19,7 @@ segments_to_nodes AS (
 )
 
 SELECT
-    segments.lionkey_dev,
+    segments.lionkey,
     segments.segmentid,
     st_x(segments.from_geom) AS from_x,
     st_y(segments.from_geom) AS from_y,
@@ -31,9 +31,9 @@ SELECT
     to_sm.sectional_map AS to_sectionalmap
 FROM segments
 LEFT JOIN segments_to_nodes AS n_from
-    ON segments.lionkey_dev = n_from.lionkey_dev AND n_from.direction = 'from'
+    ON segments.lionkey = n_from.lionkey AND n_from.direction = 'from'
 LEFT JOIN segments_to_nodes AS n_to
-    ON segments.lionkey_dev = n_to.lionkey_dev AND n_to.direction = 'to'
+    ON segments.lionkey = n_to.lionkey AND n_to.direction = 'to'
 LEFT JOIN
     {{ source("recipe_sources", "dcp_cscl_sectionalmap") }} AS from_sm
     ON st_contains(from_sm.geom, segments.from_geom)

products/cscl/models/intermediate/other_lion_intermediates/int__segment_locational_status.sql

@@ -1,13 +1,13 @@
 {{ config(
     materialized = 'table',
     indexes=[
-      {'columns': ['lionkey_dev']},
+      {'columns': ['lionkey']},
     ]
 ) }}
 
 WITH atomicpolygons AS (
     SELECT
-        lionkey_dev,
+        lionkey,
         segmentid,
         segment_borocode,
         left_atomicid,
@@ -35,11 +35,11 @@ segments_by_node AS (
 
 segments_n_neighbors AS (
     SELECT
-        lionkey_dev,
+        lionkey,
         min(n_segments) AS minimum_neighbors -- between the two nodes, minimum number of segments joined
     FROM segments_to_nodes AS s2n
     INNER JOIN segments_by_node AS sbn ON s2n.nodeid = sbn.nodeid
-    GROUP BY lionkey_dev
+    GROUP BY lionkey
 ),
 
 -- All CTEs below divide the atomicpolygons into different categories. 
@@ -74,7 +74,7 @@ same_ap AS (
             WHEN snn.minimum_neighbors > 1 THEN 'H'
         END AS segment_locational_status
     FROM atomicpolygons AS ap
-    LEFT JOIN segments_n_neighbors AS snn ON ap.lionkey_dev = snn.lionkey_dev
+    LEFT JOIN segments_n_neighbors AS snn ON ap.lionkey = snn.lionkey
     WHERE ap.left_atomicid = ap.right_atomicid
 ),
 different_aps_different_boros AS (