Pull country from metrics in quick_suggest ping

sql/moz-fx-data-shared-prod/contextual_services_derived/event_aggregates_suggest_v1/query.sql

@@ -14,7 +14,14 @@ combined AS (
     metrics.uuid.quick_suggest_context_id AS context_id,
     DATE(submission_timestamp) AS submission_date,
     'desktop' AS form_factor,
-    normalized_country_code AS country,
+    -- As of Firefox 140, the quick_suggest ping is sent via OHTTP and now
+    -- receives geo information from the client rather than from Glean ingestion's
+    -- IP geolocation. We no longer send subdivision, only country.
+    IF(
+      SAFE_CAST(metadata.user_agent.version AS INT64) >= 140,
+      metrics.string.quick_suggest_country,
+      normalized_country_code
+    ) AS country,
     LOWER(metrics.string.quick_suggest_advertiser) AS advertiser,
     SPLIT(metadata.user_agent.os, ' ')[SAFE_OFFSET(0)] AS normalized_os,
     client_info.app_channel AS release_channel,

sql/moz-fx-data-shared-prod/contextual_services_derived/event_aggregates_v1/query.sql

@@ -20,8 +20,19 @@ combined AS (
       "impression"
     ) AS event_type,
     'desktop' AS form_factor,
-    normalized_country_code AS country,
-    metadata.geo.subdivision1 AS subdivision1,
+    -- As of Firefox 140, the quick_suggest ping is sent via OHTTP and now
+    -- receives geo information from the client rather than from Glean ingestion's
+    -- IP geolocation. We no longer send subdivision, only country.
+    IF(
+      SAFE_CAST(metadata.user_agent.version AS INT64) >= 140,
+      metrics.string.quick_suggest_country,
+      normalized_country_code
+    ) AS country,
+    IF(
+      SAFE_CAST(metadata.user_agent.version AS INT64) >= 140,
+      CAST(NULL AS STRING),
+      metadata.geo.subdivision1
+    ) AS subdivision1,
     metrics.string.quick_suggest_advertiser AS advertiser,
     client_info.app_channel AS release_channel,
     metrics.quantity.quick_suggest_position AS position,

sql/moz-fx-data-shared-prod/contextual_services_derived/request_payload_suggest_v2/query.sql

@@ -18,8 +18,19 @@ ping_data AS (
       "click",
       "impression"
     ) AS interaction_type,
-    metadata.geo.country AS country_code,
-    metadata.geo.subdivision1 AS region_code,
+    -- As of Firefox 140, the quick_suggest ping is sent via OHTTP and now
+    -- receives geo information from the client rather than from Glean ingestion's
+    -- IP geolocation. We no longer send subdivision, only country.
+    IF(
+      SAFE_CAST(metadata.user_agent.version AS INT64) >= 140,
+      metrics.string.quick_suggest_country,
+      metadata.geo.country
+    ) AS country_code,
+    IF(
+      SAFE_CAST(metadata.user_agent.version AS INT64) >= 140,
+      CAST(NULL AS STRING),
+      metadata.geo.subdivision1
+    ) AS region_code,
     metadata.user_agent.os AS os_family,
     metadata.user_agent.version AS product_version,
   FROM

sql/moz-fx-data-shared-prod/search_terms_derived/suggest_impression_sanitized_v3/query.sql

@@ -45,8 +45,16 @@ WITH impressions AS (
     sample_id,
     metrics.boolean.quick_suggest_is_clicked AS is_clicked,
     client_info.locale AS locale,
-    metadata.geo.country,
-    metadata.geo.subdivision1 AS region,
+    IF(
+      SAFE_CAST(metadata.user_agent.version AS INT64) >= 140,
+      metrics.string.quick_suggest_country,
+      metadata.geo.country
+    ) AS country,
+    IF(
+      SAFE_CAST(metadata.user_agent.version AS INT64) >= 140,
+      CAST(NULL AS STRING),
+      metadata.geo.subdivision1
+    ) AS region,
     normalized_os,
     normalized_os_version,
     normalized_channel,

tests/sql/moz-fx-data-shared-prod/contextual_services_derived/event_aggregates_v1/test_aggregation/expect.yaml

@@ -3,10 +3,9 @@
   submission_date: "2030-01-01"
   source: suggest
   provider: merino
-  event_type: impression
+  event_type: click
   form_factor: desktop
-  country: US
-  subdivision1: AZ
+  country: DE
   advertiser: ad1
   release_channel: release
   match_type: "firefox-suggest"
@@ -17,7 +16,12 @@
   user_count: 1
   query_type: branded
 - <<: *suggest_base
-  event_type: click
+  event_type: impression
+  country: US
+  subdivision1: AZ
+- <<: *suggest_base
+  country: US
+  subdivision1: AZ
 - &suggest_mobile
   submission_date: "2030-01-01"
   source: suggest

tests/sql/moz-fx-data-shared-prod/contextual_services_derived/event_aggregates_v1/test_aggregation/moz-fx-data-shared-prod.firefox_desktop.quick_suggest.yaml

@@ -8,6 +8,7 @@
       subdivision1: AZ
     user_agent:
       os: "Windows 11"
+      version: "130"
   metrics: &click_metrics
     boolean:
       quick_suggest_improve_suggest_experience: true
@@ -28,3 +29,13 @@
     string:
       <<: *click_strings
       quick_suggest_ping_type: quicksuggest-impression
+- <<: *click
+  metadata:
+    user_agent:
+      os: "Windows 11"
+      version: "141"
+  metrics:
+    <<: *click_metrics
+    string:
+      <<: *click_strings
+      quick_suggest_country: DE

tests/sql/moz-fx-data-shared-prod/contextual_services_derived/event_aggregates_v1/test_overactive_filter/moz-fx-data-shared-prod.firefox_desktop.quick_suggest.yaml

@@ -8,6 +8,7 @@
       subdivision1: AZ
     user_agent:
       os: "Windows 11"
+      version: 130
   metrics: &click_metrics
     boolean:
       quick_suggest_improve_suggest_experience: true
@@ -19,6 +20,7 @@
       quick_suggest_match_type: firefox-suggest
       quick_suggest_ping_type: quicksuggest-click
       quick_suggest_request_id: HASH123
+      quick_suggest_country: null
     uuid:
       quick_suggest_context_id: a
   normalized_country_code: US

tests/sql/moz-fx-data-shared-prod/search_terms_derived/suggest_impression_sanitized_v3/test_single_day/expect.yaml

@@ -116,3 +116,24 @@
   reporting_url: https://example.com/duff-beer
   submission_timestamp: "2021-01-01T13:03:00+00:00"
   version: "134"
+- advertiser: omni consumer products
+  block_id: 205
+  browser: firefox(141)
+  context_id: cid15
+  country: US
+  form_factor: desktop
+  is_clicked: false
+  locale: en-US
+  normalized_channel: release
+  normalized_os: windows
+  normalized_os_version: 15.2
+  os_family: windows
+  position: 15
+  query: robot police
+  reporting_url: https://example.com/ocp
+  request_id: rid15
+  sequence_no: 15
+  session_id: sid15
+  submission_timestamp: "2021-01-01T13:05:00+00:00"
+  timestamp: "2021-01-01T13:05:00+00:00"
+  version: "141"

tests/sql/moz-fx-data-shared-prod/search_terms_derived/suggest_impression_sanitized_v3/test_single_day/moz-fx-data-shared-prod.firefox_desktop_stable.quick_suggest_v1.yaml

@@ -6,6 +6,8 @@
     geo:
       country: US
       subdivision1: CA
+    user_agent:
+      version: "134"
   metrics:
     boolean:
       quick_suggest_is_clicked: false
@@ -32,6 +34,8 @@
     geo:
       country: US
       subdivision1: PA
+    user_agent:
+      version: "134"
   metrics:
     boolean:
       quick_suggest_is_clicked: true
@@ -58,6 +62,8 @@
     geo:
       country: US
       subdivision1: OR
+    user_agent:
+      version: "134"
   metrics:
     boolean:
       quick_suggest_is_clicked: false
@@ -84,6 +90,8 @@
     geo:
       country: US
       subdivision1: WA
+    user_agent:
+      version: "134"
   metrics:
     boolean:
       quick_suggest_is_clicked: true
@@ -102,3 +110,32 @@
   normalized_os: Windows
   normalized_os_version: "11.0"
   sample_id: null
+- submission_timestamp: "2021-01-01 13:05:00"
+  client_info:
+    app_display_version: "141.0.1"
+    locale: en-US
+  metadata:
+    geo:
+      country: null
+      subdivision1: null
+    user_agent:
+      version: "141"
+  metrics:
+    boolean:
+      quick_suggest_is_clicked: false
+    quantity:
+      quick_suggest_position: 15
+    string:
+      quick_suggest_advertiser: omni consumer products
+      quick_suggest_block_id: 205
+      quick_suggest_ping_type: "quicksuggest-impression"
+      quick_suggest_request_id: rid15
+      quick_suggest_country: US
+    url2:
+      quick_suggest_reporting_url: "https://example.com/ocp"
+    uuid:
+      quick_suggest_context_id: cid15
+  normalized_channel: release
+  normalized_os: windows
+  normalized_os_version: "15.2"
+  sample_id: null

tests/sql/moz-fx-data-shared-prod/search_terms_derived/suggest_impression_sanitized_v3/test_single_day/moz-fx-data-shared-prod.search_terms_derived.merino_log_sanitized_v3.yaml

@@ -64,3 +64,14 @@
   form_factor: desktop
   browser: firefox(134)
   os_family: windows
+- timestamp: "2021-01-01 13:05:00.000000"
+  request_id: rid15
+  session_id: sid15
+  sequence_no: 15
+  query: "robot police"
+  country: null
+  region: null
+  dma: null
+  form_factor: desktop
+  browser: firefox(141)
+  os_family: windows