[COST-4751] GCP Scrub disabled tags in trino. (#4966)

Author: myersCody

koku/masu/database/gcp_report_db_accessor.py

@@ -227,30 +227,6 @@ def populate_enabled_tag_keys(self, start_date, end_date, bill_ids):
         }
         self._prepare_and_execute_raw_sql_query(table_name, sql, sql_params)
 
-    def update_line_item_daily_summary_with_enabled_tags(self, start_date, end_date, bill_ids):
-        """Populate the enabled tag key table.
-
-        Args:
-            start_date (datetime.date) The date to start populating the table.
-            end_date (datetime.date) The date to end on.
-            bill_ids (list) A list of bill IDs.
-
-        Returns
-            (None)
-        """
-        table_name = self._table_map["line_item_daily_summary"]
-        sql = pkgutil.get_data(
-            "masu.database", "sql/reporting_gcpcostentryline_item_daily_summary_update_enabled_tags.sql"
-        )
-        sql = sql.decode("utf-8")
-        sql_params = {
-            "start_date": start_date,
-            "end_date": end_date,
-            "bill_ids": bill_ids,
-            "schema": self.schema,
-        }
-        self._prepare_and_execute_raw_sql_query(table_name, sql, sql_params)
-
     def populate_gcp_topology_information_tables(self, provider, start_date, end_date, invoice_month_date):
         """Populate the GCP topology table."""
         ctx = {

koku/masu/database/sql/reporting_gcpcostentryline_item_daily_summary_update_enabled_tags.sql

@@ -23,6 +23,12 @@ INSERT INTO postgres.{{schema | sqlsafe}}.reporting_gcpcostentrylineitem_daily_s
     invoice_month,
     credit_amount
 )
+with cte_pg_enabled_keys as (
+    select array_agg(key order by key) as keys
+      from postgres.{{schema | sqlsafe}}.reporting_enabledtagkeys
+     where enabled = true
+     and provider_type = 'GCP'
+)
 SELECT uuid() as uuid,
     INTEGER '{{bill_id | sqlsafe}}' as cost_entry_bill_id,
     billing_account_id as account_id,
@@ -38,7 +44,12 @@ SELECT uuid() as uuid,
     json_extract_scalar(json_parse(system_labels), '$["compute.googleapis.com/machine_spec"]') as instance_type,
     max(usage_pricing_unit) as unit,
     cast(sum(usage_amount_in_pricing_units) AS decimal(24,9)) as usage_amount,
-    json_parse(labels) as tags,
+    cast(
+        map_filter(
+            cast(json_parse(labels) as map(varchar, varchar)),
+            (k,v) -> contains(pek.keys, k)
+        ) as json
+     ) as tags,
     max(currency) as currency,
     cost_type as line_item_type,
     cast(sum(cost) AS decimal(24,9)) as unblended_cost,
@@ -47,6 +58,8 @@ SELECT uuid() as uuid,
     invoice_month,
     sum(((cast(COALESCE(json_extract_scalar(json_parse(credits), '$["amount"]'), '0')AS decimal(24,9)))*1000000)/1000000) as credit_amount
 FROM hive.{{schema | sqlsafe}}.{{table | sqlsafe}}
+CROSS JOIN
+    cte_pg_enabled_keys as pek
 WHERE source = '{{source_uuid | sqlsafe}}'
     AND year = '{{year | sqlsafe}}'
     AND month = '{{month | sqlsafe}}'
@@ -61,6 +74,6 @@ GROUP BY billing_account_id,
     date(usage_end_time),
     location_region,
     json_extract_scalar(json_parse(system_labels), '$["compute.googleapis.com/machine_spec"]'),
-    labels,
+    16, -- matches column num of tag's map_filter
     cost_type,
     invoice_month

koku/masu/database/trino_sql/reporting_gcpcostentrylineitem_daily_summary.sql

@@ -116,7 +116,6 @@ def update_summary_tables(self, start_date, end_date, **kwargs):
                 accessor.populate_ui_summary_tables(start, end, self._provider.uuid)
             accessor.populate_tags_summary_table(bill_ids, start_date, end_date)
             accessor.populate_gcp_topology_information_tables(self._provider, start_date, end_date, invoice_month_date)
-            accessor.update_line_item_daily_summary_with_enabled_tags(start_date, end_date, bill_ids)
             accessor.update_line_item_daily_summary_with_tag_mapping(start_date, end_date, bill_ids)
             for bill in bills:
                 if bill.summary_data_creation_datetime is None:

koku/masu/processor/gcp/gcp_report_parquet_summary_updater.py

@@ -185,35 +185,6 @@ def test_populate_enabled_tag_keys(self):
             self.accessor.populate_enabled_tag_keys(start_date, end_date, bill_ids)
             self.assertNotEqual(EnabledTagKeys.objects.filter(provider_type=Provider.PROVIDER_GCP).count(), 0)
 
-    def test_update_line_item_daily_summary_with_enabled_tags(self):
-        """Test that we filter the daily summary table's tags with only enabled tags."""
-        start_date = self.dh.this_month_start.date()
-        end_date = self.dh.this_month_end.date()
-
-        bills = self.accessor.bills_for_provider_uuid(self.gcp_provider_uuid, start_date)
-        with schema_context(self.schema):
-            GCPTagsSummary.objects.all().delete()
-            key_to_keep = EnabledTagKeys.objects.filter(provider_type=Provider.PROVIDER_GCP).filter(key="app").first()
-            EnabledTagKeys.objects.filter(provider_type=Provider.PROVIDER_GCP).update(enabled=False)
-            EnabledTagKeys.objects.filter(provider_type=Provider.PROVIDER_GCP).filter(key="app").update(enabled=True)
-            bill_ids = [bill.id for bill in bills]
-            self.accessor.update_line_item_daily_summary_with_enabled_tags(start_date, end_date, bill_ids)
-            tags = (
-                GCPCostEntryLineItemDailySummary.objects.filter(
-                    usage_start__gte=start_date, cost_entry_bill_id__in=bill_ids
-                )
-                .values_list("tags")
-                .distinct()
-            )
-
-            for tag in tags:
-                tag_dict = tag[0] if tag[0] is not None else {}  # account for null tags value
-                tag_keys = list(tag_dict.keys())
-                if tag_keys:
-                    self.assertEqual([key_to_keep.key], tag_keys)
-                else:
-                    self.assertEqual([], tag_keys)
-
     def test_table_properties(self):
         self.assertEqual(self.accessor.line_item_daily_summary_table, GCPCostEntryLineItemDailySummary)
 

koku/masu/test/database/test_gcp_report_db_accessor.py

@@ -66,16 +66,11 @@ def test_get_sql_inputs(self):
     @patch(
         "masu.processor.gcp.gcp_report_parquet_summary_updater.GCPReportDBAccessor.delete_line_item_daily_summary_entries_for_date_range_raw"  # noqa: E501
     )
-    @patch(
-        "masu.processor.gcp.gcp_report_parquet_summary_updater.GCPReportDBAccessor.update_line_item_daily_summary_with_enabled_tags"  # noqa: E501
-    )
     @patch("masu.processor.gcp.gcp_report_parquet_summary_updater.GCPReportDBAccessor.populate_tags_summary_table")
     @patch(
         "masu.processor.gcp.gcp_report_parquet_summary_updater.GCPReportDBAccessor.populate_line_item_daily_summary_table_trino"  # noqa: E501
     )
-    def test_update_daily_summary_tables(
-        self, mock_trino, mock_tag_update, mock_summary_update, mock_delete, mock_topo
-    ):
+    def test_update_daily_summary_tables(self, mock_trino, mock_tag_update, mock_delete, mock_topo):
         """Test that we run Trino summary."""
         start_str = self.dh.this_month_start.isoformat()
         end_str = self.dh.this_month_end.isoformat()
@@ -103,7 +98,6 @@ def test_update_daily_summary_tables(
             expected_start, expected_end, self.gcp_provider.uuid, current_bill_id, markup_value, start
         )
         mock_tag_update.assert_called_with(bill_ids, start, end)
-        mock_summary_update.assert_called_with(start, end, bill_ids)
 
         self.assertEqual(start_return, start)
         self.assertEqual(end_return, end)
@@ -114,16 +108,11 @@ def test_update_daily_summary_tables(
     @patch(
         "masu.processor.gcp.gcp_report_parquet_summary_updater.GCPReportDBAccessor.delete_line_item_daily_summary_entries_for_date_range_raw"  # noqa: E501
     )
-    @patch(
-        "masu.processor.gcp.gcp_report_parquet_summary_updater.GCPReportDBAccessor.update_line_item_daily_summary_with_enabled_tags"  # noqa: E501
-    )
     @patch("masu.processor.gcp.gcp_report_parquet_summary_updater.GCPReportDBAccessor.populate_tags_summary_table")
     @patch(
         "masu.processor.gcp.gcp_report_parquet_summary_updater.GCPReportDBAccessor.populate_line_item_daily_summary_table_trino"  # noqa: E501
     )
-    def test_update_daily_summary_tables_no_invoice_month(
-        self, mock_trino, mock_tag_update, mock_summary_update, mock_delete, mock_topo
-    ):
+    def test_update_daily_summary_tables_no_invoice_month(self, mock_trino, mock_tag_update, mock_delete, mock_topo):
         """Test that we run Trino summary."""
         start_str = self.dh.this_month_start.isoformat()
         end_str = self.dh.this_month_end.isoformat()
@@ -132,7 +121,6 @@ def test_update_daily_summary_tables_no_invoice_month(
         mock_delete.assert_not_called()
         mock_trino.assert_not_called()
         mock_tag_update.assert_not_called()
-        mock_summary_update.assert_not_called()
 
         self.assertEqual(start_return, start)
         self.assertEqual(end_return, end)