[COST-4807] Make provider_uuid not a query param and parameterize bigquery queries (#5005)

* update monthly_sql query to be parameterized

* remove custom SQL

* make source uuid part of the URL instead of a param

---------

Co-authored-by: Michael Skarbek <[email protected]>

Author: cgoodfred

koku/masu/api/bigquery_cost.py

@@ -48,13 +48,19 @@ def daily_sql(self, invoice_month, results):
                 SELECT DATE(usage_start_time) as usage_date,
                     sum(cost) as cost,
                     FROM {self.table_name}
-                    WHERE invoice.month = '{invoice_month}'
-                    AND DATE(_PARTITIONTIME) BETWEEN "{str(self.start_date)}" AND "{str(self.end_date)}"
+                    WHERE invoice.month = @invoice_month
+                    AND DATE(_PARTITIONTIME) BETWEEN @start_date AND @end_date
                     GROUP BY usage_date ORDER BY usage_date;
                 """
-
+        job_config = bigquery.QueryJobConfig(
+            query_parameters=[
+                bigquery.ScalarQueryParameter("invoice_month", "STRING", invoice_month),
+                bigquery.ScalarQueryParameter("start_date", "DATE", self.start_date),
+                bigquery.ScalarQueryParameter("end_date", "DATE", self.end_date),
+            ]
+        )
         LOG.info(daily_query)
-        rows = self.client.query(daily_query).result()
+        rows = self.client.query(daily_query, job_config=job_config).result()
         daily_dict = {}
         for row in rows:
             daily_dict[str(row["usage_date"])] = {"cost": row.get("cost")}
@@ -66,10 +72,17 @@ def monthly_sql(self, invoice_month, results, key):
         monthly_query = f"""
                 SELECT sum(cost) as cost,
                     FROM {self.table_name}
-                    WHERE DATE(_PARTITIONTIME) BETWEEN "{str(self.start_date)}" AND "{str(self.end_date)}"
-                    AND invoice.month = '{invoice_month}'
+                    WHERE DATE(_PARTITIONTIME) BETWEEN @start_date AND @end_date
+                    AND invoice.month = @invoice_month
                 """
-        rows = self.client.query(monthly_query).result()
+        job_config = bigquery.QueryJobConfig(
+            query_parameters=[
+                bigquery.ScalarQueryParameter("invoice_month", "STRING", invoice_month),
+                bigquery.ScalarQueryParameter("start_date", "DATE", self.start_date),
+                bigquery.ScalarQueryParameter("end_date", "DATE", self.end_date),
+            ]
+        )
+        rows = self.client.query(monthly_query, job_config=job_config).result()
         for row in rows:
             results[key] = row[0]  # TODO: Remove once bigquery is updated.
             metadata = {
@@ -79,30 +92,22 @@ def monthly_sql(self, invoice_month, results, key):
             results[key + "_metadata"] = metadata
         return results
 
-    def custom_sql(self, query):
-        """Takes a custom query and replaces the table_name."""
-        query = query.replace("table_name", self.table_name)
-        rows = self.client.query(query).result()
-        return rows
-
 
 @never_cache
-@api_view(http_method_names=["GET", "POST"])
+@api_view(http_method_names=["GET"])
 @permission_classes((AllowAny,))
 @renderer_classes(tuple(api_settings.DEFAULT_RENDERER_CLASSES))
-def bigquery_cost(request):  # noqa: C901
+def bigquery_cost(request, *args, **kwargs):  # noqa: C901
     """Returns the invoice monthly cost."""
     params = request.query_params
-    provider_uuid = params.get("provider_uuid")
-    return_daily = True if "daily" in params.keys() else False
-
-    if provider_uuid is None:
-        errmsg = "provider_uuid is a required parameter."
-        return Response({"Error": errmsg}, status=status.HTTP_400_BAD_REQUEST)
+    provider_uuid = kwargs.get("source_uuid")
+    return_daily = "daily" in params.keys()
 
-    provider = Provider.objects.filter(uuid=provider_uuid).first()
+    provider = Provider.objects.filter(
+        uuid=provider_uuid, type__in=[Provider.PROVIDER_GCP, Provider.PROVIDER_GCP_LOCAL]
+    ).first()
     if not provider:
-        errmsg = f"The provider_uuid {provider_uuid} does not exist."
+        errmsg = f"The *GCP* provider_uuid {provider_uuid} does not exist."
         return Response({"Error": errmsg}, status=status.HTTP_400_BAD_REQUEST)
     credentials = provider.authentication.credentials
     data_source = provider.billing_source.data_source
@@ -122,19 +127,13 @@ def bigquery_cost(request):  # noqa: C901
     results = {}
     try:
         bq_helper = BigQueryHelper(table_name)
-        if request.method == "POST":
-            data = request.data
-            query = data.get("query")
-            results = bq_helper.custom_sql(query)
-            resp_key = "custom_query_results"
-        else:
-            for key, invoice_month in mapping.items():
-                if return_daily:
-                    results = bq_helper.daily_sql(invoice_month, results)
-                    resp_key = "daily_invoice_cost_mapping"
-                else:
-                    results = bq_helper.monthly_sql(invoice_month, results, key)
-                    resp_key = "monthly_invoice_cost_mapping"
+        for key, invoice_month in mapping.items():
+            if return_daily:
+                results = bq_helper.daily_sql(invoice_month, results)
+                resp_key = "daily_invoice_cost_mapping"
+            else:
+                results = bq_helper.monthly_sql(invoice_month, results, key)
+                resp_key = "monthly_invoice_cost_mapping"
     except GoogleCloudError as err:
         return Response({"Error": err.message}, status=status.HTTP_400_BAD_REQUEST)
 

koku/masu/api/urls.py

@@ -73,10 +73,7 @@
     path("celery_queue_tasks/", celery_queue_tasks, name="celery_queue_tasks"),
     path("celery_queue_lengths/", celery_queue_lengths, name="celery_queue_lengths"),
     path("clear_celery_queues/", clear_celery_queues, name="clear_celery_queues"),
-    path(
-        "gcp_invoice_monthly_cost/", bigquery_cost, name="gcp_invoice_monthly_cost"
-    ),  # TODO: Remove once iqe is updated
-    path("bigquery_cost/", bigquery_cost, name="bigquery_cost"),
+    path("bigquery_cost/<uuid:source_uuid>/", bigquery_cost, name="bigquery_cost"),
     path("purge_trino_files/", purge_trino_files, name="purge_trino_files"),
     path("db-performance", db_performance_redirect, name="db_perf_no_slash_redirect"),
     path("db-performance/", db_performance_redirect, name="db_perf_slash_redirect"),

koku/masu/openapi.json

@@ -1593,50 +1593,15 @@
           ]
         }
       },
-      "/gcp_invoice_monthly_cost/": {
-      "get": {
-        "summary": "Returns dictionary with cost mappings for continuity checks",
-        "operationId": "gcpInvoiceMonthlyCost",
-        "description": "Returns a dictionary with cost mappings for continuity checks",
-        "parameters": [
-          {
-            "name": "provider_uuid",
-            "in": "query",
-            "description": "The provider UUID",
-            "required": true,
-            "schema": {
-              "type": "string",
-              "format": "uuid",
-              "example": "83ee048e-3c1d-43ef-b945-108225ae52f4"
-            }
-          }
-        ],
-        "responses": {
-          "200": {
-            "description": "Returns a dictionary with cost mappings for continuity checks",
-            "content": {
-              "application/json": {
-                "schema": {
-                  "$ref": "#/components/schemas/gcpInvoiceMonthlyCost"
-                }
-              }
-            }
-          }
-        },
-        "tags": [
-          "GCP Invoice Monthly Cost"
-        ]
-      }
-    },
-    "/bigquery_cost/": {
+    "/bigquery_cost/{provider_uuid}/": {
       "get": {
         "summary": "Returns dictionary with cost mappings for continuity checks",
         "operationId": "BigQueryCost",
         "description": "Returns a dictionary with cost mappings for continuity checks",
         "parameters": [
           {
             "name": "provider_uuid",
-            "in": "query",
+            "in": "path",
             "description": "The provider UUID",
             "required": true,
             "schema": {

koku/masu/test/api/test_bigquery_cost.py

@@ -5,12 +5,12 @@
 """Test the update_cost_model_costs endpoint view."""
 from collections import OrderedDict
 from unittest.mock import patch
-from urllib.parse import urlencode
 from uuid import uuid4
 
 from django.test.utils import override_settings
 from django.urls import reverse
 
+from api.models import Provider
 from masu.api.bigquery_cost import get_total
 from masu.test import MasuTestCase
 
@@ -23,13 +23,12 @@ class BigQueryCostTest(MasuTestCase):
     def test_success_gcp_monthly_return(self, _):
         """Test successful endpoint return"""
         expected_value = 308.45
-        dict = OrderedDict()
-        dict["cost"] = expected_value
-        dict[0] = expected_value
-        mocked_value = [dict]
-        params = {"provider_uuid": self.gcp_provider_uuid}
-        query_string = urlencode(params)
-        url = reverse("bigquery_cost") + "?" + query_string
+        expected_dict = OrderedDict(
+            cost=expected_value,
+        )
+        expected_dict[0] = expected_value
+        mocked_value = [expected_dict]
+        url = reverse("bigquery_cost", kwargs={"source_uuid": self.gcp_provider_uuid})
         with patch("masu.api.bigquery_cost.bigquery") as bigquery:
             bigquery.Client.return_value.query.return_value.result.return_value = mocked_value
             response = self.client.get(url)
@@ -42,51 +41,39 @@ def test_success_gcp_daily_return(self, _):
         """Test successful endpoint return"""
         expected_cost = 308.45
         expected_credit = 10
-        dict = OrderedDict()
-        dict["usage_date"] = "2022-08-01"
-        dict["cost"] = expected_cost
-        dict["credit_amount"] = expected_credit
-        dict[0] = expected_cost + expected_credit
-        mocked_value = [dict]
-        params = {"provider_uuid": self.gcp_provider_uuid}
-        query_string = urlencode(params)
-        url = reverse("bigquery_cost") + "?" + query_string + "&daily"
+        expected_dict = OrderedDict(
+            usage_date="2022-08-01",
+            cost=expected_cost,
+            credit_amount=expected_credit,
+        )
+        expected_dict[0] = expected_cost + expected_credit
+        mocked_value = [expected_dict]
+        url = reverse("bigquery_cost", kwargs={"source_uuid": self.gcp_provider_uuid}) + "?daily"
         with patch("masu.api.bigquery_cost.bigquery") as bigquery:
             bigquery.Client.return_value.query.return_value.result.return_value = mocked_value
             response = self.client.get(url)
         body = response.json()
         self.assertIsNotNone(body)
 
-    @patch("koku.middleware.MASU", return_value=True)
-    def test_require_provider_uuid(self, _):
-        """Test the GET bigquery_cost endpoint with no provider uuid."""
-        response = self.client.get(reverse("bigquery_cost"))
-        body = response.json()
-        errmsg = body.get("Error")
-        expected_errmsg = "provider_uuid is a required parameter."
-        self.assertEqual(response.status_code, 400)
-        self.assertEqual(errmsg, expected_errmsg)
-
     @patch("koku.middleware.MASU", return_value=True)
     def test_bigquery_request_with_bad_provider_uuid(self, _):
         """Test the GET bigquery_cost endpoint with incorrect provider uuid."""
         provider_uuid = uuid4()
-        params = {"provider_uuid": provider_uuid}
-        query_string = urlencode(params)
-        url = reverse("bigquery_cost") + "?" + query_string
+        url = reverse("bigquery_cost", kwargs={"source_uuid": provider_uuid})
         response = self.client.get(url)
         body = response.json()
         errmsg = body.get("Error")
-        expected_errmsg = f"The provider_uuid {provider_uuid} does not exist."
+        expected_errmsg = f"The *GCP* provider_uuid {provider_uuid} does not exist."
         self.assertEqual(response.status_code, 400)
         self.assertEqual(errmsg, expected_errmsg)
 
     @patch("koku.middleware.MASU", return_value=True)
     def test_unable_to_build_gcp_table_name(self, _):
         """Test the GET bigquery_cost endpoint with no provider uuid."""
-        params = {"provider_uuid": self.aws_provider_uuid}
-        query_string = urlencode(params)
-        url = reverse("bigquery_cost") + "?" + query_string
+        p = Provider.objects.get(uuid=self.gcp_provider_uuid)
+        p.authentication.credentials = {}
+        p.authentication.save()
+        url = reverse("bigquery_cost", kwargs={"source_uuid": self.gcp_provider_uuid})
         response = self.client.get(url)
         body = response.json()
         errmsg = body.get("Error")