DR-3313: Add POST endpoint for retrieving data in order to enable lar…

…ger filter queries in UI (#1535) * wip * Update Tests to pass with post endpoint - Update DatasetsApiControllerTest - Fix snapshot unit tests * Add back GET endpoints and mark as deprecated * Have to rename the post method because it can't be the same as the get method; Do the same for column stats endpoint * Rename new endpoints; Make old endpoints "hidden" Rename to QueryDataRequestModel * Update Datasets Api Controller * Add SnapshotsApiControllerTest * Remove tests for deprecated endpoints * Fix test * Rename Lookup to Query for Column Stats Request * Spotless * spotless * Revert sharing test code * Remove hidden annotation * Cleanup - We handle default direction now Fix test * Add back empty direction handling spotless * Update datasetsApiControllerTest to still test deprecated endpoints * Make snapshots api controller parameterzied test; share query and lookup code in datasets and snapshots * bad merge
DataBiosphere · Nov 27, 2023 · 45159af · 45159af
1 parent 2f9ea0c
commit 45159af
Show file tree

Hide file tree

Showing 8 changed files with 465 additions and 137 deletions.
diff --git a/src/main/java/bio/terra/app/controller/DatasetsApiController.java b/src/main/java/bio/terra/app/controller/DatasetsApiController.java
@@ -34,6 +34,8 @@
 import bio.terra.model.PolicyMemberRequest;
 import bio.terra.model.PolicyModel;
 import bio.terra.model.PolicyResponse;
+import bio.terra.model.QueryColumnStatisticsRequestModel;
+import bio.terra.model.QueryDataRequestModel;
 import bio.terra.model.SamPolicyModel;
 import bio.terra.model.SnapshotBuilderAccessRequest;
 import bio.terra.model.SnapshotBuilderGetConceptsResponse;
@@ -80,7 +82,6 @@
 @Controller
 @Api(tags = {"datasets"})
 public class DatasetsApiController implements DatasetsApi {
-
   public static final String RETRIEVE_INCLUDE_DEFAULT_VALUE = "SCHEMA,PROFILE,DATA_PROJECT,STORAGE";
 
   private final HttpServletRequest request;
@@ -195,6 +196,27 @@ public ResponseEntity<SnapshotBuilderGetConceptsResponse> getConcepts(
     return ResponseEntity.ok(snapshotBuilderService.getConceptChildren(id, conceptId));
   }
 
+  @Override
+  public ResponseEntity<DatasetDataModel> queryDatasetDataById(
+      UUID id, String table, QueryDataRequestModel queryDataRequest) {
+    AuthenticatedUserRequest userReq = getAuthenticatedInfo();
+    verifyDatasetAuthorization(userReq, id.toString(), IamAction.READ_DATA);
+    // TODO: Remove after https://broadworkbench.atlassian.net/browse/DR-2588 is fixed
+    SqlSortDirection sortDirection =
+        Objects.requireNonNullElse(queryDataRequest.getDirection(), SqlSortDirection.ASC);
+    DatasetDataModel previewModel =
+        datasetService.retrieveData(
+            userReq,
+            id,
+            table,
+            queryDataRequest.getLimit(),
+            queryDataRequest.getOffset(),
+            queryDataRequest.getSort(),
+            sortDirection,
+            queryDataRequest.getFilter());
+    return ResponseEntity.ok(previewModel);
+  }
+
   @Override
   public ResponseEntity<DatasetDataModel> lookupDatasetDataById(
       UUID id,
@@ -204,22 +226,31 @@ public ResponseEntity<DatasetDataModel> lookupDatasetDataById(
       String sort,
       SqlSortDirection direction,
       String filter) {
-    AuthenticatedUserRequest userReq = getAuthenticatedInfo();
-    verifyDatasetAuthorization(userReq, id.toString(), IamAction.READ_DATA);
-    // TODO: Remove after https://broadworkbench.atlassian.net/browse/DR-2588 is fixed
-    SqlSortDirection sortDirection = Objects.requireNonNullElse(direction, SqlSortDirection.ASC);
-    DatasetDataModel previewModel =
-        datasetService.retrieveData(userReq, id, table, limit, offset, sort, sortDirection, filter);
-    return ResponseEntity.ok(previewModel);
+    var request =
+        new QueryDataRequestModel()
+            .offset(offset)
+            .limit(limit)
+            .sort(sort)
+            .direction(direction)
+            .filter(filter);
+    return queryDatasetDataById(id, table, request);
   }
 
   @Override
   public ResponseEntity<ColumnStatisticsModel> lookupDatasetColumnStatisticsById(
       UUID id, String table, String column, String filter) {
+    return queryDatasetColumnStatisticsById(
+        id, table, column, new QueryColumnStatisticsRequestModel().filter(filter));
+  }
+
+  @Override
+  public ResponseEntity<ColumnStatisticsModel> queryDatasetColumnStatisticsById(
+      UUID id, String table, String column, QueryColumnStatisticsRequestModel requestModel) {
     AuthenticatedUserRequest userReq = getAuthenticatedInfo();
     verifyDatasetAuthorization(userReq, id.toString(), IamAction.READ_DATA);
     ColumnStatisticsModel columnStatisticsModel =
-        datasetService.retrieveColumnStatistics(userReq, id, table, column, filter);
+        datasetService.retrieveColumnStatistics(
+            userReq, id, table, column, requestModel.getFilter());
     return ResponseEntity.ok(columnStatisticsModel);
   }
 

diff --git a/src/main/java/bio/terra/app/controller/SnapshotsApiController.java b/src/main/java/bio/terra/app/controller/SnapshotsApiController.java
@@ -16,6 +16,7 @@
 import bio.terra.model.PolicyMemberRequest;
 import bio.terra.model.PolicyModel;
 import bio.terra.model.PolicyResponse;
+import bio.terra.model.QueryDataRequestModel;
 import bio.terra.model.SnapshotIdsAndRolesModel;
 import bio.terra.model.SnapshotLinkDuosDatasetResponse;
 import bio.terra.model.SnapshotModel;
@@ -265,6 +266,26 @@ public ResponseEntity<FileModel> lookupSnapshotFileByPath(
     return ResponseEntity.ok(fileModel);
   }
 
+  @Override
+  public ResponseEntity<SnapshotPreviewModel> querySnapshotDataById(
+      UUID id, String table, QueryDataRequestModel queryDataRequest) {
+    snapshotService.verifySnapshotReadable(id, getAuthenticatedInfo());
+    // TODO: Remove after https://broadworkbench.atlassian.net/browse/DR-2588 is fixed
+    SqlSortDirection sortDirection =
+        Objects.requireNonNullElse(queryDataRequest.getDirection(), SqlSortDirection.ASC);
+    SnapshotPreviewModel previewModel =
+        snapshotService.retrievePreview(
+            getAuthenticatedInfo(),
+            id,
+            table,
+            queryDataRequest.getLimit(),
+            queryDataRequest.getOffset(),
+            queryDataRequest.getSort(),
+            sortDirection,
+            queryDataRequest.getFilter());
+    return ResponseEntity.ok(previewModel);
+  }
+
   @Override
   public ResponseEntity<SnapshotPreviewModel> lookupSnapshotPreviewById(
       UUID id,
@@ -274,15 +295,15 @@ public ResponseEntity<SnapshotPreviewModel> lookupSnapshotPreviewById(
       String sort,
       SqlSortDirection direction,
       String filter) {
-    logger.debug("Verifying user access");
-    snapshotService.verifySnapshotReadable(id, getAuthenticatedInfo());
-    logger.debug("Retrieving snapshot id {}", id);
-    // TODO: Remove after https://broadworkbench.atlassian.net/browse/DR-2588 is fixed
-    SqlSortDirection sortDirection = Objects.requireNonNullElse(direction, SqlSortDirection.ASC);
-    SnapshotPreviewModel previewModel =
-        snapshotService.retrievePreview(
-            getAuthenticatedInfo(), id, table, limit, offset, sort, sortDirection, filter);
-    return ResponseEntity.ok(previewModel);
+    return querySnapshotDataById(
+        id,
+        table,
+        new QueryDataRequestModel()
+            .offset(offset)
+            .limit(limit)
+            .sort(sort)
+            .direction(direction)
+            .filter(filter));
   }
 
   // --snapshot auth domains --

diff --git a/src/main/resources/api/data-repository-openapi.yaml b/src/main/resources/api/data-repository-openapi.yaml
@@ -826,7 +826,8 @@ paths:
         - snapshots
         - search
         - repository
-      description: Retrieve data for a table in a snapshot
+      deprecated: true
+      description: Retrieve data for a table in a snapshot. This endpoint is deprecated, please use the POST version.
       operationId: lookupSnapshotPreviewById
       parameters:
         - $ref: '#/components/parameters/Id'
@@ -875,6 +876,34 @@ paths:
             application/json:
               schema:
                 $ref: '#/components/schemas/SnapshotPreviewModel'
+    post:
+      tags:
+        - snapshots
+        - search
+        - repository
+      description: Retrieve data for a table in a snapshot.
+      operationId: querySnapshotDataById
+      parameters:
+        - $ref: '#/components/parameters/Id'
+        - name: table
+          in: path
+          description: Name of table to get data from
+          required: true
+          schema:
+            type: string
+      requestBody:
+        description: Parameters to filter results
+        content:
+          application/json:
+            schema:
+              $ref: '#/components/schemas/QueryDataRequestModel'
+      responses:
+        200:
+          description: Returns the table data from a snapshot
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/SnapshotPreviewModel'
   /api/repository/v1/snapshots/{id}/export:
     get:
       tags:
@@ -1898,7 +1927,8 @@ paths:
         - datasets
         - search
         - repository
-      description: Retrieve data for a table in a dataset
+      deprecated: true
+      description: Retrieve data for a table in a dataset. This endpoint is deprecated, please use the POST version.
       operationId: lookupDatasetDataById
       parameters:
         - $ref: '#/components/parameters/Id'
@@ -1947,11 +1977,42 @@ paths:
             application/json:
               schema:
                 $ref: '#/components/schemas/DatasetDataModel'
+    post:
+      tags:
+        - datasets
+        - search
+        - repository
+      description: Retrieve data for a table in a dataset.
+      operationId: queryDatasetDataById
+      parameters:
+        - $ref: '#/components/parameters/Id'
+        - name: table
+          in: path
+          description: Name of table to get data from
+          required: true
+          schema:
+            type: string
+      requestBody:
+        description: Parameters to filter results
+        content:
+          application/json:
+            schema:
+              $ref: '#/components/schemas/QueryDataRequestModel'
+        required: false
+      responses:
+        200:
+          description: Returns the table data from a dataset
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/DatasetDataModel'
   /api/repository/v1/datasets/{id}/data/{table}/statistics/{column}:
     get:
       tags:
         - datasets
-      description: Retrieve statiscs about data for a column in a table in a dataset
+      deprecated: true
+      description: >-
+        Retrieve statistics about data for a column in a table in a dataset. This endpoint is deprecated, please use the POST version.
       operationId: lookupDatasetColumnStatisticsById
       parameters:
         - $ref: '#/components/parameters/Id'
@@ -1963,7 +2024,7 @@ paths:
             type: string
         - name: column
           in: path
-          description: Name of column in the table to get statisitcs about
+          description: Name of column in the table to get statistics about
           required: true
           schema:
             type: string
@@ -1976,7 +2037,8 @@ paths:
             For GCP array string columns, if you wanted to include all rows that contain 'value1' in column1,
             the filter clause would look like 'WHERE 'value1' IN UNNEST(column1)'. Note that "count" value
             includes all occurrences of a value including duplicates of the same value in a single array.
-            i.e. if we had two rows in a table where the value for column1, row1 = ['value1', 'value1', 'value2'] and column1, row2 = ['value1'] the count for 'value1' would be 3.
+            i.e. if we had two rows in a table where the value for column1, row1 = ['value1', 'value1', 'value2']
+            and column1, row2 = ['value1'] the count for 'value1' would be 3.
           schema:
             type: string
       responses:
@@ -1986,6 +2048,38 @@ paths:
             application/json:
               schema:
                 $ref: '#/components/schemas/ColumnStatisticsModel'
+    post:
+      tags:
+        - datasets
+      description: Retrieve statistics about data for a column in a table in a dataset.
+      operationId: queryDatasetColumnStatisticsById
+      parameters:
+        - $ref: '#/components/parameters/Id'
+        - name: table
+          in: path
+          description: Name of table where column lives
+          required: true
+          schema:
+            type: string
+        - name: column
+          in: path
+          description: Name of column in the table to get statistics about
+          required: true
+          schema:
+            type: string
+      requestBody:
+        description: Parameters to filter results
+        content:
+          application/json:
+            schema:
+              $ref: '#/components/schemas/QueryColumnStatisticsRequestModel'
+      responses:
+        200:
+          description: Returns the statistics about a column from a dataset's table
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/ColumnStatisticsModel'
   /api/repository/v1/datasets/{id}/summary:
     get:
       tags:
@@ -5160,6 +5254,46 @@ components:
         dataset bucket.
         Eventually, this will include attributes of the storage including
         billing, temperature, geography, etc. But for now...
+    QueryDataRequestModel:
+      type: object
+      properties:
+        offset:
+          description: The number of rows to skip when retrieving the next page
+          type: integer
+          default: 0
+          minimum: 0
+        limit:
+          description: The number of rows to return for the data
+          type: integer
+          default: 30
+          minimum: 1
+          maximum: 1000
+        sort:
+          description: The table column to sort by
+          type: string
+          default: "datarepo_row_id"
+        direction:
+          $ref: '#/components/schemas/SqlSortDirection'
+          description: The direction to sort.
+          default: asc
+        filter:
+          description: A SQL WHERE clause to filter the table results.
+          type: string
+          default: ""
+    QueryColumnStatisticsRequestModel:
+      type: object
+      properties:
+        filter:
+          description: >-
+            A SQL WHERE clause to filter results included in column statistics.
+
+
+            For GCP array string columns, if you wanted to include all rows that contain 'value1' in column1,
+            the filter clause would look like 'WHERE 'value1' IN UNNEST(column1)'. Note that "count" value
+            includes all occurrences of a value including duplicates of the same value in a single array.
+            i.e. if we had two rows in a table where the value for column1, row1 = ['value1', 'value1', 'value2']
+            and column1, row2 = ['value1'] the count for 'value1' would be 3.
+          type: string
     DatasetDataModel:
       type: object
       properties: