Merge remote-tracking branch 'origin/master' into load-indices

Author: david-leifker

.github/workflows/build-and-test.yml

@@ -64,8 +64,15 @@ jobs:
       - name: Free up disk space
         run: |
           sudo apt-get remove 'dotnet-*' azure-cli || true
+          sudo rm -rf /usr/local/.ghcup || true
+          sudo rm -rf /usr/share/dotnet || true
+          sudo rm -rf /usr/share/swift || true
+          sudo rm -rf /usr/local/julia* || true
+          sudo rm -rf /usr/local/share/powershell || true
+          sudo rm -rf /usr/share/miniconda || true
           sudo rm -rf /usr/local/lib/android/ || true
-          sudo docker image prune -a -f || true
+          sudo docker system prune -a -f || true
+          df -h
       - uses: szenius/[email protected]
         with:
           timezoneLinux: ${{ matrix.timezone }}
@@ -89,6 +96,13 @@ jobs:
           distribution: "zulu"
           java-version: 17
       - uses: gradle/actions/setup-gradle@v4
+      - name: Disk Space Analysis
+        run: |
+          echo "=== Disk Usage Overview ==="
+          df -h
+
+          echo -e "\n=== Docker Disk Usage ==="
+          docker system df -v
       - name: Gradle build (and test) for NOT metadata ingestion
         if: ${{  matrix.command == 'except_metadata_ingestion' && needs.setup.outputs.backend_change == 'true' }}
         # datahub-schematron:cli excluded due to dependency on metadata-ingestion
@@ -112,6 +126,13 @@ jobs:
             -x :metadata-integration:java:datahub-schematron:cli:test
         env:
           CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
+      - name: Disk Space Analysis
+        run: |
+          echo "=== Disk Usage Overview ==="
+          df -h
+
+          echo -e "\n=== Docker Disk Usage ==="
+          docker system df -v
       - name: Gradle build (and test) for frontend
         if: ${{  matrix.command == 'frontend' && needs.setup.outputs.frontend_change == 'true' }}
         run: |

.github/workflows/docker-unified.yml

@@ -422,11 +422,17 @@ jobs:
       MIXPANEL_PROJECT_ID: ${{ secrets.MIXPANEL_PROJECT_ID }}
     steps:
       - name: Free up disk space
-        if: ${{ !contains(needs.setup.outputs.test_runner_type, 'depot') }}
         run: |
           sudo apt-get remove 'dotnet-*' azure-cli || true
+          sudo rm -rf /usr/local/.ghcup || true
+          sudo rm -rf /usr/share/dotnet || true
+          sudo rm -rf /usr/share/swift || true
+          sudo rm -rf /usr/local/julia* || true
+          sudo rm -rf /usr/local/share/powershell || true
+          sudo rm -rf /usr/share/miniconda || true
           sudo rm -rf /usr/local/lib/android/ || true
-          sudo docker image prune -a -f || true
+          sudo docker system prune -a -f || true
+          df -h
 
       - uses: actions/cache/restore@v4
         with:
@@ -468,6 +474,14 @@ jobs:
           username: ${{ secrets.ACRYL_DOCKER_USERNAME }}
           password: ${{ secrets.ACRYL_DOCKER_PASSWORD }}
 
+      - name: Disk Space Analysis
+        run: |
+          echo "=== Disk Usage Overview ==="
+          df -h
+
+          echo -e "\n=== Docker Disk Usage ==="
+          docker system df -v
+
       - name: build images
         if: ${{ needs.setup.outputs.use_depot_cache != 'true' }}
         run: |
@@ -482,6 +496,13 @@ jobs:
           depot pull --project ${{ env.DEPOT_PROJECT_ID }} ${{ needs.base_build.outputs.build_id }}
           docker images
 
+      - name: Disk Space Analysis
+        run: |
+          echo "=== Disk Usage Overview ==="
+          df -h
+
+          echo -e "\n=== Docker Disk Usage ==="
+          docker system df -v
       - name: run quickstart
         env:
           DATAHUB_TELEMETRY_ENABLED: false

build.gradle

@@ -46,6 +46,7 @@ buildscript {
   ext.neo4jApocVersion = '5.20.0'
   ext.testContainersVersion = '1.21.1'
   ext.elasticsearchVersion = '2.11.1' // ES 7.10, Opensearch 1.x, 2.x
+  ext.elasticsearch8Version = '8.17.4' // ES 8.x Java client
   ext.jacksonVersion = '2.18.4'
   ext.jettyVersion = '12.0.21'
   // see also datahub-frontend/play.gradle
@@ -151,6 +152,8 @@ project.ext.externalDependency = [
     'ebeanDdl': 'io.ebean:ebean-ddl-generator:' + ebeanVersion,
     'ebeanQueryBean': 'io.ebean:querybean-generator:' + ebeanVersion,
     'elasticSearchRest': 'org.opensearch.client:opensearch-rest-high-level-client:' + elasticsearchVersion,
+    // Multi-client shim dependencies
+    'elasticsearch8Client': 'co.elastic.clients:elasticsearch-java:' + elasticsearch8Version,
     'findbugsAnnotations': 'com.google.code.findbugs:annotations:3.0.1',
     'graphqlJava': 'com.graphql-java:graphql-java:22.3',
     'graphqlJavaScalars': 'com.graphql-java:graphql-java-extended-scalars:22.0',

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/analytics/service/AnalyticsService.java

@@ -13,6 +13,7 @@
 import com.linkedin.datahub.graphql.types.entitytype.EntityTypeMapper;
 import com.linkedin.metadata.datahubusage.DataHubUsageEventConstants;
 import com.linkedin.metadata.utils.elasticsearch.IndexConvention;
+import com.linkedin.metadata.utils.elasticsearch.SearchClientShim;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
@@ -24,7 +25,6 @@
 import org.opensearch.action.search.SearchRequest;
 import org.opensearch.action.search.SearchResponse;
 import org.opensearch.client.RequestOptions;
-import org.opensearch.client.RestHighLevelClient;
 import org.opensearch.index.query.BoolQueryBuilder;
 import org.opensearch.index.query.QueryBuilder;
 import org.opensearch.index.query.QueryBuilders;
@@ -45,7 +45,7 @@
 @RequiredArgsConstructor
 public class AnalyticsService {
 
-  private final RestHighLevelClient _elasticClient;
+  private final SearchClientShim<?> _elasticClient;
   private final IndexConvention _indexConvention;
 
   private static final String FILTERED = "filtered";

datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/config/NonBlockingConfigs.java

@@ -19,8 +19,8 @@
 import com.linkedin.metadata.search.SearchService;
 import com.linkedin.metadata.search.elasticsearch.ElasticSearchService;
 import com.linkedin.metadata.search.elasticsearch.update.ESWriteDAO;
+import com.linkedin.metadata.utils.elasticsearch.SearchClientShim;
 import io.datahubproject.metadata.context.OperationContext;
-import org.opensearch.client.RestHighLevelClient;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Qualifier;
 import org.springframework.beans.factory.annotation.Value;
@@ -76,7 +76,7 @@ public NonBlockingSystemUpgrade backfillProcessInstancesHasRunEvents(
       final OperationContext opContext,
       EntityService<?> entityService,
       ElasticSearchService elasticSearchService,
-      RestHighLevelClient restHighLevelClient,
+      SearchClientShim<?> restHighLevelClient,
       @Value("${systemUpdate.processInstanceHasRunEvents.enabled}") final boolean enabled,
       @Value("${systemUpdate.processInstanceHasRunEvents.reprocess.enabled}")
           boolean reprocessEnabled,

datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/loadindices/LoadIndicesIndexManager.java

@@ -3,6 +3,8 @@
 import com.linkedin.metadata.models.AspectSpec;
 import com.linkedin.metadata.models.registry.EntityRegistry;
 import com.linkedin.metadata.utils.elasticsearch.IndexConvention;
+import com.linkedin.metadata.utils.elasticsearch.SearchClientShim;
+import com.linkedin.metadata.utils.elasticsearch.responses.GetIndexResponse;
 import java.io.IOException;
 import java.util.HashSet;
 import java.util.Set;
@@ -11,9 +13,7 @@
 import org.opensearch.action.admin.indices.settings.get.GetSettingsResponse;
 import org.opensearch.action.admin.indices.settings.put.UpdateSettingsRequest;
 import org.opensearch.client.RequestOptions;
-import org.opensearch.client.RestHighLevelClient;
 import org.opensearch.client.indices.GetIndexRequest;
-import org.opensearch.client.indices.GetIndexResponse;
 import org.opensearch.common.settings.Settings;
 
 /**
@@ -26,7 +26,7 @@ public class LoadIndicesIndexManager {
   private static final String REFRESH_INTERVAL_SETTING = "index.refresh_interval";
   private static final String DISABLED_REFRESH_INTERVAL = "-1";
 
-  private final RestHighLevelClient searchClient;
+  private final SearchClientShim<?> searchClient;
   private final IndexConvention indexConvention;
   private final EntityRegistry entityRegistry;
   private final String configuredRefreshInterval;
@@ -35,7 +35,7 @@ public class LoadIndicesIndexManager {
   private boolean indicesDiscovered = false;
 
   public LoadIndicesIndexManager(
-      RestHighLevelClient searchClient,
+      SearchClientShim<?> searchClient,
       IndexConvention indexConvention,
       EntityRegistry entityRegistry,
       String configuredRefreshInterval) {
@@ -56,7 +56,7 @@ public Set<String> discoverDataHubIndices() throws IOException {
 
     // Get all existing indices
     GetIndexRequest request = new GetIndexRequest("*");
-    GetIndexResponse response = searchClient.indices().get(request, RequestOptions.DEFAULT);
+    GetIndexResponse response = searchClient.getIndex(request, RequestOptions.DEFAULT);
     String[] allIndices = response.getIndices();
 
     log.info("Found {} total indices in Elasticsearch", allIndices.length);
@@ -201,8 +201,7 @@ private String getIndexRefreshInterval(String indexName) throws IOException {
             .includeDefaults(true)
             .names(REFRESH_INTERVAL_SETTING);
 
-    GetSettingsResponse response =
-        searchClient.indices().getSettings(request, RequestOptions.DEFAULT);
+    GetSettingsResponse response = searchClient.getIndexSettings(request, RequestOptions.DEFAULT);
     return response.getSetting(indexName, REFRESH_INTERVAL_SETTING);
   }
 
@@ -212,7 +211,7 @@ private void setIndexRefreshInterval(String indexName, String interval) throws I
     Settings settings = Settings.builder().put(REFRESH_INTERVAL_SETTING, interval).build();
     request.settings(settings);
 
-    searchClient.indices().putSettings(request, RequestOptions.DEFAULT);
+    searchClient.updateIndexSettings(request, RequestOptions.DEFAULT);
   }
 
   /** Returns true if refresh intervals are currently disabled. */

datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/loadindices/config/LoadIndicesConfig.java

@@ -12,11 +12,11 @@
 import com.linkedin.metadata.service.UpdateIndicesService;
 import com.linkedin.metadata.systemmetadata.SystemMetadataService;
 import com.linkedin.metadata.timeseries.TimeseriesAspectService;
+import com.linkedin.metadata.utils.elasticsearch.SearchClientShim;
 import io.datahubproject.metadata.context.OperationContext;
 import io.ebean.Database;
 import javax.annotation.Nonnull;
 import lombok.extern.slf4j.Slf4j;
-import org.opensearch.client.RestHighLevelClient;
 import org.springframework.beans.factory.annotation.Qualifier;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean;
@@ -47,7 +47,7 @@ public KafkaEventProducer noOpKafkaEventProducer() {
   @Nonnull
   public LoadIndicesIndexManager createIndexManager(
       @Qualifier("systemOperationContext") final OperationContext systemOperationContext,
-      @Qualifier("elasticSearchRestHighLevelClient") final RestHighLevelClient searchClient,
+      @Qualifier("searchClientShim") SearchClientShim<?> searchClient,
       @Value("${elasticsearch.index.refreshIntervalSeconds:3}")
           final String configuredRefreshInterval)
       throws Exception {

datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/dataprocessinstances/BackfillDataProcessInstances.java

@@ -5,9 +5,9 @@
 import com.linkedin.datahub.upgrade.system.NonBlockingSystemUpgrade;
 import com.linkedin.metadata.entity.EntityService;
 import com.linkedin.metadata.search.elasticsearch.ElasticSearchService;
+import com.linkedin.metadata.utils.elasticsearch.SearchClientShim;
 import io.datahubproject.metadata.context.OperationContext;
 import java.util.List;
-import org.opensearch.client.RestHighLevelClient;
 
 public class BackfillDataProcessInstances implements NonBlockingSystemUpgrade {
 
@@ -17,7 +17,7 @@ public BackfillDataProcessInstances(
       OperationContext opContext,
       EntityService<?> entityService,
       ElasticSearchService elasticSearchService,
-      RestHighLevelClient restHighLevelClient,
+      SearchClientShim<?> restHighLevelClient,
       boolean enabled,
       boolean reprocessEnabled,
       Integer batchSize,

datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/dataprocessinstances/BackfillDataProcessInstancesHasRunEventsStep.java

@@ -14,6 +14,7 @@
 import com.linkedin.metadata.entity.EntityService;
 import com.linkedin.metadata.search.elasticsearch.ElasticSearchService;
 import com.linkedin.metadata.utils.elasticsearch.IndexConvention;
+import com.linkedin.metadata.utils.elasticsearch.SearchClientShim;
 import com.linkedin.upgrade.DataHubUpgradeState;
 import io.datahubproject.metadata.context.OperationContext;
 import java.io.IOException;
@@ -28,7 +29,6 @@
 import org.opensearch.action.search.SearchRequest;
 import org.opensearch.action.search.SearchResponse;
 import org.opensearch.client.RequestOptions;
-import org.opensearch.client.RestHighLevelClient;
 import org.opensearch.index.query.QueryBuilder;
 import org.opensearch.index.query.QueryBuilders;
 import org.opensearch.search.aggregations.Aggregation;
@@ -47,7 +47,7 @@ public class BackfillDataProcessInstancesHasRunEventsStep implements UpgradeStep
   private final OperationContext opContext;
   private final EntityService<?> entityService;
   private final ElasticSearchService elasticSearchService;
-  private final RestHighLevelClient restHighLevelClient;
+  private final SearchClientShim<?> restHighLevelClient;
 
   private final boolean reprocessEnabled;
   private final Integer batchSize;
@@ -60,7 +60,7 @@ public BackfillDataProcessInstancesHasRunEventsStep(
       OperationContext opContext,
       EntityService<?> entityService,
       ElasticSearchService elasticSearchService,
-      RestHighLevelClient restHighLevelClient,
+      SearchClientShim<?> restHighLevelClient,
       boolean reprocessEnabled,
       Integer batchSize,
       Integer batchDelayMs,

datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/elasticsearch/steps/BuildIndicesPostStep.java

@@ -63,8 +63,7 @@ public Function<UpgradeContext, UpgradeStepResult> executable() {
           boolean ack =
               esComponents
                   .getSearchClient()
-                  .indices()
-                  .putSettings(request, RequestOptions.DEFAULT)
+                  .updateIndexSettings(request, RequestOptions.DEFAULT)
                   .isAcknowledged();
           log.info(
               "Updated index {} with new settings. Settings: {}, Acknowledged: {}",