Merge pull request #49 from Terradue/httpclient

aligned with Terradue.Opensearch.SciHub 1.40.0
Terradue · Mar 19, 2024 · 7408e80 · 7408e80
2 parents 1024f65 + f1bf55c
commit 7408e80
Show file tree

Hide file tree

Showing 5 changed files with 23 additions and 26 deletions.
diff --git a/src/Stars.Data.Tests/TestBase.cs b/src/Stars.Data.Tests/TestBase.cs
@@ -71,6 +71,7 @@ protected TestBase()
             Collection.AddSingleton<IVectorService, TestVectorService>();
             Collection.AddSingleton<TranslatorManager, TranslatorManager>();
             Collection.AddSingleton<ICredentials, ConfigurationCredentialsManager>();
+            Collection.AddSingleton<IS3ClientFactory, S3ClientFactory>();
             Collection.AddHttpClient();
             var builder = new ConfigurationBuilder();
             // tell the builder to look for the appsettings.json file
@@ -81,6 +82,7 @@ protected TestBase()
             configFile.OpenRead();
             builder.AddNewtonsoftJsonFile(configFile.FullName, optional: false, reloadOnChange: false);
             configuration = builder.Build();
+            Collection.AddSingleton<IConfiguration>(configuration);
             LoadPlugins();
         }
 

diff --git a/src/Stars.Data/Suppliers/DataHubSourceSupplier.cs b/src/Stars.Data/Suppliers/DataHubSourceSupplier.cs
@@ -3,7 +3,6 @@
 using System.Threading.Tasks;
 using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.Logging;
-using Terradue.Stars.Interface.Router;
 using Terradue.OpenSearch;
 using Terradue.OpenSearch.DataHub;
 using Terradue.OpenSearch.DataHub.Aws;
@@ -15,10 +14,10 @@
 using Terradue.Stars.Services.Translator;
 using Terradue.Stars.Services.Supplier;
 using Terradue.Stars.Interface.Supplier;
-using Terradue.Stars.Services.Supplier.Carrier;
 using Terradue.Stars.Interface;
 using Terradue.Stars.Services.Plugins;
 using System.Threading;
+using Terradue.Stars.Services.Resources;
 
 
 namespace Terradue.Stars.Data.Suppliers
@@ -27,11 +26,17 @@ public class DataHubSourceSupplier : OpenSearchableSupplier, ISupplier
     {
         private IDataHubSourceWrapper wrapper;
         private ICredentials credentialsManager;
+        private readonly IS3ClientFactory _s3ClientFactory;
 
-        public DataHubSourceSupplier(ILogger<DataHubSourceSupplier> logger, TranslatorManager translatorManager, ICredentials credentialsManager, IPluginOption pluginOption) : base(logger, translatorManager)
+        public DataHubSourceSupplier(ILogger<DataHubSourceSupplier> logger,
+                                     TranslatorManager translatorManager,
+                                     ICredentials credentialsManager,
+                                     IS3ClientFactory s3ClientFactory,
+                                     IPluginOption pluginOption) : base(logger, translatorManager)
         {
             Logger4Net.Setup(logger);
             this.credentialsManager = credentialsManager;
+            _s3ClientFactory = s3ClientFactory;
             SupplierPluginOption supplierPluginOption = pluginOption as SupplierPluginOption;
             ConfigureWrapper(new Uri(supplierPluginOption.ServiceUrl));
         }
@@ -101,22 +106,26 @@ public void ConfigureWrapper(Uri serviceUrl)
             }
 
             // USGS case
-            if (target_uri.Host  == "earthexplorer.usgs.gov")
+            if (target_uri.Host == "earthexplorer.usgs.gov")
             {
                 // usgsOpenSearchable
                 wrapper = new Terradue.OpenSearch.Usgs.UsgsDataWrapper(new Uri("https://m2m.cr.usgs.gov"), target_creds);
             }
 
             if (target_uri.Host.EndsWith("amazon.com"))
             {
-                var searchWrapper = new DHuSWrapper(new Uri("https://scihub.copernicus.eu/apihub"), target_creds);
-                wrapper = new AmazonWrapper(null, null, searchWrapper);
+                Amazon.Runtime.AWSCredentials awscreds = _s3ClientFactory.GetConfiguredCredentials(S3Url.Parse("s3://sentinel-s2-l1c/test.tif"));
+                if (awscreds.GetType().ToString().Contains("DefaultInstanceProfileAWSCredentials"))
+                {
+                    awscreds = null;
+                }
+                var amazonWrapper = new AmazonStacWrapper(awscreds?.GetCredentials().AccessKey, awscreds?.GetCredentials().SecretKey, target_creds);
+                wrapper = amazonWrapper;
             }
 
             if (target_uri.Host.EndsWith("googleapis.com") || target_uri.Host.EndsWith("google.com"))
             {
-                var searchWrapper = new DHuSWrapper(new Uri("https://scihub.copernicus.eu/apihub"), target_creds);
-                wrapper = new GoogleWrapper(null, null, searchWrapper);
+                wrapper = new GoogleWrapper(null, null, target_creds, "https://cloud.google.com");
             }
 
             this.openSearchable = wrapper.CreateOpenSearchable(new OpenSearchableFactorySettings(this.opensearchEngine));

diff --git a/src/Stars.Data/Terradue.Stars.Data.csproj b/src/Stars.Data/Terradue.Stars.Data.csproj
@@ -31,9 +31,9 @@
         <PackageReference Include="Terradue.MetadataExtractor" Version="1.3.0" />
         <PackageReference Include="ProjNet" Version="2.0.0" />
         <PackageReference Include="MimeTypes" Version="2.0.2" />
-        <PackageReference Include="Terradue.OpenSearch.SciHub" Version="1.28.*"/>
-        <PackageReference Include="Terradue.OpenSearch.Asf" Version="1.2.*" />
-        <PackageReference Include="Terradue.OpenSearch.Usgs" Version="1.6.*" />
+        <PackageReference Include="Terradue.OpenSearch.SciHub" Version="1.40.*"/>
+        <PackageReference Include="Terradue.OpenSearch.Asf" Version="1.4.*" />
+        <PackageReference Include="Terradue.OpenSearch.Usgs" Version="1.7.*" />
         <PackageReference Include="Terradue.OpenSearch.GeoJson" Version="1.4.5" />
         <PackageReference Include="Terradue.GeoJson" Version="1.12.1" />
         <PackageReference Include="MaxRev.Gdal.Core" Version="3.3.3.110" />

diff --git a/src/Stars.Services/Resources/S3ClientFactory.cs b/src/Stars.Services/Resources/S3ClientFactory.cs
@@ -368,7 +368,7 @@ public AWSOptions GetNamedAWSOptionsOrDefault(string key)
                 return servicesSection.GetAWSOptions(key);
             }
 
-            return s3Options.CurrentValue.RootConfiguration.GetAWSOptions();
+            return configuration.GetAWSOptions();
         }
 
         public async Task<AWSCredentials> GetWebIdentityCredentialsAsync(string serviceURL, JwtSecurityToken jwt, string policy)

diff --git a/src/Stars.Tests/S3Tests.cs b/src/Stars.Tests/S3Tests.cs
@@ -97,20 +97,6 @@ public async Task ImportUnlimitedStreamabletoS3()
             Assert.Equal(new FileInfo(Path.Join(Environment.CurrentDirectory, "../../../In/items/test502.json")).Length, metadata.ContentLength);
         }
 
-        // Test disabled due to retirement of SciHub
-        /*[Fact]
-        public async Task ImportHttpStreamabletoS3()
-        {
-            await CreateBucketAsync("s3://local-http");
-            S3Resource s3Route = await s3ClientFactory.CreateAsync(S3Url.Parse("s3://local-http/S2B_MSIL2A_20211022T045839_N0301_R119_T44NLN_20211022T071547.jpg"));
-            BlockingStream stream = new BlockingStream(0, 100);
-            S3StreamingCarrier s3StreamingCarrier = serviceProvider.GetRequiredService<S3StreamingCarrier>();
-            var httpRoute = await resourceServiceProvider.CreateStreamResourceAsync(new GenericResource(new Uri("https://store.terradue.com/scihub/sentinel2/S2MSI2A/2023/09/17/quicklooks/v1/S2A_MSIL2A_20230917T100031_N0509_R122_T32TQM_20230917T143103.jpg")), CancellationToken.None);
-            var newRoute = await s3StreamingCarrier.StreamToS3Object(httpRoute, s3Route, CancellationToken.None);
-            var metadata = await s3Route.Client.GetObjectMetadataAsync(s3Route.S3Uri.Bucket, s3Route.S3Uri.Key);
-            Assert.Equal(httpRoute.ContentLength, Convert.ToUInt64(metadata.ContentLength));
-        }*/
-
         [Fact]
         public async Task AdaptRegion()
         {