Skip to content

Commit

Permalink
Merge branch 'develop' into IM-459-New-Configure-keycloak-in-remote-e…
Browse files Browse the repository at this point in the history 
…ngine
  • Loading branch information
@kristinaBc3
kristinaBc3 committed Nov 12, 2024
2 parents 63a50fa + 3b3985f commit 86f1587
Show file tree
Hide file tree
Showing 222 changed files with 499 additions and 882,559 deletions.
3 changes: 1 addition & 2 deletions ...s/klab.ogc/src/main/java/org/integratedmodelling/klab/ogc/vector/files/VectorEncoder.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,6 @@
import org.integratedmodelling.klab.utils.Utils;
import org.locationtech.jts.geom.Geometry;
import org.locationtech.jts.geom.Polygon;
import org.locationtech.jts.geom.prep.PreparedGeometryFactory;
import org.opengis.feature.simple.SimpleFeature;
import org.opengis.feature.simple.SimpleFeatureType;
import org.opengis.feature.type.AttributeDescriptor;
Expand Down Expand Up @@ -134,7 +133,7 @@ protected FeatureSource<SimpleFeatureType, SimpleFeature> getFeatureSource(IReso
* @param builder
* @param scope
*/
private void encodeFromFeatures(FeatureSource<SimpleFeatureType, SimpleFeature> source, IResource resource,
public void encodeFromFeatures(FeatureSource<SimpleFeatureType, SimpleFeature> source, IResource resource,
Map<String, String> urnParameters, IGeometry geometry, Builder builder, IContextualizationScope scope) {

Filter filter = null;
Expand Down
11 changes: 7 additions & 4 deletions adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACCollectionParser.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,8 +51,11 @@ public static IGeometry readGeometry(JSONObject collection) {
gBuilder.time().end(Instant.parse(interval.get(1).toString()).toEpochMilli());
}

return gBuilder.build().withProjection(Projection.DEFAULT_PROJECTION_CODE)
.withTimeType("grid");
// TODO find non-ad-hoc cases
if (collection.getString("id").equals("slovak_SK_v5_reference-points_EUNIS2012")) {
return gBuilder.build().withProjection(Projection.DEFAULT_PROJECTION_CODE).withTimeType("logical");
}
return gBuilder.build().withProjection(Projection.DEFAULT_PROJECTION_CODE).withTimeType("grid");
}

/**
Expand All @@ -63,6 +66,7 @@ public static IGeometry readGeometry(JSONObject collection) {
*/
public static JSONObject readAssetsFromCollection(String collectionUrl, JSONObject collection) throws KlabResourceAccessException {
String catalogUrl = STACUtils.getCatalogUrl(collection);
String collectonId = collection.getString("id");
JSONObject catalogData = STACUtils.requestMetadata(catalogUrl, "catalog");

Optional<String> searchEndpoint = STACUtils.containsLinkTo(catalogData, "search")
Expand All @@ -85,9 +89,8 @@ public static JSONObject readAssetsFromCollection(String collectionUrl, JSONObje
}

// TODO Move the query to another place.
String parameters = "?collections=" + collectionUrl + "&limit=1";
String parameters = "?collections=" + collectonId + "&limit=1";
HttpResponse<JsonNode> response = Unirest.get(searchEndpoint.get() + parameters).asJson();
// HttpResponse<JsonNode> response = Unirest.post(searchEndpoint.get()).body(body).asJson();

if (!response.isSuccess()) {
throw new KlabResourceAccessException(); //TODO set message
Expand Down
67 changes: 48 additions & 19 deletions adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,9 +9,11 @@
import java.util.stream.Collectors;

import org.geotools.coverage.grid.GridCoverage2D;
import org.geotools.data.FeatureSource;
import org.geotools.geometry.jts.ReferencedEnvelope;
import org.hortonmachine.gears.io.stac.HMStacCollection;
import org.hortonmachine.gears.io.stac.HMStacItem;
import org.hortonmachine.gears.io.stac.HMStacManager;
import org.hortonmachine.gears.libs.modules.HMRaster;
import org.hortonmachine.gears.libs.monitor.LogProgressMonitor;
import org.hortonmachine.gears.utils.RegionMap;
Expand Down Expand Up @@ -40,29 +42,32 @@
import org.integratedmodelling.klab.exceptions.KlabIOException;
import org.integratedmodelling.klab.exceptions.KlabIllegalStateException;
import org.integratedmodelling.klab.exceptions.KlabInternalErrorException;
import org.integratedmodelling.klab.exceptions.KlabResourceAccessException;
import org.integratedmodelling.klab.exceptions.KlabUnimplementedException;
import org.integratedmodelling.klab.ogc.STACAdapter;
import org.integratedmodelling.klab.ogc.vector.files.VectorEncoder;
import org.integratedmodelling.klab.raster.files.RasterEncoder;
import org.integratedmodelling.klab.rest.ExternalAuthenticationCredentials;
import org.integratedmodelling.klab.scale.Scale;
import org.integratedmodelling.klab.stac.extensions.STACIIASAExtension;
import org.integratedmodelling.klab.utils.s3.S3URLUtils;
import org.locationtech.jts.geom.Envelope;
import org.locationtech.jts.geom.Polygon;
import org.opengis.feature.simple.SimpleFeature;
import org.opengis.feature.simple.SimpleFeatureType;

import com.amazonaws.auth.AWSStaticCredentialsProvider;
import com.amazonaws.auth.BasicAWSCredentials;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3Client;

import kong.unirest.json.JSONObject;

public class STACEncoder implements IResourceEncoder {

/**
* The raster encoder that does the actual work after we get our coverage from
* the service.
* The raster or vector encoder that does the actual work after we get our coverage from the service.
*/
RasterEncoder encoder = new RasterEncoder();
IResourceEncoder encoder;

@Override
public boolean isOnline(IResource resource, IMonitor monitor) {
Expand Down Expand Up @@ -178,31 +183,54 @@ private AmazonS3 buildS3Client(String bucketRegion) throws IOException {
@Override
public void getEncodedData(IResource resource, Map<String, String> urnParameters, IGeometry geometry, Builder builder,
IContextualizationScope scope) {
IObservable targetSemantics = scope.getTargetArtifact() instanceof Observation
? ((Observation) scope.getTargetArtifact()).getObservable()
: null;
HMRaster.MergeMode mergeMode = chooseMergeMode(targetSemantics, scope.getMonitor());

String collectionUrl = resource.getParameters().get("collection", String.class);
JSONObject collectionData = STACUtils.requestMetadata(collectionUrl, "collection");
String catalogUrl = STACUtils.getCatalogUrl(collectionData);
JSONObject catalogData = STACUtils.requestMetadata(catalogUrl, "catalog");

boolean hasSearchOption = STACUtils.containsLinkTo(catalogData, "search");
if (!hasSearchOption) {
// This is part of a WIP that will be removed in the future
boolean isIIASA = catalogUrl.contains("iiasa.blob");
if (!hasSearchOption && !isIIASA) {
// TODO implement how to read static collections
throw new KlabUnimplementedException("Cannot read a static collection.");
}

STACService service = STACAdapter.getService(collectionUrl);
HMStacCollection collection = service.getCollection();
if (collection == null) {
scope.getMonitor().error("Collection " + resource.getParameters().get("collection", String.class) + " cannot be find.");
}

Space space = (Space) geometry.getDimensions().stream().filter(d -> d instanceof Space)
.findFirst().orElseThrow();
IEnvelope envelope = space.getEnvelope();
List<Double> bbox = List.of(envelope.getMinX(), envelope.getMaxX(), envelope.getMinY(), envelope.getMaxY());
if (isIIASA) {
FeatureSource<SimpleFeatureType, SimpleFeature> source;
try {
source = STACIIASAExtension.getFeatures(collectionData, bbox);
} catch (IOException e) {
throw new KlabResourceAccessException("Cannot extract features from IIASA catalog - " + e.getMessage());
}
encoder = new VectorEncoder();
((VectorEncoder)encoder).encodeFromFeatures(source, resource, urnParameters, geometry, builder, scope);
return;
}

LogProgressMonitor lpm = new LogProgressMonitor();
HMStacManager manager = new HMStacManager(catalogUrl, lpm);
HMStacCollection collection = null;
try {
manager.open();
collection = manager.getCollectionById(resource.getParameters().get("collectionId", String.class));
} catch (Exception e1) {
throw new KlabResourceAccessException("Cannot access to STAC collection " + collectionUrl);
}

if (collection == null) {
scope.getMonitor().error("Collection " + resource.getParameters().get("collection", String.class) + " cannot be found.");
}

IObservable targetSemantics = scope.getTargetArtifact() instanceof Observation
? ((Observation) scope.getTargetArtifact()).getObservable()
: null;
HMRaster.MergeMode mergeMode = chooseMergeMode(targetSemantics, scope.getMonitor());

Envelope env = new Envelope(envelope.getMinX(), envelope.getMaxX(), envelope.getMinY(), envelope.getMaxY());
Polygon poly = GeometryUtilities.createPolygonFromEnvelope(env);
collection.setGeometryFilter(poly);
Expand All @@ -223,6 +251,7 @@ public void getEncodedData(IResource resource, Map<String, String> urnParameters
List<HMStacItem> items = collection.searchItems();

if (items.isEmpty()) {
manager.close();
throw new KlabIllegalStateException("No STAC items found for this context.");
}
scope.getMonitor().debug("Found " + items.size() + " STAC items.");
Expand All @@ -231,7 +260,6 @@ public void getEncodedData(IResource resource, Map<String, String> urnParameters
sortByDate(items, scope.getMonitor());
}

LogProgressMonitor lpm = new LogProgressMonitor();
IGrid grid = space.getGrid();

RegionMap region = RegionMap.fromBoundsAndGrid(space.getEnvelope().getMinX(), space.getEnvelope().getMaxX(),
Expand Down Expand Up @@ -260,11 +288,12 @@ public void getEncodedData(IResource resource, Map<String, String> urnParameters
String assetId = resource.getParameters().get("asset", String.class);
HMRaster outRaster = collection.readRasterBandOnRegion(regionTransformed, assetId, items, allowTransform, mergeMode, lpm);
coverage = outRaster.buildCoverage();
manager.close();
} catch (Exception e) {
throw new KlabInternalErrorException("Cannot build STAC raster output. Reason " + e.getMessage());
}

encoder.encodeFromCoverage(resource, urnParameters, coverage, geometry, builder, scope);
encoder = new RasterEncoder();
((RasterEncoder)encoder).encodeFromCoverage(resource, urnParameters, coverage, geometry, builder, scope);
}

@Override
Expand Down
34 changes: 27 additions & 7 deletions adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACImporter.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,17 @@ private void importCollection(List<Builder> ret, IParameters<String> parameters,
parameters.remove(Resources.REGEX_ENTRY);
}

boolean isBulkImport = parameters.contains("bulkImport");
parameters.remove("bulkImport");
if (!parameters.contains("asset") && !isBulkImport) {
Builder builder = buildResource(parameters, project, monitor, collectionId);
if (builder != null) {
ret.add(builder);
} else {
monitor.warn("STAC collection " + collectionId + " is invalid and cannot be imported");
}
return;
}
JSONObject assets = STACCollectionParser.readAssetsFromCollection(collectionUrl, collectionData);
Set<String> assetIds = STACAssetMapParser.readAssetNames(assets);
for(String assetId : assetIds) {
Expand All @@ -72,20 +83,28 @@ private void importCollection(List<Builder> ret, IParameters<String> parameters,
parameters.put("asset", assetId);
String resourceUrn = collectionId + "-" + assetId;

Builder builder = validator.validate(
Resources.INSTANCE.createLocalResourceUrn(resourceUrn, project), new URL(collectionUrl),
parameters, monitor);

Builder builder = buildResource(parameters, project, monitor, resourceUrn);
if (builder != null) {
builder.withLocalName(resourceUrn).setResourceId(resourceUrn);
ret.add(builder);
monitor.info("STAC collection " + collectionId + " added");
} else {
monitor.warn("STAC collection " + collectionId + " is invalid");
monitor.warn("STAC resource with asset " + resourceUrn + " is invalid and cannot be imported");
}
}
}

private Builder buildResource(IParameters<String> parameters, IProject project, IMonitor monitor, String resourceUrn) throws MalformedURLException {
Builder builder = validator.validate(
Resources.INSTANCE.createLocalResourceUrn(resourceUrn, project), new URL(parameters.get("collection", String.class)),
parameters, monitor);

if (builder == null) {
return null;
}
builder.withLocalName(resourceUrn).setResourceId(resourceUrn);
monitor.info("STAC collection " + resourceUrn + " added");
return builder;
}

@Override
public Collection<Builder> importResources(String collectionUrl, IProject project, IParameters<String> userData,
IMonitor monitor) {
Expand All @@ -96,6 +115,7 @@ public Collection<Builder> importResources(String collectionUrl, IProject projec
Parameters<String> parameters = new Parameters<>();
parameters.putAll(userData);
parameters.put("collection", collectionUrl);
parameters.put("bulkImport", true);

importCollection(ret, parameters, project, monitor);
} catch (Exception e) {
Expand Down
32 changes: 0 additions & 32 deletions adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACService.java
View file
Original file line number Diff line number Diff line change
@@ -1,40 +1,8 @@
package org.integratedmodelling.klab.stac;

import org.hortonmachine.gears.io.stac.HMStacCollection;
import org.hortonmachine.gears.io.stac.HMStacManager;
import org.hortonmachine.gears.libs.monitor.LogProgressMonitor;
import org.integratedmodelling.klab.exceptions.KlabInternalErrorException;

import kong.unirest.json.JSONObject;

public class STACService {
private HMStacManager catalog;
private HMStacCollection collection;

public STACService(String collectionUrl) {
JSONObject collectionData = STACUtils.requestMetadata(collectionUrl, "collection");
String collectionId = STACCollectionParser.readCollectionId(collectionData);
String catalogUrl = STACUtils.getCatalogUrl(collectionData);

LogProgressMonitor lpm = new LogProgressMonitor();
this.catalog = new HMStacManager(catalogUrl, lpm);
try {
this.catalog.open();
} catch (Exception e) {
throw new KlabInternalErrorException("Error at STAC service. Cannot read catalog at '" + catalogUrl + "'.");
}

try {
this.collection = catalog.getCollectionById(collectionId);
} catch (Exception e) {
throw new KlabInternalErrorException("Error at STAC service. Cannot read collection at '" + collectionUrl + "'.");
}
if (collection == null) {
throw new KlabInternalErrorException("Error at STAC service. Endpoint '" + catalogUrl + "' has no collection '" + collectionId + "'.");
}
}

public HMStacCollection getCollection() {
return collection;
}
}
49 changes: 27 additions & 22 deletions adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACValidator.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,29 +49,34 @@ public Builder validate(String urn, URL url, IParameters<String> userData, IMoni
}
IGeometry geometry = STACCollectionParser.readGeometry(collectionData);

Builder builder = new ResourceBuilder(urn).withParameters(userData).withGeometry(geometry);
Builder builder = new ResourceBuilder(urn)
.withParameters(userData)
.withGeometry(geometry)
.withType(Type.OBJECT);

// The default URL of the resource is the collection endpoint. May be overwritten.
builder.withMetadata(IMetadata.DC_URL, collectionUrl);

String assetId = userData.get("asset", String.class);
JSONObject assets = STACCollectionParser.readAssetsFromCollection(collectionUrl, collectionData);
JSONObject asset = STACAssetMapParser.getAsset(assets, assetId);

Type type = readRasterDataType(asset);
// Currently, only files:values is supported. If needed, the classification extension could be used too.
Map<String, Object> vals = STACAssetParser.getFileValues(asset);
if (!vals.isEmpty()) {
CodelistReference codelist = populateCodelist(assetId, vals);
if (type == null) {
type = codelist.getType();
if (userData.contains("asset")) {
String assetId = userData.get("asset", String.class);
JSONObject assets = STACCollectionParser.readAssetsFromCollection(collectionUrl, collectionData);
JSONObject asset = STACAssetMapParser.getAsset(assets, assetId);

Type type = readRasterDataType(asset);
// Currently, only files:values is supported. If needed, the classification extension could be used too.
Map<String, Object> vals = STACAssetParser.getFileValues(asset);
if (!vals.isEmpty()) {
CodelistReference codelist = populateCodelist(assetId, vals);
if (type == null) {
type = codelist.getType();
}
builder.addCodeList(codelist);
}
if (type != null) {
builder.withType(type);
}
builder.addCodeList(codelist);
}

if (type != null) {
builder.withType(type);
}
readMetadata(collectionData, builder);
return builder;
}
Expand Down Expand Up @@ -122,7 +127,7 @@ private CodelistReference populateCodelist(String assetId, Map<String, Object> v
private void readMetadata(final JSONObject json, Builder builder) {
// We could check the doi only if the Scientific Notation extension is provided, but we can try anyway
String doi = STACUtils.readDOI(json);
if (doi != null) {
if (doi != null && !doi.isBlank()) {
builder.withMetadata(IMetadata.DC_URL, doi);
String authors = STACUtils.readDOIAuthors(doi);
if (authors != null) {
Expand All @@ -131,22 +136,22 @@ private void readMetadata(final JSONObject json, Builder builder) {
}

String description = STACUtils.readDescription(json);
if (description != null) {
if (description != null && !description.isBlank()) {
builder.withMetadata(IMetadata.DC_COMMENT, description);
}

String keywords = STACUtils.readKeywords(json);
if (keywords != null) {
if (keywords != null && !keywords.isBlank()) {
builder.withMetadata(IMetadata.IM_KEYWORDS, keywords);
}

String title = STACUtils.readTitle(json);
if (title != null) {
if (title != null && !title.isBlank()) {
builder.withMetadata(IMetadata.DC_TITLE, title);
}

String license = STACUtils.readLicense(json);
if (license != null) {
if (license != null && !license.isBlank()) {
builder.withMetadata(IMetadata.DC_RIGHTS, license);
}
}
Expand Down Expand Up @@ -177,7 +182,7 @@ public IResource performOperation(IResource resource, String operationName, IPar

@Override
public boolean canHandle(File resource, IParameters<String> parameters) {
return resource == null && parameters.contains("collection") && parameters.contains("asset");
return resource == null && parameters.contains("collection");
}

@Override
Expand Down
Loading

0 comments on commit 86f1587

Please sign in to comment.