Skip to content

Commit

Permalink
Merge branch 'develop'
Browse files Browse the repository at this point in the history
  • Loading branch information
euskalhenriko committed Jun 29, 2023
2 parents 866caac + 00d9c6a commit 2b46cd9
Show file tree
Hide file tree
Showing 15 changed files with 769 additions and 26 deletions.
Original file line number Diff line number Diff line change
@@ -0,0 +1,74 @@
package org.integratedmodelling.klab;

import java.util.Collections;
import java.util.HashSet;
import java.util.Set;

import kong.unirest.HttpResponse;
import kong.unirest.JsonNode;
import kong.unirest.Unirest;
import kong.unirest.json.JSONArray;
import kong.unirest.json.JSONObject;

/**
* A utility class to obtain the data of the DOI via multiple APIs.
* DataCite API: https://support.datacite.org/docs/api-get-doi
* Crossref API: https://api.crossref.org/swagger-ui/index.html
*
*/
public class DOIReader {
private static String DATACITE_GET_DOI = "https://api.datacite.org/dois/";
private static String CROSSREF_GET_DOI = "https://api.crossref.org/works/";

public static HttpResponse<JsonNode> getDOIInformationFromDatacite(String doi) {
return Unirest.get(DATACITE_GET_DOI + doi).asJson();
}

public static HttpResponse<JsonNode> getDOIInformationFromCrossref(String doi) {
return Unirest.get(CROSSREF_GET_DOI + doi).asJson();
}

public static Set<String> readAuthorsDatacite(String doi) {
HttpResponse<JsonNode> response = getDOIInformationFromDatacite(doi);
if (!response.isSuccess()) {
return Collections.emptySet();
}
JsonNode json = response.getBody();
JSONArray authorsArray = json.getObject().getJSONObject("data").getJSONObject("attributes").getJSONArray("creators");
Set<String> authors = new HashSet<>();
for (Object author : authorsArray) {
if (!((JSONObject) author).has("name")) {
continue;
}
authors.add(((JSONObject) author).getString("name"));
}
return authors;
}

public static Set<String> readAuthorsCrossref(String doi) {
HttpResponse<JsonNode> response = getDOIInformationFromCrossref(doi);
if (!response.isSuccess()) {
return Collections.emptySet();
}
JsonNode json = response.getBody();
JSONArray authorsArray = json.getObject().getJSONObject("message").getJSONArray("author");
Set<String> authors = new HashSet<>();
for (Object author : authorsArray) {
if (!((JSONObject) author).has("given") || !((JSONObject) author).has("family")) {
continue;
}
String name = ((JSONObject) author).getString("family").concat(",")
.concat(((JSONObject) author).getString("given"));
authors.add(name);
}
return authors;
}

public static Set<String> readAuthors(String doi) {
Set<String> authors = readAuthorsDatacite(doi);
if (!authors.isEmpty()) {
return authors;
}
return readAuthorsCrossref(doi);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
import org.integratedmodelling.klab.stac.STACEncoder;
import org.integratedmodelling.klab.stac.STACImporter;
import org.integratedmodelling.klab.stac.STACPublisher;
import org.integratedmodelling.klab.stac.STACService;
import org.integratedmodelling.klab.stac.STACValidator;

@ResourceAdapter(type = STACAdapter.ID, version = Version.CURRENT, canCreateEmpty = true, handlesFiles = false)
Expand Down Expand Up @@ -64,4 +65,7 @@ public Collection<IPrototype> getResourceConfiguration() {
null));
}

public static STACService getService(String catalogUrl) {
return new STACService(catalogUrl);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -108,7 +108,7 @@ public void encodeFromCoverage(IResource resource, Map<String, String> urnParame
int band = 0;
if (urnParameters.containsKey("band")) {
band = Integer.parseInt(urnParameters.get("band"));
} else {
} else if (!resource.getAdapterType().equals("stac")) {
resource.getParameters().get("band", 0);
}
Set<Double> nodata = getNodata(resource, coverage, band);
Expand Down
Original file line number Diff line number Diff line change
@@ -1,31 +1,81 @@
package org.integratedmodelling.klab.stac;

import java.io.OutputStream;
import java.util.Date;
import java.util.List;
import java.util.Map;
import java.util.Optional;

import org.geotools.coverage.grid.GridCoverage2D;
import org.geotools.geometry.jts.ReferencedEnvelope;
import org.geotools.referencing.crs.DefaultGeographicCRS;
import org.hortonmachine.gears.io.stac.HMStacCollection;
import org.hortonmachine.gears.io.stac.HMStacItem;
import org.hortonmachine.gears.libs.modules.HMRaster;
import org.hortonmachine.gears.libs.monitor.LogProgressMonitor;
import org.hortonmachine.gears.utils.CrsUtilities;
import org.hortonmachine.gears.utils.RegionMap;
import org.hortonmachine.gears.utils.geometry.GeometryUtilities;
import org.integratedmodelling.klab.api.data.IGeometry;
import org.integratedmodelling.klab.api.data.IResource;
import org.integratedmodelling.klab.api.data.adapters.IKlabData.Builder;
import org.integratedmodelling.klab.api.data.adapters.IResourceEncoder;
import org.integratedmodelling.klab.api.knowledge.ICodelist;
import org.integratedmodelling.klab.api.observations.scale.IScale;
import org.integratedmodelling.klab.api.observations.scale.space.IEnvelope;
import org.integratedmodelling.klab.api.observations.scale.space.IGrid;
import org.integratedmodelling.klab.api.observations.scale.time.ITimeInstant;
import org.integratedmodelling.klab.api.provenance.IArtifact;
import org.integratedmodelling.klab.api.runtime.IContextualizationScope;
import org.integratedmodelling.klab.api.runtime.monitoring.IMonitor;
import org.integratedmodelling.klab.components.geospace.extents.Space;
import org.integratedmodelling.klab.components.time.extents.Time;
import org.integratedmodelling.klab.ogc.STACAdapter;
import org.integratedmodelling.klab.raster.files.RasterEncoder;
import org.locationtech.jts.geom.Envelope;
import org.locationtech.jts.geom.Polygon;
import org.opengis.referencing.crs.CoordinateReferenceSystem;

public class STACEncoder implements IResourceEncoder {

/**
* The raster encoder that does the actual work after we get our coverage from the service.
*/
RasterEncoder encoder = new RasterEncoder();

@Override
public boolean isOnline(IResource resource, IMonitor monitor) {
// TODO Auto-generated method stub
return false;
STACService service = STACAdapter.getService(resource.getParameters().get("catalogUrl", String.class));

if (service == null) {
monitor.error("Service " + resource.getParameters().get("catalogUrl", String.class)
+ " does not exist: likely the service URL is wrong or offline");
return false;
}

Optional<HMStacCollection> collection;
try {
collection = service.getCollectionById(resource.getParameters().get("collectionId", String.class));
} catch (Exception e) {
monitor.error("Collection " + resource.getParameters().get("catalogUrl", String.class)
+ " cannot be find.");
return false;
}

if (collection.isEmpty()) {
monitor.error("Collection " + resource.getParameters().get("catalogUrl", String.class)
+ " cannot be find.");
return false;
}

return true;
}

@Override
public IResource contextualize(IResource resource, IScale scale, IArtifact targetObservation,
Map<String, String> urnParameters, IContextualizationScope scope) {
// TODO Auto-generated method stub
return null;
return resource;
}

@Override
Expand All @@ -37,8 +87,55 @@ public ICodelist categorize(IResource resource, String attribute, IMonitor monit
@Override
public void getEncodedData(IResource resource, Map<String, String> urnParameters, IGeometry geometry, Builder builder,
IContextualizationScope scope) {
// TODO Auto-generated method stub
STACService service = STACAdapter.getService(resource.getParameters().get("catalogUrl", String.class));
Optional<HMStacCollection> collection = null;
try {
collection = service.getCollectionById(resource.getParameters().get("collectionId", String.class));
} catch (Exception e) {
scope.getMonitor().error("Collection " + resource.getParameters().get("catalogUrl", String.class)
+ " cannot be find.");
}

GridCoverage2D coverage = null;

Space space = (Space) geometry.getDimensions().stream().filter(d -> d instanceof Space).findFirst().orElseThrow();
Time time = (Time) geometry.getDimensions().stream().filter(d -> d instanceof Time).findFirst().orElseThrow();

ITimeInstant start = time.getStart();
ITimeInstant end = time.getEnd();

IEnvelope envelope = space.getEnvelope();
Envelope env = new Envelope(envelope.getMinX(), envelope.getMaxX(), envelope.getMinY(), envelope.getMaxY());
Polygon poly = GeometryUtilities.createPolygonFromEnvelope(env);

try {

List<HMStacItem> items = collection.get().setGeometryFilter(poly)
.setTimestampFilter(new Date(start.getMilliseconds()), new Date(end.getMilliseconds()))
.searchItems();

LogProgressMonitor lpm = new LogProgressMonitor();
IGrid grid = space.getGrid();

RegionMap region = RegionMap.fromBoundsAndGrid(space.getEnvelope().getMinX(), space.getEnvelope().getMaxX(),
space.getEnvelope().getMinY(), space.getEnvelope().getMaxY(),
(int) grid.getXCells(), (int) grid.getYCells());

Integer srid = items.get(0).getEpsg();
CoordinateReferenceSystem outputCrs = CrsUtilities.getCrsFromSrid(srid);
ReferencedEnvelope regionEnvelope = new ReferencedEnvelope(region.toEnvelope(),
DefaultGeographicCRS.WGS84).transform(outputCrs, true);
RegionMap regionTransformed = RegionMap.fromEnvelopeAndGrid(regionEnvelope, (int) grid.getXCells(), (int) grid.getYCells());
String stacBand = resource.getParameters().get("band", String.class);
HMRaster outRaster = HMStacCollection.readRasterBandOnRegion(regionTransformed, stacBand, items, lpm);

coverage = outRaster.buildCoverage();
scope.getMonitor().info("Coverage: " + coverage);
} catch (Exception e) {
scope.getMonitor().error("Cannot create STAC file." + e.getMessage());
}

encoder.encodeFromCoverage(resource, urnParameters, coverage, geometry, builder, scope);
}

@Override
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,78 @@
package org.integratedmodelling.klab.stac;

import org.apache.commons.lang3.StringUtils;
import org.integratedmodelling.klab.Version;

/**
* An list of the most popular extensions for STAC.
* Currently, we only define extensions with a maturity of Stable, Candidate, Pilot or Deprecated.
* https://stac-extensions.github.io/
*/
public enum STACExtension {
ElectroOptical("eo"),
FileInfo("file"),
ItemAssetsDefinition("item-assets"),
Projection("projection"),
ScientificCitation("scientific"),
ViewGeometry("view"),
Datacube("datacube"),
Processing("processing"),
Raster("raster"),
SAR("sar"),
Satellite("sat"),
VersioningIndicators("version"),
AlternateAssets("alternate-assets"),
AnonymizedLocation("anonymized-location"),
CARD4L_OpticalAndSAR("card4l"),
Classification("classification"),
Grid("grid"),
Label("label"),
MilitaryGridReferenceSystem("mgrs"),
NOAA_GOES("goes"), // NOAA Geostationary Operational Environmental Satellite
NOAA_MRMS_QPE("noaa-mrms-qpe"),
Order("order"),
PointCloud("pointcloud"),
Stats("stats"),
Storage("storage"),
Table("table"),
Timestamps("timestamps"),
XarrayAssets("xarray-assets"),
SingleFileSTAC("single-file-stac"),
TimeSeries("timeseries");

private String name;

STACExtension(String name) {
this.name = name;
}

public String getName() {
return name;
}

public static String getExtensionName(String identifier) {
return StringUtils.substringBetween(identifier, "https://stac-extensions.github.io/", "/v");
}

public static Version getVersion(String identifier) {
return Version.create(StringUtils.substringBetween(identifier, "/v", "/schema.json"));
}

public boolean isDeprecated() {
return this.name.equals(SingleFileSTAC.name) || this.name.equals(TimeSeries.name);
}

public boolean isSupported() {
// TODO
return true;
}

public static STACExtension valueOfLabel(String label) {
for (STACExtension e : values()) {
if (e.name.equals(label)) {
return e;
}
}
return null;
}
}
Loading

0 comments on commit 2b46cd9

Please sign in to comment.