From 5be24519029b4dbf9e314678025a7a203d6768e1 Mon Sep 17 00:00:00 2001 From: Enrico Girotto Date: Mon, 3 Feb 2025 10:23:48 +0100 Subject: [PATCH 01/12] Try to implement the new CDS api --- .../copernicus/datacubes/CopernicusCDSDatacube.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/adapters/klab.adapter.copernicus/src/main/java/org/integratedmodelling/adapter/copernicus/datacubes/CopernicusCDSDatacube.java b/adapters/klab.adapter.copernicus/src/main/java/org/integratedmodelling/adapter/copernicus/datacubes/CopernicusCDSDatacube.java index daa041885..be86699ed 100644 --- a/adapters/klab.adapter.copernicus/src/main/java/org/integratedmodelling/adapter/copernicus/datacubes/CopernicusCDSDatacube.java +++ b/adapters/klab.adapter.copernicus/src/main/java/org/integratedmodelling/adapter/copernicus/datacubes/CopernicusCDSDatacube.java @@ -64,7 +64,7 @@ public CopernicusCDSDatacube(String dataset, ITimeInstant dataStart, double noDa this.dataset = dataset; this.user = Configuration.INSTANCE.getProperties().getProperty(CDS_USER_NUMBER_PROPERTY); this.apiKey = Configuration.INSTANCE.getProperties().getProperty(CDS_API_KEY_PROPERTY); - if (this.apiKey == null || this.user == null) { + if (this.apiKey == null) { setOnline(false, "Copernicus CDS datacube: no CDS credentials provided in configuration"); } else { setOnline(true, null); @@ -128,7 +128,7 @@ protected boolean downloadChunk(int chunk, String variable, File destinationDire body.put("month", this.monts[(date.getMonth() - 1) / 3]); body.put("day", this.days); body.put("version", CDS_API_VERSION); - body.put("format", CDS_API_FORMAT); + body.put("download_format", CDS_API_FORMAT); configureRequest(variable, body); @@ -136,8 +136,8 @@ protected boolean downloadChunk(int chunk, String variable, File destinationDire Logging.INSTANCE.info("requesting chunk " + chunk + " of " + variable + " to CDS API: " + jsonBody); - HttpResponse response = Unirest.post(getEndpointUrl("resources/datasets/" + this.dataset)) - .basicAuth(user, apiKey).header("Accept", "application/json").body(jsonBody).asJson(); + HttpResponse response = Unirest.post(getEndpointUrl("/datasets/" + this.dataset)) + .header("PRIVATE-TOKEN", apiKey).header("Accept", "application/json").body(jsonBody).asJson(); if (response.isSuccess()) { @@ -221,7 +221,7 @@ protected boolean downloadChunk(int chunk, String variable, File destinationDire } public String getEndpointUrl(String request) { - return "https://cds.climate.copernicus.eu/api/v2/" + request; + return "https://cds.climate.copernicus.eu/api/" + request; } @Override From 6860d3ec2269d82db47f33810cbdbe6f083b5eaa Mon Sep 17 00:00:00 2001 From: Enrico Girotto Date: Mon, 3 Feb 2025 14:06:04 +0100 Subject: [PATCH 02/12] Change endpoint --- .../adapter/copernicus/datacubes/CopernicusCDSDatacube.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/adapters/klab.adapter.copernicus/src/main/java/org/integratedmodelling/adapter/copernicus/datacubes/CopernicusCDSDatacube.java b/adapters/klab.adapter.copernicus/src/main/java/org/integratedmodelling/adapter/copernicus/datacubes/CopernicusCDSDatacube.java index be86699ed..d9d90806b 100644 --- a/adapters/klab.adapter.copernicus/src/main/java/org/integratedmodelling/adapter/copernicus/datacubes/CopernicusCDSDatacube.java +++ b/adapters/klab.adapter.copernicus/src/main/java/org/integratedmodelling/adapter/copernicus/datacubes/CopernicusCDSDatacube.java @@ -136,7 +136,7 @@ protected boolean downloadChunk(int chunk, String variable, File destinationDire Logging.INSTANCE.info("requesting chunk " + chunk + " of " + variable + " to CDS API: " + jsonBody); - HttpResponse response = Unirest.post(getEndpointUrl("/datasets/" + this.dataset)) + HttpResponse response = Unirest.post(getEndpointUrl("/resources/" + this.dataset)) .header("PRIVATE-TOKEN", apiKey).header("Accept", "application/json").body(jsonBody).asJson(); if (response.isSuccess()) { From d6038ffd2122d8b1727fe28d4c6240fd46d08a8e Mon Sep 17 00:00:00 2001 From: Enrico Girotto Date: Mon, 10 Feb 2025 16:30:46 +0100 Subject: [PATCH 03/12] First try --- .../datacubes/CopernicusCDSDatacube.java | 183 +++++++++--------- 1 file changed, 96 insertions(+), 87 deletions(-) diff --git a/adapters/klab.adapter.copernicus/src/main/java/org/integratedmodelling/adapter/copernicus/datacubes/CopernicusCDSDatacube.java b/adapters/klab.adapter.copernicus/src/main/java/org/integratedmodelling/adapter/copernicus/datacubes/CopernicusCDSDatacube.java index d9d90806b..88d8d3992 100644 --- a/adapters/klab.adapter.copernicus/src/main/java/org/integratedmodelling/adapter/copernicus/datacubes/CopernicusCDSDatacube.java +++ b/adapters/klab.adapter.copernicus/src/main/java/org/integratedmodelling/adapter/copernicus/datacubes/CopernicusCDSDatacube.java @@ -25,6 +25,8 @@ import kong.unirest.HttpResponse; import kong.unirest.JsonNode; import kong.unirest.Unirest; +import kong.unirest.json.JSONException; +import kong.unirest.json.JSONObject; import ucar.nc2.dt.grid.GridDataset; /** @@ -37,12 +39,12 @@ public abstract class CopernicusCDSDatacube extends ChunkedDatacubeRepository { private String dataset; private String apiKey; - private String user; - public static final String CDS_USER_NUMBER_PROPERTY = "klab.copernicus.cds.user"; public static final String CDS_API_KEY_PROPERTY = "klab.copernicus.cds.apikey"; public static final String CDS_API_VERSION = "1_1"; public static final String CDS_API_FORMAT = "zip"; + public static final String CDS_API_KEY_HEADER = "PRIVATE-TOKEN"; + private int TIMEOUT_SECONDS = 30; private static Pattern pattern = Pattern.compile(".*(_[0-9]{8}_).*"); @@ -62,7 +64,7 @@ public CopernicusCDSDatacube(String dataset, ITimeInstant dataStart, double noDa super(Time.resolution(1, Type.DAY), Time.resolution(3, Type.MONTH), dataStart, Configuration.INSTANCE.getDataPath("copernicus/" + dataset), noDataValue); this.dataset = dataset; - this.user = Configuration.INSTANCE.getProperties().getProperty(CDS_USER_NUMBER_PROPERTY); + this.apiKey = Configuration.INSTANCE.getProperties().getProperty(CDS_API_KEY_PROPERTY); if (this.apiKey == null) { setOnline(false, "Copernicus CDS datacube: no CDS credentials provided in configuration"); @@ -94,7 +96,9 @@ protected Geoserver initializeGeoserver() { @Override protected boolean downloadChunk(int chunk, String variable, File destinationDirectory) { - Map body = new HashMap<>(); + Map bodyWrapper = new HashMap<>(); + Map body = new HashMap<>(); + boolean ret = false; ITimeInstant date = getChunkStart(chunk); @@ -123,105 +127,110 @@ protected boolean downloadChunk(int chunk, String variable, File destinationDire FileUtils.deleteQuietly(destinationDirectory); destinationDirectory.mkdirs(); } - + body.put("year", "" + date.getYear()); body.put("month", this.monts[(date.getMonth() - 1) / 3]); body.put("day", this.days); body.put("version", CDS_API_VERSION); - body.put("download_format", CDS_API_FORMAT); + //body.put("download_format", CDS_API_FORMAT); configureRequest(variable, body); - - String jsonBody = JsonUtils.printAsJson(body); + + bodyWrapper.put("inputs", body); + String jsonBody = JsonUtils.printAsJson(bodyWrapper); Logging.INSTANCE.info("requesting chunk " + chunk + " of " + variable + " to CDS API: " + jsonBody); - HttpResponse response = Unirest.post(getEndpointUrl("/resources/" + this.dataset)) - .header("PRIVATE-TOKEN", apiKey).header("Accept", "application/json").body(jsonBody).asJson(); + // retrieve the job id + + HttpResponse response = Unirest.post(getEndpointUrl("/processes/" + this.dataset + "/execute")) + .header(CDS_API_KEY_HEADER, apiKey).header("Accept", "application/json").body(jsonBody).asJson(); if (response.isSuccess()) { - - System.out.println(response.getBody().getObject()); - - if (response.getBody().getObject().has("state")) { - - int time = 0; - int tryafter = 5; - String url = null; - - while (time < TIMEOUT_SECONDS && !"completed".equals(response.getBody().getObject().get("state"))) { - - String requestId = response.getBody().getObject().has("request_id") - ? response.getBody().getObject().getString("request_id") - : null; - - if (requestId == null || response.getBody().getObject().has("error")) { - break; - } - - try { - Thread.sleep(tryafter * 1000); - } catch (InterruptedException e) { - break; - } - - time += tryafter; - - /* - * inquire about task - */ - response = Unirest.get(getEndpointUrl("tasks/" + requestId)).basicAuth(user, apiKey).asJson(); - - System.out.println(response.getBody().getObject()); - - if (response.getBody().getObject().has("error")) { - break; - } - - // heed their fucking advice - if (response.getHeaders().containsKey("Retry-After")) { - tryafter = (int) Math.ceil(Double.parseDouble(response.getHeaders().get("Retry-After").get(0))); - } - } - - if (response.getBody().getObject().has("location")) { - - url = response.getBody().getObject().getString("location"); - if (url.endsWith(".zip")) { - - Logging.INSTANCE.info("chunk " + chunk + " data for " + variable + " ready: downloading...."); - - /* - * Download the zip and unzip in chunk directory - */ - try { - URL uurl = new URL(url); - File zipFile = File.createTempFile("agera", ".zip"); - URLUtils.copyChanneled(uurl, zipFile); - ZipUtils.unzip(zipFile, destinationDirectory); - FileUtils.deleteQuietly(zipFile); - ret = true; - Logging.INSTANCE.info("download of chunk " + chunk + " data for " + variable + " successful"); - } catch (Throwable e) { - Logging.INSTANCE.warn("Download of CDS chunk " + variable + "/" + chunk - + " threw exception: " + e.getMessage()); - } - } - } - - } else { - Logging.INSTANCE.warn("Retrieval of CDS chunk " + variable + "/" + chunk + " threw exception: " - + response.getBody().getObject().get("message")); - } + + if (response.getBody().getObject().has("status") && "accepted".equals(response.getBody().getObject().get("status"))) { + // check the status of job + int time = 0; + int tryafter = 5; + String url = null; + String requestId = response.getBody().getObject().has("jobID") + ? response.getBody().getObject().getString("jobID") + : null; + if (requestId == null) { + Logging.INSTANCE.warn("Retrieval of CDS chunk " + variable + "/" + chunk + " didn't return a job ID"); + Logging.INSTANCE.warn(response.getBody().toPrettyString()); + return ret; + } + String status = null; + do { + try { + Thread.sleep(tryafter * 1000); + } catch (InterruptedException e) { + break; + } + + time += tryafter; + /* + * inquire about task + */ + response = Unirest.get(getEndpointUrl("jobs/" + requestId)).header("PRIVATE-TOKEN", apiKey).asJson(); + status = response.getBody().getObject().getString("status"); + Logging.INSTANCE.info("Status of retrieval of CDS chunk " + variable + "/" + chunk + ": " + status); + if ("failed".equals(status)){ + break; + } + } while (time < TIMEOUT_SECONDS && !"successful".equals(status) && !"failed".equals(status)); + + // retrieve the job results + response = Unirest.get(getEndpointUrl("jobs/" + requestId + "/results")).header("PRIVATE-TOKEN", apiKey).asJson(); + if (response.isSuccess()) { + // retrieve the url + String href = null; + JSONObject rBody = response.getBody().getObject(); + try { + href = rBody.getJSONObject("asset").getJSONObject("value").getString("href"); + } catch (JSONException e) { + Logging.INSTANCE.warn("The result is not API compliant: " + response.getBody().toPrettyString()); + return false; + } + if (href.endsWith(".zip")) { + Logging.INSTANCE.info("chunk " + chunk + " data for " + variable + " ready: downloading...."); + /* + * Download the zip and unzip in chunk directory + */ + try { + URL uurl = new URL(url); + File zipFile = File.createTempFile("agera", ".zip"); + URLUtils.copyChanneled(uurl, zipFile); + ZipUtils.unzip(zipFile, destinationDirectory); + FileUtils.deleteQuietly(zipFile); + ret = true; + Logging.INSTANCE.info("download of chunk " + chunk + " data for " + variable + " successful"); + } catch (Throwable e) { + Logging.INSTANCE.warn("Download of CDS chunk " + variable + "/" + chunk + + " threw exception: " + e.getMessage()); + } + } else { + Logging.INSTANCE.warn("The returned file is not .zip" + href); + return false; + } + } else { + Logging.INSTANCE.warn("The job has failed\n" + response.getBody().getObject().getString("status")+" - " + response.getBody().getObject().getString("traceback")); + return false; + } + } else { + Logging.INSTANCE.error("API request made to CDS Service didn't get accepted: " + response.getBody().toPrettyString()); + return false; + } } else { - Logging.INSTANCE.error("API request to CDS service returned error " + response.getStatusText()); - } - + Logging.INSTANCE.error("API request to CDS service returned error " + response.getStatusText()); + return false; + } return ret; } public String getEndpointUrl(String request) { - return "https://cds.climate.copernicus.eu/api/" + request; + return "https://cds.climate.copernicus.eu/api/retrieve/v1/" + request; } @Override From 728d8c88a186f80c0aae5dd7374120b10959271f Mon Sep 17 00:00:00 2001 From: Enrico Girotto Date: Mon, 10 Feb 2025 17:10:07 +0100 Subject: [PATCH 04/12] [docker build] Try to use the tag, add logs --- .../datacubes/CopernicusCDSDatacube.java | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/adapters/klab.adapter.copernicus/src/main/java/org/integratedmodelling/adapter/copernicus/datacubes/CopernicusCDSDatacube.java b/adapters/klab.adapter.copernicus/src/main/java/org/integratedmodelling/adapter/copernicus/datacubes/CopernicusCDSDatacube.java index 88d8d3992..b1fbc1e6a 100644 --- a/adapters/klab.adapter.copernicus/src/main/java/org/integratedmodelling/adapter/copernicus/datacubes/CopernicusCDSDatacube.java +++ b/adapters/klab.adapter.copernicus/src/main/java/org/integratedmodelling/adapter/copernicus/datacubes/CopernicusCDSDatacube.java @@ -139,11 +139,12 @@ protected boolean downloadChunk(int chunk, String variable, File destinationDire bodyWrapper.put("inputs", body); String jsonBody = JsonUtils.printAsJson(bodyWrapper); - Logging.INSTANCE.info("requesting chunk " + chunk + " of " + variable + " to CDS API: " + jsonBody); + Logging.INSTANCE.info("requesting chunk " + chunk + " of " + variable + " to CDS API"); // retrieve the job id - - HttpResponse response = Unirest.post(getEndpointUrl("/processes/" + this.dataset + "/execute")) + String endpoint = getEndpointUrl("/processes/" + this.dataset + "/execute"); + Logging.INSTANCE.info("Ask for job id: " + endpoint + " with key " + apiKey + "\n" + jsonBody); + HttpResponse response = Unirest.post(endpoint) .header(CDS_API_KEY_HEADER, apiKey).header("Accept", "application/json").body(jsonBody).asJson(); if (response.isSuccess()) { @@ -173,7 +174,9 @@ protected boolean downloadChunk(int chunk, String variable, File destinationDire /* * inquire about task */ - response = Unirest.get(getEndpointUrl("jobs/" + requestId)).header("PRIVATE-TOKEN", apiKey).asJson(); + endpoint = getEndpointUrl("jobs/" + requestId); + Logging.INSTANCE.info("Ask for job status: " + endpoint + " with key " + apiKey + "\n" + jsonBody); + response = Unirest.get(endpoint).header("PRIVATE-TOKEN", apiKey).asJson(); status = response.getBody().getObject().getString("status"); Logging.INSTANCE.info("Status of retrieval of CDS chunk " + variable + "/" + chunk + ": " + status); if ("failed".equals(status)){ @@ -182,7 +185,9 @@ protected boolean downloadChunk(int chunk, String variable, File destinationDire } while (time < TIMEOUT_SECONDS && !"successful".equals(status) && !"failed".equals(status)); // retrieve the job results - response = Unirest.get(getEndpointUrl("jobs/" + requestId + "/results")).header("PRIVATE-TOKEN", apiKey).asJson(); + endpoint = getEndpointUrl("jobs/" + requestId + "/results"); + Logging.INSTANCE.info("Ask for job results: " + endpoint + " with key " + apiKey + "\n" + jsonBody); + response = Unirest.get(endpoint).header("PRIVATE-TOKEN", apiKey).asJson(); if (response.isSuccess()) { // retrieve the url String href = null; From 35aced7f79f2d1fd92f2fe57a482db1e9cd42f04 Mon Sep 17 00:00:00 2001 From: Enrico Girotto Date: Mon, 10 Feb 2025 17:29:00 +0100 Subject: [PATCH 05/12] [docker build] double slash :( --- .../adapter/copernicus/datacubes/CopernicusCDSDatacube.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/adapters/klab.adapter.copernicus/src/main/java/org/integratedmodelling/adapter/copernicus/datacubes/CopernicusCDSDatacube.java b/adapters/klab.adapter.copernicus/src/main/java/org/integratedmodelling/adapter/copernicus/datacubes/CopernicusCDSDatacube.java index b1fbc1e6a..5de52572e 100644 --- a/adapters/klab.adapter.copernicus/src/main/java/org/integratedmodelling/adapter/copernicus/datacubes/CopernicusCDSDatacube.java +++ b/adapters/klab.adapter.copernicus/src/main/java/org/integratedmodelling/adapter/copernicus/datacubes/CopernicusCDSDatacube.java @@ -235,7 +235,7 @@ protected boolean downloadChunk(int chunk, String variable, File destinationDire } public String getEndpointUrl(String request) { - return "https://cds.climate.copernicus.eu/api/retrieve/v1/" + request; + return "https://cds.climate.copernicus.eu/api/retrieve/v1" + request; } @Override From 8a958eae02b791dfc7b85a23a166d56cbfeaefb6 Mon Sep 17 00:00:00 2001 From: Enrico Girotto Date: Mon, 10 Feb 2025 17:53:37 +0100 Subject: [PATCH 06/12] [docker build] Add content type header --- .../adapter/copernicus/datacubes/CopernicusCDSDatacube.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/adapters/klab.adapter.copernicus/src/main/java/org/integratedmodelling/adapter/copernicus/datacubes/CopernicusCDSDatacube.java b/adapters/klab.adapter.copernicus/src/main/java/org/integratedmodelling/adapter/copernicus/datacubes/CopernicusCDSDatacube.java index 5de52572e..fec039c02 100644 --- a/adapters/klab.adapter.copernicus/src/main/java/org/integratedmodelling/adapter/copernicus/datacubes/CopernicusCDSDatacube.java +++ b/adapters/klab.adapter.copernicus/src/main/java/org/integratedmodelling/adapter/copernicus/datacubes/CopernicusCDSDatacube.java @@ -145,7 +145,10 @@ protected boolean downloadChunk(int chunk, String variable, File destinationDire String endpoint = getEndpointUrl("/processes/" + this.dataset + "/execute"); Logging.INSTANCE.info("Ask for job id: " + endpoint + " with key " + apiKey + "\n" + jsonBody); HttpResponse response = Unirest.post(endpoint) - .header(CDS_API_KEY_HEADER, apiKey).header("Accept", "application/json").body(jsonBody).asJson(); + .header(CDS_API_KEY_HEADER, apiKey) + .header("Content-Type", "application/json") + .header("Accept", "application/json") + .body(jsonBody).asJson(); if (response.isSuccess()) { From d5c02c624aa6eb009fdc4e149af69652fd7c41b0 Mon Sep 17 00:00:00 2001 From: Enrico Girotto Date: Mon, 10 Feb 2025 18:19:45 +0100 Subject: [PATCH 07/12] [docker build] add more headers --- .../datacubes/CopernicusCDSDatacube.java | 35 ++++++++++++++----- 1 file changed, 27 insertions(+), 8 deletions(-) diff --git a/adapters/klab.adapter.copernicus/src/main/java/org/integratedmodelling/adapter/copernicus/datacubes/CopernicusCDSDatacube.java b/adapters/klab.adapter.copernicus/src/main/java/org/integratedmodelling/adapter/copernicus/datacubes/CopernicusCDSDatacube.java index fec039c02..3d3a2805c 100644 --- a/adapters/klab.adapter.copernicus/src/main/java/org/integratedmodelling/adapter/copernicus/datacubes/CopernicusCDSDatacube.java +++ b/adapters/klab.adapter.copernicus/src/main/java/org/integratedmodelling/adapter/copernicus/datacubes/CopernicusCDSDatacube.java @@ -179,18 +179,30 @@ protected boolean downloadChunk(int chunk, String variable, File destinationDire */ endpoint = getEndpointUrl("jobs/" + requestId); Logging.INSTANCE.info("Ask for job status: " + endpoint + " with key " + apiKey + "\n" + jsonBody); - response = Unirest.get(endpoint).header("PRIVATE-TOKEN", apiKey).asJson(); - status = response.getBody().getObject().getString("status"); - Logging.INSTANCE.info("Status of retrieval of CDS chunk " + variable + "/" + chunk + ": " + status); - if ("failed".equals(status)){ - break; + response = Unirest.get(endpoint) + .header("PRIVATE-TOKEN", apiKey) + .header("Content-Type", "application/json") + .header("Accept", "application/json").asJson(); + if (response.isSuccess()) { + status = response.getBody().getObject().getString("status"); + Logging.INSTANCE.info("Status of retrieval of CDS chunk " + variable + "/" + chunk + ": " + status); + if ("failed".equals(status)){ + break; + } + } else { + Logging.INSTANCE.warn("Ask for job status return an error " + response.getStatus() + ": " + response.getStatusText()); + return false; } + } while (time < TIMEOUT_SECONDS && !"successful".equals(status) && !"failed".equals(status)); // retrieve the job results endpoint = getEndpointUrl("jobs/" + requestId + "/results"); Logging.INSTANCE.info("Ask for job results: " + endpoint + " with key " + apiKey + "\n" + jsonBody); - response = Unirest.get(endpoint).header("PRIVATE-TOKEN", apiKey).asJson(); + response = Unirest.get(endpoint) + .header("PRIVATE-TOKEN", apiKey) + .header("Content-Type", "application/json") + .header("Accept", "application/json").asJson(); if (response.isSuccess()) { // retrieve the url String href = null; @@ -223,7 +235,14 @@ protected boolean downloadChunk(int chunk, String variable, File destinationDire return false; } } else { - Logging.INSTANCE.warn("The job has failed\n" + response.getBody().getObject().getString("status")+" - " + response.getBody().getObject().getString("traceback")); + Logging.INSTANCE.warn("The job results return an error " + response.getStatus() + ": " + response.getStatusText()); + if (response.getBody().getObject().has("status")) { + StringBuffer details = new StringBuffer().append("Details:\n").append(response.getBody().getObject().getString("status")); + if (response.getBody().getObject().has("traceback")) { + details.append("\nTraceback: ").append(response.getBody().getObject().getString("traceback")); + } + Logging.INSTANCE.warn(details.toString()); + } return false; } } else { @@ -231,7 +250,7 @@ protected boolean downloadChunk(int chunk, String variable, File destinationDire return false; } } else { - Logging.INSTANCE.error("API request to CDS service returned error " + response.getStatusText()); + Logging.INSTANCE.error("API request to CDS service returned error " + response.getStatus() + ": " + response.getStatusText()); return false; } return ret; From 81032d6c02a2b54d955f396d4f2a50a1f17dd954 Mon Sep 17 00:00:00 2001 From: Enrico Girotto Date: Mon, 10 Feb 2025 18:39:36 +0100 Subject: [PATCH 08/12] [docker build] Forgot a slash :( --- .../copernicus/datacubes/CopernicusCDSDatacube.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/adapters/klab.adapter.copernicus/src/main/java/org/integratedmodelling/adapter/copernicus/datacubes/CopernicusCDSDatacube.java b/adapters/klab.adapter.copernicus/src/main/java/org/integratedmodelling/adapter/copernicus/datacubes/CopernicusCDSDatacube.java index 3d3a2805c..cf59187fd 100644 --- a/adapters/klab.adapter.copernicus/src/main/java/org/integratedmodelling/adapter/copernicus/datacubes/CopernicusCDSDatacube.java +++ b/adapters/klab.adapter.copernicus/src/main/java/org/integratedmodelling/adapter/copernicus/datacubes/CopernicusCDSDatacube.java @@ -143,7 +143,7 @@ protected boolean downloadChunk(int chunk, String variable, File destinationDire // retrieve the job id String endpoint = getEndpointUrl("/processes/" + this.dataset + "/execute"); - Logging.INSTANCE.info("Ask for job id: " + endpoint + " with key " + apiKey + "\n" + jsonBody); + Logging.INSTANCE.info("Ask for job id: " + endpoint + "\n" + jsonBody); HttpResponse response = Unirest.post(endpoint) .header(CDS_API_KEY_HEADER, apiKey) .header("Content-Type", "application/json") @@ -177,8 +177,8 @@ protected boolean downloadChunk(int chunk, String variable, File destinationDire /* * inquire about task */ - endpoint = getEndpointUrl("jobs/" + requestId); - Logging.INSTANCE.info("Ask for job status: " + endpoint + " with key " + apiKey + "\n" + jsonBody); + endpoint = getEndpointUrl("/jobs/" + requestId); + Logging.INSTANCE.info("Ask for job status: " + endpoint); response = Unirest.get(endpoint) .header("PRIVATE-TOKEN", apiKey) .header("Content-Type", "application/json") @@ -197,7 +197,7 @@ protected boolean downloadChunk(int chunk, String variable, File destinationDire } while (time < TIMEOUT_SECONDS && !"successful".equals(status) && !"failed".equals(status)); // retrieve the job results - endpoint = getEndpointUrl("jobs/" + requestId + "/results"); + endpoint = getEndpointUrl("/jobs/" + requestId + "/results"); Logging.INSTANCE.info("Ask for job results: " + endpoint + " with key " + apiKey + "\n" + jsonBody); response = Unirest.get(endpoint) .header("PRIVATE-TOKEN", apiKey) From ce4ec3e769d9a3769db716ab9d2de0aab14ae735 Mon Sep 17 00:00:00 2001 From: Enrico Girotto Date: Tue, 11 Feb 2025 09:42:08 +0100 Subject: [PATCH 09/12] [docker build] solve a variable problem --- .../adapter/copernicus/datacubes/CopernicusCDSDatacube.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/adapters/klab.adapter.copernicus/src/main/java/org/integratedmodelling/adapter/copernicus/datacubes/CopernicusCDSDatacube.java b/adapters/klab.adapter.copernicus/src/main/java/org/integratedmodelling/adapter/copernicus/datacubes/CopernicusCDSDatacube.java index cf59187fd..2abaaf7ea 100644 --- a/adapters/klab.adapter.copernicus/src/main/java/org/integratedmodelling/adapter/copernicus/datacubes/CopernicusCDSDatacube.java +++ b/adapters/klab.adapter.copernicus/src/main/java/org/integratedmodelling/adapter/copernicus/datacubes/CopernicusCDSDatacube.java @@ -219,7 +219,7 @@ protected boolean downloadChunk(int chunk, String variable, File destinationDire * Download the zip and unzip in chunk directory */ try { - URL uurl = new URL(url); + URL uurl = new URL(href); File zipFile = File.createTempFile("agera", ".zip"); URLUtils.copyChanneled(uurl, zipFile); ZipUtils.unzip(zipFile, destinationDirectory); From fc948f9cfe0d9202bdcf639d472b97c51c9a63f8 Mon Sep 17 00:00:00 2001 From: Enrico Girotto Date: Tue, 11 Feb 2025 11:15:39 +0100 Subject: [PATCH 10/12] Remove unused variable --- .../adapter/copernicus/datacubes/CopernicusCDSDatacube.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/adapters/klab.adapter.copernicus/src/main/java/org/integratedmodelling/adapter/copernicus/datacubes/CopernicusCDSDatacube.java b/adapters/klab.adapter.copernicus/src/main/java/org/integratedmodelling/adapter/copernicus/datacubes/CopernicusCDSDatacube.java index 2abaaf7ea..b4ebc1eb6 100644 --- a/adapters/klab.adapter.copernicus/src/main/java/org/integratedmodelling/adapter/copernicus/datacubes/CopernicusCDSDatacube.java +++ b/adapters/klab.adapter.copernicus/src/main/java/org/integratedmodelling/adapter/copernicus/datacubes/CopernicusCDSDatacube.java @@ -132,7 +132,6 @@ protected boolean downloadChunk(int chunk, String variable, File destinationDire body.put("month", this.monts[(date.getMonth() - 1) / 3]); body.put("day", this.days); body.put("version", CDS_API_VERSION); - //body.put("download_format", CDS_API_FORMAT); configureRequest(variable, body); @@ -156,7 +155,6 @@ protected boolean downloadChunk(int chunk, String variable, File destinationDire // check the status of job int time = 0; int tryafter = 5; - String url = null; String requestId = response.getBody().getObject().has("jobID") ? response.getBody().getObject().getString("jobID") : null; From 2ce6d092e7319ecd9fa8f7b7e0809ca0d9ac52a0 Mon Sep 17 00:00:00 2001 From: Enrico Girotto Date: Tue, 11 Feb 2025 11:16:04 +0100 Subject: [PATCH 11/12] Format code --- .../datacubes/CopernicusCDSDatacube.java | 468 +++++++++--------- 1 file changed, 233 insertions(+), 235 deletions(-) diff --git a/adapters/klab.adapter.copernicus/src/main/java/org/integratedmodelling/adapter/copernicus/datacubes/CopernicusCDSDatacube.java b/adapters/klab.adapter.copernicus/src/main/java/org/integratedmodelling/adapter/copernicus/datacubes/CopernicusCDSDatacube.java index b4ebc1eb6..c92342831 100644 --- a/adapters/klab.adapter.copernicus/src/main/java/org/integratedmodelling/adapter/copernicus/datacubes/CopernicusCDSDatacube.java +++ b/adapters/klab.adapter.copernicus/src/main/java/org/integratedmodelling/adapter/copernicus/datacubes/CopernicusCDSDatacube.java @@ -37,123 +37,120 @@ */ public abstract class CopernicusCDSDatacube extends ChunkedDatacubeRepository { - private String dataset; - private String apiKey; - - public static final String CDS_API_KEY_PROPERTY = "klab.copernicus.cds.apikey"; - public static final String CDS_API_VERSION = "1_1"; - public static final String CDS_API_FORMAT = "zip"; - public static final String CDS_API_KEY_HEADER = "PRIVATE-TOKEN"; - - private int TIMEOUT_SECONDS = 30; - private static Pattern pattern = Pattern.compile(".*(_[0-9]{8}_).*"); - - /* - * tabulate and shut up - */ - String[][] monts = { { "01", "02", "03" }, { "04", "05", "06" }, { "07", "08", "09" }, { "10", "11", "12" } }; - - /** - * These don't have to be right for the months - */ - String[] days = { "01", "02", "03", "04", "05", "06", "07", "08", "09", "10", "11", "12", "13", "14", "15", "16", - "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31" }; - - public CopernicusCDSDatacube(String dataset, ITimeInstant dataStart, double noDataValue) { - - super(Time.resolution(1, Type.DAY), Time.resolution(3, Type.MONTH), dataStart, - Configuration.INSTANCE.getDataPath("copernicus/" + dataset), noDataValue); - this.dataset = dataset; - - this.apiKey = Configuration.INSTANCE.getProperties().getProperty(CDS_API_KEY_PROPERTY); - if (this.apiKey == null) { - setOnline(false, "Copernicus CDS datacube: no CDS credentials provided in configuration"); - } else { - setOnline(true, null); - } - } - - protected Geoserver initializeGeoserver() { - return Geoserver.create(); - } - - /** - * Put the necessary informations in the map that will be sent as json in the - * CDS API request based on the variable string in the URN. At a minimum it - * should contain the "variable" field so that the CDS API for this dataset will - * recognize it, but it could also encode statistical variants, time points and - * other selectors. - *

- * This is called after setting year, months and day fields for the passed - * chunk. For now these are fixed for 3-month chunks of daily data, changing - * them will need more configuration. - * - * @param variable - * @param payload - */ - protected abstract void configureRequest(String variable, Map payload); - - @Override - protected boolean downloadChunk(int chunk, String variable, File destinationDirectory) { - - Map bodyWrapper = new HashMap<>(); - Map body = new HashMap<>(); - - boolean ret = false; - ITimeInstant date = getChunkStart(chunk); - - /* - * check if it was downloaded and not processed, or added by hand. If we have - * the expected number of files, all with the expected tick number, we have them - * and we can return true. - */ - boolean partial = false; - boolean present = true; - for (int tick : getChunkTicks(chunk)) { - File tickFile = new File(destinationDirectory + File.separator - + getOriginalDataFilename(variable, tick, destinationDirectory)); - if (tickFile.exists()) { - partial = true; - } else { - present = false; - } - } - - if (present) { - return true; - } - - if (partial) { - FileUtils.deleteQuietly(destinationDirectory); - destinationDirectory.mkdirs(); - } - - body.put("year", "" + date.getYear()); - body.put("month", this.monts[(date.getMonth() - 1) / 3]); - body.put("day", this.days); - body.put("version", CDS_API_VERSION); - - configureRequest(variable, body); - - bodyWrapper.put("inputs", body); - String jsonBody = JsonUtils.printAsJson(bodyWrapper); - - Logging.INSTANCE.info("requesting chunk " + chunk + " of " + variable + " to CDS API"); - - // retrieve the job id - String endpoint = getEndpointUrl("/processes/" + this.dataset + "/execute"); - Logging.INSTANCE.info("Ask for job id: " + endpoint + "\n" + jsonBody); - HttpResponse response = Unirest.post(endpoint) - .header(CDS_API_KEY_HEADER, apiKey) - .header("Content-Type", "application/json") - .header("Accept", "application/json") - .body(jsonBody).asJson(); - - if (response.isSuccess()) { - - if (response.getBody().getObject().has("status") && "accepted".equals(response.getBody().getObject().get("status"))) { - // check the status of job - int time = 0; + private String dataset; + private String apiKey; + + public static final String CDS_API_KEY_PROPERTY = "klab.copernicus.cds.apikey"; + public static final String CDS_API_VERSION = "1_1"; + public static final String CDS_API_FORMAT = "zip"; + public static final String CDS_API_KEY_HEADER = "PRIVATE-TOKEN"; + + private int TIMEOUT_SECONDS = 30; + private static Pattern pattern = Pattern.compile(".*(_[0-9]{8}_).*"); + + /* + * tabulate and shut up + */ + String[][] monts = {{"01", "02", "03"}, {"04", "05", "06"}, {"07", "08", "09"}, {"10", "11", "12"}}; + + /** + * These don't have to be right for the months + */ + String[] days = {"01", "02", "03", "04", "05", "06", "07", "08", "09", "10", "11", "12", "13", "14", "15", "16", "17", "18", + "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31"}; + + public CopernicusCDSDatacube(String dataset, ITimeInstant dataStart, double noDataValue) { + + super(Time.resolution(1, Type.DAY), Time.resolution(3, Type.MONTH), dataStart, + Configuration.INSTANCE.getDataPath("copernicus/" + dataset), noDataValue); + this.dataset = dataset; + + this.apiKey = Configuration.INSTANCE.getProperties().getProperty(CDS_API_KEY_PROPERTY); + if (this.apiKey == null) { + setOnline(false, "Copernicus CDS datacube: no CDS credentials provided in configuration"); + } else { + setOnline(true, null); + } + } + + protected Geoserver initializeGeoserver() { + return Geoserver.create(); + } + + /** + * Put the necessary informations in the map that will be sent as json in the + * CDS API request based on the variable string in the URN. At a minimum it + * should contain the "variable" field so that the CDS API for this dataset will + * recognize it, but it could also encode statistical variants, time points and + * other selectors. + *

+ * This is called after setting year, months and day fields for the passed + * chunk. For now these are fixed for 3-month chunks of daily data, changing + * them will need more configuration. + * + * @param variable + * @param payload + */ + protected abstract void configureRequest(String variable, Map payload); + + @Override + protected boolean downloadChunk(int chunk, String variable, File destinationDirectory) { + + Map bodyWrapper = new HashMap<>(); + Map body = new HashMap<>(); + + boolean ret = false; + ITimeInstant date = getChunkStart(chunk); + + /* + * check if it was downloaded and not processed, or added by hand. If we have + * the expected number of files, all with the expected tick number, we have them + * and we can return true. + */ + boolean partial = false; + boolean present = true; + for(int tick : getChunkTicks(chunk)) { + File tickFile = new File( + destinationDirectory + File.separator + getOriginalDataFilename(variable, tick, destinationDirectory)); + if (tickFile.exists()) { + partial = true; + } else { + present = false; + } + } + + if (present) { + return true; + } + + if (partial) { + FileUtils.deleteQuietly(destinationDirectory); + destinationDirectory.mkdirs(); + } + + body.put("year", "" + date.getYear()); + body.put("month", this.monts[(date.getMonth() - 1) / 3]); + body.put("day", this.days); + body.put("version", CDS_API_VERSION); + + configureRequest(variable, body); + + bodyWrapper.put("inputs", body); + String jsonBody = JsonUtils.printAsJson(bodyWrapper); + + Logging.INSTANCE.info("requesting chunk " + chunk + " of " + variable + " to CDS API"); + + // retrieve the job id + String endpoint = getEndpointUrl("/processes/" + this.dataset + "/execute"); + Logging.INSTANCE.info("Ask for job id: " + endpoint + "\n" + jsonBody); + HttpResponse response = Unirest.post(endpoint).header(CDS_API_KEY_HEADER, apiKey) + .header("Content-Type", "application/json").header("Accept", "application/json").body(jsonBody).asJson(); + + if (response.isSuccess()) { + + if (response.getBody().getObject().has("status") && "accepted".equals(response.getBody().getObject().get("status"))) { + // check the status of job + int time = 0; int tryafter = 5; String requestId = response.getBody().getObject().has("jobID") ? response.getBody().getObject().getString("jobID") @@ -164,50 +161,47 @@ protected boolean downloadChunk(int chunk, String variable, File destinationDire return ret; } String status = null; - do { - try { + do { + try { Thread.sleep(tryafter * 1000); } catch (InterruptedException e) { break; } time += tryafter; - /* + /* * inquire about task */ endpoint = getEndpointUrl("/jobs/" + requestId); Logging.INSTANCE.info("Ask for job status: " + endpoint); - response = Unirest.get(endpoint) - .header("PRIVATE-TOKEN", apiKey) - .header("Content-Type", "application/json") + response = Unirest.get(endpoint).header("PRIVATE-TOKEN", apiKey).header("Content-Type", "application/json") .header("Accept", "application/json").asJson(); if (response.isSuccess()) { status = response.getBody().getObject().getString("status"); Logging.INSTANCE.info("Status of retrieval of CDS chunk " + variable + "/" + chunk + ": " + status); - if ("failed".equals(status)){ - break; + if ("failed".equals(status)) { + break; } } else { - Logging.INSTANCE.warn("Ask for job status return an error " + response.getStatus() + ": " + response.getStatusText()); + Logging.INSTANCE.warn( + "Ask for job status return an error " + response.getStatus() + ": " + response.getStatusText()); return false; } - - } while (time < TIMEOUT_SECONDS && !"successful".equals(status) && !"failed".equals(status)); - - // retrieve the job results - endpoint = getEndpointUrl("/jobs/" + requestId + "/results"); - Logging.INSTANCE.info("Ask for job results: " + endpoint + " with key " + apiKey + "\n" + jsonBody); - response = Unirest.get(endpoint) - .header("PRIVATE-TOKEN", apiKey) - .header("Content-Type", "application/json") + + } while (time < TIMEOUT_SECONDS && !"successful".equals(status) && !"failed".equals(status)); + + // retrieve the job results + endpoint = getEndpointUrl("/jobs/" + requestId + "/results"); + Logging.INSTANCE.info("Ask for job results: " + endpoint + " with key " + apiKey + "\n" + jsonBody); + response = Unirest.get(endpoint).header("PRIVATE-TOKEN", apiKey).header("Content-Type", "application/json") .header("Accept", "application/json").asJson(); - if (response.isSuccess()) { - // retrieve the url - String href = null; - JSONObject rBody = response.getBody().getObject(); - try { - href = rBody.getJSONObject("asset").getJSONObject("value").getString("href"); - } catch (JSONException e) { + if (response.isSuccess()) { + // retrieve the url + String href = null; + JSONObject rBody = response.getBody().getObject(); + try { + href = rBody.getJSONObject("asset").getJSONObject("value").getString("href"); + } catch (JSONException e) { Logging.INSTANCE.warn("The result is not API compliant: " + response.getBody().toPrettyString()); return false; } @@ -225,110 +219,114 @@ protected boolean downloadChunk(int chunk, String variable, File destinationDire ret = true; Logging.INSTANCE.info("download of chunk " + chunk + " data for " + variable + " successful"); } catch (Throwable e) { - Logging.INSTANCE.warn("Download of CDS chunk " + variable + "/" + chunk - + " threw exception: " + e.getMessage()); + Logging.INSTANCE.warn( + "Download of CDS chunk " + variable + "/" + chunk + " threw exception: " + e.getMessage()); } } else { Logging.INSTANCE.warn("The returned file is not .zip" + href); return false; } - } else { - Logging.INSTANCE.warn("The job results return an error " + response.getStatus() + ": " + response.getStatusText()); - if (response.getBody().getObject().has("status")) { - StringBuffer details = new StringBuffer().append("Details:\n").append(response.getBody().getObject().getString("status")); - if (response.getBody().getObject().has("traceback")) { - details.append("\nTraceback: ").append(response.getBody().getObject().getString("traceback")); - } - Logging.INSTANCE.warn(details.toString()); - } - return false; - } - } else { - Logging.INSTANCE.error("API request made to CDS Service didn't get accepted: " + response.getBody().toPrettyString()); - return false; - } - } else { - Logging.INSTANCE.error("API request to CDS service returned error " + response.getStatus() + ": " + response.getStatusText()); + } else { + Logging.INSTANCE + .warn("The job results return an error " + response.getStatus() + ": " + response.getStatusText()); + if (response.getBody().getObject().has("status")) { + StringBuffer details = new StringBuffer().append("Details:\n") + .append(response.getBody().getObject().getString("status")); + if (response.getBody().getObject().has("traceback")) { + details.append("\nTraceback: ").append(response.getBody().getObject().getString("traceback")); + } + Logging.INSTANCE.warn(details.toString()); + } + return false; + } + } else { + Logging.INSTANCE + .error("API request made to CDS Service didn't get accepted: " + response.getBody().toPrettyString()); + return false; + } + } else { + Logging.INSTANCE + .error("API request to CDS service returned error " + response.getStatus() + ": " + response.getStatusText()); return false; } - return ret; - } - - public String getEndpointUrl(String request) { - return "https://cds.climate.copernicus.eu/api/retrieve/v1" + request; - } - - @Override - protected boolean processChunk(int chunk, String variable, File destinationDirectory) { - - String[] fields = variable.split("\\."); - String cdsname = fields[0]; - String nativeName = null; - + return ret; + } + + public String getEndpointUrl(String request) { + return "https://cds.climate.copernicus.eu/api/retrieve/v1" + request; + } + + @Override + protected boolean processChunk(int chunk, String variable, File destinationDirectory) { + + String[] fields = variable.split("\\."); + String cdsname = fields[0]; + String nativeName = null; + Logging.INSTANCE.info("chunk " + chunk + " data for " + variable + " being ingested in local Geoserver"); - for (File f : destinationDirectory.listFiles(new FilenameFilter() { - - @Override - public boolean accept(File dir, String name) { - return name.endsWith(".nc"); - } - })) { - - /* - * parse the file name and get year, month and day - */ - Matcher matcher = pattern.matcher(MiscUtilities.getFileBaseName(f)); - if (!matcher.matches()) { - Logging.INSTANCE.warn("CDS file does not match naming pattern: ignoring " + f); - continue; - } - - // layer naming logic is repeated in #getDataLayer() - String daySignature = matcher.group(1).substring(1, 9); - String layerId = cdsname + "_" + daySignature; - - if (nativeName == null) { - try (GridDataset dataset = GridDataset.open(f.toString())) { - nativeName = dataset.getGrids().get(0).getName(); - } catch (IOException e) { - Logging.INSTANCE.warn("Can't open CDS file : " + f); - return false; - } - } - - if (!geoserver.createCoverageLayer(dataset, layerId, f, nativeName)) { - Logging.INSTANCE.warn("Geoserver ingestion of " + f + " returned a failure code"); - return false; - } - } + for(File f : destinationDirectory.listFiles(new FilenameFilter(){ + + @Override + public boolean accept(File dir, String name) { + return name.endsWith(".nc"); + } + })) { + + /* + * parse the file name and get year, month and day + */ + Matcher matcher = pattern.matcher(MiscUtilities.getFileBaseName(f)); + if (!matcher.matches()) { + Logging.INSTANCE.warn("CDS file does not match naming pattern: ignoring " + f); + continue; + } + + // layer naming logic is repeated in #getDataLayer() + String daySignature = matcher.group(1).substring(1, 9); + String layerId = cdsname + "_" + daySignature; + + if (nativeName == null) { + try (GridDataset dataset = GridDataset.open(f.toString())) { + nativeName = dataset.getGrids().get(0).getName(); + } catch (IOException e) { + Logging.INSTANCE.warn("Can't open CDS file : " + f); + return false; + } + } + + if (!geoserver.createCoverageLayer(dataset, layerId, f, nativeName)) { + Logging.INSTANCE.warn("Geoserver ingestion of " + f + " returned a failure code"); + return false; + } + } Logging.INSTANCE.info("Geoserver ingestion of chunk " + chunk + " data for " + variable + " terminated successfully"); - return true; - } - - @Override - protected String getDataLayer(String variable, int tick) { - - String[] fields = variable.split("\\."); - String cdsname = fields[0]; - String file = getOriginalFile(variable, tick); - Matcher matcher = pattern.matcher(MiscUtilities.getFileBaseName(file)); - if (matcher.matches()) { - String daySignature = matcher.group(1).substring(1, 9); - return cdsname + "_" + daySignature; - } - return null; - } - - protected Geoserver getGeoserver() { - return geoserver; - } - - @Override - public String getName() { - return dataset; - } + return true; + } + + @Override + protected String getDataLayer(String variable, int tick) { + + String[] fields = variable.split("\\."); + String cdsname = fields[0]; + String file = getOriginalFile(variable, tick); + Matcher matcher = pattern.matcher(MiscUtilities.getFileBaseName(file)); + if (matcher.matches()) { + String daySignature = matcher.group(1).substring(1, 9); + return cdsname + "_" + daySignature; + } + return null; + } + + protected Geoserver getGeoserver() { + return geoserver; + } + + @Override + public String getName() { + return dataset; + } } From 8592b5b0fb0082177e920ae1d37fe8e1d9404011 Mon Sep 17 00:00:00 2001 From: Enrico Girotto Date: Tue, 11 Feb 2025 11:26:33 +0100 Subject: [PATCH 12/12] [docker build] Add CDS API url to the application.yml --- .../adapter/copernicus/datacubes/CopernicusCDSDatacube.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/adapters/klab.adapter.copernicus/src/main/java/org/integratedmodelling/adapter/copernicus/datacubes/CopernicusCDSDatacube.java b/adapters/klab.adapter.copernicus/src/main/java/org/integratedmodelling/adapter/copernicus/datacubes/CopernicusCDSDatacube.java index c92342831..897ebf791 100644 --- a/adapters/klab.adapter.copernicus/src/main/java/org/integratedmodelling/adapter/copernicus/datacubes/CopernicusCDSDatacube.java +++ b/adapters/klab.adapter.copernicus/src/main/java/org/integratedmodelling/adapter/copernicus/datacubes/CopernicusCDSDatacube.java @@ -41,6 +41,7 @@ public abstract class CopernicusCDSDatacube extends ChunkedDatacubeRepository { private String apiKey; public static final String CDS_API_KEY_PROPERTY = "klab.copernicus.cds.apikey"; + public static final String CDS_API_URL = "klab.copernicus.cds.url"; public static final String CDS_API_VERSION = "1_1"; public static final String CDS_API_FORMAT = "zip"; public static final String CDS_API_KEY_HEADER = "PRIVATE-TOKEN"; @@ -253,7 +254,7 @@ protected boolean downloadChunk(int chunk, String variable, File destinationDire } public String getEndpointUrl(String request) { - return "https://cds.climate.copernicus.eu/api/retrieve/v1" + request; + return CDS_API_URL + request; } @Override