Skip to content

Commit 1d3e683

Browse files
sjoerdbeentjessjoerdbeentjes
authored
Merge pull request #12 from openearth/sync-metadata
Sync metadata
2 parents 891173b + 55603d3 commit 1d3e683

8 files changed

+465
-9
lines changed

Diff for: ‎.env.example

+4-1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,7 @@
11
DATO_API_TOKEN=
22
GEONETWORK_API_USERNAME=
33
GEONETWORK_API_PASSWORD=
4-
SYNC_LAYER_API_TOKEN=
4+
SYNC_LAYER_API_TOKEN=
5+
6+
DATO_API_KEY_OPENEARTH_RWS_VIEWER=
7+
DATO_API_KEY_NL2120=

Diff for: ‎.github/workflows/sync-external-layers.yml

+19
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
name: Sync external layers
2+
on:
3+
schedule:
4+
- cron: '0 0 * * *'
5+
jobs:
6+
report:
7+
runs-on: ubuntu-latest
8+
steps:
9+
- uses: actions/checkout@v3
10+
- uses: actions/setup-node@v3
11+
with:
12+
node-version: 16
13+
cache: 'npm'
14+
- run: npm ci
15+
- run: npm run sync-external-layers
16+
env:
17+
DATO_API_TOKEN: ${{ secrets.DATO_API_TOKEN }}
18+
DATO_API_KEY_NL2120: ${{ secrets.DATO_API_KEY_NL2120 }}
19+
DATO_API_KEY_OPENEARTH_RWS_VIEWER: ${{ secrets.DATO_API_KEY_OPENEARTH_RWS_VIEWER }}

Diff for: ‎package.json

+2-1
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,8 @@
33
"scripts": {
44
"dev": "netlify dev -p 8080",
55
"ngrok": "ngrok http 8080",
6-
"report": "node src/scripts/report-dead-layer-links.js"
6+
"report": "node src/scripts/report-dead-layer-links.js",
7+
"sync-external-metadata": "node src/scripts/sync-external-metadata.js"
78
},
89
"devDependencies": {
910
"netlify-cli": "^10.15.0",

Diff for: ‎src/lib/build-children-tree.js

+6
Original file line numberDiff line numberDiff line change
@@ -2,9 +2,15 @@ export function buildChildrenTree(items) {
22
items.forEach((item) => {
33
if (item.parent) {
44
const parent = items.find((p) => p.id === item.parent.id)
5+
6+
if (!parent) {
7+
return
8+
}
9+
510
if (parent.children == null) {
611
parent.children = []
712
}
13+
814
parent.children.push(item)
915
}
1016
})

Diff for: ‎src/lib/datocms.js

+4-4
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ const camelCase = pipe(
1616

1717
const defaultFirst = 100
1818

19-
function executeFetch(query, variables = {}, preview = false) {
19+
function executeFetch(query, variables = {}, preview = false, token = process.env.DATO_API_TOKEN) {
2020
const endpoint = preview
2121
? 'https://graphql.datocms.com/preview'
2222
: 'https://graphql.datocms.com/'
@@ -25,7 +25,7 @@ function executeFetch(query, variables = {}, preview = false) {
2525
method: 'post',
2626
headers: {
2727
'Content-Type': 'application/json',
28-
Authorization: process.env.DATO_API_TOKEN,
28+
Authorization: `Bearer ${token}`,
2929
'X-Environment': 'main',
3030
},
3131
body: JSON.stringify({ query, variables }),
@@ -85,8 +85,8 @@ function returnData(response) {
8585
return response.data
8686
}
8787

88-
export const datocmsRequest = curry(({ query, variables, preview }) => {
89-
const args = [query, { first: defaultFirst, ...variables }, preview]
88+
export const datocmsRequest = curry(({ query, variables, preview, token: tokenOverride }) => {
89+
const args = [query, { first: defaultFirst, ...variables }, preview, tokenOverride]
9090

9191
return executeFetch(...args)
9292
.then(getPaginatedData(...args))

Diff for: ‎src/lib/geonetwork.js

+30-3
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@ export class Geonetwork {
2727
options = {
2828
responseText: false,
2929
},
30+
params = {},
3031
}) {
3132
// Request to get X-XSRF-TOKEN and Cookie: see docs https://geonetwork-opensource.org/manuals/3.10.x/en/customizing-application/misc.html
3233
const me = await fetch(this.#baseUrl + '/me', {
@@ -37,25 +38,33 @@ export class Geonetwork {
3738
})
3839

3940
const cookie = me.headers.get('set-cookie')
41+
4042
const token = cookie.split(';')[0].split('=')[1]
4143
const basicAuth =
4244
'Basic ' +
4345
Buffer.from(this.#username + ':' + this.#password).toString('base64')
4446

47+
const requestUrl = new URL(this.#baseUrl + url)
48+
49+
Object.entries(params).forEach(([key, value]) => {
50+
requestUrl.searchParams.set(key, value)
51+
})
52+
4553
// Use X-XSRF-TOKEN and Cookie in the request
46-
return fetch(this.#baseUrl + url, {
54+
return fetch(requestUrl.toString(), {
4755
method: method,
4856
...(method !== 'GET' && { body }),
4957
headers: {
5058
Authorization: basicAuth,
5159
'X-XSRF-TOKEN': token,
5260
Cookie: cookie.toString(),
53-
accept: 'application/json',
61+
Accept: 'application/json',
5462
...headers,
5563
},
5664
}).then(async (res) => {
57-
if(!res.ok) {
65+
if (!res.ok) {
5866
const error = await res.json()
67+
5968
throw new GeoNetworkError(`Error while posting to ${url}`, error)
6069
}
6170

@@ -79,4 +88,22 @@ export class Geonetwork {
7988
url,
8089
})
8190
}
91+
92+
putRecord(arg) {
93+
let url = '/records'
94+
95+
if (arg.url) {
96+
url += arg.url
97+
}
98+
99+
return this.#request({
100+
...arg,
101+
url,
102+
method: 'PUT',
103+
body: null,
104+
params: {
105+
...arg.params,
106+
},
107+
})
108+
}
82109
}

Diff for: ‎src/lib/metadata-formats.js

+245
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,245 @@
1+
/**
2+
* GeoNetwork XML Format Detection Utilities
3+
*
4+
* This file contains detection methods for various XML formats that
5+
* GeoNetwork transforms to ISO19139.
6+
*/
7+
8+
import { JSDOM } from 'jsdom';
9+
10+
/**
11+
* Main function to detect the format of an XML string from GeoNetwork
12+
* @param {string} xmlString - The XML response from GeoNetwork
13+
* @returns {string} The detected format transformation type
14+
*/
15+
export function detectFormatWith(xmlString) {
16+
const { window } = new JSDOM();
17+
const parser = new window.DOMParser();
18+
const xmlDoc = parser.parseFromString(xmlString, "text/xml");
19+
20+
// Check for parsing errors
21+
if (xmlDoc.documentElement.nodeName === "parsererror") {
22+
console.error("Error parsing XML:", xmlDoc.documentElement.textContent);
23+
return "Invalid XML";
24+
}
25+
26+
const rootElement = xmlDoc.documentElement;
27+
const namespaces = getNamespaces(rootElement);
28+
29+
// Run through various detection methods in order
30+
if (isDIF(rootElement, namespaces))
31+
return "DIF-to-ISO19139";
32+
33+
if (isEsriGeosticker(rootElement, namespaces))
34+
return "EsriGeosticker-to-ISO19139";
35+
36+
if (isISO19115(rootElement, namespaces))
37+
return "ISO19115-to-ISO19139";
38+
39+
if (isCSWGetCapabilities(rootElement, namespaces))
40+
return "OGCCSWGetCapabilities-to-ISO19119_ISO19139";
41+
42+
if (isOGCSLD(rootElement, namespaces))
43+
return "OGCSLD-to-ISO19139";
44+
45+
if (isSOSGetCapabilities(rootElement, namespaces))
46+
return "OGCSOSGetCapabilities-to-ISO19119_ISO19139";
47+
48+
if (isWCSGetCapabilities(rootElement, namespaces))
49+
return "OGCWCSGetCapabilities-to-ISO19119_ISO19139";
50+
51+
if (isWFSDescribeFeatureType(rootElement, namespaces))
52+
return "OGCWFSDescribeFeatureType-to-ISO19110";
53+
54+
if (isWFSGetCapabilities(rootElement, namespaces))
55+
return "OGCWFSGetCapabilities-to-ISO19119_ISO19139";
56+
57+
if (isWMCorOWSC(rootElement, namespaces))
58+
return "OGCWMC-OR-OWSC-to-ISO19139";
59+
60+
if (isWMSGetCapabilities(rootElement, namespaces))
61+
return "OGCWMSGetCapabilities-to-ISO19119_ISO19139";
62+
63+
if (isWPSGetCapabilities(rootElement, namespaces))
64+
return "OGCWPSGetCapabilities-to-ISO19119_ISO19139";
65+
66+
if (isGenericWxSGetCapabilities(rootElement, namespaces))
67+
return "OGCWxSGetCapabilities-to-ISO19119_ISO19139";
68+
69+
if (isThreddsCatalog(rootElement, namespaces))
70+
return "ThreddsCatalog-to-ISO19119_ISO19139";
71+
72+
// If already ISO19139, report it as such
73+
if (isISO19139(rootElement, namespaces))
74+
return null;
75+
76+
return "Unknown XML Format";
77+
}
78+
79+
/**
80+
* Helper function to extract all namespaces from an XML element
81+
* @param {Element} element - The XML element to extract namespaces from
82+
* @returns {Object} Object with namespace prefixes as keys and URIs as values
83+
*/
84+
function getNamespaces(element) {
85+
const namespaces = {};
86+
87+
// Get all attributes that define namespaces
88+
for (const attr of element.attributes) {
89+
if (attr.name.startsWith('xmlns:')) {
90+
const prefix = attr.name.split(':')[1];
91+
namespaces[prefix] = attr.value;
92+
} else if (attr.name === 'xmlns') {
93+
namespaces['default'] = attr.value;
94+
}
95+
}
96+
97+
return namespaces;
98+
}
99+
100+
/**
101+
* Detects if the XML is in DIF (Directory Interchange Format) format
102+
*/
103+
function isDIF(rootElement, namespaces) {
104+
return rootElement.nodeName === 'DIF' ||
105+
(namespaces.dif && rootElement.getElementsByTagNameNS(namespaces.dif, 'DIF').length > 0);
106+
}
107+
108+
/**
109+
* Detects if the XML is in ESRI Geosticker format
110+
*/
111+
function isEsriGeosticker(rootElement, namespaces) {
112+
return rootElement.nodeName === 'metadata' &&
113+
(rootElement.getAttribute('esri_format') === 'geosticker' ||
114+
rootElement.getElementsByTagName('esri').length > 0);
115+
}
116+
117+
/**
118+
* Detects if the XML is in ISO 19115 format
119+
*/
120+
function isISO19115(rootElement, namespaces) {
121+
return rootElement.nodeName === 'MD_Metadata' &&
122+
!namespaces.gmd && !namespaces.gco;
123+
}
124+
125+
/**
126+
* Detects if the XML is already in ISO 19139 format
127+
*/
128+
function isISO19139(rootElement, namespaces) {
129+
return (rootElement.nodeName === 'MD_Metadata' || rootElement.nodeName.endsWith(':MD_Metadata')) &&
130+
(namespaces.gmd || namespaces.gco);
131+
}
132+
133+
/**
134+
* Detects if the XML is a CSW GetCapabilities document
135+
*/
136+
function isCSWGetCapabilities(rootElement, namespaces) {
137+
return (rootElement.nodeName === 'Capabilities' || rootElement.nodeName.endsWith(':Capabilities')) &&
138+
(namespaces.csw ||
139+
(rootElement.getAttribute('service') === 'CSW' &&
140+
rootElement.getElementsByTagName('OperationsMetadata').length > 0));
141+
}
142+
143+
/**
144+
* Detects if the XML is an OGC SLD (Styled Layer Descriptor) document
145+
*/
146+
function isOGCSLD(rootElement, namespaces) {
147+
return rootElement.nodeName === 'StyledLayerDescriptor' ||
148+
rootElement.nodeName.endsWith(':StyledLayerDescriptor') ||
149+
namespaces.sld;
150+
}
151+
152+
/**
153+
* Detects if the XML is a SOS GetCapabilities document
154+
*/
155+
function isSOSGetCapabilities(rootElement, namespaces) {
156+
return (rootElement.nodeName === 'Capabilities' || rootElement.nodeName.endsWith(':Capabilities')) &&
157+
(namespaces.sos ||
158+
(rootElement.getAttribute('service') === 'SOS' &&
159+
rootElement.getElementsByTagName('OperationsMetadata').length > 0));
160+
}
161+
162+
/**
163+
* Detects if the XML is a WCS GetCapabilities document
164+
*/
165+
function isWCSGetCapabilities(rootElement, namespaces) {
166+
return (rootElement.nodeName === 'Capabilities' || rootElement.nodeName.endsWith(':Capabilities')) &&
167+
(namespaces.wcs ||
168+
(rootElement.getAttribute('service') === 'WCS' &&
169+
(rootElement.getElementsByTagName('ContentMetadata').length > 0 ||
170+
rootElement.getElementsByTagName('CoverageOfferingBrief').length > 0 ||
171+
rootElement.getElementsByTagName('Contents').length > 0)));
172+
}
173+
174+
/**
175+
* Detects if the XML is a WFS DescribeFeatureType document
176+
*/
177+
function isWFSDescribeFeatureType(rootElement, namespaces) {
178+
return (rootElement.nodeName === 'schema' || rootElement.nodeName.endsWith(':schema')) &&
179+
(namespaces.xsd || namespaces.xs) &&
180+
(rootElement.getAttribute('targetNamespace') &&
181+
rootElement.getAttribute('targetNamespace').includes('wfs'));
182+
}
183+
184+
/**
185+
* Detects if the XML is a WFS GetCapabilities document
186+
*/
187+
function isWFSGetCapabilities(rootElement, namespaces) {
188+
return (rootElement.nodeName === 'Capabilities' || rootElement.nodeName.endsWith(':Capabilities')) &&
189+
(namespaces.wfs ||
190+
(rootElement.getAttribute('service') === 'WFS' &&
191+
(rootElement.getElementsByTagName('FeatureTypeList').length > 0 ||
192+
rootElement.getElementsByTagName('FeatureType').length > 0)));
193+
}
194+
195+
/**
196+
* Detects if the XML is a Web Map Context (WMC) or OWS Context (OWSC) document
197+
*/
198+
function isWMCorOWSC(rootElement, namespaces) {
199+
return (rootElement.nodeName === 'ViewContext' || rootElement.nodeName.endsWith(':ViewContext') ||
200+
rootElement.nodeName === 'OWSContext' || rootElement.nodeName.endsWith(':OWSContext')) &&
201+
(namespaces.wmc || namespaces.owc || namespaces.owsc);
202+
}
203+
204+
/**
205+
* Detects if the XML is a WMS GetCapabilities document
206+
*/
207+
function isWMSGetCapabilities(rootElement, namespaces) {
208+
return (rootElement.nodeName === 'Capabilities' || rootElement.nodeName.endsWith(':Capabilities') ||
209+
rootElement.nodeName === 'WMS_Capabilities' || rootElement.nodeName.endsWith(':WMS_Capabilities')) &&
210+
(namespaces.wms ||
211+
(rootElement.getAttribute('service') === 'WMS' &&
212+
(rootElement.getElementsByTagName('Layer').length > 0 ||
213+
rootElement.getElementsByTagName('Capability').length > 0)));
214+
}
215+
216+
/**
217+
* Detects if the XML is a WPS GetCapabilities document
218+
*/
219+
function isWPSGetCapabilities(rootElement, namespaces) {
220+
return (rootElement.nodeName === 'Capabilities' || rootElement.nodeName.endsWith(':Capabilities')) &&
221+
(namespaces.wps ||
222+
(rootElement.getAttribute('service') === 'WPS' &&
223+
rootElement.getElementsByTagName('ProcessOfferings').length > 0));
224+
}
225+
226+
/**
227+
* Detects if the XML is a generic WxS GetCapabilities document
228+
* This is a fallback for other OGC web services not specifically handled
229+
*/
230+
function isGenericWxSGetCapabilities(rootElement, namespaces) {
231+
return (rootElement.nodeName === 'Capabilities' || rootElement.nodeName.endsWith(':Capabilities')) &&
232+
(rootElement.getAttribute('service') &&
233+
rootElement.getAttribute('service').match(/^W[A-Z]S$/) &&
234+
rootElement.getElementsByTagName('OperationsMetadata').length > 0);
235+
}
236+
237+
/**
238+
* Detects if the XML is a THREDDS catalog
239+
*/
240+
function isThreddsCatalog(rootElement, namespaces) {
241+
return (rootElement.nodeName === 'catalog' || rootElement.nodeName.endsWith(':catalog')) &&
242+
(namespaces.thredds ||
243+
rootElement.getAttribute('xmlns') === 'http://www.unidata.ucar.edu/namespaces/thredds/InvCatalog/v1.0' ||
244+
rootElement.getElementsByTagName('dataset').length > 0);
245+
}

0 commit comments

Comments
 (0)