Skip to content

Commit a2475fd

Browse files
DrabbleDrabble
authored
Create a simple Overture example (#862)
* Create a simple overture example * Fix OPTIONAL group cardinality * Clean up code * Clean up code 2 * Clean up code 3
1 parent cd2018d commit a2475fd

File tree

17 files changed

+307
-36
lines changed

17 files changed

+307
-36
lines changed

‎.run/overture-serve.run.xml

+11
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
<component name="ProjectRunConfigurationManager">
2+
<configuration default="false" name="overture-serve" type="Application" factoryName="Application">
3+
<option name="MAIN_CLASS_NAME" value="org.apache.baremaps.cli.Baremaps" />
4+
<module name="baremaps-cli" />
5+
<option name="PROGRAM_PARAMETERS" value="map serve --tileset tileset.json --style style.json" />
6+
<option name="WORKING_DIRECTORY" value="$PROJECT_DIR$/examples/overture" />
7+
<method v="2">
8+
<option name="Make" enabled="true" />
9+
</method>
10+
</configuration>
11+
</component>

‎.run/overture-workflow.run.xml

+11
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
<component name="ProjectRunConfigurationManager">
2+
<configuration default="false" name="overture-workflow" type="Application" factoryName="Application">
3+
<option name="MAIN_CLASS_NAME" value="org.apache.baremaps.cli.Baremaps" />
4+
<module name="baremaps-cli" />
5+
<option name="PROGRAM_PARAMETERS" value="workflow execute --file workflow.json" />
6+
<option name="WORKING_DIRECTORY" value="$PROJECT_DIR$/examples/overture" />
7+
<method v="2">
8+
<option name="Make" enabled="true" />
9+
</method>
10+
</configuration>
11+
</component>

‎baremaps-cli/src/main/resources/log4j.properties

+8
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
# Root logger option
2+
log4j.rootLogger=INFO, stdout
3+
4+
# Direct log messages to console
5+
log4j.appender.stdout=org.apache.log4j.ConsoleAppender
6+
log4j.appender.stdout.Target=System.out
7+
log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
8+
log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n

‎baremaps-core/src/main/java/org/apache/baremaps/storage/geoparquet/GeoParquetDataTable.java

+9-1
Original file line numberDiff line numberDiff line change
@@ -101,9 +101,17 @@ public DataSchema schema() {
101101
this.schema = GeoParquetTypeConversion.asSchema(path.toString(), schema);
102102
return this.schema;
103103
} catch (URISyntaxException e) {
104-
throw new GeoParquetException("Fail toe get the schema.", e);
104+
throw new GeoParquetException("Failed to get the schema.", e);
105105
}
106106
}
107107
return schema;
108108
}
109+
110+
public int srid(String column) {
111+
try {
112+
return reader().getGeoParquetMetadata().getSrid(column);
113+
} catch (Exception e) {
114+
throw new GeoParquetException("Fail to read the SRID from the GeoParquet metadata", e);
115+
}
116+
}
109117
}

‎baremaps-core/src/main/java/org/apache/baremaps/storage/geoparquet/GeoParquetTypeConversion.java

+7
Original file line numberDiff line numberDiff line change
@@ -70,6 +70,10 @@ public static List<Object> asRowValues(GeoParquetGroup group) {
7070
Schema schema = group.getSchema();
7171
List<Field> fields = schema.fields();
7272
for (int i = 0; i < fields.size(); i++) {
73+
if (group.getValues(i).isEmpty()) {
74+
values.add(null);
75+
continue;
76+
}
7377
Field field = fields.get(i);
7478
switch (field.type()) {
7579
case BINARY -> values.add(group.getBinaryValue(i).getBytes());
@@ -92,6 +96,9 @@ public static Map<String, Object> asNested(GeoParquetGroup group) {
9296
Schema schema = group.getSchema();
9397
List<Field> fields = schema.fields();
9498
for (int i = 0; i < fields.size(); i++) {
99+
if (group.getValues(i).isEmpty()) {
100+
continue;
101+
}
95102
Field field = fields.get(i);
96103
nested.put(field.name(), switch (field.type()) {
97104
case BINARY -> group.getBinaryValue(i).getBytes();

‎baremaps-core/src/main/java/org/apache/baremaps/workflow/Task.java

+1
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,7 @@
4747
@Type(value = ImportDaylightFeatures.class, name = "ImportDaylightFeatures"),
4848
@Type(value = ImportDaylightTranslations.class, name = "ImportDaylightTranslations"),
4949
@Type(value = ImportGeoPackage.class, name = "ImportGeoPackage"),
50+
@Type(value = ImportGeoParquet.class, name = "ImportGeoParquet"),
5051
@Type(value = ImportOsmOsc.class, name = "ImportOsmOsc"),
5152
@Type(value = ImportOsmPbf.class, name = "ImportOsmPbf"),
5253
@Type(value = ImportShapefile.class, name = "ImportShapefile"),

‎baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportGeoParquet.java

+97
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,97 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one or more
3+
* contributor license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright ownership.
5+
* The ASF licenses this file to you under the Apache License, Version 2.0
6+
* (the "License"); you may not use this file except in compliance with
7+
* the License. You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
18+
package org.apache.baremaps.workflow.tasks;
19+
20+
import java.net.URI;
21+
import java.util.StringJoiner;
22+
import org.apache.baremaps.data.storage.DataTableGeometryMapper;
23+
import org.apache.baremaps.data.storage.DataTableMapper;
24+
import org.apache.baremaps.openstreetmap.function.ProjectionTransformer;
25+
import org.apache.baremaps.storage.geoparquet.GeoParquetDataStore;
26+
import org.apache.baremaps.storage.geoparquet.GeoParquetDataTable;
27+
import org.apache.baremaps.storage.postgres.PostgresDataStore;
28+
import org.apache.baremaps.workflow.Task;
29+
import org.apache.baremaps.workflow.WorkflowContext;
30+
import org.slf4j.Logger;
31+
import org.slf4j.LoggerFactory;
32+
33+
/**
34+
* Import a GeoParquet into a database.
35+
*/
36+
public class ImportGeoParquet implements Task {
37+
38+
private static final Logger logger = LoggerFactory.getLogger(ImportGeoParquet.class);
39+
40+
private URI uri;
41+
private String tableName;
42+
private Object database;
43+
private Integer databaseSrid;
44+
45+
/**
46+
* Constructs a {@code ImportGeoParquet}.
47+
*/
48+
public ImportGeoParquet() {
49+
50+
}
51+
52+
/**
53+
* Constructs an {@code ImportGeoParquet}.
54+
*
55+
* @param uri the GeoParquet uri
56+
* @param database the database
57+
* @param databaseSrid the target SRID
58+
*/
59+
public ImportGeoParquet(URI uri, String tableName, Object database, Integer databaseSrid) {
60+
this.uri = uri;
61+
this.tableName = tableName;
62+
this.database = database;
63+
this.databaseSrid = databaseSrid;
64+
}
65+
66+
/**
67+
* {@inheritDoc}
68+
*/
69+
@Override
70+
public void execute(WorkflowContext context) throws Exception {
71+
var geoParquetDataStore = new GeoParquetDataStore(uri);
72+
var dataSource = context.getDataSource(database);
73+
var postgresDataStore = new PostgresDataStore(dataSource);
74+
for (var name : geoParquetDataStore.list()) {
75+
var geoParquetTable = (GeoParquetDataTable) geoParquetDataStore.get(name);
76+
var projectionTransformer =
77+
new ProjectionTransformer(geoParquetTable.srid("geometry"), databaseSrid);
78+
var rowTransformer =
79+
new DataTableGeometryMapper(geoParquetTable, projectionTransformer);
80+
var transformedDataTable =
81+
new DataTableMapper(geoParquetDataStore.get(name), rowTransformer);
82+
postgresDataStore.add(tableName, transformedDataTable);
83+
}
84+
}
85+
86+
/**
87+
* {@inheritDoc}
88+
*/
89+
@Override
90+
public String toString() {
91+
return new StringJoiner(", ", ImportGeoParquet.class.getSimpleName() + "[", "]")
92+
.add("uri=" + uri)
93+
.add("database=" + database)
94+
.add("databaseSrid=" + databaseSrid)
95+
.toString();
96+
}
97+
}

‎baremaps-geoparquet/src/main/java/org/apache/baremaps/geoparquet/GeoParquetReader.java

+13-33
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,8 @@
3636
import org.apache.hadoop.fs.FileStatus;
3737
import org.apache.hadoop.fs.FileSystem;
3838
import org.apache.hadoop.fs.Path;
39+
import org.apache.hadoop.fs.s3a.AnonymousAWSCredentialsProvider;
40+
import org.apache.hadoop.fs.s3a.S3AFileSystem;
3941
import org.apache.parquet.hadoop.ParquetFileReader;
4042
import org.apache.parquet.hadoop.ParquetReader;
4143
import org.apache.parquet.schema.MessageType;
@@ -91,16 +93,14 @@ public Long size() throws URISyntaxException {
9193
return files().values().stream().map(FileInfo::recordCount).reduce(0L, Long::sum);
9294
}
9395

94-
private synchronized Map<FileStatus, FileInfo> files() throws URISyntaxException {
96+
private synchronized Map<FileStatus, FileInfo> files() {
9597
try {
9698
if (files == null) {
9799
files = new HashMap<>();
98-
Path globPath = new Path(uri.getPath());
99-
URI rootUri = getRootUri(uri);
100-
FileSystem fileSystem = FileSystem.get(rootUri, configuration);
100+
FileSystem fs = FileSystem.get(uri, configuration);
101+
FileStatus[] fileStatuses = fs.globStatus(new Path(uri));
101102

102-
// Iterate over all the files in the path
103-
for (FileStatus file : fileSystem.globStatus(globPath)) {
103+
for (FileStatus file : fileStatuses) {
104104
files.put(file, buildFileInfo(file));
105105
}
106106

@@ -115,7 +115,7 @@ private synchronized Map<FileStatus, FileInfo> files() throws URISyntaxException
115115
}
116116
}
117117
} catch (IOException e) {
118-
throw new GeoParquetException("IOException while attempting to list files.", e);
118+
throw new GeoParquetException("IOException while attempting to list files.", e);
119119
}
120120
return files;
121121
}
@@ -254,31 +254,11 @@ public int characteristics() {
254254
}
255255

256256
private static Configuration createConfiguration() {
257-
Configuration configuration = new Configuration();
258-
configuration.set("fs.s3a.aws.credentials.provider",
259-
"org.apache.hadoop.fs.s3a.AnonymousAWSCredentialsProvider");
260-
configuration.setBoolean("fs.s3a.path.style.access", true);
261-
return configuration;
257+
Configuration conf = new Configuration();
258+
conf.set("fs.s3a.endpoint", "s3.us-west-2.amazonaws.com");
259+
conf.set("fs.s3a.aws.credentials.provider", AnonymousAWSCredentialsProvider.class.getName());
260+
conf.set("fs.s3a.impl", S3AFileSystem.class.getName());
261+
conf.set("fs.s3a.path.style.access", "true");
262+
return conf;
262263
}
263-
264-
private static URI getRootUri(URI uri) throws URISyntaxException {
265-
// TODO:
266-
// This is a quick and dirty way to get the root uri of the path.
267-
// We take everything before the first wildcard in the path.
268-
// This is not a perfect solution, and we should probably look for a better way to do this.
269-
String path = uri.getPath();
270-
int index = path.indexOf("*");
271-
if (index != -1) {
272-
path = path.substring(0, path.lastIndexOf("/", index) + 1);
273-
}
274-
return new URI(
275-
uri.getScheme(),
276-
uri.getUserInfo(),
277-
uri.getHost(),
278-
uri.getPort(),
279-
path,
280-
null,
281-
null);
282-
}
283-
284264
}

‎baremaps-geoparquet/src/main/java/org/apache/baremaps/geoparquet/data/GeoParquetGroup.java

+2
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,8 @@ public interface GeoParquetGroup {
5959
*/
6060
GeoParquetGroup createGroup(int fieldIndex);
6161

62+
List<Primitive> getValues(int fieldIndex);
63+
6264
Binary getBinaryValue(int fieldIndex);
6365

6466
List<Binary> getBinaryValues(int fieldIndex);

‎baremaps-geoparquet/src/main/java/org/apache/baremaps/geoparquet/data/GeoParquetGroupFactory.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -69,7 +69,7 @@ else if (!field.isPrimitive()) {
6969
GeoParquetGroup.Schema geoParquetSchema = createGeoParquetSchema(groupType, metadata);
7070
return (Field) new GeoParquetGroup.GroupField(
7171
groupType.getName(),
72-
GeoParquetGroup.Cardinality.REQUIRED,
72+
cardinality,
7373
geoParquetSchema);
7474
}
7575

‎baremaps-geoparquet/src/main/java/org/apache/baremaps/geoparquet/data/GeoParquetGroupImpl.java

+2-1
Original file line numberDiff line numberDiff line change
@@ -278,7 +278,8 @@ private void appendToString(StringBuilder builder, String indent) {
278278
}
279279
}
280280

281-
private List<Primitive> getValues(int fieldIndex) {
281+
@Override
282+
public List<Primitive> getValues(int fieldIndex) {
282283
return (List<Primitive>) data[fieldIndex];
283284
}
284285

‎examples/overture/indexes.sql

+15
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
-- Licensed to the Apache Software Foundation (ASF) under one or more
2+
-- contributor license agreements. See the NOTICE file distributed with
3+
-- this work for additional information regarding copyright ownership.
4+
-- The ASF licenses this file to you under the Apache License, Version 2.0
5+
-- (the "License"); you may not use this file except in compliance with
6+
-- the License. You may obtain a copy of the License at
7+
--
8+
-- http://www.apache.org/licenses/LICENSE-2.0
9+
--
10+
-- Unless required by applicable law or agreed to in writing, software
11+
-- distributed under the License is distributed on an "AS IS" BASIS,
12+
-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
-- See the License for the specific language governing permissions and
14+
-- limitations under the License.
15+
CREATE INDEX IF NOT EXISTS overture_admins_administrative_boundary_materialized_view_gist ON overture_admins_administrative_boundary_materialized_view USING GIST(geom);

‎examples/overture/style.json

+24
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
{
2+
"version" : 8,
3+
"sources" : {
4+
"baremaps" : {
5+
"type" : "vector",
6+
"url" : "http://localhost:9000/tiles.json"
7+
}
8+
},
9+
"layers" : [ {
10+
"id" : "administrative_boundary",
11+
"type" : "line",
12+
"source" : "baremaps",
13+
"source-layer" : "administrative_boundary",
14+
"layout" : {
15+
"visibility" : "visible"
16+
},
17+
"paint" : {
18+
"line-color": "black",
19+
"line-width": 1
20+
}
21+
}],
22+
"center" : [ 0, 0 ],
23+
"zoom" : 2
24+
}

‎examples/overture/tiles.json

+24
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
{
2+
"tilejson": "2.2.0",
3+
"tiles": [
4+
"http://localhost:9000/tiles/{z}/{x}/{y}.mvt"
5+
],
6+
"minzoom": 0.0,
7+
"maxzoom": 14.0,
8+
"center": [0, 0],
9+
"bounds": [-180, -85, 180, 85],
10+
"zoom": 2,
11+
"database": "jdbc:postgresql://localhost:5432/baremaps?&user=baremaps&password=baremaps",
12+
"vector_layers": [
13+
{
14+
"id": "administrative_boundary",
15+
"queries": [
16+
{
17+
"minzoom": 0,
18+
"maxzoom": 14,
19+
"sql": "SELECT id, tags, geom FROM overture_admins_administrative_boundary_materialized_view"
20+
}
21+
]
22+
}
23+
]
24+
}

‎examples/overture/tileset.json

+24
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
{
2+
"tilejson": "2.2.0",
3+
"minzoom": 0,
4+
"maxzoom": 14,
5+
"center": [0, 0],
6+
"bounds": [-180, -85, 180, 85],
7+
"zoom": 2,
8+
"tiles": [
9+
"http://localhost:9000/tiles/{z}/{x}/{y}.mvt"
10+
],
11+
"database": "jdbc:postgresql://localhost:5432/baremaps?&user=baremaps&password=baremaps",
12+
"vector_layers": [
13+
{
14+
"id": "administrative_boundary",
15+
"queries": [
16+
{
17+
"minzoom": 0,
18+
"maxzoom": 14,
19+
"sql": "SELECT id, tags, geom FROM overture_admins_administrative_boundary_materialized_view"
20+
}
21+
]
22+
}
23+
]
24+
}

0 commit comments

Comments
 (0)