Refactor to allow layers from various datasources to be used together.

Preparation for a JTS-based centroids layer.

geocode-api/README.md

@@ -1,8 +1,12 @@
-#geocode-api
+# Geocode API
 
-Service and model classes for the reverse geocode service, it only contains the GeocodeService.get(lat,lon) service and the Location instance returned by it.
-
-##How to build this project
+Service and model classes for the reverse geocode service.
+
+It contains the `GeocodeService.get(latitude, longitude, uncertaintyDegrees, uncertaintyMeters, layers)` service and the Location instance returned by it.
+
+It also contains a bitmap cache implementation for use by clients.
+
+## How to build this project
 Execute the Maven command:
 
 ```

geocode-api/src/main/java/org/gbif/geocode/api/cache/AbstractBitmapCachedLayer.java

@@ -1,6 +1,5 @@
 package org.gbif.geocode.api.cache;
 
-import org.gbif.common.shaded.com.google.common.base.Stopwatch;
 import org.gbif.geocode.api.model.Location;
 
 import java.awt.image.BufferedImage;
@@ -10,7 +9,6 @@
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
-import java.util.concurrent.TimeUnit;
 import java.util.stream.Collectors;
 
 import javax.imageio.ImageIO;
@@ -37,6 +35,8 @@ public abstract class AbstractBitmapCachedLayer {
   private final Map<Integer, List<Location>> colourKey = new HashMap<>();
   public long queries = 0, border = 0, empty = 0, miss = 0, hit = 0;
 
+  private static final double BITMAP_UNCERTAINTY_DEGREES = 0.05d;
+
   public AbstractBitmapCachedLayer(InputStream bitmap) {
     this(bitmap, 1);
   }
@@ -52,15 +52,43 @@ public AbstractBitmapCachedLayer(InputStream bitmap, int maxLocations) {
     }
   }
 
+  /**
+   * Layer name, used in API responses and logging.
+   */
   public abstract String name();
 
+  /**
+   * Layer source (e.g. a URL), used in API responses.
+   */
   public abstract String source();
 
+  /**
+   * Query the layer, using the bitmap cache first if the uncertainty allows it.
+   */
+  public final List<Location> query(double latitude, double longitude, double uncertaintyDegrees) {
+    List<Location> found = null;
+    if (uncertaintyDegrees <= BITMAP_UNCERTAINTY_DEGREES) { // TODO per layer
+      found = checkBitmap(latitude, longitude);
+    }
+
+    if (found == null) {
+      found = queryDatasource(latitude, longitude, uncertaintyDegrees);
+      putBitmap(latitude, longitude, found);
+    }
+
+    return found;
+  }
+
+  /**
+   * Query the underlying datasource (PostGIS, shapefile etc).
+   */
+  protected abstract List<Location> queryDatasource(double latitude, double longitude, double uncertainty);
+
   /**
    * Check the colour of a pixel from the map image to determine the country.
    * @return Locations or null if the bitmap can't answer.
    */
-  public List<Location> checkBitmap(double lat, double lng) {
+  List<Location> checkBitmap(double lat, double lng) {
     // Convert the latitude and longitude to x,y coordinates on the image.
     // The axes are swapped, and the image's origin is the top left.
     int x = (int) (Math.round((lng + 180d) / 360d * (imgWidth - 1)));
@@ -106,9 +134,9 @@ public List<Location> checkBitmap(double lat, double lng) {
   }
 
   /**
-   * Store a result in the bitmap's cache, if it's not a border region.
+   * Store a result in the bitmap cache, if it's not a border region.
    */
-  public void putBitmap(double lat, double lng, List<Location> locations) {
+  void putBitmap(double lat, double lng, List<Location> locations) {
     // Convert the latitude and longitude to x,y coordinates on the image.
     // The axes are swapped, and the image's origin is the top left.
     int x = (int) (Math.round ((lng+180d)/360d*(imgWidth -1)));

geocode-api/src/main/java/org/gbif/geocode/api/cache/GeocodeBitmapCache.java

@@ -86,6 +86,11 @@ public List<Location> get(Double lat, Double lng, Double uncertaintyDegrees, Dou
     return locations;
   }
 
+  @Override
+  protected List<Location> queryDatasource(double latitude, double longitude, double uncertainty) {
+    return get(latitude, longitude, uncertainty, null);
+  }
+
   @Override
   public byte[] bitmap() {
     return new byte[0];

geocode-api/src/main/java/org/gbif/geocode/api/model/Location.java

@@ -58,19 +58,22 @@ public int compare(Location o1, Location o2) {
    */
   private Double distance;
 
+  private Double distanceMeters;
+
   public Location() {
     // For mybatis & jackson
   }
 
   public Location(
-    String id, String type, String source, String title, String isoCountryCode2Digit, Double distance
+    String id, String type, String source, String title, String isoCountryCode2Digit, Double distance, Double distanceMeters
   ) {
     this.id = id;
     this.type = type;
     this.source = source;
     this.title = title;
     this.isoCountryCode2Digit = isoCountryCode2Digit;
     this.distance = distance;
+    this.distanceMeters = distanceMeters;
   }
 
   public String getIsoCountryCode2Digit() {
@@ -121,6 +124,22 @@ public void setDistance(Double distance) {
     this.distance = distance;
   }
 
+  public Double getDistanceMeters() {
+    return distanceMeters;
+  }
+
+  public void setDistanceMeters(Double distanceMeters) {
+    this.distanceMeters = distanceMeters;
+  }
+
+  public void calculateDistanceMetersFromLatitude(double latitude) {
+    this.distanceMeters = 111_319.491 * distance * Math.cos(Math.toRadians(latitude));
+  }
+
+  public void calculateDistanceDegreesFromMeters(double latitude) {
+    this.distance = distanceMeters / (111_319.491 * Math.cos(Math.toRadians(latitude)));
+  }
+
   @Override
   public boolean equals(Object o) {
     if (this == o) return true;
@@ -131,7 +150,8 @@ public boolean equals(Object o) {
       Objects.equals(source, location.source) &&
       Objects.equals(title, location.title) &&
       Objects.equals(isoCountryCode2Digit, location.isoCountryCode2Digit) &&
-      Objects.equals(distance, location.distance);
+      Objects.equals(distance, location.distance) &&
+      Objects.equals(distanceMeters, location.distanceMeters);
   }
 
   @Override
@@ -148,6 +168,7 @@ public String toString() {
       ", title='" + title + '\'' +
       ", isoCountryCode2Digit='" + isoCountryCode2Digit + '\'' +
       ", distance=" + distance +
+      ", distanceMeters=" + distanceMeters +
       '}';
   }
 

geocode-api/src/test/java/org/gbif/geocode/api/model/LocationTest.java

@@ -9,7 +9,7 @@ public class LocationTest {
 
   @Test
   public void testEquals() {
-    Location locationOne = new Location("id", "type", "source", "title", "iso", 0d);
+    Location locationOne = new Location("id", "type", "source", "title", "iso", 0d, 0d);
     Location locationTwo = new Location();
 
     MatcherAssert.assertThat(locationOne, CoreMatchers.not(CoreMatchers.equalTo(locationTwo)));
@@ -25,16 +25,17 @@ public void testEquals() {
 
     locationTwo.setIsoCountryCode2Digit("iso");
     locationTwo.setDistance(0d);
+    locationTwo.setDistanceMeters(0d);
     MatcherAssert.assertThat(locationOne, CoreMatchers.equalTo(locationTwo));
     MatcherAssert.assertThat(locationOne.hashCode(), CoreMatchers.equalTo(locationTwo.hashCode()));
   }
 
   @Test
   public void testCompare() {
-    Location locationOne = new Location("EC", "EEZ", "EEZ", "EC", "EC", 0d);
-    Location locationOneB = new Location("EC", "EEZ", "EEZ", "EC", "EC", 0d);
-    Location locationTwo = new Location("EC", "Political", "Political", "EC", "EC", 0.015848712600069228d);
-    Location locationThree = new Location("EC", "Political", "Political", "EC", "EC", 0d);
+    Location locationOne = new Location("EC", "EEZ", "EEZ", "EC", "EC", 0d, 0d);
+    Location locationOneB = new Location("EC", "EEZ", "EEZ", "EC", "EC", 0d, 0d);
+    Location locationTwo = new Location("EC", "Political", "Political", "EC", "EC", 0.015848712600069228d, 111_319.491 * 0.015848712600069228d);
+    Location locationThree = new Location("EC", "Political", "Political", "EC", "EC", 0d, 0d);
 
     Assertions.assertTrue(Location.DISTANCE_COMPARATOR.compare(locationOne, locationTwo) < 0);
     Assertions.assertTrue(Location.DISTANCE_COMPARATOR.compare(locationOne, locationOneB) == 0);

geocode-ws/README.md

@@ -1,10 +1,12 @@
-# geocode-ws
+# Geocode WS
 
 RESTful service that provides the reverse geocode functionality. The REST resource should be accessible at this URL: `http://{server}:{httpPort}/reverse`.
 
+The GBIF instance is available at `https://api.gbif.org/v1/geocode/reverse`.
+
 ## How to build and run this project
 
-The database url and credentials are required to start the application. You can provide them in different ways.
+Shapefiles and the database URL and credentials are required to start the application. You can provide them in different ways.
 
 ### Using maven
 You can put the DB credentials in a maven profile:
@@ -20,9 +22,9 @@ You can put the DB credentials in a maven profile:
 </profile>
 ```
 
-Then start the application with maven and your profile: ` mvn -Pgeocode-ws spring-boot:run`
+Then start the application with maven and your profile: `mvn -Pgeocode-ws spring-boot:run`
 
-### Using a java command
+### Using a Java command
 
 You can create an [application.yml](src/resources/application.yml) and a [bootstrap.yml](src/resources/bootstrap.yml) files
 and pass them to the application:
@@ -46,18 +48,32 @@ Response should be:
 
 ```json
 [
-  {
-    "title": "PA:6",
-    "id": "6"
-  }
-]
+    {
+        "id": "EUROPE",
+        "type": "Continent",
+        "source": "https://github.com/gbif/continents",
+        "title": "EUROPE",
+        "isoCountryCode2Digit": "",
+        "distance": 0.0,
+        "distanceMeters": 0.0
+    },
+    {
+        "id": "http://marineregions.org/mrgid/5674",
+        "type": "Political",
+        "source": "https://www.marineregions.org/",
+        "title": "Denmark",
+        "isoCountryCode2Digit": "DK",
+        "distance": 0.0,
+        "distanceMeters": 0.0
+    },
+    …
 ```
 
-## Create Database
+You can also try http://localhost:8080/geocode/debug/map.html
 
-At the moment we have two sources of data: Natural Earth and EEZ.
+## Create the database and shapefiles
 
-See [../database/scripts/import.sh](../database/scripts/import.sh) for a script to import the database. With appropriate environment variables, it can be used against non-Docker databases.
+See [../database/scripts/import.sh](../database/scripts/import.sh) for a script to import the database, and from it export shapefiles. With appropriate environment variables, it can be used against non-Docker databases.
 
 ## Map image for faster lookups
 
@@ -68,4 +84,3 @@ There is a PNG image used to speed up queries — after an initial query, roughl
 See [Map Image Lookup](./MapImageLookup.adoc) for how the image is created.
 
 The layers GADM0, GADM1 and GADM2 are queried at the same time as GADM3.  Since GBIF usage is for GADM3, efficient caching of the lower layers isn't implemented
-

geocode-ws/pom.xml

@@ -28,6 +28,13 @@
       </resource>
     </resources>
 
+    <testResources>
+      <testResource>
+        <directory>src/test/resources</directory>
+        <filtering>true</filtering>
+      </testResource>
+    </testResources>
+
     <plugins>
       <plugin>
         <groupId>org.codehaus.mojo</groupId>
@@ -122,13 +129,6 @@
     </plugins>
   </build>
 
-  <repositories>
-    <repository>
-      <id>gbif-all</id>
-      <url>http://repository.gbif.org/content/groups/gbif</url>
-    </repository>
-  </repositories>
-
   <dependencies>
     <!-- Spring dependencies -->
     <dependency>
@@ -221,6 +221,14 @@
       <artifactId>logstash-logback-encoder</artifactId>
       <scope>runtime</scope>
     </dependency>
+    <dependency>
+      <groupId>org.geotools</groupId>
+      <artifactId>gt-epsg-hsql</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.geotools</groupId>
+      <artifactId>gt-referencing</artifactId>
+    </dependency>
 
     <!-- dependencies to register to ZK -->
     <dependency>

geocode-ws/src/main/java/org/gbif/geocode/ws/layers/Bitmap.java

@@ -0,0 +1,7 @@
+package org.gbif.geocode.ws.layers;
+
+/**
+ * This class exists as a convenience for loading the bitmap resources.
+ */
+public class Bitmap {
+}