Skip to content

Commit 258cc46

Browse files
takbtakb
authored
Merge pull request #94 from GIScience/feature/update-gh-dataaccess
feat(gh update dataaccess): Update DataAccess to latest graphhoper version
2 parents d1493d0 + f278a8a commit 258cc46

31 files changed

+417
-424
lines changed

‎core/pom.xml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -189,8 +189,8 @@
189189
<groupId>org.apache.maven.plugins</groupId>
190190
<artifactId>maven-compiler-plugin</artifactId>
191191
<configuration>
192-
<source>8</source>
193-
<target>8</target>
192+
<source>17</source>
193+
<target>17</target>
194194
</configuration>
195195
</plugin>
196196
</plugins>

‎core/src/main/java/com/graphhopper/GraphHopper.java

Lines changed: 21 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -96,7 +96,8 @@ public class GraphHopper {
9696
private EncodingManager encodingManager;
9797
private int defaultSegmentSize = -1;
9898
private String ghLocation = "";
99-
private DAType dataAccessType = DAType.RAM_STORE;
99+
private DAType dataAccessDefaultType = DAType.RAM_STORE;
100+
private final LinkedHashMap<String, String> dataAccessConfig = new LinkedHashMap<>();
100101
private boolean sortGraph = false;
101102
private boolean elevation = false;
102103
private LockFactory lockFactory = new NativeFSLockFactory();
@@ -227,9 +228,9 @@ public GraphHopper setMinNetworkSize(int minNetworkSize) {
227228
public GraphHopper setStoreOnFlush(boolean storeOnFlush) {
228229
ensureNotLoaded();
229230
if (storeOnFlush)
230-
dataAccessType = DAType.RAM_STORE;
231+
dataAccessDefaultType = DAType.RAM_STORE;
231232
else
232-
dataAccessType = DAType.RAM;
233+
dataAccessDefaultType = DAType.RAM;
233234
return this;
234235
}
235236

@@ -502,8 +503,14 @@ public GraphHopper init(GraphHopperConfig ghConfig) {
502503
setGraphHopperLocation(graphHopperFolder);
503504
defaultSegmentSize = ghConfig.getInt("graph.dataaccess.segment_size", defaultSegmentSize);
504505

505-
String graphDATypeStr = ghConfig.getString("graph.dataaccess", "RAM_STORE");
506-
dataAccessType = DAType.fromString(graphDATypeStr);
506+
String daTypeString = ghConfig.getString("graph.dataaccess.default_type", ghConfig.getString("graph.dataaccess", "RAM_STORE"));
507+
dataAccessDefaultType = DAType.fromString(daTypeString);
508+
for (Map.Entry<String, Object> entry : ghConfig.asPMap().toMap().entrySet()) {
509+
if (entry.getKey().startsWith("graph.dataaccess.type."))
510+
dataAccessConfig.put(entry.getKey().substring("graph.dataaccess.type.".length()), entry.getValue().toString());
511+
if (entry.getKey().startsWith("graph.dataaccess.mmap.preload."))
512+
dataAccessConfig.put(entry.getKey().substring("graph.dataaccess.mmap.".length()), entry.getValue().toString());
513+
}
507514

508515
sortGraph = ghConfig.getBool("graph.do_sort", sortGraph);
509516
removeZipped = ghConfig.getBool("graph.remove_zipped", removeZipped);
@@ -811,22 +818,24 @@ public boolean load(String graphHopperFolder) {
811818

812819
setGraphHopperLocation(graphHopperFolder);
813820

814-
if (!allowWrites && dataAccessType.isMMap())
815-
dataAccessType = DAType.MMAP_RO;
821+
if (!allowWrites && dataAccessDefaultType.isMMap())
822+
dataAccessDefaultType = DAType.MMAP_RO;
816823
if (encodingManager == null) {
817-
StorableProperties properties = new StorableProperties(new GHDirectory(ghLocation, dataAccessType));
824+
StorableProperties properties = new StorableProperties(new GHDirectory(ghLocation, dataAccessDefaultType));
818825
encodingManager = properties.loadExisting()
819826
? EncodingManager.create(emBuilder, encodedValueFactory, flagEncoderFactory, properties)
820827
: buildEncodingManager(new GraphHopperConfig());
821828
}
822829

823-
GHDirectory dir = new GHDirectory(ghLocation, dataAccessType);
830+
GHDirectory directory = new GHDirectory(ghLocation, dataAccessDefaultType);
831+
directory.configure(dataAccessConfig);
824832

825833
// ORS-GH MOD START use storage factory in ORSGraphHopper
826834
if (graphStorageFactory != null) {
827-
ghStorage = graphStorageFactory.createStorage(dir, this);
835+
ghStorage = graphStorageFactory.createStorage(directory, this);
828836
} else {
829-
ghStorage = new GraphHopperStorage(dir, encodingManager, hasElevation(), encodingManager.needsTurnCostsSupport(), defaultSegmentSize);
837+
838+
ghStorage = new GraphHopperStorage(directory, encodingManager, hasElevation(), encodingManager.needsTurnCostsSupport(), defaultSegmentSize);
830839
}
831840
// ORS-GH MOD END
832841
checkProfilesConsistency();
@@ -866,6 +875,7 @@ public boolean load(String graphHopperFolder) {
866875
return false;
867876

868877
postProcessing(false);
878+
directory.loadMMap();
869879
setFullyLoaded();
870880
return true;
871881
} finally {

‎core/src/main/java/com/graphhopper/reader/dem/AbstractSRTMElevationProvider.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -119,7 +119,7 @@ public double getEle(double lat, double lon) {
119119
demProvider.setHeights(heights);
120120
demProvider.setSeaLevel(true);
121121
// use small size on disc and in-memory
122-
heights.setSegmentSize(100).create(10).
122+
heights.create(10).
123123
flush();
124124
return 0;
125125
}

‎core/src/main/java/com/graphhopper/reader/dem/AbstractTiffElevationProvider.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -135,7 +135,7 @@ public double getEle(double lat, double lon) {
135135
} catch (IOException e) {
136136
demProvider.setSeaLevel(true);
137137
// use small size on disc and in-memory
138-
heights.setSegmentSize(100).create(10).
138+
heights.create(10).
139139
flush();
140140
return 0;
141141
}

‎core/src/main/java/com/graphhopper/routing/lm/LandmarkStorage.java

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -141,7 +141,7 @@ public String toString() {
141141
// In this sense its even 'better' to use node-based.
142142
this.traversalMode = TraversalMode.NODE_BASED;
143143
// ORS-GH MOD START
144-
this.landmarkWeightDA = dir.find(getLandmarksFileName() + lmConfig.getName());
144+
this.landmarkWeightDA = dir.create(getLandmarksFileName() + lmConfig.getName());
145145
// ORS-GH MOD END
146146

147147
this.landmarks = landmarks;
@@ -837,6 +837,14 @@ public interface LandmarkExplorer extends RoutingAlgorithm{
837837

838838
void initLandmarkWeights(final int lmIdx, int lmNodeId, final long rowSize, final int offset);
839839
}
840+
841+
/**
842+
* For testing only
843+
*/
844+
DataAccess _getInternalDA() {
845+
return landmarkWeightDA;
846+
}
847+
840848
/**
841849
* This class is used to calculate landmark location (equally distributed).
842850
* It derives from DijkstraBidirectionRef, but is only used as forward or backward search.

‎core/src/main/java/com/graphhopper/search/NameIndex.java

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -137,10 +137,6 @@ public boolean isClosed() {
137137
return names.isClosed();
138138
}
139139

140-
public void setSegmentSize(int segments) {
141-
names.setSegmentSize(segments);
142-
}
143-
144140
public long getCapacity() {
145141
return names.getCapacity();
146142
}

‎core/src/main/java/com/graphhopper/search/StringIndex.java

Lines changed: 5 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -50,17 +50,12 @@ public class StringIndex {
5050
private long lastEntryPointer = -1;
5151
private Map<String, String> lastEntryMap;
5252

53-
public StringIndex(Directory dir) {
54-
this(dir, 1000);
55-
}
56-
5753
/**
5854
* Specify a larger cacheSize to reduce disk usage. Note that this increases the memory usage of this object.
5955
*/
60-
public StringIndex(Directory dir, final int cacheSize) {
61-
keys = dir.find("string_index_keys");
62-
keys.setSegmentSize(10 * 1024);
63-
vals = dir.find("string_index_vals");
56+
public StringIndex(Directory dir, final int cacheSize, final int segmentSize) {
57+
keys = dir.create("string_index_keys", segmentSize);
58+
vals = dir.create("string_index_vals", segmentSize);
6459
smallCache = new LinkedHashMap<String, Long>(cacheSize, 0.75f, true) {
6560
@Override
6661
protected boolean removeEldestEntry(Map.Entry<String, Long> entry) {
@@ -196,7 +191,7 @@ public Map<String, String> getAll(final long entryPointer) {
196191
if (keyCount == 0)
197192
return Collections.emptyMap();
198193

199-
Map<String, String> map = new HashMap<>(keyCount);
194+
Map<String, String> map = new LinkedHashMap<>(keyCount);
200195
long tmpPointer = entryPointer + 1;
201196
for (int i = 0; i < keyCount; i++) {
202197
int currentKeyIndex = vals.getShort(tmpPointer);
@@ -282,7 +277,7 @@ public String get(final long entryPointer, String key) {
282277
tmpPointer += 1 + valueLength;
283278
}
284279

285-
// value for specified key does not existing for the specified pointer
280+
// value for specified key does not exist for the specified pointer
286281
return null;
287282
}
288283

@@ -327,11 +322,6 @@ public boolean isClosed() {
327322
return vals.isClosed() && keys.isClosed();
328323
}
329324

330-
public void setSegmentSize(int segments) {
331-
keys.setSegmentSize(segments);
332-
vals.setSegmentSize(segments);
333-
}
334-
335325
public long getCapacity() {
336326
return vals.getCapacity() + keys.getCapacity();
337327
}

‎core/src/main/java/com/graphhopper/storage/AbstractDataAccess.java

Lines changed: 10 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,6 @@
2020
import com.graphhopper.util.BitUtil;
2121
import com.graphhopper.util.Helper;
2222

23-
import java.io.File;
2423
import java.io.IOException;
2524
import java.io.RandomAccessFile;
2625
import java.nio.ByteOrder;
@@ -34,24 +33,25 @@ public abstract class AbstractDataAccess implements DataAccess {
3433
protected static final int HEADER_OFFSET = 20 * 4 + 20;
3534
protected static final byte[] EMPTY = new byte[1024];
3635
private static final int SEGMENT_SIZE_DEFAULT = 1 << 20;
37-
protected final ByteOrder byteOrder;
38-
protected final BitUtil bitUtil;
36+
protected final ByteOrder byteOrder = ByteOrder.LITTLE_ENDIAN;
37+
protected final BitUtil bitUtil = BitUtil.LITTLE;
3938
private final String location;
40-
protected int header[] = new int[(HEADER_OFFSET - 20) / 4];
39+
protected int[] header = new int[(HEADER_OFFSET - 20) / 4];
4140
protected String name;
42-
protected int segmentSizeInBytes = SEGMENT_SIZE_DEFAULT;
41+
protected int segmentSizeInBytes;
4342
protected int segmentSizePower;
4443
protected int indexDivisor;
4544
protected boolean closed = false;
4645

47-
public AbstractDataAccess(String name, String location, ByteOrder order) {
48-
byteOrder = order;
49-
bitUtil = BitUtil.get(order);
46+
public AbstractDataAccess(String name, String location, int segmentSize) {
5047
this.name = name;
5148
if (!Helper.isEmpty(location) && !location.endsWith("/"))
5249
throw new IllegalArgumentException("Create DataAccess object via its corresponding Directory!");
5350

5451
this.location = location;
52+
if (segmentSize < 0)
53+
segmentSize = SEGMENT_SIZE_DEFAULT;
54+
setSegmentSize(segmentSize);
5555
}
5656

5757
@Override
@@ -105,7 +105,7 @@ protected long readHeader(RandomAccessFile raFile) throws IOException {
105105

106106
String versionHint = raFile.readUTF();
107107
if (!"GH".equals(versionHint))
108-
throw new IllegalArgumentException("Not a GraphHopper file! Expected 'GH' as file marker but was " + versionHint);
108+
throw new IllegalArgumentException("Not a GraphHopper file " + getFullName() + "! Expected 'GH' as file marker but was " + versionHint);
109109

110110
long bytes = raFile.readLong();
111111
setSegmentSize(raFile.readInt());
@@ -121,8 +121,7 @@ protected void copyHeader(DataAccess da) {
121121
}
122122
}
123123

124-
@Override
125-
public DataAccess setSegmentSize(int bytes) {
124+
DataAccess setSegmentSize(int bytes) {
126125
if (bytes > 0) {
127126
// segment size should be a power of 2
128127
int tmp = (int) (Math.log(bytes) / Math.log(2));

‎core/src/main/java/com/graphhopper/storage/BaseGraph.java

Lines changed: 7 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -90,20 +90,16 @@ class BaseGraph implements Graph {
9090
public BaseGraph(Directory dir, int intsForFlags, boolean withElevation, boolean withTurnCosts, int segmentSize) {
9191
this.dir = dir;
9292
this.intsForFlags = intsForFlags;
93-
this.bitUtil = BitUtil.get(dir.getByteOrder());
94-
this.wayGeometry = dir.find("geometry");
95-
this.stringIndex = new StringIndex(dir);
96-
this.nodes = dir.find("nodes", DAType.getPreferredInt(dir.getDefaultType()));
97-
this.edges = dir.find("edges", DAType.getPreferredInt(dir.getDefaultType()));
93+
this.bitUtil = BitUtil.LITTLE;
94+
this.wayGeometry = dir.create("geometry", segmentSize);
95+
this.stringIndex = new StringIndex(dir, 1000, segmentSize);
96+
this.nodes = dir.create("nodes", dir.getDefaultType("nodes", true), segmentSize);
97+
this.edges = dir.create("edges", dir.getDefaultType("edges", true), segmentSize);
9898
this.bounds = BBox.createInverse(withElevation);
9999
this.nodeAccess = new GHNodeAccess(this, withElevation);
100-
if (withTurnCosts) {
101-
turnCostStorage = new TurnCostStorage(this, dir.find("turn_costs"));
102-
} else {
103-
turnCostStorage = null;
104-
}
100+
this.turnCostStorage = withTurnCosts ? new TurnCostStorage(this, dir.create("turn_costs", dir.getDefaultType("turn_costs", true), segmentSize)) : null;
105101
if (segmentSize >= 0) {
106-
setSegmentSize(segmentSize);
102+
checkNotInitialized();
107103
}
108104
}
109105

@@ -330,17 +326,6 @@ public BBox getBounds() {
330326
return bounds;
331327
}
332328

333-
private void setSegmentSize(int bytes) {
334-
checkNotInitialized();
335-
nodes.setSegmentSize(bytes);
336-
edges.setSegmentSize(bytes);
337-
wayGeometry.setSegmentSize(bytes);
338-
stringIndex.setSegmentSize(bytes);
339-
if (supportsTurnCosts()) {
340-
turnCostStorage.setSegmentSize(bytes);
341-
}
342-
}
343-
344329
synchronized void freeze() {
345330
if (isFrozen())
346331
throw new IllegalStateException("base graph already frozen");

‎core/src/main/java/com/graphhopper/storage/CHStorage.java

Lines changed: 21 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -75,14 +75,10 @@ public CHStorage(Directory dir, String name, int segmentSize, boolean edgeBased)
7575

7676
public CHStorage(Directory dir, String name, int segmentSize, boolean edgeBased, String type) {
7777
this.isTypeCore = CHConfig.TYPE_CORE.equals(type);
78-
this.nodesCH = dir.find("nodes_" + type + "_" + name, DAType.getPreferredInt(dir.getDefaultType()));
79-
this.shortcuts = dir.find("shortcuts_" + type + "_" + name, DAType.getPreferredInt(dir.getDefaultType()));
78+
this.nodesCH = dir.create("nodes_" + type + "_" + name, DAType.getPreferredInt(dir.getDefaultType()), segmentSize);
79+
this.shortcuts = dir.create("shortcuts_" + type + "_" + name, DAType.getPreferredInt(dir.getDefaultType()), segmentSize);
8080
// ORS-GH MOD END
8181
this.edgeBased = edgeBased;
82-
if (segmentSize >= 0) {
83-
nodesCH.setSegmentSize(segmentSize);
84-
shortcuts.setSegmentSize(segmentSize);
85-
}
8682
// shortcuts are stored consecutively using this layout (the last two entries only exist for edge-based):
8783
// NODEA | NODEB | WEIGHT | SKIP_EDGE1 | SKIP_EDGE2 | S_ORIG_FIRST | S_ORIG_LAST
8884
S_NODEA = 0;
@@ -124,6 +120,25 @@ public void create() {
124120
shortcuts.create(0);
125121
}
126122

123+
/**
124+
* Creates a new storage. Alternatively we could load an existing one using {@link #loadExisting()}}.
125+
* The number of nodes must be given here while the expected number of shortcuts can
126+
* be given to prevent some memory allocations, but is not a requirement. When in doubt rather use a small value
127+
* so the resulting files/byte arrays won't be unnecessarily large.
128+
* todo: we could also trim down the shortcuts DataAccess when we are done adding shortcuts
129+
*/
130+
public void create(int nodes, int expectedShortcuts) {
131+
if (nodeCount >= 0)
132+
throw new IllegalStateException("CHStorage can only be created once");
133+
if (nodes < 0)
134+
throw new IllegalStateException("CHStorage must be created with a positive number of nodes");
135+
nodesCH.create((long) nodes * nodeCHEntryBytes);
136+
nodeCount = nodes;
137+
for (int node = 0; node < nodes; node++)
138+
setLastShortcut(toNodePointer(node), -1);
139+
shortcuts.create((long) expectedShortcuts * shortcutEntryBytes);
140+
}
141+
127142
/**
128143
* Initializes the storage. The number of nodes must be given here while the expected number of shortcuts can
129144
* be given to prevent some memory allocations, but is not a requirement. When in doubt rather use a small value

0 commit comments

Comments
 (0)