Indexer ES document id (#1028)

* ES document id can be a random UUID Signed-off-by: Mathieu Bret <[email protected]> * Remove the random option. Signed-off-by: Mathieu Bret <[email protected]> * Apply getDocumentId to all AbstractIndexerBolt implementation. Signed-off-by: Mathieu Bret <[email protected]> * Rollback SOLR and SQL. Signed-off-by: Mathieu Bret <[email protected]> Signed-off-by: Mathieu Bret <[email protected]>
apache · Jan 10, 2023 · 9bc2be9 · 9bc2be9
1 parent a575df5
commit 9bc2be9
Show file tree

Hide file tree

Showing 4 changed files with 19 additions and 3 deletions.
diff --git a/core/src/main/java/com/digitalpebble/stormcrawler/indexing/AbstractIndexerBolt.java b/core/src/main/java/com/digitalpebble/stormcrawler/indexing/AbstractIndexerBolt.java
@@ -175,6 +175,17 @@ protected Map<String, String[]> filterMetadata(Metadata meta) {
         return fieldVals;
     }
 
+    /**
+     * Get the document id.
+     *
+     * @param metadata The {@link Metadata}.
+     * @param normalisedUrl The normalised url.
+     * @return Return the normalised url SHA-256 digest as String.
+     */
+    protected String getDocumentID(Metadata metadata, String normalisedUrl) {
+        return org.apache.commons.codec.digest.DigestUtils.sha256Hex(normalisedUrl);
+    }
+
     /**
      * Returns the value to be used as the URL for indexing purposes, if present the canonical value
      * is used instead

diff --git a/...nal/aws/src/main/java/com/digitalpebble/stormcrawler/aws/bolt/CloudSearchIndexerBolt.java b/...nal/aws/src/main/java/com/digitalpebble/stormcrawler/aws/bolt/CloudSearchIndexerBolt.java
@@ -165,6 +165,11 @@ public void prepare(
         client.setEndpoint(endpoint);
     }
 
+    @Override
+    protected String getDocumentID(Metadata metadata, String normalisedUrl) {
+        return CloudSearchUtils.getID(normalisedUrl);
+    }
+
     @Override
     public void execute(Tuple tuple) {
 
@@ -204,7 +209,7 @@ public void execute(Tuple tuple) {
             doc_builder.put("type", "add");
 
             // generate the id from the normalised url
-            String ID = CloudSearchUtils.getID(normalisedurl);
+            String ID = getDocumentID(metadata, normalisedurl);
             doc_builder.put("id", ID);
 
             ObjectNode fields = objectMapper.createObjectNode();

diff --git a/...icsearch/src/main/java/com/digitalpebble/stormcrawler/elasticsearch/bolt/IndexerBolt.java b/...icsearch/src/main/java/com/digitalpebble/stormcrawler/elasticsearch/bolt/IndexerBolt.java
@@ -177,7 +177,7 @@ public void execute(Tuple tuple) {
             return;
         }
 
-        String docID = org.apache.commons.codec.digest.DigestUtils.sha256Hex(normalisedurl);
+        String docID = getDocumentID(metadata, normalisedurl);
 
         try {
             XContentBuilder builder = jsonBuilder().startObject();

diff --git a/.../opensearch/src/main/java/com/digitalpebble/stormcrawler/opensearch/bolt/IndexerBolt.java b/.../opensearch/src/main/java/com/digitalpebble/stormcrawler/opensearch/bolt/IndexerBolt.java
@@ -193,7 +193,7 @@ public void execute(Tuple tuple) {
             return;
         }
 
-        final String docID = org.apache.commons.codec.digest.DigestUtils.sha256Hex(normalisedurl);
+        final String docID = getDocumentID(metadata, normalisedurl);
 
         try {
             final XContentBuilder builder = jsonBuilder().startObject();
-Original file line number
+Diff line change
@@ Expand Up / @@ -177,7 +177,7 @@ public void execute(Tuple tuple) { @@
                 return;
             }
-            String docID = org.apache.commons.codec.digest.DigestUtils.sha256Hex(normalisedurl);
+            String docID = getDocumentID(metadata, normalisedurl);
             try {
                 XContentBuilder builder = jsonBuilder().startObject();
@@ Expand Down @@