Indexer ES document id

core/src/main/java/com/digitalpebble/stormcrawler/indexing/AbstractIndexerBolt.java

@@ -175,6 +175,17 @@ protected Map<String, String[]> filterMetadata(Metadata meta) {
         return fieldVals;
     }
 
+    /**
+     * Get the document id.
+     *
+     * @param metadata The {@link Metadata}.
+     * @param normalisedUrl The normalised url.
+     * @return Return the normalised url SHA-256 digest as String.
+     */
+    protected String getDocumentID(Metadata metadata, String normalisedUrl) {
+        return org.apache.commons.codec.digest.DigestUtils.sha256Hex(normalisedUrl);
+    }
+
     /**
      * Returns the value to be used as the URL for indexing purposes, if present the canonical value
      * is used instead

external/aws/src/main/java/com/digitalpebble/stormcrawler/aws/bolt/CloudSearchIndexerBolt.java

@@ -165,6 +165,11 @@ public void prepare(
         client.setEndpoint(endpoint);
     }
 
+    @Override
+    protected String getDocumentID(Metadata metadata, String normalisedUrl) {
+        return CloudSearchUtils.getID(normalisedUrl);
+    }
+
     @Override
     public void execute(Tuple tuple) {
 
@@ -204,7 +209,7 @@ public void execute(Tuple tuple) {
             doc_builder.put("type", "add");
 
             // generate the id from the normalised url
-            String ID = CloudSearchUtils.getID(normalisedurl);
+            String ID = getDocumentID(metadata, normalisedurl);
             doc_builder.put("id", ID);
 
             ObjectNode fields = objectMapper.createObjectNode();

external/elasticsearch/src/main/java/com/digitalpebble/stormcrawler/elasticsearch/bolt/IndexerBolt.java

@@ -177,7 +177,7 @@ public void execute(Tuple tuple) {
             return;
         }
 
-        String docID = org.apache.commons.codec.digest.DigestUtils.sha256Hex(normalisedurl);
+        String docID = getDocumentID(metadata, normalisedurl);
 
         try {
             XContentBuilder builder = jsonBuilder().startObject();

external/opensearch/src/main/java/com/digitalpebble/stormcrawler/opensearch/bolt/IndexerBolt.java

@@ -193,7 +193,7 @@ public void execute(Tuple tuple) {
             return;
         }
 
-        final String docID = org.apache.commons.codec.digest.DigestUtils.sha256Hex(normalisedurl);
+        final String docID = getDocumentID(metadata, normalisedurl);
 
         try {
             final XContentBuilder builder = jsonBuilder().startObject();

external/solr/src/main/java/com/digitalpebble/stormcrawler/solr/bolt/IndexerBolt.java

@@ -104,7 +104,7 @@ public void execute(Tuple tuple) {
                 // Distinguish the value used for indexing
                 // from the one used for the status
                 String normalisedurl = valueForURL(tuple);
-                doc.addField(fieldNameForURL(), normalisedurl);
+                doc.addField(fieldNameForURL(), getDocumentID(metadata, normalisedurl));
             }
 
             // select which metadata to index
@@ -133,4 +133,9 @@ public void execute(Tuple tuple) {
             _collector.fail(tuple);
         }
     }
+
+    @Override
+    protected String getDocumentID(Metadata metadata, String normalisedUrl) {
+        return normalisedUrl;
+    }
 }

external/sql/src/main/java/com/digitalpebble/stormcrawler/sql/IndexerBolt.java

@@ -139,7 +139,7 @@ public void execute(Tuple tuple) {
 
             // send URL as field?
             if (fieldNameForURL() != null) {
-                preparedStmt.setString(1, normalisedurl);
+                preparedStmt.setString(1, getDocumentID(metadata, normalisedurl));
             }
 
             for (int i = 0; i < keys.length; i++) {
@@ -168,6 +168,11 @@ public void execute(Tuple tuple) {
         }
     }
 
+    @Override
+    protected String getDocumentID(Metadata metadata, String normalisedUrl) {
+        return normalisedUrl;
+    }
+
     private void insert(
             PreparedStatement preparedStmt,
             int position,