Tika can handle embedded docs, fixes #358

external/tika/pom.xml

@@ -1,5 +1,6 @@
 <?xml version="1.0"?>
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+	xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
 	<modelVersion>4.0.0</modelVersion>
 
 	<parent>
@@ -18,6 +19,7 @@
 
 	<properties>
 		<tika-parsers.version>1.13</tika-parsers.version>
+		<mockito-all.version>1.10.8</mockito-all.version>
 	</properties>
 
 	<build>
@@ -68,11 +70,6 @@
 			<artifactId>storm-core</artifactId>
 		</dependency>
 
-		<dependency>
-			<groupId>junit</groupId>
-			<artifactId>junit</artifactId>
-		</dependency>
-
 		<dependency>
 			<groupId>com.digitalpebble.stormcrawler</groupId>
 			<artifactId>storm-crawler-core</artifactId>
@@ -90,5 +87,26 @@
 				</exclusion>
 			</exclusions>
 		</dependency>
+
+		<dependency>
+			<groupId>junit</groupId>
+			<artifactId>junit</artifactId>
+		</dependency>
+
+		<dependency>
+			<groupId>com.digitalpebble.stormcrawler</groupId>
+			<artifactId>storm-crawler-core</artifactId>
+			<version>${project.version}</version>
+			<type>test-jar</type>
+			<scope>test</scope>
+		</dependency>
+
+		<dependency>
+			<groupId>org.mockito</groupId>
+			<artifactId>mockito-all</artifactId>
+			<version>${mockito-all.version}</version>
+			<scope>test</scope>
+		</dependency>
+
 	</dependencies>
 </project>

external/tika/src/main/java/com/digitalpebble/stormcrawler/tika/ParserBolt.java

@@ -33,6 +33,7 @@
 import org.apache.html.dom.HTMLDocumentImpl;
 import org.apache.tika.Tika;
 import org.apache.tika.parser.ParseContext;
+import org.apache.tika.parser.Parser;
 import org.apache.tika.parser.html.HtmlMapper;
 import org.apache.tika.parser.html.IdentityHtmlMapper;
 import org.apache.tika.sax.BodyContentHandler;
@@ -88,6 +89,8 @@ public class ParserBolt extends BaseRichBolt {
     private boolean upperCaseElementNames = true;
     private Class<?> HTMLMapperClass = IdentityHtmlMapper.class;
 
+    private boolean extractEmbedded = false;
+
     private MetadataTransfer metadataTransfer;
     private boolean emitOutlinks = true;
 
@@ -105,6 +108,9 @@ public void prepare(Map conf, TopologyContext context,
         upperCaseElementNames = ConfUtils.getBoolean(conf,
                 "parser.uppercase.element.names", true);
 
+        extractEmbedded = ConfUtils.getBoolean(conf, "parser.extract.embedded",
+                false);
+
         String htmlmapperClassName = ConfUtils.getString(conf,
                 "parser.htmlmapper.classname",
                 "org.apache.tika.parser.html.IdentityHtmlMapper");
@@ -174,6 +180,10 @@ public void execute(Tuple tuple) {
                 textHandler);
         ParseContext parseContext = new ParseContext();
 
+        if (extractEmbedded) {
+            parseContext.set(Parser.class, tika.getParser());
+        }
+
         try {
             parseContext.set(HtmlMapper.class,
                     (HtmlMapper) HTMLMapperClass.newInstance());

external/tika/src/test/java/com/digitalpebble/stormcrawler/tika/ParserBoltTest.java

@@ -0,0 +1,71 @@
+/**
+ * Licensed to DigitalPebble Ltd under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * DigitalPebble licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.digitalpebble.stormcrawler.tika;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.storm.task.OutputCollector;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import com.digitalpebble.stormcrawler.TestUtil;
+import com.digitalpebble.stormcrawler.parse.ParsingTester;
+
+public class ParserBoltTest extends ParsingTester {
+
+    @Before
+    public void setupParserBolt() {
+        bolt = new ParserBolt();
+        setupParserBolt(bolt);
+    }
+
+    @Test
+    /**
+     * Checks that recursive docs are handled correctly
+     * 
+     * @see https://issues.apache.org/jira/browse/TIKA-2096
+     **/
+    public void testRecursiveDoc() throws IOException {
+
+        Map conf = new HashMap();
+
+        conf.put("parser.extract.embedded", true);
+
+        bolt.prepare(conf, TestUtil.getMockedTopologyContext(),
+                new OutputCollector(output));
+
+        parse("http://www.digitalpebble.com/test_recursive_embedded.docx",
+                "test_recursive_embedded.docx");
+
+        List<List<Object>> outTuples = output.getEmitted();
+
+        // TODO could we get as many subdocs as embedded in the original one?
+        // or just one for now? but should at least contain the text of the
+        // subdocs
+
+        Assert.assertEquals(1, outTuples.size());
+        Assert.assertTrue(outTuples.get(0).get(3).toString()
+                .contains("Life, Liberty and the pursuit of Happiness"));
+
+    }
+
+}