Added checks for html entity names

nitishgupta · Jan 10, 2018 · 57ba9bc · 57ba9bc
1 parent 0927351
commit 57ba9bc
Show file tree

Hide file tree

Showing 2 changed files with 3 additions and 1 deletion.
diff --git a/src/main/java/edu/illinois/cs/cogcomp/wikiparser/jwpl/jwplparsers/PageMapLineParser.java b/src/main/java/edu/illinois/cs/cogcomp/wikiparser/jwpl/jwplparsers/PageMapLineParser.java
@@ -205,7 +205,7 @@ public void parsePageMap(String pageMapFile){
                 pageTitle = pageTitle.replace("\\", "");  // Removes escape character '\'
                 pageTitle = pageTitle.trim(); // Removes trailing and leading space
                 if(pageTitle.isEmpty()) continue;
-                if (pageTitle.startsWith("List_of") || pageTitle.startsWith("Lists_of")){
+                if (pageTitle.startsWith("List_of") || pageTitle.startsWith("Lists_of") && (id == resolvedId)){
                     listPages.add(resolvedId);  // Only add resolved pages
                 }
                 curIds.add(id);  // Adds unresolved Cur Ids

diff --git a/src/main/java/edu/illinois/cs/cogcomp/wikiparser/wikiparse/PageIdTitleParser.java b/src/main/java/edu/illinois/cs/cogcomp/wikiparser/wikiparse/PageIdTitleParser.java
@@ -77,6 +77,8 @@ public void ParseDoc(String doc){
                 logger.severe("Exception: " + e.toString());
             }
             wikiTitle = wikiTitle.replaceAll(" ", "_");
+            wikiTitle = wikiTitle.replaceAll("&amp;", "&"); // Replaces entity name for the character '&'
+            wikiTitle = wikiTitle.replaceAll("&quot;", "\""); // Replaces entity name for the character '"'
             if (wikiTitle.startsWith("List_of") || wikiTitle.startsWith("Lists_of")) return;
 
             // Gets curID