get NELL labels

update script to get the labels of the NELL KG

Author: Daniel

.idea/workspace.xml

@@ -24,75 +24,93 @@ public static void main(String[] args) {
 
 		boolean dbpedia = false;
 		boolean yago = false;
-		boolean opencyc = true;
+		boolean opencyc = false;
+		boolean nell = true;
 
-		String fInstanceTypesTransitive  = "";
-		String fInstanceTypes = "";
+		String fType1  = "";
+		String fType2 = "";
 		String fLabels = "";
 
 		if (dbpedia) {
 			//DBpedia files
 			if (useSamples) {
-				fInstanceTypesTransitive = "/Users/curtis/SeminarPaper_KG_files/DBpedia/instance_types_transitive_en_s.ttl";
-				fInstanceTypes = "/Users/curtis/SeminarPaper_KG_files/DBpedia/instance_types_en_s.ttl";
+				fType1 = "/Users/curtis/SeminarPaper_KG_files/DBpedia/instance_types_transitive_en_s.ttl";
+				fType2 = "/Users/curtis/SeminarPaper_KG_files/DBpedia/instance_types_en_s.ttl";
 				fLabels = "/Users/curtis/SeminarPaper_KG_files/DBpedia/labels_en_s.ttl";
 			} else { //full files
-				fInstanceTypesTransitive = "/Users/curtis/SeminarPaper_KG_files/DBpedia/instance_types_transitive_en.ttl";
-				fInstanceTypes = "/Users/curtis/SeminarPaper_KG_files/DBpedia/instance_types_en.ttl";
+				fType1 = "/Users/curtis/SeminarPaper_KG_files/DBpedia/instance_types_transitive_en.ttl";
+				fType2 = "/Users/curtis/SeminarPaper_KG_files/DBpedia/instance_types_en.ttl";
 				fLabels = "/Users/curtis/SeminarPaper_KG_files/DBpedia/labels_en.ttl";
 			}
 			// get all classes for DBpedia
 			HashSet<String> classes = getDBpediaClasses();
 			System.out.println(classes);
 
-			runProcess(0, fInstanceTypesTransitive, fInstanceTypes, fLabels, classes);
+			runProcess(0, fType1, fType2, fLabels, classes);
 		}
 		if (yago) {
 			//YAGO files
 			if (useSamples) {
-				fInstanceTypesTransitive = "/Users/curtis/SeminarPaper_KG_files/YAGO/yagoTransitiveType_s.ttl";
-				fInstanceTypes = "/Users/curtis/SeminarPaper_KG_files/YAGO/yagoTypes_s.ttl";
+				fType1 = "/Users/curtis/SeminarPaper_KG_files/YAGO/yagoTransitiveType_s.ttl";
+				fType2 = "/Users/curtis/SeminarPaper_KG_files/YAGO/yagoTypes_s.ttl";
 				fLabels = "/Users/curtis/SeminarPaper_KG_files/YAGO/yagoLabels_s.ttl";
 			} else {
-				fInstanceTypesTransitive = "/Users/curtis/SeminarPaper_KG_files/YAGO/yagoTransitiveType.ttl";
-				fInstanceTypes = "/Users/curtis/SeminarPaper_KG_files/YAGO/yagoTypes.ttl";
+				fType1 = "/Users/curtis/SeminarPaper_KG_files/YAGO/yagoTransitiveType.ttl";
+				fType2 = "/Users/curtis/SeminarPaper_KG_files/YAGO/yagoTypes.ttl";
 				fLabels = "/Users/curtis/SeminarPaper_KG_files/YAGO/yagoLabels.ttl";
 			}
 			// get all classes for YAGO
 			HashSet<String> classes = getYagoClasses();
 			System.out.println(classes);
 
-			runProcess(1, fInstanceTypesTransitive, fInstanceTypes, fLabels, classes);
+			runProcess(1, fType1, fType2, fLabels, classes);
 
 		}
 
 		if (opencyc) {
 			//OpenCyc files
 			if (useSamples) {
-				fInstanceTypes = "/Users/curtis/SeminarPaper_KG_files/OpenCyc/opencyc-latest_sample.nt";
+				fType1 = "/Users/curtis/SeminarPaper_KG_files/OpenCyc/opencyc-latest_sample.nt";
 				fLabels = "/Users/curtis/SeminarPaper_KG_files/OpenCyc/opencyc-latest_sample.nt";
 			} else {
-				fInstanceTypes = "/Users/curtis/SeminarPaper_KG_files/OpenCyc/opencyc-latest.nt";
+				fType1 = "/Users/curtis/SeminarPaper_KG_files/OpenCyc/opencyc-latest.nt";
 				fLabels = "/Users/curtis/SeminarPaper_KG_files/OpenCyc/opencyc-latest.nt";
 			}
 			// get all classes for OpenCyc
 			HashSet<String> classes = getOpenCycClasses();
 			System.out.println(classes);
 
-			runProcess(2, fInstanceTypesTransitive, fInstanceTypes, fLabels, classes);
+			runProcess(2, fType1, fType2, fLabels, classes);
+		}
+		if (nell) {
+			//NELL files
+			if (useSamples) {
+				fType1 = "/Users/curtis/SeminarPaper_KG_files/NELL/NELL.08m.995.esv_s.csv";
+				fType2 = "/Users/curtis/SeminarPaper_KG_files/NELL/NELL.08m.995.ontology_s.csv";
+				fLabels = "/Users/curtis/SeminarPaper_KG_files/NELL/NELL.08m.995.esv_s.csv";
+			} else {
+				fType1 = "/Users/curtis/SeminarPaper_KG_files/NELL/NELL.08m.995.esv.csv";
+				fType2 = "/Users/curtis/SeminarPaper_KG_files/NELL/NELL.08m.995.ontology.csv";
+				fLabels = "/Users/curtis/SeminarPaper_KG_files/NELL/NELL.08m.995.esv.csv";
+			}
+			// get all classes for OpenCyc
+			HashSet<String> classes = getNellClasses();
+			System.out.println(classes);
+						
+			runProcess(3, fType1, fType2, fLabels, classes);
 		}
 
 		System.out.println("EXECUTION TIME: " +  ((System.nanoTime() - startTime)/1000000000) + " seconds." );
 	}
 
 
 
-		private static void runProcess(int kg, String fInstanceTypesTransitive,
-				String fInstanceTypes, String fLabels, HashSet<String> classes) {
+		private static void runProcess(int kg, String fType1,
+				String fType2, String fLabels, HashSet<String> classes) {
 
 			int skipRows = 0; 
-			if (kg == 0)
-				skipRows = 1; //skip first row for dbpedia
+			if (kg == 0 || kg == 3)
+				skipRows = 1; //skip first row for dbpedia and nell
 			else if (kg == 1)
 				skipRows = 10; //skip first ten rows for yago 
 
@@ -101,9 +119,8 @@ else if (kg == 1)
 
 			try {
 			// GET ALL INSTANCES FOR ALL CLASSES
-				Map<String, Set<String>> classInstances = getClassInstances(kg, fInstanceTypesTransitive, fInstanceTypes, classes, allInstancesSet, skipRows);
+				Map<String, Set<String>> classInstances = getClassInstances(kg, fType1, fType2, classes, allInstancesSet, skipRows);
 
-				
 				System.out.println("allInstancesSet.size():" + allInstancesSet.size());
 				int instanceCount = 0;
 				for (Entry<String, Set<String>> entry : classInstances.entrySet()) {
@@ -120,7 +137,7 @@ else if (kg == 1)
 								.skip(skipRows) //skip first row
 								.filter(line -> containsInstanceNameEn(kg, line, classInstances, allInstancesSet, labeledInstancesSet))
 								//.collect(Collectors.toMap(line -> getS(line), Collectors.toSet(line -> getLabel(getO(line)))));
-								.collect(Collectors.groupingBy(line -> getS(kg, line), Collectors.mapping(line -> getLabel(kg, getO(kg, line)), Collectors.toSet())));
+								.collect(Collectors.groupingBy(line -> getS(kg, line), Collectors.mapping(line -> getLabel(kg, line), Collectors.toSet())));
 
 				//System.out.println(classInstances);
 				//System.out.println(instancesWithLabel);
@@ -166,6 +183,8 @@ else if (kg == 1)
 					resultFolder = "yagoResults/";
 				} else if (kg == 2) {
 					resultFolder = "opencycResults/";
+				} else if (kg == 3) {
+					resultFolder ="nellResults/";
 				}
 				//http://stackoverflow.com/questions/2885173/how-to-create-a-file-and-write-to-a-file-in-java
 				//for (Entry<String, Set<String>> entry : classInstances.entrySet()) {
@@ -188,14 +207,14 @@ else if (kg == 1)
 
 
 	private static Map<String, Set<String>> getClassInstances(
-				int kg, String fInstanceTypesTransitive, String fInstanceTypes, HashSet<String> classes, HashSet<String> allInstancesSet, int skipRows) {
+				int kg, String fType1, String fType2, HashSet<String> classes, HashSet<String> allInstancesSet, int skipRows) {
 		//create stream objects of the files
 		//http://www.oracle.com/technetwork/articles/java/ma14-java-se-8-streams-2177646.html
 		Map<String, Set<String>> classInstances = null;
 		try {
-			if (fInstanceTypesTransitive != "") { //check transitive type file
-				Stream<String> itTransitive = Files.lines(Paths.get(fInstanceTypesTransitive));
-				Stream<String> it = Files.lines(Paths.get(fInstanceTypes));
+			if (fType2 != "") { //concat if two files are passed to method
+				Stream<String> itTransitive = Files.lines(Paths.get(fType1));
+				Stream<String> it = Files.lines(Paths.get(fType2));
 				// read files
 				classInstances =
 						Stream.concat(itTransitive, it)
@@ -204,7 +223,7 @@ private static Map<String, Set<String>> getClassInstances(
 							//collect: group by className (third argument), set of all instance names (first argument): instance a className
 							.collect(Collectors.groupingBy(line -> getO(kg, line), Collectors.mapping(line -> getS(kg, line), Collectors.toSet())));
 			} else { //only one file
-				Stream<String> it = Files.lines(Paths.get(fInstanceTypes));
+				Stream<String> it = Files.lines(Paths.get(fType2));
 				// read files
 				classInstances =	
 						it
@@ -241,10 +260,14 @@ private static boolean containsInstanceNameEn(int kg, String line,
 			//DBpedia and OpenCyc
 			labelString = "<http://www.w3.org/2000/01/rdf-schema#label>";
 			englishLabel = "@en";
-		} else {
+		} else if (kg == 1) {
 			//YAGO
 			labelString = "rdfs:label";
 			englishLabel = "@eng";
+		} else if (kg == 3) {
+			labelString = "generalizations";
+			englishLabel = "concept";
+			
 		}
 		//check if line was complete (yago contains single element references as line)
 		if (spo.length >= 3) {
@@ -290,22 +313,38 @@ private static String getS(int kg, String line) {
 	}
 	/**
 	  * Get label of string
-	  * @param o (string)
+	  * @param line (string)
 	  * @returns substring without label
 	 */
-	private static String getLabel(int kg, String o) {
+	private static String getLabel(int kg, String line) {
 		String returnString = "";
+		
+		
 		if (kg==0 || kg == 2) {
 			//DBpedia and OpenCyc: LABEL_TO_KEEP@en\s
+			String o = getO(kg, line);
 			returnString = o.substring(0, o.length()-4); //-4 due to whitespace created by getSPO in the end
 		} else if (kg == 1) {
 			//YAGO: "LABEL_TO_KEEP"@eng .\n
+			String o = getO(kg, line);
 			returnString = o.substring(1, o.length()-7);			
+		} else if (kg == 3) { //NELL
+			returnString = getEntityLiteralStringsInNell(kg, line);
 		}
 		//System.out.println("getLabel: " + returnString);
 		return  returnString;
 	}
+	/**
+	  * Get seventh argument of line
+	  * @param line
+	  * @returns spo[6] (String)
+	 */
+	private static String getEntityLiteralStringsInNell(int kg, String line) {
+		String spo[] = getSPO(kg, line);
+		return spo[6];
+	}
 
+
 	/**
 	  * Check if line contains className
 	  * @param kg 0:dbpedia, 1:yago
@@ -333,8 +372,11 @@ private static boolean containsClassName(int kg, String line, HashSet<String> cl
 		} else if (kg == 2) {//opencyc
 			typeString = "<http://www.w3.org/1999/02/22-rdf-syntax-ns#type>";
 			classString = "<http://sw.opencyc.org/concept/";
-			classString2 = "<http://sw.opencyc.org/concept/";
-			
+			classString2 = "<http://sw.opencyc.org/concept/";	
+		} else if (kg == 3) { //nell
+			typeString = "generalizations";
+			classString = "concept:";
+			classString2 = "concept:";
 		}
 
 		//check if line was complete (yago contains single element references as line)
@@ -373,6 +415,9 @@ private static String getClassNameOfURI(int kg, String o) {
 		} else if (kg == 2) {
 			//<http://sw.opencyc.org/concept/CLASSNAME_TO_KEEP>
 			returnString = o.substring(31, o.length()-1);
+		} else if (kg == 3) {
+			//"concept:CLASSNAME_TO_KEEP"
+			returnString = o.substring(8, o.length());
 		}
 		//System.out.println("getClassNameOfURI: " + returnString);
 		return returnString;
@@ -417,7 +462,7 @@ private static String[] getSPO(int kg, String line) {
 				String[] preWords = line.split("\\s+"); //split on whitespace
 				words = preWords;
 			}
-		} else { //YAGO
+		} else { //YAGO and NELL
 			String[] preWords = line.split("\\t"); //split on tab
 			words = preWords;
 		}
@@ -529,6 +574,10 @@ private static HashSet<String> getYagoClasses() {
 									));
 		return classNameArray;
 	}
+	/**
+	   * Get HashSet containing all class names in OpenCyc 
+	   * @return Array of all OpenCyc classes
+	   */
 	private static HashSet<String> getOpenCycClasses() {
 		HashSet<String> classNameArray = new HashSet<String>();
 		classNameArray.addAll(Arrays.asList(
@@ -576,5 +625,47 @@ private static HashSet<String> getOpenCycClasses() {
 							));
 		return classNameArray;
 	}
+	/**
+	   * Get HashSet containing all class names in NELL 
+	   * @return Array of all NELL classes
+	   */
+	private static HashSet<String> getNellClasses() {
+		HashSet<String> classNameArray = new HashSet<String>();
+		classNameArray.addAll(Arrays.asList(
+							//PERSON
+                 				"humanagent",
+                 				"agent",
+                 				"person",
+                 				"politician",
+                 				"athlete",
+                 				"actor",
+                 			//ORGANIZATION
+                 				"governmentorganization",
+                 				"company",
+                 				"politicalparty",
+    						//PLACE
+                 				"location",
+                                "geopoliticallocation",
+                                "city",
+                                "country",
+    						//ART
+                                "creativework",
+                                "musicalbum",
+                                "musicsong",
+                                "movie",
+                                "book",
+    						//EVENT
+                                "event",
+                                "militaryeventtype",
+                                "militaryconflict",
+                                "sportsevent",
+    						//TRANSPORT
+                                "vehicle",
+    						//OTHER
+                                "chemical",
+                                "planet"
+							));
+		return classNameArray;
+	}
 }