Corrections + README.md config_files

augment_mentions.py

@@ -142,6 +142,9 @@ def write_mentions(writing_q, process_q, num_processes, number_documents):
     for index in range(num_processes):
         process_q.put("exit")
 
+    collection_write.drop_index("article_id_1_sent_index_1")
+    collection_write.create_index([("article_id", 1), ("sent_index", 1)])
+
 
 if __name__ == "__main__":
     client = pymongo.MongoClient("localhost", 27017)

config_files/README.md

@@ -0,0 +1,5 @@
+Those two files are configuration files for the extraction process that creates DataNER.
+They serve as parameters for the `extract_collection.py` script.
+
+* anchors.yaml : only extracts anchors from the MongoDB collections
+* dataner_config.yaml : extracts all entities that have been created through the augmentation process. You need to have run `augment_mentions.py` to use this one.

extract_collection.py

@@ -44,7 +44,7 @@ def process_article(process_q, writing_q, config_p):
     db_w = client_w.wikipedia
 
     collection_tokens = db_w["dump_tokens"]
-    coll_w = db_w[args.collection]
+    coll_w = db_w["dump_mentions"]
 
     while True:
         article_id_p = process_q.get()
@@ -159,7 +159,6 @@ def write_article(process_q, writing_q, number_processes, number_documents):
     pool = multiprocessing.Pool(processes=num_workers - 1, initializer=process_article, initargs=(processing_queue,
                                                                                                   writing_queue,
                                                                                                   config))
-
     writing_process = multiprocessing.Process(target=write_article,
                                               args=(processing_queue, writing_queue, num_workers,
                                                     doc_count))