fix adad

Author: masoudjs

induce_alignments_AdAd.py

@@ -6,7 +6,7 @@
 from collections import defaultdict
 import numpy as np
 
-from utils import LOG, load_editions
+from utils import LOG, load_editions, load_gold
 
 class MyWG:
 	def __init__(self, nodes=[]):
@@ -65,14 +65,6 @@ def calc_wadar(self, edges, verbose=False):
 		return scores
 
 
-def load_gold(gold_path="golds/eng-fra-new.gold"):
-	golds = {}
-	with open(gold_path, "r") as fi:
-		for l in fi:
-			l = l.split("\t")
-			golds[l[0]] = list(set(l[1].split()))
-	return golds
-
 def load_texts_and_alignments(editions_file, lang_files_path, verse_alignments_path, aligner="inter", golds=None):
 	# Get languages and editions
 	editions, langs = load_editions(editions_file)
@@ -83,7 +75,10 @@ def load_texts_and_alignments(editions_file, lang_files_path, verse_alignments_p
 	texts = {}
 	for langl in langs:
 		verses = {}
-		lang_path = lang_files_path + "/" + l[1] + ".txt"
+		if langl == "eng":
+			lang_path = os.path.join("/mounts/work/mjalili/projects/graph_align_base/data/pbc/", editions[langl] + ".txt")
+		else:
+			lang_path = os.path.join(lang_files_path, editions[langl] + ".txt")
 		with codecs.open(lang_path, "r", "utf-8") as fi:
 			for l in fi:
 				if l[0] == "#": continue
@@ -102,7 +97,7 @@ def load_texts_and_alignments(editions_file, lang_files_path, verse_alignments_p
 		v_path = F"{verse_alignments_path}/{verse}_{aligner}.txt"
 		if not os.path.exists(v_path):
 			LOG.info(v_path)
-			LOG.info(f"================================== dos not exist ==================================")
+			LOG.info(f"================================== does not exist ==================================")
 			return None
 		with open(v_path, "r") as f_al:
 			for vl in f_al:
@@ -120,7 +115,7 @@ def load_texts_and_alignments(editions_file, lang_files_path, verse_alignments_p
 				else:
 					init_aligns[(l1, l2)][verse] = [[int(alp.split("-")[1]), int(alp.split("-")[0]), 1.0] for alp in vl[2].strip().split()]
 
-	return langs, texts, lang_pairs, init_aligns
+	return lang_code_map, langs, texts, lang_pairs, init_aligns
 
 def get_alignment_matrix(sim_matrix):
 	m, n = sim_matrix.shape
@@ -232,18 +227,18 @@ def add_edges_to_align_argmax(texts, waligns, out_path="", target_pair=("eng", "
 	return all_cnt
 
 def main(args):
-	target_pair = (args.source_lang, args.target_lang)
 	if args.gold_file != "":
 		pros, surs = load_gold(args.gold_file)
 		all_verses = list(pros.keys())
 	else:
 		all_verses = None
 
 	# Get languages and initial alignments
-	langs, texts, lang_pairs, init_aligns = load_texts_and_alignments(args.editions_file, args.lang_files_path, args.verse_alignments_path, args.aligner, golds=all_verses)
+	lang_code_map, langs, texts, lang_pairs, init_aligns = load_texts_and_alignments(args.editions_file, args.lang_files_path, args.verse_alignments_path, args.aligner, golds=all_verses)
+	target_pair = (lang_code_map[args.source_edition], lang_code_map[args.target_edition])
 
 	# print some info
-	LOG.info(f"Inferring alignments from {args.source_lang} to {args.target_lang}")
+	LOG.info(f"Inferring alignments from {args.source_edition} to {args.target_edition}")
 	LOG.info(f"Number of verses whose alignments will be inferred: {len(all_verses)}")
 	LOG.info(f"Number of editions to use for the graph algorithms: {len(langs)}")
 
@@ -259,13 +254,13 @@ def main(args):
 if __name__ == "__main__":
 	current_path = os.path.dirname(os.path.realpath(__file__))
 	parser = argparse.ArgumentParser()
-	
+
 	parser.add_argument('--save_path', default=os.path.join(current_path, "predicted_alignments"), type=str)
-	parser.add_argument('--gold_file', default=os.path.join(current_path, "data/gold-standards/blinker/eng-fra.gold"), type=str)   
+	parser.add_argument('--gold_file', default=os.path.join(current_path, "data/gold-standards/blinker/eng-fra.gold"), type=str)
 	parser.add_argument('--verse_alignments_path', default="/mounts/data/proj/ayyoob/align_induction/verse_alignments/", type=str)
 	parser.add_argument('--lang_files_path', default="/nfs/datc/pbc/", type=str)
-	parser.add_argument('--source_lang', default="eng", type=str)
-	parser.add_argument('--target_lang', default="fra", type=str)
+	parser.add_argument('--source_edition', default="eng-x-bible-mixed", type=str)
+	parser.add_argument('--target_edition', default="fra-x-bible-louissegond", type=str)
 	parser.add_argument('--editions_file',  default=os.path.join(current_path, "data/edition_lists/blinker_edition_list.txt" ), type=str)
 	parser.add_argument('--aligner', default="inter", type=str)
 

induce_alignments_NMF.py

@@ -55,7 +55,7 @@ def get_aligns(rf, cf, alignments):
             res.append( ( int(x[0]), int(x[1]) ) )
     else:
         return None
-    
+
     return res
 
 def add_aligns(aligns, aligns_dict, token_counts, re, ce, existing_items):
@@ -69,7 +69,7 @@ def add_aligns(aligns, aligns_dict, token_counts, re, ce, existing_items):
             token_counts[re] = align[0]
         if align[1] > token_counts[ce]:
             token_counts[ce] = align[1]
-        
+
         existing_items[re][ce].append(f"{align[0]},{align[1]}")
 
 def add_negative_samples(aligns_dict, existing_items, token_counts, verse_id):
@@ -89,7 +89,7 @@ def add_negative_samples(aligns_dict, existing_items, token_counts, verse_id):
                 aligns_dict['userID'].append(re + str(i))
                 aligns_dict['itemID'].append(ce + str(jp))
                 aligns_dict['rating'].append(1)
-                
+
                 ip %= (token_counts[re] + 1) 
                 aligns_dict['userID'].append(re + str(ip))
                 aligns_dict['itemID'].append(ce + str(j))
@@ -112,11 +112,11 @@ def get_alignments_df(row_editions, col_editions, verse_alignments,
 
             if not aligns is None:
                 add_aligns(aligns, aligns_dict, token_counts, re, ce, existing_items)
-        
+
     add_negative_samples(aligns_dict, existing_items, token_counts, verse_id)
 
     return pd.DataFrame(aligns_dict), token_counts[source_edition], token_counts[target_edition]
-    
+
 def iter_max(sim_matrix: np.ndarray, max_count: int=2, alpha_ratio = 0.7) -> np.ndarray:
     m, n = sim_matrix.shape
     forward = np.eye(n)[sim_matrix.argmax(axis=1)]  # m x n
@@ -156,14 +156,14 @@ def get_itermax_predictions(raw_s_predictions, max_count=2, alpha_ratio=0.9):
     for i in raw_s_predictions:
         for j, s in raw_s_predictions[i]:
             matrix[i,j] = s
-    
+
     itermax_res = iter_max(matrix, max_count, alpha_ratio)
     res = []
     for i in range(rows):
         for j in range(cols):
             if itermax_res[i,j] != 0:
                 res.append((i,j))
-    
+
     return res
 
 def predict_alignments(algo, source_edition, target_edition):
@@ -197,13 +197,13 @@ def train_model(df,  s_tok_count, t_tok_count, row_editions, col_editions):
     algo.row_editions = row_editions
     algo.col_editions = col_editions
     algo.df = df
-    
+
     return algo
 
 def get_induced_alignments(source_edition, target_edition, verse_alignments_path, verse_id, all_editions):
 
     verse_alignments = get_verse_alignments(verse_alignments_path, verse_id, editions=all_editions)
-    
+
     # this is only for saving the gdfa alignments from source to target for the evauation
     verse_alignments_gdfa = get_verse_alignments(verse_alignments_path, verse_id, editions=[source_edition, target_edition], gdfa=True)
 
@@ -213,17 +213,17 @@ def get_induced_alignments(source_edition, target_edition, verse_alignments_path
     df, s_tok_count, t_tok_count = get_alignments_df(row_editions, col_editions, verse_alignments, source_edition, target_edition, verse_id)
 
     algo = train_model(df, s_tok_count, t_tok_count, row_editions, col_editions)
-    
+
     predicted_alignments = predict_alignments(algo, source_edition, target_edition)
     base_inter_alignments = verse_alignments[source_edition][target_edition]
     base_gdfa_alignments = verse_alignments_gdfa[source_edition][target_edition]
-    
+
     with cnt.get_lock():
         cnt.value += 1
         if cnt.value % 20 == 0:
             LOG.info(f"Done inferring alignments for {cnt.value} verses")
 
-    return predicted_alignments, base_inter_alignments, base_gdfa_alignments,  len(algo.col_editions)+1
+    return predicted_alignments, base_inter_alignments, base_gdfa_alignments,  len(algo.col_editions) + 1
 
 
 def init_globals(counter):
@@ -255,7 +255,7 @@ def main(args):
 
     # get predicted alignments using parallel processing
     cnt = Value('i', 0)
-    with Pool(processes=args.core_count, initializer=init_globals, initargs=(cnt,)) as p:  
+    with Pool(processes=args.core_count, initializer=init_globals, initargs=(cnt,)) as p:
         all_alignments = p.starmap(get_induced_alignments, starmap_args)
 
     out_NMF_f_name = f"predicted_alignments_from_{args.source_edition}_to_{args.target_edition}_with_max_{len(all_editions)}_editions_for_{len(all_verses)}_verses_NMF.txt"
@@ -283,14 +283,14 @@ def main(args):
     parser = argparse.ArgumentParser()
 
     parser.add_argument('--save_path', default=os.path.join(current_path, "predicted_alignments"), type=str)
-    parser.add_argument('--gold_file', default=os.path.join(current_path, "data/gold-standards/blinker/eng-fra.gold"), type=str)    
+    parser.add_argument('--gold_file', default=os.path.join(current_path, "data/gold-standards/blinker/eng-fra.gold"), type=str)
     parser.add_argument('--verse_alignments_path', default="/mounts/data/proj/ayyoob/align_induction/verse_alignments/", type=str)
-    parser.add_argument('--source_edition', default="eng-x-bible-mixed", type=str) 
-    parser.add_argument('--target_edition', default="fra-x-bible-louissegond", type=str) 
+    parser.add_argument('--source_edition', default="eng-x-bible-mixed", type=str)
+    parser.add_argument('--target_edition', default="fra-x-bible-louissegond", type=str)
     parser.add_argument('--editions_file', default=os.path.join(current_path, "data/edition_lists/blinker_edition_list.txt" ), type=str)
     parser.add_argument('--core_count', default=80, type=int)
     parser.add_argument('--seed', default=42, type=int)
 
     args = parser.parse_args()
     main(args)
-    
+