Declare the directory structure in a file

Create a configuration file with directories listed in it.
The paths are specified inside a singleton - Directories.
All the files now references this class and use no hardcoded paths.

Author: xxdavid

src/analyses.py

@@ -1,6 +1,7 @@
 from abc import ABC, abstractmethod
 from file_utils import save_json, load_txt_into_set
 import re
+from config import Directories
 
 
 class Analysis(ABC):
@@ -27,13 +28,13 @@ def finalize(self):
         pass
 
     def save(self):
-        save_json(self.state, f"../outputs/json/{self.name}.json")
+        save_json(self.state, f"{Directories.json_outputs}/{self.name}.json")
 
 
 class WordFrequencyAnalysis(Analysis):
     def __init__(self):
         self.words = {}
-        self.stopwords = load_txt_into_set("../data/raw/stopwords.txt")
+        self.stopwords = load_txt_into_set(f"{Directories.raw_data}/stopwords.txt")
 
     @property
     def name(self):
@@ -90,7 +91,7 @@ def __init__(self):
 
         self.lists = {}
         for form in self.forms:
-            self.lists[form] = load_txt_into_set(f"../data/processed/{form}.txt")
+            self.lists[form] = load_txt_into_set(f"{Directories.processed_data}/{form}.txt")
 
         self.counts = {}
         for form in self.forms:

src/analyze.py

@@ -2,7 +2,7 @@
 from analyzer import Analyzer
 from analyses import *
 
-Analyzer("../data/processed/commits.txt", [
+Analyzer([
     WordFrequencyAnalysis(),
     FirstWordFrequencyAnalysis(),
     VerbFormAnalysis(),

src/analyzer.py

@@ -1,6 +1,8 @@
+from config import Directories
+
+
 class Analyzer:
-    def __init__(self, commits_path, analyses):
-        self.commits_path = commits_path
+    def __init__(self, analyses):
         self.analyses = analyses
         self.authors = {}
         self.total_number = 0
@@ -9,7 +11,7 @@ def __init__(self, commits_path, analyses):
         self.MAX_COMMITS_BY_AUTHOR = 3
 
     def analyze(self):
-        file = open(self.commits_path)
+        file = open(f"{Directories.processed_data}/commits.txt")
         with file:
             for line in file:
                 [author, repo, lines, message] = line.split("::", 3)

src/config.py

@@ -0,0 +1,10 @@
+class Directories:
+    root = ".."
+
+    data = root + "/" + "data"
+    processed_data = data + "/" + "processed"
+    raw_data = data + "/" + "raw"
+
+    outputs = root + "/" + "outputs"
+    charts = outputs + "/" + "charts"
+    json_outputs = outputs + "/" + "json"

src/download_and_parse.py

@@ -5,18 +5,19 @@
 import gzip
 import urllib.request
 import io
+
+from config import Directories
 from file_utils import open_file_dir_safe
 
 date = sys.argv[1]
 
 url = f"http://data.githubarchive.org/{date}.json.gz"
-print(url)
 
 response = urllib.request.urlopen(url)
 compressed_file = io.BytesIO(response.read())
 file = gzip.GzipFile(fileobj=compressed_file)
 
-output_filename = "../data/processed/commits.txt"
+output_filename = f"{Directories.processed_data}/commits.txt"
 output_mode = "a" if os.path.exists(output_filename) else "w"
 output_file = open_file_dir_safe(output_filename, output_mode)
 

src/generate_conjugations.py

@@ -1,14 +1,15 @@
 #!/usr/bin/env python3
+from config import Directories
 from file_utils import load_txt_into_set, load_json, open_file_dir_safe
 import re
 
-infinitives = load_txt_into_set("../data/processed/infinitive.txt")
-irregular = load_json("../data/processed/irregular_verbs.json")
+infinitives = load_txt_into_set(f"{Directories.processed_data}/infinitive.txt")
+irregular = load_json(f"{Directories.processed_data}/irregular_verbs.json")
 
-imperative_file = open_file_dir_safe("../data/processed/imperative.txt", 'w')
-third_person_file = open_file_dir_safe("../data/processed/third_person.txt", 'w')
-gerund_file = open_file_dir_safe("../data/processed/gerund.txt", 'w')
-past_tense_file = open_file_dir_safe("../data/processed/past_tense.txt", 'w')
+imperative_file = open_file_dir_safe(f"{Directories.processed_data}/imperative.txt", 'w')
+third_person_file = open_file_dir_safe(f"{Directories.processed_data}/third_person.txt", 'w')
+gerund_file = open_file_dir_safe(f"{Directories.processed_data}/gerund.txt", 'w')
+past_tense_file = open_file_dir_safe(f"{Directories.processed_data}/past_tense.txt", 'w')
 
 vowel = "[aeiouy]"
 consonant = "[b-df-hj-np-tv-z]"

src/plot.py

@@ -1,8 +1,9 @@
 #!/usr/bin/env python3
+from config import Directories
 from plotters import *
 from os import makedirs
 
-makedirs("../outputs/charts", exist_ok=True)
+makedirs(Directories.charts, exist_ok=True)
 
 FirstWordFrequencyPlotter().plot()
 WordFrequencyPlotter().plot()

src/plotters.py

@@ -1,12 +1,14 @@
 from abc import ABC, abstractmethod
 import matplotlib.pyplot as plt
 import numpy as np
+
+from config import Directories
 from file_utils import load_json
 
 
 class Plotter(ABC):
     def __init__(self):
-        self.data = load_json(f"../outputs/json/{self.input_file_name}.json")
+        self.data = load_json(f"{Directories.json_outputs}/{self.input_file_name}.json")
 
     @property
     @abstractmethod
@@ -53,8 +55,8 @@ def plot(self):
 
         plt.tight_layout()
 
-        plt.savefig(f"../outputs/charts/{self.name}.png", transparent=True)
-        plt.savefig(f"../outputs/charts/{self.name}.svg")
+        plt.savefig(f"{Directories.charts}/{self.name}.png", transparent=True)
+        plt.savefig(f"{Directories.charts}/{self.name}.svg")
         plt.close()
 
 

src/process_irregular_verbs.py

@@ -1,9 +1,11 @@
 #!/usr/bin/env python3
 from pyquery import PyQuery as pq
 import re
+
+from config import Directories
 from file_utils import save_json
 
-doc = pq(filename='../data/raw/irregular_verbs.html')
+doc = pq(filename=f'{Directories.raw_data}/irregular_verbs.html')
 
 table = doc(".wikitable")
 tds = table("tr td:first")
@@ -20,4 +22,4 @@
         infinitive = conjugations.pop(0)
         words[infinitive] = conjugations[0]
 
-save_json(words, "../data/processed/irregular_verbs.json")
+save_json(words, f"{Directories.processed_data}/irregular_verbs.json")

src/process_verbs.py

@@ -1,9 +1,11 @@
 #!/usr/bin/env python3
 import re
+
+from config import Directories
 from file_utils import open_file_dir_safe
 
-input_file = open("../data/raw/verbs.txt")
-output_file = open_file_dir_safe("../data/processed/infinitive.txt", "w")
+input_file = open(f"{Directories.raw_data}/verbs.txt")
+output_file = open_file_dir_safe(f"{Directories.processed_data}/infinitive.txt", "w")
 
 with input_file, output_file:
     for line in input_file: