cleanup & updates

.gitignore

@@ -0,0 +1,3 @@
+**/__pycache__
+models
+tmp

.vscode/launch.json

@@ -0,0 +1,15 @@
+{
+    // Use IntelliSense to learn about possible attributes.
+    // Hover to view descriptions of existing attributes.
+    // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+    "version": "0.2.0",
+    "configurations": [
+        {
+            "name": "Python: Current File",
+            "type": "python",
+            "request": "launch",
+            "program": "${file}",
+            "console": "integratedTerminal"
+        }
+    ]
+}

.vscode/settings.json

@@ -0,0 +1,2 @@
+{
+}

LibriSpeech-Transcriptions/README.md

@@ -0,0 +1,5 @@
+# SIMI experiments
+
+## Setup
+
+GitHub does not allow to push files larger than 100MB, so after cloning you need to run script `prepare.py` from folder `LibriSpeech-Transcriptions`.

LibriSpeech-Transcriptions/conf/m1.yaml

@@ -1,4 +1,3 @@
-name: 'm1'
 extend_length:
   - 3
   - 4

LibriSpeech-Transcriptions/conf/m2.yaml

@@ -0,0 +1,12 @@
+extend_length:
+  - 3
+  - 4
+  - 5
+  - 6
+  - 7
+  - 8
+  - 9
+  - 10
+replace_prob: 0.1
+replace_length:
+  - 1

LibriSpeech-Transcriptions/conf/m3.yaml

@@ -0,0 +1,12 @@
+extend_length:
+  - 3
+  - 4
+  - 5
+  - 6
+  - 7
+  - 8
+  - 9
+  - 10
+replace_prob: 0.0
+replace_length:
+  - 0

LibriSpeech-Transcriptions/prepare.py

@@ -4,7 +4,6 @@
 import random
 import os
 
-random.seed(290956)
 
 DATASETS = ['dev-clean', 'dev-other', 'test-clean', 'test-other', 'train-clean-100', 'train-clean-360', 'train-other-500']
 
@@ -39,6 +38,7 @@ def load_dataset(name):
     return dataset, dictionary
 
 def process(config_file_name):
+    random.seed(290956)
     config = None
     with open(Path('conf') / config_file_name) as config_file:
         config = yaml.full_load(config_file)
@@ -51,29 +51,32 @@ def process(config_file_name):
     replace_test = lambda: random.uniform(0, 1) < config['replace_prob']
     replace_length = lambda: random.choice(config['replace_length'])
 
-    print("Preparing train-full-960...", flush=True)
+    if not os.path.exists('train-full-960.txt'):
+        print("Preparing train-full-960...", flush=True)
 
-    with open('train-full-960.txt', 'w', encoding='utf8') as out:
-        for name in ['train-clean-100', 'train-clean-360', 'train-other-500']:
-            for line in open(name + '.txt', 'r', encoding='utf8'):
-                out.write(line)
+        with open('train-full-960.txt', 'w', encoding='utf8') as out:
+            for name in ['train-clean-100', 'train-clean-360', 'train-other-500']:
+                for line in open(name + '.txt', 'r', encoding='utf8'):
+                    out.write(line)
 
     print("Preparing dataset modifications", flush=True)
     for dataset_name in DATASETS:
         dataset, dictionary = load_dataset(dataset_name)
-        out_file = open(outPath / (dataset_name + '.txt'), 'w', encoding='utf8')
-        print(f"Processing dataset {dataset_name}", flush=True)
-        modify(dataset, dictionary, extend_range, replace_test, replace_length, out_file)
+        if not os.path.exists(outPath / (dataset_name + '.txt')):
+            print(f"Processing dataset {dataset_name}", flush=True)
+            with open(outPath / (dataset_name + '.txt'), 'w', encoding='utf8') as out_file:
+                modify(dataset, dictionary, extend_range, replace_test, replace_length, out_file)
 
-
-    with open(outPath / 'train-full-960.txt', 'w', encoding='utf8') as out:
-        for name in ['train-clean-100', 'train-clean-360', 'train-other-500']:
-            for line in open(outPath / (name + '.txt'), 'r', encoding='utf8'):
-                out.write(line)
+    if not os.path.exists(outPath / 'train-full-960.txt'):
+        with open(outPath / 'train-full-960.txt', 'w', encoding='utf8') as out:
+            for name in ['train-clean-100', 'train-clean-360', 'train-other-500']:
+                for line in open(outPath / (name + '.txt'), 'r', encoding='utf8'):
+                    out.write(line)
 
 def main():
     for file in os.listdir('./conf'):
         if file.endswith('.yaml'):
+            print(f"Processing {file}", flush=True)
             process(file)
 
 if __name__ == '__main__':

env.py

@@ -0,0 +1,3 @@
+
+ROOTPATH = '/pio/scratch/1/i290956/zs2021/simi'
+TRANSCRIPTIONS_DIR = 'LibriSpeech-Transcriptions'