Merge pull request #146 from SnowletTV/dev

Update 1.6

.gitattributes

@@ -0,0 +1,7 @@
+*.shader linguist-language=HLSL
+*.fxh linguist-language=HLSL
+*.yml linguist-language=YAML
+*.txt linguist-detectable
+*.yml linguist-detectable
+*.asset linguist-detectable
+*.csv linguist-detectable

.github/workflows/ck3_validator.yml

@@ -0,0 +1,20 @@
+name: Validate the mod with CK3_Validator scripts
+
+on: [push]
+
+jobs:
+  check_localization_file_endings:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 2
+          submodules: recursive
+      - name: "Clone CK3_Validator"
+        run: |
+          mkdir CK3_Validator
+          git clone https://github.com/LT-Rascek/CK3_Validator.git CK3_Validator
+      - name: "Check localization file endings"
+        run: |
+          find localization -name "*.txt" | xargs rm
+          python3 CK3_Validator/test_scripts/check_localization_file_endings.py "."

.github/workflows/validate_localization.yml

@@ -0,0 +1,28 @@
+name: Validate YML files
+
+on:
+  pull_request:
+    types: [opened, synchronize]
+
+jobs:
+  validate:
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v2
+
+    - name: Set up Python
+      uses: actions/setup-python@v2
+      with:
+        python-version: '3.10'
+
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install -r auto_translate/requirements.txt
+
+    - name: Validate files
+      run: |
+        python auto_translate/file_handling.py
+

.gitignore

@@ -1 +1,2 @@
-*.bak
+.idea
+py310

auto_translate/.gitignore

@@ -0,0 +1 @@
+*.json

auto_translate/extras.py

@@ -0,0 +1,127 @@
+POSSIBLE_AGENTS = [
+    {
+        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"
+                      "(KHTML, like Gecko) Chrome/70.0.3538.77 Safari/537.36"
+    },
+    {
+        "User-Agent": "Mozilla/5.0 (X11; Ubuntu; Linux x86_64) AppleWebKit/537.36"
+                      "(KHTML, like Gecko) Chrome/55.0.2919.83 Safari/537.36"
+    },
+    {
+        "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_3) AppleWebKit/537.36"
+                      "(KHTML, like Gecko) Chrome/54.0.2866.71 Safari/537.36"
+    },
+    {
+        "User-Agent": "Mozilla/5.0 (X11; Ubuntu; Linux i686 on x86_64) AppleWebKit/537.36"
+                      "(KHTML, like Gecko) Chrome/53.0.2820.59 Safari/537.36"
+    },
+    {
+        "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36"
+                      "(KHTML, like Gecko) Chrome/52.0.2762.73 Safari/537.36"
+    },
+    {
+        "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_4) AppleWebKit/537.36"
+                      "(KHTML, like Gecko) Chrome/49.0.2656.18 Safari/537.36"
+    },
+    {
+        "User-Agent": "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36"
+                      "(KHTML like Gecko) Chrome/44.0.2403.155 Safari/537.36"
+    },
+    {
+        "User-Agent": "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36"
+                      "(KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36"
+    },
+    {
+        "User-Agent": "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:77.0) Gecko/20190101"
+                      "Firefox/77.0"
+    },
+    {
+        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64; rv:77.0) Gecko/20100101"
+                      "Firefox/77.0"
+    },
+    {
+        "User-Agent": "Mozilla/5.0 (X11; Linux ppc64le; rv:75.0) Gecko/20100101"
+                      "Firefox/75.0"
+    },
+    {
+        "User-Agent": "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:39.0) Gecko/20100101"
+                      "Firefox/75.0"
+    },
+    {
+        "User-Agent": "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.10; rv:75.0) Gecko/20100101"
+                      "Firefox/75.0"
+    },
+    {"User-Agent": "Mozilla/5.0 (X11; Linux; rv:74.0) Gecko/20100101" "Firefox/74.0"},
+    {
+        "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.13; rv:61.0) Gecko/20100101"
+                      "Firefox/73.0"
+    },
+    {
+        "User-Agent": "Mozilla/5.0 (X11; OpenBSD i386; rv:72.0) Gecko/20100101"
+                      "Firefox/72.0"
+    },
+    {
+        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"
+                      "(KHTML, like Gecko) Chrome/70.0.3538.102 Safari/537.36 Edge/18.19582"
+    },
+    {
+        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"
+                      "(KHTML, like Gecko) Chrome/70.0.3538.102 Safari/537.36 Edge/18.19577"
+    },
+    {
+        "User-Agent": "Mozilla/5.0 (X11) AppleWebKit/62.41"
+                      "(KHTML, like Gecko) Edge/17.10859 Safari/452.6"
+    },
+    {
+        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"
+                      "(KHTML like Gecko) Chrome/51.0.2704.79 Safari/537.36 Edge/14.14931"
+    },
+    {
+        "User-Agent": "Chrome (AppleWebKit/537.1; Chrome50.0; Windows NT 6.3) AppleWebKit/537.36"
+                      "(KHTML like Gecko) Chrome/51.0.2704.79 Safari/537.36 Edge/14.14393"
+    },
+    {
+        "User-Agent": "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36"
+                      "KHTML like Gecko) Chrome/46.0.2486.0 Safari/537.36 Edge/13.9200"
+    },
+    {
+        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"
+                      "(KHTML like Gecko) Chrome/46.0.2486.0 Safari/537.36 Edge/13.10586"
+    },
+    {
+        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"
+                      "(KHTML, like Gecko) Chrome/42.0.2311.135 Safari/537.36 Edge/12.246"
+    },
+    {
+        "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_3) AppleWebKit/537.75.14"
+                      "(KHTML, like Gecko) Version/7.0.3 Safari/7046A194A"
+    },
+    {
+        "User-Agent": "Mozilla/5.0 (iPad; CPU OS 6_0 like Mac OS X) AppleWebKit/536.26"
+                      "(KHTML, like Gecko) Version/6.0 Mobile/10A5355d Safari/8536.25"
+    },
+    {
+        "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_8) AppleWebKit/537.13+"
+                      "(KHTML, like Gecko) Version/5.1.7 Safari/534.57.2"
+    },
+    {
+        "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_3) AppleWebKit/534.55.3"
+                      "(KHTML, like Gecko) Version/5.1.3 Safari/534.53.10"
+    },
+    {
+        "User-Agent": "Mozilla/5.0 (iPad; CPU OS 5_1 like Mac OS X) AppleWebKit/534.46"
+                      "(KHTML, like Gecko ) Version/5.1 Mobile/9B176 Safari/7534.48.3"
+    },
+    {
+        "User-Agent": "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_8; de-at) AppleWebKit/533.21.1"
+                      "KHTML, like Gecko) Version/5.0.5 Safari/533.21.1"
+    },
+    {
+        "User-Agent": "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_7; da-dk) AppleWebKit/533.21.1"
+                      "(KHTML, like Gecko) Version/5.0.5 Safari/533.21.1"
+    },
+    {
+        "User-Agent": "Mozilla/5.0 (Windows; U; Windows NT 6.1; tr-TR) AppleWebKit/533.20.25"
+                      "(KHTML, like Gecko) Version/5.0.4 Safari/533.20.27"
+    },
+]

auto_translate/file_handling.py

@@ -0,0 +1,76 @@
+import os
+from os import path as path
+
+import yaml
+
+LANGUAGE_MAPPER = {
+    'english': 'en',
+    'german': 'de',
+    'italian': 'it',
+    'french': 'fr',
+    'spanish': 'es'
+}
+
+
+def clean_yaml(text: str) -> str:
+    for i in range(0, 10):
+        text = text.replace(f":{i}", ":")
+    return text
+
+
+def read_file_into_string(file_name: str) -> str:
+    with open(file_name, encoding='utf_8_sig') as f:
+        text = f.read()
+    return text
+
+
+def load_yaml(file_name: str) -> dict:
+    text = read_file_into_string(file_name)
+    clean_text = clean_yaml(text)
+    data = yaml.safe_load(clean_text)
+    return data
+
+
+def get_all_files(source_language: str, target_language: str) -> list[tuple]:
+    all_source_files = []
+    source_folder: str = path.abspath(path.join(__file__, f'../../localization/{source_language}'))
+    for _path, sub_dir, files in os.walk(source_folder):
+        _files = [path.abspath(path.join(_path, file)) for file in files]
+        all_source_files.extend(_files)
+    # create the equivalent target language path e.g. _english.yml to _german.yml
+    all_files = [(source, source.replace(f'_{source_language}.yml', f'_{target_language}.yml')) for source in
+                 all_source_files]
+    # also adjust the target language folder name, e.g. /english/... to /german/...
+    all_files = [(x[0], x[1].replace(f'\\{source_language}\\', f'\\{target_language}\\')) for x in all_files]
+    return all_files
+
+
+def validate_all_files(source: str, target: str = 'english') -> bool:
+    error_found = False
+    all_files = get_all_files(source, target)
+    for source_file, _ in all_files:
+        try:
+            load_yaml(source_file)
+        except Exception as e:
+            print(f"Error in file '{source_file}': {e}")
+            error_found = True
+    return error_found
+
+
+if __name__ == '__main__':
+    language_errors = {}
+    for source, source_code in LANGUAGE_MAPPER.items():
+        has_syntax_error = validate_all_files(source)
+        if has_syntax_error:
+            language_errors[source] = 'ERROR'
+        else:
+            language_errors[source] = 'OK'
+
+    print(f'{"Language":<15} {"Status":<15}')
+    print('-' * 30)
+    for source, status in language_errors.items():
+        print(f'{source:<15} {status:<15}')
+    print('-' * 30)
+
+    if 'ERROR' in language_errors.values():
+        raise ValueError(f"Aborting translation, there are YML syntax errors!")