translation: Move site verification after downloading all languages

Do not attempt to verify correctness of the website before refreshing
all languages. If any structural change happened (rename, permalink
change etc), it will fail because of the old files, not the freshly
downloaded ones.

Make htmlproofer postprocessing script language agnostic. This is mostly
about removing "language" parameter, which wasn't used anyway.

Author: marmarek

_utils/_translation_utils/check_all_langs.sh

@@ -0,0 +1,28 @@
+#!/bin/bash
+# to be run from the git root
+# $1 is directory where translated files reside and language needs to be added to internal urls
+# TODO param check
+
+set -e
+
+echo "================================= build site =================================="
+#read b
+bundle exec jekyll b
+
+all_ok=true
+echo "================================= run htmlproofer ==============================="
+htmlproofer ./_site   --disable-external   --checks-to-ignore ImageCheck   --file-ignore "./_site/video-tours/index.html,./_site/.*/video-tours/index.html" --url-ignore "/qubes-issues/" --log-level debug 2&> /tmp/html.output || all_ok=false
+
+# exit here if all is ok
+if $all_ok; then
+    echo 'All checks passed!'
+    exit
+fi
+
+echo "================================== as a last resort in case of errors process html proofer errors ================================="
+python3 _utils/_translation_utils/postprocess_htmlproofer.py /tmp/html.output "$1"
+
+echo "================================= build the site and run htmlproofer ===================================="
+rm -rf ./_site/
+bundle exec jekyll b
+htmlproofer ./_site   --disable-external   --checks-to-ignore ImageCheck   --file-ignore "./_site/video-tours/index.html,./_site/.*/video-tours/index.html" --url-ignore "/qubes-issues/" --log-level debug

_utils/_translation_utils/post_transifex_pull.sh

@@ -25,26 +25,3 @@ python3 _utils/_translation_utils/postprocess_translation.py "$1" "$2"  /tmp/tx-
 echo "============================ post processing step 4 press to cont ======================================"
 #read b
 bash _utils/_translation_utils/postprocess_translation.sh "$1" "$2" /tmp/translated_href_urls.txt
-
-
-echo "================================= build site =================================="
-#read b
-bundle exec jekyll b
-
-all_ok=true
-echo "================================= run htmlproofer ==============================="
-htmlproofer ./_site   --disable-external   --checks-to-ignore ImageCheck   --file-ignore "./_site/video-tours/index.html,./_site/$1/video-tours/index.html" --url-ignore "/qubes-issues/" --log-level debug 2&> /tmp/html.output || all_ok=false
-
-# exit here if all is ok
-if $all_ok; then
-    echo 'All checks passed!'
-    exit
-fi
-
-echo "================================== as a last resort in case of errors process html proofer errors ================================="
-python3 _utils/_translation_utils/postprocess_htmlproofer.py "$1" /tmp/html.output "$2"
-
-echo "================================= build the site and run htmlproofer ===================================="
-rm -rf ./_site/
-bundle exec jekyll b
-htmlproofer ./_site   --disable-external   --checks-to-ignore ImageCheck   --file-ignore "./_site/video-tours/index.html,./_site/$1/video-tours/index.html" --url-ignore "/qubes-issues/" --log-level debug

_utils/_translation_utils/postprocess_htmlproofer.py

@@ -1,8 +1,7 @@
 #!/usr/bin/python3
 '''
-python _utils/_translation_utils/postprocess_htmlproofer.py <LANG> <OUTPUT_FROM_HTMLPROOFER> <TRANSLATED_DIRECTORY>
+python _utils/_translation_utils/postprocess_htmlproofer.py <OUTPUT_FROM_HTMLPROOFER> <TRANSLATED_DIRECTORY>
 invoke: python _utils/_translation_utils/postprocess_htmlproofer.py de /tmp/html.output _translated/de/
-<LANG>[de]: translation language 
 <OUTPUT_FROM_HTMLPROOFER>[/tmp/html.output]: output from htmlproofer 
 <TRANSLATED_DIRECTORY>[_translated/de/]: the directory with the downloaded translated files from transifex
 '''
@@ -123,12 +122,11 @@ def process_markdown(translated_file, internal_links):
 
 
 
-def get_all_translated_permalinks_and_redirects_to_file_mapping(translated_dir, lang):
+def get_all_translated_permalinks_and_redirects_to_file_mapping(translated_dir):
     """
-    traverse the already updated (via tx pull) root directory with all the translated files for a specific language
-    and get their permalinks and redirects without the specific language
-    translated_dir: root directory with all the translated files for a specific language
-    lang: the specific language
+    traverse the already updated (via tx pull) root directory with all the translated files
+    and get their permalinks and redirects
+    translated_dir: root directory with all the translated files
     return: set holding the translated permalinks and redirects
     """
     mapping = {}
@@ -246,8 +244,6 @@ def process_yml(translated, errorlinks):
 if __name__ == '__main__':
     # python _utils/_translation_utils/postprocess_htmlproofer.py de /tmp/html.output _translated/de/
     parser = ArgumentParser()
-    # for which language should we do this
-    parser.add_argument("language")
     # the file containing the output of htmlproofer
     parser.add_argument("htmlproofer_output")
     # the directory containing the translated (downloaded via tx pull) files
@@ -262,11 +258,6 @@ def process_yml(translated, errorlinks):
         logger.error("please check your translated directory")
         exit(1)
 
-    if not args.language in TRANSLATED_LANGS:
-        print("language not in the expected translation languages")
-        logger.error("please check your translation language")
-        exit(1)
-
     if not isfile(args.htmlproofer_output):
         print("please check your html proofer output file")
         logger.error("please check your html proofer output file")
@@ -293,7 +284,7 @@ def process_yml(translated, errorlinks):
     logger.debug("------------------------------------------------")
     logger.debug("------------------------------------------------")
 
-    mapping, yml_files = get_all_translated_permalinks_and_redirects_to_file_mapping(args.translated_dir, args.language)
+    mapping, yml_files = get_all_translated_permalinks_and_redirects_to_file_mapping(args.translated_dir)
 
 
     log_debug('mapping ', mapping)

_utils/transifex-pull

@@ -23,6 +23,8 @@ for lang in "$@"; do
     bash _utils/_translation_utils/post_transifex_pull.sh "$lang" _translated/"$lang"
 done
 
+bash _utils/_translation_utils/check_all_langs.sh _translated
+
 # switch to ssh for push
 git -C _translated remote set-url origin [email protected]:QubesOS/qubes-translated