Changed the source to use TRANS files to update master

IAEA-NDS · Jun 1, 2024 · 46f00f4 · 46f00f4
1 parent 2cf3309
commit 46f00f4
Show file tree

Hide file tree

Showing 6 changed files with 248 additions and 216 deletions.
diff --git a/.gitignore b/.gitignore
@@ -141,6 +141,7 @@ pip-log.txt
 pip-delete-this-directory.txt
 
 
+
 # Environments
 .env
 .venv
@@ -149,7 +150,7 @@ venv/
 ENV/
 env.bak/
 venv.bak/
-
+.python-version
 
 
 

diff --git a/config.py b/config.py
@@ -2,11 +2,11 @@
 EXFOR_TRANS_URL = "https://nds.iaea.org/nrdc/exfor-master/trans/"
 
 
-# GIT_REPO_PATH = "./"  # here
-GIT_REPO_PATH = "/Users/okumuras/Documents/nucleardata/test/exfor_master/" # Private Repo
+GIT_REPO_PATH = "./"  # here
+# GIT_REPO_PATH = "/Users/sin/Documents/test/exfor_master/"  # Private Repo
 
-# GIT_REPO_URL = "https://api.github.com/repos/IAEA-NDS/exfor_master/releases"
-GIT_REPO_URL = "https://api.github.com/repos/shinokumura/exfor_master/releases"
+GIT_REPO_URL = "https://api.github.com/repos/IAEA-NDS/exfor_master/releases"
+# GIT_REPO_URL = "https://api.github.com/repos/shinokumura/exfor_master/releases" # Private Repo
 
 
 """
@@ -21,8 +21,8 @@
 
 
 headers = {
-            "User-Agent": "Mozilla/5.0 (X11; Linux x86_64)",
-            "X-Requested-With": "XMLHttpRequest",
-            "Referer": "https://127.0.0.1:5000",
-            'content-type': 'application/vnd.github+json',
-        }
+    "User-Agent": "Mozilla/5.0 (X11; Linux x86_64)",
+    "X-Requested-With": "XMLHttpRequest",
+    "Referer": "https://127.0.0.1:5000",
+    "content-type": "application/vnd.github+json",
+}
diff --git a/exfor_backups.py b/exfor_backups.py
@@ -26,9 +26,17 @@
 from zipfile import ZipFile
 
 import logging
+
 logging.basicConfig(filename="process.log", level=logging.DEBUG, filemode="w")
 
-from config import EXFOR_ALL_URL, EXFOR_ALL_PATH, EXFOR_ALL_TEMP, headers,  GIT_REPO_PATH, GIT_REPO_URL
+from config import (
+    EXFOR_ALL_URL,
+    EXFOR_ALL_PATH,
+    EXFOR_ALL_TEMP,
+    headers,
+    GIT_REPO_PATH,
+    GIT_REPO_URL,
+)
 
 
 ####################################################################
@@ -46,7 +54,6 @@ def get_local_files():
     return x  # get_latest_date(x)
 
 
-
 def get_server_files():
     x = []
     r = requests.get(EXFOR_ALL_URL)
@@ -66,14 +73,12 @@ def get_server_files():
     return x  # get_latest_date(x)
 
 
-
 def download_master_backup_sequential():
     x = get_server_files()
     for d2 in x:
         download_backup_zip(d2)
 
 
-
 def get_latest_date(x: list):
     if x:
         d = []
@@ -84,8 +89,6 @@ def get_latest_date(x: list):
         return None
 
 
-
-
 def download_latest_master_from_bk():
     # get the latest dictionary from https://nds.iaea.org/exfor-master/backup/
 
@@ -107,7 +110,6 @@ def download_latest_master_from_bk():
     return d2
 
 
-
 ####################################################################
 # General functions
 ####################################################################
@@ -124,8 +126,6 @@ def convert_dtform(dtstring: str):
     )
 
 
-
-
 def zip_filename(date_str):
     # date_dt: datatime.date(YYYY, MM, DD) format
     date_dt = convert_dtform(date_str.replace("EXFOR-", "").replace("exfor-", ""))
@@ -135,8 +135,6 @@ def zip_filename(date_str):
         return "".join(["EXFOR-", date_str, ".zip"])
 
 
-
-
 def bck_filename(date_dt):
     # date_dt: datatime.date(YYYY, MM, DD) format
     if date_dt < datetime.date(2010, 7, 1):
@@ -149,8 +147,6 @@ def bck_filename(date_dt):
         return "".join(["EXFOR-", date_dt.strftime("%Y-%m-%d"), ".bck"])
 
 
-
-
 def download_backup_zip(date_str):
     zipfile = zip_filename(date_str)
 
@@ -168,16 +164,16 @@ def download_backup_zip(date_str):
         r = requests.get(url, allow_redirects=True)
 
         if r.status_code == 404:
-            logging.error(f"Something wrong with retrieving new dictionary from the IAEA-NDS.")
+            logging.error(
+                f"Something wrong with retrieving new dictionary from the IAEA-NDS."
+            )
             sys.exit()
 
         open(zipfile, "wb").write(r.content)
         logging.info(f"zip file downloaded")
     return zipfile
 
 
-
-
 def unzip_file(zipfile: str):
     with ZipFile(zipfile, "r") as zipObj:
         listOfFileNames = zipObj.namelist()
@@ -190,9 +186,6 @@ def unzip_file(zipfile: str):
                 exit()
 
 
-
-
-
 def split_bck_file(filename: str):
     logging.info(f"split started")
     with open(filename, "r", encoding="cp1250") as infile:
@@ -227,9 +220,6 @@ def split_bck_file(filename: str):
     logging.info(f"split finished")
 
 
-
-
-
 def run_rsync():
     ## not used
     print("rsync")
@@ -239,8 +229,6 @@ def run_rsync():
     logging.info(f"rsync finished")
 
 
-
-
 def del_files(filename: str):
     if os.path.isfile(filename):
         os.remove(filename)
@@ -249,9 +237,6 @@ def del_files(filename: str):
         logging.info(f"File does not exist")
 
 
-
-
-
 ####################################################################
 # Git functions
 ####################################################################
@@ -265,24 +250,18 @@ def git_new_branch(date_str):
     logging.info(f"repo.active_branch {repo.active_branch}")
 
 
-
-
 def git_add_commit(date_str):
     repo.git.add("exforall/")
     repo.git.commit(m=date_str)
     logging.info(f"branch commit {date_str}")
 
 
-
-
 def git_push_branch(date_str):
     origin = repo.remote(name="origin")
     origin.push(date_str)
     logging.info(f"origin.push {date_str}")
 
 
-
-
 def git_merge_to_main(date_str):
     repo.git.checkout("main")
     repo.git.merge(date_str)
@@ -291,80 +270,76 @@ def git_merge_to_main(date_str):
     logging.info(f"repo.git.merge and origin.push master")
 
 
-
 def git_branches():
     branches = []
     remote_refs = repo.remote().refs
 
     for refs in remote_refs:
-        branches.append(refs.name.replace("origin/",""))
+        branches.append(refs.name.replace("origin/", ""))
 
     return branches
 
 
-
-
 def git_tags():
     tags = []
     for tag in repo.tags:
-        tags.append(tag.name.replace("Backup-",""))
+        tags.append(tag.name.replace("Backup-", ""))
     return tags
 
 
-
-
-
 def semantic_release_name(branch_name):
     name_sp = branch_name.split("-")
     if len(branch_name) == 10:
-        return "v" + str(name_sp[0]) + str(name_sp[1])+ str(name_sp[2]) + ".0"
+        return "v" + str(name_sp[0]) + str(name_sp[1]) + str(name_sp[2]) + ".0"
     elif branch_name.count("-") == 3:
-        return "v" + str(name_sp[0]) + str(name_sp[1])+ str(name_sp[2]) + "." + name_sp[3]
+        return (
+            "v" + str(name_sp[0]) + str(name_sp[1]) + str(name_sp[2]) + "." + name_sp[3]
+        )
     elif len(branch_name) == 11:
-        return "v" + str(name_sp[0]) + str(name_sp[1])+ str(name_sp[2])[0:2] + "." + name_sp[2][2:]
-
-
+        return (
+            "v"
+            + str(name_sp[0])
+            + str(name_sp[1])
+            + str(name_sp[2])[0:2]
+            + "."
+            + name_sp[2][2:]
+        )
 
 
 def git_log():
     return repo.git.log("--name-status", "--no-merges", "-n1", "--pretty=oneline")
 
 
-
-
 def git_tagging(branch_name):
     repo.git.checkout(branch_name)
     repo.git.pull("origin", branch_name)
     repo.git.fetch("origin")
 
     msg = git_log()
     repo.create_tag("Backup-" + branch_name, ref=branch_name, message=msg)
-    
+
     origin = repo.remote(name="origin")
     origin.push("Backup-" + branch_name)
     logging.info(f"tagging branch_name: {branch_name}")
 
 
-
-
 def git_release(branch_name):
     data_body = {
         "tag_name": "Backup-" + branch_name,
         "name": semantic_release_name(branch_name),
         "draft": False,
         "prerelease": False,
-        "generate_release_notes": False
-        }
+        "generate_release_notes": False,
+    }
 
     r = requests.post(
-        GIT_REPO_URL, 
-        data=json.dumps(data_body), 
-        headers=headers, 
+        GIT_REPO_URL,
+        data=json.dumps(data_body),
+        headers=headers,
         verify=False,
     )
 
 
-
 def git_delete_branch(date_str):
     repo.git.checkout("main")
     repo.git.branch("-d", date_str)
@@ -373,8 +348,6 @@ def git_delete_branch(date_str):
     logging.info(f"repo.git.branch -d")
 
 
-
-
 def process_zip_file(date_str):
     ## download and unzip .zip file
     download_backup_zip(date_str)
@@ -392,8 +365,8 @@ def process_zip_file(date_str):
     ## create Release in Github
     git_tagging(date_str)
 
-    ## release does not work from the CLI from Github/Actions, 
-    ## it needs curl to post to REST API 
+    ## release does not work from the CLI from Github/Actions,
+    ## it needs curl to post to REST API
     ## (see .github/workflows/manual.yaml)
     # git_release(date_str)
 
@@ -420,7 +393,6 @@ def runall():
         process_zip_file(date_str)
 
 
-
 ####################################################################
 # This is the process to update the repository.
 ####################################################################
@@ -443,9 +415,8 @@ def update():
         else:
             not_processed.append(date_str)
 
-
     ## because exfor-2010-07-12.zip is broken
-    not_processed.remove('2010-07-12')
+    not_processed.remove("2010-07-12")
     logging.info(f"process starts for {not_processed}")
 
     if not_processed:
@@ -455,13 +426,8 @@ def update():
     else:
         logging.info(f"repository is up-to-date")
 
-
     return " ".join(not_processed)
 
 
-
 if __name__ == "__main__":
-
     print(update())
-
-