diff --git a/.gitignore b/.gitignore index 18c94b8..948ba78 100644 --- a/.gitignore +++ b/.gitignore @@ -153,6 +153,8 @@ dmypy.json .vscode/ # under development +*/pickles* +*.pickle evaluations/ geo/ docker/ @@ -188,6 +190,4 @@ sqldb.py check* json.schema *.sh - -# other packages -exfor_dictionary/ +uninstall.txt \ No newline at end of file diff --git a/.gitmodules b/.gitmodules index fb2c7e9..6e2dd47 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,3 @@ [submodule "submodules"] - path = submodules + path = src/exforparser/submodules url = https://github.com/shinokumura/exparser-submodule.git diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..a014cc9 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,3 @@ +include src/tabulated/MTall.dat +include src/tabulated/*.json +include src/submodules/utilities/*.txt \ No newline at end of file diff --git a/VERSION b/VERSION deleted file mode 100644 index 9dd28c2..0000000 --- a/VERSION +++ /dev/null @@ -1 +0,0 @@ -1.0.0beta \ No newline at end of file diff --git a/pickles/entry.pickle b/pickles/entry.pickle deleted file mode 100644 index 56450be..0000000 Binary files a/pickles/entry.pickle and /dev/null differ diff --git a/pickles/institute.pickle b/pickles/institute.pickle deleted file mode 100644 index b8290c1..0000000 Binary files a/pickles/institute.pickle and /dev/null differ diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..70baea7 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,51 @@ +[build-system] +requires = ["setuptools", "setuptools-scm"] +build-backend = "setuptools.build_meta" + + +[project] +name = "exforparser" +authors = [ + {name = "Shin Okumura/IAEA-NDS", email = "s.okumura@iaea.org"}, +] +version = "1.0.0b" +description = "EXFOR Parser" +readme = "README.md" +requires-python = ">=3.8" +keywords = ["EXFOR", "Nuclear Data"] +license = {file = "LICENSE"} +classifiers = [ + "Development Status :: 4 - Beta", + "Operating System :: OS Independent", + "License :: OSI Approved :: MIT License", + "Programming Language :: Python :: 3", +] +dynamic = ["dependencies"] + +[tool.setuptools.dynamic] +dependencies = {file = ["requirements.txt"]} + + +[tool.setuptools.packages.find] +where = ["src"] + + +[tool.setuptools] +include-package-data = true + + +[tool.setuptools.package-data] +"exforparser.tabulated" = ["*.dat", "*.json"] +"submodules.utilities" = ["*.txt"] + + + + +[project.urls] +Homepage = "https://github.com/shinokumura/exforparser" + + +[project.scripts] +exparser = "exforparser.exparser:main" +tabulated = "exforparser.tabulated:main" +test = "exforparser.pa:main" \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index f673513..903d31b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,8 +1,12 @@ common==0.1.2 -numpy==1.23.0 +greenlet==2.0.2 +numpy==1.25.2 pandas==2.0.3 pyparsing==3.1.1 -setuptools==57.5.0 +python-dateutil==2.8.2 +pytz==2023.3 +git+https://github.com/shinokumura/ripl3_json.git +six==1.16.0 SQLAlchemy==2.0.18 -exfor_dictionary @ git+https://github.com/IAEA-NDS/exfor_dictionary@main -ripl3_json @ git+https://github.com/shinokumura/ripl3_json@main \ No newline at end of file +typing_extensions==4.7.1 +tzdata==2023.3 diff --git a/setup.py b/setup.py index c319b7c..97ad1c8 100644 --- a/setup.py +++ b/setup.py @@ -1,33 +1,41 @@ -from setuptools import setup, find_packages +import setuptools +if __name__ == "__main__": + setuptools.setup() -def read_requirements(file): - with open(file) as f: - return f.read().splitlines() +# from setuptools import setup, find_packages -def read_file(file): - with open(file) as f: - return f.read() +# # print(find_packages()) +# # --> return ['tabulated', 'parser', 'sql'] -version = read_file("VERSION") -requirements = read_requirements("requirements.txt") +# def read_requirements(file): +# with open(file) as f: +# return f.read().splitlines() -setup( - name="exforparser", - description="EXFOR Parser", - packages=find_packages(exclude=["test"]), - version=version, - author="Shin Okumura/IAEA-NDS", - author_email="s.okumura@iaea.org", - maintainer="IAEA-NDS", - maintainer_email="nds.contact-point@iaea.org", - license="GPL-2.0 license", - url="https://github.com/s.okumura@iaea.org/exforparser", - python_requires=">=3.8", - install_requires=requirements, - classifiers=[ - "Programming Language :: Python :: 3", - "Operating System :: OS Independent", - "License :: OSI Approved :: GPL-2.0 license", - ], -) +# def read_file(file): +# with open(file) as f: +# return f.read() + +# version = read_file("VERSION") +# requirements = read_requirements("requirements.txt") + +# setup( +# name="exforparser", +# description="EXFOR Parser", +# packages=find_packages(exclude=["test"]), +# py_modules=['exparser', 'tabulated'], +# version=version, +# author="Shin Okumura/IAEA-NDS", +# author_email="s.okumura@iaea.org", +# maintainer="IAEA-NDS", +# maintainer_email="nds.contact-point@iaea.org", +# license="GPL-2.0 license", +# url="https://github.com/s.okumura@iaea.org/exforparser", +# python_requires=">=3.8", +# install_requires=requirements, +# classifiers=[ +# "Programming Language :: Python :: 3", +# "Operating System :: OS Independent", +# "License :: OSI Approved :: GPL-2.0 license", +# ], +# ) diff --git a/sql/queries.py b/sql/queries.py deleted file mode 100644 index 50841eb..0000000 --- a/sql/queries.py +++ /dev/null @@ -1,388 +0,0 @@ -import pandas as pd -from collections import OrderedDict -from operator import getitem - - -from libraries.datahandle.list import elemtoz_nz -from sql.models import Exfor_Bib, Exfor_Reactions, Exfor_Data, Exfor_Indexes -from endftables_sql.models import Endf_Reactions, Endf_XS_Data, Endf_Residual_Data, Endf_FY_Data -from config import session, session_lib, engines -from common import energy_range_conversion, get_number_from_string, get_str_from_string - - -######## -------------------------------------- ########## -## Join table for AGGrid -######## -------------------------------------- ########## -def join_reac_bib(): - connection = engines["exfor"].connect() - all = session().query(Exfor_Indexes.entry_id, - Exfor_Indexes.target, - Exfor_Indexes.process, - Exfor_Indexes.residual, - Exfor_Indexes.e_inc_min, - Exfor_Indexes.e_inc_max, - Exfor_Indexes.sf5, - Exfor_Indexes.sf6, - Exfor_Indexes.sf7, - Exfor_Indexes.sf8, - Exfor_Bib.authors, - Exfor_Bib.year, - Exfor_Bib.main_facility_institute, - Exfor_Bib.main_facility_type, - ).join(Exfor_Bib, Exfor_Bib.entry == Exfor_Indexes.entry - ).order_by(Exfor_Bib.year.desc()) - df = pd.read_sql( - sql=all.statement, - con=connection, - ) - - return df - - - - - - -def reaction_query_simple(type, elem, mass, reaction, branch): - # https://zenn.dev/shimakaze_soft/articles/6e5e47851459f5 - connection = engines["exfor"].connect() - target = elemtoz_nz(elem) + "-" + elem.upper() + "-" + mass - - queries = [Exfor_Indexes.target == target, - Exfor_Indexes.process == reaction.upper(), - Exfor_Indexes.sf6 == type.upper(), - Exfor_Indexes.arbitrary_data == False] - - - - if branch == "PAR": - queries.append(Exfor_Indexes.sf5 == branch) - - elif isinstance(branch, int): - queries.append(Exfor_Indexes.level_num == branch) - - - reac = session().query(Exfor_Indexes, Exfor_Bib).filter(*queries - ).join(Exfor_Bib, Exfor_Indexes.entry == Exfor_Bib.entry, isouter=True).distinct() - - df = pd.read_sql( - sql=reac.statement, - con=connection, - ) - # print(df) - return df - - - -######## -------------------------------------- ########## -## Reaction queries for the dataexplorer -######## -------------------------------------- ########## -def reaction_query(type, elem, mass, reaction, branch=None, rp_elem=None, rp_mass=None): - # https://zenn.dev/shimakaze_soft/articles/6e5e47851459f5 - reac = None - target = elemtoz_nz(elem) + "-" + elem.upper() + "-" + mass - - queries = [Exfor_Indexes.target == target, - Exfor_Indexes.arbitrary_data == False] - - if branch: - if branch == "PAR": - queries.append(Exfor_Indexes.sf5 == branch) - - elif isinstance(branch, int): - queries.append(Exfor_Indexes.level_num == branch) - - - elif type == "FY": - queries.append(Exfor_Indexes.sf5 == branch.upper()) - - else: - queries.append(Exfor_Indexes.sf5 == None) - - - if type == "Residual": - type = "SIG" - - if rp_mass.endswith(("m", "M", "g", "G")): - residual = rp_elem.capitalize() + "-" + str( get_number_from_string(rp_mass) ) + "-" + get_str_from_string(str(rp_mass)).upper() - - else: - residual = rp_elem.capitalize() + "-" + str(rp_mass) - - queries.append(Exfor_Indexes.projectile == reaction.upper()) - queries.append(Exfor_Indexes.residual == residual) - - - elif type != "Residual": - queries.append(Exfor_Indexes.process == reaction.upper()) - - if not "TOT" in reaction: - queries.append(~Exfor_Indexes.sf4.endswith("-G")) - queries.append(~Exfor_Indexes.sf4.endswith("-M")) - - - queries.append(Exfor_Indexes.sf6 == type.upper()) - - reac = session().query(Exfor_Indexes).filter(*queries).all() - - entries = {} - - if reac: - for ent in reac: - entries[ent.entry_id] = { - "e_inc_min": ent.e_inc_min, - "e_inc_max": ent.e_inc_max, - "points": ent.points, - "sf5": ent.sf5, - "sf8": ent.sf8, - "x4_code": ent.x4_code, - } - - return entries - - - -def get_entry_bib(entries): - - bib = session().query(Exfor_Bib).filter(Exfor_Bib.entry.in_(tuple(entries))).all() - - legend = {} - - for b in bib: - legend[b.entry] = { - "author": b.first_author, - "year": b.year if b.year else 1900, ## Comments SO: should be int in SQL - } - - return OrderedDict( - sorted(legend.items(), key=lambda x: getitem(x[1], "year"), reverse=True), - ) - - - -def data_query(entids): - - connection = engines["exfor"].connect() - - data = session().query(Exfor_Data).filter( - Exfor_Data.entry_id.in_(tuple(entids)) - ) - - df = pd.read_sql( - sql=data.statement, - con=connection, - ) - - return df - - - -######## -------------------------------------- ######## -# Queries for endftables -######## -------------------------------------- ######## - - -def lib_query(type, elem, mass, reaction, mt, rp_elem, rp_mass): - - target = elem + mass.zfill(3) - queries = [Endf_Reactions.target == target, - Endf_Reactions.projectile == reaction.split(",")[0].lower()] - - if type == "SIG": - type = "xs" - queries.append(Endf_Reactions.process == reaction.split(",")[1].upper()) - queries.append(Endf_Reactions.mt == mt) # if mt is not None else Endf_Reactions.mt is not None) - - elif type=="Residual": - if rp_mass.endswith(("m", "M", "g", "G")): - residual = rp_elem.capitalize() + str(get_number_from_string(rp_mass)) + get_str_from_string(str(rp_mass)).lower() - - else: - residual = rp_elem.capitalize() + str(rp_mass) - - queries.append(Endf_Reactions.residual == residual) - - elif type=="FY": - queries.append(Endf_Reactions.mt == mt) - - - queries.append(Endf_Reactions.type == type.lower()) - ## Establish session to the endftable database - reac = session_lib().query(Endf_Reactions).filter(*queries).all() - - libs = {} - for r in reac: - # print(r.reaction_id, r.evaluation, r.target, r.projectile, r.process, r.residual, r.mt) - libs[r.reaction_id] = r.evaluation - - return libs - - - -def lib_xs_data_query(ids): - connection = engines["endftables"].connect() - data = session().query(Endf_XS_Data).filter(Endf_XS_Data.reaction_id.in_(tuple(ids))) - - df = pd.read_sql( - sql=data.statement, - con=connection, - ) - - return df - - - -def lib_residual_data_query(ids): - connection = engines["endftables"].connect() - data = session().query(Endf_Residual_Data).filter(Endf_Residual_Data.reaction_id.in_(tuple(ids))) - - df = pd.read_sql( - sql=data.statement, - con=connection, - ) - - return df - - - - -def lib_data_query_fy(ids, en_lower, en_upper): - - connection = engines["endftables"].connect() - data = session().query(Endf_FY_Data).filter( - Endf_FY_Data.reaction_id.in_(tuple(ids)), - # Endf_FY_Data.en_inc >= en_lower, - # Endf_FY_Data.en_inc <= en_upper, - ) - - df = pd.read_sql( - sql=data.statement, - con=connection, - ) - - return df - - - -######## -------------------------------------- ######## -# Queries for FY -######## -------------------------------------- ######## - - -def fy_branch(branch): - if branch == "PRE": - return ["PRE", "TER", "QTR", "PRV", "TER/CHG"] - - if branch == "IND": - return ["IND", "SEC", "MAS", "CHG", "SEC/CHN"] - - if branch == "CUM": - return ["CUM", "CHN"] - - - - -def reaction_query_fy(type, elem, mass, reaction, branch, mesurement_opt_fy, energy_range): - # https://zenn.dev/shimakaze_soft/articles/6e5e47851459f5 - - target = elemtoz_nz(elem) + "-" + elem.upper() + "-" + mass - lower, upper = energy_range_conversion(energy_range) - - - reac = session().query(Exfor_Indexes).filter( - Exfor_Indexes.target == target, - Exfor_Indexes.process == reaction.upper(), - Exfor_Indexes.sf5.in_(tuple(fy_branch(branch))), - Exfor_Indexes.sf6 == type.upper(), - Exfor_Indexes.e_inc_min >= lower, - Exfor_Indexes.e_inc_max <= upper, - Exfor_Indexes.arbitrary_data == False, - (Exfor_Indexes.sf4 == "MASS" if mesurement_opt_fy=="A" else Exfor_Indexes.sf4 == "ELEM" if mesurement_opt_fy=="Z" else Exfor_Indexes.sf4.isnot(None) ), - ).all() - - entries = {} - - for ent in reac: - entries[ent.entry_id] = { - "e_inc_min": ent.e_inc_min, - "e_inc_max": ent.e_inc_max, - "points": ent.points, - "sf8": ent.sf8, - "residual": ent.residual, - "x4_code": ent.x4_code, - } - - return entries - - - - - - - - -def reaction_query_fission(type, elem, mass, reaction, branch, energy_range): - # https://zenn.dev/shimakaze_soft/articles/6e5e47851459f5 - sf4 = None - sf5 = None - sf6 = None - target = elemtoz_nz(elem) + "-" + elem.upper() + "-" + mass - - queries = [Exfor_Indexes.target == target, - Exfor_Indexes.process == reaction.upper(), - Exfor_Indexes.arbitrary_data == False] - - if branch == "nu_n": - sf5 = ["PR"] - sf6 = ["NU"] - elif branch == "nu_g": - sf4 = "0-G-0" - sf5 = ["PR"] - sf6 = ["FY"] - elif branch == "dn": - sf5 = ["DL"] - sf6 = ["NU"] - elif branch == "pfns": - sf5 = ["PR"] - sf6 = ["NU/DE"] - elif branch == "pfgs": - sf4 = "0-G-0" - sf5 = ["PR"] - sf6 = ["FY/DE"] - else: - ## to avoid large query - return None, None - - if sf4: - queries.append(Exfor_Indexes.sf4 == sf4) - - if sf5: - queries.append(Exfor_Indexes.sf5.in_(tuple(sf5))) - - if sf6: - queries.append(Exfor_Indexes.sf6.in_(tuple(sf6))) - - if energy_range: - lower, upper = energy_range_conversion(energy_range) - queries.append(Exfor_Indexes.e_inc_min >= lower) - queries.append(Exfor_Indexes.e_inc_max <= upper) - - reac = session().query(Exfor_Indexes).filter(*queries).all() - - entids = {} - entries = [] - - for ent in reac: - entids[ent.entry_id] = { - "e_inc_min": ent.e_inc_min, - "e_inc_max": ent.e_inc_max, - "points": ent.points, - "sf5": ent.sf5, - "sf8": ent.sf8, - "x4_code": ent.x4_code, - } - entries += [ent.entry] - # print(entids) - - return entids, entries - diff --git a/config.py b/src/exforparser/config.py similarity index 84% rename from config.py rename to src/exforparser/config.py index c6deea7..dabe289 100644 --- a/config.py +++ b/src/exforparser/config.py @@ -9,6 +9,7 @@ # Contact: nds.contact-point@iaea.org # #################################################################### +import os import sqlalchemy as db from sqlalchemy.orm import sessionmaker @@ -16,26 +17,18 @@ if DEVENV: DATA_DIR = "/Users/okumuras/Documents/nucleardata/EXFOR/" - MODULES_DIR = "/Users/okumuras/Dropbox/Development/" - else: DATA_DIR = "/srv/data/dataexplorer2/" - MODULES_DIR = "/srv/data/dataexplorer2/" - - -EXFOR_DB = DATA_DIR + "exfor.sqlite" +EXFOR_DB = os.path.join(DATA_DIR, "exfor_tmp.sqlite") +EXFOR_MASTER_REPO_PATH = os.path.join(DATA_DIR, "exfor_master") """ Pickle path of list of EXFOR master files made by parser.list_x4files.py""" ENTRY_INDEX_PICKLE = "pickles/entry.pickle" -TO_JSON = True -POST_DB = False - - OUT_PATH = DATA_DIR + "../../../Desktop/" diff --git a/exparser.py b/src/exforparser/exparser.py similarity index 94% rename from exparser.py rename to src/exforparser/exparser.py index 5e2b2c2..8b753f7 100644 --- a/exparser.py +++ b/src/exforparser/exparser.py @@ -10,10 +10,7 @@ # #################################################################### -from logging import raiseExceptions -import pandas as pd import os -import random import json import logging @@ -21,12 +18,13 @@ from config import OUT_PATH -from submodules.utilities.util import del_outputs, print_time, get_entry_update_date +from submodules.utilities.util import del_outputs, print_time from parser.list_x4files import list_entries_from_df -from parser.exfor_entry import Entry +from parser.exfor_entry import Entry, get_entry_update_date from parser.exfor_subentry import Subentry + ## get update data from git commit and store info to Python dictionary update_date = get_entry_update_date() diff --git a/parser/exceptions.py b/src/exforparser/parser/exceptions.py similarity index 100% rename from parser/exceptions.py rename to src/exforparser/parser/exceptions.py diff --git a/parser/exfor_bib.py b/src/exforparser/parser/exfor_bib.py similarity index 100% rename from parser/exfor_bib.py rename to src/exforparser/parser/exfor_bib.py diff --git a/parser/exfor_block.py b/src/exforparser/parser/exfor_block.py similarity index 100% rename from parser/exfor_block.py rename to src/exforparser/parser/exfor_block.py diff --git a/parser/exfor_data.py b/src/exforparser/parser/exfor_data.py similarity index 99% rename from parser/exfor_data.py rename to src/exforparser/parser/exfor_data.py index 0f105e3..12c128c 100644 --- a/parser/exfor_data.py +++ b/src/exforparser/parser/exfor_data.py @@ -9,15 +9,14 @@ # Contact: nds.contact-point@iaea.org # #################################################################### +import re from pyparsing import * from .exfor_field import data_header from submodules.utilities.util import flatten_list from submodules.utilities.elem import ztoelem -# from dictionary.exfor_dictionary import Diction -# sys.path.append("../exfor_dictionary/") -from exfor_dictionary import Diction +from exfor_dictionary.exfor_dictionary import Diction def corr(invalue): if re.search(r"\d|\.[+]\d", invalue): diff --git a/parser/exfor_entry.py b/src/exforparser/parser/exfor_entry.py similarity index 92% rename from parser/exfor_entry.py rename to src/exforparser/parser/exfor_entry.py index 7912466..efb6222 100644 --- a/parser/exfor_entry.py +++ b/src/exforparser/parser/exfor_entry.py @@ -14,11 +14,21 @@ import sys sys.path.append("../") -from config import EXFOR_ALL_PATH +from config import EXFOR_MASTER_REPO_PATH from .exfor_subentry import Subentry from .exfor_block import get_block +def get_entry_update_date(): + d = {} + file = os.path.join(EXFOR_MASTER_REPO_PATH, "entry_updatedate.dat") + with open(file) as f: + for line in f: + x = line.split(" ") + d.update({x[0]: {"last_update": x[1], "revisions": x[2].strip()}}) + return d + + def open_read_file(filename=""): """ separate and convert all subentries into dictionary @@ -71,7 +81,7 @@ def _check_entry_nlen(entnum): def x4filename(self): if self._check_entry_nlen(self.entnum): - return os.path.join(EXFOR_ALL_PATH, self.entnum[:3], self.entnum + ".x4") + return os.path.join(EXFOR_MASTER_REPO_PATH, "exforall", self.entnum[:3], self.entnum + ".x4") def get_entry_exfor(self) -> dict: """return as EXFOR format""" diff --git a/parser/exfor_field.py b/src/exforparser/parser/exfor_field.py similarity index 100% rename from parser/exfor_field.py rename to src/exforparser/parser/exfor_field.py diff --git a/parser/exfor_reaction.py b/src/exforparser/parser/exfor_reaction.py similarity index 100% rename from parser/exfor_reaction.py rename to src/exforparser/parser/exfor_reaction.py diff --git a/parser/exfor_subentry.py b/src/exforparser/parser/exfor_subentry.py similarity index 100% rename from parser/exfor_subentry.py rename to src/exforparser/parser/exfor_subentry.py diff --git a/parser/exfor_unit.py b/src/exforparser/parser/exfor_unit.py similarity index 96% rename from parser/exfor_unit.py rename to src/exforparser/parser/exfor_unit.py index 8454570..e9ffba8 100644 --- a/parser/exfor_unit.py +++ b/src/exforparser/parser/exfor_unit.py @@ -1,7 +1,7 @@ import math -from exfor_dictionary import Diction +from exfor_dictionary.exfor_dictionary import Diction d = Diction() # en_heads = d.get_incident_en_heads() diff --git a/parser/list_x4files.py b/src/exforparser/parser/list_x4files.py similarity index 98% rename from parser/list_x4files.py rename to src/exforparser/parser/list_x4files.py index b2e5cdd..eae46b8 100644 --- a/parser/list_x4files.py +++ b/src/exforparser/parser/list_x4files.py @@ -20,9 +20,9 @@ from .exceptions import * -sys.path.append("../") -from config import EXFOR_ALL_PATH, ENTRY_INDEX_PICKLE +from config import EXFOR_MASTER_REPO_PATH, ENTRY_INDEX_PICKLE +EXFOR_ALL_PATH = os.path.join(EXFOR_MASTER_REPO_PATH, "exforall") """ Run dos2unix for all .x4 files beforehand as follows: @@ -37,15 +37,6 @@ def rundos2unix(): -def _check_x4alldir(): - from parser.exceptions import Nox4AllDirExistenceError - - if os.path.isdir(EXFOR_ALL_PATH): - return True - else: - raise Nox4AllDirExistenceError(EXFOR_ALL_PATH) - - def _index_pickle_road(): ''' reado exfor entry index pickle ''' try: @@ -103,6 +94,7 @@ def list_exfor_files(): files = [] entries = [] + if os.path.exists(EXFOR_ALL_PATH): dirs = [f for f in os.listdir(EXFOR_ALL_PATH) if not f.startswith(".")] for d in dirs: diff --git a/sql/creation.py b/src/exforparser/sql/creation.py similarity index 100% rename from sql/creation.py rename to src/exforparser/sql/creation.py diff --git a/sql/models.py b/src/exforparser/sql/models.py similarity index 100% rename from sql/models.py rename to src/exforparser/sql/models.py diff --git a/src/exforparser/submodules/utilities/A_min_max.txt b/src/exforparser/submodules/utilities/A_min_max.txt new file mode 100644 index 0000000..25d3551 --- /dev/null +++ b/src/exforparser/submodules/utilities/A_min_max.txt @@ -0,0 +1,122 @@ +# Minimum and maximum mass number in charge number +# Provided by Marco VERPELLI (IAEA-NDS) +# z,min,max +0,1,6 +1,1,7 +2,3,10 +3,3,13 +4,5,16 +5,6,21 +6,8,22 +7,10,25 +8,11,28 +9,13,31 +10,15,34 +11,17,37 +12,18,40 +13,21,43 +14,22,44 +15,25,46 +16,26,49 +17,29,52 +18,30,53 +19,31,56 +20,34,58 +21,38,61 +22,38,64 +23,42,67 +24,42,70 +25,44,73 +26,45,76 +27,47,78 +28,48,82 +29,52,83 +30,54,85 +31,59,87 +32,59,90 +33,62,92 +34,64,95 +35,65,101 +36,69,102 +37,72,106 +38,73,108 +39,76,111 +40,77,114 +41,81,117 +42,81,119 +43,85,122 +44,85,125 +45,89,128 +46,90,131 +47,92,132 +48,94,134 +49,96,137 +50,99,140 +51,103,142 +52,104,145 +53,108,147 +54,108,150 +55,112,152 +56,114,154 +57,116,157 +58,120,158 +59,121,161 +60,125,163 +61,128,165 +62,129,168 +63,130,169 +64,134,172 +65,135,174 +66,139,176 +67,140,178 +68,143,180 +69,144,181 +70,149,185 +71,150,188 +72,151,190 +73,155,194 +74,157,197 +75,159,199 +76,161,203 +77,164,205 +78,165,208 +79,170,210 +80,170,216 +81,176,218 +82,178,220 +83,184,224 +84,186,227 +85,191,229 +86,193,231 +87,197,233 +88,201,234 +89,205,236 +90,208,238 +91,211,239 +92,214,255 +93,219,244 +94,228,247 +95,223,249 +96,233,251 +97,233,254 +98,236,256 +99,240,257 +100,241,259 +101,244,260 +102,248,262 +103,252,266 +104,253,267 +105,255,270 +106,258,271 +107,260,278 +108,263,278 +109,266,282 +110,267,282 +111,272,286 +112,277,286 +113,278,290 +114,284,290 +115,287,290 +116,289,294 +117,293,294 +118,293,295 diff --git a/src/exforparser/submodules/utilities/elem.py b/src/exforparser/submodules/utilities/elem.py new file mode 100644 index 0000000..ee36e51 --- /dev/null +++ b/src/exforparser/submodules/utilities/elem.py @@ -0,0 +1,174 @@ +#################################################################### +# +# This file is part of exfor-parser. +# Copyright (C) 2022 International Atomic Energy Agency (IAEA) +# +# Disclaimer: The code is still under developments and not ready +# to use. It has been made public to share the progress +# among collaborators. +# Contact: nds.contact-point@iaea.org +# +#################################################################### +ELEMS = [ + "H", + "He", + "Li", + "Be", + "B", + "C", + "N", + "O", + "F", + "Ne", + "Na", + "Mg", + "Al", + "Si", + "P", + "S", + "Cl", + "Ar", + "K", + "Ca", + "Sc", + "Ti", + "V", + "Cr", + "Mn", + "Fe", + "Co", + "Ni", + "Cu", + "Zn", + "Ga", + "Ge", + "As", + "Se", + "Br", + "Kr", + "Rb", + "Sr", + "Y", + "Zr", + "Nb", + "Mo", + "Tc", + "Ru", + "Rh", + "Pd", + "Ag", + "Cd", + "In", + "Sn", + "Sb", + "Te", + "I", + "Xe", + "Cs", + "Ba", + "La", + "Ce", + "Pr", + "Nd", + "Pm", + "Sm", + "Eu", + "Gd", + "Tb", + "Dy", + "Ho", + "Er", + "Tm", + "Yb", + "Lu", + "Hf", + "Ta", + "W", + "Re", + "Os", + "Ir", + "Pt", + "Au", + "Hg", + "Tl", + "Pb", + "Bi", + "Po", + "At", + "Rn", + "Fr", + "Ra", + "Ac", + "Th", + "Pa", + "U", + "Np", + "Pu", + "Am", + "Cm", + "Bk", + "Cf", + "Es", + "Fm", + "Md", + "No", + "Lr", + "Rf", + "Db", + "Sg", + "Bh", + "Hs", + "Mt", + "Ds", + "Rg", + "Cn", + "Nh", + "Fl", + "Mc", + "Lv", + "Ts", + "Og", + "119", + "120", + "121", + "122", + "123", + "124", + "125", +] + + +def ztoelem(z): + if z == 0: + elem_name = "n" + else: + try: + z = z - 1 + elem_name = ELEMS[z] + except ValueError: + elem_name = "" + # print(elem_name.capitalize()) + # return elem_name.upper() + return elem_name.capitalize() + + +def elemtoz(elem): + try: + z = ELEMS.index(elem) + z = z + 1 + z = str(z).zfill(3) + except ValueError: + z = "" + return z + + + +def numtoisomer(num): + if num == "0": + return "G" + elif num == "1": + return "M1" + elif num == "2": + return "M2" + elif num == "3": + return "M3" diff --git a/src/exforparser/submodules/utilities/mass.py b/src/exforparser/submodules/utilities/mass.py new file mode 100644 index 0000000..94c7b79 --- /dev/null +++ b/src/exforparser/submodules/utilities/mass.py @@ -0,0 +1,14 @@ +MASS_RANGE_FILE = "A_min_max.txt" + +# ------------------------------------------------------------------------------ +# Mass +# ------------------------------------------------------------------------------ +def read_mass_range(): + mass_range = {} + with open(MASS_RANGE_FILE) as f: + for z in f.readlines(): + if z.startswith("#"): + continue + z_mass = z.split(",") + mass_range[z_mass[0]] = {"min": z_mass[1].strip(), "max": z_mass[2].strip()} + return mass_range diff --git a/src/exforparser/submodules/utilities/util.py b/src/exforparser/submodules/utilities/util.py new file mode 100644 index 0000000..f570f98 --- /dev/null +++ b/src/exforparser/submodules/utilities/util.py @@ -0,0 +1,112 @@ +import os +import shutil +import time +import json + + +def slices(s, *args): + position = 0 + for length in args: + yield s[position : position + length] + position += length + + +def flatten(xs): + from collections.abc import Iterable + + for x in xs: + if isinstance(x, Iterable) and not isinstance(x, (str, bytes)): + yield from flatten(x) + else: + yield x + + +def flatten_list(list): + return [item for sublist in list for item in sublist] + + +def check_list(init_list): + print(init_list) + # print(any(isinstance(i, list) for i in init_list)) + + def _is_list_instance(init_list): + print(isinstance(init_list, list)) + + sub_list = flatten_list(init_list) + _is_list_instance(sub_list) + + return isinstance(init_list, list) + + +def dict_merge(dicts_list): + d = {**dicts_list[0]} + for entry in dicts_list[1:]: + # print("entry:", entry) + for k, v in entry.items(): + d[k] = ( + [d[k], v] + if k in d and type(d[k]) != list + else [*d[k] + v] + if k in d + else v + ) + return d + + + +def combine_dict(d1, d2): + return { + k: list(d[k] for d in (d1, d2) if k in d) + for k in set(d1.keys()) | set(d2.keys()) + } + + + +def get_key_from_value(d, val): + keys = [k for k, v in d.items() if v == val] + if keys: + return keys[0] + return None + + +def toJSON(self): + return json.dumps(self, default=lambda o: o.__dict__, sort_keys=True, indent=4) + + +def del_file(fname): + os.remove(fname) + + + +def del_outputs(name, outpath): + + path = os.path.join(outpath, name) + + if os.path.exists(path): + shutil.rmtree(path) + + os.mkdir(path) + + +def process_time(func): + """ + for debugging purpose, delete @decorator + """ + + def inner(*args): + start_time = time.time() + func(*args) + print(str(func), "--- %s seconds ---" % (time.time() - start_time)) + + return inner + + +def print_time(start_time=None): + if start_time: + str = "--- %s seconds ---" % (time.time() - start_time) + return str + + else: + return time.time() + + diff --git a/tabulated.py b/src/exforparser/tabulated.py similarity index 99% rename from tabulated.py rename to src/exforparser/tabulated.py index 80873a4..02f7d1b 100644 --- a/tabulated.py +++ b/src/exforparser/tabulated.py @@ -12,19 +12,23 @@ import random import pandas as pd import logging -import math -import json -import sys -from collections import defaultdict + FORMATTER = logging.Formatter("%(asctime)s — %(name)s — %(levelname)s — %(message)s") logging.basicConfig(filename="tabulated.log", level=logging.DEBUG, filemode="w") from config import OUT_PATH -from utilities.utilities import dict_merge, del_outputs, print_time +from submodules.utilities.util import dict_merge, del_outputs, print_time from exparser import convert_exfor_to_json, write_dict_to_json from parser.list_x4files import list_entries_from_df, good_example_entries from parser.exfor_unit import unify_units + + +# initialize exfor_dictionary +from exfor_dictionary.exfor_dictionary import Diction +D = Diction() + + from tabulated.exfor_reaction_mt import ( sf_to_mf, sf3_dict, @@ -40,9 +44,6 @@ from tabulated.data_process import * from sql.creation import * -# initialize exfor_dictionary -from exfor_dictionary import Diction -D = Diction() # get heading list x_en_heads = D.get_incident_en_heads() @@ -973,8 +974,9 @@ def main(entnum): if __name__ == "__main__": - ent = list_entries_from_df() - entries = random.sample(ent, len(ent)) + # ent = list_entries_from_df() + # entries = random.sample(ent, len(ent)) + entries = good_example_entries start_time = print_time() logging.info(f"Start processing {start_time}") diff --git a/tabulated/MTall.dat b/src/exforparser/tabulated/MTall.dat similarity index 100% rename from tabulated/MTall.dat rename to src/exforparser/tabulated/MTall.dat diff --git a/tabulated/data_dir_files.py b/src/exforparser/tabulated/data_dir_files.py similarity index 100% rename from tabulated/data_dir_files.py rename to src/exforparser/tabulated/data_dir_files.py diff --git a/tabulated/data_locations.py b/src/exforparser/tabulated/data_locations.py similarity index 98% rename from tabulated/data_locations.py rename to src/exforparser/tabulated/data_locations.py index 2bb5268..8929207 100644 --- a/tabulated/data_locations.py +++ b/src/exforparser/tabulated/data_locations.py @@ -10,7 +10,7 @@ # #################################################################### from parser.exfor_data import get_colmun_indexes -from exfor_dictionary import Diction +from exfor_dictionary.exfor_dictionary import Diction d = Diction() ## get possible heading list diff --git a/tabulated/data_process.py b/src/exforparser/tabulated/data_process.py similarity index 99% rename from tabulated/data_process.py rename to src/exforparser/tabulated/data_process.py index 83e4e07..2fec498 100644 --- a/tabulated/data_process.py +++ b/src/exforparser/tabulated/data_process.py @@ -17,7 +17,7 @@ from submodules.utilities.elem import ztoelem from tabulated.data_locations import * from tabulated.exfor_reaction_mt import get_mf, get_mt, e_lvl_to_mt50 -from ripl3_json.ripl3_descretelevel import RIPL_Level +from ripl3_json.ripl3_discretelevel import RIPL_Level def limit_data_dict_by_locs(locs, data_dict): diff --git a/tabulated/data_write.py b/src/exforparser/tabulated/data_write.py similarity index 99% rename from tabulated/data_write.py rename to src/exforparser/tabulated/data_write.py index d2480d3..7d997a1 100644 --- a/tabulated/data_write.py +++ b/src/exforparser/tabulated/data_write.py @@ -14,7 +14,7 @@ from contextlib import redirect_stdout from .data_dir_files import target_reformat -from exfor_dictionary import Diction +from exfor_dictionary.exfor_dictionary import Diction D = Diction() def bib_table(id, main_bib_dict, react_dict, mfmt, df): diff --git a/tabulated/exfor_reaction_mt.py b/src/exforparser/tabulated/exfor_reaction_mt.py similarity index 100% rename from tabulated/exfor_reaction_mt.py rename to src/exforparser/tabulated/exfor_reaction_mt.py diff --git a/tabulated/mf3.json b/src/exforparser/tabulated/mf3.json similarity index 100% rename from tabulated/mf3.json rename to src/exforparser/tabulated/mf3.json diff --git a/tabulated/mt.json b/src/exforparser/tabulated/mt.json similarity index 100% rename from tabulated/mt.json rename to src/exforparser/tabulated/mt.json diff --git a/submodules b/submodules deleted file mode 160000 index 6401183..0000000 --- a/submodules +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 64011838b02dbeb752b857aec9dd8fdc589e6f92