update package setup

.github/workflows/auto_update.yml

@@ -56,9 +56,7 @@ jobs:
           EMAIL_RECIPIENT: ${{ secrets.EMAIL_RECIPIENT }}
         run: |
           cd ${GITHUB_WORKSPACE}
-          git clone https://github.com/IAEA-NDS/exfor_dictionary.git
           git clone https://github.com/IAEA-NDS/exfor_master.git
-          git clone https://github.com/IAEA-NDS/exfor_json.git
           python exparser.py
 
 

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "submodules"]
+	path = submodules
+	url = https://github.com/shinokumura/exparser-submodule.git

VERSION

@@ -0,0 +1 @@
+1.0.0beta

config.py

@@ -9,8 +9,8 @@
 # Contact:    [email protected]
 #
 ####################################################################
-import sys
-import os
+import sqlalchemy as db
+from sqlalchemy.orm import sessionmaker
 
 DEVENV = True
 
@@ -25,47 +25,20 @@
 
 
 
-EXFOR_DB = DATA_DIR + "exfor_tmp.sqlite"
-ENDFTAB_DB = DATA_DIR + "endftables.sqlite"
-
-
-## Package locations
-EXFOR_PARSER = "exforparser/"
-EXFOR_DICTIONARY = "exfor_dictionary/"
-RIPL3 = "ripl3_json/"
-
-
-sys.path.append(os.path.join(MODULES_DIR, EXFOR_DICTIONARY))
-sys.path.append(os.path.join(MODULES_DIR, RIPL3))
-
-
-
-""" pickel file path """
-PICKLE_PATH = os.path.join(MODULES_DIR, EXFOR_DICTIONARY, "pickles/")
-
+EXFOR_DB = DATA_DIR + "exfor.sqlite"
 
 
 """ Pickle path of list of EXFOR master files made by parser.list_x4files.py"""
 ENTRY_INDEX_PICKLE = "pickles/entry.pickle"
 
 
-
-""" EXFOR master file path """
-EXFOR_MASTER_REPO_PATH = os.path.join(DATA_DIR, "exfor_master/")
-EXFOR_ALL_PATH = os.path.join(EXFOR_MASTER_REPO_PATH,  "exforall/")
-
-
 TO_JSON = True
 POST_DB = False
 
 
 OUT_PATH = DATA_DIR +  "../../../Desktop/"
 
 
-import sqlalchemy as db
-from sqlalchemy.orm import sessionmaker
-
-
 """ SQL database """
 engine = db.create_engine("sqlite:///" + EXFOR_DB)
 session = sessionmaker(autocommit=False, autoflush=True, bind=engine)

exparser.py

@@ -21,11 +21,10 @@
 
 
 from config import OUT_PATH
-from utilities.utilities import del_outputs, print_time, get_entry_update_date
+from submodules.utilities.util import del_outputs, print_time, get_entry_update_date
 from parser.list_x4files import list_entries_from_df
 from parser.exfor_entry import Entry
 from parser.exfor_subentry import Subentry
-#from mongodb import post_one_mongodb, post_many_mongodb
 
 
 ## get update data from git commit and store info to Python dictionary
@@ -97,15 +96,13 @@ def convert_exfor_to_json(entnum=None):
 
 
 def main(entnum):
-    # del_outputs(OUT_PATH + "json/")
 
     start_time = print_time()
     logging.info(f"Start processing {start_time}")
 
     try:
         entry_json = convert_exfor_to_json(entnum)
         write_dict_to_json(entnum, entry_json)
-        # post_one_mongodb("exfor_json", entry_json)
 
     except:
         logging.error(f"ERROR: at ENTRY: {entnum}")
@@ -114,10 +111,9 @@ def main(entnum):
 
 
 if __name__ == "__main__":
-    # ent = list_entries_from_df()
+    ent = list_entries_from_df()
     # entries = random.sample(ent, len(ent))
-    # entries = ent
-    entries=["32617", "40016", "30936", "C1823", "E2286"] #, "12240", "41185", "41102", "30010", "11210"]
+    entries = ent
     for entnum in entries:
         print(entnum)
         main(entnum)

parser/exfor_bib.py

@@ -16,7 +16,7 @@
 
 
 from .exfor_field import *
-from utilities.utilities import flatten_list
+from submodules.utilities.util import flatten_list
 from .exfor_block import get_identifier_details
 
 

parser/exfor_data.py

@@ -9,17 +9,24 @@
 # Contact:    [email protected]
 #
 ####################################################################
-import sys
 from pyparsing import *
 
 from .exfor_field import data_header
-from utilities.utilities import corr, flatten_list
-from utilities.elem import ztoelem, numtoisomer
+from submodules.utilities.util import flatten_list
+from submodules.utilities.elem import ztoelem
 
 # from dictionary.exfor_dictionary import Diction
 # sys.path.append("../exfor_dictionary/")
 from exfor_dictionary import Diction
 
+def corr(invalue):
+    if re.search(r"\d|\.[+]\d", invalue):
+        # invalue = invalue.replace("+", "E+")
+        invalue = re.sub(r"(\d|\.)([+])(\d)", r"\1E+\3", invalue)
+    if re.search(r"\d|\.[-]\d", invalue):
+        # invalue = invalue.replace("-", "E-")
+        invalue = re.sub(r"(\d|\.)([-])(\d)", r"\1E-\3", invalue)
+    return invalue
 
 def data_column_read(line):
     column = [0, 11, 22, 33, 44, 55]

requirements.txt

@@ -1,11 +1,8 @@
-requests
-dnspython
-matplotlib
-numpy
-pandas
-plotly
-pymongo==4.1.1
-pymongo[srv]
-ripl3_json @ https://github.com/shinokumura/ripl3_json.git
-exfor_master @ https://github.com/IAEA-NDS/exfor_master.git
-exfor_dictionary @ https://github.com/IAEA-NDS/exfor_dictionary.git
+common==0.1.2
+numpy==1.23.0
+pandas==2.0.3
+pyparsing==3.1.1
+setuptools==57.5.0
+SQLAlchemy==2.0.18
+exfor_dictionary @ git+https://github.com/IAEA-NDS/exfor_dictionary@main
+ripl3_json @ git+https://github.com/shinokumura/ripl3_json@main

setup.py

@@ -0,0 +1,33 @@
+from setuptools import setup, find_packages
+
+
+def read_requirements(file):
+    with open(file) as f:
+        return f.read().splitlines()
+
+def read_file(file):
+   with open(file) as f:
+        return f.read()
+
+version = read_file("VERSION")
+requirements = read_requirements("requirements.txt")
+
+setup(
+    name="exforparser",
+    description="EXFOR Parser",
+    packages=find_packages(exclude=["test"]), 
+    version=version,
+    author="Shin Okumura/IAEA-NDS",
+    author_email="[email protected]",
+    maintainer="IAEA-NDS",
+    maintainer_email="[email protected]",
+    license="GPL-2.0 license",
+    url="https://github.com/[email protected]/exforparser",
+    python_requires=">=3.8",
+    install_requires=requirements,
+    classifiers=[
+        "Programming Language :: Python :: 3",
+        "Operating System :: OS Independent",
+        "License :: OSI Approved :: GPL-2.0 license",
+    ],
+)

sql/creation.py

@@ -38,7 +38,7 @@ def drop_tables():
     "exfor_reactions",
     metadata,
     db.Column("entry_id", db.String(255), primary_key=True, index=True),
-    db.Column("entry", db.String(255)),
+    db.Column("entry", db.String(5)),
     db.Column("target", db.String(255), index=True),
     db.Column("projectile", db.String(255)),
     db.Column("process", db.String(255), index=True),
@@ -56,24 +56,24 @@ def drop_tables():
     "exfor_index",
     metadata,
     db.Column("id", db.Integer(), primary_key=True),
-    db.Column("entry_id", db.String(255), index=True),
-    db.Column("entry", db.String(255)),
-    db.Column("target", db.String(255), index=True),
-    db.Column("projectile", db.String(255)),
-    db.Column("process", db.String(255), index=True),
-    db.Column("sf4", db.String(255), index=True),
-    db.Column("residual", db.String(255), index=True),
+    db.Column("entry_id", db.String(20), index=True),
+    db.Column("entry", db.String(5)),
+    db.Column("target", db.String(20), index=True),
+    db.Column("projectile", db.String(20)),
+    db.Column("process", db.String(40), index=True),
+    db.Column("sf4", db.String(20), index=True),
+    db.Column("residual", db.String(20), index=True),
     db.Column("level_num", db.Integer(), index=True),
     db.Column("e_out", db.Float(), index=True),
     db.Column("e_inc_min", db.Float(), index=True),
     db.Column("e_inc_max", db.Float(), index=True),
     db.Column("points", db.Integer()),
     db.Column("arbitrary_data", db.Boolean(), index=True),
-    db.Column("sf5", db.String(255)),
-    db.Column("sf6", db.String(255), index=True),
-    db.Column("sf7", db.String(255)),
-    db.Column("sf8", db.String(255)),
-    db.Column("sf9", db.String(255)),
+    db.Column("sf5", db.String(10)),
+    db.Column("sf6", db.String(10), index=True),
+    db.Column("sf7", db.String(10)),
+    db.Column("sf8", db.String(10)),
+    db.Column("sf9", db.String(10)),
     db.Column("x4_code", db.String(255)),
     db.Column("mf", db.Integer(), index=True),
     db.Column("mt", db.Integer(), index=True),
@@ -84,13 +84,13 @@ def drop_tables():
     "exfor_data",
     metadata,
     db.Column("id", db.Integer(), primary_key=True),
-    db.Column("entry_id", db.String(255), index=True),
+    db.Column("entry_id", db.String(20), index=True),
     db.Column("en_inc", db.Float()),
     db.Column("den_inc", db.Float()),
     db.Column("charge", db.Float()),
-    db.Column("mass", db.Float()),
-    db.Column("isomer", db.String(255)),
-    db.Column("residual", db.String(255), index=True),
+    db.Column("mass", db.String(20)),   # must be string for CMP cases
+    db.Column("isomer", db.String(20)),
+    db.Column("residual", db.String(20), index=True),
     db.Column("level_num", db.Integer(), index=True),
     db.Column("data", db.Float()),
     db.Column("ddata", db.Float()),
@@ -110,13 +110,12 @@ def drop_tables():
 
 def insert_df_to_data(df):
     df2 = df.astype(object).where(pd.notnull(df), None)
-    # print(df2.to_dict(orient='records'))
-    # list = []
     for record in df2.to_dict(orient="records"):
         query = db.insert(exfor_data).values(record)
         ResultProxy = connection.execute(query)
 
 
+
 def insert_bib(dictlist):
     connection.execute(exfor_bib.insert(), dictlist)
 

sql/models.py

@@ -56,7 +56,7 @@ class Exfor_Indexes(Base):
     e_inc_min = db.Column(db.Float)
     e_inc_max = db.Column(db.Float)
     points = db.Column(db.Integer)
-    # arbitrary_data = db.Column(db.Boolean)
+    arbitrary_data = db.Column(db.Boolean)
     sf5 = db.Column(db.String)
     sf6 = db.Column(db.String)
     sf7 = db.Column(db.String)
@@ -73,14 +73,14 @@ class Exfor_Data(Base):
     en_inc = db.Column(db.String)
     den_inc = db.Column(db.String)
     charge = db.Column(db.Float)
-    mass = db.Column(db.Float)
+    mass = db.Column(db.String)
     isomer = db.Column(db.String)
     residual = db.Column(db.String)
     level_num = db.Column(db.Integer)
     data = db.Column(db.Float)
     ddata = db.Column(db.Float)
-    # arbitrary_data = db.Column(db.Boolean)
-    # arbitrary_ddata = db.Column(db.Boolean)
+    arbitrary_data = db.Column(db.Boolean)
+    arbitrary_ddata = db.Column(db.Boolean)
     e_out = db.Column(db.Float)
     de_out = db.Column(db.Float)
     angle = db.Column(db.Float)

sql/queries.py

@@ -50,7 +50,8 @@ def reaction_query_simple(type, elem, mass, reaction, branch):
 
     queries = [Exfor_Indexes.target == target, 
                Exfor_Indexes.process == reaction.upper(),
-               Exfor_Indexes.sf6 == type.upper()]
+               Exfor_Indexes.sf6 == type.upper(),
+               Exfor_Indexes.arbitrary_data == False]
 
 
 
@@ -81,7 +82,8 @@ def reaction_query(type, elem, mass, reaction, branch=None, rp_elem=None, rp_mas
     reac = None
     target = elemtoz_nz(elem) + "-" + elem.upper() + "-" + mass
 
-    queries = [Exfor_Indexes.target == target]
+    queries = [Exfor_Indexes.target == target,
+               Exfor_Indexes.arbitrary_data == False]
 
     if branch:
         if branch == "PAR":
@@ -294,6 +296,7 @@ def reaction_query_fy(type, elem, mass, reaction, branch, mesurement_opt_fy, ene
             Exfor_Indexes.sf6 == type.upper(),
             Exfor_Indexes.e_inc_min >= lower,
             Exfor_Indexes.e_inc_max <= upper,
+            Exfor_Indexes.arbitrary_data == False,
             (Exfor_Indexes.sf4 == "MASS" if mesurement_opt_fy=="A" else Exfor_Indexes.sf4 == "ELEM" if mesurement_opt_fy=="Z" else Exfor_Indexes.sf4.isnot(None) ),
         ).all()
 
@@ -326,7 +329,8 @@ def reaction_query_fission(type, elem, mass, reaction, branch, energy_range):
     target = elemtoz_nz(elem) + "-" + elem.upper() + "-" + mass
 
     queries = [Exfor_Indexes.target == target, 
-               Exfor_Indexes.process == reaction.upper(),]
+               Exfor_Indexes.process == reaction.upper(),
+               Exfor_Indexes.arbitrary_data == False]
 
     if branch == "nu_n":
         sf5 = ["PR"]