moved some pickles to exfor_dictionary

MANIFEST.in

@@ -1,3 +1,4 @@
-include src/tabulated/MTall.dat
-include src/tabulated/*.json
-include src/submodules/utilities/*.txt
+include src/exforparser/tabulated/MTall.dat
+include src/exforparsertabulated/*.json
+include src/exforparser/submodules/utilities/*.txt
+include src/exforparser/pickles/*.pickle

pyproject.toml

@@ -36,11 +36,11 @@ include-package-data = true
 
 [tool.setuptools.package-data]
 "exforparser.tabulated" = ["*.dat", "*.json"]
+"exforparser.pickles" = ["*.pickle"]
 "submodules.utilities" = ["*.txt"]
 
 
 
-
 [project.urls]
 Homepage = "https://github.com/shinokumura/exforparser"
 

src/exforparser/config.py

@@ -10,28 +10,41 @@
 #
 ####################################################################
 import os
-import sqlalchemy as db
-from sqlalchemy.orm import sessionmaker
+import site
+
+from sqlalchemy import create_engine
+from sqlalchemy.orm import scoped_session, sessionmaker
 
 DEVENV = True
 
 if DEVENV:
     DATA_DIR = "/Users/okumuras/Documents/nucleardata/EXFOR/"
+    OUT_PATH = "/Users/okumuras/Desktop/"
+    EXFOR_PARSER = "./"
 
 else:
     DATA_DIR = "/srv/data/dataexplorer2/"
+    OUT_PATH = "/srv/data/dataexplorer2/out/"
+
+    from importlib.resources import files
+    EXFOR_PARSER = files("exforparser")
 
 
-EXFOR_DB = os.path.join(DATA_DIR, "exfor_tmp.sqlite")
 EXFOR_MASTER_REPO_PATH = os.path.join(DATA_DIR, "exfor_master")
+EXFOR_DB = os.path.join(DATA_DIR, "exfor_tmp.sqlite")
+# print(EXFOR_DB)
+
 
-""" Pickle path of list of EXFOR master files made by parser.list_x4files.py"""
-ENTRY_INDEX_PICKLE = "pickles/entry.pickle"
+""" Pickle path of list of EXFOR master files made by parser.list_x4files.py """
+ENTRY_INDEX_PICKLE = os.path.join( EXFOR_PARSER, "pickles/entry.pickle" )
 
 
-OUT_PATH = DATA_DIR +  "../../../Desktop/"
+""" Pickle path of list of EXFOR master files made by parser.list_x4files.py """
+SITE_DIR = site.getsitepackages()[0]
+INSTITUTE_PICKLE = os.path.join( SITE_DIR, "exfor_dictionary", "pickles/institute.pickle" )
 
 
 """ SQL database """
-engine = db.create_engine("sqlite:///" + EXFOR_DB)
-session = sessionmaker(autocommit=False, autoflush=True, bind=engine)
+engine = create_engine("sqlite:///" + EXFOR_DB)
+Session = scoped_session(sessionmaker(autocommit=False, autoflush=False, bind=engine))
+session = Session()

src/exforparser/parser/exfor_reaction.py

@@ -23,6 +23,7 @@ def split_sf(sf49):
         sf7 = len(sf49) > 3 and sf49[3] or None
         sf8 = len(sf49) > 4 and sf49[4] or None
         sf9 = len(sf49) > 5 and sf49[5] or None
+
     return {
         "sf4": sf4,
         "sf5": sf5,
@@ -99,7 +100,7 @@ def parse_reaction(reaction_field) -> dict:
             except:
                 reaction_info = {
                     "x4_code": x4_code,
-                    "children": {"code": b[0], "type": None},
+                    "children": [ {"code": b[0], "type": None} ],
                     "type": "?",
                     "free_text": free_text,
                 }

src/exforparser/parser/list_x4files.py

@@ -175,8 +175,6 @@ def compare_hash(df):
     "14605",  # Istitute two rows with free text
     # "G0509",  # facility has pointer
     "O0191",
-    "12898",
-    "11945",
     "22436",
     "D0047",  # ((92-U-CMP(A,N),,PY,,TT)=(6-C-CMP(A,N),,PY,,TT))
     "12898",  # pointer ratio normal, from Amanda, ((23-V-51(N,P)22-TI-51,,SIG)/(92-U-238(N,F),,SIG))
@@ -203,7 +201,6 @@ def compare_hash(df):
     "21107",
     "13500",
     "23313",
-    "10377",
     "20010",  # REFERENCE  ((R,KFK-1000,1968)=(R,EUR-3963E,1968)= + \n (R,EANDC(E)-111,1968))
     "21902",
     "30328",

src/exforparser/sql/models.py

@@ -8,9 +8,10 @@
 
 import sqlalchemy as db
 from sqlalchemy.ext.declarative import declarative_base
+from config import engine
 
-Base = declarative_base()
 
+Base = declarative_base()
 
 class Exfor_Bib(Base):
     __tablename__ = "exfor_bib"
@@ -63,6 +64,7 @@ class Exfor_Indexes(Base):
     sf8 = db.Column(db.String)
     sf9 = db.Column(db.String)
     x4_code = db.Column(db.String)
+    mf = db.Column(db.Integer)
     mt = db.Column(db.Integer)
 
 
@@ -85,10 +87,13 @@ class Exfor_Data(Base):
     de_out = db.Column(db.Float)
     angle = db.Column(db.Float)
     dangle = db.Column(db.Float)
+    mf = db.Column(db.Integer)
     mt = db.Column(db.Integer)
 
+Base.metadata.create_all(bind=engine)
 
-if __name__ == "__main__":
-    from settings import engine
 
-    Base.metadata.create_all(bind=engine)
+if __name__ == "__main__":
+    # from config import engine
+    # Base.metadata.create_all(bind=engine)
+    pass

src/exforparser/sql/queries.py

@@ -0,0 +1,68 @@
+
+import sqlalchemy as db
+from sqlalchemy import insert
+import pandas as pd
+
+
+# from sql.creation import exfor_bib, exfor_reactions, exfor_index, exfor_data 
+from sql.models import Exfor_Bib, Exfor_Reactions, Exfor_Indexes, Exfor_Data
+from config import engine, session
+
+connection = engine.connect()
+metadata = db.MetaData()
+
+
+def insert_df_to_data(df):
+    df2 = df.astype(object).where(pd.notnull(df), None)
+    # for record in df2.to_dict(orient="records"):
+    #     query = db.insert(exfor_data).values(record)
+    #     ResultProxy = connection.execute(query)
+
+    df2.to_sql(
+        "exfor_data",
+        connection,
+        index=False,
+        if_exists="append",
+    )
+
+
+def insert_bib(dictlist):
+    # connection.execute(exfor_bib.insert(), dictlist)
+
+    data = Exfor_Bib(**dictlist)
+    session.add(data)
+    session.commit()
+
+
+
+def insert_reaction(dictlist):
+    # connection.execute(exfor_reactions.insert(), dictlist)
+    for dict in dictlist:
+        data = Exfor_Reactions(**dict)
+        session.add(data)
+    session.commit()
+
+
+
+def insert_reaction_index(dictlist):
+    # connection.execute(exfor_index.insert(), dictlist)
+    for dict in dictlist:
+        data = Exfor_Indexes(**dict)
+        session.add(data)
+    session.commit()
+
+
+def show():
+    results = connection.execute(db.select([exfor_data])).fetchall()
+    df = pd.DataFrame(results)
+    df.columns = results[0].keys()
+    df.head(4)
+
+
+
+def drop_tables():
+    for tbl in reversed(metadata.sorted_tables):
+        engine.execute(tbl.delete())
+
+
+