Lots of debugging with simple_test.

- Better track of progress by calling `recover` more often.
- queue ID checking had an issue in some cases.
- Less useless messages.
- Other small fixes here and there.

Author: bbusemeyer

TODO.md

@@ -5,3 +5,4 @@
 - [ ] Comparable defaults between PySCF and Crystal are not the same.
 - [ ] Cannot yet do different paths with QMC and crystal because the basis and the orb file needs to be in the same place.
   - May need to have the crystal manager export the results to the same directory (this means there are redundant files then).
+- [ ] test case where the trial function gets called before the function itself.

autopyscf.py

@@ -223,7 +223,7 @@ def __init__(self,options={}):
     self.direct_scf_tol=1e-7
     self.pyscf_path=[]
     self.spin=0
-    self.gs=[4,4,4]
+    self.gmesh=[4,4,4]
     self.xyz=""
     self.latticevec=""
     self.kpts=[2,2,2]
@@ -331,7 +331,7 @@ def pyscf_input(self,fname,chkfile):
     # The cell/molecule
     outlines+=[
         "mol=gto.M(verbose=4,",
-        "gs="+str(self.gs)+",",
+        "mesh="+str(self.gmesh)+",",
         "atom='''"+self.xyz+"''',",
         "a='''"+str(self.latticevec) +"''',",
         "basis=basis,",

autorunner.py

@@ -360,7 +360,7 @@ def submit(self,jobname=None):
       f.write('\n'.join(qsublines))
     try: 
       result = sub.check_output("qsub %s"%(qsubfile),shell=True)
-      self.queueid.append(result.decode().split()[0])
+      self.queueid.append(result.decode().split()[0].split('.')[0])
       print(self.__class__.__name__,": Submitted as %s"%self.queueid)
     except sub.CalledProcessError:
       print(self.__class__.__name__,": Error submitting job. Check queue settings.")

crystal2qmc.py

@@ -758,7 +758,7 @@ def convert_crystal(
 
   #  All the files that will get produced.
   files={
-      'kpoints':[],
+      'kpoints':{},
       'basis':base+".basis",
       'jastrow2':base+".jast2",
       'orbplot':{},
@@ -772,7 +772,7 @@ def convert_crystal(
   for kpt in eigsys['kpt_coords']:
     if eigsys['ikpt_iscmpx'][kpt] and kset=='real': continue
     kidx=eigsys['kpt_index'][kpt]
-    files['kpoints'].append(kpt)
+    files['kpoints'][kidx]=kpt
     files['orbplot'][kidx]="%s_%d.plot"%(base,kidx)
     files['slater'][kidx]="%s_%d.slater"%(base,kidx)
     files['orb'][kidx]="%s_%d.orb"%(base,kidx)

dmc.py

@@ -67,7 +67,8 @@ def qwalk_input(self,infile):
 
       with open(infile,'w') as f:
         f.write('\n'.join(outlines))
-    self.completed=True
+
+      self.completed=True
 
 
 ####################################################

linear.py

@@ -66,8 +66,7 @@ def qwalk_input(self,infile):
         f.write("total_fit %i \n"%self.total_fit)
         f.write("}\n")
         f.write(self.trialfunc)
-
-    self.completed=True
+      self.completed=True
 
 
 ####################################################

manager.py

@@ -1,3 +1,17 @@
+''' Manager classes handle the file maintainance of various runs.
+
+They:
+  - Call write, run, and read in the appropriate order.
+  - Keep a record of progress on disk.
+  - Hold a list of all file paths together in one place.
+  - Report progress to the user.
+
+All choices revolving around file names should be handled here.
+
+Tips on avoiding bugs with the write/run/read cycle:
+  - Every time data is updated, write the updates to the disk.
+  - Every time the data is used, update the Manager from the disk.
+'''
 import crystal
 import propertiesreader
 import os
@@ -122,7 +136,7 @@ def __init__(self,writer,runner,creader=None,name='crystal_run',path=None,
 
     self.logname="%s@%s"%(self.__class__.__name__,self.path+self.name)
 
-    print(self.logname,": initializing")
+    #print(self.logname,": initializing")
 
     # Handle reader and runner defaults.
     self.writer=writer
@@ -163,7 +177,7 @@ def __init__(self,writer,runner,creader=None,name='crystal_run',path=None,
 
     # Handle old results if present.
     if os.path.exists(self.path+self.pickle):
-      print(self.logname,": rebooting old manager.")
+      #print(self.logname,": rebooting old manager.")
       old=pkl.load(open(self.path+self.pickle,'rb'))
       self.recover(old)
 
@@ -188,7 +202,7 @@ def recover(self,other):
     update_attributes(copyto=self.runner,copyfrom=other.runner,
         skip_keys=['queue','walltime','np','nn','jobname'],
         take_keys=['queueid'])
-    update_attributes(copyto=self.runner,copyfrom=other.runner,
+    update_attributes(copyto=self.prunner,copyfrom=other.prunner,
         skip_keys=['queue','walltime','np','nn','jobname'],
         take_keys=['queueid'])
 
@@ -209,6 +223,7 @@ def recover(self,other):
   #----------------------------------------
   def nextstep(self):
     ''' Determine and perform the next step in the calculation.'''
+    self.recover(pkl.load(open(self.path+self.pickle,'rb')))
 
     print(self.logname,": next step.")
     cwd=os.getcwd()
@@ -321,6 +336,9 @@ def write_summary(self):
   #------------------------------------------------
   def export_qwalk(self):
     ''' Export QWalk input files into current directory.'''
+    self.recover(pkl.load(open(self.path+self.pickle,'rb')))
+
+    ready=False
     if len(self.qwfiles['slater'])==0:
       self.nextstep()
 
@@ -336,6 +354,7 @@ def export_qwalk(self):
       status=resolve_status(self.prunner,self.preader,self.propoutfn)
       print(self.logname,": properties status= %s"%(status))
       if status=='not_started':
+        ready=False
         self.prunner.add_command("cp %s INPUT"%self.propinpfn)
         self.prunner.add_task("Pproperties &> %s"%self.propoutfn)
 
@@ -348,14 +367,18 @@ def export_qwalk(self):
         self.preader.collect(self.propoutfn)
 
       if self.preader.completed:
+        ready=True
         self.qwfiles=crystal2qmc.convert_crystal(base=self.name,propoutfn=self.propoutfn)
         print(self.logname,": crystal converted to QWalk input.")
 
       os.chdir(cwd)
+    else:
+      ready=True
+
     with open(self.path+self.pickle,'wb') as outf:
       pkl.dump(self,outf)
 
-    return True
+    return ready
 
   #----------------------------------------
   def status(self):
@@ -388,7 +411,7 @@ def __init__(self,writer,reader=None,runner=None,name='psycf_run',path=None,bund
 
     self.logname="%s@%s"%(self.__class__.__name__,self.path+self.name)
 
-    print(self.logname,": initializing")
+    #print(self.logname,": initializing")
 
     self.writer=writer
     if reader is not None: self.reader=reader
@@ -417,24 +440,33 @@ def __init__(self,writer,reader=None,runner=None,name='psycf_run',path=None,bund
     if os.path.exists(self.path+self.pickle):
       print(self.logname,": rebooting old manager.")
       old=pkl.load(open(self.path+self.pickle,'rb'))
-      if False: #not self.is_consistent(old): TODO check consistency.
-        raise NotImplementedError("Handling updated input files is not implemented yet.")
-      else:
-        self.__dict__=old.__dict__
+      self.recover(old)
 
     # Update the file.
     if not os.path.exists(self.path): os.mkdir(self.path)
     with open(self.path+self.pickle,'wb') as outf:
       pkl.dump(self,outf)
 
   #------------------------------------------------
-  def update_options(self,other):
+  def recover(self,other):
     ''' Safe copy options from other to self. '''
-    updated=update_attributes(old=self.writer,new=other.writer,
-        safe_keys=['max_cycle'],
-        skip_keys=['completed','chkfile','dm_generator'])
-    if updated:
-      self.writer.completed=False
+    # Practically speaking, the run will preserve old `take_keys` and allow new changes to `skip_keys`.
+    # This is because you are taking the attributes from the older instance, and copying into the new instance.
+    updated=update_attributes(copyto=self,copyfrom=other,
+        skip_keys=['writer','runner','reader', 'path','logname','name','max_restarts','bundle'],
+        take_keys=['restarts','completed','qwfiles'])
+
+    update_attributes(copyto=self.runner,copyfrom=other.runner,
+        skip_keys=['queue','walltime','np','nn','jobname'],
+        take_keys=['queueid'])
+
+    update_attributes(copyto=self.reader,copyfrom=other.reader,
+        skip_keys=[],
+        take_keys=['completed','output'])
+
+    updated=update_attributes(copyto=self.writer,copyfrom=other.writer,
+        skip_keys=['max_cycle'],
+        take_keys=['completed','dm_generator'])
 
     # Update the file.
     with open(self.path+self.pickle,'wb') as outf:
@@ -443,6 +475,9 @@ def update_options(self,other):
   #------------------------------------------------
   def nextstep(self):
     ''' Determine and perform the next step in the calculation.'''
+    # Recover old data.
+    self.recover(pkl.load(open(self.path+self.pickle,'rb')))
+
     print(self.logname,": next step.")
     cwd=os.getcwd()
     os.chdir(self.path)
@@ -495,6 +530,9 @@ def update_queueid(self,qid):
   #------------------------------------------------
   def export_qwalk(self):
     ''' Export QWalk input files into current directory.'''
+    # Recover old data.
+    self.recover(pkl.load(open(self.path+self.pickle,'rb')))
+
     if len(self.qwfiles['slater'])==0:
       self.nextstep()
       if not self.completed:
@@ -546,7 +584,7 @@ def __init__(self,writer,reader,runner=None,trialfunc=None,
 
     self.logname="%s@%s"%(self.__class__.__name__,self.path+self.name)
 
-    print(self.logname,": initializing")
+    #print(self.logname,": initializing")
 
     self.writer=writer
     self.reader=reader
@@ -604,42 +642,23 @@ def recover(self,other):
     if updated:
       self.writer.completed=False
 
-  #------------------------------------------------
-  def update_options(self,other):
-    ''' Safe copy options from other to self. 
-    
-    Args: 
-      other (QWRunManager): New object to copy attributes from.
-    '''
-
-    update_attributes(old=self.runner,new=other.runner,
-        safe_keys=['queue','walltime','np','nn','jobname','prefix','postfix'],
-        skip_keys=['queueid'])
-
-    # trialfunc gets updated as the manager generating it finishes its calculation.
-    update_attributes(old=self.writer,new=other.writer,
-        skip_keys=['completed','trialfunc'])
-
-    # Update the file.
-    with open(self.path+self.pickle,'wb') as outf:
-      pkl.dump(self,outf)
-    
   #------------------------------------------------
   def nextstep(self):
     ''' Perform next step in calculation. trialfunc managers are updated if they aren't completed yet.'''
+    # Recover old data.
+    self.recover(pkl.load(open(self.path+self.pickle,'rb')))
 
     print(self.logname,": next step.")
 
     # Check dependency is completed first.
     if self.writer.trialfunc=='':
+      print(self.logname,": checking trial function.")
       self.writer.trialfunc=self.trialfunc.export(self.path)
 
     # Work on this job.
     cwd=os.getcwd()
     os.chdir(self.path)
 
-    dir(self.reader)
-
     # Write the input file.
     if not self.writer.completed:
       self.writer.qwalk_input(self.infile)
@@ -662,8 +681,6 @@ def nextstep(self):
         self.runner.add_task(exestr)
     elif status=='done':
       self.completed=True
-    else:
-      print(self.logname,": %s status= %s"%(self.name,status))
 
     # Ready for bundler or else just submit the jobs as needed.
     if self.bundle:
@@ -717,6 +734,9 @@ def export_qwalk(self):
     ''' Extract jastrow from the optimization run and return that file name.'''
     # Theoretically more than just Jastrow can be provided, but practically that's the only type of wavefunction we tend to export.
 
+    # Recover old data.
+    self.recover(pkl.load(open(self.path+self.pickle,'rb')))
+
     assert self.writer.qmc_abr!='dmc',"DMC doesn't provide a wave function."
 
     if self.qwfiles['wfout']=='':

tests/simple_test/simple_test.py

@@ -46,7 +46,7 @@ def h2_tests():
       name='scf',
       path='h2stretch',
       writer=stwriter,
-      runner=PySCFRunnerPBS(nn=1,np=2,ppath=sys.path),
+      runner=PySCFRunnerPBS(nn=1,walltime='0:05:00',np=16,queue='secondary',ppath=sys.path),
     )
 
   return [eqman,stman]
@@ -82,7 +82,7 @@ def si_crystal_test():
       runner=RunnerPBS(
           nn=1,queue='secondary',walltime='0:10:00'
         ),
-      trialfunc=SlaterJastrow(cman,kpoint=(0,0,0))
+      trialfunc=SlaterJastrow(cman,kpoint=0)
     )
 
   jobs.append(var)
@@ -95,21 +95,22 @@ def si_crystal_test():
       runner=RunnerPBS(
           nn=1,queue='secondary',walltime='1:00:00'
         ),
-      trialfunc=SlaterJastrow(slatman=cman,jastman=var,kpoint=(0,0,0))
+      trialfunc=SlaterJastrow(slatman=cman,jastman=var,kpoint=0)
     )
 
   jobs.append(lin)
 
-  for kpt in cman.qwfiles['kpts']:
+  for kidx in cman.qwfiles['kpoints']:
+    kpt=cman.qwfiles['kpoints'][kidx]
     dmc=QWalkManager(
-        name='dmc_%d%d%d'%kpt,
+        name='dmc_%d'%kidx,
         path=cman.path,
         writer=DMCWriter(),
         reader=DMCReader(),
         runner=RunnerPBS(
             nn=1,queue='secondary',walltime='1:00:00',
           ),
-        trialfunc=SlaterJastrow(slatman=cman,jastman=lin,kpoint=kpt)
+        trialfunc=SlaterJastrow(slatman=cman,jastman=lin,kpoint=kidx)
       )
     jobs.append(dmc)
 
@@ -129,7 +130,8 @@ def si_pyscf_test():
       runner=PySCFRunnerPBS(
           queue='secondary',
           nn=1,
-          walltime='0:20:00',
+          np=16,
+          walltime='4:00:00',
           ppath=sys.path
         )
     )
@@ -205,29 +207,30 @@ def mno_test():
       trialfunc=SlaterJastrow(slatman=cman,jastman=var,kpoint=(0,0,0))
     )
 
-  for ki,kpt in enumerate(cman.qwfiles['kpoints']):
+  for kidx in cman.qwfiles['kpoints']:
+    kpt=cman.qwfiles['kpoints'][kidx]
     if not sum(np.array(kpt)%(np.array(cman.kmesh)/2))<1e-14:
       print("complex kpt %d%d%d skipped"%kpt)
       continue
     dmc=QWalkManager(
-        name='dmc_%d%d%d'%kpt,
+        name='dmc_%d'%kidx,
         path=cman.path,
         writer=DMCWriter(),
         reader=DMCReader(),
         runner=RunnerPBS(
             nn=1,queue='secondary',walltime='1:00:00',
           ),
-        trialfunc=SlaterJastrow(slatman=cman,jastman=lin,kpoint=kpt)
+        trialfunc=SlaterJastrow(slatman=cman,jastman=lin,kpoint=kidx)
       )
     jobs.append(dmc)
   return jobs
 
 def run_tests():
   ''' Choose which tests to run and execute `nextstep()`.'''
   jobs=[]
-  #jobs+=h2_tests()
-  #jobs+=si_crystal_test()
-  #jobs+=si_pyscf_test()
+  jobs+=h2_tests()
+  jobs+=si_crystal_test()
+  jobs+=si_pyscf_test()
   jobs+=mno_test()
 
   for job in jobs: