Making progress testing parallel db with ssh support

APLProcess.dyalog

@@ -24,7 +24,6 @@
       :Trap 6
          source←⍎'(⊃⊃⎕CLASS ⎕THIS).SALT_Data.SourceFile' ⍝ ⍎ works around a bug
       :Else
-          source←(⊃⊃⎕CLASS ⎕THIS).SALT_Data.SourceFile
           :If 0=⍴source←{((⊃¨⍵)⍳⊃⊃⎕CLASS ⎕THIS)⊃⍵,⊂''}5177⌶⍬
               source←⎕WSID
           :Else ⋄ source←4⊃source
@@ -71,7 +70,8 @@
           psi.WindowStyle←Diagnostics.ProcessWindowStyle.Minimized
           Proc←Diagnostics.Process.Start psi
       :Else ⍝ Unix     
-          :If IsSsh←326=⎕DR Exe
+          :If IsSsh←326=⎕DR Exe 
+              ∘∘∘
               Proc←SshProc Exe
           :Else
           pid←_SH'{ ',args,' ',Exe,' +s ',ws,' -c APLppid=',(⍕GetCurrentProcessId),' </dev/null >/dev/null 2>&1 & } ; echo $!'

BootServers.dyalog

@@ -2,6 +2,8 @@
 ⍝ Start a vecdb server process if VECDBSRV="config.json" PORT=nnnn
 ⍝         vecdb slave process if  VECDBSLAVE="file" SHARDS="n" PORT=nnnn
 
+ ⎕←'Command Line:'
+ ⎕←2 ⎕NQ'.' 'GetCommandLine'
  getenv←{0=≢r←2 ⎕NQ'.' 'GetEnvironment'⍵:⍺ ⋄ r}
  getnum←{⊃2⊃⎕VFI ⍵}
  path←'file://',⊃⎕NPARTS ⎕WSID

TestVecdbSrv.dyalog

@@ -16,11 +16,6 @@
       :Trap 22 ⋄ {}#.vecdb.Delete folder ⋄ :EndTrap
       ⎕MKDIR folder   
 
-      :If 0=⎕NC '#.PASS'
-         ⎕←'Enter password from mkrom@Mortens-Macbook-Air: '
-         #.PASS←⍞
-      :EndIf
-
       ⍝ --- Create configuration file ---
 
       config←CreateBenchConfig folder,'config.json'
@@ -85,11 +80,11 @@
       ##.APLProcess.NewSshClient '192.168.17.129' 'mkrom' 'c:\docs\personal\macbook-air' cmd
     ∇
 
-    ∇ config←CreateBenchConfig filename;db;config;user;vecdbsrv;cmd;host;keyfile
+    ∇ config←CreateBenchConfig filename;db;config;user;vecdbsrv;cmd;host;keyfile;userid
      ⍝ 
-      cmd←'RIDE_INIT=SERVE::5678 /Applications/Dyalog-15.0.app/Contents/Resources/Dyalog/mapl'        
+      cmd←'RIDE_SPAWNED=1 RIDE_INIT=SERVE::5678 /Applications/Dyalog-15.0.app/Contents/Resources/Dyalog/mapl'        
       host←'Mortens-Macbook-Air' 
-      user←'mkrom'
+      userid←'mkrom'
       keyfile←'c:\docs\personal\macbook_air' 
 
       user←⎕NS ''
@@ -102,7 +97,7 @@
       db.Slaves←⎕NS¨2⍴⊂''
       db.Slaves.Shards←,¨1 2 ⍝ Distribution of shards to slave processors
       db.Slaves[1].(Launch←⎕NS '').Type←'local'                                                                
-      db.Slaves[2].(Launch←⎕NS '').(Type Host User KeyFile Cmd)←'ssh' host user keyfile cmd
+      db.Slaves[2].(Launch←⎕NS '').(Type Host User KeyFile Cmd)←'ssh' host userid keyfile cmd
       config←⎕NS''
       config.Server←vecdbsrv
       config.DBs←,db
@@ -134,7 +129,7 @@
 
       options←⎕NS''
       options.BlockSize←10000
-      options.ShardFolders←(folder,'Shard')∘,¨'12'
+      options.ShardFolders←(folder,'Shard1') '\\Mortens-Macbook-Air\vecdb\Shard2'
       options.(ShardFn ShardCols)←'{2-2|⎕UCS ⊃¨⊃⍵}' 1
 
       params←name folder columns types options data
@@ -148,8 +143,8 @@
       data←('IBM' 'AAPL' 'MSFT' 'GOOG' 'DYALOG')(160.97 112.6 47.21 531.23 999.99)(5⍴'Buy' 'Sell')
 
       options←⎕NS''
-      options.BlockSize←10000
       options.ShardFolders←(folder,'Shard')∘,¨'12'
+      options.BlockSize←10000
       options.(ShardFn ShardCols)←'{2-2|⎕UCS ⊃¨⊃⍵}' 1
 
       params←name folder columns types options data

vecdb.dyalog

@@ -4,7 +4,7 @@
     (⎕IO ⎕ML)←1 1
 
     :Section Constants
-    :Field Public Shared Version←'0.2.5' ⍝ Added ability to use Calculated columns
+    :Field Public Shared Version←'0.2.6' ⍝ Parallel DB
     :Field Public Shared TypeNames←,¨'I1' 'I2' 'I4' 'F' 'B' 'C'
     ⍝ To come: C4=323 indexed chars
     ⍝          Tn=Fixed with text (no index table)
@@ -348,10 +348,10 @@
 
       :For i :In newchars/newcols ⍝ Create symbol files for CHAR fields
           col←i⊃mappings
-          dix←newcols⍳i                       ⍝ data index  
+          dix←newcols⍳i                       ⍝ data index
           col.symbol←{⍵[∪⍳⍨↑⍵]}dix⊃data       ⍝ Unique symbols in input data
           col.file←folder,(⍕i),'.symbol'      ⍝ Symbol file name in main folder
-          col.symbol PutSymbols col.file      ⍝ Read symbols         
+          col.symbol PutSymbols col.file      ⍝ Read symbols
           col.(SymbolIndex←symbol∘⍳)          ⍝ Create lookup function
           (dix⊃data)←col.SymbolIndex dix⊃data ⍝ Convert indices
       :EndFor
@@ -377,8 +377,8 @@
               ⎕NUNTIE tn
           :EndIf
 
-          :For i :In newcols            ⍝ For each column being added  
-              ai3←⎕ai[3]
+          :For i :In newcols            ⍝ For each column being added
+              ai3←⎕AI[3]
               dr←(TypeNames⍳_Types[i])⊃TypeNums
               tn←(filename←sf,(⍕i),'.vector')⎕NCREATE 0
               (sizeOf size dr)⎕NRESIZE tn
@@ -460,7 +460,7 @@
           (offset+sizeOf count dr)⎕NRESIZE tn  ⍝ Extend the file
           ⎕NUNTIE tn
           n←≢col.vector←dr ¯1 ⎕MAP col.file'W' ⍝ Re-establish map
-          col.vector[(n-count)+⍳count]←count↑i⊃data ⍝ update with new data     
+          col.vector[(n-count)+⍳count]←count↑i⊃data ⍝ update with new data
       :EndFor
     ∇
 
@@ -812,7 +812,7 @@
       r←0
     ∇
 
-    ∇ r←Delete folder;file;tn;folders;files;f
+    ∇ r←Delete folder;file;tn;folders;files;f;shards
       :Access Public Shared
       ⍝ Erase a vecdb file without opening it first (it might be too damaged to open)
       ⍝   Does check whether there is a meta file in the folder
@@ -821,12 +821,17 @@
       folder←AddSlash folder
       'Folder not found'⎕SIGNAL(DirExists folder)↓22           ⍝ Not there
       'Not a vecdb'⎕SIGNAL(Exists file←folder,'meta.vecdb')↓22 ⍝ Paranoia
+      tn←file ⎕FTIE 0
+      folders←(⎕FREAD tn 6),⊂folder ⍝ shards first   
+      file ⎕FERASE tn
 
-      :If isWindows
-          ⎕CMD'rmdir "',folder,'" /s /q'
-      :Else
-          1 _SH'rm -r ',folder
-      :EndIf
+      :For folder :In folders
+          :If isWindows
+              ⎕CMD'rmdir "',folder,'" /s /q'
+          :Else
+              1 _SH'rm -r ',folder
+          :EndIf
+      :EndFor
 
       r←~DirExists folder
     ∇