From 60cb4cbc7a10d69238fa92c6460ef33ba9827fd7 Mon Sep 17 00:00:00 2001 From: Morten Kromberg Date: Thu, 29 Sep 2016 18:40:25 +0200 Subject: [PATCH] Making progress testing parallel db with ssh support --- APLProcess.dyalog | 4 ++-- BootServers.dyalog | 2 ++ TestVecdbSrv.dyalog | 17 ++++++----------- vecdb.dyalog | 29 +++++++++++++++++------------ 4 files changed, 27 insertions(+), 25 deletions(-) diff --git a/APLProcess.dyalog b/APLProcess.dyalog index 36feeba..061ccc4 100644 --- a/APLProcess.dyalog +++ b/APLProcess.dyalog @@ -24,7 +24,6 @@ :Trap 6 source←⍎'(⊃⊃⎕CLASS ⎕THIS).SALT_Data.SourceFile' ⍝ ⍎ works around a bug :Else - source←(⊃⊃⎕CLASS ⎕THIS).SALT_Data.SourceFile :If 0=⍴source←{((⊃¨⍵)⍳⊃⊃⎕CLASS ⎕THIS)⊃⍵,⊂''}5177⌶⍬ source←⎕WSID :Else ⋄ source←4⊃source @@ -71,7 +70,8 @@ psi.WindowStyle←Diagnostics.ProcessWindowStyle.Minimized Proc←Diagnostics.Process.Start psi :Else ⍝ Unix - :If IsSsh←326=⎕DR Exe + :If IsSsh←326=⎕DR Exe + ∘∘∘ Proc←SshProc Exe :Else pid←_SH'{ ',args,' ',Exe,' +s ',ws,' -c APLppid=',(⍕GetCurrentProcessId),' /dev/null 2>&1 & } ; echo $!' diff --git a/BootServers.dyalog b/BootServers.dyalog index 6a191c6..ccf9ccb 100644 --- a/BootServers.dyalog +++ b/BootServers.dyalog @@ -2,6 +2,8 @@ ⍝ Start a vecdb server process if VECDBSRV="config.json" PORT=nnnn ⍝ vecdb slave process if VECDBSLAVE="file" SHARDS="n" PORT=nnnn + ⎕←'Command Line:' + ⎕←2 ⎕NQ'.' 'GetCommandLine' getenv←{0=≢r←2 ⎕NQ'.' 'GetEnvironment'⍵:⍺ ⋄ r} getnum←{⊃2⊃⎕VFI ⍵} path←'file://',⊃⎕NPARTS ⎕WSID diff --git a/TestVecdbSrv.dyalog b/TestVecdbSrv.dyalog index 69324f2..b27497a 100644 --- a/TestVecdbSrv.dyalog +++ b/TestVecdbSrv.dyalog @@ -16,11 +16,6 @@ :Trap 22 ⋄ {}#.vecdb.Delete folder ⋄ :EndTrap ⎕MKDIR folder - :If 0=⎕NC '#.PASS' - ⎕←'Enter password from mkrom@Mortens-Macbook-Air: ' - #.PASS←⍞ - :EndIf - ⍝ --- Create configuration file --- config←CreateBenchConfig folder,'config.json' @@ -85,11 +80,11 @@ ##.APLProcess.NewSshClient '192.168.17.129' 'mkrom' 'c:\docs\personal\macbook-air' cmd ∇ - ∇ config←CreateBenchConfig filename;db;config;user;vecdbsrv;cmd;host;keyfile + ∇ config←CreateBenchConfig filename;db;config;user;vecdbsrv;cmd;host;keyfile;userid ⍝ - cmd←'RIDE_INIT=SERVE::5678 /Applications/Dyalog-15.0.app/Contents/Resources/Dyalog/mapl' + cmd←'RIDE_SPAWNED=1 RIDE_INIT=SERVE::5678 /Applications/Dyalog-15.0.app/Contents/Resources/Dyalog/mapl' host←'Mortens-Macbook-Air' - user←'mkrom' + userid←'mkrom' keyfile←'c:\docs\personal\macbook_air' user←⎕NS '' @@ -102,7 +97,7 @@ db.Slaves←⎕NS¨2⍴⊂'' db.Slaves.Shards←,¨1 2 ⍝ Distribution of shards to slave processors db.Slaves[1].(Launch←⎕NS '').Type←'local' - db.Slaves[2].(Launch←⎕NS '').(Type Host User KeyFile Cmd)←'ssh' host user keyfile cmd + db.Slaves[2].(Launch←⎕NS '').(Type Host User KeyFile Cmd)←'ssh' host userid keyfile cmd config←⎕NS'' config.Server←vecdbsrv config.DBs←,db @@ -134,7 +129,7 @@ options←⎕NS'' options.BlockSize←10000 - options.ShardFolders←(folder,'Shard')∘,¨'12' + options.ShardFolders←(folder,'Shard1') '\\Mortens-Macbook-Air\vecdb\Shard2' options.(ShardFn ShardCols)←'{2-2|⎕UCS ⊃¨⊃⍵}' 1 params←name folder columns types options data @@ -148,8 +143,8 @@ data←('IBM' 'AAPL' 'MSFT' 'GOOG' 'DYALOG')(160.97 112.6 47.21 531.23 999.99)(5⍴'Buy' 'Sell') options←⎕NS'' - options.BlockSize←10000 options.ShardFolders←(folder,'Shard')∘,¨'12' + options.BlockSize←10000 options.(ShardFn ShardCols)←'{2-2|⎕UCS ⊃¨⊃⍵}' 1 params←name folder columns types options data diff --git a/vecdb.dyalog b/vecdb.dyalog index ce227f0..521e7fd 100644 --- a/vecdb.dyalog +++ b/vecdb.dyalog @@ -4,7 +4,7 @@ (⎕IO ⎕ML)←1 1 :Section Constants - :Field Public Shared Version←'0.2.5' ⍝ Added ability to use Calculated columns + :Field Public Shared Version←'0.2.6' ⍝ Parallel DB :Field Public Shared TypeNames←,¨'I1' 'I2' 'I4' 'F' 'B' 'C' ⍝ To come: C4=323 indexed chars ⍝ Tn=Fixed with text (no index table) @@ -348,10 +348,10 @@ :For i :In newchars/newcols ⍝ Create symbol files for CHAR fields col←i⊃mappings - dix←newcols⍳i ⍝ data index + dix←newcols⍳i ⍝ data index col.symbol←{⍵[∪⍳⍨↑⍵]}dix⊃data ⍝ Unique symbols in input data col.file←folder,(⍕i),'.symbol' ⍝ Symbol file name in main folder - col.symbol PutSymbols col.file ⍝ Read symbols + col.symbol PutSymbols col.file ⍝ Read symbols col.(SymbolIndex←symbol∘⍳) ⍝ Create lookup function (dix⊃data)←col.SymbolIndex dix⊃data ⍝ Convert indices :EndFor @@ -377,8 +377,8 @@ ⎕NUNTIE tn :EndIf - :For i :In newcols ⍝ For each column being added - ai3←⎕ai[3] + :For i :In newcols ⍝ For each column being added + ai3←⎕AI[3] dr←(TypeNames⍳_Types[i])⊃TypeNums tn←(filename←sf,(⍕i),'.vector')⎕NCREATE 0 (sizeOf size dr)⎕NRESIZE tn @@ -460,7 +460,7 @@ (offset+sizeOf count dr)⎕NRESIZE tn ⍝ Extend the file ⎕NUNTIE tn n←≢col.vector←dr ¯1 ⎕MAP col.file'W' ⍝ Re-establish map - col.vector[(n-count)+⍳count]←count↑i⊃data ⍝ update with new data + col.vector[(n-count)+⍳count]←count↑i⊃data ⍝ update with new data :EndFor ∇ @@ -812,7 +812,7 @@ r←0 ∇ - ∇ r←Delete folder;file;tn;folders;files;f + ∇ r←Delete folder;file;tn;folders;files;f;shards :Access Public Shared ⍝ Erase a vecdb file without opening it first (it might be too damaged to open) ⍝ Does check whether there is a meta file in the folder @@ -821,12 +821,17 @@ folder←AddSlash folder 'Folder not found'⎕SIGNAL(DirExists folder)↓22 ⍝ Not there 'Not a vecdb'⎕SIGNAL(Exists file←folder,'meta.vecdb')↓22 ⍝ Paranoia + tn←file ⎕FTIE 0 + folders←(⎕FREAD tn 6),⊂folder ⍝ shards first + file ⎕FERASE tn - :If isWindows - ⎕CMD'rmdir "',folder,'" /s /q' - :Else - 1 _SH'rm -r ',folder - :EndIf + :For folder :In folders + :If isWindows + ⎕CMD'rmdir "',folder,'" /s /q' + :Else + 1 _SH'rm -r ',folder + :EndIf + :EndFor r←~DirExists folder ∇