diff --git a/APLProcess.dyalog b/APLProcess.dyalog index 061ccc4..5d12e33 100644 --- a/APLProcess.dyalog +++ b/APLProcess.dyalog @@ -51,28 +51,29 @@ Start(Ws Args RunTime) ∇ - ∇ Start(ws args rt);psi;pid + ∇ Start(ws args rt);psi;pid;cmd;host;port;keyfile;exe (Ws Args)←ws args :If 0≠⍴RIDE_INIT args←args,' RIDE_SPAWNED=1 RIDE_INIT=',RIDE_INIT :EndIf - :If ~0 2∊⍨10|⎕DR rt ⍝ if rt is character, it's the executable name - Exe←(RunTimeName⍣rt)GetCurrentExecutable + :If ~0 2 6∊⍨10|⎕DR rt ⍝ if rt is character or nested, it defines what to start + Exe←(RunTimeName⍣rt) GetCurrentExecutable ⍝ else, deduce it :Else Exe←rt rt←0 :EndIf - ⍝ ws,←rt/' salt' ⍝ if runtime, load the salt workspace first, which will subsequently load the target workspace - :If IsWin←IsWindows + + :If IsWin←IsWindows∧~IsSsh←326=⎕DR Exe ⎕USING←'System,System.dll' psi←⎕NEW Diagnostics.ProcessStartInfo,⊂Exe(ws,' ',args) psi.WindowStyle←Diagnostics.ProcessWindowStyle.Minimized Proc←Diagnostics.Process.Start psi :Else ⍝ Unix - :If IsSsh←326=⎕DR Exe - ∘∘∘ - Proc←SshProc Exe + :If IsSsh + (host port keyfile exe)←Exe + cmd←args,' ',exe,' -q +s ',ws + Proc←SshProc host port keyfile cmd :Else pid←_SH'{ ',args,' ',Exe,' +s ',ws,' -c APLppid=',(⍕GetCurrentProcessId),' /dev/null 2>&1 & } ; echo $!' Proc.Id←pid @@ -111,7 +112,6 @@ :If IsWin r←⍎'t'⎕NA'U4 kernel32|GetCurrentProcessId' :ElseIf IsSsh - ∘∘∘ r←Proc.Pid :Else r←tonum⊃_SH'echo $PPID' @@ -120,7 +120,7 @@ ∇ r←GetCurrentExecutable;⎕USING;t;gmfn :Access Public Shared - :If IsWin + :If IsWindows r←'' :Trap 0 'gmfn'⎕NA'U4 kernel32|GetModuleFileName* P =T[] U4' @@ -305,10 +305,8 @@ ∇ r←HasExited :Access public instance - :If IsWin + :If IsWin∨IsSsh r←{0::⍵ ⋄ Proc.HasExited}1 - :ElseIf IsSsh - ∘∘∘ :Else r←~UNIXIsRunning Proc.Id ⍝ AWS :EndIf @@ -480,14 +478,21 @@ listpids←{0~⍨2⊃(⎕UCS 10)⎕VFI (conn.RunCommand ⊂'ps -u ',user,' | grep dyalog | grep -v grep | awk ''{print $2}''').Result} guid←'dyalog-ssh-',(⍕⎕TS)~' ' pids←listpids ⍬ - tid←conn.RunCommand&⊂cmd ⍝ ,' -c ''',guid,'''' - :If 1=⍴pid←(listpids ⍬)~pids ⋄ pid←⊃pid - :Else ⋄ ∘∘∘ ⋄ :EndIf ⍝ failed to start Proc←⎕NS '' Proc.SshConn←conn + Proc.HasExited←0 + tid←{SshRun conn ⍵ Proc}&⊂cmd Proc.tid←tid - Proc.Pid← - + ⎕DL 1 + :If 1=⍴pid←(listpids ⍬)~pids ⋄ pid←⊃pid + :Else ⋄ ∘∘∘ ⋄ :EndIf ⍝ failed to start + Proc.Pid←pid + ∇ + + ∇SshRun (conn cmd proc) + ⍝ Wait until APL exits, then set HasExited←1 + conn.RunCommand cmd + proc.HasExited←1 ∇ :EndClass diff --git a/TestVecdbSrv.dyalog b/TestVecdbSrv.dyalog index b27497a..b4f6d3d 100644 --- a/TestVecdbSrv.dyalog +++ b/TestVecdbSrv.dyalog @@ -6,11 +6,12 @@ LOG←1 toJson←(0 1)∘(7160⌶) - ∇ z←Benchmark;columns;data;options;params;folder;types;name;ix;users;srvproc;clt;TEST;config;db;path + ∇ z←Benchmark;columns;data;options;params;folder;types;name;ix;users;srvproc;clt;TEST;config;db;path;tn;folders ⍝ Test database with 2 shards ⍝ Also acts as test for add/remove columns - path←Init + path←Init + path←'//Mortens-Macbook-Air/vecdb' folder←path,'/',(name←'srvtest'),'/' ⎕←'Clearing: ',folder :Trap 22 ⋄ {}#.vecdb.Delete folder ⋄ :EndTrap @@ -23,7 +24,13 @@ ⍝ --- Launch and connect to server, open database --- - srvproc←#.vecdbsrv.Launch folder 8100 + tn←(folder,'meta.vecdb') ⎕FSTIE 0 + ⍝ ↓↓↓ Arrgh, need to rename shards with local view from PC and Mac + folders←⎕FREAD tn,6 + 'c:\devt\vecdb\srvtest\shard1\' '//Users/mkrom/vecdb/srvtest/shard2/' ⎕FREPLACE tn,6 + ⎕FUNTIE tn + + srvproc←#.vecdbsrv.Launch folder 8100 ⍬ 'c:\devt\vecdb\vecdbboot.dws' #.vecdbclt.Connect '127.0.0.1' 8100 'mkrom' db←#.vecdbclt.Open folder @@ -54,7 +61,12 @@ :If ~srvproc.HasExited ⋄ srvproc.Kill ⋄ :EndIf ⎕DL 3 - TEST←'Erase database' + TEST←'Erase database' + tn←(folder,'meta.vecdb') ⎕FSTIE 0 + ⍝ ↓↓↓ Arrgh, need to rename shards with local view from PC and Mac + folders ⎕FREPLACE tn,6 + ⎕FUNTIE tn + db←⎕NEW #.vecdb(,⊂folder) assert 0={db.Erase}time ⍬ @@ -74,18 +86,12 @@ ⎕←ServerBasic ∇ - ∇ TestSsh;cmd - cmd←'RIDE_SPAWNED=1 RIDE_INIT=SERVE::5678 /Applications/Dyalog-15.0.app/Contents/Resources/Dyalog/mapl -q +s' -⍝ ##.APLProcess.NewSshClient '192.168.6.120' 'mkrom' 'c:\docs\personal\macbook-air' cmd - ##.APLProcess.NewSshClient '192.168.17.129' 'mkrom' 'c:\docs\personal\macbook-air' cmd - ∇ - ∇ config←CreateBenchConfig filename;db;config;user;vecdbsrv;cmd;host;keyfile;userid ⍝ cmd←'RIDE_SPAWNED=1 RIDE_INIT=SERVE::5678 /Applications/Dyalog-15.0.app/Contents/Resources/Dyalog/mapl' host←'Mortens-Macbook-Air' userid←'mkrom' - keyfile←'c:\docs\personal\macbook_air' + keyfile←'c:\docs\personal\macbook-air' user←⎕NS '' user.(Name Id Admin)←'mkrom' 1001 1 @@ -96,8 +102,11 @@ db.Folder←folder db.Slaves←⎕NS¨2⍴⊂'' db.Slaves.Shards←,¨1 2 ⍝ Distribution of shards to slave processors + db.Slaves.Folder←⊂folder db.Slaves[1].(Launch←⎕NS '').Type←'local' + db.Slaves[1].Folder←'//Mortens-Macbook-Air/vecdb/srvtest/' ⍝ If different seen from this slave db.Slaves[2].(Launch←⎕NS '').(Type Host User KeyFile Cmd)←'ssh' host userid keyfile cmd + db.Slaves[2].Folder←'/Users/mkrom/vecdb/srvtest/' config←⎕NS'' config.Server←vecdbsrv config.DBs←,db @@ -129,7 +138,7 @@ options←⎕NS'' options.BlockSize←10000 - options.ShardFolders←(folder,'Shard1') '\\Mortens-Macbook-Air\vecdb\Shard2' + options.ShardFolders←'c:\devt\vecdb\srvtest\shard1' '//Mortens-Macbook-Air/vecdb/srvtest/shard2' options.(ShardFn ShardCols)←'{2-2|⎕UCS ⊃¨⊃⍵}' 1 params←name folder columns types options data diff --git a/vecdb.dyalog b/vecdb.dyalog index 521e7fd..86138d3 100644 --- a/vecdb.dyalog +++ b/vecdb.dyalog @@ -72,9 +72,9 @@ ⍝ Extract calculation data from meta file :If 8=2⊃⎕FSIZE tn ⍝ If File format pre-dates calculated columns - 'unused'⎕FAPPEND tn ⍝ 8 - 'unused'⎕FAPPEND tn ⍝ 9 - (⍬ ⍬ ⍬)⎕FAPPEND tn ⍝ 10 Calc Col Names, Source Columns, Data Type + 'unused'∆FAPPEND tn ⍝ 8 + 'unused'∆FAPPEND tn ⍝ 9 + (⍬ ⍬ ⍬)∆FAPPEND tn ⍝ 10 Calc Col Names, Source Columns, Data Type :EndIf (_CalcCols _CalcSources _CalcTypes)←⎕FREAD tn,10 ⍝ Calculated column definitions @@ -365,8 +365,7 @@ :EndIf :For f :In ⍳≢ShardFolders - :If ~Exists sf←f⊃ShardFolders ⋄ MkDir sf ⋄ :EndIf - + 3 ⎕MKDIR sf←f⊃ShardFolders d←data[;shards⍳f] ⍝ extract records for one shard n←≢⊃d size←BlockSize×1⌈⌈n÷BlockSize ⍝ At least one block @@ -391,20 +390,28 @@ :EndFor :If create - tn←metafile ⎕FCREATE 0 - ('vecdb ',Version)⎕FAPPEND tn ⍝ 1 - 'See github.com/Dyalog/vecdb/doc/Implementation.md'⎕FAPPEND tn ⍝ 2 - 'unused'⎕FAPPEND tn ⍝ 3 - (fileprops(⍎¨fileprops))⎕FAPPEND tn ⍝ 4 (Name BlockSize) - (_Columns _Types)⎕FAPPEND tn ⍝ 5 - ShardFolders ⎕FAPPEND tn ⍝ 6 - (ShardFn ShardCols)⎕FAPPEND tn ⍝ 7 + tn←metafile (⎕FCREATE⍠3) 0 + ('vecdb ',Version)∆FAPPEND tn ⍝ 1 + 'See github.com/Dyalog/vecdb/doc/Implementation.md'∆FAPPEND tn ⍝ 2 + 'unused'∆FAPPEND tn ⍝ 3 + (fileprops(⍎¨fileprops))∆FAPPEND tn ⍝ 4 (Name BlockSize) + (_Columns _Types)∆FAPPEND tn ⍝ 5 + ShardFolders ∆FAPPEND tn ⍝ 6 + (ShardFn ShardCols)∆FAPPEND tn ⍝ 7 + 'unused'∆FAPPEND tn ⍝ 8 + 'unused'∆FAPPEND tn ⍝ 9 + (⍬ ⍬ ⍬)∆FAPPEND tn ⍝ 10 Calc Col Names, Source Columns, Data Type :Else ⍝ Extending tn←metafile ⎕FTIE 0 (_Columns _Types)⎕FREPLACE tn 5 :EndIf ⎕FUNTIE tn + ∇ + + ∇X ∆FAPPEND Y + X ⎕FAPPEND Y + ⎕FUNTIE ⍬ ∇ ∇ (shards data)←cix ShardData data;six;s;char;rawdata;sym;c;counts;m diff --git a/vecdbboot.dws b/vecdbboot.dws index 4276cb1..fe51ca7 100644 Binary files a/vecdbboot.dws and b/vecdbboot.dws differ diff --git a/vecdbslave.dyalog b/vecdbslave.dyalog index 6b07fa7..7a348e0 100644 --- a/vecdbslave.dyalog +++ b/vecdbslave.dyalog @@ -24,9 +24,10 @@ DB←⎕NEW ##.vecdb(folder shards) STATE←0 1 Log'Slave startup completed, ',STATUS←'Folder= ',folder,', shards= ',⍕shards - :Else + :Else STATE←2 ⍝ Startup Failed 3 Log STATUS←'Startup failed: ',∊⎕DM + ∘∘∘ :EndTrap ∇ diff --git a/vecdbsrv.dyalog b/vecdbsrv.dyalog index 83916f0..17e2f03 100644 --- a/vecdbsrv.dyalog +++ b/vecdbsrv.dyalog @@ -41,9 +41,15 @@ r←⍬ ⍝ Need a result ∇ - ∇ {r}←Init config;db;i;j;slave + ∇ {r}←Init config;db;i;j;slave;path;ws ⍝ Intialise the vecdb server + + :Trap 6 ⋄ source←SALT_Data.SourceFile + :Else ⋄ source←⎕WSID + :EndTrap + path←{(-⌊/(⌽⍵)⍳'\/')↓⍵}source + CONNS←TASKS←USERS←TOKENS←⍬ NEXTTASK←1000 LOGLEVEL←0 @@ -57,9 +63,20 @@ :For j :In ⍳≢db.Slaves slave←j⊃db.Slaves slave.Port←NEXTPORT - slave.Address←'127.0.0.1' ⍝ /// for now - slave.UserId←¯1 ⍝ /// ditto - slave.Proc←slave.Shards Launch db.Folder slave.Port + :Select slave.Launch.Type + :Case 'local' + slave.Address←'127.0.0.1' ⍝ /// for now + slave.UserId←¯1 ⍝ /// ditto + ws←path,'/vecdbboot.dws' + slave.Proc←slave.Shards Launch slave.Folder slave.Port RUNTIME ws + :Case 'ssh' + slave.Address←slave.Launch.Host + slave.UserId←¯1 + ws←({(-⌊/(⌽⍵)⍳'\/')↓⍵}¯1↓slave.Folder),'/vecdbboot.dws' + slave.Proc←slave.Shards Launch slave.Folder slave.Port slave.Launch.(Host User KeyFile Cmd) ws + :Else + ∘∘∘ ⍝ Unknown launch type + :EndSelect NEXTPORT←NEXTPORT+1 :EndFor :EndFor @@ -81,28 +98,28 @@ r←⍬ ⍝ Need a result ∇ - ∇ (sdata is)←SlavePartition (cmd slaves data);ixs;cols;new;slave;slave_recs - + ∇ (sdata is)←SlavePartition(cmd slaves data);ixs;cols;new;slave;slave_recs + Shards←∊slaves.Shards ShardSlaves←(≢¨slaves.Shards)/⍳≢slaves.Shards - + :Select cmd - :Case 'Read' - (ixs cols)←data - slave←ShardSlaves[Shards⍳ixs[;1]] - sdata←↓(⍪slave{⊂⍵}⌸ixs),⊂cols - is←⍳≢sdata + :Case 'Read' + (ixs cols)←data + slave←ShardSlaves[Shards⍳ixs[;1]] + sdata←↓(⍪slave{⊂⍵}⌸ixs),⊂cols + is←⍳≢sdata :Case 'Update' - (ixs cols new)←data - slave←ShardSlaves[Shards⍳ixs[;1]] - slave_recs←(≢¨ixs[;2])/slave - :If 1=≡,cols ⍝ Simple column name - sdata←↓(slave{⊂⍵}⌸ixs),(⊂cols),⍪slave_recs{⊂⍵}⌸new - :Else ⍝ Multiple columns - sdata←↓(slave{⊂⍵}⌸ixs),(⊂cols),⍪{slave_recs{⊂⍵}⌸⍵}¨new - ∘∘∘ - :EndIf - is←⍳≢sdata + (ixs cols new)←data + slave←ShardSlaves[Shards⍳ixs[;1]] + slave_recs←(≢¨ixs[;2])/slave + :If 1=≡,cols ⍝ Simple column name + sdata←↓(slave{⊂⍵}⌸ixs),(⊂cols),⍪slave_recs{⊂⍵}⌸new + :Else ⍝ Multiple columns + sdata←↓(slave{⊂⍵}⌸ixs),(⊂cols),⍪{slave_recs{⊂⍵}⌸⍵}¨new + ∘∘∘ + :EndIf + is←⍳≢sdata :Else sdata←,⊂data ⋄ is←(≢slaves)/1 :EndSelect @@ -128,19 +145,19 @@ r←999('Database not found: ',⊃arg) :Else slaves←(db←i⊃DBs).Slaves - (sdata ixs)←SlavePartition cmd slaves (2⊃arg) + (sdata ixs)←SlavePartition cmd slaves(2⊃arg) cmds←(≢slaves)⍴⊂'' - + :For s :In ⍳≢slaves ⍝ Send all the commands slave←s⊃slaves - (s⊃cmds)←#.vecdbclt.SrvSend slave.Connection(cmd ((s⊃ixs)⊃sdata)) + (s⊃cmds)←#.vecdbclt.SrvSend slave.Connection(cmd((s⊃ixs)⊃sdata)) :EndFor - - rs←⍬ + + rs←⍬ :For s :In cmds ⍝ Receive all the results rs,←⊂#.vecdbclt.SrvRcv s :EndFor - + r←0 rs :EndIf :Else @@ -154,17 +171,11 @@ :EndIf ∇ - ∇ proc←{shards}Launch(target port);path;runtime;args;slave;ws;source + ∇ proc←{shards}Launch(target port runtime ws);path;runtime;args;slave;source ⍝ Launch a full vecdbsrv or, if shards is defined, a slave - - :Trap 6 ⋄ source←SALT_Data.SourceFile - :Else ⋄ source←⎕WSID - :EndTrap - - path←{(-⌊/(⌽⍵)⍳'\/')↓⍵}source - ws←path,'/vecdbboot.dws' - runtime←RUNTIME - + + :If 0=≢runtime ⋄ runtime←RUNTIME ⋄ :EndIf + :If slave←2=⎕NC'shards' args←'VECDBSLAVE="',target,'" SHARDS="',(⍕shards),'" PORT=',(⍕port),' TOKEN="',TOKEN,'"' :Else