From 28f0fff3c9c23fb99a60c36a7e4adf39f7ed721b Mon Sep 17 00:00:00 2001 From: Morten Kromberg Date: Fri, 30 Sep 2016 00:28:52 +0200 Subject: [PATCH] Sharding between Mac & Windows almost working, if it wasn't for Samba errors --- APLProcess.dyalog | 41 ++++++++++++--------- TestVecdbSrv.dyalog | 33 +++++++++++------ vecdb.dyalog | 33 ++++++++++------- vecdbboot.dws | Bin 41920 -> 42152 bytes vecdbslave.dyalog | 3 +- vecdbsrv.dyalog | 87 +++++++++++++++++++++++++------------------- 6 files changed, 115 insertions(+), 82 deletions(-) diff --git a/APLProcess.dyalog b/APLProcess.dyalog index 061ccc4..5d12e33 100644 --- a/APLProcess.dyalog +++ b/APLProcess.dyalog @@ -51,28 +51,29 @@ Start(Ws Args RunTime) ∇ - ∇ Start(ws args rt);psi;pid + ∇ Start(ws args rt);psi;pid;cmd;host;port;keyfile;exe (Ws Args)←ws args :If 0≠⍴RIDE_INIT args←args,' RIDE_SPAWNED=1 RIDE_INIT=',RIDE_INIT :EndIf - :If ~0 2∊⍨10|⎕DR rt ⍝ if rt is character, it's the executable name - Exe←(RunTimeName⍣rt)GetCurrentExecutable + :If ~0 2 6∊⍨10|⎕DR rt ⍝ if rt is character or nested, it defines what to start + Exe←(RunTimeName⍣rt) GetCurrentExecutable ⍝ else, deduce it :Else Exe←rt rt←0 :EndIf - ⍝ ws,←rt/' salt' ⍝ if runtime, load the salt workspace first, which will subsequently load the target workspace - :If IsWin←IsWindows + + :If IsWin←IsWindows∧~IsSsh←326=⎕DR Exe ⎕USING←'System,System.dll' psi←⎕NEW Diagnostics.ProcessStartInfo,⊂Exe(ws,' ',args) psi.WindowStyle←Diagnostics.ProcessWindowStyle.Minimized Proc←Diagnostics.Process.Start psi :Else ⍝ Unix - :If IsSsh←326=⎕DR Exe - ∘∘∘ - Proc←SshProc Exe + :If IsSsh + (host port keyfile exe)←Exe + cmd←args,' ',exe,' -q +s ',ws + Proc←SshProc host port keyfile cmd :Else pid←_SH'{ ',args,' ',Exe,' +s ',ws,' -c APLppid=',(⍕GetCurrentProcessId),' /dev/null 2>&1 & } ; echo $!' Proc.Id←pid @@ -111,7 +112,6 @@ :If IsWin r←⍎'t'⎕NA'U4 kernel32|GetCurrentProcessId' :ElseIf IsSsh - ∘∘∘ r←Proc.Pid :Else r←tonum⊃_SH'echo $PPID' @@ -120,7 +120,7 @@ ∇ r←GetCurrentExecutable;⎕USING;t;gmfn :Access Public Shared - :If IsWin + :If IsWindows r←'' :Trap 0 'gmfn'⎕NA'U4 kernel32|GetModuleFileName* P =T[] U4' @@ -305,10 +305,8 @@ ∇ r←HasExited :Access public instance - :If IsWin + :If IsWin∨IsSsh r←{0::⍵ ⋄ Proc.HasExited}1 - :ElseIf IsSsh - ∘∘∘ :Else r←~UNIXIsRunning Proc.Id ⍝ AWS :EndIf @@ -480,14 +478,21 @@ listpids←{0~⍨2⊃(⎕UCS 10)⎕VFI (conn.RunCommand ⊂'ps -u ',user,' | grep dyalog | grep -v grep | awk ''{print $2}''').Result} guid←'dyalog-ssh-',(⍕⎕TS)~' ' pids←listpids ⍬ - tid←conn.RunCommand&⊂cmd ⍝ ,' -c ''',guid,'''' - :If 1=⍴pid←(listpids ⍬)~pids ⋄ pid←⊃pid - :Else ⋄ ∘∘∘ ⋄ :EndIf ⍝ failed to start Proc←⎕NS '' Proc.SshConn←conn + Proc.HasExited←0 + tid←{SshRun conn ⍵ Proc}&⊂cmd Proc.tid←tid - Proc.Pid← - + ⎕DL 1 + :If 1=⍴pid←(listpids ⍬)~pids ⋄ pid←⊃pid + :Else ⋄ ∘∘∘ ⋄ :EndIf ⍝ failed to start + Proc.Pid←pid + ∇ + + ∇SshRun (conn cmd proc) + ⍝ Wait until APL exits, then set HasExited←1 + conn.RunCommand cmd + proc.HasExited←1 ∇ :EndClass diff --git a/TestVecdbSrv.dyalog b/TestVecdbSrv.dyalog index b27497a..b4f6d3d 100644 --- a/TestVecdbSrv.dyalog +++ b/TestVecdbSrv.dyalog @@ -6,11 +6,12 @@ LOG←1 toJson←(0 1)∘(7160⌶) - ∇ z←Benchmark;columns;data;options;params;folder;types;name;ix;users;srvproc;clt;TEST;config;db;path + ∇ z←Benchmark;columns;data;options;params;folder;types;name;ix;users;srvproc;clt;TEST;config;db;path;tn;folders ⍝ Test database with 2 shards ⍝ Also acts as test for add/remove columns - path←Init + path←Init + path←'//Mortens-Macbook-Air/vecdb' folder←path,'/',(name←'srvtest'),'/' ⎕←'Clearing: ',folder :Trap 22 ⋄ {}#.vecdb.Delete folder ⋄ :EndTrap @@ -23,7 +24,13 @@ ⍝ --- Launch and connect to server, open database --- - srvproc←#.vecdbsrv.Launch folder 8100 + tn←(folder,'meta.vecdb') ⎕FSTIE 0 + ⍝ ↓↓↓ Arrgh, need to rename shards with local view from PC and Mac + folders←⎕FREAD tn,6 + 'c:\devt\vecdb\srvtest\shard1\' '//Users/mkrom/vecdb/srvtest/shard2/' ⎕FREPLACE tn,6 + ⎕FUNTIE tn + + srvproc←#.vecdbsrv.Launch folder 8100 ⍬ 'c:\devt\vecdb\vecdbboot.dws' #.vecdbclt.Connect '127.0.0.1' 8100 'mkrom' db←#.vecdbclt.Open folder @@ -54,7 +61,12 @@ :If ~srvproc.HasExited ⋄ srvproc.Kill ⋄ :EndIf ⎕DL 3 - TEST←'Erase database' + TEST←'Erase database' + tn←(folder,'meta.vecdb') ⎕FSTIE 0 + ⍝ ↓↓↓ Arrgh, need to rename shards with local view from PC and Mac + folders ⎕FREPLACE tn,6 + ⎕FUNTIE tn + db←⎕NEW #.vecdb(,⊂folder) assert 0={db.Erase}time ⍬ @@ -74,18 +86,12 @@ ⎕←ServerBasic ∇ - ∇ TestSsh;cmd - cmd←'RIDE_SPAWNED=1 RIDE_INIT=SERVE::5678 /Applications/Dyalog-15.0.app/Contents/Resources/Dyalog/mapl -q +s' -⍝ ##.APLProcess.NewSshClient '192.168.6.120' 'mkrom' 'c:\docs\personal\macbook-air' cmd - ##.APLProcess.NewSshClient '192.168.17.129' 'mkrom' 'c:\docs\personal\macbook-air' cmd - ∇ - ∇ config←CreateBenchConfig filename;db;config;user;vecdbsrv;cmd;host;keyfile;userid ⍝ cmd←'RIDE_SPAWNED=1 RIDE_INIT=SERVE::5678 /Applications/Dyalog-15.0.app/Contents/Resources/Dyalog/mapl' host←'Mortens-Macbook-Air' userid←'mkrom' - keyfile←'c:\docs\personal\macbook_air' + keyfile←'c:\docs\personal\macbook-air' user←⎕NS '' user.(Name Id Admin)←'mkrom' 1001 1 @@ -96,8 +102,11 @@ db.Folder←folder db.Slaves←⎕NS¨2⍴⊂'' db.Slaves.Shards←,¨1 2 ⍝ Distribution of shards to slave processors + db.Slaves.Folder←⊂folder db.Slaves[1].(Launch←⎕NS '').Type←'local' + db.Slaves[1].Folder←'//Mortens-Macbook-Air/vecdb/srvtest/' ⍝ If different seen from this slave db.Slaves[2].(Launch←⎕NS '').(Type Host User KeyFile Cmd)←'ssh' host userid keyfile cmd + db.Slaves[2].Folder←'/Users/mkrom/vecdb/srvtest/' config←⎕NS'' config.Server←vecdbsrv config.DBs←,db @@ -129,7 +138,7 @@ options←⎕NS'' options.BlockSize←10000 - options.ShardFolders←(folder,'Shard1') '\\Mortens-Macbook-Air\vecdb\Shard2' + options.ShardFolders←'c:\devt\vecdb\srvtest\shard1' '//Mortens-Macbook-Air/vecdb/srvtest/shard2' options.(ShardFn ShardCols)←'{2-2|⎕UCS ⊃¨⊃⍵}' 1 params←name folder columns types options data diff --git a/vecdb.dyalog b/vecdb.dyalog index 521e7fd..86138d3 100644 --- a/vecdb.dyalog +++ b/vecdb.dyalog @@ -72,9 +72,9 @@ ⍝ Extract calculation data from meta file :If 8=2⊃⎕FSIZE tn ⍝ If File format pre-dates calculated columns - 'unused'⎕FAPPEND tn ⍝ 8 - 'unused'⎕FAPPEND tn ⍝ 9 - (⍬ ⍬ ⍬)⎕FAPPEND tn ⍝ 10 Calc Col Names, Source Columns, Data Type + 'unused'∆FAPPEND tn ⍝ 8 + 'unused'∆FAPPEND tn ⍝ 9 + (⍬ ⍬ ⍬)∆FAPPEND tn ⍝ 10 Calc Col Names, Source Columns, Data Type :EndIf (_CalcCols _CalcSources _CalcTypes)←⎕FREAD tn,10 ⍝ Calculated column definitions @@ -365,8 +365,7 @@ :EndIf :For f :In ⍳≢ShardFolders - :If ~Exists sf←f⊃ShardFolders ⋄ MkDir sf ⋄ :EndIf - + 3 ⎕MKDIR sf←f⊃ShardFolders d←data[;shards⍳f] ⍝ extract records for one shard n←≢⊃d size←BlockSize×1⌈⌈n÷BlockSize ⍝ At least one block @@ -391,20 +390,28 @@ :EndFor :If create - tn←metafile ⎕FCREATE 0 - ('vecdb ',Version)⎕FAPPEND tn ⍝ 1 - 'See github.com/Dyalog/vecdb/doc/Implementation.md'⎕FAPPEND tn ⍝ 2 - 'unused'⎕FAPPEND tn ⍝ 3 - (fileprops(⍎¨fileprops))⎕FAPPEND tn ⍝ 4 (Name BlockSize) - (_Columns _Types)⎕FAPPEND tn ⍝ 5 - ShardFolders ⎕FAPPEND tn ⍝ 6 - (ShardFn ShardCols)⎕FAPPEND tn ⍝ 7 + tn←metafile (⎕FCREATE⍠3) 0 + ('vecdb ',Version)∆FAPPEND tn ⍝ 1 + 'See github.com/Dyalog/vecdb/doc/Implementation.md'∆FAPPEND tn ⍝ 2 + 'unused'∆FAPPEND tn ⍝ 3 + (fileprops(⍎¨fileprops))∆FAPPEND tn ⍝ 4 (Name BlockSize) + (_Columns _Types)∆FAPPEND tn ⍝ 5 + ShardFolders ∆FAPPEND tn ⍝ 6 + (ShardFn ShardCols)∆FAPPEND tn ⍝ 7 + 'unused'∆FAPPEND tn ⍝ 8 + 'unused'∆FAPPEND tn ⍝ 9 + (⍬ ⍬ ⍬)∆FAPPEND tn ⍝ 10 Calc Col Names, Source Columns, Data Type :Else ⍝ Extending tn←metafile ⎕FTIE 0 (_Columns _Types)⎕FREPLACE tn 5 :EndIf ⎕FUNTIE tn + ∇ + + ∇X ∆FAPPEND Y + X ⎕FAPPEND Y + ⎕FUNTIE ⍬ ∇ ∇ (shards data)←cix ShardData data;six;s;char;rawdata;sym;c;counts;m diff --git a/vecdbboot.dws b/vecdbboot.dws index 4276cb11ed575ee652d5b295dec7b0eefc975f2c..fe51ca72b93b6c8e21ee88dca138e52b29e0eb54 100644 GIT binary patch delta 2340 zcmZ8heQZ-z6uR<};ewr;?94#BrgNhukiCVU04khr%%1O{WT5Lx#DI+(ce z6+YLmGuC!&*hV7+|0(pJNusG}B49!miO3%si+||i2bshuA0~R{b`Wyb&^*v^1zZVv=~8e+D!>z-=_=4Yj`I9B z+nIl&AJ&IgmKb|E9y4>2?M*OOhZ*u=ce!EW1-2!qpLg}4o;D~E*_r1DjP7GL=R&X} z(ntxcjLm`Zh~g}b3TMWf!cY_4NON#3x(PBdGfYHRIQ^0T8}&tCRjd`ux)4*P6;ZH4 zBiLKN=gl!Zyc$*vc%1G0SeZ9)GzNPIUeGVaV&_E}y@!c` zx#fm`hUXCB9szE4->B7ps-AT%!PNmiQCA>&YTa_9Telu5RpBOo{LMZS&yyUa$`8#+(C0NUbJmT`Wh zKHpEm#rS+lPQv%`1(rMyF%Ff9<(6EW9U()KM2v7_3B_XOSuq<=Qnn;` z@I0(dF0phcMVfkyOj4lWEHjiFZ_e=5i4n#Hjqsnxx{hB}q0@O{7^C%OADNDiCF8Xb(taInzC_$ae2w@9@gu@P38_ZZ zBI*!J5o-_|5COz?L?@yP@fPARB7}$|QiwDni}(ofX)lt`5Z4hm5Z@twM*N1ji}({! zj6eAj#5}|*gdg!LVgsTXu}#w2J6bwhx>|O%?AAzuO(WB6nzqPiR*_iL7aOc+?-lODp#KoNCwOs&>d&9oAPRs&)i}qZ=+tRqfd1;we?rI(@rP?MBI>&TQCS zIYaH`Qklwyav@X*!Gq?6UM70xZ4H$w-AJlzTBX|4FjMWCrQ+WEQ-iw6gkjQ&HP2SH zYPFJ`G3P(BjMmcTDQKPa)u!Z zg32?f+}~*>t*Cy*6g)(wI!CaOw7$dl4j<{n6>hG__=|js9^Q?K5qedpwKllkQw4#v d;tZs4ittOM3`XR1_I!FVl}6Jzkg<1j{{WV?8u$PJ delta 2281 zcmY*beN0nV6hHSCqLqLDbWur)E=_K)+Ai6)y(jE3!>xn)_L`(tR91>NR$-h1~MedKq~=RNn_ zbI-kH{3^e6g=e3)%;j^T?hJ6yKXoLMz09(;rX3w@aoSX6B%PQS)2^?Gq(^-7(*E%n z^z8G(Fs-WTSj=BZ1elv;$3~4P=2$y|%(6K##go>Oyb)7tQ0q>ERth>Zn)p2P3x=8Y z`H2pD8_Srb**%$whjSR4s|(5F#*|n*A!RKW%B)^P2|c*c(JZBen6xJ@J8spp5dIX) zO-yXJ=JAQyi7yXKFw5Eu7j3EnM-Qg>ql`#rX(D#0g-M}^5olk3Lh9(^RZWlvwd%%n zO*;k_xIp-nkRtp_;LHL}LLKAqF5x1fkaL*e3}5egE-~G;NUB-`>tLq)h#urkcn98q zO|S+w61NE!gOBhMpKA{yK*$Ry1M( zT*A)Cv*^Vd=EmJ9^P?(W!_86+8V=)#J1Bl&ppw<1Z@`7)10Fu!jU@wR{ZS9zj#_Ok z;*4i>yS9hMs8cJIBPn(s5ScyH3p5dZp-69_X^~2UD%KKi4|KPz6BT+PtavZ#!hMER z9uj%|^`G%s#6Fl8j%utrXee$A+Hw-_pIBi_MGbXQ>pu((dZb#RQKQ&3xRwE49b89S zygFEjWw8>QXTX@CH!Ox?Ue<#5hI26)bKBe}44a37t+Ro-5;}(S@ztSHDJkr!D1JD! z)>ix2p>vShW5{Eh5}R<-l$Ew*%*;)*hlag;3UO?>jt=DB@Jhur{*A=9vf=!h0kBC}NH9Wje(c8*y)^I0)_&3sq9eOP`fhc)2( zQ?9Ui;mixOPxO;{G0dJedt^rJ%Ip=lIEWVWKm>&$u{m*UvA7}MeY)EC$Hz9d2HEMY zzDjV@E253d2|id#<5JSNNmrrk&cSxL3Af-+xB=(rb@9zv!tV^@<f#juMU$;)Jt=PY4$X7d7Koge!y_ zgzpJI5Pl^5OL#<>PrrHPgi1m+VKrerVFOn}yPH~@+M3#%_9!sRu0V!eQQG_fzdM6g zdA@7EUy@f_8P$b1wkM(Qy8S4yj2ub+WnNA-5IQql?!eB2miew~m&~?++pf6e1J%WHTZv4z-t?g6GNrq;Dpmf4va(1n zHr9CN3BHuPd2Jeb5Bfc_!q{)lAY;E21*J5nN8SKz!4P2>SYdq@t;)C?f`?{7bDXp% zNgHmJpqbP!TY_DT7dV2M(0oeR4j*YHiHxf#ewDwB!gC8H2JAC!T!M?oHsyDSe