diff --git a/TODO.md b/TODO.md index 025492f..b6e1184 100644 --- a/TODO.md +++ b/TODO.md @@ -2,11 +2,11 @@ ## Currently prioritised ## -List updated February 23rd, 2016. +List updated December 14th, 2016. -1. Parallel queries +1. Parallel queries (first pass done) -## To be done sometime later ## +## To be done soon ## 1. Generalization of Symbol Tables + Add One, Four & Eight Byte Symbol Tables 1. Enhance queries to support conditional functions... Eg. ('price' '>' 100)('Name' 'like' 'A%') @@ -14,6 +14,8 @@ List updated February 23rd, 2016. 1. Database status reporting function (# shards, records in each, statistics, etc) 1. Add a "Char" type which does not use a symbol table 1. User Guide + +## More speculative ideas ## 1. RESTful / ODATA? API 1. Timestamped non-overwriting updates 1. Delete records (AFTER non-overwriting updates) diff --git a/TestVecdb.dyalog b/TestVecdb.dyalog index e9264be..aa19f49 100644 --- a/TestVecdb.dyalog +++ b/TestVecdb.dyalog @@ -99,8 +99,8 @@ numrecs←10000000 ⍝ 10 million records memstats 1 ⍝ Clear memory statistics - :If (8×numrecs)>2000⌶16 - ⎕←'*** Warning: workspace size should be at least: ',(⍕⌈(8×numrecs)÷1000000)',Mb ***' + :If (100×numrecs)>2000⌶16 + ⎕←'*** Warning: workspace size should be at least: ',(⍕⌈(100×numrecs)÷1000000),'MB ***' :EndIf folder←path,'/',(name←'testdb1'),'/' @@ -119,7 +119,7 @@ recs←numrecs(⌊÷)2 (options←⎕NS'').BlockSize←numrecs(⌊×)0.6 ⍝ Provoke block overflow - params←name folder columns types options(recs↑¨data) + params←name folder columns types options (recs↑¨data) TEST←'Creating db & inserting ',(fmtnum recs),' records' db←⎕NEW time #.vecdb params assert db.isOpen diff --git a/TestVecdbSrv.dyalog b/TestVecdbSrv.dyalog index b4f6d3d..68dad7c 100644 --- a/TestVecdbSrv.dyalog +++ b/TestVecdbSrv.dyalog @@ -5,117 +5,158 @@ (⎕IO ⎕ML)←1 1 LOG←1 toJson←(0 1)∘(7160⌶) - - ∇ z←Benchmark;columns;data;options;params;folder;types;name;ix;users;srvproc;clt;TEST;config;db;path;tn;folders + + ∇ z←Benchmark;columns;data;options;params;folder;types;name;ix;users;srvproc;clt;TEST;config;db;path;tn;folders;run;sym;date;price;lix;expect;⎕RL;m ⍝ Test database with 2 shards ⍝ Also acts as test for add/remove columns - - path←Init - path←'//Mortens-Macbook-Air/vecdb' - folder←path,'/',(name←'srvtest'),'/' - ⎕←'Clearing: ',folder - :Trap 22 ⋄ {}#.vecdb.Delete folder ⋄ :EndTrap - ⎕MKDIR folder - + ⍝ --- Create configuration file --- + + :For run :In 1↓'serial' 'parallel' ⍝ /// While debugging parallel + ⎕←'*** ',run,' run begins...' + + columns←'Day' 'Sym' 'Price' + types←,¨'I2' 'C' 'F' + data←⍬ ⍬ ⍬ + + :Select run + :Case 'serial' ⋄ path←Init + :Case 'parallel' ⋄ path←Init ⍝ '//Mortens-Macbook-Air/vecdb' + :EndSelect + + folder←path,'/',(name←'srvtest'),'/' + + ⎕←'Clearing: ',folder + :Trap 22 ⋄ {}#.vecdb.Delete folder ⋄ :EndTrap + 1 ⎕MKDIR folder + + config←CreateBenchConfig folder,'config.json' + + options←⎕NS'' + options.BlockSize←10000 ⍝ 10,000 records - config←CreateBenchConfig folder,'config.json' - (db (name folder columns types options data))←CreateBenchDB - - ⍝ --- Launch and connect to server, open database --- - - tn←(folder,'meta.vecdb') ⎕FSTIE 0 - ⍝ ↓↓↓ Arrgh, need to rename shards with local view from PC and Mac - folders←⎕FREAD tn,6 - 'c:\devt\vecdb\srvtest\shard1\' '//Users/mkrom/vecdb/srvtest/shard2/' ⎕FREPLACE tn,6 - ⎕FUNTIE tn - - srvproc←#.vecdbsrv.Launch folder 8100 ⍬ 'c:\devt\vecdb\vecdbboot.dws' - #.vecdbclt.Connect '127.0.0.1' 8100 'mkrom' - db←#.vecdbclt.Open folder - - TEST←'Count records' - assert (≢⊃data)={db.Count} time ⍬ - - TEST←'Search for all records' - ix←db.Query time ('Name'((columns⍳⊂'Name')⊃data))⍬ ⍝ Should find everything - assert(1 2,⍪⍳¨4 1)≡ix - - TEST←'Read it all back' - assert data≡db.Read time ix columns + date←10000/⍳2×365 ⍝ 10,000 records × 2 × 365 days = ~36M records + sym←(≢date)⍴'MSFT' 'IBM' 'AAPL' 'GOOG' 'DYLG' + price←100+0.1×⍳⍴date ⍝ It's a good market! + + :Select run + :Case 'serial' + options.ShardFolders←⍬ + params←name folder columns types options (5↑¨date sym price) + TEST←'Create empty database' + db←{⎕NEW #.vecdb params} time ⍬ + assert 5=db.Count + + :Case 'parallel' + ⍝ options.ShardFolders←'c:\devt\vecdb\srvtest\shard1' '//Mortens-Macbook-Air/vecdb/srvtest/shard2' + ⍝ options.LocalFolders←'c:\devt\vecdb\srvtest\shard1' '//Users/mkrom/vecdb/srvtest/shard2' + options.ShardFolders←'c:\devt\vecdb\srvtest\shard1' 'c:\devt\vecdb\srvtest\shard2' + options.LocalFolders←'c:\devt\vecdb\srvtest\shard1' 'c:\devt\vecdb\srvtest\shard2' + + options.(ShardFn ShardCols)←'{1+2|⊃⍵}' 1 ⍝ Odd/Even day number + params←name folder columns types options (5↑¨date sym price) ⍝ pre-populate symbols + TEST←'Create empty database' + db←{⎕NEW #.vecdb params} time ⍬ + assert 5=db.Count + db.Close - (2⊃data)×←1.1 ⍝ Add 10% to all prices - TEST←'Update prices' - z←db.Update time ix 'Price' (2⊃data) ⍝ Update price - assert data≡db.Read ix columns - - ⍝ /// Tests to do: - ⍝ Append data - to all and less that all shards - ⍝ Update multiple columns - ⍝ Read & update records with shards "out of order" - ⍝ Read & update not from all shards - - ⎕←'Closing down server...' - z←db.Shutdown 'Shutting down now!' - ⎕DL 3 - :If ~srvproc.HasExited ⋄ srvproc.Kill ⋄ :EndIf - ⎕DL 3 + ⍝ ↓↓↓ rewrite this only require "folder" and read the rest from JSON config + ∘∘∘ + srvproc←#.vecdbsrv.Launch folder 8100 ⍬'c:\devt\vecdb\vecdbboot.dws' + #.vecdbclt.Connect'127.0.0.1' 8100 'mkrom' + db←#.vecdbclt.Open folder + :EndSelect + + TEST←'Append Data' + z←db.Append time columns(5↓¨date sym price) - TEST←'Erase database' - tn←(folder,'meta.vecdb') ⎕FSTIE 0 - ⍝ ↓↓↓ Arrgh, need to rename shards with local view from PC and Mac - folders ⎕FREPLACE tn,6 - ⎕FUNTIE tn + TEST←'Count records' + assert(≢date)={db.Count}time ⍬ + + TEST←'Search for two days of records' + ix←db.Query time('Day'(200 201))⍬ ⍝ Should find 2,000 records + assert (+/date∊200 201)=≢∊ix[;2] + + TEST←'Read 2 days records (2,000)' + lix←{⍵/⍳⍴⍵}date∊200 201 + expect←{⍵[lix]}¨date sym price + assert expect≡db.Query time ('Day'(200 201)) columns + + TEST←'count Price, max Price, min Price group by Day (1st run)' + m←1 + expect←(m/date){(⊂⍺),(≢⍵),(⌈/⍵),⌊/⍵}⌸m/price + assert expect≡{⍵[⍋⍵[;1];]}db.Query time ⍬ ('count Price' 'max Price' 'min Price')'Day' - db←⎕NEW #.vecdb(,⊂folder) - assert 0={db.Erase}time ⍬ + TEST←'count Price, max Price, min Price group by Day (2nd run)' + assert expect≡{⍵[⍋⍵[;1];]}db.Query time ⍬ ('count Price' 'max Price' 'min Price')'Day' + + :If run≡'parallel' + ⎕←'Closing down server...' + z←db.Shutdown'Shutting down now!' + ⎕DL 3 + :If ~srvproc.HasExited ⋄ z←srvproc.Kill ⋄ :EndIf + ⎕DL 3 + :EndIf + + TEST←'Erase database' + db←⎕NEW #.vecdb(,⊂folder) + assert 0={db.Erase}time ⍬ + 1 ⎕NDELETE folder + + :EndFor z←'Server Tests Completed' ∇ - - ∇path←Init;source + + ∇ path←Init;source ⎕FUNTIE ⎕FNUMS ⋄ ⎕NUNTIE ⎕NNUMS :Trap 6 ⋄ source←SALT_Data.SourceFile :Else ⋄ source←⎕WSID :EndTrap - path←{(-⌊/(⌽⍵)⍳'\/')↓⍵}source - :If 0=⎕NC '#.DRC' ⋄ 'DRC' #.⎕CY 'conga' ⋄ :EndIf - ∇ + path←{(-⌊/(⌽⍵)⍳'\/')↓⍵}source + :If 0=⎕NC'#.DRC' ⋄ 'DRC'#.⎕CY'conga' ⋄ :EndIf + ∇ ∇ z←RunAll;path;source ⎕←ServerBasic ∇ - ∇ config←CreateBenchConfig filename;db;config;user;vecdbsrv;cmd;host;keyfile;userid - ⍝ - cmd←'RIDE_SPAWNED=1 RIDE_INIT=SERVE::5678 /Applications/Dyalog-15.0.app/Contents/Resources/Dyalog/mapl' - host←'Mortens-Macbook-Air' + ∇ config←CreateBenchConfig filename;db;config;user;vecdbsrv;cmd;host;keyfile;userid;port + ⍝ + cmd←'RIDE_SPAWNED=1 RIDE_INIT=SERVE::5678 /Applications/Dyalog-15.0.app/Contents/Resources/Dyalog/mapl' + host←'Mortens-Macbook-Air' userid←'mkrom' - keyfile←'c:\docs\personal\macbook-air' - - user←⎕NS '' + keyfile←'c:\docs\personal\macbook-air' + port←8100 + + user←⎕NS'' user.(Name Id Admin)←'mkrom' 1001 1 vecdbsrv←⎕NS'' vecdbsrv.Name←'Test Server' + vecdbsrv.BootWs←'c:\devt\vecdb\bootvecdb.dws' + vecdbsrv.Port←port vecdbsrv.Users←,user db←⎕NS'' db.Folder←folder db.Slaves←⎕NS¨2⍴⊂'' db.Slaves.Shards←,¨1 2 ⍝ Distribution of shards to slave processors - db.Slaves.Folder←⊂folder - db.Slaves[1].(Launch←⎕NS '').Type←'local' - db.Slaves[1].Folder←'//Mortens-Macbook-Air/vecdb/srvtest/' ⍝ If different seen from this slave - db.Slaves[2].(Launch←⎕NS '').(Type Host User KeyFile Cmd)←'ssh' host userid keyfile cmd - db.Slaves[2].Folder←'/Users/mkrom/vecdb/srvtest/' + db.Slaves.Folder←⊂folder ⍝ Root folder + db.Slaves[1].(Launch←⎕NS'').Type←'local' + db.Slaves[2].(Launch←⎕NS'').Type←'local' + db.Slaves.Folder←⊂folder ⍝ Local test only right now + ⍝ db.Slaves[1].Folder←'//Mortens-Macbook-Air/vecdb/srvtest/' ⍝ If different seen from this slave + ⍝ db.Slaves[2].(Launch←⎕NS'').(Type Host User KeyFile Cmd)←'ssh'host userid keyfile cmd + ⍝ db.Slaves[2].Folder←'/Users/mkrom/vecdb/srvtest/' + config←⎕NS'' config.Server←vecdbsrv config.DBs←,db (toJson config)⎕NPUT filename ∇ - + ∇ config←CreateTestConfig filename;db;config;user;vecdbsrv - ⍝ - user←⎕NS '' + ⍝ + user←⎕NS'' user.(Name Id Admin)←'mkrom' 1001 1 vecdbsrv←⎕NS'' vecdbsrv.Name←'Test Server' @@ -123,28 +164,13 @@ db←⎕NS'' db.Folder←folder db.Slaves←⎕NS¨2⍴⊂'' - db.Slaves.Shards←,¨1 2 ⍝ Distribution of shards to slave processors - db.Slaves.(Launch←⎕NS '').Type←⊂'local' + db.Slaves.Shards←,¨1 2 ⍝ Distribution of shards to slave processors + db.Slaves.(Launch←⎕NS'').Type←⊂'local' config←⎕NS'' config.Server←vecdbsrv config.DBs←,db (toJson config)⎕NPUT filename ∇ - - ∇ (db params)←CreateBenchDB;columns;types;data;options - columns←'Name' 'Price' 'Flag' - types←,¨'C' 'F' 'C' - data←('IBM' 'AAPL' 'MSFT' 'GOOG' 'DYALOG')(160.97 112.6 47.21 531.23 999.99)(5⍴'Buy' 'Sell') - - options←⎕NS'' - options.BlockSize←10000 - options.ShardFolders←'c:\devt\vecdb\srvtest\shard1' '//Mortens-Macbook-Air/vecdb/srvtest/shard2' - options.(ShardFn ShardCols)←'{2-2|⎕UCS ⊃¨⊃⍵}' 1 - - params←name folder columns types options data - db←⎕NEW #.vecdb params - assert (≢⊃data)=db.Count - ∇ ∇ (db params)←CreateTestDB;columns;types;data;options columns←'Name' 'Price' 'Flag' @@ -158,53 +184,53 @@ params←name folder columns types options data db←⎕NEW #.vecdb params - assert (≢⊃data)=db.Count + assert(≢⊃data)=db.Count ∇ ∇ z←ServerBasic;columns;data;options;params;folder;types;name;ix;users;srvproc;clt;TEST;config;db;path ⍝ Test database with 2 shards ⍝ Also acts as test for add/remove columns - - path←Init + + path←Init folder←path,'/',(name←'srvtest'),'/' ⎕←'Clearing: ',folder :Trap 22 ⋄ {}#.vecdb.Delete folder ⋄ :EndTrap ⎕MKDIR folder ⍝ --- Create configuration file --- - + config←CreateBenchConfig folder,'config.json' - (db (name folder columns types options data))←CreateTestDB - + (db(name folder columns types options data))←CreateTestDB + ⍝ --- Launch and connect to server, open database --- - - srvproc←#.vecdbsrv.Launch folder 8100 - #.vecdbclt.Connect '127.0.0.1' 8100 'mkrom' + + srvproc←#.vecdbsrv.Launch folder 8100 + #.vecdbclt.Connect'127.0.0.1' 8100 'mkrom' db←#.vecdbclt.Open folder - + TEST←'Count records' - assert (≢⊃data)={db.Count} time ⍬ - + assert(≢⊃data)={db.Count}time ⍬ + TEST←'Search for all records' - ix←db.Query time ('Name'((columns⍳⊂'Name')⊃data))⍬ ⍝ Should find everything + ix←db.Query time('Name'((columns⍳⊂'Name')⊃data))⍬ ⍝ Should find everything assert(1 2,⍪⍳¨4 1)≡ix - + TEST←'Read it all back' assert data≡db.Read time ix columns - + (2⊃data)×←1.1 ⍝ Add 10% to all prices TEST←'Update prices' - z←db.Update time ix 'Price' (2⊃data) ⍝ Update price + z←db.Update time ix'Price'(2⊃data) ⍝ Update price assert data≡db.Read ix columns - - ⍝ /// Tests to do: + + ⍝ /// Tests to do: ⍝ Append data - to all and less that all shards ⍝ Update multiple columns ⍝ Read & update records with shards "out of order" ⍝ Read & update not from all shards - - ⎕←'Closing down server...' - z←db.Shutdown 'Shutting down now!' + + ⎕←'Closing down server...' + z←db.Shutdown'Shutting down now!' ⎕DL 3 :If ~srvproc.HasExited ⋄ srvproc.Kill ⋄ :EndIf ⎕DL 3 @@ -238,6 +264,6 @@ 0::⎕SIGNAL(⍺≡⊃⎕DMX.DM)↓11 z←⍺⍺ ⍵ ⎕SIGNAL 11 - } + } :EndNamespace diff --git a/vecdb.dyalog b/vecdb.dyalog index 86138d3..f0a8288 100644 --- a/vecdb.dyalog +++ b/vecdb.dyalog @@ -21,7 +21,8 @@ :Field Public NumBlocks←1 ⍝ We start with one block :Field Public noFiles←0 ⍝ in-memory database (not supported) :Field Public isOpen←0 ⍝ Not yet open - :Field Public ShardFolders←⍬ ⍝ List of Shard Folders + :Field Public ShardFolders←⍬ ⍝ List of Shard Folders + :Field Public LocalFolders←⍬ ⍝ Shard Folders as seen by slave task :Field Public ShardFn←⍬ ⍝ Shard Calculation Function :Field Public ShardCols←⍬ ⍝ ShardFn input column indices :Field Public ShardSelected←⍬ ⍝ Shards selected @@ -227,7 +228,11 @@ :Trap 0 ⋄ tn←(file←folder,'meta.vecdb')⎕FSTIE 0 :Else ⋄ ('Unable to open ',file)⎕SIGNAL 11 :EndTrap - (props(_Columns _Types)ShardFolders(ShardFn ShardCols))←⎕FREAD tn(4 5 6 7) + (props(_Columns _Types)ShardFolders(ShardFn ShardCols))←⎕FREAD tn(4 5 6 7) + + :If 1=⍴⍴ShardFolders ⋄ LocalFolders←ShardFolders ⍝ Old format: no local folders + :Else ⋄ ShardFolders LocalFolders←↓ShardFolders ⋄ :EndIf ⍝ New format allows spec of both + n←≢_Columns mappings←⎕NS¨n⍴⊂'' mappings.Type←0 ⍝ Not cal'd or mapped (yet) @@ -238,17 +243,19 @@ s←≢ShardFolders :If 0=⍴shards ⍝ No explicit selection of shards - ShardSelected←⍳s + ShardSelected←⍳s + shardfolders←ShardFolders :Else 'Invalid Shard Selection'⎕SIGNAL(∧/shards∊⍳s)↓11 - ShardSelected←shards + ShardSelected←shards + shardfolders←LocalFolders :EndIf AllShards←s=≢ShardSelected Shards←⎕NS¨¨s⍴⊂n⍴⊂'' Shards.name←s⍴⊂_Columns ⍝ Column Names Shards.type←s⍴⊂_Types ⍝ Types - Shards.file←(n/¨⊂¨ShardFolders),¨¨(,∘'.vector')¨¨s⍴⊂(⍕¨⍳n) ⍝ Vector file names + Shards.file←(n/¨⊂¨shardfolders),¨¨(,∘'.vector')¨¨s⍴⊂(⍕¨⍳n) ⍝ Vector file names :If 0≠⍴ShardFn ⋄ findshard←⍎ShardFn ⋄ :EndIf ⍝ Define shard calculation function @@ -271,7 +278,7 @@ :For i :In ShardSelected s←i⊃Shards - _Counts[i].counter←645 1 ⎕MAP((i⊃ShardFolders),'counters.vecdb')'W' ⍝ Map record counter + _Counts[i].counter←645 1 ⎕MAP((i⊃shardfolders),'counters.vecdb')'W' ⍝ Map record counter :For col :In ⍳≢s (col⊃s).vector←(types[col],¯1)⎕MAP(col⊃s).file'W' :EndFor @@ -287,12 +294,14 @@ :Implements constructor :Access Public 0 CreateOrExtend name folder columns types'' '' ⍝ No data or option + Open,⊂folder ⍝ now open it properly ∇ ∇ make5(name folder columns types options) :Implements constructor :Access Public 0 CreateOrExtend name folder columns types options'' ⍝ No data or option + Open,⊂folder ⍝ now open it properly ∇ ∇ make6(name folder columns types options data) @@ -332,7 +341,10 @@ ⍝ Set defaults for sharding (1 shard) ShardFolders,←(0=⍴ShardFolders)/⊂folder - ShardFolders←AddSlash¨ShardFolders + ShardFolders←AddSlash¨ShardFolders + :If 0=⎕NC 'LocalFolders' ⋄ LocalFolders←ShardFolders ⋄ :EndIf + LocalFolders←AddSlash¨LocalFolders + shardfolders←ShardFolders ⍝ When creating we have the full view ShardCols←,ShardCols :If 0≠⍴ShardFn ⋄ findshard←⍎ShardFn ⋄ :EndIf ⍝ Define shard calculation function (Name _Columns _Types)←name columns types ⍝ Set Class fields @@ -382,9 +394,11 @@ tn←(filename←sf,(⍕i),'.vector')⎕NCREATE 0 (sizeOf size dr)⎕NRESIZE tn ⎕NUNTIE tn + :If 0≠≢⊃d ⍝ if there is some data to write temp←dr ¯1 ⎕MAP filename'W' temp[]←size↑(newcols⍳i)⊃d - ⎕EX'temp' + ⎕EX'temp' + :EndIf ⍝ 'col ',(⍕i),': ',⍕⎕ai[3]-ai3 :EndFor :EndFor @@ -396,7 +410,7 @@ 'unused'∆FAPPEND tn ⍝ 3 (fileprops(⍎¨fileprops))∆FAPPEND tn ⍝ 4 (Name BlockSize) (_Columns _Types)∆FAPPEND tn ⍝ 5 - ShardFolders ∆FAPPEND tn ⍝ 6 + ((2,≢ShardFolders)⍴ShardFolders,LocalFolders) ∆FAPPEND tn ⍝ 6 (ShardFn ShardCols)∆FAPPEND tn ⍝ 7 'unused'∆FAPPEND tn ⍝ 8 'unused'∆FAPPEND tn ⍝ 9 @@ -410,16 +424,21 @@ ∇ ∇X ∆FAPPEND Y - X ⎕FAPPEND Y - ⎕FUNTIE ⍬ + ⍝ Work-around for Samba on Mac + X ⎕FAPPEND Y + ⎕FUNTIE ⍬ ∇ ∇ (shards data)←cix ShardData data;six;s;char;rawdata;sym;c;counts;m ⍝ Shards is a vector of shards to be updated ⍝ data has one column per shard, and one row per column - rawdata←data - :If 1=≢ShardFolders ⍝ Data will necessarily all be in the 1st shard then! + :If 0=≢⊃rawdata←data + shards←⍬ ⋄ data←0/⍪data + →0 + :EndIf + + :If 1=≢shardfolders ⍝ Data will necessarily all be in the 1st shard then! shards←,1 ⋄ data←⍪data :Else ⍝ Database *is* sharded @@ -487,7 +506,7 @@ :If 9=⎕NC'options' :For name :In options.⎕NL-2 - :If (⊂name)∊'BlockSize' 'InitBlocks' 'Folders' 'ShardCols' 'ShardFolders' 'ShardFn' + :If (⊂name)∊'BlockSize' 'InitBlocks' 'Folders' 'ShardCols' 'ShardFolders' 'LocalFolders' 'ShardFn' ⍎name,'←options.',name :Else ('Invalid option name: ',name)⎕SIGNAL 11 @@ -532,9 +551,9 @@ ⎕EX'Shards' ⍝ We will reopen the file at the end, need to remove maps - :For f :In ⍳≢ShardFolders + :For f :In ⍳≢shardfolders colix←1 - sf←f⊃ShardFolders + sf←f⊃shardfolders :For c :In ⍳≢_Columns tn←(file←sf,(⍕c),'.vector')⎕NTIE 0 sym←{22::0 ⋄ (Folder,(⍕c),'.symbol')⎕NTIE ⍵}0 @@ -786,7 +805,7 @@ append←(≢_Columns)⍴⊂⍬ append[cix]←canupdate↓¨d ⍝ Data which was not updated growth←BlockSize×(length-canupdate)(⌈÷)BlockSize ⍝ How many records to add to the Shard - ExtendShard(s⊃ShardFolders)Cols growth append + ExtendShard(s⊃shardfolders)Cols growth append :EndIf _Counts[s].counter[1]←count+length ⍝ Update (mapped) counter @@ -910,9 +929,10 @@ :If (⊂APLVersion)∊'*nix' 'Mac' ⋄ ((f='\')/f)←'/' ⋄ :EndIf ∇ - ∇ r←AddSlash path - ⍝ Ensure folder name has trailing slash - r←path,((¯1↑path)∊'/\')↓⊃isWindows⌽'/\' + ∇ r←AddSlash path;slash + ⍝ Ensure folder name has trailing slash + slash←⊃('/\'∩path),'/' ⍝ use / unless path ONLY contains \ + r←path,((¯1↑path)∊'/\')↓slash ∇ ∇ r←Exists path;GFA diff --git a/vecdbboot.dws b/vecdbboot.dws index fe51ca7..d44ae41 100644 Binary files a/vecdbboot.dws and b/vecdbboot.dws differ diff --git a/vecdbclt.dyalog b/vecdbclt.dyalog index 13d17ff..f21b6f7 100644 --- a/vecdbclt.dyalog +++ b/vecdbclt.dyalog @@ -31,44 +31,44 @@ :EndIf ∇ - ∇ r←SrvDo (client cmd) + ∇ r←SrvDo(client cmd) ⍝ Send a command to vecdb and await the result r←SrvRcv SrvSend client cmd ∇ - ∇cmd←SrvSend (client cmd);r + ∇ cmd←SrvSend(client cmd);r ⍝ Return command name to wait on :If 0=⊃r←##.DRC.Send client cmd cmd←2⊃r :Else - (⍕r) ⎕SIGNAL 11 + (⍕r)⎕SIGNAL 11 :EndIf ∇ ∇ r←SrvRcv c;done;wr;z ⍝ Wait for result from vecdb, signal DOMAIN ERROR if it fails - :Repeat - :If ~done←∧/100 0≠1⊃r←##.DRC.Wait c 10000 ⍝ Only wait 10 seconds + :Repeat + :If ~done←∧/100 0≠1⊃r←##.DRC.Wait c 10000 ⍝ Only wait 10 seconds - :Select 3⊃r - :Case 'Error' + :Select 3⊃r + :Case 'Error' + done←1 + :Case 'Progress' + ⎕←'Progress: ',4⊃r + :Case 'Receive' + :If 0=⊃r + r←4⊃r + :AndIf 0=⊃r + r←2⊃r done←1 - :Case 'Progress' - ⎕←'Progress: ',4⊃r - :Case 'Receive' - :If 0=⊃r - r←4⊃r - :AndIf 0=⊃r - r←2⊃r - done←1 - :Else - ('Error: ',,⍕r)⎕SIGNAL 11 - :EndIf - :EndSelect - :EndIf - :Until done + :Else + ('Error: ',,⍕r)⎕SIGNAL 11 + :EndIf + :EndSelect + :EndIf + :Until done ∇ ∇ r←Open folder @@ -93,7 +93,7 @@ ∇ {r}←Shutdown msg :Access Public - :If 0=⊃r←##.SrvDo CONNECTION('Shutdown' msg) + :If 0=⊃r←##.SrvDo CONNECTION('Shutdown'msg) {}#.DRC.Close CONNECTION CONNECTION←'' :EndIf @@ -101,7 +101,7 @@ ∇ Close :Access Public - :If 0=⊃r←##.SrvDo CONNECTION('Close' ⍬) + :If 0=⊃r←##.SrvDo CONNECTION('Close'⍬) {}#.DRC.Close CONNECTION CONNECTION←'' :EndIf @@ -110,50 +110,54 @@ ∇ r←Count :Access Public :If 0≠⍴CONNECTION - r←##.SrvDo CONNECTION('Count' (FOLDER ⍬)) - r←+/r + r←##.SrvDo CONNECTION('Count'(FOLDER ⍬)) + r←+/r :Else - 'CONNECTION CLOSED' ⎕SIGNAL 11 + 'CONNECTION CLOSED'⎕SIGNAL 11 :EndIf ∇ ∇ r←Append args :Access Public :If 0≠⍴CONNECTION - r←##.SrvDo CONNECTION('Append' (FOLDER args)) + r←##.SrvDo CONNECTION('Append'(FOLDER args)) :Else - 'CONNECTION CLOSED' ⎕SIGNAL 11 + 'CONNECTION CLOSED'⎕SIGNAL 11 :EndIf ∇ ∇ r←Query args :Access Public :If 0≠⍴CONNECTION - r←##.SrvDo CONNECTION('Query' (FOLDER args)) - r←⊃⍪/r + r←##.SrvDo CONNECTION('Query'(FOLDER args)) + :If 2=⍴⍴⊃r + r←⊃⍪/r + :Else + r←⊃,¨/r + :EndIf :Else - 'CONNECTION CLOSED' ⎕SIGNAL 11 + 'CONNECTION CLOSED'⎕SIGNAL 11 :EndIf ∇ ∇ r←Read args :Access Public :If 0≠⍴CONNECTION - r←##.SrvDo CONNECTION('Read' (FOLDER args)) - r←⊃,¨/r + r←##.SrvDo CONNECTION('Read'(FOLDER args)) + r←⊃,¨/r :Else - 'CONNECTION CLOSED' ⎕SIGNAL 11 + 'CONNECTION CLOSED'⎕SIGNAL 11 :EndIf ∇ ∇ r←Update args - :Access Public + :Access Public :If 0≠⍴CONNECTION - r←##.SrvDo CONNECTION('Update' (FOLDER args)) + r←##.SrvDo CONNECTION('Update'(FOLDER args)) :Else - 'CONNECTION CLOSED' ⎕SIGNAL 11 + 'CONNECTION CLOSED'⎕SIGNAL 11 :EndIf - + ∇ :EndClass diff --git a/vecdbslave.dyalog b/vecdbslave.dyalog index 7a348e0..9ae6904 100644 --- a/vecdbslave.dyalog +++ b/vecdbslave.dyalog @@ -92,8 +92,9 @@ r←0 (⍎cmd,' obj arg') :ElseIf (⊂cmd)∊'Append' 'Count' 'Query' 'Update' 'Read' - :If 0≠≢(CONNS⍳⊂Conn)⊃TOKENS,⊂'' - :Trap 9999 + :If 0≠≢(CONNS⍳⊂Conn)⊃TOKENS,⊂'' + :Trap 9999 + ∘∘∘ :If cmd≡'Count' ⋄ r←0 DB.Count :Else ⋄ r←0 ((DB⍎cmd) arg) :EndIf diff --git a/vecdbsrv.dyalog b/vecdbsrv.dyalog index 17e2f03..b8d57e3 100644 --- a/vecdbsrv.dyalog +++ b/vecdbsrv.dyalog @@ -41,8 +41,9 @@ r←⍬ ⍝ Need a result ∇ - ∇ {r}←Init config;db;i;j;slave;path;ws - ⍝ Intialise the vecdb server + ∇ {r}←Init config;db;i;j;slave;path;ws;tn + ⍝ Intialise the vecdb server + ∘∘∘ :Trap 6 ⋄ source←SALT_Data.SourceFile :Else ⋄ source←⎕WSID @@ -56,10 +57,14 @@ (DBs Server)←config.(DBs Server) TOKEN←{⎕RL←0 ⋄ ⎕PP←10 ⋄ 2↓⍕?0}0 - DBFolders←DBs.Folder + DBFolders←DBs.Folder :For i :In ⍳≢DBs ⍝ Launch all the processes - db←i⊃DBs + db←i⊃DBs + tn←(db.Folder,'meta.vecdb') ⎕FSTIE 0 + db.((_Columns _Types) (ShardFn ShardCols))←⎕FREAD tn (5 7) + ⎕funtie tn + :For j :In ⍳≢db.Slaves slave←j⊃db.Slaves slave.Port←NEXTPORT @@ -98,8 +103,10 @@ r←⍬ ⍝ Need a result ∇ - ∇ (sdata is)←SlavePartition(cmd slaves data);ixs;cols;new;slave;slave_recs - + ∇ (sdata is)←SlavePartition(cmd slaves data);ixs;cols;new;slave;slave_recs;cix + ⍝ Return sdata: command to send to each slave + ⍝ is: indices into slaves + Shards←∊slaves.Shards ShardSlaves←(≢¨slaves.Shards)/⍳≢slaves.Shards @@ -119,7 +126,17 @@ sdata←↓(slave{⊂⍵}⌸ixs),(⊂cols),⍪{slave_recs{⊂⍵}⌸⍵}¨new ∘∘∘ :EndIf - is←⍳≢sdata + is←⍳≢sdata + :Case 'Append' + (cols new)←data + :If ∧/db.ShardCols∊cix←db._Columns⍳cols + slave←ShardSlaves⍳(⍎db.ShardFn) new[cix⍳db.ShardCols] + sdata←{slave{⍺,⊂⍵}⌸⍵}¨new + is←(⊃sdata)[;1] ⍝ slave indices + sdata←↓(⊂cols),⍪↓⊃,/0 1∘↓¨sdata ⍝ data pertaining to each slave + :Else + ∘∘∘ ⍝ Unable to shard the data + :EndIf :Else sdata←,⊂data ⋄ is←(≢slaves)/1 :EndSelect @@ -143,7 +160,7 @@ :ElseIf (⊂cmd)∊'Append' 'Count' 'Query' 'Update' 'Read' :If (≢DBs)