Skip to content

Commit d300138

Browse files
prataprcprataprc
committed
make Generatecreate() to return unique keys.
That way it is easy to match the counts. Also, updated Generateupdate, Generateread, Generatereadseq, Generatedelete.
1 parent 34a9cd3 commit d300138

File tree

9 files changed

+114
-59
lines changed

9 files changed

+114
-59
lines changed

‎generate.go

Lines changed: 14 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -66,16 +66,19 @@ func Generateloadr(
6666

6767
// Generate keys greater than loadn, always return unique keys.
6868
func Generatecreate(
69-
klen, vlen, loadn, seed int64) func(k, v []byte) ([]byte, []byte) {
69+
klen, vlen, loadn, insertn,
70+
seed int64) func(k, v []byte) ([]byte, []byte) {
7071

7172
var textint [1024]byte
7273

7374
loadn = int64(loadn * rndscale)
74-
intn := int64(9223372036854775807) - loadn
75+
intn := (insertn * rndscale)
7576
rnd := rand.New(rand.NewSource(seed))
77+
bitmap := make([]byte, ((intn / 8) + 1))
7678

7779
return func(key, value []byte) ([]byte, []byte) {
78-
keynum := int64(rnd.Intn(int(intn))) + loadn
80+
keynum := makeuniquekey(rnd, bitmap, intn)
81+
keynum += loadn
7982
ascii := strconv.AppendInt(textint[:0], int64(keynum), 10)
8083
// create key
8184
key = Fixbuffer(key, int64(klen))
@@ -91,14 +94,14 @@ func Generatecreate(
9194
}
9295

9396
func Generateupdate(
94-
klen, vlen, loadn,
97+
klen, vlen, loadn, insertn,
9598
seedl, seedc, mod int64) func(k, v []byte) ([]byte, []byte) {
9699

97100
var textint [1024]byte
98101
var getkey func()
99102

100103
loadn1 := loadn * rndscale
101-
intn := int64(9223372036854775807) - loadn1
104+
intn := insertn * rndscale
102105
rndl := rand.New(rand.NewSource(seedl))
103106
rndc := rand.New(rand.NewSource(seedc))
104107
keynum, lcount := int64(0), int64(0)
@@ -136,12 +139,13 @@ func Generateupdate(
136139
}
137140
}
138141

139-
func Generateread(klen, loadn, seedl, seedc int64) func([]byte, int64) []byte {
142+
func Generateread(
143+
klen, loadn, insertn, seedl, seedc int64) func([]byte, int64) []byte {
140144
var textint [1024]byte
141145
var getkey func(int64)
142146

143147
loadn1 := loadn * rndscale
144-
intn := int64(9223372036854775807) - loadn1
148+
intn := insertn * rndscale
145149
rndl := rand.New(rand.NewSource(seedl))
146150
rndc := rand.New(rand.NewSource(seedc))
147151
keynum, lcount := int64(0), int64(0)
@@ -196,13 +200,14 @@ func Generatereadseq(klen, loadn, seedl int64) func([]byte, int64) []byte {
196200

197201
func Generatedelete(
198202
klen, vlen,
199-
loadn, seedl, seedc, mod int64) func(k, v []byte) ([]byte, []byte) {
203+
loadn, insertn,
204+
seedl, seedc, mod int64) func(k, v []byte) ([]byte, []byte) {
200205

201206
var textint [1024]byte
202207
var getkey func()
203208

204209
loadn1 := loadn * rndscale
205-
intn := int64(9223372036854775807) - loadn1
210+
intn := insertn * rndscale
206211
rndl := rand.New(rand.NewSource(seedl))
207212
rndc := rand.New(rand.NewSource(seedc))
208213
keynum, lcount := int64(0), int64(0)

‎generate_test.go

Lines changed: 15 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -92,7 +92,7 @@ func TestGenerateCRUD(t *testing.T) {
9292
// Create load
9393
klen, vlen = int64(32), int64(32)
9494
createn, seedc := int64(2000000), int64(200)
95-
g = Generatecreate(klen, vlen, loadn, seedc)
95+
g = Generatecreate(klen, vlen, loadn, createn, seedc)
9696
key, value := g(nil, nil)
9797
for i := int64(0); i < createn; i++ {
9898
keynum, err := strconv.ParseInt(Bytes2str(key), 10, 64)
@@ -115,7 +115,7 @@ func TestGenerateCRUD(t *testing.T) {
115115
readl, readc := map[int64]bool{}, map[int64]bool{}
116116
// read load
117117
klen, readn := int64(32), int64(2000000)
118-
gr := Generateread(klen, loadn, seedl, seedc)
118+
gr := Generateread(klen, loadn, createn, seedl, seedc)
119119
key = gr(nil, int64(len(createm)))
120120
for i := int64(0); i < readn; i++ {
121121
keynum, err := strconv.ParseInt(Bytes2str(key), 10, 64)
@@ -132,14 +132,14 @@ func TestGenerateCRUD(t *testing.T) {
132132
}
133133
if len(readl) != 850300 {
134134
t.Fatalf("%v != %v", len(readl), 850300)
135-
} else if len(readc) != 500000 {
136-
t.Fatalf("%v != %v", len(readc), 500000)
135+
} else if len(readc) != 479726 {
136+
t.Fatalf("%v != %v", len(readc), 479726)
137137
}
138138

139139
updatel, updatec := map[int64]bool{}, map[int64]bool{}
140140
// update load
141141
klen, vlen, updaten := int64(32), int64(32), int64(2000000)
142-
g = Generateupdate(klen, vlen, loadn, seedl, seedc, -1)
142+
g = Generateupdate(klen, vlen, loadn, createn, seedl, seedc, -1)
143143
key, value = g(nil, nil)
144144
for i := int64(0); i < updaten; i++ {
145145
keynum, err := strconv.ParseInt(Bytes2str(key), 10, 64)
@@ -156,14 +156,14 @@ func TestGenerateCRUD(t *testing.T) {
156156
}
157157
if len(updatel) != 850300 {
158158
t.Fatalf("%v != %v", len(updatel), 850300)
159-
} else if len(updatec) != 333333 {
160-
t.Fatalf("%v != %v", len(updatec), 333333)
159+
} else if len(updatec) != 324297 {
160+
t.Fatalf("%v != %v", len(updatec), 324297)
161161
}
162162

163163
deletel, deletec := map[int64]bool{}, map[int64]bool{}
164164
// delete load
165165
klen, deleten := int64(32), int64(1000000)
166-
gd := Generatedelete(klen, vlen, loadn, seedl, seedc, delmod)
166+
gd := Generatedelete(klen, vlen, loadn, createn, seedl, seedc, delmod)
167167
key, value = gd(nil, nil)
168168
for i := int64(0); i < deleten; i++ {
169169
keynum, err := strconv.ParseInt(Bytes2str(key), 10, 64)
@@ -180,8 +180,8 @@ func TestGenerateCRUD(t *testing.T) {
180180
}
181181
if len(deletel) != 424961 {
182182
t.Fatalf("%v != %v", len(deletel), 424961)
183-
} else if len(deletec) != 166235 {
184-
t.Fatalf("%v != %v", len(deletec), 166235)
183+
} else if len(deletec) != 162202 {
184+
t.Fatalf("%v != %v", len(deletec), 162202)
185185
}
186186
}
187187

@@ -204,8 +204,8 @@ func BenchmarkGenerateloadr(b *testing.B) {
204204
}
205205

206206
func BenchmarkGeneratecreate(b *testing.B) {
207-
klen, vlen, n, seedc := int64(32), int64(32), int64(1*1000*1000), int64(100)
208-
g := Generatecreate(klen, vlen, n, seedc)
207+
klen, vlen, seedc := int64(32), int64(32), int64(100)
208+
g := Generatecreate(klen, vlen, 0, int64(b.N), seedc)
209209
key, value := g(nil, nil)
210210
for i := 0; i < b.N; i++ {
211211
key, value = g(key, value)
@@ -215,7 +215,7 @@ func BenchmarkGeneratecreate(b *testing.B) {
215215
func BenchmarkGenerateupdate(b *testing.B) {
216216
klen, vlen, n := int64(32), int64(32), int64(1*1000*1000)
217217
seedl, seedc := int64(100), int64(200)
218-
g := Generateupdate(klen, vlen, n, seedl, seedc, -1)
218+
g := Generateupdate(klen, vlen, n, int64(b.N), seedl, seedc, -1)
219219
key, value := g(nil, nil)
220220
for i := 0; i < b.N; i++ {
221221
key, value = g(key, value)
@@ -225,7 +225,7 @@ func BenchmarkGenerateupdate(b *testing.B) {
225225
func BenchmarkGenerateread(b *testing.B) {
226226
klen, n := int64(32), int64(1*1000*1000)
227227
seedl, seedc := int64(100), int64(200)
228-
g := Generateread(klen, n, seedl, seedc)
228+
g := Generateread(klen, n, int64(b.N), seedl, seedc)
229229
key := g(nil, 0)
230230
for i := 0; i < b.N; i++ {
231231
g(key, 0)
@@ -235,7 +235,7 @@ func BenchmarkGenerateread(b *testing.B) {
235235
func BenchmarkGeneratedelete(b *testing.B) {
236236
klen, n := int64(32), int64(1*1000*1000)
237237
seedl, seedc := int64(100), int64(200)
238-
g := Generatedelete(klen, klen, n, seedl, seedc, delmod)
238+
g := Generatedelete(klen, klen, n, int64(b.N), seedl, seedc, delmod)
239239
key, value := g(nil, nil)
240240
for i := 0; i < b.N; i++ {
241241
g(key, value)

‎mod_badger.go

Lines changed: 13 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -140,9 +140,13 @@ func badgerWriter(
140140
fin chan struct{}, wg *sync.WaitGroup) {
141141

142142
var x, y, z int64
143-
gcreate := Generatecreate(klen, vlen, loadn, seedc)
144-
gupdate := Generateupdate(klen, vlen, loadn, seedl, seedc, -1)
145-
gdelete := Generatedelete(klen, vlen, loadn, seedl, seedc, delmod)
143+
gcreate := Generatecreate(klen, vlen, loadn, int64(options.inserts), seedc)
144+
gupdate := Generateupdate(
145+
klen, vlen, loadn, int64(options.inserts), seedl, seedc, -1,
146+
)
147+
gdelete := Generatedelete(
148+
klen, vlen, loadn, int64(options.inserts), seedl, seedc, delmod,
149+
)
146150

147151
cmds := make([]*badgerop, 500)
148152
for off := range cmds {
@@ -254,7 +258,9 @@ func badgerGetter(
254258
time.Sleep(time.Duration(rand.Intn(100)+300) * time.Millisecond)
255259

256260
var key []byte
257-
g := Generateread(int64(options.keylen), loadn, seedl, seedc)
261+
g := Generateread(
262+
int64(options.keylen), loadn, int64(options.inserts), seedl, seedc,
263+
)
258264

259265
get := func(txn *badger.Txn) (err error) {
260266
if _, err = txn.Get(key); err != nil {
@@ -299,7 +305,9 @@ func badgerRanger(
299305
time.Sleep(time.Duration(rand.Intn(100)+300) * time.Millisecond)
300306

301307
var key []byte
302-
g := Generateread(int64(options.keylen), loadn, seedl, seedc)
308+
g := Generateread(
309+
int64(options.keylen), loadn, int64(options.inserts), seedl, seedc,
310+
)
303311

304312
ranger := func(txn *badger.Txn) error {
305313
opts := badger.DefaultIteratorOptions

‎mod_bogn.go

Lines changed: 19 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -115,9 +115,13 @@ func bognWriter(
115115
var x, y, z int64
116116

117117
klen, vlen := int64(options.keylen), int64(options.vallen)
118-
gcreate := Generatecreate(klen, vlen, n, seedc)
119-
gupdate := Generateupdate(klen, vlen, n, seedl, seedc, -1)
120-
gdelete := Generatedelete(klen, vlen, n, seedl, seedc, delmod)
118+
gcreate := Generatecreate(klen, vlen, n, int64(options.inserts), seedc)
119+
gupdate := Generateupdate(
120+
klen, vlen, n, int64(options.inserts), seedl, seedc, -1,
121+
)
122+
gdelete := Generatedelete(
123+
klen, vlen, n, int64(options.inserts), seedl, seedc, delmod,
124+
)
121125

122126
value, oldvalue := make([]byte, vlen), make([]byte, vlen)
123127
if options.vallen <= 0 {
@@ -128,6 +132,7 @@ func bognWriter(
128132
insn, upsn, deln := options.inserts, options.upserts, options.deletes
129133

130134
as, bs := bognsets[options.setas], bogndels[options.delas]
135+
data := map[string]bool{}
131136
for totalops := insn + upsn + deln; totalops > 0; {
132137
bognset := as[rnd.Intn(len(as))]
133138
bogndel := bs[rnd.Intn(len(bs))]
@@ -136,22 +141,22 @@ func bognWriter(
136141
switch {
137142
case idx < insn:
138143
key, value = gcreate(key, value)
139-
//fmt.Printf("create %s %s\n", key, value)
144+
data[string(key)] = true
140145
bognset(index, key, value, oldvalue)
141146
atomic.AddInt64(&numentries, 1)
142147
x = atomic.AddInt64(&ninserts, 1)
143148
insn--
144149

145150
case idx < (insn + upsn):
146151
key, value = gupdate(key, value)
147-
//fmt.Printf("update %s %s\n", key, value)
152+
fmt.Printf("update %s %s\n", key, value)
148153
bognset(index, key, value, oldvalue)
149154
y = atomic.AddInt64(&nupserts, 1)
150155
upsn--
151156

152157
case idx < (insn + upsn + deln):
153158
key, value = gdelete(key, value)
154-
//fmt.Printf("delete %s %s\n", key, value)
159+
fmt.Printf("delete %s %s\n", key, value)
155160
bogndel(index, key, value, options.lsm /*lsm*/)
156161
atomic.AddInt64(&numentries, -1)
157162
z = atomic.AddInt64(&ndeletes, 1)
@@ -170,6 +175,7 @@ func bognWriter(
170175
now = time.Now()
171176
}
172177
}
178+
fmt.Println("total", len(data))
173179
took := time.Since(epoch).Round(time.Second)
174180
wg.Done()
175181
<-fin
@@ -319,7 +325,9 @@ func bognGetter(
319325

320326
var ngets, nmisses int64
321327
var key []byte
322-
g := Generateread(int64(options.keylen), n, seedl, seedc)
328+
g := Generateread(
329+
int64(options.keylen), n, int64(options.inserts), seedl, seedc,
330+
)
323331

324332
rnd := rand.New(rand.NewSource(int64(seedl)))
325333
value := make([]byte, options.vallen)
@@ -420,7 +428,9 @@ func bognRanger(
420428

421429
var nranges int64
422430
var key []byte
423-
g := Generateread(int64(options.keylen), n, seedl, seedc)
431+
g := Generateread(
432+
int64(options.keylen), n, int64(options.inserts), seedl, seedc,
433+
)
424434

425435
rnd := rand.New(rand.NewSource(int64(seedl)))
426436
value := make([]byte, options.vallen)
@@ -560,7 +570,7 @@ func bognsettings(seed int) s.Settings {
560570
setts["memstore"] = options.memstore
561571
//setts["flushratio"] = flushratios[rnd.Intn(10000)%len(flushratios)]
562572
setts["flushratio"] = 0.25
563-
setts["flushperiod"] = int64(options.period)
573+
setts["flushperiod"] = 5 // int64(options.period)
564574
setts["bubt.mmap"] = []bool{true, false}[rnd.Intn(10000)%2]
565575
//setts["bubt.msize"] = msizes[rnd.Intn(10000)%len(msizes)]
566576
//setts["bubt.zsize"] = zsizes[rnd.Intn(10000)%len(msizes)]

‎mod_bogn_dgm_initial.sh

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
#! /usr/bin/env bash
22

33
ARGS="-db bogn -bogn dgm -klen 32 -vlen 200 -memcap 4096 -log bogn"
4-
UPSERTS="-inserts 30000000 -setas set"
4+
UPSERTS="-inserts 1000000 -setas set"
55

66
rm -rf *.svg dbperf; go build
77

‎mod_bolt.go

Lines changed: 13 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -138,9 +138,13 @@ func boltWriter(
138138

139139
var key, value []byte
140140
var x, y, z int64
141-
gcreate := Generatecreate(klen, vlen, loadn, seedc)
142-
gupdate := Generateupdate(klen, vlen, loadn, seedl, seedc, -1)
143-
gdelete := Generatedelete(klen, vlen, loadn, seedl, seedc, delmod)
141+
gcreate := Generatecreate(klen, vlen, loadn, int64(options.inserts), seedc)
142+
gupdate := Generateupdate(
143+
klen, vlen, loadn, int64(options.inserts), seedl, seedc, -1,
144+
)
145+
gdelete := Generatedelete(
146+
klen, vlen, loadn, int64(options.inserts), seedl, seedc, delmod,
147+
)
144148

145149
bname := []byte(bucketname)
146150
put := func(tx *bolt.Tx) (err error) {
@@ -244,7 +248,9 @@ func boltGetter(
244248
time.Sleep(time.Duration(rand.Intn(100)+300) * time.Millisecond)
245249

246250
var key []byte
247-
g := Generateread(int64(options.keylen), loadn, seedl, seedc)
251+
g := Generateread(
252+
int64(options.keylen), loadn, int64(options.inserts), seedl, seedc,
253+
)
248254

249255
bname := []byte(bucketname)
250256
get := func(tx *bolt.Tx) (err error) {
@@ -292,7 +298,9 @@ func boltRanger(
292298
time.Sleep(time.Duration(rand.Intn(100)+300) * time.Millisecond)
293299

294300
var key []byte
295-
g := Generateread(int64(options.keylen), loadn, seedl, seedc)
301+
g := Generateread(
302+
int64(options.keylen), loadn, int64(options.inserts), seedl, seedc,
303+
)
296304

297305
bname := []byte(bucketname)
298306
ranger := func(tx *bolt.Tx) error {

‎mod_llrb.go

Lines changed: 13 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -93,9 +93,13 @@ func llrbWriter(
9393
var x, y, z int64
9494

9595
klen, vlen := int64(options.keylen), int64(options.vallen)
96-
gcreate := Generatecreate(klen, vlen, n, seedc)
97-
gupdate := Generateupdate(klen, vlen, n, seedl, seedc, -1)
98-
gdelete := Generatedelete(klen, vlen, n, seedl, seedc, delmod)
96+
gcreate := Generatecreate(klen, vlen, n, int64(options.inserts), seedc)
97+
gupdate := Generateupdate(
98+
klen, vlen, n, int64(options.inserts), seedl, seedc, -1,
99+
)
100+
gdelete := Generatedelete(
101+
klen, vlen, n, int64(options.inserts), seedl, seedc, delmod,
102+
)
99103

100104
value, oldvalue := make([]byte, vlen), make([]byte, vlen)
101105
if options.vallen <= 0 {
@@ -293,7 +297,9 @@ func llrbGetter(
293297

294298
var ngets, nmisses int64
295299
var key []byte
296-
g := Generateread(int64(options.keylen), n, seedl, seedc)
300+
g := Generateread(
301+
int64(options.keylen), n, int64(options.inserts), seedl, seedc,
302+
)
297303

298304
rnd := rand.New(rand.NewSource(seedl))
299305
epoch, now, markercount := time.Now(), time.Now(), int64(10000000)
@@ -396,7 +402,9 @@ func llrbRanger(
396402

397403
var nranges int64
398404
var key []byte
399-
g := Generateread(int64(options.keylen), n, seedl, seedc)
405+
g := Generateread(
406+
int64(options.keylen), n, int64(options.inserts), seedl, seedc,
407+
)
400408

401409
rnd := rand.New(rand.NewSource(seedl))
402410
epoch, value := time.Now(), make([]byte, options.vallen)

0 commit comments

Comments
 (0)