Skip to content

Commit 4c9fd51

Browse files
ccecce
committed
more DRY
1 parent f7359e9 commit 4c9fd51

File tree

1 file changed

+48
-66
lines changed

1 file changed

+48
-66
lines changed

‎tools/debug/algodump/compress_bench_test.go‎

Lines changed: 48 additions & 66 deletions
Original file line numberDiff line numberDiff line change
@@ -282,7 +282,13 @@ type levelWindowLog struct {
282282
windowLog int
283283
}
284284

285-
func parseLevelWindowLogList(keys []string, defaults []levelWindowLog) []levelWindowLog {
285+
type levelWindow struct {
286+
level int
287+
window int
288+
}
289+
290+
// parseConfigFromEnv parses configuration from environment variables with a generic two-field struct
291+
func parseConfigFromEnv[T comparable](keys []string, defaults []T, parse func(field1, field2 string) (T, error)) []T {
286292
for _, key := range keys {
287293
if key == "" {
288294
continue
@@ -291,19 +297,17 @@ func parseLevelWindowLogList(keys []string, defaults []levelWindowLog) []levelWi
291297
if val == "" {
292298
continue
293299
}
294-
var result []levelWindowLog
295-
seen := make(map[levelWindowLog]struct{})
300+
var result []T
301+
seen := make(map[T]struct{})
296302
for _, part := range strings.Split(val, ",") {
297303
fields := strings.Split(part, ":")
298304
if len(fields) != 2 {
299305
continue
300306
}
301-
level, err1 := strconv.Atoi(strings.TrimSpace(fields[0]))
302-
windowLog, err2 := strconv.Atoi(strings.TrimSpace(fields[1]))
303-
if err1 != nil || err2 != nil {
307+
cfg, err := parse(strings.TrimSpace(fields[0]), strings.TrimSpace(fields[1]))
308+
if err != nil {
304309
continue
305310
}
306-
cfg := levelWindowLog{level: level, windowLog: windowLog}
307311
if _, ok := seen[cfg]; ok {
308312
continue
309313
}
@@ -317,51 +321,41 @@ func parseLevelWindowLogList(keys []string, defaults []levelWindowLog) []levelWi
317321
return defaults
318322
}
319323

320-
type levelWindow struct {
321-
level int
322-
window int
324+
func parseLevelWindowLogList(keys []string, defaults []levelWindowLog) []levelWindowLog {
325+
return parseConfigFromEnv(keys, defaults, func(f1, f2 string) (levelWindowLog, error) {
326+
level, err1 := strconv.Atoi(f1)
327+
windowLog, err2 := strconv.Atoi(f2)
328+
if err1 != nil {
329+
return levelWindowLog{}, err1
330+
}
331+
if err2 != nil {
332+
return levelWindowLog{}, err2
333+
}
334+
return levelWindowLog{level: level, windowLog: windowLog}, nil
335+
})
323336
}
324337

325338
func parseLevelWindowList(keys []string, defaults []levelWindow) []levelWindow {
326-
for _, key := range keys {
327-
if key == "" {
328-
continue
329-
}
330-
val := strings.TrimSpace(os.Getenv(key))
331-
if val == "" {
332-
continue
333-
}
334-
var result []levelWindow
335-
seen := make(map[levelWindow]struct{})
336-
for _, part := range strings.Split(val, ",") {
337-
fields := strings.Split(part, ":")
338-
if len(fields) != 2 {
339-
continue
340-
}
341-
level, err1 := strconv.Atoi(strings.TrimSpace(fields[0]))
342-
window, err2 := strconv.Atoi(strings.TrimSpace(fields[1]))
343-
if err1 != nil || err2 != nil {
344-
continue
345-
}
346-
cfg := levelWindow{level: level, window: window}
347-
if _, ok := seen[cfg]; ok {
348-
continue
349-
}
350-
seen[cfg] = struct{}{}
351-
result = append(result, cfg)
339+
return parseConfigFromEnv(keys, defaults, func(f1, f2 string) (levelWindow, error) {
340+
level, err1 := strconv.Atoi(f1)
341+
window, err2 := strconv.Atoi(f2)
342+
if err1 != nil {
343+
return levelWindow{}, err1
352344
}
353-
if len(result) > 0 {
354-
return result
345+
if err2 != nil {
346+
return levelWindow{}, err2
355347
}
356-
}
357-
return defaults
348+
return levelWindow{level: level, window: window}, nil
349+
})
358350
}
359351

360352
var (
361-
defaultGozstdLevels = []int{1, 3, 7, 11}
362-
defaultZstdLevels = []int{1, 3, 7, 11}
363-
defaultKlauspostLevels = []int{1, 3}
364-
defaultGozstdWindowLog = []levelWindowLog{
353+
// Default compression levels for benchmarks
354+
defaultLevels = []int{1, 3}
355+
defaultLevelsExtend = []int{1, 3, 7, 11}
356+
357+
// Default window configurations for benchmarks
358+
defaultWindowLog = []levelWindowLog{
365359
{level: 1, windowLog: 15},
366360
{level: 1, windowLog: 18},
367361
{level: 1, windowLog: 20},
@@ -371,19 +365,7 @@ var (
371365
{level: 7, windowLog: 18},
372366
{level: 11, windowLog: 18},
373367
}
374-
defaultKlauspostWindows = []levelWindow{
375-
{level: 1, window: 1 << 12},
376-
{level: 1, window: 1 << 15},
377-
{level: 1, window: 1 << 16},
378-
{level: 1, window: 1 << 18},
379-
{level: 1, window: 1 << 20},
380-
{level: 2, window: 1 << 15},
381-
{level: 2, window: 1 << 18},
382-
{level: 2, window: 1 << 20},
383-
{level: 3, window: 1 << 18},
384-
{level: 3, window: 1 << 20},
385-
}
386-
defaultKlauspostStreamConfigs = []levelWindow{
368+
defaultWindows = []levelWindow{
387369
{level: 1, window: 1 << 12},
388370
{level: 1, window: 1 << 15},
389371
{level: 1, window: 1 << 18},
@@ -795,7 +777,7 @@ func BenchmarkVPackDynamicDecompression(b *testing.B) {
795777

796778
func BenchmarkGozstdSimple(b *testing.B) {
797779
corpus := loadTestCorpus(b)
798-
levels := parseIntListFromEnv([]string{"ALGODUMP_GOZSTD_LEVELS", "ALGODUMP_BENCH_LEVELS"}, defaultGozstdLevels)
780+
levels := parseIntListFromEnv([]string{"ALGODUMP_GOZSTD_LEVELS", "ALGODUMP_BENCH_LEVELS"}, defaultLevelsExtend)
799781
filters := parseFilterOptions([]string{"ALGODUMP_GOZSTD_FILTERS", "ALGODUMP_BENCH_FILTERS"})
800782
runSimpleCompressionMatrix(b, "gozstd/simple", corpus, levels, filters, func(level int) (simpleCompressor, func(), error) {
801783
return func(dst, src []byte) ([]byte, error) {
@@ -806,7 +788,7 @@ func BenchmarkGozstdSimple(b *testing.B) {
806788

807789
func BenchmarkZstdSimple(b *testing.B) {
808790
corpus := loadTestCorpus(b)
809-
levels := parseIntListFromEnv([]string{"ALGODUMP_ZSTD_LEVELS", "ALGODUMP_BENCH_LEVELS"}, defaultZstdLevels)
791+
levels := parseIntListFromEnv([]string{"ALGODUMP_ZSTD_LEVELS", "ALGODUMP_BENCH_LEVELS"}, defaultLevelsExtend)
810792
filters := parseFilterOptions([]string{"ALGODUMP_ZSTD_FILTERS", "ALGODUMP_BENCH_FILTERS"})
811793
runSimpleCompressionMatrix(b, "zstd/simple", corpus, levels, filters, func(level int) (simpleCompressor, func(), error) {
812794
return func(dst, src []byte) ([]byte, error) {
@@ -817,7 +799,7 @@ func BenchmarkZstdSimple(b *testing.B) {
817799

818800
func BenchmarkKlauspostSimple(b *testing.B) {
819801
corpus := loadTestCorpus(b)
820-
levels := parseIntListFromEnv([]string{"ALGODUMP_KLAUSPOST_LEVELS", "ALGODUMP_BENCH_LEVELS"}, defaultKlauspostLevels)
802+
levels := parseIntListFromEnv([]string{"ALGODUMP_KLAUSPOST_LEVELS", "ALGODUMP_BENCH_LEVELS"}, defaultLevels)
821803
filters := parseFilterOptions([]string{"ALGODUMP_KLAUSPOST_FILTERS", "ALGODUMP_BENCH_FILTERS"})
822804
runSimpleCompressionMatrix(b, "klauspost/simple", corpus, levels, filters, func(level int) (simpleCompressor, func(), error) {
823805
enc, err := kzstd.NewWriter(nil, kzstd.WithEncoderLevel(kzstd.EncoderLevel(level)))
@@ -834,7 +816,7 @@ func BenchmarkKlauspostSimple(b *testing.B) {
834816

835817
func BenchmarkGozstdDecompress(b *testing.B) {
836818
corpus := loadTestCorpus(b)
837-
levels := parseIntListFromEnv([]string{"ALGODUMP_GOZSTD_LEVELS", "ALGODUMP_BENCH_LEVELS"}, defaultGozstdLevels)
819+
levels := parseIntListFromEnv([]string{"ALGODUMP_GOZSTD_LEVELS", "ALGODUMP_BENCH_LEVELS"}, defaultLevelsExtend)
838820
filters := parseFilterOptions([]string{"ALGODUMP_GOZSTD_FILTERS", "ALGODUMP_BENCH_FILTERS"})
839821
runSimpleDecompressionMatrix(b, "gozstd/decompress", corpus, levels, filters, func(level int, msgs []StoredMessage) ([][]byte, simpleDecompressor, func(), error) {
840822
compressed := make([][]byte, len(msgs))
@@ -850,7 +832,7 @@ func BenchmarkGozstdDecompress(b *testing.B) {
850832

851833
func BenchmarkZstdDecompress(b *testing.B) {
852834
corpus := loadTestCorpus(b)
853-
levels := parseIntListFromEnv([]string{"ALGODUMP_ZSTD_LEVELS", "ALGODUMP_BENCH_LEVELS"}, defaultZstdLevels)
835+
levels := parseIntListFromEnv([]string{"ALGODUMP_ZSTD_LEVELS", "ALGODUMP_BENCH_LEVELS"}, defaultLevelsExtend)
854836
filters := parseFilterOptions([]string{"ALGODUMP_ZSTD_FILTERS", "ALGODUMP_BENCH_FILTERS"})
855837
runSimpleDecompressionMatrix(b, "zstd/decompress", corpus, levels, filters, func(level int, msgs []StoredMessage) ([][]byte, simpleDecompressor, func(), error) {
856838
compressed := make([][]byte, len(msgs))
@@ -870,7 +852,7 @@ func BenchmarkZstdDecompress(b *testing.B) {
870852

871853
func BenchmarkKlauspostDecompress(b *testing.B) {
872854
corpus := loadTestCorpus(b)
873-
levels := parseIntListFromEnv([]string{"ALGODUMP_KLAUSPOST_LEVELS", "ALGODUMP_BENCH_LEVELS"}, defaultKlauspostLevels)
855+
levels := parseIntListFromEnv([]string{"ALGODUMP_KLAUSPOST_LEVELS", "ALGODUMP_BENCH_LEVELS"}, defaultLevels)
874856
filters := parseFilterOptions([]string{"ALGODUMP_KLAUSPOST_FILTERS", "ALGODUMP_BENCH_FILTERS"})
875857
runSimpleDecompressionMatrix(b, "klauspost/decompress", corpus, levels, filters, func(level int, msgs []StoredMessage) ([][]byte, simpleDecompressor, func(), error) {
876858
enc, err := kzstd.NewWriter(nil, kzstd.WithEncoderLevel(kzstd.EncoderLevel(level)))
@@ -898,7 +880,7 @@ func BenchmarkKlauspostDecompress(b *testing.B) {
898880
}
899881

900882
func BenchmarkGozstdWindow(b *testing.B) {
901-
configs := parseLevelWindowLogList([]string{"ALGODUMP_GOZSTD_WINDOW_LOGS", "ALGODUMP_BENCH_WINDOW_LOGS"}, defaultGozstdWindowLog)
883+
configs := parseLevelWindowLogList([]string{"ALGODUMP_GOZSTD_WINDOW_LOGS", "ALGODUMP_BENCH_WINDOW_LOGS"}, defaultWindowLog)
902884
for _, cfg := range configs {
903885
b.Run(fmt.Sprintf("gozstd/window/level=%d/windowLog=%d", cfg.level, cfg.windowLog), func(b *testing.B) {
904886
benchmarkGozstdWindow(b, cfg.level, cfg.windowLog)
@@ -907,7 +889,7 @@ func BenchmarkGozstdWindow(b *testing.B) {
907889
}
908890

909891
func BenchmarkKlauspostWindow(b *testing.B) {
910-
configs := parseLevelWindowList([]string{"ALGODUMP_KLAUSPOST_WINDOWS", "ALGODUMP_BENCH_WINDOWS"}, defaultKlauspostWindows)
892+
configs := parseLevelWindowList([]string{"ALGODUMP_KLAUSPOST_WINDOWS", "ALGODUMP_BENCH_WINDOWS"}, defaultWindows)
911893
for _, cfg := range configs {
912894
b.Run(fmt.Sprintf("klauspost/window/level=%d/window=%d", cfg.level, cfg.window), func(b *testing.B) {
913895
benchmarkKlauspostWindow(b, cfg.level, cfg.window)
@@ -916,7 +898,7 @@ func BenchmarkKlauspostWindow(b *testing.B) {
916898
}
917899

918900
func BenchmarkKlauspostStream(b *testing.B) {
919-
configs := parseLevelWindowList([]string{"ALGODUMP_KLAUSPOST_STREAM_WINDOWS", "ALGODUMP_BENCH_STREAM_WINDOWS"}, defaultKlauspostStreamConfigs)
901+
configs := parseLevelWindowList([]string{"ALGODUMP_KLAUSPOST_STREAM_WINDOWS", "ALGODUMP_BENCH_STREAM_WINDOWS"}, defaultWindows)
920902
for _, cfg := range configs {
921903
b.Run(fmt.Sprintf("klauspost/stream/level=%d/window=%d", cfg.level, cfg.window), func(b *testing.B) {
922904
benchmarkKlauspostStream(b, cfg.level, cfg.window)

0 commit comments

Comments
 (0)