From eb89693509d575384b74443029f37119b53ee39a Mon Sep 17 00:00:00 2001 From: Daniel Lemire Date: Mon, 9 Oct 2023 15:06:08 -0400 Subject: [PATCH 1/4] Minor fixes --- .drone.yml | 1 - .github/workflows/basictests.yml | 1 - .github/workflows/macostests.yml | 1 - .github/workflows/windowstests.yml | 1 - Makefile | 3 +- aggregation_test.go | 7 +-- benchmark_test.go | 29 ++++++----- byte_input_test.go | 1 - roaring.go | 24 ++++----- roaring64/Makefile | 1 - roaring64/bsi64.go | 2 + roaring64/fastaggregation64_test.go | 13 ++--- roaring64/roaring64_test.go | 10 ++-- serialization_littleendian.go | 79 ++++++++++++++++------------- 14 files changed, 91 insertions(+), 82 deletions(-) diff --git a/.drone.yml b/.drone.yml index 698cd0e7..7936bfe8 100644 --- a/.drone.yml +++ b/.drone.yml @@ -11,7 +11,6 @@ steps: commands: - go get -t - go test - - go test -race -run TestConcurrent* - go build -tags appengine - go test -tags appengine - GOARCH=386 go build diff --git a/.github/workflows/basictests.yml b/.github/workflows/basictests.yml index 32d93e6b..5c6a5d94 100644 --- a/.github/workflows/basictests.yml +++ b/.github/workflows/basictests.yml @@ -19,7 +19,6 @@ jobs: uses: actions/checkout@v3 - name: Test run: | - go test -race -run TestConcurrent* go build -tags appengine go test -tags appengine go test -v diff --git a/.github/workflows/macostests.yml b/.github/workflows/macostests.yml index c705cf6c..10e74ec3 100644 --- a/.github/workflows/macostests.yml +++ b/.github/workflows/macostests.yml @@ -19,7 +19,6 @@ jobs: uses: actions/checkout@v3 - name: Test run: | - go test -race -run TestConcurrent* go build -tags appengine go test -tags appengine go test -v diff --git a/.github/workflows/windowstests.yml b/.github/workflows/windowstests.yml index 98b77f85..30ea614d 100644 --- a/.github/workflows/windowstests.yml +++ b/.github/workflows/windowstests.yml @@ -19,7 +19,6 @@ jobs: uses: actions/checkout@v3 - name: Test run: | - go test -race -run TestConcurrent* go build -tags appengine go test -tags appengine go test -v diff --git a/Makefile b/Makefile index 0a4f9f0a..4024436a 100644 --- a/Makefile +++ b/Makefile @@ -33,7 +33,6 @@ help: all: help test: go test - go test -race -run TestConcurrent* # Format the source code format: @find ./ -type f -name "*.go" -exec gofmt -w {} \; @@ -65,7 +64,7 @@ qa: fmtcheck test vet lint deps: GOPATH=$(GOPATH) go get github.com/stretchr/testify GOPATH=$(GOPATH) go get github.com/bits-and-blooms/bitset - GOPATH=$(GOPATH) go get github.com/golang/lint/golint + GOPATH=$(GOPATH) go get golang.org/x/lint/golint GOPATH=$(GOPATH) go get github.com/mschoch/smat GOPATH=$(GOPATH) go get github.com/dvyukov/go-fuzz/go-fuzz GOPATH=$(GOPATH) go get github.com/dvyukov/go-fuzz/go-fuzz-build diff --git a/aggregation_test.go b/aggregation_test.go index 8011689c..63026b2b 100644 --- a/aggregation_test.go +++ b/aggregation_test.go @@ -4,9 +4,10 @@ package roaring import ( "fmt" - "github.com/stretchr/testify/assert" "sort" "testing" + + "github.com/stretchr/testify/assert" ) func testAggregations(t *testing.T, @@ -288,9 +289,9 @@ func TestIssue330(t *testing.T) { bitmap := BitmapOf(v...) bitmap.RunOptimize() bitmaps = append(bitmaps, bitmap) - array_result := bitmap.ToArray() + arrayResult := bitmap.ToArray() sort.Sort(uints(v)) - assert.Equal(t, array_result, v) + assert.Equal(t, arrayResult, v) } assert.Equal(t, FastAnd(bitmaps[0], bitmaps[1]).GetCardinality(), uint64(0)) assert.Equal(t, FastAnd(bitmaps[0], bitmaps[2]).GetCardinality(), uint64(0)) diff --git a/benchmark_test.go b/benchmark_test.go index ffc29f93..84849119 100644 --- a/benchmark_test.go +++ b/benchmark_test.go @@ -3,10 +3,11 @@ package roaring import ( "bytes" "fmt" - "github.com/stretchr/testify/require" "math/rand" "testing" + "github.com/stretchr/testify/require" + "github.com/bits-and-blooms/bitset" ) @@ -22,7 +23,7 @@ func BenchmarkIteratorAlloc(b *testing.B) { bm.Add(v) } i := IntIterator{} - expected_cardinality := bm.GetCardinality() + expectedCardinality := bm.GetCardinality() counter := uint64(0) b.Run("simple iteration with alloc", func(b *testing.B) { for n := 0; n < b.N; n++ { @@ -35,8 +36,8 @@ func BenchmarkIteratorAlloc(b *testing.B) { } b.StopTimer() }) - if counter != expected_cardinality { - b.Fatalf("Cardinalities don't match: %d, %d", counter, expected_cardinality) + if counter != expectedCardinality { + b.Fatalf("Cardinalities don't match: %d, %d", counter, expectedCardinality) } b.Run("simple iteration", func(b *testing.B) { for n := 0; n < b.N; n++ { @@ -49,8 +50,8 @@ func BenchmarkIteratorAlloc(b *testing.B) { } b.StopTimer() }) - if counter != expected_cardinality { - b.Fatalf("Cardinalities don't match: %d, %d", counter, expected_cardinality) + if counter != expectedCardinality { + b.Fatalf("Cardinalities don't match: %d, %d", counter, expectedCardinality) } b.Run("reverse iteration with alloc", func(b *testing.B) { for n := 0; n < b.N; n++ { @@ -63,8 +64,8 @@ func BenchmarkIteratorAlloc(b *testing.B) { } b.StopTimer() }) - if counter != expected_cardinality { - b.Fatalf("Cardinalities don't match: %d, %d", counter, expected_cardinality) + if counter != expectedCardinality { + b.Fatalf("Cardinalities don't match: %d, %d", counter, expectedCardinality) } ir := IntReverseIterator{} @@ -79,8 +80,8 @@ func BenchmarkIteratorAlloc(b *testing.B) { } b.StopTimer() }) - if counter != expected_cardinality { - b.Fatalf("Cardinalities don't match: %d, %d", counter, expected_cardinality) + if counter != expectedCardinality { + b.Fatalf("Cardinalities don't match: %d, %d", counter, expectedCardinality) } b.Run("many iteration with alloc", func(b *testing.B) { @@ -94,8 +95,8 @@ func BenchmarkIteratorAlloc(b *testing.B) { } b.StopTimer() }) - if counter != expected_cardinality { - b.Fatalf("Cardinalities don't match: %d, %d", counter, expected_cardinality) + if counter != expectedCardinality { + b.Fatalf("Cardinalities don't match: %d, %d", counter, expectedCardinality) } im := ManyIntIterator{} buf := make([]uint32, 1024) @@ -110,8 +111,8 @@ func BenchmarkIteratorAlloc(b *testing.B) { } b.StopTimer() }) - if counter != expected_cardinality { - b.Fatalf("Cardinalities don't match: %d, %d", counter, expected_cardinality) + if counter != expectedCardinality { + b.Fatalf("Cardinalities don't match: %d, %d", counter, expectedCardinality) } } diff --git a/byte_input_test.go b/byte_input_test.go index c4407f19..75d55ff8 100644 --- a/byte_input_test.go +++ b/byte_input_test.go @@ -4,7 +4,6 @@ import ( "bytes" "testing" - "github.com/RoaringBitmap/roaring/internal" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) diff --git a/roaring.go b/roaring.go index cdf543ce..f7c1652b 100644 --- a/roaring.go +++ b/roaring.go @@ -369,10 +369,10 @@ type IntIterator = intIterator // Initialize configures the existing iterator so that it can iterate through the values of // the provided bitmap. // The iteration results are undefined if the bitmap is modified (e.g., with Add or Remove). -func (p *intIterator) Initialize(a *Bitmap) { - p.pos = 0 - p.highlowcontainer = &a.highlowcontainer - p.init() +func (ii *intIterator) Initialize(a *Bitmap) { + ii.pos = 0 + ii.highlowcontainer = &a.highlowcontainer + ii.init() } type intReverseIterator struct { @@ -438,10 +438,10 @@ type IntReverseIterator = intReverseIterator // Initialize configures the existing iterator so that it can iterate through the values of // the provided bitmap. // The iteration results are undefined if the bitmap is modified (e.g., with Add or Remove). -func (p *intReverseIterator) Initialize(a *Bitmap) { - p.highlowcontainer = &a.highlowcontainer - p.pos = a.highlowcontainer.size() - 1 - p.init() +func (ii *intReverseIterator) Initialize(a *Bitmap) { + ii.highlowcontainer = &a.highlowcontainer + ii.pos = a.highlowcontainer.size() - 1 + ii.init() } // ManyIntIterable allows you to iterate over the values in a Bitmap @@ -525,10 +525,10 @@ type ManyIntIterator = manyIntIterator // Initialize configures the existing iterator so that it can iterate through the values of // the provided bitmap. // The iteration results are undefined if the bitmap is modified (e.g., with Add or Remove). -func (p *manyIntIterator) Initialize(a *Bitmap) { - p.pos = 0 - p.highlowcontainer = &a.highlowcontainer - p.init() +func (ii *manyIntIterator) Initialize(a *Bitmap) { + ii.pos = 0 + ii.highlowcontainer = &a.highlowcontainer + ii.init() } // String creates a string representation of the Bitmap diff --git a/roaring64/Makefile b/roaring64/Makefile index 7e8953c7..cb36d867 100644 --- a/roaring64/Makefile +++ b/roaring64/Makefile @@ -33,7 +33,6 @@ help: all: help test: go test - go test -race -run TestConcurrent* # Format the source code format: @find ./ -type f -name "*.go" -exec gofmt -w {} \; diff --git a/roaring64/bsi64.go b/roaring64/bsi64.go index 0e49b19a..6cae3284 100644 --- a/roaring64/bsi64.go +++ b/roaring64/bsi64.go @@ -880,6 +880,7 @@ func (b *BSI) IncrementAll() { b.Increment(b.GetExistenceBitmap()) } +// Equals - Check for semantic equality of two BSIs. func (b *BSI) Equals(other *BSI) bool { if !b.eBM.Equals(&other.eBM) { return false @@ -902,6 +903,7 @@ func (b *BSI) Equals(other *BSI) bool { return true } +// GetSizeInBytes - the size in bytes of the data structure func (b *BSI) GetSizeInBytes() int { size := b.eBM.GetSizeInBytes() for _, bm := range b.bA { diff --git a/roaring64/fastaggregation64_test.go b/roaring64/fastaggregation64_test.go index 153e9031..72ddd8cb 100644 --- a/roaring64/fastaggregation64_test.go +++ b/roaring64/fastaggregation64_test.go @@ -4,8 +4,8 @@ package roaring64 import ( "github.com/stretchr/testify/assert" - "testing" "sort" + "testing" ) func TestFastAggregationsAdvanced_run(t *testing.T) { @@ -38,6 +38,7 @@ func TestFastAggregationsAdvanced_run(t *testing.T) { assert.True(t, FastOr(rb1, rb2, rb3).Equals(rb1)) assert.True(t, FastAnd(rb1, rb2, rb3).Equals(bigand)) } + type uints []uint64 func (u uints) Len() int { return len(u) } @@ -53,16 +54,16 @@ func TestIssue330_64bits(t *testing.T) { {1332290, 1447737, 1549291, 1187244, 1598129, 1579851, 1424171, 1538999, 1445358, 1586739, 1575050, 1437657, 1366343, 1062799, 1421550, 1460317, 1474875, 1060737, 1330773, 1447797, 1348633, 1559437, 1556214, 1187305, 1187234, 1187240, 1464660, 1567794, 1187260, 1260646, 1311938, 1573195, 1318525, 1484524, 1456152, 1087954, 1556007, 1187265, 1460920, 1485316, 1447849, 1447744, 1474001, 1537891, 1478211, 1313292, 1488405, 1187239, 1378814, 1343620, 1500498, 1567809, 1435838, 1506575, 1368282, 1447441, 1598101, 1067076, 1572997, 1598102, 1332697, 1324653, 1561437, 1187290, 1059945, 1187278, 1457187, 1430003, 1450643, 1447436, 1260650, 1473393, 1187247, 1087323, 1324967, 1187291, 1480771, 1472729, 1555881, 1187253, 1456481, 1452672, 1447435, 1378603, 1574771, 1187235, 1417857, 1568706, 1576739, 1534662, 1410189, 1587745, 1473791, 1187308, 1447730, 1328158, 1409164, 1506591, 1500147, 1433961, 1483709, 1187227, 1456479, 1562595, 1314333, 1187281, 1598012, 1415200, 1447791, 1371379, 1598019, 1435836, 1457188, 1457351, 1187248, 1417111, 1187289, 1187252, 1187257, 1316665, 1473464, 1187263, 1447732, 1520371, 1525651, 1598177, 1406947, 1465787, 1524659, 1324213, 1418439, 1575816, 1522696, 1187286, 1497821, 1333179, 1187282, 1335988, 1548952, 1556066, 1314993, 1187276, 1420503, 1187301, 1456468, 1423939, 1598089, 1504357, 1343247, 1437659, 1525768, 1539279, 1307385, 1187275, 1524305, 1332938, 1516498, 1303247, 1304237, 1187238, 1385283, 1595495, 1187300, 1187241, 1061740, 1316383, 1187307, 1062037, 1538693, 1454292, 1447731, 1187272, 1561442, 1187268, 1567150, 1597966, 1447745, 1598178, 1187262, 1464067, 1325394, 1537893, 1332693, 1479200, 1522335, 1589378, 1450573, 1399161, 1421274, 1561501, 1187232, 1187302, 1258469, 1331600, 1447740, 1187242, 1328147, 1264069, 1187294, 1299943, 1598013, 1526975, 1260604, 1487518, 1187229, 1487617, 1354087, 1456595, 1462047, 1561438, 1598363, 1332691, 1424655, 1567105, 1574774, 1598035, 1526981, 1384038, 1475987, 1343587, 1447437, 1454912, 1382215, 1447739, 1456512, 1447779, 1187283, 1440988, 1187293, 1187298, 1574754, 1354772, 1598018, 1447429, 1598181, 1447738, 1598273, 1312197, 1574752, 1572995, 1526127, 1473908, 1437660, 1447743, 1362262, 1456513, 1539280, 1348625, 1415878, 1332694, 1471020, 1432462, 1058088, 1526710, 1371788, 1187288, 1537984, 1316874, 1187270, 1333565, 1187292, 1447796, 1187311, 1187237, 1187231, 1574755, 1553822, 1522019, 1447418, 1187269, 1332692, 1447735, 1529638, 1468154, 1328031, 1447733, 1447402, 1593884, 1332696, 1560622, 1564819, 1538967, 1315756, 1328338, 1598113, 1324212, 1449895, 1567793, 1260629, 1430010, 1187266, 1187256, 1312754, 1449417, 1595494, 1529054, 1187261, 1187306, 1526976, 1425490, 1366922, 1527390, 1187299, 1561510, 1319222, 1187250, 1057262, 1457999, 1332937, 1187243, 1556213, 1278602, 1546839, 1187296, 1548950, 1580141, 1187303, 1187255, 1525650, 1572998, 1576740, 1187267, 1464664, 1440427, 1456467, 1187271, 1187258, 1585428, 1548760, 1342254, 1447793, 1406348, 1500177, 1260644, 1416954, 1323722, 1412713, 1187280, 1187310, 1538015, 1537285, 1187285, 1456482, 1260611, 1490508, 1187274, 1585641, 1416648, 1484655, 1421520, 1347485, 1525652, 1568987, 1526974, 1314375, 1187246, 1455623, 1488117, 1445025, 1447401, 1478237, 1561440, 1187287, 1561434, 1509337, 1451859, 1599630, 1348639, 1449436, 1361844, 1464661, 1263064, 1526973, 1187279, 1562080, 1354770, 1454521, 1520719, 1478236, 1526972, 1423948, 1334866, 1325026, 1438275, 1422582, 1437646, 1315530, 1458323, 1447795, 1528218, 1187254, 1598056, 1417853, 1423514, 1187297, 1187245, 1187264, 1524662, 1187251, 1524660, 1328113, 1187304, 1374767, 1474057, 1187284, 1331601, 1598180, 1062814, 1488818, 1187309, 1087494, 1063499, 1458325, 1187295, 1432336, 1260001, 1597982, 1537147, 1445355, 1595491, 1396111, 1546848, 1474048, 1495251, 1447734, 1464071, 1526978, 1187236, 1526977, 1566267, 1187277, 1421549, 1430015, 1316024, 1332695, 1561433, 1435837, 1087250, 1574753, 1476183, 1325395, 1561432, 1447736, 1500181, 1424164, 1456483, 1187228, 1573384, 1273769, 1598085, 1437661, 1306415, 1407257, 1187249, 1338215, 1458047, 1520791, 1447741, 1537263, 1472490, 1524661, 1061729, 1187273, 1417861, 1470196, 1485881, 1260595, 1538846, 1568762, 1315170, 1500469, 1372455, 1558140, 1425202, 1432702, 1472734, 1187230, 1187312, 1598191, 1569680, 1187233, 1263091, 1447417, 1504429, 1430016, 1435839, 1458324, 1546845, 1575027, 1187259, 1464610}, {1578810, 1166701, 1335901, 1063397, 1578812, 1526471, 1166702, 1067008, 1412862, 1059750, 1060729, 1166812, 1493768, 1335772, 1336194, 1166772, 1465775, 1166704, 1355314, 1314138, 1060727, 1388260, 1465786, 1565378, 1522096, 1312490, 1319284, 1418573, 1319169, 1060730, 1307995, 1465780, 1060728, 1404061, 1407842, 1326256, 1578754, 1410577, 1060732, 1461338, 1264209, 1166705, 1581273, 1414196, 1565398, 1355106, 1060731, 1412641, 1306618, 1060726, 1356302, 1310896, 1457885, 1497721, 1166709, 1067009, 1310812, 1578800, 1422203, 1484409, 1485278, 1322057, 1369956, 1311089, 1576614, 1355711, 1355798, 1564832, 1304166, 1166773, 1319071, 1578805, 1575869, 1403066}, } - + bitmaps := []*Bitmap{} - + for _, v := range values { bitmap := BitmapOf(v...) bitmap.RunOptimize() bitmaps = append(bitmaps, bitmap) sort.Sort(uints(v)) - array_result := bitmap.ToArray() - assert.Equal(t, array_result, v) + arrayResult := bitmap.ToArray() + assert.Equal(t, arrayResult, v) } assert.Equal(t, FastAnd(bitmaps[0], bitmaps[1]).GetCardinality(), uint64(0)) assert.Equal(t, FastAnd(bitmaps[0], bitmaps[2]).GetCardinality(), uint64(0)) @@ -76,4 +77,4 @@ func TestIssue330_64bits(t *testing.T) { assert.True(t, FastOr(bitmaps[2], bitmaps[1], bitmaps[0]).Equals(FastOr(bitmaps[0], bitmaps[1], bitmaps[2]))) assert.Equal(t, FastOr(bitmaps[2], bitmaps[1], bitmaps[0]).GetCardinality(), uint64(1040)) assert.Equal(t, FastOr(bitmaps[0], bitmaps[1], bitmaps[2]).GetCardinality(), uint64(1040)) -} \ No newline at end of file +} diff --git a/roaring64/roaring64_test.go b/roaring64/roaring64_test.go index b0d09a71..99803f49 100644 --- a/roaring64/roaring64_test.go +++ b/roaring64/roaring64_test.go @@ -1,6 +1,7 @@ package roaring64 import ( + "fmt" "io/ioutil" "math" "math/rand" @@ -8,7 +9,6 @@ import ( "path/filepath" "strconv" "testing" - "fmt" "github.com/RoaringBitmap/roaring" "github.com/bits-and-blooms/bitset" @@ -2002,7 +2002,7 @@ func Test_tryReadFromRoaring32(t *testing.T) { func Test_tryReadFromRoaring32_File(t *testing.T) { tempDir, err := ioutil.TempDir("./", "testdata") - if(err != nil) { + if err != nil { fmt.Fprintf(os.Stderr, "\n\nIMPORTANT: For testing file IO, the roaring library requires disk access.\nWe omit some tests for now.\n\n") return } @@ -2018,7 +2018,7 @@ func Test_tryReadFromRoaring32_File(t *testing.T) { t.Fatal(err) } file, err := os.Open(name) - if(err != nil) { + if err != nil { fmt.Fprintf(os.Stderr, "\n\nIMPORTANT: For testing file IO, the roaring library requires disk access.\nWe omit some tests for now.\n\n") return } @@ -2048,7 +2048,7 @@ func Test_tryReadFromRoaring32WithRoaring64(t *testing.T) { func Test_tryReadFromRoaring32WithRoaring64_File(t *testing.T) { tempDir, err := ioutil.TempDir("./", "testdata") - if(err != nil) { + if err != nil { fmt.Fprintf(os.Stderr, "\n\nIMPORTANT: For testing file IO, the roaring library requires disk access.\nWe omit some tests for now.\n\n") return } @@ -2065,7 +2065,7 @@ func Test_tryReadFromRoaring32WithRoaring64_File(t *testing.T) { t.Fatal(err) } file, err := os.Open(name) - if(err != nil) { + if err != nil { fmt.Fprintf(os.Stderr, "\n\nIMPORTANT: For testing file IO, the roaring library requires disk access.\nWe omit some tests for now.\n\n") return } diff --git a/serialization_littleendian.go b/serialization_littleendian.go index b6fa7b05..6e3a5d55 100644 --- a/serialization_littleendian.go +++ b/serialization_littleendian.go @@ -312,7 +312,7 @@ func (rb *Bitmap) FrozenView(buf []byte) error { * uint8_t[num_containers] *
uint32_t * - *
is a 4-byte value which is a bit union of FROZEN_COOKIE (15 bits) + *
is a 4-byte value which is a bit union of frozenCookie (15 bits) * and the number of containers (17 bits). * * stores number of elements for every container. @@ -328,43 +328,50 @@ func (rb *Bitmap) FrozenView(buf []byte) error { * All members have their native alignments during deserilization except
, * which is not guaranteed to be aligned by 4 bytes. */ -const FROZEN_COOKIE = 13766 +const frozenCookie = 13766 var ( - FrozenBitmapInvalidCookie = errors.New("header does not contain the FROZEN_COOKIE") - FrozenBitmapBigEndian = errors.New("loading big endian frozen bitmaps is not supported") - FrozenBitmapIncomplete = errors.New("input buffer too small to contain a frozen bitmap") - FrozenBitmapOverpopulated = errors.New("too many containers") - FrozenBitmapUnexpectedData = errors.New("spurious data in input") - FrozenBitmapInvalidTypecode = errors.New("unrecognized typecode") - FrozenBitmapBufferTooSmall = errors.New("buffer too small") + // ErrFrozenBitmapInvalidCookie is returned when the header does not contain the frozenCookie. + ErrFrozenBitmapInvalidCookie = errors.New("header does not contain the frozenCookie") + // ErrFrozenBitmapBigEndian is returned when the header is big endian. + ErrFrozenBitmapBigEndian = errors.New("loading big endian frozen bitmaps is not supported") + // ErrFrozenBitmapIncomplete is returned when the buffer is too small to contain a frozen bitmap. + ErrFrozenBitmapIncomplete = errors.New("input buffer too small to contain a frozen bitmap") + // ErrFrozenBitmapOverpopulated is returned when the number of containers is too large. + ErrFrozenBitmapOverpopulated = errors.New("too many containers") + // ErrFrozenBitmapUnexpectedData is returned when the buffer contains unexpected data. + ErrFrozenBitmapUnexpectedData = errors.New("spurious data in input") + // ErrFrozenBitmapInvalidTypecode is returned when the typecode is invalid. + ErrFrozenBitmapInvalidTypecode = errors.New("unrecognized typecode") + // ErrFrozenBitmapBufferTooSmall is returned when the buffer is too small. + ErrFrozenBitmapBufferTooSmall = errors.New("buffer too small") ) func (ra *roaringArray) frozenView(buf []byte) error { if len(buf) < 4 { - return FrozenBitmapIncomplete + return ErrFrozenBitmapIncomplete } headerBE := binary.BigEndian.Uint32(buf[len(buf)-4:]) - if headerBE&0x7fff == FROZEN_COOKIE { - return FrozenBitmapBigEndian + if headerBE&0x7fff == frozenCookie { + return ErrFrozenBitmapBigEndian } header := binary.LittleEndian.Uint32(buf[len(buf)-4:]) buf = buf[:len(buf)-4] - if header&0x7fff != FROZEN_COOKIE { - return FrozenBitmapInvalidCookie + if header&0x7fff != frozenCookie { + return ErrFrozenBitmapInvalidCookie } nCont := int(header >> 15) if nCont > (1 << 16) { - return FrozenBitmapOverpopulated + return ErrFrozenBitmapOverpopulated } // 1 byte per type, 2 bytes per key, 2 bytes per count. if len(buf) < 5*nCont { - return FrozenBitmapIncomplete + return ErrFrozenBitmapIncomplete } types := buf[len(buf)-nCont:] @@ -389,12 +396,12 @@ func (ra *roaringArray) frozenView(buf []byte) error { nRun++ nRunEl += int(counts[i]) default: - return FrozenBitmapInvalidTypecode + return ErrFrozenBitmapInvalidTypecode } } if len(buf) < (1<<13)*nBitmap+4*nRunEl+2*nArrayEl { - return FrozenBitmapIncomplete + return ErrFrozenBitmapIncomplete } bitsetsArena := byteSliceAsUint64Slice(buf[:(1<<13)*nBitmap]) @@ -407,7 +414,7 @@ func (ra *roaringArray) frozenView(buf []byte) error { buf = buf[2*nArrayEl:] if len(buf) != 0 { - return FrozenBitmapUnexpectedData + return ErrFrozenBitmapUnexpectedData } var c container @@ -474,9 +481,10 @@ func (ra *roaringArray) frozenView(buf []byte) error { return nil } -func (bm *Bitmap) GetFrozenSizeInBytes() uint64 { +// GetFrozenSizeInBytes returns the size in bytes of the frozen bitmap. +func (rb *Bitmap) GetFrozenSizeInBytes() uint64 { nBits, nArrayEl, nRunEl := uint64(0), uint64(0), uint64(0) - for _, c := range bm.highlowcontainer.containers { + for _, c := range rb.highlowcontainer.containers { switch v := c.(type) { case *bitmapContainer: nBits++ @@ -486,19 +494,21 @@ func (bm *Bitmap) GetFrozenSizeInBytes() uint64 { nRunEl += uint64(len(v.iv)) } } - return 4 + 5*uint64(len(bm.highlowcontainer.containers)) + + return 4 + 5*uint64(len(rb.highlowcontainer.containers)) + (nBits << 13) + 2*nArrayEl + 4*nRunEl } -func (bm *Bitmap) Freeze() ([]byte, error) { - sz := bm.GetFrozenSizeInBytes() +// Freeze serializes the bitmap in the CRoaring's frozen format. +func (rb *Bitmap) Freeze() ([]byte, error) { + sz := rb.GetFrozenSizeInBytes() buf := make([]byte, sz) - _, err := bm.FreezeTo(buf) + _, err := rb.FreezeTo(buf) return buf, err } -func (bm *Bitmap) FreezeTo(buf []byte) (int, error) { - containers := bm.highlowcontainer.containers +// FreezeTo serializes the bitmap in the CRoaring's frozen format. +func (rb *Bitmap) FreezeTo(buf []byte) (int, error) { + containers := rb.highlowcontainer.containers nCont := len(containers) nBits, nArrayEl, nRunEl := 0, 0, 0 @@ -515,7 +525,7 @@ func (bm *Bitmap) FreezeTo(buf []byte) (int, error) { serialSize := 4 + 5*nCont + (1<<13)*nBits + 4*nRunEl + 2*nArrayEl if len(buf) < serialSize { - return 0, FrozenBitmapBufferTooSmall + return 0, ErrFrozenBitmapBufferTooSmall } bitsArena := byteSliceAsUint64Slice(buf[:(1<<13)*nBits]) @@ -536,10 +546,10 @@ func (bm *Bitmap) FreezeTo(buf []byte) (int, error) { types := buf[:nCont] buf = buf[nCont:] - header := uint32(FROZEN_COOKIE | (nCont << 15)) + header := uint32(frozenCookie | (nCont << 15)) binary.LittleEndian.PutUint32(buf[:4], header) - copy(keys, bm.highlowcontainer.keys[:]) + copy(keys, rb.highlowcontainer.keys[:]) for i, c := range containers { switch v := c.(type) { @@ -566,11 +576,12 @@ func (bm *Bitmap) FreezeTo(buf []byte) (int, error) { return serialSize, nil } -func (bm *Bitmap) WriteFrozenTo(wr io.Writer) (int, error) { +// WriteFrozenTo serializes the bitmap in the CRoaring's frozen format. +func (rb *Bitmap) WriteFrozenTo(wr io.Writer) (int, error) { // FIXME: this is a naive version that iterates 4 times through the // containers and allocates 3*len(containers) bytes; it's quite likely // it can be done more efficiently. - containers := bm.highlowcontainer.containers + containers := rb.highlowcontainer.containers written := 0 for _, c := range containers { @@ -609,7 +620,7 @@ func (bm *Bitmap) WriteFrozenTo(wr io.Writer) (int, error) { } } - n, err := wr.Write(uint16SliceAsByteSlice(bm.highlowcontainer.keys)) + n, err := wr.Write(uint16SliceAsByteSlice(rb.highlowcontainer.keys)) written += n if err != nil { return written, err @@ -641,7 +652,7 @@ func (bm *Bitmap) WriteFrozenTo(wr io.Writer) (int, error) { return written, err } - header := uint32(FROZEN_COOKIE | (len(containers) << 15)) + header := uint32(frozenCookie | (len(containers) << 15)) if err := binary.Write(wr, binary.LittleEndian, header); err != nil { return written, err } From cabdd65295fe7270cdd7640a20469ec4108fb446 Mon Sep 17 00:00:00 2001 From: Daniel Lemire Date: Mon, 9 Oct 2023 15:18:50 -0400 Subject: [PATCH 2/4] missing import --- byte_input_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/byte_input_test.go b/byte_input_test.go index 75d55ff8..c4407f19 100644 --- a/byte_input_test.go +++ b/byte_input_test.go @@ -4,6 +4,7 @@ import ( "bytes" "testing" + "github.com/RoaringBitmap/roaring/internal" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) From c530e5c9d0f033c0d1f0d16d6de1a89d73d30f41 Mon Sep 17 00:00:00 2001 From: Daniel Lemire Date: Mon, 9 Oct 2023 15:24:39 -0400 Subject: [PATCH 3/4] removing unmaintained Makefile. --- Makefile | 106 ------------------------------------------------------- 1 file changed, 106 deletions(-) delete mode 100644 Makefile diff --git a/Makefile b/Makefile deleted file mode 100644 index 4024436a..00000000 --- a/Makefile +++ /dev/null @@ -1,106 +0,0 @@ -.PHONY: help all test format fmtcheck vet lint qa deps clean nuke ser fetch-real-roaring-datasets - - - - - - - - -# Display general help about this command -help: - @echo "" - @echo "The following commands are available:" - @echo "" - @echo " make qa : Run all the tests" - @echo " make test : Run the unit tests" - @echo "" - @echo " make format : Format the source code" - @echo " make fmtcheck : Check if the source code has been formatted" - @echo " make vet : Check for suspicious constructs" - @echo " make lint : Check for style errors" - @echo "" - @echo " make deps : Get the dependencies" - @echo " make clean : Remove any build artifact" - @echo " make nuke : Deletes any intermediate file" - @echo "" - @echo " make fuzz-smat : Fuzzy testing with smat" - @echo " make fuzz-stream : Fuzzy testing with stream deserialization" - @echo " make fuzz-buffer : Fuzzy testing with buffer deserialization" - @echo "" - -# Alias for help target -all: help -test: - go test -# Format the source code -format: - @find ./ -type f -name "*.go" -exec gofmt -w {} \; - -# Check if the source code has been formatted -fmtcheck: - @mkdir -p target - @find ./ -type f -name "*.go" -exec gofmt -d {} \; | tee target/format.diff - @test ! -s target/format.diff || { echo "ERROR: the source code has not been formatted - please use 'make format' or 'gofmt'"; exit 1; } - -# Check for syntax errors -vet: - GOPATH=$(GOPATH) go vet ./... - -# Check for style errors -lint: - GOPATH=$(GOPATH) PATH=$(GOPATH)/bin:$(PATH) golint ./... - - - - - -# Alias to run all quality-assurance checks -qa: fmtcheck test vet lint - -# --- INSTALL --- - -# Get the dependencies -deps: - GOPATH=$(GOPATH) go get github.com/stretchr/testify - GOPATH=$(GOPATH) go get github.com/bits-and-blooms/bitset - GOPATH=$(GOPATH) go get golang.org/x/lint/golint - GOPATH=$(GOPATH) go get github.com/mschoch/smat - GOPATH=$(GOPATH) go get github.com/dvyukov/go-fuzz/go-fuzz - GOPATH=$(GOPATH) go get github.com/dvyukov/go-fuzz/go-fuzz-build - GOPATH=$(GOPATH) go get github.com/glycerine/go-unsnap-stream - GOPATH=$(GOPATH) go get github.com/philhofer/fwd - GOPATH=$(GOPATH) go get github.com/jtolds/gls - -fuzz-smat: - go test -tags=gofuzz -run=TestGenerateSmatCorpus - go-fuzz-build -func FuzzSmat github.com/RoaringBitmap/roaring - go-fuzz -bin=./roaring-fuzz.zip -workdir=workdir/ -timeout=200 - - -fuzz-stream: - go-fuzz-build -func FuzzSerializationStream github.com/RoaringBitmap/roaring - go-fuzz -bin=./roaring-fuzz.zip -workdir=workdir/ -timeout=200 - - -fuzz-buffer: - go-fuzz-build -func FuzzSerializationBuffer github.com/RoaringBitmap/roaring - go-fuzz -bin=./roaring-fuzz.zip -workdir=workdir/ -timeout=200 - -# Remove any build artifact -clean: - GOPATH=$(GOPATH) go clean ./... - -# Deletes any intermediate file -nuke: - rm -rf ./target - GOPATH=$(GOPATH) go clean -i ./... - -cover: - go test -coverprofile=coverage.out - go tool cover -html=coverage.out - -fetch-real-roaring-datasets: - # pull github.com/RoaringBitmap/real-roaring-datasets -> testdata/real-roaring-datasets - git submodule init - git submodule update From 5e9100efe8f1293ff33688a32ed94e3782605eed Mon Sep 17 00:00:00 2001 From: Daniel Lemire Date: Mon, 9 Oct 2023 15:36:28 -0400 Subject: [PATCH 4/4] we do not support 1.13 --- .github/workflows/bigendiantests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/bigendiantests.yml b/.github/workflows/bigendiantests.yml index 97726b8e..4c4d9415 100644 --- a/.github/workflows/bigendiantests.yml +++ b/.github/workflows/bigendiantests.yml @@ -7,7 +7,7 @@ jobs: test: strategy: matrix: - go-version: [1.13.x, 1.14.x] + go-version: [1.14.x] platform: [ubuntu-latest] runs-on: ${{ matrix.platform }} steps: