feat: 对slice编码时支持根据数据频率确定编码权重

Author: CC11001100

go.mod

@@ -6,6 +6,8 @@ require (
 	github.com/compression-algorithm-research-lab/go-bit-buffer v0.0.0-20231024145723-e6a2ab4a8de4 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/golang-infrastructure/go-gtypes v0.0.1 // indirect
+	github.com/golang-infrastructure/go-reflect-utils v0.0.0-20221130143747-965ef2eb09c3 // indirect
+	github.com/golang-infrastructure/go-tuple v0.0.0-20221215155811-4ed54fe7d579 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/stretchr/objx v0.5.1 // indirect
 	github.com/stretchr/testify v1.8.4 // indirect

go.sum

@@ -9,6 +9,10 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/golang-infrastructure/go-gtypes v0.0.1 h1:hnM1OYSwLPLGkZ4C6ecAxgmAUaPTjnhnUtRNmJj4p6c=
 github.com/golang-infrastructure/go-gtypes v0.0.1/go.mod h1:vFMCxFzxdMInvTtgLZRlWI1rS+mui88sMbL5I+zu1hg=
+github.com/golang-infrastructure/go-reflect-utils v0.0.0-20221130143747-965ef2eb09c3 h1:jJ7AdpNdLQudsx1hiXY9iwmauHARV4/UB52KnBh9Se0=
+github.com/golang-infrastructure/go-reflect-utils v0.0.0-20221130143747-965ef2eb09c3/go.mod h1:zqXYxqOBa1mL2ilBK6PuH/Wb/Iego7en6XhiKWdZQHI=
+github.com/golang-infrastructure/go-tuple v0.0.0-20221215155811-4ed54fe7d579 h1:pQV2/ichhyLoR3aJSNXByuxtdPM2y229Rq5x9DGl5OU=
+github.com/golang-infrastructure/go-tuple v0.0.0-20221215155811-4ed54fe7d579/go.mod h1:cn8fHK0Sjxh7nSrnNpRa9wi1wIsmBLsjOip4LTjQz+Q=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=

slice_with_frequency.go

@@ -0,0 +1,81 @@
+package unary_coding
+
+import (
+	bit_buffer "github.com/compression-algorithm-research-lab/go-bit-buffer"
+	"github.com/golang-infrastructure/go-gtypes"
+	"github.com/golang-infrastructure/go-tuple"
+	"sort"
+)
+
+// EncodeSliceWithFrequency 基于频率对切片进行一元数组编码
+func EncodeSliceWithFrequency[T gtypes.Unsigned](slice []T) ([]byte, []T) {
+
+	// 对无符号整数切片中的数字统计词频
+	countMap := make(map[T]int, 0)
+	for _, v := range slice {
+		countMap[v] += 1
+	}
+	// 根据词频倒序排序
+	countSlice := make([]*tuple.Tuple2[T, int], 0)
+	for v, c := range countMap {
+		countSlice = append(countSlice, tuple.New2(v, c))
+	}
+	sort.Slice(countSlice, func(i, j int) bool {
+		// 逆序
+		return countSlice[i].V2 > countSlice[j].V2
+	})
+	weightSlice := make([]T, 0)
+	weightMap := make(map[T]int, 0)
+	for i, t := range countSlice {
+		weightSlice = append(weightSlice, t.V1)
+		weightMap[t.V1] = i + 1
+	}
+
+	// 好了，现在，终于可以开始一元编码进行压缩了
+	buffer := bit_buffer.New()
+	for _, v := range slice {
+		weight := weightMap[v]
+		for weight > 0 {
+			buffer.WriteBit(1)
+			weight--
+		}
+		buffer.WriteBit(0)
+	}
+	return buffer.Bytes(), weightSlice
+}
+
+// DecodeSliceWithFrequency 基于频率对切片进行一元数组编码
+func DecodeSliceWithFrequency[T gtypes.Unsigned](bytes []byte, weightSlice []T) []T {
+
+	// 生成权重到无符号整数的映射
+	weightMap := make(map[int]T, 0)
+	for weight, v := range weightSlice {
+		weightMap[weight+1] = v
+	}
+
+	// 然后开始解压
+	slice := make([]T, 0)
+	buffer := bit_buffer.New().SetBytes(bytes)
+	for !buffer.IsTail() {
+
+		// 先读取一位看看
+		bit := buffer.ReadBit()
+		if bit == 0 {
+			// 说明出现了两个连续的0，那意味着已经读取结束了
+			break
+		}
+
+		// 然后开始读取连续的1，直到结束
+		c := 1
+		for {
+			bit := buffer.ReadBit()
+			if bit == 1 {
+				c++
+			} else {
+				break
+			}
+		}
+		slice = append(slice, weightMap[c])
+	}
+	return slice
+}

slice_with_frequency_test.go

@@ -0,0 +1,40 @@
+package unary_coding
+
+import (
+	"github.com/stretchr/testify/assert"
+	"testing"
+)
+
+func TestEncodeSliceWithFrequency(t *testing.T) {
+	rawSlice := []uint64{
+		10086, 10086, 1024, 10086, 10086, 1024, 10086, 1024, 10086, 10086, 10086, 1024, 10086, 10086,
+	}
+	bytes, frequency := EncodeSliceWithFrequency(rawSlice)
+	assert.Equal(t, 4, len(bytes))
+	binaryString := ToBinaryString(bytes)
+	assert.Equal(t, "10101101010110101101010101101010", binaryString)
+	assert.Equal(t, []uint64{
+		uint64(10086),
+		uint64(1024),
+	}, frequency)
+
+}
+
+func TestDecodeSliceWithFrequency(t *testing.T) {
+
+	rawSlice := []uint64{
+		10086, 10086, 1024, 10086, 10086, 1024, 10086, 1024, 10086, 10086, 10086, 1024, 10086, 10086,
+	}
+	bytes, frequency := EncodeSliceWithFrequency(rawSlice)
+	assert.Equal(t, 4, len(bytes))
+	binaryString := ToBinaryString(bytes)
+	assert.Equal(t, "10101101010110101101010101101010", binaryString)
+	assert.Equal(t, []uint64{
+		uint64(10086),
+		uint64(1024),
+	}, frequency)
+
+	unzipSlice := DecodeSliceWithFrequency[uint64](bytes, frequency)
+	assert.Equal(t, rawSlice, unzipSlice)
+
+}