chore: fix ci fail

Signed-off-by: Kould <[email protected]>

Author: KKould

src/query/functions/src/scalars/mod.rs

@@ -48,7 +48,6 @@ mod vector;
 pub use comparison::ALL_COMP_FUNC_NAMES;
 use databend_functions_scalar_arithmetic::arithmetic;
 use databend_functions_scalar_numeric_basic_arithmetic::register_numeric_basic_arithmetic;
-pub use hash::DFHash;
 pub use string::ALL_STRING_FUNC_NAMES;
 
 pub fn register(registry: &mut FunctionRegistry) {

src/query/service/src/test_kits/block_writer.rs

@@ -109,7 +109,6 @@ impl<'a> BlockWriter<'a> {
             bloom_filter_index_size,
             None,
             None,
-            None,
             Compression::Lz4Raw,
             Some(Utc::now()),
         );

src/query/service/tests/it/storages/fuse/bloom_index_meta_size.rs

@@ -334,7 +334,6 @@ fn build_test_segment_info(
         bloom_filter_index_location: Some(location_gen.block_bloom_index_location(&block_uuid)),
         bloom_filter_index_size: 0,
         inverted_index_size: None,
-        ngram_filter_index_size: None,
         virtual_block_meta: None,
         compression: Compression::Lz4,
         create_on: Some(Utc::now()),

src/query/service/tests/it/storages/fuse/operations/mutation/recluster_mutator.rs

@@ -77,7 +77,6 @@ async fn test_recluster_mutator_block_select() -> Result<()> {
             0,
             None,
             None,
-            None,
             meta::Compression::Lz4Raw,
             Some(Utc::now()),
         ));

src/query/service/tests/it/storages/fuse/operations/mutation/segments_compact_mutator.rs

@@ -773,7 +773,6 @@ impl CompactSegmentTestFixture {
                         0,
                         None,
                         None,
-                        None,
                         Compression::Lz4Raw,
                         Some(Utc::now()),
                     );

src/query/service/tests/it/storages/fuse/operations/read_plan.rs

@@ -104,7 +104,6 @@ fn test_to_partitions() -> Result<()> {
         bloom_filter_size,
         None,
         None,
-        None,
         meta::Compression::Lz4Raw,
         Some(Utc::now()),
     ));

src/query/service/tests/it/storages/fuse/statistics.rs

@@ -628,7 +628,6 @@ fn test_reduce_block_meta() -> databend_common_exception::Result<()> {
             bloom_filter_index_size,
             None,
             None,
-            None,
             Compression::Lz4Raw,
             Some(Utc::now()),
         );

src/query/storages/common/index/src/bloom_index.rs

@@ -530,7 +530,7 @@ impl BloomIndex {
 
 pub struct BloomIndexBuilder {
     func_ctx: FunctionContext,
-    bloom_columns: Vec<ColumnFilterBuilder>,
+    columns: Vec<ColumnFilterBuilder>,
 }
 
 struct ColumnFilterBuilder {
@@ -596,7 +596,7 @@ impl BloomIndexBuilder {
 
         Ok(Self {
             func_ctx,
-            bloom_columns,
+            columns: bloom_columns,
         })
     }
 }
@@ -610,8 +610,8 @@ impl BloomIndexBuilder {
             return Ok(());
         }
 
-        let mut keys_to_remove = Vec::with_capacity(self.bloom_columns.len());
-        for (index, index_column) in self.bloom_columns.iter_mut().enumerate() {
+        let mut keys_to_remove = Vec::with_capacity(self.columns.len());
+        for (index, index_column) in self.columns.iter_mut().enumerate() {
             let field_type = &block.get_by_offset(index_column.index).data_type;
             if !Xor8Filter::supported_type(field_type) {
                 keys_to_remove.push(index);
@@ -721,16 +721,16 @@ impl BloomIndexBuilder {
             }
         }
         for k in keys_to_remove {
-            self.bloom_columns.remove(k);
+            self.columns.remove(k);
         }
         Ok(())
     }
 
     pub fn finalize(&mut self) -> Result<Option<BloomIndex>> {
-        let mut column_distinct_count = HashMap::with_capacity(self.bloom_columns.len());
-        let mut filters = Vec::with_capacity(self.bloom_columns.len());
-        let mut filter_fields = Vec::with_capacity(self.bloom_columns.len());
-        for column in self.bloom_columns.iter_mut() {
+        let mut column_distinct_count = HashMap::with_capacity(self.columns.len());
+        let mut filters = Vec::with_capacity(self.columns.len());
+        let mut filter_fields = Vec::with_capacity(self.columns.len());
+        for column in self.columns.iter_mut() {
             let filter = column.builder.build()?;
             let filter_name = if column.is_ngram {
                 BloomIndex::build_filter_ngram_name(&column.field)

src/query/storages/common/index/src/filters/xor8/mod.rs

@@ -76,6 +76,13 @@ impl FilterImpl {
 impl Filter for FilterImpl {
     type CodecError = ErrorCode;
 
+    fn len(&self) -> Option<usize> {
+        match self {
+            FilterImpl::Xor(filter) => filter.len(),
+            FilterImpl::Bloom(filter) => filter.len(),
+        }
+    }
+
     fn contains<K: ?Sized + Hash>(&self, key: &K) -> bool {
         match self {
             FilterImpl::Xor(filter) => filter.contains(key),

src/query/storages/common/table_meta/src/meta/v2/segment.rs

@@ -169,7 +169,6 @@ pub struct BlockMeta {
     #[serde(default)]
     pub bloom_filter_index_size: u64,
     pub inverted_index_size: Option<u64>,
-    pub ngram_filter_index_size: Option<u64>,
     /// The block meta of virtual columns.
     pub virtual_block_meta: Option<VirtualBlockMeta>,
     pub compression: Compression,
@@ -191,7 +190,6 @@ impl BlockMeta {
         bloom_filter_index_location: Option<Location>,
         bloom_filter_index_size: u64,
         inverted_index_size: Option<u64>,
-        ngram_filter_index_size: Option<u64>,
         virtual_block_meta: Option<VirtualBlockMeta>,
         compression: Compression,
         create_on: Option<DateTime<Utc>>,
@@ -207,7 +205,6 @@ impl BlockMeta {
             bloom_filter_index_location,
             bloom_filter_index_size,
             inverted_index_size,
-            ngram_filter_index_size,
             virtual_block_meta,
             compression,
             create_on,
@@ -370,7 +367,6 @@ impl BlockMeta {
             inverted_index_size: None,
             virtual_block_meta: None,
             create_on: None,
-            ngram_filter_index_size: None,
         }
     }
 
@@ -396,7 +392,6 @@ impl BlockMeta {
             inverted_index_size: None,
             virtual_block_meta: None,
             create_on: None,
-            ngram_filter_index_size: None,
         }
     }
 }

src/query/storages/common/table_meta/src/meta/v3/frozen/block_meta.rs

@@ -62,7 +62,6 @@ impl From<BlockMeta> for crate::meta::BlockMeta {
             bloom_filter_index_location: value.bloom_filter_index_location,
             bloom_filter_index_size: value.bloom_filter_index_size,
             inverted_index_size: None,
-            ngram_filter_index_size: None,
             virtual_block_meta: None,
             compression: value.compression.into(),
             create_on: None,

src/query/storages/fuse/src/io/write/block_writer.rs

@@ -226,8 +226,6 @@ impl BlockBuilder {
             inverted_index_size,
             virtual_block_meta: None,
             create_on: Some(Utc::now()),
-            // TODO(kould): ngram index
-            ngram_filter_index_size: None,
         };
 
         let serialized = BlockSerialization {

src/query/storages/fuse/src/io/write/stream/block_builder.rs

@@ -332,7 +332,6 @@ impl StreamBlockBuilder {
             compression: self.properties.write_settings.table_compression.into(),
             inverted_index_size,
             create_on: Some(Utc::now()),
-            ngram_filter_index_size: None,
             virtual_block_meta: None,
         };
         let serialized = BlockSerialization {

src/query/storages/fuse/src/statistics/reducers.rs

@@ -234,7 +234,6 @@ pub fn reduce_block_metas<T: Borrow<BlockMeta>>(
         compressed_byte_size += b.file_size;
         index_size += b.bloom_filter_index_size;
         index_size += b.inverted_index_size.unwrap_or_default();
-        index_size += b.ngram_filter_index_size.unwrap_or_default();
         if let Some(virtual_block_meta) = &b.virtual_block_meta {
             index_size += virtual_block_meta.virtual_column_size;
             virtual_block_count += 1;

src/query/storages/fuse/src/table_functions/fuse_block.rs

@@ -65,10 +65,6 @@ impl TableMetaFunc for FuseBlock {
                 "inverted_index_size",
                 TableDataType::Nullable(Box::new(TableDataType::Number(NumberDataType::UInt64))),
             ),
-            TableField::new(
-                "ngram_index_size",
-                TableDataType::Nullable(Box::new(TableDataType::Number(NumberDataType::UInt64))),
-            ),
         ])
     }
 
@@ -90,7 +86,6 @@ impl TableMetaFunc for FuseBlock {
         let mut bloom_filter_location = vec![];
         let mut bloom_filter_size = Vec::with_capacity(len);
         let mut inverted_index_size = Vec::with_capacity(len);
-        let mut ngram_index_size = Vec::with_capacity(len);
 
         let segments_io = SegmentsIO::create(ctx.clone(), tbl.operator.clone(), tbl.schema());
 
@@ -118,7 +113,6 @@ impl TableMetaFunc for FuseBlock {
                     );
                     bloom_filter_size.push(block.bloom_filter_index_size);
                     inverted_index_size.push(block.inverted_index_size);
-                    ngram_index_size.push(block.ngram_filter_index_size);
 
                     row_num += 1;
                     if row_num >= limit {
@@ -163,10 +157,6 @@ impl TableMetaFunc for FuseBlock {
                     DataType::Nullable(Box::new(DataType::Number(NumberDataType::UInt64))),
                     Value::Column(UInt64Type::from_opt_data(inverted_index_size)),
                 ),
-                BlockEntry::new(
-                    DataType::Nullable(Box::new(DataType::Number(NumberDataType::UInt64))),
-                    Value::Column(UInt64Type::from_opt_data(ngram_index_size)),
-                ),
             ],
             row_num,
         ))

tests/sqllogictests/suites/base/09_fuse_engine/09_0006_func_fuse_history.test

@@ -34,7 +34,7 @@ insert into t values (2),(3)
 query III
 select block_count, row_count, index_size from fuse_snapshot('db_09_0006', 't') order by row_count desc limit 1
 ----
-2 3 425
+2 3 426
 
 query II
 select block_count, row_count from fuse_snapshot('db_09_0006', 't') order by row_count;
@@ -52,7 +52,7 @@ query I
 select bloom_filter_size from fuse_block('db_09_0006', 't') order by bloom_filter_size
 ----
 0
-425
+426
 
 statement ok
 create table t1(a int not null) row_per_block=3