refactor: impl new BloomFilter

Signed-off-by: Kould <[email protected]>

Author: KKould

src/query/service/src/interpreters/interpreter_table_show_create.rs

@@ -27,6 +27,7 @@ use databend_common_expression::ComputedExpr;
 use databend_common_expression::DataBlock;
 use databend_common_expression::Scalar;
 use databend_common_expression::Value;
+use databend_common_meta_app::schema::TableIndexType;
 use databend_common_sql::plans::ShowCreateTablePlan;
 use databend_common_storages_fuse::FUSE_OPT_KEY_ATTACH_COLUMN_IDS;
 use databend_common_storages_stream::stream_table::StreamTable;
@@ -242,9 +243,14 @@ impl ShowCreateTableInterpreter {
                     let option = format!("{} = '{}'", key, value);
                     options.push(option);
                 }
+                let index_type = match index_field.index_type {
+                    TableIndexType::Inverted => "INVERTED",
+                    TableIndexType::Ngram => "NGRAM",
+                };
                 let mut index_str = format!(
-                    "  {} INVERTED INDEX {} ({})",
+                    "  {} {} INDEX {} ({})",
                     sync,
+                    index_type,
                     display_ident(
                         &index_field.name,
                         force_quoted_ident,

src/query/service/src/test_kits/block_writer.rs

@@ -109,6 +109,7 @@ impl<'a> BlockWriter<'a> {
             bloom_filter_index_size,
             None,
             None,
+            None,
             Compression::Lz4Raw,
             Some(Utc::now()),
         );

src/query/sql/src/planner/binder/ddl/index.rs

@@ -65,7 +65,7 @@ use crate::MetadataRef;
 use crate::RefreshAggregatingIndexRewriter;
 use crate::SUPPORTED_AGGREGATING_INDEX_FUNCTIONS;
 
-const MAXIMUM_BLOOM_BITMAP_SIZE: usize = 128 * 1024 * 1024;
+const MAXIMUM_BLOOM_SIZE: usize = 1 << 30;
 
 // valid values for inverted index option tokenizer
 static INDEX_TOKENIZER_VALUES: LazyLock<HashSet<&'static str>> = LazyLock::new(|| {
@@ -598,17 +598,17 @@ impl Binder {
                     }
                     options.insert("gram_size".to_string(), value);
                 }
-                "bitmap_size" => {
+                "bloom_size" => {
                     match value.parse::<usize>() {
                         Ok(num) => {
                             if num == 0 {
                                 return Err(ErrorCode::IndexOptionInvalid(
-                                    "`bitmap_size` cannot be 0",
+                                    "`bloom_size` cannot be 0",
                                 ));
                             }
-                            if num > MAXIMUM_BLOOM_BITMAP_SIZE {
+                            if num > MAXIMUM_BLOOM_SIZE {
                                 return Err(ErrorCode::IndexOptionInvalid(format!(
-                                    "bitmap_size: `{num}` is too large (bitmap_size is maximum: {MAXIMUM_BLOOM_BITMAP_SIZE})",
+                                    "bloom_size: `{num}` is too large (bloom_size is maximum: {MAXIMUM_BLOOM_SIZE})",
                                 )));
                             }
                         }
@@ -618,7 +618,7 @@ impl Binder {
                             )));
                         }
                     }
-                    options.insert("bitmap_size".to_string(), value);
+                    options.insert("bloom_size".to_string(), value);
                 }
                 _ => {
                     return Err(ErrorCode::IndexOptionInvalid(format!(

src/query/storages/common/cache/src/cache_items.rs

@@ -17,7 +17,7 @@ pub use databend_common_catalog::plan::PartStatistics;
 pub use databend_common_catalog::plan::Partitions;
 pub use databend_common_catalog::table::Table;
 use databend_common_exception::ErrorCode;
-pub use databend_storages_common_index::filters::Xor8Filter;
+pub use databend_storages_common_index::filters::FilterImpl;
 pub use databend_storages_common_index::BloomIndexMeta;
 pub use databend_storages_common_index::InvertedIndexFile;
 pub use databend_storages_common_index::InvertedIndexMeta;

src/query/storages/common/cache/src/caches.rs

@@ -45,7 +45,7 @@ pub type TableSnapshotCache = InMemoryLruCache<TableSnapshot>;
 pub type TableSnapshotStatisticCache = InMemoryLruCache<TableSnapshotStatistics>;
 /// In memory object cache of bloom filter.
 /// For each indexed data block, the bloom xor8 filter of column is cached individually
-pub type BloomIndexFilterCache = HybridCache<Xor8Filter>;
+pub type BloomIndexFilterCache = HybridCache<FilterImpl>;
 /// In memory object cache of parquet FileMetaData of bloom index data
 pub type BloomIndexMetaCache = HybridCache<BloomIndexMeta>;
 
@@ -123,7 +123,7 @@ impl CachedObject<(PartStatistics, Partitions)> for (PartStatistics, Partitions)
     }
 }
 
-impl CachedObject<Xor8Filter> for Xor8Filter {
+impl CachedObject<FilterImpl> for FilterImpl {
     type Cache = BloomIndexFilterCache;
     fn cache() -> Option<Self::Cache> {
         CacheManager::instance().get_bloom_index_filter_cache()
@@ -235,10 +235,10 @@ impl From<TableSnapshotStatistics> for CacheValue<TableSnapshotStatistics> {
     }
 }
 
-impl From<Xor8Filter> for CacheValue<Xor8Filter> {
-    fn from(value: Xor8Filter) -> Self {
+impl From<FilterImpl> for CacheValue<FilterImpl> {
+    fn from(value: FilterImpl) -> Self {
         CacheValue {
-            mem_bytes: std::mem::size_of::<Xor8Filter>() + value.filter.finger_prints.len(),
+            mem_bytes: value.mem_bytes(),
             inner: Arc::new(value),
         }
     }