tmp

Signed-off-by: Kould <[email protected]>

Author: KKould

src/query/functions/src/scalars/binary.rs

@@ -279,11 +279,9 @@ fn eval_binary_to_string(val: Value<BinaryType>, ctx: &mut EvalContext) -> Value
     vectorize_binary_to_string(
         |col| col.total_bytes_len(),
         |val, output, ctx| {
-            if let Ok(val) = simdutf8::basic::from_utf8(val) {
-                output.put_str(val);
-            } else {
-                ctx.set_error(output.len(), "invalid utf8 sequence");
-            }
+            // FIXME
+            let cow = String::from_utf8_lossy(val);
+            output.put_str(cow.as_ref());
             output.commit_row();
         },
     )(val, ctx)

src/query/storages/common/cache/src/caches.rs

@@ -18,7 +18,6 @@ use std::time::Instant;
 
 use arrow::array::ArrayRef;
 use databend_common_cache::MemSized;
-use databend_storages_common_index::filters::FilterImpl;
 
 pub use crate::cache_items::*;
 use crate::manager::CacheManager;
@@ -46,7 +45,7 @@ pub type TableSnapshotCache = InMemoryLruCache<TableSnapshot>;
 pub type TableSnapshotStatisticCache = InMemoryLruCache<TableSnapshotStatistics>;
 /// In memory object cache of bloom filter.
 /// For each indexed data block, the bloom xor8 filter of column is cached individually
-pub type BloomIndexFilterCache = HybridCache<FilterImpl>;
+pub type BloomIndexFilterCache = HybridCache<Xor8Filter>;
 /// In memory object cache of parquet FileMetaData of bloom index data
 pub type BloomIndexMetaCache = HybridCache<BloomIndexMeta>;
 
@@ -124,7 +123,7 @@ impl CachedObject<(PartStatistics, Partitions)> for (PartStatistics, Partitions)
     }
 }
 
-impl CachedObject<FilterImpl> for FilterImpl {
+impl CachedObject<Xor8Filter> for Xor8Filter {
     type Cache = BloomIndexFilterCache;
     fn cache() -> Option<Self::Cache> {
         CacheManager::instance().get_bloom_index_filter_cache()
@@ -236,10 +235,10 @@ impl From<TableSnapshotStatistics> for CacheValue<TableSnapshotStatistics> {
     }
 }
 
-impl From<FilterImpl> for CacheValue<FilterImpl> {
-    fn from(value: FilterImpl) -> Self {
+impl From<Xor8Filter> for CacheValue<Xor8Filter> {
+    fn from(value: Xor8Filter) -> Self {
         CacheValue {
-            mem_bytes: value.mem_bytes(),
+            mem_bytes: std::mem::size_of::<Xor8Filter>() + value.filter.finger_prints.len(),
             inner: Arc::new(value),
         }
     }

src/query/storages/common/index/src/bloom_index.rs

@@ -74,18 +74,16 @@ use super::eliminate_cast::is_injective_cast;
 use crate::eliminate_cast::cast_const;
 use crate::filters::BlockBloomFilterIndexVersion;
 use crate::filters::BlockFilter;
-use crate::filters::BloomBuilder;
-use crate::filters::BloomFilter;
 use crate::filters::Filter;
 use crate::filters::FilterBuilder;
-use crate::filters::FilterImpl;
-use crate::filters::FilterImplBuilder;
 use crate::filters::V2BloomBlock;
 use crate::filters::Xor8Builder;
 use crate::filters::Xor8Filter;
 use crate::statistics_to_domain;
 use crate::Index;
 
+const NGRAM_HASH_SEED: u64 = 1575457558;
+
 #[derive(Clone, Serialize, Deserialize)]
 pub struct BloomIndexMeta {
     pub columns: Vec<(String, SingleColumnMeta)>,
@@ -187,7 +185,7 @@ pub struct BloomIndex {
     pub version: u64,
 
     ///  filters.
-    pub filters: Vec<Arc<FilterImpl>>,
+    pub filters: Vec<Arc<Xor8Filter>>,
 
     /// Approximate distinct count of columns generated by xor hash function.
     pub column_distinct_count: HashMap<ColumnId, usize>,
@@ -218,7 +216,7 @@ impl BloomIndex {
     pub fn from_filter_block(
         func_ctx: FunctionContext,
         filter_schema: TableSchemaRef,
-        filters: Vec<Arc<FilterImpl>>,
+        filters: Vec<Arc<Xor8Filter>>,
         version: u64,
     ) -> Result<Self> {
         Ok(Self {
@@ -370,10 +368,11 @@ impl BloomIndex {
     pub fn calculate_ngram_nullable_column<'a, F, T>(
         arg: Value<AnyType>,
         gram_size: usize,
+        bitmap_size: usize,
         fn_call: F,
     ) -> impl Iterator<Item = Vec<T>> + 'a
     where
-        F: Fn(&str) -> T + 'a,
+        F: Fn(&str, usize) -> T + 'a,
     {
         (0..arg.len()).filter_map(move |i| {
             arg.index(i).and_then(|scalar| {
@@ -387,7 +386,7 @@ impl BloomIndex {
                     let char_count = indices.len();
 
                     if gram_size > char_count {
-                        return Some(vec![fn_call(&text)]);
+                        return None;
                     }
 
                     let times = char_count - gram_size + 1;
@@ -399,18 +398,23 @@ impl BloomIndex {
                         } else {
                             text.len()
                         };
-                        words.push(fn_call(&text[start..end]));
+                        words.push(fn_call(&text[start..end], bitmap_size));
                     }
                     Some(words)
                 })
             })
         })
     }
 
-    pub fn ngram_hash(s: &str) -> u64 {
-        let mut hasher = CityHasher64::with_seed(1575457558);
+    pub fn ngram_hash(s: &str, bitmap_size: usize) -> u64 {
+        let mut hasher = CityHasher64::with_seed(NGRAM_HASH_SEED);
         DFHash::hash(s, &mut hasher);
-        hasher.finish()
+
+        if bitmap_size > 0 {
+            hasher.finish() % bitmap_size as u64
+        } else {
+            hasher.finish()
+        }
     }
 
     /// calculate digest for constant scalar
@@ -433,7 +437,6 @@ impl BloomIndex {
     }
 
     /// Find all columns that can be use for index in the expression.
-    #[expect(clippy::type_complexity)]
     pub fn filter_index_field(
         expr: &Expr<String>,
         bloom_fields: Vec<TableField>,
@@ -478,45 +481,56 @@ impl BloomIndex {
         }
     }
 
-    pub fn build_filter_ngram_name(field: &TableField) -> String {
-        format!("Ngram({})", field.column_id())
+    pub fn build_filter_ngram_name(field: &TableField, gram_size: usize, bitmap_size: usize) -> String {
+        format!("Ngram({}, gram_size: {gram_size}, bitmap_size: {bitmap_size})", field.column_id())
     }
 
     fn find(
         &self,
-        filter_column: &str,
+        table_field: &TableField,
         target: &Scalar,
         ty: &DataType,
         scalar_map: &HashMap<Scalar, u64>,
         ngram_args: &[NgramArgs],
-        is_ngram: bool,
+        is_like: bool,
     ) -> Result<FilterEvalResult> {
-        if !self.filter_schema.has_field(filter_column)
+        let (filter_column, ngram_arg) = if is_like {
+            let Some(ngram_arg) = ngram_args.iter().find(|arg| &arg.field == table_field) else {
+                // The column doesn't have a Ngram Arg.
+                return Ok(FilterEvalResult::Uncertain);
+            };
+            (BloomIndex::build_filter_ngram_name(table_field, ngram_arg.gram_size, ngram_arg.bitmap_size), Some(ngram_arg))
+        } else {
+            (BloomIndex::build_filter_bloom_name(self.version, table_field)?, None)
+        };
+
+        if !self.filter_schema.has_field(&filter_column)
             || !Xor8Filter::supported_type(ty)
             || target.is_null()
         {
             // The column doesn't have a filter.
             return Ok(FilterEvalResult::Uncertain);
         }
 
-        let idx = self.filter_schema.index_of(filter_column)?;
+        let idx = self.filter_schema.index_of(&filter_column)?;
         let filter = &self.filters[idx];
 
         let contains = if self.version == V2BloomBlock::VERSION {
             let data_value = scalar_to_datavalue(target);
             filter.contains(&data_value)
-        } else if is_ngram && !ngram_args.is_empty() {
-            // NgramFilter is always placed after BloomFilter
-            let offset = self.filters.len() - ngram_args.len();
-            let arg = &ngram_args[idx - offset];
+        } else if let Some(arg) = ngram_arg {
             let Some(words) = BloomIndex::calculate_ngram_nullable_column(
                 Value::Scalar(target.clone()),
                 arg.gram_size,
-                |text| text.to_string(),
+                arg.bitmap_size,
+                |text, _| text.to_string(),
             )
             .next() else {
                 return Ok(FilterEvalResult::Uncertain);
             };
+            if words.is_empty() {
+                return Ok(FilterEvalResult::Uncertain);
+            }
             !words.into_iter().any(|word| {
                 scalar_map
                     .get(&Scalar::String(word))
@@ -563,7 +577,8 @@ struct ColumnFilterBuilder {
     field: TableField,
     is_ngram: bool,
     gram_size: usize,
-    builder: FilterImplBuilder,
+    bitmap_size: usize,
+    builder: Xor8Builder,
 }
 
 #[derive(Clone)]
@@ -591,6 +606,10 @@ impl NgramArgs {
     pub fn gram_size(&self) -> usize {
         self.gram_size
     }
+
+    pub fn bitmap_size(&self) -> usize {
+        self.bitmap_size
+    }
 }
 
 impl BloomIndexBuilder {
@@ -606,7 +625,8 @@ impl BloomIndexBuilder {
                 field: field.clone(),
                 is_ngram: false,
                 gram_size: 0,
-                builder: FilterImplBuilder::Xor(Xor8Builder::create()),
+                bitmap_size: 0,
+                builder: Xor8Builder::create(),
             });
         }
         for arg in ngram_args.iter() {
@@ -615,7 +635,8 @@ impl BloomIndexBuilder {
                 field: arg.field.clone(),
                 is_ngram: true,
                 gram_size: arg.gram_size,
-                builder: FilterImplBuilder::Ngram(BloomBuilder::create(arg.bitmap_size)?),
+                bitmap_size: arg.bitmap_size,
+                builder: Xor8Builder::create(),
             });
         }
 
@@ -737,7 +758,7 @@ impl BloomIndexBuilder {
         }
         for (index, index_column) in ngram_iter {
             let field_type = &block.get_by_offset(index_column.index).data_type;
-            if !BloomFilter::supported_type(field_type) {
+            if !Xor8Filter::supported_type(field_type) {
                 keys_to_remove.push(index);
                 continue;
             }
@@ -753,7 +774,8 @@ impl BloomIndexBuilder {
             for digests in BloomIndex::calculate_ngram_nullable_column(
                 Value::Column(column),
                 index_column.gram_size,
-                BloomIndex::ngram_hash,
+                index_column.bitmap_size,
+                BloomIndex::ngram_hash
             ) {
                 if digests.is_empty() {
                     continue;
@@ -774,7 +796,7 @@ impl BloomIndexBuilder {
         for column in self.columns.iter_mut() {
             let filter = column.builder.build()?;
             let filter_name = if column.is_ngram {
-                BloomIndex::build_filter_ngram_name(&column.field)
+                BloomIndex::build_filter_ngram_name(&column.field, column.gram_size, column.bitmap_size)
             } else {
                 if let Some(len) = filter.len() {
                     if !matches!(
@@ -844,19 +866,32 @@ where T: EqVisitor
                     ..
                 })] => {
                     if let Some(pattern) = scalar.as_string() {
-                        if let LikePattern::SurroundByPercent(v) =
-                            generate_like_pattern(pattern.as_bytes(), 1)
-                        {
-                            let string = String::from_utf8_lossy(v.needle()).to_string();
-
-                            result = self.0.enter_target(
-                                *span,
-                                id,
-                                &Scalar::String(string),
-                                column_type,
-                                return_type,
-                                true,
-                            )?;
+                        match generate_like_pattern(pattern.as_bytes(), 1) {
+                            LikePattern::StartOfPercent(v) | LikePattern::EndOfPercent(v) => {
+                                let string = String::from_utf8_lossy(v.as_ref()).to_string();
+
+                                result = self.0.enter_target(
+                                    *span,
+                                    id,
+                                    &Scalar::String(string),
+                                    column_type,
+                                    return_type,
+                                    true,
+                                )?;
+                            }
+                            LikePattern::SurroundByPercent(v) => {
+                                let string = String::from_utf8_lossy(v.needle()).to_string();
+
+                                result = self.0.enter_target(
+                                    *span,
+                                    id,
+                                    &Scalar::String(string),
+                                    column_type,
+                                    return_type,
+                                    true,
+                                )?;
+                            }
+                            _ => (),
                         }
                     }
                 }
@@ -1099,16 +1134,11 @@ impl EqVisitor for RewriteVisitor<'_> {
         is_like: bool,
     ) -> ResultRewrite {
         let table_field = self.data_schema.field_with_name(col_name)?;
-        let filter_column = if is_like {
-            BloomIndex::build_filter_ngram_name(table_field)
-        } else {
-            BloomIndex::build_filter_bloom_name(self.index.version, table_field)?
-        };
 
         // If the column doesn't contain the constant,
         // we rewrite the expression to a new column with `false` domain.
         if self.index.find(
-            &filter_column,
+            table_field,
             scalar,
             ty,
             self.scalar_map,
@@ -1251,17 +1281,15 @@ impl EqVisitor for ShortListVisitor {
     ) -> ResultRewrite {
         if is_like {
             if let Some((i, v)) = Self::found_field(&self.ngram_fields, col_name) {
-                if !scalar.is_null() && BloomFilter::supported_type(ty) {
+                if !scalar.is_null() && Xor8Filter::supported_type(ty) {
                     self.ngram_founds.push(v.clone());
                     self.ngram_scalars.push((i, scalar.clone()));
                 }
             }
-        } else {
-            if let Some((i, v)) = Self::found_field(&self.bloom_fields, col_name) {
-                if !scalar.is_null() && Xor8Filter::supported_type(ty) {
-                    self.bloom_founds.push(v.clone());
-                    self.bloom_scalars.push((i, scalar.clone(), ty.clone()));
-                }
+        } else if let Some((i, v)) = Self::found_field(&self.bloom_fields, col_name) {
+            if !scalar.is_null() && Xor8Filter::supported_type(ty) {
+                self.bloom_founds.push(v.clone());
+                self.bloom_scalars.push((i, scalar.clone(), ty.clone()));
             }
         }
         Ok(ControlFlow::Break(None))
@@ -1292,7 +1320,7 @@ impl EqVisitor for ShortListVisitor {
             let Some((i, field)) = Self::found_field(&self.ngram_fields, id) else {
                 return Ok(ControlFlow::Break(None));
             };
-            if !BloomFilter::supported_type(src_type) || !is_injective_cast(src_type, dest_type) {
+            if !Xor8Filter::supported_type(src_type) || !is_injective_cast(src_type, dest_type) {
                 return Ok(ControlFlow::Break(None));
             }
 

src/query/storages/common/index/src/filters/mod.rs

@@ -21,12 +21,6 @@ pub use filter::Filter;
 pub use filter::FilterBuilder;
 pub use xor8::BlockBloomFilterIndexVersion;
 pub use xor8::BlockFilter;
-pub use xor8::BloomBuilder;
-pub use xor8::BloomBuildingError;
-pub use xor8::BloomCodecError;
-pub use xor8::BloomFilter;
-pub use xor8::FilterImpl;
-pub use xor8::FilterImplBuilder;
 pub use xor8::V2BloomBlock;
 pub use xor8::Xor8Builder;
 pub use xor8::Xor8BuildingError;