Skip to content

[Chore](profile) attach always true counter to column predicate #58463

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open

[Chore](profile) attach always true counter to column predicate #58463

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 19 additions & 10 deletions be/src/exec/olap_common.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -210,7 +210,8 @@ class ColumnValueRange {
if (null_pred.condition_values.size() != 0) {
filters.emplace_back(_column_name, null_pred, _runtime_filter_id,
_predicate_filtered_rows_counter,
_predicate_input_rows_counter);
_predicate_input_rows_counter,
_predicate_always_true_rows_counter);
return;
}

Expand All @@ -223,9 +224,9 @@ class ColumnValueRange {
}

if (low.condition_values.size() != 0) {
filters.emplace_back(_column_name, low, _runtime_filter_id,
_predicate_filtered_rows_counter,
_predicate_input_rows_counter);
filters.emplace_back(
_column_name, low, _runtime_filter_id, _predicate_filtered_rows_counter,
_predicate_input_rows_counter, _predicate_always_true_rows_counter);
}

TCondition high;
Expand All @@ -237,9 +238,9 @@ class ColumnValueRange {
}

if (high.condition_values.size() != 0) {
filters.emplace_back(_column_name, high, _runtime_filter_id,
_predicate_filtered_rows_counter,
_predicate_input_rows_counter);
filters.emplace_back(
_column_name, high, _runtime_filter_id, _predicate_filtered_rows_counter,
_predicate_input_rows_counter, _predicate_always_true_rows_counter);
}
} else {
// 3. convert to is null and is not null filter condition
Expand All @@ -254,7 +255,8 @@ class ColumnValueRange {
if (null_pred.condition_values.size() != 0) {
filters.emplace_back(_column_name, null_pred, _runtime_filter_id,
_predicate_filtered_rows_counter,
_predicate_input_rows_counter);
_predicate_input_rows_counter,
_predicate_always_true_rows_counter);
}
}
}
Expand All @@ -271,7 +273,8 @@ class ColumnValueRange {

if (condition.condition_values.size() != 0) {
filters.emplace_back(_column_name, condition, _runtime_filter_id,
_predicate_filtered_rows_counter, _predicate_input_rows_counter);
_predicate_filtered_rows_counter, _predicate_input_rows_counter,
_predicate_always_true_rows_counter);
}
}

Expand Down Expand Up @@ -311,7 +314,8 @@ class ColumnValueRange {
void attach_profile_counter(
int runtime_filter_id,
std::shared_ptr<RuntimeProfile::Counter> predicate_filtered_rows_counter,
std::shared_ptr<RuntimeProfile::Counter> predicate_input_rows_counter) {
std::shared_ptr<RuntimeProfile::Counter> predicate_input_rows_counter,
std::shared_ptr<RuntimeProfile::Counter> predicate_always_true_rows_counter) {
DCHECK(predicate_filtered_rows_counter != nullptr);
DCHECK(predicate_input_rows_counter != nullptr);

Expand All @@ -323,6 +327,9 @@ class ColumnValueRange {
if (predicate_input_rows_counter != nullptr) {
_predicate_input_rows_counter = predicate_input_rows_counter;
}
if (predicate_always_true_rows_counter != nullptr) {
_predicate_always_true_rows_counter = predicate_always_true_rows_counter;
}
}

int precision() const { return _precision; }
Expand Down Expand Up @@ -397,6 +404,8 @@ class ColumnValueRange {
std::make_shared<RuntimeProfile::Counter>(TUnit::UNIT, 0);
std::shared_ptr<RuntimeProfile::Counter> _predicate_input_rows_counter =
std::make_shared<RuntimeProfile::Counter>(TUnit::UNIT, 0);
std::shared_ptr<RuntimeProfile::Counter> _predicate_always_true_rows_counter =
std::make_shared<RuntimeProfile::Counter>(TUnit::UNIT, 0);
};
template <>
const typename ColumnValueRange<TYPE_FLOAT>::CppType ColumnValueRange<TYPE_FLOAT>::TYPE_MIN;
Expand Down
2 changes: 1 addition & 1 deletion be/src/olap/bitmap_filter_predicate.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -111,7 +111,7 @@ uint16_t BitmapFilterColumnPredicate<T>::_evaluate_inner(const vectorized::IColu
} else {
new_size = evaluate<false>(column, nullptr, sel, size);
}
update_filter_info(size - new_size, size);
update_filter_info(size - new_size, size, 0);
return new_size;
}
} //namespace doris
21 changes: 17 additions & 4 deletions be/src/olap/column_predicate.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -187,14 +187,15 @@ class ColumnPredicate {
// a short circuit eval way
uint16_t evaluate(const vectorized::IColumn& column, uint16_t* sel, uint16_t size) const {
if (always_true()) {
update_filter_info(0, 0, size);
return size;
}

uint16_t new_size = _evaluate_inner(column, sel, size);
if (_can_ignore()) {
do_judge_selectivity(size - new_size, size);
}
update_filter_info(size - new_size, size);
update_filter_info(size - new_size, size, 0);
return new_size;
}
virtual void evaluate_and(const vectorized::IColumn& column, const uint16_t* sel, uint16_t size,
Expand Down Expand Up @@ -273,7 +274,8 @@ class ColumnPredicate {

void attach_profile_counter(
int filter_id, std::shared_ptr<RuntimeProfile::Counter> predicate_filtered_rows_counter,
std::shared_ptr<RuntimeProfile::Counter> predicate_input_rows_counter) {
std::shared_ptr<RuntimeProfile::Counter> predicate_input_rows_counter,
std::shared_ptr<RuntimeProfile::Counter> predicate_always_true_rows_counter) {
_runtime_filter_id = filter_id;
DCHECK(predicate_filtered_rows_counter != nullptr);
DCHECK(predicate_input_rows_counter != nullptr);
Expand All @@ -284,12 +286,22 @@ class ColumnPredicate {
if (predicate_input_rows_counter != nullptr) {
_predicate_input_rows_counter = predicate_input_rows_counter;
}
if (predicate_always_true_rows_counter != nullptr) {
_predicate_always_true_rows_counter = predicate_always_true_rows_counter;
}
}

/// TODO: Currently we only record statistics for runtime filters, in the future we should record for all predicates
void update_filter_info(int64_t filter_rows, int64_t input_rows) const {
void update_filter_info(int64_t filter_rows, int64_t input_rows,
int64_t always_true_rows) const {
if (_predicate_input_rows_counter == nullptr ||
_predicate_filtered_rows_counter == nullptr ||
_predicate_always_true_rows_counter == nullptr) {
throw Exception(INTERNAL_ERROR, "Predicate profile counters are not initialized");
}
COUNTER_UPDATE(_predicate_input_rows_counter, input_rows);
COUNTER_UPDATE(_predicate_filtered_rows_counter, filter_rows);
COUNTER_UPDATE(_predicate_always_true_rows_counter, always_true_rows);
}

static std::string pred_type_string(PredicateType type) {
Expand Down Expand Up @@ -378,9 +390,10 @@ class ColumnPredicate {

std::shared_ptr<RuntimeProfile::Counter> _predicate_filtered_rows_counter =
std::make_shared<RuntimeProfile::Counter>(TUnit::UNIT, 0);

std::shared_ptr<RuntimeProfile::Counter> _predicate_input_rows_counter =
std::make_shared<RuntimeProfile::Counter>(TUnit::UNIT, 0);
std::shared_ptr<RuntimeProfile::Counter> _predicate_always_true_rows_counter =
std::make_shared<RuntimeProfile::Counter>(TUnit::UNIT, 0);
};

} //namespace doris
19 changes: 10 additions & 9 deletions be/src/olap/comparison_predicate.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -349,8 +349,8 @@ class ComparisonPredicateBase : public ColumnPredicate {
}

template <bool is_and>
void __attribute__((flatten))
_evaluate_vec_internal(const vectorized::IColumn& column, uint16_t size, bool* flags) const {
void __attribute__((flatten)) _evaluate_vec_internal(const vectorized::IColumn& column,
uint16_t size, bool* flags) const {
uint16_t current_evaluated_rows = 0;
uint16_t current_passed_rows = 0;
if (_can_ignore()) {
Expand All @@ -368,8 +368,8 @@ class ComparisonPredicateBase : public ColumnPredicate {
// so reference here is safe.
// https://stackoverflow.com/questions/14688285/c-local-variable-destruction-order
Defer defer([&]() {
update_filter_info(current_evaluated_rows - current_passed_rows,
current_evaluated_rows);
update_filter_info(current_evaluated_rows - current_passed_rows, current_evaluated_rows,
0);
});

if (column.is_nullable()) {
Expand Down Expand Up @@ -577,9 +577,10 @@ class ComparisonPredicateBase : public ColumnPredicate {
}

template <bool is_nullable, bool is_and, typename TArray, typename TValue>
void __attribute__((flatten))
_base_loop_vec(uint16_t size, bool* __restrict bflags, const uint8_t* __restrict null_map,
const TArray* __restrict data_array, const TValue& value) const {
void __attribute__((flatten)) _base_loop_vec(uint16_t size, bool* __restrict bflags,
const uint8_t* __restrict null_map,
const TArray* __restrict data_array,
const TValue& value) const {
//uint8_t helps compiler to generate vectorized code
auto* flags = reinterpret_cast<uint8_t*>(bflags);
if constexpr (is_and) {
Expand Down Expand Up @@ -694,8 +695,8 @@ class ComparisonPredicateBase : public ColumnPredicate {
}
}

int32_t __attribute__((flatten))
_find_code_from_dictionary_column(const vectorized::ColumnDictI32& column) const {
int32_t __attribute__((flatten)) _find_code_from_dictionary_column(
const vectorized::ColumnDictI32& column) const {
int32_t code = 0;
if (_segment_id_to_cached_code.if_contains(
column.get_rowset_segment_id(),
Expand Down
8 changes: 7 additions & 1 deletion be/src/olap/filter_olap_param.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,8 @@ template <typename T>
struct FilterOlapParam {
FilterOlapParam(std::string column_name, T filter, int runtime_filter_id,
std::shared_ptr<RuntimeProfile::Counter> filtered_counter,
std::shared_ptr<RuntimeProfile::Counter> input_counter)
std::shared_ptr<RuntimeProfile::Counter> input_counter,
std::shared_ptr<RuntimeProfile::Counter> always_true_counter)
: column_name(std::move(column_name)),
filter(std::move(filter)),
runtime_filter_id(runtime_filter_id) {
Expand All @@ -37,6 +38,9 @@ struct FilterOlapParam {
if (input_counter != nullptr) {
input_rows_counter = input_counter;
}
if (always_true_counter != nullptr) {
always_true_rows_counter = always_true_counter;
}
}

std::string column_name;
Expand All @@ -46,6 +50,8 @@ struct FilterOlapParam {
std::make_shared<RuntimeProfile::Counter>(TUnit::UNIT, 0);
std::shared_ptr<RuntimeProfile::Counter> input_rows_counter =
std::make_shared<RuntimeProfile::Counter>(TUnit::UNIT, 0);
std::shared_ptr<RuntimeProfile::Counter> always_true_rows_counter =
std::make_shared<RuntimeProfile::Counter>(TUnit::UNIT, 0);
};

} // namespace doris
3 changes: 2 additions & 1 deletion be/src/olap/rowset/segment_v2/segment_iterator.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2254,7 +2254,8 @@ uint16_t SegmentIterator::_evaluate_vectorization_predicate(uint16_t* sel_rowid_
for (const auto& pred : _pre_eval_block_predicate) {
if (!pred->always_true()) {
all_pred_always_true = false;
break;
} else {
pred->update_filter_info(0, 0, selected_size);
}
}
if (all_pred_always_true) {
Expand Down
4 changes: 2 additions & 2 deletions be/src/olap/tablet_reader.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -527,7 +527,7 @@ Status TabletReader::_init_conditions_param(const ReaderParams& read_params) {
for (const auto& param : params) {
ColumnPredicate* predicate = _parse_to_predicate({param.column_name, param.filter});
predicate->attach_profile_counter(param.runtime_filter_id, param.filtered_rows_counter,
param.input_rows_counter);
param.input_rows_counter, param.always_true_rows_counter);
predicates.emplace_back(predicate);
}
};
Expand All @@ -545,7 +545,7 @@ Status TabletReader::_init_conditions_param(const ReaderParams& read_params) {
// record condition value into predicate_params in order to pushdown segment_iterator,
// _gen_predicate_result_sign will build predicate result unique sign with condition value
predicate->attach_profile_counter(param.runtime_filter_id, param.filtered_rows_counter,
param.input_rows_counter);
param.input_rows_counter, param.always_true_rows_counter);
predicates.emplace_back(predicate);
}
parse_and_emplace_predicates(read_params.bloom_filters);
Expand Down
16 changes: 12 additions & 4 deletions be/src/pipeline/exec/scan_operator.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -335,7 +335,8 @@ Status ScanLocalState<Derived>::_normalize_predicate(
value_range.attach_profile_counter(
rf_expr->filter_id(),
rf_expr->predicate_filtered_rows_counter(),
rf_expr->predicate_input_rows_counter());
rf_expr->predicate_input_rows_counter(),
rf_expr->predicate_always_true_rows_counter());
}
}};
RETURN_IF_PUSH_DOWN(_normalize_in_and_eq_predicate(
Expand Down Expand Up @@ -402,7 +403,8 @@ Status ScanLocalState<Derived>::_normalize_bloom_filter(vectorized::VExpr* expr,
_filter_predicates.bloom_filters.emplace_back(
slot->col_name(), expr->get_bloom_filter_func(), rf_expr->filter_id(),
rf_expr->predicate_filtered_rows_counter(),
rf_expr->predicate_input_rows_counter());
rf_expr->predicate_input_rows_counter(),
rf_expr->predicate_always_true_rows_counter());
*pdt = temp_pdt;
}
}
Expand All @@ -422,7 +424,8 @@ Status ScanLocalState<Derived>::_normalize_bitmap_filter(vectorized::VExpr* expr
_filter_predicates.bitmap_filters.emplace_back(
slot->col_name(), expr->get_bitmap_filter_func(), rf_expr->filter_id(),
rf_expr->predicate_filtered_rows_counter(),
rf_expr->predicate_input_rows_counter());
rf_expr->predicate_input_rows_counter(),
rf_expr->predicate_always_true_rows_counter());
*pdt = temp_pdt;
}
}
Expand Down Expand Up @@ -634,16 +637,21 @@ Status ScanLocalState<Derived>::_normalize_in_and_eq_predicate(vectorized::VExpr
int runtime_filter_id = -1;
std::shared_ptr<RuntimeProfile::Counter> predicate_filtered_rows_counter = nullptr;
std::shared_ptr<RuntimeProfile::Counter> predicate_input_rows_counter = nullptr;
std::shared_ptr<RuntimeProfile::Counter> predicate_always_true_rows_counter =
nullptr;
if (expr_ctx->root()->is_rf_wrapper()) {
auto* rf_expr =
assert_cast<vectorized::VRuntimeFilterWrapper*>(expr_ctx->root().get());
runtime_filter_id = rf_expr->filter_id();
predicate_filtered_rows_counter = rf_expr->predicate_filtered_rows_counter();
predicate_input_rows_counter = rf_expr->predicate_input_rows_counter();
predicate_always_true_rows_counter =
rf_expr->predicate_always_true_rows_counter();
}
_filter_predicates.in_filters.emplace_back(
slot->col_name(), expr->get_set_func(), runtime_filter_id,
predicate_filtered_rows_counter, predicate_input_rows_counter);
predicate_filtered_rows_counter, predicate_input_rows_counter,
predicate_always_true_rows_counter);
*pdt = PushDownType::ACCEPTABLE;
return Status::OK();
}
Expand Down
3 changes: 3 additions & 0 deletions be/src/vec/exprs/vruntimefilter_wrapper.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -124,6 +124,9 @@ class VRuntimeFilterWrapper final : public VExpr {
std::shared_ptr<RuntimeProfile::Counter> predicate_input_rows_counter() const {
return _rf_input_rows;
}
std::shared_ptr<RuntimeProfile::Counter> predicate_always_true_rows_counter() const {
return _always_true_filter_rows;
}

private:
void reset_judge_selectivity() const {
Expand Down
Loading