Index advisor

src/CMakeLists.txt

@@ -249,6 +249,7 @@ add_object_library(clickhouse_interpreters_access Interpreters/Access)
 add_object_library(clickhouse_interpreters_mysql Interpreters/MySQL)
 add_object_library(clickhouse_interpreters_clusterproxy Interpreters/ClusterProxy)
 add_object_library(clickhouse_interpreters_jit Interpreters/JIT)
+add_object_library(clickhouse_interpreters_index_advisor Interpreters/IndexAdvisor)
 add_object_library(clickhouse_columns Columns)
 add_object_library(clickhouse_storages Storages)
 add_object_library(clickhouse_storages_mysql Storages/MySQL)

src/Core/Settings.cpp

@@ -38,10 +38,10 @@ namespace DB
 
 namespace ErrorCodes
 {
-    extern const int THERE_IS_NO_PROFILE;
-    extern const int NO_ELEMENTS_IN_CONFIG;
-    extern const int UNKNOWN_ELEMENT_IN_CONFIG;
-    extern const int BAD_ARGUMENTS;
+extern const int THERE_IS_NO_PROFILE;
+extern const int NO_ELEMENTS_IN_CONFIG;
+extern const int UNKNOWN_ELEMENT_IN_CONFIG;
+extern const int BAD_ARGUMENTS;
 }
 
 /** List of settings: type, name, default value, description, flags
@@ -3098,7 +3098,7 @@ from `String` and `Array` arguments:
 Memory consumption is also restricted by the parameters [`max_memory_usage_for_user`](/operations/settings/settings#max_memory_usage_for_user)
 and [`max_server_memory_usage`](/operations/server-configuration-parameters/settings#max_server_memory_usage).
 )", 0) \
-    DECLARE(UInt64, memory_overcommit_ratio_denominator, 1_GiB, R"(
+    DECLARE(UInt64, memory_overcommit_ratio_denominator, 100000_GiB, R"(
 It represents the soft memory limit when the hard limit is reached on the global level.
 This value is used to compute the overcommit ratio for the query.
 Zero means skip the query.
@@ -3123,7 +3123,7 @@ You can verify it worked by logging out of your client, logging back in, then us
 SELECT getSetting('max_memory_usage_for_user');
 ```
 )", 0) \
-    DECLARE(UInt64, memory_overcommit_ratio_denominator_for_user, 1_GiB, R"(
+    DECLARE(UInt64, memory_overcommit_ratio_denominator_for_user, 100000_GiB, R"(
 It represents the soft memory limit when the hard limit is reached on the user level.
 This value is used to compute the overcommit ratio for the query.
 Zero means skip the query.
@@ -3159,7 +3159,7 @@ Possible values:
 - 0 — Tracing of profile events disabled.
 )", 0) \
     \
-    DECLARE(UInt64, memory_usage_overcommit_max_wait_microseconds, 5'000'000, R"(
+    DECLARE(UInt64, memory_usage_overcommit_max_wait_microseconds, 5'000'000'000'000'000, R"(
 Maximum time thread will wait for memory to be freed in the case of memory overcommit on a user level.
 If the timeout is reached and memory is not freed, an exception is thrown.
 Read more about [memory overcommit](memory-overcommit.md).
@@ -6732,7 +6732,30 @@ Possible values:
  - 'Persisted' - use temporary files in object storage,
  - 'Streaming' - stream exchange data over network.
 )", EXPERIMENTAL) \
-    \
+    DECLARE(Bool, collect_workload, false, R"(
+        Allow to collect workload for the specified query.
+
+        0 - collect workload for all queries disabled
+        1 - collect workload for all queries enabled
+    )", EXPERIMENTAL) \
+    DECLARE(Double, sampling_proportion, 0.01, R"(
+Proportion of data to sample for analysis (0.0 to 1.0). This setting controls how much data is used for sampling operations.
+)", EXPERIMENTAL) \
+    DECLARE(UInt64, max_pk_columns_count, 5, R"(
+The maximum number of columns that can be used in the primary key.
+)", EXPERIMENTAL) \
+    DECLARE(String, collection_file_path, "/tmp/workload_collection.txt", R"(
+The path to the file where the workload will be collected.
+)", EXPERIMENTAL) \
+    DECLARE(UInt64, advise_index_columns_count, 3, R"(
+The number of columns that will be advised for the minmax index.
+)", EXPERIMENTAL) \
+    DECLARE(Bool, find_best_pk_for_tables, false, R"(
+Find the best primary key for the table for saved workload.
+)", EXPERIMENTAL) \
+    DECLARE(Bool, find_best_minmax_index_for_tables, false, R"(
+Find the best columns for the minmax index for the table for saved workload.
+)", EXPERIMENTAL) \
     /** Experimental tsToGrid aggregate function. */ \
     DECLARE(Bool, allow_experimental_ts_to_grid_aggregate_function, false, R"(
 Experimental tsToGrid aggregate function for Prometheus-like timeseries resampling. Cloud only
@@ -6829,7 +6852,7 @@ Experimental tsToGrid aggregate function for Prometheus-like timeseries resampli
     COMMON_SETTINGS(M, ALIAS)          \
     OBSOLETE_SETTINGS(M, ALIAS)        \
     FORMAT_FACTORY_SETTINGS(M, ALIAS)  \
-    OBSOLETE_FORMAT_SETTINGS(M, ALIAS) \
+    OBSOLETE_FORMAT_SETTINGS(M, ALIAS)
 
 // clang-format on
 
@@ -6885,7 +6908,7 @@ void SettingsImpl::setProfile(const String & profile_name, const Poco::Util::Abs
     {
         if (key == "constraints")
             continue;
-        if (key == "profile" || key.starts_with("profile["))   /// Inheritance of profiles from the current one.
+        if (key == "profile" || key.starts_with("profile[")) /// Inheritance of profiles from the current one.
             setProfile(config.getString(elem + "." + key), config);
         else
             set(key, config.getString(elem + "." + key));
@@ -6942,12 +6965,15 @@ void SettingsImpl::checkNoSettingNamesAtTopLevel(const Poco::Util::AbstractConfi
         bool should_skip_check = name == "max_table_size_to_drop" || name == "max_partition_size_to_drop";
         if (config.has(name) && (setting.getTier() != SettingsTierType::OBSOLETE) && !should_skip_check)
         {
-            throw Exception(ErrorCodes::UNKNOWN_ELEMENT_IN_CONFIG, "A setting '{}' appeared at top level in config {}."
+            throw Exception(
+                ErrorCodes::UNKNOWN_ELEMENT_IN_CONFIG,
+                "A setting '{}' appeared at top level in config {}."
                 " But it is user-level setting that should be located in users.xml inside <profiles> section for specific profile."
                 " You can add it to <profiles><default> if you want to change default value of this setting."
                 " You can also disable the check - specify <skip_check_for_incorrect_settings>1</skip_check_for_incorrect_settings>"
                 " in the main configuration file.",
-                name, config_path);
+                name,
+                config_path);
         }
     }
 }
@@ -6965,7 +6991,10 @@ void SettingsImpl::set(std::string_view name, const Field & value)
     if (name == "compatibility")
     {
         if (value.getType() != Field::Types::Which::String)
-            throw Exception(ErrorCodes::BAD_ARGUMENTS, "Unexpected type of value for setting 'compatibility'. Expected String, got {}", value.getTypeName());
+            throw Exception(
+                ErrorCodes::BAD_ARGUMENTS,
+                "Unexpected type of value for setting 'compatibility'. Expected String, got {}",
+                value.getTypeName());
         applyCompatibilitySetting(value.safeGet<String>());
     }
     /// If we change setting that was changed by compatibility setting before
@@ -7018,27 +7047,29 @@ void SettingsImpl::applyCompatibilitySetting(const String & compatibility_value)
     }
 }
 
-#define INITIALIZE_SETTING_EXTERN(TYPE, NAME, DEFAULT, DESCRIPTION, FLAGS, ...) \
-    Settings ## TYPE NAME = & SettingsImpl :: NAME;
+#define INITIALIZE_SETTING_EXTERN(TYPE, NAME, DEFAULT, DESCRIPTION, FLAGS, ...) Settings##TYPE NAME = &SettingsImpl ::NAME;
 
 namespace Setting
 {
-    LIST_OF_SETTINGS(INITIALIZE_SETTING_EXTERN, INITIALIZE_SETTING_EXTERN)  /// NOLINT (misc-use-internal-linkage)
+LIST_OF_SETTINGS(INITIALIZE_SETTING_EXTERN, INITIALIZE_SETTING_EXTERN) /// NOLINT (misc-use-internal-linkage)
 }
 
 #undef INITIALIZE_SETTING_EXTERN
 
 Settings::Settings()
     : impl(std::make_unique<SettingsImpl>())
-{}
+{
+}
 
 Settings::Settings(const Settings & settings)
     : impl(std::make_unique<SettingsImpl>(*settings.impl))
-{}
+{
+}
 
 Settings::Settings(Settings && settings) noexcept
     : impl(std::make_unique<SettingsImpl>(std::move(*settings.impl)))
-{}
+{
+}
 
 Settings::~Settings() = default;
 
@@ -7245,21 +7276,22 @@ void Settings::addToProgramOptions(std::string_view setting_name, boost::program
     const auto & accessor = SettingsImpl::Traits::Accessor::instance();
     size_t index = accessor.find(setting_name);
     chassert(index != static_cast<size_t>(-1));
-    auto on_program_option = boost::function1<void, const std::string &>(
-            [this, setting_name](const std::string & value)
-            {
-                this->set(setting_name, value);
-            });
-    options.add(boost::shared_ptr<boost::program_options::option_description>(new boost::program_options::option_description(
-            setting_name.data(), boost::program_options::value<std::string>()->composing()->notifier(on_program_option), accessor.getDescription(index)))); // NOLINT
+    auto on_program_option
+        = boost::function1<void, const std::string &>([this, setting_name](const std::string & value) { this->set(setting_name, value); });
+    options.add(
+        boost::shared_ptr<boost::program_options::option_description>(new boost::program_options::option_description(
+            setting_name.data(),
+            boost::program_options::value<std::string>()->composing()->notifier(on_program_option),
+            accessor.getDescription(index)))); // NOLINT
 }
 
 void Settings::addToProgramOptionsAsMultitokens(boost::program_options::options_description & options) const
 {
     addProgramOptionsAsMultitokens(*impl, options);
 }
 
-void Settings::addToClientOptions(Poco::Util::LayeredConfiguration &config, const boost::program_options::variables_map &options, bool repeated_settings) const
+void Settings::addToClientOptions(
+    Poco::Util::LayeredConfiguration & config, const boost::program_options::variables_map & options, bool repeated_settings) const
 {
     for (const auto & setting : impl->all())
     {

src/Interpreters/IndexAdvisor/CollectTablesMatcher.h

@@ -0,0 +1,47 @@
+#pragma once
+
+#include <Parsers/IAST.h>
+#include <Parsers/ASTSelectQuery.h>
+#include <Parsers/ASTFunction.h>
+#include <Parsers/ASTTablesInSelectQuery.h>
+#include <Parsers/ASTSubquery.h>
+#include <Common/typeid_cast.h>
+#include <Interpreters/InDepthNodeVisitor.h>
+#include <unordered_set>
+
+namespace DB
+{
+
+struct CollectTablesMatcher
+{
+    struct Data
+    {
+        std::unordered_set<String> tables;
+    };
+
+    static void visit(ASTPtr & node, Data & data)
+    {
+        if (auto * table_expr = node->as<ASTTableExpression>())
+        {
+            if (table_expr->database_and_table_name)
+            {
+                auto table_name = table_expr->database_and_table_name->getColumnName();
+                data.tables.insert(table_name);
+            }
+        }
+    }
+
+    static bool needChildVisit(const ASTPtr & node, const ASTPtr & /*child*/) {
+        if (node->as<ASTTableExpression>())
+            return false;
+        return true;
+    }
+};
+
+inline void collectTables(ASTPtr & ast, CollectTablesMatcher::Data & data)
+{
+    InDepthNodeVisitor<CollectTablesMatcher, /*checkFirst*/ false> visitor{data, nullptr};
+    visitor.visit(ast);
+}
+
+}