refactor SQL translators YQL-19594

commit_hash:401d21dd23ee9bb7ee52b2fc42e596cb3e4bdda7
ydb-platform · Feb 15, 2025 · 802da27 · 802da27
1 parent b49c2fc
commit 802da27
Show file tree

Hide file tree

Showing 34 changed files with 211 additions and 134 deletions.
diff --git a/yql/essentials/core/extract_predicate/ut/extract_predicate_ut.cpp b/yql/essentials/core/extract_predicate/ut/extract_predicate_ut.cpp
@@ -6,6 +6,7 @@
 #include <yql/essentials/providers/common/provider/yql_provider_names.h>
 #include <yql/essentials/sql/settings/translation_settings.h>
 #include <yql/essentials/sql/sql.h>
+#include <yql/essentials/sql/v1/sql.h>
 #include <yql/essentials/ast/yql_ast_annotation.h>
 #include <yql/essentials/ast/yql_expr.h>
 #include <yql/essentials/core/cbo/simple/cbo_simple.h>
@@ -53,7 +54,13 @@ Y_UNIT_TEST_SUITE(TYqlExtractPredicate) {
         NSQLTranslation::TTranslationSettings settings;
         settings.SyntaxVersion = 1;
 
-        TAstParseResult astRes = SqlToYql(program, settings);
+        NSQLTranslation::TTranslators translators(
+            nullptr,
+            NSQLTranslationV1::MakeTranslator(),
+            nullptr
+        );
+
+        TAstParseResult astRes = SqlToYql(translators, program, settings);
         UNIT_ASSERT(astRes.IsOk());
         TExprNode::TPtr exprRoot;
         UNIT_ASSERT(CompileExpr(*astRes.Root, exprRoot, exprCtx, nullptr, nullptr));

diff --git a/yql/essentials/core/extract_predicate/ut/ya.make b/yql/essentials/core/extract_predicate/ut/ya.make
@@ -21,6 +21,8 @@ PEERDIR(
     yql/essentials/minikql/invoke_builtins/llvm16
     yql/essentials/sql/pg
     yql/essentials/parser/pg_wrapper
+    yql/essentials/sql
+    yql/essentials/sql/v1
 )
 
 YQL_LAST_ABI_VERSION()

diff --git a/yql/essentials/core/facade/ya.make b/yql/essentials/core/facade/ya.make
@@ -21,6 +21,8 @@ PEERDIR(
     yql/essentials/core/qplayer/storage/interface
     yql/essentials/core/qplayer/udf_resolver
     yql/essentials/sql
+    yql/essentials/sql/v1
+    yql/essentials/parser/pg_wrapper/interface
     yql/essentials/utils/log
     yql/essentials/core
     yql/essentials/core/type_ann

diff --git a/yql/essentials/core/facade/yql_facade.cpp b/yql/essentials/core/facade/yql_facade.cpp
@@ -10,6 +10,9 @@
 #include <yql/essentials/core/type_ann/type_ann_expr.h>
 #include <yql/essentials/core/services/yql_plan.h>
 #include <yql/essentials/core/services/yql_eval_params.h>
+#include <yql/essentials/sql/sql.h>
+#include <yql/essentials/sql/v1/sql.h>
+#include <yql/essentials/parser/pg_wrapper/interface/parser.h>
 #include <yql/essentials/utils/log/context.h>
 #include <yql/essentials/utils/log/profile.h>
 #include <yql/essentials/utils/limiting_allocator.h>
@@ -728,7 +731,13 @@ bool TProgram::ParseSql(const NSQLTranslation::TTranslationSettings& settings)
     }
 
     currentSettings->EmitReadsForExists = true;
-    return FillParseResult(SqlToYql(sourceCode, *currentSettings, &warningRules), &warningRules);
+    NSQLTranslation::TTranslators translators(
+        nullptr,
+        NSQLTranslationV1::MakeTranslator(),
+        NSQLTranslationPG::MakeTranslator()
+    );
+
+    return FillParseResult(SqlToYql(translators, sourceCode, *currentSettings, &warningRules), &warningRules);
 }
 
 bool TProgram::Compile(const TString& username, bool skipLibraries) {

diff --git a/yql/essentials/core/services/mounts/ya.make b/yql/essentials/core/services/mounts/ya.make
@@ -10,6 +10,8 @@ PEERDIR(
     yql/essentials/core/user_data
     yql/essentials/core
     yql/essentials/utils/log
+    yql/essentials/sql
+    yql/essentials/sql/v1
 )
 
 YQL_LAST_ABI_VERSION()

diff --git a/yql/essentials/core/services/mounts/yql_mounts.cpp b/yql/essentials/core/services/mounts/yql_mounts.cpp
@@ -1,6 +1,8 @@
 #include "yql_mounts.h"
 
 #include <yql/essentials/core/yql_library_compiler.h>
+#include <yql/essentials/sql/sql.h>
+#include <yql/essentials/sql/v1/sql.h>
 #include <yql/essentials/utils/log/profile.h>
 
 #include <library/cpp/resource/resource.h>
@@ -124,16 +126,22 @@ namespace NYql {
         TUserDataTable mounts;
         LoadYqlDefaultMounts(mounts);
 
+        NSQLTranslation::TTranslators translators(
+            nullptr,
+            NSQLTranslationV1::MakeTranslator(),
+            nullptr
+        );
+
         TModulesTable modulesTable;
-        if (!CompileLibraries(mounts, *ctx, modulesTable, optimizeLibraries)) {
+        if (!CompileLibraries(translators, mounts, *ctx, modulesTable, optimizeLibraries)) {
             return {};
         }
 
         for (const auto& item : userData) {
             AddUserDataToTable(mounts, item);
         }
 
-        moduleResolver = std::make_shared<TModuleResolver>(std::move(modulesTable), ctx->NextUniqueId,
+        moduleResolver = std::make_shared<TModuleResolver>(translators, std::move(modulesTable), ctx->NextUniqueId,
             clusterMapping, sqlFlags, optimizeLibraries, std::move(ownedCtx));
         return mounts;
     }

diff --git a/yql/essentials/core/ut/ya.make b/yql/essentials/core/ut/ya.make
@@ -32,6 +32,8 @@ PEERDIR(
     yql/essentials/minikql/invoke_builtins/llvm16
     yql/essentials/parser/pg_wrapper
     yql/essentials/sql/pg
+    yql/essentials/sql
+    yql/essentials/sql/v1
     yql/essentials/udfs/common/string
 )
 

diff --git a/yql/essentials/core/ut/yql_library_compiler_ut.cpp b/yql/essentials/core/ut/yql_library_compiler_ut.cpp
@@ -2,16 +2,25 @@
 
 #include "yql_library_compiler.h"
 
+#include <yql/essentials/sql/sql.h>
+#include <yql/essentials/sql/v1/sql.h>
+
 namespace NYql {
 
 Y_UNIT_TEST_SUITE(TLibraryCompilerTests) {
 
     static const char* alias = "/lib/ut.yql";
 
     static bool CompileAndLink(const THashMap<TString, TString>& libs, TExprContext& ctx) {
+        NSQLTranslation::TTranslators translators(
+            nullptr,
+            NSQLTranslationV1::MakeTranslator(),
+            nullptr
+        );
+
         THashMap<TString, TLibraryCohesion> compiled;
         for (const auto& lib : libs)
-            if (!CompileLibrary(alias, lib.second, ctx, compiled[lib.first]))
+            if (!CompileLibrary(translators, alias, lib.second, ctx, compiled[lib.first]))
                 return false;
 
         return LinkLibraries(compiled, ctx, ctx);
@@ -25,9 +34,15 @@ Y_UNIT_TEST_SUITE(TLibraryCompilerTests) {
             "(export X)\n"
             ")\n";
 
+        NSQLTranslation::TTranslators translators(
+            nullptr,
+            NSQLTranslationV1::MakeTranslator(),
+            nullptr
+        );
+
         TExprContext ctx;
         TLibraryCohesion cohesion;
-        UNIT_ASSERT(CompileLibrary(alias, s, ctx, cohesion));
+        UNIT_ASSERT(CompileLibrary(translators, alias, s, ctx, cohesion));
         UNIT_ASSERT_VALUES_EQUAL(2, cohesion.Exports.Symbols().size());
         UNIT_ASSERT(cohesion.Imports.empty());
     }
@@ -41,9 +56,15 @@ Y_UNIT_TEST_SUITE(TLibraryCompilerTests) {
             "(export ex)\n"
             ")\n";
 
+        NSQLTranslation::TTranslators translators(
+            nullptr,
+            NSQLTranslationV1::MakeTranslator(),
+            nullptr
+        );
+
         TExprContext ctx;
         TLibraryCohesion cohesion;
-        UNIT_ASSERT(CompileLibrary(alias, s, ctx, cohesion));
+        UNIT_ASSERT(CompileLibrary(translators, alias, s, ctx, cohesion));
         UNIT_ASSERT_VALUES_EQUAL(1, cohesion.Exports.Symbols().size());
         UNIT_ASSERT_VALUES_EQUAL(2, cohesion.Imports.size());
     }

diff --git a/yql/essentials/core/yql_library_compiler.cpp b/yql/essentials/core/yql_library_compiler.cpp
@@ -89,14 +89,15 @@ bool OptimizeLibrary(TLibraryCohesion& cohesion, TExprContext& ctx) {
     return true;
 }
 
-bool CompileLibrary(const TString& alias, const TString& script, TExprContext& ctx, TLibraryCohesion& cohesion, bool optimize)
+bool CompileLibrary(const NSQLTranslation::TTranslators& translators, const TString& alias,
+    const TString& script, TExprContext& ctx, TLibraryCohesion& cohesion, bool optimize)
 {
     TAstParseResult res;
     if (alias.EndsWith(".sql")) {
         NSQLTranslation::TTranslationSettings translationSettings;
         translationSettings.SyntaxVersion = 1;
         translationSettings.Mode = NSQLTranslation::ESqlMode::LIBRARY;
-        res = NSQLTranslation::SqlToYql(script, translationSettings);
+        res = NSQLTranslation::SqlToYql(translators, script, translationSettings);
     } else {
         res = ParseAst(script, nullptr, alias);
     }
@@ -198,7 +199,8 @@ bool LinkLibraries(THashMap<TString, TLibraryCohesion>& libs, TExprContext& ctx,
     return true;
 }
 
-bool CompileLibraries(const TUserDataTable& userData, TExprContext& ctx, TModulesTable& modules, bool optimize)
+bool CompileLibraries(const NSQLTranslation::TTranslators& translators, const TUserDataTable& userData,
+    TExprContext& ctx, TModulesTable& modules, bool optimize)
 {
     THashMap<TString, TLibraryCohesion> libs;
     for (const auto& data : userData) {
@@ -212,7 +214,7 @@ bool CompileLibraries(const TUserDataTable& userData, TExprContext& ctx, TModule
             }
 
             if (!libraryData.empty()) {
-                if (CompileLibrary(alias, libraryData, ctx, libs[alias], optimize))
+                if (CompileLibrary(translators, alias, libraryData, ctx, libs[alias], optimize))
                     modules[TModuleResolver::NormalizeModuleName(alias)] = libs[alias].Exports;
                 else
                     return false;
@@ -223,4 +225,8 @@ bool CompileLibraries(const TUserDataTable& userData, TExprContext& ctx, TModule
     return LinkLibraries(libs, ctx, ctx);
 }
 
+bool CompileLibraries(const TUserDataTable& userData, TExprContext& ctx, TModulesTable& modules, bool optimize) {
+    return CompileLibraries(NSQLTranslation::MakeAllTranslators(), userData, ctx, modules, optimize);
+}
+
 }
diff --git a/yql/essentials/core/yql_library_compiler.h b/yql/essentials/core/yql_library_compiler.h
@@ -2,14 +2,19 @@
 
 #include <yql/essentials/ast/yql_expr.h>
 #include "yql_type_annotation.h"
+#include <yql/essentials/sql/sql.h>
 
 namespace NYql {
 
 bool OptimizeLibrary(TLibraryCohesion& cohesion, TExprContext& ctx);
-bool CompileLibrary(const TString& alias, const TString& script, TExprContext& ctx, TLibraryCohesion& cohesion, bool optimize = true);
+bool CompileLibrary(const NSQLTranslation::TTranslators& translators, const TString& alias,
+    const TString& script, TExprContext& ctx, TLibraryCohesion& cohesion, bool optimize = true);
 
 bool LinkLibraries(THashMap<TString, TLibraryCohesion>& libs, TExprContext& ctx, TExprContext& ctxToClone, const std::function<const TExportTable*(const TString&)>& module2ExportTable);
 bool LinkLibraries(THashMap<TString, TLibraryCohesion>& libs, TExprContext& ctx, TExprContext& ctxToClone, const TModulesTable* loadedModules = nullptr);
 
+bool CompileLibraries(const NSQLTranslation::TTranslators& translators, const TUserDataTable& userData,
+    TExprContext& ctx, TModulesTable& modules, bool optimize = true);
+//FIXME remove
 bool CompileLibraries(const TUserDataTable& userData, TExprContext& ctx, TModulesTable& modules, bool optimize = true);
 }
diff --git a/yql/essentials/core/yql_type_annotation.cpp b/yql/essentials/core/yql_type_annotation.cpp
@@ -4,8 +4,6 @@
 #include "yql_library_compiler.h"
 #include "yql_type_helpers.h"
 
-#include <yql/essentials/sql/sql.h>
-#include <yql/essentials/sql/settings/translation_settings.h>
 #include <yql/essentials/ast/yql_constraint.h>
 #include <yql/essentials/utils/log/log.h>
 
@@ -529,7 +527,7 @@ bool TModuleResolver::AddFromMemory(const TString& fullName, const TString& modu
         settings.SyntaxVersion = syntaxVersion;
         settings.V0Behavior = NSQLTranslation::EV0Behavior::Silent;
         settings.FileAliasPrefix = FileAliasPrefix;
-        astRes = SqlToYql(query, settings);
+        astRes = SqlToYql(Translators, query, settings);
         if (!astRes.IsOk()) {
             ctx.AddError(addSubIssues(TIssue(pos, TStringBuilder() << "Failed to parse SQL: " << fullName), astRes.Issues));
             return false;
@@ -646,7 +644,7 @@ IModuleResolver::TPtr TModuleResolver::CreateMutableChild() const {
         throw yexception() << "Module resolver should not contain user data and URL loader";
     }
 
-    return std::make_shared<TModuleResolver>(&Modules, LibsContext.NextUniqueId, ClusterMapping, SqlFlags, OptimizeLibraries, KnownPackages, Libs, FileAliasPrefix);
+    return std::make_shared<TModuleResolver>(Translators, &Modules, LibsContext.NextUniqueId, ClusterMapping, SqlFlags, OptimizeLibraries, KnownPackages, Libs, FileAliasPrefix);
 }
 
 void TModuleResolver::SetFileAliasPrefix(TString&& prefix) {

diff --git a/yql/essentials/core/yql_type_annotation.h b/yql/essentials/core/yql_type_annotation.h
@@ -12,6 +12,7 @@
 #include <yql/essentials/core/url_lister/interface/url_lister_manager.h>
 #include <yql/essentials/core/qplayer/storage/interface/yql_qstorage.h>
 #include <yql/essentials/ast/yql_expr.h>
+#include <yql/essentials/sql/sql.h>
 
 #include <library/cpp/yson/node/node.h>
 #include <library/cpp/time_provider/time_provider.h>
@@ -39,9 +40,11 @@ class IUrlLoader : public TThrRefBase {
 
 class TModuleResolver : public IModuleResolver {
 public:
-    TModuleResolver(TModulesTable&& modules, ui64 nextUniqueId, const THashMap<TString, TString>& clusterMapping,
+    TModuleResolver(const NSQLTranslation::TTranslators& translators, TModulesTable&& modules,
+        ui64 nextUniqueId, const THashMap<TString, TString>& clusterMapping,
         const THashSet<TString>& sqlFlags, bool optimizeLibraries = true, THolder<TExprContext> ownedCtx = {})
-        : OwnedCtx(std::move(ownedCtx))
+        : Translators(translators)
+        , OwnedCtx(std::move(ownedCtx))
         , LibsContext(nextUniqueId)
         , Modules(std::move(modules))
         , ClusterMapping(clusterMapping)
@@ -53,10 +56,19 @@ class TModuleResolver : public IModuleResolver {
         }
     }
 
-    TModuleResolver(const TModulesTable* parentModules, ui64 nextUniqueId, const THashMap<TString, TString>& clusterMapping,
+    //FIXME remove
+    TModuleResolver(TModulesTable&& modules,
+        ui64 nextUniqueId, const THashMap<TString, TString>& clusterMapping,
+        const THashSet<TString>& sqlFlags, bool optimizeLibraries = true, THolder<TExprContext> ownedCtx = {})
+        : TModuleResolver(NSQLTranslation::MakeAllTranslators(), std::move(modules), nextUniqueId, clusterMapping, sqlFlags, optimizeLibraries, std::move(ownedCtx))
+    {}
+
+    TModuleResolver(const NSQLTranslation::TTranslators& translators, const TModulesTable* parentModules,
+        ui64 nextUniqueId, const THashMap<TString, TString>& clusterMapping,
         const THashSet<TString>& sqlFlags, bool optimizeLibraries, const TSet<TString>& knownPackages, const THashMap<TString,
         THashMap<int, TLibraryCohesion>>& libs, const TString& fileAliasPrefix)
-        : ParentModules(parentModules)
+        : Translators(translators)
+        , ParentModules(parentModules)
         , LibsContext(nextUniqueId)
         , KnownPackages(knownPackages)
         , Libs(libs)
@@ -67,6 +79,15 @@ class TModuleResolver : public IModuleResolver {
     {
     }
 
+    //FIXME remove
+    TModuleResolver(const TModulesTable* parentModules,
+        ui64 nextUniqueId, const THashMap<TString, TString>& clusterMapping,
+        const THashSet<TString>& sqlFlags, bool optimizeLibraries, const TSet<TString>& knownPackages, const THashMap<TString,
+        THashMap<int, TLibraryCohesion>>& libs, const TString& fileAliasPrefix)
+        : TModuleResolver(NSQLTranslation::MakeAllTranslators(), parentModules, nextUniqueId,
+            clusterMapping, sqlFlags, optimizeLibraries, knownPackages, libs, fileAliasPrefix)
+    {}
+
     static TString NormalizeModuleName(const TString& path);
 
     void AttachUserData(TUserDataStorage::TPtr userData) {
@@ -111,6 +132,7 @@ class TModuleResolver : public IModuleResolver {
     TString SubstParameters(const TString& str);
 
 private:
+    const NSQLTranslation::TTranslators Translators;
     THolder<TExprContext> OwnedCtx;
     const TModulesTable* ParentModules = nullptr;
     TUserDataStorage::TPtr UserData;

diff --git a/yql/essentials/public/fastcheck/fastcheck.cpp b/yql/essentials/public/fastcheck/fastcheck.cpp
@@ -6,11 +6,19 @@
 #include <yql/essentials/core/yql_type_annotation.h>
 #include <yql/essentials/core/yql_user_data_storage.h>
 #include <yql/essentials/sql/sql.h>
+#include <yql/essentials/sql/v1/sql.h>
+#include <yql/essentials/parser/pg_wrapper/interface/parser.h>
 
 namespace NYql {
 namespace NFastCheck {
 
 bool CheckProgram(const TString& program, const TOptions& options, TIssues& errors) {
+    NSQLTranslation::TTranslators translators(
+        nullptr,
+        NSQLTranslationV1::MakeTranslator(),
+        NSQLTranslationPG::MakeTranslator()
+    );
+
     TAstParseResult astRes;
     if (options.IsSql) {
         NSQLTranslation::TTranslationSettings settings;
@@ -22,7 +30,7 @@ bool CheckProgram(const TString& program, const TOptions& options, TIssues& erro
             settings.Mode = NSQLTranslation::ESqlMode::LIBRARY;
         }
 
-        astRes = SqlToYql(program, settings);
+        astRes = SqlToYql(translators, program, settings);
     } else {
         astRes = ParseAst(program);
     }
@@ -46,7 +54,7 @@ bool CheckProgram(const TString& program, const TOptions& options, TIssues& erro
             settings.File = x.first;
             settings.Mode = NSQLTranslation::ESqlMode::LIBRARY;
 
-            astRes = SqlToYql(x.second, settings);
+            astRes = SqlToYql(translators, x.second, settings);
             if (!astRes.IsOk()) {
                 errors = std::move(astRes.Issues);
                 return false;

diff --git a/yql/essentials/public/fastcheck/ya.make b/yql/essentials/public/fastcheck/ya.make
@@ -23,6 +23,7 @@ PEERDIR(
     yql/essentials/sql/v1/format
     yql/essentials/sql/settings
     yql/essentials/parser/pg_wrapper/interface
+    yql/essentials/sql/v1
 )
 
 GENERATE_ENUM_SERIALIZATION(linter.h)