Merge branch 'master' of github.com:ClickHouse/ClickHouse into limit-tests-more

Author: alexey-milovidov

.github/workflows/master.yml

@@ -776,7 +776,7 @@ jobs:
           fi
 
   build_amd_compat:
-    runs-on: [self-hosted, builder-aarch64]
+    runs-on: [self-hosted, builder]
     needs: [config_workflow, dockers_build_amd_and_merge, build_amd_tidy, build_amd_debug, build_amd_release, build_amd_asan, build_amd_tsan, build_amd_msan, build_amd_ubsan, build_amd_binary, build_arm_release, build_arm_asan]
     if: ${{ !failure() && !cancelled() && !contains(fromJson(needs.config_workflow.outputs.data).cache_success_base64, 'QnVpbGQgKGFtZF9jb21wYXQp') }}
     name: "Build (amd_compat)"

.github/workflows/pull_request.yml

@@ -930,7 +930,7 @@ jobs:
           fi
 
   build_amd_compat:
-    runs-on: [self-hosted, builder-aarch64]
+    runs-on: [self-hosted, builder]
     needs: [config_workflow, dockers_build_amd_and_merge, build_amd_debug, build_amd_release, build_amd_asan, build_amd_tsan, build_amd_msan, build_amd_ubsan, build_amd_binary, build_arm_release, build_arm_asan]
     if: ${{ !failure() && !cancelled() && !contains(fromJson(needs.config_workflow.outputs.data).cache_success_base64, 'QnVpbGQgKGFtZF9jb21wYXQp') }}
     name: "Build (amd_compat)"

README.md

@@ -53,6 +53,7 @@ Upcoming meetups
 * [Tokyo Meetup with Confluent](https://www.meetup.com/clickhouse-tokyo-user-group/events/306832118/) - April 15, 2025
 * [Kuala Lumper Meetup with CNCF](https://www.meetup.com/clickhouse-malaysia-meetup-group/events/306697678/) - April 16, 2025
 * [Jakarta Meetup with AWS](https://www.meetup.com/clickhouse-indonesia-user-group/events/306973747/) - April 22, 2025
+* [Denver Meetup](https://www.meetup.com/clickhouse-denver-user-group/events/306934991/) - April 23, 2025
 * [London Meetup](https://www.meetup.com/clickhouse-london-user-group/events/306047172/) - May 14, 2025
 * [Istanbul Meetup](https://www.meetup.com/clickhouse-turkiye-meetup-group/events/306978337/) - May 15, 2025
 

base/poco/NetSSL_OpenSSL/src/Context.cpp

@@ -504,18 +504,18 @@ void Context::createSSLContext()
  * if TLS1.x was removed at OpenSSL library build time via Configure options.
  */
         case TLSV1_1_CLIENT_USE:
-            _pSSLContext = SSL_CTX_new(TLSv1_1_client_method());
+            _pSSLContext = SSL_CTX_new(TLS_client_method());
             break;
         case TLSV1_1_SERVER_USE:
-            _pSSLContext = SSL_CTX_new(TLSv1_1_server_method());
+            _pSSLContext = SSL_CTX_new(TLS_server_method());
             break;
 #endif
 #if defined(SSL_OP_NO_TLSv1_2) && !defined(OPENSSL_NO_TLS1)
         case TLSV1_2_CLIENT_USE:
-            _pSSLContext = SSL_CTX_new(TLSv1_2_client_method());
+            _pSSLContext = SSL_CTX_new(TLS_client_method());
             break;
         case TLSV1_2_SERVER_USE:
-            _pSSLContext = SSL_CTX_new(TLSv1_2_server_method());
+            _pSSLContext = SSL_CTX_new(TLS_server_method());
             break;
 #endif
 		default:

ci/defs/job_configs.py

@@ -273,7 +273,7 @@ class JobConfigs:
             RunnerLabels.BUILDER_ARM,  # BuildTypes.ARM_V80COMPAT,
             RunnerLabels.BUILDER_AMD,  # BuildTypes.AMD_FREEBSD,
             RunnerLabels.BUILDER_ARM,  # BuildTypes.PPC64LE,
-            RunnerLabels.BUILDER_ARM,  # BuildTypes.AMD_COMPAT,
+            RunnerLabels.BUILDER_AMD,  # BuildTypes.AMD_COMPAT,
             RunnerLabels.BUILDER_AMD,  # BuildTypes.AMD_MUSL,
             RunnerLabels.BUILDER_ARM,  # BuildTypes.RISCV64,
             RunnerLabels.BUILDER_AMD,  # BuildTypes.S390X,

ci/praktika/_environment.py

@@ -56,7 +56,7 @@ def from_env(cls) -> "_Environment":
         RUN_URL = f"https://github.com/{REPOSITORY}/actions/runs/{RUN_ID}"
         BASE_BRANCH = os.getenv("GITHUB_BASE_REF", "")
         USER_LOGIN = ""
-        FORK_NAME = ""
+        FORK_NAME = REPOSITORY
         PR_BODY = ""
         PR_TITLE = ""
         PR_LABELS = []

ci/praktika/cidb.py

@@ -75,7 +75,7 @@ def json_data_generator(cls, result: Result, result_name_for_cidb):
             base_ref=env.BASE_BRANCH,
             base_repo=env.REPOSITORY,
             head_ref=env.BRANCH,
-            head_repo=env.REPOSITORY,  # TODO: remove from table?
+            head_repo=env.FORK_NAME,
             task_url="",
             instance_type=",".join(
                 filter(None, [env.INSTANCE_TYPE, env.INSTANCE_LIFE_CYCLE])

ci/praktika/hook_html.py

@@ -142,9 +142,11 @@ def push_pending_ci_report(cls, _workflow):
                 # fetch running status with start_time for current job
                 result = Result.from_fs(job.name)
             else:
-                result = Result.generate_pending(job.name)
+                result = Result.create_new(job.name, Result.Status.PENDING)
             results.append(result)
-        summary_result = Result.generate_pending(_workflow.name, results=results)
+        summary_result = Result.create_new(
+            _workflow.name, Result.Status.RUNNING, results=results
+        )
         summary_result.start_time = Utils.timestamp()
         summary_result.links.append(env.CHANGE_URL)
         summary_result.links.append(env.RUN_URL)
@@ -200,18 +202,26 @@ def configure(cls, _workflow):
                         sha=cache_record.sha,
                         job_name=skipped_job,
                     )
-                    result = Result.generate_skipped(
-                        skipped_job, [report_link], "reused from cache"
+                    result = Result.create_new(
+                        skipped_job,
+                        Result.Status.SKIPPED,
+                        [report_link],
+                        "reused from cache",
                     )
                 else:
-                    result = Result.generate_skipped(
-                        skipped_job, info=filtered_job_and_reason[skipped_job]
+                    result = Result.create_new(
+                        skipped_job,
+                        Result.Status.SKIPPED,
+                        info=filtered_job_and_reason[skipped_job],
                     )
                 results.append(result)
             if results:
-                assert _ResultS3.update_workflow_results(
-                    _workflow.name, new_sub_results=results
-                )
+                assert (
+                    _ResultS3.update_workflow_results(
+                        _workflow.name, new_sub_results=results
+                    )
+                    is None
+                ), "Workflow status supposed to remain 'running'"
 
     @classmethod
     def pre_run(cls, _workflow, _job):

ci/praktika/parser.py

@@ -236,10 +236,6 @@ def parse(self):
                     assert (
                         False
                     ), f"Artifact [{artifact_name}] has unsupported type [{artifact.type}]"
-            if not artifact.required_by and artifact.type != Artifact.Type.PHONY:
-                print(
-                    f"WARNING: Artifact [{artifact_name}] provided by job [{artifact.provided_by}] in workflow [{self.workflow_name}] has no job that requires it"
-                )
             if artifact.type == Artifact.Type.GH:
                 self.workflow_yaml_config.job_to_config[
                     artifact.provided_by

ci/praktika/result.py

@@ -344,23 +344,10 @@ def add_ext_key_value(self, key, value):
         return self
 
     @classmethod
-    def generate_pending(cls, name, results=None):
+    def create_new(cls, name, status, links=None, info="", results=None):
         return Result(
             name=name,
-            status=Result.Status.PENDING,
-            start_time=None,
-            duration=None,
-            results=results or [],
-            files=[],
-            links=[],
-            info="",
-        )
-
-    @classmethod
-    def generate_skipped(cls, name, links=None, info="", results=None):
-        return Result(
-            name=name,
-            status=Result.Status.SKIPPED,
+            status=status,
             start_time=None,
             duration=None,
             results=results or [],
@@ -511,10 +498,10 @@ def to_stdout_formatted(self, indent="", res=""):
 
 class ResultInfo:
     SETUP_ENV_JOB_FAILED = (
-        "Failed to set up job env, it's praktika bug or misconfiguration"
+        "Failed to set up job env, it is praktika bug or misconfiguration"
     )
     PRE_JOB_FAILED = (
-        "Failed to do a job pre-run step, it's praktika bug or misconfiguration"
+        "Failed to do a job pre-run step, it is praktika bug or misconfiguration"
     )
     KILLED = "Job killed or terminated, no Result provided"
     NOT_FOUND_IMPOSSIBLE = (
@@ -556,20 +543,21 @@ def copy_result_from_s3_with_version(cls, local_path):
         env = _Environment.get()
         file_name = Path(local_path).name
         local_dir = Path(local_path).parent
-        file_name_pattern = f"{file_name}_*"
-        for file_path in local_dir.glob(file_name_pattern):
-            file_path.unlink()
-        s3_path = f"{Settings.HTML_S3_PATH}/{env.get_s3_prefix()}/"
-        S3.copy_file_from_s3_matching_pattern(
-            s3_path=s3_path, local_path=local_dir, include=file_name_pattern
+        s3_path = f"{Settings.HTML_S3_PATH}/{env.get_s3_prefix()}"
+        latest_result_file = Shell.get_output(
+            f"aws s3 ls {s3_path}/{file_name}_ | awk '{{print $4}}' | sort -r | head -n 1",
+            strict=True,
+            verbose=True,
+        )
+        version = int(latest_result_file.split("_")[-1])
+        S3.copy_file_from_s3(
+            s3_path=f"{s3_path}/{latest_result_file}", local_path=local_dir
+        )
+        Shell.check(
+            f"cp {local_dir}/{latest_result_file} {local_path}",
+            strict=True,
+            verbose=True,
         )
-        result_files = []
-        for file_path in local_dir.glob(file_name_pattern):
-            result_files.append(file_path)
-        assert result_files, "No result files found"
-        result_files.sort()
-        version = int(result_files[-1].name.split("_")[-1])
-        Shell.check(f"cp {result_files[-1]} {local_path}", strict=True, verbose=True)
         return version
 
     @classmethod
@@ -698,6 +686,7 @@ def update_workflow_results(
             # when multiple concurrent jobs attempt to update the workflow report
             time.sleep(random.uniform(0, 2))
 
+        print(f"Workflow status changed: [{prev_status}] -> [{new_status}]")
         if prev_status != new_status:
             return new_status
         else:

ci/praktika/runner.py

@@ -73,7 +73,7 @@ def generate_local_run_environment(workflow, job, pr=None, sha=None):
 
         workflow_config.dump()
 
-        Result.generate_pending(job.name).dump()
+        Result.create_from(name=job.name, status=Result.Status.PENDING).dump()
 
     def _setup_env(self, _workflow, job):
         # source env file to write data into fs (workflow config json, workflow status json)
@@ -323,7 +323,7 @@ def _post_run(
                 info=info,
             ).dump()
         elif prerun_exit_code != 0:
-            info = f"ERROR: {ResultInfo.PRE_JOB_FAILED}"
+            info = ResultInfo.PRE_JOB_FAILED
             print(info)
             # set Result with error and logs
             Result(

ci/praktika/s3.py

@@ -72,7 +72,8 @@ def add_uploaded(cls, file_path):
         file_zize = cls.get_size_bytes(file_path)
         usage.uploaded += file_zize
         if file_name in usage.uploaded_details:
-            print(f"WARNING: Duplicated upload for filename [{file_name}]")
+            if not file_name.startswith("result_"):
+                print(f"WARNING: Duplicated upload for filename [{file_name}]")
             usage.uploaded_details[file_name] += file_zize
         else:
             usage.uploaded_details[file_name] = file_zize
@@ -109,7 +110,7 @@ def clean_s3_directory(cls, s3_path, include=""):
         cmd = f"aws s3 rm s3://{s3_path} --recursive"
         if include:
             cmd += f' --exclude "*" --include "{include}"'
-        cls.run_command_with_retries(cmd, retries=1)
+        cls.run_command_with_retries(cmd, retries=1, with_stderr=True)
         return
 
     @classmethod
@@ -196,7 +197,11 @@ def put(
 
     @classmethod
     def run_command_with_retries(
-        cls, command, retries=Settings.MAX_RETRIES_S3, no_strict=False
+        cls,
+        command,
+        retries=Settings.MAX_RETRIES_S3,
+        no_strict=False,
+        with_stderr=False,
     ):
         i = 0
         res = False
@@ -224,6 +229,8 @@ def run_command_with_retries(
                 print(
                     f"ERROR: aws s3 cp failed, stdout/stderr err: [{stderr}], out [{stdout}]"
                 )
+            elif with_stderr and (stdout or stderr):
+                print(f"stdout: {stdout}\nstderr: {stderr}")
             res = ret_code == 0
         if not res and not no_strict:
             raise RuntimeError(f"s3 command failed: [{stderr}]")
@@ -275,7 +282,7 @@ def copy_file_from_s3_matching_pattern(
         ).is_dir(), f"Path [{local_path}] does not exist or not a directory"
         assert s3_path.endswith("/"), f"s3 path is invalid [{s3_path}]"
         cmd = f'aws s3 cp s3://{s3_path}  {local_path} --exclude "{exclude}" --include "{include}" --recursive'
-        res = cls.run_command_with_retries(cmd, no_strict=no_strict)
+        res = cls.run_command_with_retries(cmd, no_strict=no_strict, with_stderr=True)
         if res:
             print(
                 "TODO: support StorageUsage.add_downloaded with matching pattern download"

contrib/aws-s2n-tls

@@ -168,6 +168,7 @@ target_compile_definitions (_curl PRIVATE
     BUILDING_LIBCURL
     CURL_HIDDEN_SYMBOLS
     libcurl_EXPORTS
+    OPENSSL_NO_ENGINE
     CURL_OS="${CMAKE_SYSTEM_NAME}"
 )
 

contrib/curl-cmake/CMakeLists.txt

@@ -13,6 +13,8 @@ set(_gRPC_BINARY_DIR "${ClickHouse_BINARY_DIR}/contrib/grpc")
 
 if(TARGET OpenSSL::SSL)
   set(gRPC_USE_UNSECURE_LIBRARIES FALSE)
+
+  add_definitions(-DOPENSSL_NO_ENGINE)
 else()
   set(gRPC_USE_UNSECURE_LIBRARIES TRUE)
 endif()

contrib/grpc-cmake/CMakeLists.txt

@@ -117,6 +117,7 @@ if(WITH_SASL_CYRUS)
    set(WITH_SASL_SCRAM 1)
    set(WITH_SASL_OAUTHBEARER 1)
 endif()
+add_definitions(-DOPENSSL_NO_ENGINE)
 list(APPEND SRCS "${RDKAFKA_SOURCE_DIR}/rdkafka_ssl.c")
 
 if(WITH_SSL AND WITH_CURL)