Skip to content

Commit 3c4cf2b

Browse files
semaphore-agent-production[bot]semaphore-agent-production[bot]
committed
Merge branch 'master' of https://github.com/confluentinc/kafka into trunk
2 parents b11b2cd + e6cabac commit 3c4cf2b

File tree

89 files changed

+3049
-109
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

89 files changed

+3049
-109
lines changed

‎.github/CODEOWNERS‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
* @confluentinc/kafka-eng

‎.semaphore/semaphore.yml‎

Lines changed: 166 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,166 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one or more
2+
# contributor license agreements. See the NOTICE file distributed with
3+
# this work for additional information regarding copyright ownership.
4+
# The ASF licenses this file to You under the Apache License, Version 2.0
5+
# (the "License"); you may not use this file except in compliance with
6+
# the License. You may obtain a copy of the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
16+
version: v1.0
17+
name: build-test-release
18+
agent:
19+
machine:
20+
type: s1-prod-ubuntu24-04-arm64-3
21+
fail_fast:
22+
cancel:
23+
when: "true"
24+
execution_time_limit:
25+
hours: 4
26+
queue:
27+
- when: "branch != 'master' and branch !~ '[0-9]+\\.[0-9]+\\.x'"
28+
processing: parallel
29+
global_job_config:
30+
env_vars:
31+
- name: NANO_VERSION
32+
value: "true"
33+
- name: PUBLISH
34+
value: "true"
35+
- name: ENABLE_PUBLISH_ARTIFACTS
36+
value: "true"
37+
- name: ENABLE_DOWNSTREAM_TRIGGER
38+
value: "true"
39+
- name: DOWNSTREAM_PROJECTS
40+
value: "common"
41+
prologue:
42+
commands:
43+
- echo $SEMAPHORE_WORKFLOW_ID
44+
- echo $SEMAPHORE_GIT_REPO_SLUG
45+
- checkout
46+
- sem-version java 21
47+
- sem-version go 1.16.15
48+
- sem-version python 3.11
49+
- python --version
50+
- python -m venv junitparser
51+
- source junitparser/bin/activate
52+
- pip install junitparser
53+
- git config --global url."[email protected]:".insteadOf "https://github.com/"
54+
- export SEMAPHORE_CACHE_DIR=/home/semaphore
55+
- source scripts/set_env_vars.sh
56+
- source scripts/set_downstream_branch.sh
57+
blocks:
58+
- name: Gradle Build
59+
dependencies: []
60+
task:
61+
jobs:
62+
- name: Build, Compile, Validations, Publish
63+
commands:
64+
- |
65+
if [[ "${SEMAPHORE_ORGANIZATION_URL}" != *".semaphoreci.com" ]]; then
66+
. vault-setup
67+
fi
68+
- >-
69+
if [[ "${SEMAPHORE_ORGANIZATION_URL}" != *".semaphoreci.com" ]] && [ "$NANO_VERSION" = "true" ] && [ "$RELEASE_JOB" = "false" ] && [ "$ENABLE_PUBLISH_ARTIFACTS" = "true" ]; then
70+
. ci-tools ci-update-version
71+
fi
72+
- |
73+
if [[ "${SEMAPHORE_ORGANIZATION_URL}" != *".semaphoreci.com" ]]; then
74+
make compile-validate
75+
fi
76+
- |
77+
if [[ "${SEMAPHORE_ORGANIZATION_URL}" != *".semaphoreci.com" ]] && [ "$PUBLISH" = "true" ] && [ "$SEMAPHORE_GIT_REF_TYPE" != "pull-request" ] && [ "$ENABLE_PUBLISH_ARTIFACTS" = "true" ]; then \
78+
if [[ "$RELEASE_JOB" = "false" ]]; then \
79+
. ci-tools ci-push-tag; \
80+
mavenUrl=$(vault kv get v1/ci/kv/gradle/artifactory_snapshots_settings | grep mavenUrl | cut -d "," -f 2 | cut -d "'" -f 2); \
81+
elif [[ "$SEMAPHORE_GIT_BRANCH" == *-alpha* ]]; then \
82+
mavenUrl=$(vault kv get v1/ci/kv/gradle/artifactory_preview_release_settings | grep mavenUrl | cut -d "," -f 2 | cut -d "'" -f 2); \
83+
fi; \
84+
./gradlewAll -PmavenUrl=$mavenUrl -PkeepAliveMode=session uploadArchives; \
85+
fi
86+
- |
87+
echo "PUBLISH: $PUBLISH, RELEASE_JOB: $RELEASE_JOB, SEMAPHORE_GIT_REF_TYPE: $SEMAPHORE_GIT_REF_TYPE, ENABLE_DOWNSTREAM_TRIGGER: $ENABLE_DOWNSTREAM_TRIGGER"
88+
if [[ "${SEMAPHORE_ORGANIZATION_URL}" != *".semaphoreci.com" ]] && [ "$PUBLISH" = "true" ] && [ "$RELEASE_JOB" = "false" ] && [ "$SEMAPHORE_GIT_REF_TYPE" != "pull-request" ] && [ "$ENABLE_DOWNSTREAM_TRIGGER" = "true" ]; then
89+
for project in $DOWNSTREAM_PROJECTS; do
90+
sem-trigger -p $project -b $DOWNSTREAM_BRANCH_NAME -f .semaphore/semaphore.yml
91+
done
92+
fi
93+
- name: Tests
94+
dependencies: []
95+
task:
96+
jobs:
97+
- name: Unit tests and Integration tests
98+
commands:
99+
- |
100+
./gradlew \
101+
unitTest integrationTest --no-daemon --stacktrace --continue \
102+
-Dorg.gradle.jvmargs="-Xmx6g -Xss4m -XX:+UseParallelGC" \
103+
-PtestLoggingEvents=started,passed,skipped,failed -PmaxParallelForks=2 \
104+
-PignoreFailures=true -PmaxTestRetries=1 -PmaxTestRetryFailures=5
105+
gradle_exit=$?
106+
107+
# Check Gradle exit code and fail the job if non-zero
108+
if [ "$gradle_exit" -ne 0 ]; then
109+
echo "❌ Gradle exited with code $gradle_exit — failing the job."
110+
exit "$gradle_exit"
111+
else
112+
echo "✅ Gradle completed successfully (exit code 0)."
113+
fi
114+
115+
shopt -s globstar
116+
python scripts/check_test_failures.py **/build/test-results/**/TEST-*.xml
117+
check_exit=$?
118+
119+
test-results publish --name Test-Suite --trim-output-to 1024 --omit-output-for-passed **/build/test-results/**/TEST-*.xml || true
120+
121+
# Check if check_test_failures.py returned non-zero exit code
122+
if [ "$check_exit" -ne 0 ]; then
123+
echo "❌ check_test_failures.py returned $check_exit"
124+
exit "$check_exit"
125+
fi
126+
execution_time_limit:
127+
minutes: 120
128+
epilogue:
129+
always:
130+
commands:
131+
- |
132+
echo "Job creation time: $((SEMAPHORE_JOB_CREATION_TIME * 1000))"
133+
echo "Current time: $(date +%s%3N)"
134+
echo $(( ($(date +%s%3N) - $SEMAPHORE_JOB_CREATION_TIME * 1000) )) > duration.txt
135+
artifact push workflow duration.txt --destination $SEMAPHORE_JOB_INDEX-test-duration-$SEMAPHORE_WORKFLOW_ID || true
136+
after_pipeline:
137+
task:
138+
agent:
139+
machine:
140+
type: s1-prod-ubuntu24-04-arm64-0
141+
jobs:
142+
- name: Metrics
143+
commands:
144+
- emit-ci-metrics -p -a test-results
145+
- name: Publish Test Results
146+
commands:
147+
- test-results gen-pipeline-report || true
148+
- name: SonarQube
149+
commands:
150+
- checkout
151+
- sem-version java 11
152+
- emit-sonarqube-data -a test-results
153+
- name: Trigger cp-jar-build to verify CP packaging
154+
commands:
155+
- |
156+
if [[ -z "$SEMAPHORE_GIT_PR_BRANCH" ]] && [[ "$SEMAPHORE_GIT_BRANCH" == "master" ]]; then \
157+
echo "Commit to master (not a PR), triggering cp-jar-build task to verify CP packaging"; \
158+
sem-trigger -p packaging \
159+
-t cp-jar-build-on-commit \
160+
-d "|" -i "UPSTREAM_COMPONENT|$SEMAPHORE_PROJECT_NAME" \
161+
-i "UPSTREAM_GIT_SHA|$SEMAPHORE_GIT_SHA" \
162+
-i "UPSTREAM_WORKFLOW_LINK|https://semaphore.ci.confluent.io/workflows/${SEMAPHORE_WORKFLOW_ID}" \
163+
-b $SEMAPHORE_GIT_BRANCH; \
164+
else \
165+
echo "Skipping: either it's a PR or not a commit to master branch"; \
166+
fi;

‎Confluent-README.md‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
Confluent Open Source
2+
==========================
3+
This is a [Confluent](https://www.confluent.io/) Open Source fork of Apache Kafka.
4+
5+
This version includes several modifications to enhance maintainability and ease-of-use.
6+
Just like Apache Kafka, COS is distributed under the Apache 2.0 license.

‎Makefile‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one or more
2+
# contributor license agreements. See the NOTICE file distributed with
3+
# this work for additional information regarding copyright ownership.
4+
# The ASF licenses this file to You under the Apache License, Version 2.0
5+
# (the "License"); you may not use this file except in compliance with
6+
# the License. You may obtain a copy of the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
16+
#Runs the compile and checkstyle error check
17+
.PHONY: compile-validate
18+
compile-validate:
19+
./gradlew clean -PskipSigning=true publishToMavenLocal build -x test --no-daemon --stacktrace -PxmlSpotBugsReport=true 2>&1 | tee build.log
20+
@error_count=$$(grep -c -E "(ERROR|error:|\[Error\]|FAILED)" build.log); \
21+
if [ $$error_count -ne 0 ]; then \
22+
echo "Compile, checkstyle or spotbugs error found"; \
23+
grep -E "(ERROR|error:|\[Error\]|FAILED)" build.log | while read -r line; do \
24+
echo "$$line"; \
25+
done; \
26+
echo "Number of compile, checkstyle and spotbug errors: $$error_count"; \
27+
exit $$error_count; \
28+
else \
29+
echo "No errors found"; \
30+
fi
31+
32+
# Below targets are used during kafka packaging for debian.
33+
34+
.PHONY: clean
35+
clean:
36+
37+
.PHONY: distclean
38+
distclean:
39+
40+
%:
41+
$(MAKE) -f debian/Makefile $@

‎Vagrantfile‎

Lines changed: 16 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,8 @@ num_workers = 0 # Generic workers that get the code, but don't start any service
3131
ram_megabytes = 1280
3232
base_box = "ubuntu/trusty64"
3333

34+
semaphore_job_id = ENV['SEMAPHORE_JOB_ID']
35+
3436
# EC2
3537
ec2_access_key = ENV['AWS_ACCESS_KEY']
3638
ec2_secret_key = ENV['AWS_SECRET_KEY']
@@ -41,7 +43,6 @@ ec2_keypair_file = nil
4143
ec2_region = "us-east-1"
4244
ec2_az = nil # Uses set by AWS
4345
ec2_ami = "ami-29ebb519"
44-
ec2_instance_type = "m3.medium"
4546
ec2_spot_instance = ENV['SPOT_INSTANCE'] ? ENV['SPOT_INSTANCE'] == 'true' : true
4647
ec2_spot_max_price = "0.113" # On-demand price for instance type
4748
ec2_user = "ubuntu"
@@ -51,7 +52,7 @@ ec2_subnet_id = nil
5152
# Only override this by setting it to false if you're running in a VPC and you
5253
# are running Vagrant from within that VPC as well.
5354
ec2_associate_public_ip = nil
54-
ec2_iam_instance_profile_name = nil
55+
ec2_iam_instance_profile_name = "semaphore-access"
5556

5657
ebs_volume_type = 'gp3'
5758

@@ -63,6 +64,7 @@ if File.exist?(local_config_file) then
6364
eval(File.read(local_config_file), binding, "Vagrantfile.local")
6465
end
6566

67+
ec2_instance_type = "c4.xlarge"
6668
# override any instance type set by Vagrantfile.local or above via an environment variable
6769
if ENV['INSTANCE_TYPE'] then
6870
ec2_instance_type = ENV['INSTANCE_TYPE']
@@ -161,7 +163,16 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
161163
node.vm.provider :aws do |aws|
162164
aws.tags = {
163165
'Name' => ec2_instance_name_prefix + "-" + Socket.gethostname + "-" + name,
164-
'JenkinsBuildUrl' => ENV['BUILD_URL']
166+
'role' => 'ce-kafka',
167+
'Owner' => 'ce-kafka',
168+
'JenkinsBuildUrl' => ENV['BUILD_URL'],
169+
'SemaphoreWorkflowUrl' => ENV['SEMAPHORE_WORKFLOW_URL'],
170+
'SemaphoreJobId' => ENV['SEMAPHORE_JOB_ID'],
171+
'cflt_environment' => 'devel',
172+
'cflt_partition' => 'onprem',
173+
'cflt_managed_by' => 'iac',
174+
'cflt_managed_id' => 'kafka',
175+
'cflt_service' => 'kafka'
165176
}
166177
end
167178
end
@@ -210,6 +221,8 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
210221
name_node(worker, name, ec2_instance_name_prefix)
211222
ip_address = "192.168.50." + (100 + i).to_s
212223
assign_local_ip(worker, ip_address)
224+
worker.vm.provision "file", source: "vagrant/cloudwatch-agent-configuration.json", destination: "/tmp/cloudwatch-agent-configuration.json"
225+
worker.vm.provision "shell", path: "vagrant/cloudwatch-agent-setup.sh", env: {"SEMAPHORE_JOB_ID" => semaphore_job_id}
213226
worker.vm.provision "shell", path: "vagrant/base.sh", env: {"JDK_MAJOR" => jdk_major, "JDK_FULL" => jdk_full}
214227
end
215228
}

‎bin/connect-distributed‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
#!/usr/bin/env bash
2+
# Licensed to the Apache Software Foundation (ASF) under one or more
3+
# contributor license agreements. See the NOTICE file distributed with
4+
# this work for additional information regarding copyright ownership.
5+
# The ASF licenses this file to You under the Apache License, Version 2.0
6+
# (the "License"); you may not use this file except in compliance with
7+
# the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
17+
exec "$0.sh" "$@"

‎bin/connect-distributed.sh‎

Lines changed: 30 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,9 +22,37 @@ fi
2222

2323
base_dir=$(dirname $0)
2424

25+
###
26+
### Classpath additions for Confluent Platform releases (LSB-style layout)
27+
###
28+
#cd -P deals with symlink from /bin to /usr/bin
29+
java_base_dir=$( cd -P "$base_dir/../share/java" && pwd )
30+
31+
# confluent-common: required by kafka-serde-tools
32+
# kafka-serde-tools (e.g. Avro serializer): bundled with confluent-schema-registry package
33+
for library in "kafka" "confluent-common" "kafka-serde-tools" "monitoring-interceptors"; do
34+
dir="$java_base_dir/$library"
35+
if [ -d "$dir" ]; then
36+
classpath_prefix="$CLASSPATH:"
37+
if [ "x$CLASSPATH" = "x" ]; then
38+
classpath_prefix=""
39+
fi
40+
CLASSPATH="$classpath_prefix$dir/*"
41+
fi
42+
done
43+
2544
if [ -z "$KAFKA_LOG4J_OPTS" ]; then
26-
export KAFKA_LOG4J_OPTS="-Dlog4j2.configurationFile=$base_dir/../config/connect-log4j2.yaml"
45+
LOG4J_CONFIG_NORMAL_INSTALL="/etc/kafka/connect-log4j2.yaml"
46+
LOG4J_CONFIG_ZIP_INSTALL="$base_dir/../etc/kafka/connect-log4j2.yaml"
47+
if [ -e "$LOG4J_CONFIG_NORMAL_INSTALL" ]; then # Normal install layout
48+
KAFKA_LOG4J_OPTS="-Dlog4j2.configurationFile=${LOG4J_CONFIG_NORMAL_INSTALL}"
49+
elif [ -e "${LOG4J_CONFIG_ZIP_INSTALL}" ]; then # Simple zip file layout
50+
KAFKA_LOG4J_OPTS="-Dlog4j2.configurationFile=${LOG4J_CONFIG_ZIP_INSTALL}"
51+
else # Fallback to normal default
52+
KAFKA_LOG4J_OPTS="-Dlog4j2.configurationFile=$base_dir/../config/connect-log4j2.yaml"
53+
fi
2754
fi
55+
export KAFKA_LOG4J_OPTS
2856

2957
if [ "x$KAFKA_HEAP_OPTS" = "x" ]; then
3058
export KAFKA_HEAP_OPTS="-Xms256M -Xmx2G"
@@ -42,4 +70,5 @@ case $COMMAND in
4270
;;
4371
esac
4472

73+
export CLASSPATH
4574
exec $(dirname $0)/kafka-run-class.sh $EXTRA_ARGS org.apache.kafka.connect.cli.ConnectDistributed "$@"

‎bin/connect-standalone‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
#!/usr/bin/env bash
2+
# Licensed to the Apache Software Foundation (ASF) under one or more
3+
# contributor license agreements. See the NOTICE file distributed with
4+
# this work for additional information regarding copyright ownership.
5+
# The ASF licenses this file to You under the Apache License, Version 2.0
6+
# (the "License"); you may not use this file except in compliance with
7+
# the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
17+
exec "$0.sh" "$@"

0 commit comments

Comments
 (0)