Backport: Enable parallel download artifacts (#768) (#792)

In the previous versions, it download artifacts from GitHub in serial not parallel, so it takes huge amount of times. In this version, it tries to download them in parallel, so it fixes above issue. Signed-off-by: Kentaro Hayashi <[email protected]>
fluent · Feb 10, 2025 · b9ad9b5 · b9ad9b5
1 parent 6595142
commit b9ad9b5
Showing 1 changed file with 39 additions and 8 deletions.
diff --git a/fluent-package/manage-fluent-repositories.sh b/fluent-package/manage-fluent-repositories.sh
@@ -195,39 +195,70 @@ EOF
 	     https://api.github.com/repos/fluent/fluent-package-builder/pulls/$PULL_NUMBER | jq --raw-output '.head | .ref + " " + .sha')
 	head_branch=$(echo $response | cut -d' ' -f1)
 	head_sha=$(echo $response | cut -d' ' -f2)
+	rm -f dl.list
 	curl --silent --location \
 	     -H "Accept: application/vnd.github+json" \
 	     -H "Authorization: Bearer $GITHUB_ACCESS_TOKEN" \
 	     -H "X-GitHub-Api-Version: 2022-11-28" \
 	     "https://api.github.com/repos/fluent/fluent-package-builder/actions/artifacts?per_page=100&page=$d" | \
-	    jq --raw-output '.artifacts[] | select(.workflow_run.head_branch == "'$head_branch'" and .workflow_run.head_sha == "'$head_sha'") | .name + " " + .archive_download_url' | while read line
+	    jq --raw-output '.artifacts[] | select(.workflow_run.head_branch == "'$head_branch'" and .workflow_run.head_sha == "'$head_sha'") | .name + " " + (.size_in_bytes|tostring) + " " + .archive_download_url' > dl.list
+	while read line
 	do
 	    package=$(echo $line | cut -d' ' -f1)
-	    download_url=$(echo $line | cut -d' ' -f2)
+	    download_url=$(echo $line | cut -d' ' -f3)
 	    echo "Downloading $package.zip from $download_url"
 	    case $package in
 		*debian*|*ubuntu*)
 		    mkdir -p apt/repositories
 		    (cd apt/repositories &&
 			 rm -f $package.zip &&
 			 curl --silent --location --output $package.zip \
-			      -H "Authorization: Bearer $GITHUB_ACCESS_TOKEN" $download_url &&
-			 unzip -u -o $package.zip)
+			      -H "Authorization: Bearer $GITHUB_ACCESS_TOKEN" $download_url ) &
 		    ;;
 		*centos*|*rockylinux*|*almalinux*|*amazonlinux*)
 		    mkdir -p yum/repositories
 		    (cd yum/repositories &&
 			 rm -f $package.zip &&
 			 curl --silent --location --output $package.zip \
-			      -H "Authorization: Bearer $GITHUB_ACCESS_TOKEN" $download_url &&
-			 unzip -u -o $package.zip)
+			      -H "Authorization: Bearer $GITHUB_ACCESS_TOKEN" $download_url ) &
 		    ;;
 		*)
 		    curl --silent --location --output $package.zip \
-			 -H "Authorization: Bearer $GITHUB_ACCESS_TOKEN" $download_url
+			 -H "Authorization: Bearer $GITHUB_ACCESS_TOKEN" $download_url &
 		    ;;
 	    esac
-	done
+	done < dl.list
+	wait
+
+	verified=1
+	while read line
+	do
+	    package=$(echo $line | cut -d' ' -f1)
+	    download_size=$(echo $line | cut -d' ' -f2)
+	    case $package in
+		*debian*|*ubuntu*)
+		    actual_size=$(stat --format="%s" apt/repositories/$package.zip)
+		    ;;
+		*rockylinux*|*almalinux*|*amazonlinux*)
+		    actual_size=$(stat --format="%s" yum/repositories/$package.zip)
+		    ;;
+		*)
+		    actual_size=$(stat --format="%s" $package.zip)
+		    ;;
+	    esac
+	    if [ $download_size = "$actual_size" ]; then
+		echo -e "[\e[32m\e[40mOK\e[0m] Verify apt/repositories/$package.zip"
+	    else
+		echo -e "[\e[31m\e[40mNG\e[0m] Verify apt/repositories/$package.zip (expected: $download_size actual: $actual_size)"
+		verified=0
+	    fi
+	done < dl.list
+	if [ $verified -eq 0 ]; then
+	    echo "Downloaded artifacts were corrupted! Check the downloaded artifacts."
+	    exit 1
+	fi
+	(cd apt/repositories && find . -name '*.zip' -exec unzip -u -o {} \;)
+	(cd yum/repositories && find . -name '*.zip' -exec unzip -u -o {} \;)
 	;;
     *)
 	usage