Skip to content

Find V19+ basenames automatically. #471

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 14 commits into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions .github/workflows/tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,15 @@ jobs:
USE_BAZEL_VERSION: ${{ matrix.bazel_version }}
USE_BZLMOD: ${{ matrix.bzlmod }}
run: tests/scripts/run_tests.sh
toolchain_test:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: Test
env:
USE_BAZEL_VERSION: latest
USE_BZLMOD: true
run: tests/scripts/run_toolchain_tests.sh
external_test:
strategy:
fail-fast: false
Expand Down
45 changes: 45 additions & 0 deletions tests/scripts/run_toolchain_tests.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
#!/bin/bash
# Copyright 2018 The Bazel Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

set -euo pipefail

while getopts "h" opt; do
case "${opt}" in
"h")
echo "Usage: No options"
exit 2
;;
*)
echo "invalid option: -${OPTARG}"
exit 1
;;
esac
done

scripts_dir="$(dirname "${BASH_SOURCE[0]}")"
source "${scripts_dir}/bazel.sh"
"${bazel}" version

set -x
test_args=(
"--check_direct_dependencies=off"
)

targets=(
"//toolchain/..."
)

"${bazel}" ${TEST_MIGRATION:+"--strict"} --bazelrc=/dev/null test \
"${common_test_args[@]}" "${test_args[@]}" "${targets[@]}"
15 changes: 15 additions & 0 deletions toolchain/internal/BUILD.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -12,4 +12,19 @@
# See the License for the specific language governing permissions and
# limitations under the License.

load("@bazel_skylib//rules:diff_test.bzl", "diff_test")
load("llvm_distributions.bzl", "write_distributions")

exports_files(["template.modulemap"])

write_distributions(
name = "llvm_distributions",
testonly = True,
visibility = ["//visibility:private"],
)

diff_test(
name = "llvm_distributions_test",
file1 = "llvm_distributions.golden.txt",
file2 = "llvm_distributions",
)
272 changes: 234 additions & 38 deletions toolchain/internal/llvm_distributions.bzl

Large diffs are not rendered by default.

164 changes: 164 additions & 0 deletions toolchain/internal/llvm_distributions.golden.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,164 @@
del: clang+llvm-6.0.0-amd64-unknown-freebsd-10.tar.xz
del: clang+llvm-6.0.0-i386-unknown-freebsd-10.tar.xz
del: clang+llvm-6.0.0-i686-linux-gnu-Fedora27.tar.xz
del: clang+llvm-6.0.0-mips-linux-gnu.tar.xz
del: clang+llvm-6.0.0-mipsel-linux-gnu.tar.xz
del: clang+llvm-6.0.0-x86_64-linux-gnu-Fedora27.tar.xz
del: clang+llvm-6.0.0-x86_64-linux-gnu-debian8.tar.xz
del: clang+llvm-6.0.0-x86_64-linux-gnu-ubuntu-14.04.tar.xz
del: clang+llvm-6.0.0-x86_64-linux-gnu-ubuntu-16.04.tar.xz
del: clang+llvm-6.0.0-x86_64-linux-sles11.3.tar.xz
del: clang+llvm-6.0.0-x86_64-linux-sles12.2.tar.xz
del: clang+llvm-6.0.1-amd64-unknown-freebsd10.tar.xz
del: clang+llvm-6.0.1-i386-unknown-freebsd10.tar.xz
del: clang+llvm-6.0.1-x86_64-linux-gnu-ubuntu-14.04.tar.xz
del: clang+llvm-6.0.1-x86_64-linux-gnu-ubuntu-16.04.tar.xz
del: clang+llvm-6.0.1-x86_64-linux-sles11.3.tar.xz
del: clang+llvm-6.0.1-x86_64-linux-sles12.3.tar.xz
del: clang+llvm-7.0.0-amd64-unknown-freebsd-10.tar.xz
del: clang+llvm-7.0.0-i386-unknown-freebsd-10.tar.xz
del: clang+llvm-7.0.0-x86_64-linux-gnu-ubuntu-14.04.tar.xz
del: clang+llvm-7.0.0-x86_64-linux-gnu-ubuntu-16.04.tar.xz
del: clang+llvm-7.0.0-x86_64-linux-sles11.3.tar.xz
del: clang+llvm-7.0.0-x86_64-linux-sles12.3.tar.xz
del: clang+llvm-8.0.0-amd64-unknown-freebsd11.tar.xz
del: clang+llvm-8.0.0-i386-unknown-freebsd11.tar.xz
del: clang+llvm-8.0.0-x86_64-linux-gnu-ubuntu-14.04.tar.xz
del: clang+llvm-8.0.0-x86_64-linux-gnu-ubuntu-16.04.tar.xz
del: clang+llvm-8.0.0-x86_64-linux-gnu-ubuntu-18.04.tar.xz
del: clang+llvm-8.0.0-x86_64-linux-sles11.3.tar.xz
del: clang+llvm-8.0.1-amd64-unknown-freebsd11.tar.xz
del: clang+llvm-8.0.1-i386-unknown-freebsd11.tar.xz
del: clang+llvm-8.0.1-powerpc64le-linux-rhel-7.4.tar.xz
del: clang+llvm-8.0.1-powerpc64le-linux-ubuntu-16.04.tar.xz
del: clang+llvm-8.0.1-x86_64-linux-sles11.3.tar.xz
del: clang+llvm-9.0.0-amd64-unknown-freebsd11.tar.xz
del: clang+llvm-9.0.0-i386-unknown-freebsd11.tar.xz
del: clang+llvm-9.0.0-powerpc64le-linux-rhel-7.4.tar.xz
del: clang+llvm-9.0.0-powerpc64le-linux-ubuntu-16.04.tar.xz
del: clang+llvm-9.0.0-x86_64-darwin-apple.tar.xz
del: clang+llvm-9.0.0-x86_64-linux-gnu-ubuntu-14.04.tar.xz
del: clang+llvm-9.0.0-x86_64-linux-gnu-ubuntu-16.04.tar.xz
del: clang+llvm-9.0.0-x86_64-linux-gnu-ubuntu-18.04.tar.xz
del: clang+llvm-9.0.0-x86_64-linux-sles11.3.tar.xz
del: clang+llvm-9.0.0-x86_64-pc-linux-gnu.tar.xz
del: clang+llvm-10.0.0-amd64-unknown-freebsd11.tar.xz
del: clang+llvm-10.0.0-i386-unknown-freebsd11.tar.xz
del: clang+llvm-10.0.0-powerpc64le-linux-rhel-7.4.tar.xz
del: clang+llvm-10.0.0-powerpc64le-linux-ubuntu-16.04.tar.xz
del: clang+llvm-10.0.0-x86_64-linux-sles11.3.tar.xz
del: clang+llvm-10.0.1-amd64-unknown-freebsd11.tar.xz
del: clang+llvm-10.0.1-i386-unknown-freebsd11.tar.xz
del: clang+llvm-10.0.1-powerpc64le-linux-rhel-7.4.tar.xz
del: clang+llvm-10.0.1-powerpc64le-linux-ubuntu-16.04.tar.xz
del: clang+llvm-10.0.1-x86_64-linux-sles12.4.tar.xz
del: clang+llvm-11.0.0-x86_64-linux-gnu-ubuntu-16.04.tar.xz
del: clang+llvm-11.0.0-x86_64-linux-gnu-ubuntu-20.04.tar.xz
del: clang+llvm-11.0.0-x86_64-linux-sles12.4.tar.xz
del: clang+llvm-11.0.1-amd64-unknown-freebsd11.tar.xz
del: clang+llvm-11.0.1-amd64-unknown-freebsd12.tar.xz
del: clang+llvm-11.0.1-i386-unknown-freebsd11.tar.xz
del: clang+llvm-11.0.1-i386-unknown-freebsd12.tar.xz
del: clang+llvm-11.0.1-powerpc64le-linux-rhel-7.4.tar.xz
del: clang+llvm-11.0.1-powerpc64le-linux-ubuntu-18.04.tar.xz
del: clang+llvm-11.0.1-x86_64-linux-gnu-ubuntu-16.04.tar.xz
del: clang+llvm-11.0.1-x86_64-linux-gnu-ubuntu-20.10.tar.xz
del: clang+llvm-11.0.1-x86_64-linux-sles12.4.tar.xz
del: clang+llvm-11.1.0-amd64-unknown-freebsd11.tar.xz
del: clang+llvm-11.1.0-amd64-unknown-freebsd12.tar.xz
del: clang+llvm-11.1.0-i386-unknown-freebsd11.tar.xz
del: clang+llvm-11.1.0-i386-unknown-freebsd12.tar.xz
del: clang+llvm-11.1.0-powerpc64le-linux-rhel-7.4.tar.xz
del: clang+llvm-11.1.0-powerpc64le-linux-ubuntu-18.04.tar.xz
del: clang+llvm-11.1.0-x86_64-linux-gnu-ubuntu-16.04.tar.xz
del: clang+llvm-11.1.0-x86_64-linux-gnu-ubuntu-20.10.tar.xz
del: clang+llvm-12.0.0-amd64-unknown-freebsd11.tar.xz
del: clang+llvm-12.0.0-amd64-unknown-freebsd12.tar.xz
del: clang+llvm-12.0.0-i386-unknown-freebsd11.tar.xz
del: clang+llvm-12.0.0-i386-unknown-freebsd12.tar.xz
del: clang+llvm-12.0.0-x86_64-linux-gnu-ubuntu-16.04.tar.xz
del: clang+llvm-12.0.0-x86_64-linux-gnu-ubuntu-20.04.tar.xz
del: clang+llvm-12.0.0-x86_64-linux-sles12.4.tar.xz
del: clang+llvm-12.0.1-amd64-unknown-freebsd11.tar.xz
del: clang+llvm-12.0.1-amd64-unknown-freebsd12.tar.xz
del: clang+llvm-12.0.1-i386-unknown-freebsd11.tar.xz
del: clang+llvm-12.0.1-i386-unknown-freebsd12.tar.xz
del: clang+llvm-12.0.1-powerpc64le-linux-rhel-7.9.tar.xz
del: clang+llvm-12.0.1-powerpc64le-linux-ubuntu-18.04.tar.xz
del: clang+llvm-13.0.0-amd64-unknown-freebsd12.tar.xz
del: clang+llvm-13.0.0-amd64-unknown-freebsd13.tar.xz
del: clang+llvm-13.0.0-i386-unknown-freebsd12.tar.xz
del: clang+llvm-13.0.0-i386-unknown-freebsd13.tar.xz
del: clang+llvm-13.0.0-powerpc64le-linux-rhel-7.9.tar.xz
del: clang+llvm-13.0.0-powerpc64le-linux-ubuntu-18.04.tar.xz
del: clang+llvm-13.0.0-x86_64-linux-gnu-ubuntu-16.04.tar.xz
del: clang+llvm-13.0.0-x86_64-linux-gnu-ubuntu-20.04.tar.xz
del: clang+llvm-13.0.1-amd64-unknown-freebsd12.tar.xz
del: clang+llvm-13.0.1-amd64-unknown-freebsd13.tar.xz
del: clang+llvm-13.0.1-i386-unknown-freebsd12.tar.xz
del: clang+llvm-13.0.1-i386-unknown-freebsd13.tar.xz
del: clang+llvm-13.0.1-powerpc64le-linux-rhel-7.9.tar.xz
del: clang+llvm-13.0.1-powerpc64le-linux-ubuntu-18.04.5.tar.xz
del: clang+llvm-14.0.0-amd64-unknown-freebsd12.tar.xz
del: clang+llvm-14.0.0-amd64-unknown-freebsd13.tar.xz
del: clang+llvm-14.0.0-i386-unknown-freebsd12.tar.xz
del: clang+llvm-14.0.0-i386-unknown-freebsd13.tar.xz
del: clang+llvm-14.0.0-powerpc64le-linux-rhel-7.9.tar.xz
del: clang+llvm-14.0.0-powerpc64le-linux-ubuntu-18.04.tar.xz
del: clang+llvm-14.0.0-x86_64-linux-sles12.4.tar.xz
del: clang+llvm-14.0.1-amd64-unknown-freebsd12.tar.xz
del: clang+llvm-14.0.1-amd64-unknown-freebsd13.tar.xz
del: clang+llvm-14.0.1-powerpc64le-linux-rhel-8.4.tar.xz
del: clang+llvm-14.0.1-powerpc64le-linux-ubuntu-18.04.tar.xz
del: clang+llvm-14.0.2-amd64-unknown-freebsd12.tar.xz
del: clang+llvm-14.0.2-amd64-unknown-freebsd13.tar.xz
del: clang+llvm-14.0.3-amd64-unknown-freebsd12.tar.xz
del: clang+llvm-14.0.3-amd64-unknown-freebsd13.tar.xz
del: clang+llvm-14.0.3-powerpc64le-linux-rhel-8.4.tar.xz
del: clang+llvm-14.0.3-powerpc64le-linux-ubuntu-18.04.tar.xz
del: clang+llvm-14.0.4-amd64-unknown-freebsd12.tar.xz
del: clang+llvm-14.0.4-amd64-unknown-freebsd13.tar.xz
del: clang+llvm-14.0.4-powerpc64le-linux-rhel-8.4.tar.xz
del: clang+llvm-14.0.4-powerpc64le-linux-ubuntu-18.04.5.tar.xz
del: clang+llvm-14.0.5-amd64-unknown-freebsd12.tar.xz
del: clang+llvm-14.0.5-amd64-unknown-freebsd13.tar.xz
del: clang+llvm-14.0.5-powerpc64le-linux-rhel-8.4.tar.xz
del: clang+llvm-14.0.5-powerpc64le-linux-ubuntu-18.04.5.tar.xz
del: clang+llvm-14.0.6-amd64-unknown-freebsd12.tar.xz
del: clang+llvm-14.0.6-amd64-unknown-freebsd13.tar.xz
del: clang+llvm-14.0.6-powerpc64le-linux-rhel-8.4.tar.xz
del: clang+llvm-14.0.6-powerpc64le-linux-ubuntu-18.04.5.tar.xz
del: clang+llvm-15.0.0-powerpc64le-linux-rhel-8.4.tar.xz
del: clang+llvm-15.0.0-powerpc64le-linux-ubuntu-18.04.6.tar.xz
del: clang+llvm-15.0.0-sparc64-unknown-linux-gnu.tar.xz
del: clang+llvm-15.0.1-powerpc64le-linux-rhel-8.4.tar.xz
del: clang+llvm-15.0.1-powerpc64le-linux-ubuntu-18.04.5.tar.xz
del: clang+llvm-15.0.2-powerpc64le-linux-rhel-8.4.tar.xz
del: clang+llvm-15.0.2-powerpc64le-linux-ubuntu-18.04.5.tar.xz
del: clang+llvm-15.0.2-x86_64-unknown-linux-gnu-rhel86.tar.xz
del: clang+llvm-15.0.2-x86_64-unknown-linux-gnu-sles15.tar.xz
del: clang+llvm-15.0.3-powerpc64le-linux-rhel-8.4.tar.xz
del: clang+llvm-15.0.3-powerpc64le-linux-ubuntu-18.04.5.tar.xz
del: clang+llvm-15.0.4-powerpc64le-linux-rhel-8.4.tar.xz
del: clang+llvm-15.0.4-powerpc64le-linux-ubuntu-18.04.5.tar.xz
del: clang+llvm-15.0.5-powerpc64le-linux-rhel-8.4.tar.xz
del: clang+llvm-15.0.5-powerpc64le-linux-ubuntu-18.04.5.tar.xz
del: clang+llvm-15.0.6-powerpc64le-linux-rhel-8.4.tar.xz
del: clang+llvm-15.0.6-powerpc64le-linux-ubuntu-18.04.tar.xz
del: clang+llvm-15.0.7-powerpc64le-linux-rhel-8.4.tar.xz
del: clang+llvm-15.0.7-powerpc64le-linux-ubuntu-18.04.tar.xz
del: clang+llvm-16.0.0-amd64-unknown-freebsd13.tar.xz
del: clang+llvm-16.0.0-powerpc64le-linux-rhel-8.4.tar.xz
del: clang+llvm-16.0.0-powerpc64le-linux-ubuntu-18.04.tar.xz
del: clang+llvm-16.0.0-sparc64-unknown-linux-gnu.tar.xz
del: clang+llvm-16.0.1-amd64-unknown-freebsd13.tar.xz
del: clang+llvm-16.0.1-powerpc64le-linux-rhel-8.4.tar.xz
del: clang+llvm-16.0.1-powerpc64le-linux-ubuntu-20.04.tar.xz
del: clang+llvm-16.0.2-amd64-unknown-freebsd13.tar.xz
del: clang+llvm-16.0.4-amd64-unknown-freebsd13.tar.xz
del: clang+llvm-16.0.5-amd64-unknown-freebsd13.tar.xz
del: clang+llvm-17.0.1-final_powerpc64-ibm-aix-7.2.tar.xz
del: clang+llvm-17.0.1-sparc64-unknown-linux-gnu.tar.xz
del: clang+llvm-17.0.2-sparc64-unknown-linux-gnu.tar.xz
del: clang+llvm-18.1.0-sparcv9-sun-solaris2.11.tar.xz
del: clang+llvm-18.1.0-sparcv9-unknown-linux-gnu.tar.xz
23 changes: 2 additions & 21 deletions toolchain/internal/release_name.bzl
Original file line number Diff line number Diff line change
@@ -1,7 +1,5 @@
load(
"//toolchain/internal:common.bzl",
_arch = "arch",
_os = "os",
_os_version_arch = "os_version_arch",
)

Expand Down Expand Up @@ -211,27 +209,10 @@ def _resolve_version_for_suse(major_llvm_version, llvm_version):
os_name = _ubuntu_osname("x86_64", "20.04", major_llvm_version, llvm_version)
return os_name

def llvm_release_name(rctx, llvm_version):
def llvm_release_name_context(rctx, llvm_version):
major_llvm_version = _major_llvm_version(llvm_version)
if major_llvm_version >= 19:
arch = {
"aarch64": "ARM64",
"x86_64": "X64",
}[_arch(rctx)]
os = {
"darwin": "macOS",
"linux": "Linux",
"windows": "Windows",
}[_os(rctx)]
if arch == "ARM64" and os == "Linux" and llvm_version in ["19.1.7", "19.1.6", "19.1.5", "19.1.4", "19.1.3"]:
return "clang+llvm-{llvm_version}-aarch64-linux-gnu.tar.xz".format(
llvm_version = llvm_version,
)
return "LLVM-{llvm_version}-{os}-{arch}.tar.xz".format(
llvm_version = llvm_version,
arch = arch,
os = os,
)
fail("May not use 'llvm_release_name_context' for releases starting with version 19!")
else:
(os, version, arch) = _os_version_arch(rctx)
if os == "darwin":
Expand Down
39 changes: 24 additions & 15 deletions utils/llvm_checksums.sh
Original file line number Diff line number Diff line change
Expand Up @@ -16,15 +16,18 @@
set -euo pipefail

use_github_host=0
tmp_dir=

while getopts "v:gh" opt; do
while getopts "t:v:gh" opt; do
case "${opt}" in
"t") tmp_dir="${OPTARG}" ;;
"v") llvm_version="${OPTARG}" ;;
"g") use_github_host=1 ;;
"h")
echo "Usage:"
echo "-v - Version of clang+llvm to use"
echo "-g - Use github to download releases"
echo "-t <tempdir> - Optional: Specify a temp directory to download distributions to."
echo "-v <version> - Version of clang+llvm to use."
echo "-g - Use github to download releases."
exit 2
;;
*)
Expand All @@ -35,36 +38,42 @@ while getopts "v:gh" opt; do
done

if [[ -z ${llvm_version-} ]]; then
echo "Usage: ${BASH_SOURCE[0]} [-g] -v llvm_version"
echo "Usage: ${BASH_SOURCE[0]} [-t <tempdir>] [-g] -v <llvm_version>"
exit 1
fi

tmp_dir="$(mktemp -d)"

cleanup() {
rc=$?
rm -rf "${tmp_dir}"
exit "${rc}"
}
trap 'cleanup' INT HUP QUIT TERM EXIT

if [[ -z "${tmp_dir}" ]]; then
tmp_dir="$(mktemp -d)"
echo "Using temp dir: '${tmp_dir}'"
trap 'cleanup' INT HUP QUIT TERM EXIT
elif [[ ! -r "${tmp_dir}" ]]; then
echo "Temp directory does not exist: '${tmp_dir}'."
exit 2
fi

llvm_host() {
local url_base="releases.llvm.org/${llvm_version}"
output_dir="${tmp_dir}/${url_base}"
wget --recursive --level 1 --directory-prefix="${tmp_dir}" \
--accept-regex "(clang%2bllvm|LLVM)-.*tar.xz$" "http://${url_base}/"
--accept-regex "(clang%2bllvm|LLVM)-.*tar.(xz|gz)$" "http://${url_base}/"
}

github_host() {
output_dir="${tmp_dir}"
output_dir="${tmp_dir}/${llvm_version}"
mkdir -p "${output_dir}"
(
cd "${output_dir}"
curl -s "https://api.github.com/repos/llvm/llvm-project/releases/tags/llvmorg-${llvm_version}" |
jq .assets[].browser_download_url |
tee ./urls.txt |
grep -E '(clang%2Bllvm|LLVM)-.*tar.xz"$' |
tee ./releases.json |
jq '.assets[]|select(any(.name; test("^(clang[+]llvm|LLVM)-.*tar.(xz|gz)$")))|.browser_download_url' |
tee ./filtered_urls.txt |
xargs -n1 curl -L -O
xargs -n1 curl -L -O -C -
)
}

Expand All @@ -76,8 +85,8 @@ fi

echo ""
echo "===="
echo "Checksums for clang+llvm distributions are:"
find "${output_dir}" -type f -name '*.xz' -exec shasum -a 256 {} \; |
echo "Checksums for clang+llvm distributions are (${output_dir}):"
find "${output_dir}" -type f \( -name 'clang%2?llvm-*.tar.*' -o -name 'LLVM-*.tar.*' \) \( -name '*.gz' -o -name '*.xz' \) -exec shasum -a 256 {} \; |
sed -e "s@${output_dir}/@@" |
awk '{ printf "\"%s\": \"%s\",\n", $2, $1 }' |
sed -e 's/%2[Bb]/+/' |
Expand Down