Skip to content

Commit 36d9ce1

Browse files
bors[bot]Alexhuszagh
bors[bot]
and
Alexhuszagh
authored
Merge #1164
1164: Add package site mirrors for FreeBSD. r=Emilgardis a=Alexhuszagh Closes #1162. Co-authored-by: Alex Huszagh <[email protected]>
2 parents f253bd0 + bdcaeec commit 36d9ce1

File tree

5 files changed

+207
-82
lines changed

5 files changed

+207
-82
lines changed

‎.changes/1164.json

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
{
2+
"type": "internal",
3+
"description": "add fallback mirrors for FreeBSD packages.",
4+
"issues": [1162]
5+
}

‎docker/freebsd-common.sh

Lines changed: 4 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -6,57 +6,14 @@ set -euo pipefail
66
# shellcheck disable=SC1091
77
. freebsd-arch.sh
88

9-
export BSD_ARCH=
9+
export FREEBSD_ARCH=
1010
case "${ARCH}" in
1111
x86_64)
12-
BSD_ARCH=amd64
12+
FREEBSD_ARCH=amd64
1313
;;
1414
i686)
15-
BSD_ARCH=i386
15+
FREEBSD_ARCH=i386
1616
;;
1717
esac
1818

19-
# we prefer those closer in geography to the US. they're triaged in
20-
# order of ease of use, reliability, and then geography. the mirror
21-
# list is at https://docs.freebsd.org/en/books/handbook/mirrors/.
22-
# these mirrors were known to work as of 2022-11-27. this does
23-
# not include any mirrors that are known to be rate-limited or
24-
# commercial.
25-
export BSD_HOME=(
26-
# these do not return HTML, and only list the directories
27-
"ftp.freebsd.org/pub/FreeBSD/releases"
28-
# these return HTML output, and therefore are lower priority
29-
"ftp11.freebsd.org/pub/FreeBSD/releases"
30-
"ftp3.br.freebsd.org/pub/FreeBSD/releases"
31-
"ftp2.uk.freebsd.org/pub/FreeBSD/releases"
32-
"ftp2.nl.freebsd.org/pub/FreeBSD/releases"
33-
"ftp6.fr.freebsd.org/pub/FreeBSD/releases"
34-
"ftp1.de.freebsd.org/pub/FreeBSD/releases"
35-
"ftp2.de.freebsd.org/pub/FreeBSD/releases"
36-
"ftp5.de.freebsd.org/pub/FreeBSD/releases"
37-
"ftp2.ru.freebsd.org/pub/FreeBSD/releases"
38-
"ftp2.gr.freebsd.org/pub/FreeBSD/releases"
39-
"ftp4.za.freebsd.org/pub/FreeBSD/releases"
40-
"ftp2.za.freebsd.org/pub/FreeBSD/releases"
41-
"ftp4.tw.freebsd.org/pub/FreeBSD/releases"
42-
"ftp3.jp.freebsd.org/pub/FreeBSD/releases"
43-
"ftp6.jp.freebsd.org/pub/FreeBSD/releases"
44-
# these only support HTTP, and not implicit
45-
# FTP as well, and have HTML output
46-
"http://ftp.uk.freebsd.org/pub/FreeBSD/releases"
47-
"http://ftp.nl.freebsd.org/pub/FreeBSD/releases"
48-
"http://ftp.fr.freebsd.org/pub/FreeBSD/releases"
49-
"http://ftp.at.freebsd.org/pub/FreeBSD/releases"
50-
"http://ftp.dk.freebsd.org/FreeBSD/releases"
51-
"http://ftp.cz.freebsd.org/pub/FreeBSD/releases"
52-
"http://ftp.se.freebsd.org/pub/FreeBSD/releases"
53-
"http://ftp.lv.freebsd.org/freebsd/releases"
54-
"http://ftp.pl.freebsd.org/pub/FreeBSD/releases"
55-
"http://ftp.ua.freebsd.org/pub/FreeBSD/releases"
56-
"http://ftp.gr.freebsd.org/pub/FreeBSD/releases"
57-
"http://ftp.ru.freebsd.org/pub/FreeBSD/releases"
58-
"http://ftp.nz.freebsd.org/pub/FreeBSD/releases"
59-
"http://ftp.kr.freebsd.org/pub/FreeBSD/releases"
60-
"http://ftp.jp.freebsd.org/pub/FreeBSD/releases"
61-
)
62-
export BSD_MAJOR=12
19+
export FREEBSD_MAJOR=12

‎docker/freebsd-extras.sh

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -11,8 +11,16 @@ set -euo pipefail
1111
. freebsd-install.sh
1212

1313
main() {
14-
setup_packagesite
15-
install_freebsd_package openssl sqlite3
14+
apt-get update && apt-get install --assume-yes --no-install-recommends \
15+
curl \
16+
dnsutils \
17+
jq \
18+
xz-utils
19+
20+
local url=
21+
url=$(fetch_best_freebsd_mirror)
22+
FREEBSD_MIRROR="${url}" setup_freebsd_packagesite
23+
FREEBSD_MIRROR="${url}" install_freebsd_package openssl sqlite3
1624

1725
rm "${0}"
1826
}

‎docker/freebsd-install.sh

Lines changed: 126 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -6,35 +6,147 @@ set -euo pipefail
66
# shellcheck disable=SC1091
77
. freebsd-common.sh
88

9-
export PACKAGESITE=/opt/freebsd-packagesite/packagesite.yaml
10-
export PKG_SOURCE="https://pkg.freebsd.org/FreeBSD:${BSD_MAJOR}:${BSD_ARCH}/quarterly"
11-
export TARGET="${ARCH}-unknown-freebsd${BSD_MAJOR}"
9+
# list of SRV records to query if the default mirror fails
10+
FREEBSD_HTTP_TCP_SOURCES=(
11+
# these return all mirrors, including local ones
12+
"_http._tcp.pkg.all.freebsd.org"
13+
# this only returns geodns mirrors
14+
"_http._tcp.pkg.freebsd.org"
15+
)
16+
FREEBSD_PACKAGEDIR="/opt/freebsd-packagesite"
17+
FREEBSD_PACKAGESITE="${FREEBSD_PACKAGEDIR}/packagesite.yaml"
18+
FREEBSD_TARGET="${ARCH}-unknown-freebsd${FREEBSD_MAJOR}"
19+
FREEBSD_DEFAULT_MIRROR="pkg.freebsd.org"
20+
# NOTE: these mirrors were known to work as of 2022-11-28.
21+
# no availability guarantees are made for any of them.
22+
FREEBSD_BACKUP_MIRRORS=(
23+
"pkg0.syd.freebsd.org"
24+
"pkg0.bme.freebsd.org"
25+
"pkg0.bra.freebsd.org"
26+
"pkg0.fra.freebsd.org"
27+
"pkg0.jinx.freebsd.org"
28+
"pkg0.kul.freebsd.org"
29+
"pkg0.kwc.freebsd.org"
30+
"pkg0.nyi.freebsd.org"
31+
"pkg0.tuk.freebsd.org"
32+
"pkg0.twn.freebsd.org"
33+
)
1234

13-
setup_packagesite() {
14-
apt-get update && apt-get install --assume-yes --no-install-recommends \
15-
curl \
16-
jq \
17-
xz-utils
35+
# NOTE: out of convention, we use `url` for mirrors with the scheme,
36+
# and `mirror` for those without the scheme for consistent naming.
37+
freebsd_package_source() {
38+
local url="${1}"
39+
echo "${url}/FreeBSD:${FREEBSD_MAJOR}:${FREEBSD_ARCH}/quarterly"
40+
}
41+
42+
freebsd_mirror_works() {
43+
local mirror="${1}"
44+
local scheme="${2}"
45+
local pkg_source=
46+
47+
# meta.conf is a small file for quick confirmation the mirror works
48+
pkg_source=$(freebsd_package_source "${scheme}://${mirror}")
49+
local path="${pkg_source}/meta.conf"
50+
51+
timeout 20s curl --retry 3 -sSfL "${path}" >/dev/null 2>&1
52+
}
1853

19-
mkdir /opt/freebsd-packagesite
20-
curl --retry 3 -sSfL "${PKG_SOURCE}/packagesite.txz" -O
21-
tar -C /opt/freebsd-packagesite -xJf packagesite.txz
54+
_fetch_best_freebsd_mirror() {
55+
# in case if the default mirror is down, we can use various known
56+
# fallbacks, or at worst, SRV fallbacks to find the ideal package
57+
# site. no individual mirror other than the default mirror is
58+
# guaranteed to exist, so we use a tiered approach. only
59+
# the default mirror supports https.
60+
if freebsd_mirror_works "${FREEBSD_DEFAULT_MIRROR}" "https"; then
61+
echo "https://${FREEBSD_DEFAULT_MIRROR}"
62+
return 0
63+
fi
64+
65+
# if we've gotten here, it could be a DNS issue, so using a DNS
66+
# resolver to fetch SRV fallbacks may not work. let's first try
67+
# a few previously tested mirrors and see if any work.
68+
local mirror=
69+
for mirror in "${FREEBSD_BACKUP_MIRRORS[@]}"; do
70+
if freebsd_mirror_works "${mirror}" "http"; then
71+
echo "http://${mirror}"
72+
return 0
73+
fi
74+
done
75+
76+
local http_tcp_source=
77+
local response=
78+
local lines=
79+
# shellcheck disable=SC2016
80+
local regex='/\d+\s+\d+\s+\d+\s+(.*)\./; print $1'
81+
for http_tcp_source in "${FREEBSD_HTTP_TCP_SOURCES[@]}"; do
82+
# the output will have the following format, but we only want the
83+
# target and ignore everything else:
84+
# $priority $port $weight $target.
85+
#
86+
# some output may not match, so we skip those lines, for example:
87+
# 96.47.72.71
88+
response=$(dig +short srv "${http_tcp_source}")
89+
readarray -t lines <<< "${response}"
90+
for line in "${lines[@]}"; do
91+
mirror=$(echo "${line}" | perl -nle "${regex}")
92+
if [[ -n "${mirror}" ]]; then
93+
if freebsd_mirror_works "${mirror}" "http"; then
94+
echo "http://${mirror}"
95+
return 0
96+
fi
97+
fi
98+
done
99+
done
100+
101+
echo -e "\e[31merror:\e[0m could not find a working FreeBSD package mirror." 1>&2
102+
exit 1
103+
}
104+
105+
fetch_best_freebsd_mirror() {
106+
set +e
107+
_fetch_best_freebsd_mirror
108+
code=$?
109+
set -e
110+
111+
return "${code}"
112+
}
113+
114+
setup_freebsd_packagesite() {
115+
local url="${FREEBSD_MIRROR:-}"
116+
local pkg_source=
117+
118+
if [[ -z "${url}" ]]; then
119+
url=$(fetch_best_freebsd_mirror)
120+
fi
121+
pkg_source=$(freebsd_package_source "${url}")
122+
123+
mkdir -p "${FREEBSD_PACKAGEDIR}"
124+
curl --retry 3 -sSfL "${pkg_source}/packagesite.txz" -O
125+
tar -C "${FREEBSD_PACKAGEDIR}" -xJf packagesite.txz
22126

23127
rm packagesite.txz
24128
}
25129

130+
# don't provide the mirror as a positional argument, so it can be optional
26131
install_freebsd_package() {
132+
local url="${FREEBSD_MIRROR:-}"
133+
local pkg_source=
27134
local name
28135
local path
29136
local pkg
30137
local td
31-
local destdir="/usr/local/${TARGET}"
138+
local destdir="/usr/local/${FREEBSD_TARGET}"
139+
140+
if [[ -z "${url}" ]]; then
141+
url=$(fetch_best_freebsd_mirror)
142+
fi
143+
pkg_source=$(freebsd_package_source "${url}")
32144

33145
td="$(mktemp -d)"
34146
pushd "${td}"
35147

36148
for name in "${@}"; do
37-
path=$(jq -c '. | select ( .name == "'"${name}"'" ) | .repopath' "${PACKAGESITE}")
149+
path=$(jq -c '. | select ( .name == "'"${name}"'" ) | .repopath' "${FREEBSD_PACKAGESITE}")
38150
if [[ -z "${path}" ]]; then
39151
echo "Unable to find package ${name}" >&2
40152
exit 1
@@ -43,7 +155,7 @@ install_freebsd_package() {
43155
pkg=$(basename "${path}")
44156

45157
mkdir "${td}"/package
46-
curl --retry 3 -sSfL "${PKG_SOURCE}/${path}" -O
158+
curl --retry 3 -sSfL "${pkg_source}/${path}" -O
47159
tar -C "${td}/package" -xJf "${pkg}"
48160
cp -r "${td}/package/usr/local"/* "${destdir}"/
49161

‎docker/freebsd.sh

Lines changed: 62 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,47 @@ set -euo pipefail
88
# shellcheck disable=SC1091
99
. lib.sh
1010

11+
# we prefer those closer in geography to the US. they're triaged in
12+
# order of ease of use, reliability, and then geography. the mirror
13+
# list is at https://docs.freebsd.org/en/books/handbook/mirrors/.
14+
# these mirrors were known to work as of 2022-11-27. this does
15+
# not include any mirrors that are known to be rate-limited or
16+
# commercial. everything returns HTML output.
17+
MIRRORS=(
18+
# this is a guaranteed mirror, unlike those below.
19+
"http://ftp.freebsd.org/pub/FreeBSD/releases"
20+
"http://ftp11.freebsd.org/pub/FreeBSD/releases"
21+
"http://ftp3.br.freebsd.org/pub/FreeBSD/releases"
22+
"http://ftp2.uk.freebsd.org/pub/FreeBSD/releases"
23+
"http://ftp2.nl.freebsd.org/pub/FreeBSD/releases"
24+
"http://ftp6.fr.freebsd.org/pub/FreeBSD/releases"
25+
"http://ftp1.de.freebsd.org/pub/FreeBSD/releases"
26+
"http://ftp2.de.freebsd.org/pub/FreeBSD/releases"
27+
"http://ftp5.de.freebsd.org/pub/FreeBSD/releases"
28+
"http://ftp2.ru.freebsd.org/pub/FreeBSD/releases"
29+
"http://ftp2.gr.freebsd.org/pub/FreeBSD/releases"
30+
"http://ftp4.za.freebsd.org/pub/FreeBSD/releases"
31+
"http://ftp2.za.freebsd.org/pub/FreeBSD/releases"
32+
"http://ftp4.tw.freebsd.org/pub/FreeBSD/releases"
33+
"http://ftp3.jp.freebsd.org/pub/FreeBSD/releases"
34+
"http://ftp6.jp.freebsd.org/pub/FreeBSD/releases"
35+
"http://ftp.uk.freebsd.org/pub/FreeBSD/releases"
36+
"http://ftp.nl.freebsd.org/pub/FreeBSD/releases"
37+
"http://ftp.fr.freebsd.org/pub/FreeBSD/releases"
38+
"http://ftp.at.freebsd.org/pub/FreeBSD/releases"
39+
"http://ftp.dk.freebsd.org/FreeBSD/releases"
40+
"http://ftp.cz.freebsd.org/pub/FreeBSD/releases"
41+
"http://ftp.se.freebsd.org/pub/FreeBSD/releases"
42+
"http://ftp.lv.freebsd.org/freebsd/releases"
43+
"http://ftp.pl.freebsd.org/pub/FreeBSD/releases"
44+
"http://ftp.ua.freebsd.org/pub/FreeBSD/releases"
45+
"http://ftp.gr.freebsd.org/pub/FreeBSD/releases"
46+
"http://ftp.ru.freebsd.org/pub/FreeBSD/releases"
47+
"http://ftp.nz.freebsd.org/pub/FreeBSD/releases"
48+
"http://ftp.kr.freebsd.org/pub/FreeBSD/releases"
49+
"http://ftp.jp.freebsd.org/pub/FreeBSD/releases"
50+
)
51+
1152
max_freebsd() {
1253
local best=
1354
local minor=0
@@ -18,13 +59,13 @@ max_freebsd() {
1859
version=$(echo "${release}" | cut -d '-' -f 1)
1960
release_major=$(echo "${version}"| cut -d '.' -f 1)
2061
release_minor=$(echo "${version}"| cut -d '.' -f 2)
21-
if [ "${release_major}" == "${BSD_MAJOR}" ] && [ "${release_minor}" -gt "${minor}" ]; then
62+
if [ "${release_major}" == "${FREEBSD_MAJOR}" ] && [ "${release_minor}" -gt "${minor}" ]; then
2263
best="${release}"
2364
minor="${release_minor}"
2465
fi
2566
done
2667
if [[ -z "$best" ]]; then
27-
echo -e "\e[31merror:\e[0m could not find best release for FreeBSD ${BSD_MAJOR}." 1>&2
68+
echo -e "\e[31merror:\e[0m could not find best release for FreeBSD ${FREEBSD_MAJOR}." 1>&2
2869
exit 1
2970
fi
3071
echo "${best}"
@@ -38,9 +79,9 @@ latest_freebsd() {
3879
local releases=
3980
local max_release=
4081

41-
response=$(curl --silent --list-only --location "${mirror}/${BSD_ARCH}/" | grep RELEASE)
82+
response=$(curl --retry 3 -sSfL "${mirror}/${FREEBSD_ARCH}/" | grep RELEASE)
4283
if [[ "${response}" != *RELEASE* ]]; then
43-
echo -e "\e[31merror:\e[0m could not find a candidate release for FreeBSD ${BSD_MAJOR}." 1>&2
84+
echo -e "\e[31merror:\e[0m could not find a candidate release for FreeBSD ${FREEBSD_MAJOR}." 1>&2
4485
exit 1
4586
fi
4687
readarray -t lines <<< "${response}"
@@ -66,43 +107,45 @@ latest_freebsd() {
66107
echo "${max_release//-RELEASE/}"
67108
}
68109

69-
freebsd_mirror() {
70-
local home=
110+
_freebsd_mirror() {
111+
local mirror=
71112
local code=
72113

73-
set +e
74-
for home in "${BSD_HOME[@]}"; do
114+
for mirror in "${MIRRORS[@]}"; do
75115
# we need a timeout in case the server is down to avoid
76116
# infinitely hanging. timeout error code is always 124
77117
# these mirrors can be quite slow, so have a long timeout
78-
timeout 20s curl --silent --list-only --location "${home}/${BSD_ARCH}/" >/dev/null
118+
timeout 20s curl --retry 3 -sSfL "${mirror}/${FREEBSD_ARCH}/" >/dev/null
79119
code=$?
80120
if [[ "${code}" == 0 ]]; then
81-
echo "${home}"
121+
echo "${mirror}"
82122
return 0
83123
elif [[ "${code}" != 124 ]]; then
84-
echo -e "\e[1;33mwarning:\e[0m mirror ${home} does not seem to work." 1>&2
124+
echo -e "\e[1;33mwarning:\e[0m mirror ${mirror} does not seem to work." 1>&2
85125
fi
86126
done
87-
set -e
88127

89128
echo -e "\e[31merror:\e[0m could not find a working FreeBSD mirror." 1>&2
90129
exit 1
91130
}
92131

132+
freebsd_mirror() {
133+
set +e
134+
_freebsd_mirror
135+
code=$?
136+
set -e
137+
138+
return "${code}"
139+
}
140+
93141
mirror=$(freebsd_mirror)
94142
base_release=$(latest_freebsd "${mirror}")
95-
bsd_base_url="${mirror}/${BSD_ARCH}/${base_release}-RELEASE"
96-
if [[ "${bsd_base_url}" == "http"* ]]; then
97-
bsd_url="${bsd_base_url}"
98-
else
99-
bsd_url="http://${bsd_base_url}"
100-
fi
143+
bsd_url="${mirror}/${FREEBSD_ARCH}/${base_release}-RELEASE"
101144

102145
main() {
103146
local binutils=2.32 \
104147
gcc=6.4.0 \
105-
target="${ARCH}-unknown-freebsd${BSD_MAJOR}"
148+
target="${ARCH}-unknown-freebsd${FREEBSD_MAJOR}"
106149

107150
install_packages ca-certificates \
108151
curl \

0 commit comments

Comments
 (0)