Skip to content

Commit 0255649

Browse files
rcowniercownie
committed
add support for --internalOnlySourceWritesDoneBarrier
1 parent 8ece9ae commit 0255649

File tree

6 files changed

+388
-0
lines changed

6 files changed

+388
-0
lines changed

‎etc/mise.run.sh

Lines changed: 301 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,301 @@
1+
#!/bin/sh
2+
3+
# This is copied from mise.run.
4+
5+
set -o errexit
6+
set -o nounset
7+
8+
#region logging setup
9+
if [ "${MISE_DEBUG-}" = "true" ] || [ "${MISE_DEBUG-}" = "1" ]; then
10+
debug() {
11+
echo "$@" >&2
12+
}
13+
else
14+
debug() {
15+
:
16+
}
17+
fi
18+
19+
if [ "${MISE_QUIET-}" = "1" ] || [ "${MISE_QUIET-}" = "true" ]; then
20+
info() {
21+
:
22+
}
23+
else
24+
info() {
25+
echo "$@" >&2
26+
}
27+
fi
28+
29+
error() {
30+
echo "$@" >&2
31+
exit 1
32+
}
33+
#endregion
34+
35+
#region environment setup
36+
get_os() {
37+
os="$(uname -s)"
38+
if [ "$os" = Darwin ]; then
39+
echo "macos"
40+
elif [ "$os" = Linux ]; then
41+
echo "linux"
42+
else
43+
error "unsupported OS: $os"
44+
fi
45+
}
46+
47+
get_arch() {
48+
musl=""
49+
# XXX - MongoDB-specific change - this lets us install Mise on RHEL 7.0 systems, which have an
50+
# older glibc than the one needed by the non-musl mise binary.
51+
if [ "$(uname -s)" = Linux ]; then
52+
musl="-musl"
53+
fi
54+
if type ldd >/dev/null 2>/dev/null; then
55+
libc=$(ldd /bin/ls | grep 'musl' | head -1 | cut -d ' ' -f1)
56+
if [ -n "$libc" ]; then
57+
musl="-musl"
58+
fi
59+
fi
60+
arch="$(uname -m)"
61+
if [ "$arch" = x86_64 ]; then
62+
echo "x64$musl"
63+
elif [ "$arch" = aarch64 ] || [ "$arch" = arm64 ]; then
64+
echo "arm64$musl"
65+
elif [ "$arch" = armv7l ]; then
66+
echo "armv7$musl"
67+
else
68+
error "unsupported architecture: $arch"
69+
fi
70+
}
71+
72+
get_ext() {
73+
if [ -n "${MISE_INSTALL_EXT:-}" ]; then
74+
echo "$MISE_INSTALL_EXT"
75+
elif [ -n "${MISE_VERSION:-}" ] && echo "$MISE_VERSION" | grep -q '^v2024'; then
76+
# 2024 versions don't have zstd tarballs
77+
echo "tar.gz"
78+
elif tar_supports_zstd; then
79+
echo "tar.zst"
80+
elif command -v zstd >/dev/null 2>&1; then
81+
echo "tar.zst"
82+
else
83+
echo "tar.gz"
84+
fi
85+
}
86+
87+
tar_supports_zstd() {
88+
# tar is bsdtar or version is >= 1.31
89+
if tar --version | grep -q 'bsdtar' && command -v zstd >/dev/null 2>&1; then
90+
true
91+
elif tar --version | grep -q '1\.(3[1-9]|[4-9][0-9]'; then
92+
true
93+
else
94+
false
95+
fi
96+
}
97+
98+
shasum_bin() {
99+
if command -v shasum >/dev/null 2>&1; then
100+
echo "shasum"
101+
elif command -v sha256sum >/dev/null 2>&1; then
102+
echo "sha256sum"
103+
else
104+
error "mise install requires shasum or sha256sum but neither is installed. Aborting."
105+
fi
106+
}
107+
108+
get_checksum() {
109+
version=$1
110+
os="$(get_os)"
111+
arch="$(get_arch)"
112+
ext="$(get_ext)"
113+
url="https://github.com/jdx/mise/releases/download/v${version}/SHASUMS256.txt"
114+
115+
# For current version use static checksum otherwise
116+
# use checksum from releases
117+
if [ "$version" = "v2025.2.7" ]; then
118+
checksum_linux_x86_64="a29c2600fcefc0488d3e0e16d5cfc9ccfccdb06f6348c4cd4050a5eeaf9dd53d ./mise-v2025.2.7-linux-x64.tar.gz"
119+
checksum_linux_x86_64_musl="bed90247c1f637377855bba1c90056d7d1d1a3c36c6f2370054152df45b57f02 ./mise-v2025.2.7-linux-x64-musl.tar.gz"
120+
checksum_linux_arm64="fe4f89997d8b4c9e7a1d3ac8733117d5463021c8c87f04a40deb27d34752c15f ./mise-v2025.2.7-linux-arm64.tar.gz"
121+
checksum_linux_arm64_musl="d0c2519b2db28cd51ac9ffd966bc6a484f8942e7fa19943460a9717ccc68e85e ./mise-v2025.2.7-linux-arm64-musl.tar.gz"
122+
checksum_linux_armv7="7a1d9d36135b35a82a636780acc638ca3f9cb240efb4786ce3905c8a187c0d19 ./mise-v2025.2.7-linux-armv7.tar.gz"
123+
checksum_linux_armv7_musl="abb499fff1b20dc4044206d4a00753c9e69e26d64c05b489285f7730de43075b ./mise-v2025.2.7-linux-armv7-musl.tar.gz"
124+
checksum_macos_x86_64="b8c04d4099b9956b48e3942a43e6425e6c3a5c813da8b5c18a9d7eb473cf518f ./mise-v2025.2.7-macos-x64.tar.gz"
125+
checksum_macos_arm64="376f463de5266b446da78c76de0030c9d2df8a4b5ca58c06cb608a05c2cb6d0a ./mise-v2025.2.7-macos-arm64.tar.gz"
126+
checksum_linux_x86_64_zstd="f1af6e34936224d0a2d2f8dfc123e3cd69863718722c3eef75962ed18db0dfd2 ./mise-v2025.2.7-linux-x64.tar.zst"
127+
checksum_linux_x86_64_musl_zstd="be4f0ab5945f0c821f0b60c12935e8e6cae9ebc3807a941cccf36af4bee42b48 ./mise-v2025.2.7-linux-x64-musl.tar.zst"
128+
checksum_linux_arm64_zstd="58ac3e32bea8b931619d02a1fdb9c86d990835b2f125e44caa9f4a689ed5f4e8 ./mise-v2025.2.7-linux-arm64.tar.zst"
129+
checksum_linux_arm64_musl_zstd="0d22664b59dc930ae58048671de5284122dbd66f409fd573c2160d94e6f9f9b7 ./mise-v2025.2.7-linux-arm64-musl.tar.zst"
130+
checksum_linux_armv7_zstd="7bf64f9562d505a31c17c164bfb2dde13e92babda5aace6f1de54d594c70f908 ./mise-v2025.2.7-linux-armv7.tar.zst"
131+
checksum_linux_armv7_musl_zstd="51481e505a74cbed5a66860e54b4e3cfe26c28e7b95cd44f15f9b38075b9c3b6 ./mise-v2025.2.7-linux-armv7-musl.tar.zst"
132+
checksum_macos_x86_64_zstd="b17fde3a4bbfb98347ae8a93662e4b12285b6e7e8a7b57c0c6db9e1e4a0420e9 ./mise-v2025.2.7-macos-x64.tar.zst"
133+
checksum_macos_arm64_zstd="181243d3901a31430e2214bc137f915a4960b207cd15bac9ae503996e4e25c1e ./mise-v2025.2.7-macos-arm64.tar.zst"
134+
135+
# TODO: refactor this, it's a bit messy
136+
if [ "$(get_ext)" = "tar.zst" ]; then
137+
if [ "$os" = "linux" ]; then
138+
if [ "$arch" = "x64" ]; then
139+
echo "$checksum_linux_x86_64_zstd"
140+
elif [ "$arch" = "x64-musl" ]; then
141+
echo "$checksum_linux_x86_64_musl_zstd"
142+
elif [ "$arch" = "arm64" ]; then
143+
echo "$checksum_linux_arm64_zstd"
144+
elif [ "$arch" = "arm64-musl" ]; then
145+
echo "$checksum_linux_arm64_musl_zstd"
146+
elif [ "$arch" = "armv7" ]; then
147+
echo "$checksum_linux_armv7_zstd"
148+
elif [ "$arch" = "armv7-musl" ]; then
149+
echo "$checksum_linux_armv7_musl_zstd"
150+
else
151+
warn "no checksum for $os-$arch"
152+
fi
153+
elif [ "$os" = "macos" ]; then
154+
if [ "$arch" = "x64" ]; then
155+
echo "$checksum_macos_x86_64_zstd"
156+
elif [ "$arch" = "arm64" ]; then
157+
echo "$checksum_macos_arm64_zstd"
158+
else
159+
warn "no checksum for $os-$arch"
160+
fi
161+
else
162+
warn "no checksum for $os-$arch"
163+
fi
164+
else
165+
if [ "$os" = "linux" ]; then
166+
if [ "$arch" = "x64" ]; then
167+
echo "$checksum_linux_x86_64"
168+
elif [ "$arch" = "x64-musl" ]; then
169+
echo "$checksum_linux_x86_64_musl"
170+
elif [ "$arch" = "arm64" ]; then
171+
echo "$checksum_linux_arm64"
172+
elif [ "$arch" = "arm64-musl" ]; then
173+
echo "$checksum_linux_arm64_musl"
174+
elif [ "$arch" = "armv7" ]; then
175+
echo "$checksum_linux_armv7"
176+
elif [ "$arch" = "armv7-musl" ]; then
177+
echo "$checksum_linux_armv7_musl"
178+
else
179+
warn "no checksum for $os-$arch"
180+
fi
181+
elif [ "$os" = "macos" ]; then
182+
if [ "$arch" = "x64" ]; then
183+
echo "$checksum_macos_x86_64"
184+
elif [ "$arch" = "arm64" ]; then
185+
echo "$checksum_macos_arm64"
186+
else
187+
warn "no checksum for $os-$arch"
188+
fi
189+
else
190+
warn "no checksum for $os-$arch"
191+
fi
192+
fi
193+
else
194+
if command -v curl >/dev/null 2>&1; then
195+
debug ">" curl -fsSL "$url"
196+
checksums="$(curl --compressed -fsSL "$url")"
197+
else
198+
if command -v wget >/dev/null 2>&1; then
199+
debug ">" wget -qO - "$url"
200+
stderr=$(mktemp)
201+
checksums="$(wget -qO - "$url")"
202+
else
203+
error "mise standalone install specific version requires curl or wget but neither is installed. Aborting."
204+
fi
205+
fi
206+
# TODO: verify with minisign or gpg if available
207+
208+
checksum="$(echo "$checksums" | grep "$os-$arch.$ext")"
209+
if ! echo "$checksum" | grep -Eq "^([0-9a-f]{32}|[0-9a-f]{64})"; then
210+
warn "no checksum for mise $version and $os-$arch"
211+
else
212+
echo "$checksum"
213+
fi
214+
fi
215+
}
216+
217+
#endregion
218+
219+
download_file() {
220+
url="$1"
221+
filename="$(basename "$url")"
222+
cache_dir="$(mktemp -d)"
223+
file="$cache_dir/$filename"
224+
225+
info "mise: installing mise..."
226+
227+
if command -v curl >/dev/null 2>&1; then
228+
debug ">" curl -#fLo "$file" "$url"
229+
curl -#fLo "$file" "$url"
230+
else
231+
if command -v wget >/dev/null 2>&1; then
232+
debug ">" wget -qO "$file" "$url"
233+
stderr=$(mktemp)
234+
wget -O "$file" "$url" >"$stderr" 2>&1 || error "wget failed: $(cat "$stderr")"
235+
else
236+
error "mise standalone install requires curl or wget but neither is installed. Aborting."
237+
fi
238+
fi
239+
240+
echo "$file"
241+
}
242+
243+
install_mise() {
244+
version="${MISE_VERSION:-v2025.2.7}"
245+
version="${version#v}"
246+
os="$(get_os)"
247+
arch="$(get_arch)"
248+
ext="$(get_ext)"
249+
install_path="${MISE_INSTALL_PATH:-$HOME/.local/bin/mise}"
250+
install_dir="$(dirname "$install_path")"
251+
tarball_url="https://github.com/jdx/mise/releases/download/v${version}/mise-v${version}-${os}-${arch}.${ext}"
252+
253+
cache_file=$(download_file "$tarball_url")
254+
debug "mise-setup: tarball=$cache_file"
255+
256+
debug "validating checksum"
257+
cd "$(dirname "$cache_file")" && get_checksum "$version" | "$(shasum_bin)" -c >/dev/null
258+
259+
# extract tarball
260+
mkdir -p "$install_dir"
261+
rm -rf "$install_path"
262+
cd "$(mktemp -d)"
263+
if [ "$(get_ext)" = "tar.zst" ] && ! tar_supports_zstd; then
264+
zstd -d -c "$cache_file" | tar -xf -
265+
else
266+
tar -xf "$cache_file"
267+
fi
268+
mv mise/bin/mise "$install_path"
269+
info "mise: installed successfully to $install_path"
270+
}
271+
272+
after_finish_help() {
273+
case "${SHELL:-}" in
274+
*/zsh)
275+
info "mise: run the following to activate mise in your shell:"
276+
info "echo \"eval \\\"\\\$($install_path activate zsh)\\\"\" >> \"${ZDOTDIR-$HOME}/.zshrc\""
277+
info ""
278+
info "mise: run \`mise doctor\` to verify this is setup correctly"
279+
;;
280+
*/bash)
281+
info "mise: run the following to activate mise in your shell:"
282+
info "echo \"eval \\\"\\\$($install_path activate bash)\\\"\" >> ~/.bashrc"
283+
info ""
284+
info "mise: run \`mise doctor\` to verify this is setup correctly"
285+
;;
286+
*/fish)
287+
info "mise: run the following to activate mise in your shell:"
288+
info "echo \"$install_path activate fish | source\" >> ~/.config/fish/config.fish"
289+
info ""
290+
info "mise: run \`mise doctor\` to verify this is setup correctly"
291+
;;
292+
*)
293+
info "mise: run \`$install_path --help\` to get started"
294+
;;
295+
esac
296+
}
297+
298+
install_mise
299+
if [ "${MISE_INSTALL_HELP-}" != 0 ]; then
300+
after_finish_help
301+
fi

‎mise.toml

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
# We use ubi for nearly everything we can for two reasons. One is consistency. This way we don't
2+
# have two developers who compile a Go package with different Go versions. The other is that it's
3+
# the fastest way to install these packages. In many cases, these tools are available in the mise
4+
# registry with multiple backends, so spelling out "ubi:..." ensures we use the ubi backend.
5+
[tools]
6+
# Go stuff
7+
go = "1.23.6"
8+
"go:golang.org/x/tools/cmd/goimports" = "0.24.0"
9+
"go:golang.org/x/tools/cmd/stringer" = "0.14.0"
10+
"ubi:golangci/golangci-lint" = "1.63.4"
11+
"ubi:securego/gosec" = "2.22.0"
12+
"ubi:segmentio/golines" = "0.12.2"
13+
"ubi:vektra/mockery" = "2.46.3"
14+
15+
# Other stuff
16+
"node" = "22.14.0"
17+
"npm:eslint" = "9.19.0"
18+
"npm:jest" = "29.7.0"
19+
"npm:prettier" = "3.4.2"
20+
"ubi:astral-sh/ruff" = "0.9.4"
21+
"ubi:houseabsolute/omegasort" = "0.1.3"
22+
"ubi:houseabsolute/precious" = "0.7.3"
23+
"ubi:koalaman/shellcheck" = "0.10.0"
24+
"ubi:mvdan/sh" = { exe = "shfmt", version = "3.10.0" }

‎mongodump/barrier.go

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
// Copyright (C) MongoDB, Inc. 2014-present.
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License"); you may
4+
// not use this file except in compliance with the License. You may obtain
5+
// a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
6+
package mongodump
7+
8+
import (
9+
"os"
10+
"time"
11+
12+
"github.com/mongodb/mongo-tools/common/log"
13+
)
14+
15+
// Wait until a file exists and can be opened for reading
16+
// This is used only for testing mongodump/mongorestore with resmoke
17+
// test infrastructure. The tests will create the barrier file when they have
18+
// finshed writes to the source cluster.
19+
func waitForSourceWritesDoneBarrier(barrierName string) {
20+
log.Logvf(log.Always, "waitForSourceWritesDoneBarrier: %s", barrierName)
21+
start := time.Now()
22+
logInterval := time.Minute
23+
prevLogTime := start
24+
for {
25+
f, err := os.Open(barrierName)
26+
if err == nil {
27+
// We opened the file for reading, so it does exist.
28+
f.Close()
29+
return
30+
}
31+
if os.IsNotExist(err) {
32+
if time.Since(prevLogTime) >= logInterval {
33+
prevLogTime = time.Now()
34+
log.Logvf(log.Always, "waitForSourceWritesDoneBarrier: waited %d secs for %s",
35+
prevLogTime.Sub(start).Seconds(),
36+
barrierName)
37+
}
38+
// Poll for existence of the barrier file every 500msec
39+
time.Sleep(500*time.Millisecond)
40+
} else {
41+
panic(err)
42+
}
43+
}
44+
}

‎mongodump/mongodump.go

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -226,6 +226,11 @@ func (dump *MongoDump) Dump() (err error) {
226226
return nil
227227
}
228228

229+
if !dump.OutputOptions.Oplog && (dump.InputOptions.SourceWritesDoneBarrier != "") {
230+
// Wait for tests to stop writes before dumping any collections
231+
waitForSourceWritesDoneBarrier(dump.InputOptions.SourceWritesDoneBarrier)
232+
}
233+
229234
log.Logvf(log.DebugHigh, "starting Dump()")
230235

231236
dump.shutdownIntentsNotifier = newNotifier()
@@ -430,6 +435,10 @@ func (dump *MongoDump) Dump() (err error) {
430435
// we check to see if the oplog has rolled over (i.e. the most recent entry when
431436
// we started still exist, so we know we haven't lost data)
432437
if dump.OutputOptions.Oplog {
438+
if dump.InputOptions.SourceWritesDoneBarrier != "" {
439+
// Wait for tests to stop writes before choosing the oplogEnd time
440+
waitForSourceWritesDoneBarrier(dump.InputOptions.SourceWritesDoneBarrier)
441+
}
433442
dump.oplogEnd, err = dump.getCurrentOplogTime()
434443
if err != nil {
435444
return fmt.Errorf("error getting oplog end: %v", err)

0 commit comments

Comments
 (0)