Skip to content

Commit ee1fbd7

Browse files
committed
miri-script: start and stop josh automatically
1 parent 6117fa1 commit ee1fbd7

File tree

7 files changed

+212
-25
lines changed

7 files changed

+212
-25
lines changed

src/tools/miri/.github/workflows/ci.yml

-2
Original file line numberDiff line numberDiff line change
@@ -189,8 +189,6 @@ jobs:
189189
fetch-depth: 256 # get a bit more of the history
190190
- name: install josh-proxy
191191
run: cargo +stable install josh-proxy --git https://github.com/josh-project/josh --tag r22.12.06
192-
- name: start josh-proxy
193-
run: josh-proxy --local=$HOME/.cache/josh --remote=https://github.com --no-background &
194192
- name: setup bot git name and email
195193
run: |
196194
git config --global user.name 'The Miri Conjob Bot'

src/tools/miri/CONTRIBUTING.md

+17-20
Original file line numberDiff line numberDiff line change
@@ -231,25 +231,16 @@ You can also directly run Miri on a Rust source file:
231231
## Advanced topic: Syncing with the rustc repo
232232

233233
We use the [`josh` proxy](https://github.com/josh-project/josh) to transmit changes between the
234-
rustc and Miri repositories.
234+
rustc and Miri repositories. You can install it as follows:
235235

236236
```sh
237237
cargo +stable install josh-proxy --git https://github.com/josh-project/josh --tag r22.12.06
238-
josh-proxy --local=$HOME/.cache/josh --remote=https://github.com --no-background
239238
```
240239

241-
This uses a directory `$HOME/.cache/josh` as a cache, to speed up repeated pulling/pushing.
242-
243-
To make josh push via ssh instead of https, you can add the following to your `.gitconfig`:
244-
245-
```toml
246-
[url "git@github.com:"]
247-
pushInsteadOf = https://github.com/
248-
```
240+
Josh will automatically be started and stopped by `./miri`.
249241

250242
### Importing changes from the rustc repo
251243

252-
Josh needs to be running, as described above.
253244
We assume we start on an up-to-date master branch in the Miri repo.
254245

255246
```sh
@@ -268,16 +259,14 @@ needed.
268259

269260
### Exporting changes to the rustc repo
270261

271-
Keep in mind that pushing is the most complicated job that josh has to do --
272-
pulling just filters the rustc history, but pushing needs to construct a new
273-
rustc history that would filter to the given Miri history! To avoid problems, it
274-
is a good idea to always pull immediately before you push. In particular, you
275-
should never do two josh pushes without an intermediate pull; that can lead to
276-
duplicated commits.
262+
Keep in mind that pushing is the most complicated job that josh has to do -- pulling just filters
263+
the rustc history, but pushing needs to construct a new rustc history that would filter to the given
264+
Miri history! To avoid problems, it is a good idea to always pull immediately before you push. If
265+
you are getting strange errors, chances are you are running into [this josh
266+
bug](https://github.com/josh-project/josh/issues/998). In that case, please get in touch on Zulip.
277267

278-
Josh needs to be running, as described above. We will use the josh proxy to push
279-
to your fork of rustc. Run the following in the Miri repo, assuming we are on an
280-
up-to-date master branch:
268+
We will use the josh proxy to push to your fork of rustc. Run the following in the Miri repo,
269+
assuming we are on an up-to-date master branch:
281270

282271
```sh
283272
# Push the Miri changes to your rustc fork (substitute your github handle for YOUR_NAME).
@@ -287,3 +276,11 @@ up-to-date master branch:
287276
This will create a new branch called 'miri' in your fork, and the output should
288277
include a link to create a rustc PR that will integrate those changes into the
289278
main repository.
279+
280+
If this fails due to authentication problems, it can help to make josh push via ssh instead of
281+
https. Add the following to your `.gitconfig`:
282+
283+
```toml
284+
[url "git@github.com:"]
285+
pushInsteadOf = https://github.com/
286+
```

src/tools/miri/miri-script/Cargo.lock

+125
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,38 @@ version = "1.0.71"
88
source = "registry+https://github.com/rust-lang/crates.io-index"
99
checksum = "9c7d0618f0e0b7e8ff11427422b64564d5fb0be1940354bfe2e0529b18a9d9b8"
1010

11+
[[package]]
12+
name = "bitflags"
13+
version = "1.3.2"
14+
source = "registry+https://github.com/rust-lang/crates.io-index"
15+
checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
16+
17+
[[package]]
18+
name = "cfg-if"
19+
version = "1.0.0"
20+
source = "registry+https://github.com/rust-lang/crates.io-index"
21+
checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
22+
23+
[[package]]
24+
name = "directories"
25+
version = "4.0.1"
26+
source = "registry+https://github.com/rust-lang/crates.io-index"
27+
checksum = "f51c5d4ddabd36886dd3e1438cb358cdcb0d7c499cb99cb4ac2e38e18b5cb210"
28+
dependencies = [
29+
"dirs-sys",
30+
]
31+
32+
[[package]]
33+
name = "dirs-sys"
34+
version = "0.3.7"
35+
source = "registry+https://github.com/rust-lang/crates.io-index"
36+
checksum = "1b1d1d91c932ef41c0f2663aa8b0ca0342d444d842c06914aa0a7e352d0bada6"
37+
dependencies = [
38+
"libc",
39+
"redox_users",
40+
"winapi",
41+
]
42+
1143
[[package]]
1244
name = "dunce"
1345
version = "1.0.4"
@@ -20,6 +52,17 @@ version = "1.8.1"
2052
source = "registry+https://github.com/rust-lang/crates.io-index"
2153
checksum = "7fcaabb2fef8c910e7f4c7ce9f67a1283a1715879a7c230ca9d6d1ae31f16d91"
2254

55+
[[package]]
56+
name = "getrandom"
57+
version = "0.2.10"
58+
source = "registry+https://github.com/rust-lang/crates.io-index"
59+
checksum = "be4136b2a15dd319360be1c07d9933517ccf0be8f16bf62a3bee4f0d618df427"
60+
dependencies = [
61+
"cfg-if",
62+
"libc",
63+
"wasi",
64+
]
65+
2366
[[package]]
2467
name = "itertools"
2568
version = "0.10.5"
@@ -40,6 +83,7 @@ name = "miri-script"
4083
version = "0.1.0"
4184
dependencies = [
4285
"anyhow",
86+
"directories",
4387
"dunce",
4488
"itertools",
4589
"path_macro",
@@ -62,6 +106,44 @@ version = "1.0.0"
62106
source = "registry+https://github.com/rust-lang/crates.io-index"
63107
checksum = "a6e819bbd49d5939f682638fa54826bf1650abddcd65d000923de8ad63cc7d15"
64108

109+
[[package]]
110+
name = "proc-macro2"
111+
version = "1.0.66"
112+
source = "registry+https://github.com/rust-lang/crates.io-index"
113+
checksum = "18fb31db3f9bddb2ea821cde30a9f70117e3f119938b5ee630b7403aa6e2ead9"
114+
dependencies = [
115+
"unicode-ident",
116+
]
117+
118+
[[package]]
119+
name = "quote"
120+
version = "1.0.33"
121+
source = "registry+https://github.com/rust-lang/crates.io-index"
122+
checksum = "5267fca4496028628a95160fc423a33e8b2e6af8a5302579e322e4b520293cae"
123+
dependencies = [
124+
"proc-macro2",
125+
]
126+
127+
[[package]]
128+
name = "redox_syscall"
129+
version = "0.2.16"
130+
source = "registry+https://github.com/rust-lang/crates.io-index"
131+
checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a"
132+
dependencies = [
133+
"bitflags",
134+
]
135+
136+
[[package]]
137+
name = "redox_users"
138+
version = "0.4.3"
139+
source = "registry+https://github.com/rust-lang/crates.io-index"
140+
checksum = "b033d837a7cf162d7993aded9304e30a83213c648b6e389db233191f891e5c2b"
141+
dependencies = [
142+
"getrandom",
143+
"redox_syscall",
144+
"thiserror",
145+
]
146+
65147
[[package]]
66148
name = "rustc_version"
67149
version = "0.4.0"
@@ -92,6 +174,43 @@ version = "1.1.0"
92174
source = "registry+https://github.com/rust-lang/crates.io-index"
93175
checksum = "24188a676b6ae68c3b2cb3a01be17fbf7240ce009799bb56d5b1409051e78fde"
94176

177+
[[package]]
178+
name = "syn"
179+
version = "2.0.29"
180+
source = "registry+https://github.com/rust-lang/crates.io-index"
181+
checksum = "c324c494eba9d92503e6f1ef2e6df781e78f6a7705a0202d9801b198807d518a"
182+
dependencies = [
183+
"proc-macro2",
184+
"quote",
185+
"unicode-ident",
186+
]
187+
188+
[[package]]
189+
name = "thiserror"
190+
version = "1.0.47"
191+
source = "registry+https://github.com/rust-lang/crates.io-index"
192+
checksum = "97a802ec30afc17eee47b2855fc72e0c4cd62be9b4efe6591edde0ec5bd68d8f"
193+
dependencies = [
194+
"thiserror-impl",
195+
]
196+
197+
[[package]]
198+
name = "thiserror-impl"
199+
version = "1.0.47"
200+
source = "registry+https://github.com/rust-lang/crates.io-index"
201+
checksum = "6bb623b56e39ab7dcd4b1b98bb6c8f8d907ed255b18de254088016b27a8ee19b"
202+
dependencies = [
203+
"proc-macro2",
204+
"quote",
205+
"syn",
206+
]
207+
208+
[[package]]
209+
name = "unicode-ident"
210+
version = "1.0.11"
211+
source = "registry+https://github.com/rust-lang/crates.io-index"
212+
checksum = "301abaae475aa91687eb82514b328ab47a211a533026cb25fc3e519b86adfc3c"
213+
95214
[[package]]
96215
name = "walkdir"
97216
version = "2.3.3"
@@ -102,6 +221,12 @@ dependencies = [
102221
"winapi-util",
103222
]
104223

224+
[[package]]
225+
name = "wasi"
226+
version = "0.11.0+wasi-snapshot-preview1"
227+
source = "registry+https://github.com/rust-lang/crates.io-index"
228+
checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423"
229+
105230
[[package]]
106231
name = "which"
107232
version = "4.4.0"

src/tools/miri/miri-script/Cargo.toml

+1
Original file line numberDiff line numberDiff line change
@@ -20,3 +20,4 @@ anyhow = "1.0"
2020
xshell = "0.2"
2121
rustc_version = "0.4"
2222
dunce = "1.0.4"
23+
directories = "4"

src/tools/miri/miri-script/src/commands.rs

+66-3
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,9 @@ use std::env;
22
use std::ffi::OsString;
33
use std::io::Write;
44
use std::ops::Not;
5+
use std::process;
6+
use std::thread;
7+
use std::time;
58

69
use anyhow::{anyhow, bail, Context, Result};
710
use path_macro::path;
@@ -14,6 +17,7 @@ use crate::Command;
1417
/// Used for rustc syncs.
1518
const JOSH_FILTER: &str =
1619
":rev(75dd959a3a40eb5b4574f8d2e23aa6efbeb33573:prefix=src/tools/miri):/src/tools/miri";
20+
const JOSH_PORT: &str = "42042";
1721

1822
impl MiriEnv {
1923
fn build_miri_sysroot(&mut self, quiet: bool) -> Result<()> {
@@ -81,6 +85,55 @@ impl Command {
8185
Ok(())
8286
}
8387

88+
fn start_josh() -> Result<impl Drop> {
89+
// Determine cache directory.
90+
let local_dir = {
91+
let user_dirs =
92+
directories::ProjectDirs::from("org", "rust-lang", "miri-josh").unwrap();
93+
user_dirs.cache_dir().to_owned()
94+
};
95+
96+
// Start josh, silencing its output.
97+
let mut cmd = process::Command::new("josh-proxy");
98+
cmd.arg("--local").arg(local_dir);
99+
cmd.arg("--remote").arg("https://github.com");
100+
cmd.arg("--port").arg(JOSH_PORT);
101+
cmd.arg("--no-background");
102+
cmd.stdout(process::Stdio::null());
103+
cmd.stderr(process::Stdio::null());
104+
let josh = cmd.spawn().context("failed to start josh-proxy, make sure it is installed")?;
105+
// Give it some time so hopefully the port is open. (10ms was not enough.)
106+
thread::sleep(time::Duration::from_millis(100));
107+
108+
// Create a wrapper that stops it on drop.
109+
struct Josh(process::Child);
110+
impl Drop for Josh {
111+
fn drop(&mut self) {
112+
#[cfg(unix)]
113+
{
114+
// Try to gracefully shut it down.
115+
process::Command::new("kill")
116+
.args(["-s", "INT", &self.0.id().to_string()])
117+
.output()
118+
.expect("failed to SIGINT josh-proxy");
119+
// Sadly there is no "wait with timeout"... so we just give it some time to finish.
120+
thread::sleep(time::Duration::from_millis(100));
121+
// Now hopefully it is gone.
122+
if self.0.try_wait().expect("failed to wait for josh-proxy").is_some() {
123+
return;
124+
}
125+
}
126+
// If that didn't work (or we're not on Unix), kill it hard.
127+
eprintln!(
128+
"I have to kill josh-proxy the hard way, let's hope this does not break anything."
129+
);
130+
self.0.kill().expect("failed to SIGKILL josh-proxy");
131+
}
132+
}
133+
134+
Ok(Josh(josh))
135+
}
136+
84137
pub fn exec(self) -> Result<()> {
85138
// First, and crucially only once, run the auto-actions -- but not for all commands.
86139
match &self {
@@ -174,6 +227,8 @@ impl Command {
174227
if cmd!(sh, "git status --untracked-files=no --porcelain").read()?.is_empty().not() {
175228
bail!("working directory must be clean before running `./miri rustc-pull`");
176229
}
230+
// Make sure josh is running.
231+
let josh = Self::start_josh()?;
177232

178233
// Update rust-version file. As a separate commit, since making it part of
179234
// the merge has confused the heck out of josh in the past.
@@ -186,7 +241,7 @@ impl Command {
186241
.context("FAILED to commit rust-version file, something went wrong")?;
187242

188243
// Fetch given rustc commit.
189-
cmd!(sh, "git fetch http://localhost:8000/rust-lang/rust.git@{commit}{JOSH_FILTER}.git")
244+
cmd!(sh, "git fetch http://localhost:{JOSH_PORT}/rust-lang/rust.git@{commit}{JOSH_FILTER}.git")
190245
.run()
191246
.map_err(|e| {
192247
// Try to un-do the previous `git commit`, to leave the repo in the state we found it it.
@@ -202,6 +257,8 @@ impl Command {
202257
cmd!(sh, "git merge FETCH_HEAD --no-verify --no-ff -m {MERGE_COMMIT_MESSAGE}")
203258
.run()
204259
.context("FAILED to merge new commits, something went wrong")?;
260+
261+
drop(josh);
205262
Ok(())
206263
}
207264

@@ -213,6 +270,8 @@ impl Command {
213270
if cmd!(sh, "git status --untracked-files=no --porcelain").read()?.is_empty().not() {
214271
bail!("working directory must be clean before running `./miri rustc-push`");
215272
}
273+
// Make sure josh is running.
274+
let josh = Self::start_josh()?;
216275

217276
// Find a repo we can do our preparation in.
218277
if let Ok(rustc_git) = env::var("RUSTC_GIT") {
@@ -249,6 +308,8 @@ impl Command {
249308
}
250309
cmd!(sh, "git fetch https://github.com/rust-lang/rust {base}").run()?;
251310
cmd!(sh, "git push https://github.com/{github_user}/rust {base}:refs/heads/{branch}")
311+
.ignore_stdout()
312+
.ignore_stderr() // silence the "create GitHub PR" message
252313
.run()?;
253314
println!();
254315

@@ -257,15 +318,15 @@ impl Command {
257318
println!("Pushing miri changes...");
258319
cmd!(
259320
sh,
260-
"git push http://localhost:8000/{github_user}/rust.git{JOSH_FILTER}.git HEAD:{branch}"
321+
"git push http://localhost:{JOSH_PORT}/{github_user}/rust.git{JOSH_FILTER}.git HEAD:{branch}"
261322
)
262323
.run()?;
263324
println!();
264325

265326
// Do a round-trip check to make sure the push worked as expected.
266327
cmd!(
267328
sh,
268-
"git fetch http://localhost:8000/{github_user}/rust.git{JOSH_FILTER}.git {branch}"
329+
"git fetch http://localhost:{JOSH_PORT}/{github_user}/rust.git{JOSH_FILTER}.git {branch}"
269330
)
270331
.ignore_stderr()
271332
.read()?;
@@ -278,6 +339,8 @@ impl Command {
278339
"Confirmed that the push round-trips back to Miri properly. Please create a rustc PR:"
279340
);
280341
println!(" https://github.com/{github_user}/rust/pull/new/{branch}");
342+
343+
drop(josh);
281344
Ok(())
282345
}
283346

src/tools/miri/miri-script/src/main.rs

+2
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
1+
#![allow(clippy::needless_question_mark)]
2+
13
mod commands;
24
mod util;
35

0 commit comments

Comments
 (0)