Skip to content

Commit 9fa6bdd

Browse files
borsbors
committed
Auto merge of #112482 - tgross35:ci-non-rust-linters, r=pietroalbini
Add support for tidy linting via external tools for non-rust files This change adds the flag `--check-extras` to `tidy`. It accepts a comma separated list of any of the options: * py (test everything applicable for python files) * py:lint (lint python files using `ruff`) * py:fmt (check formatting for python files using `black`) * shell or shell:lint (lint shell files using `shellcheck`) Specific files to check can also be specified via positional args. Examples: * `./x test tidy --check-extras=shell,py` * `./x test tidy --check-extras=py:fmt -- src/bootstrap/bootstrap.py` * `./x test tidy --check-extras=shell -- src/ci/*.sh` * Python formatting can be applied with bless: `./x test tidy --ckeck-extras=py:fmt --bless` `ruff` and `black` need to be installed via pip; this tool manages these within a virtual environment at `build/venv`. `shellcheck` needs to be installed on the system already. --- This PR doesn't fix any of the errors that show up (I will likely go through those at some point) and it doesn't enforce anything new in CI. Relevant zulip discussion: https://rust-lang.zulipchat.com/#narrow/stream/242791-t-infra/topic/Other.20linters.20in.20CI
2 parents 307c573 + 9df0f5d commit 9fa6bdd

File tree

18 files changed

+682
-17
lines changed

18 files changed

+682
-17
lines changed

‎src/bootstrap/bootstrap.py

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -472,7 +472,9 @@ def __init__(self):
472472

473473
class RustBuild(object):
474474
"""Provide all the methods required to build Rust"""
475-
def __init__(self, config_toml="", args=FakeArgs()):
475+
def __init__(self, config_toml="", args=None):
476+
if args is None:
477+
args = FakeArgs()
476478
self.git_version = None
477479
self.nix_deps_dir = None
478480
self._should_fix_bins_and_dylibs = None

‎src/bootstrap/bootstrap_test.py

Lines changed: 14 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,6 @@
44

55
from __future__ import absolute_import, division, print_function
66
import os
7-
import doctest
87
import unittest
98
import tempfile
109
import hashlib
@@ -16,12 +15,15 @@
1615
bootstrap_dir = os.path.dirname(os.path.abspath(__file__))
1716
# For the import below, have Python search in src/bootstrap first.
1817
sys.path.insert(0, bootstrap_dir)
19-
import bootstrap
20-
import configure
18+
import bootstrap # noqa: E402
19+
import configure # noqa: E402
2120

22-
def serialize_and_parse(configure_args, bootstrap_args=bootstrap.FakeArgs()):
21+
def serialize_and_parse(configure_args, bootstrap_args=None):
2322
from io import StringIO
2423

24+
if bootstrap_args is None:
25+
bootstrap_args = bootstrap.FakeArgs()
26+
2527
section_order, sections, targets = configure.parse_args(configure_args)
2628
buffer = StringIO()
2729
configure.write_config_toml(buffer, section_order, targets, sections)
@@ -129,7 +131,14 @@ def test_set_codegen_backends(self):
129131
class BuildBootstrap(unittest.TestCase):
130132
"""Test that we generate the appropriate arguments when building bootstrap"""
131133

132-
def build_args(self, configure_args=[], args=[], env={}):
134+
def build_args(self, configure_args=None, args=None, env=None):
135+
if configure_args is None:
136+
configure_args = []
137+
if args is None:
138+
args = []
139+
if env is None:
140+
env = {}
141+
133142
env = env.copy()
134143
env["PATH"] = os.environ["PATH"]
135144

‎src/bootstrap/builder/tests.rs

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -587,6 +587,7 @@ mod dist {
587587
run: None,
588588
only_modified: false,
589589
skip: vec![],
590+
extra_checks: None,
590591
};
591592

592593
let build = Build::new(config);
@@ -658,6 +659,7 @@ mod dist {
658659
pass: None,
659660
run: None,
660661
only_modified: false,
662+
extra_checks: None,
661663
};
662664
// Make sure rustfmt binary not being found isn't an error.
663665
config.channel = "beta".to_string();

‎src/bootstrap/flags.rs

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -339,6 +339,10 @@ pub enum Subcommand {
339339
/// whether to automatically update stderr/stdout files
340340
bless: bool,
341341
#[arg(long)]
342+
/// comma-separated list of other files types to check (accepts py, py:lint,
343+
/// py:fmt, shell)
344+
extra_checks: Option<String>,
345+
#[arg(long)]
342346
/// rerun tests even if the inputs are unchanged
343347
force_rerun: bool,
344348
#[arg(long)]
@@ -476,6 +480,13 @@ impl Subcommand {
476480
}
477481
}
478482

483+
pub fn extra_checks(&self) -> Option<&str> {
484+
match *self {
485+
Subcommand::Test { ref extra_checks, .. } => extra_checks.as_ref().map(String::as_str),
486+
_ => None,
487+
}
488+
}
489+
479490
pub fn only_modified(&self) -> bool {
480491
match *self {
481492
Subcommand::Test { only_modified, .. } => only_modified,

‎src/bootstrap/test.rs

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
//! our CI.
55
66
use std::env;
7+
use std::ffi::OsStr;
78
use std::ffi::OsString;
89
use std::fs;
910
use std::iter;
@@ -1094,6 +1095,14 @@ impl Step for Tidy {
10941095
if builder.config.cmd.bless() {
10951096
cmd.arg("--bless");
10961097
}
1098+
if let Some(s) = builder.config.cmd.extra_checks() {
1099+
cmd.arg(format!("--extra-checks={s}"));
1100+
}
1101+
let mut args = std::env::args_os();
1102+
if let Some(_) = args.find(|arg| arg == OsStr::new("--")) {
1103+
cmd.arg("--");
1104+
cmd.args(args);
1105+
}
10971106

10981107
if builder.config.channel == "dev" || builder.config.channel == "nightly" {
10991108
builder.info("fmt check");

‎src/ci/docker/host-x86_64/mingw-check-tidy/Dockerfile

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -26,11 +26,13 @@ COPY scripts/sccache.sh /scripts/
2626
RUN sh /scripts/sccache.sh
2727

2828
COPY host-x86_64/mingw-check/reuse-requirements.txt /tmp/
29-
RUN pip3 install --no-deps --no-cache-dir --require-hashes -r /tmp/reuse-requirements.txt
29+
RUN pip3 install --no-deps --no-cache-dir --require-hashes -r /tmp/reuse-requirements.txt \
30+
&& pip3 install virtualenv
3031

3132
COPY host-x86_64/mingw-check/validate-toolstate.sh /scripts/
3233
COPY host-x86_64/mingw-check/validate-error-codes.sh /scripts/
3334

3435
# NOTE: intentionally uses python2 for x.py so we can test it still works.
3536
# validate-toolstate only runs in our CI, so it's ok for it to only support python3.
36-
ENV SCRIPT python2.7 ../x.py test --stage 0 src/tools/tidy tidyselftest
37+
ENV SCRIPT TIDY_PRINT_DIFF=1 python2.7 ../x.py test \
38+
--stage 0 src/tools/tidy tidyselftest --extra-checks=py:lint

‎src/ci/docker/scripts/fuchsia-test-runner.py

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -15,12 +15,10 @@
1515
import json
1616
import os
1717
import platform
18-
import re
1918
import shutil
20-
import signal
2119
import subprocess
2220
import sys
23-
from typing import ClassVar, List, Optional
21+
from typing import ClassVar, List
2422

2523

2624
@dataclass
@@ -523,7 +521,7 @@ def log(msg):
523521
env_vars += '\n "RUST_BACKTRACE=0",'
524522

525523
# Use /tmp as the test temporary directory
526-
env_vars += f'\n "RUST_TEST_TMPDIR=/tmp",'
524+
env_vars += '\n "RUST_TEST_TMPDIR=/tmp",'
527525

528526
cml.write(
529527
self.CML_TEMPLATE.format(env_vars=env_vars, exe_name=exe_name)

‎src/etc/completions/x.py.fish

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -234,6 +234,7 @@ complete -c x.py -n "__fish_seen_subcommand_from doc" -s h -l help -d 'Print hel
234234
complete -c x.py -n "__fish_seen_subcommand_from test" -l skip -d 'skips tests matching SUBSTRING, if supported by test tool. May be passed multiple times' -r -F
235235
complete -c x.py -n "__fish_seen_subcommand_from test" -l test-args -d 'extra arguments to be passed for the test tool being used (e.g. libtest, compiletest or rustdoc)' -r
236236
complete -c x.py -n "__fish_seen_subcommand_from test" -l rustc-args -d 'extra options to pass the compiler when running tests' -r
237+
complete -c x.py -n "__fish_seen_subcommand_from test" -l extra-checks -d 'comma-separated list of other files types to check (accepts py, py:lint, py:fmt, shell)' -r
237238
complete -c x.py -n "__fish_seen_subcommand_from test" -l compare-mode -d 'mode describing what file the actual ui output will be compared to' -r
238239
complete -c x.py -n "__fish_seen_subcommand_from test" -l pass -d 'force {check,build,run}-pass tests to this mode' -r
239240
complete -c x.py -n "__fish_seen_subcommand_from test" -l run -d 'whether to execute run-* tests' -r

‎src/etc/completions/x.py.ps1

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -306,6 +306,7 @@ Register-ArgumentCompleter -Native -CommandName 'x.py' -ScriptBlock {
306306
[CompletionResult]::new('--skip', 'skip', [CompletionResultType]::ParameterName, 'skips tests matching SUBSTRING, if supported by test tool. May be passed multiple times')
307307
[CompletionResult]::new('--test-args', 'test-args', [CompletionResultType]::ParameterName, 'extra arguments to be passed for the test tool being used (e.g. libtest, compiletest or rustdoc)')
308308
[CompletionResult]::new('--rustc-args', 'rustc-args', [CompletionResultType]::ParameterName, 'extra options to pass the compiler when running tests')
309+
[CompletionResult]::new('--extra-checks', 'extra-checks', [CompletionResultType]::ParameterName, 'comma-separated list of other files types to check (accepts py, py:lint, py:fmt, shell)')
309310
[CompletionResult]::new('--compare-mode', 'compare-mode', [CompletionResultType]::ParameterName, 'mode describing what file the actual ui output will be compared to')
310311
[CompletionResult]::new('--pass', 'pass', [CompletionResultType]::ParameterName, 'force {check,build,run}-pass tests to this mode')
311312
[CompletionResult]::new('--run', 'run', [CompletionResultType]::ParameterName, 'whether to execute run-* tests')

‎src/etc/completions/x.py.sh

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1625,7 +1625,7 @@ _x.py() {
16251625
return 0
16261626
;;
16271627
x.py__test)
1628-
opts="-v -i -j -h --no-fail-fast --skip --test-args --rustc-args --no-doc --doc --bless --force-rerun --only-modified --compare-mode --pass --run --rustfix-coverage --verbose --incremental --config --build-dir --build --host --target --exclude --include-default-paths --rustc-error-format --on-fail --dry-run --stage --keep-stage --keep-stage-std --src --jobs --warnings --error-format --json-output --color --llvm-skip-rebuild --rust-profile-generate --rust-profile-use --llvm-profile-use --llvm-profile-generate --reproducible-artifact --set --help [PATHS]... [ARGS]..."
1628+
opts="-v -i -j -h --no-fail-fast --skip --test-args --rustc-args --no-doc --doc --bless --extra-checks --force-rerun --only-modified --compare-mode --pass --run --rustfix-coverage --verbose --incremental --config --build-dir --build --host --target --exclude --include-default-paths --rustc-error-format --on-fail --dry-run --stage --keep-stage --keep-stage-std --src --jobs --warnings --error-format --json-output --color --llvm-skip-rebuild --rust-profile-generate --rust-profile-use --llvm-profile-use --llvm-profile-generate --reproducible-artifact --set --help [PATHS]... [ARGS]..."
16291629
if [[ ${cur} == -* || ${COMP_CWORD} -eq 2 ]] ; then
16301630
COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") )
16311631
return 0
@@ -1643,6 +1643,10 @@ _x.py() {
16431643
COMPREPLY=($(compgen -f "${cur}"))
16441644
return 0
16451645
;;
1646+
--extra-checks)
1647+
COMPREPLY=($(compgen -f "${cur}"))
1648+
return 0
1649+
;;
16461650
--compare-mode)
16471651
COMPREPLY=($(compgen -f "${cur}"))
16481652
return 0

0 commit comments

Comments
 (0)