Skip to content

Commit f45565a

Browse files
authored
Merge pull request #418 from pietroalbini/tests
Add a test suite to the team repository
2 parents 820ed9b + 10b3a86 commit f45565a

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

61 files changed

+889
-56
lines changed

.github/workflows/main.yml

+3
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,9 @@ jobs:
3131
- name: Run clippy
3232
run: cargo clippy -- -Dwarnings
3333

34+
- name: Run tests
35+
run: cargo test
36+
3437
- name: Build the contents of the static API
3538
run: |
3639
cargo run -- static-api build

.gitignore

+1
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,3 @@
11
/target
22
**/*.rs.bk
3+
/tests/static-api/_output

Cargo.lock

+115-48
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

Cargo.toml

+7
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,13 @@ indexmap = "1.0.2"
1919
reqwest = "0.9.18"
2020
base64 = "0.10.1"
2121

22+
[dev-dependencies]
23+
duct = "0.13.4"
24+
atty = "0.2.14"
25+
ansi_term = "0.12.1"
26+
difference = "2.0.0"
27+
walkdir = "2.3.1"
28+
2229
[workspace]
2330
members = [
2431
"rust_team_data"

src/github.rs

+1-1
Original file line numberDiff line numberDiff line change
@@ -86,7 +86,7 @@ impl GitHubApi {
8686
.send()?
8787
.error_for_status()?
8888
.json()?;
89-
if let Some(error) = res.errors.iter().next() {
89+
if let Some(error) = res.errors.get(0) {
9090
bail!("graphql error: {}", error.message);
9191
} else if let Some(data) = res.data {
9292
Ok(data)

src/main.rs

+15-2
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,12 @@ enum Cli {
2121
Check {
2222
#[structopt(long = "strict", help = "fail if optional checks are not executed")]
2323
strict: bool,
24+
#[structopt(
25+
long = "skip",
26+
multiple = true,
27+
help = "skip one or more validation steps"
28+
)]
29+
skip: Vec<String>,
2430
},
2531
#[structopt(
2632
name = "add-person",
@@ -50,6 +56,9 @@ fn main() {
5056
env.default_format_timestamp(false);
5157
env.default_format_module_path(false);
5258
env.filter_module("rust_team", log::LevelFilter::Info);
59+
if std::env::var("RUST_TEAM_FORCE_COLORS").is_ok() {
60+
env.write_style(env_logger::WriteStyle::Always);
61+
}
5362
if let Ok(content) = std::env::var("RUST_LOG") {
5463
env.parse(&content);
5564
}
@@ -68,8 +77,12 @@ fn run() -> Result<(), Error> {
6877
let cli = Cli::from_args();
6978
let data = Data::load()?;
7079
match cli {
71-
Cli::Check { strict } => {
72-
crate::validate::validate(&data, strict)?;
80+
Cli::Check { strict, skip } => {
81+
crate::validate::validate(
82+
&data,
83+
strict,
84+
&skip.iter().map(|s| s.as_ref()).collect::<Vec<_>>(),
85+
)?;
7386
}
7487
Cli::AddPerson { ref github_name } => {
7588
#[derive(serde::Serialize)]

src/validate.rs

+34-5
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,19 @@ use log::{error, warn};
66
use regex::Regex;
77
use std::collections::{HashMap, HashSet};
88

9-
static CHECKS: &[fn(&Data, &mut Vec<String>)] = &[
9+
macro_rules! checks {
10+
($($f:ident,)*) => {
11+
&[$(
12+
Check {
13+
f: $f,
14+
name: stringify!($f)
15+
}
16+
),*]
17+
}
18+
}
19+
20+
#[allow(clippy::type_complexity)]
21+
static CHECKS: &[Check<fn(&Data, &mut Vec<String>)>] = checks![
1022
validate_name_prefixes,
1123
validate_subteam_of,
1224
validate_team_leads,
@@ -28,13 +40,25 @@ static CHECKS: &[fn(&Data, &mut Vec<String>)] = &[
2840
validate_project_groups_have_parent_teams,
2941
];
3042

31-
static GITHUB_CHECKS: &[fn(&Data, &GitHubApi, &mut Vec<String>)] = &[validate_github_usernames];
43+
#[allow(clippy::type_complexity)]
44+
static GITHUB_CHECKS: &[Check<fn(&Data, &GitHubApi, &mut Vec<String>)>] =
45+
checks![validate_github_usernames,];
46+
47+
struct Check<F> {
48+
f: F,
49+
name: &'static str,
50+
}
3251

33-
pub(crate) fn validate(data: &Data, strict: bool) -> Result<(), Error> {
52+
pub(crate) fn validate(data: &Data, strict: bool, skip: &[&str]) -> Result<(), Error> {
3453
let mut errors = Vec::new();
3554

3655
for check in CHECKS {
37-
check(data, &mut errors);
56+
if skip.contains(&check.name) {
57+
warn!("skipped check: {}", check.name);
58+
continue;
59+
}
60+
61+
(check.f)(data, &mut errors);
3862
}
3963

4064
let github = GitHubApi::new();
@@ -47,7 +71,12 @@ pub(crate) fn validate(data: &Data, strict: bool) -> Result<(), Error> {
4771
}
4872
} else {
4973
for check in GITHUB_CHECKS {
50-
check(data, &github, &mut errors);
74+
if skip.contains(&check.name) {
75+
warn!("skipped check: {}", check.name);
76+
continue;
77+
}
78+
79+
(check.f)(data, &github, &mut errors);
5180
}
5281
}
5382

tests/README.md

+28
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
# Automated tests
2+
3+
The tool developed in this repository is the cornerstone of the Rust Team's
4+
access control, and a bug in it could result in unauthorized people gaining
5+
access to protected resources and systems. To avoid that, the repository
6+
employs automated tests to discover regressions quickly.
7+
8+
You can run all the tests with the command:
9+
10+
```
11+
cargo test
12+
```
13+
14+
## Static API test
15+
16+
The Static API test ensures the output of the Static API for a dummy repository
17+
is what we expect. This test uses the snapshot testing technique, and the
18+
expected copy is a build of the Static API stored in the git repository.
19+
20+
To add something to the Static API test make the changes you need to the files
21+
in `tests/static-api` direcory: its layout is the same as the top-level
22+
contents. Once you made the changes, run `cargo test` to preview the diff. If
23+
the diff is what you expect, run the following command to mark it as the
24+
expected one:
25+
26+
```
27+
tests/bless.sh
28+
```

tests/bless.sh

+12
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
#!/bin/bash
2+
3+
set -euo pipefail
4+
IFS=$'\n\t'
5+
6+
cd "$(dirname "$0")"
7+
if [[ -d static-api/_output ]]; then
8+
rm -rf static-api/_expected
9+
cp -r static-api/_output static-api/_expected
10+
else
11+
echo "didn't bless static-api as there is no output to bless"
12+
fi

tests/snapshot.rs

+119
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,119 @@
1+
use duct::{cmd, Expression};
2+
use failure::Error;
3+
use std::{
4+
collections::HashSet,
5+
path::{Path, PathBuf},
6+
};
7+
8+
#[test]
9+
fn static_api() -> Result<(), Error> {
10+
let dir_output = dir_valid().join("_output");
11+
let dir_expected = dir_valid().join("_expected");
12+
13+
if dir_output.exists() {
14+
std::fs::remove_dir_all(&dir_output)?;
15+
}
16+
17+
step("checking whether the data is valid");
18+
cmd!(bin(), "check", "--skip", "validate_github_usernames")
19+
.dir(dir_valid())
20+
.assert_success()?;
21+
22+
step("generating the static api contents");
23+
cmd!(bin(), "static-api", &dir_output)
24+
.dir(dir_valid())
25+
.assert_success()?;
26+
27+
step("checking whether the output matched the expected one");
28+
29+
// Collect all the files present in either the output or expected dirs
30+
let mut files = HashSet::new();
31+
for dir in &[&dir_output, &dir_expected] {
32+
for entry in walkdir::WalkDir::new(dir) {
33+
let entry = entry?;
34+
if !entry.file_type().is_file() {
35+
continue;
36+
}
37+
files.insert(entry.path().strip_prefix(dir)?.to_path_buf());
38+
}
39+
}
40+
41+
// Check whether any file is different
42+
let mut failed = false;
43+
for file in &files {
44+
let expected = std::fs::read_to_string(dir_expected.join(file))
45+
.ok()
46+
.unwrap_or_default();
47+
let output = std::fs::read_to_string(dir_output.join(file))
48+
.ok()
49+
.unwrap_or_default();
50+
51+
let changeset = difference::Changeset::new(&expected, &output, "\n");
52+
if changeset.distance != 0 {
53+
failed = true;
54+
println!(
55+
"{} {} {}",
56+
ansi_term::Color::Red.bold().paint("!!! the file"),
57+
ansi_term::Color::White
58+
.bold()
59+
.paint(&file.to_str().unwrap().to_string()),
60+
ansi_term::Color::Red.bold().paint("does not match"),
61+
);
62+
println!("{}", changeset);
63+
}
64+
}
65+
if failed {
66+
println!(
67+
"{} {}",
68+
ansi_term::Color::Cyan
69+
.bold()
70+
.paint("==> If you believe the new content is right, run:"),
71+
ansi_term::Color::White.bold().paint("tests/bless.sh")
72+
);
73+
println!();
74+
panic!("there were differences in the expected output");
75+
}
76+
77+
Ok(())
78+
}
79+
80+
fn bin() -> &'static str {
81+
env!("CARGO_BIN_EXE_rust-team")
82+
}
83+
84+
fn dir_valid() -> PathBuf {
85+
Path::new(env!("CARGO_MANIFEST_DIR"))
86+
.join("tests")
87+
.join("static-api")
88+
}
89+
90+
fn step(name: &str) {
91+
println!(
92+
"{}",
93+
ansi_term::Color::White
94+
.bold()
95+
.paint(&format!("==> {}", name))
96+
);
97+
}
98+
99+
trait ExpressionExt {
100+
fn assert_success(self) -> Result<(), Error>;
101+
}
102+
103+
impl ExpressionExt for Expression {
104+
fn assert_success(mut self) -> Result<(), Error> {
105+
// If the environment variable is not passed colors will never be shown.
106+
if atty::is(atty::Stream::Stdout) {
107+
self = self.env("RUST_TEAM_FORCE_COLORS", "1");
108+
}
109+
110+
// Redirects stderr to stdout to be able to print the output in the correct order.
111+
let res = self.stderr_to_stdout().stdout_capture().unchecked().run()?;
112+
print!("{}", String::from_utf8_lossy(&res.stdout));
113+
114+
if !res.status.success() {
115+
failure::bail!("command returned a non-zero exit code!");
116+
}
117+
Ok(())
118+
}
119+
}
+19
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
{
2+
"lists": {
3+
4+
"address": "[email protected]",
5+
"members": [
6+
7+
8+
9+
]
10+
},
11+
12+
"address": "[email protected]",
13+
"members": [
14+
15+
16+
]
17+
}
18+
}
19+
}
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
{
2+
"github_users": [],
3+
"github_ids": [],
4+
"discord_ids": []
5+
}
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
{
2+
"github_users": [],
3+
"github_ids": [],
4+
"discord_ids": []
5+
}
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
{
2+
"github_users": [],
3+
"github_ids": [],
4+
"discord_ids": []
5+
}
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
{
2+
"github_users": [],
3+
"github_ids": [],
4+
"discord_ids": []
5+
}
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
{
2+
"github_users": [],
3+
"github_ids": [],
4+
"discord_ids": []
5+
}
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
{
2+
"github_users": [],
3+
"github_ids": [],
4+
"discord_ids": []
5+
}
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
{
2+
"github_users": [],
3+
"github_ids": [],
4+
"discord_ids": []
5+
}
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
{
2+
"github_users": [],
3+
"github_ids": [],
4+
"discord_ids": []
5+
}
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
{
2+
"github_users": [],
3+
"github_ids": [],
4+
"discord_ids": []
5+
}

0 commit comments

Comments
 (0)