Merge pull request #316 from kirtchev-adacore/wip/issue-315-add-libtest-json-emitter

Add libtest JSON emitter

Author: oli-obk

CHANGELOG.md

@@ -9,6 +9,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Added
 
+* support for JSON output via argument `--format=json`
+
 ### Fixed
 
 * missing lines in diff output 

src/config/args.rs

@@ -40,6 +40,8 @@ pub struct Args {
 /// Possible choices for styling the output.
 #[derive(Debug, Copy, Clone, Default)]
 pub enum Format {
+    /// JSON format
+    JSON,
     /// Print one line per test
     #[default]
     Pretty,
@@ -76,8 +78,9 @@ impl Args {
                 // We ignore this flag for now.
             } else if let Some(format) = parse_value("--format", &arg, &mut iter)? {
                 self.format = match &*format {
-                    "terse" => Format::Terse,
+                    "json" => Format::JSON,
                     "pretty" => Format::Pretty,
+                    "terse" => Format::Terse,
                     _ => bail!("unsupported format `{format}`"),
                 };
             } else if let Some(skip) = parse_value("--skip", &arg, &mut iter)? {

src/lib.rs

@@ -81,18 +81,16 @@ pub fn run_tests(mut config: Config) -> Result<()> {
     #[cfg(feature = "gha")]
     let name = display(&config.root_dir);
 
-    let text = match args.format {
-        Format::Terse => status_emitter::Text::quiet(),
-        Format::Pretty => status_emitter::Text::verbose(),
-    };
+    let emitter: Box<dyn StatusEmitter> = args.format.into();
+
     config.with_args(&args);
 
     run_tests_generic(
         vec![config],
         default_file_filter,
         default_per_file_config,
         (
-            text,
+            emitter,
             #[cfg(feature = "gha")]
             status_emitter::Gha::<true> { name },
         ),
@@ -180,7 +178,7 @@ pub fn run_tests_generic(
     mut configs: Vec<Config>,
     file_filter: impl Fn(&Path, &Config) -> Option<bool> + Sync,
     per_file_config: impl Copy + Fn(&mut Config, &Spanned<Vec<u8>>) + Send + Sync + 'static,
-    status_emitter: impl StatusEmitter + Send,
+    status_emitter: impl StatusEmitter,
 ) -> Result<()> {
     if nextest::emulate(&mut configs) {
         return Ok(());

src/status_emitter.rs

@@ -1,22 +1,42 @@
 //! Various schemes for reporting messages during testing or after testing is done.
-
-use crate::{test_result::TestResult, Errors};
+//!
+//! The testing framework employs the implementations of the various emitter traits
+//! as follows:
+//!
+//! The framework first creates an instance of a `StatusEmitter`.
+//!
+//! The framework then searches for tests in its perview, and if it finds one, it
+//! calls `StatusEmitter::register_test()` to obtain a `TestStatus` for that test.
+//! The tests are then executed in an asynchonous manner.
+//!
+//! Once a single test finish executing, the framework calls `TestStatus::done()`.
+//!
+//! Once all tests finish executing, the framework calls `StatusEmitter::finalize()`
+//! to obtain a Summary.
+//!
+//! For each failed test, the framework calls both `TestStatus::failed_test()` and
+//! `Summary::test_failure()`.
+
+use crate::{test_result::TestResult, Errors, Format};
 
 use std::{
+    boxed::Box,
     fmt::Debug,
     panic::RefUnwindSafe,
     path::{Path, PathBuf},
 };
-pub use text::*;
 pub mod debug;
-mod text;
 #[cfg(feature = "gha")]
 pub use gha::*;
 #[cfg(feature = "gha")]
 mod gha;
+pub use json::*;
+mod json;
+pub use text::*;
+mod text;
 
 /// A generic way to handle the output of this crate.
-pub trait StatusEmitter: Sync + RefUnwindSafe {
+pub trait StatusEmitter: Send + Sync + RefUnwindSafe {
     /// Invoked the moment we know a test will later be run.
     /// Useful for progress bars and such.
     fn register_test(&self, path: PathBuf) -> Box<dyn TestStatus + 'static>;
@@ -33,6 +53,16 @@ pub trait StatusEmitter: Sync + RefUnwindSafe {
     ) -> Box<dyn Summary>;
 }
 
+impl From<Format> for Box<dyn StatusEmitter> {
+    fn from(format: Format) -> Box<dyn StatusEmitter> {
+        match format {
+            Format::JSON => Box::new(JSON::new()),
+            Format::Pretty => Box::new(Text::verbose()),
+            Format::Terse => Box::new(Text::quiet()),
+        }
+    }
+}
+
 /// Some configuration options for revisions
 #[derive(Debug, Clone, Copy)]
 pub enum RevisionStyle {
@@ -52,7 +82,7 @@ pub trait TestStatus: Send + Sync + RefUnwindSafe {
     fn for_path(&self, path: &Path) -> Box<dyn TestStatus>;
 
     /// Invoked before each failed test prints its errors along with a drop guard that can
-    /// gets invoked afterwards.
+    /// get invoked afterwards.
     fn failed_test<'a>(
         &'a self,
         cmd: &'a str,

src/status_emitter/json.rs

@@ -0,0 +1,188 @@
+use super::RevisionStyle;
+use super::StatusEmitter;
+use super::Summary;
+use super::TestStatus;
+use crate::test_result::TestResult;
+use crate::TestOk;
+
+use std::boxed::Box;
+use std::fmt::Debug;
+use std::path::{Path, PathBuf};
+
+use bstr::ByteSlice;
+
+// MAINTENANCE REGION START
+
+// When integrating with a new libtest version, update all emit_xxx functions.
+
+fn emit_suite_end(failed: usize, filtered_out: usize, ignored: usize, passed: usize, status: &str) {
+    // Adapted from test::formatters::json::write_run_finish().
+    println!(
+        r#"{{ "type": "suite", "event": "{status}", "passed": {passed}, "failed": {failed}, "ignored": {ignored}, "measured": 0, "filtered_out": {filtered_out} }}"#
+    );
+}
+
+fn emit_suite_start() {
+    // Adapted from test::formatters::json::write_run_start().
+    println!(r#"{{ "type": "suite", "event": "started" }}"#);
+}
+
+fn emit_test_end(name: &String, revision: &String, path: &Path, status: &str, diags: &str) {
+    let displayed_path = path.display();
+    let stdout = if diags.is_empty() {
+        String::new()
+    } else {
+        let triaged_diags = serde_json::to_string(diags).unwrap();
+        format!(r#", "stdout": {triaged_diags}"#)
+    };
+
+    // Adapted from test::formatters::json::write_event().
+    println!(
+        r#"{{ "type": "test", "event": "{status}", "name": "{name} ({revision}) - {displayed_path}"{stdout} }}"#
+    );
+}
+
+fn emit_test_start(name: &String, revision: &String, path: &Path) {
+    let displayed_path = path.display();
+
+    // Adapted from test::formatters::json::write_test_start().
+    println!(
+        r#"{{ "type": "test", "event": "started", "name": "{name} ({revision}) - {displayed_path}" }}"#
+    );
+}
+
+// MAINTENANCE REGION END
+
+/// A JSON output emitter.
+#[derive(Clone)]
+pub struct JSON {}
+
+impl JSON {
+    /// Create a new instance of a JSON output emitter.
+    pub fn new() -> Self {
+        emit_suite_start();
+
+        JSON {}
+    }
+}
+
+impl Default for JSON {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+impl StatusEmitter for JSON {
+    /// Create a report about the entire test run at the end.
+    fn finalize(
+        &self,
+        failed: usize,
+        succeeded: usize,
+        ignored: usize,
+        filtered: usize,
+        aborted: bool,
+    ) -> Box<dyn Summary> {
+        let status = if aborted || failed > 0 {
+            "failed"
+        } else {
+            "ok"
+        };
+
+        emit_suite_end(failed, filtered, ignored, succeeded, status);
+
+        Box::new(())
+    }
+
+    /// Invoked the moment we know a test will later be run.
+    /// Emits a JSON start event.
+    fn register_test(&self, path: PathBuf) -> Box<dyn TestStatus + 'static> {
+        let name = path.to_str().unwrap().to_string();
+        let revision = String::new();
+
+        emit_test_start(&name, &revision, &path);
+
+        Box::new(JSONStatus {
+            name,
+            path,
+            revision: String::new(),
+            style: RevisionStyle::Show,
+        })
+    }
+}
+
+/// Information about a specific test run.
+pub struct JSONStatus {
+    name: String,
+    path: PathBuf,
+    revision: String,
+    style: RevisionStyle,
+}
+
+impl TestStatus for JSONStatus {
+    /// A test has finished, handle the result immediately.
+    fn done(&self, result: &TestResult, aborted: bool) {
+        let status = if aborted {
+            "timeout"
+        } else {
+            match result {
+                Ok(TestOk::Ignored) => "ignored",
+                Ok(TestOk::Ok) => "ok",
+                Err(_) => "failed",
+            }
+        };
+        let diags = if let Err(errored) = result {
+            let command = errored.command.as_str();
+            let stdout = errored.stderr.to_str_lossy();
+            let stderr = errored.stdout.to_str_lossy();
+
+            format!(r#"command: <{command}> stdout: <{stdout}> stderr: <{stderr}>"#)
+        } else {
+            String::new()
+        };
+
+        emit_test_end(&self.name, &self.revision, self.path(), status, &diags);
+    }
+
+    /// Invoked before each failed test prints its errors along with a drop guard that can
+    /// get invoked afterwards.
+    fn failed_test<'a>(
+        &'a self,
+        _cmd: &'a str,
+        _stderr: &'a [u8],
+        _stdout: &'a [u8],
+    ) -> Box<dyn Debug + 'a> {
+        Box::new(())
+    }
+
+    /// Create a copy of this test for a new path.
+    fn for_path(&self, path: &Path) -> Box<dyn TestStatus> {
+        let status = JSONStatus {
+            name: self.name.clone(),
+            path: path.to_path_buf(),
+            revision: self.revision.clone(),
+            style: self.style,
+        };
+        Box::new(status)
+    }
+
+    /// Create a copy of this test for a new revision.
+    fn for_revision(&self, revision: &str, style: RevisionStyle) -> Box<dyn TestStatus> {
+        let status = JSONStatus {
+            name: self.name.clone(),
+            path: self.path.clone(),
+            revision: revision.to_owned(),
+            style,
+        };
+        Box::new(status)
+    }
+
+    /// The path of the test file.
+    fn path(&self) -> &Path {
+        &self.path
+    }
+
+    /// The revision, usually an empty string.
+    fn revision(&self) -> &str {
+        &self.revision
+    }
+}

src/status_emitter/text.rs

@@ -11,7 +11,6 @@ use crate::test_result::TestOk;
 use crate::test_result::TestResult;
 use crate::Error;
 use crate::Errors;
-use crate::Format;
 use annotate_snippets::Renderer;
 use annotate_snippets::Snippet;
 use colored::Colorize;
@@ -341,15 +340,6 @@ impl Text {
     }
 }
 
-impl From<Format> for Text {
-    fn from(format: Format) -> Self {
-        match format {
-            Format::Terse => Text::quiet(),
-            Format::Pretty => Text::verbose(),
-        }
-    }
-}
-
 struct TextTest {
     text: Text,
     #[cfg(feature = "indicatif")]

tests/integration.rs

@@ -97,7 +97,7 @@ fn main() -> Result<()> {
     config.stdout_filter(r#"exit status: 0xc0000409"#, "signal: 6 (SIGABRT)");
     config.filter("\"--target=[^\"]+\"", "");
 
-    let text = ui_test::status_emitter::Text::from(args.format);
+    let emitter: Box<dyn ui_test::status_emitter::StatusEmitter> = args.format.into();
 
     let mut pass_config = config.clone();
     pass_config.comment_defaults.base().exit_status = Some(Spanned::dummy(0)).into();
@@ -139,7 +139,7 @@ fn main() -> Result<()> {
         },
         |_, _| {},
         (
-            text,
+            emitter,
             #[cfg(feature = "gha")]
             ui_test::status_emitter::Gha::<true> {
                 name: "integration tests".into(),

tests/integrations/basic-fail/Cargo.stderr

@@ -1,5 +1,13 @@
 Error: tests failed
 
+Location:
+    $DIR/src/lib.rs:LL:CC
+error: test failed, to rerun pass `--test json`
+
+Caused by:
+  process didn't exit successfully: `$DIR/target/ui/1/tests/integrations/basic-fail/debug/deps/json-HASH` (exit status: 1)
+Error: tests failed
+
 Location:
     $DIR/src/lib.rs:LL:CC
 error: test failed, to rerun pass `--test ui_tests`
@@ -32,7 +40,8 @@ error: test failed, to rerun pass `--test ui_tests_invalid_program2`
 
 Caused by:
   process didn't exit successfully: `$DIR/target/ui/1/tests/integrations/basic-fail/debug/deps/ui_tests_invalid_program2-HASH` (exit status: 1)
-error: 4 targets failed:
+error: 5 targets failed:
+    `--test json`
     `--test ui_tests`
     `--test ui_tests_diff_only`
     `--test ui_tests_invalid_program`