write complement test result summary to a tsv file

xtask/src/complement.rs

@@ -1,9 +1,13 @@
-use std::path::PathBuf;
+use std::{
+    fs::{self},
+    path::{Path, PathBuf},
+};
 
-use miette::{IntoDiagnostic, Result, WrapErr};
+use miette::{miette, IntoDiagnostic, Result, WrapErr};
 use xshell::{cmd, Shell};
 
 mod docker;
+mod summary;
 mod test2json;
 
 use self::{
@@ -12,23 +16,59 @@ use self::{
 };
 
 #[derive(clap::Args)]
-pub(crate) struct Args;
+pub(crate) struct Args {
+    /// Directory to write test results
+    ///
+    /// This directory will be created automatically, but it must be empty.
+    /// If it exists and is not empty, an error will be returned.
+    #[clap(short, long)]
+    out: PathBuf,
+}
 
 #[allow(clippy::needless_pass_by_value)]
-pub(crate) fn main(_args: Args) -> Result<()> {
+pub(crate) fn main(args: Args) -> Result<()> {
     let sh = Shell::new().unwrap();
     let toplevel = get_toplevel_path(&sh)
         .wrap_err("failed to determine repository root directory")?;
+    create_out_dir(&args.out).wrap_err_with(|| {
+        format!("error initializing output directory {:?}", args.out)
+    })?;
     let docker_image = load_docker_image(&sh, &toplevel).wrap_err(
         "failed to build and load complement-grapevine docker image",
     )?;
     let test_count = count_complement_tests(&sh, &docker_image)
         .wrap_err("failed to determine total complement test count")?;
-    run_complement(&sh, &docker_image, test_count)
+    run_complement(&sh, &args.out, &docker_image, test_count)
         .wrap_err("failed to run complement tests")?;
     Ok(())
 }
 
+/// Ensures that output directory exists and is empty
+///
+/// If the directory does not exist, it will be created. If it is not empty, an
+/// error will be returned.
+///
+/// We have no protection against concurrent programs modifying the contents of
+/// the directory while the complement wrapper tool is running.
+fn create_out_dir(out: &Path) -> Result<()> {
+    fs::create_dir_all(out)
+        .into_diagnostic()
+        .wrap_err("error creating output directory")?;
+    let mut entries = fs::read_dir(out)
+        .into_diagnostic()
+        .wrap_err("error checking current contents of output directory")?;
+    if entries.next().is_some() {
+        return Err(miette!(
+            "output directory is not empty. Refusing to run, instead of \
+             possibly overwriting existing files."
+        ));
+    }
+    fs::create_dir(out.join("logs"))
+        .into_diagnostic()
+        .wrap_err("error creating logs subdirectory in output directory")?;
+    Ok(())
+}
+
 /// Returns the path to the repository root
 fn get_toplevel_path(sh: &Shell) -> Result<PathBuf> {
     let path =

xtask/src/complement/summary.rs

@@ -0,0 +1,50 @@
+//! Functions for working with the `summary.tsv` files emitted by the complement
+//! wrapper.
+//!
+//! This file is a TSV containing test names and results for each test in a
+//! complement run.
+
+use std::{
+    collections::BTreeMap,
+    io::{BufWriter, Write},
+};
+
+use miette::{IntoDiagnostic, Result};
+
+use super::test2json::TestResult;
+
+pub(crate) type TestResults = BTreeMap<String, TestResult>;
+
+/// Escape a string value for use in a TSV file.
+///
+/// According to the [tsv spec][1], the only characters that need to be escaped
+/// are `\n`, `\t`, `\r`, and `\`.
+///
+/// [1]: https://www.loc.gov/preservation/digital/formats/fdd/fdd000533.shtml
+fn escape_tsv_value(value: &str) -> String {
+    value
+        .replace('\\', "\\\\")
+        .replace('\n', "\\n")
+        .replace('\t', "\\t")
+        .replace('\r', "\\r")
+}
+
+/// Write a test result summary to a writer.
+pub(crate) fn write_summary<W: Write>(
+    w: &mut BufWriter<W>,
+    summary: &TestResults,
+) -> Result<()> {
+    // Write header line
+    writeln!(w, "test\tresult").into_diagnostic()?;
+    // Write rows
+    for (test, result) in summary {
+        writeln!(
+            w,
+            "{}\t{}",
+            escape_tsv_value(test),
+            escape_tsv_value(&result.to_string())
+        )
+        .into_diagnostic()?;
+    }
+    Ok(())
+}

xtask/src/complement/test2json.rs

@@ -3,7 +3,10 @@
 //! [test2json]: https://pkg.go.dev/cmd/test2json@go1.22.4
 
 use std::{
-    io::{BufRead, BufReader},
+    collections::BTreeMap,
+    fs::File,
+    io::{BufRead, BufReader, BufWriter, Seek, SeekFrom, Write},
+    path::Path,
     process::{Command, Stdio},
     time::Duration,
 };
@@ -14,6 +17,8 @@ use serde::Deserialize;
 use strum::Display;
 use xshell::{cmd, Shell};
 
+use super::summary::{write_summary, TestResults};
+
 /// Returns the total number of complement tests that will be run
 ///
 /// This is only able to count toplevel tests, and will not included subtests
@@ -35,6 +40,7 @@ pub(crate) fn count_complement_tests(
 /// Runs complement test suite
 pub(crate) fn run_complement(
     sh: &Shell,
+    out: &Path,
     docker_image: &str,
     test_count: u64,
 ) -> Result<()> {
@@ -56,12 +62,12 @@ pub(crate) fn run_complement(
         .expect("child process spawned with piped stdout should have stdout");
     let lines = BufReader::new(stdout).lines();
 
-    let mut ctx = TestContext::new(test_count);
+    let mut ctx = TestContext::new(out, test_count)?;
     for line in lines {
         let line = line
             .into_diagnostic()
             .wrap_err("error reading output from complement process")?;
-        ctx.handle_line(&line);
+        ctx.handle_line(&line)?;
     }
 
     Ok(())
@@ -95,7 +101,7 @@ enum GoTestEvent {
 
 #[derive(Copy, Clone, Display, Debug)]
 #[strum(serialize_all = "UPPERCASE")]
-enum TestResult {
+pub(crate) enum TestResult {
     Pass,
     Fail,
     Skip,
@@ -106,6 +112,13 @@ struct TestContext {
     pass_count: u64,
     fail_count: u64,
     skip_count: u64,
+    // We do not need a specific method to flush this before dropping
+    // `TestContext`, because the file is only written from the
+    // `update_summary_file` method. This method always calls flush on
+    // a non-error path, and the file is left in an inconsistent state on an
+    // error anyway.
+    summary_file: BufWriter<File>,
+    results: TestResults,
 }
 
 /// Returns a string to use for displaying a test name
@@ -132,7 +145,7 @@ fn test_is_toplevel(test: &str) -> bool {
 }
 
 impl TestContext {
-    fn new(test_count: u64) -> TestContext {
+    fn new(out: &Path, test_count: u64) -> Result<TestContext> {
         // TODO: figure out how to display ETA without it fluctuating wildly.
         let style = ProgressStyle::with_template(
             "({msg})  {pos}/{len}  [{elapsed}] {wide_bar}",
@@ -141,14 +154,23 @@ impl TestContext {
         .progress_chars("##-");
         let pb = ProgressBar::new(test_count).with_style(style);
         pb.enable_steady_tick(Duration::from_secs(1));
+
+        let summary_file = File::create(out.join("summary.tsv"))
+            .into_diagnostic()
+            .wrap_err("failed to create summary file in output dir")?;
+        let summary_file = BufWriter::new(summary_file);
+
         let ctx = TestContext {
             pb,
             pass_count: 0,
             fail_count: 0,
             skip_count: 0,
+            summary_file,
+            results: BTreeMap::new(),
         };
+
         ctx.update_progress();
-        ctx
+        Ok(ctx)
     }
 
     fn update_progress(&self) {
@@ -160,8 +182,25 @@ impl TestContext {
         ));
     }
 
-    fn handle_test_result(&mut self, test: &str, result: TestResult) {
+    fn update_summary_file(&mut self) -> Result<()> {
+        // Truncate the file to clear existing contents
+        self.summary_file
+            .get_mut()
+            .seek(SeekFrom::Start(0))
+            .into_diagnostic()?;
+        self.summary_file.get_mut().set_len(0).into_diagnostic()?;
+        write_summary(&mut self.summary_file, &self.results)?;
+        self.summary_file.flush().into_diagnostic()?;
+        Ok(())
+    }
+
+    fn handle_test_result(
+        &mut self,
+        test: &str,
+        result: TestResult,
+    ) -> Result<()> {
         self.pb.println(format!("=== {result}\t{test}"));
+        self.results.insert(test.to_owned(), result);
         // 'complement.test -test.list' is only able to count toplevel tests
         // ahead-of-time, so we don't include subtests in the pass/fail/skip
         // counts.
@@ -173,9 +212,11 @@ impl TestContext {
             }
             self.update_progress();
         }
+        self.update_summary_file().wrap_err("error writing summary file")?;
+        Ok(())
     }
 
-    fn handle_event(&mut self, event: GoTestEvent) {
+    fn handle_event(&mut self, event: GoTestEvent) -> Result<()> {
         match event {
             GoTestEvent::OtherAction => (),
             GoTestEvent::Run {
@@ -186,25 +227,26 @@ impl TestContext {
             GoTestEvent::Pass {
                 test,
             } => {
-                self.handle_test_result(test_str(&test), TestResult::Pass);
+                self.handle_test_result(test_str(&test), TestResult::Pass)?;
             }
             GoTestEvent::Fail {
                 test,
             } => {
-                self.handle_test_result(test_str(&test), TestResult::Fail);
+                self.handle_test_result(test_str(&test), TestResult::Fail)?;
             }
             GoTestEvent::Skip {
                 test,
             } => {
-                self.handle_test_result(test_str(&test), TestResult::Skip);
+                self.handle_test_result(test_str(&test), TestResult::Skip)?;
             }
         }
+        Ok(())
     }
 
     /// Processes a line of output from `test2json`
-    fn handle_line(&mut self, line: &str) {
+    fn handle_line(&mut self, line: &str) -> Result<()> {
         match serde_json::from_str(line) {
-            Ok(event) => self.handle_event(event),
+            Ok(event) => self.handle_event(event)?,
             Err(e) => {
                 let label =
                     LabeledSpan::at_offset(e.column() - 1, "error here");
@@ -217,5 +259,6 @@ impl TestContext {
                 eprintln!("{report:?}");
             }
         };
+        Ok(())
     }
 }