From b3b1d43d8eb2a29c5935ce896580f62be3231bfe Mon Sep 17 00:00:00 2001 From: Benjamin Lee Date: Fri, 21 Jun 2024 12:02:27 -0700 Subject: [PATCH] write complement test result summary to a tsv file --- xtask/src/complement.rs | 50 +++++++++++++++++++--- xtask/src/complement/summary.rs | 50 ++++++++++++++++++++++ xtask/src/complement/test2json.rs | 69 +++++++++++++++++++++++++------ 3 files changed, 151 insertions(+), 18 deletions(-) create mode 100644 xtask/src/complement/summary.rs diff --git a/xtask/src/complement.rs b/xtask/src/complement.rs index 78ac6247..4b9b55a2 100644 --- a/xtask/src/complement.rs +++ b/xtask/src/complement.rs @@ -1,9 +1,13 @@ -use std::path::PathBuf; +use std::{ + fs::{self}, + path::{Path, PathBuf}, +}; -use miette::{IntoDiagnostic, Result, WrapErr}; +use miette::{miette, IntoDiagnostic, Result, WrapErr}; use xshell::{cmd, Shell}; mod docker; +mod summary; mod test2json; use self::{ @@ -12,23 +16,59 @@ use self::{ }; #[derive(clap::Args)] -pub(crate) struct Args; +pub(crate) struct Args { + /// Directory to write test results + /// + /// This directory will be created automatically, but it must be empty. + /// If it exists and is not empty, an error will be returned. + #[clap(short, long)] + out: PathBuf, +} #[allow(clippy::needless_pass_by_value)] -pub(crate) fn main(_args: Args) -> Result<()> { +pub(crate) fn main(args: Args) -> Result<()> { let sh = Shell::new().unwrap(); let toplevel = get_toplevel_path(&sh) .wrap_err("failed to determine repository root directory")?; + create_out_dir(&args.out).wrap_err_with(|| { + format!("error initializing output directory {:?}", args.out) + })?; let docker_image = load_docker_image(&sh, &toplevel).wrap_err( "failed to build and load complement-grapevine docker image", )?; let test_count = count_complement_tests(&sh, &docker_image) .wrap_err("failed to determine total complement test count")?; - run_complement(&sh, &docker_image, test_count) + run_complement(&sh, &args.out, &docker_image, test_count) .wrap_err("failed to run complement tests")?; Ok(()) } +/// Ensures that output directory exists and is empty +/// +/// If the directory does not exist, it will be created. If it is not empty, an +/// error will be returned. +/// +/// We have no protection against concurrent programs modifying the contents of +/// the directory while the complement wrapper tool is running. +fn create_out_dir(out: &Path) -> Result<()> { + fs::create_dir_all(out) + .into_diagnostic() + .wrap_err("error creating output directory")?; + let mut entries = fs::read_dir(out) + .into_diagnostic() + .wrap_err("error checking current contents of output directory")?; + if entries.next().is_some() { + return Err(miette!( + "output directory is not empty. Refusing to run, instead of \ + possibly overwriting existing files." + )); + } + fs::create_dir(out.join("logs")) + .into_diagnostic() + .wrap_err("error creating logs subdirectory in output directory")?; + Ok(()) +} + /// Returns the path to the repository root fn get_toplevel_path(sh: &Shell) -> Result { let path = diff --git a/xtask/src/complement/summary.rs b/xtask/src/complement/summary.rs new file mode 100644 index 00000000..d95f819f --- /dev/null +++ b/xtask/src/complement/summary.rs @@ -0,0 +1,50 @@ +//! Functions for working with the `summary.tsv` files emitted by the complement +//! wrapper. +//! +//! This file is a TSV containing test names and results for each test in a +//! complement run. + +use std::{ + collections::BTreeMap, + io::{BufWriter, Write}, +}; + +use miette::{IntoDiagnostic, Result}; + +use super::test2json::TestResult; + +pub(crate) type TestResults = BTreeMap; + +/// Escape a string value for use in a TSV file. +/// +/// According to the [tsv spec][1], the only characters that need to be escaped +/// are `\n`, `\t`, `\r`, and `\`. +/// +/// [1]: https://www.loc.gov/preservation/digital/formats/fdd/fdd000533.shtml +fn escape_tsv_value(value: &str) -> String { + value + .replace('\\', "\\\\") + .replace('\n', "\\n") + .replace('\t', "\\t") + .replace('\r', "\\r") +} + +/// Write a test result summary to a writer. +pub(crate) fn write_summary( + w: &mut BufWriter, + summary: &TestResults, +) -> Result<()> { + // Write header line + writeln!(w, "test\tresult").into_diagnostic()?; + // Write rows + for (test, result) in summary { + writeln!( + w, + "{}\t{}", + escape_tsv_value(test), + escape_tsv_value(&result.to_string()) + ) + .into_diagnostic()?; + } + Ok(()) +} diff --git a/xtask/src/complement/test2json.rs b/xtask/src/complement/test2json.rs index bfc80aaa..121aebbf 100644 --- a/xtask/src/complement/test2json.rs +++ b/xtask/src/complement/test2json.rs @@ -3,7 +3,10 @@ //! [test2json]: https://pkg.go.dev/cmd/test2json@go1.22.4 use std::{ - io::{BufRead, BufReader}, + collections::BTreeMap, + fs::File, + io::{BufRead, BufReader, BufWriter, Seek, SeekFrom, Write}, + path::Path, process::{Command, Stdio}, time::Duration, }; @@ -14,6 +17,8 @@ use serde::Deserialize; use strum::Display; use xshell::{cmd, Shell}; +use super::summary::{write_summary, TestResults}; + /// Returns the total number of complement tests that will be run /// /// This is only able to count toplevel tests, and will not included subtests @@ -35,6 +40,7 @@ pub(crate) fn count_complement_tests( /// Runs complement test suite pub(crate) fn run_complement( sh: &Shell, + out: &Path, docker_image: &str, test_count: u64, ) -> Result<()> { @@ -56,12 +62,12 @@ pub(crate) fn run_complement( .expect("child process spawned with piped stdout should have stdout"); let lines = BufReader::new(stdout).lines(); - let mut ctx = TestContext::new(test_count); + let mut ctx = TestContext::new(out, test_count)?; for line in lines { let line = line .into_diagnostic() .wrap_err("error reading output from complement process")?; - ctx.handle_line(&line); + ctx.handle_line(&line)?; } Ok(()) @@ -95,7 +101,7 @@ enum GoTestEvent { #[derive(Copy, Clone, Display, Debug)] #[strum(serialize_all = "UPPERCASE")] -enum TestResult { +pub(crate) enum TestResult { Pass, Fail, Skip, @@ -106,6 +112,13 @@ struct TestContext { pass_count: u64, fail_count: u64, skip_count: u64, + // We do not need a specific method to flush this before dropping + // `TestContext`, because the file is only written from the + // `update_summary_file` method. This method always calls flush on + // a non-error path, and the file is left in an inconsistent state on an + // error anyway. + summary_file: BufWriter, + results: TestResults, } /// Returns a string to use for displaying a test name @@ -132,7 +145,7 @@ fn test_is_toplevel(test: &str) -> bool { } impl TestContext { - fn new(test_count: u64) -> TestContext { + fn new(out: &Path, test_count: u64) -> Result { // TODO: figure out how to display ETA without it fluctuating wildly. let style = ProgressStyle::with_template( "({msg}) {pos}/{len} [{elapsed}] {wide_bar}", @@ -141,14 +154,23 @@ impl TestContext { .progress_chars("##-"); let pb = ProgressBar::new(test_count).with_style(style); pb.enable_steady_tick(Duration::from_secs(1)); + + let summary_file = File::create(out.join("summary.tsv")) + .into_diagnostic() + .wrap_err("failed to create summary file in output dir")?; + let summary_file = BufWriter::new(summary_file); + let ctx = TestContext { pb, pass_count: 0, fail_count: 0, skip_count: 0, + summary_file, + results: BTreeMap::new(), }; + ctx.update_progress(); - ctx + Ok(ctx) } fn update_progress(&self) { @@ -160,8 +182,25 @@ impl TestContext { )); } - fn handle_test_result(&mut self, test: &str, result: TestResult) { + fn update_summary_file(&mut self) -> Result<()> { + // Truncate the file to clear existing contents + self.summary_file + .get_mut() + .seek(SeekFrom::Start(0)) + .into_diagnostic()?; + self.summary_file.get_mut().set_len(0).into_diagnostic()?; + write_summary(&mut self.summary_file, &self.results)?; + self.summary_file.flush().into_diagnostic()?; + Ok(()) + } + + fn handle_test_result( + &mut self, + test: &str, + result: TestResult, + ) -> Result<()> { self.pb.println(format!("=== {result}\t{test}")); + self.results.insert(test.to_owned(), result); // 'complement.test -test.list' is only able to count toplevel tests // ahead-of-time, so we don't include subtests in the pass/fail/skip // counts. @@ -173,9 +212,11 @@ impl TestContext { } self.update_progress(); } + self.update_summary_file().wrap_err("error writing summary file")?; + Ok(()) } - fn handle_event(&mut self, event: GoTestEvent) { + fn handle_event(&mut self, event: GoTestEvent) -> Result<()> { match event { GoTestEvent::OtherAction => (), GoTestEvent::Run { @@ -186,25 +227,26 @@ impl TestContext { GoTestEvent::Pass { test, } => { - self.handle_test_result(test_str(&test), TestResult::Pass); + self.handle_test_result(test_str(&test), TestResult::Pass)?; } GoTestEvent::Fail { test, } => { - self.handle_test_result(test_str(&test), TestResult::Fail); + self.handle_test_result(test_str(&test), TestResult::Fail)?; } GoTestEvent::Skip { test, } => { - self.handle_test_result(test_str(&test), TestResult::Skip); + self.handle_test_result(test_str(&test), TestResult::Skip)?; } } + Ok(()) } /// Processes a line of output from `test2json` - fn handle_line(&mut self, line: &str) { + fn handle_line(&mut self, line: &str) -> Result<()> { match serde_json::from_str(line) { - Ok(event) => self.handle_event(event), + Ok(event) => self.handle_event(event)?, Err(e) => { let label = LabeledSpan::at_offset(e.column() - 1, "error here"); @@ -217,5 +259,6 @@ impl TestContext { eprintln!("{report:?}"); } }; + Ok(()) } }