From 8eab6eea204d09df0f7bb0886c654f41db97f3cd Mon Sep 17 00:00:00 2001 From: Benjamin Lee Date: Fri, 21 Jun 2024 16:33:51 -0700 Subject: [PATCH] compare complement test results to a baseline One thing that might be neat in the future is noticing differing results while the tests are still running, and modifying the log messages to indicate them. I can imagine situations where you would want to abort the test run immediately after seeing the first regression. --- complement-baseline.tsv | 220 +++++++++++++++++++++++++++ xtask/src/complement.rs | 21 ++- xtask/src/complement/summary.rs | 238 +++++++++++++++++++++++++++++- xtask/src/complement/test2json.rs | 8 +- 4 files changed, 481 insertions(+), 6 deletions(-) create mode 100644 complement-baseline.tsv diff --git a/complement-baseline.tsv b/complement-baseline.tsv new file mode 100644 index 00000000..cc489de8 --- /dev/null +++ b/complement-baseline.tsv @@ -0,0 +1,220 @@ +test result +GLOBAL FAIL +TestACLs PASS +TestBannedUserCannotSendJoin FAIL +TestCannotSendKnockViaSendKnockInMSC3787Room FAIL +TestCannotSendKnockViaSendKnockInMSC3787Room/event_with_mismatched_state_key FAIL +TestCannotSendKnockViaSendKnockInMSC3787Room/invite_event FAIL +TestCannotSendKnockViaSendKnockInMSC3787Room/join_event FAIL +TestCannotSendKnockViaSendKnockInMSC3787Room/leave_event FAIL +TestCannotSendKnockViaSendKnockInMSC3787Room/non-state_membership_event FAIL +TestCannotSendKnockViaSendKnockInMSC3787Room/regular_event FAIL +TestCannotSendNonJoinViaSendJoinV1 FAIL +TestCannotSendNonJoinViaSendJoinV1/event_with_mismatched_state_key PASS +TestCannotSendNonJoinViaSendJoinV1/invite_event PASS +TestCannotSendNonJoinViaSendJoinV1/knock_event PASS +TestCannotSendNonJoinViaSendJoinV1/leave_event FAIL +TestCannotSendNonJoinViaSendJoinV1/non-state_membership_event PASS +TestCannotSendNonJoinViaSendJoinV1/regular_event FAIL +TestCannotSendNonJoinViaSendJoinV2 FAIL +TestCannotSendNonJoinViaSendJoinV2/event_with_mismatched_state_key PASS +TestCannotSendNonJoinViaSendJoinV2/invite_event PASS +TestCannotSendNonJoinViaSendJoinV2/knock_event PASS +TestCannotSendNonJoinViaSendJoinV2/leave_event FAIL +TestCannotSendNonJoinViaSendJoinV2/non-state_membership_event PASS +TestCannotSendNonJoinViaSendJoinV2/regular_event FAIL +TestCannotSendNonKnockViaSendKnock FAIL +TestCannotSendNonLeaveViaSendLeaveV1 FAIL +TestCannotSendNonLeaveViaSendLeaveV1/event_with_mismatched_state_key FAIL +TestCannotSendNonLeaveViaSendLeaveV1/invite_event FAIL +TestCannotSendNonLeaveViaSendLeaveV1/join_event FAIL +TestCannotSendNonLeaveViaSendLeaveV1/knock_event FAIL +TestCannotSendNonLeaveViaSendLeaveV1/non-state_membership_event FAIL +TestCannotSendNonLeaveViaSendLeaveV1/regular_event FAIL +TestCannotSendNonLeaveViaSendLeaveV2 FAIL +TestCannotSendNonLeaveViaSendLeaveV2/event_with_mismatched_state_key FAIL +TestCannotSendNonLeaveViaSendLeaveV2/invite_event FAIL +TestCannotSendNonLeaveViaSendLeaveV2/join_event FAIL +TestCannotSendNonLeaveViaSendLeaveV2/knock_event FAIL +TestCannotSendNonLeaveViaSendLeaveV2/non-state_membership_event FAIL +TestCannotSendNonLeaveViaSendLeaveV2/regular_event FAIL +TestClientSpacesSummary FAIL +TestClientSpacesSummary/max_depth FAIL +TestClientSpacesSummary/pagination FAIL +TestClientSpacesSummary/query_whole_graph FAIL +TestClientSpacesSummary/redact_link FAIL +TestClientSpacesSummary/suggested_only FAIL +TestClientSpacesSummaryJoinRules FAIL +TestContentMediaV1 PASS +TestDeviceListsUpdateOverFederation FAIL +TestDeviceListsUpdateOverFederation/good_connectivity FAIL +TestDeviceListsUpdateOverFederation/interrupted_connectivity FAIL +TestDeviceListsUpdateOverFederation/stopped_server FAIL +TestDeviceListsUpdateOverFederationOnRoomJoin FAIL +TestEventAuth FAIL +TestFederatedClientSpaces FAIL +TestFederationKeyUploadQuery FAIL +TestFederationKeyUploadQuery/Can_claim_remote_one_time_key_using_POST FAIL +TestFederationKeyUploadQuery/Can_query_remote_device_keys_using_POST FAIL +TestFederationRedactSendsWithoutEvent PASS +TestFederationRejectInvite FAIL +TestFederationRoomsInvite FAIL +TestFederationRoomsInvite/Parallel FAIL +TestFederationRoomsInvite/Parallel/Invited_user_can_reject_invite_over_federation FAIL +TestFederationRoomsInvite/Parallel/Invited_user_can_reject_invite_over_federation_for_empty_room PASS +TestFederationRoomsInvite/Parallel/Invited_user_can_reject_invite_over_federation_several_times FAIL +TestFederationRoomsInvite/Parallel/Invited_user_has_'is_direct'_flag_in_prev_content_after_joining PASS +TestFederationRoomsInvite/Parallel/Remote_invited_user_can_see_room_metadata PASS +TestFederationThumbnail PASS +TestGetMissingEventsGapFilling FAIL +TestInboundCanReturnMissingEvents FAIL +TestInboundCanReturnMissingEvents/Inbound_federation_can_return_missing_events_for_invited_visibility FAIL +TestInboundCanReturnMissingEvents/Inbound_federation_can_return_missing_events_for_joined_visibility FAIL +TestInboundCanReturnMissingEvents/Inbound_federation_can_return_missing_events_for_shared_visibility FAIL +TestInboundCanReturnMissingEvents/Inbound_federation_can_return_missing_events_for_world_readable_visibility FAIL +TestInboundFederationKeys PASS +TestInboundFederationProfile PASS +TestInboundFederationProfile/Inbound_federation_can_query_profile_data PASS +TestInboundFederationProfile/Non-numeric_ports_in_server_names_are_rejected PASS +TestInboundFederationRejectsEventsWithRejectedAuthEvents FAIL +TestIsDirectFlagFederation PASS +TestIsDirectFlagLocal PASS +TestJoinFederatedRoomFailOver PASS +TestJoinFederatedRoomFromApplicationServiceBridgeUser FAIL +TestJoinFederatedRoomFromApplicationServiceBridgeUser/join_remote_federated_room_as_application_service_user FAIL +TestJoinFederatedRoomWithUnverifiableEvents PASS +TestJoinFederatedRoomWithUnverifiableEvents//send_join_response_missing_signatures_shouldn't_block_room_join PASS +TestJoinFederatedRoomWithUnverifiableEvents//send_join_response_with_bad_signatures_shouldn't_block_room_join PASS +TestJoinFederatedRoomWithUnverifiableEvents//send_join_response_with_state_with_unverifiable_auth_events_shouldn't_block_room_join PASS +TestJoinFederatedRoomWithUnverifiableEvents//send_join_response_with_unobtainable_keys_shouldn't_block_room_join PASS +TestJoinViaRoomIDAndServerName PASS +TestJumpToDateEndpoint FAIL +TestJumpToDateEndpoint/parallel FAIL +TestJumpToDateEndpoint/parallel/federation FAIL +TestJumpToDateEndpoint/parallel/federation/can_paginate_after_getting_remote_event_from_timestamp_to_event_endpoint FAIL +TestJumpToDateEndpoint/parallel/federation/looking_backwards,_should_be_able_to_find_event_that_was_sent_before_we_joined FAIL +TestJumpToDateEndpoint/parallel/federation/looking_forwards,_should_be_able_to_find_event_that_was_sent_before_we_joined FAIL +TestJumpToDateEndpoint/parallel/federation/when_looking_backwards_before_the_room_was_created,_should_be_able_to_find_event_that_was_imported FAIL +TestJumpToDateEndpoint/parallel/should_find_event_after_given_timestmap FAIL +TestJumpToDateEndpoint/parallel/should_find_event_before_given_timestmap FAIL +TestJumpToDateEndpoint/parallel/should_find_next_event_topologically_after_given_timestmap_when_all_message_timestamps_are_the_same FAIL +TestJumpToDateEndpoint/parallel/should_find_next_event_topologically_before_given_timestamp_when_all_message_timestamps_are_the_same FAIL +TestJumpToDateEndpoint/parallel/should_find_nothing_after_the_latest_timestmap PASS +TestJumpToDateEndpoint/parallel/should_find_nothing_before_the_earliest_timestmap PASS +TestJumpToDateEndpoint/parallel/should_not_be_able_to_query_a_private_room_you_are_not_a_member_of FAIL +TestJumpToDateEndpoint/parallel/should_not_be_able_to_query_a_public_room_you_are_not_a_member_of FAIL +TestKnockRoomsInPublicRoomsDirectory FAIL +TestKnockRoomsInPublicRoomsDirectoryInMSC3787Room FAIL +TestKnocking FAIL +TestKnockingInMSC3787Room FAIL +TestKnockingInMSC3787Room/A_user_can_knock_on_a_room_without_a_reason FAIL +TestKnockingInMSC3787Room/A_user_can_knock_on_a_room_without_a_reason#01 FAIL +TestKnockingInMSC3787Room/A_user_cannot_knock_on_a_room_they_are_already_in FAIL +TestKnockingInMSC3787Room/A_user_cannot_knock_on_a_room_they_are_already_in#01 FAIL +TestKnockingInMSC3787Room/A_user_cannot_knock_on_a_room_they_are_already_invited_to FAIL +TestKnockingInMSC3787Room/A_user_cannot_knock_on_a_room_they_are_already_invited_to#01 FAIL +TestKnockingInMSC3787Room/A_user_in_the_room_can_accept_a_knock PASS +TestKnockingInMSC3787Room/A_user_in_the_room_can_accept_a_knock#01 PASS +TestKnockingInMSC3787Room/A_user_in_the_room_can_reject_a_knock FAIL +TestKnockingInMSC3787Room/A_user_in_the_room_can_reject_a_knock#01 FAIL +TestKnockingInMSC3787Room/A_user_that_has_already_knocked_is_allowed_to_knock_again_on_the_same_room FAIL +TestKnockingInMSC3787Room/A_user_that_has_already_knocked_is_allowed_to_knock_again_on_the_same_room#01 FAIL +TestKnockingInMSC3787Room/A_user_that_has_knocked_on_a_local_room_can_rescind_their_knock_and_then_knock_again FAIL +TestKnockingInMSC3787Room/A_user_that_is_banned_from_a_room_cannot_knock_on_it FAIL +TestKnockingInMSC3787Room/A_user_that_is_banned_from_a_room_cannot_knock_on_it#01 FAIL +TestKnockingInMSC3787Room/Attempting_to_join_a_room_with_join_rule_'knock'_without_an_invite_should_fail FAIL +TestKnockingInMSC3787Room/Attempting_to_join_a_room_with_join_rule_'knock'_without_an_invite_should_fail#01 FAIL +TestKnockingInMSC3787Room/Change_the_join_rule_of_a_room_from_'invite'_to_'knock' PASS +TestKnockingInMSC3787Room/Change_the_join_rule_of_a_room_from_'invite'_to_'knock'#01 PASS +TestKnockingInMSC3787Room/Knocking_on_a_room_with_a_join_rule_other_than_'knock'_should_fail FAIL +TestKnockingInMSC3787Room/Knocking_on_a_room_with_a_join_rule_other_than_'knock'_should_fail#01 FAIL +TestKnockingInMSC3787Room/Knocking_on_a_room_with_join_rule_'knock'_should_succeed FAIL +TestKnockingInMSC3787Room/Knocking_on_a_room_with_join_rule_'knock'_should_succeed#01 FAIL +TestKnockingInMSC3787Room/Users_in_the_room_see_a_user's_membership_update_when_they_knock FAIL +TestKnockingInMSC3787Room/Users_in_the_room_see_a_user's_membership_update_when_they_knock#01 FAIL +TestLocalPngThumbnail PASS +TestLocalPngThumbnail/test_/_matrix/client/v1/media_endpoint PASS +TestMediaFilenames FAIL +TestMediaFilenames/Parallel FAIL +TestMediaFilenames/Parallel/ASCII FAIL +TestMediaFilenames/Parallel/ASCII/Can_download_file_'ascii' FAIL +TestMediaFilenames/Parallel/ASCII/Can_download_file_'ascii'_over_/_matrix/client/v1/media/download FAIL +TestMediaFilenames/Parallel/ASCII/Can_download_file_'name;with;semicolons' FAIL +TestMediaFilenames/Parallel/ASCII/Can_download_file_'name;with;semicolons'_over_/_matrix/client/v1/media/download FAIL +TestMediaFilenames/Parallel/ASCII/Can_download_file_'name_with_spaces' FAIL +TestMediaFilenames/Parallel/ASCII/Can_download_file_'name_with_spaces'_over_/_matrix/client/v1/media/download FAIL +TestMediaFilenames/Parallel/ASCII/Can_download_specifying_a_different_ASCII_file_name PASS +TestMediaFilenames/Parallel/ASCII/Can_download_specifying_a_different_ASCII_file_name_over__matrix/client/v1/media/download PASS +TestMediaFilenames/Parallel/ASCII/Can_upload_with_ASCII_file_name PASS +TestMediaFilenames/Parallel/Unicode FAIL +TestMediaFilenames/Parallel/Unicode/Can_download_specifying_a_different_Unicode_file_name PASS +TestMediaFilenames/Parallel/Unicode/Can_download_specifying_a_different_Unicode_file_name_over__matrix/client/v1/media/download PASS +TestMediaFilenames/Parallel/Unicode/Can_download_with_Unicode_file_name_locally FAIL +TestMediaFilenames/Parallel/Unicode/Can_download_with_Unicode_file_name_locally_over__matrix/client/v1/media/download FAIL +TestMediaFilenames/Parallel/Unicode/Can_download_with_Unicode_file_name_over_federation FAIL +TestMediaFilenames/Parallel/Unicode/Can_download_with_Unicode_file_name_over_federation_via__matrix/client/v1/media/download FAIL +TestMediaFilenames/Parallel/Unicode/Can_upload_with_Unicode_file_name PASS +TestMediaFilenames/Parallel/Unicode/Will_serve_safe_media_types_as_inline SKIP +TestMediaFilenames/Parallel/Unicode/Will_serve_safe_media_types_as_inline_via__matrix/client/v1/media/download SKIP +TestMediaFilenames/Parallel/Unicode/Will_serve_safe_media_types_with_parameters_as_inline SKIP +TestMediaFilenames/Parallel/Unicode/Will_serve_safe_media_types_with_parameters_as_inline_via__matrix/client/v1/media/download SKIP +TestMediaFilenames/Parallel/Unicode/Will_serve_unsafe_media_types_as_attachments SKIP +TestMediaFilenames/Parallel/Unicode/Will_serve_unsafe_media_types_as_attachments_via__matrix/client/v1/media/download SKIP +TestMediaWithoutFileName FAIL +TestMediaWithoutFileName/parallel FAIL +TestMediaWithoutFileName/parallel/Can_download_without_a_file_name_locally PASS +TestMediaWithoutFileName/parallel/Can_download_without_a_file_name_over_federation FAIL +TestMediaWithoutFileName/parallel/Can_upload_without_a_file_name PASS +TestMediaWithoutFileNameCSMediaV1 PASS +TestMediaWithoutFileNameCSMediaV1/parallel PASS +TestMediaWithoutFileNameCSMediaV1/parallel/Can_download_without_a_file_name_locally PASS +TestMediaWithoutFileNameCSMediaV1/parallel/Can_download_without_a_file_name_over_federation PASS +TestMediaWithoutFileNameCSMediaV1/parallel/Can_upload_without_a_file_name PASS +TestNetworkPartitionOrdering FAIL +TestOutboundFederationIgnoresMissingEventWithBadJSONForRoomVersion6 FAIL +TestOutboundFederationProfile PASS +TestOutboundFederationProfile/Outbound_federation_can_query_profile_data PASS +TestOutboundFederationSend PASS +TestRemoteAliasRequestsUnderstandUnicode PASS +TestRemotePngThumbnail PASS +TestRemotePngThumbnail/test_/_matrix/client/v1/media_endpoint PASS +TestRemotePresence FAIL +TestRemotePresence/Presence_changes_are_also_reported_to_remote_room_members FAIL +TestRemotePresence/Presence_changes_to_UNAVAILABLE_are_reported_to_remote_room_members FAIL +TestRemoteTyping FAIL +TestRestrictedRoomsLocalJoin FAIL +TestRestrictedRoomsLocalJoinInMSC3787Room FAIL +TestRestrictedRoomsLocalJoinInMSC3787Room/Join_should_fail_initially PASS +TestRestrictedRoomsLocalJoinInMSC3787Room/Join_should_fail_when_left_allowed_room FAIL +TestRestrictedRoomsLocalJoinInMSC3787Room/Join_should_fail_with_mangled_join_rules PASS +TestRestrictedRoomsLocalJoinInMSC3787Room/Join_should_succeed_when_invited FAIL +TestRestrictedRoomsLocalJoinInMSC3787Room/Join_should_succeed_when_joined_to_allowed_room PASS +TestRestrictedRoomsRemoteJoin FAIL +TestRestrictedRoomsRemoteJoinFailOver FAIL +TestRestrictedRoomsRemoteJoinFailOverInMSC3787Room FAIL +TestRestrictedRoomsRemoteJoinInMSC3787Room FAIL +TestRestrictedRoomsRemoteJoinInMSC3787Room/Join_should_fail_initially PASS +TestRestrictedRoomsRemoteJoinInMSC3787Room/Join_should_fail_when_left_allowed_room PASS +TestRestrictedRoomsRemoteJoinInMSC3787Room/Join_should_fail_with_mangled_join_rules PASS +TestRestrictedRoomsRemoteJoinInMSC3787Room/Join_should_succeed_when_invited PASS +TestRestrictedRoomsRemoteJoinInMSC3787Room/Join_should_succeed_when_joined_to_allowed_room FAIL +TestRestrictedRoomsRemoteJoinLocalUser FAIL +TestRestrictedRoomsRemoteJoinLocalUserInMSC3787Room FAIL +TestRestrictedRoomsSpacesSummaryFederation FAIL +TestRestrictedRoomsSpacesSummaryLocal FAIL +TestSendJoinPartialStateResponse SKIP +TestSyncOmitsStateChangeOnFilteredEvents PASS +TestToDeviceMessagesOverFederation FAIL +TestToDeviceMessagesOverFederation/good_connectivity PASS +TestToDeviceMessagesOverFederation/interrupted_connectivity FAIL +TestToDeviceMessagesOverFederation/stopped_server FAIL +TestUnbanViaInvite FAIL +TestUnknownEndpoints FAIL +TestUnknownEndpoints/Client-server_endpoints PASS +TestUnknownEndpoints/Key_endpoints FAIL +TestUnknownEndpoints/Media_endpoints PASS +TestUnknownEndpoints/Server-server_endpoints PASS +TestUnknownEndpoints/Unknown_prefix PASS +TestUnrejectRejectedEvents FAIL +TestUserAppearsInChangedDeviceListOnJoinOverFederation PASS +TestWriteMDirectAccountData PASS diff --git a/xtask/src/complement.rs b/xtask/src/complement.rs index 4b9b55a2..fa04eb59 100644 --- a/xtask/src/complement.rs +++ b/xtask/src/complement.rs @@ -12,6 +12,7 @@ mod test2json; use self::{ docker::load_docker_image, + summary::{compare_summary, read_summary}, test2json::{count_complement_tests, run_complement}, }; @@ -23,6 +24,12 @@ pub(crate) struct Args { /// If it exists and is not empty, an error will be returned. #[clap(short, long)] out: PathBuf, + + /// Baseline test summary file to compare with + /// + /// If unspecified, defaults to `$repo_root/complement-baseline.tsv` + #[clap(short, long)] + baseline: Option, } #[allow(clippy::needless_pass_by_value)] @@ -30,6 +37,15 @@ pub(crate) fn main(args: Args) -> Result<()> { let sh = Shell::new().unwrap(); let toplevel = get_toplevel_path(&sh) .wrap_err("failed to determine repository root directory")?; + let baseline_path = args + .baseline + .unwrap_or_else(|| toplevel.join("complement-baseline.tsv")); + let baseline = read_summary(&baseline_path).wrap_err_with(|| { + format!( + "failed to read baseline test result summary from \ + {baseline_path:?}" + ) + })?; create_out_dir(&args.out).wrap_err_with(|| { format!("error initializing output directory {:?}", args.out) })?; @@ -38,8 +54,11 @@ pub(crate) fn main(args: Args) -> Result<()> { )?; let test_count = count_complement_tests(&sh, &docker_image) .wrap_err("failed to determine total complement test count")?; - run_complement(&sh, &args.out, &docker_image, test_count) + let results = run_complement(&sh, &args.out, &docker_image, test_count) .wrap_err("failed to run complement tests")?; + let summary_path = args.out.join("summary.tsv"); + compare_summary(&baseline, &results, &baseline_path, &summary_path)?; + println!("\nTest results were identical to baseline."); Ok(()) } diff --git a/xtask/src/complement/summary.rs b/xtask/src/complement/summary.rs index d95f819f..f3b03ba9 100644 --- a/xtask/src/complement/summary.rs +++ b/xtask/src/complement/summary.rs @@ -6,10 +6,14 @@ use std::{ collections::BTreeMap, + fs, io::{BufWriter, Write}, + path::Path, }; -use miette::{IntoDiagnostic, Result}; +use miette::{ + miette, IntoDiagnostic, LabeledSpan, NamedSource, Result, WrapErr, +}; use super::test2json::TestResult; @@ -29,6 +33,30 @@ fn escape_tsv_value(value: &str) -> String { .replace('\r', "\\r") } +/// Converts a string from a TSV value from to unescaped form. +fn unescape_tsv_value(value: &str) -> String { + let mut chars = value.chars(); + let mut out = String::new(); + while let Some(c) = chars.next() { + if c == '\\' { + match chars.next() { + Some('\\') => out.push('\\'), + Some('n') => out.push('\n'), + Some('t') => out.push('\t'), + Some('r') => out.push('\r'), + Some(c2) => { + out.push(c); + out.push(c2); + } + None => out.push(c), + } + } else { + out.push(c); + } + } + out +} + /// Write a test result summary to a writer. pub(crate) fn write_summary( w: &mut BufWriter, @@ -48,3 +76,211 @@ pub(crate) fn write_summary( } Ok(()) } + +/// Reads test result summary from a TSV file written by a previous run of the +/// complement wrapper. +pub(crate) fn read_summary( + path: &Path, +) -> Result> { + let contents = fs::read_to_string(path) + .into_diagnostic() + .wrap_err("failed to read summary file contents")?; + let source = NamedSource::new(path.to_string_lossy(), contents); + let contents = &source.inner(); + + let mut offset = 0; + // The TSV spec allows CRLF, but we never emit these ourselves + let mut lines = contents.split('\n'); + + let header_line = lines.next().ok_or_else(|| { + miette!( + labels = vec![LabeledSpan::at_offset(0, "expected header row")], + "summary file missing header row", + ) + .with_source_code(source.clone()) + })?; + let expected_header_line = "test\tresult"; + if header_line != expected_header_line { + return Err(miette!( + labels = vec![LabeledSpan::at( + 0..header_line.len(), + "unexpected header" + )], + "summary file header row has unexpected columns. Expecting \ + {expected_header_line:?}." + ) + .with_source_code(source)); + } + offset += header_line.len() + 1; + + let mut results = BTreeMap::new(); + for line in lines { + if line.is_empty() { + continue; + } + + let tabs = line.match_indices('\t').collect::>(); + let column_count = tabs.len() + 1; + let (result_span, test, result) = match tabs[..] { + [(first_tab, _)] => { + let result_span = offset + first_tab + 1..offset + line.len(); + let test = line.get(..first_tab).expect( + "index should be valid because it was returned from \ + 'match_indices'", + ); + let result = line.get(first_tab + 1..).expect( + "index should be valid because it was returned from \ + 'match_indices'", + ); + (result_span, test, result) + } + [] => { + return Err(miette!( + labels = vec![LabeledSpan::at_offset( + offset + line.len(), + "expected more columns here" + )], + "each row in the summary file should have exactly two \ + columns. This row only has {column_count} columns.", + ) + .with_source_code(source)) + } + [_, (first_bad_tab, _), ..] => { + let span = offset + first_bad_tab..offset + line.len(); + return Err(miette!( + labels = + vec![LabeledSpan::at(span, "unexpected extra columns")], + "each row in the summary file should have exactly two \ + columns. This row has {column_count} columns.", + ) + .with_source_code(source)); + } + }; + + let test = unescape_tsv_value(test); + let result = unescape_tsv_value(result); + + let result = result.parse().map_err(|_| { + miette!( + labels = + vec![LabeledSpan::at(result_span, "invalid result value")], + "test result value must be one of 'PASS', 'FAIL', or 'SKIP'." + ) + .with_source_code(source.clone()) + })?; + + results.insert(test, result); + offset += line.len() + 1; + } + Ok(results) +} + +/// Print a bulleted list of test names, truncating if there are too many. +fn print_truncated_tests(tests: &[&str]) { + let max = 5; + for test in &tests[..max.min(tests.len())] { + println!(" - {test}"); + } + if tests.len() > max { + println!(" ... ({} more)", tests.len() - max); + } +} + +/// Compares new test results against older results, returning a error if they +/// differ. +/// +/// A description of the differences will be logged separately from the returned +/// error. +pub(crate) fn compare_summary( + old: &TestResults, + new: &TestResults, + old_path: &Path, + new_path: &Path, +) -> Result<()> { + let mut unexpected_pass: Vec<&str> = Vec::new(); + let mut unexpected_fail: Vec<&str> = Vec::new(); + let mut unexpected_skip: Vec<&str> = Vec::new(); + let mut added: Vec<&str> = Vec::new(); + let mut removed: Vec<&str> = Vec::new(); + + for (test, new_result) in new { + if let Some(old_result) = old.get(test) { + if old_result != new_result { + match new_result { + TestResult::Pass => unexpected_pass.push(test), + TestResult::Fail => unexpected_fail.push(test), + TestResult::Skip => unexpected_skip.push(test), + } + } + } else { + added.push(test); + } + } + for test in old.keys() { + if !new.contains_key(test) { + removed.push(test); + } + } + + let mut differences = false; + if !added.is_empty() { + differences = true; + println!( + "\n{} tests were added that were not present in the baseline:", + added.len() + ); + print_truncated_tests(&added); + } + if !removed.is_empty() { + differences = true; + println!( + "\n{} tests present in the baseline were removed:", + removed.len() + ); + print_truncated_tests(&removed); + } + if !unexpected_pass.is_empty() { + differences = true; + println!( + "\n{} tests passed that did not pass in the baseline:", + unexpected_pass.len() + ); + print_truncated_tests(&unexpected_pass); + } + if !unexpected_skip.is_empty() { + differences = true; + println!( + "\n{} tests skipped that were not skipped in the baseline:", + unexpected_skip.len() + ); + print_truncated_tests(&unexpected_skip); + } + if !unexpected_fail.is_empty() { + differences = true; + println!( + "\n{} tests failed that did not fail in the baseline (these are \ + likely regressions):", + unexpected_fail.len() + ); + print_truncated_tests(&unexpected_fail); + } + + if differences { + Err(miette!( + help = format!( + "Evaluate each of the differences to determine whether they \ + are expected. If all differences are expected, copy the new \ + summary file {new_path:?} to {old_path:?} and commit the \ + change. If some differences are unexpected, fix them and try \ + another test run.\n\nAn example of an expected change would \ + be a test that is now passing after your changes fixed it. \ + An example of an unexpected change would be an unrelated \ + test that is now failing, which would be a regression." + ), + "Test results differed from baseline in {old_path:?}. The \ + differences are described above." + )) + } else { + Ok(()) + } +} diff --git a/xtask/src/complement/test2json.rs b/xtask/src/complement/test2json.rs index c08580a9..59253b14 100644 --- a/xtask/src/complement/test2json.rs +++ b/xtask/src/complement/test2json.rs @@ -14,7 +14,7 @@ use std::{ use indicatif::{ProgressBar, ProgressStyle}; use miette::{miette, IntoDiagnostic, LabeledSpan, Result, WrapErr}; use serde::Deserialize; -use strum::Display; +use strum::{Display, EnumString}; use xshell::{cmd, Shell}; use super::summary::{write_summary, TestResults}; @@ -43,7 +43,7 @@ pub(crate) fn run_complement( out: &Path, docker_image: &str, test_count: u64, -) -> Result<()> { +) -> Result { // TODO: handle SIG{INT,TERM} // TODO: XTASK_PATH variable, so that we don't need to pollute devshell with // go @@ -70,7 +70,7 @@ pub(crate) fn run_complement( ctx.handle_line(&line)?; } - Ok(()) + Ok(ctx.results) } /// Schema from @@ -103,7 +103,7 @@ enum GoTestEvent { OtherAction, } -#[derive(Copy, Clone, Display, Debug)] +#[derive(Copy, Clone, Display, EnumString, Eq, PartialEq, Debug)] #[strum(serialize_all = "UPPERCASE")] pub(crate) enum TestResult { Pass,