compare complement test results to a baseline

One thing that might be neat in the future is noticing differing results
while the tests are still running, and modifying the log messages to
indicate them. I can imagine situations where you would want to abort
the test run immediately after seeing the first regression.

complement-baseline.tsv

@@ -0,0 +1,220 @@
+test	result
+GLOBAL	FAIL
+TestACLs	PASS
+TestBannedUserCannotSendJoin	FAIL
+TestCannotSendKnockViaSendKnockInMSC3787Room	FAIL
+TestCannotSendKnockViaSendKnockInMSC3787Room/event_with_mismatched_state_key	FAIL
+TestCannotSendKnockViaSendKnockInMSC3787Room/invite_event	FAIL
+TestCannotSendKnockViaSendKnockInMSC3787Room/join_event	FAIL
+TestCannotSendKnockViaSendKnockInMSC3787Room/leave_event	FAIL
+TestCannotSendKnockViaSendKnockInMSC3787Room/non-state_membership_event	FAIL
+TestCannotSendKnockViaSendKnockInMSC3787Room/regular_event	FAIL
+TestCannotSendNonJoinViaSendJoinV1	FAIL
+TestCannotSendNonJoinViaSendJoinV1/event_with_mismatched_state_key	PASS
+TestCannotSendNonJoinViaSendJoinV1/invite_event	PASS
+TestCannotSendNonJoinViaSendJoinV1/knock_event	PASS
+TestCannotSendNonJoinViaSendJoinV1/leave_event	FAIL
+TestCannotSendNonJoinViaSendJoinV1/non-state_membership_event	PASS
+TestCannotSendNonJoinViaSendJoinV1/regular_event	FAIL
+TestCannotSendNonJoinViaSendJoinV2	FAIL
+TestCannotSendNonJoinViaSendJoinV2/event_with_mismatched_state_key	PASS
+TestCannotSendNonJoinViaSendJoinV2/invite_event	PASS
+TestCannotSendNonJoinViaSendJoinV2/knock_event	PASS
+TestCannotSendNonJoinViaSendJoinV2/leave_event	FAIL
+TestCannotSendNonJoinViaSendJoinV2/non-state_membership_event	PASS
+TestCannotSendNonJoinViaSendJoinV2/regular_event	FAIL
+TestCannotSendNonKnockViaSendKnock	FAIL
+TestCannotSendNonLeaveViaSendLeaveV1	FAIL
+TestCannotSendNonLeaveViaSendLeaveV1/event_with_mismatched_state_key	FAIL
+TestCannotSendNonLeaveViaSendLeaveV1/invite_event	FAIL
+TestCannotSendNonLeaveViaSendLeaveV1/join_event	FAIL
+TestCannotSendNonLeaveViaSendLeaveV1/knock_event	FAIL
+TestCannotSendNonLeaveViaSendLeaveV1/non-state_membership_event	FAIL
+TestCannotSendNonLeaveViaSendLeaveV1/regular_event	FAIL
+TestCannotSendNonLeaveViaSendLeaveV2	FAIL
+TestCannotSendNonLeaveViaSendLeaveV2/event_with_mismatched_state_key	FAIL
+TestCannotSendNonLeaveViaSendLeaveV2/invite_event	FAIL
+TestCannotSendNonLeaveViaSendLeaveV2/join_event	FAIL
+TestCannotSendNonLeaveViaSendLeaveV2/knock_event	FAIL
+TestCannotSendNonLeaveViaSendLeaveV2/non-state_membership_event	FAIL
+TestCannotSendNonLeaveViaSendLeaveV2/regular_event	FAIL
+TestClientSpacesSummary	FAIL
+TestClientSpacesSummary/max_depth	FAIL
+TestClientSpacesSummary/pagination	FAIL
+TestClientSpacesSummary/query_whole_graph	FAIL
+TestClientSpacesSummary/redact_link	FAIL
+TestClientSpacesSummary/suggested_only	FAIL
+TestClientSpacesSummaryJoinRules	FAIL
+TestContentMediaV1	PASS
+TestDeviceListsUpdateOverFederation	FAIL
+TestDeviceListsUpdateOverFederation/good_connectivity	FAIL
+TestDeviceListsUpdateOverFederation/interrupted_connectivity	FAIL
+TestDeviceListsUpdateOverFederation/stopped_server	FAIL
+TestDeviceListsUpdateOverFederationOnRoomJoin	FAIL
+TestEventAuth	FAIL
+TestFederatedClientSpaces	FAIL
+TestFederationKeyUploadQuery	FAIL
+TestFederationKeyUploadQuery/Can_claim_remote_one_time_key_using_POST	FAIL
+TestFederationKeyUploadQuery/Can_query_remote_device_keys_using_POST	FAIL
+TestFederationRedactSendsWithoutEvent	PASS
+TestFederationRejectInvite	FAIL
+TestFederationRoomsInvite	FAIL
+TestFederationRoomsInvite/Parallel	FAIL
+TestFederationRoomsInvite/Parallel/Invited_user_can_reject_invite_over_federation	FAIL
+TestFederationRoomsInvite/Parallel/Invited_user_can_reject_invite_over_federation_for_empty_room	PASS
+TestFederationRoomsInvite/Parallel/Invited_user_can_reject_invite_over_federation_several_times	FAIL
+TestFederationRoomsInvite/Parallel/Invited_user_has_'is_direct'_flag_in_prev_content_after_joining	PASS
+TestFederationRoomsInvite/Parallel/Remote_invited_user_can_see_room_metadata	PASS
+TestFederationThumbnail	PASS
+TestGetMissingEventsGapFilling	FAIL
+TestInboundCanReturnMissingEvents	FAIL
+TestInboundCanReturnMissingEvents/Inbound_federation_can_return_missing_events_for_invited_visibility	FAIL
+TestInboundCanReturnMissingEvents/Inbound_federation_can_return_missing_events_for_joined_visibility	FAIL
+TestInboundCanReturnMissingEvents/Inbound_federation_can_return_missing_events_for_shared_visibility	FAIL
+TestInboundCanReturnMissingEvents/Inbound_federation_can_return_missing_events_for_world_readable_visibility	FAIL
+TestInboundFederationKeys	PASS
+TestInboundFederationProfile	PASS
+TestInboundFederationProfile/Inbound_federation_can_query_profile_data	PASS
+TestInboundFederationProfile/Non-numeric_ports_in_server_names_are_rejected	PASS
+TestInboundFederationRejectsEventsWithRejectedAuthEvents	FAIL
+TestIsDirectFlagFederation	PASS
+TestIsDirectFlagLocal	PASS
+TestJoinFederatedRoomFailOver	PASS
+TestJoinFederatedRoomFromApplicationServiceBridgeUser	FAIL
+TestJoinFederatedRoomFromApplicationServiceBridgeUser/join_remote_federated_room_as_application_service_user	FAIL
+TestJoinFederatedRoomWithUnverifiableEvents	PASS
+TestJoinFederatedRoomWithUnverifiableEvents//send_join_response_missing_signatures_shouldn't_block_room_join	PASS
+TestJoinFederatedRoomWithUnverifiableEvents//send_join_response_with_bad_signatures_shouldn't_block_room_join	PASS
+TestJoinFederatedRoomWithUnverifiableEvents//send_join_response_with_state_with_unverifiable_auth_events_shouldn't_block_room_join	PASS
+TestJoinFederatedRoomWithUnverifiableEvents//send_join_response_with_unobtainable_keys_shouldn't_block_room_join	PASS
+TestJoinViaRoomIDAndServerName	PASS
+TestJumpToDateEndpoint	FAIL
+TestJumpToDateEndpoint/parallel	FAIL
+TestJumpToDateEndpoint/parallel/federation	FAIL
+TestJumpToDateEndpoint/parallel/federation/can_paginate_after_getting_remote_event_from_timestamp_to_event_endpoint	FAIL
+TestJumpToDateEndpoint/parallel/federation/looking_backwards,_should_be_able_to_find_event_that_was_sent_before_we_joined	FAIL
+TestJumpToDateEndpoint/parallel/federation/looking_forwards,_should_be_able_to_find_event_that_was_sent_before_we_joined	FAIL
+TestJumpToDateEndpoint/parallel/federation/when_looking_backwards_before_the_room_was_created,_should_be_able_to_find_event_that_was_imported	FAIL
+TestJumpToDateEndpoint/parallel/should_find_event_after_given_timestmap	FAIL
+TestJumpToDateEndpoint/parallel/should_find_event_before_given_timestmap	FAIL
+TestJumpToDateEndpoint/parallel/should_find_next_event_topologically_after_given_timestmap_when_all_message_timestamps_are_the_same	FAIL
+TestJumpToDateEndpoint/parallel/should_find_next_event_topologically_before_given_timestamp_when_all_message_timestamps_are_the_same	FAIL
+TestJumpToDateEndpoint/parallel/should_find_nothing_after_the_latest_timestmap	PASS
+TestJumpToDateEndpoint/parallel/should_find_nothing_before_the_earliest_timestmap	PASS
+TestJumpToDateEndpoint/parallel/should_not_be_able_to_query_a_private_room_you_are_not_a_member_of	FAIL
+TestJumpToDateEndpoint/parallel/should_not_be_able_to_query_a_public_room_you_are_not_a_member_of	FAIL
+TestKnockRoomsInPublicRoomsDirectory	FAIL
+TestKnockRoomsInPublicRoomsDirectoryInMSC3787Room	FAIL
+TestKnocking	FAIL
+TestKnockingInMSC3787Room	FAIL
+TestKnockingInMSC3787Room/A_user_can_knock_on_a_room_without_a_reason	FAIL
+TestKnockingInMSC3787Room/A_user_can_knock_on_a_room_without_a_reason#01	FAIL
+TestKnockingInMSC3787Room/A_user_cannot_knock_on_a_room_they_are_already_in	FAIL
+TestKnockingInMSC3787Room/A_user_cannot_knock_on_a_room_they_are_already_in#01	FAIL
+TestKnockingInMSC3787Room/A_user_cannot_knock_on_a_room_they_are_already_invited_to	FAIL
+TestKnockingInMSC3787Room/A_user_cannot_knock_on_a_room_they_are_already_invited_to#01	FAIL
+TestKnockingInMSC3787Room/A_user_in_the_room_can_accept_a_knock	PASS
+TestKnockingInMSC3787Room/A_user_in_the_room_can_accept_a_knock#01	PASS
+TestKnockingInMSC3787Room/A_user_in_the_room_can_reject_a_knock	FAIL
+TestKnockingInMSC3787Room/A_user_in_the_room_can_reject_a_knock#01	FAIL
+TestKnockingInMSC3787Room/A_user_that_has_already_knocked_is_allowed_to_knock_again_on_the_same_room	FAIL
+TestKnockingInMSC3787Room/A_user_that_has_already_knocked_is_allowed_to_knock_again_on_the_same_room#01	FAIL
+TestKnockingInMSC3787Room/A_user_that_has_knocked_on_a_local_room_can_rescind_their_knock_and_then_knock_again	FAIL
+TestKnockingInMSC3787Room/A_user_that_is_banned_from_a_room_cannot_knock_on_it	FAIL
+TestKnockingInMSC3787Room/A_user_that_is_banned_from_a_room_cannot_knock_on_it#01	FAIL
+TestKnockingInMSC3787Room/Attempting_to_join_a_room_with_join_rule_'knock'_without_an_invite_should_fail	FAIL
+TestKnockingInMSC3787Room/Attempting_to_join_a_room_with_join_rule_'knock'_without_an_invite_should_fail#01	FAIL
+TestKnockingInMSC3787Room/Change_the_join_rule_of_a_room_from_'invite'_to_'knock'	PASS
+TestKnockingInMSC3787Room/Change_the_join_rule_of_a_room_from_'invite'_to_'knock'#01	PASS
+TestKnockingInMSC3787Room/Knocking_on_a_room_with_a_join_rule_other_than_'knock'_should_fail	FAIL
+TestKnockingInMSC3787Room/Knocking_on_a_room_with_a_join_rule_other_than_'knock'_should_fail#01	FAIL
+TestKnockingInMSC3787Room/Knocking_on_a_room_with_join_rule_'knock'_should_succeed	FAIL
+TestKnockingInMSC3787Room/Knocking_on_a_room_with_join_rule_'knock'_should_succeed#01	FAIL
+TestKnockingInMSC3787Room/Users_in_the_room_see_a_user's_membership_update_when_they_knock	FAIL
+TestKnockingInMSC3787Room/Users_in_the_room_see_a_user's_membership_update_when_they_knock#01	FAIL
+TestLocalPngThumbnail	PASS
+TestLocalPngThumbnail/test_/_matrix/client/v1/media_endpoint	PASS
+TestMediaFilenames	FAIL
+TestMediaFilenames/Parallel	FAIL
+TestMediaFilenames/Parallel/ASCII	FAIL
+TestMediaFilenames/Parallel/ASCII/Can_download_file_'ascii'	FAIL
+TestMediaFilenames/Parallel/ASCII/Can_download_file_'ascii'_over_/_matrix/client/v1/media/download	FAIL
+TestMediaFilenames/Parallel/ASCII/Can_download_file_'name;with;semicolons'	FAIL
+TestMediaFilenames/Parallel/ASCII/Can_download_file_'name;with;semicolons'_over_/_matrix/client/v1/media/download	FAIL
+TestMediaFilenames/Parallel/ASCII/Can_download_file_'name_with_spaces'	FAIL
+TestMediaFilenames/Parallel/ASCII/Can_download_file_'name_with_spaces'_over_/_matrix/client/v1/media/download	FAIL
+TestMediaFilenames/Parallel/ASCII/Can_download_specifying_a_different_ASCII_file_name	PASS
+TestMediaFilenames/Parallel/ASCII/Can_download_specifying_a_different_ASCII_file_name_over__matrix/client/v1/media/download	PASS
+TestMediaFilenames/Parallel/ASCII/Can_upload_with_ASCII_file_name	PASS
+TestMediaFilenames/Parallel/Unicode	FAIL
+TestMediaFilenames/Parallel/Unicode/Can_download_specifying_a_different_Unicode_file_name	PASS
+TestMediaFilenames/Parallel/Unicode/Can_download_specifying_a_different_Unicode_file_name_over__matrix/client/v1/media/download	PASS
+TestMediaFilenames/Parallel/Unicode/Can_download_with_Unicode_file_name_locally	FAIL
+TestMediaFilenames/Parallel/Unicode/Can_download_with_Unicode_file_name_locally_over__matrix/client/v1/media/download	FAIL
+TestMediaFilenames/Parallel/Unicode/Can_download_with_Unicode_file_name_over_federation	FAIL
+TestMediaFilenames/Parallel/Unicode/Can_download_with_Unicode_file_name_over_federation_via__matrix/client/v1/media/download	FAIL
+TestMediaFilenames/Parallel/Unicode/Can_upload_with_Unicode_file_name	PASS
+TestMediaFilenames/Parallel/Unicode/Will_serve_safe_media_types_as_inline	SKIP
+TestMediaFilenames/Parallel/Unicode/Will_serve_safe_media_types_as_inline_via__matrix/client/v1/media/download	SKIP
+TestMediaFilenames/Parallel/Unicode/Will_serve_safe_media_types_with_parameters_as_inline	SKIP
+TestMediaFilenames/Parallel/Unicode/Will_serve_safe_media_types_with_parameters_as_inline_via__matrix/client/v1/media/download	SKIP
+TestMediaFilenames/Parallel/Unicode/Will_serve_unsafe_media_types_as_attachments	SKIP
+TestMediaFilenames/Parallel/Unicode/Will_serve_unsafe_media_types_as_attachments_via__matrix/client/v1/media/download	SKIP
+TestMediaWithoutFileName	FAIL
+TestMediaWithoutFileName/parallel	FAIL
+TestMediaWithoutFileName/parallel/Can_download_without_a_file_name_locally	PASS
+TestMediaWithoutFileName/parallel/Can_download_without_a_file_name_over_federation	FAIL
+TestMediaWithoutFileName/parallel/Can_upload_without_a_file_name	PASS
+TestMediaWithoutFileNameCSMediaV1	PASS
+TestMediaWithoutFileNameCSMediaV1/parallel	PASS
+TestMediaWithoutFileNameCSMediaV1/parallel/Can_download_without_a_file_name_locally	PASS
+TestMediaWithoutFileNameCSMediaV1/parallel/Can_download_without_a_file_name_over_federation	PASS
+TestMediaWithoutFileNameCSMediaV1/parallel/Can_upload_without_a_file_name	PASS
+TestNetworkPartitionOrdering	FAIL
+TestOutboundFederationIgnoresMissingEventWithBadJSONForRoomVersion6	FAIL
+TestOutboundFederationProfile	PASS
+TestOutboundFederationProfile/Outbound_federation_can_query_profile_data	PASS
+TestOutboundFederationSend	PASS
+TestRemoteAliasRequestsUnderstandUnicode	PASS
+TestRemotePngThumbnail	PASS
+TestRemotePngThumbnail/test_/_matrix/client/v1/media_endpoint	PASS
+TestRemotePresence	FAIL
+TestRemotePresence/Presence_changes_are_also_reported_to_remote_room_members	FAIL
+TestRemotePresence/Presence_changes_to_UNAVAILABLE_are_reported_to_remote_room_members	FAIL
+TestRemoteTyping	FAIL
+TestRestrictedRoomsLocalJoin	FAIL
+TestRestrictedRoomsLocalJoinInMSC3787Room	FAIL
+TestRestrictedRoomsLocalJoinInMSC3787Room/Join_should_fail_initially	PASS
+TestRestrictedRoomsLocalJoinInMSC3787Room/Join_should_fail_when_left_allowed_room	FAIL
+TestRestrictedRoomsLocalJoinInMSC3787Room/Join_should_fail_with_mangled_join_rules	PASS
+TestRestrictedRoomsLocalJoinInMSC3787Room/Join_should_succeed_when_invited	FAIL
+TestRestrictedRoomsLocalJoinInMSC3787Room/Join_should_succeed_when_joined_to_allowed_room	PASS
+TestRestrictedRoomsRemoteJoin	FAIL
+TestRestrictedRoomsRemoteJoinFailOver	FAIL
+TestRestrictedRoomsRemoteJoinFailOverInMSC3787Room	FAIL
+TestRestrictedRoomsRemoteJoinInMSC3787Room	FAIL
+TestRestrictedRoomsRemoteJoinInMSC3787Room/Join_should_fail_initially	PASS
+TestRestrictedRoomsRemoteJoinInMSC3787Room/Join_should_fail_when_left_allowed_room	PASS
+TestRestrictedRoomsRemoteJoinInMSC3787Room/Join_should_fail_with_mangled_join_rules	PASS
+TestRestrictedRoomsRemoteJoinInMSC3787Room/Join_should_succeed_when_invited	PASS
+TestRestrictedRoomsRemoteJoinInMSC3787Room/Join_should_succeed_when_joined_to_allowed_room	FAIL
+TestRestrictedRoomsRemoteJoinLocalUser	FAIL
+TestRestrictedRoomsRemoteJoinLocalUserInMSC3787Room	FAIL
+TestRestrictedRoomsSpacesSummaryFederation	FAIL
+TestRestrictedRoomsSpacesSummaryLocal	FAIL
+TestSendJoinPartialStateResponse	SKIP
+TestSyncOmitsStateChangeOnFilteredEvents	PASS
+TestToDeviceMessagesOverFederation	FAIL
+TestToDeviceMessagesOverFederation/good_connectivity	PASS
+TestToDeviceMessagesOverFederation/interrupted_connectivity	FAIL
+TestToDeviceMessagesOverFederation/stopped_server	FAIL
+TestUnbanViaInvite	FAIL
+TestUnknownEndpoints	FAIL
+TestUnknownEndpoints/Client-server_endpoints	PASS
+TestUnknownEndpoints/Key_endpoints	FAIL
+TestUnknownEndpoints/Media_endpoints	PASS
+TestUnknownEndpoints/Server-server_endpoints	PASS
+TestUnknownEndpoints/Unknown_prefix	PASS
+TestUnrejectRejectedEvents	FAIL
+TestUserAppearsInChangedDeviceListOnJoinOverFederation	PASS
+TestWriteMDirectAccountData	PASS

xtask/src/complement.rs

@@ -12,6 +12,7 @@ mod test2json;
 
 use self::{
     docker::load_docker_image,
+    summary::{compare_summary, read_summary},
     test2json::{count_complement_tests, run_complement},
 };
 
@@ -23,6 +24,12 @@ pub(crate) struct Args {
     /// If it exists and is not empty, an error will be returned.
     #[clap(short, long)]
     out: PathBuf,
+
+    /// Baseline test summary file to compare with
+    ///
+    /// If unspecified, defaults to `$repo_root/complement-baseline.tsv`
+    #[clap(short, long)]
+    baseline: Option<PathBuf>,
 }
 
 #[allow(clippy::needless_pass_by_value)]
@@ -30,6 +37,15 @@ pub(crate) fn main(args: Args) -> Result<()> {
     let sh = Shell::new().unwrap();
     let toplevel = get_toplevel_path(&sh)
         .wrap_err("failed to determine repository root directory")?;
+    let baseline_path = args
+        .baseline
+        .unwrap_or_else(|| toplevel.join("complement-baseline.tsv"));
+    let baseline = read_summary(&baseline_path).wrap_err_with(|| {
+        format!(
+            "failed to read baseline test result summary from \
+             {baseline_path:?}"
+        )
+    })?;
     create_out_dir(&args.out).wrap_err_with(|| {
         format!("error initializing output directory {:?}", args.out)
     })?;
@@ -38,8 +54,11 @@ pub(crate) fn main(args: Args) -> Result<()> {
     )?;
     let test_count = count_complement_tests(&sh, &docker_image)
         .wrap_err("failed to determine total complement test count")?;
-    run_complement(&sh, &args.out, &docker_image, test_count)
+    let results = run_complement(&sh, &args.out, &docker_image, test_count)
         .wrap_err("failed to run complement tests")?;
+    let summary_path = args.out.join("summary.tsv");
+    compare_summary(&baseline, &results, &baseline_path, &summary_path)?;
+    println!("\nTest results were identical to baseline.");
     Ok(())
 }
 

xtask/src/complement/summary.rs

@@ -6,10 +6,14 @@
 
 use std::{
     collections::BTreeMap,
+    fs,
     io::{BufWriter, Write},
+    path::Path,
 };
 
-use miette::{IntoDiagnostic, Result};
+use miette::{
+    miette, IntoDiagnostic, LabeledSpan, NamedSource, Result, WrapErr,
+};
 
 use super::test2json::TestResult;
 
@@ -29,6 +33,30 @@ fn escape_tsv_value(value: &str) -> String {
         .replace('\r', "\\r")
 }
 
+/// Converts a string from a TSV value from to unescaped form.
+fn unescape_tsv_value(value: &str) -> String {
+    let mut chars = value.chars();
+    let mut out = String::new();
+    while let Some(c) = chars.next() {
+        if c == '\\' {
+            match chars.next() {
+                Some('\\') => out.push('\\'),
+                Some('n') => out.push('\n'),
+                Some('t') => out.push('\t'),
+                Some('r') => out.push('\r'),
+                Some(c2) => {
+                    out.push(c);
+                    out.push(c2);
+                }
+                None => out.push(c),
+            }
+        } else {
+            out.push(c);
+        }
+    }
+    out
+}
+
 /// Write a test result summary to a writer.
 pub(crate) fn write_summary<W: Write>(
     w: &mut BufWriter<W>,
@@ -48,3 +76,211 @@ pub(crate) fn write_summary<W: Write>(
     }
     Ok(())
 }
+
+/// Reads test result summary from a TSV file written by a previous run of the
+/// complement wrapper.
+pub(crate) fn read_summary(
+    path: &Path,
+) -> Result<BTreeMap<String, TestResult>> {
+    let contents = fs::read_to_string(path)
+        .into_diagnostic()
+        .wrap_err("failed to read summary file contents")?;
+    let source = NamedSource::new(path.to_string_lossy(), contents);
+    let contents = &source.inner();
+
+    let mut offset = 0;
+    // The TSV spec allows CRLF, but we never emit these ourselves
+    let mut lines = contents.split('\n');
+
+    let header_line = lines.next().ok_or_else(|| {
+        miette!(
+            labels = vec![LabeledSpan::at_offset(0, "expected header row")],
+            "summary file missing header row",
+        )
+        .with_source_code(source.clone())
+    })?;
+    let expected_header_line = "test\tresult";
+    if header_line != expected_header_line {
+        return Err(miette!(
+            labels = vec![LabeledSpan::at(
+                0..header_line.len(),
+                "unexpected header"
+            )],
+            "summary file header row has unexpected columns. Expecting \
+             {expected_header_line:?}."
+        )
+        .with_source_code(source));
+    }
+    offset += header_line.len() + 1;
+
+    let mut results = BTreeMap::new();
+    for line in lines {
+        if line.is_empty() {
+            continue;
+        }
+
+        let tabs = line.match_indices('\t').collect::<Vec<_>>();
+        let column_count = tabs.len() + 1;
+        let (result_span, test, result) = match tabs[..] {
+            [(first_tab, _)] => {
+                let result_span = offset + first_tab + 1..offset + line.len();
+                let test = line.get(..first_tab).expect(
+                    "index should be valid because it was returned from \
+                     'match_indices'",
+                );
+                let result = line.get(first_tab + 1..).expect(
+                    "index should be valid because it was returned from \
+                     'match_indices'",
+                );
+                (result_span, test, result)
+            }
+            [] => {
+                return Err(miette!(
+                    labels = vec![LabeledSpan::at_offset(
+                        offset + line.len(),
+                        "expected more columns here"
+                    )],
+                    "each row in the summary file should have exactly two \
+                     columns. This row only has {column_count} columns.",
+                )
+                .with_source_code(source))
+            }
+            [_, (first_bad_tab, _), ..] => {
+                let span = offset + first_bad_tab..offset + line.len();
+                return Err(miette!(
+                    labels =
+                        vec![LabeledSpan::at(span, "unexpected extra columns")],
+                    "each row in the summary file should have exactly two \
+                     columns. This row has {column_count} columns.",
+                )
+                .with_source_code(source));
+            }
+        };
+
+        let test = unescape_tsv_value(test);
+        let result = unescape_tsv_value(result);
+
+        let result = result.parse().map_err(|_| {
+            miette!(
+                labels =
+                    vec![LabeledSpan::at(result_span, "invalid result value")],
+                "test result value must be one of 'PASS', 'FAIL', or 'SKIP'."
+            )
+            .with_source_code(source.clone())
+        })?;
+
+        results.insert(test, result);
+        offset += line.len() + 1;
+    }
+    Ok(results)
+}
+
+/// Print a bulleted list of test names, truncating if there are too many.
+fn print_truncated_tests(tests: &[&str]) {
+    let max = 5;
+    for test in &tests[..max.min(tests.len())] {
+        println!("    - {test}");
+    }
+    if tests.len() > max {
+        println!("    ... ({} more)", tests.len() - max);
+    }
+}
+
+/// Compares new test results against older results, returning a error if they
+/// differ.
+///
+/// A description of the differences will be logged separately from the returned
+/// error.
+pub(crate) fn compare_summary(
+    old: &TestResults,
+    new: &TestResults,
+    old_path: &Path,
+    new_path: &Path,
+) -> Result<()> {
+    let mut unexpected_pass: Vec<&str> = Vec::new();
+    let mut unexpected_fail: Vec<&str> = Vec::new();
+    let mut unexpected_skip: Vec<&str> = Vec::new();
+    let mut added: Vec<&str> = Vec::new();
+    let mut removed: Vec<&str> = Vec::new();
+
+    for (test, new_result) in new {
+        if let Some(old_result) = old.get(test) {
+            if old_result != new_result {
+                match new_result {
+                    TestResult::Pass => unexpected_pass.push(test),
+                    TestResult::Fail => unexpected_fail.push(test),
+                    TestResult::Skip => unexpected_skip.push(test),
+                }
+            }
+        } else {
+            added.push(test);
+        }
+    }
+    for test in old.keys() {
+        if !new.contains_key(test) {
+            removed.push(test);
+        }
+    }
+
+    let mut differences = false;
+    if !added.is_empty() {
+        differences = true;
+        println!(
+            "\n{} tests were added that were not present in the baseline:",
+            added.len()
+        );
+        print_truncated_tests(&added);
+    }
+    if !removed.is_empty() {
+        differences = true;
+        println!(
+            "\n{} tests present in the baseline were removed:",
+            removed.len()
+        );
+        print_truncated_tests(&removed);
+    }
+    if !unexpected_pass.is_empty() {
+        differences = true;
+        println!(
+            "\n{} tests passed that did not pass in the baseline:",
+            unexpected_pass.len()
+        );
+        print_truncated_tests(&unexpected_pass);
+    }
+    if !unexpected_skip.is_empty() {
+        differences = true;
+        println!(
+            "\n{} tests skipped that were not skipped in the baseline:",
+            unexpected_skip.len()
+        );
+        print_truncated_tests(&unexpected_skip);
+    }
+    if !unexpected_fail.is_empty() {
+        differences = true;
+        println!(
+            "\n{} tests failed that did not fail in the baseline (these are \
+             likely regressions):",
+            unexpected_fail.len()
+        );
+        print_truncated_tests(&unexpected_fail);
+    }
+
+    if differences {
+        Err(miette!(
+            help = format!(
+                "Evaluate each of the differences to determine whether they \
+                 are expected. If all differences are expected, copy the new \
+                 summary file {new_path:?} to {old_path:?} and commit the \
+                 change. If some differences are unexpected, fix them and try \
+                 another test run.\n\nAn example of an expected change would \
+                 be a test that is now passing after your changes fixed it. \
+                 An example of an unexpected change would be an unrelated \
+                 test that is now failing, which would be a regression."
+            ),
+            "Test results differed from baseline in {old_path:?}. The \
+             differences are described above."
+        ))
+    } else {
+        Ok(())
+    }
+}

xtask/src/complement/test2json.rs

@@ -14,7 +14,7 @@ use std::{
 use indicatif::{ProgressBar, ProgressStyle};
 use miette::{miette, IntoDiagnostic, LabeledSpan, Result, WrapErr};
 use serde::Deserialize;
-use strum::Display;
+use strum::{Display, EnumString};
 use xshell::{cmd, Shell};
 
 use super::summary::{write_summary, TestResults};
@@ -43,7 +43,7 @@ pub(crate) fn run_complement(
     out: &Path,
     docker_image: &str,
     test_count: u64,
-) -> Result<()> {
+) -> Result<TestResults> {
     // TODO: handle SIG{INT,TERM}
     // TODO: XTASK_PATH variable, so that we don't need to pollute devshell with
     // go
@@ -70,7 +70,7 @@ pub(crate) fn run_complement(
         ctx.handle_line(&line)?;
     }
 
-    Ok(())
+    Ok(ctx.results)
 }
 
 /// Schema from <https://pkg.go.dev/cmd/test2json#hdr-Output_Format>
@@ -103,7 +103,7 @@ enum GoTestEvent {
     OtherAction,
 }
 
-#[derive(Copy, Clone, Display, Debug)]
+#[derive(Copy, Clone, Display, EnumString, Eq, PartialEq, Debug)]
 #[strum(serialize_all = "UPPERCASE")]
 pub(crate) enum TestResult {
     Pass,