add option for which errors will cause a backoff for outgoing requests

src/service/sending.rs

@@ -118,11 +118,29 @@ pub(crate) struct RequestData {
     requester_span: Span,
 }
 
+/// Which types of errors should cause us to backoff requests to this server
+/// globally.
+///
+/// The default is [`BackoffOn::AllExceptWellFormed`], which is conservative,
+/// with a high false negative rate and low false positive rate. For endpoints
+/// where we have additional information, we should pick a less conservative
+/// setting.
+#[derive(Copy, Clone, Debug)]
+pub(crate) enum BackoffOn {
+    /// All errors except for error responses that match the expected
+    /// `{ "errcode": ... }` format for the matrix protocol.
+    AllExceptWellFormed,
+
+    /// All errors
+    AllErrors,
+}
+
 #[must_use = "The request builder must be awaited for the request to be sent"]
 pub(crate) struct SendFederationRequestBuilder<'a, T> {
     destination: &'a ServerName,
     request: T,
     log_errors: LogRequestError,
+    backoff_on: BackoffOn,
 }
 
 pub(crate) struct Service {
@@ -688,6 +706,7 @@ impl Service {
             destination,
             request,
             log_errors: LogRequestError::Yes,
+            backoff_on: BackoffOn::AllExceptWellFormed,
         }
     }
 
@@ -725,6 +744,13 @@ impl<T> SendFederationRequestBuilder<'_, T> {
         self.log_errors = log_errors;
         self
     }
+
+    /// Set the types of errors that will cause us to backoff future requests to
+    /// this server globally.
+    pub(crate) fn backoff_on(mut self, backoff_on: BackoffOn) -> Self {
+        self.backoff_on = backoff_on;
+        self
+    }
 }
 
 impl<'a, T> IntoFuture for SendFederationRequestBuilder<'a, T>
@@ -771,14 +797,15 @@ where
             match &response {
                 Err(Error::Federation(_, error)) => {
                     if error.error_kind().is_some() {
-                        // Other errors may occur during normal operation with a
-                        // healthy server, so don't increment the failure
-                        // counter.
-                        backoff_guard.soft_failure();
+                        if let BackoffOn::AllExceptWellFormed = self.backoff_on
+                        {
+                            backoff_guard.soft_failure();
+                        } else {
+                            backoff_guard.hard_failure();
+                        }
                     } else {
                         // The error wasn't in the expected format for matrix
-                        // API responses. This almost certainly indicates the
-                        // server is unhealthy or offline.
+                        // API responses.
                         backoff_guard.hard_failure();
                     }
                 }