log when servers switch between online and offline

This commit is contained in:
Olivia Lee 2024-08-25 01:01:02 -07:00
parent 080fe5af42
commit 5b6aaa19b9
No known key found for this signature in database
GPG key ID: 54D568A15B9CD1F9

View file

@ -6,7 +6,7 @@ use std::{
use rand::{thread_rng, Rng}; use rand::{thread_rng, Rng};
use ruma::{OwnedServerName, ServerName}; use ruma::{OwnedServerName, ServerName};
use tracing::{debug, instrument}; use tracing::{debug, info, instrument};
use crate::{services, Error, Result}; use crate::{services, Error, Result};
@ -169,6 +169,12 @@ impl BackoffState {
let delay = Duration::from_secs_f64(delay_secs); let delay = Duration::from_secs_f64(delay_secs);
delay.checked_sub(last_failure.elapsed()) delay.checked_sub(last_failure.elapsed())
} }
/// Returns whether this server is marked as online (no backoff delay).
fn is_online(&self) -> bool {
let config = &services().globals.config.federation.backoff;
self.failure_count <= config.failure_threshold
}
} }
impl BackoffGuard { impl BackoffGuard {
@ -176,6 +182,7 @@ impl BackoffGuard {
#[instrument(skip(self))] #[instrument(skip(self))]
pub(crate) fn success(self) { pub(crate) fn success(self) {
let mut state = self.backoff.write().unwrap(); let mut state = self.backoff.write().unwrap();
let was_online = state.is_online();
if state.failure_count != 0 { if state.failure_count != 0 {
debug!( debug!(
@ -185,6 +192,11 @@ impl BackoffGuard {
} }
state.failure_count = 0; state.failure_count = 0;
if state.is_online() != was_online {
let server_name = &state.server_name;
info!(%server_name, "server transitioned from offline to online");
}
} }
/// Record a failed request indicating that the server may be unavailable. /// Record a failed request indicating that the server may be unavailable.
@ -196,6 +208,7 @@ impl BackoffGuard {
let config = &services().globals.config.federation.backoff; let config = &services().globals.config.federation.backoff;
let mut state = self.backoff.write().unwrap(); let mut state = self.backoff.write().unwrap();
let was_online = state.is_online();
if state.last_failure == self.last_failure { if state.last_failure == self.last_failure {
state.failure_count = state.failure_count.saturating_add(1); state.failure_count = state.failure_count.saturating_add(1);
@ -208,6 +221,11 @@ impl BackoffGuard {
failure_count = state.failure_count, failure_count = state.failure_count,
"hard failure sending request to server, incrementing failure count" "hard failure sending request to server, incrementing failure count"
); );
if state.is_online() != was_online {
let server_name = &state.server_name;
info!(%server_name, "server transitioned from online to offline");
}
} }
} }