Fix spans in tokio::spawn-ed tasks

tokio::spawn is a span boundary, the spawned future has no parent span.

For short futures, we simply inherit the current span with
`.in_current_span()`.

For long running futures containing a sleeping infinite loop, we don't
actually want a span on the entire task or even the entire loop body,
both would result in very long spans. Instead, we put the outermost span
(created using #[tracing::instrument] or .instrument()) around the
actual work happening after the sleep, which results in a new root span
being created after every sleep.
This commit is contained in:
Lambda 2024-05-19 21:39:13 +00:00
parent 5e9e5b76bc
commit ac42e0bfff
3 changed files with 109 additions and 70 deletions

View file

@ -100,6 +100,7 @@ pub(crate) struct Service {
>,
}
#[derive(Debug)]
enum TransactionStatus {
Running,
// number of times failed, time of last failure
@ -114,6 +115,12 @@ struct HandlerInputs {
}
type HandlerResponse = Result<OutgoingKind, (OutgoingKind, Error)>;
fn outgoing_kind_from_response(response: &HandlerResponse) -> &OutgoingKind {
match response {
Ok(kind) | Err((kind, _)) => kind,
}
}
type TransactionStatusMap = HashMap<OutgoingKind, TransactionStatus>;
impl Service {
@ -197,6 +204,14 @@ impl Service {
}
}
#[tracing::instrument(
skip(self, current_transaction_status),
fields(
current_status = ?current_transaction_status.get(
outgoing_kind_from_response(&response)
),
),
)]
fn handle_futures(
&self,
response: HandlerResponse,
@ -259,6 +274,12 @@ impl Service {
}
}
#[tracing::instrument(
skip(self, event, key, current_transaction_status),
fields(
current_status = ?current_transaction_status.get(&outgoing_kind),
),
)]
fn handle_receiver(
&self,
outgoing_kind: OutgoingKind,