From 836b8fea8d1886a16cc1b103079901a7a34bb68d Mon Sep 17 00:00:00 2001 From: Ian Clarke Date: Sat, 1 Nov 2025 01:49:05 +0100 Subject: [PATCH 1/2] fix: guard op state timeout notifications --- crates/core/src/node/op_state_manager.rs | 77 +++++++++++++++++++++++- 1 file changed, 75 insertions(+), 2 deletions(-) diff --git a/crates/core/src/node/op_state_manager.rs b/crates/core/src/node/op_state_manager.rs index 541a71c27..4ce6a9e95 100644 --- a/crates/core/src/node/op_state_manager.rs +++ b/crates/core/src/node/op_state_manager.rs @@ -331,6 +331,27 @@ impl OpManager { } } +async fn notify_transaction_timeout( + event_loop_notifier: &EventLoopNotificationsSender, + tx: Transaction, +) -> bool { + match event_loop_notifier + .notifications_sender + .send(Either::Right(NodeEvent::TransactionTimedOut(tx))) + .await + { + Ok(()) => true, + Err(err) => { + tracing::debug!( + tx = %tx, + error = ?err, + "Failed to notify event loop about timed out transaction; receiver likely dropped" + ); + false + } + } +} + async fn garbage_cleanup_task( mut new_transactions: tokio::sync::mpsc::Receiver, ops: Arc, @@ -376,7 +397,7 @@ async fn garbage_cleanup_task( ops.under_progress.remove(&tx); ops.completed.remove(&tx); tracing::debug!("Transaction timed out: {tx}"); - event_loop_notifier.notifications_sender.send(Either::Right(NodeEvent::TransactionTimedOut(tx))).await.unwrap(); + notify_transaction_timeout(&event_loop_notifier, tx).await; live_tx_tracker.remove_finished_transaction(tx); } } @@ -405,7 +426,7 @@ async fn garbage_cleanup_task( }; if removed { tracing::debug!("Transaction timed out: {tx}"); - event_loop_notifier.notifications_sender.send(Either::Right(NodeEvent::TransactionTimedOut(tx))).await.unwrap(); + notify_transaction_timeout(&event_loop_notifier, tx).await; live_tx_tracker.remove_finished_transaction(tx); } } @@ -413,3 +434,55 @@ async fn garbage_cleanup_task( } } } + +#[cfg(test)] +mod tests { + use super::super::network_bridge::event_loop_notification_channel; + use super::*; + use crate::node::network_bridge::EventLoopNotificationsReceiver; + use either::Either; + use tokio::time::{timeout, Duration}; + + #[tokio::test] + async fn notify_timeout_succeeds_when_receiver_alive() { + let (receiver, notifier) = event_loop_notification_channel(); + let EventLoopNotificationsReceiver { + mut notifications_receiver, + .. + } = receiver; + + let tx = Transaction::ttl_transaction(); + + let delivered = notify_transaction_timeout(¬ifier, tx).await; + assert!( + delivered, + "notification should be delivered while receiver is alive" + ); + + let received = timeout(Duration::from_millis(100), notifications_receiver.recv()) + .await + .expect("timed out waiting for notification") + .expect("notification channel closed"); + + match received { + Either::Right(NodeEvent::TransactionTimedOut(observed)) => { + assert_eq!(observed, tx, "unexpected transaction in notification"); + } + other => panic!("unexpected notification: {other:?}"), + } + } + + #[tokio::test] + async fn notify_timeout_handles_dropped_receiver() { + let (receiver, notifier) = event_loop_notification_channel(); + drop(receiver); + + let tx = Transaction::ttl_transaction(); + + let delivered = notify_transaction_timeout(¬ifier, tx).await; + assert!( + !delivered, + "notification delivery should fail once receiver is dropped" + ); + } +} From 09ac66f950c5fd892ca3c79f147840534a8ba5c1 Mon Sep 17 00:00:00 2001 From: "claude[bot]" <41898282+claude[bot]@users.noreply.github.com> Date: Sat, 1 Nov 2025 05:26:32 +0000 Subject: [PATCH 2/2] chore: change timeout notification log level to warn Co-authored-by: nacho.d.g --- crates/core/src/node/op_state_manager.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/core/src/node/op_state_manager.rs b/crates/core/src/node/op_state_manager.rs index 4ce6a9e95..4018a6141 100644 --- a/crates/core/src/node/op_state_manager.rs +++ b/crates/core/src/node/op_state_manager.rs @@ -342,7 +342,7 @@ async fn notify_transaction_timeout( { Ok(()) => true, Err(err) => { - tracing::debug!( + tracing::warn!( tx = %tx, error = ?err, "Failed to notify event loop about timed out transaction; receiver likely dropped"