v0.3.13-pre.011

crates/ksp-worker-raw-transaction-ingest-lib/unit_tests/runtime.rs

@@ -1,5 +1,5 @@
 // file: crates/ksp-worker-raw-transaction-ingest-lib/unit_tests/runtime.rs
-// version: 10
+// version: 11
 
 struct ActiveTaskGuard {
     active: std::sync::Arc<std::sync::atomic::AtomicUsize>,
@@ -899,3 +899,98 @@ async fn pre_009_drain_timeout_aborts_and_joins_all_owned_source_and_persistence
     assert_eq!(port.completed.load(std::sync::atomic::Ordering::Acquire), 0);
     return;
 }
+
+#[tokio::test(flavor = "current_thread")]
+async fn v0_3_13_pre_011_source_counter_exhaustion_stays_terminal_and_is_not_collapsed_to_source_failed() {
+    let settings = match settings("mainnet") {
+        std::option::Option::Some(value) => value,
+        std::option::Option::None => return,
+    };
+    let handle = match super::start_foundation_with_source_spawner(
+        settings,
+        tokio::runtime::Handle::current(),
+        std::option::Option::None,
+        move |children: &mut tokio::task::JoinSet<ksp_core_lib::Result<()>>, _stop_receiver, _admission_sender| {
+            let _abort_handle = children.spawn(async move {
+                return std::result::Result::Err(crate::counter_exhausted_error("source_reconnect_total"));
+            });
+        },
+    ) {
+        std::result::Result::Ok(value) => value,
+        std::result::Result::Err(_) => return,
+    };
+    let source = handle.snapshot_source();
+    let terminal = match handle.wait_terminal().await {
+        std::result::Result::Ok(value) => value,
+        std::result::Result::Err(_) => return,
+    };
+    assert_eq!(terminal, ksp_worker_api::WorkerState::Faulted(crate::ERROR_CODE_RAW_TRANSACTION_INGEST_COUNTER_EXHAUSTED));
+    let snapshot = source.current();
+    assert_eq!(snapshot.source_failure_total(), 1);
+    assert_eq!(snapshot.worker_snapshot().state(), terminal);
+    assert_eq!(snapshot.worker_snapshot().health(), ksp_worker_api::WorkerHealth::Unhealthy);
+    let terminal_sequence = snapshot.worker_snapshot().sequence();
+    assert!(!handle.request_stop());
+    let retained = source.current();
+    assert_eq!(retained.worker_snapshot().state(), terminal);
+    assert_eq!(retained.worker_snapshot().sequence(), terminal_sequence);
+    return;
+}
+
+#[tokio::test(flavor = "current_thread")]
+async fn v0_3_13_pre_011_drain_timeout_prevents_late_persistence_completion_after_terminal() {
+    let settings = match settings_with_runtime_limits(1, 1, crate::MIN_RAW_TRANSACTION_INGEST_SHUTDOWN_DRAIN_TIMEOUT) {
+        std::option::Option::Some(value) => value,
+        std::option::Option::None => return,
+    };
+    let network = settings.network().clone();
+    let port = std::sync::Arc::new(RuntimePersistencePort::new(network.clone(), RuntimePortResponse::SuccessBlocked, false));
+    let runtime_port: super::PersistencePort = port.clone();
+    let source_active = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0));
+    let source_active_for_task = std::sync::Arc::clone(&source_active);
+    let handle = match super::start_foundation_with_port_and_source_spawner(
+        settings,
+        tokio::runtime::Handle::current(),
+        std::option::Option::Some(runtime_port),
+        move |children, _stop_receiver, admission_sender| {
+            let _abort_handle = children.spawn(async move {
+                let ingress = match runtime_ingress(&network, 81) {
+                    std::option::Option::Some(value) => value,
+                    std::option::Option::None => return std::result::Result::Err(crate::runtime_error("test.ingress_invalid")),
+                };
+                if admission_sender.send(ingress).await.is_err() {
+                    return std::result::Result::Err(crate::runtime_error("test.source_failed"));
+                }
+                let _guard = ActiveTaskGuard::new(source_active_for_task);
+                return std::future::pending::<ksp_core_lib::Result<()>>().await;
+            });
+        },
+    ) {
+        std::result::Result::Ok(value) => value,
+        std::result::Result::Err(_) => return,
+    };
+    assert!(wait_for_max_active(&port, 1).await);
+    assert!(wait_for_active_count(&source_active, 1).await);
+    assert!(handle.request_stop());
+    let terminal = match handle.wait_terminal().await {
+        std::result::Result::Ok(value) => value,
+        std::result::Result::Err(_) => return,
+    };
+    assert_eq!(terminal, ksp_worker_api::WorkerState::Faulted(crate::ERROR_CODE_RAW_TRANSACTION_INGEST_DRAIN_TIMEOUT));
+    let snapshot = handle.snapshot_source().current();
+    let terminal_sequence = snapshot.worker_snapshot().sequence();
+    assert_eq!(snapshot.in_flight_persistence(), 0);
+    assert_eq!(port.active.load(std::sync::atomic::Ordering::Acquire), 0);
+    assert_eq!(port.completed.load(std::sync::atomic::Ordering::Acquire), 0);
+    assert_eq!(source_active.load(std::sync::atomic::Ordering::Acquire), 0);
+    port.release();
+    for _ in 0..64 {
+        tokio::task::yield_now().await;
+    }
+    assert_eq!(port.completed.load(std::sync::atomic::Ordering::Acquire), 0);
+    let retained = handle.snapshot_source().current();
+    assert_eq!(retained.worker_snapshot().state(), terminal);
+    assert_eq!(retained.worker_snapshot().sequence(), terminal_sequence);
+    assert_eq!(retained.in_flight_persistence(), 0);
+    return;
+}