v0.3.14-pre.009

crates/ksp-worker-raw-transaction-ingest-lib/unit_tests/continuity.rs

@@ -1,5 +1,5 @@
 // file: crates/ksp-worker-raw-transaction-ingest-lib/unit_tests/continuity.rs
-// version: 6
+// version: 7
 
 fn network() -> std::option::Option<ksp_store_lib::RawNetworkId> {
     return match ksp_store_lib::RawNetworkId::new("mainnet") {
@@ -566,3 +566,84 @@ fn pre_008_known_reference_missing_moves_to_continuity_ledger_and_reconciles_fro
     assert_eq!(contracts.continuity_frontier(std::option::Option::Some(120)), std::option::Option::Some(120));
     return;
 }
+
+#[test]
+fn pre_009_health_projection_distinguishes_future_coverage_from_open_gap_reconciliation() {
+    let first = match capability(1, ksp_onchain_transport_lib::SolanaCommitment::Confirmed, exact_scope("standard_logs", 7)) {
+        std::option::Option::Some(value) => value,
+        std::option::Option::None => panic!("first capability fixture unavailable"),
+    };
+    let second = match capability(2, ksp_onchain_transport_lib::SolanaCommitment::Confirmed, exact_scope("standard_logs", 7)) {
+        std::option::Option::Some(value) => value,
+        std::option::Option::None => panic!("second capability fixture unavailable"),
+    };
+    let mut contracts = match crate::RawTransactionIngestContinuityContracts::new(std::vec![first, second]) {
+        std::result::Result::Ok(value) => value,
+        std::result::Result::Err(error) => panic!("continuity contracts fixture failed: {error}"),
+    };
+    let healthy = match contracts.health_projection(&[[1_u8; 32], [2_u8; 32]], &[], std::option::Option::Some(120)) {
+        std::result::Result::Ok(value) => value,
+        std::result::Result::Err(error) => panic!("health projection failed: {error}"),
+    };
+    assert_eq!(healthy, (std::option::Option::Some(120), false, true, true));
+    let redundant_future = match contracts.health_projection(&[[2_u8; 32]], &[[1_u8; 32]], std::option::Option::Some(120)) {
+        std::result::Result::Ok(value) => value,
+        std::result::Result::Err(error) => panic!("redundant health projection failed: {error}"),
+    };
+    assert_eq!(redundant_future, (std::option::Option::Some(120), false, true, false));
+    assert!(contracts.record_source_loss_gap([1_u8; 32], 100, 110).is_ok());
+    let pending = match contracts.health_projection(&[[2_u8; 32]], &[[1_u8; 32]], std::option::Option::Some(120)) {
+        std::result::Result::Ok(value) => value,
+        std::result::Result::Err(error) => panic!("pending health projection failed: {error}"),
+    };
+    assert_eq!(pending, (std::option::Option::Some(99), true, true, false));
+    assert!(contracts.record_coverage_epoch([2_u8; 32], 90, 120).is_ok());
+    let reconciled = match contracts.health_projection(&[[2_u8; 32]], &[[1_u8; 32]], std::option::Option::Some(120)) {
+        std::result::Result::Ok(value) => value,
+        std::result::Result::Err(error) => panic!("reconciled health projection failed: {error}"),
+    };
+    assert_eq!(reconciled, (std::option::Option::Some(120), false, true, true));
+    let uncovered_future = match contracts.health_projection(&[], &[], std::option::Option::Some(120)) {
+        std::result::Result::Ok(value) => value,
+        std::result::Result::Err(error) => panic!("uncovered health projection failed: {error}"),
+    };
+    assert_eq!(uncovered_future, (std::option::Option::Some(120), false, false, true));
+    return;
+}
+
+#[test]
+fn pre_009_health_projection_rejects_duplicate_and_unknown_active_sources() {
+    let capability = match capability(1, ksp_onchain_transport_lib::SolanaCommitment::Confirmed, exact_scope("standard_logs", 7)) {
+        std::option::Option::Some(value) => value,
+        std::option::Option::None => panic!("capability fixture unavailable"),
+    };
+    let mut contracts = match crate::RawTransactionIngestContinuityContracts::new(std::vec![capability]) {
+        std::result::Result::Ok(value) => value,
+        std::result::Result::Err(error) => panic!("continuity contracts fixture failed: {error}"),
+    };
+    let duplicate = match contracts.health_projection(&[[1_u8; 32], [1_u8; 32]], &[], std::option::Option::Some(20)) {
+        std::result::Result::Ok(_) => return,
+        std::result::Result::Err(value) => value,
+    };
+    assert_eq!(duplicate.code(), crate::ERROR_CODE_RAW_TRANSACTION_INGEST_RUNTIME_INVALID);
+    assert!(duplicate.context().iter().any(|context| return context.value() == "continuity.health_active_set_invalid"));
+    let unknown = match contracts.health_projection(&[[9_u8; 32]], &[], std::option::Option::Some(20)) {
+        std::result::Result::Ok(_) => return,
+        std::result::Result::Err(value) => value,
+    };
+    assert_eq!(unknown.code(), crate::ERROR_CODE_RAW_TRANSACTION_INGEST_RUNTIME_INVALID);
+    assert!(unknown.context().iter().any(|context| return context.value() == "continuity.health_active_source_unknown"));
+    let overlap = match contracts.health_projection(&[[1_u8; 32]], &[[1_u8; 32]], std::option::Option::Some(20)) {
+        std::result::Result::Ok(_) => return,
+        std::result::Result::Err(value) => value,
+    };
+    assert_eq!(overlap.code(), crate::ERROR_CODE_RAW_TRANSACTION_INGEST_RUNTIME_INVALID);
+    assert!(overlap.context().iter().any(|context| return context.value() == "continuity.health_failed_set_invalid"));
+    let unknown_failed = match contracts.health_projection(&[], &[[9_u8; 32]], std::option::Option::Some(20)) {
+        std::result::Result::Ok(_) => return,
+        std::result::Result::Err(value) => value,
+    };
+    assert_eq!(unknown_failed.code(), crate::ERROR_CODE_RAW_TRANSACTION_INGEST_RUNTIME_INVALID);
+    assert!(unknown_failed.context().iter().any(|context| return context.value() == "continuity.health_failed_source_unknown"));
+    return;
+}

crates/ksp-worker-raw-transaction-ingest-lib/unit_tests/runtime_resources.rs

@@ -1,5 +1,5 @@
 // file: crates/ksp-worker-raw-transaction-ingest-lib/unit_tests/runtime_resources.rs
-// version: 30
+// version: 31
 
 fn grpc_endpoint(cluster: &str) -> std::option::Option<ksp_onchain_transport_lib::YellowstoneGrpcEndpointSettings> {
     return grpc_endpoint_with_identity(cluster, "yellowstone-fixture", "fixture-provider");
@@ -1051,15 +1051,21 @@ async fn v0_3_13_pre_002_runtime_resources_reject_duplicate_and_cross_network_so
 #[test]
 fn v0_3_13_pre_007_source_inventory_aggregates_frontier_and_lifecycle_conservatively() {
     let source_keys = std::vec![[1_u8; 32], [2_u8; 32]];
+    let continuity_contracts = match supervisor_contracts(&[(1, "fixture-a", 1), (2, "fixture-b", 2)]) {
+        std::option::Option::Some(value) => value,
+        std::option::Option::None => return,
+    };
     let inventory = std::sync::Arc::new(std::sync::Mutex::new(super::RawTransactionIngestSourceInventory::new(source_keys)));
     let (aggregate_sender, aggregate_receiver) = tokio::sync::watch::channel(crate::RawTransactionIngestProcessingFrontierProjection::empty());
     let first = super::RawTransactionIngestSourceInventoryPublisher {
         aggregate_sender: aggregate_sender.clone(),
+        continuity_contracts: std::sync::Arc::clone(&continuity_contracts),
         entry_index: 0,
         inventory: std::sync::Arc::clone(&inventory),
         source_key: [1_u8; 32],
     };
-    let second = super::RawTransactionIngestSourceInventoryPublisher { aggregate_sender, entry_index: 1, inventory, source_key: [2_u8; 32] };
+    let second =
+        super::RawTransactionIngestSourceInventoryPublisher { aggregate_sender, continuity_contracts, entry_index: 1, inventory, source_key: [2_u8; 32] };
     assert!(
         first
             .publish(
@@ -1077,6 +1083,8 @@ fn v0_3_13_pre_007_source_inventory_aggregates_frontier_and_lifecycle_conservati
     assert_eq!(first_aggregate.source_active(), 1);
     assert_eq!(first_aggregate.source_reconnecting(), 0);
     assert_eq!(first_aggregate.source_failed(), 0);
+    assert!(first_aggregate.continuity_policy_observed());
+    assert!(!first_aggregate.future_target_coverage());
     assert!(
         second
             .publish(
@@ -1097,6 +1105,9 @@ fn v0_3_13_pre_007_source_inventory_aggregates_frontier_and_lifecycle_conservati
     assert_eq!(aggregate.source_reconnect_total(), 4);
     assert_eq!(aggregate.source_replay_attempt_total(), 6);
     assert_eq!(aggregate.source_continuity_gap_total(), 1);
+    assert!(aggregate.continuity_policy_observed());
+    assert!(!aggregate.continuity_has_open_gaps());
+    assert!(!aggregate.future_target_coverage());
     return;
 }
 
@@ -1137,7 +1148,15 @@ async fn v0_3_13_pre_007_source_supervisor_joins_all_children_on_stop() {
         });
     }
     let supervisor = tokio::spawn(async move {
-        return super::supervise_live_source_tasks(stop_receiver, source_stop_sender, children, contracts, inventory).await;
+        return super::supervise_live_source_tasks(
+            stop_receiver,
+            source_stop_sender,
+            children,
+            contracts,
+            inventory,
+            tokio::sync::watch::channel(crate::RawTransactionIngestProcessingFrontierProjection::empty()).0,
+        )
+        .await;
     });
     for _ in 0..64 {
         if active.load(std::sync::atomic::Ordering::Acquire) == 3 {
@@ -1197,7 +1216,15 @@ async fn v0_3_13_pre_007_source_failure_stops_and_joins_sibling_sources() {
             }
         }
     });
-    let result = super::supervise_live_source_tasks(stop_receiver, source_stop_sender, children, contracts, inventory).await;
+    let result = super::supervise_live_source_tasks(
+        stop_receiver,
+        source_stop_sender,
+        children,
+        contracts,
+        inventory,
+        tokio::sync::watch::channel(crate::RawTransactionIngestProcessingFrontierProjection::empty()).0,
+    )
+    .await;
     assert!(result.is_err());
     assert!(!sibling_active.load(std::sync::atomic::Ordering::Acquire));
     return;
@@ -3804,7 +3831,15 @@ async fn v0_3_13_pre_011_aborting_outer_source_supervisor_aborts_nested_source_t
         return std::future::pending::<([u8; 32], ksp_core_lib::Result<()>)>().await;
     });
     let supervisor = tokio::spawn(async move {
-        return super::supervise_live_source_tasks(stop_receiver, source_stop_sender, children, contracts, inventory).await;
+        return super::supervise_live_source_tasks(
+            stop_receiver,
+            source_stop_sender,
+            children,
+            contracts,
+            inventory,
+            tokio::sync::watch::channel(crate::RawTransactionIngestProcessingFrontierProjection::empty()).0,
+        )
+        .await;
     });
     for _ in 0..64 {
         if active.load(std::sync::atomic::Ordering::Acquire) == 1 {
@@ -3863,7 +3898,15 @@ async fn v0_3_13_pre_011_stop_racing_ready_source_fault_preserves_fault_and_join
         }
     });
     let supervisor = tokio::spawn(async move {
-        return super::supervise_live_source_tasks(stop_receiver, source_stop_sender, children, contracts, inventory).await;
+        return super::supervise_live_source_tasks(
+            stop_receiver,
+            source_stop_sender,
+            children,
+            contracts,
+            inventory,
+            tokio::sync::watch::channel(crate::RawTransactionIngestProcessingFrontierProjection::empty()).0,
+        )
+        .await;
     });
     for _ in 0..64 {
         if sibling_active.load(std::sync::atomic::Ordering::Acquire) == 1 {
@@ -3929,18 +3972,22 @@ async fn v0_3_14_pre_008_proven_full_ledger_epoch_survives_filtered_peer_loss_wi
             }
         }
     });
+    let (health_sender, health_receiver) = tokio::sync::watch::channel(crate::RawTransactionIngestProcessingFrontierProjection::empty());
     let supervisor = tokio::spawn(async move {
-        return super::supervise_live_source_tasks(stop_receiver, source_stop_sender, children, contracts, inventory).await;
+        return super::supervise_live_source_tasks(stop_receiver, source_stop_sender, children, contracts, inventory, health_sender).await;
     });
     for _ in 0..64 {
-        if sibling_active.load(std::sync::atomic::Ordering::Acquire) {
+        if sibling_active.load(std::sync::atomic::Ordering::Acquire) && health_receiver.borrow().failed_source_losses_reconciled() {
             break;
         }
         tokio::task::yield_now().await;
     }
     assert!(sibling_active.load(std::sync::atomic::Ordering::Acquire));
-    tokio::task::yield_now().await;
-    assert!(sibling_active.load(std::sync::atomic::Ordering::Acquire));
+    let health = *health_receiver.borrow();
+    assert!(health.continuity_policy_observed());
+    assert!(!health.continuity_has_open_gaps());
+    assert!(health.future_target_coverage());
+    assert!(health.failed_source_losses_reconciled());
     stop_sender.send_replace(true);
     let result = match supervisor.await {
         std::result::Result::Ok(value) => value,
@@ -4036,3 +4083,62 @@ fn v0_3_14_pre_003_websocket_incident_without_observed_slot_is_unbounded_and_cou
     assert!(reporter.observe_websocket_continuity(crate::RawTransactionIngestSourceState::Active, 0, 0).is_err());
     return;
 }
+
+#[test]
+fn v0_3_14_pre_009_inventory_health_projection_tracks_reconciled_coverage() {
+    let source_keys = std::vec![[1_u8; 32], [2_u8; 32]];
+    let continuity_contracts = match supervisor_contracts(&[(1, "standard_logs", 7), (2, "standard_logs", 7)]) {
+        std::option::Option::Some(value) => value,
+        std::option::Option::None => return,
+    };
+    let inventory = std::sync::Arc::new(std::sync::Mutex::new(super::RawTransactionIngestSourceInventory::new(source_keys)));
+    let (aggregate_sender, aggregate_receiver) = tokio::sync::watch::channel(crate::RawTransactionIngestProcessingFrontierProjection::empty());
+    let first = super::RawTransactionIngestSourceInventoryPublisher {
+        aggregate_sender: aggregate_sender.clone(),
+        continuity_contracts: std::sync::Arc::clone(&continuity_contracts),
+        entry_index: 0,
+        inventory: std::sync::Arc::clone(&inventory),
+        source_key: [1_u8; 32],
+    };
+    let second = super::RawTransactionIngestSourceInventoryPublisher {
+        aggregate_sender,
+        continuity_contracts: std::sync::Arc::clone(&continuity_contracts),
+        entry_index: 1,
+        inventory,
+        source_key: [2_u8; 32],
+    };
+    let active = crate::RawTransactionIngestProcessingFrontierProjection::new(0, std::option::Option::Some(60), std::option::Option::None)
+        .with_source_continuity(std::option::Option::Some(crate::RawTransactionIngestSourceState::Active), 0, 0, 0);
+    assert!(first.publish(active).is_ok());
+    assert!(second.publish(active).is_ok());
+    let healthy = *aggregate_receiver.borrow();
+    assert!(healthy.continuity_policy_observed());
+    assert_eq!(healthy.continuity_frontier_slot(), std::option::Option::Some(60));
+    assert!(!healthy.continuity_has_open_gaps());
+    assert!(healthy.future_target_coverage());
+    {
+        let mut contracts = match continuity_contracts.lock() {
+            std::result::Result::Ok(value) => value,
+            std::result::Result::Err(poisoned) => poisoned.into_inner(),
+        };
+        assert!(contracts.record_source_loss_gap([2_u8; 32], 50, 55).is_ok());
+    }
+    assert!(first.publish(active).is_ok());
+    let pending = *aggregate_receiver.borrow();
+    assert_eq!(pending.continuity_frontier_slot(), std::option::Option::Some(49));
+    assert!(pending.continuity_has_open_gaps());
+    assert!(pending.future_target_coverage());
+    {
+        let mut contracts = match continuity_contracts.lock() {
+            std::result::Result::Ok(value) => value,
+            std::result::Result::Err(poisoned) => poisoned.into_inner(),
+        };
+        assert!(contracts.record_coverage_epoch([1_u8; 32], 40, 60).is_ok());
+    }
+    assert!(first.publish(active).is_ok());
+    let reconciled = *aggregate_receiver.borrow();
+    assert_eq!(reconciled.continuity_frontier_slot(), std::option::Option::Some(60));
+    assert!(!reconciled.continuity_has_open_gaps());
+    assert!(reconciled.future_target_coverage());
+    return;
+}

crates/ksp-worker-raw-transaction-ingest-lib/unit_tests/snapshot.rs

@@ -1,5 +1,5 @@
 // file: crates/ksp-worker-raw-transaction-ingest-lib/unit_tests/snapshot.rs
-// version: 5
+// version: 6
 
 fn snapshot_foundation_with_source_total(
     source_total: usize,
@@ -249,3 +249,58 @@ fn v0_3_13_pre_010_multi_source_counts_and_health_are_conservative_and_source_ne
     assert_eq!(unhealthy.worker_snapshot().health(), ksp_worker_api::WorkerHealth::Unhealthy);
     return;
 }
+
+#[test]
+fn v0_3_14_pre_009_health_requires_present_and_future_coverage_before_healthy() {
+    let (mut publisher, source) = match snapshot_foundation_with_source_total(2) {
+        std::option::Option::Some(value) => value,
+        std::option::Option::None => return,
+    };
+    let reconnecting = crate::RawTransactionIngestProcessingFrontierProjection::new(0, std::option::Option::Some(50), std::option::Option::None)
+        .with_source_continuity(std::option::Option::Some(crate::RawTransactionIngestSourceState::Reconnecting), 1, 1, 1)
+        .with_source_counts(2, 1, 1, 0)
+        .with_continuity_health(std::option::Option::Some(50), false, true)
+        .with_failed_source_losses_reconciled(true);
+    assert!(publisher.record_processing_frontier(ksp_worker_api::WorkerState::Running, 0, 0, reconnecting).is_ok());
+    assert_eq!(source.current().worker_snapshot().health(), ksp_worker_api::WorkerHealth::Unhealthy);
+    let gap_pending = crate::RawTransactionIngestProcessingFrontierProjection::new(0, std::option::Option::Some(50), std::option::Option::None)
+        .with_source_continuity(std::option::Option::Some(crate::RawTransactionIngestSourceState::Active), 1, 1, 1)
+        .with_source_counts(2, 2, 0, 0)
+        .with_continuity_health(std::option::Option::Some(49), true, true)
+        .with_failed_source_losses_reconciled(true);
+    assert!(publisher.record_processing_frontier(ksp_worker_api::WorkerState::Running, 0, 0, gap_pending).is_ok());
+    assert_eq!(source.current().worker_snapshot().health(), ksp_worker_api::WorkerHealth::Unhealthy);
+    let healthy = crate::RawTransactionIngestProcessingFrontierProjection::new(0, std::option::Option::Some(50), std::option::Option::None)
+        .with_source_continuity(std::option::Option::Some(crate::RawTransactionIngestSourceState::Active), 1, 1, 1)
+        .with_source_counts(2, 2, 0, 0)
+        .with_continuity_health(std::option::Option::Some(50), false, true)
+        .with_failed_source_losses_reconciled(true);
+    assert!(publisher.record_processing_frontier(ksp_worker_api::WorkerState::Running, 0, 0, healthy).is_ok());
+    assert_eq!(source.current().worker_snapshot().health(), ksp_worker_api::WorkerHealth::Healthy);
+    let failed_unreconciled = crate::RawTransactionIngestProcessingFrontierProjection::new(0, std::option::Option::Some(50), std::option::Option::None)
+        .with_source_continuity(std::option::Option::Some(crate::RawTransactionIngestSourceState::Failed), 1, 1, 1)
+        .with_source_counts(2, 1, 0, 1)
+        .with_continuity_health(std::option::Option::Some(50), false, true)
+        .with_failed_source_losses_reconciled(false);
+    assert!(publisher.record_processing_frontier(ksp_worker_api::WorkerState::Running, 0, 0, failed_unreconciled).is_ok());
+    assert_eq!(source.current().worker_snapshot().health(), ksp_worker_api::WorkerHealth::Unhealthy);
+    let degraded = crate::RawTransactionIngestProcessingFrontierProjection::new(0, std::option::Option::Some(50), std::option::Option::None)
+        .with_source_continuity(std::option::Option::Some(crate::RawTransactionIngestSourceState::Failed), 1, 1, 1)
+        .with_source_counts(2, 1, 0, 1)
+        .with_continuity_health(std::option::Option::Some(50), false, true)
+        .with_failed_source_losses_reconciled(true);
+    assert!(publisher.record_processing_frontier(ksp_worker_api::WorkerState::Running, 0, 0, degraded).is_ok());
+    assert_eq!(source.current().worker_snapshot().health(), ksp_worker_api::WorkerHealth::Degraded);
+    let future_uncovered = crate::RawTransactionIngestProcessingFrontierProjection::new(0, std::option::Option::Some(50), std::option::Option::None)
+        .with_source_continuity(std::option::Option::Some(crate::RawTransactionIngestSourceState::Failed), 1, 1, 1)
+        .with_source_counts(2, 1, 0, 1)
+        .with_continuity_health(std::option::Option::Some(50), false, false)
+        .with_failed_source_losses_reconciled(true);
+    assert!(publisher.record_processing_frontier(ksp_worker_api::WorkerState::Running, 0, 0, future_uncovered).is_ok());
+    assert_eq!(source.current().worker_snapshot().health(), ksp_worker_api::WorkerHealth::Unhealthy);
+    assert!(publisher.publish_state(ksp_worker_api::WorkerState::Faulted(crate::ERROR_CODE_RAW_TRANSACTION_INGEST_SOURCE_FAILED), 0, 0).is_ok());
+    let faulted = source.current();
+    assert!(matches!(faulted.worker_snapshot().state(), ksp_worker_api::WorkerState::Faulted(_)));
+    assert_eq!(faulted.worker_snapshot().health(), ksp_worker_api::WorkerHealth::Unhealthy);
+    return;
+}