// file: crates/ksp-worker-raw-transaction-ingest-lib/unit_tests/snapshot.rs // version: 5 fn snapshot_foundation_with_source_total( source_total: usize, ) -> std::option::Option<(crate::RawTransactionIngestSnapshotPublisher, crate::RawTransactionIngestSnapshotSource)> { let network = match ksp_store_lib::RawNetworkId::new("mainnet") { std::result::Result::Ok(value) => value, std::result::Result::Err(_) => return std::option::Option::None, }; let worker_id = match ksp_worker_api::WorkerId::new("raw-ingest-snapshot-001") { std::result::Result::Ok(value) => value, std::result::Result::Err(_) => return std::option::Option::None, }; let settings = crate::RawTransactionIngestSettings::with_defaults(network, worker_id.clone()); let kind = match ksp_worker_api::WorkerKindCode::new(crate::RAW_TRANSACTION_INGEST_WORKER_KIND_CODE) { std::result::Result::Ok(value) => value, std::result::Result::Err(_) => return std::option::Option::None, }; let mut lifecycle = ksp_worker_api::WorkerLifecycle::new(worker_id, kind); if lifecycle.start().is_err() { return std::option::Option::None; } return std::option::Option::Some(crate::RawTransactionIngestSnapshotPublisher::new(&settings, &lifecycle, source_total)); } fn snapshot_foundation() -> std::option::Option<(crate::RawTransactionIngestSnapshotPublisher, crate::RawTransactionIngestSnapshotSource)> { return snapshot_foundation_with_source_total(0); } #[test] fn pre_008_initial_snapshot_and_common_projection_are_exact() { let (publisher, source) = match snapshot_foundation() { std::option::Option::Some(value) => value, std::option::Option::None => return, }; let _publisher = publisher; let concrete = source.current(); assert_eq!(concrete.worker_snapshot().sequence().value(), 0); assert_eq!(concrete.worker_snapshot().state(), ksp_worker_api::WorkerState::Starting); assert_eq!(concrete.worker_snapshot().health(), ksp_worker_api::WorkerHealth::Unknown); assert_eq!(concrete.worker_snapshot().activity(), ksp_worker_api::WorkerActivity::Unknown); assert_eq!(concrete.admission_queue_capacity(), crate::DEFAULT_RAW_TRANSACTION_INGEST_ADMISSION_QUEUE_CAPACITY); assert_eq!(concrete.admission_queue_depth(), 0); assert_eq!(concrete.persistence_concurrency(), crate::DEFAULT_RAW_TRANSACTION_INGEST_PERSISTENCE_CONCURRENCY); assert_eq!(concrete.in_flight_persistence(), 0); assert_eq!(concrete.admitted_total(), 0); assert_eq!(concrete.canonicalized_total(), 0); assert_eq!(concrete.persisted_total(), 0); assert_eq!(concrete.entity_inserted_total(), 0); assert_eq!(concrete.entity_already_present_total(), 0); assert_eq!(concrete.entity_skipped_purged_total(), 0); assert_eq!(concrete.observation_inserted_total(), 0); assert_eq!(concrete.observation_already_present_total(), 0); assert_eq!(concrete.content_conflict_total(), 0); assert_eq!(concrete.store_failure_total(), 0); assert_eq!(concrete.source_failure_total(), 0); assert_eq!(concrete.backpressure_wait_total(), 0); assert_eq!(concrete.hydration_pending(), 0); assert_eq!(concrete.processing_frontier_slot(), std::option::Option::None); assert_eq!(concrete.oldest_pending_slot(), std::option::Option::None); assert_eq!(concrete.source_state(), std::option::Option::None); assert_eq!(concrete.source_total(), 0); assert_eq!(concrete.source_active(), 0); assert_eq!(concrete.source_reconnecting(), 0); assert_eq!(concrete.source_failed(), 0); assert_eq!(concrete.source_reconnect_total(), 0); assert_eq!(concrete.source_replay_attempt_total(), 0); assert_eq!(concrete.source_continuity_gap_total(), 0); let common = ksp_worker_api::WorkerSnapshotSource::current(&source); assert_eq!(&common, concrete.worker_snapshot()); return; } #[test] fn pre_008_checked_counter_exhaustion_is_stable_and_never_wraps() { let result = super::checked_counter(u64::MAX, "admitted_total"); assert!(result.is_err()); let error = match result { std::result::Result::Ok(_) => return, std::result::Result::Err(value) => value, }; assert_eq!(error.code(), crate::ERROR_CODE_RAW_TRANSACTION_INGEST_COUNTER_EXHAUSTED); assert_eq!(error.context().len(), 1); assert_eq!(error.context()[0].key(), "field"); assert_eq!(error.context()[0].value(), "admitted_total"); return; } #[tokio::test(flavor = "current_thread")] async fn pre_008_slow_concrete_and_common_listeners_coalesce_to_same_latest_sequence() { let (mut publisher, source) = match snapshot_foundation() { std::option::Option::Some(value) => value, std::option::Option::None => return, }; let observed = source.current().worker_snapshot().sequence(); assert!(publisher.publish_state(ksp_worker_api::WorkerState::Running, 0, 0).is_ok()); assert!(publisher.record_admission_success(ksp_worker_api::WorkerState::Running, 1, 1, false).is_ok()); let concrete = source.wait_for_change(observed).await; let common = ksp_worker_api::WorkerSnapshotSource::wait_for_change(&source, observed).await; assert_eq!(concrete.worker_snapshot().sequence(), common.sequence()); assert_eq!(concrete.worker_snapshot().sequence().value(), 2); assert_eq!(concrete.worker_snapshot().state(), ksp_worker_api::WorkerState::Running); assert_eq!(concrete.worker_snapshot().health(), ksp_worker_api::WorkerHealth::Healthy); assert_eq!(concrete.worker_snapshot().activity(), ksp_worker_api::WorkerActivity::Active); assert_eq!(concrete.admission_queue_depth(), 1); assert_eq!(concrete.in_flight_persistence(), 1); assert_eq!(concrete.admitted_total(), 1); assert_eq!(concrete.canonicalized_total(), 1); return; } #[tokio::test(flavor = "current_thread")] async fn pre_008_terminal_latest_value_is_retained_after_publisher_drop() { let (mut publisher, source) = match snapshot_foundation() { std::option::Option::Some(value) => value, std::option::Option::None => return, }; assert!(publisher.publish_state(ksp_worker_api::WorkerState::Running, 0, 0).is_ok()); assert!(publisher.publish_state(ksp_worker_api::WorkerState::Stopping, 0, 0).is_ok()); assert!(publisher.publish_state(ksp_worker_api::WorkerState::Stopped, 0, 0).is_ok()); std::mem::drop(publisher); let retained = source.current(); assert_eq!(retained.worker_snapshot().state(), ksp_worker_api::WorkerState::Stopped); assert_eq!(retained.worker_snapshot().health(), ksp_worker_api::WorkerHealth::Healthy); assert_eq!(retained.worker_snapshot().activity(), ksp_worker_api::WorkerActivity::Idle); let same_terminal = source.wait_for_change(retained.worker_snapshot().sequence()).await; assert_eq!(same_terminal, retained); let common = ksp_worker_api::WorkerSnapshotSource::wait_for_change(&source, retained.worker_snapshot().sequence()).await; assert_eq!(common, retained.worker_snapshot().clone()); return; } #[test] fn pre_009_source_failure_and_backpressure_counters_advance_without_changing_projection_contract() { let (mut publisher, source) = match snapshot_foundation() { std::option::Option::Some(value) => value, std::option::Option::None => return, }; assert!(publisher.publish_state(ksp_worker_api::WorkerState::Running, 0, 0).is_ok()); assert!(publisher.record_admission_success(ksp_worker_api::WorkerState::Running, 0, 1, true).is_ok()); assert!(publisher.record_source_failure(ksp_worker_api::WorkerState::Running, 0, 1).is_ok()); let concrete = source.current(); assert_eq!(concrete.admitted_total(), 1); assert_eq!(concrete.canonicalized_total(), 1); assert_eq!(concrete.backpressure_wait_total(), 1); assert_eq!(concrete.source_failure_total(), 1); assert_eq!(concrete.worker_snapshot().health(), ksp_worker_api::WorkerHealth::Healthy); assert_eq!(concrete.worker_snapshot().activity(), ksp_worker_api::WorkerActivity::Active); let common = ksp_worker_api::WorkerSnapshotSource::current(&source); assert_eq!(&common, concrete.worker_snapshot()); return; } #[test] fn pre_007_processing_frontier_projection_is_latest_value_and_activity_aware() { let (mut publisher, source) = match snapshot_foundation() { std::option::Option::Some(value) => value, std::option::Option::None => return, }; let initial = source.current(); assert_eq!(initial.hydration_pending(), 0); assert_eq!(initial.processing_frontier_slot(), std::option::Option::None); assert_eq!(initial.oldest_pending_slot(), std::option::Option::None); let projection = crate::RawTransactionIngestProcessingFrontierProjection::new(2, std::option::Option::Some(40), std::option::Option::Some(41)); assert!(publisher.record_processing_frontier(ksp_worker_api::WorkerState::Running, 0, 0, projection).is_ok()); let pending = source.current(); assert_eq!(pending.hydration_pending(), 2); assert_eq!(pending.processing_frontier_slot(), std::option::Option::Some(40)); assert_eq!(pending.oldest_pending_slot(), std::option::Option::Some(41)); assert_eq!(pending.worker_snapshot().activity(), ksp_worker_api::WorkerActivity::Active); let projection = crate::RawTransactionIngestProcessingFrontierProjection::new(0, std::option::Option::Some(45), std::option::Option::None); assert!(publisher.record_processing_frontier(ksp_worker_api::WorkerState::Running, 0, 0, projection).is_ok()); let settled = source.current(); assert_eq!(settled.hydration_pending(), 0); assert_eq!(settled.processing_frontier_slot(), std::option::Option::Some(45)); assert_eq!(settled.oldest_pending_slot(), std::option::Option::None); assert_eq!(settled.worker_snapshot().activity(), ksp_worker_api::WorkerActivity::Idle); return; } #[test] fn pre_008_source_continuity_projection_preserves_processing_frontier_and_distinguishes_replay() { let (mut publisher, source) = match snapshot_foundation() { std::option::Option::Some(value) => value, std::option::Option::None => return, }; let projection = crate::RawTransactionIngestProcessingFrontierProjection::new(2, std::option::Option::Some(40), std::option::Option::Some(41)) .with_source_continuity(std::option::Option::Some(crate::RawTransactionIngestSourceState::Reconnecting), 0, 1, 0); assert!(publisher.record_processing_frontier(ksp_worker_api::WorkerState::Running, 0, 0, projection).is_ok()); let reconnecting = source.current(); assert_eq!(reconnecting.hydration_pending(), 2); assert_eq!(reconnecting.processing_frontier_slot(), std::option::Option::Some(40)); assert_eq!(reconnecting.oldest_pending_slot(), std::option::Option::Some(41)); assert_eq!(reconnecting.source_state(), std::option::Option::Some(crate::RawTransactionIngestSourceState::Reconnecting)); assert_eq!(reconnecting.source_reconnect_total(), 0); assert_eq!(reconnecting.source_replay_attempt_total(), 1); assert_eq!(reconnecting.source_continuity_gap_total(), 0); assert_eq!(reconnecting.worker_snapshot().activity(), ksp_worker_api::WorkerActivity::Active); let projection = crate::RawTransactionIngestProcessingFrontierProjection::new(0, std::option::Option::Some(45), std::option::Option::None) .with_source_continuity(std::option::Option::Some(crate::RawTransactionIngestSourceState::Active), 1, 1, 0); assert!(publisher.record_processing_frontier(ksp_worker_api::WorkerState::Running, 0, 0, projection).is_ok()); let resumed = source.current(); assert_eq!(resumed.hydration_pending(), 0); assert_eq!(resumed.processing_frontier_slot(), std::option::Option::Some(45)); assert_eq!(resumed.source_state(), std::option::Option::Some(crate::RawTransactionIngestSourceState::Active)); assert_eq!(resumed.source_reconnect_total(), 1); assert_eq!(resumed.source_replay_attempt_total(), 1); assert_eq!(resumed.worker_snapshot().activity(), ksp_worker_api::WorkerActivity::Idle); return; } #[test] fn v0_3_13_pre_010_multi_source_counts_and_health_are_conservative_and_source_neutral() { let (mut publisher, source) = match snapshot_foundation_with_source_total(3) { std::option::Option::Some(value) => value, std::option::Option::None => return, }; let initial = source.current(); assert_eq!(initial.source_total(), 3); assert_eq!(initial.source_active(), 0); assert_eq!(initial.source_reconnecting(), 0); assert_eq!(initial.source_failed(), 0); assert_eq!(initial.worker_snapshot().health(), ksp_worker_api::WorkerHealth::Unknown); let reconnecting = crate::RawTransactionIngestProcessingFrontierProjection::new(1, std::option::Option::Some(50), std::option::Option::Some(50)) .with_source_continuity(std::option::Option::Some(crate::RawTransactionIngestSourceState::Reconnecting), 0, 1, 0) .with_source_counts(3, 2, 1, 0); assert!(publisher.record_processing_frontier(ksp_worker_api::WorkerState::Running, 0, 0, reconnecting).is_ok()); let degraded = source.current(); assert_eq!(degraded.source_total(), 3); assert_eq!(degraded.source_active(), 2); assert_eq!(degraded.source_reconnecting(), 1); assert_eq!(degraded.source_failed(), 0); assert_eq!(degraded.worker_snapshot().health(), ksp_worker_api::WorkerHealth::Degraded); assert_eq!(degraded.worker_snapshot().activity(), ksp_worker_api::WorkerActivity::Active); let active = crate::RawTransactionIngestProcessingFrontierProjection::new(0, std::option::Option::Some(51), std::option::Option::None) .with_source_continuity(std::option::Option::Some(crate::RawTransactionIngestSourceState::Active), 1, 1, 0) .with_source_counts(3, 3, 0, 0); assert!(publisher.record_processing_frontier(ksp_worker_api::WorkerState::Running, 0, 0, active).is_ok()); let healthy = source.current(); assert_eq!(healthy.worker_snapshot().health(), ksp_worker_api::WorkerHealth::Healthy); assert_eq!(healthy.worker_snapshot().activity(), ksp_worker_api::WorkerActivity::Idle); let failed = crate::RawTransactionIngestProcessingFrontierProjection::new(0, std::option::Option::Some(51), std::option::Option::None) .with_source_continuity(std::option::Option::Some(crate::RawTransactionIngestSourceState::Failed), 1, 1, 0) .with_source_counts(3, 2, 0, 1); assert!(publisher.record_processing_frontier(ksp_worker_api::WorkerState::Running, 0, 0, failed).is_ok()); let unhealthy = source.current(); assert_eq!(unhealthy.source_failed(), 1); assert_eq!(unhealthy.worker_snapshot().health(), ksp_worker_api::WorkerHealth::Unhealthy); return; }