v0.3.11-pre.008
This commit is contained in:
@@ -1,5 +1,5 @@
|
||||
// file: crates/ksp-worker-raw-transaction-ingest-lib/unit_tests/runtime.rs
|
||||
// version: 4
|
||||
// version: 5
|
||||
|
||||
struct ActiveTaskGuard {
|
||||
active: std::sync::Arc<std::sync::atomic::AtomicUsize>,
|
||||
@@ -88,7 +88,7 @@ async fn pre_004_immediate_stop_is_idempotent_and_reaches_stopped_terminal() {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(_) => return,
|
||||
};
|
||||
assert_eq!(*handle.terminal_receiver.borrow(), ksp_worker_api::WorkerState::Starting);
|
||||
assert_eq!(handle.snapshot_source().current().worker_snapshot().state(), ksp_worker_api::WorkerState::Starting);
|
||||
assert!(handle.request_stop());
|
||||
assert!(!handle.request_stop());
|
||||
let terminal = handle.wait_terminal().await;
|
||||
@@ -113,7 +113,7 @@ async fn pre_004_running_lifecycle_stops_through_cloned_handle() {
|
||||
std::result::Result::Err(_) => return,
|
||||
};
|
||||
tokio::task::yield_now().await;
|
||||
assert_eq!(*handle.terminal_receiver.borrow(), ksp_worker_api::WorkerState::Running);
|
||||
assert_eq!(handle.snapshot_source().current().worker_snapshot().state(), ksp_worker_api::WorkerState::Running);
|
||||
let clone = handle.clone();
|
||||
assert!(clone.request_stop());
|
||||
assert!(!handle.request_stop());
|
||||
@@ -136,21 +136,15 @@ async fn pre_004_dropping_last_control_handle_causes_private_runtime_exit() {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(_) => return,
|
||||
};
|
||||
let mut observer = handle.terminal_receiver.clone();
|
||||
let observer = handle.snapshot_source();
|
||||
std::mem::drop(handle);
|
||||
loop {
|
||||
let current = *observer.borrow();
|
||||
if current.is_terminal() {
|
||||
let closed = observer.changed().await;
|
||||
assert!(closed.is_err());
|
||||
assert_eq!(current, ksp_worker_api::WorkerState::Stopped);
|
||||
return;
|
||||
}
|
||||
let changed = observer.changed().await;
|
||||
assert!(changed.is_ok(), "runtime closed before publishing terminal state");
|
||||
if changed.is_err() {
|
||||
let current = observer.current();
|
||||
if current.worker_snapshot().state().is_terminal() {
|
||||
assert_eq!(current.worker_snapshot().state(), ksp_worker_api::WorkerState::Stopped);
|
||||
return;
|
||||
}
|
||||
let _changed = observer.wait_for_change(current.worker_snapshot().sequence()).await;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -485,6 +479,7 @@ async fn pre_007_content_conflict_becomes_terminal_after_private_drain() {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(_) => return,
|
||||
};
|
||||
let source = handle.snapshot_source();
|
||||
let terminal = handle.wait_terminal().await;
|
||||
let terminal = match terminal {
|
||||
std::result::Result::Ok(value) => value,
|
||||
@@ -492,6 +487,10 @@ async fn pre_007_content_conflict_becomes_terminal_after_private_drain() {
|
||||
};
|
||||
assert_eq!(terminal, ksp_worker_api::WorkerState::Faulted(crate::ERROR_CODE_RAW_TRANSACTION_INGEST_CONTENT_CONFLICT));
|
||||
assert_eq!(port.completed.load(std::sync::atomic::Ordering::Acquire), 1);
|
||||
let snapshot = source.current();
|
||||
assert_eq!(snapshot.content_conflict_total(), 1);
|
||||
assert_eq!(snapshot.store_failure_total(), 0);
|
||||
assert_eq!(snapshot.worker_snapshot().health(), ksp_worker_api::WorkerHealth::Unhealthy);
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -532,3 +531,115 @@ async fn pre_007_store_failure_becomes_terminal_without_exposing_remote_error_te
|
||||
assert!(!std::format!("{handle:?}").contains("synthetic store failure"));
|
||||
return;
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "current_thread")]
|
||||
async fn pre_008_runtime_snapshots_count_successful_pipeline_and_retain_terminal() {
|
||||
let settings = match settings_with_persistence_concurrency(2) {
|
||||
std::option::Option::Some(value) => value,
|
||||
std::option::Option::None => return,
|
||||
};
|
||||
let network = settings.network().clone();
|
||||
let port = std::sync::Arc::new(RuntimePersistencePort::new(network.clone(), RuntimePortResponse::SuccessBlocked, true));
|
||||
let runtime_port: super::PersistencePort = port.clone();
|
||||
let handle = match super::start_foundation_with_port_and_source_spawner(
|
||||
settings,
|
||||
tokio::runtime::Handle::current(),
|
||||
std::option::Option::Some(runtime_port),
|
||||
move |children, _stop_receiver, admission_sender| {
|
||||
let _abort_handle = children.spawn(async move {
|
||||
for signature_byte in 21..=23 {
|
||||
let ingress = match runtime_ingress(&network, signature_byte) {
|
||||
std::option::Option::Some(value) => value,
|
||||
std::option::Option::None => return,
|
||||
};
|
||||
if admission_sender.send(ingress).await.is_err() {
|
||||
return;
|
||||
}
|
||||
}
|
||||
return;
|
||||
});
|
||||
},
|
||||
) {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(_) => return,
|
||||
};
|
||||
let concrete_source = handle.snapshot_source();
|
||||
let common_source = handle.worker_snapshot_source();
|
||||
assert!(wait_for_completed(&port, 3).await);
|
||||
assert!(handle.request_stop());
|
||||
let terminal = handle.wait_terminal().await;
|
||||
let terminal = match terminal {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(_) => return,
|
||||
};
|
||||
assert_eq!(terminal, ksp_worker_api::WorkerState::Stopped);
|
||||
let snapshot = concrete_source.current();
|
||||
assert_eq!(snapshot.worker_snapshot().state(), ksp_worker_api::WorkerState::Stopped);
|
||||
assert_eq!(snapshot.worker_snapshot().health(), ksp_worker_api::WorkerHealth::Healthy);
|
||||
assert_eq!(snapshot.worker_snapshot().activity(), ksp_worker_api::WorkerActivity::Idle);
|
||||
assert_eq!(snapshot.admitted_total(), 3);
|
||||
assert_eq!(snapshot.canonicalized_total(), 3);
|
||||
assert_eq!(snapshot.persisted_total(), 3);
|
||||
assert_eq!(snapshot.entity_inserted_total(), 3);
|
||||
assert_eq!(snapshot.entity_already_present_total(), 0);
|
||||
assert_eq!(snapshot.entity_skipped_purged_total(), 0);
|
||||
assert_eq!(snapshot.observation_inserted_total(), 3);
|
||||
assert_eq!(snapshot.observation_already_present_total(), 0);
|
||||
assert_eq!(snapshot.content_conflict_total(), 0);
|
||||
assert_eq!(snapshot.store_failure_total(), 0);
|
||||
assert_eq!(snapshot.source_failure_total(), 0);
|
||||
assert_eq!(snapshot.backpressure_wait_total(), 0);
|
||||
assert_eq!(snapshot.admission_queue_depth(), 0);
|
||||
assert_eq!(snapshot.in_flight_persistence(), 0);
|
||||
let common = ksp_worker_api::WorkerSnapshotSource::current(&common_source);
|
||||
assert_eq!(&common, snapshot.worker_snapshot());
|
||||
let retained = concrete_source.wait_for_change(snapshot.worker_snapshot().sequence()).await;
|
||||
assert_eq!(retained, snapshot);
|
||||
return;
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "current_thread")]
|
||||
async fn pre_008_fault_snapshots_count_classified_store_failures_and_project_unhealthy() {
|
||||
let settings = match settings_with_persistence_concurrency(1) {
|
||||
std::option::Option::Some(value) => value,
|
||||
std::option::Option::None => return,
|
||||
};
|
||||
let network = settings.network().clone();
|
||||
let port = std::sync::Arc::new(RuntimePersistencePort::new(network.clone(), RuntimePortResponse::StoreFailure, true));
|
||||
let runtime_port: super::PersistencePort = port.clone();
|
||||
let handle = match super::start_foundation_with_port_and_source_spawner(
|
||||
settings,
|
||||
tokio::runtime::Handle::current(),
|
||||
std::option::Option::Some(runtime_port),
|
||||
move |children, _stop_receiver, admission_sender| {
|
||||
let _abort_handle = children.spawn(async move {
|
||||
let ingress = match runtime_ingress(&network, 24) {
|
||||
std::option::Option::Some(value) => value,
|
||||
std::option::Option::None => return,
|
||||
};
|
||||
let _sent = admission_sender.send(ingress).await;
|
||||
return;
|
||||
});
|
||||
},
|
||||
) {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(_) => return,
|
||||
};
|
||||
let source = handle.snapshot_source();
|
||||
let terminal = handle.wait_terminal().await;
|
||||
let terminal = match terminal {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(_) => return,
|
||||
};
|
||||
assert_eq!(terminal, ksp_worker_api::WorkerState::Faulted(crate::ERROR_CODE_RAW_TRANSACTION_INGEST_STORE_FAILED));
|
||||
let snapshot = source.current();
|
||||
assert_eq!(snapshot.worker_snapshot().state(), terminal);
|
||||
assert_eq!(snapshot.worker_snapshot().health(), ksp_worker_api::WorkerHealth::Unhealthy);
|
||||
assert_eq!(snapshot.worker_snapshot().activity(), ksp_worker_api::WorkerActivity::Idle);
|
||||
assert_eq!(snapshot.admitted_total(), 1);
|
||||
assert_eq!(snapshot.canonicalized_total(), 1);
|
||||
assert_eq!(snapshot.persisted_total(), 0);
|
||||
assert_eq!(snapshot.store_failure_total(), 1);
|
||||
assert_eq!(snapshot.content_conflict_total(), 0);
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,115 @@
|
||||
// file: crates/ksp-worker-raw-transaction-ingest-lib/unit_tests/snapshot.rs
|
||||
// version: 1
|
||||
|
||||
fn snapshot_foundation() -> std::option::Option<(crate::RawTransactionIngestSnapshotPublisher, crate::RawTransactionIngestSnapshotSource)> {
|
||||
let network = match ksp_store_lib::RawNetworkId::new("mainnet") {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(_) => return std::option::Option::None,
|
||||
};
|
||||
let worker_id = match ksp_worker_api::WorkerId::new("raw-ingest-snapshot-001") {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(_) => return std::option::Option::None,
|
||||
};
|
||||
let settings = crate::RawTransactionIngestSettings::with_defaults(network, worker_id.clone());
|
||||
let kind = match ksp_worker_api::WorkerKindCode::new(crate::RAW_TRANSACTION_INGEST_WORKER_KIND_CODE) {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(_) => return std::option::Option::None,
|
||||
};
|
||||
let mut lifecycle = ksp_worker_api::WorkerLifecycle::new(worker_id, kind);
|
||||
if lifecycle.start().is_err() {
|
||||
return std::option::Option::None;
|
||||
}
|
||||
return std::option::Option::Some(crate::RawTransactionIngestSnapshotPublisher::new(&settings, &lifecycle));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn pre_008_initial_snapshot_and_common_projection_are_exact() {
|
||||
let (publisher, source) = match snapshot_foundation() {
|
||||
std::option::Option::Some(value) => value,
|
||||
std::option::Option::None => return,
|
||||
};
|
||||
let _publisher = publisher;
|
||||
let concrete = source.current();
|
||||
assert_eq!(concrete.worker_snapshot().sequence().value(), 0);
|
||||
assert_eq!(concrete.worker_snapshot().state(), ksp_worker_api::WorkerState::Starting);
|
||||
assert_eq!(concrete.worker_snapshot().health(), ksp_worker_api::WorkerHealth::Unknown);
|
||||
assert_eq!(concrete.worker_snapshot().activity(), ksp_worker_api::WorkerActivity::Unknown);
|
||||
assert_eq!(concrete.admission_queue_capacity(), crate::DEFAULT_RAW_TRANSACTION_INGEST_ADMISSION_QUEUE_CAPACITY);
|
||||
assert_eq!(concrete.admission_queue_depth(), 0);
|
||||
assert_eq!(concrete.persistence_concurrency(), crate::DEFAULT_RAW_TRANSACTION_INGEST_PERSISTENCE_CONCURRENCY);
|
||||
assert_eq!(concrete.in_flight_persistence(), 0);
|
||||
assert_eq!(concrete.admitted_total(), 0);
|
||||
assert_eq!(concrete.canonicalized_total(), 0);
|
||||
assert_eq!(concrete.persisted_total(), 0);
|
||||
assert_eq!(concrete.entity_inserted_total(), 0);
|
||||
assert_eq!(concrete.entity_already_present_total(), 0);
|
||||
assert_eq!(concrete.entity_skipped_purged_total(), 0);
|
||||
assert_eq!(concrete.observation_inserted_total(), 0);
|
||||
assert_eq!(concrete.observation_already_present_total(), 0);
|
||||
assert_eq!(concrete.content_conflict_total(), 0);
|
||||
assert_eq!(concrete.store_failure_total(), 0);
|
||||
assert_eq!(concrete.source_failure_total(), 0);
|
||||
assert_eq!(concrete.backpressure_wait_total(), 0);
|
||||
let common = ksp_worker_api::WorkerSnapshotSource::current(&source);
|
||||
assert_eq!(&common, concrete.worker_snapshot());
|
||||
return;
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn pre_008_checked_counter_exhaustion_is_stable_and_never_wraps() {
|
||||
let result = super::checked_counter(u64::MAX, "admitted_total");
|
||||
assert!(result.is_err());
|
||||
let error = match result {
|
||||
std::result::Result::Ok(_) => return,
|
||||
std::result::Result::Err(value) => value,
|
||||
};
|
||||
assert_eq!(error.code(), crate::ERROR_CODE_RAW_TRANSACTION_INGEST_COUNTER_EXHAUSTED);
|
||||
assert_eq!(error.context().len(), 1);
|
||||
assert_eq!(error.context()[0].key(), "field");
|
||||
assert_eq!(error.context()[0].value(), "admitted_total");
|
||||
return;
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "current_thread")]
|
||||
async fn pre_008_slow_concrete_and_common_listeners_coalesce_to_same_latest_sequence() {
|
||||
let (mut publisher, source) = match snapshot_foundation() {
|
||||
std::option::Option::Some(value) => value,
|
||||
std::option::Option::None => return,
|
||||
};
|
||||
let observed = source.current().worker_snapshot().sequence();
|
||||
assert!(publisher.publish_state(ksp_worker_api::WorkerState::Running, 0, 0).is_ok());
|
||||
assert!(publisher.record_admission_success(ksp_worker_api::WorkerState::Running, 1, 1).is_ok());
|
||||
let concrete = source.wait_for_change(observed).await;
|
||||
let common = ksp_worker_api::WorkerSnapshotSource::wait_for_change(&source, observed).await;
|
||||
assert_eq!(concrete.worker_snapshot().sequence(), common.sequence());
|
||||
assert_eq!(concrete.worker_snapshot().sequence().value(), 2);
|
||||
assert_eq!(concrete.worker_snapshot().state(), ksp_worker_api::WorkerState::Running);
|
||||
assert_eq!(concrete.worker_snapshot().health(), ksp_worker_api::WorkerHealth::Healthy);
|
||||
assert_eq!(concrete.worker_snapshot().activity(), ksp_worker_api::WorkerActivity::Active);
|
||||
assert_eq!(concrete.admission_queue_depth(), 1);
|
||||
assert_eq!(concrete.in_flight_persistence(), 1);
|
||||
assert_eq!(concrete.admitted_total(), 1);
|
||||
assert_eq!(concrete.canonicalized_total(), 1);
|
||||
return;
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "current_thread")]
|
||||
async fn pre_008_terminal_latest_value_is_retained_after_publisher_drop() {
|
||||
let (mut publisher, source) = match snapshot_foundation() {
|
||||
std::option::Option::Some(value) => value,
|
||||
std::option::Option::None => return,
|
||||
};
|
||||
assert!(publisher.publish_state(ksp_worker_api::WorkerState::Running, 0, 0).is_ok());
|
||||
assert!(publisher.publish_state(ksp_worker_api::WorkerState::Stopping, 0, 0).is_ok());
|
||||
assert!(publisher.publish_state(ksp_worker_api::WorkerState::Stopped, 0, 0).is_ok());
|
||||
std::mem::drop(publisher);
|
||||
let retained = source.current();
|
||||
assert_eq!(retained.worker_snapshot().state(), ksp_worker_api::WorkerState::Stopped);
|
||||
assert_eq!(retained.worker_snapshot().health(), ksp_worker_api::WorkerHealth::Healthy);
|
||||
assert_eq!(retained.worker_snapshot().activity(), ksp_worker_api::WorkerActivity::Idle);
|
||||
let same_terminal = source.wait_for_change(retained.worker_snapshot().sequence()).await;
|
||||
assert_eq!(same_terminal, retained);
|
||||
let common = ksp_worker_api::WorkerSnapshotSource::wait_for_change(&source, retained.worker_snapshot().sequence()).await;
|
||||
assert_eq!(common, retained.worker_snapshot().clone());
|
||||
return;
|
||||
}
|
||||
Reference in New Issue
Block a user