349 lines
21 KiB
Rust
349 lines
21 KiB
Rust
// file: crates/ksp-worker-raw-transaction-ingest-lib/unit_tests/snapshot.rs
|
|
// version: 7
|
|
|
|
fn snapshot_foundation_with_source_total(
|
|
source_total: usize,
|
|
) -> std::option::Option<(crate::RawTransactionIngestSnapshotPublisher, crate::RawTransactionIngestSnapshotSource)> {
|
|
let network = match ksp_store_lib::RawNetworkId::new("mainnet") {
|
|
std::result::Result::Ok(value) => value,
|
|
std::result::Result::Err(_) => return std::option::Option::None,
|
|
};
|
|
let worker_id = match ksp_worker_api::WorkerId::new("raw-ingest-snapshot-001") {
|
|
std::result::Result::Ok(value) => value,
|
|
std::result::Result::Err(_) => return std::option::Option::None,
|
|
};
|
|
let settings = crate::RawTransactionIngestSettings::with_defaults(network, worker_id.clone());
|
|
let kind = match ksp_worker_api::WorkerKindCode::new(crate::RAW_TRANSACTION_INGEST_WORKER_KIND_CODE) {
|
|
std::result::Result::Ok(value) => value,
|
|
std::result::Result::Err(_) => return std::option::Option::None,
|
|
};
|
|
let mut lifecycle = ksp_worker_api::WorkerLifecycle::new(worker_id, kind);
|
|
if lifecycle.start().is_err() {
|
|
return std::option::Option::None;
|
|
}
|
|
return std::option::Option::Some(crate::RawTransactionIngestSnapshotPublisher::new(&settings, &lifecycle, source_total));
|
|
}
|
|
|
|
fn snapshot_foundation() -> std::option::Option<(crate::RawTransactionIngestSnapshotPublisher, crate::RawTransactionIngestSnapshotSource)> {
|
|
return snapshot_foundation_with_source_total(0);
|
|
}
|
|
|
|
#[test]
|
|
fn pre_008_initial_snapshot_and_common_projection_are_exact() {
|
|
let (publisher, source) = match snapshot_foundation() {
|
|
std::option::Option::Some(value) => value,
|
|
std::option::Option::None => return,
|
|
};
|
|
let _publisher = publisher;
|
|
let concrete = source.current();
|
|
assert_eq!(concrete.worker_snapshot().sequence().value(), 0);
|
|
assert_eq!(concrete.worker_snapshot().state(), ksp_worker_api::WorkerState::Starting);
|
|
assert_eq!(concrete.worker_snapshot().health(), ksp_worker_api::WorkerHealth::Unknown);
|
|
assert_eq!(concrete.worker_snapshot().activity(), ksp_worker_api::WorkerActivity::Unknown);
|
|
assert_eq!(concrete.admission_queue_capacity(), crate::DEFAULT_RAW_TRANSACTION_INGEST_ADMISSION_QUEUE_CAPACITY);
|
|
assert_eq!(concrete.admission_queue_depth(), 0);
|
|
assert_eq!(concrete.persistence_concurrency(), crate::DEFAULT_RAW_TRANSACTION_INGEST_PERSISTENCE_CONCURRENCY);
|
|
assert_eq!(concrete.in_flight_persistence(), 0);
|
|
assert_eq!(concrete.admitted_total(), 0);
|
|
assert_eq!(concrete.canonicalized_total(), 0);
|
|
assert_eq!(concrete.persisted_total(), 0);
|
|
assert_eq!(concrete.entity_inserted_total(), 0);
|
|
assert_eq!(concrete.entity_already_present_total(), 0);
|
|
assert_eq!(concrete.entity_skipped_purged_total(), 0);
|
|
assert_eq!(concrete.observation_inserted_total(), 0);
|
|
assert_eq!(concrete.observation_already_present_total(), 0);
|
|
assert_eq!(concrete.content_conflict_total(), 0);
|
|
assert_eq!(concrete.store_failure_total(), 0);
|
|
assert_eq!(concrete.source_failure_total(), 0);
|
|
assert_eq!(concrete.backpressure_wait_total(), 0);
|
|
assert_eq!(concrete.hydration_pending(), 0);
|
|
assert_eq!(concrete.processing_frontier_slot(), std::option::Option::None);
|
|
assert_eq!(concrete.oldest_pending_slot(), std::option::Option::None);
|
|
assert_eq!(concrete.source_state(), std::option::Option::None);
|
|
assert_eq!(concrete.source_total(), 0);
|
|
assert_eq!(concrete.source_active(), 0);
|
|
assert_eq!(concrete.source_reconnecting(), 0);
|
|
assert_eq!(concrete.source_failed(), 0);
|
|
assert_eq!(concrete.source_reconnect_total(), 0);
|
|
assert_eq!(concrete.source_replay_attempt_total(), 0);
|
|
assert_eq!(concrete.source_continuity_gap_total(), 0);
|
|
let common = ksp_worker_api::WorkerSnapshotSource::current(&source);
|
|
assert_eq!(&common, concrete.worker_snapshot());
|
|
return;
|
|
}
|
|
|
|
#[test]
|
|
fn pre_008_checked_counter_exhaustion_is_stable_and_never_wraps() {
|
|
let result = super::checked_counter(u64::MAX, "admitted_total");
|
|
assert!(result.is_err());
|
|
let error = match result {
|
|
std::result::Result::Ok(_) => return,
|
|
std::result::Result::Err(value) => value,
|
|
};
|
|
assert_eq!(error.code(), crate::ERROR_CODE_RAW_TRANSACTION_INGEST_COUNTER_EXHAUSTED);
|
|
assert_eq!(error.context().len(), 1);
|
|
assert_eq!(error.context()[0].key(), "field");
|
|
assert_eq!(error.context()[0].value(), "admitted_total");
|
|
return;
|
|
}
|
|
|
|
#[tokio::test(flavor = "current_thread")]
|
|
async fn pre_008_slow_concrete_and_common_listeners_coalesce_to_same_latest_sequence() {
|
|
let (mut publisher, source) = match snapshot_foundation() {
|
|
std::option::Option::Some(value) => value,
|
|
std::option::Option::None => return,
|
|
};
|
|
let observed = source.current().worker_snapshot().sequence();
|
|
assert!(publisher.publish_state(ksp_worker_api::WorkerState::Running, 0, 0).is_ok());
|
|
assert!(publisher.record_admission_success(ksp_worker_api::WorkerState::Running, 1, 1, false).is_ok());
|
|
let concrete = source.wait_for_change(observed).await;
|
|
let common = ksp_worker_api::WorkerSnapshotSource::wait_for_change(&source, observed).await;
|
|
assert_eq!(concrete.worker_snapshot().sequence(), common.sequence());
|
|
assert_eq!(concrete.worker_snapshot().sequence().value(), 2);
|
|
assert_eq!(concrete.worker_snapshot().state(), ksp_worker_api::WorkerState::Running);
|
|
assert_eq!(concrete.worker_snapshot().health(), ksp_worker_api::WorkerHealth::Healthy);
|
|
assert_eq!(concrete.worker_snapshot().activity(), ksp_worker_api::WorkerActivity::Active);
|
|
assert_eq!(concrete.admission_queue_depth(), 1);
|
|
assert_eq!(concrete.in_flight_persistence(), 1);
|
|
assert_eq!(concrete.admitted_total(), 1);
|
|
assert_eq!(concrete.canonicalized_total(), 1);
|
|
return;
|
|
}
|
|
|
|
#[tokio::test(flavor = "current_thread")]
|
|
async fn pre_008_terminal_latest_value_is_retained_after_publisher_drop() {
|
|
let (mut publisher, source) = match snapshot_foundation() {
|
|
std::option::Option::Some(value) => value,
|
|
std::option::Option::None => return,
|
|
};
|
|
assert!(publisher.publish_state(ksp_worker_api::WorkerState::Running, 0, 0).is_ok());
|
|
assert!(publisher.publish_state(ksp_worker_api::WorkerState::Stopping, 0, 0).is_ok());
|
|
assert!(publisher.publish_state(ksp_worker_api::WorkerState::Stopped, 0, 0).is_ok());
|
|
std::mem::drop(publisher);
|
|
let retained = source.current();
|
|
assert_eq!(retained.worker_snapshot().state(), ksp_worker_api::WorkerState::Stopped);
|
|
assert_eq!(retained.worker_snapshot().health(), ksp_worker_api::WorkerHealth::Healthy);
|
|
assert_eq!(retained.worker_snapshot().activity(), ksp_worker_api::WorkerActivity::Idle);
|
|
let same_terminal = source.wait_for_change(retained.worker_snapshot().sequence()).await;
|
|
assert_eq!(same_terminal, retained);
|
|
let common = ksp_worker_api::WorkerSnapshotSource::wait_for_change(&source, retained.worker_snapshot().sequence()).await;
|
|
assert_eq!(common, retained.worker_snapshot().clone());
|
|
return;
|
|
}
|
|
|
|
#[test]
|
|
fn pre_009_source_failure_and_backpressure_counters_advance_without_changing_projection_contract() {
|
|
let (mut publisher, source) = match snapshot_foundation() {
|
|
std::option::Option::Some(value) => value,
|
|
std::option::Option::None => return,
|
|
};
|
|
assert!(publisher.publish_state(ksp_worker_api::WorkerState::Running, 0, 0).is_ok());
|
|
assert!(publisher.record_admission_success(ksp_worker_api::WorkerState::Running, 0, 1, true).is_ok());
|
|
assert!(publisher.record_source_failure(ksp_worker_api::WorkerState::Running, 0, 1).is_ok());
|
|
let concrete = source.current();
|
|
assert_eq!(concrete.admitted_total(), 1);
|
|
assert_eq!(concrete.canonicalized_total(), 1);
|
|
assert_eq!(concrete.backpressure_wait_total(), 1);
|
|
assert_eq!(concrete.source_failure_total(), 1);
|
|
assert_eq!(concrete.worker_snapshot().health(), ksp_worker_api::WorkerHealth::Healthy);
|
|
assert_eq!(concrete.worker_snapshot().activity(), ksp_worker_api::WorkerActivity::Active);
|
|
let common = ksp_worker_api::WorkerSnapshotSource::current(&source);
|
|
assert_eq!(&common, concrete.worker_snapshot());
|
|
return;
|
|
}
|
|
|
|
#[test]
|
|
fn pre_007_processing_frontier_projection_is_latest_value_and_activity_aware() {
|
|
let (mut publisher, source) = match snapshot_foundation() {
|
|
std::option::Option::Some(value) => value,
|
|
std::option::Option::None => return,
|
|
};
|
|
let initial = source.current();
|
|
assert_eq!(initial.hydration_pending(), 0);
|
|
assert_eq!(initial.processing_frontier_slot(), std::option::Option::None);
|
|
assert_eq!(initial.oldest_pending_slot(), std::option::Option::None);
|
|
let projection = crate::RawTransactionIngestProcessingFrontierProjection::new(2, std::option::Option::Some(40), std::option::Option::Some(41));
|
|
assert!(publisher.record_processing_frontier(ksp_worker_api::WorkerState::Running, 0, 0, projection).is_ok());
|
|
let pending = source.current();
|
|
assert_eq!(pending.hydration_pending(), 2);
|
|
assert_eq!(pending.processing_frontier_slot(), std::option::Option::Some(40));
|
|
assert_eq!(pending.oldest_pending_slot(), std::option::Option::Some(41));
|
|
assert_eq!(pending.worker_snapshot().activity(), ksp_worker_api::WorkerActivity::Active);
|
|
let projection = crate::RawTransactionIngestProcessingFrontierProjection::new(0, std::option::Option::Some(45), std::option::Option::None);
|
|
assert!(publisher.record_processing_frontier(ksp_worker_api::WorkerState::Running, 0, 0, projection).is_ok());
|
|
let settled = source.current();
|
|
assert_eq!(settled.hydration_pending(), 0);
|
|
assert_eq!(settled.processing_frontier_slot(), std::option::Option::Some(45));
|
|
assert_eq!(settled.oldest_pending_slot(), std::option::Option::None);
|
|
assert_eq!(settled.worker_snapshot().activity(), ksp_worker_api::WorkerActivity::Idle);
|
|
return;
|
|
}
|
|
|
|
#[test]
|
|
fn pre_008_source_continuity_projection_preserves_processing_frontier_and_distinguishes_replay() {
|
|
let (mut publisher, source) = match snapshot_foundation() {
|
|
std::option::Option::Some(value) => value,
|
|
std::option::Option::None => return,
|
|
};
|
|
let projection = crate::RawTransactionIngestProcessingFrontierProjection::new(2, std::option::Option::Some(40), std::option::Option::Some(41))
|
|
.with_source_continuity(std::option::Option::Some(crate::RawTransactionIngestSourceState::Reconnecting), 0, 1, 0);
|
|
assert!(publisher.record_processing_frontier(ksp_worker_api::WorkerState::Running, 0, 0, projection).is_ok());
|
|
let reconnecting = source.current();
|
|
assert_eq!(reconnecting.hydration_pending(), 2);
|
|
assert_eq!(reconnecting.processing_frontier_slot(), std::option::Option::Some(40));
|
|
assert_eq!(reconnecting.oldest_pending_slot(), std::option::Option::Some(41));
|
|
assert_eq!(reconnecting.source_state(), std::option::Option::Some(crate::RawTransactionIngestSourceState::Reconnecting));
|
|
assert_eq!(reconnecting.source_reconnect_total(), 0);
|
|
assert_eq!(reconnecting.source_replay_attempt_total(), 1);
|
|
assert_eq!(reconnecting.source_continuity_gap_total(), 0);
|
|
assert_eq!(reconnecting.worker_snapshot().activity(), ksp_worker_api::WorkerActivity::Active);
|
|
let projection = crate::RawTransactionIngestProcessingFrontierProjection::new(0, std::option::Option::Some(45), std::option::Option::None)
|
|
.with_source_continuity(std::option::Option::Some(crate::RawTransactionIngestSourceState::Active), 1, 1, 0);
|
|
assert!(publisher.record_processing_frontier(ksp_worker_api::WorkerState::Running, 0, 0, projection).is_ok());
|
|
let resumed = source.current();
|
|
assert_eq!(resumed.hydration_pending(), 0);
|
|
assert_eq!(resumed.processing_frontier_slot(), std::option::Option::Some(45));
|
|
assert_eq!(resumed.source_state(), std::option::Option::Some(crate::RawTransactionIngestSourceState::Active));
|
|
assert_eq!(resumed.source_reconnect_total(), 1);
|
|
assert_eq!(resumed.source_replay_attempt_total(), 1);
|
|
assert_eq!(resumed.worker_snapshot().activity(), ksp_worker_api::WorkerActivity::Idle);
|
|
return;
|
|
}
|
|
|
|
#[test]
|
|
fn v0_3_13_pre_010_multi_source_counts_and_health_are_conservative_and_source_neutral() {
|
|
let (mut publisher, source) = match snapshot_foundation_with_source_total(3) {
|
|
std::option::Option::Some(value) => value,
|
|
std::option::Option::None => return,
|
|
};
|
|
let initial = source.current();
|
|
assert_eq!(initial.source_total(), 3);
|
|
assert_eq!(initial.source_active(), 0);
|
|
assert_eq!(initial.source_reconnecting(), 0);
|
|
assert_eq!(initial.source_failed(), 0);
|
|
assert_eq!(initial.worker_snapshot().health(), ksp_worker_api::WorkerHealth::Unknown);
|
|
let reconnecting = crate::RawTransactionIngestProcessingFrontierProjection::new(1, std::option::Option::Some(50), std::option::Option::Some(50))
|
|
.with_source_continuity(std::option::Option::Some(crate::RawTransactionIngestSourceState::Reconnecting), 0, 1, 0)
|
|
.with_source_counts(3, 2, 1, 0);
|
|
assert!(publisher.record_processing_frontier(ksp_worker_api::WorkerState::Running, 0, 0, reconnecting).is_ok());
|
|
let degraded = source.current();
|
|
assert_eq!(degraded.source_total(), 3);
|
|
assert_eq!(degraded.source_active(), 2);
|
|
assert_eq!(degraded.source_reconnecting(), 1);
|
|
assert_eq!(degraded.source_failed(), 0);
|
|
assert_eq!(degraded.worker_snapshot().health(), ksp_worker_api::WorkerHealth::Degraded);
|
|
assert_eq!(degraded.worker_snapshot().activity(), ksp_worker_api::WorkerActivity::Active);
|
|
let active = crate::RawTransactionIngestProcessingFrontierProjection::new(0, std::option::Option::Some(51), std::option::Option::None)
|
|
.with_source_continuity(std::option::Option::Some(crate::RawTransactionIngestSourceState::Active), 1, 1, 0)
|
|
.with_source_counts(3, 3, 0, 0);
|
|
assert!(publisher.record_processing_frontier(ksp_worker_api::WorkerState::Running, 0, 0, active).is_ok());
|
|
let healthy = source.current();
|
|
assert_eq!(healthy.worker_snapshot().health(), ksp_worker_api::WorkerHealth::Healthy);
|
|
assert_eq!(healthy.worker_snapshot().activity(), ksp_worker_api::WorkerActivity::Idle);
|
|
let failed = crate::RawTransactionIngestProcessingFrontierProjection::new(0, std::option::Option::Some(51), std::option::Option::None)
|
|
.with_source_continuity(std::option::Option::Some(crate::RawTransactionIngestSourceState::Failed), 1, 1, 0)
|
|
.with_source_counts(3, 2, 0, 1);
|
|
assert!(publisher.record_processing_frontier(ksp_worker_api::WorkerState::Running, 0, 0, failed).is_ok());
|
|
let unhealthy = source.current();
|
|
assert_eq!(unhealthy.source_failed(), 1);
|
|
assert_eq!(unhealthy.worker_snapshot().health(), ksp_worker_api::WorkerHealth::Unhealthy);
|
|
return;
|
|
}
|
|
|
|
#[test]
|
|
fn v0_3_14_pre_009_health_requires_present_and_future_coverage_before_healthy() {
|
|
let (mut publisher, source) = match snapshot_foundation_with_source_total(2) {
|
|
std::option::Option::Some(value) => value,
|
|
std::option::Option::None => return,
|
|
};
|
|
let reconnecting = crate::RawTransactionIngestProcessingFrontierProjection::new(0, std::option::Option::Some(50), std::option::Option::None)
|
|
.with_source_continuity(std::option::Option::Some(crate::RawTransactionIngestSourceState::Reconnecting), 1, 1, 1)
|
|
.with_source_counts(2, 1, 1, 0)
|
|
.with_continuity_health(std::option::Option::Some(50), false, true)
|
|
.with_failed_source_losses_reconciled(true);
|
|
assert!(publisher.record_processing_frontier(ksp_worker_api::WorkerState::Running, 0, 0, reconnecting).is_ok());
|
|
assert_eq!(source.current().worker_snapshot().health(), ksp_worker_api::WorkerHealth::Unhealthy);
|
|
let gap_pending = crate::RawTransactionIngestProcessingFrontierProjection::new(0, std::option::Option::Some(50), std::option::Option::None)
|
|
.with_source_continuity(std::option::Option::Some(crate::RawTransactionIngestSourceState::Active), 1, 1, 1)
|
|
.with_source_counts(2, 2, 0, 0)
|
|
.with_continuity_health(std::option::Option::Some(49), true, true)
|
|
.with_failed_source_losses_reconciled(true);
|
|
assert!(publisher.record_processing_frontier(ksp_worker_api::WorkerState::Running, 0, 0, gap_pending).is_ok());
|
|
assert_eq!(source.current().worker_snapshot().health(), ksp_worker_api::WorkerHealth::Unhealthy);
|
|
let healthy = crate::RawTransactionIngestProcessingFrontierProjection::new(0, std::option::Option::Some(50), std::option::Option::None)
|
|
.with_source_continuity(std::option::Option::Some(crate::RawTransactionIngestSourceState::Active), 1, 1, 1)
|
|
.with_source_counts(2, 2, 0, 0)
|
|
.with_continuity_health(std::option::Option::Some(50), false, true)
|
|
.with_failed_source_losses_reconciled(true);
|
|
assert!(publisher.record_processing_frontier(ksp_worker_api::WorkerState::Running, 0, 0, healthy).is_ok());
|
|
assert_eq!(source.current().worker_snapshot().health(), ksp_worker_api::WorkerHealth::Healthy);
|
|
let failed_unreconciled = crate::RawTransactionIngestProcessingFrontierProjection::new(0, std::option::Option::Some(50), std::option::Option::None)
|
|
.with_source_continuity(std::option::Option::Some(crate::RawTransactionIngestSourceState::Failed), 1, 1, 1)
|
|
.with_source_counts(2, 1, 0, 1)
|
|
.with_continuity_health(std::option::Option::Some(50), false, true)
|
|
.with_failed_source_losses_reconciled(false);
|
|
assert!(publisher.record_processing_frontier(ksp_worker_api::WorkerState::Running, 0, 0, failed_unreconciled).is_ok());
|
|
assert_eq!(source.current().worker_snapshot().health(), ksp_worker_api::WorkerHealth::Unhealthy);
|
|
let degraded = crate::RawTransactionIngestProcessingFrontierProjection::new(0, std::option::Option::Some(50), std::option::Option::None)
|
|
.with_source_continuity(std::option::Option::Some(crate::RawTransactionIngestSourceState::Failed), 1, 1, 1)
|
|
.with_source_counts(2, 1, 0, 1)
|
|
.with_continuity_health(std::option::Option::Some(50), false, true)
|
|
.with_failed_source_losses_reconciled(true);
|
|
assert!(publisher.record_processing_frontier(ksp_worker_api::WorkerState::Running, 0, 0, degraded).is_ok());
|
|
assert_eq!(source.current().worker_snapshot().health(), ksp_worker_api::WorkerHealth::Degraded);
|
|
let future_uncovered = crate::RawTransactionIngestProcessingFrontierProjection::new(0, std::option::Option::Some(50), std::option::Option::None)
|
|
.with_source_continuity(std::option::Option::Some(crate::RawTransactionIngestSourceState::Failed), 1, 1, 1)
|
|
.with_source_counts(2, 1, 0, 1)
|
|
.with_continuity_health(std::option::Option::Some(50), false, false)
|
|
.with_failed_source_losses_reconciled(true);
|
|
assert!(publisher.record_processing_frontier(ksp_worker_api::WorkerState::Running, 0, 0, future_uncovered).is_ok());
|
|
assert_eq!(source.current().worker_snapshot().health(), ksp_worker_api::WorkerHealth::Unhealthy);
|
|
assert!(publisher.publish_state(ksp_worker_api::WorkerState::Faulted(crate::ERROR_CODE_RAW_TRANSACTION_INGEST_SOURCE_FAILED), 0, 0).is_ok());
|
|
let faulted = source.current();
|
|
assert!(matches!(faulted.worker_snapshot().state(), ksp_worker_api::WorkerState::Faulted(_)));
|
|
assert_eq!(faulted.worker_snapshot().health(), ksp_worker_api::WorkerHealth::Unhealthy);
|
|
return;
|
|
}
|
|
|
|
#[test]
|
|
fn v0_3_14_pre_011_snapshot_carries_checked_gap_observability() {
|
|
let (mut publisher, source) = match snapshot_foundation_with_source_total(2) {
|
|
std::option::Option::Some(value) => value,
|
|
std::option::Option::None => return,
|
|
};
|
|
let gap = crate::RawTransactionIngestGapSnapshot::new(
|
|
crate::RawTransactionIngestGapId::new(7),
|
|
100,
|
|
110,
|
|
crate::RawTransactionIngestGapState::Pending,
|
|
crate::RawTransactionIngestGapReason::SourceFailure,
|
|
std::option::Option::None,
|
|
);
|
|
let continuity_snapshot = crate::RawTransactionIngestContinuitySnapshotProjection::empty()
|
|
.with_gap_state(std::vec![gap], 1, 0, std::option::Option::Some(100))
|
|
.with_recovery_totals(3, 1, 2, 1, 0)
|
|
.with_material_totals(4, 5);
|
|
let projection = crate::RawTransactionIngestProcessingFrontierProjection::new(0, std::option::Option::Some(120), std::option::Option::None)
|
|
.with_continuity_snapshot(continuity_snapshot);
|
|
assert!(publisher.record_processing_frontier(ksp_worker_api::WorkerState::Running, 0, 0, projection).is_ok());
|
|
let snapshot = source.current();
|
|
assert_eq!(snapshot.gaps().len(), 1);
|
|
assert_eq!(snapshot.gaps()[0].gap_id().value(), 7);
|
|
assert_eq!(snapshot.gaps()[0].start_slot(), 100);
|
|
assert_eq!(snapshot.gaps()[0].end_slot(), 110);
|
|
assert_eq!(snapshot.gaps()[0].state(), crate::RawTransactionIngestGapState::Pending);
|
|
assert_eq!(snapshot.gaps()[0].reason(), crate::RawTransactionIngestGapReason::SourceFailure);
|
|
assert_eq!(snapshot.gaps()[0].last_method(), std::option::Option::None);
|
|
assert_eq!(snapshot.open_gap_count(), 1);
|
|
assert_eq!(snapshot.repairing_gap_count(), 0);
|
|
assert_eq!(snapshot.repaired_gap_total(), 3);
|
|
assert_eq!(snapshot.unresolved_gap_total(), 1);
|
|
assert_eq!(snapshot.replay_repair_total(), 2);
|
|
assert_eq!(snapshot.redundant_coverage_repair_total(), 1);
|
|
assert_eq!(snapshot.http_scan_repair_total(), 0);
|
|
assert_eq!(snapshot.repair_block_fetch_total(), 4);
|
|
assert_eq!(snapshot.repair_transaction_hydration_total(), 5);
|
|
assert_eq!(snapshot.oldest_open_gap_start_slot(), std::option::Option::Some(100));
|
|
return;
|
|
}
|