// file: crates/ksp-onchain-transport-lib/unit_tests/resilience.rs // version: 2 fn non_zero(value: u32) -> std::num::NonZeroU32 { return std::num::NonZeroU32::new(value).expect("test limit must be non-zero"); } fn retry_settings() -> crate::HttpRetrySettings { return crate::HttpRetrySettings::new(4, std::time::Duration::from_millis(100), std::time::Duration::from_millis(500)); } fn method(name: &str) -> &'static crate::HttpRpcMethodDescriptor { return crate::find_http_rpc_method(name).expect("audited test method must exist"); } fn role_limits( requests_per_second: std::option::Option, burst_capacity: std::option::Option, max_concurrent_requests: std::option::Option, cooldown: std::option::Option, ) -> crate::HttpEndpointRoleSettings { return crate::HttpEndpointRoleSettings::new( crate::HttpRoleName::new("default"), true, std::vec![crate::HttpRequestKind::wildcard()], 10, crate::HttpRoleLimits::new( requests_per_second.map(|value| return non_zero(value)), burst_capacity.map(|value| return non_zero(value)), max_concurrent_requests.map(|value| return non_zero(value)), cooldown, ), ); } #[test] fn retry_backoff_is_exponential_and_bounded() { let settings = retry_settings(); assert_eq!(super::retry_backoff(&settings, 1), std::time::Duration::from_millis(100)); assert_eq!(super::retry_backoff(&settings, 2), std::time::Duration::from_millis(200)); assert_eq!(super::retry_backoff(&settings, 3), std::time::Duration::from_millis(400)); assert_eq!(super::retry_backoff(&settings, 4), std::time::Duration::from_millis(500)); assert_eq!(super::retry_backoff(&settings, 32), std::time::Duration::from_millis(500)); } #[test] fn retry_safe_timeout_is_retried_until_budget_is_exhausted() { let settings = retry_settings(); let first = crate::evaluate_transport_retry( method("getBalance"), &settings, crate::HttpRetryCause::Timeout, crate::HttpDispatchState::DispatchedAmbiguous, 0, std::option::Option::None, ); assert_eq!(first, crate::HttpRetryDecision::RetryAfter(std::time::Duration::from_millis(100))); let exhausted = crate::evaluate_transport_retry( method("getBalance"), &settings, crate::HttpRetryCause::Timeout, crate::HttpDispatchState::DispatchedAmbiguous, settings.max_retries(), std::option::Option::None, ); assert_eq!(exhausted, crate::HttpRetryDecision::Stop); } #[test] fn write_submission_never_retries_after_ambiguous_dispatch() { let decision = crate::evaluate_transport_retry( method("sendTransaction"), &retry_settings(), crate::HttpRetryCause::Connection, crate::HttpDispatchState::DispatchedAmbiguous, 0, std::option::Option::None, ); assert_eq!(decision, crate::HttpRetryDecision::Stop); } #[test] fn write_submission_can_retry_when_transport_proves_no_dispatch() { let decision = crate::evaluate_transport_retry( method("sendTransaction"), &retry_settings(), crate::HttpRetryCause::Connection, crate::HttpDispatchState::NotDispatched, 0, std::option::Option::None, ); assert!(decision.should_retry()); } #[test] fn rpc_application_and_invalid_response_are_not_transport_retries() { for cause in [crate::HttpRetryCause::RpcApplication, crate::HttpRetryCause::InvalidResponse, crate::HttpRetryCause::Request] { let decision = crate::evaluate_transport_retry( method("getBalance"), &retry_settings(), cause, crate::HttpDispatchState::NotDispatched, 0, std::option::Option::None, ); assert_eq!(decision, crate::HttpRetryDecision::Stop); } } #[test] fn provider_retry_after_can_extend_backoff_but_is_defensively_bounded() { let settings = retry_settings(); let extended = crate::evaluate_transport_retry( method("getBalance"), &settings, crate::HttpRetryCause::RateLimited, crate::HttpDispatchState::DispatchedAmbiguous, 0, std::option::Option::Some(std::time::Duration::from_secs(3)), ); assert_eq!(extended.delay(), std::option::Option::Some(std::time::Duration::from_secs(3))); let bounded = crate::evaluate_transport_retry( method("getBalance"), &settings, crate::HttpRetryCause::RateLimited, crate::HttpDispatchState::DispatchedAmbiguous, 0, std::option::Option::Some(std::time::Duration::from_secs(600)), ); assert_eq!(bounded.delay(), std::option::Option::Some(std::time::Duration::from_secs(60))); } #[test] fn token_bucket_consumes_burst_then_refills_from_elapsed_time() { let start = std::time::Instant::now(); let mut bucket = super::HttpTokenBucketState::new(2, 2, start); assert!(bucket.try_consume_at(start).is_none()); assert!(bucket.try_consume_at(start).is_none()); assert!(bucket.try_consume_at(start).is_some()); let later = start.checked_add(std::time::Duration::from_millis(500)).expect("test instant must advance"); assert!(bucket.try_consume_at(later).is_none()); } #[test] fn absent_burst_capacity_defaults_to_one_second_of_rps_capacity() { let role = role_limits(std::option::Option::Some(2), std::option::Option::None, std::option::Option::None, std::option::Option::None); let runtime = std::sync::Arc::new(crate::HttpRoleRuntime::new(&role, std::sync::Arc::new(tokio::sync::Notify::new()))); let now = std::time::Instant::now(); let first = runtime.try_acquire(now); let second = runtime.try_acquire(now); let third = runtime.try_acquire(now); assert!(matches!(first, crate::RoleAdmissionAttempt::Ready(_))); assert!(matches!(second, crate::RoleAdmissionAttempt::Ready(_))); assert!(matches!(third, crate::RoleAdmissionAttempt::BlockedUntil(_))); } #[test] fn concurrency_semaphore_releases_capacity_when_permit_is_dropped() { let role = role_limits(std::option::Option::None, std::option::Option::None, std::option::Option::Some(1), std::option::Option::None); let runtime = std::sync::Arc::new(crate::HttpRoleRuntime::new(&role, std::sync::Arc::new(tokio::sync::Notify::new()))); let now = std::time::Instant::now(); let first = runtime.try_acquire(now); let held = match first { crate::RoleAdmissionAttempt::Ready(permit) => permit, _ => panic!("first concurrency permit must be available"), }; assert!(matches!(runtime.try_acquire(now), crate::RoleAdmissionAttempt::ConcurrencySaturated)); drop(held); assert!(matches!(runtime.try_acquire(now), crate::RoleAdmissionAttempt::Ready(_))); } #[test] fn rate_limit_cooldown_marks_role_and_caps_provider_delay() { let role = role_limits( std::option::Option::None, std::option::Option::None, std::option::Option::None, std::option::Option::Some(std::time::Duration::from_millis(10)), ); let runtime = crate::HttpRoleRuntime::new(&role, std::sync::Arc::new(tokio::sync::Notify::new())); let pause = runtime.record_rate_limited(std::option::Option::Some(std::time::Duration::from_secs(600))); assert_eq!(pause, std::time::Duration::from_secs(60)); assert_eq!(runtime.rate_limit_count(), 1); assert_eq!(runtime.failure_count(), 1); assert_eq!(runtime.availability(std::time::Instant::now()), crate::HttpEndpointAvailability::RateLimited); }