v0.2.1-pre.004
This commit is contained in:
@@ -1,5 +1,5 @@
|
||||
// file: crates/ksp-onchain-transport-lib/unit_tests/client.rs
|
||||
// version: 1
|
||||
// version: 2
|
||||
|
||||
fn endpoint(enabled: bool, url_text: &str) -> crate::HttpEndpointSettings {
|
||||
let url = crate::HttpEndpointUrl::parse(url_text).expect("test endpoint URL must parse");
|
||||
@@ -47,3 +47,16 @@ fn endpoint_client_matches_exact_and_wildcard_capabilities() {
|
||||
assert!(client.supports(&crate::HttpRoleName::new("default"), &crate::HttpRequestKind::new("get_balance")));
|
||||
assert!(!client.supports(&crate::HttpRoleName::new("write"), &crate::HttpRequestKind::new("get_balance")));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn endpoint_role_snapshot_exposes_safe_resilience_state() {
|
||||
let client = super::HttpEndpointClient::new(endpoint(true, "https://api.devnet.solana.com")).expect("client must build");
|
||||
let snapshot = client.snapshot();
|
||||
let role = &snapshot.roles()[0];
|
||||
assert_eq!(role.availability(), crate::HttpEndpointAvailability::Available);
|
||||
assert_eq!(role.in_flight_requests(), std::option::Option::None);
|
||||
assert_eq!(role.cooldown_remaining(), std::option::Option::None);
|
||||
assert_eq!(role.success_count(), 0);
|
||||
assert_eq!(role.failure_count(), 0);
|
||||
assert_eq!(role.rate_limit_count(), 0);
|
||||
}
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
// file: crates/ksp-onchain-transport-lib/unit_tests/pool.rs
|
||||
// version: 2
|
||||
// version: 3
|
||||
|
||||
fn role(name: &str, priority: u32, request_kinds: std::vec::Vec<crate::HttpRequestKind>) -> crate::HttpEndpointRoleSettings {
|
||||
return crate::HttpEndpointRoleSettings::new(
|
||||
@@ -25,6 +25,38 @@ fn endpoint(name: &str, enabled: bool, priority: u32, request_kinds: std::vec::V
|
||||
);
|
||||
}
|
||||
|
||||
fn non_zero(value: u32) -> std::num::NonZeroU32 {
|
||||
return std::num::NonZeroU32::new(value).expect("test limit must be non-zero");
|
||||
}
|
||||
|
||||
fn limited_endpoint(
|
||||
name: &str,
|
||||
priority: u32,
|
||||
requests_per_second: std::option::Option<u32>,
|
||||
burst_capacity: std::option::Option<u32>,
|
||||
max_concurrent_requests: std::option::Option<u32>,
|
||||
cooldown: std::option::Option<std::time::Duration>,
|
||||
) -> crate::HttpEndpointSettings {
|
||||
let limits = crate::HttpRoleLimits::new(
|
||||
requests_per_second.map(|value| return non_zero(value)),
|
||||
burst_capacity.map(|value| return non_zero(value)),
|
||||
max_concurrent_requests.map(|value| return non_zero(value)),
|
||||
cooldown,
|
||||
);
|
||||
let role = crate::HttpEndpointRoleSettings::new(crate::HttpRoleName::new("default"), true, std::vec![crate::HttpRequestKind::wildcard()], priority, limits);
|
||||
return crate::HttpEndpointSettings::new(
|
||||
name,
|
||||
true,
|
||||
crate::HttpProviderName::new("provider"),
|
||||
crate::HttpClusterName::new("devnet"),
|
||||
crate::HttpEndpointUrl::parse(format!("https://{name}.invalid/rpc?token=SECRET-CANARY")).expect("test URL must parse"),
|
||||
std::time::Duration::from_secs(1),
|
||||
std::time::Duration::from_secs(2),
|
||||
std::option::Option::Some(4),
|
||||
std::vec![role],
|
||||
);
|
||||
}
|
||||
|
||||
fn settings(endpoints: std::vec::Vec<crate::HttpEndpointSettings>) -> crate::HttpTransportSettings {
|
||||
return crate::HttpTransportSettings::new(
|
||||
endpoints,
|
||||
@@ -172,3 +204,133 @@ fn removed_standard_method_is_rejected_before_endpoint_routing() {
|
||||
let error = pool.select_for_method(&crate::HttpRoleName::new("default"), method).expect_err("removed standard method must be rejected before routing");
|
||||
assert_eq!(error.code(), crate::ERROR_CODE_METHOD_REMOVED);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn runtime_concurrency_saturation_falls_back_to_lower_priority_tier() {
|
||||
let pool = super::HttpTransportPool::new(settings(std::vec![
|
||||
limited_endpoint("primary", 1, std::option::Option::None, std::option::Option::None, std::option::Option::Some(1), std::option::Option::None),
|
||||
limited_endpoint("fallback", 20, std::option::Option::None, std::option::Option::None, std::option::Option::Some(1), std::option::Option::None),
|
||||
]))
|
||||
.expect("pool must build");
|
||||
let role = crate::HttpRoleName::new("default");
|
||||
let kind = crate::HttpRequestKind::new("get_balance");
|
||||
let first = pool.acquire_for_request_kind(&role, &kind).await.expect("first request must acquire primary");
|
||||
assert_eq!(first.selection().endpoint_name(), "primary");
|
||||
let second = pool.acquire_for_request_kind(&role, &kind).await.expect("second request must fall back while primary is saturated");
|
||||
assert_eq!(second.selection().endpoint_name(), "fallback");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn runtime_token_bucket_exhaustion_falls_back_without_busy_waiting() {
|
||||
let pool = super::HttpTransportPool::new(settings(std::vec![
|
||||
limited_endpoint("primary", 1, std::option::Option::Some(1), std::option::Option::Some(1), std::option::Option::None, std::option::Option::None),
|
||||
limited_endpoint("fallback", 20, std::option::Option::None, std::option::Option::None, std::option::Option::None, std::option::Option::None),
|
||||
]))
|
||||
.expect("pool must build");
|
||||
let role = crate::HttpRoleName::new("default");
|
||||
let kind = crate::HttpRequestKind::new("get_balance");
|
||||
let first = pool.acquire_for_request_kind(&role, &kind).await.expect("first request must consume primary token");
|
||||
assert_eq!(first.selection().endpoint_name(), "primary");
|
||||
drop(first);
|
||||
let second = pool.acquire_for_request_kind(&role, &kind).await.expect("fallback must be used while primary token bucket refills");
|
||||
assert_eq!(second.selection().endpoint_name(), "fallback");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn provider_cooldown_excludes_rate_limited_role_and_uses_fallback() {
|
||||
let pool = super::HttpTransportPool::new(settings(std::vec![
|
||||
limited_endpoint(
|
||||
"primary",
|
||||
1,
|
||||
std::option::Option::None,
|
||||
std::option::Option::None,
|
||||
std::option::Option::None,
|
||||
std::option::Option::Some(std::time::Duration::from_millis(50)),
|
||||
),
|
||||
limited_endpoint("fallback", 20, std::option::Option::None, std::option::Option::None, std::option::Option::None, std::option::Option::None),
|
||||
]))
|
||||
.expect("pool must build");
|
||||
let role = crate::HttpRoleName::new("default");
|
||||
let kind = crate::HttpRequestKind::new("get_balance");
|
||||
let primary = pool.acquire_for_request_kind(&role, &kind).await.expect("primary must be acquired");
|
||||
assert_eq!(primary.selection().endpoint_name(), "primary");
|
||||
let pause = primary.record_rate_limited(std::option::Option::None);
|
||||
assert_eq!(pause, std::time::Duration::from_millis(50));
|
||||
drop(primary);
|
||||
let fallback = pool.acquire_for_request_kind(&role, &kind).await.expect("fallback must be selected during primary cooldown");
|
||||
assert_eq!(fallback.selection().endpoint_name(), "fallback");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn admission_waits_for_released_concurrency_without_holding_a_sync_mutex_across_await() {
|
||||
let pool = super::HttpTransportPool::new(settings(std::vec![limited_endpoint(
|
||||
"primary",
|
||||
1,
|
||||
std::option::Option::None,
|
||||
std::option::Option::None,
|
||||
std::option::Option::Some(1),
|
||||
std::option::Option::None,
|
||||
)]))
|
||||
.expect("pool must build");
|
||||
let role = crate::HttpRoleName::new("default");
|
||||
let kind = crate::HttpRequestKind::new("get_balance");
|
||||
let first = pool.acquire_for_request_kind(&role, &kind).await.expect("first permit must be acquired");
|
||||
let release_task = tokio::spawn(async move {
|
||||
tokio::time::sleep(std::time::Duration::from_millis(10)).await;
|
||||
drop(first);
|
||||
});
|
||||
let second = pool
|
||||
.acquire_for_request_kind_with_timeout(&role, &kind, std::time::Duration::from_millis(100))
|
||||
.await
|
||||
.expect("second permit must wake after concurrency release");
|
||||
assert_eq!(second.selection().endpoint_name(), "primary");
|
||||
release_task.await.expect("release task must complete");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn admission_timeout_is_bounded_when_concurrency_never_becomes_available() {
|
||||
let pool = super::HttpTransportPool::new(settings(std::vec![limited_endpoint(
|
||||
"primary",
|
||||
1,
|
||||
std::option::Option::None,
|
||||
std::option::Option::None,
|
||||
std::option::Option::Some(1),
|
||||
std::option::Option::None,
|
||||
)]))
|
||||
.expect("pool must build");
|
||||
let role = crate::HttpRoleName::new("default");
|
||||
let kind = crate::HttpRequestKind::new("get_balance");
|
||||
let _held = pool.acquire_for_request_kind(&role, &kind).await.expect("first permit must be acquired");
|
||||
let error = pool
|
||||
.acquire_for_request_kind_with_timeout(&role, &kind, std::time::Duration::from_millis(20))
|
||||
.await
|
||||
.expect_err("second permit must time out while concurrency remains saturated");
|
||||
assert_eq!(error.code(), crate::ERROR_CODE_TIMEOUT);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn passive_health_snapshot_moves_from_degraded_back_to_available_after_success() {
|
||||
let pool = super::HttpTransportPool::new(settings(std::vec![limited_endpoint(
|
||||
"primary",
|
||||
1,
|
||||
std::option::Option::None,
|
||||
std::option::Option::None,
|
||||
std::option::Option::None,
|
||||
std::option::Option::None,
|
||||
)]))
|
||||
.expect("pool must build");
|
||||
let role = crate::HttpRoleName::new("default");
|
||||
let kind = crate::HttpRequestKind::new("get_balance");
|
||||
let first = pool.acquire_for_request_kind(&role, &kind).await.expect("request permit must be acquired");
|
||||
first.record_failure();
|
||||
drop(first);
|
||||
let degraded = pool.snapshot();
|
||||
assert_eq!(degraded.endpoints()[0].availability(), crate::HttpEndpointAvailability::Degraded);
|
||||
assert_eq!(degraded.endpoints()[0].roles()[0].failure_count(), 1);
|
||||
let second = pool.acquire_for_request_kind(&role, &kind).await.expect("degraded endpoint remains eligible for passive recovery");
|
||||
second.record_success();
|
||||
drop(second);
|
||||
let recovered = pool.snapshot();
|
||||
assert_eq!(recovered.endpoints()[0].availability(), crate::HttpEndpointAvailability::Available);
|
||||
assert_eq!(recovered.endpoints()[0].roles()[0].success_count(), 1);
|
||||
}
|
||||
|
||||
186
crates/ksp-onchain-transport-lib/unit_tests/resilience.rs
Normal file
186
crates/ksp-onchain-transport-lib/unit_tests/resilience.rs
Normal file
@@ -0,0 +1,186 @@
|
||||
// file: crates/ksp-onchain-transport-lib/unit_tests/resilience.rs
|
||||
// version: 1
|
||||
|
||||
fn non_zero(value: u32) -> std::num::NonZeroU32 {
|
||||
return std::num::NonZeroU32::new(value).expect("test limit must be non-zero");
|
||||
}
|
||||
|
||||
fn retry_settings() -> crate::HttpRetrySettings {
|
||||
return crate::HttpRetrySettings::new(4, std::time::Duration::from_millis(100), std::time::Duration::from_millis(500));
|
||||
}
|
||||
|
||||
fn method(name: &str) -> &'static crate::HttpRpcMethodDescriptor {
|
||||
return crate::find_http_rpc_method(name).expect("audited test method must exist");
|
||||
}
|
||||
|
||||
fn role_limits(
|
||||
requests_per_second: std::option::Option<u32>,
|
||||
burst_capacity: std::option::Option<u32>,
|
||||
max_concurrent_requests: std::option::Option<u32>,
|
||||
cooldown: std::option::Option<std::time::Duration>,
|
||||
) -> crate::HttpEndpointRoleSettings {
|
||||
return crate::HttpEndpointRoleSettings::new(
|
||||
crate::HttpRoleName::new("default"),
|
||||
true,
|
||||
std::vec![crate::HttpRequestKind::wildcard()],
|
||||
10,
|
||||
crate::HttpRoleLimits::new(
|
||||
requests_per_second.map(|value| return non_zero(value)),
|
||||
burst_capacity.map(|value| return non_zero(value)),
|
||||
max_concurrent_requests.map(|value| return non_zero(value)),
|
||||
cooldown,
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn retry_backoff_is_exponential_and_bounded() {
|
||||
let settings = retry_settings();
|
||||
assert_eq!(super::retry_backoff(&settings, 1), std::time::Duration::from_millis(100));
|
||||
assert_eq!(super::retry_backoff(&settings, 2), std::time::Duration::from_millis(200));
|
||||
assert_eq!(super::retry_backoff(&settings, 3), std::time::Duration::from_millis(400));
|
||||
assert_eq!(super::retry_backoff(&settings, 4), std::time::Duration::from_millis(500));
|
||||
assert_eq!(super::retry_backoff(&settings, 32), std::time::Duration::from_millis(500));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn retry_safe_timeout_is_retried_until_budget_is_exhausted() {
|
||||
let settings = retry_settings();
|
||||
let first = super::evaluate_transport_retry(
|
||||
method("getBalance"),
|
||||
&settings,
|
||||
super::HttpRetryCause::Timeout,
|
||||
super::HttpDispatchState::DispatchedAmbiguous,
|
||||
0,
|
||||
std::option::Option::None,
|
||||
);
|
||||
assert_eq!(first, super::HttpRetryDecision::RetryAfter(std::time::Duration::from_millis(100)));
|
||||
let exhausted = super::evaluate_transport_retry(
|
||||
method("getBalance"),
|
||||
&settings,
|
||||
super::HttpRetryCause::Timeout,
|
||||
super::HttpDispatchState::DispatchedAmbiguous,
|
||||
settings.max_retries(),
|
||||
std::option::Option::None,
|
||||
);
|
||||
assert_eq!(exhausted, super::HttpRetryDecision::Stop);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn write_submission_never_retries_after_ambiguous_dispatch() {
|
||||
let decision = super::evaluate_transport_retry(
|
||||
method("sendTransaction"),
|
||||
&retry_settings(),
|
||||
super::HttpRetryCause::Connection,
|
||||
super::HttpDispatchState::DispatchedAmbiguous,
|
||||
0,
|
||||
std::option::Option::None,
|
||||
);
|
||||
assert_eq!(decision, super::HttpRetryDecision::Stop);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn write_submission_can_retry_when_transport_proves_no_dispatch() {
|
||||
let decision = super::evaluate_transport_retry(
|
||||
method("sendTransaction"),
|
||||
&retry_settings(),
|
||||
super::HttpRetryCause::Connection,
|
||||
super::HttpDispatchState::NotDispatched,
|
||||
0,
|
||||
std::option::Option::None,
|
||||
);
|
||||
assert!(decision.should_retry());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rpc_application_and_invalid_response_are_not_transport_retries() {
|
||||
for cause in [super::HttpRetryCause::RpcApplication, super::HttpRetryCause::InvalidResponse, super::HttpRetryCause::Request] {
|
||||
let decision = super::evaluate_transport_retry(
|
||||
method("getBalance"),
|
||||
&retry_settings(),
|
||||
cause,
|
||||
super::HttpDispatchState::NotDispatched,
|
||||
0,
|
||||
std::option::Option::None,
|
||||
);
|
||||
assert_eq!(decision, super::HttpRetryDecision::Stop);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn provider_retry_after_can_extend_backoff_but_is_defensively_bounded() {
|
||||
let settings = retry_settings();
|
||||
let extended = super::evaluate_transport_retry(
|
||||
method("getBalance"),
|
||||
&settings,
|
||||
super::HttpRetryCause::RateLimited,
|
||||
super::HttpDispatchState::DispatchedAmbiguous,
|
||||
0,
|
||||
std::option::Option::Some(std::time::Duration::from_secs(3)),
|
||||
);
|
||||
assert_eq!(extended.delay(), std::option::Option::Some(std::time::Duration::from_secs(3)));
|
||||
let bounded = super::evaluate_transport_retry(
|
||||
method("getBalance"),
|
||||
&settings,
|
||||
super::HttpRetryCause::RateLimited,
|
||||
super::HttpDispatchState::DispatchedAmbiguous,
|
||||
0,
|
||||
std::option::Option::Some(std::time::Duration::from_secs(600)),
|
||||
);
|
||||
assert_eq!(bounded.delay(), std::option::Option::Some(std::time::Duration::from_secs(60)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn token_bucket_consumes_burst_then_refills_from_elapsed_time() {
|
||||
let start = std::time::Instant::now();
|
||||
let mut bucket = super::HttpTokenBucketState::new(2, 2, start);
|
||||
assert!(bucket.try_consume_at(start).is_none());
|
||||
assert!(bucket.try_consume_at(start).is_none());
|
||||
assert!(bucket.try_consume_at(start).is_some());
|
||||
let later = start.checked_add(std::time::Duration::from_millis(500)).expect("test instant must advance");
|
||||
assert!(bucket.try_consume_at(later).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn absent_burst_capacity_defaults_to_one_second_of_rps_capacity() {
|
||||
let role = role_limits(std::option::Option::Some(2), std::option::Option::None, std::option::Option::None, std::option::Option::None);
|
||||
let runtime = std::sync::Arc::new(super::HttpRoleRuntime::new(&role, std::sync::Arc::new(tokio::sync::Notify::new())));
|
||||
let now = std::time::Instant::now();
|
||||
let first = runtime.try_acquire(now);
|
||||
let second = runtime.try_acquire(now);
|
||||
let third = runtime.try_acquire(now);
|
||||
assert!(matches!(first, super::RoleAdmissionAttempt::Ready(_)));
|
||||
assert!(matches!(second, super::RoleAdmissionAttempt::Ready(_)));
|
||||
assert!(matches!(third, super::RoleAdmissionAttempt::BlockedUntil(_)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn concurrency_semaphore_releases_capacity_when_permit_is_dropped() {
|
||||
let role = role_limits(std::option::Option::None, std::option::Option::None, std::option::Option::Some(1), std::option::Option::None);
|
||||
let runtime = std::sync::Arc::new(super::HttpRoleRuntime::new(&role, std::sync::Arc::new(tokio::sync::Notify::new())));
|
||||
let now = std::time::Instant::now();
|
||||
let first = runtime.try_acquire(now);
|
||||
let held = match first {
|
||||
super::RoleAdmissionAttempt::Ready(permit) => permit,
|
||||
_ => panic!("first concurrency permit must be available"),
|
||||
};
|
||||
assert!(matches!(runtime.try_acquire(now), super::RoleAdmissionAttempt::ConcurrencySaturated));
|
||||
drop(held);
|
||||
assert!(matches!(runtime.try_acquire(now), super::RoleAdmissionAttempt::Ready(_)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rate_limit_cooldown_marks_role_and_caps_provider_delay() {
|
||||
let role = role_limits(
|
||||
std::option::Option::None,
|
||||
std::option::Option::None,
|
||||
std::option::Option::None,
|
||||
std::option::Option::Some(std::time::Duration::from_millis(10)),
|
||||
);
|
||||
let runtime = super::HttpRoleRuntime::new(&role, std::sync::Arc::new(tokio::sync::Notify::new()));
|
||||
let pause = runtime.record_rate_limited(std::option::Option::Some(std::time::Duration::from_secs(600)));
|
||||
assert_eq!(pause, std::time::Duration::from_secs(60));
|
||||
assert_eq!(runtime.rate_limit_count(), 1);
|
||||
assert_eq!(runtime.failure_count(), 1);
|
||||
assert_eq!(runtime.availability(std::time::Instant::now()), crate::HttpEndpointAvailability::RateLimited);
|
||||
}
|
||||
Reference in New Issue
Block a user