337 lines
16 KiB
Rust
337 lines
16 KiB
Rust
// file: crates/ksp-onchain-transport-lib/unit_tests/pool.rs
|
|
// version: 3
|
|
|
|
fn role(name: &str, priority: u32, request_kinds: std::vec::Vec<crate::HttpRequestKind>) -> crate::HttpEndpointRoleSettings {
|
|
return crate::HttpEndpointRoleSettings::new(
|
|
crate::HttpRoleName::new(name),
|
|
true,
|
|
request_kinds,
|
|
priority,
|
|
crate::HttpRoleLimits::new(std::option::Option::None, std::option::Option::None, std::option::Option::None, std::option::Option::None),
|
|
);
|
|
}
|
|
|
|
fn endpoint(name: &str, enabled: bool, priority: u32, request_kinds: std::vec::Vec<crate::HttpRequestKind>) -> crate::HttpEndpointSettings {
|
|
return crate::HttpEndpointSettings::new(
|
|
name,
|
|
enabled,
|
|
crate::HttpProviderName::new("provider"),
|
|
crate::HttpClusterName::new("devnet"),
|
|
crate::HttpEndpointUrl::parse(format!("https://{name}.invalid/rpc?token=SECRET-CANARY")).expect("test URL must parse"),
|
|
std::time::Duration::from_secs(1),
|
|
std::time::Duration::from_secs(2),
|
|
std::option::Option::Some(4),
|
|
std::vec![role("default", priority, request_kinds)],
|
|
);
|
|
}
|
|
|
|
fn non_zero(value: u32) -> std::num::NonZeroU32 {
|
|
return std::num::NonZeroU32::new(value).expect("test limit must be non-zero");
|
|
}
|
|
|
|
fn limited_endpoint(
|
|
name: &str,
|
|
priority: u32,
|
|
requests_per_second: std::option::Option<u32>,
|
|
burst_capacity: std::option::Option<u32>,
|
|
max_concurrent_requests: std::option::Option<u32>,
|
|
cooldown: std::option::Option<std::time::Duration>,
|
|
) -> crate::HttpEndpointSettings {
|
|
let limits = crate::HttpRoleLimits::new(
|
|
requests_per_second.map(|value| return non_zero(value)),
|
|
burst_capacity.map(|value| return non_zero(value)),
|
|
max_concurrent_requests.map(|value| return non_zero(value)),
|
|
cooldown,
|
|
);
|
|
let role = crate::HttpEndpointRoleSettings::new(crate::HttpRoleName::new("default"), true, std::vec![crate::HttpRequestKind::wildcard()], priority, limits);
|
|
return crate::HttpEndpointSettings::new(
|
|
name,
|
|
true,
|
|
crate::HttpProviderName::new("provider"),
|
|
crate::HttpClusterName::new("devnet"),
|
|
crate::HttpEndpointUrl::parse(format!("https://{name}.invalid/rpc?token=SECRET-CANARY")).expect("test URL must parse"),
|
|
std::time::Duration::from_secs(1),
|
|
std::time::Duration::from_secs(2),
|
|
std::option::Option::Some(4),
|
|
std::vec![role],
|
|
);
|
|
}
|
|
|
|
fn settings(endpoints: std::vec::Vec<crate::HttpEndpointSettings>) -> crate::HttpTransportSettings {
|
|
return crate::HttpTransportSettings::new(
|
|
endpoints,
|
|
crate::HttpRetrySettings::new(2, std::time::Duration::from_millis(10), std::time::Duration::from_millis(50)),
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn pool_prefers_lowest_priority_tier() {
|
|
let pool = super::HttpTransportPool::new(settings(std::vec![
|
|
endpoint("secondary", true, 20, std::vec![crate::HttpRequestKind::wildcard()]),
|
|
endpoint("primary", true, 10, std::vec![crate::HttpRequestKind::wildcard()]),
|
|
]))
|
|
.expect("pool must build");
|
|
let selection = pool
|
|
.select_for_request_kind(&crate::HttpRoleName::new("default"), &crate::HttpRequestKind::new("get_balance"))
|
|
.expect("selection must succeed");
|
|
assert_eq!(selection.endpoint_name(), "primary");
|
|
assert_eq!(selection.priority(), 10);
|
|
}
|
|
|
|
#[test]
|
|
fn pool_round_robins_fairly_inside_best_priority_tier() {
|
|
let pool = super::HttpTransportPool::new(settings(std::vec![
|
|
endpoint("one", true, 10, std::vec![crate::HttpRequestKind::wildcard()]),
|
|
endpoint("two", true, 10, std::vec![crate::HttpRequestKind::wildcard()]),
|
|
endpoint("fallback", true, 20, std::vec![crate::HttpRequestKind::wildcard()]),
|
|
]))
|
|
.expect("pool must build");
|
|
let role = crate::HttpRoleName::new("default");
|
|
let kind = crate::HttpRequestKind::new("get_balance");
|
|
let first = pool.select_for_request_kind(&role, &kind).expect("first selection must succeed");
|
|
let second = pool.select_for_request_kind(&role, &kind).expect("second selection must succeed");
|
|
let third = pool.select_for_request_kind(&role, &kind).expect("third selection must succeed");
|
|
assert_eq!(first.endpoint_name(), "one");
|
|
assert_eq!(second.endpoint_name(), "two");
|
|
assert_eq!(third.endpoint_name(), "one");
|
|
}
|
|
|
|
#[test]
|
|
fn disabled_best_priority_endpoint_falls_back_to_next_tier() {
|
|
let pool = super::HttpTransportPool::new(settings(std::vec![
|
|
endpoint("disabled-primary", false, 1, std::vec![crate::HttpRequestKind::wildcard()]),
|
|
endpoint("fallback", true, 20, std::vec![crate::HttpRequestKind::wildcard()]),
|
|
]))
|
|
.expect("pool must build");
|
|
let selection = pool
|
|
.select_for_request_kind(&crate::HttpRoleName::new("default"), &crate::HttpRequestKind::new("get_balance"))
|
|
.expect("fallback must be selected");
|
|
assert_eq!(selection.endpoint_name(), "fallback");
|
|
}
|
|
|
|
#[test]
|
|
fn pool_filters_role_and_capability_before_priority() {
|
|
let pool = super::HttpTransportPool::new(settings(std::vec![
|
|
endpoint("wrong-capability", true, 1, std::vec![crate::HttpRequestKind::new("send_transaction")]),
|
|
endpoint("matching", true, 50, std::vec![crate::HttpRequestKind::new("get_balance")]),
|
|
]))
|
|
.expect("pool must build");
|
|
let selection = pool
|
|
.select_for_request_kind(&crate::HttpRoleName::new("default"), &crate::HttpRequestKind::new("get_balance"))
|
|
.expect("matching capability must be selected");
|
|
assert_eq!(selection.endpoint_name(), "matching");
|
|
}
|
|
|
|
#[test]
|
|
fn pool_returns_structured_error_when_no_endpoint_matches() {
|
|
let pool = super::HttpTransportPool::new(settings(std::vec![endpoint("read-only", true, 10, std::vec![crate::HttpRequestKind::new("get_balance")],)]))
|
|
.expect("pool must build");
|
|
let error = pool
|
|
.select_for_request_kind(&crate::HttpRoleName::new("default"), &crate::HttpRequestKind::new("send_transaction"))
|
|
.expect_err("unsupported request kind must fail selection");
|
|
assert_eq!(error.code(), crate::ERROR_CODE_ENDPOINT_SELECTION_FAILED);
|
|
assert!(!format!("{error:?}").contains("SECRET-CANARY"));
|
|
}
|
|
|
|
#[test]
|
|
fn standard_method_selection_uses_registry_request_kind() {
|
|
let pool = super::HttpTransportPool::new(settings(std::vec![endpoint("balance", true, 10, std::vec![crate::HttpRequestKind::new("get_balance")],)]))
|
|
.expect("pool must build");
|
|
let method = crate::find_http_rpc_method("getBalance").expect("audited method must exist");
|
|
let selection = pool.select_for_method(&crate::HttpRoleName::new("default"), method).expect("standard method must route");
|
|
assert_eq!(selection.request_kind().as_str(), "get_balance");
|
|
}
|
|
|
|
#[test]
|
|
fn pool_snapshot_is_safe_and_preserves_disabled_endpoints() {
|
|
let pool = super::HttpTransportPool::new(settings(std::vec![
|
|
endpoint("enabled", true, 10, std::vec![crate::HttpRequestKind::wildcard()]),
|
|
endpoint("disabled", false, 10, std::vec![crate::HttpRequestKind::wildcard()]),
|
|
]))
|
|
.expect("pool must build");
|
|
let snapshot = pool.snapshot();
|
|
let rendered = format!("{snapshot:?} {pool:?}");
|
|
assert_eq!(snapshot.endpoint_count(), 2);
|
|
assert_eq!(snapshot.available_endpoint_count(), 1);
|
|
assert!(!rendered.contains("SECRET-CANARY"));
|
|
assert!(!rendered.contains(".invalid/rpc"));
|
|
}
|
|
|
|
#[test]
|
|
fn disabled_role_is_excluded_before_priority_selection() {
|
|
let base = endpoint("disabled-role", true, 1, std::vec![crate::HttpRequestKind::wildcard()]);
|
|
let disabled_role = crate::HttpEndpointRoleSettings::new(
|
|
crate::HttpRoleName::new("default"),
|
|
false,
|
|
std::vec![crate::HttpRequestKind::wildcard()],
|
|
1,
|
|
crate::HttpRoleLimits::new(std::option::Option::None, std::option::Option::None, std::option::Option::None, std::option::Option::None),
|
|
);
|
|
let enabled_non_matching_role = crate::HttpEndpointRoleSettings::new(
|
|
crate::HttpRoleName::new("maintenance"),
|
|
true,
|
|
std::vec![crate::HttpRequestKind::wildcard()],
|
|
1,
|
|
crate::HttpRoleLimits::new(std::option::Option::None, std::option::Option::None, std::option::Option::None, std::option::Option::None),
|
|
);
|
|
let disabled_role_endpoint = crate::HttpEndpointSettings::new(
|
|
base.name(),
|
|
true,
|
|
base.provider().clone(),
|
|
base.cluster().clone(),
|
|
base.url().clone(),
|
|
base.connect_timeout(),
|
|
base.request_timeout(),
|
|
base.max_idle_connections_per_host(),
|
|
std::vec![disabled_role, enabled_non_matching_role],
|
|
);
|
|
let pool = super::HttpTransportPool::new(settings(std::vec![
|
|
disabled_role_endpoint,
|
|
endpoint("fallback", true, 20, std::vec![crate::HttpRequestKind::wildcard()]),
|
|
]))
|
|
.expect("pool must build");
|
|
let selection = pool
|
|
.select_for_request_kind(&crate::HttpRoleName::new("default"), &crate::HttpRequestKind::new("get_balance"))
|
|
.expect("enabled fallback role must be selected");
|
|
assert_eq!(selection.endpoint_name(), "fallback");
|
|
}
|
|
|
|
#[test]
|
|
fn removed_standard_method_is_rejected_before_endpoint_routing() {
|
|
let pool = super::HttpTransportPool::new(settings(std::vec![endpoint("wildcard", true, 10, std::vec![crate::HttpRequestKind::wildcard()],)]))
|
|
.expect("pool must build");
|
|
let method = crate::find_http_rpc_method("confirmTransaction").expect("historical method must exist");
|
|
let error = pool.select_for_method(&crate::HttpRoleName::new("default"), method).expect_err("removed standard method must be rejected before routing");
|
|
assert_eq!(error.code(), crate::ERROR_CODE_METHOD_REMOVED);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn runtime_concurrency_saturation_falls_back_to_lower_priority_tier() {
|
|
let pool = super::HttpTransportPool::new(settings(std::vec![
|
|
limited_endpoint("primary", 1, std::option::Option::None, std::option::Option::None, std::option::Option::Some(1), std::option::Option::None),
|
|
limited_endpoint("fallback", 20, std::option::Option::None, std::option::Option::None, std::option::Option::Some(1), std::option::Option::None),
|
|
]))
|
|
.expect("pool must build");
|
|
let role = crate::HttpRoleName::new("default");
|
|
let kind = crate::HttpRequestKind::new("get_balance");
|
|
let first = pool.acquire_for_request_kind(&role, &kind).await.expect("first request must acquire primary");
|
|
assert_eq!(first.selection().endpoint_name(), "primary");
|
|
let second = pool.acquire_for_request_kind(&role, &kind).await.expect("second request must fall back while primary is saturated");
|
|
assert_eq!(second.selection().endpoint_name(), "fallback");
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn runtime_token_bucket_exhaustion_falls_back_without_busy_waiting() {
|
|
let pool = super::HttpTransportPool::new(settings(std::vec![
|
|
limited_endpoint("primary", 1, std::option::Option::Some(1), std::option::Option::Some(1), std::option::Option::None, std::option::Option::None),
|
|
limited_endpoint("fallback", 20, std::option::Option::None, std::option::Option::None, std::option::Option::None, std::option::Option::None),
|
|
]))
|
|
.expect("pool must build");
|
|
let role = crate::HttpRoleName::new("default");
|
|
let kind = crate::HttpRequestKind::new("get_balance");
|
|
let first = pool.acquire_for_request_kind(&role, &kind).await.expect("first request must consume primary token");
|
|
assert_eq!(first.selection().endpoint_name(), "primary");
|
|
drop(first);
|
|
let second = pool.acquire_for_request_kind(&role, &kind).await.expect("fallback must be used while primary token bucket refills");
|
|
assert_eq!(second.selection().endpoint_name(), "fallback");
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn provider_cooldown_excludes_rate_limited_role_and_uses_fallback() {
|
|
let pool = super::HttpTransportPool::new(settings(std::vec![
|
|
limited_endpoint(
|
|
"primary",
|
|
1,
|
|
std::option::Option::None,
|
|
std::option::Option::None,
|
|
std::option::Option::None,
|
|
std::option::Option::Some(std::time::Duration::from_millis(50)),
|
|
),
|
|
limited_endpoint("fallback", 20, std::option::Option::None, std::option::Option::None, std::option::Option::None, std::option::Option::None),
|
|
]))
|
|
.expect("pool must build");
|
|
let role = crate::HttpRoleName::new("default");
|
|
let kind = crate::HttpRequestKind::new("get_balance");
|
|
let primary = pool.acquire_for_request_kind(&role, &kind).await.expect("primary must be acquired");
|
|
assert_eq!(primary.selection().endpoint_name(), "primary");
|
|
let pause = primary.record_rate_limited(std::option::Option::None);
|
|
assert_eq!(pause, std::time::Duration::from_millis(50));
|
|
drop(primary);
|
|
let fallback = pool.acquire_for_request_kind(&role, &kind).await.expect("fallback must be selected during primary cooldown");
|
|
assert_eq!(fallback.selection().endpoint_name(), "fallback");
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn admission_waits_for_released_concurrency_without_holding_a_sync_mutex_across_await() {
|
|
let pool = super::HttpTransportPool::new(settings(std::vec![limited_endpoint(
|
|
"primary",
|
|
1,
|
|
std::option::Option::None,
|
|
std::option::Option::None,
|
|
std::option::Option::Some(1),
|
|
std::option::Option::None,
|
|
)]))
|
|
.expect("pool must build");
|
|
let role = crate::HttpRoleName::new("default");
|
|
let kind = crate::HttpRequestKind::new("get_balance");
|
|
let first = pool.acquire_for_request_kind(&role, &kind).await.expect("first permit must be acquired");
|
|
let release_task = tokio::spawn(async move {
|
|
tokio::time::sleep(std::time::Duration::from_millis(10)).await;
|
|
drop(first);
|
|
});
|
|
let second = pool
|
|
.acquire_for_request_kind_with_timeout(&role, &kind, std::time::Duration::from_millis(100))
|
|
.await
|
|
.expect("second permit must wake after concurrency release");
|
|
assert_eq!(second.selection().endpoint_name(), "primary");
|
|
release_task.await.expect("release task must complete");
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn admission_timeout_is_bounded_when_concurrency_never_becomes_available() {
|
|
let pool = super::HttpTransportPool::new(settings(std::vec![limited_endpoint(
|
|
"primary",
|
|
1,
|
|
std::option::Option::None,
|
|
std::option::Option::None,
|
|
std::option::Option::Some(1),
|
|
std::option::Option::None,
|
|
)]))
|
|
.expect("pool must build");
|
|
let role = crate::HttpRoleName::new("default");
|
|
let kind = crate::HttpRequestKind::new("get_balance");
|
|
let _held = pool.acquire_for_request_kind(&role, &kind).await.expect("first permit must be acquired");
|
|
let error = pool
|
|
.acquire_for_request_kind_with_timeout(&role, &kind, std::time::Duration::from_millis(20))
|
|
.await
|
|
.expect_err("second permit must time out while concurrency remains saturated");
|
|
assert_eq!(error.code(), crate::ERROR_CODE_TIMEOUT);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn passive_health_snapshot_moves_from_degraded_back_to_available_after_success() {
|
|
let pool = super::HttpTransportPool::new(settings(std::vec![limited_endpoint(
|
|
"primary",
|
|
1,
|
|
std::option::Option::None,
|
|
std::option::Option::None,
|
|
std::option::Option::None,
|
|
std::option::Option::None,
|
|
)]))
|
|
.expect("pool must build");
|
|
let role = crate::HttpRoleName::new("default");
|
|
let kind = crate::HttpRequestKind::new("get_balance");
|
|
let first = pool.acquire_for_request_kind(&role, &kind).await.expect("request permit must be acquired");
|
|
first.record_failure();
|
|
drop(first);
|
|
let degraded = pool.snapshot();
|
|
assert_eq!(degraded.endpoints()[0].availability(), crate::HttpEndpointAvailability::Degraded);
|
|
assert_eq!(degraded.endpoints()[0].roles()[0].failure_count(), 1);
|
|
let second = pool.acquire_for_request_kind(&role, &kind).await.expect("degraded endpoint remains eligible for passive recovery");
|
|
second.record_success();
|
|
drop(second);
|
|
let recovered = pool.snapshot();
|
|
assert_eq!(recovered.endpoints()[0].availability(), crate::HttpEndpointAvailability::Available);
|
|
assert_eq!(recovered.endpoints()[0].roles()[0].success_count(), 1);
|
|
}
|