// file: crates/ksp-onchain-transport-lib/unit_tests/pool.rs // version: 4 fn role(name: &str, priority: u32, request_kinds: std::vec::Vec) -> crate::HttpEndpointRoleSettings { return crate::HttpEndpointRoleSettings::new( crate::HttpRoleName::new(name), true, request_kinds, priority, crate::HttpRoleLimits::new(std::option::Option::None, std::option::Option::None, std::option::Option::None, std::option::Option::None), ); } fn endpoint(name: &str, enabled: bool, priority: u32, request_kinds: std::vec::Vec) -> crate::HttpEndpointSettings { return crate::HttpEndpointSettings::new( name, enabled, crate::HttpProviderName::new("provider"), crate::HttpClusterName::new("devnet"), crate::HttpEndpointUrl::parse(format!("https://{name}.invalid/rpc?token=SECRET-CANARY")).expect("test URL must parse"), std::time::Duration::from_secs(1), std::time::Duration::from_secs(2), std::option::Option::Some(4), std::vec![role("default", priority, request_kinds)], ); } fn non_zero(value: u32) -> std::num::NonZeroU32 { return std::num::NonZeroU32::new(value).expect("test limit must be non-zero"); } fn limited_endpoint( name: &str, priority: u32, requests_per_second: std::option::Option, burst_capacity: std::option::Option, max_concurrent_requests: std::option::Option, cooldown: std::option::Option, ) -> crate::HttpEndpointSettings { let limits = crate::HttpRoleLimits::new( requests_per_second.map(|value| return non_zero(value)), burst_capacity.map(|value| return non_zero(value)), max_concurrent_requests.map(|value| return non_zero(value)), cooldown, ); let role = crate::HttpEndpointRoleSettings::new(crate::HttpRoleName::new("default"), true, std::vec![crate::HttpRequestKind::wildcard()], priority, limits); return crate::HttpEndpointSettings::new( name, true, crate::HttpProviderName::new("provider"), crate::HttpClusterName::new("devnet"), crate::HttpEndpointUrl::parse(format!("https://{name}.invalid/rpc?token=SECRET-CANARY")).expect("test URL must parse"), std::time::Duration::from_secs(1), std::time::Duration::from_secs(2), std::option::Option::Some(4), std::vec![role], ); } fn settings(endpoints: std::vec::Vec) -> crate::HttpTransportSettings { return crate::HttpTransportSettings::new( endpoints, crate::HttpRetrySettings::new(2, std::time::Duration::from_millis(10), std::time::Duration::from_millis(50)), ); } #[test] fn pool_prefers_lowest_priority_tier() { let pool = crate::HttpTransportPool::new(settings(std::vec![ endpoint("secondary", true, 20, std::vec![crate::HttpRequestKind::wildcard()]), endpoint("primary", true, 10, std::vec![crate::HttpRequestKind::wildcard()]), ])) .expect("pool must build"); let selection = pool .select_for_request_kind(&crate::HttpRoleName::new("default"), &crate::HttpRequestKind::new("get_balance")) .expect("selection must succeed"); assert_eq!(selection.endpoint_name(), "primary"); assert_eq!(selection.priority(), 10); } #[test] fn pool_round_robins_fairly_inside_best_priority_tier() { let pool = crate::HttpTransportPool::new(settings(std::vec![ endpoint("one", true, 10, std::vec![crate::HttpRequestKind::wildcard()]), endpoint("two", true, 10, std::vec![crate::HttpRequestKind::wildcard()]), endpoint("fallback", true, 20, std::vec![crate::HttpRequestKind::wildcard()]), ])) .expect("pool must build"); let role = crate::HttpRoleName::new("default"); let kind = crate::HttpRequestKind::new("get_balance"); let first = pool.select_for_request_kind(&role, &kind).expect("first selection must succeed"); let second = pool.select_for_request_kind(&role, &kind).expect("second selection must succeed"); let third = pool.select_for_request_kind(&role, &kind).expect("third selection must succeed"); assert_eq!(first.endpoint_name(), "one"); assert_eq!(second.endpoint_name(), "two"); assert_eq!(third.endpoint_name(), "one"); } #[test] fn disabled_best_priority_endpoint_falls_back_to_next_tier() { let pool = crate::HttpTransportPool::new(settings(std::vec![ endpoint("disabled-primary", false, 1, std::vec![crate::HttpRequestKind::wildcard()]), endpoint("fallback", true, 20, std::vec![crate::HttpRequestKind::wildcard()]), ])) .expect("pool must build"); let selection = pool .select_for_request_kind(&crate::HttpRoleName::new("default"), &crate::HttpRequestKind::new("get_balance")) .expect("fallback must be selected"); assert_eq!(selection.endpoint_name(), "fallback"); } #[test] fn pool_filters_role_and_capability_before_priority() { let pool = crate::HttpTransportPool::new(settings(std::vec![ endpoint("wrong-capability", true, 1, std::vec![crate::HttpRequestKind::new("send_transaction")]), endpoint("matching", true, 50, std::vec![crate::HttpRequestKind::new("get_balance")]), ])) .expect("pool must build"); let selection = pool .select_for_request_kind(&crate::HttpRoleName::new("default"), &crate::HttpRequestKind::new("get_balance")) .expect("matching capability must be selected"); assert_eq!(selection.endpoint_name(), "matching"); } #[test] fn pool_returns_structured_error_when_no_endpoint_matches() { let pool = crate::HttpTransportPool::new(settings(std::vec![endpoint("read-only", true, 10, std::vec![crate::HttpRequestKind::new("get_balance")],)])) .expect("pool must build"); let error = pool .select_for_request_kind(&crate::HttpRoleName::new("default"), &crate::HttpRequestKind::new("send_transaction")) .expect_err("unsupported request kind must fail selection"); assert_eq!(error.code(), crate::ERROR_CODE_ENDPOINT_SELECTION_FAILED); assert!(!format!("{error:?}").contains("SECRET-CANARY")); } #[test] fn standard_method_selection_uses_registry_request_kind() { let pool = crate::HttpTransportPool::new(settings(std::vec![endpoint("balance", true, 10, std::vec![crate::HttpRequestKind::new("get_balance")],)])) .expect("pool must build"); let method = crate::find_http_rpc_method("getBalance").expect("audited method must exist"); let selection = pool.select_for_method(&crate::HttpRoleName::new("default"), method).expect("standard method must route"); assert_eq!(selection.request_kind().as_str(), "get_balance"); } #[test] fn pool_snapshot_is_safe_and_preserves_disabled_endpoints() { let pool = crate::HttpTransportPool::new(settings(std::vec![ endpoint("enabled", true, 10, std::vec![crate::HttpRequestKind::wildcard()]), endpoint("disabled", false, 10, std::vec![crate::HttpRequestKind::wildcard()]), ])) .expect("pool must build"); let snapshot = pool.snapshot(); let rendered = format!("{snapshot:?} {pool:?}"); assert_eq!(snapshot.endpoint_count(), 2); assert_eq!(snapshot.available_endpoint_count(), 1); assert!(!rendered.contains("SECRET-CANARY")); assert!(!rendered.contains(".invalid/rpc")); } #[test] fn disabled_role_is_excluded_before_priority_selection() { let base = endpoint("disabled-role", true, 1, std::vec![crate::HttpRequestKind::wildcard()]); let disabled_role = crate::HttpEndpointRoleSettings::new( crate::HttpRoleName::new("default"), false, std::vec![crate::HttpRequestKind::wildcard()], 1, crate::HttpRoleLimits::new(std::option::Option::None, std::option::Option::None, std::option::Option::None, std::option::Option::None), ); let enabled_non_matching_role = crate::HttpEndpointRoleSettings::new( crate::HttpRoleName::new("maintenance"), true, std::vec![crate::HttpRequestKind::wildcard()], 1, crate::HttpRoleLimits::new(std::option::Option::None, std::option::Option::None, std::option::Option::None, std::option::Option::None), ); let disabled_role_endpoint = crate::HttpEndpointSettings::new( base.name(), true, base.provider().clone(), base.cluster().clone(), base.url().clone(), base.connect_timeout(), base.request_timeout(), base.max_idle_connections_per_host(), std::vec![disabled_role, enabled_non_matching_role], ); let pool = crate::HttpTransportPool::new(settings(std::vec![ disabled_role_endpoint, endpoint("fallback", true, 20, std::vec![crate::HttpRequestKind::wildcard()]), ])) .expect("pool must build"); let selection = pool .select_for_request_kind(&crate::HttpRoleName::new("default"), &crate::HttpRequestKind::new("get_balance")) .expect("enabled fallback role must be selected"); assert_eq!(selection.endpoint_name(), "fallback"); } #[test] fn removed_standard_method_is_rejected_before_endpoint_routing() { let pool = crate::HttpTransportPool::new(settings(std::vec![endpoint("wildcard", true, 10, std::vec![crate::HttpRequestKind::wildcard()],)])) .expect("pool must build"); let method = crate::find_http_rpc_method("confirmTransaction").expect("historical method must exist"); let error = pool.select_for_method(&crate::HttpRoleName::new("default"), method).expect_err("removed standard method must be rejected before routing"); assert_eq!(error.code(), crate::ERROR_CODE_METHOD_REMOVED); } #[tokio::test] async fn runtime_concurrency_saturation_falls_back_to_lower_priority_tier() { let pool = crate::HttpTransportPool::new(settings(std::vec![ limited_endpoint("primary", 1, std::option::Option::None, std::option::Option::None, std::option::Option::Some(1), std::option::Option::None), limited_endpoint("fallback", 20, std::option::Option::None, std::option::Option::None, std::option::Option::Some(1), std::option::Option::None), ])) .expect("pool must build"); let role = crate::HttpRoleName::new("default"); let kind = crate::HttpRequestKind::new("get_balance"); let first = pool.acquire_for_request_kind(&role, &kind).await.expect("first request must acquire primary"); assert_eq!(first.selection().endpoint_name(), "primary"); let second = pool.acquire_for_request_kind(&role, &kind).await.expect("second request must fall back while primary is saturated"); assert_eq!(second.selection().endpoint_name(), "fallback"); } #[tokio::test] async fn runtime_token_bucket_exhaustion_falls_back_without_busy_waiting() { let pool = crate::HttpTransportPool::new(settings(std::vec![ limited_endpoint("primary", 1, std::option::Option::Some(1), std::option::Option::Some(1), std::option::Option::None, std::option::Option::None), limited_endpoint("fallback", 20, std::option::Option::None, std::option::Option::None, std::option::Option::None, std::option::Option::None), ])) .expect("pool must build"); let role = crate::HttpRoleName::new("default"); let kind = crate::HttpRequestKind::new("get_balance"); let first = pool.acquire_for_request_kind(&role, &kind).await.expect("first request must consume primary token"); assert_eq!(first.selection().endpoint_name(), "primary"); drop(first); let second = pool.acquire_for_request_kind(&role, &kind).await.expect("fallback must be used while primary token bucket refills"); assert_eq!(second.selection().endpoint_name(), "fallback"); } #[tokio::test] async fn provider_cooldown_excludes_rate_limited_role_and_uses_fallback() { let pool = crate::HttpTransportPool::new(settings(std::vec![ limited_endpoint( "primary", 1, std::option::Option::None, std::option::Option::None, std::option::Option::None, std::option::Option::Some(std::time::Duration::from_millis(50)), ), limited_endpoint("fallback", 20, std::option::Option::None, std::option::Option::None, std::option::Option::None, std::option::Option::None), ])) .expect("pool must build"); let role = crate::HttpRoleName::new("default"); let kind = crate::HttpRequestKind::new("get_balance"); let primary = pool.acquire_for_request_kind(&role, &kind).await.expect("primary must be acquired"); assert_eq!(primary.selection().endpoint_name(), "primary"); let pause = primary.record_rate_limited(std::option::Option::None); assert_eq!(pause, std::time::Duration::from_millis(50)); drop(primary); let fallback = pool.acquire_for_request_kind(&role, &kind).await.expect("fallback must be selected during primary cooldown"); assert_eq!(fallback.selection().endpoint_name(), "fallback"); } #[tokio::test] async fn admission_waits_for_released_concurrency_without_holding_a_sync_mutex_across_await() { let pool = crate::HttpTransportPool::new(settings(std::vec![limited_endpoint( "primary", 1, std::option::Option::None, std::option::Option::None, std::option::Option::Some(1), std::option::Option::None, )])) .expect("pool must build"); let role = crate::HttpRoleName::new("default"); let kind = crate::HttpRequestKind::new("get_balance"); let first = pool.acquire_for_request_kind(&role, &kind).await.expect("first permit must be acquired"); let release_task = tokio::spawn(async move { tokio::time::sleep(std::time::Duration::from_millis(10)).await; drop(first); }); let second = pool .acquire_for_request_kind_with_timeout(&role, &kind, std::time::Duration::from_millis(100)) .await .expect("second permit must wake after concurrency release"); assert_eq!(second.selection().endpoint_name(), "primary"); release_task.await.expect("release task must complete"); } #[tokio::test] async fn admission_timeout_is_bounded_when_concurrency_never_becomes_available() { let pool = crate::HttpTransportPool::new(settings(std::vec![limited_endpoint( "primary", 1, std::option::Option::None, std::option::Option::None, std::option::Option::Some(1), std::option::Option::None, )])) .expect("pool must build"); let role = crate::HttpRoleName::new("default"); let kind = crate::HttpRequestKind::new("get_balance"); let _held = pool.acquire_for_request_kind(&role, &kind).await.expect("first permit must be acquired"); let error = pool .acquire_for_request_kind_with_timeout(&role, &kind, std::time::Duration::from_millis(20)) .await .expect_err("second permit must time out while concurrency remains saturated"); assert_eq!(error.code(), crate::ERROR_CODE_TIMEOUT); } #[tokio::test] async fn passive_health_snapshot_moves_from_degraded_back_to_available_after_success() { let pool = crate::HttpTransportPool::new(settings(std::vec![limited_endpoint( "primary", 1, std::option::Option::None, std::option::Option::None, std::option::Option::None, std::option::Option::None, )])) .expect("pool must build"); let role = crate::HttpRoleName::new("default"); let kind = crate::HttpRequestKind::new("get_balance"); let first = pool.acquire_for_request_kind(&role, &kind).await.expect("request permit must be acquired"); first.record_failure(); drop(first); let degraded = pool.snapshot(); assert_eq!(degraded.endpoints()[0].availability(), crate::HttpEndpointAvailability::Degraded); assert_eq!(degraded.endpoints()[0].roles()[0].failure_count(), 1); let second = pool.acquire_for_request_kind(&role, &kind).await.expect("degraded endpoint remains eligible for passive recovery"); second.record_success(); drop(second); let recovered = pool.snapshot(); assert_eq!(recovered.endpoints()[0].availability(), crate::HttpEndpointAvailability::Available); assert_eq!(recovered.endpoints()[0].roles()[0].success_count(), 1); }