// file: crates/ksp-onchain-transport-lib/src/http_pool.rs // version: 8 /// Safe snapshot of the logical HTTP endpoint pool. #[derive(Clone, Debug, Eq, PartialEq)] pub struct HttpTransportPoolSnapshot { endpoints: std::vec::Vec, } impl HttpTransportPoolSnapshot { /// Returns safe endpoint snapshots in configured declaration order. #[must_use] pub fn endpoints(&self) -> &[crate::HttpEndpointSnapshot] { return self.endpoints.as_slice(); } /// Returns the total number of configured logical endpoints. #[must_use] pub fn endpoint_count(&self) -> usize { return self.endpoints.len(); } /// Returns the number of endpoints currently eligible for normal routing. #[must_use] pub fn available_endpoint_count(&self) -> usize { return self.endpoints.iter().filter(|endpoint| return endpoint.availability() == crate::HttpEndpointAvailability::Available).count(); } } /// Result of one logical endpoint selection. #[derive(Clone, Debug)] pub struct HttpEndpointSelection { client: crate::HttpEndpointClient, role: crate::HttpRoleName, request_kind: crate::HttpRequestKind, priority: u32, } impl HttpEndpointSelection { /// Returns the selected endpoint client. #[must_use] pub const fn client(&self) -> &crate::HttpEndpointClient { return &self.client; } /// Returns the selected endpoint identity. #[must_use] pub fn endpoint_name(&self) -> &str { return self.client.name(); } /// Returns the matched logical role. #[must_use] pub const fn role(&self) -> &crate::HttpRoleName { return &self.role; } /// Returns the matched request-kind capability. #[must_use] pub const fn request_kind(&self) -> &crate::HttpRequestKind { return &self.request_kind; } /// Returns the selected role priority where lower values are preferred. #[must_use] pub const fn priority(&self) -> u32 { return self.priority; } } /// Runtime admission permit for one HTTP request. /// /// The permit reserves configured concurrency capacity and carries the common request deadline. Dropping it releases any semaphore capacity immediately. pub struct HttpRequestPermit { selection: crate::HttpEndpointSelection, deadline: std::time::Instant, role_runtime: std::sync::Arc, _concurrency_permit: crate::HttpConcurrencyPermit, } impl HttpRequestPermit { /// Returns the selected logical endpoint and role. #[must_use] pub const fn selection(&self) -> &crate::HttpEndpointSelection { return &self.selection; } /// Returns the selected endpoint client. #[must_use] pub fn client(&self) -> &crate::HttpEndpointClient { return self.selection.client(); } /// Returns the remaining duration in the common request budget. #[must_use] pub fn remaining_timeout(&self) -> std::time::Duration { let now = std::time::Instant::now(); if now >= self.deadline { return std::time::Duration::ZERO; } return self.deadline.duration_since(now); } /// Records a successful request and clears the passive degraded state for this endpoint role. pub fn record_success(&self) { self.role_runtime.record_success(); ksp_logging_lib::trace!( target: crate::TRACING_TARGET, endpoint_name = self.selection.endpoint_name(), role = self.selection.role().as_str(), request_kind = self.selection.request_kind().as_str(), "recorded successful HTTP endpoint observation" ); return; } /// Records a transport failure and marks the endpoint role degraded until a later success. pub fn record_failure(&self) { self.role_runtime.record_failure(); ksp_logging_lib::warn!( target: crate::TRACING_TARGET, endpoint_name = self.selection.endpoint_name(), provider = self.selection.client().provider().as_str(), cluster = self.selection.client().cluster().as_str(), role = self.selection.role().as_str(), request_kind = self.selection.request_kind().as_str(), "HTTP endpoint role marked degraded after transport failure" ); return; } /// Records provider rate limiting and applies the role cooldown. /// /// A provider delay can extend the configured cooldown but is defensively capped by the Transport runtime before use. pub fn record_rate_limited(&self, provider_retry_after: std::option::Option) -> std::time::Duration { let pause = self.role_runtime.record_rate_limited(provider_retry_after); let cooldown_ms = duration_millis_u64(pause); ksp_logging_lib::warn!( target: crate::TRACING_TARGET, endpoint_name = self.selection.endpoint_name(), provider = self.selection.client().provider().as_str(), cluster = self.selection.client().cluster().as_str(), role = self.selection.role().as_str(), request_kind = self.selection.request_kind().as_str(), cooldown_ms, "HTTP endpoint role entered provider rate-limit cooldown" ); return pause; } } impl std::fmt::Debug for HttpRequestPermit { fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { return formatter .debug_struct("HttpRequestPermit") .field("selection", &self.selection) .field("remaining_timeout", &self.remaining_timeout()) .finish(); } } /// Shareable logical HTTP endpoint pool with priority routing, admission limits and bounded request deadlines. #[derive(Clone)] pub struct HttpTransportPool { inner: std::sync::Arc, } impl HttpTransportPool { /// Builds a logical endpoint pool after validating all Transport-owned runtime settings. pub fn new(settings: crate::HttpTransportSettings) -> ksp_core_lib::Result { let validation = settings.validate(); if let std::result::Result::Err(error) = validation { return std::result::Result::Err(error); } let notify = std::sync::Arc::new(tokio::sync::Notify::new()); let mut clients = std::vec::Vec::with_capacity(settings.endpoints().len()); for endpoint in settings.endpoints() { let client_result = crate::HttpEndpointClient::new_with_notify(endpoint.clone(), std::sync::Arc::clone(¬ify)); let client = match client_result { std::result::Result::Ok(client) => client, std::result::Result::Err(error) => return std::result::Result::Err(error), }; clients.push(client); } let pool = Self { inner: std::sync::Arc::new(HttpTransportPoolInner { clients, retry: settings.retry().clone(), notify, request_ids: std::sync::atomic::AtomicU64::new(1), cursors: std::sync::Mutex::new(std::collections::BTreeMap::new()), }), }; ksp_logging_lib::debug!( target: crate::TRACING_TARGET, endpoint_count = pool.inner.clients.len(), available_endpoint_count = pool.snapshot().available_endpoint_count(), max_retries = pool.inner.retry.max_retries(), "created logical HTTP endpoint pool" ); return std::result::Result::Ok(pool); } /// Returns the bounded transport retry settings owned by this pool. #[must_use] pub fn retry_settings(&self) -> &crate::HttpRetrySettings { return &self.inner.retry; } /// Executes the crate-internal next request id operation for `HttpTransportPool`. pub(crate) fn next_request_id(&self) -> u64 { let id = self.inner.request_ids.fetch_add(1, std::sync::atomic::Ordering::Relaxed); if id == 0 { return self.inner.request_ids.fetch_add(1, std::sync::atomic::Ordering::Relaxed); } return id; } /// Selects an endpoint for one standard audited RPC method without reserving runtime capacity. /// /// Request execution should use `acquire_for_method` so rate, cooldown and concurrency limits are enforced. pub fn select_for_method(&self, role: &crate::HttpRoleName, method: &crate::HttpRpcMethodDescriptor) -> ksp_core_lib::Result { let support = method.ensure_runtime_supported(); if let std::result::Result::Err(error) = support { return std::result::Result::Err(error); } return self.select_for_request_kind(role, &crate::HttpRequestKind::new(method.request_kind())); } /// Selects an endpoint for an open request-kind descriptor without reserving runtime capacity. pub fn select_for_request_kind( &self, role: &crate::HttpRoleName, request_kind: &crate::HttpRequestKind, ) -> ksp_core_lib::Result { let candidates = self.static_candidates(role, request_kind); if candidates.is_empty() { return selection_failed(role, request_kind); } let best_priority = candidates[0].priority; let mut tier_size = 0_usize; for candidate in &candidates { if candidate.priority != best_priority { break; } tier_size = tier_size.saturating_add(1); } let selected_position = self.next_position(role, request_kind, best_priority, tier_size); let selected = match candidates.get(selected_position) { std::option::Option::Some(selected) => selected, std::option::Option::None => return selection_failed(role, request_kind), }; return self.selection_from_candidate(role, request_kind, selected); } /// Acquires runtime capacity for one standard audited RPC method using the common timeout of matching endpoints. pub async fn acquire_for_method( &self, role: &crate::HttpRoleName, method: &crate::HttpRpcMethodDescriptor, ) -> ksp_core_lib::Result { let support = method.ensure_runtime_supported(); if let std::result::Result::Err(error) = support { return std::result::Result::Err(error); } return self.acquire_for_request_kind(role, &crate::HttpRequestKind::new(method.request_kind())).await; } /// Acquires runtime capacity for an open request-kind descriptor using the shortest configured request timeout among matching endpoints as the common /// deadline. pub async fn acquire_for_request_kind( &self, role: &crate::HttpRoleName, request_kind: &crate::HttpRequestKind, ) -> ksp_core_lib::Result { let timeout_result = self.common_request_timeout(role, request_kind); let timeout = match timeout_result { std::result::Result::Ok(timeout) => timeout, std::result::Result::Err(error) => return std::result::Result::Err(error), }; return self.acquire_for_request_kind_with_timeout(role, request_kind, timeout).await; } /// Acquires runtime capacity with an explicit end-to-end admission budget. /// /// This is primarily useful when a higher layer already owns a stricter request deadline. A zero timeout is rejected as immediately expired. pub async fn acquire_for_request_kind_with_timeout( &self, role: &crate::HttpRoleName, request_kind: &crate::HttpRequestKind, timeout: std::time::Duration, ) -> ksp_core_lib::Result { if timeout.is_zero() { return request_timeout(role, request_kind, "HTTP request admission deadline expired before selection"); } let now = std::time::Instant::now(); let deadline = match now.checked_add(timeout) { std::option::Option::Some(value) => value, std::option::Option::None => return request_timeout(role, request_kind, "HTTP request admission deadline could not be represented"), }; return self.acquire_until(role, request_kind, deadline).await; } /// Returns a safe pool snapshot without endpoint URLs. #[must_use] pub fn snapshot(&self) -> crate::HttpTransportPoolSnapshot { let endpoints = self.inner.clients.iter().map(|client| return client.snapshot()).collect(); return crate::HttpTransportPoolSnapshot { endpoints }; } async fn acquire_until( &self, role: &crate::HttpRoleName, request_kind: &crate::HttpRequestKind, deadline: std::time::Instant, ) -> ksp_core_lib::Result { let candidates = self.runtime_candidates(role, request_kind); if candidates.is_empty() { return request_selection_failed(role, request_kind); } loop { let now = std::time::Instant::now(); if now >= deadline { return request_timeout(role, request_kind, "HTTP request admission deadline expired while waiting for endpoint capacity"); } let attempt = self.try_candidates(role, request_kind, candidates.as_slice(), now, deadline); match attempt { RuntimeSelectionAttempt::Ready(permit) => return std::result::Result::Ok(permit), RuntimeSelectionAttempt::Blocked { earliest_ready, concurrency_saturated } => { let wait_result = self.wait_for_capacity(earliest_ready, concurrency_saturated, deadline).await; if !wait_result { return request_timeout(role, request_kind, "HTTP request admission deadline expired while waiting for endpoint capacity"); } }, RuntimeSelectionAttempt::Unavailable => return request_selection_failed(role, request_kind), } } } fn try_candidates( &self, role: &crate::HttpRoleName, request_kind: &crate::HttpRequestKind, candidates: &[RuntimePoolCandidate], now: std::time::Instant, deadline: std::time::Instant, ) -> RuntimeSelectionAttempt { let mut earliest_ready: std::option::Option = std::option::Option::None; let mut concurrency_saturated = false; let mut tier_start = 0_usize; while tier_start < candidates.len() { let priority = candidates[tier_start].priority; let mut tier_end = tier_start; while tier_end < candidates.len() && candidates[tier_end].priority == priority { tier_end = tier_end.saturating_add(1); } let tier_size = tier_end.saturating_sub(tier_start); let start_position = self.next_position(role, request_kind, priority, tier_size); let mut offset = 0_usize; while offset < tier_size { let position = tier_start.saturating_add((start_position.saturating_add(offset)) % tier_size); let candidate = &candidates[position]; let admission = candidate.runtime.try_acquire(now); match admission { crate::RoleAdmissionAttempt::Ready(concurrency_permit) => { let selection_result = self.selection_from_runtime_candidate(role, request_kind, candidate); let selection = match selection_result { std::result::Result::Ok(value) => value, std::result::Result::Err(_) => return RuntimeSelectionAttempt::Unavailable, }; ksp_logging_lib::debug!( target: crate::TRACING_TARGET, endpoint_name = selection.endpoint_name(), role = role.as_str(), request_kind = request_kind.as_str(), priority, remaining_deadline_ms = duration_millis_u64(deadline.saturating_duration_since(now)), "admitted HTTP request through logical endpoint pool" ); return RuntimeSelectionAttempt::Ready(crate::HttpRequestPermit { selection, deadline, role_runtime: std::sync::Arc::clone(&candidate.runtime), _concurrency_permit: concurrency_permit, }); }, crate::RoleAdmissionAttempt::BlockedUntil(ready_at) => { earliest_ready = earlier_instant(earliest_ready, ready_at); }, crate::RoleAdmissionAttempt::ConcurrencySaturated => { concurrency_saturated = true; }, crate::RoleAdmissionAttempt::Unavailable => {}, } offset = offset.saturating_add(1); } tier_start = tier_end; } if earliest_ready.is_none() && !concurrency_saturated { return RuntimeSelectionAttempt::Unavailable; } return RuntimeSelectionAttempt::Blocked { earliest_ready, concurrency_saturated }; } async fn wait_for_capacity( &self, earliest_ready: std::option::Option, concurrency_saturated: bool, deadline: std::time::Instant, ) -> bool { let now = std::time::Instant::now(); if now >= deadline { return false; } let wake_at = match earliest_ready { std::option::Option::Some(ready_at) => std::cmp::min(ready_at, deadline), std::option::Option::None => deadline, }; if concurrency_saturated { tokio::select! { () = self.inner.notify.notified() => {}, () = tokio::time::sleep_until(tokio::time::Instant::from_std(wake_at)) => {}, } } else { tokio::time::sleep_until(tokio::time::Instant::from_std(wake_at)).await; } return std::time::Instant::now() < deadline; } /// Executes the crate-internal common request timeout operation for `HttpTransportPool`. pub(crate) fn common_request_timeout( &self, role: &crate::HttpRoleName, request_kind: &crate::HttpRequestKind, ) -> ksp_core_lib::Result { let mut timeout: std::option::Option = std::option::Option::None; for client in &self.inner.clients { if client.matching_role(role, request_kind).is_none() { continue; } timeout = match timeout { std::option::Option::Some(current) => std::option::Option::Some(std::cmp::min(current, client.request_timeout())), std::option::Option::None => std::option::Option::Some(client.request_timeout()), }; } return match timeout { std::option::Option::Some(value) => std::result::Result::Ok(value), std::option::Option::None => selection_failed_duration(role, request_kind), }; } fn static_candidates(&self, role: &crate::HttpRoleName, request_kind: &crate::HttpRequestKind) -> std::vec::Vec { let mut candidates = std::vec::Vec::new(); for (client_index, client) in self.inner.clients.iter().enumerate() { let matching_role = client.matching_role(role, request_kind); if let std::option::Option::Some(matching_role) = matching_role { candidates.push(PoolCandidate { client_index, priority: matching_role.priority() }); } } candidates.sort_by_key(|candidate| return candidate.priority); return candidates; } fn runtime_candidates(&self, role: &crate::HttpRoleName, request_kind: &crate::HttpRequestKind) -> std::vec::Vec { let mut candidates = std::vec::Vec::new(); for (client_index, client) in self.inner.clients.iter().enumerate() { let runtime_match = client.matching_role_runtime(role, request_kind); if let std::option::Option::Some((priority, runtime)) = runtime_match { candidates.push(RuntimePoolCandidate { client_index, priority, runtime }); } } candidates.sort_by_key(|candidate| return candidate.priority); return candidates; } fn selection_from_candidate( &self, role: &crate::HttpRoleName, request_kind: &crate::HttpRequestKind, candidate: &PoolCandidate, ) -> ksp_core_lib::Result { let selected_client = self.inner.clients.get(candidate.client_index); let client = match selected_client { std::option::Option::Some(client) => client.clone(), std::option::Option::None => return selection_failed(role, request_kind), }; ksp_logging_lib::debug!( target: crate::TRACING_TARGET, endpoint_name = client.name(), role = role.as_str(), request_kind = request_kind.as_str(), priority = candidate.priority, "selected logical HTTP endpoint without runtime admission" ); return std::result::Result::Ok(crate::HttpEndpointSelection { client, role: role.clone(), request_kind: request_kind.clone(), priority: candidate.priority, }); } fn selection_from_runtime_candidate( &self, role: &crate::HttpRoleName, request_kind: &crate::HttpRequestKind, candidate: &RuntimePoolCandidate, ) -> ksp_core_lib::Result { let selected_client = self.inner.clients.get(candidate.client_index); let client = match selected_client { std::option::Option::Some(client) => client.clone(), std::option::Option::None => return selection_failed(role, request_kind), }; return std::result::Result::Ok(crate::HttpEndpointSelection { client, role: role.clone(), request_kind: request_kind.clone(), priority: candidate.priority, }); } fn next_position(&self, role: &crate::HttpRoleName, request_kind: &crate::HttpRequestKind, priority: u32, tier_size: usize) -> usize { let key = (role.as_str().to_owned(), request_kind.as_str().to_owned(), priority); let lock_result = self.inner.cursors.lock(); let mut cursors = match lock_result { std::result::Result::Ok(cursors) => cursors, std::result::Result::Err(poisoned) => poisoned.into_inner(), }; let cursor = cursors.entry(key).or_insert(0); if tier_size == 0 { return 0; } let selected = *cursor % tier_size; *cursor = (*cursor).wrapping_add(1); return selected; } } impl std::fmt::Debug for HttpTransportPool { fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { return formatter.debug_struct("HttpTransportPool").field("snapshot", &self.snapshot()).finish(); } } struct HttpTransportPoolInner { clients: std::vec::Vec, retry: crate::HttpRetrySettings, notify: std::sync::Arc, request_ids: std::sync::atomic::AtomicU64, cursors: std::sync::Mutex>, } #[derive(Clone, Copy)] struct PoolCandidate { client_index: usize, priority: u32, } struct RuntimePoolCandidate { client_index: usize, priority: u32, runtime: std::sync::Arc, } enum RuntimeSelectionAttempt { Ready(crate::HttpRequestPermit), Blocked { earliest_ready: std::option::Option, concurrency_saturated: bool }, Unavailable, } fn duration_millis_u64(duration: std::time::Duration) -> u64 { let converted = u64::try_from(duration.as_millis()); return match converted { std::result::Result::Ok(value) => value, std::result::Result::Err(_) => u64::MAX, }; } fn earlier_instant(current: std::option::Option, candidate: std::time::Instant) -> std::option::Option { return match current { std::option::Option::Some(value) => std::option::Option::Some(std::cmp::min(value, candidate)), std::option::Option::None => std::option::Option::Some(candidate), }; } fn selection_failed(role: &crate::HttpRoleName, request_kind: &crate::HttpRequestKind) -> ksp_core_lib::Result { return std::result::Result::Err(selection_error(role, request_kind)); } fn selection_failed_duration(role: &crate::HttpRoleName, request_kind: &crate::HttpRequestKind) -> ksp_core_lib::Result { return std::result::Result::Err(selection_error(role, request_kind)); } fn request_selection_failed(role: &crate::HttpRoleName, request_kind: &crate::HttpRequestKind) -> ksp_core_lib::Result { return std::result::Result::Err(selection_error(role, request_kind)); } fn selection_error(role: &crate::HttpRoleName, request_kind: &crate::HttpRequestKind) -> ksp_core_lib::Error { return ksp_core_lib::Error::new(crate::ERROR_CODE_ENDPOINT_SELECTION_FAILED, "no HTTP endpoint can satisfy the requested role and request kind") .with_context("role", role.as_str()) .with_context("request_kind", request_kind.as_str()); } fn request_timeout(role: &crate::HttpRoleName, request_kind: &crate::HttpRequestKind, message: &str) -> ksp_core_lib::Result { ksp_logging_lib::warn!( target: crate::TRACING_TARGET, role = role.as_str(), request_kind = request_kind.as_str(), "HTTP request admission deadline expired" ); return std::result::Result::Err( ksp_core_lib::Error::new(crate::ERROR_CODE_TIMEOUT, message) .with_context("role", role.as_str()) .with_context("request_kind", request_kind.as_str()), ); } #[cfg(test)] #[path = "../unit_tests/http_pool.rs"] mod tests;