621 lines
26 KiB
Rust
621 lines
26 KiB
Rust
// file: crates/ksp-onchain-transport-lib/src/pool.rs
|
|
// version: 7
|
|
|
|
/// Safe snapshot of the logical HTTP endpoint pool.
|
|
#[derive(Clone, Debug, Eq, PartialEq)]
|
|
pub struct HttpTransportPoolSnapshot {
|
|
endpoints: std::vec::Vec<crate::HttpEndpointSnapshot>,
|
|
}
|
|
|
|
impl HttpTransportPoolSnapshot {
|
|
/// Returns safe endpoint snapshots in configured declaration order.
|
|
#[must_use]
|
|
pub fn endpoints(&self) -> &[crate::HttpEndpointSnapshot] {
|
|
return self.endpoints.as_slice();
|
|
}
|
|
|
|
/// Returns the total number of configured logical endpoints.
|
|
#[must_use]
|
|
pub fn endpoint_count(&self) -> usize {
|
|
return self.endpoints.len();
|
|
}
|
|
|
|
/// Returns the number of endpoints currently eligible for normal routing.
|
|
#[must_use]
|
|
pub fn available_endpoint_count(&self) -> usize {
|
|
return self.endpoints.iter().filter(|endpoint| return endpoint.availability() == crate::HttpEndpointAvailability::Available).count();
|
|
}
|
|
}
|
|
|
|
/// Result of one logical endpoint selection.
|
|
#[derive(Clone, Debug)]
|
|
pub struct HttpEndpointSelection {
|
|
client: crate::HttpEndpointClient,
|
|
role: crate::HttpRoleName,
|
|
request_kind: crate::HttpRequestKind,
|
|
priority: u32,
|
|
}
|
|
|
|
impl HttpEndpointSelection {
|
|
/// Returns the selected endpoint client.
|
|
#[must_use]
|
|
pub const fn client(&self) -> &crate::HttpEndpointClient {
|
|
return &self.client;
|
|
}
|
|
|
|
/// Returns the selected endpoint identity.
|
|
#[must_use]
|
|
pub fn endpoint_name(&self) -> &str {
|
|
return self.client.name();
|
|
}
|
|
|
|
/// Returns the matched logical role.
|
|
#[must_use]
|
|
pub const fn role(&self) -> &crate::HttpRoleName {
|
|
return &self.role;
|
|
}
|
|
|
|
/// Returns the matched request-kind capability.
|
|
#[must_use]
|
|
pub const fn request_kind(&self) -> &crate::HttpRequestKind {
|
|
return &self.request_kind;
|
|
}
|
|
|
|
/// Returns the selected role priority where lower values are preferred.
|
|
#[must_use]
|
|
pub const fn priority(&self) -> u32 {
|
|
return self.priority;
|
|
}
|
|
}
|
|
|
|
/// Runtime admission permit for one HTTP request.
|
|
///
|
|
/// The permit reserves configured concurrency capacity and carries the common request deadline. Dropping it releases any semaphore capacity immediately.
|
|
pub struct HttpRequestPermit {
|
|
selection: crate::HttpEndpointSelection,
|
|
deadline: std::time::Instant,
|
|
role_runtime: std::sync::Arc<crate::HttpRoleRuntime>,
|
|
_concurrency_permit: crate::HttpConcurrencyPermit,
|
|
}
|
|
|
|
impl HttpRequestPermit {
|
|
/// Returns the selected logical endpoint and role.
|
|
#[must_use]
|
|
pub const fn selection(&self) -> &crate::HttpEndpointSelection {
|
|
return &self.selection;
|
|
}
|
|
|
|
/// Returns the selected endpoint client.
|
|
#[must_use]
|
|
pub fn client(&self) -> &crate::HttpEndpointClient {
|
|
return self.selection.client();
|
|
}
|
|
|
|
/// Returns the remaining duration in the common request budget.
|
|
#[must_use]
|
|
pub fn remaining_timeout(&self) -> std::time::Duration {
|
|
let now = std::time::Instant::now();
|
|
if now >= self.deadline {
|
|
return std::time::Duration::ZERO;
|
|
}
|
|
return self.deadline.duration_since(now);
|
|
}
|
|
|
|
/// Records a successful request and clears the passive degraded state for this endpoint role.
|
|
pub fn record_success(&self) {
|
|
self.role_runtime.record_success();
|
|
ksp_logging_lib::trace!(
|
|
target: crate::TRACING_TARGET,
|
|
endpoint_name = self.selection.endpoint_name(),
|
|
role = self.selection.role().as_str(),
|
|
request_kind = self.selection.request_kind().as_str(),
|
|
"recorded successful HTTP endpoint observation"
|
|
);
|
|
return;
|
|
}
|
|
|
|
/// Records a transport failure and marks the endpoint role degraded until a later success.
|
|
pub fn record_failure(&self) {
|
|
self.role_runtime.record_failure();
|
|
ksp_logging_lib::warn!(
|
|
target: crate::TRACING_TARGET,
|
|
endpoint_name = self.selection.endpoint_name(),
|
|
provider = self.selection.client().provider().as_str(),
|
|
cluster = self.selection.client().cluster().as_str(),
|
|
role = self.selection.role().as_str(),
|
|
request_kind = self.selection.request_kind().as_str(),
|
|
"HTTP endpoint role marked degraded after transport failure"
|
|
);
|
|
return;
|
|
}
|
|
|
|
/// Records provider rate limiting and applies the role cooldown.
|
|
///
|
|
/// A provider delay can extend the configured cooldown but is defensively capped by the Transport runtime before use.
|
|
pub fn record_rate_limited(&self, provider_retry_after: std::option::Option<std::time::Duration>) -> std::time::Duration {
|
|
let pause = self.role_runtime.record_rate_limited(provider_retry_after);
|
|
let cooldown_ms = duration_millis_u64(pause);
|
|
ksp_logging_lib::warn!(
|
|
target: crate::TRACING_TARGET,
|
|
endpoint_name = self.selection.endpoint_name(),
|
|
provider = self.selection.client().provider().as_str(),
|
|
cluster = self.selection.client().cluster().as_str(),
|
|
role = self.selection.role().as_str(),
|
|
request_kind = self.selection.request_kind().as_str(),
|
|
cooldown_ms,
|
|
"HTTP endpoint role entered provider rate-limit cooldown"
|
|
);
|
|
return pause;
|
|
}
|
|
}
|
|
|
|
impl std::fmt::Debug for HttpRequestPermit {
|
|
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
|
return formatter
|
|
.debug_struct("HttpRequestPermit")
|
|
.field("selection", &self.selection)
|
|
.field("remaining_timeout", &self.remaining_timeout())
|
|
.finish();
|
|
}
|
|
}
|
|
|
|
/// Shareable logical HTTP endpoint pool with priority routing, admission limits and bounded request deadlines.
|
|
#[derive(Clone)]
|
|
pub struct HttpTransportPool {
|
|
inner: std::sync::Arc<HttpTransportPoolInner>,
|
|
}
|
|
|
|
impl HttpTransportPool {
|
|
/// Builds a logical endpoint pool after validating all Transport-owned runtime settings.
|
|
pub fn new(settings: crate::HttpTransportSettings) -> ksp_core_lib::Result<Self> {
|
|
let validation = settings.validate();
|
|
if let std::result::Result::Err(error) = validation {
|
|
return std::result::Result::Err(error);
|
|
}
|
|
let notify = std::sync::Arc::new(tokio::sync::Notify::new());
|
|
let mut clients = std::vec::Vec::with_capacity(settings.endpoints().len());
|
|
for endpoint in settings.endpoints() {
|
|
let client_result = crate::HttpEndpointClient::new_with_notify(endpoint.clone(), std::sync::Arc::clone(¬ify));
|
|
let client = match client_result {
|
|
std::result::Result::Ok(client) => client,
|
|
std::result::Result::Err(error) => return std::result::Result::Err(error),
|
|
};
|
|
clients.push(client);
|
|
}
|
|
let pool = Self {
|
|
inner: std::sync::Arc::new(HttpTransportPoolInner {
|
|
clients,
|
|
retry: settings.retry().clone(),
|
|
notify,
|
|
request_ids: std::sync::atomic::AtomicU64::new(1),
|
|
cursors: std::sync::Mutex::new(std::collections::BTreeMap::new()),
|
|
}),
|
|
};
|
|
ksp_logging_lib::debug!(
|
|
target: crate::TRACING_TARGET,
|
|
endpoint_count = pool.inner.clients.len(),
|
|
available_endpoint_count = pool.snapshot().available_endpoint_count(),
|
|
max_retries = pool.inner.retry.max_retries(),
|
|
"created logical HTTP endpoint pool"
|
|
);
|
|
return std::result::Result::Ok(pool);
|
|
}
|
|
|
|
/// Returns the bounded transport retry settings owned by this pool.
|
|
#[must_use]
|
|
pub fn retry_settings(&self) -> &crate::HttpRetrySettings {
|
|
return &self.inner.retry;
|
|
}
|
|
|
|
/// Executes the crate-internal next request id operation for `HttpTransportPool`.
|
|
pub(crate) fn next_request_id(&self) -> u64 {
|
|
let id = self.inner.request_ids.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
|
|
if id == 0 {
|
|
return self.inner.request_ids.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
|
|
}
|
|
return id;
|
|
}
|
|
|
|
/// Selects an endpoint for one standard audited RPC method without reserving runtime capacity.
|
|
///
|
|
/// Request execution should use `acquire_for_method` so rate, cooldown and concurrency limits are enforced.
|
|
pub fn select_for_method(&self, role: &crate::HttpRoleName, method: &crate::HttpRpcMethodDescriptor) -> ksp_core_lib::Result<crate::HttpEndpointSelection> {
|
|
let support = method.ensure_runtime_supported();
|
|
if let std::result::Result::Err(error) = support {
|
|
return std::result::Result::Err(error);
|
|
}
|
|
return self.select_for_request_kind(role, &crate::HttpRequestKind::new(method.request_kind()));
|
|
}
|
|
|
|
/// Selects an endpoint for an open request-kind descriptor without reserving runtime capacity.
|
|
pub fn select_for_request_kind(
|
|
&self,
|
|
role: &crate::HttpRoleName,
|
|
request_kind: &crate::HttpRequestKind,
|
|
) -> ksp_core_lib::Result<crate::HttpEndpointSelection> {
|
|
let candidates = self.static_candidates(role, request_kind);
|
|
if candidates.is_empty() {
|
|
return selection_failed(role, request_kind);
|
|
}
|
|
let best_priority = candidates[0].priority;
|
|
let mut tier_size = 0_usize;
|
|
for candidate in &candidates {
|
|
if candidate.priority != best_priority {
|
|
break;
|
|
}
|
|
tier_size = tier_size.saturating_add(1);
|
|
}
|
|
let selected_position = self.next_position(role, request_kind, best_priority, tier_size);
|
|
let selected = match candidates.get(selected_position) {
|
|
std::option::Option::Some(selected) => selected,
|
|
std::option::Option::None => return selection_failed(role, request_kind),
|
|
};
|
|
return self.selection_from_candidate(role, request_kind, selected);
|
|
}
|
|
|
|
/// Acquires runtime capacity for one standard audited RPC method using the common timeout of matching endpoints.
|
|
pub async fn acquire_for_method(
|
|
&self,
|
|
role: &crate::HttpRoleName,
|
|
method: &crate::HttpRpcMethodDescriptor,
|
|
) -> ksp_core_lib::Result<crate::HttpRequestPermit> {
|
|
let support = method.ensure_runtime_supported();
|
|
if let std::result::Result::Err(error) = support {
|
|
return std::result::Result::Err(error);
|
|
}
|
|
return self.acquire_for_request_kind(role, &crate::HttpRequestKind::new(method.request_kind())).await;
|
|
}
|
|
|
|
/// Acquires runtime capacity for an open request-kind descriptor using the shortest configured request timeout among matching endpoints as the common
|
|
/// deadline.
|
|
pub async fn acquire_for_request_kind(
|
|
&self,
|
|
role: &crate::HttpRoleName,
|
|
request_kind: &crate::HttpRequestKind,
|
|
) -> ksp_core_lib::Result<crate::HttpRequestPermit> {
|
|
let timeout_result = self.common_request_timeout(role, request_kind);
|
|
let timeout = match timeout_result {
|
|
std::result::Result::Ok(timeout) => timeout,
|
|
std::result::Result::Err(error) => return std::result::Result::Err(error),
|
|
};
|
|
return self.acquire_for_request_kind_with_timeout(role, request_kind, timeout).await;
|
|
}
|
|
|
|
/// Acquires runtime capacity with an explicit end-to-end admission budget.
|
|
///
|
|
/// This is primarily useful when a higher layer already owns a stricter request deadline. A zero timeout is rejected as immediately expired.
|
|
pub async fn acquire_for_request_kind_with_timeout(
|
|
&self,
|
|
role: &crate::HttpRoleName,
|
|
request_kind: &crate::HttpRequestKind,
|
|
timeout: std::time::Duration,
|
|
) -> ksp_core_lib::Result<crate::HttpRequestPermit> {
|
|
if timeout.is_zero() {
|
|
return request_timeout(role, request_kind, "HTTP request admission deadline expired before selection");
|
|
}
|
|
let now = std::time::Instant::now();
|
|
let deadline = match now.checked_add(timeout) {
|
|
std::option::Option::Some(value) => value,
|
|
std::option::Option::None => return request_timeout(role, request_kind, "HTTP request admission deadline could not be represented"),
|
|
};
|
|
return self.acquire_until(role, request_kind, deadline).await;
|
|
}
|
|
|
|
/// Returns a safe pool snapshot without endpoint URLs.
|
|
#[must_use]
|
|
pub fn snapshot(&self) -> crate::HttpTransportPoolSnapshot {
|
|
let endpoints = self.inner.clients.iter().map(|client| return client.snapshot()).collect();
|
|
return crate::HttpTransportPoolSnapshot { endpoints };
|
|
}
|
|
|
|
async fn acquire_until(
|
|
&self,
|
|
role: &crate::HttpRoleName,
|
|
request_kind: &crate::HttpRequestKind,
|
|
deadline: std::time::Instant,
|
|
) -> ksp_core_lib::Result<crate::HttpRequestPermit> {
|
|
let candidates = self.runtime_candidates(role, request_kind);
|
|
if candidates.is_empty() {
|
|
return request_selection_failed(role, request_kind);
|
|
}
|
|
loop {
|
|
let now = std::time::Instant::now();
|
|
if now >= deadline {
|
|
return request_timeout(role, request_kind, "HTTP request admission deadline expired while waiting for endpoint capacity");
|
|
}
|
|
let attempt = self.try_candidates(role, request_kind, candidates.as_slice(), now, deadline);
|
|
match attempt {
|
|
RuntimeSelectionAttempt::Ready(permit) => return std::result::Result::Ok(permit),
|
|
RuntimeSelectionAttempt::Blocked { earliest_ready, concurrency_saturated } => {
|
|
let wait_result = self.wait_for_capacity(earliest_ready, concurrency_saturated, deadline).await;
|
|
if !wait_result {
|
|
return request_timeout(role, request_kind, "HTTP request admission deadline expired while waiting for endpoint capacity");
|
|
}
|
|
},
|
|
RuntimeSelectionAttempt::Unavailable => return request_selection_failed(role, request_kind),
|
|
}
|
|
}
|
|
}
|
|
|
|
fn try_candidates(
|
|
&self,
|
|
role: &crate::HttpRoleName,
|
|
request_kind: &crate::HttpRequestKind,
|
|
candidates: &[RuntimePoolCandidate],
|
|
now: std::time::Instant,
|
|
deadline: std::time::Instant,
|
|
) -> RuntimeSelectionAttempt {
|
|
let mut earliest_ready: std::option::Option<std::time::Instant> = std::option::Option::None;
|
|
let mut concurrency_saturated = false;
|
|
let mut tier_start = 0_usize;
|
|
while tier_start < candidates.len() {
|
|
let priority = candidates[tier_start].priority;
|
|
let mut tier_end = tier_start;
|
|
while tier_end < candidates.len() && candidates[tier_end].priority == priority {
|
|
tier_end = tier_end.saturating_add(1);
|
|
}
|
|
let tier_size = tier_end.saturating_sub(tier_start);
|
|
let start_position = self.next_position(role, request_kind, priority, tier_size);
|
|
let mut offset = 0_usize;
|
|
while offset < tier_size {
|
|
let position = tier_start.saturating_add((start_position.saturating_add(offset)) % tier_size);
|
|
let candidate = &candidates[position];
|
|
let admission = candidate.runtime.try_acquire(now);
|
|
match admission {
|
|
crate::RoleAdmissionAttempt::Ready(concurrency_permit) => {
|
|
let selection_result = self.selection_from_runtime_candidate(role, request_kind, candidate);
|
|
let selection = match selection_result {
|
|
std::result::Result::Ok(value) => value,
|
|
std::result::Result::Err(_) => return RuntimeSelectionAttempt::Unavailable,
|
|
};
|
|
ksp_logging_lib::debug!(
|
|
target: crate::TRACING_TARGET,
|
|
endpoint_name = selection.endpoint_name(),
|
|
role = role.as_str(),
|
|
request_kind = request_kind.as_str(),
|
|
priority,
|
|
remaining_deadline_ms = duration_millis_u64(deadline.saturating_duration_since(now)),
|
|
"admitted HTTP request through logical endpoint pool"
|
|
);
|
|
return RuntimeSelectionAttempt::Ready(crate::HttpRequestPermit {
|
|
selection,
|
|
deadline,
|
|
role_runtime: std::sync::Arc::clone(&candidate.runtime),
|
|
_concurrency_permit: concurrency_permit,
|
|
});
|
|
},
|
|
crate::RoleAdmissionAttempt::BlockedUntil(ready_at) => {
|
|
earliest_ready = earlier_instant(earliest_ready, ready_at);
|
|
},
|
|
crate::RoleAdmissionAttempt::ConcurrencySaturated => {
|
|
concurrency_saturated = true;
|
|
},
|
|
crate::RoleAdmissionAttempt::Unavailable => {},
|
|
}
|
|
offset = offset.saturating_add(1);
|
|
}
|
|
tier_start = tier_end;
|
|
}
|
|
if earliest_ready.is_none() && !concurrency_saturated {
|
|
return RuntimeSelectionAttempt::Unavailable;
|
|
}
|
|
return RuntimeSelectionAttempt::Blocked { earliest_ready, concurrency_saturated };
|
|
}
|
|
|
|
async fn wait_for_capacity(
|
|
&self,
|
|
earliest_ready: std::option::Option<std::time::Instant>,
|
|
concurrency_saturated: bool,
|
|
deadline: std::time::Instant,
|
|
) -> bool {
|
|
let now = std::time::Instant::now();
|
|
if now >= deadline {
|
|
return false;
|
|
}
|
|
let wake_at = match earliest_ready {
|
|
std::option::Option::Some(ready_at) => std::cmp::min(ready_at, deadline),
|
|
std::option::Option::None => deadline,
|
|
};
|
|
if concurrency_saturated {
|
|
tokio::select! {
|
|
() = self.inner.notify.notified() => {},
|
|
() = tokio::time::sleep_until(tokio::time::Instant::from_std(wake_at)) => {},
|
|
}
|
|
} else {
|
|
tokio::time::sleep_until(tokio::time::Instant::from_std(wake_at)).await;
|
|
}
|
|
return std::time::Instant::now() < deadline;
|
|
}
|
|
|
|
/// Executes the crate-internal common request timeout operation for `HttpTransportPool`.
|
|
pub(crate) fn common_request_timeout(
|
|
&self,
|
|
role: &crate::HttpRoleName,
|
|
request_kind: &crate::HttpRequestKind,
|
|
) -> ksp_core_lib::Result<std::time::Duration> {
|
|
let mut timeout: std::option::Option<std::time::Duration> = std::option::Option::None;
|
|
for client in &self.inner.clients {
|
|
if client.matching_role(role, request_kind).is_none() {
|
|
continue;
|
|
}
|
|
timeout = match timeout {
|
|
std::option::Option::Some(current) => std::option::Option::Some(std::cmp::min(current, client.request_timeout())),
|
|
std::option::Option::None => std::option::Option::Some(client.request_timeout()),
|
|
};
|
|
}
|
|
return match timeout {
|
|
std::option::Option::Some(value) => std::result::Result::Ok(value),
|
|
std::option::Option::None => selection_failed_duration(role, request_kind),
|
|
};
|
|
}
|
|
|
|
fn static_candidates(&self, role: &crate::HttpRoleName, request_kind: &crate::HttpRequestKind) -> std::vec::Vec<PoolCandidate> {
|
|
let mut candidates = std::vec::Vec::new();
|
|
for (client_index, client) in self.inner.clients.iter().enumerate() {
|
|
let matching_role = client.matching_role(role, request_kind);
|
|
if let std::option::Option::Some(matching_role) = matching_role {
|
|
candidates.push(PoolCandidate { client_index, priority: matching_role.priority() });
|
|
}
|
|
}
|
|
candidates.sort_by_key(|candidate| return candidate.priority);
|
|
return candidates;
|
|
}
|
|
|
|
fn runtime_candidates(&self, role: &crate::HttpRoleName, request_kind: &crate::HttpRequestKind) -> std::vec::Vec<RuntimePoolCandidate> {
|
|
let mut candidates = std::vec::Vec::new();
|
|
for (client_index, client) in self.inner.clients.iter().enumerate() {
|
|
let runtime_match = client.matching_role_runtime(role, request_kind);
|
|
if let std::option::Option::Some((priority, runtime)) = runtime_match {
|
|
candidates.push(RuntimePoolCandidate { client_index, priority, runtime });
|
|
}
|
|
}
|
|
candidates.sort_by_key(|candidate| return candidate.priority);
|
|
return candidates;
|
|
}
|
|
|
|
fn selection_from_candidate(
|
|
&self,
|
|
role: &crate::HttpRoleName,
|
|
request_kind: &crate::HttpRequestKind,
|
|
candidate: &PoolCandidate,
|
|
) -> ksp_core_lib::Result<crate::HttpEndpointSelection> {
|
|
let selected_client = self.inner.clients.get(candidate.client_index);
|
|
let client = match selected_client {
|
|
std::option::Option::Some(client) => client.clone(),
|
|
std::option::Option::None => return selection_failed(role, request_kind),
|
|
};
|
|
ksp_logging_lib::debug!(
|
|
target: crate::TRACING_TARGET,
|
|
endpoint_name = client.name(),
|
|
role = role.as_str(),
|
|
request_kind = request_kind.as_str(),
|
|
priority = candidate.priority,
|
|
"selected logical HTTP endpoint without runtime admission"
|
|
);
|
|
return std::result::Result::Ok(crate::HttpEndpointSelection {
|
|
client,
|
|
role: role.clone(),
|
|
request_kind: request_kind.clone(),
|
|
priority: candidate.priority,
|
|
});
|
|
}
|
|
|
|
fn selection_from_runtime_candidate(
|
|
&self,
|
|
role: &crate::HttpRoleName,
|
|
request_kind: &crate::HttpRequestKind,
|
|
candidate: &RuntimePoolCandidate,
|
|
) -> ksp_core_lib::Result<crate::HttpEndpointSelection> {
|
|
let selected_client = self.inner.clients.get(candidate.client_index);
|
|
let client = match selected_client {
|
|
std::option::Option::Some(client) => client.clone(),
|
|
std::option::Option::None => return selection_failed(role, request_kind),
|
|
};
|
|
return std::result::Result::Ok(crate::HttpEndpointSelection {
|
|
client,
|
|
role: role.clone(),
|
|
request_kind: request_kind.clone(),
|
|
priority: candidate.priority,
|
|
});
|
|
}
|
|
|
|
fn next_position(&self, role: &crate::HttpRoleName, request_kind: &crate::HttpRequestKind, priority: u32, tier_size: usize) -> usize {
|
|
let key = (role.as_str().to_owned(), request_kind.as_str().to_owned(), priority);
|
|
let lock_result = self.inner.cursors.lock();
|
|
let mut cursors = match lock_result {
|
|
std::result::Result::Ok(cursors) => cursors,
|
|
std::result::Result::Err(poisoned) => poisoned.into_inner(),
|
|
};
|
|
let cursor = cursors.entry(key).or_insert(0);
|
|
if tier_size == 0 {
|
|
return 0;
|
|
}
|
|
let selected = *cursor % tier_size;
|
|
*cursor = (*cursor).wrapping_add(1);
|
|
return selected;
|
|
}
|
|
}
|
|
|
|
impl std::fmt::Debug for HttpTransportPool {
|
|
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
|
return formatter.debug_struct("HttpTransportPool").field("snapshot", &self.snapshot()).finish();
|
|
}
|
|
}
|
|
|
|
struct HttpTransportPoolInner {
|
|
clients: std::vec::Vec<crate::HttpEndpointClient>,
|
|
retry: crate::HttpRetrySettings,
|
|
notify: std::sync::Arc<tokio::sync::Notify>,
|
|
request_ids: std::sync::atomic::AtomicU64,
|
|
cursors: std::sync::Mutex<std::collections::BTreeMap<(std::string::String, std::string::String, u32), usize>>,
|
|
}
|
|
|
|
#[derive(Clone, Copy)]
|
|
struct PoolCandidate {
|
|
client_index: usize,
|
|
priority: u32,
|
|
}
|
|
|
|
struct RuntimePoolCandidate {
|
|
client_index: usize,
|
|
priority: u32,
|
|
runtime: std::sync::Arc<crate::HttpRoleRuntime>,
|
|
}
|
|
|
|
enum RuntimeSelectionAttempt {
|
|
Ready(crate::HttpRequestPermit),
|
|
Blocked { earliest_ready: std::option::Option<std::time::Instant>, concurrency_saturated: bool },
|
|
Unavailable,
|
|
}
|
|
|
|
fn duration_millis_u64(duration: std::time::Duration) -> u64 {
|
|
let converted = u64::try_from(duration.as_millis());
|
|
return match converted {
|
|
std::result::Result::Ok(value) => value,
|
|
std::result::Result::Err(_) => u64::MAX,
|
|
};
|
|
}
|
|
|
|
fn earlier_instant(current: std::option::Option<std::time::Instant>, candidate: std::time::Instant) -> std::option::Option<std::time::Instant> {
|
|
return match current {
|
|
std::option::Option::Some(value) => std::option::Option::Some(std::cmp::min(value, candidate)),
|
|
std::option::Option::None => std::option::Option::Some(candidate),
|
|
};
|
|
}
|
|
|
|
fn selection_failed(role: &crate::HttpRoleName, request_kind: &crate::HttpRequestKind) -> ksp_core_lib::Result<crate::HttpEndpointSelection> {
|
|
return std::result::Result::Err(selection_error(role, request_kind));
|
|
}
|
|
|
|
fn selection_failed_duration(role: &crate::HttpRoleName, request_kind: &crate::HttpRequestKind) -> ksp_core_lib::Result<std::time::Duration> {
|
|
return std::result::Result::Err(selection_error(role, request_kind));
|
|
}
|
|
|
|
fn request_selection_failed(role: &crate::HttpRoleName, request_kind: &crate::HttpRequestKind) -> ksp_core_lib::Result<crate::HttpRequestPermit> {
|
|
return std::result::Result::Err(selection_error(role, request_kind));
|
|
}
|
|
|
|
fn selection_error(role: &crate::HttpRoleName, request_kind: &crate::HttpRequestKind) -> ksp_core_lib::Error {
|
|
return ksp_core_lib::Error::new(crate::ERROR_CODE_ENDPOINT_SELECTION_FAILED, "no HTTP endpoint can satisfy the requested role and request kind")
|
|
.with_context("role", role.as_str())
|
|
.with_context("request_kind", request_kind.as_str());
|
|
}
|
|
|
|
fn request_timeout(role: &crate::HttpRoleName, request_kind: &crate::HttpRequestKind, message: &str) -> ksp_core_lib::Result<crate::HttpRequestPermit> {
|
|
ksp_logging_lib::warn!(
|
|
target: crate::TRACING_TARGET,
|
|
role = role.as_str(),
|
|
request_kind = request_kind.as_str(),
|
|
"HTTP request admission deadline expired"
|
|
);
|
|
return std::result::Result::Err(
|
|
ksp_core_lib::Error::new(crate::ERROR_CODE_TIMEOUT, message)
|
|
.with_context("role", role.as_str())
|
|
.with_context("request_kind", request_kind.as_str()),
|
|
);
|
|
}
|
|
|
|
#[cfg(test)]
|
|
#[path = "../unit_tests/pool.rs"]
|
|
mod tests;
|