v0.2.11-pre.003
This commit is contained in:
@@ -1,12 +1,12 @@
|
||||
# file: Cargo.toml
|
||||
# version: 271
|
||||
# version: 272
|
||||
|
||||
[workspace]
|
||||
resolver = "3"
|
||||
members = ["crates/ksp-app-config-desk", "crates/ksp-app-wallet-desk", "crates/ksp-config-lib", "crates/ksp-core-lib", "crates/ksp-logging-lib", "crates/ksp-offchain-transport-lib", "crates/ksp-onchain-transport-lib", "crates/ksp-wallet-lib"]
|
||||
|
||||
[workspace.package]
|
||||
version = "0.2.11-pre.2.fix.3"
|
||||
version = "0.2.11-pre.3"
|
||||
edition = "2024"
|
||||
license = "MIT"
|
||||
repository = "https://git.sasedev.com/Sasedev/khadhroony-solana-project"
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
# file: crates/ksp-offchain-transport-lib/Cargo.toml
|
||||
# version: 2
|
||||
# version: 3
|
||||
|
||||
[package]
|
||||
name = "ksp-offchain-transport-lib"
|
||||
@@ -10,10 +10,12 @@ repository.workspace = true
|
||||
[dependencies]
|
||||
ksp-core-lib = { path = "../ksp-core-lib" }
|
||||
ksp-logging-lib = { path = "../ksp-logging-lib" }
|
||||
reqwest = { workspace = true, features = ["rustls"] }
|
||||
serde = { workspace = true, features = ["derive"] }
|
||||
serde_json.workspace = true
|
||||
|
||||
[dev-dependencies]
|
||||
serde_json.workspace = true
|
||||
tokio = { workspace = true, features = ["io-util", "macros", "net", "rt", "time"] }
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
|
||||
@@ -1,6 +1,30 @@
|
||||
// file: crates/ksp-offchain-transport-lib/src/error.rs
|
||||
// version: 3
|
||||
// version: 4
|
||||
|
||||
/// Stable off-chain transport error for an invalid crate-wide HTTP runtime configuration.
|
||||
pub const ERROR_CODE_HTTP_SETTINGS_INVALID: ksp_core_lib::ErrorCode = ksp_core_lib::ErrorCode::new("offchain_transport", "http_settings_invalid");
|
||||
/// Stable off-chain transport error when the hardened reqwest client cannot be initialized.
|
||||
pub const ERROR_CODE_HTTP_CLIENT_BUILD_FAILED: ksp_core_lib::ErrorCode = ksp_core_lib::ErrorCode::new("offchain_transport", "http_client_build_failed");
|
||||
/// Stable off-chain transport error for an invalid crate-owned HTTP request definition.
|
||||
pub const ERROR_CODE_HTTP_REQUEST_INVALID: ksp_core_lib::ErrorCode = ksp_core_lib::ErrorCode::new("offchain_transport", "http_request_invalid");
|
||||
/// Stable off-chain transport error for a connection failure without exposing the provider URL.
|
||||
pub const ERROR_CODE_HTTP_CONNECTION_FAILED: ksp_core_lib::ErrorCode = ksp_core_lib::ErrorCode::new("offchain_transport", "http_connection_failed");
|
||||
/// Stable off-chain transport error for an end-to-end HTTP timeout.
|
||||
pub const ERROR_CODE_HTTP_TIMEOUT: ksp_core_lib::ErrorCode = ksp_core_lib::ErrorCode::new("offchain_transport", "http_timeout");
|
||||
/// Stable off-chain transport error for a generic unsuccessful or transport-level HTTP request.
|
||||
pub const ERROR_CODE_HTTP_REQUEST_FAILED: ksp_core_lib::ErrorCode = ksp_core_lib::ErrorCode::new("offchain_transport", "http_request_failed");
|
||||
/// Stable off-chain transport error for HTTP 401/403 access denial.
|
||||
pub const ERROR_CODE_HTTP_ACCESS_DENIED: ksp_core_lib::ErrorCode = ksp_core_lib::ErrorCode::new("offchain_transport", "http_access_denied");
|
||||
/// Stable off-chain transport error for HTTP 429 rate limiting.
|
||||
pub const ERROR_CODE_HTTP_RATE_LIMITED: ksp_core_lib::ErrorCode = ksp_core_lib::ErrorCode::new("offchain_transport", "http_rate_limited");
|
||||
/// Stable off-chain transport error for transient HTTP status failures such as 408 or 5xx.
|
||||
pub const ERROR_CODE_HTTP_TEMPORARY_FAILURE: ksp_core_lib::ErrorCode = ksp_core_lib::ErrorCode::new("offchain_transport", "http_temporary_failure");
|
||||
/// Stable off-chain transport error when a response exceeds the defensive body limit.
|
||||
pub const ERROR_CODE_HTTP_RESPONSE_TOO_LARGE: ksp_core_lib::ErrorCode = ksp_core_lib::ErrorCode::new("offchain_transport", "http_response_too_large");
|
||||
/// Stable off-chain transport error when a successful HTTP response is not syntactically valid JSON.
|
||||
pub const ERROR_CODE_HTTP_INVALID_JSON: ksp_core_lib::ErrorCode = ksp_core_lib::ErrorCode::new("offchain_transport", "http_invalid_json");
|
||||
/// Stable off-chain transport error for an invalid local request-admission/rate-limit policy.
|
||||
pub const ERROR_CODE_HTTP_RATE_LIMIT_INVALID: ksp_core_lib::ErrorCode = ksp_core_lib::ErrorCode::new("offchain_transport", "http_rate_limit_invalid");
|
||||
/// Stable off-chain transport error for an invalid exact market-price decimal.
|
||||
pub const ERROR_CODE_MARKET_PRICE_DECIMAL_INVALID: ksp_core_lib::ErrorCode = ksp_core_lib::ErrorCode::new("offchain_transport", "market_price_decimal_invalid");
|
||||
/// Stable off-chain transport error for an invalid normalized market-price observation.
|
||||
|
||||
218
crates/ksp-offchain-transport-lib/src/http_admission.rs
Normal file
218
crates/ksp-offchain-transport-lib/src/http_admission.rs
Normal file
@@ -0,0 +1,218 @@
|
||||
// file: crates/ksp-offchain-transport-lib/src/http_admission.rs
|
||||
// version: 1
|
||||
|
||||
//! Provider-neutral local request admission and rate-limit cooldown primitives.
|
||||
|
||||
/// Maximum provider-directed cooldown accepted from `Retry-After`.
|
||||
pub(crate) const HTTP_MAX_RETRY_AFTER: std::time::Duration = std::time::Duration::from_secs(3_600);
|
||||
|
||||
const DEFAULT_RATE_LIMIT_COOLDOWN: std::time::Duration = std::time::Duration::from_secs(1);
|
||||
const MAX_RATE_LIMIT_WINDOW: std::time::Duration = std::time::Duration::from_secs(3_600);
|
||||
|
||||
/// Crate-internal local request-admission policy.
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
pub(crate) enum HttpAdmissionPolicy {
|
||||
/// No stable local cadence is known; only provider-driven cooldown is enforced.
|
||||
Dynamic,
|
||||
/// Enforce a smooth token bucket for a documented request budget and window.
|
||||
Fixed { requests: u32, window: std::time::Duration, burst: u32 },
|
||||
/// No local cadence limit is configured; provider-driven cooldown still applies.
|
||||
Unlimited,
|
||||
}
|
||||
|
||||
impl HttpAdmissionPolicy {
|
||||
/// Creates a validated fixed-window admission policy.
|
||||
pub(crate) fn fixed(requests: u32, window: std::time::Duration, burst: std::option::Option<u32>) -> ksp_core_lib::Result<Self> {
|
||||
let burst = match burst {
|
||||
std::option::Option::Some(value) => value,
|
||||
std::option::Option::None => 1,
|
||||
};
|
||||
if requests == 0 || window.is_zero() || window > MAX_RATE_LIMIT_WINDOW || burst == 0 || burst > requests {
|
||||
return std::result::Result::Err(
|
||||
ksp_core_lib::Error::new(crate::ERROR_CODE_HTTP_RATE_LIMIT_INVALID, "HTTP request-admission policy is invalid")
|
||||
.with_context("field", "rate_limit"),
|
||||
);
|
||||
}
|
||||
return std::result::Result::Ok(Self::Fixed { requests, window, burst });
|
||||
}
|
||||
}
|
||||
|
||||
/// Result of one non-blocking local admission attempt.
|
||||
pub(crate) enum HttpAdmissionDecision {
|
||||
/// The request may dispatch now and one local token has been consumed when applicable.
|
||||
Ready,
|
||||
/// The request must be deferred for at least this duration.
|
||||
Deferred(std::time::Duration),
|
||||
}
|
||||
|
||||
/// Shared non-blocking limiter used by provider adapters and later refresh orchestration.
|
||||
pub(crate) struct HttpAdmissionController {
|
||||
cooldown_until: std::sync::Mutex<std::option::Option<std::time::Instant>>,
|
||||
fallback_cooldown: std::time::Duration,
|
||||
policy: HttpAdmissionPolicy,
|
||||
token_bucket: std::sync::Mutex<std::option::Option<HttpTokenBucketState>>,
|
||||
}
|
||||
|
||||
impl HttpAdmissionController {
|
||||
/// Creates one limiter from a provider-owned local admission policy.
|
||||
pub(crate) fn new(policy: HttpAdmissionPolicy, fallback_cooldown: std::option::Option<std::time::Duration>) -> ksp_core_lib::Result<Self> {
|
||||
let fallback_cooldown = match fallback_cooldown {
|
||||
std::option::Option::Some(value) => value,
|
||||
std::option::Option::None => DEFAULT_RATE_LIMIT_COOLDOWN,
|
||||
};
|
||||
if fallback_cooldown.is_zero() || fallback_cooldown > HTTP_MAX_RETRY_AFTER {
|
||||
return std::result::Result::Err(
|
||||
ksp_core_lib::Error::new(crate::ERROR_CODE_HTTP_RATE_LIMIT_INVALID, "HTTP fallback cooldown is outside the supported bounds")
|
||||
.with_context("field", "fallback_cooldown"),
|
||||
);
|
||||
}
|
||||
let token_bucket = match policy {
|
||||
HttpAdmissionPolicy::Fixed { requests, window, burst } => {
|
||||
std::option::Option::Some(HttpTokenBucketState::new(requests, window, burst, std::time::Instant::now()))
|
||||
},
|
||||
HttpAdmissionPolicy::Dynamic | HttpAdmissionPolicy::Unlimited => std::option::Option::None,
|
||||
};
|
||||
return std::result::Result::Ok(Self {
|
||||
cooldown_until: std::sync::Mutex::new(std::option::Option::None),
|
||||
fallback_cooldown,
|
||||
policy,
|
||||
token_bucket: std::sync::Mutex::new(token_bucket),
|
||||
});
|
||||
}
|
||||
|
||||
/// Returns the configured local policy.
|
||||
#[must_use]
|
||||
pub(crate) const fn policy(&self) -> HttpAdmissionPolicy {
|
||||
return self.policy;
|
||||
}
|
||||
|
||||
/// Tries to admit one request immediately without sleeping.
|
||||
pub(crate) fn try_admit(&self) -> HttpAdmissionDecision {
|
||||
return self.try_admit_at(std::time::Instant::now());
|
||||
}
|
||||
|
||||
/// Records a provider 429 and extends cooldown using a bounded `Retry-After` value when present.
|
||||
pub(crate) fn record_rate_limited(&self, provider_retry_after: std::option::Option<std::time::Duration>) -> std::time::Duration {
|
||||
let provider_delay = match provider_retry_after {
|
||||
std::option::Option::Some(value) => std::cmp::min(value, HTTP_MAX_RETRY_AFTER),
|
||||
std::option::Option::None => std::time::Duration::ZERO,
|
||||
};
|
||||
let effective = std::cmp::max(self.fallback_cooldown, provider_delay);
|
||||
self.record_cooldown_until(std::time::Instant::now(), effective);
|
||||
ksp_logging_lib::debug!(
|
||||
target: crate::TRACING_TARGET,
|
||||
cooldown_ms = duration_millis_u64(effective),
|
||||
provider_retry_after_present = provider_retry_after.is_some(),
|
||||
"recorded off-chain HTTP provider cooldown"
|
||||
);
|
||||
return effective;
|
||||
}
|
||||
|
||||
/// Returns the remaining provider cooldown, if any.
|
||||
pub(crate) fn cooldown_remaining(&self) -> std::option::Option<std::time::Duration> {
|
||||
return self.cooldown_remaining_at(std::time::Instant::now());
|
||||
}
|
||||
|
||||
fn try_admit_at(&self, now: std::time::Instant) -> HttpAdmissionDecision {
|
||||
if let std::option::Option::Some(remaining) = self.cooldown_remaining_at(now) {
|
||||
return HttpAdmissionDecision::Deferred(remaining);
|
||||
}
|
||||
let lock_result = self.token_bucket.lock();
|
||||
let mut token_bucket = match lock_result {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(poisoned) => poisoned.into_inner(),
|
||||
};
|
||||
let state = match token_bucket.as_mut() {
|
||||
std::option::Option::Some(value) => value,
|
||||
std::option::Option::None => return HttpAdmissionDecision::Ready,
|
||||
};
|
||||
return match state.try_consume_at(now) {
|
||||
std::option::Option::Some(delay) => HttpAdmissionDecision::Deferred(delay),
|
||||
std::option::Option::None => HttpAdmissionDecision::Ready,
|
||||
};
|
||||
}
|
||||
|
||||
fn cooldown_remaining_at(&self, now: std::time::Instant) -> std::option::Option<std::time::Duration> {
|
||||
let lock_result = self.cooldown_until.lock();
|
||||
let mut cooldown_until = match lock_result {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(poisoned) => poisoned.into_inner(),
|
||||
};
|
||||
let deadline = match *cooldown_until {
|
||||
std::option::Option::Some(value) => value,
|
||||
std::option::Option::None => return std::option::Option::None,
|
||||
};
|
||||
if deadline <= now {
|
||||
*cooldown_until = std::option::Option::None;
|
||||
return std::option::Option::None;
|
||||
}
|
||||
return std::option::Option::Some(deadline.duration_since(now));
|
||||
}
|
||||
|
||||
fn record_cooldown_until(&self, now: std::time::Instant, delay: std::time::Duration) {
|
||||
let candidate = match now.checked_add(delay) {
|
||||
std::option::Option::Some(value) => value,
|
||||
std::option::Option::None => now,
|
||||
};
|
||||
let lock_result = self.cooldown_until.lock();
|
||||
let mut cooldown_until = match lock_result {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(poisoned) => poisoned.into_inner(),
|
||||
};
|
||||
let replace = match *cooldown_until {
|
||||
std::option::Option::Some(current) => candidate > current,
|
||||
std::option::Option::None => true,
|
||||
};
|
||||
if replace {
|
||||
*cooldown_until = std::option::Option::Some(candidate);
|
||||
}
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
struct HttpTokenBucketState {
|
||||
available_tokens: f64,
|
||||
burst: u32,
|
||||
last_refill: std::time::Instant,
|
||||
refill_per_second: f64,
|
||||
}
|
||||
|
||||
impl HttpTokenBucketState {
|
||||
fn new(requests: u32, window: std::time::Duration, burst: u32, now: std::time::Instant) -> Self {
|
||||
let refill_per_second = f64::from(requests) / window.as_secs_f64();
|
||||
return Self { available_tokens: f64::from(burst), burst, last_refill: now, refill_per_second };
|
||||
}
|
||||
|
||||
fn try_consume_at(&mut self, now: std::time::Instant) -> std::option::Option<std::time::Duration> {
|
||||
self.refill_at(now);
|
||||
if self.available_tokens >= 1.0 {
|
||||
self.available_tokens -= 1.0;
|
||||
return std::option::Option::None;
|
||||
}
|
||||
let missing = 1.0 - self.available_tokens;
|
||||
let wait_seconds = missing / self.refill_per_second;
|
||||
return std::option::Option::Some(std::time::Duration::from_secs_f64(wait_seconds));
|
||||
}
|
||||
|
||||
fn refill_at(&mut self, now: std::time::Instant) {
|
||||
if now <= self.last_refill {
|
||||
return;
|
||||
}
|
||||
let elapsed = now.duration_since(self.last_refill).as_secs_f64();
|
||||
self.available_tokens = (self.available_tokens + elapsed * self.refill_per_second).min(f64::from(self.burst));
|
||||
self.last_refill = now;
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
fn duration_millis_u64(duration: std::time::Duration) -> u64 {
|
||||
return match u64::try_from(duration.as_millis()) {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(_) => u64::MAX,
|
||||
};
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[path = "../unit_tests/http_admission.rs"]
|
||||
mod tests;
|
||||
331
crates/ksp-offchain-transport-lib/src/http_client.rs
Normal file
331
crates/ksp-offchain-transport-lib/src/http_client.rs
Normal file
@@ -0,0 +1,331 @@
|
||||
// file: crates/ksp-offchain-transport-lib/src/http_client.rs
|
||||
// version: 1
|
||||
|
||||
//! Crate-wide hardened REST client used internally by off-chain capability adapters.
|
||||
|
||||
/// Bounded successful JSON document returned by the crate-internal REST client.
|
||||
pub(crate) struct HttpJsonDocument {
|
||||
bytes: std::vec::Vec<u8>,
|
||||
}
|
||||
|
||||
impl HttpJsonDocument {
|
||||
/// Returns the validated raw JSON bytes for provider-specific typed deserialization.
|
||||
#[must_use]
|
||||
pub(crate) fn as_bytes(&self) -> &[u8] {
|
||||
return self.bytes.as_slice();
|
||||
}
|
||||
}
|
||||
|
||||
impl std::fmt::Debug for HttpJsonDocument {
|
||||
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
return formatter.debug_struct("HttpJsonDocument").field("byte_len", &self.bytes.len()).finish();
|
||||
}
|
||||
}
|
||||
|
||||
/// Crate-internal fixed-origin GET request.
|
||||
///
|
||||
/// URLs and headers are deliberately absent from [`std::fmt::Debug`] because future provider adapters can attach credentials to headers.
|
||||
pub(crate) struct HttpGetRequest {
|
||||
headers: reqwest::header::HeaderMap,
|
||||
url: reqwest::Url,
|
||||
}
|
||||
|
||||
impl HttpGetRequest {
|
||||
/// Creates one HTTPS GET request from a crate-owned official provider URL.
|
||||
pub(crate) fn new_https(url: &'static str) -> ksp_core_lib::Result<Self> {
|
||||
return Self::parse(url, false);
|
||||
}
|
||||
|
||||
/// Appends one non-secret query pair using URL encoding.
|
||||
pub(crate) fn append_query_pair(&mut self, name: &'static str, value: &str) {
|
||||
self.url.query_pairs_mut().append_pair(name, value);
|
||||
return;
|
||||
}
|
||||
|
||||
/// Adds one sensitive header without exposing its value through this type's debug representation.
|
||||
pub(crate) fn insert_sensitive_header(&mut self, name: &'static str, value: &str) -> ksp_core_lib::Result<()> {
|
||||
let name_result = reqwest::header::HeaderName::from_bytes(name.as_bytes());
|
||||
let name = match name_result {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(error) => {
|
||||
return std::result::Result::Err(
|
||||
ksp_core_lib::Error::new(crate::ERROR_CODE_HTTP_REQUEST_INVALID, "Off-chain HTTP header name is invalid")
|
||||
.with_context("field", "header_name")
|
||||
.with_source(error),
|
||||
);
|
||||
},
|
||||
};
|
||||
let value_result = reqwest::header::HeaderValue::from_bytes(value.as_bytes());
|
||||
let mut value = match value_result {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(error) => {
|
||||
return std::result::Result::Err(
|
||||
ksp_core_lib::Error::new(crate::ERROR_CODE_HTTP_REQUEST_INVALID, "Off-chain HTTP header value is invalid")
|
||||
.with_context("field", "header_value")
|
||||
.with_source(error),
|
||||
);
|
||||
},
|
||||
};
|
||||
value.set_sensitive(true);
|
||||
self.headers.insert(name, value);
|
||||
return std::result::Result::Ok(());
|
||||
}
|
||||
|
||||
/// Creates a plain-HTTP request for loopback-only deterministic unit tests.
|
||||
#[cfg(test)]
|
||||
pub(crate) fn new_test_http(url: &str) -> ksp_core_lib::Result<Self> {
|
||||
return Self::parse(url, true);
|
||||
}
|
||||
|
||||
fn parse(url: &str, allow_http_for_tests: bool) -> ksp_core_lib::Result<Self> {
|
||||
let parsed_result = reqwest::Url::parse(url);
|
||||
let parsed = match parsed_result {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(error) => {
|
||||
return std::result::Result::Err(
|
||||
ksp_core_lib::Error::new(crate::ERROR_CODE_HTTP_REQUEST_INVALID, "Off-chain provider URL is invalid")
|
||||
.with_context("field", "provider_url")
|
||||
.with_source(error),
|
||||
);
|
||||
},
|
||||
};
|
||||
let scheme_allowed = parsed.scheme() == "https" || (allow_http_for_tests && parsed.scheme() == "http");
|
||||
if !scheme_allowed || parsed.host_str().is_none() {
|
||||
return std::result::Result::Err(
|
||||
ksp_core_lib::Error::new(crate::ERROR_CODE_HTTP_REQUEST_INVALID, "Off-chain provider URL must use an allowed scheme and host")
|
||||
.with_context("field", "provider_url"),
|
||||
);
|
||||
}
|
||||
if !parsed.username().is_empty() || parsed.password().is_some() {
|
||||
return std::result::Result::Err(
|
||||
ksp_core_lib::Error::new(crate::ERROR_CODE_HTTP_REQUEST_INVALID, "Off-chain provider URL cannot embed credentials")
|
||||
.with_context("field", "provider_url"),
|
||||
);
|
||||
}
|
||||
return std::result::Result::Ok(Self { headers: reqwest::header::HeaderMap::new(), url: parsed });
|
||||
}
|
||||
}
|
||||
|
||||
impl std::fmt::Debug for HttpGetRequest {
|
||||
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
return formatter.write_str("HttpGetRequest(<redacted>)");
|
||||
}
|
||||
}
|
||||
|
||||
/// Shareable hardened REST client owned by Off-chain Transport.
|
||||
#[derive(Clone)]
|
||||
pub(crate) struct HttpRestClient {
|
||||
client: reqwest::Client,
|
||||
settings: crate::HttpClientSettings,
|
||||
}
|
||||
|
||||
impl HttpRestClient {
|
||||
/// Builds one hardened client with redirects, system proxies and reqwest automatic retries disabled.
|
||||
pub(crate) fn new(settings: crate::HttpClientSettings) -> ksp_core_lib::Result<Self> {
|
||||
let client_result = reqwest::Client::builder()
|
||||
.connect_timeout(settings.connect_timeout())
|
||||
.timeout(settings.request_timeout())
|
||||
.redirect(reqwest::redirect::Policy::none())
|
||||
.referer(false)
|
||||
.retry(reqwest::retry::never())
|
||||
.no_proxy()
|
||||
.user_agent(concat!(env!("CARGO_PKG_NAME"), "/", env!("CARGO_PKG_VERSION")))
|
||||
.build();
|
||||
let client = match client_result {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(error) => {
|
||||
return std::result::Result::Err(
|
||||
ksp_core_lib::Error::new(crate::ERROR_CODE_HTTP_CLIENT_BUILD_FAILED, "Off-chain HTTP client could not be initialized")
|
||||
.with_source(error.without_url()),
|
||||
);
|
||||
},
|
||||
};
|
||||
ksp_logging_lib::debug!(
|
||||
target: crate::TRACING_TARGET,
|
||||
connect_timeout_ms = duration_millis_u64(settings.connect_timeout()),
|
||||
request_timeout_ms = duration_millis_u64(settings.request_timeout()),
|
||||
max_response_body_bytes = settings.max_response_body_bytes(),
|
||||
"created hardened off-chain HTTP REST client"
|
||||
);
|
||||
return std::result::Result::Ok(Self { client, settings });
|
||||
}
|
||||
|
||||
/// Executes one GET request and returns only a bounded syntactically valid JSON document.
|
||||
pub(crate) async fn get_json(&self, provider: &'static str, operation: &'static str, request: HttpGetRequest) -> ksp_core_lib::Result<HttpJsonDocument> {
|
||||
let send_result = self.client.get(request.url).headers(request.headers).send().await;
|
||||
let mut response = match send_result {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(error) => return std::result::Result::Err(map_reqwest_error(provider, operation, error)),
|
||||
};
|
||||
let status = response.status().as_u16();
|
||||
let retry_after = parse_retry_after(response.headers());
|
||||
if !(200..300).contains(&status) {
|
||||
return classify_http_status(provider, operation, status, retry_after);
|
||||
}
|
||||
if let std::option::Option::Some(content_length) = response.content_length()
|
||||
&& content_length > usize_to_u64(self.settings.max_response_body_bytes())
|
||||
{
|
||||
return response_too_large(provider, operation, self.settings.max_response_body_bytes());
|
||||
}
|
||||
let mut body = std::vec::Vec::new();
|
||||
loop {
|
||||
let chunk_result = response.chunk().await;
|
||||
let chunk = match chunk_result {
|
||||
std::result::Result::Ok(std::option::Option::Some(value)) => value,
|
||||
std::result::Result::Ok(std::option::Option::None) => break,
|
||||
std::result::Result::Err(error) => return std::result::Result::Err(map_reqwest_error(provider, operation, error)),
|
||||
};
|
||||
let next_len = body.len().saturating_add(chunk.len());
|
||||
if next_len > self.settings.max_response_body_bytes() {
|
||||
return response_too_large(provider, operation, self.settings.max_response_body_bytes());
|
||||
}
|
||||
body.extend_from_slice(chunk.as_ref());
|
||||
}
|
||||
let json_validation = serde_json::from_slice::<serde::de::IgnoredAny>(body.as_slice());
|
||||
if let std::result::Result::Err(error) = json_validation {
|
||||
return std::result::Result::Err(
|
||||
ksp_core_lib::Error::new(crate::ERROR_CODE_HTTP_INVALID_JSON, "Off-chain provider returned invalid JSON")
|
||||
.with_context("provider", provider)
|
||||
.with_context("operation", operation)
|
||||
.with_source(error),
|
||||
);
|
||||
}
|
||||
ksp_logging_lib::debug!(
|
||||
target: crate::TRACING_TARGET,
|
||||
provider = provider,
|
||||
operation = operation,
|
||||
http_status = status,
|
||||
response_body_bytes = body.len(),
|
||||
"completed off-chain HTTP REST request"
|
||||
);
|
||||
return std::result::Result::Ok(HttpJsonDocument { bytes: body });
|
||||
}
|
||||
}
|
||||
|
||||
fn classify_http_status(
|
||||
provider: &'static str,
|
||||
operation: &'static str,
|
||||
status: u16,
|
||||
retry_after: std::option::Option<std::time::Duration>,
|
||||
) -> ksp_core_lib::Result<HttpJsonDocument> {
|
||||
if status == 401 || status == 403 {
|
||||
return std::result::Result::Err(http_status_error(
|
||||
crate::ERROR_CODE_HTTP_ACCESS_DENIED,
|
||||
"Off-chain provider denied HTTP access",
|
||||
provider,
|
||||
operation,
|
||||
status,
|
||||
std::option::Option::None,
|
||||
));
|
||||
}
|
||||
if status == 429 {
|
||||
return std::result::Result::Err(http_status_error(
|
||||
crate::ERROR_CODE_HTTP_RATE_LIMITED,
|
||||
"Off-chain provider rate-limited the request",
|
||||
provider,
|
||||
operation,
|
||||
status,
|
||||
retry_after,
|
||||
));
|
||||
}
|
||||
if status == 408 || (500..600).contains(&status) {
|
||||
return std::result::Result::Err(http_status_error(
|
||||
crate::ERROR_CODE_HTTP_TEMPORARY_FAILURE,
|
||||
"Off-chain provider returned a temporary HTTP failure",
|
||||
provider,
|
||||
operation,
|
||||
status,
|
||||
retry_after,
|
||||
));
|
||||
}
|
||||
return std::result::Result::Err(http_status_error(
|
||||
crate::ERROR_CODE_HTTP_REQUEST_FAILED,
|
||||
"Off-chain provider returned an unsuccessful HTTP status",
|
||||
provider,
|
||||
operation,
|
||||
status,
|
||||
std::option::Option::None,
|
||||
));
|
||||
}
|
||||
|
||||
fn http_status_error(
|
||||
code: ksp_core_lib::ErrorCode,
|
||||
message: &'static str,
|
||||
provider: &'static str,
|
||||
operation: &'static str,
|
||||
status: u16,
|
||||
retry_after: std::option::Option<std::time::Duration>,
|
||||
) -> ksp_core_lib::Error {
|
||||
let mut error = ksp_core_lib::Error::new(code, message)
|
||||
.with_context("provider", provider)
|
||||
.with_context("operation", operation)
|
||||
.with_context("http_status", status.to_string());
|
||||
if let std::option::Option::Some(value) = retry_after {
|
||||
error = error.with_context("retry_after_seconds", value.as_secs().to_string());
|
||||
}
|
||||
return error;
|
||||
}
|
||||
|
||||
fn map_reqwest_error(provider: &'static str, operation: &'static str, error: reqwest::Error) -> ksp_core_lib::Error {
|
||||
let code = if error.is_timeout() {
|
||||
crate::ERROR_CODE_HTTP_TIMEOUT
|
||||
} else if error.is_connect() {
|
||||
crate::ERROR_CODE_HTTP_CONNECTION_FAILED
|
||||
} else {
|
||||
crate::ERROR_CODE_HTTP_REQUEST_FAILED
|
||||
};
|
||||
let message = if code == crate::ERROR_CODE_HTTP_TIMEOUT {
|
||||
"Off-chain HTTP request timed out"
|
||||
} else if code == crate::ERROR_CODE_HTTP_CONNECTION_FAILED {
|
||||
"Off-chain HTTP connection failed"
|
||||
} else {
|
||||
"Off-chain HTTP request failed"
|
||||
};
|
||||
return ksp_core_lib::Error::new(code, message)
|
||||
.with_context("provider", provider)
|
||||
.with_context("operation", operation)
|
||||
.with_source(error.without_url());
|
||||
}
|
||||
|
||||
fn parse_retry_after(headers: &reqwest::header::HeaderMap) -> std::option::Option<std::time::Duration> {
|
||||
let value = match headers.get(reqwest::header::RETRY_AFTER) {
|
||||
std::option::Option::Some(value) => value,
|
||||
std::option::Option::None => return std::option::Option::None,
|
||||
};
|
||||
let text = match value.to_str() {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(_) => return std::option::Option::None,
|
||||
};
|
||||
let seconds = match text.parse::<u64>() {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(_) => return std::option::Option::None,
|
||||
};
|
||||
return std::option::Option::Some(std::cmp::min(std::time::Duration::from_secs(seconds), crate::HTTP_MAX_RETRY_AFTER));
|
||||
}
|
||||
|
||||
fn response_too_large(provider: &'static str, operation: &'static str, limit: usize) -> ksp_core_lib::Result<HttpJsonDocument> {
|
||||
return std::result::Result::Err(
|
||||
ksp_core_lib::Error::new(crate::ERROR_CODE_HTTP_RESPONSE_TOO_LARGE, "Off-chain provider response exceeded the configured body limit")
|
||||
.with_context("provider", provider)
|
||||
.with_context("operation", operation)
|
||||
.with_context("max_response_body_bytes", limit.to_string()),
|
||||
);
|
||||
}
|
||||
|
||||
fn duration_millis_u64(duration: std::time::Duration) -> u64 {
|
||||
return match u64::try_from(duration.as_millis()) {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(_) => u64::MAX,
|
||||
};
|
||||
}
|
||||
|
||||
fn usize_to_u64(value: usize) -> u64 {
|
||||
return match u64::try_from(value) {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(_) => u64::MAX,
|
||||
};
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[path = "../unit_tests/http_client.rs"]
|
||||
mod tests;
|
||||
79
crates/ksp-offchain-transport-lib/src/http_settings.rs
Normal file
79
crates/ksp-offchain-transport-lib/src/http_settings.rs
Normal file
@@ -0,0 +1,79 @@
|
||||
// file: crates/ksp-offchain-transport-lib/src/http_settings.rs
|
||||
// version: 1
|
||||
|
||||
//! Crate-wide bounded HTTP runtime settings shared by off-chain capability families.
|
||||
|
||||
const DEFAULT_CONNECT_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(5);
|
||||
const DEFAULT_MAX_RESPONSE_BODY_BYTES: usize = 1_048_576;
|
||||
const DEFAULT_REQUEST_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(10);
|
||||
const MAX_CONNECT_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(30);
|
||||
const MAX_REQUEST_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(120);
|
||||
const MAX_RESPONSE_BODY_BYTES: usize = 4_194_304;
|
||||
|
||||
/// Crate-internal HTTP client settings with defensive hard bounds.
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
pub(crate) struct HttpClientSettings {
|
||||
connect_timeout: std::time::Duration,
|
||||
max_response_body_bytes: usize,
|
||||
request_timeout: std::time::Duration,
|
||||
}
|
||||
|
||||
impl HttpClientSettings {
|
||||
/// Creates one validated HTTP settings value.
|
||||
pub(crate) fn new(
|
||||
connect_timeout: std::time::Duration,
|
||||
request_timeout: std::time::Duration,
|
||||
max_response_body_bytes: usize,
|
||||
) -> ksp_core_lib::Result<Self> {
|
||||
if connect_timeout.is_zero() || connect_timeout > MAX_CONNECT_TIMEOUT {
|
||||
return invalid_http_settings("HTTP connect timeout is outside the supported bounds", "connect_timeout");
|
||||
}
|
||||
if request_timeout.is_zero() || request_timeout > MAX_REQUEST_TIMEOUT {
|
||||
return invalid_http_settings("HTTP request timeout is outside the supported bounds", "request_timeout");
|
||||
}
|
||||
if connect_timeout > request_timeout {
|
||||
return invalid_http_settings("HTTP connect timeout cannot exceed the total request timeout", "connect_timeout");
|
||||
}
|
||||
if max_response_body_bytes == 0 || max_response_body_bytes > MAX_RESPONSE_BODY_BYTES {
|
||||
return invalid_http_settings("HTTP response-body limit is outside the supported bounds", "max_response_body_bytes");
|
||||
}
|
||||
return std::result::Result::Ok(Self { connect_timeout, max_response_body_bytes, request_timeout });
|
||||
}
|
||||
|
||||
/// Returns the bounded connect timeout.
|
||||
#[must_use]
|
||||
pub(crate) const fn connect_timeout(&self) -> std::time::Duration {
|
||||
return self.connect_timeout;
|
||||
}
|
||||
|
||||
/// Returns the maximum decoded response body accepted before JSON parsing.
|
||||
#[must_use]
|
||||
pub(crate) const fn max_response_body_bytes(&self) -> usize {
|
||||
return self.max_response_body_bytes;
|
||||
}
|
||||
|
||||
/// Returns the end-to-end request timeout.
|
||||
#[must_use]
|
||||
pub(crate) const fn request_timeout(&self) -> std::time::Duration {
|
||||
return self.request_timeout;
|
||||
}
|
||||
}
|
||||
|
||||
impl std::default::Default for HttpClientSettings {
|
||||
fn default() -> Self {
|
||||
return Self {
|
||||
connect_timeout: DEFAULT_CONNECT_TIMEOUT,
|
||||
max_response_body_bytes: DEFAULT_MAX_RESPONSE_BODY_BYTES,
|
||||
request_timeout: DEFAULT_REQUEST_TIMEOUT,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
fn invalid_http_settings(message: &str, field: &'static str) -> ksp_core_lib::Result<HttpClientSettings> {
|
||||
ksp_logging_lib::warn!(target: crate::TRACING_TARGET, field = field, "rejected invalid off-chain HTTP client settings");
|
||||
return std::result::Result::Err(ksp_core_lib::Error::new(crate::ERROR_CODE_HTTP_SETTINGS_INVALID, message).with_context("field", field));
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[path = "../unit_tests/http_settings.rs"]
|
||||
mod tests;
|
||||
@@ -1,5 +1,5 @@
|
||||
// file: crates/ksp-offchain-transport-lib/src/lib.rs
|
||||
// version: 3
|
||||
// version: 4
|
||||
|
||||
#![warn(missing_docs)]
|
||||
#![deny(unreachable_pub)]
|
||||
@@ -7,19 +7,48 @@
|
||||
|
||||
//! KSP-owned off-chain transport foundation.
|
||||
//!
|
||||
//! `0.2.11-pre.002` opens the first deliberately narrow capability family: `market_price`, currently limited to exact SOL/USD values, provider-neutral
|
||||
//! observations, provider descriptors, common provider settings and generic availability/rate-limit metadata. The crate itself is deliberately broader than
|
||||
//! market prices: future off-chain capabilities such as amount-specific swap quotes belong to separate capability families instead of extending the
|
||||
//! `market_price_*` modules. Shared transport/runtime concerns stay crate-wide. No provider wire type, HTTP client, provider SDK, Config dependency or active
|
||||
//! rate limiter is introduced by this tranche.
|
||||
//! `0.2.11-pre.003` keeps the first capability family deliberately narrow (`market_price`, SOL/USD only) while materializing crate-wide hardened HTTP REST
|
||||
//! primitives and non-blocking local admission/cooldown machinery. These `http_*` internals are shared transport mechanics for future off-chain capability
|
||||
//! families such as `swap_quote_*`; they are intentionally not re-exported as a generic consumer HTTP client. Provider wire DTOs, provider adapters, Config
|
||||
//! integration and refresh orchestration remain outside this tranche.
|
||||
|
||||
mod constants;
|
||||
mod error;
|
||||
#[allow(dead_code)] // Staged in pre.003 and consumed by provider adapters starting in pre.004.
|
||||
mod http_admission;
|
||||
#[allow(dead_code)] // Staged in pre.003 and consumed by provider adapters starting in pre.004.
|
||||
mod http_client;
|
||||
#[allow(dead_code)] // Staged in pre.003 and consumed by provider adapters starting in pre.004.
|
||||
mod http_settings;
|
||||
mod market_price_decimal;
|
||||
mod market_price_observation;
|
||||
mod market_price_provider;
|
||||
mod market_price_settings;
|
||||
|
||||
/// Stable error code for HTTP access denial.
|
||||
pub use self::error::ERROR_CODE_HTTP_ACCESS_DENIED;
|
||||
/// Stable error code for hardened HTTP client initialization failure.
|
||||
pub use self::error::ERROR_CODE_HTTP_CLIENT_BUILD_FAILED;
|
||||
/// Stable error code for an off-chain HTTP connection failure.
|
||||
pub use self::error::ERROR_CODE_HTTP_CONNECTION_FAILED;
|
||||
/// Stable error code for a syntactically invalid JSON response.
|
||||
pub use self::error::ERROR_CODE_HTTP_INVALID_JSON;
|
||||
/// Stable error code for an invalid local HTTP rate-limit policy.
|
||||
pub use self::error::ERROR_CODE_HTTP_RATE_LIMIT_INVALID;
|
||||
/// Stable error code for HTTP 429 rate limiting.
|
||||
pub use self::error::ERROR_CODE_HTTP_RATE_LIMITED;
|
||||
/// Stable error code for a generic unsuccessful HTTP request.
|
||||
pub use self::error::ERROR_CODE_HTTP_REQUEST_FAILED;
|
||||
/// Stable error code for an invalid crate-owned HTTP request definition.
|
||||
pub use self::error::ERROR_CODE_HTTP_REQUEST_INVALID;
|
||||
/// Stable error code for an oversized HTTP response body.
|
||||
pub use self::error::ERROR_CODE_HTTP_RESPONSE_TOO_LARGE;
|
||||
/// Stable error code for invalid crate-wide HTTP runtime settings.
|
||||
pub use self::error::ERROR_CODE_HTTP_SETTINGS_INVALID;
|
||||
/// Stable error code for temporary HTTP provider failures.
|
||||
pub use self::error::ERROR_CODE_HTTP_TEMPORARY_FAILURE;
|
||||
/// Stable error code for end-to-end HTTP timeout.
|
||||
pub use self::error::ERROR_CODE_HTTP_TIMEOUT;
|
||||
/// Stable error code for an invalid exact decimal price.
|
||||
pub use self::error::ERROR_CODE_MARKET_PRICE_DECIMAL_INVALID;
|
||||
/// Stable error code for an invalid provider-neutral observation.
|
||||
@@ -79,3 +108,19 @@ pub use self::market_price_settings::MarketPriceProviderCommonSettings;
|
||||
|
||||
/// Owning tracing target for events emitted by Off-chain Transport.
|
||||
pub(crate) use self::constants::TRACING_TARGET;
|
||||
/// Maximum provider-directed cooldown accepted from a server `Retry-After` value.
|
||||
pub(crate) use self::http_admission::HTTP_MAX_RETRY_AFTER;
|
||||
/// Crate-internal non-blocking request-admission controller.
|
||||
pub(crate) use self::http_admission::HttpAdmissionController;
|
||||
/// Crate-internal result of one immediate request-admission attempt.
|
||||
pub(crate) use self::http_admission::HttpAdmissionDecision;
|
||||
/// Crate-internal provider-neutral local request-admission policy.
|
||||
pub(crate) use self::http_admission::HttpAdmissionPolicy;
|
||||
/// Crate-internal fixed-origin GET request with redacted diagnostics.
|
||||
pub(crate) use self::http_client::HttpGetRequest;
|
||||
/// Crate-internal bounded syntactically valid JSON response document.
|
||||
pub(crate) use self::http_client::HttpJsonDocument;
|
||||
/// Crate-internal hardened REST client shared by capability adapters.
|
||||
pub(crate) use self::http_client::HttpRestClient;
|
||||
/// Crate-internal bounded HTTP runtime settings.
|
||||
pub(crate) use self::http_settings::HttpClientSettings;
|
||||
|
||||
@@ -1,20 +1,21 @@
|
||||
// file: crates/ksp-offchain-transport-lib/tests/dependency_boundary.rs
|
||||
// version: 3
|
||||
// version: 4
|
||||
|
||||
#![warn(missing_docs)]
|
||||
#![deny(unreachable_pub)]
|
||||
#![forbid(unsafe_code)]
|
||||
|
||||
//! Dependency, observability and module-taxonomy canaries for the Off-chain Transport foundation.
|
||||
//! Dependency, observability, hardened HTTP and module-taxonomy canaries for Off-chain Transport.
|
||||
|
||||
#[test]
|
||||
fn pre_002_manifest_keeps_foundation_provider_and_config_independent() {
|
||||
fn pre_003_manifest_adds_only_shared_http_runtime_dependencies() {
|
||||
let manifest = include_str!("../Cargo.toml");
|
||||
assert!(manifest.contains("ksp-core-lib"));
|
||||
assert!(manifest.contains("ksp-logging-lib"));
|
||||
assert!(manifest.contains("reqwest = { workspace = true, features = [\"rustls\"] }"));
|
||||
assert!(manifest.contains("serde"));
|
||||
assert!(manifest.contains("serde_json.workspace = true"));
|
||||
assert!(!manifest.contains("ksp-config-lib"));
|
||||
assert!(!manifest.contains("reqwest"));
|
||||
assert!(!manifest.contains("coingecko"));
|
||||
assert!(!manifest.contains("coinmarketcap"));
|
||||
assert!(!manifest.contains("jupiter"));
|
||||
@@ -22,15 +23,30 @@ fn pre_002_manifest_keeps_foundation_provider_and_config_independent() {
|
||||
assert!(!manifest.contains("dexscreener"));
|
||||
assert!(!manifest.lines().any(|line| return line.trim_start().starts_with("tracing =")));
|
||||
let crate_root = include_str!("../src/lib.rs");
|
||||
assert!(crate_root.contains("mod constants;"));
|
||||
assert!(crate_root.contains("mod http_admission;"));
|
||||
assert!(crate_root.contains("mod http_client;"));
|
||||
assert!(crate_root.contains("mod http_settings;"));
|
||||
assert!(crate_root.contains("mod market_price_decimal;"));
|
||||
assert!(crate_root.contains("mod market_price_observation;"));
|
||||
assert!(crate_root.contains("mod market_price_provider;"));
|
||||
assert!(crate_root.contains("mod market_price_settings;"));
|
||||
assert!(!crate_root.contains("mod decimal;"));
|
||||
assert!(!crate_root.contains("mod observation;"));
|
||||
assert!(!crate_root.contains("mod provider;"));
|
||||
assert!(!crate_root.contains("mod settings;"));
|
||||
assert!(!crate_root.contains("pub use self::http_client::HttpRestClient"));
|
||||
let constants = include_str!("../src/constants.rs");
|
||||
assert!(constants.contains("pub(crate) const TRACING_TARGET: &str = \"ksp-offchain-transport-lib\";"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn pre_003_http_client_hardening_is_explicit_and_provider_neutral() {
|
||||
let client = include_str!("../src/http_client.rs");
|
||||
assert!(client.contains(".redirect(reqwest::redirect::Policy::none())"));
|
||||
assert!(client.contains(".referer(false)"));
|
||||
assert!(client.contains(".retry(reqwest::retry::never())"));
|
||||
assert!(client.contains(".no_proxy()"));
|
||||
assert!(client.contains("error.without_url()"));
|
||||
assert!(client.contains("response.chunk().await"));
|
||||
assert!(!client.contains("coingecko"));
|
||||
assert!(!client.contains("coinmarketcap"));
|
||||
assert!(!client.contains("jupiter"));
|
||||
assert!(!client.contains("birdeye"));
|
||||
assert!(!client.contains("dexscreener"));
|
||||
}
|
||||
|
||||
@@ -1,11 +1,11 @@
|
||||
// file: crates/ksp-offchain-transport-lib/tests/public_api.rs
|
||||
// version: 3
|
||||
// version: 4
|
||||
|
||||
#![warn(missing_docs)]
|
||||
#![deny(unreachable_pub)]
|
||||
#![forbid(unsafe_code)]
|
||||
|
||||
//! Public API canaries for the first Off-chain Transport market-price capability family.
|
||||
//! Public API canaries for the market-price foundation and stable Off-chain Transport error-code surface.
|
||||
|
||||
#[test]
|
||||
fn public_pre_002_market_price_foundation_is_available_from_crate_root() -> ksp_core_lib::Result<()> {
|
||||
@@ -68,6 +68,18 @@ fn public_pre_002_market_price_foundation_is_available_from_crate_root() -> ksp_
|
||||
#[test]
|
||||
fn offchain_error_codes_use_owned_domain() {
|
||||
let codes = [
|
||||
ksp_offchain_transport_lib::ERROR_CODE_HTTP_ACCESS_DENIED,
|
||||
ksp_offchain_transport_lib::ERROR_CODE_HTTP_CLIENT_BUILD_FAILED,
|
||||
ksp_offchain_transport_lib::ERROR_CODE_HTTP_CONNECTION_FAILED,
|
||||
ksp_offchain_transport_lib::ERROR_CODE_HTTP_INVALID_JSON,
|
||||
ksp_offchain_transport_lib::ERROR_CODE_HTTP_RATE_LIMIT_INVALID,
|
||||
ksp_offchain_transport_lib::ERROR_CODE_HTTP_RATE_LIMITED,
|
||||
ksp_offchain_transport_lib::ERROR_CODE_HTTP_REQUEST_FAILED,
|
||||
ksp_offchain_transport_lib::ERROR_CODE_HTTP_REQUEST_INVALID,
|
||||
ksp_offchain_transport_lib::ERROR_CODE_HTTP_RESPONSE_TOO_LARGE,
|
||||
ksp_offchain_transport_lib::ERROR_CODE_HTTP_SETTINGS_INVALID,
|
||||
ksp_offchain_transport_lib::ERROR_CODE_HTTP_TEMPORARY_FAILURE,
|
||||
ksp_offchain_transport_lib::ERROR_CODE_HTTP_TIMEOUT,
|
||||
ksp_offchain_transport_lib::ERROR_CODE_MARKET_PRICE_DECIMAL_INVALID,
|
||||
ksp_offchain_transport_lib::ERROR_CODE_MARKET_PRICE_PROVIDER_DESCRIPTOR_INVALID,
|
||||
ksp_offchain_transport_lib::ERROR_CODE_MARKET_PRICE_PROVIDER_ID_INVALID,
|
||||
|
||||
@@ -0,0 +1,73 @@
|
||||
// file: crates/ksp-offchain-transport-lib/unit_tests/http_admission.rs
|
||||
// version: 1
|
||||
|
||||
#[test]
|
||||
fn fixed_admission_smooths_undocumented_burst_and_refills_deterministically() -> ksp_core_lib::Result<()> {
|
||||
let policy = match crate::HttpAdmissionPolicy::fixed(2, std::time::Duration::from_secs(1), std::option::Option::None) {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(error) => return std::result::Result::Err(error),
|
||||
};
|
||||
let controller = match crate::HttpAdmissionController::new(policy, std::option::Option::None) {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(error) => return std::result::Result::Err(error),
|
||||
};
|
||||
let start = std::time::Instant::now();
|
||||
assert!(matches!(controller.try_admit_at(start), crate::HttpAdmissionDecision::Ready));
|
||||
let deferred = controller.try_admit_at(start);
|
||||
let delay = match deferred {
|
||||
crate::HttpAdmissionDecision::Deferred(value) => value,
|
||||
crate::HttpAdmissionDecision::Ready => {
|
||||
return std::result::Result::Err(ksp_core_lib::Error::new(crate::ERROR_CODE_HTTP_RATE_LIMIT_INVALID, "test expected local deferral"));
|
||||
},
|
||||
};
|
||||
assert!(delay > std::time::Duration::ZERO);
|
||||
let half_second = match start.checked_add(std::time::Duration::from_millis(500)) {
|
||||
std::option::Option::Some(value) => value,
|
||||
std::option::Option::None => {
|
||||
return std::result::Result::Err(ksp_core_lib::Error::new(crate::ERROR_CODE_HTTP_RATE_LIMIT_INVALID, "test instant overflow"));
|
||||
},
|
||||
};
|
||||
assert!(matches!(controller.try_admit_at(half_second), crate::HttpAdmissionDecision::Ready));
|
||||
return std::result::Result::Ok(());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn documented_burst_is_consumed_atomically_before_refill() -> ksp_core_lib::Result<()> {
|
||||
let policy = match crate::HttpAdmissionPolicy::fixed(1, std::time::Duration::from_secs(1), std::option::Option::Some(2)) {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(error) => return std::result::Result::Err(error),
|
||||
};
|
||||
let controller = match crate::HttpAdmissionController::new(policy, std::option::Option::None) {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(error) => return std::result::Result::Err(error),
|
||||
};
|
||||
let start = std::time::Instant::now();
|
||||
assert!(matches!(controller.try_admit_at(start), crate::HttpAdmissionDecision::Ready));
|
||||
assert!(matches!(controller.try_admit_at(start), crate::HttpAdmissionDecision::Ready));
|
||||
assert!(matches!(controller.try_admit_at(start), crate::HttpAdmissionDecision::Deferred(_)));
|
||||
return std::result::Result::Ok(());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn provider_retry_after_extends_but_cannot_pathologically_lock_cooldown() -> ksp_core_lib::Result<()> {
|
||||
let controller =
|
||||
match crate::HttpAdmissionController::new(crate::HttpAdmissionPolicy::Dynamic, std::option::Option::Some(std::time::Duration::from_secs(2))) {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(error) => return std::result::Result::Err(error),
|
||||
};
|
||||
assert_eq!(controller.policy(), crate::HttpAdmissionPolicy::Dynamic);
|
||||
let applied = controller.record_rate_limited(std::option::Option::Some(std::time::Duration::from_secs(99_999)));
|
||||
assert_eq!(applied, crate::HTTP_MAX_RETRY_AFTER);
|
||||
let remaining = controller.cooldown_remaining();
|
||||
assert!(remaining.is_some());
|
||||
assert!(matches!(controller.try_admit(), crate::HttpAdmissionDecision::Deferred(_)));
|
||||
return std::result::Result::Ok(());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn admission_policy_rejects_zero_values_and_invalid_fallback() {
|
||||
assert!(crate::HttpAdmissionPolicy::fixed(0, std::time::Duration::from_secs(1), std::option::Option::None).is_err());
|
||||
assert!(crate::HttpAdmissionPolicy::fixed(1, std::time::Duration::ZERO, std::option::Option::None).is_err());
|
||||
assert!(crate::HttpAdmissionPolicy::fixed(1, std::time::Duration::from_secs(1), std::option::Option::Some(0)).is_err());
|
||||
assert!(crate::HttpAdmissionController::new(crate::HttpAdmissionPolicy::Unlimited, std::option::Option::Some(std::time::Duration::ZERO)).is_err());
|
||||
}
|
||||
220
crates/ksp-offchain-transport-lib/unit_tests/http_client.rs
Normal file
220
crates/ksp-offchain-transport-lib/unit_tests/http_client.rs
Normal file
@@ -0,0 +1,220 @@
|
||||
// file: crates/ksp-offchain-transport-lib/unit_tests/http_client.rs
|
||||
// version: 1
|
||||
|
||||
#[tokio::test]
|
||||
async fn rest_client_accepts_bounded_json_and_never_exposes_request_debug() -> ksp_core_lib::Result<()> {
|
||||
let server_result =
|
||||
spawn_single_response("HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: 13\r\nConnection: close\r\n\r\n{\"price\":123}").await;
|
||||
let server = match server_result {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(error) => return std::result::Result::Err(error),
|
||||
};
|
||||
let mut request = match crate::HttpGetRequest::new_test_http(server.url.as_str()) {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(error) => return std::result::Result::Err(error),
|
||||
};
|
||||
request.append_query_pair("symbol", "SOL/USD");
|
||||
let secret = "sensitive-canary-value";
|
||||
if let std::result::Result::Err(error) = request.insert_sensitive_header("x-api-key", secret) {
|
||||
return std::result::Result::Err(error);
|
||||
}
|
||||
let debug = format!("{request:?}");
|
||||
assert!(!debug.contains(secret));
|
||||
assert!(!debug.contains(server.url.as_str()));
|
||||
let client = match crate::HttpRestClient::new(crate::HttpClientSettings::default()) {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(error) => return std::result::Result::Err(error),
|
||||
};
|
||||
let document = match client.get_json("test-provider", "sol_usd", request).await {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(error) => return std::result::Result::Err(error),
|
||||
};
|
||||
assert_eq!(document.as_bytes(), br#"{"price":123}"#);
|
||||
if let std::result::Result::Err(error) = finish_server(server.join).await {
|
||||
return std::result::Result::Err(error);
|
||||
}
|
||||
return std::result::Result::Ok(());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn rest_client_rejects_redirects_instead_of_following_them() -> ksp_core_lib::Result<()> {
|
||||
let server_result =
|
||||
spawn_single_response("HTTP/1.1 302 Found\r\nLocation: https://example.invalid/secret\r\nContent-Length: 0\r\nConnection: close\r\n\r\n").await;
|
||||
let server = match server_result {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(error) => return std::result::Result::Err(error),
|
||||
};
|
||||
let request = match crate::HttpGetRequest::new_test_http(server.url.as_str()) {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(error) => return std::result::Result::Err(error),
|
||||
};
|
||||
let client = match crate::HttpRestClient::new(crate::HttpClientSettings::default()) {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(error) => return std::result::Result::Err(error),
|
||||
};
|
||||
let error = match client.get_json("test-provider", "redirect", request).await {
|
||||
std::result::Result::Ok(_) => {
|
||||
return std::result::Result::Err(ksp_core_lib::Error::new(crate::ERROR_CODE_HTTP_REQUEST_FAILED, "test expected redirect rejection"));
|
||||
},
|
||||
std::result::Result::Err(error) => error,
|
||||
};
|
||||
assert_eq!(error.code(), crate::ERROR_CODE_HTTP_REQUEST_FAILED);
|
||||
if let std::result::Result::Err(error) = finish_server(server.join).await {
|
||||
return std::result::Result::Err(error);
|
||||
}
|
||||
return std::result::Result::Ok(());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn rest_client_bounds_chunked_body_before_json_deserialization() -> ksp_core_lib::Result<()> {
|
||||
let server_result = spawn_single_response(concat!(
|
||||
"HTTP/1.1 200 OK\r\n",
|
||||
"Content-Type: application/json\r\n",
|
||||
"Transfer-Encoding: chunked\r\n",
|
||||
"Connection: close\r\n\r\n",
|
||||
"8\r\n{\"aaaa\":\r\n",
|
||||
"8\r\n\"bbbbbb\"\r\n",
|
||||
"1\r\n}\r\n0\r\n\r\n",
|
||||
))
|
||||
.await;
|
||||
let server = match server_result {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(error) => return std::result::Result::Err(error),
|
||||
};
|
||||
let settings = match crate::HttpClientSettings::new(std::time::Duration::from_secs(1), std::time::Duration::from_secs(2), 8) {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(error) => return std::result::Result::Err(error),
|
||||
};
|
||||
let request = match crate::HttpGetRequest::new_test_http(server.url.as_str()) {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(error) => return std::result::Result::Err(error),
|
||||
};
|
||||
let client = match crate::HttpRestClient::new(settings) {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(error) => return std::result::Result::Err(error),
|
||||
};
|
||||
let error = match client.get_json("test-provider", "bounded", request).await {
|
||||
std::result::Result::Ok(_) => {
|
||||
return std::result::Result::Err(ksp_core_lib::Error::new(crate::ERROR_CODE_HTTP_RESPONSE_TOO_LARGE, "test expected body bound"));
|
||||
},
|
||||
std::result::Result::Err(error) => error,
|
||||
};
|
||||
assert_eq!(error.code(), crate::ERROR_CODE_HTTP_RESPONSE_TOO_LARGE);
|
||||
if let std::result::Result::Err(error) = finish_server(server.join).await {
|
||||
return std::result::Result::Err(error);
|
||||
}
|
||||
return std::result::Result::Ok(());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn rest_client_classifies_429_retry_after_and_never_copies_remote_body() -> ksp_core_lib::Result<()> {
|
||||
let remote_canary = "REMOTE_SECRET_CANARY";
|
||||
let response =
|
||||
format!("HTTP/1.1 429 Too Many Requests\r\nRetry-After: 7\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}", remote_canary.len(), remote_canary);
|
||||
let server_result = spawn_single_response(response.as_str()).await;
|
||||
let server = match server_result {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(error) => return std::result::Result::Err(error),
|
||||
};
|
||||
let request = match crate::HttpGetRequest::new_test_http(server.url.as_str()) {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(error) => return std::result::Result::Err(error),
|
||||
};
|
||||
let client = match crate::HttpRestClient::new(crate::HttpClientSettings::default()) {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(error) => return std::result::Result::Err(error),
|
||||
};
|
||||
let error = match client.get_json("test-provider", "rate_limited", request).await {
|
||||
std::result::Result::Ok(_) => return std::result::Result::Err(ksp_core_lib::Error::new(crate::ERROR_CODE_HTTP_RATE_LIMITED, "test expected 429")),
|
||||
std::result::Result::Err(error) => error,
|
||||
};
|
||||
assert_eq!(error.code(), crate::ERROR_CODE_HTTP_RATE_LIMITED);
|
||||
assert!(!format!("{error:?}").contains(remote_canary));
|
||||
assert!(!error.to_string().contains(remote_canary));
|
||||
if let std::result::Result::Err(error) = finish_server(server.join).await {
|
||||
return std::result::Result::Err(error);
|
||||
}
|
||||
return std::result::Result::Ok(());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn rest_client_rejects_invalid_json_after_success_status() -> ksp_core_lib::Result<()> {
|
||||
let server_result = spawn_single_response("HTTP/1.1 200 OK\r\nContent-Length: 8\r\nConnection: close\r\n\r\nnot-json").await;
|
||||
let server = match server_result {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(error) => return std::result::Result::Err(error),
|
||||
};
|
||||
let request = match crate::HttpGetRequest::new_test_http(server.url.as_str()) {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(error) => return std::result::Result::Err(error),
|
||||
};
|
||||
let client = match crate::HttpRestClient::new(crate::HttpClientSettings::default()) {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(error) => return std::result::Result::Err(error),
|
||||
};
|
||||
let error = match client.get_json("test-provider", "invalid_json", request).await {
|
||||
std::result::Result::Ok(_) => {
|
||||
return std::result::Result::Err(ksp_core_lib::Error::new(crate::ERROR_CODE_HTTP_INVALID_JSON, "test expected invalid JSON"));
|
||||
},
|
||||
std::result::Result::Err(error) => error,
|
||||
};
|
||||
assert_eq!(error.code(), crate::ERROR_CODE_HTTP_INVALID_JSON);
|
||||
if let std::result::Result::Err(error) = finish_server(server.join).await {
|
||||
return std::result::Result::Err(error);
|
||||
}
|
||||
return std::result::Result::Ok(());
|
||||
}
|
||||
|
||||
struct TestServer {
|
||||
join: tokio::task::JoinHandle<std::io::Result<()>>,
|
||||
url: std::string::String,
|
||||
}
|
||||
|
||||
async fn spawn_single_response(response: &str) -> ksp_core_lib::Result<TestServer> {
|
||||
let listener_result = tokio::net::TcpListener::bind("127.0.0.1:0").await;
|
||||
let listener = match listener_result {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(error) => return std::result::Result::Err(io_error(error)),
|
||||
};
|
||||
let address = match listener.local_addr() {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(error) => return std::result::Result::Err(io_error(error)),
|
||||
};
|
||||
let response = response.as_bytes().to_vec();
|
||||
let join = tokio::spawn(async move {
|
||||
let accept_result = listener.accept().await;
|
||||
let (mut socket, _) = match accept_result {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(error) => return std::result::Result::Err(error),
|
||||
};
|
||||
let mut request = [0_u8; 4096];
|
||||
if let std::result::Result::Err(error) = tokio::io::AsyncReadExt::read(&mut socket, &mut request).await {
|
||||
return std::result::Result::Err(error);
|
||||
}
|
||||
if let std::result::Result::Err(error) = tokio::io::AsyncWriteExt::write_all(&mut socket, response.as_slice()).await {
|
||||
return std::result::Result::Err(error);
|
||||
}
|
||||
if let std::result::Result::Err(error) = tokio::io::AsyncWriteExt::shutdown(&mut socket).await {
|
||||
return std::result::Result::Err(error);
|
||||
}
|
||||
return std::io::Result::Ok(());
|
||||
});
|
||||
return std::result::Result::Ok(TestServer { join, url: format!("http://{address}/price") });
|
||||
}
|
||||
|
||||
async fn finish_server(join: tokio::task::JoinHandle<std::io::Result<()>>) -> ksp_core_lib::Result<()> {
|
||||
let joined = join.await;
|
||||
return match joined {
|
||||
std::result::Result::Ok(std::result::Result::Ok(())) => std::result::Result::Ok(()),
|
||||
std::result::Result::Ok(std::result::Result::Err(error)) => std::result::Result::Err(io_error(error)),
|
||||
std::result::Result::Err(error) => std::result::Result::Err(join_error(error)),
|
||||
};
|
||||
}
|
||||
|
||||
fn io_error(error: std::io::Error) -> ksp_core_lib::Error {
|
||||
return ksp_core_lib::Error::new(crate::ERROR_CODE_HTTP_REQUEST_FAILED, "test HTTP server failed").with_source(error);
|
||||
}
|
||||
|
||||
fn join_error(error: tokio::task::JoinError) -> ksp_core_lib::Error {
|
||||
return ksp_core_lib::Error::new(crate::ERROR_CODE_HTTP_REQUEST_FAILED, "test HTTP server task failed").with_source(error);
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
// file: crates/ksp-offchain-transport-lib/unit_tests/http_settings.rs
|
||||
// version: 1
|
||||
|
||||
#[test]
|
||||
fn http_settings_defaults_are_bounded_and_ordered() {
|
||||
let settings = crate::HttpClientSettings::default();
|
||||
assert!(!settings.connect_timeout().is_zero());
|
||||
assert!(settings.connect_timeout() <= settings.request_timeout());
|
||||
assert!(settings.max_response_body_bytes() > 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn http_settings_reject_zero_reversed_and_pathological_bounds() {
|
||||
assert!(crate::HttpClientSettings::new(std::time::Duration::ZERO, std::time::Duration::from_secs(1), 1024).is_err());
|
||||
assert!(crate::HttpClientSettings::new(std::time::Duration::from_secs(2), std::time::Duration::from_secs(1), 1024).is_err());
|
||||
assert!(crate::HttpClientSettings::new(std::time::Duration::from_secs(1), std::time::Duration::from_secs(2), 0).is_err());
|
||||
assert!(crate::HttpClientSettings::new(std::time::Duration::from_secs(31), std::time::Duration::from_secs(31), 1024).is_err());
|
||||
assert!(crate::HttpClientSettings::new(std::time::Duration::from_secs(1), std::time::Duration::from_secs(121), 1024).is_err());
|
||||
assert!(crate::HttpClientSettings::new(std::time::Duration::from_secs(1), std::time::Duration::from_secs(2), 4_194_305).is_err());
|
||||
}
|
||||
@@ -1,5 +1,5 @@
|
||||
// file: crates/ksp-onchain-transport-lib/unit_tests/grpc_stream.rs
|
||||
// version: 2
|
||||
// version: 3
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
enum FixtureMode {
|
||||
@@ -27,7 +27,6 @@ struct FixtureGeyser {
|
||||
#[tonic::async_trait]
|
||||
impl yellowstone_grpc_proto::geyser::geyser_server::Geyser for FixtureGeyser {
|
||||
type SubscribeStream = super::MpscStream<std::result::Result<yellowstone_grpc_proto::geyser::SubscribeUpdate, tonic::Status>>;
|
||||
type SubscribeDeshredStream = futures_util::stream::Empty<std::result::Result<yellowstone_grpc_proto::geyser::SubscribeUpdateDeshred, tonic::Status>>;
|
||||
|
||||
async fn subscribe(
|
||||
&self,
|
||||
@@ -163,6 +162,8 @@ impl yellowstone_grpc_proto::geyser::geyser_server::Geyser for FixtureGeyser {
|
||||
return std::result::Result::Ok(tonic::Response::new(super::MpscStream::new(outbound_rx)));
|
||||
}
|
||||
|
||||
type SubscribeDeshredStream = futures_util::stream::Empty<std::result::Result<yellowstone_grpc_proto::geyser::SubscribeUpdateDeshred, tonic::Status>>;
|
||||
|
||||
async fn subscribe_deshred(
|
||||
&self,
|
||||
_request: tonic::Request<tonic::Streaming<yellowstone_grpc_proto::geyser::SubscribeDeshredRequest>>,
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
// file: crates/ksp-onchain-transport-lib/unit_tests/grpc_unary.rs
|
||||
// version: 2
|
||||
// version: 3
|
||||
|
||||
#[derive(Clone, Default)]
|
||||
struct FixtureGeyser;
|
||||
@@ -10,11 +10,6 @@ impl yellowstone_grpc_proto::geyser::geyser_server::Geyser for FixtureGeyser {
|
||||
type SubscribeStream = std::pin::Pin<
|
||||
std::boxed::Box<dyn futures_util::Stream<Item = std::result::Result<yellowstone_grpc_proto::geyser::SubscribeUpdate, tonic::Status>> + Send + 'static>,
|
||||
>;
|
||||
type SubscribeDeshredStream = std::pin::Pin<
|
||||
std::boxed::Box<
|
||||
dyn futures_util::Stream<Item = std::result::Result<yellowstone_grpc_proto::geyser::SubscribeUpdateDeshred, tonic::Status>> + Send + 'static,
|
||||
>,
|
||||
>;
|
||||
|
||||
async fn subscribe(
|
||||
&self,
|
||||
@@ -23,6 +18,12 @@ impl yellowstone_grpc_proto::geyser::geyser_server::Geyser for FixtureGeyser {
|
||||
return std::result::Result::Err(tonic::Status::unimplemented("streaming is outside the pre.003 fixture"));
|
||||
}
|
||||
|
||||
type SubscribeDeshredStream = std::pin::Pin<
|
||||
std::boxed::Box<
|
||||
dyn futures_util::Stream<Item = std::result::Result<yellowstone_grpc_proto::geyser::SubscribeUpdateDeshred, tonic::Status>> + Send + 'static,
|
||||
>,
|
||||
>;
|
||||
|
||||
async fn subscribe_deshred(
|
||||
&self,
|
||||
_request: tonic::Request<tonic::Streaming<yellowstone_grpc_proto::geyser::SubscribeDeshredRequest>>,
|
||||
|
||||
300
deltas/0.2.11/pre.003.md
Normal file
300
deltas/0.2.11/pre.003.md
Normal file
@@ -0,0 +1,300 @@
|
||||
<!-- file: deltas/0.2.11/pre.003.md -->
|
||||
<!-- version: 1 -->
|
||||
|
||||
# Delta `0.2.11-pre.003` — HTTP REST commun et rate limiting
|
||||
|
||||
## 1. Base requise
|
||||
|
||||
Cette tranche s'applique exclusivement après :
|
||||
|
||||
```text
|
||||
v0.2.10
|
||||
+ 0.2.11-pre.001
|
||||
+ 0.2.11-pre.001-fix.001
|
||||
+ 0.2.11-pre.002
|
||||
+ 0.2.11-pre.002-fix.001
|
||||
+ 0.2.11-pre.002-fix.002
|
||||
+ 0.2.11-pre.002-fix.003
|
||||
```
|
||||
|
||||
La version Cargo attendue à l'entrée est :
|
||||
|
||||
```text
|
||||
0.2.11-pre.2.fix.3
|
||||
```
|
||||
|
||||
La version Cargo de sortie est :
|
||||
|
||||
```text
|
||||
0.2.11-pre.3
|
||||
```
|
||||
|
||||
## 2. Gate d'entrée acquis
|
||||
|
||||
L'opérateur a validé `0.2.11-pre.002-fix.003` le `2026-08-25` avec :
|
||||
|
||||
```text
|
||||
cargo fmt --all exécuté
|
||||
python3 scripts/audit_rust_workspace_rules.py clean
|
||||
python3 scripts/audit_markdown_tables.py clean, 116 tables / 105 files
|
||||
cargo check --workspace PASS
|
||||
cargo clippy --workspace --all-targets PASS
|
||||
cargo test -p ksp-offchain-transport-lib PASS, 10 unit + 1 boundary + 2 public API
|
||||
```
|
||||
|
||||
`cargo test --workspace` n'a pas été fourni pour cet état exact et n'est pas déclaré PASS ici.
|
||||
|
||||
## 3. Objectif
|
||||
|
||||
Matérialiser les primitives HTTP REST communes de `ksp-offchain-transport-lib` et le rate limiting local sans avancer aucun provider.
|
||||
|
||||
La tranche doit préparer directement `pre.004+` tout en conservant les frontières :
|
||||
|
||||
```text
|
||||
http_* = mécanique transport crate-wide et crate-private
|
||||
market_price_* = première capacité métier, SOL/USD uniquement
|
||||
aucun client HTTP générique exporté aux consumers
|
||||
aucun SDK provider
|
||||
aucun Config -> Off-chain ajouté avant pre.009
|
||||
aucun refresh registry/service avant pre.007/pre.008
|
||||
```
|
||||
|
||||
## 4. HTTP REST commun
|
||||
|
||||
`http_client.rs` matérialise un `reqwest::Client` réutilisable par les adapters internes.
|
||||
|
||||
La construction impose explicitement :
|
||||
|
||||
```text
|
||||
rustls
|
||||
redirects désactivés
|
||||
Referer automatique désactivé
|
||||
proxy système désactivé
|
||||
retries implicites reqwest désactivés
|
||||
User-Agent KSP explicite
|
||||
connect timeout borné
|
||||
request timeout borné
|
||||
```
|
||||
|
||||
`reqwest` reste une dépendance directe de la crate et aucun SDK provider n'est ajouté.
|
||||
|
||||
La requête GET interne :
|
||||
|
||||
```text
|
||||
accepte uniquement HTTPS pour les adapters runtime
|
||||
refuse credentials dans l'URL
|
||||
encode les query pairs via Url
|
||||
permet des headers sensibles marqués sensitive
|
||||
redacte URL et headers dans Debug
|
||||
```
|
||||
|
||||
Un constructeur HTTP non HTTPS existe uniquement sous `cfg(test)` pour les serveurs loopback déterministes.
|
||||
|
||||
## 5. Bornes de réponse et JSON
|
||||
|
||||
Les valeurs initiales sont :
|
||||
|
||||
```text
|
||||
connect timeout défaut = 5 s
|
||||
connect timeout hard max = 30 s
|
||||
request timeout défaut = 10 s
|
||||
request timeout hard max = 120 s
|
||||
body défaut = 1 MiB
|
||||
body hard max = 4 MiB
|
||||
```
|
||||
|
||||
Le body est borné pendant sa lecture par chunks. La présence ou l'absence de `Content-Length` ne permet donc pas de contourner la limite.
|
||||
|
||||
Après lecture, la syntaxe JSON est validée avec `serde::de::IgnoredAny`. Le document brut borné reste disponible pour le futur adapter typed, ce qui évite de faire passer un nombre provider par une représentation générique `f64` avant le parsing `MarketPriceDecimal`.
|
||||
|
||||
## 6. Classification HTTP
|
||||
|
||||
Les codes crate-wide ajoutés sous le domaine `offchain_transport` distinguent :
|
||||
|
||||
```text
|
||||
http_settings_invalid
|
||||
http_client_build_failed
|
||||
http_request_invalid
|
||||
http_connection_failed
|
||||
http_timeout
|
||||
http_request_failed
|
||||
http_access_denied
|
||||
http_rate_limited
|
||||
http_temporary_failure
|
||||
http_response_too_large
|
||||
http_invalid_json
|
||||
http_rate_limit_invalid
|
||||
```
|
||||
|
||||
La classification de statut est :
|
||||
|
||||
```text
|
||||
401/403 -> access denied
|
||||
429 -> rate limited
|
||||
408 et 5xx -> temporary failure
|
||||
autre non-2xx -> request failed
|
||||
2xx -> body borné puis JSON syntaxiquement validé
|
||||
```
|
||||
|
||||
Les erreurs `reqwest` sont converties après `without_url()`. Aucun body remote n'est recopié dans `KspError`.
|
||||
|
||||
`Retry-After` est actuellement exploité sous sa forme `delta-seconds`, bornée défensivement à une heure. Une forme HTTP-date non parseable reste ignorée plutôt que devinée.
|
||||
|
||||
## 7. Admission et cooldown
|
||||
|
||||
`http_admission.rs` fournit un limiter non bloquant.
|
||||
|
||||
Les policies internes sont :
|
||||
|
||||
```text
|
||||
Fixed
|
||||
Dynamic
|
||||
Unlimited
|
||||
```
|
||||
|
||||
Une policy fixe utilise un token bucket lissé :
|
||||
|
||||
```text
|
||||
requests/window
|
||||
burst explicite uniquement lorsqu'il est documenté
|
||||
burst absent -> capacité locale conservatrice de 1
|
||||
```
|
||||
|
||||
Une policy dynamique n'invente aucune cadence locale. Elle applique seulement les cooldowns appris du provider.
|
||||
|
||||
`try_admit()` ne dort jamais : il retourne immédiatement `Ready` ou `Deferred(duration)`. Cette propriété prépare le `refresh multiple/all` de `pre.008`, qui devra continuer avec les autres providers au lieu d'attendre un provider en cooldown.
|
||||
|
||||
Un `429` peut étendre le cooldown avec `Retry-After`; la durée provider est bornée à une heure et comparée au fallback local.
|
||||
|
||||
## 8. Tests ajoutés
|
||||
|
||||
Les nouveaux tests unitaires couvrent :
|
||||
|
||||
```text
|
||||
settings HTTP par défaut et bornes pathologiques
|
||||
policy fixe invalide
|
||||
burst absent lissé conservativement
|
||||
burst documenté consommé atomiquement
|
||||
refill déterministe du token bucket
|
||||
cooldown provider et borne Retry-After
|
||||
Debug request sans URL/credential
|
||||
GET JSON valide
|
||||
redirect refusé
|
||||
body chunked dépassant la limite
|
||||
429 + Retry-After sans propagation du body canari
|
||||
JSON invalide après statut 2xx
|
||||
```
|
||||
|
||||
Les canaries d'intégration vérifient aussi :
|
||||
|
||||
```text
|
||||
reqwest rustls présent
|
||||
Config toujours absent
|
||||
SDK providers toujours absents
|
||||
tracing direct toujours absent
|
||||
modules http_* présents
|
||||
HttpRestClient non exporté publiquement
|
||||
redirect/no-proxy/no-retry/without_url explicites dans le client
|
||||
codes d'erreur HTTP dans le domaine offchain_transport
|
||||
```
|
||||
|
||||
## 9. Fichiers ajoutés
|
||||
|
||||
```text
|
||||
crates/ksp-offchain-transport-lib/src/http_admission.rs
|
||||
crates/ksp-offchain-transport-lib/src/http_client.rs
|
||||
crates/ksp-offchain-transport-lib/src/http_settings.rs
|
||||
crates/ksp-offchain-transport-lib/unit_tests/http_admission.rs
|
||||
crates/ksp-offchain-transport-lib/unit_tests/http_client.rs
|
||||
crates/ksp-offchain-transport-lib/unit_tests/http_settings.rs
|
||||
deltas/0.2.11/pre.003.md
|
||||
```
|
||||
|
||||
## 10. Fichiers modifiés
|
||||
|
||||
```text
|
||||
Cargo.toml
|
||||
crates/ksp-offchain-transport-lib/Cargo.toml
|
||||
crates/ksp-offchain-transport-lib/src/error.rs
|
||||
crates/ksp-offchain-transport-lib/src/lib.rs
|
||||
crates/ksp-offchain-transport-lib/tests/dependency_boundary.rs
|
||||
crates/ksp-offchain-transport-lib/tests/public_api.rs
|
||||
docs/plans/018-V0_2_11_OFFCHAIN_PRICE_TRANSPORT_PLAN.md
|
||||
docs/validation/014-V0_2_11_OFFCHAIN_PRICE_TRANSPORT.md
|
||||
```
|
||||
|
||||
## 11. Fichiers supprimés
|
||||
|
||||
```text
|
||||
aucun
|
||||
```
|
||||
|
||||
## 12. Fichiers volontairement inchangés
|
||||
|
||||
```text
|
||||
CHANGELOG.md
|
||||
ROADMAP.md
|
||||
README.md
|
||||
.env.example
|
||||
config/**
|
||||
crates/ksp-config-lib/**
|
||||
crates/ksp-onchain-transport-lib/**
|
||||
prompts/**
|
||||
```
|
||||
|
||||
`ROADMAP.md` et `CHANGELOG.md` restent hors de la tranche conformément à leur ownership de release.
|
||||
|
||||
## 13. Validations exécutées dans le sandbox
|
||||
|
||||
```text
|
||||
python3 scripts/audit_rust_workspace_rules.py
|
||||
General Rust rule audit: clean
|
||||
Rust export completeness audit: 0 candidate(s)
|
||||
KSP workspace Rust rule audit: clean
|
||||
|
||||
python3 scripts/audit_markdown_tables.py README.md RULES.md ROADMAP.md CHANGELOG.md docs prompts crates deltas/0.2.11
|
||||
clean
|
||||
|
||||
contrôle lignes Rust > 160 sur ksp-offchain-transport-lib
|
||||
PASS
|
||||
|
||||
inspection diff contre 0.2.11-pre.002-fix.003
|
||||
aucun provider adapter ajouté
|
||||
aucune dépendance ksp-config-lib ajoutée
|
||||
aucune dépendance tracing directe ajoutée
|
||||
```
|
||||
|
||||
## 14. Validations non exécutées dans le sandbox
|
||||
|
||||
Le sandbox de génération ne fournit pas `cargo`, `rustc` ou `rustfmt`.
|
||||
|
||||
Après application, exécuter :
|
||||
|
||||
```bash
|
||||
cargo fmt --all
|
||||
python3 scripts/audit_rust_workspace_rules.py
|
||||
python3 scripts/audit_markdown_tables.py README.md RULES.md ROADMAP.md CHANGELOG.md docs prompts crates deltas/0.2.11
|
||||
cargo check --workspace
|
||||
cargo clippy --workspace --all-targets
|
||||
cargo test -p ksp-offchain-transport-lib
|
||||
cargo test --workspace
|
||||
```
|
||||
|
||||
## 15. Décisions et points différés
|
||||
|
||||
Aucune question ne bloque `pre.004`.
|
||||
|
||||
Restent volontairement différés :
|
||||
|
||||
```text
|
||||
mapping MarketPriceProviderRateLimit -> HttpAdmissionPolicy par provider
|
||||
CoinGecko/CoinMarketCap/CoinPaprika wire DTOs et endpoints
|
||||
credentials provider effectifs
|
||||
registry et availability runtime
|
||||
refresh single/multiple
|
||||
Config std.offchain_transport
|
||||
Retry-After HTTP-date si un provider retenu l'exige réellement
|
||||
POST/écritures HTTP pour futures familles off-chain
|
||||
```
|
||||
|
||||
`pre.004` reste propriétaire du premier lot d'adapters : CoinGecko, CoinMarketCap et CoinPaprika.
|
||||
@@ -1,9 +1,9 @@
|
||||
<!-- file: docs/plans/018-V0_2_11_OFFCHAIN_PRICE_TRANSPORT_PLAN.md -->
|
||||
<!-- version: 6 -->
|
||||
<!-- version: 7 -->
|
||||
|
||||
# Plan `0.2.11` — Off-chain price transport SOL/USD multi-provider
|
||||
|
||||
**Statut courant : `0.2.11-pre.002-fix.003` ferme le nommage public de la famille `market_price` avant `pre.003`. Les types, constantes et codes d'erreur publics spécifiques à cette famille portent désormais le préfixe `MarketPrice` / `MARKET_PRICE`, afin d'éviter une collision future avec `swap_quote_*` ou une autre capacité off-chain. Le contrat V1 reste limité à SOL/USD et aucun client HTTP/provider n'est encore ajouté.**
|
||||
**Statut courant : `0.2.11-pre.003` matérialise le HTTP REST commun et le rate limiting provider-neutral sans encore ajouter d'adapter provider. Les primitives `http_*` restent internes à la crate et réutilisables par les futures familles off-chain ; `market_price_*` reste la seule famille métier actuelle et demeure limitée à SOL/USD.**
|
||||
|
||||
## 1. Base et autorité
|
||||
|
||||
@@ -482,24 +482,30 @@ Décisions V1 :
|
||||
|
||||
```text
|
||||
reqwest uniquement
|
||||
HTTPS officiel fixe lorsque le provider le supporte
|
||||
HTTPS officiel fixe dans les adapters V1
|
||||
redirect désactivé
|
||||
proxy implicite désactivé selon le pattern KSP retenu
|
||||
connect timeout borné
|
||||
request timeout borné
|
||||
Referer automatique désactivé
|
||||
proxy système/implicite désactivé
|
||||
retries implicites reqwest désactivés
|
||||
connect timeout borné : défaut 5 s, hard max 30 s
|
||||
request timeout borné : défaut 10 s, hard max 120 s
|
||||
GET uniquement pour les prix V1
|
||||
body réponse borné
|
||||
JSON attendu et validé
|
||||
body réponse borné avant désérialisation : défaut 1 MiB, hard max 4 MiB
|
||||
JSON syntaxiquement validé avant remise à l'adapter typed
|
||||
429 classé explicitement
|
||||
Retry-After honoré lorsqu'il est exploitable
|
||||
5xx classé transitoire
|
||||
401/403 classé auth/access
|
||||
Retry-After delta-seconds honoré et borné à 1 h lorsqu'il est exploitable
|
||||
408 et 5xx classés transitoires
|
||||
401/403 classés auth/access
|
||||
URL retirée des erreurs reqwest avant contexte KSP
|
||||
aucun body remote brut dans KspError
|
||||
```
|
||||
|
||||
Il n'existe pas d'URL provider arbitraire dans la Config V1. Cela évite SSRF, redirection de credential et confusion de provenance.
|
||||
|
||||
Les primitives `http_*` sont **crate-private** : elles ne créent pas un client HTTP générique public contournant les capacités métier. Les adapters `market_price_*`, puis de futures familles telles que `swap_quote_*`, les consomment derrière leur propre contrat.
|
||||
|
||||
Le limiter `pre.003` est non bloquant. Une cadence fixe est matérialisée par un token bucket lissé ; lorsqu'aucun burst n'est documenté, la capacité locale initiale reste volontairement `1`. Une limite dynamique n'invente aucune cadence locale et apprend seulement des réponses provider, notamment `429`. Un `Retry-After` serveur peut prolonger le cooldown mais ne peut pas dépasser une borne défensive d'une heure. Aucune primitive HTTP commune ne dort en attendant la disponibilité : elle expose un délai de defer que l'orchestrateur `pre.008` pourra projeter provider par provider.
|
||||
|
||||
## 15. Config provider-capability-aware
|
||||
|
||||
Config reste l'unique propriétaire des documents/env/secrets.
|
||||
@@ -686,9 +692,9 @@ Préfixage de toute la surface publique spécifique à la famille `market_price`
|
||||
|
||||
### `pre.003` — HTTP REST commun et rate limiting
|
||||
|
||||
**Statut : planifié.**
|
||||
**Statut : réalisé.**
|
||||
|
||||
Client HTTP REST commun, classification d'erreurs, bornes de body et timeouts, redaction, limiter et cooldown génériques.
|
||||
Client HTTP REST commun crate-private, classification d'erreurs HTTP/reqwest, bornes de body et timeouts, redaction, désactivation explicite des redirects/proxy/retries implicites, JSON syntaxiquement validé, limiter token-bucket générique non bloquant et cooldown provider-driven borné. Aucun adapter provider n'est avancé.
|
||||
|
||||
### `pre.004` — CoinGecko, CoinMarketCap et CoinPaprika
|
||||
|
||||
|
||||
@@ -1157,4 +1157,3 @@ warnings 0
|
||||
```
|
||||
|
||||
Après validation de `rel.001`, le tag unique de la release est `v0.2.8` et `prompts/014-V0_2_9_START_PROMPT.md` devient le contrat actif pour `0.2.9-pre.001`.
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
<!-- file: docs/validation/014-V0_2_11_OFFCHAIN_PRICE_TRANSPORT.md -->
|
||||
<!-- version: 5 -->
|
||||
<!-- version: 6 -->
|
||||
|
||||
# Validation `0.2.11` — Off-chain price transport SOL/USD
|
||||
|
||||
@@ -103,6 +103,29 @@ ERROR_CODE_MARKET_PRICE_*
|
||||
|
||||
Cette correction reste dans la responsabilité de `pre.002` : elle stabilise la fondation et sa façade avant l'introduction du HTTP runtime en `pre.003`.
|
||||
|
||||
## 3.4 Gate `0.2.11-pre.003`
|
||||
|
||||
| Critère | Statut | Preuve |
|
||||
|-----------------------------------------------------------------------|---------|-----------------------------------------------------|
|
||||
| version workspace `0.2.11-pre.3` | PASS | `Cargo.toml` racine |
|
||||
| `reqwest` utilisé directement, sans SDK provider | PASS | manifest `ksp-offchain-transport-lib` |
|
||||
| primitives HTTP communes sous modules `http_*` crate-private | PASS | `http_client`, `http_settings`, `http_admission` |
|
||||
| client HTTP générique non exporté aux consumers | PASS | façade `lib.rs` |
|
||||
| redirects, Referer, proxy système et retries reqwest désactivés | PASS | construction `HttpRestClient` |
|
||||
| connect/request timeouts bornés | PASS | `HttpClientSettings` |
|
||||
| body borné pendant lecture chunked avant désérialisation | PASS | `HttpRestClient::get_json` |
|
||||
| JSON syntaxiquement validé sans normalisation numérique intermédiaire | PASS | `serde::de::IgnoredAny` sur bytes bruts |
|
||||
| 401/403, 429, 408/5xx et autres statuts distingués | PASS | codes d'erreur HTTP dédiés |
|
||||
| URL reqwest supprimée des sources d'erreur | PASS | `reqwest::Error::without_url` |
|
||||
| body remote non recopié dans les erreurs | PASS | erreurs construites uniquement depuis metadata safe |
|
||||
| limiter fixe lissé et burst absent traité conservativement | PASS | `HttpAdmissionController` |
|
||||
| limite dynamique n'invente pas de cadence locale | PASS | `HttpAdmissionPolicy::Dynamic` |
|
||||
| cooldown `Retry-After` borné et admission non bloquante | PASS | `record_rate_limited` + `try_admit` |
|
||||
| adapters CoinGecko/Jupiter/etc. ajoutés | N/A | explicitement réservés à `pre.004+` |
|
||||
| gate Cargo complet sur l'état livré | PENDING | à exécuter par l'opérateur |
|
||||
|
||||
Les tests déterministes de `pre.003` couvrent en particulier le body chunked dépassant la limite, le refus des redirects, un `429` avec `Retry-After`, la non-propagation d'un body distant canari, le JSON invalide et le refill/cooldown du limiter.
|
||||
|
||||
## 4. Matrice provider prévue
|
||||
|
||||
| Provider | SOL/USD V1 | Gratuit V1 | Mode auth prévu | Test déterministe | Smoke live | Statut courant |
|
||||
|
||||
Reference in New Issue
Block a user