v0.3.14-pre.006
This commit is contained in:
@@ -1,12 +1,12 @@
|
||||
# file: Cargo.toml
|
||||
# version: 572
|
||||
# version: 573
|
||||
|
||||
[workspace]
|
||||
resolver = "3"
|
||||
members = ["crates/ksp-app-backfill-desk", "crates/ksp-app-config-desk", "crates/ksp-app-solprices-desk", "crates/ksp-app-store-desk", "crates/ksp-app-wallet-desk", "crates/ksp-config-lib", "crates/ksp-core-lib", "crates/ksp-interface-lib", "crates/ksp-job-api", "crates/ksp-job-backfill-lib", "crates/ksp-logging-lib", "crates/ksp-offchain-transport-lib", "crates/ksp-onchain-transport-lib", "crates/ksp-program-api", "crates/ksp-raw-transaction-lib", "crates/ksp-store-api", "crates/ksp-store-lib", "crates/ksp-store-postgres-lib", "crates/ksp-wallet-lib", "crates/ksp-worker-api", "crates/ksp-worker-raw-transaction-ingest-lib"]
|
||||
|
||||
[workspace.package]
|
||||
version = "0.3.14-pre.5"
|
||||
version = "0.3.14-pre.6"
|
||||
edition = "2024"
|
||||
license = "MIT"
|
||||
repository = "https://git.sasedev.com/Sasedev/khadhroony-solana-project"
|
||||
|
||||
@@ -1,5 +1,8 @@
|
||||
// file: crates/ksp-worker-raw-transaction-ingest-lib/src/continuity.rs
|
||||
// version: 5
|
||||
// version: 6
|
||||
|
||||
/// Maximum number of slots admitted by one private continuity HTTP discovery window outside this module.
|
||||
pub(crate) const MAX_RAW_TRANSACTION_INGEST_CONTINUITY_DISCOVERY_WINDOW_SLOTS: u64 = MAX_RAW_TRANSACTION_INGEST_REPAIR_DISCOVERY_WINDOW_SLOTS;
|
||||
|
||||
/// Maximum number of proven redundant coverage epochs retained by one run-local continuity contract.
|
||||
const MAX_RAW_TRANSACTION_INGEST_COVERAGE_EPOCHS: usize = 256;
|
||||
@@ -627,8 +630,8 @@ impl crate::RawTransactionIngestContinuityContracts {
|
||||
return std::result::Result::Err(crate::runtime_error("continuity.coverage_scope_catalog_invalid"));
|
||||
}
|
||||
if MAX_RAW_TRANSACTION_INGEST_REPAIR_BLOCK_FETCH_IN_FLIGHT == 0
|
||||
|| MAX_RAW_TRANSACTION_INGEST_REPAIR_DISCOVERY_WINDOW_SLOTS == 0
|
||||
|| MAX_RAW_TRANSACTION_INGEST_REPAIR_DISCOVERY_WINDOW_SLOTS > MAX_RAW_TRANSACTION_INGEST_REPAIR_RANGE_SLOTS
|
||||
|| crate::MAX_RAW_TRANSACTION_INGEST_CONTINUITY_DISCOVERY_WINDOW_SLOTS == 0
|
||||
|| crate::MAX_RAW_TRANSACTION_INGEST_CONTINUITY_DISCOVERY_WINDOW_SLOTS > MAX_RAW_TRANSACTION_INGEST_REPAIR_RANGE_SLOTS
|
||||
{
|
||||
return std::result::Result::Err(crate::runtime_error("continuity.repair_bounds_invalid"));
|
||||
}
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
// file: crates/ksp-worker-raw-transaction-ingest-lib/src/lib.rs
|
||||
// version: 29
|
||||
// version: 30
|
||||
|
||||
#![warn(missing_docs)]
|
||||
#![deny(unreachable_pub)]
|
||||
@@ -111,6 +111,8 @@ pub use self::snapshot::RawTransactionIngestSourceState;
|
||||
pub(crate) use self::admission::RawTransactionAdmission;
|
||||
/// Crate-private source-neutral ingress sent through the bounded central admission queue.
|
||||
pub(crate) use self::admission::RawTransactionIngress;
|
||||
/// Maximum number of slots admitted by one private continuity HTTP discovery window.
|
||||
pub(crate) use self::continuity::MAX_RAW_TRANSACTION_INGEST_CONTINUITY_DISCOVERY_WINDOW_SLOTS;
|
||||
/// Private source continuity-capability descriptor prepared without network or Store I/O.
|
||||
pub(crate) use self::continuity::RawTransactionIngestContinuityCapabilityDescriptor;
|
||||
/// Private run-local continuity aggregate containing capabilities, TargetCoverage and the gap ledger.
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
// file: crates/ksp-worker-raw-transaction-ingest-lib/src/runtime_resources.rs
|
||||
// version: 31
|
||||
// version: 32
|
||||
|
||||
use sha2::Digest; // rust-rules: trait-import
|
||||
|
||||
@@ -36,6 +36,42 @@ const RAW_TRANSACTION_INGEST_YELLOWSTONE_FILTER_FINGERPRINT_DOMAIN: &[u8] = b"ks
|
||||
const RAW_TRANSACTION_INGEST_YELLOWSTONE_HTTP_PROTOCOL: &str = "yellowstone_http";
|
||||
const RAW_TRANSACTION_INGEST_YELLOWSTONE_HTTP_SOURCE_KEY_DOMAIN: &[u8] = b"ksp.raw_transaction_ingest.yellowstone_http.source_key.v1\0";
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
enum RawTransactionIngestHttpDiscoveryStrategy {
|
||||
ClosedRange,
|
||||
WithLimitBoundary,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Eq, PartialEq)]
|
||||
struct RawTransactionIngestHttpDiscoveryWindow {
|
||||
produced_slots: std::vec::Vec<u64>,
|
||||
proven_end_slot: std::option::Option<u64>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
struct RawTransactionIngestHttpScanCapabilities {
|
||||
get_block: bool,
|
||||
get_blocks: bool,
|
||||
get_blocks_with_limit: bool,
|
||||
get_slot: bool,
|
||||
}
|
||||
|
||||
impl RawTransactionIngestHttpScanCapabilities {
|
||||
const fn can_scan(self) -> bool {
|
||||
return self.get_block && (self.get_blocks || (self.get_blocks_with_limit && self.get_slot));
|
||||
}
|
||||
|
||||
fn strategy(self) -> ksp_core_lib::Result<RawTransactionIngestHttpDiscoveryStrategy> {
|
||||
if self.get_block && self.get_blocks {
|
||||
return std::result::Result::Ok(RawTransactionIngestHttpDiscoveryStrategy::ClosedRange);
|
||||
}
|
||||
if self.get_block && self.get_blocks_with_limit && self.get_slot {
|
||||
return std::result::Result::Ok(RawTransactionIngestHttpDiscoveryStrategy::WithLimitBoundary);
|
||||
}
|
||||
return std::result::Result::Err(crate::runtime_error("continuity.http_scan_capability_incomplete"));
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
enum RawTransactionIngestSourceFamily {
|
||||
Block,
|
||||
@@ -1340,33 +1376,40 @@ impl crate::RawTransactionIngestHttpBlockPollingSource {
|
||||
},
|
||||
};
|
||||
if next_scan_slot <= current_tip {
|
||||
let discovered = tokio::select! {
|
||||
biased;
|
||||
_ = stop_receiver.changed() => {
|
||||
break;
|
||||
}
|
||||
result = self.http_pool.get_blocks_with_limit(
|
||||
&self.polling_role,
|
||||
next_scan_slot,
|
||||
u64::from(self.max_discovered_blocks_per_cycle),
|
||||
std::option::Option::Some(&context_config),
|
||||
) => result,
|
||||
};
|
||||
let discovered = match discovered {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(error) => {
|
||||
fault = std::option::Option::Some(source_transport_error(error.code()));
|
||||
let configured_window_slots =
|
||||
u64::from(self.max_discovered_blocks_per_cycle).min(crate::MAX_RAW_TRANSACTION_INGEST_CONTINUITY_DISCOVERY_WINDOW_SLOTS);
|
||||
let window_offset = match configured_window_slots.checked_sub(1) {
|
||||
std::option::Option::Some(value) => value,
|
||||
std::option::Option::None => {
|
||||
fault = std::option::Option::Some(crate::runtime_error("source.http_block_polling_window_invalid"));
|
||||
break;
|
||||
},
|
||||
};
|
||||
let candidate_end = match next_scan_slot.checked_add(window_offset) {
|
||||
std::option::Option::Some(value) => value,
|
||||
std::option::Option::None => u64::MAX,
|
||||
};
|
||||
let window_end = current_tip.min(candidate_end);
|
||||
let discovered = discover_http_block_window(
|
||||
&self.http_pool,
|
||||
&self.polling_role,
|
||||
self.network.as_str(),
|
||||
self.commitment,
|
||||
next_scan_slot,
|
||||
window_end,
|
||||
&mut stop_receiver,
|
||||
)
|
||||
.await;
|
||||
let discovery = match discovered {
|
||||
std::result::Result::Ok(std::option::Option::Some(value)) => value,
|
||||
std::result::Result::Ok(std::option::Option::None) => break,
|
||||
std::result::Result::Err(error) => {
|
||||
fault = std::option::Option::Some(error);
|
||||
break;
|
||||
},
|
||||
};
|
||||
let validated = validate_http_block_polling_discovery(next_scan_slot, discovered.as_slice());
|
||||
if let std::result::Result::Err(error) = validated {
|
||||
fault = std::option::Option::Some(error);
|
||||
break;
|
||||
}
|
||||
let discovered_count = discovered.len();
|
||||
let mut blocked_by_null = false;
|
||||
for slot in discovered {
|
||||
for slot in discovery.produced_slots {
|
||||
let observed = tokio::select! {
|
||||
biased;
|
||||
_ = stop_receiver.changed() => {
|
||||
@@ -1424,16 +1467,9 @@ impl crate::RawTransactionIngestHttpBlockPollingSource {
|
||||
fault = std::option::Option::Some(error);
|
||||
break 'source;
|
||||
}
|
||||
next_scan_slot = match slot.checked_add(1) {
|
||||
std::option::Option::Some(value) => value,
|
||||
std::option::Option::None => {
|
||||
fault = std::option::Option::Some(crate::runtime_error("source.http_block_polling_slot_exhausted"));
|
||||
break 'source;
|
||||
},
|
||||
};
|
||||
}
|
||||
if !blocked_by_null && discovered_count < usize::from(self.max_discovered_blocks_per_cycle) && next_scan_slot <= current_tip {
|
||||
next_scan_slot = match current_tip.checked_add(1) {
|
||||
if !blocked_by_null && let std::option::Option::Some(proven_end_slot) = discovery.proven_end_slot {
|
||||
next_scan_slot = match proven_end_slot.checked_add(1) {
|
||||
std::option::Option::Some(value) => value,
|
||||
std::option::Option::None => {
|
||||
fault = std::option::Option::Some(crate::runtime_error("source.http_block_polling_slot_exhausted"));
|
||||
@@ -2399,28 +2435,40 @@ async fn drain_live_source_tasks(
|
||||
};
|
||||
}
|
||||
|
||||
fn http_role_scan_capabilities(
|
||||
pool: &ksp_onchain_transport_lib::HttpTransportPool,
|
||||
role: &ksp_onchain_transport_lib::HttpRoleName,
|
||||
expected_cluster: &str,
|
||||
) -> ksp_core_lib::Result<RawTransactionIngestHttpScanCapabilities> {
|
||||
let get_block = match http_role_supports_rpc_method(pool, role, "getBlock", expected_cluster) {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(error) => return std::result::Result::Err(error),
|
||||
};
|
||||
let get_blocks = match http_role_supports_rpc_method(pool, role, "getBlocks", expected_cluster) {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(error) => return std::result::Result::Err(error),
|
||||
};
|
||||
let get_blocks_with_limit = match http_role_supports_rpc_method(pool, role, "getBlocksWithLimit", expected_cluster) {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(error) => return std::result::Result::Err(error),
|
||||
};
|
||||
let get_slot = match http_role_supports_rpc_method(pool, role, "getSlot", expected_cluster) {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(error) => return std::result::Result::Err(error),
|
||||
};
|
||||
return std::result::Result::Ok(RawTransactionIngestHttpScanCapabilities { get_block, get_blocks, get_blocks_with_limit, get_slot });
|
||||
}
|
||||
|
||||
fn http_role_supports_repair_scan(
|
||||
pool: &ksp_onchain_transport_lib::HttpTransportPool,
|
||||
role: &ksp_onchain_transport_lib::HttpRoleName,
|
||||
expected_cluster: &str,
|
||||
) -> ksp_core_lib::Result<bool> {
|
||||
let has_get_block = match http_role_supports_rpc_method(pool, role, "getBlock", expected_cluster) {
|
||||
let capabilities = match http_role_scan_capabilities(pool, role, expected_cluster) {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(error) => return std::result::Result::Err(error),
|
||||
};
|
||||
let has_get_blocks = match http_role_supports_rpc_method(pool, role, "getBlocks", expected_cluster) {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(error) => return std::result::Result::Err(error),
|
||||
};
|
||||
let has_get_blocks_with_limit = match http_role_supports_rpc_method(pool, role, "getBlocksWithLimit", expected_cluster) {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(error) => return std::result::Result::Err(error),
|
||||
};
|
||||
let has_get_slot = match http_role_supports_rpc_method(pool, role, "getSlot", expected_cluster) {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(error) => return std::result::Result::Err(error),
|
||||
};
|
||||
return std::result::Result::Ok(has_get_block && has_get_slot && (has_get_blocks || has_get_blocks_with_limit));
|
||||
return std::result::Result::Ok(capabilities.can_scan());
|
||||
}
|
||||
|
||||
fn http_role_supports_rpc_method(
|
||||
@@ -2561,6 +2609,125 @@ fn http_role_supports_request_kind(role: &ksp_onchain_transport_lib::HttpEndpoin
|
||||
return role.request_kinds().iter().any(|kind| return kind.as_str() == "*" || kind.as_str() == request_kind);
|
||||
}
|
||||
|
||||
fn http_discovery_window_slot_count(start_slot: u64, end_slot: u64) -> ksp_core_lib::Result<u64> {
|
||||
if end_slot < start_slot {
|
||||
return std::result::Result::Err(crate::runtime_error("continuity.http_discovery_range_reversed"));
|
||||
}
|
||||
let slot_count = match end_slot.checked_sub(start_slot).and_then(|value| return value.checked_add(1)) {
|
||||
std::option::Option::Some(value) => value,
|
||||
std::option::Option::None => return std::result::Result::Err(crate::runtime_error("continuity.http_discovery_range_overflow")),
|
||||
};
|
||||
if slot_count > crate::MAX_RAW_TRANSACTION_INGEST_CONTINUITY_DISCOVERY_WINDOW_SLOTS {
|
||||
return std::result::Result::Err(crate::runtime_error("continuity.http_discovery_window_too_large"));
|
||||
}
|
||||
return std::result::Result::Ok(slot_count);
|
||||
}
|
||||
|
||||
fn validate_http_block_discovery_result(
|
||||
start_slot: u64,
|
||||
end_slot: u64,
|
||||
strategy: RawTransactionIngestHttpDiscoveryStrategy,
|
||||
current_tip: std::option::Option<u64>,
|
||||
discovered: &[u64],
|
||||
) -> ksp_core_lib::Result<RawTransactionIngestHttpDiscoveryWindow> {
|
||||
if let std::result::Result::Err(error) = http_discovery_window_slot_count(start_slot, end_slot) {
|
||||
return std::result::Result::Err(error);
|
||||
}
|
||||
if let std::result::Result::Err(error) = validate_http_block_polling_discovery(start_slot, discovered) {
|
||||
return std::result::Result::Err(error);
|
||||
}
|
||||
return match strategy {
|
||||
RawTransactionIngestHttpDiscoveryStrategy::ClosedRange => {
|
||||
if discovered.iter().any(|slot| return *slot > end_slot) {
|
||||
return std::result::Result::Err(crate::runtime_error("continuity.http_closed_range_exceeded"));
|
||||
}
|
||||
std::result::Result::Ok(RawTransactionIngestHttpDiscoveryWindow {
|
||||
produced_slots: discovered.to_vec(),
|
||||
proven_end_slot: std::option::Option::Some(end_slot),
|
||||
})
|
||||
},
|
||||
RawTransactionIngestHttpDiscoveryStrategy::WithLimitBoundary => {
|
||||
let current_tip = match current_tip {
|
||||
std::option::Option::Some(value) => value,
|
||||
std::option::Option::None => return std::result::Result::Err(crate::runtime_error("continuity.http_discovery_tip_missing")),
|
||||
};
|
||||
if current_tip < start_slot && discovered.is_empty() {
|
||||
return std::result::Result::Ok(RawTransactionIngestHttpDiscoveryWindow {
|
||||
produced_slots: std::vec::Vec::new(),
|
||||
proven_end_slot: std::option::Option::None,
|
||||
});
|
||||
}
|
||||
let proven_end_slot = discovered.last().copied().map(|slot| return slot.min(end_slot));
|
||||
let produced_slots = discovered.iter().copied().take_while(|slot| return *slot <= end_slot).collect();
|
||||
std::result::Result::Ok(RawTransactionIngestHttpDiscoveryWindow { produced_slots, proven_end_slot })
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
async fn discover_http_block_window(
|
||||
pool: &ksp_onchain_transport_lib::HttpTransportPool,
|
||||
role: &ksp_onchain_transport_lib::HttpRoleName,
|
||||
expected_cluster: &str,
|
||||
commitment: ksp_onchain_transport_lib::SolanaCommitment,
|
||||
start_slot: u64,
|
||||
end_slot: u64,
|
||||
stop_receiver: &mut tokio::sync::watch::Receiver<bool>,
|
||||
) -> ksp_core_lib::Result<std::option::Option<RawTransactionIngestHttpDiscoveryWindow>> {
|
||||
let slot_count = match http_discovery_window_slot_count(start_slot, end_slot) {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(error) => return std::result::Result::Err(error),
|
||||
};
|
||||
let capabilities = match http_role_scan_capabilities(pool, role, expected_cluster) {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(error) => return std::result::Result::Err(error),
|
||||
};
|
||||
let strategy = match capabilities.strategy() {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(error) => return std::result::Result::Err(error),
|
||||
};
|
||||
let context_config = ksp_onchain_transport_lib::SolanaContextConfig::new(std::option::Option::Some(commitment), std::option::Option::None);
|
||||
let mut current_tip = std::option::Option::None;
|
||||
let discovered = match strategy {
|
||||
RawTransactionIngestHttpDiscoveryStrategy::ClosedRange => tokio::select! {
|
||||
biased;
|
||||
_ = stop_receiver.changed() => {
|
||||
return std::result::Result::Ok(std::option::Option::None);
|
||||
}
|
||||
result = pool.get_blocks(role, start_slot, std::option::Option::Some(end_slot), std::option::Option::Some(&context_config)) => result,
|
||||
},
|
||||
RawTransactionIngestHttpDiscoveryStrategy::WithLimitBoundary => {
|
||||
let tip = tokio::select! {
|
||||
biased;
|
||||
_ = stop_receiver.changed() => {
|
||||
return std::result::Result::Ok(std::option::Option::None);
|
||||
}
|
||||
result = pool.get_slot(role, std::option::Option::Some(&context_config)) => result,
|
||||
};
|
||||
let tip = match tip {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(error) => return std::result::Result::Err(source_transport_error(error.code())),
|
||||
};
|
||||
current_tip = std::option::Option::Some(tip);
|
||||
tokio::select! {
|
||||
biased;
|
||||
_ = stop_receiver.changed() => {
|
||||
return std::result::Result::Ok(std::option::Option::None);
|
||||
}
|
||||
result = pool.get_blocks_with_limit(role, start_slot, slot_count, std::option::Option::Some(&context_config)) => result,
|
||||
}
|
||||
},
|
||||
};
|
||||
let discovered = match discovered {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(error) => return std::result::Result::Err(source_transport_error(error.code())),
|
||||
};
|
||||
let validated = validate_http_block_discovery_result(start_slot, end_slot, strategy, current_tip, discovered.as_slice());
|
||||
return match validated {
|
||||
std::result::Result::Ok(value) => std::result::Result::Ok(std::option::Option::Some(value)),
|
||||
std::result::Result::Err(error) => std::result::Result::Err(error),
|
||||
};
|
||||
}
|
||||
|
||||
fn validate_http_block_polling_discovery(next_scan_slot: u64, discovered: &[u64]) -> ksp_core_lib::Result<()> {
|
||||
let mut previous = std::option::Option::None;
|
||||
for slot in discovered {
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
// file: crates/ksp-worker-raw-transaction-ingest-lib/tests/hardening.rs
|
||||
// version: 29
|
||||
// version: 30
|
||||
|
||||
//! External public, security, redaction and release-boundary hardening canaries through `v0.3.14-pre.005`.
|
||||
//! External public, security, redaction and release-boundary hardening canaries through `v0.3.14-pre.006`.
|
||||
|
||||
fn network(value: &'static str) -> std::option::Option<ksp_store_lib::RawNetworkId> {
|
||||
let result = ksp_store_lib::RawNetworkId::new(value);
|
||||
@@ -1090,3 +1090,29 @@ fn v0_3_14_pre_005_redundant_coverage_requires_exact_or_superset_proof_over_full
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn v0_3_14_pre_006_http_discovery_is_bounded_prefers_closed_range_and_never_uses_tip_alone_as_coverage() {
|
||||
let resources = include_str!("../src/runtime_resources.rs");
|
||||
let root = include_str!("../src/lib.rs");
|
||||
for required in [
|
||||
"RawTransactionIngestHttpDiscoveryStrategy",
|
||||
"RawTransactionIngestHttpDiscoveryWindow",
|
||||
"RawTransactionIngestHttpScanCapabilities",
|
||||
"ClosedRange",
|
||||
"WithLimitBoundary",
|
||||
"MAX_RAW_TRANSACTION_INGEST_CONTINUITY_DISCOVERY_WINDOW_SLOTS",
|
||||
"pool.get_blocks(role, start_slot, std::option::Option::Some(end_slot)",
|
||||
"pool.get_blocks_with_limit(role, start_slot, slot_count",
|
||||
"self.http_pool.get_block_observed",
|
||||
"discovered.last().copied().map(|slot| return slot.min(end_slot))",
|
||||
] {
|
||||
assert!(resources.contains(required), "required pre.006 bounded HTTP discovery guard missing: {required}");
|
||||
}
|
||||
assert!(resources.contains("http_role_scan_capabilities"));
|
||||
assert!(resources.contains("http_role_supports_repair_scan"));
|
||||
assert!(!resources.contains("ksp_job_backfill_lib::"));
|
||||
assert!(!resources.contains("get_blocks_with_limit(role, start_slot, u64::MAX"));
|
||||
assert!(!root.contains("repair"), "pre.006 leaked lower-case repair responsibility through crate root");
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
// file: crates/ksp-worker-raw-transaction-ingest-lib/tests/release_completeness.rs
|
||||
// version: 24
|
||||
// version: 25
|
||||
|
||||
//! Release-completeness canaries through the `v0.3.14-pre.005` redundant-coverage proof tranche.
|
||||
//! Release-completeness canaries through the `v0.3.14-pre.006` bounded HTTP continuity-discovery tranche.
|
||||
|
||||
#[test]
|
||||
fn pre_010_production_module_inventory_is_exact() -> std::io::Result<()> {
|
||||
@@ -282,3 +282,23 @@ fn v0_3_14_pre_005_redundant_coverage_canaries_are_present_without_public_surfac
|
||||
assert!(!root.contains("pub use self::continuity::RawTransactionIngestCoverageRelation"));
|
||||
return;
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn v0_3_14_pre_006_bounded_http_discovery_canaries_are_present_without_backfill_or_public_surface_growth() {
|
||||
let hardening = include_str!("hardening.rs");
|
||||
let resource_tests = include_str!("../unit_tests/runtime_resources.rs");
|
||||
let root = include_str!("../src/lib.rs");
|
||||
for required in [
|
||||
"v0_3_14_pre_006_http_scan_capabilities_prefer_closed_range_and_require_safe_fallback",
|
||||
"v0_3_14_pre_006_http_scan_capability_detection_is_role_local_and_io_free",
|
||||
"v0_3_14_pre_006_http_discovery_window_is_inclusive_bounded_and_overflow_safe",
|
||||
"v0_3_14_pre_006_closed_range_discovery_proves_skipped_slots_only_inside_exact_window",
|
||||
"v0_3_14_pre_006_with_limit_discovery_never_uses_tip_alone_as_tail_coverage_proof",
|
||||
] {
|
||||
assert!(resource_tests.contains(required), "required pre.006 HTTP discovery canary missing: {required}");
|
||||
}
|
||||
assert!(hardening.contains("v0_3_14_pre_006_http_discovery_is_bounded_prefers_closed_range_and_never_uses_tip_alone_as_coverage"));
|
||||
assert!(!root.contains("pub use self::runtime_resources::RawTransactionIngestHttpDiscovery"));
|
||||
assert!(!root.contains("backfill"));
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
// file: crates/ksp-worker-raw-transaction-ingest-lib/unit_tests/runtime_resources.rs
|
||||
// version: 25
|
||||
// version: 26
|
||||
|
||||
fn grpc_endpoint(cluster: &str) -> std::option::Option<ksp_onchain_transport_lib::YellowstoneGrpcEndpointSettings> {
|
||||
return grpc_endpoint_with_identity(cluster, "yellowstone-fixture", "fixture-provider");
|
||||
@@ -2968,6 +2968,178 @@ fn v0_3_13_pre_006_http_block_polling_discovery_never_moves_before_run_frontier_
|
||||
return;
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn v0_3_14_pre_006_http_scan_capabilities_prefer_closed_range_and_require_safe_fallback() {
|
||||
let exact = super::RawTransactionIngestHttpScanCapabilities { get_block: true, get_blocks: true, get_blocks_with_limit: true, get_slot: true };
|
||||
assert!(exact.can_scan());
|
||||
let exact_strategy = match exact.strategy() {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(_) => return,
|
||||
};
|
||||
assert_eq!(exact_strategy, super::RawTransactionIngestHttpDiscoveryStrategy::ClosedRange);
|
||||
let fallback = super::RawTransactionIngestHttpScanCapabilities { get_block: true, get_blocks: false, get_blocks_with_limit: true, get_slot: true };
|
||||
assert!(fallback.can_scan());
|
||||
let fallback_strategy = match fallback.strategy() {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(_) => return,
|
||||
};
|
||||
assert_eq!(fallback_strategy, super::RawTransactionIngestHttpDiscoveryStrategy::WithLimitBoundary);
|
||||
for incomplete in [
|
||||
super::RawTransactionIngestHttpScanCapabilities { get_block: false, get_blocks: true, get_blocks_with_limit: true, get_slot: true },
|
||||
super::RawTransactionIngestHttpScanCapabilities { get_block: true, get_blocks: false, get_blocks_with_limit: true, get_slot: false },
|
||||
super::RawTransactionIngestHttpScanCapabilities { get_block: true, get_blocks: false, get_blocks_with_limit: false, get_slot: true },
|
||||
] {
|
||||
assert!(!incomplete.can_scan());
|
||||
assert!(incomplete.strategy().is_err());
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn v0_3_14_pre_006_http_scan_capability_detection_is_role_local_and_io_free() {
|
||||
let cases = [
|
||||
(&["get_block", "get_blocks"][..], std::option::Option::Some(super::RawTransactionIngestHttpDiscoveryStrategy::ClosedRange)),
|
||||
(
|
||||
&["get_block", "get_blocks_with_limit", "get_slot"][..],
|
||||
std::option::Option::Some(super::RawTransactionIngestHttpDiscoveryStrategy::WithLimitBoundary),
|
||||
),
|
||||
(&["get_transaction"][..], std::option::Option::None),
|
||||
];
|
||||
for (request_kinds, expected_strategy) in cases {
|
||||
let pool = match http_polling_pool_with_identity("devnet", "scan", "scan-fixture", "fixture-provider", request_kinds) {
|
||||
std::option::Option::Some(value) => value,
|
||||
std::option::Option::None => return,
|
||||
};
|
||||
let capabilities = match super::http_role_scan_capabilities(&pool, &ksp_onchain_transport_lib::HttpRoleName::new("scan"), "devnet") {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(_) => return,
|
||||
};
|
||||
match expected_strategy {
|
||||
std::option::Option::Some(expected) => {
|
||||
assert!(capabilities.can_scan());
|
||||
let actual = match capabilities.strategy() {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(_) => return,
|
||||
};
|
||||
assert_eq!(actual, expected);
|
||||
},
|
||||
std::option::Option::None => {
|
||||
assert!(!capabilities.can_scan());
|
||||
assert!(capabilities.strategy().is_err());
|
||||
},
|
||||
}
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn v0_3_14_pre_006_http_discovery_window_is_inclusive_bounded_and_overflow_safe() {
|
||||
let single = match super::http_discovery_window_slot_count(100, 100) {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(_) => return,
|
||||
};
|
||||
assert_eq!(single, 1);
|
||||
let maximum = match super::http_discovery_window_slot_count(100, 611) {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(_) => return,
|
||||
};
|
||||
assert_eq!(maximum, crate::MAX_RAW_TRANSACTION_INGEST_CONTINUITY_DISCOVERY_WINDOW_SLOTS);
|
||||
assert!(super::http_discovery_window_slot_count(100, 612).is_err());
|
||||
assert!(super::http_discovery_window_slot_count(101, 100).is_err());
|
||||
assert!(super::http_discovery_window_slot_count(0, u64::MAX).is_err());
|
||||
return;
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn v0_3_14_pre_006_closed_range_discovery_proves_skipped_slots_only_inside_exact_window() {
|
||||
let complete = match super::validate_http_block_discovery_result(
|
||||
100,
|
||||
105,
|
||||
super::RawTransactionIngestHttpDiscoveryStrategy::ClosedRange,
|
||||
std::option::Option::None,
|
||||
&[100, 102, 105],
|
||||
) {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(_) => return,
|
||||
};
|
||||
assert_eq!(complete.produced_slots, std::vec![100, 102, 105]);
|
||||
assert_eq!(complete.proven_end_slot, std::option::Option::Some(105));
|
||||
let all_skipped = match super::validate_http_block_discovery_result(
|
||||
200,
|
||||
203,
|
||||
super::RawTransactionIngestHttpDiscoveryStrategy::ClosedRange,
|
||||
std::option::Option::None,
|
||||
&[],
|
||||
) {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(_) => return,
|
||||
};
|
||||
assert!(all_skipped.produced_slots.is_empty());
|
||||
assert_eq!(all_skipped.proven_end_slot, std::option::Option::Some(203));
|
||||
assert!(
|
||||
super::validate_http_block_discovery_result(
|
||||
100,
|
||||
105,
|
||||
super::RawTransactionIngestHttpDiscoveryStrategy::ClosedRange,
|
||||
std::option::Option::None,
|
||||
&[100, 106],
|
||||
)
|
||||
.is_err()
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn v0_3_14_pre_006_with_limit_discovery_never_uses_tip_alone_as_tail_coverage_proof() {
|
||||
let crossed = match super::validate_http_block_discovery_result(
|
||||
100,
|
||||
105,
|
||||
super::RawTransactionIngestHttpDiscoveryStrategy::WithLimitBoundary,
|
||||
std::option::Option::Some(110),
|
||||
&[100, 103, 107],
|
||||
) {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(_) => return,
|
||||
};
|
||||
assert_eq!(crossed.produced_slots, std::vec![100, 103]);
|
||||
assert_eq!(crossed.proven_end_slot, std::option::Option::Some(105));
|
||||
let partial = match super::validate_http_block_discovery_result(
|
||||
100,
|
||||
105,
|
||||
super::RawTransactionIngestHttpDiscoveryStrategy::WithLimitBoundary,
|
||||
std::option::Option::Some(110),
|
||||
&[100, 103],
|
||||
) {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(_) => return,
|
||||
};
|
||||
assert_eq!(partial.produced_slots, std::vec![100, 103]);
|
||||
assert_eq!(partial.proven_end_slot, std::option::Option::Some(103));
|
||||
let tip_only = match super::validate_http_block_discovery_result(
|
||||
100,
|
||||
105,
|
||||
super::RawTransactionIngestHttpDiscoveryStrategy::WithLimitBoundary,
|
||||
std::option::Option::Some(110),
|
||||
&[],
|
||||
) {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(_) => return,
|
||||
};
|
||||
assert!(tip_only.produced_slots.is_empty());
|
||||
assert_eq!(tip_only.proven_end_slot, std::option::Option::None);
|
||||
assert!(
|
||||
super::validate_http_block_discovery_result(
|
||||
100,
|
||||
105,
|
||||
super::RawTransactionIngestHttpDiscoveryStrategy::WithLimitBoundary,
|
||||
std::option::Option::None,
|
||||
&[100, 103, 107],
|
||||
)
|
||||
.is_err()
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn v0_3_13_pre_006_http_block_polling_qualifies_legacy_v0_v1_and_rejects_ambiguous_or_future_versions() {
|
||||
let cases = [
|
||||
|
||||
274
deltas/0.3.14/pre.006.md
Normal file
274
deltas/0.3.14/pre.006.md
Normal file
@@ -0,0 +1,274 @@
|
||||
<!-- file: deltas/0.3.14/pre.006.md -->
|
||||
<!-- version: 1 -->
|
||||
|
||||
# Delta `0.3.14-pre.006` — discovery HTTP bornée pour continuité
|
||||
|
||||
## Base requise
|
||||
|
||||
```text
|
||||
0.3.14-pre.005
|
||||
workspace.package.version = 0.3.14-pre.5
|
||||
deltas/0.3.14/pre.005.md présent
|
||||
```
|
||||
|
||||
## Gate de la base
|
||||
|
||||
Le gate opérateur de `0.3.14-pre.005` est validé avant ouverture de cette tranche :
|
||||
|
||||
```text
|
||||
cargo fmt --all : PASS
|
||||
cargo fmt --all -- --check : PASS
|
||||
audit Rust workspace rules : PASS
|
||||
audit Markdown tables : PASS
|
||||
cargo check --workspace : PASS
|
||||
cargo clippy --workspace --all-targets --all-features -- -D warnings : PASS
|
||||
cargo test -p ksp-worker-raw-transaction-ingest-lib --all-targets --all-features : PASS
|
||||
```
|
||||
|
||||
Le gate Worker comprend notamment :
|
||||
|
||||
```text
|
||||
124 unit tests : PASS
|
||||
cross_layer_completeness : 8 PASS
|
||||
dependency_boundary : 19 PASS
|
||||
hardening : 30 PASS
|
||||
public_api : 20 PASS
|
||||
release_completeness : 7 PASS
|
||||
```
|
||||
|
||||
## Objectif
|
||||
|
||||
Implémenter strictement la tranche `pre.006` du plan `035` :
|
||||
|
||||
```text
|
||||
détecter les primitives HTTP de scan réellement disponibles sur le rôle same-network
|
||||
borner chaque fenêtre de discovery à 512 slots inclusifs
|
||||
préférer getBlocks(start,end) lorsqu'il est réellement supporté
|
||||
utiliser getBlocksWithLimit uniquement avec une preuve de progression issue de sa propre réponse
|
||||
ne jamais utiliser getSlot seul comme preuve de queue skipped
|
||||
réutiliser getBlock observed pour les slots explicitement produits
|
||||
conserver getBlock = null comme obstacle, jamais comme slot skipped
|
||||
ne pas élargir un scope filtré vers FullLedgerTransactions
|
||||
ne pas modifier encore la politique de supervisor ou de health
|
||||
ne pas introduire de logique caller-driven de backfill
|
||||
```
|
||||
|
||||
## Capability HTTP run-wide
|
||||
|
||||
`runtime_resources.rs` matérialise maintenant un inventaire privé :
|
||||
|
||||
```text
|
||||
RawTransactionIngestHttpScanCapabilities
|
||||
get_block
|
||||
get_blocks
|
||||
get_blocks_with_limit
|
||||
get_slot
|
||||
```
|
||||
|
||||
Le choix de primitive est conservative :
|
||||
|
||||
```text
|
||||
getBlock + getBlocks
|
||||
-> ClosedRange
|
||||
|
||||
getBlock + getBlocksWithLimit + getSlot
|
||||
-> WithLimitBoundary
|
||||
|
||||
autres combinaisons
|
||||
-> scan indisponible
|
||||
```
|
||||
|
||||
`ClosedRange` est toujours préféré lorsque les deux stratégies sont disponibles.
|
||||
|
||||
La détection reste fondée sur le snapshot Transport du rôle exact et du cluster attendu. La seule présence d'un pool HTTP ne crée donc aucune capability implicite.
|
||||
|
||||
## Fenêtres bornées
|
||||
|
||||
La borne canonique de `continuity.rs` reste :
|
||||
|
||||
```text
|
||||
MAX_RAW_TRANSACTION_INGEST_REPAIR_DISCOVERY_WINDOW_SLOTS = 512
|
||||
```
|
||||
|
||||
Un alias crate-wide de même valeur est ajouté uniquement pour que `runtime_resources.rs` réutilise cette borne sans la dupliquer :
|
||||
|
||||
```text
|
||||
MAX_RAW_TRANSACTION_INGEST_CONTINUITY_DISCOVERY_WINDOW_SLOTS
|
||||
```
|
||||
|
||||
Une fenêtre :
|
||||
|
||||
```text
|
||||
est inclusive
|
||||
rejette end < start
|
||||
rejette les overflows
|
||||
rejette plus de 512 slots
|
||||
```
|
||||
|
||||
La source HTTP polling productive réutilise désormais cette primitive. Sa valeur `max_discovered_blocks_per_cycle` reste une borne supérieure ; une fenêtre productive ne dépasse jamais le minimum entre cette valeur et `512` slots.
|
||||
|
||||
## Stratégie `ClosedRange`
|
||||
|
||||
Lorsque `getBlocks` est supporté, le Worker appelle :
|
||||
|
||||
```text
|
||||
getBlocks(start_slot, end_slot, commitment)
|
||||
```
|
||||
|
||||
Une réponse réussie constitue une preuve fermée de la fenêtre RPC demandée. Les slots absents de la liste sont alors qualifiables comme `skipped/non-produced` uniquement à l'intérieur de cette fenêtre.
|
||||
|
||||
Tout slot retourné au-delà de `end_slot` est rejeté comme violation d'invariant.
|
||||
|
||||
## Stratégie `WithLimitBoundary`
|
||||
|
||||
Lorsque seul le chemin `getBlocksWithLimit` qualifié est disponible :
|
||||
|
||||
```text
|
||||
getSlot(commitment)
|
||||
getBlocksWithLimit(start_slot, window_slot_count, commitment)
|
||||
```
|
||||
|
||||
`getSlot` indique uniquement la position du cluster observée par ce rôle. Il ne ferme jamais à lui seul la queue de la fenêtre.
|
||||
|
||||
La preuve avance seulement jusqu'au dernier slot produit effectivement énuméré par la réponse `getBlocksWithLimit` :
|
||||
|
||||
```text
|
||||
last_discovered < end_slot
|
||||
-> preuve partielle jusqu'à last_discovered
|
||||
|
||||
last_discovered >= end_slot
|
||||
-> fenêtre cible entièrement prouvée
|
||||
|
||||
aucun slot découvert
|
||||
-> aucune preuve de tail, même si getSlot >= end_slot
|
||||
```
|
||||
|
||||
Ainsi, une queue constituée uniquement de slots skipped n'est pas inventée à partir de deux réponses pouvant provenir de routes HTTP différentes. La fenêtre restera en attente jusqu'à ce qu'une primitive plus forte ou un bloc ultérieur fournisse une borne observable.
|
||||
|
||||
## `getBlock observed`
|
||||
|
||||
Chaque slot explicitement produit continue à passer par :
|
||||
|
||||
```text
|
||||
HttpTransportPool::get_block_observed
|
||||
```
|
||||
|
||||
Les règles existantes restent inchangées :
|
||||
|
||||
```text
|
||||
Some(block)
|
||||
-> projection Common RAW puis admission centrale
|
||||
|
||||
None
|
||||
-> la progression se bloque sur ce slot
|
||||
le slot n'est pas reclassé skipped
|
||||
```
|
||||
|
||||
Aucun second pipeline de persistence n'est ajouté.
|
||||
|
||||
## Scope et intégration
|
||||
|
||||
Cette tranche utilise immédiatement la primitive bornée uniquement dans `RawTransactionIngestHttpBlockPollingSource`, dont le scope est déjà `FullLedgerTransactions`.
|
||||
|
||||
Pour Yellowstone, Standard Logs et Helius Transaction, la détection run-wide de capability HTTP demeure disponible dans les continuity descriptors, mais aucun scan full-block n'est déclenché ici. Leur activation dépendra de la preuve de scope et de la réconciliation des tranches suivantes.
|
||||
|
||||
`Standard Block` conserve l'absence de pool HTTP attaché.
|
||||
|
||||
La politique de terminalité des incidents `pre.003/pre.004` n'est pas encore remplacée. `pre.008` reste responsable de l'intégration supervisor/gap-ledger.
|
||||
|
||||
## Tests ajoutés
|
||||
|
||||
Les unit tests Worker couvrent :
|
||||
|
||||
```text
|
||||
préférence ClosedRange sur fallback
|
||||
capability fallback complète obligatoire
|
||||
capability détectée par rôle sans I/O
|
||||
fenêtre inclusive 1..512
|
||||
rejet 513 slots, range inversé et overflow
|
||||
ClosedRange vide comme preuve de slots skipped dans la fenêtre exacte
|
||||
rejet d'un résultat ClosedRange au-delà de end_slot
|
||||
fallback dépassant end_slot et filtrage de la cible
|
||||
fallback partiel ne prouvant que jusqu'au dernier slot énuméré
|
||||
getSlot seul + réponse vide ne prouvant aucune queue
|
||||
absence de getSlot dans le fallback rejetée
|
||||
```
|
||||
|
||||
Les canaris `hardening` et `release_completeness` garantissent en plus :
|
||||
|
||||
```text
|
||||
présence des deux stratégies HTTP
|
||||
usage borné de getBlocks/getBlocksWithLimit
|
||||
réutilisation de getBlock observed
|
||||
absence de dépendance Job Backfill
|
||||
absence de surface publique de discovery
|
||||
absence de responsabilité repair lower-case dans la crate-root historique
|
||||
```
|
||||
|
||||
## Hors périmètre inchangé
|
||||
|
||||
```text
|
||||
aucun moteur de reconciliation supervisor
|
||||
aucun changement health public
|
||||
aucune hydration repair getTransaction dans cette tranche
|
||||
aucune source/provider supplémentaire
|
||||
aucun EARLY/shred adapter
|
||||
aucun backfill historique caller-driven
|
||||
```
|
||||
|
||||
## Fichiers ajoutés
|
||||
|
||||
```text
|
||||
deltas/0.3.14/pre.006.md
|
||||
```
|
||||
|
||||
## Fichiers modifiés
|
||||
|
||||
```text
|
||||
Cargo.toml
|
||||
crates/ksp-worker-raw-transaction-ingest-lib/src/continuity.rs
|
||||
crates/ksp-worker-raw-transaction-ingest-lib/src/lib.rs
|
||||
crates/ksp-worker-raw-transaction-ingest-lib/src/runtime_resources.rs
|
||||
crates/ksp-worker-raw-transaction-ingest-lib/tests/hardening.rs
|
||||
crates/ksp-worker-raw-transaction-ingest-lib/tests/release_completeness.rs
|
||||
crates/ksp-worker-raw-transaction-ingest-lib/unit_tests/runtime_resources.rs
|
||||
```
|
||||
|
||||
## Fichiers supprimés
|
||||
|
||||
```text
|
||||
aucun
|
||||
```
|
||||
|
||||
## Version Cargo
|
||||
|
||||
Conformément à `VER-ID-009` :
|
||||
|
||||
```text
|
||||
header Cargo.toml : 572 -> 573
|
||||
workspace.package.version : 0.3.14-pre.5 -> 0.3.14-pre.6
|
||||
```
|
||||
|
||||
Versions des fichiers modifiés :
|
||||
|
||||
```text
|
||||
continuity.rs : 5 -> 6
|
||||
lib.rs : 29 -> 30
|
||||
runtime_resources.rs : 31 -> 32
|
||||
unit_tests/runtime_resources.rs : 25 -> 26
|
||||
tests/hardening.rs : 29 -> 30
|
||||
tests/release_completeness.rs : 24 -> 25
|
||||
```
|
||||
|
||||
## Validation exécutée dans l'environnement de préparation
|
||||
|
||||
```text
|
||||
python3 scripts/audit_rust_workspace_rules.py : PASS
|
||||
python3 scripts/audit_markdown_tables.py README.md RULES.md ROADMAP.md CHANGELOG.md docs prompts crates deltas : PASS
|
||||
```
|
||||
|
||||
Les gates Cargo ne sont pas déclarés PASS dans l'environnement de préparation lorsqu'ils ne peuvent pas y être exécutés. Ils restent obligatoires côté opérateur avant `pre.007`.
|
||||
|
||||
## Prochaine tranche
|
||||
|
||||
`pre.007` : hydration des références connues avec `getTransaction observed`, coalescence globale existante et conservation de `Missing` comme obligation non résolue ; aucun `getTransaction = null` ne devra devenir une preuve d'absence.
|
||||
Reference in New Issue
Block a user