diff --git a/Cargo.toml b/Cargo.toml index 63f3145..c58f04a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,12 +1,12 @@ # file: Cargo.toml -# version: 572 +# version: 573 [workspace] resolver = "3" members = ["crates/ksp-app-backfill-desk", "crates/ksp-app-config-desk", "crates/ksp-app-solprices-desk", "crates/ksp-app-store-desk", "crates/ksp-app-wallet-desk", "crates/ksp-config-lib", "crates/ksp-core-lib", "crates/ksp-interface-lib", "crates/ksp-job-api", "crates/ksp-job-backfill-lib", "crates/ksp-logging-lib", "crates/ksp-offchain-transport-lib", "crates/ksp-onchain-transport-lib", "crates/ksp-program-api", "crates/ksp-raw-transaction-lib", "crates/ksp-store-api", "crates/ksp-store-lib", "crates/ksp-store-postgres-lib", "crates/ksp-wallet-lib", "crates/ksp-worker-api", "crates/ksp-worker-raw-transaction-ingest-lib"] [workspace.package] -version = "0.3.14-pre.5" +version = "0.3.14-pre.6" edition = "2024" license = "MIT" repository = "https://git.sasedev.com/Sasedev/khadhroony-solana-project" diff --git a/crates/ksp-worker-raw-transaction-ingest-lib/src/continuity.rs b/crates/ksp-worker-raw-transaction-ingest-lib/src/continuity.rs index 86ee013..1c03bdf 100644 --- a/crates/ksp-worker-raw-transaction-ingest-lib/src/continuity.rs +++ b/crates/ksp-worker-raw-transaction-ingest-lib/src/continuity.rs @@ -1,5 +1,8 @@ // file: crates/ksp-worker-raw-transaction-ingest-lib/src/continuity.rs -// version: 5 +// version: 6 + +/// Maximum number of slots admitted by one private continuity HTTP discovery window outside this module. +pub(crate) const MAX_RAW_TRANSACTION_INGEST_CONTINUITY_DISCOVERY_WINDOW_SLOTS: u64 = MAX_RAW_TRANSACTION_INGEST_REPAIR_DISCOVERY_WINDOW_SLOTS; /// Maximum number of proven redundant coverage epochs retained by one run-local continuity contract. const MAX_RAW_TRANSACTION_INGEST_COVERAGE_EPOCHS: usize = 256; @@ -627,8 +630,8 @@ impl crate::RawTransactionIngestContinuityContracts { return std::result::Result::Err(crate::runtime_error("continuity.coverage_scope_catalog_invalid")); } if MAX_RAW_TRANSACTION_INGEST_REPAIR_BLOCK_FETCH_IN_FLIGHT == 0 - || MAX_RAW_TRANSACTION_INGEST_REPAIR_DISCOVERY_WINDOW_SLOTS == 0 - || MAX_RAW_TRANSACTION_INGEST_REPAIR_DISCOVERY_WINDOW_SLOTS > MAX_RAW_TRANSACTION_INGEST_REPAIR_RANGE_SLOTS + || crate::MAX_RAW_TRANSACTION_INGEST_CONTINUITY_DISCOVERY_WINDOW_SLOTS == 0 + || crate::MAX_RAW_TRANSACTION_INGEST_CONTINUITY_DISCOVERY_WINDOW_SLOTS > MAX_RAW_TRANSACTION_INGEST_REPAIR_RANGE_SLOTS { return std::result::Result::Err(crate::runtime_error("continuity.repair_bounds_invalid")); } diff --git a/crates/ksp-worker-raw-transaction-ingest-lib/src/lib.rs b/crates/ksp-worker-raw-transaction-ingest-lib/src/lib.rs index bb0f069..887ce88 100644 --- a/crates/ksp-worker-raw-transaction-ingest-lib/src/lib.rs +++ b/crates/ksp-worker-raw-transaction-ingest-lib/src/lib.rs @@ -1,5 +1,5 @@ // file: crates/ksp-worker-raw-transaction-ingest-lib/src/lib.rs -// version: 29 +// version: 30 #![warn(missing_docs)] #![deny(unreachable_pub)] @@ -111,6 +111,8 @@ pub use self::snapshot::RawTransactionIngestSourceState; pub(crate) use self::admission::RawTransactionAdmission; /// Crate-private source-neutral ingress sent through the bounded central admission queue. pub(crate) use self::admission::RawTransactionIngress; +/// Maximum number of slots admitted by one private continuity HTTP discovery window. +pub(crate) use self::continuity::MAX_RAW_TRANSACTION_INGEST_CONTINUITY_DISCOVERY_WINDOW_SLOTS; /// Private source continuity-capability descriptor prepared without network or Store I/O. pub(crate) use self::continuity::RawTransactionIngestContinuityCapabilityDescriptor; /// Private run-local continuity aggregate containing capabilities, TargetCoverage and the gap ledger. diff --git a/crates/ksp-worker-raw-transaction-ingest-lib/src/runtime_resources.rs b/crates/ksp-worker-raw-transaction-ingest-lib/src/runtime_resources.rs index d257ed1..cfa23bd 100644 --- a/crates/ksp-worker-raw-transaction-ingest-lib/src/runtime_resources.rs +++ b/crates/ksp-worker-raw-transaction-ingest-lib/src/runtime_resources.rs @@ -1,5 +1,5 @@ // file: crates/ksp-worker-raw-transaction-ingest-lib/src/runtime_resources.rs -// version: 31 +// version: 32 use sha2::Digest; // rust-rules: trait-import @@ -36,6 +36,42 @@ const RAW_TRANSACTION_INGEST_YELLOWSTONE_FILTER_FINGERPRINT_DOMAIN: &[u8] = b"ks const RAW_TRANSACTION_INGEST_YELLOWSTONE_HTTP_PROTOCOL: &str = "yellowstone_http"; const RAW_TRANSACTION_INGEST_YELLOWSTONE_HTTP_SOURCE_KEY_DOMAIN: &[u8] = b"ksp.raw_transaction_ingest.yellowstone_http.source_key.v1\0"; +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum RawTransactionIngestHttpDiscoveryStrategy { + ClosedRange, + WithLimitBoundary, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +struct RawTransactionIngestHttpDiscoveryWindow { + produced_slots: std::vec::Vec, + proven_end_slot: std::option::Option, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +struct RawTransactionIngestHttpScanCapabilities { + get_block: bool, + get_blocks: bool, + get_blocks_with_limit: bool, + get_slot: bool, +} + +impl RawTransactionIngestHttpScanCapabilities { + const fn can_scan(self) -> bool { + return self.get_block && (self.get_blocks || (self.get_blocks_with_limit && self.get_slot)); + } + + fn strategy(self) -> ksp_core_lib::Result { + if self.get_block && self.get_blocks { + return std::result::Result::Ok(RawTransactionIngestHttpDiscoveryStrategy::ClosedRange); + } + if self.get_block && self.get_blocks_with_limit && self.get_slot { + return std::result::Result::Ok(RawTransactionIngestHttpDiscoveryStrategy::WithLimitBoundary); + } + return std::result::Result::Err(crate::runtime_error("continuity.http_scan_capability_incomplete")); + } +} + #[derive(Clone, Copy, Debug, Eq, PartialEq)] enum RawTransactionIngestSourceFamily { Block, @@ -1340,33 +1376,40 @@ impl crate::RawTransactionIngestHttpBlockPollingSource { }, }; if next_scan_slot <= current_tip { - let discovered = tokio::select! { - biased; - _ = stop_receiver.changed() => { - break; - } - result = self.http_pool.get_blocks_with_limit( - &self.polling_role, - next_scan_slot, - u64::from(self.max_discovered_blocks_per_cycle), - std::option::Option::Some(&context_config), - ) => result, - }; - let discovered = match discovered { - std::result::Result::Ok(value) => value, - std::result::Result::Err(error) => { - fault = std::option::Option::Some(source_transport_error(error.code())); + let configured_window_slots = + u64::from(self.max_discovered_blocks_per_cycle).min(crate::MAX_RAW_TRANSACTION_INGEST_CONTINUITY_DISCOVERY_WINDOW_SLOTS); + let window_offset = match configured_window_slots.checked_sub(1) { + std::option::Option::Some(value) => value, + std::option::Option::None => { + fault = std::option::Option::Some(crate::runtime_error("source.http_block_polling_window_invalid")); + break; + }, + }; + let candidate_end = match next_scan_slot.checked_add(window_offset) { + std::option::Option::Some(value) => value, + std::option::Option::None => u64::MAX, + }; + let window_end = current_tip.min(candidate_end); + let discovered = discover_http_block_window( + &self.http_pool, + &self.polling_role, + self.network.as_str(), + self.commitment, + next_scan_slot, + window_end, + &mut stop_receiver, + ) + .await; + let discovery = match discovered { + std::result::Result::Ok(std::option::Option::Some(value)) => value, + std::result::Result::Ok(std::option::Option::None) => break, + std::result::Result::Err(error) => { + fault = std::option::Option::Some(error); break; }, }; - let validated = validate_http_block_polling_discovery(next_scan_slot, discovered.as_slice()); - if let std::result::Result::Err(error) = validated { - fault = std::option::Option::Some(error); - break; - } - let discovered_count = discovered.len(); let mut blocked_by_null = false; - for slot in discovered { + for slot in discovery.produced_slots { let observed = tokio::select! { biased; _ = stop_receiver.changed() => { @@ -1424,16 +1467,9 @@ impl crate::RawTransactionIngestHttpBlockPollingSource { fault = std::option::Option::Some(error); break 'source; } - next_scan_slot = match slot.checked_add(1) { - std::option::Option::Some(value) => value, - std::option::Option::None => { - fault = std::option::Option::Some(crate::runtime_error("source.http_block_polling_slot_exhausted")); - break 'source; - }, - }; } - if !blocked_by_null && discovered_count < usize::from(self.max_discovered_blocks_per_cycle) && next_scan_slot <= current_tip { - next_scan_slot = match current_tip.checked_add(1) { + if !blocked_by_null && let std::option::Option::Some(proven_end_slot) = discovery.proven_end_slot { + next_scan_slot = match proven_end_slot.checked_add(1) { std::option::Option::Some(value) => value, std::option::Option::None => { fault = std::option::Option::Some(crate::runtime_error("source.http_block_polling_slot_exhausted")); @@ -2399,28 +2435,40 @@ async fn drain_live_source_tasks( }; } +fn http_role_scan_capabilities( + pool: &ksp_onchain_transport_lib::HttpTransportPool, + role: &ksp_onchain_transport_lib::HttpRoleName, + expected_cluster: &str, +) -> ksp_core_lib::Result { + let get_block = match http_role_supports_rpc_method(pool, role, "getBlock", expected_cluster) { + std::result::Result::Ok(value) => value, + std::result::Result::Err(error) => return std::result::Result::Err(error), + }; + let get_blocks = match http_role_supports_rpc_method(pool, role, "getBlocks", expected_cluster) { + std::result::Result::Ok(value) => value, + std::result::Result::Err(error) => return std::result::Result::Err(error), + }; + let get_blocks_with_limit = match http_role_supports_rpc_method(pool, role, "getBlocksWithLimit", expected_cluster) { + std::result::Result::Ok(value) => value, + std::result::Result::Err(error) => return std::result::Result::Err(error), + }; + let get_slot = match http_role_supports_rpc_method(pool, role, "getSlot", expected_cluster) { + std::result::Result::Ok(value) => value, + std::result::Result::Err(error) => return std::result::Result::Err(error), + }; + return std::result::Result::Ok(RawTransactionIngestHttpScanCapabilities { get_block, get_blocks, get_blocks_with_limit, get_slot }); +} + fn http_role_supports_repair_scan( pool: &ksp_onchain_transport_lib::HttpTransportPool, role: &ksp_onchain_transport_lib::HttpRoleName, expected_cluster: &str, ) -> ksp_core_lib::Result { - let has_get_block = match http_role_supports_rpc_method(pool, role, "getBlock", expected_cluster) { + let capabilities = match http_role_scan_capabilities(pool, role, expected_cluster) { std::result::Result::Ok(value) => value, std::result::Result::Err(error) => return std::result::Result::Err(error), }; - let has_get_blocks = match http_role_supports_rpc_method(pool, role, "getBlocks", expected_cluster) { - std::result::Result::Ok(value) => value, - std::result::Result::Err(error) => return std::result::Result::Err(error), - }; - let has_get_blocks_with_limit = match http_role_supports_rpc_method(pool, role, "getBlocksWithLimit", expected_cluster) { - std::result::Result::Ok(value) => value, - std::result::Result::Err(error) => return std::result::Result::Err(error), - }; - let has_get_slot = match http_role_supports_rpc_method(pool, role, "getSlot", expected_cluster) { - std::result::Result::Ok(value) => value, - std::result::Result::Err(error) => return std::result::Result::Err(error), - }; - return std::result::Result::Ok(has_get_block && has_get_slot && (has_get_blocks || has_get_blocks_with_limit)); + return std::result::Result::Ok(capabilities.can_scan()); } fn http_role_supports_rpc_method( @@ -2561,6 +2609,125 @@ fn http_role_supports_request_kind(role: &ksp_onchain_transport_lib::HttpEndpoin return role.request_kinds().iter().any(|kind| return kind.as_str() == "*" || kind.as_str() == request_kind); } +fn http_discovery_window_slot_count(start_slot: u64, end_slot: u64) -> ksp_core_lib::Result { + if end_slot < start_slot { + return std::result::Result::Err(crate::runtime_error("continuity.http_discovery_range_reversed")); + } + let slot_count = match end_slot.checked_sub(start_slot).and_then(|value| return value.checked_add(1)) { + std::option::Option::Some(value) => value, + std::option::Option::None => return std::result::Result::Err(crate::runtime_error("continuity.http_discovery_range_overflow")), + }; + if slot_count > crate::MAX_RAW_TRANSACTION_INGEST_CONTINUITY_DISCOVERY_WINDOW_SLOTS { + return std::result::Result::Err(crate::runtime_error("continuity.http_discovery_window_too_large")); + } + return std::result::Result::Ok(slot_count); +} + +fn validate_http_block_discovery_result( + start_slot: u64, + end_slot: u64, + strategy: RawTransactionIngestHttpDiscoveryStrategy, + current_tip: std::option::Option, + discovered: &[u64], +) -> ksp_core_lib::Result { + if let std::result::Result::Err(error) = http_discovery_window_slot_count(start_slot, end_slot) { + return std::result::Result::Err(error); + } + if let std::result::Result::Err(error) = validate_http_block_polling_discovery(start_slot, discovered) { + return std::result::Result::Err(error); + } + return match strategy { + RawTransactionIngestHttpDiscoveryStrategy::ClosedRange => { + if discovered.iter().any(|slot| return *slot > end_slot) { + return std::result::Result::Err(crate::runtime_error("continuity.http_closed_range_exceeded")); + } + std::result::Result::Ok(RawTransactionIngestHttpDiscoveryWindow { + produced_slots: discovered.to_vec(), + proven_end_slot: std::option::Option::Some(end_slot), + }) + }, + RawTransactionIngestHttpDiscoveryStrategy::WithLimitBoundary => { + let current_tip = match current_tip { + std::option::Option::Some(value) => value, + std::option::Option::None => return std::result::Result::Err(crate::runtime_error("continuity.http_discovery_tip_missing")), + }; + if current_tip < start_slot && discovered.is_empty() { + return std::result::Result::Ok(RawTransactionIngestHttpDiscoveryWindow { + produced_slots: std::vec::Vec::new(), + proven_end_slot: std::option::Option::None, + }); + } + let proven_end_slot = discovered.last().copied().map(|slot| return slot.min(end_slot)); + let produced_slots = discovered.iter().copied().take_while(|slot| return *slot <= end_slot).collect(); + std::result::Result::Ok(RawTransactionIngestHttpDiscoveryWindow { produced_slots, proven_end_slot }) + }, + }; +} + +async fn discover_http_block_window( + pool: &ksp_onchain_transport_lib::HttpTransportPool, + role: &ksp_onchain_transport_lib::HttpRoleName, + expected_cluster: &str, + commitment: ksp_onchain_transport_lib::SolanaCommitment, + start_slot: u64, + end_slot: u64, + stop_receiver: &mut tokio::sync::watch::Receiver, +) -> ksp_core_lib::Result> { + let slot_count = match http_discovery_window_slot_count(start_slot, end_slot) { + std::result::Result::Ok(value) => value, + std::result::Result::Err(error) => return std::result::Result::Err(error), + }; + let capabilities = match http_role_scan_capabilities(pool, role, expected_cluster) { + std::result::Result::Ok(value) => value, + std::result::Result::Err(error) => return std::result::Result::Err(error), + }; + let strategy = match capabilities.strategy() { + std::result::Result::Ok(value) => value, + std::result::Result::Err(error) => return std::result::Result::Err(error), + }; + let context_config = ksp_onchain_transport_lib::SolanaContextConfig::new(std::option::Option::Some(commitment), std::option::Option::None); + let mut current_tip = std::option::Option::None; + let discovered = match strategy { + RawTransactionIngestHttpDiscoveryStrategy::ClosedRange => tokio::select! { + biased; + _ = stop_receiver.changed() => { + return std::result::Result::Ok(std::option::Option::None); + } + result = pool.get_blocks(role, start_slot, std::option::Option::Some(end_slot), std::option::Option::Some(&context_config)) => result, + }, + RawTransactionIngestHttpDiscoveryStrategy::WithLimitBoundary => { + let tip = tokio::select! { + biased; + _ = stop_receiver.changed() => { + return std::result::Result::Ok(std::option::Option::None); + } + result = pool.get_slot(role, std::option::Option::Some(&context_config)) => result, + }; + let tip = match tip { + std::result::Result::Ok(value) => value, + std::result::Result::Err(error) => return std::result::Result::Err(source_transport_error(error.code())), + }; + current_tip = std::option::Option::Some(tip); + tokio::select! { + biased; + _ = stop_receiver.changed() => { + return std::result::Result::Ok(std::option::Option::None); + } + result = pool.get_blocks_with_limit(role, start_slot, slot_count, std::option::Option::Some(&context_config)) => result, + } + }, + }; + let discovered = match discovered { + std::result::Result::Ok(value) => value, + std::result::Result::Err(error) => return std::result::Result::Err(source_transport_error(error.code())), + }; + let validated = validate_http_block_discovery_result(start_slot, end_slot, strategy, current_tip, discovered.as_slice()); + return match validated { + std::result::Result::Ok(value) => std::result::Result::Ok(std::option::Option::Some(value)), + std::result::Result::Err(error) => std::result::Result::Err(error), + }; +} + fn validate_http_block_polling_discovery(next_scan_slot: u64, discovered: &[u64]) -> ksp_core_lib::Result<()> { let mut previous = std::option::Option::None; for slot in discovered { diff --git a/crates/ksp-worker-raw-transaction-ingest-lib/tests/hardening.rs b/crates/ksp-worker-raw-transaction-ingest-lib/tests/hardening.rs index 28761aa..ac49fc1 100644 --- a/crates/ksp-worker-raw-transaction-ingest-lib/tests/hardening.rs +++ b/crates/ksp-worker-raw-transaction-ingest-lib/tests/hardening.rs @@ -1,7 +1,7 @@ // file: crates/ksp-worker-raw-transaction-ingest-lib/tests/hardening.rs -// version: 29 +// version: 30 -//! External public, security, redaction and release-boundary hardening canaries through `v0.3.14-pre.005`. +//! External public, security, redaction and release-boundary hardening canaries through `v0.3.14-pre.006`. fn network(value: &'static str) -> std::option::Option { let result = ksp_store_lib::RawNetworkId::new(value); @@ -1090,3 +1090,29 @@ fn v0_3_14_pre_005_redundant_coverage_requires_exact_or_superset_proof_over_full } return; } + +#[test] +fn v0_3_14_pre_006_http_discovery_is_bounded_prefers_closed_range_and_never_uses_tip_alone_as_coverage() { + let resources = include_str!("../src/runtime_resources.rs"); + let root = include_str!("../src/lib.rs"); + for required in [ + "RawTransactionIngestHttpDiscoveryStrategy", + "RawTransactionIngestHttpDiscoveryWindow", + "RawTransactionIngestHttpScanCapabilities", + "ClosedRange", + "WithLimitBoundary", + "MAX_RAW_TRANSACTION_INGEST_CONTINUITY_DISCOVERY_WINDOW_SLOTS", + "pool.get_blocks(role, start_slot, std::option::Option::Some(end_slot)", + "pool.get_blocks_with_limit(role, start_slot, slot_count", + "self.http_pool.get_block_observed", + "discovered.last().copied().map(|slot| return slot.min(end_slot))", + ] { + assert!(resources.contains(required), "required pre.006 bounded HTTP discovery guard missing: {required}"); + } + assert!(resources.contains("http_role_scan_capabilities")); + assert!(resources.contains("http_role_supports_repair_scan")); + assert!(!resources.contains("ksp_job_backfill_lib::")); + assert!(!resources.contains("get_blocks_with_limit(role, start_slot, u64::MAX")); + assert!(!root.contains("repair"), "pre.006 leaked lower-case repair responsibility through crate root"); + return; +} diff --git a/crates/ksp-worker-raw-transaction-ingest-lib/tests/release_completeness.rs b/crates/ksp-worker-raw-transaction-ingest-lib/tests/release_completeness.rs index 1da5541..15a7b44 100644 --- a/crates/ksp-worker-raw-transaction-ingest-lib/tests/release_completeness.rs +++ b/crates/ksp-worker-raw-transaction-ingest-lib/tests/release_completeness.rs @@ -1,7 +1,7 @@ // file: crates/ksp-worker-raw-transaction-ingest-lib/tests/release_completeness.rs -// version: 24 +// version: 25 -//! Release-completeness canaries through the `v0.3.14-pre.005` redundant-coverage proof tranche. +//! Release-completeness canaries through the `v0.3.14-pre.006` bounded HTTP continuity-discovery tranche. #[test] fn pre_010_production_module_inventory_is_exact() -> std::io::Result<()> { @@ -282,3 +282,23 @@ fn v0_3_14_pre_005_redundant_coverage_canaries_are_present_without_public_surfac assert!(!root.contains("pub use self::continuity::RawTransactionIngestCoverageRelation")); return; } + +#[test] +fn v0_3_14_pre_006_bounded_http_discovery_canaries_are_present_without_backfill_or_public_surface_growth() { + let hardening = include_str!("hardening.rs"); + let resource_tests = include_str!("../unit_tests/runtime_resources.rs"); + let root = include_str!("../src/lib.rs"); + for required in [ + "v0_3_14_pre_006_http_scan_capabilities_prefer_closed_range_and_require_safe_fallback", + "v0_3_14_pre_006_http_scan_capability_detection_is_role_local_and_io_free", + "v0_3_14_pre_006_http_discovery_window_is_inclusive_bounded_and_overflow_safe", + "v0_3_14_pre_006_closed_range_discovery_proves_skipped_slots_only_inside_exact_window", + "v0_3_14_pre_006_with_limit_discovery_never_uses_tip_alone_as_tail_coverage_proof", + ] { + assert!(resource_tests.contains(required), "required pre.006 HTTP discovery canary missing: {required}"); + } + assert!(hardening.contains("v0_3_14_pre_006_http_discovery_is_bounded_prefers_closed_range_and_never_uses_tip_alone_as_coverage")); + assert!(!root.contains("pub use self::runtime_resources::RawTransactionIngestHttpDiscovery")); + assert!(!root.contains("backfill")); + return; +} diff --git a/crates/ksp-worker-raw-transaction-ingest-lib/unit_tests/runtime_resources.rs b/crates/ksp-worker-raw-transaction-ingest-lib/unit_tests/runtime_resources.rs index 620f0dc..34366ab 100644 --- a/crates/ksp-worker-raw-transaction-ingest-lib/unit_tests/runtime_resources.rs +++ b/crates/ksp-worker-raw-transaction-ingest-lib/unit_tests/runtime_resources.rs @@ -1,5 +1,5 @@ // file: crates/ksp-worker-raw-transaction-ingest-lib/unit_tests/runtime_resources.rs -// version: 25 +// version: 26 fn grpc_endpoint(cluster: &str) -> std::option::Option { return grpc_endpoint_with_identity(cluster, "yellowstone-fixture", "fixture-provider"); @@ -2968,6 +2968,178 @@ fn v0_3_13_pre_006_http_block_polling_discovery_never_moves_before_run_frontier_ return; } +#[test] +fn v0_3_14_pre_006_http_scan_capabilities_prefer_closed_range_and_require_safe_fallback() { + let exact = super::RawTransactionIngestHttpScanCapabilities { get_block: true, get_blocks: true, get_blocks_with_limit: true, get_slot: true }; + assert!(exact.can_scan()); + let exact_strategy = match exact.strategy() { + std::result::Result::Ok(value) => value, + std::result::Result::Err(_) => return, + }; + assert_eq!(exact_strategy, super::RawTransactionIngestHttpDiscoveryStrategy::ClosedRange); + let fallback = super::RawTransactionIngestHttpScanCapabilities { get_block: true, get_blocks: false, get_blocks_with_limit: true, get_slot: true }; + assert!(fallback.can_scan()); + let fallback_strategy = match fallback.strategy() { + std::result::Result::Ok(value) => value, + std::result::Result::Err(_) => return, + }; + assert_eq!(fallback_strategy, super::RawTransactionIngestHttpDiscoveryStrategy::WithLimitBoundary); + for incomplete in [ + super::RawTransactionIngestHttpScanCapabilities { get_block: false, get_blocks: true, get_blocks_with_limit: true, get_slot: true }, + super::RawTransactionIngestHttpScanCapabilities { get_block: true, get_blocks: false, get_blocks_with_limit: true, get_slot: false }, + super::RawTransactionIngestHttpScanCapabilities { get_block: true, get_blocks: false, get_blocks_with_limit: false, get_slot: true }, + ] { + assert!(!incomplete.can_scan()); + assert!(incomplete.strategy().is_err()); + } + return; +} + +#[test] +fn v0_3_14_pre_006_http_scan_capability_detection_is_role_local_and_io_free() { + let cases = [ + (&["get_block", "get_blocks"][..], std::option::Option::Some(super::RawTransactionIngestHttpDiscoveryStrategy::ClosedRange)), + ( + &["get_block", "get_blocks_with_limit", "get_slot"][..], + std::option::Option::Some(super::RawTransactionIngestHttpDiscoveryStrategy::WithLimitBoundary), + ), + (&["get_transaction"][..], std::option::Option::None), + ]; + for (request_kinds, expected_strategy) in cases { + let pool = match http_polling_pool_with_identity("devnet", "scan", "scan-fixture", "fixture-provider", request_kinds) { + std::option::Option::Some(value) => value, + std::option::Option::None => return, + }; + let capabilities = match super::http_role_scan_capabilities(&pool, &ksp_onchain_transport_lib::HttpRoleName::new("scan"), "devnet") { + std::result::Result::Ok(value) => value, + std::result::Result::Err(_) => return, + }; + match expected_strategy { + std::option::Option::Some(expected) => { + assert!(capabilities.can_scan()); + let actual = match capabilities.strategy() { + std::result::Result::Ok(value) => value, + std::result::Result::Err(_) => return, + }; + assert_eq!(actual, expected); + }, + std::option::Option::None => { + assert!(!capabilities.can_scan()); + assert!(capabilities.strategy().is_err()); + }, + } + } + return; +} + +#[test] +fn v0_3_14_pre_006_http_discovery_window_is_inclusive_bounded_and_overflow_safe() { + let single = match super::http_discovery_window_slot_count(100, 100) { + std::result::Result::Ok(value) => value, + std::result::Result::Err(_) => return, + }; + assert_eq!(single, 1); + let maximum = match super::http_discovery_window_slot_count(100, 611) { + std::result::Result::Ok(value) => value, + std::result::Result::Err(_) => return, + }; + assert_eq!(maximum, crate::MAX_RAW_TRANSACTION_INGEST_CONTINUITY_DISCOVERY_WINDOW_SLOTS); + assert!(super::http_discovery_window_slot_count(100, 612).is_err()); + assert!(super::http_discovery_window_slot_count(101, 100).is_err()); + assert!(super::http_discovery_window_slot_count(0, u64::MAX).is_err()); + return; +} + +#[test] +fn v0_3_14_pre_006_closed_range_discovery_proves_skipped_slots_only_inside_exact_window() { + let complete = match super::validate_http_block_discovery_result( + 100, + 105, + super::RawTransactionIngestHttpDiscoveryStrategy::ClosedRange, + std::option::Option::None, + &[100, 102, 105], + ) { + std::result::Result::Ok(value) => value, + std::result::Result::Err(_) => return, + }; + assert_eq!(complete.produced_slots, std::vec![100, 102, 105]); + assert_eq!(complete.proven_end_slot, std::option::Option::Some(105)); + let all_skipped = match super::validate_http_block_discovery_result( + 200, + 203, + super::RawTransactionIngestHttpDiscoveryStrategy::ClosedRange, + std::option::Option::None, + &[], + ) { + std::result::Result::Ok(value) => value, + std::result::Result::Err(_) => return, + }; + assert!(all_skipped.produced_slots.is_empty()); + assert_eq!(all_skipped.proven_end_slot, std::option::Option::Some(203)); + assert!( + super::validate_http_block_discovery_result( + 100, + 105, + super::RawTransactionIngestHttpDiscoveryStrategy::ClosedRange, + std::option::Option::None, + &[100, 106], + ) + .is_err() + ); + return; +} + +#[test] +fn v0_3_14_pre_006_with_limit_discovery_never_uses_tip_alone_as_tail_coverage_proof() { + let crossed = match super::validate_http_block_discovery_result( + 100, + 105, + super::RawTransactionIngestHttpDiscoveryStrategy::WithLimitBoundary, + std::option::Option::Some(110), + &[100, 103, 107], + ) { + std::result::Result::Ok(value) => value, + std::result::Result::Err(_) => return, + }; + assert_eq!(crossed.produced_slots, std::vec![100, 103]); + assert_eq!(crossed.proven_end_slot, std::option::Option::Some(105)); + let partial = match super::validate_http_block_discovery_result( + 100, + 105, + super::RawTransactionIngestHttpDiscoveryStrategy::WithLimitBoundary, + std::option::Option::Some(110), + &[100, 103], + ) { + std::result::Result::Ok(value) => value, + std::result::Result::Err(_) => return, + }; + assert_eq!(partial.produced_slots, std::vec![100, 103]); + assert_eq!(partial.proven_end_slot, std::option::Option::Some(103)); + let tip_only = match super::validate_http_block_discovery_result( + 100, + 105, + super::RawTransactionIngestHttpDiscoveryStrategy::WithLimitBoundary, + std::option::Option::Some(110), + &[], + ) { + std::result::Result::Ok(value) => value, + std::result::Result::Err(_) => return, + }; + assert!(tip_only.produced_slots.is_empty()); + assert_eq!(tip_only.proven_end_slot, std::option::Option::None); + assert!( + super::validate_http_block_discovery_result( + 100, + 105, + super::RawTransactionIngestHttpDiscoveryStrategy::WithLimitBoundary, + std::option::Option::None, + &[100, 103, 107], + ) + .is_err() + ); + return; +} + #[test] fn v0_3_13_pre_006_http_block_polling_qualifies_legacy_v0_v1_and_rejects_ambiguous_or_future_versions() { let cases = [ diff --git a/deltas/0.3.14/pre.006.md b/deltas/0.3.14/pre.006.md new file mode 100644 index 0000000..b80d882 --- /dev/null +++ b/deltas/0.3.14/pre.006.md @@ -0,0 +1,274 @@ + + + +# Delta `0.3.14-pre.006` — discovery HTTP bornée pour continuité + +## Base requise + +```text +0.3.14-pre.005 +workspace.package.version = 0.3.14-pre.5 +deltas/0.3.14/pre.005.md présent +``` + +## Gate de la base + +Le gate opérateur de `0.3.14-pre.005` est validé avant ouverture de cette tranche : + +```text +cargo fmt --all : PASS +cargo fmt --all -- --check : PASS +audit Rust workspace rules : PASS +audit Markdown tables : PASS +cargo check --workspace : PASS +cargo clippy --workspace --all-targets --all-features -- -D warnings : PASS +cargo test -p ksp-worker-raw-transaction-ingest-lib --all-targets --all-features : PASS +``` + +Le gate Worker comprend notamment : + +```text +124 unit tests : PASS +cross_layer_completeness : 8 PASS +dependency_boundary : 19 PASS +hardening : 30 PASS +public_api : 20 PASS +release_completeness : 7 PASS +``` + +## Objectif + +Implémenter strictement la tranche `pre.006` du plan `035` : + +```text +détecter les primitives HTTP de scan réellement disponibles sur le rôle same-network +borner chaque fenêtre de discovery à 512 slots inclusifs +préférer getBlocks(start,end) lorsqu'il est réellement supporté +utiliser getBlocksWithLimit uniquement avec une preuve de progression issue de sa propre réponse +ne jamais utiliser getSlot seul comme preuve de queue skipped +réutiliser getBlock observed pour les slots explicitement produits +conserver getBlock = null comme obstacle, jamais comme slot skipped +ne pas élargir un scope filtré vers FullLedgerTransactions +ne pas modifier encore la politique de supervisor ou de health +ne pas introduire de logique caller-driven de backfill +``` + +## Capability HTTP run-wide + +`runtime_resources.rs` matérialise maintenant un inventaire privé : + +```text +RawTransactionIngestHttpScanCapabilities + get_block + get_blocks + get_blocks_with_limit + get_slot +``` + +Le choix de primitive est conservative : + +```text +getBlock + getBlocks + -> ClosedRange + +getBlock + getBlocksWithLimit + getSlot + -> WithLimitBoundary + +autres combinaisons + -> scan indisponible +``` + +`ClosedRange` est toujours préféré lorsque les deux stratégies sont disponibles. + +La détection reste fondée sur le snapshot Transport du rôle exact et du cluster attendu. La seule présence d'un pool HTTP ne crée donc aucune capability implicite. + +## Fenêtres bornées + +La borne canonique de `continuity.rs` reste : + +```text +MAX_RAW_TRANSACTION_INGEST_REPAIR_DISCOVERY_WINDOW_SLOTS = 512 +``` + +Un alias crate-wide de même valeur est ajouté uniquement pour que `runtime_resources.rs` réutilise cette borne sans la dupliquer : + +```text +MAX_RAW_TRANSACTION_INGEST_CONTINUITY_DISCOVERY_WINDOW_SLOTS +``` + +Une fenêtre : + +```text +est inclusive +rejette end < start +rejette les overflows +rejette plus de 512 slots +``` + +La source HTTP polling productive réutilise désormais cette primitive. Sa valeur `max_discovered_blocks_per_cycle` reste une borne supérieure ; une fenêtre productive ne dépasse jamais le minimum entre cette valeur et `512` slots. + +## Stratégie `ClosedRange` + +Lorsque `getBlocks` est supporté, le Worker appelle : + +```text +getBlocks(start_slot, end_slot, commitment) +``` + +Une réponse réussie constitue une preuve fermée de la fenêtre RPC demandée. Les slots absents de la liste sont alors qualifiables comme `skipped/non-produced` uniquement à l'intérieur de cette fenêtre. + +Tout slot retourné au-delà de `end_slot` est rejeté comme violation d'invariant. + +## Stratégie `WithLimitBoundary` + +Lorsque seul le chemin `getBlocksWithLimit` qualifié est disponible : + +```text +getSlot(commitment) +getBlocksWithLimit(start_slot, window_slot_count, commitment) +``` + +`getSlot` indique uniquement la position du cluster observée par ce rôle. Il ne ferme jamais à lui seul la queue de la fenêtre. + +La preuve avance seulement jusqu'au dernier slot produit effectivement énuméré par la réponse `getBlocksWithLimit` : + +```text +last_discovered < end_slot + -> preuve partielle jusqu'à last_discovered + +last_discovered >= end_slot + -> fenêtre cible entièrement prouvée + +aucun slot découvert + -> aucune preuve de tail, même si getSlot >= end_slot +``` + +Ainsi, une queue constituée uniquement de slots skipped n'est pas inventée à partir de deux réponses pouvant provenir de routes HTTP différentes. La fenêtre restera en attente jusqu'à ce qu'une primitive plus forte ou un bloc ultérieur fournisse une borne observable. + +## `getBlock observed` + +Chaque slot explicitement produit continue à passer par : + +```text +HttpTransportPool::get_block_observed +``` + +Les règles existantes restent inchangées : + +```text +Some(block) + -> projection Common RAW puis admission centrale + +None + -> la progression se bloque sur ce slot + le slot n'est pas reclassé skipped +``` + +Aucun second pipeline de persistence n'est ajouté. + +## Scope et intégration + +Cette tranche utilise immédiatement la primitive bornée uniquement dans `RawTransactionIngestHttpBlockPollingSource`, dont le scope est déjà `FullLedgerTransactions`. + +Pour Yellowstone, Standard Logs et Helius Transaction, la détection run-wide de capability HTTP demeure disponible dans les continuity descriptors, mais aucun scan full-block n'est déclenché ici. Leur activation dépendra de la preuve de scope et de la réconciliation des tranches suivantes. + +`Standard Block` conserve l'absence de pool HTTP attaché. + +La politique de terminalité des incidents `pre.003/pre.004` n'est pas encore remplacée. `pre.008` reste responsable de l'intégration supervisor/gap-ledger. + +## Tests ajoutés + +Les unit tests Worker couvrent : + +```text +préférence ClosedRange sur fallback +capability fallback complète obligatoire +capability détectée par rôle sans I/O +fenêtre inclusive 1..512 +rejet 513 slots, range inversé et overflow +ClosedRange vide comme preuve de slots skipped dans la fenêtre exacte +rejet d'un résultat ClosedRange au-delà de end_slot +fallback dépassant end_slot et filtrage de la cible +fallback partiel ne prouvant que jusqu'au dernier slot énuméré +getSlot seul + réponse vide ne prouvant aucune queue +absence de getSlot dans le fallback rejetée +``` + +Les canaris `hardening` et `release_completeness` garantissent en plus : + +```text +présence des deux stratégies HTTP +usage borné de getBlocks/getBlocksWithLimit +réutilisation de getBlock observed +absence de dépendance Job Backfill +absence de surface publique de discovery +absence de responsabilité repair lower-case dans la crate-root historique +``` + +## Hors périmètre inchangé + +```text +aucun moteur de reconciliation supervisor +aucun changement health public +aucune hydration repair getTransaction dans cette tranche +aucune source/provider supplémentaire +aucun EARLY/shred adapter +aucun backfill historique caller-driven +``` + +## Fichiers ajoutés + +```text +deltas/0.3.14/pre.006.md +``` + +## Fichiers modifiés + +```text +Cargo.toml +crates/ksp-worker-raw-transaction-ingest-lib/src/continuity.rs +crates/ksp-worker-raw-transaction-ingest-lib/src/lib.rs +crates/ksp-worker-raw-transaction-ingest-lib/src/runtime_resources.rs +crates/ksp-worker-raw-transaction-ingest-lib/tests/hardening.rs +crates/ksp-worker-raw-transaction-ingest-lib/tests/release_completeness.rs +crates/ksp-worker-raw-transaction-ingest-lib/unit_tests/runtime_resources.rs +``` + +## Fichiers supprimés + +```text +aucun +``` + +## Version Cargo + +Conformément à `VER-ID-009` : + +```text +header Cargo.toml : 572 -> 573 +workspace.package.version : 0.3.14-pre.5 -> 0.3.14-pre.6 +``` + +Versions des fichiers modifiés : + +```text +continuity.rs : 5 -> 6 +lib.rs : 29 -> 30 +runtime_resources.rs : 31 -> 32 +unit_tests/runtime_resources.rs : 25 -> 26 +tests/hardening.rs : 29 -> 30 +tests/release_completeness.rs : 24 -> 25 +``` + +## Validation exécutée dans l'environnement de préparation + +```text +python3 scripts/audit_rust_workspace_rules.py : PASS +python3 scripts/audit_markdown_tables.py README.md RULES.md ROADMAP.md CHANGELOG.md docs prompts crates deltas : PASS +``` + +Les gates Cargo ne sont pas déclarés PASS dans l'environnement de préparation lorsqu'ils ne peuvent pas y être exécutés. Ils restent obligatoires côté opérateur avant `pre.007`. + +## Prochaine tranche + +`pre.007` : hydration des références connues avec `getTransaction observed`, coalescence globale existante et conservation de `Missing` comme obligation non résolue ; aucun `getTransaction = null` ne devra devenir une preuve d'absence.