diff --git a/Cargo.toml b/Cargo.toml index 8f3b925..91d791e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,12 +1,12 @@ # file: Cargo.toml -# version: 524 +# version: 525 [workspace] resolver = "3" members = ["crates/ksp-app-backfill-desk", "crates/ksp-app-config-desk", "crates/ksp-app-solprices-desk", "crates/ksp-app-store-desk", "crates/ksp-app-wallet-desk", "crates/ksp-config-lib", "crates/ksp-core-lib", "crates/ksp-interface-lib", "crates/ksp-job-api", "crates/ksp-job-backfill-lib", "crates/ksp-logging-lib", "crates/ksp-offchain-transport-lib", "crates/ksp-onchain-transport-lib", "crates/ksp-program-api", "crates/ksp-raw-transaction-lib", "crates/ksp-store-api", "crates/ksp-store-lib", "crates/ksp-store-postgres-lib", "crates/ksp-wallet-lib", "crates/ksp-worker-api", "crates/ksp-worker-raw-transaction-ingest-lib"] [workspace.package] -version = "0.3.12-pre.4.fix.1" +version = "0.3.12-pre.5" edition = "2024" license = "MIT" repository = "https://git.sasedev.com/Sasedev/khadhroony-solana-project" diff --git a/crates/ksp-worker-raw-transaction-ingest-lib/src/lib.rs b/crates/ksp-worker-raw-transaction-ingest-lib/src/lib.rs index 40b62c0..a37739c 100644 --- a/crates/ksp-worker-raw-transaction-ingest-lib/src/lib.rs +++ b/crates/ksp-worker-raw-transaction-ingest-lib/src/lib.rs @@ -1,5 +1,5 @@ // file: crates/ksp-worker-raw-transaction-ingest-lib/src/lib.rs -// version: 11 +// version: 12 #![warn(missing_docs)] #![deny(unreachable_pub)] @@ -12,8 +12,9 @@ //! owns bounded source-neutral admission, common RAW canonicalization/assembly and backend-neutral //! Store persistence in normal mode plus concrete latest-value snapshots projected onto Worker API; //! the first validated Yellowstone/HTTP runtime-resource contract also closes the deterministic hydration -//! qualification path in test configuration. Transaction/TransactionStatus signal projection and HTTP hydration -//! remain inactive in production until the productive source task is wired; no live stream is opened yet. +//! qualification path in test configuration. Transaction/TransactionStatus/Block signal projection, BlockMeta/Slot +//! continuity projection and HTTP hydration remain inactive in production until the productive source task is wired; +//! no live stream is opened yet. mod admission; mod error; diff --git a/crates/ksp-worker-raw-transaction-ingest-lib/src/runtime_resources.rs b/crates/ksp-worker-raw-transaction-ingest-lib/src/runtime_resources.rs index cead8ca..153b824 100644 --- a/crates/ksp-worker-raw-transaction-ingest-lib/src/runtime_resources.rs +++ b/crates/ksp-worker-raw-transaction-ingest-lib/src/runtime_resources.rs @@ -1,5 +1,5 @@ // file: crates/ksp-worker-raw-transaction-ingest-lib/src/runtime_resources.rs -// version: 5 +// version: 6 #[cfg(test)] use sha2::Digest; // rust-rules: trait-import @@ -14,10 +14,64 @@ const RAW_TRANSACTION_INGEST_YELLOWSTONE_HTTP_SOURCE_KEY_DOMAIN: &[u8] = b"ksp.r #[cfg(test)] #[derive(Clone, Copy, Debug, Eq, PartialEq)] enum RawTransactionIngestSourceFamily { + Block, Transaction, TransactionStatus, } +#[cfg(test)] +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum RawTransactionIngestContinuityFamily { + BlockMeta, + Slot, +} + +#[cfg(test)] +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum RawTransactionIngestContinuityStatus { + Completed, + Confirmed, + CreatedBank, + Dead, + Finalized, + FirstShredReceived, + Processed, +} + +#[cfg(test)] +#[derive(Clone, Eq, PartialEq)] +struct RawTransactionIngestContinuitySignal { + created_at: std::option::Option, + family: RawTransactionIngestContinuityFamily, + matched_filter_count: usize, + matched_filter_fingerprint: [u8; 32], + matched_filter_id: std::option::Option, + network: ksp_store_lib::RawNetworkId, + parent_slot: std::option::Option, + route: RawTransactionIngestSourceRoute, + slot: u64, + status: std::option::Option, +} + +#[cfg(test)] +impl std::fmt::Debug for RawTransactionIngestContinuitySignal { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + return formatter + .debug_struct("RawTransactionIngestContinuitySignal") + .field("created_at", &self.created_at) + .field("family", &self.family) + .field("matched_filter_count", &self.matched_filter_count) + .field("matched_filter_fingerprint_bytes", &self.matched_filter_fingerprint.len()) + .field("has_direct_filter_id", &self.matched_filter_id.is_some()) + .field("network", &self.network) + .field("parent_slot", &self.parent_slot) + .field("route", &self.route) + .field("slot", &self.slot) + .field("status", &self.status) + .finish_non_exhaustive(); + } +} + #[derive(Clone, Debug, Eq, PartialEq)] struct RawTransactionIngestSourceRoute { endpoint_id: ksp_store_lib::RawProvenanceCode, @@ -141,6 +195,115 @@ impl RawTransactionIngestYellowstoneSignalView for ksp_onchain_transport_lib::Ye } } +#[cfg(test)] +trait RawTransactionIngestYellowstoneBlockView { + fn created_at(&self) -> std::option::Option; + + fn filters(&self) -> &[ksp_onchain_transport_lib::YellowstoneSubscribeFilterName]; + + fn slot(&self) -> u64; + + fn transaction_count(&self) -> usize; + + fn transaction_identity(&self, position: usize) -> ksp_core_lib::Result<(ksp_onchain_transport_lib::YellowstoneTransactionSignature, u64)>; +} + +#[cfg(test)] +impl RawTransactionIngestYellowstoneBlockView for ksp_onchain_transport_lib::YellowstoneBlockUpdate { + fn created_at(&self) -> std::option::Option { + return ksp_onchain_transport_lib::YellowstoneBlockUpdate::created_at(self); + } + + fn filters(&self) -> &[ksp_onchain_transport_lib::YellowstoneSubscribeFilterName] { + return ksp_onchain_transport_lib::YellowstoneBlockUpdate::filters(self); + } + + fn slot(&self) -> u64 { + return ksp_onchain_transport_lib::YellowstoneBlockUpdate::slot(self); + } + + fn transaction_count(&self) -> usize { + return ksp_onchain_transport_lib::YellowstoneBlockUpdate::transactions(self).len(); + } + + fn transaction_identity(&self, position: usize) -> ksp_core_lib::Result<(ksp_onchain_transport_lib::YellowstoneTransactionSignature, u64)> { + let transaction = match ksp_onchain_transport_lib::YellowstoneBlockUpdate::transactions(self).get(position) { + std::option::Option::Some(value) => value, + std::option::Option::None => return std::result::Result::Err(crate::runtime_error("source.block_transaction_missing")), + }; + return std::result::Result::Ok((transaction.signature(), transaction.index())); + } +} + +#[cfg(test)] +trait RawTransactionIngestYellowstoneContinuityView { + fn created_at(&self) -> std::option::Option; + + fn family(&self) -> RawTransactionIngestContinuityFamily; + + fn filters(&self) -> &[ksp_onchain_transport_lib::YellowstoneSubscribeFilterName]; + + fn parent_slot(&self) -> std::option::Option; + + fn slot(&self) -> u64; + + fn status(&self) -> std::option::Option; +} + +#[cfg(test)] +impl RawTransactionIngestYellowstoneContinuityView for ksp_onchain_transport_lib::YellowstoneBlockMetaUpdate { + fn created_at(&self) -> std::option::Option { + return ksp_onchain_transport_lib::YellowstoneBlockMetaUpdate::created_at(self); + } + + fn family(&self) -> RawTransactionIngestContinuityFamily { + return RawTransactionIngestContinuityFamily::BlockMeta; + } + + fn filters(&self) -> &[ksp_onchain_transport_lib::YellowstoneSubscribeFilterName] { + return ksp_onchain_transport_lib::YellowstoneBlockMetaUpdate::filters(self); + } + + fn parent_slot(&self) -> std::option::Option { + return std::option::Option::Some(ksp_onchain_transport_lib::YellowstoneBlockMetaUpdate::parent_slot(self)); + } + + fn slot(&self) -> u64 { + return ksp_onchain_transport_lib::YellowstoneBlockMetaUpdate::slot(self); + } + + fn status(&self) -> std::option::Option { + return std::option::Option::None; + } +} + +#[cfg(test)] +impl RawTransactionIngestYellowstoneContinuityView for ksp_onchain_transport_lib::YellowstoneSlotUpdate { + fn created_at(&self) -> std::option::Option { + return ksp_onchain_transport_lib::YellowstoneSlotUpdate::created_at(self); + } + + fn family(&self) -> RawTransactionIngestContinuityFamily { + return RawTransactionIngestContinuityFamily::Slot; + } + + fn filters(&self) -> &[ksp_onchain_transport_lib::YellowstoneSubscribeFilterName] { + return ksp_onchain_transport_lib::YellowstoneSlotUpdate::filters(self); + } + + fn parent_slot(&self) -> std::option::Option { + return ksp_onchain_transport_lib::YellowstoneSlotUpdate::parent(self); + } + + fn slot(&self) -> u64 { + return ksp_onchain_transport_lib::YellowstoneSlotUpdate::slot(self); + } + + fn status(&self) -> std::option::Option { + return std::option::Option::Some(map_yellowstone_slot_status(ksp_onchain_transport_lib::YellowstoneSlotUpdate::status(self))); + } +} + #[cfg(test)] impl std::convert::From<(&crate::RawTransactionIngestYellowstoneSource, &ksp_onchain_transport_lib::YellowstoneTransactionUpdate)> for RawTransactionIngestSourceSignal @@ -234,6 +397,8 @@ impl std::fmt::Debug for crate::RawTransactionIngestYellowstoneSource { .field("transaction_filter_count", &self.subscribe_request.transaction_filter_count()) .field("transaction_status_filter_count", &self.subscribe_request.transaction_status_filter_count()) .field("block_filter_count", &self.subscribe_request.block_filter_count()) + .field("blocks_meta_filter_count", &self.subscribe_request.blocks_meta_filter_count()) + .field("slot_filter_count", &self.subscribe_request.slot_filter_count()) .field("commitment", &self.subscribe_request.commitment()) .field("has_from_slot", &self.subscribe_request.from_slot().is_some()) .field("hydration_role", &self.hydration_role.as_str()) @@ -361,6 +526,80 @@ fn matched_filter_fingerprint(filters: &[ksp_onchain_transport_lib::YellowstoneS return hasher.finalize().into(); } +#[cfg(test)] +fn project_yellowstone_block_signals( + source: &crate::RawTransactionIngestYellowstoneSource, + update: &T, +) -> ksp_core_lib::Result> { + let filters = update.filters(); + let matched_filter_count = filters.len(); + let matched_filter_fingerprint = matched_filter_fingerprint(filters); + let matched_filter_id = matched_filter_direct_id(filters); + let created_at = map_yellowstone_source_timestamp(update.created_at()); + let mut signals = std::vec::Vec::with_capacity(update.transaction_count()); + for position in 0..update.transaction_count() { + let (signature, transaction_index) = match update.transaction_identity(position) { + std::result::Result::Ok(value) => value, + std::result::Result::Err(error) => return std::result::Result::Err(error), + }; + signals.push(RawTransactionIngestSourceSignal { + created_at, + family: RawTransactionIngestSourceFamily::Block, + matched_filter_count, + matched_filter_fingerprint, + matched_filter_id: matched_filter_id.clone(), + network: source.network.clone(), + route: source.route.clone(), + signature: ksp_store_lib::RawTransactionSignature::new(signature.into_bytes()), + slot: update.slot(), + transaction_index: std::option::Option::Some(transaction_index), + }); + } + return std::result::Result::Ok(signals); +} + +#[cfg(test)] +fn project_yellowstone_continuity_signal( + source: &crate::RawTransactionIngestYellowstoneSource, + update: &T, +) -> RawTransactionIngestContinuitySignal { + let filters = update.filters(); + return RawTransactionIngestContinuitySignal { + created_at: map_yellowstone_source_timestamp(update.created_at()), + family: update.family(), + matched_filter_count: filters.len(), + matched_filter_fingerprint: matched_filter_fingerprint(filters), + matched_filter_id: matched_filter_direct_id(filters), + network: source.network.clone(), + parent_slot: update.parent_slot(), + route: source.route.clone(), + slot: update.slot(), + status: update.status(), + }; +} + +#[cfg(test)] +fn map_yellowstone_slot_status(status: ksp_onchain_transport_lib::YellowstoneSlotStatus) -> RawTransactionIngestContinuityStatus { + return match status { + ksp_onchain_transport_lib::YellowstoneSlotStatus::Processed => RawTransactionIngestContinuityStatus::Processed, + ksp_onchain_transport_lib::YellowstoneSlotStatus::Confirmed => RawTransactionIngestContinuityStatus::Confirmed, + ksp_onchain_transport_lib::YellowstoneSlotStatus::Finalized => RawTransactionIngestContinuityStatus::Finalized, + ksp_onchain_transport_lib::YellowstoneSlotStatus::FirstShredReceived => RawTransactionIngestContinuityStatus::FirstShredReceived, + ksp_onchain_transport_lib::YellowstoneSlotStatus::Completed => RawTransactionIngestContinuityStatus::Completed, + ksp_onchain_transport_lib::YellowstoneSlotStatus::CreatedBank => RawTransactionIngestContinuityStatus::CreatedBank, + ksp_onchain_transport_lib::YellowstoneSlotStatus::Dead => RawTransactionIngestContinuityStatus::Dead, + }; +} + +#[cfg(test)] +fn map_yellowstone_source_timestamp( + created_at: std::option::Option, +) -> std::option::Option { + return created_at.map(|value| { + return RawTransactionIngestSourceTimestamp { nanos: value.nanos(), seconds: value.seconds() }; + }); +} + #[cfg(test)] fn project_yellowstone_signal( source: &crate::RawTransactionIngestYellowstoneSource, @@ -543,6 +782,7 @@ fn build_hydration_provenance( #[cfg(test)] fn hydration_method_code(family: RawTransactionIngestSourceFamily) -> &'static str { return match family { + RawTransactionIngestSourceFamily::Block => "block_get_transaction", RawTransactionIngestSourceFamily::Transaction => "transaction_get_transaction", RawTransactionIngestSourceFamily::TransactionStatus => "status_get_transaction", }; diff --git a/crates/ksp-worker-raw-transaction-ingest-lib/tests/dependency_boundary.rs b/crates/ksp-worker-raw-transaction-ingest-lib/tests/dependency_boundary.rs index dd4031e..a024989 100644 --- a/crates/ksp-worker-raw-transaction-ingest-lib/tests/dependency_boundary.rs +++ b/crates/ksp-worker-raw-transaction-ingest-lib/tests/dependency_boundary.rs @@ -1,5 +1,5 @@ // file: crates/ksp-worker-raw-transaction-ingest-lib/tests/dependency_boundary.rs -// version: 11 +// version: 12 //! Dependency firewall canaries for the RAW transaction ingest Worker foundation. @@ -144,6 +144,38 @@ fn v0_3_12_pre_004_hydration_contract_uses_only_transport_facade_common_raw_and_ return; } +#[test] +fn v0_3_12_pre_005_block_and_continuity_adapters_remain_private_offline_and_transport_facade_only() { + let root = include_str!("../src/lib.rs"); + let resources = include_str!("../src/runtime_resources.rs"); + for required in [ + "RawTransactionIngestYellowstoneBlockView", + "YellowstoneBlockUpdate", + "project_yellowstone_block_signals", + "RawTransactionIngestYellowstoneContinuityView", + "YellowstoneBlockMetaUpdate", + "YellowstoneSlotUpdate", + "RawTransactionIngestContinuitySignal", + "block_get_transaction", + ] { + assert!(resources.contains(required), "required pre.005 private adapter contract missing: {required}"); + } + for forbidden in [ + "open_standard_subscribe", + "next_update", + "get_block_observed", + "ksp_config_lib::", + "ksp_job_backfill_lib::", + "ksp_store_postgres_lib::", + "reqwest::", + "tonic::", + "yellowstone_grpc_proto::", + ] { + assert!(!resources.contains(forbidden) && !root.contains(forbidden), "pre.005 crossed a forbidden boundary: {forbidden}"); + } + return; +} + #[test] fn v0_3_12_pre_002_source_surface_hardens_shutdown_and_faults_without_backend_or_premature_live_io() { let root = include_str!("../src/lib.rs"); diff --git a/crates/ksp-worker-raw-transaction-ingest-lib/tests/hardening.rs b/crates/ksp-worker-raw-transaction-ingest-lib/tests/hardening.rs index dd2c376..9307910 100644 --- a/crates/ksp-worker-raw-transaction-ingest-lib/tests/hardening.rs +++ b/crates/ksp-worker-raw-transaction-ingest-lib/tests/hardening.rs @@ -1,5 +1,5 @@ // file: crates/ksp-worker-raw-transaction-ingest-lib/tests/hardening.rs -// version: 6 +// version: 7 //! External public, security, redaction and release-boundary hardening canaries for `pre.010`. @@ -338,6 +338,49 @@ fn v0_3_12_pre_004_hydration_provenance_and_remote_material_are_bounded_and_reda return; } +#[test] +fn v0_3_12_pre_005_block_and_continuity_shapes_are_bounded_redacted_and_raw_separated() { + let resources = include_str!("../src/runtime_resources.rs"); + for required in [ + "impl RawTransactionIngestYellowstoneBlockView for ksp_onchain_transport_lib::YellowstoneBlockUpdate", + "impl RawTransactionIngestYellowstoneContinuityView for ksp_onchain_transport_lib::YellowstoneBlockMetaUpdate", + "impl RawTransactionIngestYellowstoneContinuityView for ksp_onchain_transport_lib::YellowstoneSlotUpdate", + "RawTransactionIngestSourceFamily::Block", + "project_yellowstone_block_signals", + "project_yellowstone_continuity_signal", + "RawTransactionIngestContinuityStatus::Dead", + "block_get_transaction", + ] { + assert!(resources.contains(required), "required pre.005 bounded adapter guard missing: {required}"); + } + let continuity_struct = match resources.split_once("struct RawTransactionIngestContinuitySignal {") { + std::option::Option::Some((_, tail)) => match tail.split_once("impl std::fmt::Debug for RawTransactionIngestContinuitySignal") { + std::option::Option::Some((value, _)) => value, + std::option::Option::None => "", + }, + std::option::Option::None => "", + }; + for required in [ + "created_at:", + "family:", + "matched_filter_count:", + "matched_filter_fingerprint:", + "matched_filter_id:", + "network:", + "parent_slot:", + "route:", + "slot:", + "status:", + ] { + assert!(continuity_struct.contains(required), "required continuity-only field missing: {required}"); + } + for forbidden in ["signature:", "transaction:", "meta:", "payload:", "body:", "error:", "dead_error:"] { + assert!(!continuity_struct.contains(forbidden), "RAW/remote material leaked into continuity-only signal: {forbidden}"); + } + assert!(!resources.contains("YellowstoneSlotUpdate::dead_error")); + return; +} + #[test] fn v0_3_12_pre_002_runtime_resource_contract_performs_no_live_io_or_source_spawn() { let resources = include_str!("../src/runtime_resources.rs"); diff --git a/crates/ksp-worker-raw-transaction-ingest-lib/tests/release_completeness.rs b/crates/ksp-worker-raw-transaction-ingest-lib/tests/release_completeness.rs index e021813..e859bc1 100644 --- a/crates/ksp-worker-raw-transaction-ingest-lib/tests/release_completeness.rs +++ b/crates/ksp-worker-raw-transaction-ingest-lib/tests/release_completeness.rs @@ -1,5 +1,5 @@ // file: crates/ksp-worker-raw-transaction-ingest-lib/tests/release_completeness.rs -// version: 4 +// version: 5 //! Release-completeness canaries through the `pre.010` public/release/security hardening tranche. @@ -100,6 +100,7 @@ fn pre_010_external_hardening_suite_is_present_and_scoped() { "v0_3_12_pre_002_manifest_dependency_surface_opens_only_transport_and_remains_backend_neutral", "v0_3_12_pre_003_private_signal_debug_and_shape_do_not_expose_signature_filters_or_payload", "v0_3_12_pre_004_hydration_provenance_and_remote_material_are_bounded_and_redacted", + "v0_3_12_pre_005_block_and_continuity_shapes_are_bounded_redacted_and_raw_separated", "pre_010_source_visibility_contract_uses_crate_root_for_shared_items", "pre_010_production_surface_has_no_historical_backfill_or_retriever_contract", "v0_3_12_pre_002_production_sources_keep_transport_confined_to_runtime_resources", @@ -115,6 +116,7 @@ fn pre_010_external_hardening_suite_is_present_and_scoped() { assert!(dependency_boundary.contains("v0_3_12_pre_002_manifest_opens_only_the_onchain_transport_live_source_edge")); assert!(dependency_boundary.contains("v0_3_12_pre_003_transaction_and_status_adapters_are_private_offline_and_transport_facade_only")); assert!(dependency_boundary.contains("v0_3_12_pre_004_hydration_contract_uses_only_transport_facade_common_raw_and_existing_ingress")); + assert!(dependency_boundary.contains("v0_3_12_pre_005_block_and_continuity_adapters_remain_private_offline_and_transport_facade_only")); let public_api = include_str!("public_api.rs"); assert!(public_api.contains("pre_003_kind_code_and_settings_are_consumable_from_crate_root")); assert!(public_api.contains("pre_004_start_handle_and_terminal_future_are_consumable_without_public_join_handle")); diff --git a/crates/ksp-worker-raw-transaction-ingest-lib/unit_tests/runtime_resources.rs b/crates/ksp-worker-raw-transaction-ingest-lib/unit_tests/runtime_resources.rs index b602383..02cbc48 100644 --- a/crates/ksp-worker-raw-transaction-ingest-lib/unit_tests/runtime_resources.rs +++ b/crates/ksp-worker-raw-transaction-ingest-lib/unit_tests/runtime_resources.rs @@ -1,5 +1,5 @@ // file: crates/ksp-worker-raw-transaction-ingest-lib/unit_tests/runtime_resources.rs -// version: 4 +// version: 5 fn grpc_endpoint(cluster: &str) -> std::option::Option { return grpc_endpoint_with_identity(cluster, "yellowstone-fixture", "fixture-provider"); @@ -112,6 +112,73 @@ impl super::RawTransactionIngestYellowstoneSignalView for SignalViewFixture { } } +struct BlockViewFixture { + created_at: std::option::Option, + filters: std::vec::Vec, + slot: u64, + transactions: std::vec::Vec<(ksp_onchain_transport_lib::YellowstoneTransactionSignature, u64)>, +} + +impl super::RawTransactionIngestYellowstoneBlockView for BlockViewFixture { + fn created_at(&self) -> std::option::Option { + return self.created_at; + } + + fn filters(&self) -> &[ksp_onchain_transport_lib::YellowstoneSubscribeFilterName] { + return self.filters.as_slice(); + } + + fn slot(&self) -> u64 { + return self.slot; + } + + fn transaction_count(&self) -> usize { + return self.transactions.len(); + } + + fn transaction_identity(&self, position: usize) -> ksp_core_lib::Result<(ksp_onchain_transport_lib::YellowstoneTransactionSignature, u64)> { + return match self.transactions.get(position) { + std::option::Option::Some(value) => std::result::Result::Ok(*value), + std::option::Option::None => std::result::Result::Err(crate::runtime_error("fixture.block_transaction_missing")), + }; + } +} + +struct ContinuityViewFixture { + created_at: std::option::Option, + family: super::RawTransactionIngestContinuityFamily, + filters: std::vec::Vec, + parent_slot: std::option::Option, + slot: u64, + status: std::option::Option, +} + +impl super::RawTransactionIngestYellowstoneContinuityView for ContinuityViewFixture { + fn created_at(&self) -> std::option::Option { + return self.created_at; + } + + fn family(&self) -> super::RawTransactionIngestContinuityFamily { + return self.family; + } + + fn filters(&self) -> &[ksp_onchain_transport_lib::YellowstoneSubscribeFilterName] { + return self.filters.as_slice(); + } + + fn parent_slot(&self) -> std::option::Option { + return self.parent_slot; + } + + fn slot(&self) -> u64 { + return self.slot; + } + + fn status(&self) -> std::option::Option { + return self.status; + } +} + fn filter_names(values: &[&str]) -> std::option::Option> { let mut filters = std::vec::Vec::with_capacity(values.len()); for value in values { @@ -478,6 +545,8 @@ async fn pre_002_runtime_resource_debug_is_safe_and_exposes_no_client_inner() { let debug = std::format!("{resources:?}"); assert!(debug.contains("RawTransactionIngestRuntimeResources")); assert!(debug.contains("devnet")); + assert!(debug.contains("blocks_meta_filter_count")); + assert!(debug.contains("slot_filter_count")); assert!(!debug.contains("GRPC-SECRET-CANARY")); assert!(!debug.contains("HTTP-SECRET-CANARY")); assert!(!debug.contains("127.0.0.1")); @@ -668,6 +737,150 @@ async fn pre_003_source_rejects_route_identity_that_cannot_become_safe_provenanc return; } +#[tokio::test(flavor = "current_thread")] +async fn pre_005_block_projects_transaction_signals_in_exact_source_order() { + let source = match signal_source() { + std::option::Option::Some(value) => value, + std::option::Option::None => return, + }; + let filters = match filter_names(&["block-filter-b", "block-filter-a"]) { + std::option::Option::Some(value) => value, + std::option::Option::None => return, + }; + let created_at = match ksp_onchain_transport_lib::YellowstoneUpdateTimestamp::new(1_760_000_321, 123_000_000) { + std::result::Result::Ok(value) => value, + std::result::Result::Err(_) => return, + }; + let fixture = BlockViewFixture { + created_at: std::option::Option::Some(created_at), + filters, + slot: 777, + transactions: std::vec![ + (ksp_onchain_transport_lib::YellowstoneTransactionSignature::new([0x11; 64]), 9), + (ksp_onchain_transport_lib::YellowstoneTransactionSignature::new([0x22; 64]), 3), + (ksp_onchain_transport_lib::YellowstoneTransactionSignature::new([0x33; 64]), 17), + ], + }; + let signals = match super::project_yellowstone_block_signals(&source, &fixture) { + std::result::Result::Ok(value) => value, + std::result::Result::Err(_) => return, + }; + assert_eq!(signals.len(), 3); + assert_eq!(signals[0].signature.as_bytes(), &[0x11; 64]); + assert_eq!(signals[1].signature.as_bytes(), &[0x22; 64]); + assert_eq!(signals[2].signature.as_bytes(), &[0x33; 64]); + assert_eq!(signals[0].transaction_index, std::option::Option::Some(9)); + assert_eq!(signals[1].transaction_index, std::option::Option::Some(3)); + assert_eq!(signals[2].transaction_index, std::option::Option::Some(17)); + for signal in &signals { + assert_eq!(signal.family, super::RawTransactionIngestSourceFamily::Block); + assert_eq!(signal.slot, 777); + assert_eq!(signal.network.as_str(), "devnet"); + assert_eq!(signal.matched_filter_count, 2); + assert_eq!(signal.created_at, std::option::Option::Some(super::RawTransactionIngestSourceTimestamp { nanos: 123_000_000, seconds: 1_760_000_321 })); + } + return; +} + +#[tokio::test(flavor = "current_thread")] +async fn pre_005_block_hydration_uses_distinct_composite_method() { + let source = match signal_source() { + std::option::Option::Some(value) => value, + std::option::Option::None => return, + }; + let settings = match pre_004_settings() { + std::option::Option::Some(value) => value, + std::option::Option::None => return, + }; + let signal = match pre_004_signal(&source, super::RawTransactionIngestSourceFamily::Block, 42, 7, &["block-filter"], 0) { + std::option::Option::Some(value) => value, + std::option::Option::None => return, + }; + let received_at = match ksp_store_lib::RawTimestamp::from_unix_millis(1_760_000_124_000) { + std::result::Result::Ok(value) => value, + std::result::Result::Err(_) => return, + }; + let provenance = match super::build_hydration_provenance( + &settings, + &signal, + "fixture-http-provider", + "http-hydration-fixture", + ksp_onchain_transport_lib::SolanaCommitment::Confirmed, + received_at, + ) { + std::result::Result::Ok(value) => value, + std::result::Result::Err(_) => return, + }; + assert_eq!(provenance.acquisition_method().as_str(), "block_get_transaction"); + return; +} + +#[tokio::test(flavor = "current_thread")] +async fn pre_005_block_meta_and_slot_project_continuity_only_without_remote_dead_error() { + let source = match signal_source() { + std::option::Option::Some(value) => value, + std::option::Option::None => return, + }; + let block_filters = match filter_names(&["block-meta-filter"]) { + std::option::Option::Some(value) => value, + std::option::Option::None => return, + }; + let slot_filters = match filter_names(&["slot-filter-secret", "slot-filter-secret"]) { + std::option::Option::Some(value) => value, + std::option::Option::None => return, + }; + let block_meta = ContinuityViewFixture { + created_at: std::option::Option::None, + family: super::RawTransactionIngestContinuityFamily::BlockMeta, + filters: block_filters, + parent_slot: std::option::Option::Some(899), + slot: 900, + status: std::option::Option::None, + }; + let slot = ContinuityViewFixture { + created_at: std::option::Option::None, + family: super::RawTransactionIngestContinuityFamily::Slot, + filters: slot_filters, + parent_slot: std::option::Option::Some(900), + slot: 901, + status: std::option::Option::Some(super::RawTransactionIngestContinuityStatus::Dead), + }; + let block_meta_signal = super::project_yellowstone_continuity_signal(&source, &block_meta); + let slot_signal = super::project_yellowstone_continuity_signal(&source, &slot); + assert_eq!(block_meta_signal.family, super::RawTransactionIngestContinuityFamily::BlockMeta); + assert_eq!(block_meta_signal.slot, 900); + assert_eq!(block_meta_signal.parent_slot, std::option::Option::Some(899)); + assert_eq!(block_meta_signal.status, std::option::Option::None); + assert_eq!(slot_signal.family, super::RawTransactionIngestContinuityFamily::Slot); + assert_eq!(slot_signal.slot, 901); + assert_eq!(slot_signal.parent_slot, std::option::Option::Some(900)); + assert_eq!(slot_signal.status, std::option::Option::Some(super::RawTransactionIngestContinuityStatus::Dead)); + assert_eq!(slot_signal.matched_filter_count, 2); + let debug = std::format!("{slot_signal:?}"); + assert!(!debug.contains("slot-filter-secret")); + assert!(!debug.contains("dead_error")); + assert!(!debug.contains("transaction")); + assert!(!debug.contains("meta")); + assert!(!debug.contains("payload")); + return; +} + +#[test] +fn pre_005_slot_status_projection_is_exact_and_source_neutral() { + for (source, expected) in [ + (ksp_onchain_transport_lib::YellowstoneSlotStatus::Processed, super::RawTransactionIngestContinuityStatus::Processed), + (ksp_onchain_transport_lib::YellowstoneSlotStatus::Confirmed, super::RawTransactionIngestContinuityStatus::Confirmed), + (ksp_onchain_transport_lib::YellowstoneSlotStatus::Finalized, super::RawTransactionIngestContinuityStatus::Finalized), + (ksp_onchain_transport_lib::YellowstoneSlotStatus::FirstShredReceived, super::RawTransactionIngestContinuityStatus::FirstShredReceived), + (ksp_onchain_transport_lib::YellowstoneSlotStatus::Completed, super::RawTransactionIngestContinuityStatus::Completed), + (ksp_onchain_transport_lib::YellowstoneSlotStatus::CreatedBank, super::RawTransactionIngestContinuityStatus::CreatedBank), + (ksp_onchain_transport_lib::YellowstoneSlotStatus::Dead, super::RawTransactionIngestContinuityStatus::Dead), + ] { + assert_eq!(super::map_yellowstone_slot_status(source), expected); + } + return; +} + #[test] fn pre_004_composite_provenance_rejects_overflow_without_truncation() { let provider = match ksp_store_lib::RawProvenanceCode::new("yellowstone-provider") { diff --git a/deltas/0.3.12/pre.005.md b/deltas/0.3.12/pre.005.md new file mode 100644 index 0000000..ba953b2 --- /dev/null +++ b/deltas/0.3.12/pre.005.md @@ -0,0 +1,188 @@ + + + +# Delta `0.3.12-pre.005` — Block transactionnel + BlockMeta/Slot continuity-only + +## Base requise + +```text +0.3.12-pre.004-fix.001 +workspace.package.version = 0.3.12-pre.4.fix.1 +``` + +Le gate opérateur communiqué pour cette base est entièrement vert : audits Rust/Markdown, `cargo check --workspace`, Clippy strict, 17 tests Common RAW et 52 tests Worker avec toutes les suites d'intégration associées. + +## Objectif + +Fermer les adapters privés Yellowstone restants nécessaires avant le source task productif : + +```text +Block -> N signaux transactionnels d'hydration dans l'ordre source +BlockMeta -> signal continuity-only +Slot -> signal continuity-only +``` + +La tranche ne branche ni session live, ni reconnect policy Worker, ni `from_slot`, ni `ReplayInfo`, ni processing frontier. + +## Version + +```text +workspace.package.version = 0.3.12-pre.5 +``` + +## Block + +`YellowstoneBlockUpdate` implémente un adapter privé borné qui lit uniquement les identités nécessaires au P0 : + +```text +filters +created_at +slot +transactions[].signature +transactions[].index +``` + +Chaque transaction incluse produit un `RawTransactionIngestSourceSignal` avec `family = Block`. L'ordre du `Vec` de sortie suit strictement l'ordre `transactions[]` du DTO Transport. Aucun body/meta Yellowstone n'entre dans Common RAW directement. + +La future hydration réutilise `getTransaction` et la provenance composite `yellowstone_http`; la méthode d'acquisition devient `block_get_transaction`, distincte de `transaction_get_transaction` et `status_get_transaction`. + +`get_block_observed` reste hors P0 et n'est pas introduit. + +## BlockMeta et Slot + +Les deux DTOs convergent vers un signal Worker privé séparé, `RawTransactionIngestContinuitySignal`. Sa forme est strictement bornée à : + +```text +created_at +family +matched_filter_count +matched_filter_fingerprint +matched_filter_id +network +parent_slot +route +slot +status +``` + +`BlockMeta` fournit slot/parent et aucun status. Ses blockhash, rewards, block_time, block_height et compteurs ne sont pas utilisés comme preuve de complétude transactionnelle dans cette tranche. + +`Slot` fournit slot/parent/status. Les sept états Yellowstone sont mappés vers un enum Worker privé source-neutral. Le diagnostic `dead_error` n'est jamais lu. + +Aucun signal continuity-only ne peut produire un `RawTransactionIngress`. + +## `RUST-API-008` + +Le consumer productif n'arrive qu'en `pre.006`. Les adapters et projecteurs privés `Block`/`BlockMeta`/`Slot` restent donc sous `#[cfg(test)]`, comme la chaîne Transaction/Status/hydration déjà qualifiée. + +Aucun `#[allow(dead_code)]`, `#[expect(dead_code)]` ou export public artificiel n'est ajouté. + +## Canaris + +```text +Block -> 3 signatures exactes dans l'ordre source, avec indexes non triés conservés +Block -> slot/network/filter context commun +Block -> méthode de provenance block_get_transaction +BlockMeta -> continuity-only slot/parent sans status +Slot -> continuity-only slot/parent/status +mapping exact Processed/Confirmed/Finalized/FirstShredReceived/Completed/CreatedBank/Dead +Debug continuity sans filter names ni RAW material +scanner continuity sans signature/transaction/meta/payload/body/error/dead_error +impls réels YellowstoneBlockUpdate/YellowstoneBlockMetaUpdate/YellowstoneSlotUpdate présents +aucun open_standard_subscribe/next_update/get_block_observed +aucune dépendance directe Config/Backfill/backend/reqwest/tonic/proto +``` + +## Décisions prises + +```text +Block est une troisième famille de discovery/hydration, pas un second pipeline RAW +BlockMeta et Slot sont continuity-only +les compteurs BlockMeta ne prouvent jamais la complétude +dead_error ne traverse jamais la frontière Worker +getBlock reste hors P0 +reconnect/from_slot/ReplayInfo restent pre.008 +activation source productive et coalescence restent pre.006 +``` + +## Questions ouvertes + +Aucune question ouverte ne bloque `pre.005`. + +Restent réservés : + +```text +source task productive + coalescence : pre.006 +processing frontier run-local : pre.007 +reconnect/from_slot/ReplayInfo : pre.008 +races/retry/backpressure : pre.009 +``` + +## Fichiers ajoutés + +```text +deltas/0.3.12/pre.005.md +``` + +## Fichiers modifiés + +```text +Cargo.toml +crates/ksp-worker-raw-transaction-ingest-lib/src/lib.rs +crates/ksp-worker-raw-transaction-ingest-lib/src/runtime_resources.rs +crates/ksp-worker-raw-transaction-ingest-lib/tests/dependency_boundary.rs +crates/ksp-worker-raw-transaction-ingest-lib/tests/hardening.rs +crates/ksp-worker-raw-transaction-ingest-lib/tests/release_completeness.rs +crates/ksp-worker-raw-transaction-ingest-lib/unit_tests/runtime_resources.rs +docs/validation/029-V0_3_12_YELLOWSTONE_HYDRATION_CONTINUITY.md +``` + +## Fichiers supprimés + +Aucun. + +## Validations exécutées dans l'environnement d'assemblage + +```text +python3 scripts/audit_rust_workspace_rules.py +General Rust rule audit: clean +Rust export completeness audit: 0 candidate(s) +KSP workspace Rust rule audit: clean + +python3 scripts/audit_markdown_tables.py README.md RULES.md ROADMAP.md CHANGELOG.md docs prompts crates deltas +Markdown table audit: clean +``` + +Les scanners source ciblés vérifient également l'absence d'I/O live prématurée, de `get_block_observed`, de `dead_error` dans l'adapter Slot et de dépendances d'implémentation directes interdites. + +## Validations non exécutées dans l'environnement d'assemblage + +`cargo`, `rustc` et `rustfmt` ne sont pas installés dans l'environnement d'assemblage. Aucun résultat Cargo local n'est déclaré PASS. + +Gate opérateur requis : + +```bash +cargo fmt --all +python3 scripts/audit_rust_workspace_rules.py +python3 scripts/audit_markdown_tables.py README.md RULES.md ROADMAP.md CHANGELOG.md docs prompts crates deltas +cargo check --workspace +cargo clippy --workspace --all-targets --all-features -- -D warnings +cargo test -p ksp-worker-raw-transaction-ingest-lib +cargo tree -p ksp-worker-raw-transaction-ingest-lib --edges normal +cargo tree -p ksp-worker-raw-transaction-ingest-lib -e features +cargo tree --duplicates +``` + +## Non-claims + +```text +pas de source Yellowstone productive +pas de stream ouvert +pas de getBlock +pas de RAW direct depuis Block +pas de RAW depuis BlockMeta/Slot +pas de coalescence productive +pas de frontier +pas de reconnect/replay/repair Worker +pas de smoke live Worker +``` diff --git a/docs/validation/029-V0_3_12_YELLOWSTONE_HYDRATION_CONTINUITY.md b/docs/validation/029-V0_3_12_YELLOWSTONE_HYDRATION_CONTINUITY.md index 40a7624..ee8499e 100644 --- a/docs/validation/029-V0_3_12_YELLOWSTONE_HYDRATION_CONTINUITY.md +++ b/docs/validation/029-V0_3_12_YELLOWSTONE_HYDRATION_CONTINUITY.md @@ -1,5 +1,5 @@ - + # Validation v0.3.12 — Yellowstone + hydration HTTP + continuité de run du Worker RawTransaction @@ -968,3 +968,119 @@ Version du workspace après correctif : ``` Aucune API publique, dépendance, provenance, politique retry, surface Block/continuity ou source task productive n'est modifiée. + +## 41. Gate opérateur reçu pour `pre.004-fix.001` + +Après application de `pre.004-fix.001`, l'opérateur a exécuté : + +```text +cargo fmt --all : PASS visible +audit Rust : clean +Rust export completeness : 0 candidate +KSP workspace Rust rule audit : clean +audit Markdown : clean (340 tables, 807 fichiers) +cargo check --workspace : PASS visible +cargo clippy --workspace --all-targets --all-features -- -D warnings : PASS visible +cargo test -p ksp-raw-transaction-lib : PASS +unit tests Common RAW : 17 PASS, 0 fail +integration Common RAW dependency_boundary : 2 PASS +integration Common RAW public_api : 5 PASS +integration Common RAW release_completeness : 2 PASS +integration Common RAW security_hardening : 2 PASS +cargo test -p ksp-worker-raw-transaction-ingest-lib : PASS +unit tests Worker : 52 PASS, 0 fail +dependency_boundary Worker : 5 PASS +hardening Worker : 11 PASS +public_api Worker : 8 PASS +release_completeness Worker : 3 PASS +doc-tests : 0 fail +``` + +Résultat : `0.3.12-pre.4.fix.1` devient la base autoritaire de `pre.005`. + +## 42. Gate `pre.005` — Block transactionnel et continuité BlockMeta/Slot + +La tranche porte `workspace.package.version = 0.3.12-pre.5`. Elle reste sans source task productive et ferme trois adapters privés contre la façade Transport existante. + +`YellowstoneBlockUpdate` est adapté vers zéro ou plusieurs `RawTransactionIngestSourceSignal` : + +```text +un signal par transaction réellement incluse dans transactions[] +ordre des signaux == ordre source +slot == slot du Block +transaction_index == index Yellowstone de la transaction +signature == signature Yellowstone 64 bytes +family = Block +filters/created_at/network/route réutilisent les mêmes règles que Transaction/Status +``` + +Aucun body/meta Yellowstone du Block n'est admis directement. Ces signaux suivent la même hydration `getTransaction` que les autres familles et utilisent la méthode de provenance `block_get_transaction`. Aucun `get_block_observed` n'est introduit. + +`YellowstoneBlockMetaUpdate` et `YellowstoneSlotUpdate` convergent séparément vers `RawTransactionIngestContinuitySignal`, strictement continuity-only : + +```text +created_at +family = BlockMeta | Slot +matched_filter_count +matched_filter_fingerprint +matched_filter_id optionnel +network +parent_slot optionnel +route +slot +status optionnel +``` + +Le signal de continuité ne contient ni signature transactionnelle, ni transaction/meta/payload, ni remote error. Pour `Slot`, les sept états Yellowstone sont mappés vers un enum Worker privé. Le texte `dead_error` n'est jamais lu ni recopié. Pour `BlockMeta`, aucun compteur n'est interprété comme preuve de complétude transactionnelle. + +## 43. `RUST-API-008` et frontière productive `pre.005` + +Comme `pre.006` reste la première tranche qui ouvre réellement une session Yellowstone et consomme ces adapters dans le runtime normal, les nouveaux helpers `Block`/`BlockMeta`/`Slot` demeurent sous `#[cfg(test)]`. + +Cette décision évite simultanément : + +```text +code privé mort dans le build normal +surface publique artificielle +#[allow(dead_code)] / #[expect(dead_code)] +activation prématurée du source task +``` + +Les impls contre les DTOs Transport réels sont néanmoins compilés dans les tests. Les fixtures génériques qualifient l'ordre, les identités et la séparation RAW/continuité sans nécessiter de constructeur public supplémentaire dans Transport. + +## 44. Canaris déterministes `pre.005` + +Les preuves ajoutées couvrent : + +```text +Block -> N signaux dans l'ordre exact source +signature/index/slot de chaque transaction Block conservés +family Block distincte +provenance hydration Block -> block_get_transaction +BlockMeta -> continuity-only +Slot -> continuity-only +mapping exact des sept YellowstoneSlotStatus +absence de signature/transaction/meta/payload/error dans le signal continuity +absence de YellowstoneSlotUpdate::dead_error +absence de getBlock/get_block_observed +absence d'open_standard_subscribe/next_update/source spawn +absence Config/Backfill/backend/reqwest/tonic/proto direct dans le Worker +``` + +## 45. Non-claims `pre.005` + +`pre.005` ne prétend pas avoir : + +```text +ouvert un stream Yellowstone productif +hydraté une transaction Block reçue en live +coalescé Block + Transaction + TransactionStatus +persisté un RAW depuis une update Block live +utilisé BlockMeta ou Slot comme preuve de complétude transactionnelle +branché la processing frontier +interprété from_slot ou ReplayInfo +détecté/réparé un gap de continuité +introduit une politique reconnect Worker +``` + +La tranche suivante reste `pre.006` : source task productive, session Yellowstone via Transport, hydration/admission et coalescence bornée dans le supervisor existant.