diff --git a/Cargo.toml b/Cargo.toml index 1e9fb01..5c69351 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,12 +1,12 @@ # file: Cargo.toml -# version: 520 +# version: 521 [workspace] resolver = "3" members = ["crates/ksp-app-backfill-desk", "crates/ksp-app-config-desk", "crates/ksp-app-solprices-desk", "crates/ksp-app-store-desk", "crates/ksp-app-wallet-desk", "crates/ksp-config-lib", "crates/ksp-core-lib", "crates/ksp-interface-lib", "crates/ksp-job-api", "crates/ksp-job-backfill-lib", "crates/ksp-logging-lib", "crates/ksp-offchain-transport-lib", "crates/ksp-onchain-transport-lib", "crates/ksp-program-api", "crates/ksp-raw-transaction-lib", "crates/ksp-store-api", "crates/ksp-store-lib", "crates/ksp-store-postgres-lib", "crates/ksp-wallet-lib", "crates/ksp-worker-api", "crates/ksp-worker-raw-transaction-ingest-lib"] [workspace.package] -version = "0.3.12-pre.2" +version = "0.3.12-pre.3" edition = "2024" license = "MIT" repository = "https://git.sasedev.com/Sasedev/khadhroony-solana-project" diff --git a/crates/ksp-worker-raw-transaction-ingest-lib/src/lib.rs b/crates/ksp-worker-raw-transaction-ingest-lib/src/lib.rs index 56b5d5f..5957f47 100644 --- a/crates/ksp-worker-raw-transaction-ingest-lib/src/lib.rs +++ b/crates/ksp-worker-raw-transaction-ingest-lib/src/lib.rs @@ -1,5 +1,5 @@ // file: crates/ksp-worker-raw-transaction-ingest-lib/src/lib.rs -// version: 9 +// version: 10 #![warn(missing_docs)] #![deny(unreachable_pub)] @@ -11,7 +11,8 @@ //! and the caller-runtime-owned lifecycle with private child-task supervision. This tranche also //! owns bounded source-neutral admission, common RAW canonicalization/assembly and backend-neutral //! Store persistence in normal mode plus concrete latest-value snapshots projected onto Worker API; -//! the first validated Yellowstone/HTTP runtime-resource contract exists, but no live stream is opened yet. +//! the first validated Yellowstone/HTTP runtime-resource contract exists and deterministically projects +//! Transaction/TransactionStatus updates into private source-neutral signals, but no live stream is opened yet. mod admission; mod error; diff --git a/crates/ksp-worker-raw-transaction-ingest-lib/src/runtime_resources.rs b/crates/ksp-worker-raw-transaction-ingest-lib/src/runtime_resources.rs index a8cac1d..f26e6bc 100644 --- a/crates/ksp-worker-raw-transaction-ingest-lib/src/runtime_resources.rs +++ b/crates/ksp-worker-raw-transaction-ingest-lib/src/runtime_resources.rs @@ -1,5 +1,145 @@ // file: crates/ksp-worker-raw-transaction-ingest-lib/src/runtime_resources.rs -// version: 1 +// version: 2 + +use sha2::Digest; // rust-rules: trait-import + +const RAW_TRANSACTION_INGEST_YELLOWSTONE_FILTER_FINGERPRINT_DOMAIN: &[u8] = b"ksp.raw_transaction_ingest.yellowstone.filters.v1\0"; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum RawTransactionIngestSourceFamily { + Transaction, + TransactionStatus, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +struct RawTransactionIngestSourceRoute { + endpoint_id: ksp_store_lib::RawProvenanceCode, + provider: ksp_store_lib::RawProvenanceCode, +} + +#[derive(Clone, Copy, Eq, PartialEq)] +struct RawTransactionIngestSourceTimestamp { + nanos: u32, + seconds: i64, +} + +impl std::fmt::Debug for RawTransactionIngestSourceTimestamp { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + return formatter.debug_struct("RawTransactionIngestSourceTimestamp").field("nanos", &self.nanos).field("seconds", &self.seconds).finish(); + } +} + +#[derive(Clone, Eq, PartialEq)] +struct RawTransactionIngestSourceSignal { + created_at: std::option::Option, + family: RawTransactionIngestSourceFamily, + matched_filter_count: usize, + matched_filter_fingerprint: [u8; 32], + network: ksp_store_lib::RawNetworkId, + route: RawTransactionIngestSourceRoute, + signature: ksp_store_lib::RawTransactionSignature, + slot: u64, + transaction_index: std::option::Option, +} + +impl std::fmt::Debug for RawTransactionIngestSourceSignal { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + return formatter + .debug_struct("RawTransactionIngestSourceSignal") + .field("created_at", &self.created_at) + .field("family", &self.family) + .field("matched_filter_count", &self.matched_filter_count) + .field("matched_filter_fingerprint_bytes", &self.matched_filter_fingerprint.len()) + .field("network", &self.network) + .field("route", &self.route) + .field("signature_bytes", &self.signature.as_bytes().len()) + .field("slot", &self.slot) + .field("has_transaction_index", &self.transaction_index.is_some()) + .finish_non_exhaustive(); + } +} + +trait RawTransactionIngestYellowstoneSignalView { + fn created_at(&self) -> std::option::Option; + + fn family(&self) -> RawTransactionIngestSourceFamily; + + fn filters(&self) -> &[ksp_onchain_transport_lib::YellowstoneSubscribeFilterName]; + + fn index(&self) -> u64; + + fn signature(&self) -> ksp_onchain_transport_lib::YellowstoneTransactionSignature; + + fn slot(&self) -> u64; +} + +impl RawTransactionIngestYellowstoneSignalView for ksp_onchain_transport_lib::YellowstoneTransactionUpdate { + fn created_at(&self) -> std::option::Option { + return ksp_onchain_transport_lib::YellowstoneTransactionUpdate::created_at(self); + } + + fn family(&self) -> RawTransactionIngestSourceFamily { + return RawTransactionIngestSourceFamily::Transaction; + } + + fn filters(&self) -> &[ksp_onchain_transport_lib::YellowstoneSubscribeFilterName] { + return ksp_onchain_transport_lib::YellowstoneTransactionUpdate::filters(self); + } + + fn index(&self) -> u64 { + return ksp_onchain_transport_lib::YellowstoneTransactionUpdate::transaction(self).index(); + } + + fn signature(&self) -> ksp_onchain_transport_lib::YellowstoneTransactionSignature { + return ksp_onchain_transport_lib::YellowstoneTransactionUpdate::transaction(self).signature(); + } + + fn slot(&self) -> u64 { + return ksp_onchain_transport_lib::YellowstoneTransactionUpdate::slot(self); + } +} + +impl RawTransactionIngestYellowstoneSignalView for ksp_onchain_transport_lib::YellowstoneTransactionStatusUpdate { + fn created_at(&self) -> std::option::Option { + return ksp_onchain_transport_lib::YellowstoneTransactionStatusUpdate::created_at(self); + } + + fn family(&self) -> RawTransactionIngestSourceFamily { + return RawTransactionIngestSourceFamily::TransactionStatus; + } + + fn filters(&self) -> &[ksp_onchain_transport_lib::YellowstoneSubscribeFilterName] { + return ksp_onchain_transport_lib::YellowstoneTransactionStatusUpdate::filters(self); + } + + fn index(&self) -> u64 { + return ksp_onchain_transport_lib::YellowstoneTransactionStatusUpdate::index(self); + } + + fn signature(&self) -> ksp_onchain_transport_lib::YellowstoneTransactionSignature { + return ksp_onchain_transport_lib::YellowstoneTransactionStatusUpdate::signature(self); + } + + fn slot(&self) -> u64 { + return ksp_onchain_transport_lib::YellowstoneTransactionStatusUpdate::slot(self); + } +} + +impl std::convert::From<(&crate::RawTransactionIngestYellowstoneSource, &ksp_onchain_transport_lib::YellowstoneTransactionUpdate)> + for RawTransactionIngestSourceSignal +{ + fn from(value: (&crate::RawTransactionIngestYellowstoneSource, &ksp_onchain_transport_lib::YellowstoneTransactionUpdate)) -> Self { + return project_yellowstone_signal(value.0, value.1); + } +} + +impl std::convert::From<(&crate::RawTransactionIngestYellowstoneSource, &ksp_onchain_transport_lib::YellowstoneTransactionStatusUpdate)> + for RawTransactionIngestSourceSignal +{ + fn from(value: (&crate::RawTransactionIngestYellowstoneSource, &ksp_onchain_transport_lib::YellowstoneTransactionStatusUpdate)) -> Self { + return project_yellowstone_signal(value.0, value.1); + } +} /// Validated Yellowstone plus HTTP runtime source owned by the continuous RAW transaction ingest Worker. /// @@ -10,6 +150,8 @@ pub struct RawTransactionIngestYellowstoneSource { subscribe_request: ksp_onchain_transport_lib::YellowstoneSubscribeRequest, http_pool: ksp_onchain_transport_lib::HttpTransportPool, hydration_role: ksp_onchain_transport_lib::HttpRoleName, + network: ksp_store_lib::RawNetworkId, + route: RawTransactionIngestSourceRoute, } impl crate::RawTransactionIngestYellowstoneSource { @@ -33,11 +175,23 @@ impl crate::RawTransactionIngestYellowstoneSource { return std::result::Result::Err(crate::runtime_error("runtime_resources.hydration_commitment_invalid")); }, } + let network = match ksp_store_lib::RawNetworkId::new(yellowstone_channel.cluster().as_str()) { + std::result::Result::Ok(value) => value, + std::result::Result::Err(_) => return std::result::Result::Err(crate::runtime_error("runtime_resources.yellowstone_network_unrepresentable")), + }; + let provider = match ksp_store_lib::RawProvenanceCode::new(yellowstone_channel.provider().as_str()) { + std::result::Result::Ok(value) => value, + std::result::Result::Err(_) => return std::result::Result::Err(crate::runtime_error("runtime_resources.yellowstone_provider_unrepresentable")), + }; + let endpoint_id = match ksp_store_lib::RawProvenanceCode::new(yellowstone_channel.endpoint_name()) { + std::result::Result::Ok(value) => value, + std::result::Result::Err(_) => return std::result::Result::Err(crate::runtime_error("runtime_resources.yellowstone_endpoint_unrepresentable")), + }; let method = match ksp_onchain_transport_lib::find_http_rpc_method("getTransaction") { std::option::Option::Some(value) => value, std::option::Option::None => return std::result::Result::Err(crate::runtime_error("runtime_resources.hydration_method_missing")), }; - let compatible_http_routes = compatible_http_route_count(&http_pool, &hydration_role, method.request_kind(), yellowstone_channel.cluster().as_str()); + let compatible_http_routes = compatible_http_route_count(&http_pool, &hydration_role, method.request_kind(), network.as_str()); let compatible_http_routes = match compatible_http_routes { std::result::Result::Ok(value) => value, std::result::Result::Err(error) => return std::result::Result::Err(error), @@ -45,7 +199,8 @@ impl crate::RawTransactionIngestYellowstoneSource { if compatible_http_routes == 0 { return std::result::Result::Err(crate::runtime_error("runtime_resources.hydration_role_unsupported")); } - return std::result::Result::Ok(Self { yellowstone_channel, subscribe_request, http_pool, hydration_role }); + let route = RawTransactionIngestSourceRoute { endpoint_id, provider }; + return std::result::Result::Ok(Self { yellowstone_channel, subscribe_request, http_pool, hydration_role, network, route }); } } @@ -56,7 +211,7 @@ impl std::fmt::Debug for crate::RawTransactionIngestYellowstoneSource { .debug_struct("RawTransactionIngestYellowstoneSource") .field("yellowstone_endpoint_name", &self.yellowstone_channel.endpoint_name()) .field("yellowstone_provider", &self.yellowstone_channel.provider().as_str()) - .field("network", &self.yellowstone_channel.cluster().as_str()) + .field("network", &self.network.as_str()) .field("transaction_filter_count", &self.subscribe_request.transaction_filter_count()) .field("transaction_status_filter_count", &self.subscribe_request.transaction_status_filter_count()) .field("block_filter_count", &self.subscribe_request.block_filter_count()) @@ -85,7 +240,7 @@ impl crate::RawTransactionIngestRuntimeResources { /// Validates that caller-owned Worker settings target the same logical network as the composed Yellowstone/HTTP source. pub(crate) fn validate_network(&self, network: &ksp_store_lib::RawNetworkId) -> ksp_core_lib::Result<()> { - if self.yellowstone_source.yellowstone_channel.cluster().as_str() != network.as_str() { + if &self.yellowstone_source.network != network { return std::result::Result::Err(crate::runtime_error("runtime_resources.worker_network_mismatch")); } return std::result::Result::Ok(()); @@ -131,6 +286,41 @@ fn ingestion_filter_count(request: &ksp_onchain_transport_lib::YellowstoneSubscr return request.transaction_filter_count().saturating_add(request.transaction_status_filter_count()).saturating_add(request.block_filter_count()); } +fn matched_filter_fingerprint(filters: &[ksp_onchain_transport_lib::YellowstoneSubscribeFilterName]) -> [u8; 32] { + let mut names = filters.iter().map(ksp_onchain_transport_lib::YellowstoneSubscribeFilterName::as_str).collect::>(); + names.sort_unstable(); + names.dedup(); + let mut hasher = sha2::Sha256::new(); + hasher.update(RAW_TRANSACTION_INGEST_YELLOWSTONE_FILTER_FINGERPRINT_DOMAIN); + hasher.update((names.len() as u64).to_be_bytes()); + for name in names { + hasher.update((name.len() as u64).to_be_bytes()); + hasher.update(name.as_bytes()); + } + return hasher.finalize().into(); +} + +fn project_yellowstone_signal( + source: &crate::RawTransactionIngestYellowstoneSource, + update: &T, +) -> RawTransactionIngestSourceSignal { + let signature = ksp_store_lib::RawTransactionSignature::new(*update.signature().as_bytes()); + let created_at = update.created_at().map(|value| { + return RawTransactionIngestSourceTimestamp { nanos: value.nanos(), seconds: value.seconds() }; + }); + return RawTransactionIngestSourceSignal { + created_at, + family: update.family(), + matched_filter_count: update.filters().len(), + matched_filter_fingerprint: matched_filter_fingerprint(update.filters()), + network: source.network.clone(), + route: source.route.clone(), + signature, + slot: update.slot(), + transaction_index: std::option::Option::Some(update.index()), + }; +} + #[cfg(test)] #[path = "../unit_tests/runtime_resources.rs"] mod tests; diff --git a/crates/ksp-worker-raw-transaction-ingest-lib/tests/dependency_boundary.rs b/crates/ksp-worker-raw-transaction-ingest-lib/tests/dependency_boundary.rs index 8ba56f0..98c4098 100644 --- a/crates/ksp-worker-raw-transaction-ingest-lib/tests/dependency_boundary.rs +++ b/crates/ksp-worker-raw-transaction-ingest-lib/tests/dependency_boundary.rs @@ -1,5 +1,5 @@ // file: crates/ksp-worker-raw-transaction-ingest-lib/tests/dependency_boundary.rs -// version: 9 +// version: 10 //! Dependency firewall canaries for the RAW transaction ingest Worker foundation. @@ -63,6 +63,51 @@ fn v0_3_12_pre_002_manifest_opens_only_the_onchain_transport_live_source_edge() return; } +#[test] +fn v0_3_12_pre_003_transaction_and_status_adapters_are_private_offline_and_transport_facade_only() { + let root = include_str!("../src/lib.rs"); + let resources = include_str!("../src/runtime_resources.rs"); + for required in [ + "RawTransactionIngestYellowstoneSignalView for ksp_onchain_transport_lib::YellowstoneTransactionUpdate", + "RawTransactionIngestYellowstoneSignalView for ksp_onchain_transport_lib::YellowstoneTransactionStatusUpdate", + "YellowstoneTransactionUpdate::transaction(self).signature()", + "YellowstoneTransactionUpdate::transaction(self).index()", + "YellowstoneTransactionStatusUpdate::signature(self)", + "YellowstoneTransactionStatusUpdate::index(self)", + "RawTransactionSignature::new", + "matched_filter_fingerprint", + "yellowstone_provider_unrepresentable", + "yellowstone_endpoint_unrepresentable", + ] { + assert!(resources.contains(required), "required pre.003 signal-adapter contract missing: {required}"); + } + for forbidden in [ + "pub struct RawTransactionIngestSourceSignal", + "pub(crate) struct RawTransactionIngestSourceSignal", + "pub use self::runtime_resources::RawTransactionIngestSourceSignal", + "open_standard_subscribe", + "next_update", + "get_transaction_observed", + "get_block_observed", + "ksp_store_postgres_lib::", + "reqwest::", + "tonic::", + "yellowstone_grpc_proto::", + ] { + assert!(!resources.contains(forbidden) && !root.contains(forbidden), "pre.003 crossed a private/offline boundary: {forbidden}"); + } + let status_impl = + match resources.split_once("impl RawTransactionIngestYellowstoneSignalView for ksp_onchain_transport_lib::YellowstoneTransactionStatusUpdate") { + std::option::Option::Some((_, tail)) => match tail.split_once("impl std::convert::From<") { + std::option::Option::Some((value, _)) => value, + std::option::Option::None => tail, + }, + std::option::Option::None => "", + }; + assert!(!status_impl.contains("::error(self)"), "TransactionStatus remote error material must not enter the private signal"); + return; +} + #[test] fn v0_3_12_pre_002_source_surface_hardens_shutdown_and_faults_without_backend_or_premature_live_io() { let root = include_str!("../src/lib.rs"); diff --git a/crates/ksp-worker-raw-transaction-ingest-lib/tests/hardening.rs b/crates/ksp-worker-raw-transaction-ingest-lib/tests/hardening.rs index 4918754..127d466 100644 --- a/crates/ksp-worker-raw-transaction-ingest-lib/tests/hardening.rs +++ b/crates/ksp-worker-raw-transaction-ingest-lib/tests/hardening.rs @@ -1,5 +1,5 @@ // file: crates/ksp-worker-raw-transaction-ingest-lib/tests/hardening.rs -// version: 3 +// version: 4 //! External public, security, redaction and release-boundary hardening canaries for `pre.010`. @@ -278,6 +278,40 @@ fn v0_3_12_pre_002_production_sources_keep_transport_confined_to_runtime_resourc return; } +#[test] +fn v0_3_12_pre_003_private_signal_debug_and_shape_do_not_expose_signature_filters_or_payload() { + let root = include_str!("../src/lib.rs"); + let resources = include_str!("../src/runtime_resources.rs"); + assert!(resources.contains("struct RawTransactionIngestSourceSignal")); + assert!(!root.contains("RawTransactionIngestSourceSignal")); + assert!(!resources.contains(".field(\"matched_filter_fingerprint\", &self.matched_filter_fingerprint)")); + assert!(!resources.contains(".field(\"signature\", &self.signature)")); + let signal_struct = match resources.split_once("struct RawTransactionIngestSourceSignal {") { + std::option::Option::Some((_, tail)) => match tail.split_once("}\n\nimpl std::fmt::Debug for RawTransactionIngestSourceSignal") { + std::option::Option::Some((value, _)) => value, + std::option::Option::None => "", + }, + std::option::Option::None => "", + }; + for forbidden in ["transaction:", "meta:", "error:", "payload:", "body:", "is_vote:"] { + assert!(!signal_struct.contains(forbidden), "payload/provider-specific field leaked into private source signal: {forbidden}"); + } + for required in [ + "created_at:", + "family:", + "matched_filter_count:", + "matched_filter_fingerprint:", + "network:", + "route:", + "signature:", + "slot:", + "transaction_index:", + ] { + assert!(signal_struct.contains(required), "required private signal field missing: {required}"); + } + return; +} + #[test] fn v0_3_12_pre_002_runtime_resource_contract_performs_no_live_io_or_source_spawn() { let resources = include_str!("../src/runtime_resources.rs"); diff --git a/crates/ksp-worker-raw-transaction-ingest-lib/tests/release_completeness.rs b/crates/ksp-worker-raw-transaction-ingest-lib/tests/release_completeness.rs index e43ce5a..9acf226 100644 --- a/crates/ksp-worker-raw-transaction-ingest-lib/tests/release_completeness.rs +++ b/crates/ksp-worker-raw-transaction-ingest-lib/tests/release_completeness.rs @@ -1,5 +1,5 @@ // file: crates/ksp-worker-raw-transaction-ingest-lib/tests/release_completeness.rs -// version: 2 +// version: 3 //! Release-completeness canaries through the `pre.010` public/release/security hardening tranche. @@ -98,6 +98,7 @@ fn pre_010_external_hardening_suite_is_present_and_scoped() { "pre_010_external_error_codes_are_stable_unique_and_domain_scoped", "pre_010_debug_and_settings_errors_redact_worker_identity_and_invalid_values", "v0_3_12_pre_002_manifest_dependency_surface_opens_only_transport_and_remains_backend_neutral", + "v0_3_12_pre_003_private_signal_debug_and_shape_do_not_expose_signature_filters_or_payload", "pre_010_source_visibility_contract_uses_crate_root_for_shared_items", "pre_010_production_surface_has_no_historical_backfill_or_retriever_contract", "v0_3_12_pre_002_production_sources_keep_transport_confined_to_runtime_resources", @@ -111,6 +112,7 @@ fn pre_010_external_hardening_suite_is_present_and_scoped() { assert!(dependency_boundary.contains("pre_002_manifest_dependency_surface_is_exact")); assert!(dependency_boundary.contains("v0_3_12_pre_002_source_surface_hardens_shutdown_and_faults_without_backend_or_premature_live_io")); assert!(dependency_boundary.contains("v0_3_12_pre_002_manifest_opens_only_the_onchain_transport_live_source_edge")); + assert!(dependency_boundary.contains("v0_3_12_pre_003_transaction_and_status_adapters_are_private_offline_and_transport_facade_only")); let public_api = include_str!("public_api.rs"); assert!(public_api.contains("pre_003_kind_code_and_settings_are_consumable_from_crate_root")); assert!(public_api.contains("pre_004_start_handle_and_terminal_future_are_consumable_without_public_join_handle")); diff --git a/crates/ksp-worker-raw-transaction-ingest-lib/unit_tests/runtime_resources.rs b/crates/ksp-worker-raw-transaction-ingest-lib/unit_tests/runtime_resources.rs index 6804f13..57a4e32 100644 --- a/crates/ksp-worker-raw-transaction-ingest-lib/unit_tests/runtime_resources.rs +++ b/crates/ksp-worker-raw-transaction-ingest-lib/unit_tests/runtime_resources.rs @@ -1,15 +1,23 @@ // file: crates/ksp-worker-raw-transaction-ingest-lib/unit_tests/runtime_resources.rs -// version: 1 +// version: 2 fn grpc_endpoint(cluster: &str) -> std::option::Option { + return grpc_endpoint_with_identity(cluster, "yellowstone-fixture", "fixture-provider"); +} + +fn grpc_endpoint_with_identity( + cluster: &str, + endpoint_name: &str, + provider: &str, +) -> std::option::Option { let url = match ksp_onchain_transport_lib::YellowstoneGrpcEndpointUrl::parse("http://127.0.0.1:10000/GRPC-SECRET-CANARY") { std::result::Result::Ok(value) => value, std::result::Result::Err(_) => return std::option::Option::None, }; return std::option::Option::Some(ksp_onchain_transport_lib::YellowstoneGrpcEndpointSettings::new( - "yellowstone-fixture", + endpoint_name, true, - ksp_onchain_transport_lib::YellowstoneGrpcProviderName::new("fixture-provider"), + ksp_onchain_transport_lib::YellowstoneGrpcProviderName::new(provider), ksp_onchain_transport_lib::YellowstoneGrpcClusterName::new(cluster), url, ksp_onchain_transport_lib::YellowstoneGrpcSessionSettings::default(), @@ -69,6 +77,76 @@ fn transaction_request( return std::option::Option::Some(request); } +struct SignalViewFixture { + created_at: std::option::Option, + family: super::RawTransactionIngestSourceFamily, + filters: std::vec::Vec, + index: u64, + signature: ksp_onchain_transport_lib::YellowstoneTransactionSignature, + slot: u64, +} + +impl super::RawTransactionIngestYellowstoneSignalView for SignalViewFixture { + fn created_at(&self) -> std::option::Option { + return self.created_at; + } + + fn family(&self) -> super::RawTransactionIngestSourceFamily { + return self.family; + } + + fn filters(&self) -> &[ksp_onchain_transport_lib::YellowstoneSubscribeFilterName] { + return self.filters.as_slice(); + } + + fn index(&self) -> u64 { + return self.index; + } + + fn signature(&self) -> ksp_onchain_transport_lib::YellowstoneTransactionSignature { + return self.signature; + } + + fn slot(&self) -> u64 { + return self.slot; + } +} + +fn filter_names(values: &[&str]) -> std::option::Option> { + let mut filters = std::vec::Vec::with_capacity(values.len()); + for value in values { + let filter = match ksp_onchain_transport_lib::YellowstoneSubscribeFilterName::new(*value) { + std::result::Result::Ok(value) => value, + std::result::Result::Err(_) => return std::option::Option::None, + }; + filters.push(filter); + } + return std::option::Option::Some(filters); +} + +fn signal_source() -> std::option::Option { + let endpoint = match grpc_endpoint("devnet") { + std::option::Option::Some(value) => value, + std::option::Option::None => return std::option::Option::None, + }; + let channel = match ksp_onchain_transport_lib::YellowstoneGrpcChannel::prepare(&endpoint) { + std::result::Result::Ok(value) => value, + std::result::Result::Err(_) => return std::option::Option::None, + }; + let request = match transaction_request(std::option::Option::Some(ksp_onchain_transport_lib::SolanaCommitment::Confirmed)) { + std::option::Option::Some(value) => value, + std::option::Option::None => return std::option::Option::None, + }; + let pool = match http_pool("devnet", "hydration", "get_transaction") { + std::option::Option::Some(value) => value, + std::option::Option::None => return std::option::Option::None, + }; + return match crate::RawTransactionIngestYellowstoneSource::new(channel, request, pool, ksp_onchain_transport_lib::HttpRoleName::new("hydration")) { + std::result::Result::Ok(value) => std::option::Option::Some(value), + std::result::Result::Err(_) => std::option::Option::None, + }; +} + #[tokio::test(flavor = "current_thread")] async fn pre_002_source_accepts_matching_confirmed_transaction_and_get_transaction_route_without_io() { let endpoint = match grpc_endpoint("devnet") { @@ -250,3 +328,149 @@ async fn pre_002_runtime_resources_reject_worker_network_mismatch_without_starti assert!(error.context().iter().any(|context| return context.value() == "runtime_resources.worker_network_mismatch")); return; } + +#[tokio::test(flavor = "current_thread")] +async fn pre_003_transaction_signal_fixture_preserves_exact_source_neutral_identity() { + let source = match signal_source() { + std::option::Option::Some(value) => value, + std::option::Option::None => return, + }; + let filters = match filter_names(&["filter-b", "filter-a", "filter-a"]) { + std::option::Option::Some(value) => value, + std::option::Option::None => return, + }; + let created_at = match ksp_onchain_transport_lib::YellowstoneUpdateTimestamp::new(1_760_000_123, 456_000_000) { + std::result::Result::Ok(value) => value, + std::result::Result::Err(_) => return, + }; + let fixture = SignalViewFixture { + created_at: std::option::Option::Some(created_at), + family: super::RawTransactionIngestSourceFamily::Transaction, + filters, + index: 7, + signature: ksp_onchain_transport_lib::YellowstoneTransactionSignature::new([0xAB; 64]), + slot: 42, + }; + let signal = super::project_yellowstone_signal(&source, &fixture); + assert_eq!(signal.family, super::RawTransactionIngestSourceFamily::Transaction); + assert_eq!(signal.network.as_str(), "devnet"); + assert_eq!(signal.signature.as_bytes(), &[0xAB; 64]); + assert_eq!(signal.slot, 42); + assert_eq!(signal.transaction_index, std::option::Option::Some(7)); + assert_eq!(signal.route.provider.as_str(), "fixture-provider"); + assert_eq!(signal.route.endpoint_id.as_str(), "yellowstone-fixture"); + assert_eq!(signal.matched_filter_count, 3); + assert_eq!( + signal.matched_filter_fingerprint, + [ + 0x20, 0x83, 0x1d, 0xc8, 0x37, 0x8b, 0x60, 0x70, 0xf9, 0x1d, 0x9d, 0xe6, 0xd3, 0xa1, 0xd2, 0xb3, 0xd7, 0x72, 0x98, 0xe3, 0xad, 0xa7, 0x6c, 0x01, + 0xfa, 0xa3, 0x08, 0x26, 0xff, 0xab, 0xcd, 0x84, + ] + ); + assert_eq!(signal.created_at, std::option::Option::Some(super::RawTransactionIngestSourceTimestamp { nanos: 456_000_000, seconds: 1_760_000_123 })); + return; +} + +#[tokio::test(flavor = "current_thread")] +async fn pre_003_transaction_status_signal_has_no_payload_or_remote_error_material() { + let source = match signal_source() { + std::option::Option::Some(value) => value, + std::option::Option::None => return, + }; + let filters = match filter_names(&["status-secret-canary"]) { + std::option::Option::Some(value) => value, + std::option::Option::None => return, + }; + let fixture = SignalViewFixture { + created_at: std::option::Option::None, + family: super::RawTransactionIngestSourceFamily::TransactionStatus, + filters, + index: 9, + signature: ksp_onchain_transport_lib::YellowstoneTransactionSignature::new([0xCD; 64]), + slot: 84, + }; + let signal = super::project_yellowstone_signal(&source, &fixture); + assert_eq!(signal.family, super::RawTransactionIngestSourceFamily::TransactionStatus); + assert_eq!(signal.signature.as_bytes(), &[0xCD; 64]); + assert_eq!(signal.slot, 84); + assert_eq!(signal.transaction_index, std::option::Option::Some(9)); + assert_eq!(signal.created_at, std::option::Option::None); + let debug = std::format!("{signal:?}"); + assert!(!debug.contains("status-secret-canary")); + assert!(debug.contains("signature_bytes")); + assert!(!debug.contains("YellowstoneTransactionError")); + assert!(!debug.contains("transaction_body")); + assert!(!debug.contains("transaction_meta")); + return; +} + +#[tokio::test(flavor = "current_thread")] +async fn pre_003_filter_fingerprint_is_order_and_duplicate_insensitive() { + let source = match signal_source() { + std::option::Option::Some(value) => value, + std::option::Option::None => return, + }; + let first_filters = match filter_names(&["filter-b", "filter-a", "filter-a"]) { + std::option::Option::Some(value) => value, + std::option::Option::None => return, + }; + let second_filters = match filter_names(&["filter-a", "filter-b"]) { + std::option::Option::Some(value) => value, + std::option::Option::None => return, + }; + let first = SignalViewFixture { + created_at: std::option::Option::None, + family: super::RawTransactionIngestSourceFamily::Transaction, + filters: first_filters, + index: 1, + signature: ksp_onchain_transport_lib::YellowstoneTransactionSignature::new([1; 64]), + slot: 1, + }; + let second = SignalViewFixture { + created_at: std::option::Option::None, + family: super::RawTransactionIngestSourceFamily::Transaction, + filters: second_filters, + index: 1, + signature: ksp_onchain_transport_lib::YellowstoneTransactionSignature::new([1; 64]), + slot: 1, + }; + let first_signal = super::project_yellowstone_signal(&source, &first); + let second_signal = super::project_yellowstone_signal(&source, &second); + assert_eq!(first_signal.matched_filter_fingerprint, second_signal.matched_filter_fingerprint); + assert_eq!(first_signal.matched_filter_count, 3); + assert_eq!(second_signal.matched_filter_count, 2); + return; +} + +#[tokio::test(flavor = "current_thread")] +async fn pre_003_source_rejects_route_identity_that_cannot_become_safe_provenance() { + for (endpoint_name, provider, expected_condition) in [ + ("yellowstone-fixture", "fixture/provider", "runtime_resources.yellowstone_provider_unrepresentable"), + ("yellowstone/fixture", "fixture-provider", "runtime_resources.yellowstone_endpoint_unrepresentable"), + ] { + let endpoint = match grpc_endpoint_with_identity("devnet", endpoint_name, provider) { + std::option::Option::Some(value) => value, + std::option::Option::None => return, + }; + let channel = match ksp_onchain_transport_lib::YellowstoneGrpcChannel::prepare(&endpoint) { + std::result::Result::Ok(value) => value, + std::result::Result::Err(_) => return, + }; + let request = match transaction_request(std::option::Option::Some(ksp_onchain_transport_lib::SolanaCommitment::Confirmed)) { + std::option::Option::Some(value) => value, + std::option::Option::None => return, + }; + let pool = match http_pool("devnet", "hydration", "get_transaction") { + std::option::Option::Some(value) => value, + std::option::Option::None => return, + }; + let result = crate::RawTransactionIngestYellowstoneSource::new(channel, request, pool, ksp_onchain_transport_lib::HttpRoleName::new("hydration")); + let error = match result { + std::result::Result::Ok(_) => return, + std::result::Result::Err(error) => error, + }; + assert_eq!(error.code(), crate::ERROR_CODE_RAW_TRANSACTION_INGEST_RUNTIME_INVALID); + assert!(error.context().iter().any(|context| return context.value() == expected_condition)); + } + return; +} diff --git a/deltas/0.3.12/pre.003.md b/deltas/0.3.12/pre.003.md new file mode 100644 index 0000000..e0d57ee --- /dev/null +++ b/deltas/0.3.12/pre.003.md @@ -0,0 +1,255 @@ + + + +# Delta `0.3.12-pre.003` — signaux privés Yellowstone Transaction + TransactionStatus + +## Base requise + +```text +0.3.12-pre.002 +workspace.package.version = 0.3.12-pre.2 +``` + +Le gate opérateur communiqué pour `pre.002` est vert sur : + +```text +cargo fmt --all +Rust rule audit +Markdown table audit +cargo check --workspace +cargo clippy --workspace --all-targets --all-features -- -D warnings +cargo test -p ksp-worker-raw-transaction-ingest-lib : 41 PASS +cargo tree Worker normal/features +cargo tree --duplicates exécuté pour inspection +``` + +Le graphe confirme que `reqwest`, `tonic` et `yellowstone-grpc-proto` restent transitifs via `ksp-onchain-transport-lib`, jamais directs dans le Worker. + +## Objectif + +Matérialiser uniquement les deux adapters déterministes prévus : + +```text +YellowstoneTransactionUpdate -> signal Worker privé +YellowstoneTransactionStatusUpdate -> signal Worker privé +``` + +sans ouvrir de session Yellowstone, sans HTTP et sans persistence réseau. + +## Version + +```text +workspace.package.version = 0.3.12-pre.3 +``` + +## Signal Worker privé + +Le signal est strictement privé à `runtime_resources.rs`. Sa forme contient : + +```text +RawNetworkId +RawTransactionSignature +slot u64 +transaction_index Option +family Transaction | TransactionStatus +route sûre provider + endpoint_id +matched_filter_count +matched_filter_fingerprint [u8; 32] +created_at Option +``` + +Il ne contient volontairement pas : + +```text +transaction body Yellowstone +transaction meta Yellowstone +is_vote +TransactionStatus.error +raw/provider payload +HTTP material +Store acquisition +``` + +La signature Yellowstone `[u8; 64]` est convertie directement vers `RawTransactionSignature`; aucune Base58 intermédiaire n'est introduite. + +## Adapter `Transaction` + +Le mapping utilise exactement : + +```text +update.slot() +update.transaction().signature() +update.transaction().index() +update.filters() +update.created_at() +``` + +Le body et la meta présents dans `YellowstoneTransactionInfo` restent ignorés jusqu'à l'hydration HTTP qualifiée de `pre.004`. + +## Adapter `TransactionStatus` + +Le mapping utilise exactement : + +```text +update.slot() +update.signature() +update.index() +update.filters() +update.created_at() +``` + +`update.error()` n'est pas consulté. Le remote error opaque n'est donc ni copié dans le signal, ni transformé en contexte Worker, ni loggé. + +## Fingerprint des filtres + +Le fingerprint est Worker-owned, SHA-256 et domain-separated : + +```text +ksp.raw_transaction_ingest.yellowstone.filters.v1\0 +``` + +Les noms sont triés puis dédupliqués avant hashing ; chaque élément est length-prefixed par un `u64` big-endian. Le résultat est donc déterministe pour le même ensemble de filtres, même si l'ordre d'enveloppe diffère. + +Le golden de fixture `filter-a + filter-b` est : + +```text +20831dc8378b6070f91d9de6d3a1d2b3d77298e3ada76c01faa30826ffabcd84 +``` + +Les noms eux-mêmes ne sont pas conservés dans le signal et n'apparaissent pas dans son `Debug`. + +## Route source sûre + +La source Yellowstone convertit dès construction : + +```text +cluster -> RawNetworkId +provider -> RawProvenanceCode +logical endpoint name -> RawProvenanceCode +``` + +Une valeur impossible à représenter selon les bornes Store est rejetée par `worker_raw_transaction_ingest.runtime_invalid` avec une condition statique. Il n'y a ni troncature ni fuite de valeur. + +Cela prépare `pre.004`, où le provider/endpoint HTTP gagnant sera combiné avec cette identité Yellowstone dans la provenance composite retenue par le plan. + +## Absence volontaire de comportement live + +Toujours absents en `pre.003` : + +```text +open_standard_subscribe +next_update +get_transaction_observed +get_block_observed +source task Yellowstone +HTTP hydration +RawTransactionIngress issue du réseau +Store write issue du réseau +coalescence hydration +frontier / reconnect / replay +``` + +L'activation productive de la session reste `pre.006`. + +## Tests et hardening ajoutés + +```text +Transaction fixture -> identité source-neutral exacte +TransactionStatus fixture -> même forme sans payload/error +fingerprint golden exact +fingerprint order/duplicate insensitive +signal Debug redacted +route provider/endpoint non représentable rejetée +source-scan des deux adapters Transport +TransactionStatus.error absent du mapping +signal non public +aucun I/O live prématuré +``` + +## Décisions prises + +```text +un seul signal privé commun aux deux familles +signature canonique conservée en bytes fixes +transaction_index toujours Some pour ces deux DTOs mais champ du signal reste optionnel pour les familles suivantes +filter fingerprint canonique indépendant de l'ordre +gRPC route identity validée tôt pour la future provenance +remote TransactionStatus.error ignoré au boundary Worker +``` + +## Questions ouvertes + +Aucune question ouverte n'est bloquante pour `pre.003`. Les choix HTTP missing/mismatch, provenance composite finale et fan-out/coalescence restent volontairement dans `pre.004`, conformément au plan. + +## Fichiers ajoutés + +```text +deltas/0.3.12/pre.003.md +``` + +## Fichiers modifiés + +```text +Cargo.toml +crates/ksp-worker-raw-transaction-ingest-lib/src/lib.rs +crates/ksp-worker-raw-transaction-ingest-lib/src/runtime_resources.rs +crates/ksp-worker-raw-transaction-ingest-lib/unit_tests/runtime_resources.rs +crates/ksp-worker-raw-transaction-ingest-lib/tests/dependency_boundary.rs +crates/ksp-worker-raw-transaction-ingest-lib/tests/hardening.rs +crates/ksp-worker-raw-transaction-ingest-lib/tests/release_completeness.rs +docs/validation/029-V0_3_12_YELLOWSTONE_HYDRATION_CONTINUITY.md +``` + +## Fichiers supprimés + +Aucun. + +## Validations exécutées dans l'environnement d'assemblage + +```text +python3 scripts/audit_rust_workspace_rules.py +General Rust rule audit: clean +Rust export completeness audit: 0 candidate(s) +KSP workspace Rust rule audit: clean +``` + +L'audit Markdown final est exécuté après création de ce delta. + +`cargo`, `rustc` et `rustfmt` ne sont pas installés dans l'environnement d'assemblage. Aucun gate Cargo local n'est déclaré PASS. + +## Gate opérateur demandé + +```bash +cargo fmt --all +python3 scripts/audit_rust_workspace_rules.py +python3 scripts/audit_markdown_tables.py README.md RULES.md ROADMAP.md CHANGELOG.md docs prompts crates deltas +cargo check --workspace +cargo clippy --workspace --all-targets --all-features -- -D warnings +cargo test -p ksp-worker-raw-transaction-ingest-lib +cargo tree -p ksp-worker-raw-transaction-ingest-lib --edges normal +cargo tree -p ksp-worker-raw-transaction-ingest-lib -e features +cargo tree --duplicates +``` + +## Non-claims + +Cette tranche ne revendique pas : + +```text +session Yellowstone ouverte +update réseau réellement consommée +hydration HTTP +conversion réseau vers Common RAW +provenance composite finale +persistence réseau +Block / BlockMeta / Slot +coalescence +processing frontier +from_slot / ReplayInfo +smoke live Worker +réconciliation README/USAGE/architecture finale +``` + +## Suite + +Après gate opérateur vert, `pre.004` ferme `signal -> get_transaction_observed -> Common RAW -> RawTransactionIngress`, avec `missing`, mismatches et provenance Yellowstone + HTTP observée. Block et la continuité/replay restent hors scope. diff --git a/docs/validation/029-V0_3_12_YELLOWSTONE_HYDRATION_CONTINUITY.md b/docs/validation/029-V0_3_12_YELLOWSTONE_HYDRATION_CONTINUITY.md index dbf76e4..0b9f628 100644 --- a/docs/validation/029-V0_3_12_YELLOWSTONE_HYDRATION_CONTINUITY.md +++ b/docs/validation/029-V0_3_12_YELLOWSTONE_HYDRATION_CONTINUITY.md @@ -1,5 +1,5 @@ - + # Validation v0.3.12 — Yellowstone + hydration HTTP + continuité de run du Worker RawTransaction @@ -648,3 +648,134 @@ réconcilié README/USAGE/architecture pour la surface finale 0.3.12 La tranche suivante reste `pre.003` : adaptation déterministe `Transaction` + `TransactionStatus` vers le signal Worker privé, sans HTTP ni persistence réseau. +## 27. Gate opérateur reçu pour `pre.002` + +L'opérateur a exécuté après application de `pre.002` : + +```text +cargo fmt --all : sans erreur visible +audit Rust : clean +Rust export completeness : 0 candidate +KSP workspace Rust rule audit : clean +audit Markdown : clean (340 tables, 802 fichiers) +cargo check --workspace : PASS visible +cargo clippy --workspace --all-targets --all-features -- -D warnings : PASS visible +cargo test -p ksp-worker-raw-transaction-ingest-lib : 41 PASS, 0 fail +cargo tree Worker normal : exécuté et inspecté +cargo tree Worker features : exécuté et inspecté +cargo tree --duplicates : exécuté ; inventaire informatif des versions dupliquées, sans gate d'unicité artificiel +``` + +Le graphe confirme que le Worker possède directement uniquement l'edge `ksp-onchain-transport-lib`; `reqwest`, `tonic`, `tonic-prost` et `yellowstone-grpc-proto` restent transitifs sous Transport. + +Résultat : `pre.002` est accepté comme base technique de `pre.003`. + +## 28. Gate `pre.003` — signal privé Transaction + TransactionStatus + +La tranche matérialise : + +```text +workspace.package.version = 0.3.12-pre.3 +projection Yellowstone Transaction -> signal Worker privé +projection Yellowstone TransactionStatus -> même signal Worker privé +aucune ouverture de stream +aucune hydration HTTP +aucune persistence réseau +``` + +Le signal reste strictement interne à `runtime_resources.rs` et n'est ni `pub`, ni `pub(crate)`, ni réexporté au crate-root. + +Sa forme source-neutral actuelle est : + +```text +network +signature RawTransactionSignature +slot +transaction_index optionnel +source family Transaction | TransactionStatus +safe Yellowstone source route provider + endpoint_id +matched_filter_count +matched_filter_fingerprint SHA-256 déterministe +created_at source-neutral optionnel seconds+nanos +``` + +Aucun transaction body, meta Yellowstone, `is_vote`, remote error ou payload provider n'est conservé dans ce signal. + +## 29. Projection déterministe des filtres et de la route + +Les filter names Yellowstone sont consommés uniquement pour produire un fingerprint Worker-owned. Le calcul : + +```text +domain = ksp.raw_transaction_ingest.yellowstone.filters.v1\0 +tri lexical des noms +suppression des doublons +nombre de noms uniques encodé u64 big-endian +pour chaque nom : longueur u64 big-endian + bytes UTF-8 +SHA-256 final +``` + +Le signal conserve séparément le nombre de matches reçu afin de ne pas confondre le fingerprint canonique de l'ensemble et la cardinalité de l'enveloppe reçue. + +La source valide aussi dès sa construction que : + +```text +cluster Yellowstone -> RawNetworkId représentable +provider Yellowstone -> RawProvenanceCode représentable +endpoint logical name Yellowstone -> RawProvenanceCode représentable +``` + +Une identité non représentable est rejetée par une condition Worker stable plutôt que tronquée. Cette fermeture prépare la provenance composite de `pre.004` sans modifier Store. + +## 30. Canaris déterministes `pre.003` + +Les nouvelles preuves couvrent : + +```text +fixture Transaction : signature/slot/index/network/route/timestamp exacts +fixture TransactionStatus : même forme sans payload ni remote error +fingerprint golden exact +fingerprint indépendant de l'ordre et des doublons de filter names +Debug du signal sans filter name, signature brute, payload ou remote error +provider/endpoint Yellowstone non représentables rejetés +adapters réels typés YellowstoneTransactionUpdate / YellowstoneTransactionStatusUpdate présents +TransactionStatus.error non consulté par l'adapter +signal absent de la façade publique +aucun open_standard_subscribe / next_update / get_transaction_observed +aucun backend Store / reqwest / tonic / proto direct ajouté +``` + +La tranche ne modifie pas le graphe Cargo du Worker au-delà de celui déjà accepté en `pre.002`. + +## 31. Validation locale et non-claims `pre.003` + +Exécuté dans l'environnement d'assemblage après les modifications Rust : + +```text +python3 scripts/audit_rust_workspace_rules.py +General Rust rule audit: clean +Rust export completeness audit: 0 candidate(s) +KSP workspace Rust rule audit: clean +``` + +L'audit Markdown est rejoué après création du delta final. + +`cargo`, `rustc` et `rustfmt` ne sont pas disponibles dans l'environnement d'assemblage ; aucun gate Cargo local n'est inventé. + +`pre.003` ne prétend pas avoir : + +```text +ouvert une session Yellowstone +consommé une update depuis un réseau live +appelé getTransaction +créé une RawTransactionMaterial depuis le réseau +assemblé la provenance composite Yellowstone + HTTP +persisté une transaction issue de Yellowstone +coalescé les hydrations +traité Block / BlockMeta / Slot +modifié la processing frontier +branché from_slot / ReplayInfo +exécuté un smoke live Worker +``` + +La tranche suivante reste `pre.004` : fermeture déterministe `signal -> get_transaction_observed -> Common RAW -> ingress` avec missing/mismatch et provenance composite, sans Block ni continuité/replay. +