diff --git a/Cargo.toml b/Cargo.toml index b357f3d..7ca4679 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,12 +1,12 @@ # file: Cargo.toml -# version: 522 +# version: 523 [workspace] resolver = "3" members = ["crates/ksp-app-backfill-desk", "crates/ksp-app-config-desk", "crates/ksp-app-solprices-desk", "crates/ksp-app-store-desk", "crates/ksp-app-wallet-desk", "crates/ksp-config-lib", "crates/ksp-core-lib", "crates/ksp-interface-lib", "crates/ksp-job-api", "crates/ksp-job-backfill-lib", "crates/ksp-logging-lib", "crates/ksp-offchain-transport-lib", "crates/ksp-onchain-transport-lib", "crates/ksp-program-api", "crates/ksp-raw-transaction-lib", "crates/ksp-store-api", "crates/ksp-store-lib", "crates/ksp-store-postgres-lib", "crates/ksp-wallet-lib", "crates/ksp-worker-api", "crates/ksp-worker-raw-transaction-ingest-lib"] [workspace.package] -version = "0.3.12-pre.3.fix.2" +version = "0.3.12-pre.4" edition = "2024" license = "MIT" repository = "https://git.sasedev.com/Sasedev/khadhroony-solana-project" diff --git a/crates/ksp-raw-transaction-lib/README.md b/crates/ksp-raw-transaction-lib/README.md index 77650d2..6ed0d05 100644 --- a/crates/ksp-raw-transaction-lib/README.md +++ b/crates/ksp-raw-transaction-lib/README.md @@ -1,5 +1,5 @@ - + # ksp-raw-transaction-lib @@ -26,6 +26,7 @@ La canonicalisation produit un `ksp_store_api::RawTransaction` avec payload KSP ### Signature Solana - `parse_raw_transaction_signature` valide et décode une signature Base58 bornée vers exactement 64 bytes ; +- `format_raw_transaction_signature` encode exactement 64 bytes canoniques vers la forme Base58 bornée correspondante ; - `extract_raw_transaction_signature_from_binary_base64` extrait la première signature canonique d'un wire transactionnel Base64 complet ; - les bornes textuelles publiques permettent l'admission avant décodage. diff --git a/crates/ksp-raw-transaction-lib/USAGE.md b/crates/ksp-raw-transaction-lib/USAGE.md index 3b493c2..e782b50 100644 --- a/crates/ksp-raw-transaction-lib/USAGE.md +++ b/crates/ksp-raw-transaction-lib/USAGE.md @@ -1,5 +1,5 @@ - + # Utilisation de ksp-raw-transaction-lib @@ -17,6 +17,16 @@ fn parse_signature(value: &str) -> ksp_core_lib::Result std::string::String { + return ksp_raw_transaction_lib::format_raw_transaction_signature(value); +} +``` + +Le texte produit respecte les bornes publiques de signature et effectue un round-trip exact avec `parse_raw_transaction_signature`. + Lorsqu'un wire Base64 complet contient déjà son tableau de signatures, utiliser : ```rust diff --git a/crates/ksp-raw-transaction-lib/src/lib.rs b/crates/ksp-raw-transaction-lib/src/lib.rs index 5c36a8f..b964565 100644 --- a/crates/ksp-raw-transaction-lib/src/lib.rs +++ b/crates/ksp-raw-transaction-lib/src/lib.rs @@ -1,5 +1,5 @@ // file: crates/ksp-raw-transaction-lib/src/lib.rs -// version: 3 +// version: 4 #![warn(missing_docs)] #![deny(unreachable_pub)] @@ -7,7 +7,7 @@ //! Source-neutral canonicalization of complete Solana transactions into KSP RAW transaction v1. //! -//! This lower layer owns the frozen RAW transaction format, textual signature parsing, canonical +//! This lower layer owns the frozen RAW transaction format, textual signature parsing/formatting, canonical //! JSON construction, content hashing and assembly of one canonical transaction with one //! producer-owned observation. It owns no Transport, Config, Job, Worker, async runtime or Store //! backend behavior. @@ -46,6 +46,8 @@ pub use self::signature::MAX_RAW_TRANSACTION_SIGNATURE_TEXT_BYTES; pub use self::signature::MIN_RAW_TRANSACTION_SIGNATURE_TEXT_BYTES; /// Extracts the first canonical 64-byte Solana signature from one complete Base64 transaction wire. pub use self::signature::extract_raw_transaction_signature_from_binary_base64; +/// Encodes one canonical 64-byte Solana transaction signature as bounded Base58 text. +pub use self::signature::format_raw_transaction_signature; /// Parses one bounded Base58 Solana transaction signature to exactly 64 canonical bytes. pub use self::signature::parse_raw_transaction_signature; /// One source-neutral v0 address-table lookup. diff --git a/crates/ksp-raw-transaction-lib/src/signature.rs b/crates/ksp-raw-transaction-lib/src/signature.rs index c9e3e22..2bff7ce 100644 --- a/crates/ksp-raw-transaction-lib/src/signature.rs +++ b/crates/ksp-raw-transaction-lib/src/signature.rs @@ -1,5 +1,5 @@ // file: crates/ksp-raw-transaction-lib/src/signature.rs -// version: 3 +// version: 4 use base64::Engine; // rust-rules: trait-import @@ -8,6 +8,35 @@ pub const MAX_RAW_TRANSACTION_SIGNATURE_TEXT_BYTES: usize = 88; /// Minimum UTF-8 byte length admitted for one textual Base58 Solana transaction signature. pub const MIN_RAW_TRANSACTION_SIGNATURE_TEXT_BYTES: usize = 64; +/// Encodes one canonical 64-byte Solana transaction signature as bounded Base58 text. +#[must_use] +pub fn format_raw_transaction_signature(signature: &ksp_store_api::RawTransactionSignature) -> std::string::String { + const ALPHABET: &[u8; 58] = b"123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz"; + let bytes = signature.as_bytes(); + let leading_zeroes = bytes.iter().take_while(|byte| return **byte == 0).count(); + let mut digits = std::vec::Vec::with_capacity(crate::MAX_RAW_TRANSACTION_SIGNATURE_TEXT_BYTES); + for byte in bytes { + let mut carry = u32::from(*byte); + for digit in &mut digits { + let expanded = (u32::from(*digit) * 256) + carry; + *digit = (expanded % 58) as u8; + carry = expanded / 58; + } + while carry != 0 { + digits.push((carry % 58) as u8); + carry /= 58; + } + } + let mut output = std::string::String::with_capacity(leading_zeroes + digits.len()); + for _ in 0..leading_zeroes { + output.push('1'); + } + for digit in digits.iter().rev() { + output.push(char::from(ALPHABET[usize::from(*digit)])); + } + return output; +} + /// Parses one bounded Base58 Solana transaction signature to exactly 64 canonical bytes. pub fn parse_raw_transaction_signature(value: &str) -> ksp_core_lib::Result { let text = value.as_bytes(); diff --git a/crates/ksp-raw-transaction-lib/tests/public_api.rs b/crates/ksp-raw-transaction-lib/tests/public_api.rs index 7ae22ce..0a26fb4 100644 --- a/crates/ksp-raw-transaction-lib/tests/public_api.rs +++ b/crates/ksp-raw-transaction-lib/tests/public_api.rs @@ -1,5 +1,5 @@ // file: crates/ksp-raw-transaction-lib/tests/public_api.rs -// version: 4 +// version: 5 //! Integration canaries for the public common RAW Transaction crate-root surface. @@ -120,3 +120,16 @@ fn pre_006_source_neutral_wire_contract_is_available_from_crate_root() { } return; } + +#[test] +fn pre_004_signature_formatter_is_consumable_from_crate_root() { + let signature = ksp_store_api::RawTransactionSignature::new([0_u8; 64]); + let text = ksp_raw_transaction_lib::format_raw_transaction_signature(&signature); + assert_eq!(text, "1".repeat(64)); + let parsed = ksp_raw_transaction_lib::parse_raw_transaction_signature(text.as_str()); + assert!(parsed.is_ok()); + if let std::result::Result::Ok(parsed) = parsed { + assert_eq!(parsed, signature); + } + return; +} diff --git a/crates/ksp-raw-transaction-lib/unit_tests/signature.rs b/crates/ksp-raw-transaction-lib/unit_tests/signature.rs index a66d5e1..d033649 100644 --- a/crates/ksp-raw-transaction-lib/unit_tests/signature.rs +++ b/crates/ksp-raw-transaction-lib/unit_tests/signature.rs @@ -1,5 +1,5 @@ // file: crates/ksp-raw-transaction-lib/unit_tests/signature.rs -// version: 3 +// version: 4 use base64::Engine; // rust-rules: trait-import @@ -79,3 +79,19 @@ fn pre_006_embedded_signature_extracts_terminal_v1_signature_and_rejects_trailin assert!(crate::extract_raw_transaction_signature_from_binary_base64(trailing.as_str()).is_err()); } } + +#[test] +fn pre_004_base58_formatter_round_trips_canonical_signatures() { + for bytes in [[0_u8; 64], [1_u8; 64], [0xAB_u8; 64]] { + let signature = ksp_store_api::RawTransactionSignature::new(bytes); + let text = crate::format_raw_transaction_signature(&signature); + assert!((crate::MIN_RAW_TRANSACTION_SIGNATURE_TEXT_BYTES..=crate::MAX_RAW_TRANSACTION_SIGNATURE_TEXT_BYTES).contains(&text.len())); + let parsed = crate::parse_raw_transaction_signature(text.as_str()); + assert!(parsed.is_ok()); + if let std::result::Result::Ok(parsed) = parsed { + assert_eq!(parsed, signature); + } + } + assert_eq!(crate::format_raw_transaction_signature(&ksp_store_api::RawTransactionSignature::new([0_u8; 64])), "1".repeat(64)); + return; +} diff --git a/crates/ksp-worker-raw-transaction-ingest-lib/src/lib.rs b/crates/ksp-worker-raw-transaction-ingest-lib/src/lib.rs index 5957f47..40b62c0 100644 --- a/crates/ksp-worker-raw-transaction-ingest-lib/src/lib.rs +++ b/crates/ksp-worker-raw-transaction-ingest-lib/src/lib.rs @@ -1,5 +1,5 @@ // file: crates/ksp-worker-raw-transaction-ingest-lib/src/lib.rs -// version: 10 +// version: 11 #![warn(missing_docs)] #![deny(unreachable_pub)] @@ -11,8 +11,9 @@ //! and the caller-runtime-owned lifecycle with private child-task supervision. This tranche also //! owns bounded source-neutral admission, common RAW canonicalization/assembly and backend-neutral //! Store persistence in normal mode plus concrete latest-value snapshots projected onto Worker API; -//! the first validated Yellowstone/HTTP runtime-resource contract exists and deterministically projects -//! Transaction/TransactionStatus updates into private source-neutral signals, but no live stream is opened yet. +//! the first validated Yellowstone/HTTP runtime-resource contract also closes the deterministic hydration +//! qualification path in test configuration. Transaction/TransactionStatus signal projection and HTTP hydration +//! remain inactive in production until the productive source task is wired; no live stream is opened yet. mod admission; mod error; diff --git a/crates/ksp-worker-raw-transaction-ingest-lib/src/runtime_resources.rs b/crates/ksp-worker-raw-transaction-ingest-lib/src/runtime_resources.rs index 9f7aa40..98c5ba5 100644 --- a/crates/ksp-worker-raw-transaction-ingest-lib/src/runtime_resources.rs +++ b/crates/ksp-worker-raw-transaction-ingest-lib/src/runtime_resources.rs @@ -1,11 +1,15 @@ // file: crates/ksp-worker-raw-transaction-ingest-lib/src/runtime_resources.rs -// version: 3 +// version: 4 #[cfg(test)] use sha2::Digest; // rust-rules: trait-import #[cfg(test)] const RAW_TRANSACTION_INGEST_YELLOWSTONE_FILTER_FINGERPRINT_DOMAIN: &[u8] = b"ksp.raw_transaction_ingest.yellowstone.filters.v1\0"; +#[cfg(test)] +const RAW_TRANSACTION_INGEST_YELLOWSTONE_HTTP_PROTOCOL: &str = "yellowstone_http"; +#[cfg(test)] +const RAW_TRANSACTION_INGEST_YELLOWSTONE_HTTP_SOURCE_KEY_DOMAIN: &[u8] = b"ksp.raw_transaction_ingest.yellowstone_http.source_key.v1\0"; #[cfg(test)] #[derive(Clone, Copy, Debug, Eq, PartialEq)] @@ -41,6 +45,7 @@ struct RawTransactionIngestSourceSignal { family: RawTransactionIngestSourceFamily, matched_filter_count: usize, matched_filter_fingerprint: [u8; 32], + matched_filter_id: std::option::Option, network: ksp_store_lib::RawNetworkId, route: RawTransactionIngestSourceRoute, signature: ksp_store_lib::RawTransactionSignature, @@ -57,6 +62,7 @@ impl std::fmt::Debug for RawTransactionIngestSourceSignal { .field("family", &self.family) .field("matched_filter_count", &self.matched_filter_count) .field("matched_filter_fingerprint_bytes", &self.matched_filter_fingerprint.len()) + .field("has_direct_filter_id", &self.matched_filter_id.is_some()) .field("network", &self.network) .field("route", &self.route) .field("signature_bytes", &self.signature.as_bytes().len()) @@ -199,11 +205,12 @@ impl crate::RawTransactionIngestYellowstoneSource { std::result::Result::Ok(value) => value, std::result::Result::Err(_) => return std::result::Result::Err(crate::runtime_error("runtime_resources.yellowstone_endpoint_unrepresentable")), }; + let route = RawTransactionIngestSourceRoute { endpoint_id, provider }; let method = match ksp_onchain_transport_lib::find_http_rpc_method("getTransaction") { std::option::Option::Some(value) => value, std::option::Option::None => return std::result::Result::Err(crate::runtime_error("runtime_resources.hydration_method_missing")), }; - let compatible_http_routes = compatible_http_route_count(&http_pool, &hydration_role, method.request_kind(), network.as_str()); + let compatible_http_routes = compatible_http_route_count(&http_pool, &hydration_role, method.request_kind(), network.as_str(), &route); let compatible_http_routes = match compatible_http_routes { std::result::Result::Ok(value) => value, std::result::Result::Err(error) => return std::result::Result::Err(error), @@ -211,7 +218,6 @@ impl crate::RawTransactionIngestYellowstoneSource { if compatible_http_routes == 0 { return std::result::Result::Err(crate::runtime_error("runtime_resources.hydration_role_unsupported")); } - let route = RawTransactionIngestSourceRoute { endpoint_id, provider }; return std::result::Result::Ok(Self { yellowstone_channel, subscribe_request, http_pool, hydration_role, network, route }); } } @@ -271,6 +277,7 @@ fn compatible_http_route_count( hydration_role: &ksp_onchain_transport_lib::HttpRoleName, request_kind: &str, expected_cluster: &str, + yellowstone_route: &RawTransactionIngestSourceRoute, ) -> ksp_core_lib::Result { let snapshot = pool.snapshot(); let mut compatible = 0_usize; @@ -289,16 +296,56 @@ fn compatible_http_route_count( if endpoint.cluster() != expected_cluster { return std::result::Result::Err(crate::runtime_error("runtime_resources.transport_network_mismatch")); } + let composite = composite_provenance_codes(yellowstone_route, endpoint.provider(), endpoint.name()); + if let std::result::Result::Err(error) = composite { + return std::result::Result::Err(error); + } compatible = compatible.saturating_add(1); } } return std::result::Result::Ok(compatible); } +fn composite_provenance_codes( + yellowstone_route: &RawTransactionIngestSourceRoute, + http_provider: &str, + http_endpoint: &str, +) -> ksp_core_lib::Result<(ksp_store_lib::RawProvenanceCode, ksp_store_lib::RawProvenanceCode)> { + let provider = std::format!("ys.{}:http.{}", yellowstone_route.provider.as_str(), http_provider); + let provider = match ksp_store_lib::RawProvenanceCode::new(provider) { + std::result::Result::Ok(value) => value, + std::result::Result::Err(_) => { + return std::result::Result::Err(crate::runtime_error("runtime_resources.composite_provider_unrepresentable")); + }, + }; + let endpoint = std::format!("ys.{}:http.{}", yellowstone_route.endpoint_id.as_str(), http_endpoint); + let endpoint = match ksp_store_lib::RawProvenanceCode::new(endpoint) { + std::result::Result::Ok(value) => value, + std::result::Result::Err(_) => { + return std::result::Result::Err(crate::runtime_error("runtime_resources.composite_endpoint_unrepresentable")); + }, + }; + return std::result::Result::Ok((provider, endpoint)); +} + fn ingestion_filter_count(request: &ksp_onchain_transport_lib::YellowstoneSubscribeRequest) -> usize { return request.transaction_filter_count().saturating_add(request.transaction_status_filter_count()).saturating_add(request.block_filter_count()); } +#[cfg(test)] +fn matched_filter_direct_id(filters: &[ksp_onchain_transport_lib::YellowstoneSubscribeFilterName]) -> std::option::Option { + let mut names = filters.iter().map(ksp_onchain_transport_lib::YellowstoneSubscribeFilterName::as_str).collect::>(); + names.sort_unstable(); + names.dedup(); + if names.len() != 1 { + return std::option::Option::None; + } + return match ksp_store_lib::RawProvenanceCode::new(names[0]) { + std::result::Result::Ok(value) => std::option::Option::Some(value), + std::result::Result::Err(_) => std::option::Option::None, + }; +} + #[cfg(test)] fn matched_filter_fingerprint(filters: &[ksp_onchain_transport_lib::YellowstoneSubscribeFilterName]) -> [u8; 32] { let mut names = filters.iter().map(ksp_onchain_transport_lib::YellowstoneSubscribeFilterName::as_str).collect::>(); @@ -328,6 +375,7 @@ fn project_yellowstone_signal( family: update.family(), matched_filter_count: update.filters().len(), matched_filter_fingerprint: matched_filter_fingerprint(update.filters()), + matched_filter_id: matched_filter_direct_id(update.filters()), network: source.network.clone(), route: source.route.clone(), signature, @@ -336,6 +384,264 @@ fn project_yellowstone_signal( }; } +#[cfg(test)] +enum RawTransactionIngestHydrationOutcome { + Available(crate::RawTransactionIngress), + Missing(ksp_store_lib::RawTransactionReference), +} + +#[cfg(test)] +async fn hydrate_yellowstone_signal( + source: &crate::RawTransactionIngestYellowstoneSource, + settings: &crate::RawTransactionIngestSettings, + signal: RawTransactionIngestSourceSignal, + received_at: ksp_store_lib::RawTimestamp, +) -> ksp_core_lib::Result { + if &signal.network != settings.network() || signal.network != source.network { + return std::result::Result::Err(crate::runtime_error("hydration.network_mismatch")); + } + if signal.route != source.route { + return std::result::Result::Err(crate::runtime_error("hydration.source_route_mismatch")); + } + let commitment = match source.subscribe_request.commitment() { + std::option::Option::Some(ksp_onchain_transport_lib::SolanaCommitment::Confirmed) => ksp_onchain_transport_lib::SolanaCommitment::Confirmed, + std::option::Option::Some(ksp_onchain_transport_lib::SolanaCommitment::Finalized) => ksp_onchain_transport_lib::SolanaCommitment::Finalized, + std::option::Option::Some(ksp_onchain_transport_lib::SolanaCommitment::Processed) | std::option::Option::None => { + return std::result::Result::Err(crate::runtime_error("hydration.commitment_invalid")); + }, + }; + let config = ksp_onchain_transport_lib::SolanaGetTransactionConfig::new( + std::option::Option::Some(commitment), + std::option::Option::Some(ksp_onchain_transport_lib::SolanaTransactionEncoding::Base64), + std::option::Option::Some(0), + ); + let signature_text = ksp_raw_transaction_lib::format_raw_transaction_signature(&signal.signature); + let observed = source.http_pool.get_transaction_observed(&source.hydration_role, signature_text.as_str(), std::option::Option::Some(&config)).await; + let observed = match observed { + std::result::Result::Ok(value) => value, + std::result::Result::Err(error) => return std::result::Result::Err(hydration_transport_error(error.code())), + }; + let http_provider = observed.provider().as_str().to_owned(); + let http_endpoint = observed.endpoint_name().to_owned(); + let transaction = observed.into_value(); + let transaction = match transaction { + std::option::Option::Some(value) => value, + std::option::Option::None => { + let reference = ksp_store_lib::RawTransactionReference::new(signal.network, signal.signature); + return std::result::Result::Ok(RawTransactionIngestHydrationOutcome::Missing(reference)); + }, + }; + if transaction.slot() != signal.slot { + return std::result::Result::Err(crate::runtime_error("hydration.slot_mismatch")); + } + if let std::option::Option::Some(expected_index) = signal.transaction_index { + if let ksp_onchain_transport_lib::SolanaWireField::Value(actual_index) = transaction.transaction_index() { + if expected_index != u64::from(*actual_index) { + return std::result::Result::Err(crate::runtime_error("hydration.transaction_index_mismatch")); + } + } + } + let transaction_data = match transaction.transaction() { + ksp_onchain_transport_lib::SolanaEncodedTransaction::Binary { data, encoding } + if *encoding == ksp_onchain_transport_lib::SolanaTransactionBinaryEncoding::Base64 => + { + data.clone() + }, + ksp_onchain_transport_lib::SolanaEncodedTransaction::Binary { .. } + | ksp_onchain_transport_lib::SolanaEncodedTransaction::LegacyBinary(_) + | ksp_onchain_transport_lib::SolanaEncodedTransaction::Json(_) => { + return std::result::Result::Err(crate::runtime_error("hydration.transaction_encoding_invalid")); + }, + }; + let embedded_signature = ksp_raw_transaction_lib::extract_raw_transaction_signature_from_binary_base64(transaction_data.as_str()); + let embedded_signature = match embedded_signature { + std::result::Result::Ok(value) => value, + std::result::Result::Err(_) => return std::result::Result::Err(crate::runtime_error("hydration.transaction_signature_invalid")), + }; + if embedded_signature != signal.signature { + return std::result::Result::Err(crate::runtime_error("hydration.signature_mismatch")); + } + let provenance = build_hydration_provenance(settings, &signal, http_provider.as_str(), http_endpoint.as_str(), commitment, received_at); + let provenance = match provenance { + std::result::Result::Ok(value) => value, + std::result::Result::Err(error) => return std::result::Result::Err(error), + }; + let source_key = hydration_source_key(&provenance); + let material = ksp_raw_transaction_lib::RawTransactionMaterial::binary_base64( + signal.network.clone(), + signal.signature, + transaction.slot(), + transaction.block_time(), + transaction_data, + map_hydration_wire_field(transaction.meta(), |value| return value.clone()), + map_hydration_wire_field(transaction.version(), |value| { + return match value { + ksp_onchain_transport_lib::SolanaTransactionVersion::Legacy => ksp_raw_transaction_lib::RawTransactionVersion::Legacy, + ksp_onchain_transport_lib::SolanaTransactionVersion::Number(number) => ksp_raw_transaction_lib::RawTransactionVersion::Number(*number), + }; + }), + map_hydration_wire_field(transaction.transaction_index(), |value| return *value), + ); + return std::result::Result::Ok(RawTransactionIngestHydrationOutcome::Available(crate::RawTransactionIngress { + material, + network: signal.network, + provenance, + source_key, + })); +} + +#[cfg(test)] +fn build_hydration_provenance( + settings: &crate::RawTransactionIngestSettings, + signal: &RawTransactionIngestSourceSignal, + http_provider: &str, + http_endpoint: &str, + commitment: ksp_onchain_transport_lib::SolanaCommitment, + received_at: ksp_store_lib::RawTimestamp, +) -> ksp_core_lib::Result { + let (provider, endpoint_id) = match composite_provenance_codes(&signal.route, http_provider, http_endpoint) { + std::result::Result::Ok(value) => value, + std::result::Result::Err(error) => return std::result::Result::Err(error), + }; + let protocol = match ksp_store_lib::RawProvenanceCode::new(RAW_TRANSACTION_INGEST_YELLOWSTONE_HTTP_PROTOCOL) { + std::result::Result::Ok(value) => value, + std::result::Result::Err(_) => return std::result::Result::Err(crate::runtime_error("hydration.protocol_unrepresentable")), + }; + let acquisition_method = match ksp_store_lib::RawProvenanceCode::new(hydration_method_code(signal.family)) { + std::result::Result::Ok(value) => value, + std::result::Result::Err(_) => return std::result::Result::Err(crate::runtime_error("hydration.method_unrepresentable")), + }; + let capture_session = match ksp_store_lib::RawProvenanceCode::new(settings.worker_id().as_str()) { + std::result::Result::Ok(value) => value, + std::result::Result::Err(_) => return std::result::Result::Err(crate::runtime_error("hydration.capture_session_unrepresentable")), + }; + let commitment = match ksp_store_lib::RawProvenanceCode::new(commitment.as_str()) { + std::result::Result::Ok(value) => value, + std::result::Result::Err(_) => return std::result::Result::Err(crate::runtime_error("hydration.commitment_unrepresentable")), + }; + let filter_id = match signal.matched_filter_id.clone() { + std::option::Option::Some(value) => value, + std::option::Option::None => match fingerprint_filter_code(&signal.matched_filter_fingerprint) { + std::result::Result::Ok(value) => value, + std::result::Result::Err(error) => return std::result::Result::Err(error), + }, + }; + let mut provenance = + ksp_store_lib::RawAcquisitionProvenance::new(provider, protocol, acquisition_method, ksp_store_lib::RawAcquisitionOrigin::Live, received_at) + .with_endpoint_id(endpoint_id) + .with_commitment(commitment) + .with_capture_session_id(capture_session) + .with_filter_id(filter_id); + if let std::option::Option::Some(observed_at) = representable_observed_at(signal.created_at, received_at) { + provenance = match provenance.try_with_observed_at(observed_at) { + std::result::Result::Ok(value) => value, + std::result::Result::Err(_) => return std::result::Result::Err(crate::runtime_error("hydration.observed_at_invalid")), + }; + } + return std::result::Result::Ok(provenance); +} + +#[cfg(test)] +fn hydration_method_code(family: RawTransactionIngestSourceFamily) -> &'static str { + return match family { + RawTransactionIngestSourceFamily::Transaction => "transaction_get_transaction", + RawTransactionIngestSourceFamily::TransactionStatus => "status_get_transaction", + }; +} + +#[cfg(test)] +fn fingerprint_filter_code(fingerprint: &[u8; 32]) -> ksp_core_lib::Result { + const HEX: &[u8; 16] = b"0123456789abcdef"; + let mut value = std::string::String::with_capacity(71); + value.push_str("sha256."); + for byte in fingerprint { + value.push(char::from(HEX[usize::from(*byte >> 4)])); + value.push(char::from(HEX[usize::from(*byte & 0x0f)])); + } + return match ksp_store_lib::RawProvenanceCode::new(value) { + std::result::Result::Ok(value) => std::result::Result::Ok(value), + std::result::Result::Err(_) => std::result::Result::Err(crate::runtime_error("hydration.filter_fingerprint_unrepresentable")), + }; +} + +#[cfg(test)] +fn representable_observed_at( + created_at: std::option::Option, + received_at: ksp_store_lib::RawTimestamp, +) -> std::option::Option { + let created_at = match created_at { + std::option::Option::Some(value) => value, + std::option::Option::None => return std::option::Option::None, + }; + let seconds = match u64::try_from(created_at.seconds) { + std::result::Result::Ok(value) => value, + std::result::Result::Err(_) => return std::option::Option::None, + }; + let millis = match seconds.checked_mul(1_000).and_then(|value| return value.checked_add(u64::from(created_at.nanos / 1_000_000))) { + std::option::Option::Some(value) => value, + std::option::Option::None => return std::option::Option::None, + }; + let observed_at = match ksp_store_lib::RawTimestamp::from_unix_millis(millis) { + std::result::Result::Ok(value) => value, + std::result::Result::Err(_) => return std::option::Option::None, + }; + if observed_at > received_at { + return std::option::Option::None; + } + return std::option::Option::Some(observed_at); +} + +#[cfg(test)] +fn hydration_source_key(provenance: &ksp_store_lib::RawAcquisitionProvenance) -> [u8; 32] { + let mut hasher = sha2::Sha256::new(); + hasher.update(RAW_TRANSACTION_INGEST_YELLOWSTONE_HTTP_SOURCE_KEY_DOMAIN); + hash_hydration_source_key(&mut hasher, provenance.provider().as_str().as_bytes()); + hash_hydration_source_key(&mut hasher, provenance.protocol().as_str().as_bytes()); + hash_hydration_source_key(&mut hasher, provenance.acquisition_method().as_str().as_bytes()); + if let std::option::Option::Some(value) = provenance.endpoint_id() { + hash_hydration_source_key(&mut hasher, value.as_str().as_bytes()); + } + if let std::option::Option::Some(value) = provenance.commitment() { + hash_hydration_source_key(&mut hasher, value.as_str().as_bytes()); + } + if let std::option::Option::Some(value) = provenance.capture_session_id() { + hash_hydration_source_key(&mut hasher, value.as_str().as_bytes()); + } + if let std::option::Option::Some(value) = provenance.filter_id() { + hash_hydration_source_key(&mut hasher, value.as_str().as_bytes()); + } + return hasher.finalize().into(); +} + +#[cfg(test)] +fn hash_hydration_source_key(hasher: &mut sha2::Sha256, value: &[u8]) { + hasher.update((value.len() as u64).to_be_bytes()); + hasher.update(value); + return; +} + +#[cfg(test)] +fn hydration_transport_error(code: ksp_core_lib::ErrorCode) -> ksp_core_lib::Error { + return ksp_core_lib::Error::new(crate::ERROR_CODE_RAW_TRANSACTION_INGEST_SOURCE_FAILED, "RAW transaction ingest hydration transport failed") + .with_context("transport_domain", code.domain()) + .with_context("transport_code", code.code()); +} + +#[cfg(test)] +fn map_hydration_wire_field( + field: &ksp_onchain_transport_lib::SolanaWireField, + mut map_value: F, +) -> ksp_raw_transaction_lib::RawTransactionWireField +where + F: FnMut(&T) -> U, +{ + return match field { + ksp_onchain_transport_lib::SolanaWireField::Omitted => ksp_raw_transaction_lib::RawTransactionWireField::Omitted, + ksp_onchain_transport_lib::SolanaWireField::Null => ksp_raw_transaction_lib::RawTransactionWireField::Null, + ksp_onchain_transport_lib::SolanaWireField::Value(value) => ksp_raw_transaction_lib::RawTransactionWireField::Value(map_value(value)), + }; +} + #[cfg(test)] #[path = "../unit_tests/runtime_resources.rs"] mod tests; diff --git a/crates/ksp-worker-raw-transaction-ingest-lib/tests/dependency_boundary.rs b/crates/ksp-worker-raw-transaction-ingest-lib/tests/dependency_boundary.rs index 98c4098..dd4031e 100644 --- a/crates/ksp-worker-raw-transaction-ingest-lib/tests/dependency_boundary.rs +++ b/crates/ksp-worker-raw-transaction-ingest-lib/tests/dependency_boundary.rs @@ -1,5 +1,5 @@ // file: crates/ksp-worker-raw-transaction-ingest-lib/tests/dependency_boundary.rs -// version: 10 +// version: 11 //! Dependency firewall canaries for the RAW transaction ingest Worker foundation. @@ -87,7 +87,6 @@ fn v0_3_12_pre_003_transaction_and_status_adapters_are_private_offline_and_trans "pub use self::runtime_resources::RawTransactionIngestSourceSignal", "open_standard_subscribe", "next_update", - "get_transaction_observed", "get_block_observed", "ksp_store_postgres_lib::", "reqwest::", @@ -108,6 +107,43 @@ fn v0_3_12_pre_003_transaction_and_status_adapters_are_private_offline_and_trans return; } +#[test] +fn v0_3_12_pre_004_hydration_contract_uses_only_transport_facade_common_raw_and_existing_ingress() { + let root = include_str!("../src/lib.rs"); + let resources = include_str!("../src/runtime_resources.rs"); + for required in [ + "get_transaction_observed", + "SolanaGetTransactionConfig::new", + "SolanaTransactionEncoding::Base64", + "format_raw_transaction_signature", + "extract_raw_transaction_signature_from_binary_base64", + "RawTransactionMaterial::binary_base64", + "RawTransactionIngress", + "yellowstone_http", + "transaction_get_transaction", + "status_get_transaction", + "RawAcquisitionOrigin::Live", + "composite_provenance_codes", + ] { + assert!(resources.contains(required), "required pre.004 hydration contract missing: {required}"); + } + assert!(resources.contains("#[cfg(test)]\nasync fn hydrate_yellowstone_signal")); + for forbidden in [ + "open_standard_subscribe", + "next_update", + "get_block_observed", + "ksp_config_lib::", + "ksp_job_backfill_lib::", + "ksp_store_postgres_lib::", + "reqwest::", + "tonic::", + "yellowstone_grpc_proto::", + ] { + assert!(!resources.contains(forbidden) && !root.contains(forbidden), "pre.004 crossed a forbidden boundary: {forbidden}"); + } + return; +} + #[test] fn v0_3_12_pre_002_source_surface_hardens_shutdown_and_faults_without_backend_or_premature_live_io() { let root = include_str!("../src/lib.rs"); diff --git a/crates/ksp-worker-raw-transaction-ingest-lib/tests/hardening.rs b/crates/ksp-worker-raw-transaction-ingest-lib/tests/hardening.rs index 65d3c4b..dd2c376 100644 --- a/crates/ksp-worker-raw-transaction-ingest-lib/tests/hardening.rs +++ b/crates/ksp-worker-raw-transaction-ingest-lib/tests/hardening.rs @@ -1,5 +1,5 @@ // file: crates/ksp-worker-raw-transaction-ingest-lib/tests/hardening.rs -// version: 5 +// version: 6 //! External public, security, redaction and release-boundary hardening canaries for `pre.010`. @@ -301,6 +301,7 @@ fn v0_3_12_pre_003_private_signal_debug_and_shape_do_not_expose_signature_filter "family:", "matched_filter_count:", "matched_filter_fingerprint:", + "matched_filter_id:", "network:", "route:", "signature:", @@ -312,13 +313,40 @@ fn v0_3_12_pre_003_private_signal_debug_and_shape_do_not_expose_signature_filter return; } +#[test] +fn v0_3_12_pre_004_hydration_provenance_and_remote_material_are_bounded_and_redacted() { + let resources = include_str!("../src/runtime_resources.rs"); + for required in [ + "ys.{}:http.{}", + "composite_provider_unrepresentable", + "composite_endpoint_unrepresentable", + "RawAcquisitionOrigin::Live", + "with_capture_session_id", + "with_commitment", + "with_endpoint_id", + "with_filter_id", + "try_with_observed_at", + "hydration.signature_mismatch", + "hydration.slot_mismatch", + "hydration.transaction_index_mismatch", + ] { + assert!(resources.contains(required), "required pre.004 bounded provenance/mismatch guard missing: {required}"); + } + for forbidden in ["source_payload_hash", "source_payload_size_bytes", "HTTP-SECRET-CANARY", "GRPC-SECRET-CANARY", "TransactionStatus.error", ".error()"] { + assert!(!resources.contains(forbidden), "pre.004 retained forbidden remote/source material: {forbidden}"); + } + return; +} + #[test] fn v0_3_12_pre_002_runtime_resource_contract_performs_no_live_io_or_source_spawn() { let resources = include_str!("../src/runtime_resources.rs"); let runtime = include_str!("../src/runtime.rs"); - for forbidden in ["open_standard_subscribe", "next_update", "get_transaction_observed", "get_block_observed", "tokio::spawn", "JoinSet"] { - assert!(!resources.contains(forbidden), "pre.002 runtime-resource contract opened premature live behavior: {forbidden}"); + for forbidden in ["open_standard_subscribe", "next_update", "get_block_observed", "tokio::spawn", "JoinSet"] { + assert!(!resources.contains(forbidden), "runtime-resource contract opened premature live source behavior: {forbidden}"); } + assert!(resources.contains("#[cfg(test)]\nasync fn hydrate_yellowstone_signal")); + assert!(resources.contains("get_transaction_observed")); assert!(runtime.contains("start_with_runtime_resources")); for forbidden in ["open_standard_subscribe", "next_update", "get_transaction_observed", "get_block_observed"] { assert!(!runtime.contains(forbidden), "pre.002 runtime start opened premature live behavior: {forbidden}"); diff --git a/crates/ksp-worker-raw-transaction-ingest-lib/tests/release_completeness.rs b/crates/ksp-worker-raw-transaction-ingest-lib/tests/release_completeness.rs index 9acf226..e021813 100644 --- a/crates/ksp-worker-raw-transaction-ingest-lib/tests/release_completeness.rs +++ b/crates/ksp-worker-raw-transaction-ingest-lib/tests/release_completeness.rs @@ -1,5 +1,5 @@ // file: crates/ksp-worker-raw-transaction-ingest-lib/tests/release_completeness.rs -// version: 3 +// version: 4 //! Release-completeness canaries through the `pre.010` public/release/security hardening tranche. @@ -99,6 +99,7 @@ fn pre_010_external_hardening_suite_is_present_and_scoped() { "pre_010_debug_and_settings_errors_redact_worker_identity_and_invalid_values", "v0_3_12_pre_002_manifest_dependency_surface_opens_only_transport_and_remains_backend_neutral", "v0_3_12_pre_003_private_signal_debug_and_shape_do_not_expose_signature_filters_or_payload", + "v0_3_12_pre_004_hydration_provenance_and_remote_material_are_bounded_and_redacted", "pre_010_source_visibility_contract_uses_crate_root_for_shared_items", "pre_010_production_surface_has_no_historical_backfill_or_retriever_contract", "v0_3_12_pre_002_production_sources_keep_transport_confined_to_runtime_resources", @@ -113,6 +114,7 @@ fn pre_010_external_hardening_suite_is_present_and_scoped() { assert!(dependency_boundary.contains("v0_3_12_pre_002_source_surface_hardens_shutdown_and_faults_without_backend_or_premature_live_io")); assert!(dependency_boundary.contains("v0_3_12_pre_002_manifest_opens_only_the_onchain_transport_live_source_edge")); assert!(dependency_boundary.contains("v0_3_12_pre_003_transaction_and_status_adapters_are_private_offline_and_transport_facade_only")); + assert!(dependency_boundary.contains("v0_3_12_pre_004_hydration_contract_uses_only_transport_facade_common_raw_and_existing_ingress")); let public_api = include_str!("public_api.rs"); assert!(public_api.contains("pre_003_kind_code_and_settings_are_consumable_from_crate_root")); assert!(public_api.contains("pre_004_start_handle_and_terminal_future_are_consumable_without_public_join_handle")); diff --git a/crates/ksp-worker-raw-transaction-ingest-lib/unit_tests/runtime_resources.rs b/crates/ksp-worker-raw-transaction-ingest-lib/unit_tests/runtime_resources.rs index 57a4e32..2000767 100644 --- a/crates/ksp-worker-raw-transaction-ingest-lib/unit_tests/runtime_resources.rs +++ b/crates/ksp-worker-raw-transaction-ingest-lib/unit_tests/runtime_resources.rs @@ -1,5 +1,5 @@ // file: crates/ksp-worker-raw-transaction-ingest-lib/unit_tests/runtime_resources.rs -// version: 2 +// version: 3 fn grpc_endpoint(cluster: &str) -> std::option::Option { return grpc_endpoint_with_identity(cluster, "yellowstone-fixture", "fixture-provider"); @@ -147,6 +147,199 @@ fn signal_source() -> std::option::Option std::option::Option { + let url = match ksp_onchain_transport_lib::HttpEndpointUrl::parse(url) { + std::result::Result::Ok(value) => value, + std::result::Result::Err(_) => return std::option::Option::None, + }; + let role = ksp_onchain_transport_lib::HttpEndpointRoleSettings::new( + ksp_onchain_transport_lib::HttpRoleName::new("hydration"), + true, + std::vec![ksp_onchain_transport_lib::HttpRequestKind::new("get_transaction")], + 10, + ksp_onchain_transport_lib::HttpRoleLimits::new( + std::option::Option::None, + std::option::Option::None, + std::option::Option::None, + std::option::Option::None, + ), + ); + let endpoint = ksp_onchain_transport_lib::HttpEndpointSettings::new( + endpoint_name, + true, + ksp_onchain_transport_lib::HttpProviderName::new(provider), + ksp_onchain_transport_lib::HttpClusterName::new(cluster), + url, + std::time::Duration::from_secs(1), + std::time::Duration::from_secs(1), + std::option::Option::Some(1), + std::vec![role], + ); + let settings = ksp_onchain_transport_lib::HttpTransportSettings::new( + std::vec![endpoint], + ksp_onchain_transport_lib::HttpRetrySettings::new(0, std::time::Duration::from_millis(1), std::time::Duration::from_millis(1)), + ); + return match ksp_onchain_transport_lib::HttpTransportPool::new(settings) { + std::result::Result::Ok(value) => std::option::Option::Some(value), + std::result::Result::Err(_) => std::option::Option::None, + }; +} + +fn signal_source_for_http_url(url: &str) -> std::option::Option { + return signal_source_for_http_url_with_commitment(url, ksp_onchain_transport_lib::SolanaCommitment::Confirmed); +} + +fn signal_source_for_http_url_with_commitment( + url: &str, + commitment: ksp_onchain_transport_lib::SolanaCommitment, +) -> std::option::Option { + let endpoint = match grpc_endpoint("devnet") { + std::option::Option::Some(value) => value, + std::option::Option::None => return std::option::Option::None, + }; + let channel = match ksp_onchain_transport_lib::YellowstoneGrpcChannel::prepare(&endpoint) { + std::result::Result::Ok(value) => value, + std::result::Result::Err(_) => return std::option::Option::None, + }; + let request = match transaction_request(std::option::Option::Some(commitment)) { + std::option::Option::Some(value) => value, + std::option::Option::None => return std::option::Option::None, + }; + let pool = match http_pool_for_url(url, "devnet", "http-hydration-fixture", "fixture-http-provider") { + std::option::Option::Some(value) => value, + std::option::Option::None => return std::option::Option::None, + }; + return match crate::RawTransactionIngestYellowstoneSource::new(channel, request, pool, ksp_onchain_transport_lib::HttpRoleName::new("hydration")) { + std::result::Result::Ok(value) => std::option::Option::Some(value), + std::result::Result::Err(_) => std::option::Option::None, + }; +} + +fn pre_004_settings() -> std::option::Option { + let network = match ksp_store_lib::RawNetworkId::new("devnet") { + std::result::Result::Ok(value) => value, + std::result::Result::Err(_) => return std::option::Option::None, + }; + let worker_id = match ksp_worker_api::WorkerId::new("worker-fixture") { + std::result::Result::Ok(value) => value, + std::result::Result::Err(_) => return std::option::Option::None, + }; + return std::option::Option::Some(crate::RawTransactionIngestSettings::with_defaults(network, worker_id)); +} + +fn pre_004_signal( + source: &crate::RawTransactionIngestYellowstoneSource, + family: super::RawTransactionIngestSourceFamily, + slot: u64, + index: u64, + filters: &[&str], + signature_byte: u8, +) -> std::option::Option { + let filters = match filter_names(filters) { + std::option::Option::Some(value) => value, + std::option::Option::None => return std::option::Option::None, + }; + let created_at = match ksp_onchain_transport_lib::YellowstoneUpdateTimestamp::new(1_760_000_123, 456_000_000) { + std::result::Result::Ok(value) => value, + std::result::Result::Err(_) => return std::option::Option::None, + }; + let fixture = SignalViewFixture { + created_at: std::option::Option::Some(created_at), + family, + filters, + index, + signature: ksp_onchain_transport_lib::YellowstoneTransactionSignature::new([signature_byte; 64]), + slot, + }; + return std::option::Option::Some(super::project_yellowstone_signal(source, &fixture)); +} + +fn serve_http_once(body: std::string::String) -> std::io::Result<(std::string::String, std::thread::JoinHandle>)> { + let listener = match std::net::TcpListener::bind("127.0.0.1:0") { + std::result::Result::Ok(value) => value, + std::result::Result::Err(error) => return std::result::Result::Err(error), + }; + let address = match listener.local_addr() { + std::result::Result::Ok(value) => value, + std::result::Result::Err(error) => return std::result::Result::Err(error), + }; + let handle = std::thread::spawn(move || { + let (mut stream, _) = match listener.accept() { + std::result::Result::Ok(value) => value, + std::result::Result::Err(error) => return std::result::Result::Err(error), + }; + let request = match read_http_request(&mut stream) { + std::result::Result::Ok(value) => value, + std::result::Result::Err(error) => return std::result::Result::Err(error), + }; + let response = + std::format!("HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}", body.len(), body,); + if let std::result::Result::Err(error) = std::io::Write::write_all(&mut stream, response.as_bytes()) { + return std::result::Result::Err(error); + } + return std::result::Result::Ok(request); + }); + return std::result::Result::Ok((std::format!("http://{address}"), handle)); +} + +fn read_http_request(stream: &mut std::net::TcpStream) -> std::io::Result { + let mut bytes = std::vec::Vec::new(); + let mut buffer = [0_u8; 1024]; + loop { + let count = match std::io::Read::read(stream, &mut buffer) { + std::result::Result::Ok(value) => value, + std::result::Result::Err(error) => return std::result::Result::Err(error), + }; + if count == 0 { + break; + } + bytes.extend_from_slice(&buffer[..count]); + if http_request_complete(bytes.as_slice()) { + break; + } + } + return match std::string::String::from_utf8(bytes) { + std::result::Result::Ok(value) => std::result::Result::Ok(value), + std::result::Result::Err(_) => std::result::Result::Err(std::io::Error::new(std::io::ErrorKind::InvalidData, "fixture HTTP request is not UTF-8")), + }; +} + +fn http_request_complete(bytes: &[u8]) -> bool { + let text = match std::str::from_utf8(bytes) { + std::result::Result::Ok(value) => value, + std::result::Result::Err(_) => return false, + }; + let header_end = match text.find("\r\n\r\n") { + std::option::Option::Some(value) => value, + std::option::Option::None => return false, + }; + let mut content_length = 0_usize; + for line in text[..header_end].lines() { + let (name, value) = match line.split_once(':') { + std::option::Option::Some(value) => value, + std::option::Option::None => continue, + }; + if name.eq_ignore_ascii_case("content-length") { + content_length = match value.trim().parse::() { + std::result::Result::Ok(value) => value, + std::result::Result::Err(_) => return false, + }; + } + } + return bytes.len() >= header_end.saturating_add(4).saturating_add(content_length); +} + +fn http_request_body(request: &str) -> &str { + return match request.split_once("\r\n\r\n") { + std::option::Option::Some((_, body)) => body, + std::option::Option::None => "", + }; +} + #[tokio::test(flavor = "current_thread")] async fn pre_002_source_accepts_matching_confirmed_transaction_and_get_transaction_route_without_io() { let endpoint = match grpc_endpoint("devnet") { @@ -474,3 +667,368 @@ async fn pre_003_source_rejects_route_identity_that_cannot_become_safe_provenanc } return; } + +#[test] +fn pre_004_composite_provenance_rejects_overflow_without_truncation() { + let provider = match ksp_store_lib::RawProvenanceCode::new("yellowstone-provider") { + std::result::Result::Ok(value) => value, + std::result::Result::Err(_) => return, + }; + let endpoint_id = match ksp_store_lib::RawProvenanceCode::new("yellowstone-endpoint") { + std::result::Result::Ok(value) => value, + std::result::Result::Err(_) => return, + }; + let route = super::RawTransactionIngestSourceRoute { endpoint_id, provider }; + let result = super::composite_provenance_codes(&route, "h".repeat(120).as_str(), "http-endpoint"); + let error = match result { + std::result::Result::Ok(_) => return, + std::result::Result::Err(error) => error, + }; + assert_eq!(error.code(), crate::ERROR_CODE_RAW_TRANSACTION_INGEST_RUNTIME_INVALID); + assert!(error.context().iter().any(|context| return context.value() == "runtime_resources.composite_provider_unrepresentable")); + return; +} + +#[tokio::test(flavor = "current_thread")] +async fn pre_004_observed_get_transaction_closes_signal_to_common_raw_ingress_with_composite_provenance() { + let body = std::format!( + "{{\"jsonrpc\":\"2.0\",\"result\":{{\"slot\":42,\"blockTime\":1760000120,\"transaction\":[\"{}\",\"base64\"],\"meta\":{{\"err\":null,\"fee\":5000}},\"version\":\"legacy\",\"transactionIndex\":7}},\"id\":1}}", + PRE_004_ZERO_TRANSACTION_BASE64, + ); + let (url, server) = match serve_http_once(body) { + std::result::Result::Ok(value) => value, + std::result::Result::Err(_) => return, + }; + let source = match signal_source_for_http_url(url.as_str()) { + std::option::Option::Some(value) => value, + std::option::Option::None => return, + }; + let settings = match pre_004_settings() { + std::option::Option::Some(value) => value, + std::option::Option::None => return, + }; + let signal = match pre_004_signal(&source, super::RawTransactionIngestSourceFamily::Transaction, 42, 7, &["tx-fixture"], 0) { + std::option::Option::Some(value) => value, + std::option::Option::None => return, + }; + let received_at = match ksp_store_lib::RawTimestamp::from_unix_millis(1_760_000_124_000) { + std::result::Result::Ok(value) => value, + std::result::Result::Err(_) => return, + }; + let outcome = super::hydrate_yellowstone_signal(&source, &settings, signal, received_at).await; + assert!(outcome.is_ok()); + let outcome = match outcome { + std::result::Result::Ok(value) => value, + std::result::Result::Err(_) => return, + }; + let request = match server.join() { + std::result::Result::Ok(std::result::Result::Ok(value)) => value, + _ => return, + }; + let request_body = http_request_body(request.as_str()); + assert!(request_body.contains("\"method\":\"getTransaction\"")); + assert!(request_body.contains(PRE_004_ZERO_SIGNATURE_TEXT)); + assert!(request_body.contains("\"commitment\":\"confirmed\"")); + assert!(request_body.contains("\"encoding\":\"base64\"")); + assert!(request_body.contains("\"maxSupportedTransactionVersion\":0")); + let ingress = match outcome { + super::RawTransactionIngestHydrationOutcome::Available(value) => value, + super::RawTransactionIngestHydrationOutcome::Missing(_) => return, + }; + assert_eq!(ingress.provenance.provider().as_str(), "ys.fixture-provider:http.fixture-http-provider"); + assert_eq!(ingress.provenance.protocol().as_str(), "yellowstone_http"); + assert_eq!(ingress.provenance.acquisition_method().as_str(), "transaction_get_transaction"); + assert_eq!(ingress.provenance.origin(), ksp_store_lib::RawAcquisitionOrigin::Live); + assert_eq!(ingress.provenance.received_at(), received_at); + assert_eq!( + ingress.provenance.endpoint_id().map(ksp_store_lib::RawProvenanceCode::as_str), + std::option::Option::Some("ys.yellowstone-fixture:http.http-hydration-fixture"), + ); + assert_eq!(ingress.provenance.commitment().map(ksp_store_lib::RawProvenanceCode::as_str), std::option::Option::Some("confirmed")); + assert_eq!(ingress.provenance.capture_session_id().map(ksp_store_lib::RawProvenanceCode::as_str), std::option::Option::Some("worker-fixture")); + assert_eq!(ingress.provenance.filter_id().map(ksp_store_lib::RawProvenanceCode::as_str), std::option::Option::Some("tx-fixture")); + assert_eq!(ingress.provenance.observed_at().map(|value| return value.unix_millis()), std::option::Option::Some(1_760_000_123_456)); + let (mut admission, sender) = crate::RawTransactionAdmission::new(1); + if sender.send(ingress).await.is_err() { + return; + } + std::mem::drop(sender); + let acquisition = match admission.receive(settings.network()).await { + std::result::Result::Ok(std::option::Option::Some(value)) => value, + _ => return, + }; + assert_eq!(acquisition.transaction().reference().network().as_str(), "devnet"); + assert_eq!(acquisition.transaction().reference().signature().as_bytes(), &[0_u8; 64]); + assert_eq!(acquisition.transaction().slot(), 42); + assert_eq!(acquisition.transaction().block_time().map(|value| return value.unix_millis()), std::option::Option::Some(1_760_000_120_000),); + assert_eq!(acquisition.observation().provenance().provider().as_str(), "ys.fixture-provider:http.fixture-http-provider"); + return; +} + +#[tokio::test(flavor = "current_thread")] +async fn pre_004_finalized_hydration_preserves_request_and_provenance_commitment() { + let body = std::format!( + "{{\"jsonrpc\":\"2.0\",\"result\":{{\"slot\":42,\"blockTime\":1760000120,\"transaction\":[\"{}\",\"base64\"],\"meta\":null,\"version\":\"legacy\",\"transactionIndex\":7}},\"id\":1}}", + PRE_004_ZERO_TRANSACTION_BASE64, + ); + let (url, server) = match serve_http_once(body) { + std::result::Result::Ok(value) => value, + std::result::Result::Err(_) => return, + }; + let source = match signal_source_for_http_url_with_commitment(url.as_str(), ksp_onchain_transport_lib::SolanaCommitment::Finalized) { + std::option::Option::Some(value) => value, + std::option::Option::None => return, + }; + let settings = match pre_004_settings() { + std::option::Option::Some(value) => value, + std::option::Option::None => return, + }; + let signal = match pre_004_signal(&source, super::RawTransactionIngestSourceFamily::Transaction, 42, 7, &["tx-fixture"], 0) { + std::option::Option::Some(value) => value, + std::option::Option::None => return, + }; + let received_at = match ksp_store_lib::RawTimestamp::from_unix_millis(1_760_000_124_000) { + std::result::Result::Ok(value) => value, + std::result::Result::Err(_) => return, + }; + let outcome = match super::hydrate_yellowstone_signal(&source, &settings, signal, received_at).await { + std::result::Result::Ok(value) => value, + std::result::Result::Err(_) => return, + }; + let request = match server.join() { + std::result::Result::Ok(std::result::Result::Ok(value)) => value, + _ => return, + }; + let request_body = http_request_body(request.as_str()); + assert!(request_body.contains("\"commitment\":\"finalized\"")); + assert!(request_body.contains("\"encoding\":\"base64\"")); + assert!(request_body.contains("\"maxSupportedTransactionVersion\":0")); + let ingress = match outcome { + super::RawTransactionIngestHydrationOutcome::Available(value) => value, + super::RawTransactionIngestHydrationOutcome::Missing(_) => return, + }; + assert_eq!(ingress.provenance.commitment().map(ksp_store_lib::RawProvenanceCode::as_str), std::option::Option::Some("finalized")); + return; +} + +#[tokio::test(flavor = "current_thread")] +async fn pre_004_get_transaction_null_is_missing_without_ingress_or_worker_retry() { + let (url, server) = match serve_http_once("{\"jsonrpc\":\"2.0\",\"result\":null,\"id\":1}".to_owned()) { + std::result::Result::Ok(value) => value, + std::result::Result::Err(_) => return, + }; + let source = match signal_source_for_http_url(url.as_str()) { + std::option::Option::Some(value) => value, + std::option::Option::None => return, + }; + let settings = match pre_004_settings() { + std::option::Option::Some(value) => value, + std::option::Option::None => return, + }; + let signal = match pre_004_signal(&source, super::RawTransactionIngestSourceFamily::TransactionStatus, 42, 7, &["status-fixture"], 0) { + std::option::Option::Some(value) => value, + std::option::Option::None => return, + }; + let received_at = match ksp_store_lib::RawTimestamp::from_unix_millis(1_760_000_124_000) { + std::result::Result::Ok(value) => value, + std::result::Result::Err(_) => return, + }; + let outcome = match super::hydrate_yellowstone_signal(&source, &settings, signal, received_at).await { + std::result::Result::Ok(value) => value, + std::result::Result::Err(_) => return, + }; + let request = match server.join() { + std::result::Result::Ok(std::result::Result::Ok(value)) => value, + _ => return, + }; + assert!(http_request_body(request.as_str()).contains(PRE_004_ZERO_SIGNATURE_TEXT)); + let reference = match outcome { + super::RawTransactionIngestHydrationOutcome::Available(_) => return, + super::RawTransactionIngestHydrationOutcome::Missing(value) => value, + }; + assert_eq!(reference.network().as_str(), "devnet"); + assert_eq!(reference.signature().as_bytes(), &[0_u8; 64]); + return; +} + +#[tokio::test(flavor = "current_thread")] +async fn pre_004_v0_null_and_omitted_wire_fields_preserve_common_raw_semantics() { + let body = std::format!( + "{{\"jsonrpc\":\"2.0\",\"result\":{{\"slot\":42,\"blockTime\":null,\"transaction\":[\"{}\",\"base64\"],\"meta\":null,\"version\":0}},\"id\":1}}", + PRE_004_ZERO_TRANSACTION_BASE64, + ); + let (url, server) = match serve_http_once(body) { + std::result::Result::Ok(value) => value, + std::result::Result::Err(_) => return, + }; + let source = match signal_source_for_http_url(url.as_str()) { + std::option::Option::Some(value) => value, + std::option::Option::None => return, + }; + let settings = match pre_004_settings() { + std::option::Option::Some(value) => value, + std::option::Option::None => return, + }; + let signal = match pre_004_signal(&source, super::RawTransactionIngestSourceFamily::Transaction, 42, 7, &["filter-a", "filter-b"], 0) { + std::option::Option::Some(value) => value, + std::option::Option::None => return, + }; + let received_at = match ksp_store_lib::RawTimestamp::from_unix_millis(1_760_000_124_000) { + std::result::Result::Ok(value) => value, + std::result::Result::Err(_) => return, + }; + let outcome = match super::hydrate_yellowstone_signal(&source, &settings, signal, received_at).await { + std::result::Result::Ok(value) => value, + std::result::Result::Err(_) => return, + }; + let _request = match server.join() { + std::result::Result::Ok(std::result::Result::Ok(value)) => value, + _ => return, + }; + let ingress = match outcome { + super::RawTransactionIngestHydrationOutcome::Available(value) => value, + super::RawTransactionIngestHydrationOutcome::Missing(_) => return, + }; + let filter_id = match ingress.provenance.filter_id() { + std::option::Option::Some(value) => value.as_str(), + std::option::Option::None => return, + }; + assert!(filter_id.starts_with("sha256.")); + assert_eq!(filter_id.len(), 71); + let (mut admission, sender) = crate::RawTransactionAdmission::new(1); + if sender.send(ingress).await.is_err() { + return; + } + std::mem::drop(sender); + let acquisition = match admission.receive(settings.network()).await { + std::result::Result::Ok(std::option::Option::Some(value)) => value, + _ => return, + }; + assert_eq!(acquisition.transaction().block_time(), std::option::Option::None); + let payload = match std::str::from_utf8(acquisition.transaction().payload().bytes()) { + std::result::Result::Ok(value) => value, + std::result::Result::Err(_) => return, + }; + assert!(payload.contains("\"meta\":null")); + assert!(payload.contains("\"version\":0")); + assert!(!payload.contains("transactionIndex")); + return; +} + +#[tokio::test(flavor = "current_thread")] +async fn pre_004_source_hydration_rejects_slot_index_signature_and_network_mismatch_before_admission() { + for (slot, index, transaction_data, expected_condition) in [ + (43_u64, 7_u32, PRE_004_ZERO_TRANSACTION_BASE64, "hydration.slot_mismatch"), + (42_u64, 8_u32, PRE_004_ZERO_TRANSACTION_BASE64, "hydration.transaction_index_mismatch"), + (42_u64, 7_u32, PRE_004_OTHER_TRANSACTION_BASE64, "hydration.signature_mismatch"), + ] { + let body = std::format!( + "{{\"jsonrpc\":\"2.0\",\"result\":{{\"slot\":{},\"blockTime\":1760000120,\"transaction\":[\"{}\",\"base64\"],\"meta\":null,\"version\":\"legacy\",\"transactionIndex\":{}}},\"id\":1}}", + slot, + transaction_data, + index, + ); + let (url, server) = match serve_http_once(body) { + std::result::Result::Ok(value) => value, + std::result::Result::Err(_) => return, + }; + let source = match signal_source_for_http_url(url.as_str()) { + std::option::Option::Some(value) => value, + std::option::Option::None => return, + }; + let settings = match pre_004_settings() { + std::option::Option::Some(value) => value, + std::option::Option::None => return, + }; + let signal = match pre_004_signal(&source, super::RawTransactionIngestSourceFamily::Transaction, 42, 7, &["tx-fixture"], 0) { + std::option::Option::Some(value) => value, + std::option::Option::None => return, + }; + let received_at = match ksp_store_lib::RawTimestamp::from_unix_millis(1_760_000_124_000) { + std::result::Result::Ok(value) => value, + std::result::Result::Err(_) => return, + }; + let error = match super::hydrate_yellowstone_signal(&source, &settings, signal, received_at).await { + std::result::Result::Ok(_) => return, + std::result::Result::Err(error) => error, + }; + let _request = match server.join() { + std::result::Result::Ok(std::result::Result::Ok(value)) => value, + _ => return, + }; + assert_eq!(error.code(), crate::ERROR_CODE_RAW_TRANSACTION_INGEST_RUNTIME_INVALID); + assert!(error.context().iter().any(|context| return context.value() == expected_condition)); + } + let source = match signal_source() { + std::option::Option::Some(value) => value, + std::option::Option::None => return, + }; + let settings = match pre_004_settings() { + std::option::Option::Some(value) => value, + std::option::Option::None => return, + }; + let mut signal = match pre_004_signal(&source, super::RawTransactionIngestSourceFamily::Transaction, 42, 7, &["tx-fixture"], 0) { + std::option::Option::Some(value) => value, + std::option::Option::None => return, + }; + signal.network = match ksp_store_lib::RawNetworkId::new("mainnet") { + std::result::Result::Ok(value) => value, + std::result::Result::Err(_) => return, + }; + let received_at = match ksp_store_lib::RawTimestamp::from_unix_millis(1_760_000_124_000) { + std::result::Result::Ok(value) => value, + std::result::Result::Err(_) => return, + }; + let error = match super::hydrate_yellowstone_signal(&source, &settings, signal, received_at).await { + std::result::Result::Ok(_) => return, + std::result::Result::Err(error) => error, + }; + assert_eq!(error.code(), crate::ERROR_CODE_RAW_TRANSACTION_INGEST_RUNTIME_INVALID); + assert!(error.context().iter().any(|context| return context.value() == "hydration.network_mismatch")); + return; +} + +#[tokio::test(flavor = "current_thread")] +async fn pre_004_status_provenance_and_future_source_timestamp_are_bounded_and_redacted() { + let source = match signal_source() { + std::option::Option::Some(value) => value, + std::option::Option::None => return, + }; + let settings = match pre_004_settings() { + std::option::Option::Some(value) => value, + std::option::Option::None => return, + }; + let mut signal = match pre_004_signal(&source, super::RawTransactionIngestSourceFamily::TransactionStatus, 42, 7, &["filter-b", "filter-a", "filter-a"], 0) + { + std::option::Option::Some(value) => value, + std::option::Option::None => return, + }; + signal.created_at = std::option::Option::Some(super::RawTransactionIngestSourceTimestamp { seconds: 1_760_000_200, nanos: 0 }); + let received_at = match ksp_store_lib::RawTimestamp::from_unix_millis(1_760_000_124_000) { + std::result::Result::Ok(value) => value, + std::result::Result::Err(_) => return, + }; + let provenance = match super::build_hydration_provenance( + &settings, + &signal, + "fixture-http-provider", + "http-hydration-fixture", + ksp_onchain_transport_lib::SolanaCommitment::Confirmed, + received_at, + ) { + std::result::Result::Ok(value) => value, + std::result::Result::Err(_) => return, + }; + assert_eq!(provenance.acquisition_method().as_str(), "status_get_transaction"); + assert_eq!(provenance.observed_at(), std::option::Option::None); + let filter_id = match provenance.filter_id() { + std::option::Option::Some(value) => value.as_str(), + std::option::Option::None => return, + }; + assert!(filter_id.starts_with("sha256.")); + let debug = std::format!("{provenance:?}"); + assert!(!debug.contains("filter-a")); + assert!(!debug.contains("filter-b")); + assert!(!debug.contains(PRE_004_ZERO_SIGNATURE_TEXT)); + return; +} diff --git a/deltas/0.3.12/pre.004.md b/deltas/0.3.12/pre.004.md new file mode 100644 index 0000000..586e9ed --- /dev/null +++ b/deltas/0.3.12/pre.004.md @@ -0,0 +1,292 @@ + + + +# Delta `0.3.12-pre.004` — hydration `getTransaction` et provenance composite Yellowstone + HTTP + +## Base requise + +```text +0.3.12-pre.003-fix.002 +workspace.package.version = 0.3.12-pre.3.fix.2 +``` + +Le gate opérateur communiqué pour cette base est entièrement vert : + +```text +cargo fmt --all +Rust rule audit : clean +Markdown table audit : clean (340 tables, 805 fichiers) +cargo check --workspace : PASS +cargo clippy --workspace --all-targets --all-features -- -D warnings : PASS +cargo test -p ksp-worker-raw-transaction-ingest-lib : PASS +unit tests Worker : 45 PASS +integration dependency_boundary : 4 PASS +integration hardening : 10 PASS +integration public_api : 8 PASS +integration release_completeness : 3 PASS +doc-tests : 0 fail +``` + +## Objectif + +Fermer et qualifier déterministiquement, sans source task productive, la chaîne : + +```text +signal Yellowstone Transaction / TransactionStatus + -> signature canonique 64 bytes -> Base58 commun + -> get_transaction_observed + Base64 + confirmed | finalized + maxSupportedTransactionVersion = 0 + -> contrôles source/hydration + -> RawTransactionMaterial commun + -> RawTransactionIngress existant +``` + +La tranche ne branche ni Block, ni continuité/replay, ni session Yellowstone productive. + +## Version + +```text +workspace.package.version = 0.3.12-pre.4 +``` + +## Common RAW — formatage Base58 + +`ksp-raw-transaction-lib` expose désormais `format_raw_transaction_signature` depuis la racine de crate. + +La fonction encode exactement les 64 bytes de `RawTransactionSignature` vers le texte Base58 canonique attendu par les RPC Solana. Elle ne dépend d'aucune crate Transport ni d'un second codec externe. Les bornes restent celles déjà publiques : + +```text +MIN_RAW_TRANSACTION_SIGNATURE_TEXT_BYTES = 64 +MAX_RAW_TRANSACTION_SIGNATURE_TEXT_BYTES = 88 +``` + +Le round-trip `bytes -> format -> parse -> bytes` est qualifié sur plusieurs signatures, dont la signature nulle représentée par exactement 64 caractères `1`. + +## Hydration HTTP qualifiée + +Le helper privé de qualification construit `SolanaGetTransactionConfig` avec : + +```text +encoding = base64 +commitment = confirmed | finalized +maxSupportedTransactionVersion = 0 +``` + +La signature RPC est obtenue exclusivement via `ksp_raw_transaction_lib::format_raw_transaction_signature`. + +`HttpTransportPool::get_transaction_observed` reste propriétaire du routage, retry/backoff et rate-limit Transport. Le Worker n'ajoute aucune boucle retry autour d'une erreur Transport. + +## Contrôles avant admission + +Avant de créer un ingress, la qualification impose : + +```text +network signal == network Worker == network source +route signal == route source +slot HTTP == slot signal +transaction_index HTTP == index signal lorsque les deux côtés le fournissent +transaction HTTP réellement encodée en Base64 +signature extraite du wire Base64 HTTP == signature signal +``` + +Un mismatch devient `worker_raw_transaction_ingest.runtime_invalid` avec une condition `hydration.*` stable. Il n'est jamais converti en content conflict Store. + +Les erreurs Transport sont reclassées en `worker_raw_transaction_ingest.source_failed` en ne conservant que le domain/code lower-layer déjà sûrs. + +## Missing + +`getTransaction -> null` retourne : + +```text +Missing(RawTransactionReference) +``` + +Aucun `RawTransactionMaterial`, aucune provenance et aucun ingress ne sont inventés. Aucun retry Worker supplémentaire n'est ajouté dans cette tranche. + +## Provenance composite + +La provenance réutilise sans migration `RawAcquisitionProvenance` et `RawProvenanceCode` : + +```text +protocol = yellowstone_http +provider = ys.:http. +endpoint_id = ys.:http. +acquisition_method = transaction_get_transaction | status_get_transaction +origin = Live +commitment = confirmed | finalized +capture_session_id = WorkerId sûr +filter_id = nom direct unique représentable | sha256. +observed_at = created_at Yellowstone seulement si représentable et <= received_at +``` + +Le provider et l'endpoint HTTP proviennent exclusivement du `HttpObservedValue` gagnant. Les URL, headers, tokens, bodies HTTP bruts et remote errors ne sont pas recopiés. + +Le constructeur `RawTransactionIngestYellowstoneSource::new` valide maintenant que chaque route HTTP compatible peut former les codes composites provider/endpoint sans dépasser `RawProvenanceCode`. Aucune troncature n'est admise. + +## Common RAW et wire + +Après contrôles, la réponse HTTP est projetée vers `RawTransactionMaterial::binary_base64` en préservant : + +```text +blockTime -> Option puis RawTimestamp commun en millisecondes +meta omitted | null | value +version omitted | null | legacy | numeric +transactionIndex omitted | null | value +``` + +La signature réellement embarquée dans le wire Base64 est extraite via `extract_raw_transaction_signature_from_binary_base64` avant admission. + +Les fixtures couvrent notamment : + +```text +Legacy + meta valeur + block_time valeur + index présent +V0 + meta null + block_time null + index omis +``` + +## `RUST-API-008` + +Le source task productif n'est pas créé avant `pre.006`. Les adapters Transaction/TransactionStatus et la chaîne d'hydration, encore sans consumer productif, restent donc sous `#[cfg(test)]` conformément à `RUST-API-008`. + +Aucun `#[allow(dead_code)]` ni `#[expect(dead_code)]` n'est introduit et aucune surface publique artificielle n'est créée. + +Les éléments immédiatement productifs de `pre.004` sont limités à : + +```text +format_raw_transaction_signature dans la common RAW +validation de représentabilité de provenance composite au constructeur de source +``` + +## Canaris déterministes ajoutés ou étendus + +```text +Base58 formatter round-trip et crate-root API +getTransaction confirmed/base64/max version 0 +getTransaction finalized/base64/max version 0 +commitment finalized conservé dans la provenance +winner HTTP observé dans provider/endpoint composites +provenance Transaction et TransactionStatus distincte +filter unique direct +multi-filter -> fingerprint sha256 borné +created_at futur ignoré pour observed_at +getTransaction null -> Missing sans ingress/retry Worker +slot mismatch +transaction index mismatch +signature wire mismatch +network mismatch avant I/O +composite provider overflow rejeté sans troncature +absence Block/getBlock/stream/source spawn +absence Config/Backfill/backend/reqwest/tonic/proto direct dans le Worker +``` + +## Décisions prises + +```text +aucune migration Store pour une chaîne de provenance multi-hop +Base58 appartient à la common RAW, pas au Worker +HTTP observed est la source de vérité du provider/endpoint gagnant +processed reste exclu du P0 hydration +missing n'ajoute pas de retry Worker par défaut +mismatch source/hydration est un fault Worker avant Store +pas de RAW direct Yellowstone avant qualification byte-exacte dédiée +adapters/hydration restent test-only jusqu'au source task productif de pre.006 +``` + +## Questions ouvertes + +Aucune question ouverte n'est bloquante pour `pre.004`. + +Les sujets suivants restent volontairement réservés aux tranches ultérieures : + +```text +Block / BlockMeta / Slot : pre.005 +source task productive et coalescence : pre.006 +processing frontier : pre.007 +reconnect/from_slot/ReplayInfo : pre.008 +races/retry/backpressure final : pre.009 +``` + +## Fichiers ajoutés + +```text +deltas/0.3.12/pre.004.md +``` + +## Fichiers modifiés + +```text +Cargo.toml +crates/ksp-raw-transaction-lib/README.md +crates/ksp-raw-transaction-lib/USAGE.md +crates/ksp-raw-transaction-lib/src/lib.rs +crates/ksp-raw-transaction-lib/src/signature.rs +crates/ksp-raw-transaction-lib/tests/public_api.rs +crates/ksp-raw-transaction-lib/unit_tests/signature.rs +crates/ksp-worker-raw-transaction-ingest-lib/src/lib.rs +crates/ksp-worker-raw-transaction-ingest-lib/src/runtime_resources.rs +crates/ksp-worker-raw-transaction-ingest-lib/tests/dependency_boundary.rs +crates/ksp-worker-raw-transaction-ingest-lib/tests/hardening.rs +crates/ksp-worker-raw-transaction-ingest-lib/tests/release_completeness.rs +crates/ksp-worker-raw-transaction-ingest-lib/unit_tests/runtime_resources.rs +docs/plans/033-V0_3_12_YELLOWSTONE_HYDRATION_CONTINUITY_PLAN.md +docs/validation/029-V0_3_12_YELLOWSTONE_HYDRATION_CONTINUITY.md +``` + +## Fichiers supprimés + +Aucun. + +## Validations exécutées dans l'environnement d'assemblage + +```text +python3 scripts/audit_rust_workspace_rules.py +General Rust rule audit: clean +Rust export completeness audit: 0 candidate(s) +KSP workspace Rust rule audit: clean + +python3 scripts/audit_markdown_tables.py README.md RULES.md ROADMAP.md CHANGELOG.md docs prompts crates deltas +Markdown table audit: clean (340 table(s), 806 file(s)) +``` + +## Validations non exécutées dans l'environnement d'assemblage + +`cargo`, `rustc` et `rustfmt` ne sont pas installés dans l'environnement d'assemblage. Aucun résultat Cargo local n'est déclaré PASS. + +Le gate opérateur requis reste : + +```bash +cargo fmt --all +python3 scripts/audit_rust_workspace_rules.py +python3 scripts/audit_markdown_tables.py README.md RULES.md ROADMAP.md CHANGELOG.md docs prompts crates deltas +cargo check --workspace +cargo clippy --workspace --all-targets --all-features -- -D warnings +cargo test -p ksp-raw-transaction-lib +cargo test -p ksp-worker-raw-transaction-ingest-lib +cargo tree -p ksp-worker-raw-transaction-ingest-lib --edges normal +cargo tree -p ksp-worker-raw-transaction-ingest-lib -e features +cargo tree --duplicates +``` + +## Non-claims + +Cette tranche ne revendique pas : + +```text +open_standard_subscribe +consommation d'une update réseau live +source task Yellowstone productif +persistence d'une acquisition issue d'un stream réel +Block / BlockMeta / Slot +getBlock hydration +coalescence productive +processing frontier +from_slot / ReplayInfo +repair historique +smoke live Worker +réconciliation documentaire finale de pre.012 +``` + +## Suite + +Après gate opérateur vert, `pre.005` adapte `Block` en signaux transactionnels et `BlockMeta`/`Slot` en signaux continuity-only, sans encore introduire la politique reconnect Worker. diff --git a/docs/plans/033-V0_3_12_YELLOWSTONE_HYDRATION_CONTINUITY_PLAN.md b/docs/plans/033-V0_3_12_YELLOWSTONE_HYDRATION_CONTINUITY_PLAN.md index 36ff831..d0f1d02 100644 --- a/docs/plans/033-V0_3_12_YELLOWSTONE_HYDRATION_CONTINUITY_PLAN.md +++ b/docs/plans/033-V0_3_12_YELLOWSTONE_HYDRATION_CONTINUITY_PLAN.md @@ -1,5 +1,5 @@ - + # Plan v0.3.12 — Yellowstone + hydration HTTP + continuité de run du Worker RawTransaction @@ -962,7 +962,7 @@ Adapter les deux DTOs vers le signal Worker privé, sans HTTP ni persistence ré ### `pre.004` — hydration `getTransaction` + provenance composite -Fermer signal -> HTTP observed -> Common RAW -> ingress existant, missing/mismatch/provenance, sans Block ni continuity. +Fermer et qualifier déterministiquement signal -> HTTP observed -> Common RAW -> ingress existant, missing/mismatch/provenance, sans Block ni continuity. Tant que `pre.006` n'apporte pas le source task productif qui consomme cette chaîne, les adapters/hydration strictement privés sans consumer productif restent sous `#[cfg(test)]` conformément à `RUST-API-008`; `pre.004` ne crée pas artificiellement une surface publique pour contourner `dead_code`. ### `pre.005` — Block + BlockMeta + Slot diff --git a/docs/validation/029-V0_3_12_YELLOWSTONE_HYDRATION_CONTINUITY.md b/docs/validation/029-V0_3_12_YELLOWSTONE_HYDRATION_CONTINUITY.md index 9849f64..6b9d427 100644 --- a/docs/validation/029-V0_3_12_YELLOWSTONE_HYDRATION_CONTINUITY.md +++ b/docs/validation/029-V0_3_12_YELLOWSTONE_HYDRATION_CONTINUITY.md @@ -1,5 +1,5 @@ - + # Validation v0.3.12 — Yellowstone + hydration HTTP + continuité de run du Worker RawTransaction @@ -812,3 +812,138 @@ Le correctif ne touche pas au contrat de production. Le canari recherche désorm Aucun lint n'est neutralisé, aucun helper n'est remis prématurément en production et aucune fonctionnalité de `pre.004` n'est avancée. +## 34. Gate opérateur reçu pour `pre.003-fix.002` + +L'opérateur a exécuté après application de `pre.003-fix.002` : + +```text +cargo fmt --all : PASS visible +audit Rust : clean +Rust export completeness : 0 candidate +KSP workspace Rust rule audit : clean +audit Markdown : clean (340 tables, 805 fichiers) +cargo check --workspace : PASS visible +cargo clippy --workspace --all-targets --all-features -- -D warnings : PASS visible +cargo test -p ksp-worker-raw-transaction-ingest-lib : PASS +unit tests Worker : 45 PASS, 0 fail +dependency_boundary : 4 PASS +hardening : 10 PASS +public_api : 8 PASS +release_completeness : 3 PASS +doc-tests : 0 fail +``` + +Résultat : les deux correctifs `pre.003` sont clos et `0.3.12-pre.3.fix.2` devient la base autoritaire de `pre.004`. + +## 35. Gate `pre.004` — hydration HTTP qualifiée et provenance composite + +La tranche porte `workspace.package.version = 0.3.12-pre.4` et ferme la qualification déterministe suivante : + +```text +signal Transaction / TransactionStatus + -> signature canonique 64 bytes -> Base58 commun + -> get_transaction_observed + encoding = base64 + commitment = confirmed | finalized + maxSupportedTransactionVersion = 0 + -> contrôle slot/index/signature embarquée + -> RawTransactionMaterial source-neutral + -> RawTransactionIngress existant +``` + +`getTransaction -> null` produit explicitement un résultat `Missing(RawTransactionReference)` et ne fabrique ni matériau RAW, ni provenance, ni ingress. Aucun retry Worker n'est ajouté autour du Transport dans cette tranche. + +La common RAW reçoit `format_raw_transaction_signature`, encodeur Base58 borné de `RawTransactionSignature` 64 bytes. Le round-trip avec `parse_raw_transaction_signature` est qualifié sur plusieurs signatures canoniques, y compris la signature nulle représentée par 64 caractères `1`. Le Worker n'embarque donc pas un second codec Base58. + +Avant toute admission, la hydration vérifie : + +```text +network signal == network Worker == network source +route signal == route source +slot HTTP == slot signal +transaction_index HTTP == signal lorsqu'il est présent des deux côtés +signature extraite du wire Base64 HTTP == signature signal +transaction HTTP réellement encodée en Base64 +``` + +Un mismatch devient un fault Worker sûr `runtime_invalid` avec condition stable `hydration.*`; il n'est pas délégué au Store sous forme de content conflict. + +## 36. Provenance composite `pre.004` + +La provenance réutilise strictement `RawAcquisitionProvenance` et `RawProvenanceCode`; aucune migration Store n'est créée. + +Forme qualifiée : + +```text +protocol = yellowstone_http +provider = ys.:http. +endpoint_id = ys.:http. +acquisition_method = transaction_get_transaction | status_get_transaction +origin = Live +commitment = confirmed | finalized +capture_session_id = WorkerId sûr +filter_id = nom unique représentable | sha256. +observed_at = created_at Yellowstone seulement s'il est représentable et <= received_at +``` + +Les identités HTTP proviennent de `HttpObservedValue`, donc de l'endpoint réellement gagnant après routage/retry Transport. URL, headers, token, body HTTP brut et remote error ne sont jamais copiés dans la provenance ou les diagnostics Worker. + +Le constructeur de `RawTransactionIngestYellowstoneSource` valide désormais que chaque route HTTP compatible peut former les codes composites provider/endpoint sans dépasser les bornes Store. Une route non représentable est rejetée ; aucune troncature n'est autorisée. + +## 37. Application de `RUST-API-008` pendant `pre.004` + +`pre.004` ne crée toujours pas le source task productif : son ouverture reste réservée à `pre.006`. La chaîne signal/hydration n'a donc pas encore de consumer productif dans le build normal. + +Conformément à `RUST-API-008`, les adapters privés Transaction/TransactionStatus et le helper de hydration restent sous `#[cfg(test)]` pendant cette tranche. Ils sont exécutés par des fixtures déterministes réelles contre le `HttpTransportPool`, mais aucune fonction privée morte n'est laissée dans le build normal et aucun `#[allow(dead_code)]`/`#[expect(dead_code)]` n'est utilisé. + +Cela précise l'anticipation écrite dans `pre.003-fix.002` selon laquelle `pre.004` remettrait nécessairement ces helpers en production : la première consommation productive est en réalité celle du source task `pre.006`. La frontière fonctionnelle ne change pas ; seule la date d'activation de compilation productive est alignée avec la règle Rust du workspace. + +`pre.004` possède néanmoins deux effets productifs immédiatement valides : + +```text +API common RAW de formatage Base58 +validation de représentabilité de la future provenance composite dans le constructeur de source +``` + +## 38. Canaris déterministes `pre.004` + +Les nouvelles preuves couvrent : + +```text +round-trip Base58 64 bytes <-> texte canonique +requête getTransaction avec confirmed/base64/max version 0 +winner HTTP observé dans provider/endpoint composites +Legacy + meta valeur + block_time valeur + index présent +V0 + meta null + block_time null + index omis +filter unique direct et multi-filter fingerprint borné +Transaction et TransactionStatus vers méthodes de provenance distinctes +created_at futur ignoré pour observed_at +getTransaction null -> Missing sans ingress et sans retry Worker +slot mismatch +index mismatch +signature wire mismatch +network mismatch avant I/O +absence de Block/getBlock/stream Yellowstone/source spawn +absence de Config/Backfill/backend/reqwest/tonic/proto direct dans le Worker +``` + +## 39. Validation locale et non-claims `pre.004` + +L'environnement d'assemblage exécute les audits Python et l'audit Markdown après finalisation du delta. `cargo`, `rustc` et `rustfmt` n'y sont pas disponibles ; aucun résultat Cargo local n'est inventé. + +`pre.004` ne prétend pas avoir : + +```text +ouvert open_standard_subscribe +consommé une update réseau live +spawn un source task Yellowstone +branché Block / BlockMeta / Slot +coalescé des hydrations productives +modifié la processing frontier +branché from_slot / ReplayInfo +persisté une transaction issue d'un stream Yellowstone réel +exécuté un smoke live Worker +``` + +La tranche suivante reste `pre.005` : adapters Block transactionnels et signaux continuity-only BlockMeta/Slot, toujours sans politique reconnect Worker. +