From ad24cb36fa4e35f82080dad266d25618b51b45ac Mon Sep 17 00:00:00 2001 From: SinuS Von SifriduS Date: Thu, 17 Sep 2026 13:02:10 +0200 Subject: [PATCH] 0.3.15-pre.014 --- Cargo.toml | 4 +- .../src/raw_transaction.rs | 175 +++++++++++++++++- .../tests/hardening_completeness.rs | 39 +++- .../unit_tests/raw_transaction.rs | 48 ++++- .../src/runtime_resources.rs | 60 +++++- .../tests/hardening.rs | 34 +++- .../tests/release_completeness.rs | 12 +- deltas/0.3.15/pre.014.md | 111 +++++++++++ ...0_3_15_RAW_TRANSACTION_INGEST_DESK_PLAN.md | 14 +- ...032-V0_3_15_RAW_TRANSACTION_INGEST_DESK.md | 77 +++++++- 10 files changed, 561 insertions(+), 13 deletions(-) create mode 100644 deltas/0.3.15/pre.014.md diff --git a/Cargo.toml b/Cargo.toml index 34c217d..99f4ea9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,12 +1,12 @@ # file: Cargo.toml -# version: 627 +# version: 628 [workspace] resolver = "3" members = ["crates/ksp-app-backfill-desk", "crates/ksp-app-config-desk", "crates/ksp-app-raw-transaction-ingest-desk", "crates/ksp-app-solprices-desk", "crates/ksp-app-store-desk", "crates/ksp-app-wallet-desk", "crates/ksp-config-lib", "crates/ksp-core-lib", "crates/ksp-interface-lib", "crates/ksp-job-api", "crates/ksp-job-backfill-lib", "crates/ksp-logging-lib", "crates/ksp-offchain-transport-lib", "crates/ksp-onchain-transport-lib", "crates/ksp-program-api", "crates/ksp-raw-transaction-lib", "crates/ksp-store-api", "crates/ksp-store-lib", "crates/ksp-store-postgres-lib", "crates/ksp-wallet-lib", "crates/ksp-worker-api", "crates/ksp-worker-raw-transaction-ingest-lib"] [workspace.package] -version = "0.3.15-pre.13" +version = "0.3.15-pre.14" edition = "2024" license = "MIT" repository = "https://git.sasedev.com/Sasedev/khadhroony-solana-project" diff --git a/crates/ksp-store-postgres-lib/src/raw_transaction.rs b/crates/ksp-store-postgres-lib/src/raw_transaction.rs index 1c41053..bd06c8c 100644 --- a/crates/ksp-store-postgres-lib/src/raw_transaction.rs +++ b/crates/ksp-store-postgres-lib/src/raw_transaction.rs @@ -1,5 +1,5 @@ // file: crates/ksp-store-postgres-lib/src/raw_transaction.rs -// version: 8 +// version: 9 pub(crate) mod cursor; @@ -48,8 +48,26 @@ const UPDATE_ARCHIVED_TRANSACTION_SQL: &str = "UPDATE ksp_raw_transactions SET payload = NULL, retention_state = 'archived' WHERE signature = $1 AND retention_state = 'full'"; const UPDATE_PURGED_TRANSACTION_SQL: &str = "UPDATE ksp_raw_transactions SET block_time_unix_millis = NULL, payload = NULL, retention_state = 'purged' WHERE signature = $1 AND retention_state = 'archived'"; +struct RawLogMessagesContentConflictDiagnostic { + available: bool, + first_mismatch_index: std::string::String, + incoming_count: usize, + incoming_first_kind: &'static str, + incoming_first_length: usize, + incoming_has_truncation_marker: bool, + incoming_prefix_of_stored: bool, + incoming_state: &'static str, + stored_count: usize, + stored_first_kind: &'static str, + stored_first_length: usize, + stored_has_truncation_marker: bool, + stored_prefix_of_incoming: bool, + stored_state: &'static str, +} + struct RawPayloadContentConflictDiagnostic { available: bool, + log_messages: RawLogMessagesContentConflictDiagnostic, meta_mismatch: bool, meta_mismatch_fields: std::string::String, other_mismatch: bool, @@ -63,6 +81,7 @@ struct RawTransactionContentConflictDiagnostic { content_hash_mismatch: bool, format_id_mismatch: bool, format_version_mismatch: bool, + log_messages: RawLogMessagesContentConflictDiagnostic, meta_mismatch_fields: std::string::String, payload_bytes_mismatch: bool, payload_diagnostic_available: bool, @@ -1138,6 +1157,7 @@ fn raw_transaction_content_conflict_diagnostic( content_hash_mismatch: stored.payload().content_hash() != incoming.payload().content_hash(), format_id_mismatch: stored.payload().format_id() != incoming.payload().format_id(), format_version_mismatch: stored.payload().format_version() != incoming.payload().format_version(), + log_messages: payload.log_messages, meta_mismatch_fields: payload.meta_mismatch_fields, payload_bytes_mismatch: stored.payload().bytes() != incoming.payload().bytes(), payload_diagnostic_available: payload.available, @@ -1158,6 +1178,7 @@ fn raw_payload_content_conflict_diagnostic(stored: &[u8], incoming: &[u8]) -> Ra _ => { return RawPayloadContentConflictDiagnostic { available: false, + log_messages: raw_log_messages_content_conflict_diagnostic(std::option::Option::None, std::option::Option::None), meta_mismatch: false, meta_mismatch_fields: "unavailable".to_owned(), other_mismatch: false, @@ -1173,8 +1194,10 @@ fn raw_payload_content_conflict_diagnostic(stored: &[u8], incoming: &[u8]) -> Ra let transaction_index_mismatch = stored.get("transactionIndex") != incoming.get("transactionIndex"); let other_mismatch = json_object_other_fields_mismatch(&stored, &incoming, RAW_TRANSACTION_CONTENT_CONFLICT_PAYLOAD_FIELDS.as_slice()); let meta_mismatch_fields = raw_meta_content_conflict_fields(stored.get("meta"), incoming.get("meta")); + let log_messages = raw_log_messages_content_conflict_diagnostic(stored.get("meta"), incoming.get("meta")); return RawPayloadContentConflictDiagnostic { available: true, + log_messages, meta_mismatch, meta_mismatch_fields, other_mismatch, @@ -1184,6 +1207,140 @@ fn raw_payload_content_conflict_diagnostic(stored: &[u8], incoming: &[u8]) -> Ra }; } +fn raw_log_messages_content_conflict_diagnostic( + stored_meta: std::option::Option<&serde_json::Value>, + incoming_meta: std::option::Option<&serde_json::Value>, +) -> RawLogMessagesContentConflictDiagnostic { + let stored = raw_log_messages_value(stored_meta); + let incoming = raw_log_messages_value(incoming_meta); + let stored_state = raw_log_messages_state(stored); + let incoming_state = raw_log_messages_state(incoming); + let (stored_values, incoming_values) = match (stored, incoming) { + (std::option::Option::Some(serde_json::Value::Array(stored)), std::option::Option::Some(serde_json::Value::Array(incoming))) => { + (stored.as_slice(), incoming.as_slice()) + }, + _ => { + return RawLogMessagesContentConflictDiagnostic { + available: false, + first_mismatch_index: "unavailable".to_owned(), + incoming_count: raw_log_messages_count(incoming), + incoming_first_kind: "unavailable", + incoming_first_length: 0, + incoming_has_truncation_marker: raw_log_messages_has_truncation_marker(incoming), + incoming_prefix_of_stored: false, + incoming_state, + stored_count: raw_log_messages_count(stored), + stored_first_kind: "unavailable", + stored_first_length: 0, + stored_has_truncation_marker: raw_log_messages_has_truncation_marker(stored), + stored_prefix_of_incoming: false, + stored_state, + }; + }, + }; + let shared_len = stored_values.len().min(incoming_values.len()); + let first_mismatch = stored_values.iter().zip(incoming_values.iter()).position(|(stored, incoming)| return stored != incoming).or_else(|| { + if stored_values.len() == incoming_values.len() { + return std::option::Option::None; + } + return std::option::Option::Some(shared_len); + }); + let (stored_first_kind, stored_first_length) = raw_log_message_kind_and_length(first_mismatch.and_then(|index| return stored_values.get(index))); + let (incoming_first_kind, incoming_first_length) = raw_log_message_kind_and_length(first_mismatch.and_then(|index| return incoming_values.get(index))); + let first_mismatch_index = match first_mismatch { + std::option::Option::Some(value) => value.to_string(), + std::option::Option::None => "none".to_owned(), + }; + return RawLogMessagesContentConflictDiagnostic { + available: true, + first_mismatch_index, + incoming_count: incoming_values.len(), + incoming_first_kind, + incoming_first_length, + incoming_has_truncation_marker: raw_log_message_array_has_truncation_marker(incoming_values), + incoming_prefix_of_stored: raw_json_array_is_prefix(incoming_values, stored_values), + incoming_state, + stored_count: stored_values.len(), + stored_first_kind, + stored_first_length, + stored_has_truncation_marker: raw_log_message_array_has_truncation_marker(stored_values), + stored_prefix_of_incoming: raw_json_array_is_prefix(stored_values, incoming_values), + stored_state, + }; +} + +fn raw_log_messages_value(meta: std::option::Option<&serde_json::Value>) -> std::option::Option<&serde_json::Value> { + return match meta { + std::option::Option::Some(serde_json::Value::Object(meta)) => meta.get("logMessages"), + _ => std::option::Option::None, + }; +} + +fn raw_log_messages_state(value: std::option::Option<&serde_json::Value>) -> &'static str { + return match value { + std::option::Option::None => "missing", + std::option::Option::Some(serde_json::Value::Null) => "null", + std::option::Option::Some(serde_json::Value::Array(_)) => "array", + std::option::Option::Some(_) => "other", + }; +} + +fn raw_log_messages_count(value: std::option::Option<&serde_json::Value>) -> usize { + return match value { + std::option::Option::Some(serde_json::Value::Array(values)) => values.len(), + _ => 0, + }; +} + +fn raw_json_array_is_prefix(prefix: &[serde_json::Value], values: &[serde_json::Value]) -> bool { + if prefix.len() > values.len() { + return false; + } + return prefix.iter().zip(values.iter()).all(|(left, right)| return left == right); +} + +fn raw_log_message_kind_and_length(value: std::option::Option<&serde_json::Value>) -> (&'static str, usize) { + let line = match value { + std::option::Option::Some(serde_json::Value::String(value)) => value.as_str(), + std::option::Option::Some(_) => return ("non_string", 0), + std::option::Option::None => return ("missing", 0), + }; + let kind = if line.starts_with("Log truncated") || line.starts_with("log truncated") { + "log_truncated" + } else if line.starts_with("Program log:") { + "program_log" + } else if line.starts_with("Program data:") { + "program_data" + } else if line.starts_with("Program ") && line.contains(" invoke [") { + "program_invoke" + } else if line.starts_with("Program ") && line.ends_with(" success") { + "program_success" + } else if line.starts_with("Program ") && line.contains(" failed:") { + "program_failed" + } else if line.starts_with("Program ") && line.contains(" consumed ") && line.contains(" compute units") { + "compute_units" + } else { + "other" + }; + return (kind, line.len()); +} + +fn raw_log_messages_has_truncation_marker(value: std::option::Option<&serde_json::Value>) -> bool { + return match value { + std::option::Option::Some(serde_json::Value::Array(values)) => raw_log_message_array_has_truncation_marker(values.as_slice()), + _ => false, + }; +} + +fn raw_log_message_array_has_truncation_marker(values: &[serde_json::Value]) -> bool { + return values.iter().any(|value| { + return match value { + serde_json::Value::String(line) => line.starts_with("Log truncated") || line.starts_with("log truncated"), + _ => false, + }; + }); +} + fn raw_meta_content_conflict_fields(stored: std::option::Option<&serde_json::Value>, incoming: std::option::Option<&serde_json::Value>) -> std::string::String { if stored == incoming { return "none".to_owned(); @@ -1227,6 +1384,8 @@ fn log_raw_transaction_content_conflict( target: crate::TRACING_TARGET, domain = "store.raw_transaction.content_conflict", network = network.as_str(), + stored_slot = stored.slot(), + incoming_slot = incoming.slot(), slot_mismatch = diagnostic.slot_mismatch, block_time_mismatch = diagnostic.block_time_mismatch, format_id_mismatch = diagnostic.format_id_mismatch, @@ -1240,6 +1399,20 @@ fn log_raw_transaction_content_conflict( transaction_index_mismatch = diagnostic.payload_transaction_index_mismatch, payload_other_mismatch = diagnostic.payload_other_mismatch, meta_mismatch_fields = diagnostic.meta_mismatch_fields.as_str(), + log_messages_diagnostic_available = diagnostic.log_messages.available, + stored_log_messages_state = diagnostic.log_messages.stored_state, + incoming_log_messages_state = diagnostic.log_messages.incoming_state, + stored_log_messages_count = diagnostic.log_messages.stored_count, + incoming_log_messages_count = diagnostic.log_messages.incoming_count, + log_messages_first_mismatch_index = diagnostic.log_messages.first_mismatch_index.as_str(), + stored_log_messages_first_kind = diagnostic.log_messages.stored_first_kind, + incoming_log_messages_first_kind = diagnostic.log_messages.incoming_first_kind, + stored_log_messages_first_length = diagnostic.log_messages.stored_first_length, + incoming_log_messages_first_length = diagnostic.log_messages.incoming_first_length, + stored_log_messages_prefix_of_incoming = diagnostic.log_messages.stored_prefix_of_incoming, + incoming_log_messages_prefix_of_stored = diagnostic.log_messages.incoming_prefix_of_stored, + stored_log_messages_has_truncation_marker = diagnostic.log_messages.stored_has_truncation_marker, + incoming_log_messages_has_truncation_marker = diagnostic.log_messages.incoming_has_truncation_marker, "PostgreSQL Store rejected divergent canonical RAW transaction" ); return; diff --git a/crates/ksp-store-postgres-lib/tests/hardening_completeness.rs b/crates/ksp-store-postgres-lib/tests/hardening_completeness.rs index e37e5dd..c5f758c 100644 --- a/crates/ksp-store-postgres-lib/tests/hardening_completeness.rs +++ b/crates/ksp-store-postgres-lib/tests/hardening_completeness.rs @@ -1,5 +1,5 @@ // file: crates/ksp-store-postgres-lib/tests/hardening_completeness.rs -// version: 24 +// version: 25 #![warn(missing_docs)] #![deny(unreachable_pub)] @@ -606,3 +606,40 @@ fn v0_3_15_pre_012_fix_004_content_conflict_diagnostic_is_field_only_bounded_and assert!(source.contains("fields.push(\"other\")")); return; } + +#[test] +fn v0_3_15_pre_014_content_conflict_diagnostic_exposes_only_block_slot_and_bounded_log_shape() { + let source = include_str!("../src/raw_transaction.rs"); + for required in [ + "stored_slot", + "incoming_slot", + "log_messages_first_mismatch_index", + "stored_log_messages_count", + "incoming_log_messages_count", + "stored_log_messages_first_kind", + "incoming_log_messages_first_kind", + "stored_log_messages_prefix_of_incoming", + "incoming_log_messages_prefix_of_stored", + "stored_log_messages_has_truncation_marker", + "incoming_log_messages_has_truncation_marker", + "raw_log_message_kind_and_length", + "raw_log_message_array_has_truncation_marker", + ] { + assert!(source.contains(required), "missing pre.014 bounded logMessages diagnostic field: {required}"); + } + let start = source.find("fn log_raw_transaction_content_conflict("); + assert!(start.is_some(), "missing pre.014 conflict logger"); + let Some(start) = start else { + return; + }; + let end = source[start..].find("async fn log_raw_transaction_content_conflict_provenance("); + assert!(end.is_some(), "missing pre.014 conflict logger end marker"); + let Some(end) = end else { + return; + }; + let logger = &source[start..start + end]; + for forbidden in ["signature =", "blockhash =", "previous_blockhash =", "log_message =", "log_line =", "payload ="] { + assert!(!logger.contains(forbidden), "pre.014 conflict logger exposes forbidden material: {forbidden}"); + } + return; +} diff --git a/crates/ksp-store-postgres-lib/unit_tests/raw_transaction.rs b/crates/ksp-store-postgres-lib/unit_tests/raw_transaction.rs index d7bb4a1..3ae6c22 100644 --- a/crates/ksp-store-postgres-lib/unit_tests/raw_transaction.rs +++ b/crates/ksp-store-postgres-lib/unit_tests/raw_transaction.rs @@ -1,5 +1,5 @@ // file: crates/ksp-store-postgres-lib/unit_tests/raw_transaction.rs -// version: 7 +// version: 8 fn network() -> ksp_store_api::RawNetworkId { return match ksp_store_api::RawNetworkId::new("devnet") { @@ -429,6 +429,52 @@ fn v0_3_15_pre_012_fix_004_content_conflict_diagnostic_collapses_unknown_meta_ke return; } +#[test] +fn v0_3_15_pre_014_log_messages_conflict_diagnostic_distinguishes_prefix_truncation_and_line_kind_without_text() { + let stored = raw_transaction( + 9, + br#"{"transaction":["AQID","base64"],"meta":{"logMessages":["Program Vote111 invoke [1]","Program log: stable","Program Vote111 success"]},"version":0}"#, + 7, + ); + let incoming = raw_transaction( + 9, + br#"{"transaction":["AQID","base64"],"meta":{"logMessages":["Program Vote111 invoke [1]","Program log: stable","Log truncated"]},"version":0}"#, + 8, + ); + let diagnostic = super::raw_transaction_content_conflict_diagnostic(&stored, &incoming); + assert!(diagnostic.log_messages.available); + assert_eq!(diagnostic.log_messages.stored_state, "array"); + assert_eq!(diagnostic.log_messages.incoming_state, "array"); + assert_eq!(diagnostic.log_messages.stored_count, 3); + assert_eq!(diagnostic.log_messages.incoming_count, 3); + assert_eq!(diagnostic.log_messages.first_mismatch_index, "2"); + assert_eq!(diagnostic.log_messages.stored_first_kind, "program_success"); + assert_eq!(diagnostic.log_messages.incoming_first_kind, "log_truncated"); + assert!(!diagnostic.log_messages.stored_has_truncation_marker); + assert!(diagnostic.log_messages.incoming_has_truncation_marker); + assert!(!diagnostic.log_messages.stored_prefix_of_incoming); + assert!(!diagnostic.log_messages.incoming_prefix_of_stored); + return; +} + +#[test] +fn v0_3_15_pre_014_log_messages_conflict_diagnostic_detects_strict_prefix_without_exposing_line_material() { + let stored = raw_transaction( + 9, + br#"{"transaction":["AQID","base64"],"meta":{"logMessages":["Program Vote111 invoke [1]","Program Vote111 success"]},"version":0}"#, + 7, + ); + let incoming = raw_transaction(9, br#"{"transaction":["AQID","base64"],"meta":{"logMessages":["Program Vote111 invoke [1]"]},"version":0}"#, 8); + let diagnostic = super::raw_transaction_content_conflict_diagnostic(&stored, &incoming); + assert!(diagnostic.log_messages.available); + assert_eq!(diagnostic.log_messages.first_mismatch_index, "1"); + assert_eq!(diagnostic.log_messages.stored_first_kind, "program_success"); + assert_eq!(diagnostic.log_messages.incoming_first_kind, "missing"); + assert!(!diagnostic.log_messages.stored_prefix_of_incoming); + assert!(diagnostic.log_messages.incoming_prefix_of_stored); + return; +} + #[test] fn pre_005_observation_idempotence_compares_reference_and_complete_provenance() { let incoming_transaction = raw_transaction(8, &[1, 2, 3], 4); diff --git a/crates/ksp-worker-raw-transaction-ingest-lib/src/runtime_resources.rs b/crates/ksp-worker-raw-transaction-ingest-lib/src/runtime_resources.rs index fad6106..f4138d6 100644 --- a/crates/ksp-worker-raw-transaction-ingest-lib/src/runtime_resources.rs +++ b/crates/ksp-worker-raw-transaction-ingest-lib/src/runtime_resources.rs @@ -1,5 +1,5 @@ // file: crates/ksp-worker-raw-transaction-ingest-lib/src/runtime_resources.rs -// version: 48 +// version: 49 use sha2::Digest; // rust-rules: trait-import @@ -19,6 +19,7 @@ pub const MIN_RAW_TRANSACTION_INGEST_HTTP_POLL_INTERVAL: std::time::Duration = s pub const MIN_RAW_TRANSACTION_INGEST_HTTP_POLL_MAX_BLOCKS_PER_CYCLE: u16 = 1; const MAX_RAW_TRANSACTION_INGEST_REPAIR_BURST: usize = 1; +const RAW_TRANSACTION_INGEST_BLOCK_IDENTITY_FINGERPRINT_DOMAIN: &[u8] = b"ksp.raw_transaction_ingest.block_identity.v1\0"; const RAW_TRANSACTION_INGEST_HELIUS_TRANSACTION_FILTER_FINGERPRINT_DOMAIN: &[u8] = b"ksp.raw_transaction_ingest.helius_transaction.filter.v1\0"; const RAW_TRANSACTION_INGEST_HELIUS_TRANSACTION_HTTP_PROTOCOL: &str = "helius_ws_http"; const RAW_TRANSACTION_INGEST_HELIUS_TRANSACTION_HTTP_SOURCE_KEY_DOMAIN: &[u8] = b"ksp.raw_transaction_ingest.helius_transaction_http.source_key.v1\0"; @@ -3595,6 +3596,7 @@ async fn fetch_yellowstone_block_ingresses( std::option::Option::Some(value) => value, std::option::Option::None => return std::result::Result::Err(crate::runtime_error("source.yellowstone_block_not_available")), }; + log_block_identity("yellowstone_block_hydration", &hydration.network, slot, hydration.commitment, observed.endpoint_name(), observed.provider(), block); let received_at = match current_raw_timestamp() { std::result::Result::Ok(value) => value, std::result::Result::Err(error) => return std::result::Result::Err(error), @@ -3665,6 +3667,7 @@ fn project_http_block_polling_ingresses( observed: &ksp_onchain_transport_lib::HttpObservedValue>, received_at: ksp_store_lib::RawTimestamp, ) -> ksp_core_lib::Result> { + log_block_identity("http_block_polling", &source.network, slot, source.commitment, observed.endpoint_name(), observed.provider(), block); let transactions = match block.transactions() { ksp_onchain_transport_lib::SolanaWireField::Value(value) => value, ksp_onchain_transport_lib::SolanaWireField::Omitted | ksp_onchain_transport_lib::SolanaWireField::Null => { @@ -5573,6 +5576,61 @@ fn hydration_method_code(family: RawTransactionIngestSourceFamily) -> &'static s }; } +fn block_identity_fingerprint(block: &ksp_onchain_transport_lib::SolanaConfirmedBlock) -> std::string::String { + const HEX: &[u8; 16] = b"0123456789abcdef"; + let mut hasher = sha2::Sha256::new(); + hasher.update(RAW_TRANSACTION_INGEST_BLOCK_IDENTITY_FINGERPRINT_DOMAIN); + hash_live_source_key_component(&mut hasher, block.blockhash().as_bytes()); + hash_live_source_key_component(&mut hasher, block.previous_blockhash().as_bytes()); + hasher.update(block.parent_slot().to_be_bytes()); + match block.block_height() { + std::option::Option::Some(value) => { + hasher.update([1_u8]); + hasher.update(value.to_be_bytes()); + }, + std::option::Option::None => hasher.update([0_u8]), + } + let digest: [u8; 32] = hasher.finalize().into(); + let mut value = std::string::String::with_capacity(71); + value.push_str("sha256."); + for byte in digest { + value.push(char::from(HEX[usize::from(byte >> 4)])); + value.push(char::from(HEX[usize::from(byte & 0x0f)])); + } + return value; +} + +fn log_block_identity( + source_kind: &'static str, + network: &ksp_store_lib::RawNetworkId, + slot: u64, + commitment: ksp_onchain_transport_lib::SolanaCommitment, + endpoint_name: &str, + provider_name: &ksp_onchain_transport_lib::HttpProviderName, + block: &ksp_onchain_transport_lib::SolanaConfirmedBlock, +) { + let fingerprint = block_identity_fingerprint(block); + let block_height = match block.block_height() { + std::option::Option::Some(value) => value.to_string(), + std::option::Option::None => "none".to_owned(), + }; + ksp_logging_lib::debug!( + target: crate::TRACING_TARGET, + domain = "raw_transaction_ingest.block_identity", + source_kind = source_kind, + network = network.as_str(), + slot = slot, + provider = provider_name.as_str(), + endpoint_id = endpoint_name, + commitment = commitment.as_str(), + parent_slot = block.parent_slot(), + block_height = block_height.as_str(), + block_identity_fingerprint = fingerprint.as_str(), + "RAW transaction ingest observed HTTP block identity" + ); + return; +} + fn fingerprint_filter_code(fingerprint: &[u8; 32]) -> ksp_core_lib::Result { const HEX: &[u8; 16] = b"0123456789abcdef"; let mut value = std::string::String::with_capacity(71); diff --git a/crates/ksp-worker-raw-transaction-ingest-lib/tests/hardening.rs b/crates/ksp-worker-raw-transaction-ingest-lib/tests/hardening.rs index b0ea833..710b049 100644 --- a/crates/ksp-worker-raw-transaction-ingest-lib/tests/hardening.rs +++ b/crates/ksp-worker-raw-transaction-ingest-lib/tests/hardening.rs @@ -1,5 +1,5 @@ // file: crates/ksp-worker-raw-transaction-ingest-lib/tests/hardening.rs -// version: 39 +// version: 40 //! External public, security, redaction and release-boundary hardening canaries through `v0.3.15-pre.004`. @@ -1449,3 +1449,35 @@ fn v0_3_15_pre_013_yellowstone_block_hydration_is_bounded_concurrent_and_stop_pr } return; } + +#[test] +fn v0_3_15_pre_014_cross_provider_block_identity_diagnostic_is_safe_and_covers_both_get_block_routes() { + let resources = include_str!("../src/runtime_resources.rs"); + for required in [ + "RAW_TRANSACTION_INGEST_BLOCK_IDENTITY_FINGERPRINT_DOMAIN", + "fn block_identity_fingerprint(", + "fn log_block_identity(", + "domain = \"raw_transaction_ingest.block_identity\"", + "\"yellowstone_block_hydration\"", + "\"http_block_polling\"", + "block_identity_fingerprint = fingerprint.as_str()", + "parent_slot = block.parent_slot()", + ] { + assert!(resources.contains(required), "missing pre.014 block identity diagnostic guard: {required}"); + } + let start = resources.find("fn log_block_identity("); + assert!(start.is_some(), "missing pre.014 block identity logger"); + let Some(start) = start else { + return; + }; + let end = resources[start..].find("fn fingerprint_filter_code("); + assert!(end.is_some(), "missing pre.014 block identity logger end marker"); + let Some(end) = end else { + return; + }; + let logger = &resources[start..start + end]; + for forbidden in ["blockhash =", "previous_blockhash =", "signature =", "payload =", "transaction ="] { + assert!(!logger.contains(forbidden), "pre.014 block identity logger exposes forbidden material: {forbidden}"); + } + return; +} diff --git a/crates/ksp-worker-raw-transaction-ingest-lib/tests/release_completeness.rs b/crates/ksp-worker-raw-transaction-ingest-lib/tests/release_completeness.rs index 883d958..391ab66 100644 --- a/crates/ksp-worker-raw-transaction-ingest-lib/tests/release_completeness.rs +++ b/crates/ksp-worker-raw-transaction-ingest-lib/tests/release_completeness.rs @@ -1,5 +1,5 @@ // file: crates/ksp-worker-raw-transaction-ingest-lib/tests/release_completeness.rs -// version: 35 +// version: 36 //! Release-completeness canaries through the `v0.3.15-pre.004` WebSocket capability-enforcement tranche. @@ -496,3 +496,13 @@ fn v0_3_15_pre_013_yellowstone_block_backpressure_fix_adds_no_public_surface() { assert!(!root.contains("YellowstoneBlockHydrationContext")); return; } + +#[test] +fn v0_3_15_pre_014_cross_provider_convergence_diagnostic_adds_no_public_worker_surface() { + let hardening = include_str!("hardening.rs"); + let root = include_str!("../src/lib.rs"); + assert!(hardening.contains("v0_3_15_pre_014_cross_provider_block_identity_diagnostic_is_safe_and_covers_both_get_block_routes")); + assert!(!root.contains("BlockIdentityFingerprint")); + assert!(!root.contains("LogMessagesConflictDiagnostic")); + return; +} diff --git a/deltas/0.3.15/pre.014.md b/deltas/0.3.15/pre.014.md new file mode 100644 index 0000000..ff717a1 --- /dev/null +++ b/deltas/0.3.15/pre.014.md @@ -0,0 +1,111 @@ + + + +# Delta v0.3.15-pre.014 — convergence inter-provider diagnostiquable + +## Base + +```text +ksp-general-0.3.15-pre.013.zip +SHA-256: 4d463230f45a55f68994443f138afd246b6ec34bea21357c2cedfe5d8fe1c5c6 +``` + +Le gate opérateur de `pre.013` est propre et le live ne reproduit plus le `grpc_backpressure_overflow` fermé par cette tranche. Yellowstone devient néanmoins terminal `content_conflict` une fois pendant l'exécution parallèle avec HTTP Block Polling. + +Le diagnostic de `pre.012-fix.004` prouve que le conflit ne porte ni sur le slot, ni sur `block_time`, ni sur les octets/instructions de transaction, ni sur la version, ni sur `transactionIndex`. Seul `meta.logMessages` diffère. La version déjà canonique provient de `solana_mainnet_public`; la version entrante est hydratée par `publicnode_solana_mainnet_rpc` après trigger Yellowstone, toutes deux en `confirmed`. + +## Diagnostic ajouté + +Chaque `getBlock` réussi des voies Yellowstone Block Hydration et HTTP Block Polling émet désormais un diagnostic DEBUG sûr : + +```text +domain=raw_transaction_ingest.block_identity +source_kind +network +slot +provider +endpoint_id +commitment +parent_slot +block_height +block_identity_fingerprint +``` + +`block_identity_fingerprint` est un SHA-256 domain-separated calculé à partir de `blockhash`, `previousBlockhash`, `parentSlot` et `blockHeight`. Les blockhash bruts ne sont jamais journalisés. + +Lors d'un `content_conflict`, Store complète son masque existant avec : + +```text +stored_slot +incoming_slot +log_messages_diagnostic_available +stored_log_messages_state +incoming_log_messages_state +stored_log_messages_count +incoming_log_messages_count +log_messages_first_mismatch_index +stored_log_messages_first_kind +incoming_log_messages_first_kind +stored_log_messages_first_length +incoming_log_messages_first_length +stored_log_messages_prefix_of_incoming +incoming_log_messages_prefix_of_stored +stored_log_messages_has_truncation_marker +incoming_log_messages_has_truncation_marker +``` + +Les types de ligne sont réduits à une taxonomie statique ; aucun texte arbitraire de log n'est recopié. + +## Frontières + +Aucun changement de : + +```text +RawTransaction canonical identity +égalité / content_hash +politique fail-closed content_conflict +Store schema +Transport gRPC / HTTP +Config +Desk +Backfill +public Worker API +public Store API +``` + +`logMessages` reste partie intégrante du payload canonique tant que la cause de la divergence n'est pas prouvée. + +## Canaris + +```text +v0_3_15_pre_014_log_messages_conflict_diagnostic_distinguishes_prefix_truncation_and_line_kind_without_text +v0_3_15_pre_014_log_messages_conflict_diagnostic_detects_strict_prefix_without_exposing_line_material +v0_3_15_pre_014_content_conflict_diagnostic_exposes_only_block_slot_and_bounded_log_shape +v0_3_15_pre_014_cross_provider_block_identity_diagnostic_is_safe_and_covers_both_get_block_routes +v0_3_15_pre_014_cross_provider_convergence_diagnostic_adds_no_public_worker_surface +``` + +## Version + +```text +workspace.package.version : 0.3.15-pre.14 +root Cargo header counter : 628 +``` + +## Validation attendue + +```text +cargo fmt --all +cargo fmt --all -- --check +python3 scripts/audit_rust_workspace_rules.py +python3 scripts/audit_markdown_tables.py README.md RULES.md ROADMAP.md CHANGELOG.md docs prompts crates deltas/0.3.15 +cargo check --workspace +cargo clippy --workspace --all-targets --all-features -- -D warnings +cargo test -p ksp-store-postgres-lib --all-targets --all-features +cargo test -p ksp-worker-raw-transaction-ingest-lib --all-targets --all-features +cargo test -p ksp-app-raw-transaction-ingest-desk --all-targets --all-features +cargo test --workspace --all-targets --all-features +(cd crates/ksp-app-raw-transaction-ingest-desk && cargo tauri dev) +``` + +Live : Yellowstone + HTTP Block Polling en `confirmed` jusqu'à reproduction d'un conflit. Au slot annoncé par Store, comparer les deux `block_identity_fingerprint`, puis lire la forme bornée de `logMessages`. Si nécessaire, répéter ensuite la même expérience en `finalized` comme diagnostic de finalité, sans changer le défaut produit par défaut. diff --git a/docs/plans/036-V0_3_15_RAW_TRANSACTION_INGEST_DESK_PLAN.md b/docs/plans/036-V0_3_15_RAW_TRANSACTION_INGEST_DESK_PLAN.md index b25b0c6..daab699 100644 --- a/docs/plans/036-V0_3_15_RAW_TRANSACTION_INGEST_DESK_PLAN.md +++ b/docs/plans/036-V0_3_15_RAW_TRANSACTION_INGEST_DESK_PLAN.md @@ -1,5 +1,5 @@ - + # Plan v0.3.15 — Raw Transaction Ingest Desk @@ -855,11 +855,17 @@ Le mode Yellowstone Block reçoit désormais une partition déterministe des bud Budget cible : une tranche corrective acquisition/throughput, suivie d'un live Mainnet prolongé. -### `pre.014` — completeness/security cross-layer +### `pre.014` — convergence inter-provider + completeness/security cross-layer -Fermer les canaris Config/Transport/Worker/Common RAW/Store/Desk, dependency firewalls, API publique, redaction et absence de seconde pipeline ou d'orchestration Backfill. +**État : implémenté, gate/live opérateur requis.** Le gate de `pre.013` compile et teste proprement le workspace et le live ne reproduit plus `grpc_backpressure_overflow`. Un unique `content_conflict` Mainnet réapparaît toutefois entre HTTP Block Polling (`solana_mainnet_public`) et Yellowstone hydraté par HTTP (`publicnode_solana_mainnet_rpc`), tous deux en `confirmed`. Le diagnostic `pre.012-fix.004` prouve que `slot`, `block_time`, transaction wire, version et `transactionIndex` sont identiques ; seule la valeur `meta.logMessages` diverge. -Budget cible : une tranche de clôture fonctionnelle. +La tranche ne normalise ni n'ignore `logMessages`. Elle ajoute un diagnostic de convergence sûr permettant de distinguer un fork `confirmed` d'une divergence de représentation/provider : chaque `getBlock` réussi des deux routes journalise le slot et un fingerprint SHA-256 irréversible de l'identité de bloc (`blockhash`, `previousBlockhash`, `parentSlot`, `blockHeight`) sans exposer les blockhash. En cas de conflit RAW, Store journalise désormais les slots exacts et une forme bornée de la divergence `logMessages` : état absent/null/array, cardinalités, premier index divergent, type statique de la première ligne divergente, longueur, relation de préfixe et présence d'un marqueur de troncature. Aucun texte de log distant n'est copié. + +Le live doit comparer les fingerprints au slot du conflit. Fingerprints différents au même slot indiquent deux vues de bloc/fork différentes au commitment observé ; fingerprint identique avec `logMessages` divergents indique une divergence de réponse/recording RPC entre providers pour le même bloc. Ce diagnostic précède toute décision de canonicalisation durable. + +Les canaris de clôture continuent de vérifier dependency firewalls, API publique, redaction et absence de seconde pipeline ou d'orchestration Backfill. + +Budget cible : une tranche de convergence/completeness cross-layer, sans modification de la politique fail-closed. ### `pre.015` — gate technique/live final diff --git a/docs/validation/032-V0_3_15_RAW_TRANSACTION_INGEST_DESK.md b/docs/validation/032-V0_3_15_RAW_TRANSACTION_INGEST_DESK.md index 71a63df..ae6e19c 100644 --- a/docs/validation/032-V0_3_15_RAW_TRANSACTION_INGEST_DESK.md +++ b/docs/validation/032-V0_3_15_RAW_TRANSACTION_INGEST_DESK.md @@ -1,5 +1,5 @@ - + # Validation v0.3.15 — Raw Transaction Ingest Desk @@ -885,3 +885,78 @@ Aucun type du coordinateur n'est exporté publiquement. Validation live requise : Yellowstone Mainnet seul pendant une durée supérieure aux ~80 s de reproduction, puis Yellowstone + HTTP Block Polling en parallèle. Le run doit rester `Healthy`, sans `grpc_backpressure_overflow`, et les Stop ciblés doivent finir `Stopped`. Le diagnostic `pre.012-fix.004` reste actif si un `content_conflict` réapparaît. +### `pre.014` — diagnostic de convergence inter-provider au niveau bloc / `logMessages` + +Le gate opérateur de `pre.013` est propre : audits Rust/Markdown, `cargo check --workspace`, Clippy strict, Worker (`161/161` unitaires + suites externes), Desk (`50/50` unitaires + suites externes) et workspace passent. Le live Mainnet confirme également l'objectif de `pre.013` : aucune occurrence de `grpc_backpressure_overflow` ni de `Yellowstone subscribe update queue overflowed` n'est observée dans le run transmis. + +Le live reproduit en revanche un unique terminal Yellowstone `content_conflict`. Le diagnostic Store de `pre.012-fix.004` donne exactement : + +```text +slot_mismatch=false +block_time_mismatch=false +format_id_mismatch=false +format_version_mismatch=false +transaction_mismatch=false +meta_mismatch=true +version_mismatch=false +transaction_index_mismatch=false +payload_other_mismatch=false +meta_mismatch_fields="logMessages" +``` + +La provenance durable classe les deux acquisitions : + +```text +stored : solana-public / solana_http / block_polling_get_block / solana_mainnet_public / confirmed +incoming : ys.publicnode:http.publicnode / yellowstone_http / block_subscribe_get_block / + ys.publicnode_solana_mainnet_yellowstone:http.publicnode_solana_mainnet_rpc / confirmed +``` + +La transaction canonique elle-même est donc identique ; seule la métadonnée d'exécution `meta.logMessages` diffère. Le diagnostic existant ne permet toutefois pas encore de savoir si les deux RPC décrivent le même bloc `confirmed` ou deux forks/vues de bloc différents au même slot. + +`pre.014` ajoute deux niveaux de diagnostic sans changer la politique de conflit : + +```text +Worker / chaque getBlock réussi + source_kind + network + slot + provider + endpoint_id + commitment + parent_slot + block_height + block_identity_fingerprint = SHA-256(domain || blockhash || previousBlockhash || parentSlot || blockHeight) + +Store / seulement lors d'un content_conflict + stored_slot / incoming_slot + logMessages state : missing | null | array | other + count de chaque côté + premier index divergent + type statique de la première ligne divergente : + program_invoke | program_success | program_failed | program_log | + program_data | compute_units | log_truncated | other | non_string | missing + longueur de la première ligne divergente + stored_prefix_of_incoming / incoming_prefix_of_stored + has_truncation_marker de chaque côté +``` + +Le blockhash, le previousBlockhash, les signatures, les payloads et le texte réel de `logMessages` ne sont jamais journalisés. Le fingerprint de bloc est irréversible et sert uniquement à comparer deux acquisitions du même slot. + +Interprétation du prochain conflit : + +```text +même slot + fingerprints différents + => vues/forks confirmed différents entre providers au moment des getBlock + +même slot + même fingerprint + logMessages différents + => divergence provider/node sur la représentation ou l'enregistrement des logs du même bloc + +incoming_prefix_of_stored=true ou stored_prefix_of_incoming=true + => divergence compatible avec une liste tronquée/incomplète + +*_has_truncation_marker=true + => marqueur explicite de troncature présent dans une des réponses +``` + +Non-claims : `pre.014` ne retire pas `logMessages` du contenu canonique, ne choisit aucun provider comme vérité, ne transforme pas `content_conflict` en succès, ne modifie pas le schema Store et n'expose aucun nouveau type public Worker/Store.