0.3.15-pre.014

Cargo.toml

@@ -1,12 +1,12 @@
 # file: Cargo.toml
-# version: 627
+# version: 628
 
 [workspace]
 resolver = "3"
 members = ["crates/ksp-app-backfill-desk", "crates/ksp-app-config-desk", "crates/ksp-app-raw-transaction-ingest-desk", "crates/ksp-app-solprices-desk", "crates/ksp-app-store-desk", "crates/ksp-app-wallet-desk", "crates/ksp-config-lib", "crates/ksp-core-lib", "crates/ksp-interface-lib", "crates/ksp-job-api", "crates/ksp-job-backfill-lib", "crates/ksp-logging-lib", "crates/ksp-offchain-transport-lib", "crates/ksp-onchain-transport-lib", "crates/ksp-program-api", "crates/ksp-raw-transaction-lib", "crates/ksp-store-api", "crates/ksp-store-lib", "crates/ksp-store-postgres-lib", "crates/ksp-wallet-lib", "crates/ksp-worker-api", "crates/ksp-worker-raw-transaction-ingest-lib"]
 
 [workspace.package]
-version = "0.3.15-pre.13"
+version = "0.3.15-pre.14"
 edition = "2024"
 license = "MIT"
 repository = "https://git.sasedev.com/Sasedev/khadhroony-solana-project"

crates/ksp-store-postgres-lib/src/raw_transaction.rs

@@ -1,5 +1,5 @@
 // file: crates/ksp-store-postgres-lib/src/raw_transaction.rs
-// version: 8
+// version: 9
 
 pub(crate) mod cursor;
 
@@ -48,8 +48,26 @@ const UPDATE_ARCHIVED_TRANSACTION_SQL: &str =
     "UPDATE ksp_raw_transactions SET payload = NULL, retention_state = 'archived' WHERE signature = $1 AND retention_state = 'full'";
 const UPDATE_PURGED_TRANSACTION_SQL: &str = "UPDATE ksp_raw_transactions SET block_time_unix_millis = NULL, payload = NULL, retention_state = 'purged' WHERE signature = $1 AND retention_state = 'archived'";
 
+struct RawLogMessagesContentConflictDiagnostic {
+    available: bool,
+    first_mismatch_index: std::string::String,
+    incoming_count: usize,
+    incoming_first_kind: &'static str,
+    incoming_first_length: usize,
+    incoming_has_truncation_marker: bool,
+    incoming_prefix_of_stored: bool,
+    incoming_state: &'static str,
+    stored_count: usize,
+    stored_first_kind: &'static str,
+    stored_first_length: usize,
+    stored_has_truncation_marker: bool,
+    stored_prefix_of_incoming: bool,
+    stored_state: &'static str,
+}
+
 struct RawPayloadContentConflictDiagnostic {
     available: bool,
+    log_messages: RawLogMessagesContentConflictDiagnostic,
     meta_mismatch: bool,
     meta_mismatch_fields: std::string::String,
     other_mismatch: bool,
@@ -63,6 +81,7 @@ struct RawTransactionContentConflictDiagnostic {
     content_hash_mismatch: bool,
     format_id_mismatch: bool,
     format_version_mismatch: bool,
+    log_messages: RawLogMessagesContentConflictDiagnostic,
     meta_mismatch_fields: std::string::String,
     payload_bytes_mismatch: bool,
     payload_diagnostic_available: bool,
@@ -1138,6 +1157,7 @@ fn raw_transaction_content_conflict_diagnostic(
         content_hash_mismatch: stored.payload().content_hash() != incoming.payload().content_hash(),
         format_id_mismatch: stored.payload().format_id() != incoming.payload().format_id(),
         format_version_mismatch: stored.payload().format_version() != incoming.payload().format_version(),
+        log_messages: payload.log_messages,
         meta_mismatch_fields: payload.meta_mismatch_fields,
         payload_bytes_mismatch: stored.payload().bytes() != incoming.payload().bytes(),
         payload_diagnostic_available: payload.available,
@@ -1158,6 +1178,7 @@ fn raw_payload_content_conflict_diagnostic(stored: &[u8], incoming: &[u8]) -> Ra
         _ => {
             return RawPayloadContentConflictDiagnostic {
                 available: false,
+                log_messages: raw_log_messages_content_conflict_diagnostic(std::option::Option::None, std::option::Option::None),
                 meta_mismatch: false,
                 meta_mismatch_fields: "unavailable".to_owned(),
                 other_mismatch: false,
@@ -1173,8 +1194,10 @@ fn raw_payload_content_conflict_diagnostic(stored: &[u8], incoming: &[u8]) -> Ra
     let transaction_index_mismatch = stored.get("transactionIndex") != incoming.get("transactionIndex");
     let other_mismatch = json_object_other_fields_mismatch(&stored, &incoming, RAW_TRANSACTION_CONTENT_CONFLICT_PAYLOAD_FIELDS.as_slice());
     let meta_mismatch_fields = raw_meta_content_conflict_fields(stored.get("meta"), incoming.get("meta"));
+    let log_messages = raw_log_messages_content_conflict_diagnostic(stored.get("meta"), incoming.get("meta"));
     return RawPayloadContentConflictDiagnostic {
         available: true,
+        log_messages,
         meta_mismatch,
         meta_mismatch_fields,
         other_mismatch,
@@ -1184,6 +1207,140 @@ fn raw_payload_content_conflict_diagnostic(stored: &[u8], incoming: &[u8]) -> Ra
     };
 }
 
+fn raw_log_messages_content_conflict_diagnostic(
+    stored_meta: std::option::Option<&serde_json::Value>,
+    incoming_meta: std::option::Option<&serde_json::Value>,
+) -> RawLogMessagesContentConflictDiagnostic {
+    let stored = raw_log_messages_value(stored_meta);
+    let incoming = raw_log_messages_value(incoming_meta);
+    let stored_state = raw_log_messages_state(stored);
+    let incoming_state = raw_log_messages_state(incoming);
+    let (stored_values, incoming_values) = match (stored, incoming) {
+        (std::option::Option::Some(serde_json::Value::Array(stored)), std::option::Option::Some(serde_json::Value::Array(incoming))) => {
+            (stored.as_slice(), incoming.as_slice())
+        },
+        _ => {
+            return RawLogMessagesContentConflictDiagnostic {
+                available: false,
+                first_mismatch_index: "unavailable".to_owned(),
+                incoming_count: raw_log_messages_count(incoming),
+                incoming_first_kind: "unavailable",
+                incoming_first_length: 0,
+                incoming_has_truncation_marker: raw_log_messages_has_truncation_marker(incoming),
+                incoming_prefix_of_stored: false,
+                incoming_state,
+                stored_count: raw_log_messages_count(stored),
+                stored_first_kind: "unavailable",
+                stored_first_length: 0,
+                stored_has_truncation_marker: raw_log_messages_has_truncation_marker(stored),
+                stored_prefix_of_incoming: false,
+                stored_state,
+            };
+        },
+    };
+    let shared_len = stored_values.len().min(incoming_values.len());
+    let first_mismatch = stored_values.iter().zip(incoming_values.iter()).position(|(stored, incoming)| return stored != incoming).or_else(|| {
+        if stored_values.len() == incoming_values.len() {
+            return std::option::Option::None;
+        }
+        return std::option::Option::Some(shared_len);
+    });
+    let (stored_first_kind, stored_first_length) = raw_log_message_kind_and_length(first_mismatch.and_then(|index| return stored_values.get(index)));
+    let (incoming_first_kind, incoming_first_length) = raw_log_message_kind_and_length(first_mismatch.and_then(|index| return incoming_values.get(index)));
+    let first_mismatch_index = match first_mismatch {
+        std::option::Option::Some(value) => value.to_string(),
+        std::option::Option::None => "none".to_owned(),
+    };
+    return RawLogMessagesContentConflictDiagnostic {
+        available: true,
+        first_mismatch_index,
+        incoming_count: incoming_values.len(),
+        incoming_first_kind,
+        incoming_first_length,
+        incoming_has_truncation_marker: raw_log_message_array_has_truncation_marker(incoming_values),
+        incoming_prefix_of_stored: raw_json_array_is_prefix(incoming_values, stored_values),
+        incoming_state,
+        stored_count: stored_values.len(),
+        stored_first_kind,
+        stored_first_length,
+        stored_has_truncation_marker: raw_log_message_array_has_truncation_marker(stored_values),
+        stored_prefix_of_incoming: raw_json_array_is_prefix(stored_values, incoming_values),
+        stored_state,
+    };
+}
+
+fn raw_log_messages_value(meta: std::option::Option<&serde_json::Value>) -> std::option::Option<&serde_json::Value> {
+    return match meta {
+        std::option::Option::Some(serde_json::Value::Object(meta)) => meta.get("logMessages"),
+        _ => std::option::Option::None,
+    };
+}
+
+fn raw_log_messages_state(value: std::option::Option<&serde_json::Value>) -> &'static str {
+    return match value {
+        std::option::Option::None => "missing",
+        std::option::Option::Some(serde_json::Value::Null) => "null",
+        std::option::Option::Some(serde_json::Value::Array(_)) => "array",
+        std::option::Option::Some(_) => "other",
+    };
+}
+
+fn raw_log_messages_count(value: std::option::Option<&serde_json::Value>) -> usize {
+    return match value {
+        std::option::Option::Some(serde_json::Value::Array(values)) => values.len(),
+        _ => 0,
+    };
+}
+
+fn raw_json_array_is_prefix(prefix: &[serde_json::Value], values: &[serde_json::Value]) -> bool {
+    if prefix.len() > values.len() {
+        return false;
+    }
+    return prefix.iter().zip(values.iter()).all(|(left, right)| return left == right);
+}
+
+fn raw_log_message_kind_and_length(value: std::option::Option<&serde_json::Value>) -> (&'static str, usize) {
+    let line = match value {
+        std::option::Option::Some(serde_json::Value::String(value)) => value.as_str(),
+        std::option::Option::Some(_) => return ("non_string", 0),
+        std::option::Option::None => return ("missing", 0),
+    };
+    let kind = if line.starts_with("Log truncated") || line.starts_with("log truncated") {
+        "log_truncated"
+    } else if line.starts_with("Program log:") {
+        "program_log"
+    } else if line.starts_with("Program data:") {
+        "program_data"
+    } else if line.starts_with("Program ") && line.contains(" invoke [") {
+        "program_invoke"
+    } else if line.starts_with("Program ") && line.ends_with(" success") {
+        "program_success"
+    } else if line.starts_with("Program ") && line.contains(" failed:") {
+        "program_failed"
+    } else if line.starts_with("Program ") && line.contains(" consumed ") && line.contains(" compute units") {
+        "compute_units"
+    } else {
+        "other"
+    };
+    return (kind, line.len());
+}
+
+fn raw_log_messages_has_truncation_marker(value: std::option::Option<&serde_json::Value>) -> bool {
+    return match value {
+        std::option::Option::Some(serde_json::Value::Array(values)) => raw_log_message_array_has_truncation_marker(values.as_slice()),
+        _ => false,
+    };
+}
+
+fn raw_log_message_array_has_truncation_marker(values: &[serde_json::Value]) -> bool {
+    return values.iter().any(|value| {
+        return match value {
+            serde_json::Value::String(line) => line.starts_with("Log truncated") || line.starts_with("log truncated"),
+            _ => false,
+        };
+    });
+}
+
 fn raw_meta_content_conflict_fields(stored: std::option::Option<&serde_json::Value>, incoming: std::option::Option<&serde_json::Value>) -> std::string::String {
     if stored == incoming {
         return "none".to_owned();
@@ -1227,6 +1384,8 @@ fn log_raw_transaction_content_conflict(
         target: crate::TRACING_TARGET,
         domain = "store.raw_transaction.content_conflict",
         network = network.as_str(),
+        stored_slot = stored.slot(),
+        incoming_slot = incoming.slot(),
         slot_mismatch = diagnostic.slot_mismatch,
         block_time_mismatch = diagnostic.block_time_mismatch,
         format_id_mismatch = diagnostic.format_id_mismatch,
@@ -1240,6 +1399,20 @@ fn log_raw_transaction_content_conflict(
         transaction_index_mismatch = diagnostic.payload_transaction_index_mismatch,
         payload_other_mismatch = diagnostic.payload_other_mismatch,
         meta_mismatch_fields = diagnostic.meta_mismatch_fields.as_str(),
+        log_messages_diagnostic_available = diagnostic.log_messages.available,
+        stored_log_messages_state = diagnostic.log_messages.stored_state,
+        incoming_log_messages_state = diagnostic.log_messages.incoming_state,
+        stored_log_messages_count = diagnostic.log_messages.stored_count,
+        incoming_log_messages_count = diagnostic.log_messages.incoming_count,
+        log_messages_first_mismatch_index = diagnostic.log_messages.first_mismatch_index.as_str(),
+        stored_log_messages_first_kind = diagnostic.log_messages.stored_first_kind,
+        incoming_log_messages_first_kind = diagnostic.log_messages.incoming_first_kind,
+        stored_log_messages_first_length = diagnostic.log_messages.stored_first_length,
+        incoming_log_messages_first_length = diagnostic.log_messages.incoming_first_length,
+        stored_log_messages_prefix_of_incoming = diagnostic.log_messages.stored_prefix_of_incoming,
+        incoming_log_messages_prefix_of_stored = diagnostic.log_messages.incoming_prefix_of_stored,
+        stored_log_messages_has_truncation_marker = diagnostic.log_messages.stored_has_truncation_marker,
+        incoming_log_messages_has_truncation_marker = diagnostic.log_messages.incoming_has_truncation_marker,
         "PostgreSQL Store rejected divergent canonical RAW transaction"
     );
     return;

crates/ksp-store-postgres-lib/tests/hardening_completeness.rs

@@ -1,5 +1,5 @@
 // file: crates/ksp-store-postgres-lib/tests/hardening_completeness.rs
-// version: 24
+// version: 25
 
 #![warn(missing_docs)]
 #![deny(unreachable_pub)]
@@ -606,3 +606,40 @@ fn v0_3_15_pre_012_fix_004_content_conflict_diagnostic_is_field_only_bounded_and
     assert!(source.contains("fields.push(\"other\")"));
     return;
 }
+
+#[test]
+fn v0_3_15_pre_014_content_conflict_diagnostic_exposes_only_block_slot_and_bounded_log_shape() {
+    let source = include_str!("../src/raw_transaction.rs");
+    for required in [
+        "stored_slot",
+        "incoming_slot",
+        "log_messages_first_mismatch_index",
+        "stored_log_messages_count",
+        "incoming_log_messages_count",
+        "stored_log_messages_first_kind",
+        "incoming_log_messages_first_kind",
+        "stored_log_messages_prefix_of_incoming",
+        "incoming_log_messages_prefix_of_stored",
+        "stored_log_messages_has_truncation_marker",
+        "incoming_log_messages_has_truncation_marker",
+        "raw_log_message_kind_and_length",
+        "raw_log_message_array_has_truncation_marker",
+    ] {
+        assert!(source.contains(required), "missing pre.014 bounded logMessages diagnostic field: {required}");
+    }
+    let start = source.find("fn log_raw_transaction_content_conflict(");
+    assert!(start.is_some(), "missing pre.014 conflict logger");
+    let Some(start) = start else {
+        return;
+    };
+    let end = source[start..].find("async fn log_raw_transaction_content_conflict_provenance(");
+    assert!(end.is_some(), "missing pre.014 conflict logger end marker");
+    let Some(end) = end else {
+        return;
+    };
+    let logger = &source[start..start + end];
+    for forbidden in ["signature =", "blockhash =", "previous_blockhash =", "log_message =", "log_line =", "payload ="] {
+        assert!(!logger.contains(forbidden), "pre.014 conflict logger exposes forbidden material: {forbidden}");
+    }
+    return;
+}

crates/ksp-store-postgres-lib/unit_tests/raw_transaction.rs

@@ -1,5 +1,5 @@
 // file: crates/ksp-store-postgres-lib/unit_tests/raw_transaction.rs
-// version: 7
+// version: 8
 
 fn network() -> ksp_store_api::RawNetworkId {
     return match ksp_store_api::RawNetworkId::new("devnet") {
@@ -429,6 +429,52 @@ fn v0_3_15_pre_012_fix_004_content_conflict_diagnostic_collapses_unknown_meta_ke
     return;
 }
 
+#[test]
+fn v0_3_15_pre_014_log_messages_conflict_diagnostic_distinguishes_prefix_truncation_and_line_kind_without_text() {
+    let stored = raw_transaction(
+        9,
+        br#"{"transaction":["AQID","base64"],"meta":{"logMessages":["Program Vote111 invoke [1]","Program log: stable","Program Vote111 success"]},"version":0}"#,
+        7,
+    );
+    let incoming = raw_transaction(
+        9,
+        br#"{"transaction":["AQID","base64"],"meta":{"logMessages":["Program Vote111 invoke [1]","Program log: stable","Log truncated"]},"version":0}"#,
+        8,
+    );
+    let diagnostic = super::raw_transaction_content_conflict_diagnostic(&stored, &incoming);
+    assert!(diagnostic.log_messages.available);
+    assert_eq!(diagnostic.log_messages.stored_state, "array");
+    assert_eq!(diagnostic.log_messages.incoming_state, "array");
+    assert_eq!(diagnostic.log_messages.stored_count, 3);
+    assert_eq!(diagnostic.log_messages.incoming_count, 3);
+    assert_eq!(diagnostic.log_messages.first_mismatch_index, "2");
+    assert_eq!(diagnostic.log_messages.stored_first_kind, "program_success");
+    assert_eq!(diagnostic.log_messages.incoming_first_kind, "log_truncated");
+    assert!(!diagnostic.log_messages.stored_has_truncation_marker);
+    assert!(diagnostic.log_messages.incoming_has_truncation_marker);
+    assert!(!diagnostic.log_messages.stored_prefix_of_incoming);
+    assert!(!diagnostic.log_messages.incoming_prefix_of_stored);
+    return;
+}
+
+#[test]
+fn v0_3_15_pre_014_log_messages_conflict_diagnostic_detects_strict_prefix_without_exposing_line_material() {
+    let stored = raw_transaction(
+        9,
+        br#"{"transaction":["AQID","base64"],"meta":{"logMessages":["Program Vote111 invoke [1]","Program Vote111 success"]},"version":0}"#,
+        7,
+    );
+    let incoming = raw_transaction(9, br#"{"transaction":["AQID","base64"],"meta":{"logMessages":["Program Vote111 invoke [1]"]},"version":0}"#, 8);
+    let diagnostic = super::raw_transaction_content_conflict_diagnostic(&stored, &incoming);
+    assert!(diagnostic.log_messages.available);
+    assert_eq!(diagnostic.log_messages.first_mismatch_index, "1");
+    assert_eq!(diagnostic.log_messages.stored_first_kind, "program_success");
+    assert_eq!(diagnostic.log_messages.incoming_first_kind, "missing");
+    assert!(!diagnostic.log_messages.stored_prefix_of_incoming);
+    assert!(diagnostic.log_messages.incoming_prefix_of_stored);
+    return;
+}
+
 #[test]
 fn pre_005_observation_idempotence_compares_reference_and_complete_provenance() {
     let incoming_transaction = raw_transaction(8, &[1, 2, 3], 4);

crates/ksp-worker-raw-transaction-ingest-lib/src/runtime_resources.rs

@@ -1,5 +1,5 @@
 // file: crates/ksp-worker-raw-transaction-ingest-lib/src/runtime_resources.rs
-// version: 48
+// version: 49
 
 use sha2::Digest; // rust-rules: trait-import
 
@@ -19,6 +19,7 @@ pub const MIN_RAW_TRANSACTION_INGEST_HTTP_POLL_INTERVAL: std::time::Duration = s
 pub const MIN_RAW_TRANSACTION_INGEST_HTTP_POLL_MAX_BLOCKS_PER_CYCLE: u16 = 1;
 
 const MAX_RAW_TRANSACTION_INGEST_REPAIR_BURST: usize = 1;
+const RAW_TRANSACTION_INGEST_BLOCK_IDENTITY_FINGERPRINT_DOMAIN: &[u8] = b"ksp.raw_transaction_ingest.block_identity.v1\0";
 const RAW_TRANSACTION_INGEST_HELIUS_TRANSACTION_FILTER_FINGERPRINT_DOMAIN: &[u8] = b"ksp.raw_transaction_ingest.helius_transaction.filter.v1\0";
 const RAW_TRANSACTION_INGEST_HELIUS_TRANSACTION_HTTP_PROTOCOL: &str = "helius_ws_http";
 const RAW_TRANSACTION_INGEST_HELIUS_TRANSACTION_HTTP_SOURCE_KEY_DOMAIN: &[u8] = b"ksp.raw_transaction_ingest.helius_transaction_http.source_key.v1\0";
@@ -3595,6 +3596,7 @@ async fn fetch_yellowstone_block_ingresses(
         std::option::Option::Some(value) => value,
         std::option::Option::None => return std::result::Result::Err(crate::runtime_error("source.yellowstone_block_not_available")),
     };
+    log_block_identity("yellowstone_block_hydration", &hydration.network, slot, hydration.commitment, observed.endpoint_name(), observed.provider(), block);
     let received_at = match current_raw_timestamp() {
         std::result::Result::Ok(value) => value,
         std::result::Result::Err(error) => return std::result::Result::Err(error),
@@ -3665,6 +3667,7 @@ fn project_http_block_polling_ingresses(
     observed: &ksp_onchain_transport_lib::HttpObservedValue<std::option::Option<ksp_onchain_transport_lib::SolanaConfirmedBlock>>,
     received_at: ksp_store_lib::RawTimestamp,
 ) -> ksp_core_lib::Result<std::vec::Vec<crate::RawTransactionIngress>> {
+    log_block_identity("http_block_polling", &source.network, slot, source.commitment, observed.endpoint_name(), observed.provider(), block);
     let transactions = match block.transactions() {
         ksp_onchain_transport_lib::SolanaWireField::Value(value) => value,
         ksp_onchain_transport_lib::SolanaWireField::Omitted | ksp_onchain_transport_lib::SolanaWireField::Null => {
@@ -5573,6 +5576,61 @@ fn hydration_method_code(family: RawTransactionIngestSourceFamily) -> &'static s
     };
 }
 
+fn block_identity_fingerprint(block: &ksp_onchain_transport_lib::SolanaConfirmedBlock) -> std::string::String {
+    const HEX: &[u8; 16] = b"0123456789abcdef";
+    let mut hasher = sha2::Sha256::new();
+    hasher.update(RAW_TRANSACTION_INGEST_BLOCK_IDENTITY_FINGERPRINT_DOMAIN);
+    hash_live_source_key_component(&mut hasher, block.blockhash().as_bytes());
+    hash_live_source_key_component(&mut hasher, block.previous_blockhash().as_bytes());
+    hasher.update(block.parent_slot().to_be_bytes());
+    match block.block_height() {
+        std::option::Option::Some(value) => {
+            hasher.update([1_u8]);
+            hasher.update(value.to_be_bytes());
+        },
+        std::option::Option::None => hasher.update([0_u8]),
+    }
+    let digest: [u8; 32] = hasher.finalize().into();
+    let mut value = std::string::String::with_capacity(71);
+    value.push_str("sha256.");
+    for byte in digest {
+        value.push(char::from(HEX[usize::from(byte >> 4)]));
+        value.push(char::from(HEX[usize::from(byte & 0x0f)]));
+    }
+    return value;
+}
+
+fn log_block_identity(
+    source_kind: &'static str,
+    network: &ksp_store_lib::RawNetworkId,
+    slot: u64,
+    commitment: ksp_onchain_transport_lib::SolanaCommitment,
+    endpoint_name: &str,
+    provider_name: &ksp_onchain_transport_lib::HttpProviderName,
+    block: &ksp_onchain_transport_lib::SolanaConfirmedBlock,
+) {
+    let fingerprint = block_identity_fingerprint(block);
+    let block_height = match block.block_height() {
+        std::option::Option::Some(value) => value.to_string(),
+        std::option::Option::None => "none".to_owned(),
+    };
+    ksp_logging_lib::debug!(
+        target: crate::TRACING_TARGET,
+        domain = "raw_transaction_ingest.block_identity",
+        source_kind = source_kind,
+        network = network.as_str(),
+        slot = slot,
+        provider = provider_name.as_str(),
+        endpoint_id = endpoint_name,
+        commitment = commitment.as_str(),
+        parent_slot = block.parent_slot(),
+        block_height = block_height.as_str(),
+        block_identity_fingerprint = fingerprint.as_str(),
+        "RAW transaction ingest observed HTTP block identity"
+    );
+    return;
+}
+
 fn fingerprint_filter_code(fingerprint: &[u8; 32]) -> ksp_core_lib::Result<ksp_store_lib::RawProvenanceCode> {
     const HEX: &[u8; 16] = b"0123456789abcdef";
     let mut value = std::string::String::with_capacity(71);

crates/ksp-worker-raw-transaction-ingest-lib/tests/hardening.rs

@@ -1,5 +1,5 @@
 // file: crates/ksp-worker-raw-transaction-ingest-lib/tests/hardening.rs
-// version: 39
+// version: 40
 
 //! External public, security, redaction and release-boundary hardening canaries through `v0.3.15-pre.004`.
 
@@ -1449,3 +1449,35 @@ fn v0_3_15_pre_013_yellowstone_block_hydration_is_bounded_concurrent_and_stop_pr
     }
     return;
 }
+
+#[test]
+fn v0_3_15_pre_014_cross_provider_block_identity_diagnostic_is_safe_and_covers_both_get_block_routes() {
+    let resources = include_str!("../src/runtime_resources.rs");
+    for required in [
+        "RAW_TRANSACTION_INGEST_BLOCK_IDENTITY_FINGERPRINT_DOMAIN",
+        "fn block_identity_fingerprint(",
+        "fn log_block_identity(",
+        "domain = \"raw_transaction_ingest.block_identity\"",
+        "\"yellowstone_block_hydration\"",
+        "\"http_block_polling\"",
+        "block_identity_fingerprint = fingerprint.as_str()",
+        "parent_slot = block.parent_slot()",
+    ] {
+        assert!(resources.contains(required), "missing pre.014 block identity diagnostic guard: {required}");
+    }
+    let start = resources.find("fn log_block_identity(");
+    assert!(start.is_some(), "missing pre.014 block identity logger");
+    let Some(start) = start else {
+        return;
+    };
+    let end = resources[start..].find("fn fingerprint_filter_code(");
+    assert!(end.is_some(), "missing pre.014 block identity logger end marker");
+    let Some(end) = end else {
+        return;
+    };
+    let logger = &resources[start..start + end];
+    for forbidden in ["blockhash =", "previous_blockhash =", "signature =", "payload =", "transaction ="] {
+        assert!(!logger.contains(forbidden), "pre.014 block identity logger exposes forbidden material: {forbidden}");
+    }
+    return;
+}

crates/ksp-worker-raw-transaction-ingest-lib/tests/release_completeness.rs

@@ -1,5 +1,5 @@
 // file: crates/ksp-worker-raw-transaction-ingest-lib/tests/release_completeness.rs
-// version: 35
+// version: 36
 
 //! Release-completeness canaries through the `v0.3.15-pre.004` WebSocket capability-enforcement tranche.
 
@@ -496,3 +496,13 @@ fn v0_3_15_pre_013_yellowstone_block_backpressure_fix_adds_no_public_surface() {
     assert!(!root.contains("YellowstoneBlockHydrationContext"));
     return;
 }
+
+#[test]
+fn v0_3_15_pre_014_cross_provider_convergence_diagnostic_adds_no_public_worker_surface() {
+    let hardening = include_str!("hardening.rs");
+    let root = include_str!("../src/lib.rs");
+    assert!(hardening.contains("v0_3_15_pre_014_cross_provider_block_identity_diagnostic_is_safe_and_covers_both_get_block_routes"));
+    assert!(!root.contains("BlockIdentityFingerprint"));
+    assert!(!root.contains("LogMessagesConflictDiagnostic"));
+    return;
+}

deltas/0.3.15/pre.014.md

@@ -0,0 +1,111 @@
+<!-- file: deltas/0.3.15/pre.014.md -->
+<!-- version: 1 -->
+
+# Delta v0.3.15-pre.014 — convergence inter-provider diagnostiquable
+
+## Base
+
+```text
+ksp-general-0.3.15-pre.013.zip
+SHA-256: 4d463230f45a55f68994443f138afd246b6ec34bea21357c2cedfe5d8fe1c5c6
+```
+
+Le gate opérateur de `pre.013` est propre et le live ne reproduit plus le `grpc_backpressure_overflow` fermé par cette tranche. Yellowstone devient néanmoins terminal `content_conflict` une fois pendant l'exécution parallèle avec HTTP Block Polling.
+
+Le diagnostic de `pre.012-fix.004` prouve que le conflit ne porte ni sur le slot, ni sur `block_time`, ni sur les octets/instructions de transaction, ni sur la version, ni sur `transactionIndex`. Seul `meta.logMessages` diffère. La version déjà canonique provient de `solana_mainnet_public`; la version entrante est hydratée par `publicnode_solana_mainnet_rpc` après trigger Yellowstone, toutes deux en `confirmed`.
+
+## Diagnostic ajouté
+
+Chaque `getBlock` réussi des voies Yellowstone Block Hydration et HTTP Block Polling émet désormais un diagnostic DEBUG sûr :
+
+```text
+domain=raw_transaction_ingest.block_identity
+source_kind
+network
+slot
+provider
+endpoint_id
+commitment
+parent_slot
+block_height
+block_identity_fingerprint
+```
+
+`block_identity_fingerprint` est un SHA-256 domain-separated calculé à partir de `blockhash`, `previousBlockhash`, `parentSlot` et `blockHeight`. Les blockhash bruts ne sont jamais journalisés.
+
+Lors d'un `content_conflict`, Store complète son masque existant avec :
+
+```text
+stored_slot
+incoming_slot
+log_messages_diagnostic_available
+stored_log_messages_state
+incoming_log_messages_state
+stored_log_messages_count
+incoming_log_messages_count
+log_messages_first_mismatch_index
+stored_log_messages_first_kind
+incoming_log_messages_first_kind
+stored_log_messages_first_length
+incoming_log_messages_first_length
+stored_log_messages_prefix_of_incoming
+incoming_log_messages_prefix_of_stored
+stored_log_messages_has_truncation_marker
+incoming_log_messages_has_truncation_marker
+```
+
+Les types de ligne sont réduits à une taxonomie statique ; aucun texte arbitraire de log n'est recopié.
+
+## Frontières
+
+Aucun changement de :
+
+```text
+RawTransaction canonical identity
+égalité / content_hash
+politique fail-closed content_conflict
+Store schema
+Transport gRPC / HTTP
+Config
+Desk
+Backfill
+public Worker API
+public Store API
+```
+
+`logMessages` reste partie intégrante du payload canonique tant que la cause de la divergence n'est pas prouvée.
+
+## Canaris
+
+```text
+v0_3_15_pre_014_log_messages_conflict_diagnostic_distinguishes_prefix_truncation_and_line_kind_without_text
+v0_3_15_pre_014_log_messages_conflict_diagnostic_detects_strict_prefix_without_exposing_line_material
+v0_3_15_pre_014_content_conflict_diagnostic_exposes_only_block_slot_and_bounded_log_shape
+v0_3_15_pre_014_cross_provider_block_identity_diagnostic_is_safe_and_covers_both_get_block_routes
+v0_3_15_pre_014_cross_provider_convergence_diagnostic_adds_no_public_worker_surface
+```
+
+## Version
+
+```text
+workspace.package.version : 0.3.15-pre.14
+root Cargo header counter  : 628
+```
+
+## Validation attendue
+
+```text
+cargo fmt --all
+cargo fmt --all -- --check
+python3 scripts/audit_rust_workspace_rules.py
+python3 scripts/audit_markdown_tables.py README.md RULES.md ROADMAP.md CHANGELOG.md docs prompts crates deltas/0.3.15
+cargo check --workspace
+cargo clippy --workspace --all-targets --all-features -- -D warnings
+cargo test -p ksp-store-postgres-lib --all-targets --all-features
+cargo test -p ksp-worker-raw-transaction-ingest-lib --all-targets --all-features
+cargo test -p ksp-app-raw-transaction-ingest-desk --all-targets --all-features
+cargo test --workspace --all-targets --all-features
+(cd crates/ksp-app-raw-transaction-ingest-desk && cargo tauri dev)
+```
+
+Live : Yellowstone + HTTP Block Polling en `confirmed` jusqu'à reproduction d'un conflit. Au slot annoncé par Store, comparer les deux `block_identity_fingerprint`, puis lire la forme bornée de `logMessages`. Si nécessaire, répéter ensuite la même expérience en `finalized` comme diagnostic de finalité, sans changer le défaut produit par défaut.

docs/plans/036-V0_3_15_RAW_TRANSACTION_INGEST_DESK_PLAN.md

@@ -1,5 +1,5 @@
 <!-- file: docs/plans/036-V0_3_15_RAW_TRANSACTION_INGEST_DESK_PLAN.md -->
-<!-- version: 21 -->
+<!-- version: 22 -->
 
 # Plan v0.3.15 — Raw Transaction Ingest Desk
 
@@ -855,11 +855,17 @@ Le mode Yellowstone Block reçoit désormais une partition déterministe des bud
 
 Budget cible : une tranche corrective acquisition/throughput, suivie d'un live Mainnet prolongé.
 
-### `pre.014` — completeness/security cross-layer
+### `pre.014` — convergence inter-provider + completeness/security cross-layer
 
-Fermer les canaris Config/Transport/Worker/Common RAW/Store/Desk, dependency firewalls, API publique, redaction et absence de seconde pipeline ou d'orchestration Backfill.
+**État : implémenté, gate/live opérateur requis.** Le gate de `pre.013` compile et teste proprement le workspace et le live ne reproduit plus `grpc_backpressure_overflow`. Un unique `content_conflict` Mainnet réapparaît toutefois entre HTTP Block Polling (`solana_mainnet_public`) et Yellowstone hydraté par HTTP (`publicnode_solana_mainnet_rpc`), tous deux en `confirmed`. Le diagnostic `pre.012-fix.004` prouve que `slot`, `block_time`, transaction wire, version et `transactionIndex` sont identiques ; seule la valeur `meta.logMessages` diverge.
 
-Budget cible : une tranche de clôture fonctionnelle.
+La tranche ne normalise ni n'ignore `logMessages`. Elle ajoute un diagnostic de convergence sûr permettant de distinguer un fork `confirmed` d'une divergence de représentation/provider : chaque `getBlock` réussi des deux routes journalise le slot et un fingerprint SHA-256 irréversible de l'identité de bloc (`blockhash`, `previousBlockhash`, `parentSlot`, `blockHeight`) sans exposer les blockhash. En cas de conflit RAW, Store journalise désormais les slots exacts et une forme bornée de la divergence `logMessages` : état absent/null/array, cardinalités, premier index divergent, type statique de la première ligne divergente, longueur, relation de préfixe et présence d'un marqueur de troncature. Aucun texte de log distant n'est copié.
+
+Le live doit comparer les fingerprints au slot du conflit. Fingerprints différents au même slot indiquent deux vues de bloc/fork différentes au commitment observé ; fingerprint identique avec `logMessages` divergents indique une divergence de réponse/recording RPC entre providers pour le même bloc. Ce diagnostic précède toute décision de canonicalisation durable.
+
+Les canaris de clôture continuent de vérifier dependency firewalls, API publique, redaction et absence de seconde pipeline ou d'orchestration Backfill.
+
+Budget cible : une tranche de convergence/completeness cross-layer, sans modification de la politique fail-closed.
 
 ### `pre.015` — gate technique/live final
 

docs/validation/032-V0_3_15_RAW_TRANSACTION_INGEST_DESK.md

@@ -1,5 +1,5 @@
 <!-- file: docs/validation/032-V0_3_15_RAW_TRANSACTION_INGEST_DESK.md -->
-<!-- version: 27 -->
+<!-- version: 28 -->
 
 # Validation v0.3.15 — Raw Transaction Ingest Desk
 
@@ -885,3 +885,78 @@ Aucun type du coordinateur n'est exporté publiquement.
 
 Validation live requise : Yellowstone Mainnet seul pendant une durée supérieure aux ~80 s de reproduction, puis Yellowstone + HTTP Block Polling en parallèle. Le run doit rester `Healthy`, sans `grpc_backpressure_overflow`, et les Stop ciblés doivent finir `Stopped`. Le diagnostic `pre.012-fix.004` reste actif si un `content_conflict` réapparaît.
 
+### `pre.014` — diagnostic de convergence inter-provider au niveau bloc / `logMessages`
+
+Le gate opérateur de `pre.013` est propre : audits Rust/Markdown, `cargo check --workspace`, Clippy strict, Worker (`161/161` unitaires + suites externes), Desk (`50/50` unitaires + suites externes) et workspace passent. Le live Mainnet confirme également l'objectif de `pre.013` : aucune occurrence de `grpc_backpressure_overflow` ni de `Yellowstone subscribe update queue overflowed` n'est observée dans le run transmis.
+
+Le live reproduit en revanche un unique terminal Yellowstone `content_conflict`. Le diagnostic Store de `pre.012-fix.004` donne exactement :
+
+```text
+slot_mismatch=false
+block_time_mismatch=false
+format_id_mismatch=false
+format_version_mismatch=false
+transaction_mismatch=false
+meta_mismatch=true
+version_mismatch=false
+transaction_index_mismatch=false
+payload_other_mismatch=false
+meta_mismatch_fields="logMessages"
+```
+
+La provenance durable classe les deux acquisitions :
+
+```text
+stored   : solana-public / solana_http / block_polling_get_block / solana_mainnet_public / confirmed
+incoming : ys.publicnode:http.publicnode / yellowstone_http / block_subscribe_get_block /
+           ys.publicnode_solana_mainnet_yellowstone:http.publicnode_solana_mainnet_rpc / confirmed
+```
+
+La transaction canonique elle-même est donc identique ; seule la métadonnée d'exécution `meta.logMessages` diffère. Le diagnostic existant ne permet toutefois pas encore de savoir si les deux RPC décrivent le même bloc `confirmed` ou deux forks/vues de bloc différents au même slot.
+
+`pre.014` ajoute deux niveaux de diagnostic sans changer la politique de conflit :
+
+```text
+Worker / chaque getBlock réussi
+    source_kind
+    network
+    slot
+    provider
+    endpoint_id
+    commitment
+    parent_slot
+    block_height
+    block_identity_fingerprint = SHA-256(domain || blockhash || previousBlockhash || parentSlot || blockHeight)
+
+Store / seulement lors d'un content_conflict
+    stored_slot / incoming_slot
+    logMessages state : missing | null | array | other
+    count de chaque côté
+    premier index divergent
+    type statique de la première ligne divergente :
+        program_invoke | program_success | program_failed | program_log |
+        program_data | compute_units | log_truncated | other | non_string | missing
+    longueur de la première ligne divergente
+    stored_prefix_of_incoming / incoming_prefix_of_stored
+    has_truncation_marker de chaque côté
+```
+
+Le blockhash, le previousBlockhash, les signatures, les payloads et le texte réel de `logMessages` ne sont jamais journalisés. Le fingerprint de bloc est irréversible et sert uniquement à comparer deux acquisitions du même slot.
+
+Interprétation du prochain conflit :
+
+```text
+même slot + fingerprints différents
+    => vues/forks confirmed différents entre providers au moment des getBlock
+
+même slot + même fingerprint + logMessages différents
+    => divergence provider/node sur la représentation ou l'enregistrement des logs du même bloc
+
+incoming_prefix_of_stored=true ou stored_prefix_of_incoming=true
+    => divergence compatible avec une liste tronquée/incomplète
+
+*_has_truncation_marker=true
+    => marqueur explicite de troncature présent dans une des réponses
+```
+
+Non-claims : `pre.014` ne retire pas `logMessages` du contenu canonique, ne choisit aucun provider comme vérité, ne transforme pas `content_conflict` en succès, ne modifie pas le schema Store et n'expose aucun nouveau type public Worker/Store.