diff --git a/Cargo.toml b/Cargo.toml index c94766b..a81d91d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,12 +1,12 @@ # file: Cargo.toml -# version: 642 +# version: 644 [workspace] resolver = "3" members = ["crates/ksp-app-backfill-desk", "crates/ksp-app-config-desk", "crates/ksp-app-raw-transaction-ingest-desk", "crates/ksp-app-solprices-desk", "crates/ksp-app-store-desk", "crates/ksp-app-wallet-desk", "crates/ksp-config-lib", "crates/ksp-core-lib", "crates/ksp-interface-lib", "crates/ksp-job-api", "crates/ksp-job-backfill-lib", "crates/ksp-logging-lib", "crates/ksp-offchain-transport-lib", "crates/ksp-onchain-transport-lib", "crates/ksp-program-api", "crates/ksp-raw-transaction-lib", "crates/ksp-store-api", "crates/ksp-store-lib", "crates/ksp-store-postgres-lib", "crates/ksp-wallet-lib", "crates/ksp-worker-api", "crates/ksp-worker-raw-transaction-ingest-lib"] [workspace.package] -version = "0.3.16-pre.4" +version = "0.3.16-pre.5.fix.1" edition = "2024" license = "MIT" repository = "https://git.sasedev.com/Sasedev/khadhroony-solana-project" diff --git a/crates/ksp-store-api/src/lib.rs b/crates/ksp-store-api/src/lib.rs index 5ba08b8..5b6e415 100644 --- a/crates/ksp-store-api/src/lib.rs +++ b/crates/ksp-store-api/src/lib.rs @@ -1,5 +1,5 @@ // file: crates/ksp-store-api/src/lib.rs -// version: 8 +// version: 9 #![warn(missing_docs)] #![deny(unreachable_pub)] #![forbid(unsafe_code)] @@ -170,6 +170,8 @@ pub use self::model::raw_transaction::RawTransactionVariantReference; pub use self::model::raw_transaction::RawTransactionVariantRelation; /// Stable backend-neutral reason explaining one RAW transaction variant relation. pub use self::model::raw_transaction::RawTransactionVariantRelationReason; +/// Compares one incoming RAW transaction variant against the currently canonical variant. +pub use self::model::raw_transaction::compare_raw_transaction_variants; /// Common KSP error type used by Store-facing contracts. pub use ksp_core_lib::Error; /// Stable structured code identifying a KSP error category and condition. diff --git a/crates/ksp-store-api/src/model/raw_transaction.rs b/crates/ksp-store-api/src/model/raw_transaction.rs index ddb28f7..e65f0c0 100644 --- a/crates/ksp-store-api/src/model/raw_transaction.rs +++ b/crates/ksp-store-api/src/model/raw_transaction.rs @@ -1,5 +1,5 @@ // file: crates/ksp-store-api/src/model/raw_transaction.rs -// version: 3 +// version: 5 /// Durable backend-independent identity of one canonical RAW transaction. #[derive(Clone, Debug, Eq, Hash, PartialEq)] @@ -203,6 +203,51 @@ impl RawTransactionVariantComparison { } } +/// Compares one incoming RAW transaction variant against the currently canonical variant. +/// +/// The comparator is pure and backend-neutral. Automatic dominance is intentionally limited to +/// a strictly proved `logMessages` truncation relation. Every other payload divergence fails closed +/// as either [`RawTransactionVariantRelation::Conflict`] or [`RawTransactionVariantRelation::Incomparable`]. +pub fn compare_raw_transaction_variants( + canonical: &crate::RawTransaction, + incoming: &crate::RawTransaction, +) -> crate::Result { + if canonical.reference() != incoming.reference() { + return std::result::Result::Err(raw_variant_model_error("comparison.reference")); + } + if canonical.slot() != incoming.slot() { + return crate::RawTransactionVariantComparison::try_new( + crate::RawTransactionVariantRelation::Conflict, + crate::RawTransactionVariantRelationReason::SlotMismatch, + ); + } + if canonical.block_time() != incoming.block_time() { + return crate::RawTransactionVariantComparison::try_new( + crate::RawTransactionVariantRelation::Conflict, + crate::RawTransactionVariantRelationReason::BlockTimeMismatch, + ); + } + if canonical.payload().format_id() != incoming.payload().format_id() || canonical.payload().format_version() != incoming.payload().format_version() { + return crate::RawTransactionVariantComparison::try_new( + crate::RawTransactionVariantRelation::Incomparable, + crate::RawTransactionVariantRelationReason::PayloadFormatMismatch, + ); + } + if canonical.payload().bytes() == incoming.payload().bytes() { + return crate::RawTransactionVariantComparison::try_new( + crate::RawTransactionVariantRelation::Exact, + crate::RawTransactionVariantRelationReason::ExactCanonicalContent, + ); + } + if canonical.payload().content_hash() == incoming.payload().content_hash() { + return crate::RawTransactionVariantComparison::try_new( + crate::RawTransactionVariantRelation::Conflict, + crate::RawTransactionVariantRelationReason::ContentHashCollision, + ); + } + return compare_raw_variant_payloads(canonical.payload().bytes(), incoming.payload().bytes()); +} + /// Durable lifecycle status of one RAW transaction conflict case. #[non_exhaustive] #[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)] @@ -304,6 +349,384 @@ impl RawTransactionObservation { } } +const MAX_RAW_VARIANT_JSON_DEPTH: usize = 128; +const RAW_LOG_MESSAGES_KEY: &[u8] = b"\"logMessages\""; +const RAW_META_KEY: &[u8] = b"\"meta\""; +const RAW_TRUNCATION_MARKER_JSON: &[u8] = b"\"Log truncated\""; + +#[derive(Clone, Copy)] +struct RawVariantJsonMember<'a> { + key: &'a [u8], + value: &'a [u8], +} + +fn compare_raw_variant_payloads(canonical: &[u8], incoming: &[u8]) -> crate::Result { + let canonical_members = raw_variant_json_object_members(canonical); + let incoming_members = raw_variant_json_object_members(incoming); + let (canonical_members, incoming_members) = match (canonical_members, incoming_members) { + (std::option::Option::Some(canonical_members), std::option::Option::Some(incoming_members)) => (canonical_members, incoming_members), + _ => return raw_variant_incomparable(), + }; + if canonical_members.len() != incoming_members.len() { + return raw_variant_incomparable(); + } + let mut canonical_meta = std::option::Option::None; + let mut incoming_meta = std::option::Option::None; + for (canonical_member, incoming_member) in canonical_members.iter().zip(incoming_members.iter()) { + if canonical_member.key != incoming_member.key { + return raw_variant_incomparable(); + } + if canonical_member.key == RAW_META_KEY { + canonical_meta = std::option::Option::Some(canonical_member.value); + incoming_meta = std::option::Option::Some(incoming_member.value); + continue; + } + if canonical_member.value != incoming_member.value { + return raw_variant_concrete_difference(canonical_member.value, incoming_member.value); + } + } + let (canonical_meta, incoming_meta) = match (canonical_meta, incoming_meta) { + (std::option::Option::Some(canonical_meta), std::option::Option::Some(incoming_meta)) => (canonical_meta, incoming_meta), + _ => return raw_variant_incomparable(), + }; + return compare_raw_variant_meta(canonical_meta, incoming_meta); +} + +fn compare_raw_variant_meta(canonical: &[u8], incoming: &[u8]) -> crate::Result { + if canonical == incoming { + return raw_variant_incomparable(); + } + if canonical == b"null" || incoming == b"null" { + return raw_variant_incomparable(); + } + let canonical_members = raw_variant_json_object_members(canonical); + let incoming_members = raw_variant_json_object_members(incoming); + let (canonical_members, incoming_members) = match (canonical_members, incoming_members) { + (std::option::Option::Some(canonical_members), std::option::Option::Some(incoming_members)) => (canonical_members, incoming_members), + _ => return raw_variant_concrete_difference(canonical, incoming), + }; + if canonical_members.len() != incoming_members.len() { + return raw_variant_incomparable(); + } + let mut canonical_logs = std::option::Option::None; + let mut incoming_logs = std::option::Option::None; + for (canonical_member, incoming_member) in canonical_members.iter().zip(incoming_members.iter()) { + if canonical_member.key != incoming_member.key { + return raw_variant_incomparable(); + } + if canonical_member.key == RAW_LOG_MESSAGES_KEY { + canonical_logs = std::option::Option::Some(canonical_member.value); + incoming_logs = std::option::Option::Some(incoming_member.value); + continue; + } + if canonical_member.value != incoming_member.value { + return raw_variant_concrete_difference(canonical_member.value, incoming_member.value); + } + } + let (canonical_logs, incoming_logs) = match (canonical_logs, incoming_logs) { + (std::option::Option::Some(canonical_logs), std::option::Option::Some(incoming_logs)) => (canonical_logs, incoming_logs), + _ => return raw_variant_incomparable(), + }; + if canonical_logs == incoming_logs { + return raw_variant_incomparable(); + } + let canonical_items = raw_variant_json_array_items(canonical_logs); + let incoming_items = raw_variant_json_array_items(incoming_logs); + let (canonical_items, incoming_items) = match (canonical_items, incoming_items) { + (std::option::Option::Some(canonical_items), std::option::Option::Some(incoming_items)) => (canonical_items, incoming_items), + _ => return raw_variant_concrete_difference(canonical_logs, incoming_logs), + }; + if raw_variant_log_messages_truncated(incoming_items.as_slice(), canonical_items.as_slice()) { + return crate::RawTransactionVariantComparison::try_new( + crate::RawTransactionVariantRelation::CompatibleLessComplete, + crate::RawTransactionVariantRelationReason::IncomingLogMessagesTruncated, + ); + } + if raw_variant_log_messages_truncated(canonical_items.as_slice(), incoming_items.as_slice()) { + return crate::RawTransactionVariantComparison::try_new( + crate::RawTransactionVariantRelation::CompatibleMoreComplete, + crate::RawTransactionVariantRelationReason::CanonicalLogMessagesTruncated, + ); + } + return raw_variant_concrete_difference(canonical_logs, incoming_logs); +} + +fn raw_variant_concrete_difference(canonical: &[u8], incoming: &[u8]) -> crate::Result { + if canonical == b"null" || incoming == b"null" { + return raw_variant_incomparable(); + } + return crate::RawTransactionVariantComparison::try_new( + crate::RawTransactionVariantRelation::Conflict, + crate::RawTransactionVariantRelationReason::CanonicalPayloadConflict, + ); +} + +fn raw_variant_incomparable() -> crate::Result { + return crate::RawTransactionVariantComparison::try_new( + crate::RawTransactionVariantRelation::Incomparable, + crate::RawTransactionVariantRelationReason::UnsupportedCanonicalDifference, + ); +} + +fn raw_variant_log_messages_truncated(truncated: &[&[u8]], full: &[&[u8]]) -> bool { + if truncated.is_empty() || full.is_empty() { + return false; + } + if !truncated.iter().all(|value| return raw_variant_json_is_string(value)) || !full.iter().all(|value| return raw_variant_json_is_string(value)) { + return false; + } + let marker_count = truncated.iter().filter(|value| return **value == RAW_TRUNCATION_MARKER_JSON).count(); + if marker_count != 1 || truncated.last().copied() != std::option::Option::Some(RAW_TRUNCATION_MARKER_JSON) { + return false; + } + if full.iter().any(|value| return *value == RAW_TRUNCATION_MARKER_JSON) { + return false; + } + let prefix_len = truncated.len() - 1; + if prefix_len >= full.len() { + return false; + } + return truncated[..prefix_len] == full[..prefix_len]; +} + +fn raw_variant_json_object_members(value: &[u8]) -> std::option::Option>> { + if std::str::from_utf8(value).is_err() { + return std::option::Option::None; + } + let start = raw_variant_json_skip_whitespace(value, 0); + if value.get(start) != std::option::Option::Some(&b'{') { + return std::option::Option::None; + } + let end = match raw_variant_json_value_end(value, start, 0) { + std::option::Option::Some(value) => value, + std::option::Option::None => return std::option::Option::None, + }; + if raw_variant_json_skip_whitespace(value, end) != value.len() { + return std::option::Option::None; + } + let mut index = raw_variant_json_skip_whitespace(value, start + 1); + let mut members = std::vec::Vec::new(); + if value.get(index) == std::option::Option::Some(&b'}') { + return std::option::Option::Some(members); + } + loop { + let key_start = index; + let key_end = match raw_variant_json_string_end(value, key_start) { + std::option::Option::Some(value) => value, + std::option::Option::None => return std::option::Option::None, + }; + index = raw_variant_json_skip_whitespace(value, key_end); + if value.get(index) != std::option::Option::Some(&b':') { + return std::option::Option::None; + } + index = raw_variant_json_skip_whitespace(value, index + 1); + let value_start = index; + let value_end = match raw_variant_json_value_end(value, value_start, 1) { + std::option::Option::Some(value) => value, + std::option::Option::None => return std::option::Option::None, + }; + members.push(RawVariantJsonMember { key: &value[key_start..key_end], value: &value[value_start..value_end] }); + index = raw_variant_json_skip_whitespace(value, value_end); + match value.get(index).copied() { + std::option::Option::Some(b',') => index = raw_variant_json_skip_whitespace(value, index + 1), + std::option::Option::Some(b'}') => return std::option::Option::Some(members), + _ => return std::option::Option::None, + } + } +} + +fn raw_variant_json_array_items(value: &[u8]) -> std::option::Option> { + let start = raw_variant_json_skip_whitespace(value, 0); + if value.get(start) != std::option::Option::Some(&b'[') { + return std::option::Option::None; + } + let end = match raw_variant_json_value_end(value, start, 0) { + std::option::Option::Some(value) => value, + std::option::Option::None => return std::option::Option::None, + }; + if raw_variant_json_skip_whitespace(value, end) != value.len() { + return std::option::Option::None; + } + let mut index = raw_variant_json_skip_whitespace(value, start + 1); + let mut items = std::vec::Vec::new(); + if value.get(index) == std::option::Option::Some(&b']') { + return std::option::Option::Some(items); + } + loop { + let item_start = index; + let item_end = match raw_variant_json_value_end(value, item_start, 1) { + std::option::Option::Some(value) => value, + std::option::Option::None => return std::option::Option::None, + }; + items.push(&value[item_start..item_end]); + index = raw_variant_json_skip_whitespace(value, item_end); + match value.get(index).copied() { + std::option::Option::Some(b',') => index = raw_variant_json_skip_whitespace(value, index + 1), + std::option::Option::Some(b']') => return std::option::Option::Some(items), + _ => return std::option::Option::None, + } + } +} + +fn raw_variant_json_value_end(value: &[u8], start: usize, depth: usize) -> std::option::Option { + if depth > MAX_RAW_VARIANT_JSON_DEPTH { + return std::option::Option::None; + } + let start = raw_variant_json_skip_whitespace(value, start); + return match value.get(start).copied() { + std::option::Option::Some(b'"') => raw_variant_json_string_end(value, start), + std::option::Option::Some(b'{') => raw_variant_json_object_end(value, start, depth + 1), + std::option::Option::Some(b'[') => raw_variant_json_array_end(value, start, depth + 1), + std::option::Option::Some(b't') => raw_variant_json_literal_end(value, start, b"true"), + std::option::Option::Some(b'f') => raw_variant_json_literal_end(value, start, b"false"), + std::option::Option::Some(b'n') => raw_variant_json_literal_end(value, start, b"null"), + std::option::Option::Some(b'-' | b'0'..=b'9') => raw_variant_json_number_end(value, start), + _ => std::option::Option::None, + }; +} + +fn raw_variant_json_object_end(value: &[u8], start: usize, depth: usize) -> std::option::Option { + let mut index = raw_variant_json_skip_whitespace(value, start + 1); + if value.get(index) == std::option::Option::Some(&b'}') { + return std::option::Option::Some(index + 1); + } + loop { + index = match raw_variant_json_string_end(value, index) { + std::option::Option::Some(value) => value, + std::option::Option::None => return std::option::Option::None, + }; + index = raw_variant_json_skip_whitespace(value, index); + if value.get(index) != std::option::Option::Some(&b':') { + return std::option::Option::None; + } + index = match raw_variant_json_value_end(value, index + 1, depth) { + std::option::Option::Some(value) => value, + std::option::Option::None => return std::option::Option::None, + }; + index = raw_variant_json_skip_whitespace(value, index); + match value.get(index).copied() { + std::option::Option::Some(b',') => index = raw_variant_json_skip_whitespace(value, index + 1), + std::option::Option::Some(b'}') => return std::option::Option::Some(index + 1), + _ => return std::option::Option::None, + } + } +} + +fn raw_variant_json_array_end(value: &[u8], start: usize, depth: usize) -> std::option::Option { + let mut index = raw_variant_json_skip_whitespace(value, start + 1); + if value.get(index) == std::option::Option::Some(&b']') { + return std::option::Option::Some(index + 1); + } + loop { + index = match raw_variant_json_value_end(value, index, depth) { + std::option::Option::Some(value) => value, + std::option::Option::None => return std::option::Option::None, + }; + index = raw_variant_json_skip_whitespace(value, index); + match value.get(index).copied() { + std::option::Option::Some(b',') => index = raw_variant_json_skip_whitespace(value, index + 1), + std::option::Option::Some(b']') => return std::option::Option::Some(index + 1), + _ => return std::option::Option::None, + } + } +} + +fn raw_variant_json_string_end(value: &[u8], start: usize) -> std::option::Option { + if value.get(start) != std::option::Option::Some(&b'"') { + return std::option::Option::None; + } + let mut index = start + 1; + while index < value.len() { + match value[index] { + b'"' => return std::option::Option::Some(index + 1), + b'\\' => { + index += 1; + let escape = match value.get(index).copied() { + std::option::Option::Some(value) => value, + std::option::Option::None => return std::option::Option::None, + }; + if escape == b'u' { + if index + 4 >= value.len() || !value[index + 1..=index + 4].iter().all(|byte| return byte.is_ascii_hexdigit()) { + return std::option::Option::None; + } + index += 5; + continue; + } + if !matches!(escape, b'"' | b'\\' | b'/' | b'b' | b'f' | b'n' | b'r' | b't') { + return std::option::Option::None; + } + index += 1; + }, + 0x00..=0x1f => return std::option::Option::None, + _ => index += 1, + } + } + return std::option::Option::None; +} + +fn raw_variant_json_literal_end(value: &[u8], start: usize, literal: &[u8]) -> std::option::Option { + let end = match start.checked_add(literal.len()) { + std::option::Option::Some(value) => value, + std::option::Option::None => return std::option::Option::None, + }; + if value.get(start..end) == std::option::Option::Some(literal) { + return std::option::Option::Some(end); + } + return std::option::Option::None; +} + +fn raw_variant_json_number_end(value: &[u8], start: usize) -> std::option::Option { + let mut index = start; + if value.get(index) == std::option::Option::Some(&b'-') { + index += 1; + } + match value.get(index).copied() { + std::option::Option::Some(b'0') => index += 1, + std::option::Option::Some(b'1'..=b'9') => { + index += 1; + while matches!(value.get(index).copied(), std::option::Option::Some(b'0'..=b'9')) { + index += 1; + } + }, + _ => return std::option::Option::None, + } + if value.get(index) == std::option::Option::Some(&b'.') { + index += 1; + let fraction_start = index; + while matches!(value.get(index).copied(), std::option::Option::Some(b'0'..=b'9')) { + index += 1; + } + if index == fraction_start { + return std::option::Option::None; + } + } + if matches!(value.get(index).copied(), std::option::Option::Some(b'e' | b'E')) { + index += 1; + if matches!(value.get(index).copied(), std::option::Option::Some(b'+' | b'-')) { + index += 1; + } + let exponent_start = index; + while matches!(value.get(index).copied(), std::option::Option::Some(b'0'..=b'9')) { + index += 1; + } + if index == exponent_start { + return std::option::Option::None; + } + } + return std::option::Option::Some(index); +} + +fn raw_variant_json_skip_whitespace(value: &[u8], mut index: usize) -> usize { + while matches!(value.get(index).copied(), std::option::Option::Some(b' ' | b'\n' | b'\r' | b'\t')) { + index += 1; + } + return index; +} + +fn raw_variant_json_is_string(value: &[u8]) -> bool { + return raw_variant_json_string_end(value, 0) == std::option::Option::Some(value.len()); +} + fn raw_variant_model_error(field: &'static str) -> crate::Error { return crate::Error::new(crate::ERROR_CODE_RAW_MODEL_INVALID, "invalid RAW transaction variant model").with_context("field", field); } diff --git a/crates/ksp-store-api/tests/dependency_boundary.rs b/crates/ksp-store-api/tests/dependency_boundary.rs index a2a8cc9..3d57384 100644 --- a/crates/ksp-store-api/tests/dependency_boundary.rs +++ b/crates/ksp-store-api/tests/dependency_boundary.rs @@ -1,5 +1,5 @@ // file: crates/ksp-store-api/tests/dependency_boundary.rs -// version: 7 +// version: 8 //! Dependency canaries for the Store API RAW foundation. @@ -97,6 +97,7 @@ fn pre_006_source_boundary_keeps_models_and_capabilities_backend_free() { "ksp_onchain_transport_lib", "ksp_program_api", "serde::", + "serde_json::", "sqlx::", "tokio::", "tokio_postgres::", diff --git a/crates/ksp-store-api/tests/public_api.rs b/crates/ksp-store-api/tests/public_api.rs index 1859418..0a7b9c4 100644 --- a/crates/ksp-store-api/tests/public_api.rs +++ b/crates/ksp-store-api/tests/public_api.rs @@ -1,5 +1,5 @@ // file: crates/ksp-store-api/tests/public_api.rs -// version: 9 +// version: 10 //! Integration canaries for the public `ksp-store-api` surface. @@ -208,3 +208,10 @@ fn public_v0_3_8_pre_003_inspection_contracts_are_backend_neutral_and_dyn_compat assert!(account_observation_inspection.is_none()); return; } + +#[test] +fn public_v0_3_16_pre_005_variant_comparator_is_available_from_crate_root() { + let comparator = ksp_store_api::compare_raw_transaction_variants; + let _ = comparator; + return; +} diff --git a/crates/ksp-store-api/tests/release_completeness.rs b/crates/ksp-store-api/tests/release_completeness.rs index ef7634f..de8370c 100644 --- a/crates/ksp-store-api/tests/release_completeness.rs +++ b/crates/ksp-store-api/tests/release_completeness.rs @@ -1,5 +1,5 @@ // file: crates/ksp-store-api/tests/release_completeness.rs -// version: 5 +// version: 6 //! Release-level boundary and completeness canaries for the backend-neutral Store API RAW surface. #[test] @@ -87,6 +87,7 @@ fn v0_3_8_pre_003_exact_crate_root_export_inventory_is_stable() { "pub use self::model::raw_retention::RawTransactionRetentionTransition;", "pub use self::model::raw_retention::RawTransactionTombstone;", "pub use self::model::raw_transaction::RawTransaction;", + "pub use self::model::raw_transaction::compare_raw_transaction_variants;", "pub use self::model::raw_transaction::RawTransactionConflictStatus;", "pub use self::model::raw_transaction::RawTransactionObservation;", "pub use self::model::raw_transaction::RawTransactionReference;", diff --git a/crates/ksp-store-api/unit_tests/model/raw_transaction.rs b/crates/ksp-store-api/unit_tests/model/raw_transaction.rs index 5f0fde9..69a3c02 100644 --- a/crates/ksp-store-api/unit_tests/model/raw_transaction.rs +++ b/crates/ksp-store-api/unit_tests/model/raw_transaction.rs @@ -1,5 +1,5 @@ // file: crates/ksp-store-api/unit_tests/model/raw_transaction.rs -// version: 3 +// version: 5 fn network() -> std::option::Option { return match crate::RawNetworkId::new("mainnet".to_owned()) { @@ -140,3 +140,121 @@ fn transaction_variant_codes_and_outcomes_are_stable_backend_neutral_contracts() assert_ne!(crate::RawTransactionVariantWriteOutcome::QuarantinedConflict, crate::RawTransactionVariantWriteOutcome::PromotedCompatibleMoreComplete); return; } + +fn comparison_transaction(payload_bytes: &[u8], hash_byte: u8) -> std::option::Option { + let network = match network() { + std::option::Option::Some(value) => value, + std::option::Option::None => return std::option::Option::None, + }; + let format = match crate::RawFormatId::new("ksp.solana.raw_transaction".to_owned()) { + std::result::Result::Ok(value) => value, + std::result::Result::Err(_) => return std::option::Option::None, + }; + let payload = match crate::RawPayload::try_new(format, 1, payload_bytes.to_vec().into_boxed_slice(), crate::RawContentHash::new([hash_byte; 32])) { + std::result::Result::Ok(value) => value, + std::result::Result::Err(_) => return std::option::Option::None, + }; + let reference = crate::RawTransactionReference::new(network, crate::RawTransactionSignature::new([21_u8; 64])); + return std::option::Option::Some(crate::RawTransaction::new(reference, 88, std::option::Option::None, payload)); +} + +fn comparison_relation( + canonical_payload: &[u8], + canonical_hash: u8, + incoming_payload: &[u8], + incoming_hash: u8, +) -> std::option::Option { + let canonical = match comparison_transaction(canonical_payload, canonical_hash) { + std::option::Option::Some(value) => value, + std::option::Option::None => return std::option::Option::None, + }; + let incoming = match comparison_transaction(incoming_payload, incoming_hash) { + std::option::Option::Some(value) => value, + std::option::Option::None => return std::option::Option::None, + }; + return crate::compare_raw_transaction_variants(&canonical, &incoming).ok(); +} + +#[test] +fn v0_3_16_pre_005_comparator_proves_bidirectional_log_messages_truncation_only() { + let full = br#"{"transaction":["AQID","base64"],"meta":{"err":null,"fee":5000,"logMessages":["Program Vote111 invoke [1]","Program log: stable","Program Vote111 success"],"rewards":[]},"version":0,"transactionIndex":7}"#; + let truncated = br#"{"transaction":["AQID","base64"],"meta":{"err":null,"fee":5000,"logMessages":["Program Vote111 invoke [1]","Program log: stable","Log truncated"],"rewards":[]},"version":0,"transactionIndex":7}"#; + let less = match comparison_relation(full, 31, truncated, 32) { + std::option::Option::Some(value) => value, + std::option::Option::None => return, + }; + assert_eq!(less.relation(), crate::RawTransactionVariantRelation::CompatibleLessComplete); + assert_eq!(less.reason(), crate::RawTransactionVariantRelationReason::IncomingLogMessagesTruncated); + let more = match comparison_relation(truncated, 32, full, 31) { + std::option::Option::Some(value) => value, + std::option::Option::None => return, + }; + assert_eq!(more.relation(), crate::RawTransactionVariantRelation::CompatibleMoreComplete); + assert_eq!(more.reason(), crate::RawTransactionVariantRelationReason::CanonicalLogMessagesTruncated); + return; +} + +#[test] +fn v0_3_16_pre_005_comparator_rejects_unproved_log_shortening_and_marker_shapes() { + let full = br#"{"transaction":["AQID","base64"],"meta":{"err":null,"fee":5000,"logMessages":["a","b","c"],"rewards":[]},"version":0}"#; + let shorter = br#"{"transaction":["AQID","base64"],"meta":{"err":null,"fee":5000,"logMessages":["a","b"],"rewards":[]},"version":0}"#; + let marker_not_last = + br#"{"transaction":["AQID","base64"],"meta":{"err":null,"fee":5000,"logMessages":["a","Log truncated","c"],"rewards":[]},"version":0}"#; + let marker_not_exact = + br#"{"transaction":["AQID","base64"],"meta":{"err":null,"fee":5000,"logMessages":["a","b","Log truncated by provider"],"rewards":[]},"version":0}"#; + let bad_prefix = + br#"{"transaction":["AQID","base64"],"meta":{"err":null,"fee":5000,"logMessages":["a","changed","Log truncated"],"rewards":[]},"version":0}"#; + let candidates: [&[u8]; 4] = [shorter, marker_not_last, marker_not_exact, bad_prefix]; + for (index, candidate) in candidates.iter().enumerate() { + let hash_offset = match u8::try_from(index) { + std::result::Result::Ok(value) => value, + std::result::Result::Err(_) => return, + }; + let comparison = match comparison_relation(full, 40, candidate, 41 + hash_offset) { + std::option::Option::Some(value) => value, + std::option::Option::None => return, + }; + assert_eq!(comparison.relation(), crate::RawTransactionVariantRelation::Conflict); + assert_eq!(comparison.reason(), crate::RawTransactionVariantRelationReason::CanonicalPayloadConflict); + } + return; +} + +#[test] +fn v0_3_16_pre_005_comparator_is_fail_closed_for_optional_and_other_payload_differences() { + let canonical = br#"{"transaction":["AQID","base64"],"meta":{"fee":5000,"logMessages":["a"],"returnData":null},"version":0}"#; + let optional_value = + br#"{"transaction":["AQID","base64"],"meta":{"fee":5000,"logMessages":["a"],"returnData":{"data":["","base64"],"programId":"Vote111"}},"version":0}"#; + let concrete_conflict = br#"{"transaction":["AQID","base64"],"meta":{"fee":5001,"logMessages":["a"],"returnData":null},"version":0}"#; + let optional = match comparison_relation(canonical, 50, optional_value, 51) { + std::option::Option::Some(value) => value, + std::option::Option::None => return, + }; + assert_eq!(optional.relation(), crate::RawTransactionVariantRelation::Incomparable); + assert_eq!(optional.reason(), crate::RawTransactionVariantRelationReason::UnsupportedCanonicalDifference); + let conflict = match comparison_relation(canonical, 50, concrete_conflict, 52) { + std::option::Option::Some(value) => value, + std::option::Option::None => return, + }; + assert_eq!(conflict.relation(), crate::RawTransactionVariantRelation::Conflict); + assert_eq!(conflict.reason(), crate::RawTransactionVariantRelationReason::CanonicalPayloadConflict); + return; +} + +#[test] +fn v0_3_16_pre_005_comparator_uses_exact_bytes_and_detects_hash_collision_without_hash_equality_shortcut() { + let canonical = br#"{"transaction":["AQID","base64"],"meta":{"logMessages":["a"]}}"#; + let same_bytes = match comparison_relation(canonical, 60, canonical, 61) { + std::option::Option::Some(value) => value, + std::option::Option::None => return, + }; + assert_eq!(same_bytes.relation(), crate::RawTransactionVariantRelation::Exact); + let different = br#"{"transaction":["AQID","base64"],"meta":{"logMessages":["b"]}}"#; + let collision = match comparison_relation(canonical, 62, different, 62) { + std::option::Option::Some(value) => value, + std::option::Option::None => return, + }; + assert_eq!(collision.relation(), crate::RawTransactionVariantRelation::Conflict); + assert_eq!(collision.reason(), crate::RawTransactionVariantRelationReason::ContentHashCollision); + return; +} diff --git a/crates/ksp-store-postgres-lib/src/raw_transaction.rs b/crates/ksp-store-postgres-lib/src/raw_transaction.rs index 6e2741a..eb8f7c8 100644 --- a/crates/ksp-store-postgres-lib/src/raw_transaction.rs +++ b/crates/ksp-store-postgres-lib/src/raw_transaction.rs @@ -1,5 +1,5 @@ // file: crates/ksp-store-postgres-lib/src/raw_transaction.rs -// version: 11 +// version: 12 pub(crate) mod cursor; @@ -712,7 +712,7 @@ pub(crate) async fn persist_raw_transaction_acquisition( }; match comparison { ExistingTransactionMatch::Active => (ksp_store_api::RawEntityWriteOutcome::AlreadyPresent, canonical_variant_id), - ExistingTransactionMatch::ActiveIncomingTruncatedLogs => { + ExistingTransactionMatch::ActiveCompatibleLessComplete => { let variant_result = persist_or_reuse_native_transaction_variant(&sql_transaction, &raw_transaction, linked_at).await; let variant_id = match variant_result { std::result::Result::Ok(value) => value, @@ -896,7 +896,7 @@ pub(crate) async fn transition_raw_transaction_retention( #[derive(Clone, Copy, Debug, Eq, PartialEq)] enum ExistingTransactionMatch { Active, - ActiveIncomingTruncatedLogs, + ActiveCompatibleLessComplete, Purged, } @@ -1439,87 +1439,24 @@ fn compare_existing_transaction( std::result::Result::Ok(std::option::Option::None) => return std::result::Result::Err(data_invalid("raw_acquisition_active_shape")), std::result::Result::Err(error) => return std::result::Result::Err(error), }; - if raw_transactions_equal(&stored, incoming) { - return std::result::Result::Ok(ExistingTransactionMatch::Active); - } - if raw_transaction_incoming_truncated_log_messages_compatible(&stored, incoming) { - log_raw_transaction_compatible_truncated_log_messages(network, &stored, incoming); - return std::result::Result::Ok(ExistingTransactionMatch::ActiveIncomingTruncatedLogs); - } - log_raw_transaction_content_conflict(network, &stored, incoming); - return std::result::Result::Err(conflict("raw_acquisition_content_conflict")); -} - -fn raw_transaction_incoming_truncated_log_messages_compatible(stored: &ksp_store_api::RawTransaction, incoming: &ksp_store_api::RawTransaction) -> bool { - if stored.reference() != incoming.reference() - || stored.slot() != incoming.slot() - || stored.block_time() != incoming.block_time() - || stored.payload().format_id() != incoming.payload().format_id() - || stored.payload().format_version() != incoming.payload().format_version() - { - return false; - } - let stored_payload = serde_json::from_slice::(stored.payload().bytes()); - let incoming_payload = serde_json::from_slice::(incoming.payload().bytes()); - let (stored_payload, incoming_payload) = match (stored_payload, incoming_payload) { - (std::result::Result::Ok(serde_json::Value::Object(stored)), std::result::Result::Ok(serde_json::Value::Object(incoming))) => (stored, incoming), - _ => return false, + let comparison = match ksp_store_api::compare_raw_transaction_variants(&stored, incoming) { + std::result::Result::Ok(value) => value, + std::result::Result::Err(_) => return std::result::Result::Err(data_invalid("raw_acquisition_variant_comparison")), }; - if stored_payload.get("transaction") != incoming_payload.get("transaction") - || stored_payload.get("version") != incoming_payload.get("version") - || stored_payload.get("transactionIndex") != incoming_payload.get("transactionIndex") - || json_object_other_fields_mismatch(&stored_payload, &incoming_payload, RAW_TRANSACTION_CONTENT_CONFLICT_PAYLOAD_FIELDS.as_slice()) - { - return false; - } - let (stored_meta, incoming_meta) = match (stored_payload.get("meta"), incoming_payload.get("meta")) { - (std::option::Option::Some(serde_json::Value::Object(stored)), std::option::Option::Some(serde_json::Value::Object(incoming))) => (stored, incoming), - _ => return false, - }; - if raw_meta_other_than_log_messages_mismatch(stored_meta, incoming_meta) { - return false; - } - let (stored_logs, incoming_logs) = match (stored_meta.get("logMessages"), incoming_meta.get("logMessages")) { - (std::option::Option::Some(serde_json::Value::Array(stored)), std::option::Option::Some(serde_json::Value::Array(incoming))) => { - (stored.as_slice(), incoming.as_slice()) + return match comparison.relation() { + ksp_store_api::RawTransactionVariantRelation::Exact => std::result::Result::Ok(ExistingTransactionMatch::Active), + ksp_store_api::RawTransactionVariantRelation::CompatibleLessComplete => { + log_raw_transaction_compatible_truncated_log_messages(network, &stored, incoming); + std::result::Result::Ok(ExistingTransactionMatch::ActiveCompatibleLessComplete) }, - _ => return false, + ksp_store_api::RawTransactionVariantRelation::CompatibleMoreComplete + | ksp_store_api::RawTransactionVariantRelation::Conflict + | ksp_store_api::RawTransactionVariantRelation::Incomparable => { + log_raw_transaction_content_conflict(network, &stored, incoming); + std::result::Result::Err(conflict("raw_acquisition_content_conflict")) + }, + _ => std::result::Result::Err(data_invalid("raw_acquisition_variant_relation")), }; - return raw_log_messages_incoming_truncated_compatible(stored_logs, incoming_logs); -} - -fn raw_meta_other_than_log_messages_mismatch( - stored: &serde_json::Map, - incoming: &serde_json::Map, -) -> bool { - let stored_without_logs = stored.iter().filter(|(key, _)| return key.as_str() != "logMessages").collect::>(); - let incoming_without_logs = incoming.iter().filter(|(key, _)| return key.as_str() != "logMessages").collect::>(); - return stored_without_logs != incoming_without_logs; -} - -fn raw_log_messages_incoming_truncated_compatible(stored: &[serde_json::Value], incoming: &[serde_json::Value]) -> bool { - if raw_log_messages_exact_truncation_marker_count(stored) != 0 { - return false; - } - let marker_index = match incoming.iter().position(raw_log_message_is_exact_truncation_marker) { - std::option::Option::Some(value) => value, - std::option::Option::None => return false, - }; - if raw_log_messages_exact_truncation_marker_count(incoming) != 1 || marker_index >= stored.len() { - return false; - } - if incoming[..marker_index] != stored[..marker_index] { - return false; - } - return true; -} - -fn raw_log_messages_exact_truncation_marker_count(values: &[serde_json::Value]) -> usize { - return values.iter().filter(|value| return raw_log_message_is_exact_truncation_marker(value)).count(); -} - -fn raw_log_message_is_exact_truncation_marker(value: &serde_json::Value) -> bool { - return matches!(value, serde_json::Value::String(line) if line == "Log truncated"); } fn raw_transaction_content_conflict_diagnostic( @@ -1863,16 +1800,6 @@ async fn log_raw_transaction_content_conflict_provenance( return; } -fn raw_transactions_equal(left: &ksp_store_api::RawTransaction, right: &ksp_store_api::RawTransaction) -> bool { - return left.reference() == right.reference() - && left.slot() == right.slot() - && left.block_time() == right.block_time() - && left.payload().format_id() == right.payload().format_id() - && left.payload().format_version() == right.payload().format_version() - && left.payload().content_hash() == right.payload().content_hash() - && left.payload().bytes() == right.payload().bytes(); -} - async fn rehydrate_transaction( sql_transaction: &deadpool_postgres::Transaction<'_>, raw_transaction: &ksp_store_api::RawTransaction, diff --git a/crates/ksp-store-postgres-lib/tests/hardening_completeness.rs b/crates/ksp-store-postgres-lib/tests/hardening_completeness.rs index e194e4e..9b1c09f 100644 --- a/crates/ksp-store-postgres-lib/tests/hardening_completeness.rs +++ b/crates/ksp-store-postgres-lib/tests/hardening_completeness.rs @@ -1,5 +1,5 @@ // file: crates/ksp-store-postgres-lib/tests/hardening_completeness.rs -// version: 27 +// version: 28 #![warn(missing_docs)] #![deny(unreachable_pub)] @@ -594,7 +594,7 @@ fn v0_3_15_pre_012_fix_004_content_conflict_diagnostic_is_field_only_bounded_and let Some(start) = start else { return; }; - let end = source[start..].find("fn raw_transactions_equal("); + let end = source[start..].find("async fn rehydrate_transaction("); assert!(end.is_some(), "missing RAW conflict logger end marker"); let Some(end) = end else { return; @@ -645,33 +645,42 @@ fn v0_3_15_pre_014_content_conflict_diagnostic_exposes_only_block_slot_and_bound } #[test] -fn v0_3_15_pre_014_fix_001_truncated_log_compatibility_is_narrow_and_keeps_canonical_content() { +fn v0_3_16_pre_005_postgres_uses_shared_fail_closed_variant_comparator_without_promotion() { let source = include_str!("../src/raw_transaction.rs"); for required in [ - "ActiveIncomingTruncatedLogs", + "ksp_store_api::compare_raw_transaction_variants", + "RawTransactionVariantRelation::Exact", + "RawTransactionVariantRelation::CompatibleLessComplete", + "RawTransactionVariantRelation::CompatibleMoreComplete", + "RawTransactionVariantRelation::Conflict", + "RawTransactionVariantRelation::Incomparable", + "ExistingTransactionMatch::ActiveCompatibleLessComplete", + "raw_acquisition_content_conflict", + ] { + assert!(source.contains(required), "missing shared comparator integration guard: {required}"); + } + for forbidden in [ "raw_transaction_incoming_truncated_log_messages_compatible", "raw_meta_other_than_log_messages_mismatch", "raw_log_messages_incoming_truncated_compatible", "raw_log_message_is_exact_truncation_marker", - "marker_index >= stored.len()", - "incoming[..marker_index] != stored[..marker_index]", - "PostgreSQL Store accepted compatible truncated RAW transaction logs without replacing canonical content", ] { - assert!(source.contains(required), "missing narrow truncated-log compatibility guard: {required}"); + assert!(!source.contains(forbidden), "backend-private convergence heuristic survived pre.005: {forbidden}"); } let start = source.find("fn compare_existing_transaction("); - assert!(start.is_some(), "missing fix.001 existing-transaction comparison path"); + assert!(start.is_some(), "missing pre.005 existing-transaction comparison path"); let Some(start) = start else { return; }; - let end = source[start..].find("fn raw_transaction_incoming_truncated_log_messages_compatible("); - assert!(end.is_some(), "missing fix.001 compatibility helper boundary"); + let end = source[start..].find("fn raw_transaction_content_conflict_diagnostic("); + assert!(end.is_some(), "missing pre.005 comparison boundary"); let Some(end) = end else { return; }; let comparison = &source[start..start + end]; - assert!(comparison.contains("ExistingTransactionMatch::ActiveIncomingTruncatedLogs")); + assert!(comparison.contains("CompatibleLessComplete")); + assert!(comparison.contains("CompatibleMoreComplete")); assert!(!comparison.contains("UPDATE ksp_raw_transactions SET payload")); - assert!(!comparison.contains("DELETE FROM ksp_raw_transactions")); + assert!(!comparison.contains("canonical_revision")); return; } diff --git a/crates/ksp-store-postgres-lib/tests/v003_variant_persistence.rs b/crates/ksp-store-postgres-lib/tests/v003_variant_persistence.rs index 2fc2f55..7d0b4c0 100644 --- a/crates/ksp-store-postgres-lib/tests/v003_variant_persistence.rs +++ b/crates/ksp-store-postgres-lib/tests/v003_variant_persistence.rs @@ -1,5 +1,5 @@ // file: crates/ksp-store-postgres-lib/tests/v003_variant_persistence.rs -// version: 1 +// version: 2 #![warn(missing_docs)] #![deny(unreachable_pub)] @@ -59,9 +59,11 @@ fn v0_3_16_pre_004_legacy_observation_without_mapping_is_not_fabricated() { } #[test] -fn v0_3_16_pre_004_keeps_comparator_and_durable_conflict_scope_deferred() { +fn v0_3_16_pre_005_shared_comparator_is_active_but_durable_conflict_and_promotion_stay_deferred() { let source = include_str!("../src/raw_transaction.rs"); - assert!(!source.contains("RawTransactionVariantRelation::")); + assert!(source.contains("ksp_store_api::compare_raw_transaction_variants")); + assert!(source.contains("ksp_store_api::RawTransactionVariantRelation::CompatibleLessComplete")); + assert!(source.contains("ksp_store_api::RawTransactionVariantRelation::CompatibleMoreComplete")); assert!(!source.contains("INSERT INTO ksp_raw_transaction_conflicts")); assert!(!source.contains("UPDATE ksp_raw_transaction_canonical_selectors SET canonical_variant_id")); return; diff --git a/crates/ksp-store-postgres-lib/unit_tests/raw_transaction.rs b/crates/ksp-store-postgres-lib/unit_tests/raw_transaction.rs index a4fc746..4ca07b0 100644 --- a/crates/ksp-store-postgres-lib/unit_tests/raw_transaction.rs +++ b/crates/ksp-store-postgres-lib/unit_tests/raw_transaction.rs @@ -1,5 +1,5 @@ // file: crates/ksp-store-postgres-lib/unit_tests/raw_transaction.rs -// version: 11 +// version: 12 fn network() -> ksp_store_api::RawNetworkId { return match ksp_store_api::RawNetworkId::new("devnet") { @@ -458,7 +458,7 @@ fn v0_3_15_pre_014_log_messages_conflict_diagnostic_distinguishes_prefix_truncat } #[test] -fn v0_3_15_pre_014_fix_001_incoming_truncated_logs_are_compatible_only_when_missing_middle_is_proven() { +fn v0_3_16_pre_005_shared_comparator_accepts_only_proved_incoming_truncation_in_postgres_path() { let stored = raw_transaction( 9, br#"{"transaction":["AQID","base64"],"meta":{"err":null,"fee":5000,"logMessages":["Program Vote111 invoke [1]","Program log: stable","Program log: detail-a","Program log: detail-b","Program Vote111 success"],"rewards":[]},"version":0,"transactionIndex":7}"#, @@ -466,10 +466,15 @@ fn v0_3_15_pre_014_fix_001_incoming_truncated_logs_are_compatible_only_when_miss ); let incoming = raw_transaction( 9, - br#"{"transaction":["AQID","base64"],"meta":{"err":null,"fee":5000,"logMessages":["Program Vote111 invoke [1]","Program log: stable","Log truncated","Program Vote111 success"],"rewards":[]},"version":0,"transactionIndex":7}"#, + br#"{"transaction":["AQID","base64"],"meta":{"err":null,"fee":5000,"logMessages":["Program Vote111 invoke [1]","Program log: stable","Log truncated"],"rewards":[]},"version":0,"transactionIndex":7}"#, 8, ); - assert!(super::raw_transaction_incoming_truncated_log_messages_compatible(&stored, &incoming)); + let relation = ksp_store_api::compare_raw_transaction_variants(&stored, &incoming); + let relation = match relation { + std::result::Result::Ok(value) => value, + std::result::Result::Err(_) => return, + }; + assert_eq!(relation.relation(), ksp_store_api::RawTransactionVariantRelation::CompatibleLessComplete); let mut stored_row = transaction_row("full"); stored_row.signature = stored.reference().signature().as_bytes().to_vec(); stored_row.slot_text = stored.slot().to_string(); @@ -479,12 +484,12 @@ fn v0_3_15_pre_014_fix_001_incoming_truncated_logs_are_compatible_only_when_miss stored_row.content_hash = stored.payload().content_hash().as_bytes().to_vec(); stored_row.payload = std::option::Option::Some(stored.payload().bytes().to_vec()); let comparison = super::compare_existing_transaction(&network(), stored_row, &incoming); - assert!(matches!(comparison, std::result::Result::Ok(super::ExistingTransactionMatch::ActiveIncomingTruncatedLogs))); + assert!(matches!(comparison, std::result::Result::Ok(super::ExistingTransactionMatch::ActiveCompatibleLessComplete))); return; } #[test] -fn v0_3_15_pre_014_fix_001_truncated_log_compatibility_rejects_unproven_or_reverse_changes() { +fn v0_3_16_pre_005_shared_comparator_rejects_unproved_shapes_and_defers_more_complete_promotion() { let stored = raw_transaction( 9, br#"{"transaction":["AQID","base64"],"meta":{"err":null,"fee":5000,"logMessages":["Program Vote111 invoke [1]","Program log: stable","Program log: detail-a","Program log: detail-b","Program Vote111 success"],"rewards":[]},"version":0,"transactionIndex":7}"#, @@ -492,34 +497,52 @@ fn v0_3_15_pre_014_fix_001_truncated_log_compatibility_rejects_unproven_or_rever ); let bad_prefix = raw_transaction( 9, - br#"{"transaction":["AQID","base64"],"meta":{"err":null,"fee":5000,"logMessages":["Program Vote111 invoke [1]","Program log: changed","Log truncated","Program Vote111 success"],"rewards":[]},"version":0,"transactionIndex":7}"#, + br#"{"transaction":["AQID","base64"],"meta":{"err":null,"fee":5000,"logMessages":["Program Vote111 invoke [1]","Program log: changed","Log truncated"],"rewards":[]},"version":0,"transactionIndex":7}"#, 8, ); - assert!(!super::raw_transaction_incoming_truncated_log_messages_compatible(&stored, &bad_prefix)); let post_marker_material = raw_transaction( 9, br#"{"transaction":["AQID","base64"],"meta":{"err":null,"fee":5000,"logMessages":["Program Vote111 invoke [1]","Program log: stable","Log truncated","Program Other111 success"],"rewards":[]},"version":0,"transactionIndex":7}"#, 8, ); - assert!(super::raw_transaction_incoming_truncated_log_messages_compatible(&stored, &post_marker_material)); let other_meta = raw_transaction( 9, - br#"{"transaction":["AQID","base64"],"meta":{"err":null,"fee":5001,"logMessages":["Program Vote111 invoke [1]","Program log: stable","Log truncated","Program Vote111 success"],"rewards":[]},"version":0,"transactionIndex":7}"#, + br#"{"transaction":["AQID","base64"],"meta":{"err":null,"fee":5001,"logMessages":["Program Vote111 invoke [1]","Program log: stable","Log truncated"],"rewards":[]},"version":0,"transactionIndex":7}"#, 8, ); - assert!(!super::raw_transaction_incoming_truncated_log_messages_compatible(&stored, &other_meta)); let non_exact_marker = raw_transaction( 9, br#"{"transaction":["AQID","base64"],"meta":{"err":null,"fee":5000,"logMessages":["Program Vote111 invoke [1]","Program log: stable","Log truncated by provider"],"rewards":[]},"version":0,"transactionIndex":7}"#, 8, ); - assert!(!super::raw_transaction_incoming_truncated_log_messages_compatible(&stored, &non_exact_marker)); + for candidate in [&bad_prefix, &post_marker_material, &other_meta, &non_exact_marker] { + let relation = match ksp_store_api::compare_raw_transaction_variants(&stored, candidate) { + std::result::Result::Ok(value) => value, + std::result::Result::Err(_) => return, + }; + assert_ne!(relation.relation(), ksp_store_api::RawTransactionVariantRelation::CompatibleLessComplete); + assert_ne!(relation.relation(), ksp_store_api::RawTransactionVariantRelation::CompatibleMoreComplete); + } let stored_truncated = raw_transaction( 9, - br#"{"transaction":["AQID","base64"],"meta":{"err":null,"fee":5000,"logMessages":["Program Vote111 invoke [1]","Program log: stable","Log truncated","Program Vote111 success"],"rewards":[]},"version":0,"transactionIndex":7}"#, + br#"{"transaction":["AQID","base64"],"meta":{"err":null,"fee":5000,"logMessages":["Program Vote111 invoke [1]","Program log: stable","Log truncated"],"rewards":[]},"version":0,"transactionIndex":7}"#, 8, ); - assert!(!super::raw_transaction_incoming_truncated_log_messages_compatible(&stored_truncated, &stored)); + let more = match ksp_store_api::compare_raw_transaction_variants(&stored_truncated, &stored) { + std::result::Result::Ok(value) => value, + std::result::Result::Err(_) => return, + }; + assert_eq!(more.relation(), ksp_store_api::RawTransactionVariantRelation::CompatibleMoreComplete); + let mut stored_row = transaction_row("full"); + stored_row.signature = stored_truncated.reference().signature().as_bytes().to_vec(); + stored_row.slot_text = stored_truncated.slot().to_string(); + stored_row.block_time_unix_millis = stored_truncated.block_time().and_then(|value| return i64::try_from(value.unix_millis()).ok()); + stored_row.format_id = stored_truncated.payload().format_id().as_str().to_owned(); + stored_row.format_version = i64::from(stored_truncated.payload().format_version()); + stored_row.content_hash = stored_truncated.payload().content_hash().as_bytes().to_vec(); + stored_row.payload = std::option::Option::Some(stored_truncated.payload().bytes().to_vec()); + let comparison = super::compare_existing_transaction(&network(), stored_row, &stored); + assert!(matches!(comparison, std::result::Result::Err(error) if error.phase() == "raw_acquisition_content_conflict")); return; } diff --git a/deltas/0.3.16/pre.005-fix.001.md b/deltas/0.3.16/pre.005-fix.001.md new file mode 100644 index 0000000..d8c6bd3 --- /dev/null +++ b/deltas/0.3.16/pre.005-fix.001.md @@ -0,0 +1,77 @@ + + + +# Delta `0.3.16-pre.005-fix.001` — conformité Clippy et canari de scope `pre.005` + +## Base requise + +```text +0.3.16-pre.005 appliquée +workspace.package.version = 0.3.16-pre.5 +``` + +Le gate opérateur de `pre.005` confirme que les audits Rust/export/KSP/Markdown sont propres, que `cargo check --workspace` passe et que les suites ciblées exécutées sont fonctionnellement vertes à l'exception d'un canari PostgreSQL devenu obsolète. Clippy `-D warnings` refuse toutefois treize usages de l'opérateur `?` sous la règle workspace `clippy::question_mark_used` : dix dans le scanner JSON privé du comparateur et trois dans ses helpers de tests. + +Le seul échec de test est `v0_3_16_pre_004_keeps_comparator_and_durable_conflict_scope_deferred`, qui interdit encore toute référence à `RawTransactionVariantRelation::` alors que `pre.005` a précisément introduit le comparateur partagé et ses relations dans le chemin PostgreSQL. + +## Version + +Cette correction modifie du Rust : + +```text +workspace.package.version = 0.3.16-pre.5.fix.1 +``` + +## Corrections + +Le correctif ne modifie aucune règle métier du comparateur : + +- les dix propagations `Option` par `?` du scanner JSON privé sont remplacées par des `match` explicites qui retournent `None` sur le même chemin d'échec ; +- les trois usages `?` des helpers de tests Store API sont remplacés par les mêmes retours explicites ; +- aucune autorisation locale `#[allow(clippy::question_mark_used)]` n'est ajoutée ; +- aucune dépendance runtime n'est ajoutée à `ksp-store-api` ; +- le canari V003 hérité de `pre.004` est mis à jour pour exiger désormais l'appel à `ksp_store_api::compare_raw_transaction_variants` et les relations `CompatibleLessComplete` / `CompatibleMoreComplete` ; +- ce même canari continue d'interdire l'écriture dans `ksp_raw_transaction_conflicts` et toute promotion du selector canonique avant les tranches prévues. + +## Invariants conservés + +La sémantique de `pre.005` reste inchangée : + +```text +Exact -> idempotence canonique +CompatibleLessComplete -> canonique conservé, variante entrante persistée/rattachée +CompatibleMoreComplete -> reconnue mais fail-closed jusqu'à pre.006 +Conflict / Incomparable -> conflit existant sans conflict case durable avant pre.007 +``` + +La preuve `logMessages` reste strictement bidirectionnelle, avec marqueur exact `"Log truncated"`, unique et terminal, préfixe byte-identique et matériel supplémentaire obligatoire du côté complet. + +Aucune migration SQL, ressource V003, promotion canonique, `canonical_revision`, journal de transition ou conflit durable n'est modifié par ce fix. + +## Validation exécutée lors de la génération + +Les scripts Python réels du checkout complet sont exécutés sur l'état final puis sur le ZIP rejoué sur une copie fraîche de `pre.005`. + +Le toolchain Rust n'est pas disponible dans l'environnement de génération ; les gates Cargo restent opérateur. + +## Validation opérateur demandée + +```bash +cargo fmt --all +cargo fmt --all -- --check + +python3 scripts/audit_rust_workspace_rules.py +python3 scripts/audit_markdown_tables.py README.md RULES.md ROADMAP.md CHANGELOG.md docs prompts crates deltas + +cargo check --workspace +cargo clippy --workspace --all-targets --all-features -- -D warnings + +cargo test -p ksp-store-api --all-targets --all-features +cargo test -p ksp-store-postgres-lib --all-targets --all-features +``` + +Les trois preuves PostgreSQL live restent opt-in. + +## Suite + +Après gate propre : `0.3.16-pre.006` — selector canonique, promotion atomique de `CompatibleMoreComplete`, `canonical_revision`, projection V001 cohérente et conservation de l'ancien canonique. diff --git a/deltas/0.3.16/pre.005.md b/deltas/0.3.16/pre.005.md new file mode 100644 index 0000000..bea4fd5 --- /dev/null +++ b/deltas/0.3.16/pre.005.md @@ -0,0 +1,173 @@ + + + +# Delta `0.3.16-pre.005` — comparateur backend-neutral des variantes RAW + +## Base requise + +```text +0.3.16-pre.004 appliquée +workspace.package.version = 0.3.16-pre.4 +``` + +Le gate opérateur de `pre.004` est confirmé propre : audits Rust/export/KSP/Markdown, `cargo check --workspace`, Clippy `-D warnings`, 85 tests unitaires `ksp-store-postgres-lib` et toutes les suites d'intégration ciblées passent. Les trois preuves PostgreSQL live restent opt-in. + +## Version + +Cette tranche modifie le contrat Store API et son intégration PostgreSQL : + +```text +workspace.package.version = 0.3.16-pre.5 +``` + +## Portée + +Cette tranche rend effectif le comparateur pur backend-neutral prévu depuis `pre.002`. + +`ksp-store-api` expose désormais : + +```text +compare_raw_transaction_variants(canonical, incoming) + -> RawTransactionVariantComparison +``` + +Le comparateur produit exclusivement les relations déjà figées : + +```text +Exact +CompatibleLessComplete +CompatibleMoreComplete +Conflict +Incomparable +``` + +Le comparateur n'ajoute aucune dépendance runtime à `ksp-store-api`. En particulier, `serde_json` reste interdit par le canari de dépendances existant. L'analyse du payload RAW v1 utilise un scanner JSON privé, borné en profondeur et fondé uniquement sur `std`. + +## Ordre de décision + +La comparaison est fail-closed et applique l'ordre suivant : + +1. les deux variantes doivent appartenir à la même identité `network + signature`, sinon l'appel est invalide ; +2. `slot` différent -> `Conflict / SlotMismatch` ; +3. `block_time` différent -> `Conflict / BlockTimeMismatch` ; +4. format id/version différent -> `Incomparable / PayloadFormatMismatch` ; +5. payload bytes exactement égaux -> `Exact / ExactCanonicalContent` ; +6. hash égal mais bytes différents -> `Conflict / ContentHashCollision` ; +7. seulement ensuite, analyse structurelle du RAW v1 pour une éventuelle preuve de troncature `logMessages` ; +8. toute autre divergence est `Conflict` ou `Incomparable` sans dominance automatique. + +Le `content_hash` reste donc un marqueur d'intégrité/préfiltre, jamais une preuve d'égalité. + +## Preuve `logMessages` + +La seule dominance automatique admise reste la troncature SVM strictement prouvée. + +Pour reconnaître une variante tronquée, toutes les conditions suivantes sont nécessaires : + +- les autres composantes top-level sont identiques ; +- les autres champs `meta` sont identiques ; +- les deux `logMessages` sont des tableaux de chaînes JSON valides ; +- le côté tronqué contient exactement un marqueur exact `"Log truncated"` ; +- ce marqueur est le dernier élément ; +- le côté complet ne contient aucun marqueur exact ; +- le préfixe avant le marqueur est byte-identique ; +- le côté complet possède au moins une ligne au-delà de ce préfixe. + +Ainsi : + +```text +canonique complet + entrant tronqué -> CompatibleLessComplete +canonique tronqué + entrant complet -> CompatibleMoreComplete +``` + +Une liste plus courte sans marqueur, un préfixe divergent, un marqueur non exact ou du matériel placé après `"Log truncated"` ne prouve aucune dominance. + +## Politique fail-closed des autres champs + +Les différences de présence/omission ou `null` contre une valeur concrète ne sont pas transformées en score de qualité. Elles produisent `Incomparable / UnsupportedCanonicalDifference`. + +Les valeurs concrètes présentes des deux côtés mais contradictoires produisent `Conflict / CanonicalPayloadConflict`. + +Cette politique couvre notamment les champs optionnels documentés dans le plan (`innerInstructions`, `loadedAddresses`, `returnData`, unités consommées, token balances, rewards) sans inventer de hiérarchie de complétude. + +## Intégration PostgreSQL + +`ksp-store-postgres-lib` ne possède plus l'heuristique privée unidirectionnelle `raw_transaction_incoming_truncated_log_messages_compatible`. + +`compare_existing_transaction` appelle désormais directement : + +```text +ksp_store_api::compare_raw_transaction_variants(...) +``` + +Comportement de cette tranche : + +- `Exact` -> idempotence canonique existante ; +- `CompatibleLessComplete` -> conservation du canonique et persistance/rattachement de la variante entrante comme en `pre.004` ; +- `CompatibleMoreComplete` -> relation reconnue mais encore fail-closed côté mutation, car la promotion atomique appartient à `pre.006` ; +- `Conflict` / `Incomparable` -> conflit PostgreSQL existant, sans ouverture de conflict case durable avant `pre.007`. + +Aucune mise à jour du selector, de `canonical_revision` ou de la projection V001 n'est ajoutée ici. + +## Canaris + +Les tests Store API ajoutés verrouillent : + +- la relation bidirectionnelle `LessComplete` / `MoreComplete` ; +- le marqueur exact, unique et terminal ; +- le rejet d'une simple liste plus courte ; +- le rejet d'un préfixe divergent ; +- le rejet d'un marqueur provider non exact ; +- le rejet de matériel après le marqueur ; +- la politique `Incomparable` sur optionalité `null`/valeur ; +- le `Conflict` sur valeurs concrètes contradictoires ; +- l'égalité par bytes même si les hashes fournis diffèrent ; +- la détection d'une collision de hash lorsque les bytes diffèrent. + +Les canaris PostgreSQL verrouillent en plus : + +- l'usage exclusif du comparateur Store API pour la décision de convergence ; +- l'absence des anciens helpers privés de compatibilité ; +- l'absence de promotion/écriture canonique dans `pre.005` ; +- le refus temporaire de `CompatibleMoreComplete` jusqu'à `pre.006`. + +## Hors scope maintenu + +Cette tranche n'ajoute pas : + +- promotion du selector canonique ; +- incrément de `canonical_revision` ; +- remplacement atomique de la projection V001 ; +- journal de transition canonique ; +- ouverture de `ksp_raw_transaction_conflicts` ; +- outcome Worker non terminal pour les conflits ; +- Store Desk ; +- retry Store/Transport ; +- politique de rétention des variantes. + +Aucune migration SQL ni ressource V003 n'est modifiée. + +## Validation exécutée lors de la génération + +Les scripts Python réels du checkout complet sont exécutés sur l'état final puis sur le ZIP rejoué sur une copie fraîche de `pre.004`. + +Le toolchain Rust n'est pas disponible dans l'environnement de génération ; les gates Cargo restent opérateur. + +## Validation opérateur demandée + +```bash +cargo fmt --all +cargo fmt --all -- --check + +python3 scripts/audit_rust_workspace_rules.py +python3 scripts/audit_markdown_tables.py README.md RULES.md ROADMAP.md CHANGELOG.md docs prompts crates deltas + +cargo check --workspace +cargo clippy --workspace --all-targets --all-features -- -D warnings +cargo test -p ksp-store-api --all-targets --all-features +cargo test -p ksp-store-postgres-lib --all-targets --all-features +``` + +## Suite + +Après gate propre : `0.3.16-pre.006` — selector canonique et promotion atomique avec revision, projection V001 cohérente, conservation de l'ancien canonique et journal minimal des transitions. diff --git a/docs/plans/038-V0_3_16_RAW_RESILIENCE_CONFLICT_PLAN.md b/docs/plans/038-V0_3_16_RAW_RESILIENCE_CONFLICT_PLAN.md index c9a7536..62e345e 100644 --- a/docs/plans/038-V0_3_16_RAW_RESILIENCE_CONFLICT_PLAN.md +++ b/docs/plans/038-V0_3_16_RAW_RESILIENCE_CONFLICT_PLAN.md @@ -970,15 +970,15 @@ Contrats Store API backend-neutral : variant identity, relation de qualité, out #### `0.3.16-pre.003` -Migration V003 : registre d'extension, resources et schéma multi-variantes/selector/conflict minimal ; canaris garantissant l'immuabilité byte/checksum de V000/V001/V002. Implémentation livrée par cette tranche ; gate opérateur requis avant `pre.004`. +Migration V003 : registre d'extension, resources et schéma multi-variantes/selector/conflict minimal ; canaris garantissant l'immuabilité byte/checksum de V000/V001/V002. Implémentation livrée et gate opérateur confirmé propre après `pre.003-fix.002`. #### `0.3.16-pre.004` -Backend PostgreSQL : bootstrap V003, variant ledger, rattachement exact des nouvelles observations à leur variante, idempotence et concurrence d'insertion. +Backend PostgreSQL : bootstrap V003, variant ledger, rattachement exact des nouvelles observations à leur variante, idempotence et concurrence d'insertion. Implémentation livrée et gate opérateur confirmé propre. #### `0.3.16-pre.005` -Comparateur partagé : `Exact`, `CompatibleLessComplete`, `CompatibleMoreComplete`, `Conflict`, `Incomparable` ; canaris `logMessages` bidirectionnels et politique fail-closed sur les autres champs. +Comparateur partagé : `Exact`, `CompatibleLessComplete`, `CompatibleMoreComplete`, `Conflict`, `Incomparable` ; canaris `logMessages` bidirectionnels et politique fail-closed sur les autres champs. Implémentation livrée par cette tranche ; gate opérateur requis avant `pre.006`. #### `0.3.16-pre.006`