0.3.16-pre.007
This commit is contained in:
@@ -1,5 +1,5 @@
|
||||
// file: crates/ksp-job-backfill-lib/src/persistence.rs
|
||||
// version: 3
|
||||
// version: 4
|
||||
|
||||
/// Canonical entity disposition produced by one Backfill Store persistence attempt.
|
||||
#[non_exhaustive]
|
||||
@@ -13,7 +13,7 @@ pub enum BackfillEntityPersistence {
|
||||
SkippedPurged,
|
||||
/// `getTransaction` returned JSON `null`, so no canonical Store write was attempted.
|
||||
Missing,
|
||||
/// Store reported divergent canonical content for the same network-scoped transaction identity.
|
||||
/// Store durably quarantined divergent/incomparable content, or a legacy backend reported a conflict error.
|
||||
Conflict,
|
||||
}
|
||||
|
||||
@@ -25,7 +25,7 @@ pub enum BackfillObservationPersistence {
|
||||
Inserted,
|
||||
/// The same deterministic acquisition observation was already durable.
|
||||
AlreadyPresent,
|
||||
/// Store intentionally recorded no observation because normal persistence skipped a purged entity or detected a conflict.
|
||||
/// Store intentionally recorded no observation because normal persistence skipped a purged entity or a legacy conflict error predated durable quarantine.
|
||||
NotRecorded,
|
||||
/// No observation existed because hydration returned `Missing` before persistence.
|
||||
NotApplicable,
|
||||
@@ -75,9 +75,9 @@ impl crate::BackfillPersistenceOutcome {
|
||||
///
|
||||
/// Available acquisitions use the existing atomic transaction-plus-observation Store contract in
|
||||
/// `Normal` mode. A durable purge is therefore respected and never rehydrated implicitly.
|
||||
/// `Missing` performs no Store write. Stable Store content conflicts are projected explicitly as
|
||||
/// [`BackfillEntityPersistence::Conflict`] rather than being silently treated as idempotent skips.
|
||||
/// Other Store failures remain errors.
|
||||
/// `Missing` performs no Store write. Durable quarantined conflicts are projected explicitly as
|
||||
/// [`BackfillEntityPersistence::Conflict`] while preserving the actual observation disposition;
|
||||
/// legacy Store conflict errors remain compatible and other Store failures remain errors.
|
||||
pub async fn persist_backfill_hydration(
|
||||
store: &ksp_store_lib::Store,
|
||||
hydration: crate::BackfillHydrationOutcome,
|
||||
@@ -170,6 +170,17 @@ fn map_store_outcome(
|
||||
) -> ksp_core_lib::Result<crate::BackfillPersistenceOutcome> {
|
||||
let entity = outcome.entity();
|
||||
let observation = outcome.observation();
|
||||
if outcome.transaction_variant() == std::option::Option::Some(ksp_store_lib::RawTransactionVariantWriteOutcome::QuarantinedConflict) {
|
||||
let projected_observation = match observation {
|
||||
ksp_store_lib::RawObservationWriteOutcome::Inserted => crate::BackfillObservationPersistence::Inserted,
|
||||
ksp_store_lib::RawObservationWriteOutcome::AlreadyPresent => crate::BackfillObservationPersistence::AlreadyPresent,
|
||||
_ => return std::result::Result::Err(persistence_error("store.conflict_observation")),
|
||||
};
|
||||
if entity != ksp_store_lib::RawEntityWriteOutcome::AlreadyPresent {
|
||||
return std::result::Result::Err(persistence_error("store.conflict_entity"));
|
||||
}
|
||||
return std::result::Result::Ok(crate::BackfillPersistenceOutcome::new(reference, crate::BackfillEntityPersistence::Conflict, projected_observation));
|
||||
}
|
||||
if entity == ksp_store_lib::RawEntityWriteOutcome::Inserted && observation == ksp_store_lib::RawObservationWriteOutcome::Inserted {
|
||||
return std::result::Result::Ok(crate::BackfillPersistenceOutcome::new(
|
||||
reference,
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
// file: crates/ksp-job-backfill-lib/unit_tests/persistence.rs
|
||||
// version: 1
|
||||
// version: 2
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
enum FakeResponse {
|
||||
@@ -143,6 +143,14 @@ fn store_outcome(
|
||||
return ksp_store_lib::RawAcquisitionWriteOutcome::new(entity, observation);
|
||||
}
|
||||
|
||||
fn store_variant_outcome(
|
||||
entity: ksp_store_lib::RawEntityWriteOutcome,
|
||||
observation: ksp_store_lib::RawObservationWriteOutcome,
|
||||
transaction_variant: ksp_store_lib::RawTransactionVariantWriteOutcome,
|
||||
) -> ksp_store_lib::RawAcquisitionWriteOutcome {
|
||||
return ksp_store_lib::RawAcquisitionWriteOutcome::with_transaction_variant(entity, observation, transaction_variant);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn pre_007_missing_skips_store_and_preserves_network_scoped_identity() {
|
||||
let reference = match raw_reference("devnet", 1) {
|
||||
@@ -266,7 +274,33 @@ async fn pre_007_normal_backfill_respects_purged_tombstone_without_observation()
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn pre_007_store_content_conflict_is_explicit_and_not_idempotent_success() {
|
||||
async fn v0_3_16_pre_007_durable_quarantined_conflict_preserves_observation_projection() {
|
||||
let (reference, transaction, observation) = match raw_acquisition_parts("devnet", 6, 16) {
|
||||
std::option::Option::Some(parts) => parts,
|
||||
std::option::Option::None => return,
|
||||
};
|
||||
let network = match raw_network("devnet") {
|
||||
std::option::Option::Some(network) => network,
|
||||
std::option::Option::None => return,
|
||||
};
|
||||
let response = store_variant_outcome(
|
||||
ksp_store_lib::RawEntityWriteOutcome::AlreadyPresent,
|
||||
ksp_store_lib::RawObservationWriteOutcome::Inserted,
|
||||
ksp_store_lib::RawTransactionVariantWriteOutcome::QuarantinedConflict,
|
||||
);
|
||||
let port = FakePersistencePort::new(network, &[FakeResponse::Outcome(response)]);
|
||||
let result = super::persist_available_with_port(&port, reference, transaction, observation).await;
|
||||
assert!(result.is_ok());
|
||||
if let std::result::Result::Ok(result) = result {
|
||||
assert_eq!(result.entity(), crate::BackfillEntityPersistence::Conflict);
|
||||
assert_eq!(result.observation(), crate::BackfillObservationPersistence::Inserted);
|
||||
assert_ne!(result.entity(), crate::BackfillEntityPersistence::AlreadyPresent);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn pre_007_legacy_store_content_conflict_error_is_explicit_and_not_idempotent_success() {
|
||||
let (reference, transaction, observation) = match raw_acquisition_parts("devnet", 6, 16) {
|
||||
std::option::Option::Some(parts) => parts,
|
||||
std::option::Option::None => return,
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
// file: crates/ksp-store-api/src/capability/raw_transaction.rs
|
||||
// version: 4
|
||||
// version: 5
|
||||
|
||||
/// Read capability for canonical RAW transactions.
|
||||
///
|
||||
@@ -42,8 +42,9 @@ pub trait RawTransactionInspectionRead: std::marker::Send + std::marker::Sync {
|
||||
///
|
||||
/// The transaction and its acquisition observation form one logical persistence
|
||||
/// operation. An implementation must not leave one side durable if the other
|
||||
/// side fails. Divergent content for one logical identity is a stable conflict
|
||||
/// error, never a silent overwrite.
|
||||
/// side fails. Divergent or incomparable content must be preserved as a durable
|
||||
/// variant/conflict outcome rather than silently overwriting the canonical transaction
|
||||
/// or being projected as a terminal persistence failure.
|
||||
pub trait RawTransactionWrite: std::marker::Send + std::marker::Sync {
|
||||
/// Persists one complete RAW transaction together with one observation atomically.
|
||||
fn persist_raw_transaction_acquisition<'a>(
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
// file: crates/ksp-store-api/src/model/raw_outcome.rs
|
||||
// version: 3
|
||||
// version: 4
|
||||
/// Outcome for one canonical RAW entity in an idempotent persistence operation.
|
||||
#[non_exhaustive]
|
||||
#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)]
|
||||
@@ -31,13 +31,24 @@ pub enum RawObservationWriteOutcome {
|
||||
pub struct RawAcquisitionWriteOutcome {
|
||||
entity: crate::RawEntityWriteOutcome,
|
||||
observation: crate::RawObservationWriteOutcome,
|
||||
transaction_variant: std::option::Option<crate::RawTransactionVariantWriteOutcome>,
|
||||
}
|
||||
|
||||
impl RawAcquisitionWriteOutcome {
|
||||
/// Creates one backend-independent atomic acquisition outcome.
|
||||
#[must_use]
|
||||
pub const fn new(entity: crate::RawEntityWriteOutcome, observation: crate::RawObservationWriteOutcome) -> Self {
|
||||
return Self { entity, observation };
|
||||
return Self { entity, observation, transaction_variant: std::option::Option::None };
|
||||
}
|
||||
|
||||
/// Creates one backend-independent atomic RAW transaction acquisition outcome with variant convergence classification.
|
||||
#[must_use]
|
||||
pub const fn with_transaction_variant(
|
||||
entity: crate::RawEntityWriteOutcome,
|
||||
observation: crate::RawObservationWriteOutcome,
|
||||
transaction_variant: crate::RawTransactionVariantWriteOutcome,
|
||||
) -> Self {
|
||||
return Self { entity, observation, transaction_variant: std::option::Option::Some(transaction_variant) };
|
||||
}
|
||||
|
||||
/// Returns the canonical RAW entity write outcome.
|
||||
@@ -51,6 +62,12 @@ impl RawAcquisitionWriteOutcome {
|
||||
pub const fn observation(&self) -> crate::RawObservationWriteOutcome {
|
||||
return self.observation;
|
||||
}
|
||||
|
||||
/// Returns the variant-aware transaction outcome when the persistence operation was a RAW transaction acquisition.
|
||||
#[must_use]
|
||||
pub const fn transaction_variant(&self) -> std::option::Option<crate::RawTransactionVariantWriteOutcome> {
|
||||
return self.transaction_variant;
|
||||
}
|
||||
}
|
||||
|
||||
/// Durable Store outcome for one RAW transaction acquisition after variant-aware convergence.
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
// file: crates/ksp-store-api/tests/public_api.rs
|
||||
// version: 10
|
||||
// version: 11
|
||||
|
||||
//! Integration canaries for the public `ksp-store-api` surface.
|
||||
|
||||
@@ -170,6 +170,13 @@ fn public_pre_006_query_outcome_and_retention_contracts_are_available_from_crate
|
||||
ksp_store_api::RawAcquisitionWriteOutcome::new(ksp_store_api::RawEntityWriteOutcome::Inserted, ksp_store_api::RawObservationWriteOutcome::Inserted);
|
||||
assert_eq!(outcome.entity(), ksp_store_api::RawEntityWriteOutcome::Inserted);
|
||||
assert_eq!(outcome.observation(), ksp_store_api::RawObservationWriteOutcome::Inserted);
|
||||
assert_eq!(outcome.transaction_variant(), std::option::Option::None);
|
||||
let variant_outcome = ksp_store_api::RawAcquisitionWriteOutcome::with_transaction_variant(
|
||||
ksp_store_api::RawEntityWriteOutcome::AlreadyPresent,
|
||||
ksp_store_api::RawObservationWriteOutcome::Inserted,
|
||||
ksp_store_api::RawTransactionVariantWriteOutcome::QuarantinedConflict,
|
||||
);
|
||||
assert_eq!(variant_outcome.transaction_variant(), std::option::Option::Some(ksp_store_api::RawTransactionVariantWriteOutcome::QuarantinedConflict));
|
||||
let retention_read: std::option::Option<&dyn ksp_store_api::RawTransactionRetentionRead> = std::option::Option::None;
|
||||
let retention_write: std::option::Option<&dyn ksp_store_api::RawTransactionRetentionWrite> = std::option::Option::None;
|
||||
assert!(retention_read.is_none());
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
// file: crates/ksp-store-lib/src/lib.rs
|
||||
// version: 12
|
||||
// version: 13
|
||||
|
||||
#![warn(missing_docs)]
|
||||
#![deny(unreachable_pub)]
|
||||
@@ -229,6 +229,8 @@ pub use ksp_store_api::RawTransactionSignature;
|
||||
pub use ksp_store_api::RawTransactionSummary;
|
||||
/// Minimal durable identity retained after a canonical RAW transaction payload is purged.
|
||||
pub use ksp_store_api::RawTransactionTombstone;
|
||||
/// Durable Store outcome for one variant-aware RAW transaction acquisition.
|
||||
pub use ksp_store_api::RawTransactionVariantWriteOutcome;
|
||||
/// Write capability for canonical RAW transaction acquisitions.
|
||||
pub use ksp_store_api::RawTransactionWrite;
|
||||
/// Common KSP result alias using [`Error`].
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
// file: crates/ksp-store-lib/tests/hardening_completeness.rs
|
||||
// version: 10
|
||||
// version: 11
|
||||
|
||||
#![warn(missing_docs)]
|
||||
#![deny(unreachable_pub)]
|
||||
@@ -209,6 +209,7 @@ fn pre_009_facade_modules_and_crate_root_exports_are_exact() {
|
||||
"RawTransactionSignature",
|
||||
"RawTransactionSummary",
|
||||
"RawTransactionTombstone",
|
||||
"RawTransactionVariantWriteOutcome",
|
||||
"RawTransactionWrite",
|
||||
"Result",
|
||||
"Store",
|
||||
@@ -222,7 +223,7 @@ fn pre_009_facade_modules_and_crate_root_exports_are_exact() {
|
||||
];
|
||||
expected.sort_unstable();
|
||||
assert_eq!(actual.as_slice(), expected.as_slice());
|
||||
assert_eq!(actual.len(), 105);
|
||||
assert_eq!(actual.len(), 106);
|
||||
return;
|
||||
}
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
// file: crates/ksp-store-postgres-lib/src/raw_transaction.rs
|
||||
// version: 13
|
||||
// version: 14
|
||||
|
||||
pub(crate) mod cursor;
|
||||
|
||||
@@ -12,6 +12,7 @@ const GET_TOMBSTONE_SQL: &str = "SELECT signature, slot::text AS slot_text, bloc
|
||||
const GET_TRANSACTION_SQL: &str = "SELECT transaction_row.signature, transaction_row.slot::text AS slot_text, transaction_row.block_time_unix_millis, transaction_row.format_id, transaction_row.format_version, transaction_row.content_hash, transaction_row.payload, transaction_row.retention_state, archive_row.payload AS archive_payload FROM ksp_raw_transactions AS transaction_row LEFT JOIN ksp_raw_transaction_archive_payloads AS archive_row ON archive_row.signature = transaction_row.signature WHERE transaction_row.signature = $1";
|
||||
const INSERT_ARCHIVE_PAYLOAD_SQL: &str = "INSERT INTO ksp_raw_transaction_archive_payloads (signature, payload) VALUES ($1, $2)";
|
||||
const INSERT_OBSERVATION_SQL: &str = "INSERT INTO ksp_raw_transaction_observations (observation_key, transaction_signature, provider, protocol, acquisition_method, origin, received_at_unix_millis, capture_session_id, commitment, endpoint_id, filter_id, observed_at_unix_millis, source_payload_hash, source_payload_size_bytes) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14) ON CONFLICT (observation_key) DO NOTHING RETURNING observation_key";
|
||||
const INSERT_TRANSACTION_CONFLICT_SQL: &str = "INSERT INTO ksp_raw_transaction_conflicts (transaction_signature, status, revision, canonical_variant_id, incoming_variant_id, latest_relation, latest_reason_code, created_at_unix_millis, updated_at_unix_millis) VALUES ($1, 'open', 1, $2::TEXT::NUMERIC, $3::TEXT::NUMERIC, $4, $5, $6, $6)";
|
||||
const INSERT_TRANSACTION_SQL: &str = "INSERT INTO ksp_raw_transactions (signature, slot, block_time_unix_millis, format_id, format_version, content_hash, payload, retention_state) VALUES ($1, $2::TEXT::NUMERIC, $3, $4, $5, $6, $7, 'full') ON CONFLICT (signature) DO NOTHING RETURNING signature";
|
||||
const INSERT_TRANSACTION_VARIANT_LINK_SQL: &str = "INSERT INTO ksp_raw_transaction_observation_variants (observation_key, transaction_signature, variant_id, linked_at_unix_millis) VALUES ($1, $2, $3::TEXT::NUMERIC, $4) RETURNING observation_key";
|
||||
const INSERT_TRANSACTION_VARIANT_SELECTOR_SQL: &str = "INSERT INTO ksp_raw_transaction_canonical_selectors (transaction_signature, canonical_variant_id, canonical_revision, updated_at_unix_millis) VALUES ($1, $2::TEXT::NUMERIC, 1, $3)";
|
||||
@@ -25,6 +26,7 @@ const LIST_TRANSACTIONS_DESC_SQL: &str = "SELECT signature, slot::text AS slot_t
|
||||
const LOCK_OBSERVATION_SQL: &str = "SELECT observation_key, transaction_signature, provider, protocol, acquisition_method, origin, received_at_unix_millis, capture_session_id, commitment, endpoint_id, filter_id, observed_at_unix_millis, source_payload_hash, source_payload_size_bytes FROM ksp_raw_transaction_observations WHERE observation_key = $1 FOR UPDATE";
|
||||
const LOCK_RETENTION_TRANSACTION_SQL: &str =
|
||||
"SELECT block_time_unix_millis, payload, retention_state FROM ksp_raw_transactions WHERE signature = $1 FOR UPDATE";
|
||||
const LOCK_TRANSACTION_CONFLICT_SQL: &str = "SELECT status, revision::TEXT AS revision_text, canonical_variant_id::TEXT AS canonical_variant_id_text, incoming_variant_id::TEXT AS incoming_variant_id_text, latest_relation, latest_reason_code, created_at_unix_millis, updated_at_unix_millis FROM ksp_raw_transaction_conflicts WHERE transaction_signature = $1 FOR UPDATE";
|
||||
const LOCK_TRANSACTION_SQL: &str = "SELECT signature, slot::text AS slot_text, block_time_unix_millis, format_id, format_version, content_hash, payload, retention_state, NULL::BYTEA AS archive_payload FROM ksp_raw_transactions WHERE signature = $1 FOR UPDATE";
|
||||
const RAW_TRANSACTION_CONTENT_CONFLICT_META_FIELDS: [&str; 15] = [
|
||||
"err",
|
||||
@@ -59,6 +61,7 @@ const UPDATE_ARCHIVED_TRANSACTION_SQL: &str =
|
||||
const UPDATE_CANONICAL_TRANSACTION_PROJECTION_SQL: &str = "UPDATE ksp_raw_transactions SET slot = $2::TEXT::NUMERIC, block_time_unix_millis = $3, format_id = $4, format_version = $5, content_hash = $6, payload = $7, retention_state = 'full' WHERE signature = $1";
|
||||
const UPDATE_PURGED_TRANSACTION_SQL: &str = "UPDATE ksp_raw_transactions SET block_time_unix_millis = NULL, payload = NULL, retention_state = 'purged' WHERE signature = $1 AND retention_state = 'archived'";
|
||||
const UPDATE_REHYDRATED_TRANSACTION_VARIANT_SQL: &str = "UPDATE ksp_raw_transaction_variants SET block_time_unix_millis = $3, payload = $4, retention_state = 'full' WHERE transaction_signature = $1 AND variant_id = $2::TEXT::NUMERIC";
|
||||
const UPDATE_TRANSACTION_CONFLICT_SQL: &str = "UPDATE ksp_raw_transaction_conflicts SET status = 'open', revision = $2::TEXT::NUMERIC, canonical_variant_id = $3::TEXT::NUMERIC, incoming_variant_id = $4::TEXT::NUMERIC, latest_relation = $5, latest_reason_code = $6, updated_at_unix_millis = $7 WHERE transaction_signature = $1 AND revision = $8::TEXT::NUMERIC";
|
||||
const UPDATE_TRANSACTION_VARIANT_SELECTOR_SQL: &str = "UPDATE ksp_raw_transaction_canonical_selectors SET canonical_variant_id = $4::TEXT::NUMERIC, canonical_revision = $5::TEXT::NUMERIC, updated_at_unix_millis = $6 WHERE transaction_signature = $1 AND canonical_variant_id = $2::TEXT::NUMERIC AND canonical_revision = $3::TEXT::NUMERIC";
|
||||
const UPDATE_TRANSACTION_VARIANT_TO_FULL_SQL: &str = "UPDATE ksp_raw_transaction_variants SET payload = $8, retention_state = 'full' WHERE transaction_signature = $1 AND variant_id = $2::TEXT::NUMERIC AND slot = $3::TEXT::NUMERIC AND block_time_unix_millis IS NOT DISTINCT FROM $4 AND format_id = $5 AND format_version = $6 AND content_hash = $7 AND ((retention_state = 'archived' AND payload IS NULL) OR (retention_state = 'full' AND payload = $8))";
|
||||
|
||||
@@ -701,42 +704,86 @@ pub(crate) async fn persist_raw_transaction_acquisition(
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(error) => return std::result::Result::Err(error),
|
||||
};
|
||||
let (entity_outcome, observed_variant_id, canonical_promotion) = if inserted {
|
||||
(ksp_store_api::RawEntityWriteOutcome::Inserted, canonical_variant_id, std::option::Option::None)
|
||||
let (entity_outcome, observed_variant_id, transaction_variant_outcome, canonical_promotion, quarantined_conflict) = if inserted {
|
||||
(
|
||||
ksp_store_api::RawEntityWriteOutcome::Inserted,
|
||||
canonical_variant_id,
|
||||
ksp_store_api::RawTransactionVariantWriteOutcome::InsertedCanonical,
|
||||
std::option::Option::None,
|
||||
std::option::Option::None,
|
||||
)
|
||||
} else {
|
||||
let comparison_result = compare_existing_transaction(network, locked.clone(), &raw_transaction);
|
||||
let comparison = match comparison_result {
|
||||
let comparison = match compare_existing_transaction(network, locked.clone(), &raw_transaction) {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(error) => {
|
||||
if error.kind() == crate::PostgresBackendErrorKind::Conflict && error.phase() == "raw_acquisition_content_conflict" {
|
||||
log_raw_transaction_content_conflict_provenance(&sql_transaction, network, &observation).await;
|
||||
}
|
||||
return std::result::Result::Err(error);
|
||||
},
|
||||
std::result::Result::Err(error) => return std::result::Result::Err(error),
|
||||
};
|
||||
match comparison {
|
||||
ExistingTransactionMatch::Active => (ksp_store_api::RawEntityWriteOutcome::AlreadyPresent, canonical_variant_id, std::option::Option::None),
|
||||
ExistingTransactionMatch::Active => (
|
||||
ksp_store_api::RawEntityWriteOutcome::AlreadyPresent,
|
||||
canonical_variant_id,
|
||||
ksp_store_api::RawTransactionVariantWriteOutcome::ObservedExact,
|
||||
std::option::Option::None,
|
||||
std::option::Option::None,
|
||||
),
|
||||
ExistingTransactionMatch::ActiveCompatibleLessComplete => {
|
||||
let variant_result = persist_or_reuse_native_transaction_variant(&sql_transaction, &raw_transaction, linked_at).await;
|
||||
let variant_id = match variant_result {
|
||||
let variant_id = match persist_or_reuse_native_transaction_variant(&sql_transaction, &raw_transaction, linked_at).await {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(error) => return std::result::Result::Err(error),
|
||||
};
|
||||
(ksp_store_api::RawEntityWriteOutcome::AlreadyPresent, variant_id, std::option::Option::None)
|
||||
(
|
||||
ksp_store_api::RawEntityWriteOutcome::AlreadyPresent,
|
||||
variant_id,
|
||||
ksp_store_api::RawTransactionVariantWriteOutcome::ObservedCompatibleLessComplete,
|
||||
std::option::Option::None,
|
||||
std::option::Option::None,
|
||||
)
|
||||
},
|
||||
ExistingTransactionMatch::ActiveCompatibleMoreComplete => {
|
||||
let variant_result = persist_or_reuse_native_transaction_variant(&sql_transaction, &raw_transaction, linked_at).await;
|
||||
let variant_id = match variant_result {
|
||||
let variant_id = match persist_or_reuse_native_transaction_variant(&sql_transaction, &raw_transaction, linked_at).await {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(error) => return std::result::Result::Err(error),
|
||||
};
|
||||
let promotion_result =
|
||||
promote_more_complete_transaction_variant(&sql_transaction, &locked, &raw_transaction, canonical_variant_id, variant_id, linked_at).await;
|
||||
let promotion = match promotion_result {
|
||||
let promotion =
|
||||
match promote_more_complete_transaction_variant(&sql_transaction, &locked, &raw_transaction, canonical_variant_id, variant_id, linked_at)
|
||||
.await
|
||||
{
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(error) => return std::result::Result::Err(error),
|
||||
};
|
||||
(
|
||||
ksp_store_api::RawEntityWriteOutcome::AlreadyPresent,
|
||||
variant_id,
|
||||
ksp_store_api::RawTransactionVariantWriteOutcome::PromotedCompatibleMoreComplete,
|
||||
std::option::Option::Some(promotion),
|
||||
std::option::Option::None,
|
||||
)
|
||||
},
|
||||
ExistingTransactionMatch::ActiveConflict(variant_comparison) => {
|
||||
let variant_id = match persist_or_reuse_native_transaction_variant(&sql_transaction, &raw_transaction, linked_at).await {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(error) => return std::result::Result::Err(error),
|
||||
};
|
||||
(ksp_store_api::RawEntityWriteOutcome::AlreadyPresent, variant_id, std::option::Option::Some(promotion))
|
||||
let conflict_transition = match open_or_update_raw_transaction_conflict(
|
||||
&sql_transaction,
|
||||
raw_transaction.reference(),
|
||||
canonical_variant_id,
|
||||
variant_id,
|
||||
variant_comparison,
|
||||
linked_at,
|
||||
)
|
||||
.await
|
||||
{
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(error) => return std::result::Result::Err(error),
|
||||
};
|
||||
log_raw_transaction_content_conflict_provenance(&sql_transaction, network, &observation).await;
|
||||
(
|
||||
ksp_store_api::RawEntityWriteOutcome::AlreadyPresent,
|
||||
variant_id,
|
||||
ksp_store_api::RawTransactionVariantWriteOutcome::QuarantinedConflict,
|
||||
std::option::Option::None,
|
||||
std::option::Option::Some(conflict_transition),
|
||||
)
|
||||
},
|
||||
ExistingTransactionMatch::Purged => {
|
||||
if mode == ksp_store_api::RawTransactionAcquisitionMode::ForceRehydrate {
|
||||
@@ -748,15 +795,22 @@ pub(crate) async fn persist_raw_transaction_acquisition(
|
||||
if let std::result::Result::Err(error) = variant_result {
|
||||
return std::result::Result::Err(error);
|
||||
}
|
||||
(ksp_store_api::RawEntityWriteOutcome::Rehydrated, canonical_variant_id, std::option::Option::None)
|
||||
(
|
||||
ksp_store_api::RawEntityWriteOutcome::Rehydrated,
|
||||
canonical_variant_id,
|
||||
ksp_store_api::RawTransactionVariantWriteOutcome::Rehydrated,
|
||||
std::option::Option::None,
|
||||
std::option::Option::None,
|
||||
)
|
||||
} else {
|
||||
let commit_result = sql_transaction.commit().await;
|
||||
if commit_result.is_err() {
|
||||
return std::result::Result::Err(write_failed("raw_acquisition_commit"));
|
||||
}
|
||||
return std::result::Result::Ok(ksp_store_api::RawAcquisitionWriteOutcome::new(
|
||||
return std::result::Result::Ok(ksp_store_api::RawAcquisitionWriteOutcome::with_transaction_variant(
|
||||
ksp_store_api::RawEntityWriteOutcome::SkippedPurged,
|
||||
ksp_store_api::RawObservationWriteOutcome::NotRecorded,
|
||||
ksp_store_api::RawTransactionVariantWriteOutcome::SkippedPurged,
|
||||
));
|
||||
}
|
||||
},
|
||||
@@ -778,7 +832,18 @@ pub(crate) async fn persist_raw_transaction_acquisition(
|
||||
if let std::option::Option::Some(transition) = canonical_promotion {
|
||||
log_raw_transaction_canonical_promotion_committed(network, raw_transaction.slot(), transition);
|
||||
}
|
||||
return std::result::Result::Ok(ksp_store_api::RawAcquisitionWriteOutcome::new(entity_outcome, observation_outcome));
|
||||
if let std::option::Option::Some(transition) = quarantined_conflict {
|
||||
let stored = decode_raw_transaction_row(network, locked.clone());
|
||||
if let std::result::Result::Ok(std::option::Option::Some(stored)) = stored {
|
||||
log_raw_transaction_content_conflict(network, &stored, &raw_transaction);
|
||||
}
|
||||
log_raw_transaction_conflict_quarantined_committed(network, raw_transaction.slot(), transition);
|
||||
}
|
||||
return std::result::Result::Ok(ksp_store_api::RawAcquisitionWriteOutcome::with_transaction_variant(
|
||||
entity_outcome,
|
||||
observation_outcome,
|
||||
transaction_variant_outcome,
|
||||
));
|
||||
}
|
||||
|
||||
/// Persists one additional observation for an already known RAW transaction.
|
||||
@@ -922,11 +987,21 @@ struct RawCanonicalPromotionTransition {
|
||||
to_variant_id: u64,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
struct RawTransactionConflictTransition {
|
||||
canonical_variant_id: u64,
|
||||
incoming_variant_id: u64,
|
||||
reason: ksp_store_api::RawTransactionVariantRelationReason,
|
||||
relation: ksp_store_api::RawTransactionVariantRelation,
|
||||
revision: u64,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
enum ExistingTransactionMatch {
|
||||
Active,
|
||||
ActiveCompatibleLessComplete,
|
||||
ActiveCompatibleMoreComplete,
|
||||
ActiveConflict(ksp_store_api::RawTransactionVariantComparison),
|
||||
Purged,
|
||||
}
|
||||
|
||||
@@ -1259,6 +1334,155 @@ async fn persist_or_reuse_native_transaction_variant(
|
||||
};
|
||||
}
|
||||
|
||||
async fn open_or_update_raw_transaction_conflict(
|
||||
sql_transaction: &deadpool_postgres::Transaction<'_>,
|
||||
reference: &ksp_store_api::RawTransactionReference,
|
||||
canonical_variant_id: u64,
|
||||
incoming_variant_id: u64,
|
||||
comparison: ksp_store_api::RawTransactionVariantComparison,
|
||||
observed_at_unix_millis: i64,
|
||||
) -> std::result::Result<RawTransactionConflictTransition, crate::PostgresBackendError> {
|
||||
if canonical_variant_id == incoming_variant_id {
|
||||
return std::result::Result::Err(data_invalid("raw_conflict_variant_identity"));
|
||||
}
|
||||
if !matches!(comparison.relation(), ksp_store_api::RawTransactionVariantRelation::Conflict | ksp_store_api::RawTransactionVariantRelation::Incomparable) {
|
||||
return std::result::Result::Err(data_invalid("raw_conflict_relation"));
|
||||
}
|
||||
let signature = reference.signature();
|
||||
let signature_bytes: &[u8] = signature.as_bytes();
|
||||
let relation_code = comparison.relation().code();
|
||||
let reason_code = comparison.reason().code();
|
||||
let row_result = sql_transaction.query_opt(LOCK_TRANSACTION_CONFLICT_SQL, &[&signature_bytes]).await;
|
||||
let row = match row_result {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(_) => return std::result::Result::Err(write_failed("raw_conflict_lock")),
|
||||
};
|
||||
let canonical_variant_id_text = canonical_variant_id.to_string();
|
||||
let incoming_variant_id_text = incoming_variant_id.to_string();
|
||||
if row.is_none() {
|
||||
let insert_result = sql_transaction
|
||||
.execute(
|
||||
INSERT_TRANSACTION_CONFLICT_SQL,
|
||||
&[
|
||||
&signature_bytes,
|
||||
&canonical_variant_id_text.as_str(),
|
||||
&incoming_variant_id_text.as_str(),
|
||||
&relation_code,
|
||||
&reason_code,
|
||||
&observed_at_unix_millis,
|
||||
],
|
||||
)
|
||||
.await;
|
||||
return match insert_result {
|
||||
std::result::Result::Ok(1) => std::result::Result::Ok(RawTransactionConflictTransition {
|
||||
canonical_variant_id,
|
||||
incoming_variant_id,
|
||||
reason: comparison.reason(),
|
||||
relation: comparison.relation(),
|
||||
revision: 1,
|
||||
}),
|
||||
std::result::Result::Ok(_) => std::result::Result::Err(data_invalid("raw_conflict_insert_count")),
|
||||
std::result::Result::Err(_) => std::result::Result::Err(write_failed("raw_conflict_insert")),
|
||||
};
|
||||
}
|
||||
let row = match row {
|
||||
std::option::Option::Some(value) => value,
|
||||
std::option::Option::None => return std::result::Result::Err(data_invalid("raw_conflict_lock_state")),
|
||||
};
|
||||
let status = match row.try_get::<_, std::string::String>("status") {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(_) => return std::result::Result::Err(data_invalid("raw_conflict_status")),
|
||||
};
|
||||
if status != ksp_store_api::RawTransactionConflictStatus::Open.code() && status != ksp_store_api::RawTransactionConflictStatus::Resolved.code() {
|
||||
return std::result::Result::Err(data_invalid("raw_conflict_status"));
|
||||
}
|
||||
let revision_text = match row.try_get::<_, std::string::String>("revision_text") {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(_) => return std::result::Result::Err(data_invalid("raw_conflict_revision")),
|
||||
};
|
||||
let revision = match decode_u64_decimal(revision_text.as_str(), "raw_conflict_revision") {
|
||||
std::result::Result::Ok(value) if value > 0 => value,
|
||||
std::result::Result::Ok(_) => return std::result::Result::Err(data_invalid("raw_conflict_revision")),
|
||||
std::result::Result::Err(error) => return std::result::Result::Err(error),
|
||||
};
|
||||
let existing_canonical_text = match row.try_get::<_, std::string::String>("canonical_variant_id_text") {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(_) => return std::result::Result::Err(data_invalid("raw_conflict_canonical_variant_id")),
|
||||
};
|
||||
let existing_canonical = match decode_u64_decimal(existing_canonical_text.as_str(), "raw_conflict_canonical_variant_id") {
|
||||
std::result::Result::Ok(value) if value > 0 => value,
|
||||
std::result::Result::Ok(_) => return std::result::Result::Err(data_invalid("raw_conflict_canonical_variant_id")),
|
||||
std::result::Result::Err(error) => return std::result::Result::Err(error),
|
||||
};
|
||||
let existing_incoming_text = match row.try_get::<_, std::string::String>("incoming_variant_id_text") {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(_) => return std::result::Result::Err(data_invalid("raw_conflict_incoming_variant_id")),
|
||||
};
|
||||
let existing_incoming = match decode_u64_decimal(existing_incoming_text.as_str(), "raw_conflict_incoming_variant_id") {
|
||||
std::result::Result::Ok(value) if value > 0 => value,
|
||||
std::result::Result::Ok(_) => return std::result::Result::Err(data_invalid("raw_conflict_incoming_variant_id")),
|
||||
std::result::Result::Err(error) => return std::result::Result::Err(error),
|
||||
};
|
||||
let existing_relation = match row.try_get::<_, std::string::String>("latest_relation") {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(_) => return std::result::Result::Err(data_invalid("raw_conflict_relation")),
|
||||
};
|
||||
let existing_reason = match row.try_get::<_, std::string::String>("latest_reason_code") {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(_) => return std::result::Result::Err(data_invalid("raw_conflict_reason")),
|
||||
};
|
||||
if status == ksp_store_api::RawTransactionConflictStatus::Open.code()
|
||||
&& existing_canonical == canonical_variant_id
|
||||
&& existing_incoming == incoming_variant_id
|
||||
&& existing_relation == relation_code
|
||||
&& existing_reason == reason_code
|
||||
{
|
||||
return std::result::Result::Ok(RawTransactionConflictTransition {
|
||||
canonical_variant_id,
|
||||
incoming_variant_id,
|
||||
reason: comparison.reason(),
|
||||
relation: comparison.relation(),
|
||||
revision,
|
||||
});
|
||||
}
|
||||
let updated_at = match row.try_get::<_, i64>("updated_at_unix_millis") {
|
||||
std::result::Result::Ok(value) => std::cmp::max(value, observed_at_unix_millis),
|
||||
std::result::Result::Err(_) => return std::result::Result::Err(data_invalid("raw_conflict_updated_at")),
|
||||
};
|
||||
let next_revision = match revision.checked_add(1) {
|
||||
std::option::Option::Some(value) if value != 0 => value,
|
||||
_ => return std::result::Result::Err(data_invalid("raw_conflict_revision_exhausted")),
|
||||
};
|
||||
let next_revision_text = next_revision.to_string();
|
||||
let update_result = sql_transaction
|
||||
.execute(
|
||||
UPDATE_TRANSACTION_CONFLICT_SQL,
|
||||
&[
|
||||
&signature_bytes,
|
||||
&next_revision_text.as_str(),
|
||||
&canonical_variant_id_text.as_str(),
|
||||
&incoming_variant_id_text.as_str(),
|
||||
&relation_code,
|
||||
&reason_code,
|
||||
&updated_at,
|
||||
&revision_text.as_str(),
|
||||
],
|
||||
)
|
||||
.await;
|
||||
return match update_result {
|
||||
std::result::Result::Ok(1) => std::result::Result::Ok(RawTransactionConflictTransition {
|
||||
canonical_variant_id,
|
||||
incoming_variant_id,
|
||||
reason: comparison.reason(),
|
||||
relation: comparison.relation(),
|
||||
revision: next_revision,
|
||||
}),
|
||||
std::result::Result::Ok(0) => std::result::Result::Err(data_invalid("raw_conflict_stale_revision")),
|
||||
std::result::Result::Ok(_) => std::result::Result::Err(data_invalid("raw_conflict_update_count")),
|
||||
std::result::Result::Err(_) => std::result::Result::Err(write_failed("raw_conflict_update")),
|
||||
};
|
||||
}
|
||||
|
||||
async fn promote_more_complete_transaction_variant(
|
||||
sql_transaction: &deadpool_postgres::Transaction<'_>,
|
||||
locked: &RawTransactionDbRow,
|
||||
@@ -1657,8 +1881,7 @@ fn compare_existing_transaction(
|
||||
},
|
||||
ksp_store_api::RawTransactionVariantRelation::CompatibleMoreComplete => std::result::Result::Ok(ExistingTransactionMatch::ActiveCompatibleMoreComplete),
|
||||
ksp_store_api::RawTransactionVariantRelation::Conflict | ksp_store_api::RawTransactionVariantRelation::Incomparable => {
|
||||
log_raw_transaction_content_conflict(network, &stored, incoming);
|
||||
std::result::Result::Err(conflict("raw_acquisition_content_conflict"))
|
||||
std::result::Result::Ok(ExistingTransactionMatch::ActiveConflict(comparison))
|
||||
},
|
||||
_ => std::result::Result::Err(data_invalid("raw_acquisition_variant_relation")),
|
||||
};
|
||||
@@ -1925,6 +2148,22 @@ fn log_raw_transaction_canonical_promotion_committed(network: &ksp_store_api::Ra
|
||||
return;
|
||||
}
|
||||
|
||||
fn log_raw_transaction_conflict_quarantined_committed(network: &ksp_store_api::RawNetworkId, slot: u64, transition: RawTransactionConflictTransition) {
|
||||
ksp_logging_lib::warn!(
|
||||
target: crate::TRACING_TARGET,
|
||||
domain = "store.raw_transaction.quarantined_conflict",
|
||||
network = network.as_str(),
|
||||
slot = slot,
|
||||
canonical_variant_id = transition.canonical_variant_id,
|
||||
incoming_variant_id = transition.incoming_variant_id,
|
||||
conflict_revision = transition.revision,
|
||||
relation = transition.relation.code(),
|
||||
reason = transition.reason.code(),
|
||||
"PostgreSQL Store committed a durable RAW transaction conflict case"
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
fn log_raw_transaction_content_conflict(
|
||||
network: &ksp_store_api::RawNetworkId,
|
||||
stored: &ksp_store_api::RawTransaction,
|
||||
@@ -1964,7 +2203,7 @@ fn log_raw_transaction_content_conflict(
|
||||
incoming_log_messages_prefix_of_stored = diagnostic.log_messages.incoming_prefix_of_stored,
|
||||
stored_log_messages_has_truncation_marker = diagnostic.log_messages.stored_has_truncation_marker,
|
||||
incoming_log_messages_has_truncation_marker = diagnostic.log_messages.incoming_has_truncation_marker,
|
||||
"PostgreSQL Store rejected divergent canonical RAW transaction"
|
||||
"PostgreSQL Store classified divergent canonical RAW transaction content"
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
// file: crates/ksp-store-postgres-lib/tests/hardening_completeness.rs
|
||||
// version: 28
|
||||
// version: 29
|
||||
|
||||
#![warn(missing_docs)]
|
||||
#![deny(unreachable_pub)]
|
||||
@@ -645,7 +645,7 @@ fn v0_3_15_pre_014_content_conflict_diagnostic_exposes_only_block_slot_and_bound
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn v0_3_16_pre_005_postgres_uses_shared_fail_closed_variant_comparator_without_promotion() {
|
||||
fn v0_3_16_pre_007_postgres_uses_shared_fail_closed_variant_comparator_with_durable_conflict() {
|
||||
let source = include_str!("../src/raw_transaction.rs");
|
||||
for required in [
|
||||
"ksp_store_api::compare_raw_transaction_variants",
|
||||
@@ -655,7 +655,8 @@ fn v0_3_16_pre_005_postgres_uses_shared_fail_closed_variant_comparator_without_p
|
||||
"RawTransactionVariantRelation::Conflict",
|
||||
"RawTransactionVariantRelation::Incomparable",
|
||||
"ExistingTransactionMatch::ActiveCompatibleLessComplete",
|
||||
"raw_acquisition_content_conflict",
|
||||
"ExistingTransactionMatch::ActiveConflict",
|
||||
"RawTransactionVariantWriteOutcome::QuarantinedConflict",
|
||||
] {
|
||||
assert!(source.contains(required), "missing shared comparator integration guard: {required}");
|
||||
}
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
// file: crates/ksp-store-postgres-lib/tests/v003_variant_persistence.rs
|
||||
// version: 3
|
||||
// version: 4
|
||||
|
||||
#![warn(missing_docs)]
|
||||
#![deny(unreachable_pub)]
|
||||
@@ -59,7 +59,7 @@ fn v0_3_16_pre_004_legacy_observation_without_mapping_is_not_fabricated() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn v0_3_16_pre_006_shared_comparator_promotes_only_more_complete_and_keeps_conflict_deferred() {
|
||||
fn v0_3_16_pre_007_shared_comparator_promotes_more_complete_and_quarantines_conflict() {
|
||||
let source = include_str!("../src/raw_transaction.rs");
|
||||
assert!(source.contains("ksp_store_api::compare_raw_transaction_variants"));
|
||||
assert!(source.contains("ExistingTransactionMatch::ActiveCompatibleLessComplete"));
|
||||
@@ -70,7 +70,44 @@ fn v0_3_16_pre_006_shared_comparator_promotes_only_more_complete_and_keeps_confl
|
||||
assert!(source.contains("UPDATE ksp_raw_transactions SET slot = $2::TEXT::NUMERIC"));
|
||||
assert!(source.contains("PostgreSQL Store committed a proved RAW transaction canonical promotion"));
|
||||
assert!(!source.contains("log_raw_transaction_compatible_more_complete_log_messages"));
|
||||
assert!(!source.contains("INSERT INTO ksp_raw_transaction_conflicts"));
|
||||
assert!(source.contains("INSERT INTO ksp_raw_transaction_conflicts"));
|
||||
assert!(source.contains("ExistingTransactionMatch::ActiveConflict"));
|
||||
assert!(source.contains("RawTransactionVariantWriteOutcome::QuarantinedConflict"));
|
||||
return;
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn v0_3_16_pre_007_conflict_is_persisted_inside_acquisition_transaction_before_observation_and_commit() {
|
||||
let source = include_str!("../src/raw_transaction.rs");
|
||||
let start = source.find("pub(crate) async fn persist_raw_transaction_acquisition(");
|
||||
let end = source.find("pub(crate) async fn record_raw_transaction_observation(");
|
||||
let (start, end) = match (start, end) {
|
||||
(std::option::Option::Some(start), std::option::Option::Some(end)) if start < end => (start, end),
|
||||
_ => panic!("pre.007 acquisition function markers are missing"),
|
||||
};
|
||||
let body = &source[start..end];
|
||||
let persist_variant = body.find("persist_or_reuse_native_transaction_variant");
|
||||
let conflict = body.find("open_or_update_raw_transaction_conflict");
|
||||
let observation = body.find("persist_observation_row");
|
||||
let commit = body.rfind("sql_transaction.commit().await");
|
||||
let transition_log = body.find("log_raw_transaction_conflict_quarantined_committed");
|
||||
assert!(matches!(
|
||||
(persist_variant, conflict, observation, commit, transition_log),
|
||||
(Some(persist_variant), Some(conflict), Some(observation), Some(commit), Some(transition_log))
|
||||
if persist_variant < conflict && conflict < observation && observation < commit && commit < transition_log
|
||||
));
|
||||
assert!(body.contains("RawTransactionVariantWriteOutcome::QuarantinedConflict"));
|
||||
let conflict_start = source.find("async fn open_or_update_raw_transaction_conflict(");
|
||||
let conflict_end = source.find("async fn persist_observation_row(");
|
||||
let (conflict_start, conflict_end) = match (conflict_start, conflict_end) {
|
||||
(std::option::Option::Some(start), std::option::Option::Some(end)) if start < end => (start, end),
|
||||
_ => panic!("pre.007 conflict helper markers are missing"),
|
||||
};
|
||||
let conflict_helper = &source[conflict_start..conflict_end];
|
||||
assert!(conflict_helper.contains("LOCK_TRANSACTION_CONFLICT_SQL"));
|
||||
assert!(conflict_helper.contains("INSERT_TRANSACTION_CONFLICT_SQL"));
|
||||
assert!(conflict_helper.contains("UPDATE_TRANSACTION_CONFLICT_SQL"));
|
||||
assert!(conflict_helper.contains("checked_add(1)"));
|
||||
return;
|
||||
}
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
// file: crates/ksp-store-postgres-lib/unit_tests/raw_transaction.rs
|
||||
// version: 13
|
||||
// version: 14
|
||||
|
||||
fn network() -> ksp_store_api::RawNetworkId {
|
||||
return match ksp_store_api::RawNetworkId::new("devnet") {
|
||||
@@ -900,3 +900,16 @@ fn v0_3_16_pre_006_promotion_sql_is_compare_guarded_and_projection_complete() {
|
||||
assert!(!super::UPDATE_TRANSACTION_VARIANT_TO_FULL_SQL.contains("DELETE"));
|
||||
return;
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn v0_3_16_pre_007_conflict_sql_is_locked_revision_guarded_and_reopenable() {
|
||||
assert!(super::LOCK_TRANSACTION_CONFLICT_SQL.contains("FOR UPDATE"));
|
||||
assert!(super::INSERT_TRANSACTION_CONFLICT_SQL.contains("status, revision"));
|
||||
assert!(super::INSERT_TRANSACTION_CONFLICT_SQL.contains("'open', 1"));
|
||||
assert!(super::UPDATE_TRANSACTION_CONFLICT_SQL.contains("status = 'open'"));
|
||||
assert!(super::UPDATE_TRANSACTION_CONFLICT_SQL.contains("revision = $2::TEXT::NUMERIC"));
|
||||
assert!(super::UPDATE_TRANSACTION_CONFLICT_SQL.contains("AND revision = $8::TEXT::NUMERIC"));
|
||||
assert!(super::UPDATE_TRANSACTION_CONFLICT_SQL.contains("canonical_variant_id = $3::TEXT::NUMERIC"));
|
||||
assert!(super::UPDATE_TRANSACTION_CONFLICT_SQL.contains("incoming_variant_id = $4::TEXT::NUMERIC"));
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
<!-- file: crates/ksp-worker-raw-transaction-ingest-lib/README.md -->
|
||||
<!-- version: 17 -->
|
||||
<!-- version: 18 -->
|
||||
|
||||
# ksp-worker-raw-transaction-ingest-lib
|
||||
|
||||
@@ -187,7 +187,7 @@ source hydratante active => quota pending >= 1 et quota in-flight >= 1
|
||||
|
||||
Pour garantir simultanément ces bornes et l'absence de starvation structurelle, le démarrage échoue avant spawn si le nombre de sources hydratantes dépasse `admission_queue_capacity` ou `persistence_concurrency`. Le registre des hydrations transactionnelles protège en plus l'ouverture effective des `getTransaction`; Yellowstone Block reste borné par sa partition déterministe et son `JoinSet` possédé.
|
||||
|
||||
Les signaux partageant le même `(network, signature, commitment)` sont coalescés cross-source avant le fan-out HTTP. La publication du résultat partagé notifie les followers sous le verrou de registry avant de retirer la clé : une nouvelle génération de leader ne peut donc pas s'intercaler entre retrait et notification. Après canonicalisation, une cache run-local bornée sérialise les acquisitions de même `(network, signature)` : la première passe par l'écriture atomique entity + observation, les suivantes de contenu canonique identique ajoutent uniquement leur observation déterministe. Le Store conserve le guard durable final. En `0.3.15`, il accepte aussi un entrant strictement moins complet lorsque l'unique divergence prouvée est `meta.logMessages` avec exactement un marqueur `Log truncated` après un préfixe identique alors que le canonique stocké n'est pas tronqué ; le canonique complet reste inchangé et l'observation est conservée. Toute autre divergence, ainsi que le sens tronqué -> complet, reste un `content_conflict` terminal. Aucune majorité, préférence provider ou overwrite n'est appliqué.
|
||||
Les signaux partageant le même `(network, signature, commitment)` sont coalescés cross-source avant le fan-out HTTP. La publication du résultat partagé notifie les followers sous le verrou de registry avant de retirer la clé : une nouvelle génération de leader ne peut donc pas s'intercaler entre retrait et notification. Après canonicalisation, une cache run-local bornée sérialise uniquement les acquisitions de même `(network, signature)` ; elle ne décide jamais du contenu. Chaque acquisition repasse par l'écriture atomique du Store, qui reste l'autorité durable finale. Le comparateur partagé accepte un entrant strictement moins complet lorsque l'unique divergence prouvée est `meta.logMessages` tronqué, promeut atomiquement un entrant prouvé plus complet, et conserve toute divergence conflictuelle/incomparable comme variante durable avec un conflict case ouvert. Aucune majorité, préférence provider ou overwrite n'est appliqué.
|
||||
|
||||
Les retries/reroutages HTTP appartiennent à `ksp-onchain-transport-lib`. Le Worker ne possède pas une seconde boucle de retry autour de `getTransaction`.
|
||||
|
||||
@@ -283,11 +283,13 @@ entity skipped purged
|
||||
observation inserted
|
||||
observation already present
|
||||
observation not recorded for purged entity
|
||||
content conflict
|
||||
variant inserted canonical / exact / compatible less complete
|
||||
variant promoted compatible more complete
|
||||
variant quarantined conflict
|
||||
store failure
|
||||
```
|
||||
|
||||
Un content conflict réel reste terminal en `0.3.15` et n'est jamais converti en succès idempotent. L'exception `logMessages` tronqués décrite ci-dessus est classée avant le conflit comme observation compatible moins complète ; elle ne remplace pas le canonique et n'incrémente pas le terminal `content_conflict`. La conservation de variantes conflictuelles et leur résolution sans arrêt du Worker appartiennent à `0.3.16`.
|
||||
En `0.3.16-pre.007`, un conflit de contenu classifié par le Store n'est plus une faute terminale : la variante entrante est conservée, le conflict case durable est ouvert ou mis à jour idempotemment et l'outcome `QuarantinedConflict` maintient le Worker `Running` avec une health `Degraded`. Un ancien backend qui renvoie encore `ERROR_CODE_RAW_CONFLICT` reste traité comme erreur terminale de compatibilité. La résolution explicite du conflict case n'appartient pas à cette tranche.
|
||||
|
||||
## Snapshots et erreurs
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
<!-- file: crates/ksp-worker-raw-transaction-ingest-lib/USAGE.md -->
|
||||
<!-- version: 15 -->
|
||||
<!-- version: 16 -->
|
||||
|
||||
# Utilisation de ksp-worker-raw-transaction-ingest-lib
|
||||
|
||||
@@ -266,7 +266,7 @@ somme de leurs tâches d'hydration actives <= persistence_concurrency
|
||||
|
||||
Ces contraintes garantissent au moins une part à chaque source reference-bearing sans introduire de scheduler pondéré. Si les settings ne permettent pas cette répartition, le démarrage échoue avant spawn avec `runtime_invalid`; le caller doit augmenter la capacité concernée ou réduire le nombre de sources nécessitant une hydration.
|
||||
|
||||
Pour une même identité `(network, signature)`, le Worker ne choisit ni majorité ni provider préféré et n'écrase pas un contenu divergent ; le Store reste l'autorité durable finale. En `0.3.15`, un canonique complet peut accepter comme observation supplémentaire un entrant identique hors `meta.logMessages` lorsque celui-ci contient exactement un marqueur `Log truncated` après un préfixe identique. Le canonique n'est pas réécrit. Le sens tronqué -> complet et toute autre divergence non prouvée restent des `content_conflict` terminaux ; leur stockage en variantes et leur résolution sont réservés à `0.3.16`.
|
||||
Pour une même identité `(network, signature)`, le Worker ne choisit ni majorité ni provider préféré et n'écrase pas un contenu divergent ; le Store reste l'autorité durable finale. La cache run-local ne fait que sérialiser cette identité et chaque acquisition atteint le Store. Un entrant prouvé moins complet reste une observation compatible, un entrant prouvé plus complet peut devenir canonique atomiquement, et une divergence conflictuelle/incomparable est conservée comme variante avec un conflict case durable. L'outcome `QuarantinedConflict` n'arrête pas le Worker : il reste `Running` et sa health devient `Degraded`.
|
||||
|
||||
## Observer le snapshot concret
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
// file: crates/ksp-worker-raw-transaction-ingest-lib/src/persistence.rs
|
||||
// version: 4
|
||||
// version: 5
|
||||
|
||||
/// Canonical entity disposition produced by one Worker Store persistence attempt.
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
@@ -28,6 +28,7 @@ pub(crate) enum RawTransactionIngestObservationPersistence {
|
||||
pub(crate) struct RawTransactionIngestPersistenceOutcome {
|
||||
entity: crate::RawTransactionIngestEntityPersistence,
|
||||
observation: crate::RawTransactionIngestObservationPersistence,
|
||||
quarantined_conflict: bool,
|
||||
}
|
||||
|
||||
impl crate::RawTransactionIngestPersistenceOutcome {
|
||||
@@ -42,6 +43,12 @@ impl crate::RawTransactionIngestPersistenceOutcome {
|
||||
pub(crate) const fn observation(self) -> crate::RawTransactionIngestObservationPersistence {
|
||||
return self.observation;
|
||||
}
|
||||
|
||||
/// Reports whether Store durably quarantined divergent or incomparable content without failing the acquisition.
|
||||
#[must_use]
|
||||
pub(crate) const fn quarantined_conflict(self) -> bool {
|
||||
return self.quarantined_conflict;
|
||||
}
|
||||
}
|
||||
|
||||
/// Private backend-neutral Store persistence port used by the Worker and deterministic tests.
|
||||
@@ -56,12 +63,6 @@ pub(crate) trait RawTransactionIngestPersistencePort: std::marker::Send + std::m
|
||||
observation: ksp_store_lib::RawTransactionObservation,
|
||||
mode: ksp_store_lib::RawTransactionAcquisitionMode,
|
||||
) -> ksp_store_lib::StoreApiFuture<'a, ksp_store_lib::Result<ksp_store_lib::RawAcquisitionWriteOutcome>>;
|
||||
|
||||
/// Records one additional observation for an already durable canonical RAW transaction.
|
||||
fn record_observation<'a>(
|
||||
&'a self,
|
||||
observation: ksp_store_lib::RawTransactionObservation,
|
||||
) -> ksp_store_lib::StoreApiFuture<'a, ksp_store_lib::Result<ksp_store_lib::RawObservationWriteOutcome>>;
|
||||
}
|
||||
|
||||
impl crate::RawTransactionIngestPersistencePort for ksp_store_lib::Store {
|
||||
@@ -77,13 +78,6 @@ impl crate::RawTransactionIngestPersistencePort for ksp_store_lib::Store {
|
||||
) -> ksp_store_lib::StoreApiFuture<'a, ksp_store_lib::Result<ksp_store_lib::RawAcquisitionWriteOutcome>> {
|
||||
return ksp_store_lib::RawTransactionWrite::persist_raw_transaction_acquisition(self, transaction, observation, mode);
|
||||
}
|
||||
|
||||
fn record_observation<'a>(
|
||||
&'a self,
|
||||
observation: ksp_store_lib::RawTransactionObservation,
|
||||
) -> ksp_store_lib::StoreApiFuture<'a, ksp_store_lib::Result<ksp_store_lib::RawObservationWriteOutcome>> {
|
||||
return ksp_store_lib::RawTransactionObservationWrite::record_raw_transaction_observation(self, observation);
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Eq, Ord, PartialEq, PartialOrd)]
|
||||
@@ -92,24 +86,10 @@ struct RawTransactionIngestPersistenceKey {
|
||||
signature: ksp_store_lib::RawTransactionSignature,
|
||||
}
|
||||
|
||||
#[derive(Clone, Eq, PartialEq)]
|
||||
struct RawTransactionIngestCanonicalState {
|
||||
block_time: std::option::Option<ksp_store_lib::RawTimestamp>,
|
||||
content_hash: ksp_store_lib::RawContentHash,
|
||||
format_id: ksp_store_lib::RawFormatId,
|
||||
format_version: u32,
|
||||
slot: u64,
|
||||
}
|
||||
|
||||
/// Private bounded run-local cache serializing repeated canonical identities before Store writes.
|
||||
pub(crate) struct RawTransactionIngestPersistenceConvergence {
|
||||
max_entries: usize,
|
||||
entries: std::sync::Mutex<
|
||||
std::collections::BTreeMap<
|
||||
RawTransactionIngestPersistenceKey,
|
||||
std::sync::Arc<tokio::sync::Mutex<std::option::Option<RawTransactionIngestCanonicalState>>>,
|
||||
>,
|
||||
>,
|
||||
entries: std::sync::Mutex<std::collections::BTreeMap<RawTransactionIngestPersistenceKey, std::sync::Arc<tokio::sync::Mutex<()>>>>,
|
||||
}
|
||||
|
||||
impl crate::RawTransactionIngestPersistenceConvergence {
|
||||
@@ -119,10 +99,7 @@ impl crate::RawTransactionIngestPersistenceConvergence {
|
||||
return Self { max_entries, entries: std::sync::Mutex::new(std::collections::BTreeMap::new()) };
|
||||
}
|
||||
|
||||
fn entry(
|
||||
&self,
|
||||
key: RawTransactionIngestPersistenceKey,
|
||||
) -> std::option::Option<std::sync::Arc<tokio::sync::Mutex<std::option::Option<RawTransactionIngestCanonicalState>>>> {
|
||||
fn entry(&self, key: RawTransactionIngestPersistenceKey) -> std::option::Option<std::sync::Arc<tokio::sync::Mutex<()>>> {
|
||||
let mut entries = match self.entries.lock() {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(poisoned) => poisoned.into_inner(),
|
||||
@@ -144,7 +121,7 @@ impl crate::RawTransactionIngestPersistenceConvergence {
|
||||
if entries.len() >= self.max_entries {
|
||||
return std::option::Option::None;
|
||||
}
|
||||
let entry = std::sync::Arc::new(tokio::sync::Mutex::new(std::option::Option::None));
|
||||
let entry = std::sync::Arc::new(tokio::sync::Mutex::new(()));
|
||||
entries.insert(key, std::sync::Arc::clone(&entry));
|
||||
return std::option::Option::Some(entry);
|
||||
}
|
||||
@@ -189,44 +166,14 @@ where
|
||||
if acquisition.observation().transaction() != &reference {
|
||||
return std::result::Result::Err(crate::runtime_error("persistence.acquisition_reference_mismatch"));
|
||||
}
|
||||
let canonical_state = canonical_state(acquisition.transaction());
|
||||
let key = RawTransactionIngestPersistenceKey { network: reference.network().clone(), signature: reference.signature() };
|
||||
let entry = convergence.entry(key);
|
||||
let entry = match entry {
|
||||
std::option::Option::Some(value) => value,
|
||||
std::option::Option::None => return crate::persist_raw_transaction_ingest_acquisition(port, acquisition).await,
|
||||
};
|
||||
let mut known_state = entry.lock().await;
|
||||
if let std::option::Option::Some(known_state_value) = known_state.as_ref() {
|
||||
if known_state_value != &canonical_state {
|
||||
return std::result::Result::Err(crate::content_conflict_error());
|
||||
}
|
||||
let (_transaction, observation) = acquisition.into_parts();
|
||||
let result = port.record_observation(observation).await;
|
||||
return match result {
|
||||
std::result::Result::Ok(outcome) => map_additional_observation_outcome(outcome),
|
||||
std::result::Result::Err(error) => map_store_error(error),
|
||||
};
|
||||
}
|
||||
let outcome = crate::persist_raw_transaction_ingest_acquisition(port, acquisition).await;
|
||||
let outcome = match outcome {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(error) => return std::result::Result::Err(error),
|
||||
};
|
||||
if outcome.entity() != crate::RawTransactionIngestEntityPersistence::SkippedPurged {
|
||||
*known_state = std::option::Option::Some(canonical_state);
|
||||
}
|
||||
return std::result::Result::Ok(outcome);
|
||||
}
|
||||
|
||||
fn canonical_state(transaction: &ksp_store_lib::RawTransaction) -> RawTransactionIngestCanonicalState {
|
||||
return RawTransactionIngestCanonicalState {
|
||||
block_time: transaction.block_time(),
|
||||
content_hash: transaction.payload().content_hash(),
|
||||
format_id: transaction.payload().format_id().clone(),
|
||||
format_version: transaction.payload().format_version(),
|
||||
slot: transaction.slot(),
|
||||
};
|
||||
let _identity_guard = entry.lock().await;
|
||||
return crate::persist_raw_transaction_ingest_acquisition(port, acquisition).await;
|
||||
}
|
||||
|
||||
fn map_store_error(error: ksp_core_lib::Error) -> ksp_core_lib::Result<crate::RawTransactionIngestPersistenceOutcome> {
|
||||
@@ -239,49 +186,74 @@ fn map_store_error(error: ksp_core_lib::Error) -> ksp_core_lib::Result<crate::Ra
|
||||
fn map_store_outcome(outcome: ksp_store_lib::RawAcquisitionWriteOutcome) -> ksp_core_lib::Result<crate::RawTransactionIngestPersistenceOutcome> {
|
||||
let entity = outcome.entity();
|
||||
let observation = outcome.observation();
|
||||
let quarantined_conflict = match validate_transaction_variant_outcome(outcome.transaction_variant(), entity, observation) {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(error) => return std::result::Result::Err(error),
|
||||
};
|
||||
if entity == ksp_store_lib::RawEntityWriteOutcome::Inserted && observation == ksp_store_lib::RawObservationWriteOutcome::Inserted {
|
||||
return std::result::Result::Ok(crate::RawTransactionIngestPersistenceOutcome {
|
||||
entity: crate::RawTransactionIngestEntityPersistence::Inserted,
|
||||
observation: crate::RawTransactionIngestObservationPersistence::Inserted,
|
||||
quarantined_conflict,
|
||||
});
|
||||
}
|
||||
if entity == ksp_store_lib::RawEntityWriteOutcome::AlreadyPresent && observation == ksp_store_lib::RawObservationWriteOutcome::Inserted {
|
||||
return std::result::Result::Ok(crate::RawTransactionIngestPersistenceOutcome {
|
||||
entity: crate::RawTransactionIngestEntityPersistence::AlreadyPresent,
|
||||
observation: crate::RawTransactionIngestObservationPersistence::Inserted,
|
||||
quarantined_conflict,
|
||||
});
|
||||
}
|
||||
if entity == ksp_store_lib::RawEntityWriteOutcome::AlreadyPresent && observation == ksp_store_lib::RawObservationWriteOutcome::AlreadyPresent {
|
||||
return std::result::Result::Ok(crate::RawTransactionIngestPersistenceOutcome {
|
||||
entity: crate::RawTransactionIngestEntityPersistence::AlreadyPresent,
|
||||
observation: crate::RawTransactionIngestObservationPersistence::AlreadyPresent,
|
||||
quarantined_conflict,
|
||||
});
|
||||
}
|
||||
if entity == ksp_store_lib::RawEntityWriteOutcome::SkippedPurged && observation == ksp_store_lib::RawObservationWriteOutcome::NotRecorded {
|
||||
return std::result::Result::Ok(crate::RawTransactionIngestPersistenceOutcome {
|
||||
entity: crate::RawTransactionIngestEntityPersistence::SkippedPurged,
|
||||
observation: crate::RawTransactionIngestObservationPersistence::NotRecorded,
|
||||
quarantined_conflict,
|
||||
});
|
||||
}
|
||||
return std::result::Result::Err(crate::runtime_error("persistence.store_outcome_invalid"));
|
||||
}
|
||||
|
||||
fn map_additional_observation_outcome(
|
||||
outcome: ksp_store_lib::RawObservationWriteOutcome,
|
||||
) -> ksp_core_lib::Result<crate::RawTransactionIngestPersistenceOutcome> {
|
||||
return match outcome {
|
||||
ksp_store_lib::RawObservationWriteOutcome::Inserted => std::result::Result::Ok(crate::RawTransactionIngestPersistenceOutcome {
|
||||
entity: crate::RawTransactionIngestEntityPersistence::AlreadyPresent,
|
||||
observation: crate::RawTransactionIngestObservationPersistence::Inserted,
|
||||
}),
|
||||
ksp_store_lib::RawObservationWriteOutcome::AlreadyPresent => std::result::Result::Ok(crate::RawTransactionIngestPersistenceOutcome {
|
||||
entity: crate::RawTransactionIngestEntityPersistence::AlreadyPresent,
|
||||
observation: crate::RawTransactionIngestObservationPersistence::AlreadyPresent,
|
||||
}),
|
||||
ksp_store_lib::RawObservationWriteOutcome::NotRecorded => {
|
||||
std::result::Result::Err(crate::runtime_error("persistence.additional_observation_not_recorded"))
|
||||
fn validate_transaction_variant_outcome(
|
||||
transaction_variant: std::option::Option<ksp_store_lib::RawTransactionVariantWriteOutcome>,
|
||||
entity: ksp_store_lib::RawEntityWriteOutcome,
|
||||
observation: ksp_store_lib::RawObservationWriteOutcome,
|
||||
) -> ksp_core_lib::Result<bool> {
|
||||
let recorded_observation =
|
||||
matches!(observation, ksp_store_lib::RawObservationWriteOutcome::Inserted | ksp_store_lib::RawObservationWriteOutcome::AlreadyPresent);
|
||||
return match transaction_variant {
|
||||
std::option::Option::None => std::result::Result::Ok(false),
|
||||
std::option::Option::Some(ksp_store_lib::RawTransactionVariantWriteOutcome::InsertedCanonical)
|
||||
if entity == ksp_store_lib::RawEntityWriteOutcome::Inserted && observation == ksp_store_lib::RawObservationWriteOutcome::Inserted =>
|
||||
{
|
||||
std::result::Result::Ok(false)
|
||||
},
|
||||
_ => std::result::Result::Err(crate::runtime_error("persistence.additional_observation_outcome_invalid")),
|
||||
std::option::Option::Some(
|
||||
ksp_store_lib::RawTransactionVariantWriteOutcome::ObservedExact
|
||||
| ksp_store_lib::RawTransactionVariantWriteOutcome::ObservedCompatibleLessComplete
|
||||
| ksp_store_lib::RawTransactionVariantWriteOutcome::PromotedCompatibleMoreComplete,
|
||||
) if entity == ksp_store_lib::RawEntityWriteOutcome::AlreadyPresent && recorded_observation => std::result::Result::Ok(false),
|
||||
std::option::Option::Some(ksp_store_lib::RawTransactionVariantWriteOutcome::QuarantinedConflict)
|
||||
if entity == ksp_store_lib::RawEntityWriteOutcome::AlreadyPresent && recorded_observation =>
|
||||
{
|
||||
std::result::Result::Ok(true)
|
||||
},
|
||||
std::option::Option::Some(ksp_store_lib::RawTransactionVariantWriteOutcome::SkippedPurged)
|
||||
if entity == ksp_store_lib::RawEntityWriteOutcome::SkippedPurged && observation == ksp_store_lib::RawObservationWriteOutcome::NotRecorded =>
|
||||
{
|
||||
std::result::Result::Ok(false)
|
||||
},
|
||||
std::option::Option::Some(ksp_store_lib::RawTransactionVariantWriteOutcome::Rehydrated) => {
|
||||
std::result::Result::Err(crate::runtime_error("persistence.store_variant_rehydrated_in_normal_mode"))
|
||||
},
|
||||
_ => std::result::Result::Err(crate::runtime_error("persistence.store_variant_outcome_invalid")),
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
// file: crates/ksp-worker-raw-transaction-ingest-lib/src/snapshot.rs
|
||||
// version: 10
|
||||
// version: 11
|
||||
|
||||
/// Runtime-neutral boxed future resolving to one newer concrete RAW transaction ingest Worker snapshot.
|
||||
pub type RawTransactionIngestSnapshotFuture<'a> =
|
||||
@@ -1132,6 +1132,13 @@ impl crate::RawTransactionIngestSnapshotPublisher {
|
||||
let mut entity_skipped_purged_total = self.snapshot.entity_skipped_purged_total;
|
||||
let mut observation_inserted_total = self.snapshot.observation_inserted_total;
|
||||
let mut observation_already_present_total = self.snapshot.observation_already_present_total;
|
||||
let mut content_conflict_total = self.snapshot.content_conflict_total;
|
||||
if outcome.quarantined_conflict() {
|
||||
content_conflict_total = match checked_counter(content_conflict_total, "content_conflict_total") {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(error) => return std::result::Result::Err(error),
|
||||
};
|
||||
}
|
||||
match outcome.entity() {
|
||||
crate::RawTransactionIngestEntityPersistence::Inserted => {
|
||||
entity_inserted_total = match checked_counter(entity_inserted_total, "entity_inserted_total") {
|
||||
@@ -1173,6 +1180,7 @@ impl crate::RawTransactionIngestSnapshotPublisher {
|
||||
self.snapshot.entity_skipped_purged_total = entity_skipped_purged_total;
|
||||
self.snapshot.observation_inserted_total = observation_inserted_total;
|
||||
self.snapshot.observation_already_present_total = observation_already_present_total;
|
||||
self.snapshot.content_conflict_total = content_conflict_total;
|
||||
return self.publish(state, admission_queue_depth, in_flight_persistence);
|
||||
}
|
||||
|
||||
@@ -1279,6 +1287,9 @@ fn health_for_state(
|
||||
return ksp_worker_api::WorkerHealth::Unhealthy;
|
||||
}
|
||||
if source_total > 0 && snapshot.source_active == snapshot.source_total {
|
||||
if snapshot.content_conflict_total > 0 {
|
||||
return ksp_worker_api::WorkerHealth::Degraded;
|
||||
}
|
||||
return ksp_worker_api::WorkerHealth::Healthy;
|
||||
}
|
||||
if source_total > 0
|
||||
@@ -1297,6 +1308,7 @@ fn health_for_state(
|
||||
ksp_worker_api::WorkerState::Running if snapshot.source_total > 0 && snapshot.source_active < snapshot.source_total => {
|
||||
ksp_worker_api::WorkerHealth::Degraded
|
||||
},
|
||||
ksp_worker_api::WorkerState::Running if snapshot.content_conflict_total > 0 => ksp_worker_api::WorkerHealth::Degraded,
|
||||
ksp_worker_api::WorkerState::Running => ksp_worker_api::WorkerHealth::Healthy,
|
||||
ksp_worker_api::WorkerState::Stopping | ksp_worker_api::WorkerState::Stopped => previous,
|
||||
ksp_worker_api::WorkerState::Faulted(_) => ksp_worker_api::WorkerHealth::Unhealthy,
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
// file: crates/ksp-worker-raw-transaction-ingest-lib/tests/cross_layer_completeness.rs
|
||||
// version: 3
|
||||
// version: 4
|
||||
|
||||
//! Cross-layer completeness and security canaries through `0.3.14-pre.013`.
|
||||
|
||||
@@ -215,7 +215,12 @@ fn v0_3_13_pre_012_all_five_live_source_families_share_one_worker_common_raw_sto
|
||||
] {
|
||||
assert!(worker_resources.contains(required), "pre.012 Worker source/pipeline contract missing: {required}");
|
||||
}
|
||||
for required in ["persist_raw_transaction_ingest_converged_acquisition", "RawTransactionIngestPersistenceConvergence", "record_observation"] {
|
||||
for required in [
|
||||
"persist_raw_transaction_ingest_converged_acquisition",
|
||||
"RawTransactionIngestPersistenceConvergence",
|
||||
"let _identity_guard = entry.lock().await",
|
||||
"persist_raw_transaction_ingest_acquisition(port, acquisition).await",
|
||||
] {
|
||||
assert!(worker_persistence.contains(required), "pre.012 Worker persistence contract missing: {required}");
|
||||
}
|
||||
for required in ["SolanaStandardWsSession", "HeliusLaserStreamWsSession"] {
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
// file: crates/ksp-worker-raw-transaction-ingest-lib/tests/dependency_boundary.rs
|
||||
// version: 31
|
||||
// version: 32
|
||||
|
||||
//! Dependency firewall canaries for the RAW transaction ingest Worker foundation.
|
||||
|
||||
@@ -585,13 +585,16 @@ fn v0_3_13_pre_008_cross_source_convergence_reuses_store_and_transport_facades_o
|
||||
"RawTransactionIngestPersistenceConvergence",
|
||||
"persist_raw_transaction_ingest_converged_acquisition",
|
||||
"RawTransactionIngestPersistencePort",
|
||||
"record_raw_transaction_observation",
|
||||
"RawTransactionObservationWrite",
|
||||
"payload().content_hash()",
|
||||
"let _identity_guard = entry.lock().await",
|
||||
"persist_raw_transaction_ingest_acquisition(port, acquisition).await",
|
||||
"RawTransactionVariantWriteOutcome::QuarantinedConflict",
|
||||
"content_conflict_error()",
|
||||
] {
|
||||
assert!(persistence.contains(required) || root.contains(required), "required pre.008 persistence convergence contract missing: {required}");
|
||||
}
|
||||
for forbidden in ["record_raw_transaction_observation", "RawTransactionObservationWrite", "known_state_value", "canonical_state("] {
|
||||
assert!(!persistence.contains(forbidden), "pre.007 Worker persistence retained stronger run-local arbitration: {forbidden}");
|
||||
}
|
||||
for required in [
|
||||
"RawTransactionIngestGlobalHydrationRegistry",
|
||||
"fetch_hydration_shared",
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
// file: crates/ksp-worker-raw-transaction-ingest-lib/tests/hardening.rs
|
||||
// version: 40
|
||||
// version: 41
|
||||
|
||||
//! External public, security, redaction and release-boundary hardening canaries through `v0.3.15-pre.004`.
|
||||
|
||||
@@ -764,11 +764,15 @@ fn v0_3_13_pre_008_cross_source_convergence_is_bounded_conflict_checked_and_priv
|
||||
"max_entries: usize",
|
||||
"entries.len() >= self.max_entries",
|
||||
"std::sync::Arc::strong_count(entry) == 1",
|
||||
"known_state_value != &canonical_state",
|
||||
"record_observation(observation)",
|
||||
"persistence.additional_observation_not_recorded",
|
||||
"let _identity_guard = entry.lock().await",
|
||||
"persist_raw_transaction_ingest_acquisition(port, acquisition).await",
|
||||
"outcome.transaction_variant()",
|
||||
"RawTransactionVariantWriteOutcome::QuarantinedConflict",
|
||||
] {
|
||||
assert!(persistence.contains(required), "required pre.008 persistence hardening guard missing: {required}");
|
||||
assert!(persistence.contains(required), "required persistence serialization/Store-authority guard missing: {required}");
|
||||
}
|
||||
for forbidden in ["known_state_value", "canonical_state(", "record_observation(", "RawTransactionObservationWrite"] {
|
||||
assert!(!persistence.contains(forbidden), "run-local persistence cache still arbitrates before Store: {forbidden}");
|
||||
}
|
||||
for required in [
|
||||
"max_pending: usize",
|
||||
@@ -803,9 +807,9 @@ fn v0_3_13_pre_009_duplicate_storm_disagreement_and_starvation_guards_are_explic
|
||||
"tokio::sync::Semaphore::new(max_in_flight)",
|
||||
"RawTransactionIngestHydrationLeaderGuard",
|
||||
"source.global_hydration_pending_saturated",
|
||||
"block_time: transaction.block_time()",
|
||||
"slot: transaction.slot()",
|
||||
"content_hash: transaction.payload().content_hash()",
|
||||
"network: ksp_store_lib::RawNetworkId",
|
||||
"signature: ksp_store_lib::RawTransactionSignature",
|
||||
"let _identity_guard = entry.lock().await",
|
||||
] {
|
||||
assert!(
|
||||
admission.contains(required) || persistence.contains(required) || resources.contains(required),
|
||||
|
||||
@@ -1,9 +1,10 @@
|
||||
// file: crates/ksp-worker-raw-transaction-ingest-lib/unit_tests/persistence.rs
|
||||
// version: 3
|
||||
// version: 4
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
enum PortResponse {
|
||||
Outcome(ksp_store_lib::RawEntityWriteOutcome, ksp_store_lib::RawObservationWriteOutcome),
|
||||
VariantOutcome(ksp_store_lib::RawEntityWriteOutcome, ksp_store_lib::RawObservationWriteOutcome, ksp_store_lib::RawTransactionVariantWriteOutcome),
|
||||
Conflict,
|
||||
StoreFailure,
|
||||
}
|
||||
@@ -12,8 +13,6 @@ struct FakePersistencePort {
|
||||
calls: std::sync::atomic::AtomicUsize,
|
||||
network: ksp_store_lib::RawNetworkId,
|
||||
normal_mode_seen: std::sync::atomic::AtomicBool,
|
||||
observation_calls: std::sync::atomic::AtomicUsize,
|
||||
observation_response: std::option::Option<ksp_store_lib::RawObservationWriteOutcome>,
|
||||
response: PortResponse,
|
||||
}
|
||||
|
||||
@@ -23,16 +22,9 @@ impl FakePersistencePort {
|
||||
calls: std::sync::atomic::AtomicUsize::new(0),
|
||||
network,
|
||||
normal_mode_seen: std::sync::atomic::AtomicBool::new(false),
|
||||
observation_calls: std::sync::atomic::AtomicUsize::new(0),
|
||||
observation_response: std::option::Option::None,
|
||||
response,
|
||||
};
|
||||
}
|
||||
|
||||
fn with_observation_response(mut self, observation_response: ksp_store_lib::RawObservationWriteOutcome) -> Self {
|
||||
self.observation_response = std::option::Option::Some(observation_response);
|
||||
return self;
|
||||
}
|
||||
}
|
||||
|
||||
impl crate::RawTransactionIngestPersistencePort for FakePersistencePort {
|
||||
@@ -52,28 +44,9 @@ impl crate::RawTransactionIngestPersistencePort for FakePersistencePort {
|
||||
return std::boxed::Box::pin(async move {
|
||||
return match response {
|
||||
PortResponse::Outcome(entity, observation) => std::result::Result::Ok(ksp_store_lib::RawAcquisitionWriteOutcome::new(entity, observation)),
|
||||
PortResponse::Conflict => std::result::Result::Err(ksp_core_lib::Error::new(ksp_store_lib::ERROR_CODE_RAW_CONFLICT, "synthetic conflict")),
|
||||
PortResponse::StoreFailure => std::result::Result::Err(ksp_core_lib::Error::new(
|
||||
ksp_core_lib::ErrorCode::new("synthetic_store", "write_failed"),
|
||||
"synthetic remote failure text that must not escape",
|
||||
)),
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
fn record_observation<'a>(
|
||||
&'a self,
|
||||
_observation: ksp_store_lib::RawTransactionObservation,
|
||||
) -> ksp_store_lib::StoreApiFuture<'a, ksp_store_lib::Result<ksp_store_lib::RawObservationWriteOutcome>> {
|
||||
self.observation_calls.fetch_add(1, std::sync::atomic::Ordering::AcqRel);
|
||||
let observation_response = self.observation_response;
|
||||
let response = self.response;
|
||||
return std::boxed::Box::pin(async move {
|
||||
if let std::option::Option::Some(outcome) = observation_response {
|
||||
return std::result::Result::Ok(outcome);
|
||||
}
|
||||
return match response {
|
||||
PortResponse::Outcome(_, observation) => std::result::Result::Ok(observation),
|
||||
PortResponse::VariantOutcome(entity, observation, transaction_variant) => {
|
||||
std::result::Result::Ok(ksp_store_lib::RawAcquisitionWriteOutcome::with_transaction_variant(entity, observation, transaction_variant))
|
||||
},
|
||||
PortResponse::Conflict => std::result::Result::Err(ksp_core_lib::Error::new(ksp_store_lib::ERROR_CODE_RAW_CONFLICT, "synthetic conflict")),
|
||||
PortResponse::StoreFailure => std::result::Result::Err(ksp_core_lib::Error::new(
|
||||
ksp_core_lib::ErrorCode::new("synthetic_store", "write_failed"),
|
||||
@@ -203,7 +176,36 @@ async fn pre_007_normal_mode_classifies_new_idempotent_and_purged_outcomes() {
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "current_thread")]
|
||||
async fn pre_007_content_conflict_maps_to_terminal_worker_code_without_remote_text() {
|
||||
async fn v0_3_16_pre_007_quarantined_conflict_is_successful_and_explicit() {
|
||||
let network = match network("mainnet") {
|
||||
std::option::Option::Some(value) => value,
|
||||
std::option::Option::None => return,
|
||||
};
|
||||
let port = FakePersistencePort::new(
|
||||
network,
|
||||
PortResponse::VariantOutcome(
|
||||
ksp_store_lib::RawEntityWriteOutcome::AlreadyPresent,
|
||||
ksp_store_lib::RawObservationWriteOutcome::Inserted,
|
||||
ksp_store_lib::RawTransactionVariantWriteOutcome::QuarantinedConflict,
|
||||
),
|
||||
);
|
||||
let acquisition = match acquisition("mainnet", 8) {
|
||||
std::option::Option::Some(value) => value,
|
||||
std::option::Option::None => return,
|
||||
};
|
||||
let result = crate::persist_raw_transaction_ingest_acquisition(&port, acquisition).await;
|
||||
let outcome = match result {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(_) => return,
|
||||
};
|
||||
assert_eq!(outcome.entity(), crate::RawTransactionIngestEntityPersistence::AlreadyPresent);
|
||||
assert_eq!(outcome.observation(), crate::RawTransactionIngestObservationPersistence::Inserted);
|
||||
assert!(outcome.quarantined_conflict());
|
||||
return;
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "current_thread")]
|
||||
async fn pre_007_legacy_store_conflict_error_maps_to_terminal_worker_code_without_remote_text() {
|
||||
let network = match network("mainnet") {
|
||||
std::option::Option::Some(value) => value,
|
||||
std::option::Option::None => return,
|
||||
@@ -303,14 +305,14 @@ async fn pre_007_store_network_mismatch_is_rejected_before_write() {
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "current_thread")]
|
||||
async fn v0_3_13_pre_008_same_canonical_identity_uses_atomic_entity_once_then_additional_observation() {
|
||||
async fn v0_3_16_pre_007_same_identity_is_serialized_but_every_acquisition_reaches_store() {
|
||||
let network = match network("mainnet") {
|
||||
std::option::Option::Some(value) => value,
|
||||
std::option::Option::None => return,
|
||||
};
|
||||
let port = FakePersistencePort::new(
|
||||
network,
|
||||
PortResponse::Outcome(ksp_store_lib::RawEntityWriteOutcome::Inserted, ksp_store_lib::RawObservationWriteOutcome::Inserted),
|
||||
PortResponse::Outcome(ksp_store_lib::RawEntityWriteOutcome::AlreadyPresent, ksp_store_lib::RawObservationWriteOutcome::Inserted),
|
||||
);
|
||||
let convergence = crate::RawTransactionIngestPersistenceConvergence::new(8);
|
||||
let first = match acquisition_with_observation_key("mainnet", 41, 1) {
|
||||
@@ -321,56 +323,14 @@ async fn v0_3_13_pre_008_same_canonical_identity_uses_atomic_entity_once_then_ad
|
||||
std::option::Option::Some(value) => value,
|
||||
std::option::Option::None => return,
|
||||
};
|
||||
let first_outcome = crate::persist_raw_transaction_ingest_converged_acquisition(&port, first, &convergence).await;
|
||||
let second_outcome = crate::persist_raw_transaction_ingest_converged_acquisition(&port, second, &convergence).await;
|
||||
assert!(first_outcome.is_ok());
|
||||
assert!(second_outcome.is_ok());
|
||||
assert_eq!(port.calls.load(std::sync::atomic::Ordering::Acquire), 1);
|
||||
assert_eq!(port.observation_calls.load(std::sync::atomic::Ordering::Acquire), 1);
|
||||
let second_outcome = match second_outcome {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(_) => return,
|
||||
};
|
||||
assert_eq!(second_outcome.entity(), crate::RawTransactionIngestEntityPersistence::AlreadyPresent);
|
||||
assert_eq!(second_outcome.observation(), crate::RawTransactionIngestObservationPersistence::Inserted);
|
||||
assert!(crate::persist_raw_transaction_ingest_converged_acquisition(&port, first, &convergence).await.is_ok());
|
||||
assert!(crate::persist_raw_transaction_ingest_converged_acquisition(&port, second, &convergence).await.is_ok());
|
||||
assert_eq!(port.calls.load(std::sync::atomic::Ordering::Acquire), 2);
|
||||
return;
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "current_thread")]
|
||||
async fn v0_3_13_pre_008_replayed_same_observation_is_idempotent_without_raw_resubmission() {
|
||||
let network = match network("mainnet") {
|
||||
std::option::Option::Some(value) => value,
|
||||
std::option::Option::None => return,
|
||||
};
|
||||
let port = FakePersistencePort::new(
|
||||
network,
|
||||
PortResponse::Outcome(ksp_store_lib::RawEntityWriteOutcome::Inserted, ksp_store_lib::RawObservationWriteOutcome::Inserted),
|
||||
)
|
||||
.with_observation_response(ksp_store_lib::RawObservationWriteOutcome::AlreadyPresent);
|
||||
let convergence = crate::RawTransactionIngestPersistenceConvergence::new(8);
|
||||
let first = match acquisition_with_observation_key("mainnet", 42, 3) {
|
||||
std::option::Option::Some(value) => value,
|
||||
std::option::Option::None => return,
|
||||
};
|
||||
let second = match acquisition_with_observation_key("mainnet", 42, 3) {
|
||||
std::option::Option::Some(value) => value,
|
||||
std::option::Option::None => return,
|
||||
};
|
||||
let first_result = crate::persist_raw_transaction_ingest_converged_acquisition(&port, first, &convergence).await;
|
||||
assert!(first_result.is_ok());
|
||||
let second_result = crate::persist_raw_transaction_ingest_converged_acquisition(&port, second, &convergence).await;
|
||||
let second_outcome = match second_result {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(_) => return,
|
||||
};
|
||||
assert_eq!(port.calls.load(std::sync::atomic::Ordering::Acquire), 1);
|
||||
assert_eq!(port.observation_calls.load(std::sync::atomic::Ordering::Acquire), 1);
|
||||
assert_eq!(second_outcome.observation(), crate::RawTransactionIngestObservationPersistence::AlreadyPresent);
|
||||
return;
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "current_thread")]
|
||||
async fn v0_3_13_pre_009_source_skew_and_provider_disagreement_fail_before_additional_observation() {
|
||||
async fn v0_3_16_pre_007_run_local_cache_never_rejects_divergent_shapes_before_store() {
|
||||
let cases = [
|
||||
(43_u64, std::option::Option::Some(1_700_000_000_i64), "AQID"),
|
||||
(42_u64, std::option::Option::Some(1_700_000_001_i64), "AQID"),
|
||||
@@ -383,7 +343,11 @@ async fn v0_3_13_pre_009_source_skew_and_provider_disagreement_fail_before_addit
|
||||
};
|
||||
let port = FakePersistencePort::new(
|
||||
network,
|
||||
PortResponse::Outcome(ksp_store_lib::RawEntityWriteOutcome::Inserted, ksp_store_lib::RawObservationWriteOutcome::Inserted),
|
||||
PortResponse::VariantOutcome(
|
||||
ksp_store_lib::RawEntityWriteOutcome::AlreadyPresent,
|
||||
ksp_store_lib::RawObservationWriteOutcome::Inserted,
|
||||
ksp_store_lib::RawTransactionVariantWriteOutcome::QuarantinedConflict,
|
||||
),
|
||||
);
|
||||
let convergence = crate::RawTransactionIngestPersistenceConvergence::new(8);
|
||||
let first = match acquisition_with_shape("mainnet", 51, 10, 42, std::option::Option::Some(1_700_000_000), "AQID") {
|
||||
@@ -394,15 +358,11 @@ async fn v0_3_13_pre_009_source_skew_and_provider_disagreement_fail_before_addit
|
||||
std::option::Option::Some(value) => value,
|
||||
std::option::Option::None => return,
|
||||
};
|
||||
assert!(crate::persist_raw_transaction_ingest_converged_acquisition(&port, first, &convergence).await.is_ok());
|
||||
let conflict = crate::persist_raw_transaction_ingest_converged_acquisition(&port, divergent, &convergence).await;
|
||||
let error = match conflict {
|
||||
std::result::Result::Ok(_) => return,
|
||||
std::result::Result::Err(value) => value,
|
||||
};
|
||||
assert_eq!(error.code(), crate::ERROR_CODE_RAW_TRANSACTION_INGEST_CONTENT_CONFLICT);
|
||||
assert_eq!(port.calls.load(std::sync::atomic::Ordering::Acquire), 1);
|
||||
assert_eq!(port.observation_calls.load(std::sync::atomic::Ordering::Acquire), 0);
|
||||
let first_outcome = crate::persist_raw_transaction_ingest_converged_acquisition(&port, first, &convergence).await;
|
||||
let second_outcome = crate::persist_raw_transaction_ingest_converged_acquisition(&port, divergent, &convergence).await;
|
||||
assert!(matches!(first_outcome, std::result::Result::Ok(value) if value.quarantined_conflict()));
|
||||
assert!(matches!(second_outcome, std::result::Result::Ok(value) if value.quarantined_conflict()));
|
||||
assert_eq!(port.calls.load(std::sync::atomic::Ordering::Acquire), 2);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
// file: crates/ksp-worker-raw-transaction-ingest-lib/unit_tests/runtime.rs
|
||||
// version: 12
|
||||
// version: 13
|
||||
|
||||
struct ActiveTaskGuard {
|
||||
active: std::sync::Arc<std::sync::atomic::AtomicUsize>,
|
||||
@@ -249,6 +249,7 @@ async fn pre_005_supervisor_reaps_completed_source_task_and_still_joins_live_chi
|
||||
#[derive(Clone, Copy)]
|
||||
enum RuntimePortResponse {
|
||||
Conflict,
|
||||
QuarantinedConflict,
|
||||
StoreFailure,
|
||||
StoreFailureBlocked,
|
||||
SuccessBlocked,
|
||||
@@ -337,35 +338,13 @@ impl crate::RawTransactionIngestPersistencePort for RuntimePersistencePort {
|
||||
));
|
||||
}
|
||||
}
|
||||
if matches!(response, RuntimePortResponse::Conflict) {
|
||||
if matches!(response, RuntimePortResponse::QuarantinedConflict) {
|
||||
self.completed.fetch_add(1, std::sync::atomic::Ordering::AcqRel);
|
||||
return std::result::Result::Err(ksp_core_lib::Error::new(ksp_store_lib::ERROR_CODE_RAW_CONFLICT, "synthetic conflict"));
|
||||
}
|
||||
self.completed.fetch_add(1, std::sync::atomic::Ordering::AcqRel);
|
||||
return std::result::Result::Err(ksp_core_lib::Error::new(
|
||||
ksp_core_lib::ErrorCode::new("synthetic_store", "write_failed"),
|
||||
"synthetic store failure",
|
||||
));
|
||||
});
|
||||
}
|
||||
|
||||
fn record_observation<'a>(
|
||||
&'a self,
|
||||
_observation: ksp_store_lib::RawTransactionObservation,
|
||||
) -> ksp_store_lib::StoreApiFuture<'a, ksp_store_lib::Result<ksp_store_lib::RawObservationWriteOutcome>> {
|
||||
let response = self.response;
|
||||
return std::boxed::Box::pin(async move {
|
||||
if matches!(response, RuntimePortResponse::StoreFailureBlocked | RuntimePortResponse::SuccessBlocked) {
|
||||
let current = self.active.fetch_add(1, std::sync::atomic::Ordering::AcqRel) + 1;
|
||||
let _active_guard = RuntimePortActiveGuard { active: &self.active };
|
||||
self.update_max_active(current);
|
||||
while !self.released.load(std::sync::atomic::Ordering::Acquire) {
|
||||
self.notify.notified().await;
|
||||
}
|
||||
if matches!(response, RuntimePortResponse::SuccessBlocked) {
|
||||
self.completed.fetch_add(1, std::sync::atomic::Ordering::AcqRel);
|
||||
return std::result::Result::Ok(ksp_store_lib::RawObservationWriteOutcome::Inserted);
|
||||
}
|
||||
return std::result::Result::Ok(ksp_store_lib::RawAcquisitionWriteOutcome::with_transaction_variant(
|
||||
ksp_store_lib::RawEntityWriteOutcome::AlreadyPresent,
|
||||
ksp_store_lib::RawObservationWriteOutcome::Inserted,
|
||||
ksp_store_lib::RawTransactionVariantWriteOutcome::QuarantinedConflict,
|
||||
));
|
||||
}
|
||||
if matches!(response, RuntimePortResponse::Conflict) {
|
||||
self.completed.fetch_add(1, std::sync::atomic::Ordering::AcqRel);
|
||||
@@ -505,7 +484,50 @@ async fn pre_007_runtime_bounds_in_flight_store_persistence_to_configured_concur
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "current_thread")]
|
||||
async fn pre_007_content_conflict_becomes_terminal_after_private_drain() {
|
||||
async fn v0_3_16_pre_007_quarantined_conflict_keeps_worker_running_and_degraded() {
|
||||
let settings = match settings_with_persistence_concurrency(1) {
|
||||
std::option::Option::Some(value) => value,
|
||||
std::option::Option::None => return,
|
||||
};
|
||||
let network = settings.network().clone();
|
||||
let port = std::sync::Arc::new(RuntimePersistencePort::new(network.clone(), RuntimePortResponse::QuarantinedConflict, true));
|
||||
let runtime_port: super::PersistencePort = port.clone();
|
||||
let handle = match super::start_foundation_with_port_and_source_spawner(
|
||||
settings,
|
||||
tokio::runtime::Handle::current(),
|
||||
std::option::Option::Some(runtime_port),
|
||||
move |children, _stop_receiver, admission_sender| {
|
||||
let _abort_handle = children.spawn(async move {
|
||||
let ingress = match runtime_ingress(&network, 10) {
|
||||
std::option::Option::Some(value) => value,
|
||||
std::option::Option::None => return std::result::Result::Err(crate::runtime_error("test.source_failed")),
|
||||
};
|
||||
if admission_sender.send(ingress).await.is_err() {
|
||||
return std::result::Result::Err(crate::runtime_error("test.source_failed"));
|
||||
}
|
||||
return std::result::Result::Ok(());
|
||||
});
|
||||
},
|
||||
) {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(_) => return,
|
||||
};
|
||||
let source = handle.snapshot_source();
|
||||
assert!(wait_for_completed(&port, 1).await);
|
||||
let snapshot = source.current();
|
||||
assert_eq!(snapshot.worker_snapshot().state(), ksp_worker_api::WorkerState::Running);
|
||||
assert_eq!(snapshot.worker_snapshot().health(), ksp_worker_api::WorkerHealth::Degraded);
|
||||
assert_eq!(snapshot.persisted_total(), 1);
|
||||
assert_eq!(snapshot.content_conflict_total(), 1);
|
||||
assert_eq!(snapshot.store_failure_total(), 0);
|
||||
assert!(handle.request_stop());
|
||||
let terminal = handle.wait_terminal().await;
|
||||
assert_eq!(terminal, std::result::Result::Ok(ksp_worker_api::WorkerState::Stopped));
|
||||
return;
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "current_thread")]
|
||||
async fn pre_007_legacy_store_conflict_error_becomes_terminal_after_private_drain() {
|
||||
let settings = match settings_with_persistence_concurrency(1) {
|
||||
std::option::Option::Some(value) => value,
|
||||
std::option::Option::None => return,
|
||||
|
||||
Reference in New Issue
Block a user