v0.5.3-pre.003

This commit is contained in:
2026-08-12 11:00:59 +02:00
parent 8448ad1079
commit 400ced4832
313 changed files with 7773 additions and 2623 deletions

View File

@@ -1,9 +1,9 @@
// file: ks-store/src/constants.rs
// version: 5
// version: 6
//! Local constants for the `ks-store` crate.
/// Canonical tracing target for backend-independent storage operations.
pub(crate) const TRACING_TARGET: &str = "ks-store";
/// Transitional schema contract identifier used until the `pre.003` baseline rebuild.
pub const STORE_SCHEMA_CONTRACT_VERSION: &str = "0.5.3-pre.2-legacy-schema";
/// Candidate N1-N3 store schema contract identifier established during `0.5.3`.
pub const STORE_SCHEMA_CONTRACT_VERSION: &str = "0.5.3";

View File

@@ -1,5 +1,5 @@
// file: ks-store/src/contracts.rs
// version: 6
// version: 7
//! Backend-neutral storage contracts used by pipeline crates.
@@ -11,10 +11,18 @@ mod pagination;
mod replay;
mod repository;
/// Generic account acquisition observation insert contract.
pub use self::dto::AccountObservationInsert;
/// Transaction acquisition observation origin.
pub use self::dto::AcquisitionObservationOrigin;
/// Generic account acquisition observation status.
pub use self::dto::AcquisitionObservationStatus;
/// Core account key insert contract.
pub use self::dto::CoreAccountKeyInsert;
/// Core account key source category.
pub use self::dto::CoreAccountKeySource;
/// Core account-state insert contract.
pub use self::dto::CoreAccountStateInsert;
/// Core balance change insert contract.
pub use self::dto::CoreBalanceChangeInsert;
/// Core balance change kind.
@@ -37,6 +45,8 @@ pub use self::dto::CoreInstructionProcessingState;
pub use self::dto::CoreInstructionReplayFilter;
/// Core log insert contract.
pub use self::dto::CoreLogInsert;
/// Core return-data insert contract.
pub use self::dto::CoreReturnDataInsert;
/// Core transaction insert contract.
pub use self::dto::CoreTransactionInsert;
/// One machine-readable decoder coverage declaration row.
@@ -51,20 +61,21 @@ pub use self::dto::DecodeFailure;
pub use self::dto::DecodeObservationInsert;
/// Atomic persistence bundle for one decoder and one contextual input.
pub use self::dto::DecodePersistenceBundle;
pub use self::dto::DecodeSchemaProvenance;
/// Bounded contextual instruction selection filter for decode campaigns.
pub use self::dto::DecodeSelectionFilter;
/// Insert or upsert result contract returned by repositories.
pub use self::dto::InsertOutcome;
/// Maximum number of materialized rows returned by one bounded query.
pub use self::dto::MAX_MATERIALIZED_EVENT_QUERY_ROWS;
pub use self::dto::MAX_MATERIALIZED_OUTPUT_QUERY_ROWS;
/// Atomic persistence bundle for one materializer and one decoded observation.
pub use self::dto::MaterializationPersistenceBundle;
/// Bounded read-only materialized event selection.
pub use self::dto::MaterializedEventFilter;
/// One materialized output returned by a bounded query.
pub use self::dto::MaterializedEventQueryRow;
/// Bounded read-only materialized output selection.
pub use self::dto::MaterializedOutputFilter;
/// One processor-owned materialized output row.
pub use self::dto::MaterializedOutputInsert;
/// One materialized output returned by a bounded query.
pub use self::dto::MaterializedOutputQueryRow;
/// Stable processing ledger identity.
pub use self::dto::ProcessingLedgerIdentity;
/// Stable processing ledger status.
@@ -105,12 +116,13 @@ pub use self::dto::StoreResourceStatistics;
pub use self::dto::StoreRuntimeSummary;
/// Transaction acquisition observation insert contract.
pub use self::dto::TransactionObservationInsert;
/// Transaction acquisition observation origin.
pub use self::dto::TransactionObservationOrigin;
/// Transaction acquisition observation status.
pub use self::dto::TransactionObservationStatus;
/// Generic account acquisition observation SQL-like row contract.
pub use self::entity::AccountObservationRow;
/// Core account key SQL-like row contract.
pub use self::entity::CoreAccountKeyRow;
/// Core account-state SQL-like row contract.
pub use self::entity::CoreAccountStateRow;
/// Core balance change SQL-like row contract.
pub use self::entity::CoreBalanceChangeRow;
/// Core inner instruction SQL-like row contract.
@@ -119,6 +131,8 @@ pub use self::entity::CoreInnerInstructionRow;
pub use self::entity::CoreInstructionRow;
/// Core log SQL-like row contract.
pub use self::entity::CoreLogRow;
/// Core return-data SQL-like row contract.
pub use self::entity::CoreReturnDataRow;
/// Core transaction SQL-like row contract.
pub use self::entity::CoreTransactionRow;
/// Canonical raw Solana transaction SQL-like row contract.

View File

@@ -1,5 +1,5 @@
// file: ks-store/src/contracts/dto.rs
// version: 6
// version: 7
//! Backend-neutral DTO exports for storage repository contracts.
@@ -14,6 +14,8 @@ mod store;
pub use self::core::CoreAccountKeyInsert;
/// Core account key source category.
pub use self::core::CoreAccountKeySource;
/// Core account-state insert contract.
pub use self::core::CoreAccountStateInsert;
/// Core balance change insert contract.
pub use self::core::CoreBalanceChangeInsert;
/// Core balance change kind.
@@ -30,6 +32,8 @@ pub use self::core::CoreInstructionProcessingState;
pub use self::core::CoreInstructionReplayFilter;
/// Core log insert contract.
pub use self::core::CoreLogInsert;
/// Core return-data insert contract.
pub use self::core::CoreReturnDataInsert;
/// Core transaction insert contract.
pub use self::core::CoreTransactionInsert;
/// Complete normalized core extraction write bundle.
@@ -54,20 +58,27 @@ pub use self::decode::DecodeFailure;
pub use self::decode::DecodeObservationInsert;
/// Atomic persistence bundle for one decoder and one contextual input.
pub use self::decode::DecodePersistenceBundle;
pub use self::decode::DecodeSchemaProvenance;
/// Bounded contextual instruction selection filter for decode campaigns.
pub use self::decode::DecodeSelectionFilter;
/// Maximum number of materialized rows returned by one bounded query.
pub use self::decode::MAX_MATERIALIZED_EVENT_QUERY_ROWS;
pub use self::decode::MAX_MATERIALIZED_OUTPUT_QUERY_ROWS;
/// Atomic persistence bundle for one materializer and one decoded observation.
pub use self::decode::MaterializationPersistenceBundle;
/// Bounded read-only materialized event selection.
pub use self::decode::MaterializedEventFilter;
/// One materialized output returned by a bounded query.
pub use self::decode::MaterializedEventQueryRow;
/// Bounded read-only materialized output selection.
pub use self::decode::MaterializedOutputFilter;
/// One processor-owned materialized output row.
pub use self::decode::MaterializedOutputInsert;
/// One materialized output returned by a bounded query.
pub use self::decode::MaterializedOutputQueryRow;
/// Insert or upsert result contract returned by repositories.
pub use self::outcome::InsertOutcome;
/// Generic account acquisition observation insert contract.
pub use self::raw::AccountObservationInsert;
/// Transaction acquisition observation origin.
pub use self::raw::AcquisitionObservationOrigin;
/// Generic account acquisition observation status.
pub use self::raw::AcquisitionObservationStatus;
/// Raw payload lifecycle mark request.
pub use self::raw::RawPayloadLifecycleMark;
/// Raw payload processing state.
@@ -78,10 +89,7 @@ pub use self::raw::RawPayloadRetentionState;
pub use self::raw::RawTransactionInsert;
/// Transaction acquisition observation insert contract.
pub use self::raw::TransactionObservationInsert;
/// Transaction acquisition observation origin.
pub use self::raw::TransactionObservationOrigin;
/// Transaction acquisition observation status.
pub use self::raw::TransactionObservationStatus;
/// Store backend diagnostic contract.
pub use self::store::StoreBackendDescriptor;
/// Backend-neutral diagnostic snapshot.

View File

@@ -1,5 +1,5 @@
// file: ks-store/src/contracts/dto/core.rs
// version: 2
// version: 3
//! Core Solana storage DTOs.
@@ -47,6 +47,8 @@ pub struct CoreTransactionInsert {
pub signature: std::string::String,
/// Transaction slot in Solana unsigned representation.
pub slot: u64,
/// Optional block timestamp as Unix seconds supplied by Solana.
pub block_time: std::option::Option<i64>,
/// Whether the transaction failed on-chain.
pub failed: bool,
/// Optional raw error JSON extracted from transaction metadata.
@@ -55,11 +57,12 @@ pub struct CoreTransactionInsert {
pub raw_transaction_id: std::option::Option<i64>,
}
impl CoreTransactionInsert {
impl crate::CoreTransactionInsert {
/// Builds a core transaction insert contract after minimal validation.
pub fn new(
signature: impl std::convert::Into<std::string::String>,
slot: u64,
block_time: std::option::Option<i64>,
failed: bool,
err_json: std::option::Option<serde_json::Value>,
) -> ks_core::Result<Self> {
@@ -74,6 +77,7 @@ impl CoreTransactionInsert {
return std::result::Result::Ok(Self {
signature: signature_value,
slot,
block_time,
failed,
err_json,
raw_transaction_id: std::option::Option::None,
@@ -99,7 +103,7 @@ pub struct CoreAccountKeyInsert {
/// Account public key as non-empty base58 text.
pub account_key: std::string::String,
/// Source category for this account key.
pub source: CoreAccountKeySource,
pub source: crate::CoreAccountKeySource,
/// Whether the resolved account is writable for the transaction.
pub writable: bool,
/// Whether the resolved account signed the transaction.
@@ -108,7 +112,7 @@ pub struct CoreAccountKeyInsert {
pub executable: std::option::Option<bool>,
}
impl CoreAccountKeyInsert {
impl crate::CoreAccountKeyInsert {
/// Builds a core account key insert contract after minimal validation.
#[allow(clippy::too_many_arguments)]
pub fn new(
@@ -116,7 +120,7 @@ impl CoreAccountKeyInsert {
slot: u64,
account_index: u32,
account_key: impl std::convert::Into<std::string::String>,
source: CoreAccountKeySource,
source: crate::CoreAccountKeySource,
writable: bool,
signer: bool,
executable: std::option::Option<bool>,
@@ -148,6 +152,49 @@ impl CoreAccountKeyInsert {
}
}
/// Canonical Core account-state observation produced from a raw account observation.
#[derive(Clone, Debug, PartialEq, serde::Deserialize, serde::Serialize)]
pub struct CoreAccountStateInsert {
/// Source raw account observation technical id.
pub source_observation_id: i64,
/// Observed account public key.
pub account_key: std::string::String,
/// Context slot of the normalized account state.
pub slot: u64,
/// Owning program id.
pub owner: std::string::String,
/// Exact lamports value.
pub lamports: u64,
/// Whether the account is executable.
pub executable: bool,
/// Exact rent epoch.
pub rent_epoch: u64,
/// Complete account-data length.
pub space: u64,
/// Exact account bytes encoded as standard base64.
pub data_base64: std::string::String,
/// Deterministic digest of decoded account bytes.
pub data_hash: std::string::String,
}
impl crate::CoreAccountStateInsert {
/// Validates a complete source-independent account state before persistence.
pub fn validate(&self) -> ks_core::Result<()> {
if self.source_observation_id <= 0 {
return std::result::Result::Err(ks_core::Error::db(
"core account state source observation id must be positive",
));
}
let required = [self.account_key.as_str(), self.owner.as_str(), self.data_hash.as_str()];
if required.iter().any(|value| return value.trim().is_empty()) {
return std::result::Result::Err(ks_core::Error::db(
"core account state identity fields must not be empty",
));
}
return std::result::Result::Ok(());
}
}
/// Core instruction insert contract.
#[derive(Clone, Debug, PartialEq, serde::Deserialize, serde::Serialize)]
pub struct CoreInstructionInsert {
@@ -159,6 +206,8 @@ pub struct CoreInstructionInsert {
pub instruction_path: std::string::String,
/// Program id as non-empty base58 text.
pub program_id: std::string::String,
/// Optional runtime invocation stack height.
pub stack_height: std::option::Option<u32>,
/// Instruction accounts as JSON, preserving unresolved forms when needed.
pub accounts_json: serde_json::Value,
/// Instruction payload JSON, preserving raw and partially decoded forms when needed.
@@ -166,16 +215,17 @@ pub struct CoreInstructionInsert {
/// Optional deterministic payload JSON hash.
pub payload_json_hash: std::option::Option<std::string::String>,
/// Initial processing state used by replay schedulers.
pub processing_state: CoreInstructionProcessingState,
pub processing_state: crate::CoreInstructionProcessingState,
}
impl CoreInstructionInsert {
impl crate::CoreInstructionInsert {
/// Builds a core instruction insert contract after minimal validation.
pub fn new(
signature: impl std::convert::Into<std::string::String>,
slot: u64,
instruction_path: impl std::convert::Into<std::string::String>,
program_id: impl std::convert::Into<std::string::String>,
stack_height: std::option::Option<u32>,
accounts_json: serde_json::Value,
payload_json: serde_json::Value,
) -> ks_core::Result<Self> {
@@ -191,15 +241,21 @@ impl CoreInstructionInsert {
if let std::result::Result::Err(error) = validation_result {
return std::result::Result::Err(error);
}
if stack_height.is_some_and(|value| return value == 0) {
return std::result::Result::Err(ks_core::Error::db(
"core instruction stack height must be greater than zero when present",
));
}
return std::result::Result::Ok(Self {
signature: signature_value,
slot,
instruction_path: instruction_path_value,
program_id: program_id_value,
stack_height,
accounts_json,
payload_json,
payload_json_hash: std::option::Option::None,
processing_state: CoreInstructionProcessingState::Pending,
processing_state: crate::CoreInstructionProcessingState::Pending,
});
}
@@ -232,15 +288,19 @@ pub struct CoreInnerInstructionInsert {
pub instruction_path: std::string::String,
/// Program id as non-empty base58 text.
pub program_id: std::string::String,
/// Optional runtime invocation stack height.
pub stack_height: std::option::Option<u32>,
/// Inner instruction accounts as JSON.
pub accounts_json: serde_json::Value,
/// Inner instruction payload JSON.
pub payload_json: serde_json::Value,
/// Optional deterministic payload JSON hash.
pub payload_json_hash: std::option::Option<std::string::String>,
/// Initial processing state used by CPI replay schedulers.
pub processing_state: crate::CoreInstructionProcessingState,
}
impl CoreInnerInstructionInsert {
impl crate::CoreInnerInstructionInsert {
/// Builds a core inner instruction insert contract after minimal validation.
pub fn new(
signature: impl std::convert::Into<std::string::String>,
@@ -248,6 +308,7 @@ impl CoreInnerInstructionInsert {
parent_instruction_path: impl std::convert::Into<std::string::String>,
instruction_path: impl std::convert::Into<std::string::String>,
program_id: impl std::convert::Into<std::string::String>,
stack_height: std::option::Option<u32>,
accounts_json: serde_json::Value,
payload_json: serde_json::Value,
) -> ks_core::Result<Self> {
@@ -271,15 +332,22 @@ impl CoreInnerInstructionInsert {
if let std::result::Result::Err(error) = validation_result {
return std::result::Result::Err(error);
}
if stack_height.is_some_and(|value| return value <= 1) {
return std::result::Result::Err(ks_core::Error::db(
"core inner instruction stack height must be greater than one when present",
));
}
return std::result::Result::Ok(Self {
signature: signature_value,
slot,
parent_instruction_path: parent_instruction_path_value,
instruction_path: instruction_path_value,
program_id: program_id_value,
stack_height,
accounts_json,
payload_json,
payload_json_hash: std::option::Option::None,
processing_state: crate::CoreInstructionProcessingState::Pending,
});
}
@@ -299,6 +367,52 @@ impl CoreInnerInstructionInsert {
}
}
/// Transaction return-data fact retained in Core for future decoders.
#[derive(Clone, Debug, PartialEq, serde::Deserialize, serde::Serialize)]
pub struct CoreReturnDataInsert {
/// Transaction signature.
pub signature: std::string::String,
/// Transaction slot.
pub slot: u64,
/// Program id that produced the return data.
pub program_id: std::string::String,
/// Exact return bytes encoded as standard base64.
pub data_base64: std::string::String,
}
impl crate::CoreReturnDataInsert {
/// Builds a validated Core return-data fact.
pub fn new(
signature: impl std::convert::Into<std::string::String>,
slot: u64,
program_id: impl std::convert::Into<std::string::String>,
data_base64: impl std::convert::Into<std::string::String>,
) -> ks_core::Result<Self> {
let signature_value = signature.into();
let program_id_value = program_id.into();
let signature_result = validate_required_text(
&signature_value,
"core return-data signature must not be empty",
);
if let std::result::Result::Err(error) = signature_result {
return std::result::Result::Err(error);
}
let program_result = validate_required_text(
&program_id_value,
"core return-data program id must not be empty",
);
if let std::result::Result::Err(error) = program_result {
return std::result::Result::Err(error);
}
return std::result::Result::Ok(Self {
signature: signature_value,
slot,
program_id: program_id_value,
data_base64: data_base64.into(),
});
}
}
/// Core log insert contract.
#[derive(Clone, Debug, PartialEq, serde::Deserialize, serde::Serialize)]
pub struct CoreLogInsert {
@@ -318,7 +432,7 @@ pub struct CoreLogInsert {
pub log_text_hash: std::option::Option<std::string::String>,
}
impl CoreLogInsert {
impl crate::CoreLogInsert {
/// Builds a core log insert contract after minimal validation.
pub fn new(
signature: impl std::convert::Into<std::string::String>,
@@ -391,7 +505,7 @@ pub struct CoreBalanceChangeInsert {
/// Stable balance change index preserving extraction order.
pub balance_change_index: u32,
/// Balance change family.
pub balance_kind: CoreBalanceChangeKind,
pub balance_kind: crate::CoreBalanceChangeKind,
/// Optional account index when available.
pub account_index: std::option::Option<u32>,
/// Optional account public key when available.
@@ -408,14 +522,14 @@ pub struct CoreBalanceChangeInsert {
pub delta_json: std::option::Option<serde_json::Value>,
}
impl CoreBalanceChangeInsert {
impl crate::CoreBalanceChangeInsert {
/// Builds a core balance change insert contract after minimal validation.
#[allow(clippy::too_many_arguments)]
pub fn new(
signature: impl std::convert::Into<std::string::String>,
slot: u64,
balance_change_index: u32,
balance_kind: CoreBalanceChangeKind,
balance_kind: crate::CoreBalanceChangeKind,
account_index: std::option::Option<u32>,
account_key: std::option::Option<std::string::String>,
mint: std::option::Option<std::string::String>,
@@ -473,7 +587,7 @@ impl CoreBalanceChangeInsert {
#[derive(Clone, Debug, Eq, PartialEq, serde::Deserialize, serde::Serialize)]
pub struct CoreInstructionReplayFilter {
/// Optional processing state to select, usually `Pending`, `Failed` or `ReplayRequested`.
pub processing_state: std::option::Option<CoreInstructionProcessingState>,
pub processing_state: std::option::Option<crate::CoreInstructionProcessingState>,
/// Optional program id filter.
pub program_id: std::option::Option<std::string::String>,
/// Optional inclusive minimum slot.
@@ -482,10 +596,10 @@ pub struct CoreInstructionReplayFilter {
pub max_slot: std::option::Option<u64>,
}
impl CoreInstructionReplayFilter {
impl crate::CoreInstructionReplayFilter {
/// Builds a replay filter after minimal validation.
pub fn new(
processing_state: std::option::Option<CoreInstructionProcessingState>,
processing_state: std::option::Option<crate::CoreInstructionProcessingState>,
program_id: std::option::Option<std::string::String>,
min_slot: std::option::Option<u64>,
max_slot: std::option::Option<u64>,
@@ -519,7 +633,9 @@ impl CoreInstructionReplayFilter {
/// Builds the default pending instruction replay filter.
pub fn pending() -> Self {
return Self {
processing_state: std::option::Option::Some(CoreInstructionProcessingState::Pending),
processing_state: std::option::Option::Some(
crate::CoreInstructionProcessingState::Pending,
),
program_id: std::option::Option::None,
min_slot: std::option::Option::None,
max_slot: std::option::Option::None,
@@ -535,7 +651,7 @@ pub struct CoreInstructionLifecycleMark {
/// Stable instruction path, for example `0` or `2/1`.
pub instruction_path: std::string::String,
/// New processing state.
pub processing_state: CoreInstructionProcessingState,
pub processing_state: crate::CoreInstructionProcessingState,
/// Optional processor name that produced the lifecycle transition.
pub processor_name: std::option::Option<std::string::String>,
/// Optional processor version that produced the lifecycle transition.
@@ -544,12 +660,12 @@ pub struct CoreInstructionLifecycleMark {
pub reason: std::option::Option<std::string::String>,
}
impl CoreInstructionLifecycleMark {
impl crate::CoreInstructionLifecycleMark {
/// Builds a core instruction lifecycle mark after minimal validation.
pub fn new(
signature: impl std::convert::Into<std::string::String>,
instruction_path: impl std::convert::Into<std::string::String>,
processing_state: CoreInstructionProcessingState,
processing_state: crate::CoreInstructionProcessingState,
processor_name: std::option::Option<std::string::String>,
processor_version: std::option::Option<std::string::String>,
reason: std::option::Option<std::string::String>,
@@ -658,13 +774,25 @@ fn validate_optional_text(
mod tests {
#[test]
fn core_transaction_rejects_empty_signature() {
let result = crate::CoreTransactionInsert::new(" ", 1, false, std::option::Option::None);
let result = crate::CoreTransactionInsert::new(
" ",
1,
std::option::Option::None,
false,
std::option::Option::None,
);
assert!(result.is_err());
}
#[test]
fn core_transaction_accepts_optional_raw_lineage() {
let result = crate::CoreTransactionInsert::new("abc", 1, false, std::option::Option::None);
let result = crate::CoreTransactionInsert::new(
"abc",
1,
std::option::Option::None,
false,
std::option::Option::None,
);
let input = match result {
std::result::Result::Ok(value) => value.with_raw_transaction_id(7),
std::result::Result::Err(error) => panic!("unexpected transaction error: {error}"),
@@ -694,6 +822,7 @@ mod tests {
1,
"0",
" ",
std::option::Option::None,
serde_json::json!([]),
serde_json::json!({}),
);
@@ -707,6 +836,7 @@ mod tests {
1,
"0",
"program",
std::option::Option::None,
serde_json::json!([]),
serde_json::json!({}),
);
@@ -725,6 +855,36 @@ mod tests {
" ",
"0/0",
"program",
std::option::Option::None,
serde_json::json!([]),
serde_json::json!({}),
);
assert!(result.is_err());
}
#[test]
fn core_instruction_rejects_zero_stack_height() {
let result = crate::CoreInstructionInsert::new(
"abc",
1,
"0",
"program",
std::option::Option::Some(0),
serde_json::json!([]),
serde_json::json!({}),
);
assert!(result.is_err());
}
#[test]
fn core_inner_instruction_rejects_top_level_stack_height() {
let result = crate::CoreInnerInstructionInsert::new(
"abc",
1,
"0",
"0/0",
"program",
std::option::Option::Some(1),
serde_json::json!([]),
serde_json::json!({}),
);
@@ -762,6 +922,23 @@ mod tests {
assert!(result.is_err());
}
#[test]
fn core_account_state_requires_positive_lineage() {
let state = crate::CoreAccountStateInsert {
source_observation_id: 0,
account_key: "11111111111111111111111111111111".to_string(),
slot: 1,
owner: "11111111111111111111111111111111".to_string(),
lamports: 0,
executable: false,
rent_epoch: 0,
space: 0,
data_base64: std::string::String::new(),
data_hash: "hash".to_string(),
};
assert!(state.validate().is_err());
}
#[test]
fn replay_input_rejects_empty_key() {
let result = crate::MdCoreInstructionReplayInput::new(
@@ -785,7 +962,7 @@ mod tests {
}
#[test]
fn replay_input_accepts_ordered_outer_instruction_array() {
fn replay_input_accepts_ordered_top_level_instruction_array() {
let result = crate::MdCoreInstructionReplayInput::new(
"abc:2",
"abc",
@@ -822,15 +999,15 @@ mod tests {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => panic!("unexpected replay input error: {error}"),
};
assert_eq!(input.core_contract_version, 2);
assert_eq!(input.core_contract_version, crate::MD_CORE_REPLAY_INPUT_CONTRACT_VERSION);
assert_eq!(
input.outer_instructions_json.as_array().map(std::vec::Vec::len),
input.top_level_instructions_json.as_array().map(std::vec::Vec::len),
std::option::Option::Some(2)
);
}
#[test]
fn replay_input_rejects_non_array_outer_instruction_context() {
fn replay_input_rejects_non_array_top_level_instruction_context() {
let result = crate::MdCoreInstructionReplayInput::new(
"abc:0",
"abc",

View File

@@ -1,5 +1,5 @@
// file: ks-store/src/contracts/dto/core_extraction.rs
// version: 2
// version: 3
//! Canonical transaction to core extraction storage contracts.
@@ -31,7 +31,7 @@ pub struct CoreExtractionSelectionFilter {
pub limit: u32,
}
impl CoreExtractionSelectionFilter {
impl crate::CoreExtractionSelectionFilter {
/// Builds a validated extraction selection filter.
pub fn new(
signatures: std::vec::Vec<std::string::String>,
@@ -105,7 +105,7 @@ pub struct ProcessingLedgerIdentity {
pub input_hash: std::string::String,
}
impl ProcessingLedgerIdentity {
impl crate::ProcessingLedgerIdentity {
/// Builds a validated processing ledger identity.
pub fn new(
stage: impl std::convert::Into<std::string::String>,
@@ -154,9 +154,11 @@ pub struct CoreExtractionBundle {
pub logs: std::vec::Vec<crate::CoreLogInsert>,
/// Native and token balance changes.
pub balance_changes: std::vec::Vec<crate::CoreBalanceChangeInsert>,
/// Optional transaction return data retained by Core.
pub return_data: std::option::Option<crate::CoreReturnDataInsert>,
}
impl CoreExtractionBundle {
impl crate::CoreExtractionBundle {
/// Validates lineage and stable signature consistency across the bundle.
pub fn validate(&self) -> ks_core::Result<()> {
if self.raw_transaction_id <= 0 {
@@ -211,6 +213,13 @@ impl CoreExtractionBundle {
));
}
}
if let std::option::Option::Some(return_data) = &self.return_data {
if return_data.signature != signature {
return std::result::Result::Err(ks_core::Error::db(
"core extraction return-data signature mismatch",
));
}
}
return std::result::Result::Ok(());
}
}
@@ -228,7 +237,7 @@ pub struct CoreExtractionFailure {
pub error_message: std::string::String,
}
impl CoreExtractionFailure {
impl crate::CoreExtractionFailure {
/// Builds a validated failure record.
pub fn new(
raw_transaction_id: i64,

View File

@@ -1,14 +1,14 @@
// file: ks-store/src/contracts/dto/decode.rs
// version: 5
// version: 6
//! Backend-neutral decode, coverage and materialization persistence DTOs.
/// Maximum number of materialized rows returned by one bounded query.
pub const MAX_MATERIALIZED_EVENT_QUERY_ROWS: u32 = 500;
pub const MAX_MATERIALIZED_OUTPUT_QUERY_ROWS: u32 = 500;
/// Bounded read-only materialized event selection.
/// Bounded read-only materialized output selection.
#[derive(Clone, Debug, Eq, PartialEq, serde::Deserialize, serde::Serialize)]
pub struct MaterializedEventFilter {
pub struct MaterializedOutputFilter {
/// Optional exact materializer processor name.
pub processor_name: std::option::Option<std::string::String>,
/// Optional exact materialized family code.
@@ -19,18 +19,18 @@ pub struct MaterializedEventFilter {
pub limit: u32,
}
impl MaterializedEventFilter {
/// Builds and validates a bounded materialized event filter.
impl crate::MaterializedOutputFilter {
/// Builds and validates a bounded materialized output filter.
pub fn new(
processor_name: std::option::Option<std::string::String>,
materialized_family: std::option::Option<std::string::String>,
signature_contains: std::option::Option<std::string::String>,
limit: u32,
) -> ks_core::Result<Self> {
if limit == 0 || limit > crate::MAX_MATERIALIZED_EVENT_QUERY_ROWS {
if limit == 0 || limit > crate::MAX_MATERIALIZED_OUTPUT_QUERY_ROWS {
return std::result::Result::Err(ks_core::Error::db(format!(
"materialized event query limit must be between 1 and {}",
crate::MAX_MATERIALIZED_EVENT_QUERY_ROWS
"materialized output query limit must be between 1 and {}",
crate::MAX_MATERIALIZED_OUTPUT_QUERY_ROWS
)));
}
return std::result::Result::Ok(Self {
@@ -48,7 +48,7 @@ impl MaterializedEventFilter {
/// One materialized output returned by the common bounded query contract.
#[derive(Clone, Debug, PartialEq, serde::Deserialize, serde::Serialize)]
pub struct MaterializedEventQueryRow {
pub struct MaterializedOutputQueryRow {
/// Materializer processor name.
pub processor_name: std::string::String,
/// Materializer processor version.
@@ -98,7 +98,7 @@ pub struct DecodeSelectionFilter {
pub limit: u32,
}
impl DecodeSelectionFilter {
impl crate::DecodeSelectionFilter {
/// Builds a validated bounded decode selection filter.
#[allow(clippy::too_many_arguments)]
pub fn new(
@@ -165,6 +165,38 @@ impl DecodeSelectionFilter {
}
}
/// Optional external schema provenance used by a decoder for one observation.
#[derive(Clone, Debug, Eq, PartialEq, serde::Deserialize, serde::Serialize)]
pub struct DecodeSchemaProvenance {
/// Stable schema mechanism code, for example an IDL family or another external schema system.
pub schema_kind: std::string::String,
/// Stable schema identity within the selected mechanism.
pub schema_id: std::string::String,
/// Optional source schema version when one is published independently from the decoder.
pub schema_version: std::option::Option<std::string::String>,
/// Deterministic schema content hash used to reproduce the decode.
pub schema_hash: std::string::String,
}
impl crate::DecodeSchemaProvenance {
/// Validates the source-neutral schema identity without assuming a specific IDL mechanism.
pub fn validate(&self) -> ks_core::Result<()> {
if self.schema_kind.trim().is_empty()
|| self.schema_id.trim().is_empty()
|| self.schema_hash.trim().is_empty()
|| self
.schema_version
.as_deref()
.is_some_and(|value| return value.trim().is_empty())
{
return std::result::Result::Err(ks_core::Error::db(
"decode schema provenance fields must not be empty",
));
}
return std::result::Result::Ok(());
}
}
/// One processor-owned decoded observation row.
#[derive(Clone, Debug, PartialEq, serde::Deserialize, serde::Serialize)]
pub struct DecodeObservationInsert {
@@ -206,6 +238,8 @@ pub struct DecodeObservationInsert {
pub proof_json: serde_json::Value,
/// Typed decoded payload JSON.
pub payload_json: serde_json::Value,
/// Optional external schema provenance when decoding depends on a schema independent of decoder code.
pub schema_provenance: std::option::Option<crate::DecodeSchemaProvenance>,
/// Whether the source transaction failed on-chain.
pub transaction_failed: bool,
/// Optional source transaction error JSON.
@@ -214,7 +248,7 @@ pub struct DecodeObservationInsert {
pub observation_committed: bool,
}
impl DecodeObservationInsert {
impl crate::DecodeObservationInsert {
/// Validates stable identities and failed transaction commit semantics.
pub fn validate(&self) -> ks_core::Result<()> {
let fields = [
@@ -240,6 +274,12 @@ impl DecodeObservationInsert {
"decoded observation identity fields must not be empty",
));
}
if let std::option::Option::Some(schema_provenance) = &self.schema_provenance {
let schema_result = schema_provenance.validate();
if let std::result::Result::Err(error) = schema_result {
return std::result::Result::Err(error);
}
}
if self.transaction_failed && self.observation_committed {
return std::result::Result::Err(ks_core::Error::db(
"failed transaction decoded observations must not be committed",
@@ -325,12 +365,12 @@ pub struct DecodePersistenceBundle {
/// Optional human-readable terminal error message.
pub error_message: std::option::Option<std::string::String>,
/// Processor-owned decoded observations.
pub observations: std::vec::Vec<DecodeObservationInsert>,
pub observations: std::vec::Vec<crate::DecodeObservationInsert>,
/// Coverage observation for this attempt.
pub coverage: DecodeCoverageObservationInsert,
pub coverage: crate::DecodeCoverageObservationInsert,
}
impl DecodePersistenceBundle {
impl crate::DecodePersistenceBundle {
/// Validates identities shared by every atomic decode output.
pub fn validate(&self) -> ks_core::Result<()> {
if self.ledger_identity.stage != "instruction_decode"
@@ -471,10 +511,10 @@ pub struct MaterializationPersistenceBundle {
/// Optional human-readable terminal error message.
pub error_message: std::option::Option<std::string::String>,
/// Processor-owned materialized outputs.
pub outputs: std::vec::Vec<MaterializedOutputInsert>,
pub outputs: std::vec::Vec<crate::MaterializedOutputInsert>,
}
impl MaterializationPersistenceBundle {
impl crate::MaterializationPersistenceBundle {
/// Validates the materializer, source decoder and output identities.
pub fn validate(&self) -> ks_core::Result<()> {
if self.ledger_identity.stage != "event_materialization"
@@ -630,12 +670,12 @@ mod tests {
}
#[test]
fn materialized_event_filter_is_bounded_and_trims_optional_text() {
let result = crate::MaterializedEventFilter::new(
fn materialized_output_filter_is_bounded_and_trims_optional_text() {
let result = crate::MaterializedOutputFilter::new(
std::option::Option::Some(" materializer.transaction.annotations ".to_string()),
std::option::Option::Some(" transaction_annotation ".to_string()),
std::option::Option::Some(" signature ".to_string()),
crate::MAX_MATERIALIZED_EVENT_QUERY_ROWS,
crate::MAX_MATERIALIZED_OUTPUT_QUERY_ROWS,
);
let filter = match result {
std::result::Result::Ok(value) => value,
@@ -647,7 +687,7 @@ mod tests {
);
assert_eq!(filter.signature_contains.as_deref(), std::option::Option::Some("signature"));
assert!(
crate::MaterializedEventFilter::new(
crate::MaterializedOutputFilter::new(
std::option::Option::None,
std::option::Option::None,
std::option::Option::None,
@@ -657,6 +697,24 @@ mod tests {
);
}
#[test]
fn decode_schema_provenance_requires_reproducible_identity() {
let valid = crate::DecodeSchemaProvenance {
schema_kind: "idl".to_string(),
schema_id: "program-schema".to_string(),
schema_version: std::option::Option::Some("1".to_string()),
schema_hash: "abc123".to_string(),
};
assert!(valid.validate().is_ok());
let invalid = crate::DecodeSchemaProvenance {
schema_kind: "idl".to_string(),
schema_id: " ".to_string(),
schema_version: std::option::Option::None,
schema_hash: "abc123".to_string(),
};
assert!(invalid.validate().is_err());
}
#[test]
fn failed_decoded_observation_cannot_be_committed() {
let input = crate::DecodeObservationInsert {
@@ -679,6 +737,7 @@ mod tests {
proof_kind: "exact_layout".to_string(),
proof_json: serde_json::json!({}),
payload_json: serde_json::json!({}),
schema_provenance: std::option::Option::None,
transaction_failed: true,
transaction_error: std::option::Option::Some(serde_json::json!({})),
observation_committed: true,

View File

@@ -1,5 +1,5 @@
// file: ks-store/src/contracts/dto/raw.rs
// version: 2
// version: 3
//! Canonical Solana transaction and acquisition observation storage DTOs.
@@ -31,35 +31,35 @@ pub enum RawPayloadProcessingState {
Failed,
}
/// Origin of one transaction acquisition observation.
/// Origin of one source acquisition observation.
#[derive(Clone, Copy, Debug, Eq, PartialEq, serde::Deserialize, serde::Serialize)]
pub enum TransactionObservationOrigin {
/// Transaction was observed from a current live stream.
pub enum AcquisitionObservationOrigin {
/// Data was observed from a current live stream.
Live,
/// Transaction was acquired by an explicit historical backfill.
/// Data was acquired by an explicit historical backfill.
Backfill,
/// Transaction was replayed from an already captured source.
/// Data was replayed from an already captured source.
Replay,
/// Transaction was fetched to repair an acquisition gap.
/// Data was fetched to repair an acquisition gap.
Repair,
/// Observation was converted from a historical storage table.
Migration,
}
/// Technical status of one transaction acquisition observation.
/// Technical status of one source acquisition observation.
#[derive(Clone, Copy, Debug, Eq, PartialEq, serde::Deserialize, serde::Serialize)]
pub enum TransactionObservationStatus {
/// A transaction candidate was detected before a complete payload was received.
pub enum AcquisitionObservationStatus {
/// A source candidate was detected before a complete payload was received.
Detected,
/// A source payload was received.
Received,
/// A source payload was normalized into the canonical transaction contract.
/// A source payload was normalized into its canonical N1 contract.
Normalized,
/// The observation and any linked canonical transaction were persisted.
/// The observation and any linked canonical payload were persisted.
Persisted,
/// Acquisition or normalization failed.
Failed,
/// The source reported or implied a transaction that was temporarily unavailable.
/// The source reported that the requested source object was unavailable.
Missing,
}
@@ -70,6 +70,8 @@ pub struct RawTransactionInsert {
pub signature: std::string::String,
/// Transaction slot in Solana unsigned representation.
pub slot: u64,
/// Optional block timestamp as Unix seconds supplied by Solana.
pub block_time: std::option::Option<i64>,
/// Canonical source-independent transaction document.
pub canonical_json: serde_json::Value,
/// Optional deterministic digest of the canonical document.
@@ -78,11 +80,12 @@ pub struct RawTransactionInsert {
pub canonical_format_version: u32,
}
impl RawTransactionInsert {
impl crate::RawTransactionInsert {
/// Builds a canonical raw transaction insert contract after minimal validation.
pub fn new(
signature: impl std::convert::Into<std::string::String>,
slot: u64,
block_time: std::option::Option<i64>,
canonical_json: serde_json::Value,
canonical_format_version: u32,
) -> ks_core::Result<Self> {
@@ -100,6 +103,7 @@ impl RawTransactionInsert {
return std::result::Result::Ok(Self {
signature: signature_value,
slot,
block_time,
canonical_json,
canonical_json_hash: std::option::Option::None,
canonical_format_version,
@@ -121,6 +125,7 @@ impl RawTransactionInsert {
let insert_result = Self::new(
transaction.primary_signature.clone(),
transaction.slot,
transaction.block_time,
canonical_json,
transaction.format_version,
);
@@ -147,6 +152,115 @@ impl RawTransactionInsert {
}
}
/// Generic Solana account acquisition observation retained for future account decoders.
#[derive(Clone, Debug, PartialEq, serde::Deserialize, serde::Serialize)]
pub struct AccountObservationInsert {
/// Stable observation key used for idempotence.
pub observation_key: std::string::String,
/// Observed account public key.
pub account_key: std::string::String,
/// RPC or stream context slot associated with this observation.
pub context_slot: u64,
/// Account owner when the account exists.
pub owner: std::option::Option<std::string::String>,
/// Account lamports when the account exists.
pub lamports: std::option::Option<u64>,
/// Executable flag when the account exists.
pub executable: std::option::Option<bool>,
/// Rent epoch when the account exists.
pub rent_epoch: std::option::Option<u64>,
/// Complete account-data size when known.
pub space: std::option::Option<u64>,
/// Exact account bytes encoded as standard base64 when captured.
pub data_base64: std::option::Option<std::string::String>,
/// Deterministic digest of the decoded account bytes when captured.
pub data_hash: std::option::Option<std::string::String>,
/// Acquisition provider code.
pub provider: std::string::String,
/// Optional configured endpoint code.
pub endpoint_code: std::option::Option<std::string::String>,
/// Acquisition protocol.
pub protocol: std::string::String,
/// Acquisition method.
pub acquisition_method: std::string::String,
/// Optional commitment used for the read or subscription.
pub commitment: std::option::Option<std::string::String>,
/// Optional capture session identifier.
pub capture_session_id: std::option::Option<std::string::String>,
/// Optional configured filter code.
pub filter_code: std::option::Option<std::string::String>,
/// Acquisition origin.
pub origin: crate::AcquisitionObservationOrigin,
/// Optional first detection timestamp.
pub detected_at: std::option::Option<chrono::DateTime<chrono::Utc>>,
/// Timestamp at which the observation was received locally.
pub received_at: chrono::DateTime<chrono::Utc>,
/// Optional timestamp at which generic normalization completed.
pub normalized_at: std::option::Option<chrono::DateTime<chrono::Utc>>,
/// Optional uncompressed source payload size in bytes.
pub payload_size_bytes: std::option::Option<u64>,
/// Optional digest of the source-specific payload without retaining that payload.
pub source_payload_hash: std::option::Option<std::string::String>,
/// Technical observation status.
pub status: crate::AcquisitionObservationStatus,
/// Optional stable error code.
pub error_code: std::option::Option<std::string::String>,
/// Optional bounded diagnostic message.
pub error_message: std::option::Option<std::string::String>,
}
impl crate::AccountObservationInsert {
/// Validates the replayable account observation shape without assuming a decoder schema.
pub fn validate(&self) -> ks_core::Result<()> {
let required_result = validate_required_observation_texts(
self.observation_key.as_str(),
self.provider.as_str(),
self.protocol.as_str(),
self.acquisition_method.as_str(),
);
if let std::result::Result::Err(error) = required_result {
return std::result::Result::Err(error);
}
if self.account_key.trim().is_empty() {
return std::result::Result::Err(ks_core::Error::db(
"account observation account key must not be empty",
));
}
let optional_texts = [
self.owner.as_deref(),
self.endpoint_code.as_deref(),
self.commitment.as_deref(),
self.capture_session_id.as_deref(),
self.filter_code.as_deref(),
self.data_hash.as_deref(),
self.source_payload_hash.as_deref(),
self.error_code.as_deref(),
self.error_message.as_deref(),
];
if optional_texts.iter().flatten().any(|value| return value.trim().is_empty()) {
return std::result::Result::Err(ks_core::Error::db(
"account observation optional text fields must not be empty when present",
));
}
let state_fields_present = [
self.owner.is_some(),
self.lamports.is_some(),
self.executable.is_some(),
self.rent_epoch.is_some(),
self.space.is_some(),
self.data_base64.is_some(),
self.data_hash.is_some(),
];
let present_count = state_fields_present.iter().filter(|present| return **present).count();
if present_count != 0 && present_count != state_fields_present.len() {
return std::result::Result::Err(ks_core::Error::db(
"account observation state fields must be complete or entirely absent",
));
}
return std::result::Result::Ok(());
}
}
/// Lightweight transaction acquisition observation insert contract.
#[derive(Clone, Debug, PartialEq, serde::Deserialize, serde::Serialize)]
pub struct TransactionObservationInsert {
@@ -167,7 +281,7 @@ pub struct TransactionObservationInsert {
/// Source acquisition method.
pub acquisition_method: std::string::String,
/// Observation origin.
pub origin: TransactionObservationOrigin,
pub origin: crate::AcquisitionObservationOrigin,
/// Optional Solana commitment.
pub commitment: std::option::Option<std::string::String>,
/// Optional capture session identifier.
@@ -185,21 +299,21 @@ pub struct TransactionObservationInsert {
/// Optional digest of the source-specific payload without retaining that payload.
pub source_payload_hash: std::option::Option<std::string::String>,
/// Observation status.
pub status: TransactionObservationStatus,
pub status: crate::AcquisitionObservationStatus,
/// Optional machine-readable error code.
pub error_code: std::option::Option<std::string::String>,
/// Optional diagnostic error message.
pub error_message: std::option::Option<std::string::String>,
}
impl TransactionObservationInsert {
impl crate::TransactionObservationInsert {
/// Builds a lightweight transaction observation after validating required source metadata.
pub fn new(
observation_key: impl std::convert::Into<std::string::String>,
provider: impl std::convert::Into<std::string::String>,
protocol: impl std::convert::Into<std::string::String>,
acquisition_method: impl std::convert::Into<std::string::String>,
origin: TransactionObservationOrigin,
origin: crate::AcquisitionObservationOrigin,
received_at: chrono::DateTime<chrono::Utc>,
) -> ks_core::Result<Self> {
let observation_key_value = observation_key.into();
@@ -233,7 +347,7 @@ impl TransactionObservationInsert {
normalized_at: std::option::Option::None,
payload_size_bytes: std::option::Option::None,
source_payload_hash: std::option::Option::None,
status: TransactionObservationStatus::Received,
status: crate::AcquisitionObservationStatus::Received,
error_code: std::option::Option::None,
error_message: std::option::Option::None,
});
@@ -356,7 +470,7 @@ impl TransactionObservationInsert {
}
/// Replaces the current observation status.
pub fn with_status(mut self, status: TransactionObservationStatus) -> Self {
pub fn with_status(mut self, status: crate::AcquisitionObservationStatus) -> Self {
self.status = status;
return self;
}
@@ -384,7 +498,7 @@ impl TransactionObservationInsert {
}
self.error_code = std::option::Option::Some(error_code_value);
self.error_message = error_message;
self.status = TransactionObservationStatus::Failed;
self.status = crate::AcquisitionObservationStatus::Failed;
return std::result::Result::Ok(self);
}
}
@@ -392,25 +506,25 @@ impl TransactionObservationInsert {
/// Canonical raw transaction lifecycle mark request.
#[derive(Clone, Debug, PartialEq, serde::Deserialize, serde::Serialize)]
pub struct RawPayloadLifecycleMark {
/// Physical canonical raw table name using the `kb_sol_<domain>_<name>` convention.
/// Physical canonical raw table name using the `k_sol_<domain>_<name>` convention.
pub raw_table_name: std::string::String,
/// Stable canonical raw row key, currently the transaction signature.
pub raw_row_key: std::string::String,
/// Retention state to record for the canonical payload.
pub retention_state: RawPayloadRetentionState,
pub retention_state: crate::RawPayloadRetentionState,
/// Processing state to record for the canonical payload.
pub processing_state: RawPayloadProcessingState,
pub processing_state: crate::RawPayloadProcessingState,
/// Optional reason visible in diagnostics.
pub reason: std::option::Option<std::string::String>,
}
impl RawPayloadLifecycleMark {
impl crate::RawPayloadLifecycleMark {
/// Builds a canonical raw payload lifecycle mark after minimal validation.
pub fn new(
raw_table_name: impl std::convert::Into<std::string::String>,
raw_row_key: impl std::convert::Into<std::string::String>,
retention_state: RawPayloadRetentionState,
processing_state: RawPayloadProcessingState,
retention_state: crate::RawPayloadRetentionState,
processing_state: crate::RawPayloadProcessingState,
reason: std::option::Option<std::string::String>,
) -> ks_core::Result<Self> {
let raw_table_name_value = raw_table_name.into();
@@ -488,13 +602,25 @@ fn validate_optional_owned_text(
mod tests {
#[test]
fn raw_transaction_rejects_empty_signature() {
let result = crate::RawTransactionInsert::new(" ", 1, serde_json::json!({"ok": true}), 1);
let result = crate::RawTransactionInsert::new(
" ",
1,
std::option::Option::None,
serde_json::json!({"ok": true}),
1,
);
assert!(result.is_err());
}
#[test]
fn raw_transaction_rejects_zero_format_version() {
let result = crate::RawTransactionInsert::new("abc", 1, serde_json::json!({"ok": true}), 0);
let result = crate::RawTransactionInsert::new(
"abc",
1,
std::option::Option::None,
serde_json::json!({"ok": true}),
0,
);
assert!(result.is_err());
}
@@ -565,10 +691,49 @@ mod tests {
#[test]
fn raw_transaction_accepts_canonical_payload() {
let result = crate::RawTransactionInsert::new("abc", 1, serde_json::json!({"ok": true}), 1);
let result = crate::RawTransactionInsert::new(
"abc",
1,
std::option::Option::None,
serde_json::json!({"ok": true}),
1,
);
assert!(result.is_ok());
}
#[test]
fn account_observation_requires_complete_or_absent_state() {
let observation = crate::AccountObservationInsert {
observation_key: "account:1".to_string(),
account_key: "11111111111111111111111111111111".to_string(),
context_slot: 1,
owner: std::option::Option::Some("11111111111111111111111111111111".to_string()),
lamports: std::option::Option::None,
executable: std::option::Option::Some(false),
rent_epoch: std::option::Option::Some(0),
space: std::option::Option::Some(0),
data_base64: std::option::Option::Some(std::string::String::new()),
data_hash: std::option::Option::Some("hash".to_string()),
provider: "rpc".to_string(),
endpoint_code: std::option::Option::None,
protocol: "solana_http".to_string(),
acquisition_method: "getAccountInfo".to_string(),
commitment: std::option::Option::None,
capture_session_id: std::option::Option::None,
filter_code: std::option::Option::None,
origin: crate::AcquisitionObservationOrigin::Backfill,
detected_at: std::option::Option::None,
received_at: chrono::Utc::now(),
normalized_at: std::option::Option::None,
payload_size_bytes: std::option::Option::None,
source_payload_hash: std::option::Option::None,
status: crate::AcquisitionObservationStatus::Received,
error_code: std::option::Option::None,
error_message: std::option::Option::None,
};
assert!(observation.validate().is_err());
}
#[test]
fn transaction_observation_rejects_empty_provider() {
let result = crate::TransactionObservationInsert::new(
@@ -576,7 +741,7 @@ mod tests {
" ",
"solana_http",
"getTransaction",
crate::TransactionObservationOrigin::Backfill,
crate::AcquisitionObservationOrigin::Backfill,
chrono::Utc::now(),
);
assert!(result.is_err());
@@ -589,7 +754,7 @@ mod tests {
"helius",
"solana_http",
"getTransaction",
crate::TransactionObservationOrigin::Repair,
crate::AcquisitionObservationOrigin::Repair,
chrono::Utc::now(),
);
assert!(result.is_ok());
@@ -598,7 +763,7 @@ mod tests {
#[test]
fn raw_lifecycle_rejects_empty_reason_when_present() {
let result = crate::RawPayloadLifecycleMark::new(
"kb_sol_raw_transactions",
"k_sol_raw_transactions",
"signature",
crate::RawPayloadRetentionState::Full,
crate::RawPayloadProcessingState::Received,

View File

@@ -1,22 +1,22 @@
// file: ks-store/src/contracts/dto/store.rs
// version: 5
// version: 6
//! Backend-agnostic store configuration and diagnostic DTOs.
/// Store migration status contract.
/// Store schema evolution status contract.
#[derive(Clone, Copy, Debug, Eq, PartialEq, serde::Deserialize, serde::Serialize)]
pub enum StoreMigrationStatus {
/// Migration status is not known yet.
/// Schema evolution status is not known yet.
Unknown,
/// Store has no migration table or migration history yet.
/// Store baseline schema is not initialized yet.
NotInitialized,
/// Store migrations are current.
/// Store schema contract is current.
Current,
/// Store has pending migrations.
/// Store has pending schema evolution work.
Pending,
/// Store migration history is inconsistent.
/// Store schema contract is partially applied or inconsistent.
Drift,
/// Store migration check failed.
/// Store schema contract verification failed.
Failed,
}
@@ -99,21 +99,21 @@ impl crate::StoreBackendDescriptor {
}
}
/// Store migration diagnostic snapshot contract.
/// Store schema evolution diagnostic snapshot contract.
#[derive(Clone, Debug, Eq, PartialEq, serde::Deserialize, serde::Serialize)]
pub struct StoreMigrationSnapshot {
/// Current migration status.
/// Current schema evolution status.
pub status: crate::StoreMigrationStatus,
/// Last applied migration identifier when known.
/// Current schema contract version when known.
pub current_version: std::option::Option<std::string::String>,
/// Pending migration identifiers when known.
/// Pending schema evolution identifiers when known.
pub pending_versions: std::vec::Vec<std::string::String>,
/// Optional human-readable diagnostic message.
pub message: std::option::Option<std::string::String>,
}
impl crate::StoreMigrationSnapshot {
/// Builds a migration snapshot from explicit values.
/// Builds a schema evolution snapshot from explicit values.
pub fn new(
status: crate::StoreMigrationStatus,
current_version: std::option::Option<std::string::String>,

View File

@@ -1,5 +1,5 @@
// file: ks-store/src/contracts/entity.rs
// version: 5
// version: 6
//! Backend-neutral persisted entity exports for storage adapters.
@@ -8,6 +8,8 @@ mod raw;
/// Core account key persisted row contract.
pub use self::core::CoreAccountKeyRow;
/// Core account-state persisted row contract.
pub use self::core::CoreAccountStateRow;
/// Core balance change persisted row contract.
pub use self::core::CoreBalanceChangeRow;
/// Core inner instruction persisted row contract.
@@ -16,8 +18,12 @@ pub use self::core::CoreInnerInstructionRow;
pub use self::core::CoreInstructionRow;
/// Core log persisted row contract.
pub use self::core::CoreLogRow;
/// Core return-data persisted row contract.
pub use self::core::CoreReturnDataRow;
/// Core transaction persisted row contract.
pub use self::core::CoreTransactionRow;
/// Generic account acquisition observation persisted row contract.
pub use self::raw::AccountObservationRow;
/// Canonical raw Solana transaction persisted row contract.
pub use self::raw::RawTransactionRow;
/// Transaction acquisition observation persisted row contract.

View File

@@ -1,5 +1,5 @@
// file: ks-store/src/contracts/entity/core.rs
// version: 3
// version: 4
//! Core Solana persisted entities.
@@ -12,6 +12,8 @@ pub struct CoreTransactionRow {
pub signature: std::string::String,
/// Transaction slot stored as SQL `BIGINT`.
pub slot: i64,
/// Optional block timestamp as Unix seconds supplied by Solana.
pub block_time: std::option::Option<i64>,
/// Whether the transaction failed on-chain.
pub failed: bool,
/// Optional canonical raw transaction row id used for lineage when available.
@@ -51,6 +53,35 @@ pub struct CoreAccountKeyRow {
pub created_at: chrono::DateTime<chrono::Utc>,
}
/// Canonical Core account-state persisted row contract.
#[derive(Clone, Debug, PartialEq, serde::Deserialize, serde::Serialize)]
pub struct CoreAccountStateRow {
/// Technical primary key.
pub id: i64,
/// Source account observation technical key.
pub source_observation_id: i64,
/// Account public key.
pub account_key: std::string::String,
/// Context slot stored as SQL `BIGINT`.
pub slot: i64,
/// Owning program id.
pub owner: std::string::String,
/// Exact lamports representation returned by PostgreSQL.
pub lamports: std::string::String,
/// Whether the account is executable.
pub executable: bool,
/// Exact rent-epoch representation returned by PostgreSQL.
pub rent_epoch: std::string::String,
/// Complete account-data length.
pub space: i64,
/// Exact account bytes encoded as standard base64.
pub data_base64: std::string::String,
/// Deterministic digest of decoded account bytes.
pub data_hash: std::string::String,
/// Insert timestamp.
pub created_at: chrono::DateTime<chrono::Utc>,
}
/// Core instruction persisted row contract.
#[derive(Clone, Debug, PartialEq, serde::Deserialize, serde::Serialize)]
pub struct CoreInstructionRow {
@@ -66,6 +97,8 @@ pub struct CoreInstructionRow {
pub instruction_path: std::string::String,
/// Program id as non-empty base58 text.
pub program_id: std::string::String,
/// Optional runtime invocation stack height.
pub stack_height: std::option::Option<i32>,
/// Instruction accounts as JSON, preserving unresolved forms when needed.
pub accounts_json: serde_json::Value,
/// Instruction payload JSON while it is still retained in the hot store.
@@ -97,12 +130,37 @@ pub struct CoreInnerInstructionRow {
pub instruction_path: std::string::String,
/// Program id as non-empty base58 text.
pub program_id: std::string::String,
/// Optional runtime invocation stack height.
pub stack_height: std::option::Option<i32>,
/// Inner instruction accounts as JSON.
pub accounts_json: serde_json::Value,
/// Inner instruction payload JSON while it is retained in the hot store.
pub payload_json: std::option::Option<serde_json::Value>,
/// Optional digest of the inner instruction payload after compaction or purge.
pub payload_json_hash: std::option::Option<std::string::String>,
/// Current processing state used by CPI replay.
pub processing_state: crate::CoreInstructionProcessingState,
/// Insert timestamp.
pub created_at: chrono::DateTime<chrono::Utc>,
/// Last lifecycle update timestamp.
pub updated_at: chrono::DateTime<chrono::Utc>,
}
/// Core return-data persisted row contract.
#[derive(Clone, Debug, PartialEq, serde::Deserialize, serde::Serialize)]
pub struct CoreReturnDataRow {
/// Technical primary key.
pub id: i64,
/// Parent transaction technical key.
pub transaction_id: i64,
/// Transaction signature.
pub signature: std::string::String,
/// Transaction slot stored as SQL `BIGINT`.
pub slot: i64,
/// Program id that produced the return data.
pub program_id: std::string::String,
/// Exact return bytes encoded as standard base64.
pub data_base64: std::string::String,
/// Insert timestamp.
pub created_at: chrono::DateTime<chrono::Utc>,
}

View File

@@ -1,5 +1,5 @@
// file: ks-store/src/contracts/entity/raw.rs
// version: 3
// version: 4
//! Canonical Solana transaction and acquisition observation persisted entities.
@@ -12,6 +12,8 @@ pub struct RawTransactionRow {
pub signature: std::string::String,
/// Transaction slot stored as SQL `BIGINT`.
pub slot: i64,
/// Optional block timestamp as Unix seconds supplied by Solana.
pub block_time: std::option::Option<i64>,
/// Canonical source-independent transaction document while retained in the hot store.
pub canonical_json: std::option::Option<serde_json::Value>,
/// Optional deterministic digest of the canonical document.
@@ -50,7 +52,7 @@ pub struct TransactionObservationRow {
/// Acquisition method, for example `getTransaction`, `transactionSubscribe` or `transactions`.
pub acquisition_method: std::string::String,
/// Acquisition origin category.
pub origin: crate::TransactionObservationOrigin,
pub origin: crate::AcquisitionObservationOrigin,
/// Optional Solana commitment.
pub commitment: std::option::Option<std::string::String>,
/// Optional capture session identifier.
@@ -70,9 +72,70 @@ pub struct TransactionObservationRow {
/// Optional digest of the source-specific payload without retaining that payload.
pub source_payload_hash: std::option::Option<std::string::String>,
/// Current observation status.
pub status: crate::TransactionObservationStatus,
pub status: crate::AcquisitionObservationStatus,
/// Optional machine-readable error code.
pub error_code: std::option::Option<std::string::String>,
/// Optional diagnostic error message.
pub error_message: std::option::Option<std::string::String>,
}
/// Generic account acquisition observation persisted row contract.
#[derive(Clone, Debug, PartialEq, serde::Deserialize, serde::Serialize)]
pub struct AccountObservationRow {
/// Technical primary key.
pub id: i64,
/// Stable observation key.
pub observation_key: std::string::String,
/// Observed account public key.
pub account_key: std::string::String,
/// Context slot stored as SQL `BIGINT`.
pub context_slot: i64,
/// Account owner when present.
pub owner: std::option::Option<std::string::String>,
/// Exact lamports rendered as decimal text when present.
pub lamports: std::option::Option<std::string::String>,
/// Executable flag when present.
pub executable: std::option::Option<bool>,
/// Exact rent epoch rendered as decimal text when present.
pub rent_epoch: std::option::Option<std::string::String>,
/// Account data length when present.
pub space: std::option::Option<i64>,
/// Exact account bytes encoded as standard base64 when captured.
pub data_base64: std::option::Option<std::string::String>,
/// Deterministic digest of decoded account bytes when captured.
pub data_hash: std::option::Option<std::string::String>,
/// Acquisition provider code.
pub provider: std::string::String,
/// Optional endpoint code.
pub endpoint_code: std::option::Option<std::string::String>,
/// Acquisition protocol.
pub protocol: std::string::String,
/// Acquisition method.
pub acquisition_method: std::string::String,
/// Optional commitment.
pub commitment: std::option::Option<std::string::String>,
/// Optional capture session id.
pub capture_session_id: std::option::Option<std::string::String>,
/// Optional configured filter code.
pub filter_code: std::option::Option<std::string::String>,
/// Acquisition origin.
pub origin: crate::AcquisitionObservationOrigin,
/// Optional detection timestamp.
pub detected_at: std::option::Option<chrono::DateTime<chrono::Utc>>,
/// Receive timestamp.
pub received_at: chrono::DateTime<chrono::Utc>,
/// Optional normalization timestamp.
pub normalized_at: std::option::Option<chrono::DateTime<chrono::Utc>>,
/// Persist timestamp.
pub persisted_at: chrono::DateTime<chrono::Utc>,
/// Optional uncompressed source payload size.
pub payload_size_bytes: std::option::Option<i64>,
/// Optional source-specific payload digest.
pub source_payload_hash: std::option::Option<std::string::String>,
/// Technical observation status.
pub status: crate::AcquisitionObservationStatus,
/// Optional error code.
pub error_code: std::option::Option<std::string::String>,
/// Optional diagnostic message.
pub error_message: std::option::Option<std::string::String>,
}

View File

@@ -1,5 +1,5 @@
// file: ks-store/src/contracts/health.rs
// version: 2
// version: 3
//! Backend-neutral health contracts for storage implementations.
@@ -22,16 +22,16 @@ pub struct StoreHealthSnapshot {
/// Stable backend code such as `postgres` or `sqlite`.
pub backend: std::string::String,
/// Current health status.
pub status: StoreHealthStatus,
pub status: crate::StoreHealthStatus,
/// Optional human-readable diagnostic message.
pub message: std::option::Option<std::string::String>,
}
impl StoreHealthSnapshot {
impl crate::StoreHealthSnapshot {
/// Builds a store health snapshot after minimal validation.
pub fn new(
backend: impl std::convert::Into<std::string::String>,
status: StoreHealthStatus,
status: crate::StoreHealthStatus,
message: std::option::Option<std::string::String>,
) -> ks_core::Result<Self> {
let backend_value = backend.into();

View File

@@ -1,5 +1,5 @@
// file: ks-store/src/contracts/pagination.rs
// version: 2
// version: 3
//! Backend-neutral pagination and sorting contracts for repository operations.
@@ -26,7 +26,7 @@ pub struct PageRequest {
pub offset: u64,
}
impl PageRequest {
impl crate::PageRequest {
/// Builds a page request after minimal bounds validation.
pub fn new(limit: u16, offset: u64) -> ks_core::Result<Self> {
if limit == 0 {

View File

@@ -1,5 +1,5 @@
// file: ks-store/src/contracts/repository.rs
// version: 3
// version: 4
//! Storage trait definitions shared by concrete stores.
@@ -187,8 +187,8 @@ pub trait DecodePipelineStore {
) -> ks_core::Result<std::vec::Vec<crate::DecodeCoverageSummaryRow>>;
/// Lists bounded materialized outputs for read-only application views.
async fn list_materialized_events(
async fn list_materialized_outputs(
&self,
filter: &crate::MaterializedEventFilter,
) -> ks_core::Result<std::vec::Vec<crate::MaterializedEventQueryRow>>;
filter: &crate::MaterializedOutputFilter,
) -> ks_core::Result<std::vec::Vec<crate::MaterializedOutputQueryRow>>;
}

View File

@@ -1,5 +1,5 @@
// file: ks-store/src/lib.rs
// version: 10
// version: 12
//! Backend-agnostic Solana storage contracts and store facade.
#![warn(missing_docs)]
@@ -13,8 +13,12 @@ mod store;
/// Canonical crate-internal tracing target shared through the crate-root facade.
pub(crate) use self::constants::TRACING_TARGET;
/// Crate-internal storage symbol `ACCOUNT_OBSERVATIONS_TABLE_NAME` shared through the crate-root facade.
pub(crate) use self::postgres::ACCOUNT_OBSERVATIONS_TABLE_NAME;
/// Crate-internal storage symbol `CORE_ACCOUNT_KEYS_TABLE_NAME` shared through the crate-root facade.
pub(crate) use self::postgres::CORE_ACCOUNT_KEYS_TABLE_NAME;
/// Crate-internal storage symbol `CORE_ACCOUNT_STATES_TABLE_NAME` shared through the crate-root facade.
pub(crate) use self::postgres::CORE_ACCOUNT_STATES_TABLE_NAME;
/// Crate-internal storage symbol `CORE_BALANCE_CHANGES_TABLE_NAME` shared through the crate-root facade.
pub(crate) use self::postgres::CORE_BALANCE_CHANGES_TABLE_NAME;
/// Crate-internal storage symbol `CORE_INNER_INSTRUCTIONS_TABLE_NAME` shared through the crate-root facade.
@@ -23,6 +27,8 @@ pub(crate) use self::postgres::CORE_INNER_INSTRUCTIONS_TABLE_NAME;
pub(crate) use self::postgres::CORE_INSTRUCTIONS_TABLE_NAME;
/// Crate-internal storage symbol `CORE_LOGS_TABLE_NAME` shared through the crate-root facade.
pub(crate) use self::postgres::CORE_LOGS_TABLE_NAME;
/// Crate-internal storage symbol `CORE_RETURN_DATA_TABLE_NAME` shared through the crate-root facade.
pub(crate) use self::postgres::CORE_RETURN_DATA_TABLE_NAME;
/// Crate-internal storage symbol `CORE_TRANSACTIONS_TABLE_NAME` shared through the crate-root facade.
pub(crate) use self::postgres::CORE_TRANSACTIONS_TABLE_NAME;
/// Crate-internal storage symbol `DECODE_COVERAGE_DECLARATIONS_TABLE_NAME` shared through the crate-root facade.
@@ -31,8 +37,10 @@ pub(crate) use self::postgres::DECODE_COVERAGE_DECLARATIONS_TABLE_NAME;
pub(crate) use self::postgres::DECODE_COVERAGE_OBSERVATIONS_TABLE_NAME;
/// Crate-internal storage symbol `DECODE_EVENTS_TABLE_NAME` shared through the crate-root facade.
pub(crate) use self::postgres::DECODE_EVENTS_TABLE_NAME;
/// Crate-internal storage symbol `MATERIALIZED_EVENTS_TABLE_NAME` shared through the crate-root facade.
pub(crate) use self::postgres::MATERIALIZED_EVENTS_TABLE_NAME;
/// Crate-internal storage symbol `LEGACY_SOLANA_TABLE_PREFIX` shared through the crate-root facade.
pub(crate) use self::postgres::LEGACY_SOLANA_TABLE_PREFIX;
/// Crate-internal storage symbol `MATERIALIZED_OUTPUTS_TABLE_NAME` shared through the crate-root facade.
pub(crate) use self::postgres::MATERIALIZED_OUTPUTS_TABLE_NAME;
/// Crate-internal storage symbol `PROCESSING_LEDGER_TABLE_NAME` shared through the crate-root facade.
pub(crate) use self::postgres::PROCESSING_LEDGER_TABLE_NAME;
/// Crate-internal storage symbol `PostgresStore` shared through the crate-root facade.
@@ -47,12 +55,8 @@ pub(crate) use self::postgres::RAW_TRANSACTIONS_TABLE_NAME;
pub(crate) use self::postgres::STORE_SCHEMA_ADVISORY_LOCK_ID;
/// Crate-internal storage symbol `TRANSACTION_OBSERVATIONS_TABLE_NAME` shared through the crate-root facade.
pub(crate) use self::postgres::TRANSACTION_OBSERVATIONS_TABLE_NAME;
/// Crate-internal storage symbol `apply_core_store_schema` shared through the crate-root facade.
pub(crate) use self::postgres::apply_core_store_schema;
/// Crate-internal storage symbol `apply_decode_store_schema` shared through the crate-root facade.
pub(crate) use self::postgres::apply_decode_store_schema;
/// Crate-internal storage symbol `apply_raw_store_schema` shared through the crate-root facade.
pub(crate) use self::postgres::apply_raw_store_schema;
/// Crate-internal complete PostgreSQL baseline initializer shared through the crate-root facade.
pub(crate) use self::postgres::apply_store_schema;
/// Crate-internal storage symbol `core_store_schema_statements` shared through the crate-root facade.
pub(crate) use self::postgres::core_store_schema_statements;
/// Crate-internal storage symbol `core_store_table_diagnostic_specs` shared through the crate-root facade.
@@ -61,6 +65,8 @@ pub(crate) use self::postgres::core_store_table_diagnostic_specs;
pub(crate) use self::postgres::decode_store_schema_statements;
/// Crate-internal storage symbol `decode_store_table_diagnostic_specs` shared through the crate-root facade.
pub(crate) use self::postgres::decode_store_table_diagnostic_specs;
/// Crate-internal expected PostgreSQL index inventory shared through the crate-root facade.
pub(crate) use self::postgres::expected_postgres_index_names;
/// Crate-internal storage symbol `has_raw_transaction_signature` shared through the crate-root facade.
pub(crate) use self::postgres::has_raw_transaction_signature;
/// Crate-internal storage symbol `has_transaction_observation_key` shared through the crate-root facade.
@@ -95,8 +101,8 @@ pub(crate) use self::postgres::list_decode_coverage_summary;
pub(crate) use self::postgres::list_decode_inputs;
/// Crate-internal storage symbol `list_decode_replay_inputs` shared through the crate-root facade.
pub(crate) use self::postgres::list_decode_replay_inputs;
/// Crate-internal storage symbol `list_materialized_events` shared through the crate-root facade.
pub(crate) use self::postgres::list_materialized_events;
/// Crate-internal storage symbol `list_materialized_outputs` shared through the crate-root facade.
pub(crate) use self::postgres::list_materialized_outputs;
/// Crate-internal storage symbol `list_raw_transactions_for_core_extraction` shared through the crate-root facade.
pub(crate) use self::postgres::list_raw_transactions_for_core_extraction;
/// Crate-internal storage symbol `list_replay_entity_summaries` shared through the crate-root facade.
@@ -107,10 +113,8 @@ pub(crate) use self::postgres::list_replay_program_summaries;
pub(crate) use self::postgres::list_replay_transaction_candidates;
/// Crate-internal storage symbol `load_current_schema` shared through the crate-root facade.
pub(crate) use self::postgres::load_current_schema;
/// Crate-internal storage symbol `load_latest_migration_version` shared through the crate-root facade.
pub(crate) use self::postgres::load_latest_migration_version;
/// Crate-internal storage symbol `load_migration_table_name` shared through the crate-root facade.
pub(crate) use self::postgres::load_migration_table_name;
/// Crate-internal expected PostgreSQL index counts shared through the crate-root facade.
pub(crate) use self::postgres::load_expected_index_counts;
/// Crate-internal storage symbol `load_server_version` shared through the crate-root facade.
pub(crate) use self::postgres::load_server_version;
/// Crate-internal storage symbol `load_table_statistics` shared through the crate-root facade.
@@ -138,51 +142,65 @@ pub(crate) use self::postgres::raw_store_table_diagnostic_specs;
pub(crate) use self::postgres::run_health_check;
/// Crate-internal storage symbol `table_exists` shared through the crate-root facade.
pub(crate) use self::postgres::table_exists;
/// Crate-internal storage symbol `table_stats_kb_sol_core_account_keys_sql` shared through the crate-root facade.
pub(crate) use self::postgres::table_stats_kb_sol_core_account_keys_sql;
/// Crate-internal storage symbol `table_stats_kb_sol_core_balance_changes_sql` shared through the crate-root facade.
pub(crate) use self::postgres::table_stats_kb_sol_core_balance_changes_sql;
/// Crate-internal storage symbol `table_stats_kb_sol_core_inner_instructions_sql` shared through the crate-root facade.
pub(crate) use self::postgres::table_stats_kb_sol_core_inner_instructions_sql;
/// Crate-internal storage symbol `table_stats_kb_sol_core_instructions_sql` shared through the crate-root facade.
pub(crate) use self::postgres::table_stats_kb_sol_core_instructions_sql;
/// Crate-internal storage symbol `table_stats_kb_sol_core_logs_sql` shared through the crate-root facade.
pub(crate) use self::postgres::table_stats_kb_sol_core_logs_sql;
/// Crate-internal storage symbol `table_stats_kb_sol_core_transactions_sql` shared through the crate-root facade.
pub(crate) use self::postgres::table_stats_kb_sol_core_transactions_sql;
/// Crate-internal storage symbol `table_stats_kb_sol_decode_coverage_declarations_sql` shared through the crate-root facade.
pub(crate) use self::postgres::table_stats_kb_sol_decode_coverage_declarations_sql;
/// Crate-internal storage symbol `table_stats_kb_sol_decode_coverage_observations_sql` shared through the crate-root facade.
pub(crate) use self::postgres::table_stats_kb_sol_decode_coverage_observations_sql;
/// Crate-internal storage symbol `table_stats_kb_sol_decode_events_sql` shared through the crate-root facade.
pub(crate) use self::postgres::table_stats_kb_sol_decode_events_sql;
/// Crate-internal storage symbol `table_stats_kb_sol_mat_events_sql` shared through the crate-root facade.
pub(crate) use self::postgres::table_stats_kb_sol_mat_events_sql;
/// Crate-internal storage symbol `table_stats_kb_sol_obs_transaction_observations_sql` shared through the crate-root facade.
pub(crate) use self::postgres::table_stats_kb_sol_obs_transaction_observations_sql;
/// Crate-internal storage symbol `table_stats_kb_sol_ops_processing_ledger_sql` shared through the crate-root facade.
pub(crate) use self::postgres::table_stats_kb_sol_ops_processing_ledger_sql;
/// Crate-internal storage symbol `table_stats_kb_sol_raw_transactions_sql` shared through the crate-root facade.
pub(crate) use self::postgres::table_stats_kb_sol_raw_transactions_sql;
/// Crate-internal storage symbol `table_stats_k_sol_core_account_keys_sql` shared through the crate-root facade.
pub(crate) use self::postgres::table_stats_k_sol_core_account_keys_sql;
/// Crate-internal storage symbol `table_stats_k_sol_core_account_states_sql` shared through the crate-root facade.
pub(crate) use self::postgres::table_stats_k_sol_core_account_states_sql;
/// Crate-internal storage symbol `table_stats_k_sol_core_balance_changes_sql` shared through the crate-root facade.
pub(crate) use self::postgres::table_stats_k_sol_core_balance_changes_sql;
/// Crate-internal storage symbol `table_stats_k_sol_core_inner_instructions_sql` shared through the crate-root facade.
pub(crate) use self::postgres::table_stats_k_sol_core_inner_instructions_sql;
/// Crate-internal storage symbol `table_stats_k_sol_core_instructions_sql` shared through the crate-root facade.
pub(crate) use self::postgres::table_stats_k_sol_core_instructions_sql;
/// Crate-internal storage symbol `table_stats_k_sol_core_logs_sql` shared through the crate-root facade.
pub(crate) use self::postgres::table_stats_k_sol_core_logs_sql;
/// Crate-internal storage symbol `table_stats_k_sol_core_return_data_sql` shared through the crate-root facade.
pub(crate) use self::postgres::table_stats_k_sol_core_return_data_sql;
/// Crate-internal storage symbol `table_stats_k_sol_core_transactions_sql` shared through the crate-root facade.
pub(crate) use self::postgres::table_stats_k_sol_core_transactions_sql;
/// Crate-internal storage symbol `table_stats_k_sol_decode_coverage_declarations_sql` shared through the crate-root facade.
pub(crate) use self::postgres::table_stats_k_sol_decode_coverage_declarations_sql;
/// Crate-internal storage symbol `table_stats_k_sol_decode_coverage_observations_sql` shared through the crate-root facade.
pub(crate) use self::postgres::table_stats_k_sol_decode_coverage_observations_sql;
/// Crate-internal storage symbol `table_stats_k_sol_decode_events_sql` shared through the crate-root facade.
pub(crate) use self::postgres::table_stats_k_sol_decode_events_sql;
/// Crate-internal storage symbol `table_stats_k_sol_mat_outputs_sql` shared through the crate-root facade.
pub(crate) use self::postgres::table_stats_k_sol_mat_outputs_sql;
/// Crate-internal storage symbol `table_stats_k_sol_obs_account_observations_sql` shared through the crate-root facade.
pub(crate) use self::postgres::table_stats_k_sol_obs_account_observations_sql;
/// Crate-internal storage symbol `table_stats_k_sol_obs_transaction_observations_sql` shared through the crate-root facade.
pub(crate) use self::postgres::table_stats_k_sol_obs_transaction_observations_sql;
/// Crate-internal storage symbol `table_stats_k_sol_ops_processing_ledger_sql` shared through the crate-root facade.
pub(crate) use self::postgres::table_stats_k_sol_ops_processing_ledger_sql;
/// Crate-internal storage symbol `table_stats_k_sol_raw_transactions_sql` shared through the crate-root facade.
pub(crate) use self::postgres::table_stats_k_sol_raw_transactions_sql;
/// Crate-internal storage symbol `update_core_instruction_lifecycle` shared through the crate-root facade.
pub(crate) use self::postgres::update_core_instruction_lifecycle;
/// Crate-internal storage symbol `update_raw_payload_lifecycle` shared through the crate-root facade.
pub(crate) use self::postgres::update_raw_payload_lifecycle;
/// Crate-internal storage symbol `validate_core_store_table_names` shared through the crate-root facade.
pub(crate) use self::postgres::validate_core_store_table_names;
/// Crate-internal storage symbol `validate_decode_store_table_names` shared through the crate-root facade.
pub(crate) use self::postgres::validate_decode_store_table_names;
/// Crate-internal storage symbol `validate_raw_store_table_names` shared through the crate-root facade.
pub(crate) use self::postgres::validate_raw_store_table_names;
/// Crate-internal PostgreSQL SQL-resource canary shared through the crate-root facade.
pub(crate) use self::postgres::validate_postgres_sql_resources;
/// Transitional logical schema contract identifier used until the `0.5.3-pre.3` rebuild.
pub use self::constants::STORE_SCHEMA_CONTRACT_VERSION;
/// Generic account acquisition observation insert contract.
pub use self::contracts::AccountObservationInsert;
/// Generic account acquisition observation persisted row contract.
pub use self::contracts::AccountObservationRow;
/// Transaction acquisition observation origin.
pub use self::contracts::AcquisitionObservationOrigin;
/// Generic account acquisition observation status.
pub use self::contracts::AcquisitionObservationStatus;
/// Core account key insert contract.
pub use self::contracts::CoreAccountKeyInsert;
/// Core account key persisted row contract.
pub use self::contracts::CoreAccountKeyRow;
/// Core account key source category.
pub use self::contracts::CoreAccountKeySource;
/// Core account-state insert contract.
pub use self::contracts::CoreAccountStateInsert;
/// Core account-state persisted row contract.
pub use self::contracts::CoreAccountStateRow;
/// Core balance change insert contract.
pub use self::contracts::CoreBalanceChangeInsert;
/// Core balance change kind.
@@ -215,6 +233,10 @@ pub use self::contracts::CoreInstructionRow;
pub use self::contracts::CoreLogInsert;
/// Core log persisted row contract.
pub use self::contracts::CoreLogRow;
/// Core return-data insert contract.
pub use self::contracts::CoreReturnDataInsert;
/// Core return-data persisted row contract.
pub use self::contracts::CoreReturnDataRow;
/// Core transaction insert contract.
pub use self::contracts::CoreTransactionInsert;
/// Core transaction persisted row contract.
@@ -237,24 +259,26 @@ pub use self::contracts::DecodeObservationInsert;
pub use self::contracts::DecodePersistenceBundle;
/// Contextual instruction decode and materialization storage behavior.
pub use self::contracts::DecodePipelineStore;
/// Source-neutral external decode schema provenance.
pub use self::contracts::DecodeSchemaProvenance;
/// Bounded contextual instruction selection filter for decode campaigns.
pub use self::contracts::DecodeSelectionFilter;
/// Insert or upsert result contract returned by repositories.
pub use self::contracts::InsertOutcome;
/// Maximum number of materialized rows returned by one bounded query.
pub use self::contracts::MAX_MATERIALIZED_EVENT_QUERY_ROWS;
pub use self::contracts::MAX_MATERIALIZED_OUTPUT_QUERY_ROWS;
/// Maximum repository page size.
pub use self::contracts::MAX_PAGE_SIZE;
/// Maximum number of rows returned by one replay candidate query.
pub use self::contracts::MAX_REPLAY_CANDIDATE_ROWS;
/// Atomic persistence bundle for one materializer and one decoded observation.
pub use self::contracts::MaterializationPersistenceBundle;
/// Bounded read-only materialized event selection.
pub use self::contracts::MaterializedEventFilter;
/// One materialized output returned by a bounded query.
pub use self::contracts::MaterializedEventQueryRow;
/// Bounded read-only materialized output selection.
pub use self::contracts::MaterializedOutputFilter;
/// One processor-owned generic materialized output row.
pub use self::contracts::MaterializedOutputInsert;
/// One materialized output returned by a bounded query.
pub use self::contracts::MaterializedOutputQueryRow;
/// Page request contract for repository list operations.
pub use self::contracts::PageRequest;
/// Stable processing ledger identity.
@@ -325,12 +349,8 @@ pub use self::contracts::StoreResourceStatistics;
pub use self::contracts::StoreRuntimeSummary;
/// Transaction acquisition observation insert contract.
pub use self::contracts::TransactionObservationInsert;
/// Transaction acquisition observation origin.
pub use self::contracts::TransactionObservationOrigin;
/// Transaction acquisition observation persisted row contract.
pub use self::contracts::TransactionObservationRow;
/// Transaction acquisition observation status.
pub use self::contracts::TransactionObservationStatus;
/// Storage error helper functions.
pub use self::contracts::storage_contract_error;
/// Backend-agnostic persistent store facade.

View File

@@ -1,5 +1,5 @@
// file: ks-store/src/postgres.rs
// version: 11
// version: 13
//! Private PostgreSQL implementation of the backend-agnostic store contracts.
@@ -10,16 +10,20 @@ mod store;
#[cfg(test)]
mod test_serial;
pub(crate) use self::migrations::ACCOUNT_OBSERVATIONS_TABLE_NAME;
pub(crate) use self::migrations::CORE_ACCOUNT_KEYS_TABLE_NAME;
pub(crate) use self::migrations::CORE_ACCOUNT_STATES_TABLE_NAME;
pub(crate) use self::migrations::CORE_BALANCE_CHANGES_TABLE_NAME;
pub(crate) use self::migrations::CORE_INNER_INSTRUCTIONS_TABLE_NAME;
pub(crate) use self::migrations::CORE_INSTRUCTIONS_TABLE_NAME;
pub(crate) use self::migrations::CORE_LOGS_TABLE_NAME;
pub(crate) use self::migrations::CORE_RETURN_DATA_TABLE_NAME;
pub(crate) use self::migrations::CORE_TRANSACTIONS_TABLE_NAME;
pub(crate) use self::migrations::DECODE_COVERAGE_DECLARATIONS_TABLE_NAME;
pub(crate) use self::migrations::DECODE_COVERAGE_OBSERVATIONS_TABLE_NAME;
pub(crate) use self::migrations::DECODE_EVENTS_TABLE_NAME;
pub(crate) use self::migrations::MATERIALIZED_EVENTS_TABLE_NAME;
pub(crate) use self::migrations::LEGACY_SOLANA_TABLE_PREFIX;
pub(crate) use self::migrations::MATERIALIZED_OUTPUTS_TABLE_NAME;
pub(crate) use self::migrations::PROCESSING_LEDGER_TABLE_NAME;
pub(crate) use self::migrations::PostgresTableDiagnosticSpec;
pub(crate) use self::migrations::RAW_TRANSACTIONS_TABLE_NAME;
@@ -29,27 +33,27 @@ pub(crate) use self::migrations::core_store_schema_statements;
pub(crate) use self::migrations::core_store_table_diagnostic_specs;
pub(crate) use self::migrations::decode_store_schema_statements;
pub(crate) use self::migrations::decode_store_table_diagnostic_specs;
pub(crate) use self::migrations::expected_postgres_index_names;
pub(crate) use self::migrations::raw_store_schema_statements;
pub(crate) use self::migrations::raw_store_table_diagnostic_specs;
pub(crate) use self::migrations::table_stats_kb_sol_core_account_keys_sql;
pub(crate) use self::migrations::table_stats_kb_sol_core_balance_changes_sql;
pub(crate) use self::migrations::table_stats_kb_sol_core_inner_instructions_sql;
pub(crate) use self::migrations::table_stats_kb_sol_core_instructions_sql;
pub(crate) use self::migrations::table_stats_kb_sol_core_logs_sql;
pub(crate) use self::migrations::table_stats_kb_sol_core_transactions_sql;
pub(crate) use self::migrations::table_stats_kb_sol_decode_coverage_declarations_sql;
pub(crate) use self::migrations::table_stats_kb_sol_decode_coverage_observations_sql;
pub(crate) use self::migrations::table_stats_kb_sol_decode_events_sql;
pub(crate) use self::migrations::table_stats_kb_sol_mat_events_sql;
pub(crate) use self::migrations::table_stats_kb_sol_obs_transaction_observations_sql;
pub(crate) use self::migrations::table_stats_kb_sol_ops_processing_ledger_sql;
pub(crate) use self::migrations::table_stats_kb_sol_raw_transactions_sql;
pub(crate) use self::migrations::validate_core_store_table_names;
pub(crate) use self::migrations::validate_decode_store_table_names;
pub(crate) use self::migrations::validate_raw_store_table_names;
pub(crate) use self::query::apply_core_store_schema;
pub(crate) use self::query::apply_decode_store_schema;
pub(crate) use self::query::apply_raw_store_schema;
pub(crate) use self::migrations::table_stats_k_sol_core_account_keys_sql;
pub(crate) use self::migrations::table_stats_k_sol_core_account_states_sql;
pub(crate) use self::migrations::table_stats_k_sol_core_balance_changes_sql;
pub(crate) use self::migrations::table_stats_k_sol_core_inner_instructions_sql;
pub(crate) use self::migrations::table_stats_k_sol_core_instructions_sql;
pub(crate) use self::migrations::table_stats_k_sol_core_logs_sql;
pub(crate) use self::migrations::table_stats_k_sol_core_return_data_sql;
pub(crate) use self::migrations::table_stats_k_sol_core_transactions_sql;
pub(crate) use self::migrations::table_stats_k_sol_decode_coverage_declarations_sql;
pub(crate) use self::migrations::table_stats_k_sol_decode_coverage_observations_sql;
pub(crate) use self::migrations::table_stats_k_sol_decode_events_sql;
pub(crate) use self::migrations::table_stats_k_sol_mat_outputs_sql;
pub(crate) use self::migrations::table_stats_k_sol_obs_account_observations_sql;
pub(crate) use self::migrations::table_stats_k_sol_obs_transaction_observations_sql;
pub(crate) use self::migrations::table_stats_k_sol_ops_processing_ledger_sql;
pub(crate) use self::migrations::table_stats_k_sol_raw_transactions_sql;
pub(crate) use self::migrations::validate_postgres_sql_resources;
pub(crate) use self::query::apply_store_schema;
pub(crate) use self::query::has_raw_transaction_signature;
pub(crate) use self::query::has_transaction_observation_key;
pub(crate) use self::query::insert_core_account_keys;
@@ -67,14 +71,13 @@ pub(crate) use self::query::list_core_instructions_for_replay;
pub(crate) use self::query::list_decode_coverage_summary;
pub(crate) use self::query::list_decode_inputs;
pub(crate) use self::query::list_decode_replay_inputs;
pub(crate) use self::query::list_materialized_events;
pub(crate) use self::query::list_materialized_outputs;
pub(crate) use self::query::list_raw_transactions_for_core_extraction;
pub(crate) use self::query::list_replay_entity_summaries;
pub(crate) use self::query::list_replay_program_summaries;
pub(crate) use self::query::list_replay_transaction_candidates;
pub(crate) use self::query::load_current_schema;
pub(crate) use self::query::load_latest_migration_version;
pub(crate) use self::query::load_migration_table_name;
pub(crate) use self::query::load_expected_index_counts;
pub(crate) use self::query::load_server_version;
pub(crate) use self::query::load_table_statistics;
pub(crate) use self::query::mark_core_extraction_failed;

File diff suppressed because it is too large Load Diff

View File

@@ -1,5 +1,5 @@
// file: ks-store/src/postgres/query.rs
// version: 4
// version: 5
//! PostgreSQL query modules.
@@ -9,13 +9,13 @@ mod decode_pipeline_queries;
mod health_queries;
mod raw_queries;
mod replay_candidate_queries;
mod schema_queries;
mod table_diagnostics_queries;
pub(crate) use self::core_extraction_queries::is_core_extraction_current;
pub(crate) use self::core_extraction_queries::list_raw_transactions_for_core_extraction;
pub(crate) use self::core_extraction_queries::mark_core_extraction_failed;
pub(crate) use self::core_extraction_queries::persist_core_extraction;
pub(crate) use self::core_queries::apply_core_store_schema;
pub(crate) use self::core_queries::insert_core_account_keys;
pub(crate) use self::core_queries::insert_core_balance_changes;
pub(crate) use self::core_queries::insert_core_inner_instructions;
@@ -26,21 +26,17 @@ pub(crate) use self::core_queries::list_core_instruction_replay_inputs;
pub(crate) use self::core_queries::list_core_instructions_for_replay;
pub(crate) use self::core_queries::list_decode_replay_inputs;
pub(crate) use self::core_queries::update_core_instruction_lifecycle;
pub(crate) use self::decode_pipeline_queries::apply_decode_store_schema;
pub(crate) use self::decode_pipeline_queries::is_decode_current;
pub(crate) use self::decode_pipeline_queries::list_decode_coverage_summary;
pub(crate) use self::decode_pipeline_queries::list_decode_inputs;
pub(crate) use self::decode_pipeline_queries::list_materialized_events;
pub(crate) use self::decode_pipeline_queries::list_materialized_outputs;
pub(crate) use self::decode_pipeline_queries::mark_decode_failed;
pub(crate) use self::decode_pipeline_queries::persist_decode_coverage_declarations;
pub(crate) use self::decode_pipeline_queries::persist_decode_result;
pub(crate) use self::decode_pipeline_queries::persist_materialization_result;
pub(crate) use self::health_queries::load_current_schema;
pub(crate) use self::health_queries::load_latest_migration_version;
pub(crate) use self::health_queries::load_migration_table_name;
pub(crate) use self::health_queries::load_server_version;
pub(crate) use self::health_queries::run_health_check;
pub(crate) use self::raw_queries::apply_raw_store_schema;
pub(crate) use self::raw_queries::has_raw_transaction_signature;
pub(crate) use self::raw_queries::has_transaction_observation_key;
pub(crate) use self::raw_queries::insert_raw_transaction;
@@ -49,5 +45,7 @@ pub(crate) use self::raw_queries::update_raw_payload_lifecycle;
pub(crate) use self::replay_candidate_queries::list_replay_entity_summaries;
pub(crate) use self::replay_candidate_queries::list_replay_program_summaries;
pub(crate) use self::replay_candidate_queries::list_replay_transaction_candidates;
pub(crate) use self::schema_queries::apply_store_schema;
pub(crate) use self::schema_queries::load_expected_index_counts;
pub(crate) use self::table_diagnostics_queries::load_table_statistics;
pub(crate) use self::table_diagnostics_queries::table_exists;

View File

@@ -1,5 +1,5 @@
// file: ks-store/src/postgres/query/core_extraction_queries.rs
// version: 6
// version: 8
//! PostgreSQL queries for atomic canonical transaction to core extraction.
@@ -9,25 +9,21 @@ pub(crate) async fn list_raw_transactions_for_core_extraction(
pool: &sqlx::PgPool,
filter: &crate::CoreExtractionSelectionFilter,
) -> ks_core::Result<std::vec::Vec<crate::RawTransactionRow>> {
let min_slot_result =
crate::postgres::query::core_extraction_queries::optional_sql_slot(filter.min_slot);
let min_slot_result = optional_sql_slot(filter.min_slot);
let min_slot = match min_slot_result {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let max_slot_result =
crate::postgres::query::core_extraction_queries::optional_sql_slot(filter.max_slot);
let max_slot_result = optional_sql_slot(filter.max_slot);
let max_slot = match max_slot_result {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let processing_state = filter
.processing_state
.map(crate::postgres::query::core_extraction_queries::raw_processing_state_to_sql);
let processing_state = filter.processing_state.map(raw_processing_state_to_sql);
let limit = i64::from(filter.limit);
let signatures_are_empty = filter.signatures.is_empty();
let query_result = sqlx::query(
"SELECT raw.id, raw.signature, raw.slot, raw.canonical_json, raw.canonical_json_hash, raw.canonical_format_version, raw.retention_state, raw.processing_state, raw.created_at, raw.updated_at FROM kb_sol_raw_transactions raw WHERE ($1 OR raw.signature = ANY($2)) AND ($3::BIGINT IS NULL OR raw.slot >= $3) AND ($4::BIGINT IS NULL OR raw.slot <= $4) AND ($5::TEXT IS NULL OR raw.processing_state = $5) AND ($6::TEXT IS NULL OR EXISTS (SELECT 1 FROM kb_sol_core_instructions instruction WHERE instruction.signature = raw.signature AND instruction.program_id = $6)) ORDER BY raw.slot ASC, raw.signature ASC LIMIT $7",
"SELECT raw.id, raw.signature, raw.slot, raw.block_time, raw.canonical_json, raw.canonical_json_hash, raw.canonical_format_version, raw.retention_state, raw.processing_state, raw.created_at, raw.updated_at FROM k_sol_raw_transactions raw WHERE ($1 OR raw.signature = ANY($2)) AND ($3::BIGINT IS NULL OR raw.slot >= $3) AND ($4::BIGINT IS NULL OR raw.slot <= $4) AND ($5::TEXT IS NULL OR raw.processing_state = $5) AND ($6::TEXT IS NULL OR EXISTS (SELECT 1 FROM k_sol_core_instructions instruction WHERE instruction.signature = raw.signature AND instruction.program_id = $6)) ORDER BY raw.slot ASC, raw.signature ASC LIMIT $7",
)
.bind(signatures_are_empty)
.bind(&filter.signatures)
@@ -48,8 +44,7 @@ pub(crate) async fn list_raw_transactions_for_core_extraction(
};
let mut output = std::vec::Vec::with_capacity(rows.len());
for row in rows {
let mapped_result =
crate::postgres::query::core_extraction_queries::map_raw_transaction_row(&row);
let mapped_result = map_raw_transaction_row(&row);
let mapped = match mapped_result {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
@@ -64,7 +59,7 @@ pub(crate) async fn is_core_extraction_current(
identity: &crate::ProcessingLedgerIdentity,
) -> ks_core::Result<bool> {
let query_result = sqlx::query_scalar::<sqlx::Postgres, bool>(
"SELECT EXISTS(SELECT 1 FROM kb_sol_ops_processing_ledger WHERE stage = $1 AND processor_name = $2 AND processor_version = $3 AND input_key = $4 AND input_hash = $5 AND status = 'succeeded')",
"SELECT EXISTS(SELECT 1 FROM k_sol_ops_processing_ledger WHERE stage = $1 AND processor_name = $2 AND processor_version = $3 AND input_key = $4 AND input_hash = $5 AND status = 'succeeded')",
)
.bind(identity.stage.as_str())
.bind(identity.processor_name.as_str())
@@ -100,7 +95,7 @@ pub(crate) async fn persist_core_extraction(
},
};
let existing_result = sqlx::query_scalar::<sqlx::Postgres, i64>(
"SELECT id FROM kb_sol_core_transactions WHERE signature = $1 LIMIT 1",
"SELECT id FROM k_sol_core_transactions WHERE signature = $1 LIMIT 1",
)
.bind(bundle.transaction.signature.as_str())
.fetch_optional(&mut *transaction)
@@ -114,11 +109,10 @@ pub(crate) async fn persist_core_extraction(
},
};
if existing.is_some() {
let delete_result =
sqlx::query("DELETE FROM kb_sol_core_transactions WHERE signature = $1")
.bind(bundle.transaction.signature.as_str())
.execute(&mut *transaction)
.await;
let delete_result = sqlx::query("DELETE FROM k_sol_core_transactions WHERE signature = $1")
.bind(bundle.transaction.signature.as_str())
.execute(&mut *transaction)
.await;
if let std::result::Result::Err(error) = delete_result {
return std::result::Result::Err(ks_core::Error::db(format!(
"postgres core extraction replacement delete failed: {error}"
@@ -132,56 +126,51 @@ pub(crate) async fn persist_core_extraction(
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let account_result =
crate::postgres::query::core_extraction_queries::insert_account_keys_in_transaction(
&mut transaction,
transaction_id,
&bundle.account_keys,
)
.await;
insert_account_keys_in_transaction(&mut transaction, transaction_id, &bundle.account_keys)
.await;
if let std::result::Result::Err(error) = account_result {
return std::result::Result::Err(error);
}
let instruction_result =
crate::postgres::query::core_extraction_queries::insert_instructions_in_transaction(
&mut transaction,
transaction_id,
&bundle.instructions,
)
.await;
insert_instructions_in_transaction(&mut transaction, transaction_id, &bundle.instructions)
.await;
if let std::result::Result::Err(error) = instruction_result {
return std::result::Result::Err(error);
}
let inner_result =
crate::postgres::query::core_extraction_queries::insert_inner_instructions_in_transaction(
&mut transaction,
transaction_id,
&bundle.inner_instructions,
)
.await;
let inner_result = insert_inner_instructions_in_transaction(
&mut transaction,
transaction_id,
&bundle.inner_instructions,
)
.await;
if let std::result::Result::Err(error) = inner_result {
return std::result::Result::Err(error);
}
let log_result = crate::postgres::query::core_extraction_queries::insert_logs_in_transaction(
&mut transaction,
transaction_id,
&bundle.logs,
)
.await;
let log_result =
insert_logs_in_transaction(&mut transaction, transaction_id, &bundle.logs).await;
if let std::result::Result::Err(error) = log_result {
return std::result::Result::Err(error);
}
let balance_result =
crate::postgres::query::core_extraction_queries::insert_balance_changes_in_transaction(
&mut transaction,
transaction_id,
&bundle.balance_changes,
)
.await;
let balance_result = insert_balance_changes_in_transaction(
&mut transaction,
transaction_id,
&bundle.balance_changes,
)
.await;
if let std::result::Result::Err(error) = balance_result {
return std::result::Result::Err(error);
}
let return_data_result = insert_return_data_in_transaction(
&mut transaction,
transaction_id,
bundle.return_data.as_ref(),
)
.await;
if let std::result::Result::Err(error) = return_data_result {
return std::result::Result::Err(error);
}
let raw_update_result = sqlx::query(
"UPDATE kb_sol_raw_transactions SET processing_state = 'core_extracted', lifecycle_reason = NULL, updated_at = NOW() WHERE id = $1",
"UPDATE k_sol_raw_transactions SET processing_state = 'core_extracted', lifecycle_reason = NULL, updated_at = NOW() WHERE id = $1",
)
.bind(bundle.raw_transaction_id)
.execute(&mut *transaction)
@@ -191,15 +180,14 @@ pub(crate) async fn persist_core_extraction(
"postgres core extraction raw lifecycle update failed: {error}"
)));
}
let ledger_result =
crate::postgres::query::core_extraction_queries::upsert_ledger_terminal_in_transaction(
&mut transaction,
&bundle.ledger_identity,
"succeeded",
std::option::Option::None,
std::option::Option::None,
)
.await;
let ledger_result = upsert_ledger_terminal_in_transaction(
&mut transaction,
&bundle.ledger_identity,
"succeeded",
std::option::Option::None,
std::option::Option::None,
)
.await;
if let std::result::Result::Err(error) = ledger_result {
return std::result::Result::Err(error);
}
@@ -209,8 +197,7 @@ pub(crate) async fn persist_core_extraction(
"postgres core extraction commit failed: {error}"
)));
}
let inserted_count_result =
crate::postgres::query::core_extraction_queries::bundle_inserted_count(bundle);
let inserted_count_result = bundle_inserted_count(bundle);
let inserted_count = match inserted_count_result {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
@@ -236,7 +223,7 @@ pub(crate) async fn mark_core_extraction_failed(
},
};
let raw_update_result = sqlx::query(
"UPDATE kb_sol_raw_transactions SET processing_state = 'failed', lifecycle_reason = $2, updated_at = NOW() WHERE id = $1",
"UPDATE k_sol_raw_transactions SET processing_state = 'failed', lifecycle_reason = $2, updated_at = NOW() WHERE id = $1",
)
.bind(failure.raw_transaction_id)
.bind(failure.error_message.as_str())
@@ -247,15 +234,14 @@ pub(crate) async fn mark_core_extraction_failed(
"postgres core extraction failed lifecycle update failed: {error}"
)));
}
let ledger_result =
crate::postgres::query::core_extraction_queries::upsert_ledger_terminal_in_transaction(
&mut transaction,
&failure.ledger_identity,
"failed",
std::option::Option::Some(failure.error_code.as_str()),
std::option::Option::Some(failure.error_message.as_str()),
)
.await;
let ledger_result = upsert_ledger_terminal_in_transaction(
&mut transaction,
&failure.ledger_identity,
"failed",
std::option::Option::Some(failure.error_code.as_str()),
std::option::Option::Some(failure.error_message.as_str()),
)
.await;
if let std::result::Result::Err(error) = ledger_result {
return std::result::Result::Err(error);
}
@@ -272,17 +258,18 @@ async fn insert_core_transaction_in_transaction(
transaction: &mut sqlx::Transaction<'_, sqlx::Postgres>,
input: &crate::CoreTransactionInsert,
) -> ks_core::Result<i64> {
let slot_result = crate::postgres::query::core_extraction_queries::sql_slot(input.slot);
let slot_result = sql_slot(input.slot);
let slot = match slot_result {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let query_result = sqlx::query_scalar::<sqlx::Postgres, i64>(
"INSERT INTO kb_sol_core_transactions (raw_transaction_id, signature, slot, failed, err_json) VALUES ($1, $2, $3, $4, $5) RETURNING id",
"INSERT INTO k_sol_core_transactions (raw_transaction_id, signature, slot, block_time, failed, err_json) VALUES ($1, $2, $3, $4, $5, $6) RETURNING id",
)
.bind(input.raw_transaction_id)
.bind(input.signature.as_str())
.bind(slot)
.bind(input.block_time)
.bind(input.failed)
.bind(input.err_json.clone())
.fetch_one(&mut **transaction)
@@ -301,7 +288,7 @@ async fn insert_account_keys_in_transaction(
inputs: &[crate::CoreAccountKeyInsert],
) -> ks_core::Result<()> {
for input in inputs {
let slot_result = crate::postgres::query::core_extraction_queries::sql_slot(input.slot);
let slot_result = sql_slot(input.slot);
let slot = match slot_result {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
@@ -316,14 +303,14 @@ async fn insert_account_keys_in_transaction(
},
};
let query_result = sqlx::query(
"INSERT INTO kb_sol_core_account_keys (transaction_id, signature, slot, account_index, account_key, source, writable, signer, executable) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)",
"INSERT INTO k_sol_core_account_keys (transaction_id, signature, slot, account_index, account_key, source, writable, signer, executable) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)",
)
.bind(transaction_id)
.bind(input.signature.as_str())
.bind(slot)
.bind(account_index)
.bind(input.account_key.as_str())
.bind(crate::postgres::query::core_extraction_queries::account_key_source_to_sql(input.source))
.bind(account_key_source_to_sql(input.source))
.bind(input.writable)
.bind(input.signer)
.bind(input.executable)
@@ -344,19 +331,25 @@ async fn insert_instructions_in_transaction(
inputs: &[crate::CoreInstructionInsert],
) -> ks_core::Result<()> {
for input in inputs {
let slot_result = crate::postgres::query::core_extraction_queries::sql_slot(input.slot);
let slot_result = sql_slot(input.slot);
let slot = match slot_result {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let stack_height_result = sql_stack_height(input.stack_height, "core instruction");
let stack_height = match stack_height_result {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let query_result = sqlx::query(
"INSERT INTO kb_sol_core_instructions (transaction_id, signature, slot, instruction_path, program_id, accounts_json, payload_json, payload_json_hash, processing_state) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, 'pending')",
"INSERT INTO k_sol_core_instructions (transaction_id, signature, slot, instruction_path, program_id, stack_height, accounts_json, payload_json, payload_json_hash, processing_state) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, 'pending')",
)
.bind(transaction_id)
.bind(input.signature.as_str())
.bind(slot)
.bind(input.instruction_path.as_str())
.bind(input.program_id.as_str())
.bind(stack_height)
.bind(&input.accounts_json)
.bind(&input.payload_json)
.bind(input.payload_json_hash.as_deref())
@@ -377,13 +370,18 @@ async fn insert_inner_instructions_in_transaction(
inputs: &[crate::CoreInnerInstructionInsert],
) -> ks_core::Result<()> {
for input in inputs {
let slot_result = crate::postgres::query::core_extraction_queries::sql_slot(input.slot);
let slot_result = sql_slot(input.slot);
let slot = match slot_result {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let stack_height_result = sql_stack_height(input.stack_height, "core inner instruction");
let stack_height = match stack_height_result {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let query_result = sqlx::query(
"INSERT INTO kb_sol_core_inner_instructions (transaction_id, signature, slot, parent_instruction_path, instruction_path, program_id, accounts_json, payload_json, payload_json_hash) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)",
"INSERT INTO k_sol_core_inner_instructions (transaction_id, signature, slot, parent_instruction_path, instruction_path, program_id, stack_height, accounts_json, payload_json, payload_json_hash, processing_state) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, 'pending')",
)
.bind(transaction_id)
.bind(input.signature.as_str())
@@ -391,6 +389,7 @@ async fn insert_inner_instructions_in_transaction(
.bind(input.parent_instruction_path.as_str())
.bind(input.instruction_path.as_str())
.bind(input.program_id.as_str())
.bind(stack_height)
.bind(&input.accounts_json)
.bind(&input.payload_json)
.bind(input.payload_json_hash.as_deref())
@@ -405,13 +404,45 @@ async fn insert_inner_instructions_in_transaction(
return std::result::Result::Ok(());
}
async fn insert_return_data_in_transaction(
transaction: &mut sqlx::Transaction<'_, sqlx::Postgres>,
transaction_id: i64,
input: std::option::Option<&crate::CoreReturnDataInsert>,
) -> ks_core::Result<()> {
let value = match input {
std::option::Option::Some(value) => value,
std::option::Option::None => return std::result::Result::Ok(()),
};
let slot_result = sql_slot(value.slot);
let slot = match slot_result {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let query_result = sqlx::query(
"INSERT INTO k_sol_core_return_data (transaction_id, signature, slot, program_id, data_base64) VALUES ($1, $2, $3, $4, $5)",
)
.bind(transaction_id)
.bind(value.signature.as_str())
.bind(slot)
.bind(value.program_id.as_str())
.bind(value.data_base64.as_str())
.execute(&mut **transaction)
.await;
return match query_result {
std::result::Result::Ok(_) => std::result::Result::Ok(()),
std::result::Result::Err(error) => std::result::Result::Err(ks_core::Error::db(format!(
"postgres core extraction return-data insert failed: {error}"
))),
};
}
async fn insert_logs_in_transaction(
transaction: &mut sqlx::Transaction<'_, sqlx::Postgres>,
transaction_id: i64,
inputs: &[crate::CoreLogInsert],
) -> ks_core::Result<()> {
for input in inputs {
let slot_result = crate::postgres::query::core_extraction_queries::sql_slot(input.slot);
let slot_result = sql_slot(input.slot);
let slot = match slot_result {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
@@ -426,7 +457,7 @@ async fn insert_logs_in_transaction(
},
};
let query_result = sqlx::query(
"INSERT INTO kb_sol_core_logs (transaction_id, signature, slot, log_index, instruction_path, program_id, log_text, log_text_hash) VALUES ($1, $2, $3, $4, $5, $6, $7, $8)",
"INSERT INTO k_sol_core_logs (transaction_id, signature, slot, log_index, instruction_path, program_id, log_text, log_text_hash) VALUES ($1, $2, $3, $4, $5, $6, $7, $8)",
)
.bind(transaction_id)
.bind(input.signature.as_str())
@@ -453,7 +484,7 @@ async fn insert_balance_changes_in_transaction(
inputs: &[crate::CoreBalanceChangeInsert],
) -> ks_core::Result<()> {
for input in inputs {
let slot_result = crate::postgres::query::core_extraction_queries::sql_slot(input.slot);
let slot_result = sql_slot(input.slot);
let slot = match slot_result {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
@@ -477,13 +508,13 @@ async fn insert_balance_changes_in_transaction(
},
};
let query_result = sqlx::query(
"INSERT INTO kb_sol_core_balance_changes (transaction_id, signature, slot, balance_change_index, balance_kind, account_index, account_key, mint, owner, pre_balance_json, post_balance_json, delta_json) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12)",
"INSERT INTO k_sol_core_balance_changes (transaction_id, signature, slot, balance_change_index, balance_kind, account_index, account_key, mint, owner, pre_balance_json, post_balance_json, delta_json) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12)",
)
.bind(transaction_id)
.bind(input.signature.as_str())
.bind(slot)
.bind(balance_index)
.bind(crate::postgres::query::core_extraction_queries::balance_change_kind_to_sql(input.balance_kind))
.bind(balance_change_kind_to_sql(input.balance_kind))
.bind(account_index)
.bind(input.account_key.as_deref())
.bind(input.mint.as_deref())
@@ -510,7 +541,7 @@ async fn upsert_ledger_terminal_in_transaction(
error_message: std::option::Option<&str>,
) -> ks_core::Result<()> {
let query_result = sqlx::query(
"INSERT INTO kb_sol_ops_processing_ledger (stage, processor_name, processor_version, input_key, input_hash, status, attempt_count, started_at, finished_at, error_code, error_message) VALUES ($1, $2, $3, $4, $5, $6, 1, NOW(), NOW(), $7, $8) ON CONFLICT (stage, processor_name, processor_version, input_key) DO UPDATE SET input_hash = EXCLUDED.input_hash, status = EXCLUDED.status, attempt_count = kb_sol_ops_processing_ledger.attempt_count + 1, started_at = NOW(), finished_at = NOW(), error_code = EXCLUDED.error_code, error_message = EXCLUDED.error_message, updated_at = NOW()",
"INSERT INTO k_sol_ops_processing_ledger (stage, processor_name, processor_version, input_key, input_hash, status, attempt_count, started_at, finished_at, error_code, error_message) VALUES ($1, $2, $3, $4, $5, $6, 1, NOW(), NOW(), $7, $8) ON CONFLICT (stage, processor_name, processor_version, input_key) DO UPDATE SET input_hash = EXCLUDED.input_hash, status = EXCLUDED.status, attempt_count = k_sol_ops_processing_ledger.attempt_count + 1, started_at = NOW(), finished_at = NOW(), error_code = EXCLUDED.error_code, error_message = EXCLUDED.error_message, updated_at = NOW()",
)
.bind(identity.stage.as_str())
.bind(identity.processor_name.as_str())
@@ -551,18 +582,12 @@ fn map_raw_transaction_row(
)));
},
};
let retention_result =
crate::postgres::query::core_extraction_queries::raw_retention_state_from_sql(
retention_text.as_str(),
);
let retention_result = raw_retention_state_from_sql(retention_text.as_str());
let retention_state = match retention_result {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let processing_result =
crate::postgres::query::core_extraction_queries::raw_processing_state_from_sql(
processing_text.as_str(),
);
let processing_result = raw_processing_state_from_sql(processing_text.as_str());
let processing_state = match processing_result {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
@@ -591,6 +616,14 @@ fn map_raw_transaction_row(
)));
},
};
let block_time = match row.try_get::<std::option::Option<i64>, _>("block_time") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => {
return std::result::Result::Err(ks_core::Error::db(format!(
"postgres raw block time mapping failed: {error}"
)));
},
};
let canonical_json =
match row.try_get::<std::option::Option<serde_json::Value>, _>("canonical_json") {
std::result::Result::Ok(value) => value,
@@ -637,6 +670,7 @@ fn map_raw_transaction_row(
id,
signature,
slot,
block_time,
canonical_json,
canonical_json_hash,
canonical_format_version,
@@ -655,6 +689,7 @@ fn bundle_inserted_count(bundle: &crate::CoreExtractionBundle) -> ks_core::Resul
bundle.inner_instructions.len(),
bundle.logs.len(),
bundle.balance_changes.len(),
usize::from(bundle.return_data.is_some()),
] {
let value_result = u64::try_from(length);
let value = match value_result {
@@ -678,10 +713,23 @@ fn bundle_inserted_count(bundle: &crate::CoreExtractionBundle) -> ks_core::Resul
return std::result::Result::Ok(total);
}
fn sql_stack_height(
value: std::option::Option<u32>,
context: &str,
) -> ks_core::Result<std::option::Option<i32>> {
let conversion_result = value.map(i32::try_from).transpose();
return match conversion_result {
std::result::Result::Ok(converted) => std::result::Result::Ok(converted),
std::result::Result::Err(error) => std::result::Result::Err(ks_core::Error::db(format!(
"{context} stack height does not fit into SQL INTEGER: {error}"
))),
};
}
fn optional_sql_slot(value: std::option::Option<u64>) -> ks_core::Result<std::option::Option<i64>> {
return match value {
std::option::Option::Some(slot) => {
let converted_result = crate::postgres::query::core_extraction_queries::sql_slot(slot);
let converted_result = sql_slot(slot);
match converted_result {
std::result::Result::Ok(converted) => {
std::result::Result::Ok(std::option::Option::Some(converted))
@@ -760,16 +808,14 @@ mod tests {
#[test]
fn processing_state_serializes_to_expected_sql() {
assert_eq!(
crate::postgres::query::core_extraction_queries::raw_processing_state_to_sql(
crate::RawPayloadProcessingState::CoreExtracted,
),
super::raw_processing_state_to_sql(crate::RawPayloadProcessingState::CoreExtracted,),
"core_extracted"
);
}
#[test]
fn slot_rejects_values_above_bigint() {
let result = crate::postgres::query::core_extraction_queries::sql_slot(u64::MAX);
let result = super::sql_slot(u64::MAX);
assert!(result.is_err());
}
@@ -790,7 +836,7 @@ mod tests {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => panic!("unexpected connect error: {error}"),
};
let schema_result = store.initialize_core_store_schema().await;
let schema_result = store.initialize_store_schema().await;
if let std::result::Result::Err(error) = schema_result {
panic!("unexpected schema error: {error}");
}
@@ -798,6 +844,7 @@ mod tests {
let raw_input_result = crate::RawTransactionInsert::new(
signature.clone(),
1,
std::option::Option::None,
serde_json::json!({"test": "core_extraction_rollback"}),
1,
);
@@ -816,7 +863,7 @@ mod tests {
panic!("unexpected raw insert error: {error}");
}
let raw_id_result = sqlx::query_scalar::<sqlx::Postgres, i64>(
"SELECT id FROM kb_sol_raw_transactions WHERE signature = $1",
"SELECT id FROM k_sol_raw_transactions WHERE signature = $1",
)
.bind(signature.as_str())
.fetch_one(store.pool())
@@ -855,8 +902,13 @@ mod tests {
signature: &str,
duplicate_account_index: bool,
) -> crate::CoreExtractionBundle {
let transaction_result =
crate::CoreTransactionInsert::new(signature, 1, false, std::option::Option::None);
let transaction_result = crate::CoreTransactionInsert::new(
signature,
1,
std::option::Option::None,
false,
std::option::Option::None,
);
let transaction = match transaction_result {
std::result::Result::Ok(value) => value.with_raw_transaction_id(raw_transaction_id),
std::result::Result::Err(error) => panic!("unexpected transaction error: {error}"),
@@ -915,6 +967,7 @@ mod tests {
inner_instructions: std::vec::Vec::new(),
logs: std::vec::Vec::new(),
balance_changes: std::vec::Vec::new(),
return_data: std::option::Option::None,
};
}
@@ -926,7 +979,7 @@ mod tests {
expected_ledger_count: i64,
) {
let core_count_result = sqlx::query_scalar::<sqlx::Postgres, i64>(
"SELECT COUNT(*)::BIGINT FROM kb_sol_core_transactions WHERE signature = $1",
"SELECT COUNT(*)::BIGINT FROM k_sol_core_transactions WHERE signature = $1",
)
.bind(signature)
.fetch_one(store.pool())
@@ -936,7 +989,7 @@ mod tests {
std::result::Result::Err(error) => panic!("unexpected core count error: {error}"),
};
let account_count_result = sqlx::query_scalar::<sqlx::Postgres, i64>(
"SELECT COUNT(*)::BIGINT FROM kb_sol_core_account_keys WHERE signature = $1",
"SELECT COUNT(*)::BIGINT FROM k_sol_core_account_keys WHERE signature = $1",
)
.bind(signature)
.fetch_one(store.pool())
@@ -946,7 +999,7 @@ mod tests {
std::result::Result::Err(error) => panic!("unexpected account count error: {error}"),
};
let processing_state_result = sqlx::query_scalar::<sqlx::Postgres, std::string::String>(
"SELECT processing_state FROM kb_sol_raw_transactions WHERE signature = $1",
"SELECT processing_state FROM k_sol_raw_transactions WHERE signature = $1",
)
.bind(signature)
.fetch_one(store.pool())
@@ -956,7 +1009,7 @@ mod tests {
std::result::Result::Err(error) => panic!("unexpected processing state error: {error}"),
};
let ledger_count_result = sqlx::query_scalar::<sqlx::Postgres, i64>(
"SELECT COUNT(*)::BIGINT FROM kb_sol_ops_processing_ledger WHERE stage = 'core_extraction' AND processor_name = 'canonical_to_core' AND processor_version = 'rollback-test' AND input_key = $1",
"SELECT COUNT(*)::BIGINT FROM k_sol_ops_processing_ledger WHERE stage = 'core_extraction' AND processor_name = 'canonical_to_core' AND processor_version = 'rollback-test' AND input_key = $1",
)
.bind(signature)
.fetch_one(store.pool())
@@ -973,7 +1026,7 @@ mod tests {
async fn cleanup_rollback_test(store: &crate::PostgresStore, signature: &str) {
let ledger_result = sqlx::query(
"DELETE FROM kb_sol_ops_processing_ledger WHERE processor_version = 'rollback-test' AND input_key = $1",
"DELETE FROM k_sol_ops_processing_ledger WHERE processor_version = 'rollback-test' AND input_key = $1",
)
.bind(signature)
.execute(store.pool())
@@ -981,14 +1034,14 @@ mod tests {
if let std::result::Result::Err(error) = ledger_result {
panic!("unexpected ledger cleanup error: {error}");
}
let core_result = sqlx::query("DELETE FROM kb_sol_core_transactions WHERE signature = $1")
let core_result = sqlx::query("DELETE FROM k_sol_core_transactions WHERE signature = $1")
.bind(signature)
.execute(store.pool())
.await;
if let std::result::Result::Err(error) = core_result {
panic!("unexpected core cleanup error: {error}");
}
let raw_result = sqlx::query("DELETE FROM kb_sol_raw_transactions WHERE signature = $1")
let raw_result = sqlx::query("DELETE FROM k_sol_raw_transactions WHERE signature = $1")
.bind(signature)
.execute(store.pool())
.await;

View File

@@ -1,75 +1,34 @@
// file: ks-store/src/postgres/query/core_queries.rs
// version: 6
// version: 9
//! PostgreSQL queries for normalized Solana core storage.
use sqlx::Row; // rust-rules: trait-import
const OUTER_INSTRUCTIONS_CONTEXT_SQL: &str = "SELECT COALESCE(jsonb_agg(jsonb_build_object('instructionIndex', instruction_path::BIGINT, 'instructionPath', instruction_path, 'programId', program_id, 'payloadJson', payload_json, 'payloadHash', payload_json_hash) ORDER BY instruction_path::BIGINT, instruction_path ASC), '[]'::jsonb) FROM kb_sol_core_instructions WHERE signature = $1 AND instruction_path ~ '^[0-9]+$'";
pub(crate) async fn apply_core_store_schema(pool: &sqlx::PgPool) -> ks_core::Result<()> {
let validation_result = crate::validate_core_store_table_names();
if let std::result::Result::Err(error) = validation_result {
return std::result::Result::Err(error);
}
let transaction_result = pool.begin().await;
let mut transaction = match transaction_result {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => {
return std::result::Result::Err(ks_core::Error::db(format!(
"postgres core store schema transaction failed: {error}"
)));
},
};
let lock_result = sqlx::query("SELECT pg_advisory_xact_lock($1)")
.bind(crate::STORE_SCHEMA_ADVISORY_LOCK_ID)
.execute(&mut *transaction)
.await;
if let std::result::Result::Err(error) = lock_result {
return std::result::Result::Err(ks_core::Error::db(format!(
"postgres core store schema advisory lock failed: {error}"
)));
}
for statement in crate::core_store_schema_statements() {
let execution_result = sqlx::query(statement).execute(&mut *transaction).await;
if let std::result::Result::Err(error) = execution_result {
return std::result::Result::Err(ks_core::Error::db(format!(
"postgres core store schema initialization failed: {error}"
)));
}
}
let commit_result = transaction.commit().await;
if let std::result::Result::Err(error) = commit_result {
return std::result::Result::Err(ks_core::Error::db(format!(
"postgres core store schema commit failed: {error}"
)));
}
return std::result::Result::Ok(());
}
const TOP_LEVEL_INSTRUCTIONS_CONTEXT_SQL: &str = "SELECT COALESCE(jsonb_agg(jsonb_build_object('instructionIndex', instruction_path::BIGINT, 'instructionPath', instruction_path, 'programId', program_id, 'stackHeight', stack_height, 'payloadJson', payload_json, 'payloadHash', payload_json_hash) ORDER BY instruction_path::BIGINT, instruction_path ASC), '[]'::jsonb) FROM k_sol_core_instructions WHERE signature = $1 AND instruction_path ~ '^[0-9]+$'";
const INCOMPLETE_DECODE_INPUTS_SQL: &str = "WITH incomplete AS (SELECT signature, MIN(slot) AS first_slot FROM k_sol_core_instructions WHERE (cardinality($1::text[]) = 0 OR signature = ANY($1)) AND ($3::bigint IS NULL OR slot >= $3) AND ($4::bigint IS NULL OR slot <= $4) AND (cardinality($5::text[]) = 0 OR program_id = ANY($5)) AND (processing_state = ANY($2) OR (processing_state = 'ignored' AND lifecycle_reason = 'unsupported')) GROUP BY signature ORDER BY MIN(slot), signature LIMIT $7) SELECT i.id, i.transaction_id, i.signature, i.slot, i.instruction_path, i.program_id, i.stack_height, i.accounts_json, i.payload_json, i.payload_json_hash, i.processing_state, i.created_at, i.updated_at FROM k_sol_core_instructions i JOIN incomplete s ON s.signature = i.signature WHERE (cardinality($5::text[]) = 0 OR i.program_id = ANY($5)) AND (cardinality($6::text[]) = 0 OR i.instruction_path = ANY($6)) ORDER BY i.slot ASC, i.signature ASC, i.instruction_path ASC";
pub(crate) async fn insert_core_transaction(
pool: &sqlx::PgPool,
input: &crate::CoreTransactionInsert,
) -> ks_core::Result<crate::InsertOutcome> {
let slot_result = crate::postgres::query::core_queries::sql_slot_from_u64(input.slot);
let slot_result = sql_slot_from_u64(input.slot);
let slot = match slot_result {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let query_result = sqlx::query_scalar::<sqlx::Postgres, i64>(
"INSERT INTO kb_sol_core_transactions (raw_transaction_id, signature, slot, failed, err_json) VALUES ($1, $2, $3, $4, $5) ON CONFLICT (signature) DO NOTHING RETURNING id",
"INSERT INTO k_sol_core_transactions (raw_transaction_id, signature, slot, block_time, failed, err_json) VALUES ($1, $2, $3, $4, $5, $6) ON CONFLICT (signature) DO NOTHING RETURNING id",
)
.bind(input.raw_transaction_id)
.bind(input.signature.as_str())
.bind(slot)
.bind(input.block_time)
.bind(input.failed)
.bind(input.err_json.clone())
.fetch_optional(pool)
.await;
return crate::postgres::query::core_queries::outcome_from_insert_optional(
query_result,
"postgres core transaction insert failed",
);
return outcome_from_insert_optional(query_result, "postgres core transaction insert failed");
}
pub(crate) async fn insert_core_account_keys(
@@ -79,12 +38,12 @@ pub(crate) async fn insert_core_account_keys(
let mut inserted_count = 0_u64;
let mut skipped_count = 0_u64;
for input in inputs {
let slot_result = crate::postgres::query::core_queries::sql_slot_from_u64(input.slot);
let slot_result = sql_slot_from_u64(input.slot);
let slot = match slot_result {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let account_index_result = crate::postgres::query::core_queries::sql_i32_from_u32(
let account_index_result = sql_i32_from_u32(
input.account_index,
"core account index does not fit into SQL INTEGER",
);
@@ -92,9 +51,9 @@ pub(crate) async fn insert_core_account_keys(
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let source = crate::postgres::query::core_queries::account_key_source_to_sql(input.source);
let source = account_key_source_to_sql(input.source);
let query_result = sqlx::query_scalar::<sqlx::Postgres, i64>(
"INSERT INTO kb_sol_core_account_keys (transaction_id, signature, slot, account_index, account_key, source, writable, signer, executable) VALUES ((SELECT id FROM kb_sol_core_transactions WHERE signature = $1 LIMIT 1), $1, $2, $3, $4, $5, $6, $7, $8) ON CONFLICT (signature, account_index) DO NOTHING RETURNING id",
"INSERT INTO k_sol_core_account_keys (transaction_id, signature, slot, account_index, account_key, source, writable, signer, executable) VALUES ((SELECT id FROM k_sol_core_transactions WHERE signature = $1 LIMIT 1), $1, $2, $3, $4, $5, $6, $7, $8) ON CONFLICT (signature, account_index) DO NOTHING RETURNING id",
)
.bind(input.signature.as_str())
.bind(slot)
@@ -106,10 +65,8 @@ pub(crate) async fn insert_core_account_keys(
.bind(input.executable)
.fetch_optional(pool)
.await;
let outcome_result = crate::postgres::query::core_queries::outcome_from_insert_optional(
query_result,
"postgres core account key insert failed",
);
let outcome_result =
outcome_from_insert_optional(query_result, "postgres core account key insert failed");
let outcome = match outcome_result {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
@@ -127,32 +84,36 @@ pub(crate) async fn insert_core_instructions(
let mut inserted_count = 0_u64;
let mut skipped_count = 0_u64;
for input in inputs {
let slot_result = crate::postgres::query::core_queries::sql_slot_from_u64(input.slot);
let slot_result = sql_slot_from_u64(input.slot);
let slot = match slot_result {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let processing_state =
crate::postgres::query::core_queries::instruction_processing_state_to_sql(
input.processing_state,
);
let stack_height_result = optional_sql_i32_from_u32(
input.stack_height,
"core instruction stack height does not fit into SQL INTEGER",
);
let stack_height = match stack_height_result {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let processing_state = instruction_processing_state_to_sql(input.processing_state);
let query_result = sqlx::query_scalar::<sqlx::Postgres, i64>(
"INSERT INTO kb_sol_core_instructions (transaction_id, signature, slot, instruction_path, program_id, accounts_json, payload_json, payload_json_hash, processing_state) VALUES ((SELECT id FROM kb_sol_core_transactions WHERE signature = $1 LIMIT 1), $1, $2, $3, $4, $5, $6, $7, $8) ON CONFLICT (signature, instruction_path) DO NOTHING RETURNING id",
"INSERT INTO k_sol_core_instructions (transaction_id, signature, slot, instruction_path, program_id, stack_height, accounts_json, payload_json, payload_json_hash, processing_state) VALUES ((SELECT id FROM k_sol_core_transactions WHERE signature = $1 LIMIT 1), $1, $2, $3, $4, $5, $6, $7, $8, $9) ON CONFLICT (signature, instruction_path) DO NOTHING RETURNING id",
)
.bind(input.signature.as_str())
.bind(slot)
.bind(input.instruction_path.as_str())
.bind(input.program_id.as_str())
.bind(stack_height)
.bind(&input.accounts_json)
.bind(&input.payload_json)
.bind(input.payload_json_hash.as_deref())
.bind(processing_state)
.fetch_optional(pool)
.await;
let outcome_result = crate::postgres::query::core_queries::outcome_from_insert_optional(
query_result,
"postgres core instruction insert failed",
);
let outcome_result =
outcome_from_insert_optional(query_result, "postgres core instruction insert failed");
let outcome = match outcome_result {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
@@ -170,25 +131,36 @@ pub(crate) async fn insert_core_inner_instructions(
let mut inserted_count = 0_u64;
let mut skipped_count = 0_u64;
for input in inputs {
let slot_result = crate::postgres::query::core_queries::sql_slot_from_u64(input.slot);
let slot_result = sql_slot_from_u64(input.slot);
let slot = match slot_result {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let stack_height_result = optional_sql_i32_from_u32(
input.stack_height,
"core inner instruction stack height does not fit into SQL INTEGER",
);
let stack_height = match stack_height_result {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let processing_state = instruction_processing_state_to_sql(input.processing_state);
let query_result = sqlx::query_scalar::<sqlx::Postgres, i64>(
"INSERT INTO kb_sol_core_inner_instructions (transaction_id, signature, slot, parent_instruction_path, instruction_path, program_id, accounts_json, payload_json, payload_json_hash) VALUES ((SELECT id FROM kb_sol_core_transactions WHERE signature = $1 LIMIT 1), $1, $2, $3, $4, $5, $6, $7, $8) ON CONFLICT (signature, instruction_path) DO NOTHING RETURNING id",
"INSERT INTO k_sol_core_inner_instructions (transaction_id, signature, slot, parent_instruction_path, instruction_path, program_id, stack_height, accounts_json, payload_json, payload_json_hash, processing_state) VALUES ((SELECT id FROM k_sol_core_transactions WHERE signature = $1 LIMIT 1), $1, $2, $3, $4, $5, $6, $7, $8, $9, $10) ON CONFLICT (signature, instruction_path) DO NOTHING RETURNING id",
)
.bind(input.signature.as_str())
.bind(slot)
.bind(input.parent_instruction_path.as_str())
.bind(input.instruction_path.as_str())
.bind(input.program_id.as_str())
.bind(stack_height)
.bind(&input.accounts_json)
.bind(&input.payload_json)
.bind(input.payload_json_hash.as_deref())
.bind(processing_state)
.fetch_optional(pool)
.await;
let outcome_result = crate::postgres::query::core_queries::outcome_from_insert_optional(
let outcome_result = outcome_from_insert_optional(
query_result,
"postgres core inner instruction insert failed",
);
@@ -209,21 +181,19 @@ pub(crate) async fn insert_core_logs(
let mut inserted_count = 0_u64;
let mut skipped_count = 0_u64;
for input in inputs {
let slot_result = crate::postgres::query::core_queries::sql_slot_from_u64(input.slot);
let slot_result = sql_slot_from_u64(input.slot);
let slot = match slot_result {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let log_index_result = crate::postgres::query::core_queries::sql_i32_from_u32(
input.log_index,
"core log index does not fit into SQL INTEGER",
);
let log_index_result =
sql_i32_from_u32(input.log_index, "core log index does not fit into SQL INTEGER");
let log_index = match log_index_result {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let query_result = sqlx::query_scalar::<sqlx::Postgres, i64>(
"INSERT INTO kb_sol_core_logs (transaction_id, signature, slot, log_index, instruction_path, program_id, log_text, log_text_hash) VALUES ((SELECT id FROM kb_sol_core_transactions WHERE signature = $1 LIMIT 1), $1, $2, $3, $4, $5, $6, $7) ON CONFLICT (signature, log_index) DO NOTHING RETURNING id",
"INSERT INTO k_sol_core_logs (transaction_id, signature, slot, log_index, instruction_path, program_id, log_text, log_text_hash) VALUES ((SELECT id FROM k_sol_core_transactions WHERE signature = $1 LIMIT 1), $1, $2, $3, $4, $5, $6, $7) ON CONFLICT (signature, log_index) DO NOTHING RETURNING id",
)
.bind(input.signature.as_str())
.bind(slot)
@@ -234,10 +204,8 @@ pub(crate) async fn insert_core_logs(
.bind(input.log_text_hash.as_deref())
.fetch_optional(pool)
.await;
let outcome_result = crate::postgres::query::core_queries::outcome_from_insert_optional(
query_result,
"postgres core log insert failed",
);
let outcome_result =
outcome_from_insert_optional(query_result, "postgres core log insert failed");
let outcome = match outcome_result {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
@@ -255,12 +223,12 @@ pub(crate) async fn insert_core_balance_changes(
let mut inserted_count = 0_u64;
let mut skipped_count = 0_u64;
for input in inputs {
let slot_result = crate::postgres::query::core_queries::sql_slot_from_u64(input.slot);
let slot_result = sql_slot_from_u64(input.slot);
let slot = match slot_result {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let balance_change_index_result = crate::postgres::query::core_queries::sql_i32_from_u32(
let balance_change_index_result = sql_i32_from_u32(
input.balance_change_index,
"core balance change index does not fit into SQL INTEGER",
);
@@ -268,7 +236,7 @@ pub(crate) async fn insert_core_balance_changes(
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let account_index_result = crate::postgres::query::core_queries::optional_sql_i32_from_u32(
let account_index_result = optional_sql_i32_from_u32(
input.account_index,
"core balance account index does not fit into SQL INTEGER",
);
@@ -276,10 +244,9 @@ pub(crate) async fn insert_core_balance_changes(
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let balance_kind =
crate::postgres::query::core_queries::balance_change_kind_to_sql(input.balance_kind);
let balance_kind = balance_change_kind_to_sql(input.balance_kind);
let query_result = sqlx::query_scalar::<sqlx::Postgres, i64>(
"INSERT INTO kb_sol_core_balance_changes (transaction_id, signature, slot, balance_change_index, balance_kind, account_index, account_key, mint, owner, pre_balance_json, post_balance_json, delta_json) VALUES ((SELECT id FROM kb_sol_core_transactions WHERE signature = $1 LIMIT 1), $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11) ON CONFLICT (signature, balance_change_index) DO NOTHING RETURNING id",
"INSERT INTO k_sol_core_balance_changes (transaction_id, signature, slot, balance_change_index, balance_kind, account_index, account_key, mint, owner, pre_balance_json, post_balance_json, delta_json) VALUES ((SELECT id FROM k_sol_core_transactions WHERE signature = $1 LIMIT 1), $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11) ON CONFLICT (signature, balance_change_index) DO NOTHING RETURNING id",
)
.bind(input.signature.as_str())
.bind(slot)
@@ -294,7 +261,7 @@ pub(crate) async fn insert_core_balance_changes(
.bind(input.delta_json.clone())
.fetch_optional(pool)
.await;
let outcome_result = crate::postgres::query::core_queries::outcome_from_insert_optional(
let outcome_result = outcome_from_insert_optional(
query_result,
"postgres core balance change insert failed",
);
@@ -313,31 +280,25 @@ pub(crate) async fn list_core_instructions_for_replay(
filter: &crate::CoreInstructionReplayFilter,
page_request: &crate::PageRequest,
) -> ks_core::Result<std::vec::Vec<crate::CoreInstructionRow>> {
let min_slot_result =
crate::postgres::query::core_queries::optional_sql_slot_from_u64(filter.min_slot);
let min_slot_result = optional_sql_slot_from_u64(filter.min_slot);
let min_slot = match min_slot_result {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let max_slot_result =
crate::postgres::query::core_queries::optional_sql_slot_from_u64(filter.max_slot);
let max_slot_result = optional_sql_slot_from_u64(filter.max_slot);
let max_slot = match max_slot_result {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let offset_result = crate::postgres::query::core_queries::sql_i64_from_u64(
page_request.offset,
"page offset does not fit into SQL BIGINT",
);
let offset_result =
sql_i64_from_u64(page_request.offset, "page offset does not fit into SQL BIGINT");
let offset = match offset_result {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let processing_state = filter
.processing_state
.map(crate::postgres::query::core_queries::instruction_processing_state_to_sql);
let processing_state = filter.processing_state.map(instruction_processing_state_to_sql);
let query_result = sqlx::query(
"SELECT id, transaction_id, signature, slot, instruction_path, program_id, accounts_json, payload_json, payload_json_hash, processing_state, created_at, updated_at FROM kb_sol_core_instructions WHERE ($1::TEXT IS NULL OR processing_state = $1) AND ($2::TEXT IS NULL OR program_id = $2) AND ($3::BIGINT IS NULL OR slot >= $3) AND ($4::BIGINT IS NULL OR slot <= $4) ORDER BY slot ASC, id ASC LIMIT $5 OFFSET $6",
"SELECT id, transaction_id, signature, slot, instruction_path, program_id, stack_height, accounts_json, payload_json, payload_json_hash, processing_state, created_at, updated_at FROM k_sol_core_instructions WHERE ($1::TEXT IS NULL OR processing_state = $1) AND ($2::TEXT IS NULL OR program_id = $2) AND ($3::BIGINT IS NULL OR slot >= $3) AND ($4::BIGINT IS NULL OR slot <= $4) ORDER BY slot ASC, id ASC LIMIT $5 OFFSET $6",
)
.bind(processing_state)
.bind(filter.program_id.as_deref())
@@ -357,7 +318,7 @@ pub(crate) async fn list_core_instructions_for_replay(
};
let mut output = std::vec::Vec::with_capacity(rows.len());
for row in rows {
let mapped_result = crate::postgres::query::core_queries::row_to_core_instruction_row(&row);
let mapped_result = row_to_core_instruction_row(&row);
match mapped_result {
std::result::Result::Ok(value) => output.push(value),
std::result::Result::Err(error) => return std::result::Result::Err(error),
@@ -379,11 +340,7 @@ pub(crate) async fn list_core_instruction_replay_inputs(
};
let mut output = std::vec::Vec::with_capacity(instructions.len());
for instruction in instructions {
let input_result = crate::postgres::query::core_queries::load_replay_input_for_instruction(
pool,
&instruction,
)
.await;
let input_result = load_replay_input_for_instruction(pool, &instruction).await;
match input_result {
std::result::Result::Ok(value) => output.push(value),
std::result::Result::Err(error) => return std::result::Result::Err(error),
@@ -396,14 +353,12 @@ pub(crate) async fn list_decode_replay_inputs(
pool: &sqlx::PgPool,
filter: &crate::DecodeSelectionFilter,
) -> ks_core::Result<std::vec::Vec<crate::MdCoreInstructionReplayInput>> {
let min_slot_result =
crate::postgres::query::core_queries::optional_sql_slot_from_u64(filter.min_slot);
let min_slot_result = optional_sql_slot_from_u64(filter.min_slot);
let min_slot = match min_slot_result {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let max_slot_result =
crate::postgres::query::core_queries::optional_sql_slot_from_u64(filter.max_slot);
let max_slot_result = optional_sql_slot_from_u64(filter.max_slot);
let max_slot = match max_slot_result {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
@@ -413,28 +368,23 @@ pub(crate) async fn list_decode_replay_inputs(
.processing_states
.iter()
.map(|state| {
return crate::postgres::query::core_queries::instruction_processing_state_to_sql(
*state,
)
.to_string();
return instruction_processing_state_to_sql(*state).to_string();
})
.collect::<std::vec::Vec<_>>();
let rows_result = if filter.incomplete_signatures {
sqlx::query(
"WITH incomplete AS (SELECT signature, MIN(slot) AS first_slot FROM kb_sol_core_instructions WHERE (cardinality($1::text[]) = 0 OR signature = ANY($1)) AND ($3::bigint IS NULL OR slot >= $3) AND ($4::bigint IS NULL OR slot <= $4) AND (cardinality($5::text[]) = 0 OR program_id = ANY($5)) AND (processing_state = ANY($2) OR (processing_state = 'ignored' AND lifecycle_reason = 'unsupported')) GROUP BY signature ORDER BY MIN(slot), signature LIMIT $7) SELECT i.id, i.transaction_id, i.signature, i.slot, i.instruction_path, i.program_id, i.accounts_json, i.payload_json, i.payload_json_hash, i.processing_state, i.created_at, i.updated_at FROM kb_sol_core_instructions i JOIN incomplete s ON s.signature = i.signature WHERE (cardinality($5::text[]) = 0 OR i.program_id = ANY($5)) AND (cardinality($6::text[]) = 0 OR i.instruction_path = ANY($6)) ORDER BY i.slot ASC, i.signature ASC, i.instruction_path ASC",
)
.bind(&filter.signatures)
.bind(&processing_states)
.bind(min_slot)
.bind(max_slot)
.bind(&filter.program_ids)
.bind(&filter.instruction_paths)
.bind(limit)
.fetch_all(pool)
.await
sqlx::query(INCOMPLETE_DECODE_INPUTS_SQL)
.bind(&filter.signatures)
.bind(&processing_states)
.bind(min_slot)
.bind(max_slot)
.bind(&filter.program_ids)
.bind(&filter.instruction_paths)
.bind(limit)
.fetch_all(pool)
.await
} else {
sqlx::query(
"SELECT id, transaction_id, signature, slot, instruction_path, program_id, accounts_json, payload_json, payload_json_hash, processing_state, created_at, updated_at FROM kb_sol_core_instructions WHERE (cardinality($1::text[]) = 0 OR signature = ANY($1)) AND (cardinality($2::text[]) = 0 OR processing_state = ANY($2)) AND ($3::bigint IS NULL OR slot >= $3) AND ($4::bigint IS NULL OR slot <= $4) AND (cardinality($5::text[]) = 0 OR program_id = ANY($5)) AND (cardinality($6::text[]) = 0 OR instruction_path = ANY($6)) ORDER BY slot ASC, signature ASC, instruction_path ASC LIMIT $7",
"SELECT id, transaction_id, signature, slot, instruction_path, program_id, stack_height, accounts_json, payload_json, payload_json_hash, processing_state, created_at, updated_at FROM k_sol_core_instructions WHERE (cardinality($1::text[]) = 0 OR signature = ANY($1)) AND (cardinality($2::text[]) = 0 OR processing_state = ANY($2)) AND ($3::bigint IS NULL OR slot >= $3) AND ($4::bigint IS NULL OR slot <= $4) AND (cardinality($5::text[]) = 0 OR program_id = ANY($5)) AND (cardinality($6::text[]) = 0 OR instruction_path = ANY($6)) ORDER BY slot ASC, signature ASC, instruction_path ASC LIMIT $7",
)
.bind(&filter.signatures)
.bind(&processing_states)
@@ -456,17 +406,12 @@ pub(crate) async fn list_decode_replay_inputs(
};
let mut output = std::vec::Vec::with_capacity(rows.len());
for row in rows {
let instruction_result =
crate::postgres::query::core_queries::row_to_core_instruction_row(&row);
let instruction_result = row_to_core_instruction_row(&row);
let instruction = match instruction_result {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let input_result = crate::postgres::query::core_queries::load_replay_input_for_instruction(
pool,
&instruction,
)
.await;
let input_result = load_replay_input_for_instruction(pool, &instruction).await;
match input_result {
std::result::Result::Ok(value) => output.push(value),
std::result::Result::Err(error) => return std::result::Result::Err(error),
@@ -479,12 +424,9 @@ pub(crate) async fn update_core_instruction_lifecycle(
pool: &sqlx::PgPool,
mark: &crate::CoreInstructionLifecycleMark,
) -> ks_core::Result<crate::InsertOutcome> {
let processing_state =
crate::postgres::query::core_queries::instruction_processing_state_to_sql(
mark.processing_state,
);
let processing_state = instruction_processing_state_to_sql(mark.processing_state);
let query_result = sqlx::query(
"UPDATE kb_sol_core_instructions SET processing_state = $1, processor_name = $2, processor_version = $3, lifecycle_reason = $4, updated_at = NOW() WHERE signature = $5 AND instruction_path = $6",
"UPDATE k_sol_core_instructions SET processing_state = $1, processor_name = $2, processor_version = $3, lifecycle_reason = $4, updated_at = NOW() WHERE signature = $5 AND instruction_path = $6",
)
.bind(processing_state)
.bind(mark.processor_name.as_deref())
@@ -494,7 +436,7 @@ pub(crate) async fn update_core_instruction_lifecycle(
.bind(mark.instruction_path.as_str())
.execute(pool)
.await;
return crate::postgres::query::core_queries::outcome_from_update_result(
return outcome_from_update_result(
query_result,
"postgres core instruction lifecycle update failed",
);
@@ -549,10 +491,7 @@ fn balance_change_kind_to_sql(kind: crate::CoreBalanceChangeKind) -> &'static st
}
fn sql_slot_from_u64(slot: u64) -> ks_core::Result<i64> {
return crate::postgres::query::core_queries::sql_i64_from_u64(
slot,
"solana slot does not fit into SQL BIGINT",
);
return sql_i64_from_u64(slot, "solana slot does not fit into SQL BIGINT");
}
fn optional_sql_slot_from_u64(
@@ -560,7 +499,7 @@ fn optional_sql_slot_from_u64(
) -> ks_core::Result<std::option::Option<i64>> {
return match slot {
std::option::Option::Some(value) => {
let conversion_result = crate::postgres::query::core_queries::sql_slot_from_u64(value);
let conversion_result = sql_slot_from_u64(value);
match conversion_result {
std::result::Result::Ok(converted) => {
std::result::Result::Ok(std::option::Option::Some(converted))
@@ -586,8 +525,7 @@ fn optional_sql_i32_from_u32(
) -> ks_core::Result<std::option::Option<i32>> {
return match value {
std::option::Option::Some(inner) => {
let conversion_result =
crate::postgres::query::core_queries::sql_i32_from_u32(inner, message);
let conversion_result = sql_i32_from_u32(inner, message);
match conversion_result {
std::result::Result::Ok(converted) => {
std::result::Result::Ok(std::option::Option::Some(converted))
@@ -599,6 +537,26 @@ fn optional_sql_i32_from_u32(
};
}
fn optional_u32_from_sql_i32(
value: std::option::Option<i32>,
message: &str,
) -> ks_core::Result<std::option::Option<u32>> {
return match value {
std::option::Option::Some(inner) => {
let conversion_result = u32::try_from(inner);
match conversion_result {
std::result::Result::Ok(converted) => {
std::result::Result::Ok(std::option::Option::Some(converted))
},
std::result::Result::Err(_error) => {
std::result::Result::Err(ks_core::Error::db(message))
},
}
},
std::option::Option::None => std::result::Result::Ok(std::option::Option::None),
};
}
fn sql_i64_from_u64(value: u64, message: &str) -> ks_core::Result<i64> {
let conversion_result = i64::try_from(value);
return match conversion_result {
@@ -663,59 +621,56 @@ fn row_to_core_instruction_row(
)));
},
};
let state_result = crate::postgres::query::core_queries::instruction_processing_state_from_sql(
state_text.as_str(),
);
let state_result = instruction_processing_state_from_sql(state_text.as_str());
let processing_state = match state_result {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let id = match crate::postgres::query::core_queries::read_row_value(row, "id") {
let id = match read_row_value(row, "id") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let transaction_id =
match crate::postgres::query::core_queries::read_row_value(row, "transaction_id") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let signature = match crate::postgres::query::core_queries::read_row_value(row, "signature") {
let transaction_id = match read_row_value(row, "transaction_id") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let slot = match crate::postgres::query::core_queries::read_row_value(row, "slot") {
let signature = match read_row_value(row, "signature") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let instruction_path =
match crate::postgres::query::core_queries::read_row_value(row, "instruction_path") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let program_id = match crate::postgres::query::core_queries::read_row_value(row, "program_id") {
let slot = match read_row_value(row, "slot") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let accounts_json =
match crate::postgres::query::core_queries::read_row_value(row, "accounts_json") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let payload_json =
match crate::postgres::query::core_queries::read_row_value(row, "payload_json") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let payload_json_hash =
match crate::postgres::query::core_queries::read_row_value(row, "payload_json_hash") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let created_at = match crate::postgres::query::core_queries::read_row_value(row, "created_at") {
let instruction_path = match read_row_value(row, "instruction_path") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let updated_at = match crate::postgres::query::core_queries::read_row_value(row, "updated_at") {
let program_id = match read_row_value(row, "program_id") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let stack_height = match read_row_value(row, "stack_height") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let accounts_json = match read_row_value(row, "accounts_json") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let payload_json = match read_row_value(row, "payload_json") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let payload_json_hash = match read_row_value(row, "payload_json_hash") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let created_at = match read_row_value(row, "created_at") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let updated_at = match read_row_value(row, "updated_at") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
@@ -726,6 +681,7 @@ fn row_to_core_instruction_row(
slot,
instruction_path,
program_id,
stack_height,
accounts_json,
payload_json,
payload_json_hash,
@@ -753,8 +709,8 @@ async fn load_replay_input_for_instruction(
instruction: &crate::CoreInstructionRow,
) -> ks_core::Result<crate::MdCoreInstructionReplayInput> {
let transaction_context_result =
sqlx::query_as::<sqlx::Postgres, (bool, std::option::Option<serde_json::Value>)>(
"SELECT failed, err_json FROM kb_sol_core_transactions WHERE signature = $1 LIMIT 1",
sqlx::query_as::<sqlx::Postgres, (bool, std::option::Option<serde_json::Value>, std::option::Option<i64>)>(
"SELECT failed, err_json, block_time FROM k_sol_core_transactions WHERE signature = $1 LIMIT 1",
)
.bind(instruction.signature.as_str())
.fetch_one(pool)
@@ -767,63 +723,59 @@ async fn load_replay_input_for_instruction(
)));
},
};
let account_keys_json_result = crate::postgres::query::core_queries::load_account_keys_json(
pool,
instruction.signature.as_str(),
)
.await;
let account_keys_json_result =
load_account_keys_json(pool, instruction.signature.as_str()).await;
let account_keys_json = match account_keys_json_result {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let outer_instructions_json_result =
crate::postgres::query::core_queries::load_outer_instructions_json(
pool,
instruction.signature.as_str(),
)
.await;
let outer_instructions_json = match outer_instructions_json_result {
let top_level_instructions_json_result =
load_top_level_instructions_json(pool, instruction.signature.as_str()).await;
let top_level_instructions_json = match top_level_instructions_json_result {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let inner_instructions_json_result =
crate::postgres::query::core_queries::load_inner_instructions_json(
pool,
instruction.signature.as_str(),
instruction.instruction_path.as_str(),
)
.await;
let inner_instructions_json_result = load_inner_instructions_json(
pool,
instruction.signature.as_str(),
instruction.instruction_path.as_str(),
)
.await;
let inner_instructions_json = match inner_instructions_json_result {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let logs_json_result =
crate::postgres::query::core_queries::load_logs_json(pool, instruction.signature.as_str())
.await;
let logs_json_result = load_logs_json(pool, instruction.signature.as_str()).await;
let logs_json = match logs_json_result {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let balance_changes_json_result =
crate::postgres::query::core_queries::load_balance_changes_json(
pool,
instruction.signature.as_str(),
)
.await;
load_balance_changes_json(pool, instruction.signature.as_str()).await;
let balance_changes_json = match balance_changes_json_result {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let replay_input_key = format!("{}:{}", instruction.signature, instruction.instruction_path);
let slot_result = crate::postgres::query::core_queries::u64_from_sql_i64(
instruction.slot,
"core instruction slot is negative",
let return_data_result = load_return_data(pool, instruction.signature.as_str()).await;
let return_data = match return_data_result {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let stack_height_result = optional_u32_from_sql_i32(
instruction.stack_height,
"core instruction stack height is negative",
);
let stack_height = match stack_height_result {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let replay_input_key = format!("{}:{}", instruction.signature, instruction.instruction_path);
let slot_result = u64_from_sql_i64(instruction.slot, "core instruction slot is negative");
let slot = match slot_result {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
return crate::MdCoreInstructionReplayInput::new(
let input_result = crate::MdCoreInstructionReplayInput::new(
replay_input_key,
instruction.signature.clone(),
slot,
@@ -835,35 +787,45 @@ async fn load_replay_input_for_instruction(
instruction.accounts_json.clone(),
instruction.payload_json.clone(),
instruction.payload_json_hash.clone(),
outer_instructions_json,
top_level_instructions_json,
inner_instructions_json,
logs_json,
balance_changes_json,
);
let input = match input_result {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
return input.with_core_context(
transaction_context.2,
std::option::Option::None,
stack_height,
return_data,
);
}
async fn load_account_keys_json(
pool: &sqlx::PgPool,
signature: &str,
) -> ks_core::Result<serde_json::Value> {
return crate::postgres::query::core_queries::load_json_aggregate(
return load_json_aggregate(
pool,
"SELECT COALESCE(jsonb_agg(jsonb_build_object('accountIndex', account_index, 'accountKey', account_key, 'source', source, 'writable', writable, 'signer', signer, 'executable', executable) ORDER BY account_index), '[]'::jsonb) FROM kb_sol_core_account_keys WHERE signature = $1",
"SELECT COALESCE(jsonb_agg(jsonb_build_object('accountIndex', account_index, 'accountKey', account_key, 'source', source, 'writable', writable, 'signer', signer, 'executable', executable) ORDER BY account_index), '[]'::jsonb) FROM k_sol_core_account_keys WHERE signature = $1",
signature,
"postgres core account keys context query failed",
)
.await;
}
async fn load_outer_instructions_json(
async fn load_top_level_instructions_json(
pool: &sqlx::PgPool,
signature: &str,
) -> ks_core::Result<serde_json::Value> {
return crate::postgres::query::core_queries::load_json_aggregate(
return load_json_aggregate(
pool,
crate::postgres::query::core_queries::OUTER_INSTRUCTIONS_CONTEXT_SQL,
TOP_LEVEL_INSTRUCTIONS_CONTEXT_SQL,
signature,
"postgres core outer instructions context query failed",
"postgres core top-level instructions context query failed",
)
.await;
}
@@ -875,7 +837,7 @@ async fn load_inner_instructions_json(
) -> ks_core::Result<serde_json::Value> {
let like_pattern = format!("{instruction_path}/%");
let query_result = sqlx::query_scalar::<sqlx::Postgres, serde_json::Value>(
"SELECT COALESCE(jsonb_agg(jsonb_build_object('parentInstructionPath', parent_instruction_path, 'instructionPath', instruction_path, 'programId', program_id, 'accounts', accounts_json, 'payload', payload_json) ORDER BY instruction_path), '[]'::jsonb) FROM kb_sol_core_inner_instructions WHERE signature = $1 AND (parent_instruction_path = $2 OR instruction_path LIKE $3)",
"SELECT COALESCE(jsonb_agg(jsonb_build_object('parentInstructionPath', parent_instruction_path, 'instructionPath', instruction_path, 'programId', program_id, 'stackHeight', stack_height, 'accounts', accounts_json, 'payload', payload_json, 'payloadHash', payload_json_hash) ORDER BY instruction_path), '[]'::jsonb) FROM k_sol_core_inner_instructions WHERE signature = $1 AND (parent_instruction_path = $2 OR instruction_path LIKE $3)",
)
.bind(signature)
.bind(instruction_path)
@@ -890,13 +852,39 @@ async fn load_inner_instructions_json(
};
}
async fn load_return_data(
pool: &sqlx::PgPool,
signature: &str,
) -> ks_core::Result<std::option::Option<ks_lib::MdCanonicalReturnData>> {
let query_result = sqlx::query_as::<sqlx::Postgres, (std::string::String, std::string::String)>(
"SELECT program_id, data_base64 FROM k_sol_core_return_data WHERE signature = $1 LIMIT 1",
)
.bind(signature)
.fetch_optional(pool)
.await;
return match query_result {
std::result::Result::Ok(std::option::Option::Some((program_id, data_base64))) => {
std::result::Result::Ok(std::option::Option::Some(ks_lib::MdCanonicalReturnData {
program_id,
data_base64,
}))
},
std::result::Result::Ok(std::option::Option::None) => {
std::result::Result::Ok(std::option::Option::None)
},
std::result::Result::Err(error) => std::result::Result::Err(ks_core::Error::db(format!(
"postgres core return-data context query failed: {error}"
))),
};
}
async fn load_logs_json(
pool: &sqlx::PgPool,
signature: &str,
) -> ks_core::Result<serde_json::Value> {
return crate::postgres::query::core_queries::load_json_aggregate(
return load_json_aggregate(
pool,
"SELECT COALESCE(jsonb_agg(jsonb_build_object('logIndex', log_index, 'instructionPath', instruction_path, 'programId', program_id, 'text', log_text) ORDER BY log_index), '[]'::jsonb) FROM kb_sol_core_logs WHERE signature = $1",
"SELECT COALESCE(jsonb_agg(jsonb_build_object('logIndex', log_index, 'instructionPath', instruction_path, 'programId', program_id, 'text', log_text) ORDER BY log_index), '[]'::jsonb) FROM k_sol_core_logs WHERE signature = $1",
signature,
"postgres core logs context query failed",
)
@@ -907,9 +895,9 @@ async fn load_balance_changes_json(
pool: &sqlx::PgPool,
signature: &str,
) -> ks_core::Result<serde_json::Value> {
return crate::postgres::query::core_queries::load_json_aggregate(
return load_json_aggregate(
pool,
"SELECT COALESCE(jsonb_agg(jsonb_build_object('balanceChangeIndex', balance_change_index, 'balanceKind', balance_kind, 'accountIndex', account_index, 'accountKey', account_key, 'mint', mint, 'owner', owner, 'pre', pre_balance_json, 'post', post_balance_json, 'delta', delta_json) ORDER BY balance_change_index), '[]'::jsonb) FROM kb_sol_core_balance_changes WHERE signature = $1",
"SELECT COALESCE(jsonb_agg(jsonb_build_object('balanceChangeIndex', balance_change_index, 'balanceKind', balance_kind, 'accountIndex', account_index, 'accountKey', account_key, 'mint', mint, 'owner', owner, 'pre', pre_balance_json, 'post', post_balance_json, 'delta', delta_json) ORDER BY balance_change_index), '[]'::jsonb) FROM k_sol_core_balance_changes WHERE signature = $1",
signature,
"postgres core balance changes context query failed",
)
@@ -937,9 +925,32 @@ async fn load_json_aggregate(
#[cfg(test)]
mod tests {
#[test]
fn outer_instruction_context_query_is_read_only_and_numeric() {
let normalized = crate::postgres::query::core_queries::OUTER_INSTRUCTIONS_CONTEXT_SQL
.to_ascii_lowercase();
fn incomplete_decode_input_query_projects_every_core_instruction_row_column() {
for column in [
"i.id",
"i.transaction_id",
"i.signature",
"i.slot",
"i.instruction_path",
"i.program_id",
"i.stack_height",
"i.accounts_json",
"i.payload_json",
"i.payload_json_hash",
"i.processing_state",
"i.created_at",
"i.updated_at",
] {
assert!(
super::INCOMPLETE_DECODE_INPUTS_SQL.contains(column),
"incomplete decode input query is missing {column}"
);
}
}
#[test]
fn top_level_instruction_context_query_is_read_only_and_numeric() {
let normalized = super::TOP_LEVEL_INSTRUCTIONS_CONTEXT_SQL.to_ascii_lowercase();
assert!(normalized.starts_with("select "));
assert!(normalized.contains("order by instruction_path::bigint, instruction_path asc"));
assert!(!normalized.contains("create "));
@@ -949,7 +960,7 @@ mod tests {
#[test]
fn instruction_processing_state_serializes_to_lower_snake_case() {
let value = crate::postgres::query::core_queries::instruction_processing_state_to_sql(
let value = super::instruction_processing_state_to_sql(
crate::CoreInstructionProcessingState::ReplayRequested,
);
assert_eq!(value, "replay_requested");
@@ -957,23 +968,19 @@ mod tests {
#[test]
fn account_key_source_serializes_to_lower_snake_case() {
let value = crate::postgres::query::core_queries::account_key_source_to_sql(
crate::CoreAccountKeySource::LoadedWritable,
);
let value = super::account_key_source_to_sql(crate::CoreAccountKeySource::LoadedWritable);
assert_eq!(value, "loaded_writable");
}
#[test]
fn balance_change_kind_serializes_to_lower_snake_case() {
let value = crate::postgres::query::core_queries::balance_change_kind_to_sql(
crate::CoreBalanceChangeKind::NativeLamports,
);
let value = super::balance_change_kind_to_sql(crate::CoreBalanceChangeKind::NativeLamports);
assert_eq!(value, "native_lamports");
}
#[test]
fn sql_slot_rejects_values_above_bigint() {
let result = crate::postgres::query::core_queries::sql_slot_from_u64(u64::MAX);
let result = super::sql_slot_from_u64(u64::MAX);
assert!(result.is_err());
}
@@ -994,7 +1001,7 @@ mod tests {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => panic!("unexpected connect error: {error}"),
};
let schema_result = store.initialize_core_store_schema().await;
let schema_result = store.initialize_store_schema().await;
if let std::result::Result::Err(error) = schema_result {
panic!("unexpected schema error: {error}");
}
@@ -1059,19 +1066,21 @@ mod tests {
});
let target = match target {
std::option::Option::Some(value) => value,
std::option::Option::None => panic!("expected replay input for outer instruction 2"),
std::option::Option::None => {
panic!("expected replay input for top-level instruction 2")
},
};
let outer = match target.outer_instructions_json.as_array() {
let top_level = match target.top_level_instructions_json.as_array() {
std::option::Option::Some(value) => value,
std::option::Option::None => panic!("outer instruction context must be an array"),
std::option::Option::None => panic!("top-level instruction context must be an array"),
};
let ordered_indexes = outer
let ordered_indexes = top_level
.iter()
.filter_map(|value| return value.get("instructionIndex"))
.filter_map(serde_json::Value::as_i64)
.collect::<std::vec::Vec<_>>();
assert_eq!(ordered_indexes, vec![0, 2, 10]);
assert!(outer.iter().any(|value| {
assert!(top_level.iter().any(|value| {
return value.get("instructionPath").and_then(serde_json::Value::as_str)
== std::option::Option::Some("2");
}));
@@ -1109,8 +1118,13 @@ mod tests {
}
fn test_core_transaction(signature: &str) -> crate::CoreTransactionInsert {
let result =
crate::CoreTransactionInsert::new(signature, 1, false, std::option::Option::None);
let result = crate::CoreTransactionInsert::new(
signature,
1,
std::option::Option::None,
false,
std::option::Option::None,
);
match result {
std::result::Result::Ok(value) => return value,
std::result::Result::Err(error) => {
@@ -1144,6 +1158,7 @@ mod tests {
1,
instruction_path,
"program",
std::option::Option::None,
serde_json::json!([0]),
serde_json::json!({"dataBase64": "YWJj"}),
);
@@ -1162,6 +1177,7 @@ mod tests {
"0",
"0/0",
"inner_program",
std::option::Option::None,
serde_json::json!([0]),
serde_json::json!({"data": "inner"}),
);

View File

@@ -1,12 +1,12 @@
// file: ks-store/src/postgres/query/decode_pipeline_queries.rs
// version: 10
// version: 11
//! PostgreSQL queries for contextual decode, coverage and materialization persistence.
use sqlx::Row; // rust-rules: trait-import
#[derive(sqlx::FromRow)]
struct MaterializedEventDatabaseRow {
struct MaterializedOutputDatabaseRow {
processor_name: std::string::String,
processor_version: std::string::String,
input_key: std::string::String,
@@ -22,48 +22,6 @@ struct MaterializedEventDatabaseRow {
updated_at: std::string::String,
}
pub(crate) async fn apply_decode_store_schema(pool: &sqlx::PgPool) -> ks_core::Result<()> {
tracing::debug!(target: crate::TRACING_TARGET, backend = "postgres", domain = "ks-store.pg", action = "apply_decode_store_schema", statement_count = crate::decode_store_schema_statements().len(), "apply PostgreSQL decode store schema");
let validation_result = crate::validate_decode_store_table_names();
if let std::result::Result::Err(error) = validation_result {
return std::result::Result::Err(error);
}
let transaction_result = pool.begin().await;
let mut transaction = match transaction_result {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => {
return std::result::Result::Err(ks_core::Error::db(format!(
"postgres decode store schema transaction failed: {error}"
)));
},
};
let lock_result = sqlx::query("SELECT pg_advisory_xact_lock($1)")
.bind(crate::STORE_SCHEMA_ADVISORY_LOCK_ID)
.execute(&mut *transaction)
.await;
if let std::result::Result::Err(error) = lock_result {
return std::result::Result::Err(ks_core::Error::db(format!(
"postgres decode store schema advisory lock failed: {error}"
)));
}
for statement in crate::decode_store_schema_statements() {
let execution_result = sqlx::query(statement).execute(&mut *transaction).await;
if let std::result::Result::Err(error) = execution_result {
return std::result::Result::Err(ks_core::Error::db(format!(
"postgres decode store schema initialization failed: {error}"
)));
}
}
let commit_result = transaction.commit().await;
if let std::result::Result::Err(error) = commit_result {
return std::result::Result::Err(ks_core::Error::db(format!(
"postgres decode store schema commit failed: {error}"
)));
}
tracing::debug!(target: crate::TRACING_TARGET, backend = "postgres", domain = "ks-store.pg", action = "apply_decode_store_schema", committed = true, "PostgreSQL decode store schema applied");
return std::result::Result::Ok(());
}
pub(crate) async fn list_decode_inputs(
pool: &sqlx::PgPool,
filter: &crate::DecodeSelectionFilter,
@@ -87,19 +45,19 @@ pub(crate) async fn list_decode_inputs(
};
}
pub(crate) async fn list_materialized_events(
pub(crate) async fn list_materialized_outputs(
pool: &sqlx::PgPool,
filter: &crate::MaterializedEventFilter,
) -> ks_core::Result<std::vec::Vec<crate::MaterializedEventQueryRow>> {
tracing::debug!(target: crate::TRACING_TARGET, backend = "postgres", domain = "ks-store.pg", action = "list_materialized_events", processor_name = ?filter.processor_name, materialized_family = ?filter.materialized_family, signature_contains = ?filter.signature_contains, limit = filter.limit, "query bounded PostgreSQL materialized events");
if filter.limit == 0 || filter.limit > crate::MAX_MATERIALIZED_EVENT_QUERY_ROWS {
filter: &crate::MaterializedOutputFilter,
) -> ks_core::Result<std::vec::Vec<crate::MaterializedOutputQueryRow>> {
tracing::debug!(target: crate::TRACING_TARGET, backend = "postgres", domain = "ks-store.pg", action = "list_materialized_outputs", processor_name = ?filter.processor_name, materialized_family = ?filter.materialized_family, signature_contains = ?filter.signature_contains, limit = filter.limit, "query bounded PostgreSQL materialized outputs");
if filter.limit == 0 || filter.limit > crate::MAX_MATERIALIZED_OUTPUT_QUERY_ROWS {
return std::result::Result::Err(ks_core::Error::db(format!(
"materialized event query limit must be between 1 and {}",
crate::MAX_MATERIALIZED_EVENT_QUERY_ROWS
"materialized output query limit must be between 1 and {}",
crate::MAX_MATERIALIZED_OUTPUT_QUERY_ROWS
)));
}
let query_result = sqlx::query_as::<sqlx::Postgres, MaterializedEventDatabaseRow>(
"SELECT processor_name, processor_version, input_key, output_key, source_event_key, source_decoder_name, source_decoder_version, signature, slot, materialized_family, payload_jsonb AS payload_json, created_at::text AS created_at, updated_at::text AS updated_at FROM kb_sol_mat_events WHERE ($1::text IS NULL OR processor_name = $1) AND ($2::text IS NULL OR materialized_family = $2) AND ($3::text IS NULL OR POSITION(LOWER($3) IN LOWER(signature)) > 0) ORDER BY slot DESC, id DESC LIMIT $4",
let query_result = sqlx::query_as::<sqlx::Postgres, MaterializedOutputDatabaseRow>(
"SELECT processor_name, processor_version, input_key, output_key, source_event_key, source_decoder_name, source_decoder_version, signature, slot, materialized_family, payload_jsonb AS payload_json, created_at::text AS created_at, updated_at::text AS updated_at FROM k_sol_mat_outputs WHERE ($1::text IS NULL OR processor_name = $1) AND ($2::text IS NULL OR materialized_family = $2) AND ($3::text IS NULL OR POSITION(LOWER($3) IN LOWER(signature)) > 0) ORDER BY slot DESC, id DESC LIMIT $4",
)
.bind(filter.processor_name.as_deref())
.bind(filter.materialized_family.as_deref())
@@ -111,7 +69,7 @@ pub(crate) async fn list_materialized_events(
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => {
return std::result::Result::Err(ks_core::Error::db(format!(
"postgres materialized event query failed: {error}"
"postgres materialized output query failed: {error}"
)));
},
};
@@ -122,11 +80,11 @@ pub(crate) async fn list_materialized_events(
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => {
return std::result::Result::Err(ks_core::Error::db(format!(
"postgres materialized event slot conversion failed: {error}"
"postgres materialized output slot conversion failed: {error}"
)));
},
};
output.push(crate::MaterializedEventQueryRow {
output.push(crate::MaterializedOutputQueryRow {
processor_name: row.processor_name,
processor_version: row.processor_version,
input_key: row.input_key,
@@ -142,7 +100,7 @@ pub(crate) async fn list_materialized_events(
updated_at: row.updated_at,
});
}
tracing::debug!(target: crate::TRACING_TARGET, backend = "postgres", domain = "ks-store.pg", action = "list_materialized_events", row_count = output.len(), "bounded PostgreSQL materialized events loaded");
tracing::debug!(target: crate::TRACING_TARGET, backend = "postgres", domain = "ks-store.pg", action = "list_materialized_outputs", row_count = output.len(), "bounded PostgreSQL materialized outputs loaded");
return std::result::Result::Ok(output);
}
@@ -152,7 +110,7 @@ pub(crate) async fn is_decode_current(
) -> ks_core::Result<bool> {
tracing::debug!(target: crate::TRACING_TARGET, backend = "postgres", domain = "ks-store.pg", action = "is_decode_current", stage = %identity.stage, processor_name = %identity.processor_name, processor_version = %identity.processor_version, input_key = %identity.input_key, input_hash = %identity.input_hash, "query PostgreSQL processing ledger current state");
let query_result = sqlx::query_scalar::<sqlx::Postgres, bool>(
"SELECT EXISTS(SELECT 1 FROM kb_sol_ops_processing_ledger WHERE stage = $1 AND processor_name = $2 AND processor_version = $3 AND input_key = $4 AND input_hash = $5 AND status = 'succeeded')",
"SELECT EXISTS(SELECT 1 FROM k_sol_ops_processing_ledger WHERE stage = $1 AND processor_name = $2 AND processor_version = $3 AND input_key = $4 AND input_hash = $5 AND status = 'succeeded')",
)
.bind(identity.stage.as_str())
.bind(identity.processor_name.as_str())
@@ -236,7 +194,7 @@ pub(crate) async fn persist_decode_coverage_declarations(
)));
}
let existing_result = sqlx::query(
"SELECT program_id, surface_code, entry_kind, entry_code, discriminator_hex, historical FROM kb_sol_decode_coverage_declarations WHERE processor_name = $1 AND processor_version = $2 FOR UPDATE",
"SELECT program_id, surface_code, entry_kind, entry_code, discriminator_hex, historical FROM k_sol_decode_coverage_declarations WHERE processor_name = $1 AND processor_version = $2 FOR UPDATE",
)
.bind(processor_name)
.bind(processor_version)
@@ -276,7 +234,7 @@ pub(crate) async fn persist_decode_coverage_declarations(
match existing_historical {
std::option::Option::None => {
let insert_result = sqlx::query(
"INSERT INTO kb_sol_decode_coverage_declarations (processor_name, processor_version, program_id, surface_code, entry_kind, entry_code, discriminator_hex, historical) VALUES ($1, $2, $3, $4, $5, $6, $7, $8)",
"INSERT INTO k_sol_decode_coverage_declarations (processor_name, processor_version, program_id, surface_code, entry_kind, entry_code, discriminator_hex, historical) VALUES ($1, $2, $3, $4, $5, $6, $7, $8)",
)
.bind(declaration.processor_name.as_str())
.bind(declaration.processor_version.as_str())
@@ -300,7 +258,7 @@ pub(crate) async fn persist_decode_coverage_declarations(
},
std::option::Option::Some(_historical) => {
let update_result = sqlx::query(
"UPDATE kb_sol_decode_coverage_declarations SET historical = $1, updated_at = NOW() WHERE processor_name = $2 AND processor_version = $3 AND program_id = $4 AND COALESCE(surface_code, '') = COALESCE($5::text, '') AND entry_kind = $6 AND entry_code = $7 AND COALESCE(discriminator_hex, '') = COALESCE($8::text, '')",
"UPDATE k_sol_decode_coverage_declarations SET historical = $1, updated_at = NOW() WHERE processor_name = $2 AND processor_version = $3 AND program_id = $4 AND COALESCE(surface_code, '') = COALESCE($5::text, '') AND entry_kind = $6 AND entry_code = $7 AND COALESCE(discriminator_hex, '') = COALESCE($8::text, '')",
)
.bind(declaration.historical)
.bind(declaration.processor_name.as_str())
@@ -323,7 +281,7 @@ pub(crate) async fn persist_decode_coverage_declarations(
}
for (key, _historical) in existing {
let delete_result = sqlx::query(
"DELETE FROM kb_sol_decode_coverage_declarations WHERE processor_name = $1 AND processor_version = $2 AND program_id = $3 AND COALESCE(surface_code, '') = COALESCE($4::text, '') AND entry_kind = $5 AND entry_code = $6 AND COALESCE(discriminator_hex, '') = COALESCE($7::text, '')",
"DELETE FROM k_sol_decode_coverage_declarations WHERE processor_name = $1 AND processor_version = $2 AND program_id = $3 AND COALESCE(surface_code, '') = COALESCE($4::text, '') AND entry_kind = $5 AND entry_code = $6 AND COALESCE(discriminator_hex, '') = COALESCE($7::text, '')",
)
.bind(processor_name)
.bind(processor_version)
@@ -372,7 +330,7 @@ pub(crate) async fn persist_decode_result(
},
};
let delete_events_result = sqlx::query(
"DELETE FROM kb_sol_decode_events WHERE processor_name = $1 AND processor_version = $2 AND input_key = $3",
"DELETE FROM k_sol_decode_events WHERE processor_name = $1 AND processor_version = $2 AND input_key = $3",
)
.bind(bundle.ledger_identity.processor_name.as_str())
.bind(bundle.ledger_identity.processor_version.as_str())
@@ -385,7 +343,7 @@ pub(crate) async fn persist_decode_result(
)));
}
let delete_coverage_result = sqlx::query(
"DELETE FROM kb_sol_decode_coverage_observations WHERE processor_name = $1 AND processor_version = $2 AND input_key = $3",
"DELETE FROM k_sol_decode_coverage_observations WHERE processor_name = $1 AND processor_version = $2 AND input_key = $3",
)
.bind(bundle.ledger_identity.processor_name.as_str())
.bind(bundle.ledger_identity.processor_version.as_str())
@@ -407,8 +365,24 @@ pub(crate) async fn persist_decode_result(
)));
},
};
let schema_kind = observation
.schema_provenance
.as_ref()
.map(|value| return value.schema_kind.as_str());
let schema_id = observation
.schema_provenance
.as_ref()
.map(|value| return value.schema_id.as_str());
let schema_version = observation
.schema_provenance
.as_ref()
.and_then(|value| return value.schema_version.as_deref());
let schema_hash = observation
.schema_provenance
.as_ref()
.map(|value| return value.schema_hash.as_str());
let insert_result = sqlx::query(
"INSERT INTO kb_sol_decode_events (processor_name, processor_version, input_key, input_hash, event_key, signature, slot, instruction_path, program_id, protocol_code, surface_code, event_code, event_name, event_family, source_kind, confidence, proof_kind, proof_jsonb, payload_jsonb, transaction_failed, transaction_error_jsonb, observation_committed) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22) ON CONFLICT (processor_name, processor_version, input_key, event_key) DO UPDATE SET input_hash = EXCLUDED.input_hash, payload_jsonb = EXCLUDED.payload_jsonb, proof_jsonb = EXCLUDED.proof_jsonb, transaction_failed = EXCLUDED.transaction_failed, transaction_error_jsonb = EXCLUDED.transaction_error_jsonb, observation_committed = EXCLUDED.observation_committed, updated_at = NOW()",
"INSERT INTO k_sol_decode_events (processor_name, processor_version, input_key, input_hash, event_key, signature, slot, instruction_path, program_id, protocol_code, surface_code, event_code, event_name, event_family, source_kind, confidence, proof_kind, proof_jsonb, payload_jsonb, schema_kind, schema_id, schema_version, schema_hash, transaction_failed, transaction_error_jsonb, observation_committed) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26) ON CONFLICT (processor_name, processor_version, input_key, event_key) DO UPDATE SET input_hash = EXCLUDED.input_hash, payload_jsonb = EXCLUDED.payload_jsonb, proof_jsonb = EXCLUDED.proof_jsonb, schema_kind = EXCLUDED.schema_kind, schema_id = EXCLUDED.schema_id, schema_version = EXCLUDED.schema_version, schema_hash = EXCLUDED.schema_hash, transaction_failed = EXCLUDED.transaction_failed, transaction_error_jsonb = EXCLUDED.transaction_error_jsonb, observation_committed = EXCLUDED.observation_committed, updated_at = NOW()",
)
.bind(observation.processor_name.as_str())
.bind(observation.processor_version.as_str())
@@ -429,6 +403,10 @@ pub(crate) async fn persist_decode_result(
.bind(observation.proof_kind.as_str())
.bind(&observation.proof_json)
.bind(&observation.payload_json)
.bind(schema_kind)
.bind(schema_id)
.bind(schema_version)
.bind(schema_hash)
.bind(observation.transaction_failed)
.bind(observation.transaction_error.clone())
.bind(observation.observation_committed)
@@ -467,7 +445,7 @@ pub(crate) async fn persist_decode_result(
_unknown => "decoded",
};
let lifecycle_result = sqlx::query(
"UPDATE kb_sol_core_instructions SET processing_state = CASE WHEN $1 = 'decoded' AND processing_state <> 'materialized' THEN 'decoded' WHEN $1 IN ('ignored', 'failed') AND processing_state IN ('pending', 'failed', 'replay_requested', 'ignored') THEN $1 ELSE processing_state END, processor_name = CASE WHEN $1 = 'decoded' AND processing_state <> 'materialized' THEN $2 WHEN $1 IN ('ignored', 'failed') AND processing_state IN ('pending', 'failed', 'replay_requested', 'ignored') THEN $2 ELSE processor_name END, processor_version = CASE WHEN $1 = 'decoded' AND processing_state <> 'materialized' THEN $3 WHEN $1 IN ('ignored', 'failed') AND processing_state IN ('pending', 'failed', 'replay_requested', 'ignored') THEN $3 ELSE processor_version END, lifecycle_reason = CASE WHEN $1 = 'decoded' AND processing_state <> 'materialized' THEN $4 WHEN $1 IN ('ignored', 'failed') AND processing_state IN ('pending', 'failed', 'replay_requested', 'ignored') THEN $4 ELSE lifecycle_reason END, updated_at = NOW() WHERE signature = $5 AND instruction_path = $6",
"UPDATE k_sol_core_instructions SET processing_state = CASE WHEN $1 = 'decoded' AND processing_state <> 'materialized' THEN 'decoded' WHEN $1 IN ('ignored', 'failed') AND processing_state IN ('pending', 'failed', 'replay_requested', 'ignored') THEN $1 ELSE processing_state END, processor_name = CASE WHEN $1 = 'decoded' AND processing_state <> 'materialized' THEN $2 WHEN $1 IN ('ignored', 'failed') AND processing_state IN ('pending', 'failed', 'replay_requested', 'ignored') THEN $2 ELSE processor_name END, processor_version = CASE WHEN $1 = 'decoded' AND processing_state <> 'materialized' THEN $3 WHEN $1 IN ('ignored', 'failed') AND processing_state IN ('pending', 'failed', 'replay_requested', 'ignored') THEN $3 ELSE processor_version END, lifecycle_reason = CASE WHEN $1 = 'decoded' AND processing_state <> 'materialized' THEN $4 WHEN $1 IN ('ignored', 'failed') AND processing_state IN ('pending', 'failed', 'replay_requested', 'ignored') THEN $4 ELSE lifecycle_reason END, updated_at = NOW() WHERE signature = $5 AND instruction_path = $6",
)
.bind(lifecycle_state)
.bind(bundle.ledger_identity.processor_name.as_str())
@@ -537,7 +515,7 @@ pub(crate) async fn mark_decode_failed(
},
};
let lifecycle_result = sqlx::query(
"UPDATE kb_sol_core_instructions SET processing_state = CASE WHEN processing_state IN ('pending', 'failed', 'replay_requested', 'ignored') THEN 'failed' ELSE processing_state END, processor_name = CASE WHEN processing_state IN ('pending', 'failed', 'replay_requested', 'ignored') THEN $1 ELSE processor_name END, processor_version = CASE WHEN processing_state IN ('pending', 'failed', 'replay_requested', 'ignored') THEN $2 ELSE processor_version END, lifecycle_reason = CASE WHEN processing_state IN ('pending', 'failed', 'replay_requested', 'ignored') THEN $3 ELSE lifecycle_reason END, updated_at = NOW() WHERE signature = $4 AND instruction_path = $5",
"UPDATE k_sol_core_instructions SET processing_state = CASE WHEN processing_state IN ('pending', 'failed', 'replay_requested', 'ignored') THEN 'failed' ELSE processing_state END, processor_name = CASE WHEN processing_state IN ('pending', 'failed', 'replay_requested', 'ignored') THEN $1 ELSE processor_name END, processor_version = CASE WHEN processing_state IN ('pending', 'failed', 'replay_requested', 'ignored') THEN $2 ELSE processor_version END, lifecycle_reason = CASE WHEN processing_state IN ('pending', 'failed', 'replay_requested', 'ignored') THEN $3 ELSE lifecycle_reason END, updated_at = NOW() WHERE signature = $4 AND instruction_path = $5",
)
.bind(failure.ledger_identity.processor_name.as_str())
.bind(failure.ledger_identity.processor_version.as_str())
@@ -610,7 +588,7 @@ pub(crate) async fn persist_materialization_result(
},
};
let delete_result = sqlx::query(
"DELETE FROM kb_sol_mat_events WHERE processor_name = $1 AND processor_version = $2 AND input_key = $3",
"DELETE FROM k_sol_mat_outputs WHERE processor_name = $1 AND processor_version = $2 AND input_key = $3",
)
.bind(bundle.ledger_identity.processor_name.as_str())
.bind(bundle.ledger_identity.processor_version.as_str())
@@ -633,7 +611,7 @@ pub(crate) async fn persist_materialization_result(
},
};
let insert_result = sqlx::query(
"INSERT INTO kb_sol_mat_events (processor_name, processor_version, input_key, input_hash, output_key, source_event_key, source_decoder_name, source_decoder_version, source_decode_input_key, signature, slot, materialized_family, payload_jsonb) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13) ON CONFLICT (processor_name, processor_version, input_key, output_key) DO UPDATE SET input_hash = EXCLUDED.input_hash, source_decoder_name = EXCLUDED.source_decoder_name, source_decoder_version = EXCLUDED.source_decoder_version, source_decode_input_key = EXCLUDED.source_decode_input_key, payload_jsonb = EXCLUDED.payload_jsonb, updated_at = NOW()",
"INSERT INTO k_sol_mat_outputs (processor_name, processor_version, input_key, input_hash, output_key, source_event_key, source_decoder_name, source_decoder_version, source_decode_input_key, signature, slot, materialized_family, payload_jsonb) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13) ON CONFLICT (processor_name, processor_version, input_key, output_key) DO UPDATE SET input_hash = EXCLUDED.input_hash, source_decoder_name = EXCLUDED.source_decoder_name, source_decoder_version = EXCLUDED.source_decoder_version, source_decode_input_key = EXCLUDED.source_decode_input_key, payload_jsonb = EXCLUDED.payload_jsonb, updated_at = NOW()",
)
.bind(output.processor_name.as_str())
.bind(output.processor_version.as_str())
@@ -670,7 +648,7 @@ pub(crate) async fn persist_materialization_result(
}
if !bundle.outputs.is_empty() {
let lifecycle_result = sqlx::query(
"UPDATE kb_sol_core_instructions SET processing_state = 'materialized', processor_name = $1, processor_version = $2, lifecycle_reason = $3, updated_at = NOW() WHERE signature = $4 AND instruction_path = $5",
"UPDATE k_sol_core_instructions SET processing_state = 'materialized', processor_name = $1, processor_version = $2, lifecycle_reason = $3, updated_at = NOW() WHERE signature = $4 AND instruction_path = $5",
)
.bind(bundle.ledger_identity.processor_name.as_str())
.bind(bundle.ledger_identity.processor_version.as_str())
@@ -729,7 +707,7 @@ pub(crate) async fn list_decode_coverage_summary(
));
}
let query_result = sqlx::query(
"WITH declared AS (SELECT processor_name, processor_version, program_id, surface_code, entry_code, COUNT(*)::BIGINT AS declared_count FROM kb_sol_decode_coverage_declarations WHERE ($1::text IS NULL OR processor_name = $1) AND ($2::text IS NULL OR processor_version = $2) GROUP BY processor_name, processor_version, program_id, surface_code, entry_code), observed AS (SELECT processor_name, processor_version, program_id, surface_code, COALESCE(entry_code, 'unknown') AS entry_code, COUNT(*)::BIGINT AS observed_count, COUNT(*) FILTER (WHERE recognized)::BIGINT AS recognized_count, COALESCE(SUM(decoded_count), 0)::BIGINT AS decoded_count, COALESCE(SUM(materialized_count), 0)::BIGINT AS materialized_count, COALESCE(SUM(error_count), 0)::BIGINT AS error_count, COUNT(*) FILTER (WHERE status = 'unsupported' OR entry_code IS NULL)::BIGINT AS unknown_count, COUNT(*) FILTER (WHERE NOT transaction_failed)::BIGINT AS successful_transaction_count, COUNT(*) FILTER (WHERE transaction_failed)::BIGINT AS failed_transaction_count FROM kb_sol_decode_coverage_observations WHERE ($1::text IS NULL OR processor_name = $1) AND ($2::text IS NULL OR processor_version = $2) GROUP BY processor_name, processor_version, program_id, surface_code, COALESCE(entry_code, 'unknown')) SELECT COALESCE(d.processor_name, o.processor_name) AS processor_name, COALESCE(d.processor_version, o.processor_version) AS processor_version, COALESCE(d.program_id, o.program_id) AS program_id, COALESCE(d.surface_code, o.surface_code) AS surface_code, COALESCE(d.entry_code, o.entry_code) AS entry_code, COALESCE(d.declared_count, 0)::BIGINT AS declared_count, COALESCE(o.observed_count, 0)::BIGINT AS observed_count, COALESCE(o.recognized_count, 0)::BIGINT AS recognized_count, COALESCE(o.decoded_count, 0)::BIGINT AS decoded_count, COALESCE(o.materialized_count, 0)::BIGINT AS materialized_count, COALESCE(o.error_count, 0)::BIGINT AS error_count, COALESCE(o.unknown_count, 0)::BIGINT AS unknown_count, COALESCE(o.successful_transaction_count, 0)::BIGINT AS successful_transaction_count, COALESCE(o.failed_transaction_count, 0)::BIGINT AS failed_transaction_count FROM declared d FULL OUTER JOIN observed o ON d.processor_name = o.processor_name AND d.processor_version = o.processor_version AND d.program_id = o.program_id AND COALESCE(d.surface_code, '') = COALESCE(o.surface_code, '') AND d.entry_code = o.entry_code ORDER BY observed_count DESC, processor_name, program_id, entry_code LIMIT $3",
"WITH declared AS (SELECT processor_name, processor_version, program_id, surface_code, entry_code, COUNT(*)::BIGINT AS declared_count FROM k_sol_decode_coverage_declarations WHERE ($1::text IS NULL OR processor_name = $1) AND ($2::text IS NULL OR processor_version = $2) GROUP BY processor_name, processor_version, program_id, surface_code, entry_code), observed AS (SELECT processor_name, processor_version, program_id, surface_code, COALESCE(entry_code, 'unknown') AS entry_code, COUNT(*)::BIGINT AS observed_count, COUNT(*) FILTER (WHERE recognized)::BIGINT AS recognized_count, COALESCE(SUM(decoded_count), 0)::BIGINT AS decoded_count, COALESCE(SUM(materialized_count), 0)::BIGINT AS materialized_count, COALESCE(SUM(error_count), 0)::BIGINT AS error_count, COUNT(*) FILTER (WHERE status = 'unsupported' OR entry_code IS NULL)::BIGINT AS unknown_count, COUNT(*) FILTER (WHERE NOT transaction_failed)::BIGINT AS successful_transaction_count, COUNT(*) FILTER (WHERE transaction_failed)::BIGINT AS failed_transaction_count FROM k_sol_decode_coverage_observations WHERE ($1::text IS NULL OR processor_name = $1) AND ($2::text IS NULL OR processor_version = $2) GROUP BY processor_name, processor_version, program_id, surface_code, COALESCE(entry_code, 'unknown')) SELECT COALESCE(d.processor_name, o.processor_name) AS processor_name, COALESCE(d.processor_version, o.processor_version) AS processor_version, COALESCE(d.program_id, o.program_id) AS program_id, COALESCE(d.surface_code, o.surface_code) AS surface_code, COALESCE(d.entry_code, o.entry_code) AS entry_code, COALESCE(d.declared_count, 0)::BIGINT AS declared_count, COALESCE(o.observed_count, 0)::BIGINT AS observed_count, COALESCE(o.recognized_count, 0)::BIGINT AS recognized_count, COALESCE(o.decoded_count, 0)::BIGINT AS decoded_count, COALESCE(o.materialized_count, 0)::BIGINT AS materialized_count, COALESCE(o.error_count, 0)::BIGINT AS error_count, COALESCE(o.unknown_count, 0)::BIGINT AS unknown_count, COALESCE(o.successful_transaction_count, 0)::BIGINT AS successful_transaction_count, COALESCE(o.failed_transaction_count, 0)::BIGINT AS failed_transaction_count FROM declared d FULL OUTER JOIN observed o ON d.processor_name = o.processor_name AND d.processor_version = o.processor_version AND d.program_id = o.program_id AND COALESCE(d.surface_code, '') = COALESCE(o.surface_code, '') AND d.entry_code = o.entry_code ORDER BY observed_count DESC, processor_name, program_id, entry_code LIMIT $3",
)
.bind(processor_name)
.bind(processor_version)
@@ -798,7 +776,7 @@ async fn upsert_coverage_observation(
},
};
let query_result = sqlx::query(
"INSERT INTO kb_sol_decode_coverage_observations (processor_name, processor_version, input_key, input_hash, signature, slot, instruction_path, program_id, surface_code, entry_code, discriminator_hex, status, recognized, decoded_count, materialized_count, error_count, transaction_failed) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17) ON CONFLICT (processor_name, processor_version, input_key) DO UPDATE SET input_hash = EXCLUDED.input_hash, surface_code = EXCLUDED.surface_code, entry_code = EXCLUDED.entry_code, discriminator_hex = EXCLUDED.discriminator_hex, status = EXCLUDED.status, recognized = EXCLUDED.recognized, decoded_count = EXCLUDED.decoded_count, materialized_count = EXCLUDED.materialized_count, error_count = EXCLUDED.error_count, transaction_failed = EXCLUDED.transaction_failed, updated_at = NOW()",
"INSERT INTO k_sol_decode_coverage_observations (processor_name, processor_version, input_key, input_hash, signature, slot, instruction_path, program_id, surface_code, entry_code, discriminator_hex, status, recognized, decoded_count, materialized_count, error_count, transaction_failed) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17) ON CONFLICT (processor_name, processor_version, input_key) DO UPDATE SET input_hash = EXCLUDED.input_hash, surface_code = EXCLUDED.surface_code, entry_code = EXCLUDED.entry_code, discriminator_hex = EXCLUDED.discriminator_hex, status = EXCLUDED.status, recognized = EXCLUDED.recognized, decoded_count = EXCLUDED.decoded_count, materialized_count = EXCLUDED.materialized_count, error_count = EXCLUDED.error_count, transaction_failed = EXCLUDED.transaction_failed, updated_at = NOW()",
)
.bind(coverage.processor_name.as_str())
.bind(coverage.processor_version.as_str())
@@ -834,7 +812,7 @@ async fn refresh_materialized_coverage_count(
decode_input_key: &str,
) -> ks_core::Result<()> {
let query_result = sqlx::query(
"UPDATE kb_sol_decode_coverage_observations SET materialized_count = (SELECT COUNT(*)::INTEGER FROM kb_sol_mat_events WHERE source_decoder_name = $1 AND source_decoder_version = $2 AND source_decode_input_key = $3), updated_at = NOW() WHERE processor_name = $1 AND processor_version = $2 AND input_key = $3",
"UPDATE k_sol_decode_coverage_observations SET materialized_count = (SELECT COUNT(*)::INTEGER FROM k_sol_mat_outputs WHERE source_decoder_name = $1 AND source_decoder_version = $2 AND source_decode_input_key = $3), updated_at = NOW() WHERE processor_name = $1 AND processor_version = $2 AND input_key = $3",
)
.bind(decoder_name)
.bind(decoder_version)
@@ -857,7 +835,7 @@ async fn upsert_ledger_terminal(
error_message: std::option::Option<&str>,
) -> ks_core::Result<()> {
let query_result = sqlx::query(
"INSERT INTO kb_sol_ops_processing_ledger (stage, processor_name, processor_version, input_key, input_hash, status, attempt_count, started_at, finished_at, error_code, error_message) VALUES ($1, $2, $3, $4, $5, $6, 1, NOW(), NOW(), $7, $8) ON CONFLICT (stage, processor_name, processor_version, input_key) DO UPDATE SET input_hash = EXCLUDED.input_hash, status = EXCLUDED.status, attempt_count = kb_sol_ops_processing_ledger.attempt_count + 1, started_at = NOW(), finished_at = NOW(), error_code = EXCLUDED.error_code, error_message = EXCLUDED.error_message, updated_at = NOW()",
"INSERT INTO k_sol_ops_processing_ledger (stage, processor_name, processor_version, input_key, input_hash, status, attempt_count, started_at, finished_at, error_code, error_message) VALUES ($1, $2, $3, $4, $5, $6, 1, NOW(), NOW(), $7, $8) ON CONFLICT (stage, processor_name, processor_version, input_key) DO UPDATE SET input_hash = EXCLUDED.input_hash, status = EXCLUDED.status, attempt_count = k_sol_ops_processing_ledger.attempt_count + 1, started_at = NOW(), finished_at = NOW(), error_code = EXCLUDED.error_code, error_message = EXCLUDED.error_message, updated_at = NOW()",
)
.bind(identity.stage.as_str())
.bind(identity.processor_name.as_str())
@@ -1032,9 +1010,7 @@ mod tests {
};
let pool_result = sqlx::PgPool::connect(url.as_str()).await;
let pool = result_or_panic(pool_result);
result_or_panic(crate::apply_raw_store_schema(&pool).await);
result_or_panic(crate::apply_core_store_schema(&pool).await);
result_or_panic(crate::apply_decode_store_schema(&pool).await);
result_or_panic(crate::apply_store_schema(&pool).await);
return std::option::Option::Some(pool);
}
@@ -1069,12 +1045,11 @@ mod tests {
crate::persist_decode_coverage_declarations(&pool, &[declaration]).await,
);
assert_eq!(third, crate::InsertOutcome::new(0, 1, 0));
let cleanup_result = sqlx::query(
"DELETE FROM kb_sol_decode_coverage_declarations WHERE processor_name = $1",
)
.bind(processor_name.as_str())
.execute(&pool)
.await;
let cleanup_result =
sqlx::query("DELETE FROM k_sol_decode_coverage_declarations WHERE processor_name = $1")
.bind(processor_name.as_str())
.execute(&pool)
.await;
result_or_panic(cleanup_result);
}
@@ -1126,15 +1101,14 @@ mod tests {
result_or_panic(crate::persist_decode_result(&pool, &bundle, true).await);
let current = result_or_panic(crate::is_decode_current(&pool, &identity).await);
assert!(current);
let cleanup_coverage = sqlx::query(
"DELETE FROM kb_sol_decode_coverage_observations WHERE processor_name = $1",
)
.bind(processor_name.as_str())
.execute(&pool)
.await;
let cleanup_coverage =
sqlx::query("DELETE FROM k_sol_decode_coverage_observations WHERE processor_name = $1")
.bind(processor_name.as_str())
.execute(&pool)
.await;
result_or_panic(cleanup_coverage);
let cleanup_ledger =
sqlx::query("DELETE FROM kb_sol_ops_processing_ledger WHERE processor_name = $1")
sqlx::query("DELETE FROM k_sol_ops_processing_ledger WHERE processor_name = $1")
.bind(processor_name.as_str())
.execute(&pool)
.await;
@@ -1142,7 +1116,7 @@ mod tests {
}
#[tokio::test]
async fn optional_postgres_materialized_event_query_is_bounded_and_typed_from_env() {
async fn optional_postgres_materialized_output_query_is_bounded_and_typed_from_env() {
let _postgres_guard = crate::postgres_test_guard().await;
let pool = match test_pool_from_env().await {
std::option::Option::Some(value) => value,
@@ -1150,7 +1124,7 @@ mod tests {
};
let input_key = unique_processor_name("annotation_query_test");
let insert_result = sqlx::query(
"INSERT INTO kb_sol_mat_events (processor_name, processor_version, input_key, input_hash, output_key, source_event_key, source_decoder_name, source_decoder_version, source_decode_input_key, signature, slot, materialized_family, payload_jsonb) VALUES ('materializer.transaction.annotations', '0.4.3', $1, 'hash', 'annotation', 'memo:0', 'ks-lib-decoder.spl.memo', '0.4.3', 'decode-input', $2, 42, 'transaction_annotation', $3)",
"INSERT INTO k_sol_mat_outputs (processor_name, processor_version, input_key, input_hash, output_key, source_event_key, source_decoder_name, source_decoder_version, source_decode_input_key, signature, slot, materialized_family, payload_jsonb) VALUES ('materializer.transaction.annotations', '0.4.3', $1, 'hash', 'annotation', 'memo:0', 'ks-lib-decoder.spl.memo', '0.4.3', 'decode-input', $2, 42, 'transaction_annotation', $3)",
)
.bind(input_key.as_str())
.bind(input_key.as_str())
@@ -1158,17 +1132,17 @@ mod tests {
.execute(&pool)
.await;
result_or_panic(insert_result);
let filter = result_or_panic(crate::MaterializedEventFilter::new(
let filter = result_or_panic(crate::MaterializedOutputFilter::new(
std::option::Option::Some("materializer.transaction.annotations".to_string()),
std::option::Option::Some("transaction_annotation".to_string()),
std::option::Option::Some(input_key.clone()),
1,
));
let rows = result_or_panic(crate::list_materialized_events(&pool, &filter).await);
let rows = result_or_panic(crate::list_materialized_outputs(&pool, &filter).await);
assert_eq!(rows.len(), 1);
assert_eq!(rows[0].slot, 42);
assert_eq!(rows[0].payload_json["text"], "postgres annotation");
let cleanup_result = sqlx::query("DELETE FROM kb_sol_mat_events WHERE input_key = $1")
let cleanup_result = sqlx::query("DELETE FROM k_sol_mat_outputs WHERE input_key = $1")
.bind(input_key.as_str())
.execute(&pool)
.await;
@@ -1184,22 +1158,20 @@ mod tests {
let _postgres_guard = crate::postgres_test_guard().await;
let pool_result = sqlx::PgPool::connect(url.as_str()).await;
let pool = result_or_panic(pool_result);
result_or_panic(crate::apply_raw_store_schema(&pool).await);
result_or_panic(crate::apply_core_store_schema(&pool).await);
result_or_panic(crate::apply_decode_store_schema(&pool).await);
result_or_panic(crate::apply_store_schema(&pool).await);
execute_sql(
&pool,
"DELETE FROM kb_sol_decode_events WHERE processor_name = 'decode_atomic_rollback_test'",
"DELETE FROM k_sol_decode_events WHERE processor_name = 'decode_atomic_rollback_test'",
)
.await;
execute_sql(
&pool,
"DELETE FROM kb_sol_decode_coverage_observations WHERE processor_name = 'decode_atomic_rollback_test'",
"DELETE FROM k_sol_decode_coverage_observations WHERE processor_name = 'decode_atomic_rollback_test'",
)
.await;
execute_sql(
&pool,
"DELETE FROM kb_sol_ops_processing_ledger WHERE processor_name = 'decode_atomic_rollback_test'",
"DELETE FROM k_sol_ops_processing_ledger WHERE processor_name = 'decode_atomic_rollback_test'",
)
.await;
execute_sql(
@@ -1209,12 +1181,12 @@ mod tests {
.await;
execute_sql(
&pool,
"DROP TRIGGER IF EXISTS ks_test_reject_decode_ledger_trigger ON kb_sol_ops_processing_ledger",
"DROP TRIGGER IF EXISTS ks_test_reject_decode_ledger_trigger ON k_sol_ops_processing_ledger",
)
.await;
execute_sql(
&pool,
"CREATE TRIGGER ks_test_reject_decode_ledger_trigger BEFORE INSERT OR UPDATE ON kb_sol_ops_processing_ledger FOR EACH ROW EXECUTE FUNCTION ks_test_reject_decode_ledger()",
"CREATE TRIGGER ks_test_reject_decode_ledger_trigger BEFORE INSERT OR UPDATE ON k_sol_ops_processing_ledger FOR EACH ROW EXECUTE FUNCTION ks_test_reject_decode_ledger()",
)
.await;
let identity_result = crate::ProcessingLedgerIdentity::new(
@@ -1264,6 +1236,7 @@ mod tests {
proof_kind: "exact_layout".to_string(),
proof_json: serde_json::json!({"source": "test"}),
payload_json: serde_json::json!({"value": 1}),
schema_provenance: std::option::Option::None,
transaction_failed: false,
transaction_error: std::option::Option::None,
observation_committed: true,
@@ -1281,13 +1254,13 @@ mod tests {
let persistence_result = crate::persist_decode_result(&pool, &bundle, true).await;
assert!(persistence_result.is_err());
let event_count_result = sqlx::query_scalar::<sqlx::Postgres, i64>(
"SELECT COUNT(*) FROM kb_sol_decode_events WHERE processor_name = 'decode_atomic_rollback_test'",
"SELECT COUNT(*) FROM k_sol_decode_events WHERE processor_name = 'decode_atomic_rollback_test'",
)
.fetch_one(&pool)
.await;
let event_count = result_or_panic(event_count_result);
let ledger_count_result = sqlx::query_scalar::<sqlx::Postgres, i64>(
"SELECT COUNT(*) FROM kb_sol_ops_processing_ledger WHERE processor_name = 'decode_atomic_rollback_test'",
"SELECT COUNT(*) FROM k_sol_ops_processing_ledger WHERE processor_name = 'decode_atomic_rollback_test'",
)
.fetch_one(&pool)
.await;
@@ -1296,7 +1269,7 @@ mod tests {
assert_eq!(ledger_count, 0);
execute_sql(
&pool,
"DROP TRIGGER IF EXISTS ks_test_reject_decode_ledger_trigger ON kb_sol_ops_processing_ledger",
"DROP TRIGGER IF EXISTS ks_test_reject_decode_ledger_trigger ON k_sol_ops_processing_ledger",
)
.await;
execute_sql(&pool, "DROP FUNCTION IF EXISTS ks_test_reject_decode_ledger()").await;

View File

@@ -1,5 +1,5 @@
// file: ks-store/src/postgres/query/health_queries.rs
// version: 3
// version: 4
//! PostgreSQL health and diagnostic SQL queries.
@@ -42,36 +42,3 @@ pub(crate) async fn load_server_version(
))),
};
}
pub(crate) async fn load_migration_table_name(
pool: &sqlx::PgPool,
) -> ks_core::Result<std::option::Option<std::string::String>> {
let query_result =
sqlx::query_scalar::<sqlx::Postgres, std::option::Option<std::string::String>>(
"SELECT to_regclass('_sqlx_migrations')::text",
)
.fetch_one(pool)
.await;
return match query_result {
std::result::Result::Ok(table_name) => std::result::Result::Ok(table_name),
std::result::Result::Err(error) => std::result::Result::Err(ks_core::Error::db(format!(
"postgres migration table query failed: {error}"
))),
};
}
pub(crate) async fn load_latest_migration_version(
pool: &sqlx::PgPool,
) -> ks_core::Result<std::option::Option<std::string::String>> {
let query_result = sqlx::query_scalar::<sqlx::Postgres, std::option::Option<std::string::String>>(
"SELECT version::text FROM _sqlx_migrations WHERE success = true ORDER BY version DESC LIMIT 1",
)
.fetch_optional(pool)
.await;
return match query_result {
std::result::Result::Ok(version) => std::result::Result::Ok(version.flatten()),
std::result::Result::Err(error) => std::result::Result::Err(ks_core::Error::db(format!(
"postgres latest migration query failed: {error}"
))),
};
}

View File

@@ -1,61 +1,19 @@
// file: ks-store/src/postgres/query/raw_queries.rs
// version: 6
// version: 8
//! PostgreSQL canonical transaction and acquisition observation SQL queries.
pub(crate) async fn apply_raw_store_schema(pool: &sqlx::PgPool) -> ks_core::Result<()> {
let validation_result = crate::validate_raw_store_table_names();
if let std::result::Result::Err(error) = validation_result {
return std::result::Result::Err(error);
}
let transaction_result = pool.begin().await;
let mut transaction = match transaction_result {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => {
return std::result::Result::Err(ks_core::Error::db(format!(
"postgres canonical raw store schema transaction failed: {error}"
)));
},
};
let lock_result = sqlx::query("SELECT pg_advisory_xact_lock($1)")
.bind(crate::STORE_SCHEMA_ADVISORY_LOCK_ID)
.execute(&mut *transaction)
.await;
if let std::result::Result::Err(error) = lock_result {
return std::result::Result::Err(ks_core::Error::db(format!(
"postgres canonical raw store schema advisory lock failed: {error}"
)));
}
for statement in crate::raw_store_schema_statements() {
let execution_result = sqlx::query(statement).execute(&mut *transaction).await;
if let std::result::Result::Err(error) = execution_result {
return std::result::Result::Err(ks_core::Error::db(format!(
"postgres canonical raw store schema statement failed: {error}"
)));
}
}
let commit_result = transaction.commit().await;
if let std::result::Result::Err(error) = commit_result {
return std::result::Result::Err(ks_core::Error::db(format!(
"postgres canonical raw store schema commit failed: {error}"
)));
}
return std::result::Result::Ok(());
}
pub(crate) async fn has_raw_transaction_signature(
pool: &sqlx::PgPool,
signature: &str,
) -> ks_core::Result<bool> {
let validation_result = crate::postgres::query::raw_queries::validate_required_text(
signature,
"canonical raw transaction signature must not be empty",
);
let validation_result =
validate_required_text(signature, "canonical raw transaction signature must not be empty");
if let std::result::Result::Err(error) = validation_result {
return std::result::Result::Err(error);
}
let query_result = sqlx::query_scalar::<sqlx::Postgres, bool>(
"SELECT EXISTS(SELECT 1 FROM kb_sol_raw_transactions WHERE signature = $1)",
"SELECT EXISTS(SELECT 1 FROM k_sol_raw_transactions WHERE signature = $1)",
)
.bind(signature)
.fetch_one(pool)
@@ -72,15 +30,13 @@ pub(crate) async fn has_transaction_observation_key(
pool: &sqlx::PgPool,
observation_key: &str,
) -> ks_core::Result<bool> {
let validation_result = crate::postgres::query::raw_queries::validate_required_text(
observation_key,
"transaction observation key must not be empty",
);
let validation_result =
validate_required_text(observation_key, "transaction observation key must not be empty");
if let std::result::Result::Err(error) = validation_result {
return std::result::Result::Err(error);
}
let query_result = sqlx::query_scalar::<sqlx::Postgres, bool>(
"SELECT EXISTS(SELECT 1 FROM kb_sol_obs_transaction_observations WHERE observation_key = $1)",
"SELECT EXISTS(SELECT 1 FROM k_sol_obs_transaction_observations WHERE observation_key = $1)",
)
.bind(observation_key)
.fetch_one(pool)
@@ -97,7 +53,7 @@ pub(crate) async fn insert_raw_transaction(
pool: &sqlx::PgPool,
input: &crate::RawTransactionInsert,
) -> ks_core::Result<crate::InsertOutcome> {
let slot_result = crate::postgres::query::raw_queries::sql_slot_from_u64(input.slot);
let slot_result = sql_slot_from_u64(input.slot);
let slot = match slot_result {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
@@ -112,10 +68,11 @@ pub(crate) async fn insert_raw_transaction(
},
};
let query_result = sqlx::query_scalar::<sqlx::Postgres, i64>(
"INSERT INTO kb_sol_raw_transactions (signature, slot, canonical_json, canonical_json_hash, canonical_format_version, retention_state, processing_state) VALUES ($1, $2, $3, $4, $5, 'full', 'received') ON CONFLICT (signature) DO NOTHING RETURNING id",
"INSERT INTO k_sol_raw_transactions (signature, slot, block_time, canonical_json, canonical_json_hash, canonical_format_version, retention_state, processing_state) VALUES ($1, $2, $3, $4, $5, $6, 'full', 'received') ON CONFLICT (signature) DO NOTHING RETURNING id",
)
.bind(input.signature.as_str())
.bind(slot)
.bind(input.block_time)
.bind(&input.canonical_json)
.bind(input.canonical_json_hash.as_deref())
.bind(canonical_format_version)
@@ -138,23 +95,20 @@ pub(crate) async fn insert_transaction_observation(
pool: &sqlx::PgPool,
input: &crate::TransactionObservationInsert,
) -> ks_core::Result<crate::InsertOutcome> {
let slot_result = crate::postgres::query::raw_queries::optional_sql_slot_from_u64(input.slot);
let slot_result = optional_sql_slot_from_u64(input.slot);
let slot = match slot_result {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let payload_size_result =
crate::postgres::query::raw_queries::optional_sql_bigint_from_u64(input.payload_size_bytes);
let payload_size_result = optional_sql_bigint_from_u64(input.payload_size_bytes);
let payload_size_bytes = match payload_size_result {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let origin =
crate::postgres::query::raw_queries::transaction_observation_origin_to_sql(input.origin);
let status =
crate::postgres::query::raw_queries::transaction_observation_status_to_sql(input.status);
let origin = transaction_observation_origin_to_sql(input.origin);
let status = transaction_observation_status_to_sql(input.status);
let query_result = sqlx::query_scalar::<sqlx::Postgres, i64>(
"INSERT INTO kb_sol_obs_transaction_observations (raw_transaction_id, observation_key, signature, slot, provider, endpoint_code, protocol, acquisition_method, origin, commitment, capture_session_id, filter_code, detected_at, received_at, normalized_at, payload_size_bytes, source_payload_hash, status, error_code, error_message) VALUES (COALESCE($1, (SELECT id FROM kb_sol_raw_transactions WHERE signature = $3 LIMIT 1)), $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20) ON CONFLICT (observation_key) DO NOTHING RETURNING id",
"INSERT INTO k_sol_obs_transaction_observations (raw_transaction_id, observation_key, signature, slot, provider, endpoint_code, protocol, acquisition_method, origin, commitment, capture_session_id, filter_code, detected_at, received_at, normalized_at, payload_size_bytes, source_payload_hash, status, error_code, error_message) VALUES (COALESCE($1, (SELECT id FROM k_sol_raw_transactions WHERE signature = $3 LIMIT 1)), $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20) ON CONFLICT (observation_key) DO NOTHING RETURNING id",
)
.bind(input.raw_transaction_id)
.bind(input.observation_key.as_str())
@@ -200,7 +154,7 @@ pub(crate) async fn update_raw_payload_lifecycle(
"raw lifecycle table name is not supported by the PostgreSQL canonical raw store",
));
}
return crate::postgres::query::raw_queries::update_raw_transaction_lifecycle(pool, mark).await;
return update_raw_transaction_lifecycle(pool, mark).await;
}
fn raw_retention_state_to_sql(state: crate::RawPayloadRetentionState) -> &'static str {
@@ -223,27 +177,27 @@ fn raw_processing_state_to_sql(state: crate::RawPayloadProcessingState) -> &'sta
}
fn transaction_observation_origin_to_sql(
origin: crate::TransactionObservationOrigin,
origin: crate::AcquisitionObservationOrigin,
) -> &'static str {
return match origin {
crate::TransactionObservationOrigin::Live => "live",
crate::TransactionObservationOrigin::Backfill => "backfill",
crate::TransactionObservationOrigin::Replay => "replay",
crate::TransactionObservationOrigin::Repair => "repair",
crate::TransactionObservationOrigin::Migration => "migration",
crate::AcquisitionObservationOrigin::Live => "live",
crate::AcquisitionObservationOrigin::Backfill => "backfill",
crate::AcquisitionObservationOrigin::Replay => "replay",
crate::AcquisitionObservationOrigin::Repair => "repair",
crate::AcquisitionObservationOrigin::Migration => "migration",
};
}
fn transaction_observation_status_to_sql(
status: crate::TransactionObservationStatus,
status: crate::AcquisitionObservationStatus,
) -> &'static str {
return match status {
crate::TransactionObservationStatus::Detected => "detected",
crate::TransactionObservationStatus::Received => "received",
crate::TransactionObservationStatus::Normalized => "normalized",
crate::TransactionObservationStatus::Persisted => "persisted",
crate::TransactionObservationStatus::Failed => "failed",
crate::TransactionObservationStatus::Missing => "missing",
crate::AcquisitionObservationStatus::Detected => "detected",
crate::AcquisitionObservationStatus::Received => "received",
crate::AcquisitionObservationStatus::Normalized => "normalized",
crate::AcquisitionObservationStatus::Persisted => "persisted",
crate::AcquisitionObservationStatus::Failed => "failed",
crate::AcquisitionObservationStatus::Missing => "missing",
};
}
@@ -260,7 +214,7 @@ fn sql_slot_from_u64(slot: u64) -> ks_core::Result<i64> {
fn optional_sql_slot_from_u64(
slot: std::option::Option<u64>,
) -> ks_core::Result<std::option::Option<i64>> {
return crate::postgres::query::raw_queries::optional_sql_bigint_from_u64(slot);
return optional_sql_bigint_from_u64(slot);
}
fn optional_sql_bigint_from_u64(
@@ -293,12 +247,10 @@ async fn update_raw_transaction_lifecycle(
pool: &sqlx::PgPool,
mark: &crate::RawPayloadLifecycleMark,
) -> ks_core::Result<crate::InsertOutcome> {
let retention_state =
crate::postgres::query::raw_queries::raw_retention_state_to_sql(mark.retention_state);
let processing_state =
crate::postgres::query::raw_queries::raw_processing_state_to_sql(mark.processing_state);
let retention_state = raw_retention_state_to_sql(mark.retention_state);
let processing_state = raw_processing_state_to_sql(mark.processing_state);
let query_result = sqlx::query(
"UPDATE kb_sol_raw_transactions SET retention_state = $1, processing_state = $2, lifecycle_reason = $3, updated_at = NOW() WHERE signature = $4",
"UPDATE k_sol_raw_transactions SET retention_state = $1, processing_state = $2, lifecycle_reason = $3, updated_at = NOW() WHERE signature = $4",
)
.bind(retention_state)
.bind(processing_state)
@@ -306,7 +258,7 @@ async fn update_raw_transaction_lifecycle(
.bind(mark.raw_row_key.as_str())
.execute(pool)
.await;
return crate::postgres::query::raw_queries::outcome_from_update_result(
return outcome_from_update_result(
query_result,
"postgres canonical raw transaction lifecycle update failed",
);
@@ -334,39 +286,36 @@ fn outcome_from_update_result(
mod tests {
#[test]
fn retention_state_serializes_to_lower_snake_case() {
let value = crate::postgres::query::raw_queries::raw_retention_state_to_sql(
crate::RawPayloadRetentionState::Compacted,
);
let value = super::raw_retention_state_to_sql(crate::RawPayloadRetentionState::Compacted);
assert_eq!(value, "compacted");
}
#[test]
fn processing_state_serializes_to_lower_snake_case() {
let value = crate::postgres::query::raw_queries::raw_processing_state_to_sql(
crate::RawPayloadProcessingState::CoreExtracted,
);
let value =
super::raw_processing_state_to_sql(crate::RawPayloadProcessingState::CoreExtracted);
assert_eq!(value, "core_extracted");
}
#[test]
fn observation_origin_serializes_to_lower_snake_case() {
let value = crate::postgres::query::raw_queries::transaction_observation_origin_to_sql(
crate::TransactionObservationOrigin::Backfill,
let value = super::transaction_observation_origin_to_sql(
crate::AcquisitionObservationOrigin::Backfill,
);
assert_eq!(value, "backfill");
}
#[test]
fn observation_status_serializes_to_lower_snake_case() {
let value = crate::postgres::query::raw_queries::transaction_observation_status_to_sql(
crate::TransactionObservationStatus::Normalized,
let value = super::transaction_observation_status_to_sql(
crate::AcquisitionObservationStatus::Normalized,
);
assert_eq!(value, "normalized");
}
#[test]
fn sql_slot_rejects_values_above_bigint() {
let result = crate::postgres::query::raw_queries::sql_slot_from_u64(u64::MAX);
let result = super::sql_slot_from_u64(u64::MAX);
assert!(result.is_err());
}
@@ -387,7 +336,7 @@ mod tests {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => panic!("unexpected connect error: {error}"),
};
let schema_result = store.initialize_raw_store_schema().await;
let schema_result = store.initialize_store_schema().await;
if let std::result::Result::Err(error) = schema_result {
panic!("unexpected schema error: {error}");
}
@@ -395,6 +344,7 @@ mod tests {
let raw_input_result = crate::RawTransactionInsert::new(
signature.clone(),
1,
std::option::Option::None,
serde_json::json!({"source": "test"}),
1,
);
@@ -432,7 +382,7 @@ mod tests {
"test_provider",
"solana_http",
"getTransaction",
crate::TransactionObservationOrigin::Backfill,
crate::AcquisitionObservationOrigin::Backfill,
chrono::Utc::now(),
);
let observation = match observation_result {

View File

@@ -1,5 +1,5 @@
// file: ks-store/src/postgres/query/replay_candidate_queries.rs
// version: 6
// version: 7
//! Read-only PostgreSQL queries for replay candidate discovery.
@@ -119,11 +119,11 @@ pub(crate) async fn list_replay_program_summaries(
) -> ks_core::Result<std::vec::Vec<crate::ReplayProgramSummary>> {
let query_result = sqlx::query_as::<sqlx::Postgres, crate::postgres::query::replay_candidate_queries::ReplayProgramSummaryRow>(
r#"WITH occurrences AS (
SELECT program_id, signature, slot, 'top_level'::TEXT AS scope FROM kb_sol_core_instructions
SELECT program_id, signature, slot, 'top_level'::TEXT AS scope FROM k_sol_core_instructions
UNION ALL
SELECT program_id, signature, slot, 'inner'::TEXT AS scope FROM kb_sol_core_inner_instructions
SELECT program_id, signature, slot, 'inner'::TEXT AS scope FROM k_sol_core_inner_instructions
UNION ALL
SELECT program_id, signature, slot, 'logs'::TEXT AS scope FROM kb_sol_core_logs WHERE program_id IS NOT NULL
SELECT program_id, signature, slot, 'logs'::TEXT AS scope FROM k_sol_core_logs WHERE program_id IS NOT NULL
)
SELECT program_id,
COUNT(DISTINCT signature)::BIGINT AS transaction_count,
@@ -176,15 +176,15 @@ pub(crate) async fn list_replay_entity_summaries(
>(
r#"WITH entities AS (
SELECT 'mint'::TEXT AS entity_kind, mint AS entity_value, signature, slot
FROM kb_sol_core_balance_changes
FROM k_sol_core_balance_changes
WHERE mint IS NOT NULL
UNION ALL
SELECT 'owner'::TEXT AS entity_kind, owner AS entity_value, signature, slot
FROM kb_sol_core_balance_changes
FROM k_sol_core_balance_changes
WHERE owner IS NOT NULL
UNION ALL
SELECT 'account_key'::TEXT AS entity_kind, account_key AS entity_value, signature, slot
FROM kb_sol_core_account_keys
FROM k_sol_core_account_keys
)
SELECT entity_kind,
entity_value,
@@ -250,11 +250,11 @@ fn transaction_candidate_sql(order: &str) -> &'static str {
COALESCE(top_level_stats.program_count, 0)::BIGINT AS top_level_program_count,
COALESCE(inner_stats.program_count, 0)::BIGINT AS inner_program_count,
raw.updated_at::TEXT AS updated_at
FROM kb_sol_raw_transactions raw
LEFT JOIN kb_sol_core_transactions core ON core.signature = raw.signature
FROM k_sol_raw_transactions raw
LEFT JOIN k_sol_core_transactions core ON core.signature = raw.signature
LEFT JOIN LATERAL (
SELECT status, processor_version, attempt_count
FROM kb_sol_ops_processing_ledger
FROM k_sol_ops_processing_ledger
WHERE stage = 'core_extraction'
AND processor_name = 'canonical_to_core'
AND input_key = raw.signature
@@ -263,12 +263,12 @@ fn transaction_candidate_sql(order: &str) -> &'static str {
) ledger ON TRUE
LEFT JOIN LATERAL (
SELECT COUNT(*)::BIGINT AS instruction_count, COUNT(DISTINCT program_id)::BIGINT AS program_count
FROM kb_sol_core_instructions
FROM k_sol_core_instructions
WHERE signature = raw.signature
) top_level_stats ON TRUE
LEFT JOIN LATERAL (
SELECT COUNT(*)::BIGINT AS instruction_count, COUNT(DISTINCT program_id)::BIGINT AS program_count
FROM kb_sol_core_inner_instructions
FROM k_sol_core_inner_instructions
WHERE signature = raw.signature
) inner_stats ON TRUE
WHERE ($1::TEXT IS NULL OR raw.signature ILIKE '%' || $1 || '%')
@@ -278,26 +278,26 @@ fn transaction_candidate_sql(order: &str) -> &'static str {
AND ($5::TEXT IS NULL OR ($5 = 'not_started' AND ledger.status IS NULL) OR ledger.status = $5)
AND ($6::TEXT IS NULL OR
($7 = 'any' AND (
EXISTS (SELECT 1 FROM kb_sol_core_instructions candidate_top_level WHERE candidate_top_level.signature = raw.signature AND candidate_top_level.program_id = $6) OR
EXISTS (SELECT 1 FROM kb_sol_core_inner_instructions candidate_inner WHERE candidate_inner.signature = raw.signature AND candidate_inner.program_id = $6) OR
EXISTS (SELECT 1 FROM kb_sol_core_logs candidate_log WHERE candidate_log.signature = raw.signature AND candidate_log.program_id = $6)
EXISTS (SELECT 1 FROM k_sol_core_instructions candidate_top_level WHERE candidate_top_level.signature = raw.signature AND candidate_top_level.program_id = $6) OR
EXISTS (SELECT 1 FROM k_sol_core_inner_instructions candidate_inner WHERE candidate_inner.signature = raw.signature AND candidate_inner.program_id = $6) OR
EXISTS (SELECT 1 FROM k_sol_core_logs candidate_log WHERE candidate_log.signature = raw.signature AND candidate_log.program_id = $6)
)) OR
($7 = 'top_level' AND EXISTS (SELECT 1 FROM kb_sol_core_instructions candidate_top_level WHERE candidate_top_level.signature = raw.signature AND candidate_top_level.program_id = $6)) OR
($7 = 'inner' AND EXISTS (SELECT 1 FROM kb_sol_core_inner_instructions candidate_inner WHERE candidate_inner.signature = raw.signature AND candidate_inner.program_id = $6)) OR
($7 = 'logs' AND EXISTS (SELECT 1 FROM kb_sol_core_logs candidate_log WHERE candidate_log.signature = raw.signature AND candidate_log.program_id = $6)))
($7 = 'top_level' AND EXISTS (SELECT 1 FROM k_sol_core_instructions candidate_top_level WHERE candidate_top_level.signature = raw.signature AND candidate_top_level.program_id = $6)) OR
($7 = 'inner' AND EXISTS (SELECT 1 FROM k_sol_core_inner_instructions candidate_inner WHERE candidate_inner.signature = raw.signature AND candidate_inner.program_id = $6)) OR
($7 = 'logs' AND EXISTS (SELECT 1 FROM k_sol_core_logs candidate_log WHERE candidate_log.signature = raw.signature AND candidate_log.program_id = $6)))
AND ($8::TEXT IS NULL OR
($8 = 'mint' AND EXISTS (
SELECT 1 FROM kb_sol_core_balance_changes candidate_balance
SELECT 1 FROM k_sol_core_balance_changes candidate_balance
WHERE candidate_balance.signature = raw.signature
AND candidate_balance.mint = $9
)) OR
($8 = 'owner' AND EXISTS (
SELECT 1 FROM kb_sol_core_balance_changes candidate_balance
SELECT 1 FROM k_sol_core_balance_changes candidate_balance
WHERE candidate_balance.signature = raw.signature
AND candidate_balance.owner = $9
)) OR
($8 = 'account_key' AND EXISTS (
SELECT 1 FROM kb_sol_core_account_keys candidate_account
SELECT 1 FROM k_sol_core_account_keys candidate_account
WHERE candidate_account.signature = raw.signature
AND candidate_account.account_key = $9
)))
@@ -318,11 +318,11 @@ fn transaction_candidate_sql(order: &str) -> &'static str {
COALESCE(top_level_stats.program_count, 0)::BIGINT AS top_level_program_count,
COALESCE(inner_stats.program_count, 0)::BIGINT AS inner_program_count,
raw.updated_at::TEXT AS updated_at
FROM kb_sol_raw_transactions raw
LEFT JOIN kb_sol_core_transactions core ON core.signature = raw.signature
FROM k_sol_raw_transactions raw
LEFT JOIN k_sol_core_transactions core ON core.signature = raw.signature
LEFT JOIN LATERAL (
SELECT status, processor_version, attempt_count
FROM kb_sol_ops_processing_ledger
FROM k_sol_ops_processing_ledger
WHERE stage = 'core_extraction'
AND processor_name = 'canonical_to_core'
AND input_key = raw.signature
@@ -331,12 +331,12 @@ fn transaction_candidate_sql(order: &str) -> &'static str {
) ledger ON TRUE
LEFT JOIN LATERAL (
SELECT COUNT(*)::BIGINT AS instruction_count, COUNT(DISTINCT program_id)::BIGINT AS program_count
FROM kb_sol_core_instructions
FROM k_sol_core_instructions
WHERE signature = raw.signature
) top_level_stats ON TRUE
LEFT JOIN LATERAL (
SELECT COUNT(*)::BIGINT AS instruction_count, COUNT(DISTINCT program_id)::BIGINT AS program_count
FROM kb_sol_core_inner_instructions
FROM k_sol_core_inner_instructions
WHERE signature = raw.signature
) inner_stats ON TRUE
WHERE ($1::TEXT IS NULL OR raw.signature ILIKE '%' || $1 || '%')
@@ -346,26 +346,26 @@ fn transaction_candidate_sql(order: &str) -> &'static str {
AND ($5::TEXT IS NULL OR ($5 = 'not_started' AND ledger.status IS NULL) OR ledger.status = $5)
AND ($6::TEXT IS NULL OR
($7 = 'any' AND (
EXISTS (SELECT 1 FROM kb_sol_core_instructions candidate_top_level WHERE candidate_top_level.signature = raw.signature AND candidate_top_level.program_id = $6) OR
EXISTS (SELECT 1 FROM kb_sol_core_inner_instructions candidate_inner WHERE candidate_inner.signature = raw.signature AND candidate_inner.program_id = $6) OR
EXISTS (SELECT 1 FROM kb_sol_core_logs candidate_log WHERE candidate_log.signature = raw.signature AND candidate_log.program_id = $6)
EXISTS (SELECT 1 FROM k_sol_core_instructions candidate_top_level WHERE candidate_top_level.signature = raw.signature AND candidate_top_level.program_id = $6) OR
EXISTS (SELECT 1 FROM k_sol_core_inner_instructions candidate_inner WHERE candidate_inner.signature = raw.signature AND candidate_inner.program_id = $6) OR
EXISTS (SELECT 1 FROM k_sol_core_logs candidate_log WHERE candidate_log.signature = raw.signature AND candidate_log.program_id = $6)
)) OR
($7 = 'top_level' AND EXISTS (SELECT 1 FROM kb_sol_core_instructions candidate_top_level WHERE candidate_top_level.signature = raw.signature AND candidate_top_level.program_id = $6)) OR
($7 = 'inner' AND EXISTS (SELECT 1 FROM kb_sol_core_inner_instructions candidate_inner WHERE candidate_inner.signature = raw.signature AND candidate_inner.program_id = $6)) OR
($7 = 'logs' AND EXISTS (SELECT 1 FROM kb_sol_core_logs candidate_log WHERE candidate_log.signature = raw.signature AND candidate_log.program_id = $6)))
($7 = 'top_level' AND EXISTS (SELECT 1 FROM k_sol_core_instructions candidate_top_level WHERE candidate_top_level.signature = raw.signature AND candidate_top_level.program_id = $6)) OR
($7 = 'inner' AND EXISTS (SELECT 1 FROM k_sol_core_inner_instructions candidate_inner WHERE candidate_inner.signature = raw.signature AND candidate_inner.program_id = $6)) OR
($7 = 'logs' AND EXISTS (SELECT 1 FROM k_sol_core_logs candidate_log WHERE candidate_log.signature = raw.signature AND candidate_log.program_id = $6)))
AND ($8::TEXT IS NULL OR
($8 = 'mint' AND EXISTS (
SELECT 1 FROM kb_sol_core_balance_changes candidate_balance
SELECT 1 FROM k_sol_core_balance_changes candidate_balance
WHERE candidate_balance.signature = raw.signature
AND candidate_balance.mint = $9
)) OR
($8 = 'owner' AND EXISTS (
SELECT 1 FROM kb_sol_core_balance_changes candidate_balance
SELECT 1 FROM k_sol_core_balance_changes candidate_balance
WHERE candidate_balance.signature = raw.signature
AND candidate_balance.owner = $9
)) OR
($8 = 'account_key' AND EXISTS (
SELECT 1 FROM kb_sol_core_account_keys candidate_account
SELECT 1 FROM k_sol_core_account_keys candidate_account
WHERE candidate_account.signature = raw.signature
AND candidate_account.account_key = $9
)))
@@ -411,13 +411,9 @@ mod tests {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => panic!("unexpected connect error: {error}"),
};
let raw_schema_result = store.initialize_raw_store_schema().await;
if let std::result::Result::Err(error) = raw_schema_result {
panic!("unexpected raw schema error: {error}");
}
let core_schema_result = store.initialize_core_store_schema().await;
if let std::result::Result::Err(error) = core_schema_result {
panic!("unexpected core schema error: {error}");
let schema_result = store.initialize_store_schema().await;
if let std::result::Result::Err(error) = schema_result {
panic!("unexpected store schema error: {error}");
}
let transaction_filter_result = crate::ReplayTransactionFilter::new(
std::option::Option::None,

View File

@@ -0,0 +1,162 @@
// file: ks-store/src/postgres/query/schema_queries.rs
// version: 1
//! PostgreSQL orchestration for the complete store baseline schema.
pub(crate) async fn apply_store_schema(pool: &sqlx::PgPool) -> ks_core::Result<()> {
let resource_validation_result = crate::validate_postgres_sql_resources();
if let std::result::Result::Err(error) = resource_validation_result {
return std::result::Result::Err(error);
}
let legacy_result = legacy_store_schema_detected(pool).await;
let legacy_detected = match legacy_result {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
if legacy_detected {
return std::result::Result::Err(crate::storage_contract_error(
"store_legacy_schema_detected",
"historical store schema detected; rebuild the database before initialization",
));
}
let transaction_result = pool.begin().await;
let mut transaction = match transaction_result {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => {
return std::result::Result::Err(ks_core::Error::db(format!(
"postgres store schema transaction failed: {error}"
)));
},
};
let lock_result = sqlx::query("SELECT pg_advisory_xact_lock($1)")
.bind(crate::STORE_SCHEMA_ADVISORY_LOCK_ID)
.execute(&mut *transaction)
.await;
if let std::result::Result::Err(error) = lock_result {
return std::result::Result::Err(ks_core::Error::db(format!(
"postgres store schema advisory lock failed: {error}"
)));
}
let raw_result =
execute_schema_group(&mut transaction, "raw", crate::raw_store_schema_statements()).await;
if let std::result::Result::Err(error) = raw_result {
return std::result::Result::Err(error);
}
let core_result =
execute_schema_group(&mut transaction, "core", crate::core_store_schema_statements()).await;
if let std::result::Result::Err(error) = core_result {
return std::result::Result::Err(error);
}
let decode_result =
execute_schema_group(&mut transaction, "decode", crate::decode_store_schema_statements())
.await;
if let std::result::Result::Err(error) = decode_result {
return std::result::Result::Err(error);
}
let commit_result = transaction.commit().await;
if let std::result::Result::Err(error) = commit_result {
return std::result::Result::Err(ks_core::Error::db(format!(
"postgres store schema commit failed: {error}"
)));
}
return std::result::Result::Ok(());
}
async fn execute_schema_group(
transaction: &mut sqlx::Transaction<'_, sqlx::Postgres>,
model: &str,
statements: std::vec::Vec<&'static str>,
) -> ks_core::Result<()> {
for (statement_index, statement) in statements.into_iter().enumerate() {
tracing::trace!(
target: crate::TRACING_TARGET,
backend = "postgres",
domain = "ks-store.pg",
action = "schema_statement",
model,
statement_index,
sql = statement,
"execute PostgreSQL schema statement"
);
let execution_result = sqlx::query(statement).execute(&mut **transaction).await;
if let std::result::Result::Err(error) = execution_result {
tracing::error!(
target: crate::TRACING_TARGET,
backend = "postgres",
domain = "ks-store.pg",
action = "schema_statement",
model,
statement_index,
"PostgreSQL schema statement failed"
);
return std::result::Result::Err(ks_core::Error::db(format!(
"postgres store schema statement failed for model {model} at index {statement_index}: {error}"
)));
}
}
return std::result::Result::Ok(());
}
async fn legacy_store_schema_detected(pool: &sqlx::PgPool) -> ks_core::Result<bool> {
let pattern = format!("{}%", crate::LEGACY_SOLANA_TABLE_PREFIX);
let query_result = sqlx::query_scalar::<sqlx::Postgres, bool>(
"SELECT EXISTS(SELECT 1 FROM pg_class object JOIN pg_namespace namespace ON namespace.oid = object.relnamespace WHERE namespace.nspname = current_schema() AND object.relkind IN ('r', 'p', 'S', 'i') AND object.relname LIKE $1)",
)
.bind(pattern)
.fetch_one(pool)
.await;
return match query_result {
std::result::Result::Ok(value) => std::result::Result::Ok(value),
std::result::Result::Err(error) => std::result::Result::Err(ks_core::Error::db(format!(
"postgres historical schema detection failed: {error}"
))),
};
}
pub(crate) async fn load_expected_index_counts(pool: &sqlx::PgPool) -> ks_core::Result<(u32, u32)> {
let names = crate::expected_postgres_index_names()
.iter()
.map(|value| return (*value).to_string())
.collect::<std::vec::Vec<_>>();
let expected_count = match u32::try_from(names.len()) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_error) => {
return std::result::Result::Err(ks_core::Error::db(
"postgres expected index count exceeds u32",
));
},
};
let query_result = sqlx::query_scalar::<sqlx::Postgres, i64>(
"SELECT COUNT(*)::BIGINT FROM pg_indexes WHERE schemaname = current_schema() AND indexname = ANY($1::TEXT[])",
)
.bind(&names)
.fetch_one(pool)
.await;
let available_i64 = match query_result {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => {
return std::result::Result::Err(ks_core::Error::db(format!(
"postgres expected index verification failed: {error}"
)));
},
};
let available_count = match u32::try_from(available_i64) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_error) => {
return std::result::Result::Err(ks_core::Error::db(
"postgres available index count exceeds u32",
));
},
};
return std::result::Result::Ok((expected_count, available_count));
}
#[cfg(test)]
mod tests {
#[test]
fn legacy_prefix_is_distinct_from_the_candidate_baseline_prefix() {
assert_eq!(crate::LEGACY_SOLANA_TABLE_PREFIX, "kb_sol_");
assert!(crate::RAW_TRANSACTIONS_TABLE_NAME.starts_with("k_sol_"));
assert!(!crate::RAW_TRANSACTIONS_TABLE_NAME.starts_with(crate::LEGACY_SOLANA_TABLE_PREFIX));
}
}

View File

@@ -1,5 +1,5 @@
// file: ks-store/src/postgres/query/table_diagnostics_queries.rs
// version: 6
// version: 7
//! Read-only PostgreSQL diagnostics for known Solana store tables.
@@ -27,7 +27,7 @@ pub(crate) async fn load_table_statistics(
crate::RAW_TRANSACTIONS_TABLE_NAME => {
load_table_statistics_from_sql(
pool,
crate::table_stats_kb_sol_raw_transactions_sql(),
crate::table_stats_k_sol_raw_transactions_sql(),
table_name,
)
.await
@@ -35,7 +35,15 @@ pub(crate) async fn load_table_statistics(
crate::TRANSACTION_OBSERVATIONS_TABLE_NAME => {
load_table_statistics_from_sql(
pool,
crate::table_stats_kb_sol_obs_transaction_observations_sql(),
crate::table_stats_k_sol_obs_transaction_observations_sql(),
table_name,
)
.await
},
crate::ACCOUNT_OBSERVATIONS_TABLE_NAME => {
load_table_statistics_from_sql(
pool,
crate::table_stats_k_sol_obs_account_observations_sql(),
table_name,
)
.await
@@ -43,7 +51,7 @@ pub(crate) async fn load_table_statistics(
crate::CORE_TRANSACTIONS_TABLE_NAME => {
load_table_statistics_from_sql(
pool,
crate::table_stats_kb_sol_core_transactions_sql(),
crate::table_stats_k_sol_core_transactions_sql(),
table_name,
)
.await
@@ -51,7 +59,7 @@ pub(crate) async fn load_table_statistics(
crate::CORE_ACCOUNT_KEYS_TABLE_NAME => {
load_table_statistics_from_sql(
pool,
crate::table_stats_kb_sol_core_account_keys_sql(),
crate::table_stats_k_sol_core_account_keys_sql(),
table_name,
)
.await
@@ -59,7 +67,7 @@ pub(crate) async fn load_table_statistics(
crate::CORE_INSTRUCTIONS_TABLE_NAME => {
load_table_statistics_from_sql(
pool,
crate::table_stats_kb_sol_core_instructions_sql(),
crate::table_stats_k_sol_core_instructions_sql(),
table_name,
)
.await
@@ -67,7 +75,7 @@ pub(crate) async fn load_table_statistics(
crate::CORE_INNER_INSTRUCTIONS_TABLE_NAME => {
load_table_statistics_from_sql(
pool,
crate::table_stats_kb_sol_core_inner_instructions_sql(),
crate::table_stats_k_sol_core_inner_instructions_sql(),
table_name,
)
.await
@@ -75,7 +83,7 @@ pub(crate) async fn load_table_statistics(
crate::CORE_LOGS_TABLE_NAME => {
load_table_statistics_from_sql(
pool,
crate::table_stats_kb_sol_core_logs_sql(),
crate::table_stats_k_sol_core_logs_sql(),
table_name,
)
.await
@@ -83,7 +91,23 @@ pub(crate) async fn load_table_statistics(
crate::CORE_BALANCE_CHANGES_TABLE_NAME => {
load_table_statistics_from_sql(
pool,
crate::table_stats_kb_sol_core_balance_changes_sql(),
crate::table_stats_k_sol_core_balance_changes_sql(),
table_name,
)
.await
},
crate::CORE_RETURN_DATA_TABLE_NAME => {
load_table_statistics_from_sql(
pool,
crate::table_stats_k_sol_core_return_data_sql(),
table_name,
)
.await
},
crate::CORE_ACCOUNT_STATES_TABLE_NAME => {
load_table_statistics_from_sql(
pool,
crate::table_stats_k_sol_core_account_states_sql(),
table_name,
)
.await
@@ -91,7 +115,7 @@ pub(crate) async fn load_table_statistics(
crate::PROCESSING_LEDGER_TABLE_NAME => {
load_table_statistics_from_sql(
pool,
crate::table_stats_kb_sol_ops_processing_ledger_sql(),
crate::table_stats_k_sol_ops_processing_ledger_sql(),
table_name,
)
.await
@@ -99,7 +123,7 @@ pub(crate) async fn load_table_statistics(
crate::DECODE_EVENTS_TABLE_NAME => {
load_table_statistics_from_sql(
pool,
crate::table_stats_kb_sol_decode_events_sql(),
crate::table_stats_k_sol_decode_events_sql(),
table_name,
)
.await
@@ -107,7 +131,7 @@ pub(crate) async fn load_table_statistics(
crate::DECODE_COVERAGE_DECLARATIONS_TABLE_NAME => {
load_table_statistics_from_sql(
pool,
crate::table_stats_kb_sol_decode_coverage_declarations_sql(),
crate::table_stats_k_sol_decode_coverage_declarations_sql(),
table_name,
)
.await
@@ -115,15 +139,15 @@ pub(crate) async fn load_table_statistics(
crate::DECODE_COVERAGE_OBSERVATIONS_TABLE_NAME => {
load_table_statistics_from_sql(
pool,
crate::table_stats_kb_sol_decode_coverage_observations_sql(),
crate::table_stats_k_sol_decode_coverage_observations_sql(),
table_name,
)
.await
},
crate::MATERIALIZED_EVENTS_TABLE_NAME => {
crate::MATERIALIZED_OUTPUTS_TABLE_NAME => {
load_table_statistics_from_sql(
pool,
crate::table_stats_kb_sol_mat_events_sql(),
crate::table_stats_k_sol_mat_outputs_sql(),
table_name,
)
.await

View File

@@ -1,5 +1,5 @@
// file: ks-store/src/postgres/repository/decode_pipeline_repository.rs
// version: 3
// version: 4
//! PostgreSQL contextual decode and materialization repository.
@@ -96,10 +96,10 @@ impl crate::DecodePipelineStore for crate::PostgresStore {
clippy::implicit_return,
reason = "async_trait expansion triggers implicit_return on generated async trait methods."
)]
async fn list_materialized_events(
async fn list_materialized_outputs(
&self,
filter: &crate::MaterializedEventFilter,
) -> ks_core::Result<std::vec::Vec<crate::MaterializedEventQueryRow>> {
return crate::list_materialized_events(self.pool(), filter).await;
filter: &crate::MaterializedOutputFilter,
) -> ks_core::Result<std::vec::Vec<crate::MaterializedOutputQueryRow>> {
return crate::list_materialized_outputs(self.pool(), filter).await;
}
}

View File

@@ -1,5 +1,5 @@
// file: ks-store/src/postgres/store.rs
// version: 9
// version: 13
//! PostgreSQL store implementation kept behind the backend-agnostic `Store` facade.
@@ -171,10 +171,24 @@ impl crate::PostgresStore {
/// Connects to PostgreSQL from validated backend options.
pub(crate) async fn connect(options: crate::PostgresStoreOptions) -> ks_core::Result<Self> {
tracing::debug!(target: crate::TRACING_TARGET, backend = "postgres", domain = "ks-store.pg", action = "connection_open", "open PostgreSQL store connection");
let connect_options_result =
options.database_url.parse::<sqlx::postgres::PgConnectOptions>();
let connect_options = match connect_options_result {
std::result::Result::Ok(value) => {
sqlx::ConnectOptions::disable_statement_logging(value)
},
std::result::Result::Err(_error) => {
tracing::error!(target: crate::TRACING_TARGET, backend = "postgres", domain = "ks-store.pg", action = "connection_open", connected = false, "PostgreSQL store connection options are invalid");
return std::result::Result::Err(crate::storage_contract_error(
"store_backend_connection_failed",
"postgres connection failed",
));
},
};
let pool_options = sqlx::postgres::PgPoolOptions::new()
.max_connections(options.max_connections)
.acquire_timeout(std::time::Duration::from_millis(options.connect_timeout_ms));
let connect_result = pool_options.connect(options.database_url.as_str()).await;
let connect_result = pool_options.connect_with(connect_options).await;
return match connect_result {
std::result::Result::Ok(pool) => {
tracing::debug!(target: crate::TRACING_TARGET, backend = "postgres", domain = "ks-store.pg", action = "connection_open", connected = true, "PostgreSQL store connection opened");
@@ -195,40 +209,21 @@ impl crate::PostgresStore {
return &self.pool;
}
/// Reads expected and available PostgreSQL index counts without exposing physical names.
pub(crate) async fn index_object_counts(&self) -> ks_core::Result<(u32, u32)> {
return crate::load_expected_index_counts(&self.pool).await;
}
/// Applies each idempotent store schema once per invocation in dependency order.
pub(crate) async fn initialize_store_schema(&self) -> ks_core::Result<()> {
tracing::debug!(target: crate::TRACING_TARGET, backend = "postgres", domain = "ks-store.pg", action = "schema_initialize", "initialize PostgreSQL store schema");
let raw_result = crate::apply_raw_store_schema(&self.pool).await;
if let std::result::Result::Err(error) = raw_result {
return std::result::Result::Err(error);
}
let core_result = crate::apply_core_store_schema(&self.pool).await;
if let std::result::Result::Err(error) = core_result {
return std::result::Result::Err(error);
}
let result = crate::apply_decode_store_schema(&self.pool).await;
let result = crate::apply_store_schema(&self.pool).await;
if result.is_ok() {
tracing::debug!(target: crate::TRACING_TARGET, backend = "postgres", domain = "ks-store.pg", action = "schema_initialize", initialized = true, "PostgreSQL store schema initialized");
}
return result;
}
/// Applies the idempotent minimal raw Solana store schema.
#[cfg(test)]
pub(crate) async fn initialize_raw_store_schema(&self) -> ks_core::Result<()> {
return crate::apply_raw_store_schema(&self.pool).await;
}
/// Applies the idempotent minimal Core Solana store schema.
#[cfg(test)]
pub(crate) async fn initialize_core_store_schema(&self) -> ks_core::Result<()> {
let raw_result = self.initialize_raw_store_schema().await;
if let std::result::Result::Err(error) = raw_result {
return std::result::Result::Err(error);
}
return crate::apply_core_store_schema(&self.pool).await;
}
/// Reads a UI-safe backend descriptor.
pub(crate) async fn backend_descriptor(
&self,
@@ -265,27 +260,45 @@ impl crate::PostgresStore {
};
}
/// Reads a non-destructive migration snapshot.
/// Reads the current store schema contract status without relying on backend migration history.
pub(crate) async fn migration_snapshot(
&self,
) -> ks_core::Result<crate::StoreMigrationSnapshot> {
let migration_table_result = crate::load_migration_table_name(&self.pool).await;
return match migration_table_result {
std::result::Result::Ok(std::option::Option::None) => {
std::result::Result::Ok(crate::StoreMigrationSnapshot::new(
crate::StoreMigrationStatus::NotInitialized,
std::option::Option::None,
std::vec::Vec::new(),
std::option::Option::Some(std::string::String::from(
"no migration history table detected; crate-managed schema initialization is active",
)),
))
},
std::result::Result::Ok(std::option::Option::Some(_table_name)) => {
self.migration_snapshot_from_existing_table().await
},
std::result::Result::Err(error) => std::result::Result::Err(error),
let resources_result = self.known_resource_diagnostics().await;
let resources = match resources_result {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let expected_count = resources.len();
let available_count = resources.iter().filter(|resource| return resource.available).count();
if available_count == 0 {
return std::result::Result::Ok(crate::StoreMigrationSnapshot::new(
crate::StoreMigrationStatus::NotInitialized,
std::option::Option::None,
std::vec::Vec::new(),
std::option::Option::Some(std::string::String::from(
"store baseline schema is not initialized",
)),
));
}
if available_count == expected_count {
return std::result::Result::Ok(crate::StoreMigrationSnapshot::new(
crate::StoreMigrationStatus::Current,
std::option::Option::Some(crate::STORE_SCHEMA_CONTRACT_VERSION.to_string()),
std::vec::Vec::new(),
std::option::Option::Some(std::string::String::from(
"store baseline schema contract is current",
)),
));
}
return std::result::Result::Ok(crate::StoreMigrationSnapshot::new(
crate::StoreMigrationStatus::Drift,
std::option::Option::None,
std::vec::Vec::new(),
std::option::Option::Some(std::string::String::from(
"store baseline schema is partially initialized",
)),
));
}
/// Reads a complete backend-neutral diagnostic snapshot.
@@ -420,25 +433,6 @@ impl crate::PostgresStore {
}
return std::result::Result::Ok(diagnostics);
}
async fn migration_snapshot_from_existing_table(
&self,
) -> ks_core::Result<crate::StoreMigrationSnapshot> {
let version_result = crate::load_latest_migration_version(&self.pool).await;
return match version_result {
std::result::Result::Ok(current_version) => {
std::result::Result::Ok(crate::StoreMigrationSnapshot::new(
crate::StoreMigrationStatus::Current,
current_version,
std::vec::Vec::new(),
std::option::Option::Some(std::string::String::from(
"migration history table detected",
)),
))
},
std::result::Result::Err(error) => std::result::Result::Err(error),
};
}
}
/// Returns a PostgreSQL connection descriptor masked for logs and diagnostics.
@@ -489,6 +483,14 @@ fn query_suffix(had_query: bool) -> std::string::String {
#[cfg(test)]
mod tests {
#[test]
fn postgres_connection_disables_statement_logging() {
let source = include_str!("store.rs");
assert!(source.contains("parse::<sqlx::postgres::PgConnectOptions>()"));
assert!(source.contains("sqlx::ConnectOptions::disable_statement_logging(value)"));
assert!(source.contains("connect_with(connect_options)"));
}
#[test]
fn backend_options_reject_empty_database_url() {
let result = crate::PostgresStoreOptions::new(" ", 1, 1000, false);

View File

@@ -1,5 +1,5 @@
// file: ks-store/src/store.rs
// version: 9
// version: 10
//! Backend-agnostic store facade and connection ownership.
@@ -266,6 +266,15 @@ async fn open_postgres(backend_options: serde_json::Value) -> ks_core::Result<cr
));
},
};
let before_indexes = match store.index_object_counts().await {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_error) => {
return std::result::Result::Err(crate::storage_contract_error(
"store_initialization_verification_failed",
"store index verification failed before initialization",
));
},
};
if auto_initialize_schema {
let initialize_result = store.initialize_store_schema().await;
if let std::result::Result::Err(_error) = initialize_result {
@@ -284,10 +293,24 @@ async fn open_postgres(backend_options: serde_json::Value) -> ks_core::Result<cr
));
},
};
let after_indexes = match store.index_object_counts().await {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_error) => {
return std::result::Result::Err(crate::storage_contract_error(
"store_initialization_verification_failed",
"store index verification failed after initialization",
));
},
};
return std::result::Result::Ok(crate::Store {
backend: StoreBackend::Postgres(store),
configuration,
initialization: build_initialization_summary(&before, &after),
initialization: build_initialization_summary(
&before,
&after,
before_indexes,
after_indexes,
),
});
}
@@ -334,6 +357,8 @@ fn disabled_initialization_summary() -> crate::StoreInitializationSummary {
fn build_initialization_summary(
before: &[crate::StoreResourceDiagnostics],
after: &[crate::StoreResourceDiagnostics],
before_indexes: (u32, u32),
after_indexes: (u32, u32),
) -> crate::StoreInitializationSummary {
let model_codes = ["raw", "core", "processing", "decode", "materialization"];
let mut models = std::vec::Vec::new();
@@ -378,7 +403,9 @@ fn build_initialization_summary(
available_resource_count += available;
created_resource_count += created;
}
let status = if expected_resource_count == available_resource_count {
let resources_ready = expected_resource_count == available_resource_count;
let indexes_ready = after_indexes.0 == after_indexes.1;
let status = if resources_ready && indexes_ready {
crate::StoreInitializationStatus::Ready
} else {
crate::StoreInitializationStatus::Partial
@@ -390,12 +417,20 @@ fn build_initialization_summary(
expected_resource_count,
available_resource_count,
created_resource_count,
objects: std::vec![crate::StoreObjectVerificationSummary {
object_kind: "table".to_string(),
expected_count: expected_resource_count,
available_count: available_resource_count,
created_count: created_resource_count,
}],
objects: std::vec![
crate::StoreObjectVerificationSummary {
object_kind: "table".to_string(),
expected_count: expected_resource_count,
available_count: available_resource_count,
created_count: created_resource_count,
},
crate::StoreObjectVerificationSummary {
object_kind: "index".to_string(),
expected_count: after_indexes.0,
available_count: after_indexes.1,
created_count: after_indexes.1.saturating_sub(before_indexes.1),
},
],
};
}
@@ -911,14 +946,14 @@ impl crate::DecodePipelineStore for crate::Store {
clippy::implicit_return,
reason = "async_trait expansion triggers implicit_return on generated async trait methods."
)]
async fn list_materialized_events(
async fn list_materialized_outputs(
&self,
filter: &crate::MaterializedEventFilter,
) -> ks_core::Result<std::vec::Vec<crate::MaterializedEventQueryRow>> {
filter: &crate::MaterializedOutputFilter,
) -> ks_core::Result<std::vec::Vec<crate::MaterializedOutputQueryRow>> {
return match &self.backend {
StoreBackend::Disabled => std::result::Result::Err(disabled_store_error()),
StoreBackend::Postgres(store) => {
crate::DecodePipelineStore::list_materialized_events(store, filter).await
crate::DecodePipelineStore::list_materialized_outputs(store, filter).await
},
};
}
@@ -929,7 +964,7 @@ mod tests {
#[test]
fn backend_diagnostic_errors_are_sanitized_at_the_facade_boundary() {
let result: ks_core::Result<()> = std::result::Result::Err(ks_core::Error::db(
"backend detail references kb_sol_private_table",
"backend detail references k_sol_private_table",
));
let error = match super::sanitize_backend_result(
result,
@@ -939,7 +974,7 @@ mod tests {
std::result::Result::Ok(()) => panic!("backend error must remain an error"),
std::result::Result::Err(error) => error,
};
assert!(!error.to_string().contains("kb_sol_private_table"));
assert!(!error.to_string().contains("k_sol_private_table"));
assert!(error.to_string().contains("store resource diagnostics failed"));
}
@@ -959,17 +994,39 @@ mod tests {
available: true,
statistics: std::option::Option::None,
}];
let summary = super::build_initialization_summary(&before, &after);
assert_eq!(summary.objects.len(), 1);
let summary = super::build_initialization_summary(&before, &after, (3, 1), (3, 3));
assert_eq!(summary.objects.len(), 2);
assert_eq!(summary.objects[0].object_kind, "table");
assert_eq!(summary.objects[0].expected_count, 1);
assert_eq!(summary.objects[0].available_count, 1);
assert_eq!(summary.objects[0].created_count, 1);
assert_eq!(summary.objects[1].object_kind, "index");
assert_eq!(summary.objects[1].expected_count, 3);
assert_eq!(summary.objects[1].available_count, 3);
assert_eq!(summary.objects[1].created_count, 2);
let serialized = match serde_json::to_string(&summary) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => panic!("summary must serialize: {error}"),
};
assert!(!serialized.contains("kb_sol_"));
assert!(!serialized.contains("k_sol_"));
}
#[test]
fn initialization_summary_is_partial_when_expected_indexes_are_missing() {
let resource = crate::StoreResourceDiagnostics {
resource_code: "raw_transactions".to_string(),
model_code: "raw".to_string(),
role: "canonical raw transaction".to_string(),
available: true,
statistics: std::option::Option::None,
};
let summary = super::build_initialization_summary(
std::slice::from_ref(&resource),
std::slice::from_ref(&resource),
(3, 2),
(3, 2),
);
assert_eq!(summary.status, crate::StoreInitializationStatus::Partial);
}
#[test]