v0.5.3-pre.004

This commit is contained in:
2026-08-12 14:31:48 +02:00
parent 400ced4832
commit b6583bd9fb
47 changed files with 2978 additions and 291 deletions

View File

@@ -1,5 +1,5 @@
// file: ks-store/src/contracts.rs
// version: 7
// version: 8
//! Backend-neutral storage contracts used by pipeline crates.
@@ -11,8 +11,16 @@ mod pagination;
mod replay;
mod repository;
/// Stable processing stage used for generic account-state normalization.
pub use self::dto::ACCOUNT_STATE_NORMALIZATION_STAGE;
/// Generic account acquisition observation insert contract.
pub use self::dto::AccountObservationInsert;
/// Failed generic account-state normalization attempt.
pub use self::dto::AccountStateNormalizationFailure;
/// Atomic generic account-state persistence bundle.
pub use self::dto::AccountStatePersistenceBundle;
/// Bounded generic account-state selection filter.
pub use self::dto::AccountStateSelectionFilter;
/// Transaction acquisition observation origin.
pub use self::dto::AcquisitionObservationOrigin;
/// Generic account acquisition observation status.
@@ -43,6 +51,8 @@ pub use self::dto::CoreInstructionLifecycleMark;
pub use self::dto::CoreInstructionProcessingState;
/// Core instruction replay filter contract.
pub use self::dto::CoreInstructionReplayFilter;
/// Logical Core instruction scope.
pub use self::dto::CoreInstructionScope;
/// Core log insert contract.
pub use self::dto::CoreLogInsert;
/// Core return-data insert contract.
@@ -61,11 +71,18 @@ pub use self::dto::DecodeFailure;
pub use self::dto::DecodeObservationInsert;
/// Atomic persistence bundle for one decoder and one contextual input.
pub use self::dto::DecodePersistenceBundle;
/// Source-neutral external decode schema provenance.
pub use self::dto::DecodeSchemaProvenance;
/// Bounded contextual instruction selection filter for decode campaigns.
pub use self::dto::DecodeSelectionFilter;
/// Insert or upsert result contract returned by repositories.
pub use self::dto::InsertOutcome;
/// Maximum number of account observations selected by one normalization query.
pub use self::dto::MAX_ACCOUNT_STATE_SELECTION_ROWS;
/// Maximum canonical transactions selected by one Core extraction batch.
pub use self::dto::MAX_CORE_EXTRACTION_SELECTION_ROWS;
/// Maximum contextual inputs selected by one decode processing batch.
pub use self::dto::MAX_DECODE_SELECTION_ROWS;
/// Maximum number of materialized rows returned by one bounded query.
pub use self::dto::MAX_MATERIALIZED_OUTPUT_QUERY_ROWS;
/// Atomic persistence bundle for one materializer and one decoded observation.
@@ -116,7 +133,6 @@ pub use self::dto::StoreResourceStatistics;
pub use self::dto::StoreRuntimeSummary;
/// Transaction acquisition observation insert contract.
pub use self::dto::TransactionObservationInsert;
/// Transaction acquisition observation status.
/// Generic account acquisition observation SQL-like row contract.
pub use self::entity::AccountObservationRow;
/// Core account key SQL-like row contract.
@@ -127,6 +143,8 @@ pub use self::entity::CoreAccountStateRow;
pub use self::entity::CoreBalanceChangeRow;
/// Core inner instruction SQL-like row contract.
pub use self::entity::CoreInnerInstructionRow;
/// Logical top-level or CPI instruction SQL-like replay row.
pub use self::entity::CoreInstructionReplayRow;
/// Core instruction SQL-like row contract.
pub use self::entity::CoreInstructionRow;
/// Core log SQL-like row contract.
@@ -147,10 +165,14 @@ pub use self::health::StoreHealthSnapshot;
pub use self::health::StoreHealthStatus;
/// Default repository page size.
pub use self::pagination::DEFAULT_PAGE_SIZE;
/// Maximum encoded cursor size.
pub use self::pagination::MAX_PAGE_CURSOR_LENGTH;
/// Maximum repository page size.
pub use self::pagination::MAX_PAGE_SIZE;
/// Page request contract for repository list operations.
pub use self::pagination::PageRequest;
/// One bounded page and its continuation cursor.
pub use self::pagination::PageSlice;
/// Sort direction for repository list operations.
pub use self::pagination::SortDirection;
/// Maximum number of rows returned by one replay candidate query.
@@ -171,6 +193,8 @@ pub use self::replay::ReplayProgramSummary;
pub use self::replay::ReplayTransactionCandidate;
/// Bounded read-only filter for transaction replay candidates.
pub use self::replay::ReplayTransactionFilter;
/// Generic account observation to Core account-state storage behavior.
pub use self::repository::AccountStateStore;
/// Canonical transaction to core extraction storage behavior.
pub use self::repository::CoreExtractionStore;
/// Core Solana storage behavior.

View File

@@ -1,8 +1,9 @@
// file: ks-store/src/contracts/dto.rs
// version: 7
// version: 8
//! Backend-neutral DTO exports for storage repository contracts.
mod account_state;
mod core;
mod core_extraction;
mod decode;
@@ -10,6 +11,16 @@ mod outcome;
mod raw;
mod store;
/// Stable processing stage used for generic account-state normalization.
pub use self::account_state::ACCOUNT_STATE_NORMALIZATION_STAGE;
/// Failed generic account-state normalization attempt.
pub use self::account_state::AccountStateNormalizationFailure;
/// Atomic generic account-state persistence bundle.
pub use self::account_state::AccountStatePersistenceBundle;
/// Bounded account-state normalization selection filter.
pub use self::account_state::AccountStateSelectionFilter;
/// Maximum number of account observations selected by one normalization query.
pub use self::account_state::MAX_ACCOUNT_STATE_SELECTION_ROWS;
/// Core account key insert contract.
pub use self::core::CoreAccountKeyInsert;
/// Core account key source category.
@@ -30,6 +41,8 @@ pub use self::core::CoreInstructionLifecycleMark;
pub use self::core::CoreInstructionProcessingState;
/// Core instruction replay filter contract.
pub use self::core::CoreInstructionReplayFilter;
/// Logical scope of one replayable Core instruction.
pub use self::core::CoreInstructionScope;
/// Core log insert contract.
pub use self::core::CoreLogInsert;
/// Core return-data insert contract.
@@ -42,6 +55,8 @@ pub use self::core_extraction::CoreExtractionBundle;
pub use self::core_extraction::CoreExtractionFailure;
/// Bounded canonical transaction selection filter for core extraction.
pub use self::core_extraction::CoreExtractionSelectionFilter;
/// Maximum canonical transactions selected by one Core extraction batch.
pub use self::core_extraction::MAX_CORE_EXTRACTION_SELECTION_ROWS;
/// Stable processing ledger identity.
pub use self::core_extraction::ProcessingLedgerIdentity;
/// Stable processing ledger status.
@@ -61,6 +76,8 @@ pub use self::decode::DecodePersistenceBundle;
pub use self::decode::DecodeSchemaProvenance;
/// Bounded contextual instruction selection filter for decode campaigns.
pub use self::decode::DecodeSelectionFilter;
/// Maximum contextual inputs selected by one decode processing batch.
pub use self::decode::MAX_DECODE_SELECTION_ROWS;
/// Maximum number of materialized rows returned by one bounded query.
pub use self::decode::MAX_MATERIALIZED_OUTPUT_QUERY_ROWS;
/// Atomic persistence bundle for one materializer and one decoded observation.

View File

@@ -0,0 +1,200 @@
// file: ks-store/src/contracts/dto/account_state.rs
// version: 1
//! Generic account-observation to Core account-state replay contracts.
/// Stable processing stage used for generic account-state normalization.
pub const ACCOUNT_STATE_NORMALIZATION_STAGE: &str = "account_state_normalization";
/// Maximum number of account observations selected by one normalization query.
pub const MAX_ACCOUNT_STATE_SELECTION_ROWS: u32 = 1000;
/// Bounded selection filter for replayable account observations.
#[derive(Clone, Debug, Eq, PartialEq, serde::Deserialize, serde::Serialize)]
pub struct AccountStateSelectionFilter {
/// Processor name whose current succeeded inputs may be excluded.
pub processor_name: std::string::String,
/// Processor version whose current succeeded inputs may be excluded.
pub processor_version: std::string::String,
/// Optional exact account keys.
pub account_keys: std::vec::Vec<std::string::String>,
/// Optional inclusive minimum context slot.
pub min_slot: std::option::Option<u64>,
/// Optional inclusive maximum context slot.
pub max_slot: std::option::Option<u64>,
/// Whether observations already succeeded for the selected processor identity may be selected again.
pub include_current: bool,
/// Maximum selected observations.
pub limit: u32,
}
impl crate::AccountStateSelectionFilter {
/// Builds a bounded account-state normalization selection.
pub fn new(
processor_name: impl std::convert::Into<std::string::String>,
processor_version: impl std::convert::Into<std::string::String>,
account_keys: std::vec::Vec<std::string::String>,
min_slot: std::option::Option<u64>,
max_slot: std::option::Option<u64>,
include_current: bool,
limit: u32,
) -> ks_core::Result<Self> {
let processor_name_value = processor_name.into();
let processor_version_value = processor_version.into();
if processor_name_value.trim().is_empty() || processor_version_value.trim().is_empty() {
return std::result::Result::Err(ks_core::Error::db(
"account-state selection processor identity must not be empty",
));
}
if limit == 0 || limit > crate::MAX_ACCOUNT_STATE_SELECTION_ROWS {
return std::result::Result::Err(ks_core::Error::db(format!(
"account-state selection limit must be between 1 and {}",
crate::MAX_ACCOUNT_STATE_SELECTION_ROWS
)));
}
if let (std::option::Option::Some(minimum), std::option::Option::Some(maximum)) =
(min_slot, max_slot)
{
if minimum > maximum {
return std::result::Result::Err(ks_core::Error::db(
"account-state selection minimum slot must not exceed maximum slot",
));
}
}
for account_key in &account_keys {
if account_key.trim().is_empty() {
return std::result::Result::Err(ks_core::Error::db(
"account-state selection account keys must not be empty",
));
}
}
return std::result::Result::Ok(Self {
processor_name: processor_name_value,
processor_version: processor_version_value,
account_keys,
min_slot,
max_slot,
include_current,
limit,
});
}
/// Builds a default pending account-state selection.
pub fn pending(
processor_name: impl std::convert::Into<std::string::String>,
processor_version: impl std::convert::Into<std::string::String>,
limit: u32,
) -> ks_core::Result<Self> {
return Self::new(
processor_name,
processor_version,
std::vec::Vec::new(),
std::option::Option::None,
std::option::Option::None,
false,
limit,
);
}
}
/// Atomic persistence bundle for one generic account observation normalized to Core state.
#[derive(Clone, Debug, PartialEq, serde::Deserialize, serde::Serialize)]
pub struct AccountStatePersistenceBundle {
/// Processing ledger identity.
pub ledger_identity: crate::ProcessingLedgerIdentity,
/// Canonical Core account state produced from the observation.
pub state: crate::CoreAccountStateInsert,
}
impl crate::AccountStatePersistenceBundle {
/// Validates the account-state normalization identity and lineage.
pub fn validate(&self) -> ks_core::Result<()> {
if self.ledger_identity.stage != crate::ACCOUNT_STATE_NORMALIZATION_STAGE {
return std::result::Result::Err(ks_core::Error::db(
"account-state persistence stage is invalid",
));
}
let state_result = self.state.validate();
if let std::result::Result::Err(error) = state_result {
return std::result::Result::Err(error);
}
return std::result::Result::Ok(());
}
}
/// Failed generic account-state normalization attempt.
#[derive(Clone, Debug, Eq, PartialEq, serde::Deserialize, serde::Serialize)]
pub struct AccountStateNormalizationFailure {
/// Source raw account observation technical id.
pub source_observation_id: i64,
/// Processing ledger identity.
pub ledger_identity: crate::ProcessingLedgerIdentity,
/// Stable machine-readable error code.
pub error_code: std::string::String,
/// Bounded diagnostic message.
pub error_message: std::string::String,
}
impl crate::AccountStateNormalizationFailure {
/// Validates a failure record before persistence.
pub fn validate(&self) -> ks_core::Result<()> {
if self.source_observation_id <= 0
|| self.ledger_identity.stage != crate::ACCOUNT_STATE_NORMALIZATION_STAGE
|| self.error_code.trim().is_empty()
|| self.error_message.trim().is_empty()
{
return std::result::Result::Err(ks_core::Error::db(
"account-state normalization failure is invalid",
));
}
return std::result::Result::Ok(());
}
}
#[cfg(test)]
mod tests {
#[test]
fn account_state_selection_is_bounded() {
assert!(crate::AccountStateSelectionFilter::pending("normalizer", "1", 0).is_err());
assert!(
crate::AccountStateSelectionFilter::pending(
"normalizer",
"1",
crate::MAX_ACCOUNT_STATE_SELECTION_ROWS + 1,
)
.is_err()
);
}
#[test]
fn account_state_selection_requires_processor_identity() {
assert!(crate::AccountStateSelectionFilter::pending("", "1", 10).is_err());
assert!(crate::AccountStateSelectionFilter::pending("normalizer", "", 10).is_err());
}
#[test]
fn account_state_bundle_requires_the_canonical_stage() {
let identity = crate::ProcessingLedgerIdentity {
stage: "other".to_string(),
processor_name: "normalizer".to_string(),
processor_version: "1".to_string(),
input_key: "observation".to_string(),
input_hash: "hash".to_string(),
};
let bundle = crate::AccountStatePersistenceBundle {
ledger_identity: identity,
state: crate::CoreAccountStateInsert {
source_observation_id: 1,
account_key: "account".to_string(),
slot: 1,
owner: "owner".to_string(),
lamports: 0,
executable: false,
rent_epoch: 0,
space: 0,
data_base64: std::string::String::new(),
data_hash: "hash".to_string(),
},
};
assert!(bundle.validate().is_err());
}
}

View File

@@ -1,8 +1,51 @@
// file: ks-store/src/contracts/dto/core.rs
// version: 3
// version: 4
//! Core Solana storage DTOs.
/// Logical scope of one Core instruction persisted in the top-level or CPI table.
#[derive(Clone, Copy, Debug, Eq, PartialEq, serde::Deserialize, serde::Serialize)]
pub enum CoreInstructionScope {
/// Transaction message top-level instruction.
TopLevel,
/// Inner/CPI instruction emitted during program execution.
Inner,
}
impl crate::CoreInstructionScope {
/// Returns the stable storage code used by backend-neutral query contracts.
pub fn as_code(self) -> &'static str {
return match self {
Self::TopLevel => "top_level",
Self::Inner => "inner",
};
}
/// Resolves a stable scope code returned by a storage adapter.
pub fn from_code(value: &str) -> ks_core::Result<Self> {
return match value {
"top_level" => std::result::Result::Ok(Self::TopLevel),
"inner" => std::result::Result::Ok(Self::Inner),
_unknown => {
std::result::Result::Err(ks_core::Error::db("unknown core instruction scope"))
},
};
}
/// Infers the scope from the canonical instruction path convention.
pub fn from_instruction_path(instruction_path: &str) -> ks_core::Result<Self> {
if instruction_path.trim().is_empty() {
return std::result::Result::Err(ks_core::Error::db(
"core instruction path must not be empty",
));
}
if instruction_path.contains('/') {
return std::result::Result::Ok(Self::Inner);
}
return std::result::Result::Ok(Self::TopLevel);
}
}
/// Processing state for one normalized core instruction.
#[derive(Clone, Copy, Debug, Eq, PartialEq, serde::Deserialize, serde::Serialize)]
pub enum CoreInstructionProcessingState {
@@ -590,6 +633,8 @@ pub struct CoreInstructionReplayFilter {
pub processing_state: std::option::Option<crate::CoreInstructionProcessingState>,
/// Optional program id filter.
pub program_id: std::option::Option<std::string::String>,
/// Optional logical instruction scope.
pub scope: std::option::Option<crate::CoreInstructionScope>,
/// Optional inclusive minimum slot.
pub min_slot: std::option::Option<u64>,
/// Optional inclusive maximum slot.
@@ -601,6 +646,7 @@ impl crate::CoreInstructionReplayFilter {
pub fn new(
processing_state: std::option::Option<crate::CoreInstructionProcessingState>,
program_id: std::option::Option<std::string::String>,
scope: std::option::Option<crate::CoreInstructionScope>,
min_slot: std::option::Option<u64>,
max_slot: std::option::Option<u64>,
) -> ks_core::Result<Self> {
@@ -625,6 +671,7 @@ impl crate::CoreInstructionReplayFilter {
return std::result::Result::Ok(Self {
processing_state,
program_id,
scope,
min_slot,
max_slot,
});
@@ -637,6 +684,7 @@ impl crate::CoreInstructionReplayFilter {
crate::CoreInstructionProcessingState::Pending,
),
program_id: std::option::Option::None,
scope: std::option::Option::None,
min_slot: std::option::Option::None,
max_slot: std::option::Option::None,
};
@@ -1035,13 +1083,27 @@ mod tests {
std::option::Option::Some(" ".to_string()),
std::option::Option::None,
std::option::Option::None,
std::option::Option::None,
);
assert!(result.is_err());
}
#[test]
fn instruction_scope_is_inferred_from_canonical_path() {
assert_eq!(
crate::CoreInstructionScope::from_instruction_path("2"),
std::result::Result::Ok(crate::CoreInstructionScope::TopLevel)
);
assert_eq!(
crate::CoreInstructionScope::from_instruction_path("2/1"),
std::result::Result::Ok(crate::CoreInstructionScope::Inner)
);
}
#[test]
fn replay_filter_rejects_inverted_slots() {
let result = crate::CoreInstructionReplayFilter::new(
std::option::Option::None,
std::option::Option::None,
std::option::Option::None,
std::option::Option::Some(10),

View File

@@ -1,8 +1,11 @@
// file: ks-store/src/contracts/dto/core_extraction.rs
// version: 3
// version: 4
//! Canonical transaction to core extraction storage contracts.
/// Maximum canonical transactions selected by one Core extraction batch.
pub const MAX_CORE_EXTRACTION_SELECTION_ROWS: u32 = 1000;
/// Stable processing status for one extraction ledger entry.
#[derive(Clone, Copy, Debug, Eq, PartialEq, serde::Deserialize, serde::Serialize)]
pub enum ProcessingLedgerStatus {
@@ -41,10 +44,11 @@ impl crate::CoreExtractionSelectionFilter {
program_id: std::option::Option<std::string::String>,
limit: u32,
) -> ks_core::Result<Self> {
if limit == 0 {
return std::result::Result::Err(ks_core::Error::db(
"core extraction selection limit must be greater than zero",
));
if limit == 0 || limit > crate::MAX_CORE_EXTRACTION_SELECTION_ROWS {
return std::result::Result::Err(ks_core::Error::db(format!(
"core extraction selection limit must be between 1 and {}",
crate::MAX_CORE_EXTRACTION_SELECTION_ROWS
)));
}
if let (std::option::Option::Some(minimum), std::option::Option::Some(maximum)) =
(min_slot, max_slot)
@@ -271,6 +275,14 @@ mod tests {
assert!(result.is_err());
}
#[test]
fn selection_filter_rejects_limit_above_processing_bound() {
let result = crate::CoreExtractionSelectionFilter::pending(
crate::MAX_CORE_EXTRACTION_SELECTION_ROWS + 1,
);
assert!(result.is_err());
}
#[test]
fn selection_filter_rejects_inverted_slots() {
let result = crate::CoreExtractionSelectionFilter::new(

View File

@@ -1,8 +1,10 @@
// file: ks-store/src/contracts/dto/decode.rs
// version: 6
// version: 7
//! Backend-neutral decode, coverage and materialization persistence DTOs.
/// Maximum contextual inputs selected by one decode processing batch.
pub const MAX_DECODE_SELECTION_ROWS: u32 = 1000;
/// Maximum number of materialized rows returned by one bounded query.
pub const MAX_MATERIALIZED_OUTPUT_QUERY_ROWS: u32 = 500;
@@ -111,10 +113,11 @@ impl crate::DecodeSelectionFilter {
incomplete_signatures: bool,
limit: u32,
) -> ks_core::Result<Self> {
if limit == 0 {
return std::result::Result::Err(ks_core::Error::db(
"decode selection limit must be greater than zero",
));
if limit == 0 || limit > crate::MAX_DECODE_SELECTION_ROWS {
return std::result::Result::Err(ks_core::Error::db(format!(
"decode selection limit must be between 1 and {}",
crate::MAX_DECODE_SELECTION_ROWS
)));
}
if min_slot.is_some() && max_slot.is_some() && min_slot > max_slot {
return std::result::Result::Err(ks_core::Error::db(
@@ -634,6 +637,13 @@ mod tests {
assert!(crate::DecodeSelectionFilter::actionable(0).is_err());
}
#[test]
fn actionable_filter_rejects_limit_above_processing_bound() {
assert!(
crate::DecodeSelectionFilter::actionable(crate::MAX_DECODE_SELECTION_ROWS + 1).is_err()
);
}
#[test]
fn incomplete_signature_filter_preserves_signature_limit_semantics() {
let result = crate::DecodeSelectionFilter::new(

View File

@@ -1,5 +1,5 @@
// file: ks-store/src/contracts/entity.rs
// version: 6
// version: 7
//! Backend-neutral persisted entity exports for storage adapters.
@@ -14,6 +14,8 @@ pub use self::core::CoreAccountStateRow;
pub use self::core::CoreBalanceChangeRow;
/// Core inner instruction persisted row contract.
pub use self::core::CoreInnerInstructionRow;
/// Logical Core instruction replay row spanning top-level and inner/CPI storage.
pub use self::core::CoreInstructionReplayRow;
/// Core instruction persisted row contract.
pub use self::core::CoreInstructionRow;
/// Core log persisted row contract.

View File

@@ -1,5 +1,5 @@
// file: ks-store/src/contracts/entity/core.rs
// version: 4
// version: 5
//! Core Solana persisted entities.
@@ -82,6 +82,41 @@ pub struct CoreAccountStateRow {
pub created_at: chrono::DateTime<chrono::Utc>,
}
/// Logical Core instruction row spanning top-level and inner/CPI storage.
#[derive(Clone, Debug, PartialEq, serde::Deserialize, serde::Serialize)]
pub struct CoreInstructionReplayRow {
/// Technical primary key inside the physical instruction scope.
pub id: i64,
/// Parent core transaction technical key.
pub transaction_id: i64,
/// Transaction signature.
pub signature: std::string::String,
/// Transaction slot stored as SQL `BIGINT`.
pub slot: i64,
/// Logical top-level or inner/CPI scope.
pub scope: crate::CoreInstructionScope,
/// Optional immediate parent instruction path for inner/CPI rows.
pub parent_instruction_path: std::option::Option<std::string::String>,
/// Stable instruction path.
pub instruction_path: std::string::String,
/// Program id.
pub program_id: std::string::String,
/// Optional runtime invocation stack height.
pub stack_height: std::option::Option<i32>,
/// Instruction accounts as JSON.
pub accounts_json: serde_json::Value,
/// Optional instruction payload retained in the hot store.
pub payload_json: std::option::Option<serde_json::Value>,
/// Optional deterministic payload digest.
pub payload_json_hash: std::option::Option<std::string::String>,
/// Current processing state.
pub processing_state: crate::CoreInstructionProcessingState,
/// Insert timestamp.
pub created_at: chrono::DateTime<chrono::Utc>,
/// Last lifecycle update timestamp.
pub updated_at: chrono::DateTime<chrono::Utc>,
}
/// Core instruction persisted row contract.
#[derive(Clone, Debug, PartialEq, serde::Deserialize, serde::Serialize)]
pub struct CoreInstructionRow {

View File

@@ -1,12 +1,14 @@
// file: ks-store/src/contracts/pagination.rs
// version: 3
// version: 4
//! Backend-neutral pagination and sorting contracts for repository operations.
//! Backend-neutral pagination contracts for repository operations.
/// Default page size for repository list operations.
pub const DEFAULT_PAGE_SIZE: u16 = 100;
/// Maximum page size for repository list operations.
pub const MAX_PAGE_SIZE: u16 = 1000;
pub const MAX_PAGE_SIZE: u16 = 500;
/// Maximum serialized opaque cursor length accepted by repository list operations.
pub const MAX_PAGE_CURSOR_LENGTH: usize = 512;
/// Sort direction for repository list operations.
#[derive(Clone, Copy, Debug, Eq, PartialEq, serde::Deserialize, serde::Serialize)]
@@ -17,18 +19,21 @@ pub enum SortDirection {
Desc,
}
/// Page request contract for repository list operations.
#[derive(Clone, Copy, Debug, Eq, PartialEq, serde::Deserialize, serde::Serialize)]
/// Stable cursor request contract for repository list operations.
#[derive(Clone, Debug, Eq, PartialEq, serde::Deserialize, serde::Serialize)]
pub struct PageRequest {
/// Maximum number of rows to return.
pub limit: u16,
/// Zero-based row offset.
pub offset: u64,
/// Optional opaque cursor returned by the previous page.
pub cursor: std::option::Option<std::string::String>,
}
impl crate::PageRequest {
/// Builds a page request after minimal bounds validation.
pub fn new(limit: u16, offset: u64) -> ks_core::Result<Self> {
/// Builds a page request after validating the limit and opaque cursor bounds.
pub fn new(
limit: u16,
cursor: std::option::Option<std::string::String>,
) -> ks_core::Result<Self> {
if limit == 0 {
return std::result::Result::Err(ks_core::Error::db(
"page limit must be greater than zero",
@@ -39,36 +44,95 @@ impl crate::PageRequest {
"page limit exceeds maximum page size",
));
}
return std::result::Result::Ok(Self { limit, offset });
let cursor_value = cursor.and_then(|value| {
let trimmed = value.trim();
if trimmed.is_empty() {
return std::option::Option::None;
}
return std::option::Option::Some(trimmed.to_string());
});
if cursor_value
.as_deref()
.is_some_and(|value| return value.len() > crate::MAX_PAGE_CURSOR_LENGTH)
{
return std::result::Result::Err(ks_core::Error::db(
"page cursor exceeds maximum encoded length",
));
}
return std::result::Result::Ok(Self { limit, cursor: cursor_value });
}
/// Builds the default first page request.
pub fn first_page() -> Self {
return Self {
limit: crate::DEFAULT_PAGE_SIZE,
offset: 0,
cursor: std::option::Option::None,
};
}
/// Builds a request continuing after an opaque cursor returned by a previous page.
pub fn after(
limit: u16,
cursor: impl std::convert::Into<std::string::String>,
) -> ks_core::Result<Self> {
return Self::new(limit, std::option::Option::Some(cursor.into()));
}
}
/// One bounded page of rows and its continuation cursor.
#[derive(Clone, Debug, PartialEq, serde::Deserialize, serde::Serialize)]
pub struct PageSlice<T> {
/// Rows returned for this page.
pub rows: std::vec::Vec<T>,
/// Opaque cursor for the next page when additional rows exist.
pub next_cursor: std::option::Option<std::string::String>,
}
impl<T> crate::PageSlice<T> {
/// Builds one page from already bounded rows and an optional continuation cursor.
pub fn new(
rows: std::vec::Vec<T>,
next_cursor: std::option::Option<std::string::String>,
) -> Self {
return Self { rows, next_cursor };
}
/// Returns true when this page has no continuation cursor.
pub fn is_last_page(&self) -> bool {
return self.next_cursor.is_none();
}
}
#[cfg(test)]
mod tests {
#[test]
fn page_request_rejects_zero_limit() {
let result = crate::PageRequest::new(0, 0);
let result = crate::PageRequest::new(0, std::option::Option::None);
assert!(result.is_err());
}
#[test]
fn page_request_rejects_limit_above_maximum() {
let result = crate::PageRequest::new(crate::MAX_PAGE_SIZE + 1, 0);
let result = crate::PageRequest::new(crate::MAX_PAGE_SIZE + 1, std::option::Option::None);
assert!(result.is_err());
}
#[test]
fn first_page_uses_default_limit() {
fn page_request_rejects_oversized_cursor() {
let result = crate::PageRequest::after(10, "x".repeat(crate::MAX_PAGE_CURSOR_LENGTH + 1));
assert!(result.is_err());
}
#[test]
fn first_page_uses_default_limit_without_cursor() {
let request = crate::PageRequest::first_page();
assert_eq!(request.limit, crate::DEFAULT_PAGE_SIZE);
assert_eq!(request.offset, 0);
assert!(request.cursor.is_none());
}
#[test]
fn page_slice_reports_last_page_from_cursor_presence() {
let page = crate::PageSlice::new(vec![1_u32], std::option::Option::None);
assert!(page.is_last_page());
}
}

View File

@@ -1,10 +1,10 @@
// file: ks-store/src/contracts/replay.rs
// version: 6
// version: 7
//! Backend-agnostic replay candidate filters and read models.
/// Maximum number of rows returned by one replay candidate query.
pub const MAX_REPLAY_CANDIDATE_ROWS: u32 = 100_000;
pub const MAX_REPLAY_CANDIDATE_ROWS: u32 = 500;
/// Program occurrence scope used while filtering replay candidates.
#[derive(Clone, Copy, Debug, Eq, PartialEq)]

View File

@@ -1,5 +1,5 @@
// file: ks-store/src/contracts/repository.rs
// version: 4
// version: 5
//! Storage trait definitions shared by concrete stores.
@@ -29,6 +29,9 @@ pub trait RawTransactionStore {
async fn has_transaction_observation_key(&self, observation_key: &str)
-> ks_core::Result<bool>;
/// Returns true when a generic account observation key is already stored.
async fn has_account_observation_key(&self, observation_key: &str) -> ks_core::Result<bool>;
/// Stores one canonical source-independent transaction payload.
async fn insert_raw_transaction(
&self,
@@ -41,6 +44,12 @@ pub trait RawTransactionStore {
input: &crate::TransactionObservationInsert,
) -> ks_core::Result<crate::InsertOutcome>;
/// Stores one generic replayable account acquisition observation.
async fn insert_account_observation(
&self,
input: &crate::AccountObservationInsert,
) -> ks_core::Result<crate::InsertOutcome>;
/// Updates canonical raw transaction retention and processing metadata.
async fn mark_raw_payload_lifecycle(
&self,
@@ -92,14 +101,14 @@ pub trait CoreTransactionStore {
&self,
filter: &crate::CoreInstructionReplayFilter,
page_request: &crate::PageRequest,
) -> ks_core::Result<std::vec::Vec<crate::CoreInstructionRow>>;
) -> ks_core::Result<crate::PageSlice<crate::CoreInstructionReplayRow>>;
/// Lists replay inputs with instruction context, logs, balances and account keys.
async fn list_core_instruction_replay_inputs(
&self,
filter: &crate::CoreInstructionReplayFilter,
page_request: &crate::PageRequest,
) -> ks_core::Result<std::vec::Vec<crate::MdCoreInstructionReplayInput>>;
) -> ks_core::Result<crate::PageSlice<crate::MdCoreInstructionReplayInput>>;
/// Updates one normalized core instruction lifecycle state.
async fn mark_core_instruction_lifecycle(
@@ -108,6 +117,35 @@ pub trait CoreTransactionStore {
) -> ks_core::Result<crate::InsertOutcome>;
}
/// Generic account observation to Core account-state normalization storage behavior.
#[async_trait::async_trait]
pub trait AccountStateStore {
/// Lists complete account observations selected for first normalization or replay.
async fn list_account_observations_for_normalization(
&self,
filter: &crate::AccountStateSelectionFilter,
) -> ks_core::Result<std::vec::Vec<crate::AccountObservationRow>>;
/// Returns true when the same account-state normalizer version succeeded for the same input hash.
async fn is_account_state_current(
&self,
identity: &crate::ProcessingLedgerIdentity,
) -> ks_core::Result<bool>;
/// Atomically persists one Core account state and its processing ledger result.
async fn persist_account_state(
&self,
bundle: &crate::AccountStatePersistenceBundle,
force_replay: bool,
) -> ks_core::Result<crate::InsertOutcome>;
/// Persists one failed account-state normalization attempt.
async fn mark_account_state_failed(
&self,
failure: &crate::AccountStateNormalizationFailure,
) -> ks_core::Result<crate::InsertOutcome>;
}
/// Canonical transaction to core extraction storage behavior.
#[async_trait::async_trait]
pub trait CoreExtractionStore {

View File

@@ -1,5 +1,5 @@
// file: ks-store/src/lib.rs
// version: 12
// version: 13
//! Backend-agnostic Solana storage contracts and store facade.
#![warn(missing_docs)]
@@ -67,10 +67,14 @@ pub(crate) use self::postgres::decode_store_schema_statements;
pub(crate) use self::postgres::decode_store_table_diagnostic_specs;
/// Crate-internal expected PostgreSQL index inventory shared through the crate-root facade.
pub(crate) use self::postgres::expected_postgres_index_names;
/// Crate-internal account-observation key lookup shared through the crate-root facade.
pub(crate) use self::postgres::has_account_observation_key;
/// Crate-internal storage symbol `has_raw_transaction_signature` shared through the crate-root facade.
pub(crate) use self::postgres::has_raw_transaction_signature;
/// Crate-internal storage symbol `has_transaction_observation_key` shared through the crate-root facade.
pub(crate) use self::postgres::has_transaction_observation_key;
/// Crate-internal account-observation insert shared through the crate-root facade.
pub(crate) use self::postgres::insert_account_observation;
/// Crate-internal storage symbol `insert_core_account_keys` shared through the crate-root facade.
pub(crate) use self::postgres::insert_core_account_keys;
/// Crate-internal storage symbol `insert_core_balance_changes` shared through the crate-root facade.
@@ -87,10 +91,18 @@ pub(crate) use self::postgres::insert_core_transaction;
pub(crate) use self::postgres::insert_raw_transaction;
/// Crate-internal storage symbol `insert_transaction_observation` shared through the crate-root facade.
pub(crate) use self::postgres::insert_transaction_observation;
/// Crate-internal Core descendant invalidation shared through the crate-root facade.
pub(crate) use self::postgres::invalidate_core_descendants_in_transaction;
/// Crate-internal decode descendant invalidation shared through the crate-root facade.
pub(crate) use self::postgres::invalidate_decode_descendants_in_transaction;
/// Crate-internal account-state currentness lookup shared through the crate-root facade.
pub(crate) use self::postgres::is_account_state_current;
/// Crate-internal storage symbol `is_core_extraction_current` shared through the crate-root facade.
pub(crate) use self::postgres::is_core_extraction_current;
/// Crate-internal storage symbol `is_decode_current` shared through the crate-root facade.
pub(crate) use self::postgres::is_decode_current;
/// Crate-internal account-observation normalization selection shared through the crate-root facade.
pub(crate) use self::postgres::list_account_observations_for_normalization;
/// Crate-internal storage symbol `list_core_instruction_replay_inputs` shared through the crate-root facade.
pub(crate) use self::postgres::list_core_instruction_replay_inputs;
/// Crate-internal storage symbol `list_core_instructions_for_replay` shared through the crate-root facade.
@@ -119,10 +131,14 @@ pub(crate) use self::postgres::load_expected_index_counts;
pub(crate) use self::postgres::load_server_version;
/// Crate-internal storage symbol `load_table_statistics` shared through the crate-root facade.
pub(crate) use self::postgres::load_table_statistics;
/// Crate-internal account-state failure persistence shared through the crate-root facade.
pub(crate) use self::postgres::mark_account_state_failed;
/// Crate-internal storage symbol `mark_core_extraction_failed` shared through the crate-root facade.
pub(crate) use self::postgres::mark_core_extraction_failed;
/// Crate-internal storage symbol `mark_decode_failed` shared through the crate-root facade.
pub(crate) use self::postgres::mark_decode_failed;
/// Crate-internal account-state persistence shared through the crate-root facade.
pub(crate) use self::postgres::persist_account_state;
/// Crate-internal storage symbol `persist_core_extraction` shared through the crate-root facade.
pub(crate) use self::postgres::persist_core_extraction;
/// Crate-internal storage symbol `persist_decode_coverage_declarations` shared through the crate-root facade.
@@ -138,6 +154,8 @@ pub(crate) use self::postgres::postgres_test_guard;
pub(crate) use self::postgres::raw_store_schema_statements;
/// Crate-internal storage symbol `raw_store_table_diagnostic_specs` shared through the crate-root facade.
pub(crate) use self::postgres::raw_store_table_diagnostic_specs;
/// Crate-internal instruction lifecycle recomputation shared through the crate-root facade.
pub(crate) use self::postgres::recompute_instruction_lifecycle_in_transaction;
/// Crate-internal storage symbol `run_health_check` shared through the crate-root facade.
pub(crate) use self::postgres::run_health_check;
/// Crate-internal storage symbol `table_exists` shared through the crate-root facade.
@@ -183,10 +201,20 @@ pub(crate) use self::postgres::validate_postgres_sql_resources;
/// Transitional logical schema contract identifier used until the `0.5.3-pre.3` rebuild.
pub use self::constants::STORE_SCHEMA_CONTRACT_VERSION;
/// Account-state normalization processing-stage identifier.
pub use self::contracts::ACCOUNT_STATE_NORMALIZATION_STAGE;
/// Generic account acquisition observation insert contract.
pub use self::contracts::AccountObservationInsert;
/// Generic account acquisition observation persisted row contract.
pub use self::contracts::AccountObservationRow;
/// Failed generic account-state normalization attempt.
pub use self::contracts::AccountStateNormalizationFailure;
/// Atomic persistence bundle for one generic Core account state.
pub use self::contracts::AccountStatePersistenceBundle;
/// Bounded generic account-observation selection for Core normalization.
pub use self::contracts::AccountStateSelectionFilter;
/// Generic account observation to Core account-state storage behavior.
pub use self::contracts::AccountStateStore;
/// Transaction acquisition observation origin.
pub use self::contracts::AcquisitionObservationOrigin;
/// Generic account acquisition observation status.
@@ -227,8 +255,12 @@ pub use self::contracts::CoreInstructionLifecycleMark;
pub use self::contracts::CoreInstructionProcessingState;
/// Core instruction replay filter contract.
pub use self::contracts::CoreInstructionReplayFilter;
/// Logical top-level or CPI instruction row selected for replay.
pub use self::contracts::CoreInstructionReplayRow;
/// Core instruction persisted row contract.
pub use self::contracts::CoreInstructionRow;
/// Logical scope of one replayable Core instruction.
pub use self::contracts::CoreInstructionScope;
/// Core log insert contract.
pub use self::contracts::CoreLogInsert;
/// Core log persisted row contract.
@@ -265,8 +297,16 @@ pub use self::contracts::DecodeSchemaProvenance;
pub use self::contracts::DecodeSelectionFilter;
/// Insert or upsert result contract returned by repositories.
pub use self::contracts::InsertOutcome;
/// Maximum number of account observations selected by one normalization query.
pub use self::contracts::MAX_ACCOUNT_STATE_SELECTION_ROWS;
/// Maximum canonical transactions selected by one Core extraction batch.
pub use self::contracts::MAX_CORE_EXTRACTION_SELECTION_ROWS;
/// Maximum contextual inputs selected by one decode processing batch.
pub use self::contracts::MAX_DECODE_SELECTION_ROWS;
/// Maximum number of materialized rows returned by one bounded query.
pub use self::contracts::MAX_MATERIALIZED_OUTPUT_QUERY_ROWS;
/// Maximum encoded opaque page-cursor length.
pub use self::contracts::MAX_PAGE_CURSOR_LENGTH;
/// Maximum repository page size.
pub use self::contracts::MAX_PAGE_SIZE;
/// Maximum number of rows returned by one replay candidate query.
@@ -281,6 +321,8 @@ pub use self::contracts::MaterializedOutputInsert;
pub use self::contracts::MaterializedOutputQueryRow;
/// Page request contract for repository list operations.
pub use self::contracts::PageRequest;
/// One bounded repository page and its optional continuation cursor.
pub use self::contracts::PageSlice;
/// Stable processing ledger identity.
pub use self::contracts::ProcessingLedgerIdentity;
/// Stable processing ledger status.

View File

@@ -1,5 +1,5 @@
// file: ks-store/src/postgres.rs
// version: 13
// version: 14
//! Private PostgreSQL implementation of the backend-agnostic store contracts.
@@ -54,8 +54,10 @@ pub(crate) use self::migrations::table_stats_k_sol_ops_processing_ledger_sql;
pub(crate) use self::migrations::table_stats_k_sol_raw_transactions_sql;
pub(crate) use self::migrations::validate_postgres_sql_resources;
pub(crate) use self::query::apply_store_schema;
pub(crate) use self::query::has_account_observation_key;
pub(crate) use self::query::has_raw_transaction_signature;
pub(crate) use self::query::has_transaction_observation_key;
pub(crate) use self::query::insert_account_observation;
pub(crate) use self::query::insert_core_account_keys;
pub(crate) use self::query::insert_core_balance_changes;
pub(crate) use self::query::insert_core_inner_instructions;
@@ -64,8 +66,12 @@ pub(crate) use self::query::insert_core_logs;
pub(crate) use self::query::insert_core_transaction;
pub(crate) use self::query::insert_raw_transaction;
pub(crate) use self::query::insert_transaction_observation;
pub(crate) use self::query::invalidate_core_descendants_in_transaction;
pub(crate) use self::query::invalidate_decode_descendants_in_transaction;
pub(crate) use self::query::is_account_state_current;
pub(crate) use self::query::is_core_extraction_current;
pub(crate) use self::query::is_decode_current;
pub(crate) use self::query::list_account_observations_for_normalization;
pub(crate) use self::query::list_core_instruction_replay_inputs;
pub(crate) use self::query::list_core_instructions_for_replay;
pub(crate) use self::query::list_decode_coverage_summary;
@@ -80,12 +86,15 @@ pub(crate) use self::query::load_current_schema;
pub(crate) use self::query::load_expected_index_counts;
pub(crate) use self::query::load_server_version;
pub(crate) use self::query::load_table_statistics;
pub(crate) use self::query::mark_account_state_failed;
pub(crate) use self::query::mark_core_extraction_failed;
pub(crate) use self::query::mark_decode_failed;
pub(crate) use self::query::persist_account_state;
pub(crate) use self::query::persist_core_extraction;
pub(crate) use self::query::persist_decode_coverage_declarations;
pub(crate) use self::query::persist_decode_result;
pub(crate) use self::query::persist_materialization_result;
pub(crate) use self::query::recompute_instruction_lifecycle_in_transaction;
pub(crate) use self::query::run_health_check;
pub(crate) use self::query::table_exists;
pub(crate) use self::query::update_core_instruction_lifecycle;

View File

@@ -1,5 +1,5 @@
// file: ks-store/src/postgres/migrations.rs
// version: 10
// version: 11
//! PostgreSQL schema resources and migration conventions for the storage backend.
@@ -83,6 +83,7 @@ const EXPECTED_POSTGRES_INDEX_NAMES: &[&str] = &[
"ix_k_sol_core_inner_instructions_parent",
"ix_k_sol_core_inner_instructions_processing",
"ix_k_sol_core_inner_instructions_program",
"ix_k_sol_core_inner_instructions_slot",
"ix_k_sol_core_instructions_processing",
"ix_k_sol_core_instructions_program",
"ix_k_sol_core_instructions_slot",
@@ -94,10 +95,13 @@ const EXPECTED_POSTGRES_INDEX_NAMES: &[&str] = &[
"ix_k_sol_decode_coverage_declarations_program",
"ix_k_sol_decode_coverage_observations_entry",
"ix_k_sol_decode_coverage_observations_program_status",
"ix_k_sol_decode_coverage_observations_signature",
"ix_k_sol_decode_events_family_commit",
"ix_k_sol_decode_events_program_surface",
"ix_k_sol_decode_events_signature_path",
"ix_k_sol_mat_outputs_processor_family_slot",
"ix_k_sol_mat_outputs_signature",
"ix_k_sol_mat_outputs_source_decode_input",
"ix_k_sol_mat_outputs_signature_family",
"ix_k_sol_obs_account_observations_account_slot",
"ix_k_sol_obs_account_observations_provider_method",
@@ -1277,6 +1281,12 @@ fn create_pg_index_if_not_exists_ix_k_sol_core_inner_instructions_program() -> &
);
}
fn create_pg_index_if_not_exists_ix_k_sol_core_inner_instructions_slot() -> &'static str {
return include_str!(
"../../migrations/postgres/schema/indexes/create_index_if_not_exists_ix_k_sol_core_inner_instructions_slot.sql"
);
}
fn create_pg_index_if_not_exists_ix_k_sol_core_instructions_processing() -> &'static str {
return include_str!(
"../../migrations/postgres/schema/indexes/create_index_if_not_exists_ix_k_sol_core_instructions_processing.sql"
@@ -1344,6 +1354,12 @@ fn create_pg_index_if_not_exists_ix_k_sol_decode_coverage_observations_program_s
);
}
fn create_pg_index_if_not_exists_ix_k_sol_decode_coverage_observations_signature() -> &'static str {
return include_str!(
"../../migrations/postgres/schema/indexes/create_index_if_not_exists_ix_k_sol_decode_coverage_observations_signature.sql"
);
}
fn create_pg_index_if_not_exists_ix_k_sol_decode_events_family_commit() -> &'static str {
return include_str!(
"../../migrations/postgres/schema/indexes/create_index_if_not_exists_ix_k_sol_decode_events_family_commit.sql"
@@ -1374,6 +1390,18 @@ fn create_pg_index_if_not_exists_ix_k_sol_mat_outputs_signature_family() -> &'st
);
}
fn create_pg_index_if_not_exists_ix_k_sol_mat_outputs_signature() -> &'static str {
return include_str!(
"../../migrations/postgres/schema/indexes/create_index_if_not_exists_ix_k_sol_mat_outputs_signature.sql"
);
}
fn create_pg_index_if_not_exists_ix_k_sol_mat_outputs_source_decode_input() -> &'static str {
return include_str!(
"../../migrations/postgres/schema/indexes/create_index_if_not_exists_ix_k_sol_mat_outputs_source_decode_input.sql"
);
}
fn create_pg_index_if_not_exists_ix_k_sol_obs_account_observations_account_slot() -> &'static str {
return include_str!(
"../../migrations/postgres/schema/indexes/create_index_if_not_exists_ix_k_sol_obs_account_observations_account_slot.sql"
@@ -1942,6 +1970,7 @@ pub(crate) fn core_store_schema_statements() -> std::vec::Vec<&'static str> {
create_pg_index_if_not_exists_ix_k_sol_core_inner_instructions_parent(),
create_pg_index_if_not_exists_ix_k_sol_core_inner_instructions_processing(),
create_pg_index_if_not_exists_ix_k_sol_core_inner_instructions_program(),
create_pg_index_if_not_exists_ix_k_sol_core_inner_instructions_slot(),
create_pg_index_if_not_exists_ix_k_sol_core_instructions_processing(),
create_pg_index_if_not_exists_ix_k_sol_core_instructions_program(),
create_pg_index_if_not_exists_ix_k_sol_core_instructions_slot(),
@@ -1985,11 +2014,14 @@ pub(crate) fn decode_store_schema_statements() -> std::vec::Vec<&'static str> {
create_pg_index_if_not_exists_ix_k_sol_decode_coverage_declarations_program(),
create_pg_index_if_not_exists_ix_k_sol_decode_coverage_observations_entry(),
create_pg_index_if_not_exists_ix_k_sol_decode_coverage_observations_program_status(),
create_pg_index_if_not_exists_ix_k_sol_decode_coverage_observations_signature(),
create_pg_index_if_not_exists_ix_k_sol_decode_events_family_commit(),
create_pg_index_if_not_exists_ix_k_sol_decode_events_program_surface(),
create_pg_index_if_not_exists_ix_k_sol_decode_events_signature_path(),
create_pg_index_if_not_exists_ix_k_sol_mat_outputs_processor_family_slot(),
create_pg_index_if_not_exists_ix_k_sol_mat_outputs_signature(),
create_pg_index_if_not_exists_ix_k_sol_mat_outputs_signature_family(),
create_pg_index_if_not_exists_ix_k_sol_mat_outputs_source_decode_input(),
create_pg_index_if_not_exists_ux_k_sol_decode_coverage_declarations_identity(),
create_pg_index_if_not_exists_ux_k_sol_decode_coverage_observations_identity(),
create_pg_index_if_not_exists_ux_k_sol_decode_events_processor_input_event(),
@@ -2208,12 +2240,12 @@ mod tests {
}
#[test]
fn candidate_baseline_inventory_has_sixteen_tables_and_seventy_five_indexes() {
fn candidate_baseline_inventory_has_sixteen_tables_and_seventy_nine_indexes() {
let table_count = super::RAW_STORE_TABLE_NAMES.len()
+ super::CORE_STORE_TABLE_NAMES.len()
+ super::DECODE_STORE_TABLE_NAMES.len();
assert_eq!(table_count, 16);
assert_eq!(crate::expected_postgres_index_names().len(), 75);
assert_eq!(crate::expected_postgres_index_names().len(), 79);
}
#[test]
@@ -2232,7 +2264,7 @@ mod tests {
}
let maintenance_count = super::truncate_store_schema_statements().len()
+ super::drop_store_schema_statements().len();
assert_eq!(schema_count + maintenance_count, 236);
assert_eq!(schema_count + maintenance_count, 240);
}
#[test]

View File

@@ -1,17 +1,23 @@
// file: ks-store/src/postgres/query.rs
// version: 5
// version: 6
//! PostgreSQL query modules.
mod account_state_queries;
mod core_extraction_queries;
mod core_queries;
mod decode_pipeline_queries;
mod health_queries;
mod invalidation_queries;
mod raw_queries;
mod replay_candidate_queries;
mod schema_queries;
mod table_diagnostics_queries;
pub(crate) use self::account_state_queries::is_account_state_current;
pub(crate) use self::account_state_queries::list_account_observations_for_normalization;
pub(crate) use self::account_state_queries::mark_account_state_failed;
pub(crate) use self::account_state_queries::persist_account_state;
pub(crate) use self::core_extraction_queries::is_core_extraction_current;
pub(crate) use self::core_extraction_queries::list_raw_transactions_for_core_extraction;
pub(crate) use self::core_extraction_queries::mark_core_extraction_failed;
@@ -37,8 +43,13 @@ pub(crate) use self::decode_pipeline_queries::persist_materialization_result;
pub(crate) use self::health_queries::load_current_schema;
pub(crate) use self::health_queries::load_server_version;
pub(crate) use self::health_queries::run_health_check;
pub(crate) use self::invalidation_queries::invalidate_core_descendants_in_transaction;
pub(crate) use self::invalidation_queries::invalidate_decode_descendants_in_transaction;
pub(crate) use self::invalidation_queries::recompute_instruction_lifecycle_in_transaction;
pub(crate) use self::raw_queries::has_account_observation_key;
pub(crate) use self::raw_queries::has_raw_transaction_signature;
pub(crate) use self::raw_queries::has_transaction_observation_key;
pub(crate) use self::raw_queries::insert_account_observation;
pub(crate) use self::raw_queries::insert_raw_transaction;
pub(crate) use self::raw_queries::insert_transaction_observation;
pub(crate) use self::raw_queries::update_raw_payload_lifecycle;

View File

@@ -0,0 +1,708 @@
// file: ks-store/src/postgres/query/account_state_queries.rs
// version: 4
//! PostgreSQL queries for generic account-observation to Core account-state normalization.
use sqlx::Row; // rust-rules: trait-import
pub(crate) async fn list_account_observations_for_normalization(
pool: &sqlx::PgPool,
filter: &crate::AccountStateSelectionFilter,
) -> ks_core::Result<std::vec::Vec<crate::AccountObservationRow>> {
let min_slot_result = optional_sql_bigint_from_u64(filter.min_slot);
let min_slot = match min_slot_result {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let max_slot_result = optional_sql_bigint_from_u64(filter.max_slot);
let max_slot = match max_slot_result {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let started_at = std::time::Instant::now();
let query_result = sqlx::query(
"SELECT o.id, o.observation_key, o.account_key, o.context_slot, o.owner, o.lamports::TEXT AS lamports, o.executable, o.rent_epoch::TEXT AS rent_epoch, o.space, o.data_base64, o.data_hash, o.provider, o.endpoint_code, o.protocol, o.acquisition_method, o.commitment, o.capture_session_id, o.filter_code, o.origin, o.detected_at, o.received_at, o.normalized_at, o.persisted_at, o.payload_size_bytes, o.source_payload_hash, o.status, o.error_code, o.error_message FROM k_sol_obs_account_observations o WHERE o.owner IS NOT NULL AND o.lamports IS NOT NULL AND o.executable IS NOT NULL AND o.rent_epoch IS NOT NULL AND o.space IS NOT NULL AND o.data_base64 IS NOT NULL AND o.data_hash IS NOT NULL AND (cardinality($1::TEXT[]) = 0 OR o.account_key = ANY($1)) AND ($2::BIGINT IS NULL OR o.context_slot >= $2) AND ($3::BIGINT IS NULL OR o.context_slot <= $3) AND ($4 OR NOT EXISTS (SELECT 1 FROM k_sol_ops_processing_ledger l WHERE l.stage = 'account_state_normalization' AND l.processor_name = $5 AND l.processor_version = $6 AND l.input_key = o.observation_key AND l.status = 'succeeded')) ORDER BY o.context_slot ASC, o.id ASC LIMIT $7",
)
.bind(&filter.account_keys)
.bind(min_slot)
.bind(max_slot)
.bind(filter.include_current)
.bind(filter.processor_name.as_str())
.bind(filter.processor_version.as_str())
.bind(i64::from(filter.limit))
.fetch_all(pool)
.await;
let rows = match query_result {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => {
trace_query_failure("list_account_observations", started_at, &error);
return std::result::Result::Err(ks_core::Error::db(format!(
"postgres account observation normalization query failed: {error}"
)));
},
};
let mut output = std::vec::Vec::with_capacity(rows.len());
for row in rows {
let mapped_result = map_account_observation_row(&row);
match mapped_result {
std::result::Result::Ok(value) => output.push(value),
std::result::Result::Err(error) => return std::result::Result::Err(error),
}
}
trace_query_success("list_account_observations", started_at, output.len());
return std::result::Result::Ok(output);
}
pub(crate) async fn is_account_state_current(
pool: &sqlx::PgPool,
identity: &crate::ProcessingLedgerIdentity,
) -> ks_core::Result<bool> {
if identity.stage != crate::ACCOUNT_STATE_NORMALIZATION_STAGE {
return std::result::Result::Err(ks_core::Error::db(
"account-state current check stage is invalid",
));
}
let started_at = std::time::Instant::now();
let query_result = sqlx::query_scalar::<sqlx::Postgres, bool>(
"SELECT EXISTS(SELECT 1 FROM k_sol_ops_processing_ledger WHERE stage = $1 AND processor_name = $2 AND processor_version = $3 AND input_key = $4 AND input_hash = $5 AND status = 'succeeded')",
)
.bind(identity.stage.as_str())
.bind(identity.processor_name.as_str())
.bind(identity.processor_version.as_str())
.bind(identity.input_key.as_str())
.bind(identity.input_hash.as_str())
.fetch_one(pool)
.await;
return match query_result {
std::result::Result::Ok(value) => {
trace_query_success(
"is_account_state_current",
started_at,
if value { 1_usize } else { 0_usize },
);
std::result::Result::Ok(value)
},
std::result::Result::Err(error) => {
trace_query_failure("is_account_state_current", started_at, &error);
std::result::Result::Err(ks_core::Error::db(format!(
"postgres account-state ledger current check failed: {error}"
)))
},
};
}
pub(crate) async fn persist_account_state(
pool: &sqlx::PgPool,
bundle: &crate::AccountStatePersistenceBundle,
force_replay: bool,
) -> ks_core::Result<crate::InsertOutcome> {
let validation_result = bundle.validate();
if let std::result::Result::Err(error) = validation_result {
return std::result::Result::Err(error);
}
if !force_replay {
let current_result = crate::is_account_state_current(pool, &bundle.ledger_identity).await;
match current_result {
std::result::Result::Ok(true) => {
return std::result::Result::Ok(crate::InsertOutcome::new(0, 0, 1));
},
std::result::Result::Ok(false) => {},
std::result::Result::Err(error) => return std::result::Result::Err(error),
}
}
let started_at = std::time::Instant::now();
let transaction_result = pool.begin().await;
let mut transaction = match transaction_result {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => {
return std::result::Result::Err(ks_core::Error::db(format!(
"postgres account-state transaction begin failed: {error}"
)));
},
};
let source_result =
load_source_observation(&mut transaction, bundle.state.source_observation_id).await;
let source = match source_result {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let lineage_result = validate_source_lineage(&source, bundle);
if let std::result::Result::Err(error) = lineage_result {
return std::result::Result::Err(error);
}
let existed_result = sqlx::query_scalar::<sqlx::Postgres, bool>(
"SELECT EXISTS(SELECT 1 FROM k_sol_core_account_states WHERE source_observation_id = $1)",
)
.bind(bundle.state.source_observation_id)
.fetch_one(&mut *transaction)
.await;
let existed = match existed_result {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => {
return std::result::Result::Err(ks_core::Error::db(format!(
"postgres account-state existing row query failed: {error}"
)));
},
};
let slot_result = sql_bigint_from_u64(
bundle.state.slot,
"account-state slot does not fit into PostgreSQL BIGINT",
);
let slot = match slot_result {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let space_result = sql_bigint_from_u64(
bundle.state.space,
"account-state space does not fit into PostgreSQL BIGINT",
);
let space = match space_result {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let lamports = bundle.state.lamports.to_string();
let rent_epoch = bundle.state.rent_epoch.to_string();
let upsert_result = sqlx::query(
"INSERT INTO k_sol_core_account_states (source_observation_id, account_key, slot, owner, lamports, executable, rent_epoch, space, data_base64, data_hash) VALUES ($1, $2, $3, $4, $5::NUMERIC, $6, $7::NUMERIC, $8, $9, $10) ON CONFLICT (source_observation_id) DO UPDATE SET account_key = EXCLUDED.account_key, slot = EXCLUDED.slot, owner = EXCLUDED.owner, lamports = EXCLUDED.lamports, executable = EXCLUDED.executable, rent_epoch = EXCLUDED.rent_epoch, space = EXCLUDED.space, data_base64 = EXCLUDED.data_base64, data_hash = EXCLUDED.data_hash",
)
.bind(bundle.state.source_observation_id)
.bind(bundle.state.account_key.as_str())
.bind(slot)
.bind(bundle.state.owner.as_str())
.bind(lamports.as_str())
.bind(bundle.state.executable)
.bind(rent_epoch.as_str())
.bind(space)
.bind(bundle.state.data_base64.as_str())
.bind(bundle.state.data_hash.as_str())
.execute(&mut *transaction)
.await;
if let std::result::Result::Err(error) = upsert_result {
return std::result::Result::Err(ks_core::Error::db(format!(
"postgres account-state upsert failed: {error}"
)));
}
let observation_result = sqlx::query(
"UPDATE k_sol_obs_account_observations SET status = 'normalized', normalized_at = NOW(), error_code = NULL, error_message = NULL WHERE id = $1",
)
.bind(bundle.state.source_observation_id)
.execute(&mut *transaction)
.await;
if let std::result::Result::Err(error) = observation_result {
return std::result::Result::Err(ks_core::Error::db(format!(
"postgres account observation normalization update failed: {error}"
)));
}
let ledger_result = upsert_ledger_terminal(
&mut transaction,
&bundle.ledger_identity,
"succeeded",
std::option::Option::None,
std::option::Option::None,
)
.await;
if let std::result::Result::Err(error) = ledger_result {
return std::result::Result::Err(error);
}
let commit_result = transaction.commit().await;
if let std::result::Result::Err(error) = commit_result {
return std::result::Result::Err(ks_core::Error::db(format!(
"postgres account-state transaction commit failed: {error}"
)));
}
let outcome = if existed {
crate::InsertOutcome::new(0, 1, 0)
} else {
crate::InsertOutcome::new(1, 0, 0)
};
trace_query_success("persist_account_state", started_at, 1);
return std::result::Result::Ok(outcome);
}
pub(crate) async fn mark_account_state_failed(
pool: &sqlx::PgPool,
failure: &crate::AccountStateNormalizationFailure,
) -> ks_core::Result<crate::InsertOutcome> {
let validation_result = failure.validate();
if let std::result::Result::Err(error) = validation_result {
return std::result::Result::Err(error);
}
let started_at = std::time::Instant::now();
let transaction_result = pool.begin().await;
let mut transaction = match transaction_result {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => {
return std::result::Result::Err(ks_core::Error::db(format!(
"postgres account-state failure transaction begin failed: {error}"
)));
},
};
let update_result = sqlx::query(
"UPDATE k_sol_obs_account_observations SET status = 'failed', error_code = $1, error_message = $2 WHERE id = $3",
)
.bind(failure.error_code.as_str())
.bind(failure.error_message.as_str())
.bind(failure.source_observation_id)
.execute(&mut *transaction)
.await;
let rows_affected = match update_result {
std::result::Result::Ok(value) => value.rows_affected(),
std::result::Result::Err(error) => {
return std::result::Result::Err(ks_core::Error::db(format!(
"postgres account observation failure update failed: {error}"
)));
},
};
if rows_affected == 0 {
return std::result::Result::Err(ks_core::Error::db(
"account-state normalization source observation does not exist",
));
}
let ledger_result = upsert_ledger_terminal(
&mut transaction,
&failure.ledger_identity,
"failed",
std::option::Option::Some(failure.error_code.as_str()),
std::option::Option::Some(failure.error_message.as_str()),
)
.await;
if let std::result::Result::Err(error) = ledger_result {
return std::result::Result::Err(error);
}
let commit_result = transaction.commit().await;
if let std::result::Result::Err(error) = commit_result {
return std::result::Result::Err(ks_core::Error::db(format!(
"postgres account-state failure transaction commit failed: {error}"
)));
}
trace_query_success("mark_account_state_failed", started_at, 1);
return std::result::Result::Ok(crate::InsertOutcome::new(0, 1, 0));
}
struct SourceAccountObservation {
observation_key: std::string::String,
account_key: std::string::String,
context_slot: i64,
owner: std::string::String,
lamports: std::string::String,
executable: bool,
rent_epoch: std::string::String,
space: i64,
data_base64: std::string::String,
data_hash: std::string::String,
}
async fn load_source_observation(
transaction: &mut sqlx::Transaction<'_, sqlx::Postgres>,
source_observation_id: i64,
) -> ks_core::Result<SourceAccountObservation> {
let query_result = sqlx::query(
"SELECT observation_key, account_key, context_slot, owner, lamports::TEXT AS lamports, executable, rent_epoch::TEXT AS rent_epoch, space, data_base64, data_hash FROM k_sol_obs_account_observations WHERE id = $1 AND owner IS NOT NULL AND lamports IS NOT NULL AND executable IS NOT NULL AND rent_epoch IS NOT NULL AND space IS NOT NULL AND data_base64 IS NOT NULL AND data_hash IS NOT NULL LIMIT 1",
)
.bind(source_observation_id)
.fetch_optional(&mut **transaction)
.await;
let row = match query_result {
std::result::Result::Ok(std::option::Option::Some(value)) => value,
std::result::Result::Ok(std::option::Option::None) => {
return std::result::Result::Err(ks_core::Error::db(
"account-state normalization requires one complete source observation",
));
},
std::result::Result::Err(error) => {
return std::result::Result::Err(ks_core::Error::db(format!(
"postgres account-state source observation query failed: {error}"
)));
},
};
let observation_key = match read_row(&row, "observation_key") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let account_key = match read_row(&row, "account_key") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let context_slot = match read_row(&row, "context_slot") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let owner = match read_row(&row, "owner") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let lamports = match read_row(&row, "lamports") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let executable = match read_row(&row, "executable") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let rent_epoch = match read_row(&row, "rent_epoch") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let space = match read_row(&row, "space") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let data_base64 = match read_row(&row, "data_base64") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let data_hash = match read_row(&row, "data_hash") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
return std::result::Result::Ok(SourceAccountObservation {
observation_key,
account_key,
context_slot,
owner,
lamports,
executable,
rent_epoch,
space,
data_base64,
data_hash,
});
}
fn validate_source_lineage(
source: &SourceAccountObservation,
bundle: &crate::AccountStatePersistenceBundle,
) -> ks_core::Result<()> {
let slot_result = u64::try_from(source.context_slot);
let source_slot = match slot_result {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => {
return std::result::Result::Err(ks_core::Error::db(format!(
"account observation context slot is negative: {error}"
)));
},
};
let space_result = u64::try_from(source.space);
let source_space = match space_result {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => {
return std::result::Result::Err(ks_core::Error::db(format!(
"account observation space is negative: {error}"
)));
},
};
if source.observation_key != bundle.ledger_identity.input_key
|| source.account_key != bundle.state.account_key
|| source_slot != bundle.state.slot
|| source.owner != bundle.state.owner
|| source.lamports != bundle.state.lamports.to_string()
|| source.executable != bundle.state.executable
|| source.rent_epoch != bundle.state.rent_epoch.to_string()
|| source_space != bundle.state.space
|| source.data_base64 != bundle.state.data_base64
|| source.data_hash != bundle.state.data_hash
{
return std::result::Result::Err(ks_core::Error::db(
"account-state persistence bundle does not match its source observation",
));
}
return std::result::Result::Ok(());
}
async fn upsert_ledger_terminal(
transaction: &mut sqlx::Transaction<'_, sqlx::Postgres>,
identity: &crate::ProcessingLedgerIdentity,
status: &str,
error_code: std::option::Option<&str>,
error_message: std::option::Option<&str>,
) -> ks_core::Result<()> {
let query_result = sqlx::query(
"INSERT INTO k_sol_ops_processing_ledger (stage, processor_name, processor_version, input_key, input_hash, status, attempt_count, started_at, finished_at, error_code, error_message) VALUES ($1, $2, $3, $4, $5, $6, 1, NOW(), NOW(), $7, $8) ON CONFLICT (stage, processor_name, processor_version, input_key) DO UPDATE SET input_hash = EXCLUDED.input_hash, status = EXCLUDED.status, attempt_count = k_sol_ops_processing_ledger.attempt_count + 1, started_at = NOW(), finished_at = NOW(), error_code = EXCLUDED.error_code, error_message = EXCLUDED.error_message, updated_at = NOW()",
)
.bind(identity.stage.as_str())
.bind(identity.processor_name.as_str())
.bind(identity.processor_version.as_str())
.bind(identity.input_key.as_str())
.bind(identity.input_hash.as_str())
.bind(status)
.bind(error_code)
.bind(error_message)
.execute(&mut **transaction)
.await;
if let std::result::Result::Err(error) = query_result {
return std::result::Result::Err(ks_core::Error::db(format!(
"postgres account-state processing ledger upsert failed: {error}"
)));
}
return std::result::Result::Ok(());
}
fn map_account_observation_row(
row: &sqlx::postgres::PgRow,
) -> ks_core::Result<crate::AccountObservationRow> {
let origin_text: std::string::String = match read_row(row, "origin") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let status_text: std::string::String = match read_row(row, "status") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let id = match read_row(row, "id") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let observation_key = match read_row(row, "observation_key") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let account_key = match read_row(row, "account_key") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let context_slot = match read_row(row, "context_slot") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let owner = match read_row(row, "owner") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let lamports = match read_row(row, "lamports") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let executable = match read_row(row, "executable") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let rent_epoch = match read_row(row, "rent_epoch") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let space = match read_row(row, "space") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let data_base64 = match read_row(row, "data_base64") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let data_hash = match read_row(row, "data_hash") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let provider = match read_row(row, "provider") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let endpoint_code = match read_row(row, "endpoint_code") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let protocol = match read_row(row, "protocol") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let acquisition_method = match read_row(row, "acquisition_method") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let commitment = match read_row(row, "commitment") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let capture_session_id = match read_row(row, "capture_session_id") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let filter_code = match read_row(row, "filter_code") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let detected_at = match read_row(row, "detected_at") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let received_at = match read_row(row, "received_at") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let normalized_at = match read_row(row, "normalized_at") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let persisted_at = match read_row(row, "persisted_at") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let payload_size_bytes = match read_row(row, "payload_size_bytes") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let source_payload_hash = match read_row(row, "source_payload_hash") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let error_code = match read_row(row, "error_code") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let error_message = match read_row(row, "error_message") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let origin = match observation_origin_from_sql(origin_text.as_str()) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let status = match observation_status_from_sql(status_text.as_str()) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
return std::result::Result::Ok(crate::AccountObservationRow {
id,
observation_key,
account_key,
context_slot,
owner,
lamports,
executable,
rent_epoch,
space,
data_base64,
data_hash,
provider,
endpoint_code,
protocol,
acquisition_method,
commitment,
capture_session_id,
filter_code,
origin,
detected_at,
received_at,
normalized_at,
persisted_at,
payload_size_bytes,
source_payload_hash,
status,
error_code,
error_message,
});
}
fn observation_origin_from_sql(
value: &str,
) -> ks_core::Result<crate::AcquisitionObservationOrigin> {
return match value {
"live" => std::result::Result::Ok(crate::AcquisitionObservationOrigin::Live),
"backfill" => std::result::Result::Ok(crate::AcquisitionObservationOrigin::Backfill),
"replay" => std::result::Result::Ok(crate::AcquisitionObservationOrigin::Replay),
"repair" => std::result::Result::Ok(crate::AcquisitionObservationOrigin::Repair),
"migration" => std::result::Result::Ok(crate::AcquisitionObservationOrigin::Migration),
_unknown => {
std::result::Result::Err(ks_core::Error::db("unknown account observation origin"))
},
};
}
fn observation_status_from_sql(
value: &str,
) -> ks_core::Result<crate::AcquisitionObservationStatus> {
return match value {
"detected" => std::result::Result::Ok(crate::AcquisitionObservationStatus::Detected),
"received" => std::result::Result::Ok(crate::AcquisitionObservationStatus::Received),
"normalized" => std::result::Result::Ok(crate::AcquisitionObservationStatus::Normalized),
"persisted" => std::result::Result::Ok(crate::AcquisitionObservationStatus::Persisted),
"failed" => std::result::Result::Ok(crate::AcquisitionObservationStatus::Failed),
"missing" => std::result::Result::Ok(crate::AcquisitionObservationStatus::Missing),
_unknown => {
std::result::Result::Err(ks_core::Error::db("unknown account observation status"))
},
};
}
fn optional_sql_bigint_from_u64(
value: std::option::Option<u64>,
) -> ks_core::Result<std::option::Option<i64>> {
return match value {
std::option::Option::Some(raw) => {
let converted_result = i64::try_from(raw);
match converted_result {
std::result::Result::Ok(converted) => {
std::result::Result::Ok(std::option::Option::Some(converted))
},
std::result::Result::Err(error) => std::result::Result::Err(ks_core::Error::db(
format!("unsigned value does not fit into PostgreSQL BIGINT: {error}"),
)),
}
},
std::option::Option::None => std::result::Result::Ok(std::option::Option::None),
};
}
fn sql_bigint_from_u64(value: u64, message: &str) -> ks_core::Result<i64> {
let result = i64::try_from(value);
return match result {
std::result::Result::Ok(converted) => std::result::Result::Ok(converted),
std::result::Result::Err(error) => {
std::result::Result::Err(ks_core::Error::db(format!("{message}: {error}")))
},
};
}
fn read_row<'row, T>(row: &'row sqlx::postgres::PgRow, column: &str) -> ks_core::Result<T>
where
T: sqlx::Decode<'row, sqlx::Postgres> + sqlx::Type<sqlx::Postgres>,
{
let value_result = row.try_get::<T, &str>(column);
return match value_result {
std::result::Result::Ok(value) => std::result::Result::Ok(value),
std::result::Result::Err(error) => std::result::Result::Err(ks_core::Error::db(format!(
"postgres account-state row read failed for {column}: {error}"
))),
};
}
fn trace_query_success(action: &str, started_at: std::time::Instant, rows: usize) {
let elapsed_ms = started_at.elapsed().as_secs_f64() * 1000.0;
tracing::trace!(target: crate::TRACING_TARGET, backend = "postgres", domain = "ks-store.pg", action = action, elapsed_ms, rows, outcome = "success", "PostgreSQL store operation completed");
}
fn trace_query_failure(action: &str, started_at: std::time::Instant, error: &sqlx::Error) {
let elapsed_ms = started_at.elapsed().as_secs_f64() * 1000.0;
tracing::trace!(target: crate::TRACING_TARGET, backend = "postgres", domain = "ks-store.pg", action = action, elapsed_ms, error = %error, outcome = "error", "PostgreSQL store operation failed");
}
#[cfg(test)]
mod tests {
#[test]
fn account_state_selection_is_processor_version_aware() {
let source = include_str!("account_state_queries.rs");
let tests_marker = source.find("#[cfg(test)]");
let production_source = match tests_marker {
std::option::Option::Some(index) => &source[..index],
std::option::Option::None => source,
};
assert!(production_source.contains("l.processor_name = $5"));
assert!(production_source.contains("l.processor_version = $6"));
assert!(production_source.contains("l.input_key = o.observation_key"));
assert!(!production_source.contains("NOT EXISTS (SELECT 1 FROM k_sol_core_account_states"));
}
#[test]
fn account_state_bigint_conversion_rejects_unsigned_overflow() {
assert!(super::sql_bigint_from_u64(u64::MAX, "overflow").is_err());
assert!(super::optional_sql_bigint_from_u64(std::option::Option::Some(u64::MAX)).is_err());
}
#[test]
fn account_state_status_mapping_is_closed() {
assert_eq!(
super::observation_status_from_sql("normalized"),
std::result::Result::Ok(crate::AcquisitionObservationStatus::Normalized)
);
assert!(super::observation_status_from_sql("other").is_err());
}
}

View File

@@ -1,5 +1,5 @@
// file: ks-store/src/postgres/query/core_extraction_queries.rs
// version: 8
// version: 9
//! PostgreSQL queries for atomic canonical transaction to core extraction.
@@ -22,6 +22,7 @@ pub(crate) async fn list_raw_transactions_for_core_extraction(
let processing_state = filter.processing_state.map(raw_processing_state_to_sql);
let limit = i64::from(filter.limit);
let signatures_are_empty = filter.signatures.is_empty();
let started_at = std::time::Instant::now();
let query_result = sqlx::query(
"SELECT raw.id, raw.signature, raw.slot, raw.block_time, raw.canonical_json, raw.canonical_json_hash, raw.canonical_format_version, raw.retention_state, raw.processing_state, raw.created_at, raw.updated_at FROM k_sol_raw_transactions raw WHERE ($1 OR raw.signature = ANY($2)) AND ($3::BIGINT IS NULL OR raw.slot >= $3) AND ($4::BIGINT IS NULL OR raw.slot <= $4) AND ($5::TEXT IS NULL OR raw.processing_state = $5) AND ($6::TEXT IS NULL OR EXISTS (SELECT 1 FROM k_sol_core_instructions instruction WHERE instruction.signature = raw.signature AND instruction.program_id = $6)) ORDER BY raw.slot ASC, raw.signature ASC LIMIT $7",
)
@@ -37,6 +38,7 @@ pub(crate) async fn list_raw_transactions_for_core_extraction(
let rows = match query_result {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => {
trace_query_failure("list_raw_transactions_for_core_extraction", started_at, &error);
return std::result::Result::Err(ks_core::Error::db(format!(
"postgres core extraction raw selection failed: {error}"
)));
@@ -51,6 +53,7 @@ pub(crate) async fn list_raw_transactions_for_core_extraction(
};
output.push(mapped);
}
trace_query_success("list_raw_transactions_for_core_extraction", started_at, output.len());
return std::result::Result::Ok(output);
}
@@ -58,6 +61,7 @@ pub(crate) async fn is_core_extraction_current(
pool: &sqlx::PgPool,
identity: &crate::ProcessingLedgerIdentity,
) -> ks_core::Result<bool> {
let started_at = std::time::Instant::now();
let query_result = sqlx::query_scalar::<sqlx::Postgres, bool>(
"SELECT EXISTS(SELECT 1 FROM k_sol_ops_processing_ledger WHERE stage = $1 AND processor_name = $2 AND processor_version = $3 AND input_key = $4 AND input_hash = $5 AND status = 'succeeded')",
)
@@ -69,10 +73,20 @@ pub(crate) async fn is_core_extraction_current(
.fetch_one(pool)
.await;
return match query_result {
std::result::Result::Ok(value) => std::result::Result::Ok(value),
std::result::Result::Err(error) => std::result::Result::Err(ks_core::Error::db(format!(
"postgres core extraction ledger lookup failed: {error}"
))),
std::result::Result::Ok(value) => {
trace_query_success(
"is_core_extraction_current",
started_at,
if value { 1_usize } else { 0_usize },
);
std::result::Result::Ok(value)
},
std::result::Result::Err(error) => {
trace_query_failure("is_core_extraction_current", started_at, &error);
std::result::Result::Err(ks_core::Error::db(format!(
"postgres core extraction ledger lookup failed: {error}"
)))
},
};
}
@@ -81,6 +95,7 @@ pub(crate) async fn persist_core_extraction(
bundle: &crate::CoreExtractionBundle,
_force_replay: bool,
) -> ks_core::Result<crate::InsertOutcome> {
let started_at = std::time::Instant::now();
let validation_result = bundle.validate();
if let std::result::Result::Err(error) = validation_result {
return std::result::Result::Err(error);
@@ -109,6 +124,14 @@ pub(crate) async fn persist_core_extraction(
},
};
if existing.is_some() {
let invalidation_result = crate::invalidate_core_descendants_in_transaction(
&mut transaction,
bundle.transaction.signature.as_str(),
)
.await;
if let std::result::Result::Err(error) = invalidation_result {
return std::result::Result::Err(error);
}
let delete_result = sqlx::query("DELETE FROM k_sol_core_transactions WHERE signature = $1")
.bind(bundle.transaction.signature.as_str())
.execute(&mut *transaction)
@@ -202,17 +225,18 @@ pub(crate) async fn persist_core_extraction(
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
return std::result::Result::Ok(crate::InsertOutcome::new(
inserted_count,
if existing.is_some() { 1 } else { 0 },
0,
));
let outcome =
crate::InsertOutcome::new(inserted_count, if existing.is_some() { 1 } else { 0 }, 0);
let elapsed_ms = started_at.elapsed().as_secs_f64() * 1000.0;
tracing::trace!(target: crate::TRACING_TARGET, backend = "postgres", domain = "ks-store.pg", action = "persist_core_extraction", elapsed_ms, rows = inserted_count, replaced = existing.is_some(), outcome = "success", "PostgreSQL store operation completed");
return std::result::Result::Ok(outcome);
}
pub(crate) async fn mark_core_extraction_failed(
pool: &sqlx::PgPool,
failure: &crate::CoreExtractionFailure,
) -> ks_core::Result<crate::InsertOutcome> {
let started_at = std::time::Instant::now();
let transaction_result = pool.begin().await;
let mut transaction = match transaction_result {
std::result::Result::Ok(value) => value,
@@ -251,6 +275,7 @@ pub(crate) async fn mark_core_extraction_failed(
"postgres core extraction failure commit failed: {error}"
)));
}
trace_query_success("mark_core_extraction_failed", started_at, 1);
return std::result::Result::Ok(crate::InsertOutcome::new(0, 1, 0));
}
@@ -803,6 +828,16 @@ fn balance_change_kind_to_sql(value: crate::CoreBalanceChangeKind) -> &'static s
};
}
fn trace_query_success(action: &str, started_at: std::time::Instant, rows: usize) {
let elapsed_ms = started_at.elapsed().as_secs_f64() * 1000.0;
tracing::trace!(target: crate::TRACING_TARGET, backend = "postgres", domain = "ks-store.pg", action = action, elapsed_ms, rows, outcome = "success", "PostgreSQL store operation completed");
}
fn trace_query_failure(action: &str, started_at: std::time::Instant, error: &sqlx::Error) {
let elapsed_ms = started_at.elapsed().as_secs_f64() * 1000.0;
tracing::trace!(target: crate::TRACING_TARGET, backend = "postgres", domain = "ks-store.pg", action = action, elapsed_ms, error = %error, outcome = "error", "PostgreSQL store operation failed");
}
#[cfg(test)]
mod tests {
#[test]

View File

@@ -1,12 +1,22 @@
// file: ks-store/src/postgres/query/core_queries.rs
// version: 9
// version: 11
//! PostgreSQL queries for normalized Solana core storage.
use sqlx::Row; // rust-rules: trait-import
const TOP_LEVEL_INSTRUCTIONS_CONTEXT_SQL: &str = "SELECT COALESCE(jsonb_agg(jsonb_build_object('instructionIndex', instruction_path::BIGINT, 'instructionPath', instruction_path, 'programId', program_id, 'stackHeight', stack_height, 'payloadJson', payload_json, 'payloadHash', payload_json_hash) ORDER BY instruction_path::BIGINT, instruction_path ASC), '[]'::jsonb) FROM k_sol_core_instructions WHERE signature = $1 AND instruction_path ~ '^[0-9]+$'";
const INCOMPLETE_DECODE_INPUTS_SQL: &str = "WITH incomplete AS (SELECT signature, MIN(slot) AS first_slot FROM k_sol_core_instructions WHERE (cardinality($1::text[]) = 0 OR signature = ANY($1)) AND ($3::bigint IS NULL OR slot >= $3) AND ($4::bigint IS NULL OR slot <= $4) AND (cardinality($5::text[]) = 0 OR program_id = ANY($5)) AND (processing_state = ANY($2) OR (processing_state = 'ignored' AND lifecycle_reason = 'unsupported')) GROUP BY signature ORDER BY MIN(slot), signature LIMIT $7) SELECT i.id, i.transaction_id, i.signature, i.slot, i.instruction_path, i.program_id, i.stack_height, i.accounts_json, i.payload_json, i.payload_json_hash, i.processing_state, i.created_at, i.updated_at FROM k_sol_core_instructions i JOIN incomplete s ON s.signature = i.signature WHERE (cardinality($5::text[]) = 0 OR i.program_id = ANY($5)) AND (cardinality($6::text[]) = 0 OR i.instruction_path = ANY($6)) ORDER BY i.slot ASC, i.signature ASC, i.instruction_path ASC";
const CORE_INSTRUCTION_REPLAY_SQL: &str = "WITH instruction_rows AS (SELECT id, transaction_id, signature, slot, 'top_level'::TEXT AS scope_code, 0::INTEGER AS scope_rank, NULL::TEXT AS parent_instruction_path, instruction_path, program_id, stack_height, accounts_json, payload_json, payload_json_hash, processing_state, lifecycle_reason, created_at, updated_at FROM k_sol_core_instructions UNION ALL SELECT id, transaction_id, signature, slot, 'inner'::TEXT AS scope_code, 1::INTEGER AS scope_rank, parent_instruction_path, instruction_path, program_id, stack_height, accounts_json, payload_json, payload_json_hash, processing_state, lifecycle_reason, created_at, updated_at FROM k_sol_core_inner_instructions) SELECT id, transaction_id, signature, slot, scope_code, scope_rank, parent_instruction_path, instruction_path, program_id, stack_height, accounts_json, payload_json, payload_json_hash, processing_state, created_at, updated_at FROM instruction_rows WHERE ($1::TEXT IS NULL OR processing_state = $1) AND ($2::TEXT IS NULL OR program_id = $2) AND ($3::TEXT IS NULL OR scope_code = $3) AND ($4::BIGINT IS NULL OR slot >= $4) AND ($5::BIGINT IS NULL OR slot <= $5) AND ($6::BIGINT IS NULL OR (slot, signature, scope_rank, instruction_path) > ($6, $7, $8, $9)) ORDER BY slot ASC, signature ASC, scope_rank ASC, instruction_path ASC LIMIT $10";
const DECODE_INPUTS_SQL: &str = "WITH instruction_rows AS (SELECT id, transaction_id, signature, slot, 'top_level'::TEXT AS scope_code, 0::INTEGER AS scope_rank, NULL::TEXT AS parent_instruction_path, instruction_path, program_id, stack_height, accounts_json, payload_json, payload_json_hash, processing_state, lifecycle_reason, created_at, updated_at FROM k_sol_core_instructions UNION ALL SELECT id, transaction_id, signature, slot, 'inner'::TEXT AS scope_code, 1::INTEGER AS scope_rank, parent_instruction_path, instruction_path, program_id, stack_height, accounts_json, payload_json, payload_json_hash, processing_state, lifecycle_reason, created_at, updated_at FROM k_sol_core_inner_instructions) SELECT id, transaction_id, signature, slot, scope_code, scope_rank, parent_instruction_path, instruction_path, program_id, stack_height, accounts_json, payload_json, payload_json_hash, processing_state, created_at, updated_at FROM instruction_rows WHERE (cardinality($1::TEXT[]) = 0 OR signature = ANY($1)) AND (cardinality($2::TEXT[]) = 0 OR processing_state = ANY($2)) AND ($3::BIGINT IS NULL OR slot >= $3) AND ($4::BIGINT IS NULL OR slot <= $4) AND (cardinality($5::TEXT[]) = 0 OR program_id = ANY($5)) AND (cardinality($6::TEXT[]) = 0 OR instruction_path = ANY($6)) ORDER BY slot ASC, signature ASC, scope_rank ASC, instruction_path ASC LIMIT $7";
const INCOMPLETE_DECODE_INPUTS_SQL: &str = "WITH instruction_rows AS (SELECT id, transaction_id, signature, slot, 'top_level'::TEXT AS scope_code, 0::INTEGER AS scope_rank, NULL::TEXT AS parent_instruction_path, instruction_path, program_id, stack_height, accounts_json, payload_json, payload_json_hash, processing_state, lifecycle_reason, created_at, updated_at FROM k_sol_core_instructions UNION ALL SELECT id, transaction_id, signature, slot, 'inner'::TEXT AS scope_code, 1::INTEGER AS scope_rank, parent_instruction_path, instruction_path, program_id, stack_height, accounts_json, payload_json, payload_json_hash, processing_state, lifecycle_reason, created_at, updated_at FROM k_sol_core_inner_instructions), incomplete AS (SELECT signature, MIN(slot) AS first_slot FROM instruction_rows WHERE (cardinality($1::TEXT[]) = 0 OR signature = ANY($1)) AND ($3::BIGINT IS NULL OR slot >= $3) AND ($4::BIGINT IS NULL OR slot <= $4) AND (cardinality($5::TEXT[]) = 0 OR program_id = ANY($5)) AND (processing_state = ANY($2) OR (processing_state = 'ignored' AND lifecycle_reason = 'unsupported')) GROUP BY signature ORDER BY MIN(slot), signature LIMIT $7) SELECT i.id, i.transaction_id, i.signature, i.slot, i.scope_code, i.scope_rank, i.parent_instruction_path, i.instruction_path, i.program_id, i.stack_height, i.accounts_json, i.payload_json, i.payload_json_hash, i.processing_state, i.created_at, i.updated_at FROM instruction_rows i JOIN incomplete s ON s.signature = i.signature WHERE (cardinality($5::TEXT[]) = 0 OR i.program_id = ANY($5)) AND (cardinality($6::TEXT[]) = 0 OR i.instruction_path = ANY($6)) ORDER BY i.slot ASC, i.signature ASC, i.scope_rank ASC, i.instruction_path ASC";
#[derive(Clone, Debug, serde::Deserialize, serde::Serialize)]
struct CoreInstructionPageCursor {
slot: i64,
signature: std::string::String,
scope_rank: i32,
instruction_path: std::string::String,
}
pub(crate) async fn insert_core_transaction(
pool: &sqlx::PgPool,
@@ -279,7 +289,7 @@ pub(crate) async fn list_core_instructions_for_replay(
pool: &sqlx::PgPool,
filter: &crate::CoreInstructionReplayFilter,
page_request: &crate::PageRequest,
) -> ks_core::Result<std::vec::Vec<crate::CoreInstructionRow>> {
) -> ks_core::Result<crate::PageSlice<crate::CoreInstructionReplayRow>> {
let min_slot_result = optional_sql_slot_from_u64(filter.min_slot);
let min_slot = match min_slot_result {
std::result::Result::Ok(value) => value,
@@ -290,63 +300,84 @@ pub(crate) async fn list_core_instructions_for_replay(
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let offset_result =
sql_i64_from_u64(page_request.offset, "page offset does not fit into SQL BIGINT");
let offset = match offset_result {
let cursor_result = decode_core_instruction_cursor(page_request.cursor.as_deref());
let cursor = match cursor_result {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let processing_state = filter.processing_state.map(instruction_processing_state_to_sql);
let query_result = sqlx::query(
"SELECT id, transaction_id, signature, slot, instruction_path, program_id, stack_height, accounts_json, payload_json, payload_json_hash, processing_state, created_at, updated_at FROM k_sol_core_instructions WHERE ($1::TEXT IS NULL OR processing_state = $1) AND ($2::TEXT IS NULL OR program_id = $2) AND ($3::BIGINT IS NULL OR slot >= $3) AND ($4::BIGINT IS NULL OR slot <= $4) ORDER BY slot ASC, id ASC LIMIT $5 OFFSET $6",
)
.bind(processing_state)
.bind(filter.program_id.as_deref())
.bind(min_slot)
.bind(max_slot)
.bind(i64::from(page_request.limit))
.bind(offset)
.fetch_all(pool)
.await;
let scope_code = filter.scope.map(crate::CoreInstructionScope::as_code);
let query_limit = i64::from(page_request.limit) + 1;
let started_at = std::time::Instant::now();
let query_result = sqlx::query(CORE_INSTRUCTION_REPLAY_SQL)
.bind(processing_state)
.bind(filter.program_id.as_deref())
.bind(scope_code)
.bind(min_slot)
.bind(max_slot)
.bind(cursor.as_ref().map(|value| return value.slot))
.bind(cursor.as_ref().map(|value| return value.signature.as_str()))
.bind(cursor.as_ref().map(|value| return value.scope_rank))
.bind(cursor.as_ref().map(|value| return value.instruction_path.as_str()))
.bind(query_limit)
.fetch_all(pool)
.await;
let rows = match query_result {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => {
trace_query_failure("list_core_instruction_page", started_at, &error);
return std::result::Result::Err(ks_core::Error::db(format!(
"postgres core instruction replay query failed: {error}"
)));
},
};
let mut output = std::vec::Vec::with_capacity(rows.len());
for row in rows {
let mapped_result = row_to_core_instruction_row(&row);
let mut output = std::vec::Vec::with_capacity(rows.len().min(usize::from(page_request.limit)));
for row in rows.iter().take(usize::from(page_request.limit)) {
let mapped_result = row_to_core_instruction_replay_row(row);
match mapped_result {
std::result::Result::Ok(value) => output.push(value),
std::result::Result::Err(error) => return std::result::Result::Err(error),
}
}
return std::result::Result::Ok(output);
let next_cursor = if rows.len() > usize::from(page_request.limit) {
let last = output.last();
match last {
std::option::Option::Some(value) => {
let cursor_result = encode_core_instruction_cursor(value);
match cursor_result {
std::result::Result::Ok(encoded) => std::option::Option::Some(encoded),
std::result::Result::Err(error) => return std::result::Result::Err(error),
}
},
std::option::Option::None => std::option::Option::None,
}
} else {
std::option::Option::None
};
trace_query_success("list_core_instruction_page", started_at, output.len());
return std::result::Result::Ok(crate::PageSlice::new(output, next_cursor));
}
pub(crate) async fn list_core_instruction_replay_inputs(
pool: &sqlx::PgPool,
filter: &crate::CoreInstructionReplayFilter,
page_request: &crate::PageRequest,
) -> ks_core::Result<std::vec::Vec<crate::MdCoreInstructionReplayInput>> {
) -> ks_core::Result<crate::PageSlice<crate::MdCoreInstructionReplayInput>> {
let instructions_result =
crate::list_core_instructions_for_replay(pool, filter, page_request).await;
let instructions = match instructions_result {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let mut output = std::vec::Vec::with_capacity(instructions.len());
for instruction in instructions {
let input_result = load_replay_input_for_instruction(pool, &instruction).await;
let mut output = std::vec::Vec::with_capacity(instructions.rows.len());
for instruction in &instructions.rows {
let input_result = load_replay_input_for_instruction(pool, instruction).await;
match input_result {
std::result::Result::Ok(value) => output.push(value),
std::result::Result::Err(error) => return std::result::Result::Err(error),
}
}
return std::result::Result::Ok(output);
return std::result::Result::Ok(crate::PageSlice::new(output, instructions.next_cursor));
}
pub(crate) async fn list_decode_replay_inputs(
@@ -367,10 +398,9 @@ pub(crate) async fn list_decode_replay_inputs(
let processing_states = filter
.processing_states
.iter()
.map(|state| {
return instruction_processing_state_to_sql(*state).to_string();
})
.map(|state| return instruction_processing_state_to_sql(*state).to_string())
.collect::<std::vec::Vec<_>>();
let started_at = std::time::Instant::now();
let rows_result = if filter.incomplete_signatures {
sqlx::query(INCOMPLETE_DECODE_INPUTS_SQL)
.bind(&filter.signatures)
@@ -383,22 +413,21 @@ pub(crate) async fn list_decode_replay_inputs(
.fetch_all(pool)
.await
} else {
sqlx::query(
"SELECT id, transaction_id, signature, slot, instruction_path, program_id, stack_height, accounts_json, payload_json, payload_json_hash, processing_state, created_at, updated_at FROM k_sol_core_instructions WHERE (cardinality($1::text[]) = 0 OR signature = ANY($1)) AND (cardinality($2::text[]) = 0 OR processing_state = ANY($2)) AND ($3::bigint IS NULL OR slot >= $3) AND ($4::bigint IS NULL OR slot <= $4) AND (cardinality($5::text[]) = 0 OR program_id = ANY($5)) AND (cardinality($6::text[]) = 0 OR instruction_path = ANY($6)) ORDER BY slot ASC, signature ASC, instruction_path ASC LIMIT $7",
)
.bind(&filter.signatures)
.bind(&processing_states)
.bind(min_slot)
.bind(max_slot)
.bind(&filter.program_ids)
.bind(&filter.instruction_paths)
.bind(limit)
.fetch_all(pool)
.await
sqlx::query(DECODE_INPUTS_SQL)
.bind(&filter.signatures)
.bind(&processing_states)
.bind(min_slot)
.bind(max_slot)
.bind(&filter.program_ids)
.bind(&filter.instruction_paths)
.bind(limit)
.fetch_all(pool)
.await
};
let rows = match rows_result {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => {
trace_query_failure("list_decode_replay_inputs", started_at, &error);
return std::result::Result::Err(ks_core::Error::db(format!(
"postgres decode replay input selection failed: {error}"
)));
@@ -406,7 +435,7 @@ pub(crate) async fn list_decode_replay_inputs(
};
let mut output = std::vec::Vec::with_capacity(rows.len());
for row in rows {
let instruction_result = row_to_core_instruction_row(&row);
let instruction_result = row_to_core_instruction_replay_row(&row);
let instruction = match instruction_result {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
@@ -417,6 +446,7 @@ pub(crate) async fn list_decode_replay_inputs(
std::result::Result::Err(error) => return std::result::Result::Err(error),
}
}
trace_query_success("list_decode_replay_inputs", started_at, output.len());
return std::result::Result::Ok(output);
}
@@ -425,21 +455,48 @@ pub(crate) async fn update_core_instruction_lifecycle(
mark: &crate::CoreInstructionLifecycleMark,
) -> ks_core::Result<crate::InsertOutcome> {
let processing_state = instruction_processing_state_to_sql(mark.processing_state);
let query_result = sqlx::query(
"UPDATE k_sol_core_instructions SET processing_state = $1, processor_name = $2, processor_version = $3, lifecycle_reason = $4, updated_at = NOW() WHERE signature = $5 AND instruction_path = $6",
)
.bind(processing_state)
.bind(mark.processor_name.as_deref())
.bind(mark.processor_version.as_deref())
.bind(mark.reason.as_deref())
.bind(mark.signature.as_str())
.bind(mark.instruction_path.as_str())
.execute(pool)
.await;
return outcome_from_update_result(
let scope_result =
crate::CoreInstructionScope::from_instruction_path(mark.instruction_path.as_str());
let scope = match scope_result {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let sql = match scope {
crate::CoreInstructionScope::TopLevel => {
"UPDATE k_sol_core_instructions SET processing_state = $1, processor_name = $2, processor_version = $3, lifecycle_reason = $4, updated_at = NOW() WHERE signature = $5 AND instruction_path = $6"
},
crate::CoreInstructionScope::Inner => {
"UPDATE k_sol_core_inner_instructions SET processing_state = $1, processor_name = $2, processor_version = $3, lifecycle_reason = $4, updated_at = NOW() WHERE signature = $5 AND instruction_path = $6"
},
};
let started_at = std::time::Instant::now();
let query_result = sqlx::query(sql)
.bind(processing_state)
.bind(mark.processor_name.as_deref())
.bind(mark.processor_version.as_deref())
.bind(mark.reason.as_deref())
.bind(mark.signature.as_str())
.bind(mark.instruction_path.as_str())
.execute(pool)
.await;
let outcome_result = outcome_from_update_result(
query_result,
"postgres core instruction lifecycle update failed",
);
return match outcome_result {
std::result::Result::Ok(outcome) => {
trace_query_success(
"update_core_instruction_lifecycle",
started_at,
match usize::try_from(outcome.updated_count) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => usize::MAX,
},
);
std::result::Result::Ok(outcome)
},
std::result::Result::Err(error) => std::result::Result::Err(error),
};
}
fn instruction_processing_state_to_sql(
@@ -608,21 +665,24 @@ fn outcome_from_update_result(
};
}
fn row_to_core_instruction_row(
fn row_to_core_instruction_replay_row(
row: &sqlx::postgres::PgRow,
) -> ks_core::Result<crate::CoreInstructionRow> {
let state_text_result: std::result::Result<std::string::String, sqlx::Error> =
row.try_get("processing_state");
let state_text = match state_text_result {
) -> ks_core::Result<crate::CoreInstructionReplayRow> {
let state_text: std::string::String = match read_row_value(row, "processing_state") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => {
return std::result::Result::Err(ks_core::Error::db(format!(
"postgres core instruction processing state read failed: {error}"
)));
},
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let state_result = instruction_processing_state_from_sql(state_text.as_str());
let processing_state = match state_result {
let processing_state_result = instruction_processing_state_from_sql(state_text.as_str());
let processing_state = match processing_state_result {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let scope_text: std::string::String = match read_row_value(row, "scope_code") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let scope_result = crate::CoreInstructionScope::from_code(scope_text.as_str());
let scope = match scope_result {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
@@ -642,6 +702,10 @@ fn row_to_core_instruction_row(
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let parent_instruction_path = match read_row_value(row, "parent_instruction_path") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let instruction_path = match read_row_value(row, "instruction_path") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
@@ -674,11 +738,13 @@ fn row_to_core_instruction_row(
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
return std::result::Result::Ok(crate::CoreInstructionRow {
return std::result::Result::Ok(crate::CoreInstructionReplayRow {
id,
transaction_id,
signature,
slot,
scope,
parent_instruction_path,
instruction_path,
program_id,
stack_height,
@@ -706,7 +772,7 @@ where
async fn load_replay_input_for_instruction(
pool: &sqlx::PgPool,
instruction: &crate::CoreInstructionRow,
instruction: &crate::CoreInstructionReplayRow,
) -> ks_core::Result<crate::MdCoreInstructionReplayInput> {
let transaction_context_result =
sqlx::query_as::<sqlx::Postgres, (bool, std::option::Option<serde_json::Value>, std::option::Option<i64>)>(
@@ -798,7 +864,7 @@ async fn load_replay_input_for_instruction(
};
return input.with_core_context(
transaction_context.2,
std::option::Option::None,
instruction.parent_instruction_path.clone(),
stack_height,
return_data,
);
@@ -922,8 +988,120 @@ async fn load_json_aggregate(
};
}
fn decode_core_instruction_cursor(
cursor: std::option::Option<&str>,
) -> ks_core::Result<std::option::Option<CoreInstructionPageCursor>> {
let raw = match cursor {
std::option::Option::Some(value) => value,
std::option::Option::None => return std::result::Result::Ok(std::option::Option::None),
};
let decode_result = serde_json::from_str::<CoreInstructionPageCursor>(raw);
let decoded = match decode_result {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_error) => {
return std::result::Result::Err(ks_core::Error::db(
"core instruction page cursor is invalid",
));
},
};
if decoded.slot < 0
|| decoded.signature.trim().is_empty()
|| decoded.instruction_path.trim().is_empty()
|| !matches!(decoded.scope_rank, 0 | 1)
{
return std::result::Result::Err(ks_core::Error::db(
"core instruction page cursor values are invalid",
));
}
return std::result::Result::Ok(std::option::Option::Some(decoded));
}
fn encode_core_instruction_cursor(
row: &crate::CoreInstructionReplayRow,
) -> ks_core::Result<std::string::String> {
let scope_rank = match row.scope {
crate::CoreInstructionScope::TopLevel => 0,
crate::CoreInstructionScope::Inner => 1,
};
let cursor = CoreInstructionPageCursor {
slot: row.slot,
signature: row.signature.clone(),
scope_rank,
instruction_path: row.instruction_path.clone(),
};
let encode_result = serde_json::to_string(&cursor);
return match encode_result {
std::result::Result::Ok(value) if value.len() <= crate::MAX_PAGE_CURSOR_LENGTH => {
std::result::Result::Ok(value)
},
std::result::Result::Ok(_value) => std::result::Result::Err(ks_core::Error::db(
"core instruction page cursor exceeds maximum encoded length",
)),
std::result::Result::Err(error) => std::result::Result::Err(ks_core::Error::db(format!(
"core instruction page cursor serialization failed: {error}"
))),
};
}
fn trace_query_success(action: &str, started_at: std::time::Instant, rows: usize) {
let elapsed_ms = started_at.elapsed().as_secs_f64() * 1000.0;
tracing::trace!(target: crate::TRACING_TARGET, backend = "postgres", domain = "ks-store.pg", action = action, elapsed_ms, rows, outcome = "success", "PostgreSQL store operation completed");
}
fn trace_query_failure(action: &str, started_at: std::time::Instant, error: &sqlx::Error) {
let elapsed_ms = started_at.elapsed().as_secs_f64() * 1000.0;
tracing::trace!(target: crate::TRACING_TARGET, backend = "postgres", domain = "ks-store.pg", action = action, elapsed_ms, error = %error, outcome = "error", "PostgreSQL store operation failed");
}
#[cfg(test)]
mod tests {
#[test]
fn core_instruction_replay_query_is_union_based_and_offset_free() {
assert!(super::CORE_INSTRUCTION_REPLAY_SQL.contains("k_sol_core_instructions"));
assert!(super::CORE_INSTRUCTION_REPLAY_SQL.contains("k_sol_core_inner_instructions"));
assert!(super::CORE_INSTRUCTION_REPLAY_SQL.contains("scope_code"));
assert!(!super::CORE_INSTRUCTION_REPLAY_SQL.to_ascii_uppercase().contains(" OFFSET "));
}
#[test]
fn core_instruction_cursor_round_trips_top_level_and_inner_order_fields() {
let row = crate::CoreInstructionReplayRow {
id: 1,
transaction_id: 2,
signature: "signature".to_string(),
slot: 3,
scope: crate::CoreInstructionScope::Inner,
parent_instruction_path: std::option::Option::Some("0".to_string()),
instruction_path: "0/1".to_string(),
program_id: "program".to_string(),
stack_height: std::option::Option::Some(2),
accounts_json: serde_json::json!([]),
payload_json: std::option::Option::None,
payload_json_hash: std::option::Option::None,
processing_state: crate::CoreInstructionProcessingState::Pending,
created_at: chrono::Utc::now(),
updated_at: chrono::Utc::now(),
};
let encoded_result = super::encode_core_instruction_cursor(&row);
let encoded = match encoded_result {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => panic!("unexpected cursor encode error: {error}"),
};
let decoded_result =
super::decode_core_instruction_cursor(std::option::Option::Some(encoded.as_str()));
let decoded = match decoded_result {
std::result::Result::Ok(std::option::Option::Some(value)) => value,
std::result::Result::Ok(std::option::Option::None) => {
panic!("cursor unexpectedly absent")
},
std::result::Result::Err(error) => panic!("unexpected cursor decode error: {error}"),
};
assert_eq!(decoded.slot, 3);
assert_eq!(decoded.signature, "signature");
assert_eq!(decoded.scope_rank, 1);
assert_eq!(decoded.instruction_path, "0/1");
}
#[test]
fn incomplete_decode_input_query_projects_every_core_instruction_row_column() {
for column in [
@@ -931,6 +1109,8 @@ mod tests {
"i.transaction_id",
"i.signature",
"i.slot",
"i.scope_code",
"i.parent_instruction_path",
"i.instruction_path",
"i.program_id",
"i.stack_height",
@@ -1061,7 +1241,7 @@ mod tests {
panic!("unexpected replay input query error: {error}")
},
};
let target = replay_inputs.iter().find(|input| {
let target = replay_inputs.rows.iter().find(|input| {
return input.signature == signature && input.instruction_path == "2";
});
let target = match target {

View File

@@ -1,5 +1,5 @@
// file: ks-store/src/postgres/query/decode_pipeline_queries.rs
// version: 11
// version: 12
//! PostgreSQL queries for contextual decode, coverage and materialization persistence.
@@ -26,36 +26,20 @@ pub(crate) async fn list_decode_inputs(
pool: &sqlx::PgPool,
filter: &crate::DecodeSelectionFilter,
) -> ks_core::Result<std::vec::Vec<crate::MdCoreInstructionReplayInput>> {
tracing::debug!(target: crate::TRACING_TARGET, backend = "postgres", domain = "ks-store.pg", action = "list_decode_inputs", signature_count = filter.signatures.len(), signature_sample = ?filter.signatures.iter().take(5).map(std::string::String::as_str).collect::<std::vec::Vec<_>>(), processing_states = ?filter.processing_states, min_slot = ?filter.min_slot, max_slot = ?filter.max_slot, program_ids = ?filter.program_ids, instruction_paths = ?filter.instruction_paths, limit = filter.limit, "query PostgreSQL contextual decode inputs");
let result = crate::list_decode_replay_inputs(pool, filter).await;
return match result {
std::result::Result::Ok(inputs) => {
let selected_input_keys = inputs
.iter()
.take(10)
.map(|input| return input.replay_input_key.as_str())
.collect::<std::vec::Vec<_>>();
tracing::debug!(target: crate::TRACING_TARGET, backend = "postgres", domain = "ks-store.pg", action = "list_decode_inputs", selected_count = inputs.len(), input_key_sample = ?selected_input_keys, "PostgreSQL contextual decode inputs selected");
std::result::Result::Ok(inputs)
},
std::result::Result::Err(error) => {
tracing::error!(target: crate::TRACING_TARGET, backend = "postgres", domain = "ks-store.pg", action = "list_decode_inputs", error = %error, "PostgreSQL contextual decode input query failed");
std::result::Result::Err(error)
},
};
return crate::list_decode_replay_inputs(pool, filter).await;
}
pub(crate) async fn list_materialized_outputs(
pool: &sqlx::PgPool,
filter: &crate::MaterializedOutputFilter,
) -> ks_core::Result<std::vec::Vec<crate::MaterializedOutputQueryRow>> {
tracing::debug!(target: crate::TRACING_TARGET, backend = "postgres", domain = "ks-store.pg", action = "list_materialized_outputs", processor_name = ?filter.processor_name, materialized_family = ?filter.materialized_family, signature_contains = ?filter.signature_contains, limit = filter.limit, "query bounded PostgreSQL materialized outputs");
if filter.limit == 0 || filter.limit > crate::MAX_MATERIALIZED_OUTPUT_QUERY_ROWS {
return std::result::Result::Err(ks_core::Error::db(format!(
"materialized output query limit must be between 1 and {}",
crate::MAX_MATERIALIZED_OUTPUT_QUERY_ROWS
)));
}
let started_at = std::time::Instant::now();
let query_result = sqlx::query_as::<sqlx::Postgres, MaterializedOutputDatabaseRow>(
"SELECT processor_name, processor_version, input_key, output_key, source_event_key, source_decoder_name, source_decoder_version, signature, slot, materialized_family, payload_jsonb AS payload_json, created_at::text AS created_at, updated_at::text AS updated_at FROM k_sol_mat_outputs WHERE ($1::text IS NULL OR processor_name = $1) AND ($2::text IS NULL OR materialized_family = $2) AND ($3::text IS NULL OR POSITION(LOWER($3) IN LOWER(signature)) > 0) ORDER BY slot DESC, id DESC LIMIT $4",
)
@@ -68,6 +52,7 @@ pub(crate) async fn list_materialized_outputs(
let rows = match query_result {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => {
trace_query_failure("list_materialized_outputs", started_at, &error);
return std::result::Result::Err(ks_core::Error::db(format!(
"postgres materialized output query failed: {error}"
)));
@@ -100,7 +85,7 @@ pub(crate) async fn list_materialized_outputs(
updated_at: row.updated_at,
});
}
tracing::debug!(target: crate::TRACING_TARGET, backend = "postgres", domain = "ks-store.pg", action = "list_materialized_outputs", row_count = output.len(), "bounded PostgreSQL materialized outputs loaded");
trace_query_success("list_materialized_outputs", started_at, output.len());
return std::result::Result::Ok(output);
}
@@ -108,7 +93,7 @@ pub(crate) async fn is_decode_current(
pool: &sqlx::PgPool,
identity: &crate::ProcessingLedgerIdentity,
) -> ks_core::Result<bool> {
tracing::debug!(target: crate::TRACING_TARGET, backend = "postgres", domain = "ks-store.pg", action = "is_decode_current", stage = %identity.stage, processor_name = %identity.processor_name, processor_version = %identity.processor_version, input_key = %identity.input_key, input_hash = %identity.input_hash, "query PostgreSQL processing ledger current state");
let started_at = std::time::Instant::now();
let query_result = sqlx::query_scalar::<sqlx::Postgres, bool>(
"SELECT EXISTS(SELECT 1 FROM k_sol_ops_processing_ledger WHERE stage = $1 AND processor_name = $2 AND processor_version = $3 AND input_key = $4 AND input_hash = $5 AND status = 'succeeded')",
)
@@ -121,11 +106,15 @@ pub(crate) async fn is_decode_current(
.await;
return match query_result {
std::result::Result::Ok(value) => {
tracing::debug!(target: crate::TRACING_TARGET, backend = "postgres", domain = "ks-store.pg", action = "is_decode_current", stage = %identity.stage, processor_name = %identity.processor_name, processor_version = %identity.processor_version, input_key = %identity.input_key, input_hash = %identity.input_hash, current = value, "PostgreSQL processing ledger current state loaded");
trace_query_success(
"is_decode_current",
started_at,
if value { 1_usize } else { 0_usize },
);
std::result::Result::Ok(value)
},
std::result::Result::Err(error) => {
tracing::error!(target: crate::TRACING_TARGET, backend = "postgres", domain = "ks-store.pg", action = "is_decode_current", stage = %identity.stage, processor_name = %identity.processor_name, processor_version = %identity.processor_version, input_key = %identity.input_key, error = %error, "PostgreSQL processing ledger current check failed");
trace_query_failure("is_decode_current", started_at, &error);
std::result::Result::Err(ks_core::Error::db(format!(
"postgres decode ledger current check failed: {error}"
)))
@@ -138,16 +127,11 @@ pub(crate) async fn persist_decode_coverage_declarations(
declarations: &[crate::DecodeCoverageDeclarationInsert],
) -> ks_core::Result<crate::InsertOutcome> {
if declarations.is_empty() {
tracing::debug!(target: crate::TRACING_TARGET, backend = "postgres", domain = "ks-store.pg", action = "persist_coverage_declarations", declaration_count = 0_usize, "skip empty PostgreSQL decode coverage declarations");
return std::result::Result::Ok(crate::InsertOutcome::new(0, 0, 0));
}
let started_at = std::time::Instant::now();
let processor_name = declarations[0].processor_name.as_str();
let processor_version = declarations[0].processor_version.as_str();
let program_ids = declarations
.iter()
.map(|entry| return entry.program_id.as_str())
.collect::<std::vec::Vec<_>>();
tracing::debug!(target: crate::TRACING_TARGET, backend = "postgres", domain = "ks-store.pg", action = "persist_coverage_declarations", processor_name = %processor_name, processor_version = %processor_version, declaration_count = declarations.len(), program_ids = ?program_ids, "persist PostgreSQL decode coverage declarations");
if declarations.iter().any(|entry| {
return entry.processor_name != processor_name
|| entry.processor_version != processor_version
@@ -306,16 +290,21 @@ pub(crate) async fn persist_decode_coverage_declarations(
)));
}
let outcome = crate::InsertOutcome::new(inserted_count, updated_count, skipped_count);
tracing::debug!(target: crate::TRACING_TARGET, backend = "postgres", domain = "ks-store.pg", action = "persist_coverage_declarations", processor_name = %processor_name, processor_version = %processor_version, outcome = ?outcome, "PostgreSQL decode coverage declarations persisted");
let row_count = inserted_count.saturating_add(updated_count).saturating_add(skipped_count);
let trace_rows = match usize::try_from(row_count) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_error) => usize::MAX,
};
trace_query_success("persist_coverage_declarations", started_at, trace_rows);
return std::result::Result::Ok(outcome);
}
pub(crate) async fn persist_decode_result(
pool: &sqlx::PgPool,
bundle: &crate::DecodePersistenceBundle,
force_replay: bool,
_force_replay: bool,
) -> ks_core::Result<crate::InsertOutcome> {
tracing::debug!(target: crate::TRACING_TARGET, backend = "postgres", domain = "ks-store.pg", action = "persist_decode_result", signature = %bundle.signature, instruction_path = %bundle.instruction_path, stage = %bundle.ledger_identity.stage, processor_name = %bundle.ledger_identity.processor_name, processor_version = %bundle.ledger_identity.processor_version, input_key = %bundle.ledger_identity.input_key, input_hash = %bundle.ledger_identity.input_hash, status = %bundle.status, observation_count = bundle.observations.len(), coverage_program_id = %bundle.coverage.program_id, force_replay, "persist PostgreSQL contextual decode result");
let started_at = std::time::Instant::now();
let validation_result = bundle.validate();
if let std::result::Result::Err(error) = validation_result {
return std::result::Result::Err(error);
@@ -329,6 +318,16 @@ pub(crate) async fn persist_decode_result(
)));
},
};
let invalidation_result = crate::invalidate_decode_descendants_in_transaction(
&mut transaction,
bundle.ledger_identity.processor_name.as_str(),
bundle.ledger_identity.processor_version.as_str(),
bundle.ledger_identity.input_key.as_str(),
)
.await;
if let std::result::Result::Err(error) = invalidation_result {
return std::result::Result::Err(error);
}
let delete_events_result = sqlx::query(
"DELETE FROM k_sol_decode_events WHERE processor_name = $1 AND processor_version = $2 AND input_key = $3",
)
@@ -438,27 +437,14 @@ pub(crate) async fn persist_decode_result(
if let std::result::Result::Err(error) = materialized_count_result {
return std::result::Result::Err(error);
}
let lifecycle_state = match bundle.status.as_str() {
"decoded" => "decoded",
"ignored" | "unsupported" => "ignored",
"failed" => "failed",
_unknown => "decoded",
};
let lifecycle_result = sqlx::query(
"UPDATE k_sol_core_instructions SET processing_state = CASE WHEN $1 = 'decoded' AND processing_state <> 'materialized' THEN 'decoded' WHEN $1 IN ('ignored', 'failed') AND processing_state IN ('pending', 'failed', 'replay_requested', 'ignored') THEN $1 ELSE processing_state END, processor_name = CASE WHEN $1 = 'decoded' AND processing_state <> 'materialized' THEN $2 WHEN $1 IN ('ignored', 'failed') AND processing_state IN ('pending', 'failed', 'replay_requested', 'ignored') THEN $2 ELSE processor_name END, processor_version = CASE WHEN $1 = 'decoded' AND processing_state <> 'materialized' THEN $3 WHEN $1 IN ('ignored', 'failed') AND processing_state IN ('pending', 'failed', 'replay_requested', 'ignored') THEN $3 ELSE processor_version END, lifecycle_reason = CASE WHEN $1 = 'decoded' AND processing_state <> 'materialized' THEN $4 WHEN $1 IN ('ignored', 'failed') AND processing_state IN ('pending', 'failed', 'replay_requested', 'ignored') THEN $4 ELSE lifecycle_reason END, updated_at = NOW() WHERE signature = $5 AND instruction_path = $6",
let lifecycle_result = crate::recompute_instruction_lifecycle_in_transaction(
&mut transaction,
bundle.signature.as_str(),
bundle.instruction_path.as_str(),
)
.bind(lifecycle_state)
.bind(bundle.ledger_identity.processor_name.as_str())
.bind(bundle.ledger_identity.processor_version.as_str())
.bind(bundle.status.as_str())
.bind(bundle.signature.as_str())
.bind(bundle.instruction_path.as_str())
.execute(&mut *transaction)
.await;
if let std::result::Result::Err(error) = lifecycle_result {
return std::result::Result::Err(ks_core::Error::db(format!(
"postgres decode instruction lifecycle update failed: {error}"
)));
return std::result::Result::Err(error);
}
let ledger_status = if bundle.status == "failed" { "failed" } else { "succeeded" };
let ledger_result = crate::postgres::query::decode_pipeline_queries::upsert_ledger_terminal(
@@ -488,7 +474,8 @@ pub(crate) async fn persist_decode_result(
},
};
let outcome = crate::InsertOutcome::new(count, 1, 0);
tracing::debug!(target: crate::TRACING_TARGET, backend = "postgres", domain = "ks-store.pg", action = "persist_decode_result", signature = %bundle.signature, instruction_path = %bundle.instruction_path, processor_name = %bundle.ledger_identity.processor_name, processor_version = %bundle.ledger_identity.processor_version, status = %bundle.status, outcome = ?outcome, committed = true, "PostgreSQL contextual decode result persisted");
let elapsed_ms = started_at.elapsed().as_secs_f64() * 1000.0;
tracing::trace!(target: crate::TRACING_TARGET, backend = "postgres", domain = "ks-store.pg", action = "persist_decode_result", elapsed_ms, rows = count, outcome = "success", "PostgreSQL store operation completed");
return std::result::Result::Ok(outcome);
}
@@ -496,7 +483,7 @@ pub(crate) async fn mark_decode_failed(
pool: &sqlx::PgPool,
failure: &crate::DecodeFailure,
) -> ks_core::Result<crate::InsertOutcome> {
tracing::error!(target: crate::TRACING_TARGET, backend = "postgres", domain = "ks-store.pg", action = "mark_decode_failed", signature = %failure.signature, instruction_path = %failure.instruction_path, processor_name = %failure.ledger_identity.processor_name, processor_version = %failure.ledger_identity.processor_version, input_key = %failure.ledger_identity.input_key, input_hash = %failure.ledger_identity.input_hash, error_code = %failure.error_code, error_message = %failure.error_message, "persist PostgreSQL contextual decode failure");
let started_at = std::time::Instant::now();
if failure.ledger_identity.stage != "instruction_decode"
|| failure.signature.trim().is_empty()
|| failure.instruction_path.trim().is_empty()
@@ -514,20 +501,31 @@ pub(crate) async fn mark_decode_failed(
)));
},
};
let lifecycle_result = sqlx::query(
"UPDATE k_sol_core_instructions SET processing_state = CASE WHEN processing_state IN ('pending', 'failed', 'replay_requested', 'ignored') THEN 'failed' ELSE processing_state END, processor_name = CASE WHEN processing_state IN ('pending', 'failed', 'replay_requested', 'ignored') THEN $1 ELSE processor_name END, processor_version = CASE WHEN processing_state IN ('pending', 'failed', 'replay_requested', 'ignored') THEN $2 ELSE processor_version END, lifecycle_reason = CASE WHEN processing_state IN ('pending', 'failed', 'replay_requested', 'ignored') THEN $3 ELSE lifecycle_reason END, updated_at = NOW() WHERE signature = $4 AND instruction_path = $5",
let invalidation_result = crate::invalidate_decode_descendants_in_transaction(
&mut transaction,
failure.ledger_identity.processor_name.as_str(),
failure.ledger_identity.processor_version.as_str(),
failure.ledger_identity.input_key.as_str(),
)
.bind(failure.ledger_identity.processor_name.as_str())
.bind(failure.ledger_identity.processor_version.as_str())
.bind(failure.error_code.as_str())
.bind(failure.signature.as_str())
.bind(failure.instruction_path.as_str())
.execute(&mut *transaction)
.await;
if let std::result::Result::Err(error) = lifecycle_result {
return std::result::Result::Err(ks_core::Error::db(format!(
"postgres decode failure lifecycle update failed: {error}"
)));
if let std::result::Result::Err(error) = invalidation_result {
return std::result::Result::Err(error);
}
for sql in [
"DELETE FROM k_sol_decode_events WHERE processor_name = $1 AND processor_version = $2 AND input_key = $3",
"DELETE FROM k_sol_decode_coverage_observations WHERE processor_name = $1 AND processor_version = $2 AND input_key = $3",
] {
let delete_result = sqlx::query(sql)
.bind(failure.ledger_identity.processor_name.as_str())
.bind(failure.ledger_identity.processor_version.as_str())
.bind(failure.ledger_identity.input_key.as_str())
.execute(&mut *transaction)
.await;
if let std::result::Result::Err(error) = delete_result {
return std::result::Result::Err(ks_core::Error::db(format!(
"postgres failed decode replacement cleanup failed: {error}"
)));
}
}
let ledger_result = crate::postgres::query::decode_pipeline_queries::upsert_ledger_terminal(
&mut transaction,
@@ -540,6 +538,15 @@ pub(crate) async fn mark_decode_failed(
if let std::result::Result::Err(error) = ledger_result {
return std::result::Result::Err(error);
}
let lifecycle_result = crate::recompute_instruction_lifecycle_in_transaction(
&mut transaction,
failure.signature.as_str(),
failure.instruction_path.as_str(),
)
.await;
if let std::result::Result::Err(error) = lifecycle_result {
return std::result::Result::Err(error);
}
let commit_result = transaction.commit().await;
if let std::result::Result::Err(error) = commit_result {
return std::result::Result::Err(ks_core::Error::db(format!(
@@ -547,33 +554,16 @@ pub(crate) async fn mark_decode_failed(
)));
}
let outcome = crate::InsertOutcome::new(0, 1, 0);
tracing::debug!(target: crate::TRACING_TARGET, backend = "postgres", domain = "ks-store.pg", action = "mark_decode_failed", signature = %failure.signature, instruction_path = %failure.instruction_path, processor_name = %failure.ledger_identity.processor_name, processor_version = %failure.ledger_identity.processor_version, outcome = ?outcome, committed = true, "PostgreSQL contextual decode failure persisted");
trace_query_success("mark_decode_failed", started_at, 1);
return std::result::Result::Ok(outcome);
}
pub(crate) async fn persist_materialization_result(
pool: &sqlx::PgPool,
bundle: &crate::MaterializationPersistenceBundle,
force_replay: bool,
_force_replay: bool,
) -> ks_core::Result<crate::InsertOutcome> {
if bundle.status == "failed" {
tracing::error!(
target: crate::TRACING_TARGET, backend = "postgres", domain = "ks-store.pg",
action = "persist_materialization_failure",
signature = %bundle.signature,
instruction_path = %bundle.instruction_path,
processor_name = %bundle.ledger_identity.processor_name,
processor_version = %bundle.ledger_identity.processor_version,
input_key = %bundle.ledger_identity.input_key,
input_hash = %bundle.ledger_identity.input_hash,
source_decoder_name = %bundle.source_decoder_name,
source_decoder_version = %bundle.source_decoder_version,
error_code = ?bundle.error_code,
error_message = ?bundle.error_message,
"persist PostgreSQL materialization failure"
);
}
tracing::debug!(target: crate::TRACING_TARGET, backend = "postgres", domain = "ks-store.pg", action = "persist_materialization_result", signature = %bundle.signature, instruction_path = %bundle.instruction_path, stage = %bundle.ledger_identity.stage, processor_name = %bundle.ledger_identity.processor_name, processor_version = %bundle.ledger_identity.processor_version, input_key = %bundle.ledger_identity.input_key, input_hash = %bundle.ledger_identity.input_hash, source_decoder_name = %bundle.source_decoder_name, source_decoder_version = %bundle.source_decoder_version, source_decode_input_key = %bundle.source_decode_input_key, status = %bundle.status, output_count = bundle.outputs.len(), force_replay, "persist PostgreSQL materialization result");
let started_at = std::time::Instant::now();
let validation_result = bundle.validate();
if let std::result::Result::Err(error) = validation_result {
return std::result::Result::Err(error);
@@ -646,23 +636,6 @@ pub(crate) async fn persist_materialization_result(
if let std::result::Result::Err(error) = ledger_result {
return std::result::Result::Err(error);
}
if !bundle.outputs.is_empty() {
let lifecycle_result = sqlx::query(
"UPDATE k_sol_core_instructions SET processing_state = 'materialized', processor_name = $1, processor_version = $2, lifecycle_reason = $3, updated_at = NOW() WHERE signature = $4 AND instruction_path = $5",
)
.bind(bundle.ledger_identity.processor_name.as_str())
.bind(bundle.ledger_identity.processor_version.as_str())
.bind(bundle.status.as_str())
.bind(bundle.signature.as_str())
.bind(bundle.instruction_path.as_str())
.execute(&mut *transaction)
.await;
if let std::result::Result::Err(error) = lifecycle_result {
return std::result::Result::Err(ks_core::Error::db(format!(
"postgres materialization lifecycle update failed: {error}"
)));
}
}
let coverage_result =
crate::postgres::query::decode_pipeline_queries::refresh_materialized_coverage_count(
&mut transaction,
@@ -674,6 +647,15 @@ pub(crate) async fn persist_materialization_result(
if let std::result::Result::Err(error) = coverage_result {
return std::result::Result::Err(error);
}
let lifecycle_result = crate::recompute_instruction_lifecycle_in_transaction(
&mut transaction,
bundle.signature.as_str(),
bundle.instruction_path.as_str(),
)
.await;
if let std::result::Result::Err(error) = lifecycle_result {
return std::result::Result::Err(error);
}
let commit_result = transaction.commit().await;
if let std::result::Result::Err(error) = commit_result {
return std::result::Result::Err(ks_core::Error::db(format!(
@@ -690,7 +672,8 @@ pub(crate) async fn persist_materialization_result(
},
};
let outcome = crate::InsertOutcome::new(count, 1, 0);
tracing::debug!(target: crate::TRACING_TARGET, backend = "postgres", domain = "ks-store.pg", action = "persist_materialization_result", signature = %bundle.signature, instruction_path = %bundle.instruction_path, processor_name = %bundle.ledger_identity.processor_name, processor_version = %bundle.ledger_identity.processor_version, status = %bundle.status, outcome = ?outcome, committed = true, "PostgreSQL materialization result persisted");
let elapsed_ms = started_at.elapsed().as_secs_f64() * 1000.0;
tracing::trace!(target: crate::TRACING_TARGET, backend = "postgres", domain = "ks-store.pg", action = "persist_materialization_result", elapsed_ms, rows = count, outcome = "success", "PostgreSQL store operation completed");
return std::result::Result::Ok(outcome);
}
@@ -700,12 +683,12 @@ pub(crate) async fn list_decode_coverage_summary(
processor_version: std::option::Option<&str>,
limit: u32,
) -> ks_core::Result<std::vec::Vec<crate::DecodeCoverageSummaryRow>> {
tracing::debug!(target: crate::TRACING_TARGET, backend = "postgres", domain = "ks-store.pg", action = "list_decode_coverage_summary", processor_name = ?processor_name, processor_version = ?processor_version, limit, "query PostgreSQL decode coverage summary");
if limit == 0 {
return std::result::Result::Err(ks_core::Error::db(
"decode coverage summary limit must be greater than zero",
));
}
let started_at = std::time::Instant::now();
let query_result = sqlx::query(
"WITH declared AS (SELECT processor_name, processor_version, program_id, surface_code, entry_code, COUNT(*)::BIGINT AS declared_count FROM k_sol_decode_coverage_declarations WHERE ($1::text IS NULL OR processor_name = $1) AND ($2::text IS NULL OR processor_version = $2) GROUP BY processor_name, processor_version, program_id, surface_code, entry_code), observed AS (SELECT processor_name, processor_version, program_id, surface_code, COALESCE(entry_code, 'unknown') AS entry_code, COUNT(*)::BIGINT AS observed_count, COUNT(*) FILTER (WHERE recognized)::BIGINT AS recognized_count, COALESCE(SUM(decoded_count), 0)::BIGINT AS decoded_count, COALESCE(SUM(materialized_count), 0)::BIGINT AS materialized_count, COALESCE(SUM(error_count), 0)::BIGINT AS error_count, COUNT(*) FILTER (WHERE status = 'unsupported' OR entry_code IS NULL)::BIGINT AS unknown_count, COUNT(*) FILTER (WHERE NOT transaction_failed)::BIGINT AS successful_transaction_count, COUNT(*) FILTER (WHERE transaction_failed)::BIGINT AS failed_transaction_count FROM k_sol_decode_coverage_observations WHERE ($1::text IS NULL OR processor_name = $1) AND ($2::text IS NULL OR processor_version = $2) GROUP BY processor_name, processor_version, program_id, surface_code, COALESCE(entry_code, 'unknown')) SELECT COALESCE(d.processor_name, o.processor_name) AS processor_name, COALESCE(d.processor_version, o.processor_version) AS processor_version, COALESCE(d.program_id, o.program_id) AS program_id, COALESCE(d.surface_code, o.surface_code) AS surface_code, COALESCE(d.entry_code, o.entry_code) AS entry_code, COALESCE(d.declared_count, 0)::BIGINT AS declared_count, COALESCE(o.observed_count, 0)::BIGINT AS observed_count, COALESCE(o.recognized_count, 0)::BIGINT AS recognized_count, COALESCE(o.decoded_count, 0)::BIGINT AS decoded_count, COALESCE(o.materialized_count, 0)::BIGINT AS materialized_count, COALESCE(o.error_count, 0)::BIGINT AS error_count, COALESCE(o.unknown_count, 0)::BIGINT AS unknown_count, COALESCE(o.successful_transaction_count, 0)::BIGINT AS successful_transaction_count, COALESCE(o.failed_transaction_count, 0)::BIGINT AS failed_transaction_count FROM declared d FULL OUTER JOIN observed o ON d.processor_name = o.processor_name AND d.processor_version = o.processor_version AND d.program_id = o.program_id AND COALESCE(d.surface_code, '') = COALESCE(o.surface_code, '') AND d.entry_code = o.entry_code ORDER BY observed_count DESC, processor_name, program_id, entry_code LIMIT $3",
)
@@ -717,6 +700,7 @@ pub(crate) async fn list_decode_coverage_summary(
let rows = match query_result {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => {
trace_query_failure("list_decode_coverage_summary", started_at, &error);
return std::result::Result::Err(ks_core::Error::db(format!(
"postgres decode coverage summary query failed: {error}"
)));
@@ -731,7 +715,7 @@ pub(crate) async fn list_decode_coverage_summary(
std::result::Result::Err(error) => return std::result::Result::Err(error),
}
}
tracing::debug!(target: crate::TRACING_TARGET, backend = "postgres", domain = "ks-store.pg", action = "list_decode_coverage_summary", processor_name = ?processor_name, processor_version = ?processor_version, row_count = output.len(), "PostgreSQL decode coverage summary loaded");
trace_query_success("list_decode_coverage_summary", started_at, output.len());
return std::result::Result::Ok(output);
}
@@ -975,6 +959,16 @@ fn map_error<T>(column: &str, error: sqlx::Error) -> ks_core::Result<T> {
)));
}
fn trace_query_success(action: &str, started_at: std::time::Instant, rows: usize) {
let elapsed_ms = started_at.elapsed().as_secs_f64() * 1000.0;
tracing::trace!(target: crate::TRACING_TARGET, backend = "postgres", domain = "ks-store.pg", action = action, elapsed_ms, rows, outcome = "success", "PostgreSQL store operation completed");
}
fn trace_query_failure(action: &str, started_at: std::time::Instant, error: &sqlx::Error) {
let elapsed_ms = started_at.elapsed().as_secs_f64() * 1000.0;
tracing::trace!(target: crate::TRACING_TARGET, backend = "postgres", domain = "ks-store.pg", action = action, elapsed_ms, error = %error, outcome = "error", "PostgreSQL store operation failed");
}
#[cfg(test)]
mod tests {
fn result_or_panic<T, E>(result: std::result::Result<T, E>) -> T

View File

@@ -0,0 +1,275 @@
// file: ks-store/src/postgres/query/invalidation_queries.rs
// version: 2
//! PostgreSQL descendant invalidation and aggregate instruction lifecycle helpers.
pub(crate) async fn invalidate_core_descendants_in_transaction(
transaction: &mut sqlx::Transaction<'_, sqlx::Postgres>,
signature: &str,
) -> ks_core::Result<u64> {
let input_prefix = format!("{signature}:");
let mut deleted = 0_u64;
for (sql, stage) in [
(
"DELETE FROM k_sol_ops_processing_ledger WHERE stage = $1 AND LEFT(input_key, LENGTH($2)) = $2",
"event_materialization",
),
(
"DELETE FROM k_sol_ops_processing_ledger WHERE stage = $1 AND LEFT(input_key, LENGTH($2)) = $2",
"instruction_decode",
),
] {
let result = sqlx::query(sql)
.bind(stage)
.bind(input_prefix.as_str())
.execute(&mut **transaction)
.await;
match result {
std::result::Result::Ok(value) => {
deleted = deleted.saturating_add(value.rows_affected())
},
std::result::Result::Err(error) => {
return std::result::Result::Err(ks_core::Error::db(format!(
"postgres descendant ledger invalidation failed: {error}"
)));
},
}
}
for sql in [
"DELETE FROM k_sol_mat_outputs WHERE signature = $1",
"DELETE FROM k_sol_decode_events WHERE signature = $1",
"DELETE FROM k_sol_decode_coverage_observations WHERE signature = $1",
] {
let result = sqlx::query(sql).bind(signature).execute(&mut **transaction).await;
match result {
std::result::Result::Ok(value) => {
deleted = deleted.saturating_add(value.rows_affected())
},
std::result::Result::Err(error) => {
return std::result::Result::Err(ks_core::Error::db(format!(
"postgres descendant output invalidation failed: {error}"
)));
},
}
}
return std::result::Result::Ok(deleted);
}
pub(crate) async fn invalidate_decode_descendants_in_transaction(
transaction: &mut sqlx::Transaction<'_, sqlx::Postgres>,
decoder_name: &str,
decoder_version: &str,
decode_input_key: &str,
) -> ks_core::Result<u64> {
let materialization_prefix = format!("{decode_input_key}:{decoder_name}:{decoder_version}:");
let ledger_result = sqlx::query(
"DELETE FROM k_sol_ops_processing_ledger WHERE stage = 'event_materialization' AND LEFT(input_key, LENGTH($1)) = $1",
)
.bind(materialization_prefix.as_str())
.execute(&mut **transaction)
.await;
let ledger_deleted = match ledger_result {
std::result::Result::Ok(value) => value.rows_affected(),
std::result::Result::Err(error) => {
return std::result::Result::Err(ks_core::Error::db(format!(
"postgres materialization ledger invalidation failed: {error}"
)));
},
};
let output_result = sqlx::query(
"DELETE FROM k_sol_mat_outputs WHERE source_decoder_name = $1 AND source_decoder_version = $2 AND source_decode_input_key = $3",
)
.bind(decoder_name)
.bind(decoder_version)
.bind(decode_input_key)
.execute(&mut **transaction)
.await;
let output_deleted = match output_result {
std::result::Result::Ok(value) => value.rows_affected(),
std::result::Result::Err(error) => {
return std::result::Result::Err(ks_core::Error::db(format!(
"postgres materialization output invalidation failed: {error}"
)));
},
};
return std::result::Result::Ok(ledger_deleted.saturating_add(output_deleted));
}
pub(crate) async fn recompute_instruction_lifecycle_in_transaction(
transaction: &mut sqlx::Transaction<'_, sqlx::Postgres>,
signature: &str,
instruction_path: &str,
) -> ks_core::Result<u64> {
let decode_input_key = format!("{signature}:{instruction_path}");
let decode_result = sqlx::query_as::<
sqlx::Postgres,
(std::string::String, std::string::String, std::string::String),
>(
"SELECT processor_name, processor_version, status FROM k_sol_decode_coverage_observations WHERE input_key = $1 ORDER BY updated_at DESC, id DESC LIMIT 1",
)
.bind(decode_input_key.as_str())
.fetch_optional(&mut **transaction)
.await;
let decode = match decode_result {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => {
return std::result::Result::Err(ks_core::Error::db(format!(
"postgres decode lifecycle lookup failed: {error}"
)));
},
};
if let std::option::Option::Some((decoder_name, decoder_version, status)) = decode {
let materialized_result = sqlx::query_as::<
sqlx::Postgres,
(std::string::String, std::string::String),
>(
"SELECT processor_name, processor_version FROM k_sol_mat_outputs WHERE source_decode_input_key = $1 AND source_decoder_name = $2 AND source_decoder_version = $3 ORDER BY updated_at DESC, id DESC LIMIT 1",
)
.bind(decode_input_key.as_str())
.bind(decoder_name.as_str())
.bind(decoder_version.as_str())
.fetch_optional(&mut **transaction)
.await;
let materialized = match materialized_result {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => {
return std::result::Result::Err(ks_core::Error::db(format!(
"postgres materialized lifecycle lookup failed: {error}"
)));
},
};
if let std::option::Option::Some((processor_name, processor_version)) = materialized {
return update_instruction_lifecycle_in_transaction(
transaction,
signature,
instruction_path,
"materialized",
std::option::Option::Some(processor_name.as_str()),
std::option::Option::Some(processor_version.as_str()),
std::option::Option::Some("materialized"),
)
.await;
}
let processing_state = match status.as_str() {
"decoded" => "decoded",
"ignored" | "unsupported" => "ignored",
"failed" => "failed",
_unknown => "decoded",
};
return update_instruction_lifecycle_in_transaction(
transaction,
signature,
instruction_path,
processing_state,
std::option::Option::Some(decoder_name.as_str()),
std::option::Option::Some(decoder_version.as_str()),
std::option::Option::Some(status.as_str()),
)
.await;
}
let ledger_result = sqlx::query_as::<
sqlx::Postgres,
(std::string::String, std::string::String, std::string::String),
>(
"SELECT processor_name, processor_version, status FROM k_sol_ops_processing_ledger WHERE stage = 'instruction_decode' AND input_key = $1 ORDER BY updated_at DESC, id DESC LIMIT 1",
)
.bind(decode_input_key.as_str())
.fetch_optional(&mut **transaction)
.await;
let ledger = match ledger_result {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => {
return std::result::Result::Err(ks_core::Error::db(format!(
"postgres decode ledger lifecycle lookup failed: {error}"
)));
},
};
if let std::option::Option::Some((processor_name, processor_version, status)) = ledger {
let processing_state = if status == "failed" { "failed" } else { "decoded" };
return update_instruction_lifecycle_in_transaction(
transaction,
signature,
instruction_path,
processing_state,
std::option::Option::Some(processor_name.as_str()),
std::option::Option::Some(processor_version.as_str()),
std::option::Option::Some(status.as_str()),
)
.await;
}
return update_instruction_lifecycle_in_transaction(
transaction,
signature,
instruction_path,
"pending",
std::option::Option::None,
std::option::Option::None,
std::option::Option::Some("descendants_invalidated"),
)
.await;
}
async fn update_instruction_lifecycle_in_transaction(
transaction: &mut sqlx::Transaction<'_, sqlx::Postgres>,
signature: &str,
instruction_path: &str,
processing_state: &str,
processor_name: std::option::Option<&str>,
processor_version: std::option::Option<&str>,
reason: std::option::Option<&str>,
) -> ks_core::Result<u64> {
let scope_result = crate::CoreInstructionScope::from_instruction_path(instruction_path);
let scope = match scope_result {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let sql = match scope {
crate::CoreInstructionScope::TopLevel => {
"UPDATE k_sol_core_instructions SET processing_state = $1, processor_name = $2, processor_version = $3, lifecycle_reason = $4, updated_at = NOW() WHERE signature = $5 AND instruction_path = $6"
},
crate::CoreInstructionScope::Inner => {
"UPDATE k_sol_core_inner_instructions SET processing_state = $1, processor_name = $2, processor_version = $3, lifecycle_reason = $4, updated_at = NOW() WHERE signature = $5 AND instruction_path = $6"
},
};
let result = sqlx::query(sql)
.bind(processing_state)
.bind(processor_name)
.bind(processor_version)
.bind(reason)
.bind(signature)
.bind(instruction_path)
.execute(&mut **transaction)
.await;
return match result {
std::result::Result::Ok(value) => std::result::Result::Ok(value.rows_affected()),
std::result::Result::Err(error) => std::result::Result::Err(ks_core::Error::db(format!(
"postgres aggregate instruction lifecycle update failed: {error}"
))),
};
}
#[cfg(test)]
mod tests {
#[test]
fn lifecycle_materialization_is_scoped_to_the_current_decoder_identity() {
let source = include_str!("invalidation_queries.rs");
assert!(source.contains("source_decode_input_key = $1 AND source_decoder_name = $2 AND source_decoder_version = $3"));
}
#[test]
fn descendant_invalidation_stages_are_exact() {
let source = include_str!("invalidation_queries.rs");
let tests_marker = source.find("#[cfg(test)]");
let production_source = match tests_marker {
std::option::Option::Some(index) => &source[..index],
std::option::Option::None => source,
};
assert!(production_source.contains("stage = 'event_materialization'"));
assert!(production_source.contains("\"instruction_decode\""));
assert!(production_source.contains("k_sol_mat_outputs"));
assert!(production_source.contains("k_sol_decode_events"));
assert!(production_source.contains("k_sol_decode_coverage_observations"));
assert!(production_source.contains("LEFT(input_key, LENGTH("));
assert!(!production_source.contains("input_key LIKE"));
}
}

View File

@@ -1,5 +1,5 @@
// file: ks-store/src/postgres/query/raw_queries.rs
// version: 8
// version: 9
//! PostgreSQL canonical transaction and acquisition observation SQL queries.
@@ -49,6 +49,40 @@ pub(crate) async fn has_transaction_observation_key(
};
}
pub(crate) async fn has_account_observation_key(
pool: &sqlx::PgPool,
observation_key: &str,
) -> ks_core::Result<bool> {
let validation_result =
validate_required_text(observation_key, "account observation key must not be empty");
if let std::result::Result::Err(error) = validation_result {
return std::result::Result::Err(error);
}
let started_at = std::time::Instant::now();
let query_result = sqlx::query_scalar::<sqlx::Postgres, bool>(
"SELECT EXISTS(SELECT 1 FROM k_sol_obs_account_observations WHERE observation_key = $1)",
)
.bind(observation_key)
.fetch_one(pool)
.await;
return match query_result {
std::result::Result::Ok(value) => {
trace_query_success(
"has_account_observation_key",
started_at,
if value { 1_usize } else { 0_usize },
);
std::result::Result::Ok(value)
},
std::result::Result::Err(error) => {
trace_query_failure("has_account_observation_key", started_at, &error);
std::result::Result::Err(ks_core::Error::db(format!(
"postgres account observation lookup failed: {error}"
)))
},
};
}
pub(crate) async fn insert_raw_transaction(
pool: &sqlx::PgPool,
input: &crate::RawTransactionInsert,
@@ -145,6 +179,83 @@ pub(crate) async fn insert_transaction_observation(
};
}
pub(crate) async fn insert_account_observation(
pool: &sqlx::PgPool,
input: &crate::AccountObservationInsert,
) -> ks_core::Result<crate::InsertOutcome> {
let validation_result = input.validate();
if let std::result::Result::Err(error) = validation_result {
return std::result::Result::Err(error);
}
let context_slot_result = sql_slot_from_u64(input.context_slot);
let context_slot = match context_slot_result {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let space_result = optional_sql_bigint_from_u64(input.space);
let space = match space_result {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let payload_size_result = optional_sql_bigint_from_u64(input.payload_size_bytes);
let payload_size_bytes = match payload_size_result {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let lamports = input.lamports.map(|value| return value.to_string());
let rent_epoch = input.rent_epoch.map(|value| return value.to_string());
let origin = transaction_observation_origin_to_sql(input.origin);
let status = transaction_observation_status_to_sql(input.status);
let started_at = std::time::Instant::now();
let query_result = sqlx::query_scalar::<sqlx::Postgres, i64>(
"INSERT INTO k_sol_obs_account_observations (observation_key, account_key, context_slot, owner, lamports, executable, rent_epoch, space, data_base64, data_hash, provider, endpoint_code, protocol, acquisition_method, commitment, capture_session_id, filter_code, origin, detected_at, received_at, normalized_at, payload_size_bytes, source_payload_hash, status, error_code, error_message) VALUES ($1, $2, $3, $4, $5::NUMERIC, $6, $7::NUMERIC, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26) ON CONFLICT (observation_key) DO NOTHING RETURNING id",
)
.bind(input.observation_key.as_str())
.bind(input.account_key.as_str())
.bind(context_slot)
.bind(input.owner.as_deref())
.bind(lamports.as_deref())
.bind(input.executable)
.bind(rent_epoch.as_deref())
.bind(space)
.bind(input.data_base64.as_deref())
.bind(input.data_hash.as_deref())
.bind(input.provider.as_str())
.bind(input.endpoint_code.as_deref())
.bind(input.protocol.as_str())
.bind(input.acquisition_method.as_str())
.bind(input.commitment.as_deref())
.bind(input.capture_session_id.as_deref())
.bind(input.filter_code.as_deref())
.bind(origin)
.bind(input.detected_at.as_ref())
.bind(input.received_at)
.bind(input.normalized_at.as_ref())
.bind(payload_size_bytes)
.bind(input.source_payload_hash.as_deref())
.bind(status)
.bind(input.error_code.as_deref())
.bind(input.error_message.as_deref())
.fetch_optional(pool)
.await;
return match query_result {
std::result::Result::Ok(std::option::Option::Some(_id)) => {
trace_query_success("insert_account_observation", started_at, 1);
std::result::Result::Ok(crate::InsertOutcome::new(1, 0, 0))
},
std::result::Result::Ok(std::option::Option::None) => {
trace_query_success("insert_account_observation", started_at, 0);
std::result::Result::Ok(crate::InsertOutcome::new(0, 0, 1))
},
std::result::Result::Err(error) => {
trace_query_failure("insert_account_observation", started_at, &error);
std::result::Result::Err(ks_core::Error::db(format!(
"postgres account observation insert failed: {error}"
)))
},
};
}
pub(crate) async fn update_raw_payload_lifecycle(
pool: &sqlx::PgPool,
mark: &crate::RawPayloadLifecycleMark,
@@ -282,6 +393,16 @@ fn outcome_from_update_result(
};
}
fn trace_query_success(action: &str, started_at: std::time::Instant, rows: usize) {
let elapsed_ms = started_at.elapsed().as_secs_f64() * 1000.0;
tracing::trace!(target: crate::TRACING_TARGET, backend = "postgres", domain = "ks-store.pg", action = action, elapsed_ms, rows, outcome = "success", "PostgreSQL store operation completed");
}
fn trace_query_failure(action: &str, started_at: std::time::Instant, error: &sqlx::Error) {
let elapsed_ms = started_at.elapsed().as_secs_f64() * 1000.0;
tracing::trace!(target: crate::TRACING_TARGET, backend = "postgres", domain = "ks-store.pg", action = action, elapsed_ms, error = %error, outcome = "error", "PostgreSQL store operation failed");
}
#[cfg(test)]
mod tests {
#[test]
@@ -313,6 +434,12 @@ mod tests {
assert_eq!(value, "normalized");
}
#[test]
fn optional_bigint_rejects_values_above_bigint() {
let result = super::optional_sql_bigint_from_u64(std::option::Option::Some(u64::MAX));
assert!(result.is_err());
}
#[test]
fn sql_slot_rejects_values_above_bigint() {
let result = super::sql_slot_from_u64(u64::MAX);

View File

@@ -1,5 +1,5 @@
// file: ks-store/src/postgres/query/replay_candidate_queries.rs
// version: 7
// version: 8
//! Read-only PostgreSQL queries for replay candidate discovery.
@@ -67,6 +67,7 @@ pub(crate) async fn list_replay_transaction_candidates(
} else {
crate::postgres::query::replay_candidate_queries::transaction_candidate_sql_asc()
};
let started_at = std::time::Instant::now();
let query_result = sqlx::query_as::<
sqlx::Postgres,
crate::postgres::query::replay_candidate_queries::ReplayTransactionCandidateRow,
@@ -86,6 +87,7 @@ pub(crate) async fn list_replay_transaction_candidates(
let rows = match query_result {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => {
trace_query_failure("list_replay_transaction_candidates", started_at, &error);
return std::result::Result::Err(ks_core::Error::db(format!(
"postgres replay transaction candidate query failed: {error}"
)));
@@ -110,6 +112,7 @@ pub(crate) async fn list_replay_transaction_candidates(
updated_at: row.updated_at,
});
}
trace_query_success("list_replay_transaction_candidates", started_at, output.len());
return std::result::Result::Ok(output);
}
@@ -117,6 +120,7 @@ pub(crate) async fn list_replay_program_summaries(
pool: &sqlx::PgPool,
filter: &crate::ReplayProgramFilter,
) -> ks_core::Result<std::vec::Vec<crate::ReplayProgramSummary>> {
let started_at = std::time::Instant::now();
let query_result = sqlx::query_as::<sqlx::Postgres, crate::postgres::query::replay_candidate_queries::ReplayProgramSummaryRow>(
r#"WITH occurrences AS (
SELECT program_id, signature, slot, 'top_level'::TEXT AS scope FROM k_sol_core_instructions
@@ -145,6 +149,7 @@ pub(crate) async fn list_replay_program_summaries(
let rows = match query_result {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => {
trace_query_failure("list_replay_program_summaries", started_at, &error);
return std::result::Result::Err(ks_core::Error::db(format!(
"postgres replay program summary query failed: {error}"
)));
@@ -162,6 +167,7 @@ pub(crate) async fn list_replay_program_summaries(
max_slot: row.max_slot,
});
}
trace_query_success("list_replay_program_summaries", started_at, output.len());
return std::result::Result::Ok(output);
}
@@ -170,6 +176,7 @@ pub(crate) async fn list_replay_entity_summaries(
filter: &crate::ReplayEntityFilter,
) -> ks_core::Result<std::vec::Vec<crate::ReplayEntitySummary>> {
let entity_kind = filter.entity_kind.as_sql();
let started_at = std::time::Instant::now();
let query_result = sqlx::query_as::<
sqlx::Postgres,
crate::postgres::query::replay_candidate_queries::ReplayEntitySummaryRow,
@@ -207,6 +214,7 @@ pub(crate) async fn list_replay_entity_summaries(
let rows = match query_result {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => {
trace_query_failure("list_replay_entity_summaries", started_at, &error);
return std::result::Result::Err(ks_core::Error::db(format!(
"postgres replay entity summary query failed: {error}"
)));
@@ -223,6 +231,7 @@ pub(crate) async fn list_replay_entity_summaries(
max_slot: row.max_slot,
});
}
trace_query_success("list_replay_entity_summaries", started_at, output.len());
return std::result::Result::Ok(output);
}
@@ -392,6 +401,16 @@ fn optional_sql_bigint(
};
}
fn trace_query_success(action: &str, started_at: std::time::Instant, rows: usize) {
let elapsed_ms = started_at.elapsed().as_secs_f64() * 1000.0;
tracing::trace!(target: crate::TRACING_TARGET, backend = "postgres", domain = "ks-store.pg", action = action, elapsed_ms, rows, outcome = "success", "PostgreSQL store operation completed");
}
fn trace_query_failure(action: &str, started_at: std::time::Instant, error: &sqlx::Error) {
let elapsed_ms = started_at.elapsed().as_secs_f64() * 1000.0;
tracing::trace!(target: crate::TRACING_TARGET, backend = "postgres", domain = "ks-store.pg", action = action, elapsed_ms, error = %error, outcome = "error", "PostgreSQL store operation failed");
}
#[cfg(test)]
mod tests {
#[tokio::test]

View File

@@ -1,8 +1,9 @@
// file: ks-store/src/postgres/repository.rs
// version: 2
// version: 3
//! PostgreSQL repository implementations.
mod account_state_repository;
mod core_extraction_repository;
mod core_transaction_repository;
mod decode_pipeline_repository;

View File

@@ -0,0 +1,52 @@
// file: ks-store/src/postgres/repository/account_state_repository.rs
// version: 1
//! PostgreSQL generic account observation to Core account-state repository.
#[async_trait::async_trait]
impl crate::AccountStateStore for crate::PostgresStore {
#[expect(
clippy::implicit_return,
reason = "async_trait expansion triggers implicit_return on generated async trait methods."
)]
async fn list_account_observations_for_normalization(
&self,
filter: &crate::AccountStateSelectionFilter,
) -> ks_core::Result<std::vec::Vec<crate::AccountObservationRow>> {
return crate::list_account_observations_for_normalization(self.pool(), filter).await;
}
#[expect(
clippy::implicit_return,
reason = "async_trait expansion triggers implicit_return on generated async trait methods."
)]
async fn is_account_state_current(
&self,
identity: &crate::ProcessingLedgerIdentity,
) -> ks_core::Result<bool> {
return crate::is_account_state_current(self.pool(), identity).await;
}
#[expect(
clippy::implicit_return,
reason = "async_trait expansion triggers implicit_return on generated async trait methods."
)]
async fn persist_account_state(
&self,
bundle: &crate::AccountStatePersistenceBundle,
force_replay: bool,
) -> ks_core::Result<crate::InsertOutcome> {
return crate::persist_account_state(self.pool(), bundle, force_replay).await;
}
#[expect(
clippy::implicit_return,
reason = "async_trait expansion triggers implicit_return on generated async trait methods."
)]
async fn mark_account_state_failed(
&self,
failure: &crate::AccountStateNormalizationFailure,
) -> ks_core::Result<crate::InsertOutcome> {
return crate::mark_account_state_failed(self.pool(), failure).await;
}
}

View File

@@ -1,5 +1,5 @@
// file: ks-store/src/postgres/repository/core_transaction_repository.rs
// version: 3
// version: 4
//! PostgreSQL core Solana repository implementation.
@@ -79,7 +79,7 @@ impl crate::CoreTransactionStore for crate::PostgresStore {
&self,
filter: &crate::CoreInstructionReplayFilter,
page_request: &crate::PageRequest,
) -> ks_core::Result<std::vec::Vec<crate::CoreInstructionRow>> {
) -> ks_core::Result<crate::PageSlice<crate::CoreInstructionReplayRow>> {
return crate::list_core_instructions_for_replay(self.pool(), filter, page_request).await;
}
@@ -91,7 +91,7 @@ impl crate::CoreTransactionStore for crate::PostgresStore {
&self,
filter: &crate::CoreInstructionReplayFilter,
page_request: &crate::PageRequest,
) -> ks_core::Result<std::vec::Vec<crate::MdCoreInstructionReplayInput>> {
) -> ks_core::Result<crate::PageSlice<crate::MdCoreInstructionReplayInput>> {
return crate::list_core_instruction_replay_inputs(self.pool(), filter, page_request).await;
}

View File

@@ -1,5 +1,5 @@
// file: ks-store/src/postgres/repository/raw_transaction_repository.rs
// version: 3
// version: 4
//! PostgreSQL canonical transaction and acquisition observation repository implementation.
@@ -27,6 +27,14 @@ impl crate::RawTransactionStore for crate::PostgresStore {
return crate::has_transaction_observation_key(self.pool(), observation_key).await;
}
#[expect(
clippy::implicit_return,
reason = "async_trait expansion triggers implicit_return on generated async trait methods."
)]
async fn has_account_observation_key(&self, observation_key: &str) -> ks_core::Result<bool> {
return crate::has_account_observation_key(self.pool(), observation_key).await;
}
#[expect(
clippy::implicit_return,
reason = "async_trait expansion triggers implicit_return on generated async trait methods."
@@ -49,6 +57,17 @@ impl crate::RawTransactionStore for crate::PostgresStore {
return crate::insert_transaction_observation(self.pool(), input).await;
}
#[expect(
clippy::implicit_return,
reason = "async_trait expansion triggers implicit_return on generated async trait methods."
)]
async fn insert_account_observation(
&self,
input: &crate::AccountObservationInsert,
) -> ks_core::Result<crate::InsertOutcome> {
return crate::insert_account_observation(self.pool(), input).await;
}
#[expect(
clippy::implicit_return,
reason = "async_trait expansion triggers implicit_return on generated async trait methods."

View File

@@ -1,5 +1,5 @@
// file: ks-store/src/store.rs
// version: 10
// version: 11
//! Backend-agnostic store facade and connection ownership.
@@ -531,6 +531,20 @@ impl crate::RawTransactionStore for crate::Store {
};
}
#[expect(
clippy::implicit_return,
reason = "async_trait expansion triggers implicit_return on generated async trait methods."
)]
async fn has_account_observation_key(&self, observation_key: &str) -> ks_core::Result<bool> {
return match &self.backend {
StoreBackend::Disabled => std::result::Result::Err(disabled_store_error()),
StoreBackend::Postgres(store) => {
crate::RawTransactionStore::has_account_observation_key(store, observation_key)
.await
},
};
}
#[expect(
clippy::implicit_return,
reason = "async_trait expansion triggers implicit_return on generated async trait methods."
@@ -563,6 +577,22 @@ impl crate::RawTransactionStore for crate::Store {
};
}
#[expect(
clippy::implicit_return,
reason = "async_trait expansion triggers implicit_return on generated async trait methods."
)]
async fn insert_account_observation(
&self,
input: &crate::AccountObservationInsert,
) -> ks_core::Result<crate::InsertOutcome> {
return match &self.backend {
StoreBackend::Disabled => std::result::Result::Err(disabled_store_error()),
StoreBackend::Postgres(store) => {
crate::RawTransactionStore::insert_account_observation(store, input).await
},
};
}
#[expect(
clippy::implicit_return,
reason = "async_trait expansion triggers implicit_return on generated async trait methods."
@@ -686,7 +716,7 @@ impl crate::CoreTransactionStore for crate::Store {
&self,
filter: &crate::CoreInstructionReplayFilter,
page_request: &crate::PageRequest,
) -> ks_core::Result<std::vec::Vec<crate::CoreInstructionRow>> {
) -> ks_core::Result<crate::PageSlice<crate::CoreInstructionReplayRow>> {
return match &self.backend {
StoreBackend::Disabled => std::result::Result::Err(disabled_store_error()),
StoreBackend::Postgres(store) => {
@@ -708,7 +738,7 @@ impl crate::CoreTransactionStore for crate::Store {
&self,
filter: &crate::CoreInstructionReplayFilter,
page_request: &crate::PageRequest,
) -> ks_core::Result<std::vec::Vec<crate::MdCoreInstructionReplayInput>> {
) -> ks_core::Result<crate::PageSlice<crate::MdCoreInstructionReplayInput>> {
return match &self.backend {
StoreBackend::Disabled => std::result::Result::Err(disabled_store_error()),
StoreBackend::Postgres(store) => {
@@ -739,6 +769,75 @@ impl crate::CoreTransactionStore for crate::Store {
}
}
#[async_trait::async_trait]
impl crate::AccountStateStore for crate::Store {
#[expect(
clippy::implicit_return,
reason = "async_trait expansion triggers implicit_return on generated async trait methods."
)]
async fn list_account_observations_for_normalization(
&self,
filter: &crate::AccountStateSelectionFilter,
) -> ks_core::Result<std::vec::Vec<crate::AccountObservationRow>> {
return match &self.backend {
StoreBackend::Disabled => std::result::Result::Err(disabled_store_error()),
StoreBackend::Postgres(store) => {
crate::AccountStateStore::list_account_observations_for_normalization(store, filter)
.await
},
};
}
#[expect(
clippy::implicit_return,
reason = "async_trait expansion triggers implicit_return on generated async trait methods."
)]
async fn is_account_state_current(
&self,
identity: &crate::ProcessingLedgerIdentity,
) -> ks_core::Result<bool> {
return match &self.backend {
StoreBackend::Disabled => std::result::Result::Err(disabled_store_error()),
StoreBackend::Postgres(store) => {
crate::AccountStateStore::is_account_state_current(store, identity).await
},
};
}
#[expect(
clippy::implicit_return,
reason = "async_trait expansion triggers implicit_return on generated async trait methods."
)]
async fn persist_account_state(
&self,
bundle: &crate::AccountStatePersistenceBundle,
force_replay: bool,
) -> ks_core::Result<crate::InsertOutcome> {
return match &self.backend {
StoreBackend::Disabled => std::result::Result::Err(disabled_store_error()),
StoreBackend::Postgres(store) => {
crate::AccountStateStore::persist_account_state(store, bundle, force_replay).await
},
};
}
#[expect(
clippy::implicit_return,
reason = "async_trait expansion triggers implicit_return on generated async trait methods."
)]
async fn mark_account_state_failed(
&self,
failure: &crate::AccountStateNormalizationFailure,
) -> ks_core::Result<crate::InsertOutcome> {
return match &self.backend {
StoreBackend::Disabled => std::result::Result::Err(disabled_store_error()),
StoreBackend::Postgres(store) => {
crate::AccountStateStore::mark_account_state_failed(store, failure).await
},
};
}
}
#[async_trait::async_trait]
impl crate::CoreExtractionStore for crate::Store {
#[expect(