Files
khadhroony-bot3/ks-store/src/postgres/query/replay_candidate_queries.rs

944 lines
43 KiB
Rust

// file: ks-store/src/postgres/query/replay_candidate_queries.rs
// version: 10
//! Read-only PostgreSQL queries for replay candidate discovery.
#[derive(sqlx::FromRow)]
struct ReplayTransactionCandidateRow {
signature: std::string::String,
slot: i64,
raw_processing_state: std::string::String,
retention_state: std::string::String,
has_core_transaction: bool,
transaction_failed: std::option::Option<bool>,
ledger_status: std::string::String,
processor_version: std::option::Option<std::string::String>,
attempt_count: i32,
top_level_instruction_count: i64,
inner_instruction_count: i64,
top_level_program_count: i64,
inner_program_count: i64,
updated_at: std::string::String,
}
#[derive(sqlx::FromRow)]
struct ReplayProgramSummaryRow {
program_id: std::string::String,
transaction_count: i64,
top_level_instruction_count: i64,
inner_instruction_count: i64,
log_count: i64,
min_slot: i64,
max_slot: i64,
}
#[derive(sqlx::FromRow)]
struct ReplayEntitySummaryRow {
entity_kind: std::string::String,
entity_value: std::string::String,
transaction_count: i64,
occurrence_count: i64,
min_slot: i64,
max_slot: i64,
}
#[derive(Clone, Debug, serde::Deserialize, serde::Serialize)]
struct ReplayTransactionPageCursor {
slot: i64,
signature: std::string::String,
newest_first: bool,
}
#[derive(Clone, Debug, serde::Deserialize, serde::Serialize)]
struct ReplayProgramPageCursor {
transaction_count: i64,
program_id: std::string::String,
}
#[derive(Clone, Debug, serde::Deserialize, serde::Serialize)]
struct ReplayEntityPageCursor {
entity_kind: std::string::String,
transaction_count: i64,
entity_value: std::string::String,
}
pub(crate) async fn list_replay_transaction_candidates(
pool: &sqlx::PgPool,
filter: &crate::ReplayTransactionFilter,
page_request: &crate::PageRequest,
) -> ks_core::Result<crate::CountedPageSlice<crate::ReplayTransactionCandidate>> {
let page_result = validate_page_request(page_request);
if let std::result::Result::Err(error) = page_result {
return std::result::Result::Err(error);
}
let min_slot_result = optional_sql_bigint(filter.min_slot);
let min_slot = match min_slot_result {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let max_slot_result = optional_sql_bigint(filter.max_slot);
let max_slot = match max_slot_result {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let entity_kind = match filter.entity_kind {
std::option::Option::Some(value) => std::option::Option::Some(value.as_sql()),
std::option::Option::None => std::option::Option::None,
};
let cursor_result =
decode_transaction_cursor(page_request.cursor.as_deref(), filter.newest_first);
let cursor = match cursor_result {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let total_rows_result =
count_replay_transaction_candidates(pool, filter, min_slot, max_slot, entity_kind).await;
let total_rows = match total_rows_result {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let sql = if filter.newest_first {
transaction_candidate_sql_desc()
} else {
transaction_candidate_sql_asc()
};
let query_limit = i64::from(page_request.limit) + 1;
let started_at = std::time::Instant::now();
let query_result = sqlx::query_as::<sqlx::Postgres, ReplayTransactionCandidateRow>(sql)
.bind(filter.signature_contains.as_deref())
.bind(min_slot)
.bind(max_slot)
.bind(filter.raw_processing_state.as_deref())
.bind(filter.ledger_status.as_deref())
.bind(filter.program_id.as_deref())
.bind(filter.program_scope.as_sql())
.bind(entity_kind)
.bind(filter.entity_value.as_deref())
.bind(cursor.as_ref().map(|value| return value.slot))
.bind(cursor.as_ref().map(|value| return value.signature.as_str()))
.bind(query_limit)
.fetch_all(pool)
.await;
let rows = match query_result {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => {
trace_query_failure("list_replay_transaction_candidate_page", started_at, &error);
return std::result::Result::Err(ks_core::Error::db(format!(
"postgres replay transaction candidate query failed: {error}"
)));
},
};
let mut output = std::vec::Vec::with_capacity(rows.len().min(usize::from(page_request.limit)));
for row in rows.iter().take(usize::from(page_request.limit)) {
output.push(crate::ReplayTransactionCandidate {
signature: row.signature.clone(),
slot: row.slot,
raw_processing_state: row.raw_processing_state.clone(),
retention_state: row.retention_state.clone(),
has_core_transaction: row.has_core_transaction,
transaction_failed: row.transaction_failed,
ledger_status: row.ledger_status.clone(),
processor_version: row.processor_version.clone(),
attempt_count: row.attempt_count,
top_level_instruction_count: row.top_level_instruction_count,
inner_instruction_count: row.inner_instruction_count,
top_level_program_count: row.top_level_program_count,
inner_program_count: row.inner_program_count,
updated_at: row.updated_at.clone(),
});
}
let next_cursor = if rows.len() > usize::from(page_request.limit) {
match output.last() {
std::option::Option::Some(value) => {
let encoded_result = encode_transaction_cursor(value, filter.newest_first);
match encoded_result {
std::result::Result::Ok(value) => std::option::Option::Some(value),
std::result::Result::Err(error) => return std::result::Result::Err(error),
}
},
std::option::Option::None => std::option::Option::None,
}
} else {
std::option::Option::None
};
trace_query_success("list_replay_transaction_candidate_page", started_at, output.len());
return std::result::Result::Ok(crate::CountedPageSlice::new(output, next_cursor, total_rows));
}
pub(crate) async fn list_replay_program_summaries(
pool: &sqlx::PgPool,
filter: &crate::ReplayProgramFilter,
page_request: &crate::PageRequest,
) -> ks_core::Result<crate::CountedPageSlice<crate::ReplayProgramSummary>> {
let page_result = validate_page_request(page_request);
if let std::result::Result::Err(error) = page_result {
return std::result::Result::Err(error);
}
let cursor_result = decode_program_cursor(page_request.cursor.as_deref());
let cursor = match cursor_result {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let total_rows_result = count_replay_program_summaries(pool, filter).await;
let total_rows = match total_rows_result {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let query_limit = i64::from(page_request.limit) + 1;
let started_at = std::time::Instant::now();
let query_result = sqlx::query_as::<sqlx::Postgres, ReplayProgramSummaryRow>(PROGRAM_PAGE_SQL)
.bind(filter.program_id_contains.as_deref())
.bind(cursor.as_ref().map(|value| return value.transaction_count))
.bind(cursor.as_ref().map(|value| return value.program_id.as_str()))
.bind(query_limit)
.fetch_all(pool)
.await;
let rows = match query_result {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => {
trace_query_failure("list_replay_program_summary_page", started_at, &error);
return std::result::Result::Err(ks_core::Error::db(format!(
"postgres replay program summary query failed: {error}"
)));
},
};
let mut output = std::vec::Vec::with_capacity(rows.len().min(usize::from(page_request.limit)));
for row in rows.iter().take(usize::from(page_request.limit)) {
output.push(crate::ReplayProgramSummary {
program_id: row.program_id.clone(),
transaction_count: row.transaction_count,
top_level_instruction_count: row.top_level_instruction_count,
inner_instruction_count: row.inner_instruction_count,
log_count: row.log_count,
min_slot: row.min_slot,
max_slot: row.max_slot,
});
}
let next_cursor = if rows.len() > usize::from(page_request.limit) {
match output.last() {
std::option::Option::Some(value) => {
let encoded_result = encode_program_cursor(value);
match encoded_result {
std::result::Result::Ok(value) => std::option::Option::Some(value),
std::result::Result::Err(error) => return std::result::Result::Err(error),
}
},
std::option::Option::None => std::option::Option::None,
}
} else {
std::option::Option::None
};
trace_query_success("list_replay_program_summary_page", started_at, output.len());
return std::result::Result::Ok(crate::CountedPageSlice::new(output, next_cursor, total_rows));
}
pub(crate) async fn list_replay_entity_summaries(
pool: &sqlx::PgPool,
filter: &crate::ReplayEntityFilter,
page_request: &crate::PageRequest,
) -> ks_core::Result<crate::CountedPageSlice<crate::ReplayEntitySummary>> {
let page_result = validate_page_request(page_request);
if let std::result::Result::Err(error) = page_result {
return std::result::Result::Err(error);
}
let entity_kind = filter.entity_kind.as_sql();
let cursor_result = decode_entity_cursor(page_request.cursor.as_deref(), entity_kind);
let cursor = match cursor_result {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let total_rows_result = count_replay_entity_summaries(pool, filter).await;
let total_rows = match total_rows_result {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let query_limit = i64::from(page_request.limit) + 1;
let started_at = std::time::Instant::now();
let query_result = sqlx::query_as::<sqlx::Postgres, ReplayEntitySummaryRow>(ENTITY_PAGE_SQL)
.bind(entity_kind)
.bind(filter.entity_value_contains.as_deref())
.bind(cursor.as_ref().map(|value| return value.transaction_count))
.bind(cursor.as_ref().map(|value| return value.entity_value.as_str()))
.bind(query_limit)
.fetch_all(pool)
.await;
let rows = match query_result {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => {
trace_query_failure("list_replay_entity_summary_page", started_at, &error);
return std::result::Result::Err(ks_core::Error::db(format!(
"postgres replay entity summary query failed: {error}"
)));
},
};
let mut output = std::vec::Vec::with_capacity(rows.len().min(usize::from(page_request.limit)));
for row in rows.iter().take(usize::from(page_request.limit)) {
output.push(crate::ReplayEntitySummary {
entity_kind: row.entity_kind.clone(),
entity_value: row.entity_value.clone(),
transaction_count: row.transaction_count,
occurrence_count: row.occurrence_count,
min_slot: row.min_slot,
max_slot: row.max_slot,
});
}
let next_cursor = if rows.len() > usize::from(page_request.limit) {
match output.last() {
std::option::Option::Some(value) => {
let encoded_result = encode_entity_cursor(value);
match encoded_result {
std::result::Result::Ok(value) => std::option::Option::Some(value),
std::result::Result::Err(error) => return std::result::Result::Err(error),
}
},
std::option::Option::None => std::option::Option::None,
}
} else {
std::option::Option::None
};
trace_query_success("list_replay_entity_summary_page", started_at, output.len());
return std::result::Result::Ok(crate::CountedPageSlice::new(output, next_cursor, total_rows));
}
async fn count_replay_transaction_candidates(
pool: &sqlx::PgPool,
filter: &crate::ReplayTransactionFilter,
min_slot: std::option::Option<i64>,
max_slot: std::option::Option<i64>,
entity_kind: std::option::Option<&str>,
) -> ks_core::Result<u64> {
let started_at = std::time::Instant::now();
let query_result = sqlx::query_scalar::<sqlx::Postgres, i64>(TRANSACTION_COUNT_SQL)
.bind(filter.signature_contains.as_deref())
.bind(min_slot)
.bind(max_slot)
.bind(filter.raw_processing_state.as_deref())
.bind(filter.ledger_status.as_deref())
.bind(filter.program_id.as_deref())
.bind(filter.program_scope.as_sql())
.bind(entity_kind)
.bind(filter.entity_value.as_deref())
.fetch_one(pool)
.await;
let count = match query_result {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => {
trace_query_failure("count_replay_transaction_candidates", started_at, &error);
return std::result::Result::Err(ks_core::Error::db(format!(
"postgres replay transaction candidate count failed: {error}"
)));
},
};
trace_query_success("count_replay_transaction_candidates", started_at, 1);
return unsigned_count(count, "replay transaction candidate count is negative");
}
async fn count_replay_program_summaries(
pool: &sqlx::PgPool,
filter: &crate::ReplayProgramFilter,
) -> ks_core::Result<u64> {
let started_at = std::time::Instant::now();
let query_result = sqlx::query_scalar::<sqlx::Postgres, i64>(PROGRAM_COUNT_SQL)
.bind(filter.program_id_contains.as_deref())
.fetch_one(pool)
.await;
let count = match query_result {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => {
trace_query_failure("count_replay_program_summaries", started_at, &error);
return std::result::Result::Err(ks_core::Error::db(format!(
"postgres replay program summary count failed: {error}"
)));
},
};
trace_query_success("count_replay_program_summaries", started_at, 1);
return unsigned_count(count, "replay program summary count is negative");
}
async fn count_replay_entity_summaries(
pool: &sqlx::PgPool,
filter: &crate::ReplayEntityFilter,
) -> ks_core::Result<u64> {
let started_at = std::time::Instant::now();
let query_result = sqlx::query_scalar::<sqlx::Postgres, i64>(ENTITY_COUNT_SQL)
.bind(filter.entity_kind.as_sql())
.bind(filter.entity_value_contains.as_deref())
.fetch_one(pool)
.await;
let count = match query_result {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => {
trace_query_failure("count_replay_entity_summaries", started_at, &error);
return std::result::Result::Err(ks_core::Error::db(format!(
"postgres replay entity summary count failed: {error}"
)));
},
};
trace_query_success("count_replay_entity_summaries", started_at, 1);
return unsigned_count(count, "replay entity summary count is negative");
}
fn validate_page_request(page_request: &crate::PageRequest) -> ks_core::Result<()> {
if page_request.limit == 0 || page_request.limit > crate::MAX_PAGE_SIZE {
return std::result::Result::Err(ks_core::Error::db(
"replay candidate page size is outside repository bounds",
));
}
return std::result::Result::Ok(());
}
fn unsigned_count(value: i64, message: &str) -> ks_core::Result<u64> {
let conversion_result = u64::try_from(value);
return match conversion_result {
std::result::Result::Ok(converted) => std::result::Result::Ok(converted),
std::result::Result::Err(_error) => std::result::Result::Err(ks_core::Error::db(message)),
};
}
fn decode_transaction_cursor(
cursor: std::option::Option<&str>,
newest_first: bool,
) -> ks_core::Result<std::option::Option<ReplayTransactionPageCursor>> {
let decoded_result = decode_cursor::<ReplayTransactionPageCursor>(cursor, "transaction");
let decoded = match decoded_result {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
if decoded.as_ref().is_some_and(|value| {
return value.slot < 0
|| value.signature.trim().is_empty()
|| value.newest_first != newest_first;
}) {
return std::result::Result::Err(ks_core::Error::db(
"replay transaction page cursor values are invalid",
));
}
return std::result::Result::Ok(decoded);
}
fn decode_program_cursor(
cursor: std::option::Option<&str>,
) -> ks_core::Result<std::option::Option<ReplayProgramPageCursor>> {
let decoded_result = decode_cursor::<ReplayProgramPageCursor>(cursor, "program");
let decoded = match decoded_result {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
if decoded.as_ref().is_some_and(|value| {
return value.transaction_count < 0 || value.program_id.trim().is_empty();
}) {
return std::result::Result::Err(ks_core::Error::db(
"replay program page cursor values are invalid",
));
}
return std::result::Result::Ok(decoded);
}
fn decode_entity_cursor(
cursor: std::option::Option<&str>,
entity_kind: &str,
) -> ks_core::Result<std::option::Option<ReplayEntityPageCursor>> {
let decoded_result = decode_cursor::<ReplayEntityPageCursor>(cursor, "entity");
let decoded = match decoded_result {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
if decoded.as_ref().is_some_and(|value| {
return value.transaction_count < 0
|| value.entity_value.trim().is_empty()
|| value.entity_kind != entity_kind;
}) {
return std::result::Result::Err(ks_core::Error::db(
"replay entity page cursor values are invalid",
));
}
return std::result::Result::Ok(decoded);
}
fn decode_cursor<T>(
cursor: std::option::Option<&str>,
label: &str,
) -> ks_core::Result<std::option::Option<T>>
where
T: serde::de::DeserializeOwned,
{
let raw = match cursor {
std::option::Option::Some(value) => value,
std::option::Option::None => return std::result::Result::Ok(std::option::Option::None),
};
let decode_result = serde_json::from_str::<T>(raw);
return match decode_result {
std::result::Result::Ok(value) => std::result::Result::Ok(std::option::Option::Some(value)),
std::result::Result::Err(_error) => std::result::Result::Err(ks_core::Error::db(format!(
"replay {label} page cursor is invalid"
))),
};
}
fn encode_transaction_cursor(
row: &crate::ReplayTransactionCandidate,
newest_first: bool,
) -> ks_core::Result<std::string::String> {
return encode_cursor(
&ReplayTransactionPageCursor {
slot: row.slot,
signature: row.signature.clone(),
newest_first,
},
"transaction",
);
}
fn encode_program_cursor(
row: &crate::ReplayProgramSummary,
) -> ks_core::Result<std::string::String> {
return encode_cursor(
&ReplayProgramPageCursor {
transaction_count: row.transaction_count,
program_id: row.program_id.clone(),
},
"program",
);
}
fn encode_entity_cursor(row: &crate::ReplayEntitySummary) -> ks_core::Result<std::string::String> {
return encode_cursor(
&ReplayEntityPageCursor {
entity_kind: row.entity_kind.clone(),
transaction_count: row.transaction_count,
entity_value: row.entity_value.clone(),
},
"entity",
);
}
fn encode_cursor<T>(value: &T, label: &str) -> ks_core::Result<std::string::String>
where
T: serde::Serialize,
{
let encode_result = serde_json::to_string(value);
return match encode_result {
std::result::Result::Ok(encoded) if encoded.len() <= crate::MAX_PAGE_CURSOR_LENGTH => {
std::result::Result::Ok(encoded)
},
std::result::Result::Ok(_encoded) => std::result::Result::Err(ks_core::Error::db(format!(
"replay {label} page cursor exceeds maximum encoded length"
))),
std::result::Result::Err(error) => std::result::Result::Err(ks_core::Error::db(format!(
"replay {label} page cursor serialization failed: {error}"
))),
};
}
const TRANSACTION_COUNT_SQL: &str = r#"SELECT COUNT(*)::BIGINT
FROM k_sol_raw_transactions raw
LEFT JOIN LATERAL (
SELECT status
FROM k_sol_ops_processing_ledger
WHERE stage = 'core_extraction'
AND processor_name = 'canonical_to_core'
AND input_key = raw.signature
ORDER BY updated_at DESC, id DESC
LIMIT 1
) ledger ON TRUE
WHERE ($1::TEXT IS NULL OR raw.signature ILIKE '%' || $1 || '%')
AND ($2::BIGINT IS NULL OR raw.slot >= $2)
AND ($3::BIGINT IS NULL OR raw.slot <= $3)
AND ($4::TEXT IS NULL OR raw.processing_state = $4)
AND ($5::TEXT IS NULL OR ($5 = 'not_started' AND ledger.status IS NULL) OR ledger.status = $5)
AND ($6::TEXT IS NULL OR
($7 = 'any' AND (
EXISTS (SELECT 1 FROM k_sol_core_instructions candidate_top_level WHERE candidate_top_level.signature = raw.signature AND candidate_top_level.program_id = $6) OR
EXISTS (SELECT 1 FROM k_sol_core_inner_instructions candidate_inner WHERE candidate_inner.signature = raw.signature AND candidate_inner.program_id = $6) OR
EXISTS (SELECT 1 FROM k_sol_core_logs candidate_log WHERE candidate_log.signature = raw.signature AND candidate_log.program_id = $6)
)) OR
($7 = 'top_level' AND EXISTS (SELECT 1 FROM k_sol_core_instructions candidate_top_level WHERE candidate_top_level.signature = raw.signature AND candidate_top_level.program_id = $6)) OR
($7 = 'inner' AND EXISTS (SELECT 1 FROM k_sol_core_inner_instructions candidate_inner WHERE candidate_inner.signature = raw.signature AND candidate_inner.program_id = $6)) OR
($7 = 'logs' AND EXISTS (SELECT 1 FROM k_sol_core_logs candidate_log WHERE candidate_log.signature = raw.signature AND candidate_log.program_id = $6)))
AND ($8::TEXT IS NULL OR
($8 = 'mint' AND EXISTS (SELECT 1 FROM k_sol_core_balance_changes candidate_balance WHERE candidate_balance.signature = raw.signature AND candidate_balance.mint = $9)) OR
($8 = 'owner' AND EXISTS (SELECT 1 FROM k_sol_core_balance_changes candidate_balance WHERE candidate_balance.signature = raw.signature AND candidate_balance.owner = $9)) OR
($8 = 'account_key' AND EXISTS (SELECT 1 FROM k_sol_core_account_keys candidate_account WHERE candidate_account.signature = raw.signature AND candidate_account.account_key = $9)))"#;
const PROGRAM_PAGE_SQL: &str = r#"WITH occurrences AS (
SELECT program_id, signature, slot, 'top_level'::TEXT AS scope FROM k_sol_core_instructions
UNION ALL
SELECT program_id, signature, slot, 'inner'::TEXT AS scope FROM k_sol_core_inner_instructions
UNION ALL
SELECT program_id, signature, slot, 'logs'::TEXT AS scope FROM k_sol_core_logs WHERE program_id IS NOT NULL
), summaries AS (
SELECT program_id,
COUNT(DISTINCT signature)::BIGINT AS transaction_count,
COUNT(*) FILTER (WHERE scope = 'top_level')::BIGINT AS top_level_instruction_count,
COUNT(*) FILTER (WHERE scope = 'inner')::BIGINT AS inner_instruction_count,
COUNT(*) FILTER (WHERE scope = 'logs')::BIGINT AS log_count,
MIN(slot)::BIGINT AS min_slot,
MAX(slot)::BIGINT AS max_slot
FROM occurrences
WHERE ($1::TEXT IS NULL OR program_id ILIKE '%' || $1 || '%')
GROUP BY program_id
)
SELECT program_id, transaction_count, top_level_instruction_count, inner_instruction_count, log_count, min_slot, max_slot
FROM summaries
WHERE ($2::BIGINT IS NULL OR transaction_count < $2 OR (transaction_count = $2 AND program_id > $3))
ORDER BY transaction_count DESC, program_id ASC
LIMIT $4"#;
const PROGRAM_COUNT_SQL: &str = r#"WITH occurrences AS (
SELECT program_id FROM k_sol_core_instructions
UNION ALL
SELECT program_id FROM k_sol_core_inner_instructions
UNION ALL
SELECT program_id FROM k_sol_core_logs WHERE program_id IS NOT NULL
), summaries AS (
SELECT program_id
FROM occurrences
WHERE ($1::TEXT IS NULL OR program_id ILIKE '%' || $1 || '%')
GROUP BY program_id
)
SELECT COUNT(*)::BIGINT FROM summaries"#;
const ENTITY_PAGE_SQL: &str = r#"WITH entities AS (
SELECT 'mint'::TEXT AS entity_kind, mint AS entity_value, signature, slot FROM k_sol_core_balance_changes WHERE mint IS NOT NULL
UNION ALL
SELECT 'owner'::TEXT AS entity_kind, owner AS entity_value, signature, slot FROM k_sol_core_balance_changes WHERE owner IS NOT NULL
UNION ALL
SELECT 'account_key'::TEXT AS entity_kind, account_key AS entity_value, signature, slot FROM k_sol_core_account_keys
), summaries AS (
SELECT entity_kind,
entity_value,
COUNT(DISTINCT signature)::BIGINT AS transaction_count,
COUNT(*)::BIGINT AS occurrence_count,
MIN(slot)::BIGINT AS min_slot,
MAX(slot)::BIGINT AS max_slot
FROM entities
WHERE entity_kind = $1
AND ($2::TEXT IS NULL OR entity_value ILIKE '%' || $2 || '%')
GROUP BY entity_kind, entity_value
)
SELECT entity_kind, entity_value, transaction_count, occurrence_count, min_slot, max_slot
FROM summaries
WHERE ($3::BIGINT IS NULL OR transaction_count < $3 OR (transaction_count = $3 AND entity_value > $4))
ORDER BY transaction_count DESC, entity_value ASC
LIMIT $5"#;
const ENTITY_COUNT_SQL: &str = r#"WITH entities AS (
SELECT 'mint'::TEXT AS entity_kind, mint AS entity_value FROM k_sol_core_balance_changes WHERE mint IS NOT NULL
UNION ALL
SELECT 'owner'::TEXT AS entity_kind, owner AS entity_value FROM k_sol_core_balance_changes WHERE owner IS NOT NULL
UNION ALL
SELECT 'account_key'::TEXT AS entity_kind, account_key AS entity_value FROM k_sol_core_account_keys
), summaries AS (
SELECT entity_kind, entity_value
FROM entities
WHERE entity_kind = $1
AND ($2::TEXT IS NULL OR entity_value ILIKE '%' || $2 || '%')
GROUP BY entity_kind, entity_value
)
SELECT COUNT(*)::BIGINT FROM summaries"#;
fn transaction_candidate_sql_desc() -> &'static str {
return crate::postgres::query::replay_candidate_queries::transaction_candidate_sql("DESC");
}
fn transaction_candidate_sql_asc() -> &'static str {
return crate::postgres::query::replay_candidate_queries::transaction_candidate_sql("ASC");
}
fn transaction_candidate_sql(order: &str) -> &'static str {
if order == "DESC" {
return r#"SELECT raw.signature,
raw.slot,
raw.processing_state AS raw_processing_state,
raw.retention_state,
(core.id IS NOT NULL) AS has_core_transaction,
core.failed AS transaction_failed,
COALESCE(ledger.status, 'not_started') AS ledger_status,
ledger.processor_version,
COALESCE(ledger.attempt_count, 0)::INTEGER AS attempt_count,
COALESCE(top_level_stats.instruction_count, 0)::BIGINT AS top_level_instruction_count,
COALESCE(inner_stats.instruction_count, 0)::BIGINT AS inner_instruction_count,
COALESCE(top_level_stats.program_count, 0)::BIGINT AS top_level_program_count,
COALESCE(inner_stats.program_count, 0)::BIGINT AS inner_program_count,
raw.updated_at::TEXT AS updated_at
FROM k_sol_raw_transactions raw
LEFT JOIN k_sol_core_transactions core ON core.signature = raw.signature
LEFT JOIN LATERAL (
SELECT status, processor_version, attempt_count
FROM k_sol_ops_processing_ledger
WHERE stage = 'core_extraction'
AND processor_name = 'canonical_to_core'
AND input_key = raw.signature
ORDER BY updated_at DESC, id DESC
LIMIT 1
) ledger ON TRUE
LEFT JOIN LATERAL (
SELECT COUNT(*)::BIGINT AS instruction_count, COUNT(DISTINCT program_id)::BIGINT AS program_count
FROM k_sol_core_instructions
WHERE signature = raw.signature
) top_level_stats ON TRUE
LEFT JOIN LATERAL (
SELECT COUNT(*)::BIGINT AS instruction_count, COUNT(DISTINCT program_id)::BIGINT AS program_count
FROM k_sol_core_inner_instructions
WHERE signature = raw.signature
) inner_stats ON TRUE
WHERE ($1::TEXT IS NULL OR raw.signature ILIKE '%' || $1 || '%')
AND ($2::BIGINT IS NULL OR raw.slot >= $2)
AND ($3::BIGINT IS NULL OR raw.slot <= $3)
AND ($4::TEXT IS NULL OR raw.processing_state = $4)
AND ($5::TEXT IS NULL OR ($5 = 'not_started' AND ledger.status IS NULL) OR ledger.status = $5)
AND ($6::TEXT IS NULL OR
($7 = 'any' AND (
EXISTS (SELECT 1 FROM k_sol_core_instructions candidate_top_level WHERE candidate_top_level.signature = raw.signature AND candidate_top_level.program_id = $6) OR
EXISTS (SELECT 1 FROM k_sol_core_inner_instructions candidate_inner WHERE candidate_inner.signature = raw.signature AND candidate_inner.program_id = $6) OR
EXISTS (SELECT 1 FROM k_sol_core_logs candidate_log WHERE candidate_log.signature = raw.signature AND candidate_log.program_id = $6)
)) OR
($7 = 'top_level' AND EXISTS (SELECT 1 FROM k_sol_core_instructions candidate_top_level WHERE candidate_top_level.signature = raw.signature AND candidate_top_level.program_id = $6)) OR
($7 = 'inner' AND EXISTS (SELECT 1 FROM k_sol_core_inner_instructions candidate_inner WHERE candidate_inner.signature = raw.signature AND candidate_inner.program_id = $6)) OR
($7 = 'logs' AND EXISTS (SELECT 1 FROM k_sol_core_logs candidate_log WHERE candidate_log.signature = raw.signature AND candidate_log.program_id = $6)))
AND ($8::TEXT IS NULL OR
($8 = 'mint' AND EXISTS (
SELECT 1 FROM k_sol_core_balance_changes candidate_balance
WHERE candidate_balance.signature = raw.signature
AND candidate_balance.mint = $9
)) OR
($8 = 'owner' AND EXISTS (
SELECT 1 FROM k_sol_core_balance_changes candidate_balance
WHERE candidate_balance.signature = raw.signature
AND candidate_balance.owner = $9
)) OR
($8 = 'account_key' AND EXISTS (
SELECT 1 FROM k_sol_core_account_keys candidate_account
WHERE candidate_account.signature = raw.signature
AND candidate_account.account_key = $9
)))
AND ($10::BIGINT IS NULL OR raw.slot < $10 OR (raw.slot = $10 AND raw.signature < $11))
ORDER BY raw.slot DESC, raw.signature DESC
LIMIT $12"#;
}
return r#"SELECT raw.signature,
raw.slot,
raw.processing_state AS raw_processing_state,
raw.retention_state,
(core.id IS NOT NULL) AS has_core_transaction,
core.failed AS transaction_failed,
COALESCE(ledger.status, 'not_started') AS ledger_status,
ledger.processor_version,
COALESCE(ledger.attempt_count, 0)::INTEGER AS attempt_count,
COALESCE(top_level_stats.instruction_count, 0)::BIGINT AS top_level_instruction_count,
COALESCE(inner_stats.instruction_count, 0)::BIGINT AS inner_instruction_count,
COALESCE(top_level_stats.program_count, 0)::BIGINT AS top_level_program_count,
COALESCE(inner_stats.program_count, 0)::BIGINT AS inner_program_count,
raw.updated_at::TEXT AS updated_at
FROM k_sol_raw_transactions raw
LEFT JOIN k_sol_core_transactions core ON core.signature = raw.signature
LEFT JOIN LATERAL (
SELECT status, processor_version, attempt_count
FROM k_sol_ops_processing_ledger
WHERE stage = 'core_extraction'
AND processor_name = 'canonical_to_core'
AND input_key = raw.signature
ORDER BY updated_at DESC, id DESC
LIMIT 1
) ledger ON TRUE
LEFT JOIN LATERAL (
SELECT COUNT(*)::BIGINT AS instruction_count, COUNT(DISTINCT program_id)::BIGINT AS program_count
FROM k_sol_core_instructions
WHERE signature = raw.signature
) top_level_stats ON TRUE
LEFT JOIN LATERAL (
SELECT COUNT(*)::BIGINT AS instruction_count, COUNT(DISTINCT program_id)::BIGINT AS program_count
FROM k_sol_core_inner_instructions
WHERE signature = raw.signature
) inner_stats ON TRUE
WHERE ($1::TEXT IS NULL OR raw.signature ILIKE '%' || $1 || '%')
AND ($2::BIGINT IS NULL OR raw.slot >= $2)
AND ($3::BIGINT IS NULL OR raw.slot <= $3)
AND ($4::TEXT IS NULL OR raw.processing_state = $4)
AND ($5::TEXT IS NULL OR ($5 = 'not_started' AND ledger.status IS NULL) OR ledger.status = $5)
AND ($6::TEXT IS NULL OR
($7 = 'any' AND (
EXISTS (SELECT 1 FROM k_sol_core_instructions candidate_top_level WHERE candidate_top_level.signature = raw.signature AND candidate_top_level.program_id = $6) OR
EXISTS (SELECT 1 FROM k_sol_core_inner_instructions candidate_inner WHERE candidate_inner.signature = raw.signature AND candidate_inner.program_id = $6) OR
EXISTS (SELECT 1 FROM k_sol_core_logs candidate_log WHERE candidate_log.signature = raw.signature AND candidate_log.program_id = $6)
)) OR
($7 = 'top_level' AND EXISTS (SELECT 1 FROM k_sol_core_instructions candidate_top_level WHERE candidate_top_level.signature = raw.signature AND candidate_top_level.program_id = $6)) OR
($7 = 'inner' AND EXISTS (SELECT 1 FROM k_sol_core_inner_instructions candidate_inner WHERE candidate_inner.signature = raw.signature AND candidate_inner.program_id = $6)) OR
($7 = 'logs' AND EXISTS (SELECT 1 FROM k_sol_core_logs candidate_log WHERE candidate_log.signature = raw.signature AND candidate_log.program_id = $6)))
AND ($8::TEXT IS NULL OR
($8 = 'mint' AND EXISTS (
SELECT 1 FROM k_sol_core_balance_changes candidate_balance
WHERE candidate_balance.signature = raw.signature
AND candidate_balance.mint = $9
)) OR
($8 = 'owner' AND EXISTS (
SELECT 1 FROM k_sol_core_balance_changes candidate_balance
WHERE candidate_balance.signature = raw.signature
AND candidate_balance.owner = $9
)) OR
($8 = 'account_key' AND EXISTS (
SELECT 1 FROM k_sol_core_account_keys candidate_account
WHERE candidate_account.signature = raw.signature
AND candidate_account.account_key = $9
)))
AND ($10::BIGINT IS NULL OR raw.slot > $10 OR (raw.slot = $10 AND raw.signature > $11))
ORDER BY raw.slot ASC, raw.signature ASC
LIMIT $12"#;
}
fn optional_sql_bigint(
value: std::option::Option<u64>,
) -> ks_core::Result<std::option::Option<i64>> {
return match value {
std::option::Option::Some(raw_value) => {
let conversion_result = i64::try_from(raw_value);
match conversion_result {
std::result::Result::Ok(converted) => {
std::result::Result::Ok(std::option::Option::Some(converted))
},
std::result::Result::Err(error) => std::result::Result::Err(ks_core::Error::db(
format!("replay candidate slot does not fit into PostgreSQL BIGINT: {error}"),
)),
}
},
std::option::Option::None => std::result::Result::Ok(std::option::Option::None),
};
}
fn trace_query_success(action: &str, started_at: std::time::Instant, rows: usize) {
let elapsed_ms = started_at.elapsed().as_secs_f64() * 1000.0;
tracing::trace!(target: crate::TRACING_TARGET, backend = "postgres", domain = "ks-store.pg", action = action, elapsed_ms, rows, outcome = "success", "PostgreSQL store operation completed");
}
fn trace_query_failure(action: &str, started_at: std::time::Instant, error: &sqlx::Error) {
let elapsed_ms = started_at.elapsed().as_secs_f64() * 1000.0;
tracing::trace!(target: crate::TRACING_TARGET, backend = "postgres", domain = "ks-store.pg", action = action, elapsed_ms, error = %error, outcome = "error", "PostgreSQL store operation failed");
}
#[cfg(test)]
mod tests {
#[test]
fn replay_candidate_queries_are_cursor_based_and_counted() {
let source = include_str!("replay_candidate_queries.rs");
let tests_marker = source.find("#[cfg(test)]");
let production_source = match tests_marker {
std::option::Option::Some(index) => &source[..index],
std::option::Option::None => source,
};
assert!(!production_source.contains(" OFFSET "));
assert!(production_source.contains("TRANSACTION_COUNT_SQL"));
assert!(production_source.contains("PROGRAM_COUNT_SQL"));
assert!(production_source.contains("ENTITY_COUNT_SQL"));
assert!(production_source.contains("CountedPageSlice::new"));
}
#[test]
fn transaction_cursor_rejects_order_direction_changes() {
let row = crate::ReplayTransactionCandidate {
signature: "signature".to_string(),
slot: 42,
raw_processing_state: "received".to_string(),
retention_state: "retained".to_string(),
has_core_transaction: false,
transaction_failed: std::option::Option::None,
ledger_status: "not_started".to_string(),
processor_version: std::option::Option::None,
attempt_count: 0,
top_level_instruction_count: 0,
inner_instruction_count: 0,
top_level_program_count: 0,
inner_program_count: 0,
updated_at: "now".to_string(),
};
let encoded_result = super::encode_transaction_cursor(&row, true);
let encoded = match encoded_result {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => panic!("unexpected encode error: {error}"),
};
let decoded_result =
super::decode_transaction_cursor(std::option::Option::Some(encoded.as_str()), false);
assert!(decoded_result.is_err());
}
#[tokio::test]
async fn optional_postgres_replay_candidate_queries_from_env() {
let database_url = match std::env::var("KS_SECRET_POSTGRES_TEST_URL") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_error) => return,
};
let _postgres_guard = crate::postgres_test_guard().await;
let options_result = crate::PostgresStoreOptions::new(database_url, 1, 5000, false);
let options = match options_result {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => panic!("unexpected options error: {error}"),
};
let store_result = crate::PostgresStore::connect(options).await;
let store = match store_result {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => panic!("unexpected connect error: {error}"),
};
let schema_result = store.initialize_missing_store_schema().await;
if let std::result::Result::Err(error) = schema_result {
panic!("unexpected store schema error: {error}");
}
let transaction_filter_result = crate::ReplayTransactionFilter::new(
std::option::Option::None,
std::option::Option::None,
std::option::Option::None,
std::option::Option::None,
std::option::Option::None,
std::option::Option::None,
crate::ReplayProgramScope::Any,
std::option::Option::None,
std::option::Option::None,
true,
);
let transaction_filter = match transaction_filter_result {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => {
panic!("unexpected transaction filter error: {error}")
},
};
let transaction_page = crate::PageRequest::new(10, std::option::Option::None);
let transaction_page = match transaction_page {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => panic!("unexpected transaction page error: {error}"),
};
let transaction_result = store
.replay_transaction_candidates(&transaction_filter, &transaction_page)
.await;
if let std::result::Result::Err(error) = transaction_result {
panic!("unexpected transaction candidate query error: {error}");
}
let program_filter_result = crate::ReplayProgramFilter::new(std::option::Option::None);
let program_filter = match program_filter_result {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => panic!("unexpected program filter error: {error}"),
};
let program_result =
store.replay_program_summaries(&program_filter, &transaction_page).await;
if let std::result::Result::Err(error) = program_result {
panic!("unexpected program summary query error: {error}");
}
for entity_kind in [
crate::ReplayEntityKind::Mint,
crate::ReplayEntityKind::Owner,
crate::ReplayEntityKind::AccountKey,
] {
let entity_filter_result =
crate::ReplayEntityFilter::new(entity_kind, std::option::Option::None);
let entity_filter = match entity_filter_result {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => {
panic!("unexpected entity filter error: {error}")
},
};
let entity_result =
store.replay_entity_summaries(&entity_filter, &transaction_page).await;
if let std::result::Result::Err(error) = entity_result {
panic!("unexpected entity summary query error: {error}");
}
}
}
}