Sasedev/khadhroony-bot3
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

0.5.1-pre.002

Browse Source
This commit is contained in:
SinuS von SifriduS
2026-08-09 19:34:08 +02:00
parent 816eee59a9
commit 6a680767ae
767 changed files with 12257 additions and 12195 deletions
Download Patch File Download Diff File Expand all files Collapse all files

474
ks-store/src/postgres/query/replay_candidate_queries.rs Normal file
View File

@@ -0,0 +1,474 @@
// file: ks-store/src/postgres/query/replay_candidate_queries.rs
// version: 3
//! Read-only PostgreSQL queries for replay candidate discovery.
#[derive(sqlx::FromRow)]
struct ReplayTransactionCandidateRow {
signature: std::string::String,
slot: i64,
raw_processing_state: std::string::String,
retention_state: std::string::String,
has_core_transaction: bool,
transaction_failed: std::option::Option<bool>,
ledger_status: std::string::String,
processor_version: std::option::Option<std::string::String>,
attempt_count: i32,
outer_instruction_count: i64,
inner_instruction_count: i64,
outer_program_count: i64,
inner_program_count: i64,
updated_at: std::string::String,
}
#[derive(sqlx::FromRow)]
struct ReplayProgramSummaryRow {
program_id: std::string::String,
transaction_count: i64,
outer_instruction_count: i64,
inner_instruction_count: i64,
log_count: i64,
min_slot: i64,
max_slot: i64,
}
#[derive(sqlx::FromRow)]
struct ReplayEntitySummaryRow {
entity_kind: std::string::String,
entity_value: std::string::String,
transaction_count: i64,
occurrence_count: i64,
min_slot: i64,
max_slot: i64,
}
pub(in crate::postgres) async fn list_replay_transaction_candidates(
pool: &sqlx::PgPool,
filter: &crate::PostgresReplayTransactionFilter,
) -> ks_core::Result<std::vec::Vec<crate::PostgresReplayTransactionCandidate>> {
let min_slot_result =
crate::postgres::query::replay_candidate_queries::optional_sql_bigint(filter.min_slot);
let min_slot = match min_slot_result {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let max_slot_result =
crate::postgres::query::replay_candidate_queries::optional_sql_bigint(filter.max_slot);
let max_slot = match max_slot_result {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let entity_kind = match filter.entity_kind {
std::option::Option::Some(value) => std::option::Option::Some(value.as_sql()),
std::option::Option::None => std::option::Option::None,
};
let sql = if filter.newest_first {
crate::postgres::query::replay_candidate_queries::transaction_candidate_sql_desc()
} else {
crate::postgres::query::replay_candidate_queries::transaction_candidate_sql_asc()
};
let query_result = sqlx::query_as::<
sqlx::Postgres,
crate::postgres::query::replay_candidate_queries::ReplayTransactionCandidateRow,
>(sql)
.bind(filter.signature_contains.as_deref())
.bind(min_slot)
.bind(max_slot)
.bind(filter.raw_processing_state.as_deref())
.bind(filter.ledger_status.as_deref())
.bind(filter.program_id.as_deref())
.bind(filter.program_scope.as_sql())
.bind(entity_kind)
.bind(filter.entity_value.as_deref())
.bind(i64::from(filter.limit))
.fetch_all(pool)
.await;
let rows = match query_result {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => {
return std::result::Result::Err(ks_core::Error::db(format!(
"postgres replay transaction candidate query failed: {error}"
)));
},
};
let mut output = std::vec::Vec::with_capacity(rows.len());
for row in rows {
output.push(crate::PostgresReplayTransactionCandidate {
signature: row.signature,
slot: row.slot,
raw_processing_state: row.raw_processing_state,
retention_state: row.retention_state,
has_core_transaction: row.has_core_transaction,
transaction_failed: row.transaction_failed,
ledger_status: row.ledger_status,
processor_version: row.processor_version,
attempt_count: row.attempt_count,
outer_instruction_count: row.outer_instruction_count,
inner_instruction_count: row.inner_instruction_count,
outer_program_count: row.outer_program_count,
inner_program_count: row.inner_program_count,
updated_at: row.updated_at,
});
}
return std::result::Result::Ok(output);
}
pub(in crate::postgres) async fn list_replay_program_summaries(
pool: &sqlx::PgPool,
filter: &crate::PostgresReplayProgramFilter,
) -> ks_core::Result<std::vec::Vec<crate::PostgresReplayProgramSummary>> {
let query_result = sqlx::query_as::<sqlx::Postgres, crate::postgres::query::replay_candidate_queries::ReplayProgramSummaryRow>(
r#"WITH occurrences AS (
SELECT program_id, signature, slot, 'outer'::TEXT AS scope FROM kb_sol_core_instructions
UNION ALL
SELECT program_id, signature, slot, 'inner'::TEXT AS scope FROM kb_sol_core_inner_instructions
UNION ALL
SELECT program_id, signature, slot, 'logs'::TEXT AS scope FROM kb_sol_core_logs WHERE program_id IS NOT NULL
)
SELECT program_id,
COUNT(DISTINCT signature)::BIGINT AS transaction_count,
COUNT(*) FILTER (WHERE scope = 'outer')::BIGINT AS outer_instruction_count,
COUNT(*) FILTER (WHERE scope = 'inner')::BIGINT AS inner_instruction_count,
COUNT(*) FILTER (WHERE scope = 'logs')::BIGINT AS log_count,
MIN(slot)::BIGINT AS min_slot,
MAX(slot)::BIGINT AS max_slot
FROM occurrences
WHERE ($1::TEXT IS NULL OR program_id ILIKE '%' || $1 || '%')
GROUP BY program_id
ORDER BY transaction_count DESC, program_id ASC
LIMIT $2"#,
)
.bind(filter.program_id_contains.as_deref())
.bind(i64::from(filter.limit))
.fetch_all(pool)
.await;
let rows = match query_result {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => {
return std::result::Result::Err(ks_core::Error::db(format!(
"postgres replay program summary query failed: {error}"
)));
},
};
let mut output = std::vec::Vec::with_capacity(rows.len());
for row in rows {
output.push(crate::PostgresReplayProgramSummary {
program_id: row.program_id,
transaction_count: row.transaction_count,
outer_instruction_count: row.outer_instruction_count,
inner_instruction_count: row.inner_instruction_count,
log_count: row.log_count,
min_slot: row.min_slot,
max_slot: row.max_slot,
});
}
return std::result::Result::Ok(output);
}
pub(in crate::postgres) async fn list_replay_entity_summaries(
pool: &sqlx::PgPool,
filter: &crate::PostgresReplayEntityFilter,
) -> ks_core::Result<std::vec::Vec<crate::PostgresReplayEntitySummary>> {
let entity_kind = filter.entity_kind.as_sql();
let query_result = sqlx::query_as::<
sqlx::Postgres,
crate::postgres::query::replay_candidate_queries::ReplayEntitySummaryRow,
>(
r#"WITH entities AS (
SELECT 'mint'::TEXT AS entity_kind, mint AS entity_value, signature, slot
FROM kb_sol_core_balance_changes
WHERE mint IS NOT NULL
UNION ALL
SELECT 'owner'::TEXT AS entity_kind, owner AS entity_value, signature, slot
FROM kb_sol_core_balance_changes
WHERE owner IS NOT NULL
UNION ALL
SELECT 'account_key'::TEXT AS entity_kind, account_key AS entity_value, signature, slot
FROM kb_sol_core_account_keys
)
SELECT entity_kind,
entity_value,
COUNT(DISTINCT signature)::BIGINT AS transaction_count,
COUNT(*)::BIGINT AS occurrence_count,
MIN(slot)::BIGINT AS min_slot,
MAX(slot)::BIGINT AS max_slot
FROM entities
WHERE entity_kind = $1
AND ($2::TEXT IS NULL OR entity_value ILIKE '%' || $2 || '%')
GROUP BY entity_kind, entity_value
ORDER BY transaction_count DESC, entity_value ASC
LIMIT $3"#,
)
.bind(entity_kind)
.bind(filter.entity_value_contains.as_deref())
.bind(i64::from(filter.limit))
.fetch_all(pool)
.await;
let rows = match query_result {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => {
return std::result::Result::Err(ks_core::Error::db(format!(
"postgres replay entity summary query failed: {error}"
)));
},
};
let mut output = std::vec::Vec::with_capacity(rows.len());
for row in rows {
output.push(crate::PostgresReplayEntitySummary {
entity_kind: row.entity_kind,
entity_value: row.entity_value,
transaction_count: row.transaction_count,
occurrence_count: row.occurrence_count,
min_slot: row.min_slot,
max_slot: row.max_slot,
});
}
return std::result::Result::Ok(output);
}
fn transaction_candidate_sql_desc() -> &'static str {
return crate::postgres::query::replay_candidate_queries::transaction_candidate_sql("DESC");
}
fn transaction_candidate_sql_asc() -> &'static str {
return crate::postgres::query::replay_candidate_queries::transaction_candidate_sql("ASC");
}
fn transaction_candidate_sql(order: &str) -> &'static str {
if order == "DESC" {
return r#"SELECT raw.signature,
raw.slot,
raw.processing_state AS raw_processing_state,
raw.retention_state,
(core.id IS NOT NULL) AS has_core_transaction,
core.failed AS transaction_failed,
COALESCE(ledger.status, 'not_started') AS ledger_status,
ledger.processor_version,
COALESCE(ledger.attempt_count, 0)::INTEGER AS attempt_count,
COALESCE(outer_stats.instruction_count, 0)::BIGINT AS outer_instruction_count,
COALESCE(inner_stats.instruction_count, 0)::BIGINT AS inner_instruction_count,
COALESCE(outer_stats.program_count, 0)::BIGINT AS outer_program_count,
COALESCE(inner_stats.program_count, 0)::BIGINT AS inner_program_count,
raw.updated_at::TEXT AS updated_at
FROM kb_sol_raw_transactions raw
LEFT JOIN kb_sol_core_transactions core ON core.signature = raw.signature
LEFT JOIN LATERAL (
SELECT status, processor_version, attempt_count
FROM kb_sol_ops_processing_ledger
WHERE stage = 'core_extraction'
AND processor_name = 'canonical_to_core'
AND input_key = raw.signature
ORDER BY updated_at DESC, id DESC
LIMIT 1
) ledger ON TRUE
LEFT JOIN LATERAL (
SELECT COUNT(*)::BIGINT AS instruction_count, COUNT(DISTINCT program_id)::BIGINT AS program_count
FROM kb_sol_core_instructions
WHERE signature = raw.signature
) outer_stats ON TRUE
LEFT JOIN LATERAL (
SELECT COUNT(*)::BIGINT AS instruction_count, COUNT(DISTINCT program_id)::BIGINT AS program_count
FROM kb_sol_core_inner_instructions
WHERE signature = raw.signature
) inner_stats ON TRUE
WHERE ($1::TEXT IS NULL OR raw.signature ILIKE '%' || $1 || '%')
AND ($2::BIGINT IS NULL OR raw.slot >= $2)
AND ($3::BIGINT IS NULL OR raw.slot <= $3)
AND ($4::TEXT IS NULL OR raw.processing_state = $4)
AND ($5::TEXT IS NULL OR ($5 = 'not_started' AND ledger.status IS NULL) OR ledger.status = $5)
AND ($6::TEXT IS NULL OR
($7 = 'any' AND (
EXISTS (SELECT 1 FROM kb_sol_core_instructions candidate_outer WHERE candidate_outer.signature = raw.signature AND candidate_outer.program_id = $6) OR
EXISTS (SELECT 1 FROM kb_sol_core_inner_instructions candidate_inner WHERE candidate_inner.signature = raw.signature AND candidate_inner.program_id = $6) OR
EXISTS (SELECT 1 FROM kb_sol_core_logs candidate_log WHERE candidate_log.signature = raw.signature AND candidate_log.program_id = $6)
)) OR
($7 = 'outer' AND EXISTS (SELECT 1 FROM kb_sol_core_instructions candidate_outer WHERE candidate_outer.signature = raw.signature AND candidate_outer.program_id = $6)) OR
($7 = 'inner' AND EXISTS (SELECT 1 FROM kb_sol_core_inner_instructions candidate_inner WHERE candidate_inner.signature = raw.signature AND candidate_inner.program_id = $6)) OR
($7 = 'logs' AND EXISTS (SELECT 1 FROM kb_sol_core_logs candidate_log WHERE candidate_log.signature = raw.signature AND candidate_log.program_id = $6)))
AND ($8::TEXT IS NULL OR
($8 = 'mint' AND EXISTS (
SELECT 1 FROM kb_sol_core_balance_changes candidate_balance
WHERE candidate_balance.signature = raw.signature
AND candidate_balance.mint = $9
)) OR
($8 = 'owner' AND EXISTS (
SELECT 1 FROM kb_sol_core_balance_changes candidate_balance
WHERE candidate_balance.signature = raw.signature
AND candidate_balance.owner = $9
)) OR
($8 = 'account_key' AND EXISTS (
SELECT 1 FROM kb_sol_core_account_keys candidate_account
WHERE candidate_account.signature = raw.signature
AND candidate_account.account_key = $9
)))
ORDER BY raw.slot DESC, raw.signature DESC
LIMIT $10"#;
}
return r#"SELECT raw.signature,
raw.slot,
raw.processing_state AS raw_processing_state,
raw.retention_state,
(core.id IS NOT NULL) AS has_core_transaction,
core.failed AS transaction_failed,
COALESCE(ledger.status, 'not_started') AS ledger_status,
ledger.processor_version,
COALESCE(ledger.attempt_count, 0)::INTEGER AS attempt_count,
COALESCE(outer_stats.instruction_count, 0)::BIGINT AS outer_instruction_count,
COALESCE(inner_stats.instruction_count, 0)::BIGINT AS inner_instruction_count,
COALESCE(outer_stats.program_count, 0)::BIGINT AS outer_program_count,
COALESCE(inner_stats.program_count, 0)::BIGINT AS inner_program_count,
raw.updated_at::TEXT AS updated_at
FROM kb_sol_raw_transactions raw
LEFT JOIN kb_sol_core_transactions core ON core.signature = raw.signature
LEFT JOIN LATERAL (
SELECT status, processor_version, attempt_count
FROM kb_sol_ops_processing_ledger
WHERE stage = 'core_extraction'
AND processor_name = 'canonical_to_core'
AND input_key = raw.signature
ORDER BY updated_at DESC, id DESC
LIMIT 1
) ledger ON TRUE
LEFT JOIN LATERAL (
SELECT COUNT(*)::BIGINT AS instruction_count, COUNT(DISTINCT program_id)::BIGINT AS program_count
FROM kb_sol_core_instructions
WHERE signature = raw.signature
) outer_stats ON TRUE
LEFT JOIN LATERAL (
SELECT COUNT(*)::BIGINT AS instruction_count, COUNT(DISTINCT program_id)::BIGINT AS program_count
FROM kb_sol_core_inner_instructions
WHERE signature = raw.signature
) inner_stats ON TRUE
WHERE ($1::TEXT IS NULL OR raw.signature ILIKE '%' || $1 || '%')
AND ($2::BIGINT IS NULL OR raw.slot >= $2)
AND ($3::BIGINT IS NULL OR raw.slot <= $3)
AND ($4::TEXT IS NULL OR raw.processing_state = $4)
AND ($5::TEXT IS NULL OR ($5 = 'not_started' AND ledger.status IS NULL) OR ledger.status = $5)
AND ($6::TEXT IS NULL OR
($7 = 'any' AND (
EXISTS (SELECT 1 FROM kb_sol_core_instructions candidate_outer WHERE candidate_outer.signature = raw.signature AND candidate_outer.program_id = $6) OR
EXISTS (SELECT 1 FROM kb_sol_core_inner_instructions candidate_inner WHERE candidate_inner.signature = raw.signature AND candidate_inner.program_id = $6) OR
EXISTS (SELECT 1 FROM kb_sol_core_logs candidate_log WHERE candidate_log.signature = raw.signature AND candidate_log.program_id = $6)
)) OR
($7 = 'outer' AND EXISTS (SELECT 1 FROM kb_sol_core_instructions candidate_outer WHERE candidate_outer.signature = raw.signature AND candidate_outer.program_id = $6)) OR
($7 = 'inner' AND EXISTS (SELECT 1 FROM kb_sol_core_inner_instructions candidate_inner WHERE candidate_inner.signature = raw.signature AND candidate_inner.program_id = $6)) OR
($7 = 'logs' AND EXISTS (SELECT 1 FROM kb_sol_core_logs candidate_log WHERE candidate_log.signature = raw.signature AND candidate_log.program_id = $6)))
AND ($8::TEXT IS NULL OR
($8 = 'mint' AND EXISTS (
SELECT 1 FROM kb_sol_core_balance_changes candidate_balance
WHERE candidate_balance.signature = raw.signature
AND candidate_balance.mint = $9
)) OR
($8 = 'owner' AND EXISTS (
SELECT 1 FROM kb_sol_core_balance_changes candidate_balance
WHERE candidate_balance.signature = raw.signature
AND candidate_balance.owner = $9
)) OR
($8 = 'account_key' AND EXISTS (
SELECT 1 FROM kb_sol_core_account_keys candidate_account
WHERE candidate_account.signature = raw.signature
AND candidate_account.account_key = $9
)))
ORDER BY raw.slot ASC, raw.signature ASC
LIMIT $10"#;
}
fn optional_sql_bigint(
value: std::option::Option<u64>,
) -> ks_core::Result<std::option::Option<i64>> {
return match value {
std::option::Option::Some(raw_value) => {
let conversion_result = i64::try_from(raw_value);
match conversion_result {
std::result::Result::Ok(converted) => {
std::result::Result::Ok(std::option::Option::Some(converted))
},
std::result::Result::Err(error) => std::result::Result::Err(ks_core::Error::db(
format!("replay candidate slot does not fit into PostgreSQL BIGINT: {error}"),
)),
}
},
std::option::Option::None => std::result::Result::Ok(std::option::Option::None),
};
}
#[cfg(test)]
mod tests {
#[tokio::test]
async fn optional_postgres_replay_candidate_queries_from_env() {
let database_url = match std::env::var("KB_POSTGRES_TEST_URL") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_error) => return,
};
let _postgres_guard = crate::postgres::test_serial::postgres_test_guard().await;
let options_result = crate::PostgresStoreOptions::new(database_url, 1, 5000, false);
let options = match options_result {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => panic!("unexpected options error: {error}"),
};
let store_result = crate::PostgresStore::connect(options).await;
let store = match store_result {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => panic!("unexpected connect error: {error}"),
};
let raw_schema_result = store.initialize_raw_store_schema().await;
if let std::result::Result::Err(error) = raw_schema_result {
panic!("unexpected raw schema error: {error}");
}
let core_schema_result = store.initialize_core_store_schema().await;
if let std::result::Result::Err(error) = core_schema_result {
panic!("unexpected core schema error: {error}");
}
let transaction_filter_result = crate::PostgresReplayTransactionFilter::new(
std::option::Option::None,
std::option::Option::None,
std::option::Option::None,
std::option::Option::None,
std::option::Option::None,
std::option::Option::None,
crate::PostgresReplayProgramScope::Any,
std::option::Option::None,
std::option::Option::None,
10,
true,
);
let transaction_filter = match transaction_filter_result {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => {
panic!("unexpected transaction filter error: {error}")
},
};
let transaction_result = store.replay_transaction_candidates(&transaction_filter).await;
if let std::result::Result::Err(error) = transaction_result {
panic!("unexpected transaction candidate query error: {error}");
}
let program_filter_result =
crate::PostgresReplayProgramFilter::new(std::option::Option::None, 10);
let program_filter = match program_filter_result {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => panic!("unexpected program filter error: {error}"),
};
let program_result = store.replay_program_summaries(&program_filter).await;
if let std::result::Result::Err(error) = program_result {
panic!("unexpected program summary query error: {error}");
}
for entity_kind in [
crate::PostgresReplayEntityKind::Mint,
crate::PostgresReplayEntityKind::Owner,
crate::PostgresReplayEntityKind::AccountKey,
] {
let entity_filter_result =
crate::PostgresReplayEntityFilter::new(entity_kind, std::option::Option::None, 10);
let entity_filter = match entity_filter_result {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => {
panic!("unexpected entity filter error: {error}")
},
};
let entity_result = store.replay_entity_summaries(&entity_filter).await;
if let std::result::Result::Err(error) = entity_result {
panic!("unexpected entity summary query error: {error}");
}
}
}
}