v0.5.3-pre.005-fix010
This commit is contained in:
@@ -1,5 +1,5 @@
|
||||
// file: ks-store/src/postgres/query/replay_candidate_queries.rs
|
||||
// version: 8
|
||||
// version: 10
|
||||
|
||||
//! Read-only PostgreSQL queries for replay candidate discovery.
|
||||
|
||||
@@ -42,18 +42,41 @@ struct ReplayEntitySummaryRow {
|
||||
max_slot: i64,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, serde::Deserialize, serde::Serialize)]
|
||||
struct ReplayTransactionPageCursor {
|
||||
slot: i64,
|
||||
signature: std::string::String,
|
||||
newest_first: bool,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, serde::Deserialize, serde::Serialize)]
|
||||
struct ReplayProgramPageCursor {
|
||||
transaction_count: i64,
|
||||
program_id: std::string::String,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, serde::Deserialize, serde::Serialize)]
|
||||
struct ReplayEntityPageCursor {
|
||||
entity_kind: std::string::String,
|
||||
transaction_count: i64,
|
||||
entity_value: std::string::String,
|
||||
}
|
||||
|
||||
pub(crate) async fn list_replay_transaction_candidates(
|
||||
pool: &sqlx::PgPool,
|
||||
filter: &crate::ReplayTransactionFilter,
|
||||
) -> ks_core::Result<std::vec::Vec<crate::ReplayTransactionCandidate>> {
|
||||
let min_slot_result =
|
||||
crate::postgres::query::replay_candidate_queries::optional_sql_bigint(filter.min_slot);
|
||||
page_request: &crate::PageRequest,
|
||||
) -> ks_core::Result<crate::CountedPageSlice<crate::ReplayTransactionCandidate>> {
|
||||
let page_result = validate_page_request(page_request);
|
||||
if let std::result::Result::Err(error) = page_result {
|
||||
return std::result::Result::Err(error);
|
||||
}
|
||||
let min_slot_result = optional_sql_bigint(filter.min_slot);
|
||||
let min_slot = match min_slot_result {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(error) => return std::result::Result::Err(error),
|
||||
};
|
||||
let max_slot_result =
|
||||
crate::postgres::query::replay_candidate_queries::optional_sql_bigint(filter.max_slot);
|
||||
let max_slot_result = optional_sql_bigint(filter.max_slot);
|
||||
let max_slot = match max_slot_result {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(error) => return std::result::Result::Err(error),
|
||||
@@ -62,103 +85,127 @@ pub(crate) async fn list_replay_transaction_candidates(
|
||||
std::option::Option::Some(value) => std::option::Option::Some(value.as_sql()),
|
||||
std::option::Option::None => std::option::Option::None,
|
||||
};
|
||||
let sql = if filter.newest_first {
|
||||
crate::postgres::query::replay_candidate_queries::transaction_candidate_sql_desc()
|
||||
} else {
|
||||
crate::postgres::query::replay_candidate_queries::transaction_candidate_sql_asc()
|
||||
let cursor_result =
|
||||
decode_transaction_cursor(page_request.cursor.as_deref(), filter.newest_first);
|
||||
let cursor = match cursor_result {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(error) => return std::result::Result::Err(error),
|
||||
};
|
||||
let total_rows_result =
|
||||
count_replay_transaction_candidates(pool, filter, min_slot, max_slot, entity_kind).await;
|
||||
let total_rows = match total_rows_result {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(error) => return std::result::Result::Err(error),
|
||||
};
|
||||
let sql = if filter.newest_first {
|
||||
transaction_candidate_sql_desc()
|
||||
} else {
|
||||
transaction_candidate_sql_asc()
|
||||
};
|
||||
let query_limit = i64::from(page_request.limit) + 1;
|
||||
let started_at = std::time::Instant::now();
|
||||
let query_result = sqlx::query_as::<
|
||||
sqlx::Postgres,
|
||||
crate::postgres::query::replay_candidate_queries::ReplayTransactionCandidateRow,
|
||||
>(sql)
|
||||
.bind(filter.signature_contains.as_deref())
|
||||
.bind(min_slot)
|
||||
.bind(max_slot)
|
||||
.bind(filter.raw_processing_state.as_deref())
|
||||
.bind(filter.ledger_status.as_deref())
|
||||
.bind(filter.program_id.as_deref())
|
||||
.bind(filter.program_scope.as_sql())
|
||||
.bind(entity_kind)
|
||||
.bind(filter.entity_value.as_deref())
|
||||
.bind(i64::from(filter.limit))
|
||||
.fetch_all(pool)
|
||||
.await;
|
||||
let query_result = sqlx::query_as::<sqlx::Postgres, ReplayTransactionCandidateRow>(sql)
|
||||
.bind(filter.signature_contains.as_deref())
|
||||
.bind(min_slot)
|
||||
.bind(max_slot)
|
||||
.bind(filter.raw_processing_state.as_deref())
|
||||
.bind(filter.ledger_status.as_deref())
|
||||
.bind(filter.program_id.as_deref())
|
||||
.bind(filter.program_scope.as_sql())
|
||||
.bind(entity_kind)
|
||||
.bind(filter.entity_value.as_deref())
|
||||
.bind(cursor.as_ref().map(|value| return value.slot))
|
||||
.bind(cursor.as_ref().map(|value| return value.signature.as_str()))
|
||||
.bind(query_limit)
|
||||
.fetch_all(pool)
|
||||
.await;
|
||||
let rows = match query_result {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(error) => {
|
||||
trace_query_failure("list_replay_transaction_candidates", started_at, &error);
|
||||
trace_query_failure("list_replay_transaction_candidate_page", started_at, &error);
|
||||
return std::result::Result::Err(ks_core::Error::db(format!(
|
||||
"postgres replay transaction candidate query failed: {error}"
|
||||
)));
|
||||
},
|
||||
};
|
||||
let mut output = std::vec::Vec::with_capacity(rows.len());
|
||||
for row in rows {
|
||||
let mut output = std::vec::Vec::with_capacity(rows.len().min(usize::from(page_request.limit)));
|
||||
for row in rows.iter().take(usize::from(page_request.limit)) {
|
||||
output.push(crate::ReplayTransactionCandidate {
|
||||
signature: row.signature,
|
||||
signature: row.signature.clone(),
|
||||
slot: row.slot,
|
||||
raw_processing_state: row.raw_processing_state,
|
||||
retention_state: row.retention_state,
|
||||
raw_processing_state: row.raw_processing_state.clone(),
|
||||
retention_state: row.retention_state.clone(),
|
||||
has_core_transaction: row.has_core_transaction,
|
||||
transaction_failed: row.transaction_failed,
|
||||
ledger_status: row.ledger_status,
|
||||
processor_version: row.processor_version,
|
||||
ledger_status: row.ledger_status.clone(),
|
||||
processor_version: row.processor_version.clone(),
|
||||
attempt_count: row.attempt_count,
|
||||
top_level_instruction_count: row.top_level_instruction_count,
|
||||
inner_instruction_count: row.inner_instruction_count,
|
||||
top_level_program_count: row.top_level_program_count,
|
||||
inner_program_count: row.inner_program_count,
|
||||
updated_at: row.updated_at,
|
||||
updated_at: row.updated_at.clone(),
|
||||
});
|
||||
}
|
||||
trace_query_success("list_replay_transaction_candidates", started_at, output.len());
|
||||
return std::result::Result::Ok(output);
|
||||
let next_cursor = if rows.len() > usize::from(page_request.limit) {
|
||||
match output.last() {
|
||||
std::option::Option::Some(value) => {
|
||||
let encoded_result = encode_transaction_cursor(value, filter.newest_first);
|
||||
match encoded_result {
|
||||
std::result::Result::Ok(value) => std::option::Option::Some(value),
|
||||
std::result::Result::Err(error) => return std::result::Result::Err(error),
|
||||
}
|
||||
},
|
||||
std::option::Option::None => std::option::Option::None,
|
||||
}
|
||||
} else {
|
||||
std::option::Option::None
|
||||
};
|
||||
trace_query_success("list_replay_transaction_candidate_page", started_at, output.len());
|
||||
return std::result::Result::Ok(crate::CountedPageSlice::new(output, next_cursor, total_rows));
|
||||
}
|
||||
|
||||
pub(crate) async fn list_replay_program_summaries(
|
||||
pool: &sqlx::PgPool,
|
||||
filter: &crate::ReplayProgramFilter,
|
||||
) -> ks_core::Result<std::vec::Vec<crate::ReplayProgramSummary>> {
|
||||
page_request: &crate::PageRequest,
|
||||
) -> ks_core::Result<crate::CountedPageSlice<crate::ReplayProgramSummary>> {
|
||||
let page_result = validate_page_request(page_request);
|
||||
if let std::result::Result::Err(error) = page_result {
|
||||
return std::result::Result::Err(error);
|
||||
}
|
||||
let cursor_result = decode_program_cursor(page_request.cursor.as_deref());
|
||||
let cursor = match cursor_result {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(error) => return std::result::Result::Err(error),
|
||||
};
|
||||
let total_rows_result = count_replay_program_summaries(pool, filter).await;
|
||||
let total_rows = match total_rows_result {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(error) => return std::result::Result::Err(error),
|
||||
};
|
||||
let query_limit = i64::from(page_request.limit) + 1;
|
||||
let started_at = std::time::Instant::now();
|
||||
let query_result = sqlx::query_as::<sqlx::Postgres, crate::postgres::query::replay_candidate_queries::ReplayProgramSummaryRow>(
|
||||
r#"WITH occurrences AS (
|
||||
SELECT program_id, signature, slot, 'top_level'::TEXT AS scope FROM k_sol_core_instructions
|
||||
UNION ALL
|
||||
SELECT program_id, signature, slot, 'inner'::TEXT AS scope FROM k_sol_core_inner_instructions
|
||||
UNION ALL
|
||||
SELECT program_id, signature, slot, 'logs'::TEXT AS scope FROM k_sol_core_logs WHERE program_id IS NOT NULL
|
||||
)
|
||||
SELECT program_id,
|
||||
COUNT(DISTINCT signature)::BIGINT AS transaction_count,
|
||||
COUNT(*) FILTER (WHERE scope = 'top_level')::BIGINT AS top_level_instruction_count,
|
||||
COUNT(*) FILTER (WHERE scope = 'inner')::BIGINT AS inner_instruction_count,
|
||||
COUNT(*) FILTER (WHERE scope = 'logs')::BIGINT AS log_count,
|
||||
MIN(slot)::BIGINT AS min_slot,
|
||||
MAX(slot)::BIGINT AS max_slot
|
||||
FROM occurrences
|
||||
WHERE ($1::TEXT IS NULL OR program_id ILIKE '%' || $1 || '%')
|
||||
GROUP BY program_id
|
||||
ORDER BY transaction_count DESC, program_id ASC
|
||||
LIMIT $2"#,
|
||||
)
|
||||
.bind(filter.program_id_contains.as_deref())
|
||||
.bind(i64::from(filter.limit))
|
||||
.fetch_all(pool)
|
||||
.await;
|
||||
let query_result = sqlx::query_as::<sqlx::Postgres, ReplayProgramSummaryRow>(PROGRAM_PAGE_SQL)
|
||||
.bind(filter.program_id_contains.as_deref())
|
||||
.bind(cursor.as_ref().map(|value| return value.transaction_count))
|
||||
.bind(cursor.as_ref().map(|value| return value.program_id.as_str()))
|
||||
.bind(query_limit)
|
||||
.fetch_all(pool)
|
||||
.await;
|
||||
let rows = match query_result {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(error) => {
|
||||
trace_query_failure("list_replay_program_summaries", started_at, &error);
|
||||
trace_query_failure("list_replay_program_summary_page", started_at, &error);
|
||||
return std::result::Result::Err(ks_core::Error::db(format!(
|
||||
"postgres replay program summary query failed: {error}"
|
||||
)));
|
||||
},
|
||||
};
|
||||
let mut output = std::vec::Vec::with_capacity(rows.len());
|
||||
for row in rows {
|
||||
let mut output = std::vec::Vec::with_capacity(rows.len().min(usize::from(page_request.limit)));
|
||||
for row in rows.iter().take(usize::from(page_request.limit)) {
|
||||
output.push(crate::ReplayProgramSummary {
|
||||
program_id: row.program_id,
|
||||
program_id: row.program_id.clone(),
|
||||
transaction_count: row.transaction_count,
|
||||
top_level_instruction_count: row.top_level_instruction_count,
|
||||
inner_instruction_count: row.inner_instruction_count,
|
||||
@@ -167,74 +214,429 @@ pub(crate) async fn list_replay_program_summaries(
|
||||
max_slot: row.max_slot,
|
||||
});
|
||||
}
|
||||
trace_query_success("list_replay_program_summaries", started_at, output.len());
|
||||
return std::result::Result::Ok(output);
|
||||
let next_cursor = if rows.len() > usize::from(page_request.limit) {
|
||||
match output.last() {
|
||||
std::option::Option::Some(value) => {
|
||||
let encoded_result = encode_program_cursor(value);
|
||||
match encoded_result {
|
||||
std::result::Result::Ok(value) => std::option::Option::Some(value),
|
||||
std::result::Result::Err(error) => return std::result::Result::Err(error),
|
||||
}
|
||||
},
|
||||
std::option::Option::None => std::option::Option::None,
|
||||
}
|
||||
} else {
|
||||
std::option::Option::None
|
||||
};
|
||||
trace_query_success("list_replay_program_summary_page", started_at, output.len());
|
||||
return std::result::Result::Ok(crate::CountedPageSlice::new(output, next_cursor, total_rows));
|
||||
}
|
||||
|
||||
pub(crate) async fn list_replay_entity_summaries(
|
||||
pool: &sqlx::PgPool,
|
||||
filter: &crate::ReplayEntityFilter,
|
||||
) -> ks_core::Result<std::vec::Vec<crate::ReplayEntitySummary>> {
|
||||
page_request: &crate::PageRequest,
|
||||
) -> ks_core::Result<crate::CountedPageSlice<crate::ReplayEntitySummary>> {
|
||||
let page_result = validate_page_request(page_request);
|
||||
if let std::result::Result::Err(error) = page_result {
|
||||
return std::result::Result::Err(error);
|
||||
}
|
||||
let entity_kind = filter.entity_kind.as_sql();
|
||||
let cursor_result = decode_entity_cursor(page_request.cursor.as_deref(), entity_kind);
|
||||
let cursor = match cursor_result {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(error) => return std::result::Result::Err(error),
|
||||
};
|
||||
let total_rows_result = count_replay_entity_summaries(pool, filter).await;
|
||||
let total_rows = match total_rows_result {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(error) => return std::result::Result::Err(error),
|
||||
};
|
||||
let query_limit = i64::from(page_request.limit) + 1;
|
||||
let started_at = std::time::Instant::now();
|
||||
let query_result = sqlx::query_as::<
|
||||
sqlx::Postgres,
|
||||
crate::postgres::query::replay_candidate_queries::ReplayEntitySummaryRow,
|
||||
>(
|
||||
r#"WITH entities AS (
|
||||
SELECT 'mint'::TEXT AS entity_kind, mint AS entity_value, signature, slot
|
||||
FROM k_sol_core_balance_changes
|
||||
WHERE mint IS NOT NULL
|
||||
UNION ALL
|
||||
SELECT 'owner'::TEXT AS entity_kind, owner AS entity_value, signature, slot
|
||||
FROM k_sol_core_balance_changes
|
||||
WHERE owner IS NOT NULL
|
||||
UNION ALL
|
||||
SELECT 'account_key'::TEXT AS entity_kind, account_key AS entity_value, signature, slot
|
||||
FROM k_sol_core_account_keys
|
||||
)
|
||||
SELECT entity_kind,
|
||||
entity_value,
|
||||
COUNT(DISTINCT signature)::BIGINT AS transaction_count,
|
||||
COUNT(*)::BIGINT AS occurrence_count,
|
||||
MIN(slot)::BIGINT AS min_slot,
|
||||
MAX(slot)::BIGINT AS max_slot
|
||||
FROM entities
|
||||
WHERE entity_kind = $1
|
||||
AND ($2::TEXT IS NULL OR entity_value ILIKE '%' || $2 || '%')
|
||||
GROUP BY entity_kind, entity_value
|
||||
ORDER BY transaction_count DESC, entity_value ASC
|
||||
LIMIT $3"#,
|
||||
)
|
||||
.bind(entity_kind)
|
||||
.bind(filter.entity_value_contains.as_deref())
|
||||
.bind(i64::from(filter.limit))
|
||||
.fetch_all(pool)
|
||||
.await;
|
||||
let query_result = sqlx::query_as::<sqlx::Postgres, ReplayEntitySummaryRow>(ENTITY_PAGE_SQL)
|
||||
.bind(entity_kind)
|
||||
.bind(filter.entity_value_contains.as_deref())
|
||||
.bind(cursor.as_ref().map(|value| return value.transaction_count))
|
||||
.bind(cursor.as_ref().map(|value| return value.entity_value.as_str()))
|
||||
.bind(query_limit)
|
||||
.fetch_all(pool)
|
||||
.await;
|
||||
let rows = match query_result {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(error) => {
|
||||
trace_query_failure("list_replay_entity_summaries", started_at, &error);
|
||||
trace_query_failure("list_replay_entity_summary_page", started_at, &error);
|
||||
return std::result::Result::Err(ks_core::Error::db(format!(
|
||||
"postgres replay entity summary query failed: {error}"
|
||||
)));
|
||||
},
|
||||
};
|
||||
let mut output = std::vec::Vec::with_capacity(rows.len());
|
||||
for row in rows {
|
||||
let mut output = std::vec::Vec::with_capacity(rows.len().min(usize::from(page_request.limit)));
|
||||
for row in rows.iter().take(usize::from(page_request.limit)) {
|
||||
output.push(crate::ReplayEntitySummary {
|
||||
entity_kind: row.entity_kind,
|
||||
entity_value: row.entity_value,
|
||||
entity_kind: row.entity_kind.clone(),
|
||||
entity_value: row.entity_value.clone(),
|
||||
transaction_count: row.transaction_count,
|
||||
occurrence_count: row.occurrence_count,
|
||||
min_slot: row.min_slot,
|
||||
max_slot: row.max_slot,
|
||||
});
|
||||
}
|
||||
trace_query_success("list_replay_entity_summaries", started_at, output.len());
|
||||
return std::result::Result::Ok(output);
|
||||
let next_cursor = if rows.len() > usize::from(page_request.limit) {
|
||||
match output.last() {
|
||||
std::option::Option::Some(value) => {
|
||||
let encoded_result = encode_entity_cursor(value);
|
||||
match encoded_result {
|
||||
std::result::Result::Ok(value) => std::option::Option::Some(value),
|
||||
std::result::Result::Err(error) => return std::result::Result::Err(error),
|
||||
}
|
||||
},
|
||||
std::option::Option::None => std::option::Option::None,
|
||||
}
|
||||
} else {
|
||||
std::option::Option::None
|
||||
};
|
||||
trace_query_success("list_replay_entity_summary_page", started_at, output.len());
|
||||
return std::result::Result::Ok(crate::CountedPageSlice::new(output, next_cursor, total_rows));
|
||||
}
|
||||
|
||||
async fn count_replay_transaction_candidates(
|
||||
pool: &sqlx::PgPool,
|
||||
filter: &crate::ReplayTransactionFilter,
|
||||
min_slot: std::option::Option<i64>,
|
||||
max_slot: std::option::Option<i64>,
|
||||
entity_kind: std::option::Option<&str>,
|
||||
) -> ks_core::Result<u64> {
|
||||
let started_at = std::time::Instant::now();
|
||||
let query_result = sqlx::query_scalar::<sqlx::Postgres, i64>(TRANSACTION_COUNT_SQL)
|
||||
.bind(filter.signature_contains.as_deref())
|
||||
.bind(min_slot)
|
||||
.bind(max_slot)
|
||||
.bind(filter.raw_processing_state.as_deref())
|
||||
.bind(filter.ledger_status.as_deref())
|
||||
.bind(filter.program_id.as_deref())
|
||||
.bind(filter.program_scope.as_sql())
|
||||
.bind(entity_kind)
|
||||
.bind(filter.entity_value.as_deref())
|
||||
.fetch_one(pool)
|
||||
.await;
|
||||
let count = match query_result {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(error) => {
|
||||
trace_query_failure("count_replay_transaction_candidates", started_at, &error);
|
||||
return std::result::Result::Err(ks_core::Error::db(format!(
|
||||
"postgres replay transaction candidate count failed: {error}"
|
||||
)));
|
||||
},
|
||||
};
|
||||
trace_query_success("count_replay_transaction_candidates", started_at, 1);
|
||||
return unsigned_count(count, "replay transaction candidate count is negative");
|
||||
}
|
||||
|
||||
async fn count_replay_program_summaries(
|
||||
pool: &sqlx::PgPool,
|
||||
filter: &crate::ReplayProgramFilter,
|
||||
) -> ks_core::Result<u64> {
|
||||
let started_at = std::time::Instant::now();
|
||||
let query_result = sqlx::query_scalar::<sqlx::Postgres, i64>(PROGRAM_COUNT_SQL)
|
||||
.bind(filter.program_id_contains.as_deref())
|
||||
.fetch_one(pool)
|
||||
.await;
|
||||
let count = match query_result {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(error) => {
|
||||
trace_query_failure("count_replay_program_summaries", started_at, &error);
|
||||
return std::result::Result::Err(ks_core::Error::db(format!(
|
||||
"postgres replay program summary count failed: {error}"
|
||||
)));
|
||||
},
|
||||
};
|
||||
trace_query_success("count_replay_program_summaries", started_at, 1);
|
||||
return unsigned_count(count, "replay program summary count is negative");
|
||||
}
|
||||
|
||||
async fn count_replay_entity_summaries(
|
||||
pool: &sqlx::PgPool,
|
||||
filter: &crate::ReplayEntityFilter,
|
||||
) -> ks_core::Result<u64> {
|
||||
let started_at = std::time::Instant::now();
|
||||
let query_result = sqlx::query_scalar::<sqlx::Postgres, i64>(ENTITY_COUNT_SQL)
|
||||
.bind(filter.entity_kind.as_sql())
|
||||
.bind(filter.entity_value_contains.as_deref())
|
||||
.fetch_one(pool)
|
||||
.await;
|
||||
let count = match query_result {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(error) => {
|
||||
trace_query_failure("count_replay_entity_summaries", started_at, &error);
|
||||
return std::result::Result::Err(ks_core::Error::db(format!(
|
||||
"postgres replay entity summary count failed: {error}"
|
||||
)));
|
||||
},
|
||||
};
|
||||
trace_query_success("count_replay_entity_summaries", started_at, 1);
|
||||
return unsigned_count(count, "replay entity summary count is negative");
|
||||
}
|
||||
|
||||
fn validate_page_request(page_request: &crate::PageRequest) -> ks_core::Result<()> {
|
||||
if page_request.limit == 0 || page_request.limit > crate::MAX_PAGE_SIZE {
|
||||
return std::result::Result::Err(ks_core::Error::db(
|
||||
"replay candidate page size is outside repository bounds",
|
||||
));
|
||||
}
|
||||
return std::result::Result::Ok(());
|
||||
}
|
||||
|
||||
fn unsigned_count(value: i64, message: &str) -> ks_core::Result<u64> {
|
||||
let conversion_result = u64::try_from(value);
|
||||
return match conversion_result {
|
||||
std::result::Result::Ok(converted) => std::result::Result::Ok(converted),
|
||||
std::result::Result::Err(_error) => std::result::Result::Err(ks_core::Error::db(message)),
|
||||
};
|
||||
}
|
||||
|
||||
fn decode_transaction_cursor(
|
||||
cursor: std::option::Option<&str>,
|
||||
newest_first: bool,
|
||||
) -> ks_core::Result<std::option::Option<ReplayTransactionPageCursor>> {
|
||||
let decoded_result = decode_cursor::<ReplayTransactionPageCursor>(cursor, "transaction");
|
||||
let decoded = match decoded_result {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(error) => return std::result::Result::Err(error),
|
||||
};
|
||||
if decoded.as_ref().is_some_and(|value| {
|
||||
return value.slot < 0
|
||||
|| value.signature.trim().is_empty()
|
||||
|| value.newest_first != newest_first;
|
||||
}) {
|
||||
return std::result::Result::Err(ks_core::Error::db(
|
||||
"replay transaction page cursor values are invalid",
|
||||
));
|
||||
}
|
||||
return std::result::Result::Ok(decoded);
|
||||
}
|
||||
|
||||
fn decode_program_cursor(
|
||||
cursor: std::option::Option<&str>,
|
||||
) -> ks_core::Result<std::option::Option<ReplayProgramPageCursor>> {
|
||||
let decoded_result = decode_cursor::<ReplayProgramPageCursor>(cursor, "program");
|
||||
let decoded = match decoded_result {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(error) => return std::result::Result::Err(error),
|
||||
};
|
||||
if decoded.as_ref().is_some_and(|value| {
|
||||
return value.transaction_count < 0 || value.program_id.trim().is_empty();
|
||||
}) {
|
||||
return std::result::Result::Err(ks_core::Error::db(
|
||||
"replay program page cursor values are invalid",
|
||||
));
|
||||
}
|
||||
return std::result::Result::Ok(decoded);
|
||||
}
|
||||
|
||||
fn decode_entity_cursor(
|
||||
cursor: std::option::Option<&str>,
|
||||
entity_kind: &str,
|
||||
) -> ks_core::Result<std::option::Option<ReplayEntityPageCursor>> {
|
||||
let decoded_result = decode_cursor::<ReplayEntityPageCursor>(cursor, "entity");
|
||||
let decoded = match decoded_result {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(error) => return std::result::Result::Err(error),
|
||||
};
|
||||
if decoded.as_ref().is_some_and(|value| {
|
||||
return value.transaction_count < 0
|
||||
|| value.entity_value.trim().is_empty()
|
||||
|| value.entity_kind != entity_kind;
|
||||
}) {
|
||||
return std::result::Result::Err(ks_core::Error::db(
|
||||
"replay entity page cursor values are invalid",
|
||||
));
|
||||
}
|
||||
return std::result::Result::Ok(decoded);
|
||||
}
|
||||
|
||||
fn decode_cursor<T>(
|
||||
cursor: std::option::Option<&str>,
|
||||
label: &str,
|
||||
) -> ks_core::Result<std::option::Option<T>>
|
||||
where
|
||||
T: serde::de::DeserializeOwned,
|
||||
{
|
||||
let raw = match cursor {
|
||||
std::option::Option::Some(value) => value,
|
||||
std::option::Option::None => return std::result::Result::Ok(std::option::Option::None),
|
||||
};
|
||||
let decode_result = serde_json::from_str::<T>(raw);
|
||||
return match decode_result {
|
||||
std::result::Result::Ok(value) => std::result::Result::Ok(std::option::Option::Some(value)),
|
||||
std::result::Result::Err(_error) => std::result::Result::Err(ks_core::Error::db(format!(
|
||||
"replay {label} page cursor is invalid"
|
||||
))),
|
||||
};
|
||||
}
|
||||
|
||||
fn encode_transaction_cursor(
|
||||
row: &crate::ReplayTransactionCandidate,
|
||||
newest_first: bool,
|
||||
) -> ks_core::Result<std::string::String> {
|
||||
return encode_cursor(
|
||||
&ReplayTransactionPageCursor {
|
||||
slot: row.slot,
|
||||
signature: row.signature.clone(),
|
||||
newest_first,
|
||||
},
|
||||
"transaction",
|
||||
);
|
||||
}
|
||||
|
||||
fn encode_program_cursor(
|
||||
row: &crate::ReplayProgramSummary,
|
||||
) -> ks_core::Result<std::string::String> {
|
||||
return encode_cursor(
|
||||
&ReplayProgramPageCursor {
|
||||
transaction_count: row.transaction_count,
|
||||
program_id: row.program_id.clone(),
|
||||
},
|
||||
"program",
|
||||
);
|
||||
}
|
||||
|
||||
fn encode_entity_cursor(row: &crate::ReplayEntitySummary) -> ks_core::Result<std::string::String> {
|
||||
return encode_cursor(
|
||||
&ReplayEntityPageCursor {
|
||||
entity_kind: row.entity_kind.clone(),
|
||||
transaction_count: row.transaction_count,
|
||||
entity_value: row.entity_value.clone(),
|
||||
},
|
||||
"entity",
|
||||
);
|
||||
}
|
||||
|
||||
fn encode_cursor<T>(value: &T, label: &str) -> ks_core::Result<std::string::String>
|
||||
where
|
||||
T: serde::Serialize,
|
||||
{
|
||||
let encode_result = serde_json::to_string(value);
|
||||
return match encode_result {
|
||||
std::result::Result::Ok(encoded) if encoded.len() <= crate::MAX_PAGE_CURSOR_LENGTH => {
|
||||
std::result::Result::Ok(encoded)
|
||||
},
|
||||
std::result::Result::Ok(_encoded) => std::result::Result::Err(ks_core::Error::db(format!(
|
||||
"replay {label} page cursor exceeds maximum encoded length"
|
||||
))),
|
||||
std::result::Result::Err(error) => std::result::Result::Err(ks_core::Error::db(format!(
|
||||
"replay {label} page cursor serialization failed: {error}"
|
||||
))),
|
||||
};
|
||||
}
|
||||
|
||||
const TRANSACTION_COUNT_SQL: &str = r#"SELECT COUNT(*)::BIGINT
|
||||
FROM k_sol_raw_transactions raw
|
||||
LEFT JOIN LATERAL (
|
||||
SELECT status
|
||||
FROM k_sol_ops_processing_ledger
|
||||
WHERE stage = 'core_extraction'
|
||||
AND processor_name = 'canonical_to_core'
|
||||
AND input_key = raw.signature
|
||||
ORDER BY updated_at DESC, id DESC
|
||||
LIMIT 1
|
||||
) ledger ON TRUE
|
||||
WHERE ($1::TEXT IS NULL OR raw.signature ILIKE '%' || $1 || '%')
|
||||
AND ($2::BIGINT IS NULL OR raw.slot >= $2)
|
||||
AND ($3::BIGINT IS NULL OR raw.slot <= $3)
|
||||
AND ($4::TEXT IS NULL OR raw.processing_state = $4)
|
||||
AND ($5::TEXT IS NULL OR ($5 = 'not_started' AND ledger.status IS NULL) OR ledger.status = $5)
|
||||
AND ($6::TEXT IS NULL OR
|
||||
($7 = 'any' AND (
|
||||
EXISTS (SELECT 1 FROM k_sol_core_instructions candidate_top_level WHERE candidate_top_level.signature = raw.signature AND candidate_top_level.program_id = $6) OR
|
||||
EXISTS (SELECT 1 FROM k_sol_core_inner_instructions candidate_inner WHERE candidate_inner.signature = raw.signature AND candidate_inner.program_id = $6) OR
|
||||
EXISTS (SELECT 1 FROM k_sol_core_logs candidate_log WHERE candidate_log.signature = raw.signature AND candidate_log.program_id = $6)
|
||||
)) OR
|
||||
($7 = 'top_level' AND EXISTS (SELECT 1 FROM k_sol_core_instructions candidate_top_level WHERE candidate_top_level.signature = raw.signature AND candidate_top_level.program_id = $6)) OR
|
||||
($7 = 'inner' AND EXISTS (SELECT 1 FROM k_sol_core_inner_instructions candidate_inner WHERE candidate_inner.signature = raw.signature AND candidate_inner.program_id = $6)) OR
|
||||
($7 = 'logs' AND EXISTS (SELECT 1 FROM k_sol_core_logs candidate_log WHERE candidate_log.signature = raw.signature AND candidate_log.program_id = $6)))
|
||||
AND ($8::TEXT IS NULL OR
|
||||
($8 = 'mint' AND EXISTS (SELECT 1 FROM k_sol_core_balance_changes candidate_balance WHERE candidate_balance.signature = raw.signature AND candidate_balance.mint = $9)) OR
|
||||
($8 = 'owner' AND EXISTS (SELECT 1 FROM k_sol_core_balance_changes candidate_balance WHERE candidate_balance.signature = raw.signature AND candidate_balance.owner = $9)) OR
|
||||
($8 = 'account_key' AND EXISTS (SELECT 1 FROM k_sol_core_account_keys candidate_account WHERE candidate_account.signature = raw.signature AND candidate_account.account_key = $9)))"#;
|
||||
|
||||
const PROGRAM_PAGE_SQL: &str = r#"WITH occurrences AS (
|
||||
SELECT program_id, signature, slot, 'top_level'::TEXT AS scope FROM k_sol_core_instructions
|
||||
UNION ALL
|
||||
SELECT program_id, signature, slot, 'inner'::TEXT AS scope FROM k_sol_core_inner_instructions
|
||||
UNION ALL
|
||||
SELECT program_id, signature, slot, 'logs'::TEXT AS scope FROM k_sol_core_logs WHERE program_id IS NOT NULL
|
||||
), summaries AS (
|
||||
SELECT program_id,
|
||||
COUNT(DISTINCT signature)::BIGINT AS transaction_count,
|
||||
COUNT(*) FILTER (WHERE scope = 'top_level')::BIGINT AS top_level_instruction_count,
|
||||
COUNT(*) FILTER (WHERE scope = 'inner')::BIGINT AS inner_instruction_count,
|
||||
COUNT(*) FILTER (WHERE scope = 'logs')::BIGINT AS log_count,
|
||||
MIN(slot)::BIGINT AS min_slot,
|
||||
MAX(slot)::BIGINT AS max_slot
|
||||
FROM occurrences
|
||||
WHERE ($1::TEXT IS NULL OR program_id ILIKE '%' || $1 || '%')
|
||||
GROUP BY program_id
|
||||
)
|
||||
SELECT program_id, transaction_count, top_level_instruction_count, inner_instruction_count, log_count, min_slot, max_slot
|
||||
FROM summaries
|
||||
WHERE ($2::BIGINT IS NULL OR transaction_count < $2 OR (transaction_count = $2 AND program_id > $3))
|
||||
ORDER BY transaction_count DESC, program_id ASC
|
||||
LIMIT $4"#;
|
||||
|
||||
const PROGRAM_COUNT_SQL: &str = r#"WITH occurrences AS (
|
||||
SELECT program_id FROM k_sol_core_instructions
|
||||
UNION ALL
|
||||
SELECT program_id FROM k_sol_core_inner_instructions
|
||||
UNION ALL
|
||||
SELECT program_id FROM k_sol_core_logs WHERE program_id IS NOT NULL
|
||||
), summaries AS (
|
||||
SELECT program_id
|
||||
FROM occurrences
|
||||
WHERE ($1::TEXT IS NULL OR program_id ILIKE '%' || $1 || '%')
|
||||
GROUP BY program_id
|
||||
)
|
||||
SELECT COUNT(*)::BIGINT FROM summaries"#;
|
||||
|
||||
const ENTITY_PAGE_SQL: &str = r#"WITH entities AS (
|
||||
SELECT 'mint'::TEXT AS entity_kind, mint AS entity_value, signature, slot FROM k_sol_core_balance_changes WHERE mint IS NOT NULL
|
||||
UNION ALL
|
||||
SELECT 'owner'::TEXT AS entity_kind, owner AS entity_value, signature, slot FROM k_sol_core_balance_changes WHERE owner IS NOT NULL
|
||||
UNION ALL
|
||||
SELECT 'account_key'::TEXT AS entity_kind, account_key AS entity_value, signature, slot FROM k_sol_core_account_keys
|
||||
), summaries AS (
|
||||
SELECT entity_kind,
|
||||
entity_value,
|
||||
COUNT(DISTINCT signature)::BIGINT AS transaction_count,
|
||||
COUNT(*)::BIGINT AS occurrence_count,
|
||||
MIN(slot)::BIGINT AS min_slot,
|
||||
MAX(slot)::BIGINT AS max_slot
|
||||
FROM entities
|
||||
WHERE entity_kind = $1
|
||||
AND ($2::TEXT IS NULL OR entity_value ILIKE '%' || $2 || '%')
|
||||
GROUP BY entity_kind, entity_value
|
||||
)
|
||||
SELECT entity_kind, entity_value, transaction_count, occurrence_count, min_slot, max_slot
|
||||
FROM summaries
|
||||
WHERE ($3::BIGINT IS NULL OR transaction_count < $3 OR (transaction_count = $3 AND entity_value > $4))
|
||||
ORDER BY transaction_count DESC, entity_value ASC
|
||||
LIMIT $5"#;
|
||||
|
||||
const ENTITY_COUNT_SQL: &str = r#"WITH entities AS (
|
||||
SELECT 'mint'::TEXT AS entity_kind, mint AS entity_value FROM k_sol_core_balance_changes WHERE mint IS NOT NULL
|
||||
UNION ALL
|
||||
SELECT 'owner'::TEXT AS entity_kind, owner AS entity_value FROM k_sol_core_balance_changes WHERE owner IS NOT NULL
|
||||
UNION ALL
|
||||
SELECT 'account_key'::TEXT AS entity_kind, account_key AS entity_value FROM k_sol_core_account_keys
|
||||
), summaries AS (
|
||||
SELECT entity_kind, entity_value
|
||||
FROM entities
|
||||
WHERE entity_kind = $1
|
||||
AND ($2::TEXT IS NULL OR entity_value ILIKE '%' || $2 || '%')
|
||||
GROUP BY entity_kind, entity_value
|
||||
)
|
||||
SELECT COUNT(*)::BIGINT FROM summaries"#;
|
||||
|
||||
fn transaction_candidate_sql_desc() -> &'static str {
|
||||
return crate::postgres::query::replay_candidate_queries::transaction_candidate_sql("DESC");
|
||||
}
|
||||
@@ -310,8 +712,9 @@ fn transaction_candidate_sql(order: &str) -> &'static str {
|
||||
WHERE candidate_account.signature = raw.signature
|
||||
AND candidate_account.account_key = $9
|
||||
)))
|
||||
AND ($10::BIGINT IS NULL OR raw.slot < $10 OR (raw.slot = $10 AND raw.signature < $11))
|
||||
ORDER BY raw.slot DESC, raw.signature DESC
|
||||
LIMIT $10"#;
|
||||
LIMIT $12"#;
|
||||
}
|
||||
return r#"SELECT raw.signature,
|
||||
raw.slot,
|
||||
@@ -378,8 +781,9 @@ fn transaction_candidate_sql(order: &str) -> &'static str {
|
||||
WHERE candidate_account.signature = raw.signature
|
||||
AND candidate_account.account_key = $9
|
||||
)))
|
||||
AND ($10::BIGINT IS NULL OR raw.slot > $10 OR (raw.slot = $10 AND raw.signature > $11))
|
||||
ORDER BY raw.slot ASC, raw.signature ASC
|
||||
LIMIT $10"#;
|
||||
LIMIT $12"#;
|
||||
}
|
||||
|
||||
fn optional_sql_bigint(
|
||||
@@ -413,6 +817,49 @@ fn trace_query_failure(action: &str, started_at: std::time::Instant, error: &sql
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
#[test]
|
||||
fn replay_candidate_queries_are_cursor_based_and_counted() {
|
||||
let source = include_str!("replay_candidate_queries.rs");
|
||||
let tests_marker = source.find("#[cfg(test)]");
|
||||
let production_source = match tests_marker {
|
||||
std::option::Option::Some(index) => &source[..index],
|
||||
std::option::Option::None => source,
|
||||
};
|
||||
assert!(!production_source.contains(" OFFSET "));
|
||||
assert!(production_source.contains("TRANSACTION_COUNT_SQL"));
|
||||
assert!(production_source.contains("PROGRAM_COUNT_SQL"));
|
||||
assert!(production_source.contains("ENTITY_COUNT_SQL"));
|
||||
assert!(production_source.contains("CountedPageSlice::new"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn transaction_cursor_rejects_order_direction_changes() {
|
||||
let row = crate::ReplayTransactionCandidate {
|
||||
signature: "signature".to_string(),
|
||||
slot: 42,
|
||||
raw_processing_state: "received".to_string(),
|
||||
retention_state: "retained".to_string(),
|
||||
has_core_transaction: false,
|
||||
transaction_failed: std::option::Option::None,
|
||||
ledger_status: "not_started".to_string(),
|
||||
processor_version: std::option::Option::None,
|
||||
attempt_count: 0,
|
||||
top_level_instruction_count: 0,
|
||||
inner_instruction_count: 0,
|
||||
top_level_program_count: 0,
|
||||
inner_program_count: 0,
|
||||
updated_at: "now".to_string(),
|
||||
};
|
||||
let encoded_result = super::encode_transaction_cursor(&row, true);
|
||||
let encoded = match encoded_result {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(error) => panic!("unexpected encode error: {error}"),
|
||||
};
|
||||
let decoded_result =
|
||||
super::decode_transaction_cursor(std::option::Option::Some(encoded.as_str()), false);
|
||||
assert!(decoded_result.is_err());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn optional_postgres_replay_candidate_queries_from_env() {
|
||||
let database_url = match std::env::var("KS_SECRET_POSTGRES_TEST_URL") {
|
||||
@@ -430,7 +877,7 @@ mod tests {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(error) => panic!("unexpected connect error: {error}"),
|
||||
};
|
||||
let schema_result = store.initialize_store_schema().await;
|
||||
let schema_result = store.initialize_missing_store_schema().await;
|
||||
if let std::result::Result::Err(error) = schema_result {
|
||||
panic!("unexpected store schema error: {error}");
|
||||
}
|
||||
@@ -444,7 +891,6 @@ mod tests {
|
||||
crate::ReplayProgramScope::Any,
|
||||
std::option::Option::None,
|
||||
std::option::Option::None,
|
||||
10,
|
||||
true,
|
||||
);
|
||||
let transaction_filter = match transaction_filter_result {
|
||||
@@ -453,16 +899,24 @@ mod tests {
|
||||
panic!("unexpected transaction filter error: {error}")
|
||||
},
|
||||
};
|
||||
let transaction_result = store.replay_transaction_candidates(&transaction_filter).await;
|
||||
let transaction_page = crate::PageRequest::new(10, std::option::Option::None);
|
||||
let transaction_page = match transaction_page {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(error) => panic!("unexpected transaction page error: {error}"),
|
||||
};
|
||||
let transaction_result = store
|
||||
.replay_transaction_candidates(&transaction_filter, &transaction_page)
|
||||
.await;
|
||||
if let std::result::Result::Err(error) = transaction_result {
|
||||
panic!("unexpected transaction candidate query error: {error}");
|
||||
}
|
||||
let program_filter_result = crate::ReplayProgramFilter::new(std::option::Option::None, 10);
|
||||
let program_filter_result = crate::ReplayProgramFilter::new(std::option::Option::None);
|
||||
let program_filter = match program_filter_result {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(error) => panic!("unexpected program filter error: {error}"),
|
||||
};
|
||||
let program_result = store.replay_program_summaries(&program_filter).await;
|
||||
let program_result =
|
||||
store.replay_program_summaries(&program_filter, &transaction_page).await;
|
||||
if let std::result::Result::Err(error) = program_result {
|
||||
panic!("unexpected program summary query error: {error}");
|
||||
}
|
||||
@@ -472,14 +926,15 @@ mod tests {
|
||||
crate::ReplayEntityKind::AccountKey,
|
||||
] {
|
||||
let entity_filter_result =
|
||||
crate::ReplayEntityFilter::new(entity_kind, std::option::Option::None, 10);
|
||||
crate::ReplayEntityFilter::new(entity_kind, std::option::Option::None);
|
||||
let entity_filter = match entity_filter_result {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(error) => {
|
||||
panic!("unexpected entity filter error: {error}")
|
||||
},
|
||||
};
|
||||
let entity_result = store.replay_entity_summaries(&entity_filter).await;
|
||||
let entity_result =
|
||||
store.replay_entity_summaries(&entity_filter, &transaction_page).await;
|
||||
if let std::result::Result::Err(error) = entity_result {
|
||||
panic!("unexpected entity summary query error: {error}");
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user