// file: ks-store/src/postgres/query/replay_candidate_queries.rs // version: 10 //! Read-only PostgreSQL queries for replay candidate discovery. #[derive(sqlx::FromRow)] struct ReplayTransactionCandidateRow { signature: std::string::String, slot: i64, raw_processing_state: std::string::String, retention_state: std::string::String, has_core_transaction: bool, transaction_failed: std::option::Option, ledger_status: std::string::String, processor_version: std::option::Option, attempt_count: i32, top_level_instruction_count: i64, inner_instruction_count: i64, top_level_program_count: i64, inner_program_count: i64, updated_at: std::string::String, } #[derive(sqlx::FromRow)] struct ReplayProgramSummaryRow { program_id: std::string::String, transaction_count: i64, top_level_instruction_count: i64, inner_instruction_count: i64, log_count: i64, min_slot: i64, max_slot: i64, } #[derive(sqlx::FromRow)] struct ReplayEntitySummaryRow { entity_kind: std::string::String, entity_value: std::string::String, transaction_count: i64, occurrence_count: i64, min_slot: i64, max_slot: i64, } #[derive(Clone, Debug, serde::Deserialize, serde::Serialize)] struct ReplayTransactionPageCursor { slot: i64, signature: std::string::String, newest_first: bool, } #[derive(Clone, Debug, serde::Deserialize, serde::Serialize)] struct ReplayProgramPageCursor { transaction_count: i64, program_id: std::string::String, } #[derive(Clone, Debug, serde::Deserialize, serde::Serialize)] struct ReplayEntityPageCursor { entity_kind: std::string::String, transaction_count: i64, entity_value: std::string::String, } pub(crate) async fn list_replay_transaction_candidates( pool: &sqlx::PgPool, filter: &crate::ReplayTransactionFilter, page_request: &crate::PageRequest, ) -> ks_core::Result> { let page_result = validate_page_request(page_request); if let std::result::Result::Err(error) = page_result { return std::result::Result::Err(error); } let min_slot_result = optional_sql_bigint(filter.min_slot); let min_slot = match min_slot_result { std::result::Result::Ok(value) => value, std::result::Result::Err(error) => return std::result::Result::Err(error), }; let max_slot_result = optional_sql_bigint(filter.max_slot); let max_slot = match max_slot_result { std::result::Result::Ok(value) => value, std::result::Result::Err(error) => return std::result::Result::Err(error), }; let entity_kind = match filter.entity_kind { std::option::Option::Some(value) => std::option::Option::Some(value.as_sql()), std::option::Option::None => std::option::Option::None, }; let cursor_result = decode_transaction_cursor(page_request.cursor.as_deref(), filter.newest_first); let cursor = match cursor_result { std::result::Result::Ok(value) => value, std::result::Result::Err(error) => return std::result::Result::Err(error), }; let total_rows_result = count_replay_transaction_candidates(pool, filter, min_slot, max_slot, entity_kind).await; let total_rows = match total_rows_result { std::result::Result::Ok(value) => value, std::result::Result::Err(error) => return std::result::Result::Err(error), }; let sql = if filter.newest_first { transaction_candidate_sql_desc() } else { transaction_candidate_sql_asc() }; let query_limit = i64::from(page_request.limit) + 1; let started_at = std::time::Instant::now(); let query_result = sqlx::query_as::(sql) .bind(filter.signature_contains.as_deref()) .bind(min_slot) .bind(max_slot) .bind(filter.raw_processing_state.as_deref()) .bind(filter.ledger_status.as_deref()) .bind(filter.program_id.as_deref()) .bind(filter.program_scope.as_sql()) .bind(entity_kind) .bind(filter.entity_value.as_deref()) .bind(cursor.as_ref().map(|value| return value.slot)) .bind(cursor.as_ref().map(|value| return value.signature.as_str())) .bind(query_limit) .fetch_all(pool) .await; let rows = match query_result { std::result::Result::Ok(value) => value, std::result::Result::Err(error) => { trace_query_failure("list_replay_transaction_candidate_page", started_at, &error); return std::result::Result::Err(ks_core::Error::db(format!( "postgres replay transaction candidate query failed: {error}" ))); }, }; let mut output = std::vec::Vec::with_capacity(rows.len().min(usize::from(page_request.limit))); for row in rows.iter().take(usize::from(page_request.limit)) { output.push(crate::ReplayTransactionCandidate { signature: row.signature.clone(), slot: row.slot, raw_processing_state: row.raw_processing_state.clone(), retention_state: row.retention_state.clone(), has_core_transaction: row.has_core_transaction, transaction_failed: row.transaction_failed, ledger_status: row.ledger_status.clone(), processor_version: row.processor_version.clone(), attempt_count: row.attempt_count, top_level_instruction_count: row.top_level_instruction_count, inner_instruction_count: row.inner_instruction_count, top_level_program_count: row.top_level_program_count, inner_program_count: row.inner_program_count, updated_at: row.updated_at.clone(), }); } let next_cursor = if rows.len() > usize::from(page_request.limit) { match output.last() { std::option::Option::Some(value) => { let encoded_result = encode_transaction_cursor(value, filter.newest_first); match encoded_result { std::result::Result::Ok(value) => std::option::Option::Some(value), std::result::Result::Err(error) => return std::result::Result::Err(error), } }, std::option::Option::None => std::option::Option::None, } } else { std::option::Option::None }; trace_query_success("list_replay_transaction_candidate_page", started_at, output.len()); return std::result::Result::Ok(crate::CountedPageSlice::new(output, next_cursor, total_rows)); } pub(crate) async fn list_replay_program_summaries( pool: &sqlx::PgPool, filter: &crate::ReplayProgramFilter, page_request: &crate::PageRequest, ) -> ks_core::Result> { let page_result = validate_page_request(page_request); if let std::result::Result::Err(error) = page_result { return std::result::Result::Err(error); } let cursor_result = decode_program_cursor(page_request.cursor.as_deref()); let cursor = match cursor_result { std::result::Result::Ok(value) => value, std::result::Result::Err(error) => return std::result::Result::Err(error), }; let total_rows_result = count_replay_program_summaries(pool, filter).await; let total_rows = match total_rows_result { std::result::Result::Ok(value) => value, std::result::Result::Err(error) => return std::result::Result::Err(error), }; let query_limit = i64::from(page_request.limit) + 1; let started_at = std::time::Instant::now(); let query_result = sqlx::query_as::(PROGRAM_PAGE_SQL) .bind(filter.program_id_contains.as_deref()) .bind(cursor.as_ref().map(|value| return value.transaction_count)) .bind(cursor.as_ref().map(|value| return value.program_id.as_str())) .bind(query_limit) .fetch_all(pool) .await; let rows = match query_result { std::result::Result::Ok(value) => value, std::result::Result::Err(error) => { trace_query_failure("list_replay_program_summary_page", started_at, &error); return std::result::Result::Err(ks_core::Error::db(format!( "postgres replay program summary query failed: {error}" ))); }, }; let mut output = std::vec::Vec::with_capacity(rows.len().min(usize::from(page_request.limit))); for row in rows.iter().take(usize::from(page_request.limit)) { output.push(crate::ReplayProgramSummary { program_id: row.program_id.clone(), transaction_count: row.transaction_count, top_level_instruction_count: row.top_level_instruction_count, inner_instruction_count: row.inner_instruction_count, log_count: row.log_count, min_slot: row.min_slot, max_slot: row.max_slot, }); } let next_cursor = if rows.len() > usize::from(page_request.limit) { match output.last() { std::option::Option::Some(value) => { let encoded_result = encode_program_cursor(value); match encoded_result { std::result::Result::Ok(value) => std::option::Option::Some(value), std::result::Result::Err(error) => return std::result::Result::Err(error), } }, std::option::Option::None => std::option::Option::None, } } else { std::option::Option::None }; trace_query_success("list_replay_program_summary_page", started_at, output.len()); return std::result::Result::Ok(crate::CountedPageSlice::new(output, next_cursor, total_rows)); } pub(crate) async fn list_replay_entity_summaries( pool: &sqlx::PgPool, filter: &crate::ReplayEntityFilter, page_request: &crate::PageRequest, ) -> ks_core::Result> { let page_result = validate_page_request(page_request); if let std::result::Result::Err(error) = page_result { return std::result::Result::Err(error); } let entity_kind = filter.entity_kind.as_sql(); let cursor_result = decode_entity_cursor(page_request.cursor.as_deref(), entity_kind); let cursor = match cursor_result { std::result::Result::Ok(value) => value, std::result::Result::Err(error) => return std::result::Result::Err(error), }; let total_rows_result = count_replay_entity_summaries(pool, filter).await; let total_rows = match total_rows_result { std::result::Result::Ok(value) => value, std::result::Result::Err(error) => return std::result::Result::Err(error), }; let query_limit = i64::from(page_request.limit) + 1; let started_at = std::time::Instant::now(); let query_result = sqlx::query_as::(ENTITY_PAGE_SQL) .bind(entity_kind) .bind(filter.entity_value_contains.as_deref()) .bind(cursor.as_ref().map(|value| return value.transaction_count)) .bind(cursor.as_ref().map(|value| return value.entity_value.as_str())) .bind(query_limit) .fetch_all(pool) .await; let rows = match query_result { std::result::Result::Ok(value) => value, std::result::Result::Err(error) => { trace_query_failure("list_replay_entity_summary_page", started_at, &error); return std::result::Result::Err(ks_core::Error::db(format!( "postgres replay entity summary query failed: {error}" ))); }, }; let mut output = std::vec::Vec::with_capacity(rows.len().min(usize::from(page_request.limit))); for row in rows.iter().take(usize::from(page_request.limit)) { output.push(crate::ReplayEntitySummary { entity_kind: row.entity_kind.clone(), entity_value: row.entity_value.clone(), transaction_count: row.transaction_count, occurrence_count: row.occurrence_count, min_slot: row.min_slot, max_slot: row.max_slot, }); } let next_cursor = if rows.len() > usize::from(page_request.limit) { match output.last() { std::option::Option::Some(value) => { let encoded_result = encode_entity_cursor(value); match encoded_result { std::result::Result::Ok(value) => std::option::Option::Some(value), std::result::Result::Err(error) => return std::result::Result::Err(error), } }, std::option::Option::None => std::option::Option::None, } } else { std::option::Option::None }; trace_query_success("list_replay_entity_summary_page", started_at, output.len()); return std::result::Result::Ok(crate::CountedPageSlice::new(output, next_cursor, total_rows)); } async fn count_replay_transaction_candidates( pool: &sqlx::PgPool, filter: &crate::ReplayTransactionFilter, min_slot: std::option::Option, max_slot: std::option::Option, entity_kind: std::option::Option<&str>, ) -> ks_core::Result { let started_at = std::time::Instant::now(); let query_result = sqlx::query_scalar::(TRANSACTION_COUNT_SQL) .bind(filter.signature_contains.as_deref()) .bind(min_slot) .bind(max_slot) .bind(filter.raw_processing_state.as_deref()) .bind(filter.ledger_status.as_deref()) .bind(filter.program_id.as_deref()) .bind(filter.program_scope.as_sql()) .bind(entity_kind) .bind(filter.entity_value.as_deref()) .fetch_one(pool) .await; let count = match query_result { std::result::Result::Ok(value) => value, std::result::Result::Err(error) => { trace_query_failure("count_replay_transaction_candidates", started_at, &error); return std::result::Result::Err(ks_core::Error::db(format!( "postgres replay transaction candidate count failed: {error}" ))); }, }; trace_query_success("count_replay_transaction_candidates", started_at, 1); return unsigned_count(count, "replay transaction candidate count is negative"); } async fn count_replay_program_summaries( pool: &sqlx::PgPool, filter: &crate::ReplayProgramFilter, ) -> ks_core::Result { let started_at = std::time::Instant::now(); let query_result = sqlx::query_scalar::(PROGRAM_COUNT_SQL) .bind(filter.program_id_contains.as_deref()) .fetch_one(pool) .await; let count = match query_result { std::result::Result::Ok(value) => value, std::result::Result::Err(error) => { trace_query_failure("count_replay_program_summaries", started_at, &error); return std::result::Result::Err(ks_core::Error::db(format!( "postgres replay program summary count failed: {error}" ))); }, }; trace_query_success("count_replay_program_summaries", started_at, 1); return unsigned_count(count, "replay program summary count is negative"); } async fn count_replay_entity_summaries( pool: &sqlx::PgPool, filter: &crate::ReplayEntityFilter, ) -> ks_core::Result { let started_at = std::time::Instant::now(); let query_result = sqlx::query_scalar::(ENTITY_COUNT_SQL) .bind(filter.entity_kind.as_sql()) .bind(filter.entity_value_contains.as_deref()) .fetch_one(pool) .await; let count = match query_result { std::result::Result::Ok(value) => value, std::result::Result::Err(error) => { trace_query_failure("count_replay_entity_summaries", started_at, &error); return std::result::Result::Err(ks_core::Error::db(format!( "postgres replay entity summary count failed: {error}" ))); }, }; trace_query_success("count_replay_entity_summaries", started_at, 1); return unsigned_count(count, "replay entity summary count is negative"); } fn validate_page_request(page_request: &crate::PageRequest) -> ks_core::Result<()> { if page_request.limit == 0 || page_request.limit > crate::MAX_PAGE_SIZE { return std::result::Result::Err(ks_core::Error::db( "replay candidate page size is outside repository bounds", )); } return std::result::Result::Ok(()); } fn unsigned_count(value: i64, message: &str) -> ks_core::Result { let conversion_result = u64::try_from(value); return match conversion_result { std::result::Result::Ok(converted) => std::result::Result::Ok(converted), std::result::Result::Err(_error) => std::result::Result::Err(ks_core::Error::db(message)), }; } fn decode_transaction_cursor( cursor: std::option::Option<&str>, newest_first: bool, ) -> ks_core::Result> { let decoded_result = decode_cursor::(cursor, "transaction"); let decoded = match decoded_result { std::result::Result::Ok(value) => value, std::result::Result::Err(error) => return std::result::Result::Err(error), }; if decoded.as_ref().is_some_and(|value| { return value.slot < 0 || value.signature.trim().is_empty() || value.newest_first != newest_first; }) { return std::result::Result::Err(ks_core::Error::db( "replay transaction page cursor values are invalid", )); } return std::result::Result::Ok(decoded); } fn decode_program_cursor( cursor: std::option::Option<&str>, ) -> ks_core::Result> { let decoded_result = decode_cursor::(cursor, "program"); let decoded = match decoded_result { std::result::Result::Ok(value) => value, std::result::Result::Err(error) => return std::result::Result::Err(error), }; if decoded.as_ref().is_some_and(|value| { return value.transaction_count < 0 || value.program_id.trim().is_empty(); }) { return std::result::Result::Err(ks_core::Error::db( "replay program page cursor values are invalid", )); } return std::result::Result::Ok(decoded); } fn decode_entity_cursor( cursor: std::option::Option<&str>, entity_kind: &str, ) -> ks_core::Result> { let decoded_result = decode_cursor::(cursor, "entity"); let decoded = match decoded_result { std::result::Result::Ok(value) => value, std::result::Result::Err(error) => return std::result::Result::Err(error), }; if decoded.as_ref().is_some_and(|value| { return value.transaction_count < 0 || value.entity_value.trim().is_empty() || value.entity_kind != entity_kind; }) { return std::result::Result::Err(ks_core::Error::db( "replay entity page cursor values are invalid", )); } return std::result::Result::Ok(decoded); } fn decode_cursor( cursor: std::option::Option<&str>, label: &str, ) -> ks_core::Result> where T: serde::de::DeserializeOwned, { let raw = match cursor { std::option::Option::Some(value) => value, std::option::Option::None => return std::result::Result::Ok(std::option::Option::None), }; let decode_result = serde_json::from_str::(raw); return match decode_result { std::result::Result::Ok(value) => std::result::Result::Ok(std::option::Option::Some(value)), std::result::Result::Err(_error) => std::result::Result::Err(ks_core::Error::db(format!( "replay {label} page cursor is invalid" ))), }; } fn encode_transaction_cursor( row: &crate::ReplayTransactionCandidate, newest_first: bool, ) -> ks_core::Result { return encode_cursor( &ReplayTransactionPageCursor { slot: row.slot, signature: row.signature.clone(), newest_first, }, "transaction", ); } fn encode_program_cursor( row: &crate::ReplayProgramSummary, ) -> ks_core::Result { return encode_cursor( &ReplayProgramPageCursor { transaction_count: row.transaction_count, program_id: row.program_id.clone(), }, "program", ); } fn encode_entity_cursor(row: &crate::ReplayEntitySummary) -> ks_core::Result { return encode_cursor( &ReplayEntityPageCursor { entity_kind: row.entity_kind.clone(), transaction_count: row.transaction_count, entity_value: row.entity_value.clone(), }, "entity", ); } fn encode_cursor(value: &T, label: &str) -> ks_core::Result where T: serde::Serialize, { let encode_result = serde_json::to_string(value); return match encode_result { std::result::Result::Ok(encoded) if encoded.len() <= crate::MAX_PAGE_CURSOR_LENGTH => { std::result::Result::Ok(encoded) }, std::result::Result::Ok(_encoded) => std::result::Result::Err(ks_core::Error::db(format!( "replay {label} page cursor exceeds maximum encoded length" ))), std::result::Result::Err(error) => std::result::Result::Err(ks_core::Error::db(format!( "replay {label} page cursor serialization failed: {error}" ))), }; } const TRANSACTION_COUNT_SQL: &str = r#"SELECT COUNT(*)::BIGINT FROM k_sol_raw_transactions raw LEFT JOIN LATERAL ( SELECT status FROM k_sol_ops_processing_ledger WHERE stage = 'core_extraction' AND processor_name = 'canonical_to_core' AND input_key = raw.signature ORDER BY updated_at DESC, id DESC LIMIT 1 ) ledger ON TRUE WHERE ($1::TEXT IS NULL OR raw.signature ILIKE '%' || $1 || '%') AND ($2::BIGINT IS NULL OR raw.slot >= $2) AND ($3::BIGINT IS NULL OR raw.slot <= $3) AND ($4::TEXT IS NULL OR raw.processing_state = $4) AND ($5::TEXT IS NULL OR ($5 = 'not_started' AND ledger.status IS NULL) OR ledger.status = $5) AND ($6::TEXT IS NULL OR ($7 = 'any' AND ( EXISTS (SELECT 1 FROM k_sol_core_instructions candidate_top_level WHERE candidate_top_level.signature = raw.signature AND candidate_top_level.program_id = $6) OR EXISTS (SELECT 1 FROM k_sol_core_inner_instructions candidate_inner WHERE candidate_inner.signature = raw.signature AND candidate_inner.program_id = $6) OR EXISTS (SELECT 1 FROM k_sol_core_logs candidate_log WHERE candidate_log.signature = raw.signature AND candidate_log.program_id = $6) )) OR ($7 = 'top_level' AND EXISTS (SELECT 1 FROM k_sol_core_instructions candidate_top_level WHERE candidate_top_level.signature = raw.signature AND candidate_top_level.program_id = $6)) OR ($7 = 'inner' AND EXISTS (SELECT 1 FROM k_sol_core_inner_instructions candidate_inner WHERE candidate_inner.signature = raw.signature AND candidate_inner.program_id = $6)) OR ($7 = 'logs' AND EXISTS (SELECT 1 FROM k_sol_core_logs candidate_log WHERE candidate_log.signature = raw.signature AND candidate_log.program_id = $6))) AND ($8::TEXT IS NULL OR ($8 = 'mint' AND EXISTS (SELECT 1 FROM k_sol_core_balance_changes candidate_balance WHERE candidate_balance.signature = raw.signature AND candidate_balance.mint = $9)) OR ($8 = 'owner' AND EXISTS (SELECT 1 FROM k_sol_core_balance_changes candidate_balance WHERE candidate_balance.signature = raw.signature AND candidate_balance.owner = $9)) OR ($8 = 'account_key' AND EXISTS (SELECT 1 FROM k_sol_core_account_keys candidate_account WHERE candidate_account.signature = raw.signature AND candidate_account.account_key = $9)))"#; const PROGRAM_PAGE_SQL: &str = r#"WITH occurrences AS ( SELECT program_id, signature, slot, 'top_level'::TEXT AS scope FROM k_sol_core_instructions UNION ALL SELECT program_id, signature, slot, 'inner'::TEXT AS scope FROM k_sol_core_inner_instructions UNION ALL SELECT program_id, signature, slot, 'logs'::TEXT AS scope FROM k_sol_core_logs WHERE program_id IS NOT NULL ), summaries AS ( SELECT program_id, COUNT(DISTINCT signature)::BIGINT AS transaction_count, COUNT(*) FILTER (WHERE scope = 'top_level')::BIGINT AS top_level_instruction_count, COUNT(*) FILTER (WHERE scope = 'inner')::BIGINT AS inner_instruction_count, COUNT(*) FILTER (WHERE scope = 'logs')::BIGINT AS log_count, MIN(slot)::BIGINT AS min_slot, MAX(slot)::BIGINT AS max_slot FROM occurrences WHERE ($1::TEXT IS NULL OR program_id ILIKE '%' || $1 || '%') GROUP BY program_id ) SELECT program_id, transaction_count, top_level_instruction_count, inner_instruction_count, log_count, min_slot, max_slot FROM summaries WHERE ($2::BIGINT IS NULL OR transaction_count < $2 OR (transaction_count = $2 AND program_id > $3)) ORDER BY transaction_count DESC, program_id ASC LIMIT $4"#; const PROGRAM_COUNT_SQL: &str = r#"WITH occurrences AS ( SELECT program_id FROM k_sol_core_instructions UNION ALL SELECT program_id FROM k_sol_core_inner_instructions UNION ALL SELECT program_id FROM k_sol_core_logs WHERE program_id IS NOT NULL ), summaries AS ( SELECT program_id FROM occurrences WHERE ($1::TEXT IS NULL OR program_id ILIKE '%' || $1 || '%') GROUP BY program_id ) SELECT COUNT(*)::BIGINT FROM summaries"#; const ENTITY_PAGE_SQL: &str = r#"WITH entities AS ( SELECT 'mint'::TEXT AS entity_kind, mint AS entity_value, signature, slot FROM k_sol_core_balance_changes WHERE mint IS NOT NULL UNION ALL SELECT 'owner'::TEXT AS entity_kind, owner AS entity_value, signature, slot FROM k_sol_core_balance_changes WHERE owner IS NOT NULL UNION ALL SELECT 'account_key'::TEXT AS entity_kind, account_key AS entity_value, signature, slot FROM k_sol_core_account_keys ), summaries AS ( SELECT entity_kind, entity_value, COUNT(DISTINCT signature)::BIGINT AS transaction_count, COUNT(*)::BIGINT AS occurrence_count, MIN(slot)::BIGINT AS min_slot, MAX(slot)::BIGINT AS max_slot FROM entities WHERE entity_kind = $1 AND ($2::TEXT IS NULL OR entity_value ILIKE '%' || $2 || '%') GROUP BY entity_kind, entity_value ) SELECT entity_kind, entity_value, transaction_count, occurrence_count, min_slot, max_slot FROM summaries WHERE ($3::BIGINT IS NULL OR transaction_count < $3 OR (transaction_count = $3 AND entity_value > $4)) ORDER BY transaction_count DESC, entity_value ASC LIMIT $5"#; const ENTITY_COUNT_SQL: &str = r#"WITH entities AS ( SELECT 'mint'::TEXT AS entity_kind, mint AS entity_value FROM k_sol_core_balance_changes WHERE mint IS NOT NULL UNION ALL SELECT 'owner'::TEXT AS entity_kind, owner AS entity_value FROM k_sol_core_balance_changes WHERE owner IS NOT NULL UNION ALL SELECT 'account_key'::TEXT AS entity_kind, account_key AS entity_value FROM k_sol_core_account_keys ), summaries AS ( SELECT entity_kind, entity_value FROM entities WHERE entity_kind = $1 AND ($2::TEXT IS NULL OR entity_value ILIKE '%' || $2 || '%') GROUP BY entity_kind, entity_value ) SELECT COUNT(*)::BIGINT FROM summaries"#; fn transaction_candidate_sql_desc() -> &'static str { return crate::postgres::query::replay_candidate_queries::transaction_candidate_sql("DESC"); } fn transaction_candidate_sql_asc() -> &'static str { return crate::postgres::query::replay_candidate_queries::transaction_candidate_sql("ASC"); } fn transaction_candidate_sql(order: &str) -> &'static str { if order == "DESC" { return r#"SELECT raw.signature, raw.slot, raw.processing_state AS raw_processing_state, raw.retention_state, (core.id IS NOT NULL) AS has_core_transaction, core.failed AS transaction_failed, COALESCE(ledger.status, 'not_started') AS ledger_status, ledger.processor_version, COALESCE(ledger.attempt_count, 0)::INTEGER AS attempt_count, COALESCE(top_level_stats.instruction_count, 0)::BIGINT AS top_level_instruction_count, COALESCE(inner_stats.instruction_count, 0)::BIGINT AS inner_instruction_count, COALESCE(top_level_stats.program_count, 0)::BIGINT AS top_level_program_count, COALESCE(inner_stats.program_count, 0)::BIGINT AS inner_program_count, raw.updated_at::TEXT AS updated_at FROM k_sol_raw_transactions raw LEFT JOIN k_sol_core_transactions core ON core.signature = raw.signature LEFT JOIN LATERAL ( SELECT status, processor_version, attempt_count FROM k_sol_ops_processing_ledger WHERE stage = 'core_extraction' AND processor_name = 'canonical_to_core' AND input_key = raw.signature ORDER BY updated_at DESC, id DESC LIMIT 1 ) ledger ON TRUE LEFT JOIN LATERAL ( SELECT COUNT(*)::BIGINT AS instruction_count, COUNT(DISTINCT program_id)::BIGINT AS program_count FROM k_sol_core_instructions WHERE signature = raw.signature ) top_level_stats ON TRUE LEFT JOIN LATERAL ( SELECT COUNT(*)::BIGINT AS instruction_count, COUNT(DISTINCT program_id)::BIGINT AS program_count FROM k_sol_core_inner_instructions WHERE signature = raw.signature ) inner_stats ON TRUE WHERE ($1::TEXT IS NULL OR raw.signature ILIKE '%' || $1 || '%') AND ($2::BIGINT IS NULL OR raw.slot >= $2) AND ($3::BIGINT IS NULL OR raw.slot <= $3) AND ($4::TEXT IS NULL OR raw.processing_state = $4) AND ($5::TEXT IS NULL OR ($5 = 'not_started' AND ledger.status IS NULL) OR ledger.status = $5) AND ($6::TEXT IS NULL OR ($7 = 'any' AND ( EXISTS (SELECT 1 FROM k_sol_core_instructions candidate_top_level WHERE candidate_top_level.signature = raw.signature AND candidate_top_level.program_id = $6) OR EXISTS (SELECT 1 FROM k_sol_core_inner_instructions candidate_inner WHERE candidate_inner.signature = raw.signature AND candidate_inner.program_id = $6) OR EXISTS (SELECT 1 FROM k_sol_core_logs candidate_log WHERE candidate_log.signature = raw.signature AND candidate_log.program_id = $6) )) OR ($7 = 'top_level' AND EXISTS (SELECT 1 FROM k_sol_core_instructions candidate_top_level WHERE candidate_top_level.signature = raw.signature AND candidate_top_level.program_id = $6)) OR ($7 = 'inner' AND EXISTS (SELECT 1 FROM k_sol_core_inner_instructions candidate_inner WHERE candidate_inner.signature = raw.signature AND candidate_inner.program_id = $6)) OR ($7 = 'logs' AND EXISTS (SELECT 1 FROM k_sol_core_logs candidate_log WHERE candidate_log.signature = raw.signature AND candidate_log.program_id = $6))) AND ($8::TEXT IS NULL OR ($8 = 'mint' AND EXISTS ( SELECT 1 FROM k_sol_core_balance_changes candidate_balance WHERE candidate_balance.signature = raw.signature AND candidate_balance.mint = $9 )) OR ($8 = 'owner' AND EXISTS ( SELECT 1 FROM k_sol_core_balance_changes candidate_balance WHERE candidate_balance.signature = raw.signature AND candidate_balance.owner = $9 )) OR ($8 = 'account_key' AND EXISTS ( SELECT 1 FROM k_sol_core_account_keys candidate_account WHERE candidate_account.signature = raw.signature AND candidate_account.account_key = $9 ))) AND ($10::BIGINT IS NULL OR raw.slot < $10 OR (raw.slot = $10 AND raw.signature < $11)) ORDER BY raw.slot DESC, raw.signature DESC LIMIT $12"#; } return r#"SELECT raw.signature, raw.slot, raw.processing_state AS raw_processing_state, raw.retention_state, (core.id IS NOT NULL) AS has_core_transaction, core.failed AS transaction_failed, COALESCE(ledger.status, 'not_started') AS ledger_status, ledger.processor_version, COALESCE(ledger.attempt_count, 0)::INTEGER AS attempt_count, COALESCE(top_level_stats.instruction_count, 0)::BIGINT AS top_level_instruction_count, COALESCE(inner_stats.instruction_count, 0)::BIGINT AS inner_instruction_count, COALESCE(top_level_stats.program_count, 0)::BIGINT AS top_level_program_count, COALESCE(inner_stats.program_count, 0)::BIGINT AS inner_program_count, raw.updated_at::TEXT AS updated_at FROM k_sol_raw_transactions raw LEFT JOIN k_sol_core_transactions core ON core.signature = raw.signature LEFT JOIN LATERAL ( SELECT status, processor_version, attempt_count FROM k_sol_ops_processing_ledger WHERE stage = 'core_extraction' AND processor_name = 'canonical_to_core' AND input_key = raw.signature ORDER BY updated_at DESC, id DESC LIMIT 1 ) ledger ON TRUE LEFT JOIN LATERAL ( SELECT COUNT(*)::BIGINT AS instruction_count, COUNT(DISTINCT program_id)::BIGINT AS program_count FROM k_sol_core_instructions WHERE signature = raw.signature ) top_level_stats ON TRUE LEFT JOIN LATERAL ( SELECT COUNT(*)::BIGINT AS instruction_count, COUNT(DISTINCT program_id)::BIGINT AS program_count FROM k_sol_core_inner_instructions WHERE signature = raw.signature ) inner_stats ON TRUE WHERE ($1::TEXT IS NULL OR raw.signature ILIKE '%' || $1 || '%') AND ($2::BIGINT IS NULL OR raw.slot >= $2) AND ($3::BIGINT IS NULL OR raw.slot <= $3) AND ($4::TEXT IS NULL OR raw.processing_state = $4) AND ($5::TEXT IS NULL OR ($5 = 'not_started' AND ledger.status IS NULL) OR ledger.status = $5) AND ($6::TEXT IS NULL OR ($7 = 'any' AND ( EXISTS (SELECT 1 FROM k_sol_core_instructions candidate_top_level WHERE candidate_top_level.signature = raw.signature AND candidate_top_level.program_id = $6) OR EXISTS (SELECT 1 FROM k_sol_core_inner_instructions candidate_inner WHERE candidate_inner.signature = raw.signature AND candidate_inner.program_id = $6) OR EXISTS (SELECT 1 FROM k_sol_core_logs candidate_log WHERE candidate_log.signature = raw.signature AND candidate_log.program_id = $6) )) OR ($7 = 'top_level' AND EXISTS (SELECT 1 FROM k_sol_core_instructions candidate_top_level WHERE candidate_top_level.signature = raw.signature AND candidate_top_level.program_id = $6)) OR ($7 = 'inner' AND EXISTS (SELECT 1 FROM k_sol_core_inner_instructions candidate_inner WHERE candidate_inner.signature = raw.signature AND candidate_inner.program_id = $6)) OR ($7 = 'logs' AND EXISTS (SELECT 1 FROM k_sol_core_logs candidate_log WHERE candidate_log.signature = raw.signature AND candidate_log.program_id = $6))) AND ($8::TEXT IS NULL OR ($8 = 'mint' AND EXISTS ( SELECT 1 FROM k_sol_core_balance_changes candidate_balance WHERE candidate_balance.signature = raw.signature AND candidate_balance.mint = $9 )) OR ($8 = 'owner' AND EXISTS ( SELECT 1 FROM k_sol_core_balance_changes candidate_balance WHERE candidate_balance.signature = raw.signature AND candidate_balance.owner = $9 )) OR ($8 = 'account_key' AND EXISTS ( SELECT 1 FROM k_sol_core_account_keys candidate_account WHERE candidate_account.signature = raw.signature AND candidate_account.account_key = $9 ))) AND ($10::BIGINT IS NULL OR raw.slot > $10 OR (raw.slot = $10 AND raw.signature > $11)) ORDER BY raw.slot ASC, raw.signature ASC LIMIT $12"#; } fn optional_sql_bigint( value: std::option::Option, ) -> ks_core::Result> { return match value { std::option::Option::Some(raw_value) => { let conversion_result = i64::try_from(raw_value); match conversion_result { std::result::Result::Ok(converted) => { std::result::Result::Ok(std::option::Option::Some(converted)) }, std::result::Result::Err(error) => std::result::Result::Err(ks_core::Error::db( format!("replay candidate slot does not fit into PostgreSQL BIGINT: {error}"), )), } }, std::option::Option::None => std::result::Result::Ok(std::option::Option::None), }; } fn trace_query_success(action: &str, started_at: std::time::Instant, rows: usize) { let elapsed_ms = started_at.elapsed().as_secs_f64() * 1000.0; tracing::trace!(target: crate::TRACING_TARGET, backend = "postgres", domain = "ks-store.pg", action = action, elapsed_ms, rows, outcome = "success", "PostgreSQL store operation completed"); } fn trace_query_failure(action: &str, started_at: std::time::Instant, error: &sqlx::Error) { let elapsed_ms = started_at.elapsed().as_secs_f64() * 1000.0; tracing::trace!(target: crate::TRACING_TARGET, backend = "postgres", domain = "ks-store.pg", action = action, elapsed_ms, error = %error, outcome = "error", "PostgreSQL store operation failed"); } #[cfg(test)] mod tests { #[test] fn replay_candidate_queries_are_cursor_based_and_counted() { let source = include_str!("replay_candidate_queries.rs"); let tests_marker = source.find("#[cfg(test)]"); let production_source = match tests_marker { std::option::Option::Some(index) => &source[..index], std::option::Option::None => source, }; assert!(!production_source.contains(" OFFSET ")); assert!(production_source.contains("TRANSACTION_COUNT_SQL")); assert!(production_source.contains("PROGRAM_COUNT_SQL")); assert!(production_source.contains("ENTITY_COUNT_SQL")); assert!(production_source.contains("CountedPageSlice::new")); } #[test] fn transaction_cursor_rejects_order_direction_changes() { let row = crate::ReplayTransactionCandidate { signature: "signature".to_string(), slot: 42, raw_processing_state: "received".to_string(), retention_state: "retained".to_string(), has_core_transaction: false, transaction_failed: std::option::Option::None, ledger_status: "not_started".to_string(), processor_version: std::option::Option::None, attempt_count: 0, top_level_instruction_count: 0, inner_instruction_count: 0, top_level_program_count: 0, inner_program_count: 0, updated_at: "now".to_string(), }; let encoded_result = super::encode_transaction_cursor(&row, true); let encoded = match encoded_result { std::result::Result::Ok(value) => value, std::result::Result::Err(error) => panic!("unexpected encode error: {error}"), }; let decoded_result = super::decode_transaction_cursor(std::option::Option::Some(encoded.as_str()), false); assert!(decoded_result.is_err()); } #[tokio::test] async fn optional_postgres_replay_candidate_queries_from_env() { let database_url = match std::env::var("KS_SECRET_POSTGRES_TEST_URL") { std::result::Result::Ok(value) => value, std::result::Result::Err(_error) => return, }; let _postgres_guard = crate::postgres_test_guard().await; let options_result = crate::PostgresStoreOptions::new(database_url, 1, 5000, false); let options = match options_result { std::result::Result::Ok(value) => value, std::result::Result::Err(error) => panic!("unexpected options error: {error}"), }; let store_result = crate::PostgresStore::connect(options).await; let store = match store_result { std::result::Result::Ok(value) => value, std::result::Result::Err(error) => panic!("unexpected connect error: {error}"), }; let schema_result = store.initialize_missing_store_schema().await; if let std::result::Result::Err(error) = schema_result { panic!("unexpected store schema error: {error}"); } let transaction_filter_result = crate::ReplayTransactionFilter::new( std::option::Option::None, std::option::Option::None, std::option::Option::None, std::option::Option::None, std::option::Option::None, std::option::Option::None, crate::ReplayProgramScope::Any, std::option::Option::None, std::option::Option::None, true, ); let transaction_filter = match transaction_filter_result { std::result::Result::Ok(value) => value, std::result::Result::Err(error) => { panic!("unexpected transaction filter error: {error}") }, }; let transaction_page = crate::PageRequest::new(10, std::option::Option::None); let transaction_page = match transaction_page { std::result::Result::Ok(value) => value, std::result::Result::Err(error) => panic!("unexpected transaction page error: {error}"), }; let transaction_result = store .replay_transaction_candidates(&transaction_filter, &transaction_page) .await; if let std::result::Result::Err(error) = transaction_result { panic!("unexpected transaction candidate query error: {error}"); } let program_filter_result = crate::ReplayProgramFilter::new(std::option::Option::None); let program_filter = match program_filter_result { std::result::Result::Ok(value) => value, std::result::Result::Err(error) => panic!("unexpected program filter error: {error}"), }; let program_result = store.replay_program_summaries(&program_filter, &transaction_page).await; if let std::result::Result::Err(error) = program_result { panic!("unexpected program summary query error: {error}"); } for entity_kind in [ crate::ReplayEntityKind::Mint, crate::ReplayEntityKind::Owner, crate::ReplayEntityKind::AccountKey, ] { let entity_filter_result = crate::ReplayEntityFilter::new(entity_kind, std::option::Option::None); let entity_filter = match entity_filter_result { std::result::Result::Ok(value) => value, std::result::Result::Err(error) => { panic!("unexpected entity filter error: {error}") }, }; let entity_result = store.replay_entity_summaries(&entity_filter, &transaction_page).await; if let std::result::Result::Err(error) = entity_result { panic!("unexpected entity summary query error: {error}"); } } } }