v0.3.8-pre.009

This commit is contained in:
2026-09-03 20:44:21 +02:00
parent 8c23625fee
commit 32c4b67541
25 changed files with 1319 additions and 44 deletions

View File

@@ -1,5 +1,5 @@
// file: crates/ksp-store-postgres-lib/src/raw_account.rs
// version: 6
// version: 7
pub(crate) mod cursor;
@@ -7,6 +7,8 @@ const GET_ACCOUNT_OBSERVATION_SQL: &str = "SELECT observation_key, account_pubke
const GET_ACCOUNT_STATE_SQL: &str = "SELECT pubkey, slot::text AS slot_text, state_hash, lamports::text AS lamports_text, owner, executable, rent_epoch::text AS rent_epoch_text, data FROM ksp_raw_account_states WHERE pubkey = $1 AND slot = $2::TEXT::NUMERIC AND state_hash = $3";
const INSERT_ACCOUNT_OBSERVATION_SQL: &str = "INSERT INTO ksp_raw_account_observations (observation_key, account_pubkey, account_slot, account_state_hash, provider, protocol, acquisition_method, origin, received_at_unix_millis, capture_session_id, commitment, endpoint_id, filter_id, observed_at_unix_millis, source_payload_hash, source_payload_size_bytes, is_startup, transaction_signature, write_version) VALUES ($1, $2, $3::TEXT::NUMERIC, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19::TEXT::NUMERIC) ON CONFLICT (observation_key) DO NOTHING RETURNING observation_key";
const INSERT_ACCOUNT_STATE_SQL: &str = "INSERT INTO ksp_raw_account_states (pubkey, slot, state_hash, lamports, owner, executable, rent_epoch, data) VALUES ($1, $2::TEXT::NUMERIC, $3, $4::TEXT::NUMERIC, $5, $6, $7::TEXT::NUMERIC, $8) ON CONFLICT (pubkey, slot, state_hash) DO NOTHING RETURNING pubkey";
const INSPECT_ACCOUNT_OBSERVATIONS_ASC_SQL: &str = "WITH filtered_count AS (SELECT COUNT(*)::TEXT AS filtered_count_text FROM ksp_raw_account_observations WHERE ($1::BYTEA IS NULL OR (account_pubkey = $1 AND account_slot = $2::TEXT::NUMERIC AND account_state_hash = $3))), counts AS (SELECT filtered_count_text, CASE WHEN $1::BYTEA IS NULL THEN filtered_count_text ELSE (SELECT COUNT(*)::TEXT FROM ksp_raw_account_observations) END AS total_count_text FROM filtered_count) SELECT counts.total_count_text, counts.filtered_count_text, page.observation_key IS NOT NULL AS page_present, page.observation_key, page.account_pubkey, page.account_slot_text, page.account_state_hash, page.provider, page.protocol, page.acquisition_method, page.origin, page.received_at_unix_millis, page.capture_session_id, page.commitment, page.endpoint_id, page.filter_id, page.observed_at_unix_millis, page.source_payload_hash, page.source_payload_size_bytes, page.is_startup, page.transaction_signature, page.write_version_text FROM counts LEFT JOIN LATERAL (SELECT observation_key, account_pubkey, account_slot::TEXT AS account_slot_text, account_state_hash, provider, protocol, acquisition_method, origin, received_at_unix_millis, capture_session_id, commitment, endpoint_id, filter_id, observed_at_unix_millis, source_payload_hash, source_payload_size_bytes, is_startup, transaction_signature, write_version::TEXT AS write_version_text FROM ksp_raw_account_observations WHERE ($1::BYTEA IS NULL OR (account_pubkey = $1 AND account_slot = $2::TEXT::NUMERIC AND account_state_hash = $3)) ORDER BY received_at_unix_millis ASC, observation_key ASC LIMIT $4 OFFSET $5) AS page ON TRUE";
const INSPECT_ACCOUNT_OBSERVATIONS_DESC_SQL: &str = "WITH filtered_count AS (SELECT COUNT(*)::TEXT AS filtered_count_text FROM ksp_raw_account_observations WHERE ($1::BYTEA IS NULL OR (account_pubkey = $1 AND account_slot = $2::TEXT::NUMERIC AND account_state_hash = $3))), counts AS (SELECT filtered_count_text, CASE WHEN $1::BYTEA IS NULL THEN filtered_count_text ELSE (SELECT COUNT(*)::TEXT FROM ksp_raw_account_observations) END AS total_count_text FROM filtered_count) SELECT counts.total_count_text, counts.filtered_count_text, page.observation_key IS NOT NULL AS page_present, page.observation_key, page.account_pubkey, page.account_slot_text, page.account_state_hash, page.provider, page.protocol, page.acquisition_method, page.origin, page.received_at_unix_millis, page.capture_session_id, page.commitment, page.endpoint_id, page.filter_id, page.observed_at_unix_millis, page.source_payload_hash, page.source_payload_size_bytes, page.is_startup, page.transaction_signature, page.write_version_text FROM counts LEFT JOIN LATERAL (SELECT observation_key, account_pubkey, account_slot::TEXT AS account_slot_text, account_state_hash, provider, protocol, acquisition_method, origin, received_at_unix_millis, capture_session_id, commitment, endpoint_id, filter_id, observed_at_unix_millis, source_payload_hash, source_payload_size_bytes, is_startup, transaction_signature, write_version::TEXT AS write_version_text FROM ksp_raw_account_observations WHERE ($1::BYTEA IS NULL OR (account_pubkey = $1 AND account_slot = $2::TEXT::NUMERIC AND account_state_hash = $3)) ORDER BY received_at_unix_millis DESC, observation_key DESC LIMIT $4 OFFSET $5) AS page ON TRUE";
const INSPECT_ACCOUNT_STATES_ASC_SQL: &str = "WITH filtered_count AS (SELECT COUNT(*)::TEXT AS filtered_count_text FROM ksp_raw_account_states WHERE ($1::BYTEA IS NULL OR pubkey = $1::BYTEA) AND ($2::TEXT IS NULL OR slot >= $2::TEXT::NUMERIC) AND ($3::TEXT IS NULL OR slot <= $3::TEXT::NUMERIC)), counts AS (SELECT filtered_count_text, CASE WHEN $1::BYTEA IS NULL AND $2::TEXT IS NULL AND $3::TEXT IS NULL THEN filtered_count_text ELSE (SELECT COUNT(*)::TEXT FROM ksp_raw_account_states) END AS total_count_text FROM filtered_count) SELECT counts.total_count_text, counts.filtered_count_text, page.pubkey IS NOT NULL AS page_present, page.pubkey, page.slot_text, page.state_hash, page.lamports_text, page.owner, page.executable, page.rent_epoch_text, page.data_length_bytes FROM counts LEFT JOIN LATERAL (SELECT account_row.pubkey, account_row.slot::TEXT AS slot_text, account_row.state_hash, account_row.lamports::TEXT AS lamports_text, account_row.owner, account_row.executable, account_row.rent_epoch::TEXT AS rent_epoch_text, OCTET_LENGTH(account_row.data)::BIGINT AS data_length_bytes FROM ksp_raw_account_states AS account_row WHERE ($1::BYTEA IS NULL OR account_row.pubkey = $1::BYTEA) AND ($2::TEXT IS NULL OR account_row.slot >= $2::TEXT::NUMERIC) AND ($3::TEXT IS NULL OR account_row.slot <= $3::TEXT::NUMERIC) ORDER BY account_row.slot ASC, account_row.pubkey ASC, account_row.state_hash ASC LIMIT $4 OFFSET $5) AS page ON TRUE";
const INSPECT_ACCOUNT_STATES_DESC_SQL: &str = "WITH filtered_count AS (SELECT COUNT(*)::TEXT AS filtered_count_text FROM ksp_raw_account_states WHERE ($1::BYTEA IS NULL OR pubkey = $1::BYTEA) AND ($2::TEXT IS NULL OR slot >= $2::TEXT::NUMERIC) AND ($3::TEXT IS NULL OR slot <= $3::TEXT::NUMERIC)), counts AS (SELECT filtered_count_text, CASE WHEN $1::BYTEA IS NULL AND $2::TEXT IS NULL AND $3::TEXT IS NULL THEN filtered_count_text ELSE (SELECT COUNT(*)::TEXT FROM ksp_raw_account_states) END AS total_count_text FROM filtered_count) SELECT counts.total_count_text, counts.filtered_count_text, page.pubkey IS NOT NULL AS page_present, page.pubkey, page.slot_text, page.state_hash, page.lamports_text, page.owner, page.executable, page.rent_epoch_text, page.data_length_bytes FROM counts LEFT JOIN LATERAL (SELECT account_row.pubkey, account_row.slot::TEXT AS slot_text, account_row.state_hash, account_row.lamports::TEXT AS lamports_text, account_row.owner, account_row.executable, account_row.rent_epoch::TEXT AS rent_epoch_text, OCTET_LENGTH(account_row.data)::BIGINT AS data_length_bytes FROM ksp_raw_account_states AS account_row WHERE ($1::BYTEA IS NULL OR account_row.pubkey = $1::BYTEA) AND ($2::TEXT IS NULL OR account_row.slot >= $2::TEXT::NUMERIC) AND ($3::TEXT IS NULL OR account_row.slot <= $3::TEXT::NUMERIC) ORDER BY account_row.slot DESC, account_row.pubkey DESC, account_row.state_hash DESC LIMIT $4 OFFSET $5) AS page ON TRUE";
const LIST_ACCOUNT_STATES_ASC_SQL: &str = "SELECT pubkey, slot::text AS slot_text, state_hash FROM ksp_raw_account_states WHERE ($1::TEXT IS NULL OR slot >= $1::TEXT::NUMERIC) AND ($2::TEXT IS NULL OR slot <= $2::TEXT::NUMERIC) AND ($3::TEXT IS NULL OR (slot, pubkey, state_hash) > ($3::TEXT::NUMERIC, $4::BYTEA, $5::BYTEA)) ORDER BY slot ASC, pubkey ASC, state_hash ASC LIMIT $6";
@@ -38,6 +40,31 @@ struct RawAccountInspectionDbRow {
total_count_text: std::string::String,
}
struct RawAccountObservationInspectionDbRow {
account_pubkey: std::option::Option<std::vec::Vec<u8>>,
account_slot_text: std::option::Option<std::string::String>,
account_state_hash: std::option::Option<std::vec::Vec<u8>>,
acquisition_method: std::option::Option<std::string::String>,
capture_session_id: std::option::Option<std::string::String>,
commitment: std::option::Option<std::string::String>,
endpoint_id: std::option::Option<std::string::String>,
filter_id: std::option::Option<std::string::String>,
filtered_count_text: std::string::String,
is_startup: std::option::Option<bool>,
observation_key: std::option::Option<std::vec::Vec<u8>>,
observed_at_unix_millis: std::option::Option<i64>,
origin: std::option::Option<std::string::String>,
page_present: bool,
protocol: std::option::Option<std::string::String>,
provider: std::option::Option<std::string::String>,
received_at_unix_millis: std::option::Option<i64>,
source_payload_hash: std::option::Option<std::vec::Vec<u8>>,
source_payload_size_bytes: std::option::Option<i64>,
total_count_text: std::string::String,
transaction_signature: std::option::Option<std::vec::Vec<u8>>,
write_version_text: std::option::Option<std::string::String>,
}
struct RawAccountObservationDbRow {
account_pubkey: std::vec::Vec<u8>,
account_slot_text: std::string::String,
@@ -317,6 +344,121 @@ pub(crate) async fn inspect_raw_account_states(
};
}
/// Inspects one safe random-access RAW account-observation window with exact counts.
pub(crate) async fn inspect_raw_account_observations(
pool: &deadpool_postgres::Pool,
network: &ksp_store_api::RawNetworkId,
query: &ksp_store_api::RawAccountObservationInspectionQuery,
) -> std::result::Result<ksp_store_api::RawInspectionPage<ksp_store_api::RawAccountObservationSummary>, crate::PostgresBackendError> {
if query.network() != network {
return std::result::Result::Err(crate::PostgresBackendError::new(
crate::PostgresBackendErrorKind::WrongNetwork,
"raw_account_observation_inspection_network",
));
}
let sql = match query.direction() {
ksp_store_api::RawSortDirection::Ascending => INSPECT_ACCOUNT_OBSERVATIONS_ASC_SQL,
ksp_store_api::RawSortDirection::Descending => INSPECT_ACCOUNT_OBSERVATIONS_DESC_SQL,
_ => {
return std::result::Result::Err(crate::PostgresBackendError::new(
crate::PostgresBackendErrorKind::QueryInvalid,
"raw_account_observation_inspection_direction",
));
},
};
let (sql_limit, sql_offset) = match raw_account_inspection_sql_window(query.page()) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let account_pubkey = query.account().map(|reference| return reference.pubkey().to_bytes().to_vec());
let account_slot_text = query.account().map(|reference| return reference.slot().to_string());
let account_state_hash = query.account().map(|reference| return reference.state_hash().as_bytes().to_vec());
let client_result = pool.get().await;
let client = match client_result {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(crate::map_pool_error(error)),
};
let rows_result = client.query(sql, &[&account_pubkey, &account_slot_text, &account_state_hash, &sql_limit, &sql_offset]).await;
let rows = match rows_result {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => {
return std::result::Result::Err(crate::PostgresBackendError::new(
crate::PostgresBackendErrorKind::ReadFailed,
"raw_account_observation_inspection_query",
));
},
};
if rows.is_empty() {
return std::result::Result::Err(data_invalid("raw_account_observation_inspection_cardinality"));
}
let row_count = rows.len();
let mut total_items = std::option::Option::None;
let mut filtered_items = std::option::Option::None;
let mut items = std::vec::Vec::new();
for row in rows {
let physical = match raw_account_observation_inspection_db_row(&row) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let row_total = match decode_u64_decimal(physical.total_count_text.as_str(), "raw_account_observation_inspection_total") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let row_filtered = match decode_u64_decimal(physical.filtered_count_text.as_str(), "raw_account_observation_inspection_filtered") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
match (total_items, filtered_items) {
(std::option::Option::None, std::option::Option::None) => {
total_items = std::option::Option::Some(row_total);
filtered_items = std::option::Option::Some(row_filtered);
},
(std::option::Option::Some(total), std::option::Option::Some(filtered)) if total == row_total && filtered == row_filtered => {},
_ => return std::result::Result::Err(data_invalid("raw_account_observation_inspection_counts")),
}
if physical.page_present {
let observation = match decode_raw_account_observation_inspection_row(network, physical) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
if let std::option::Option::Some(reference) = query.account() {
if observation.account() != reference {
return std::result::Result::Err(data_invalid("raw_account_observation_inspection_reference"));
}
}
items.push(ksp_store_api::RawAccountObservationSummary::new(
observation.observation_key(),
observation.account().clone(),
observation.provenance().clone(),
observation.is_startup(),
observation.transaction_signature(),
observation.write_version(),
));
} else if row_count != 1 || !raw_account_observation_inspection_empty_page_is_clean(&physical) {
return std::result::Result::Err(data_invalid("raw_account_observation_inspection_page"));
}
}
let total_items = match total_items {
std::option::Option::Some(value) => value,
std::option::Option::None => return std::result::Result::Err(data_invalid("raw_account_observation_inspection_total")),
};
let filtered_items = match filtered_items {
std::option::Option::Some(value) => value,
std::option::Option::None => return std::result::Result::Err(data_invalid("raw_account_observation_inspection_filtered")),
};
let item_count = match u64::try_from(items.len()) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return std::result::Result::Err(data_invalid("raw_account_observation_inspection_page")),
};
if item_count > query.page().limit().get() {
return std::result::Result::Err(data_invalid("raw_account_observation_inspection_page"));
}
return match ksp_store_api::RawInspectionPage::try_new(items, total_items, filtered_items) {
std::result::Result::Ok(value) => std::result::Result::Ok(value),
std::result::Result::Err(_) => std::result::Result::Err(data_invalid("raw_account_observation_inspection_page")),
};
}
/// Reads one persisted RAW account observation by producer-owned idempotence key.
pub(crate) async fn get_raw_account_observation(
pool: &deadpool_postgres::Pool,
@@ -495,6 +637,141 @@ fn decode_raw_account_list_row(
return std::result::Result::Ok((slot, reference));
}
fn raw_account_observation_inspection_empty_page_is_clean(row: &RawAccountObservationInspectionDbRow) -> bool {
return row.account_pubkey.is_none()
&& row.account_slot_text.is_none()
&& row.account_state_hash.is_none()
&& row.acquisition_method.is_none()
&& row.capture_session_id.is_none()
&& row.commitment.is_none()
&& row.endpoint_id.is_none()
&& row.filter_id.is_none()
&& row.is_startup.is_none()
&& row.observation_key.is_none()
&& row.observed_at_unix_millis.is_none()
&& row.origin.is_none()
&& row.protocol.is_none()
&& row.provider.is_none()
&& row.received_at_unix_millis.is_none()
&& row.source_payload_hash.is_none()
&& row.source_payload_size_bytes.is_none()
&& row.transaction_signature.is_none()
&& row.write_version_text.is_none();
}
fn raw_account_observation_inspection_db_row(
row: &tokio_postgres::Row,
) -> std::result::Result<RawAccountObservationInspectionDbRow, crate::PostgresBackendError> {
macro_rules! get_optional {
($name:literal, $ty:ty) => {
match row.try_get::<_, std::option::Option<$ty>>($name) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return std::result::Result::Err(data_invalid("raw_account_observation_inspection_decode")),
}
};
}
let filtered_count_text = match row.try_get::<_, std::string::String>("filtered_count_text") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return std::result::Result::Err(data_invalid("raw_account_observation_inspection_decode")),
};
let page_present = match row.try_get::<_, bool>("page_present") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return std::result::Result::Err(data_invalid("raw_account_observation_inspection_decode")),
};
let total_count_text = match row.try_get::<_, std::string::String>("total_count_text") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return std::result::Result::Err(data_invalid("raw_account_observation_inspection_decode")),
};
return std::result::Result::Ok(RawAccountObservationInspectionDbRow {
account_pubkey: get_optional!("account_pubkey", std::vec::Vec<u8>),
account_slot_text: get_optional!("account_slot_text", std::string::String),
account_state_hash: get_optional!("account_state_hash", std::vec::Vec<u8>),
acquisition_method: get_optional!("acquisition_method", std::string::String),
capture_session_id: get_optional!("capture_session_id", std::string::String),
commitment: get_optional!("commitment", std::string::String),
endpoint_id: get_optional!("endpoint_id", std::string::String),
filter_id: get_optional!("filter_id", std::string::String),
filtered_count_text,
is_startup: get_optional!("is_startup", bool),
observation_key: get_optional!("observation_key", std::vec::Vec<u8>),
observed_at_unix_millis: get_optional!("observed_at_unix_millis", i64),
origin: get_optional!("origin", std::string::String),
page_present,
protocol: get_optional!("protocol", std::string::String),
provider: get_optional!("provider", std::string::String),
received_at_unix_millis: get_optional!("received_at_unix_millis", i64),
source_payload_hash: get_optional!("source_payload_hash", std::vec::Vec<u8>),
source_payload_size_bytes: get_optional!("source_payload_size_bytes", i64),
total_count_text,
transaction_signature: get_optional!("transaction_signature", std::vec::Vec<u8>),
write_version_text: get_optional!("write_version_text", std::string::String),
});
}
fn decode_raw_account_observation_inspection_row(
network: &ksp_store_api::RawNetworkId,
row: RawAccountObservationInspectionDbRow,
) -> std::result::Result<ksp_store_api::RawAccountObservation, crate::PostgresBackendError> {
let account_pubkey = match inspection_required(row.account_pubkey, "raw_account_observation_inspection_shape") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let account_slot_text = match inspection_required(row.account_slot_text, "raw_account_observation_inspection_shape") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let account_state_hash = match inspection_required(row.account_state_hash, "raw_account_observation_inspection_shape") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let acquisition_method = match inspection_required(row.acquisition_method, "raw_account_observation_inspection_shape") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let observation_key = match inspection_required(row.observation_key, "raw_account_observation_inspection_shape") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let origin = match inspection_required(row.origin, "raw_account_observation_inspection_shape") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let protocol = match inspection_required(row.protocol, "raw_account_observation_inspection_shape") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let provider = match inspection_required(row.provider, "raw_account_observation_inspection_shape") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let received_at_unix_millis = match inspection_required(row.received_at_unix_millis, "raw_account_observation_inspection_shape") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let physical = RawAccountObservationDbRow {
account_pubkey,
account_slot_text,
account_state_hash,
acquisition_method,
capture_session_id: row.capture_session_id,
commitment: row.commitment,
endpoint_id: row.endpoint_id,
filter_id: row.filter_id,
is_startup: row.is_startup,
observation_key,
observed_at_unix_millis: row.observed_at_unix_millis,
origin,
protocol,
provider,
received_at_unix_millis,
source_payload_hash: row.source_payload_hash,
source_payload_size_bytes: row.source_payload_size_bytes,
transaction_signature: row.transaction_signature,
write_version_text: row.write_version_text,
};
return decode_raw_account_observation_row(network, physical);
}
fn raw_account_observation_db_row(row: &tokio_postgres::Row) -> std::result::Result<RawAccountObservationDbRow, crate::PostgresBackendError> {
let account_pubkey: std::vec::Vec<u8> = match row.try_get("account_pubkey") {
std::result::Result::Ok(value) => value,