v0.3.8-pre.009
This commit is contained in:
@@ -1,5 +1,5 @@
|
||||
// file: crates/ksp-store-postgres-lib/src/lib.rs
|
||||
// version: 23
|
||||
// version: 24
|
||||
|
||||
#![warn(missing_docs)]
|
||||
#![deny(unreachable_pub)]
|
||||
@@ -36,6 +36,8 @@
|
||||
//! `0.3.8-pre.004` adds payload-free random-access RawTransaction inspection;
|
||||
//! `0.3.8-pre.005` adds the corresponding data-free RawAccountState inspection
|
||||
//! with exact counts while preserving both keyset traversal families unchanged.
|
||||
//! `0.3.8-pre.009` adds safe random-access observation inspection for both RAW
|
||||
//! families without changing the physical schema or read-by-key contracts.
|
||||
//!
|
||||
//! This crate depends on `ksp-store-api` and never on `ksp-store-lib`. The
|
||||
//! common facade consumes only this crate's narrow backend bridge and never
|
||||
@@ -85,6 +87,8 @@ pub(crate) use self::raw_account::cursor::raw_account_physical_page_limit;
|
||||
pub(crate) use self::raw_account::get_raw_account_observation;
|
||||
/// Private RAW account state reader consumed by the physical backend runtime.
|
||||
pub(crate) use self::raw_account::get_raw_account_state;
|
||||
/// Private safe RAW account-observation inspection reader consumed by the physical backend runtime.
|
||||
pub(crate) use self::raw_account::inspect_raw_account_observations;
|
||||
/// Private data-free RAW account-state inspection reader consumed by the physical backend runtime.
|
||||
pub(crate) use self::raw_account::inspect_raw_account_states;
|
||||
/// Private RAW account-state list reader consumed by the physical backend runtime.
|
||||
@@ -107,6 +111,8 @@ pub(crate) use self::raw_transaction::get_raw_transaction_observation;
|
||||
pub(crate) use self::raw_transaction::get_raw_transaction_retention_state;
|
||||
/// Private RAW transaction tombstone reader consumed by the physical backend runtime.
|
||||
pub(crate) use self::raw_transaction::get_raw_transaction_tombstone;
|
||||
/// Private safe RAW transaction-observation inspection reader consumed by the physical backend runtime.
|
||||
pub(crate) use self::raw_transaction::inspect_raw_transaction_observations;
|
||||
/// Private payload-free RAW transaction inspection reader consumed by the physical backend runtime.
|
||||
pub(crate) use self::raw_transaction::inspect_raw_transactions;
|
||||
/// Private RAW transaction list reader consumed by the physical backend runtime.
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
// file: crates/ksp-store-postgres-lib/src/raw_account.rs
|
||||
// version: 6
|
||||
// version: 7
|
||||
|
||||
pub(crate) mod cursor;
|
||||
|
||||
@@ -7,6 +7,8 @@ const GET_ACCOUNT_OBSERVATION_SQL: &str = "SELECT observation_key, account_pubke
|
||||
const GET_ACCOUNT_STATE_SQL: &str = "SELECT pubkey, slot::text AS slot_text, state_hash, lamports::text AS lamports_text, owner, executable, rent_epoch::text AS rent_epoch_text, data FROM ksp_raw_account_states WHERE pubkey = $1 AND slot = $2::TEXT::NUMERIC AND state_hash = $3";
|
||||
const INSERT_ACCOUNT_OBSERVATION_SQL: &str = "INSERT INTO ksp_raw_account_observations (observation_key, account_pubkey, account_slot, account_state_hash, provider, protocol, acquisition_method, origin, received_at_unix_millis, capture_session_id, commitment, endpoint_id, filter_id, observed_at_unix_millis, source_payload_hash, source_payload_size_bytes, is_startup, transaction_signature, write_version) VALUES ($1, $2, $3::TEXT::NUMERIC, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19::TEXT::NUMERIC) ON CONFLICT (observation_key) DO NOTHING RETURNING observation_key";
|
||||
const INSERT_ACCOUNT_STATE_SQL: &str = "INSERT INTO ksp_raw_account_states (pubkey, slot, state_hash, lamports, owner, executable, rent_epoch, data) VALUES ($1, $2::TEXT::NUMERIC, $3, $4::TEXT::NUMERIC, $5, $6, $7::TEXT::NUMERIC, $8) ON CONFLICT (pubkey, slot, state_hash) DO NOTHING RETURNING pubkey";
|
||||
const INSPECT_ACCOUNT_OBSERVATIONS_ASC_SQL: &str = "WITH filtered_count AS (SELECT COUNT(*)::TEXT AS filtered_count_text FROM ksp_raw_account_observations WHERE ($1::BYTEA IS NULL OR (account_pubkey = $1 AND account_slot = $2::TEXT::NUMERIC AND account_state_hash = $3))), counts AS (SELECT filtered_count_text, CASE WHEN $1::BYTEA IS NULL THEN filtered_count_text ELSE (SELECT COUNT(*)::TEXT FROM ksp_raw_account_observations) END AS total_count_text FROM filtered_count) SELECT counts.total_count_text, counts.filtered_count_text, page.observation_key IS NOT NULL AS page_present, page.observation_key, page.account_pubkey, page.account_slot_text, page.account_state_hash, page.provider, page.protocol, page.acquisition_method, page.origin, page.received_at_unix_millis, page.capture_session_id, page.commitment, page.endpoint_id, page.filter_id, page.observed_at_unix_millis, page.source_payload_hash, page.source_payload_size_bytes, page.is_startup, page.transaction_signature, page.write_version_text FROM counts LEFT JOIN LATERAL (SELECT observation_key, account_pubkey, account_slot::TEXT AS account_slot_text, account_state_hash, provider, protocol, acquisition_method, origin, received_at_unix_millis, capture_session_id, commitment, endpoint_id, filter_id, observed_at_unix_millis, source_payload_hash, source_payload_size_bytes, is_startup, transaction_signature, write_version::TEXT AS write_version_text FROM ksp_raw_account_observations WHERE ($1::BYTEA IS NULL OR (account_pubkey = $1 AND account_slot = $2::TEXT::NUMERIC AND account_state_hash = $3)) ORDER BY received_at_unix_millis ASC, observation_key ASC LIMIT $4 OFFSET $5) AS page ON TRUE";
|
||||
const INSPECT_ACCOUNT_OBSERVATIONS_DESC_SQL: &str = "WITH filtered_count AS (SELECT COUNT(*)::TEXT AS filtered_count_text FROM ksp_raw_account_observations WHERE ($1::BYTEA IS NULL OR (account_pubkey = $1 AND account_slot = $2::TEXT::NUMERIC AND account_state_hash = $3))), counts AS (SELECT filtered_count_text, CASE WHEN $1::BYTEA IS NULL THEN filtered_count_text ELSE (SELECT COUNT(*)::TEXT FROM ksp_raw_account_observations) END AS total_count_text FROM filtered_count) SELECT counts.total_count_text, counts.filtered_count_text, page.observation_key IS NOT NULL AS page_present, page.observation_key, page.account_pubkey, page.account_slot_text, page.account_state_hash, page.provider, page.protocol, page.acquisition_method, page.origin, page.received_at_unix_millis, page.capture_session_id, page.commitment, page.endpoint_id, page.filter_id, page.observed_at_unix_millis, page.source_payload_hash, page.source_payload_size_bytes, page.is_startup, page.transaction_signature, page.write_version_text FROM counts LEFT JOIN LATERAL (SELECT observation_key, account_pubkey, account_slot::TEXT AS account_slot_text, account_state_hash, provider, protocol, acquisition_method, origin, received_at_unix_millis, capture_session_id, commitment, endpoint_id, filter_id, observed_at_unix_millis, source_payload_hash, source_payload_size_bytes, is_startup, transaction_signature, write_version::TEXT AS write_version_text FROM ksp_raw_account_observations WHERE ($1::BYTEA IS NULL OR (account_pubkey = $1 AND account_slot = $2::TEXT::NUMERIC AND account_state_hash = $3)) ORDER BY received_at_unix_millis DESC, observation_key DESC LIMIT $4 OFFSET $5) AS page ON TRUE";
|
||||
const INSPECT_ACCOUNT_STATES_ASC_SQL: &str = "WITH filtered_count AS (SELECT COUNT(*)::TEXT AS filtered_count_text FROM ksp_raw_account_states WHERE ($1::BYTEA IS NULL OR pubkey = $1::BYTEA) AND ($2::TEXT IS NULL OR slot >= $2::TEXT::NUMERIC) AND ($3::TEXT IS NULL OR slot <= $3::TEXT::NUMERIC)), counts AS (SELECT filtered_count_text, CASE WHEN $1::BYTEA IS NULL AND $2::TEXT IS NULL AND $3::TEXT IS NULL THEN filtered_count_text ELSE (SELECT COUNT(*)::TEXT FROM ksp_raw_account_states) END AS total_count_text FROM filtered_count) SELECT counts.total_count_text, counts.filtered_count_text, page.pubkey IS NOT NULL AS page_present, page.pubkey, page.slot_text, page.state_hash, page.lamports_text, page.owner, page.executable, page.rent_epoch_text, page.data_length_bytes FROM counts LEFT JOIN LATERAL (SELECT account_row.pubkey, account_row.slot::TEXT AS slot_text, account_row.state_hash, account_row.lamports::TEXT AS lamports_text, account_row.owner, account_row.executable, account_row.rent_epoch::TEXT AS rent_epoch_text, OCTET_LENGTH(account_row.data)::BIGINT AS data_length_bytes FROM ksp_raw_account_states AS account_row WHERE ($1::BYTEA IS NULL OR account_row.pubkey = $1::BYTEA) AND ($2::TEXT IS NULL OR account_row.slot >= $2::TEXT::NUMERIC) AND ($3::TEXT IS NULL OR account_row.slot <= $3::TEXT::NUMERIC) ORDER BY account_row.slot ASC, account_row.pubkey ASC, account_row.state_hash ASC LIMIT $4 OFFSET $5) AS page ON TRUE";
|
||||
const INSPECT_ACCOUNT_STATES_DESC_SQL: &str = "WITH filtered_count AS (SELECT COUNT(*)::TEXT AS filtered_count_text FROM ksp_raw_account_states WHERE ($1::BYTEA IS NULL OR pubkey = $1::BYTEA) AND ($2::TEXT IS NULL OR slot >= $2::TEXT::NUMERIC) AND ($3::TEXT IS NULL OR slot <= $3::TEXT::NUMERIC)), counts AS (SELECT filtered_count_text, CASE WHEN $1::BYTEA IS NULL AND $2::TEXT IS NULL AND $3::TEXT IS NULL THEN filtered_count_text ELSE (SELECT COUNT(*)::TEXT FROM ksp_raw_account_states) END AS total_count_text FROM filtered_count) SELECT counts.total_count_text, counts.filtered_count_text, page.pubkey IS NOT NULL AS page_present, page.pubkey, page.slot_text, page.state_hash, page.lamports_text, page.owner, page.executable, page.rent_epoch_text, page.data_length_bytes FROM counts LEFT JOIN LATERAL (SELECT account_row.pubkey, account_row.slot::TEXT AS slot_text, account_row.state_hash, account_row.lamports::TEXT AS lamports_text, account_row.owner, account_row.executable, account_row.rent_epoch::TEXT AS rent_epoch_text, OCTET_LENGTH(account_row.data)::BIGINT AS data_length_bytes FROM ksp_raw_account_states AS account_row WHERE ($1::BYTEA IS NULL OR account_row.pubkey = $1::BYTEA) AND ($2::TEXT IS NULL OR account_row.slot >= $2::TEXT::NUMERIC) AND ($3::TEXT IS NULL OR account_row.slot <= $3::TEXT::NUMERIC) ORDER BY account_row.slot DESC, account_row.pubkey DESC, account_row.state_hash DESC LIMIT $4 OFFSET $5) AS page ON TRUE";
|
||||
const LIST_ACCOUNT_STATES_ASC_SQL: &str = "SELECT pubkey, slot::text AS slot_text, state_hash FROM ksp_raw_account_states WHERE ($1::TEXT IS NULL OR slot >= $1::TEXT::NUMERIC) AND ($2::TEXT IS NULL OR slot <= $2::TEXT::NUMERIC) AND ($3::TEXT IS NULL OR (slot, pubkey, state_hash) > ($3::TEXT::NUMERIC, $4::BYTEA, $5::BYTEA)) ORDER BY slot ASC, pubkey ASC, state_hash ASC LIMIT $6";
|
||||
@@ -38,6 +40,31 @@ struct RawAccountInspectionDbRow {
|
||||
total_count_text: std::string::String,
|
||||
}
|
||||
|
||||
struct RawAccountObservationInspectionDbRow {
|
||||
account_pubkey: std::option::Option<std::vec::Vec<u8>>,
|
||||
account_slot_text: std::option::Option<std::string::String>,
|
||||
account_state_hash: std::option::Option<std::vec::Vec<u8>>,
|
||||
acquisition_method: std::option::Option<std::string::String>,
|
||||
capture_session_id: std::option::Option<std::string::String>,
|
||||
commitment: std::option::Option<std::string::String>,
|
||||
endpoint_id: std::option::Option<std::string::String>,
|
||||
filter_id: std::option::Option<std::string::String>,
|
||||
filtered_count_text: std::string::String,
|
||||
is_startup: std::option::Option<bool>,
|
||||
observation_key: std::option::Option<std::vec::Vec<u8>>,
|
||||
observed_at_unix_millis: std::option::Option<i64>,
|
||||
origin: std::option::Option<std::string::String>,
|
||||
page_present: bool,
|
||||
protocol: std::option::Option<std::string::String>,
|
||||
provider: std::option::Option<std::string::String>,
|
||||
received_at_unix_millis: std::option::Option<i64>,
|
||||
source_payload_hash: std::option::Option<std::vec::Vec<u8>>,
|
||||
source_payload_size_bytes: std::option::Option<i64>,
|
||||
total_count_text: std::string::String,
|
||||
transaction_signature: std::option::Option<std::vec::Vec<u8>>,
|
||||
write_version_text: std::option::Option<std::string::String>,
|
||||
}
|
||||
|
||||
struct RawAccountObservationDbRow {
|
||||
account_pubkey: std::vec::Vec<u8>,
|
||||
account_slot_text: std::string::String,
|
||||
@@ -317,6 +344,121 @@ pub(crate) async fn inspect_raw_account_states(
|
||||
};
|
||||
}
|
||||
|
||||
/// Inspects one safe random-access RAW account-observation window with exact counts.
|
||||
pub(crate) async fn inspect_raw_account_observations(
|
||||
pool: &deadpool_postgres::Pool,
|
||||
network: &ksp_store_api::RawNetworkId,
|
||||
query: &ksp_store_api::RawAccountObservationInspectionQuery,
|
||||
) -> std::result::Result<ksp_store_api::RawInspectionPage<ksp_store_api::RawAccountObservationSummary>, crate::PostgresBackendError> {
|
||||
if query.network() != network {
|
||||
return std::result::Result::Err(crate::PostgresBackendError::new(
|
||||
crate::PostgresBackendErrorKind::WrongNetwork,
|
||||
"raw_account_observation_inspection_network",
|
||||
));
|
||||
}
|
||||
let sql = match query.direction() {
|
||||
ksp_store_api::RawSortDirection::Ascending => INSPECT_ACCOUNT_OBSERVATIONS_ASC_SQL,
|
||||
ksp_store_api::RawSortDirection::Descending => INSPECT_ACCOUNT_OBSERVATIONS_DESC_SQL,
|
||||
_ => {
|
||||
return std::result::Result::Err(crate::PostgresBackendError::new(
|
||||
crate::PostgresBackendErrorKind::QueryInvalid,
|
||||
"raw_account_observation_inspection_direction",
|
||||
));
|
||||
},
|
||||
};
|
||||
let (sql_limit, sql_offset) = match raw_account_inspection_sql_window(query.page()) {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(error) => return std::result::Result::Err(error),
|
||||
};
|
||||
let account_pubkey = query.account().map(|reference| return reference.pubkey().to_bytes().to_vec());
|
||||
let account_slot_text = query.account().map(|reference| return reference.slot().to_string());
|
||||
let account_state_hash = query.account().map(|reference| return reference.state_hash().as_bytes().to_vec());
|
||||
let client_result = pool.get().await;
|
||||
let client = match client_result {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(error) => return std::result::Result::Err(crate::map_pool_error(error)),
|
||||
};
|
||||
let rows_result = client.query(sql, &[&account_pubkey, &account_slot_text, &account_state_hash, &sql_limit, &sql_offset]).await;
|
||||
let rows = match rows_result {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(_) => {
|
||||
return std::result::Result::Err(crate::PostgresBackendError::new(
|
||||
crate::PostgresBackendErrorKind::ReadFailed,
|
||||
"raw_account_observation_inspection_query",
|
||||
));
|
||||
},
|
||||
};
|
||||
if rows.is_empty() {
|
||||
return std::result::Result::Err(data_invalid("raw_account_observation_inspection_cardinality"));
|
||||
}
|
||||
let row_count = rows.len();
|
||||
let mut total_items = std::option::Option::None;
|
||||
let mut filtered_items = std::option::Option::None;
|
||||
let mut items = std::vec::Vec::new();
|
||||
for row in rows {
|
||||
let physical = match raw_account_observation_inspection_db_row(&row) {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(error) => return std::result::Result::Err(error),
|
||||
};
|
||||
let row_total = match decode_u64_decimal(physical.total_count_text.as_str(), "raw_account_observation_inspection_total") {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(error) => return std::result::Result::Err(error),
|
||||
};
|
||||
let row_filtered = match decode_u64_decimal(physical.filtered_count_text.as_str(), "raw_account_observation_inspection_filtered") {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(error) => return std::result::Result::Err(error),
|
||||
};
|
||||
match (total_items, filtered_items) {
|
||||
(std::option::Option::None, std::option::Option::None) => {
|
||||
total_items = std::option::Option::Some(row_total);
|
||||
filtered_items = std::option::Option::Some(row_filtered);
|
||||
},
|
||||
(std::option::Option::Some(total), std::option::Option::Some(filtered)) if total == row_total && filtered == row_filtered => {},
|
||||
_ => return std::result::Result::Err(data_invalid("raw_account_observation_inspection_counts")),
|
||||
}
|
||||
if physical.page_present {
|
||||
let observation = match decode_raw_account_observation_inspection_row(network, physical) {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(error) => return std::result::Result::Err(error),
|
||||
};
|
||||
if let std::option::Option::Some(reference) = query.account() {
|
||||
if observation.account() != reference {
|
||||
return std::result::Result::Err(data_invalid("raw_account_observation_inspection_reference"));
|
||||
}
|
||||
}
|
||||
items.push(ksp_store_api::RawAccountObservationSummary::new(
|
||||
observation.observation_key(),
|
||||
observation.account().clone(),
|
||||
observation.provenance().clone(),
|
||||
observation.is_startup(),
|
||||
observation.transaction_signature(),
|
||||
observation.write_version(),
|
||||
));
|
||||
} else if row_count != 1 || !raw_account_observation_inspection_empty_page_is_clean(&physical) {
|
||||
return std::result::Result::Err(data_invalid("raw_account_observation_inspection_page"));
|
||||
}
|
||||
}
|
||||
let total_items = match total_items {
|
||||
std::option::Option::Some(value) => value,
|
||||
std::option::Option::None => return std::result::Result::Err(data_invalid("raw_account_observation_inspection_total")),
|
||||
};
|
||||
let filtered_items = match filtered_items {
|
||||
std::option::Option::Some(value) => value,
|
||||
std::option::Option::None => return std::result::Result::Err(data_invalid("raw_account_observation_inspection_filtered")),
|
||||
};
|
||||
let item_count = match u64::try_from(items.len()) {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(_) => return std::result::Result::Err(data_invalid("raw_account_observation_inspection_page")),
|
||||
};
|
||||
if item_count > query.page().limit().get() {
|
||||
return std::result::Result::Err(data_invalid("raw_account_observation_inspection_page"));
|
||||
}
|
||||
return match ksp_store_api::RawInspectionPage::try_new(items, total_items, filtered_items) {
|
||||
std::result::Result::Ok(value) => std::result::Result::Ok(value),
|
||||
std::result::Result::Err(_) => std::result::Result::Err(data_invalid("raw_account_observation_inspection_page")),
|
||||
};
|
||||
}
|
||||
|
||||
/// Reads one persisted RAW account observation by producer-owned idempotence key.
|
||||
pub(crate) async fn get_raw_account_observation(
|
||||
pool: &deadpool_postgres::Pool,
|
||||
@@ -495,6 +637,141 @@ fn decode_raw_account_list_row(
|
||||
return std::result::Result::Ok((slot, reference));
|
||||
}
|
||||
|
||||
fn raw_account_observation_inspection_empty_page_is_clean(row: &RawAccountObservationInspectionDbRow) -> bool {
|
||||
return row.account_pubkey.is_none()
|
||||
&& row.account_slot_text.is_none()
|
||||
&& row.account_state_hash.is_none()
|
||||
&& row.acquisition_method.is_none()
|
||||
&& row.capture_session_id.is_none()
|
||||
&& row.commitment.is_none()
|
||||
&& row.endpoint_id.is_none()
|
||||
&& row.filter_id.is_none()
|
||||
&& row.is_startup.is_none()
|
||||
&& row.observation_key.is_none()
|
||||
&& row.observed_at_unix_millis.is_none()
|
||||
&& row.origin.is_none()
|
||||
&& row.protocol.is_none()
|
||||
&& row.provider.is_none()
|
||||
&& row.received_at_unix_millis.is_none()
|
||||
&& row.source_payload_hash.is_none()
|
||||
&& row.source_payload_size_bytes.is_none()
|
||||
&& row.transaction_signature.is_none()
|
||||
&& row.write_version_text.is_none();
|
||||
}
|
||||
|
||||
fn raw_account_observation_inspection_db_row(
|
||||
row: &tokio_postgres::Row,
|
||||
) -> std::result::Result<RawAccountObservationInspectionDbRow, crate::PostgresBackendError> {
|
||||
macro_rules! get_optional {
|
||||
($name:literal, $ty:ty) => {
|
||||
match row.try_get::<_, std::option::Option<$ty>>($name) {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(_) => return std::result::Result::Err(data_invalid("raw_account_observation_inspection_decode")),
|
||||
}
|
||||
};
|
||||
}
|
||||
let filtered_count_text = match row.try_get::<_, std::string::String>("filtered_count_text") {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(_) => return std::result::Result::Err(data_invalid("raw_account_observation_inspection_decode")),
|
||||
};
|
||||
let page_present = match row.try_get::<_, bool>("page_present") {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(_) => return std::result::Result::Err(data_invalid("raw_account_observation_inspection_decode")),
|
||||
};
|
||||
let total_count_text = match row.try_get::<_, std::string::String>("total_count_text") {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(_) => return std::result::Result::Err(data_invalid("raw_account_observation_inspection_decode")),
|
||||
};
|
||||
return std::result::Result::Ok(RawAccountObservationInspectionDbRow {
|
||||
account_pubkey: get_optional!("account_pubkey", std::vec::Vec<u8>),
|
||||
account_slot_text: get_optional!("account_slot_text", std::string::String),
|
||||
account_state_hash: get_optional!("account_state_hash", std::vec::Vec<u8>),
|
||||
acquisition_method: get_optional!("acquisition_method", std::string::String),
|
||||
capture_session_id: get_optional!("capture_session_id", std::string::String),
|
||||
commitment: get_optional!("commitment", std::string::String),
|
||||
endpoint_id: get_optional!("endpoint_id", std::string::String),
|
||||
filter_id: get_optional!("filter_id", std::string::String),
|
||||
filtered_count_text,
|
||||
is_startup: get_optional!("is_startup", bool),
|
||||
observation_key: get_optional!("observation_key", std::vec::Vec<u8>),
|
||||
observed_at_unix_millis: get_optional!("observed_at_unix_millis", i64),
|
||||
origin: get_optional!("origin", std::string::String),
|
||||
page_present,
|
||||
protocol: get_optional!("protocol", std::string::String),
|
||||
provider: get_optional!("provider", std::string::String),
|
||||
received_at_unix_millis: get_optional!("received_at_unix_millis", i64),
|
||||
source_payload_hash: get_optional!("source_payload_hash", std::vec::Vec<u8>),
|
||||
source_payload_size_bytes: get_optional!("source_payload_size_bytes", i64),
|
||||
total_count_text,
|
||||
transaction_signature: get_optional!("transaction_signature", std::vec::Vec<u8>),
|
||||
write_version_text: get_optional!("write_version_text", std::string::String),
|
||||
});
|
||||
}
|
||||
|
||||
fn decode_raw_account_observation_inspection_row(
|
||||
network: &ksp_store_api::RawNetworkId,
|
||||
row: RawAccountObservationInspectionDbRow,
|
||||
) -> std::result::Result<ksp_store_api::RawAccountObservation, crate::PostgresBackendError> {
|
||||
let account_pubkey = match inspection_required(row.account_pubkey, "raw_account_observation_inspection_shape") {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(error) => return std::result::Result::Err(error),
|
||||
};
|
||||
let account_slot_text = match inspection_required(row.account_slot_text, "raw_account_observation_inspection_shape") {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(error) => return std::result::Result::Err(error),
|
||||
};
|
||||
let account_state_hash = match inspection_required(row.account_state_hash, "raw_account_observation_inspection_shape") {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(error) => return std::result::Result::Err(error),
|
||||
};
|
||||
let acquisition_method = match inspection_required(row.acquisition_method, "raw_account_observation_inspection_shape") {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(error) => return std::result::Result::Err(error),
|
||||
};
|
||||
let observation_key = match inspection_required(row.observation_key, "raw_account_observation_inspection_shape") {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(error) => return std::result::Result::Err(error),
|
||||
};
|
||||
let origin = match inspection_required(row.origin, "raw_account_observation_inspection_shape") {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(error) => return std::result::Result::Err(error),
|
||||
};
|
||||
let protocol = match inspection_required(row.protocol, "raw_account_observation_inspection_shape") {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(error) => return std::result::Result::Err(error),
|
||||
};
|
||||
let provider = match inspection_required(row.provider, "raw_account_observation_inspection_shape") {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(error) => return std::result::Result::Err(error),
|
||||
};
|
||||
let received_at_unix_millis = match inspection_required(row.received_at_unix_millis, "raw_account_observation_inspection_shape") {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(error) => return std::result::Result::Err(error),
|
||||
};
|
||||
let physical = RawAccountObservationDbRow {
|
||||
account_pubkey,
|
||||
account_slot_text,
|
||||
account_state_hash,
|
||||
acquisition_method,
|
||||
capture_session_id: row.capture_session_id,
|
||||
commitment: row.commitment,
|
||||
endpoint_id: row.endpoint_id,
|
||||
filter_id: row.filter_id,
|
||||
is_startup: row.is_startup,
|
||||
observation_key,
|
||||
observed_at_unix_millis: row.observed_at_unix_millis,
|
||||
origin,
|
||||
protocol,
|
||||
provider,
|
||||
received_at_unix_millis,
|
||||
source_payload_hash: row.source_payload_hash,
|
||||
source_payload_size_bytes: row.source_payload_size_bytes,
|
||||
transaction_signature: row.transaction_signature,
|
||||
write_version_text: row.write_version_text,
|
||||
};
|
||||
return decode_raw_account_observation_row(network, physical);
|
||||
}
|
||||
|
||||
fn raw_account_observation_db_row(row: &tokio_postgres::Row) -> std::result::Result<RawAccountObservationDbRow, crate::PostgresBackendError> {
|
||||
let account_pubkey: std::vec::Vec<u8> = match row.try_get("account_pubkey") {
|
||||
std::result::Result::Ok(value) => value,
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
// file: crates/ksp-store-postgres-lib/src/raw_transaction.rs
|
||||
// version: 5
|
||||
// version: 6
|
||||
|
||||
pub(crate) mod cursor;
|
||||
|
||||
@@ -12,6 +12,8 @@ const GET_TRANSACTION_SQL: &str = "SELECT transaction_row.signature, transaction
|
||||
const INSERT_ARCHIVE_PAYLOAD_SQL: &str = "INSERT INTO ksp_raw_transaction_archive_payloads (signature, payload) VALUES ($1, $2)";
|
||||
const INSERT_OBSERVATION_SQL: &str = "INSERT INTO ksp_raw_transaction_observations (observation_key, transaction_signature, provider, protocol, acquisition_method, origin, received_at_unix_millis, capture_session_id, commitment, endpoint_id, filter_id, observed_at_unix_millis, source_payload_hash, source_payload_size_bytes) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14) ON CONFLICT (observation_key) DO NOTHING RETURNING observation_key";
|
||||
const INSERT_TRANSACTION_SQL: &str = "INSERT INTO ksp_raw_transactions (signature, slot, block_time_unix_millis, format_id, format_version, content_hash, payload, retention_state) VALUES ($1, $2::TEXT::NUMERIC, $3, $4, $5, $6, $7, 'full') ON CONFLICT (signature) DO NOTHING RETURNING signature";
|
||||
const INSPECT_OBSERVATIONS_ASC_SQL: &str = "WITH filtered_count AS (SELECT COUNT(*)::TEXT AS filtered_count_text FROM ksp_raw_transaction_observations WHERE ($1::BYTEA IS NULL OR transaction_signature = $1)), counts AS (SELECT filtered_count_text, CASE WHEN $1::BYTEA IS NULL THEN filtered_count_text ELSE (SELECT COUNT(*)::TEXT FROM ksp_raw_transaction_observations) END AS total_count_text FROM filtered_count) SELECT counts.total_count_text, counts.filtered_count_text, page.observation_key IS NOT NULL AS page_present, page.observation_key, page.transaction_signature, page.provider, page.protocol, page.acquisition_method, page.origin, page.received_at_unix_millis, page.capture_session_id, page.commitment, page.endpoint_id, page.filter_id, page.observed_at_unix_millis, page.source_payload_hash, page.source_payload_size_bytes FROM counts LEFT JOIN LATERAL (SELECT observation_key, transaction_signature, provider, protocol, acquisition_method, origin, received_at_unix_millis, capture_session_id, commitment, endpoint_id, filter_id, observed_at_unix_millis, source_payload_hash, source_payload_size_bytes FROM ksp_raw_transaction_observations WHERE ($1::BYTEA IS NULL OR transaction_signature = $1) ORDER BY received_at_unix_millis ASC, observation_key ASC LIMIT $2 OFFSET $3) AS page ON TRUE";
|
||||
const INSPECT_OBSERVATIONS_DESC_SQL: &str = "WITH filtered_count AS (SELECT COUNT(*)::TEXT AS filtered_count_text FROM ksp_raw_transaction_observations WHERE ($1::BYTEA IS NULL OR transaction_signature = $1)), counts AS (SELECT filtered_count_text, CASE WHEN $1::BYTEA IS NULL THEN filtered_count_text ELSE (SELECT COUNT(*)::TEXT FROM ksp_raw_transaction_observations) END AS total_count_text FROM filtered_count) SELECT counts.total_count_text, counts.filtered_count_text, page.observation_key IS NOT NULL AS page_present, page.observation_key, page.transaction_signature, page.provider, page.protocol, page.acquisition_method, page.origin, page.received_at_unix_millis, page.capture_session_id, page.commitment, page.endpoint_id, page.filter_id, page.observed_at_unix_millis, page.source_payload_hash, page.source_payload_size_bytes FROM counts LEFT JOIN LATERAL (SELECT observation_key, transaction_signature, provider, protocol, acquisition_method, origin, received_at_unix_millis, capture_session_id, commitment, endpoint_id, filter_id, observed_at_unix_millis, source_payload_hash, source_payload_size_bytes FROM ksp_raw_transaction_observations WHERE ($1::BYTEA IS NULL OR transaction_signature = $1) ORDER BY received_at_unix_millis DESC, observation_key DESC LIMIT $2 OFFSET $3) AS page ON TRUE";
|
||||
const INSPECT_TRANSACTIONS_ASC_SQL: &str = "WITH filtered_count AS (SELECT COUNT(*)::TEXT AS filtered_count_text FROM ksp_raw_transactions WHERE ($1::TEXT IS NULL OR slot >= $1::TEXT::NUMERIC) AND ($2::TEXT IS NULL OR slot <= $2::TEXT::NUMERIC)), counts AS (SELECT filtered_count_text, CASE WHEN $1::TEXT IS NULL AND $2::TEXT IS NULL THEN filtered_count_text ELSE (SELECT COUNT(*)::TEXT FROM ksp_raw_transactions) END AS total_count_text FROM filtered_count) SELECT counts.total_count_text, counts.filtered_count_text, page.signature IS NOT NULL AS page_present, page.signature, page.slot_text, page.block_time_unix_millis, page.format_id, page.format_version, page.content_hash, page.retention_state, page.payload_size_bytes, page.hot_payload_present, page.archive_payload_present FROM counts LEFT JOIN LATERAL (SELECT transaction_row.signature, transaction_row.slot::TEXT AS slot_text, transaction_row.block_time_unix_millis, transaction_row.format_id, transaction_row.format_version, transaction_row.content_hash, transaction_row.retention_state, CASE WHEN transaction_row.retention_state = 'full' THEN OCTET_LENGTH(transaction_row.payload)::BIGINT WHEN transaction_row.retention_state = 'archived' THEN OCTET_LENGTH(archive_row.payload)::BIGINT ELSE NULL END AS payload_size_bytes, transaction_row.payload IS NOT NULL AS hot_payload_present, archive_row.payload IS NOT NULL AS archive_payload_present FROM ksp_raw_transactions AS transaction_row LEFT JOIN ksp_raw_transaction_archive_payloads AS archive_row ON archive_row.signature = transaction_row.signature WHERE ($1::TEXT IS NULL OR transaction_row.slot >= $1::TEXT::NUMERIC) AND ($2::TEXT IS NULL OR transaction_row.slot <= $2::TEXT::NUMERIC) ORDER BY transaction_row.slot ASC, transaction_row.signature ASC LIMIT $3 OFFSET $4) AS page ON TRUE";
|
||||
const INSPECT_TRANSACTIONS_DESC_SQL: &str = "WITH filtered_count AS (SELECT COUNT(*)::TEXT AS filtered_count_text FROM ksp_raw_transactions WHERE ($1::TEXT IS NULL OR slot >= $1::TEXT::NUMERIC) AND ($2::TEXT IS NULL OR slot <= $2::TEXT::NUMERIC)), counts AS (SELECT filtered_count_text, CASE WHEN $1::TEXT IS NULL AND $2::TEXT IS NULL THEN filtered_count_text ELSE (SELECT COUNT(*)::TEXT FROM ksp_raw_transactions) END AS total_count_text FROM filtered_count) SELECT counts.total_count_text, counts.filtered_count_text, page.signature IS NOT NULL AS page_present, page.signature, page.slot_text, page.block_time_unix_millis, page.format_id, page.format_version, page.content_hash, page.retention_state, page.payload_size_bytes, page.hot_payload_present, page.archive_payload_present FROM counts LEFT JOIN LATERAL (SELECT transaction_row.signature, transaction_row.slot::TEXT AS slot_text, transaction_row.block_time_unix_millis, transaction_row.format_id, transaction_row.format_version, transaction_row.content_hash, transaction_row.retention_state, CASE WHEN transaction_row.retention_state = 'full' THEN OCTET_LENGTH(transaction_row.payload)::BIGINT WHEN transaction_row.retention_state = 'archived' THEN OCTET_LENGTH(archive_row.payload)::BIGINT ELSE NULL END AS payload_size_bytes, transaction_row.payload IS NOT NULL AS hot_payload_present, archive_row.payload IS NOT NULL AS archive_payload_present FROM ksp_raw_transactions AS transaction_row LEFT JOIN ksp_raw_transaction_archive_payloads AS archive_row ON archive_row.signature = transaction_row.signature WHERE ($1::TEXT IS NULL OR transaction_row.slot >= $1::TEXT::NUMERIC) AND ($2::TEXT IS NULL OR transaction_row.slot <= $2::TEXT::NUMERIC) ORDER BY transaction_row.slot DESC, transaction_row.signature DESC LIMIT $3 OFFSET $4) AS page ON TRUE";
|
||||
const LIST_TRANSACTIONS_ASC_SQL: &str = "SELECT signature, slot::text AS slot_text FROM ksp_raw_transactions WHERE retention_state <> 'purged' AND ($1::TEXT IS NULL OR slot >= $1::TEXT::NUMERIC) AND ($2::TEXT IS NULL OR slot <= $2::TEXT::NUMERIC) AND ($3::TEXT IS NULL OR (slot, signature) > ($3::TEXT::NUMERIC, $4::BYTEA)) ORDER BY slot ASC, signature ASC LIMIT $5";
|
||||
@@ -48,6 +50,26 @@ struct RawInspectionDbRow {
|
||||
total_count_text: std::string::String,
|
||||
}
|
||||
|
||||
struct RawObservationInspectionDbRow {
|
||||
acquisition_method: std::option::Option<std::string::String>,
|
||||
capture_session_id: std::option::Option<std::string::String>,
|
||||
commitment: std::option::Option<std::string::String>,
|
||||
endpoint_id: std::option::Option<std::string::String>,
|
||||
filter_id: std::option::Option<std::string::String>,
|
||||
filtered_count_text: std::string::String,
|
||||
observation_key: std::option::Option<std::vec::Vec<u8>>,
|
||||
observed_at_unix_millis: std::option::Option<i64>,
|
||||
origin: std::option::Option<std::string::String>,
|
||||
page_present: bool,
|
||||
protocol: std::option::Option<std::string::String>,
|
||||
provider: std::option::Option<std::string::String>,
|
||||
received_at_unix_millis: std::option::Option<i64>,
|
||||
source_payload_hash: std::option::Option<std::vec::Vec<u8>>,
|
||||
source_payload_size_bytes: std::option::Option<i64>,
|
||||
total_count_text: std::string::String,
|
||||
transaction_signature: std::option::Option<std::vec::Vec<u8>>,
|
||||
}
|
||||
|
||||
struct RawObservationDbRow {
|
||||
acquisition_method: std::string::String,
|
||||
capture_session_id: std::option::Option<std::string::String>,
|
||||
@@ -307,6 +329,116 @@ pub(crate) async fn inspect_raw_transactions(
|
||||
};
|
||||
}
|
||||
|
||||
/// Inspects one safe random-access RAW transaction-observation window with exact counts.
|
||||
pub(crate) async fn inspect_raw_transaction_observations(
|
||||
pool: &deadpool_postgres::Pool,
|
||||
network: &ksp_store_api::RawNetworkId,
|
||||
query: &ksp_store_api::RawTransactionObservationInspectionQuery,
|
||||
) -> std::result::Result<ksp_store_api::RawInspectionPage<ksp_store_api::RawTransactionObservationSummary>, crate::PostgresBackendError> {
|
||||
if query.network() != network {
|
||||
return std::result::Result::Err(crate::PostgresBackendError::new(
|
||||
crate::PostgresBackendErrorKind::WrongNetwork,
|
||||
"raw_transaction_observation_inspection_network",
|
||||
));
|
||||
}
|
||||
let sql = match query.direction() {
|
||||
ksp_store_api::RawSortDirection::Ascending => INSPECT_OBSERVATIONS_ASC_SQL,
|
||||
ksp_store_api::RawSortDirection::Descending => INSPECT_OBSERVATIONS_DESC_SQL,
|
||||
_ => {
|
||||
return std::result::Result::Err(crate::PostgresBackendError::new(
|
||||
crate::PostgresBackendErrorKind::QueryInvalid,
|
||||
"raw_transaction_observation_inspection_direction",
|
||||
));
|
||||
},
|
||||
};
|
||||
let (sql_limit, sql_offset) = match raw_transaction_inspection_sql_window(query.page()) {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(error) => return std::result::Result::Err(error),
|
||||
};
|
||||
let signature = query.transaction().map(|reference| return reference.signature().as_bytes().to_vec());
|
||||
let client_result = pool.get().await;
|
||||
let client = match client_result {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(error) => return std::result::Result::Err(crate::map_pool_error(error)),
|
||||
};
|
||||
let rows_result = client.query(sql, &[&signature, &sql_limit, &sql_offset]).await;
|
||||
let rows = match rows_result {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(_) => {
|
||||
return std::result::Result::Err(crate::PostgresBackendError::new(
|
||||
crate::PostgresBackendErrorKind::ReadFailed,
|
||||
"raw_transaction_observation_inspection_query",
|
||||
));
|
||||
},
|
||||
};
|
||||
if rows.is_empty() {
|
||||
return std::result::Result::Err(data_invalid("raw_transaction_observation_inspection_cardinality"));
|
||||
}
|
||||
let row_count = rows.len();
|
||||
let mut total_items = std::option::Option::None;
|
||||
let mut filtered_items = std::option::Option::None;
|
||||
let mut items = std::vec::Vec::new();
|
||||
for row in rows {
|
||||
let physical = match raw_observation_inspection_db_row(&row) {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(error) => return std::result::Result::Err(error),
|
||||
};
|
||||
let row_total = match decode_u64_decimal(physical.total_count_text.as_str(), "raw_transaction_observation_inspection_total") {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(error) => return std::result::Result::Err(error),
|
||||
};
|
||||
let row_filtered = match decode_u64_decimal(physical.filtered_count_text.as_str(), "raw_transaction_observation_inspection_filtered") {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(error) => return std::result::Result::Err(error),
|
||||
};
|
||||
match (total_items, filtered_items) {
|
||||
(std::option::Option::None, std::option::Option::None) => {
|
||||
total_items = std::option::Option::Some(row_total);
|
||||
filtered_items = std::option::Option::Some(row_filtered);
|
||||
},
|
||||
(std::option::Option::Some(total), std::option::Option::Some(filtered)) if total == row_total && filtered == row_filtered => {},
|
||||
_ => return std::result::Result::Err(data_invalid("raw_transaction_observation_inspection_counts")),
|
||||
}
|
||||
if physical.page_present {
|
||||
let observation = match decode_raw_observation_inspection_row(network, physical) {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(error) => return std::result::Result::Err(error),
|
||||
};
|
||||
if let std::option::Option::Some(reference) = query.transaction() {
|
||||
if observation.transaction() != reference {
|
||||
return std::result::Result::Err(data_invalid("raw_transaction_observation_inspection_reference"));
|
||||
}
|
||||
}
|
||||
items.push(ksp_store_api::RawTransactionObservationSummary::new(
|
||||
observation.observation_key(),
|
||||
observation.transaction().clone(),
|
||||
observation.provenance().clone(),
|
||||
));
|
||||
} else if row_count != 1 || !raw_observation_inspection_empty_page_is_clean(&physical) {
|
||||
return std::result::Result::Err(data_invalid("raw_transaction_observation_inspection_page"));
|
||||
}
|
||||
}
|
||||
let total_items = match total_items {
|
||||
std::option::Option::Some(value) => value,
|
||||
std::option::Option::None => return std::result::Result::Err(data_invalid("raw_transaction_observation_inspection_total")),
|
||||
};
|
||||
let filtered_items = match filtered_items {
|
||||
std::option::Option::Some(value) => value,
|
||||
std::option::Option::None => return std::result::Result::Err(data_invalid("raw_transaction_observation_inspection_filtered")),
|
||||
};
|
||||
let item_count = match u64::try_from(items.len()) {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(_) => return std::result::Result::Err(data_invalid("raw_transaction_observation_inspection_page")),
|
||||
};
|
||||
if item_count > query.page().limit().get() {
|
||||
return std::result::Result::Err(data_invalid("raw_transaction_observation_inspection_page"));
|
||||
}
|
||||
return match ksp_store_api::RawInspectionPage::try_new(items, total_items, filtered_items) {
|
||||
std::result::Result::Ok(value) => std::result::Result::Ok(value),
|
||||
std::result::Result::Err(_) => std::result::Result::Err(data_invalid("raw_transaction_observation_inspection_page")),
|
||||
};
|
||||
}
|
||||
|
||||
/// Reads one RAW transaction observation from the physical PostgreSQL backend.
|
||||
pub(crate) async fn get_raw_transaction_observation(
|
||||
pool: &deadpool_postgres::Pool,
|
||||
@@ -1401,6 +1533,116 @@ fn raw_transaction_db_row(row: &tokio_postgres::Row) -> std::result::Result<RawT
|
||||
});
|
||||
}
|
||||
|
||||
fn raw_observation_inspection_empty_page_is_clean(row: &RawObservationInspectionDbRow) -> bool {
|
||||
return row.acquisition_method.is_none()
|
||||
&& row.capture_session_id.is_none()
|
||||
&& row.commitment.is_none()
|
||||
&& row.endpoint_id.is_none()
|
||||
&& row.filter_id.is_none()
|
||||
&& row.observation_key.is_none()
|
||||
&& row.observed_at_unix_millis.is_none()
|
||||
&& row.origin.is_none()
|
||||
&& row.protocol.is_none()
|
||||
&& row.provider.is_none()
|
||||
&& row.received_at_unix_millis.is_none()
|
||||
&& row.source_payload_hash.is_none()
|
||||
&& row.source_payload_size_bytes.is_none()
|
||||
&& row.transaction_signature.is_none();
|
||||
}
|
||||
|
||||
fn raw_observation_inspection_db_row(row: &tokio_postgres::Row) -> std::result::Result<RawObservationInspectionDbRow, crate::PostgresBackendError> {
|
||||
macro_rules! get_optional {
|
||||
($name:literal, $ty:ty) => {
|
||||
match row.try_get::<_, std::option::Option<$ty>>($name) {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(_) => return std::result::Result::Err(data_invalid("raw_transaction_observation_inspection_decode")),
|
||||
}
|
||||
};
|
||||
}
|
||||
let filtered_count_text = match row.try_get::<_, std::string::String>("filtered_count_text") {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(_) => return std::result::Result::Err(data_invalid("raw_transaction_observation_inspection_decode")),
|
||||
};
|
||||
let page_present = match row.try_get::<_, bool>("page_present") {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(_) => return std::result::Result::Err(data_invalid("raw_transaction_observation_inspection_decode")),
|
||||
};
|
||||
let total_count_text = match row.try_get::<_, std::string::String>("total_count_text") {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(_) => return std::result::Result::Err(data_invalid("raw_transaction_observation_inspection_decode")),
|
||||
};
|
||||
return std::result::Result::Ok(RawObservationInspectionDbRow {
|
||||
acquisition_method: get_optional!("acquisition_method", std::string::String),
|
||||
capture_session_id: get_optional!("capture_session_id", std::string::String),
|
||||
commitment: get_optional!("commitment", std::string::String),
|
||||
endpoint_id: get_optional!("endpoint_id", std::string::String),
|
||||
filter_id: get_optional!("filter_id", std::string::String),
|
||||
filtered_count_text,
|
||||
observation_key: get_optional!("observation_key", std::vec::Vec<u8>),
|
||||
observed_at_unix_millis: get_optional!("observed_at_unix_millis", i64),
|
||||
origin: get_optional!("origin", std::string::String),
|
||||
page_present,
|
||||
protocol: get_optional!("protocol", std::string::String),
|
||||
provider: get_optional!("provider", std::string::String),
|
||||
received_at_unix_millis: get_optional!("received_at_unix_millis", i64),
|
||||
source_payload_hash: get_optional!("source_payload_hash", std::vec::Vec<u8>),
|
||||
source_payload_size_bytes: get_optional!("source_payload_size_bytes", i64),
|
||||
total_count_text,
|
||||
transaction_signature: get_optional!("transaction_signature", std::vec::Vec<u8>),
|
||||
});
|
||||
}
|
||||
|
||||
fn decode_raw_observation_inspection_row(
|
||||
network: &ksp_store_api::RawNetworkId,
|
||||
row: RawObservationInspectionDbRow,
|
||||
) -> std::result::Result<ksp_store_api::RawTransactionObservation, crate::PostgresBackendError> {
|
||||
let observation_key = match inspection_required(row.observation_key, "raw_transaction_observation_inspection_shape") {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(error) => return std::result::Result::Err(error),
|
||||
};
|
||||
let received_at_unix_millis = match inspection_required(row.received_at_unix_millis, "raw_transaction_observation_inspection_shape") {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(error) => return std::result::Result::Err(error),
|
||||
};
|
||||
let transaction_signature = match inspection_required(row.transaction_signature, "raw_transaction_observation_inspection_shape") {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(error) => return std::result::Result::Err(error),
|
||||
};
|
||||
let acquisition_method = match inspection_required(row.acquisition_method, "raw_transaction_observation_inspection_shape") {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(error) => return std::result::Result::Err(error),
|
||||
};
|
||||
let origin = match inspection_required(row.origin, "raw_transaction_observation_inspection_shape") {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(error) => return std::result::Result::Err(error),
|
||||
};
|
||||
let protocol = match inspection_required(row.protocol, "raw_transaction_observation_inspection_shape") {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(error) => return std::result::Result::Err(error),
|
||||
};
|
||||
let provider = match inspection_required(row.provider, "raw_transaction_observation_inspection_shape") {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(error) => return std::result::Result::Err(error),
|
||||
};
|
||||
let physical = RawObservationDbRow {
|
||||
acquisition_method,
|
||||
capture_session_id: row.capture_session_id,
|
||||
commitment: row.commitment,
|
||||
endpoint_id: row.endpoint_id,
|
||||
filter_id: row.filter_id,
|
||||
observation_key,
|
||||
observed_at_unix_millis: row.observed_at_unix_millis,
|
||||
origin,
|
||||
protocol,
|
||||
provider,
|
||||
received_at_unix_millis,
|
||||
source_payload_hash: row.source_payload_hash,
|
||||
source_payload_size_bytes: row.source_payload_size_bytes,
|
||||
transaction_signature,
|
||||
};
|
||||
return decode_raw_observation_row(network, physical);
|
||||
}
|
||||
|
||||
fn raw_observation_db_row(row: &tokio_postgres::Row) -> std::result::Result<RawObservationDbRow, crate::PostgresBackendError> {
|
||||
macro_rules! required {
|
||||
($name:literal, $ty:ty) => {
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
// file: crates/ksp-store-postgres-lib/src/runtime.rs
|
||||
// version: 17
|
||||
// version: 18
|
||||
|
||||
const APPLICATION_NAME: &str = "ksp-store";
|
||||
const MAX_CONNECTION_URI_BYTES: usize = 4_096;
|
||||
@@ -356,6 +356,14 @@ impl PostgresBackend {
|
||||
return crate::inspect_raw_account_states(&self.pool, &self.network, query).await;
|
||||
}
|
||||
|
||||
/// Inspects one safe random-access RAW account-observation window with exact logical counts.
|
||||
pub async fn inspect_raw_account_observations(
|
||||
&self,
|
||||
query: &ksp_store_api::RawAccountObservationInspectionQuery,
|
||||
) -> std::result::Result<ksp_store_api::RawInspectionPage<ksp_store_api::RawAccountObservationSummary>, crate::PostgresBackendError> {
|
||||
return crate::inspect_raw_account_observations(&self.pool, &self.network, query).await;
|
||||
}
|
||||
|
||||
/// Reads one persisted RAW account observation by producer-owned idempotence key.
|
||||
pub async fn get_raw_account_observation(
|
||||
&self,
|
||||
@@ -405,6 +413,14 @@ impl PostgresBackend {
|
||||
return crate::inspect_raw_transactions(&self.pool, &self.network, query).await;
|
||||
}
|
||||
|
||||
/// Inspects one safe random-access RAW transaction-observation window with exact logical counts.
|
||||
pub async fn inspect_raw_transaction_observations(
|
||||
&self,
|
||||
query: &ksp_store_api::RawTransactionObservationInspectionQuery,
|
||||
) -> std::result::Result<ksp_store_api::RawInspectionPage<ksp_store_api::RawTransactionObservationSummary>, crate::PostgresBackendError> {
|
||||
return crate::inspect_raw_transaction_observations(&self.pool, &self.network, query).await;
|
||||
}
|
||||
|
||||
/// Reads one persisted RAW transaction observation by producer-owned idempotence key.
|
||||
pub async fn get_raw_transaction_observation(
|
||||
&self,
|
||||
@@ -485,6 +501,18 @@ impl std::fmt::Debug for PostgresBackend {
|
||||
}
|
||||
}
|
||||
|
||||
impl ksp_store_api::RawAccountObservationInspectionRead for PostgresBackend {
|
||||
fn inspect_raw_account_observations<'a>(
|
||||
&'a self,
|
||||
query: &'a ksp_store_api::RawAccountObservationInspectionQuery,
|
||||
) -> ksp_store_api::StoreApiFuture<'a, ksp_store_api::Result<ksp_store_api::RawInspectionPage<ksp_store_api::RawAccountObservationSummary>>> {
|
||||
return std::boxed::Box::pin(async move {
|
||||
let result = PostgresBackend::inspect_raw_account_observations(self, query).await;
|
||||
return result.map_err(map_capability_error);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
impl ksp_store_api::RawAccountObservationRead for PostgresBackend {
|
||||
fn get_raw_account_observation<'a>(
|
||||
&'a self,
|
||||
@@ -604,6 +632,18 @@ impl ksp_store_api::RawTransactionWrite for PostgresBackend {
|
||||
}
|
||||
}
|
||||
|
||||
impl ksp_store_api::RawTransactionObservationInspectionRead for PostgresBackend {
|
||||
fn inspect_raw_transaction_observations<'a>(
|
||||
&'a self,
|
||||
query: &'a ksp_store_api::RawTransactionObservationInspectionQuery,
|
||||
) -> ksp_store_api::StoreApiFuture<'a, ksp_store_api::Result<ksp_store_api::RawInspectionPage<ksp_store_api::RawTransactionObservationSummary>>> {
|
||||
return std::boxed::Box::pin(async move {
|
||||
let result = PostgresBackend::inspect_raw_transaction_observations(self, query).await;
|
||||
return result.map_err(map_capability_error);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
impl ksp_store_api::RawTransactionObservationRead for PostgresBackend {
|
||||
fn get_raw_transaction_observation<'a>(
|
||||
&'a self,
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
// file: crates/ksp-store-postgres-lib/tests/dependency_boundary.rs
|
||||
// version: 28
|
||||
// version: 29
|
||||
|
||||
#![warn(missing_docs)]
|
||||
#![deny(unreachable_pub)]
|
||||
@@ -187,6 +187,7 @@ fn pre_005_raw_account_acquisition_is_atomic_idempotent_and_keeps_trait_impls_ou
|
||||
"impl ksp_store_api::RawAccountStateInspectionRead for PostgresBackend",
|
||||
"impl ksp_store_api::RawAccountStateRead for PostgresBackend",
|
||||
"impl ksp_store_api::RawAccountStateWrite for PostgresBackend",
|
||||
"impl ksp_store_api::RawAccountObservationInspectionRead for PostgresBackend",
|
||||
"impl ksp_store_api::RawAccountObservationRead for PostgresBackend",
|
||||
"impl ksp_store_api::RawAccountObservationWrite for PostgresBackend",
|
||||
] {
|
||||
@@ -222,6 +223,7 @@ fn pre_006_raw_account_additional_observation_is_reference_guarded_cancellation_
|
||||
"impl ksp_store_api::RawAccountStateInspectionRead for PostgresBackend",
|
||||
"impl ksp_store_api::RawAccountStateRead for PostgresBackend",
|
||||
"impl ksp_store_api::RawAccountStateWrite for PostgresBackend",
|
||||
"impl ksp_store_api::RawAccountObservationInspectionRead for PostgresBackend",
|
||||
"impl ksp_store_api::RawAccountObservationRead for PostgresBackend",
|
||||
"impl ksp_store_api::RawAccountObservationWrite for PostgresBackend",
|
||||
] {
|
||||
@@ -408,6 +410,7 @@ fn pre_007_raw_account_pagination_is_keyset_cursor_bound_and_policy_free() {
|
||||
"impl ksp_store_api::RawAccountStateInspectionRead for PostgresBackend",
|
||||
"impl ksp_store_api::RawAccountStateRead for PostgresBackend",
|
||||
"impl ksp_store_api::RawAccountStateWrite for PostgresBackend",
|
||||
"impl ksp_store_api::RawAccountObservationInspectionRead for PostgresBackend",
|
||||
"impl ksp_store_api::RawAccountObservationRead for PostgresBackend",
|
||||
"impl ksp_store_api::RawAccountObservationWrite for PostgresBackend",
|
||||
] {
|
||||
@@ -449,15 +452,17 @@ fn pre_007_raw_retention_is_atomic_compare_and_transition_without_fake_compactio
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn v0_3_8_pre_005_backend_trait_implementations_cover_exact_twelve_raw_capabilities_in_runtime_bridge() {
|
||||
fn v0_3_8_pre_009_backend_trait_implementations_cover_exact_fourteen_raw_capabilities_in_runtime_bridge() {
|
||||
let runtime = include_str!("../src/runtime.rs");
|
||||
for implementation in [
|
||||
"impl ksp_store_api::RawAccountObservationInspectionRead for PostgresBackend",
|
||||
"impl ksp_store_api::RawAccountObservationRead for PostgresBackend",
|
||||
"impl ksp_store_api::RawAccountObservationWrite for PostgresBackend",
|
||||
"impl ksp_store_api::RawAccountStateInspectionRead for PostgresBackend",
|
||||
"impl ksp_store_api::RawAccountStateRead for PostgresBackend",
|
||||
"impl ksp_store_api::RawAccountStateWrite for PostgresBackend",
|
||||
"impl ksp_store_api::RawTransactionInspectionRead for PostgresBackend",
|
||||
"impl ksp_store_api::RawTransactionObservationInspectionRead for PostgresBackend",
|
||||
"impl ksp_store_api::RawTransactionObservationRead for PostgresBackend",
|
||||
"impl ksp_store_api::RawTransactionObservationWrite for PostgresBackend",
|
||||
"impl ksp_store_api::RawTransactionRead for PostgresBackend",
|
||||
@@ -467,6 +472,6 @@ fn v0_3_8_pre_005_backend_trait_implementations_cover_exact_twelve_raw_capabilit
|
||||
] {
|
||||
assert_eq!(runtime.matches(implementation).count(), 1, "unexpected PostgreSQL RAW capability inventory: {implementation}");
|
||||
}
|
||||
assert_eq!(runtime.matches("impl ksp_store_api::Raw").count(), 12);
|
||||
assert_eq!(runtime.matches("impl ksp_store_api::Raw").count(), 14);
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
// file: crates/ksp-store-postgres-lib/tests/hardening_completeness.rs
|
||||
// version: 21
|
||||
// version: 22
|
||||
|
||||
#![warn(missing_docs)]
|
||||
#![deny(unreachable_pub)]
|
||||
@@ -302,15 +302,17 @@ fn pre_009_live_raw_transaction_proof_is_opt_in_isolated_and_secret_safe() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn v0_3_8_pre_005_raw_capability_implementation_inventory_is_exactly_twelve() {
|
||||
fn v0_3_8_pre_009_raw_capability_implementation_inventory_is_exactly_fourteen() {
|
||||
let runtime = include_str!("../src/runtime.rs");
|
||||
let capability_impls = [
|
||||
"impl ksp_store_api::RawAccountObservationInspectionRead for PostgresBackend",
|
||||
"impl ksp_store_api::RawAccountObservationRead for PostgresBackend",
|
||||
"impl ksp_store_api::RawAccountObservationWrite for PostgresBackend",
|
||||
"impl ksp_store_api::RawAccountStateInspectionRead for PostgresBackend",
|
||||
"impl ksp_store_api::RawAccountStateRead for PostgresBackend",
|
||||
"impl ksp_store_api::RawAccountStateWrite for PostgresBackend",
|
||||
"impl ksp_store_api::RawTransactionInspectionRead for PostgresBackend",
|
||||
"impl ksp_store_api::RawTransactionObservationInspectionRead for PostgresBackend",
|
||||
"impl ksp_store_api::RawTransactionObservationRead for PostgresBackend",
|
||||
"impl ksp_store_api::RawTransactionObservationWrite for PostgresBackend",
|
||||
"impl ksp_store_api::RawTransactionRead for PostgresBackend",
|
||||
@@ -321,7 +323,7 @@ fn v0_3_8_pre_005_raw_capability_implementation_inventory_is_exactly_twelve() {
|
||||
for implementation in capability_impls {
|
||||
assert_eq!(runtime.matches(implementation).count(), 1, "unexpected PostgreSQL capability implementation inventory: {implementation}");
|
||||
}
|
||||
assert_eq!(runtime.matches("impl ksp_store_api::Raw").count(), 12);
|
||||
assert_eq!(runtime.matches("impl ksp_store_api::Raw").count(), 14);
|
||||
let migration = include_str!("../src/migration.rs");
|
||||
assert!(migration.contains("raw_account_state"));
|
||||
assert!(migration.contains("crate::V002_RESOURCES"));
|
||||
@@ -394,13 +396,77 @@ fn v0_3_8_pre_004_transaction_inspection_sql_is_single_statement_payload_free_co
|
||||
}
|
||||
assert!(ascending.contains("ORDER BY transaction_row.slot ASC, transaction_row.signature ASC"));
|
||||
assert!(descending.contains("ORDER BY transaction_row.slot DESC, transaction_row.signature DESC"));
|
||||
assert_eq!(source.matches(" OFFSET ").count(), 2);
|
||||
assert!(source.contains("raw_transaction_inspection_sql_window(query.page())"));
|
||||
assert!(source.contains("raw_transaction_inspection_offset"));
|
||||
assert!(source.contains("raw_transaction_inspection_limit"));
|
||||
return;
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn v0_3_8_pre_009_observation_inspection_sql_is_counted_random_access_safe_and_family_local() {
|
||||
let transaction = include_str!("../src/raw_transaction.rs");
|
||||
let account = include_str!("../src/raw_account.rs");
|
||||
let transaction_ascending = transaction.lines().find(|line| return line.starts_with("const INSPECT_OBSERVATIONS_ASC_SQL"));
|
||||
let transaction_ascending = match transaction_ascending {
|
||||
std::option::Option::Some(value) => value,
|
||||
std::option::Option::None => panic!("missing ascending transaction-observation inspection SQL"),
|
||||
};
|
||||
let transaction_descending = transaction.lines().find(|line| return line.starts_with("const INSPECT_OBSERVATIONS_DESC_SQL"));
|
||||
let transaction_descending = match transaction_descending {
|
||||
std::option::Option::Some(value) => value,
|
||||
std::option::Option::None => panic!("missing descending transaction-observation inspection SQL"),
|
||||
};
|
||||
for statement in [transaction_ascending, transaction_descending] {
|
||||
for required in [
|
||||
"COUNT(*)::TEXT AS filtered_count_text",
|
||||
"COUNT(*)::TEXT FROM ksp_raw_transaction_observations",
|
||||
"LEFT JOIN LATERAL",
|
||||
"transaction_signature = $1",
|
||||
"LIMIT $2 OFFSET $3",
|
||||
"received_at_unix_millis",
|
||||
"observation_key",
|
||||
] {
|
||||
assert!(statement.contains(required), "missing transaction-observation inspection SQL contract: {required}");
|
||||
}
|
||||
for forbidden in ["ksp_raw_transactions AS", "payload", "archive_payload", "SELECT *"] {
|
||||
assert!(!statement.contains(forbidden), "transaction-observation inspection leaked unrelated/raw material: {forbidden}");
|
||||
}
|
||||
}
|
||||
assert!(transaction_ascending.contains("ORDER BY received_at_unix_millis ASC, observation_key ASC"));
|
||||
assert!(transaction_descending.contains("ORDER BY received_at_unix_millis DESC, observation_key DESC"));
|
||||
let account_ascending = account.lines().find(|line| return line.starts_with("const INSPECT_ACCOUNT_OBSERVATIONS_ASC_SQL"));
|
||||
let account_ascending = match account_ascending {
|
||||
std::option::Option::Some(value) => value,
|
||||
std::option::Option::None => panic!("missing ascending account-observation inspection SQL"),
|
||||
};
|
||||
let account_descending = account.lines().find(|line| return line.starts_with("const INSPECT_ACCOUNT_OBSERVATIONS_DESC_SQL"));
|
||||
let account_descending = match account_descending {
|
||||
std::option::Option::Some(value) => value,
|
||||
std::option::Option::None => panic!("missing descending account-observation inspection SQL"),
|
||||
};
|
||||
for statement in [account_ascending, account_descending] {
|
||||
for required in [
|
||||
"COUNT(*)::TEXT AS filtered_count_text",
|
||||
"COUNT(*)::TEXT FROM ksp_raw_account_observations",
|
||||
"LEFT JOIN LATERAL",
|
||||
"account_pubkey = $1",
|
||||
"account_slot = $2::TEXT::NUMERIC",
|
||||
"account_state_hash = $3",
|
||||
"LIMIT $4 OFFSET $5",
|
||||
"received_at_unix_millis",
|
||||
"observation_key",
|
||||
] {
|
||||
assert!(statement.contains(required), "missing account-observation inspection SQL contract: {required}");
|
||||
}
|
||||
for forbidden in ["ksp_raw_account_states AS", "account_row.data", "SELECT *"] {
|
||||
assert!(!statement.contains(forbidden), "account-observation inspection leaked unrelated/raw material: {forbidden}");
|
||||
}
|
||||
}
|
||||
assert!(account_ascending.contains("ORDER BY received_at_unix_millis ASC, observation_key ASC"));
|
||||
assert!(account_descending.contains("ORDER BY received_at_unix_millis DESC, observation_key DESC"));
|
||||
return;
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn pre_010_raw_account_private_sql_is_non_destructive_keyset_and_family_local() {
|
||||
let source = include_str!("../src/raw_account.rs");
|
||||
@@ -493,6 +559,5 @@ fn v0_3_8_pre_005_raw_account_keyset_sql_remains_offset_free_while_inspection_is
|
||||
assert!(statement.contains("account_row.pubkey"));
|
||||
assert!(statement.contains("account_row.state_hash"));
|
||||
}
|
||||
assert_eq!(source.matches(" OFFSET ").count(), 2);
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
// file: crates/ksp-store-postgres-lib/tests/public_api.rs
|
||||
// version: 16
|
||||
// version: 17
|
||||
|
||||
#![warn(missing_docs)]
|
||||
#![deny(unreachable_pub)]
|
||||
@@ -120,8 +120,10 @@ fn pre_005_raw_write_bridge_uses_only_backend_independent_models_and_outcomes()
|
||||
fn v0_3_8_pre_005_raw_list_and_inspection_bridges_use_only_backend_independent_models() {
|
||||
let _account_list = ksp_store_postgres_lib::PostgresBackend::list_raw_account_states;
|
||||
let _account_inspection = ksp_store_postgres_lib::PostgresBackend::inspect_raw_account_states;
|
||||
let _account_observation_inspection = ksp_store_postgres_lib::PostgresBackend::inspect_raw_account_observations;
|
||||
let _transaction_list = ksp_store_postgres_lib::PostgresBackend::list_raw_transactions;
|
||||
let _transaction_inspection = ksp_store_postgres_lib::PostgresBackend::inspect_raw_transactions;
|
||||
let _transaction_observation_inspection = ksp_store_postgres_lib::PostgresBackend::inspect_raw_transaction_observations;
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -133,12 +135,14 @@ fn pre_007_raw_retention_write_bridge_uses_backend_independent_transition_and_ou
|
||||
|
||||
fn assert_raw_capabilities<T>()
|
||||
where
|
||||
T: ksp_store_api::RawAccountObservationRead
|
||||
T: ksp_store_api::RawAccountObservationInspectionRead
|
||||
+ ksp_store_api::RawAccountObservationRead
|
||||
+ ksp_store_api::RawAccountObservationWrite
|
||||
+ ksp_store_api::RawAccountStateInspectionRead
|
||||
+ ksp_store_api::RawAccountStateRead
|
||||
+ ksp_store_api::RawAccountStateWrite
|
||||
+ ksp_store_api::RawTransactionInspectionRead
|
||||
+ ksp_store_api::RawTransactionObservationInspectionRead
|
||||
+ ksp_store_api::RawTransactionObservationRead
|
||||
+ ksp_store_api::RawTransactionObservationWrite
|
||||
+ ksp_store_api::RawTransactionRead
|
||||
@@ -151,7 +155,7 @@ where
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn v0_3_8_pre_005_postgres_backend_implements_both_inspection_capabilities_as_12_of_12() {
|
||||
fn v0_3_8_pre_009_postgres_backend_implements_entity_and_observation_inspection_capabilities_as_14_of_14() {
|
||||
assert_raw_capabilities::<ksp_store_postgres_lib::PostgresBackend>();
|
||||
return;
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user