v0.3.3-pre.006

This commit is contained in:
2026-08-30 13:06:33 +02:00
parent 66a3926adb
commit 1352a61d70
15 changed files with 891 additions and 36 deletions

View File

@@ -1,5 +1,5 @@
// file: crates/ksp-store-postgres-lib/src/error.rs
// version: 6
// version: 7
/// Stable KSP error code reserved for PostgreSQL retention transitions that require unsupported physical compaction.
pub const ERROR_CODE_POSTGRES_RETENTION_COMPACTION_UNSUPPORTED: ksp_store_api::ErrorCode =
@@ -23,8 +23,12 @@ pub enum PostgresBackendErrorKind {
DataInvalid,
/// PostgreSQL migration/bootstrap execution failed without exposing server text or SQL.
MigrationFailed,
/// The requested RAW page size cannot be represented by PostgreSQL LIMIT plus the continuation probe row.
PageLimitUnsupported,
/// Applied PostgreSQL migration history diverges from the embedded immutable KSP history.
MigrationMismatch,
/// A bounded backend-private RAW query or opaque cursor is invalid for the requested navigation context.
QueryInvalid,
/// A PostgreSQL RAW read statement failed without exposing server text, SQL or bind values.
ReadFailed,
/// A RAW write requires an existing canonical reference that is not durable.

View File

@@ -1,5 +1,5 @@
// file: crates/ksp-store-postgres-lib/src/lib.rs
// version: 10
// version: 11
#![warn(missing_docs)]
#![deny(unreachable_pub)]
@@ -15,7 +15,8 @@
//! backend-private RAW transaction/observation/retention read mapping.
//! `0.3.3-pre.005` adds atomic canonical/observation writes, real idempotence
//! checks and safe conflict classification without exposing PostgreSQL rows or
//! SQL through the public bridge.
//! SQL through the public bridge. `0.3.3-pre.006` adds deterministic keyset
//! pagination with a fixed opaque cursor bound to network, range and direction.
//!
//! This crate depends on `ksp-store-api` and never on `ksp-store-lib`. The
//! common facade consumes only this crate's narrow backend bridge and never
@@ -54,6 +55,14 @@ pub(crate) use self::health::probe_health;
pub(crate) use self::migration::bootstrap;
/// Current embedded migration version consumed by the private health probe.
pub(crate) use self::migration::current_migration_version;
/// Private decoded RAW cursor representation consumed by tests and the physical RAW module.
pub(crate) use self::raw_transaction::cursor::RawTransactionDecodedCursor;
/// Private RAW transaction cursor decoder consumed by the physical RAW module.
pub(crate) use self::raw_transaction::cursor::decode_raw_transaction_cursor;
/// Private RAW transaction cursor encoder consumed by the physical RAW module.
pub(crate) use self::raw_transaction::cursor::encode_raw_transaction_cursor;
/// Private physical page-limit converter consumed by the physical RAW module.
pub(crate) use self::raw_transaction::cursor::raw_transaction_physical_page_limit;
/// Private RAW transaction reader consumed by the physical backend runtime.
pub(crate) use self::raw_transaction::get_raw_transaction;
/// Private RAW transaction observation reader consumed by the physical backend runtime.
@@ -62,6 +71,8 @@ pub(crate) use self::raw_transaction::get_raw_transaction_observation;
pub(crate) use self::raw_transaction::get_raw_transaction_retention_state;
/// Private RAW transaction tombstone reader consumed by the physical backend runtime.
pub(crate) use self::raw_transaction::get_raw_transaction_tombstone;
/// Private RAW transaction list reader consumed by the physical backend runtime.
pub(crate) use self::raw_transaction::list_raw_transactions;
/// Private atomic RAW transaction acquisition writer consumed by the physical backend runtime.
pub(crate) use self::raw_transaction::persist_raw_transaction_acquisition;
/// Private additional RAW transaction observation writer consumed by the physical backend runtime.

View File

@@ -1,5 +1,7 @@
// file: crates/ksp-store-postgres-lib/src/raw_transaction.rs
// version: 2
// version: 3
pub(crate) mod cursor;
const GET_ARCHIVE_PAYLOAD_SQL: &str = "SELECT payload FROM ksp_raw_transaction_archive_payloads WHERE signature = $1";
const GET_OBSERVATION_SQL: &str = "SELECT observation_key, transaction_signature, provider, protocol, acquisition_method, origin, received_at_unix_millis, capture_session_id, commitment, endpoint_id, filter_id, observed_at_unix_millis, source_payload_hash, source_payload_size_bytes FROM ksp_raw_transaction_observations WHERE observation_key = $1";
@@ -8,12 +10,19 @@ const GET_TOMBSTONE_SQL: &str = "SELECT signature, slot::text AS slot_text, bloc
const GET_TRANSACTION_SQL: &str = "SELECT transaction_row.signature, transaction_row.slot::text AS slot_text, transaction_row.block_time_unix_millis, transaction_row.format_id, transaction_row.format_version, transaction_row.content_hash, transaction_row.payload, transaction_row.retention_state, archive_row.payload AS archive_payload FROM ksp_raw_transactions AS transaction_row LEFT JOIN ksp_raw_transaction_archive_payloads AS archive_row ON archive_row.signature = transaction_row.signature WHERE transaction_row.signature = $1";
const INSERT_OBSERVATION_SQL: &str = "INSERT INTO ksp_raw_transaction_observations (observation_key, transaction_signature, provider, protocol, acquisition_method, origin, received_at_unix_millis, capture_session_id, commitment, endpoint_id, filter_id, observed_at_unix_millis, source_payload_hash, source_payload_size_bytes) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14) ON CONFLICT (observation_key) DO NOTHING RETURNING observation_key";
const INSERT_TRANSACTION_SQL: &str = "INSERT INTO ksp_raw_transactions (signature, slot, block_time_unix_millis, format_id, format_version, content_hash, payload, retention_state) VALUES ($1, $2::TEXT::NUMERIC, $3, $4, $5, $6, $7, 'full') ON CONFLICT (signature) DO NOTHING RETURNING signature";
const LIST_TRANSACTIONS_ASC_SQL: &str = "SELECT signature, slot::text AS slot_text FROM ksp_raw_transactions WHERE retention_state <> 'purged' AND ($1::TEXT IS NULL OR slot >= $1::TEXT::NUMERIC) AND ($2::TEXT IS NULL OR slot <= $2::TEXT::NUMERIC) AND ($3::TEXT IS NULL OR (slot, signature) > ($3::TEXT::NUMERIC, $4::BYTEA)) ORDER BY slot ASC, signature ASC LIMIT $5";
const LIST_TRANSACTIONS_DESC_SQL: &str = "SELECT signature, slot::text AS slot_text FROM ksp_raw_transactions WHERE retention_state <> 'purged' AND ($1::TEXT IS NULL OR slot >= $1::TEXT::NUMERIC) AND ($2::TEXT IS NULL OR slot <= $2::TEXT::NUMERIC) AND ($3::TEXT IS NULL OR (slot, signature) < ($3::TEXT::NUMERIC, $4::BYTEA)) ORDER BY slot DESC, signature DESC LIMIT $5";
const LOCK_OBSERVATION_SQL: &str = "SELECT observation_key, transaction_signature, provider, protocol, acquisition_method, origin, received_at_unix_millis, capture_session_id, commitment, endpoint_id, filter_id, observed_at_unix_millis, source_payload_hash, source_payload_size_bytes FROM ksp_raw_transaction_observations WHERE observation_key = $1 FOR UPDATE";
const LOCK_TRANSACTION_SQL: &str = "SELECT signature, slot::text AS slot_text, block_time_unix_millis, format_id, format_version, content_hash, payload, retention_state, NULL::BYTEA AS archive_payload FROM ksp_raw_transactions WHERE signature = $1 FOR UPDATE";
const LOCK_TRANSACTION_STATE_SQL: &str = "SELECT retention_state FROM ksp_raw_transactions WHERE signature = $1 FOR UPDATE";
const REHYDRATE_TRANSACTION_SQL: &str =
"UPDATE ksp_raw_transactions SET block_time_unix_millis = $2, payload = $3, retention_state = 'full' WHERE signature = $1";
struct RawListDbRow {
signature: std::vec::Vec<u8>,
slot_text: std::string::String,
}
struct RawObservationDbRow {
acquisition_method: std::string::String,
capture_session_id: std::option::Option<std::string::String>,
@@ -94,6 +103,82 @@ pub(crate) async fn get_raw_transaction(
return decode_raw_transaction_row(network, physical);
}
/// Lists deterministic canonical RAW transaction references using PostgreSQL keyset pagination.
pub(crate) async fn list_raw_transactions(
pool: &deadpool_postgres::Pool,
network: &ksp_store_api::RawNetworkId,
query: &ksp_store_api::RawTransactionQuery,
) -> std::result::Result<ksp_store_api::RawPage<ksp_store_api::RawTransactionReference>, crate::PostgresBackendError> {
if query.network() != network {
return std::result::Result::Err(crate::PostgresBackendError::new(crate::PostgresBackendErrorKind::WrongNetwork, "raw_transaction_list_network"));
}
let (requested_usize, sql_limit) = match crate::raw_transaction_physical_page_limit(query.page().limit().get()) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let decoded_cursor = match query.page().cursor() {
std::option::Option::Some(value) => match crate::decode_raw_transaction_cursor(query, value) {
std::result::Result::Ok(decoded) => std::option::Option::Some(decoded),
std::result::Result::Err(error) => return std::result::Result::Err(error),
},
std::option::Option::None => std::option::Option::None,
};
let client_result = pool.get().await;
let client = match client_result {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(crate::map_pool_error(error)),
};
let slots = query.slots();
let start_text = slots.start_inclusive().map(|value| return value.to_string());
let end_text = slots.end_inclusive().map(|value| return value.to_string());
let cursor_slot_text = decoded_cursor.as_ref().map(|value| return value.last_slot.to_string());
let cursor_signature = decoded_cursor.as_ref().map(|value| return value.last_signature.to_vec());
let sql = match query.direction() {
ksp_store_api::RawSortDirection::Ascending => LIST_TRANSACTIONS_ASC_SQL,
ksp_store_api::RawSortDirection::Descending => LIST_TRANSACTIONS_DESC_SQL,
_ => {
return std::result::Result::Err(crate::PostgresBackendError::new(crate::PostgresBackendErrorKind::QueryInvalid, "raw_transaction_list_direction"));
},
};
let rows_result = client.query(sql, &[&start_text, &end_text, &cursor_slot_text, &cursor_signature, &sql_limit]).await;
let rows = match rows_result {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => {
return std::result::Result::Err(crate::PostgresBackendError::new(crate::PostgresBackendErrorKind::ReadFailed, "raw_transaction_list_query"));
},
};
let mut decoded = std::vec::Vec::with_capacity(rows.len());
for row in rows {
let physical = match raw_list_db_row(&row) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let item = match decode_raw_list_row(network, physical) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
decoded.push(item);
}
let has_more = decoded.len() > requested_usize;
if has_more {
decoded.truncate(requested_usize);
}
let next_cursor = if has_more {
let last = match decoded.last() {
std::option::Option::Some(value) => value,
std::option::Option::None => return std::result::Result::Err(data_invalid("raw_transaction_list_page")),
};
match crate::encode_raw_transaction_cursor(query, last.0, &last.1.signature()) {
std::result::Result::Ok(value) => std::option::Option::Some(value),
std::result::Result::Err(error) => return std::result::Result::Err(error),
}
} else {
std::option::Option::None
};
let items = decoded.into_iter().map(|value| return value.1).collect();
return std::result::Result::Ok(ksp_store_api::RawPage::new(items, next_cursor));
}
/// Reads one RAW transaction observation from the physical PostgreSQL backend.
pub(crate) async fn get_raw_transaction_observation(
pool: &deadpool_postgres::Pool,
@@ -655,6 +740,18 @@ fn write_failed(phase: &'static str) -> crate::PostgresBackendError {
return crate::PostgresBackendError::new(crate::PostgresBackendErrorKind::WriteFailed, phase);
}
fn raw_list_db_row(row: &tokio_postgres::Row) -> std::result::Result<RawListDbRow, crate::PostgresBackendError> {
let signature = match row.try_get::<_, std::vec::Vec<u8>>("signature") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return std::result::Result::Err(data_invalid("raw_transaction_list_decode")),
};
let slot_text = match row.try_get::<_, std::string::String>("slot_text") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return std::result::Result::Err(data_invalid("raw_transaction_list_decode")),
};
return std::result::Result::Ok(RawListDbRow { signature, slot_text });
}
fn raw_transaction_db_row(row: &tokio_postgres::Row) -> std::result::Result<RawTransactionDbRow, crate::PostgresBackendError> {
let signature = match row.try_get::<_, std::vec::Vec<u8>>("signature") {
std::result::Result::Ok(value) => value,
@@ -772,6 +869,22 @@ fn raw_tombstone_db_row(row: &tokio_postgres::Row) -> std::result::Result<RawTom
});
}
fn decode_raw_list_row(
network: &ksp_store_api::RawNetworkId,
row: RawListDbRow,
) -> std::result::Result<(u64, ksp_store_api::RawTransactionReference), crate::PostgresBackendError> {
let signature = match fixed_bytes::<64>(row.signature) {
std::result::Result::Ok(value) => ksp_store_api::RawTransactionSignature::new(value),
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let slot = match decode_u64_decimal(row.slot_text.as_str(), "raw_transaction_list_slot") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let reference = ksp_store_api::RawTransactionReference::new(network.clone(), signature);
return std::result::Result::Ok((slot, reference));
}
fn decode_raw_transaction_row(
network: &ksp_store_api::RawNetworkId,
row: RawTransactionDbRow,

View File

@@ -0,0 +1,184 @@
// file: crates/ksp-store-postgres-lib/src/raw_transaction/cursor.rs
// version: 1
use sha2::Digest; // rust-rules: trait-import
const CURSOR_BYTES: usize = 109;
const CURSOR_DIGEST_OFFSET: usize = 77;
const CURSOR_DOMAIN: &[u8] = b"KSP/raw-transaction-cursor/v1";
const CURSOR_MAGIC: &[u8; 4] = b"KSPT";
const CURSOR_VERSION: u8 = 1;
const MAX_POSTGRES_PAGE_LIMIT: u64 = 9_223_372_036_854_775_806;
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
/// Decoded backend-private continuation key extracted from one validated RAW cursor.
pub(crate) struct RawTransactionDecodedCursor {
/// Last canonical transaction signature returned by the previous page.
pub(crate) last_signature: [u8; 64],
/// Last canonical transaction slot returned by the previous page.
pub(crate) last_slot: u64,
}
/// Decodes and validates one backend-private RAW transaction cursor against its query binding.
pub(crate) fn decode_raw_transaction_cursor(
query: &ksp_store_api::RawTransactionQuery,
cursor: &ksp_store_api::RawPageCursor,
) -> std::result::Result<RawTransactionDecodedCursor, crate::PostgresBackendError> {
let bytes = cursor.as_bytes();
if bytes.len() != CURSOR_BYTES {
return std::result::Result::Err(query_invalid("raw_cursor_size"));
}
if bytes.get(0..4) != std::option::Option::Some(CURSOR_MAGIC.as_ref()) {
return std::result::Result::Err(query_invalid("raw_cursor_magic"));
}
if bytes.get(4).copied() != std::option::Option::Some(CURSOR_VERSION) {
return std::result::Result::Err(query_invalid("raw_cursor_version"));
}
let slot_bytes = match bytes.get(5..13) {
std::option::Option::Some(value) => match <[u8; 8]>::try_from(value) {
std::result::Result::Ok(decoded) => decoded,
std::result::Result::Err(_) => return std::result::Result::Err(query_invalid("raw_cursor_slot")),
},
std::option::Option::None => return std::result::Result::Err(query_invalid("raw_cursor_slot")),
};
let last_slot = u64::from_be_bytes(slot_bytes);
let last_signature = match bytes.get(13..CURSOR_DIGEST_OFFSET) {
std::option::Option::Some(value) => match <[u8; 64]>::try_from(value) {
std::result::Result::Ok(decoded) => decoded,
std::result::Result::Err(_) => return std::result::Result::Err(query_invalid("raw_cursor_signature")),
},
std::option::Option::None => return std::result::Result::Err(query_invalid("raw_cursor_signature")),
};
let range_result = validate_slot_in_range(query.slots(), last_slot);
if let std::result::Result::Err(error) = range_result {
return std::result::Result::Err(error);
}
let expected_digest = match binding_digest(query, last_slot, &last_signature) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let stored_digest = match bytes.get(CURSOR_DIGEST_OFFSET..CURSOR_BYTES) {
std::option::Option::Some(value) => match <[u8; 32]>::try_from(value) {
std::result::Result::Ok(decoded) => decoded,
std::result::Result::Err(_) => return std::result::Result::Err(query_invalid("raw_cursor_digest")),
},
std::option::Option::None => return std::result::Result::Err(query_invalid("raw_cursor_digest")),
};
if stored_digest != expected_digest {
return std::result::Result::Err(query_invalid("raw_cursor_binding"));
}
return std::result::Result::Ok(RawTransactionDecodedCursor { last_signature, last_slot });
}
/// Encodes one backend-private RAW transaction cursor bound to the supplied query context.
pub(crate) fn encode_raw_transaction_cursor(
query: &ksp_store_api::RawTransactionQuery,
last_slot: u64,
last_signature: &ksp_store_api::RawTransactionSignature,
) -> std::result::Result<ksp_store_api::RawPageCursor, crate::PostgresBackendError> {
let range_result = validate_slot_in_range(query.slots(), last_slot);
if let std::result::Result::Err(error) = range_result {
return std::result::Result::Err(error);
}
let digest = match binding_digest(query, last_slot, last_signature.as_bytes()) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let mut bytes = std::vec::Vec::with_capacity(CURSOR_BYTES);
bytes.extend_from_slice(CURSOR_MAGIC);
bytes.push(CURSOR_VERSION);
bytes.extend_from_slice(&last_slot.to_be_bytes());
bytes.extend_from_slice(last_signature.as_bytes());
bytes.extend_from_slice(&digest);
if bytes.len() != CURSOR_BYTES {
return std::result::Result::Err(query_invalid("raw_cursor_encode_size"));
}
return match ksp_store_api::RawPageCursor::try_new(bytes.into_boxed_slice()) {
std::result::Result::Ok(value) => std::result::Result::Ok(value),
std::result::Result::Err(_) => std::result::Result::Err(query_invalid("raw_cursor_encode")),
};
}
/// Converts one caller page size into the exact PostgreSQL LIMIT+1 representation.
pub(crate) fn raw_transaction_physical_page_limit(requested: u64) -> std::result::Result<(usize, i64), crate::PostgresBackendError> {
if requested > MAX_POSTGRES_PAGE_LIMIT {
return std::result::Result::Err(crate::PostgresBackendError::new(crate::PostgresBackendErrorKind::PageLimitUnsupported, "raw_page_limit"));
}
let requested_usize = match usize::try_from(requested) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => {
return std::result::Result::Err(crate::PostgresBackendError::new(crate::PostgresBackendErrorKind::PageLimitUnsupported, "raw_page_limit"));
},
};
let requested_plus_one = match requested.checked_add(1) {
std::option::Option::Some(value) => value,
std::option::Option::None => {
return std::result::Result::Err(crate::PostgresBackendError::new(crate::PostgresBackendErrorKind::PageLimitUnsupported, "raw_page_limit"));
},
};
let sql_limit = match i64::try_from(requested_plus_one) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => {
return std::result::Result::Err(crate::PostgresBackendError::new(crate::PostgresBackendErrorKind::PageLimitUnsupported, "raw_page_limit"));
},
};
return std::result::Result::Ok((requested_usize, sql_limit));
}
fn binding_digest(
query: &ksp_store_api::RawTransactionQuery,
last_slot: u64,
last_signature: &[u8; 64],
) -> std::result::Result<[u8; 32], crate::PostgresBackendError> {
let direction = match query.direction() {
ksp_store_api::RawSortDirection::Ascending => 0_u8,
ksp_store_api::RawSortDirection::Descending => 1_u8,
_ => return std::result::Result::Err(query_invalid("raw_cursor_direction")),
};
let network_len = match u64::try_from(query.network().as_str().len()) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return std::result::Result::Err(query_invalid("raw_cursor_network")),
};
let mut hasher = sha2::Sha256::new();
hasher.update(CURSOR_DOMAIN);
hasher.update(network_len.to_be_bytes());
hasher.update(query.network().as_str().as_bytes());
hasher.update([direction]);
hash_optional_slot(&mut hasher, query.slots().start_inclusive());
hash_optional_slot(&mut hasher, query.slots().end_inclusive());
hasher.update(last_slot.to_be_bytes());
hasher.update(last_signature);
let finalized = hasher.finalize();
let mut digest = [0_u8; 32];
digest.copy_from_slice(finalized.as_ref());
return std::result::Result::Ok(digest);
}
fn hash_optional_slot(hasher: &mut sha2::Sha256, value: std::option::Option<u64>) {
match value {
std::option::Option::Some(slot) => {
hasher.update([1_u8]);
hasher.update(slot.to_be_bytes());
},
std::option::Option::None => hasher.update([0_u8]),
}
return;
}
fn validate_slot_in_range(range: ksp_store_api::RawSlotRange, slot: u64) -> std::result::Result<(), crate::PostgresBackendError> {
if let std::option::Option::Some(start) = range.start_inclusive()
&& slot < start
{
return std::result::Result::Err(query_invalid("raw_cursor_slot_range"));
}
if let std::option::Option::Some(end) = range.end_inclusive()
&& slot > end
{
return std::result::Result::Err(query_invalid("raw_cursor_slot_range"));
}
return std::result::Result::Ok(());
}
fn query_invalid(phase: &'static str) -> crate::PostgresBackendError {
return crate::PostgresBackendError::new(crate::PostgresBackendErrorKind::QueryInvalid, phase);
}

View File

@@ -1,5 +1,5 @@
// file: crates/ksp-store-postgres-lib/src/runtime.rs
// version: 7
// version: 8
const APPLICATION_NAME: &str = "ksp-store";
const MAX_CONNECTION_URI_BYTES: usize = 4_096;
@@ -340,6 +340,14 @@ impl PostgresBackend {
return crate::get_raw_transaction(&self.pool, &self.network, reference).await;
}
/// Lists deterministic canonical RAW transaction references with a backend-owned opaque continuation cursor.
pub async fn list_raw_transactions(
&self,
query: &ksp_store_api::RawTransactionQuery,
) -> std::result::Result<ksp_store_api::RawPage<ksp_store_api::RawTransactionReference>, crate::PostgresBackendError> {
return crate::list_raw_transactions(&self.pool, &self.network, query).await;
}
/// Reads one persisted RAW transaction observation by producer-owned idempotence key.
pub async fn get_raw_transaction_observation(
&self,