v0.3.6-pre.006

This commit is contained in:
2026-09-01 14:08:53 +02:00
parent ea5f756d17
commit 032919fa07
13 changed files with 1172 additions and 57 deletions

View File

@@ -1,5 +1,5 @@
# file: crates/ksp-job-backfill-lib/Cargo.toml
# version: 1
# version: 2
[package]
name = "ksp-job-backfill-lib"
@@ -13,6 +13,7 @@ ksp-job-api = { path = "../ksp-job-api" }
ksp-logging-lib = { path = "../ksp-logging-lib" }
ksp-onchain-transport-lib = { path = "../ksp-onchain-transport-lib" }
ksp-store-lib = { path = "../ksp-store-lib", default-features = false }
serde_json.workspace = true
sha2.workspace = true
[dev-dependencies]

View File

@@ -0,0 +1,475 @@
// file: crates/ksp-job-backfill-lib/src/conversion.rs
// version: 1
use sha2::Digest; // rust-rules: trait-import
/// KSP-owned source-independent RAW transaction format identifier produced by this Backfill vertical.
pub const RAW_TRANSACTION_FORMAT_ID: &str = "ksp.solana.raw_transaction";
/// Initial KSP-owned RAW transaction format version produced by this Backfill vertical.
pub const RAW_TRANSACTION_FORMAT_VERSION: u32 = 1;
const RAW_TRANSACTION_METHOD_CODE: &str = "getTransaction";
const RAW_TRANSACTION_OBSERVATION_CONTRACT_VERSION: u32 = 1;
const RAW_TRANSACTION_PROTOCOL_CODE: &str = "solana.http.json_rpc";
/// Complete in-memory RAW transaction acquisition ready for the later Store persistence tranche.
#[derive(Debug)]
pub struct BackfillRawAcquisition {
transaction: ksp_store_lib::RawTransaction,
observation: ksp_store_lib::RawTransactionObservation,
}
impl BackfillRawAcquisition {
/// Returns the canonical RAW transaction produced from the typed Transport response.
#[must_use]
pub const fn transaction(&self) -> &ksp_store_lib::RawTransaction {
return &self.transaction;
}
/// Returns the acquisition observation whose provenance records the actual successful endpoint.
#[must_use]
pub const fn observation(&self) -> &ksp_store_lib::RawTransactionObservation {
return &self.observation;
}
/// Consumes the in-memory acquisition into the canonical transaction and its observation.
#[must_use]
pub fn into_parts(self) -> (ksp_store_lib::RawTransaction, ksp_store_lib::RawTransactionObservation) {
return (self.transaction, self.observation);
}
}
/// Result of hydrating one deterministic Backfill candidate through observed `getTransaction`.
#[derive(Debug)]
pub enum BackfillHydrationOutcome {
/// The RPC returned one complete transaction and conversion produced canonical RAW plus provenance.
Available(BackfillRawAcquisition),
/// The RPC returned JSON `null`; only the canonical transaction identity exists and no provenance is fabricated.
Missing(ksp_store_lib::RawTransactionReference),
}
impl BackfillHydrationOutcome {
/// Returns the network-scoped transaction identity represented by this hydration outcome.
#[must_use]
pub fn reference(&self) -> &ksp_store_lib::RawTransactionReference {
return match self {
Self::Available(acquisition) => acquisition.transaction().reference(),
Self::Missing(reference) => reference,
};
}
/// Returns whether `getTransaction` returned JSON `null`.
#[must_use]
pub const fn is_missing(&self) -> bool {
return matches!(self, Self::Missing(_));
}
}
/// Hydrates one candidate with the typed observed Transport path and converts it to canonical RAW v1.
///
/// The caller supplies the local receipt timestamp because wall-clock ownership remains outside this
/// pure conversion tranche. Transport retains endpoint selection and retry. This function never
/// persists to Store; persistence begins in `pre.007`.
pub async fn hydrate_backfill_candidate(
transport: &ksp_onchain_transport_lib::HttpTransportPool,
request: &crate::BackfillRequest,
candidate: &crate::BackfillCandidate,
received_at: ksp_store_lib::RawTimestamp,
) -> ksp_core_lib::Result<BackfillHydrationOutcome> {
let reference = canonical_reference(request, candidate);
let reference = match reference {
std::result::Result::Ok(reference) => reference,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let config = ksp_onchain_transport_lib::SolanaGetTransactionConfig::new(
std::option::Option::Some(request.commitment().transport()),
std::option::Option::Some(ksp_onchain_transport_lib::SolanaTransactionEncoding::Base64),
std::option::Option::Some(0),
);
let observed = transport.get_transaction_observed(request.role(), candidate.identity().signature().as_str(), std::option::Option::Some(&config)).await;
let observed = match observed {
std::result::Result::Ok(observed) => observed,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let provider = observed.provider().as_str().to_owned();
let endpoint = observed.endpoint_name().to_owned();
let transaction = observed.into_value();
let transaction = match transaction {
std::option::Option::Some(transaction) => transaction,
std::option::Option::None => return std::result::Result::Ok(BackfillHydrationOutcome::Missing(reference)),
};
let fields = CanonicalTransactionFields {
slot: transaction.slot(),
block_time: transaction.block_time(),
transaction: transaction.transaction(),
meta: transaction.meta(),
version: transaction.version(),
transaction_index: transaction.transaction_index(),
};
let acquisition = convert_available_fields(request, reference, fields, provider.as_str(), endpoint.as_str(), received_at);
return match acquisition {
std::result::Result::Ok(acquisition) => std::result::Result::Ok(BackfillHydrationOutcome::Available(acquisition)),
std::result::Result::Err(error) => std::result::Result::Err(error),
};
}
/// Decodes one validated Base58 signature to exactly 64 canonical bytes without a Solana SDK dependency.
pub(crate) fn decode_backfill_signature(signature: &crate::BackfillSignature) -> ksp_core_lib::Result<ksp_store_lib::RawTransactionSignature> {
let text = signature.as_str().as_bytes();
let mut decoded = [0_u8; 64];
let mut leading_zeroes = 0_usize;
for byte in text {
if *byte != b'1' {
break;
}
leading_zeroes += 1;
}
for byte in text {
let digit = match base58_digit(*byte) {
std::option::Option::Some(digit) => digit,
std::option::Option::None => return std::result::Result::Err(conversion_error("signature")),
};
let mut carry = u32::from(digit);
for output in decoded.iter_mut().rev() {
let value = (u32::from(*output) * 58) + carry;
*output = (value & 0xff) as u8;
carry = value >> 8;
}
if carry != 0 {
return std::result::Result::Err(conversion_error("signature"));
}
}
let significant_len = match decoded.iter().position(|byte| return *byte != 0) {
std::option::Option::Some(index) => decoded.len() - index,
std::option::Option::None => 0,
};
if leading_zeroes + significant_len != decoded.len() {
return std::result::Result::Err(conversion_error("signature"));
}
return std::result::Result::Ok(ksp_store_lib::RawTransactionSignature::new(decoded));
}
struct CanonicalTransactionFields<'a> {
slot: u64,
block_time: std::option::Option<i64>,
transaction: &'a ksp_onchain_transport_lib::SolanaEncodedTransaction,
meta: &'a ksp_onchain_transport_lib::SolanaWireField<serde_json::Value>,
version: &'a ksp_onchain_transport_lib::SolanaWireField<ksp_onchain_transport_lib::SolanaTransactionVersion>,
transaction_index: &'a ksp_onchain_transport_lib::SolanaWireField<u32>,
}
fn canonical_reference(request: &crate::BackfillRequest, candidate: &crate::BackfillCandidate) -> ksp_core_lib::Result<ksp_store_lib::RawTransactionReference> {
if candidate.identity().network() != request.network() {
return std::result::Result::Err(conversion_error("candidate.network"));
}
let signature = crate::decode_backfill_signature(candidate.identity().signature());
let signature = match signature {
std::result::Result::Ok(signature) => signature,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
return std::result::Result::Ok(ksp_store_lib::RawTransactionReference::new(candidate.identity().network().clone(), signature));
}
fn convert_available_fields(
request: &crate::BackfillRequest,
reference: ksp_store_lib::RawTransactionReference,
fields: CanonicalTransactionFields<'_>,
provider: &str,
endpoint: &str,
received_at: ksp_store_lib::RawTimestamp,
) -> ksp_core_lib::Result<BackfillRawAcquisition> {
let block_time = convert_block_time(fields.block_time);
let block_time = match block_time {
std::result::Result::Ok(block_time) => block_time,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let bytes = canonical_payload_bytes(&fields);
let bytes = match bytes {
std::result::Result::Ok(bytes) => bytes,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let hash: [u8; 32] = sha2::Sha256::digest(bytes.as_slice()).into();
let format_id = ksp_store_lib::RawFormatId::new(RAW_TRANSACTION_FORMAT_ID);
let format_id = match format_id {
std::result::Result::Ok(format_id) => format_id,
std::result::Result::Err(_) => return std::result::Result::Err(conversion_error("payload.format_id")),
};
let payload =
ksp_store_lib::RawPayload::try_new(format_id, RAW_TRANSACTION_FORMAT_VERSION, bytes.into_boxed_slice(), ksp_store_lib::RawContentHash::new(hash));
let payload = match payload {
std::result::Result::Ok(payload) => payload,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let provenance = build_provenance(request, provider, endpoint, received_at);
let provenance = match provenance {
std::result::Result::Ok(provenance) => provenance,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let observation_key = observation_key(request, &reference, provider, endpoint);
let transaction = ksp_store_lib::RawTransaction::new(reference.clone(), fields.slot, block_time, payload);
let observation = ksp_store_lib::RawTransactionObservation::new(observation_key, reference, provenance);
return std::result::Result::Ok(BackfillRawAcquisition { transaction, observation });
}
fn convert_block_time(value: std::option::Option<i64>) -> ksp_core_lib::Result<std::option::Option<ksp_store_lib::RawTimestamp>> {
let seconds = match value {
std::option::Option::Some(seconds) => seconds,
std::option::Option::None => return std::result::Result::Ok(std::option::Option::None),
};
let seconds = match u64::try_from(seconds) {
std::result::Result::Ok(seconds) => seconds,
std::result::Result::Err(_) => return std::result::Result::Err(conversion_error("block_time")),
};
let millis = match seconds.checked_mul(1_000) {
std::option::Option::Some(millis) => millis,
std::option::Option::None => return std::result::Result::Err(conversion_error("block_time")),
};
let timestamp = ksp_store_lib::RawTimestamp::from_unix_millis(millis);
return match timestamp {
std::result::Result::Ok(timestamp) => std::result::Result::Ok(std::option::Option::Some(timestamp)),
std::result::Result::Err(_) => std::result::Result::Err(conversion_error("block_time")),
};
}
fn canonical_payload_bytes(fields: &CanonicalTransactionFields<'_>) -> ksp_core_lib::Result<std::vec::Vec<u8>> {
let (transaction_data, transaction_encoding) = match fields.transaction {
ksp_onchain_transport_lib::SolanaEncodedTransaction::Binary { data, encoding } => {
if *encoding != ksp_onchain_transport_lib::SolanaTransactionBinaryEncoding::Base64 {
return std::result::Result::Err(conversion_error("transaction.encoding"));
}
(data.as_str(), "base64")
},
ksp_onchain_transport_lib::SolanaEncodedTransaction::LegacyBinary(_) | ksp_onchain_transport_lib::SolanaEncodedTransaction::Json(_) => {
return std::result::Result::Err(conversion_error("transaction.encoding"));
},
};
let mut bytes = std::vec::Vec::new();
bytes.extend_from_slice(b"{\"transaction\":[");
if let std::result::Result::Err(error) = append_json_string(&mut bytes, transaction_data) {
return std::result::Result::Err(error);
}
bytes.push(b',');
if let std::result::Result::Err(error) = append_json_string(&mut bytes, transaction_encoding) {
return std::result::Result::Err(error);
}
bytes.push(b']');
if let std::result::Result::Err(error) = append_wire_value(&mut bytes, "meta", fields.meta, |output, value| return append_canonical_json(output, value)) {
return std::result::Result::Err(error);
}
let version_result = append_wire_value(&mut bytes, "version", fields.version, |output, value| {
return match value {
ksp_onchain_transport_lib::SolanaTransactionVersion::Legacy => append_json_string(output, "legacy"),
ksp_onchain_transport_lib::SolanaTransactionVersion::Number(number) => {
output.extend_from_slice(number.to_string().as_bytes());
std::result::Result::Ok(())
},
};
});
if let std::result::Result::Err(error) = version_result {
return std::result::Result::Err(error);
}
let transaction_index_result = append_wire_value(&mut bytes, "transactionIndex", fields.transaction_index, |output, value| {
output.extend_from_slice(value.to_string().as_bytes());
return std::result::Result::Ok(());
});
if let std::result::Result::Err(error) = transaction_index_result {
return std::result::Result::Err(error);
}
bytes.push(b'}');
return std::result::Result::Ok(bytes);
}
fn append_wire_value<T, F>(
output: &mut std::vec::Vec<u8>,
key: &str,
field: &ksp_onchain_transport_lib::SolanaWireField<T>,
mut append_value: F,
) -> ksp_core_lib::Result<()>
where
F: FnMut(&mut std::vec::Vec<u8>, &T) -> ksp_core_lib::Result<()>,
{
return match field {
ksp_onchain_transport_lib::SolanaWireField::Omitted => std::result::Result::Ok(()),
ksp_onchain_transport_lib::SolanaWireField::Null => {
output.push(b',');
let key_result = append_json_string(output, key);
if let std::result::Result::Err(error) = key_result {
return std::result::Result::Err(error);
}
output.extend_from_slice(b":null");
std::result::Result::Ok(())
},
ksp_onchain_transport_lib::SolanaWireField::Value(value) => {
output.push(b',');
let key_result = append_json_string(output, key);
if let std::result::Result::Err(error) = key_result {
return std::result::Result::Err(error);
}
output.push(b':');
append_value(output, value)
},
};
}
fn append_canonical_json(output: &mut std::vec::Vec<u8>, value: &serde_json::Value) -> ksp_core_lib::Result<()> {
return match value {
serde_json::Value::Null => {
output.extend_from_slice(b"null");
std::result::Result::Ok(())
},
serde_json::Value::Bool(value) => {
if *value {
output.extend_from_slice(b"true");
} else {
output.extend_from_slice(b"false");
}
std::result::Result::Ok(())
},
serde_json::Value::Number(value) => {
output.extend_from_slice(value.to_string().as_bytes());
std::result::Result::Ok(())
},
serde_json::Value::String(value) => append_json_string(output, value.as_str()),
serde_json::Value::Array(values) => {
output.push(b'[');
for (index, item) in values.iter().enumerate() {
if index != 0 {
output.push(b',');
}
let item_result = append_canonical_json(output, item);
if let std::result::Result::Err(error) = item_result {
return std::result::Result::Err(error);
}
}
output.push(b']');
std::result::Result::Ok(())
},
serde_json::Value::Object(values) => {
output.push(b'{');
let mut keys = values.keys().collect::<std::vec::Vec<_>>();
keys.sort_unstable();
for (index, key) in keys.iter().enumerate() {
if index != 0 {
output.push(b',');
}
let key_result = append_json_string(output, key.as_str());
if let std::result::Result::Err(error) = key_result {
return std::result::Result::Err(error);
}
output.push(b':');
let item = values.get(key.as_str());
let item = match item {
std::option::Option::Some(item) => item,
std::option::Option::None => return std::result::Result::Err(conversion_error("payload.meta")),
};
let item_result = append_canonical_json(output, item);
if let std::result::Result::Err(error) = item_result {
return std::result::Result::Err(error);
}
}
output.push(b'}');
std::result::Result::Ok(())
},
};
}
fn append_json_string(output: &mut std::vec::Vec<u8>, value: &str) -> ksp_core_lib::Result<()> {
let encoded = serde_json::to_vec(value);
return match encoded {
std::result::Result::Ok(encoded) => {
output.extend_from_slice(encoded.as_slice());
std::result::Result::Ok(())
},
std::result::Result::Err(_) => std::result::Result::Err(conversion_error("payload.json")),
};
}
fn build_provenance(
request: &crate::BackfillRequest,
provider: &str,
endpoint: &str,
received_at: ksp_store_lib::RawTimestamp,
) -> ksp_core_lib::Result<ksp_store_lib::RawAcquisitionProvenance> {
let provider = match provenance_code(provider, "provenance.provider") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let protocol = match provenance_code(RAW_TRANSACTION_PROTOCOL_CODE, "provenance.protocol") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let method = match provenance_code(RAW_TRANSACTION_METHOD_CODE, "provenance.method") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let endpoint = match provenance_code(endpoint, "provenance.endpoint") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let commitment = match provenance_code(request.commitment().code(), "provenance.commitment") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let capture_session = match provenance_code(request.job_id().as_str(), "provenance.capture_session") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let provenance = ksp_store_lib::RawAcquisitionProvenance::new(provider, protocol, method, ksp_store_lib::RawAcquisitionOrigin::Backfill, received_at)
.with_endpoint_id(endpoint)
.with_commitment(commitment)
.with_capture_session_id(capture_session);
return std::result::Result::Ok(provenance);
}
fn provenance_code(value: &str, field: &'static str) -> ksp_core_lib::Result<ksp_store_lib::RawProvenanceCode> {
let code = ksp_store_lib::RawProvenanceCode::new(value);
return match code {
std::result::Result::Ok(code) => std::result::Result::Ok(code),
std::result::Result::Err(_) => std::result::Result::Err(conversion_error(field)),
};
}
fn observation_key(
request: &crate::BackfillRequest,
reference: &ksp_store_lib::RawTransactionReference,
provider: &str,
endpoint: &str,
) -> ksp_store_lib::RawObservationKey {
let mut hasher = sha2::Sha256::new();
hasher.update(b"ksp.job.backfill.raw_transaction_observation.v1\0");
hash_bytes(&mut hasher, request.job_id().as_str().as_bytes());
hash_bytes(&mut hasher, request.scope_fingerprint().as_bytes());
hash_bytes(&mut hasher, reference.signature().as_bytes());
hash_bytes(&mut hasher, provider.as_bytes());
hash_bytes(&mut hasher, endpoint.as_bytes());
hash_bytes(&mut hasher, request.commitment().code().as_bytes());
hasher.update(RAW_TRANSACTION_OBSERVATION_CONTRACT_VERSION.to_be_bytes());
let bytes: [u8; 32] = hasher.finalize().into();
return ksp_store_lib::RawObservationKey::new(bytes);
}
fn hash_bytes(hasher: &mut sha2::Sha256, value: &[u8]) {
hasher.update((value.len() as u64).to_be_bytes());
hasher.update(value);
}
fn base58_digit(byte: u8) -> std::option::Option<u8> {
return match byte {
b'1'..=b'9' => std::option::Option::Some(byte - b'1'),
b'A'..=b'H' => std::option::Option::Some((byte - b'A') + 9),
b'J'..=b'N' => std::option::Option::Some((byte - b'J') + 17),
b'P'..=b'Z' => std::option::Option::Some((byte - b'P') + 22),
b'a'..=b'k' => std::option::Option::Some((byte - b'a') + 33),
b'm'..=b'z' => std::option::Option::Some((byte - b'm') + 44),
_ => std::option::Option::None,
};
}
fn conversion_error(field: &'static str) -> ksp_core_lib::Error {
return ksp_core_lib::Error::new(crate::ERROR_CODE_BACKFILL_RAW_CONVERSION_INVALID, "invalid deterministic Backfill RAW conversion")
.with_context("field", field);
}
#[cfg(test)]
#[path = "../unit_tests/conversion.rs"]
mod tests;

View File

@@ -1,10 +1,12 @@
// file: crates/ksp-job-backfill-lib/src/error.rs
// version: 1
// version: 2
/// Error code used when a signature page violates a bounded discovery invariant.
pub const ERROR_CODE_BACKFILL_DISCOVERY_INVALID: ksp_core_lib::ErrorCode = ksp_core_lib::ErrorCode::new("job_backfill", "discovery_invalid");
/// Error code used when paginated discovery cannot advance its exclusive RPC cursor safely.
pub const ERROR_CODE_BACKFILL_DISCOVERY_STALLED: ksp_core_lib::ErrorCode = ksp_core_lib::ErrorCode::new("job_backfill", "discovery_stalled");
/// Error code used when deterministic Transport-to-RAW conversion violates the v1 contract.
pub const ERROR_CODE_BACKFILL_RAW_CONVERSION_INVALID: ksp_core_lib::ErrorCode = ksp_core_lib::ErrorCode::new("job_backfill", "raw_conversion_invalid");
/// Error code used when one Backfill request violates its bounded admission contract.
pub const ERROR_CODE_BACKFILL_REQUEST_INVALID: ksp_core_lib::ErrorCode = ksp_core_lib::ErrorCode::new("job_backfill", "request_invalid");
/// Error code used when one transaction signature text violates the bounded Base58-shape contract.

View File

@@ -1,5 +1,5 @@
// file: crates/ksp-job-backfill-lib/src/lib.rs
// version: 1
// version: 2
#![warn(missing_docs)]
#![deny(unreachable_pub)]
@@ -7,16 +7,28 @@
//! Concrete bounded historical RAW transaction Backfill foundation.
//!
//! This tranche owns explicit admission, network-scoped candidate identity and deterministic
//! `getSignaturesForAddress` pagination. Transport retains provider/endpoint selection and retry;
//! Store retains durable idempotence and persistence. RAW conversion, persistence, concurrency,
//! checkpointing, cancellation and concrete latest-value snapshots are added by later v0.3.6 tranches.
//! This tranche owns explicit admission, network-scoped candidate identity, deterministic
//! `getSignaturesForAddress` pagination and canonical RAW v1 conversion through observed
//! `getTransaction`. Transport retains provider/endpoint selection and retry; Store retains
//! durable idempotence and persistence. Persistence, concurrency, checkpointing, cancellation
//! and concrete latest-value snapshots are added by later v0.3.6 tranches.
mod constants;
mod conversion;
mod discovery;
mod error;
mod request;
/// Result of hydrating one deterministic candidate through observed `getTransaction`.
pub use self::conversion::BackfillHydrationOutcome;
/// Complete in-memory RAW transaction acquisition ready for later Store persistence.
pub use self::conversion::BackfillRawAcquisition;
/// KSP-owned source-independent RAW transaction format identifier produced by this Backfill vertical.
pub use self::conversion::RAW_TRANSACTION_FORMAT_ID;
/// Initial KSP-owned RAW transaction format version produced by this Backfill vertical.
pub use self::conversion::RAW_TRANSACTION_FORMAT_VERSION;
/// Hydrates one candidate through observed Transport and converts a non-null response to canonical RAW v1.
pub use self::conversion::hydrate_backfill_candidate;
/// One deterministic transaction candidate produced by bounded discovery.
pub use self::discovery::BackfillCandidate;
/// Network-scoped identity of one discovered transaction candidate before canonical signature decoding.
@@ -31,6 +43,8 @@ pub use self::discovery::discover_backfill_candidates;
pub use self::error::ERROR_CODE_BACKFILL_DISCOVERY_INVALID;
/// Error code used when paginated discovery cannot advance its exclusive RPC cursor safely.
pub use self::error::ERROR_CODE_BACKFILL_DISCOVERY_STALLED;
/// Error code used when deterministic Transport-to-RAW conversion violates the v1 contract.
pub use self::error::ERROR_CODE_BACKFILL_RAW_CONVERSION_INVALID;
/// Error code used when one Backfill request violates its bounded admission contract.
pub use self::error::ERROR_CODE_BACKFILL_REQUEST_INVALID;
/// Error code used when one transaction signature text violates the bounded Base58-shape contract.
@@ -45,7 +59,7 @@ pub use self::request::BackfillScope;
pub use self::request::BackfillScopeFingerprint;
/// Stable category of one bounded Backfill discovery scope.
pub use self::request::BackfillScopeKind;
/// Bounded Base58-shaped transaction signature text used before canonical 64-byte decoding.
/// Bounded Base58-shaped transaction signature text with exact RAW signature conversion.
pub use self::request::BackfillSignature;
/// Maximum number of transaction candidates admitted by one bounded Backfill Job.
pub use self::request::MAX_BACKFILL_CANDIDATES;
@@ -62,3 +76,5 @@ pub use self::request::MIN_BACKFILL_SIGNATURE_TEXT_BYTES;
/// Owning tracing target used by the concrete Backfill runtime.
pub(crate) use self::constants::TRACING_TARGET;
/// Exact private Base58 decoder shared by the public signature wrapper and hydration path.
pub(crate) use self::conversion::decode_backfill_signature;

View File

@@ -1,5 +1,5 @@
// file: crates/ksp-job-backfill-lib/src/request.rs
// version: 1
// version: 2
use sha2::Digest; // rust-rules: trait-import
@@ -45,10 +45,10 @@ impl BackfillCommitment {
}
}
/// Bounded Base58-shaped transaction signature text used before canonical 64-byte decoding.
/// Bounded Base58-shaped transaction signature text used by discovery and RAW conversion.
///
/// This type deliberately validates only the encoded shape required by discovery. Exact decoding
/// to the Store-owned 64-byte signature is introduced by the RAW conversion tranche.
/// Construction validates the encoded shape required by discovery. Exact conversion to the
/// Store-owned 64-byte signature is available through [`Self::to_raw_transaction_signature`].
#[derive(Clone, Eq, Hash, Ord, PartialEq, PartialOrd)]
pub struct BackfillSignature(std::string::String);
@@ -67,6 +67,11 @@ impl BackfillSignature {
pub fn as_str(&self) -> &str {
return self.0.as_str();
}
/// Decodes this Base58 text to the exact Store-owned 64-byte Solana signature.
pub fn to_raw_transaction_signature(&self) -> ksp_core_lib::Result<ksp_store_lib::RawTransactionSignature> {
return crate::decode_backfill_signature(self);
}
}
impl std::fmt::Debug for BackfillSignature {

View File

@@ -1,10 +1,10 @@
// file: crates/ksp-job-backfill-lib/tests/dependency_boundary.rs
// version: 1
// version: 2
//! Dependency firewall canaries for the concrete Backfill foundation.
//! Dependency firewall canaries for Backfill discovery and RAW v1 conversion.
#[test]
fn pre_005_manifest_uses_only_planned_ksp_edges_and_backend_neutral_store() {
fn pre_006_manifest_uses_only_planned_ksp_edges_and_backend_neutral_store() {
let manifest = include_str!("../Cargo.toml");
for required in [
"ksp-core-lib = { path = \"../ksp-core-lib\" }",
@@ -12,6 +12,7 @@ fn pre_005_manifest_uses_only_planned_ksp_edges_and_backend_neutral_store() {
"ksp-logging-lib = { path = \"../ksp-logging-lib\" }",
"ksp-onchain-transport-lib = { path = \"../ksp-onchain-transport-lib\" }",
"ksp-store-lib = { path = \"../ksp-store-lib\", default-features = false }",
"serde_json.workspace = true",
"sha2.workspace = true",
] {
assert!(manifest.contains(required), "required Backfill dependency missing: {required}");
@@ -25,7 +26,8 @@ fn pre_005_manifest_uses_only_planned_ksp_edges_and_backend_neutral_store() {
"ksp-wallet-lib",
"solana-",
"reqwest",
"serde",
"serde.workspace = true",
"serde = {",
"tonic",
] {
assert!(!manifest.contains(forbidden), "forbidden Backfill dependency present: {forbidden}");
@@ -34,21 +36,32 @@ fn pre_005_manifest_uses_only_planned_ksp_edges_and_backend_neutral_store() {
}
#[test]
fn pre_005_production_sources_keep_transport_and_store_in_their_owned_layers() {
let sources = [
fn pre_006_production_sources_keep_transport_and_store_in_their_owned_layers() {
let non_conversion_sources = [
include_str!("../src/constants.rs"),
include_str!("../src/discovery.rs"),
include_str!("../src/error.rs"),
include_str!("../src/lib.rs"),
include_str!("../src/request.rs"),
];
for source in sources {
for source in non_conversion_sources {
for forbidden in
["ksp_config_lib::", "ksp_interface_lib::", "ksp_store_api::", "ksp_store_postgres_lib::", "reqwest::", "serde_json::", "std::env", "tonic::"]
{
assert!(!source.contains(forbidden), "forbidden concrete Backfill path detected: {forbidden}");
}
}
let conversion = include_str!("../src/conversion.rs");
assert!(conversion.contains("serde_json::"));
assert!(conversion.contains("get_transaction_observed"));
assert!(conversion.contains("SolanaTransactionEncoding::Base64"));
assert!(conversion.contains("std::option::Option::Some(0)"));
assert!(conversion.contains("BackfillHydrationOutcome::Missing(reference)"));
assert!(!conversion.contains("execute_standard_rpc"));
assert!(!conversion.contains("persist_raw_transaction_acquisition"));
for forbidden in ["ksp_config_lib::", "ksp_interface_lib::", "ksp_store_api::", "ksp_store_postgres_lib::", "reqwest::", "std::env", "tonic::"] {
assert!(!conversion.contains(forbidden), "forbidden RAW conversion path detected: {forbidden}");
}
let discovery = include_str!("../src/discovery.rs");
assert!(discovery.contains("get_signatures_for_address"));
assert!(!discovery.contains("execute_standard_rpc"));

View File

@@ -1,7 +1,7 @@
// file: crates/ksp-job-backfill-lib/tests/public_api.rs
// version: 1
// version: 2
//! Public API canaries for the bounded Backfill foundation.
//! Public API canaries for bounded Backfill discovery and RAW v1 conversion.
#[test]
fn pre_005_request_scope_and_discovery_contracts_are_available_from_crate_root() {
@@ -64,3 +64,25 @@ fn pre_005_candidate_identity_is_network_plus_signature_not_transport_source() {
assert_ne!(mainnet_identity, devnet_identity);
return;
}
#[test]
fn pre_006_raw_conversion_contract_is_available_from_crate_root() {
assert_eq!(ksp_job_backfill_lib::RAW_TRANSACTION_FORMAT_ID, "ksp.solana.raw_transaction");
assert_eq!(ksp_job_backfill_lib::RAW_TRANSACTION_FORMAT_VERSION, 1);
let signature = ksp_job_backfill_lib::BackfillSignature::new("1".repeat(64));
assert!(signature.is_ok());
let signature = match signature {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return,
};
let decoded = signature.to_raw_transaction_signature();
assert!(decoded.is_ok());
if let std::result::Result::Ok(decoded) = decoded {
assert_eq!(decoded.as_bytes(), &[0_u8; 64]);
}
let _hydrate = ksp_job_backfill_lib::hydrate_backfill_candidate;
let _outcome: std::option::Option<ksp_job_backfill_lib::BackfillHydrationOutcome> = std::option::Option::None;
let _acquisition: std::option::Option<ksp_job_backfill_lib::BackfillRawAcquisition> = std::option::Option::None;
assert_eq!(ksp_job_backfill_lib::ERROR_CODE_BACKFILL_RAW_CONVERSION_INVALID, ksp_core_lib::ErrorCode::new("job_backfill", "raw_conversion_invalid"));
return;
}

View File

@@ -1,10 +1,10 @@
// file: crates/ksp-job-backfill-lib/tests/release_completeness.rs
// version: 1
// version: 2
//! Completeness canaries for the `pre.005` Backfill foundation.
//! Completeness canaries for the `pre.006` Backfill RAW conversion tranche.
#[test]
fn pre_005_production_module_inventory_is_exact() -> std::io::Result<()> {
fn pre_006_production_module_inventory_is_exact() -> std::io::Result<()> {
let source_root = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("src");
let entries = match std::fs::read_dir(source_root) {
std::result::Result::Ok(value) => value,
@@ -32,12 +32,12 @@ fn pre_005_production_module_inventory_is_exact() -> std::io::Result<()> {
}
}
names.sort_unstable();
assert_eq!(names, std::vec!["constants.rs", "discovery.rs", "error.rs", "lib.rs", "request.rs"]);
assert_eq!(names, std::vec!["constants.rs", "conversion.rs", "discovery.rs", "error.rs", "lib.rs", "request.rs"]);
return std::result::Result::Ok(());
}
#[test]
fn pre_005_surface_is_discovery_only_without_raw_persistence_or_checkpoint_runtime() {
fn pre_006_surface_adds_raw_conversion_without_persistence_or_checkpoint_runtime() {
let root = include_str!("../src/lib.rs");
for required in [
"BackfillCandidate",
@@ -50,11 +50,17 @@ fn pre_005_surface_is_discovery_only_without_raw_persistence_or_checkpoint_runti
"BackfillScopeFingerprint",
"BackfillSignature",
"discover_backfill_candidates",
"BackfillRawAcquisition",
"BackfillHydrationOutcome",
"hydrate_backfill_candidate",
"RAW_TRANSACTION_FORMAT_ID",
"RAW_TRANSACTION_FORMAT_VERSION",
"ERROR_CODE_BACKFILL_RAW_CONVERSION_INVALID",
] {
assert!(root.contains(required), "required pre.005 public contract missing: {required}");
assert!(root.contains(required), "required pre.006 public contract missing: {required}");
}
for forbidden in ["RawTransactionObservation", "persist_raw_transaction_acquisition", "BackfillCheckpoint", "BackfillJobHandle", "JobSnapshotSource"] {
assert!(!root.contains(forbidden), "later Backfill tranche leaked into pre.005: {forbidden}");
for forbidden in ["persist_raw_transaction_acquisition", "BackfillCheckpoint", "BackfillJobHandle", "JobSnapshotSource"] {
assert!(!root.contains(forbidden), "later Backfill tranche leaked into pre.006: {forbidden}");
}
assert!(!root.contains("pub mod "));
return;

View File

@@ -0,0 +1,295 @@
// file: crates/ksp-job-backfill-lib/unit_tests/conversion.rs
// version: 1
fn signature_text() -> std::option::Option<crate::BackfillSignature> {
return match crate::BackfillSignature::new("1".repeat(64)) {
std::result::Result::Ok(value) => std::option::Option::Some(value),
std::result::Result::Err(_) => std::option::Option::None,
};
}
fn request(network: &str, signature: crate::BackfillSignature) -> std::option::Option<crate::BackfillRequest> {
let network = match ksp_store_lib::RawNetworkId::new(network) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return std::option::Option::None,
};
let job_id = match ksp_job_api::JobId::new("backfill:pre006") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return std::option::Option::None,
};
let scope = match crate::BackfillScope::explicit_signatures(std::vec![signature]) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return std::option::Option::None,
};
return match crate::BackfillRequest::new(
job_id,
network,
ksp_onchain_transport_lib::HttpRoleName::new("history"),
crate::BackfillCommitment::Finalized,
scope,
100,
10,
100,
4,
std::option::Option::None,
) {
std::result::Result::Ok(value) => std::option::Option::Some(value),
std::result::Result::Err(_) => std::option::Option::None,
};
}
fn candidate(network: &str, signature: crate::BackfillSignature) -> std::option::Option<crate::BackfillCandidate> {
let network = match ksp_store_lib::RawNetworkId::new(network) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return std::option::Option::None,
};
return std::option::Option::Some(crate::BackfillCandidate::new(crate::BackfillCandidateIdentity::new(network, signature), std::option::Option::Some(42)));
}
fn received_at() -> std::option::Option<ksp_store_lib::RawTimestamp> {
return match ksp_store_lib::RawTimestamp::from_unix_millis(1_700_000_001_000) {
std::result::Result::Ok(value) => std::option::Option::Some(value),
std::result::Result::Err(_) => std::option::Option::None,
};
}
fn fields<'a>(
transaction: &'a ksp_onchain_transport_lib::SolanaEncodedTransaction,
meta: &'a ksp_onchain_transport_lib::SolanaWireField<serde_json::Value>,
version: &'a ksp_onchain_transport_lib::SolanaWireField<ksp_onchain_transport_lib::SolanaTransactionVersion>,
transaction_index: &'a ksp_onchain_transport_lib::SolanaWireField<u32>,
block_time: std::option::Option<i64>,
) -> super::CanonicalTransactionFields<'a> {
return super::CanonicalTransactionFields { slot: 123, block_time, transaction, meta, version, transaction_index };
}
#[test]
fn pre_006_signature_decoding_requires_exactly_sixty_four_bytes() {
let signature = match signature_text() {
std::option::Option::Some(value) => value,
std::option::Option::None => return,
};
let decoded = signature.to_raw_transaction_signature();
assert!(decoded.is_ok());
if let std::result::Result::Ok(decoded) = decoded {
assert_eq!(decoded.as_bytes(), &[0_u8; 64]);
}
let too_many_zeroes = crate::BackfillSignature::new("1".repeat(65));
assert!(too_many_zeroes.is_ok());
if let std::result::Result::Ok(too_many_zeroes) = too_many_zeroes {
assert!(too_many_zeroes.to_raw_transaction_signature().is_err());
}
let overflow = crate::BackfillSignature::new("z".repeat(88));
assert!(overflow.is_ok());
if let std::result::Result::Ok(overflow) = overflow {
assert!(overflow.to_raw_transaction_signature().is_err());
}
return;
}
#[test]
fn pre_006_canonical_payload_golden_bytes_hash_and_provenance_are_exact() {
let signature = match signature_text() {
std::option::Option::Some(value) => value,
std::option::Option::None => return,
};
let request = match request("devnet", signature.clone()) {
std::option::Option::Some(value) => value,
std::option::Option::None => return,
};
let candidate = match candidate("devnet", signature) {
std::option::Option::Some(value) => value,
std::option::Option::None => return,
};
let reference = match super::canonical_reference(&request, &candidate) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return,
};
let transaction = ksp_onchain_transport_lib::SolanaEncodedTransaction::Binary {
data: "AQID".to_owned(),
encoding: ksp_onchain_transport_lib::SolanaTransactionBinaryEncoding::Base64,
};
let meta = ksp_onchain_transport_lib::SolanaWireField::Value(serde_json::json!({"z": 1, "a": {"y": true, "x": null}}));
let version = ksp_onchain_transport_lib::SolanaWireField::Value(ksp_onchain_transport_lib::SolanaTransactionVersion::Legacy);
let transaction_index = ksp_onchain_transport_lib::SolanaWireField::Value(7_u32);
let received_at = match received_at() {
std::option::Option::Some(value) => value,
std::option::Option::None => return,
};
let acquisition = super::convert_available_fields(
&request,
reference,
fields(&transaction, &meta, &version, &transaction_index, std::option::Option::Some(1_700_000_000)),
"publicnode",
"devnet-primary",
received_at,
);
assert!(acquisition.is_ok());
let acquisition = match acquisition {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return,
};
let expected = b"{\"transaction\":[\"AQID\",\"base64\"],\"meta\":{\"a\":{\"x\":null,\"y\":true},\"z\":1},\"version\":\"legacy\",\"transactionIndex\":7}";
assert_eq!(acquisition.transaction().payload().bytes(), expected);
assert_eq!(acquisition.transaction().payload().byte_len(), 112);
assert_eq!(acquisition.transaction().payload().format_id().as_str(), crate::RAW_TRANSACTION_FORMAT_ID);
assert_eq!(acquisition.transaction().payload().format_version(), crate::RAW_TRANSACTION_FORMAT_VERSION);
assert_eq!(
acquisition.transaction().payload().content_hash().as_bytes(),
&[
34, 7, 146, 210, 177, 93, 38, 47, 218, 36, 44, 178, 32, 119, 78, 233, 221, 239, 254, 191, 4, 220, 250, 218, 188, 248, 239, 118, 169, 177, 167, 195
]
);
assert_eq!(acquisition.transaction().slot(), 123);
assert_eq!(acquisition.transaction().block_time().map(|value| return value.unix_millis()), std::option::Option::Some(1_700_000_000_000));
assert_eq!(acquisition.transaction().reference().network().as_str(), "devnet");
assert_eq!(acquisition.transaction().reference().signature().as_bytes(), &[0_u8; 64]);
let provenance = acquisition.observation().provenance();
assert_eq!(provenance.provider().as_str(), "publicnode");
assert_eq!(provenance.protocol().as_str(), "solana.http.json_rpc");
assert_eq!(provenance.acquisition_method().as_str(), "getTransaction");
assert_eq!(provenance.endpoint_id().map(ksp_store_lib::RawProvenanceCode::as_str), std::option::Option::Some("devnet-primary"));
assert_eq!(provenance.commitment().map(ksp_store_lib::RawProvenanceCode::as_str), std::option::Option::Some("finalized"));
assert_eq!(provenance.capture_session_id().map(ksp_store_lib::RawProvenanceCode::as_str), std::option::Option::Some("backfill:pre006"));
assert_eq!(provenance.received_at(), received_at);
assert!(provenance.source_payload_hash().is_none());
assert!(provenance.source_payload_size_bytes().is_none());
return;
}
#[test]
fn pre_006_wire_omission_and_null_produce_distinct_canonical_bytes() {
let transaction = ksp_onchain_transport_lib::SolanaEncodedTransaction::Binary {
data: "AQID".to_owned(),
encoding: ksp_onchain_transport_lib::SolanaTransactionBinaryEncoding::Base64,
};
let meta_omitted = ksp_onchain_transport_lib::SolanaWireField::Omitted;
let meta_null = ksp_onchain_transport_lib::SolanaWireField::Null;
let version_omitted = ksp_onchain_transport_lib::SolanaWireField::Omitted;
let version_null = ksp_onchain_transport_lib::SolanaWireField::Null;
let transaction_index_omitted = ksp_onchain_transport_lib::SolanaWireField::Omitted;
let transaction_index_null = ksp_onchain_transport_lib::SolanaWireField::Null;
let omitted = super::canonical_payload_bytes(&fields(&transaction, &meta_omitted, &version_omitted, &transaction_index_omitted, std::option::Option::None));
assert!(omitted.is_ok());
let omitted = match omitted {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return,
};
assert_eq!(omitted, b"{\"transaction\":[\"AQID\",\"base64\"]}");
let nulls = super::canonical_payload_bytes(&fields(&transaction, &meta_null, &version_null, &transaction_index_null, std::option::Option::None));
assert!(nulls.is_ok());
let nulls = match nulls {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return,
};
assert_eq!(nulls, b"{\"transaction\":[\"AQID\",\"base64\"],\"meta\":null,\"version\":null,\"transactionIndex\":null}");
assert_ne!(omitted, nulls);
return;
}
#[test]
fn pre_006_non_base64_transaction_shapes_are_rejected() {
let base58 = ksp_onchain_transport_lib::SolanaEncodedTransaction::Binary {
data: "1111".to_owned(),
encoding: ksp_onchain_transport_lib::SolanaTransactionBinaryEncoding::Base58,
};
let legacy = ksp_onchain_transport_lib::SolanaEncodedTransaction::LegacyBinary("1111".to_owned());
let json = ksp_onchain_transport_lib::SolanaEncodedTransaction::Json(serde_json::json!({"message": {}}));
let meta = ksp_onchain_transport_lib::SolanaWireField::Omitted;
let version = ksp_onchain_transport_lib::SolanaWireField::Omitted;
let transaction_index = ksp_onchain_transport_lib::SolanaWireField::Omitted;
for transaction in [&base58, &legacy, &json] {
let result = super::canonical_payload_bytes(&fields(transaction, &meta, &version, &transaction_index, std::option::Option::None));
assert!(result.is_err());
if let std::result::Result::Err(error) = result {
assert_eq!(error.code(), crate::ERROR_CODE_BACKFILL_RAW_CONVERSION_INVALID);
}
}
return;
}
#[test]
fn pre_006_negative_and_unrepresentable_block_times_are_terminal_conversion_errors() {
let negative = super::convert_block_time(std::option::Option::Some(-1));
assert!(negative.is_err());
let oversized = super::convert_block_time(std::option::Option::Some(i64::MAX));
assert!(oversized.is_err());
let absent = super::convert_block_time(std::option::Option::None);
assert!(matches!(absent, std::result::Result::Ok(std::option::Option::None)));
return;
}
#[test]
fn pre_006_observation_key_is_deterministic_and_endpoint_specific() {
let signature = match signature_text() {
std::option::Option::Some(value) => value,
std::option::Option::None => return,
};
let request = match request("devnet", signature.clone()) {
std::option::Option::Some(value) => value,
std::option::Option::None => return,
};
let candidate = match candidate("devnet", signature) {
std::option::Option::Some(value) => value,
std::option::Option::None => return,
};
let reference = match super::canonical_reference(&request, &candidate) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return,
};
let first = super::observation_key(&request, &reference, "provider", "endpoint-a");
let same = super::observation_key(&request, &reference, "provider", "endpoint-a");
let other_endpoint = super::observation_key(&request, &reference, "provider", "endpoint-b");
let other_provider = super::observation_key(&request, &reference, "provider-2", "endpoint-a");
assert_eq!(first, same);
assert_ne!(first, other_endpoint);
assert_ne!(first, other_provider);
return;
}
#[test]
fn pre_006_candidate_network_mismatch_is_rejected_before_transport() {
let signature = match signature_text() {
std::option::Option::Some(value) => value,
std::option::Option::None => return,
};
let request = match request("devnet", signature.clone()) {
std::option::Option::Some(value) => value,
std::option::Option::None => return,
};
let candidate = match candidate("mainnet", signature) {
std::option::Option::Some(value) => value,
std::option::Option::None => return,
};
let reference = super::canonical_reference(&request, &candidate);
assert!(reference.is_err());
if let std::result::Result::Err(error) = reference {
assert_eq!(error.code(), crate::ERROR_CODE_BACKFILL_RAW_CONVERSION_INVALID);
}
return;
}
#[test]
fn pre_006_missing_outcome_contains_only_network_scoped_reference() {
let signature = match signature_text() {
std::option::Option::Some(value) => value,
std::option::Option::None => return,
};
let request = match request("devnet", signature.clone()) {
std::option::Option::Some(value) => value,
std::option::Option::None => return,
};
let candidate = match candidate("devnet", signature) {
std::option::Option::Some(value) => value,
std::option::Option::None => return,
};
let reference = match super::canonical_reference(&request, &candidate) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return,
};
let outcome = crate::BackfillHydrationOutcome::Missing(reference);
assert!(outcome.is_missing());
assert_eq!(outcome.reference().network().as_str(), "devnet");
assert_eq!(outcome.reference().signature().as_bytes(), &[0_u8; 64]);
return;
}