v0.3.10-pre.003

This commit is contained in:
2026-09-06 10:41:47 +02:00
parent aabe8dedb2
commit 8309d829cc
12 changed files with 546 additions and 289 deletions

View File

@@ -1,12 +1,12 @@
# file: Cargo.toml # file: Cargo.toml
# version: 492 # version: 493
[workspace] [workspace]
resolver = "3" resolver = "3"
members = ["crates/ksp-app-backfill-desk", "crates/ksp-app-config-desk", "crates/ksp-app-solprices-desk", "crates/ksp-app-store-desk", "crates/ksp-app-wallet-desk", "crates/ksp-config-lib", "crates/ksp-core-lib", "crates/ksp-interface-lib", "crates/ksp-job-api", "crates/ksp-job-backfill-lib", "crates/ksp-logging-lib", "crates/ksp-offchain-transport-lib", "crates/ksp-onchain-transport-lib", "crates/ksp-program-api", "crates/ksp-raw-transaction-lib", "crates/ksp-store-api", "crates/ksp-store-lib", "crates/ksp-store-postgres-lib", "crates/ksp-wallet-lib", "crates/ksp-worker-api"] members = ["crates/ksp-app-backfill-desk", "crates/ksp-app-config-desk", "crates/ksp-app-solprices-desk", "crates/ksp-app-store-desk", "crates/ksp-app-wallet-desk", "crates/ksp-config-lib", "crates/ksp-core-lib", "crates/ksp-interface-lib", "crates/ksp-job-api", "crates/ksp-job-backfill-lib", "crates/ksp-logging-lib", "crates/ksp-offchain-transport-lib", "crates/ksp-onchain-transport-lib", "crates/ksp-program-api", "crates/ksp-raw-transaction-lib", "crates/ksp-store-api", "crates/ksp-store-lib", "crates/ksp-store-postgres-lib", "crates/ksp-wallet-lib", "crates/ksp-worker-api"]
[workspace.package] [workspace.package]
version = "0.3.10-pre.2.fix.1" version = "0.3.10-pre.3"
edition = "2024" edition = "2024"
license = "MIT" license = "MIT"
repository = "https://git.sasedev.com/Sasedev/khadhroony-solana-project" repository = "https://git.sasedev.com/Sasedev/khadhroony-solana-project"

View File

@@ -1,5 +1,5 @@
# file: crates/ksp-job-backfill-lib/Cargo.toml # file: crates/ksp-job-backfill-lib/Cargo.toml
# version: 4 # version: 5
[package] [package]
name = "ksp-job-backfill-lib" name = "ksp-job-backfill-lib"
@@ -13,6 +13,7 @@ ksp-core-lib = { path = "../ksp-core-lib" }
ksp-job-api = { path = "../ksp-job-api" } ksp-job-api = { path = "../ksp-job-api" }
ksp-logging-lib = { path = "../ksp-logging-lib" } ksp-logging-lib = { path = "../ksp-logging-lib" }
ksp-onchain-transport-lib = { path = "../ksp-onchain-transport-lib" } ksp-onchain-transport-lib = { path = "../ksp-onchain-transport-lib" }
ksp-raw-transaction-lib = { path = "../ksp-raw-transaction-lib" }
ksp-store-lib = { path = "../ksp-store-lib", default-features = false } ksp-store-lib = { path = "../ksp-store-lib", default-features = false }
serde_json.workspace = true serde_json.workspace = true
sha2.workspace = true sha2.workspace = true

View File

@@ -1,11 +1,11 @@
<!-- file: crates/ksp-job-backfill-lib/README.md --> <!-- file: crates/ksp-job-backfill-lib/README.md -->
<!-- version: 1 --> <!-- version: 2 -->
# ksp-job-backfill-lib # ksp-job-backfill-lib
`ksp-job-backfill-lib` implémente le premier job historique concret de KSP : un backfill borné de transactions Solana vers la couche RAW durable. `ksp-job-backfill-lib` implémente le premier job historique concret de KSP : un backfill borné de transactions Solana vers la couche RAW durable.
La crate compose les contrats Job, Transport observé et Store backend-neutral sans posséder leurs politiques internes. Elle couvre l'admission, la découverte historique, l'hydratation `getTransaction`, la conversion RAW v1, la persistance atomique, la concurrence bornée, la frontier contiguë, le checkpoint caller-owned, l'annulation coopérative et les snapshots latest-value. La crate compose les contrats Job, Transport observé, common RAW et Store backend-neutral sans posséder leurs politiques internes. Elle couvre l'admission, la découverte historique, l'hydratation `getTransaction`, l'adaptation vers `ksp-raw-transaction-lib`, la persistance atomique, la concurrence bornée, la frontier contiguë, le checkpoint caller-owned, l'annulation coopérative et les snapshots latest-value.
## Identité ## Identité
@@ -94,6 +94,7 @@ ksp-core-lib
ksp-job-api ksp-job-api
ksp-logging-lib ksp-logging-lib
ksp-onchain-transport-lib ksp-onchain-transport-lib
ksp-raw-transaction-lib
ksp-store-lib (default-features = false) ksp-store-lib (default-features = false)
futures-util futures-util
serde_json serde_json

View File

@@ -1,18 +1,13 @@
// file: crates/ksp-job-backfill-lib/src/conversion.rs // file: crates/ksp-job-backfill-lib/src/conversion.rs
// version: 4 // version: 5
use sha2::Digest; // rust-rules: trait-import use sha2::Digest; // rust-rules: trait-import
/// KSP-owned source-independent RAW transaction format identifier produced by this Backfill vertical.
pub const RAW_TRANSACTION_FORMAT_ID: &str = "ksp.solana.raw_transaction";
/// Initial KSP-owned RAW transaction format version produced by this Backfill vertical.
pub const RAW_TRANSACTION_FORMAT_VERSION: u32 = 1;
const RAW_TRANSACTION_METHOD_CODE: &str = "getTransaction"; const RAW_TRANSACTION_METHOD_CODE: &str = "getTransaction";
const RAW_TRANSACTION_OBSERVATION_CONTRACT_VERSION: u32 = 1; const RAW_TRANSACTION_OBSERVATION_CONTRACT_VERSION: u32 = 1;
const RAW_TRANSACTION_PROTOCOL_CODE: &str = "solana.http.json_rpc"; const RAW_TRANSACTION_PROTOCOL_CODE: &str = "solana.http.json_rpc";
/// Complete in-memory RAW transaction acquisition ready for the later Store persistence tranche. /// Complete in-memory RAW transaction acquisition ready for Store persistence.
#[derive(Debug)] #[derive(Debug)]
pub struct BackfillRawAcquisition { pub struct BackfillRawAcquisition {
inner: Box<BackfillRawAcquisitionInner>, inner: Box<BackfillRawAcquisitionInner>,
@@ -20,35 +15,34 @@ pub struct BackfillRawAcquisition {
#[derive(Debug)] #[derive(Debug)]
struct BackfillRawAcquisitionInner { struct BackfillRawAcquisitionInner {
transaction: ksp_store_lib::RawTransaction, acquisition: ksp_raw_transaction_lib::RawTransactionAcquisition,
observation: ksp_store_lib::RawTransactionObservation,
} }
impl crate::BackfillRawAcquisition { impl crate::BackfillRawAcquisition {
/// Returns the canonical RAW transaction produced from the typed Transport response. /// Returns the canonical RAW transaction produced through the source-neutral common layer.
#[must_use] #[must_use]
pub const fn transaction(&self) -> &ksp_store_lib::RawTransaction { pub fn transaction(&self) -> &ksp_store_lib::RawTransaction {
return &self.inner.transaction; return self.inner.acquisition.transaction();
} }
/// Returns the acquisition observation whose provenance records the actual successful endpoint. /// Returns the Backfill-owned acquisition observation linked by the common assembly layer.
#[must_use] #[must_use]
pub const fn observation(&self) -> &ksp_store_lib::RawTransactionObservation { pub fn observation(&self) -> &ksp_store_lib::RawTransactionObservation {
return &self.inner.observation; return self.inner.acquisition.observation();
} }
/// Consumes the in-memory acquisition into the canonical transaction and its observation. /// Consumes the in-memory acquisition into the canonical transaction and its Backfill-owned observation.
#[must_use] #[must_use]
pub fn into_parts(self) -> (ksp_store_lib::RawTransaction, ksp_store_lib::RawTransactionObservation) { pub fn into_parts(self) -> (ksp_store_lib::RawTransaction, ksp_store_lib::RawTransactionObservation) {
let inner = *self.inner; let inner = *self.inner;
return (inner.transaction, inner.observation); return inner.acquisition.into_parts();
} }
} }
/// Result of hydrating one deterministic Backfill candidate through observed `getTransaction`. /// Result of hydrating one deterministic Backfill candidate through observed `getTransaction`.
#[derive(Debug)] #[derive(Debug)]
pub enum BackfillHydrationOutcome { pub enum BackfillHydrationOutcome {
/// The RPC returned one complete transaction and conversion produced canonical RAW plus provenance. /// The RPC returned one complete transaction and common conversion produced canonical RAW plus Backfill provenance.
Available(crate::BackfillRawAcquisition), Available(crate::BackfillRawAcquisition),
/// The RPC returned JSON `null`; only the canonical transaction identity exists and no provenance is fabricated. /// The RPC returned JSON `null`; only the canonical transaction identity exists and no provenance is fabricated.
Missing(ksp_store_lib::RawTransactionReference), Missing(ksp_store_lib::RawTransactionReference),
@@ -71,11 +65,12 @@ impl crate::BackfillHydrationOutcome {
} }
} }
/// Hydrates one candidate with the typed observed Transport path and converts it to canonical RAW v1. /// Hydrates one candidate with the typed observed Transport path and converts it through the common RAW v1 layer.
/// ///
/// The caller supplies the local receipt timestamp because wall-clock ownership remains outside this /// The caller supplies the local receipt timestamp because wall-clock ownership remains outside this
/// pure conversion tranche. Transport retains endpoint selection and retry. This function never /// pure conversion path. Transport retains endpoint selection and retry. Backfill retains campaign
/// persists to Store; persistence begins in `pre.007`. /// provenance and observation-key ownership; the common crate owns signature parsing, RAW v1
/// canonicalization, content hashing and transaction/observation assembly.
pub async fn hydrate_backfill_candidate( pub async fn hydrate_backfill_candidate(
transport: &ksp_onchain_transport_lib::HttpTransportPool, transport: &ksp_onchain_transport_lib::HttpTransportPool,
request: &crate::BackfillRequest, request: &crate::BackfillRequest,
@@ -105,12 +100,12 @@ pub async fn hydrate_backfill_candidate(
std::option::Option::None => return std::result::Result::Ok(crate::BackfillHydrationOutcome::Missing(reference)), std::option::Option::None => return std::result::Result::Ok(crate::BackfillHydrationOutcome::Missing(reference)),
}; };
let fields = CanonicalTransactionFields { let fields = CanonicalTransactionFields {
slot: transaction.slot(),
block_time: transaction.block_time(), block_time: transaction.block_time(),
transaction: transaction.transaction(),
meta: transaction.meta(), meta: transaction.meta(),
version: transaction.version(), slot: transaction.slot(),
transaction: transaction.transaction(),
transaction_index: transaction.transaction_index(), transaction_index: transaction.transaction_index(),
version: transaction.version(),
}; };
let acquisition = convert_available_fields(request, reference, fields, provider.as_str(), endpoint.as_str(), received_at); let acquisition = convert_available_fields(request, reference, fields, provider.as_str(), endpoint.as_str(), received_at);
return match acquisition { return match acquisition {
@@ -119,49 +114,22 @@ pub async fn hydrate_backfill_candidate(
}; };
} }
/// Decodes one validated Base58 signature to exactly 64 canonical bytes without a Solana SDK dependency. /// Decodes one validated Base58 signature through the source-neutral common parser while preserving Backfill error semantics.
pub(crate) fn decode_backfill_signature(signature: &crate::BackfillSignature) -> ksp_core_lib::Result<ksp_store_lib::RawTransactionSignature> { pub(crate) fn decode_backfill_signature(signature: &crate::BackfillSignature) -> ksp_core_lib::Result<ksp_store_lib::RawTransactionSignature> {
let text = signature.as_str().as_bytes(); let parsed = ksp_raw_transaction_lib::parse_raw_transaction_signature(signature.as_str());
let mut decoded = [0_u8; 64]; return match parsed {
let mut leading_zeroes = 0_usize; std::result::Result::Ok(signature) => std::result::Result::Ok(signature),
for byte in text { std::result::Result::Err(_) => std::result::Result::Err(conversion_error("signature")),
if *byte != b'1' {
break;
}
leading_zeroes += 1;
}
for byte in text {
let digit = match base58_digit(*byte) {
std::option::Option::Some(digit) => digit,
std::option::Option::None => return std::result::Result::Err(conversion_error("signature")),
};
let mut carry = u32::from(digit);
for output in decoded.iter_mut().rev() {
let value = (u32::from(*output) * 58) + carry;
*output = (value & 0xff) as u8;
carry = value >> 8;
}
if carry != 0 {
return std::result::Result::Err(conversion_error("signature"));
}
}
let significant_len = match decoded.iter().position(|byte| return *byte != 0) {
std::option::Option::Some(index) => decoded.len() - index,
std::option::Option::None => 0,
}; };
if leading_zeroes + significant_len != decoded.len() {
return std::result::Result::Err(conversion_error("signature"));
}
return std::result::Result::Ok(ksp_store_lib::RawTransactionSignature::new(decoded));
} }
struct CanonicalTransactionFields<'a> { struct CanonicalTransactionFields<'a> {
slot: u64,
block_time: std::option::Option<i64>, block_time: std::option::Option<i64>,
transaction: &'a ksp_onchain_transport_lib::SolanaEncodedTransaction,
meta: &'a ksp_onchain_transport_lib::SolanaWireField<serde_json::Value>, meta: &'a ksp_onchain_transport_lib::SolanaWireField<serde_json::Value>,
version: &'a ksp_onchain_transport_lib::SolanaWireField<ksp_onchain_transport_lib::SolanaTransactionVersion>, slot: u64,
transaction: &'a ksp_onchain_transport_lib::SolanaEncodedTransaction,
transaction_index: &'a ksp_onchain_transport_lib::SolanaWireField<u32>, transaction_index: &'a ksp_onchain_transport_lib::SolanaWireField<u32>,
version: &'a ksp_onchain_transport_lib::SolanaWireField<ksp_onchain_transport_lib::SolanaTransactionVersion>,
} }
fn canonical_reference(request: &crate::BackfillRequest, candidate: &crate::BackfillCandidate) -> ksp_core_lib::Result<ksp_store_lib::RawTransactionReference> { fn canonical_reference(request: &crate::BackfillRequest, candidate: &crate::BackfillCandidate) -> ksp_core_lib::Result<ksp_store_lib::RawTransactionReference> {
@@ -184,30 +152,9 @@ fn convert_available_fields(
endpoint: &str, endpoint: &str,
received_at: ksp_store_lib::RawTimestamp, received_at: ksp_store_lib::RawTimestamp,
) -> ksp_core_lib::Result<crate::BackfillRawAcquisition> { ) -> ksp_core_lib::Result<crate::BackfillRawAcquisition> {
let block_time = convert_block_time(fields.block_time); let transaction = canonical_transaction(&reference, fields);
let block_time = match block_time { let transaction = match transaction {
std::result::Result::Ok(block_time) => block_time, std::result::Result::Ok(transaction) => transaction,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let bytes = canonical_payload_bytes(&fields);
let bytes = match bytes {
std::result::Result::Ok(bytes) => bytes,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let hash: [u8; 32] = sha2::Sha256::digest(bytes.as_slice()).into();
let format_id = ksp_store_lib::RawFormatId::new(crate::RAW_TRANSACTION_FORMAT_ID);
let format_id = match format_id {
std::result::Result::Ok(format_id) => format_id,
std::result::Result::Err(_) => return std::result::Result::Err(conversion_error("payload.format_id")),
};
let payload = ksp_store_lib::RawPayload::try_new(
format_id,
crate::RAW_TRANSACTION_FORMAT_VERSION,
bytes.into_boxed_slice(),
ksp_store_lib::RawContentHash::new(hash),
);
let payload = match payload {
std::result::Result::Ok(payload) => payload,
std::result::Result::Err(error) => return std::result::Result::Err(error), std::result::Result::Err(error) => return std::result::Result::Err(error),
}; };
let provenance = build_provenance(request, provider, endpoint, received_at); let provenance = build_provenance(request, provider, endpoint, received_at);
@@ -215,182 +162,60 @@ fn convert_available_fields(
std::result::Result::Ok(provenance) => provenance, std::result::Result::Ok(provenance) => provenance,
std::result::Result::Err(error) => return std::result::Result::Err(error), std::result::Result::Err(error) => return std::result::Result::Err(error),
}; };
let observation_key = observation_key(request, &reference, provider, endpoint); let observation_key = observation_key(request, transaction.reference(), provider, endpoint);
let transaction = ksp_store_lib::RawTransaction::new(reference.clone(), fields.slot, block_time, payload); let acquisition = ksp_raw_transaction_lib::assemble_raw_transaction_acquisition(transaction, observation_key, provenance);
let observation = ksp_store_lib::RawTransactionObservation::new(observation_key, reference, provenance); return std::result::Result::Ok(crate::BackfillRawAcquisition { inner: Box::new(BackfillRawAcquisitionInner { acquisition }) });
return std::result::Result::Ok(crate::BackfillRawAcquisition { inner: Box::new(BackfillRawAcquisitionInner { transaction, observation }) });
} }
fn convert_block_time(value: std::option::Option<i64>) -> ksp_core_lib::Result<std::option::Option<ksp_store_lib::RawTimestamp>> { fn canonical_transaction(
let seconds = match value { reference: &ksp_store_lib::RawTransactionReference,
std::option::Option::Some(seconds) => seconds, fields: CanonicalTransactionFields<'_>,
std::option::Option::None => return std::result::Result::Ok(std::option::Option::None), ) -> ksp_core_lib::Result<ksp_store_lib::RawTransaction> {
}; let transaction_data = match fields.transaction {
let seconds = match u64::try_from(seconds) { ksp_onchain_transport_lib::SolanaEncodedTransaction::Binary { data, encoding }
std::result::Result::Ok(seconds) => seconds, if *encoding == ksp_onchain_transport_lib::SolanaTransactionBinaryEncoding::Base64 =>
std::result::Result::Err(_) => return std::result::Result::Err(conversion_error("block_time")), {
}; data.clone()
let millis = match seconds.checked_mul(1_000) {
std::option::Option::Some(millis) => millis,
std::option::Option::None => return std::result::Result::Err(conversion_error("block_time")),
};
let timestamp = ksp_store_lib::RawTimestamp::from_unix_millis(millis);
return match timestamp {
std::result::Result::Ok(timestamp) => std::result::Result::Ok(std::option::Option::Some(timestamp)),
std::result::Result::Err(_) => std::result::Result::Err(conversion_error("block_time")),
};
}
fn canonical_payload_bytes(fields: &CanonicalTransactionFields<'_>) -> ksp_core_lib::Result<std::vec::Vec<u8>> {
let (transaction_data, transaction_encoding) = match fields.transaction {
ksp_onchain_transport_lib::SolanaEncodedTransaction::Binary { data, encoding } => {
if *encoding != ksp_onchain_transport_lib::SolanaTransactionBinaryEncoding::Base64 {
return std::result::Result::Err(conversion_error("transaction.encoding"));
}
(data.as_str(), "base64")
}, },
ksp_onchain_transport_lib::SolanaEncodedTransaction::LegacyBinary(_) | ksp_onchain_transport_lib::SolanaEncodedTransaction::Json(_) => { ksp_onchain_transport_lib::SolanaEncodedTransaction::Binary { .. }
| ksp_onchain_transport_lib::SolanaEncodedTransaction::LegacyBinary(_)
| ksp_onchain_transport_lib::SolanaEncodedTransaction::Json(_) => {
return std::result::Result::Err(conversion_error("transaction.encoding")); return std::result::Result::Err(conversion_error("transaction.encoding"));
}, },
}; };
let mut bytes = std::vec::Vec::new(); let meta = map_wire_field(fields.meta, |value| return value.clone());
bytes.extend_from_slice(b"{\"transaction\":["); let version = map_wire_field(fields.version, |value| {
if let std::result::Result::Err(error) = append_json_string(&mut bytes, transaction_data) {
return std::result::Result::Err(error);
}
bytes.push(b',');
if let std::result::Result::Err(error) = append_json_string(&mut bytes, transaction_encoding) {
return std::result::Result::Err(error);
}
bytes.push(b']');
if let std::result::Result::Err(error) = append_wire_value(&mut bytes, "meta", fields.meta, |output, value| return append_canonical_json(output, value)) {
return std::result::Result::Err(error);
}
let version_result = append_wire_value(&mut bytes, "version", fields.version, |output, value| {
return match value { return match value {
ksp_onchain_transport_lib::SolanaTransactionVersion::Legacy => append_json_string(output, "legacy"), ksp_onchain_transport_lib::SolanaTransactionVersion::Legacy => ksp_raw_transaction_lib::RawTransactionVersion::Legacy,
ksp_onchain_transport_lib::SolanaTransactionVersion::Number(number) => { ksp_onchain_transport_lib::SolanaTransactionVersion::Number(number) => ksp_raw_transaction_lib::RawTransactionVersion::Number(*number),
output.extend_from_slice(number.to_string().as_bytes());
std::result::Result::Ok(())
},
}; };
}); });
if let std::result::Result::Err(error) = version_result { let transaction_index = map_wire_field(fields.transaction_index, |value| return *value);
return std::result::Result::Err(error); let material = ksp_raw_transaction_lib::RawTransactionMaterial::binary_base64(
} reference.network().clone(),
let transaction_index_result = append_wire_value(&mut bytes, "transactionIndex", fields.transaction_index, |output, value| { reference.signature(),
output.extend_from_slice(value.to_string().as_bytes()); fields.slot,
return std::result::Result::Ok(()); fields.block_time,
}); transaction_data,
if let std::result::Result::Err(error) = transaction_index_result { meta,
return std::result::Result::Err(error); version,
} transaction_index,
bytes.push(b'}'); );
return std::result::Result::Ok(bytes); let canonical = ksp_raw_transaction_lib::canonicalize_raw_transaction(material);
return match canonical {
std::result::Result::Ok(transaction) => std::result::Result::Ok(transaction),
std::result::Result::Err(_) => std::result::Result::Err(conversion_error("common.canonicalization")),
};
} }
fn append_wire_value<T, F>( fn map_wire_field<T, U, F>(field: &ksp_onchain_transport_lib::SolanaWireField<T>, mut map_value: F) -> ksp_raw_transaction_lib::RawTransactionWireField<U>
output: &mut std::vec::Vec<u8>,
key: &str,
field: &ksp_onchain_transport_lib::SolanaWireField<T>,
mut append_value: F,
) -> ksp_core_lib::Result<()>
where where
F: FnMut(&mut std::vec::Vec<u8>, &T) -> ksp_core_lib::Result<()>, F: FnMut(&T) -> U,
{ {
return match field { return match field {
ksp_onchain_transport_lib::SolanaWireField::Omitted => std::result::Result::Ok(()), ksp_onchain_transport_lib::SolanaWireField::Omitted => ksp_raw_transaction_lib::RawTransactionWireField::Omitted,
ksp_onchain_transport_lib::SolanaWireField::Null => { ksp_onchain_transport_lib::SolanaWireField::Null => ksp_raw_transaction_lib::RawTransactionWireField::Null,
output.push(b','); ksp_onchain_transport_lib::SolanaWireField::Value(value) => ksp_raw_transaction_lib::RawTransactionWireField::Value(map_value(value)),
let key_result = append_json_string(output, key);
if let std::result::Result::Err(error) = key_result {
return std::result::Result::Err(error);
}
output.extend_from_slice(b":null");
std::result::Result::Ok(())
},
ksp_onchain_transport_lib::SolanaWireField::Value(value) => {
output.push(b',');
let key_result = append_json_string(output, key);
if let std::result::Result::Err(error) = key_result {
return std::result::Result::Err(error);
}
output.push(b':');
append_value(output, value)
},
};
}
fn append_canonical_json(output: &mut std::vec::Vec<u8>, value: &serde_json::Value) -> ksp_core_lib::Result<()> {
return match value {
serde_json::Value::Null => {
output.extend_from_slice(b"null");
std::result::Result::Ok(())
},
serde_json::Value::Bool(value) => {
if *value {
output.extend_from_slice(b"true");
} else {
output.extend_from_slice(b"false");
}
std::result::Result::Ok(())
},
serde_json::Value::Number(value) => {
output.extend_from_slice(value.to_string().as_bytes());
std::result::Result::Ok(())
},
serde_json::Value::String(value) => append_json_string(output, value.as_str()),
serde_json::Value::Array(values) => {
output.push(b'[');
for (index, item) in values.iter().enumerate() {
if index != 0 {
output.push(b',');
}
let item_result = append_canonical_json(output, item);
if let std::result::Result::Err(error) = item_result {
return std::result::Result::Err(error);
}
}
output.push(b']');
std::result::Result::Ok(())
},
serde_json::Value::Object(values) => {
output.push(b'{');
let mut keys = values.keys().collect::<std::vec::Vec<_>>();
keys.sort_unstable();
for (index, key) in keys.iter().enumerate() {
if index != 0 {
output.push(b',');
}
let key_result = append_json_string(output, key.as_str());
if let std::result::Result::Err(error) = key_result {
return std::result::Result::Err(error);
}
output.push(b':');
let item = values.get(key.as_str());
let item = match item {
std::option::Option::Some(item) => item,
std::option::Option::None => return std::result::Result::Err(conversion_error("payload.meta")),
};
let item_result = append_canonical_json(output, item);
if let std::result::Result::Err(error) = item_result {
return std::result::Result::Err(error);
}
}
output.push(b'}');
std::result::Result::Ok(())
},
};
}
fn append_json_string(output: &mut std::vec::Vec<u8>, value: &str) -> ksp_core_lib::Result<()> {
let encoded = serde_json::to_vec(value);
return match encoded {
std::result::Result::Ok(encoded) => {
output.extend_from_slice(encoded.as_slice());
std::result::Result::Ok(())
},
std::result::Result::Err(_) => std::result::Result::Err(conversion_error("payload.json")),
}; };
} }
@@ -463,18 +288,6 @@ fn hash_bytes(hasher: &mut sha2::Sha256, value: &[u8]) {
hasher.update(value); hasher.update(value);
} }
fn base58_digit(byte: u8) -> std::option::Option<u8> {
return match byte {
b'1'..=b'9' => std::option::Option::Some(byte - b'1'),
b'A'..=b'H' => std::option::Option::Some((byte - b'A') + 9),
b'J'..=b'N' => std::option::Option::Some((byte - b'J') + 17),
b'P'..=b'Z' => std::option::Option::Some((byte - b'P') + 22),
b'a'..=b'k' => std::option::Option::Some((byte - b'a') + 33),
b'm'..=b'z' => std::option::Option::Some((byte - b'm') + 44),
_ => std::option::Option::None,
};
}
fn conversion_error(field: &'static str) -> ksp_core_lib::Error { fn conversion_error(field: &'static str) -> ksp_core_lib::Error {
return ksp_core_lib::Error::new(crate::ERROR_CODE_BACKFILL_RAW_CONVERSION_INVALID, "invalid deterministic Backfill RAW conversion") return ksp_core_lib::Error::new(crate::ERROR_CODE_BACKFILL_RAW_CONVERSION_INVALID, "invalid deterministic Backfill RAW conversion")
.with_context("field", field); .with_context("field", field);

View File

@@ -1,5 +1,5 @@
// file: crates/ksp-job-backfill-lib/src/lib.rs // file: crates/ksp-job-backfill-lib/src/lib.rs
// version: 6 // version: 7
#![warn(missing_docs)] #![warn(missing_docs)]
#![deny(unreachable_pub)] #![deny(unreachable_pub)]
@@ -30,10 +30,6 @@ pub use self::checkpoint::BackfillCheckpoint;
pub use self::conversion::BackfillHydrationOutcome; pub use self::conversion::BackfillHydrationOutcome;
/// Complete in-memory RAW transaction acquisition ready for later Store persistence. /// Complete in-memory RAW transaction acquisition ready for later Store persistence.
pub use self::conversion::BackfillRawAcquisition; pub use self::conversion::BackfillRawAcquisition;
/// KSP-owned source-independent RAW transaction format identifier produced by this Backfill vertical.
pub use self::conversion::RAW_TRANSACTION_FORMAT_ID;
/// Initial KSP-owned RAW transaction format version produced by this Backfill vertical.
pub use self::conversion::RAW_TRANSACTION_FORMAT_VERSION;
/// Hydrates one candidate through observed Transport and converts a non-null response to canonical RAW v1. /// Hydrates one candidate through observed Transport and converts a non-null response to canonical RAW v1.
pub use self::conversion::hydrate_backfill_candidate; pub use self::conversion::hydrate_backfill_candidate;
/// One deterministic transaction candidate produced by bounded discovery. /// One deterministic transaction candidate produced by bounded discovery.
@@ -112,6 +108,10 @@ pub use self::runtime::BackfillJobRuntime;
pub use self::runtime::BackfillJobSnapshot; pub use self::runtime::BackfillJobSnapshot;
/// Cloneable runtime-neutral-facing latest-value source for concrete Backfill snapshots. /// Cloneable runtime-neutral-facing latest-value source for concrete Backfill snapshots.
pub use self::runtime::BackfillSnapshotSource; pub use self::runtime::BackfillSnapshotSource;
/// KSP-owned source-independent RAW transaction format identifier re-exported from the common RAW layer.
pub use ksp_raw_transaction_lib::RAW_TRANSACTION_FORMAT_ID;
/// Frozen RAW transaction format version re-exported from the common RAW layer.
pub use ksp_raw_transaction_lib::RAW_TRANSACTION_FORMAT_VERSION;
/// Internal contiguous completion frontier used by bounded execution. /// Internal contiguous completion frontier used by bounded execution.
pub(crate) use self::checkpoint::CompletionFrontier; pub(crate) use self::checkpoint::CompletionFrontier;

View File

@@ -1,5 +1,5 @@
// file: crates/ksp-job-backfill-lib/src/request.rs // file: crates/ksp-job-backfill-lib/src/request.rs
// version: 6 // version: 7
use sha2::Digest; // rust-rules: trait-import use sha2::Digest; // rust-rules: trait-import
@@ -12,9 +12,9 @@ pub const MAX_BACKFILL_PAGES: usize = 10_000;
/// Maximum page size admitted for one `getSignaturesForAddress` request. /// Maximum page size admitted for one `getSignaturesForAddress` request.
pub const MAX_BACKFILL_PAGE_SIZE: usize = 1_000; pub const MAX_BACKFILL_PAGE_SIZE: usize = 1_000;
/// Maximum Base58 text length possible for one canonical 64-byte Solana signature. /// Maximum Base58 text length possible for one canonical 64-byte Solana signature.
pub const MAX_BACKFILL_SIGNATURE_TEXT_BYTES: usize = 88; pub const MAX_BACKFILL_SIGNATURE_TEXT_BYTES: usize = ksp_raw_transaction_lib::MAX_RAW_TRANSACTION_SIGNATURE_TEXT_BYTES;
/// Minimum Base58 text length possible for one canonical 64-byte Solana signature. /// Minimum Base58 text length possible for one canonical 64-byte Solana signature.
pub const MIN_BACKFILL_SIGNATURE_TEXT_BYTES: usize = 64; pub const MIN_BACKFILL_SIGNATURE_TEXT_BYTES: usize = ksp_raw_transaction_lib::MIN_RAW_TRANSACTION_SIGNATURE_TEXT_BYTES;
/// Commitment levels intentionally admitted by the historical Backfill vertical. /// Commitment levels intentionally admitted by the historical Backfill vertical.
#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)] #[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)]

View File

@@ -1,10 +1,10 @@
// file: crates/ksp-job-backfill-lib/tests/dependency_boundary.rs // file: crates/ksp-job-backfill-lib/tests/dependency_boundary.rs
// version: 6 // version: 7
//! Dependency firewall canaries through the concrete cancellation and latest-value runtime tranche. //! Dependency firewall canaries through the concrete cancellation and latest-value runtime tranche.
#[test] #[test]
fn pre_009_manifest_uses_only_planned_ksp_edges_and_private_tokio_runtime() { fn pre_003_manifest_adds_common_raw_edge_without_crossing_store_or_runtime_boundaries() {
let manifest = include_str!("../Cargo.toml"); let manifest = include_str!("../Cargo.toml");
for required in [ for required in [
"futures-util = { workspace = true, features = [\"std\"] }", "futures-util = { workspace = true, features = [\"std\"] }",
@@ -12,6 +12,7 @@ fn pre_009_manifest_uses_only_planned_ksp_edges_and_private_tokio_runtime() {
"ksp-job-api = { path = \"../ksp-job-api\" }", "ksp-job-api = { path = \"../ksp-job-api\" }",
"ksp-logging-lib = { path = \"../ksp-logging-lib\" }", "ksp-logging-lib = { path = \"../ksp-logging-lib\" }",
"ksp-onchain-transport-lib = { path = \"../ksp-onchain-transport-lib\" }", "ksp-onchain-transport-lib = { path = \"../ksp-onchain-transport-lib\" }",
"ksp-raw-transaction-lib = { path = \"../ksp-raw-transaction-lib\" }",
"ksp-store-lib = { path = \"../ksp-store-lib\", default-features = false }", "ksp-store-lib = { path = \"../ksp-store-lib\", default-features = false }",
"serde_json.workspace = true", "serde_json.workspace = true",
"sha2.workspace = true", "sha2.workspace = true",
@@ -56,7 +57,12 @@ fn pre_009_production_sources_keep_transport_store_and_scheduler_ownership_separ
} }
} }
let conversion = include_str!("../src/conversion.rs"); let conversion = include_str!("../src/conversion.rs");
assert!(conversion.contains("serde_json::")); assert!(conversion.contains("ksp_raw_transaction_lib::canonicalize_raw_transaction"));
assert!(conversion.contains("ksp_raw_transaction_lib::assemble_raw_transaction_acquisition"));
assert!(conversion.contains("ksp_raw_transaction_lib::parse_raw_transaction_signature"));
assert!(!conversion.contains("fn append_canonical_json"));
assert!(!conversion.contains("fn base58_digit"));
assert!(!conversion.contains("RawPayload::try_new"));
assert!(conversion.contains("get_transaction_observed")); assert!(conversion.contains("get_transaction_observed"));
assert!(conversion.contains("SolanaTransactionEncoding::Base64")); assert!(conversion.contains("SolanaTransactionEncoding::Base64"));
assert!(conversion.contains("std::option::Option::Some(0)")); assert!(conversion.contains("std::option::Option::Some(0)"));

View File

@@ -1,5 +1,5 @@
// file: crates/ksp-job-backfill-lib/tests/hardening.rs // file: crates/ksp-job-backfill-lib/tests/hardening.rs
// version: 4 // version: 5
//! Adversarial, security, visibility and external-boundary hardening canaries for `pre.010`. //! Adversarial, security, visibility and external-boundary hardening canaries for `pre.010`.
@@ -268,6 +268,7 @@ fn pre_010_manifest_dependency_surface_is_exact_and_backend_neutral() {
"ksp-job-api", "ksp-job-api",
"ksp-logging-lib", "ksp-logging-lib",
"ksp-onchain-transport-lib", "ksp-onchain-transport-lib",
"ksp-raw-transaction-lib",
"ksp-store-lib", "ksp-store-lib",
"serde_json.workspace", "serde_json.workspace",
"sha2.workspace", "sha2.workspace",
@@ -328,6 +329,7 @@ fn pre_010_lower_layers_have_no_dependency_return_to_job() {
include_str!("../../ksp-core-lib/Cargo.toml"), include_str!("../../ksp-core-lib/Cargo.toml"),
include_str!("../../ksp-logging-lib/Cargo.toml"), include_str!("../../ksp-logging-lib/Cargo.toml"),
include_str!("../../ksp-onchain-transport-lib/Cargo.toml"), include_str!("../../ksp-onchain-transport-lib/Cargo.toml"),
include_str!("../../ksp-raw-transaction-lib/Cargo.toml"),
include_str!("../../ksp-store-api/Cargo.toml"), include_str!("../../ksp-store-api/Cargo.toml"),
include_str!("../../ksp-store-lib/Cargo.toml"), include_str!("../../ksp-store-lib/Cargo.toml"),
include_str!("../../ksp-store-postgres-lib/Cargo.toml"), include_str!("../../ksp-store-postgres-lib/Cargo.toml"),

View File

@@ -1,5 +1,5 @@
// file: crates/ksp-job-backfill-lib/unit_tests/conversion.rs // file: crates/ksp-job-backfill-lib/unit_tests/conversion.rs
// version: 2 // version: 3
fn signature_text() -> std::option::Option<crate::BackfillSignature> { fn signature_text() -> std::option::Option<crate::BackfillSignature> {
return match crate::BackfillSignature::new("1".repeat(64)) { return match crate::BackfillSignature::new("1".repeat(64)) {
@@ -46,6 +46,14 @@ fn candidate(network: &str, signature: crate::BackfillSignature) -> std::option:
return std::option::Option::Some(crate::BackfillCandidate::new(crate::BackfillCandidateIdentity::new(network, signature), std::option::Option::Some(42))); return std::option::Option::Some(crate::BackfillCandidate::new(crate::BackfillCandidateIdentity::new(network, signature), std::option::Option::Some(42)));
} }
fn raw_reference(network: &str) -> std::option::Option<ksp_store_lib::RawTransactionReference> {
let network = match ksp_store_lib::RawNetworkId::new(network) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return std::option::Option::None,
};
return std::option::Option::Some(ksp_store_lib::RawTransactionReference::new(network, ksp_store_lib::RawTransactionSignature::new([0_u8; 64])));
}
fn received_at() -> std::option::Option<ksp_store_lib::RawTimestamp> { fn received_at() -> std::option::Option<ksp_store_lib::RawTimestamp> {
return match ksp_store_lib::RawTimestamp::from_unix_millis(1_700_000_001_000) { return match ksp_store_lib::RawTimestamp::from_unix_millis(1_700_000_001_000) {
std::result::Result::Ok(value) => std::option::Option::Some(value), std::result::Result::Ok(value) => std::option::Option::Some(value),
@@ -169,21 +177,26 @@ fn pre_006_wire_omission_and_null_produce_distinct_canonical_bytes() {
let version_null = ksp_onchain_transport_lib::SolanaWireField::Null; let version_null = ksp_onchain_transport_lib::SolanaWireField::Null;
let transaction_index_omitted = ksp_onchain_transport_lib::SolanaWireField::Omitted; let transaction_index_omitted = ksp_onchain_transport_lib::SolanaWireField::Omitted;
let transaction_index_null = ksp_onchain_transport_lib::SolanaWireField::Null; let transaction_index_null = ksp_onchain_transport_lib::SolanaWireField::Null;
let omitted = super::canonical_payload_bytes(&fields(&transaction, &meta_omitted, &version_omitted, &transaction_index_omitted, std::option::Option::None)); let reference = match raw_reference("devnet") {
std::option::Option::Some(value) => value,
std::option::Option::None => return,
};
let omitted =
super::canonical_transaction(&reference, fields(&transaction, &meta_omitted, &version_omitted, &transaction_index_omitted, std::option::Option::None));
assert!(omitted.is_ok()); assert!(omitted.is_ok());
let omitted = match omitted { let omitted = match omitted {
std::result::Result::Ok(value) => value, std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return, std::result::Result::Err(_) => return,
}; };
assert_eq!(omitted, b"{\"transaction\":[\"AQID\",\"base64\"]}"); assert_eq!(omitted.payload().bytes(), b"{\"transaction\":[\"AQID\",\"base64\"]}");
let nulls = super::canonical_payload_bytes(&fields(&transaction, &meta_null, &version_null, &transaction_index_null, std::option::Option::None)); let nulls = super::canonical_transaction(&reference, fields(&transaction, &meta_null, &version_null, &transaction_index_null, std::option::Option::None));
assert!(nulls.is_ok()); assert!(nulls.is_ok());
let nulls = match nulls { let nulls = match nulls {
std::result::Result::Ok(value) => value, std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return, std::result::Result::Err(_) => return,
}; };
assert_eq!(nulls, b"{\"transaction\":[\"AQID\",\"base64\"],\"meta\":null,\"version\":null,\"transactionIndex\":null}"); assert_eq!(nulls.payload().bytes(), b"{\"transaction\":[\"AQID\",\"base64\"],\"meta\":null,\"version\":null,\"transactionIndex\":null}");
assert_ne!(omitted, nulls); assert_ne!(omitted.payload().bytes(), nulls.payload().bytes());
return; return;
} }
@@ -198,8 +211,12 @@ fn pre_006_non_base64_transaction_shapes_are_rejected() {
let meta = ksp_onchain_transport_lib::SolanaWireField::Omitted; let meta = ksp_onchain_transport_lib::SolanaWireField::Omitted;
let version = ksp_onchain_transport_lib::SolanaWireField::Omitted; let version = ksp_onchain_transport_lib::SolanaWireField::Omitted;
let transaction_index = ksp_onchain_transport_lib::SolanaWireField::Omitted; let transaction_index = ksp_onchain_transport_lib::SolanaWireField::Omitted;
let reference = match raw_reference("devnet") {
std::option::Option::Some(value) => value,
std::option::Option::None => return,
};
for transaction in [&base58, &legacy, &json] { for transaction in [&base58, &legacy, &json] {
let result = super::canonical_payload_bytes(&fields(transaction, &meta, &version, &transaction_index, std::option::Option::None)); let result = super::canonical_transaction(&reference, fields(transaction, &meta, &version, &transaction_index, std::option::Option::None));
assert!(result.is_err()); assert!(result.is_err());
if let std::result::Result::Err(error) = result { if let std::result::Result::Err(error) = result {
assert_eq!(error.code(), crate::ERROR_CODE_BACKFILL_RAW_CONVERSION_INVALID); assert_eq!(error.code(), crate::ERROR_CODE_BACKFILL_RAW_CONVERSION_INVALID);
@@ -210,12 +227,26 @@ fn pre_006_non_base64_transaction_shapes_are_rejected() {
#[test] #[test]
fn pre_006_negative_and_unrepresentable_block_times_are_terminal_conversion_errors() { fn pre_006_negative_and_unrepresentable_block_times_are_terminal_conversion_errors() {
let negative = super::convert_block_time(std::option::Option::Some(-1)); let reference = match raw_reference("devnet") {
std::option::Option::Some(value) => value,
std::option::Option::None => return,
};
let transaction = ksp_onchain_transport_lib::SolanaEncodedTransaction::Binary {
data: "AQID".to_owned(),
encoding: ksp_onchain_transport_lib::SolanaTransactionBinaryEncoding::Base64,
};
let meta = ksp_onchain_transport_lib::SolanaWireField::Omitted;
let version = ksp_onchain_transport_lib::SolanaWireField::Omitted;
let transaction_index = ksp_onchain_transport_lib::SolanaWireField::Omitted;
let negative = super::canonical_transaction(&reference, fields(&transaction, &meta, &version, &transaction_index, std::option::Option::Some(-1)));
assert!(negative.is_err()); assert!(negative.is_err());
let oversized = super::convert_block_time(std::option::Option::Some(i64::MAX)); let oversized = super::canonical_transaction(&reference, fields(&transaction, &meta, &version, &transaction_index, std::option::Option::Some(i64::MAX)));
assert!(oversized.is_err()); assert!(oversized.is_err());
let absent = super::convert_block_time(std::option::Option::None); let absent = super::canonical_transaction(&reference, fields(&transaction, &meta, &version, &transaction_index, std::option::Option::None));
assert!(matches!(absent, std::result::Result::Ok(std::option::Option::None))); assert!(absent.is_ok());
if let std::result::Result::Ok(absent) = absent {
assert!(absent.block_time().is_none());
}
return; return;
} }
@@ -242,6 +273,12 @@ fn pre_006_observation_key_is_deterministic_and_endpoint_specific() {
let other_endpoint = super::observation_key(&request, &reference, "provider", "endpoint-b"); let other_endpoint = super::observation_key(&request, &reference, "provider", "endpoint-b");
let other_provider = super::observation_key(&request, &reference, "provider-2", "endpoint-a"); let other_provider = super::observation_key(&request, &reference, "provider-2", "endpoint-a");
assert_eq!(first, same); assert_eq!(first, same);
assert_eq!(
first.as_bytes(),
&[
184, 85, 15, 15, 33, 101, 243, 112, 222, 145, 139, 212, 251, 14, 199, 130, 57, 26, 253, 221, 184, 140, 87, 240, 16, 116, 62, 0, 11, 19, 103, 2
]
);
assert_ne!(first, other_endpoint); assert_ne!(first, other_endpoint);
assert_ne!(first, other_provider); assert_ne!(first, other_provider);
return; return;
@@ -294,6 +331,15 @@ fn pre_006_missing_outcome_contains_only_network_scoped_reference() {
return; return;
} }
#[test]
fn pre_003_backfill_uses_common_raw_v1_contract_without_changing_frozen_identity() {
assert_eq!(crate::RAW_TRANSACTION_FORMAT_ID, ksp_raw_transaction_lib::RAW_TRANSACTION_FORMAT_ID);
assert_eq!(crate::RAW_TRANSACTION_FORMAT_VERSION, ksp_raw_transaction_lib::RAW_TRANSACTION_FORMAT_VERSION);
assert_eq!(crate::MIN_BACKFILL_SIGNATURE_TEXT_BYTES, ksp_raw_transaction_lib::MIN_RAW_TRANSACTION_SIGNATURE_TEXT_BYTES);
assert_eq!(crate::MAX_BACKFILL_SIGNATURE_TEXT_BYTES, ksp_raw_transaction_lib::MAX_RAW_TRANSACTION_SIGNATURE_TEXT_BYTES);
return;
}
#[test] #[test]
fn pre_006_fix_001_raw_acquisition_uses_one_private_indirection() { fn pre_006_fix_001_raw_acquisition_uses_one_private_indirection() {
assert_eq!(std::mem::size_of::<crate::BackfillRawAcquisition>(), std::mem::size_of::<usize>(),); assert_eq!(std::mem::size_of::<crate::BackfillRawAcquisition>(), std::mem::size_of::<usize>(),);

236
deltas/0.3.10/pre.003.md Normal file
View File

@@ -0,0 +1,236 @@
<!-- file: deltas/0.3.10/pre.003.md -->
<!-- version: 1 -->
# Delta `0.3.10-pre.003` — migration du Backfill vers la common RAW Transaction
## Base requise
```text
0.3.10-pre.002-fix.001
workspace.package.version = 0.3.10-pre.2.fix.1
```
Le gate opérateur communiqué le 2026-09-06 ferme intégralement la base : audits statiques propres, `cargo check --workspace` PASS, Clippy workspace `--all-targets --all-features -- -D warnings` PASS, `cargo test -p ksp-raw-transaction-lib` PASS avec 9 tests unitaires et 8 tests dintégration, arbres normal/features conformes.
## Objectif
Migrer uniquement `ksp-job-backfill-lib` vers `ksp-raw-transaction-lib` sans modifier le comportement de campagne : common devient propriétaire du parser Base58, de RAW v1, du hash payload et de lassemblage transaction/observation ; Backfill conserve Transport, provenance, observation key, Job scope et Store persistence.
Aucun Worker, aucune nouvelle source Transport et aucune extension du format RAW ne sont introduits.
## Version
```text
workspace.package.version = 0.3.10-pre.3
```
Le header du `Cargo.toml` racine passe de `492` à `493`. Le nombre de membres workspace reste `20`.
## Migration de dépendance
`ksp-job-backfill-lib` ajoute :
```text
ksp-raw-transaction-lib = { path = "../ksp-raw-transaction-lib" }
```
Le reste de son graphe runtime est conservé. `serde_json` reste nécessaire au type Transport `SolanaWireField<serde_json::Value>`. `sha2` reste nécessaire aux fingerprints et à la `RawObservationKey` Backfill ; il nest plus utilisé pour le hash du payload RAW.
## Chemin RAW après migration
```text
BackfillSignature
-> parse_raw_transaction_signature
-> RawTransactionSignature
getTransaction observed
-> adapter SolanaEncodedTransaction/SolanaWireField
-> RawTransactionMaterial::binary_base64
-> canonicalize_raw_transaction
-> RawTransaction
Backfill provenance + Backfill RawObservationKey
-> assemble_raw_transaction_acquisition
-> BackfillRawAcquisition
-> Store persistence existante
```
Les erreurs du parser/canonicalizer common sont remappées au boundary Backfill vers :
```text
ERROR_CODE_BACKFILL_RAW_CONVERSION_INVALID
```
Le Job ne change donc pas de domaine derreur externe par effet de la migration.
## Code privé supprimé du Backfill
La migration retire de `conversion.rs` les implémentations locales de :
```text
base58_digit(...)
algorithme de décodage Base58 64-byte
convert_block_time(...)
canonical_payload_bytes(...)
append_wire_value(...)
append_canonical_json(...)
append_json_string(...)
RawPayload::try_new + SHA-256 du payload
construction directe RawTransactionObservation
```
`CanonicalTransactionFields` reste un adapter Transport privé ; il ne définit aucun format RAW.
## Propriété Backfill conservée
Le Backfill continue de posséder :
```text
getTransaction observed / rôle HTTP
provider gagnant
endpoint gagnant
commitment
job_id
scope_fingerprint
RawAcquisitionOrigin::Backfill
capture_session
observation contract version = 1
algorithme RawObservationKey Backfill
persistence Store
sémantique Missing
```
La common crate ne reçoit donc aucun vocabulaire de campagne.
## Parité déterministe
Le golden historique reste inchangé :
```text
format_id = ksp.solana.raw_transaction
format_version = 1
bytes = {"transaction":["AQID","base64"],"meta":{"a":{"x":null,"y":true},"z":1},"version":"legacy","transactionIndex":7}
byte_len = 112
sha256 = 220792d2b15d262fda242cb220774ee9ddeffebf04dcfadabcf8ef76a9b1a7c3
```
La provenance historique reste :
```text
provider = publicnode
protocol = solana.http.json_rpc
method = getTransaction
endpoint = devnet-primary
commitment = finalized
capture_session = backfill:pre006
origin = Backfill
```
Le canari dobservation key est désormais exact :
```text
provider = provider
endpoint = endpoint-a
key = b8550f0f2165f370de918bd4fb0ec782391afdddb88c57f010743e000b136702
```
## Canaris mis à jour
```text
BackfillSignature délègue au parser common
bornes Backfill == bornes common
format id/version Backfill == common
wire omitted/null reste distinct via common
non-Base64 reste rejeté avec error code Backfill
block_time invalide reste terminal avec error code Backfill
golden bytes/hash/provenance inchangés
observation key exacte inchangée
manifest Backfill exige ledge common
conversion.rs exige parser + canonicalizer + assembler common
conversion.rs interdit canonical JSON/Base58/RawPayload privés
lower-layer common interdit toute dépendance retour vers Job
```
## Fichiers ajoutés
```text
deltas/0.3.10/pre.003.md
```
## Fichiers modifiés
```text
Cargo.toml
crates/ksp-job-backfill-lib/Cargo.toml
crates/ksp-job-backfill-lib/README.md
crates/ksp-job-backfill-lib/src/conversion.rs
crates/ksp-job-backfill-lib/src/lib.rs
crates/ksp-job-backfill-lib/src/request.rs
crates/ksp-job-backfill-lib/tests/dependency_boundary.rs
crates/ksp-job-backfill-lib/tests/hardening.rs
crates/ksp-job-backfill-lib/unit_tests/conversion.rs
docs/plans/031-V0_3_10_RAW_TRANSACTION_INGEST_PLAN.md
docs/validation/027-V0_3_10_RAW_TRANSACTION_INGEST.md
```
## Fichiers supprimés
Aucun.
## Validations exécutées dans lenvironnement dassemblage
```text
python3 scripts/audit_rust_workspace_rules.py
General Rust rule audit: clean
Rust export completeness audit: 0 candidate(s)
KSP workspace Rust rule audit: clean
python3 scripts/audit_markdown_tables.py README.md RULES.md ROADMAP.md CHANGELOG.md docs prompts crates deltas
Markdown table audit: clean (339 table(s), 756 file(s))
```
Lenvironnement dassemblage ne fournit ni `cargo`, ni `rustc`, ni `rustfmt`.
## Gate opérateur demandé
```bash
cargo fmt --all
python3 scripts/audit_rust_workspace_rules.py
python3 scripts/audit_markdown_tables.py README.md RULES.md ROADMAP.md CHANGELOG.md docs prompts crates deltas
cargo check --workspace
cargo clippy --workspace --all-targets --all-features -- -D warnings
cargo test -p ksp-raw-transaction-lib
cargo test -p ksp-job-backfill-lib
cargo tree -p ksp-job-backfill-lib --edges normal
cargo tree -p ksp-job-backfill-lib -e features
```
## Validations non exécutées localement
```text
cargo fmt --all
cargo check --workspace
cargo clippy --workspace --all-targets --all-features -- -D warnings
cargo test -p ksp-raw-transaction-lib
cargo test -p ksp-job-backfill-lib
cargo tree -p ksp-job-backfill-lib --edges normal
cargo tree -p ksp-job-backfill-lib -e features
smokes live : non pertinents pour cette tranche de migration déterministe
```
## Décisions prises
```text
common possède parser + canonical RAW v1 + payload hash + assembly
Backfill conserve observation key/provenance/campagne
les erreurs common sont remappées vers le code Backfill historique
les constantes publiques RAW du Backfill sont des reexports common
les bornes publiques de signature Backfill suivent les constantes common
aucun changement du Store persistence path
aucun changement de source/acquisition réseau
aucun Worker avant la tranche planifiée
```
## Questions ouvertes
Aucune question ne bloque le gate de `pre.003`. `pre.004` reste strictement dédiée à `get_block_observed` et au matériau HTTP block après validation opérateur complète de cette migration.

View File

@@ -1,5 +1,5 @@
<!-- file: docs/plans/031-V0_3_10_RAW_TRANSACTION_INGEST_PLAN.md --> <!-- file: docs/plans/031-V0_3_10_RAW_TRANSACTION_INGEST_PLAN.md -->
<!-- version: 3 --> <!-- version: 4 -->
# Plan v0.3.10 — RAW Transaction commune + Worker dingestion multi-source # Plan v0.3.10 — RAW Transaction commune + Worker dingestion multi-source
@@ -1040,11 +1040,15 @@ Présente tranche : archive/règles, common extraction, capability matrix, fraî
La matérialisation a révélé une correction normative du sizing `pre.001` : `DEP-PIPE-006` impose `ksp-store-api` au pipeline RAW et interdit `ksp-store-lib`. Le graphe et la section 5.4 sont donc corrigés dans cette tranche sans changer le contrat fonctionnel RAW. La matérialisation a révélé une correction normative du sizing `pre.001` : `DEP-PIPE-006` impose `ksp-store-api` au pipeline RAW et interdit `ksp-store-lib`. Le graphe et la section 5.4 sont donc corrigés dans cette tranche sans changer le contrat fonctionnel RAW.
La validation opérateur du 2026-09-06 confirme `cargo check --workspace`, les audits statiques et les tests de la crate, mais révèle un échec du gate Clippy `-D warnings` limité aux tests dintégration : quatre crates de test sans rustdoc de crate et un `clippy::collapsible_if`. `pre.002-fix.001` corrige uniquement ces canaris, sans modifier le contrat RAW ni commencer `pre.003`; le signal Cargo devient `0.3.10-pre.2.fix.1`. La validation opérateur du 2026-09-06 a dabord révélé un échec Clippy limité aux tests dintégration. `pre.002-fix.001` a corrigé ces canaris sans toucher au contrat RAW. Le rejeu opérateur suivant est intégralement vert : audits, `cargo check --workspace`, Clippy `--all-targets --all-features -- -D warnings`, tests de `ksp-raw-transaction-lib`, arbre normal et arbre de features. La fondation common est donc fermée.
### `pre.003` — migration Backfill vers common ### `pre.003` — migration Backfill vers common
Remplacer la canonicalisation privée du Backfill par common, conserver son observation key/provenance/campagne et prouver zéro changement bytes/hash/comportement de campagne. **Statut : réalisé.**
La canonicalisation privée du Backfill est supprimée au profit de `ksp-raw-transaction-lib`. Le Job conserve uniquement ladaptation Transport vers `RawTransactionMaterial`, le remapping de ses erreurs publiques, la provenance/campagne et sa `RawObservationKey` producer-owned. Le parser Base58 common, `canonicalize_raw_transaction(...)` et `assemble_raw_transaction_acquisition(...)` sont désormais les chemins uniques pour la signature, RAW v1 et lassemblage.
Les invariants historiques restent gelés : `ksp.solana.raw_transaction` v1, golden `112` octets, SHA-256 `220792d2b15d262fda242cb220774ee9ddeffebf04dcfadabcf8ef76a9b1a7c3`, origin `Backfill`, provider/endpoint/commitment/capture-session, observation key de campagne et sémantique `Missing`. Les erreurs common sont remappées sur `ERROR_CODE_BACKFILL_RAW_CONVERSION_INVALID` afin de ne pas modifier le contrat externe du Job.
### `pre.004` — HTTP observed block + block material ### `pre.004` — HTTP observed block + block material

View File

@@ -1,5 +1,5 @@
<!-- file: docs/validation/027-V0_3_10_RAW_TRANSACTION_INGEST.md --> <!-- file: docs/validation/027-V0_3_10_RAW_TRANSACTION_INGEST.md -->
<!-- version: 3 --> <!-- version: 4 -->
# Validation v0.3.10 — RAW Transaction commune + Worker dingestion # Validation v0.3.10 — RAW Transaction commune + Worker dingestion
@@ -513,3 +513,151 @@ workspace.package.version = 0.3.10-pre.2.fix.1
Le gate Clippy complet doit être rejoué après application du fix avant ouverture de `pre.003`. Le gate Clippy complet doit être rejoué après application du fix avant ouverture de `pre.003`.
## 12. `pre.002-fix.001` fermé et `pre.003` — migration Backfill vers common
### 12.1 Fermeture opérateur de `pre.002-fix.001`
Le rejeu communiqué le 2026-09-06 ferme le fix précédent :
```text
cargo fmt --all : exécuté
General Rust rule audit: clean
Rust export completeness audit: 0 candidate(s)
KSP workspace Rust rule audit: clean
Markdown table audit: clean (339 table(s), 755 file(s))
cargo check --workspace : PASS
cargo clippy --workspace --all-targets --all-features -- -D warnings : PASS
cargo test -p ksp-raw-transaction-lib : PASS
```
Détail des tests common communiqué :
```text
9 unitaires : PASS
8 intégration : PASS
0 échec
0 doctest
```
Larbre normal confirme uniquement `ksp-core-lib`, `ksp-store-api`, `serde_json` et `sha2` côté common. Larbre de features ne révèle aucune feature KSP/runtime inattendue. `pre.003` peut donc être ouverte.
### 12.2 Migration réalisée dans `pre.003`
`ksp-job-backfill-lib` dépend désormais explicitement de `ksp-raw-transaction-lib`. La conversion Backfill ne possède plus :
```text
algorithme Base58 64-byte
canonical JSON récursif
construction RawPayload
SHA-256 du payload RAW
conversion block_time RAW
assemblage direct RawTransaction + RawTransactionObservation
```
Le chemin productif devient :
```text
BackfillSignature
-> ksp-raw-transaction-lib::parse_raw_transaction_signature
SolanaConfirmedTransaction
-> adapter Transport local
-> RawTransactionMaterial::binary_base64
-> ksp-raw-transaction-lib::canonicalize_raw_transaction
Backfill provenance + Backfill observation key
-> ksp-raw-transaction-lib::assemble_raw_transaction_acquisition
-> Store facade
```
Le Backfill conserve volontairement :
```text
getTransaction observed + rôle HTTP
commitment/campagne/job_id/scope_fingerprint
provider + endpoint réel gagnant
RawAcquisitionOrigin::Backfill
capture_session = JobId
algorithme de RawObservationKey Backfill v1
remapping des erreurs common vers ERROR_CODE_BACKFILL_RAW_CONVERSION_INVALID
persistence Store et sémantique Missing
```
### 12.3 Parité gelée
Les canaris Backfill continuent dexiger :
```text
RAW_TRANSACTION_FORMAT_ID = ksp.solana.raw_transaction
RAW_TRANSACTION_FORMAT_VERSION = 1
byte_len golden = 112
sha256 golden = 220792d2b15d262fda242cb220774ee9ddeffebf04dcfadabcf8ef76a9b1a7c3
block_time golden = 1700000000000 ms
network = devnet
signature = 64 octets zéro pour le vecteur de référence
provider = publicnode
protocol = solana.http.json_rpc
method = getTransaction
endpoint = devnet-primary
commitment = finalized
capture_session = backfill:pre006
```
Lobservation key Backfill du vecteur `provider` / `endpoint-a` est désormais verrouillée byte-for-byte :
```text
b8550f0f2165f370de918bd4fb0ec782391afdddb88c57f010743e000b136702
```
Ce canari prouve que la migration common ne déplace pas lidempotence producer-owned vers la lower layer.
### 12.4 Frontières de dépendances
Le graphe Backfill attendu devient :
```text
ksp-job-backfill-lib
-> ksp-core-lib
-> ksp-job-api
-> ksp-logging-lib
-> ksp-onchain-transport-lib
-> ksp-raw-transaction-lib
-> ksp-store-lib default-features=false
-> futures-util
-> serde_json
-> sha2
-> tokio macros+sync
```
`serde_json` reste un type direct de ladapter Transport (`SolanaWireField<serde_json::Value>`), mais aucune canonicalisation JSON Backfill nexiste plus. `sha2` reste Job-owned pour scope fingerprint et observation key ; le hash du payload RAW appartient désormais uniquement à common.
### 12.5 Validation locale dassemblage `pre.003`
Exécuté dans lenvironnement dassemblage :
```text
python3 scripts/audit_rust_workspace_rules.py
General Rust rule audit: clean
Rust export completeness audit: 0 candidate(s)
KSP workspace Rust rule audit: clean
python3 scripts/audit_markdown_tables.py README.md RULES.md ROADMAP.md CHANGELOG.md docs prompts crates deltas
Markdown table audit: clean (339 table(s), 756 file(s))
```
Lenvironnement dassemblage ne fournit toujours ni `cargo`, ni `rustc`, ni `rustfmt`. Les gates Cargo de `pre.003` restent donc à rejouer côté opérateur.
### 12.6 Gate opérateur demandé pour `pre.003`
```bash
cargo fmt --all
python3 scripts/audit_rust_workspace_rules.py
python3 scripts/audit_markdown_tables.py README.md RULES.md ROADMAP.md CHANGELOG.md docs prompts crates deltas
cargo check --workspace
cargo clippy --workspace --all-targets --all-features -- -D warnings
cargo test -p ksp-raw-transaction-lib
cargo test -p ksp-job-backfill-lib
cargo tree -p ksp-job-backfill-lib --edges normal
cargo tree -p ksp-job-backfill-lib -e features
```