v0.3.6-pre.006
This commit is contained in:
@@ -1,12 +1,12 @@
|
||||
# file: Cargo.toml
|
||||
# version: 401
|
||||
# version: 402
|
||||
|
||||
[workspace]
|
||||
resolver = "3"
|
||||
members = ["crates/ksp-app-config-desk", "crates/ksp-app-solprices-desk", "crates/ksp-app-wallet-desk", "crates/ksp-config-lib", "crates/ksp-core-lib", "crates/ksp-interface-lib", "crates/ksp-job-api", "crates/ksp-job-backfill-lib", "crates/ksp-logging-lib", "crates/ksp-offchain-transport-lib", "crates/ksp-onchain-transport-lib", "crates/ksp-program-api", "crates/ksp-store-api", "crates/ksp-store-lib", "crates/ksp-store-postgres-lib", "crates/ksp-wallet-lib"]
|
||||
|
||||
[workspace.package]
|
||||
version = "0.3.6-pre.5.fix.1"
|
||||
version = "0.3.6-pre.6"
|
||||
edition = "2024"
|
||||
license = "MIT"
|
||||
repository = "https://git.sasedev.com/Sasedev/khadhroony-solana-project"
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
# file: crates/ksp-job-backfill-lib/Cargo.toml
|
||||
# version: 1
|
||||
# version: 2
|
||||
|
||||
[package]
|
||||
name = "ksp-job-backfill-lib"
|
||||
@@ -13,6 +13,7 @@ ksp-job-api = { path = "../ksp-job-api" }
|
||||
ksp-logging-lib = { path = "../ksp-logging-lib" }
|
||||
ksp-onchain-transport-lib = { path = "../ksp-onchain-transport-lib" }
|
||||
ksp-store-lib = { path = "../ksp-store-lib", default-features = false }
|
||||
serde_json.workspace = true
|
||||
sha2.workspace = true
|
||||
|
||||
[dev-dependencies]
|
||||
|
||||
475
crates/ksp-job-backfill-lib/src/conversion.rs
Normal file
475
crates/ksp-job-backfill-lib/src/conversion.rs
Normal file
@@ -0,0 +1,475 @@
|
||||
// file: crates/ksp-job-backfill-lib/src/conversion.rs
|
||||
// version: 1
|
||||
|
||||
use sha2::Digest; // rust-rules: trait-import
|
||||
|
||||
/// KSP-owned source-independent RAW transaction format identifier produced by this Backfill vertical.
|
||||
pub const RAW_TRANSACTION_FORMAT_ID: &str = "ksp.solana.raw_transaction";
|
||||
/// Initial KSP-owned RAW transaction format version produced by this Backfill vertical.
|
||||
pub const RAW_TRANSACTION_FORMAT_VERSION: u32 = 1;
|
||||
|
||||
const RAW_TRANSACTION_METHOD_CODE: &str = "getTransaction";
|
||||
const RAW_TRANSACTION_OBSERVATION_CONTRACT_VERSION: u32 = 1;
|
||||
const RAW_TRANSACTION_PROTOCOL_CODE: &str = "solana.http.json_rpc";
|
||||
|
||||
/// Complete in-memory RAW transaction acquisition ready for the later Store persistence tranche.
|
||||
#[derive(Debug)]
|
||||
pub struct BackfillRawAcquisition {
|
||||
transaction: ksp_store_lib::RawTransaction,
|
||||
observation: ksp_store_lib::RawTransactionObservation,
|
||||
}
|
||||
|
||||
impl BackfillRawAcquisition {
|
||||
/// Returns the canonical RAW transaction produced from the typed Transport response.
|
||||
#[must_use]
|
||||
pub const fn transaction(&self) -> &ksp_store_lib::RawTransaction {
|
||||
return &self.transaction;
|
||||
}
|
||||
|
||||
/// Returns the acquisition observation whose provenance records the actual successful endpoint.
|
||||
#[must_use]
|
||||
pub const fn observation(&self) -> &ksp_store_lib::RawTransactionObservation {
|
||||
return &self.observation;
|
||||
}
|
||||
|
||||
/// Consumes the in-memory acquisition into the canonical transaction and its observation.
|
||||
#[must_use]
|
||||
pub fn into_parts(self) -> (ksp_store_lib::RawTransaction, ksp_store_lib::RawTransactionObservation) {
|
||||
return (self.transaction, self.observation);
|
||||
}
|
||||
}
|
||||
|
||||
/// Result of hydrating one deterministic Backfill candidate through observed `getTransaction`.
|
||||
#[derive(Debug)]
|
||||
pub enum BackfillHydrationOutcome {
|
||||
/// The RPC returned one complete transaction and conversion produced canonical RAW plus provenance.
|
||||
Available(BackfillRawAcquisition),
|
||||
/// The RPC returned JSON `null`; only the canonical transaction identity exists and no provenance is fabricated.
|
||||
Missing(ksp_store_lib::RawTransactionReference),
|
||||
}
|
||||
|
||||
impl BackfillHydrationOutcome {
|
||||
/// Returns the network-scoped transaction identity represented by this hydration outcome.
|
||||
#[must_use]
|
||||
pub fn reference(&self) -> &ksp_store_lib::RawTransactionReference {
|
||||
return match self {
|
||||
Self::Available(acquisition) => acquisition.transaction().reference(),
|
||||
Self::Missing(reference) => reference,
|
||||
};
|
||||
}
|
||||
|
||||
/// Returns whether `getTransaction` returned JSON `null`.
|
||||
#[must_use]
|
||||
pub const fn is_missing(&self) -> bool {
|
||||
return matches!(self, Self::Missing(_));
|
||||
}
|
||||
}
|
||||
|
||||
/// Hydrates one candidate with the typed observed Transport path and converts it to canonical RAW v1.
|
||||
///
|
||||
/// The caller supplies the local receipt timestamp because wall-clock ownership remains outside this
|
||||
/// pure conversion tranche. Transport retains endpoint selection and retry. This function never
|
||||
/// persists to Store; persistence begins in `pre.007`.
|
||||
pub async fn hydrate_backfill_candidate(
|
||||
transport: &ksp_onchain_transport_lib::HttpTransportPool,
|
||||
request: &crate::BackfillRequest,
|
||||
candidate: &crate::BackfillCandidate,
|
||||
received_at: ksp_store_lib::RawTimestamp,
|
||||
) -> ksp_core_lib::Result<BackfillHydrationOutcome> {
|
||||
let reference = canonical_reference(request, candidate);
|
||||
let reference = match reference {
|
||||
std::result::Result::Ok(reference) => reference,
|
||||
std::result::Result::Err(error) => return std::result::Result::Err(error),
|
||||
};
|
||||
let config = ksp_onchain_transport_lib::SolanaGetTransactionConfig::new(
|
||||
std::option::Option::Some(request.commitment().transport()),
|
||||
std::option::Option::Some(ksp_onchain_transport_lib::SolanaTransactionEncoding::Base64),
|
||||
std::option::Option::Some(0),
|
||||
);
|
||||
let observed = transport.get_transaction_observed(request.role(), candidate.identity().signature().as_str(), std::option::Option::Some(&config)).await;
|
||||
let observed = match observed {
|
||||
std::result::Result::Ok(observed) => observed,
|
||||
std::result::Result::Err(error) => return std::result::Result::Err(error),
|
||||
};
|
||||
let provider = observed.provider().as_str().to_owned();
|
||||
let endpoint = observed.endpoint_name().to_owned();
|
||||
let transaction = observed.into_value();
|
||||
let transaction = match transaction {
|
||||
std::option::Option::Some(transaction) => transaction,
|
||||
std::option::Option::None => return std::result::Result::Ok(BackfillHydrationOutcome::Missing(reference)),
|
||||
};
|
||||
let fields = CanonicalTransactionFields {
|
||||
slot: transaction.slot(),
|
||||
block_time: transaction.block_time(),
|
||||
transaction: transaction.transaction(),
|
||||
meta: transaction.meta(),
|
||||
version: transaction.version(),
|
||||
transaction_index: transaction.transaction_index(),
|
||||
};
|
||||
let acquisition = convert_available_fields(request, reference, fields, provider.as_str(), endpoint.as_str(), received_at);
|
||||
return match acquisition {
|
||||
std::result::Result::Ok(acquisition) => std::result::Result::Ok(BackfillHydrationOutcome::Available(acquisition)),
|
||||
std::result::Result::Err(error) => std::result::Result::Err(error),
|
||||
};
|
||||
}
|
||||
|
||||
/// Decodes one validated Base58 signature to exactly 64 canonical bytes without a Solana SDK dependency.
|
||||
pub(crate) fn decode_backfill_signature(signature: &crate::BackfillSignature) -> ksp_core_lib::Result<ksp_store_lib::RawTransactionSignature> {
|
||||
let text = signature.as_str().as_bytes();
|
||||
let mut decoded = [0_u8; 64];
|
||||
let mut leading_zeroes = 0_usize;
|
||||
for byte in text {
|
||||
if *byte != b'1' {
|
||||
break;
|
||||
}
|
||||
leading_zeroes += 1;
|
||||
}
|
||||
for byte in text {
|
||||
let digit = match base58_digit(*byte) {
|
||||
std::option::Option::Some(digit) => digit,
|
||||
std::option::Option::None => return std::result::Result::Err(conversion_error("signature")),
|
||||
};
|
||||
let mut carry = u32::from(digit);
|
||||
for output in decoded.iter_mut().rev() {
|
||||
let value = (u32::from(*output) * 58) + carry;
|
||||
*output = (value & 0xff) as u8;
|
||||
carry = value >> 8;
|
||||
}
|
||||
if carry != 0 {
|
||||
return std::result::Result::Err(conversion_error("signature"));
|
||||
}
|
||||
}
|
||||
let significant_len = match decoded.iter().position(|byte| return *byte != 0) {
|
||||
std::option::Option::Some(index) => decoded.len() - index,
|
||||
std::option::Option::None => 0,
|
||||
};
|
||||
if leading_zeroes + significant_len != decoded.len() {
|
||||
return std::result::Result::Err(conversion_error("signature"));
|
||||
}
|
||||
return std::result::Result::Ok(ksp_store_lib::RawTransactionSignature::new(decoded));
|
||||
}
|
||||
|
||||
struct CanonicalTransactionFields<'a> {
|
||||
slot: u64,
|
||||
block_time: std::option::Option<i64>,
|
||||
transaction: &'a ksp_onchain_transport_lib::SolanaEncodedTransaction,
|
||||
meta: &'a ksp_onchain_transport_lib::SolanaWireField<serde_json::Value>,
|
||||
version: &'a ksp_onchain_transport_lib::SolanaWireField<ksp_onchain_transport_lib::SolanaTransactionVersion>,
|
||||
transaction_index: &'a ksp_onchain_transport_lib::SolanaWireField<u32>,
|
||||
}
|
||||
|
||||
fn canonical_reference(request: &crate::BackfillRequest, candidate: &crate::BackfillCandidate) -> ksp_core_lib::Result<ksp_store_lib::RawTransactionReference> {
|
||||
if candidate.identity().network() != request.network() {
|
||||
return std::result::Result::Err(conversion_error("candidate.network"));
|
||||
}
|
||||
let signature = crate::decode_backfill_signature(candidate.identity().signature());
|
||||
let signature = match signature {
|
||||
std::result::Result::Ok(signature) => signature,
|
||||
std::result::Result::Err(error) => return std::result::Result::Err(error),
|
||||
};
|
||||
return std::result::Result::Ok(ksp_store_lib::RawTransactionReference::new(candidate.identity().network().clone(), signature));
|
||||
}
|
||||
|
||||
fn convert_available_fields(
|
||||
request: &crate::BackfillRequest,
|
||||
reference: ksp_store_lib::RawTransactionReference,
|
||||
fields: CanonicalTransactionFields<'_>,
|
||||
provider: &str,
|
||||
endpoint: &str,
|
||||
received_at: ksp_store_lib::RawTimestamp,
|
||||
) -> ksp_core_lib::Result<BackfillRawAcquisition> {
|
||||
let block_time = convert_block_time(fields.block_time);
|
||||
let block_time = match block_time {
|
||||
std::result::Result::Ok(block_time) => block_time,
|
||||
std::result::Result::Err(error) => return std::result::Result::Err(error),
|
||||
};
|
||||
let bytes = canonical_payload_bytes(&fields);
|
||||
let bytes = match bytes {
|
||||
std::result::Result::Ok(bytes) => bytes,
|
||||
std::result::Result::Err(error) => return std::result::Result::Err(error),
|
||||
};
|
||||
let hash: [u8; 32] = sha2::Sha256::digest(bytes.as_slice()).into();
|
||||
let format_id = ksp_store_lib::RawFormatId::new(RAW_TRANSACTION_FORMAT_ID);
|
||||
let format_id = match format_id {
|
||||
std::result::Result::Ok(format_id) => format_id,
|
||||
std::result::Result::Err(_) => return std::result::Result::Err(conversion_error("payload.format_id")),
|
||||
};
|
||||
let payload =
|
||||
ksp_store_lib::RawPayload::try_new(format_id, RAW_TRANSACTION_FORMAT_VERSION, bytes.into_boxed_slice(), ksp_store_lib::RawContentHash::new(hash));
|
||||
let payload = match payload {
|
||||
std::result::Result::Ok(payload) => payload,
|
||||
std::result::Result::Err(error) => return std::result::Result::Err(error),
|
||||
};
|
||||
let provenance = build_provenance(request, provider, endpoint, received_at);
|
||||
let provenance = match provenance {
|
||||
std::result::Result::Ok(provenance) => provenance,
|
||||
std::result::Result::Err(error) => return std::result::Result::Err(error),
|
||||
};
|
||||
let observation_key = observation_key(request, &reference, provider, endpoint);
|
||||
let transaction = ksp_store_lib::RawTransaction::new(reference.clone(), fields.slot, block_time, payload);
|
||||
let observation = ksp_store_lib::RawTransactionObservation::new(observation_key, reference, provenance);
|
||||
return std::result::Result::Ok(BackfillRawAcquisition { transaction, observation });
|
||||
}
|
||||
|
||||
fn convert_block_time(value: std::option::Option<i64>) -> ksp_core_lib::Result<std::option::Option<ksp_store_lib::RawTimestamp>> {
|
||||
let seconds = match value {
|
||||
std::option::Option::Some(seconds) => seconds,
|
||||
std::option::Option::None => return std::result::Result::Ok(std::option::Option::None),
|
||||
};
|
||||
let seconds = match u64::try_from(seconds) {
|
||||
std::result::Result::Ok(seconds) => seconds,
|
||||
std::result::Result::Err(_) => return std::result::Result::Err(conversion_error("block_time")),
|
||||
};
|
||||
let millis = match seconds.checked_mul(1_000) {
|
||||
std::option::Option::Some(millis) => millis,
|
||||
std::option::Option::None => return std::result::Result::Err(conversion_error("block_time")),
|
||||
};
|
||||
let timestamp = ksp_store_lib::RawTimestamp::from_unix_millis(millis);
|
||||
return match timestamp {
|
||||
std::result::Result::Ok(timestamp) => std::result::Result::Ok(std::option::Option::Some(timestamp)),
|
||||
std::result::Result::Err(_) => std::result::Result::Err(conversion_error("block_time")),
|
||||
};
|
||||
}
|
||||
|
||||
fn canonical_payload_bytes(fields: &CanonicalTransactionFields<'_>) -> ksp_core_lib::Result<std::vec::Vec<u8>> {
|
||||
let (transaction_data, transaction_encoding) = match fields.transaction {
|
||||
ksp_onchain_transport_lib::SolanaEncodedTransaction::Binary { data, encoding } => {
|
||||
if *encoding != ksp_onchain_transport_lib::SolanaTransactionBinaryEncoding::Base64 {
|
||||
return std::result::Result::Err(conversion_error("transaction.encoding"));
|
||||
}
|
||||
(data.as_str(), "base64")
|
||||
},
|
||||
ksp_onchain_transport_lib::SolanaEncodedTransaction::LegacyBinary(_) | ksp_onchain_transport_lib::SolanaEncodedTransaction::Json(_) => {
|
||||
return std::result::Result::Err(conversion_error("transaction.encoding"));
|
||||
},
|
||||
};
|
||||
let mut bytes = std::vec::Vec::new();
|
||||
bytes.extend_from_slice(b"{\"transaction\":[");
|
||||
if let std::result::Result::Err(error) = append_json_string(&mut bytes, transaction_data) {
|
||||
return std::result::Result::Err(error);
|
||||
}
|
||||
bytes.push(b',');
|
||||
if let std::result::Result::Err(error) = append_json_string(&mut bytes, transaction_encoding) {
|
||||
return std::result::Result::Err(error);
|
||||
}
|
||||
bytes.push(b']');
|
||||
if let std::result::Result::Err(error) = append_wire_value(&mut bytes, "meta", fields.meta, |output, value| return append_canonical_json(output, value)) {
|
||||
return std::result::Result::Err(error);
|
||||
}
|
||||
let version_result = append_wire_value(&mut bytes, "version", fields.version, |output, value| {
|
||||
return match value {
|
||||
ksp_onchain_transport_lib::SolanaTransactionVersion::Legacy => append_json_string(output, "legacy"),
|
||||
ksp_onchain_transport_lib::SolanaTransactionVersion::Number(number) => {
|
||||
output.extend_from_slice(number.to_string().as_bytes());
|
||||
std::result::Result::Ok(())
|
||||
},
|
||||
};
|
||||
});
|
||||
if let std::result::Result::Err(error) = version_result {
|
||||
return std::result::Result::Err(error);
|
||||
}
|
||||
let transaction_index_result = append_wire_value(&mut bytes, "transactionIndex", fields.transaction_index, |output, value| {
|
||||
output.extend_from_slice(value.to_string().as_bytes());
|
||||
return std::result::Result::Ok(());
|
||||
});
|
||||
if let std::result::Result::Err(error) = transaction_index_result {
|
||||
return std::result::Result::Err(error);
|
||||
}
|
||||
bytes.push(b'}');
|
||||
return std::result::Result::Ok(bytes);
|
||||
}
|
||||
|
||||
fn append_wire_value<T, F>(
|
||||
output: &mut std::vec::Vec<u8>,
|
||||
key: &str,
|
||||
field: &ksp_onchain_transport_lib::SolanaWireField<T>,
|
||||
mut append_value: F,
|
||||
) -> ksp_core_lib::Result<()>
|
||||
where
|
||||
F: FnMut(&mut std::vec::Vec<u8>, &T) -> ksp_core_lib::Result<()>,
|
||||
{
|
||||
return match field {
|
||||
ksp_onchain_transport_lib::SolanaWireField::Omitted => std::result::Result::Ok(()),
|
||||
ksp_onchain_transport_lib::SolanaWireField::Null => {
|
||||
output.push(b',');
|
||||
let key_result = append_json_string(output, key);
|
||||
if let std::result::Result::Err(error) = key_result {
|
||||
return std::result::Result::Err(error);
|
||||
}
|
||||
output.extend_from_slice(b":null");
|
||||
std::result::Result::Ok(())
|
||||
},
|
||||
ksp_onchain_transport_lib::SolanaWireField::Value(value) => {
|
||||
output.push(b',');
|
||||
let key_result = append_json_string(output, key);
|
||||
if let std::result::Result::Err(error) = key_result {
|
||||
return std::result::Result::Err(error);
|
||||
}
|
||||
output.push(b':');
|
||||
append_value(output, value)
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
fn append_canonical_json(output: &mut std::vec::Vec<u8>, value: &serde_json::Value) -> ksp_core_lib::Result<()> {
|
||||
return match value {
|
||||
serde_json::Value::Null => {
|
||||
output.extend_from_slice(b"null");
|
||||
std::result::Result::Ok(())
|
||||
},
|
||||
serde_json::Value::Bool(value) => {
|
||||
if *value {
|
||||
output.extend_from_slice(b"true");
|
||||
} else {
|
||||
output.extend_from_slice(b"false");
|
||||
}
|
||||
std::result::Result::Ok(())
|
||||
},
|
||||
serde_json::Value::Number(value) => {
|
||||
output.extend_from_slice(value.to_string().as_bytes());
|
||||
std::result::Result::Ok(())
|
||||
},
|
||||
serde_json::Value::String(value) => append_json_string(output, value.as_str()),
|
||||
serde_json::Value::Array(values) => {
|
||||
output.push(b'[');
|
||||
for (index, item) in values.iter().enumerate() {
|
||||
if index != 0 {
|
||||
output.push(b',');
|
||||
}
|
||||
let item_result = append_canonical_json(output, item);
|
||||
if let std::result::Result::Err(error) = item_result {
|
||||
return std::result::Result::Err(error);
|
||||
}
|
||||
}
|
||||
output.push(b']');
|
||||
std::result::Result::Ok(())
|
||||
},
|
||||
serde_json::Value::Object(values) => {
|
||||
output.push(b'{');
|
||||
let mut keys = values.keys().collect::<std::vec::Vec<_>>();
|
||||
keys.sort_unstable();
|
||||
for (index, key) in keys.iter().enumerate() {
|
||||
if index != 0 {
|
||||
output.push(b',');
|
||||
}
|
||||
let key_result = append_json_string(output, key.as_str());
|
||||
if let std::result::Result::Err(error) = key_result {
|
||||
return std::result::Result::Err(error);
|
||||
}
|
||||
output.push(b':');
|
||||
let item = values.get(key.as_str());
|
||||
let item = match item {
|
||||
std::option::Option::Some(item) => item,
|
||||
std::option::Option::None => return std::result::Result::Err(conversion_error("payload.meta")),
|
||||
};
|
||||
let item_result = append_canonical_json(output, item);
|
||||
if let std::result::Result::Err(error) = item_result {
|
||||
return std::result::Result::Err(error);
|
||||
}
|
||||
}
|
||||
output.push(b'}');
|
||||
std::result::Result::Ok(())
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
fn append_json_string(output: &mut std::vec::Vec<u8>, value: &str) -> ksp_core_lib::Result<()> {
|
||||
let encoded = serde_json::to_vec(value);
|
||||
return match encoded {
|
||||
std::result::Result::Ok(encoded) => {
|
||||
output.extend_from_slice(encoded.as_slice());
|
||||
std::result::Result::Ok(())
|
||||
},
|
||||
std::result::Result::Err(_) => std::result::Result::Err(conversion_error("payload.json")),
|
||||
};
|
||||
}
|
||||
|
||||
fn build_provenance(
|
||||
request: &crate::BackfillRequest,
|
||||
provider: &str,
|
||||
endpoint: &str,
|
||||
received_at: ksp_store_lib::RawTimestamp,
|
||||
) -> ksp_core_lib::Result<ksp_store_lib::RawAcquisitionProvenance> {
|
||||
let provider = match provenance_code(provider, "provenance.provider") {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(error) => return std::result::Result::Err(error),
|
||||
};
|
||||
let protocol = match provenance_code(RAW_TRANSACTION_PROTOCOL_CODE, "provenance.protocol") {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(error) => return std::result::Result::Err(error),
|
||||
};
|
||||
let method = match provenance_code(RAW_TRANSACTION_METHOD_CODE, "provenance.method") {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(error) => return std::result::Result::Err(error),
|
||||
};
|
||||
let endpoint = match provenance_code(endpoint, "provenance.endpoint") {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(error) => return std::result::Result::Err(error),
|
||||
};
|
||||
let commitment = match provenance_code(request.commitment().code(), "provenance.commitment") {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(error) => return std::result::Result::Err(error),
|
||||
};
|
||||
let capture_session = match provenance_code(request.job_id().as_str(), "provenance.capture_session") {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(error) => return std::result::Result::Err(error),
|
||||
};
|
||||
let provenance = ksp_store_lib::RawAcquisitionProvenance::new(provider, protocol, method, ksp_store_lib::RawAcquisitionOrigin::Backfill, received_at)
|
||||
.with_endpoint_id(endpoint)
|
||||
.with_commitment(commitment)
|
||||
.with_capture_session_id(capture_session);
|
||||
return std::result::Result::Ok(provenance);
|
||||
}
|
||||
|
||||
fn provenance_code(value: &str, field: &'static str) -> ksp_core_lib::Result<ksp_store_lib::RawProvenanceCode> {
|
||||
let code = ksp_store_lib::RawProvenanceCode::new(value);
|
||||
return match code {
|
||||
std::result::Result::Ok(code) => std::result::Result::Ok(code),
|
||||
std::result::Result::Err(_) => std::result::Result::Err(conversion_error(field)),
|
||||
};
|
||||
}
|
||||
|
||||
fn observation_key(
|
||||
request: &crate::BackfillRequest,
|
||||
reference: &ksp_store_lib::RawTransactionReference,
|
||||
provider: &str,
|
||||
endpoint: &str,
|
||||
) -> ksp_store_lib::RawObservationKey {
|
||||
let mut hasher = sha2::Sha256::new();
|
||||
hasher.update(b"ksp.job.backfill.raw_transaction_observation.v1\0");
|
||||
hash_bytes(&mut hasher, request.job_id().as_str().as_bytes());
|
||||
hash_bytes(&mut hasher, request.scope_fingerprint().as_bytes());
|
||||
hash_bytes(&mut hasher, reference.signature().as_bytes());
|
||||
hash_bytes(&mut hasher, provider.as_bytes());
|
||||
hash_bytes(&mut hasher, endpoint.as_bytes());
|
||||
hash_bytes(&mut hasher, request.commitment().code().as_bytes());
|
||||
hasher.update(RAW_TRANSACTION_OBSERVATION_CONTRACT_VERSION.to_be_bytes());
|
||||
let bytes: [u8; 32] = hasher.finalize().into();
|
||||
return ksp_store_lib::RawObservationKey::new(bytes);
|
||||
}
|
||||
|
||||
fn hash_bytes(hasher: &mut sha2::Sha256, value: &[u8]) {
|
||||
hasher.update((value.len() as u64).to_be_bytes());
|
||||
hasher.update(value);
|
||||
}
|
||||
|
||||
fn base58_digit(byte: u8) -> std::option::Option<u8> {
|
||||
return match byte {
|
||||
b'1'..=b'9' => std::option::Option::Some(byte - b'1'),
|
||||
b'A'..=b'H' => std::option::Option::Some((byte - b'A') + 9),
|
||||
b'J'..=b'N' => std::option::Option::Some((byte - b'J') + 17),
|
||||
b'P'..=b'Z' => std::option::Option::Some((byte - b'P') + 22),
|
||||
b'a'..=b'k' => std::option::Option::Some((byte - b'a') + 33),
|
||||
b'm'..=b'z' => std::option::Option::Some((byte - b'm') + 44),
|
||||
_ => std::option::Option::None,
|
||||
};
|
||||
}
|
||||
|
||||
fn conversion_error(field: &'static str) -> ksp_core_lib::Error {
|
||||
return ksp_core_lib::Error::new(crate::ERROR_CODE_BACKFILL_RAW_CONVERSION_INVALID, "invalid deterministic Backfill RAW conversion")
|
||||
.with_context("field", field);
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[path = "../unit_tests/conversion.rs"]
|
||||
mod tests;
|
||||
@@ -1,10 +1,12 @@
|
||||
// file: crates/ksp-job-backfill-lib/src/error.rs
|
||||
// version: 1
|
||||
// version: 2
|
||||
|
||||
/// Error code used when a signature page violates a bounded discovery invariant.
|
||||
pub const ERROR_CODE_BACKFILL_DISCOVERY_INVALID: ksp_core_lib::ErrorCode = ksp_core_lib::ErrorCode::new("job_backfill", "discovery_invalid");
|
||||
/// Error code used when paginated discovery cannot advance its exclusive RPC cursor safely.
|
||||
pub const ERROR_CODE_BACKFILL_DISCOVERY_STALLED: ksp_core_lib::ErrorCode = ksp_core_lib::ErrorCode::new("job_backfill", "discovery_stalled");
|
||||
/// Error code used when deterministic Transport-to-RAW conversion violates the v1 contract.
|
||||
pub const ERROR_CODE_BACKFILL_RAW_CONVERSION_INVALID: ksp_core_lib::ErrorCode = ksp_core_lib::ErrorCode::new("job_backfill", "raw_conversion_invalid");
|
||||
/// Error code used when one Backfill request violates its bounded admission contract.
|
||||
pub const ERROR_CODE_BACKFILL_REQUEST_INVALID: ksp_core_lib::ErrorCode = ksp_core_lib::ErrorCode::new("job_backfill", "request_invalid");
|
||||
/// Error code used when one transaction signature text violates the bounded Base58-shape contract.
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
// file: crates/ksp-job-backfill-lib/src/lib.rs
|
||||
// version: 1
|
||||
// version: 2
|
||||
|
||||
#![warn(missing_docs)]
|
||||
#![deny(unreachable_pub)]
|
||||
@@ -7,16 +7,28 @@
|
||||
|
||||
//! Concrete bounded historical RAW transaction Backfill foundation.
|
||||
//!
|
||||
//! This tranche owns explicit admission, network-scoped candidate identity and deterministic
|
||||
//! `getSignaturesForAddress` pagination. Transport retains provider/endpoint selection and retry;
|
||||
//! Store retains durable idempotence and persistence. RAW conversion, persistence, concurrency,
|
||||
//! checkpointing, cancellation and concrete latest-value snapshots are added by later v0.3.6 tranches.
|
||||
//! This tranche owns explicit admission, network-scoped candidate identity, deterministic
|
||||
//! `getSignaturesForAddress` pagination and canonical RAW v1 conversion through observed
|
||||
//! `getTransaction`. Transport retains provider/endpoint selection and retry; Store retains
|
||||
//! durable idempotence and persistence. Persistence, concurrency, checkpointing, cancellation
|
||||
//! and concrete latest-value snapshots are added by later v0.3.6 tranches.
|
||||
|
||||
mod constants;
|
||||
mod conversion;
|
||||
mod discovery;
|
||||
mod error;
|
||||
mod request;
|
||||
|
||||
/// Result of hydrating one deterministic candidate through observed `getTransaction`.
|
||||
pub use self::conversion::BackfillHydrationOutcome;
|
||||
/// Complete in-memory RAW transaction acquisition ready for later Store persistence.
|
||||
pub use self::conversion::BackfillRawAcquisition;
|
||||
/// KSP-owned source-independent RAW transaction format identifier produced by this Backfill vertical.
|
||||
pub use self::conversion::RAW_TRANSACTION_FORMAT_ID;
|
||||
/// Initial KSP-owned RAW transaction format version produced by this Backfill vertical.
|
||||
pub use self::conversion::RAW_TRANSACTION_FORMAT_VERSION;
|
||||
/// Hydrates one candidate through observed Transport and converts a non-null response to canonical RAW v1.
|
||||
pub use self::conversion::hydrate_backfill_candidate;
|
||||
/// One deterministic transaction candidate produced by bounded discovery.
|
||||
pub use self::discovery::BackfillCandidate;
|
||||
/// Network-scoped identity of one discovered transaction candidate before canonical signature decoding.
|
||||
@@ -31,6 +43,8 @@ pub use self::discovery::discover_backfill_candidates;
|
||||
pub use self::error::ERROR_CODE_BACKFILL_DISCOVERY_INVALID;
|
||||
/// Error code used when paginated discovery cannot advance its exclusive RPC cursor safely.
|
||||
pub use self::error::ERROR_CODE_BACKFILL_DISCOVERY_STALLED;
|
||||
/// Error code used when deterministic Transport-to-RAW conversion violates the v1 contract.
|
||||
pub use self::error::ERROR_CODE_BACKFILL_RAW_CONVERSION_INVALID;
|
||||
/// Error code used when one Backfill request violates its bounded admission contract.
|
||||
pub use self::error::ERROR_CODE_BACKFILL_REQUEST_INVALID;
|
||||
/// Error code used when one transaction signature text violates the bounded Base58-shape contract.
|
||||
@@ -45,7 +59,7 @@ pub use self::request::BackfillScope;
|
||||
pub use self::request::BackfillScopeFingerprint;
|
||||
/// Stable category of one bounded Backfill discovery scope.
|
||||
pub use self::request::BackfillScopeKind;
|
||||
/// Bounded Base58-shaped transaction signature text used before canonical 64-byte decoding.
|
||||
/// Bounded Base58-shaped transaction signature text with exact RAW signature conversion.
|
||||
pub use self::request::BackfillSignature;
|
||||
/// Maximum number of transaction candidates admitted by one bounded Backfill Job.
|
||||
pub use self::request::MAX_BACKFILL_CANDIDATES;
|
||||
@@ -62,3 +76,5 @@ pub use self::request::MIN_BACKFILL_SIGNATURE_TEXT_BYTES;
|
||||
|
||||
/// Owning tracing target used by the concrete Backfill runtime.
|
||||
pub(crate) use self::constants::TRACING_TARGET;
|
||||
/// Exact private Base58 decoder shared by the public signature wrapper and hydration path.
|
||||
pub(crate) use self::conversion::decode_backfill_signature;
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
// file: crates/ksp-job-backfill-lib/src/request.rs
|
||||
// version: 1
|
||||
// version: 2
|
||||
|
||||
use sha2::Digest; // rust-rules: trait-import
|
||||
|
||||
@@ -45,10 +45,10 @@ impl BackfillCommitment {
|
||||
}
|
||||
}
|
||||
|
||||
/// Bounded Base58-shaped transaction signature text used before canonical 64-byte decoding.
|
||||
/// Bounded Base58-shaped transaction signature text used by discovery and RAW conversion.
|
||||
///
|
||||
/// This type deliberately validates only the encoded shape required by discovery. Exact decoding
|
||||
/// to the Store-owned 64-byte signature is introduced by the RAW conversion tranche.
|
||||
/// Construction validates the encoded shape required by discovery. Exact conversion to the
|
||||
/// Store-owned 64-byte signature is available through [`Self::to_raw_transaction_signature`].
|
||||
#[derive(Clone, Eq, Hash, Ord, PartialEq, PartialOrd)]
|
||||
pub struct BackfillSignature(std::string::String);
|
||||
|
||||
@@ -67,6 +67,11 @@ impl BackfillSignature {
|
||||
pub fn as_str(&self) -> &str {
|
||||
return self.0.as_str();
|
||||
}
|
||||
|
||||
/// Decodes this Base58 text to the exact Store-owned 64-byte Solana signature.
|
||||
pub fn to_raw_transaction_signature(&self) -> ksp_core_lib::Result<ksp_store_lib::RawTransactionSignature> {
|
||||
return crate::decode_backfill_signature(self);
|
||||
}
|
||||
}
|
||||
|
||||
impl std::fmt::Debug for BackfillSignature {
|
||||
|
||||
@@ -1,10 +1,10 @@
|
||||
// file: crates/ksp-job-backfill-lib/tests/dependency_boundary.rs
|
||||
// version: 1
|
||||
// version: 2
|
||||
|
||||
//! Dependency firewall canaries for the concrete Backfill foundation.
|
||||
//! Dependency firewall canaries for Backfill discovery and RAW v1 conversion.
|
||||
|
||||
#[test]
|
||||
fn pre_005_manifest_uses_only_planned_ksp_edges_and_backend_neutral_store() {
|
||||
fn pre_006_manifest_uses_only_planned_ksp_edges_and_backend_neutral_store() {
|
||||
let manifest = include_str!("../Cargo.toml");
|
||||
for required in [
|
||||
"ksp-core-lib = { path = \"../ksp-core-lib\" }",
|
||||
@@ -12,6 +12,7 @@ fn pre_005_manifest_uses_only_planned_ksp_edges_and_backend_neutral_store() {
|
||||
"ksp-logging-lib = { path = \"../ksp-logging-lib\" }",
|
||||
"ksp-onchain-transport-lib = { path = \"../ksp-onchain-transport-lib\" }",
|
||||
"ksp-store-lib = { path = \"../ksp-store-lib\", default-features = false }",
|
||||
"serde_json.workspace = true",
|
||||
"sha2.workspace = true",
|
||||
] {
|
||||
assert!(manifest.contains(required), "required Backfill dependency missing: {required}");
|
||||
@@ -25,7 +26,8 @@ fn pre_005_manifest_uses_only_planned_ksp_edges_and_backend_neutral_store() {
|
||||
"ksp-wallet-lib",
|
||||
"solana-",
|
||||
"reqwest",
|
||||
"serde",
|
||||
"serde.workspace = true",
|
||||
"serde = {",
|
||||
"tonic",
|
||||
] {
|
||||
assert!(!manifest.contains(forbidden), "forbidden Backfill dependency present: {forbidden}");
|
||||
@@ -34,21 +36,32 @@ fn pre_005_manifest_uses_only_planned_ksp_edges_and_backend_neutral_store() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn pre_005_production_sources_keep_transport_and_store_in_their_owned_layers() {
|
||||
let sources = [
|
||||
fn pre_006_production_sources_keep_transport_and_store_in_their_owned_layers() {
|
||||
let non_conversion_sources = [
|
||||
include_str!("../src/constants.rs"),
|
||||
include_str!("../src/discovery.rs"),
|
||||
include_str!("../src/error.rs"),
|
||||
include_str!("../src/lib.rs"),
|
||||
include_str!("../src/request.rs"),
|
||||
];
|
||||
for source in sources {
|
||||
for source in non_conversion_sources {
|
||||
for forbidden in
|
||||
["ksp_config_lib::", "ksp_interface_lib::", "ksp_store_api::", "ksp_store_postgres_lib::", "reqwest::", "serde_json::", "std::env", "tonic::"]
|
||||
{
|
||||
assert!(!source.contains(forbidden), "forbidden concrete Backfill path detected: {forbidden}");
|
||||
}
|
||||
}
|
||||
let conversion = include_str!("../src/conversion.rs");
|
||||
assert!(conversion.contains("serde_json::"));
|
||||
assert!(conversion.contains("get_transaction_observed"));
|
||||
assert!(conversion.contains("SolanaTransactionEncoding::Base64"));
|
||||
assert!(conversion.contains("std::option::Option::Some(0)"));
|
||||
assert!(conversion.contains("BackfillHydrationOutcome::Missing(reference)"));
|
||||
assert!(!conversion.contains("execute_standard_rpc"));
|
||||
assert!(!conversion.contains("persist_raw_transaction_acquisition"));
|
||||
for forbidden in ["ksp_config_lib::", "ksp_interface_lib::", "ksp_store_api::", "ksp_store_postgres_lib::", "reqwest::", "std::env", "tonic::"] {
|
||||
assert!(!conversion.contains(forbidden), "forbidden RAW conversion path detected: {forbidden}");
|
||||
}
|
||||
let discovery = include_str!("../src/discovery.rs");
|
||||
assert!(discovery.contains("get_signatures_for_address"));
|
||||
assert!(!discovery.contains("execute_standard_rpc"));
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
// file: crates/ksp-job-backfill-lib/tests/public_api.rs
|
||||
// version: 1
|
||||
// version: 2
|
||||
|
||||
//! Public API canaries for the bounded Backfill foundation.
|
||||
//! Public API canaries for bounded Backfill discovery and RAW v1 conversion.
|
||||
|
||||
#[test]
|
||||
fn pre_005_request_scope_and_discovery_contracts_are_available_from_crate_root() {
|
||||
@@ -64,3 +64,25 @@ fn pre_005_candidate_identity_is_network_plus_signature_not_transport_source() {
|
||||
assert_ne!(mainnet_identity, devnet_identity);
|
||||
return;
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn pre_006_raw_conversion_contract_is_available_from_crate_root() {
|
||||
assert_eq!(ksp_job_backfill_lib::RAW_TRANSACTION_FORMAT_ID, "ksp.solana.raw_transaction");
|
||||
assert_eq!(ksp_job_backfill_lib::RAW_TRANSACTION_FORMAT_VERSION, 1);
|
||||
let signature = ksp_job_backfill_lib::BackfillSignature::new("1".repeat(64));
|
||||
assert!(signature.is_ok());
|
||||
let signature = match signature {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(_) => return,
|
||||
};
|
||||
let decoded = signature.to_raw_transaction_signature();
|
||||
assert!(decoded.is_ok());
|
||||
if let std::result::Result::Ok(decoded) = decoded {
|
||||
assert_eq!(decoded.as_bytes(), &[0_u8; 64]);
|
||||
}
|
||||
let _hydrate = ksp_job_backfill_lib::hydrate_backfill_candidate;
|
||||
let _outcome: std::option::Option<ksp_job_backfill_lib::BackfillHydrationOutcome> = std::option::Option::None;
|
||||
let _acquisition: std::option::Option<ksp_job_backfill_lib::BackfillRawAcquisition> = std::option::Option::None;
|
||||
assert_eq!(ksp_job_backfill_lib::ERROR_CODE_BACKFILL_RAW_CONVERSION_INVALID, ksp_core_lib::ErrorCode::new("job_backfill", "raw_conversion_invalid"));
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -1,10 +1,10 @@
|
||||
// file: crates/ksp-job-backfill-lib/tests/release_completeness.rs
|
||||
// version: 1
|
||||
// version: 2
|
||||
|
||||
//! Completeness canaries for the `pre.005` Backfill foundation.
|
||||
//! Completeness canaries for the `pre.006` Backfill RAW conversion tranche.
|
||||
|
||||
#[test]
|
||||
fn pre_005_production_module_inventory_is_exact() -> std::io::Result<()> {
|
||||
fn pre_006_production_module_inventory_is_exact() -> std::io::Result<()> {
|
||||
let source_root = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("src");
|
||||
let entries = match std::fs::read_dir(source_root) {
|
||||
std::result::Result::Ok(value) => value,
|
||||
@@ -32,12 +32,12 @@ fn pre_005_production_module_inventory_is_exact() -> std::io::Result<()> {
|
||||
}
|
||||
}
|
||||
names.sort_unstable();
|
||||
assert_eq!(names, std::vec!["constants.rs", "discovery.rs", "error.rs", "lib.rs", "request.rs"]);
|
||||
assert_eq!(names, std::vec!["constants.rs", "conversion.rs", "discovery.rs", "error.rs", "lib.rs", "request.rs"]);
|
||||
return std::result::Result::Ok(());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn pre_005_surface_is_discovery_only_without_raw_persistence_or_checkpoint_runtime() {
|
||||
fn pre_006_surface_adds_raw_conversion_without_persistence_or_checkpoint_runtime() {
|
||||
let root = include_str!("../src/lib.rs");
|
||||
for required in [
|
||||
"BackfillCandidate",
|
||||
@@ -50,11 +50,17 @@ fn pre_005_surface_is_discovery_only_without_raw_persistence_or_checkpoint_runti
|
||||
"BackfillScopeFingerprint",
|
||||
"BackfillSignature",
|
||||
"discover_backfill_candidates",
|
||||
"BackfillRawAcquisition",
|
||||
"BackfillHydrationOutcome",
|
||||
"hydrate_backfill_candidate",
|
||||
"RAW_TRANSACTION_FORMAT_ID",
|
||||
"RAW_TRANSACTION_FORMAT_VERSION",
|
||||
"ERROR_CODE_BACKFILL_RAW_CONVERSION_INVALID",
|
||||
] {
|
||||
assert!(root.contains(required), "required pre.005 public contract missing: {required}");
|
||||
assert!(root.contains(required), "required pre.006 public contract missing: {required}");
|
||||
}
|
||||
for forbidden in ["RawTransactionObservation", "persist_raw_transaction_acquisition", "BackfillCheckpoint", "BackfillJobHandle", "JobSnapshotSource"] {
|
||||
assert!(!root.contains(forbidden), "later Backfill tranche leaked into pre.005: {forbidden}");
|
||||
for forbidden in ["persist_raw_transaction_acquisition", "BackfillCheckpoint", "BackfillJobHandle", "JobSnapshotSource"] {
|
||||
assert!(!root.contains(forbidden), "later Backfill tranche leaked into pre.006: {forbidden}");
|
||||
}
|
||||
assert!(!root.contains("pub mod "));
|
||||
return;
|
||||
|
||||
295
crates/ksp-job-backfill-lib/unit_tests/conversion.rs
Normal file
295
crates/ksp-job-backfill-lib/unit_tests/conversion.rs
Normal file
@@ -0,0 +1,295 @@
|
||||
// file: crates/ksp-job-backfill-lib/unit_tests/conversion.rs
|
||||
// version: 1
|
||||
|
||||
fn signature_text() -> std::option::Option<crate::BackfillSignature> {
|
||||
return match crate::BackfillSignature::new("1".repeat(64)) {
|
||||
std::result::Result::Ok(value) => std::option::Option::Some(value),
|
||||
std::result::Result::Err(_) => std::option::Option::None,
|
||||
};
|
||||
}
|
||||
|
||||
fn request(network: &str, signature: crate::BackfillSignature) -> std::option::Option<crate::BackfillRequest> {
|
||||
let network = match ksp_store_lib::RawNetworkId::new(network) {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(_) => return std::option::Option::None,
|
||||
};
|
||||
let job_id = match ksp_job_api::JobId::new("backfill:pre006") {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(_) => return std::option::Option::None,
|
||||
};
|
||||
let scope = match crate::BackfillScope::explicit_signatures(std::vec![signature]) {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(_) => return std::option::Option::None,
|
||||
};
|
||||
return match crate::BackfillRequest::new(
|
||||
job_id,
|
||||
network,
|
||||
ksp_onchain_transport_lib::HttpRoleName::new("history"),
|
||||
crate::BackfillCommitment::Finalized,
|
||||
scope,
|
||||
100,
|
||||
10,
|
||||
100,
|
||||
4,
|
||||
std::option::Option::None,
|
||||
) {
|
||||
std::result::Result::Ok(value) => std::option::Option::Some(value),
|
||||
std::result::Result::Err(_) => std::option::Option::None,
|
||||
};
|
||||
}
|
||||
|
||||
fn candidate(network: &str, signature: crate::BackfillSignature) -> std::option::Option<crate::BackfillCandidate> {
|
||||
let network = match ksp_store_lib::RawNetworkId::new(network) {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(_) => return std::option::Option::None,
|
||||
};
|
||||
return std::option::Option::Some(crate::BackfillCandidate::new(crate::BackfillCandidateIdentity::new(network, signature), std::option::Option::Some(42)));
|
||||
}
|
||||
|
||||
fn received_at() -> std::option::Option<ksp_store_lib::RawTimestamp> {
|
||||
return match ksp_store_lib::RawTimestamp::from_unix_millis(1_700_000_001_000) {
|
||||
std::result::Result::Ok(value) => std::option::Option::Some(value),
|
||||
std::result::Result::Err(_) => std::option::Option::None,
|
||||
};
|
||||
}
|
||||
|
||||
fn fields<'a>(
|
||||
transaction: &'a ksp_onchain_transport_lib::SolanaEncodedTransaction,
|
||||
meta: &'a ksp_onchain_transport_lib::SolanaWireField<serde_json::Value>,
|
||||
version: &'a ksp_onchain_transport_lib::SolanaWireField<ksp_onchain_transport_lib::SolanaTransactionVersion>,
|
||||
transaction_index: &'a ksp_onchain_transport_lib::SolanaWireField<u32>,
|
||||
block_time: std::option::Option<i64>,
|
||||
) -> super::CanonicalTransactionFields<'a> {
|
||||
return super::CanonicalTransactionFields { slot: 123, block_time, transaction, meta, version, transaction_index };
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn pre_006_signature_decoding_requires_exactly_sixty_four_bytes() {
|
||||
let signature = match signature_text() {
|
||||
std::option::Option::Some(value) => value,
|
||||
std::option::Option::None => return,
|
||||
};
|
||||
let decoded = signature.to_raw_transaction_signature();
|
||||
assert!(decoded.is_ok());
|
||||
if let std::result::Result::Ok(decoded) = decoded {
|
||||
assert_eq!(decoded.as_bytes(), &[0_u8; 64]);
|
||||
}
|
||||
let too_many_zeroes = crate::BackfillSignature::new("1".repeat(65));
|
||||
assert!(too_many_zeroes.is_ok());
|
||||
if let std::result::Result::Ok(too_many_zeroes) = too_many_zeroes {
|
||||
assert!(too_many_zeroes.to_raw_transaction_signature().is_err());
|
||||
}
|
||||
let overflow = crate::BackfillSignature::new("z".repeat(88));
|
||||
assert!(overflow.is_ok());
|
||||
if let std::result::Result::Ok(overflow) = overflow {
|
||||
assert!(overflow.to_raw_transaction_signature().is_err());
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn pre_006_canonical_payload_golden_bytes_hash_and_provenance_are_exact() {
|
||||
let signature = match signature_text() {
|
||||
std::option::Option::Some(value) => value,
|
||||
std::option::Option::None => return,
|
||||
};
|
||||
let request = match request("devnet", signature.clone()) {
|
||||
std::option::Option::Some(value) => value,
|
||||
std::option::Option::None => return,
|
||||
};
|
||||
let candidate = match candidate("devnet", signature) {
|
||||
std::option::Option::Some(value) => value,
|
||||
std::option::Option::None => return,
|
||||
};
|
||||
let reference = match super::canonical_reference(&request, &candidate) {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(_) => return,
|
||||
};
|
||||
let transaction = ksp_onchain_transport_lib::SolanaEncodedTransaction::Binary {
|
||||
data: "AQID".to_owned(),
|
||||
encoding: ksp_onchain_transport_lib::SolanaTransactionBinaryEncoding::Base64,
|
||||
};
|
||||
let meta = ksp_onchain_transport_lib::SolanaWireField::Value(serde_json::json!({"z": 1, "a": {"y": true, "x": null}}));
|
||||
let version = ksp_onchain_transport_lib::SolanaWireField::Value(ksp_onchain_transport_lib::SolanaTransactionVersion::Legacy);
|
||||
let transaction_index = ksp_onchain_transport_lib::SolanaWireField::Value(7_u32);
|
||||
let received_at = match received_at() {
|
||||
std::option::Option::Some(value) => value,
|
||||
std::option::Option::None => return,
|
||||
};
|
||||
let acquisition = super::convert_available_fields(
|
||||
&request,
|
||||
reference,
|
||||
fields(&transaction, &meta, &version, &transaction_index, std::option::Option::Some(1_700_000_000)),
|
||||
"publicnode",
|
||||
"devnet-primary",
|
||||
received_at,
|
||||
);
|
||||
assert!(acquisition.is_ok());
|
||||
let acquisition = match acquisition {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(_) => return,
|
||||
};
|
||||
let expected = b"{\"transaction\":[\"AQID\",\"base64\"],\"meta\":{\"a\":{\"x\":null,\"y\":true},\"z\":1},\"version\":\"legacy\",\"transactionIndex\":7}";
|
||||
assert_eq!(acquisition.transaction().payload().bytes(), expected);
|
||||
assert_eq!(acquisition.transaction().payload().byte_len(), 112);
|
||||
assert_eq!(acquisition.transaction().payload().format_id().as_str(), crate::RAW_TRANSACTION_FORMAT_ID);
|
||||
assert_eq!(acquisition.transaction().payload().format_version(), crate::RAW_TRANSACTION_FORMAT_VERSION);
|
||||
assert_eq!(
|
||||
acquisition.transaction().payload().content_hash().as_bytes(),
|
||||
&[
|
||||
34, 7, 146, 210, 177, 93, 38, 47, 218, 36, 44, 178, 32, 119, 78, 233, 221, 239, 254, 191, 4, 220, 250, 218, 188, 248, 239, 118, 169, 177, 167, 195
|
||||
]
|
||||
);
|
||||
assert_eq!(acquisition.transaction().slot(), 123);
|
||||
assert_eq!(acquisition.transaction().block_time().map(|value| return value.unix_millis()), std::option::Option::Some(1_700_000_000_000));
|
||||
assert_eq!(acquisition.transaction().reference().network().as_str(), "devnet");
|
||||
assert_eq!(acquisition.transaction().reference().signature().as_bytes(), &[0_u8; 64]);
|
||||
let provenance = acquisition.observation().provenance();
|
||||
assert_eq!(provenance.provider().as_str(), "publicnode");
|
||||
assert_eq!(provenance.protocol().as_str(), "solana.http.json_rpc");
|
||||
assert_eq!(provenance.acquisition_method().as_str(), "getTransaction");
|
||||
assert_eq!(provenance.endpoint_id().map(ksp_store_lib::RawProvenanceCode::as_str), std::option::Option::Some("devnet-primary"));
|
||||
assert_eq!(provenance.commitment().map(ksp_store_lib::RawProvenanceCode::as_str), std::option::Option::Some("finalized"));
|
||||
assert_eq!(provenance.capture_session_id().map(ksp_store_lib::RawProvenanceCode::as_str), std::option::Option::Some("backfill:pre006"));
|
||||
assert_eq!(provenance.received_at(), received_at);
|
||||
assert!(provenance.source_payload_hash().is_none());
|
||||
assert!(provenance.source_payload_size_bytes().is_none());
|
||||
return;
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn pre_006_wire_omission_and_null_produce_distinct_canonical_bytes() {
|
||||
let transaction = ksp_onchain_transport_lib::SolanaEncodedTransaction::Binary {
|
||||
data: "AQID".to_owned(),
|
||||
encoding: ksp_onchain_transport_lib::SolanaTransactionBinaryEncoding::Base64,
|
||||
};
|
||||
let meta_omitted = ksp_onchain_transport_lib::SolanaWireField::Omitted;
|
||||
let meta_null = ksp_onchain_transport_lib::SolanaWireField::Null;
|
||||
let version_omitted = ksp_onchain_transport_lib::SolanaWireField::Omitted;
|
||||
let version_null = ksp_onchain_transport_lib::SolanaWireField::Null;
|
||||
let transaction_index_omitted = ksp_onchain_transport_lib::SolanaWireField::Omitted;
|
||||
let transaction_index_null = ksp_onchain_transport_lib::SolanaWireField::Null;
|
||||
let omitted = super::canonical_payload_bytes(&fields(&transaction, &meta_omitted, &version_omitted, &transaction_index_omitted, std::option::Option::None));
|
||||
assert!(omitted.is_ok());
|
||||
let omitted = match omitted {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(_) => return,
|
||||
};
|
||||
assert_eq!(omitted, b"{\"transaction\":[\"AQID\",\"base64\"]}");
|
||||
let nulls = super::canonical_payload_bytes(&fields(&transaction, &meta_null, &version_null, &transaction_index_null, std::option::Option::None));
|
||||
assert!(nulls.is_ok());
|
||||
let nulls = match nulls {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(_) => return,
|
||||
};
|
||||
assert_eq!(nulls, b"{\"transaction\":[\"AQID\",\"base64\"],\"meta\":null,\"version\":null,\"transactionIndex\":null}");
|
||||
assert_ne!(omitted, nulls);
|
||||
return;
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn pre_006_non_base64_transaction_shapes_are_rejected() {
|
||||
let base58 = ksp_onchain_transport_lib::SolanaEncodedTransaction::Binary {
|
||||
data: "1111".to_owned(),
|
||||
encoding: ksp_onchain_transport_lib::SolanaTransactionBinaryEncoding::Base58,
|
||||
};
|
||||
let legacy = ksp_onchain_transport_lib::SolanaEncodedTransaction::LegacyBinary("1111".to_owned());
|
||||
let json = ksp_onchain_transport_lib::SolanaEncodedTransaction::Json(serde_json::json!({"message": {}}));
|
||||
let meta = ksp_onchain_transport_lib::SolanaWireField::Omitted;
|
||||
let version = ksp_onchain_transport_lib::SolanaWireField::Omitted;
|
||||
let transaction_index = ksp_onchain_transport_lib::SolanaWireField::Omitted;
|
||||
for transaction in [&base58, &legacy, &json] {
|
||||
let result = super::canonical_payload_bytes(&fields(transaction, &meta, &version, &transaction_index, std::option::Option::None));
|
||||
assert!(result.is_err());
|
||||
if let std::result::Result::Err(error) = result {
|
||||
assert_eq!(error.code(), crate::ERROR_CODE_BACKFILL_RAW_CONVERSION_INVALID);
|
||||
}
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn pre_006_negative_and_unrepresentable_block_times_are_terminal_conversion_errors() {
|
||||
let negative = super::convert_block_time(std::option::Option::Some(-1));
|
||||
assert!(negative.is_err());
|
||||
let oversized = super::convert_block_time(std::option::Option::Some(i64::MAX));
|
||||
assert!(oversized.is_err());
|
||||
let absent = super::convert_block_time(std::option::Option::None);
|
||||
assert!(matches!(absent, std::result::Result::Ok(std::option::Option::None)));
|
||||
return;
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn pre_006_observation_key_is_deterministic_and_endpoint_specific() {
|
||||
let signature = match signature_text() {
|
||||
std::option::Option::Some(value) => value,
|
||||
std::option::Option::None => return,
|
||||
};
|
||||
let request = match request("devnet", signature.clone()) {
|
||||
std::option::Option::Some(value) => value,
|
||||
std::option::Option::None => return,
|
||||
};
|
||||
let candidate = match candidate("devnet", signature) {
|
||||
std::option::Option::Some(value) => value,
|
||||
std::option::Option::None => return,
|
||||
};
|
||||
let reference = match super::canonical_reference(&request, &candidate) {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(_) => return,
|
||||
};
|
||||
let first = super::observation_key(&request, &reference, "provider", "endpoint-a");
|
||||
let same = super::observation_key(&request, &reference, "provider", "endpoint-a");
|
||||
let other_endpoint = super::observation_key(&request, &reference, "provider", "endpoint-b");
|
||||
let other_provider = super::observation_key(&request, &reference, "provider-2", "endpoint-a");
|
||||
assert_eq!(first, same);
|
||||
assert_ne!(first, other_endpoint);
|
||||
assert_ne!(first, other_provider);
|
||||
return;
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn pre_006_candidate_network_mismatch_is_rejected_before_transport() {
|
||||
let signature = match signature_text() {
|
||||
std::option::Option::Some(value) => value,
|
||||
std::option::Option::None => return,
|
||||
};
|
||||
let request = match request("devnet", signature.clone()) {
|
||||
std::option::Option::Some(value) => value,
|
||||
std::option::Option::None => return,
|
||||
};
|
||||
let candidate = match candidate("mainnet", signature) {
|
||||
std::option::Option::Some(value) => value,
|
||||
std::option::Option::None => return,
|
||||
};
|
||||
let reference = super::canonical_reference(&request, &candidate);
|
||||
assert!(reference.is_err());
|
||||
if let std::result::Result::Err(error) = reference {
|
||||
assert_eq!(error.code(), crate::ERROR_CODE_BACKFILL_RAW_CONVERSION_INVALID);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn pre_006_missing_outcome_contains_only_network_scoped_reference() {
|
||||
let signature = match signature_text() {
|
||||
std::option::Option::Some(value) => value,
|
||||
std::option::Option::None => return,
|
||||
};
|
||||
let request = match request("devnet", signature.clone()) {
|
||||
std::option::Option::Some(value) => value,
|
||||
std::option::Option::None => return,
|
||||
};
|
||||
let candidate = match candidate("devnet", signature) {
|
||||
std::option::Option::Some(value) => value,
|
||||
std::option::Option::None => return,
|
||||
};
|
||||
let reference = match super::canonical_reference(&request, &candidate) {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(_) => return,
|
||||
};
|
||||
let outcome = crate::BackfillHydrationOutcome::Missing(reference);
|
||||
assert!(outcome.is_missing());
|
||||
assert_eq!(outcome.reference().network().as_str(), "devnet");
|
||||
assert_eq!(outcome.reference().signature().as_bytes(), &[0_u8; 64]);
|
||||
return;
|
||||
}
|
||||
272
deltas/0.3.6/pre.006.md
Normal file
272
deltas/0.3.6/pre.006.md
Normal file
@@ -0,0 +1,272 @@
|
||||
<!-- file: deltas/0.3.6/pre.006.md -->
|
||||
<!-- version: 1 -->
|
||||
|
||||
# Delta `0.3.6-pre.006` — conversion RAW transaction v1 et provenance observée
|
||||
|
||||
## Base requise
|
||||
|
||||
```text
|
||||
0.3.6-pre.005-fix.001 appliquée
|
||||
workspace.package.version = 0.3.6-pre.5.fix.1
|
||||
```
|
||||
|
||||
La base opérateur inclut également la synchronisation manuelle des versions d'en-tête omise dans l'archive initiale du fix :
|
||||
|
||||
```text
|
||||
Cargo.toml header 400 -> 401
|
||||
crates/ksp-job-backfill-lib/src/discovery.rs header 1 -> 2
|
||||
```
|
||||
|
||||
Le gate opérateur fourni pour cette base confirme :
|
||||
|
||||
```text
|
||||
cargo fmt --all PASS
|
||||
python3 scripts/audit_rust_workspace_rules.py PASS / clean
|
||||
python3 scripts/audit_markdown_tables.py ... PASS / clean (264 tables / 148 fichiers)
|
||||
cargo check --workspace PASS
|
||||
cargo clippy --workspace --all-targets PASS
|
||||
cargo test -p ksp-job-backfill-lib PASS
|
||||
unitaires 11 PASS
|
||||
dependency_boundary 2 PASS
|
||||
public_api 2 PASS
|
||||
release_completeness 2 PASS
|
||||
cargo tree -p ksp-job-backfill-lib --edges normal exécuté
|
||||
cargo tree -p ksp-job-backfill-lib -e features exécuté
|
||||
```
|
||||
|
||||
## Objectif
|
||||
|
||||
Matérialiser exclusivement la tranche `pre.006` du plan 027 : convertir un candidat Backfill déjà découvert en transaction RAW canonique v1 et en observation d'acquisition sûre à partir du chemin Transport observé `getTransaction`, sans encore persister dans Store ni ouvrir le runtime de Job.
|
||||
|
||||
L'identité logique reste strictement :
|
||||
|
||||
```text
|
||||
RawTransactionReference = (RawNetworkId, RawTransactionSignature)
|
||||
```
|
||||
|
||||
Provider, endpoint, protocole, méthode Transport et `JobId` ne deviennent jamais une identité de transaction. Ils servent uniquement à caractériser l'acquisition et, lorsqu'une transaction existe, l'identité déterministe de son observation.
|
||||
|
||||
## Signature canonique
|
||||
|
||||
`BackfillSignature::to_raw_transaction_signature` ajoute le passage de la forme Base58 bornée de `pre.005` vers `RawTransactionSignature` exactement 64 octets.
|
||||
|
||||
Le décodeur Base58 est privé, borné et spécialisé pour cette frontière :
|
||||
|
||||
- alphabet Base58 Solana exact ;
|
||||
- exactement 64 octets décodés requis ;
|
||||
- zéros initiaux `1` conservés ;
|
||||
- dépassement arithmétique rejeté ;
|
||||
- aucune dépendance SDK/protocolaire Solana ;
|
||||
- aucune nouvelle dépendance Base58 externe.
|
||||
|
||||
La conversion ne modifie pas l'identité réseau du candidat et rejette avant Transport tout candidat dont le réseau diffère de la requête.
|
||||
|
||||
## Hydratation Transport observée
|
||||
|
||||
La voie publique :
|
||||
|
||||
```text
|
||||
hydrate_backfill_candidate
|
||||
```
|
||||
|
||||
appelle exclusivement :
|
||||
|
||||
```text
|
||||
HttpTransportPool::get_transaction_observed
|
||||
```
|
||||
|
||||
avec :
|
||||
|
||||
```text
|
||||
encoding = base64
|
||||
commitment = engagement explicite du Backfill
|
||||
maxSupportedTransactionVersion = 0
|
||||
```
|
||||
|
||||
Transport reste propriétaire du routage, retry, pacing, cooldown et endpoint victorieux. Job n'introduit aucun second client ni boucle de retry.
|
||||
|
||||
Deux outcomes sont distingués :
|
||||
|
||||
- `Available(BackfillRawAcquisition)` : transaction présente, RAW v1 et observation construits en mémoire ;
|
||||
- `Missing(RawTransactionReference)` : JSON RPC `result: null`, aucune provenance ni observation fabriquée.
|
||||
|
||||
## RAW transaction v1
|
||||
|
||||
Le format KSP figé par la tranche est :
|
||||
|
||||
```text
|
||||
format_id = ksp.solana.raw_transaction
|
||||
format_version = 1
|
||||
```
|
||||
|
||||
Le payload canonique est du JSON UTF-8 compact déterministe. La forme top-level est construite dans cet ordre :
|
||||
|
||||
1. `transaction` ;
|
||||
2. `meta` si le champ wire n'est pas omitted ;
|
||||
3. `version` si le champ wire n'est pas omitted ;
|
||||
4. `transactionIndex` si le champ wire n'est pas omitted.
|
||||
|
||||
La transaction est conservée sans décodage métier sous la forme :
|
||||
|
||||
```json
|
||||
["<base64>","base64"]
|
||||
```
|
||||
|
||||
Les objets JSON imbriqués sont canonisés récursivement par tri lexical des clés ; l'ordre des tableaux reste intact. Les états wire `Omitted`, `Null` et `Value` de `meta`, `version` et `transactionIndex` restent distincts.
|
||||
|
||||
`slot` et `blockTime` appartiennent aux champs structurés de `RawTransaction` et ne sont pas dupliqués dans le payload. Un `blockTime` négatif ou non représentable dans `RawTimestamp` est une erreur de conversion terminale.
|
||||
|
||||
SHA-256 et `byte_len` sont calculés sur les octets canoniques exacts avant construction de `RawPayload`.
|
||||
|
||||
Les réponses transactionnelles non Base64 sont rejetées ; Job ne bascule pas vers une interprétation JSON/jsonParsed ou fournisseur.
|
||||
|
||||
## Provenance d'acquisition
|
||||
|
||||
Une transaction disponible produit `RawAcquisitionProvenance` avec :
|
||||
|
||||
- provider réellement victorieux ;
|
||||
- protocole sûr `solana.http.json_rpc` ;
|
||||
- méthode `getTransaction` ;
|
||||
- endpoint sûr réellement victorieux ;
|
||||
- engagement demandé ;
|
||||
- `JobId` comme capture session id ;
|
||||
- origine `Backfill` ;
|
||||
- timestamp de réception fourni par l'hôte.
|
||||
|
||||
Aucun URL, header, secret, body HTTP ou payload fournisseur brut n'est copié dans la provenance.
|
||||
|
||||
La clé `RawTransactionObservationKey` utilise SHA-256 avec séparation de domaine et couvre :
|
||||
|
||||
- `JobId` ;
|
||||
- fingerprint du scope ;
|
||||
- signature RAW 64 octets ;
|
||||
- provider ;
|
||||
- endpoint ;
|
||||
- engagement ;
|
||||
- version du contrat d'observation.
|
||||
|
||||
Même réseau + même signature + endpoint différent représente donc toujours la même transaction logique mais une observation d'acquisition distincte.
|
||||
|
||||
## Dépendances
|
||||
|
||||
La crate conserve ses dépendances KSP de `pre.005` et ajoute uniquement :
|
||||
|
||||
```text
|
||||
serde_json.workspace = true
|
||||
```
|
||||
|
||||
`serde_json` est déjà centralisé dans `[workspace.dependencies]`. `sha2` reste la primitive SHA-256 déjà présente. Aucune dépendance `bs58`, SDK Solana, Store API/backend, Config, reqwest ou tonic n'est ajoutée.
|
||||
|
||||
## Tests matérialisés
|
||||
|
||||
La tranche ajoute **8 tests unitaires**, portant le total de `ksp-job-backfill-lib` à **19 unitaires** :
|
||||
|
||||
- décodage Base58 exact 64 octets, longueur excessive et overflow ;
|
||||
- golden payload canonique, longueur, SHA-256 et provenance ;
|
||||
- distinction omitted/null des champs wire ;
|
||||
- rejet des transactions non Base64 ;
|
||||
- block times négatifs ou non représentables ;
|
||||
- clé d'observation déterministe et sensible à provider/endpoint ;
|
||||
- mismatch réseau rejeté avant Transport ;
|
||||
- outcome Missing limité à la référence réseau + signature.
|
||||
|
||||
Une canarie publique supplémentaire porte les canaries d'intégration à **7** :
|
||||
|
||||
- `dependency_boundary` : 2 ;
|
||||
- `public_api` : 3 ;
|
||||
- `release_completeness` : 2.
|
||||
|
||||
Les canaries verrouillent en outre l'usage du chemin observé, `base64`, `maxSupportedTransactionVersion = 0`, l'absence de persistance et les firewalls de dépendances.
|
||||
|
||||
## Fichiers ajoutés
|
||||
|
||||
```text
|
||||
crates/ksp-job-backfill-lib/src/conversion.rs
|
||||
crates/ksp-job-backfill-lib/unit_tests/conversion.rs
|
||||
deltas/0.3.6/pre.006.md
|
||||
```
|
||||
|
||||
## Fichiers modifiés
|
||||
|
||||
```text
|
||||
Cargo.toml
|
||||
crates/ksp-job-backfill-lib/Cargo.toml
|
||||
crates/ksp-job-backfill-lib/src/error.rs
|
||||
crates/ksp-job-backfill-lib/src/lib.rs
|
||||
crates/ksp-job-backfill-lib/src/request.rs
|
||||
crates/ksp-job-backfill-lib/tests/dependency_boundary.rs
|
||||
crates/ksp-job-backfill-lib/tests/public_api.rs
|
||||
crates/ksp-job-backfill-lib/tests/release_completeness.rs
|
||||
docs/plans/027-V0_3_6_JOB_API_BACKFILL_PLAN.md
|
||||
docs/validation/023-V0_3_6_JOB_API_BACKFILL.md
|
||||
```
|
||||
|
||||
## Versions d'en-tête
|
||||
|
||||
Tous les fichiers Rust/TOML modifiés par cette tranche incrémentent leur version d'en-tête par rapport à la base opérateur corrigée :
|
||||
|
||||
```text
|
||||
Cargo.toml 401 -> 402
|
||||
crates/ksp-job-backfill-lib/Cargo.toml 1 -> 2
|
||||
crates/ksp-job-backfill-lib/src/error.rs 1 -> 2
|
||||
crates/ksp-job-backfill-lib/src/lib.rs 1 -> 2
|
||||
crates/ksp-job-backfill-lib/src/request.rs 1 -> 2
|
||||
crates/ksp-job-backfill-lib/tests/dependency_boundary.rs 1 -> 2
|
||||
crates/ksp-job-backfill-lib/tests/public_api.rs 1 -> 2
|
||||
crates/ksp-job-backfill-lib/tests/release_completeness.rs 1 -> 2
|
||||
```
|
||||
|
||||
Les nouveaux fichiers Rust commencent à `version: 1`. Les documents Markdown modifiés incrémentent eux aussi leur en-tête documentaire : plan 027 et validation 023 passent de `9` à `10`.
|
||||
|
||||
## Version workspace
|
||||
|
||||
La tranche modifie du code Rust et le manifeste runtime de la crate :
|
||||
|
||||
```text
|
||||
workspace.package.version = 0.3.6-pre.6
|
||||
delivery = 0.3.6-pre.006
|
||||
commit = v0.3.6-pre.006
|
||||
```
|
||||
|
||||
Aucun tag prerelease.
|
||||
|
||||
## Fichiers supprimés
|
||||
|
||||
Aucun.
|
||||
|
||||
## Frontières conservées
|
||||
|
||||
- aucune persistance Store ;
|
||||
- aucune prélecture Store avant hydratation ;
|
||||
- aucun checkpoint ou frontier contiguë ;
|
||||
- aucune concurrence d'hydratation concrète ;
|
||||
- aucun snapshot Job concret ;
|
||||
- aucune Config/env ;
|
||||
- aucun Worker ou exécutable ;
|
||||
- aucune modification Transport ou Store ;
|
||||
- aucun endpoint/provider/protocole dans l'identité transactionnelle ;
|
||||
- aucun README, USAGE, CHANGELOG ou ROADMAP rouvert ;
|
||||
- aucun code kbot3 copié.
|
||||
|
||||
## Validations dans l'environnement d'assemblage
|
||||
|
||||
L'environnement d'assemblage exécute les audits statiques du dépôt et les contrôles d'archive. Il ne possède ni `cargo`, ni `rustc`, ni `rustfmt`; le gate Rust de `pre.006` doit donc être rejoué par l'opérateur avant clôture.
|
||||
|
||||
## Gate opérateur demandé
|
||||
|
||||
```bash
|
||||
cargo fmt --all
|
||||
python3 scripts/audit_rust_workspace_rules.py
|
||||
python3 scripts/audit_markdown_tables.py README.md RULES.md ROADMAP.md CHANGELOG.md docs prompts crates deltas/0.3.6
|
||||
cargo check --workspace
|
||||
cargo clippy --workspace --all-targets
|
||||
cargo test -p ksp-job-backfill-lib
|
||||
cargo tree -p ksp-job-backfill-lib --edges normal
|
||||
cargo tree -p ksp-job-backfill-lib -e features
|
||||
```
|
||||
|
||||
Résultat attendu : **19 tests unitaires + 7 canaries d'intégration**, sans persistance Store et avec un graphe normal qui conserve `ksp-store-lib` sans backend imposé.
|
||||
|
||||
## Questions ouvertes
|
||||
|
||||
Aucune pour `pre.006`. La persistance atomique Store et ses outcomes restent réservés à `pre.007` après gate vert de cette tranche.
|
||||
@@ -1,5 +1,5 @@
|
||||
<!-- file: docs/plans/027-V0_3_6_JOB_API_BACKFILL_PLAN.md -->
|
||||
<!-- version: 9 -->
|
||||
<!-- version: 10 -->
|
||||
|
||||
# Plan v0.3.6 — Job API et premier backfill RAW
|
||||
|
||||
@@ -343,7 +343,7 @@ Graphe prévu :
|
||||
|
||||
Aucun document Config `std.jobs.*` n'est introduit. Les réglages de Job sont fournis explicitement par l'hôte ; la composition Config appartient à l'application v0.3.7.
|
||||
|
||||
La dépendance Base58 exacte sera ajoutée seulement après audit de version et de features lors de la tranche de conversion.
|
||||
L'audit de `pre.006` conclut qu'aucune dépendance Base58 externe n'est nécessaire pour cette frontière étroite : un décodeur privé borné à exactement 64 octets suffit, évite une dépendance publique ou protocolaire supplémentaire et reste couvert par les golden tests. `serde_json`, déjà centralisé au workspace, devient la seule dépendance normale additionnelle de la crate pour l'écriture canonique et récursive du JSON.
|
||||
|
||||
## 20. Stratégie de tests et canaries
|
||||
|
||||
@@ -434,7 +434,7 @@ Les tests couvrent le succès direct `null`, la surface publique et surtout un r
|
||||
|
||||
### `pre.005` — Fondation Backfill et découverte
|
||||
|
||||
**Statut : réalisé ; corrigé par `pre.005-fix.001`, gate opérateur du fix à rejouer.**
|
||||
**Statut : clôturé ; corrigé par `pre.005-fix.001`, gate opérateur vert.**
|
||||
|
||||
Budget cible : **15-20 min**. Entrée : Transport observé stable et gate `pre.004` vert. La tranche crée `ksp-job-backfill-lib` avec dépendances directes Job API, Core, Logging, Onchain Transport, Store façade sans feature backend imposée et SHA-256 déjà centralisé au workspace ; Tokio reste uniquement une dev-dependency pour les tests async de cette tranche.
|
||||
|
||||
@@ -444,15 +444,19 @@ La requête impose explicitement `JobId`, `RawNetworkId`, rôle HTTP, engagement
|
||||
|
||||
#### `pre.005-fix.001` — Return explicite dans la closure du curseur Before
|
||||
|
||||
**Statut : matérialisé ; gate opérateur à rejouer.**
|
||||
**Statut : clôturé ; gate opérateur vert.**
|
||||
|
||||
Le gate opérateur de `pre.005` confirme `cargo fmt`, les audits Rust/Markdown, `cargo check --workspace`, les 11 tests unitaires, les 6 canaries d’intégration et les arbres Cargo. `cargo clippy --workspace --all-targets` échoue uniquement sur `clippy::implicit_return` à la construction optionnelle du curseur `before` dans `discover_older`. Le fix remplace l'expression implicite de la closure par `return value.as_str().to_owned()` sans modifier la valeur produite, les branches de scope, la pagination ou le contrat public. Comme un fichier Rust est corrigé, la version workspace devient `0.3.6-pre.5.fix.1`.
|
||||
Le gate opérateur de `pre.005` confirme `cargo fmt`, les audits Rust/Markdown, `cargo check --workspace`, les 11 tests unitaires, les 6 canaries d’intégration et les arbres Cargo. `cargo clippy --workspace --all-targets` échoue uniquement sur `clippy::implicit_return` à la construction optionnelle du curseur `before` dans `discover_older`. Le fix remplace l'expression implicite de la closure par `return value.as_str().to_owned()` sans modifier la valeur produite, les branches de scope, la pagination ou le contrat public. Comme un fichier Rust est corrigé, la version workspace devient `0.3.6-pre.5.fix.1`. Le gate du fix est ensuite intégralement vert : `cargo fmt`, audits Rust/Markdown, `cargo check`, Clippy, 11 unitaires, 6 canaries et arbres Cargo passent sur `0.3.6-pre.5.fix.1`. Les versions d'en-tête du `Cargo.toml` racine et du fichier Rust corrigé sont synchronisées dans la base opérateur avant ouverture de `pre.006`.
|
||||
|
||||
### `pre.006` — Conversion RAW v1 et provenance
|
||||
|
||||
**Statut : planifié.**
|
||||
**Statut : matérialisé ; gate opérateur à rejouer.**
|
||||
|
||||
Budget cible : **15-20 min**. Entrée : candidats déterministes. Implémenter décodage Base58 minimal, payload canonique et provenance. Sortie : golden bytes, hash, absent contre `null` et block time hostile couverts.
|
||||
Budget cible : **15-20 min**. Entrée : candidats déterministes et gate `pre.005-fix.001` vert. La tranche ajoute une conversion RAW v1 strictement en mémoire : `BackfillSignature::to_raw_transaction_signature` décode Base58 vers exactement 64 octets sans SDK Solana ni nouvelle dépendance Base58 ; `hydrate_backfill_candidate` appelle uniquement `HttpTransportPool::get_transaction_observed` avec `base64`, engagement explicite et `maxSupportedTransactionVersion = 0`. Un résultat `null` devient `BackfillHydrationOutcome::Missing(RawTransactionReference)` et ne contient structurellement aucune provenance.
|
||||
|
||||
Le payload `ksp.solana.raw_transaction` version `1` conserve la transaction comme tuple `[data, "base64"]`, omet `slot` et `blockTime`, préserve les états omitted/null/value de `meta`, `version` et `transactionIndex`, et canonise récursivement les objets JSON par tri lexical des clés avant calcul SHA-256. Les block times négatifs ou hors plage sont rejetés. Une acquisition disponible produit `RawTransaction` et `RawTransactionObservation` avec provider, protocole `solana.http.json_rpc`, méthode `getTransaction`, endpoint victorieux, engagement, `JobId` et timestamp de réception. La clé d'observation est domain-separated et dépend du JobId, fingerprint, signature, provider, endpoint, engagement et version du contrat ; changer d'endpoint crée donc une observation distincte sans changer l'identité `(network, signature)`.
|
||||
|
||||
La tranche matérialise 8 nouveaux tests unitaires de conversion, portant le total Backfill à 19 unitaires, et une canarie publique supplémentaire, portant les canaries d'intégration à 7. Aucun appel de persistance Store, checkpoint, concurrence runtime ou snapshot concret n'est ouvert ; ces responsabilités restent aux tranches suivantes.
|
||||
|
||||
### `pre.007` — Persistance Store et idempotence
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
<!-- file: docs/validation/023-V0_3_6_JOB_API_BACKFILL.md -->
|
||||
<!-- version: 9 -->
|
||||
<!-- version: 10 -->
|
||||
|
||||
# Validation v0.3.6 — Job API et premier backfill RAW
|
||||
|
||||
@@ -60,6 +60,8 @@ Aucune entrée absolue, traversée, avec séparateur inversé ou lien symbolique
|
||||
- [X] Gate opérateur de `pre.005` : `cargo fmt`, audits Rust/Markdown et `cargo check --workspace` verts ; les 11 tests unitaires et 6 canaries `ksp-job-backfill-lib` sont verts.
|
||||
- [X] Le même gate révèle un unique échec Clippy `clippy::implicit_return` dans la closure privée qui dérive le curseur `before`; les arbres Cargo normal/features sont néanmoins exécutés et cohérents avec le graphe prévu.
|
||||
- [X] `pre.005-fix.001` ajoute uniquement le `return` explicite exigé dans cette closure et ne modifie ni valeur, ni pagination, ni identité réseau, ni dépendance, ni API publique.
|
||||
- [X] Gate opérateur de `pre.005-fix.001` : `cargo fmt`, audits Rust/Markdown, `cargo check`, Clippy, 11 unitaires + 6 canaries `ksp-job-backfill-lib` et arbres Cargo normal/features verts sur `0.3.6-pre.5.fix.1`.
|
||||
- [X] Versions d'en-tête du `Cargo.toml` racine et de `discovery.rs` corrigées dans la base opérateur avant `pre.006`.
|
||||
|
||||
## 4. Autorités et cohérence documentaire
|
||||
|
||||
@@ -100,30 +102,30 @@ Aucune entrée absolue, traversée, avec séparateur inversé ou lien symbolique
|
||||
|
||||
## 7. Admission et portées
|
||||
|
||||
`pre.005` matérialise les contrats et leurs tests. Le gate opérateur confirme les 11 unitaires et 6 canaries, tandis que Clippy impose un fix syntaxique sans effet comportemental avant clôture complète. L'identité candidate est `(RawNetworkId, signature)` ; rôle HTTP, provider, endpoint et protocole ne participent ni à cette identité ni au fingerprint sémantique du scope.
|
||||
`pre.005` matérialise les contrats et leurs tests. Le gate du fix confirme les 11 unitaires et 6 canaries avec Clippy vert. L'identité candidate est `(RawNetworkId, signature)` ; rôle HTTP, provider, endpoint et protocole ne participent ni à cette identité ni au fingerprint sémantique du scope.
|
||||
|
||||
- [ ] `LatestAddress`, `BeforeAddress`, `AfterAddress` et `ExplicitSignatures` couverts.
|
||||
- [ ] Adresse validée par le type Core existant.
|
||||
- [X] `LatestAddress`, `BeforeAddress`, `AfterAddress` et `ExplicitSignatures` couverts.
|
||||
- [X] Adresse validée par le type Core existant.
|
||||
- [ ] Signature Base58 décodée vers exactement 64 octets.
|
||||
- [ ] Engagement limité à `Confirmed` ou `Finalized`.
|
||||
- [ ] Taille de page entre 1 et 1 000.
|
||||
- [ ] Nombre de pages entre 1 et 10 000.
|
||||
- [ ] Nombre de candidats entre 1 et 10 000.
|
||||
- [ ] Concurrence d'hydratation entre 1 et 64.
|
||||
- [ ] `min_context_slot` transmis sans en changer le sens.
|
||||
- [X] Engagement limité à `Confirmed` ou `Finalized`.
|
||||
- [X] Taille de page entre 1 et 1 000.
|
||||
- [X] Nombre de pages entre 1 et 10 000.
|
||||
- [X] Nombre de candidats entre 1 et 10 000.
|
||||
- [X] Concurrence d'hydratation entre 1 et 64.
|
||||
- [X] `min_context_slot` transmis sans en changer le sens.
|
||||
- [ ] Checkpoint rejeté si son JobId ou son fingerprint de scope diffère.
|
||||
- [ ] Aucun défaut d'application silencieux dans la bibliothèque.
|
||||
- [X] Aucun défaut d'application silencieux dans la bibliothèque.
|
||||
|
||||
## 8. Pagination, ordre et déduplication
|
||||
|
||||
- [ ] Latest respecte l'ordre officiel du plus récent au plus ancien.
|
||||
- [ ] Before utilise une ancre exclusive et avance vers l'historique plus ancien.
|
||||
- [ ] After atteint l'ancre, garde les plus proches candidats plus récents et produit un ordre déterministe.
|
||||
- [ ] Ancre before absente acceptée comme curseur.
|
||||
- [ ] Ancre after non atteinte bornée et signalée comme résultat partiel.
|
||||
- [ ] Doublons inter-pages retirés à première occurrence.
|
||||
- [ ] Doublons explicites retirés sans réordonner la liste.
|
||||
- [ ] Limites de pages et candidats respectées sur tous les chemins.
|
||||
- [X] Latest respecte l'ordre officiel du plus récent au plus ancien.
|
||||
- [X] Before utilise une ancre exclusive et avance vers l'historique plus ancien.
|
||||
- [X] After atteint l'ancre, garde les plus proches candidats plus récents et produit un ordre déterministe.
|
||||
- [X] Ancre before absente acceptée comme curseur.
|
||||
- [X] Ancre after non atteinte bornée et signalée comme résultat partiel.
|
||||
- [X] Doublons inter-pages retirés à première occurrence.
|
||||
- [X] Doublons explicites retirés sans réordonner la liste.
|
||||
- [X] Limites de pages et candidats respectées sur tous les chemins.
|
||||
|
||||
## 9. Acquisition Transport et retry
|
||||
|
||||
@@ -137,6 +139,8 @@ Aucune entrée absolue, traversée, avec séparateur inversé ou lien symbolique
|
||||
|
||||
## 10. RAW v1 et provenance
|
||||
|
||||
`pre.006` matérialise cette surface et ses tests statiques/unitaires ; les cases ci-dessous restent ouvertes jusqu'au gate Cargo opérateur de la tranche.
|
||||
|
||||
- [X] Identité logique future auditée : transaction/signature scoping par `RawNetworkId`, jamais par rôle/provider/endpoint/protocole.
|
||||
- [X] Le Store PostgreSQL actuel est mono-réseau via `ksp_store_identity`; un futur backend multi-réseaux devra préserver `(network, signature)` par clé/partition équivalente.
|
||||
- [ ] Format `ksp.solana.raw_transaction`, version `1`, figé.
|
||||
|
||||
Reference in New Issue
Block a user