v0.3.14-pre.002
This commit is contained in:
@@ -1,12 +1,12 @@
|
|||||||
# file: Cargo.toml
|
# file: Cargo.toml
|
||||||
# version: 566
|
# version: 567
|
||||||
|
|
||||||
[workspace]
|
[workspace]
|
||||||
resolver = "3"
|
resolver = "3"
|
||||||
members = ["crates/ksp-app-backfill-desk", "crates/ksp-app-config-desk", "crates/ksp-app-solprices-desk", "crates/ksp-app-store-desk", "crates/ksp-app-wallet-desk", "crates/ksp-config-lib", "crates/ksp-core-lib", "crates/ksp-interface-lib", "crates/ksp-job-api", "crates/ksp-job-backfill-lib", "crates/ksp-logging-lib", "crates/ksp-offchain-transport-lib", "crates/ksp-onchain-transport-lib", "crates/ksp-program-api", "crates/ksp-raw-transaction-lib", "crates/ksp-store-api", "crates/ksp-store-lib", "crates/ksp-store-postgres-lib", "crates/ksp-wallet-lib", "crates/ksp-worker-api", "crates/ksp-worker-raw-transaction-ingest-lib"]
|
members = ["crates/ksp-app-backfill-desk", "crates/ksp-app-config-desk", "crates/ksp-app-solprices-desk", "crates/ksp-app-store-desk", "crates/ksp-app-wallet-desk", "crates/ksp-config-lib", "crates/ksp-core-lib", "crates/ksp-interface-lib", "crates/ksp-job-api", "crates/ksp-job-backfill-lib", "crates/ksp-logging-lib", "crates/ksp-offchain-transport-lib", "crates/ksp-onchain-transport-lib", "crates/ksp-program-api", "crates/ksp-raw-transaction-lib", "crates/ksp-store-api", "crates/ksp-store-lib", "crates/ksp-store-postgres-lib", "crates/ksp-wallet-lib", "crates/ksp-worker-api", "crates/ksp-worker-raw-transaction-ingest-lib"]
|
||||||
|
|
||||||
[workspace.package]
|
[workspace.package]
|
||||||
version = "0.3.14-pre.1"
|
version = "0.3.14-pre.2"
|
||||||
edition = "2024"
|
edition = "2024"
|
||||||
license = "MIT"
|
license = "MIT"
|
||||||
repository = "https://git.sasedev.com/Sasedev/khadhroony-solana-project"
|
repository = "https://git.sasedev.com/Sasedev/khadhroony-solana-project"
|
||||||
|
|||||||
435
crates/ksp-worker-raw-transaction-ingest-lib/src/continuity.rs
Normal file
435
crates/ksp-worker-raw-transaction-ingest-lib/src/continuity.rs
Normal file
@@ -0,0 +1,435 @@
|
|||||||
|
// file: crates/ksp-worker-raw-transaction-ingest-lib/src/continuity.rs
|
||||||
|
// version: 1
|
||||||
|
|
||||||
|
/// Maximum number of simultaneously retained non-repaired run-local gaps.
|
||||||
|
pub(crate) const MAX_RAW_TRANSACTION_INGEST_OPEN_REPAIR_GAPS: usize = 64;
|
||||||
|
/// Maximum number of run-local gaps that may actively repair at once.
|
||||||
|
pub(crate) const MAX_RAW_TRANSACTION_INGEST_REPAIR_ACTIVE_GAPS: usize = 1;
|
||||||
|
/// Maximum number of logical block fetches that a later repair scheduler may admit concurrently.
|
||||||
|
pub(crate) const MAX_RAW_TRANSACTION_INGEST_REPAIR_BLOCK_FETCH_IN_FLIGHT: usize = 4;
|
||||||
|
/// Maximum number of slots admitted by one later HTTP repair discovery window.
|
||||||
|
pub(crate) const MAX_RAW_TRANSACTION_INGEST_REPAIR_DISCOVERY_WINDOW_SLOTS: u64 = 512;
|
||||||
|
/// Maximum inclusive slot span admitted for one run-local repair gap.
|
||||||
|
pub(crate) const MAX_RAW_TRANSACTION_INGEST_REPAIR_RANGE_SLOTS: u64 = 4_096;
|
||||||
|
|
||||||
|
/// Private source scope used to prove whether one configured live source can cover another run-local requirement.
|
||||||
|
#[derive(Clone, Debug, Eq, PartialEq)]
|
||||||
|
pub(crate) enum RawTransactionIngestCoverageScope {
|
||||||
|
/// Every transaction contained by every produced block at the configured commitment.
|
||||||
|
FullLedgerTransactions,
|
||||||
|
/// One opaque exact source-family scope identified independently from provider and endpoint identity.
|
||||||
|
ExactSourceScope(&'static str, [u8; 32]),
|
||||||
|
/// A bounded set of references already observed by the Worker; never sufficient as configured target coverage.
|
||||||
|
KnownReferences([u8; 32]),
|
||||||
|
}
|
||||||
|
|
||||||
|
impl crate::RawTransactionIngestCoverageScope {
|
||||||
|
/// Creates one provider-neutral exact source scope from a stable family code and semantic fingerprint.
|
||||||
|
pub(crate) const fn exact_source_scope(family_code: &'static str, fingerprint: [u8; 32]) -> Self {
|
||||||
|
return Self::ExactSourceScope(family_code, fingerprint);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Creates the full-ledger transaction scope used by complete block sources.
|
||||||
|
pub(crate) const fn full_ledger_transactions() -> Self {
|
||||||
|
return Self::FullLedgerTransactions;
|
||||||
|
}
|
||||||
|
|
||||||
|
fn validate_configured_target_scope(&self) -> ksp_core_lib::Result<()> {
|
||||||
|
return match self {
|
||||||
|
Self::FullLedgerTransactions => std::result::Result::Ok(()),
|
||||||
|
Self::ExactSourceScope(family_code, fingerprint) => {
|
||||||
|
if family_code.is_empty() {
|
||||||
|
return std::result::Result::Err(crate::runtime_error("continuity.exact_scope_family_empty"));
|
||||||
|
}
|
||||||
|
let _fingerprint_first_byte = fingerprint[0];
|
||||||
|
std::result::Result::Ok(())
|
||||||
|
},
|
||||||
|
Self::KnownReferences(fingerprint) => {
|
||||||
|
let _fingerprint_first_byte = fingerprint[0];
|
||||||
|
std::result::Result::Err(crate::runtime_error("continuity.known_references_not_target_scope"))
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
fn is_known_references(&self) -> bool {
|
||||||
|
return matches!(self, Self::KnownReferences(_));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Debug, Eq, PartialEq)]
|
||||||
|
struct RawTransactionIngestCoverageRequirement {
|
||||||
|
commitment: ksp_onchain_transport_lib::SolanaCommitment,
|
||||||
|
scope: crate::RawTransactionIngestCoverageScope,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||||
|
enum RawTransactionIngestGapReason {
|
||||||
|
HttpProducedBlockUnavailable,
|
||||||
|
SourceFailure,
|
||||||
|
TransportOverflow,
|
||||||
|
WebSocketReconnect,
|
||||||
|
YellowstoneRetention,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl RawTransactionIngestGapReason {
|
||||||
|
const ALL: [Self; 5] =
|
||||||
|
[Self::HttpProducedBlockUnavailable, Self::SourceFailure, Self::TransportOverflow, Self::WebSocketReconnect, Self::YellowstoneRetention];
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||||
|
enum RawTransactionIngestGapState {
|
||||||
|
Pending,
|
||||||
|
Repairing,
|
||||||
|
Repaired,
|
||||||
|
Unresolved,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl RawTransactionIngestGapState {
|
||||||
|
const ALL: [Self; 4] = [Self::Pending, Self::Repairing, Self::Repaired, Self::Unresolved];
|
||||||
|
|
||||||
|
const fn is_open(self) -> bool {
|
||||||
|
return !matches!(self, Self::Repaired);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||||
|
struct RawTransactionIngestGapRange {
|
||||||
|
end_slot: u64,
|
||||||
|
start_slot: u64,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl RawTransactionIngestGapRange {
|
||||||
|
fn new(start_slot: u64, end_slot: u64) -> ksp_core_lib::Result<Self> {
|
||||||
|
if end_slot < start_slot {
|
||||||
|
return std::result::Result::Err(crate::runtime_error("continuity.gap_range_reversed"));
|
||||||
|
}
|
||||||
|
let slot_count = match end_slot.checked_sub(start_slot).and_then(|value| return value.checked_add(1)) {
|
||||||
|
std::option::Option::Some(value) => value,
|
||||||
|
std::option::Option::None => return std::result::Result::Err(crate::runtime_error("continuity.gap_range_overflow")),
|
||||||
|
};
|
||||||
|
if slot_count > crate::MAX_RAW_TRANSACTION_INGEST_REPAIR_RANGE_SLOTS {
|
||||||
|
return std::result::Result::Err(crate::runtime_error("continuity.gap_range_too_large"));
|
||||||
|
}
|
||||||
|
return std::result::Result::Ok(Self { end_slot, start_slot });
|
||||||
|
}
|
||||||
|
|
||||||
|
const fn end_slot(self) -> u64 {
|
||||||
|
return self.end_slot;
|
||||||
|
}
|
||||||
|
|
||||||
|
fn overlaps_or_is_adjacent(self, other: Self) -> bool {
|
||||||
|
if self.start_slot <= other.end_slot && other.start_slot <= self.end_slot {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
let self_next = self.end_slot.checked_add(1);
|
||||||
|
if self_next == std::option::Option::Some(other.start_slot) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
let other_next = other.end_slot.checked_add(1);
|
||||||
|
return other_next == std::option::Option::Some(self.start_slot);
|
||||||
|
}
|
||||||
|
|
||||||
|
const fn start_slot(self) -> u64 {
|
||||||
|
return self.start_slot;
|
||||||
|
}
|
||||||
|
|
||||||
|
fn try_merge(self, other: Self) -> ksp_core_lib::Result<std::option::Option<Self>> {
|
||||||
|
if !self.overlaps_or_is_adjacent(other) {
|
||||||
|
return std::result::Result::Ok(std::option::Option::None);
|
||||||
|
}
|
||||||
|
let start_slot = self.start_slot.min(other.start_slot);
|
||||||
|
let end_slot = self.end_slot.max(other.end_slot);
|
||||||
|
let slot_count = match end_slot.checked_sub(start_slot).and_then(|value| return value.checked_add(1)) {
|
||||||
|
std::option::Option::Some(value) => value,
|
||||||
|
std::option::Option::None => return std::result::Result::Ok(std::option::Option::None),
|
||||||
|
};
|
||||||
|
if slot_count > crate::MAX_RAW_TRANSACTION_INGEST_REPAIR_RANGE_SLOTS {
|
||||||
|
return std::result::Result::Ok(std::option::Option::None);
|
||||||
|
}
|
||||||
|
return std::result::Result::Ok(std::option::Option::Some(Self { end_slot, start_slot }));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Copy, Debug, Eq, PartialEq, Ord, PartialOrd)]
|
||||||
|
struct RawTransactionIngestGapId(u64);
|
||||||
|
|
||||||
|
struct RawTransactionIngestGap {
|
||||||
|
commitment: ksp_onchain_transport_lib::SolanaCommitment,
|
||||||
|
coverage_requirement: crate::RawTransactionIngestCoverageScope,
|
||||||
|
gap_id: RawTransactionIngestGapId,
|
||||||
|
network: ksp_store_lib::RawNetworkId,
|
||||||
|
range: RawTransactionIngestGapRange,
|
||||||
|
reason: RawTransactionIngestGapReason,
|
||||||
|
source_key: [u8; 32],
|
||||||
|
state: RawTransactionIngestGapState,
|
||||||
|
}
|
||||||
|
|
||||||
|
struct RawTransactionIngestGapLedger {
|
||||||
|
gaps: std::vec::Vec<RawTransactionIngestGap>,
|
||||||
|
network: ksp_store_lib::RawNetworkId,
|
||||||
|
next_gap_id: u64,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl RawTransactionIngestGapLedger {
|
||||||
|
fn new(network: ksp_store_lib::RawNetworkId) -> Self {
|
||||||
|
return Self { gaps: std::vec::Vec::new(), network, next_gap_id: 1 };
|
||||||
|
}
|
||||||
|
|
||||||
|
fn validate_invariants(&self) -> ksp_core_lib::Result<()> {
|
||||||
|
if self.next_gap_id == 0 {
|
||||||
|
return std::result::Result::Err(crate::runtime_error("continuity.gap_id_exhausted"));
|
||||||
|
}
|
||||||
|
let mut ids = std::collections::BTreeSet::new();
|
||||||
|
let mut open_gap_count = 0_usize;
|
||||||
|
let mut active_gap_count = 0_usize;
|
||||||
|
let mut highest_gap_id = 0_u64;
|
||||||
|
for gap in &self.gaps {
|
||||||
|
if gap.network != self.network {
|
||||||
|
return std::result::Result::Err(crate::runtime_error("continuity.gap_network_mismatch"));
|
||||||
|
}
|
||||||
|
if gap.gap_id.0 == 0 || !ids.insert(gap.gap_id) {
|
||||||
|
return std::result::Result::Err(crate::runtime_error("continuity.gap_id_invalid"));
|
||||||
|
}
|
||||||
|
highest_gap_id = highest_gap_id.max(gap.gap_id.0);
|
||||||
|
if gap.state.is_open() {
|
||||||
|
open_gap_count = match open_gap_count.checked_add(1) {
|
||||||
|
std::option::Option::Some(value) => value,
|
||||||
|
std::option::Option::None => return std::result::Result::Err(crate::counter_exhausted_error("continuity.open_gap_count")),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
if gap.state == RawTransactionIngestGapState::Repairing {
|
||||||
|
active_gap_count = match active_gap_count.checked_add(1) {
|
||||||
|
std::option::Option::Some(value) => value,
|
||||||
|
std::option::Option::None => return std::result::Result::Err(crate::counter_exhausted_error("continuity.active_gap_count")),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
if !RawTransactionIngestGapReason::ALL.contains(&gap.reason) || !RawTransactionIngestGapState::ALL.contains(&gap.state) {
|
||||||
|
return std::result::Result::Err(crate::runtime_error("continuity.gap_catalog_invalid"));
|
||||||
|
}
|
||||||
|
if gap.range.start_slot() > gap.range.end_slot() {
|
||||||
|
return std::result::Result::Err(crate::runtime_error("continuity.gap_range_reversed"));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if open_gap_count > crate::MAX_RAW_TRANSACTION_INGEST_OPEN_REPAIR_GAPS {
|
||||||
|
return std::result::Result::Err(crate::runtime_error("continuity.open_gap_limit_exceeded"));
|
||||||
|
}
|
||||||
|
if active_gap_count > crate::MAX_RAW_TRANSACTION_INGEST_REPAIR_ACTIVE_GAPS {
|
||||||
|
return std::result::Result::Err(crate::runtime_error("continuity.active_gap_limit_exceeded"));
|
||||||
|
}
|
||||||
|
if !self.gaps.is_empty() && self.next_gap_id <= highest_gap_id {
|
||||||
|
return std::result::Result::Err(crate::runtime_error("continuity.next_gap_id_invalid"));
|
||||||
|
}
|
||||||
|
for first_index in 0..self.gaps.len() {
|
||||||
|
let first = &self.gaps[first_index];
|
||||||
|
for second in self.gaps.iter().skip(first_index + 1) {
|
||||||
|
if first.source_key != second.source_key
|
||||||
|
|| first.commitment != second.commitment
|
||||||
|
|| first.coverage_requirement != second.coverage_requirement
|
||||||
|
|| first.reason != second.reason
|
||||||
|
|| !first.state.is_open()
|
||||||
|
|| !second.state.is_open()
|
||||||
|
{
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
let merged = first.range.try_merge(second.range);
|
||||||
|
match merged {
|
||||||
|
std::result::Result::Ok(std::option::Option::Some(_)) => {
|
||||||
|
return std::result::Result::Err(crate::runtime_error("continuity.coalescible_gap_ranges"));
|
||||||
|
},
|
||||||
|
std::result::Result::Ok(std::option::Option::None) => {},
|
||||||
|
std::result::Result::Err(error) => return std::result::Result::Err(error),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if RawTransactionIngestGapReason::ALL.len() != 5 || RawTransactionIngestGapState::ALL.len() != 4 {
|
||||||
|
return std::result::Result::Err(crate::runtime_error("continuity.gap_catalog_invalid"));
|
||||||
|
}
|
||||||
|
return std::result::Result::Ok(());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Private source capability descriptor prepared without network I/O from already validated runtime resources.
|
||||||
|
pub(crate) struct RawTransactionIngestRepairCapabilityDescriptor {
|
||||||
|
block_material: bool,
|
||||||
|
commitment: ksp_onchain_transport_lib::SolanaCommitment,
|
||||||
|
http_block_scan: bool,
|
||||||
|
known_reference_hydration: bool,
|
||||||
|
native_replay: bool,
|
||||||
|
network: ksp_store_lib::RawNetworkId,
|
||||||
|
reference_bearing: bool,
|
||||||
|
slot_enumerating: bool,
|
||||||
|
source_key: [u8; 32],
|
||||||
|
source_scope: crate::RawTransactionIngestCoverageScope,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl crate::RawTransactionIngestRepairCapabilityDescriptor {
|
||||||
|
/// Creates one private descriptor from source-local capabilities that were validated without issuing repair I/O.
|
||||||
|
#[allow(clippy::too_many_arguments)]
|
||||||
|
pub(crate) fn new(
|
||||||
|
source_key: [u8; 32],
|
||||||
|
network: ksp_store_lib::RawNetworkId,
|
||||||
|
commitment: ksp_onchain_transport_lib::SolanaCommitment,
|
||||||
|
source_scope: crate::RawTransactionIngestCoverageScope,
|
||||||
|
reference_bearing: bool,
|
||||||
|
block_material: bool,
|
||||||
|
slot_enumerating: bool,
|
||||||
|
known_reference_hydration: bool,
|
||||||
|
native_replay: bool,
|
||||||
|
http_block_scan: bool,
|
||||||
|
) -> ksp_core_lib::Result<Self> {
|
||||||
|
let descriptor = Self {
|
||||||
|
block_material,
|
||||||
|
commitment,
|
||||||
|
http_block_scan,
|
||||||
|
known_reference_hydration,
|
||||||
|
native_replay,
|
||||||
|
network,
|
||||||
|
reference_bearing,
|
||||||
|
slot_enumerating,
|
||||||
|
source_key,
|
||||||
|
source_scope,
|
||||||
|
};
|
||||||
|
if let std::result::Result::Err(error) = descriptor.validate() {
|
||||||
|
return std::result::Result::Err(error);
|
||||||
|
}
|
||||||
|
return std::result::Result::Ok(descriptor);
|
||||||
|
}
|
||||||
|
|
||||||
|
fn validate(&self) -> ksp_core_lib::Result<()> {
|
||||||
|
if self.commitment == ksp_onchain_transport_lib::SolanaCommitment::Processed {
|
||||||
|
return std::result::Result::Err(crate::runtime_error("continuity.processed_commitment_unsupported"));
|
||||||
|
}
|
||||||
|
if let std::result::Result::Err(error) = self.source_scope.validate_configured_target_scope() {
|
||||||
|
return std::result::Result::Err(error);
|
||||||
|
}
|
||||||
|
if !self.reference_bearing
|
||||||
|
&& !self.block_material
|
||||||
|
&& !self.slot_enumerating
|
||||||
|
&& !self.known_reference_hydration
|
||||||
|
&& !self.native_replay
|
||||||
|
&& !self.http_block_scan
|
||||||
|
{
|
||||||
|
return std::result::Result::Err(crate::runtime_error("continuity.source_capability_empty"));
|
||||||
|
}
|
||||||
|
if self.http_block_scan && (!self.block_material || !self.slot_enumerating) {
|
||||||
|
return std::result::Result::Err(crate::runtime_error("continuity.http_scan_capability_incomplete"));
|
||||||
|
}
|
||||||
|
return std::result::Result::Ok(());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
struct RawTransactionIngestTargetCoverage {
|
||||||
|
requirements: std::vec::Vec<RawTransactionIngestCoverageRequirement>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl RawTransactionIngestTargetCoverage {
|
||||||
|
fn from_capabilities(capabilities: &[crate::RawTransactionIngestRepairCapabilityDescriptor]) -> ksp_core_lib::Result<Self> {
|
||||||
|
let mut value = Self { requirements: std::vec::Vec::new() };
|
||||||
|
for capability in capabilities {
|
||||||
|
if let std::result::Result::Err(error) = value.include(capability.commitment, capability.source_scope.clone()) {
|
||||||
|
return std::result::Result::Err(error);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if value.requirements.is_empty() {
|
||||||
|
return std::result::Result::Err(crate::runtime_error("continuity.target_coverage_empty"));
|
||||||
|
}
|
||||||
|
return std::result::Result::Ok(value);
|
||||||
|
}
|
||||||
|
|
||||||
|
fn include(
|
||||||
|
&mut self,
|
||||||
|
commitment: ksp_onchain_transport_lib::SolanaCommitment,
|
||||||
|
scope: crate::RawTransactionIngestCoverageScope,
|
||||||
|
) -> ksp_core_lib::Result<()> {
|
||||||
|
if scope.is_known_references() {
|
||||||
|
return std::result::Result::Err(crate::runtime_error("continuity.known_references_not_target_scope"));
|
||||||
|
}
|
||||||
|
if matches!(&scope, crate::RawTransactionIngestCoverageScope::FullLedgerTransactions) {
|
||||||
|
self.requirements.retain(|requirement| return requirement.commitment != commitment);
|
||||||
|
self.requirements.push(RawTransactionIngestCoverageRequirement { commitment, scope });
|
||||||
|
return std::result::Result::Ok(());
|
||||||
|
}
|
||||||
|
if self.requirements.iter().any(|requirement| {
|
||||||
|
return requirement.commitment == commitment
|
||||||
|
&& (requirement.scope == crate::RawTransactionIngestCoverageScope::FullLedgerTransactions || requirement.scope == scope);
|
||||||
|
}) {
|
||||||
|
return std::result::Result::Ok(());
|
||||||
|
}
|
||||||
|
self.requirements.push(RawTransactionIngestCoverageRequirement { commitment, scope });
|
||||||
|
return std::result::Result::Ok(());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Run-local continuity contracts prepared from the exact caller-composed live-source aggregate before productive source execution.
|
||||||
|
pub(crate) struct RawTransactionIngestContinuityContracts {
|
||||||
|
capabilities: std::vec::Vec<crate::RawTransactionIngestRepairCapabilityDescriptor>,
|
||||||
|
gap_ledger: RawTransactionIngestGapLedger,
|
||||||
|
target_coverage: RawTransactionIngestTargetCoverage,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl crate::RawTransactionIngestContinuityContracts {
|
||||||
|
/// Builds the private capability inventory, conservative `TargetCoverage` and empty run-local gap ledger without issuing network or Store I/O.
|
||||||
|
pub(crate) fn new(capabilities: std::vec::Vec<crate::RawTransactionIngestRepairCapabilityDescriptor>) -> ksp_core_lib::Result<Self> {
|
||||||
|
if capabilities.is_empty() || capabilities.len() > crate::MAX_RAW_TRANSACTION_INGEST_LIVE_SOURCES {
|
||||||
|
return std::result::Result::Err(crate::runtime_error("continuity.capability_inventory_size_invalid"));
|
||||||
|
}
|
||||||
|
let network = capabilities[0].network.clone();
|
||||||
|
let mut source_keys = std::collections::BTreeSet::new();
|
||||||
|
for capability in &capabilities {
|
||||||
|
if let std::result::Result::Err(error) = capability.validate() {
|
||||||
|
return std::result::Result::Err(error);
|
||||||
|
}
|
||||||
|
if capability.network != network {
|
||||||
|
return std::result::Result::Err(crate::runtime_error("continuity.capability_network_mismatch"));
|
||||||
|
}
|
||||||
|
if !source_keys.insert(capability.source_key) {
|
||||||
|
return std::result::Result::Err(crate::runtime_error("continuity.duplicate_capability_source"));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let target_coverage = match RawTransactionIngestTargetCoverage::from_capabilities(capabilities.as_slice()) {
|
||||||
|
std::result::Result::Ok(value) => value,
|
||||||
|
std::result::Result::Err(error) => return std::result::Result::Err(error),
|
||||||
|
};
|
||||||
|
let gap_ledger = RawTransactionIngestGapLedger::new(network);
|
||||||
|
if let std::result::Result::Err(error) = gap_ledger.validate_invariants() {
|
||||||
|
return std::result::Result::Err(error);
|
||||||
|
}
|
||||||
|
if !coverage_scope_catalog_is_complete() {
|
||||||
|
return std::result::Result::Err(crate::runtime_error("continuity.coverage_scope_catalog_invalid"));
|
||||||
|
}
|
||||||
|
if crate::MAX_RAW_TRANSACTION_INGEST_REPAIR_BLOCK_FETCH_IN_FLIGHT == 0
|
||||||
|
|| crate::MAX_RAW_TRANSACTION_INGEST_REPAIR_DISCOVERY_WINDOW_SLOTS == 0
|
||||||
|
|| crate::MAX_RAW_TRANSACTION_INGEST_REPAIR_DISCOVERY_WINDOW_SLOTS > crate::MAX_RAW_TRANSACTION_INGEST_REPAIR_RANGE_SLOTS
|
||||||
|
{
|
||||||
|
return std::result::Result::Err(crate::runtime_error("continuity.repair_bounds_invalid"));
|
||||||
|
}
|
||||||
|
return std::result::Result::Ok(Self { capabilities, gap_ledger, target_coverage });
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Revalidates that the run-local contracts still correspond exactly to the caller-composed source count before tasks are spawned.
|
||||||
|
pub(crate) fn validate_for_source_count(&self, expected_source_count: usize) -> ksp_core_lib::Result<()> {
|
||||||
|
if self.capabilities.len() != expected_source_count || self.target_coverage.requirements.is_empty() {
|
||||||
|
return std::result::Result::Err(crate::runtime_error("continuity.source_inventory_mismatch"));
|
||||||
|
}
|
||||||
|
if let std::result::Result::Err(error) = self.gap_ledger.validate_invariants() {
|
||||||
|
return std::result::Result::Err(error);
|
||||||
|
}
|
||||||
|
return std::result::Result::Ok(());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn coverage_scope_catalog_is_complete() -> bool {
|
||||||
|
let full = crate::RawTransactionIngestCoverageScope::FullLedgerTransactions;
|
||||||
|
let exact = crate::RawTransactionIngestCoverageScope::ExactSourceScope("catalog", [0_u8; 32]);
|
||||||
|
let known = crate::RawTransactionIngestCoverageScope::KnownReferences([0_u8; 32]);
|
||||||
|
return full.validate_configured_target_scope().is_ok()
|
||||||
|
&& exact.validate_configured_target_scope().is_ok()
|
||||||
|
&& known.validate_configured_target_scope().is_err()
|
||||||
|
&& !full.is_known_references()
|
||||||
|
&& !exact.is_known_references()
|
||||||
|
&& known.is_known_references();
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
#[path = "../unit_tests/continuity.rs"]
|
||||||
|
mod tests;
|
||||||
@@ -1,5 +1,5 @@
|
|||||||
// file: crates/ksp-worker-raw-transaction-ingest-lib/src/lib.rs
|
// file: crates/ksp-worker-raw-transaction-ingest-lib/src/lib.rs
|
||||||
// version: 26
|
// version: 27
|
||||||
|
|
||||||
#![warn(missing_docs)]
|
#![warn(missing_docs)]
|
||||||
#![deny(unreachable_pub)]
|
#![deny(unreachable_pub)]
|
||||||
@@ -21,6 +21,7 @@
|
|||||||
//! state and faults conservatively when Transport proves a replay-retention continuity gap; history remediation remains outside this crate.
|
//! state and faults conservatively when Transport proves a replay-retention continuity gap; history remediation remains outside this crate.
|
||||||
|
|
||||||
mod admission;
|
mod admission;
|
||||||
|
mod continuity;
|
||||||
mod error;
|
mod error;
|
||||||
mod identity;
|
mod identity;
|
||||||
mod persistence;
|
mod persistence;
|
||||||
@@ -110,6 +111,22 @@ pub use self::snapshot::RawTransactionIngestSourceState;
|
|||||||
pub(crate) use self::admission::RawTransactionAdmission;
|
pub(crate) use self::admission::RawTransactionAdmission;
|
||||||
/// Crate-private source-neutral ingress sent through the bounded central admission queue.
|
/// Crate-private source-neutral ingress sent through the bounded central admission queue.
|
||||||
pub(crate) use self::admission::RawTransactionIngress;
|
pub(crate) use self::admission::RawTransactionIngress;
|
||||||
|
/// Maximum number of simultaneously retained non-repaired run-local continuity gaps.
|
||||||
|
pub(crate) use self::continuity::MAX_RAW_TRANSACTION_INGEST_OPEN_REPAIR_GAPS;
|
||||||
|
/// Maximum number of run-local continuity gaps that may actively repair at once.
|
||||||
|
pub(crate) use self::continuity::MAX_RAW_TRANSACTION_INGEST_REPAIR_ACTIVE_GAPS;
|
||||||
|
/// Maximum number of logical block fetches reserved for the later bounded repair scheduler.
|
||||||
|
pub(crate) use self::continuity::MAX_RAW_TRANSACTION_INGEST_REPAIR_BLOCK_FETCH_IN_FLIGHT;
|
||||||
|
/// Maximum number of slots reserved for one later HTTP repair discovery window.
|
||||||
|
pub(crate) use self::continuity::MAX_RAW_TRANSACTION_INGEST_REPAIR_DISCOVERY_WINDOW_SLOTS;
|
||||||
|
/// Maximum inclusive slot span accepted by one run-local repair gap.
|
||||||
|
pub(crate) use self::continuity::MAX_RAW_TRANSACTION_INGEST_REPAIR_RANGE_SLOTS;
|
||||||
|
/// Private run-local continuity aggregate containing capabilities, TargetCoverage and the gap ledger.
|
||||||
|
pub(crate) use self::continuity::RawTransactionIngestContinuityContracts;
|
||||||
|
/// Private provider-neutral coverage scope used by run-local continuity proof contracts.
|
||||||
|
pub(crate) use self::continuity::RawTransactionIngestCoverageScope;
|
||||||
|
/// Private source capability descriptor prepared without repair I/O.
|
||||||
|
pub(crate) use self::continuity::RawTransactionIngestRepairCapabilityDescriptor;
|
||||||
/// Creates one terminal content-conflict error without copying conflicting material into diagnostics.
|
/// Creates one terminal content-conflict error without copying conflicting material into diagnostics.
|
||||||
pub(crate) use self::error::content_conflict_error;
|
pub(crate) use self::error::content_conflict_error;
|
||||||
/// Creates one terminal counter-exhaustion error without exposing runtime material.
|
/// Creates one terminal counter-exhaustion error without exposing runtime material.
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
// file: crates/ksp-worker-raw-transaction-ingest-lib/src/runtime_resources.rs
|
// file: crates/ksp-worker-raw-transaction-ingest-lib/src/runtime_resources.rs
|
||||||
// version: 26
|
// version: 27
|
||||||
|
|
||||||
use sha2::Digest; // rust-rules: trait-import
|
use sha2::Digest; // rust-rules: trait-import
|
||||||
|
|
||||||
@@ -31,6 +31,7 @@ const RAW_TRANSACTION_INGEST_STANDARD_BLOCK_PROTOCOL: &str = "solana_ws";
|
|||||||
const RAW_TRANSACTION_INGEST_STANDARD_LOGS_FILTER_FINGERPRINT_DOMAIN: &[u8] = b"ksp.raw_transaction_ingest.standard_logs.filter.v1\0";
|
const RAW_TRANSACTION_INGEST_STANDARD_LOGS_FILTER_FINGERPRINT_DOMAIN: &[u8] = b"ksp.raw_transaction_ingest.standard_logs.filter.v1\0";
|
||||||
const RAW_TRANSACTION_INGEST_STANDARD_LOGS_HTTP_PROTOCOL: &str = "solana_ws_http";
|
const RAW_TRANSACTION_INGEST_STANDARD_LOGS_HTTP_PROTOCOL: &str = "solana_ws_http";
|
||||||
const RAW_TRANSACTION_INGEST_STANDARD_LOGS_HTTP_SOURCE_KEY_DOMAIN: &[u8] = b"ksp.raw_transaction_ingest.standard_logs_http.source_key.v1\0";
|
const RAW_TRANSACTION_INGEST_STANDARD_LOGS_HTTP_SOURCE_KEY_DOMAIN: &[u8] = b"ksp.raw_transaction_ingest.standard_logs_http.source_key.v1\0";
|
||||||
|
const RAW_TRANSACTION_INGEST_YELLOWSTONE_COVERAGE_SCOPE_FINGERPRINT_DOMAIN: &[u8] = b"ksp.raw_transaction_ingest.yellowstone.coverage_scope.v1\0";
|
||||||
const RAW_TRANSACTION_INGEST_YELLOWSTONE_FILTER_FINGERPRINT_DOMAIN: &[u8] = b"ksp.raw_transaction_ingest.yellowstone.filters.v1\0";
|
const RAW_TRANSACTION_INGEST_YELLOWSTONE_FILTER_FINGERPRINT_DOMAIN: &[u8] = b"ksp.raw_transaction_ingest.yellowstone.filters.v1\0";
|
||||||
const RAW_TRANSACTION_INGEST_YELLOWSTONE_HTTP_PROTOCOL: &str = "yellowstone_http";
|
const RAW_TRANSACTION_INGEST_YELLOWSTONE_HTTP_PROTOCOL: &str = "yellowstone_http";
|
||||||
const RAW_TRANSACTION_INGEST_YELLOWSTONE_HTTP_SOURCE_KEY_DOMAIN: &[u8] = b"ksp.raw_transaction_ingest.yellowstone_http.source_key.v1\0";
|
const RAW_TRANSACTION_INGEST_YELLOWSTONE_HTTP_SOURCE_KEY_DOMAIN: &[u8] = b"ksp.raw_transaction_ingest.yellowstone_http.source_key.v1\0";
|
||||||
@@ -75,6 +76,86 @@ impl RawTransactionIngestLiveSource {
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn repair_capability_descriptor(&self) -> ksp_core_lib::Result<crate::RawTransactionIngestRepairCapabilityDescriptor> {
|
||||||
|
let commitment = match self {
|
||||||
|
Self::HeliusTransaction(source) => source.commitment,
|
||||||
|
Self::HttpBlockPolling(source) => source.commitment,
|
||||||
|
Self::StandardBlock(source) => source.commitment,
|
||||||
|
Self::StandardLogs(source) => source.commitment,
|
||||||
|
Self::Yellowstone(source) => match source.subscribe_request.commitment() {
|
||||||
|
std::option::Option::Some(value) => value,
|
||||||
|
std::option::Option::None => return std::result::Result::Err(crate::runtime_error("continuity.yellowstone_commitment_missing")),
|
||||||
|
},
|
||||||
|
};
|
||||||
|
let source_scope = match self {
|
||||||
|
Self::HeliusTransaction(source) => crate::RawTransactionIngestCoverageScope::exact_source_scope("helius_transaction", source.filter_fingerprint),
|
||||||
|
Self::HttpBlockPolling(_) => crate::RawTransactionIngestCoverageScope::full_ledger_transactions(),
|
||||||
|
Self::StandardBlock(source) => match &source.filter {
|
||||||
|
ksp_onchain_transport_lib::SolanaBlockSubscribeFilter::All => crate::RawTransactionIngestCoverageScope::full_ledger_transactions(),
|
||||||
|
ksp_onchain_transport_lib::SolanaBlockSubscribeFilter::MentionsAccountOrProgram(_) => {
|
||||||
|
crate::RawTransactionIngestCoverageScope::exact_source_scope("standard_block", source.filter_fingerprint)
|
||||||
|
},
|
||||||
|
},
|
||||||
|
Self::StandardLogs(source) => crate::RawTransactionIngestCoverageScope::exact_source_scope("standard_logs", source.filter_fingerprint),
|
||||||
|
Self::Yellowstone(source) => {
|
||||||
|
let fingerprint = match yellowstone_coverage_scope_fingerprint(&source.subscribe_request) {
|
||||||
|
std::result::Result::Ok(value) => value,
|
||||||
|
std::result::Result::Err(error) => return std::result::Result::Err(error),
|
||||||
|
};
|
||||||
|
crate::RawTransactionIngestCoverageScope::exact_source_scope("yellowstone", fingerprint)
|
||||||
|
},
|
||||||
|
};
|
||||||
|
let (reference_bearing, live_block_material, native_replay, http_block_scan, known_reference_hydration) = match self {
|
||||||
|
Self::HeliusTransaction(source) => {
|
||||||
|
let http_block_scan = match http_role_supports_repair_scan(&source.http_pool, &source.hydration_role, source.network.as_str()) {
|
||||||
|
std::result::Result::Ok(value) => value,
|
||||||
|
std::result::Result::Err(error) => return std::result::Result::Err(error),
|
||||||
|
};
|
||||||
|
(true, false, false, http_block_scan, true)
|
||||||
|
},
|
||||||
|
Self::HttpBlockPolling(source) => {
|
||||||
|
let known_reference_hydration =
|
||||||
|
match http_role_supports_rpc_method(&source.http_pool, &source.polling_role, "getTransaction", source.network.as_str()) {
|
||||||
|
std::result::Result::Ok(value) => value,
|
||||||
|
std::result::Result::Err(error) => return std::result::Result::Err(error),
|
||||||
|
};
|
||||||
|
(false, true, false, true, known_reference_hydration)
|
||||||
|
},
|
||||||
|
Self::StandardBlock(_) => (false, true, false, false, false),
|
||||||
|
Self::StandardLogs(source) => {
|
||||||
|
let http_block_scan = match http_role_supports_repair_scan(&source.http_pool, &source.hydration_role, source.network.as_str()) {
|
||||||
|
std::result::Result::Ok(value) => value,
|
||||||
|
std::result::Result::Err(error) => return std::result::Result::Err(error),
|
||||||
|
};
|
||||||
|
(true, false, false, http_block_scan, true)
|
||||||
|
},
|
||||||
|
Self::Yellowstone(source) => {
|
||||||
|
let http_block_scan = match http_role_supports_repair_scan(&source.http_pool, &source.hydration_role, source.network.as_str()) {
|
||||||
|
std::result::Result::Ok(value) => value,
|
||||||
|
std::result::Result::Err(error) => return std::result::Result::Err(error),
|
||||||
|
};
|
||||||
|
let reference_bearing =
|
||||||
|
source.subscribe_request.transaction_filter_count() > 0 || source.subscribe_request.transaction_status_filter_count() > 0;
|
||||||
|
let live_block_material = source.subscribe_request.block_filter_count() > 0;
|
||||||
|
(reference_bearing, live_block_material, true, http_block_scan, true)
|
||||||
|
},
|
||||||
|
};
|
||||||
|
let block_material = live_block_material || http_block_scan;
|
||||||
|
let slot_enumerating = http_block_scan;
|
||||||
|
return crate::RawTransactionIngestRepairCapabilityDescriptor::new(
|
||||||
|
self.source_key(),
|
||||||
|
self.network().clone(),
|
||||||
|
commitment,
|
||||||
|
source_scope,
|
||||||
|
reference_bearing,
|
||||||
|
block_material,
|
||||||
|
slot_enumerating,
|
||||||
|
known_reference_hydration,
|
||||||
|
native_replay,
|
||||||
|
http_block_scan,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
fn source_key(&self) -> [u8; 32] {
|
fn source_key(&self) -> [u8; 32] {
|
||||||
return match self {
|
return match self {
|
||||||
Self::HeliusTransaction(source) => source.source_key,
|
Self::HeliusTransaction(source) => source.source_key,
|
||||||
@@ -2023,6 +2104,22 @@ impl crate::RawTransactionIngestRuntimeResources {
|
|||||||
if self.sources.is_empty() || self.sources.len() > crate::MAX_RAW_TRANSACTION_INGEST_LIVE_SOURCES {
|
if self.sources.is_empty() || self.sources.len() > crate::MAX_RAW_TRANSACTION_INGEST_LIVE_SOURCES {
|
||||||
return std::result::Result::Err(crate::runtime_error("runtime_resources.source_collection_invalid"));
|
return std::result::Result::Err(crate::runtime_error("runtime_resources.source_collection_invalid"));
|
||||||
}
|
}
|
||||||
|
let mut repair_capabilities = std::vec::Vec::with_capacity(self.sources.len());
|
||||||
|
for source in &self.sources {
|
||||||
|
let capability = match source.repair_capability_descriptor() {
|
||||||
|
std::result::Result::Ok(value) => value,
|
||||||
|
std::result::Result::Err(error) => return std::result::Result::Err(error),
|
||||||
|
};
|
||||||
|
repair_capabilities.push(capability);
|
||||||
|
}
|
||||||
|
let continuity_contracts = match crate::RawTransactionIngestContinuityContracts::new(repair_capabilities) {
|
||||||
|
std::result::Result::Ok(value) => value,
|
||||||
|
std::result::Result::Err(error) => return std::result::Result::Err(error),
|
||||||
|
};
|
||||||
|
let continuity_source_count = self.sources.len();
|
||||||
|
if let std::result::Result::Err(error) = continuity_contracts.validate_for_source_count(continuity_source_count) {
|
||||||
|
return std::result::Result::Err(error);
|
||||||
|
}
|
||||||
let source_keys = self.sources.iter().map(RawTransactionIngestLiveSource::source_key).collect::<std::vec::Vec<_>>();
|
let source_keys = self.sources.iter().map(RawTransactionIngestLiveSource::source_key).collect::<std::vec::Vec<_>>();
|
||||||
let hydration_source_count = self.sources.iter().filter(|source| return source.uses_hydration()).count();
|
let hydration_source_count = self.sources.iter().filter(|source| return source.uses_hydration()).count();
|
||||||
if let std::result::Result::Err(error) =
|
if let std::result::Result::Err(error) =
|
||||||
@@ -2072,7 +2169,11 @@ impl crate::RawTransactionIngestRuntimeResources {
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
std::mem::drop(admission_sender);
|
std::mem::drop(admission_sender);
|
||||||
return supervise_live_source_tasks(stop_receiver, source_stop_sender, children).await;
|
let result = supervise_live_source_tasks(stop_receiver, source_stop_sender, children).await;
|
||||||
|
if let std::result::Result::Err(error) = continuity_contracts.validate_for_source_count(continuity_source_count) {
|
||||||
|
return std::result::Result::Err(error);
|
||||||
|
}
|
||||||
|
return result;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -2205,6 +2306,57 @@ async fn drain_live_source_tasks(
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn http_role_supports_repair_scan(
|
||||||
|
pool: &ksp_onchain_transport_lib::HttpTransportPool,
|
||||||
|
role: &ksp_onchain_transport_lib::HttpRoleName,
|
||||||
|
expected_cluster: &str,
|
||||||
|
) -> ksp_core_lib::Result<bool> {
|
||||||
|
let has_get_block = match http_role_supports_rpc_method(pool, role, "getBlock", expected_cluster) {
|
||||||
|
std::result::Result::Ok(value) => value,
|
||||||
|
std::result::Result::Err(error) => return std::result::Result::Err(error),
|
||||||
|
};
|
||||||
|
let has_get_blocks = match http_role_supports_rpc_method(pool, role, "getBlocks", expected_cluster) {
|
||||||
|
std::result::Result::Ok(value) => value,
|
||||||
|
std::result::Result::Err(error) => return std::result::Result::Err(error),
|
||||||
|
};
|
||||||
|
let has_get_blocks_with_limit = match http_role_supports_rpc_method(pool, role, "getBlocksWithLimit", expected_cluster) {
|
||||||
|
std::result::Result::Ok(value) => value,
|
||||||
|
std::result::Result::Err(error) => return std::result::Result::Err(error),
|
||||||
|
};
|
||||||
|
let has_get_slot = match http_role_supports_rpc_method(pool, role, "getSlot", expected_cluster) {
|
||||||
|
std::result::Result::Ok(value) => value,
|
||||||
|
std::result::Result::Err(error) => return std::result::Result::Err(error),
|
||||||
|
};
|
||||||
|
return std::result::Result::Ok(has_get_block && has_get_slot && (has_get_blocks || has_get_blocks_with_limit));
|
||||||
|
}
|
||||||
|
|
||||||
|
fn http_role_supports_rpc_method(
|
||||||
|
pool: &ksp_onchain_transport_lib::HttpTransportPool,
|
||||||
|
role: &ksp_onchain_transport_lib::HttpRoleName,
|
||||||
|
method_name: &'static str,
|
||||||
|
expected_cluster: &str,
|
||||||
|
) -> ksp_core_lib::Result<bool> {
|
||||||
|
let method = match ksp_onchain_transport_lib::find_http_rpc_method(method_name) {
|
||||||
|
std::option::Option::Some(value) => value,
|
||||||
|
std::option::Option::None => return std::result::Result::Err(crate::runtime_error("continuity.http_method_missing")),
|
||||||
|
};
|
||||||
|
let snapshot = pool.snapshot();
|
||||||
|
for endpoint in snapshot.endpoints() {
|
||||||
|
if !endpoint.enabled() || endpoint.cluster() != expected_cluster {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
for endpoint_role in endpoint.roles() {
|
||||||
|
if !endpoint_role.enabled() || endpoint_role.role() != role.as_str() {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if http_role_supports_request_kind(endpoint_role, method.request_kind()) {
|
||||||
|
return std::result::Result::Ok(true);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return std::result::Result::Ok(false);
|
||||||
|
}
|
||||||
|
|
||||||
fn http_block_polling_live_source_key(
|
fn http_block_polling_live_source_key(
|
||||||
network: &ksp_store_lib::RawNetworkId,
|
network: &ksp_store_lib::RawNetworkId,
|
||||||
polling_role: &ksp_onchain_transport_lib::HttpRoleName,
|
polling_role: &ksp_onchain_transport_lib::HttpRoleName,
|
||||||
@@ -2639,6 +2791,22 @@ fn standard_logs_live_source_key(
|
|||||||
return hasher.finalize().into();
|
return hasher.finalize().into();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn yellowstone_coverage_scope_fingerprint(request: &ksp_onchain_transport_lib::YellowstoneSubscribeRequest) -> ksp_core_lib::Result<[u8; 32]> {
|
||||||
|
let mut normalized = request.clone();
|
||||||
|
normalized.set_commitment(std::option::Option::None);
|
||||||
|
normalized.set_from_slot(std::option::Option::None);
|
||||||
|
normalized.set_ping(std::option::Option::None);
|
||||||
|
let identity = match normalized.identity() {
|
||||||
|
std::result::Result::Ok(value) => value,
|
||||||
|
std::result::Result::Err(_) => return std::result::Result::Err(crate::runtime_error("continuity.yellowstone_scope_identity_invalid")),
|
||||||
|
};
|
||||||
|
let mut hasher = sha2::Sha256::new();
|
||||||
|
hasher.update(RAW_TRANSACTION_INGEST_YELLOWSTONE_COVERAGE_SCOPE_FINGERPRINT_DOMAIN);
|
||||||
|
let mut writer = RawTransactionIngestSourceKeyHashWriter { hasher: &mut hasher };
|
||||||
|
std::hash::Hash::hash(&identity, &mut writer);
|
||||||
|
return std::result::Result::Ok(hasher.finalize().into());
|
||||||
|
}
|
||||||
|
|
||||||
fn yellowstone_live_source_key(
|
fn yellowstone_live_source_key(
|
||||||
network: &ksp_store_lib::RawNetworkId,
|
network: &ksp_store_lib::RawNetworkId,
|
||||||
route: &RawTransactionIngestSourceRoute,
|
route: &RawTransactionIngestSourceRoute,
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
// file: crates/ksp-worker-raw-transaction-ingest-lib/tests/hardening.rs
|
// file: crates/ksp-worker-raw-transaction-ingest-lib/tests/hardening.rs
|
||||||
// version: 24
|
// version: 25
|
||||||
|
|
||||||
//! External public, security, redaction and release-boundary hardening canaries through `pre.012`.
|
//! External public, security, redaction and release-boundary hardening canaries through `v0.3.14-pre.002`.
|
||||||
|
|
||||||
fn network(value: &'static str) -> std::option::Option<ksp_store_lib::RawNetworkId> {
|
fn network(value: &'static str) -> std::option::Option<ksp_store_lib::RawNetworkId> {
|
||||||
let result = ksp_store_lib::RawNetworkId::new(value);
|
let result = ksp_store_lib::RawNetworkId::new(value);
|
||||||
@@ -957,3 +957,57 @@ fn v0_3_13_pre_012_cross_layer_security_closure_is_complete_and_source_neutral()
|
|||||||
}
|
}
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn v0_3_14_pre_002_continuity_contracts_are_private_bounded_and_io_free() {
|
||||||
|
let continuity = include_str!("../src/continuity.rs");
|
||||||
|
let resources = include_str!("../src/runtime_resources.rs");
|
||||||
|
let root = include_str!("../src/lib.rs");
|
||||||
|
for required in [
|
||||||
|
"MAX_RAW_TRANSACTION_INGEST_OPEN_REPAIR_GAPS",
|
||||||
|
"MAX_RAW_TRANSACTION_INGEST_REPAIR_ACTIVE_GAPS",
|
||||||
|
"MAX_RAW_TRANSACTION_INGEST_REPAIR_BLOCK_FETCH_IN_FLIGHT",
|
||||||
|
"MAX_RAW_TRANSACTION_INGEST_REPAIR_DISCOVERY_WINDOW_SLOTS",
|
||||||
|
"MAX_RAW_TRANSACTION_INGEST_REPAIR_RANGE_SLOTS",
|
||||||
|
"FullLedgerTransactions",
|
||||||
|
"ExactSourceScope",
|
||||||
|
"KnownReferences",
|
||||||
|
"RawTransactionIngestGapLedger",
|
||||||
|
"RawTransactionIngestTargetCoverage",
|
||||||
|
"RawTransactionIngestRepairCapabilityDescriptor",
|
||||||
|
"RawTransactionIngestContinuityContracts",
|
||||||
|
"continuity.known_references_not_target_scope",
|
||||||
|
"continuity.coalescible_gap_ranges",
|
||||||
|
] {
|
||||||
|
assert!(continuity.contains(required), "required pre.002 continuity contract missing: {required}");
|
||||||
|
}
|
||||||
|
for required in [
|
||||||
|
"repair_capability_descriptor",
|
||||||
|
"yellowstone_coverage_scope_fingerprint",
|
||||||
|
"http_role_supports_repair_scan",
|
||||||
|
"RawTransactionIngestContinuityContracts::new",
|
||||||
|
"validate_for_source_count",
|
||||||
|
] {
|
||||||
|
assert!(resources.contains(required), "required pre.002 runtime-resource capability wiring missing: {required}");
|
||||||
|
}
|
||||||
|
for forbidden in [
|
||||||
|
"get_block_observed",
|
||||||
|
"get_transaction_observed",
|
||||||
|
"get_blocks_with_limit",
|
||||||
|
"get_blocks(",
|
||||||
|
"open_standard_subscribe",
|
||||||
|
"SolanaStandardWsSession::connect",
|
||||||
|
"HeliusLaserStreamWsSession::connect",
|
||||||
|
"ksp_config_lib::",
|
||||||
|
"ksp_job_backfill_lib::",
|
||||||
|
"ksp_store_postgres_lib::",
|
||||||
|
"reqwest::",
|
||||||
|
"tonic::",
|
||||||
|
"yellowstone_grpc_proto::",
|
||||||
|
] {
|
||||||
|
assert!(!continuity.contains(forbidden), "pre.002 continuity contract performed or imported forbidden I/O/boundary: {forbidden}");
|
||||||
|
}
|
||||||
|
assert!(!root.contains("pub use self::continuity::"));
|
||||||
|
assert!(root.contains("pub(crate) use self::continuity::RawTransactionIngestContinuityContracts;"));
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
// file: crates/ksp-worker-raw-transaction-ingest-lib/tests/release_completeness.rs
|
// file: crates/ksp-worker-raw-transaction-ingest-lib/tests/release_completeness.rs
|
||||||
// version: 20
|
// version: 21
|
||||||
|
|
||||||
//! Release-completeness canaries through the `pre.012` cross-layer completeness/security tranche.
|
//! Release-completeness canaries through the `v0.3.14-pre.002` continuity-contract tranche.
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn pre_010_production_module_inventory_is_exact() -> std::io::Result<()> {
|
fn pre_010_production_module_inventory_is_exact() -> std::io::Result<()> {
|
||||||
@@ -34,7 +34,18 @@ fn pre_010_production_module_inventory_is_exact() -> std::io::Result<()> {
|
|||||||
names.sort_unstable();
|
names.sort_unstable();
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
names,
|
names,
|
||||||
std::vec!["admission.rs", "error.rs", "identity.rs", "lib.rs", "persistence.rs", "runtime.rs", "runtime_resources.rs", "settings.rs", "snapshot.rs",]
|
std::vec![
|
||||||
|
"admission.rs",
|
||||||
|
"continuity.rs",
|
||||||
|
"error.rs",
|
||||||
|
"identity.rs",
|
||||||
|
"lib.rs",
|
||||||
|
"persistence.rs",
|
||||||
|
"runtime.rs",
|
||||||
|
"runtime_resources.rs",
|
||||||
|
"settings.rs",
|
||||||
|
"snapshot.rs",
|
||||||
|
]
|
||||||
);
|
);
|
||||||
return std::result::Result::Ok(());
|
return std::result::Result::Ok(());
|
||||||
}
|
}
|
||||||
@@ -132,6 +143,7 @@ fn pre_010_external_hardening_suite_is_present_and_scoped() {
|
|||||||
"v0_3_13_pre_009_duplicate_storm_disagreement_and_starvation_guards_are_explicit",
|
"v0_3_13_pre_009_duplicate_storm_disagreement_and_starvation_guards_are_explicit",
|
||||||
"v0_3_13_pre_010_multi_source_health_is_conservative_counted_and_redacted",
|
"v0_3_13_pre_010_multi_source_health_is_conservative_counted_and_redacted",
|
||||||
"v0_3_13_pre_011_shutdown_races_are_bounded_joined_atomic_and_counter_safe",
|
"v0_3_13_pre_011_shutdown_races_are_bounded_joined_atomic_and_counter_safe",
|
||||||
|
"v0_3_14_pre_002_continuity_contracts_are_private_bounded_and_io_free",
|
||||||
] {
|
] {
|
||||||
assert!(hardening.contains(required), "required pre.010 hardening canary missing: {required}");
|
assert!(hardening.contains(required), "required pre.010 hardening canary missing: {required}");
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,241 @@
|
|||||||
|
// file: crates/ksp-worker-raw-transaction-ingest-lib/unit_tests/continuity.rs
|
||||||
|
// version: 1
|
||||||
|
|
||||||
|
fn network() -> std::option::Option<ksp_store_lib::RawNetworkId> {
|
||||||
|
return match ksp_store_lib::RawNetworkId::new("mainnet") {
|
||||||
|
std::result::Result::Ok(value) => std::option::Option::Some(value),
|
||||||
|
std::result::Result::Err(_) => std::option::Option::None,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
fn exact_scope(family: &'static str, fingerprint_byte: u8) -> crate::RawTransactionIngestCoverageScope {
|
||||||
|
return crate::RawTransactionIngestCoverageScope::exact_source_scope(family, [fingerprint_byte; 32]);
|
||||||
|
}
|
||||||
|
|
||||||
|
fn capability(
|
||||||
|
source_key_byte: u8,
|
||||||
|
commitment: ksp_onchain_transport_lib::SolanaCommitment,
|
||||||
|
scope: crate::RawTransactionIngestCoverageScope,
|
||||||
|
) -> std::option::Option<crate::RawTransactionIngestRepairCapabilityDescriptor> {
|
||||||
|
let network = match network() {
|
||||||
|
std::option::Option::Some(value) => value,
|
||||||
|
std::option::Option::None => return std::option::Option::None,
|
||||||
|
};
|
||||||
|
return match crate::RawTransactionIngestRepairCapabilityDescriptor::new(
|
||||||
|
[source_key_byte; 32],
|
||||||
|
network,
|
||||||
|
commitment,
|
||||||
|
scope,
|
||||||
|
true,
|
||||||
|
false,
|
||||||
|
false,
|
||||||
|
true,
|
||||||
|
false,
|
||||||
|
false,
|
||||||
|
) {
|
||||||
|
std::result::Result::Ok(value) => std::option::Option::Some(value),
|
||||||
|
std::result::Result::Err(_) => std::option::Option::None,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
fn gap(
|
||||||
|
gap_id: u64,
|
||||||
|
source_key_byte: u8,
|
||||||
|
start_slot: u64,
|
||||||
|
end_slot: u64,
|
||||||
|
state: super::RawTransactionIngestGapState,
|
||||||
|
) -> std::option::Option<super::RawTransactionIngestGap> {
|
||||||
|
let network = match network() {
|
||||||
|
std::option::Option::Some(value) => value,
|
||||||
|
std::option::Option::None => return std::option::Option::None,
|
||||||
|
};
|
||||||
|
let range = match super::RawTransactionIngestGapRange::new(start_slot, end_slot) {
|
||||||
|
std::result::Result::Ok(value) => value,
|
||||||
|
std::result::Result::Err(_) => return std::option::Option::None,
|
||||||
|
};
|
||||||
|
return std::option::Option::Some(super::RawTransactionIngestGap {
|
||||||
|
commitment: ksp_onchain_transport_lib::SolanaCommitment::Confirmed,
|
||||||
|
coverage_requirement: exact_scope("standard_logs", 9),
|
||||||
|
gap_id: super::RawTransactionIngestGapId(gap_id),
|
||||||
|
network,
|
||||||
|
range,
|
||||||
|
reason: super::RawTransactionIngestGapReason::WebSocketReconnect,
|
||||||
|
source_key: [source_key_byte; 32],
|
||||||
|
state,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn pre_002_repair_bounds_are_exact_and_run_local() {
|
||||||
|
assert_eq!(crate::MAX_RAW_TRANSACTION_INGEST_OPEN_REPAIR_GAPS, 64);
|
||||||
|
assert_eq!(crate::MAX_RAW_TRANSACTION_INGEST_REPAIR_ACTIVE_GAPS, 1);
|
||||||
|
assert_eq!(crate::MAX_RAW_TRANSACTION_INGEST_REPAIR_BLOCK_FETCH_IN_FLIGHT, 4);
|
||||||
|
assert_eq!(crate::MAX_RAW_TRANSACTION_INGEST_REPAIR_DISCOVERY_WINDOW_SLOTS, 512);
|
||||||
|
assert_eq!(crate::MAX_RAW_TRANSACTION_INGEST_REPAIR_RANGE_SLOTS, 4_096);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn pre_002_gap_range_is_inclusive_bounded_and_overflow_safe() {
|
||||||
|
assert!(super::RawTransactionIngestGapRange::new(10, 9).is_err());
|
||||||
|
assert!(super::RawTransactionIngestGapRange::new(10, 10 + crate::MAX_RAW_TRANSACTION_INGEST_REPAIR_RANGE_SLOTS - 1).is_ok());
|
||||||
|
assert!(super::RawTransactionIngestGapRange::new(10, 10 + crate::MAX_RAW_TRANSACTION_INGEST_REPAIR_RANGE_SLOTS).is_err());
|
||||||
|
assert!(super::RawTransactionIngestGapRange::new(u64::MAX, u64::MAX).is_ok());
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn pre_002_gap_range_overlap_and_adjacency_are_explicit() {
|
||||||
|
let first = match super::RawTransactionIngestGapRange::new(100, 109) {
|
||||||
|
std::result::Result::Ok(value) => value,
|
||||||
|
std::result::Result::Err(error) => panic!("unexpected first range failure: {error}"),
|
||||||
|
};
|
||||||
|
let overlap = match super::RawTransactionIngestGapRange::new(105, 115) {
|
||||||
|
std::result::Result::Ok(value) => value,
|
||||||
|
std::result::Result::Err(error) => panic!("unexpected overlap range failure: {error}"),
|
||||||
|
};
|
||||||
|
let adjacent = match super::RawTransactionIngestGapRange::new(110, 120) {
|
||||||
|
std::result::Result::Ok(value) => value,
|
||||||
|
std::result::Result::Err(error) => panic!("unexpected adjacent range failure: {error}"),
|
||||||
|
};
|
||||||
|
let separate = match super::RawTransactionIngestGapRange::new(111, 120) {
|
||||||
|
std::result::Result::Ok(value) => value,
|
||||||
|
std::result::Result::Err(error) => panic!("unexpected separate range failure: {error}"),
|
||||||
|
};
|
||||||
|
assert!(first.overlaps_or_is_adjacent(overlap));
|
||||||
|
assert!(first.overlaps_or_is_adjacent(adjacent));
|
||||||
|
assert!(!first.overlaps_or_is_adjacent(separate));
|
||||||
|
let merged = first.try_merge(adjacent);
|
||||||
|
assert!(matches!(merged, std::result::Result::Ok(std::option::Option::Some(value)) if value.start_slot() == 100 && value.end_slot() == 120));
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn pre_002_target_coverage_deduplicates_exact_scopes_without_cross_commitment_broadening() {
|
||||||
|
let first = match capability(1, ksp_onchain_transport_lib::SolanaCommitment::Confirmed, exact_scope("standard_logs", 7)) {
|
||||||
|
std::option::Option::Some(value) => value,
|
||||||
|
std::option::Option::None => panic!("first capability fixture unavailable"),
|
||||||
|
};
|
||||||
|
let second = match capability(2, ksp_onchain_transport_lib::SolanaCommitment::Confirmed, exact_scope("standard_logs", 7)) {
|
||||||
|
std::option::Option::Some(value) => value,
|
||||||
|
std::option::Option::None => panic!("second capability fixture unavailable"),
|
||||||
|
};
|
||||||
|
let finalized_full =
|
||||||
|
match capability(3, ksp_onchain_transport_lib::SolanaCommitment::Finalized, crate::RawTransactionIngestCoverageScope::full_ledger_transactions()) {
|
||||||
|
std::option::Option::Some(value) => value,
|
||||||
|
std::option::Option::None => panic!("full capability fixture unavailable"),
|
||||||
|
};
|
||||||
|
let contracts = match crate::RawTransactionIngestContinuityContracts::new(std::vec![first, second, finalized_full]) {
|
||||||
|
std::result::Result::Ok(value) => value,
|
||||||
|
std::result::Result::Err(error) => panic!("continuity contracts fixture failed: {error}"),
|
||||||
|
};
|
||||||
|
assert_eq!(contracts.capabilities.len(), 3);
|
||||||
|
assert_eq!(contracts.target_coverage.requirements.len(), 2);
|
||||||
|
assert!(contracts.target_coverage.requirements.iter().any(|requirement| {
|
||||||
|
return requirement.commitment == ksp_onchain_transport_lib::SolanaCommitment::Confirmed && requirement.scope == exact_scope("standard_logs", 7);
|
||||||
|
}));
|
||||||
|
assert!(contracts.target_coverage.requirements.iter().any(|requirement| {
|
||||||
|
return requirement.commitment == ksp_onchain_transport_lib::SolanaCommitment::Finalized
|
||||||
|
&& requirement.scope == crate::RawTransactionIngestCoverageScope::full_ledger_transactions();
|
||||||
|
}));
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn pre_002_full_ledger_scope_subsumes_only_same_commitment_targets() {
|
||||||
|
let exact = match capability(1, ksp_onchain_transport_lib::SolanaCommitment::Confirmed, exact_scope("helius_transaction", 4)) {
|
||||||
|
std::option::Option::Some(value) => value,
|
||||||
|
std::option::Option::None => panic!("exact capability fixture unavailable"),
|
||||||
|
};
|
||||||
|
let full = match capability(2, ksp_onchain_transport_lib::SolanaCommitment::Confirmed, crate::RawTransactionIngestCoverageScope::full_ledger_transactions())
|
||||||
|
{
|
||||||
|
std::option::Option::Some(value) => value,
|
||||||
|
std::option::Option::None => panic!("full capability fixture unavailable"),
|
||||||
|
};
|
||||||
|
let contracts = match crate::RawTransactionIngestContinuityContracts::new(std::vec![exact, full]) {
|
||||||
|
std::result::Result::Ok(value) => value,
|
||||||
|
std::result::Result::Err(error) => panic!("continuity contracts fixture failed: {error}"),
|
||||||
|
};
|
||||||
|
assert_eq!(contracts.target_coverage.requirements.len(), 1);
|
||||||
|
assert_eq!(contracts.target_coverage.requirements[0].scope, crate::RawTransactionIngestCoverageScope::full_ledger_transactions());
|
||||||
|
assert_eq!(contracts.target_coverage.requirements[0].commitment, ksp_onchain_transport_lib::SolanaCommitment::Confirmed);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn pre_002_known_references_can_never_be_configured_target_coverage() {
|
||||||
|
let network = match network() {
|
||||||
|
std::option::Option::Some(value) => value,
|
||||||
|
std::option::Option::None => panic!("network fixture unavailable"),
|
||||||
|
};
|
||||||
|
let result = crate::RawTransactionIngestRepairCapabilityDescriptor::new(
|
||||||
|
[1_u8; 32],
|
||||||
|
network,
|
||||||
|
ksp_onchain_transport_lib::SolanaCommitment::Confirmed,
|
||||||
|
crate::RawTransactionIngestCoverageScope::KnownReferences([2_u8; 32]),
|
||||||
|
true,
|
||||||
|
false,
|
||||||
|
false,
|
||||||
|
true,
|
||||||
|
false,
|
||||||
|
false,
|
||||||
|
);
|
||||||
|
assert!(result.is_err());
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn pre_002_gap_ledger_rejects_coalescible_open_ranges_and_active_overflow() {
|
||||||
|
let network = match network() {
|
||||||
|
std::option::Option::Some(value) => value,
|
||||||
|
std::option::Option::None => panic!("network fixture unavailable"),
|
||||||
|
};
|
||||||
|
let first = match gap(1, 1, 100, 109, super::RawTransactionIngestGapState::Pending) {
|
||||||
|
std::option::Option::Some(value) => value,
|
||||||
|
std::option::Option::None => panic!("first gap fixture unavailable"),
|
||||||
|
};
|
||||||
|
let adjacent = match gap(2, 1, 110, 120, super::RawTransactionIngestGapState::Pending) {
|
||||||
|
std::option::Option::Some(value) => value,
|
||||||
|
std::option::Option::None => panic!("adjacent gap fixture unavailable"),
|
||||||
|
};
|
||||||
|
let adjacent_ledger = super::RawTransactionIngestGapLedger { gaps: std::vec![first, adjacent], network: network.clone(), next_gap_id: 3 };
|
||||||
|
assert!(adjacent_ledger.validate_invariants().is_err());
|
||||||
|
let first_active = match gap(1, 1, 100, 109, super::RawTransactionIngestGapState::Repairing) {
|
||||||
|
std::option::Option::Some(value) => value,
|
||||||
|
std::option::Option::None => panic!("first active gap fixture unavailable"),
|
||||||
|
};
|
||||||
|
let second_active = match gap(2, 2, 200, 209, super::RawTransactionIngestGapState::Repairing) {
|
||||||
|
std::option::Option::Some(value) => value,
|
||||||
|
std::option::Option::None => panic!("second active gap fixture unavailable"),
|
||||||
|
};
|
||||||
|
let active_ledger = super::RawTransactionIngestGapLedger { gaps: std::vec![first_active, second_active], network, next_gap_id: 3 };
|
||||||
|
assert!(active_ledger.validate_invariants().is_err());
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn pre_002_gap_ledger_enforces_open_gap_bound_and_next_id_monotonicity() {
|
||||||
|
let network = match network() {
|
||||||
|
std::option::Option::Some(value) => value,
|
||||||
|
std::option::Option::None => panic!("network fixture unavailable"),
|
||||||
|
};
|
||||||
|
let mut gaps = std::vec::Vec::new();
|
||||||
|
for index in 0..=crate::MAX_RAW_TRANSACTION_INGEST_OPEN_REPAIR_GAPS {
|
||||||
|
let gap_id = (index as u64) + 1;
|
||||||
|
let slot = (index as u64) * 2;
|
||||||
|
let gap = match gap(gap_id, 1, slot, slot, super::RawTransactionIngestGapState::Pending) {
|
||||||
|
std::option::Option::Some(value) => value,
|
||||||
|
std::option::Option::None => panic!("gap fixture unavailable"),
|
||||||
|
};
|
||||||
|
gaps.push(gap);
|
||||||
|
}
|
||||||
|
let ledger = super::RawTransactionIngestGapLedger { gaps, network: network.clone(), next_gap_id: 66 };
|
||||||
|
assert!(ledger.validate_invariants().is_err());
|
||||||
|
let single = match gap(7, 1, 100, 100, super::RawTransactionIngestGapState::Unresolved) {
|
||||||
|
std::option::Option::Some(value) => value,
|
||||||
|
std::option::Option::None => panic!("single gap fixture unavailable"),
|
||||||
|
};
|
||||||
|
let stale_next = super::RawTransactionIngestGapLedger { gaps: std::vec![single], network, next_gap_id: 7 };
|
||||||
|
assert!(stale_next.validate_invariants().is_err());
|
||||||
|
return;
|
||||||
|
}
|
||||||
257
deltas/0.3.14/pre.002.md
Normal file
257
deltas/0.3.14/pre.002.md
Normal file
@@ -0,0 +1,257 @@
|
|||||||
|
<!-- file: deltas/0.3.14/pre.002.md -->
|
||||||
|
<!-- version: 1 -->
|
||||||
|
|
||||||
|
# Delta `0.3.14-pre.002` — contrats run-local gap / range / coverage
|
||||||
|
|
||||||
|
## Base requise
|
||||||
|
|
||||||
|
```text
|
||||||
|
0.3.14-pre.001
|
||||||
|
workspace.package.version = 0.3.14-pre.1
|
||||||
|
deltas/0.3.14/pre.001.md présent
|
||||||
|
```
|
||||||
|
|
||||||
|
Gate opérateur communiqué sur `0.3.14-pre.1` :
|
||||||
|
|
||||||
|
```text
|
||||||
|
cargo fmt --all -- --check : PASS
|
||||||
|
General Rust rule audit : clean
|
||||||
|
Rust export completeness audit : 0 candidate(s)
|
||||||
|
KSP workspace Rust rule audit : clean
|
||||||
|
Markdown table audit : clean (306 tables / 181 files sur le périmètre demandé)
|
||||||
|
cargo check --workspace : PASS
|
||||||
|
```
|
||||||
|
|
||||||
|
## Objectif
|
||||||
|
|
||||||
|
Matérialiser uniquement la tranche `pre.002` du plan `035` :
|
||||||
|
|
||||||
|
```text
|
||||||
|
contrats run-local de gap et plage inclusive
|
||||||
|
TargetCoverage conservative par commitment et scope
|
||||||
|
FullLedgerTransactions / ExactSourceScope / KnownReferences
|
||||||
|
inventaire privé des capabilities de repair déjà composées
|
||||||
|
ledger run-local borné et invariants de coalescence
|
||||||
|
bornes de future discovery / fetch / active repair
|
||||||
|
raccord pur aux RuntimeResources avant spawn des sources
|
||||||
|
```
|
||||||
|
|
||||||
|
Cette tranche ne réalise aucun repair, replay, scan HTTP, hydration de gap ou I/O Store supplémentaire.
|
||||||
|
|
||||||
|
## Fichiers ajoutés
|
||||||
|
|
||||||
|
```text
|
||||||
|
crates/ksp-worker-raw-transaction-ingest-lib/src/continuity.rs
|
||||||
|
crates/ksp-worker-raw-transaction-ingest-lib/unit_tests/continuity.rs
|
||||||
|
deltas/0.3.14/pre.002.md
|
||||||
|
```
|
||||||
|
|
||||||
|
## Fichiers modifiés
|
||||||
|
|
||||||
|
```text
|
||||||
|
Cargo.toml
|
||||||
|
crates/ksp-worker-raw-transaction-ingest-lib/src/lib.rs
|
||||||
|
crates/ksp-worker-raw-transaction-ingest-lib/src/runtime_resources.rs
|
||||||
|
crates/ksp-worker-raw-transaction-ingest-lib/tests/hardening.rs
|
||||||
|
crates/ksp-worker-raw-transaction-ingest-lib/tests/release_completeness.rs
|
||||||
|
```
|
||||||
|
|
||||||
|
## Fichiers supprimés
|
||||||
|
|
||||||
|
```text
|
||||||
|
aucun
|
||||||
|
```
|
||||||
|
|
||||||
|
## Contrats introduits
|
||||||
|
|
||||||
|
Bornes run-local :
|
||||||
|
|
||||||
|
```text
|
||||||
|
MAX_RAW_TRANSACTION_INGEST_OPEN_REPAIR_GAPS = 64
|
||||||
|
MAX_RAW_TRANSACTION_INGEST_REPAIR_RANGE_SLOTS = 4096
|
||||||
|
MAX_RAW_TRANSACTION_INGEST_REPAIR_DISCOVERY_WINDOW_SLOTS = 512
|
||||||
|
MAX_RAW_TRANSACTION_INGEST_REPAIR_BLOCK_FETCH_IN_FLIGHT = 4
|
||||||
|
MAX_RAW_TRANSACTION_INGEST_REPAIR_ACTIVE_GAPS = 1
|
||||||
|
```
|
||||||
|
|
||||||
|
Le scope de coverage distingue explicitement :
|
||||||
|
|
||||||
|
```text
|
||||||
|
FullLedgerTransactions
|
||||||
|
ExactSourceScope(family, semantic_fingerprint)
|
||||||
|
KnownReferences(fingerprint)
|
||||||
|
```
|
||||||
|
|
||||||
|
`KnownReferences` est admis comme notion de capability mais rejeté comme `TargetCoverage` configurée : connaître des signatures à hydrater ne prouve jamais qu'aucune transaction n'a été perdue.
|
||||||
|
|
||||||
|
Les plages de gap sont inclusives, bornées, validées avec arithmétique checked et coalescibles seulement lorsqu'elles se chevauchent ou sont adjacentes sans dépasser la borne maximale.
|
||||||
|
|
||||||
|
Le ledger impose notamment :
|
||||||
|
|
||||||
|
```text
|
||||||
|
réseau unique du run
|
||||||
|
GapId non nul, unique et monotone
|
||||||
|
maximum 64 gaps ouverts
|
||||||
|
maximum 1 gap en Repairing
|
||||||
|
pas de ranges ouverts coalescibles du même source/scope/commitment/reason
|
||||||
|
aucun wrap implicite des compteurs ou bornes
|
||||||
|
```
|
||||||
|
|
||||||
|
## Capabilities source
|
||||||
|
|
||||||
|
L'inventaire est dérivé des `RuntimeResources` déjà validées, sans I/O :
|
||||||
|
|
||||||
|
```text
|
||||||
|
HTTP Block Polling All -> FullLedgerTransactions
|
||||||
|
Standard Block All -> FullLedgerTransactions
|
||||||
|
Standard Block filtered -> ExactSourceScope
|
||||||
|
Standard Logs -> ExactSourceScope
|
||||||
|
Helius transactionSubscribe -> ExactSourceScope
|
||||||
|
Yellowstone -> ExactSourceScope provider-neutral
|
||||||
|
```
|
||||||
|
|
||||||
|
La capability de scan HTTP n'est pas inférée de la simple présence d'un `HttpTransportPool`. Elle est déclarée seulement lorsque le rôle fourni expose réellement, sur le réseau attendu :
|
||||||
|
|
||||||
|
```text
|
||||||
|
getBlock
|
||||||
|
getSlot
|
||||||
|
et au moins getBlocks ou getBlocksWithLimit
|
||||||
|
```
|
||||||
|
|
||||||
|
La disponibilité de `getTransaction` est détectée séparément pour la future hydration de références.
|
||||||
|
|
||||||
|
Pour Yellowstone, le fingerprint de coverage retire du calcul les éléments de transport/continuité qui ne définissent pas le scope métier :
|
||||||
|
|
||||||
|
```text
|
||||||
|
commitment
|
||||||
|
from_slot
|
||||||
|
ping
|
||||||
|
```
|
||||||
|
|
||||||
|
Le scope reste volontairement conservatif : aucune équivalence cross-family ou provider n'est inventée.
|
||||||
|
|
||||||
|
## Intégration runtime
|
||||||
|
|
||||||
|
Avant le spawn des sources live, `RawTransactionIngestRuntimeResources` :
|
||||||
|
|
||||||
|
```text
|
||||||
|
construit exactement un descriptor par source
|
||||||
|
valide réseau / source keys / TargetCoverage / bornes
|
||||||
|
construit un ledger vide run-local
|
||||||
|
conserve ces contrats pendant la durée du supervisor
|
||||||
|
revalide les invariants à la fermeture
|
||||||
|
```
|
||||||
|
|
||||||
|
Aucune nouvelle tâche, socket, retry, requête HTTP/gRPC/WS, lecture Config ou opération Store n'est ajoutée par cette tranche.
|
||||||
|
|
||||||
|
## Tests ajoutés / renforcés
|
||||||
|
|
||||||
|
Les tests couvrent :
|
||||||
|
|
||||||
|
```text
|
||||||
|
valeurs exactes des cinq bornes
|
||||||
|
plages inclusives / reversed / maximum / overflow
|
||||||
|
chevauchement et adjacency
|
||||||
|
merge borné
|
||||||
|
TargetCoverage exact dédupliquée
|
||||||
|
absence de broadening entre commitments
|
||||||
|
subsomption FullLedger seulement au même commitment
|
||||||
|
rejet de KnownReferences comme target
|
||||||
|
limites open/active gaps
|
||||||
|
monotonicité GapId
|
||||||
|
module production continuity privé
|
||||||
|
absence d'I/O et de nouvelles frontières dans continuity.rs
|
||||||
|
inventaire de modules de production mis à jour
|
||||||
|
```
|
||||||
|
|
||||||
|
La surface publique du crate reste inchangée ; les nouveaux contrats sont `pub(crate)` et réexportés uniquement par la façade crate-root pour usage interne conforme aux règles KSP.
|
||||||
|
|
||||||
|
## Dépendances / features
|
||||||
|
|
||||||
|
```text
|
||||||
|
aucune nouvelle dépendance
|
||||||
|
aucune nouvelle feature
|
||||||
|
aucun SDK provider
|
||||||
|
aucun accès Config depuis le Worker
|
||||||
|
aucun accès Job Backfill depuis le Worker
|
||||||
|
aucun backend Store physique depuis le Worker
|
||||||
|
```
|
||||||
|
|
||||||
|
## Version Cargo
|
||||||
|
|
||||||
|
```text
|
||||||
|
header Cargo.toml : 566 -> 567
|
||||||
|
workspace.package.version : 0.3.14-pre.1 -> 0.3.14-pre.2
|
||||||
|
```
|
||||||
|
|
||||||
|
## Validations exécutées
|
||||||
|
|
||||||
|
Dans le sandbox de préparation :
|
||||||
|
|
||||||
|
```text
|
||||||
|
python3 scripts/audit_rust_workspace_rules.py
|
||||||
|
python3 scripts/audit_markdown_tables.py README.md RULES.md ROADMAP.md CHANGELOG.md docs prompts crates deltas/0.3.14
|
||||||
|
scan statique frontières / I/O continuity
|
||||||
|
comparaison exacte avec la base pre.001
|
||||||
|
contrôle contenu archive delta
|
||||||
|
unzip -t archive delta
|
||||||
|
```
|
||||||
|
|
||||||
|
Résultats :
|
||||||
|
|
||||||
|
```text
|
||||||
|
General Rust rule audit : clean
|
||||||
|
Rust export completeness audit : 0 candidate(s)
|
||||||
|
KSP workspace Rust rule audit : clean
|
||||||
|
Markdown table audit : clean (306 tables / 182 files)
|
||||||
|
scan continuity I/O / frontières : PASS
|
||||||
|
scan surface publique continuity : PASS
|
||||||
|
comparaison pre.001 -> pre.002 : 3 ajouts / 5 modifications / 0 suppression
|
||||||
|
```
|
||||||
|
|
||||||
|
Le contrôle ZIP est exécuté après construction de l'archive de livraison ; toute archive qui échoue ce contrôle est rejetée et n'est pas livrée.
|
||||||
|
|
||||||
|
## Validations non exécutées
|
||||||
|
|
||||||
|
Le sandbox de préparation ne fournit pas le toolchain Cargo/Rust ; les gates suivants ne peuvent donc pas constituer une preuve locale :
|
||||||
|
|
||||||
|
```text
|
||||||
|
cargo fmt --all -- --check
|
||||||
|
cargo check --workspace
|
||||||
|
cargo clippy --workspace --all-targets --all-features -- -D warnings
|
||||||
|
cargo test -p ksp-worker-raw-transaction-ingest-lib --all-targets --all-features
|
||||||
|
```
|
||||||
|
|
||||||
|
Ils doivent être exécutés côté opérateur après application du delta.
|
||||||
|
|
||||||
|
## Décisions prises
|
||||||
|
|
||||||
|
```text
|
||||||
|
pre.002 reste sans repair I/O
|
||||||
|
TargetCoverage est conservative et commitment-sensitive
|
||||||
|
FullLedger ne broaden jamais un commitment différent
|
||||||
|
KnownReferences ne prouve jamais la complétude
|
||||||
|
HTTP repair capability est détectée par méthodes réellement routables
|
||||||
|
Yellowstone coverage identity exclut cursor replay et identité provider/endpoint
|
||||||
|
les contrats restent privés au Worker
|
||||||
|
aucune nouvelle dépendance ou frontière architecturale
|
||||||
|
```
|
||||||
|
|
||||||
|
## Questions ouvertes
|
||||||
|
|
||||||
|
```text
|
||||||
|
aucune pour pre.002
|
||||||
|
```
|
||||||
|
|
||||||
|
La tranche suivante reste `pre.003` : observabilité latest-value de continuité WebSocket, sans encore déclencher le repair.
|
||||||
|
|
||||||
|
## Gate opérateur après application
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cargo fmt --all -- --check
|
||||||
|
python3 scripts/audit_rust_workspace_rules.py
|
||||||
|
python3 scripts/audit_markdown_tables.py README.md RULES.md ROADMAP.md CHANGELOG.md docs prompts crates deltas/0.3.14
|
||||||
|
cargo check --workspace
|
||||||
|
cargo clippy --workspace --all-targets --all-features -- -D warnings
|
||||||
|
cargo test -p ksp-worker-raw-transaction-ingest-lib --all-targets --all-features
|
||||||
|
```
|
||||||
Reference in New Issue
Block a user