v0.3.14-pre.002

This commit is contained in:
2026-09-11 19:58:32 +02:00
parent 2ca56dbaf4
commit b2068e9ef2
8 changed files with 1194 additions and 10 deletions

View File

@@ -0,0 +1,435 @@
// file: crates/ksp-worker-raw-transaction-ingest-lib/src/continuity.rs
// version: 1
/// Maximum number of simultaneously retained non-repaired run-local gaps.
pub(crate) const MAX_RAW_TRANSACTION_INGEST_OPEN_REPAIR_GAPS: usize = 64;
/// Maximum number of run-local gaps that may actively repair at once.
pub(crate) const MAX_RAW_TRANSACTION_INGEST_REPAIR_ACTIVE_GAPS: usize = 1;
/// Maximum number of logical block fetches that a later repair scheduler may admit concurrently.
pub(crate) const MAX_RAW_TRANSACTION_INGEST_REPAIR_BLOCK_FETCH_IN_FLIGHT: usize = 4;
/// Maximum number of slots admitted by one later HTTP repair discovery window.
pub(crate) const MAX_RAW_TRANSACTION_INGEST_REPAIR_DISCOVERY_WINDOW_SLOTS: u64 = 512;
/// Maximum inclusive slot span admitted for one run-local repair gap.
pub(crate) const MAX_RAW_TRANSACTION_INGEST_REPAIR_RANGE_SLOTS: u64 = 4_096;
/// Private source scope used to prove whether one configured live source can cover another run-local requirement.
#[derive(Clone, Debug, Eq, PartialEq)]
pub(crate) enum RawTransactionIngestCoverageScope {
/// Every transaction contained by every produced block at the configured commitment.
FullLedgerTransactions,
/// One opaque exact source-family scope identified independently from provider and endpoint identity.
ExactSourceScope(&'static str, [u8; 32]),
/// A bounded set of references already observed by the Worker; never sufficient as configured target coverage.
KnownReferences([u8; 32]),
}
impl crate::RawTransactionIngestCoverageScope {
/// Creates one provider-neutral exact source scope from a stable family code and semantic fingerprint.
pub(crate) const fn exact_source_scope(family_code: &'static str, fingerprint: [u8; 32]) -> Self {
return Self::ExactSourceScope(family_code, fingerprint);
}
/// Creates the full-ledger transaction scope used by complete block sources.
pub(crate) const fn full_ledger_transactions() -> Self {
return Self::FullLedgerTransactions;
}
fn validate_configured_target_scope(&self) -> ksp_core_lib::Result<()> {
return match self {
Self::FullLedgerTransactions => std::result::Result::Ok(()),
Self::ExactSourceScope(family_code, fingerprint) => {
if family_code.is_empty() {
return std::result::Result::Err(crate::runtime_error("continuity.exact_scope_family_empty"));
}
let _fingerprint_first_byte = fingerprint[0];
std::result::Result::Ok(())
},
Self::KnownReferences(fingerprint) => {
let _fingerprint_first_byte = fingerprint[0];
std::result::Result::Err(crate::runtime_error("continuity.known_references_not_target_scope"))
},
};
}
fn is_known_references(&self) -> bool {
return matches!(self, Self::KnownReferences(_));
}
}
#[derive(Clone, Debug, Eq, PartialEq)]
struct RawTransactionIngestCoverageRequirement {
commitment: ksp_onchain_transport_lib::SolanaCommitment,
scope: crate::RawTransactionIngestCoverageScope,
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
enum RawTransactionIngestGapReason {
HttpProducedBlockUnavailable,
SourceFailure,
TransportOverflow,
WebSocketReconnect,
YellowstoneRetention,
}
impl RawTransactionIngestGapReason {
const ALL: [Self; 5] =
[Self::HttpProducedBlockUnavailable, Self::SourceFailure, Self::TransportOverflow, Self::WebSocketReconnect, Self::YellowstoneRetention];
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
enum RawTransactionIngestGapState {
Pending,
Repairing,
Repaired,
Unresolved,
}
impl RawTransactionIngestGapState {
const ALL: [Self; 4] = [Self::Pending, Self::Repairing, Self::Repaired, Self::Unresolved];
const fn is_open(self) -> bool {
return !matches!(self, Self::Repaired);
}
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
struct RawTransactionIngestGapRange {
end_slot: u64,
start_slot: u64,
}
impl RawTransactionIngestGapRange {
fn new(start_slot: u64, end_slot: u64) -> ksp_core_lib::Result<Self> {
if end_slot < start_slot {
return std::result::Result::Err(crate::runtime_error("continuity.gap_range_reversed"));
}
let slot_count = match end_slot.checked_sub(start_slot).and_then(|value| return value.checked_add(1)) {
std::option::Option::Some(value) => value,
std::option::Option::None => return std::result::Result::Err(crate::runtime_error("continuity.gap_range_overflow")),
};
if slot_count > crate::MAX_RAW_TRANSACTION_INGEST_REPAIR_RANGE_SLOTS {
return std::result::Result::Err(crate::runtime_error("continuity.gap_range_too_large"));
}
return std::result::Result::Ok(Self { end_slot, start_slot });
}
const fn end_slot(self) -> u64 {
return self.end_slot;
}
fn overlaps_or_is_adjacent(self, other: Self) -> bool {
if self.start_slot <= other.end_slot && other.start_slot <= self.end_slot {
return true;
}
let self_next = self.end_slot.checked_add(1);
if self_next == std::option::Option::Some(other.start_slot) {
return true;
}
let other_next = other.end_slot.checked_add(1);
return other_next == std::option::Option::Some(self.start_slot);
}
const fn start_slot(self) -> u64 {
return self.start_slot;
}
fn try_merge(self, other: Self) -> ksp_core_lib::Result<std::option::Option<Self>> {
if !self.overlaps_or_is_adjacent(other) {
return std::result::Result::Ok(std::option::Option::None);
}
let start_slot = self.start_slot.min(other.start_slot);
let end_slot = self.end_slot.max(other.end_slot);
let slot_count = match end_slot.checked_sub(start_slot).and_then(|value| return value.checked_add(1)) {
std::option::Option::Some(value) => value,
std::option::Option::None => return std::result::Result::Ok(std::option::Option::None),
};
if slot_count > crate::MAX_RAW_TRANSACTION_INGEST_REPAIR_RANGE_SLOTS {
return std::result::Result::Ok(std::option::Option::None);
}
return std::result::Result::Ok(std::option::Option::Some(Self { end_slot, start_slot }));
}
}
#[derive(Clone, Copy, Debug, Eq, PartialEq, Ord, PartialOrd)]
struct RawTransactionIngestGapId(u64);
struct RawTransactionIngestGap {
commitment: ksp_onchain_transport_lib::SolanaCommitment,
coverage_requirement: crate::RawTransactionIngestCoverageScope,
gap_id: RawTransactionIngestGapId,
network: ksp_store_lib::RawNetworkId,
range: RawTransactionIngestGapRange,
reason: RawTransactionIngestGapReason,
source_key: [u8; 32],
state: RawTransactionIngestGapState,
}
struct RawTransactionIngestGapLedger {
gaps: std::vec::Vec<RawTransactionIngestGap>,
network: ksp_store_lib::RawNetworkId,
next_gap_id: u64,
}
impl RawTransactionIngestGapLedger {
fn new(network: ksp_store_lib::RawNetworkId) -> Self {
return Self { gaps: std::vec::Vec::new(), network, next_gap_id: 1 };
}
fn validate_invariants(&self) -> ksp_core_lib::Result<()> {
if self.next_gap_id == 0 {
return std::result::Result::Err(crate::runtime_error("continuity.gap_id_exhausted"));
}
let mut ids = std::collections::BTreeSet::new();
let mut open_gap_count = 0_usize;
let mut active_gap_count = 0_usize;
let mut highest_gap_id = 0_u64;
for gap in &self.gaps {
if gap.network != self.network {
return std::result::Result::Err(crate::runtime_error("continuity.gap_network_mismatch"));
}
if gap.gap_id.0 == 0 || !ids.insert(gap.gap_id) {
return std::result::Result::Err(crate::runtime_error("continuity.gap_id_invalid"));
}
highest_gap_id = highest_gap_id.max(gap.gap_id.0);
if gap.state.is_open() {
open_gap_count = match open_gap_count.checked_add(1) {
std::option::Option::Some(value) => value,
std::option::Option::None => return std::result::Result::Err(crate::counter_exhausted_error("continuity.open_gap_count")),
};
}
if gap.state == RawTransactionIngestGapState::Repairing {
active_gap_count = match active_gap_count.checked_add(1) {
std::option::Option::Some(value) => value,
std::option::Option::None => return std::result::Result::Err(crate::counter_exhausted_error("continuity.active_gap_count")),
};
}
if !RawTransactionIngestGapReason::ALL.contains(&gap.reason) || !RawTransactionIngestGapState::ALL.contains(&gap.state) {
return std::result::Result::Err(crate::runtime_error("continuity.gap_catalog_invalid"));
}
if gap.range.start_slot() > gap.range.end_slot() {
return std::result::Result::Err(crate::runtime_error("continuity.gap_range_reversed"));
}
}
if open_gap_count > crate::MAX_RAW_TRANSACTION_INGEST_OPEN_REPAIR_GAPS {
return std::result::Result::Err(crate::runtime_error("continuity.open_gap_limit_exceeded"));
}
if active_gap_count > crate::MAX_RAW_TRANSACTION_INGEST_REPAIR_ACTIVE_GAPS {
return std::result::Result::Err(crate::runtime_error("continuity.active_gap_limit_exceeded"));
}
if !self.gaps.is_empty() && self.next_gap_id <= highest_gap_id {
return std::result::Result::Err(crate::runtime_error("continuity.next_gap_id_invalid"));
}
for first_index in 0..self.gaps.len() {
let first = &self.gaps[first_index];
for second in self.gaps.iter().skip(first_index + 1) {
if first.source_key != second.source_key
|| first.commitment != second.commitment
|| first.coverage_requirement != second.coverage_requirement
|| first.reason != second.reason
|| !first.state.is_open()
|| !second.state.is_open()
{
continue;
}
let merged = first.range.try_merge(second.range);
match merged {
std::result::Result::Ok(std::option::Option::Some(_)) => {
return std::result::Result::Err(crate::runtime_error("continuity.coalescible_gap_ranges"));
},
std::result::Result::Ok(std::option::Option::None) => {},
std::result::Result::Err(error) => return std::result::Result::Err(error),
}
}
}
if RawTransactionIngestGapReason::ALL.len() != 5 || RawTransactionIngestGapState::ALL.len() != 4 {
return std::result::Result::Err(crate::runtime_error("continuity.gap_catalog_invalid"));
}
return std::result::Result::Ok(());
}
}
/// Private source capability descriptor prepared without network I/O from already validated runtime resources.
pub(crate) struct RawTransactionIngestRepairCapabilityDescriptor {
block_material: bool,
commitment: ksp_onchain_transport_lib::SolanaCommitment,
http_block_scan: bool,
known_reference_hydration: bool,
native_replay: bool,
network: ksp_store_lib::RawNetworkId,
reference_bearing: bool,
slot_enumerating: bool,
source_key: [u8; 32],
source_scope: crate::RawTransactionIngestCoverageScope,
}
impl crate::RawTransactionIngestRepairCapabilityDescriptor {
/// Creates one private descriptor from source-local capabilities that were validated without issuing repair I/O.
#[allow(clippy::too_many_arguments)]
pub(crate) fn new(
source_key: [u8; 32],
network: ksp_store_lib::RawNetworkId,
commitment: ksp_onchain_transport_lib::SolanaCommitment,
source_scope: crate::RawTransactionIngestCoverageScope,
reference_bearing: bool,
block_material: bool,
slot_enumerating: bool,
known_reference_hydration: bool,
native_replay: bool,
http_block_scan: bool,
) -> ksp_core_lib::Result<Self> {
let descriptor = Self {
block_material,
commitment,
http_block_scan,
known_reference_hydration,
native_replay,
network,
reference_bearing,
slot_enumerating,
source_key,
source_scope,
};
if let std::result::Result::Err(error) = descriptor.validate() {
return std::result::Result::Err(error);
}
return std::result::Result::Ok(descriptor);
}
fn validate(&self) -> ksp_core_lib::Result<()> {
if self.commitment == ksp_onchain_transport_lib::SolanaCommitment::Processed {
return std::result::Result::Err(crate::runtime_error("continuity.processed_commitment_unsupported"));
}
if let std::result::Result::Err(error) = self.source_scope.validate_configured_target_scope() {
return std::result::Result::Err(error);
}
if !self.reference_bearing
&& !self.block_material
&& !self.slot_enumerating
&& !self.known_reference_hydration
&& !self.native_replay
&& !self.http_block_scan
{
return std::result::Result::Err(crate::runtime_error("continuity.source_capability_empty"));
}
if self.http_block_scan && (!self.block_material || !self.slot_enumerating) {
return std::result::Result::Err(crate::runtime_error("continuity.http_scan_capability_incomplete"));
}
return std::result::Result::Ok(());
}
}
struct RawTransactionIngestTargetCoverage {
requirements: std::vec::Vec<RawTransactionIngestCoverageRequirement>,
}
impl RawTransactionIngestTargetCoverage {
fn from_capabilities(capabilities: &[crate::RawTransactionIngestRepairCapabilityDescriptor]) -> ksp_core_lib::Result<Self> {
let mut value = Self { requirements: std::vec::Vec::new() };
for capability in capabilities {
if let std::result::Result::Err(error) = value.include(capability.commitment, capability.source_scope.clone()) {
return std::result::Result::Err(error);
}
}
if value.requirements.is_empty() {
return std::result::Result::Err(crate::runtime_error("continuity.target_coverage_empty"));
}
return std::result::Result::Ok(value);
}
fn include(
&mut self,
commitment: ksp_onchain_transport_lib::SolanaCommitment,
scope: crate::RawTransactionIngestCoverageScope,
) -> ksp_core_lib::Result<()> {
if scope.is_known_references() {
return std::result::Result::Err(crate::runtime_error("continuity.known_references_not_target_scope"));
}
if matches!(&scope, crate::RawTransactionIngestCoverageScope::FullLedgerTransactions) {
self.requirements.retain(|requirement| return requirement.commitment != commitment);
self.requirements.push(RawTransactionIngestCoverageRequirement { commitment, scope });
return std::result::Result::Ok(());
}
if self.requirements.iter().any(|requirement| {
return requirement.commitment == commitment
&& (requirement.scope == crate::RawTransactionIngestCoverageScope::FullLedgerTransactions || requirement.scope == scope);
}) {
return std::result::Result::Ok(());
}
self.requirements.push(RawTransactionIngestCoverageRequirement { commitment, scope });
return std::result::Result::Ok(());
}
}
/// Run-local continuity contracts prepared from the exact caller-composed live-source aggregate before productive source execution.
pub(crate) struct RawTransactionIngestContinuityContracts {
capabilities: std::vec::Vec<crate::RawTransactionIngestRepairCapabilityDescriptor>,
gap_ledger: RawTransactionIngestGapLedger,
target_coverage: RawTransactionIngestTargetCoverage,
}
impl crate::RawTransactionIngestContinuityContracts {
/// Builds the private capability inventory, conservative `TargetCoverage` and empty run-local gap ledger without issuing network or Store I/O.
pub(crate) fn new(capabilities: std::vec::Vec<crate::RawTransactionIngestRepairCapabilityDescriptor>) -> ksp_core_lib::Result<Self> {
if capabilities.is_empty() || capabilities.len() > crate::MAX_RAW_TRANSACTION_INGEST_LIVE_SOURCES {
return std::result::Result::Err(crate::runtime_error("continuity.capability_inventory_size_invalid"));
}
let network = capabilities[0].network.clone();
let mut source_keys = std::collections::BTreeSet::new();
for capability in &capabilities {
if let std::result::Result::Err(error) = capability.validate() {
return std::result::Result::Err(error);
}
if capability.network != network {
return std::result::Result::Err(crate::runtime_error("continuity.capability_network_mismatch"));
}
if !source_keys.insert(capability.source_key) {
return std::result::Result::Err(crate::runtime_error("continuity.duplicate_capability_source"));
}
}
let target_coverage = match RawTransactionIngestTargetCoverage::from_capabilities(capabilities.as_slice()) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let gap_ledger = RawTransactionIngestGapLedger::new(network);
if let std::result::Result::Err(error) = gap_ledger.validate_invariants() {
return std::result::Result::Err(error);
}
if !coverage_scope_catalog_is_complete() {
return std::result::Result::Err(crate::runtime_error("continuity.coverage_scope_catalog_invalid"));
}
if crate::MAX_RAW_TRANSACTION_INGEST_REPAIR_BLOCK_FETCH_IN_FLIGHT == 0
|| crate::MAX_RAW_TRANSACTION_INGEST_REPAIR_DISCOVERY_WINDOW_SLOTS == 0
|| crate::MAX_RAW_TRANSACTION_INGEST_REPAIR_DISCOVERY_WINDOW_SLOTS > crate::MAX_RAW_TRANSACTION_INGEST_REPAIR_RANGE_SLOTS
{
return std::result::Result::Err(crate::runtime_error("continuity.repair_bounds_invalid"));
}
return std::result::Result::Ok(Self { capabilities, gap_ledger, target_coverage });
}
/// Revalidates that the run-local contracts still correspond exactly to the caller-composed source count before tasks are spawned.
pub(crate) fn validate_for_source_count(&self, expected_source_count: usize) -> ksp_core_lib::Result<()> {
if self.capabilities.len() != expected_source_count || self.target_coverage.requirements.is_empty() {
return std::result::Result::Err(crate::runtime_error("continuity.source_inventory_mismatch"));
}
if let std::result::Result::Err(error) = self.gap_ledger.validate_invariants() {
return std::result::Result::Err(error);
}
return std::result::Result::Ok(());
}
}
fn coverage_scope_catalog_is_complete() -> bool {
let full = crate::RawTransactionIngestCoverageScope::FullLedgerTransactions;
let exact = crate::RawTransactionIngestCoverageScope::ExactSourceScope("catalog", [0_u8; 32]);
let known = crate::RawTransactionIngestCoverageScope::KnownReferences([0_u8; 32]);
return full.validate_configured_target_scope().is_ok()
&& exact.validate_configured_target_scope().is_ok()
&& known.validate_configured_target_scope().is_err()
&& !full.is_known_references()
&& !exact.is_known_references()
&& known.is_known_references();
}
#[cfg(test)]
#[path = "../unit_tests/continuity.rs"]
mod tests;

View File

@@ -1,5 +1,5 @@
// file: crates/ksp-worker-raw-transaction-ingest-lib/src/lib.rs
// version: 26
// version: 27
#![warn(missing_docs)]
#![deny(unreachable_pub)]
@@ -21,6 +21,7 @@
//! state and faults conservatively when Transport proves a replay-retention continuity gap; history remediation remains outside this crate.
mod admission;
mod continuity;
mod error;
mod identity;
mod persistence;
@@ -110,6 +111,22 @@ pub use self::snapshot::RawTransactionIngestSourceState;
pub(crate) use self::admission::RawTransactionAdmission;
/// Crate-private source-neutral ingress sent through the bounded central admission queue.
pub(crate) use self::admission::RawTransactionIngress;
/// Maximum number of simultaneously retained non-repaired run-local continuity gaps.
pub(crate) use self::continuity::MAX_RAW_TRANSACTION_INGEST_OPEN_REPAIR_GAPS;
/// Maximum number of run-local continuity gaps that may actively repair at once.
pub(crate) use self::continuity::MAX_RAW_TRANSACTION_INGEST_REPAIR_ACTIVE_GAPS;
/// Maximum number of logical block fetches reserved for the later bounded repair scheduler.
pub(crate) use self::continuity::MAX_RAW_TRANSACTION_INGEST_REPAIR_BLOCK_FETCH_IN_FLIGHT;
/// Maximum number of slots reserved for one later HTTP repair discovery window.
pub(crate) use self::continuity::MAX_RAW_TRANSACTION_INGEST_REPAIR_DISCOVERY_WINDOW_SLOTS;
/// Maximum inclusive slot span accepted by one run-local repair gap.
pub(crate) use self::continuity::MAX_RAW_TRANSACTION_INGEST_REPAIR_RANGE_SLOTS;
/// Private run-local continuity aggregate containing capabilities, TargetCoverage and the gap ledger.
pub(crate) use self::continuity::RawTransactionIngestContinuityContracts;
/// Private provider-neutral coverage scope used by run-local continuity proof contracts.
pub(crate) use self::continuity::RawTransactionIngestCoverageScope;
/// Private source capability descriptor prepared without repair I/O.
pub(crate) use self::continuity::RawTransactionIngestRepairCapabilityDescriptor;
/// Creates one terminal content-conflict error without copying conflicting material into diagnostics.
pub(crate) use self::error::content_conflict_error;
/// Creates one terminal counter-exhaustion error without exposing runtime material.

View File

@@ -1,5 +1,5 @@
// file: crates/ksp-worker-raw-transaction-ingest-lib/src/runtime_resources.rs
// version: 26
// version: 27
use sha2::Digest; // rust-rules: trait-import
@@ -31,6 +31,7 @@ const RAW_TRANSACTION_INGEST_STANDARD_BLOCK_PROTOCOL: &str = "solana_ws";
const RAW_TRANSACTION_INGEST_STANDARD_LOGS_FILTER_FINGERPRINT_DOMAIN: &[u8] = b"ksp.raw_transaction_ingest.standard_logs.filter.v1\0";
const RAW_TRANSACTION_INGEST_STANDARD_LOGS_HTTP_PROTOCOL: &str = "solana_ws_http";
const RAW_TRANSACTION_INGEST_STANDARD_LOGS_HTTP_SOURCE_KEY_DOMAIN: &[u8] = b"ksp.raw_transaction_ingest.standard_logs_http.source_key.v1\0";
const RAW_TRANSACTION_INGEST_YELLOWSTONE_COVERAGE_SCOPE_FINGERPRINT_DOMAIN: &[u8] = b"ksp.raw_transaction_ingest.yellowstone.coverage_scope.v1\0";
const RAW_TRANSACTION_INGEST_YELLOWSTONE_FILTER_FINGERPRINT_DOMAIN: &[u8] = b"ksp.raw_transaction_ingest.yellowstone.filters.v1\0";
const RAW_TRANSACTION_INGEST_YELLOWSTONE_HTTP_PROTOCOL: &str = "yellowstone_http";
const RAW_TRANSACTION_INGEST_YELLOWSTONE_HTTP_SOURCE_KEY_DOMAIN: &[u8] = b"ksp.raw_transaction_ingest.yellowstone_http.source_key.v1\0";
@@ -75,6 +76,86 @@ impl RawTransactionIngestLiveSource {
};
}
fn repair_capability_descriptor(&self) -> ksp_core_lib::Result<crate::RawTransactionIngestRepairCapabilityDescriptor> {
let commitment = match self {
Self::HeliusTransaction(source) => source.commitment,
Self::HttpBlockPolling(source) => source.commitment,
Self::StandardBlock(source) => source.commitment,
Self::StandardLogs(source) => source.commitment,
Self::Yellowstone(source) => match source.subscribe_request.commitment() {
std::option::Option::Some(value) => value,
std::option::Option::None => return std::result::Result::Err(crate::runtime_error("continuity.yellowstone_commitment_missing")),
},
};
let source_scope = match self {
Self::HeliusTransaction(source) => crate::RawTransactionIngestCoverageScope::exact_source_scope("helius_transaction", source.filter_fingerprint),
Self::HttpBlockPolling(_) => crate::RawTransactionIngestCoverageScope::full_ledger_transactions(),
Self::StandardBlock(source) => match &source.filter {
ksp_onchain_transport_lib::SolanaBlockSubscribeFilter::All => crate::RawTransactionIngestCoverageScope::full_ledger_transactions(),
ksp_onchain_transport_lib::SolanaBlockSubscribeFilter::MentionsAccountOrProgram(_) => {
crate::RawTransactionIngestCoverageScope::exact_source_scope("standard_block", source.filter_fingerprint)
},
},
Self::StandardLogs(source) => crate::RawTransactionIngestCoverageScope::exact_source_scope("standard_logs", source.filter_fingerprint),
Self::Yellowstone(source) => {
let fingerprint = match yellowstone_coverage_scope_fingerprint(&source.subscribe_request) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
crate::RawTransactionIngestCoverageScope::exact_source_scope("yellowstone", fingerprint)
},
};
let (reference_bearing, live_block_material, native_replay, http_block_scan, known_reference_hydration) = match self {
Self::HeliusTransaction(source) => {
let http_block_scan = match http_role_supports_repair_scan(&source.http_pool, &source.hydration_role, source.network.as_str()) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
(true, false, false, http_block_scan, true)
},
Self::HttpBlockPolling(source) => {
let known_reference_hydration =
match http_role_supports_rpc_method(&source.http_pool, &source.polling_role, "getTransaction", source.network.as_str()) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
(false, true, false, true, known_reference_hydration)
},
Self::StandardBlock(_) => (false, true, false, false, false),
Self::StandardLogs(source) => {
let http_block_scan = match http_role_supports_repair_scan(&source.http_pool, &source.hydration_role, source.network.as_str()) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
(true, false, false, http_block_scan, true)
},
Self::Yellowstone(source) => {
let http_block_scan = match http_role_supports_repair_scan(&source.http_pool, &source.hydration_role, source.network.as_str()) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let reference_bearing =
source.subscribe_request.transaction_filter_count() > 0 || source.subscribe_request.transaction_status_filter_count() > 0;
let live_block_material = source.subscribe_request.block_filter_count() > 0;
(reference_bearing, live_block_material, true, http_block_scan, true)
},
};
let block_material = live_block_material || http_block_scan;
let slot_enumerating = http_block_scan;
return crate::RawTransactionIngestRepairCapabilityDescriptor::new(
self.source_key(),
self.network().clone(),
commitment,
source_scope,
reference_bearing,
block_material,
slot_enumerating,
known_reference_hydration,
native_replay,
http_block_scan,
);
}
fn source_key(&self) -> [u8; 32] {
return match self {
Self::HeliusTransaction(source) => source.source_key,
@@ -2023,6 +2104,22 @@ impl crate::RawTransactionIngestRuntimeResources {
if self.sources.is_empty() || self.sources.len() > crate::MAX_RAW_TRANSACTION_INGEST_LIVE_SOURCES {
return std::result::Result::Err(crate::runtime_error("runtime_resources.source_collection_invalid"));
}
let mut repair_capabilities = std::vec::Vec::with_capacity(self.sources.len());
for source in &self.sources {
let capability = match source.repair_capability_descriptor() {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
repair_capabilities.push(capability);
}
let continuity_contracts = match crate::RawTransactionIngestContinuityContracts::new(repair_capabilities) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let continuity_source_count = self.sources.len();
if let std::result::Result::Err(error) = continuity_contracts.validate_for_source_count(continuity_source_count) {
return std::result::Result::Err(error);
}
let source_keys = self.sources.iter().map(RawTransactionIngestLiveSource::source_key).collect::<std::vec::Vec<_>>();
let hydration_source_count = self.sources.iter().filter(|source| return source.uses_hydration()).count();
if let std::result::Result::Err(error) =
@@ -2072,7 +2169,11 @@ impl crate::RawTransactionIngestRuntimeResources {
});
}
std::mem::drop(admission_sender);
return supervise_live_source_tasks(stop_receiver, source_stop_sender, children).await;
let result = supervise_live_source_tasks(stop_receiver, source_stop_sender, children).await;
if let std::result::Result::Err(error) = continuity_contracts.validate_for_source_count(continuity_source_count) {
return std::result::Result::Err(error);
}
return result;
}
}
@@ -2205,6 +2306,57 @@ async fn drain_live_source_tasks(
};
}
fn http_role_supports_repair_scan(
pool: &ksp_onchain_transport_lib::HttpTransportPool,
role: &ksp_onchain_transport_lib::HttpRoleName,
expected_cluster: &str,
) -> ksp_core_lib::Result<bool> {
let has_get_block = match http_role_supports_rpc_method(pool, role, "getBlock", expected_cluster) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let has_get_blocks = match http_role_supports_rpc_method(pool, role, "getBlocks", expected_cluster) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let has_get_blocks_with_limit = match http_role_supports_rpc_method(pool, role, "getBlocksWithLimit", expected_cluster) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let has_get_slot = match http_role_supports_rpc_method(pool, role, "getSlot", expected_cluster) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
return std::result::Result::Ok(has_get_block && has_get_slot && (has_get_blocks || has_get_blocks_with_limit));
}
fn http_role_supports_rpc_method(
pool: &ksp_onchain_transport_lib::HttpTransportPool,
role: &ksp_onchain_transport_lib::HttpRoleName,
method_name: &'static str,
expected_cluster: &str,
) -> ksp_core_lib::Result<bool> {
let method = match ksp_onchain_transport_lib::find_http_rpc_method(method_name) {
std::option::Option::Some(value) => value,
std::option::Option::None => return std::result::Result::Err(crate::runtime_error("continuity.http_method_missing")),
};
let snapshot = pool.snapshot();
for endpoint in snapshot.endpoints() {
if !endpoint.enabled() || endpoint.cluster() != expected_cluster {
continue;
}
for endpoint_role in endpoint.roles() {
if !endpoint_role.enabled() || endpoint_role.role() != role.as_str() {
continue;
}
if http_role_supports_request_kind(endpoint_role, method.request_kind()) {
return std::result::Result::Ok(true);
}
}
}
return std::result::Result::Ok(false);
}
fn http_block_polling_live_source_key(
network: &ksp_store_lib::RawNetworkId,
polling_role: &ksp_onchain_transport_lib::HttpRoleName,
@@ -2639,6 +2791,22 @@ fn standard_logs_live_source_key(
return hasher.finalize().into();
}
fn yellowstone_coverage_scope_fingerprint(request: &ksp_onchain_transport_lib::YellowstoneSubscribeRequest) -> ksp_core_lib::Result<[u8; 32]> {
let mut normalized = request.clone();
normalized.set_commitment(std::option::Option::None);
normalized.set_from_slot(std::option::Option::None);
normalized.set_ping(std::option::Option::None);
let identity = match normalized.identity() {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return std::result::Result::Err(crate::runtime_error("continuity.yellowstone_scope_identity_invalid")),
};
let mut hasher = sha2::Sha256::new();
hasher.update(RAW_TRANSACTION_INGEST_YELLOWSTONE_COVERAGE_SCOPE_FINGERPRINT_DOMAIN);
let mut writer = RawTransactionIngestSourceKeyHashWriter { hasher: &mut hasher };
std::hash::Hash::hash(&identity, &mut writer);
return std::result::Result::Ok(hasher.finalize().into());
}
fn yellowstone_live_source_key(
network: &ksp_store_lib::RawNetworkId,
route: &RawTransactionIngestSourceRoute,

View File

@@ -1,7 +1,7 @@
// file: crates/ksp-worker-raw-transaction-ingest-lib/tests/hardening.rs
// version: 24
// version: 25
//! External public, security, redaction and release-boundary hardening canaries through `pre.012`.
//! External public, security, redaction and release-boundary hardening canaries through `v0.3.14-pre.002`.
fn network(value: &'static str) -> std::option::Option<ksp_store_lib::RawNetworkId> {
let result = ksp_store_lib::RawNetworkId::new(value);
@@ -957,3 +957,57 @@ fn v0_3_13_pre_012_cross_layer_security_closure_is_complete_and_source_neutral()
}
return;
}
#[test]
fn v0_3_14_pre_002_continuity_contracts_are_private_bounded_and_io_free() {
let continuity = include_str!("../src/continuity.rs");
let resources = include_str!("../src/runtime_resources.rs");
let root = include_str!("../src/lib.rs");
for required in [
"MAX_RAW_TRANSACTION_INGEST_OPEN_REPAIR_GAPS",
"MAX_RAW_TRANSACTION_INGEST_REPAIR_ACTIVE_GAPS",
"MAX_RAW_TRANSACTION_INGEST_REPAIR_BLOCK_FETCH_IN_FLIGHT",
"MAX_RAW_TRANSACTION_INGEST_REPAIR_DISCOVERY_WINDOW_SLOTS",
"MAX_RAW_TRANSACTION_INGEST_REPAIR_RANGE_SLOTS",
"FullLedgerTransactions",
"ExactSourceScope",
"KnownReferences",
"RawTransactionIngestGapLedger",
"RawTransactionIngestTargetCoverage",
"RawTransactionIngestRepairCapabilityDescriptor",
"RawTransactionIngestContinuityContracts",
"continuity.known_references_not_target_scope",
"continuity.coalescible_gap_ranges",
] {
assert!(continuity.contains(required), "required pre.002 continuity contract missing: {required}");
}
for required in [
"repair_capability_descriptor",
"yellowstone_coverage_scope_fingerprint",
"http_role_supports_repair_scan",
"RawTransactionIngestContinuityContracts::new",
"validate_for_source_count",
] {
assert!(resources.contains(required), "required pre.002 runtime-resource capability wiring missing: {required}");
}
for forbidden in [
"get_block_observed",
"get_transaction_observed",
"get_blocks_with_limit",
"get_blocks(",
"open_standard_subscribe",
"SolanaStandardWsSession::connect",
"HeliusLaserStreamWsSession::connect",
"ksp_config_lib::",
"ksp_job_backfill_lib::",
"ksp_store_postgres_lib::",
"reqwest::",
"tonic::",
"yellowstone_grpc_proto::",
] {
assert!(!continuity.contains(forbidden), "pre.002 continuity contract performed or imported forbidden I/O/boundary: {forbidden}");
}
assert!(!root.contains("pub use self::continuity::"));
assert!(root.contains("pub(crate) use self::continuity::RawTransactionIngestContinuityContracts;"));
return;
}

View File

@@ -1,7 +1,7 @@
// file: crates/ksp-worker-raw-transaction-ingest-lib/tests/release_completeness.rs
// version: 20
// version: 21
//! Release-completeness canaries through the `pre.012` cross-layer completeness/security tranche.
//! Release-completeness canaries through the `v0.3.14-pre.002` continuity-contract tranche.
#[test]
fn pre_010_production_module_inventory_is_exact() -> std::io::Result<()> {
@@ -34,7 +34,18 @@ fn pre_010_production_module_inventory_is_exact() -> std::io::Result<()> {
names.sort_unstable();
assert_eq!(
names,
std::vec!["admission.rs", "error.rs", "identity.rs", "lib.rs", "persistence.rs", "runtime.rs", "runtime_resources.rs", "settings.rs", "snapshot.rs",]
std::vec![
"admission.rs",
"continuity.rs",
"error.rs",
"identity.rs",
"lib.rs",
"persistence.rs",
"runtime.rs",
"runtime_resources.rs",
"settings.rs",
"snapshot.rs",
]
);
return std::result::Result::Ok(());
}
@@ -132,6 +143,7 @@ fn pre_010_external_hardening_suite_is_present_and_scoped() {
"v0_3_13_pre_009_duplicate_storm_disagreement_and_starvation_guards_are_explicit",
"v0_3_13_pre_010_multi_source_health_is_conservative_counted_and_redacted",
"v0_3_13_pre_011_shutdown_races_are_bounded_joined_atomic_and_counter_safe",
"v0_3_14_pre_002_continuity_contracts_are_private_bounded_and_io_free",
] {
assert!(hardening.contains(required), "required pre.010 hardening canary missing: {required}");
}

View File

@@ -0,0 +1,241 @@
// file: crates/ksp-worker-raw-transaction-ingest-lib/unit_tests/continuity.rs
// version: 1
fn network() -> std::option::Option<ksp_store_lib::RawNetworkId> {
return match ksp_store_lib::RawNetworkId::new("mainnet") {
std::result::Result::Ok(value) => std::option::Option::Some(value),
std::result::Result::Err(_) => std::option::Option::None,
};
}
fn exact_scope(family: &'static str, fingerprint_byte: u8) -> crate::RawTransactionIngestCoverageScope {
return crate::RawTransactionIngestCoverageScope::exact_source_scope(family, [fingerprint_byte; 32]);
}
fn capability(
source_key_byte: u8,
commitment: ksp_onchain_transport_lib::SolanaCommitment,
scope: crate::RawTransactionIngestCoverageScope,
) -> std::option::Option<crate::RawTransactionIngestRepairCapabilityDescriptor> {
let network = match network() {
std::option::Option::Some(value) => value,
std::option::Option::None => return std::option::Option::None,
};
return match crate::RawTransactionIngestRepairCapabilityDescriptor::new(
[source_key_byte; 32],
network,
commitment,
scope,
true,
false,
false,
true,
false,
false,
) {
std::result::Result::Ok(value) => std::option::Option::Some(value),
std::result::Result::Err(_) => std::option::Option::None,
};
}
fn gap(
gap_id: u64,
source_key_byte: u8,
start_slot: u64,
end_slot: u64,
state: super::RawTransactionIngestGapState,
) -> std::option::Option<super::RawTransactionIngestGap> {
let network = match network() {
std::option::Option::Some(value) => value,
std::option::Option::None => return std::option::Option::None,
};
let range = match super::RawTransactionIngestGapRange::new(start_slot, end_slot) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return std::option::Option::None,
};
return std::option::Option::Some(super::RawTransactionIngestGap {
commitment: ksp_onchain_transport_lib::SolanaCommitment::Confirmed,
coverage_requirement: exact_scope("standard_logs", 9),
gap_id: super::RawTransactionIngestGapId(gap_id),
network,
range,
reason: super::RawTransactionIngestGapReason::WebSocketReconnect,
source_key: [source_key_byte; 32],
state,
});
}
#[test]
fn pre_002_repair_bounds_are_exact_and_run_local() {
assert_eq!(crate::MAX_RAW_TRANSACTION_INGEST_OPEN_REPAIR_GAPS, 64);
assert_eq!(crate::MAX_RAW_TRANSACTION_INGEST_REPAIR_ACTIVE_GAPS, 1);
assert_eq!(crate::MAX_RAW_TRANSACTION_INGEST_REPAIR_BLOCK_FETCH_IN_FLIGHT, 4);
assert_eq!(crate::MAX_RAW_TRANSACTION_INGEST_REPAIR_DISCOVERY_WINDOW_SLOTS, 512);
assert_eq!(crate::MAX_RAW_TRANSACTION_INGEST_REPAIR_RANGE_SLOTS, 4_096);
return;
}
#[test]
fn pre_002_gap_range_is_inclusive_bounded_and_overflow_safe() {
assert!(super::RawTransactionIngestGapRange::new(10, 9).is_err());
assert!(super::RawTransactionIngestGapRange::new(10, 10 + crate::MAX_RAW_TRANSACTION_INGEST_REPAIR_RANGE_SLOTS - 1).is_ok());
assert!(super::RawTransactionIngestGapRange::new(10, 10 + crate::MAX_RAW_TRANSACTION_INGEST_REPAIR_RANGE_SLOTS).is_err());
assert!(super::RawTransactionIngestGapRange::new(u64::MAX, u64::MAX).is_ok());
return;
}
#[test]
fn pre_002_gap_range_overlap_and_adjacency_are_explicit() {
let first = match super::RawTransactionIngestGapRange::new(100, 109) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => panic!("unexpected first range failure: {error}"),
};
let overlap = match super::RawTransactionIngestGapRange::new(105, 115) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => panic!("unexpected overlap range failure: {error}"),
};
let adjacent = match super::RawTransactionIngestGapRange::new(110, 120) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => panic!("unexpected adjacent range failure: {error}"),
};
let separate = match super::RawTransactionIngestGapRange::new(111, 120) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => panic!("unexpected separate range failure: {error}"),
};
assert!(first.overlaps_or_is_adjacent(overlap));
assert!(first.overlaps_or_is_adjacent(adjacent));
assert!(!first.overlaps_or_is_adjacent(separate));
let merged = first.try_merge(adjacent);
assert!(matches!(merged, std::result::Result::Ok(std::option::Option::Some(value)) if value.start_slot() == 100 && value.end_slot() == 120));
return;
}
#[test]
fn pre_002_target_coverage_deduplicates_exact_scopes_without_cross_commitment_broadening() {
let first = match capability(1, ksp_onchain_transport_lib::SolanaCommitment::Confirmed, exact_scope("standard_logs", 7)) {
std::option::Option::Some(value) => value,
std::option::Option::None => panic!("first capability fixture unavailable"),
};
let second = match capability(2, ksp_onchain_transport_lib::SolanaCommitment::Confirmed, exact_scope("standard_logs", 7)) {
std::option::Option::Some(value) => value,
std::option::Option::None => panic!("second capability fixture unavailable"),
};
let finalized_full =
match capability(3, ksp_onchain_transport_lib::SolanaCommitment::Finalized, crate::RawTransactionIngestCoverageScope::full_ledger_transactions()) {
std::option::Option::Some(value) => value,
std::option::Option::None => panic!("full capability fixture unavailable"),
};
let contracts = match crate::RawTransactionIngestContinuityContracts::new(std::vec![first, second, finalized_full]) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => panic!("continuity contracts fixture failed: {error}"),
};
assert_eq!(contracts.capabilities.len(), 3);
assert_eq!(contracts.target_coverage.requirements.len(), 2);
assert!(contracts.target_coverage.requirements.iter().any(|requirement| {
return requirement.commitment == ksp_onchain_transport_lib::SolanaCommitment::Confirmed && requirement.scope == exact_scope("standard_logs", 7);
}));
assert!(contracts.target_coverage.requirements.iter().any(|requirement| {
return requirement.commitment == ksp_onchain_transport_lib::SolanaCommitment::Finalized
&& requirement.scope == crate::RawTransactionIngestCoverageScope::full_ledger_transactions();
}));
return;
}
#[test]
fn pre_002_full_ledger_scope_subsumes_only_same_commitment_targets() {
let exact = match capability(1, ksp_onchain_transport_lib::SolanaCommitment::Confirmed, exact_scope("helius_transaction", 4)) {
std::option::Option::Some(value) => value,
std::option::Option::None => panic!("exact capability fixture unavailable"),
};
let full = match capability(2, ksp_onchain_transport_lib::SolanaCommitment::Confirmed, crate::RawTransactionIngestCoverageScope::full_ledger_transactions())
{
std::option::Option::Some(value) => value,
std::option::Option::None => panic!("full capability fixture unavailable"),
};
let contracts = match crate::RawTransactionIngestContinuityContracts::new(std::vec![exact, full]) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => panic!("continuity contracts fixture failed: {error}"),
};
assert_eq!(contracts.target_coverage.requirements.len(), 1);
assert_eq!(contracts.target_coverage.requirements[0].scope, crate::RawTransactionIngestCoverageScope::full_ledger_transactions());
assert_eq!(contracts.target_coverage.requirements[0].commitment, ksp_onchain_transport_lib::SolanaCommitment::Confirmed);
return;
}
#[test]
fn pre_002_known_references_can_never_be_configured_target_coverage() {
let network = match network() {
std::option::Option::Some(value) => value,
std::option::Option::None => panic!("network fixture unavailable"),
};
let result = crate::RawTransactionIngestRepairCapabilityDescriptor::new(
[1_u8; 32],
network,
ksp_onchain_transport_lib::SolanaCommitment::Confirmed,
crate::RawTransactionIngestCoverageScope::KnownReferences([2_u8; 32]),
true,
false,
false,
true,
false,
false,
);
assert!(result.is_err());
return;
}
#[test]
fn pre_002_gap_ledger_rejects_coalescible_open_ranges_and_active_overflow() {
let network = match network() {
std::option::Option::Some(value) => value,
std::option::Option::None => panic!("network fixture unavailable"),
};
let first = match gap(1, 1, 100, 109, super::RawTransactionIngestGapState::Pending) {
std::option::Option::Some(value) => value,
std::option::Option::None => panic!("first gap fixture unavailable"),
};
let adjacent = match gap(2, 1, 110, 120, super::RawTransactionIngestGapState::Pending) {
std::option::Option::Some(value) => value,
std::option::Option::None => panic!("adjacent gap fixture unavailable"),
};
let adjacent_ledger = super::RawTransactionIngestGapLedger { gaps: std::vec![first, adjacent], network: network.clone(), next_gap_id: 3 };
assert!(adjacent_ledger.validate_invariants().is_err());
let first_active = match gap(1, 1, 100, 109, super::RawTransactionIngestGapState::Repairing) {
std::option::Option::Some(value) => value,
std::option::Option::None => panic!("first active gap fixture unavailable"),
};
let second_active = match gap(2, 2, 200, 209, super::RawTransactionIngestGapState::Repairing) {
std::option::Option::Some(value) => value,
std::option::Option::None => panic!("second active gap fixture unavailable"),
};
let active_ledger = super::RawTransactionIngestGapLedger { gaps: std::vec![first_active, second_active], network, next_gap_id: 3 };
assert!(active_ledger.validate_invariants().is_err());
return;
}
#[test]
fn pre_002_gap_ledger_enforces_open_gap_bound_and_next_id_monotonicity() {
let network = match network() {
std::option::Option::Some(value) => value,
std::option::Option::None => panic!("network fixture unavailable"),
};
let mut gaps = std::vec::Vec::new();
for index in 0..=crate::MAX_RAW_TRANSACTION_INGEST_OPEN_REPAIR_GAPS {
let gap_id = (index as u64) + 1;
let slot = (index as u64) * 2;
let gap = match gap(gap_id, 1, slot, slot, super::RawTransactionIngestGapState::Pending) {
std::option::Option::Some(value) => value,
std::option::Option::None => panic!("gap fixture unavailable"),
};
gaps.push(gap);
}
let ledger = super::RawTransactionIngestGapLedger { gaps, network: network.clone(), next_gap_id: 66 };
assert!(ledger.validate_invariants().is_err());
let single = match gap(7, 1, 100, 100, super::RawTransactionIngestGapState::Unresolved) {
std::option::Option::Some(value) => value,
std::option::Option::None => panic!("single gap fixture unavailable"),
};
let stale_next = super::RawTransactionIngestGapLedger { gaps: std::vec![single], network, next_gap_id: 7 };
assert!(stale_next.validate_invariants().is_err());
return;
}