1239 lines
60 KiB
Rust
1239 lines
60 KiB
Rust
// file: crates/ksp-worker-raw-transaction-ingest-lib/src/continuity.rs
|
|
// version: 11
|
|
|
|
/// Maximum number of slots admitted by one private continuity HTTP discovery window outside this module.
|
|
pub(crate) const MAX_RAW_TRANSACTION_INGEST_CONTINUITY_DISCOVERY_WINDOW_SLOTS: u64 = MAX_RAW_TRANSACTION_INGEST_REPAIR_DISCOVERY_WINDOW_SLOTS;
|
|
|
|
/// Maximum number of proven redundant coverage epochs retained by one run-local continuity contract.
|
|
const MAX_RAW_TRANSACTION_INGEST_COVERAGE_EPOCHS: usize = 256;
|
|
/// Maximum number of simultaneously retained non-repaired run-local gaps.
|
|
const MAX_RAW_TRANSACTION_INGEST_OPEN_REPAIR_GAPS: usize = 64;
|
|
/// Maximum number of run-local gaps that may actively repair at once.
|
|
const MAX_RAW_TRANSACTION_INGEST_REPAIR_ACTIVE_GAPS: usize = 1;
|
|
/// Maximum number of logical block fetches that a later repair scheduler may admit concurrently.
|
|
const MAX_RAW_TRANSACTION_INGEST_REPAIR_BLOCK_FETCH_IN_FLIGHT: usize = 4;
|
|
/// Maximum number of slots admitted by one later HTTP repair discovery window.
|
|
const MAX_RAW_TRANSACTION_INGEST_REPAIR_DISCOVERY_WINDOW_SLOTS: u64 = 512;
|
|
/// Maximum inclusive slot span admitted for one run-local repair gap.
|
|
const MAX_RAW_TRANSACTION_INGEST_REPAIR_RANGE_SLOTS: u64 = 4_096;
|
|
|
|
/// Private source scope used to prove whether one configured live source can cover another run-local requirement.
|
|
#[derive(Clone, Debug, Eq, PartialEq)]
|
|
pub(crate) enum RawTransactionIngestCoverageScope {
|
|
/// Every transaction contained by every produced block at the configured commitment.
|
|
FullLedgerTransactions,
|
|
/// One opaque exact source-family scope identified independently from provider and endpoint identity.
|
|
ExactSourceScope(&'static str, [u8; 32]),
|
|
/// A bounded set of references already observed by the Worker; never sufficient as configured target coverage.
|
|
KnownReferences([u8; 32]),
|
|
}
|
|
|
|
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
|
enum RawTransactionIngestCoverageRelation {
|
|
Exact,
|
|
Superset,
|
|
}
|
|
|
|
impl crate::RawTransactionIngestCoverageScope {
|
|
/// Creates one provider-neutral exact source scope from a stable family code and semantic fingerprint.
|
|
pub(crate) const fn exact_source_scope(family_code: &'static str, fingerprint: [u8; 32]) -> Self {
|
|
return Self::ExactSourceScope(family_code, fingerprint);
|
|
}
|
|
|
|
/// Creates the full-ledger transaction scope used by complete block sources.
|
|
pub(crate) const fn full_ledger_transactions() -> Self {
|
|
return Self::FullLedgerTransactions;
|
|
}
|
|
|
|
fn validate_configured_target_scope(&self) -> ksp_core_lib::Result<()> {
|
|
return match self {
|
|
Self::FullLedgerTransactions => std::result::Result::Ok(()),
|
|
Self::ExactSourceScope(family_code, fingerprint) => {
|
|
if family_code.is_empty() {
|
|
return std::result::Result::Err(crate::runtime_error("continuity.exact_scope_family_empty"));
|
|
}
|
|
let _fingerprint_first_byte = fingerprint[0];
|
|
std::result::Result::Ok(())
|
|
},
|
|
Self::KnownReferences(fingerprint) => {
|
|
let _fingerprint_first_byte = fingerprint[0];
|
|
std::result::Result::Err(crate::runtime_error("continuity.known_references_not_target_scope"))
|
|
},
|
|
};
|
|
}
|
|
|
|
fn is_known_references(&self) -> bool {
|
|
return matches!(self, Self::KnownReferences(_));
|
|
}
|
|
|
|
fn relation_to(&self, requirement: &Self) -> std::option::Option<RawTransactionIngestCoverageRelation> {
|
|
return match (self, requirement) {
|
|
(Self::FullLedgerTransactions, Self::FullLedgerTransactions) => std::option::Option::Some(RawTransactionIngestCoverageRelation::Exact),
|
|
(Self::FullLedgerTransactions, Self::ExactSourceScope(_, _)) => std::option::Option::Some(RawTransactionIngestCoverageRelation::Superset),
|
|
(Self::ExactSourceScope(family_code, fingerprint), Self::ExactSourceScope(required_family_code, required_fingerprint))
|
|
if family_code == required_family_code && fingerprint == required_fingerprint =>
|
|
{
|
|
std::option::Option::Some(RawTransactionIngestCoverageRelation::Exact)
|
|
},
|
|
_ => std::option::Option::None,
|
|
};
|
|
}
|
|
}
|
|
|
|
#[derive(Clone, Debug, Eq, PartialEq)]
|
|
struct RawTransactionIngestCoverageRequirement {
|
|
commitment: ksp_onchain_transport_lib::SolanaCommitment,
|
|
scope: crate::RawTransactionIngestCoverageScope,
|
|
}
|
|
|
|
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
|
enum RawTransactionIngestGapReason {
|
|
HttpProducedBlockUnavailable,
|
|
KnownReferenceMissing,
|
|
SourceFailure,
|
|
TransportOverflow,
|
|
WebSocketReconnect,
|
|
YellowstoneRetention,
|
|
}
|
|
|
|
impl RawTransactionIngestGapReason {
|
|
const ALL: [Self; 6] = [
|
|
Self::HttpProducedBlockUnavailable,
|
|
Self::KnownReferenceMissing,
|
|
Self::SourceFailure,
|
|
Self::TransportOverflow,
|
|
Self::WebSocketReconnect,
|
|
Self::YellowstoneRetention,
|
|
];
|
|
|
|
const fn public(self) -> crate::RawTransactionIngestGapReason {
|
|
return match self {
|
|
Self::HttpProducedBlockUnavailable => crate::RawTransactionIngestGapReason::HttpProducedBlockUnavailable,
|
|
Self::KnownReferenceMissing => crate::RawTransactionIngestGapReason::KnownReferenceMissing,
|
|
Self::SourceFailure => crate::RawTransactionIngestGapReason::SourceFailure,
|
|
Self::TransportOverflow => crate::RawTransactionIngestGapReason::TransportOverflow,
|
|
Self::WebSocketReconnect => crate::RawTransactionIngestGapReason::WebSocketReconnect,
|
|
Self::YellowstoneRetention => crate::RawTransactionIngestGapReason::YellowstoneRetention,
|
|
};
|
|
}
|
|
}
|
|
|
|
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
|
enum RawTransactionIngestGapState {
|
|
Pending,
|
|
Repairing,
|
|
Repaired,
|
|
Unresolved,
|
|
}
|
|
|
|
impl RawTransactionIngestGapState {
|
|
const ALL: [Self; 4] = [Self::Pending, Self::Repairing, Self::Repaired, Self::Unresolved];
|
|
|
|
const fn is_open(self) -> bool {
|
|
return !matches!(self, Self::Repaired);
|
|
}
|
|
|
|
const fn public(self) -> crate::RawTransactionIngestGapState {
|
|
return match self {
|
|
Self::Pending => crate::RawTransactionIngestGapState::Pending,
|
|
Self::Repairing => crate::RawTransactionIngestGapState::Repairing,
|
|
Self::Repaired => crate::RawTransactionIngestGapState::Repaired,
|
|
Self::Unresolved => crate::RawTransactionIngestGapState::Unresolved,
|
|
};
|
|
}
|
|
}
|
|
|
|
/// Private supervisor decision for one terminal live-source loss after conservative continuity reconciliation.
|
|
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
|
pub(crate) enum RawTransactionIngestSourceLossDecision {
|
|
/// Remaining active sources explicitly preserve all configured target coverage and no known gap blocks continuity.
|
|
Continue,
|
|
/// Coverage is incomplete, a known gap remains open, or the source-loss proof is otherwise insufficient.
|
|
Fault,
|
|
}
|
|
|
|
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
|
struct RawTransactionIngestGapRange {
|
|
end_slot: u64,
|
|
start_slot: u64,
|
|
}
|
|
|
|
impl RawTransactionIngestGapRange {
|
|
fn new(start_slot: u64, end_slot: u64) -> ksp_core_lib::Result<Self> {
|
|
if end_slot < start_slot {
|
|
return std::result::Result::Err(crate::runtime_error("continuity.gap_range_reversed"));
|
|
}
|
|
let slot_count = match end_slot.checked_sub(start_slot).and_then(|value| return value.checked_add(1)) {
|
|
std::option::Option::Some(value) => value,
|
|
std::option::Option::None => return std::result::Result::Err(crate::runtime_error("continuity.gap_range_overflow")),
|
|
};
|
|
if slot_count > MAX_RAW_TRANSACTION_INGEST_REPAIR_RANGE_SLOTS {
|
|
return std::result::Result::Err(crate::runtime_error("continuity.gap_range_too_large"));
|
|
}
|
|
return std::result::Result::Ok(Self { end_slot, start_slot });
|
|
}
|
|
|
|
const fn end_slot(self) -> u64 {
|
|
return self.end_slot;
|
|
}
|
|
|
|
fn overlaps_or_is_adjacent(self, other: Self) -> bool {
|
|
if self.start_slot <= other.end_slot && other.start_slot <= self.end_slot {
|
|
return true;
|
|
}
|
|
let self_next = self.end_slot.checked_add(1);
|
|
if self_next == std::option::Option::Some(other.start_slot) {
|
|
return true;
|
|
}
|
|
let other_next = other.end_slot.checked_add(1);
|
|
return other_next == std::option::Option::Some(self.start_slot);
|
|
}
|
|
|
|
const fn start_slot(self) -> u64 {
|
|
return self.start_slot;
|
|
}
|
|
|
|
fn try_merge(self, other: Self) -> ksp_core_lib::Result<std::option::Option<Self>> {
|
|
if !self.overlaps_or_is_adjacent(other) {
|
|
return std::result::Result::Ok(std::option::Option::None);
|
|
}
|
|
let start_slot = self.start_slot.min(other.start_slot);
|
|
let end_slot = self.end_slot.max(other.end_slot);
|
|
let slot_count = match end_slot.checked_sub(start_slot).and_then(|value| return value.checked_add(1)) {
|
|
std::option::Option::Some(value) => value,
|
|
std::option::Option::None => return std::result::Result::Ok(std::option::Option::None),
|
|
};
|
|
if slot_count > MAX_RAW_TRANSACTION_INGEST_REPAIR_RANGE_SLOTS {
|
|
return std::result::Result::Ok(std::option::Option::None);
|
|
}
|
|
return std::result::Result::Ok(std::option::Option::Some(Self { end_slot, start_slot }));
|
|
}
|
|
}
|
|
|
|
/// Private run-local obligation for one transaction reference already observed by a live source.
|
|
#[derive(Clone, Debug, Eq, PartialEq)]
|
|
pub(crate) struct RawTransactionIngestKnownReferenceObligation {
|
|
commitment: ksp_onchain_transport_lib::SolanaCommitment,
|
|
reference: ksp_store_lib::RawTransactionReference,
|
|
slot: u64,
|
|
}
|
|
|
|
impl crate::RawTransactionIngestKnownReferenceObligation {
|
|
/// Creates one known-reference obligation without inventing any absence or coverage proof.
|
|
pub(crate) fn new(
|
|
reference: ksp_store_lib::RawTransactionReference,
|
|
slot: u64,
|
|
commitment: ksp_onchain_transport_lib::SolanaCommitment,
|
|
) -> ksp_core_lib::Result<Self> {
|
|
if commitment == ksp_onchain_transport_lib::SolanaCommitment::Processed {
|
|
return std::result::Result::Err(crate::runtime_error("continuity.known_reference_processed_unsupported"));
|
|
}
|
|
return std::result::Result::Ok(Self { commitment, reference, slot });
|
|
}
|
|
|
|
/// Returns the exact commitment under which this reference must be resolved.
|
|
pub(crate) const fn commitment(&self) -> ksp_onchain_transport_lib::SolanaCommitment {
|
|
return self.commitment;
|
|
}
|
|
|
|
/// Returns the already-known canonical transaction reference.
|
|
pub(crate) const fn reference(&self) -> &ksp_store_lib::RawTransactionReference {
|
|
return &self.reference;
|
|
}
|
|
|
|
/// Returns the already-observed slot associated with the reference.
|
|
pub(crate) const fn slot(&self) -> u64 {
|
|
return self.slot;
|
|
}
|
|
}
|
|
|
|
/// Private run-local anchor for one WebSocket continuity incident.
|
|
///
|
|
/// The anchor never derives slots from wall-clock time. Its inclusive start is the latest slot actually observed by that source before Transport reported a
|
|
/// reconnect or notification overflow; the end remains absent until the first post-incident source slot is observed.
|
|
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
|
pub(crate) struct RawTransactionIngestWebSocketIncidentAnchor {
|
|
end_slot: std::option::Option<u64>,
|
|
overflow_total: u64,
|
|
reconnect_total: u64,
|
|
saw_overflow: bool,
|
|
saw_reconnect: bool,
|
|
start_slot: u64,
|
|
}
|
|
|
|
impl crate::RawTransactionIngestWebSocketIncidentAnchor {
|
|
/// Creates one open incident anchor from monotone Transport counters and one actually observed source slot.
|
|
pub(crate) fn new(start_slot: u64, reconnect_total: u64, overflow_total: u64, saw_reconnect: bool, saw_overflow: bool) -> ksp_core_lib::Result<Self> {
|
|
if !saw_reconnect && !saw_overflow {
|
|
return std::result::Result::Err(crate::runtime_error("continuity.websocket_incident_reason_missing"));
|
|
}
|
|
return std::result::Result::Ok(Self {
|
|
end_slot: std::option::Option::None,
|
|
overflow_total,
|
|
reconnect_total,
|
|
saw_overflow,
|
|
saw_reconnect,
|
|
start_slot,
|
|
});
|
|
}
|
|
|
|
/// Extends one still-open incident with newer monotone Transport counters without changing its earliest start slot.
|
|
pub(crate) fn extend(&mut self, reconnect_total: u64, overflow_total: u64, saw_reconnect: bool, saw_overflow: bool) -> ksp_core_lib::Result<()> {
|
|
if self.end_slot.is_some() {
|
|
return std::result::Result::Err(crate::runtime_error("continuity.websocket_incident_already_closed"));
|
|
}
|
|
if reconnect_total < self.reconnect_total || overflow_total < self.overflow_total {
|
|
return std::result::Result::Err(crate::runtime_error("continuity.websocket_incident_counter_regression"));
|
|
}
|
|
self.reconnect_total = reconnect_total;
|
|
self.overflow_total = overflow_total;
|
|
self.saw_reconnect = self.saw_reconnect || saw_reconnect;
|
|
self.saw_overflow = self.saw_overflow || saw_overflow;
|
|
return std::result::Result::Ok(());
|
|
}
|
|
|
|
/// Closes the inclusive incident range at the first post-incident source slot.
|
|
pub(crate) fn close_at(&mut self, end_slot: u64) -> ksp_core_lib::Result<()> {
|
|
let range = match RawTransactionIngestGapRange::new(self.start_slot, end_slot) {
|
|
std::result::Result::Ok(value) => value,
|
|
std::result::Result::Err(error) => return std::result::Result::Err(error),
|
|
};
|
|
self.end_slot = std::option::Option::Some(range.end_slot());
|
|
return std::result::Result::Ok(());
|
|
}
|
|
|
|
/// Returns the first source slot that safely anchors the incident, inclusively.
|
|
pub(crate) const fn start_slot(self) -> u64 {
|
|
return self.start_slot;
|
|
}
|
|
|
|
/// Returns the first post-incident source slot when the incident has become range-bounded.
|
|
pub(crate) const fn end_slot(self) -> std::option::Option<u64> {
|
|
return self.end_slot;
|
|
}
|
|
|
|
/// Returns whether at least one physical WebSocket reconnect contributed to this incident.
|
|
#[cfg(test)]
|
|
pub(crate) const fn saw_reconnect(self) -> bool {
|
|
return self.saw_reconnect;
|
|
}
|
|
|
|
/// Returns whether at least one Transport notification overflow contributed to this incident.
|
|
#[cfg(test)]
|
|
pub(crate) const fn saw_overflow(self) -> bool {
|
|
return self.saw_overflow;
|
|
}
|
|
}
|
|
|
|
#[derive(Clone, Copy, Debug, Eq, PartialEq, Ord, PartialOrd)]
|
|
struct RawTransactionIngestGapId(u64);
|
|
|
|
struct RawTransactionIngestGap {
|
|
commitment: ksp_onchain_transport_lib::SolanaCommitment,
|
|
coverage_requirement: crate::RawTransactionIngestCoverageScope,
|
|
gap_id: RawTransactionIngestGapId,
|
|
network: ksp_store_lib::RawNetworkId,
|
|
range: RawTransactionIngestGapRange,
|
|
reason: RawTransactionIngestGapReason,
|
|
source_key: [u8; 32],
|
|
state: RawTransactionIngestGapState,
|
|
last_method: std::option::Option<crate::RawTransactionIngestRepairMethod>,
|
|
}
|
|
|
|
impl RawTransactionIngestGap {
|
|
fn snapshot(&self) -> crate::RawTransactionIngestGapSnapshot {
|
|
return crate::RawTransactionIngestGapSnapshot::new(
|
|
crate::RawTransactionIngestGapId::new(self.gap_id.0),
|
|
self.range.start_slot(),
|
|
self.range.end_slot(),
|
|
self.state.public(),
|
|
self.reason.public(),
|
|
self.last_method,
|
|
);
|
|
}
|
|
}
|
|
|
|
struct RawTransactionIngestGapLedger {
|
|
gaps: std::vec::Vec<RawTransactionIngestGap>,
|
|
network: ksp_store_lib::RawNetworkId,
|
|
next_gap_id: u64,
|
|
}
|
|
|
|
impl RawTransactionIngestGapLedger {
|
|
fn new(network: ksp_store_lib::RawNetworkId) -> Self {
|
|
return Self { gaps: std::vec::Vec::new(), network, next_gap_id: 1 };
|
|
}
|
|
|
|
fn continuity_frontier(&self, processing_frontier_slot: std::option::Option<u64>) -> std::option::Option<u64> {
|
|
let processing_frontier_slot = match processing_frontier_slot {
|
|
std::option::Option::Some(value) => value,
|
|
std::option::Option::None => return std::option::Option::None,
|
|
};
|
|
let earliest_open_gap_start = self.gaps.iter().filter(|gap| return gap.state.is_open()).map(|gap| return gap.range.start_slot()).min();
|
|
let earliest_open_gap_start = match earliest_open_gap_start {
|
|
std::option::Option::Some(value) if value <= processing_frontier_slot => value,
|
|
std::option::Option::Some(_) | std::option::Option::None => return std::option::Option::Some(processing_frontier_slot),
|
|
};
|
|
return earliest_open_gap_start.checked_sub(1);
|
|
}
|
|
|
|
fn has_open_gaps(&self) -> bool {
|
|
return self.gaps.iter().any(|gap| return gap.state.is_open());
|
|
}
|
|
|
|
fn source_failures_reconciled(&self, source_keys: &std::collections::BTreeSet<[u8; 32]>) -> bool {
|
|
for source_key in source_keys {
|
|
let mut found = false;
|
|
for gap in &self.gaps {
|
|
if gap.source_key != *source_key || gap.reason != RawTransactionIngestGapReason::SourceFailure {
|
|
continue;
|
|
}
|
|
found = true;
|
|
if gap.state.is_open() {
|
|
return false;
|
|
}
|
|
}
|
|
if !found {
|
|
return false;
|
|
}
|
|
}
|
|
return true;
|
|
}
|
|
|
|
fn record_gap(
|
|
&mut self,
|
|
source_key: [u8; 32],
|
|
commitment: ksp_onchain_transport_lib::SolanaCommitment,
|
|
coverage_requirement: crate::RawTransactionIngestCoverageScope,
|
|
reason: RawTransactionIngestGapReason,
|
|
range: RawTransactionIngestGapRange,
|
|
) -> ksp_core_lib::Result<()> {
|
|
for gap in &mut self.gaps {
|
|
if !gap.state.is_open()
|
|
|| gap.source_key != source_key
|
|
|| gap.commitment != commitment
|
|
|| gap.coverage_requirement != coverage_requirement
|
|
|| gap.reason != reason
|
|
{
|
|
continue;
|
|
}
|
|
let merged = match gap.range.try_merge(range) {
|
|
std::result::Result::Ok(value) => value,
|
|
std::result::Result::Err(error) => return std::result::Result::Err(error),
|
|
};
|
|
if let std::option::Option::Some(merged) = merged {
|
|
gap.range = merged;
|
|
return self.validate_invariants();
|
|
}
|
|
}
|
|
let open_gap_count = self.gaps.iter().filter(|gap| return gap.state.is_open()).count();
|
|
if open_gap_count >= MAX_RAW_TRANSACTION_INGEST_OPEN_REPAIR_GAPS {
|
|
return std::result::Result::Err(crate::runtime_error("continuity.open_gap_limit_exceeded"));
|
|
}
|
|
let gap_id = self.next_gap_id;
|
|
self.next_gap_id = match self.next_gap_id.checked_add(1) {
|
|
std::option::Option::Some(value) => value,
|
|
std::option::Option::None => return std::result::Result::Err(crate::runtime_error("continuity.gap_id_exhausted")),
|
|
};
|
|
self.gaps.push(RawTransactionIngestGap {
|
|
commitment,
|
|
coverage_requirement,
|
|
gap_id: RawTransactionIngestGapId(gap_id),
|
|
network: self.network.clone(),
|
|
range,
|
|
reason,
|
|
source_key,
|
|
state: RawTransactionIngestGapState::Pending,
|
|
last_method: std::option::Option::None,
|
|
});
|
|
return self.validate_invariants();
|
|
}
|
|
|
|
fn reconcile_with_coverage_epochs(&mut self, coverage_epochs: &RawTransactionIngestCoverageEpochLedger) -> ksp_core_lib::Result<()> {
|
|
for gap in &mut self.gaps {
|
|
if !gap.state.is_open() || gap.state == RawTransactionIngestGapState::Unresolved {
|
|
continue;
|
|
}
|
|
let requirement = RawTransactionIngestCoverageRequirement { commitment: gap.commitment, scope: gap.coverage_requirement.clone() };
|
|
if coverage_epochs.redundant_relation_for_gap(gap.source_key, &requirement, gap.range).is_some() {
|
|
gap.state = RawTransactionIngestGapState::Repaired;
|
|
gap.last_method = std::option::Option::Some(crate::RawTransactionIngestRepairMethod::RedundantCoverage);
|
|
}
|
|
}
|
|
return self.validate_invariants();
|
|
}
|
|
|
|
fn observability_projection(&self) -> ksp_core_lib::Result<crate::RawTransactionIngestContinuitySnapshotProjection> {
|
|
let mut open_gap_count = 0_usize;
|
|
let mut repairing_gap_count = 0_usize;
|
|
let mut repaired_gap_total = 0_u64;
|
|
let mut unresolved_gap_total = 0_u64;
|
|
let mut replay_repair_total = 0_u64;
|
|
let mut redundant_coverage_repair_total = 0_u64;
|
|
let mut http_scan_repair_total = 0_u64;
|
|
let mut repair_block_fetch_total = 0_u64;
|
|
let mut repair_transaction_hydration_total = 0_u64;
|
|
let mut oldest_open_gap_start_slot: std::option::Option<u64> = std::option::Option::None;
|
|
let mut gaps = std::vec::Vec::with_capacity(MAX_RAW_TRANSACTION_INGEST_OPEN_REPAIR_GAPS);
|
|
for gap in &self.gaps {
|
|
if gap.state.is_open() {
|
|
open_gap_count = match open_gap_count.checked_add(1) {
|
|
std::option::Option::Some(value) => value,
|
|
std::option::Option::None => return std::result::Result::Err(crate::counter_exhausted_error("continuity.open_gap_count")),
|
|
};
|
|
if gap.state == RawTransactionIngestGapState::Repairing {
|
|
repairing_gap_count = match repairing_gap_count.checked_add(1) {
|
|
std::option::Option::Some(value) => value,
|
|
std::option::Option::None => {
|
|
return std::result::Result::Err(crate::counter_exhausted_error("continuity.repairing_gap_count"));
|
|
},
|
|
};
|
|
}
|
|
if gap.state == RawTransactionIngestGapState::Unresolved {
|
|
unresolved_gap_total = match unresolved_gap_total.checked_add(1) {
|
|
std::option::Option::Some(value) => value,
|
|
std::option::Option::None => {
|
|
return std::result::Result::Err(crate::counter_exhausted_error("continuity.unresolved_gap_total"));
|
|
},
|
|
};
|
|
}
|
|
oldest_open_gap_start_slot = match oldest_open_gap_start_slot {
|
|
std::option::Option::Some(value) => std::option::Option::Some(value.min(gap.range.start_slot())),
|
|
std::option::Option::None => std::option::Option::Some(gap.range.start_slot()),
|
|
};
|
|
gaps.push(gap.snapshot());
|
|
continue;
|
|
}
|
|
repaired_gap_total = match repaired_gap_total.checked_add(1) {
|
|
std::option::Option::Some(value) => value,
|
|
std::option::Option::None => return std::result::Result::Err(crate::counter_exhausted_error("continuity.repaired_gap_total")),
|
|
};
|
|
match gap.last_method {
|
|
std::option::Option::Some(crate::RawTransactionIngestRepairMethod::Replay) => {
|
|
replay_repair_total = match replay_repair_total.checked_add(1) {
|
|
std::option::Option::Some(value) => value,
|
|
std::option::Option::None => {
|
|
return std::result::Result::Err(crate::counter_exhausted_error("continuity.replay_repair_total"));
|
|
},
|
|
};
|
|
},
|
|
std::option::Option::Some(crate::RawTransactionIngestRepairMethod::RedundantCoverage) => {
|
|
redundant_coverage_repair_total = match redundant_coverage_repair_total.checked_add(1) {
|
|
std::option::Option::Some(value) => value,
|
|
std::option::Option::None => {
|
|
return std::result::Result::Err(crate::counter_exhausted_error("continuity.redundant_coverage_repair_total"));
|
|
},
|
|
};
|
|
},
|
|
std::option::Option::Some(crate::RawTransactionIngestRepairMethod::HttpScan) => {
|
|
http_scan_repair_total = match http_scan_repair_total.checked_add(1) {
|
|
std::option::Option::Some(value) => value,
|
|
std::option::Option::None => {
|
|
return std::result::Result::Err(crate::counter_exhausted_error("continuity.http_scan_repair_total"));
|
|
},
|
|
};
|
|
},
|
|
std::option::Option::Some(crate::RawTransactionIngestRepairMethod::BlockFetch) => {
|
|
repair_block_fetch_total = match repair_block_fetch_total.checked_add(1) {
|
|
std::option::Option::Some(value) => value,
|
|
std::option::Option::None => {
|
|
return std::result::Result::Err(crate::counter_exhausted_error("continuity.repair_block_fetch_total"));
|
|
},
|
|
};
|
|
},
|
|
std::option::Option::Some(crate::RawTransactionIngestRepairMethod::TransactionHydration) => {
|
|
repair_transaction_hydration_total = match repair_transaction_hydration_total.checked_add(1) {
|
|
std::option::Option::Some(value) => value,
|
|
std::option::Option::None => {
|
|
return std::result::Result::Err(crate::counter_exhausted_error("continuity.repair_transaction_hydration_total"));
|
|
},
|
|
};
|
|
},
|
|
std::option::Option::None => {},
|
|
}
|
|
}
|
|
if gaps.len() < MAX_RAW_TRANSACTION_INGEST_OPEN_REPAIR_GAPS {
|
|
for gap in self.gaps.iter().rev() {
|
|
if gap.state != RawTransactionIngestGapState::Repaired {
|
|
continue;
|
|
}
|
|
gaps.push(gap.snapshot());
|
|
if gaps.len() == MAX_RAW_TRANSACTION_INGEST_OPEN_REPAIR_GAPS {
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
gaps.sort_unstable_by_key(|gap| return gap.gap_id().value());
|
|
return std::result::Result::Ok(
|
|
crate::RawTransactionIngestContinuitySnapshotProjection::empty()
|
|
.with_gap_state(gaps, open_gap_count, repairing_gap_count, oldest_open_gap_start_slot)
|
|
.with_recovery_totals(repaired_gap_total, unresolved_gap_total, replay_repair_total, redundant_coverage_repair_total, http_scan_repair_total)
|
|
.with_material_totals(repair_block_fetch_total, repair_transaction_hydration_total),
|
|
);
|
|
}
|
|
|
|
fn validate_invariants(&self) -> ksp_core_lib::Result<()> {
|
|
if self.next_gap_id == 0 {
|
|
return std::result::Result::Err(crate::runtime_error("continuity.gap_id_exhausted"));
|
|
}
|
|
let mut ids = std::collections::BTreeSet::new();
|
|
let mut open_gap_count = 0_usize;
|
|
let mut active_gap_count = 0_usize;
|
|
let mut highest_gap_id = 0_u64;
|
|
for gap in &self.gaps {
|
|
if gap.network != self.network {
|
|
return std::result::Result::Err(crate::runtime_error("continuity.gap_network_mismatch"));
|
|
}
|
|
if gap.gap_id.0 == 0 || !ids.insert(gap.gap_id) {
|
|
return std::result::Result::Err(crate::runtime_error("continuity.gap_id_invalid"));
|
|
}
|
|
highest_gap_id = highest_gap_id.max(gap.gap_id.0);
|
|
if gap.state.is_open() {
|
|
open_gap_count = match open_gap_count.checked_add(1) {
|
|
std::option::Option::Some(value) => value,
|
|
std::option::Option::None => return std::result::Result::Err(crate::counter_exhausted_error("continuity.open_gap_count")),
|
|
};
|
|
}
|
|
if gap.state == RawTransactionIngestGapState::Repairing {
|
|
active_gap_count = match active_gap_count.checked_add(1) {
|
|
std::option::Option::Some(value) => value,
|
|
std::option::Option::None => return std::result::Result::Err(crate::counter_exhausted_error("continuity.active_gap_count")),
|
|
};
|
|
}
|
|
if !RawTransactionIngestGapReason::ALL.contains(&gap.reason) || !RawTransactionIngestGapState::ALL.contains(&gap.state) {
|
|
return std::result::Result::Err(crate::runtime_error("continuity.gap_catalog_invalid"));
|
|
}
|
|
let range_validation = RawTransactionIngestGapRange::new(gap.range.start_slot(), gap.range.end_slot());
|
|
if let std::result::Result::Err(error) = range_validation {
|
|
return std::result::Result::Err(error);
|
|
}
|
|
}
|
|
if open_gap_count > MAX_RAW_TRANSACTION_INGEST_OPEN_REPAIR_GAPS {
|
|
return std::result::Result::Err(crate::runtime_error("continuity.open_gap_limit_exceeded"));
|
|
}
|
|
if active_gap_count > MAX_RAW_TRANSACTION_INGEST_REPAIR_ACTIVE_GAPS {
|
|
return std::result::Result::Err(crate::runtime_error("continuity.active_gap_limit_exceeded"));
|
|
}
|
|
if !self.gaps.is_empty() && self.next_gap_id <= highest_gap_id {
|
|
return std::result::Result::Err(crate::runtime_error("continuity.next_gap_id_invalid"));
|
|
}
|
|
for first_index in 0..self.gaps.len() {
|
|
let first = &self.gaps[first_index];
|
|
for second in self.gaps.iter().skip(first_index + 1) {
|
|
if first.source_key != second.source_key
|
|
|| first.commitment != second.commitment
|
|
|| first.coverage_requirement != second.coverage_requirement
|
|
|| first.reason != second.reason
|
|
|| !first.state.is_open()
|
|
|| !second.state.is_open()
|
|
{
|
|
continue;
|
|
}
|
|
let merged = first.range.try_merge(second.range);
|
|
match merged {
|
|
std::result::Result::Ok(std::option::Option::Some(_)) => {
|
|
return std::result::Result::Err(crate::runtime_error("continuity.coalescible_gap_ranges"));
|
|
},
|
|
std::result::Result::Ok(std::option::Option::None) => {},
|
|
std::result::Result::Err(error) => return std::result::Result::Err(error),
|
|
}
|
|
}
|
|
}
|
|
if RawTransactionIngestGapReason::ALL.len() != 6 || RawTransactionIngestGapState::ALL.len() != 4 {
|
|
return std::result::Result::Err(crate::runtime_error("continuity.gap_catalog_invalid"));
|
|
}
|
|
return std::result::Result::Ok(());
|
|
}
|
|
}
|
|
|
|
/// Private source capability descriptor prepared without network I/O from already validated runtime resources.
|
|
pub(crate) struct RawTransactionIngestContinuityCapabilityDescriptor {
|
|
block_material: bool,
|
|
commitment: ksp_onchain_transport_lib::SolanaCommitment,
|
|
http_block_scan: bool,
|
|
known_reference_hydration: bool,
|
|
native_replay: bool,
|
|
network: ksp_store_lib::RawNetworkId,
|
|
reference_bearing: bool,
|
|
slot_enumerating: bool,
|
|
source_key: [u8; 32],
|
|
source_scope: crate::RawTransactionIngestCoverageScope,
|
|
}
|
|
|
|
impl crate::RawTransactionIngestContinuityCapabilityDescriptor {
|
|
/// Creates one private descriptor from source-local capabilities that were validated without issuing repair I/O.
|
|
#[allow(clippy::too_many_arguments)]
|
|
pub(crate) fn new(
|
|
source_key: [u8; 32],
|
|
network: ksp_store_lib::RawNetworkId,
|
|
commitment: ksp_onchain_transport_lib::SolanaCommitment,
|
|
source_scope: crate::RawTransactionIngestCoverageScope,
|
|
reference_bearing: bool,
|
|
block_material: bool,
|
|
slot_enumerating: bool,
|
|
known_reference_hydration: bool,
|
|
native_replay: bool,
|
|
http_block_scan: bool,
|
|
) -> ksp_core_lib::Result<Self> {
|
|
let descriptor = Self {
|
|
block_material,
|
|
commitment,
|
|
http_block_scan,
|
|
known_reference_hydration,
|
|
native_replay,
|
|
network,
|
|
reference_bearing,
|
|
slot_enumerating,
|
|
source_key,
|
|
source_scope,
|
|
};
|
|
if let std::result::Result::Err(error) = descriptor.validate() {
|
|
return std::result::Result::Err(error);
|
|
}
|
|
return std::result::Result::Ok(descriptor);
|
|
}
|
|
|
|
fn validate(&self) -> ksp_core_lib::Result<()> {
|
|
if self.commitment == ksp_onchain_transport_lib::SolanaCommitment::Processed {
|
|
return std::result::Result::Err(crate::runtime_error("continuity.processed_commitment_unsupported"));
|
|
}
|
|
if let std::result::Result::Err(error) = self.source_scope.validate_configured_target_scope() {
|
|
return std::result::Result::Err(error);
|
|
}
|
|
if !self.reference_bearing
|
|
&& !self.block_material
|
|
&& !self.slot_enumerating
|
|
&& !self.known_reference_hydration
|
|
&& !self.native_replay
|
|
&& !self.http_block_scan
|
|
{
|
|
return std::result::Result::Err(crate::runtime_error("continuity.source_capability_empty"));
|
|
}
|
|
if self.http_block_scan && (!self.block_material || !self.slot_enumerating) {
|
|
return std::result::Result::Err(crate::runtime_error("continuity.http_scan_capability_incomplete"));
|
|
}
|
|
return std::result::Result::Ok(());
|
|
}
|
|
}
|
|
|
|
struct RawTransactionIngestTargetCoverage {
|
|
requirements: std::vec::Vec<RawTransactionIngestCoverageRequirement>,
|
|
}
|
|
|
|
impl RawTransactionIngestTargetCoverage {
|
|
fn from_capabilities(capabilities: &[crate::RawTransactionIngestContinuityCapabilityDescriptor]) -> ksp_core_lib::Result<Self> {
|
|
let mut value = Self { requirements: std::vec::Vec::new() };
|
|
for capability in capabilities {
|
|
if let std::result::Result::Err(error) = value.include(capability.commitment, capability.source_scope.clone()) {
|
|
return std::result::Result::Err(error);
|
|
}
|
|
}
|
|
if value.requirements.is_empty() {
|
|
return std::result::Result::Err(crate::runtime_error("continuity.target_coverage_empty"));
|
|
}
|
|
return std::result::Result::Ok(value);
|
|
}
|
|
|
|
fn is_covered_by_active_sources(
|
|
&self,
|
|
capabilities: &[crate::RawTransactionIngestContinuityCapabilityDescriptor],
|
|
active_source_keys: &std::collections::BTreeSet<[u8; 32]>,
|
|
) -> bool {
|
|
return self.requirements.iter().all(|requirement| {
|
|
return capabilities.iter().any(|capability| {
|
|
if !active_source_keys.contains(&capability.source_key) || capability.commitment != requirement.commitment {
|
|
return false;
|
|
}
|
|
return capability.source_scope.relation_to(&requirement.scope).is_some();
|
|
});
|
|
});
|
|
}
|
|
|
|
fn include(
|
|
&mut self,
|
|
commitment: ksp_onchain_transport_lib::SolanaCommitment,
|
|
scope: crate::RawTransactionIngestCoverageScope,
|
|
) -> ksp_core_lib::Result<()> {
|
|
if scope.is_known_references() {
|
|
return std::result::Result::Err(crate::runtime_error("continuity.known_references_not_target_scope"));
|
|
}
|
|
self.requirements.retain(|requirement| {
|
|
if requirement.commitment != commitment {
|
|
return true;
|
|
}
|
|
return scope.relation_to(&requirement.scope) != std::option::Option::Some(RawTransactionIngestCoverageRelation::Superset);
|
|
});
|
|
if self.requirements.iter().any(|requirement| {
|
|
return requirement.commitment == commitment && requirement.scope.relation_to(&scope).is_some();
|
|
}) {
|
|
return std::result::Result::Ok(());
|
|
}
|
|
self.requirements.push(RawTransactionIngestCoverageRequirement { commitment, scope });
|
|
return std::result::Result::Ok(());
|
|
}
|
|
}
|
|
|
|
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
|
struct RawTransactionIngestCoverageRange {
|
|
end_slot: u64,
|
|
start_slot: u64,
|
|
}
|
|
|
|
impl RawTransactionIngestCoverageRange {
|
|
fn new(start_slot: u64, end_slot: u64) -> ksp_core_lib::Result<Self> {
|
|
if end_slot < start_slot {
|
|
return std::result::Result::Err(crate::runtime_error("continuity.coverage_range_reversed"));
|
|
}
|
|
return std::result::Result::Ok(Self { end_slot, start_slot });
|
|
}
|
|
|
|
const fn contains_gap(self, gap: RawTransactionIngestGapRange) -> bool {
|
|
return self.start_slot <= gap.start_slot() && self.end_slot >= gap.end_slot();
|
|
}
|
|
|
|
fn try_merge(self, other: Self) -> std::option::Option<Self> {
|
|
let overlaps = self.start_slot <= other.end_slot && other.start_slot <= self.end_slot;
|
|
let adjacent = self.end_slot.checked_add(1) == std::option::Option::Some(other.start_slot)
|
|
|| other.end_slot.checked_add(1) == std::option::Option::Some(self.start_slot);
|
|
if !overlaps && !adjacent {
|
|
return std::option::Option::None;
|
|
}
|
|
return std::option::Option::Some(Self { end_slot: self.end_slot.max(other.end_slot), start_slot: self.start_slot.min(other.start_slot) });
|
|
}
|
|
}
|
|
|
|
#[derive(Clone, Debug, Eq, PartialEq)]
|
|
struct RawTransactionIngestCoverageEpoch {
|
|
commitment: ksp_onchain_transport_lib::SolanaCommitment,
|
|
epoch_id: u64,
|
|
range: RawTransactionIngestCoverageRange,
|
|
scope: crate::RawTransactionIngestCoverageScope,
|
|
source_key: [u8; 32],
|
|
}
|
|
|
|
impl RawTransactionIngestCoverageEpoch {
|
|
fn new(
|
|
epoch_id: u64,
|
|
source_key: [u8; 32],
|
|
commitment: ksp_onchain_transport_lib::SolanaCommitment,
|
|
scope: crate::RawTransactionIngestCoverageScope,
|
|
range: RawTransactionIngestCoverageRange,
|
|
) -> ksp_core_lib::Result<Self> {
|
|
if epoch_id == 0 {
|
|
return std::result::Result::Err(crate::runtime_error("continuity.coverage_epoch_id_invalid"));
|
|
}
|
|
if let std::result::Result::Err(error) = scope.validate_configured_target_scope() {
|
|
return std::result::Result::Err(error);
|
|
}
|
|
return std::result::Result::Ok(Self { commitment, epoch_id, range, scope, source_key });
|
|
}
|
|
|
|
fn relation_to_requirement(
|
|
&self,
|
|
target_source_key: [u8; 32],
|
|
requirement: &RawTransactionIngestCoverageRequirement,
|
|
gap: RawTransactionIngestGapRange,
|
|
) -> std::option::Option<RawTransactionIngestCoverageRelation> {
|
|
if self.source_key == target_source_key || self.commitment != requirement.commitment || !self.range.contains_gap(gap) {
|
|
return std::option::Option::None;
|
|
}
|
|
return self.scope.relation_to(&requirement.scope);
|
|
}
|
|
}
|
|
|
|
struct RawTransactionIngestCoverageEpochLedger {
|
|
epochs: std::vec::Vec<RawTransactionIngestCoverageEpoch>,
|
|
next_epoch_id: u64,
|
|
}
|
|
|
|
impl RawTransactionIngestCoverageEpochLedger {
|
|
fn new() -> Self {
|
|
return Self { epochs: std::vec::Vec::new(), next_epoch_id: 1 };
|
|
}
|
|
|
|
fn record(
|
|
&mut self,
|
|
source_key: [u8; 32],
|
|
commitment: ksp_onchain_transport_lib::SolanaCommitment,
|
|
scope: crate::RawTransactionIngestCoverageScope,
|
|
range: RawTransactionIngestCoverageRange,
|
|
capabilities: &[crate::RawTransactionIngestContinuityCapabilityDescriptor],
|
|
) -> ksp_core_lib::Result<()> {
|
|
let matching_capability = capabilities.iter().find(|capability| return capability.source_key == source_key);
|
|
let matching_capability = match matching_capability {
|
|
std::option::Option::Some(value) => value,
|
|
std::option::Option::None => return std::result::Result::Err(crate::runtime_error("continuity.coverage_epoch_source_unknown")),
|
|
};
|
|
if matching_capability.commitment != commitment || matching_capability.source_scope != scope {
|
|
return std::result::Result::Err(crate::runtime_error("continuity.coverage_epoch_capability_mismatch"));
|
|
}
|
|
for epoch in &mut self.epochs {
|
|
if epoch.source_key != source_key || epoch.commitment != commitment || epoch.scope != scope {
|
|
continue;
|
|
}
|
|
if let std::option::Option::Some(merged) = epoch.range.try_merge(range) {
|
|
epoch.range = merged;
|
|
return self.validate_invariants(capabilities);
|
|
}
|
|
}
|
|
if self.epochs.len() >= MAX_RAW_TRANSACTION_INGEST_COVERAGE_EPOCHS {
|
|
return std::result::Result::Err(crate::runtime_error("continuity.coverage_epoch_inventory_invalid"));
|
|
}
|
|
let epoch_id = self.next_epoch_id;
|
|
self.next_epoch_id = match self.next_epoch_id.checked_add(1) {
|
|
std::option::Option::Some(value) => value,
|
|
std::option::Option::None => return std::result::Result::Err(crate::runtime_error("continuity.coverage_epoch_inventory_invalid")),
|
|
};
|
|
let epoch = match RawTransactionIngestCoverageEpoch::new(epoch_id, source_key, commitment, scope, range) {
|
|
std::result::Result::Ok(value) => value,
|
|
std::result::Result::Err(error) => return std::result::Result::Err(error),
|
|
};
|
|
self.epochs.push(epoch);
|
|
return self.validate_invariants(capabilities);
|
|
}
|
|
|
|
fn redundant_relation_for_gap(
|
|
&self,
|
|
target_source_key: [u8; 32],
|
|
requirement: &RawTransactionIngestCoverageRequirement,
|
|
gap: RawTransactionIngestGapRange,
|
|
) -> std::option::Option<RawTransactionIngestCoverageRelation> {
|
|
let mut exact = false;
|
|
for epoch in &self.epochs {
|
|
match epoch.relation_to_requirement(target_source_key, requirement, gap) {
|
|
std::option::Option::Some(RawTransactionIngestCoverageRelation::Superset) => {
|
|
return std::option::Option::Some(RawTransactionIngestCoverageRelation::Superset);
|
|
},
|
|
std::option::Option::Some(RawTransactionIngestCoverageRelation::Exact) => exact = true,
|
|
std::option::Option::None => {},
|
|
}
|
|
}
|
|
if exact {
|
|
return std::option::Option::Some(RawTransactionIngestCoverageRelation::Exact);
|
|
}
|
|
return std::option::Option::None;
|
|
}
|
|
|
|
fn validate_invariants(&self, capabilities: &[crate::RawTransactionIngestContinuityCapabilityDescriptor]) -> ksp_core_lib::Result<()> {
|
|
if self.next_epoch_id == 0 || self.epochs.len() > MAX_RAW_TRANSACTION_INGEST_COVERAGE_EPOCHS {
|
|
return std::result::Result::Err(crate::runtime_error("continuity.coverage_epoch_inventory_invalid"));
|
|
}
|
|
let mut ids = std::collections::BTreeSet::new();
|
|
let mut highest_epoch_id = 0_u64;
|
|
for epoch in &self.epochs {
|
|
if epoch.epoch_id == 0 || !ids.insert(epoch.epoch_id) {
|
|
return std::result::Result::Err(crate::runtime_error("continuity.coverage_epoch_id_invalid"));
|
|
}
|
|
highest_epoch_id = highest_epoch_id.max(epoch.epoch_id);
|
|
if let std::result::Result::Err(error) = epoch.scope.validate_configured_target_scope() {
|
|
return std::result::Result::Err(error);
|
|
}
|
|
if RawTransactionIngestCoverageRange::new(epoch.range.start_slot, epoch.range.end_slot).is_err() {
|
|
return std::result::Result::Err(crate::runtime_error("continuity.coverage_epoch_range_invalid"));
|
|
}
|
|
let matching_capability = capabilities.iter().find(|capability| return capability.source_key == epoch.source_key);
|
|
let matching_capability = match matching_capability {
|
|
std::option::Option::Some(value) => value,
|
|
std::option::Option::None => return std::result::Result::Err(crate::runtime_error("continuity.coverage_epoch_source_unknown")),
|
|
};
|
|
if matching_capability.commitment != epoch.commitment || matching_capability.source_scope != epoch.scope {
|
|
return std::result::Result::Err(crate::runtime_error("continuity.coverage_epoch_capability_mismatch"));
|
|
}
|
|
}
|
|
if !self.epochs.is_empty() && self.next_epoch_id <= highest_epoch_id {
|
|
return std::result::Result::Err(crate::runtime_error("continuity.coverage_epoch_next_id_invalid"));
|
|
}
|
|
return std::result::Result::Ok(());
|
|
}
|
|
}
|
|
|
|
/// Run-local continuity contracts prepared from the exact caller-composed live-source aggregate before productive source execution.
|
|
pub(crate) struct RawTransactionIngestContinuityContracts {
|
|
capabilities: std::vec::Vec<crate::RawTransactionIngestContinuityCapabilityDescriptor>,
|
|
coverage_epochs: RawTransactionIngestCoverageEpochLedger,
|
|
gap_ledger: RawTransactionIngestGapLedger,
|
|
target_coverage: RawTransactionIngestTargetCoverage,
|
|
}
|
|
|
|
impl crate::RawTransactionIngestContinuityContracts {
|
|
/// Builds the private capability inventory, conservative `TargetCoverage` and empty run-local gap ledger without issuing network or Store I/O.
|
|
pub(crate) fn new(capabilities: std::vec::Vec<crate::RawTransactionIngestContinuityCapabilityDescriptor>) -> ksp_core_lib::Result<Self> {
|
|
if capabilities.is_empty() || capabilities.len() > crate::MAX_RAW_TRANSACTION_INGEST_LIVE_SOURCES {
|
|
return std::result::Result::Err(crate::runtime_error("continuity.capability_inventory_size_invalid"));
|
|
}
|
|
let network = capabilities[0].network.clone();
|
|
let mut source_keys = std::collections::BTreeSet::new();
|
|
for capability in &capabilities {
|
|
if let std::result::Result::Err(error) = capability.validate() {
|
|
return std::result::Result::Err(error);
|
|
}
|
|
if capability.network != network {
|
|
return std::result::Result::Err(crate::runtime_error("continuity.capability_network_mismatch"));
|
|
}
|
|
if !source_keys.insert(capability.source_key) {
|
|
return std::result::Result::Err(crate::runtime_error("continuity.duplicate_capability_source"));
|
|
}
|
|
}
|
|
let target_coverage = match RawTransactionIngestTargetCoverage::from_capabilities(capabilities.as_slice()) {
|
|
std::result::Result::Ok(value) => value,
|
|
std::result::Result::Err(error) => return std::result::Result::Err(error),
|
|
};
|
|
let coverage_epochs = RawTransactionIngestCoverageEpochLedger::new();
|
|
if let std::result::Result::Err(error) = coverage_epochs.validate_invariants(capabilities.as_slice()) {
|
|
return std::result::Result::Err(error);
|
|
}
|
|
let gap_ledger = RawTransactionIngestGapLedger::new(network);
|
|
if let std::result::Result::Err(error) = gap_ledger.validate_invariants() {
|
|
return std::result::Result::Err(error);
|
|
}
|
|
if !coverage_scope_catalog_is_complete() || !coverage_relation_catalog_is_complete() {
|
|
return std::result::Result::Err(crate::runtime_error("continuity.coverage_scope_catalog_invalid"));
|
|
}
|
|
if MAX_RAW_TRANSACTION_INGEST_REPAIR_BLOCK_FETCH_IN_FLIGHT == 0
|
|
|| crate::MAX_RAW_TRANSACTION_INGEST_CONTINUITY_DISCOVERY_WINDOW_SLOTS == 0
|
|
|| crate::MAX_RAW_TRANSACTION_INGEST_CONTINUITY_DISCOVERY_WINDOW_SLOTS > MAX_RAW_TRANSACTION_INGEST_REPAIR_RANGE_SLOTS
|
|
{
|
|
return std::result::Result::Err(crate::runtime_error("continuity.repair_bounds_invalid"));
|
|
}
|
|
return std::result::Result::Ok(Self { capabilities, coverage_epochs, gap_ledger, target_coverage });
|
|
}
|
|
|
|
/// Records one interval that a configured source has actually proven covered during this run.
|
|
pub(crate) fn record_coverage_epoch(&mut self, source_key: [u8; 32], start_slot: u64, end_slot: u64) -> ksp_core_lib::Result<()> {
|
|
let capability = self.capabilities.iter().find(|capability| return capability.source_key == source_key);
|
|
let capability = match capability {
|
|
std::option::Option::Some(value) => value,
|
|
std::option::Option::None => return std::result::Result::Err(crate::runtime_error("continuity.coverage_epoch_source_unknown")),
|
|
};
|
|
let range = match RawTransactionIngestCoverageRange::new(start_slot, end_slot) {
|
|
std::result::Result::Ok(value) => value,
|
|
std::result::Result::Err(error) => return std::result::Result::Err(error),
|
|
};
|
|
let commitment = capability.commitment;
|
|
let scope = capability.source_scope.clone();
|
|
if let std::result::Result::Err(error) = self.coverage_epochs.record(source_key, commitment, scope, range, self.capabilities.as_slice()) {
|
|
return std::result::Result::Err(error);
|
|
}
|
|
return self.gap_ledger.reconcile_with_coverage_epochs(&self.coverage_epochs);
|
|
}
|
|
|
|
/// Records one unresolved known-reference obligation as a single-slot continuity gap without blocking the processing frontier.
|
|
pub(crate) fn record_known_reference_gap(&mut self, source_key: [u8; 32], slot: u64) -> ksp_core_lib::Result<()> {
|
|
let capability = self.capabilities.iter().find(|capability| return capability.source_key == source_key);
|
|
let capability = match capability {
|
|
std::option::Option::Some(value) => value,
|
|
std::option::Option::None => return std::result::Result::Err(crate::runtime_error("continuity.known_reference_source_unknown")),
|
|
};
|
|
let range = match RawTransactionIngestGapRange::new(slot, slot) {
|
|
std::result::Result::Ok(value) => value,
|
|
std::result::Result::Err(error) => return std::result::Result::Err(error),
|
|
};
|
|
if let std::result::Result::Err(error) = self.gap_ledger.record_gap(
|
|
source_key,
|
|
capability.commitment,
|
|
capability.source_scope.clone(),
|
|
RawTransactionIngestGapReason::KnownReferenceMissing,
|
|
range,
|
|
) {
|
|
return std::result::Result::Err(error);
|
|
}
|
|
return self.gap_ledger.reconcile_with_coverage_epochs(&self.coverage_epochs);
|
|
}
|
|
|
|
/// Records one bounded run-local source-loss gap using the exact configured capability as its required coverage.
|
|
pub(crate) fn record_source_loss_gap(&mut self, source_key: [u8; 32], start_slot: u64, end_slot: u64) -> ksp_core_lib::Result<()> {
|
|
let capability = self.capabilities.iter().find(|capability| return capability.source_key == source_key);
|
|
let capability = match capability {
|
|
std::option::Option::Some(value) => value,
|
|
std::option::Option::None => return std::result::Result::Err(crate::runtime_error("continuity.source_loss_unknown_source")),
|
|
};
|
|
let range = match RawTransactionIngestGapRange::new(start_slot, end_slot) {
|
|
std::result::Result::Ok(value) => value,
|
|
std::result::Result::Err(error) => return std::result::Result::Err(error),
|
|
};
|
|
return self.gap_ledger.record_gap(
|
|
source_key,
|
|
capability.commitment,
|
|
capability.source_scope.clone(),
|
|
RawTransactionIngestGapReason::SourceFailure,
|
|
range,
|
|
);
|
|
}
|
|
|
|
/// Returns the gap-aware continuity frontier without conflating it with the processing frontier.
|
|
///
|
|
/// The processing frontier is only an upper bound: any known open continuity gap at or below it clamps the returned frontier to the slot immediately
|
|
/// before that gap. A gap beginning at slot zero yields no continuity frontier.
|
|
pub(crate) fn continuity_frontier(&self, processing_frontier_slot: std::option::Option<u64>) -> std::option::Option<u64> {
|
|
return self.gap_ledger.continuity_frontier(processing_frontier_slot);
|
|
}
|
|
|
|
/// Returns the bounded source-neutral continuity observability projection for the latest Worker snapshot.
|
|
pub(crate) fn observability_projection(&self) -> ksp_core_lib::Result<crate::RawTransactionIngestContinuitySnapshotProjection> {
|
|
return self.gap_ledger.observability_projection();
|
|
}
|
|
|
|
/// Reconciles known gaps and projects source-neutral present/future coverage evidence for Worker health.
|
|
///
|
|
/// The returned tuple is `(continuity_frontier, has_open_gaps, future_target_coverage, failed_source_losses_reconciled)`. Configuration alone may
|
|
/// satisfy only the future-coverage component; present continuity and terminal source-loss history still require proven run-local reconciliation.
|
|
pub(crate) fn health_projection(
|
|
&mut self,
|
|
active_source_keys: &[[u8; 32]],
|
|
failed_source_keys: &[[u8; 32]],
|
|
processing_frontier_slot: std::option::Option<u64>,
|
|
) -> ksp_core_lib::Result<(std::option::Option<u64>, bool, bool, bool)> {
|
|
let mut active = std::collections::BTreeSet::new();
|
|
for source_key in active_source_keys {
|
|
if !active.insert(*source_key) {
|
|
return std::result::Result::Err(crate::runtime_error("continuity.health_active_set_invalid"));
|
|
}
|
|
if !self.capabilities.iter().any(|capability| return capability.source_key == *source_key) {
|
|
return std::result::Result::Err(crate::runtime_error("continuity.health_active_source_unknown"));
|
|
}
|
|
}
|
|
let mut failed = std::collections::BTreeSet::new();
|
|
for source_key in failed_source_keys {
|
|
if active.contains(source_key) || !failed.insert(*source_key) {
|
|
return std::result::Result::Err(crate::runtime_error("continuity.health_failed_set_invalid"));
|
|
}
|
|
if !self.capabilities.iter().any(|capability| return capability.source_key == *source_key) {
|
|
return std::result::Result::Err(crate::runtime_error("continuity.health_failed_source_unknown"));
|
|
}
|
|
}
|
|
if let std::result::Result::Err(error) = self.gap_ledger.reconcile_with_coverage_epochs(&self.coverage_epochs) {
|
|
return std::result::Result::Err(error);
|
|
}
|
|
let continuity_frontier = self.continuity_frontier(processing_frontier_slot);
|
|
let has_open_gaps = self.gap_ledger.has_open_gaps();
|
|
let future_target_coverage = self.target_coverage.is_covered_by_active_sources(self.capabilities.as_slice(), &active);
|
|
let failed_source_losses_reconciled = self.gap_ledger.source_failures_reconciled(&failed);
|
|
return std::result::Result::Ok((continuity_frontier, has_open_gaps, future_target_coverage, failed_source_losses_reconciled));
|
|
}
|
|
|
|
/// Reconciles known gaps against already-proven coverage and decides whether one lost source may remain absent without stopping sibling sources.
|
|
///
|
|
/// Continuation requires all configured `TargetCoverage` requirements to remain covered by distinct currently active sources and requires the known
|
|
/// gap ledger to be fully reconciled. The method never treats configuration alone as historical coverage evidence and never respawns a source.
|
|
pub(crate) fn source_loss_decision(
|
|
&mut self,
|
|
lost_source_key: [u8; 32],
|
|
active_source_keys: &[[u8; 32]],
|
|
processing_frontier_slot: std::option::Option<u64>,
|
|
) -> ksp_core_lib::Result<crate::RawTransactionIngestSourceLossDecision> {
|
|
if !self.capabilities.iter().any(|capability| return capability.source_key == lost_source_key) {
|
|
return std::result::Result::Err(crate::runtime_error("continuity.source_loss_unknown_source"));
|
|
}
|
|
let mut active = std::collections::BTreeSet::new();
|
|
for source_key in active_source_keys {
|
|
if *source_key == lost_source_key || !active.insert(*source_key) {
|
|
return std::result::Result::Err(crate::runtime_error("continuity.source_loss_active_set_invalid"));
|
|
}
|
|
if !self.capabilities.iter().any(|capability| return capability.source_key == *source_key) {
|
|
return std::result::Result::Err(crate::runtime_error("continuity.source_loss_active_source_unknown"));
|
|
}
|
|
}
|
|
let (continuity_frontier, has_open_gaps, future_target_coverage, failed_source_losses_reconciled) =
|
|
match self.health_projection(active_source_keys, std::slice::from_ref(&lost_source_key), processing_frontier_slot) {
|
|
std::result::Result::Ok(value) => value,
|
|
std::result::Result::Err(error) => return std::result::Result::Err(error),
|
|
};
|
|
if has_open_gaps || continuity_frontier != processing_frontier_slot || !future_target_coverage || !failed_source_losses_reconciled {
|
|
return std::result::Result::Ok(crate::RawTransactionIngestSourceLossDecision::Fault);
|
|
}
|
|
return std::result::Result::Ok(crate::RawTransactionIngestSourceLossDecision::Continue);
|
|
}
|
|
|
|
/// Revalidates that the run-local contracts still correspond exactly to the caller-composed source count before tasks are spawned.
|
|
pub(crate) fn validate_for_source_count(&self, expected_source_count: usize) -> ksp_core_lib::Result<()> {
|
|
if self.capabilities.len() != expected_source_count || self.target_coverage.requirements.is_empty() {
|
|
return std::result::Result::Err(crate::runtime_error("continuity.source_inventory_mismatch"));
|
|
}
|
|
if let std::result::Result::Err(error) = self.coverage_epochs.validate_invariants(self.capabilities.as_slice()) {
|
|
return std::result::Result::Err(error);
|
|
}
|
|
if let std::result::Result::Err(error) = self.gap_ledger.validate_invariants() {
|
|
return std::result::Result::Err(error);
|
|
}
|
|
return std::result::Result::Ok(());
|
|
}
|
|
}
|
|
|
|
fn coverage_scope_catalog_is_complete() -> bool {
|
|
let full = crate::RawTransactionIngestCoverageScope::FullLedgerTransactions;
|
|
let exact = crate::RawTransactionIngestCoverageScope::ExactSourceScope("catalog", [0_u8; 32]);
|
|
let known = crate::RawTransactionIngestCoverageScope::KnownReferences([0_u8; 32]);
|
|
return full.validate_configured_target_scope().is_ok()
|
|
&& exact.validate_configured_target_scope().is_ok()
|
|
&& known.validate_configured_target_scope().is_err()
|
|
&& !full.is_known_references()
|
|
&& !exact.is_known_references()
|
|
&& known.is_known_references();
|
|
}
|
|
|
|
fn coverage_relation_catalog_is_complete() -> bool {
|
|
let target_scope = crate::RawTransactionIngestCoverageScope::ExactSourceScope("standard_logs", [7_u8; 32]);
|
|
let target_requirement =
|
|
RawTransactionIngestCoverageRequirement { commitment: ksp_onchain_transport_lib::SolanaCommitment::Confirmed, scope: target_scope.clone() };
|
|
let gap = match RawTransactionIngestGapRange::new(100, 110) {
|
|
std::result::Result::Ok(value) => value,
|
|
std::result::Result::Err(_) => return false,
|
|
};
|
|
let full_range = match RawTransactionIngestCoverageRange::new(90, 120) {
|
|
std::result::Result::Ok(value) => value,
|
|
std::result::Result::Err(_) => return false,
|
|
};
|
|
let partial_range = match RawTransactionIngestCoverageRange::new(100, 109) {
|
|
std::result::Result::Ok(value) => value,
|
|
std::result::Result::Err(_) => return false,
|
|
};
|
|
let exact_epoch =
|
|
match RawTransactionIngestCoverageEpoch::new(1, [2_u8; 32], ksp_onchain_transport_lib::SolanaCommitment::Confirmed, target_scope, full_range) {
|
|
std::result::Result::Ok(value) => value,
|
|
std::result::Result::Err(_) => return false,
|
|
};
|
|
let full_epoch = match RawTransactionIngestCoverageEpoch::new(
|
|
2,
|
|
[3_u8; 32],
|
|
ksp_onchain_transport_lib::SolanaCommitment::Confirmed,
|
|
crate::RawTransactionIngestCoverageScope::FullLedgerTransactions,
|
|
full_range,
|
|
) {
|
|
std::result::Result::Ok(value) => value,
|
|
std::result::Result::Err(_) => return false,
|
|
};
|
|
let cross_family_epoch = match RawTransactionIngestCoverageEpoch::new(
|
|
3,
|
|
[4_u8; 32],
|
|
ksp_onchain_transport_lib::SolanaCommitment::Confirmed,
|
|
crate::RawTransactionIngestCoverageScope::ExactSourceScope("helius_transaction", [7_u8; 32]),
|
|
full_range,
|
|
) {
|
|
std::result::Result::Ok(value) => value,
|
|
std::result::Result::Err(_) => return false,
|
|
};
|
|
let partial_epoch = match RawTransactionIngestCoverageEpoch::new(
|
|
4,
|
|
[5_u8; 32],
|
|
ksp_onchain_transport_lib::SolanaCommitment::Confirmed,
|
|
crate::RawTransactionIngestCoverageScope::FullLedgerTransactions,
|
|
partial_range,
|
|
) {
|
|
std::result::Result::Ok(value) => value,
|
|
std::result::Result::Err(_) => return false,
|
|
};
|
|
let ledger = RawTransactionIngestCoverageEpochLedger {
|
|
epochs: std::vec![exact_epoch.clone(), full_epoch.clone(), cross_family_epoch.clone(), partial_epoch.clone()],
|
|
next_epoch_id: 5,
|
|
};
|
|
return ledger.redundant_relation_for_gap([1_u8; 32], &target_requirement, gap)
|
|
== std::option::Option::Some(RawTransactionIngestCoverageRelation::Superset)
|
|
&& exact_epoch.relation_to_requirement([2_u8; 32], &target_requirement, gap).is_none()
|
|
&& cross_family_epoch.relation_to_requirement([1_u8; 32], &target_requirement, gap).is_none()
|
|
&& partial_epoch.relation_to_requirement([1_u8; 32], &target_requirement, gap).is_none();
|
|
}
|
|
|
|
#[cfg(test)]
|
|
#[path = "../unit_tests/continuity.rs"]
|
|
mod tests;
|