Sasedev/khadhroony-solana-project
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

v0.3.12-pre.008

Browse Source
This commit is contained in:
SinuS von SifriduS
2026-09-09 20:42:32 +02:00
parent 90c266d68a
commit 2aa64cf0a4
17 changed files with 879 additions and 47 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
crates/ksp-worker-raw-transaction-ingest-lib/src/lib.rs
View File

@@ -1,5 +1,5 @@
// file: crates/ksp-worker-raw-transaction-ingest-lib/src/lib.rs
// version: 14
// version: 15
#![warn(missing_docs)]
#![deny(unreachable_pub)]
@@ -14,8 +14,9 @@
//! the validated Yellowstone/HTTP runtime-resource contract now drives one productive supervised source task.
//! Transaction/TransactionStatus/Block updates feed bounded HTTP `getTransaction` hydration and the existing central
//! admission path; BlockMeta/Slot remain continuity-only signals. Hydration is coalesced by network/signature/commitment
//! under bounded in-flight and pending budgets. A bounded run-local processing frontier now projects hydration pending,
//! oldest pending slot and highest unblocked actually observed slot; reconnect/replay interpretation remains deferred.
//! under bounded in-flight and pending budgets. A bounded run-local processing frontier projects hydration pending,
//! oldest pending slot and highest unblocked actually observed slot. The productive source now also projects safe Transport reconnect/replay state and faults
//! conservatively when Transport proves a replay-retention continuity gap; historical repair remains outside this crate.
mod admission;
mod error;
@@ -78,6 +79,8 @@ pub use self::snapshot::RawTransactionIngestSnapshot;
pub use self::snapshot::RawTransactionIngestSnapshotFuture;
/// Cloneable latest-value source exposing concrete and common Worker snapshots from one shared watch state.
pub use self::snapshot::RawTransactionIngestSnapshotSource;
/// Source-neutral lifecycle state of the productive RAW transaction ingest source.
pub use self::snapshot::RawTransactionIngestSourceState;
/// Receiver-side owner of the private bounded RAW transaction admission queue.
pub(crate) use self::admission::RawTransactionAdmission;

119
crates/ksp-worker-raw-transaction-ingest-lib/src/runtime_resources.rs
View File

@@ -1,5 +1,5 @@
// file: crates/ksp-worker-raw-transaction-ingest-lib/src/runtime_resources.rs
// version: 10
// version: 11
use sha2::Digest; // rust-rules: trait-import
@@ -382,6 +382,13 @@ impl crate::RawTransactionIngestYellowstoneSource {
};
let mut coordinator = RawTransactionIngestHydrationCoordinator::new(&settings);
let mut processing_frontier = RawTransactionIngestProcessingFrontierReporter::new(processing_frontier_sender);
let snapshot_source = session.snapshot_source();
if let std::result::Result::Err(error) = processing_frontier.observe_session_snapshot(snapshot_source.current()) {
processing_frontier.set_source_state(crate::RawTransactionIngestSourceState::Failed);
let _closed = session.close().await;
return std::result::Result::Err(error);
}
let mut session_snapshot_source = std::option::Option::Some(snapshot_source);
let mut fault = std::option::Option::None;
loop {
if *stop_receiver.borrow() {
@@ -401,6 +408,19 @@ impl crate::RawTransactionIngestYellowstoneSource {
_ = stop_receiver.changed() => {
break;
}
source_snapshot = wait_yellowstone_session_snapshot(&mut session_snapshot_source) => {
match source_snapshot {
std::option::Option::Some(snapshot) => {
if let std::result::Result::Err(error) = processing_frontier.observe_session_snapshot(snapshot) {
fault = std::option::Option::Some(error);
break;
}
},
std::option::Option::None => {
session_snapshot_source = std::option::Option::None;
},
}
}
joined = coordinator.tasks.join_next(), if !coordinator.tasks.is_empty() => {
let joined = match joined {
std::option::Option::Some(value) => value,
@@ -453,13 +473,21 @@ impl crate::RawTransactionIngestYellowstoneSource {
}
}
coordinator.abort_all().await;
processing_frontier.set_source_state(crate::RawTransactionIngestSourceState::Closing);
let closed = session.close().await;
if let std::option::Option::Some(error) = fault {
processing_frontier.set_source_state(crate::RawTransactionIngestSourceState::Failed);
return std::result::Result::Err(error);
}
return match closed {
std::result::Result::Ok(()) => std::result::Result::Ok(()),
std::result::Result::Err(error) => std::result::Result::Err(source_transport_error(error.code())),
std::result::Result::Ok(()) => {
processing_frontier.set_source_state(crate::RawTransactionIngestSourceState::Closed);
std::result::Result::Ok(())
},
std::result::Result::Err(error) => {
processing_frontier.set_source_state(crate::RawTransactionIngestSourceState::Failed);
std::result::Result::Err(source_transport_error(error.code()))
},
};
}
}
@@ -774,12 +802,33 @@ fn current_raw_timestamp() -> ksp_core_lib::Result<ksp_store_lib::RawTimestamp>
};
}
async fn wait_yellowstone_session_snapshot(
source: &mut std::option::Option<ksp_onchain_transport_lib::YellowstoneGrpcSubscribeSnapshotSource>,
) -> std::option::Option<ksp_onchain_transport_lib::YellowstoneGrpcSubscribeSnapshot> {
return match source {
std::option::Option::Some(value) => value.wait_for_change().await,
std::option::Option::None => {
return std::future::pending::<std::option::Option<ksp_onchain_transport_lib::YellowstoneGrpcSubscribeSnapshot>>().await;
},
};
}
fn source_transport_error(code: ksp_core_lib::ErrorCode) -> ksp_core_lib::Error {
return ksp_core_lib::Error::new(crate::ERROR_CODE_RAW_TRANSACTION_INGEST_SOURCE_FAILED, "RAW transaction ingest Yellowstone source transport failed")
.with_context("transport_domain", code.domain())
.with_context("transport_code", code.code());
}
fn map_yellowstone_source_state(state: ksp_onchain_transport_lib::YellowstoneGrpcSubscribeState) -> crate::RawTransactionIngestSourceState {
return match state {
ksp_onchain_transport_lib::YellowstoneGrpcSubscribeState::Active => crate::RawTransactionIngestSourceState::Active,
ksp_onchain_transport_lib::YellowstoneGrpcSubscribeState::Reconnecting => crate::RawTransactionIngestSourceState::Reconnecting,
ksp_onchain_transport_lib::YellowstoneGrpcSubscribeState::Closing => crate::RawTransactionIngestSourceState::Closing,
ksp_onchain_transport_lib::YellowstoneGrpcSubscribeState::Closed => crate::RawTransactionIngestSourceState::Closed,
ksp_onchain_transport_lib::YellowstoneGrpcSubscribeState::Failed => crate::RawTransactionIngestSourceState::Failed,
};
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
struct RawTransactionIngestProcessingSlotState {
pending: usize,
@@ -915,11 +964,22 @@ impl RawTransactionIngestProcessingFrontier {
struct RawTransactionIngestProcessingFrontierReporter {
frontier: RawTransactionIngestProcessingFrontier,
sender: tokio::sync::watch::Sender<crate::RawTransactionIngestProcessingFrontierProjection>,
source_state: std::option::Option<crate::RawTransactionIngestSourceState>,
source_reconnect_total: u64,
source_replay_attempt_total: u64,
source_continuity_gap_total: u64,
}
impl RawTransactionIngestProcessingFrontierReporter {
fn new(sender: tokio::sync::watch::Sender<crate::RawTransactionIngestProcessingFrontierProjection>) -> Self {
return Self { frontier: RawTransactionIngestProcessingFrontier::new(), sender };
return Self {
frontier: RawTransactionIngestProcessingFrontier::new(),
sender,
source_state: std::option::Option::None,
source_reconnect_total: 0,
source_replay_attempt_total: 0,
source_continuity_gap_total: 0,
};
}
fn observe_pending(&mut self, slot: u64) -> ksp_core_lib::Result<()> {
@@ -946,8 +1006,57 @@ impl RawTransactionIngestProcessingFrontierReporter {
return std::result::Result::Ok(());
}
fn observe_session_snapshot(&mut self, snapshot: ksp_onchain_transport_lib::YellowstoneGrpcSubscribeSnapshot) -> ksp_core_lib::Result<()> {
return self.observe_source_continuity(
map_yellowstone_source_state(snapshot.state()),
snapshot.reconnect_count(),
snapshot.replay_attempt_count(),
snapshot.continuity_gap_count(),
);
}
fn observe_source_continuity(
&mut self,
state: crate::RawTransactionIngestSourceState,
reconnect_total: u64,
replay_attempt_total: u64,
continuity_gap_total: u64,
) -> ksp_core_lib::Result<()> {
if reconnect_total < self.source_reconnect_total
|| replay_attempt_total < self.source_replay_attempt_total
|| continuity_gap_total < self.source_continuity_gap_total
{
return std::result::Result::Err(crate::runtime_error("source.continuity_counter_regression"));
}
let gap_increased = continuity_gap_total > self.source_continuity_gap_total;
self.source_state = std::option::Option::Some(state);
self.source_reconnect_total = reconnect_total;
self.source_replay_attempt_total = replay_attempt_total;
self.source_continuity_gap_total = continuity_gap_total;
self.publish();
if gap_increased {
return std::result::Result::Err(crate::runtime_error("source.continuity_gap_proven"));
}
return std::result::Result::Ok(());
}
fn set_source_state(&mut self, state: crate::RawTransactionIngestSourceState) {
self.source_state = std::option::Option::Some(state);
self.publish();
return;
}
fn projection(&self) -> crate::RawTransactionIngestProcessingFrontierProjection {
return self.frontier.projection().with_source_continuity(
self.source_state,
self.source_reconnect_total,
self.source_replay_attempt_total,
self.source_continuity_gap_total,
);
}
fn publish(&self) {
self.sender.send_replace(self.frontier.projection());
self.sender.send_replace(self.projection());
return;
}
}

125
crates/ksp-worker-raw-transaction-ingest-lib/src/snapshot.rs
View File

@@ -1,16 +1,35 @@
// file: crates/ksp-worker-raw-transaction-ingest-lib/src/snapshot.rs
// version: 3
// version: 4
/// Runtime-neutral boxed future resolving to one newer concrete RAW transaction ingest Worker snapshot.
pub type RawTransactionIngestSnapshotFuture<'a> =
std::pin::Pin<std::boxed::Box<dyn std::future::Future<Output = crate::RawTransactionIngestSnapshot> + std::marker::Send + 'a>>;
/// Private latest-value processing-frontier projection emitted by the productive source task.
/// Source-neutral lifecycle state of the productive RAW transaction ingest source.
#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)]
pub enum RawTransactionIngestSourceState {
/// The productive source is active.
Active,
/// Transport is performing one bounded reconnect/replay attempt.
Reconnecting,
/// Cooperative source shutdown has started.
Closing,
/// The productive source closed cleanly.
Closed,
/// The productive source reached a terminal failure.
Failed,
}
/// Private latest-value source-processing projection emitted by the productive source task.
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub(crate) struct RawTransactionIngestProcessingFrontierProjection {
hydration_pending: usize,
processing_frontier_slot: std::option::Option<u64>,
oldest_pending_slot: std::option::Option<u64>,
source_state: std::option::Option<crate::RawTransactionIngestSourceState>,
source_reconnect_total: u64,
source_replay_attempt_total: u64,
source_continuity_gap_total: u64,
}
impl crate::RawTransactionIngestProcessingFrontierProjection {
@@ -20,6 +39,10 @@ impl crate::RawTransactionIngestProcessingFrontierProjection {
hydration_pending: 0,
processing_frontier_slot: std::option::Option::None,
oldest_pending_slot: std::option::Option::None,
source_state: std::option::Option::None,
source_reconnect_total: 0,
source_replay_attempt_total: 0,
source_continuity_gap_total: 0,
};
}
@@ -29,7 +52,15 @@ impl crate::RawTransactionIngestProcessingFrontierProjection {
processing_frontier_slot: std::option::Option<u64>,
oldest_pending_slot: std::option::Option<u64>,
) -> Self {
return Self { hydration_pending, processing_frontier_slot, oldest_pending_slot };
return Self {
hydration_pending,
processing_frontier_slot,
oldest_pending_slot,
source_state: std::option::Option::None,
source_reconnect_total: 0,
source_replay_attempt_total: 0,
source_continuity_gap_total: 0,
};
}
/// Returns the number of source signals still pending hydration/admission processing.
@@ -46,6 +77,41 @@ impl crate::RawTransactionIngestProcessingFrontierProjection {
pub(crate) const fn oldest_pending_slot(&self) -> std::option::Option<u64> {
return self.oldest_pending_slot;
}
/// Returns a copy carrying the latest source reconnect/replay projection.
pub(crate) const fn with_source_continuity(
mut self,
source_state: std::option::Option<crate::RawTransactionIngestSourceState>,
source_reconnect_total: u64,
source_replay_attempt_total: u64,
source_continuity_gap_total: u64,
) -> Self {
self.source_state = source_state;
self.source_reconnect_total = source_reconnect_total;
self.source_replay_attempt_total = source_replay_attempt_total;
self.source_continuity_gap_total = source_continuity_gap_total;
return self;
}
/// Returns the latest source-neutral lifecycle state carried by this private projection.
pub(crate) const fn source_state(&self) -> std::option::Option<crate::RawTransactionIngestSourceState> {
return self.source_state;
}
/// Returns successful reconnects carried by this private projection.
pub(crate) const fn source_reconnect_total(&self) -> u64 {
return self.source_reconnect_total;
}
/// Returns replay-bearing reconnect attempts carried by this private projection.
pub(crate) const fn source_replay_attempt_total(&self) -> u64 {
return self.source_replay_attempt_total;
}
/// Returns proven replay-retention gaps carried by this private projection.
pub(crate) const fn source_continuity_gap_total(&self) -> u64 {
return self.source_continuity_gap_total;
}
}
/// Complete safe latest-value snapshot of one continuous RAW transaction ingest Worker.
@@ -71,6 +137,10 @@ pub struct RawTransactionIngestSnapshot {
hydration_pending: usize,
processing_frontier_slot: std::option::Option<u64>,
oldest_pending_slot: std::option::Option<u64>,
source_state: std::option::Option<crate::RawTransactionIngestSourceState>,
source_reconnect_total: u64,
source_replay_attempt_total: u64,
source_continuity_gap_total: u64,
}
impl crate::RawTransactionIngestSnapshot {
@@ -193,6 +263,30 @@ impl crate::RawTransactionIngestSnapshot {
pub const fn oldest_pending_slot(&self) -> std::option::Option<u64> {
return self.oldest_pending_slot;
}
/// Returns the latest source-neutral lifecycle state when the productive source has started.
#[must_use]
pub const fn source_state(&self) -> std::option::Option<crate::RawTransactionIngestSourceState> {
return self.source_state;
}
/// Returns successful automatic Yellowstone reconnects observed by this Worker run.
#[must_use]
pub const fn source_reconnect_total(&self) -> u64 {
return self.source_reconnect_total;
}
/// Returns Yellowstone replay-bearing reconnect attempts observed by this Worker run.
#[must_use]
pub const fn source_replay_attempt_total(&self) -> u64 {
return self.source_replay_attempt_total;
}
/// Returns replay retention gaps conservatively proven by Transport during this Worker run.
#[must_use]
pub const fn source_continuity_gap_total(&self) -> u64 {
return self.source_continuity_gap_total;
}
}
impl std::fmt::Debug for crate::RawTransactionIngestSnapshot {
@@ -219,6 +313,10 @@ impl std::fmt::Debug for crate::RawTransactionIngestSnapshot {
.field("hydration_pending", &self.hydration_pending)
.field("processing_frontier_slot", &self.processing_frontier_slot)
.field("oldest_pending_slot", &self.oldest_pending_slot)
.field("source_state", &self.source_state)
.field("source_reconnect_total", &self.source_reconnect_total)
.field("source_replay_attempt_total", &self.source_replay_attempt_total)
.field("source_continuity_gap_total", &self.source_continuity_gap_total)
.finish();
}
}
@@ -335,6 +433,10 @@ impl crate::RawTransactionIngestSnapshotPublisher {
hydration_pending: 0,
processing_frontier_slot: std::option::Option::None,
oldest_pending_slot: std::option::Option::None,
source_state: std::option::Option::None,
source_reconnect_total: 0,
source_replay_attempt_total: 0,
source_continuity_gap_total: 0,
};
let (sender, receiver) = tokio::sync::watch::channel(snapshot.clone());
return (Self { sender, snapshot }, crate::RawTransactionIngestSnapshotSource { receiver });
@@ -427,6 +529,10 @@ impl crate::RawTransactionIngestSnapshotPublisher {
self.snapshot.hydration_pending = projection.hydration_pending();
self.snapshot.processing_frontier_slot = projection.processing_frontier_slot();
self.snapshot.oldest_pending_slot = projection.oldest_pending_slot();
self.snapshot.source_state = projection.source_state();
self.snapshot.source_reconnect_total = projection.source_reconnect_total();
self.snapshot.source_replay_attempt_total = projection.source_replay_attempt_total();
self.snapshot.source_continuity_gap_total = projection.source_continuity_gap_total();
return self.publish(state, admission_queue_depth, in_flight_persistence);
}
@@ -541,7 +647,7 @@ impl crate::RawTransactionIngestSnapshotPublisher {
sequence,
state,
health_for_state(state, self.snapshot.worker.health()),
activity_for_state(state, admission_queue_depth, in_flight_persistence, self.snapshot.hydration_pending),
activity_for_state(state, admission_queue_depth, in_flight_persistence, self.snapshot.hydration_pending, self.snapshot.source_state),
);
self.snapshot.worker = worker;
self.snapshot.admission_queue_depth = admission_queue_depth;
@@ -556,8 +662,17 @@ fn activity_for_state(
admission_queue_depth: usize,
in_flight_persistence: usize,
hydration_pending: usize,
source_state: std::option::Option<crate::RawTransactionIngestSourceState>,
) -> ksp_worker_api::WorkerActivity {
if admission_queue_depth > 0 || in_flight_persistence > 0 || hydration_pending > 0 {
if admission_queue_depth > 0
|| in_flight_persistence > 0
|| hydration_pending > 0
|| matches!(
source_state,
std::option::Option::Some(crate::RawTransactionIngestSourceState::Reconnecting)
| std::option::Option::Some(crate::RawTransactionIngestSourceState::Closing)
)
{
return ksp_worker_api::WorkerActivity::Active;
}
return match state {