Sasedev/khadhroony-solana-project
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

v0.3.13-pre.010

Browse Source
This commit is contained in:
SinuS von SifriduS
2026-09-10 21:02:29 +02:00
parent 0f8f38ff52
commit f636169783
16 changed files with 476 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
crates/ksp-worker-raw-transaction-ingest-lib/src/lib.rs
View File

@@ -1,5 +1,5 @@
// file: crates/ksp-worker-raw-transaction-ingest-lib/src/lib.rs
// version: 24
// version: 25
#![warn(missing_docs)]
#![deny(unreachable_pub)]
@@ -10,11 +10,10 @@
//! This tranche owns the concrete Worker family identity, validated technical settings
//! and the caller-runtime-owned lifecycle with private child-task supervision. This tranche also
//! owns bounded source-neutral admission, common RAW canonicalization/assembly and backend-neutral
//! Store persistence in normal mode plus concrete latest-value snapshots projected onto Worker API. `pre.006` keeps the bounded 1..32 caller-composed
//! aggregate and adds productive HTTP live block polling beside Yellowstone, Standard Logs, Standard Block and Helius Transaction while
//! simultaneous multi-source activation remains gated until the dedicated supervisor tranche. Helius Full notifications, Standard Logs and Yellowstone
//! reference paths converge into one source-neutral hydration coordinator contract; qualified Standard Block Legacy/V0/V1 transactions enter the existing
//! central admission path directly. Yellowstone
//! Store persistence in normal mode plus concrete latest-value snapshots projected onto Worker API. The bounded 1..32 caller-composed aggregate starts
//! all validated Yellowstone, Standard Logs, Standard Block, Helius Transaction and HTTP live block polling sources under one private supervisor. Reference-bearing
//! sources share bounded cross-source hydration and fairness budgets; direct-qualified Standard Block and HTTP polling transactions enter the existing central
//! admission path directly. Public snapshots expose only source-neutral aggregate counts/state and conservative Worker health. Yellowstone
//! Transaction/TransactionStatus/Block and standard logs notifications become signature/slot references; BlockMeta/Slot remain continuity-only signals.
//! Hydration is coalesced by network/signature/commitment under bounded in-flight and pending budgets. A bounded run-local processing frontier projects
//! hydration pending, oldest pending slot and highest unblocked actually observed slot. The productive source also projects safe Transport reconnect/replay

9
crates/ksp-worker-raw-transaction-ingest-lib/src/runtime.rs
View File

@@ -1,5 +1,5 @@
// file: crates/ksp-worker-raw-transaction-ingest-lib/src/runtime.rs
// version: 15
// version: 16
type PersistencePort = std::sync::Arc<dyn crate::RawTransactionIngestPersistencePort + 'static>;
type PersistenceTasks = tokio::task::JoinSet<ksp_core_lib::Result<crate::RawTransactionIngestPersistenceOutcome>>;
@@ -117,6 +117,7 @@ impl crate::RawTransactionIngestWorker {
if let std::result::Result::Err(error) = runtime_resources.validate_network(settings.network()) {
return std::result::Result::Err(error);
}
let source_total = runtime_resources.source_count();
let source_settings = settings.clone();
let (processing_frontier_sender, processing_frontier_receiver) =
tokio::sync::watch::channel(crate::RawTransactionIngestProcessingFrontierProjection::empty());
@@ -126,6 +127,7 @@ impl crate::RawTransactionIngestWorker {
runtime,
std::option::Option::Some(port),
std::option::Option::Some(processing_frontier_receiver),
source_total,
move |children, stop_receiver, admission_sender| {
let _abort_handle = children.spawn(async move {
return runtime_resources.run_live_sources(source_settings, stop_receiver, admission_sender, processing_frontier_sender).await;
@@ -458,7 +460,7 @@ where
Spawner:
FnOnce(&mut SourceTasks, tokio::sync::watch::Receiver<bool>, tokio::sync::mpsc::Sender<crate::RawTransactionIngress>) + std::marker::Send + 'static,
{
return start_foundation_with_port_source_spawner_and_frontier(settings, runtime, port, std::option::Option::None, source_spawner);
return start_foundation_with_port_source_spawner_and_frontier(settings, runtime, port, std::option::Option::None, 0, source_spawner);
}
fn start_foundation_with_port_source_spawner_and_frontier<Spawner>(
@@ -466,6 +468,7 @@ fn start_foundation_with_port_source_spawner_and_frontier<Spawner>(
runtime: tokio::runtime::Handle,
port: std::option::Option<PersistencePort>,
processing_frontier_receiver: std::option::Option<ProcessingFrontierReceiver>,
source_total: usize,
source_spawner: Spawner,
) -> ksp_core_lib::Result<crate::RawTransactionIngestHandle>
where
@@ -482,7 +485,7 @@ where
}
let stop_token = ksp_worker_api::WorkerStopToken::new();
let (stop_sender, stop_receiver) = tokio::sync::watch::channel(false);
let (snapshots, snapshot_source) = crate::RawTransactionIngestSnapshotPublisher::new(&settings, &lifecycle);
let (snapshots, snapshot_source) = crate::RawTransactionIngestSnapshotPublisher::new(&settings, &lifecycle, source_total);
let handle = crate::RawTransactionIngestHandle { snapshots: snapshot_source, stop_sender, stop_token };
std::mem::drop(runtime.spawn(run_supervisor(settings, lifecycle, port, stop_receiver, snapshots, processing_frontier_receiver, source_spawner)));
return std::result::Result::Ok(handle);

12
crates/ksp-worker-raw-transaction-ingest-lib/src/runtime_resources.rs
View File

@@ -1,5 +1,5 @@
// file: crates/ksp-worker-raw-transaction-ingest-lib/src/runtime_resources.rs
// version: 24
// version: 25
use sha2::Digest; // rust-rules: trait-import
@@ -179,6 +179,10 @@ impl RawTransactionIngestSourceInventory {
let mut source_continuity_gap_total = 0_u64;
let mut source_reconnect_total = 0_u64;
let mut source_replay_attempt_total = 0_u64;
let source_total = self.source_projections.len();
let mut source_active = 0_usize;
let mut source_reconnecting = 0_usize;
let mut source_failed = 0_usize;
let mut any_active = false;
let mut any_closing = false;
let mut any_failed = false;
@@ -200,6 +204,7 @@ impl RawTransactionIngestSourceInventory {
source_replay_attempt_total = source_replay_attempt_total.saturating_add(projection.source_replay_attempt_total());
match projection.source_state() {
std::option::Option::Some(crate::RawTransactionIngestSourceState::Active) => {
source_active += 1;
any_active = true;
all_closed = false;
},
@@ -209,10 +214,12 @@ impl RawTransactionIngestSourceInventory {
},
std::option::Option::Some(crate::RawTransactionIngestSourceState::Closed) => {},
std::option::Option::Some(crate::RawTransactionIngestSourceState::Failed) => {
source_failed += 1;
any_failed = true;
all_closed = false;
},
std::option::Option::Some(crate::RawTransactionIngestSourceState::Reconnecting) => {
source_reconnecting += 1;
any_reconnecting = true;
all_closed = false;
},
@@ -238,7 +245,8 @@ impl RawTransactionIngestSourceInventory {
std::option::Option::None
};
return crate::RawTransactionIngestProcessingFrontierProjection::new(hydration_pending, processing_frontier_slot, oldest_pending_slot)
.with_source_continuity(source_state, source_reconnect_total, source_replay_attempt_total, source_continuity_gap_total);
.with_source_continuity(source_state, source_reconnect_total, source_replay_attempt_total, source_continuity_gap_total)
.with_source_counts(source_total, source_active, source_reconnecting, source_failed);
}
fn new(source_keys: std::vec::Vec<[u8; 32]>) -> Self {

116
crates/ksp-worker-raw-transaction-ingest-lib/src/snapshot.rs
View File

@@ -1,5 +1,5 @@
// file: crates/ksp-worker-raw-transaction-ingest-lib/src/snapshot.rs
// version: 4
// version: 5
/// Runtime-neutral boxed future resolving to one newer concrete RAW transaction ingest Worker snapshot.
pub type RawTransactionIngestSnapshotFuture<'a> =
@@ -27,6 +27,10 @@ pub(crate) struct RawTransactionIngestProcessingFrontierProjection {
processing_frontier_slot: std::option::Option<u64>,
oldest_pending_slot: std::option::Option<u64>,
source_state: std::option::Option<crate::RawTransactionIngestSourceState>,
source_total: usize,
source_active: usize,
source_reconnecting: usize,
source_failed: usize,
source_reconnect_total: u64,
source_replay_attempt_total: u64,
source_continuity_gap_total: u64,
@@ -40,6 +44,10 @@ impl crate::RawTransactionIngestProcessingFrontierProjection {
processing_frontier_slot: std::option::Option::None,
oldest_pending_slot: std::option::Option::None,
source_state: std::option::Option::None,
source_total: 0,
source_active: 0,
source_reconnecting: 0,
source_failed: 0,
source_reconnect_total: 0,
source_replay_attempt_total: 0,
source_continuity_gap_total: 0,
@@ -57,6 +65,10 @@ impl crate::RawTransactionIngestProcessingFrontierProjection {
processing_frontier_slot,
oldest_pending_slot,
source_state: std::option::Option::None,
source_total: 0,
source_active: 0,
source_reconnecting: 0,
source_failed: 0,
source_reconnect_total: 0,
source_replay_attempt_total: 0,
source_continuity_gap_total: 0,
@@ -93,6 +105,35 @@ impl crate::RawTransactionIngestProcessingFrontierProjection {
return self;
}
/// Returns a copy carrying source-neutral multi-source lifecycle counts.
pub(crate) const fn with_source_counts(mut self, source_total: usize, source_active: usize, source_reconnecting: usize, source_failed: usize) -> Self {
self.source_total = source_total;
self.source_active = source_active;
self.source_reconnecting = source_reconnecting;
self.source_failed = source_failed;
return self;
}
/// Returns the configured logical source count carried by this private projection.
pub(crate) const fn source_total(&self) -> usize {
return self.source_total;
}
/// Returns the number of sources currently projected Active.
pub(crate) const fn source_active(&self) -> usize {
return self.source_active;
}
/// Returns the number of sources currently projected Reconnecting.
pub(crate) const fn source_reconnecting(&self) -> usize {
return self.source_reconnecting;
}
/// Returns the number of sources currently projected Failed.
pub(crate) const fn source_failed(&self) -> usize {
return self.source_failed;
}
/// Returns the latest source-neutral lifecycle state carried by this private projection.
pub(crate) const fn source_state(&self) -> std::option::Option<crate::RawTransactionIngestSourceState> {
return self.source_state;
@@ -138,6 +179,10 @@ pub struct RawTransactionIngestSnapshot {
processing_frontier_slot: std::option::Option<u64>,
oldest_pending_slot: std::option::Option<u64>,
source_state: std::option::Option<crate::RawTransactionIngestSourceState>,
source_total: usize,
source_active: usize,
source_reconnecting: usize,
source_failed: usize,
source_reconnect_total: u64,
source_replay_attempt_total: u64,
source_continuity_gap_total: u64,
@@ -264,6 +309,30 @@ impl crate::RawTransactionIngestSnapshot {
return self.oldest_pending_slot;
}
/// Returns the configured number of logical live sources for this Worker run.
#[must_use]
pub const fn source_total(&self) -> usize {
return self.source_total;
}
/// Returns the latest number of sources projected Active.
#[must_use]
pub const fn source_active(&self) -> usize {
return self.source_active;
}
/// Returns the latest number of sources projected Reconnecting.
#[must_use]
pub const fn source_reconnecting(&self) -> usize {
return self.source_reconnecting;
}
/// Returns the latest number of sources projected Failed.
#[must_use]
pub const fn source_failed(&self) -> usize {
return self.source_failed;
}
/// Returns the latest source-neutral lifecycle state when the productive source has started.
#[must_use]
pub const fn source_state(&self) -> std::option::Option<crate::RawTransactionIngestSourceState> {
@@ -314,6 +383,10 @@ impl std::fmt::Debug for crate::RawTransactionIngestSnapshot {
.field("processing_frontier_slot", &self.processing_frontier_slot)
.field("oldest_pending_slot", &self.oldest_pending_slot)
.field("source_state", &self.source_state)
.field("source_total", &self.source_total)
.field("source_active", &self.source_active)
.field("source_reconnecting", &self.source_reconnecting)
.field("source_failed", &self.source_failed)
.field("source_reconnect_total", &self.source_reconnect_total)
.field("source_replay_attempt_total", &self.source_replay_attempt_total)
.field("source_continuity_gap_total", &self.source_continuity_gap_total)
@@ -403,6 +476,7 @@ impl crate::RawTransactionIngestSnapshotPublisher {
pub(crate) fn new(
settings: &crate::RawTransactionIngestSettings,
lifecycle: &ksp_worker_api::WorkerLifecycle,
source_total: usize,
) -> (crate::RawTransactionIngestSnapshotPublisher, crate::RawTransactionIngestSnapshotSource) {
let worker = ksp_worker_api::WorkerSnapshot::new(
lifecycle.id().clone(),
@@ -434,6 +508,10 @@ impl crate::RawTransactionIngestSnapshotPublisher {
processing_frontier_slot: std::option::Option::None,
oldest_pending_slot: std::option::Option::None,
source_state: std::option::Option::None,
source_total,
source_active: 0,
source_reconnecting: 0,
source_failed: 0,
source_reconnect_total: 0,
source_replay_attempt_total: 0,
source_continuity_gap_total: 0,
@@ -449,7 +527,14 @@ impl crate::RawTransactionIngestSnapshotPublisher {
self.snapshot.worker.kind().clone(),
self.snapshot.worker.sequence(),
state,
health_for_state(state, self.snapshot.worker.health()),
health_for_state(
state,
self.snapshot.worker.health(),
self.snapshot.source_total,
self.snapshot.source_active,
self.snapshot.source_reconnecting,
self.snapshot.source_failed,
),
ksp_worker_api::WorkerActivity::Idle,
);
self.snapshot.worker = worker;
@@ -530,6 +615,12 @@ impl crate::RawTransactionIngestSnapshotPublisher {
self.snapshot.processing_frontier_slot = projection.processing_frontier_slot();
self.snapshot.oldest_pending_slot = projection.oldest_pending_slot();
self.snapshot.source_state = projection.source_state();
if projection.source_total() > 0 {
self.snapshot.source_total = projection.source_total();
self.snapshot.source_active = projection.source_active();
self.snapshot.source_reconnecting = projection.source_reconnecting();
self.snapshot.source_failed = projection.source_failed();
}
self.snapshot.source_reconnect_total = projection.source_reconnect_total();
self.snapshot.source_replay_attempt_total = projection.source_replay_attempt_total();
self.snapshot.source_continuity_gap_total = projection.source_continuity_gap_total();
@@ -646,7 +737,14 @@ impl crate::RawTransactionIngestSnapshotPublisher {
self.snapshot.worker.kind().clone(),
sequence,
state,
health_for_state(state, self.snapshot.worker.health()),
health_for_state(
state,
self.snapshot.worker.health(),
self.snapshot.source_total,
self.snapshot.source_active,
self.snapshot.source_reconnecting,
self.snapshot.source_failed,
),
activity_for_state(state, admission_queue_depth, in_flight_persistence, self.snapshot.hydration_pending, self.snapshot.source_state),
);
self.snapshot.worker = worker;
@@ -698,8 +796,18 @@ fn checked_optional_counter(current: u64, increment: bool, field: &'static str)
return checked_counter(current, field);
}
fn health_for_state(state: ksp_worker_api::WorkerState, previous: ksp_worker_api::WorkerHealth) -> ksp_worker_api::WorkerHealth {
fn health_for_state(
state: ksp_worker_api::WorkerState,
previous: ksp_worker_api::WorkerHealth,
source_total: usize,
source_active: usize,
source_reconnecting: usize,
source_failed: usize,
) -> ksp_worker_api::WorkerHealth {
return match state {
ksp_worker_api::WorkerState::Running if source_failed > 0 => ksp_worker_api::WorkerHealth::Unhealthy,
ksp_worker_api::WorkerState::Running if source_reconnecting > 0 => ksp_worker_api::WorkerHealth::Degraded,
ksp_worker_api::WorkerState::Running if source_total > 0 && source_active < source_total => ksp_worker_api::WorkerHealth::Degraded,
ksp_worker_api::WorkerState::Running => ksp_worker_api::WorkerHealth::Healthy,
ksp_worker_api::WorkerState::Stopping | ksp_worker_api::WorkerState::Stopped => previous,
ksp_worker_api::WorkerState::Faulted(_) => ksp_worker_api::WorkerHealth::Unhealthy,