Sasedev/khadhroony-solana-project
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

v0.3.13-pre.007

Browse Source
This commit is contained in:
SinuS von SifriduS
2026-09-10 17:33:02 +02:00
parent 72df965a9c
commit 8bb530e9b4
13 changed files with 905 additions and 78 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
crates/ksp-worker-raw-transaction-ingest-lib/src/runtime.rs
View File

@@ -1,5 +1,5 @@
// file: crates/ksp-worker-raw-transaction-ingest-lib/src/runtime.rs
// version: 13
// version: 14
type PersistencePort = std::sync::Arc<dyn crate::RawTransactionIngestPersistencePort + 'static>;
type PersistenceTasks = tokio::task::JoinSet<ksp_core_lib::Result<crate::RawTransactionIngestPersistenceOutcome>>;
@@ -99,8 +99,8 @@ impl crate::RawTransactionIngestWorker {
/// Starts one Worker with caller-composed runtime resources.
///
/// `pre.003` activates either the existing Yellowstone source or one standard Solana `logsSubscribe` source through the same bounded Worker foundation.
/// Aggregates with more than one source still fail closed before any source task is spawned until the dedicated simultaneous supervisor tranche.
/// Every validated source in the bounded runtime-resource aggregate is started concurrently under one private source supervisor. Any configured source
/// failure remains terminal for the Worker because this release does not infer equivalent coverage or failover between source families.
pub fn start_with_runtime_resources(
settings: crate::RawTransactionIngestSettings,
store: std::sync::Arc<ksp_store_lib::Store>,
@@ -117,9 +117,6 @@ impl crate::RawTransactionIngestWorker {
if let std::result::Result::Err(error) = runtime_resources.validate_network(settings.network()) {
return std::result::Result::Err(error);
}
if let std::result::Result::Err(error) = runtime_resources.validate_single_source_activation() {
return std::result::Result::Err(error);
}
let source_settings = settings.clone();
let (processing_frontier_sender, processing_frontier_receiver) =
tokio::sync::watch::channel(crate::RawTransactionIngestProcessingFrontierProjection::empty());
@@ -131,7 +128,7 @@ impl crate::RawTransactionIngestWorker {
std::option::Option::Some(processing_frontier_receiver),
move |children, stop_receiver, admission_sender| {
let _abort_handle = children.spawn(async move {
return runtime_resources.run_single_live_source(source_settings, stop_receiver, admission_sender, processing_frontier_sender).await;
return runtime_resources.run_live_sources(source_settings, stop_receiver, admission_sender, processing_frontier_sender).await;
});
},
);

293
crates/ksp-worker-raw-transaction-ingest-lib/src/runtime_resources.rs
View File

@@ -1,5 +1,5 @@
// file: crates/ksp-worker-raw-transaction-ingest-lib/src/runtime_resources.rs
// version: 20
// version: 21
use sha2::Digest; // rust-rules: trait-import
@@ -51,6 +51,19 @@ enum RawTransactionIngestLiveSource {
Yellowstone(crate::RawTransactionIngestYellowstoneSource),
}
struct RawTransactionIngestSourceInventory {
source_keys: std::vec::Vec<[u8; 32]>,
source_projections: std::vec::Vec<crate::RawTransactionIngestProcessingFrontierProjection>,
}
#[derive(Clone)]
struct RawTransactionIngestSourceInventoryPublisher {
aggregate_sender: tokio::sync::watch::Sender<crate::RawTransactionIngestProcessingFrontierProjection>,
entry_index: usize,
inventory: std::sync::Arc<std::sync::Mutex<RawTransactionIngestSourceInventory>>,
source_key: [u8; 32],
}
impl RawTransactionIngestLiveSource {
fn network(&self) -> &ksp_store_lib::RawNetworkId {
return match self {
@@ -77,15 +90,148 @@ impl RawTransactionIngestLiveSource {
settings: crate::RawTransactionIngestSettings,
stop_receiver: tokio::sync::watch::Receiver<bool>,
admission_sender: tokio::sync::mpsc::Sender<crate::RawTransactionIngress>,
processing_frontier_sender: tokio::sync::watch::Sender<crate::RawTransactionIngestProcessingFrontierProjection>,
inventory_publisher: RawTransactionIngestSourceInventoryPublisher,
) -> ksp_core_lib::Result<()> {
return match self {
Self::HeliusTransaction(source) => source.run(settings, stop_receiver, admission_sender, processing_frontier_sender).await,
Self::HttpBlockPolling(source) => source.run(settings, stop_receiver, admission_sender, processing_frontier_sender).await,
Self::StandardBlock(source) => source.run(settings, stop_receiver, admission_sender, processing_frontier_sender).await,
Self::StandardLogs(source) => source.run(settings, stop_receiver, admission_sender, processing_frontier_sender).await,
Self::Yellowstone(source) => source.run(settings, stop_receiver, admission_sender, processing_frontier_sender).await,
let (source_frontier_sender, mut source_frontier_receiver) =
tokio::sync::watch::channel(crate::RawTransactionIngestProcessingFrontierProjection::empty());
let mut source_future = std::boxed::Box::pin(async move {
return match self {
Self::HeliusTransaction(source) => source.run(settings, stop_receiver, admission_sender, source_frontier_sender).await,
Self::HttpBlockPolling(source) => source.run(settings, stop_receiver, admission_sender, source_frontier_sender).await,
Self::StandardBlock(source) => source.run(settings, stop_receiver, admission_sender, source_frontier_sender).await,
Self::StandardLogs(source) => source.run(settings, stop_receiver, admission_sender, source_frontier_sender).await,
Self::Yellowstone(source) => source.run(settings, stop_receiver, admission_sender, source_frontier_sender).await,
};
});
loop {
tokio::select! {
biased;
result = &mut source_future => {
let latest = *source_frontier_receiver.borrow_and_update();
let terminal_state = if result.is_ok() {
crate::RawTransactionIngestSourceState::Closed
} else {
crate::RawTransactionIngestSourceState::Failed
};
inventory_publisher.publish(source_projection_with_state(latest, terminal_state));
return result;
}
changed = source_frontier_receiver.changed() => {
if changed.is_err() {
return std::result::Result::Err(crate::runtime_error("source.frontier_channel_closed"));
}
inventory_publisher.publish(*source_frontier_receiver.borrow_and_update());
}
}
}
}
}
impl RawTransactionIngestSourceInventory {
fn aggregate(&self) -> crate::RawTransactionIngestProcessingFrontierProjection {
let mut hydration_pending = 0_usize;
let mut oldest_pending_slot = std::option::Option::None;
let mut processing_frontier_slot = std::option::Option::None;
let mut all_frontiers_present = !self.source_projections.is_empty();
let mut source_continuity_gap_total = 0_u64;
let mut source_reconnect_total = 0_u64;
let mut source_replay_attempt_total = 0_u64;
let mut any_active = false;
let mut any_closing = false;
let mut any_failed = false;
let mut any_reconnecting = false;
let mut all_closed = !self.source_projections.is_empty();
for projection in &self.source_projections {
hydration_pending = hydration_pending.saturating_add(projection.hydration_pending());
oldest_pending_slot = minimum_optional_slot(oldest_pending_slot, projection.oldest_pending_slot());
match projection.processing_frontier_slot() {
std::option::Option::Some(slot) => {
processing_frontier_slot = minimum_optional_slot(processing_frontier_slot, std::option::Option::Some(slot));
},
std::option::Option::None => {
all_frontiers_present = false;
},
}
source_continuity_gap_total = source_continuity_gap_total.saturating_add(projection.source_continuity_gap_total());
source_reconnect_total = source_reconnect_total.saturating_add(projection.source_reconnect_total());
source_replay_attempt_total = source_replay_attempt_total.saturating_add(projection.source_replay_attempt_total());
match projection.source_state() {
std::option::Option::Some(crate::RawTransactionIngestSourceState::Active) => {
any_active = true;
all_closed = false;
},
std::option::Option::Some(crate::RawTransactionIngestSourceState::Closing) => {
any_closing = true;
all_closed = false;
},
std::option::Option::Some(crate::RawTransactionIngestSourceState::Closed) => {},
std::option::Option::Some(crate::RawTransactionIngestSourceState::Failed) => {
any_failed = true;
all_closed = false;
},
std::option::Option::Some(crate::RawTransactionIngestSourceState::Reconnecting) => {
any_reconnecting = true;
all_closed = false;
},
std::option::Option::None => {
all_closed = false;
},
}
}
if !all_frontiers_present {
processing_frontier_slot = std::option::Option::None;
}
let source_state = if any_failed {
std::option::Option::Some(crate::RawTransactionIngestSourceState::Failed)
} else if any_reconnecting {
std::option::Option::Some(crate::RawTransactionIngestSourceState::Reconnecting)
} else if any_closing {
std::option::Option::Some(crate::RawTransactionIngestSourceState::Closing)
} else if any_active {
std::option::Option::Some(crate::RawTransactionIngestSourceState::Active)
} else if all_closed {
std::option::Option::Some(crate::RawTransactionIngestSourceState::Closed)
} else {
std::option::Option::None
};
return crate::RawTransactionIngestProcessingFrontierProjection::new(hydration_pending, processing_frontier_slot, oldest_pending_slot)
.with_source_continuity(source_state, source_reconnect_total, source_replay_attempt_total, source_continuity_gap_total);
}
fn new(source_keys: std::vec::Vec<[u8; 32]>) -> Self {
let source_projections = source_keys.iter().map(|_source_key| return crate::RawTransactionIngestProcessingFrontierProjection::empty()).collect();
return Self { source_keys, source_projections };
}
fn update(&mut self, entry_index: usize, source_key: [u8; 32], projection: crate::RawTransactionIngestProcessingFrontierProjection) {
let expected_key = match self.source_keys.get(entry_index) {
std::option::Option::Some(value) => value,
std::option::Option::None => return,
};
if expected_key != &source_key {
return;
}
let current = match self.source_projections.get_mut(entry_index) {
std::option::Option::Some(value) => value,
std::option::Option::None => return,
};
*current = projection;
return;
}
}
impl RawTransactionIngestSourceInventoryPublisher {
fn publish(&self, projection: crate::RawTransactionIngestProcessingFrontierProjection) {
let aggregate = {
let mut inventory = match self.inventory.lock() {
std::result::Result::Ok(value) => value,
std::result::Result::Err(poisoned) => poisoned.into_inner(),
};
inventory.update(self.entry_index, self.source_key, projection);
inventory.aggregate()
};
self.aggregate_sender.send_replace(aggregate);
return;
}
}
@@ -1610,9 +1756,8 @@ impl std::fmt::Debug for crate::RawTransactionIngestYellowstoneSource {
/// Caller-composed runtime resources accepted by the continuous RAW transaction ingest Worker.
///
/// The aggregate owns a bounded collection of capability-specific live-source contracts. `pre.006` supports Yellowstone, standard Solana logs/block, Helius
/// transaction and HTTP live block polling sources while deliberately keeping simultaneous multi-source supervision gated until the dedicated supervisor
/// tranche.
/// The aggregate owns a bounded collection of capability-specific live-source contracts. All validated sources are started together by one private supervisor;
/// source identities and per-source lifecycle remain private while the existing Worker snapshot receives only a conservative source-neutral aggregate.
pub struct RawTransactionIngestRuntimeResources {
sources: std::vec::Vec<RawTransactionIngestLiveSource>,
}
@@ -1776,34 +1921,37 @@ impl crate::RawTransactionIngestRuntimeResources {
return std::result::Result::Ok(());
}
/// Validates that the current supervisor tranche can activate the aggregate without silently dropping configured sources.
pub(crate) fn validate_single_source_activation(&self) -> ksp_core_lib::Result<()> {
if self.sources.is_empty() {
return std::result::Result::Err(crate::runtime_error("runtime_resources.source_collection_empty"));
}
if self.sources.len() != 1 {
return std::result::Result::Err(crate::runtime_error("runtime_resources.multi_source_activation_pending"));
}
return std::result::Result::Ok(());
}
/// Runs the already validated single live source through the common Worker source-task contract.
pub(crate) async fn run_single_live_source(
/// Runs every validated source concurrently under one private bounded source supervisor.
pub(crate) async fn run_live_sources(
self,
settings: crate::RawTransactionIngestSettings,
stop_receiver: tokio::sync::watch::Receiver<bool>,
admission_sender: tokio::sync::mpsc::Sender<crate::RawTransactionIngress>,
processing_frontier_sender: tokio::sync::watch::Sender<crate::RawTransactionIngestProcessingFrontierProjection>,
) -> ksp_core_lib::Result<()> {
let mut sources = self.sources.into_iter();
let source = match sources.next() {
std::option::Option::Some(value) => value,
std::option::Option::None => return std::result::Result::Err(crate::runtime_error("runtime_resources.source_collection_empty")),
};
if sources.next().is_some() {
return std::result::Result::Err(crate::runtime_error("runtime_resources.multi_source_activation_pending"));
if self.sources.is_empty() || self.sources.len() > crate::MAX_RAW_TRANSACTION_INGEST_LIVE_SOURCES {
return std::result::Result::Err(crate::runtime_error("runtime_resources.source_collection_invalid"));
}
return source.run(settings, stop_receiver, admission_sender, processing_frontier_sender).await;
let source_keys = self.sources.iter().map(RawTransactionIngestLiveSource::source_key).collect::<std::vec::Vec<_>>();
let inventory = std::sync::Arc::new(std::sync::Mutex::new(RawTransactionIngestSourceInventory::new(source_keys)));
let (source_stop_sender, source_stop_receiver) = tokio::sync::watch::channel(false);
let mut children = tokio::task::JoinSet::new();
for (entry_index, source) in self.sources.into_iter().enumerate() {
let publisher = RawTransactionIngestSourceInventoryPublisher {
aggregate_sender: processing_frontier_sender.clone(),
entry_index,
inventory: std::sync::Arc::clone(&inventory),
source_key: source.source_key(),
};
let source_admission_sender = admission_sender.clone();
let source_settings = settings.clone();
let source_stop_receiver = source_stop_receiver.clone();
let _abort_handle = children.spawn(async move {
return source.run(source_settings, source_stop_receiver, source_admission_sender, publisher).await;
});
}
std::mem::drop(admission_sender);
return supervise_live_source_tasks(stop_receiver, source_stop_sender, children).await;
}
}
@@ -1828,6 +1976,87 @@ impl std::hash::Hasher for RawTransactionIngestSourceKeyHashWriter<'_> {
}
}
fn minimum_optional_slot(first: std::option::Option<u64>, second: std::option::Option<u64>) -> std::option::Option<u64> {
return match (first, second) {
(std::option::Option::Some(left), std::option::Option::Some(right)) => std::option::Option::Some(left.min(right)),
(std::option::Option::Some(value), std::option::Option::None) | (std::option::Option::None, std::option::Option::Some(value)) => {
std::option::Option::Some(value)
},
(std::option::Option::None, std::option::Option::None) => std::option::Option::None,
};
}
fn source_projection_with_state(
projection: crate::RawTransactionIngestProcessingFrontierProjection,
state: crate::RawTransactionIngestSourceState,
) -> crate::RawTransactionIngestProcessingFrontierProjection {
return crate::RawTransactionIngestProcessingFrontierProjection::new(
projection.hydration_pending(),
projection.processing_frontier_slot(),
projection.oldest_pending_slot(),
)
.with_source_continuity(
std::option::Option::Some(state),
projection.source_reconnect_total(),
projection.source_replay_attempt_total(),
projection.source_continuity_gap_total(),
);
}
async fn supervise_live_source_tasks(
mut stop_receiver: tokio::sync::watch::Receiver<bool>,
source_stop_sender: tokio::sync::watch::Sender<bool>,
mut children: tokio::task::JoinSet<ksp_core_lib::Result<()>>,
) -> ksp_core_lib::Result<()> {
loop {
if *stop_receiver.borrow() {
source_stop_sender.send_replace(true);
return drain_live_source_tasks(&mut children, std::option::Option::None).await;
}
let joined = tokio::select! {
biased;
changed = stop_receiver.changed() => {
if changed.is_err() || *stop_receiver.borrow() {
source_stop_sender.send_replace(true);
return drain_live_source_tasks(&mut children, std::option::Option::None).await;
}
continue;
}
value = children.join_next(), if !children.is_empty() => value,
};
let first_fault = match joined {
std::option::Option::Some(std::result::Result::Ok(std::result::Result::Ok(()))) => {
std::option::Option::Some(crate::runtime_error("source.configured_source_closed"))
},
std::option::Option::Some(std::result::Result::Ok(std::result::Result::Err(error))) => std::option::Option::Some(error),
std::option::Option::Some(std::result::Result::Err(_)) => std::option::Option::Some(crate::runtime_error("source.task_join_failed")),
std::option::Option::None => std::option::Option::Some(crate::runtime_error("source.task_set_empty")),
};
source_stop_sender.send_replace(true);
return drain_live_source_tasks(&mut children, first_fault).await;
}
}
async fn drain_live_source_tasks(
children: &mut tokio::task::JoinSet<ksp_core_lib::Result<()>>,
mut first_fault: std::option::Option<ksp_core_lib::Error>,
) -> ksp_core_lib::Result<()> {
while let std::option::Option::Some(joined) = children.join_next().await {
if first_fault.is_some() {
continue;
}
first_fault = match joined {
std::result::Result::Ok(std::result::Result::Ok(())) => std::option::Option::None,
std::result::Result::Ok(std::result::Result::Err(error)) => std::option::Option::Some(error),
std::result::Result::Err(_) => std::option::Option::Some(crate::runtime_error("source.task_join_failed")),
};
}
return match first_fault {
std::option::Option::Some(error) => std::result::Result::Err(error),
std::option::Option::None => std::result::Result::Ok(()),
};
}
fn http_block_polling_live_source_key(
network: &ksp_store_lib::RawNetworkId,
polling_role: &ksp_onchain_transport_lib::HttpRoleName,