Sasedev/khadhroony-solana-project
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

v0.3.13-pre.008

Browse Source
This commit is contained in:
SinuS von SifriduS
2026-09-10 17:52:35 +02:00
parent 8bb530e9b4
commit 1f2720463b
18 changed files with 899 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files

141
crates/ksp-worker-raw-transaction-ingest-lib/src/runtime_resources.rs
View File

@@ -1,5 +1,5 @@
// file: crates/ksp-worker-raw-transaction-ingest-lib/src/runtime_resources.rs
// version: 21
// version: 22
use sha2::Digest; // rust-rules: trait-import
@@ -90,17 +90,20 @@ impl RawTransactionIngestLiveSource {
settings: crate::RawTransactionIngestSettings,
stop_receiver: tokio::sync::watch::Receiver<bool>,
admission_sender: tokio::sync::mpsc::Sender<crate::RawTransactionIngress>,
global_hydration_registry: std::sync::Arc<RawTransactionIngestGlobalHydrationRegistry>,
inventory_publisher: RawTransactionIngestSourceInventoryPublisher,
) -> ksp_core_lib::Result<()> {
let (source_frontier_sender, mut source_frontier_receiver) =
tokio::sync::watch::channel(crate::RawTransactionIngestProcessingFrontierProjection::empty());
let mut source_future = std::boxed::Box::pin(async move {
return match self {
Self::HeliusTransaction(source) => source.run(settings, stop_receiver, admission_sender, source_frontier_sender).await,
Self::HeliusTransaction(source) => {
source.run(settings, stop_receiver, admission_sender, source_frontier_sender, global_hydration_registry).await
},
Self::HttpBlockPolling(source) => source.run(settings, stop_receiver, admission_sender, source_frontier_sender).await,
Self::StandardBlock(source) => source.run(settings, stop_receiver, admission_sender, source_frontier_sender).await,
Self::StandardLogs(source) => source.run(settings, stop_receiver, admission_sender, source_frontier_sender).await,
Self::Yellowstone(source) => source.run(settings, stop_receiver, admission_sender, source_frontier_sender).await,
Self::StandardLogs(source) => source.run(settings, stop_receiver, admission_sender, source_frontier_sender, global_hydration_registry).await,
Self::Yellowstone(source) => source.run(settings, stop_receiver, admission_sender, source_frontier_sender, global_hydration_registry).await,
};
});
loop {
@@ -624,6 +627,7 @@ impl crate::RawTransactionIngestYellowstoneSource {
mut stop_receiver: tokio::sync::watch::Receiver<bool>,
admission_sender: tokio::sync::mpsc::Sender<crate::RawTransactionIngress>,
processing_frontier_sender: tokio::sync::watch::Sender<crate::RawTransactionIngestProcessingFrontierProjection>,
global_hydration_registry: std::sync::Arc<RawTransactionIngestGlobalHydrationRegistry>,
) -> ksp_core_lib::Result<()> {
let opened = tokio::select! {
biased;
@@ -637,7 +641,7 @@ impl crate::RawTransactionIngestYellowstoneSource {
std::result::Result::Err(error) => return std::result::Result::Err(source_transport_error(error.code())),
};
let hydration = self.hydration_context();
let mut coordinator = RawTransactionIngestHydrationCoordinator::new(&settings);
let mut coordinator = RawTransactionIngestHydrationCoordinator::with_global_registry(&settings, global_hydration_registry);
let mut processing_frontier = RawTransactionIngestProcessingFrontierReporter::new(processing_frontier_sender);
let snapshot_source = session.snapshot_source();
if let std::result::Result::Err(error) = processing_frontier.observe_session_snapshot(snapshot_source.current()) {
@@ -860,6 +864,7 @@ impl crate::RawTransactionIngestHeliusTransactionSource {
mut stop_receiver: tokio::sync::watch::Receiver<bool>,
admission_sender: tokio::sync::mpsc::Sender<crate::RawTransactionIngress>,
processing_frontier_sender: tokio::sync::watch::Sender<crate::RawTransactionIngestProcessingFrontierProjection>,
global_hydration_registry: std::sync::Arc<RawTransactionIngestGlobalHydrationRegistry>,
) -> ksp_core_lib::Result<()> {
let connected = tokio::select! {
biased;
@@ -889,7 +894,7 @@ impl crate::RawTransactionIngestHeliusTransactionSource {
},
};
let hydration = self.hydration_context();
let mut coordinator = RawTransactionIngestHydrationCoordinator::new(&settings);
let mut coordinator = RawTransactionIngestHydrationCoordinator::with_global_registry(&settings, global_hydration_registry);
let mut processing_frontier = RawTransactionIngestProcessingFrontierReporter::new(processing_frontier_sender);
processing_frontier.set_source_state(crate::RawTransactionIngestSourceState::Active);
let mut fault = std::option::Option::None;
@@ -1578,6 +1583,7 @@ impl crate::RawTransactionIngestStandardLogsSource {
mut stop_receiver: tokio::sync::watch::Receiver<bool>,
admission_sender: tokio::sync::mpsc::Sender<crate::RawTransactionIngress>,
processing_frontier_sender: tokio::sync::watch::Sender<crate::RawTransactionIngestProcessingFrontierProjection>,
global_hydration_registry: std::sync::Arc<RawTransactionIngestGlobalHydrationRegistry>,
) -> ksp_core_lib::Result<()> {
let connected = tokio::select! {
biased;
@@ -1607,7 +1613,7 @@ impl crate::RawTransactionIngestStandardLogsSource {
},
};
let hydration = self.hydration_context();
let mut coordinator = RawTransactionIngestHydrationCoordinator::new(&settings);
let mut coordinator = RawTransactionIngestHydrationCoordinator::with_global_registry(&settings, global_hydration_registry);
let mut processing_frontier = RawTransactionIngestProcessingFrontierReporter::new(processing_frontier_sender);
processing_frontier.set_source_state(crate::RawTransactionIngestSourceState::Active);
let mut fault = std::option::Option::None;
@@ -1934,6 +1940,7 @@ impl crate::RawTransactionIngestRuntimeResources {
}
let source_keys = self.sources.iter().map(RawTransactionIngestLiveSource::source_key).collect::<std::vec::Vec<_>>();
let inventory = std::sync::Arc::new(std::sync::Mutex::new(RawTransactionIngestSourceInventory::new(source_keys)));
let global_hydration_registry = std::sync::Arc::new(RawTransactionIngestGlobalHydrationRegistry::new(&settings));
let (source_stop_sender, source_stop_receiver) = tokio::sync::watch::channel(false);
let mut children = tokio::task::JoinSet::new();
for (entry_index, source) in self.sources.into_iter().enumerate() {
@@ -1946,8 +1953,9 @@ impl crate::RawTransactionIngestRuntimeResources {
let source_admission_sender = admission_sender.clone();
let source_settings = settings.clone();
let source_stop_receiver = source_stop_receiver.clone();
let source_global_hydration_registry = std::sync::Arc::clone(&global_hydration_registry);
let _abort_handle = children.spawn(async move {
return source.run(source_settings, source_stop_receiver, source_admission_sender, publisher).await;
return source.run(source_settings, source_stop_receiver, source_admission_sender, source_global_hydration_registry, publisher).await;
});
}
std::mem::drop(admission_sender);
@@ -3336,9 +3344,68 @@ struct RawTransactionIngestHydrationFetch {
observed: RawTransactionIngestObservedTransaction,
}
#[derive(Clone)]
enum RawTransactionIngestSharedHydrationResult {
Available(RawTransactionIngestObservedTransaction),
Failed(ksp_core_lib::ErrorCode),
}
struct RawTransactionIngestGlobalHydrationRegistry {
max_pending: usize,
pending: std::sync::Mutex<
std::collections::BTreeMap<
RawTransactionIngestHydrationKey,
tokio::sync::watch::Sender<std::option::Option<RawTransactionIngestSharedHydrationResult>>,
>,
>,
}
impl RawTransactionIngestGlobalHydrationRegistry {
fn new(settings: &crate::RawTransactionIngestSettings) -> Self {
return Self {
max_pending: settings.admission_queue_capacity(),
pending: std::sync::Mutex::new(std::collections::BTreeMap::new()),
};
}
fn subscribe_or_lead(
&self,
key: &RawTransactionIngestHydrationKey,
) -> ksp_core_lib::Result<(bool, tokio::sync::watch::Receiver<std::option::Option<RawTransactionIngestSharedHydrationResult>>)> {
let mut pending = match self.pending.lock() {
std::result::Result::Ok(value) => value,
std::result::Result::Err(poisoned) => poisoned.into_inner(),
};
if let std::option::Option::Some(sender) = pending.get(key) {
return std::result::Result::Ok((false, sender.subscribe()));
}
if pending.len() >= self.max_pending {
return std::result::Result::Err(crate::runtime_error("source.global_hydration_pending_saturated"));
}
let (sender, receiver) = tokio::sync::watch::channel(std::option::Option::None);
pending.insert(key.clone(), sender);
return std::result::Result::Ok((true, receiver));
}
fn publish_and_remove(&self, key: &RawTransactionIngestHydrationKey, result: RawTransactionIngestSharedHydrationResult) {
let sender = {
let mut pending = match self.pending.lock() {
std::result::Result::Ok(value) => value,
std::result::Result::Err(poisoned) => poisoned.into_inner(),
};
pending.remove(key)
};
if let std::option::Option::Some(sender) = sender {
sender.send_replace(std::option::Option::Some(result));
}
return;
}
}
type RawTransactionIngestHydrationTasks = tokio::task::JoinSet<ksp_core_lib::Result<RawTransactionIngestHydrationFetch>>;
struct RawTransactionIngestHydrationCoordinator {
global_registry: std::sync::Arc<RawTransactionIngestGlobalHydrationRegistry>,
max_in_flight: usize,
max_pending_signals: usize,
pending_signal_count: usize,
@@ -3348,7 +3415,15 @@ struct RawTransactionIngestHydrationCoordinator {
impl RawTransactionIngestHydrationCoordinator {
fn new(settings: &crate::RawTransactionIngestSettings) -> Self {
return Self::with_global_registry(settings, std::sync::Arc::new(RawTransactionIngestGlobalHydrationRegistry::new(settings)));
}
fn with_global_registry(
settings: &crate::RawTransactionIngestSettings,
global_registry: std::sync::Arc<RawTransactionIngestGlobalHydrationRegistry>,
) -> Self {
return Self {
global_registry,
max_in_flight: settings.persistence_concurrency(),
max_pending_signals: settings.admission_queue_capacity(),
pending_signal_count: 0,
@@ -3419,8 +3494,9 @@ impl RawTransactionIngestHydrationCoordinator {
let role = hydration.hydration_role.clone();
let expected_network = hydration.network.clone();
let task_key = key.clone();
let global_registry = std::sync::Arc::clone(&self.global_registry);
let _abort_handle = self.tasks.spawn(async move {
return fetch_hydration(pool, role, expected_network, task_key, commitment).await;
return fetch_hydration_shared(global_registry, pool, role, expected_network, task_key, commitment).await;
});
}
return std::result::Result::Ok(());
@@ -3538,6 +3614,53 @@ async fn fetch_hydration(
return std::result::Result::Ok(RawTransactionIngestHydrationFetch { key, observed });
}
async fn fetch_hydration_shared(
global_registry: std::sync::Arc<RawTransactionIngestGlobalHydrationRegistry>,
http_pool: ksp_onchain_transport_lib::HttpTransportPool,
hydration_role: ksp_onchain_transport_lib::HttpRoleName,
expected_network: ksp_store_lib::RawNetworkId,
key: RawTransactionIngestHydrationKey,
commitment: ksp_onchain_transport_lib::SolanaCommitment,
) -> ksp_core_lib::Result<RawTransactionIngestHydrationFetch> {
let (leader, mut receiver) = match global_registry.subscribe_or_lead(&key) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
if leader {
let fetched = fetch_hydration(http_pool, hydration_role, expected_network, key.clone(), commitment).await;
return match fetched {
std::result::Result::Ok(value) => {
global_registry.publish_and_remove(&key, RawTransactionIngestSharedHydrationResult::Available(value.observed.clone()));
std::result::Result::Ok(value)
},
std::result::Result::Err(error) => {
global_registry.publish_and_remove(&key, RawTransactionIngestSharedHydrationResult::Failed(error.code()));
std::result::Result::Err(error)
},
};
}
loop {
if let std::option::Option::Some(result) = receiver.borrow().clone() {
return shared_hydration_result(key, result);
}
if receiver.changed().await.is_err() {
return std::result::Result::Err(crate::runtime_error("source.global_hydration_channel_closed"));
}
}
}
fn shared_hydration_result(
key: RawTransactionIngestHydrationKey,
result: RawTransactionIngestSharedHydrationResult,
) -> ksp_core_lib::Result<RawTransactionIngestHydrationFetch> {
return match result {
RawTransactionIngestSharedHydrationResult::Available(observed) => std::result::Result::Ok(RawTransactionIngestHydrationFetch { key, observed }),
RawTransactionIngestSharedHydrationResult::Failed(code) => {
std::result::Result::Err(ksp_core_lib::Error::new(code, "RAW transaction ingest shared hydration failed"))
},
};
}
fn finalize_hydration(
hydration: &RawTransactionIngestHydrationContext,
settings: &crate::RawTransactionIngestSettings,