v0.3.13-pre.007
This commit is contained in:
@@ -1,5 +1,5 @@
|
||||
// file: crates/ksp-worker-raw-transaction-ingest-lib/src/runtime_resources.rs
|
||||
// version: 20
|
||||
// version: 21
|
||||
|
||||
use sha2::Digest; // rust-rules: trait-import
|
||||
|
||||
@@ -51,6 +51,19 @@ enum RawTransactionIngestLiveSource {
|
||||
Yellowstone(crate::RawTransactionIngestYellowstoneSource),
|
||||
}
|
||||
|
||||
struct RawTransactionIngestSourceInventory {
|
||||
source_keys: std::vec::Vec<[u8; 32]>,
|
||||
source_projections: std::vec::Vec<crate::RawTransactionIngestProcessingFrontierProjection>,
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
struct RawTransactionIngestSourceInventoryPublisher {
|
||||
aggregate_sender: tokio::sync::watch::Sender<crate::RawTransactionIngestProcessingFrontierProjection>,
|
||||
entry_index: usize,
|
||||
inventory: std::sync::Arc<std::sync::Mutex<RawTransactionIngestSourceInventory>>,
|
||||
source_key: [u8; 32],
|
||||
}
|
||||
|
||||
impl RawTransactionIngestLiveSource {
|
||||
fn network(&self) -> &ksp_store_lib::RawNetworkId {
|
||||
return match self {
|
||||
@@ -77,15 +90,148 @@ impl RawTransactionIngestLiveSource {
|
||||
settings: crate::RawTransactionIngestSettings,
|
||||
stop_receiver: tokio::sync::watch::Receiver<bool>,
|
||||
admission_sender: tokio::sync::mpsc::Sender<crate::RawTransactionIngress>,
|
||||
processing_frontier_sender: tokio::sync::watch::Sender<crate::RawTransactionIngestProcessingFrontierProjection>,
|
||||
inventory_publisher: RawTransactionIngestSourceInventoryPublisher,
|
||||
) -> ksp_core_lib::Result<()> {
|
||||
return match self {
|
||||
Self::HeliusTransaction(source) => source.run(settings, stop_receiver, admission_sender, processing_frontier_sender).await,
|
||||
Self::HttpBlockPolling(source) => source.run(settings, stop_receiver, admission_sender, processing_frontier_sender).await,
|
||||
Self::StandardBlock(source) => source.run(settings, stop_receiver, admission_sender, processing_frontier_sender).await,
|
||||
Self::StandardLogs(source) => source.run(settings, stop_receiver, admission_sender, processing_frontier_sender).await,
|
||||
Self::Yellowstone(source) => source.run(settings, stop_receiver, admission_sender, processing_frontier_sender).await,
|
||||
let (source_frontier_sender, mut source_frontier_receiver) =
|
||||
tokio::sync::watch::channel(crate::RawTransactionIngestProcessingFrontierProjection::empty());
|
||||
let mut source_future = std::boxed::Box::pin(async move {
|
||||
return match self {
|
||||
Self::HeliusTransaction(source) => source.run(settings, stop_receiver, admission_sender, source_frontier_sender).await,
|
||||
Self::HttpBlockPolling(source) => source.run(settings, stop_receiver, admission_sender, source_frontier_sender).await,
|
||||
Self::StandardBlock(source) => source.run(settings, stop_receiver, admission_sender, source_frontier_sender).await,
|
||||
Self::StandardLogs(source) => source.run(settings, stop_receiver, admission_sender, source_frontier_sender).await,
|
||||
Self::Yellowstone(source) => source.run(settings, stop_receiver, admission_sender, source_frontier_sender).await,
|
||||
};
|
||||
});
|
||||
loop {
|
||||
tokio::select! {
|
||||
biased;
|
||||
result = &mut source_future => {
|
||||
let latest = *source_frontier_receiver.borrow_and_update();
|
||||
let terminal_state = if result.is_ok() {
|
||||
crate::RawTransactionIngestSourceState::Closed
|
||||
} else {
|
||||
crate::RawTransactionIngestSourceState::Failed
|
||||
};
|
||||
inventory_publisher.publish(source_projection_with_state(latest, terminal_state));
|
||||
return result;
|
||||
}
|
||||
changed = source_frontier_receiver.changed() => {
|
||||
if changed.is_err() {
|
||||
return std::result::Result::Err(crate::runtime_error("source.frontier_channel_closed"));
|
||||
}
|
||||
inventory_publisher.publish(*source_frontier_receiver.borrow_and_update());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl RawTransactionIngestSourceInventory {
|
||||
fn aggregate(&self) -> crate::RawTransactionIngestProcessingFrontierProjection {
|
||||
let mut hydration_pending = 0_usize;
|
||||
let mut oldest_pending_slot = std::option::Option::None;
|
||||
let mut processing_frontier_slot = std::option::Option::None;
|
||||
let mut all_frontiers_present = !self.source_projections.is_empty();
|
||||
let mut source_continuity_gap_total = 0_u64;
|
||||
let mut source_reconnect_total = 0_u64;
|
||||
let mut source_replay_attempt_total = 0_u64;
|
||||
let mut any_active = false;
|
||||
let mut any_closing = false;
|
||||
let mut any_failed = false;
|
||||
let mut any_reconnecting = false;
|
||||
let mut all_closed = !self.source_projections.is_empty();
|
||||
for projection in &self.source_projections {
|
||||
hydration_pending = hydration_pending.saturating_add(projection.hydration_pending());
|
||||
oldest_pending_slot = minimum_optional_slot(oldest_pending_slot, projection.oldest_pending_slot());
|
||||
match projection.processing_frontier_slot() {
|
||||
std::option::Option::Some(slot) => {
|
||||
processing_frontier_slot = minimum_optional_slot(processing_frontier_slot, std::option::Option::Some(slot));
|
||||
},
|
||||
std::option::Option::None => {
|
||||
all_frontiers_present = false;
|
||||
},
|
||||
}
|
||||
source_continuity_gap_total = source_continuity_gap_total.saturating_add(projection.source_continuity_gap_total());
|
||||
source_reconnect_total = source_reconnect_total.saturating_add(projection.source_reconnect_total());
|
||||
source_replay_attempt_total = source_replay_attempt_total.saturating_add(projection.source_replay_attempt_total());
|
||||
match projection.source_state() {
|
||||
std::option::Option::Some(crate::RawTransactionIngestSourceState::Active) => {
|
||||
any_active = true;
|
||||
all_closed = false;
|
||||
},
|
||||
std::option::Option::Some(crate::RawTransactionIngestSourceState::Closing) => {
|
||||
any_closing = true;
|
||||
all_closed = false;
|
||||
},
|
||||
std::option::Option::Some(crate::RawTransactionIngestSourceState::Closed) => {},
|
||||
std::option::Option::Some(crate::RawTransactionIngestSourceState::Failed) => {
|
||||
any_failed = true;
|
||||
all_closed = false;
|
||||
},
|
||||
std::option::Option::Some(crate::RawTransactionIngestSourceState::Reconnecting) => {
|
||||
any_reconnecting = true;
|
||||
all_closed = false;
|
||||
},
|
||||
std::option::Option::None => {
|
||||
all_closed = false;
|
||||
},
|
||||
}
|
||||
}
|
||||
if !all_frontiers_present {
|
||||
processing_frontier_slot = std::option::Option::None;
|
||||
}
|
||||
let source_state = if any_failed {
|
||||
std::option::Option::Some(crate::RawTransactionIngestSourceState::Failed)
|
||||
} else if any_reconnecting {
|
||||
std::option::Option::Some(crate::RawTransactionIngestSourceState::Reconnecting)
|
||||
} else if any_closing {
|
||||
std::option::Option::Some(crate::RawTransactionIngestSourceState::Closing)
|
||||
} else if any_active {
|
||||
std::option::Option::Some(crate::RawTransactionIngestSourceState::Active)
|
||||
} else if all_closed {
|
||||
std::option::Option::Some(crate::RawTransactionIngestSourceState::Closed)
|
||||
} else {
|
||||
std::option::Option::None
|
||||
};
|
||||
return crate::RawTransactionIngestProcessingFrontierProjection::new(hydration_pending, processing_frontier_slot, oldest_pending_slot)
|
||||
.with_source_continuity(source_state, source_reconnect_total, source_replay_attempt_total, source_continuity_gap_total);
|
||||
}
|
||||
|
||||
fn new(source_keys: std::vec::Vec<[u8; 32]>) -> Self {
|
||||
let source_projections = source_keys.iter().map(|_source_key| return crate::RawTransactionIngestProcessingFrontierProjection::empty()).collect();
|
||||
return Self { source_keys, source_projections };
|
||||
}
|
||||
|
||||
fn update(&mut self, entry_index: usize, source_key: [u8; 32], projection: crate::RawTransactionIngestProcessingFrontierProjection) {
|
||||
let expected_key = match self.source_keys.get(entry_index) {
|
||||
std::option::Option::Some(value) => value,
|
||||
std::option::Option::None => return,
|
||||
};
|
||||
if expected_key != &source_key {
|
||||
return;
|
||||
}
|
||||
let current = match self.source_projections.get_mut(entry_index) {
|
||||
std::option::Option::Some(value) => value,
|
||||
std::option::Option::None => return,
|
||||
};
|
||||
*current = projection;
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
impl RawTransactionIngestSourceInventoryPublisher {
|
||||
fn publish(&self, projection: crate::RawTransactionIngestProcessingFrontierProjection) {
|
||||
let aggregate = {
|
||||
let mut inventory = match self.inventory.lock() {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(poisoned) => poisoned.into_inner(),
|
||||
};
|
||||
inventory.update(self.entry_index, self.source_key, projection);
|
||||
inventory.aggregate()
|
||||
};
|
||||
self.aggregate_sender.send_replace(aggregate);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1610,9 +1756,8 @@ impl std::fmt::Debug for crate::RawTransactionIngestYellowstoneSource {
|
||||
|
||||
/// Caller-composed runtime resources accepted by the continuous RAW transaction ingest Worker.
|
||||
///
|
||||
/// The aggregate owns a bounded collection of capability-specific live-source contracts. `pre.006` supports Yellowstone, standard Solana logs/block, Helius
|
||||
/// transaction and HTTP live block polling sources while deliberately keeping simultaneous multi-source supervision gated until the dedicated supervisor
|
||||
/// tranche.
|
||||
/// The aggregate owns a bounded collection of capability-specific live-source contracts. All validated sources are started together by one private supervisor;
|
||||
/// source identities and per-source lifecycle remain private while the existing Worker snapshot receives only a conservative source-neutral aggregate.
|
||||
pub struct RawTransactionIngestRuntimeResources {
|
||||
sources: std::vec::Vec<RawTransactionIngestLiveSource>,
|
||||
}
|
||||
@@ -1776,34 +1921,37 @@ impl crate::RawTransactionIngestRuntimeResources {
|
||||
return std::result::Result::Ok(());
|
||||
}
|
||||
|
||||
/// Validates that the current supervisor tranche can activate the aggregate without silently dropping configured sources.
|
||||
pub(crate) fn validate_single_source_activation(&self) -> ksp_core_lib::Result<()> {
|
||||
if self.sources.is_empty() {
|
||||
return std::result::Result::Err(crate::runtime_error("runtime_resources.source_collection_empty"));
|
||||
}
|
||||
if self.sources.len() != 1 {
|
||||
return std::result::Result::Err(crate::runtime_error("runtime_resources.multi_source_activation_pending"));
|
||||
}
|
||||
return std::result::Result::Ok(());
|
||||
}
|
||||
|
||||
/// Runs the already validated single live source through the common Worker source-task contract.
|
||||
pub(crate) async fn run_single_live_source(
|
||||
/// Runs every validated source concurrently under one private bounded source supervisor.
|
||||
pub(crate) async fn run_live_sources(
|
||||
self,
|
||||
settings: crate::RawTransactionIngestSettings,
|
||||
stop_receiver: tokio::sync::watch::Receiver<bool>,
|
||||
admission_sender: tokio::sync::mpsc::Sender<crate::RawTransactionIngress>,
|
||||
processing_frontier_sender: tokio::sync::watch::Sender<crate::RawTransactionIngestProcessingFrontierProjection>,
|
||||
) -> ksp_core_lib::Result<()> {
|
||||
let mut sources = self.sources.into_iter();
|
||||
let source = match sources.next() {
|
||||
std::option::Option::Some(value) => value,
|
||||
std::option::Option::None => return std::result::Result::Err(crate::runtime_error("runtime_resources.source_collection_empty")),
|
||||
};
|
||||
if sources.next().is_some() {
|
||||
return std::result::Result::Err(crate::runtime_error("runtime_resources.multi_source_activation_pending"));
|
||||
if self.sources.is_empty() || self.sources.len() > crate::MAX_RAW_TRANSACTION_INGEST_LIVE_SOURCES {
|
||||
return std::result::Result::Err(crate::runtime_error("runtime_resources.source_collection_invalid"));
|
||||
}
|
||||
return source.run(settings, stop_receiver, admission_sender, processing_frontier_sender).await;
|
||||
let source_keys = self.sources.iter().map(RawTransactionIngestLiveSource::source_key).collect::<std::vec::Vec<_>>();
|
||||
let inventory = std::sync::Arc::new(std::sync::Mutex::new(RawTransactionIngestSourceInventory::new(source_keys)));
|
||||
let (source_stop_sender, source_stop_receiver) = tokio::sync::watch::channel(false);
|
||||
let mut children = tokio::task::JoinSet::new();
|
||||
for (entry_index, source) in self.sources.into_iter().enumerate() {
|
||||
let publisher = RawTransactionIngestSourceInventoryPublisher {
|
||||
aggregate_sender: processing_frontier_sender.clone(),
|
||||
entry_index,
|
||||
inventory: std::sync::Arc::clone(&inventory),
|
||||
source_key: source.source_key(),
|
||||
};
|
||||
let source_admission_sender = admission_sender.clone();
|
||||
let source_settings = settings.clone();
|
||||
let source_stop_receiver = source_stop_receiver.clone();
|
||||
let _abort_handle = children.spawn(async move {
|
||||
return source.run(source_settings, source_stop_receiver, source_admission_sender, publisher).await;
|
||||
});
|
||||
}
|
||||
std::mem::drop(admission_sender);
|
||||
return supervise_live_source_tasks(stop_receiver, source_stop_sender, children).await;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1828,6 +1976,87 @@ impl std::hash::Hasher for RawTransactionIngestSourceKeyHashWriter<'_> {
|
||||
}
|
||||
}
|
||||
|
||||
fn minimum_optional_slot(first: std::option::Option<u64>, second: std::option::Option<u64>) -> std::option::Option<u64> {
|
||||
return match (first, second) {
|
||||
(std::option::Option::Some(left), std::option::Option::Some(right)) => std::option::Option::Some(left.min(right)),
|
||||
(std::option::Option::Some(value), std::option::Option::None) | (std::option::Option::None, std::option::Option::Some(value)) => {
|
||||
std::option::Option::Some(value)
|
||||
},
|
||||
(std::option::Option::None, std::option::Option::None) => std::option::Option::None,
|
||||
};
|
||||
}
|
||||
|
||||
fn source_projection_with_state(
|
||||
projection: crate::RawTransactionIngestProcessingFrontierProjection,
|
||||
state: crate::RawTransactionIngestSourceState,
|
||||
) -> crate::RawTransactionIngestProcessingFrontierProjection {
|
||||
return crate::RawTransactionIngestProcessingFrontierProjection::new(
|
||||
projection.hydration_pending(),
|
||||
projection.processing_frontier_slot(),
|
||||
projection.oldest_pending_slot(),
|
||||
)
|
||||
.with_source_continuity(
|
||||
std::option::Option::Some(state),
|
||||
projection.source_reconnect_total(),
|
||||
projection.source_replay_attempt_total(),
|
||||
projection.source_continuity_gap_total(),
|
||||
);
|
||||
}
|
||||
|
||||
async fn supervise_live_source_tasks(
|
||||
mut stop_receiver: tokio::sync::watch::Receiver<bool>,
|
||||
source_stop_sender: tokio::sync::watch::Sender<bool>,
|
||||
mut children: tokio::task::JoinSet<ksp_core_lib::Result<()>>,
|
||||
) -> ksp_core_lib::Result<()> {
|
||||
loop {
|
||||
if *stop_receiver.borrow() {
|
||||
source_stop_sender.send_replace(true);
|
||||
return drain_live_source_tasks(&mut children, std::option::Option::None).await;
|
||||
}
|
||||
let joined = tokio::select! {
|
||||
biased;
|
||||
changed = stop_receiver.changed() => {
|
||||
if changed.is_err() || *stop_receiver.borrow() {
|
||||
source_stop_sender.send_replace(true);
|
||||
return drain_live_source_tasks(&mut children, std::option::Option::None).await;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
value = children.join_next(), if !children.is_empty() => value,
|
||||
};
|
||||
let first_fault = match joined {
|
||||
std::option::Option::Some(std::result::Result::Ok(std::result::Result::Ok(()))) => {
|
||||
std::option::Option::Some(crate::runtime_error("source.configured_source_closed"))
|
||||
},
|
||||
std::option::Option::Some(std::result::Result::Ok(std::result::Result::Err(error))) => std::option::Option::Some(error),
|
||||
std::option::Option::Some(std::result::Result::Err(_)) => std::option::Option::Some(crate::runtime_error("source.task_join_failed")),
|
||||
std::option::Option::None => std::option::Option::Some(crate::runtime_error("source.task_set_empty")),
|
||||
};
|
||||
source_stop_sender.send_replace(true);
|
||||
return drain_live_source_tasks(&mut children, first_fault).await;
|
||||
}
|
||||
}
|
||||
|
||||
async fn drain_live_source_tasks(
|
||||
children: &mut tokio::task::JoinSet<ksp_core_lib::Result<()>>,
|
||||
mut first_fault: std::option::Option<ksp_core_lib::Error>,
|
||||
) -> ksp_core_lib::Result<()> {
|
||||
while let std::option::Option::Some(joined) = children.join_next().await {
|
||||
if first_fault.is_some() {
|
||||
continue;
|
||||
}
|
||||
first_fault = match joined {
|
||||
std::result::Result::Ok(std::result::Result::Ok(())) => std::option::Option::None,
|
||||
std::result::Result::Ok(std::result::Result::Err(error)) => std::option::Option::Some(error),
|
||||
std::result::Result::Err(_) => std::option::Option::Some(crate::runtime_error("source.task_join_failed")),
|
||||
};
|
||||
}
|
||||
return match first_fault {
|
||||
std::option::Option::Some(error) => std::result::Result::Err(error),
|
||||
std::option::Option::None => std::result::Result::Ok(()),
|
||||
};
|
||||
}
|
||||
|
||||
fn http_block_polling_live_source_key(
|
||||
network: &ksp_store_lib::RawNetworkId,
|
||||
polling_role: &ksp_onchain_transport_lib::HttpRoleName,
|
||||
|
||||
Reference in New Issue
Block a user