Sasedev/khadhroony-solana-project
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

v0.3.12-pre.007

Browse Source
This commit is contained in:
SinuS von SifriduS
2026-09-09 16:21:23 +02:00
parent 07b9b20eb5
commit ba69a5ad20
13 changed files with 1037 additions and 105 deletions
Download Patch File Download Diff File Expand all files Collapse all files

319
crates/ksp-worker-raw-transaction-ingest-lib/src/runtime_resources.rs
View File

@@ -1,5 +1,5 @@
// file: crates/ksp-worker-raw-transaction-ingest-lib/src/runtime_resources.rs
// version: 9
// version: 10
use sha2::Digest; // rust-rules: trait-import
@@ -188,7 +188,10 @@ trait RawTransactionIngestYellowstoneBlockView {
fn transaction_count(&self) -> usize;
fn transaction_identity(&self, position: usize) -> ksp_core_lib::Result<(ksp_onchain_transport_lib::YellowstoneTransactionSignature, u64)>;
fn transaction_identity(
&self,
position: usize,
) -> ksp_core_lib::Result<(ksp_onchain_transport_lib::YellowstoneTransactionSignature, u64)>;
}
impl RawTransactionIngestYellowstoneBlockView for ksp_onchain_transport_lib::YellowstoneBlockUpdate {
@@ -208,7 +211,10 @@ impl RawTransactionIngestYellowstoneBlockView for ksp_onchain_transport_lib::Yel
return ksp_onchain_transport_lib::YellowstoneBlockUpdate::transactions(self).len();
}
fn transaction_identity(&self, position: usize) -> ksp_core_lib::Result<(ksp_onchain_transport_lib::YellowstoneTransactionSignature, u64)> {
fn transaction_identity(
&self,
position: usize,
) -> ksp_core_lib::Result<(ksp_onchain_transport_lib::YellowstoneTransactionSignature, u64)> {
let transaction = match ksp_onchain_transport_lib::YellowstoneBlockUpdate::transactions(self).get(position) {
std::option::Option::Some(value) => value,
std::option::Option::None => return std::result::Result::Err(crate::runtime_error("source.block_transaction_missing")),
@@ -286,7 +292,9 @@ impl RawTransactionIngestYellowstoneContinuityView for ksp_onchain_transport_lib
impl std::convert::From<(&crate::RawTransactionIngestYellowstoneSource, &ksp_onchain_transport_lib::YellowstoneTransactionUpdate)>
for RawTransactionIngestSourceSignal
{
fn from(value: (&crate::RawTransactionIngestYellowstoneSource, &ksp_onchain_transport_lib::YellowstoneTransactionUpdate)) -> Self {
fn from(
value: (&crate::RawTransactionIngestYellowstoneSource, &ksp_onchain_transport_lib::YellowstoneTransactionUpdate),
) -> Self {
return project_yellowstone_signal(value.0, value.1);
}
}
@@ -294,7 +302,9 @@ impl std::convert::From<(&crate::RawTransactionIngestYellowstoneSource, &ksp_onc
impl std::convert::From<(&crate::RawTransactionIngestYellowstoneSource, &ksp_onchain_transport_lib::YellowstoneTransactionStatusUpdate)>
for RawTransactionIngestSourceSignal
{
fn from(value: (&crate::RawTransactionIngestYellowstoneSource, &ksp_onchain_transport_lib::YellowstoneTransactionStatusUpdate)) -> Self {
fn from(
value: (&crate::RawTransactionIngestYellowstoneSource, &ksp_onchain_transport_lib::YellowstoneTransactionStatusUpdate),
) -> Self {
return project_yellowstone_signal(value.0, value.1);
}
}
@@ -367,6 +377,7 @@ impl crate::RawTransactionIngestYellowstoneSource {
settings: crate::RawTransactionIngestSettings,
mut stop_receiver: tokio::sync::watch::Receiver<bool>,
admission_sender: tokio::sync::mpsc::Sender<crate::RawTransactionIngress>,
processing_frontier_sender: tokio::sync::watch::Sender<crate::RawTransactionIngestProcessingFrontierProjection>,
) -> ksp_core_lib::Result<()> {
let opened = tokio::select! {
biased;
@@ -380,6 +391,7 @@ impl crate::RawTransactionIngestYellowstoneSource {
std::result::Result::Err(error) => return std::result::Result::Err(source_transport_error(error.code())),
};
let mut coordinator = RawTransactionIngestHydrationCoordinator::new(&settings);
let mut processing_frontier = RawTransactionIngestProcessingFrontierReporter::new(processing_frontier_sender);
let mut fault = std::option::Option::None;
loop {
if *stop_receiver.borrow() {
@@ -407,7 +419,16 @@ impl crate::RawTransactionIngestYellowstoneSource {
break;
},
};
let handled = coordinator.handle_joined(joined, &self, &settings, &admission_sender, &mut stop_receiver).await;
let handled = coordinator
.handle_joined(
joined,
&self,
&settings,
&admission_sender,
&mut stop_receiver,
&mut processing_frontier,
)
.await;
match handled {
std::result::Result::Ok(true) => {},
std::result::Result::Ok(false) => break,
@@ -429,7 +450,12 @@ impl crate::RawTransactionIngestYellowstoneSource {
break;
},
};
if let std::result::Result::Err(error) = route_yellowstone_update(&self, &mut coordinator, update) {
if let std::result::Result::Err(error) = route_yellowstone_update(
&self,
&mut coordinator,
&mut processing_frontier,
update,
) {
fault = std::option::Option::Some(error);
break;
}
@@ -565,7 +591,9 @@ fn ingestion_filter_count(request: &ksp_onchain_transport_lib::YellowstoneSubscr
return request.transaction_filter_count().saturating_add(request.transaction_status_filter_count()).saturating_add(request.block_filter_count());
}
fn matched_filter_direct_id(filters: &[ksp_onchain_transport_lib::YellowstoneSubscribeFilterName]) -> std::option::Option<ksp_store_lib::RawProvenanceCode> {
fn matched_filter_direct_id(
filters: &[ksp_onchain_transport_lib::YellowstoneSubscribeFilterName],
) -> std::option::Option<ksp_store_lib::RawProvenanceCode> {
let mut names = filters.iter().map(ksp_onchain_transport_lib::YellowstoneSubscribeFilterName::as_str).collect::<std::vec::Vec<_>>();
names.sort_unstable();
names.dedup();
@@ -687,6 +715,7 @@ fn project_yellowstone_signal<T: RawTransactionIngestYellowstoneSignalView>(
fn route_yellowstone_update(
source: &crate::RawTransactionIngestYellowstoneSource,
coordinator: &mut RawTransactionIngestHydrationCoordinator,
processing_frontier: &mut RawTransactionIngestProcessingFrontierReporter,
update: ksp_onchain_transport_lib::YellowstoneSubscribeUpdate,
) -> ksp_core_lib::Result<()> {
match update {
@@ -695,14 +724,24 @@ fn route_yellowstone_update(
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
return coordinator.queue_signal(source, RawTransactionIngestSourceSignal::from((source, value.as_ref())), received_at);
return coordinator.queue_signal(
source,
RawTransactionIngestSourceSignal::from((source, value.as_ref())),
received_at,
processing_frontier,
);
},
ksp_onchain_transport_lib::YellowstoneSubscribeUpdate::TransactionStatus(value) => {
let received_at = match current_raw_timestamp() {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
return coordinator.queue_signal(source, RawTransactionIngestSourceSignal::from((source, &value)), received_at);
return coordinator.queue_signal(
source,
RawTransactionIngestSourceSignal::from((source, &value)),
received_at,
processing_frontier,
);
},
ksp_onchain_transport_lib::YellowstoneSubscribeUpdate::Block(value) => {
let received_at = match current_raw_timestamp() {
@@ -714,18 +753,26 @@ fn route_yellowstone_update(
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
for signal in signals {
if let std::result::Result::Err(error) = coordinator.queue_signal(source, signal, received_at) {
if let std::result::Result::Err(error) =
coordinator.queue_signal(source, signal, received_at, processing_frontier)
{
return std::result::Result::Err(error);
}
}
return std::result::Result::Ok(());
},
ksp_onchain_transport_lib::YellowstoneSubscribeUpdate::BlockMeta(value) => {
let _continuity_signal = project_yellowstone_continuity_signal(source, &value);
let continuity_signal = project_yellowstone_continuity_signal(source, &value);
if let std::result::Result::Err(error) = processing_frontier.observe_settled(continuity_signal.slot) {
return std::result::Result::Err(error);
}
return std::result::Result::Ok(());
},
ksp_onchain_transport_lib::YellowstoneSubscribeUpdate::Slot(value) => {
let _continuity_signal = project_yellowstone_continuity_signal(source, &value);
let continuity_signal = project_yellowstone_continuity_signal(source, &value);
if let std::result::Result::Err(error) = processing_frontier.observe_settled(continuity_signal.slot) {
return std::result::Result::Err(error);
}
return std::result::Result::Ok(());
},
ksp_onchain_transport_lib::YellowstoneSubscribeUpdate::Account(_)
@@ -757,6 +804,178 @@ fn source_transport_error(code: ksp_core_lib::ErrorCode) -> ksp_core_lib::Error
.with_context("transport_code", code.code());
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
struct RawTransactionIngestProcessingSlotState {
pending: usize,
settled: u64,
}
struct RawTransactionIngestProcessingFrontier {
pending_total: usize,
slots: std::collections::BTreeMap<u64, RawTransactionIngestProcessingSlotState>,
}
impl RawTransactionIngestProcessingFrontier {
fn new() -> Self {
return Self { pending_total: 0, slots: std::collections::BTreeMap::new() };
}
fn observe_pending(&mut self, slot: u64) -> ksp_core_lib::Result<()> {
let pending_total = match self.pending_total.checked_add(1) {
std::option::Option::Some(value) => value,
std::option::Option::None => return std::result::Result::Err(crate::runtime_error("source.frontier_pending_counter_exhausted")),
};
let state = self.slots.entry(slot).or_insert(RawTransactionIngestProcessingSlotState { pending: 0, settled: 0 });
state.pending = match state.pending.checked_add(1) {
std::option::Option::Some(value) => value,
std::option::Option::None => return std::result::Result::Err(crate::runtime_error("source.frontier_slot_pending_counter_exhausted")),
};
self.pending_total = pending_total;
self.compact();
return std::result::Result::Ok(());
}
fn observe_settled(&mut self, slot: u64) -> ksp_core_lib::Result<()> {
let state = self.slots.entry(slot).or_insert(RawTransactionIngestProcessingSlotState { pending: 0, settled: 0 });
state.settled = match state.settled.checked_add(1) {
std::option::Option::Some(value) => value,
std::option::Option::None => return std::result::Result::Err(crate::runtime_error("source.frontier_slot_settled_counter_exhausted")),
};
self.compact();
return std::result::Result::Ok(());
}
fn settle_pending(&mut self, slot: u64) -> ksp_core_lib::Result<()> {
let state = match self.slots.get_mut(&slot) {
std::option::Option::Some(value) => value,
std::option::Option::None => return std::result::Result::Err(crate::runtime_error("source.frontier_pending_slot_missing")),
};
if state.pending == 0 || self.pending_total == 0 {
return std::result::Result::Err(crate::runtime_error("source.frontier_pending_counter_invalid"));
}
state.pending -= 1;
self.pending_total -= 1;
state.settled = match state.settled.checked_add(1) {
std::option::Option::Some(value) => value,
std::option::Option::None => return std::result::Result::Err(crate::runtime_error("source.frontier_slot_settled_counter_exhausted")),
};
self.compact();
return std::result::Result::Ok(());
}
fn projection(&self) -> crate::RawTransactionIngestProcessingFrontierProjection {
let oldest_pending_slot = self.slots.iter().find_map(|(slot, state)| {
if state.pending == 0 {
return std::option::Option::None;
}
return std::option::Option::Some(*slot);
});
let processing_frontier_slot = match oldest_pending_slot {
std::option::Option::Some(pending_slot) => self.slots.range(..pending_slot).rev().find_map(|(slot, state)| {
if state.pending == 0 && state.settled > 0 {
return std::option::Option::Some(*slot);
}
return std::option::Option::None;
}),
std::option::Option::None => self.slots.iter().rev().find_map(|(slot, state)| {
if state.pending == 0 && state.settled > 0 {
return std::option::Option::Some(*slot);
}
return std::option::Option::None;
}),
};
return crate::RawTransactionIngestProcessingFrontierProjection::new(self.pending_total, processing_frontier_slot, oldest_pending_slot);
}
fn compact(&mut self) {
let pending_slots = self
.slots
.iter()
.filter_map(|(slot, state)| {
if state.pending > 0 {
return std::option::Option::Some(*slot);
}
return std::option::Option::None;
})
.collect::<std::vec::Vec<_>>();
let mut settled_interval_highs = std::collections::BTreeMap::<std::option::Option<u64>, u64>::new();
for (slot, state) in &self.slots {
if state.pending > 0 || state.settled == 0 {
continue;
}
let upper_pending = pending_slots.iter().find_map(|pending_slot| {
if pending_slot > slot {
return std::option::Option::Some(*pending_slot);
}
return std::option::Option::None;
});
match settled_interval_highs.entry(upper_pending) {
std::collections::btree_map::Entry::Occupied(mut entry) => {
if *slot > *entry.get() {
entry.insert(*slot);
}
},
std::collections::btree_map::Entry::Vacant(entry) => {
entry.insert(*slot);
},
}
}
let mut retained = pending_slots.into_iter().collect::<std::collections::BTreeSet<_>>();
for slot in settled_interval_highs.into_values() {
retained.insert(slot);
}
self.slots.retain(|slot, _state| {
return retained.contains(slot);
});
return;
}
#[cfg(test)]
fn slot_state_count(&self) -> usize {
return self.slots.len();
}
}
struct RawTransactionIngestProcessingFrontierReporter {
frontier: RawTransactionIngestProcessingFrontier,
sender: tokio::sync::watch::Sender<crate::RawTransactionIngestProcessingFrontierProjection>,
}
impl RawTransactionIngestProcessingFrontierReporter {
fn new(sender: tokio::sync::watch::Sender<crate::RawTransactionIngestProcessingFrontierProjection>) -> Self {
return Self { frontier: RawTransactionIngestProcessingFrontier::new(), sender };
}
fn observe_pending(&mut self, slot: u64) -> ksp_core_lib::Result<()> {
if let std::result::Result::Err(error) = self.frontier.observe_pending(slot) {
return std::result::Result::Err(error);
}
self.publish();
return std::result::Result::Ok(());
}
fn observe_settled(&mut self, slot: u64) -> ksp_core_lib::Result<()> {
if let std::result::Result::Err(error) = self.frontier.observe_settled(slot) {
return std::result::Result::Err(error);
}
self.publish();
return std::result::Result::Ok(());
}
fn settle_pending(&mut self, slot: u64) -> ksp_core_lib::Result<()> {
if let std::result::Result::Err(error) = self.frontier.settle_pending(slot) {
return std::result::Result::Err(error);
}
self.publish();
return std::result::Result::Ok(());
}
fn publish(&self) {
self.sender.send_replace(self.frontier.projection());
return;
}
}
#[derive(Clone, Eq, Ord, PartialEq, PartialOrd)]
struct RawTransactionIngestHydrationKey {
commitment: &'static str,
@@ -812,6 +1031,7 @@ impl RawTransactionIngestHydrationCoordinator {
source: &crate::RawTransactionIngestYellowstoneSource,
signal: RawTransactionIngestSourceSignal,
received_at: ksp_store_lib::RawTimestamp,
processing_frontier: &mut RawTransactionIngestProcessingFrontierReporter,
) -> ksp_core_lib::Result<()> {
if self.pending_signal_count >= self.max_pending_signals {
return std::result::Result::Err(crate::runtime_error("source.hydration_pending_saturated"));
@@ -820,6 +1040,7 @@ impl RawTransactionIngestHydrationCoordinator {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let signal_slot = signal.slot;
match self.pending.entry(key) {
std::collections::btree_map::Entry::Occupied(mut entry) => {
entry.get_mut().signals.push(RawTransactionIngestPendingSignal { received_at, signal });
@@ -835,17 +1056,23 @@ impl RawTransactionIngestHydrationCoordinator {
std::option::Option::Some(value) => value,
std::option::Option::None => return std::result::Result::Err(crate::runtime_error("source.hydration_pending_counter_exhausted")),
};
if let std::result::Result::Err(error) = processing_frontier.observe_pending(signal_slot) {
return std::result::Result::Err(error);
}
return self.start_hydrations(source);
}
fn start_hydrations(&mut self, source: &crate::RawTransactionIngestYellowstoneSource) -> ksp_core_lib::Result<()> {
while self.tasks.len() < self.max_in_flight {
let key = self.pending.iter().find_map(|(key, pending)| {
if pending.in_flight {
return std::option::Option::None;
}
return std::option::Option::Some(key.clone());
});
let key = self
.pending
.iter()
.find_map(|(key, pending)| {
if pending.in_flight {
return std::option::Option::None;
}
return std::option::Option::Some(key.clone());
});
let key = match key {
std::option::Option::Some(value) => value,
std::option::Option::None => break,
@@ -877,6 +1104,7 @@ impl RawTransactionIngestHydrationCoordinator {
settings: &crate::RawTransactionIngestSettings,
admission_sender: &tokio::sync::mpsc::Sender<crate::RawTransactionIngress>,
stop_receiver: &mut tokio::sync::watch::Receiver<bool>,
processing_frontier: &mut RawTransactionIngestProcessingFrontierReporter,
) -> ksp_core_lib::Result<bool> {
let fetched = match joined {
std::result::Result::Ok(std::result::Result::Ok(value)) => value,
@@ -892,6 +1120,7 @@ impl RawTransactionIngestHydrationCoordinator {
}
self.pending_signal_count -= pending.signals.len();
for pending_signal in pending.signals {
let signal_slot = pending_signal.signal.slot;
let ingress = finalize_yellowstone_hydration(source, settings, pending_signal.signal, pending_signal.received_at, &fetched.observed);
let ingress = match ingress {
std::result::Result::Ok(value) => value,
@@ -899,7 +1128,12 @@ impl RawTransactionIngestHydrationCoordinator {
};
let ingress = match ingress {
std::option::Option::Some(value) => value,
std::option::Option::None => continue,
std::option::Option::None => {
if let std::result::Result::Err(error) = processing_frontier.settle_pending(signal_slot) {
return std::result::Result::Err(error);
}
continue;
},
};
let sent = tokio::select! {
biased;
@@ -914,6 +1148,9 @@ impl RawTransactionIngestHydrationCoordinator {
}
return std::result::Result::Err(crate::runtime_error("source.admission_closed"));
}
if let std::result::Result::Err(error) = processing_frontier.settle_pending(signal_slot) {
return std::result::Result::Err(error);
}
}
if let std::result::Result::Err(error) = self.start_hydrations(source) {
return std::result::Result::Err(error);
@@ -1039,7 +1276,14 @@ fn finalize_yellowstone_hydration(
if embedded_signature != signal.signature {
return std::result::Result::Err(crate::runtime_error("hydration.signature_mismatch"));
}
let provenance = build_hydration_provenance(settings, &signal, observed.provider().as_str(), observed.endpoint_name(), commitment, received_at);
let provenance = build_hydration_provenance(
settings,
&signal,
observed.provider().as_str(),
observed.endpoint_name(),
commitment,
received_at,
);
let provenance = match provenance {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
@@ -1060,7 +1304,12 @@ fn finalize_yellowstone_hydration(
}),
map_hydration_wire_field(transaction.transaction_index(), |value| return *value),
);
return std::result::Result::Ok(std::option::Option::Some(crate::RawTransactionIngress { material, network: signal.network, provenance, source_key }));
return std::result::Result::Ok(std::option::Option::Some(crate::RawTransactionIngress {
material,
network: signal.network,
provenance,
source_key,
}));
}
#[cfg(test)]
@@ -1085,7 +1334,14 @@ async fn hydrate_yellowstone_signal(
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let fetched = fetch_yellowstone_hydration(source.http_pool.clone(), source.hydration_role.clone(), source.network.clone(), key, commitment).await;
let fetched = fetch_yellowstone_hydration(
source.http_pool.clone(),
source.hydration_role.clone(),
source.network.clone(),
key,
commitment,
)
.await;
let fetched = match fetched {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
@@ -1135,12 +1391,17 @@ fn build_hydration_provenance(
std::result::Result::Err(error) => return std::result::Result::Err(error),
},
};
let mut provenance =
ksp_store_lib::RawAcquisitionProvenance::new(provider, protocol, acquisition_method, ksp_store_lib::RawAcquisitionOrigin::Live, received_at)
.with_endpoint_id(endpoint_id)
.with_commitment(commitment)
.with_capture_session_id(capture_session)
.with_filter_id(filter_id);
let mut provenance = ksp_store_lib::RawAcquisitionProvenance::new(
provider,
protocol,
acquisition_method,
ksp_store_lib::RawAcquisitionOrigin::Live,
received_at,
)
.with_endpoint_id(endpoint_id)
.with_commitment(commitment)
.with_capture_session_id(capture_session)
.with_filter_id(filter_id);
if let std::option::Option::Some(observed_at) = representable_observed_at(signal.created_at, received_at) {
provenance = match provenance.try_with_observed_at(observed_at) {
std::result::Result::Ok(value) => value,