Sasedev/khadhroony-solana-project
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
8ae4d0effd8394aaecc0e312a4eb0667fb200f4b
khadhroony-solana-project/crates/ksp-worker-raw-transaction-ingest-lib/src/runtime_resources.rs

5504 lines
276 KiB
Rust
Raw Blame History

// file: crates/ksp-worker-raw-transaction-ingest-lib/src/runtime_resources.rs
// version: 47
use sha2::Digest; // rust-rules: trait-import
/// Default interval between HTTP live block polling cycles.
pub const DEFAULT_RAW_TRANSACTION_INGEST_HTTP_POLL_INTERVAL: std::time::Duration = std::time::Duration::from_secs(1);
/// Default maximum number of confirmed blocks discovered during one HTTP live polling cycle.
pub const DEFAULT_RAW_TRANSACTION_INGEST_HTTP_POLL_MAX_BLOCKS_PER_CYCLE: u16 = 128;
/// Maximum interval accepted between HTTP live block polling cycles.
pub const MAX_RAW_TRANSACTION_INGEST_HTTP_POLL_INTERVAL: std::time::Duration = std::time::Duration::from_secs(30);
/// Maximum number of confirmed blocks accepted during one HTTP live polling cycle.
pub const MAX_RAW_TRANSACTION_INGEST_HTTP_POLL_MAX_BLOCKS_PER_CYCLE: u16 = 1024;
/// Maximum number of logical live sources accepted in one RAW transaction ingest runtime-resource aggregate.
pub const MAX_RAW_TRANSACTION_INGEST_LIVE_SOURCES: usize = 32;
/// Minimum interval accepted between HTTP live block polling cycles.
pub const MIN_RAW_TRANSACTION_INGEST_HTTP_POLL_INTERVAL: std::time::Duration = std::time::Duration::from_millis(100);
/// Minimum number of confirmed blocks accepted during one HTTP live polling cycle.
pub const MIN_RAW_TRANSACTION_INGEST_HTTP_POLL_MAX_BLOCKS_PER_CYCLE: u16 = 1;
const MAX_RAW_TRANSACTION_INGEST_REPAIR_BURST: usize = 1;
const RAW_TRANSACTION_INGEST_HELIUS_TRANSACTION_FILTER_FINGERPRINT_DOMAIN: &[u8] = b"ksp.raw_transaction_ingest.helius_transaction.filter.v1\0";
const RAW_TRANSACTION_INGEST_HELIUS_TRANSACTION_HTTP_PROTOCOL: &str = "helius_ws_http";
const RAW_TRANSACTION_INGEST_HELIUS_TRANSACTION_HTTP_SOURCE_KEY_DOMAIN: &[u8] = b"ksp.raw_transaction_ingest.helius_transaction_http.source_key.v1\0";
const RAW_TRANSACTION_INGEST_HTTP_BLOCK_POLLING_ACQUISITION_METHOD: &str = "block_polling_get_block";
const RAW_TRANSACTION_INGEST_HTTP_BLOCK_POLLING_PROFILE_FINGERPRINT_DOMAIN: &[u8] = b"ksp.raw_transaction_ingest.http_block_polling.profile.v1\0";
const RAW_TRANSACTION_INGEST_HTTP_BLOCK_POLLING_PROTOCOL: &str = "solana_http";
const RAW_TRANSACTION_INGEST_LIVE_SOURCE_KEY_DOMAIN: &[u8] = b"ksp.raw_transaction_ingest.live_source.source_key.v1\0";
const RAW_TRANSACTION_INGEST_STANDARD_BLOCK_ACQUISITION_METHOD: &str = "block_subscribe";
const RAW_TRANSACTION_INGEST_STANDARD_BLOCK_FILTER_FINGERPRINT_DOMAIN: &[u8] = b"ksp.raw_transaction_ingest.standard_block.filter.v1\0";
const RAW_TRANSACTION_INGEST_STANDARD_BLOCK_PROTOCOL: &str = "solana_ws";
const RAW_TRANSACTION_INGEST_STANDARD_LOGS_FILTER_FINGERPRINT_DOMAIN: &[u8] = b"ksp.raw_transaction_ingest.standard_logs.filter.v1\0";
const RAW_TRANSACTION_INGEST_STANDARD_LOGS_HTTP_PROTOCOL: &str = "solana_ws_http";
const RAW_TRANSACTION_INGEST_STANDARD_LOGS_HTTP_SOURCE_KEY_DOMAIN: &[u8] = b"ksp.raw_transaction_ingest.standard_logs_http.source_key.v1\0";
const RAW_TRANSACTION_INGEST_YELLOWSTONE_BLOCK_HTTP_ACQUISITION_METHOD: &str = "block_subscribe_get_block";
const RAW_TRANSACTION_INGEST_YELLOWSTONE_BLOCK_RECONCILIATION_ATTEMPTS: u8 = 4;
const RAW_TRANSACTION_INGEST_YELLOWSTONE_BLOCK_RECONCILIATION_DELAY: std::time::Duration = std::time::Duration::from_millis(100);
const RAW_TRANSACTION_INGEST_YELLOWSTONE_COVERAGE_SCOPE_FINGERPRINT_DOMAIN: &[u8] = b"ksp.raw_transaction_ingest.yellowstone.coverage_scope.v1\0";
const RAW_TRANSACTION_INGEST_YELLOWSTONE_FILTER_FINGERPRINT_DOMAIN: &[u8] = b"ksp.raw_transaction_ingest.yellowstone.filters.v1\0";
const RAW_TRANSACTION_INGEST_YELLOWSTONE_HTTP_PROTOCOL: &str = "yellowstone_http";
const RAW_TRANSACTION_INGEST_YELLOWSTONE_HTTP_SOURCE_KEY_DOMAIN: &[u8] = b"ksp.raw_transaction_ingest.yellowstone_http.source_key.v1\0";
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
enum RawTransactionIngestTrafficClass {
Nominal,
Repair,
}
struct RawTransactionIngestFairTurnState {
active: bool,
last_granted: std::option::Option<RawTransactionIngestTrafficClass>,
next_ticket: u64,
waiters: std::collections::VecDeque<(u64, RawTransactionIngestTrafficClass)>,
}
struct RawTransactionIngestFairTurnGate {
max_waiters: usize,
notify: tokio::sync::Notify,
state: std::sync::Mutex<RawTransactionIngestFairTurnState>,
}
struct RawTransactionIngestFairWaiterGuard {
armed: bool,
gate: std::sync::Arc<RawTransactionIngestFairTurnGate>,
ticket: u64,
}
struct RawTransactionIngestFairTurnGuard {
gate: std::sync::Arc<RawTransactionIngestFairTurnGate>,
}
impl RawTransactionIngestFairTurnGate {
fn new(max_waiters: usize) -> Self {
return Self {
max_waiters,
notify: tokio::sync::Notify::new(),
state: std::sync::Mutex::new(RawTransactionIngestFairTurnState {
active: false,
last_granted: std::option::Option::None,
next_ticket: 1,
waiters: std::collections::VecDeque::with_capacity(max_waiters),
}),
};
}
async fn acquire(self: std::sync::Arc<Self>, class: RawTransactionIngestTrafficClass) -> ksp_core_lib::Result<RawTransactionIngestFairTurnGuard> {
let mut waiter = match RawTransactionIngestFairWaiterGuard::register(std::sync::Arc::clone(&self), class) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
loop {
let notify_gate = std::sync::Arc::clone(&self);
let notified = notify_gate.notify.notified();
let granted = {
let mut state = match self.state.lock() {
std::result::Result::Ok(value) => value,
std::result::Result::Err(poisoned) => poisoned.into_inner(),
};
if state.active || fair_turn_selected_ticket(&state) != std::option::Option::Some(waiter.ticket) {
false
} else {
let position = state.waiters.iter().position(|(ticket, _)| return *ticket == waiter.ticket);
let position = match position {
std::option::Option::Some(value) => value,
std::option::Option::None => return std::result::Result::Err(crate::runtime_error("source.fairness_waiter_missing")),
};
let removed = state.waiters.remove(position);
let removed = match removed {
std::option::Option::Some(value) => value,
std::option::Option::None => return std::result::Result::Err(crate::runtime_error("source.fairness_waiter_missing")),
};
if removed.1 != class {
return std::result::Result::Err(crate::runtime_error("source.fairness_waiter_class_mismatch"));
}
state.active = true;
state.last_granted = std::option::Option::Some(class);
true
}
};
if granted {
waiter.armed = false;
std::mem::drop(notified);
std::mem::drop(notify_gate);
return std::result::Result::Ok(RawTransactionIngestFairTurnGuard { gate: self });
}
notified.await;
}
}
}
impl RawTransactionIngestFairWaiterGuard {
fn register(gate: std::sync::Arc<RawTransactionIngestFairTurnGate>, class: RawTransactionIngestTrafficClass) -> ksp_core_lib::Result<Self> {
let ticket = {
let mut state = match gate.state.lock() {
std::result::Result::Ok(value) => value,
std::result::Result::Err(poisoned) => poisoned.into_inner(),
};
if gate.max_waiters == 0 || state.waiters.len() >= gate.max_waiters {
return std::result::Result::Err(crate::runtime_error("source.fairness_waiter_capacity_exceeded"));
}
let ticket = state.next_ticket;
state.next_ticket = match state.next_ticket.checked_add(1) {
std::option::Option::Some(value) => value,
std::option::Option::None => return std::result::Result::Err(crate::runtime_error("source.fairness_ticket_exhausted")),
};
state.waiters.push_back((ticket, class));
ticket
};
gate.notify.notify_waiters();
return std::result::Result::Ok(Self { armed: true, gate, ticket });
}
}
impl std::ops::Drop for RawTransactionIngestFairWaiterGuard {
fn drop(&mut self) {
if !self.armed {
return;
}
let mut state = match self.gate.state.lock() {
std::result::Result::Ok(value) => value,
std::result::Result::Err(poisoned) => poisoned.into_inner(),
};
if let std::option::Option::Some(position) = state.waiters.iter().position(|(ticket, _)| return *ticket == self.ticket) {
let _removed = state.waiters.remove(position);
}
std::mem::drop(state);
self.gate.notify.notify_waiters();
return;
}
}
impl std::ops::Drop for RawTransactionIngestFairTurnGuard {
fn drop(&mut self) {
let mut state = match self.gate.state.lock() {
std::result::Result::Ok(value) => value,
std::result::Result::Err(poisoned) => poisoned.into_inner(),
};
state.active = false;
std::mem::drop(state);
self.gate.notify.notify_waiters();
return;
}
}
fn fair_turn_selected_ticket(state: &RawTransactionIngestFairTurnState) -> std::option::Option<u64> {
let nominal = state.waiters.iter().find(|(_, class)| return *class == RawTransactionIngestTrafficClass::Nominal);
let repair = state.waiters.iter().find(|(_, class)| return *class == RawTransactionIngestTrafficClass::Repair);
let selected = match (nominal, repair) {
(std::option::Option::Some(nominal), std::option::Option::Some(repair)) => match state.last_granted {
std::option::Option::Some(RawTransactionIngestTrafficClass::Nominal) => repair,
std::option::Option::Some(RawTransactionIngestTrafficClass::Repair) | std::option::Option::None => nominal,
},
(std::option::Option::Some(nominal), std::option::Option::None) => nominal,
(std::option::Option::None, std::option::Option::Some(repair)) => repair,
(std::option::Option::None, std::option::Option::None) => return std::option::Option::None,
};
return std::option::Option::Some(selected.0);
}
fn validate_repair_fairness_contract(admission_capacity: usize, persistence_concurrency: usize) -> ksp_core_lib::Result<()> {
if admission_capacity == 0
|| persistence_concurrency == 0
|| repair_block_fetch_limit(persistence_concurrency) == 0
|| MAX_RAW_TRANSACTION_INGEST_REPAIR_BURST != 1
|| !repair_fairness_catalog_is_complete()
{
return std::result::Result::Err(crate::runtime_error("runtime_resources.repair_fairness_invalid"));
}
return std::result::Result::Ok(());
}
fn repair_block_fetch_limit(existing_capacity: usize) -> usize {
return existing_capacity.min(4);
}
fn repair_fairness_catalog_is_complete() -> bool {
let mut state = RawTransactionIngestFairTurnState {
active: false,
last_granted: std::option::Option::None,
next_ticket: 3,
waiters: std::collections::VecDeque::from([(1, RawTransactionIngestTrafficClass::Nominal), (2, RawTransactionIngestTrafficClass::Repair)]),
};
if fair_turn_selected_ticket(&state) != std::option::Option::Some(1) {
return false;
}
state.last_granted = std::option::Option::Some(RawTransactionIngestTrafficClass::Nominal);
if fair_turn_selected_ticket(&state) != std::option::Option::Some(2) {
return false;
}
state.last_granted = std::option::Option::Some(RawTransactionIngestTrafficClass::Repair);
return fair_turn_selected_ticket(&state) == std::option::Option::Some(1);
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
enum RawTransactionIngestHttpDiscoveryStrategy {
ClosedRange,
WithLimitBoundary,
}
#[derive(Clone, Debug, Eq, PartialEq)]
struct RawTransactionIngestHttpDiscoveryWindow {
produced_slots: std::vec::Vec<u64>,
proven_end_slot: std::option::Option<u64>,
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
struct RawTransactionIngestHttpScanCapabilities {
get_block: bool,
get_blocks: bool,
get_blocks_with_limit: bool,
get_slot: bool,
}
impl RawTransactionIngestHttpScanCapabilities {
const fn can_scan(self) -> bool {
return self.get_block && (self.get_blocks || (self.get_blocks_with_limit && self.get_slot));
}
fn strategy(self) -> ksp_core_lib::Result<RawTransactionIngestHttpDiscoveryStrategy> {
if self.get_block && self.get_blocks {
return std::result::Result::Ok(RawTransactionIngestHttpDiscoveryStrategy::ClosedRange);
}
if self.get_block && self.get_blocks_with_limit && self.get_slot {
return std::result::Result::Ok(RawTransactionIngestHttpDiscoveryStrategy::WithLimitBoundary);
}
return std::result::Result::Err(crate::runtime_error("continuity.http_scan_capability_incomplete"));
}
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
enum RawTransactionIngestSourceFamily {
Block,
Logs,
Transaction,
TransactionStatus,
}
enum RawTransactionIngestLiveSource {
HeliusTransaction(crate::RawTransactionIngestHeliusTransactionSource),
HttpBlockPolling(crate::RawTransactionIngestHttpBlockPollingSource),
StandardBlock(crate::RawTransactionIngestStandardBlockSource),
StandardLogs(crate::RawTransactionIngestStandardLogsSource),
Yellowstone(crate::RawTransactionIngestYellowstoneSource),
}
#[derive(Clone)]
struct RawTransactionIngestSourceRuntimeShared {
continuity_contracts: std::sync::Arc<std::sync::Mutex<crate::RawTransactionIngestContinuityContracts>>,
global_hydration_registry: std::sync::Arc<RawTransactionIngestGlobalHydrationRegistry>,
hydration_in_flight_limit: usize,
hydration_pending_limit: usize,
}
struct RawTransactionIngestSourceInventory {
source_keys: std::vec::Vec<[u8; 32]>,
source_projections: std::vec::Vec<crate::RawTransactionIngestProcessingFrontierProjection>,
}
#[derive(Clone)]
struct RawTransactionIngestSourceInventoryPublisher {
aggregate_sender: tokio::sync::watch::Sender<crate::RawTransactionIngestProcessingFrontierProjection>,
continuity_contracts: std::sync::Arc<std::sync::Mutex<crate::RawTransactionIngestContinuityContracts>>,
entry_index: usize,
inventory: std::sync::Arc<std::sync::Mutex<RawTransactionIngestSourceInventory>>,
source_key: [u8; 32],
}
impl RawTransactionIngestLiveSource {
fn network(&self) -> &ksp_store_lib::RawNetworkId {
return match self {
Self::HeliusTransaction(source) => &source.network,
Self::HttpBlockPolling(source) => &source.network,
Self::StandardBlock(source) => &source.network,
Self::StandardLogs(source) => &source.network,
Self::Yellowstone(source) => &source.network,
};
}
fn repair_capability_descriptor(&self) -> ksp_core_lib::Result<crate::RawTransactionIngestContinuityCapabilityDescriptor> {
let commitment = match self {
Self::HeliusTransaction(source) => source.commitment,
Self::HttpBlockPolling(source) => source.commitment,
Self::StandardBlock(source) => source.commitment,
Self::StandardLogs(source) => source.commitment,
Self::Yellowstone(source) => match source.subscribe_request.commitment() {
std::option::Option::Some(value) => value,
std::option::Option::None => return std::result::Result::Err(crate::runtime_error("continuity.yellowstone_commitment_missing")),
},
};
let source_scope = match self {
Self::HeliusTransaction(source) => crate::RawTransactionIngestCoverageScope::exact_source_scope("helius_transaction", source.filter_fingerprint),
Self::HttpBlockPolling(_) => crate::RawTransactionIngestCoverageScope::full_ledger_transactions(),
Self::StandardBlock(source) => match &source.filter {
ksp_onchain_transport_lib::SolanaBlockSubscribeFilter::All => crate::RawTransactionIngestCoverageScope::full_ledger_transactions(),
ksp_onchain_transport_lib::SolanaBlockSubscribeFilter::MentionsAccountOrProgram(_) => {
crate::RawTransactionIngestCoverageScope::exact_source_scope("standard_block", source.filter_fingerprint)
},
},
Self::StandardLogs(source) => crate::RawTransactionIngestCoverageScope::exact_source_scope("standard_logs", source.filter_fingerprint),
Self::Yellowstone(source) => {
let fingerprint = match yellowstone_coverage_scope_fingerprint(&source.subscribe_request) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
crate::RawTransactionIngestCoverageScope::exact_source_scope("yellowstone", fingerprint)
},
};
let (reference_bearing, live_block_material, native_replay, http_block_scan, known_reference_hydration) = match self {
Self::HeliusTransaction(source) => {
let http_block_scan = match http_role_supports_repair_scan(&source.http_pool, &source.hydration_role, source.network.as_str()) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
(true, false, false, http_block_scan, true)
},
Self::HttpBlockPolling(source) => {
let known_reference_hydration =
match http_role_supports_rpc_method(&source.http_pool, &source.polling_role, "getTransaction", source.network.as_str()) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
(false, true, false, true, known_reference_hydration)
},
Self::StandardBlock(_) => (false, true, false, false, false),
Self::StandardLogs(source) => {
let http_block_scan = match http_role_supports_repair_scan(&source.http_pool, &source.hydration_role, source.network.as_str()) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
(true, false, false, http_block_scan, true)
},
Self::Yellowstone(source) => {
let http_block_scan = match http_role_supports_repair_scan(&source.http_pool, &source.hydration_role, source.network.as_str()) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let known_reference_hydration =
match http_role_supports_rpc_method(&source.http_pool, &source.hydration_role, "getTransaction", source.network.as_str()) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let reference_bearing =
source.subscribe_request.transaction_filter_count() > 0 || source.subscribe_request.transaction_status_filter_count() > 0;
let live_block_material = source.subscribe_request.block_filter_count() > 0;
(reference_bearing, live_block_material, true, http_block_scan, known_reference_hydration)
},
};
let block_material = live_block_material || http_block_scan;
let slot_enumerating = http_block_scan;
return crate::RawTransactionIngestContinuityCapabilityDescriptor::new(
self.source_key(),
self.network().clone(),
commitment,
source_scope,
reference_bearing,
block_material,
slot_enumerating,
known_reference_hydration,
native_replay,
http_block_scan,
);
}
fn kind_code(&self) -> &'static str {
return match self {
Self::HeliusTransaction(_) => "helius_transaction",
Self::HttpBlockPolling(_) => "http_block_polling",
Self::StandardBlock(_) => "standard_block",
Self::StandardLogs(_) => "standard_logs",
Self::Yellowstone(_) => "yellowstone",
};
}
fn source_key(&self) -> [u8; 32] {
return match self {
Self::HeliusTransaction(source) => source.source_key,
Self::HttpBlockPolling(source) => source.source_key,
Self::StandardBlock(source) => source.source_key,
Self::StandardLogs(source) => source.source_key,
Self::Yellowstone(source) => source.source_key,
};
}
fn uses_hydration(&self) -> bool {
return match self {
Self::HeliusTransaction(_) | Self::StandardLogs(_) => true,
Self::Yellowstone(source) => source.uses_transaction_hydration(),
Self::HttpBlockPolling(_) | Self::StandardBlock(_) => false,
};
}
async fn run(
self,
settings: crate::RawTransactionIngestSettings,
stop_receiver: tokio::sync::watch::Receiver<bool>,
admission_sender: tokio::sync::mpsc::Sender<crate::RawTransactionIngress>,
inventory_publisher: RawTransactionIngestSourceInventoryPublisher,
shared: RawTransactionIngestSourceRuntimeShared,
) -> ksp_core_lib::Result<()> {
let source_kind = self.kind_code();
let (source_frontier_sender, mut source_frontier_receiver) =
tokio::sync::watch::channel(crate::RawTransactionIngestProcessingFrontierProjection::empty());
let mut source_future = std::boxed::Box::pin(async move {
return match self {
Self::HeliusTransaction(source) => source.run(settings, stop_receiver, admission_sender, source_frontier_sender, shared).await,
Self::HttpBlockPolling(source) => {
source.run(settings, stop_receiver, admission_sender, source_frontier_sender, shared.continuity_contracts).await
},
Self::StandardBlock(source) => source.run(settings, stop_receiver, admission_sender, source_frontier_sender).await,
Self::StandardLogs(source) => source.run(settings, stop_receiver, admission_sender, source_frontier_sender, shared).await,
Self::Yellowstone(source) => source.run(settings, stop_receiver, admission_sender, source_frontier_sender, shared).await,
};
});
loop {
tokio::select! {
biased;
result = &mut source_future => {
let latest = source_frontier_receiver.borrow_and_update().clone();
let terminal_state = if result.is_ok() {
crate::RawTransactionIngestSourceState::Closed
} else {
crate::RawTransactionIngestSourceState::Failed
};
if let std::result::Result::Err(error) = inventory_publisher.publish(source_projection_with_state(latest, terminal_state)) {
return std::result::Result::Err(error);
}
if let std::result::Result::Err(error) = &result {
let condition = source_error_context_value(error, "condition").unwrap_or("none");
let transport_domain = source_error_context_value(error, "transport_domain").unwrap_or("none");
let transport_code = source_error_context_value(error, "transport_code").unwrap_or("none");
ksp_logging_lib::warn!(
target: crate::TRACING_TARGET,
domain = "raw_transaction_ingest.source",
source_kind = source_kind,
error_domain = error.code().domain(),
error_code = error.code().code(),
condition = condition,
transport_domain = transport_domain,
transport_code = transport_code,
"RAW transaction ingest live source reached terminal failure"
);
}
return result;
}
changed = source_frontier_receiver.changed() => {
if changed.is_err() {
return std::result::Result::Err(crate::runtime_error("source.frontier_channel_closed"));
}
if let std::result::Result::Err(error) = inventory_publisher.publish(source_frontier_receiver.borrow_and_update().clone()) {
return std::result::Result::Err(error);
}
}
}
}
}
}
impl RawTransactionIngestSourceInventory {
fn aggregate(&self) -> ksp_core_lib::Result<crate::RawTransactionIngestProcessingFrontierProjection> {
let mut hydration_pending = 0_usize;
let mut oldest_pending_slot = std::option::Option::None;
let mut processing_frontier_slot = std::option::Option::None;
let mut all_frontiers_present = !self.source_projections.is_empty();
let mut source_continuity_gap_total = 0_u64;
let mut source_reconnect_total = 0_u64;
let mut source_replay_attempt_total = 0_u64;
let source_total = self.source_projections.len();
let mut source_active = 0_usize;
let mut source_reconnecting = 0_usize;
let mut source_failed = 0_usize;
let mut any_active = false;
let mut any_closing = false;
let mut any_failed = false;
let mut any_reconnecting = false;
let mut all_closed = !self.source_projections.is_empty();
for projection in &self.source_projections {
hydration_pending = match hydration_pending.checked_add(projection.hydration_pending()) {
std::option::Option::Some(value) => value,
std::option::Option::None => return std::result::Result::Err(crate::counter_exhausted_error("hydration_pending")),
};
oldest_pending_slot = minimum_optional_slot(oldest_pending_slot, projection.oldest_pending_slot());
match projection.processing_frontier_slot() {
std::option::Option::Some(slot) => {
processing_frontier_slot = minimum_optional_slot(processing_frontier_slot, std::option::Option::Some(slot));
},
std::option::Option::None => {
all_frontiers_present = false;
},
}
source_continuity_gap_total = match source_continuity_gap_total.checked_add(projection.source_continuity_gap_total()) {
std::option::Option::Some(value) => value,
std::option::Option::None => return std::result::Result::Err(crate::counter_exhausted_error("source_continuity_gap_total")),
};
source_reconnect_total = match source_reconnect_total.checked_add(projection.source_reconnect_total()) {
std::option::Option::Some(value) => value,
std::option::Option::None => return std::result::Result::Err(crate::counter_exhausted_error("source_reconnect_total")),
};
source_replay_attempt_total = match source_replay_attempt_total.checked_add(projection.source_replay_attempt_total()) {
std::option::Option::Some(value) => value,
std::option::Option::None => return std::result::Result::Err(crate::counter_exhausted_error("source_replay_attempt_total")),
};
match projection.source_state() {
std::option::Option::Some(crate::RawTransactionIngestSourceState::Active) => {
source_active += 1;
any_active = true;
all_closed = false;
},
std::option::Option::Some(crate::RawTransactionIngestSourceState::Closing) => {
any_closing = true;
all_closed = false;
},
std::option::Option::Some(crate::RawTransactionIngestSourceState::Closed) => {},
std::option::Option::Some(crate::RawTransactionIngestSourceState::Failed) => {
source_failed += 1;
any_failed = true;
all_closed = false;
},
std::option::Option::Some(crate::RawTransactionIngestSourceState::Reconnecting) => {
source_reconnecting += 1;
any_reconnecting = true;
all_closed = false;
},
std::option::Option::None => {
all_closed = false;
},
}
}
if !all_frontiers_present {
processing_frontier_slot = std::option::Option::None;
}
let source_state = if any_failed {
std::option::Option::Some(crate::RawTransactionIngestSourceState::Failed)
} else if any_reconnecting {
std::option::Option::Some(crate::RawTransactionIngestSourceState::Reconnecting)
} else if any_closing {
std::option::Option::Some(crate::RawTransactionIngestSourceState::Closing)
} else if any_active {
std::option::Option::Some(crate::RawTransactionIngestSourceState::Active)
} else if all_closed {
std::option::Option::Some(crate::RawTransactionIngestSourceState::Closed)
} else {
std::option::Option::None
};
return std::result::Result::Ok(
crate::RawTransactionIngestProcessingFrontierProjection::new(hydration_pending, processing_frontier_slot, oldest_pending_slot)
.with_source_continuity(source_state, source_reconnect_total, source_replay_attempt_total, source_continuity_gap_total)
.with_source_counts(source_total, source_active, source_reconnecting, source_failed),
);
}
fn new(source_keys: std::vec::Vec<[u8; 32]>) -> Self {
let source_projections = source_keys.iter().map(|_source_key| return crate::RawTransactionIngestProcessingFrontierProjection::empty()).collect();
return Self { source_keys, source_projections };
}
fn active_source_keys(&self) -> ksp_core_lib::Result<std::vec::Vec<[u8; 32]>> {
if self.source_keys.len() != self.source_projections.len() {
return std::result::Result::Err(crate::runtime_error("source.inventory_shape_mismatch"));
}
let mut active_source_keys = std::vec::Vec::new();
for (source_key, projection) in self.source_keys.iter().zip(self.source_projections.iter()) {
if projection.source_state() == std::option::Option::Some(crate::RawTransactionIngestSourceState::Active) {
active_source_keys.push(*source_key);
}
}
return std::result::Result::Ok(active_source_keys);
}
fn failed_source_keys(&self) -> ksp_core_lib::Result<std::vec::Vec<[u8; 32]>> {
if self.source_keys.len() != self.source_projections.len() {
return std::result::Result::Err(crate::runtime_error("source.inventory_shape_mismatch"));
}
let mut failed_source_keys = std::vec::Vec::new();
for (source_key, projection) in self.source_keys.iter().zip(self.source_projections.iter()) {
if projection.source_state() == std::option::Option::Some(crate::RawTransactionIngestSourceState::Failed) {
failed_source_keys.push(*source_key);
}
}
return std::result::Result::Ok(failed_source_keys);
}
fn supervisor_state(&self) -> ksp_core_lib::Result<(std::vec::Vec<[u8; 32]>, std::option::Option<u64>)> {
let active_source_keys = match self.active_source_keys() {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let aggregate = match self.aggregate() {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
return std::result::Result::Ok((active_source_keys, aggregate.processing_frontier_slot()));
}
fn update(
&mut self,
entry_index: usize,
source_key: [u8; 32],
projection: crate::RawTransactionIngestProcessingFrontierProjection,
) -> ksp_core_lib::Result<()> {
let expected_key = match self.source_keys.get(entry_index) {
std::option::Option::Some(value) => value,
std::option::Option::None => return std::result::Result::Err(crate::runtime_error("source.inventory_entry_missing")),
};
if expected_key != &source_key {
return std::result::Result::Err(crate::runtime_error("source.inventory_key_mismatch"));
}
let current = match self.source_projections.get_mut(entry_index) {
std::option::Option::Some(value) => value,
std::option::Option::None => return std::result::Result::Err(crate::runtime_error("source.inventory_projection_missing")),
};
*current = projection;
return std::result::Result::Ok(());
}
}
fn source_inventory_health_projection(
inventory: &std::sync::Arc<std::sync::Mutex<RawTransactionIngestSourceInventory>>,
continuity_contracts: &std::sync::Arc<std::sync::Mutex<crate::RawTransactionIngestContinuityContracts>>,
) -> ksp_core_lib::Result<crate::RawTransactionIngestProcessingFrontierProjection> {
let (aggregate, active_source_keys, failed_source_keys) = {
let inventory = match inventory.lock() {
std::result::Result::Ok(value) => value,
std::result::Result::Err(poisoned) => poisoned.into_inner(),
};
let aggregate = match inventory.aggregate() {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let active_source_keys = match inventory.active_source_keys() {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let failed_source_keys = match inventory.failed_source_keys() {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
(aggregate, active_source_keys, failed_source_keys)
};
let (continuity_frontier_slot, continuity_has_open_gaps, future_target_coverage, failed_source_losses_reconciled, continuity_snapshot) = {
let mut contracts = match continuity_contracts.lock() {
std::result::Result::Ok(value) => value,
std::result::Result::Err(poisoned) => poisoned.into_inner(),
};
let health = match contracts.health_projection(active_source_keys.as_slice(), failed_source_keys.as_slice(), aggregate.processing_frontier_slot()) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let continuity_snapshot = match contracts.observability_projection() {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
(health.0, health.1, health.2, health.3, continuity_snapshot)
};
let aggregate = aggregate.with_continuity_health(continuity_frontier_slot, continuity_has_open_gaps, future_target_coverage);
let aggregate = aggregate.with_continuity_snapshot(continuity_snapshot);
return std::result::Result::Ok(aggregate.with_failed_source_losses_reconciled(failed_source_losses_reconciled));
}
impl RawTransactionIngestSourceInventoryPublisher {
fn publish(&self, projection: crate::RawTransactionIngestProcessingFrontierProjection) -> ksp_core_lib::Result<()> {
{
let mut inventory = match self.inventory.lock() {
std::result::Result::Ok(value) => value,
std::result::Result::Err(poisoned) => poisoned.into_inner(),
};
if let std::result::Result::Err(error) = inventory.update(self.entry_index, self.source_key, projection) {
return std::result::Result::Err(error);
}
}
let aggregate = match source_inventory_health_projection(&self.inventory, &self.continuity_contracts) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
self.aggregate_sender.send_replace(aggregate);
return std::result::Result::Ok(());
}
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
enum RawTransactionIngestContinuityFamily {
BlockMeta,
Slot,
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
enum RawTransactionIngestContinuityStatus {
Completed,
Confirmed,
CreatedBank,
Dead,
Finalized,
FirstShredReceived,
Processed,
}
#[derive(Clone, Eq, PartialEq)]
struct RawTransactionIngestContinuitySignal {
created_at: std::option::Option<RawTransactionIngestSourceTimestamp>,
family: RawTransactionIngestContinuityFamily,
matched_filter_count: usize,
matched_filter_fingerprint: [u8; 32],
matched_filter_id: std::option::Option<ksp_store_lib::RawProvenanceCode>,
network: ksp_store_lib::RawNetworkId,
parent_slot: std::option::Option<u64>,
route: RawTransactionIngestSourceRoute,
slot: u64,
status: std::option::Option<RawTransactionIngestContinuityStatus>,
}
impl std::fmt::Debug for RawTransactionIngestContinuitySignal {
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
return formatter
.debug_struct("RawTransactionIngestContinuitySignal")
.field("created_at", &self.created_at)
.field("family", &self.family)
.field("matched_filter_count", &self.matched_filter_count)
.field("matched_filter_fingerprint_bytes", &self.matched_filter_fingerprint.len())
.field("has_direct_filter_id", &self.matched_filter_id.is_some())
.field("network", &self.network)
.field("parent_slot", &self.parent_slot)
.field("route", &self.route)
.field("slot", &self.slot)
.field("status", &self.status)
.finish_non_exhaustive();
}
}
#[derive(Clone, Debug, Eq, PartialEq)]
struct RawTransactionIngestSourceRoute {
endpoint_id: ksp_store_lib::RawProvenanceCode,
provider: ksp_store_lib::RawProvenanceCode,
}
#[derive(Clone, Copy, Eq, PartialEq)]
struct RawTransactionIngestSourceTimestamp {
nanos: u32,
seconds: i64,
}
impl std::fmt::Debug for RawTransactionIngestSourceTimestamp {
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
return formatter.debug_struct("RawTransactionIngestSourceTimestamp").field("nanos", &self.nanos).field("seconds", &self.seconds).finish();
}
}
#[derive(Clone, Eq, PartialEq)]
struct RawTransactionIngestSourceSignal {
created_at: std::option::Option<RawTransactionIngestSourceTimestamp>,
family: RawTransactionIngestSourceFamily,
matched_filter_count: usize,
matched_filter_fingerprint: [u8; 32],
matched_filter_id: std::option::Option<ksp_store_lib::RawProvenanceCode>,
network: ksp_store_lib::RawNetworkId,
route: RawTransactionIngestSourceRoute,
signature: ksp_store_lib::RawTransactionSignature,
slot: u64,
transaction_index: std::option::Option<u64>,
}
impl std::fmt::Debug for RawTransactionIngestSourceSignal {
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
return formatter
.debug_struct("RawTransactionIngestSourceSignal")
.field("created_at", &self.created_at)
.field("family", &self.family)
.field("matched_filter_count", &self.matched_filter_count)
.field("matched_filter_fingerprint_bytes", &self.matched_filter_fingerprint.len())
.field("has_direct_filter_id", &self.matched_filter_id.is_some())
.field("network", &self.network)
.field("route", &self.route)
.field("signature_bytes", &self.signature.as_bytes().len())
.field("slot", &self.slot)
.field("has_transaction_index", &self.transaction_index.is_some())
.finish_non_exhaustive();
}
}
trait RawTransactionIngestYellowstoneSignalView {
fn created_at(&self) -> std::option::Option<ksp_onchain_transport_lib::YellowstoneUpdateTimestamp>;
fn family(&self) -> RawTransactionIngestSourceFamily;
fn filters(&self) -> &[ksp_onchain_transport_lib::YellowstoneSubscribeFilterName];
fn index(&self) -> u64;
fn signature(&self) -> ksp_onchain_transport_lib::YellowstoneTransactionSignature;
fn slot(&self) -> u64;
}
impl RawTransactionIngestYellowstoneSignalView for ksp_onchain_transport_lib::YellowstoneTransactionUpdate {
fn created_at(&self) -> std::option::Option<ksp_onchain_transport_lib::YellowstoneUpdateTimestamp> {
return ksp_onchain_transport_lib::YellowstoneTransactionUpdate::created_at(self);
}
fn family(&self) -> RawTransactionIngestSourceFamily {
return RawTransactionIngestSourceFamily::Transaction;
}
fn filters(&self) -> &[ksp_onchain_transport_lib::YellowstoneSubscribeFilterName] {
return ksp_onchain_transport_lib::YellowstoneTransactionUpdate::filters(self);
}
fn index(&self) -> u64 {
return ksp_onchain_transport_lib::YellowstoneTransactionUpdate::transaction(self).index();
}
fn signature(&self) -> ksp_onchain_transport_lib::YellowstoneTransactionSignature {
return ksp_onchain_transport_lib::YellowstoneTransactionUpdate::transaction(self).signature();
}
fn slot(&self) -> u64 {
return ksp_onchain_transport_lib::YellowstoneTransactionUpdate::slot(self);
}
}
impl RawTransactionIngestYellowstoneSignalView for ksp_onchain_transport_lib::YellowstoneTransactionStatusUpdate {
fn created_at(&self) -> std::option::Option<ksp_onchain_transport_lib::YellowstoneUpdateTimestamp> {
return ksp_onchain_transport_lib::YellowstoneTransactionStatusUpdate::created_at(self);
}
fn family(&self) -> RawTransactionIngestSourceFamily {
return RawTransactionIngestSourceFamily::TransactionStatus;
}
fn filters(&self) -> &[ksp_onchain_transport_lib::YellowstoneSubscribeFilterName] {
return ksp_onchain_transport_lib::YellowstoneTransactionStatusUpdate::filters(self);
}
fn index(&self) -> u64 {
return ksp_onchain_transport_lib::YellowstoneTransactionStatusUpdate::index(self);
}
fn signature(&self) -> ksp_onchain_transport_lib::YellowstoneTransactionSignature {
return ksp_onchain_transport_lib::YellowstoneTransactionStatusUpdate::signature(self);
}
fn slot(&self) -> u64 {
return ksp_onchain_transport_lib::YellowstoneTransactionStatusUpdate::slot(self);
}
}
trait RawTransactionIngestYellowstoneBlockView {
fn created_at(&self) -> std::option::Option<ksp_onchain_transport_lib::YellowstoneUpdateTimestamp>;
fn filters(&self) -> &[ksp_onchain_transport_lib::YellowstoneSubscribeFilterName];
fn slot(&self) -> u64;
fn transaction_count(&self) -> usize;
fn transaction_identity(&self, position: usize) -> ksp_core_lib::Result<(ksp_onchain_transport_lib::YellowstoneTransactionSignature, u64)>;
}
impl RawTransactionIngestYellowstoneBlockView for ksp_onchain_transport_lib::YellowstoneBlockUpdate {
fn created_at(&self) -> std::option::Option<ksp_onchain_transport_lib::YellowstoneUpdateTimestamp> {
return ksp_onchain_transport_lib::YellowstoneBlockUpdate::created_at(self);
}
fn filters(&self) -> &[ksp_onchain_transport_lib::YellowstoneSubscribeFilterName] {
return ksp_onchain_transport_lib::YellowstoneBlockUpdate::filters(self);
}
fn slot(&self) -> u64 {
return ksp_onchain_transport_lib::YellowstoneBlockUpdate::slot(self);
}
fn transaction_count(&self) -> usize {
return ksp_onchain_transport_lib::YellowstoneBlockUpdate::transactions(self).len();
}
fn transaction_identity(&self, position: usize) -> ksp_core_lib::Result<(ksp_onchain_transport_lib::YellowstoneTransactionSignature, u64)> {
let transaction = match ksp_onchain_transport_lib::YellowstoneBlockUpdate::transactions(self).get(position) {
std::option::Option::Some(value) => value,
std::option::Option::None => return std::result::Result::Err(crate::runtime_error("source.block_transaction_missing")),
};
return std::result::Result::Ok((transaction.signature(), transaction.index()));
}
}
trait RawTransactionIngestYellowstoneContinuityView {
fn created_at(&self) -> std::option::Option<ksp_onchain_transport_lib::YellowstoneUpdateTimestamp>;
fn family(&self) -> RawTransactionIngestContinuityFamily;
fn filters(&self) -> &[ksp_onchain_transport_lib::YellowstoneSubscribeFilterName];
fn parent_slot(&self) -> std::option::Option<u64>;
fn slot(&self) -> u64;
fn status(&self) -> std::option::Option<RawTransactionIngestContinuityStatus>;
}
impl RawTransactionIngestYellowstoneContinuityView for ksp_onchain_transport_lib::YellowstoneBlockMetaUpdate {
fn created_at(&self) -> std::option::Option<ksp_onchain_transport_lib::YellowstoneUpdateTimestamp> {
return ksp_onchain_transport_lib::YellowstoneBlockMetaUpdate::created_at(self);
}
fn family(&self) -> RawTransactionIngestContinuityFamily {
return RawTransactionIngestContinuityFamily::BlockMeta;
}
fn filters(&self) -> &[ksp_onchain_transport_lib::YellowstoneSubscribeFilterName] {
return ksp_onchain_transport_lib::YellowstoneBlockMetaUpdate::filters(self);
}
fn parent_slot(&self) -> std::option::Option<u64> {
return std::option::Option::Some(ksp_onchain_transport_lib::YellowstoneBlockMetaUpdate::parent_slot(self));
}
fn slot(&self) -> u64 {
return ksp_onchain_transport_lib::YellowstoneBlockMetaUpdate::slot(self);
}
fn status(&self) -> std::option::Option<RawTransactionIngestContinuityStatus> {
return std::option::Option::None;
}
}
impl RawTransactionIngestYellowstoneContinuityView for ksp_onchain_transport_lib::YellowstoneSlotUpdate {
fn created_at(&self) -> std::option::Option<ksp_onchain_transport_lib::YellowstoneUpdateTimestamp> {
return ksp_onchain_transport_lib::YellowstoneSlotUpdate::created_at(self);
}
fn family(&self) -> RawTransactionIngestContinuityFamily {
return RawTransactionIngestContinuityFamily::Slot;
}
fn filters(&self) -> &[ksp_onchain_transport_lib::YellowstoneSubscribeFilterName] {
return ksp_onchain_transport_lib::YellowstoneSlotUpdate::filters(self);
}
fn parent_slot(&self) -> std::option::Option<u64> {
return ksp_onchain_transport_lib::YellowstoneSlotUpdate::parent(self);
}
fn slot(&self) -> u64 {
return ksp_onchain_transport_lib::YellowstoneSlotUpdate::slot(self);
}
fn status(&self) -> std::option::Option<RawTransactionIngestContinuityStatus> {
return std::option::Option::Some(map_yellowstone_slot_status(ksp_onchain_transport_lib::YellowstoneSlotUpdate::status(self)));
}
}
impl std::convert::From<(&crate::RawTransactionIngestYellowstoneSource, &ksp_onchain_transport_lib::YellowstoneTransactionUpdate)>
for RawTransactionIngestSourceSignal
{
fn from(value: (&crate::RawTransactionIngestYellowstoneSource, &ksp_onchain_transport_lib::YellowstoneTransactionUpdate)) -> Self {
return project_yellowstone_signal(value.0, value.1);
}
}
impl std::convert::From<(&crate::RawTransactionIngestYellowstoneSource, &ksp_onchain_transport_lib::YellowstoneTransactionStatusUpdate)>
for RawTransactionIngestSourceSignal
{
fn from(value: (&crate::RawTransactionIngestYellowstoneSource, &ksp_onchain_transport_lib::YellowstoneTransactionStatusUpdate)) -> Self {
return project_yellowstone_signal(value.0, value.1);
}
}
#[derive(Clone)]
struct RawTransactionIngestHydrationContext {
commitment: ksp_onchain_transport_lib::SolanaCommitment,
http_pool: ksp_onchain_transport_lib::HttpTransportPool,
hydration_role: ksp_onchain_transport_lib::HttpRoleName,
network: ksp_store_lib::RawNetworkId,
protocol: &'static str,
route: RawTransactionIngestSourceRoute,
route_prefix: &'static str,
source_key: [u8; 32],
source_key_domain: &'static [u8],
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
enum RawTransactionIngestYellowstoneMode {
TransactionHydration,
BlockHydration,
}
/// Validated Yellowstone plus HTTP runtime source owned by the continuous RAW transaction ingest Worker.
///
/// The Transport-owned channel, subscribe request, HTTP pool and hydration role remain private. Construction validates deterministic source-composition
/// invariants without network I/O; runtime execution opens exactly one Transport-owned Yellowstone session.
pub struct RawTransactionIngestYellowstoneSource {
yellowstone_channel: ksp_onchain_transport_lib::YellowstoneGrpcChannel,
subscribe_request: ksp_onchain_transport_lib::YellowstoneSubscribeRequest,
http_pool: ksp_onchain_transport_lib::HttpTransportPool,
hydration_role: ksp_onchain_transport_lib::HttpRoleName,
mode: RawTransactionIngestYellowstoneMode,
network: ksp_store_lib::RawNetworkId,
route: RawTransactionIngestSourceRoute,
source_key: [u8; 32],
}
impl crate::RawTransactionIngestYellowstoneSource {
/// Creates one validated Yellowstone source contract without opening a stream or issuing HTTP requests.
pub fn new(
yellowstone_channel: ksp_onchain_transport_lib::YellowstoneGrpcChannel,
subscribe_request: ksp_onchain_transport_lib::YellowstoneSubscribeRequest,
http_pool: ksp_onchain_transport_lib::HttpTransportPool,
hydration_role: ksp_onchain_transport_lib::HttpRoleName,
) -> ksp_core_lib::Result<Self> {
if subscribe_request.validate().is_err() {
return std::result::Result::Err(crate::runtime_error("runtime_resources.yellowstone_request_invalid"));
}
if ingestion_filter_count(&subscribe_request) == 0 {
return std::result::Result::Err(crate::runtime_error("runtime_resources.ingestion_filter_missing"));
}
match subscribe_request.commitment() {
std::option::Option::Some(ksp_onchain_transport_lib::SolanaCommitment::Confirmed)
| std::option::Option::Some(ksp_onchain_transport_lib::SolanaCommitment::Finalized) => {},
std::option::Option::Some(ksp_onchain_transport_lib::SolanaCommitment::Processed) | std::option::Option::None => {
return std::result::Result::Err(crate::runtime_error("runtime_resources.hydration_commitment_invalid"));
},
}
let network = match ksp_store_lib::RawNetworkId::new(yellowstone_channel.cluster().as_str()) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return std::result::Result::Err(crate::runtime_error("runtime_resources.yellowstone_network_unrepresentable")),
};
let provider = match ksp_store_lib::RawProvenanceCode::new(yellowstone_channel.provider().as_str()) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return std::result::Result::Err(crate::runtime_error("runtime_resources.yellowstone_provider_unrepresentable")),
};
let endpoint_id = match ksp_store_lib::RawProvenanceCode::new(yellowstone_channel.endpoint_name()) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return std::result::Result::Err(crate::runtime_error("runtime_resources.yellowstone_endpoint_unrepresentable")),
};
let route = RawTransactionIngestSourceRoute { endpoint_id, provider };
let mode = if subscribe_request.block_filter_count() > 0
&& subscribe_request.transaction_filter_count() == 0
&& subscribe_request.transaction_status_filter_count() == 0
{
RawTransactionIngestYellowstoneMode::BlockHydration
} else {
RawTransactionIngestYellowstoneMode::TransactionHydration
};
let method_name = match mode {
RawTransactionIngestYellowstoneMode::TransactionHydration => "getTransaction",
RawTransactionIngestYellowstoneMode::BlockHydration => "getBlock",
};
let method = match ksp_onchain_transport_lib::find_http_rpc_method(method_name) {
std::option::Option::Some(value) => value,
std::option::Option::None => return std::result::Result::Err(crate::runtime_error("runtime_resources.hydration_method_missing")),
};
let compatible_http_routes = compatible_http_route_count(&http_pool, &hydration_role, method.request_kind(), network.as_str(), &route, "ys");
let compatible_http_routes = match compatible_http_routes {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
if compatible_http_routes == 0 {
return std::result::Result::Err(crate::runtime_error("runtime_resources.hydration_role_unsupported"));
}
let request_identity = match subscribe_request.identity() {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return std::result::Result::Err(crate::runtime_error("runtime_resources.yellowstone_identity_invalid")),
};
let source_key = yellowstone_live_source_key(&network, &route, subscribe_request.commitment(), &request_identity);
return std::result::Result::Ok(Self { yellowstone_channel, subscribe_request, http_pool, hydration_role, mode, network, route, source_key });
}
fn uses_transaction_hydration(&self) -> bool {
return self.mode == RawTransactionIngestYellowstoneMode::TransactionHydration;
}
fn hydration_context(&self) -> RawTransactionIngestHydrationContext {
let commitment = match self.subscribe_request.commitment() {
std::option::Option::Some(value) => value,
std::option::Option::None => ksp_onchain_transport_lib::SolanaCommitment::Confirmed,
};
return RawTransactionIngestHydrationContext {
commitment,
http_pool: self.http_pool.clone(),
hydration_role: self.hydration_role.clone(),
network: self.network.clone(),
protocol: RAW_TRANSACTION_INGEST_YELLOWSTONE_HTTP_PROTOCOL,
route: self.route.clone(),
route_prefix: "ys",
source_key: self.source_key,
source_key_domain: RAW_TRANSACTION_INGEST_YELLOWSTONE_HTTP_SOURCE_KEY_DOMAIN,
};
}
/// Runs the productive Yellowstone source task until cooperative stop or one safe terminal source failure.
async fn run(
self,
settings: crate::RawTransactionIngestSettings,
mut stop_receiver: tokio::sync::watch::Receiver<bool>,
admission_sender: tokio::sync::mpsc::Sender<crate::RawTransactionIngress>,
processing_frontier_sender: tokio::sync::watch::Sender<crate::RawTransactionIngestProcessingFrontierProjection>,
shared: RawTransactionIngestSourceRuntimeShared,
) -> ksp_core_lib::Result<()> {
if self.mode == RawTransactionIngestYellowstoneMode::BlockHydration {
return self.run_block_hydration(settings, stop_receiver, admission_sender, processing_frontier_sender).await;
}
let RawTransactionIngestSourceRuntimeShared { continuity_contracts, global_hydration_registry, hydration_in_flight_limit, hydration_pending_limit } =
shared;
let opened = tokio::select! {
biased;
_ = stop_receiver.changed() => {
return std::result::Result::Ok(());
}
result = self.yellowstone_channel.open_standard_subscribe(self.subscribe_request.clone()) => result,
};
let mut session = match opened {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(source_transport_error(error.code())),
};
let hydration = self.hydration_context();
let mut coordinator =
RawTransactionIngestHydrationCoordinator::with_global_registry(global_hydration_registry, hydration_pending_limit, hydration_in_flight_limit);
let mut processing_frontier = RawTransactionIngestProcessingFrontierReporter::new(processing_frontier_sender);
let snapshot_source = session.snapshot_source();
if let std::result::Result::Err(error) = processing_frontier.observe_session_snapshot(snapshot_source.current()) {
processing_frontier.set_source_state(crate::RawTransactionIngestSourceState::Failed);
let _closed = session.close().await;
return std::result::Result::Err(error);
}
let mut session_snapshot_source = std::option::Option::Some(snapshot_source);
let mut fault = std::option::Option::None;
loop {
if *stop_receiver.borrow() {
break;
}
if let std::result::Result::Err(error) = coordinator.start_hydrations(&hydration) {
fault = std::option::Option::Some(error);
break;
}
let can_receive = coordinator.can_receive();
if !can_receive && coordinator.tasks.is_empty() {
fault = std::option::Option::Some(crate::runtime_error("source.hydration_stalled"));
break;
}
tokio::select! {
biased;
_ = stop_receiver.changed() => {
break;
}
source_snapshot = wait_yellowstone_session_snapshot(&mut session_snapshot_source) => {
match source_snapshot {
std::option::Option::Some(snapshot) => {
if let std::result::Result::Err(error) = processing_frontier.observe_session_snapshot(snapshot) {
fault = std::option::Option::Some(error);
break;
}
},
std::option::Option::None => {
session_snapshot_source = std::option::Option::None;
},
}
}
joined = coordinator.tasks.join_next(), if !coordinator.tasks.is_empty() => {
let joined = match joined {
std::option::Option::Some(value) => value,
std::option::Option::None => {
fault = std::option::Option::Some(crate::runtime_error("source.hydration_join_missing"));
break;
},
};
let handled = coordinator
.handle_joined(
joined,
&hydration,
&settings,
&admission_sender,
&mut stop_receiver,
&mut processing_frontier,
&continuity_contracts,
)
.await;
match handled {
std::result::Result::Ok(true) => {},
std::result::Result::Ok(false) => break,
std::result::Result::Err(error) => {
fault = std::option::Option::Some(error);
break;
},
}
}
update = session.next_update(), if can_receive => {
let update = match update {
std::result::Result::Ok(std::option::Option::Some(value)) => value,
std::result::Result::Ok(std::option::Option::None) => {
fault = std::option::Option::Some(crate::runtime_error("source.session_closed"));
break;
},
std::result::Result::Err(error) => {
fault = std::option::Option::Some(source_transport_error(error.code()));
break;
},
};
if let std::result::Result::Err(error) = route_yellowstone_update(
&self,
&hydration,
&mut coordinator,
&mut processing_frontier,
update,
) {
fault = std::option::Option::Some(error);
break;
}
}
}
}
coordinator.abort_all(&mut processing_frontier).await;
processing_frontier.set_source_state(crate::RawTransactionIngestSourceState::Closing);
let closed = session.close().await;
if let std::option::Option::Some(error) = fault {
processing_frontier.set_source_state(crate::RawTransactionIngestSourceState::Failed);
return std::result::Result::Err(error);
}
return match resolve_yellowstone_close_after_stop(*stop_receiver.borrow(), closed) {
std::result::Result::Ok(()) => {
processing_frontier.set_source_state(crate::RawTransactionIngestSourceState::Closed);
std::result::Result::Ok(())
},
std::result::Result::Err(error) => {
processing_frontier.set_source_state(crate::RawTransactionIngestSourceState::Failed);
std::result::Result::Err(error)
},
};
}
async fn run_block_hydration(
self,
settings: crate::RawTransactionIngestSettings,
mut stop_receiver: tokio::sync::watch::Receiver<bool>,
admission_sender: tokio::sync::mpsc::Sender<crate::RawTransactionIngress>,
processing_frontier_sender: tokio::sync::watch::Sender<crate::RawTransactionIngestProcessingFrontierProjection>,
) -> ksp_core_lib::Result<()> {
let opened = tokio::select! {
biased;
_ = stop_receiver.changed() => {
return std::result::Result::Ok(());
}
result = self.yellowstone_channel.open_standard_subscribe(self.subscribe_request.clone()) => result,
};
let mut session = match opened {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(source_transport_error(error.code())),
};
let mut processing_frontier = RawTransactionIngestProcessingFrontierReporter::new(processing_frontier_sender);
let snapshot_source = session.snapshot_source();
if let std::result::Result::Err(error) = processing_frontier.observe_session_snapshot(snapshot_source.current()) {
processing_frontier.set_source_state(crate::RawTransactionIngestSourceState::Failed);
let _closed = session.close().await;
return std::result::Result::Err(error);
}
let mut session_snapshot_source = std::option::Option::Some(snapshot_source);
let mut fault = std::option::Option::None;
loop {
if *stop_receiver.borrow() {
break;
}
tokio::select! {
biased;
_ = stop_receiver.changed() => {
break;
}
source_snapshot = wait_yellowstone_session_snapshot(&mut session_snapshot_source) => {
match source_snapshot {
std::option::Option::Some(snapshot) => {
if let std::result::Result::Err(error) = processing_frontier.observe_session_snapshot(snapshot) {
fault = std::option::Option::Some(error);
break;
}
},
std::option::Option::None => session_snapshot_source = std::option::Option::None,
}
}
update = session.next_update() => {
let update = match update {
std::result::Result::Ok(std::option::Option::Some(value)) => value,
std::result::Result::Ok(std::option::Option::None) => {
fault = std::option::Option::Some(crate::runtime_error("source.session_closed"));
break;
},
std::result::Result::Err(error) => {
fault = std::option::Option::Some(source_transport_error(error.code()));
break;
},
};
if let ksp_onchain_transport_lib::YellowstoneSubscribeUpdate::Block(value) = update {
let fetched = fetch_yellowstone_block_ingresses(&self, &settings, value.slot(), &mut stop_receiver).await;
let ingresses = match fetched {
std::result::Result::Ok(std::option::Option::Some(value)) => value,
std::result::Result::Ok(std::option::Option::None) => break,
std::result::Result::Err(error) => {
fault = std::option::Option::Some(error);
break;
},
};
for ingress in ingresses {
let sent = tokio::select! {
biased;
_ = stop_receiver.changed() => {
break;
}
result = admission_sender.send(ingress) => result,
};
if sent.is_err() {
if !*stop_receiver.borrow() {
fault = std::option::Option::Some(crate::runtime_error("source.admission_closed"));
}
break;
}
}
if fault.is_some() || *stop_receiver.borrow() {
break;
}
if let std::result::Result::Err(error) = processing_frontier.observe_settled(value.slot()) {
fault = std::option::Option::Some(error);
break;
}
}
}
}
}
processing_frontier.set_source_state(crate::RawTransactionIngestSourceState::Closing);
let closed = session.close().await;
if let std::option::Option::Some(error) = fault {
processing_frontier.set_source_state(crate::RawTransactionIngestSourceState::Failed);
return std::result::Result::Err(error);
}
return match resolve_yellowstone_close_after_stop(*stop_receiver.borrow(), closed) {
std::result::Result::Ok(()) => {
processing_frontier.set_source_state(crate::RawTransactionIngestSourceState::Closed);
std::result::Result::Ok(())
},
std::result::Result::Err(error) => {
processing_frontier.set_source_state(crate::RawTransactionIngestSourceState::Failed);
std::result::Result::Err(error)
},
};
}
}
/// Validated Helius `transactionSubscribe` plus HTTP hydration source owned by the continuous RAW transaction ingest Worker.
///
/// The caller provides one Transport-owned Helius LaserStream WebSocket endpoint explicitly declaring `HeliusTransaction`, one Helius transaction filter, a
/// Confirmed/Finalized commitment and one HTTP hydration pool/role. The Worker always requests Full/Base64 notifications with `maxSupportedTransactionVersion = 1` but projects only the provider
/// signature/slot/index reference into the source-neutral hydration coordinator; the nested Helius payload is never copied into Worker state, while endpoint
/// URLs and sensitive endpoint material remain encapsulated by Transport-owned settings and are never exposed by Worker APIs or diagnostics.
pub struct RawTransactionIngestHeliusTransactionSource {
ws_endpoint: ksp_onchain_transport_lib::WsEndpointSettings,
filter: ksp_onchain_transport_lib::HeliusTransactionSubscribeFilter,
commitment: ksp_onchain_transport_lib::SolanaCommitment,
http_pool: ksp_onchain_transport_lib::HttpTransportPool,
hydration_role: ksp_onchain_transport_lib::HttpRoleName,
network: ksp_store_lib::RawNetworkId,
route: RawTransactionIngestSourceRoute,
filter_fingerprint: [u8; 32],
source_key: [u8; 32],
}
impl crate::RawTransactionIngestHeliusTransactionSource {
/// Creates one validated Helius transaction source without opening WebSocket or HTTP network I/O.
pub fn new(
ws_endpoint: ksp_onchain_transport_lib::WsEndpointSettings,
filter: ksp_onchain_transport_lib::HeliusTransactionSubscribeFilter,
commitment: ksp_onchain_transport_lib::SolanaCommitment,
http_pool: ksp_onchain_transport_lib::HttpTransportPool,
hydration_role: ksp_onchain_transport_lib::HttpRoleName,
) -> ksp_core_lib::Result<Self> {
let ws_settings = ksp_onchain_transport_lib::WsTransportSettings::new(std::vec![ws_endpoint.clone()]);
if ws_settings.validate().is_err() {
return std::result::Result::Err(crate::runtime_error("runtime_resources.helius_transaction_endpoint_invalid"));
}
if ws_endpoint.protocol() != ksp_onchain_transport_lib::WsProtocolKind::HeliusLaserStream {
return std::result::Result::Err(crate::runtime_error("runtime_resources.helius_transaction_protocol_invalid"));
}
if !ws_endpoint.supports_subscription(ksp_onchain_transport_lib::WsSubscriptionKind::HeliusTransaction) {
return std::result::Result::Err(crate::runtime_error("runtime_resources.helius_transaction_capability_missing"));
}
match commitment {
ksp_onchain_transport_lib::SolanaCommitment::Confirmed | ksp_onchain_transport_lib::SolanaCommitment::Finalized => {},
ksp_onchain_transport_lib::SolanaCommitment::Processed => {
return std::result::Result::Err(crate::runtime_error("runtime_resources.hydration_commitment_invalid"));
},
}
let request = helius_transaction_request(filter.clone(), commitment);
if request.validate().is_err() {
return std::result::Result::Err(crate::runtime_error("runtime_resources.helius_transaction_request_invalid"));
}
let network = match ksp_store_lib::RawNetworkId::new(ws_endpoint.cluster().as_str()) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => {
return std::result::Result::Err(crate::runtime_error("runtime_resources.helius_transaction_network_unrepresentable"));
},
};
let provider = match ksp_store_lib::RawProvenanceCode::new(ws_endpoint.provider().as_str()) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => {
return std::result::Result::Err(crate::runtime_error("runtime_resources.helius_transaction_provider_unrepresentable"));
},
};
let endpoint_id = match ksp_store_lib::RawProvenanceCode::new(ws_endpoint.name()) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => {
return std::result::Result::Err(crate::runtime_error("runtime_resources.helius_transaction_endpoint_unrepresentable"));
},
};
let route = RawTransactionIngestSourceRoute { endpoint_id, provider };
let method = match ksp_onchain_transport_lib::find_http_rpc_method("getTransaction") {
std::option::Option::Some(value) => value,
std::option::Option::None => return std::result::Result::Err(crate::runtime_error("runtime_resources.hydration_method_missing")),
};
let compatible_http_routes = compatible_http_route_count(&http_pool, &hydration_role, method.request_kind(), network.as_str(), &route, "hx");
let compatible_http_routes = match compatible_http_routes {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
if compatible_http_routes == 0 {
return std::result::Result::Err(crate::runtime_error("runtime_resources.hydration_role_unsupported"));
}
let filter_fingerprint = helius_transaction_filter_fingerprint(&filter);
let source_key = helius_transaction_live_source_key(&network, &route, commitment, &filter_fingerprint);
return std::result::Result::Ok(Self {
ws_endpoint,
filter,
commitment,
http_pool,
hydration_role,
network,
route,
filter_fingerprint,
source_key,
});
}
fn hydration_context(&self) -> RawTransactionIngestHydrationContext {
return RawTransactionIngestHydrationContext {
commitment: self.commitment,
http_pool: self.http_pool.clone(),
hydration_role: self.hydration_role.clone(),
network: self.network.clone(),
protocol: RAW_TRANSACTION_INGEST_HELIUS_TRANSACTION_HTTP_PROTOCOL,
route: self.route.clone(),
route_prefix: "hx",
source_key: self.source_key,
source_key_domain: RAW_TRANSACTION_INGEST_HELIUS_TRANSACTION_HTTP_SOURCE_KEY_DOMAIN,
};
}
/// Runs one productive Helius `transactionSubscribe` source until cooperative stop or one safe terminal source failure.
async fn run(
self,
settings: crate::RawTransactionIngestSettings,
mut stop_receiver: tokio::sync::watch::Receiver<bool>,
admission_sender: tokio::sync::mpsc::Sender<crate::RawTransactionIngress>,
processing_frontier_sender: tokio::sync::watch::Sender<crate::RawTransactionIngestProcessingFrontierProjection>,
shared: RawTransactionIngestSourceRuntimeShared,
) -> ksp_core_lib::Result<()> {
let RawTransactionIngestSourceRuntimeShared { continuity_contracts, global_hydration_registry, hydration_in_flight_limit, hydration_pending_limit } =
shared;
let connected = tokio::select! {
biased;
_ = stop_receiver.changed() => {
return std::result::Result::Ok(());
}
result = ksp_onchain_transport_lib::HeliusLaserStreamWsSession::connect(self.ws_endpoint.clone()) => result,
};
let session = match connected {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(source_transport_error(error.code())),
};
let request = helius_transaction_request(self.filter.clone(), self.commitment);
let subscribed = tokio::select! {
biased;
_ = stop_receiver.changed() => {
let _closed = session.close().await;
return std::result::Result::Ok(());
}
result = session.transaction_subscribe(&request) => result,
};
let mut subscription = match subscribed {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => {
let _closed = session.close().await;
return std::result::Result::Err(source_transport_error(error.code()));
},
};
let mut session_snapshot_source = session.snapshot_source();
let hydration = self.hydration_context();
let mut coordinator =
RawTransactionIngestHydrationCoordinator::with_global_registry(global_hydration_registry, hydration_pending_limit, hydration_in_flight_limit);
let mut processing_frontier = RawTransactionIngestProcessingFrontierReporter::new(processing_frontier_sender);
if let std::result::Result::Err(error) = processing_frontier.observe_websocket_session_snapshot(session_snapshot_source.current()) {
let _closed = session.close().await;
return std::result::Result::Err(error);
}
let mut fault = std::option::Option::None;
let mut bounded_websocket_incident = std::option::Option::None;
loop {
if *stop_receiver.borrow() {
break;
}
if let std::result::Result::Err(error) = coordinator.start_hydrations(&hydration) {
fault = std::option::Option::Some(error);
break;
}
if let std::option::Option::Some((start_slot, end_slot)) = bounded_websocket_incident
&& coordinator.pending_signal_count == 0
&& coordinator.tasks.is_empty()
{
fault = std::option::Option::Some(continuity_range_error("source.continuity_gap_proven", start_slot, end_slot));
break;
}
let can_receive = coordinator.can_receive() && bounded_websocket_incident.is_none();
if !can_receive && coordinator.tasks.is_empty() {
fault = std::option::Option::Some(crate::runtime_error("source.hydration_stalled"));
break;
}
tokio::select! {
biased;
_ = stop_receiver.changed() => {
break;
}
source_snapshot = session_snapshot_source.wait_for_change() => {
let source_snapshot = match source_snapshot {
std::option::Option::Some(value) => value,
std::option::Option::None => {
fault = std::option::Option::Some(crate::runtime_error("source.websocket_snapshot_closed"));
break;
},
};
if let std::result::Result::Err(error) = processing_frontier.observe_websocket_session_snapshot(source_snapshot) {
fault = std::option::Option::Some(error);
break;
}
}
joined = coordinator.tasks.join_next(), if !coordinator.tasks.is_empty() => {
let joined = match joined {
std::option::Option::Some(value) => value,
std::option::Option::None => {
fault = std::option::Option::Some(crate::runtime_error("source.hydration_join_missing"));
break;
},
};
let handled = coordinator
.handle_joined(
joined,
&hydration,
&settings,
&admission_sender,
&mut stop_receiver,
&mut processing_frontier,
&continuity_contracts,
)
.await;
match handled {
std::result::Result::Ok(true) => {},
std::result::Result::Ok(false) => break,
std::result::Result::Err(error) => {
fault = std::option::Option::Some(error);
break;
},
}
}
notification = subscription.recv(), if can_receive => {
let notification = match notification {
std::option::Option::Some(std::result::Result::Ok(value)) => value,
std::option::Option::Some(std::result::Result::Err(error)) => {
fault = std::option::Option::Some(source_transport_error(error.code()));
break;
},
std::option::Option::None => {
let error = match subscription.terminal_error_code() {
std::option::Option::Some(code) => source_transport_error(code),
std::option::Option::None => crate::runtime_error("source.helius_transaction_subscription_closed"),
};
fault = std::option::Option::Some(error);
break;
},
};
let received_at = match current_raw_timestamp() {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => {
fault = std::option::Option::Some(error);
break;
},
};
let full = match notification {
ksp_onchain_transport_lib::HeliusTransactionNotification::Full(value) => value,
_ => {
fault = std::option::Option::Some(crate::runtime_error("source.helius_transaction_notification_unqualified"));
break;
},
};
let signal = match project_helius_transaction_signal(&self, &full) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => {
fault = std::option::Option::Some(error);
break;
},
};
let incident_bounded = match processing_frontier.observe_websocket_post_incident_slot(signal.slot) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => {
fault = std::option::Option::Some(error);
break;
},
};
if let std::result::Result::Err(error) = coordinator.queue_signal(&hydration, signal, received_at, &mut processing_frontier) {
fault = std::option::Option::Some(error);
break;
}
if bounded_websocket_incident.is_none() {
bounded_websocket_incident = incident_bounded;
}
}
}
}
coordinator.abort_all(&mut processing_frontier).await;
processing_frontier.set_source_state(crate::RawTransactionIngestSourceState::Closing);
let closed = session.close().await;
if let std::option::Option::Some(error) = fault {
processing_frontier.set_source_state(crate::RawTransactionIngestSourceState::Failed);
return std::result::Result::Err(error);
}
return match closed {
std::result::Result::Ok(()) => {
processing_frontier.set_source_state(crate::RawTransactionIngestSourceState::Closed);
std::result::Result::Ok(())
},
std::result::Result::Err(error) => {
processing_frontier.set_source_state(crate::RawTransactionIngestSourceState::Failed);
std::result::Result::Err(source_transport_error(error.code()))
},
};
}
}
impl std::fmt::Debug for crate::RawTransactionIngestHeliusTransactionSource {
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
let http_snapshot = self.http_pool.snapshot();
return formatter
.debug_struct("RawTransactionIngestHeliusTransactionSource")
.field("ws_endpoint_name", &self.ws_endpoint.name())
.field("ws_provider", &self.ws_endpoint.provider().as_str())
.field("network", &self.network.as_str())
.field("commitment", &self.commitment)
.field("filter", &self.filter)
.field("filter_fingerprint_bytes", &self.filter_fingerprint.len())
.field("hydration_role", &self.hydration_role.as_str())
.field("http_endpoint_count", &http_snapshot.endpoint_count())
.field("source_key_bytes", &self.source_key.len())
.finish();
}
}
/// Validated standard Solana HTTP live block polling source owned by the continuous RAW transaction ingest Worker.
///
/// The caller provides one Transport-owned HTTP pool/role, a Confirmed/Finalized commitment and bounded polling controls. Construction proves that the role
/// has same-network routes for `getSlot`, `getBlocksWithLimit` and `getBlock` without network I/O. Runtime starts at the first observed committed slot and
/// never requests an earlier slot, so this source remains live and run-local.
pub struct RawTransactionIngestHttpBlockPollingSource {
http_pool: ksp_onchain_transport_lib::HttpTransportPool,
polling_role: ksp_onchain_transport_lib::HttpRoleName,
commitment: ksp_onchain_transport_lib::SolanaCommitment,
network: ksp_store_lib::RawNetworkId,
poll_interval: std::time::Duration,
max_discovered_blocks_per_cycle: u16,
profile_fingerprint: [u8; 32],
source_key: [u8; 32],
}
impl crate::RawTransactionIngestHttpBlockPollingSource {
/// Creates one HTTP live block polling source using the default 1-second cadence and 128-block discovery bound.
pub fn new(
http_pool: ksp_onchain_transport_lib::HttpTransportPool,
polling_role: ksp_onchain_transport_lib::HttpRoleName,
commitment: ksp_onchain_transport_lib::SolanaCommitment,
) -> ksp_core_lib::Result<Self> {
return Self::new_with_limits(
http_pool,
polling_role,
commitment,
crate::DEFAULT_RAW_TRANSACTION_INGEST_HTTP_POLL_INTERVAL,
crate::DEFAULT_RAW_TRANSACTION_INGEST_HTTP_POLL_MAX_BLOCKS_PER_CYCLE,
);
}
/// Creates one HTTP live block polling source with explicit bounded cadence and discovery controls.
pub fn new_with_limits(
http_pool: ksp_onchain_transport_lib::HttpTransportPool,
polling_role: ksp_onchain_transport_lib::HttpRoleName,
commitment: ksp_onchain_transport_lib::SolanaCommitment,
poll_interval: std::time::Duration,
max_discovered_blocks_per_cycle: u16,
) -> ksp_core_lib::Result<Self> {
match commitment {
ksp_onchain_transport_lib::SolanaCommitment::Confirmed | ksp_onchain_transport_lib::SolanaCommitment::Finalized => {},
ksp_onchain_transport_lib::SolanaCommitment::Processed => {
return std::result::Result::Err(crate::runtime_error("runtime_resources.http_block_polling_commitment_invalid"));
},
}
if !(crate::MIN_RAW_TRANSACTION_INGEST_HTTP_POLL_INTERVAL..=crate::MAX_RAW_TRANSACTION_INGEST_HTTP_POLL_INTERVAL).contains(&poll_interval) {
return std::result::Result::Err(crate::runtime_error("runtime_resources.http_block_polling_interval_invalid"));
}
if !(crate::MIN_RAW_TRANSACTION_INGEST_HTTP_POLL_MAX_BLOCKS_PER_CYCLE..=crate::MAX_RAW_TRANSACTION_INGEST_HTTP_POLL_MAX_BLOCKS_PER_CYCLE)
.contains(&max_discovered_blocks_per_cycle)
{
return std::result::Result::Err(crate::runtime_error("runtime_resources.http_block_polling_limit_invalid"));
}
let profile = validate_http_block_polling_profile(&http_pool, &polling_role);
let (network, profile_fingerprint) = match profile {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let source_key = http_block_polling_live_source_key(&network, &polling_role, commitment, &profile_fingerprint);
return std::result::Result::Ok(Self {
http_pool,
polling_role,
commitment,
network,
poll_interval,
max_discovered_blocks_per_cycle,
profile_fingerprint,
source_key,
});
}
/// Runs one productive HTTP live block polling source until cooperative stop or one safe terminal source failure.
pub(crate) async fn run(
self,
settings: crate::RawTransactionIngestSettings,
mut stop_receiver: tokio::sync::watch::Receiver<bool>,
admission_sender: tokio::sync::mpsc::Sender<crate::RawTransactionIngress>,
processing_frontier_sender: tokio::sync::watch::Sender<crate::RawTransactionIngestProcessingFrontierProjection>,
continuity_contracts: std::sync::Arc<std::sync::Mutex<crate::RawTransactionIngestContinuityContracts>>,
) -> ksp_core_lib::Result<()> {
let context_config = ksp_onchain_transport_lib::SolanaContextConfig::new(std::option::Option::Some(self.commitment), std::option::Option::None);
let get_block_config = ksp_onchain_transport_lib::SolanaGetBlockConfig::new(
std::option::Option::Some(self.commitment),
std::option::Option::Some(ksp_onchain_transport_lib::SolanaTransactionEncoding::Base64),
std::option::Option::Some(ksp_onchain_transport_lib::SolanaTransactionDetails::Full),
std::option::Option::Some(1),
std::option::Option::Some(false),
);
let start_slot = tokio::select! {
biased;
_ = stop_receiver.changed() => {
return std::result::Result::Ok(());
}
result = self.http_pool.get_slot(&self.polling_role, std::option::Option::Some(&context_config)) => result,
};
let start_slot = match start_slot {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(source_transport_error(error.code())),
};
let mut next_scan_slot = start_slot;
let mut processing_frontier = RawTransactionIngestProcessingFrontierReporter::new(processing_frontier_sender);
processing_frontier.set_source_state(crate::RawTransactionIngestSourceState::Active);
let mut fault = std::option::Option::None;
'source: loop {
if *stop_receiver.borrow() {
break;
}
let current_tip = tokio::select! {
biased;
_ = stop_receiver.changed() => {
break;
}
result = self.http_pool.get_slot(&self.polling_role, std::option::Option::Some(&context_config)) => result,
};
let current_tip = match current_tip {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => {
fault = std::option::Option::Some(source_transport_error(error.code()));
break;
},
};
if next_scan_slot <= current_tip {
let configured_window_slots =
u64::from(self.max_discovered_blocks_per_cycle).min(crate::MAX_RAW_TRANSACTION_INGEST_CONTINUITY_DISCOVERY_WINDOW_SLOTS);
let window_offset = match configured_window_slots.checked_sub(1) {
std::option::Option::Some(value) => value,
std::option::Option::None => {
fault = std::option::Option::Some(crate::runtime_error("source.http_block_polling_window_invalid"));
break;
},
};
let candidate_end = match next_scan_slot.checked_add(window_offset) {
std::option::Option::Some(value) => value,
std::option::Option::None => u64::MAX,
};
let window_end = current_tip.min(candidate_end);
let window_start = next_scan_slot;
let discovered = discover_http_block_window(
&self.http_pool,
&self.polling_role,
self.network.as_str(),
self.commitment,
next_scan_slot,
window_end,
&mut stop_receiver,
)
.await;
let discovery = match discovered {
std::result::Result::Ok(std::option::Option::Some(value)) => value,
std::result::Result::Ok(std::option::Option::None) => break,
std::result::Result::Err(error) => {
fault = std::option::Option::Some(error);
break;
},
};
let mut blocked_by_null = false;
for slot in discovery.produced_slots {
let observed = tokio::select! {
biased;
_ = stop_receiver.changed() => {
break 'source;
}
result = self.http_pool.get_block_observed(&self.polling_role, slot, std::option::Option::Some(&get_block_config)) => result,
};
let observed = match observed {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => {
fault = std::option::Option::Some(source_transport_error(error.code()));
break 'source;
},
};
let block = match observed.value() {
std::option::Option::Some(value) => value,
std::option::Option::None => {
next_scan_slot = slot;
blocked_by_null = true;
break;
},
};
let received_at = match current_raw_timestamp() {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => {
fault = std::option::Option::Some(error);
break 'source;
},
};
let ingresses = project_http_block_polling_ingresses(&self, &settings, slot, block, &observed, received_at);
let ingresses = match ingresses {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => {
fault = std::option::Option::Some(error);
break 'source;
},
};
for ingress in ingresses {
let sent = tokio::select! {
biased;
_ = stop_receiver.changed() => {
break 'source;
}
result = admission_sender.send(ingress) => result,
};
if sent.is_err() {
if *stop_receiver.borrow() {
break 'source;
}
fault = std::option::Option::Some(crate::runtime_error("source.admission_closed"));
break 'source;
}
}
if let std::result::Result::Err(error) = processing_frontier.observe_settled(slot) {
fault = std::option::Option::Some(error);
break 'source;
}
}
if *stop_receiver.borrow() {
break;
}
if !blocked_by_null && let std::option::Option::Some(proven_end_slot) = discovery.proven_end_slot {
let coverage_result = {
let mut contracts = match continuity_contracts.lock() {
std::result::Result::Ok(value) => value,
std::result::Result::Err(poisoned) => poisoned.into_inner(),
};
contracts.record_coverage_epoch(self.source_key, window_start, proven_end_slot)
};
if let std::result::Result::Err(error) = coverage_result {
fault = std::option::Option::Some(error);
break;
}
processing_frontier.publish();
next_scan_slot = match proven_end_slot.checked_add(1) {
std::option::Option::Some(value) => value,
std::option::Option::None => {
fault = std::option::Option::Some(crate::runtime_error("source.http_block_polling_slot_exhausted"));
break;
},
};
}
}
tokio::select! {
biased;
_ = stop_receiver.changed() => {
break;
}
_ = tokio::time::sleep(self.poll_interval) => {},
}
}
processing_frontier.discard_all_pending();
processing_frontier.set_source_state(crate::RawTransactionIngestSourceState::Closing);
if let std::option::Option::Some(error) = fault {
processing_frontier.set_source_state(crate::RawTransactionIngestSourceState::Failed);
return std::result::Result::Err(error);
}
processing_frontier.set_source_state(crate::RawTransactionIngestSourceState::Closed);
return std::result::Result::Ok(());
}
}
impl std::fmt::Debug for crate::RawTransactionIngestHttpBlockPollingSource {
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
return formatter
.debug_struct("RawTransactionIngestHttpBlockPollingSource")
.field("polling_role", &self.polling_role.as_str())
.field("network", &self.network.as_str())
.field("commitment", &self.commitment)
.field("poll_interval_ms", &self.poll_interval.as_millis())
.field("max_discovered_blocks_per_cycle", &self.max_discovered_blocks_per_cycle)
.field("http_endpoint_count", &self.http_pool.snapshot().endpoint_count())
.field("profile_fingerprint_bytes", &self.profile_fingerprint.len())
.field("source_key_bytes", &self.source_key.len())
.finish();
}
}
/// Validated standard Solana `blockSubscribe` direct RAW source owned by the continuous RAW transaction ingest Worker.
///
/// The caller provides one Transport-owned standard WebSocket endpoint explicitly declaring `Block`, one block filter and a Confirmed/Finalized commitment.
/// The source requests Full/Base64 blocks with `maxSupportedTransactionVersion = 1`, admits only explicitly qualified Legacy/V0/V1 transactions, and treats
/// `block: null`, remote block errors,
/// unsupported versions and incomplete block transaction shapes as safe terminal source failures rather than empty progress.
pub struct RawTransactionIngestStandardBlockSource {
ws_endpoint: ksp_onchain_transport_lib::WsEndpointSettings,
filter: ksp_onchain_transport_lib::SolanaBlockSubscribeFilter,
commitment: ksp_onchain_transport_lib::SolanaCommitment,
network: ksp_store_lib::RawNetworkId,
route: RawTransactionIngestSourceRoute,
filter_fingerprint: [u8; 32],
source_key: [u8; 32],
}
impl crate::RawTransactionIngestStandardBlockSource {
/// Creates one validated standard Solana block source without opening WebSocket network I/O.
pub fn new(
ws_endpoint: ksp_onchain_transport_lib::WsEndpointSettings,
filter: ksp_onchain_transport_lib::SolanaBlockSubscribeFilter,
commitment: ksp_onchain_transport_lib::SolanaCommitment,
) -> ksp_core_lib::Result<Self> {
let ws_settings = ksp_onchain_transport_lib::WsTransportSettings::new(std::vec![ws_endpoint.clone()]);
if ws_settings.validate().is_err() {
return std::result::Result::Err(crate::runtime_error("runtime_resources.standard_block_endpoint_invalid"));
}
if ws_endpoint.protocol() != ksp_onchain_transport_lib::WsProtocolKind::SolanaStandard {
return std::result::Result::Err(crate::runtime_error("runtime_resources.standard_block_protocol_invalid"));
}
if !ws_endpoint.supports_subscription(ksp_onchain_transport_lib::WsSubscriptionKind::Block) {
return std::result::Result::Err(crate::runtime_error("runtime_resources.standard_block_capability_missing"));
}
match commitment {
ksp_onchain_transport_lib::SolanaCommitment::Confirmed | ksp_onchain_transport_lib::SolanaCommitment::Finalized => {},
ksp_onchain_transport_lib::SolanaCommitment::Processed => {
return std::result::Result::Err(crate::runtime_error("runtime_resources.standard_block_commitment_invalid"));
},
}
let network = match ksp_store_lib::RawNetworkId::new(ws_endpoint.cluster().as_str()) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return std::result::Result::Err(crate::runtime_error("runtime_resources.standard_block_network_unrepresentable")),
};
let provider = match ksp_store_lib::RawProvenanceCode::new(ws_endpoint.provider().as_str()) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return std::result::Result::Err(crate::runtime_error("runtime_resources.standard_block_provider_unrepresentable")),
};
let endpoint_id = match ksp_store_lib::RawProvenanceCode::new(ws_endpoint.name()) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return std::result::Result::Err(crate::runtime_error("runtime_resources.standard_block_endpoint_unrepresentable")),
};
let route = RawTransactionIngestSourceRoute { endpoint_id, provider };
let filter_fingerprint = standard_block_filter_fingerprint(&filter);
let source_key = standard_block_live_source_key(&network, &route, commitment, &filter_fingerprint);
return std::result::Result::Ok(Self { ws_endpoint, filter, commitment, network, route, filter_fingerprint, source_key });
}
/// Runs one productive standard `blockSubscribe` source until cooperative stop or one safe terminal source failure.
pub(crate) async fn run(
self,
settings: crate::RawTransactionIngestSettings,
mut stop_receiver: tokio::sync::watch::Receiver<bool>,
admission_sender: tokio::sync::mpsc::Sender<crate::RawTransactionIngress>,
processing_frontier_sender: tokio::sync::watch::Sender<crate::RawTransactionIngestProcessingFrontierProjection>,
) -> ksp_core_lib::Result<()> {
let connected = tokio::select! {
biased;
_ = stop_receiver.changed() => {
return std::result::Result::Ok(());
}
result = ksp_onchain_transport_lib::SolanaStandardWsSession::connect(self.ws_endpoint.clone()) => result,
};
let session = match connected {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(source_transport_error(error.code())),
};
let config = ksp_onchain_transport_lib::SolanaBlockSubscribeConfig::new(
std::option::Option::Some(self.commitment),
std::option::Option::Some(ksp_onchain_transport_lib::SolanaTransactionEncoding::Base64),
std::option::Option::Some(ksp_onchain_transport_lib::SolanaTransactionDetails::Full),
std::option::Option::Some(1),
std::option::Option::Some(false),
);
let subscribed = tokio::select! {
biased;
_ = stop_receiver.changed() => {
let _closed = session.close().await;
return std::result::Result::Ok(());
}
result = session.block_subscribe(&self.filter, std::option::Option::Some(&config)) => result,
};
let mut subscription = match subscribed {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => {
let _closed = session.close().await;
return std::result::Result::Err(source_transport_error(error.code()));
},
};
let mut session_snapshot_source = session.snapshot_source();
let mut processing_frontier = RawTransactionIngestProcessingFrontierReporter::new(processing_frontier_sender);
if let std::result::Result::Err(error) = processing_frontier.observe_websocket_session_snapshot(session_snapshot_source.current()) {
let _closed = session.close().await;
return std::result::Result::Err(error);
}
let mut fault = std::option::Option::None;
'source: loop {
if *stop_receiver.borrow() {
break;
}
let notification = tokio::select! {
biased;
_ = stop_receiver.changed() => {
break;
}
source_snapshot = session_snapshot_source.wait_for_change() => {
let source_snapshot = match source_snapshot {
std::option::Option::Some(value) => value,
std::option::Option::None => {
fault = std::option::Option::Some(crate::runtime_error("source.websocket_snapshot_closed"));
break;
},
};
if let std::result::Result::Err(error) = processing_frontier.observe_websocket_session_snapshot(source_snapshot) {
fault = std::option::Option::Some(error);
break;
}
continue;
}
value = subscription.recv() => value,
};
let notification = match notification {
std::option::Option::Some(std::result::Result::Ok(value)) => value,
std::option::Option::Some(std::result::Result::Err(error)) => {
fault = std::option::Option::Some(source_transport_error(error.code()));
break;
},
std::option::Option::None => {
let error = match subscription.terminal_error_code() {
std::option::Option::Some(code) => source_transport_error(code),
std::option::Option::None => crate::runtime_error("source.standard_block_subscription_closed"),
};
fault = std::option::Option::Some(error);
break;
},
};
let received_at = match current_raw_timestamp() {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => {
fault = std::option::Option::Some(error);
break;
},
};
let (slot, ingresses) = match project_standard_block_ingresses(&self, &settings, &notification, received_at) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => {
fault = std::option::Option::Some(error);
break;
},
};
let incident_bounded = match processing_frontier.observe_websocket_post_incident_slot(slot) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => {
fault = std::option::Option::Some(error);
break;
},
};
if ingresses.is_empty() {
if let std::result::Result::Err(error) = processing_frontier.observe_settled(slot) {
fault = std::option::Option::Some(error);
break;
}
if let std::option::Option::Some((start_slot, end_slot)) = incident_bounded {
fault = std::option::Option::Some(continuity_range_error("source.continuity_gap_proven", start_slot, end_slot));
break;
}
continue;
}
for ingress in ingresses {
let sent = tokio::select! {
biased;
_ = stop_receiver.changed() => {
break 'source;
}
result = admission_sender.send(ingress) => result,
};
if sent.is_err() {
if *stop_receiver.borrow() {
break 'source;
}
fault = std::option::Option::Some(crate::runtime_error("source.admission_closed"));
break 'source;
}
}
if let std::result::Result::Err(error) = processing_frontier.observe_settled(slot) {
fault = std::option::Option::Some(error);
break;
}
if let std::option::Option::Some((start_slot, end_slot)) = incident_bounded {
fault = std::option::Option::Some(continuity_range_error("source.continuity_gap_proven", start_slot, end_slot));
break;
}
}
processing_frontier.discard_all_pending();
processing_frontier.set_source_state(crate::RawTransactionIngestSourceState::Closing);
let closed = session.close().await;
if let std::option::Option::Some(error) = fault {
processing_frontier.set_source_state(crate::RawTransactionIngestSourceState::Failed);
return std::result::Result::Err(error);
}
return match closed {
std::result::Result::Ok(()) => {
processing_frontier.set_source_state(crate::RawTransactionIngestSourceState::Closed);
std::result::Result::Ok(())
},
std::result::Result::Err(error) => {
processing_frontier.set_source_state(crate::RawTransactionIngestSourceState::Failed);
std::result::Result::Err(source_transport_error(error.code()))
},
};
}
}
impl std::fmt::Debug for crate::RawTransactionIngestStandardBlockSource {
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
return formatter
.debug_struct("RawTransactionIngestStandardBlockSource")
.field("ws_endpoint_name", &self.ws_endpoint.name())
.field("ws_provider", &self.ws_endpoint.provider().as_str())
.field("network", &self.network.as_str())
.field("commitment", &self.commitment)
.field("filter_kind", &standard_block_filter_kind(&self.filter))
.field("filter_fingerprint_bytes", &self.filter_fingerprint.len())
.field("source_key_bytes", &self.source_key.len())
.finish();
}
}
/// Validated standard Solana `logsSubscribe` plus HTTP hydration source owned by the continuous RAW transaction ingest Worker.
///
/// The caller provides one Transport-owned standard WebSocket endpoint explicitly declaring `Logs`, one standard logs filter, a Confirmed/Finalized commitment
/// and one HTTP hydration pool/role. The source keeps the WebSocket URL and filter private, never copies remote logs/errors into Worker state, and emits only signature/slot
/// references into the source-neutral hydration coordinator.
pub struct RawTransactionIngestStandardLogsSource {
ws_endpoint: ksp_onchain_transport_lib::WsEndpointSettings,
filter: ksp_onchain_transport_lib::SolanaLogsSubscribeFilter,
commitment: ksp_onchain_transport_lib::SolanaCommitment,
http_pool: ksp_onchain_transport_lib::HttpTransportPool,
hydration_role: ksp_onchain_transport_lib::HttpRoleName,
network: ksp_store_lib::RawNetworkId,
route: RawTransactionIngestSourceRoute,
filter_fingerprint: [u8; 32],
source_key: [u8; 32],
}
impl crate::RawTransactionIngestStandardLogsSource {
/// Creates one validated standard Solana logs source without opening WebSocket or HTTP network I/O.
pub fn new(
ws_endpoint: ksp_onchain_transport_lib::WsEndpointSettings,
filter: ksp_onchain_transport_lib::SolanaLogsSubscribeFilter,
commitment: ksp_onchain_transport_lib::SolanaCommitment,
http_pool: ksp_onchain_transport_lib::HttpTransportPool,
hydration_role: ksp_onchain_transport_lib::HttpRoleName,
) -> ksp_core_lib::Result<Self> {
let ws_settings = ksp_onchain_transport_lib::WsTransportSettings::new(std::vec![ws_endpoint.clone()]);
if ws_settings.validate().is_err() {
return std::result::Result::Err(crate::runtime_error("runtime_resources.standard_logs_endpoint_invalid"));
}
if ws_endpoint.protocol() != ksp_onchain_transport_lib::WsProtocolKind::SolanaStandard {
return std::result::Result::Err(crate::runtime_error("runtime_resources.standard_logs_protocol_invalid"));
}
if !ws_endpoint.supports_subscription(ksp_onchain_transport_lib::WsSubscriptionKind::Logs) {
return std::result::Result::Err(crate::runtime_error("runtime_resources.standard_logs_capability_missing"));
}
match commitment {
ksp_onchain_transport_lib::SolanaCommitment::Confirmed | ksp_onchain_transport_lib::SolanaCommitment::Finalized => {},
ksp_onchain_transport_lib::SolanaCommitment::Processed => {
return std::result::Result::Err(crate::runtime_error("runtime_resources.hydration_commitment_invalid"));
},
}
let network = match ksp_store_lib::RawNetworkId::new(ws_endpoint.cluster().as_str()) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return std::result::Result::Err(crate::runtime_error("runtime_resources.standard_logs_network_unrepresentable")),
};
let provider = match ksp_store_lib::RawProvenanceCode::new(ws_endpoint.provider().as_str()) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return std::result::Result::Err(crate::runtime_error("runtime_resources.standard_logs_provider_unrepresentable")),
};
let endpoint_id = match ksp_store_lib::RawProvenanceCode::new(ws_endpoint.name()) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return std::result::Result::Err(crate::runtime_error("runtime_resources.standard_logs_endpoint_unrepresentable")),
};
let route = RawTransactionIngestSourceRoute { endpoint_id, provider };
let method = match ksp_onchain_transport_lib::find_http_rpc_method("getTransaction") {
std::option::Option::Some(value) => value,
std::option::Option::None => return std::result::Result::Err(crate::runtime_error("runtime_resources.hydration_method_missing")),
};
let compatible_http_routes = compatible_http_route_count(&http_pool, &hydration_role, method.request_kind(), network.as_str(), &route, "ws");
let compatible_http_routes = match compatible_http_routes {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
if compatible_http_routes == 0 {
return std::result::Result::Err(crate::runtime_error("runtime_resources.hydration_role_unsupported"));
}
let filter_fingerprint = standard_logs_filter_fingerprint(&filter);
let source_key = standard_logs_live_source_key(&network, &route, commitment, &filter_fingerprint);
return std::result::Result::Ok(Self {
ws_endpoint,
filter,
commitment,
http_pool,
hydration_role,
network,
route,
filter_fingerprint,
source_key,
});
}
fn hydration_context(&self) -> RawTransactionIngestHydrationContext {
return RawTransactionIngestHydrationContext {
commitment: self.commitment,
http_pool: self.http_pool.clone(),
hydration_role: self.hydration_role.clone(),
network: self.network.clone(),
protocol: RAW_TRANSACTION_INGEST_STANDARD_LOGS_HTTP_PROTOCOL,
route: self.route.clone(),
route_prefix: "ws",
source_key: self.source_key,
source_key_domain: RAW_TRANSACTION_INGEST_STANDARD_LOGS_HTTP_SOURCE_KEY_DOMAIN,
};
}
/// Runs one productive standard `logsSubscribe` source until cooperative stop or one safe terminal source failure.
async fn run(
self,
settings: crate::RawTransactionIngestSettings,
mut stop_receiver: tokio::sync::watch::Receiver<bool>,
admission_sender: tokio::sync::mpsc::Sender<crate::RawTransactionIngress>,
processing_frontier_sender: tokio::sync::watch::Sender<crate::RawTransactionIngestProcessingFrontierProjection>,
shared: RawTransactionIngestSourceRuntimeShared,
) -> ksp_core_lib::Result<()> {
let RawTransactionIngestSourceRuntimeShared { continuity_contracts, global_hydration_registry, hydration_in_flight_limit, hydration_pending_limit } =
shared;
let connected = tokio::select! {
biased;
_ = stop_receiver.changed() => {
return std::result::Result::Ok(());
}
result = ksp_onchain_transport_lib::SolanaStandardWsSession::connect(self.ws_endpoint.clone()) => result,
};
let session = match connected {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(source_transport_error(error.code())),
};
let config = ksp_onchain_transport_lib::SolanaCommitmentConfig::new(std::option::Option::Some(self.commitment));
let subscribed = tokio::select! {
biased;
_ = stop_receiver.changed() => {
let _closed = session.close().await;
return std::result::Result::Ok(());
}
result = session.logs_subscribe(&self.filter, std::option::Option::Some(&config)) => result,
};
let mut subscription = match subscribed {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => {
let _closed = session.close().await;
return std::result::Result::Err(source_transport_error(error.code()));
},
};
let mut session_snapshot_source = session.snapshot_source();
let hydration = self.hydration_context();
let mut coordinator =
RawTransactionIngestHydrationCoordinator::with_global_registry(global_hydration_registry, hydration_pending_limit, hydration_in_flight_limit);
let mut processing_frontier = RawTransactionIngestProcessingFrontierReporter::new(processing_frontier_sender);
if let std::result::Result::Err(error) = processing_frontier.observe_websocket_session_snapshot(session_snapshot_source.current()) {
let _closed = session.close().await;
return std::result::Result::Err(error);
}
let mut fault = std::option::Option::None;
let mut bounded_websocket_incident = std::option::Option::None;
loop {
if *stop_receiver.borrow() {
break;
}
if let std::result::Result::Err(error) = coordinator.start_hydrations(&hydration) {
fault = std::option::Option::Some(error);
break;
}
if let std::option::Option::Some((start_slot, end_slot)) = bounded_websocket_incident
&& coordinator.pending_signal_count == 0
&& coordinator.tasks.is_empty()
{
fault = std::option::Option::Some(continuity_range_error("source.continuity_gap_proven", start_slot, end_slot));
break;
}
let can_receive = coordinator.can_receive() && bounded_websocket_incident.is_none();
if !can_receive && coordinator.tasks.is_empty() {
fault = std::option::Option::Some(crate::runtime_error("source.hydration_stalled"));
break;
}
tokio::select! {
biased;
_ = stop_receiver.changed() => {
break;
}
source_snapshot = session_snapshot_source.wait_for_change() => {
let source_snapshot = match source_snapshot {
std::option::Option::Some(value) => value,
std::option::Option::None => {
fault = std::option::Option::Some(crate::runtime_error("source.websocket_snapshot_closed"));
break;
},
};
if let std::result::Result::Err(error) = processing_frontier.observe_websocket_session_snapshot(source_snapshot) {
fault = std::option::Option::Some(error);
break;
}
}
joined = coordinator.tasks.join_next(), if !coordinator.tasks.is_empty() => {
let joined = match joined {
std::option::Option::Some(value) => value,
std::option::Option::None => {
fault = std::option::Option::Some(crate::runtime_error("source.hydration_join_missing"));
break;
},
};
let handled = coordinator
.handle_joined(
joined,
&hydration,
&settings,
&admission_sender,
&mut stop_receiver,
&mut processing_frontier,
&continuity_contracts,
)
.await;
match handled {
std::result::Result::Ok(true) => {},
std::result::Result::Ok(false) => break,
std::result::Result::Err(error) => {
fault = std::option::Option::Some(error);
break;
},
}
}
notification = subscription.recv(), if can_receive => {
let notification = match notification {
std::option::Option::Some(std::result::Result::Ok(value)) => value,
std::option::Option::Some(std::result::Result::Err(error)) => {
fault = std::option::Option::Some(source_transport_error(error.code()));
break;
},
std::option::Option::None => {
let error = match subscription.terminal_error_code() {
std::option::Option::Some(code) => source_transport_error(code),
std::option::Option::None => crate::runtime_error("source.standard_logs_subscription_closed"),
};
fault = std::option::Option::Some(error);
break;
},
};
let received_at = match current_raw_timestamp() {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => {
fault = std::option::Option::Some(error);
break;
},
};
let signal = match project_standard_logs_signal(&self, &notification) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => {
fault = std::option::Option::Some(error);
break;
},
};
let incident_bounded = match processing_frontier.observe_websocket_post_incident_slot(signal.slot) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => {
fault = std::option::Option::Some(error);
break;
},
};
if let std::result::Result::Err(error) = coordinator.queue_signal(&hydration, signal, received_at, &mut processing_frontier) {
fault = std::option::Option::Some(error);
break;
}
if bounded_websocket_incident.is_none() {
bounded_websocket_incident = incident_bounded;
}
}
}
}
coordinator.abort_all(&mut processing_frontier).await;
processing_frontier.set_source_state(crate::RawTransactionIngestSourceState::Closing);
let closed = session.close().await;
if let std::option::Option::Some(error) = fault {
processing_frontier.set_source_state(crate::RawTransactionIngestSourceState::Failed);
return std::result::Result::Err(error);
}
return match closed {
std::result::Result::Ok(()) => {
processing_frontier.set_source_state(crate::RawTransactionIngestSourceState::Closed);
std::result::Result::Ok(())
},
std::result::Result::Err(error) => {
processing_frontier.set_source_state(crate::RawTransactionIngestSourceState::Failed);
std::result::Result::Err(source_transport_error(error.code()))
},
};
}
}
impl std::fmt::Debug for crate::RawTransactionIngestStandardLogsSource {
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
let http_snapshot = self.http_pool.snapshot();
return formatter
.debug_struct("RawTransactionIngestStandardLogsSource")
.field("ws_endpoint_name", &self.ws_endpoint.name())
.field("ws_provider", &self.ws_endpoint.provider().as_str())
.field("network", &self.network.as_str())
.field("commitment", &self.commitment)
.field("filter_kind", &standard_logs_filter_kind(&self.filter))
.field("filter_fingerprint_bytes", &self.filter_fingerprint.len())
.field("hydration_role", &self.hydration_role.as_str())
.field("http_endpoint_count", &http_snapshot.endpoint_count())
.field("source_key_bytes", &self.source_key.len())
.finish();
}
}
impl std::fmt::Debug for crate::RawTransactionIngestYellowstoneSource {
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
let http_snapshot = self.http_pool.snapshot();
return formatter
.debug_struct("RawTransactionIngestYellowstoneSource")
.field("yellowstone_endpoint_name", &self.yellowstone_channel.endpoint_name())
.field("yellowstone_provider", &self.yellowstone_channel.provider().as_str())
.field("network", &self.network.as_str())
.field("route", &self.route)
.field("transaction_filter_count", &self.subscribe_request.transaction_filter_count())
.field("transaction_status_filter_count", &self.subscribe_request.transaction_status_filter_count())
.field("block_filter_count", &self.subscribe_request.block_filter_count())
.field("blocks_meta_filter_count", &self.subscribe_request.blocks_meta_filter_count())
.field("slot_filter_count", &self.subscribe_request.slot_filter_count())
.field("commitment", &self.subscribe_request.commitment())
.field("has_from_slot", &self.subscribe_request.from_slot().is_some())
.field("hydration_role", &self.hydration_role.as_str())
.field("http_endpoint_count", &http_snapshot.endpoint_count())
.field("source_key_bytes", &self.source_key.len())
.finish();
}
}
/// Caller-composed runtime resources accepted by the continuous RAW transaction ingest Worker.
///
/// The aggregate owns a bounded collection of capability-specific live-source contracts. All validated sources are started together by one private supervisor;
/// source identities and per-source lifecycle remain private while the existing Worker snapshot receives only a conservative source-neutral aggregate.
pub struct RawTransactionIngestRuntimeResources {
sources: std::vec::Vec<RawTransactionIngestLiveSource>,
}
impl crate::RawTransactionIngestRuntimeResources {
/// Creates one runtime-resource aggregate from the already productive Yellowstone source contract.
#[must_use]
pub fn new(yellowstone_source: crate::RawTransactionIngestYellowstoneSource) -> Self {
return Self { sources: std::vec![RawTransactionIngestLiveSource::Yellowstone(yellowstone_source)] };
}
/// Creates one runtime-resource aggregate from one Helius `transactionSubscribe` plus HTTP hydration source contract.
#[must_use]
pub fn from_helius_transaction_source(source: crate::RawTransactionIngestHeliusTransactionSource) -> Self {
return Self { sources: std::vec![RawTransactionIngestLiveSource::HeliusTransaction(source)] };
}
/// Creates one runtime-resource aggregate from one standard Solana HTTP live block polling source contract.
#[must_use]
pub fn from_http_block_polling_source(source: crate::RawTransactionIngestHttpBlockPollingSource) -> Self {
return Self { sources: std::vec![RawTransactionIngestLiveSource::HttpBlockPolling(source)] };
}
/// Creates one runtime-resource aggregate from one standard Solana `blockSubscribe` direct RAW source contract.
#[must_use]
pub fn from_standard_block_source(source: crate::RawTransactionIngestStandardBlockSource) -> Self {
return Self { sources: std::vec![RawTransactionIngestLiveSource::StandardBlock(source)] };
}
/// Creates one runtime-resource aggregate from one standard Solana `logsSubscribe` plus HTTP hydration source contract.
#[must_use]
pub fn from_standard_logs_source(source: crate::RawTransactionIngestStandardLogsSource) -> Self {
return Self { sources: std::vec![RawTransactionIngestLiveSource::StandardLogs(source)] };
}
/// Returns the number of validated logical live sources currently owned by this aggregate.
#[must_use]
pub fn source_count(&self) -> usize {
return self.sources.len();
}
/// Adds one validated Yellowstone source while preserving the global 1..32 bound, one-network invariant and unique logical source identity.
pub fn try_push_yellowstone_source(&mut self, yellowstone_source: crate::RawTransactionIngestYellowstoneSource) -> ksp_core_lib::Result<()> {
if self.sources.len() >= crate::MAX_RAW_TRANSACTION_INGEST_LIVE_SOURCES {
return std::result::Result::Err(crate::runtime_error("runtime_resources.source_count_exceeded"));
}
let candidate = RawTransactionIngestLiveSource::Yellowstone(yellowstone_source);
let expected_network = match self.sources.first() {
std::option::Option::Some(source) => source.network(),
std::option::Option::None => return std::result::Result::Err(crate::runtime_error("runtime_resources.source_collection_empty")),
};
if candidate.network() != expected_network {
return std::result::Result::Err(crate::runtime_error("runtime_resources.source_network_mismatch"));
}
let source_key = candidate.source_key();
if self.sources.iter().any(|source| return source.source_key() == source_key) {
return std::result::Result::Err(crate::runtime_error("runtime_resources.duplicate_source_identity"));
}
self.sources.push(candidate);
return std::result::Result::Ok(());
}
/// Adds one validated Helius transaction source while preserving the global 1..32 bound, one-network invariant and unique logical source identity.
pub fn try_push_helius_transaction_source(&mut self, source: crate::RawTransactionIngestHeliusTransactionSource) -> ksp_core_lib::Result<()> {
if self.sources.len() >= crate::MAX_RAW_TRANSACTION_INGEST_LIVE_SOURCES {
return std::result::Result::Err(crate::runtime_error("runtime_resources.source_count_exceeded"));
}
let candidate = RawTransactionIngestLiveSource::HeliusTransaction(source);
let expected_network = match self.sources.first() {
std::option::Option::Some(source) => source.network(),
std::option::Option::None => return std::result::Result::Err(crate::runtime_error("runtime_resources.source_collection_empty")),
};
if candidate.network() != expected_network {
return std::result::Result::Err(crate::runtime_error("runtime_resources.source_network_mismatch"));
}
let source_key = candidate.source_key();
if self.sources.iter().any(|source| return source.source_key() == source_key) {
return std::result::Result::Err(crate::runtime_error("runtime_resources.duplicate_source_identity"));
}
self.sources.push(candidate);
return std::result::Result::Ok(());
}
/// Adds one validated HTTP live block polling source while preserving the global 1..32 bound, one-network invariant and unique logical source identity.
pub fn try_push_http_block_polling_source(&mut self, source: crate::RawTransactionIngestHttpBlockPollingSource) -> ksp_core_lib::Result<()> {
if self.sources.len() >= crate::MAX_RAW_TRANSACTION_INGEST_LIVE_SOURCES {
return std::result::Result::Err(crate::runtime_error("runtime_resources.source_count_exceeded"));
}
let candidate = RawTransactionIngestLiveSource::HttpBlockPolling(source);
let expected_network = match self.sources.first() {
std::option::Option::Some(source) => source.network(),
std::option::Option::None => return std::result::Result::Err(crate::runtime_error("runtime_resources.source_collection_empty")),
};
if candidate.network() != expected_network {
return std::result::Result::Err(crate::runtime_error("runtime_resources.source_network_mismatch"));
}
let source_key = candidate.source_key();
if self.sources.iter().any(|source| return source.source_key() == source_key) {
return std::result::Result::Err(crate::runtime_error("runtime_resources.duplicate_source_identity"));
}
self.sources.push(candidate);
return std::result::Result::Ok(());
}
/// Adds one validated standard Solana block source while preserving the global 1..32 bound, one-network invariant and unique logical source identity.
pub fn try_push_standard_block_source(&mut self, source: crate::RawTransactionIngestStandardBlockSource) -> ksp_core_lib::Result<()> {
if self.sources.len() >= crate::MAX_RAW_TRANSACTION_INGEST_LIVE_SOURCES {
return std::result::Result::Err(crate::runtime_error("runtime_resources.source_count_exceeded"));
}
let candidate = RawTransactionIngestLiveSource::StandardBlock(source);
let expected_network = match self.sources.first() {
std::option::Option::Some(source) => source.network(),
std::option::Option::None => return std::result::Result::Err(crate::runtime_error("runtime_resources.source_collection_empty")),
};
if candidate.network() != expected_network {
return std::result::Result::Err(crate::runtime_error("runtime_resources.source_network_mismatch"));
}
let source_key = candidate.source_key();
if self.sources.iter().any(|source| return source.source_key() == source_key) {
return std::result::Result::Err(crate::runtime_error("runtime_resources.duplicate_source_identity"));
}
self.sources.push(candidate);
return std::result::Result::Ok(());
}
/// Adds one validated standard Solana logs source while preserving the global 1..32 bound, one-network invariant and unique logical source identity.
pub fn try_push_standard_logs_source(&mut self, source: crate::RawTransactionIngestStandardLogsSource) -> ksp_core_lib::Result<()> {
if self.sources.len() >= crate::MAX_RAW_TRANSACTION_INGEST_LIVE_SOURCES {
return std::result::Result::Err(crate::runtime_error("runtime_resources.source_count_exceeded"));
}
let candidate = RawTransactionIngestLiveSource::StandardLogs(source);
let expected_network = match self.sources.first() {
std::option::Option::Some(source) => source.network(),
std::option::Option::None => return std::result::Result::Err(crate::runtime_error("runtime_resources.source_collection_empty")),
};
if candidate.network() != expected_network {
return std::result::Result::Err(crate::runtime_error("runtime_resources.source_network_mismatch"));
}
let source_key = candidate.source_key();
if self.sources.iter().any(|source| return source.source_key() == source_key) {
return std::result::Result::Err(crate::runtime_error("runtime_resources.duplicate_source_identity"));
}
self.sources.push(candidate);
return std::result::Result::Ok(());
}
/// Validates that caller-owned Worker settings target the same logical network as every composed live source.
pub(crate) fn validate_network(&self, network: &ksp_store_lib::RawNetworkId) -> ksp_core_lib::Result<()> {
if self.sources.is_empty() || self.sources.len() > crate::MAX_RAW_TRANSACTION_INGEST_LIVE_SOURCES {
return std::result::Result::Err(crate::runtime_error("runtime_resources.source_collection_invalid"));
}
let mut source_keys = std::collections::BTreeSet::new();
for source in &self.sources {
if source.network() != network {
return std::result::Result::Err(crate::runtime_error("runtime_resources.worker_network_mismatch"));
}
if !source_keys.insert(source.source_key()) {
return std::result::Result::Err(crate::runtime_error("runtime_resources.duplicate_source_identity"));
}
}
return std::result::Result::Ok(());
}
/// Runs every validated source concurrently under one private bounded source supervisor.
pub(crate) async fn run_live_sources(
self,
settings: crate::RawTransactionIngestSettings,
stop_receiver: tokio::sync::watch::Receiver<bool>,
admission_sender: tokio::sync::mpsc::Sender<crate::RawTransactionIngress>,
processing_frontier_sender: tokio::sync::watch::Sender<crate::RawTransactionIngestProcessingFrontierProjection>,
) -> ksp_core_lib::Result<()> {
if self.sources.is_empty() || self.sources.len() > crate::MAX_RAW_TRANSACTION_INGEST_LIVE_SOURCES {
return std::result::Result::Err(crate::runtime_error("runtime_resources.source_collection_invalid"));
}
let mut continuity_capabilities = std::vec::Vec::with_capacity(self.sources.len());
for source in &self.sources {
let capability = match source.repair_capability_descriptor() {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
continuity_capabilities.push(capability);
}
let continuity_contracts = match crate::RawTransactionIngestContinuityContracts::new(continuity_capabilities) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let continuity_source_count = self.sources.len();
if let std::result::Result::Err(error) = continuity_contracts.validate_for_source_count(continuity_source_count) {
return std::result::Result::Err(error);
}
let continuity_contracts = std::sync::Arc::new(std::sync::Mutex::new(continuity_contracts));
let source_keys = self.sources.iter().map(RawTransactionIngestLiveSource::source_key).collect::<std::vec::Vec<_>>();
let hydration_source_count = self.sources.iter().filter(|source| return source.uses_hydration()).count();
if let std::result::Result::Err(error) =
validate_hydration_fairness_capacity(settings.admission_queue_capacity(), settings.persistence_concurrency(), hydration_source_count)
{
return std::result::Result::Err(error);
}
if let std::result::Result::Err(error) = validate_repair_fairness_contract(settings.admission_queue_capacity(), settings.persistence_concurrency()) {
return std::result::Result::Err(error);
}
let hydration_pending_budget = settings.admission_queue_capacity();
let hydration_in_flight_budget = settings.persistence_concurrency();
let inventory = std::sync::Arc::new(std::sync::Mutex::new(RawTransactionIngestSourceInventory::new(source_keys)));
let global_hydration_registry =
std::sync::Arc::new(RawTransactionIngestGlobalHydrationRegistry::new(hydration_pending_budget, hydration_in_flight_budget));
let (source_stop_sender, source_stop_receiver) = tokio::sync::watch::channel(false);
let mut children = tokio::task::JoinSet::new();
let mut hydration_entry_index = 0_usize;
for (entry_index, source) in self.sources.into_iter().enumerate() {
let (hydration_pending_limit, hydration_in_flight_limit) = if source.uses_hydration() {
let pending_limit = hydration_pending_limit(hydration_pending_budget, hydration_source_count, hydration_entry_index);
let in_flight_limit = hydration_in_flight_limit(hydration_in_flight_budget, hydration_source_count, hydration_entry_index);
hydration_entry_index += 1;
(pending_limit, in_flight_limit)
} else {
(0, 0)
};
let publisher = RawTransactionIngestSourceInventoryPublisher {
aggregate_sender: processing_frontier_sender.clone(),
continuity_contracts: std::sync::Arc::clone(&continuity_contracts),
entry_index,
inventory: std::sync::Arc::clone(&inventory),
source_key: source.source_key(),
};
let source_admission_sender = admission_sender.clone();
let source_settings = settings.clone();
let source_stop_receiver = source_stop_receiver.clone();
let source_shared = RawTransactionIngestSourceRuntimeShared {
continuity_contracts: std::sync::Arc::clone(&continuity_contracts),
global_hydration_registry: std::sync::Arc::clone(&global_hydration_registry),
hydration_in_flight_limit,
hydration_pending_limit,
};
let source_key = source.source_key();
let _abort_handle = children.spawn(async move {
let result = source.run(source_settings, source_stop_receiver, source_admission_sender, publisher, source_shared).await;
return (source_key, result);
});
}
std::mem::drop(admission_sender);
let result = supervise_live_source_tasks(
stop_receiver,
source_stop_sender,
children,
std::sync::Arc::clone(&continuity_contracts),
std::sync::Arc::clone(&inventory),
processing_frontier_sender,
settings.shutdown_drain_timeout(),
)
.await;
let validation = {
let contracts = match continuity_contracts.lock() {
std::result::Result::Ok(value) => value,
std::result::Result::Err(poisoned) => poisoned.into_inner(),
};
contracts.validate_for_source_count(continuity_source_count)
};
if let std::result::Result::Err(error) = validation {
return std::result::Result::Err(error);
}
return result;
}
}
fn validate_hydration_fairness_capacity(
admission_queue_capacity: usize,
persistence_concurrency: usize,
hydration_source_count: usize,
) -> ksp_core_lib::Result<()> {
if hydration_source_count > admission_queue_capacity {
return std::result::Result::Err(crate::runtime_error("runtime_resources.hydration_pending_capacity_insufficient"));
}
if hydration_source_count > persistence_concurrency {
return std::result::Result::Err(crate::runtime_error("runtime_resources.hydration_concurrency_insufficient"));
}
return std::result::Result::Ok(());
}
fn hydration_pending_limit(total_budget: usize, hydration_source_count: usize, hydration_entry_index: usize) -> usize {
if hydration_source_count == 0 || hydration_entry_index >= hydration_source_count {
return 0;
}
let base = total_budget / hydration_source_count;
let remainder = total_budget % hydration_source_count;
return base + usize::from(hydration_entry_index < remainder);
}
fn hydration_in_flight_limit(total_budget: usize, hydration_source_count: usize, hydration_entry_index: usize) -> usize {
return hydration_pending_limit(total_budget, hydration_source_count, hydration_entry_index);
}
impl std::fmt::Debug for crate::RawTransactionIngestRuntimeResources {
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
return formatter.debug_struct("RawTransactionIngestRuntimeResources").field("source_count", &self.sources.len()).finish();
}
}
struct RawTransactionIngestSourceKeyHashWriter<'a> {
hasher: &'a mut sha2::Sha256,
}
impl std::hash::Hasher for RawTransactionIngestSourceKeyHashWriter<'_> {
fn finish(&self) -> u64 {
return 0;
}
fn write(&mut self, bytes: &[u8]) {
self.hasher.update(bytes);
return;
}
}
fn minimum_optional_slot(first: std::option::Option<u64>, second: std::option::Option<u64>) -> std::option::Option<u64> {
return match (first, second) {
(std::option::Option::Some(left), std::option::Option::Some(right)) => std::option::Option::Some(left.min(right)),
(std::option::Option::Some(value), std::option::Option::None) | (std::option::Option::None, std::option::Option::Some(value)) => {
std::option::Option::Some(value)
},
(std::option::Option::None, std::option::Option::None) => std::option::Option::None,
};
}
fn source_projection_with_state(
projection: crate::RawTransactionIngestProcessingFrontierProjection,
state: crate::RawTransactionIngestSourceState,
) -> crate::RawTransactionIngestProcessingFrontierProjection {
return crate::RawTransactionIngestProcessingFrontierProjection::new(
projection.hydration_pending(),
projection.processing_frontier_slot(),
projection.oldest_pending_slot(),
)
.with_source_continuity(
std::option::Option::Some(state),
projection.source_reconnect_total(),
projection.source_replay_attempt_total(),
projection.source_continuity_gap_total(),
);
}
async fn supervise_live_source_tasks(
mut stop_receiver: tokio::sync::watch::Receiver<bool>,
source_stop_sender: tokio::sync::watch::Sender<bool>,
mut children: tokio::task::JoinSet<([u8; 32], ksp_core_lib::Result<()>)>,
continuity_contracts: std::sync::Arc<std::sync::Mutex<crate::RawTransactionIngestContinuityContracts>>,
inventory: std::sync::Arc<std::sync::Mutex<RawTransactionIngestSourceInventory>>,
processing_frontier_sender: tokio::sync::watch::Sender<crate::RawTransactionIngestProcessingFrontierProjection>,
shutdown_drain_timeout: std::time::Duration,
) -> ksp_core_lib::Result<()> {
loop {
if *stop_receiver.borrow() {
source_stop_sender.send_replace(true);
return drain_live_source_tasks(&mut children, std::option::Option::None, shutdown_drain_timeout).await;
}
let joined = tokio::select! {
biased;
changed = stop_receiver.changed() => {
if changed.is_err() || *stop_receiver.borrow() {
source_stop_sender.send_replace(true);
return drain_live_source_tasks(&mut children, std::option::Option::None, shutdown_drain_timeout).await;
}
continue;
}
value = children.join_next(), if !children.is_empty() => value,
};
let (source_key, source_result) = match joined {
std::option::Option::Some(std::result::Result::Ok(value)) => value,
std::option::Option::Some(std::result::Result::Err(_)) => {
source_stop_sender.send_replace(true);
return drain_live_source_tasks(
&mut children,
std::option::Option::Some(crate::runtime_error("source.task_join_failed")),
shutdown_drain_timeout,
)
.await;
},
std::option::Option::None => {
source_stop_sender.send_replace(true);
return drain_live_source_tasks(
&mut children,
std::option::Option::Some(crate::runtime_error("source.task_set_empty")),
shutdown_drain_timeout,
)
.await;
},
};
let source_fault = match source_result {
std::result::Result::Ok(()) => crate::runtime_error("source.configured_source_closed"),
std::result::Result::Err(error) => error,
};
if !source_loss_is_reconcilable(&source_fault) {
source_stop_sender.send_replace(true);
return drain_live_source_tasks(&mut children, std::option::Option::Some(source_fault), shutdown_drain_timeout).await;
}
let supervisor_state = {
let inventory = match inventory.lock() {
std::result::Result::Ok(value) => value,
std::result::Result::Err(poisoned) => poisoned.into_inner(),
};
inventory.supervisor_state()
};
let (active_source_keys, processing_frontier_slot) = match supervisor_state {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => {
source_stop_sender.send_replace(true);
return drain_live_source_tasks(&mut children, std::option::Option::Some(error), shutdown_drain_timeout).await;
},
};
let continuity_range = match source_loss_continuity_range(&source_fault) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => {
source_stop_sender.send_replace(true);
return drain_live_source_tasks(&mut children, std::option::Option::Some(error), shutdown_drain_timeout).await;
},
};
let continuity_range = match continuity_range {
std::option::Option::Some(value) => value,
std::option::Option::None => {
source_stop_sender.send_replace(true);
return drain_live_source_tasks(&mut children, std::option::Option::Some(source_fault), shutdown_drain_timeout).await;
},
};
let decision = {
let mut contracts = match continuity_contracts.lock() {
std::result::Result::Ok(value) => value,
std::result::Result::Err(poisoned) => poisoned.into_inner(),
};
if let std::result::Result::Err(error) = contracts.record_source_loss_gap(source_key, continuity_range.0, continuity_range.1) {
std::result::Result::Err(error)
} else {
contracts.source_loss_decision(source_key, active_source_keys.as_slice(), processing_frontier_slot)
}
};
let decision = match decision {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => {
source_stop_sender.send_replace(true);
return drain_live_source_tasks(&mut children, std::option::Option::Some(error), shutdown_drain_timeout).await;
},
};
match decision {
crate::RawTransactionIngestSourceLossDecision::Continue => {
let aggregate = match source_inventory_health_projection(&inventory, &continuity_contracts) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => {
source_stop_sender.send_replace(true);
return drain_live_source_tasks(&mut children, std::option::Option::Some(error), shutdown_drain_timeout).await;
},
};
processing_frontier_sender.send_replace(aggregate);
continue;
},
crate::RawTransactionIngestSourceLossDecision::Fault => {
source_stop_sender.send_replace(true);
return drain_live_source_tasks(&mut children, std::option::Option::Some(source_fault), shutdown_drain_timeout).await;
},
}
}
}
fn continuity_range_error(condition: &'static str, start_slot: u64, end_slot: u64) -> ksp_core_lib::Error {
return crate::runtime_error(condition)
.with_context("continuity_start_slot", start_slot.to_string())
.with_context("continuity_end_slot", end_slot.to_string());
}
fn source_loss_continuity_range(error: &ksp_core_lib::Error) -> ksp_core_lib::Result<std::option::Option<(u64, u64)>> {
let mut start_slot = std::option::Option::None;
let mut end_slot = std::option::Option::None;
for context in error.context() {
if context.key() == "continuity_start_slot" {
start_slot = match context.value().parse::<u64>() {
std::result::Result::Ok(value) => std::option::Option::Some(value),
std::result::Result::Err(_) => return std::result::Result::Err(crate::runtime_error("continuity.source_loss_range_invalid")),
};
} else if context.key() == "continuity_end_slot" {
end_slot = match context.value().parse::<u64>() {
std::result::Result::Ok(value) => std::option::Option::Some(value),
std::result::Result::Err(_) => return std::result::Result::Err(crate::runtime_error("continuity.source_loss_range_invalid")),
};
}
}
return match (start_slot, end_slot) {
(std::option::Option::None, std::option::Option::None) => std::result::Result::Ok(std::option::Option::None),
(std::option::Option::Some(start), std::option::Option::Some(end)) if end >= start => std::result::Result::Ok(std::option::Option::Some((start, end))),
_ => std::result::Result::Err(crate::runtime_error("continuity.source_loss_range_invalid")),
};
}
fn source_loss_is_reconcilable(error: &ksp_core_lib::Error) -> bool {
if error.code() == crate::ERROR_CODE_RAW_TRANSACTION_INGEST_SOURCE_FAILED {
return true;
}
if error.code() != crate::ERROR_CODE_RAW_TRANSACTION_INGEST_RUNTIME_INVALID {
return false;
}
return error.context().iter().any(|context| {
if context.key() != "condition" {
return false;
}
return matches!(
context.value(),
"source.configured_source_closed" | "source.continuity_gap_proven" | "source.replay_coverage_unproven" | "source.websocket_incident_unbounded"
);
});
}
async fn drain_live_source_tasks(
children: &mut tokio::task::JoinSet<([u8; 32], ksp_core_lib::Result<()>)>,
mut first_fault: std::option::Option<ksp_core_lib::Error>,
shutdown_drain_timeout: std::time::Duration,
) -> ksp_core_lib::Result<()> {
let drain = async {
while let std::option::Option::Some(joined) = children.join_next().await {
if first_fault.is_some() {
continue;
}
first_fault = match joined {
std::result::Result::Ok((_source_key, std::result::Result::Ok(()))) => std::option::Option::None,
std::result::Result::Ok((_source_key, std::result::Result::Err(error))) => std::option::Option::Some(error),
std::result::Result::Err(_) => std::option::Option::Some(crate::runtime_error("source.task_join_failed")),
};
}
};
if tokio::time::timeout(shutdown_drain_timeout, drain).await.is_err() {
children.abort_all();
while children.join_next().await.is_some() {}
if first_fault.is_none() {
first_fault = std::option::Option::Some(ksp_core_lib::Error::new(
crate::ERROR_CODE_RAW_TRANSACTION_INGEST_DRAIN_TIMEOUT,
"RAW transaction ingest source drain timed out",
));
}
}
return match first_fault {
std::option::Option::Some(error) => std::result::Result::Err(error),
std::option::Option::None => std::result::Result::Ok(()),
};
}
fn http_role_scan_capabilities(
pool: &ksp_onchain_transport_lib::HttpTransportPool,
role: &ksp_onchain_transport_lib::HttpRoleName,
expected_cluster: &str,
) -> ksp_core_lib::Result<RawTransactionIngestHttpScanCapabilities> {
let get_block = match http_role_supports_rpc_method(pool, role, "getBlock", expected_cluster) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let get_blocks = match http_role_supports_rpc_method(pool, role, "getBlocks", expected_cluster) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let get_blocks_with_limit = match http_role_supports_rpc_method(pool, role, "getBlocksWithLimit", expected_cluster) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let get_slot = match http_role_supports_rpc_method(pool, role, "getSlot", expected_cluster) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
return std::result::Result::Ok(RawTransactionIngestHttpScanCapabilities { get_block, get_blocks, get_blocks_with_limit, get_slot });
}
fn http_role_supports_repair_scan(
pool: &ksp_onchain_transport_lib::HttpTransportPool,
role: &ksp_onchain_transport_lib::HttpRoleName,
expected_cluster: &str,
) -> ksp_core_lib::Result<bool> {
let capabilities = match http_role_scan_capabilities(pool, role, expected_cluster) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
return std::result::Result::Ok(capabilities.can_scan());
}
fn http_role_supports_rpc_method(
pool: &ksp_onchain_transport_lib::HttpTransportPool,
role: &ksp_onchain_transport_lib::HttpRoleName,
method_name: &'static str,
expected_cluster: &str,
) -> ksp_core_lib::Result<bool> {
let method = match ksp_onchain_transport_lib::find_http_rpc_method(method_name) {
std::option::Option::Some(value) => value,
std::option::Option::None => return std::result::Result::Err(crate::runtime_error("continuity.http_method_missing")),
};
let snapshot = pool.snapshot();
for endpoint in snapshot.endpoints() {
if !endpoint.enabled() || endpoint.cluster() != expected_cluster {
continue;
}
for endpoint_role in endpoint.roles() {
if !endpoint_role.enabled() || endpoint_role.role() != role.as_str() {
continue;
}
if http_role_supports_request_kind(endpoint_role, method.request_kind()) {
return std::result::Result::Ok(true);
}
}
}
return std::result::Result::Ok(false);
}
fn http_block_polling_live_source_key(
network: &ksp_store_lib::RawNetworkId,
polling_role: &ksp_onchain_transport_lib::HttpRoleName,
commitment: ksp_onchain_transport_lib::SolanaCommitment,
profile_fingerprint: &[u8; 32],
) -> [u8; 32] {
let mut hasher = sha2::Sha256::new();
hasher.update(RAW_TRANSACTION_INGEST_LIVE_SOURCE_KEY_DOMAIN);
hash_live_source_key_component(&mut hasher, b"http_block_polling");
hash_live_source_key_component(&mut hasher, network.as_str().as_bytes());
hash_live_source_key_component(&mut hasher, polling_role.as_str().as_bytes());
hash_live_source_key_component(&mut hasher, commitment.as_str().as_bytes());
hash_live_source_key_component(&mut hasher, profile_fingerprint);
return hasher.finalize().into();
}
fn validate_http_block_polling_profile(
http_pool: &ksp_onchain_transport_lib::HttpTransportPool,
polling_role: &ksp_onchain_transport_lib::HttpRoleName,
) -> ksp_core_lib::Result<(ksp_store_lib::RawNetworkId, [u8; 32])> {
let get_block = match ksp_onchain_transport_lib::find_http_rpc_method("getBlock") {
std::option::Option::Some(value) => value,
std::option::Option::None => return std::result::Result::Err(crate::runtime_error("runtime_resources.http_block_polling_get_block_missing")),
};
let get_blocks = match ksp_onchain_transport_lib::find_http_rpc_method("getBlocksWithLimit") {
std::option::Option::Some(value) => value,
std::option::Option::None => return std::result::Result::Err(crate::runtime_error("runtime_resources.http_block_polling_get_blocks_missing")),
};
let get_slot = match ksp_onchain_transport_lib::find_http_rpc_method("getSlot") {
std::option::Option::Some(value) => value,
std::option::Option::None => return std::result::Result::Err(crate::runtime_error("runtime_resources.http_block_polling_get_slot_missing")),
};
let mut has_get_block = false;
let mut has_get_blocks = false;
let mut has_get_slot = false;
let mut expected_cluster: std::option::Option<&str> = std::option::Option::None;
let mut routes = std::vec::Vec::new();
let snapshot = http_pool.snapshot();
for endpoint in snapshot.endpoints() {
if !endpoint.enabled() {
continue;
}
for role in endpoint.roles() {
if !role.enabled() || role.role() != polling_role.as_str() {
continue;
}
let supports_get_block = http_role_supports_request_kind(role, get_block.request_kind());
let supports_get_blocks = http_role_supports_request_kind(role, get_blocks.request_kind());
let supports_get_slot = http_role_supports_request_kind(role, get_slot.request_kind());
if !supports_get_block && !supports_get_blocks && !supports_get_slot {
continue;
}
match expected_cluster {
std::option::Option::Some(cluster) if cluster != endpoint.cluster() => {
return std::result::Result::Err(crate::runtime_error("runtime_resources.transport_network_mismatch"));
},
std::option::Option::Some(_) => {},
std::option::Option::None => expected_cluster = std::option::Option::Some(endpoint.cluster()),
}
if ksp_store_lib::RawProvenanceCode::new(endpoint.provider()).is_err() {
return std::result::Result::Err(crate::runtime_error("runtime_resources.http_block_polling_provider_unrepresentable"));
}
if ksp_store_lib::RawProvenanceCode::new(endpoint.name()).is_err() {
return std::result::Result::Err(crate::runtime_error("runtime_resources.http_block_polling_endpoint_unrepresentable"));
}
has_get_block |= supports_get_block;
has_get_blocks |= supports_get_blocks;
has_get_slot |= supports_get_slot;
routes.push((
endpoint.cluster().to_owned(),
endpoint.provider().to_owned(),
endpoint.name().to_owned(),
supports_get_block,
supports_get_blocks,
supports_get_slot,
));
}
}
if !has_get_slot || !has_get_blocks || !has_get_block {
return std::result::Result::Err(crate::runtime_error("runtime_resources.http_block_polling_role_unsupported"));
}
let cluster = match expected_cluster {
std::option::Option::Some(value) => value,
std::option::Option::None => return std::result::Result::Err(crate::runtime_error("runtime_resources.http_block_polling_role_unsupported")),
};
let network = match ksp_store_lib::RawNetworkId::new(cluster) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => {
return std::result::Result::Err(crate::runtime_error("runtime_resources.http_block_polling_network_unrepresentable"));
},
};
routes.sort_unstable();
let mut hasher = sha2::Sha256::new();
hasher.update(RAW_TRANSACTION_INGEST_HTTP_BLOCK_POLLING_PROFILE_FINGERPRINT_DOMAIN);
hash_live_source_key_component(&mut hasher, polling_role.as_str().as_bytes());
hash_live_source_key_component(&mut hasher, &(routes.len() as u64).to_be_bytes());
for (cluster, provider, endpoint, supports_get_block, supports_get_blocks, supports_get_slot) in routes {
hash_live_source_key_component(&mut hasher, cluster.as_bytes());
hash_live_source_key_component(&mut hasher, provider.as_bytes());
hash_live_source_key_component(&mut hasher, endpoint.as_bytes());
hash_live_source_key_component(&mut hasher, if supports_get_block { b"1" } else { b"0" });
hash_live_source_key_component(&mut hasher, if supports_get_blocks { b"1" } else { b"0" });
hash_live_source_key_component(&mut hasher, if supports_get_slot { b"1" } else { b"0" });
}
return std::result::Result::Ok((network, hasher.finalize().into()));
}
fn http_role_supports_request_kind(role: &ksp_onchain_transport_lib::HttpEndpointRoleSnapshot, request_kind: &str) -> bool {
return role.request_kinds().iter().any(|kind| return kind.as_str() == "*" || kind.as_str() == request_kind);
}
fn http_discovery_window_slot_count(start_slot: u64, end_slot: u64) -> ksp_core_lib::Result<u64> {
if end_slot < start_slot {
return std::result::Result::Err(crate::runtime_error("continuity.http_discovery_range_reversed"));
}
let slot_count = match end_slot.checked_sub(start_slot).and_then(|value| return value.checked_add(1)) {
std::option::Option::Some(value) => value,
std::option::Option::None => return std::result::Result::Err(crate::runtime_error("continuity.http_discovery_range_overflow")),
};
if slot_count > crate::MAX_RAW_TRANSACTION_INGEST_CONTINUITY_DISCOVERY_WINDOW_SLOTS {
return std::result::Result::Err(crate::runtime_error("continuity.http_discovery_window_too_large"));
}
return std::result::Result::Ok(slot_count);
}
fn validate_http_block_discovery_result(
start_slot: u64,
end_slot: u64,
strategy: RawTransactionIngestHttpDiscoveryStrategy,
current_tip: std::option::Option<u64>,
discovered: &[u64],
) -> ksp_core_lib::Result<RawTransactionIngestHttpDiscoveryWindow> {
if let std::result::Result::Err(error) = http_discovery_window_slot_count(start_slot, end_slot) {
return std::result::Result::Err(error);
}
if let std::result::Result::Err(error) = validate_http_block_polling_discovery(start_slot, discovered) {
return std::result::Result::Err(error);
}
return match strategy {
RawTransactionIngestHttpDiscoveryStrategy::ClosedRange => {
if discovered.iter().any(|slot| return *slot > end_slot) {
return std::result::Result::Err(crate::runtime_error("continuity.http_closed_range_exceeded"));
}
std::result::Result::Ok(RawTransactionIngestHttpDiscoveryWindow {
produced_slots: discovered.to_vec(),
proven_end_slot: std::option::Option::Some(end_slot),
})
},
RawTransactionIngestHttpDiscoveryStrategy::WithLimitBoundary => {
let current_tip = match current_tip {
std::option::Option::Some(value) => value,
std::option::Option::None => return std::result::Result::Err(crate::runtime_error("continuity.http_discovery_tip_missing")),
};
if current_tip < start_slot && discovered.is_empty() {
return std::result::Result::Ok(RawTransactionIngestHttpDiscoveryWindow {
produced_slots: std::vec::Vec::new(),
proven_end_slot: std::option::Option::None,
});
}
let proven_end_slot = discovered.last().copied().map(|slot| return slot.min(end_slot));
let produced_slots = discovered.iter().copied().take_while(|slot| return *slot <= end_slot).collect();
std::result::Result::Ok(RawTransactionIngestHttpDiscoveryWindow { produced_slots, proven_end_slot })
},
};
}
async fn discover_http_block_window(
pool: &ksp_onchain_transport_lib::HttpTransportPool,
role: &ksp_onchain_transport_lib::HttpRoleName,
expected_cluster: &str,
commitment: ksp_onchain_transport_lib::SolanaCommitment,
start_slot: u64,
end_slot: u64,
stop_receiver: &mut tokio::sync::watch::Receiver<bool>,
) -> ksp_core_lib::Result<std::option::Option<RawTransactionIngestHttpDiscoveryWindow>> {
let slot_count = match http_discovery_window_slot_count(start_slot, end_slot) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let capabilities = match http_role_scan_capabilities(pool, role, expected_cluster) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let strategy = match capabilities.strategy() {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let context_config = ksp_onchain_transport_lib::SolanaContextConfig::new(std::option::Option::Some(commitment), std::option::Option::None);
let mut current_tip = std::option::Option::None;
let discovered = match strategy {
RawTransactionIngestHttpDiscoveryStrategy::ClosedRange => tokio::select! {
biased;
_ = stop_receiver.changed() => {
return std::result::Result::Ok(std::option::Option::None);
}
result = pool.get_blocks(role, start_slot, std::option::Option::Some(end_slot), std::option::Option::Some(&context_config)) => result,
},
RawTransactionIngestHttpDiscoveryStrategy::WithLimitBoundary => {
let tip = tokio::select! {
biased;
_ = stop_receiver.changed() => {
return std::result::Result::Ok(std::option::Option::None);
}
result = pool.get_slot(role, std::option::Option::Some(&context_config)) => result,
};
let tip = match tip {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(source_transport_error(error.code())),
};
current_tip = std::option::Option::Some(tip);
tokio::select! {
biased;
_ = stop_receiver.changed() => {
return std::result::Result::Ok(std::option::Option::None);
}
result = pool.get_blocks_with_limit(role, start_slot, slot_count, std::option::Option::Some(&context_config)) => result,
}
},
};
let discovered = match discovered {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(source_transport_error(error.code())),
};
let validated = validate_http_block_discovery_result(start_slot, end_slot, strategy, current_tip, discovered.as_slice());
return match validated {
std::result::Result::Ok(value) => std::result::Result::Ok(std::option::Option::Some(value)),
std::result::Result::Err(error) => std::result::Result::Err(error),
};
}
fn validate_http_block_polling_discovery(next_scan_slot: u64, discovered: &[u64]) -> ksp_core_lib::Result<()> {
let mut previous = std::option::Option::None;
for slot in discovered {
if *slot < next_scan_slot {
return std::result::Result::Err(crate::runtime_error("source.http_block_polling_discovery_before_frontier"));
}
if let std::option::Option::Some(previous) = previous
&& *slot <= previous
{
return std::result::Result::Err(crate::runtime_error("source.http_block_polling_discovery_not_strictly_increasing"));
}
previous = std::option::Option::Some(*slot);
}
return std::result::Result::Ok(());
}
async fn fetch_yellowstone_block_ingresses(
source: &crate::RawTransactionIngestYellowstoneSource,
settings: &crate::RawTransactionIngestSettings,
slot: u64,
stop_receiver: &mut tokio::sync::watch::Receiver<bool>,
) -> ksp_core_lib::Result<std::option::Option<std::vec::Vec<crate::RawTransactionIngress>>> {
let commitment = match source.subscribe_request.commitment() {
std::option::Option::Some(value) => value,
std::option::Option::None => return std::result::Result::Err(crate::runtime_error("source.yellowstone_commitment_missing")),
};
let config = ksp_onchain_transport_lib::SolanaGetBlockConfig::new(
std::option::Option::Some(commitment),
std::option::Option::Some(ksp_onchain_transport_lib::SolanaTransactionEncoding::Base64),
std::option::Option::Some(ksp_onchain_transport_lib::SolanaTransactionDetails::Full),
std::option::Option::Some(1),
std::option::Option::Some(false),
);
let mut attempt = 0_u8;
let observed = loop {
attempt = attempt.saturating_add(1);
let request = tokio::select! {
biased;
_ = stop_receiver.changed() => {
return std::result::Result::Ok(std::option::Option::None);
}
result = source.http_pool.get_block_observed(&source.hydration_role, slot, std::option::Option::Some(&config)) => result,
};
match request {
std::result::Result::Ok(value) if value.value().is_some() => break value,
std::result::Result::Ok(_) if attempt < RAW_TRANSACTION_INGEST_YELLOWSTONE_BLOCK_RECONCILIATION_ATTEMPTS => {},
std::result::Result::Ok(_) => {
return std::result::Result::Err(crate::runtime_error("source.yellowstone_block_not_available"));
},
std::result::Result::Err(_) if attempt < RAW_TRANSACTION_INGEST_YELLOWSTONE_BLOCK_RECONCILIATION_ATTEMPTS => {},
std::result::Result::Err(error) => return std::result::Result::Err(source_transport_error(error.code())),
}
tokio::select! {
biased;
_ = stop_receiver.changed() => {
return std::result::Result::Ok(std::option::Option::None);
}
_ = tokio::time::sleep(RAW_TRANSACTION_INGEST_YELLOWSTONE_BLOCK_RECONCILIATION_DELAY) => {},
}
};
let block = match observed.value() {
std::option::Option::Some(value) => value,
std::option::Option::None => return std::result::Result::Err(crate::runtime_error("source.yellowstone_block_not_available")),
};
let received_at = match current_raw_timestamp() {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let transactions = match block.transactions() {
ksp_onchain_transport_lib::SolanaWireField::Value(value) => value,
ksp_onchain_transport_lib::SolanaWireField::Omitted | ksp_onchain_transport_lib::SolanaWireField::Null => {
return std::result::Result::Err(crate::runtime_error("source.yellowstone_block_transactions_missing"));
},
};
let provenance = build_yellowstone_block_provenance(source, settings, observed.endpoint_name(), observed.provider(), received_at);
let provenance = match provenance {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let mut ingresses = std::vec::Vec::with_capacity(transactions.len());
for (position, transaction) in transactions.iter().enumerate() {
let material = build_http_block_polling_material_from_transaction(&source.network, slot, block.block_time(), transaction, position);
let material = match material {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
ingresses.push(crate::RawTransactionIngress {
material,
network: source.network.clone(),
provenance: provenance.clone(),
source_key: source.source_key,
});
}
return std::result::Result::Ok(std::option::Option::Some(ingresses));
}
fn build_yellowstone_block_provenance(
source: &crate::RawTransactionIngestYellowstoneSource,
settings: &crate::RawTransactionIngestSettings,
endpoint_name: &str,
provider_name: &ksp_onchain_transport_lib::HttpProviderName,
received_at: ksp_store_lib::RawTimestamp,
) -> ksp_core_lib::Result<ksp_store_lib::RawAcquisitionProvenance> {
let (provider, endpoint_id) = match composite_provenance_codes(&source.route, "ys", provider_name.as_str(), endpoint_name) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let protocol = match ksp_store_lib::RawProvenanceCode::new(RAW_TRANSACTION_INGEST_YELLOWSTONE_HTTP_PROTOCOL) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return std::result::Result::Err(crate::runtime_error("source.yellowstone_block_protocol_unrepresentable")),
};
let acquisition_method = match ksp_store_lib::RawProvenanceCode::new(RAW_TRANSACTION_INGEST_YELLOWSTONE_BLOCK_HTTP_ACQUISITION_METHOD) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return std::result::Result::Err(crate::runtime_error("source.yellowstone_block_method_unrepresentable")),
};
let capture_session = match ksp_store_lib::RawProvenanceCode::new(settings.worker_id().as_str()) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return std::result::Result::Err(crate::runtime_error("source.yellowstone_block_capture_session_unrepresentable")),
};
let commitment = match source.subscribe_request.commitment() {
std::option::Option::Some(value) => value,
std::option::Option::None => return std::result::Result::Err(crate::runtime_error("source.yellowstone_commitment_missing")),
};
let commitment = match ksp_store_lib::RawProvenanceCode::new(commitment.as_str()) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return std::result::Result::Err(crate::runtime_error("source.yellowstone_block_commitment_unrepresentable")),
};
return std::result::Result::Ok(
ksp_store_lib::RawAcquisitionProvenance::new(provider, protocol, acquisition_method, ksp_store_lib::RawAcquisitionOrigin::Live, received_at)
.with_endpoint_id(endpoint_id)
.with_commitment(commitment)
.with_capture_session_id(capture_session),
);
}
fn project_http_block_polling_ingresses(
source: &crate::RawTransactionIngestHttpBlockPollingSource,
settings: &crate::RawTransactionIngestSettings,
slot: u64,
block: &ksp_onchain_transport_lib::SolanaConfirmedBlock,
observed: &ksp_onchain_transport_lib::HttpObservedValue<std::option::Option<ksp_onchain_transport_lib::SolanaConfirmedBlock>>,
received_at: ksp_store_lib::RawTimestamp,
) -> ksp_core_lib::Result<std::vec::Vec<crate::RawTransactionIngress>> {
let transactions = match block.transactions() {
ksp_onchain_transport_lib::SolanaWireField::Value(value) => value,
ksp_onchain_transport_lib::SolanaWireField::Omitted | ksp_onchain_transport_lib::SolanaWireField::Null => {
return std::result::Result::Err(crate::runtime_error("source.http_block_polling_transactions_missing"));
},
};
let provenance = build_http_block_polling_provenance(source, settings, observed.endpoint_name(), observed.provider(), received_at);
let provenance = match provenance {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let mut ingresses = std::vec::Vec::with_capacity(transactions.len());
for (position, transaction) in transactions.iter().enumerate() {
let material = build_http_block_polling_material_from_transaction(&source.network, slot, block.block_time(), transaction, position);
let material = match material {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
ingresses.push(crate::RawTransactionIngress {
material,
network: source.network.clone(),
provenance: provenance.clone(),
source_key: source.source_key,
});
}
return std::result::Result::Ok(ingresses);
}
fn build_http_block_polling_material_from_transaction(
network: &ksp_store_lib::RawNetworkId,
slot: u64,
block_time: std::option::Option<i64>,
transaction: &ksp_onchain_transport_lib::SolanaBlockTransaction,
position: usize,
) -> ksp_core_lib::Result<ksp_raw_transaction_lib::RawTransactionMaterial> {
let transaction_data = match transaction.transaction() {
ksp_onchain_transport_lib::SolanaEncodedTransaction::Binary { data, encoding }
if *encoding == ksp_onchain_transport_lib::SolanaTransactionBinaryEncoding::Base64 =>
{
data.as_str()
},
ksp_onchain_transport_lib::SolanaEncodedTransaction::Binary { .. }
| ksp_onchain_transport_lib::SolanaEncodedTransaction::LegacyBinary(_)
| ksp_onchain_transport_lib::SolanaEncodedTransaction::Json(_) => {
return std::result::Result::Err(crate::runtime_error("source.http_block_polling_transaction_encoding_invalid"));
},
};
let version = qualify_http_block_polling_version(transaction.version());
let version = match version {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let transaction_index = match u32::try_from(position) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return std::result::Result::Err(crate::runtime_error("source.http_block_polling_transaction_index_invalid")),
};
let material = ksp_raw_transaction_lib::RawTransactionMaterial::binary_base64_with_embedded_signature(
network.clone(),
slot,
block_time,
transaction_data,
map_hydration_wire_field(transaction.meta(), |value| return value.clone()),
version,
ksp_raw_transaction_lib::RawTransactionWireField::Value(transaction_index),
);
return match material {
std::result::Result::Ok(value) => std::result::Result::Ok(value),
std::result::Result::Err(_) => std::result::Result::Err(crate::runtime_error("source.http_block_polling_transaction_signature_invalid")),
};
}
fn qualify_http_block_polling_version(
version: &ksp_onchain_transport_lib::SolanaWireField<ksp_onchain_transport_lib::SolanaTransactionVersion>,
) -> ksp_core_lib::Result<ksp_raw_transaction_lib::RawTransactionWireField<ksp_raw_transaction_lib::RawTransactionVersion>> {
return match version {
ksp_onchain_transport_lib::SolanaWireField::Value(ksp_onchain_transport_lib::SolanaTransactionVersion::Legacy) => {
std::result::Result::Ok(ksp_raw_transaction_lib::RawTransactionWireField::Value(ksp_raw_transaction_lib::RawTransactionVersion::Legacy))
},
ksp_onchain_transport_lib::SolanaWireField::Value(ksp_onchain_transport_lib::SolanaTransactionVersion::Number(0)) => {
std::result::Result::Ok(ksp_raw_transaction_lib::RawTransactionWireField::Value(ksp_raw_transaction_lib::RawTransactionVersion::Number(0)))
},
ksp_onchain_transport_lib::SolanaWireField::Value(ksp_onchain_transport_lib::SolanaTransactionVersion::Number(1)) => {
std::result::Result::Ok(ksp_raw_transaction_lib::RawTransactionWireField::Value(ksp_raw_transaction_lib::RawTransactionVersion::Number(1)))
},
ksp_onchain_transport_lib::SolanaWireField::Value(ksp_onchain_transport_lib::SolanaTransactionVersion::Number(_)) => {
std::result::Result::Err(crate::runtime_error("source.http_block_polling_transaction_version_unsupported"))
},
ksp_onchain_transport_lib::SolanaWireField::Omitted | ksp_onchain_transport_lib::SolanaWireField::Null => {
std::result::Result::Err(crate::runtime_error("source.http_block_polling_transaction_version_unqualified"))
},
};
}
fn build_http_block_polling_provenance(
source: &crate::RawTransactionIngestHttpBlockPollingSource,
settings: &crate::RawTransactionIngestSettings,
endpoint_name: &str,
provider_name: &ksp_onchain_transport_lib::HttpProviderName,
received_at: ksp_store_lib::RawTimestamp,
) -> ksp_core_lib::Result<ksp_store_lib::RawAcquisitionProvenance> {
let provider = match ksp_store_lib::RawProvenanceCode::new(provider_name.as_str()) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => {
return std::result::Result::Err(crate::runtime_error("source.http_block_polling_provider_unrepresentable"));
},
};
let endpoint_id = match ksp_store_lib::RawProvenanceCode::new(endpoint_name) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => {
return std::result::Result::Err(crate::runtime_error("source.http_block_polling_endpoint_unrepresentable"));
},
};
let protocol = match ksp_store_lib::RawProvenanceCode::new(RAW_TRANSACTION_INGEST_HTTP_BLOCK_POLLING_PROTOCOL) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return std::result::Result::Err(crate::runtime_error("source.http_block_polling_protocol_unrepresentable")),
};
let acquisition_method = match ksp_store_lib::RawProvenanceCode::new(RAW_TRANSACTION_INGEST_HTTP_BLOCK_POLLING_ACQUISITION_METHOD) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return std::result::Result::Err(crate::runtime_error("source.http_block_polling_method_unrepresentable")),
};
let capture_session = match ksp_store_lib::RawProvenanceCode::new(settings.worker_id().as_str()) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return std::result::Result::Err(crate::runtime_error("source.http_block_polling_capture_session_unrepresentable")),
};
let commitment = match ksp_store_lib::RawProvenanceCode::new(source.commitment.as_str()) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return std::result::Result::Err(crate::runtime_error("source.http_block_polling_commitment_unrepresentable")),
};
return std::result::Result::Ok(
ksp_store_lib::RawAcquisitionProvenance::new(provider, protocol, acquisition_method, ksp_store_lib::RawAcquisitionOrigin::Live, received_at)
.with_endpoint_id(endpoint_id)
.with_commitment(commitment)
.with_capture_session_id(capture_session),
);
}
fn helius_transaction_filter_fingerprint(filter: &ksp_onchain_transport_lib::HeliusTransactionSubscribeFilter) -> [u8; 32] {
let mut hasher = sha2::Sha256::new();
hasher.update(RAW_TRANSACTION_INGEST_HELIUS_TRANSACTION_FILTER_FINGERPRINT_DOMAIN);
hash_helius_optional_bool(&mut hasher, b"failed", filter.failed());
hash_helius_optional_bool(&mut hasher, b"vote", filter.vote());
hash_helius_optional_pubkey_list(&mut hasher, b"account_exclude", filter.account_exclude());
hash_helius_optional_pubkey_list(&mut hasher, b"account_include", filter.account_include());
hash_helius_optional_pubkey_list(&mut hasher, b"account_required", filter.account_required());
hash_live_source_key_component(&mut hasher, b"signature");
match filter.signature() {
std::option::Option::Some(value) => {
hash_live_source_key_component(&mut hasher, b"value");
hash_live_source_key_component(&mut hasher, value.as_bytes());
},
std::option::Option::None => hash_live_source_key_component(&mut hasher, b"omitted"),
}
hash_live_source_key_component(&mut hasher, b"token_accounts");
match filter.token_accounts() {
std::option::Option::Some(value) => {
hash_live_source_key_component(&mut hasher, b"value");
hash_live_source_key_component(&mut hasher, value.as_str().as_bytes());
},
std::option::Option::None => hash_live_source_key_component(&mut hasher, b"omitted"),
}
return hasher.finalize().into();
}
fn helius_transaction_live_source_key(
network: &ksp_store_lib::RawNetworkId,
route: &RawTransactionIngestSourceRoute,
commitment: ksp_onchain_transport_lib::SolanaCommitment,
filter_fingerprint: &[u8; 32],
) -> [u8; 32] {
let mut hasher = sha2::Sha256::new();
hasher.update(RAW_TRANSACTION_INGEST_LIVE_SOURCE_KEY_DOMAIN);
hash_live_source_key_component(&mut hasher, b"helius_transaction");
hash_live_source_key_component(&mut hasher, network.as_str().as_bytes());
hash_live_source_key_component(&mut hasher, route.provider.as_str().as_bytes());
hash_live_source_key_component(&mut hasher, route.endpoint_id.as_str().as_bytes());
hash_live_source_key_component(&mut hasher, commitment.as_str().as_bytes());
hash_live_source_key_component(&mut hasher, filter_fingerprint);
return hasher.finalize().into();
}
fn helius_transaction_request(
filter: ksp_onchain_transport_lib::HeliusTransactionSubscribeFilter,
commitment: ksp_onchain_transport_lib::SolanaCommitment,
) -> ksp_onchain_transport_lib::HeliusTransactionSubscribeRequest {
let options = ksp_onchain_transport_lib::HeliusTransactionSubscribeOptions::new(
std::option::Option::Some(commitment),
std::option::Option::Some(ksp_onchain_transport_lib::HeliusTransactionSubscribeEncoding::Base64),
std::option::Option::Some(ksp_onchain_transport_lib::SolanaTransactionDetails::Full),
std::option::Option::Some(false),
std::option::Option::Some(1),
);
return ksp_onchain_transport_lib::HeliusTransactionSubscribeRequest::new(filter, std::option::Option::Some(options));
}
fn hash_helius_optional_bool(hasher: &mut sha2::Sha256, label: &[u8], value: std::option::Option<bool>) {
hash_live_source_key_component(hasher, label);
match value {
std::option::Option::Some(true) => hash_live_source_key_component(hasher, b"true"),
std::option::Option::Some(false) => hash_live_source_key_component(hasher, b"false"),
std::option::Option::None => hash_live_source_key_component(hasher, b"omitted"),
}
return;
}
fn hash_helius_optional_pubkey_list(hasher: &mut sha2::Sha256, label: &[u8], values: std::option::Option<&[ksp_core_lib::Pubkey]>) {
hash_live_source_key_component(hasher, label);
let values = match values {
std::option::Option::Some(values) => values,
std::option::Option::None => {
hash_live_source_key_component(hasher, b"omitted");
return;
},
};
hash_live_source_key_component(hasher, b"value");
let mut normalized = values.iter().map(|value| return value.to_bytes()).collect::<std::vec::Vec<_>>();
normalized.sort_unstable();
hash_live_source_key_component(hasher, &(normalized.len() as u64).to_be_bytes());
for value in normalized {
hash_live_source_key_component(hasher, &value);
}
return;
}
fn standard_block_filter_fingerprint(filter: &ksp_onchain_transport_lib::SolanaBlockSubscribeFilter) -> [u8; 32] {
let mut hasher = sha2::Sha256::new();
hasher.update(RAW_TRANSACTION_INGEST_STANDARD_BLOCK_FILTER_FINGERPRINT_DOMAIN);
match filter {
ksp_onchain_transport_lib::SolanaBlockSubscribeFilter::All => hash_live_source_key_component(&mut hasher, b"all"),
ksp_onchain_transport_lib::SolanaBlockSubscribeFilter::MentionsAccountOrProgram(pubkey) => {
hash_live_source_key_component(&mut hasher, b"mentionsAccountOrProgram");
hash_live_source_key_component(&mut hasher, &pubkey.to_bytes());
},
}
return hasher.finalize().into();
}
fn standard_block_filter_kind(filter: &ksp_onchain_transport_lib::SolanaBlockSubscribeFilter) -> &'static str {
return match filter {
ksp_onchain_transport_lib::SolanaBlockSubscribeFilter::All => "all",
ksp_onchain_transport_lib::SolanaBlockSubscribeFilter::MentionsAccountOrProgram(_) => "mentions_account_or_program",
};
}
fn standard_block_live_source_key(
network: &ksp_store_lib::RawNetworkId,
route: &RawTransactionIngestSourceRoute,
commitment: ksp_onchain_transport_lib::SolanaCommitment,
filter_fingerprint: &[u8; 32],
) -> [u8; 32] {
let mut hasher = sha2::Sha256::new();
hasher.update(RAW_TRANSACTION_INGEST_LIVE_SOURCE_KEY_DOMAIN);
hash_live_source_key_component(&mut hasher, b"standard_block");
hash_live_source_key_component(&mut hasher, network.as_str().as_bytes());
hash_live_source_key_component(&mut hasher, route.provider.as_str().as_bytes());
hash_live_source_key_component(&mut hasher, route.endpoint_id.as_str().as_bytes());
hash_live_source_key_component(&mut hasher, commitment.as_str().as_bytes());
hash_live_source_key_component(&mut hasher, filter_fingerprint);
return hasher.finalize().into();
}
fn standard_logs_filter_fingerprint(filter: &ksp_onchain_transport_lib::SolanaLogsSubscribeFilter) -> [u8; 32] {
let mut hasher = sha2::Sha256::new();
hasher.update(RAW_TRANSACTION_INGEST_STANDARD_LOGS_FILTER_FINGERPRINT_DOMAIN);
match filter {
ksp_onchain_transport_lib::SolanaLogsSubscribeFilter::All => hash_live_source_key_component(&mut hasher, b"all"),
ksp_onchain_transport_lib::SolanaLogsSubscribeFilter::AllWithVotes => hash_live_source_key_component(&mut hasher, b"allWithVotes"),
ksp_onchain_transport_lib::SolanaLogsSubscribeFilter::Mentions(pubkey) => {
hash_live_source_key_component(&mut hasher, b"mentions");
hash_live_source_key_component(&mut hasher, &pubkey.to_bytes());
},
}
return hasher.finalize().into();
}
fn standard_logs_filter_kind(filter: &ksp_onchain_transport_lib::SolanaLogsSubscribeFilter) -> &'static str {
return match filter {
ksp_onchain_transport_lib::SolanaLogsSubscribeFilter::All => "all",
ksp_onchain_transport_lib::SolanaLogsSubscribeFilter::AllWithVotes => "all_with_votes",
ksp_onchain_transport_lib::SolanaLogsSubscribeFilter::Mentions(_) => "mentions",
};
}
fn standard_logs_live_source_key(
network: &ksp_store_lib::RawNetworkId,
route: &RawTransactionIngestSourceRoute,
commitment: ksp_onchain_transport_lib::SolanaCommitment,
filter_fingerprint: &[u8; 32],
) -> [u8; 32] {
let mut hasher = sha2::Sha256::new();
hasher.update(RAW_TRANSACTION_INGEST_LIVE_SOURCE_KEY_DOMAIN);
hash_live_source_key_component(&mut hasher, b"standard_logs");
hash_live_source_key_component(&mut hasher, network.as_str().as_bytes());
hash_live_source_key_component(&mut hasher, route.provider.as_str().as_bytes());
hash_live_source_key_component(&mut hasher, route.endpoint_id.as_str().as_bytes());
hash_live_source_key_component(&mut hasher, commitment.as_str().as_bytes());
hash_live_source_key_component(&mut hasher, filter_fingerprint);
return hasher.finalize().into();
}
fn yellowstone_coverage_scope_fingerprint(request: &ksp_onchain_transport_lib::YellowstoneSubscribeRequest) -> ksp_core_lib::Result<[u8; 32]> {
let mut normalized = request.clone();
normalized.set_commitment(std::option::Option::None);
normalized.set_ping(std::option::Option::None);
let identity = match normalized.identity() {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return std::result::Result::Err(crate::runtime_error("continuity.yellowstone_scope_identity_invalid")),
};
let mut hasher = sha2::Sha256::new();
hasher.update(RAW_TRANSACTION_INGEST_YELLOWSTONE_COVERAGE_SCOPE_FINGERPRINT_DOMAIN);
let mut writer = RawTransactionIngestSourceKeyHashWriter { hasher: &mut hasher };
std::hash::Hash::hash(&identity, &mut writer);
return std::result::Result::Ok(hasher.finalize().into());
}
fn yellowstone_live_source_key(
network: &ksp_store_lib::RawNetworkId,
route: &RawTransactionIngestSourceRoute,
commitment: std::option::Option<ksp_onchain_transport_lib::SolanaCommitment>,
request_identity: &ksp_onchain_transport_lib::YellowstoneSubscribeRequestIdentity,
) -> [u8; 32] {
let mut hasher = sha2::Sha256::new();
hasher.update(RAW_TRANSACTION_INGEST_LIVE_SOURCE_KEY_DOMAIN);
hash_live_source_key_component(&mut hasher, b"yellowstone");
hash_live_source_key_component(&mut hasher, network.as_str().as_bytes());
hash_live_source_key_component(&mut hasher, route.provider.as_str().as_bytes());
hash_live_source_key_component(&mut hasher, route.endpoint_id.as_str().as_bytes());
let commitment = match commitment {
std::option::Option::Some(value) => value.as_str(),
std::option::Option::None => "none",
};
hash_live_source_key_component(&mut hasher, commitment.as_bytes());
let mut writer = RawTransactionIngestSourceKeyHashWriter { hasher: &mut hasher };
std::hash::Hash::hash(request_identity, &mut writer);
return hasher.finalize().into();
}
fn hash_live_source_key_component(hasher: &mut sha2::Sha256, value: &[u8]) {
hasher.update((value.len() as u64).to_be_bytes());
hasher.update(value);
return;
}
fn compatible_http_route_count(
pool: &ksp_onchain_transport_lib::HttpTransportPool,
hydration_role: &ksp_onchain_transport_lib::HttpRoleName,
request_kind: &str,
expected_cluster: &str,
source_route: &RawTransactionIngestSourceRoute,
route_prefix: &str,
) -> ksp_core_lib::Result<usize> {
let snapshot = pool.snapshot();
let mut compatible = 0_usize;
for endpoint in snapshot.endpoints() {
if !endpoint.enabled() {
continue;
}
for role in endpoint.roles() {
if !role.enabled() || role.role() != hydration_role.as_str() {
continue;
}
let supports_request = role.request_kinds().iter().any(|kind| return kind.as_str() == "*" || kind.as_str() == request_kind);
if !supports_request {
continue;
}
if endpoint.cluster() != expected_cluster {
return std::result::Result::Err(crate::runtime_error("runtime_resources.transport_network_mismatch"));
}
let composite = composite_provenance_codes(source_route, route_prefix, endpoint.provider(), endpoint.name());
if let std::result::Result::Err(error) = composite {
return std::result::Result::Err(error);
}
compatible = compatible.saturating_add(1);
}
}
return std::result::Result::Ok(compatible);
}
fn composite_provenance_codes(
source_route: &RawTransactionIngestSourceRoute,
route_prefix: &str,
http_provider: &str,
http_endpoint: &str,
) -> ksp_core_lib::Result<(ksp_store_lib::RawProvenanceCode, ksp_store_lib::RawProvenanceCode)> {
let provider = std::format!("{}.{}:http.{}", route_prefix, source_route.provider.as_str(), http_provider);
let provider = match ksp_store_lib::RawProvenanceCode::new(provider) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => {
return std::result::Result::Err(crate::runtime_error("runtime_resources.composite_provider_unrepresentable"));
},
};
let endpoint = std::format!("{}.{}:http.{}", route_prefix, source_route.endpoint_id.as_str(), http_endpoint);
let endpoint = match ksp_store_lib::RawProvenanceCode::new(endpoint) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => {
return std::result::Result::Err(crate::runtime_error("runtime_resources.composite_endpoint_unrepresentable"));
},
};
return std::result::Result::Ok((provider, endpoint));
}
fn ingestion_filter_count(request: &ksp_onchain_transport_lib::YellowstoneSubscribeRequest) -> usize {
return request.transaction_filter_count().saturating_add(request.transaction_status_filter_count()).saturating_add(request.block_filter_count());
}
fn matched_filter_direct_id(filters: &[ksp_onchain_transport_lib::YellowstoneSubscribeFilterName]) -> std::option::Option<ksp_store_lib::RawProvenanceCode> {
let mut names = filters.iter().map(ksp_onchain_transport_lib::YellowstoneSubscribeFilterName::as_str).collect::<std::vec::Vec<_>>();
names.sort_unstable();
names.dedup();
if names.len() != 1 {
return std::option::Option::None;
}
return match ksp_store_lib::RawProvenanceCode::new(names[0]) {
std::result::Result::Ok(value) => std::option::Option::Some(value),
std::result::Result::Err(_) => std::option::Option::None,
};
}
fn matched_filter_fingerprint(filters: &[ksp_onchain_transport_lib::YellowstoneSubscribeFilterName]) -> [u8; 32] {
let mut names = filters.iter().map(ksp_onchain_transport_lib::YellowstoneSubscribeFilterName::as_str).collect::<std::vec::Vec<_>>();
names.sort_unstable();
names.dedup();
let mut hasher = sha2::Sha256::new();
hasher.update(RAW_TRANSACTION_INGEST_YELLOWSTONE_FILTER_FINGERPRINT_DOMAIN);
hasher.update((names.len() as u64).to_be_bytes());
for name in names {
hasher.update((name.len() as u64).to_be_bytes());
hasher.update(name.as_bytes());
}
return hasher.finalize().into();
}
fn project_yellowstone_block_signals<T: RawTransactionIngestYellowstoneBlockView>(
source: &crate::RawTransactionIngestYellowstoneSource,
update: &T,
) -> ksp_core_lib::Result<std::vec::Vec<RawTransactionIngestSourceSignal>> {
let filters = update.filters();
let matched_filter_count = filters.len();
let matched_filter_fingerprint = matched_filter_fingerprint(filters);
let matched_filter_id = matched_filter_direct_id(filters);
let created_at = map_yellowstone_source_timestamp(update.created_at());
let mut signals = std::vec::Vec::with_capacity(update.transaction_count());
for position in 0..update.transaction_count() {
let (signature, transaction_index) = match update.transaction_identity(position) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
signals.push(RawTransactionIngestSourceSignal {
created_at,
family: RawTransactionIngestSourceFamily::Block,
matched_filter_count,
matched_filter_fingerprint,
matched_filter_id: matched_filter_id.clone(),
network: source.network.clone(),
route: source.route.clone(),
signature: ksp_store_lib::RawTransactionSignature::new(*signature.as_bytes()),
slot: update.slot(),
transaction_index: std::option::Option::Some(transaction_index),
});
}
return std::result::Result::Ok(signals);
}
fn project_yellowstone_continuity_signal<T: RawTransactionIngestYellowstoneContinuityView>(
source: &crate::RawTransactionIngestYellowstoneSource,
update: &T,
) -> RawTransactionIngestContinuitySignal {
let filters = update.filters();
return RawTransactionIngestContinuitySignal {
created_at: map_yellowstone_source_timestamp(update.created_at()),
family: update.family(),
matched_filter_count: filters.len(),
matched_filter_fingerprint: matched_filter_fingerprint(filters),
matched_filter_id: matched_filter_direct_id(filters),
network: source.network.clone(),
parent_slot: update.parent_slot(),
route: source.route.clone(),
slot: update.slot(),
status: update.status(),
};
}
fn map_yellowstone_slot_status(status: ksp_onchain_transport_lib::YellowstoneSlotStatus) -> RawTransactionIngestContinuityStatus {
return match status {
ksp_onchain_transport_lib::YellowstoneSlotStatus::Processed => RawTransactionIngestContinuityStatus::Processed,
ksp_onchain_transport_lib::YellowstoneSlotStatus::Confirmed => RawTransactionIngestContinuityStatus::Confirmed,
ksp_onchain_transport_lib::YellowstoneSlotStatus::Finalized => RawTransactionIngestContinuityStatus::Finalized,
ksp_onchain_transport_lib::YellowstoneSlotStatus::FirstShredReceived => RawTransactionIngestContinuityStatus::FirstShredReceived,
ksp_onchain_transport_lib::YellowstoneSlotStatus::Completed => RawTransactionIngestContinuityStatus::Completed,
ksp_onchain_transport_lib::YellowstoneSlotStatus::CreatedBank => RawTransactionIngestContinuityStatus::CreatedBank,
ksp_onchain_transport_lib::YellowstoneSlotStatus::Dead => RawTransactionIngestContinuityStatus::Dead,
};
}
fn map_yellowstone_source_timestamp(
created_at: std::option::Option<ksp_onchain_transport_lib::YellowstoneUpdateTimestamp>,
) -> std::option::Option<RawTransactionIngestSourceTimestamp> {
return created_at.map(|value| {
return RawTransactionIngestSourceTimestamp { nanos: value.nanos(), seconds: value.seconds() };
});
}
fn project_yellowstone_signal<T: RawTransactionIngestYellowstoneSignalView>(
source: &crate::RawTransactionIngestYellowstoneSource,
update: &T,
) -> RawTransactionIngestSourceSignal {
let signature = ksp_store_lib::RawTransactionSignature::new(*update.signature().as_bytes());
let created_at = update.created_at().map(|value| {
return RawTransactionIngestSourceTimestamp { nanos: value.nanos(), seconds: value.seconds() };
});
return RawTransactionIngestSourceSignal {
created_at,
family: update.family(),
matched_filter_count: update.filters().len(),
matched_filter_fingerprint: matched_filter_fingerprint(update.filters()),
matched_filter_id: matched_filter_direct_id(update.filters()),
network: source.network.clone(),
route: source.route.clone(),
signature,
slot: update.slot(),
transaction_index: std::option::Option::Some(update.index()),
};
}
fn project_standard_block_ingresses(
source: &crate::RawTransactionIngestStandardBlockSource,
settings: &crate::RawTransactionIngestSettings,
notification: &ksp_onchain_transport_lib::SolanaRpcResponse<ksp_onchain_transport_lib::SolanaBlockNotification>,
received_at: ksp_store_lib::RawTimestamp,
) -> ksp_core_lib::Result<(u64, std::vec::Vec<crate::RawTransactionIngress>)> {
let value = notification.value();
if notification.context().slot() != value.slot() {
return std::result::Result::Err(crate::runtime_error("source.standard_block_context_slot_mismatch"));
}
if value.err().is_some() {
return std::result::Result::Err(crate::runtime_error("source.standard_block_remote_error"));
}
let block = match value.block() {
std::option::Option::Some(value) => value,
std::option::Option::None => return std::result::Result::Err(crate::runtime_error("source.standard_block_missing")),
};
let transactions = match block.transactions() {
ksp_onchain_transport_lib::SolanaWireField::Value(value) => value,
ksp_onchain_transport_lib::SolanaWireField::Omitted | ksp_onchain_transport_lib::SolanaWireField::Null => {
return std::result::Result::Err(crate::runtime_error("source.standard_block_transactions_missing"));
},
};
let provenance = build_standard_block_provenance(source, settings, received_at);
let provenance = match provenance {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let mut ingresses = std::vec::Vec::with_capacity(transactions.len());
for (position, transaction) in transactions.iter().enumerate() {
let material = build_standard_block_material_from_transaction(&source.network, value.slot(), block.block_time(), transaction, position);
let material = match material {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
ingresses.push(crate::RawTransactionIngress {
material,
network: source.network.clone(),
provenance: provenance.clone(),
source_key: source.source_key,
});
}
return std::result::Result::Ok((value.slot(), ingresses));
}
fn build_standard_block_material_from_transaction(
network: &ksp_store_lib::RawNetworkId,
slot: u64,
block_time: std::option::Option<i64>,
transaction: &ksp_onchain_transport_lib::SolanaBlockTransaction,
position: usize,
) -> ksp_core_lib::Result<ksp_raw_transaction_lib::RawTransactionMaterial> {
let transaction_data = match transaction.transaction() {
ksp_onchain_transport_lib::SolanaEncodedTransaction::Binary { data, encoding }
if *encoding == ksp_onchain_transport_lib::SolanaTransactionBinaryEncoding::Base64 =>
{
data.as_str()
},
ksp_onchain_transport_lib::SolanaEncodedTransaction::Binary { .. }
| ksp_onchain_transport_lib::SolanaEncodedTransaction::LegacyBinary(_)
| ksp_onchain_transport_lib::SolanaEncodedTransaction::Json(_) => {
return std::result::Result::Err(crate::runtime_error("source.standard_block_transaction_encoding_invalid"));
},
};
return build_standard_block_material(transaction.version(), position, |version, transaction_index| {
let material = ksp_raw_transaction_lib::RawTransactionMaterial::binary_base64_with_embedded_signature(
network.clone(),
slot,
block_time,
transaction_data,
map_hydration_wire_field(transaction.meta(), |value| return value.clone()),
version,
transaction_index,
);
return match material {
std::result::Result::Ok(value) => std::result::Result::Ok(value),
std::result::Result::Err(_) => std::result::Result::Err(crate::runtime_error("source.standard_block_transaction_signature_invalid")),
};
});
}
fn build_standard_block_material(
version: &ksp_onchain_transport_lib::SolanaWireField<ksp_onchain_transport_lib::SolanaTransactionVersion>,
position: usize,
build_material: impl std::ops::FnOnce(
ksp_raw_transaction_lib::RawTransactionWireField<ksp_raw_transaction_lib::RawTransactionVersion>,
ksp_raw_transaction_lib::RawTransactionWireField<u32>,
) -> ksp_core_lib::Result<ksp_raw_transaction_lib::RawTransactionMaterial>,
) -> ksp_core_lib::Result<ksp_raw_transaction_lib::RawTransactionMaterial> {
let version = qualify_standard_block_version(version);
let version = match version {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let transaction_index = match u32::try_from(position) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return std::result::Result::Err(crate::runtime_error("source.standard_block_transaction_index_invalid")),
};
return build_material(version, ksp_raw_transaction_lib::RawTransactionWireField::Value(transaction_index));
}
fn qualify_standard_block_version(
version: &ksp_onchain_transport_lib::SolanaWireField<ksp_onchain_transport_lib::SolanaTransactionVersion>,
) -> ksp_core_lib::Result<ksp_raw_transaction_lib::RawTransactionWireField<ksp_raw_transaction_lib::RawTransactionVersion>> {
return match version {
ksp_onchain_transport_lib::SolanaWireField::Value(ksp_onchain_transport_lib::SolanaTransactionVersion::Legacy) => {
std::result::Result::Ok(ksp_raw_transaction_lib::RawTransactionWireField::Value(ksp_raw_transaction_lib::RawTransactionVersion::Legacy))
},
ksp_onchain_transport_lib::SolanaWireField::Value(ksp_onchain_transport_lib::SolanaTransactionVersion::Number(0)) => {
std::result::Result::Ok(ksp_raw_transaction_lib::RawTransactionWireField::Value(ksp_raw_transaction_lib::RawTransactionVersion::Number(0)))
},
ksp_onchain_transport_lib::SolanaWireField::Value(ksp_onchain_transport_lib::SolanaTransactionVersion::Number(1)) => {
std::result::Result::Ok(ksp_raw_transaction_lib::RawTransactionWireField::Value(ksp_raw_transaction_lib::RawTransactionVersion::Number(1)))
},
ksp_onchain_transport_lib::SolanaWireField::Value(ksp_onchain_transport_lib::SolanaTransactionVersion::Number(_)) => {
std::result::Result::Err(crate::runtime_error("source.standard_block_transaction_version_unsupported"))
},
ksp_onchain_transport_lib::SolanaWireField::Omitted | ksp_onchain_transport_lib::SolanaWireField::Null => {
std::result::Result::Err(crate::runtime_error("source.standard_block_transaction_version_unqualified"))
},
};
}
fn build_standard_block_provenance(
source: &crate::RawTransactionIngestStandardBlockSource,
settings: &crate::RawTransactionIngestSettings,
received_at: ksp_store_lib::RawTimestamp,
) -> ksp_core_lib::Result<ksp_store_lib::RawAcquisitionProvenance> {
let protocol = match ksp_store_lib::RawProvenanceCode::new(RAW_TRANSACTION_INGEST_STANDARD_BLOCK_PROTOCOL) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return std::result::Result::Err(crate::runtime_error("source.standard_block_protocol_unrepresentable")),
};
let acquisition_method = match ksp_store_lib::RawProvenanceCode::new(RAW_TRANSACTION_INGEST_STANDARD_BLOCK_ACQUISITION_METHOD) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return std::result::Result::Err(crate::runtime_error("source.standard_block_method_unrepresentable")),
};
let capture_session = match ksp_store_lib::RawProvenanceCode::new(settings.worker_id().as_str()) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return std::result::Result::Err(crate::runtime_error("source.standard_block_capture_session_unrepresentable")),
};
let commitment = match ksp_store_lib::RawProvenanceCode::new(source.commitment.as_str()) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return std::result::Result::Err(crate::runtime_error("source.standard_block_commitment_unrepresentable")),
};
let filter_id = match fingerprint_filter_code(&source.filter_fingerprint) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
return std::result::Result::Ok(
ksp_store_lib::RawAcquisitionProvenance::new(
source.route.provider.clone(),
protocol,
acquisition_method,
ksp_store_lib::RawAcquisitionOrigin::Live,
received_at,
)
.with_endpoint_id(source.route.endpoint_id.clone())
.with_commitment(commitment)
.with_capture_session_id(capture_session)
.with_filter_id(filter_id),
);
}
trait RawTransactionIngestHeliusTransactionView {
fn signature(&self) -> &str;
fn slot(&self) -> u64;
fn transaction_index(&self) -> u64;
}
impl RawTransactionIngestHeliusTransactionView for ksp_onchain_transport_lib::HeliusFullTransactionNotification {
fn signature(&self) -> &str {
return ksp_onchain_transport_lib::HeliusFullTransactionNotification::signature(self);
}
fn slot(&self) -> u64 {
return ksp_onchain_transport_lib::HeliusFullTransactionNotification::slot(self);
}
fn transaction_index(&self) -> u64 {
return ksp_onchain_transport_lib::HeliusFullTransactionNotification::transaction_index(self);
}
}
fn project_helius_transaction_signal<T: RawTransactionIngestHeliusTransactionView>(
source: &crate::RawTransactionIngestHeliusTransactionSource,
response: &T,
) -> ksp_core_lib::Result<RawTransactionIngestSourceSignal> {
let signature = ksp_raw_transaction_lib::parse_raw_transaction_signature(response.signature());
let signature = match signature {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return std::result::Result::Err(crate::runtime_error("source.helius_transaction_signature_invalid")),
};
return std::result::Result::Ok(RawTransactionIngestSourceSignal {
created_at: std::option::Option::None,
family: RawTransactionIngestSourceFamily::Transaction,
matched_filter_count: 1,
matched_filter_fingerprint: source.filter_fingerprint,
matched_filter_id: std::option::Option::None,
network: source.network.clone(),
route: source.route.clone(),
signature,
slot: response.slot(),
transaction_index: std::option::Option::Some(response.transaction_index()),
});
}
trait RawTransactionIngestStandardLogsView {
fn signature(&self) -> &str;
fn slot(&self) -> u64;
}
impl RawTransactionIngestStandardLogsView for ksp_onchain_transport_lib::SolanaRpcResponse<ksp_onchain_transport_lib::SolanaLogsNotification> {
fn signature(&self) -> &str {
return self.value().signature();
}
fn slot(&self) -> u64 {
return self.context().slot();
}
}
fn project_standard_logs_signal<T: RawTransactionIngestStandardLogsView>(
source: &crate::RawTransactionIngestStandardLogsSource,
response: &T,
) -> ksp_core_lib::Result<RawTransactionIngestSourceSignal> {
let signature = ksp_raw_transaction_lib::parse_raw_transaction_signature(response.signature());
let signature = match signature {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return std::result::Result::Err(crate::runtime_error("source.standard_logs_signature_invalid")),
};
return std::result::Result::Ok(RawTransactionIngestSourceSignal {
created_at: std::option::Option::None,
family: RawTransactionIngestSourceFamily::Logs,
matched_filter_count: 1,
matched_filter_fingerprint: source.filter_fingerprint,
matched_filter_id: std::option::Option::None,
network: source.network.clone(),
route: source.route.clone(),
signature,
slot: response.slot(),
transaction_index: std::option::Option::None,
});
}
fn route_yellowstone_update(
source: &crate::RawTransactionIngestYellowstoneSource,
hydration: &RawTransactionIngestHydrationContext,
coordinator: &mut RawTransactionIngestHydrationCoordinator,
processing_frontier: &mut RawTransactionIngestProcessingFrontierReporter,
update: ksp_onchain_transport_lib::YellowstoneSubscribeUpdate,
) -> ksp_core_lib::Result<()> {
match update {
ksp_onchain_transport_lib::YellowstoneSubscribeUpdate::Transaction(value) => {
let received_at = match current_raw_timestamp() {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
return coordinator.queue_signal(hydration, RawTransactionIngestSourceSignal::from((source, value.as_ref())), received_at, processing_frontier);
},
ksp_onchain_transport_lib::YellowstoneSubscribeUpdate::TransactionStatus(value) => {
let received_at = match current_raw_timestamp() {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
return coordinator.queue_signal(hydration, RawTransactionIngestSourceSignal::from((source, &value)), received_at, processing_frontier);
},
ksp_onchain_transport_lib::YellowstoneSubscribeUpdate::Block(value) => {
let received_at = match current_raw_timestamp() {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let signals = match project_yellowstone_block_signals(source, &value) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
for signal in signals {
if let std::result::Result::Err(error) = coordinator.queue_signal(hydration, signal, received_at, processing_frontier) {
return std::result::Result::Err(error);
}
}
return std::result::Result::Ok(());
},
ksp_onchain_transport_lib::YellowstoneSubscribeUpdate::BlockMeta(value) => {
let continuity_signal = project_yellowstone_continuity_signal(source, &value);
if let std::result::Result::Err(error) = processing_frontier.observe_settled(continuity_signal.slot) {
return std::result::Result::Err(error);
}
return std::result::Result::Ok(());
},
ksp_onchain_transport_lib::YellowstoneSubscribeUpdate::Slot(value) => {
let continuity_signal = project_yellowstone_continuity_signal(source, &value);
if let std::result::Result::Err(error) = processing_frontier.observe_settled(continuity_signal.slot) {
return std::result::Result::Err(error);
}
return std::result::Result::Ok(());
},
ksp_onchain_transport_lib::YellowstoneSubscribeUpdate::Account(_)
| ksp_onchain_transport_lib::YellowstoneSubscribeUpdate::Ping(_)
| ksp_onchain_transport_lib::YellowstoneSubscribeUpdate::Pong(_)
| ksp_onchain_transport_lib::YellowstoneSubscribeUpdate::Entry(_) => return std::result::Result::Ok(()),
}
}
fn current_raw_timestamp() -> ksp_core_lib::Result<ksp_store_lib::RawTimestamp> {
let duration = std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH);
let duration = match duration {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return std::result::Result::Err(crate::runtime_error("source.clock_before_epoch")),
};
let millis = match u64::try_from(duration.as_millis()) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return std::result::Result::Err(crate::runtime_error("source.clock_unrepresentable")),
};
return match ksp_store_lib::RawTimestamp::from_unix_millis(millis) {
std::result::Result::Ok(value) => std::result::Result::Ok(value),
std::result::Result::Err(_) => std::result::Result::Err(crate::runtime_error("source.clock_out_of_bounds")),
};
}
async fn wait_yellowstone_session_snapshot(
source: &mut std::option::Option<ksp_onchain_transport_lib::YellowstoneGrpcSubscribeSnapshotSource>,
) -> std::option::Option<ksp_onchain_transport_lib::YellowstoneGrpcSubscribeSnapshot> {
return match source {
std::option::Option::Some(value) => value.wait_for_change().await,
std::option::Option::None => {
return std::future::pending::<std::option::Option<ksp_onchain_transport_lib::YellowstoneGrpcSubscribeSnapshot>>().await;
},
};
}
fn source_transport_error(code: ksp_core_lib::ErrorCode) -> ksp_core_lib::Error {
return ksp_core_lib::Error::new(crate::ERROR_CODE_RAW_TRANSACTION_INGEST_SOURCE_FAILED, "RAW transaction ingest source transport failed")
.with_context("transport_domain", code.domain())
.with_context("transport_code", code.code());
}
fn source_error_context_value<'a>(error: &'a ksp_core_lib::Error, key: &'static str) -> std::option::Option<&'a str> {
for context in error.context() {
if context.key() == key {
return std::option::Option::Some(context.value());
}
}
return std::option::Option::None;
}
fn resolve_yellowstone_close_after_stop(stop_requested: bool, closed: ksp_core_lib::Result<()>) -> ksp_core_lib::Result<()> {
return match closed {
std::result::Result::Ok(()) => std::result::Result::Ok(()),
std::result::Result::Err(error) if stop_requested && error.code() == ksp_onchain_transport_lib::ERROR_CODE_TIMEOUT => std::result::Result::Ok(()),
std::result::Result::Err(error) => std::result::Result::Err(source_transport_error(error.code())),
};
}
fn map_yellowstone_source_state(state: ksp_onchain_transport_lib::YellowstoneGrpcSubscribeState) -> crate::RawTransactionIngestSourceState {
return match state {
ksp_onchain_transport_lib::YellowstoneGrpcSubscribeState::Active => crate::RawTransactionIngestSourceState::Active,
ksp_onchain_transport_lib::YellowstoneGrpcSubscribeState::Reconnecting => crate::RawTransactionIngestSourceState::Reconnecting,
ksp_onchain_transport_lib::YellowstoneGrpcSubscribeState::Closing => crate::RawTransactionIngestSourceState::Closing,
ksp_onchain_transport_lib::YellowstoneGrpcSubscribeState::Closed => crate::RawTransactionIngestSourceState::Closed,
ksp_onchain_transport_lib::YellowstoneGrpcSubscribeState::Failed => crate::RawTransactionIngestSourceState::Failed,
};
}
fn map_websocket_source_state(state: ksp_onchain_transport_lib::WsSessionState) -> crate::RawTransactionIngestSourceState {
return match state {
ksp_onchain_transport_lib::WsSessionState::Disconnected
| ksp_onchain_transport_lib::WsSessionState::Connecting
| ksp_onchain_transport_lib::WsSessionState::Reconnecting { .. } => crate::RawTransactionIngestSourceState::Reconnecting,
ksp_onchain_transport_lib::WsSessionState::Active => crate::RawTransactionIngestSourceState::Active,
ksp_onchain_transport_lib::WsSessionState::Closing => crate::RawTransactionIngestSourceState::Closing,
ksp_onchain_transport_lib::WsSessionState::Closed => crate::RawTransactionIngestSourceState::Closed,
ksp_onchain_transport_lib::WsSessionState::Failed => crate::RawTransactionIngestSourceState::Failed,
};
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
struct RawTransactionIngestProcessingSlotState {
pending: usize,
settled: u64,
}
struct RawTransactionIngestProcessingFrontier {
pending_total: usize,
slots: std::collections::BTreeMap<u64, RawTransactionIngestProcessingSlotState>,
}
impl RawTransactionIngestProcessingFrontier {
fn new() -> Self {
return Self { pending_total: 0, slots: std::collections::BTreeMap::new() };
}
fn observe_pending(&mut self, slot: u64) -> ksp_core_lib::Result<()> {
let pending_total = match self.pending_total.checked_add(1) {
std::option::Option::Some(value) => value,
std::option::Option::None => return std::result::Result::Err(crate::runtime_error("source.frontier_pending_counter_exhausted")),
};
match self.slots.entry(slot) {
std::collections::btree_map::Entry::Occupied(mut entry) => {
let pending = match entry.get().pending.checked_add(1) {
std::option::Option::Some(value) => value,
std::option::Option::None => return std::result::Result::Err(crate::runtime_error("source.frontier_slot_pending_counter_exhausted")),
};
entry.get_mut().pending = pending;
},
std::collections::btree_map::Entry::Vacant(entry) => {
entry.insert(RawTransactionIngestProcessingSlotState { pending: 1, settled: 0 });
},
}
self.pending_total = pending_total;
self.compact();
return std::result::Result::Ok(());
}
fn observe_settled(&mut self, slot: u64) -> ksp_core_lib::Result<()> {
match self.slots.entry(slot) {
std::collections::btree_map::Entry::Occupied(mut entry) => {
let settled = match entry.get().settled.checked_add(1) {
std::option::Option::Some(value) => value,
std::option::Option::None => return std::result::Result::Err(crate::runtime_error("source.frontier_slot_settled_counter_exhausted")),
};
entry.get_mut().settled = settled;
},
std::collections::btree_map::Entry::Vacant(entry) => {
entry.insert(RawTransactionIngestProcessingSlotState { pending: 0, settled: 1 });
},
}
self.compact();
return std::result::Result::Ok(());
}
fn settle_pending(&mut self, slot: u64) -> ksp_core_lib::Result<()> {
let state = match self.slots.get_mut(&slot) {
std::option::Option::Some(value) => value,
std::option::Option::None => return std::result::Result::Err(crate::runtime_error("source.frontier_pending_slot_missing")),
};
if state.pending == 0 || self.pending_total == 0 {
return std::result::Result::Err(crate::runtime_error("source.frontier_pending_counter_invalid"));
}
let settled = match state.settled.checked_add(1) {
std::option::Option::Some(value) => value,
std::option::Option::None => return std::result::Result::Err(crate::runtime_error("source.frontier_slot_settled_counter_exhausted")),
};
state.pending -= 1;
self.pending_total -= 1;
state.settled = settled;
self.compact();
return std::result::Result::Ok(());
}
fn discard_all_pending(&mut self) {
self.pending_total = 0;
self.slots.retain(|_slot, state| {
return state.pending == 0 && state.settled > 0;
});
self.compact();
return;
}
fn highest_observed_slot(&self) -> std::option::Option<u64> {
return self.slots.keys().next_back().copied();
}
fn projection(&self) -> crate::RawTransactionIngestProcessingFrontierProjection {
let oldest_pending_slot = self.slots.iter().find_map(|(slot, state)| {
if state.pending == 0 {
return std::option::Option::None;
}
return std::option::Option::Some(*slot);
});
let processing_frontier_slot = match oldest_pending_slot {
std::option::Option::Some(pending_slot) => self.slots.range(..pending_slot).rev().find_map(|(slot, state)| {
if state.pending == 0 && state.settled > 0 {
return std::option::Option::Some(*slot);
}
return std::option::Option::None;
}),
std::option::Option::None => self.slots.iter().rev().find_map(|(slot, state)| {
if state.pending == 0 && state.settled > 0 {
return std::option::Option::Some(*slot);
}
return std::option::Option::None;
}),
};
return crate::RawTransactionIngestProcessingFrontierProjection::new(self.pending_total, processing_frontier_slot, oldest_pending_slot);
}
fn compact(&mut self) {
let pending_slots = self
.slots
.iter()
.filter_map(|(slot, state)| {
if state.pending > 0 {
return std::option::Option::Some(*slot);
}
return std::option::Option::None;
})
.collect::<std::vec::Vec<_>>();
let mut settled_interval_highs = std::collections::BTreeMap::<std::option::Option<u64>, u64>::new();
for (slot, state) in &self.slots {
if state.pending > 0 || state.settled == 0 {
continue;
}
let upper_pending = pending_slots.iter().find_map(|pending_slot| {
if pending_slot > slot {
return std::option::Option::Some(*pending_slot);
}
return std::option::Option::None;
});
match settled_interval_highs.entry(upper_pending) {
std::collections::btree_map::Entry::Occupied(mut entry) => {
if *slot > *entry.get() {
entry.insert(*slot);
}
},
std::collections::btree_map::Entry::Vacant(entry) => {
entry.insert(*slot);
},
}
}
let mut retained = pending_slots.into_iter().collect::<std::collections::BTreeSet<_>>();
for slot in settled_interval_highs.into_values() {
retained.insert(slot);
}
self.slots.retain(|slot, _state| {
return retained.contains(slot);
});
return;
}
#[cfg(test)]
fn slot_state_count(&self) -> usize {
return self.slots.len();
}
}
struct RawTransactionIngestProcessingFrontierReporter {
frontier: RawTransactionIngestProcessingFrontier,
sender: tokio::sync::watch::Sender<crate::RawTransactionIngestProcessingFrontierProjection>,
source_state: std::option::Option<crate::RawTransactionIngestSourceState>,
source_reconnect_total: u64,
source_replay_attempt_total: u64,
source_replay_delivery_total: u64,
source_replay_coverage_unproven_total: u64,
source_continuity_gap_total: u64,
source_overflow_total: u64,
websocket_incident_anchor: std::option::Option<crate::RawTransactionIngestWebSocketIncidentAnchor>,
}
impl RawTransactionIngestProcessingFrontierReporter {
fn new(sender: tokio::sync::watch::Sender<crate::RawTransactionIngestProcessingFrontierProjection>) -> Self {
return Self {
frontier: RawTransactionIngestProcessingFrontier::new(),
sender,
source_state: std::option::Option::None,
source_reconnect_total: 0,
source_replay_attempt_total: 0,
source_replay_delivery_total: 0,
source_replay_coverage_unproven_total: 0,
source_continuity_gap_total: 0,
source_overflow_total: 0,
websocket_incident_anchor: std::option::Option::None,
};
}
fn observe_pending(&mut self, slot: u64) -> ksp_core_lib::Result<()> {
if let std::result::Result::Err(error) = self.frontier.observe_pending(slot) {
return std::result::Result::Err(error);
}
self.publish();
return std::result::Result::Ok(());
}
fn observe_settled(&mut self, slot: u64) -> ksp_core_lib::Result<()> {
if let std::result::Result::Err(error) = self.frontier.observe_settled(slot) {
return std::result::Result::Err(error);
}
self.publish();
return std::result::Result::Ok(());
}
fn settle_pending(&mut self, slot: u64) -> ksp_core_lib::Result<()> {
if let std::result::Result::Err(error) = self.frontier.settle_pending(slot) {
return std::result::Result::Err(error);
}
self.publish();
return std::result::Result::Ok(());
}
fn discard_all_pending(&mut self) {
self.frontier.discard_all_pending();
self.publish();
return;
}
fn observe_session_snapshot(&mut self, snapshot: ksp_onchain_transport_lib::YellowstoneGrpcSubscribeSnapshot) -> ksp_core_lib::Result<()> {
return self.observe_source_continuity(
map_yellowstone_source_state(snapshot.state()),
snapshot.reconnect_count(),
snapshot.replay_attempt_count(),
snapshot.replay_delivery_count(),
snapshot.replay_coverage_unproven_count(),
snapshot.continuity_gap_count(),
);
}
fn observe_websocket_session_snapshot(&mut self, snapshot: ksp_onchain_transport_lib::WsSessionSnapshot) -> ksp_core_lib::Result<()> {
return self.observe_websocket_continuity(map_websocket_source_state(snapshot.state()), snapshot.continuity_gap_count(), snapshot.overflow_count());
}
fn observe_websocket_continuity(
&mut self,
state: crate::RawTransactionIngestSourceState,
reconnect_total: u64,
overflow_total: u64,
) -> ksp_core_lib::Result<()> {
if reconnect_total < self.source_reconnect_total || overflow_total < self.source_overflow_total {
return std::result::Result::Err(crate::runtime_error("source.websocket_continuity_counter_regression"));
}
let continuity_gap_total = match reconnect_total.checked_add(overflow_total) {
std::option::Option::Some(value) => value,
std::option::Option::None => return std::result::Result::Err(crate::counter_exhausted_error("source.websocket_continuity_gap_total")),
};
if continuity_gap_total < self.source_continuity_gap_total {
return std::result::Result::Err(crate::runtime_error("source.continuity_counter_regression"));
}
let saw_reconnect = reconnect_total > self.source_reconnect_total;
let saw_overflow = overflow_total > self.source_overflow_total;
self.source_reconnect_total = reconnect_total;
self.source_replay_attempt_total = 0;
self.source_continuity_gap_total = continuity_gap_total;
self.source_overflow_total = overflow_total;
if saw_reconnect || saw_overflow {
let start_slot = match self.frontier.highest_observed_slot() {
std::option::Option::Some(value) => value,
std::option::Option::None => {
self.publish();
return std::result::Result::Err(crate::runtime_error("source.websocket_incident_unbounded"));
},
};
match self.websocket_incident_anchor.as_mut() {
std::option::Option::Some(anchor) => {
if let std::result::Result::Err(error) = anchor.extend(reconnect_total, overflow_total, saw_reconnect, saw_overflow) {
self.publish();
return std::result::Result::Err(error);
}
},
std::option::Option::None => {
let anchor =
match crate::RawTransactionIngestWebSocketIncidentAnchor::new(start_slot, reconnect_total, overflow_total, saw_reconnect, saw_overflow)
{
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => {
self.publish();
return std::result::Result::Err(error);
},
};
self.websocket_incident_anchor = std::option::Option::Some(anchor);
},
}
}
self.source_state = if self.websocket_incident_anchor.is_some() && state == crate::RawTransactionIngestSourceState::Active {
std::option::Option::Some(crate::RawTransactionIngestSourceState::Reconnecting)
} else {
std::option::Option::Some(state)
};
self.publish();
return std::result::Result::Ok(());
}
fn observe_websocket_post_incident_slot(&mut self, slot: u64) -> ksp_core_lib::Result<std::option::Option<(u64, u64)>> {
let anchor = match self.websocket_incident_anchor.as_mut() {
std::option::Option::Some(value) => value,
std::option::Option::None => return std::result::Result::Ok(std::option::Option::None),
};
if anchor.end_slot().is_some() {
return std::result::Result::Ok(std::option::Option::None);
}
if let std::result::Result::Err(error) = anchor.close_at(slot) {
return std::result::Result::Err(error);
}
let range = (anchor.start_slot(), slot);
self.publish();
return std::result::Result::Ok(std::option::Option::Some(range));
}
fn observe_source_continuity(
&mut self,
state: crate::RawTransactionIngestSourceState,
reconnect_total: u64,
replay_attempt_total: u64,
replay_delivery_total: u64,
replay_coverage_unproven_total: u64,
continuity_gap_total: u64,
) -> ksp_core_lib::Result<()> {
if reconnect_total < self.source_reconnect_total
|| replay_attempt_total < self.source_replay_attempt_total
|| replay_delivery_total < self.source_replay_delivery_total
|| replay_coverage_unproven_total < self.source_replay_coverage_unproven_total
|| continuity_gap_total < self.source_continuity_gap_total
{
return std::result::Result::Err(crate::runtime_error("source.continuity_counter_regression"));
}
let gap_increased = continuity_gap_total > self.source_continuity_gap_total;
let replay_coverage_unproven_increased = replay_coverage_unproven_total > self.source_replay_coverage_unproven_total;
self.source_state = std::option::Option::Some(state);
self.source_reconnect_total = reconnect_total;
self.source_replay_attempt_total = replay_attempt_total;
self.source_replay_delivery_total = replay_delivery_total;
self.source_replay_coverage_unproven_total = replay_coverage_unproven_total;
self.source_continuity_gap_total = continuity_gap_total;
self.publish();
if gap_increased {
return std::result::Result::Err(crate::runtime_error("source.continuity_gap_proven"));
}
if replay_coverage_unproven_increased {
return std::result::Result::Err(crate::runtime_error("source.replay_coverage_unproven"));
}
return std::result::Result::Ok(());
}
fn set_source_state(&mut self, state: crate::RawTransactionIngestSourceState) {
self.source_state = std::option::Option::Some(state);
self.publish();
return;
}
fn projection(&self) -> crate::RawTransactionIngestProcessingFrontierProjection {
return self.frontier.projection().with_source_continuity(
self.source_state,
self.source_reconnect_total,
self.source_replay_attempt_total,
self.source_continuity_gap_total,
);
}
fn publish(&self) {
self.sender.send_replace(self.projection());
return;
}
}
#[derive(Clone, Eq, Ord, PartialEq, PartialOrd)]
struct RawTransactionIngestHydrationKey {
commitment: &'static str,
network: ksp_store_lib::RawNetworkId,
signature: ksp_store_lib::RawTransactionSignature,
}
struct RawTransactionIngestPendingSignal {
received_at: ksp_store_lib::RawTimestamp,
signal: RawTransactionIngestSourceSignal,
}
struct RawTransactionIngestPendingHydration {
in_flight: bool,
signals: std::vec::Vec<RawTransactionIngestPendingSignal>,
}
type RawTransactionIngestObservedTransaction =
ksp_onchain_transport_lib::HttpObservedValue<std::option::Option<ksp_onchain_transport_lib::SolanaConfirmedTransaction>>;
struct RawTransactionIngestHydrationFetch {
key: RawTransactionIngestHydrationKey,
observed: RawTransactionIngestObservedTransaction,
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
enum RawTransactionIngestKnownReferenceMissingDisposition {
AwaitCoverage,
PreferBlockSlot,
}
enum RawTransactionIngestKnownReferenceHydrationResolution {
Available(std::boxed::Box<crate::RawTransactionIngress>),
Missing {
disposition: RawTransactionIngestKnownReferenceMissingDisposition,
obligation: crate::RawTransactionIngestKnownReferenceObligation,
},
}
#[derive(Clone)]
enum RawTransactionIngestSharedHydrationResult {
Available(RawTransactionIngestObservedTransaction),
Failed(ksp_core_lib::ErrorCode),
}
struct RawTransactionIngestGlobalHydrationRegistry {
hydration_fairness: std::sync::Arc<RawTransactionIngestFairTurnGate>,
hydration_permits: std::sync::Arc<tokio::sync::Semaphore>,
max_pending: usize,
pending: std::sync::Mutex<
std::collections::BTreeMap<
RawTransactionIngestHydrationKey,
tokio::sync::watch::Sender<std::option::Option<RawTransactionIngestSharedHydrationResult>>,
>,
>,
}
impl RawTransactionIngestGlobalHydrationRegistry {
fn new(max_pending: usize, max_in_flight: usize) -> Self {
return Self {
hydration_fairness: std::sync::Arc::new(RawTransactionIngestFairTurnGate::new(max_pending)),
hydration_permits: std::sync::Arc::new(tokio::sync::Semaphore::new(max_in_flight)),
max_pending,
pending: std::sync::Mutex::new(std::collections::BTreeMap::new()),
};
}
async fn acquire_hydration_permit(&self, class: RawTransactionIngestTrafficClass) -> ksp_core_lib::Result<tokio::sync::OwnedSemaphorePermit> {
let turn = match std::sync::Arc::clone(&self.hydration_fairness).acquire(class).await {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let permit = std::sync::Arc::clone(&self.hydration_permits).acquire_owned().await;
std::mem::drop(turn);
return match permit {
std::result::Result::Ok(value) => std::result::Result::Ok(value),
std::result::Result::Err(_) => std::result::Result::Err(crate::runtime_error("source.global_hydration_permit_closed")),
};
}
fn subscribe_or_lead(
&self,
key: &RawTransactionIngestHydrationKey,
) -> ksp_core_lib::Result<(bool, tokio::sync::watch::Receiver<std::option::Option<RawTransactionIngestSharedHydrationResult>>)> {
let mut pending = match self.pending.lock() {
std::result::Result::Ok(value) => value,
std::result::Result::Err(poisoned) => poisoned.into_inner(),
};
if let std::option::Option::Some(sender) = pending.get(key) {
return std::result::Result::Ok((false, sender.subscribe()));
}
if pending.len() >= self.max_pending {
return std::result::Result::Err(crate::runtime_error("source.global_hydration_pending_saturated"));
}
let (sender, receiver) = tokio::sync::watch::channel(std::option::Option::None);
pending.insert(key.clone(), sender);
return std::result::Result::Ok((true, receiver));
}
fn publish_and_remove(&self, key: &RawTransactionIngestHydrationKey, result: RawTransactionIngestSharedHydrationResult) {
let mut pending = match self.pending.lock() {
std::result::Result::Ok(value) => value,
std::result::Result::Err(poisoned) => poisoned.into_inner(),
};
let published = match pending.get(key) {
std::option::Option::Some(sender) => {
sender.send_replace(std::option::Option::Some(result));
true
},
std::option::Option::None => false,
};
if published {
let _removed = pending.remove(key);
}
return;
}
}
type RawTransactionIngestHydrationTasks = tokio::task::JoinSet<ksp_core_lib::Result<RawTransactionIngestHydrationFetch>>;
struct RawTransactionIngestHydrationCoordinator {
global_registry: std::sync::Arc<RawTransactionIngestGlobalHydrationRegistry>,
max_in_flight: usize,
max_pending_signals: usize,
pending_signal_count: usize,
pending: std::collections::BTreeMap<RawTransactionIngestHydrationKey, RawTransactionIngestPendingHydration>,
tasks: RawTransactionIngestHydrationTasks,
}
impl RawTransactionIngestHydrationCoordinator {
#[cfg(test)]
fn new(settings: &crate::RawTransactionIngestSettings) -> Self {
let pending_limit = settings.admission_queue_capacity();
let in_flight_limit = settings.persistence_concurrency().min(pending_limit);
let registry = std::sync::Arc::new(RawTransactionIngestGlobalHydrationRegistry::new(pending_limit, settings.persistence_concurrency()));
return Self::with_global_registry(registry, pending_limit, in_flight_limit);
}
fn with_global_registry(
global_registry: std::sync::Arc<RawTransactionIngestGlobalHydrationRegistry>,
max_pending_signals: usize,
max_in_flight: usize,
) -> Self {
return Self {
global_registry,
max_in_flight,
max_pending_signals,
pending_signal_count: 0,
pending: std::collections::BTreeMap::new(),
tasks: RawTransactionIngestHydrationTasks::new(),
};
}
fn can_receive(&self) -> bool {
return self.pending_signal_count < self.max_pending_signals;
}
fn queue_signal(
&mut self,
hydration: &RawTransactionIngestHydrationContext,
signal: RawTransactionIngestSourceSignal,
received_at: ksp_store_lib::RawTimestamp,
processing_frontier: &mut RawTransactionIngestProcessingFrontierReporter,
) -> ksp_core_lib::Result<()> {
if self.pending_signal_count >= self.max_pending_signals {
return std::result::Result::Err(crate::runtime_error("source.hydration_pending_saturated"));
}
let key = match hydration_key(hydration, &signal) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let signal_slot = signal.slot;
match self.pending.entry(key) {
std::collections::btree_map::Entry::Occupied(mut entry) => {
entry.get_mut().signals.push(RawTransactionIngestPendingSignal { received_at, signal });
},
std::collections::btree_map::Entry::Vacant(entry) => {
entry.insert(RawTransactionIngestPendingHydration {
in_flight: false,
signals: std::vec![RawTransactionIngestPendingSignal { received_at, signal }],
});
},
}
self.pending_signal_count = match self.pending_signal_count.checked_add(1) {
std::option::Option::Some(value) => value,
std::option::Option::None => return std::result::Result::Err(crate::runtime_error("source.hydration_pending_counter_exhausted")),
};
if let std::result::Result::Err(error) = processing_frontier.observe_pending(signal_slot) {
return std::result::Result::Err(error);
}
return self.start_hydrations(hydration);
}
fn start_hydrations(&mut self, hydration: &RawTransactionIngestHydrationContext) -> ksp_core_lib::Result<()> {
while self.tasks.len() < self.max_in_flight {
let key = self.pending.iter().find_map(|(key, pending)| {
if pending.in_flight {
return std::option::Option::None;
}
return std::option::Option::Some(key.clone());
});
let key = match key {
std::option::Option::Some(value) => value,
std::option::Option::None => break,
};
let commitment = hydration.commitment;
let pending = match self.pending.get_mut(&key) {
std::option::Option::Some(value) => value,
std::option::Option::None => return std::result::Result::Err(crate::runtime_error("source.hydration_pending_missing")),
};
pending.in_flight = true;
let pool = hydration.http_pool.clone();
let role = hydration.hydration_role.clone();
let expected_network = hydration.network.clone();
let task_key = key.clone();
let global_registry = std::sync::Arc::clone(&self.global_registry);
let _abort_handle = self.tasks.spawn(async move {
return fetch_hydration_shared(global_registry, pool, role, expected_network, task_key, commitment).await;
});
}
return std::result::Result::Ok(());
}
async fn handle_joined(
&mut self,
joined: std::result::Result<ksp_core_lib::Result<RawTransactionIngestHydrationFetch>, tokio::task::JoinError>,
hydration: &RawTransactionIngestHydrationContext,
settings: &crate::RawTransactionIngestSettings,
admission_sender: &tokio::sync::mpsc::Sender<crate::RawTransactionIngress>,
stop_receiver: &mut tokio::sync::watch::Receiver<bool>,
processing_frontier: &mut RawTransactionIngestProcessingFrontierReporter,
continuity_contracts: &std::sync::Arc<std::sync::Mutex<crate::RawTransactionIngestContinuityContracts>>,
) -> ksp_core_lib::Result<bool> {
let fetched = match joined {
std::result::Result::Ok(std::result::Result::Ok(value)) => value,
std::result::Result::Ok(std::result::Result::Err(error)) => return std::result::Result::Err(error),
std::result::Result::Err(_) => return std::result::Result::Err(crate::runtime_error("source.hydration_task_join_failed")),
};
if *stop_receiver.borrow() {
return std::result::Result::Ok(false);
}
let pending = match self.pending.remove(&fetched.key) {
std::option::Option::Some(value) => value,
std::option::Option::None => return std::result::Result::Err(crate::runtime_error("source.hydration_result_without_pending")),
};
if pending.signals.len() > self.pending_signal_count {
return std::result::Result::Err(crate::runtime_error("source.hydration_pending_counter_invalid"));
}
self.pending_signal_count -= pending.signals.len();
for pending_signal in pending.signals {
if *stop_receiver.borrow() {
return std::result::Result::Ok(false);
}
let signal_slot = pending_signal.signal.slot;
let resolution = resolve_known_reference_hydration(hydration, settings, pending_signal.signal, pending_signal.received_at, &fetched.observed);
let resolution = match resolution {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let ingress = match resolution {
RawTransactionIngestKnownReferenceHydrationResolution::Available(value) => *value,
RawTransactionIngestKnownReferenceHydrationResolution::Missing { disposition, obligation } => {
if obligation.commitment() != hydration.commitment
|| obligation.reference().network() != &hydration.network
|| obligation.slot() != signal_slot
{
return std::result::Result::Err(crate::runtime_error("source.known_reference_obligation_mismatch"));
}
match disposition {
RawTransactionIngestKnownReferenceMissingDisposition::AwaitCoverage
| RawTransactionIngestKnownReferenceMissingDisposition::PreferBlockSlot => {},
}
let continuity_result = {
let mut contracts = match continuity_contracts.lock() {
std::result::Result::Ok(value) => value,
std::result::Result::Err(poisoned) => poisoned.into_inner(),
};
contracts.record_known_reference_gap(hydration.source_key, signal_slot)
};
if let std::result::Result::Err(error) = continuity_result {
return std::result::Result::Err(error);
}
if let std::result::Result::Err(error) = processing_frontier.settle_pending(signal_slot) {
return std::result::Result::Err(error);
}
continue;
},
};
let sent = tokio::select! {
biased;
_ = stop_receiver.changed() => {
return std::result::Result::Ok(false);
}
result = admission_sender.send(ingress) => result,
};
if sent.is_err() {
if *stop_receiver.borrow() {
return std::result::Result::Ok(false);
}
return std::result::Result::Err(crate::runtime_error("source.admission_closed"));
}
if let std::result::Result::Err(error) = processing_frontier.settle_pending(signal_slot) {
return std::result::Result::Err(error);
}
}
if let std::result::Result::Err(error) = self.start_hydrations(hydration) {
return std::result::Result::Err(error);
}
return std::result::Result::Ok(true);
}
async fn abort_all(&mut self, processing_frontier: &mut RawTransactionIngestProcessingFrontierReporter) {
self.tasks.abort_all();
while self.tasks.join_next().await.is_some() {}
self.pending.clear();
self.pending_signal_count = 0;
processing_frontier.discard_all_pending();
return;
}
}
fn hydration_key(
hydration: &RawTransactionIngestHydrationContext,
signal: &RawTransactionIngestSourceSignal,
) -> ksp_core_lib::Result<RawTransactionIngestHydrationKey> {
if signal.network != hydration.network {
return std::result::Result::Err(crate::runtime_error("hydration.network_mismatch"));
}
return std::result::Result::Ok(RawTransactionIngestHydrationKey {
commitment: hydration.commitment.as_str(),
network: signal.network.clone(),
signature: signal.signature,
});
}
async fn fetch_hydration(
http_pool: ksp_onchain_transport_lib::HttpTransportPool,
hydration_role: ksp_onchain_transport_lib::HttpRoleName,
expected_network: ksp_store_lib::RawNetworkId,
key: RawTransactionIngestHydrationKey,
commitment: ksp_onchain_transport_lib::SolanaCommitment,
) -> ksp_core_lib::Result<RawTransactionIngestHydrationFetch> {
if key.network != expected_network {
return std::result::Result::Err(crate::runtime_error("hydration.network_key_mismatch"));
}
if key.commitment != commitment.as_str() {
return std::result::Result::Err(crate::runtime_error("hydration.commitment_key_mismatch"));
}
let config = ksp_onchain_transport_lib::SolanaGetTransactionConfig::new(
std::option::Option::Some(commitment),
std::option::Option::Some(ksp_onchain_transport_lib::SolanaTransactionEncoding::Base64),
std::option::Option::Some(1),
);
let signature_text = ksp_raw_transaction_lib::format_raw_transaction_signature(&key.signature);
let observed = http_pool.get_transaction_observed(&hydration_role, signature_text.as_str(), std::option::Option::Some(&config)).await;
let observed = match observed {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(hydration_transport_error(error.code())),
};
return std::result::Result::Ok(RawTransactionIngestHydrationFetch { key, observed });
}
struct RawTransactionIngestHydrationLeaderGuard {
armed: bool,
key: RawTransactionIngestHydrationKey,
registry: std::sync::Arc<RawTransactionIngestGlobalHydrationRegistry>,
}
impl RawTransactionIngestHydrationLeaderGuard {
fn new(registry: std::sync::Arc<RawTransactionIngestGlobalHydrationRegistry>, key: RawTransactionIngestHydrationKey) -> Self {
return Self { armed: true, key, registry };
}
fn disarm(&mut self) {
self.armed = false;
return;
}
}
impl std::ops::Drop for RawTransactionIngestHydrationLeaderGuard {
fn drop(&mut self) {
if self.armed {
self.registry
.publish_and_remove(&self.key, RawTransactionIngestSharedHydrationResult::Failed(crate::ERROR_CODE_RAW_TRANSACTION_INGEST_SOURCE_FAILED));
}
return;
}
}
async fn fetch_hydration_shared(
global_registry: std::sync::Arc<RawTransactionIngestGlobalHydrationRegistry>,
http_pool: ksp_onchain_transport_lib::HttpTransportPool,
hydration_role: ksp_onchain_transport_lib::HttpRoleName,
expected_network: ksp_store_lib::RawNetworkId,
key: RawTransactionIngestHydrationKey,
commitment: ksp_onchain_transport_lib::SolanaCommitment,
) -> ksp_core_lib::Result<RawTransactionIngestHydrationFetch> {
let (leader, mut receiver) = match global_registry.subscribe_or_lead(&key) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
if leader {
let mut leader_guard = RawTransactionIngestHydrationLeaderGuard::new(std::sync::Arc::clone(&global_registry), key.clone());
let _permit = match global_registry.acquire_hydration_permit(RawTransactionIngestTrafficClass::Nominal).await {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let fetched = fetch_hydration(http_pool, hydration_role, expected_network, key.clone(), commitment).await;
return match fetched {
std::result::Result::Ok(value) => {
global_registry.publish_and_remove(&key, RawTransactionIngestSharedHydrationResult::Available(value.observed.clone()));
leader_guard.disarm();
std::result::Result::Ok(value)
},
std::result::Result::Err(error) => {
global_registry.publish_and_remove(&key, RawTransactionIngestSharedHydrationResult::Failed(error.code()));
leader_guard.disarm();
std::result::Result::Err(error)
},
};
}
loop {
if let std::option::Option::Some(result) = receiver.borrow().clone() {
return shared_hydration_result(key, result);
}
if receiver.changed().await.is_err() {
return std::result::Result::Err(crate::runtime_error("source.global_hydration_channel_closed"));
}
}
}
fn shared_hydration_result(
key: RawTransactionIngestHydrationKey,
result: RawTransactionIngestSharedHydrationResult,
) -> ksp_core_lib::Result<RawTransactionIngestHydrationFetch> {
return match result {
RawTransactionIngestSharedHydrationResult::Available(observed) => std::result::Result::Ok(RawTransactionIngestHydrationFetch { key, observed }),
RawTransactionIngestSharedHydrationResult::Failed(code) => {
std::result::Result::Err(ksp_core_lib::Error::new(code, "RAW transaction ingest shared hydration failed"))
},
};
}
fn resolve_known_reference_hydration(
hydration: &RawTransactionIngestHydrationContext,
settings: &crate::RawTransactionIngestSettings,
signal: RawTransactionIngestSourceSignal,
received_at: ksp_store_lib::RawTimestamp,
observed: &RawTransactionIngestObservedTransaction,
) -> ksp_core_lib::Result<RawTransactionIngestKnownReferenceHydrationResolution> {
let reference = ksp_store_lib::RawTransactionReference::new(signal.network.clone(), signal.signature);
let obligation = match crate::RawTransactionIngestKnownReferenceObligation::new(reference, signal.slot, hydration.commitment) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let ingress = finalize_hydration(hydration, settings, signal, received_at, observed);
let ingress = match ingress {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
if let std::option::Option::Some(value) = ingress {
return std::result::Result::Ok(RawTransactionIngestKnownReferenceHydrationResolution::Available(std::boxed::Box::new(value)));
}
let block_slot_supported = match http_role_supports_rpc_method(&hydration.http_pool, &hydration.hydration_role, "getBlock", hydration.network.as_str()) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let disposition = if block_slot_supported {
RawTransactionIngestKnownReferenceMissingDisposition::PreferBlockSlot
} else {
RawTransactionIngestKnownReferenceMissingDisposition::AwaitCoverage
};
return std::result::Result::Ok(RawTransactionIngestKnownReferenceHydrationResolution::Missing { disposition, obligation });
}
fn finalize_hydration(
hydration: &RawTransactionIngestHydrationContext,
settings: &crate::RawTransactionIngestSettings,
signal: RawTransactionIngestSourceSignal,
received_at: ksp_store_lib::RawTimestamp,
observed: &RawTransactionIngestObservedTransaction,
) -> ksp_core_lib::Result<std::option::Option<crate::RawTransactionIngress>> {
if &signal.network != settings.network() || signal.network != hydration.network {
return std::result::Result::Err(crate::runtime_error("hydration.network_mismatch"));
}
if signal.route != hydration.route {
return std::result::Result::Err(crate::runtime_error("hydration.source_route_mismatch"));
}
let commitment = hydration.commitment;
let transaction = match observed.value().as_ref() {
std::option::Option::Some(value) => value,
std::option::Option::None => return std::result::Result::Ok(std::option::Option::None),
};
if transaction.slot() != signal.slot {
return std::result::Result::Err(crate::runtime_error("hydration.slot_mismatch"));
}
if let std::option::Option::Some(expected_index) = signal.transaction_index
&& let ksp_onchain_transport_lib::SolanaWireField::Value(actual_index) = transaction.transaction_index()
&& expected_index != u64::from(*actual_index)
{
return std::result::Result::Err(crate::runtime_error("hydration.transaction_index_mismatch"));
}
let transaction_data = match transaction.transaction() {
ksp_onchain_transport_lib::SolanaEncodedTransaction::Binary { data, encoding }
if *encoding == ksp_onchain_transport_lib::SolanaTransactionBinaryEncoding::Base64 =>
{
data.clone()
},
ksp_onchain_transport_lib::SolanaEncodedTransaction::Binary { .. }
| ksp_onchain_transport_lib::SolanaEncodedTransaction::LegacyBinary(_)
| ksp_onchain_transport_lib::SolanaEncodedTransaction::Json(_) => {
return std::result::Result::Err(crate::runtime_error("hydration.transaction_encoding_invalid"));
},
};
let embedded_signature = ksp_raw_transaction_lib::extract_raw_transaction_signature_from_binary_base64(transaction_data.as_str());
let embedded_signature = match embedded_signature {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return std::result::Result::Err(crate::runtime_error("hydration.transaction_signature_invalid")),
};
if embedded_signature != signal.signature {
return std::result::Result::Err(crate::runtime_error("hydration.signature_mismatch"));
}
let provenance = build_hydration_provenance(hydration, settings, &signal, observed.provider().as_str(), observed.endpoint_name(), commitment, received_at);
let provenance = match provenance {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let source_key = hydration_source_key(hydration.source_key_domain, &provenance);
let material = ksp_raw_transaction_lib::RawTransactionMaterial::binary_base64(
signal.network.clone(),
signal.signature,
transaction.slot(),
transaction.block_time(),
transaction_data,
map_hydration_wire_field(transaction.meta(), |value| return value.clone()),
map_hydration_wire_field(transaction.version(), |value| {
return match value {
ksp_onchain_transport_lib::SolanaTransactionVersion::Legacy => ksp_raw_transaction_lib::RawTransactionVersion::Legacy,
ksp_onchain_transport_lib::SolanaTransactionVersion::Number(number) => ksp_raw_transaction_lib::RawTransactionVersion::Number(*number),
};
}),
map_hydration_wire_field(transaction.transaction_index(), |value| return *value),
);
return std::result::Result::Ok(std::option::Option::Some(crate::RawTransactionIngress { material, network: signal.network, provenance, source_key }));
}
#[cfg(test)]
enum RawTransactionIngestHydrationOutcome {
Available(std::boxed::Box<crate::RawTransactionIngress>),
Missing(ksp_store_lib::RawTransactionReference),
}
#[cfg(test)]
async fn hydrate_yellowstone_signal(
source: &crate::RawTransactionIngestYellowstoneSource,
settings: &crate::RawTransactionIngestSettings,
signal: RawTransactionIngestSourceSignal,
received_at: ksp_store_lib::RawTimestamp,
) -> ksp_core_lib::Result<RawTransactionIngestHydrationOutcome> {
let reference = ksp_store_lib::RawTransactionReference::new(signal.network.clone(), signal.signature);
let hydration = source.hydration_context();
let key = match hydration_key(&hydration, &signal) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let commitment = hydration.commitment;
let fetched = fetch_hydration(hydration.http_pool.clone(), hydration.hydration_role.clone(), hydration.network.clone(), key, commitment).await;
let fetched = match fetched {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let ingress = finalize_hydration(&hydration, settings, signal, received_at, &fetched.observed);
return match ingress {
std::result::Result::Ok(std::option::Option::Some(value)) => {
std::result::Result::Ok(RawTransactionIngestHydrationOutcome::Available(std::boxed::Box::new(value)))
},
std::result::Result::Ok(std::option::Option::None) => std::result::Result::Ok(RawTransactionIngestHydrationOutcome::Missing(reference)),
std::result::Result::Err(error) => std::result::Result::Err(error),
};
}
fn build_hydration_provenance(
hydration: &RawTransactionIngestHydrationContext,
settings: &crate::RawTransactionIngestSettings,
signal: &RawTransactionIngestSourceSignal,
http_provider: &str,
http_endpoint: &str,
commitment: ksp_onchain_transport_lib::SolanaCommitment,
received_at: ksp_store_lib::RawTimestamp,
) -> ksp_core_lib::Result<ksp_store_lib::RawAcquisitionProvenance> {
let (provider, endpoint_id) = match composite_provenance_codes(&signal.route, hydration.route_prefix, http_provider, http_endpoint) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let protocol = match ksp_store_lib::RawProvenanceCode::new(hydration.protocol) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return std::result::Result::Err(crate::runtime_error("hydration.protocol_unrepresentable")),
};
let acquisition_method = match ksp_store_lib::RawProvenanceCode::new(hydration_method_code(signal.family)) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return std::result::Result::Err(crate::runtime_error("hydration.method_unrepresentable")),
};
let capture_session = match ksp_store_lib::RawProvenanceCode::new(settings.worker_id().as_str()) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return std::result::Result::Err(crate::runtime_error("hydration.capture_session_unrepresentable")),
};
let commitment = match ksp_store_lib::RawProvenanceCode::new(commitment.as_str()) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return std::result::Result::Err(crate::runtime_error("hydration.commitment_unrepresentable")),
};
let filter_id = match signal.matched_filter_id.clone() {
std::option::Option::Some(value) => value,
std::option::Option::None => match fingerprint_filter_code(&signal.matched_filter_fingerprint) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
},
};
let mut provenance =
ksp_store_lib::RawAcquisitionProvenance::new(provider, protocol, acquisition_method, ksp_store_lib::RawAcquisitionOrigin::Live, received_at)
.with_endpoint_id(endpoint_id)
.with_commitment(commitment)
.with_capture_session_id(capture_session)
.with_filter_id(filter_id);
if let std::option::Option::Some(observed_at) = representable_observed_at(signal.created_at, received_at) {
provenance = match provenance.try_with_observed_at(observed_at) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return std::result::Result::Err(crate::runtime_error("hydration.observed_at_invalid")),
};
}
return std::result::Result::Ok(provenance);
}
fn hydration_method_code(family: RawTransactionIngestSourceFamily) -> &'static str {
return match family {
RawTransactionIngestSourceFamily::Block => "block_get_transaction",
RawTransactionIngestSourceFamily::Logs => "logs_get_transaction",
RawTransactionIngestSourceFamily::Transaction => "transaction_get_transaction",
RawTransactionIngestSourceFamily::TransactionStatus => "status_get_transaction",
};
}
fn fingerprint_filter_code(fingerprint: &[u8; 32]) -> ksp_core_lib::Result<ksp_store_lib::RawProvenanceCode> {
const HEX: &[u8; 16] = b"0123456789abcdef";
let mut value = std::string::String::with_capacity(71);
value.push_str("sha256.");
for byte in fingerprint {
value.push(char::from(HEX[usize::from(*byte >> 4)]));
value.push(char::from(HEX[usize::from(*byte & 0x0f)]));
}
return match ksp_store_lib::RawProvenanceCode::new(value) {
std::result::Result::Ok(value) => std::result::Result::Ok(value),
std::result::Result::Err(_) => std::result::Result::Err(crate::runtime_error("hydration.filter_fingerprint_unrepresentable")),
};
}
fn representable_observed_at(
created_at: std::option::Option<RawTransactionIngestSourceTimestamp>,
received_at: ksp_store_lib::RawTimestamp,
) -> std::option::Option<ksp_store_lib::RawTimestamp> {
let created_at = match created_at {
std::option::Option::Some(value) => value,
std::option::Option::None => return std::option::Option::None,
};
let seconds = match u64::try_from(created_at.seconds) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return std::option::Option::None,
};
let millis = match seconds.checked_mul(1_000).and_then(|value| return value.checked_add(u64::from(created_at.nanos / 1_000_000))) {
std::option::Option::Some(value) => value,
std::option::Option::None => return std::option::Option::None,
};
let observed_at = match ksp_store_lib::RawTimestamp::from_unix_millis(millis) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return std::option::Option::None,
};
if observed_at > received_at {
return std::option::Option::None;
}
return std::option::Option::Some(observed_at);
}
fn hydration_source_key(source_key_domain: &[u8], provenance: &ksp_store_lib::RawAcquisitionProvenance) -> [u8; 32] {
let mut hasher = sha2::Sha256::new();
hasher.update(source_key_domain);
hash_hydration_source_key(&mut hasher, provenance.provider().as_str().as_bytes());
hash_hydration_source_key(&mut hasher, provenance.protocol().as_str().as_bytes());
hash_hydration_source_key(&mut hasher, provenance.acquisition_method().as_str().as_bytes());
if let std::option::Option::Some(value) = provenance.endpoint_id() {
hash_hydration_source_key(&mut hasher, value.as_str().as_bytes());
}
if let std::option::Option::Some(value) = provenance.commitment() {
hash_hydration_source_key(&mut hasher, value.as_str().as_bytes());
}
if let std::option::Option::Some(value) = provenance.capture_session_id() {
hash_hydration_source_key(&mut hasher, value.as_str().as_bytes());
}
if let std::option::Option::Some(value) = provenance.filter_id() {
hash_hydration_source_key(&mut hasher, value.as_str().as_bytes());
}
return hasher.finalize().into();
}
fn hash_hydration_source_key(hasher: &mut sha2::Sha256, value: &[u8]) {
hasher.update((value.len() as u64).to_be_bytes());
hasher.update(value);
return;
}
fn hydration_transport_error(code: ksp_core_lib::ErrorCode) -> ksp_core_lib::Error {
return ksp_core_lib::Error::new(crate::ERROR_CODE_RAW_TRANSACTION_INGEST_SOURCE_FAILED, "RAW transaction ingest hydration transport failed")
.with_context("transport_domain", code.domain())
.with_context("transport_code", code.code());
}
fn map_hydration_wire_field<T, U, F>(
field: &ksp_onchain_transport_lib::SolanaWireField<T>,
mut map_value: F,
) -> ksp_raw_transaction_lib::RawTransactionWireField<U>
where
F: FnMut(&T) -> U,
{
return match field {
ksp_onchain_transport_lib::SolanaWireField::Omitted => ksp_raw_transaction_lib::RawTransactionWireField::Omitted,
ksp_onchain_transport_lib::SolanaWireField::Null => ksp_raw_transaction_lib::RawTransactionWireField::Null,
ksp_onchain_transport_lib::SolanaWireField::Value(value) => ksp_raw_transaction_lib::RawTransactionWireField::Value(map_value(value)),
};
}
#[cfg(test)]
#[path = "../unit_tests/runtime_resources.rs"]
mod tests;
Reference in New Issue View Git Blame Copy Permalink