v0.3.11-pre.009
This commit is contained in:
@@ -1,5 +1,5 @@
|
||||
// file: crates/ksp-worker-raw-transaction-ingest-lib/src/admission.rs
|
||||
// version: 2
|
||||
// version: 3
|
||||
|
||||
use sha2::Digest; // rust-rules: trait-import
|
||||
|
||||
@@ -19,6 +19,8 @@ pub(crate) struct RawTransactionIngress {
|
||||
|
||||
/// Receiver side of the bounded central RAW transaction admission queue owned by the Worker supervisor.
|
||||
pub(crate) struct RawTransactionAdmission {
|
||||
backpressure_wait_observed: bool,
|
||||
capacity: usize,
|
||||
receiver: tokio::sync::mpsc::Receiver<crate::RawTransactionIngress>,
|
||||
}
|
||||
|
||||
@@ -27,7 +29,7 @@ impl crate::RawTransactionAdmission {
|
||||
#[must_use]
|
||||
pub(crate) fn new(capacity: usize) -> (crate::RawTransactionAdmission, tokio::sync::mpsc::Sender<crate::RawTransactionIngress>) {
|
||||
let (sender, receiver) = tokio::sync::mpsc::channel(capacity);
|
||||
return (Self { receiver }, sender);
|
||||
return (Self { backpressure_wait_observed: false, capacity, receiver }, sender);
|
||||
}
|
||||
|
||||
/// Returns the latest receiver-side queue depth without consuming an ingress entry.
|
||||
@@ -47,6 +49,7 @@ impl crate::RawTransactionAdmission {
|
||||
&mut self,
|
||||
expected_network: &ksp_store_lib::RawNetworkId,
|
||||
) -> ksp_core_lib::Result<std::option::Option<ksp_raw_transaction_lib::RawTransactionAcquisition>> {
|
||||
self.backpressure_wait_observed = self.receiver.len() == self.capacity;
|
||||
let ingress = match self.receiver.recv().await {
|
||||
std::option::Option::Some(value) => value,
|
||||
std::option::Option::None => return std::result::Result::Ok(std::option::Option::None),
|
||||
@@ -57,6 +60,14 @@ impl crate::RawTransactionAdmission {
|
||||
std::result::Result::Err(error) => std::result::Result::Err(error),
|
||||
};
|
||||
}
|
||||
|
||||
/// Takes and clears the source-neutral signal that the previous dequeue observed the bounded queue at full capacity.
|
||||
#[must_use]
|
||||
pub(crate) fn take_backpressure_wait_observed(&mut self) -> bool {
|
||||
let observed = self.backpressure_wait_observed;
|
||||
self.backpressure_wait_observed = false;
|
||||
return observed;
|
||||
}
|
||||
}
|
||||
|
||||
fn canonicalize_ingress(
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
// file: crates/ksp-worker-raw-transaction-ingest-lib/src/error.rs
|
||||
// version: 5
|
||||
// version: 6
|
||||
|
||||
/// Error code used when durable Store content conflicts with one admitted canonical RAW transaction.
|
||||
pub const ERROR_CODE_RAW_TRANSACTION_INGEST_CONTENT_CONFLICT: ksp_core_lib::ErrorCode =
|
||||
@@ -7,12 +7,18 @@ pub const ERROR_CODE_RAW_TRANSACTION_INGEST_CONTENT_CONFLICT: ksp_core_lib::Erro
|
||||
/// Error code used when one monotone RAW transaction ingest Worker counter or snapshot sequence cannot advance without wrapping.
|
||||
pub const ERROR_CODE_RAW_TRANSACTION_INGEST_COUNTER_EXHAUSTED: ksp_core_lib::ErrorCode =
|
||||
ksp_core_lib::ErrorCode::new("worker_raw_transaction_ingest", "counter_exhausted");
|
||||
/// Error code used when private Worker tasks cannot drain before the configured shutdown deadline.
|
||||
pub const ERROR_CODE_RAW_TRANSACTION_INGEST_DRAIN_TIMEOUT: ksp_core_lib::ErrorCode =
|
||||
ksp_core_lib::ErrorCode::new("worker_raw_transaction_ingest", "drain_timeout");
|
||||
/// Error code used when the RAW transaction ingest Worker reaches an invalid runtime or lifecycle condition.
|
||||
pub const ERROR_CODE_RAW_TRANSACTION_INGEST_RUNTIME_INVALID: ksp_core_lib::ErrorCode =
|
||||
ksp_core_lib::ErrorCode::new("worker_raw_transaction_ingest", "runtime_invalid");
|
||||
/// Error code used when RAW transaction ingest Worker settings violate one bounded runtime invariant.
|
||||
pub const ERROR_CODE_RAW_TRANSACTION_INGEST_SETTINGS_INVALID: ksp_core_lib::ErrorCode =
|
||||
ksp_core_lib::ErrorCode::new("worker_raw_transaction_ingest", "settings_invalid");
|
||||
/// Error code used when one private source task terminates with a classified failure.
|
||||
pub const ERROR_CODE_RAW_TRANSACTION_INGEST_SOURCE_FAILED: ksp_core_lib::ErrorCode =
|
||||
ksp_core_lib::ErrorCode::new("worker_raw_transaction_ingest", "source_failed");
|
||||
/// Error code used when Store persistence fails for one admitted canonical RAW transaction.
|
||||
pub const ERROR_CODE_RAW_TRANSACTION_INGEST_STORE_FAILED: ksp_core_lib::ErrorCode =
|
||||
ksp_core_lib::ErrorCode::new("worker_raw_transaction_ingest", "store_failed");
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
// file: crates/ksp-worker-raw-transaction-ingest-lib/src/lib.rs
|
||||
// version: 7
|
||||
// version: 8
|
||||
|
||||
#![warn(missing_docs)]
|
||||
#![deny(unreachable_pub)]
|
||||
@@ -25,10 +25,14 @@ mod snapshot;
|
||||
pub use self::error::ERROR_CODE_RAW_TRANSACTION_INGEST_CONTENT_CONFLICT;
|
||||
/// Error code used when one monotone Worker counter or snapshot sequence cannot advance without wrapping.
|
||||
pub use self::error::ERROR_CODE_RAW_TRANSACTION_INGEST_COUNTER_EXHAUSTED;
|
||||
/// Error code used when private Worker tasks cannot drain before the configured shutdown deadline.
|
||||
pub use self::error::ERROR_CODE_RAW_TRANSACTION_INGEST_DRAIN_TIMEOUT;
|
||||
/// Error code used when the RAW transaction ingest Worker reaches an invalid runtime or lifecycle condition.
|
||||
pub use self::error::ERROR_CODE_RAW_TRANSACTION_INGEST_RUNTIME_INVALID;
|
||||
/// Error code used when RAW transaction ingest Worker settings violate one bounded runtime invariant.
|
||||
pub use self::error::ERROR_CODE_RAW_TRANSACTION_INGEST_SETTINGS_INVALID;
|
||||
/// Error code used when one private source task terminates with a classified failure.
|
||||
pub use self::error::ERROR_CODE_RAW_TRANSACTION_INGEST_SOURCE_FAILED;
|
||||
/// Error code used when Store persistence fails for one admitted canonical RAW transaction.
|
||||
pub use self::error::ERROR_CODE_RAW_TRANSACTION_INGEST_STORE_FAILED;
|
||||
/// Stable Worker kind code used by the continuous RAW transaction ingest vertical.
|
||||
|
||||
@@ -1,8 +1,9 @@
|
||||
// file: crates/ksp-worker-raw-transaction-ingest-lib/src/runtime.rs
|
||||
// version: 6
|
||||
// version: 7
|
||||
|
||||
type PersistencePort = std::sync::Arc<dyn crate::RawTransactionIngestPersistencePort + 'static>;
|
||||
type PersistenceTasks = tokio::task::JoinSet<ksp_core_lib::Result<crate::RawTransactionIngestPersistenceOutcome>>;
|
||||
type SourceTasks = tokio::task::JoinSet<ksp_core_lib::Result<()>>;
|
||||
|
||||
/// Runtime-neutral boxed future resolving after one RAW transaction ingest Worker has fully reached a terminal lifecycle state.
|
||||
pub type RawTransactionIngestTerminalFuture<'a> =
|
||||
@@ -118,12 +119,15 @@ fn current_runtime_handle() -> ksp_core_lib::Result<tokio::runtime::Handle> {
|
||||
};
|
||||
}
|
||||
|
||||
async fn drain_children(children: &mut tokio::task::JoinSet<()>) -> std::option::Option<ksp_core_lib::ErrorCode> {
|
||||
async fn drain_children(
|
||||
state: ksp_worker_api::WorkerState,
|
||||
children: &mut SourceTasks,
|
||||
snapshots: &mut crate::RawTransactionIngestSnapshotPublisher,
|
||||
) -> std::option::Option<ksp_core_lib::ErrorCode> {
|
||||
let mut fault = std::option::Option::None;
|
||||
while let std::option::Option::Some(joined) = children.join_next().await {
|
||||
if joined.is_err() && fault.is_none() {
|
||||
fault = std::option::Option::Some(crate::ERROR_CODE_RAW_TRANSACTION_INGEST_RUNTIME_INVALID);
|
||||
}
|
||||
let current = source_completion(joined, state, 0, 0, snapshots);
|
||||
fault = merge_fault(fault, current);
|
||||
}
|
||||
return fault;
|
||||
}
|
||||
@@ -137,9 +141,7 @@ async fn drain_persistence(
|
||||
let mut fault = std::option::Option::None;
|
||||
while let std::option::Option::Some(joined) = persistence.join_next().await {
|
||||
let current = persistence_completion(joined, lifecycle.state(), admission.queue_depth(), persistence.len(), snapshots);
|
||||
if fault.is_none() {
|
||||
fault = current;
|
||||
}
|
||||
fault = merge_fault(fault, current);
|
||||
}
|
||||
return fault;
|
||||
}
|
||||
@@ -161,15 +163,14 @@ async fn drain_admission_and_persistence(
|
||||
std::option::Option::Some(value) => persistence_completion(value, lifecycle.state(), admission.queue_depth(), persistence.len(), snapshots),
|
||||
std::option::Option::None => std::option::Option::Some(crate::ERROR_CODE_RAW_TRANSACTION_INGEST_RUNTIME_INVALID),
|
||||
};
|
||||
if fault.is_none() {
|
||||
fault = current;
|
||||
}
|
||||
fault = merge_fault(fault, current);
|
||||
}
|
||||
let received = admission.receive(settings.network()).await;
|
||||
let backpressure_wait_observed = admission.take_backpressure_wait_observed();
|
||||
match received {
|
||||
std::result::Result::Ok(std::option::Option::Some(acquisition)) => {
|
||||
let spawned = spawn_persistence(persistence, port, acquisition);
|
||||
let published = snapshots.record_admission_success(lifecycle.state(), admission.queue_depth(), persistence.len());
|
||||
let published = snapshots.record_admission_success(lifecycle.state(), admission.queue_depth(), persistence.len(), backpressure_wait_observed);
|
||||
if !spawned && fault.is_none() {
|
||||
fault = std::option::Option::Some(crate::ERROR_CODE_RAW_TRANSACTION_INGEST_RUNTIME_INVALID);
|
||||
}
|
||||
@@ -181,24 +182,56 @@ async fn drain_admission_and_persistence(
|
||||
},
|
||||
std::result::Result::Ok(std::option::Option::None) => break,
|
||||
std::result::Result::Err(_) => {
|
||||
let published = snapshots.record_admission_failure(lifecycle.state(), admission.queue_depth(), persistence.len());
|
||||
if let std::result::Result::Err(error) = published {
|
||||
if fault.is_none() {
|
||||
fault = std::option::Option::Some(error.code());
|
||||
}
|
||||
} else if fault.is_none() {
|
||||
fault = std::option::Option::Some(crate::ERROR_CODE_RAW_TRANSACTION_INGEST_RUNTIME_INVALID);
|
||||
let published = snapshots.record_admission_failure(lifecycle.state(), admission.queue_depth(), persistence.len(), backpressure_wait_observed);
|
||||
match published {
|
||||
std::result::Result::Ok(()) => {
|
||||
if fault.is_none() {
|
||||
fault = std::option::Option::Some(crate::ERROR_CODE_RAW_TRANSACTION_INGEST_RUNTIME_INVALID);
|
||||
}
|
||||
},
|
||||
std::result::Result::Err(error) => {
|
||||
if fault.is_none() {
|
||||
fault = std::option::Option::Some(error.code());
|
||||
}
|
||||
},
|
||||
}
|
||||
},
|
||||
}
|
||||
}
|
||||
let persistence_fault = drain_persistence(lifecycle, admission, persistence, snapshots).await;
|
||||
if fault.is_none() {
|
||||
fault = persistence_fault;
|
||||
}
|
||||
fault = merge_fault(fault, persistence_fault);
|
||||
return fault;
|
||||
}
|
||||
|
||||
async fn drain_owned_work(
|
||||
settings: &crate::RawTransactionIngestSettings,
|
||||
lifecycle: &ksp_worker_api::WorkerLifecycle,
|
||||
children: &mut SourceTasks,
|
||||
admission: &mut crate::RawTransactionAdmission,
|
||||
persistence: &mut PersistenceTasks,
|
||||
port: &std::option::Option<PersistencePort>,
|
||||
snapshots: &mut crate::RawTransactionIngestSnapshotPublisher,
|
||||
) -> std::option::Option<ksp_core_lib::ErrorCode> {
|
||||
let drain = async {
|
||||
let mut fault = drain_admission_and_persistence(settings, lifecycle, admission, persistence, port, snapshots).await;
|
||||
let child_fault = drain_children(lifecycle.state(), children, snapshots).await;
|
||||
fault = merge_fault(fault, child_fault);
|
||||
return fault;
|
||||
};
|
||||
let timed = tokio::time::timeout(settings.shutdown_drain_timeout(), drain).await;
|
||||
return match timed {
|
||||
std::result::Result::Ok(fault) => fault,
|
||||
std::result::Result::Err(_) => {
|
||||
admission.close();
|
||||
persistence.abort_all();
|
||||
children.abort_all();
|
||||
while persistence.join_next().await.is_some() {}
|
||||
while children.join_next().await.is_some() {}
|
||||
std::option::Option::Some(crate::ERROR_CODE_RAW_TRANSACTION_INGEST_DRAIN_TIMEOUT)
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
fn finish_faulted(
|
||||
lifecycle: &mut ksp_worker_api::WorkerLifecycle,
|
||||
snapshots: &mut crate::RawTransactionIngestSnapshotPublisher,
|
||||
@@ -261,6 +294,24 @@ fn persistence_completion(
|
||||
};
|
||||
}
|
||||
|
||||
fn source_completion(
|
||||
joined: std::result::Result<ksp_core_lib::Result<()>, tokio::task::JoinError>,
|
||||
state: ksp_worker_api::WorkerState,
|
||||
admission_queue_depth: usize,
|
||||
in_flight_persistence: usize,
|
||||
snapshots: &mut crate::RawTransactionIngestSnapshotPublisher,
|
||||
) -> std::option::Option<ksp_core_lib::ErrorCode> {
|
||||
match joined {
|
||||
std::result::Result::Ok(std::result::Result::Ok(())) => return std::option::Option::None,
|
||||
std::result::Result::Ok(std::result::Result::Err(_)) | std::result::Result::Err(_) => {},
|
||||
}
|
||||
let published = snapshots.record_source_failure(state, admission_queue_depth, in_flight_persistence);
|
||||
return match published {
|
||||
std::result::Result::Ok(()) => std::option::Option::Some(crate::ERROR_CODE_RAW_TRANSACTION_INGEST_SOURCE_FAILED),
|
||||
std::result::Result::Err(error) => std::option::Option::Some(error.code()),
|
||||
};
|
||||
}
|
||||
|
||||
async fn run_supervisor<Spawner>(
|
||||
settings: crate::RawTransactionIngestSettings,
|
||||
mut lifecycle: ksp_worker_api::WorkerLifecycle,
|
||||
@@ -269,9 +320,8 @@ async fn run_supervisor<Spawner>(
|
||||
mut snapshots: crate::RawTransactionIngestSnapshotPublisher,
|
||||
source_spawner: Spawner,
|
||||
) where
|
||||
Spawner: FnOnce(&mut tokio::task::JoinSet<()>, tokio::sync::watch::Receiver<bool>, tokio::sync::mpsc::Sender<crate::RawTransactionIngress>)
|
||||
+ std::marker::Send
|
||||
+ 'static,
|
||||
Spawner:
|
||||
FnOnce(&mut SourceTasks, tokio::sync::watch::Receiver<bool>, tokio::sync::mpsc::Sender<crate::RawTransactionIngress>) + std::marker::Send + 'static,
|
||||
{
|
||||
if *stop_receiver.borrow() {
|
||||
let stopping_fault = begin_stopping(&mut lifecycle, &mut snapshots, 0, 0);
|
||||
@@ -289,7 +339,7 @@ async fn run_supervisor<Spawner>(
|
||||
finish_faulted(&mut lifecycle, &mut snapshots, error.code());
|
||||
return;
|
||||
}
|
||||
let mut children = tokio::task::JoinSet::new();
|
||||
let mut children = SourceTasks::new();
|
||||
let mut persistence = PersistenceTasks::new();
|
||||
let (source_stop_sender, source_stop_receiver) = tokio::sync::watch::channel(false);
|
||||
let (mut admission, admission_sender) = crate::RawTransactionAdmission::new(settings.admission_queue_capacity());
|
||||
@@ -306,15 +356,19 @@ async fn run_supervisor<Spawner>(
|
||||
&mut snapshots,
|
||||
)
|
||||
.await;
|
||||
if fault.is_some() {
|
||||
source_stop_sender.send_replace(true);
|
||||
}
|
||||
source_stop_sender.send_replace(true);
|
||||
let stopping_fault = begin_stopping(&mut lifecycle, &mut snapshots, admission.queue_depth(), persistence.len());
|
||||
fault = merge_fault(fault, stopping_fault);
|
||||
let drain_fault = drain_admission_and_persistence(&settings, &lifecycle, &mut admission, &mut persistence, &port, &mut snapshots).await;
|
||||
fault = merge_fault(fault, drain_fault);
|
||||
let child_fault = drain_children(&mut children).await;
|
||||
fault = merge_fault(fault, child_fault);
|
||||
let drain_fault = drain_owned_work(&settings, &lifecycle, &mut children, &mut admission, &mut persistence, &port, &mut snapshots).await;
|
||||
match drain_fault {
|
||||
std::option::Option::Some(code) if code == crate::ERROR_CODE_RAW_TRANSACTION_INGEST_DRAIN_TIMEOUT => {
|
||||
fault = std::option::Option::Some(code);
|
||||
},
|
||||
std::option::Option::Some(code) => {
|
||||
fault = merge_fault(fault, std::option::Option::Some(code));
|
||||
},
|
||||
std::option::Option::None => {},
|
||||
}
|
||||
match fault {
|
||||
std::option::Option::Some(code) => finish_faulted(&mut lifecycle, &mut snapshots, code),
|
||||
std::option::Option::None => finish_stopped(&mut lifecycle, &mut snapshots),
|
||||
@@ -352,9 +406,8 @@ fn start_foundation_with_port_and_source_spawner<Spawner>(
|
||||
source_spawner: Spawner,
|
||||
) -> ksp_core_lib::Result<crate::RawTransactionIngestHandle>
|
||||
where
|
||||
Spawner: FnOnce(&mut tokio::task::JoinSet<()>, tokio::sync::watch::Receiver<bool>, tokio::sync::mpsc::Sender<crate::RawTransactionIngress>)
|
||||
+ std::marker::Send
|
||||
+ 'static,
|
||||
Spawner:
|
||||
FnOnce(&mut SourceTasks, tokio::sync::watch::Receiver<bool>, tokio::sync::mpsc::Sender<crate::RawTransactionIngress>) + std::marker::Send + 'static,
|
||||
{
|
||||
let kind = match ksp_worker_api::WorkerKindCode::new(crate::RAW_TRANSACTION_INGEST_WORKER_KIND_CODE) {
|
||||
std::result::Result::Ok(value) => value,
|
||||
@@ -379,9 +432,8 @@ fn start_foundation_with_source_spawner<Spawner>(
|
||||
source_spawner: Spawner,
|
||||
) -> ksp_core_lib::Result<crate::RawTransactionIngestHandle>
|
||||
where
|
||||
Spawner: FnOnce(&mut tokio::task::JoinSet<()>, tokio::sync::watch::Receiver<bool>, tokio::sync::mpsc::Sender<crate::RawTransactionIngress>)
|
||||
+ std::marker::Send
|
||||
+ 'static,
|
||||
Spawner:
|
||||
FnOnce(&mut SourceTasks, tokio::sync::watch::Receiver<bool>, tokio::sync::mpsc::Sender<crate::RawTransactionIngress>) + std::marker::Send + 'static,
|
||||
{
|
||||
let port = match store_guard {
|
||||
std::option::Option::Some(store) => {
|
||||
@@ -398,7 +450,7 @@ async fn supervise_until_stop(
|
||||
lifecycle: &ksp_worker_api::WorkerLifecycle,
|
||||
source_stop_sender: &tokio::sync::watch::Sender<bool>,
|
||||
stop_receiver: &mut tokio::sync::watch::Receiver<bool>,
|
||||
children: &mut tokio::task::JoinSet<()>,
|
||||
children: &mut SourceTasks,
|
||||
admission: &mut crate::RawTransactionAdmission,
|
||||
persistence: &mut PersistenceTasks,
|
||||
port: &std::option::Option<PersistencePort>,
|
||||
@@ -427,25 +479,38 @@ async fn supervise_until_stop(
|
||||
}
|
||||
}
|
||||
joined = children.join_next(), if !children.is_empty() => {
|
||||
if let std::option::Option::Some(std::result::Result::Err(_)) = joined {
|
||||
let source_fault = match joined {
|
||||
std::option::Option::Some(value) => source_completion(value, lifecycle.state(), admission.queue_depth(), persistence.len(), snapshots),
|
||||
std::option::Option::None => std::option::Option::None,
|
||||
};
|
||||
if source_fault.is_some() {
|
||||
source_stop_sender.send_replace(true);
|
||||
return std::option::Option::Some(crate::ERROR_CODE_RAW_TRANSACTION_INGEST_RUNTIME_INVALID);
|
||||
return source_fault;
|
||||
}
|
||||
}
|
||||
joined = persistence.join_next(), if !persistence.is_empty() => {
|
||||
if let std::option::Option::Some(value) = joined {
|
||||
let fault = persistence_completion(value, lifecycle.state(), admission.queue_depth(), persistence.len(), snapshots);
|
||||
if fault.is_some() {
|
||||
source_stop_sender.send_replace(true);
|
||||
return fault;
|
||||
}
|
||||
match joined {
|
||||
std::option::Option::Some(value) => {
|
||||
let fault = persistence_completion(value, lifecycle.state(), admission.queue_depth(), persistence.len(), snapshots);
|
||||
if fault.is_some() {
|
||||
source_stop_sender.send_replace(true);
|
||||
return fault;
|
||||
}
|
||||
},
|
||||
std::option::Option::None => {},
|
||||
}
|
||||
}
|
||||
received = admission.receive(settings.network()), if admission_open && persistence.len() < settings.persistence_concurrency() => {
|
||||
match received {
|
||||
std::result::Result::Ok(std::option::Option::Some(acquisition)) => {
|
||||
let backpressure_wait_observed = admission.take_backpressure_wait_observed();
|
||||
let spawned = spawn_persistence(persistence, port, acquisition);
|
||||
let published = snapshots.record_admission_success(lifecycle.state(), admission.queue_depth(), persistence.len());
|
||||
let published = snapshots.record_admission_success(
|
||||
lifecycle.state(),
|
||||
admission.queue_depth(),
|
||||
persistence.len(),
|
||||
backpressure_wait_observed,
|
||||
);
|
||||
if !spawned {
|
||||
source_stop_sender.send_replace(true);
|
||||
return std::option::Option::Some(crate::ERROR_CODE_RAW_TRANSACTION_INGEST_RUNTIME_INVALID);
|
||||
@@ -455,9 +520,18 @@ async fn supervise_until_stop(
|
||||
return std::option::Option::Some(error.code());
|
||||
}
|
||||
},
|
||||
std::result::Result::Ok(std::option::Option::None) => admission_open = false,
|
||||
std::result::Result::Ok(std::option::Option::None) => {
|
||||
let _backpressure_wait_observed = admission.take_backpressure_wait_observed();
|
||||
admission_open = false;
|
||||
},
|
||||
std::result::Result::Err(_) => {
|
||||
let published = snapshots.record_admission_failure(lifecycle.state(), admission.queue_depth(), persistence.len());
|
||||
let backpressure_wait_observed = admission.take_backpressure_wait_observed();
|
||||
let published = snapshots.record_admission_failure(
|
||||
lifecycle.state(),
|
||||
admission.queue_depth(),
|
||||
persistence.len(),
|
||||
backpressure_wait_observed,
|
||||
);
|
||||
source_stop_sender.send_replace(true);
|
||||
return match published {
|
||||
std::result::Result::Ok(()) => std::option::Option::Some(crate::ERROR_CODE_RAW_TRANSACTION_INGEST_RUNTIME_INVALID),
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
// file: crates/ksp-worker-raw-transaction-ingest-lib/src/snapshot.rs
|
||||
// version: 1
|
||||
// version: 2
|
||||
|
||||
/// Runtime-neutral boxed future resolving to one newer concrete RAW transaction ingest Worker snapshot.
|
||||
pub type RawTransactionIngestSnapshotFuture<'a> =
|
||||
@@ -124,7 +124,7 @@ impl crate::RawTransactionIngestSnapshot {
|
||||
return self.source_failure_total;
|
||||
}
|
||||
|
||||
/// Returns the number of source-side admission waits explicitly classified as backpressure waits.
|
||||
/// Returns the number of supervisor dequeues that observed the bounded admission queue at full capacity.
|
||||
#[must_use]
|
||||
pub const fn backpressure_wait_total(&self) -> u64 {
|
||||
return self.backpressure_wait_total;
|
||||
@@ -303,12 +303,19 @@ impl crate::RawTransactionIngestSnapshotPublisher {
|
||||
state: ksp_worker_api::WorkerState,
|
||||
admission_queue_depth: usize,
|
||||
in_flight_persistence: usize,
|
||||
backpressure_wait_observed: bool,
|
||||
) -> ksp_core_lib::Result<()> {
|
||||
let admitted_total = match checked_counter(self.snapshot.admitted_total, "admitted_total") {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(error) => return std::result::Result::Err(error),
|
||||
};
|
||||
let backpressure_wait_total =
|
||||
match checked_optional_counter(self.snapshot.backpressure_wait_total, backpressure_wait_observed, "backpressure_wait_total") {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(error) => return std::result::Result::Err(error),
|
||||
};
|
||||
self.snapshot.admitted_total = admitted_total;
|
||||
self.snapshot.backpressure_wait_total = backpressure_wait_total;
|
||||
return self.publish(state, admission_queue_depth, in_flight_persistence);
|
||||
}
|
||||
|
||||
@@ -318,20 +325,42 @@ impl crate::RawTransactionIngestSnapshotPublisher {
|
||||
state: ksp_worker_api::WorkerState,
|
||||
admission_queue_depth: usize,
|
||||
in_flight_persistence: usize,
|
||||
backpressure_wait_observed: bool,
|
||||
) -> ksp_core_lib::Result<()> {
|
||||
let admitted_total = match checked_counter(self.snapshot.admitted_total, "admitted_total") {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(error) => return std::result::Result::Err(error),
|
||||
};
|
||||
let backpressure_wait_total =
|
||||
match checked_optional_counter(self.snapshot.backpressure_wait_total, backpressure_wait_observed, "backpressure_wait_total") {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(error) => return std::result::Result::Err(error),
|
||||
};
|
||||
let canonicalized_total = match checked_counter(self.snapshot.canonicalized_total, "canonicalized_total") {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(error) => return std::result::Result::Err(error),
|
||||
};
|
||||
self.snapshot.admitted_total = admitted_total;
|
||||
self.snapshot.backpressure_wait_total = backpressure_wait_total;
|
||||
self.snapshot.canonicalized_total = canonicalized_total;
|
||||
return self.publish(state, admission_queue_depth, in_flight_persistence);
|
||||
}
|
||||
|
||||
/// Records one failed private source task without retaining provider-specific error material.
|
||||
pub(crate) fn record_source_failure(
|
||||
&mut self,
|
||||
state: ksp_worker_api::WorkerState,
|
||||
admission_queue_depth: usize,
|
||||
in_flight_persistence: usize,
|
||||
) -> ksp_core_lib::Result<()> {
|
||||
let source_failure_total = match checked_counter(self.snapshot.source_failure_total, "source_failure_total") {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(error) => return std::result::Result::Err(error),
|
||||
};
|
||||
self.snapshot.source_failure_total = source_failure_total;
|
||||
return self.publish(state, admission_queue_depth, in_flight_persistence);
|
||||
}
|
||||
|
||||
/// Records one successful Store persistence outcome and publishes the resulting latest value.
|
||||
pub(crate) fn record_persistence_success(
|
||||
&mut self,
|
||||
@@ -458,6 +487,13 @@ fn checked_counter(current: u64, field: &'static str) -> ksp_core_lib::Result<u6
|
||||
};
|
||||
}
|
||||
|
||||
fn checked_optional_counter(current: u64, increment: bool, field: &'static str) -> ksp_core_lib::Result<u64> {
|
||||
if !increment {
|
||||
return std::result::Result::Ok(current);
|
||||
}
|
||||
return checked_counter(current, field);
|
||||
}
|
||||
|
||||
fn health_for_state(state: ksp_worker_api::WorkerState, previous: ksp_worker_api::WorkerHealth) -> ksp_worker_api::WorkerHealth {
|
||||
return match state {
|
||||
ksp_worker_api::WorkerState::Running => ksp_worker_api::WorkerHealth::Healthy,
|
||||
|
||||
Reference in New Issue
Block a user