v0.3.11-pre.008
This commit is contained in:
@@ -6,7 +6,7 @@ resolver = "3"
|
|||||||
members = ["crates/ksp-app-backfill-desk", "crates/ksp-app-config-desk", "crates/ksp-app-solprices-desk", "crates/ksp-app-store-desk", "crates/ksp-app-wallet-desk", "crates/ksp-config-lib", "crates/ksp-core-lib", "crates/ksp-interface-lib", "crates/ksp-job-api", "crates/ksp-job-backfill-lib", "crates/ksp-logging-lib", "crates/ksp-offchain-transport-lib", "crates/ksp-onchain-transport-lib", "crates/ksp-program-api", "crates/ksp-raw-transaction-lib", "crates/ksp-store-api", "crates/ksp-store-lib", "crates/ksp-store-postgres-lib", "crates/ksp-wallet-lib", "crates/ksp-worker-api", "crates/ksp-worker-raw-transaction-ingest-lib"]
|
members = ["crates/ksp-app-backfill-desk", "crates/ksp-app-config-desk", "crates/ksp-app-solprices-desk", "crates/ksp-app-store-desk", "crates/ksp-app-wallet-desk", "crates/ksp-config-lib", "crates/ksp-core-lib", "crates/ksp-interface-lib", "crates/ksp-job-api", "crates/ksp-job-backfill-lib", "crates/ksp-logging-lib", "crates/ksp-offchain-transport-lib", "crates/ksp-onchain-transport-lib", "crates/ksp-program-api", "crates/ksp-raw-transaction-lib", "crates/ksp-store-api", "crates/ksp-store-lib", "crates/ksp-store-postgres-lib", "crates/ksp-wallet-lib", "crates/ksp-worker-api", "crates/ksp-worker-raw-transaction-ingest-lib"]
|
||||||
|
|
||||||
[workspace.package]
|
[workspace.package]
|
||||||
version = "0.3.11-pre.7"
|
version = "0.3.11-pre.8"
|
||||||
edition = "2024"
|
edition = "2024"
|
||||||
license = "MIT"
|
license = "MIT"
|
||||||
repository = "https://git.sasedev.com/Sasedev/khadhroony-solana-project"
|
repository = "https://git.sasedev.com/Sasedev/khadhroony-solana-project"
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
// file: crates/ksp-worker-raw-transaction-ingest-lib/src/admission.rs
|
// file: crates/ksp-worker-raw-transaction-ingest-lib/src/admission.rs
|
||||||
// version: 1
|
// version: 2
|
||||||
|
|
||||||
use sha2::Digest; // rust-rules: trait-import
|
use sha2::Digest; // rust-rules: trait-import
|
||||||
|
|
||||||
@@ -30,6 +30,12 @@ impl crate::RawTransactionAdmission {
|
|||||||
return (Self { receiver }, sender);
|
return (Self { receiver }, sender);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Returns the latest receiver-side queue depth without consuming an ingress entry.
|
||||||
|
#[must_use]
|
||||||
|
pub(crate) fn queue_depth(&self) -> usize {
|
||||||
|
return self.receiver.len();
|
||||||
|
}
|
||||||
|
|
||||||
/// Closes new admissions while preserving already queued ingress for deterministic drain.
|
/// Closes new admissions while preserving already queued ingress for deterministic drain.
|
||||||
pub(crate) fn close(&mut self) {
|
pub(crate) fn close(&mut self) {
|
||||||
self.receiver.close();
|
self.receiver.close();
|
||||||
|
|||||||
@@ -1,9 +1,12 @@
|
|||||||
// file: crates/ksp-worker-raw-transaction-ingest-lib/src/error.rs
|
// file: crates/ksp-worker-raw-transaction-ingest-lib/src/error.rs
|
||||||
// version: 4
|
// version: 5
|
||||||
|
|
||||||
/// Error code used when durable Store content conflicts with one admitted canonical RAW transaction.
|
/// Error code used when durable Store content conflicts with one admitted canonical RAW transaction.
|
||||||
pub const ERROR_CODE_RAW_TRANSACTION_INGEST_CONTENT_CONFLICT: ksp_core_lib::ErrorCode =
|
pub const ERROR_CODE_RAW_TRANSACTION_INGEST_CONTENT_CONFLICT: ksp_core_lib::ErrorCode =
|
||||||
ksp_core_lib::ErrorCode::new("worker_raw_transaction_ingest", "content_conflict");
|
ksp_core_lib::ErrorCode::new("worker_raw_transaction_ingest", "content_conflict");
|
||||||
|
/// Error code used when one monotone RAW transaction ingest Worker counter or snapshot sequence cannot advance without wrapping.
|
||||||
|
pub const ERROR_CODE_RAW_TRANSACTION_INGEST_COUNTER_EXHAUSTED: ksp_core_lib::ErrorCode =
|
||||||
|
ksp_core_lib::ErrorCode::new("worker_raw_transaction_ingest", "counter_exhausted");
|
||||||
/// Error code used when the RAW transaction ingest Worker reaches an invalid runtime or lifecycle condition.
|
/// Error code used when the RAW transaction ingest Worker reaches an invalid runtime or lifecycle condition.
|
||||||
pub const ERROR_CODE_RAW_TRANSACTION_INGEST_RUNTIME_INVALID: ksp_core_lib::ErrorCode =
|
pub const ERROR_CODE_RAW_TRANSACTION_INGEST_RUNTIME_INVALID: ksp_core_lib::ErrorCode =
|
||||||
ksp_core_lib::ErrorCode::new("worker_raw_transaction_ingest", "runtime_invalid");
|
ksp_core_lib::ErrorCode::new("worker_raw_transaction_ingest", "runtime_invalid");
|
||||||
@@ -14,6 +17,12 @@ pub const ERROR_CODE_RAW_TRANSACTION_INGEST_SETTINGS_INVALID: ksp_core_lib::Erro
|
|||||||
pub const ERROR_CODE_RAW_TRANSACTION_INGEST_STORE_FAILED: ksp_core_lib::ErrorCode =
|
pub const ERROR_CODE_RAW_TRANSACTION_INGEST_STORE_FAILED: ksp_core_lib::ErrorCode =
|
||||||
ksp_core_lib::ErrorCode::new("worker_raw_transaction_ingest", "store_failed");
|
ksp_core_lib::ErrorCode::new("worker_raw_transaction_ingest", "store_failed");
|
||||||
|
|
||||||
|
/// Creates one terminal counter-exhaustion error without exposing runtime material.
|
||||||
|
pub(crate) fn counter_exhausted_error(field: &'static str) -> ksp_core_lib::Error {
|
||||||
|
return ksp_core_lib::Error::new(crate::ERROR_CODE_RAW_TRANSACTION_INGEST_COUNTER_EXHAUSTED, "RAW transaction ingest Worker counter exhausted")
|
||||||
|
.with_context("field", field);
|
||||||
|
}
|
||||||
|
|
||||||
/// Creates one terminal content-conflict error without copying conflicting material into diagnostics.
|
/// Creates one terminal content-conflict error without copying conflicting material into diagnostics.
|
||||||
pub(crate) fn content_conflict_error() -> ksp_core_lib::Error {
|
pub(crate) fn content_conflict_error() -> ksp_core_lib::Error {
|
||||||
return ksp_core_lib::Error::new(crate::ERROR_CODE_RAW_TRANSACTION_INGEST_CONTENT_CONFLICT, "RAW transaction ingest Store content conflict");
|
return ksp_core_lib::Error::new(crate::ERROR_CODE_RAW_TRANSACTION_INGEST_CONTENT_CONFLICT, "RAW transaction ingest Store content conflict");
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
// file: crates/ksp-worker-raw-transaction-ingest-lib/src/lib.rs
|
// file: crates/ksp-worker-raw-transaction-ingest-lib/src/lib.rs
|
||||||
// version: 6
|
// version: 7
|
||||||
|
|
||||||
#![warn(missing_docs)]
|
#![warn(missing_docs)]
|
||||||
#![deny(unreachable_pub)]
|
#![deny(unreachable_pub)]
|
||||||
@@ -10,8 +10,8 @@
|
|||||||
//! This tranche owns the concrete Worker family identity, validated technical settings
|
//! This tranche owns the concrete Worker family identity, validated technical settings
|
||||||
//! and the caller-runtime-owned lifecycle with private child-task supervision. This tranche also
|
//! and the caller-runtime-owned lifecycle with private child-task supervision. This tranche also
|
||||||
//! owns bounded source-neutral admission, common RAW canonicalization/assembly and backend-neutral
|
//! owns bounded source-neutral admission, common RAW canonicalization/assembly and backend-neutral
|
||||||
//! Store persistence in normal mode; latest-value snapshots remain in a later prerelease, and no live
|
//! Store persistence in normal mode plus concrete latest-value snapshots projected onto Worker API;
|
||||||
//! source or Transport dependency exists.
|
//! no live source or Transport dependency exists.
|
||||||
|
|
||||||
mod admission;
|
mod admission;
|
||||||
mod error;
|
mod error;
|
||||||
@@ -19,9 +19,12 @@ mod identity;
|
|||||||
mod persistence;
|
mod persistence;
|
||||||
mod runtime;
|
mod runtime;
|
||||||
mod settings;
|
mod settings;
|
||||||
|
mod snapshot;
|
||||||
|
|
||||||
/// Error code used when durable Store content conflicts with one admitted canonical RAW transaction.
|
/// Error code used when durable Store content conflicts with one admitted canonical RAW transaction.
|
||||||
pub use self::error::ERROR_CODE_RAW_TRANSACTION_INGEST_CONTENT_CONFLICT;
|
pub use self::error::ERROR_CODE_RAW_TRANSACTION_INGEST_CONTENT_CONFLICT;
|
||||||
|
/// Error code used when one monotone Worker counter or snapshot sequence cannot advance without wrapping.
|
||||||
|
pub use self::error::ERROR_CODE_RAW_TRANSACTION_INGEST_COUNTER_EXHAUSTED;
|
||||||
/// Error code used when the RAW transaction ingest Worker reaches an invalid runtime or lifecycle condition.
|
/// Error code used when the RAW transaction ingest Worker reaches an invalid runtime or lifecycle condition.
|
||||||
pub use self::error::ERROR_CODE_RAW_TRANSACTION_INGEST_RUNTIME_INVALID;
|
pub use self::error::ERROR_CODE_RAW_TRANSACTION_INGEST_RUNTIME_INVALID;
|
||||||
/// Error code used when RAW transaction ingest Worker settings violate one bounded runtime invariant.
|
/// Error code used when RAW transaction ingest Worker settings violate one bounded runtime invariant.
|
||||||
@@ -56,6 +59,12 @@ pub use self::settings::MIN_RAW_TRANSACTION_INGEST_PERSISTENCE_CONCURRENCY;
|
|||||||
pub use self::settings::MIN_RAW_TRANSACTION_INGEST_SHUTDOWN_DRAIN_TIMEOUT;
|
pub use self::settings::MIN_RAW_TRANSACTION_INGEST_SHUTDOWN_DRAIN_TIMEOUT;
|
||||||
/// Validated source-neutral runtime settings for one continuous RAW transaction ingest Worker.
|
/// Validated source-neutral runtime settings for one continuous RAW transaction ingest Worker.
|
||||||
pub use self::settings::RawTransactionIngestSettings;
|
pub use self::settings::RawTransactionIngestSettings;
|
||||||
|
/// Complete safe latest-value snapshot of one continuous RAW transaction ingest Worker.
|
||||||
|
pub use self::snapshot::RawTransactionIngestSnapshot;
|
||||||
|
/// Runtime-neutral boxed future resolving to one newer concrete RAW transaction ingest Worker snapshot.
|
||||||
|
pub use self::snapshot::RawTransactionIngestSnapshotFuture;
|
||||||
|
/// Cloneable latest-value source exposing concrete and common Worker snapshots from one shared watch state.
|
||||||
|
pub use self::snapshot::RawTransactionIngestSnapshotSource;
|
||||||
|
|
||||||
/// Receiver-side owner of the private bounded RAW transaction admission queue.
|
/// Receiver-side owner of the private bounded RAW transaction admission queue.
|
||||||
pub(crate) use self::admission::RawTransactionAdmission;
|
pub(crate) use self::admission::RawTransactionAdmission;
|
||||||
@@ -63,6 +72,8 @@ pub(crate) use self::admission::RawTransactionAdmission;
|
|||||||
pub(crate) use self::admission::RawTransactionIngress;
|
pub(crate) use self::admission::RawTransactionIngress;
|
||||||
/// Creates one terminal content-conflict error without copying conflicting material into diagnostics.
|
/// Creates one terminal content-conflict error without copying conflicting material into diagnostics.
|
||||||
pub(crate) use self::error::content_conflict_error;
|
pub(crate) use self::error::content_conflict_error;
|
||||||
|
/// Creates one terminal counter-exhaustion error without exposing runtime material.
|
||||||
|
pub(crate) use self::error::counter_exhausted_error;
|
||||||
/// Creates one runtime-domain error without copying runtime/provider/Store values into diagnostics.
|
/// Creates one runtime-domain error without copying runtime/provider/Store values into diagnostics.
|
||||||
pub(crate) use self::error::runtime_error;
|
pub(crate) use self::error::runtime_error;
|
||||||
/// Creates one settings-domain error without copying caller-supplied values into diagnostics.
|
/// Creates one settings-domain error without copying caller-supplied values into diagnostics.
|
||||||
@@ -79,3 +90,5 @@ pub(crate) use self::persistence::RawTransactionIngestPersistenceOutcome;
|
|||||||
pub(crate) use self::persistence::RawTransactionIngestPersistencePort;
|
pub(crate) use self::persistence::RawTransactionIngestPersistencePort;
|
||||||
/// Persists one already-canonical Worker acquisition through the private Store port in `Normal` mode.
|
/// Persists one already-canonical Worker acquisition through the private Store port in `Normal` mode.
|
||||||
pub(crate) use self::persistence::persist_raw_transaction_ingest_acquisition;
|
pub(crate) use self::persistence::persist_raw_transaction_ingest_acquisition;
|
||||||
|
/// Private latest-value publisher and checked counter owner shared by the Worker supervisor.
|
||||||
|
pub(crate) use self::snapshot::RawTransactionIngestSnapshotPublisher;
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
// file: crates/ksp-worker-raw-transaction-ingest-lib/src/runtime.rs
|
// file: crates/ksp-worker-raw-transaction-ingest-lib/src/runtime.rs
|
||||||
// version: 4
|
// version: 5
|
||||||
|
|
||||||
type PersistencePort = std::sync::Arc<dyn crate::RawTransactionIngestPersistencePort + 'static>;
|
type PersistencePort = std::sync::Arc<dyn crate::RawTransactionIngestPersistencePort + 'static>;
|
||||||
type PersistenceTasks = tokio::task::JoinSet<ksp_core_lib::Result<crate::RawTransactionIngestPersistenceOutcome>>;
|
type PersistenceTasks = tokio::task::JoinSet<ksp_core_lib::Result<crate::RawTransactionIngestPersistenceOutcome>>;
|
||||||
@@ -11,9 +11,9 @@ pub type RawTransactionIngestTerminalFuture<'a> =
|
|||||||
/// Cloneable external control handle for one continuous RAW transaction ingest Worker.
|
/// Cloneable external control handle for one continuous RAW transaction ingest Worker.
|
||||||
#[derive(Clone)]
|
#[derive(Clone)]
|
||||||
pub struct RawTransactionIngestHandle {
|
pub struct RawTransactionIngestHandle {
|
||||||
|
snapshots: crate::RawTransactionIngestSnapshotSource,
|
||||||
stop_sender: tokio::sync::watch::Sender<bool>,
|
stop_sender: tokio::sync::watch::Sender<bool>,
|
||||||
stop_token: ksp_worker_api::WorkerStopToken,
|
stop_token: ksp_worker_api::WorkerStopToken,
|
||||||
terminal_receiver: tokio::sync::watch::Receiver<ksp_worker_api::WorkerState>,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
impl crate::RawTransactionIngestHandle {
|
impl crate::RawTransactionIngestHandle {
|
||||||
@@ -26,22 +26,36 @@ impl crate::RawTransactionIngestHandle {
|
|||||||
return self.stop_sender.send(true).is_ok();
|
return self.stop_sender.send(true).is_ok();
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Waits until the private runtime task has published and closed one terminal lifecycle state.
|
/// Returns an independent latest-value source for concrete RAW transaction ingest Worker snapshots.
|
||||||
|
#[must_use]
|
||||||
|
pub fn snapshot_source(&self) -> crate::RawTransactionIngestSnapshotSource {
|
||||||
|
return self.snapshots.clone();
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Returns an independent source implementing the common [`ksp_worker_api::WorkerSnapshotSource`] projection.
|
||||||
|
#[must_use]
|
||||||
|
pub fn worker_snapshot_source(&self) -> crate::RawTransactionIngestSnapshotSource {
|
||||||
|
return self.snapshots.clone();
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Waits until the private runtime task has published one terminal lifecycle state after draining owned work.
|
||||||
#[must_use]
|
#[must_use]
|
||||||
pub fn wait_terminal(&self) -> crate::RawTransactionIngestTerminalFuture<'_> {
|
pub fn wait_terminal(&self) -> crate::RawTransactionIngestTerminalFuture<'_> {
|
||||||
let mut receiver = self.terminal_receiver.clone();
|
let source = self.snapshots.clone();
|
||||||
return std::boxed::Box::pin(async move {
|
return std::boxed::Box::pin(async move {
|
||||||
loop {
|
loop {
|
||||||
let current = *receiver.borrow();
|
let current = source.current();
|
||||||
if current.is_terminal() {
|
let state = current.worker_snapshot().state();
|
||||||
let changed = receiver.changed().await;
|
if state.is_terminal() {
|
||||||
if changed.is_err() {
|
return std::result::Result::Ok(state);
|
||||||
return std::result::Result::Ok(current);
|
|
||||||
}
|
|
||||||
return std::result::Result::Err(crate::runtime_error("terminal.changed_after_terminal"));
|
|
||||||
}
|
}
|
||||||
let changed = receiver.changed().await;
|
let observed = current.worker_snapshot().sequence();
|
||||||
if changed.is_err() {
|
let changed = source.wait_for_change(observed).await;
|
||||||
|
let state = changed.worker_snapshot().state();
|
||||||
|
if state.is_terminal() {
|
||||||
|
return std::result::Result::Ok(state);
|
||||||
|
}
|
||||||
|
if changed.worker_snapshot().sequence() == observed && source.is_closed() {
|
||||||
return std::result::Result::Err(crate::runtime_error("terminal.closed_before_terminal"));
|
return std::result::Result::Err(crate::runtime_error("terminal.closed_before_terminal"));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -51,11 +65,12 @@ impl crate::RawTransactionIngestHandle {
|
|||||||
|
|
||||||
impl std::fmt::Debug for crate::RawTransactionIngestHandle {
|
impl std::fmt::Debug for crate::RawTransactionIngestHandle {
|
||||||
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||||
let state = *self.terminal_receiver.borrow();
|
let snapshot = self.snapshots.current();
|
||||||
return formatter
|
return formatter
|
||||||
.debug_struct("RawTransactionIngestHandle")
|
.debug_struct("RawTransactionIngestHandle")
|
||||||
.field("stop_requested", &self.stop_token.is_stop_requested())
|
.field("stop_requested", &self.stop_token.is_stop_requested())
|
||||||
.field("state", &state)
|
.field("sequence", &snapshot.worker_snapshot().sequence())
|
||||||
|
.field("state", &snapshot.worker_snapshot().state())
|
||||||
.finish();
|
.finish();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -81,6 +96,21 @@ impl crate::RawTransactionIngestWorker {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn begin_stopping(
|
||||||
|
lifecycle: &mut ksp_worker_api::WorkerLifecycle,
|
||||||
|
snapshots: &mut crate::RawTransactionIngestSnapshotPublisher,
|
||||||
|
admission_queue_depth: usize,
|
||||||
|
in_flight_persistence: usize,
|
||||||
|
) -> std::option::Option<ksp_core_lib::ErrorCode> {
|
||||||
|
if lifecycle.mark_stopping().is_err() {
|
||||||
|
return std::option::Option::Some(crate::ERROR_CODE_RAW_TRANSACTION_INGEST_RUNTIME_INVALID);
|
||||||
|
}
|
||||||
|
if let std::result::Result::Err(error) = snapshots.publish_state(lifecycle.state(), admission_queue_depth, in_flight_persistence) {
|
||||||
|
return std::option::Option::Some(error.code());
|
||||||
|
}
|
||||||
|
return std::option::Option::None;
|
||||||
|
}
|
||||||
|
|
||||||
fn current_runtime_handle() -> ksp_core_lib::Result<tokio::runtime::Handle> {
|
fn current_runtime_handle() -> ksp_core_lib::Result<tokio::runtime::Handle> {
|
||||||
return match tokio::runtime::Handle::try_current() {
|
return match tokio::runtime::Handle::try_current() {
|
||||||
std::result::Result::Ok(value) => std::result::Result::Ok(value),
|
std::result::Result::Ok(value) => std::result::Result::Ok(value),
|
||||||
@@ -98,10 +128,15 @@ async fn drain_children(children: &mut tokio::task::JoinSet<()>) -> std::option:
|
|||||||
return fault;
|
return fault;
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn drain_persistence(persistence: &mut PersistenceTasks) -> std::option::Option<ksp_core_lib::ErrorCode> {
|
async fn drain_persistence(
|
||||||
|
lifecycle: &ksp_worker_api::WorkerLifecycle,
|
||||||
|
admission: &crate::RawTransactionAdmission,
|
||||||
|
persistence: &mut PersistenceTasks,
|
||||||
|
snapshots: &mut crate::RawTransactionIngestSnapshotPublisher,
|
||||||
|
) -> std::option::Option<ksp_core_lib::ErrorCode> {
|
||||||
let mut fault = std::option::Option::None;
|
let mut fault = std::option::Option::None;
|
||||||
while let std::option::Option::Some(joined) = persistence.join_next().await {
|
while let std::option::Option::Some(joined) = persistence.join_next().await {
|
||||||
let current = persistence_fault(joined);
|
let current = persistence_completion(joined, lifecycle.state(), admission.queue_depth(), persistence.len(), snapshots);
|
||||||
if fault.is_none() {
|
if fault.is_none() {
|
||||||
fault = current;
|
fault = current;
|
||||||
}
|
}
|
||||||
@@ -111,9 +146,11 @@ async fn drain_persistence(persistence: &mut PersistenceTasks) -> std::option::O
|
|||||||
|
|
||||||
async fn drain_admission_and_persistence(
|
async fn drain_admission_and_persistence(
|
||||||
settings: &crate::RawTransactionIngestSettings,
|
settings: &crate::RawTransactionIngestSettings,
|
||||||
|
lifecycle: &ksp_worker_api::WorkerLifecycle,
|
||||||
admission: &mut crate::RawTransactionAdmission,
|
admission: &mut crate::RawTransactionAdmission,
|
||||||
persistence: &mut PersistenceTasks,
|
persistence: &mut PersistenceTasks,
|
||||||
port: &std::option::Option<PersistencePort>,
|
port: &std::option::Option<PersistencePort>,
|
||||||
|
snapshots: &mut crate::RawTransactionIngestSnapshotPublisher,
|
||||||
) -> std::option::Option<ksp_core_lib::ErrorCode> {
|
) -> std::option::Option<ksp_core_lib::ErrorCode> {
|
||||||
let mut fault = std::option::Option::None;
|
let mut fault = std::option::Option::None;
|
||||||
admission.close();
|
admission.close();
|
||||||
@@ -121,7 +158,7 @@ async fn drain_admission_and_persistence(
|
|||||||
while persistence.len() >= settings.persistence_concurrency() {
|
while persistence.len() >= settings.persistence_concurrency() {
|
||||||
let joined = persistence.join_next().await;
|
let joined = persistence.join_next().await;
|
||||||
let current = match joined {
|
let current = match joined {
|
||||||
std::option::Option::Some(value) => persistence_fault(value),
|
std::option::Option::Some(value) => persistence_completion(value, lifecycle.state(), admission.queue_depth(), persistence.len(), snapshots),
|
||||||
std::option::Option::None => std::option::Option::Some(crate::ERROR_CODE_RAW_TRANSACTION_INGEST_RUNTIME_INVALID),
|
std::option::Option::None => std::option::Option::Some(crate::ERROR_CODE_RAW_TRANSACTION_INGEST_RUNTIME_INVALID),
|
||||||
};
|
};
|
||||||
if fault.is_none() {
|
if fault.is_none() {
|
||||||
@@ -131,19 +168,31 @@ async fn drain_admission_and_persistence(
|
|||||||
let received = admission.receive(settings.network()).await;
|
let received = admission.receive(settings.network()).await;
|
||||||
match received {
|
match received {
|
||||||
std::result::Result::Ok(std::option::Option::Some(acquisition)) => {
|
std::result::Result::Ok(std::option::Option::Some(acquisition)) => {
|
||||||
if !spawn_persistence(persistence, port, acquisition) && fault.is_none() {
|
let spawned = spawn_persistence(persistence, port, acquisition);
|
||||||
|
let published = snapshots.record_admission_success(lifecycle.state(), admission.queue_depth(), persistence.len());
|
||||||
|
if !spawned && fault.is_none() {
|
||||||
fault = std::option::Option::Some(crate::ERROR_CODE_RAW_TRANSACTION_INGEST_RUNTIME_INVALID);
|
fault = std::option::Option::Some(crate::ERROR_CODE_RAW_TRANSACTION_INGEST_RUNTIME_INVALID);
|
||||||
}
|
}
|
||||||
|
if let std::result::Result::Err(error) = published {
|
||||||
|
if fault.is_none() {
|
||||||
|
fault = std::option::Option::Some(error.code());
|
||||||
|
}
|
||||||
|
}
|
||||||
},
|
},
|
||||||
std::result::Result::Ok(std::option::Option::None) => break,
|
std::result::Result::Ok(std::option::Option::None) => break,
|
||||||
std::result::Result::Err(_) => {
|
std::result::Result::Err(_) => {
|
||||||
if fault.is_none() {
|
let published = snapshots.record_admission_failure(lifecycle.state(), admission.queue_depth(), persistence.len());
|
||||||
|
if let std::result::Result::Err(error) = published {
|
||||||
|
if fault.is_none() {
|
||||||
|
fault = std::option::Option::Some(error.code());
|
||||||
|
}
|
||||||
|
} else if fault.is_none() {
|
||||||
fault = std::option::Option::Some(crate::ERROR_CODE_RAW_TRANSACTION_INGEST_RUNTIME_INVALID);
|
fault = std::option::Option::Some(crate::ERROR_CODE_RAW_TRANSACTION_INGEST_RUNTIME_INVALID);
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
let persistence_fault = drain_persistence(persistence).await;
|
let persistence_fault = drain_persistence(lifecycle, admission, persistence, snapshots).await;
|
||||||
if fault.is_none() {
|
if fault.is_none() {
|
||||||
fault = persistence_fault;
|
fault = persistence_fault;
|
||||||
}
|
}
|
||||||
@@ -152,28 +201,27 @@ async fn drain_admission_and_persistence(
|
|||||||
|
|
||||||
fn finish_faulted(
|
fn finish_faulted(
|
||||||
lifecycle: &mut ksp_worker_api::WorkerLifecycle,
|
lifecycle: &mut ksp_worker_api::WorkerLifecycle,
|
||||||
sender: &tokio::sync::watch::Sender<ksp_worker_api::WorkerState>,
|
snapshots: &mut crate::RawTransactionIngestSnapshotPublisher,
|
||||||
code: ksp_core_lib::ErrorCode,
|
code: ksp_core_lib::ErrorCode,
|
||||||
) {
|
) {
|
||||||
if lifecycle.fault(code).is_err() {
|
if lifecycle.fault(code).is_err() {
|
||||||
sender.send_replace(ksp_worker_api::WorkerState::Faulted(crate::ERROR_CODE_RAW_TRANSACTION_INGEST_RUNTIME_INVALID));
|
snapshots.force_terminal(ksp_worker_api::WorkerState::Faulted(crate::ERROR_CODE_RAW_TRANSACTION_INGEST_RUNTIME_INVALID));
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
sender.send_replace(lifecycle.state());
|
if snapshots.publish_state(lifecycle.state(), 0, 0).is_err() {
|
||||||
|
snapshots.force_terminal(ksp_worker_api::WorkerState::Faulted(crate::ERROR_CODE_RAW_TRANSACTION_INGEST_COUNTER_EXHAUSTED));
|
||||||
|
}
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
fn finish_stopped(lifecycle: &mut ksp_worker_api::WorkerLifecycle, sender: &tokio::sync::watch::Sender<ksp_worker_api::WorkerState>) {
|
fn finish_stopped(lifecycle: &mut ksp_worker_api::WorkerLifecycle, snapshots: &mut crate::RawTransactionIngestSnapshotPublisher) {
|
||||||
if lifecycle.mark_stopping().is_err() {
|
|
||||||
sender.send_replace(ksp_worker_api::WorkerState::Faulted(crate::ERROR_CODE_RAW_TRANSACTION_INGEST_RUNTIME_INVALID));
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
sender.send_replace(lifecycle.state());
|
|
||||||
if lifecycle.mark_stopped().is_err() {
|
if lifecycle.mark_stopped().is_err() {
|
||||||
sender.send_replace(ksp_worker_api::WorkerState::Faulted(crate::ERROR_CODE_RAW_TRANSACTION_INGEST_RUNTIME_INVALID));
|
snapshots.force_terminal(ksp_worker_api::WorkerState::Faulted(crate::ERROR_CODE_RAW_TRANSACTION_INGEST_RUNTIME_INVALID));
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
sender.send_replace(lifecycle.state());
|
if snapshots.publish_state(lifecycle.state(), 0, 0).is_err() {
|
||||||
|
snapshots.force_terminal(ksp_worker_api::WorkerState::Faulted(crate::ERROR_CODE_RAW_TRANSACTION_INGEST_COUNTER_EXHAUSTED));
|
||||||
|
}
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -187,16 +235,28 @@ fn merge_fault(
|
|||||||
return next;
|
return next;
|
||||||
}
|
}
|
||||||
|
|
||||||
fn persistence_fault(
|
fn persistence_completion(
|
||||||
joined: std::result::Result<ksp_core_lib::Result<crate::RawTransactionIngestPersistenceOutcome>, tokio::task::JoinError>,
|
joined: std::result::Result<ksp_core_lib::Result<crate::RawTransactionIngestPersistenceOutcome>, tokio::task::JoinError>,
|
||||||
|
state: ksp_worker_api::WorkerState,
|
||||||
|
admission_queue_depth: usize,
|
||||||
|
in_flight_persistence: usize,
|
||||||
|
snapshots: &mut crate::RawTransactionIngestSnapshotPublisher,
|
||||||
) -> std::option::Option<ksp_core_lib::ErrorCode> {
|
) -> std::option::Option<ksp_core_lib::ErrorCode> {
|
||||||
return match joined {
|
return match joined {
|
||||||
std::result::Result::Ok(std::result::Result::Ok(outcome)) => {
|
std::result::Result::Ok(std::result::Result::Ok(outcome)) => {
|
||||||
let _entity = outcome.entity();
|
match snapshots.record_persistence_success(state, admission_queue_depth, in_flight_persistence, outcome) {
|
||||||
let _observation = outcome.observation();
|
std::result::Result::Ok(()) => std::option::Option::None,
|
||||||
std::option::Option::None
|
std::result::Result::Err(error) => std::option::Option::Some(error.code()),
|
||||||
|
}
|
||||||
|
},
|
||||||
|
std::result::Result::Ok(std::result::Result::Err(error)) => {
|
||||||
|
let code = error.code();
|
||||||
|
let published = snapshots.record_persistence_fault(state, admission_queue_depth, in_flight_persistence, code);
|
||||||
|
match published {
|
||||||
|
std::result::Result::Ok(()) => std::option::Option::Some(code),
|
||||||
|
std::result::Result::Err(snapshot_error) => std::option::Option::Some(snapshot_error.code()),
|
||||||
|
}
|
||||||
},
|
},
|
||||||
std::result::Result::Ok(std::result::Result::Err(error)) => std::option::Option::Some(error.code()),
|
|
||||||
std::result::Result::Err(_) => std::option::Option::Some(crate::ERROR_CODE_RAW_TRANSACTION_INGEST_RUNTIME_INVALID),
|
std::result::Result::Err(_) => std::option::Option::Some(crate::ERROR_CODE_RAW_TRANSACTION_INGEST_RUNTIME_INVALID),
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
@@ -206,7 +266,7 @@ async fn run_supervisor<Spawner>(
|
|||||||
mut lifecycle: ksp_worker_api::WorkerLifecycle,
|
mut lifecycle: ksp_worker_api::WorkerLifecycle,
|
||||||
port: std::option::Option<PersistencePort>,
|
port: std::option::Option<PersistencePort>,
|
||||||
mut stop_receiver: tokio::sync::watch::Receiver<bool>,
|
mut stop_receiver: tokio::sync::watch::Receiver<bool>,
|
||||||
terminal_sender: tokio::sync::watch::Sender<ksp_worker_api::WorkerState>,
|
mut snapshots: crate::RawTransactionIngestSnapshotPublisher,
|
||||||
source_spawner: Spawner,
|
source_spawner: Spawner,
|
||||||
) where
|
) where
|
||||||
Spawner: FnOnce(&mut tokio::task::JoinSet<()>, tokio::sync::watch::Receiver<bool>, tokio::sync::mpsc::Sender<crate::RawTransactionIngress>)
|
Spawner: FnOnce(&mut tokio::task::JoinSet<()>, tokio::sync::watch::Receiver<bool>, tokio::sync::mpsc::Sender<crate::RawTransactionIngress>)
|
||||||
@@ -214,30 +274,50 @@ async fn run_supervisor<Spawner>(
|
|||||||
+ 'static,
|
+ 'static,
|
||||||
{
|
{
|
||||||
if *stop_receiver.borrow() {
|
if *stop_receiver.borrow() {
|
||||||
finish_stopped(&mut lifecycle, &terminal_sender);
|
let stopping_fault = begin_stopping(&mut lifecycle, &mut snapshots, 0, 0);
|
||||||
|
match stopping_fault {
|
||||||
|
std::option::Option::Some(code) => finish_faulted(&mut lifecycle, &mut snapshots, code),
|
||||||
|
std::option::Option::None => finish_stopped(&mut lifecycle, &mut snapshots),
|
||||||
|
}
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
if lifecycle.mark_running().is_err() {
|
if lifecycle.mark_running().is_err() {
|
||||||
terminal_sender.send_replace(ksp_worker_api::WorkerState::Faulted(crate::ERROR_CODE_RAW_TRANSACTION_INGEST_RUNTIME_INVALID));
|
snapshots.force_terminal(ksp_worker_api::WorkerState::Faulted(crate::ERROR_CODE_RAW_TRANSACTION_INGEST_RUNTIME_INVALID));
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if let std::result::Result::Err(error) = snapshots.publish_state(lifecycle.state(), 0, 0) {
|
||||||
|
finish_faulted(&mut lifecycle, &mut snapshots, error.code());
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
terminal_sender.send_replace(lifecycle.state());
|
|
||||||
let mut children = tokio::task::JoinSet::new();
|
let mut children = tokio::task::JoinSet::new();
|
||||||
let mut persistence = PersistenceTasks::new();
|
let mut persistence = PersistenceTasks::new();
|
||||||
let (source_stop_sender, source_stop_receiver) = tokio::sync::watch::channel(false);
|
let (source_stop_sender, source_stop_receiver) = tokio::sync::watch::channel(false);
|
||||||
let (mut admission, admission_sender) = crate::RawTransactionAdmission::new(settings.admission_queue_capacity());
|
let (mut admission, admission_sender) = crate::RawTransactionAdmission::new(settings.admission_queue_capacity());
|
||||||
source_spawner(&mut children, source_stop_receiver, admission_sender);
|
source_spawner(&mut children, source_stop_receiver, admission_sender);
|
||||||
let mut fault = supervise_until_stop(&settings, &source_stop_sender, &mut stop_receiver, &mut children, &mut admission, &mut persistence, &port).await;
|
let mut fault = supervise_until_stop(
|
||||||
|
&settings,
|
||||||
|
&lifecycle,
|
||||||
|
&source_stop_sender,
|
||||||
|
&mut stop_receiver,
|
||||||
|
&mut children,
|
||||||
|
&mut admission,
|
||||||
|
&mut persistence,
|
||||||
|
&port,
|
||||||
|
&mut snapshots,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
if fault.is_some() {
|
if fault.is_some() {
|
||||||
source_stop_sender.send_replace(true);
|
source_stop_sender.send_replace(true);
|
||||||
}
|
}
|
||||||
let drain_fault = drain_admission_and_persistence(&settings, &mut admission, &mut persistence, &port).await;
|
let stopping_fault = begin_stopping(&mut lifecycle, &mut snapshots, admission.queue_depth(), persistence.len());
|
||||||
|
fault = merge_fault(fault, stopping_fault);
|
||||||
|
let drain_fault = drain_admission_and_persistence(&settings, &lifecycle, &mut admission, &mut persistence, &port, &mut snapshots).await;
|
||||||
fault = merge_fault(fault, drain_fault);
|
fault = merge_fault(fault, drain_fault);
|
||||||
let child_fault = drain_children(&mut children).await;
|
let child_fault = drain_children(&mut children).await;
|
||||||
fault = merge_fault(fault, child_fault);
|
fault = merge_fault(fault, child_fault);
|
||||||
match fault {
|
match fault {
|
||||||
std::option::Option::Some(code) => finish_faulted(&mut lifecycle, &terminal_sender, code),
|
std::option::Option::Some(code) => finish_faulted(&mut lifecycle, &mut snapshots, code),
|
||||||
std::option::Option::None => finish_stopped(&mut lifecycle, &terminal_sender),
|
std::option::Option::None => finish_stopped(&mut lifecycle, &mut snapshots),
|
||||||
}
|
}
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
@@ -286,9 +366,9 @@ where
|
|||||||
}
|
}
|
||||||
let stop_token = ksp_worker_api::WorkerStopToken::new();
|
let stop_token = ksp_worker_api::WorkerStopToken::new();
|
||||||
let (stop_sender, stop_receiver) = tokio::sync::watch::channel(false);
|
let (stop_sender, stop_receiver) = tokio::sync::watch::channel(false);
|
||||||
let (terminal_sender, terminal_receiver) = tokio::sync::watch::channel(lifecycle.state());
|
let (snapshots, snapshot_source) = crate::RawTransactionIngestSnapshotPublisher::new(&settings, &lifecycle);
|
||||||
let handle = crate::RawTransactionIngestHandle { stop_sender, stop_token, terminal_receiver };
|
let handle = crate::RawTransactionIngestHandle { snapshots: snapshot_source, stop_sender, stop_token };
|
||||||
std::mem::drop(runtime.spawn(run_supervisor(settings, lifecycle, port, stop_receiver, terminal_sender, source_spawner)));
|
std::mem::drop(runtime.spawn(run_supervisor(settings, lifecycle, port, stop_receiver, snapshots, source_spawner)));
|
||||||
return std::result::Result::Ok(handle);
|
return std::result::Result::Ok(handle);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -315,12 +395,14 @@ where
|
|||||||
|
|
||||||
async fn supervise_until_stop(
|
async fn supervise_until_stop(
|
||||||
settings: &crate::RawTransactionIngestSettings,
|
settings: &crate::RawTransactionIngestSettings,
|
||||||
|
lifecycle: &ksp_worker_api::WorkerLifecycle,
|
||||||
source_stop_sender: &tokio::sync::watch::Sender<bool>,
|
source_stop_sender: &tokio::sync::watch::Sender<bool>,
|
||||||
stop_receiver: &mut tokio::sync::watch::Receiver<bool>,
|
stop_receiver: &mut tokio::sync::watch::Receiver<bool>,
|
||||||
children: &mut tokio::task::JoinSet<()>,
|
children: &mut tokio::task::JoinSet<()>,
|
||||||
admission: &mut crate::RawTransactionAdmission,
|
admission: &mut crate::RawTransactionAdmission,
|
||||||
persistence: &mut PersistenceTasks,
|
persistence: &mut PersistenceTasks,
|
||||||
port: &std::option::Option<PersistencePort>,
|
port: &std::option::Option<PersistencePort>,
|
||||||
|
snapshots: &mut crate::RawTransactionIngestSnapshotPublisher,
|
||||||
) -> std::option::Option<ksp_core_lib::ErrorCode> {
|
) -> std::option::Option<ksp_core_lib::ErrorCode> {
|
||||||
let mut admission_open = true;
|
let mut admission_open = true;
|
||||||
loop {
|
loop {
|
||||||
@@ -352,7 +434,7 @@ async fn supervise_until_stop(
|
|||||||
}
|
}
|
||||||
joined = persistence.join_next(), if !persistence.is_empty() => {
|
joined = persistence.join_next(), if !persistence.is_empty() => {
|
||||||
if let std::option::Option::Some(value) = joined {
|
if let std::option::Option::Some(value) = joined {
|
||||||
let fault = persistence_fault(value);
|
let fault = persistence_completion(value, lifecycle.state(), admission.queue_depth(), persistence.len(), snapshots);
|
||||||
if fault.is_some() {
|
if fault.is_some() {
|
||||||
source_stop_sender.send_replace(true);
|
source_stop_sender.send_replace(true);
|
||||||
return fault;
|
return fault;
|
||||||
@@ -362,15 +444,25 @@ async fn supervise_until_stop(
|
|||||||
received = admission.receive(settings.network()), if admission_open && persistence.len() < settings.persistence_concurrency() => {
|
received = admission.receive(settings.network()), if admission_open && persistence.len() < settings.persistence_concurrency() => {
|
||||||
match received {
|
match received {
|
||||||
std::result::Result::Ok(std::option::Option::Some(acquisition)) => {
|
std::result::Result::Ok(std::option::Option::Some(acquisition)) => {
|
||||||
if !spawn_persistence(persistence, port, acquisition) {
|
let spawned = spawn_persistence(persistence, port, acquisition);
|
||||||
|
let published = snapshots.record_admission_success(lifecycle.state(), admission.queue_depth(), persistence.len());
|
||||||
|
if !spawned {
|
||||||
source_stop_sender.send_replace(true);
|
source_stop_sender.send_replace(true);
|
||||||
return std::option::Option::Some(crate::ERROR_CODE_RAW_TRANSACTION_INGEST_RUNTIME_INVALID);
|
return std::option::Option::Some(crate::ERROR_CODE_RAW_TRANSACTION_INGEST_RUNTIME_INVALID);
|
||||||
}
|
}
|
||||||
|
if let std::result::Result::Err(error) = published {
|
||||||
|
source_stop_sender.send_replace(true);
|
||||||
|
return std::option::Option::Some(error.code());
|
||||||
|
}
|
||||||
},
|
},
|
||||||
std::result::Result::Ok(std::option::Option::None) => admission_open = false,
|
std::result::Result::Ok(std::option::Option::None) => admission_open = false,
|
||||||
std::result::Result::Err(_) => {
|
std::result::Result::Err(_) => {
|
||||||
|
let published = snapshots.record_admission_failure(lifecycle.state(), admission.queue_depth(), persistence.len());
|
||||||
source_stop_sender.send_replace(true);
|
source_stop_sender.send_replace(true);
|
||||||
return std::option::Option::Some(crate::ERROR_CODE_RAW_TRANSACTION_INGEST_RUNTIME_INVALID);
|
return match published {
|
||||||
|
std::result::Result::Ok(()) => std::option::Option::Some(crate::ERROR_CODE_RAW_TRANSACTION_INGEST_RUNTIME_INVALID),
|
||||||
|
std::result::Result::Err(error) => std::option::Option::Some(error.code()),
|
||||||
|
};
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
472
crates/ksp-worker-raw-transaction-ingest-lib/src/snapshot.rs
Normal file
472
crates/ksp-worker-raw-transaction-ingest-lib/src/snapshot.rs
Normal file
@@ -0,0 +1,472 @@
|
|||||||
|
// file: crates/ksp-worker-raw-transaction-ingest-lib/src/snapshot.rs
|
||||||
|
// version: 1
|
||||||
|
|
||||||
|
/// Runtime-neutral boxed future resolving to one newer concrete RAW transaction ingest Worker snapshot.
|
||||||
|
pub type RawTransactionIngestSnapshotFuture<'a> =
|
||||||
|
std::pin::Pin<std::boxed::Box<dyn std::future::Future<Output = crate::RawTransactionIngestSnapshot> + std::marker::Send + 'a>>;
|
||||||
|
|
||||||
|
/// Complete safe latest-value snapshot of one continuous RAW transaction ingest Worker.
|
||||||
|
#[derive(Clone, Eq, PartialEq)]
|
||||||
|
pub struct RawTransactionIngestSnapshot {
|
||||||
|
worker: ksp_worker_api::WorkerSnapshot,
|
||||||
|
admission_queue_capacity: usize,
|
||||||
|
admission_queue_depth: usize,
|
||||||
|
persistence_concurrency: usize,
|
||||||
|
in_flight_persistence: usize,
|
||||||
|
admitted_total: u64,
|
||||||
|
canonicalized_total: u64,
|
||||||
|
persisted_total: u64,
|
||||||
|
entity_inserted_total: u64,
|
||||||
|
entity_already_present_total: u64,
|
||||||
|
entity_skipped_purged_total: u64,
|
||||||
|
observation_inserted_total: u64,
|
||||||
|
observation_already_present_total: u64,
|
||||||
|
content_conflict_total: u64,
|
||||||
|
store_failure_total: u64,
|
||||||
|
source_failure_total: u64,
|
||||||
|
backpressure_wait_total: u64,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl crate::RawTransactionIngestSnapshot {
|
||||||
|
/// Returns the common Worker projection carried by this concrete snapshot.
|
||||||
|
#[must_use]
|
||||||
|
pub const fn worker_snapshot(&self) -> &ksp_worker_api::WorkerSnapshot {
|
||||||
|
return &self.worker;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Returns the configured bounded admission queue capacity.
|
||||||
|
#[must_use]
|
||||||
|
pub const fn admission_queue_capacity(&self) -> usize {
|
||||||
|
return self.admission_queue_capacity;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Returns the latest observed number of queued ingress entries awaiting supervisor admission.
|
||||||
|
#[must_use]
|
||||||
|
pub const fn admission_queue_depth(&self) -> usize {
|
||||||
|
return self.admission_queue_depth;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Returns the configured maximum number of concurrent Store persistence operations.
|
||||||
|
#[must_use]
|
||||||
|
pub const fn persistence_concurrency(&self) -> usize {
|
||||||
|
return self.persistence_concurrency;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Returns the latest observed number of in-flight Store persistence operations.
|
||||||
|
#[must_use]
|
||||||
|
pub const fn in_flight_persistence(&self) -> usize {
|
||||||
|
return self.in_flight_persistence;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Returns the number of ingress entries removed from the bounded admission queue.
|
||||||
|
#[must_use]
|
||||||
|
pub const fn admitted_total(&self) -> u64 {
|
||||||
|
return self.admitted_total;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Returns the number of admitted ingress entries successfully canonicalized through the Common RAW contract.
|
||||||
|
#[must_use]
|
||||||
|
pub const fn canonicalized_total(&self) -> u64 {
|
||||||
|
return self.canonicalized_total;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Returns the number of successful atomic Store persistence outcomes.
|
||||||
|
#[must_use]
|
||||||
|
pub const fn persisted_total(&self) -> u64 {
|
||||||
|
return self.persisted_total;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Returns the number of newly inserted canonical RAW transaction entities.
|
||||||
|
#[must_use]
|
||||||
|
pub const fn entity_inserted_total(&self) -> u64 {
|
||||||
|
return self.entity_inserted_total;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Returns the number of identical canonical RAW transaction entities already durable.
|
||||||
|
#[must_use]
|
||||||
|
pub const fn entity_already_present_total(&self) -> u64 {
|
||||||
|
return self.entity_already_present_total;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Returns the number of durable purge tombstones respected by normal persistence.
|
||||||
|
#[must_use]
|
||||||
|
pub const fn entity_skipped_purged_total(&self) -> u64 {
|
||||||
|
return self.entity_skipped_purged_total;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Returns the number of newly inserted deterministic acquisition observations.
|
||||||
|
#[must_use]
|
||||||
|
pub const fn observation_inserted_total(&self) -> u64 {
|
||||||
|
return self.observation_inserted_total;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Returns the number of deterministic acquisition observations already durable.
|
||||||
|
#[must_use]
|
||||||
|
pub const fn observation_already_present_total(&self) -> u64 {
|
||||||
|
return self.observation_already_present_total;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Returns the number of durable Store content conflicts observed by this run.
|
||||||
|
#[must_use]
|
||||||
|
pub const fn content_conflict_total(&self) -> u64 {
|
||||||
|
return self.content_conflict_total;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Returns the number of non-conflict Store persistence failures observed by this run.
|
||||||
|
#[must_use]
|
||||||
|
pub const fn store_failure_total(&self) -> u64 {
|
||||||
|
return self.store_failure_total;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Returns the number of source-task failures observed by this run.
|
||||||
|
#[must_use]
|
||||||
|
pub const fn source_failure_total(&self) -> u64 {
|
||||||
|
return self.source_failure_total;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Returns the number of source-side admission waits explicitly classified as backpressure waits.
|
||||||
|
#[must_use]
|
||||||
|
pub const fn backpressure_wait_total(&self) -> u64 {
|
||||||
|
return self.backpressure_wait_total;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl std::fmt::Debug for crate::RawTransactionIngestSnapshot {
|
||||||
|
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||||
|
return formatter
|
||||||
|
.debug_struct("RawTransactionIngestSnapshot")
|
||||||
|
.field("worker", &self.worker)
|
||||||
|
.field("admission_queue_capacity", &self.admission_queue_capacity)
|
||||||
|
.field("admission_queue_depth", &self.admission_queue_depth)
|
||||||
|
.field("persistence_concurrency", &self.persistence_concurrency)
|
||||||
|
.field("in_flight_persistence", &self.in_flight_persistence)
|
||||||
|
.field("admitted_total", &self.admitted_total)
|
||||||
|
.field("canonicalized_total", &self.canonicalized_total)
|
||||||
|
.field("persisted_total", &self.persisted_total)
|
||||||
|
.field("entity_inserted_total", &self.entity_inserted_total)
|
||||||
|
.field("entity_already_present_total", &self.entity_already_present_total)
|
||||||
|
.field("entity_skipped_purged_total", &self.entity_skipped_purged_total)
|
||||||
|
.field("observation_inserted_total", &self.observation_inserted_total)
|
||||||
|
.field("observation_already_present_total", &self.observation_already_present_total)
|
||||||
|
.field("content_conflict_total", &self.content_conflict_total)
|
||||||
|
.field("store_failure_total", &self.store_failure_total)
|
||||||
|
.field("source_failure_total", &self.source_failure_total)
|
||||||
|
.field("backpressure_wait_total", &self.backpressure_wait_total)
|
||||||
|
.finish();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Cloneable latest-value source exposing concrete and common Worker snapshots from one shared watch state.
|
||||||
|
#[derive(Clone)]
|
||||||
|
pub struct RawTransactionIngestSnapshotSource {
|
||||||
|
receiver: tokio::sync::watch::Receiver<crate::RawTransactionIngestSnapshot>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl crate::RawTransactionIngestSnapshotSource {
|
||||||
|
/// Returns the complete current concrete snapshot without replaying intermediate updates.
|
||||||
|
#[must_use]
|
||||||
|
pub fn current(&self) -> crate::RawTransactionIngestSnapshot {
|
||||||
|
return self.receiver.borrow().clone();
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Reports whether the private publisher has closed after the runtime task returned.
|
||||||
|
#[must_use]
|
||||||
|
pub(crate) fn is_closed(&self) -> bool {
|
||||||
|
return self.receiver.has_changed().is_err();
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Waits for one concrete snapshot newer than `observed`, coalescing intermediate updates to the latest value.
|
||||||
|
#[must_use]
|
||||||
|
pub fn wait_for_change(&self, observed: ksp_worker_api::WorkerSnapshotSequence) -> crate::RawTransactionIngestSnapshotFuture<'_> {
|
||||||
|
let mut receiver = self.receiver.clone();
|
||||||
|
return std::boxed::Box::pin(async move {
|
||||||
|
loop {
|
||||||
|
let current = receiver.borrow_and_update().clone();
|
||||||
|
if current.worker_snapshot().sequence().is_after(observed) {
|
||||||
|
return current;
|
||||||
|
}
|
||||||
|
let changed = receiver.changed().await;
|
||||||
|
if changed.is_err() {
|
||||||
|
return receiver.borrow_and_update().clone();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl ksp_worker_api::WorkerSnapshotSource for crate::RawTransactionIngestSnapshotSource {
|
||||||
|
fn current(&self) -> ksp_worker_api::WorkerSnapshot {
|
||||||
|
return self.receiver.borrow().worker_snapshot().clone();
|
||||||
|
}
|
||||||
|
|
||||||
|
fn wait_for_change(&self, observed: ksp_worker_api::WorkerSnapshotSequence) -> ksp_worker_api::WorkerSnapshotFuture<'_> {
|
||||||
|
let mut receiver = self.receiver.clone();
|
||||||
|
return std::boxed::Box::pin(async move {
|
||||||
|
loop {
|
||||||
|
let current = receiver.borrow_and_update().worker_snapshot().clone();
|
||||||
|
if current.sequence().is_after(observed) {
|
||||||
|
return current;
|
||||||
|
}
|
||||||
|
let changed = receiver.changed().await;
|
||||||
|
if changed.is_err() {
|
||||||
|
return receiver.borrow_and_update().worker_snapshot().clone();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl std::fmt::Debug for crate::RawTransactionIngestSnapshotSource {
|
||||||
|
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||||
|
let current = self.receiver.borrow();
|
||||||
|
return formatter
|
||||||
|
.debug_struct("RawTransactionIngestSnapshotSource")
|
||||||
|
.field("sequence", ¤t.worker_snapshot().sequence())
|
||||||
|
.field("state", ¤t.worker_snapshot().state())
|
||||||
|
.finish();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Private latest-value publisher and checked counter owner shared by the Worker supervisor.
|
||||||
|
pub(crate) struct RawTransactionIngestSnapshotPublisher {
|
||||||
|
sender: tokio::sync::watch::Sender<crate::RawTransactionIngestSnapshot>,
|
||||||
|
snapshot: crate::RawTransactionIngestSnapshot,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl crate::RawTransactionIngestSnapshotPublisher {
|
||||||
|
/// Creates the initial `Starting` snapshot stream for one validated Worker run.
|
||||||
|
pub(crate) fn new(
|
||||||
|
settings: &crate::RawTransactionIngestSettings,
|
||||||
|
lifecycle: &ksp_worker_api::WorkerLifecycle,
|
||||||
|
) -> (crate::RawTransactionIngestSnapshotPublisher, crate::RawTransactionIngestSnapshotSource) {
|
||||||
|
let worker = ksp_worker_api::WorkerSnapshot::new(
|
||||||
|
lifecycle.id().clone(),
|
||||||
|
lifecycle.kind().clone(),
|
||||||
|
ksp_worker_api::WorkerSnapshotSequence::initial(),
|
||||||
|
lifecycle.state(),
|
||||||
|
ksp_worker_api::WorkerHealth::Unknown,
|
||||||
|
ksp_worker_api::WorkerActivity::Unknown,
|
||||||
|
);
|
||||||
|
let snapshot = crate::RawTransactionIngestSnapshot {
|
||||||
|
worker,
|
||||||
|
admission_queue_capacity: settings.admission_queue_capacity(),
|
||||||
|
admission_queue_depth: 0,
|
||||||
|
persistence_concurrency: settings.persistence_concurrency(),
|
||||||
|
in_flight_persistence: 0,
|
||||||
|
admitted_total: 0,
|
||||||
|
canonicalized_total: 0,
|
||||||
|
persisted_total: 0,
|
||||||
|
entity_inserted_total: 0,
|
||||||
|
entity_already_present_total: 0,
|
||||||
|
entity_skipped_purged_total: 0,
|
||||||
|
observation_inserted_total: 0,
|
||||||
|
observation_already_present_total: 0,
|
||||||
|
content_conflict_total: 0,
|
||||||
|
store_failure_total: 0,
|
||||||
|
source_failure_total: 0,
|
||||||
|
backpressure_wait_total: 0,
|
||||||
|
};
|
||||||
|
let (sender, receiver) = tokio::sync::watch::channel(snapshot.clone());
|
||||||
|
return (Self { sender, snapshot }, crate::RawTransactionIngestSnapshotSource { receiver });
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Forces one terminal latest value without advancing the sequence, used only when sequence publication itself is exhausted or invalid.
|
||||||
|
pub(crate) fn force_terminal(&mut self, state: ksp_worker_api::WorkerState) {
|
||||||
|
let worker = ksp_worker_api::WorkerSnapshot::new(
|
||||||
|
self.snapshot.worker.id().clone(),
|
||||||
|
self.snapshot.worker.kind().clone(),
|
||||||
|
self.snapshot.worker.sequence(),
|
||||||
|
state,
|
||||||
|
health_for_state(state, self.snapshot.worker.health()),
|
||||||
|
ksp_worker_api::WorkerActivity::Idle,
|
||||||
|
);
|
||||||
|
self.snapshot.worker = worker;
|
||||||
|
self.snapshot.admission_queue_depth = 0;
|
||||||
|
self.snapshot.in_flight_persistence = 0;
|
||||||
|
self.sender.send_replace(self.snapshot.clone());
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Publishes one lifecycle/depth refresh with a strictly newer common sequence.
|
||||||
|
pub(crate) fn publish_state(
|
||||||
|
&mut self,
|
||||||
|
state: ksp_worker_api::WorkerState,
|
||||||
|
admission_queue_depth: usize,
|
||||||
|
in_flight_persistence: usize,
|
||||||
|
) -> ksp_core_lib::Result<()> {
|
||||||
|
return self.publish(state, admission_queue_depth, in_flight_persistence);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Records one dequeued ingress that failed canonicalization and publishes the resulting latest value.
|
||||||
|
pub(crate) fn record_admission_failure(
|
||||||
|
&mut self,
|
||||||
|
state: ksp_worker_api::WorkerState,
|
||||||
|
admission_queue_depth: usize,
|
||||||
|
in_flight_persistence: usize,
|
||||||
|
) -> ksp_core_lib::Result<()> {
|
||||||
|
let admitted_total = match checked_counter(self.snapshot.admitted_total, "admitted_total") {
|
||||||
|
std::result::Result::Ok(value) => value,
|
||||||
|
std::result::Result::Err(error) => return std::result::Result::Err(error),
|
||||||
|
};
|
||||||
|
self.snapshot.admitted_total = admitted_total;
|
||||||
|
return self.publish(state, admission_queue_depth, in_flight_persistence);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Records one successfully canonicalized admitted ingress and publishes the resulting latest value.
|
||||||
|
pub(crate) fn record_admission_success(
|
||||||
|
&mut self,
|
||||||
|
state: ksp_worker_api::WorkerState,
|
||||||
|
admission_queue_depth: usize,
|
||||||
|
in_flight_persistence: usize,
|
||||||
|
) -> ksp_core_lib::Result<()> {
|
||||||
|
let admitted_total = match checked_counter(self.snapshot.admitted_total, "admitted_total") {
|
||||||
|
std::result::Result::Ok(value) => value,
|
||||||
|
std::result::Result::Err(error) => return std::result::Result::Err(error),
|
||||||
|
};
|
||||||
|
let canonicalized_total = match checked_counter(self.snapshot.canonicalized_total, "canonicalized_total") {
|
||||||
|
std::result::Result::Ok(value) => value,
|
||||||
|
std::result::Result::Err(error) => return std::result::Result::Err(error),
|
||||||
|
};
|
||||||
|
self.snapshot.admitted_total = admitted_total;
|
||||||
|
self.snapshot.canonicalized_total = canonicalized_total;
|
||||||
|
return self.publish(state, admission_queue_depth, in_flight_persistence);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Records one successful Store persistence outcome and publishes the resulting latest value.
|
||||||
|
pub(crate) fn record_persistence_success(
|
||||||
|
&mut self,
|
||||||
|
state: ksp_worker_api::WorkerState,
|
||||||
|
admission_queue_depth: usize,
|
||||||
|
in_flight_persistence: usize,
|
||||||
|
outcome: crate::RawTransactionIngestPersistenceOutcome,
|
||||||
|
) -> ksp_core_lib::Result<()> {
|
||||||
|
let persisted_total = match checked_counter(self.snapshot.persisted_total, "persisted_total") {
|
||||||
|
std::result::Result::Ok(value) => value,
|
||||||
|
std::result::Result::Err(error) => return std::result::Result::Err(error),
|
||||||
|
};
|
||||||
|
let mut entity_inserted_total = self.snapshot.entity_inserted_total;
|
||||||
|
let mut entity_already_present_total = self.snapshot.entity_already_present_total;
|
||||||
|
let mut entity_skipped_purged_total = self.snapshot.entity_skipped_purged_total;
|
||||||
|
let mut observation_inserted_total = self.snapshot.observation_inserted_total;
|
||||||
|
let mut observation_already_present_total = self.snapshot.observation_already_present_total;
|
||||||
|
match outcome.entity() {
|
||||||
|
crate::RawTransactionIngestEntityPersistence::Inserted => {
|
||||||
|
entity_inserted_total = match checked_counter(entity_inserted_total, "entity_inserted_total") {
|
||||||
|
std::result::Result::Ok(value) => value,
|
||||||
|
std::result::Result::Err(error) => return std::result::Result::Err(error),
|
||||||
|
};
|
||||||
|
},
|
||||||
|
crate::RawTransactionIngestEntityPersistence::AlreadyPresent => {
|
||||||
|
entity_already_present_total = match checked_counter(entity_already_present_total, "entity_already_present_total") {
|
||||||
|
std::result::Result::Ok(value) => value,
|
||||||
|
std::result::Result::Err(error) => return std::result::Result::Err(error),
|
||||||
|
};
|
||||||
|
},
|
||||||
|
crate::RawTransactionIngestEntityPersistence::SkippedPurged => {
|
||||||
|
entity_skipped_purged_total = match checked_counter(entity_skipped_purged_total, "entity_skipped_purged_total") {
|
||||||
|
std::result::Result::Ok(value) => value,
|
||||||
|
std::result::Result::Err(error) => return std::result::Result::Err(error),
|
||||||
|
};
|
||||||
|
},
|
||||||
|
}
|
||||||
|
match outcome.observation() {
|
||||||
|
crate::RawTransactionIngestObservationPersistence::Inserted => {
|
||||||
|
observation_inserted_total = match checked_counter(observation_inserted_total, "observation_inserted_total") {
|
||||||
|
std::result::Result::Ok(value) => value,
|
||||||
|
std::result::Result::Err(error) => return std::result::Result::Err(error),
|
||||||
|
};
|
||||||
|
},
|
||||||
|
crate::RawTransactionIngestObservationPersistence::AlreadyPresent => {
|
||||||
|
observation_already_present_total = match checked_counter(observation_already_present_total, "observation_already_present_total") {
|
||||||
|
std::result::Result::Ok(value) => value,
|
||||||
|
std::result::Result::Err(error) => return std::result::Result::Err(error),
|
||||||
|
};
|
||||||
|
},
|
||||||
|
crate::RawTransactionIngestObservationPersistence::NotRecorded => {},
|
||||||
|
}
|
||||||
|
self.snapshot.persisted_total = persisted_total;
|
||||||
|
self.snapshot.entity_inserted_total = entity_inserted_total;
|
||||||
|
self.snapshot.entity_already_present_total = entity_already_present_total;
|
||||||
|
self.snapshot.entity_skipped_purged_total = entity_skipped_purged_total;
|
||||||
|
self.snapshot.observation_inserted_total = observation_inserted_total;
|
||||||
|
self.snapshot.observation_already_present_total = observation_already_present_total;
|
||||||
|
return self.publish(state, admission_queue_depth, in_flight_persistence);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Records one classified persistence fault counter and publishes the resulting latest value.
|
||||||
|
pub(crate) fn record_persistence_fault(
|
||||||
|
&mut self,
|
||||||
|
state: ksp_worker_api::WorkerState,
|
||||||
|
admission_queue_depth: usize,
|
||||||
|
in_flight_persistence: usize,
|
||||||
|
code: ksp_core_lib::ErrorCode,
|
||||||
|
) -> ksp_core_lib::Result<()> {
|
||||||
|
if code == crate::ERROR_CODE_RAW_TRANSACTION_INGEST_CONTENT_CONFLICT {
|
||||||
|
let next = match checked_counter(self.snapshot.content_conflict_total, "content_conflict_total") {
|
||||||
|
std::result::Result::Ok(value) => value,
|
||||||
|
std::result::Result::Err(error) => return std::result::Result::Err(error),
|
||||||
|
};
|
||||||
|
self.snapshot.content_conflict_total = next;
|
||||||
|
} else if code == crate::ERROR_CODE_RAW_TRANSACTION_INGEST_STORE_FAILED {
|
||||||
|
let next = match checked_counter(self.snapshot.store_failure_total, "store_failure_total") {
|
||||||
|
std::result::Result::Ok(value) => value,
|
||||||
|
std::result::Result::Err(error) => return std::result::Result::Err(error),
|
||||||
|
};
|
||||||
|
self.snapshot.store_failure_total = next;
|
||||||
|
}
|
||||||
|
return self.publish(state, admission_queue_depth, in_flight_persistence);
|
||||||
|
}
|
||||||
|
|
||||||
|
fn publish(&mut self, state: ksp_worker_api::WorkerState, admission_queue_depth: usize, in_flight_persistence: usize) -> ksp_core_lib::Result<()> {
|
||||||
|
let sequence = match self.snapshot.worker.sequence().next() {
|
||||||
|
std::result::Result::Ok(value) => value,
|
||||||
|
std::result::Result::Err(_) => return std::result::Result::Err(crate::counter_exhausted_error("snapshot_sequence")),
|
||||||
|
};
|
||||||
|
let worker = ksp_worker_api::WorkerSnapshot::new(
|
||||||
|
self.snapshot.worker.id().clone(),
|
||||||
|
self.snapshot.worker.kind().clone(),
|
||||||
|
sequence,
|
||||||
|
state,
|
||||||
|
health_for_state(state, self.snapshot.worker.health()),
|
||||||
|
activity_for_state(state, admission_queue_depth, in_flight_persistence),
|
||||||
|
);
|
||||||
|
self.snapshot.worker = worker;
|
||||||
|
self.snapshot.admission_queue_depth = admission_queue_depth;
|
||||||
|
self.snapshot.in_flight_persistence = in_flight_persistence;
|
||||||
|
self.sender.send_replace(self.snapshot.clone());
|
||||||
|
return std::result::Result::Ok(());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn activity_for_state(state: ksp_worker_api::WorkerState, admission_queue_depth: usize, in_flight_persistence: usize) -> ksp_worker_api::WorkerActivity {
|
||||||
|
if admission_queue_depth > 0 || in_flight_persistence > 0 {
|
||||||
|
return ksp_worker_api::WorkerActivity::Active;
|
||||||
|
}
|
||||||
|
return match state {
|
||||||
|
ksp_worker_api::WorkerState::Running
|
||||||
|
| ksp_worker_api::WorkerState::Stopping
|
||||||
|
| ksp_worker_api::WorkerState::Stopped
|
||||||
|
| ksp_worker_api::WorkerState::Faulted(_) => ksp_worker_api::WorkerActivity::Idle,
|
||||||
|
_ => ksp_worker_api::WorkerActivity::Unknown,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
fn checked_counter(current: u64, field: &'static str) -> ksp_core_lib::Result<u64> {
|
||||||
|
return match current.checked_add(1) {
|
||||||
|
std::option::Option::Some(value) => std::result::Result::Ok(value),
|
||||||
|
std::option::Option::None => std::result::Result::Err(crate::counter_exhausted_error(field)),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
fn health_for_state(state: ksp_worker_api::WorkerState, previous: ksp_worker_api::WorkerHealth) -> ksp_worker_api::WorkerHealth {
|
||||||
|
return match state {
|
||||||
|
ksp_worker_api::WorkerState::Running => ksp_worker_api::WorkerHealth::Healthy,
|
||||||
|
ksp_worker_api::WorkerState::Stopping | ksp_worker_api::WorkerState::Stopped => previous,
|
||||||
|
ksp_worker_api::WorkerState::Faulted(_) => ksp_worker_api::WorkerHealth::Unhealthy,
|
||||||
|
_ => ksp_worker_api::WorkerHealth::Unknown,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
#[path = "../unit_tests/snapshot.rs"]
|
||||||
|
mod tests;
|
||||||
@@ -1,5 +1,5 @@
|
|||||||
// file: crates/ksp-worker-raw-transaction-ingest-lib/tests/dependency_boundary.rs
|
// file: crates/ksp-worker-raw-transaction-ingest-lib/tests/dependency_boundary.rs
|
||||||
// version: 6
|
// version: 7
|
||||||
|
|
||||||
//! Dependency firewall canaries for the RAW transaction ingest Worker foundation.
|
//! Dependency firewall canaries for the RAW transaction ingest Worker foundation.
|
||||||
|
|
||||||
@@ -52,25 +52,31 @@ fn pre_002_manifest_keeps_forbidden_layers_and_live_sources_out() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn pre_007_source_surface_adds_private_normal_store_persistence_without_backend_or_live_source() {
|
fn pre_008_source_surface_adds_one_latest_value_snapshot_watch_without_backend_or_live_source() {
|
||||||
let root = include_str!("../src/lib.rs");
|
let root = include_str!("../src/lib.rs");
|
||||||
let runtime = include_str!("../src/runtime.rs");
|
let runtime = include_str!("../src/runtime.rs");
|
||||||
let admission = include_str!("../src/admission.rs");
|
let admission = include_str!("../src/admission.rs");
|
||||||
let persistence = include_str!("../src/persistence.rs");
|
let persistence = include_str!("../src/persistence.rs");
|
||||||
|
let snapshot = include_str!("../src/snapshot.rs");
|
||||||
for required in [
|
for required in [
|
||||||
"tokio::sync::mpsc::channel",
|
"tokio::sync::mpsc::channel",
|
||||||
"canonicalize_raw_transaction",
|
"canonicalize_raw_transaction",
|
||||||
"assemble_raw_transaction_acquisition",
|
"assemble_raw_transaction_acquisition",
|
||||||
"RawTransactionIngestPersistencePort",
|
"RawTransactionIngestPersistencePort",
|
||||||
"persist_raw_transaction_acquisition",
|
|
||||||
"RawTransactionAcquisitionMode::Normal",
|
"RawTransactionAcquisitionMode::Normal",
|
||||||
"persistence_concurrency()",
|
"RawTransactionIngestSnapshot",
|
||||||
"ERROR_CODE_RAW_TRANSACTION_INGEST_CONTENT_CONFLICT",
|
"RawTransactionIngestSnapshotSource",
|
||||||
"ERROR_CODE_RAW_TRANSACTION_INGEST_STORE_FAILED",
|
"WorkerSnapshotSource",
|
||||||
|
"tokio::sync::watch::channel",
|
||||||
|
"ERROR_CODE_RAW_TRANSACTION_INGEST_COUNTER_EXHAUSTED",
|
||||||
] {
|
] {
|
||||||
assert!(
|
assert!(
|
||||||
root.contains(required) || runtime.contains(required) || admission.contains(required) || persistence.contains(required),
|
root.contains(required)
|
||||||
"required pre.007 persistence contract missing: {required}"
|
|| runtime.contains(required)
|
||||||
|
|| admission.contains(required)
|
||||||
|
|| persistence.contains(required)
|
||||||
|
|| snapshot.contains(required),
|
||||||
|
"required pre.008 snapshot contract missing: {required}"
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
for forbidden in [
|
for forbidden in [
|
||||||
@@ -80,15 +86,21 @@ fn pre_007_source_surface_adds_private_normal_store_persistence_without_backend_
|
|||||||
"unbounded_channel",
|
"unbounded_channel",
|
||||||
"ksp_store_postgres_lib::",
|
"ksp_store_postgres_lib::",
|
||||||
"ksp_onchain_transport_lib::",
|
"ksp_onchain_transport_lib::",
|
||||||
"RawTransactionIngestSnapshot",
|
|
||||||
"WorkerSnapshotSource",
|
|
||||||
"ForceRehydrate",
|
"ForceRehydrate",
|
||||||
|
"source_failed",
|
||||||
|
"drain_timeout",
|
||||||
] {
|
] {
|
||||||
assert!(
|
assert!(
|
||||||
!root.contains(forbidden) && !runtime.contains(forbidden) && !admission.contains(forbidden) && !persistence.contains(forbidden),
|
!root.contains(forbidden)
|
||||||
"pre.007 crossed a forbidden runtime boundary: {forbidden}"
|
&& !runtime.contains(forbidden)
|
||||||
|
&& !admission.contains(forbidden)
|
||||||
|
&& !persistence.contains(forbidden)
|
||||||
|
&& !snapshot.contains(forbidden),
|
||||||
|
"pre.008 crossed a forbidden runtime boundary: {forbidden}"
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
assert_eq!(runtime.matches("tokio::sync::watch::channel").count(), 2, "runtime retains only the external-control and private-source stop watches");
|
||||||
|
assert_eq!(snapshot.matches("tokio::sync::watch::channel").count(), 1, "exactly one shared concrete snapshot watch is allowed");
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
// file: crates/ksp-worker-raw-transaction-ingest-lib/tests/public_api.rs
|
// file: crates/ksp-worker-raw-transaction-ingest-lib/tests/public_api.rs
|
||||||
// version: 6
|
// version: 7
|
||||||
|
|
||||||
//! External public-surface proofs for the RAW transaction ingest Worker identity and settings foundation.
|
//! External public-surface proofs for the RAW transaction ingest Worker foundation.
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn pre_003_kind_code_and_settings_are_consumable_from_crate_root() {
|
fn pre_003_kind_code_and_settings_are_consumable_from_crate_root() {
|
||||||
@@ -100,3 +100,29 @@ fn pre_007_store_and_content_conflict_error_codes_are_public_and_stable() {
|
|||||||
assert_eq!(ksp_worker_raw_transaction_ingest_lib::ERROR_CODE_RAW_TRANSACTION_INGEST_STORE_FAILED.code(), "store_failed");
|
assert_eq!(ksp_worker_raw_transaction_ingest_lib::ERROR_CODE_RAW_TRANSACTION_INGEST_STORE_FAILED.code(), "store_failed");
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn pre_008_snapshot_surface_and_common_projection_are_public_and_stable() {
|
||||||
|
let _snapshot_source: fn(
|
||||||
|
&ksp_worker_raw_transaction_ingest_lib::RawTransactionIngestHandle,
|
||||||
|
) -> ksp_worker_raw_transaction_ingest_lib::RawTransactionIngestSnapshotSource =
|
||||||
|
ksp_worker_raw_transaction_ingest_lib::RawTransactionIngestHandle::snapshot_source;
|
||||||
|
let _worker_snapshot_source: fn(
|
||||||
|
&ksp_worker_raw_transaction_ingest_lib::RawTransactionIngestHandle,
|
||||||
|
) -> ksp_worker_raw_transaction_ingest_lib::RawTransactionIngestSnapshotSource =
|
||||||
|
ksp_worker_raw_transaction_ingest_lib::RawTransactionIngestHandle::worker_snapshot_source;
|
||||||
|
let _current: fn(
|
||||||
|
&ksp_worker_raw_transaction_ingest_lib::RawTransactionIngestSnapshotSource,
|
||||||
|
) -> ksp_worker_raw_transaction_ingest_lib::RawTransactionIngestSnapshot =
|
||||||
|
ksp_worker_raw_transaction_ingest_lib::RawTransactionIngestSnapshotSource::current;
|
||||||
|
let _wait: for<'a> fn(
|
||||||
|
&'a ksp_worker_raw_transaction_ingest_lib::RawTransactionIngestSnapshotSource,
|
||||||
|
ksp_worker_api::WorkerSnapshotSequence,
|
||||||
|
) -> ksp_worker_raw_transaction_ingest_lib::RawTransactionIngestSnapshotFuture<'a> =
|
||||||
|
ksp_worker_raw_transaction_ingest_lib::RawTransactionIngestSnapshotSource::wait_for_change;
|
||||||
|
fn require_worker_source<T: ksp_worker_api::WorkerSnapshotSource + std::marker::Send + std::marker::Sync>() {}
|
||||||
|
require_worker_source::<ksp_worker_raw_transaction_ingest_lib::RawTransactionIngestSnapshotSource>();
|
||||||
|
assert_eq!(ksp_worker_raw_transaction_ingest_lib::ERROR_CODE_RAW_TRANSACTION_INGEST_COUNTER_EXHAUSTED.domain(), "worker_raw_transaction_ingest");
|
||||||
|
assert_eq!(ksp_worker_raw_transaction_ingest_lib::ERROR_CODE_RAW_TRANSACTION_INGEST_COUNTER_EXHAUSTED.code(), "counter_exhausted");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
// file: crates/ksp-worker-raw-transaction-ingest-lib/unit_tests/runtime.rs
|
// file: crates/ksp-worker-raw-transaction-ingest-lib/unit_tests/runtime.rs
|
||||||
// version: 4
|
// version: 5
|
||||||
|
|
||||||
struct ActiveTaskGuard {
|
struct ActiveTaskGuard {
|
||||||
active: std::sync::Arc<std::sync::atomic::AtomicUsize>,
|
active: std::sync::Arc<std::sync::atomic::AtomicUsize>,
|
||||||
@@ -88,7 +88,7 @@ async fn pre_004_immediate_stop_is_idempotent_and_reaches_stopped_terminal() {
|
|||||||
std::result::Result::Ok(value) => value,
|
std::result::Result::Ok(value) => value,
|
||||||
std::result::Result::Err(_) => return,
|
std::result::Result::Err(_) => return,
|
||||||
};
|
};
|
||||||
assert_eq!(*handle.terminal_receiver.borrow(), ksp_worker_api::WorkerState::Starting);
|
assert_eq!(handle.snapshot_source().current().worker_snapshot().state(), ksp_worker_api::WorkerState::Starting);
|
||||||
assert!(handle.request_stop());
|
assert!(handle.request_stop());
|
||||||
assert!(!handle.request_stop());
|
assert!(!handle.request_stop());
|
||||||
let terminal = handle.wait_terminal().await;
|
let terminal = handle.wait_terminal().await;
|
||||||
@@ -113,7 +113,7 @@ async fn pre_004_running_lifecycle_stops_through_cloned_handle() {
|
|||||||
std::result::Result::Err(_) => return,
|
std::result::Result::Err(_) => return,
|
||||||
};
|
};
|
||||||
tokio::task::yield_now().await;
|
tokio::task::yield_now().await;
|
||||||
assert_eq!(*handle.terminal_receiver.borrow(), ksp_worker_api::WorkerState::Running);
|
assert_eq!(handle.snapshot_source().current().worker_snapshot().state(), ksp_worker_api::WorkerState::Running);
|
||||||
let clone = handle.clone();
|
let clone = handle.clone();
|
||||||
assert!(clone.request_stop());
|
assert!(clone.request_stop());
|
||||||
assert!(!handle.request_stop());
|
assert!(!handle.request_stop());
|
||||||
@@ -136,21 +136,15 @@ async fn pre_004_dropping_last_control_handle_causes_private_runtime_exit() {
|
|||||||
std::result::Result::Ok(value) => value,
|
std::result::Result::Ok(value) => value,
|
||||||
std::result::Result::Err(_) => return,
|
std::result::Result::Err(_) => return,
|
||||||
};
|
};
|
||||||
let mut observer = handle.terminal_receiver.clone();
|
let observer = handle.snapshot_source();
|
||||||
std::mem::drop(handle);
|
std::mem::drop(handle);
|
||||||
loop {
|
loop {
|
||||||
let current = *observer.borrow();
|
let current = observer.current();
|
||||||
if current.is_terminal() {
|
if current.worker_snapshot().state().is_terminal() {
|
||||||
let closed = observer.changed().await;
|
assert_eq!(current.worker_snapshot().state(), ksp_worker_api::WorkerState::Stopped);
|
||||||
assert!(closed.is_err());
|
|
||||||
assert_eq!(current, ksp_worker_api::WorkerState::Stopped);
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
let changed = observer.changed().await;
|
|
||||||
assert!(changed.is_ok(), "runtime closed before publishing terminal state");
|
|
||||||
if changed.is_err() {
|
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
let _changed = observer.wait_for_change(current.worker_snapshot().sequence()).await;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -485,6 +479,7 @@ async fn pre_007_content_conflict_becomes_terminal_after_private_drain() {
|
|||||||
std::result::Result::Ok(value) => value,
|
std::result::Result::Ok(value) => value,
|
||||||
std::result::Result::Err(_) => return,
|
std::result::Result::Err(_) => return,
|
||||||
};
|
};
|
||||||
|
let source = handle.snapshot_source();
|
||||||
let terminal = handle.wait_terminal().await;
|
let terminal = handle.wait_terminal().await;
|
||||||
let terminal = match terminal {
|
let terminal = match terminal {
|
||||||
std::result::Result::Ok(value) => value,
|
std::result::Result::Ok(value) => value,
|
||||||
@@ -492,6 +487,10 @@ async fn pre_007_content_conflict_becomes_terminal_after_private_drain() {
|
|||||||
};
|
};
|
||||||
assert_eq!(terminal, ksp_worker_api::WorkerState::Faulted(crate::ERROR_CODE_RAW_TRANSACTION_INGEST_CONTENT_CONFLICT));
|
assert_eq!(terminal, ksp_worker_api::WorkerState::Faulted(crate::ERROR_CODE_RAW_TRANSACTION_INGEST_CONTENT_CONFLICT));
|
||||||
assert_eq!(port.completed.load(std::sync::atomic::Ordering::Acquire), 1);
|
assert_eq!(port.completed.load(std::sync::atomic::Ordering::Acquire), 1);
|
||||||
|
let snapshot = source.current();
|
||||||
|
assert_eq!(snapshot.content_conflict_total(), 1);
|
||||||
|
assert_eq!(snapshot.store_failure_total(), 0);
|
||||||
|
assert_eq!(snapshot.worker_snapshot().health(), ksp_worker_api::WorkerHealth::Unhealthy);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -532,3 +531,115 @@ async fn pre_007_store_failure_becomes_terminal_without_exposing_remote_error_te
|
|||||||
assert!(!std::format!("{handle:?}").contains("synthetic store failure"));
|
assert!(!std::format!("{handle:?}").contains("synthetic store failure"));
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test(flavor = "current_thread")]
|
||||||
|
async fn pre_008_runtime_snapshots_count_successful_pipeline_and_retain_terminal() {
|
||||||
|
let settings = match settings_with_persistence_concurrency(2) {
|
||||||
|
std::option::Option::Some(value) => value,
|
||||||
|
std::option::Option::None => return,
|
||||||
|
};
|
||||||
|
let network = settings.network().clone();
|
||||||
|
let port = std::sync::Arc::new(RuntimePersistencePort::new(network.clone(), RuntimePortResponse::SuccessBlocked, true));
|
||||||
|
let runtime_port: super::PersistencePort = port.clone();
|
||||||
|
let handle = match super::start_foundation_with_port_and_source_spawner(
|
||||||
|
settings,
|
||||||
|
tokio::runtime::Handle::current(),
|
||||||
|
std::option::Option::Some(runtime_port),
|
||||||
|
move |children, _stop_receiver, admission_sender| {
|
||||||
|
let _abort_handle = children.spawn(async move {
|
||||||
|
for signature_byte in 21..=23 {
|
||||||
|
let ingress = match runtime_ingress(&network, signature_byte) {
|
||||||
|
std::option::Option::Some(value) => value,
|
||||||
|
std::option::Option::None => return,
|
||||||
|
};
|
||||||
|
if admission_sender.send(ingress).await.is_err() {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return;
|
||||||
|
});
|
||||||
|
},
|
||||||
|
) {
|
||||||
|
std::result::Result::Ok(value) => value,
|
||||||
|
std::result::Result::Err(_) => return,
|
||||||
|
};
|
||||||
|
let concrete_source = handle.snapshot_source();
|
||||||
|
let common_source = handle.worker_snapshot_source();
|
||||||
|
assert!(wait_for_completed(&port, 3).await);
|
||||||
|
assert!(handle.request_stop());
|
||||||
|
let terminal = handle.wait_terminal().await;
|
||||||
|
let terminal = match terminal {
|
||||||
|
std::result::Result::Ok(value) => value,
|
||||||
|
std::result::Result::Err(_) => return,
|
||||||
|
};
|
||||||
|
assert_eq!(terminal, ksp_worker_api::WorkerState::Stopped);
|
||||||
|
let snapshot = concrete_source.current();
|
||||||
|
assert_eq!(snapshot.worker_snapshot().state(), ksp_worker_api::WorkerState::Stopped);
|
||||||
|
assert_eq!(snapshot.worker_snapshot().health(), ksp_worker_api::WorkerHealth::Healthy);
|
||||||
|
assert_eq!(snapshot.worker_snapshot().activity(), ksp_worker_api::WorkerActivity::Idle);
|
||||||
|
assert_eq!(snapshot.admitted_total(), 3);
|
||||||
|
assert_eq!(snapshot.canonicalized_total(), 3);
|
||||||
|
assert_eq!(snapshot.persisted_total(), 3);
|
||||||
|
assert_eq!(snapshot.entity_inserted_total(), 3);
|
||||||
|
assert_eq!(snapshot.entity_already_present_total(), 0);
|
||||||
|
assert_eq!(snapshot.entity_skipped_purged_total(), 0);
|
||||||
|
assert_eq!(snapshot.observation_inserted_total(), 3);
|
||||||
|
assert_eq!(snapshot.observation_already_present_total(), 0);
|
||||||
|
assert_eq!(snapshot.content_conflict_total(), 0);
|
||||||
|
assert_eq!(snapshot.store_failure_total(), 0);
|
||||||
|
assert_eq!(snapshot.source_failure_total(), 0);
|
||||||
|
assert_eq!(snapshot.backpressure_wait_total(), 0);
|
||||||
|
assert_eq!(snapshot.admission_queue_depth(), 0);
|
||||||
|
assert_eq!(snapshot.in_flight_persistence(), 0);
|
||||||
|
let common = ksp_worker_api::WorkerSnapshotSource::current(&common_source);
|
||||||
|
assert_eq!(&common, snapshot.worker_snapshot());
|
||||||
|
let retained = concrete_source.wait_for_change(snapshot.worker_snapshot().sequence()).await;
|
||||||
|
assert_eq!(retained, snapshot);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test(flavor = "current_thread")]
|
||||||
|
async fn pre_008_fault_snapshots_count_classified_store_failures_and_project_unhealthy() {
|
||||||
|
let settings = match settings_with_persistence_concurrency(1) {
|
||||||
|
std::option::Option::Some(value) => value,
|
||||||
|
std::option::Option::None => return,
|
||||||
|
};
|
||||||
|
let network = settings.network().clone();
|
||||||
|
let port = std::sync::Arc::new(RuntimePersistencePort::new(network.clone(), RuntimePortResponse::StoreFailure, true));
|
||||||
|
let runtime_port: super::PersistencePort = port.clone();
|
||||||
|
let handle = match super::start_foundation_with_port_and_source_spawner(
|
||||||
|
settings,
|
||||||
|
tokio::runtime::Handle::current(),
|
||||||
|
std::option::Option::Some(runtime_port),
|
||||||
|
move |children, _stop_receiver, admission_sender| {
|
||||||
|
let _abort_handle = children.spawn(async move {
|
||||||
|
let ingress = match runtime_ingress(&network, 24) {
|
||||||
|
std::option::Option::Some(value) => value,
|
||||||
|
std::option::Option::None => return,
|
||||||
|
};
|
||||||
|
let _sent = admission_sender.send(ingress).await;
|
||||||
|
return;
|
||||||
|
});
|
||||||
|
},
|
||||||
|
) {
|
||||||
|
std::result::Result::Ok(value) => value,
|
||||||
|
std::result::Result::Err(_) => return,
|
||||||
|
};
|
||||||
|
let source = handle.snapshot_source();
|
||||||
|
let terminal = handle.wait_terminal().await;
|
||||||
|
let terminal = match terminal {
|
||||||
|
std::result::Result::Ok(value) => value,
|
||||||
|
std::result::Result::Err(_) => return,
|
||||||
|
};
|
||||||
|
assert_eq!(terminal, ksp_worker_api::WorkerState::Faulted(crate::ERROR_CODE_RAW_TRANSACTION_INGEST_STORE_FAILED));
|
||||||
|
let snapshot = source.current();
|
||||||
|
assert_eq!(snapshot.worker_snapshot().state(), terminal);
|
||||||
|
assert_eq!(snapshot.worker_snapshot().health(), ksp_worker_api::WorkerHealth::Unhealthy);
|
||||||
|
assert_eq!(snapshot.worker_snapshot().activity(), ksp_worker_api::WorkerActivity::Idle);
|
||||||
|
assert_eq!(snapshot.admitted_total(), 1);
|
||||||
|
assert_eq!(snapshot.canonicalized_total(), 1);
|
||||||
|
assert_eq!(snapshot.persisted_total(), 0);
|
||||||
|
assert_eq!(snapshot.store_failure_total(), 1);
|
||||||
|
assert_eq!(snapshot.content_conflict_total(), 0);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|||||||
@@ -0,0 +1,115 @@
|
|||||||
|
// file: crates/ksp-worker-raw-transaction-ingest-lib/unit_tests/snapshot.rs
|
||||||
|
// version: 1
|
||||||
|
|
||||||
|
fn snapshot_foundation() -> std::option::Option<(crate::RawTransactionIngestSnapshotPublisher, crate::RawTransactionIngestSnapshotSource)> {
|
||||||
|
let network = match ksp_store_lib::RawNetworkId::new("mainnet") {
|
||||||
|
std::result::Result::Ok(value) => value,
|
||||||
|
std::result::Result::Err(_) => return std::option::Option::None,
|
||||||
|
};
|
||||||
|
let worker_id = match ksp_worker_api::WorkerId::new("raw-ingest-snapshot-001") {
|
||||||
|
std::result::Result::Ok(value) => value,
|
||||||
|
std::result::Result::Err(_) => return std::option::Option::None,
|
||||||
|
};
|
||||||
|
let settings = crate::RawTransactionIngestSettings::with_defaults(network, worker_id.clone());
|
||||||
|
let kind = match ksp_worker_api::WorkerKindCode::new(crate::RAW_TRANSACTION_INGEST_WORKER_KIND_CODE) {
|
||||||
|
std::result::Result::Ok(value) => value,
|
||||||
|
std::result::Result::Err(_) => return std::option::Option::None,
|
||||||
|
};
|
||||||
|
let mut lifecycle = ksp_worker_api::WorkerLifecycle::new(worker_id, kind);
|
||||||
|
if lifecycle.start().is_err() {
|
||||||
|
return std::option::Option::None;
|
||||||
|
}
|
||||||
|
return std::option::Option::Some(crate::RawTransactionIngestSnapshotPublisher::new(&settings, &lifecycle));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn pre_008_initial_snapshot_and_common_projection_are_exact() {
|
||||||
|
let (publisher, source) = match snapshot_foundation() {
|
||||||
|
std::option::Option::Some(value) => value,
|
||||||
|
std::option::Option::None => return,
|
||||||
|
};
|
||||||
|
let _publisher = publisher;
|
||||||
|
let concrete = source.current();
|
||||||
|
assert_eq!(concrete.worker_snapshot().sequence().value(), 0);
|
||||||
|
assert_eq!(concrete.worker_snapshot().state(), ksp_worker_api::WorkerState::Starting);
|
||||||
|
assert_eq!(concrete.worker_snapshot().health(), ksp_worker_api::WorkerHealth::Unknown);
|
||||||
|
assert_eq!(concrete.worker_snapshot().activity(), ksp_worker_api::WorkerActivity::Unknown);
|
||||||
|
assert_eq!(concrete.admission_queue_capacity(), crate::DEFAULT_RAW_TRANSACTION_INGEST_ADMISSION_QUEUE_CAPACITY);
|
||||||
|
assert_eq!(concrete.admission_queue_depth(), 0);
|
||||||
|
assert_eq!(concrete.persistence_concurrency(), crate::DEFAULT_RAW_TRANSACTION_INGEST_PERSISTENCE_CONCURRENCY);
|
||||||
|
assert_eq!(concrete.in_flight_persistence(), 0);
|
||||||
|
assert_eq!(concrete.admitted_total(), 0);
|
||||||
|
assert_eq!(concrete.canonicalized_total(), 0);
|
||||||
|
assert_eq!(concrete.persisted_total(), 0);
|
||||||
|
assert_eq!(concrete.entity_inserted_total(), 0);
|
||||||
|
assert_eq!(concrete.entity_already_present_total(), 0);
|
||||||
|
assert_eq!(concrete.entity_skipped_purged_total(), 0);
|
||||||
|
assert_eq!(concrete.observation_inserted_total(), 0);
|
||||||
|
assert_eq!(concrete.observation_already_present_total(), 0);
|
||||||
|
assert_eq!(concrete.content_conflict_total(), 0);
|
||||||
|
assert_eq!(concrete.store_failure_total(), 0);
|
||||||
|
assert_eq!(concrete.source_failure_total(), 0);
|
||||||
|
assert_eq!(concrete.backpressure_wait_total(), 0);
|
||||||
|
let common = ksp_worker_api::WorkerSnapshotSource::current(&source);
|
||||||
|
assert_eq!(&common, concrete.worker_snapshot());
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn pre_008_checked_counter_exhaustion_is_stable_and_never_wraps() {
|
||||||
|
let result = super::checked_counter(u64::MAX, "admitted_total");
|
||||||
|
assert!(result.is_err());
|
||||||
|
let error = match result {
|
||||||
|
std::result::Result::Ok(_) => return,
|
||||||
|
std::result::Result::Err(value) => value,
|
||||||
|
};
|
||||||
|
assert_eq!(error.code(), crate::ERROR_CODE_RAW_TRANSACTION_INGEST_COUNTER_EXHAUSTED);
|
||||||
|
assert_eq!(error.context().len(), 1);
|
||||||
|
assert_eq!(error.context()[0].key(), "field");
|
||||||
|
assert_eq!(error.context()[0].value(), "admitted_total");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test(flavor = "current_thread")]
|
||||||
|
async fn pre_008_slow_concrete_and_common_listeners_coalesce_to_same_latest_sequence() {
|
||||||
|
let (mut publisher, source) = match snapshot_foundation() {
|
||||||
|
std::option::Option::Some(value) => value,
|
||||||
|
std::option::Option::None => return,
|
||||||
|
};
|
||||||
|
let observed = source.current().worker_snapshot().sequence();
|
||||||
|
assert!(publisher.publish_state(ksp_worker_api::WorkerState::Running, 0, 0).is_ok());
|
||||||
|
assert!(publisher.record_admission_success(ksp_worker_api::WorkerState::Running, 1, 1).is_ok());
|
||||||
|
let concrete = source.wait_for_change(observed).await;
|
||||||
|
let common = ksp_worker_api::WorkerSnapshotSource::wait_for_change(&source, observed).await;
|
||||||
|
assert_eq!(concrete.worker_snapshot().sequence(), common.sequence());
|
||||||
|
assert_eq!(concrete.worker_snapshot().sequence().value(), 2);
|
||||||
|
assert_eq!(concrete.worker_snapshot().state(), ksp_worker_api::WorkerState::Running);
|
||||||
|
assert_eq!(concrete.worker_snapshot().health(), ksp_worker_api::WorkerHealth::Healthy);
|
||||||
|
assert_eq!(concrete.worker_snapshot().activity(), ksp_worker_api::WorkerActivity::Active);
|
||||||
|
assert_eq!(concrete.admission_queue_depth(), 1);
|
||||||
|
assert_eq!(concrete.in_flight_persistence(), 1);
|
||||||
|
assert_eq!(concrete.admitted_total(), 1);
|
||||||
|
assert_eq!(concrete.canonicalized_total(), 1);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test(flavor = "current_thread")]
|
||||||
|
async fn pre_008_terminal_latest_value_is_retained_after_publisher_drop() {
|
||||||
|
let (mut publisher, source) = match snapshot_foundation() {
|
||||||
|
std::option::Option::Some(value) => value,
|
||||||
|
std::option::Option::None => return,
|
||||||
|
};
|
||||||
|
assert!(publisher.publish_state(ksp_worker_api::WorkerState::Running, 0, 0).is_ok());
|
||||||
|
assert!(publisher.publish_state(ksp_worker_api::WorkerState::Stopping, 0, 0).is_ok());
|
||||||
|
assert!(publisher.publish_state(ksp_worker_api::WorkerState::Stopped, 0, 0).is_ok());
|
||||||
|
std::mem::drop(publisher);
|
||||||
|
let retained = source.current();
|
||||||
|
assert_eq!(retained.worker_snapshot().state(), ksp_worker_api::WorkerState::Stopped);
|
||||||
|
assert_eq!(retained.worker_snapshot().health(), ksp_worker_api::WorkerHealth::Healthy);
|
||||||
|
assert_eq!(retained.worker_snapshot().activity(), ksp_worker_api::WorkerActivity::Idle);
|
||||||
|
let same_terminal = source.wait_for_change(retained.worker_snapshot().sequence()).await;
|
||||||
|
assert_eq!(same_terminal, retained);
|
||||||
|
let common = ksp_worker_api::WorkerSnapshotSource::wait_for_change(&source, retained.worker_snapshot().sequence()).await;
|
||||||
|
assert_eq!(common, retained.worker_snapshot().clone());
|
||||||
|
return;
|
||||||
|
}
|
||||||
167
deltas/0.3.11/pre.008.md
Normal file
167
deltas/0.3.11/pre.008.md
Normal file
@@ -0,0 +1,167 @@
|
|||||||
|
<!-- file: deltas/0.3.11/pre.008.md -->
|
||||||
|
<!-- version: 1 -->
|
||||||
|
|
||||||
|
# Delta `0.3.11-pre.008` — snapshots latest-value et projection Worker API
|
||||||
|
|
||||||
|
## Base requise
|
||||||
|
|
||||||
|
```text
|
||||||
|
0.3.11-pre.007
|
||||||
|
workspace.package.version = 0.3.11-pre.7
|
||||||
|
```
|
||||||
|
|
||||||
|
Le gate opérateur du 8 septembre 2026 est vert sur `fmt`, audits Rust/Markdown, `cargo check --workspace`, Clippy strict, 31 tests de la crate Worker et les doc-tests. Aucun `cargo tree` intermédiaire n'était requis : `pre.007` n'avait modifié aucune dépendance ni feature.
|
||||||
|
|
||||||
|
## Objectif
|
||||||
|
|
||||||
|
Matérialiser uniquement la responsabilité `pre.008` du plan `032` : snapshot concret latest-value, compteurs checked, projection directe vers `ksp-worker-api::WorkerSnapshotSource`, coalescence des listeners et terminal retenu. Aucun hardening timeout/abort et aucune source réseau.
|
||||||
|
|
||||||
|
## Version
|
||||||
|
|
||||||
|
```text
|
||||||
|
identifiant de livraison : 0.3.11-pre.008
|
||||||
|
workspace.package.version : 0.3.11-pre.8
|
||||||
|
```
|
||||||
|
|
||||||
|
## Snapshot concret
|
||||||
|
|
||||||
|
La crate expose :
|
||||||
|
|
||||||
|
```text
|
||||||
|
RawTransactionIngestSnapshot
|
||||||
|
RawTransactionIngestSnapshotFuture
|
||||||
|
RawTransactionIngestSnapshotSource
|
||||||
|
```
|
||||||
|
|
||||||
|
Le snapshot contient uniquement la projection `WorkerSnapshot`, les capacités/profondeurs techniques bornées et les compteurs sûrs prévus par le plan.
|
||||||
|
|
||||||
|
## Un seul flux latest-value
|
||||||
|
|
||||||
|
Un unique :
|
||||||
|
|
||||||
|
```text
|
||||||
|
tokio::sync::watch::channel<RawTransactionIngestSnapshot>
|
||||||
|
```
|
||||||
|
|
||||||
|
porte l'état observable du run. Le watch terminal spécialisé des tranches précédentes disparaît : `wait_terminal()`, `snapshot_source()` et `worker_snapshot_source()` lisent tous ce même flux.
|
||||||
|
|
||||||
|
`RawTransactionIngestSnapshotSource` implémente directement `ksp_worker_api::WorkerSnapshotSource`. La sequence `WorkerSnapshotSequence` est donc identique pour la vue concrète et la projection commune.
|
||||||
|
|
||||||
|
## Compteurs checked
|
||||||
|
|
||||||
|
Les compteurs `u64` effectivement incrémentés dans cette tranche utilisent `checked_add(1)`. Aucun wrap silencieux n'est admis. Le nouveau code public est :
|
||||||
|
|
||||||
|
```text
|
||||||
|
worker_raw_transaction_ingest.counter_exhausted
|
||||||
|
```
|
||||||
|
|
||||||
|
Les compteurs sont mis à jour au point d'ownership du supervisor : dequeue/canonicalisation et récolte de persistence. `admission_queue_depth` provient directement du receiver `mpsc` borné ; `in_flight_persistence` provient du `JoinSet` privé existant.
|
||||||
|
|
||||||
|
`source_failure_total` et `backpressure_wait_total` font partie du snapshot sûr mais restent à zéro en `pre.008`; leur comportement effectif appartient à `pre.009`.
|
||||||
|
|
||||||
|
## Projection common Worker
|
||||||
|
|
||||||
|
Mapping de fondation :
|
||||||
|
|
||||||
|
```text
|
||||||
|
Starting -> health Unknown
|
||||||
|
Running -> health Healthy
|
||||||
|
Stopping / Stopped -> conserve le dernier health non terminal
|
||||||
|
Faulted -> health Unhealthy
|
||||||
|
queue > 0 ou persistence > 0 -> activity Active
|
||||||
|
sinon Running/Stopping/terminal -> activity Idle
|
||||||
|
```
|
||||||
|
|
||||||
|
La state machine reste celle de `ksp-worker-api::WorkerLifecycle`.
|
||||||
|
|
||||||
|
## Tests ajoutés ou étendus
|
||||||
|
|
||||||
|
```text
|
||||||
|
snapshot initial exact
|
||||||
|
projection common/concrete même sequence
|
||||||
|
counter exhaustion stable sans wrap
|
||||||
|
listener lent coalescé vers la dernière valeur
|
||||||
|
terminal latest-value retenu après fermeture publisher
|
||||||
|
surface publique snapshot_source/worker_snapshot_source
|
||||||
|
WorkerSnapshotSource Send + Sync
|
||||||
|
pipeline runtime réussi -> counters exacts
|
||||||
|
Store failure -> health Unhealthy + store_failure_total
|
||||||
|
absence de watch terminal séparé
|
||||||
|
absence de nouveau dependency edge
|
||||||
|
```
|
||||||
|
|
||||||
|
## Fichiers ajoutés
|
||||||
|
|
||||||
|
```text
|
||||||
|
crates/ksp-worker-raw-transaction-ingest-lib/src/snapshot.rs
|
||||||
|
crates/ksp-worker-raw-transaction-ingest-lib/unit_tests/snapshot.rs
|
||||||
|
deltas/0.3.11/pre.008.md
|
||||||
|
```
|
||||||
|
|
||||||
|
## Fichiers modifiés
|
||||||
|
|
||||||
|
```text
|
||||||
|
Cargo.toml
|
||||||
|
crates/ksp-worker-raw-transaction-ingest-lib/src/admission.rs
|
||||||
|
crates/ksp-worker-raw-transaction-ingest-lib/src/error.rs
|
||||||
|
crates/ksp-worker-raw-transaction-ingest-lib/src/lib.rs
|
||||||
|
crates/ksp-worker-raw-transaction-ingest-lib/src/runtime.rs
|
||||||
|
crates/ksp-worker-raw-transaction-ingest-lib/tests/dependency_boundary.rs
|
||||||
|
crates/ksp-worker-raw-transaction-ingest-lib/tests/public_api.rs
|
||||||
|
crates/ksp-worker-raw-transaction-ingest-lib/unit_tests/runtime.rs
|
||||||
|
docs/plans/032-V0_3_11_RAW_TRANSACTION_INGEST_WORKER_FOUNDATION_PLAN.md
|
||||||
|
docs/validation/028-V0_3_11_RAW_TRANSACTION_INGEST_WORKER_FOUNDATION.md
|
||||||
|
```
|
||||||
|
|
||||||
|
## Fichiers supprimés
|
||||||
|
|
||||||
|
Aucun.
|
||||||
|
|
||||||
|
## Hors scope conservé
|
||||||
|
|
||||||
|
```text
|
||||||
|
source live / Transport
|
||||||
|
Config
|
||||||
|
backend Store direct
|
||||||
|
source_failed public
|
||||||
|
source failure behavior
|
||||||
|
backpressure wait instrumentation productive
|
||||||
|
drain timeout / abort forcé
|
||||||
|
fault/stop precedence finalisée
|
||||||
|
retry / reconnect / gap repair
|
||||||
|
```
|
||||||
|
|
||||||
|
## Validations exécutées dans l'environnement d'assemblage
|
||||||
|
|
||||||
|
```text
|
||||||
|
General Rust rule audit: clean
|
||||||
|
Rust export completeness audit: 0 candidate(s)
|
||||||
|
KSP workspace Rust rule audit: clean
|
||||||
|
Markdown table audit: clean
|
||||||
|
|
||||||
|
scan production : aucun unwrap, expect, panic, unbounded_channel
|
||||||
|
scan scope : aucun backend Store direct, aucun Transport, aucune nouvelle dépendance
|
||||||
|
```
|
||||||
|
|
||||||
|
L'environnement d'assemblage ne fournit ni `cargo`, ni `rustc`, ni `rustfmt`. Aucun gate Cargo de `pre.008` n'est déclaré PASS localement.
|
||||||
|
|
||||||
|
## Gate opérateur demandé
|
||||||
|
|
||||||
|
Aucune dépendance ni feature n'a changé dans cette tranche ; aucun `cargo tree` intermédiaire n'est requis :
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cargo fmt --all
|
||||||
|
python3 scripts/audit_rust_workspace_rules.py
|
||||||
|
python3 scripts/audit_markdown_tables.py README.md RULES.md ROADMAP.md CHANGELOG.md docs prompts crates deltas
|
||||||
|
cargo check --workspace
|
||||||
|
cargo clippy --workspace --all-targets --all-features -- -D warnings
|
||||||
|
cargo test -p ksp-worker-raw-transaction-ingest-lib
|
||||||
|
```
|
||||||
|
|
||||||
|
## Décision
|
||||||
|
|
||||||
|
`pre.009` reste bloquée jusqu'à validation opérateur verte des snapshots common/concrete, des compteurs checked et du terminal latest-value retenu.
|
||||||
|
|
||||||
|
## Questions ouvertes
|
||||||
|
|
||||||
|
Aucune nouvelle question architecturale. Le hardening shutdown/backpressure/fault races reste dans `pre.009`.
|
||||||
@@ -1,5 +1,5 @@
|
|||||||
<!-- file: docs/plans/032-V0_3_11_RAW_TRANSACTION_INGEST_WORKER_FOUNDATION_PLAN.md -->
|
<!-- file: docs/plans/032-V0_3_11_RAW_TRANSACTION_INGEST_WORKER_FOUNDATION_PLAN.md -->
|
||||||
<!-- version: 7 -->
|
<!-- version: 8 -->
|
||||||
|
|
||||||
# Plan v0.3.11 — fondation runtime du Worker RawTransaction ingest
|
# Plan v0.3.11 — fondation runtime du Worker RawTransaction ingest
|
||||||
|
|
||||||
@@ -732,12 +732,14 @@ Budget cible : **15–20 min**. `mpsc` borné, backpressure, ingress privé, com
|
|||||||
|
|
||||||
Budget cible : **15–20 min**. Port privé Store, mode Normal, concurrency bornée, outcomes new/idempotent/purged, observation distincte, Store error et content conflict terminal.
|
Budget cible : **15–20 min**. Port privé Store, mode Normal, concurrency bornée, outcomes new/idempotent/purged, observation distincte, Store error et content conflict terminal.
|
||||||
|
|
||||||
État après matérialisation : **implémenté, gate opérateur requis**. La crate possède désormais un port de persistence Store crate-private, implémenté par la façade `ksp-store-lib::Store`, et appelle exclusivement l'opération atomique transaction + observation en `RawTransactionAcquisitionMode::Normal`. Le supervisor possède un `JoinSet` privé de persistences ; il ne lit une nouvelle acquisition que si `in_flight < persistence_concurrency`, ce qui borne les écritures sans sémaphore ni file secondaire. Les succès distinguent `Inserted`, `AlreadyPresent`, `SkippedPurged` et les observations `Inserted`, `AlreadyPresent`, `NotRecorded`; `Rehydrated` ou toute combinaison impossible en mode Normal deviennent `runtime_invalid`. `store_api.raw_conflict` est réduit à `worker_raw_transaction_ingest.content_conflict`; toute autre erreur Store devient `worker_raw_transaction_ingest.store_failed` en ne conservant que le `ErrorCode` inférieur sûr. Le premier fault de cette tranche signale le stop privé, ferme ensuite les nouvelles admissions, draine les acquisitions déjà admises, rejoint persistences et sources, puis publie le terminal. Les races terminales fortes et le timeout/abort restent réservés à `pre.009`; les compteurs/snapshots restent réservés à `pre.008`.
|
État après matérialisation : **implémenté et gate opérateur validé**. La crate possède désormais un port de persistence Store crate-private, implémenté par la façade `ksp-store-lib::Store`, et appelle exclusivement l'opération atomique transaction + observation en `RawTransactionAcquisitionMode::Normal`. Le supervisor possède un `JoinSet` privé de persistences ; il ne lit une nouvelle acquisition que si `in_flight < persistence_concurrency`, ce qui borne les écritures sans sémaphore ni file secondaire. Les succès distinguent `Inserted`, `AlreadyPresent`, `SkippedPurged` et les observations `Inserted`, `AlreadyPresent`, `NotRecorded`; `Rehydrated` ou toute combinaison impossible en mode Normal deviennent `runtime_invalid`. `store_api.raw_conflict` est réduit à `worker_raw_transaction_ingest.content_conflict`; toute autre erreur Store devient `worker_raw_transaction_ingest.store_failed` en ne conservant que le `ErrorCode` inférieur sûr. Le premier fault de cette tranche signale le stop privé, ferme ensuite les nouvelles admissions, draine les acquisitions déjà admises, rejoint persistences et sources, puis publie le terminal. Le gate communiqué le 8 septembre 2026 est vert sur `fmt`, audits, `check`, Clippy strict, 31 tests de crate et doc-tests. Aucun arbre Cargo n'a été requis car aucune dépendance/feature n'avait changé. Les races terminales fortes et le timeout/abort restent réservés à `pre.009`.
|
||||||
|
|
||||||
### `pre.008` — snapshots concrets + projection Worker API
|
### `pre.008` — snapshots concrets + projection Worker API
|
||||||
|
|
||||||
Budget cible : **15–20 min**. `watch` latest-value, compteurs checked, common projection, slow/no listener et terminal retained.
|
Budget cible : **15–20 min**. `watch` latest-value, compteurs checked, common projection, slow/no listener et terminal retained.
|
||||||
|
|
||||||
|
État après matérialisation : **implémenté, gate opérateur requis**. La crate expose `RawTransactionIngestSnapshot` et `RawTransactionIngestSnapshotSource`, ce dernier implémentant directement `ksp-worker-api::WorkerSnapshotSource`. Un unique `watch<RawTransactionIngestSnapshot>` porte la valeur latest-value concrète ; `wait_terminal()` lit ce même flux, supprimant le watch terminal spécialisé de `pre.004`. La séquence `WorkerSnapshotSequence` est donc identique pour la vue concrète et la projection commune. Les compteurs `u64` sont incrémentés par checked-add au point exact où le supervisor dequeue/canonicalise ou récolte une persistence ; l'épuisement devient `worker_raw_transaction_ingest.counter_exhausted`. Les profondeurs de queue et de persistence sont dérivées des structures bornées existantes, sans tâche de monitoring ni seconde event queue. `source_failure_total` et `backpressure_wait_total` existent dans le contrat sûr mais restent à zéro dans cette tranche source-neutral ; `pre.009` est propriétaire du fault source et de l'instrumentation de saturation. Les tests couvrent projection common/concrete, listener lent coalescé, terminal retenu, counters de pipeline réussi et Store fault projeté `Unhealthy`.
|
||||||
|
|
||||||
### `pre.009` — hardening shutdown/backpressure/fault races
|
### `pre.009` — hardening shutdown/backpressure/fault races
|
||||||
|
|
||||||
Budget cible : **15–20 min**. Drain deadline, stop/fault ordering, saturation, source failure, Store slow/failure, abort+join au timeout, no orphan tasks. Scinder immédiatement si le gate réel dépasse le budget.
|
Budget cible : **15–20 min**. Drain deadline, stop/fault ordering, saturation, source failure, Store slow/failure, abort+join au timeout, no orphan tasks. Scinder immédiatement si le gate réel dépasse le budget.
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
<!-- file: docs/validation/028-V0_3_11_RAW_TRANSACTION_INGEST_WORKER_FOUNDATION.md -->
|
<!-- file: docs/validation/028-V0_3_11_RAW_TRANSACTION_INGEST_WORKER_FOUNDATION.md -->
|
||||||
<!-- version: 10 -->
|
<!-- version: 11 -->
|
||||||
|
|
||||||
# Validation v0.3.11 — fondation runtime du Worker RawTransaction ingest
|
# Validation v0.3.11 — fondation runtime du Worker RawTransaction ingest
|
||||||
|
|
||||||
@@ -1173,3 +1173,131 @@ cargo test -p ksp-worker-raw-transaction-ingest-lib
|
|||||||
```
|
```
|
||||||
|
|
||||||
Critère de passage : persistence Normal, outcomes/idempotence/purge, concurrence bornée et terminaux conflict/Store sont verts, sans backend direct, Transport ni snapshot concret.
|
Critère de passage : persistence Normal, outcomes/idempotence/purge, concurrence bornée et terminaux conflict/Store sont verts, sans backend direct, Transport ni snapshot concret.
|
||||||
|
|
||||||
|
## 19. Fermeture opérateur `pre.007` et matérialisation `pre.008`
|
||||||
|
|
||||||
|
### 19.1 Fermeture opérateur de `pre.007`
|
||||||
|
|
||||||
|
Le journal opérateur communiqué le 8 septembre 2026 ferme `0.3.11-pre.007` (`workspace.package.version = 0.3.11-pre.7`) :
|
||||||
|
|
||||||
|
```text
|
||||||
|
cargo fmt --all : terminé sans erreur
|
||||||
|
python3 scripts/audit_rust_workspace_rules.py : clean, export completeness 0
|
||||||
|
python3 scripts/audit_markdown_tables.py ... : clean (340 tables, 783 files)
|
||||||
|
cargo check --workspace : terminé sans erreur
|
||||||
|
cargo clippy --workspace --all-targets --all-features -- -D warnings : terminé sans erreur
|
||||||
|
cargo test -p ksp-worker-raw-transaction-ingest-lib : 23 unit + 3 dependency-boundary + 5 public API PASS, 0 échec
|
||||||
|
doc-tests : PASS
|
||||||
|
```
|
||||||
|
|
||||||
|
Aucune dépendance ni feature n'avait changé dans `pre.007`; aucun `cargo tree` intermédiaire n'était requis. `pre.008` peut donc être ouverte.
|
||||||
|
|
||||||
|
### 19.2 Snapshot concret latest-value unique
|
||||||
|
|
||||||
|
`pre.008` introduit `RawTransactionIngestSnapshot` avec uniquement des données sûres : projection commune Worker, capacités/profondeurs bornées et compteurs monotones. Un seul :
|
||||||
|
|
||||||
|
```text
|
||||||
|
tokio::sync::watch::channel<RawTransactionIngestSnapshot>
|
||||||
|
```
|
||||||
|
|
||||||
|
porte la valeur latest-value concrète. Le handle ne conserve plus de watch terminal séparé : `wait_terminal()`, `snapshot_source()` et `worker_snapshot_source()` lisent le même flux.
|
||||||
|
|
||||||
|
Le source concret implémente directement `ksp_worker_api::WorkerSnapshotSource`; la même `WorkerSnapshotSequence` pilote donc les deux vues sans adaptation de sequence ou deuxième queue d'événements.
|
||||||
|
|
||||||
|
### 19.3 Compteurs checked et profondeurs runtime
|
||||||
|
|
||||||
|
Les compteurs de `RawTransactionIngestSnapshot` sont des `u64` monotones :
|
||||||
|
|
||||||
|
```text
|
||||||
|
admitted_total
|
||||||
|
canonicalized_total
|
||||||
|
persisted_total
|
||||||
|
entity_inserted_total
|
||||||
|
entity_already_present_total
|
||||||
|
entity_skipped_purged_total
|
||||||
|
observation_inserted_total
|
||||||
|
observation_already_present_total
|
||||||
|
content_conflict_total
|
||||||
|
store_failure_total
|
||||||
|
source_failure_total
|
||||||
|
backpressure_wait_total
|
||||||
|
```
|
||||||
|
|
||||||
|
Chaque incrément effectivement utilisé dans `pre.008` passe par `checked_add(1)`. L'épuisement ne wrappe jamais et devient :
|
||||||
|
|
||||||
|
```text
|
||||||
|
worker_raw_transaction_ingest.counter_exhausted
|
||||||
|
```
|
||||||
|
|
||||||
|
`admission_queue_depth` est lu directement depuis le receiver `mpsc` borné et `in_flight_persistence` depuis le `JoinSet` privé de persistences. Aucun polling, aucune tâche métrique et aucune queue de télémétrie ne sont ajoutés.
|
||||||
|
|
||||||
|
`source_failure_total` et `backpressure_wait_total` restent à zéro dans cette tranche source-neutral. Leur instrumentation comportementale appartient à `pre.009`, qui possède le fault source, la saturation et les races shutdown/fault.
|
||||||
|
|
||||||
|
### 19.4 Mapping Worker API
|
||||||
|
|
||||||
|
Mapping conservé :
|
||||||
|
|
||||||
|
```text
|
||||||
|
Starting -> health Unknown / activity Unknown si vide
|
||||||
|
Running sans fault -> health Healthy
|
||||||
|
Stopping -> conserve le dernier health non terminal
|
||||||
|
Faulted -> health Unhealthy
|
||||||
|
queue > 0 ou persistence > 0 -> activity Active
|
||||||
|
Running/Stopping terminalement vide -> activity Idle
|
||||||
|
```
|
||||||
|
|
||||||
|
Le terminal n'est publié qu'après le drain/join déjà possédé par le supervisor. La valeur terminale reste lisible après drop du publisher grâce à la sémantique latest-value de `watch`.
|
||||||
|
|
||||||
|
### 19.5 Preuves ajoutées
|
||||||
|
|
||||||
|
Les tests `pre.008` couvrent :
|
||||||
|
|
||||||
|
```text
|
||||||
|
snapshot initial Starting exact
|
||||||
|
projection common == snapshot concret pour la même sequence
|
||||||
|
compteurs/capacités initialement exacts
|
||||||
|
checked counter exhaustion -> counter_exhausted sans wrap
|
||||||
|
listener lent -> coalescence sur la dernière sequence
|
||||||
|
terminal concret retenu après fermeture du publisher
|
||||||
|
handle.snapshot_source() public
|
||||||
|
handle.worker_snapshot_source() public et WorkerSnapshotSource Send + Sync
|
||||||
|
pipeline runtime réussi -> admitted/canonicalized/persisted/outcomes exacts
|
||||||
|
Store failure terminal -> snapshot Unhealthy + store_failure_total
|
||||||
|
absence de nouveau dependency edge
|
||||||
|
un seul watch snapshot concret, aucun backend direct ni Transport
|
||||||
|
```
|
||||||
|
|
||||||
|
### 19.6 Frontière volontaire de `pre.008`
|
||||||
|
|
||||||
|
La tranche n'introduit toujours aucun :
|
||||||
|
|
||||||
|
```text
|
||||||
|
source live
|
||||||
|
Transport
|
||||||
|
Config
|
||||||
|
backend Store direct
|
||||||
|
source_failed public
|
||||||
|
source failure behavior
|
||||||
|
backpressure wait instrumentation productive
|
||||||
|
drain timeout / abort forcé
|
||||||
|
fault/stop precedence finalisée
|
||||||
|
retry / reconnect / gap repair
|
||||||
|
```
|
||||||
|
|
||||||
|
`pre.009` reste propriétaire du hardening shutdown/backpressure/fault races.
|
||||||
|
|
||||||
|
### 19.7 Gate opérateur demandé pour `pre.008`
|
||||||
|
|
||||||
|
Aucune dépendance ni feature n'est modifiée par `pre.008`; aucun `cargo tree` intermédiaire n'est requis. Le gate demandé est :
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cargo fmt --all
|
||||||
|
python3 scripts/audit_rust_workspace_rules.py
|
||||||
|
python3 scripts/audit_markdown_tables.py README.md RULES.md ROADMAP.md CHANGELOG.md docs prompts crates deltas
|
||||||
|
cargo check --workspace
|
||||||
|
cargo clippy --workspace --all-targets --all-features -- -D warnings
|
||||||
|
cargo test -p ksp-worker-raw-transaction-ingest-lib
|
||||||
|
```
|
||||||
|
|
||||||
|
Critère de passage : snapshots common/concrete latest-value, compteurs checked, terminal retenu et absence de nouveau boundary crossing sont verts.
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user