Sasedev/khadhroony-solana-project
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

v0.3.11-pre.008

Browse Source
This commit is contained in:
SinuS von SifriduS
2026-09-08 12:28:51 +02:00
parent ed6c0ac10c
commit 22c88c449d
13 changed files with 1241 additions and 88 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
Cargo.toml
View File

@@ -6,7 +6,7 @@ resolver = "3"
members = ["crates/ksp-app-backfill-desk", "crates/ksp-app-config-desk", "crates/ksp-app-solprices-desk", "crates/ksp-app-store-desk", "crates/ksp-app-wallet-desk", "crates/ksp-config-lib", "crates/ksp-core-lib", "crates/ksp-interface-lib", "crates/ksp-job-api", "crates/ksp-job-backfill-lib", "crates/ksp-logging-lib", "crates/ksp-offchain-transport-lib", "crates/ksp-onchain-transport-lib", "crates/ksp-program-api", "crates/ksp-raw-transaction-lib", "crates/ksp-store-api", "crates/ksp-store-lib", "crates/ksp-store-postgres-lib", "crates/ksp-wallet-lib", "crates/ksp-worker-api", "crates/ksp-worker-raw-transaction-ingest-lib"]
[workspace.package]
version = "0.3.11-pre.7"
version = "0.3.11-pre.8"
edition = "2024"
license = "MIT"
repository = "https://git.sasedev.com/Sasedev/khadhroony-solana-project"

8
crates/ksp-worker-raw-transaction-ingest-lib/src/admission.rs
View File

@@ -1,5 +1,5 @@
// file: crates/ksp-worker-raw-transaction-ingest-lib/src/admission.rs
// version: 1
// version: 2
use sha2::Digest; // rust-rules: trait-import
@@ -30,6 +30,12 @@ impl crate::RawTransactionAdmission {
return (Self { receiver }, sender);
}
/// Returns the latest receiver-side queue depth without consuming an ingress entry.
#[must_use]
pub(crate) fn queue_depth(&self) -> usize {
return self.receiver.len();
}
/// Closes new admissions while preserving already queued ingress for deterministic drain.
pub(crate) fn close(&mut self) {
self.receiver.close();

11
crates/ksp-worker-raw-transaction-ingest-lib/src/error.rs
View File

@@ -1,9 +1,12 @@
// file: crates/ksp-worker-raw-transaction-ingest-lib/src/error.rs
// version: 4
// version: 5
/// Error code used when durable Store content conflicts with one admitted canonical RAW transaction.
pub const ERROR_CODE_RAW_TRANSACTION_INGEST_CONTENT_CONFLICT: ksp_core_lib::ErrorCode =
ksp_core_lib::ErrorCode::new("worker_raw_transaction_ingest", "content_conflict");
/// Error code used when one monotone RAW transaction ingest Worker counter or snapshot sequence cannot advance without wrapping.
pub const ERROR_CODE_RAW_TRANSACTION_INGEST_COUNTER_EXHAUSTED: ksp_core_lib::ErrorCode =
ksp_core_lib::ErrorCode::new("worker_raw_transaction_ingest", "counter_exhausted");
/// Error code used when the RAW transaction ingest Worker reaches an invalid runtime or lifecycle condition.
pub const ERROR_CODE_RAW_TRANSACTION_INGEST_RUNTIME_INVALID: ksp_core_lib::ErrorCode =
ksp_core_lib::ErrorCode::new("worker_raw_transaction_ingest", "runtime_invalid");
@@ -14,6 +17,12 @@ pub const ERROR_CODE_RAW_TRANSACTION_INGEST_SETTINGS_INVALID: ksp_core_lib::Erro
pub const ERROR_CODE_RAW_TRANSACTION_INGEST_STORE_FAILED: ksp_core_lib::ErrorCode =
ksp_core_lib::ErrorCode::new("worker_raw_transaction_ingest", "store_failed");
/// Creates one terminal counter-exhaustion error without exposing runtime material.
pub(crate) fn counter_exhausted_error(field: &'static str) -> ksp_core_lib::Error {
return ksp_core_lib::Error::new(crate::ERROR_CODE_RAW_TRANSACTION_INGEST_COUNTER_EXHAUSTED, "RAW transaction ingest Worker counter exhausted")
.with_context("field", field);
}
/// Creates one terminal content-conflict error without copying conflicting material into diagnostics.
pub(crate) fn content_conflict_error() -> ksp_core_lib::Error {
return ksp_core_lib::Error::new(crate::ERROR_CODE_RAW_TRANSACTION_INGEST_CONTENT_CONFLICT, "RAW transaction ingest Store content conflict");

19
crates/ksp-worker-raw-transaction-ingest-lib/src/lib.rs
View File

@@ -1,5 +1,5 @@
// file: crates/ksp-worker-raw-transaction-ingest-lib/src/lib.rs
// version: 6
// version: 7
#![warn(missing_docs)]
#![deny(unreachable_pub)]
@@ -10,8 +10,8 @@
//! This tranche owns the concrete Worker family identity, validated technical settings
//! and the caller-runtime-owned lifecycle with private child-task supervision. This tranche also
//! owns bounded source-neutral admission, common RAW canonicalization/assembly and backend-neutral
//! Store persistence in normal mode; latest-value snapshots remain in a later prerelease, and no live
//! source or Transport dependency exists.
//! Store persistence in normal mode plus concrete latest-value snapshots projected onto Worker API;
//! no live source or Transport dependency exists.
mod admission;
mod error;
@@ -19,9 +19,12 @@ mod identity;
mod persistence;
mod runtime;
mod settings;
mod snapshot;
/// Error code used when durable Store content conflicts with one admitted canonical RAW transaction.
pub use self::error::ERROR_CODE_RAW_TRANSACTION_INGEST_CONTENT_CONFLICT;
/// Error code used when one monotone Worker counter or snapshot sequence cannot advance without wrapping.
pub use self::error::ERROR_CODE_RAW_TRANSACTION_INGEST_COUNTER_EXHAUSTED;
/// Error code used when the RAW transaction ingest Worker reaches an invalid runtime or lifecycle condition.
pub use self::error::ERROR_CODE_RAW_TRANSACTION_INGEST_RUNTIME_INVALID;
/// Error code used when RAW transaction ingest Worker settings violate one bounded runtime invariant.
@@ -56,6 +59,12 @@ pub use self::settings::MIN_RAW_TRANSACTION_INGEST_PERSISTENCE_CONCURRENCY;
pub use self::settings::MIN_RAW_TRANSACTION_INGEST_SHUTDOWN_DRAIN_TIMEOUT;
/// Validated source-neutral runtime settings for one continuous RAW transaction ingest Worker.
pub use self::settings::RawTransactionIngestSettings;
/// Complete safe latest-value snapshot of one continuous RAW transaction ingest Worker.
pub use self::snapshot::RawTransactionIngestSnapshot;
/// Runtime-neutral boxed future resolving to one newer concrete RAW transaction ingest Worker snapshot.
pub use self::snapshot::RawTransactionIngestSnapshotFuture;
/// Cloneable latest-value source exposing concrete and common Worker snapshots from one shared watch state.
pub use self::snapshot::RawTransactionIngestSnapshotSource;
/// Receiver-side owner of the private bounded RAW transaction admission queue.
pub(crate) use self::admission::RawTransactionAdmission;
@@ -63,6 +72,8 @@ pub(crate) use self::admission::RawTransactionAdmission;
pub(crate) use self::admission::RawTransactionIngress;
/// Creates one terminal content-conflict error without copying conflicting material into diagnostics.
pub(crate) use self::error::content_conflict_error;
/// Creates one terminal counter-exhaustion error without exposing runtime material.
pub(crate) use self::error::counter_exhausted_error;
/// Creates one runtime-domain error without copying runtime/provider/Store values into diagnostics.
pub(crate) use self::error::runtime_error;
/// Creates one settings-domain error without copying caller-supplied values into diagnostics.
@@ -79,3 +90,5 @@ pub(crate) use self::persistence::RawTransactionIngestPersistenceOutcome;
pub(crate) use self::persistence::RawTransactionIngestPersistencePort;
/// Persists one already-canonical Worker acquisition through the private Store port in `Normal` mode.
pub(crate) use self::persistence::persist_raw_transaction_ingest_acquisition;
/// Private latest-value publisher and checked counter owner shared by the Worker supervisor.
pub(crate) use self::snapshot::RawTransactionIngestSnapshotPublisher;

194
crates/ksp-worker-raw-transaction-ingest-lib/src/runtime.rs
View File

@@ -1,5 +1,5 @@
// file: crates/ksp-worker-raw-transaction-ingest-lib/src/runtime.rs
// version: 4
// version: 5
type PersistencePort = std::sync::Arc<dyn crate::RawTransactionIngestPersistencePort + 'static>;
type PersistenceTasks = tokio::task::JoinSet<ksp_core_lib::Result<crate::RawTransactionIngestPersistenceOutcome>>;
@@ -11,9 +11,9 @@ pub type RawTransactionIngestTerminalFuture<'a> =
/// Cloneable external control handle for one continuous RAW transaction ingest Worker.
#[derive(Clone)]
pub struct RawTransactionIngestHandle {
snapshots: crate::RawTransactionIngestSnapshotSource,
stop_sender: tokio::sync::watch::Sender<bool>,
stop_token: ksp_worker_api::WorkerStopToken,
terminal_receiver: tokio::sync::watch::Receiver<ksp_worker_api::WorkerState>,
}
impl crate::RawTransactionIngestHandle {
@@ -26,22 +26,36 @@ impl crate::RawTransactionIngestHandle {
return self.stop_sender.send(true).is_ok();
}
/// Waits until the private runtime task has published and closed one terminal lifecycle state.
/// Returns an independent latest-value source for concrete RAW transaction ingest Worker snapshots.
#[must_use]
pub fn snapshot_source(&self) -> crate::RawTransactionIngestSnapshotSource {
return self.snapshots.clone();
}
/// Returns an independent source implementing the common [`ksp_worker_api::WorkerSnapshotSource`] projection.
#[must_use]
pub fn worker_snapshot_source(&self) -> crate::RawTransactionIngestSnapshotSource {
return self.snapshots.clone();
}
/// Waits until the private runtime task has published one terminal lifecycle state after draining owned work.
#[must_use]
pub fn wait_terminal(&self) -> crate::RawTransactionIngestTerminalFuture<'_> {
let mut receiver = self.terminal_receiver.clone();
let source = self.snapshots.clone();
return std::boxed::Box::pin(async move {
loop {
let current = *receiver.borrow();
if current.is_terminal() {
let changed = receiver.changed().await;
if changed.is_err() {
return std::result::Result::Ok(current);
}
return std::result::Result::Err(crate::runtime_error("terminal.changed_after_terminal"));
let current = source.current();
let state = current.worker_snapshot().state();
if state.is_terminal() {
return std::result::Result::Ok(state);
}
let changed = receiver.changed().await;
if changed.is_err() {
let observed = current.worker_snapshot().sequence();
let changed = source.wait_for_change(observed).await;
let state = changed.worker_snapshot().state();
if state.is_terminal() {
return std::result::Result::Ok(state);
}
if changed.worker_snapshot().sequence() == observed && source.is_closed() {
return std::result::Result::Err(crate::runtime_error("terminal.closed_before_terminal"));
}
}
@@ -51,11 +65,12 @@ impl crate::RawTransactionIngestHandle {
impl std::fmt::Debug for crate::RawTransactionIngestHandle {
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
let state = *self.terminal_receiver.borrow();
let snapshot = self.snapshots.current();
return formatter
.debug_struct("RawTransactionIngestHandle")
.field("stop_requested", &self.stop_token.is_stop_requested())
.field("state", &state)
.field("sequence", &snapshot.worker_snapshot().sequence())
.field("state", &snapshot.worker_snapshot().state())
.finish();
}
}
@@ -81,6 +96,21 @@ impl crate::RawTransactionIngestWorker {
}
}
fn begin_stopping(
lifecycle: &mut ksp_worker_api::WorkerLifecycle,
snapshots: &mut crate::RawTransactionIngestSnapshotPublisher,
admission_queue_depth: usize,
in_flight_persistence: usize,
) -> std::option::Option<ksp_core_lib::ErrorCode> {
if lifecycle.mark_stopping().is_err() {
return std::option::Option::Some(crate::ERROR_CODE_RAW_TRANSACTION_INGEST_RUNTIME_INVALID);
}
if let std::result::Result::Err(error) = snapshots.publish_state(lifecycle.state(), admission_queue_depth, in_flight_persistence) {
return std::option::Option::Some(error.code());
}
return std::option::Option::None;
}
fn current_runtime_handle() -> ksp_core_lib::Result<tokio::runtime::Handle> {
return match tokio::runtime::Handle::try_current() {
std::result::Result::Ok(value) => std::result::Result::Ok(value),
@@ -98,10 +128,15 @@ async fn drain_children(children: &mut tokio::task::JoinSet<()>) -> std::option:
return fault;
}
async fn drain_persistence(persistence: &mut PersistenceTasks) -> std::option::Option<ksp_core_lib::ErrorCode> {
async fn drain_persistence(
lifecycle: &ksp_worker_api::WorkerLifecycle,
admission: &crate::RawTransactionAdmission,
persistence: &mut PersistenceTasks,
snapshots: &mut crate::RawTransactionIngestSnapshotPublisher,
) -> std::option::Option<ksp_core_lib::ErrorCode> {
let mut fault = std::option::Option::None;
while let std::option::Option::Some(joined) = persistence.join_next().await {
let current = persistence_fault(joined);
let current = persistence_completion(joined, lifecycle.state(), admission.queue_depth(), persistence.len(), snapshots);
if fault.is_none() {
fault = current;
}
@@ -111,9 +146,11 @@ async fn drain_persistence(persistence: &mut PersistenceTasks) -> std::option::O
async fn drain_admission_and_persistence(
settings: &crate::RawTransactionIngestSettings,
lifecycle: &ksp_worker_api::WorkerLifecycle,
admission: &mut crate::RawTransactionAdmission,
persistence: &mut PersistenceTasks,
port: &std::option::Option<PersistencePort>,
snapshots: &mut crate::RawTransactionIngestSnapshotPublisher,
) -> std::option::Option<ksp_core_lib::ErrorCode> {
let mut fault = std::option::Option::None;
admission.close();
@@ -121,7 +158,7 @@ async fn drain_admission_and_persistence(
while persistence.len() >= settings.persistence_concurrency() {
let joined = persistence.join_next().await;
let current = match joined {
std::option::Option::Some(value) => persistence_fault(value),
std::option::Option::Some(value) => persistence_completion(value, lifecycle.state(), admission.queue_depth(), persistence.len(), snapshots),
std::option::Option::None => std::option::Option::Some(crate::ERROR_CODE_RAW_TRANSACTION_INGEST_RUNTIME_INVALID),
};
if fault.is_none() {
@@ -131,19 +168,31 @@ async fn drain_admission_and_persistence(
let received = admission.receive(settings.network()).await;
match received {
std::result::Result::Ok(std::option::Option::Some(acquisition)) => {
if !spawn_persistence(persistence, port, acquisition) && fault.is_none() {
let spawned = spawn_persistence(persistence, port, acquisition);
let published = snapshots.record_admission_success(lifecycle.state(), admission.queue_depth(), persistence.len());
if !spawned && fault.is_none() {
fault = std::option::Option::Some(crate::ERROR_CODE_RAW_TRANSACTION_INGEST_RUNTIME_INVALID);
}
if let std::result::Result::Err(error) = published {
if fault.is_none() {
fault = std::option::Option::Some(error.code());
}
}
},
std::result::Result::Ok(std::option::Option::None) => break,
std::result::Result::Err(_) => {
if fault.is_none() {
let published = snapshots.record_admission_failure(lifecycle.state(), admission.queue_depth(), persistence.len());
if let std::result::Result::Err(error) = published {
if fault.is_none() {
fault = std::option::Option::Some(error.code());
}
} else if fault.is_none() {
fault = std::option::Option::Some(crate::ERROR_CODE_RAW_TRANSACTION_INGEST_RUNTIME_INVALID);
}
},
}
}
let persistence_fault = drain_persistence(persistence).await;
let persistence_fault = drain_persistence(lifecycle, admission, persistence, snapshots).await;
if fault.is_none() {
fault = persistence_fault;
}
@@ -152,28 +201,27 @@ async fn drain_admission_and_persistence(
fn finish_faulted(
lifecycle: &mut ksp_worker_api::WorkerLifecycle,
sender: &tokio::sync::watch::Sender<ksp_worker_api::WorkerState>,
snapshots: &mut crate::RawTransactionIngestSnapshotPublisher,
code: ksp_core_lib::ErrorCode,
) {
if lifecycle.fault(code).is_err() {
sender.send_replace(ksp_worker_api::WorkerState::Faulted(crate::ERROR_CODE_RAW_TRANSACTION_INGEST_RUNTIME_INVALID));
snapshots.force_terminal(ksp_worker_api::WorkerState::Faulted(crate::ERROR_CODE_RAW_TRANSACTION_INGEST_RUNTIME_INVALID));
return;
}
sender.send_replace(lifecycle.state());
if snapshots.publish_state(lifecycle.state(), 0, 0).is_err() {
snapshots.force_terminal(ksp_worker_api::WorkerState::Faulted(crate::ERROR_CODE_RAW_TRANSACTION_INGEST_COUNTER_EXHAUSTED));
}
return;
}
fn finish_stopped(lifecycle: &mut ksp_worker_api::WorkerLifecycle, sender: &tokio::sync::watch::Sender<ksp_worker_api::WorkerState>) {
if lifecycle.mark_stopping().is_err() {
sender.send_replace(ksp_worker_api::WorkerState::Faulted(crate::ERROR_CODE_RAW_TRANSACTION_INGEST_RUNTIME_INVALID));
return;
}
sender.send_replace(lifecycle.state());
fn finish_stopped(lifecycle: &mut ksp_worker_api::WorkerLifecycle, snapshots: &mut crate::RawTransactionIngestSnapshotPublisher) {
if lifecycle.mark_stopped().is_err() {
sender.send_replace(ksp_worker_api::WorkerState::Faulted(crate::ERROR_CODE_RAW_TRANSACTION_INGEST_RUNTIME_INVALID));
snapshots.force_terminal(ksp_worker_api::WorkerState::Faulted(crate::ERROR_CODE_RAW_TRANSACTION_INGEST_RUNTIME_INVALID));
return;
}
sender.send_replace(lifecycle.state());
if snapshots.publish_state(lifecycle.state(), 0, 0).is_err() {
snapshots.force_terminal(ksp_worker_api::WorkerState::Faulted(crate::ERROR_CODE_RAW_TRANSACTION_INGEST_COUNTER_EXHAUSTED));
}
return;
}
@@ -187,16 +235,28 @@ fn merge_fault(
return next;
}
fn persistence_fault(
fn persistence_completion(
joined: std::result::Result<ksp_core_lib::Result<crate::RawTransactionIngestPersistenceOutcome>, tokio::task::JoinError>,
state: ksp_worker_api::WorkerState,
admission_queue_depth: usize,
in_flight_persistence: usize,
snapshots: &mut crate::RawTransactionIngestSnapshotPublisher,
) -> std::option::Option<ksp_core_lib::ErrorCode> {
return match joined {
std::result::Result::Ok(std::result::Result::Ok(outcome)) => {
let _entity = outcome.entity();
let _observation = outcome.observation();
std::option::Option::None
match snapshots.record_persistence_success(state, admission_queue_depth, in_flight_persistence, outcome) {
std::result::Result::Ok(()) => std::option::Option::None,
std::result::Result::Err(error) => std::option::Option::Some(error.code()),
}
},
std::result::Result::Ok(std::result::Result::Err(error)) => {
let code = error.code();
let published = snapshots.record_persistence_fault(state, admission_queue_depth, in_flight_persistence, code);
match published {
std::result::Result::Ok(()) => std::option::Option::Some(code),
std::result::Result::Err(snapshot_error) => std::option::Option::Some(snapshot_error.code()),
}
},
std::result::Result::Ok(std::result::Result::Err(error)) => std::option::Option::Some(error.code()),
std::result::Result::Err(_) => std::option::Option::Some(crate::ERROR_CODE_RAW_TRANSACTION_INGEST_RUNTIME_INVALID),
};
}
@@ -206,7 +266,7 @@ async fn run_supervisor<Spawner>(
mut lifecycle: ksp_worker_api::WorkerLifecycle,
port: std::option::Option<PersistencePort>,
mut stop_receiver: tokio::sync::watch::Receiver<bool>,
terminal_sender: tokio::sync::watch::Sender<ksp_worker_api::WorkerState>,
mut snapshots: crate::RawTransactionIngestSnapshotPublisher,
source_spawner: Spawner,
) where
Spawner: FnOnce(&mut tokio::task::JoinSet<()>, tokio::sync::watch::Receiver<bool>, tokio::sync::mpsc::Sender<crate::RawTransactionIngress>)
@@ -214,30 +274,50 @@ async fn run_supervisor<Spawner>(
+ 'static,
{
if *stop_receiver.borrow() {
finish_stopped(&mut lifecycle, &terminal_sender);
let stopping_fault = begin_stopping(&mut lifecycle, &mut snapshots, 0, 0);
match stopping_fault {
std::option::Option::Some(code) => finish_faulted(&mut lifecycle, &mut snapshots, code),
std::option::Option::None => finish_stopped(&mut lifecycle, &mut snapshots),
}
return;
}
if lifecycle.mark_running().is_err() {
terminal_sender.send_replace(ksp_worker_api::WorkerState::Faulted(crate::ERROR_CODE_RAW_TRANSACTION_INGEST_RUNTIME_INVALID));
snapshots.force_terminal(ksp_worker_api::WorkerState::Faulted(crate::ERROR_CODE_RAW_TRANSACTION_INGEST_RUNTIME_INVALID));
return;
}
if let std::result::Result::Err(error) = snapshots.publish_state(lifecycle.state(), 0, 0) {
finish_faulted(&mut lifecycle, &mut snapshots, error.code());
return;
}
terminal_sender.send_replace(lifecycle.state());
let mut children = tokio::task::JoinSet::new();
let mut persistence = PersistenceTasks::new();
let (source_stop_sender, source_stop_receiver) = tokio::sync::watch::channel(false);
let (mut admission, admission_sender) = crate::RawTransactionAdmission::new(settings.admission_queue_capacity());
source_spawner(&mut children, source_stop_receiver, admission_sender);
let mut fault = supervise_until_stop(&settings, &source_stop_sender, &mut stop_receiver, &mut children, &mut admission, &mut persistence, &port).await;
let mut fault = supervise_until_stop(
&settings,
&lifecycle,
&source_stop_sender,
&mut stop_receiver,
&mut children,
&mut admission,
&mut persistence,
&port,
&mut snapshots,
)
.await;
if fault.is_some() {
source_stop_sender.send_replace(true);
}
let drain_fault = drain_admission_and_persistence(&settings, &mut admission, &mut persistence, &port).await;
let stopping_fault = begin_stopping(&mut lifecycle, &mut snapshots, admission.queue_depth(), persistence.len());
fault = merge_fault(fault, stopping_fault);
let drain_fault = drain_admission_and_persistence(&settings, &lifecycle, &mut admission, &mut persistence, &port, &mut snapshots).await;
fault = merge_fault(fault, drain_fault);
let child_fault = drain_children(&mut children).await;
fault = merge_fault(fault, child_fault);
match fault {
std::option::Option::Some(code) => finish_faulted(&mut lifecycle, &terminal_sender, code),
std::option::Option::None => finish_stopped(&mut lifecycle, &terminal_sender),
std::option::Option::Some(code) => finish_faulted(&mut lifecycle, &mut snapshots, code),
std::option::Option::None => finish_stopped(&mut lifecycle, &mut snapshots),
}
return;
}
@@ -286,9 +366,9 @@ where
}
let stop_token = ksp_worker_api::WorkerStopToken::new();
let (stop_sender, stop_receiver) = tokio::sync::watch::channel(false);
let (terminal_sender, terminal_receiver) = tokio::sync::watch::channel(lifecycle.state());
let handle = crate::RawTransactionIngestHandle { stop_sender, stop_token, terminal_receiver };
std::mem::drop(runtime.spawn(run_supervisor(settings, lifecycle, port, stop_receiver, terminal_sender, source_spawner)));
let (snapshots, snapshot_source) = crate::RawTransactionIngestSnapshotPublisher::new(&settings, &lifecycle);
let handle = crate::RawTransactionIngestHandle { snapshots: snapshot_source, stop_sender, stop_token };
std::mem::drop(runtime.spawn(run_supervisor(settings, lifecycle, port, stop_receiver, snapshots, source_spawner)));
return std::result::Result::Ok(handle);
}
@@ -315,12 +395,14 @@ where
async fn supervise_until_stop(
settings: &crate::RawTransactionIngestSettings,
lifecycle: &ksp_worker_api::WorkerLifecycle,
source_stop_sender: &tokio::sync::watch::Sender<bool>,
stop_receiver: &mut tokio::sync::watch::Receiver<bool>,
children: &mut tokio::task::JoinSet<()>,
admission: &mut crate::RawTransactionAdmission,
persistence: &mut PersistenceTasks,
port: &std::option::Option<PersistencePort>,
snapshots: &mut crate::RawTransactionIngestSnapshotPublisher,
) -> std::option::Option<ksp_core_lib::ErrorCode> {
let mut admission_open = true;
loop {
@@ -352,7 +434,7 @@ async fn supervise_until_stop(
}
joined = persistence.join_next(), if !persistence.is_empty() => {
if let std::option::Option::Some(value) = joined {
let fault = persistence_fault(value);
let fault = persistence_completion(value, lifecycle.state(), admission.queue_depth(), persistence.len(), snapshots);
if fault.is_some() {
source_stop_sender.send_replace(true);
return fault;
@@ -362,15 +444,25 @@ async fn supervise_until_stop(
received = admission.receive(settings.network()), if admission_open && persistence.len() < settings.persistence_concurrency() => {
match received {
std::result::Result::Ok(std::option::Option::Some(acquisition)) => {
if !spawn_persistence(persistence, port, acquisition) {
let spawned = spawn_persistence(persistence, port, acquisition);
let published = snapshots.record_admission_success(lifecycle.state(), admission.queue_depth(), persistence.len());
if !spawned {
source_stop_sender.send_replace(true);
return std::option::Option::Some(crate::ERROR_CODE_RAW_TRANSACTION_INGEST_RUNTIME_INVALID);
}
if let std::result::Result::Err(error) = published {
source_stop_sender.send_replace(true);
return std::option::Option::Some(error.code());
}
},
std::result::Result::Ok(std::option::Option::None) => admission_open = false,
std::result::Result::Err(_) => {
let published = snapshots.record_admission_failure(lifecycle.state(), admission.queue_depth(), persistence.len());
source_stop_sender.send_replace(true);
return std::option::Option::Some(crate::ERROR_CODE_RAW_TRANSACTION_INGEST_RUNTIME_INVALID);
return match published {
std::result::Result::Ok(()) => std::option::Option::Some(crate::ERROR_CODE_RAW_TRANSACTION_INGEST_RUNTIME_INVALID),
std::result::Result::Err(error) => std::option::Option::Some(error.code()),
};
},
}
}

472
crates/ksp-worker-raw-transaction-ingest-lib/src/snapshot.rs Normal file
View File

@@ -0,0 +1,472 @@
// file: crates/ksp-worker-raw-transaction-ingest-lib/src/snapshot.rs
// version: 1
/// Runtime-neutral boxed future resolving to one newer concrete RAW transaction ingest Worker snapshot.
pub type RawTransactionIngestSnapshotFuture<'a> =
std::pin::Pin<std::boxed::Box<dyn std::future::Future<Output = crate::RawTransactionIngestSnapshot> + std::marker::Send + 'a>>;
/// Complete safe latest-value snapshot of one continuous RAW transaction ingest Worker.
#[derive(Clone, Eq, PartialEq)]
pub struct RawTransactionIngestSnapshot {
worker: ksp_worker_api::WorkerSnapshot,
admission_queue_capacity: usize,
admission_queue_depth: usize,
persistence_concurrency: usize,
in_flight_persistence: usize,
admitted_total: u64,
canonicalized_total: u64,
persisted_total: u64,
entity_inserted_total: u64,
entity_already_present_total: u64,
entity_skipped_purged_total: u64,
observation_inserted_total: u64,
observation_already_present_total: u64,
content_conflict_total: u64,
store_failure_total: u64,
source_failure_total: u64,
backpressure_wait_total: u64,
}
impl crate::RawTransactionIngestSnapshot {
/// Returns the common Worker projection carried by this concrete snapshot.
#[must_use]
pub const fn worker_snapshot(&self) -> &ksp_worker_api::WorkerSnapshot {
return &self.worker;
}
/// Returns the configured bounded admission queue capacity.
#[must_use]
pub const fn admission_queue_capacity(&self) -> usize {
return self.admission_queue_capacity;
}
/// Returns the latest observed number of queued ingress entries awaiting supervisor admission.
#[must_use]
pub const fn admission_queue_depth(&self) -> usize {
return self.admission_queue_depth;
}
/// Returns the configured maximum number of concurrent Store persistence operations.
#[must_use]
pub const fn persistence_concurrency(&self) -> usize {
return self.persistence_concurrency;
}
/// Returns the latest observed number of in-flight Store persistence operations.
#[must_use]
pub const fn in_flight_persistence(&self) -> usize {
return self.in_flight_persistence;
}
/// Returns the number of ingress entries removed from the bounded admission queue.
#[must_use]
pub const fn admitted_total(&self) -> u64 {
return self.admitted_total;
}
/// Returns the number of admitted ingress entries successfully canonicalized through the Common RAW contract.
#[must_use]
pub const fn canonicalized_total(&self) -> u64 {
return self.canonicalized_total;
}
/// Returns the number of successful atomic Store persistence outcomes.
#[must_use]
pub const fn persisted_total(&self) -> u64 {
return self.persisted_total;
}
/// Returns the number of newly inserted canonical RAW transaction entities.
#[must_use]
pub const fn entity_inserted_total(&self) -> u64 {
return self.entity_inserted_total;
}
/// Returns the number of identical canonical RAW transaction entities already durable.
#[must_use]
pub const fn entity_already_present_total(&self) -> u64 {
return self.entity_already_present_total;
}
/// Returns the number of durable purge tombstones respected by normal persistence.
#[must_use]
pub const fn entity_skipped_purged_total(&self) -> u64 {
return self.entity_skipped_purged_total;
}
/// Returns the number of newly inserted deterministic acquisition observations.
#[must_use]
pub const fn observation_inserted_total(&self) -> u64 {
return self.observation_inserted_total;
}
/// Returns the number of deterministic acquisition observations already durable.
#[must_use]
pub const fn observation_already_present_total(&self) -> u64 {
return self.observation_already_present_total;
}
/// Returns the number of durable Store content conflicts observed by this run.
#[must_use]
pub const fn content_conflict_total(&self) -> u64 {
return self.content_conflict_total;
}
/// Returns the number of non-conflict Store persistence failures observed by this run.
#[must_use]
pub const fn store_failure_total(&self) -> u64 {
return self.store_failure_total;
}
/// Returns the number of source-task failures observed by this run.
#[must_use]
pub const fn source_failure_total(&self) -> u64 {
return self.source_failure_total;
}
/// Returns the number of source-side admission waits explicitly classified as backpressure waits.
#[must_use]
pub const fn backpressure_wait_total(&self) -> u64 {
return self.backpressure_wait_total;
}
}
impl std::fmt::Debug for crate::RawTransactionIngestSnapshot {
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
return formatter
.debug_struct("RawTransactionIngestSnapshot")
.field("worker", &self.worker)
.field("admission_queue_capacity", &self.admission_queue_capacity)
.field("admission_queue_depth", &self.admission_queue_depth)
.field("persistence_concurrency", &self.persistence_concurrency)
.field("in_flight_persistence", &self.in_flight_persistence)
.field("admitted_total", &self.admitted_total)
.field("canonicalized_total", &self.canonicalized_total)
.field("persisted_total", &self.persisted_total)
.field("entity_inserted_total", &self.entity_inserted_total)
.field("entity_already_present_total", &self.entity_already_present_total)
.field("entity_skipped_purged_total", &self.entity_skipped_purged_total)
.field("observation_inserted_total", &self.observation_inserted_total)
.field("observation_already_present_total", &self.observation_already_present_total)
.field("content_conflict_total", &self.content_conflict_total)
.field("store_failure_total", &self.store_failure_total)
.field("source_failure_total", &self.source_failure_total)
.field("backpressure_wait_total", &self.backpressure_wait_total)
.finish();
}
}
/// Cloneable latest-value source exposing concrete and common Worker snapshots from one shared watch state.
#[derive(Clone)]
pub struct RawTransactionIngestSnapshotSource {
receiver: tokio::sync::watch::Receiver<crate::RawTransactionIngestSnapshot>,
}
impl crate::RawTransactionIngestSnapshotSource {
/// Returns the complete current concrete snapshot without replaying intermediate updates.
#[must_use]
pub fn current(&self) -> crate::RawTransactionIngestSnapshot {
return self.receiver.borrow().clone();
}
/// Reports whether the private publisher has closed after the runtime task returned.
#[must_use]
pub(crate) fn is_closed(&self) -> bool {
return self.receiver.has_changed().is_err();
}
/// Waits for one concrete snapshot newer than `observed`, coalescing intermediate updates to the latest value.
#[must_use]
pub fn wait_for_change(&self, observed: ksp_worker_api::WorkerSnapshotSequence) -> crate::RawTransactionIngestSnapshotFuture<'_> {
let mut receiver = self.receiver.clone();
return std::boxed::Box::pin(async move {
loop {
let current = receiver.borrow_and_update().clone();
if current.worker_snapshot().sequence().is_after(observed) {
return current;
}
let changed = receiver.changed().await;
if changed.is_err() {
return receiver.borrow_and_update().clone();
}
}
});
}
}
impl ksp_worker_api::WorkerSnapshotSource for crate::RawTransactionIngestSnapshotSource {
fn current(&self) -> ksp_worker_api::WorkerSnapshot {
return self.receiver.borrow().worker_snapshot().clone();
}
fn wait_for_change(&self, observed: ksp_worker_api::WorkerSnapshotSequence) -> ksp_worker_api::WorkerSnapshotFuture<'_> {
let mut receiver = self.receiver.clone();
return std::boxed::Box::pin(async move {
loop {
let current = receiver.borrow_and_update().worker_snapshot().clone();
if current.sequence().is_after(observed) {
return current;
}
let changed = receiver.changed().await;
if changed.is_err() {
return receiver.borrow_and_update().worker_snapshot().clone();
}
}
});
}
}
impl std::fmt::Debug for crate::RawTransactionIngestSnapshotSource {
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
let current = self.receiver.borrow();
return formatter
.debug_struct("RawTransactionIngestSnapshotSource")
.field("sequence", &current.worker_snapshot().sequence())
.field("state", &current.worker_snapshot().state())
.finish();
}
}
/// Private latest-value publisher and checked counter owner shared by the Worker supervisor.
pub(crate) struct RawTransactionIngestSnapshotPublisher {
sender: tokio::sync::watch::Sender<crate::RawTransactionIngestSnapshot>,
snapshot: crate::RawTransactionIngestSnapshot,
}
impl crate::RawTransactionIngestSnapshotPublisher {
/// Creates the initial `Starting` snapshot stream for one validated Worker run.
pub(crate) fn new(
settings: &crate::RawTransactionIngestSettings,
lifecycle: &ksp_worker_api::WorkerLifecycle,
) -> (crate::RawTransactionIngestSnapshotPublisher, crate::RawTransactionIngestSnapshotSource) {
let worker = ksp_worker_api::WorkerSnapshot::new(
lifecycle.id().clone(),
lifecycle.kind().clone(),
ksp_worker_api::WorkerSnapshotSequence::initial(),
lifecycle.state(),
ksp_worker_api::WorkerHealth::Unknown,
ksp_worker_api::WorkerActivity::Unknown,
);
let snapshot = crate::RawTransactionIngestSnapshot {
worker,
admission_queue_capacity: settings.admission_queue_capacity(),
admission_queue_depth: 0,
persistence_concurrency: settings.persistence_concurrency(),
in_flight_persistence: 0,
admitted_total: 0,
canonicalized_total: 0,
persisted_total: 0,
entity_inserted_total: 0,
entity_already_present_total: 0,
entity_skipped_purged_total: 0,
observation_inserted_total: 0,
observation_already_present_total: 0,
content_conflict_total: 0,
store_failure_total: 0,
source_failure_total: 0,
backpressure_wait_total: 0,
};
let (sender, receiver) = tokio::sync::watch::channel(snapshot.clone());
return (Self { sender, snapshot }, crate::RawTransactionIngestSnapshotSource { receiver });
}
/// Forces one terminal latest value without advancing the sequence, used only when sequence publication itself is exhausted or invalid.
pub(crate) fn force_terminal(&mut self, state: ksp_worker_api::WorkerState) {
let worker = ksp_worker_api::WorkerSnapshot::new(
self.snapshot.worker.id().clone(),
self.snapshot.worker.kind().clone(),
self.snapshot.worker.sequence(),
state,
health_for_state(state, self.snapshot.worker.health()),
ksp_worker_api::WorkerActivity::Idle,
);
self.snapshot.worker = worker;
self.snapshot.admission_queue_depth = 0;
self.snapshot.in_flight_persistence = 0;
self.sender.send_replace(self.snapshot.clone());
return;
}
/// Publishes one lifecycle/depth refresh with a strictly newer common sequence.
pub(crate) fn publish_state(
&mut self,
state: ksp_worker_api::WorkerState,
admission_queue_depth: usize,
in_flight_persistence: usize,
) -> ksp_core_lib::Result<()> {
return self.publish(state, admission_queue_depth, in_flight_persistence);
}
/// Records one dequeued ingress that failed canonicalization and publishes the resulting latest value.
pub(crate) fn record_admission_failure(
&mut self,
state: ksp_worker_api::WorkerState,
admission_queue_depth: usize,
in_flight_persistence: usize,
) -> ksp_core_lib::Result<()> {
let admitted_total = match checked_counter(self.snapshot.admitted_total, "admitted_total") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
self.snapshot.admitted_total = admitted_total;
return self.publish(state, admission_queue_depth, in_flight_persistence);
}
/// Records one successfully canonicalized admitted ingress and publishes the resulting latest value.
pub(crate) fn record_admission_success(
&mut self,
state: ksp_worker_api::WorkerState,
admission_queue_depth: usize,
in_flight_persistence: usize,
) -> ksp_core_lib::Result<()> {
let admitted_total = match checked_counter(self.snapshot.admitted_total, "admitted_total") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let canonicalized_total = match checked_counter(self.snapshot.canonicalized_total, "canonicalized_total") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
self.snapshot.admitted_total = admitted_total;
self.snapshot.canonicalized_total = canonicalized_total;
return self.publish(state, admission_queue_depth, in_flight_persistence);
}
/// Records one successful Store persistence outcome and publishes the resulting latest value.
pub(crate) fn record_persistence_success(
&mut self,
state: ksp_worker_api::WorkerState,
admission_queue_depth: usize,
in_flight_persistence: usize,
outcome: crate::RawTransactionIngestPersistenceOutcome,
) -> ksp_core_lib::Result<()> {
let persisted_total = match checked_counter(self.snapshot.persisted_total, "persisted_total") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let mut entity_inserted_total = self.snapshot.entity_inserted_total;
let mut entity_already_present_total = self.snapshot.entity_already_present_total;
let mut entity_skipped_purged_total = self.snapshot.entity_skipped_purged_total;
let mut observation_inserted_total = self.snapshot.observation_inserted_total;
let mut observation_already_present_total = self.snapshot.observation_already_present_total;
match outcome.entity() {
crate::RawTransactionIngestEntityPersistence::Inserted => {
entity_inserted_total = match checked_counter(entity_inserted_total, "entity_inserted_total") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
},
crate::RawTransactionIngestEntityPersistence::AlreadyPresent => {
entity_already_present_total = match checked_counter(entity_already_present_total, "entity_already_present_total") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
},
crate::RawTransactionIngestEntityPersistence::SkippedPurged => {
entity_skipped_purged_total = match checked_counter(entity_skipped_purged_total, "entity_skipped_purged_total") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
},
}
match outcome.observation() {
crate::RawTransactionIngestObservationPersistence::Inserted => {
observation_inserted_total = match checked_counter(observation_inserted_total, "observation_inserted_total") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
},
crate::RawTransactionIngestObservationPersistence::AlreadyPresent => {
observation_already_present_total = match checked_counter(observation_already_present_total, "observation_already_present_total") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
},
crate::RawTransactionIngestObservationPersistence::NotRecorded => {},
}
self.snapshot.persisted_total = persisted_total;
self.snapshot.entity_inserted_total = entity_inserted_total;
self.snapshot.entity_already_present_total = entity_already_present_total;
self.snapshot.entity_skipped_purged_total = entity_skipped_purged_total;
self.snapshot.observation_inserted_total = observation_inserted_total;
self.snapshot.observation_already_present_total = observation_already_present_total;
return self.publish(state, admission_queue_depth, in_flight_persistence);
}
/// Records one classified persistence fault counter and publishes the resulting latest value.
pub(crate) fn record_persistence_fault(
&mut self,
state: ksp_worker_api::WorkerState,
admission_queue_depth: usize,
in_flight_persistence: usize,
code: ksp_core_lib::ErrorCode,
) -> ksp_core_lib::Result<()> {
if code == crate::ERROR_CODE_RAW_TRANSACTION_INGEST_CONTENT_CONFLICT {
let next = match checked_counter(self.snapshot.content_conflict_total, "content_conflict_total") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
self.snapshot.content_conflict_total = next;
} else if code == crate::ERROR_CODE_RAW_TRANSACTION_INGEST_STORE_FAILED {
let next = match checked_counter(self.snapshot.store_failure_total, "store_failure_total") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
self.snapshot.store_failure_total = next;
}
return self.publish(state, admission_queue_depth, in_flight_persistence);
}
fn publish(&mut self, state: ksp_worker_api::WorkerState, admission_queue_depth: usize, in_flight_persistence: usize) -> ksp_core_lib::Result<()> {
let sequence = match self.snapshot.worker.sequence().next() {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return std::result::Result::Err(crate::counter_exhausted_error("snapshot_sequence")),
};
let worker = ksp_worker_api::WorkerSnapshot::new(
self.snapshot.worker.id().clone(),
self.snapshot.worker.kind().clone(),
sequence,
state,
health_for_state(state, self.snapshot.worker.health()),
activity_for_state(state, admission_queue_depth, in_flight_persistence),
);
self.snapshot.worker = worker;
self.snapshot.admission_queue_depth = admission_queue_depth;
self.snapshot.in_flight_persistence = in_flight_persistence;
self.sender.send_replace(self.snapshot.clone());
return std::result::Result::Ok(());
}
}
fn activity_for_state(state: ksp_worker_api::WorkerState, admission_queue_depth: usize, in_flight_persistence: usize) -> ksp_worker_api::WorkerActivity {
if admission_queue_depth > 0 || in_flight_persistence > 0 {
return ksp_worker_api::WorkerActivity::Active;
}
return match state {
ksp_worker_api::WorkerState::Running
| ksp_worker_api::WorkerState::Stopping
| ksp_worker_api::WorkerState::Stopped
| ksp_worker_api::WorkerState::Faulted(_) => ksp_worker_api::WorkerActivity::Idle,
_ => ksp_worker_api::WorkerActivity::Unknown,
};
}
fn checked_counter(current: u64, field: &'static str) -> ksp_core_lib::Result<u64> {
return match current.checked_add(1) {
std::option::Option::Some(value) => std::result::Result::Ok(value),
std::option::Option::None => std::result::Result::Err(crate::counter_exhausted_error(field)),
};
}
fn health_for_state(state: ksp_worker_api::WorkerState, previous: ksp_worker_api::WorkerHealth) -> ksp_worker_api::WorkerHealth {
return match state {
ksp_worker_api::WorkerState::Running => ksp_worker_api::WorkerHealth::Healthy,
ksp_worker_api::WorkerState::Stopping | ksp_worker_api::WorkerState::Stopped => previous,
ksp_worker_api::WorkerState::Faulted(_) => ksp_worker_api::WorkerHealth::Unhealthy,
_ => ksp_worker_api::WorkerHealth::Unknown,
};
}
#[cfg(test)]
#[path = "../unit_tests/snapshot.rs"]
mod tests;

36
crates/ksp-worker-raw-transaction-ingest-lib/tests/dependency_boundary.rs
View File

@@ -1,5 +1,5 @@
// file: crates/ksp-worker-raw-transaction-ingest-lib/tests/dependency_boundary.rs
// version: 6
// version: 7
//! Dependency firewall canaries for the RAW transaction ingest Worker foundation.
@@ -52,25 +52,31 @@ fn pre_002_manifest_keeps_forbidden_layers_and_live_sources_out() {
}
#[test]
fn pre_007_source_surface_adds_private_normal_store_persistence_without_backend_or_live_source() {
fn pre_008_source_surface_adds_one_latest_value_snapshot_watch_without_backend_or_live_source() {
let root = include_str!("../src/lib.rs");
let runtime = include_str!("../src/runtime.rs");
let admission = include_str!("../src/admission.rs");
let persistence = include_str!("../src/persistence.rs");
let snapshot = include_str!("../src/snapshot.rs");
for required in [
"tokio::sync::mpsc::channel",
"canonicalize_raw_transaction",
"assemble_raw_transaction_acquisition",
"RawTransactionIngestPersistencePort",
"persist_raw_transaction_acquisition",
"RawTransactionAcquisitionMode::Normal",
"persistence_concurrency()",
"ERROR_CODE_RAW_TRANSACTION_INGEST_CONTENT_CONFLICT",
"ERROR_CODE_RAW_TRANSACTION_INGEST_STORE_FAILED",
"RawTransactionIngestSnapshot",
"RawTransactionIngestSnapshotSource",
"WorkerSnapshotSource",
"tokio::sync::watch::channel",
"ERROR_CODE_RAW_TRANSACTION_INGEST_COUNTER_EXHAUSTED",
] {
assert!(
root.contains(required) || runtime.contains(required) || admission.contains(required) || persistence.contains(required),
"required pre.007 persistence contract missing: {required}"
root.contains(required)
|| runtime.contains(required)
|| admission.contains(required)
|| persistence.contains(required)
|| snapshot.contains(required),
"required pre.008 snapshot contract missing: {required}"
);
}
for forbidden in [
@@ -80,15 +86,21 @@ fn pre_007_source_surface_adds_private_normal_store_persistence_without_backend_
"unbounded_channel",
"ksp_store_postgres_lib::",
"ksp_onchain_transport_lib::",
"RawTransactionIngestSnapshot",
"WorkerSnapshotSource",
"ForceRehydrate",
"source_failed",
"drain_timeout",
] {
assert!(
!root.contains(forbidden) && !runtime.contains(forbidden) && !admission.contains(forbidden) && !persistence.contains(forbidden),
"pre.007 crossed a forbidden runtime boundary: {forbidden}"
!root.contains(forbidden)
&& !runtime.contains(forbidden)
&& !admission.contains(forbidden)
&& !persistence.contains(forbidden)
&& !snapshot.contains(forbidden),
"pre.008 crossed a forbidden runtime boundary: {forbidden}"
);
}
assert_eq!(runtime.matches("tokio::sync::watch::channel").count(), 2, "runtime retains only the external-control and private-source stop watches");
assert_eq!(snapshot.matches("tokio::sync::watch::channel").count(), 1, "exactly one shared concrete snapshot watch is allowed");
return;
}

30
crates/ksp-worker-raw-transaction-ingest-lib/tests/public_api.rs
View File

@@ -1,7 +1,7 @@
// file: crates/ksp-worker-raw-transaction-ingest-lib/tests/public_api.rs
// version: 6
// version: 7
//! External public-surface proofs for the RAW transaction ingest Worker identity and settings foundation.
//! External public-surface proofs for the RAW transaction ingest Worker foundation.
#[test]
fn pre_003_kind_code_and_settings_are_consumable_from_crate_root() {
@@ -100,3 +100,29 @@ fn pre_007_store_and_content_conflict_error_codes_are_public_and_stable() {
assert_eq!(ksp_worker_raw_transaction_ingest_lib::ERROR_CODE_RAW_TRANSACTION_INGEST_STORE_FAILED.code(), "store_failed");
return;
}
#[test]
fn pre_008_snapshot_surface_and_common_projection_are_public_and_stable() {
let _snapshot_source: fn(
&ksp_worker_raw_transaction_ingest_lib::RawTransactionIngestHandle,
) -> ksp_worker_raw_transaction_ingest_lib::RawTransactionIngestSnapshotSource =
ksp_worker_raw_transaction_ingest_lib::RawTransactionIngestHandle::snapshot_source;
let _worker_snapshot_source: fn(
&ksp_worker_raw_transaction_ingest_lib::RawTransactionIngestHandle,
) -> ksp_worker_raw_transaction_ingest_lib::RawTransactionIngestSnapshotSource =
ksp_worker_raw_transaction_ingest_lib::RawTransactionIngestHandle::worker_snapshot_source;
let _current: fn(
&ksp_worker_raw_transaction_ingest_lib::RawTransactionIngestSnapshotSource,
) -> ksp_worker_raw_transaction_ingest_lib::RawTransactionIngestSnapshot =
ksp_worker_raw_transaction_ingest_lib::RawTransactionIngestSnapshotSource::current;
let _wait: for<'a> fn(
&'a ksp_worker_raw_transaction_ingest_lib::RawTransactionIngestSnapshotSource,
ksp_worker_api::WorkerSnapshotSequence,
) -> ksp_worker_raw_transaction_ingest_lib::RawTransactionIngestSnapshotFuture<'a> =
ksp_worker_raw_transaction_ingest_lib::RawTransactionIngestSnapshotSource::wait_for_change;
fn require_worker_source<T: ksp_worker_api::WorkerSnapshotSource + std::marker::Send + std::marker::Sync>() {}
require_worker_source::<ksp_worker_raw_transaction_ingest_lib::RawTransactionIngestSnapshotSource>();
assert_eq!(ksp_worker_raw_transaction_ingest_lib::ERROR_CODE_RAW_TRANSACTION_INGEST_COUNTER_EXHAUSTED.domain(), "worker_raw_transaction_ingest");
assert_eq!(ksp_worker_raw_transaction_ingest_lib::ERROR_CODE_RAW_TRANSACTION_INGEST_COUNTER_EXHAUSTED.code(), "counter_exhausted");
return;
}

139
crates/ksp-worker-raw-transaction-ingest-lib/unit_tests/runtime.rs
View File

@@ -1,5 +1,5 @@
// file: crates/ksp-worker-raw-transaction-ingest-lib/unit_tests/runtime.rs
// version: 4
// version: 5
struct ActiveTaskGuard {
active: std::sync::Arc<std::sync::atomic::AtomicUsize>,
@@ -88,7 +88,7 @@ async fn pre_004_immediate_stop_is_idempotent_and_reaches_stopped_terminal() {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return,
};
assert_eq!(*handle.terminal_receiver.borrow(), ksp_worker_api::WorkerState::Starting);
assert_eq!(handle.snapshot_source().current().worker_snapshot().state(), ksp_worker_api::WorkerState::Starting);
assert!(handle.request_stop());
assert!(!handle.request_stop());
let terminal = handle.wait_terminal().await;
@@ -113,7 +113,7 @@ async fn pre_004_running_lifecycle_stops_through_cloned_handle() {
std::result::Result::Err(_) => return,
};
tokio::task::yield_now().await;
assert_eq!(*handle.terminal_receiver.borrow(), ksp_worker_api::WorkerState::Running);
assert_eq!(handle.snapshot_source().current().worker_snapshot().state(), ksp_worker_api::WorkerState::Running);
let clone = handle.clone();
assert!(clone.request_stop());
assert!(!handle.request_stop());
@@ -136,21 +136,15 @@ async fn pre_004_dropping_last_control_handle_causes_private_runtime_exit() {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return,
};
let mut observer = handle.terminal_receiver.clone();
let observer = handle.snapshot_source();
std::mem::drop(handle);
loop {
let current = *observer.borrow();
if current.is_terminal() {
let closed = observer.changed().await;
assert!(closed.is_err());
assert_eq!(current, ksp_worker_api::WorkerState::Stopped);
return;
}
let changed = observer.changed().await;
assert!(changed.is_ok(), "runtime closed before publishing terminal state");
if changed.is_err() {
let current = observer.current();
if current.worker_snapshot().state().is_terminal() {
assert_eq!(current.worker_snapshot().state(), ksp_worker_api::WorkerState::Stopped);
return;
}
let _changed = observer.wait_for_change(current.worker_snapshot().sequence()).await;
}
}
@@ -485,6 +479,7 @@ async fn pre_007_content_conflict_becomes_terminal_after_private_drain() {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return,
};
let source = handle.snapshot_source();
let terminal = handle.wait_terminal().await;
let terminal = match terminal {
std::result::Result::Ok(value) => value,
@@ -492,6 +487,10 @@ async fn pre_007_content_conflict_becomes_terminal_after_private_drain() {
};
assert_eq!(terminal, ksp_worker_api::WorkerState::Faulted(crate::ERROR_CODE_RAW_TRANSACTION_INGEST_CONTENT_CONFLICT));
assert_eq!(port.completed.load(std::sync::atomic::Ordering::Acquire), 1);
let snapshot = source.current();
assert_eq!(snapshot.content_conflict_total(), 1);
assert_eq!(snapshot.store_failure_total(), 0);
assert_eq!(snapshot.worker_snapshot().health(), ksp_worker_api::WorkerHealth::Unhealthy);
return;
}
@@ -532,3 +531,115 @@ async fn pre_007_store_failure_becomes_terminal_without_exposing_remote_error_te
assert!(!std::format!("{handle:?}").contains("synthetic store failure"));
return;
}
#[tokio::test(flavor = "current_thread")]
async fn pre_008_runtime_snapshots_count_successful_pipeline_and_retain_terminal() {
let settings = match settings_with_persistence_concurrency(2) {
std::option::Option::Some(value) => value,
std::option::Option::None => return,
};
let network = settings.network().clone();
let port = std::sync::Arc::new(RuntimePersistencePort::new(network.clone(), RuntimePortResponse::SuccessBlocked, true));
let runtime_port: super::PersistencePort = port.clone();
let handle = match super::start_foundation_with_port_and_source_spawner(
settings,
tokio::runtime::Handle::current(),
std::option::Option::Some(runtime_port),
move |children, _stop_receiver, admission_sender| {
let _abort_handle = children.spawn(async move {
for signature_byte in 21..=23 {
let ingress = match runtime_ingress(&network, signature_byte) {
std::option::Option::Some(value) => value,
std::option::Option::None => return,
};
if admission_sender.send(ingress).await.is_err() {
return;
}
}
return;
});
},
) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return,
};
let concrete_source = handle.snapshot_source();
let common_source = handle.worker_snapshot_source();
assert!(wait_for_completed(&port, 3).await);
assert!(handle.request_stop());
let terminal = handle.wait_terminal().await;
let terminal = match terminal {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return,
};
assert_eq!(terminal, ksp_worker_api::WorkerState::Stopped);
let snapshot = concrete_source.current();
assert_eq!(snapshot.worker_snapshot().state(), ksp_worker_api::WorkerState::Stopped);
assert_eq!(snapshot.worker_snapshot().health(), ksp_worker_api::WorkerHealth::Healthy);
assert_eq!(snapshot.worker_snapshot().activity(), ksp_worker_api::WorkerActivity::Idle);
assert_eq!(snapshot.admitted_total(), 3);
assert_eq!(snapshot.canonicalized_total(), 3);
assert_eq!(snapshot.persisted_total(), 3);
assert_eq!(snapshot.entity_inserted_total(), 3);
assert_eq!(snapshot.entity_already_present_total(), 0);
assert_eq!(snapshot.entity_skipped_purged_total(), 0);
assert_eq!(snapshot.observation_inserted_total(), 3);
assert_eq!(snapshot.observation_already_present_total(), 0);
assert_eq!(snapshot.content_conflict_total(), 0);
assert_eq!(snapshot.store_failure_total(), 0);
assert_eq!(snapshot.source_failure_total(), 0);
assert_eq!(snapshot.backpressure_wait_total(), 0);
assert_eq!(snapshot.admission_queue_depth(), 0);
assert_eq!(snapshot.in_flight_persistence(), 0);
let common = ksp_worker_api::WorkerSnapshotSource::current(&common_source);
assert_eq!(&common, snapshot.worker_snapshot());
let retained = concrete_source.wait_for_change(snapshot.worker_snapshot().sequence()).await;
assert_eq!(retained, snapshot);
return;
}
#[tokio::test(flavor = "current_thread")]
async fn pre_008_fault_snapshots_count_classified_store_failures_and_project_unhealthy() {
let settings = match settings_with_persistence_concurrency(1) {
std::option::Option::Some(value) => value,
std::option::Option::None => return,
};
let network = settings.network().clone();
let port = std::sync::Arc::new(RuntimePersistencePort::new(network.clone(), RuntimePortResponse::StoreFailure, true));
let runtime_port: super::PersistencePort = port.clone();
let handle = match super::start_foundation_with_port_and_source_spawner(
settings,
tokio::runtime::Handle::current(),
std::option::Option::Some(runtime_port),
move |children, _stop_receiver, admission_sender| {
let _abort_handle = children.spawn(async move {
let ingress = match runtime_ingress(&network, 24) {
std::option::Option::Some(value) => value,
std::option::Option::None => return,
};
let _sent = admission_sender.send(ingress).await;
return;
});
},
) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return,
};
let source = handle.snapshot_source();
let terminal = handle.wait_terminal().await;
let terminal = match terminal {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return,
};
assert_eq!(terminal, ksp_worker_api::WorkerState::Faulted(crate::ERROR_CODE_RAW_TRANSACTION_INGEST_STORE_FAILED));
let snapshot = source.current();
assert_eq!(snapshot.worker_snapshot().state(), terminal);
assert_eq!(snapshot.worker_snapshot().health(), ksp_worker_api::WorkerHealth::Unhealthy);
assert_eq!(snapshot.worker_snapshot().activity(), ksp_worker_api::WorkerActivity::Idle);
assert_eq!(snapshot.admitted_total(), 1);
assert_eq!(snapshot.canonicalized_total(), 1);
assert_eq!(snapshot.persisted_total(), 0);
assert_eq!(snapshot.store_failure_total(), 1);
assert_eq!(snapshot.content_conflict_total(), 0);
return;
}

115
crates/ksp-worker-raw-transaction-ingest-lib/unit_tests/snapshot.rs Normal file
View File

@@ -0,0 +1,115 @@
// file: crates/ksp-worker-raw-transaction-ingest-lib/unit_tests/snapshot.rs
// version: 1
fn snapshot_foundation() -> std::option::Option<(crate::RawTransactionIngestSnapshotPublisher, crate::RawTransactionIngestSnapshotSource)> {
let network = match ksp_store_lib::RawNetworkId::new("mainnet") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return std::option::Option::None,
};
let worker_id = match ksp_worker_api::WorkerId::new("raw-ingest-snapshot-001") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return std::option::Option::None,
};
let settings = crate::RawTransactionIngestSettings::with_defaults(network, worker_id.clone());
let kind = match ksp_worker_api::WorkerKindCode::new(crate::RAW_TRANSACTION_INGEST_WORKER_KIND_CODE) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return std::option::Option::None,
};
let mut lifecycle = ksp_worker_api::WorkerLifecycle::new(worker_id, kind);
if lifecycle.start().is_err() {
return std::option::Option::None;
}
return std::option::Option::Some(crate::RawTransactionIngestSnapshotPublisher::new(&settings, &lifecycle));
}
#[test]
fn pre_008_initial_snapshot_and_common_projection_are_exact() {
let (publisher, source) = match snapshot_foundation() {
std::option::Option::Some(value) => value,
std::option::Option::None => return,
};
let _publisher = publisher;
let concrete = source.current();
assert_eq!(concrete.worker_snapshot().sequence().value(), 0);
assert_eq!(concrete.worker_snapshot().state(), ksp_worker_api::WorkerState::Starting);
assert_eq!(concrete.worker_snapshot().health(), ksp_worker_api::WorkerHealth::Unknown);
assert_eq!(concrete.worker_snapshot().activity(), ksp_worker_api::WorkerActivity::Unknown);
assert_eq!(concrete.admission_queue_capacity(), crate::DEFAULT_RAW_TRANSACTION_INGEST_ADMISSION_QUEUE_CAPACITY);
assert_eq!(concrete.admission_queue_depth(), 0);
assert_eq!(concrete.persistence_concurrency(), crate::DEFAULT_RAW_TRANSACTION_INGEST_PERSISTENCE_CONCURRENCY);
assert_eq!(concrete.in_flight_persistence(), 0);
assert_eq!(concrete.admitted_total(), 0);
assert_eq!(concrete.canonicalized_total(), 0);
assert_eq!(concrete.persisted_total(), 0);
assert_eq!(concrete.entity_inserted_total(), 0);
assert_eq!(concrete.entity_already_present_total(), 0);
assert_eq!(concrete.entity_skipped_purged_total(), 0);
assert_eq!(concrete.observation_inserted_total(), 0);
assert_eq!(concrete.observation_already_present_total(), 0);
assert_eq!(concrete.content_conflict_total(), 0);
assert_eq!(concrete.store_failure_total(), 0);
assert_eq!(concrete.source_failure_total(), 0);
assert_eq!(concrete.backpressure_wait_total(), 0);
let common = ksp_worker_api::WorkerSnapshotSource::current(&source);
assert_eq!(&common, concrete.worker_snapshot());
return;
}
#[test]
fn pre_008_checked_counter_exhaustion_is_stable_and_never_wraps() {
let result = super::checked_counter(u64::MAX, "admitted_total");
assert!(result.is_err());
let error = match result {
std::result::Result::Ok(_) => return,
std::result::Result::Err(value) => value,
};
assert_eq!(error.code(), crate::ERROR_CODE_RAW_TRANSACTION_INGEST_COUNTER_EXHAUSTED);
assert_eq!(error.context().len(), 1);
assert_eq!(error.context()[0].key(), "field");
assert_eq!(error.context()[0].value(), "admitted_total");
return;
}
#[tokio::test(flavor = "current_thread")]
async fn pre_008_slow_concrete_and_common_listeners_coalesce_to_same_latest_sequence() {
let (mut publisher, source) = match snapshot_foundation() {
std::option::Option::Some(value) => value,
std::option::Option::None => return,
};
let observed = source.current().worker_snapshot().sequence();
assert!(publisher.publish_state(ksp_worker_api::WorkerState::Running, 0, 0).is_ok());
assert!(publisher.record_admission_success(ksp_worker_api::WorkerState::Running, 1, 1).is_ok());
let concrete = source.wait_for_change(observed).await;
let common = ksp_worker_api::WorkerSnapshotSource::wait_for_change(&source, observed).await;
assert_eq!(concrete.worker_snapshot().sequence(), common.sequence());
assert_eq!(concrete.worker_snapshot().sequence().value(), 2);
assert_eq!(concrete.worker_snapshot().state(), ksp_worker_api::WorkerState::Running);
assert_eq!(concrete.worker_snapshot().health(), ksp_worker_api::WorkerHealth::Healthy);
assert_eq!(concrete.worker_snapshot().activity(), ksp_worker_api::WorkerActivity::Active);
assert_eq!(concrete.admission_queue_depth(), 1);
assert_eq!(concrete.in_flight_persistence(), 1);
assert_eq!(concrete.admitted_total(), 1);
assert_eq!(concrete.canonicalized_total(), 1);
return;
}
#[tokio::test(flavor = "current_thread")]
async fn pre_008_terminal_latest_value_is_retained_after_publisher_drop() {
let (mut publisher, source) = match snapshot_foundation() {
std::option::Option::Some(value) => value,
std::option::Option::None => return,
};
assert!(publisher.publish_state(ksp_worker_api::WorkerState::Running, 0, 0).is_ok());
assert!(publisher.publish_state(ksp_worker_api::WorkerState::Stopping, 0, 0).is_ok());
assert!(publisher.publish_state(ksp_worker_api::WorkerState::Stopped, 0, 0).is_ok());
std::mem::drop(publisher);
let retained = source.current();
assert_eq!(retained.worker_snapshot().state(), ksp_worker_api::WorkerState::Stopped);
assert_eq!(retained.worker_snapshot().health(), ksp_worker_api::WorkerHealth::Healthy);
assert_eq!(retained.worker_snapshot().activity(), ksp_worker_api::WorkerActivity::Idle);
let same_terminal = source.wait_for_change(retained.worker_snapshot().sequence()).await;
assert_eq!(same_terminal, retained);
let common = ksp_worker_api::WorkerSnapshotSource::wait_for_change(&source, retained.worker_snapshot().sequence()).await;
assert_eq!(common, retained.worker_snapshot().clone());
return;
}

167
deltas/0.3.11/pre.008.md Normal file
View File

@@ -0,0 +1,167 @@
<!-- file: deltas/0.3.11/pre.008.md -->
<!-- version: 1 -->
# Delta `0.3.11-pre.008` — snapshots latest-value et projection Worker API
## Base requise
```text
0.3.11-pre.007
workspace.package.version = 0.3.11-pre.7
```
Le gate opérateur du 8 septembre 2026 est vert sur `fmt`, audits Rust/Markdown, `cargo check --workspace`, Clippy strict, 31 tests de la crate Worker et les doc-tests. Aucun `cargo tree` intermédiaire n'était requis : `pre.007` n'avait modifié aucune dépendance ni feature.
## Objectif
Matérialiser uniquement la responsabilité `pre.008` du plan `032` : snapshot concret latest-value, compteurs checked, projection directe vers `ksp-worker-api::WorkerSnapshotSource`, coalescence des listeners et terminal retenu. Aucun hardening timeout/abort et aucune source réseau.
## Version
```text
identifiant de livraison : 0.3.11-pre.008
workspace.package.version : 0.3.11-pre.8
```
## Snapshot concret
La crate expose :
```text
RawTransactionIngestSnapshot
RawTransactionIngestSnapshotFuture
RawTransactionIngestSnapshotSource
```
Le snapshot contient uniquement la projection `WorkerSnapshot`, les capacités/profondeurs techniques bornées et les compteurs sûrs prévus par le plan.
## Un seul flux latest-value
Un unique :
```text
tokio::sync::watch::channel<RawTransactionIngestSnapshot>
```
porte l'état observable du run. Le watch terminal spécialisé des tranches précédentes disparaît : `wait_terminal()`, `snapshot_source()` et `worker_snapshot_source()` lisent tous ce même flux.
`RawTransactionIngestSnapshotSource` implémente directement `ksp_worker_api::WorkerSnapshotSource`. La sequence `WorkerSnapshotSequence` est donc identique pour la vue concrète et la projection commune.
## Compteurs checked
Les compteurs `u64` effectivement incrémentés dans cette tranche utilisent `checked_add(1)`. Aucun wrap silencieux n'est admis. Le nouveau code public est :
```text
worker_raw_transaction_ingest.counter_exhausted
```
Les compteurs sont mis à jour au point d'ownership du supervisor : dequeue/canonicalisation et récolte de persistence. `admission_queue_depth` provient directement du receiver `mpsc` borné ; `in_flight_persistence` provient du `JoinSet` privé existant.
`source_failure_total` et `backpressure_wait_total` font partie du snapshot sûr mais restent à zéro en `pre.008`; leur comportement effectif appartient à `pre.009`.
## Projection common Worker
Mapping de fondation :
```text
Starting -> health Unknown
Running -> health Healthy
Stopping / Stopped -> conserve le dernier health non terminal
Faulted -> health Unhealthy
queue > 0 ou persistence > 0 -> activity Active
sinon Running/Stopping/terminal -> activity Idle
```
La state machine reste celle de `ksp-worker-api::WorkerLifecycle`.
## Tests ajoutés ou étendus
```text
snapshot initial exact
projection common/concrete même sequence
counter exhaustion stable sans wrap
listener lent coalescé vers la dernière valeur
terminal latest-value retenu après fermeture publisher
surface publique snapshot_source/worker_snapshot_source
WorkerSnapshotSource Send + Sync
pipeline runtime réussi -> counters exacts
Store failure -> health Unhealthy + store_failure_total
absence de watch terminal séparé
absence de nouveau dependency edge
```
## Fichiers ajoutés
```text
crates/ksp-worker-raw-transaction-ingest-lib/src/snapshot.rs
crates/ksp-worker-raw-transaction-ingest-lib/unit_tests/snapshot.rs
deltas/0.3.11/pre.008.md
```
## Fichiers modifiés
```text
Cargo.toml
crates/ksp-worker-raw-transaction-ingest-lib/src/admission.rs
crates/ksp-worker-raw-transaction-ingest-lib/src/error.rs
crates/ksp-worker-raw-transaction-ingest-lib/src/lib.rs
crates/ksp-worker-raw-transaction-ingest-lib/src/runtime.rs
crates/ksp-worker-raw-transaction-ingest-lib/tests/dependency_boundary.rs
crates/ksp-worker-raw-transaction-ingest-lib/tests/public_api.rs
crates/ksp-worker-raw-transaction-ingest-lib/unit_tests/runtime.rs
docs/plans/032-V0_3_11_RAW_TRANSACTION_INGEST_WORKER_FOUNDATION_PLAN.md
docs/validation/028-V0_3_11_RAW_TRANSACTION_INGEST_WORKER_FOUNDATION.md
```
## Fichiers supprimés
Aucun.
## Hors scope conservé
```text
source live / Transport
Config
backend Store direct
source_failed public
source failure behavior
backpressure wait instrumentation productive
drain timeout / abort forcé
fault/stop precedence finalisée
retry / reconnect / gap repair
```
## Validations exécutées dans l'environnement d'assemblage
```text
General Rust rule audit: clean
Rust export completeness audit: 0 candidate(s)
KSP workspace Rust rule audit: clean
Markdown table audit: clean
scan production : aucun unwrap, expect, panic, unbounded_channel
scan scope : aucun backend Store direct, aucun Transport, aucune nouvelle dépendance
```
L'environnement d'assemblage ne fournit ni `cargo`, ni `rustc`, ni `rustfmt`. Aucun gate Cargo de `pre.008` n'est déclaré PASS localement.
## Gate opérateur demandé
Aucune dépendance ni feature n'a changé dans cette tranche ; aucun `cargo tree` intermédiaire n'est requis :
```bash
cargo fmt --all
python3 scripts/audit_rust_workspace_rules.py
python3 scripts/audit_markdown_tables.py README.md RULES.md ROADMAP.md CHANGELOG.md docs prompts crates deltas
cargo check --workspace
cargo clippy --workspace --all-targets --all-features -- -D warnings
cargo test -p ksp-worker-raw-transaction-ingest-lib
```
## Décision
`pre.009` reste bloquée jusqu'à validation opérateur verte des snapshots common/concrete, des compteurs checked et du terminal latest-value retenu.
## Questions ouvertes
Aucune nouvelle question architecturale. Le hardening shutdown/backpressure/fault races reste dans `pre.009`.

6
docs/plans/032-V0_3_11_RAW_TRANSACTION_INGEST_WORKER_FOUNDATION_PLAN.md
View File

@@ -1,5 +1,5 @@
<!-- file: docs/plans/032-V0_3_11_RAW_TRANSACTION_INGEST_WORKER_FOUNDATION_PLAN.md -->
<!-- version: 7 -->
<!-- version: 8 -->
# Plan v0.3.11 — fondation runtime du Worker RawTransaction ingest
@@ -732,12 +732,14 @@ Budget cible : **15–20 min**. `mpsc` borné, backpressure, ingress privé, com
Budget cible : **15–20 min**. Port privé Store, mode Normal, concurrency bornée, outcomes new/idempotent/purged, observation distincte, Store error et content conflict terminal.
État après matérialisation : **implémenté, gate opérateur requis**. La crate possède désormais un port de persistence Store crate-private, implémenté par la façade `ksp-store-lib::Store`, et appelle exclusivement l'opération atomique transaction + observation en `RawTransactionAcquisitionMode::Normal`. Le supervisor possède un `JoinSet` privé de persistences ; il ne lit une nouvelle acquisition que si `in_flight < persistence_concurrency`, ce qui borne les écritures sans sémaphore ni file secondaire. Les succès distinguent `Inserted`, `AlreadyPresent`, `SkippedPurged` et les observations `Inserted`, `AlreadyPresent`, `NotRecorded`; `Rehydrated` ou toute combinaison impossible en mode Normal deviennent `runtime_invalid`. `store_api.raw_conflict` est réduit à `worker_raw_transaction_ingest.content_conflict`; toute autre erreur Store devient `worker_raw_transaction_ingest.store_failed` en ne conservant que le `ErrorCode` inférieur sûr. Le premier fault de cette tranche signale le stop privé, ferme ensuite les nouvelles admissions, draine les acquisitions déjà admises, rejoint persistences et sources, puis publie le terminal. Les races terminales fortes et le timeout/abort restent réservés à `pre.009`; les compteurs/snapshots restent réservés à `pre.008`.
État après matérialisation : **implémenté et gate opérateur validé**. La crate possède désormais un port de persistence Store crate-private, implémenté par la façade `ksp-store-lib::Store`, et appelle exclusivement l'opération atomique transaction + observation en `RawTransactionAcquisitionMode::Normal`. Le supervisor possède un `JoinSet` privé de persistences ; il ne lit une nouvelle acquisition que si `in_flight < persistence_concurrency`, ce qui borne les écritures sans sémaphore ni file secondaire. Les succès distinguent `Inserted`, `AlreadyPresent`, `SkippedPurged` et les observations `Inserted`, `AlreadyPresent`, `NotRecorded`; `Rehydrated` ou toute combinaison impossible en mode Normal deviennent `runtime_invalid`. `store_api.raw_conflict` est réduit à `worker_raw_transaction_ingest.content_conflict`; toute autre erreur Store devient `worker_raw_transaction_ingest.store_failed` en ne conservant que le `ErrorCode` inférieur sûr. Le premier fault de cette tranche signale le stop privé, ferme ensuite les nouvelles admissions, draine les acquisitions déjà admises, rejoint persistences et sources, puis publie le terminal. Le gate communiqué le 8 septembre 2026 est vert sur `fmt`, audits, `check`, Clippy strict, 31 tests de crate et doc-tests. Aucun arbre Cargo n'a été requis car aucune dépendance/feature n'avait changé. Les races terminales fortes et le timeout/abort restent réservés à `pre.009`.
### `pre.008` — snapshots concrets + projection Worker API
Budget cible : **15–20 min**. `watch` latest-value, compteurs checked, common projection, slow/no listener et terminal retained.
État après matérialisation : **implémenté, gate opérateur requis**. La crate expose `RawTransactionIngestSnapshot` et `RawTransactionIngestSnapshotSource`, ce dernier implémentant directement `ksp-worker-api::WorkerSnapshotSource`. Un unique `watch<RawTransactionIngestSnapshot>` porte la valeur latest-value concrète ; `wait_terminal()` lit ce même flux, supprimant le watch terminal spécialisé de `pre.004`. La séquence `WorkerSnapshotSequence` est donc identique pour la vue concrète et la projection commune. Les compteurs `u64` sont incrémentés par checked-add au point exact où le supervisor dequeue/canonicalise ou récolte une persistence ; l'épuisement devient `worker_raw_transaction_ingest.counter_exhausted`. Les profondeurs de queue et de persistence sont dérivées des structures bornées existantes, sans tâche de monitoring ni seconde event queue. `source_failure_total` et `backpressure_wait_total` existent dans le contrat sûr mais restent à zéro dans cette tranche source-neutral ; `pre.009` est propriétaire du fault source et de l'instrumentation de saturation. Les tests couvrent projection common/concrete, listener lent coalescé, terminal retenu, counters de pipeline réussi et Store fault projeté `Unhealthy`.
### `pre.009` — hardening shutdown/backpressure/fault races
Budget cible : **15–20 min**. Drain deadline, stop/fault ordering, saturation, source failure, Store slow/failure, abort+join au timeout, no orphan tasks. Scinder immédiatement si le gate réel dépasse le budget.

130
docs/validation/028-V0_3_11_RAW_TRANSACTION_INGEST_WORKER_FOUNDATION.md
View File

@@ -1,5 +1,5 @@
<!-- file: docs/validation/028-V0_3_11_RAW_TRANSACTION_INGEST_WORKER_FOUNDATION.md -->
<!-- version: 10 -->
<!-- version: 11 -->
# Validation v0.3.11 — fondation runtime du Worker RawTransaction ingest
@@ -1173,3 +1173,131 @@ cargo test -p ksp-worker-raw-transaction-ingest-lib
```
Critère de passage : persistence Normal, outcomes/idempotence/purge, concurrence bornée et terminaux conflict/Store sont verts, sans backend direct, Transport ni snapshot concret.
## 19. Fermeture opérateur `pre.007` et matérialisation `pre.008`
### 19.1 Fermeture opérateur de `pre.007`
Le journal opérateur communiqué le 8 septembre 2026 ferme `0.3.11-pre.007` (`workspace.package.version = 0.3.11-pre.7`) :
```text
cargo fmt --all : terminé sans erreur
python3 scripts/audit_rust_workspace_rules.py : clean, export completeness 0
python3 scripts/audit_markdown_tables.py ... : clean (340 tables, 783 files)
cargo check --workspace : terminé sans erreur
cargo clippy --workspace --all-targets --all-features -- -D warnings : terminé sans erreur
cargo test -p ksp-worker-raw-transaction-ingest-lib : 23 unit + 3 dependency-boundary + 5 public API PASS, 0 échec
doc-tests : PASS
```
Aucune dépendance ni feature n'avait changé dans `pre.007`; aucun `cargo tree` intermédiaire n'était requis. `pre.008` peut donc être ouverte.
### 19.2 Snapshot concret latest-value unique
`pre.008` introduit `RawTransactionIngestSnapshot` avec uniquement des données sûres : projection commune Worker, capacités/profondeurs bornées et compteurs monotones. Un seul :
```text
tokio::sync::watch::channel<RawTransactionIngestSnapshot>
```
porte la valeur latest-value concrète. Le handle ne conserve plus de watch terminal séparé : `wait_terminal()`, `snapshot_source()` et `worker_snapshot_source()` lisent le même flux.
Le source concret implémente directement `ksp_worker_api::WorkerSnapshotSource`; la même `WorkerSnapshotSequence` pilote donc les deux vues sans adaptation de sequence ou deuxième queue d'événements.
### 19.3 Compteurs checked et profondeurs runtime
Les compteurs de `RawTransactionIngestSnapshot` sont des `u64` monotones :
```text
admitted_total
canonicalized_total
persisted_total
entity_inserted_total
entity_already_present_total
entity_skipped_purged_total
observation_inserted_total
observation_already_present_total
content_conflict_total
store_failure_total
source_failure_total
backpressure_wait_total
```
Chaque incrément effectivement utilisé dans `pre.008` passe par `checked_add(1)`. L'épuisement ne wrappe jamais et devient :
```text
worker_raw_transaction_ingest.counter_exhausted
```
`admission_queue_depth` est lu directement depuis le receiver `mpsc` borné et `in_flight_persistence` depuis le `JoinSet` privé de persistences. Aucun polling, aucune tâche métrique et aucune queue de télémétrie ne sont ajoutés.
`source_failure_total` et `backpressure_wait_total` restent à zéro dans cette tranche source-neutral. Leur instrumentation comportementale appartient à `pre.009`, qui possède le fault source, la saturation et les races shutdown/fault.
### 19.4 Mapping Worker API
Mapping conservé :
```text
Starting -> health Unknown / activity Unknown si vide
Running sans fault -> health Healthy
Stopping -> conserve le dernier health non terminal
Faulted -> health Unhealthy
queue > 0 ou persistence > 0 -> activity Active
Running/Stopping terminalement vide -> activity Idle
```
Le terminal n'est publié qu'après le drain/join déjà possédé par le supervisor. La valeur terminale reste lisible après drop du publisher grâce à la sémantique latest-value de `watch`.
### 19.5 Preuves ajoutées
Les tests `pre.008` couvrent :
```text
snapshot initial Starting exact
projection common == snapshot concret pour la même sequence
compteurs/capacités initialement exacts
checked counter exhaustion -> counter_exhausted sans wrap
listener lent -> coalescence sur la dernière sequence
terminal concret retenu après fermeture du publisher
handle.snapshot_source() public
handle.worker_snapshot_source() public et WorkerSnapshotSource Send + Sync
pipeline runtime réussi -> admitted/canonicalized/persisted/outcomes exacts
Store failure terminal -> snapshot Unhealthy + store_failure_total
absence de nouveau dependency edge
un seul watch snapshot concret, aucun backend direct ni Transport
```
### 19.6 Frontière volontaire de `pre.008`
La tranche n'introduit toujours aucun :
```text
source live
Transport
Config
backend Store direct
source_failed public
source failure behavior
backpressure wait instrumentation productive
drain timeout / abort forcé
fault/stop precedence finalisée
retry / reconnect / gap repair
```
`pre.009` reste propriétaire du hardening shutdown/backpressure/fault races.
### 19.7 Gate opérateur demandé pour `pre.008`
Aucune dépendance ni feature n'est modifiée par `pre.008`; aucun `cargo tree` intermédiaire n'est requis. Le gate demandé est :
```bash
cargo fmt --all
python3 scripts/audit_rust_workspace_rules.py
python3 scripts/audit_markdown_tables.py README.md RULES.md ROADMAP.md CHANGELOG.md docs prompts crates deltas
cargo check --workspace
cargo clippy --workspace --all-targets --all-features -- -D warnings
cargo test -p ksp-worker-raw-transaction-ingest-lib
```
Critère de passage : snapshots common/concrete latest-value, compteurs checked, terminal retenu et absence de nouveau boundary crossing sont verts.