v0.3.14-pre.012
This commit is contained in:
@@ -1,12 +1,12 @@
|
|||||||
# file: Cargo.toml
|
# file: Cargo.toml
|
||||||
# version: 586
|
# version: 587
|
||||||
|
|
||||||
[workspace]
|
[workspace]
|
||||||
resolver = "3"
|
resolver = "3"
|
||||||
members = ["crates/ksp-app-backfill-desk", "crates/ksp-app-config-desk", "crates/ksp-app-solprices-desk", "crates/ksp-app-store-desk", "crates/ksp-app-wallet-desk", "crates/ksp-config-lib", "crates/ksp-core-lib", "crates/ksp-interface-lib", "crates/ksp-job-api", "crates/ksp-job-backfill-lib", "crates/ksp-logging-lib", "crates/ksp-offchain-transport-lib", "crates/ksp-onchain-transport-lib", "crates/ksp-program-api", "crates/ksp-raw-transaction-lib", "crates/ksp-store-api", "crates/ksp-store-lib", "crates/ksp-store-postgres-lib", "crates/ksp-wallet-lib", "crates/ksp-worker-api", "crates/ksp-worker-raw-transaction-ingest-lib"]
|
members = ["crates/ksp-app-backfill-desk", "crates/ksp-app-config-desk", "crates/ksp-app-solprices-desk", "crates/ksp-app-store-desk", "crates/ksp-app-wallet-desk", "crates/ksp-config-lib", "crates/ksp-core-lib", "crates/ksp-interface-lib", "crates/ksp-job-api", "crates/ksp-job-backfill-lib", "crates/ksp-logging-lib", "crates/ksp-offchain-transport-lib", "crates/ksp-onchain-transport-lib", "crates/ksp-program-api", "crates/ksp-raw-transaction-lib", "crates/ksp-store-api", "crates/ksp-store-lib", "crates/ksp-store-postgres-lib", "crates/ksp-wallet-lib", "crates/ksp-worker-api", "crates/ksp-worker-raw-transaction-ingest-lib"]
|
||||||
|
|
||||||
[workspace.package]
|
[workspace.package]
|
||||||
version = "0.3.14-pre.11.fix.2"
|
version = "0.3.14-pre.12"
|
||||||
edition = "2024"
|
edition = "2024"
|
||||||
license = "MIT"
|
license = "MIT"
|
||||||
repository = "https://git.sasedev.com/Sasedev/khadhroony-solana-project"
|
repository = "https://git.sasedev.com/Sasedev/khadhroony-solana-project"
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
// file: crates/ksp-worker-raw-transaction-ingest-lib/src/runtime_resources.rs
|
// file: crates/ksp-worker-raw-transaction-ingest-lib/src/runtime_resources.rs
|
||||||
// version: 42
|
// version: 43
|
||||||
|
|
||||||
use sha2::Digest; // rust-rules: trait-import
|
use sha2::Digest; // rust-rules: trait-import
|
||||||
|
|
||||||
@@ -1726,6 +1726,9 @@ impl crate::RawTransactionIngestHttpBlockPollingSource {
|
|||||||
break 'source;
|
break 'source;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
if *stop_receiver.borrow() {
|
||||||
|
break;
|
||||||
|
}
|
||||||
if !blocked_by_null && let std::option::Option::Some(proven_end_slot) = discovery.proven_end_slot {
|
if !blocked_by_null && let std::option::Option::Some(proven_end_slot) = discovery.proven_end_slot {
|
||||||
let coverage_result = {
|
let coverage_result = {
|
||||||
let mut contracts = match continuity_contracts.lock() {
|
let mut contracts = match continuity_contracts.lock() {
|
||||||
@@ -2584,6 +2587,7 @@ impl crate::RawTransactionIngestRuntimeResources {
|
|||||||
std::sync::Arc::clone(&continuity_contracts),
|
std::sync::Arc::clone(&continuity_contracts),
|
||||||
std::sync::Arc::clone(&inventory),
|
std::sync::Arc::clone(&inventory),
|
||||||
processing_frontier_sender,
|
processing_frontier_sender,
|
||||||
|
settings.shutdown_drain_timeout(),
|
||||||
)
|
)
|
||||||
.await;
|
.await;
|
||||||
let validation = {
|
let validation = {
|
||||||
@@ -2682,18 +2686,19 @@ async fn supervise_live_source_tasks(
|
|||||||
continuity_contracts: std::sync::Arc<std::sync::Mutex<crate::RawTransactionIngestContinuityContracts>>,
|
continuity_contracts: std::sync::Arc<std::sync::Mutex<crate::RawTransactionIngestContinuityContracts>>,
|
||||||
inventory: std::sync::Arc<std::sync::Mutex<RawTransactionIngestSourceInventory>>,
|
inventory: std::sync::Arc<std::sync::Mutex<RawTransactionIngestSourceInventory>>,
|
||||||
processing_frontier_sender: tokio::sync::watch::Sender<crate::RawTransactionIngestProcessingFrontierProjection>,
|
processing_frontier_sender: tokio::sync::watch::Sender<crate::RawTransactionIngestProcessingFrontierProjection>,
|
||||||
|
shutdown_drain_timeout: std::time::Duration,
|
||||||
) -> ksp_core_lib::Result<()> {
|
) -> ksp_core_lib::Result<()> {
|
||||||
loop {
|
loop {
|
||||||
if *stop_receiver.borrow() {
|
if *stop_receiver.borrow() {
|
||||||
source_stop_sender.send_replace(true);
|
source_stop_sender.send_replace(true);
|
||||||
return drain_live_source_tasks(&mut children, std::option::Option::None).await;
|
return drain_live_source_tasks(&mut children, std::option::Option::None, shutdown_drain_timeout).await;
|
||||||
}
|
}
|
||||||
let joined = tokio::select! {
|
let joined = tokio::select! {
|
||||||
biased;
|
biased;
|
||||||
changed = stop_receiver.changed() => {
|
changed = stop_receiver.changed() => {
|
||||||
if changed.is_err() || *stop_receiver.borrow() {
|
if changed.is_err() || *stop_receiver.borrow() {
|
||||||
source_stop_sender.send_replace(true);
|
source_stop_sender.send_replace(true);
|
||||||
return drain_live_source_tasks(&mut children, std::option::Option::None).await;
|
return drain_live_source_tasks(&mut children, std::option::Option::None, shutdown_drain_timeout).await;
|
||||||
}
|
}
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
@@ -2703,11 +2708,21 @@ async fn supervise_live_source_tasks(
|
|||||||
std::option::Option::Some(std::result::Result::Ok(value)) => value,
|
std::option::Option::Some(std::result::Result::Ok(value)) => value,
|
||||||
std::option::Option::Some(std::result::Result::Err(_)) => {
|
std::option::Option::Some(std::result::Result::Err(_)) => {
|
||||||
source_stop_sender.send_replace(true);
|
source_stop_sender.send_replace(true);
|
||||||
return drain_live_source_tasks(&mut children, std::option::Option::Some(crate::runtime_error("source.task_join_failed"))).await;
|
return drain_live_source_tasks(
|
||||||
|
&mut children,
|
||||||
|
std::option::Option::Some(crate::runtime_error("source.task_join_failed")),
|
||||||
|
shutdown_drain_timeout,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
},
|
},
|
||||||
std::option::Option::None => {
|
std::option::Option::None => {
|
||||||
source_stop_sender.send_replace(true);
|
source_stop_sender.send_replace(true);
|
||||||
return drain_live_source_tasks(&mut children, std::option::Option::Some(crate::runtime_error("source.task_set_empty"))).await;
|
return drain_live_source_tasks(
|
||||||
|
&mut children,
|
||||||
|
std::option::Option::Some(crate::runtime_error("source.task_set_empty")),
|
||||||
|
shutdown_drain_timeout,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
},
|
},
|
||||||
};
|
};
|
||||||
let source_fault = match source_result {
|
let source_fault = match source_result {
|
||||||
@@ -2716,7 +2731,7 @@ async fn supervise_live_source_tasks(
|
|||||||
};
|
};
|
||||||
if !source_loss_is_reconcilable(&source_fault) {
|
if !source_loss_is_reconcilable(&source_fault) {
|
||||||
source_stop_sender.send_replace(true);
|
source_stop_sender.send_replace(true);
|
||||||
return drain_live_source_tasks(&mut children, std::option::Option::Some(source_fault)).await;
|
return drain_live_source_tasks(&mut children, std::option::Option::Some(source_fault), shutdown_drain_timeout).await;
|
||||||
}
|
}
|
||||||
let supervisor_state = {
|
let supervisor_state = {
|
||||||
let inventory = match inventory.lock() {
|
let inventory = match inventory.lock() {
|
||||||
@@ -2729,21 +2744,21 @@ async fn supervise_live_source_tasks(
|
|||||||
std::result::Result::Ok(value) => value,
|
std::result::Result::Ok(value) => value,
|
||||||
std::result::Result::Err(error) => {
|
std::result::Result::Err(error) => {
|
||||||
source_stop_sender.send_replace(true);
|
source_stop_sender.send_replace(true);
|
||||||
return drain_live_source_tasks(&mut children, std::option::Option::Some(error)).await;
|
return drain_live_source_tasks(&mut children, std::option::Option::Some(error), shutdown_drain_timeout).await;
|
||||||
},
|
},
|
||||||
};
|
};
|
||||||
let continuity_range = match source_loss_continuity_range(&source_fault) {
|
let continuity_range = match source_loss_continuity_range(&source_fault) {
|
||||||
std::result::Result::Ok(value) => value,
|
std::result::Result::Ok(value) => value,
|
||||||
std::result::Result::Err(error) => {
|
std::result::Result::Err(error) => {
|
||||||
source_stop_sender.send_replace(true);
|
source_stop_sender.send_replace(true);
|
||||||
return drain_live_source_tasks(&mut children, std::option::Option::Some(error)).await;
|
return drain_live_source_tasks(&mut children, std::option::Option::Some(error), shutdown_drain_timeout).await;
|
||||||
},
|
},
|
||||||
};
|
};
|
||||||
let continuity_range = match continuity_range {
|
let continuity_range = match continuity_range {
|
||||||
std::option::Option::Some(value) => value,
|
std::option::Option::Some(value) => value,
|
||||||
std::option::Option::None => {
|
std::option::Option::None => {
|
||||||
source_stop_sender.send_replace(true);
|
source_stop_sender.send_replace(true);
|
||||||
return drain_live_source_tasks(&mut children, std::option::Option::Some(source_fault)).await;
|
return drain_live_source_tasks(&mut children, std::option::Option::Some(source_fault), shutdown_drain_timeout).await;
|
||||||
},
|
},
|
||||||
};
|
};
|
||||||
let decision = {
|
let decision = {
|
||||||
@@ -2761,7 +2776,7 @@ async fn supervise_live_source_tasks(
|
|||||||
std::result::Result::Ok(value) => value,
|
std::result::Result::Ok(value) => value,
|
||||||
std::result::Result::Err(error) => {
|
std::result::Result::Err(error) => {
|
||||||
source_stop_sender.send_replace(true);
|
source_stop_sender.send_replace(true);
|
||||||
return drain_live_source_tasks(&mut children, std::option::Option::Some(error)).await;
|
return drain_live_source_tasks(&mut children, std::option::Option::Some(error), shutdown_drain_timeout).await;
|
||||||
},
|
},
|
||||||
};
|
};
|
||||||
match decision {
|
match decision {
|
||||||
@@ -2770,7 +2785,7 @@ async fn supervise_live_source_tasks(
|
|||||||
std::result::Result::Ok(value) => value,
|
std::result::Result::Ok(value) => value,
|
||||||
std::result::Result::Err(error) => {
|
std::result::Result::Err(error) => {
|
||||||
source_stop_sender.send_replace(true);
|
source_stop_sender.send_replace(true);
|
||||||
return drain_live_source_tasks(&mut children, std::option::Option::Some(error)).await;
|
return drain_live_source_tasks(&mut children, std::option::Option::Some(error), shutdown_drain_timeout).await;
|
||||||
},
|
},
|
||||||
};
|
};
|
||||||
processing_frontier_sender.send_replace(aggregate);
|
processing_frontier_sender.send_replace(aggregate);
|
||||||
@@ -2778,7 +2793,7 @@ async fn supervise_live_source_tasks(
|
|||||||
},
|
},
|
||||||
crate::RawTransactionIngestSourceLossDecision::Fault => {
|
crate::RawTransactionIngestSourceLossDecision::Fault => {
|
||||||
source_stop_sender.send_replace(true);
|
source_stop_sender.send_replace(true);
|
||||||
return drain_live_source_tasks(&mut children, std::option::Option::Some(source_fault)).await;
|
return drain_live_source_tasks(&mut children, std::option::Option::Some(source_fault), shutdown_drain_timeout).await;
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -2834,7 +2849,9 @@ fn source_loss_is_reconcilable(error: &ksp_core_lib::Error) -> bool {
|
|||||||
async fn drain_live_source_tasks(
|
async fn drain_live_source_tasks(
|
||||||
children: &mut tokio::task::JoinSet<([u8; 32], ksp_core_lib::Result<()>)>,
|
children: &mut tokio::task::JoinSet<([u8; 32], ksp_core_lib::Result<()>)>,
|
||||||
mut first_fault: std::option::Option<ksp_core_lib::Error>,
|
mut first_fault: std::option::Option<ksp_core_lib::Error>,
|
||||||
|
shutdown_drain_timeout: std::time::Duration,
|
||||||
) -> ksp_core_lib::Result<()> {
|
) -> ksp_core_lib::Result<()> {
|
||||||
|
let drain = async {
|
||||||
while let std::option::Option::Some(joined) = children.join_next().await {
|
while let std::option::Option::Some(joined) = children.join_next().await {
|
||||||
if first_fault.is_some() {
|
if first_fault.is_some() {
|
||||||
continue;
|
continue;
|
||||||
@@ -2845,6 +2862,17 @@ async fn drain_live_source_tasks(
|
|||||||
std::result::Result::Err(_) => std::option::Option::Some(crate::runtime_error("source.task_join_failed")),
|
std::result::Result::Err(_) => std::option::Option::Some(crate::runtime_error("source.task_join_failed")),
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
};
|
||||||
|
if tokio::time::timeout(shutdown_drain_timeout, drain).await.is_err() {
|
||||||
|
children.abort_all();
|
||||||
|
while children.join_next().await.is_some() {}
|
||||||
|
if first_fault.is_none() {
|
||||||
|
first_fault = std::option::Option::Some(ksp_core_lib::Error::new(
|
||||||
|
crate::ERROR_CODE_RAW_TRANSACTION_INGEST_DRAIN_TIMEOUT,
|
||||||
|
"RAW transaction ingest source drain timed out",
|
||||||
|
));
|
||||||
|
}
|
||||||
|
}
|
||||||
return match first_fault {
|
return match first_fault {
|
||||||
std::option::Option::Some(error) => std::result::Result::Err(error),
|
std::option::Option::Some(error) => std::result::Result::Err(error),
|
||||||
std::option::Option::None => std::result::Result::Ok(()),
|
std::option::Option::None => std::result::Result::Ok(()),
|
||||||
@@ -4656,6 +4684,9 @@ impl RawTransactionIngestHydrationCoordinator {
|
|||||||
std::result::Result::Ok(std::result::Result::Err(error)) => return std::result::Result::Err(error),
|
std::result::Result::Ok(std::result::Result::Err(error)) => return std::result::Result::Err(error),
|
||||||
std::result::Result::Err(_) => return std::result::Result::Err(crate::runtime_error("source.hydration_task_join_failed")),
|
std::result::Result::Err(_) => return std::result::Result::Err(crate::runtime_error("source.hydration_task_join_failed")),
|
||||||
};
|
};
|
||||||
|
if *stop_receiver.borrow() {
|
||||||
|
return std::result::Result::Ok(false);
|
||||||
|
}
|
||||||
let pending = match self.pending.remove(&fetched.key) {
|
let pending = match self.pending.remove(&fetched.key) {
|
||||||
std::option::Option::Some(value) => value,
|
std::option::Option::Some(value) => value,
|
||||||
std::option::Option::None => return std::result::Result::Err(crate::runtime_error("source.hydration_result_without_pending")),
|
std::option::Option::None => return std::result::Result::Err(crate::runtime_error("source.hydration_result_without_pending")),
|
||||||
@@ -4665,6 +4696,9 @@ impl RawTransactionIngestHydrationCoordinator {
|
|||||||
}
|
}
|
||||||
self.pending_signal_count -= pending.signals.len();
|
self.pending_signal_count -= pending.signals.len();
|
||||||
for pending_signal in pending.signals {
|
for pending_signal in pending.signals {
|
||||||
|
if *stop_receiver.borrow() {
|
||||||
|
return std::result::Result::Ok(false);
|
||||||
|
}
|
||||||
let signal_slot = pending_signal.signal.slot;
|
let signal_slot = pending_signal.signal.slot;
|
||||||
let resolution = resolve_known_reference_hydration(hydration, settings, pending_signal.signal, pending_signal.received_at, &fetched.observed);
|
let resolution = resolve_known_reference_hydration(hydration, settings, pending_signal.signal, pending_signal.received_at, &fetched.observed);
|
||||||
let resolution = match resolution {
|
let resolution = match resolution {
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
// file: crates/ksp-worker-raw-transaction-ingest-lib/tests/dependency_boundary.rs
|
// file: crates/ksp-worker-raw-transaction-ingest-lib/tests/dependency_boundary.rs
|
||||||
// version: 29
|
// version: 30
|
||||||
|
|
||||||
//! Dependency firewall canaries for the RAW transaction ingest Worker foundation.
|
//! Dependency firewall canaries for the RAW transaction ingest Worker foundation.
|
||||||
|
|
||||||
@@ -732,3 +732,29 @@ fn v0_3_13_pre_012_cross_layer_closure_does_not_expand_worker_dependency_graph()
|
|||||||
}
|
}
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn v0_3_14_pre_012_shutdown_hardening_stays_worker_local_and_facade_only() {
|
||||||
|
let runtime = include_str!("../src/runtime.rs");
|
||||||
|
let resources = include_str!("../src/runtime_resources.rs");
|
||||||
|
for required in [
|
||||||
|
"tokio::time::timeout(settings.shutdown_drain_timeout(), drain).await",
|
||||||
|
"tokio::time::timeout(shutdown_drain_timeout, drain).await",
|
||||||
|
"children.abort_all()",
|
||||||
|
"persistence.abort_all()",
|
||||||
|
] {
|
||||||
|
assert!(runtime.contains(required) || resources.contains(required), "required pre.012 bounded shutdown guard missing: {required}");
|
||||||
|
}
|
||||||
|
for forbidden in [
|
||||||
|
"ksp_store_postgres_lib::",
|
||||||
|
"ksp_config_lib::",
|
||||||
|
"ksp_job_backfill_lib::",
|
||||||
|
"reqwest::",
|
||||||
|
"tokio_tungstenite::",
|
||||||
|
"tonic::",
|
||||||
|
"yellowstone_grpc_proto::",
|
||||||
|
] {
|
||||||
|
assert!(!runtime.contains(forbidden) && !resources.contains(forbidden), "pre.012 added forbidden Worker dependency edge: {forbidden}");
|
||||||
|
}
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
// file: crates/ksp-worker-raw-transaction-ingest-lib/tests/hardening.rs
|
// file: crates/ksp-worker-raw-transaction-ingest-lib/tests/hardening.rs
|
||||||
// version: 35
|
// version: 36
|
||||||
|
|
||||||
//! External public, security, redaction and release-boundary hardening canaries through `v0.3.14-pre.011`.
|
//! External public, security, redaction and release-boundary hardening canaries through `v0.3.14-pre.011`.
|
||||||
|
|
||||||
@@ -1313,3 +1313,42 @@ fn v0_3_14_pre_011_gap_observability_is_bounded_checked_and_redacted() {
|
|||||||
assert!(!root.contains("source_key"));
|
assert!(!root.contains("source_key"));
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn v0_3_14_pre_012_shutdown_races_are_bounded_across_source_and_worker_drains() {
|
||||||
|
let runtime = include_str!("../src/runtime.rs");
|
||||||
|
let resources = include_str!("../src/runtime_resources.rs");
|
||||||
|
for required in [
|
||||||
|
"tokio::time::timeout(settings.shutdown_drain_timeout(), drain).await",
|
||||||
|
"persistence.abort_all()",
|
||||||
|
"while persistence.join_next().await.is_some() {}",
|
||||||
|
"while children.join_next().await.is_some() {}",
|
||||||
|
] {
|
||||||
|
assert!(runtime.contains(required), "required pre.012 outer shutdown guard missing: {required}");
|
||||||
|
}
|
||||||
|
for required in [
|
||||||
|
"shutdown_drain_timeout: std::time::Duration",
|
||||||
|
"tokio::time::timeout(shutdown_drain_timeout, drain).await",
|
||||||
|
"children.abort_all()",
|
||||||
|
"ERROR_CODE_RAW_TRANSACTION_INGEST_DRAIN_TIMEOUT",
|
||||||
|
"coordinator.abort_all(&mut processing_frontier).await",
|
||||||
|
"discover_http_block_window(",
|
||||||
|
"admission_sender.send(ingress)",
|
||||||
|
"if *stop_receiver.borrow()",
|
||||||
|
] {
|
||||||
|
assert!(resources.contains(required), "required pre.012 source shutdown guard missing: {required}");
|
||||||
|
}
|
||||||
|
for forbidden in [
|
||||||
|
"unbounded_channel",
|
||||||
|
"ksp_store_postgres_lib::",
|
||||||
|
"ksp_config_lib::",
|
||||||
|
"ksp_job_backfill_lib::",
|
||||||
|
"reqwest::",
|
||||||
|
"tokio_tungstenite::",
|
||||||
|
"tonic::",
|
||||||
|
"yellowstone_grpc_proto::",
|
||||||
|
] {
|
||||||
|
assert!(!runtime.contains(forbidden) && !resources.contains(forbidden), "pre.012 crossed a Worker/facade boundary: {forbidden}");
|
||||||
|
}
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
// file: crates/ksp-worker-raw-transaction-ingest-lib/tests/release_completeness.rs
|
// file: crates/ksp-worker-raw-transaction-ingest-lib/tests/release_completeness.rs
|
||||||
// version: 30
|
// version: 31
|
||||||
|
|
||||||
//! Release-completeness canaries through the `v0.3.14-pre.011` gap observability tranche.
|
//! Release-completeness canaries through the `v0.3.14-pre.011` gap observability tranche.
|
||||||
|
|
||||||
@@ -422,3 +422,22 @@ fn v0_3_14_pre_011_gap_observability_canaries_are_present_without_source_identit
|
|||||||
assert!(!root.contains("source_key"));
|
assert!(!root.contains("source_key"));
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn v0_3_14_pre_012_shutdown_race_canaries_cover_bounded_source_and_worker_drains() {
|
||||||
|
let dependency_boundary = include_str!("dependency_boundary.rs");
|
||||||
|
let hardening = include_str!("hardening.rs");
|
||||||
|
let resource_tests = include_str!("../unit_tests/runtime_resources.rs");
|
||||||
|
let root = include_str!("../src/lib.rs");
|
||||||
|
for required in [
|
||||||
|
"v0_3_14_pre_012_source_fault_preserves_first_fault_and_aborts_non_cooperative_sibling",
|
||||||
|
"v0_3_14_pre_012_stop_aborts_non_cooperative_source_after_bounded_drain",
|
||||||
|
] {
|
||||||
|
assert!(resource_tests.contains(required), "required pre.012 source shutdown canary missing: {required}");
|
||||||
|
}
|
||||||
|
assert!(hardening.contains("v0_3_14_pre_012_shutdown_races_are_bounded_across_source_and_worker_drains"));
|
||||||
|
assert!(dependency_boundary.contains("v0_3_14_pre_012_shutdown_hardening_stays_worker_local_and_facade_only"));
|
||||||
|
assert!(!root.contains("RawTransactionIngestShutdown"));
|
||||||
|
assert!(!root.contains("SourceDrain"));
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
// file: crates/ksp-worker-raw-transaction-ingest-lib/unit_tests/runtime_resources.rs
|
// file: crates/ksp-worker-raw-transaction-ingest-lib/unit_tests/runtime_resources.rs
|
||||||
// version: 33
|
// version: 34
|
||||||
|
|
||||||
fn grpc_endpoint(cluster: &str) -> std::option::Option<ksp_onchain_transport_lib::YellowstoneGrpcEndpointSettings> {
|
fn grpc_endpoint(cluster: &str) -> std::option::Option<ksp_onchain_transport_lib::YellowstoneGrpcEndpointSettings> {
|
||||||
return grpc_endpoint_with_identity(cluster, "yellowstone-fixture", "fixture-provider");
|
return grpc_endpoint_with_identity(cluster, "yellowstone-fixture", "fixture-provider");
|
||||||
@@ -1155,6 +1155,7 @@ async fn v0_3_13_pre_007_source_supervisor_joins_all_children_on_stop() {
|
|||||||
contracts,
|
contracts,
|
||||||
inventory,
|
inventory,
|
||||||
tokio::sync::watch::channel(crate::RawTransactionIngestProcessingFrontierProjection::empty()).0,
|
tokio::sync::watch::channel(crate::RawTransactionIngestProcessingFrontierProjection::empty()).0,
|
||||||
|
std::time::Duration::from_secs(5),
|
||||||
)
|
)
|
||||||
.await;
|
.await;
|
||||||
});
|
});
|
||||||
@@ -1223,6 +1224,7 @@ async fn v0_3_13_pre_007_source_failure_stops_and_joins_sibling_sources() {
|
|||||||
contracts,
|
contracts,
|
||||||
inventory,
|
inventory,
|
||||||
tokio::sync::watch::channel(crate::RawTransactionIngestProcessingFrontierProjection::empty()).0,
|
tokio::sync::watch::channel(crate::RawTransactionIngestProcessingFrontierProjection::empty()).0,
|
||||||
|
std::time::Duration::from_secs(5),
|
||||||
)
|
)
|
||||||
.await;
|
.await;
|
||||||
assert!(result.is_err());
|
assert!(result.is_err());
|
||||||
@@ -3838,6 +3840,7 @@ async fn v0_3_13_pre_011_aborting_outer_source_supervisor_aborts_nested_source_t
|
|||||||
contracts,
|
contracts,
|
||||||
inventory,
|
inventory,
|
||||||
tokio::sync::watch::channel(crate::RawTransactionIngestProcessingFrontierProjection::empty()).0,
|
tokio::sync::watch::channel(crate::RawTransactionIngestProcessingFrontierProjection::empty()).0,
|
||||||
|
std::time::Duration::from_secs(5),
|
||||||
)
|
)
|
||||||
.await;
|
.await;
|
||||||
});
|
});
|
||||||
@@ -3905,6 +3908,7 @@ async fn v0_3_13_pre_011_stop_racing_ready_source_fault_preserves_fault_and_join
|
|||||||
contracts,
|
contracts,
|
||||||
inventory,
|
inventory,
|
||||||
tokio::sync::watch::channel(crate::RawTransactionIngestProcessingFrontierProjection::empty()).0,
|
tokio::sync::watch::channel(crate::RawTransactionIngestProcessingFrontierProjection::empty()).0,
|
||||||
|
std::time::Duration::from_secs(5),
|
||||||
)
|
)
|
||||||
.await;
|
.await;
|
||||||
});
|
});
|
||||||
@@ -3930,6 +3934,110 @@ async fn v0_3_13_pre_011_stop_racing_ready_source_fault_preserves_fault_and_join
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test(flavor = "current_thread")]
|
||||||
|
async fn v0_3_14_pre_012_source_fault_preserves_first_fault_and_aborts_non_cooperative_sibling() {
|
||||||
|
let active = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0));
|
||||||
|
let barrier = std::sync::Arc::new(tokio::sync::Barrier::new(2));
|
||||||
|
let (_stop_sender, stop_receiver) = tokio::sync::watch::channel(false);
|
||||||
|
let (source_stop_sender, _source_stop_receiver) = tokio::sync::watch::channel(false);
|
||||||
|
let contracts = match supervisor_contracts(&[(1, "fixture-a", 1), (2, "fixture-b", 2)]) {
|
||||||
|
std::option::Option::Some(value) => value,
|
||||||
|
std::option::Option::None => return,
|
||||||
|
};
|
||||||
|
let inventory = match supervisor_inventory(
|
||||||
|
std::vec![[1_u8; 32], [2_u8; 32]],
|
||||||
|
&[
|
||||||
|
std::option::Option::Some(crate::RawTransactionIngestSourceState::Failed),
|
||||||
|
std::option::Option::Some(crate::RawTransactionIngestSourceState::Active),
|
||||||
|
],
|
||||||
|
) {
|
||||||
|
std::option::Option::Some(value) => value,
|
||||||
|
std::option::Option::None => return,
|
||||||
|
};
|
||||||
|
let mut children = tokio::task::JoinSet::new();
|
||||||
|
let failing_barrier = std::sync::Arc::clone(&barrier);
|
||||||
|
let _failing_abort_handle = children.spawn(async move {
|
||||||
|
let _barrier_wait = failing_barrier.wait().await;
|
||||||
|
return ([1_u8; 32], std::result::Result::Err(crate::runtime_error("test.pre_012_source_fault")));
|
||||||
|
});
|
||||||
|
let sibling_barrier = std::sync::Arc::clone(&barrier);
|
||||||
|
let sibling_active = std::sync::Arc::clone(&active);
|
||||||
|
let _sibling_abort_handle = children.spawn(async move {
|
||||||
|
let _guard = Pre011SourceActiveGuard::new(sibling_active);
|
||||||
|
let _barrier_wait = sibling_barrier.wait().await;
|
||||||
|
return std::future::pending::<([u8; 32], ksp_core_lib::Result<()>)>().await;
|
||||||
|
});
|
||||||
|
let result = super::supervise_live_source_tasks(
|
||||||
|
stop_receiver,
|
||||||
|
source_stop_sender,
|
||||||
|
children,
|
||||||
|
contracts,
|
||||||
|
inventory,
|
||||||
|
tokio::sync::watch::channel(crate::RawTransactionIngestProcessingFrontierProjection::empty()).0,
|
||||||
|
std::time::Duration::from_millis(5),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
let error = match result {
|
||||||
|
std::result::Result::Ok(()) => return,
|
||||||
|
std::result::Result::Err(value) => value,
|
||||||
|
};
|
||||||
|
assert_eq!(error.context()[0].value(), "test.pre_012_source_fault");
|
||||||
|
assert_eq!(active.load(std::sync::atomic::Ordering::Acquire), 0);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test(flavor = "current_thread")]
|
||||||
|
async fn v0_3_14_pre_012_stop_aborts_non_cooperative_source_after_bounded_drain() {
|
||||||
|
let active = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0));
|
||||||
|
let (stop_sender, stop_receiver) = tokio::sync::watch::channel(false);
|
||||||
|
let (source_stop_sender, _source_stop_receiver) = tokio::sync::watch::channel(false);
|
||||||
|
let contracts = match supervisor_contracts(&[(1, "fixture-a", 1)]) {
|
||||||
|
std::option::Option::Some(value) => value,
|
||||||
|
std::option::Option::None => return,
|
||||||
|
};
|
||||||
|
let inventory = match supervisor_inventory(std::vec![[1_u8; 32]], &[std::option::Option::Some(crate::RawTransactionIngestSourceState::Active)]) {
|
||||||
|
std::option::Option::Some(value) => value,
|
||||||
|
std::option::Option::None => return,
|
||||||
|
};
|
||||||
|
let mut children = tokio::task::JoinSet::new();
|
||||||
|
let source_active = std::sync::Arc::clone(&active);
|
||||||
|
let _abort_handle = children.spawn(async move {
|
||||||
|
let _guard = Pre011SourceActiveGuard::new(source_active);
|
||||||
|
return std::future::pending::<([u8; 32], ksp_core_lib::Result<()>)>().await;
|
||||||
|
});
|
||||||
|
let supervisor = tokio::spawn(async move {
|
||||||
|
return super::supervise_live_source_tasks(
|
||||||
|
stop_receiver,
|
||||||
|
source_stop_sender,
|
||||||
|
children,
|
||||||
|
contracts,
|
||||||
|
inventory,
|
||||||
|
tokio::sync::watch::channel(crate::RawTransactionIngestProcessingFrontierProjection::empty()).0,
|
||||||
|
std::time::Duration::from_millis(5),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
});
|
||||||
|
for _ in 0..64 {
|
||||||
|
if active.load(std::sync::atomic::Ordering::Acquire) == 1 {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
tokio::task::yield_now().await;
|
||||||
|
}
|
||||||
|
assert_eq!(active.load(std::sync::atomic::Ordering::Acquire), 1);
|
||||||
|
stop_sender.send_replace(true);
|
||||||
|
let result = match supervisor.await {
|
||||||
|
std::result::Result::Ok(value) => value,
|
||||||
|
std::result::Result::Err(_) => return,
|
||||||
|
};
|
||||||
|
let error = match result {
|
||||||
|
std::result::Result::Ok(()) => return,
|
||||||
|
std::result::Result::Err(value) => value,
|
||||||
|
};
|
||||||
|
assert_eq!(error.code(), crate::ERROR_CODE_RAW_TRANSACTION_INGEST_DRAIN_TIMEOUT);
|
||||||
|
assert_eq!(active.load(std::sync::atomic::Ordering::Acquire), 0);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test(flavor = "current_thread")]
|
#[tokio::test(flavor = "current_thread")]
|
||||||
async fn v0_3_14_pre_008_proven_full_ledger_epoch_survives_filtered_peer_loss_without_worker_respawn() {
|
async fn v0_3_14_pre_008_proven_full_ledger_epoch_survives_filtered_peer_loss_without_worker_respawn() {
|
||||||
let sibling_active = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false));
|
let sibling_active = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false));
|
||||||
@@ -3974,7 +4082,16 @@ async fn v0_3_14_pre_008_proven_full_ledger_epoch_survives_filtered_peer_loss_wi
|
|||||||
});
|
});
|
||||||
let (health_sender, health_receiver) = tokio::sync::watch::channel(crate::RawTransactionIngestProcessingFrontierProjection::empty());
|
let (health_sender, health_receiver) = tokio::sync::watch::channel(crate::RawTransactionIngestProcessingFrontierProjection::empty());
|
||||||
let supervisor = tokio::spawn(async move {
|
let supervisor = tokio::spawn(async move {
|
||||||
return super::supervise_live_source_tasks(stop_receiver, source_stop_sender, children, contracts, inventory, health_sender).await;
|
return super::supervise_live_source_tasks(
|
||||||
|
stop_receiver,
|
||||||
|
source_stop_sender,
|
||||||
|
children,
|
||||||
|
contracts,
|
||||||
|
inventory,
|
||||||
|
health_sender,
|
||||||
|
std::time::Duration::from_secs(5),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
});
|
});
|
||||||
for _ in 0..64 {
|
for _ in 0..64 {
|
||||||
if sibling_active.load(std::sync::atomic::Ordering::Acquire) && health_receiver.borrow().failed_source_losses_reconciled() {
|
if sibling_active.load(std::sync::atomic::Ordering::Acquire) && health_receiver.borrow().failed_source_losses_reconciled() {
|
||||||
|
|||||||
237
deltas/0.3.14/pre.012.md
Normal file
237
deltas/0.3.14/pre.012.md
Normal file
@@ -0,0 +1,237 @@
|
|||||||
|
<!-- file: deltas/0.3.14/pre.012.md -->
|
||||||
|
<!-- version: 1 -->
|
||||||
|
|
||||||
|
# Delta `0.3.14-pre.012` — hardening races / shutdown
|
||||||
|
|
||||||
|
## Base requise
|
||||||
|
|
||||||
|
```text
|
||||||
|
0.3.14-pre.011-fix.002
|
||||||
|
workspace.package.version = 0.3.14-pre.11.fix.2
|
||||||
|
deltas/0.3.14/pre.011-fix.002.md présent
|
||||||
|
```
|
||||||
|
|
||||||
|
## Gate de la base
|
||||||
|
|
||||||
|
Le gate opérateur de `0.3.14-pre.011-fix.002` est validé avant ouverture de cette tranche :
|
||||||
|
|
||||||
|
```text
|
||||||
|
cargo fmt --all : PASS
|
||||||
|
cargo fmt --all -- --check : PASS
|
||||||
|
audit Rust workspace rules : PASS
|
||||||
|
audit Markdown tables : PASS
|
||||||
|
cargo check --workspace : PASS
|
||||||
|
cargo clippy --workspace --all-targets --all-features -- -D warnings : PASS
|
||||||
|
cargo test -p ksp-worker-raw-transaction-ingest-lib --all-targets --all-features : PASS
|
||||||
|
```
|
||||||
|
|
||||||
|
Le gate Worker comprend notamment :
|
||||||
|
|
||||||
|
```text
|
||||||
|
150 unit tests : PASS
|
||||||
|
cross_layer_completeness : 8 PASS
|
||||||
|
dependency_boundary : 19 PASS
|
||||||
|
hardening : 36 PASS
|
||||||
|
public_api : 21 PASS
|
||||||
|
release_completeness : 13 PASS
|
||||||
|
```
|
||||||
|
|
||||||
|
## Objectif
|
||||||
|
|
||||||
|
Implémenter strictement la tranche `pre.012` du plan `035` :
|
||||||
|
|
||||||
|
```text
|
||||||
|
borner stop/fault pendant les sous-tâches source et les phases de continuité
|
||||||
|
préserver la première faute quand un sibling ne coopère pas au shutdown
|
||||||
|
abandonner et rejoindre les tâches source restantes après expiration de la deadline
|
||||||
|
empêcher une mutation de coverage après observation d'un stop HTTP
|
||||||
|
empêcher une hydration déjà jointe de poursuivre admission/continuity après observation d'un stop
|
||||||
|
conserver le drain/abort/join global déjà validé pour admission/persistence
|
||||||
|
ne créer aucun task pool, pipeline, timeout ou ownership parallèle
|
||||||
|
```
|
||||||
|
|
||||||
|
## Drain interne des sources désormais borné
|
||||||
|
|
||||||
|
Avant cette tranche, le drain global Worker était borné par `shutdown_drain_timeout`, mais le superviseur interne des sources pouvait encore attendre indéfiniment un sibling non coopératif après une faute de source.
|
||||||
|
|
||||||
|
`supervise_live_source_tasks` reçoit maintenant la même deadline déjà configurée dans `RawTransactionIngestSettings` :
|
||||||
|
|
||||||
|
```text
|
||||||
|
settings.shutdown_drain_timeout()
|
||||||
|
```
|
||||||
|
|
||||||
|
`drain_live_source_tasks` applique :
|
||||||
|
|
||||||
|
```text
|
||||||
|
tokio::time::timeout(shutdown_drain_timeout, drain)
|
||||||
|
```
|
||||||
|
|
||||||
|
En cas d'expiration :
|
||||||
|
|
||||||
|
```text
|
||||||
|
children.abort_all()
|
||||||
|
join de tous les children annulés
|
||||||
|
```
|
||||||
|
|
||||||
|
Aucune tâche source ne reste donc détenue après retour du superviseur interne.
|
||||||
|
|
||||||
|
## Priorité de faute
|
||||||
|
|
||||||
|
Si le shutdown interne commence après une faute source déjà observée et que le sibling ne coopère pas avant la deadline :
|
||||||
|
|
||||||
|
```text
|
||||||
|
la première faute source reste la faute retournée
|
||||||
|
```
|
||||||
|
|
||||||
|
Le timeout ne remplace pas une cause terminale déjà connue.
|
||||||
|
|
||||||
|
Si aucun défaut antérieur n'existe et qu'un stop coopératif ne peut pas joindre une source avant la deadline :
|
||||||
|
|
||||||
|
```text
|
||||||
|
ERROR_CODE_RAW_TRANSACTION_INGEST_DRAIN_TIMEOUT
|
||||||
|
```
|
||||||
|
|
||||||
|
est retourné.
|
||||||
|
|
||||||
|
Cette règle aligne le superviseur source avec le drain Worker global déjà validé.
|
||||||
|
|
||||||
|
## Stop avant mutation de coverage HTTP
|
||||||
|
|
||||||
|
Le polling HTTP vérifie maintenant explicitement le stop après le traitement des blocs découverts et avant :
|
||||||
|
|
||||||
|
```text
|
||||||
|
record_coverage_epoch(...)
|
||||||
|
processing_frontier.publish()
|
||||||
|
```
|
||||||
|
|
||||||
|
Un stop déjà observé ne peut donc pas être suivi d'une nouvelle preuve de coverage de cette fenêtre.
|
||||||
|
|
||||||
|
Les appels réseau restent stop-preemptible via les `tokio::select!` existants ; aucun retry Worker n'est ajouté.
|
||||||
|
|
||||||
|
## Stop pendant hydration/admission
|
||||||
|
|
||||||
|
`RawTransactionIngestHydrationCoordinator::handle_joined` vérifie maintenant le stop :
|
||||||
|
|
||||||
|
```text
|
||||||
|
avant de retirer/appliquer le résultat d'hydration joint
|
||||||
|
avant chaque signal coalescé de ce résultat
|
||||||
|
```
|
||||||
|
|
||||||
|
Les `tokio::select!` existants restent en place autour de l'admission async.
|
||||||
|
|
||||||
|
Un stop déjà observé ne poursuit donc pas la mutation continuity ou l'admission d'un résultat d'hydration devenu prêt au même moment.
|
||||||
|
|
||||||
|
## Invariants préservés
|
||||||
|
|
||||||
|
```text
|
||||||
|
Transport reste propriétaire des sockets, reconnects et replay natif
|
||||||
|
Worker ne modifie jamais from_slot
|
||||||
|
aucun respawn de source
|
||||||
|
aucun second scheduler ou sémaphore
|
||||||
|
fairness pre.010 inchangée
|
||||||
|
observabilité pre.011 inchangée
|
||||||
|
Common RAW/admission/persistence restent uniques
|
||||||
|
Store reste accessible via ksp-store-lib seulement
|
||||||
|
aucun Worker -> Config/Job/backend Store
|
||||||
|
aucune nouvelle dépendance
|
||||||
|
```
|
||||||
|
|
||||||
|
## Tests ajoutés
|
||||||
|
|
||||||
|
Deux canaris runtime reproduisent les races que cette tranche ferme :
|
||||||
|
|
||||||
|
```text
|
||||||
|
v0_3_14_pre_012_source_fault_preserves_first_fault_and_aborts_non_cooperative_sibling
|
||||||
|
v0_3_14_pre_012_stop_aborts_non_cooperative_source_after_bounded_drain
|
||||||
|
```
|
||||||
|
|
||||||
|
Le premier prouve :
|
||||||
|
|
||||||
|
```text
|
||||||
|
sibling réellement démarré
|
||||||
|
faute source observée
|
||||||
|
sibling volontairement non coopératif
|
||||||
|
deadline interne atteinte
|
||||||
|
abort + join du sibling
|
||||||
|
première faute préservée
|
||||||
|
aucune tâche active après retour
|
||||||
|
```
|
||||||
|
|
||||||
|
Le second prouve :
|
||||||
|
|
||||||
|
```text
|
||||||
|
stop coopératif demandé
|
||||||
|
source volontairement non coopérative
|
||||||
|
deadline interne atteinte
|
||||||
|
ERROR_CODE_RAW_TRANSACTION_INGEST_DRAIN_TIMEOUT
|
||||||
|
abort + join
|
||||||
|
aucune tâche active après retour
|
||||||
|
```
|
||||||
|
|
||||||
|
Les canaris `hardening`, `dependency_boundary` et `release_completeness` vérifient en plus :
|
||||||
|
|
||||||
|
```text
|
||||||
|
deadline source et deadline Worker présentes
|
||||||
|
abort_all + join présents aux deux niveaux
|
||||||
|
checkpoints stop avant les mutations repair/admission concernées
|
||||||
|
aucune nouvelle dépendance ni frontière backend
|
||||||
|
aucune croissance de surface publique shutdown
|
||||||
|
```
|
||||||
|
|
||||||
|
## Hors périmètre inchangé
|
||||||
|
|
||||||
|
```text
|
||||||
|
aucune nouvelle stratégie de repair
|
||||||
|
aucune nouvelle méthode HTTP
|
||||||
|
aucun changement de health policy
|
||||||
|
aucun changement de snapshot public
|
||||||
|
aucun changement de TargetCoverage
|
||||||
|
aucun changement de fairness
|
||||||
|
aucun changement de persistence semantics
|
||||||
|
aucun EARLY/shred
|
||||||
|
aucun Job Backfill depuis Worker
|
||||||
|
```
|
||||||
|
|
||||||
|
## Fichiers ajoutés
|
||||||
|
|
||||||
|
```text
|
||||||
|
deltas/0.3.14/pre.012.md
|
||||||
|
```
|
||||||
|
|
||||||
|
## Fichiers modifiés
|
||||||
|
|
||||||
|
```text
|
||||||
|
Cargo.toml
|
||||||
|
crates/ksp-worker-raw-transaction-ingest-lib/src/runtime_resources.rs
|
||||||
|
crates/ksp-worker-raw-transaction-ingest-lib/unit_tests/runtime_resources.rs
|
||||||
|
crates/ksp-worker-raw-transaction-ingest-lib/tests/dependency_boundary.rs
|
||||||
|
crates/ksp-worker-raw-transaction-ingest-lib/tests/hardening.rs
|
||||||
|
crates/ksp-worker-raw-transaction-ingest-lib/tests/release_completeness.rs
|
||||||
|
```
|
||||||
|
|
||||||
|
## Fichiers supprimés
|
||||||
|
|
||||||
|
```text
|
||||||
|
aucun
|
||||||
|
```
|
||||||
|
|
||||||
|
## Version Cargo
|
||||||
|
|
||||||
|
Conformément au workflow prerelease non-fix :
|
||||||
|
|
||||||
|
```text
|
||||||
|
header Cargo.toml : 586 -> 587
|
||||||
|
workspace.package.version : 0.3.14-pre.11.fix.2 -> 0.3.14-pre.12
|
||||||
|
```
|
||||||
|
|
||||||
|
## Gate opérateur après application
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cargo fmt --all
|
||||||
|
cargo fmt --all -- --check
|
||||||
|
python3 scripts/audit_rust_workspace_rules.py
|
||||||
|
python3 scripts/audit_markdown_tables.py README.md RULES.md ROADMAP.md CHANGELOG.md docs prompts crates deltas
|
||||||
|
cargo check --workspace
|
||||||
|
cargo clippy --workspace --all-targets --all-features -- -D warnings
|
||||||
|
cargo test -p ksp-worker-raw-transaction-ingest-lib --all-targets --all-features
|
||||||
|
```
|
||||||
Reference in New Issue
Block a user