v0.3.12-pre.009

This commit is contained in:
2026-09-09 22:06:18 +02:00
parent 223f11b989
commit f9d4e6bc74
6 changed files with 447 additions and 22 deletions

View File

@@ -1,12 +1,12 @@
# file: Cargo.toml
# version: 532
# version: 533
[workspace]
resolver = "3"
members = ["crates/ksp-app-backfill-desk", "crates/ksp-app-config-desk", "crates/ksp-app-solprices-desk", "crates/ksp-app-store-desk", "crates/ksp-app-wallet-desk", "crates/ksp-config-lib", "crates/ksp-core-lib", "crates/ksp-interface-lib", "crates/ksp-job-api", "crates/ksp-job-backfill-lib", "crates/ksp-logging-lib", "crates/ksp-offchain-transport-lib", "crates/ksp-onchain-transport-lib", "crates/ksp-program-api", "crates/ksp-raw-transaction-lib", "crates/ksp-store-api", "crates/ksp-store-lib", "crates/ksp-store-postgres-lib", "crates/ksp-wallet-lib", "crates/ksp-worker-api", "crates/ksp-worker-raw-transaction-ingest-lib"]
[workspace.package]
version = "0.3.12-pre.8.fix.2"
version = "0.3.12-pre.9"
edition = "2024"
license = "MIT"
repository = "https://git.sasedev.com/Sasedev/khadhroony-solana-project"

View File

@@ -1,5 +1,5 @@
// file: crates/ksp-worker-raw-transaction-ingest-lib/src/runtime_resources.rs
// version: 11
// version: 12
use sha2::Digest; // rust-rules: trait-import
@@ -472,7 +472,7 @@ impl crate::RawTransactionIngestYellowstoneSource {
}
}
}
coordinator.abort_all().await;
coordinator.abort_all(&mut processing_frontier).await;
processing_frontier.set_source_state(crate::RawTransactionIngestSourceState::Closing);
let closed = session.close().await;
if let std::option::Option::Some(error) = fault {
@@ -850,22 +850,36 @@ impl RawTransactionIngestProcessingFrontier {
std::option::Option::Some(value) => value,
std::option::Option::None => return std::result::Result::Err(crate::runtime_error("source.frontier_pending_counter_exhausted")),
};
let state = self.slots.entry(slot).or_insert(RawTransactionIngestProcessingSlotState { pending: 0, settled: 0 });
state.pending = match state.pending.checked_add(1) {
std::option::Option::Some(value) => value,
std::option::Option::None => return std::result::Result::Err(crate::runtime_error("source.frontier_slot_pending_counter_exhausted")),
};
match self.slots.entry(slot) {
std::collections::btree_map::Entry::Occupied(mut entry) => {
let pending = match entry.get().pending.checked_add(1) {
std::option::Option::Some(value) => value,
std::option::Option::None => return std::result::Result::Err(crate::runtime_error("source.frontier_slot_pending_counter_exhausted")),
};
entry.get_mut().pending = pending;
},
std::collections::btree_map::Entry::Vacant(entry) => {
entry.insert(RawTransactionIngestProcessingSlotState { pending: 1, settled: 0 });
},
}
self.pending_total = pending_total;
self.compact();
return std::result::Result::Ok(());
}
fn observe_settled(&mut self, slot: u64) -> ksp_core_lib::Result<()> {
let state = self.slots.entry(slot).or_insert(RawTransactionIngestProcessingSlotState { pending: 0, settled: 0 });
state.settled = match state.settled.checked_add(1) {
std::option::Option::Some(value) => value,
std::option::Option::None => return std::result::Result::Err(crate::runtime_error("source.frontier_slot_settled_counter_exhausted")),
};
match self.slots.entry(slot) {
std::collections::btree_map::Entry::Occupied(mut entry) => {
let settled = match entry.get().settled.checked_add(1) {
std::option::Option::Some(value) => value,
std::option::Option::None => return std::result::Result::Err(crate::runtime_error("source.frontier_slot_settled_counter_exhausted")),
};
entry.get_mut().settled = settled;
},
std::collections::btree_map::Entry::Vacant(entry) => {
entry.insert(RawTransactionIngestProcessingSlotState { pending: 0, settled: 1 });
},
}
self.compact();
return std::result::Result::Ok(());
}
@@ -878,16 +892,26 @@ impl RawTransactionIngestProcessingFrontier {
if state.pending == 0 || self.pending_total == 0 {
return std::result::Result::Err(crate::runtime_error("source.frontier_pending_counter_invalid"));
}
state.pending -= 1;
self.pending_total -= 1;
state.settled = match state.settled.checked_add(1) {
let settled = match state.settled.checked_add(1) {
std::option::Option::Some(value) => value,
std::option::Option::None => return std::result::Result::Err(crate::runtime_error("source.frontier_slot_settled_counter_exhausted")),
};
state.pending -= 1;
self.pending_total -= 1;
state.settled = settled;
self.compact();
return std::result::Result::Ok(());
}
fn discard_all_pending(&mut self) {
self.pending_total = 0;
self.slots.retain(|_slot, state| {
return state.pending == 0 && state.settled > 0;
});
self.compact();
return;
}
fn projection(&self) -> crate::RawTransactionIngestProcessingFrontierProjection {
let oldest_pending_slot = self.slots.iter().find_map(|(slot, state)| {
if state.pending == 0 {
@@ -1006,6 +1030,12 @@ impl RawTransactionIngestProcessingFrontierReporter {
return std::result::Result::Ok(());
}
fn discard_all_pending(&mut self) {
self.frontier.discard_all_pending();
self.publish();
return;
}
fn observe_session_snapshot(&mut self, snapshot: ksp_onchain_transport_lib::YellowstoneGrpcSubscribeSnapshot) -> ksp_core_lib::Result<()> {
return self.observe_source_continuity(
map_yellowstone_source_state(snapshot.state()),
@@ -1240,11 +1270,12 @@ impl RawTransactionIngestHydrationCoordinator {
return std::result::Result::Ok(true);
}
async fn abort_all(&mut self) {
async fn abort_all(&mut self, processing_frontier: &mut RawTransactionIngestProcessingFrontierReporter) {
self.tasks.abort_all();
while self.tasks.join_next().await.is_some() {}
self.pending.clear();
self.pending_signal_count = 0;
processing_frontier.discard_all_pending();
return;
}
}

View File

@@ -1,5 +1,5 @@
// file: crates/ksp-worker-raw-transaction-ingest-lib/tests/hardening.rs
// version: 10
// version: 11
//! External public, security, redaction and release-boundary hardening canaries for `pre.010`.
@@ -522,3 +522,22 @@ fn v0_3_12_pre_008_reconnect_projection_is_source_neutral_bounded_and_contains_n
}
return;
}
#[test]
fn v0_3_12_pre_009_hydration_retry_ownership_and_no_orphan_cleanup_are_explicit() {
let resources = include_str!("../src/runtime_resources.rs");
assert_eq!(resources.matches("get_transaction_observed(").count(), 1, "Worker must delegate one hydration attempt to Transport");
for required in [
"coordinator.abort_all(&mut processing_frontier).await",
"fn discard_all_pending(&mut self)",
"processing_frontier.discard_all_pending()",
"source.hydration_pending_saturated",
"tasks.abort_all()",
] {
assert!(resources.contains(required), "required pre.009 no-orphan/backpressure guard missing: {required}");
}
for forbidden in ["tokio::time::sleep", "tokio::time::interval", "get_block_observed", "unbounded_channel"] {
assert!(!resources.contains(forbidden), "Worker introduced forbidden retry/unbounded behavior: {forbidden}");
}
return;
}

View File

@@ -1,5 +1,5 @@
// file: crates/ksp-worker-raw-transaction-ingest-lib/unit_tests/runtime_resources.rs
// version: 9
// version: 10
fn grpc_endpoint(cluster: &str) -> std::option::Option<ksp_onchain_transport_lib::YellowstoneGrpcEndpointSettings> {
return grpc_endpoint_with_identity(cluster, "yellowstone-fixture", "fixture-provider");
@@ -1289,7 +1289,7 @@ async fn pre_006_coalescence_key_merges_transaction_and_status_before_http_fanou
assert_eq!(coordinator.pending.len(), 1);
assert_eq!(coordinator.pending_signal_count, 2);
assert_eq!(coordinator.tasks.len(), 1);
coordinator.abort_all().await;
coordinator.abort_all(&mut processing_frontier).await;
assert_eq!(coordinator.pending_signal_count, 0);
assert!(coordinator.pending.is_empty());
assert!(coordinator.tasks.is_empty());
@@ -1412,3 +1412,125 @@ fn pre_008_reconnect_replay_and_proven_gap_are_distinct_monotone_and_frontier_pr
}));
return;
}
#[tokio::test(flavor = "current_thread")]
async fn pre_009_duplicate_storm_is_bounded_coalesced_and_abort_leaves_no_orphan_frontier() {
let source = match signal_source() {
std::option::Option::Some(value) => value,
std::option::Option::None => return,
};
let settings = match pre_004_settings() {
std::option::Option::Some(value) => value,
std::option::Option::None => return,
};
let signal = match pre_004_signal(&source, super::RawTransactionIngestSourceFamily::Transaction, 10, 1, &["storm"], 0x55) {
std::option::Option::Some(value) => value,
std::option::Option::None => return,
};
let received_at = match ksp_store_lib::RawTimestamp::from_unix_millis(1_760_000_300_000) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return,
};
let mut coordinator = super::RawTransactionIngestHydrationCoordinator::new(&settings);
coordinator.max_in_flight = 0;
coordinator.max_pending_signals = 3;
let (frontier_sender, frontier_receiver) = tokio::sync::watch::channel(crate::RawTransactionIngestProcessingFrontierProjection::empty());
let mut processing_frontier = super::RawTransactionIngestProcessingFrontierReporter::new(frontier_sender);
assert!(processing_frontier.observe_settled(9).is_ok());
assert!(processing_frontier.observe_settled(10).is_ok());
for _ in 0..3 {
assert!(coordinator.queue_signal(&source, signal.clone(), received_at, &mut processing_frontier).is_ok());
}
let saturated = coordinator.queue_signal(&source, signal, received_at, &mut processing_frontier);
let saturated = match saturated {
std::result::Result::Ok(()) => return,
std::result::Result::Err(value) => value,
};
assert!(saturated.context().iter().any(|context| {
return context.value() == "source.hydration_pending_saturated";
}));
assert_eq!(coordinator.pending.len(), 1);
assert_eq!(coordinator.pending_signal_count, 3);
assert!(coordinator.tasks.is_empty());
let pending = *frontier_receiver.borrow();
assert_eq!(pending.hydration_pending(), 3);
assert_eq!(pending.oldest_pending_slot(), std::option::Option::Some(10));
assert_eq!(pending.processing_frontier_slot(), std::option::Option::Some(9));
coordinator.abort_all(&mut processing_frontier).await;
assert!(coordinator.pending.is_empty());
assert_eq!(coordinator.pending_signal_count, 0);
assert!(coordinator.tasks.is_empty());
let cleaned = *frontier_receiver.borrow();
assert_eq!(cleaned.hydration_pending(), 0);
assert_eq!(cleaned.oldest_pending_slot(), std::option::Option::None);
assert_eq!(cleaned.processing_frontier_slot(), std::option::Option::Some(9));
return;
}
#[test]
fn pre_009_frontier_counter_exhaustion_is_transactional_before_abort_cleanup() {
let mut frontier = super::RawTransactionIngestProcessingFrontier::new();
frontier.pending_total = 1;
frontier.slots.insert(77, super::RawTransactionIngestProcessingSlotState { pending: 1, settled: u64::MAX });
let before = frontier.projection();
let exhausted = frontier.settle_pending(77);
let exhausted = match exhausted {
std::result::Result::Ok(()) => return,
std::result::Result::Err(value) => value,
};
assert!(exhausted.context().iter().any(|context| {
return context.value() == "source.frontier_slot_settled_counter_exhausted";
}));
let after = frontier.projection();
assert_eq!(after.hydration_pending(), before.hydration_pending());
assert_eq!(after.oldest_pending_slot(), before.oldest_pending_slot());
assert_eq!(after.processing_frontier_slot(), before.processing_frontier_slot());
let state = match frontier.slots.get(&77) {
std::option::Option::Some(value) => value,
std::option::Option::None => return,
};
assert_eq!(state.pending, 1);
assert_eq!(state.settled, u64::MAX);
frontier.discard_all_pending();
let cleaned = frontier.projection();
assert_eq!(cleaned.hydration_pending(), 0);
assert_eq!(cleaned.oldest_pending_slot(), std::option::Option::None);
assert_eq!(cleaned.processing_frontier_slot(), std::option::Option::None);
return;
}
#[tokio::test(flavor = "current_thread")]
async fn pre_009_abort_joins_in_flight_hydration_and_clears_pending_projection() {
let source = match signal_source() {
std::option::Option::Some(value) => value,
std::option::Option::None => return,
};
let settings = match pre_004_settings() {
std::option::Option::Some(value) => value,
std::option::Option::None => return,
};
let signal = match pre_004_signal(&source, super::RawTransactionIngestSourceFamily::TransactionStatus, 81, 2, &["stop-hydration"], 0x66) {
std::option::Option::Some(value) => value,
std::option::Option::None => return,
};
let received_at = match ksp_store_lib::RawTimestamp::from_unix_millis(1_760_000_301_000) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return,
};
let mut coordinator = super::RawTransactionIngestHydrationCoordinator::new(&settings);
let (frontier_sender, frontier_receiver) = tokio::sync::watch::channel(crate::RawTransactionIngestProcessingFrontierProjection::empty());
let mut processing_frontier = super::RawTransactionIngestProcessingFrontierReporter::new(frontier_sender);
assert!(coordinator.queue_signal(&source, signal, received_at, &mut processing_frontier).is_ok());
assert_eq!(coordinator.pending_signal_count, 1);
assert_eq!(coordinator.tasks.len(), 1);
assert_eq!(frontier_receiver.borrow().hydration_pending(), 1);
coordinator.abort_all(&mut processing_frontier).await;
assert_eq!(coordinator.pending_signal_count, 0);
assert!(coordinator.pending.is_empty());
assert!(coordinator.tasks.is_empty());
let cleaned = *frontier_receiver.borrow();
assert_eq!(cleaned.hydration_pending(), 0);
assert_eq!(cleaned.oldest_pending_slot(), std::option::Option::None);
assert_eq!(cleaned.processing_frontier_slot(), std::option::Option::None);
return;
}

152
deltas/0.3.12/pre.009.md Normal file
View File

@@ -0,0 +1,152 @@
<!-- file: deltas/0.3.12/pre.009.md -->
<!-- version: 1 -->
# Delta `0.3.12-pre.009` — hardening races/retry/backpressure et no-orphan
## Base requise
```text
0.3.12-pre.008-fix.002
workspace.package.version = 0.3.12-pre.8.fix.2
```
Le gate opérateur communiqué pour cette base est entièrement vert : audits Rust/Markdown, `cargo check --workspace`, Clippy strict, 388 tests unitaires Transport, 51 `public_api` Transport, 43 `release_completeness` Transport, 63 tests unitaires Worker, 9 `dependency_boundary`, 15 `hardening`, 9 `public_api` et 3 `release_completeness` Worker sans échec.
## Objectif
Fermer les races terminales prévues pour `pre.009` sans élargir le pipeline :
```text
stop/fault pendant hydration
pending/coalescence abortée
processing frontier conservative
retry ownership Transport-only
duplicate storm bornée
counter exhaustion transactionnelle
no-orphan final
```
## Version
```text
workspace.package.version = 0.3.12-pre.9
```
## No-orphan source/hydration
`RawTransactionIngestHydrationCoordinator::abort_all` reçoit désormais le reporter de processing frontier. Le cleanup :
```text
abort_all des tâches hydration
join de toutes les tâches hydration
clear de la map pending
pending_signal_count = 0
discard_all_pending de la frontier
```
`discard_all_pending` ne convertit jamais un pending en settled. Toute slot qui contenait encore au moins un pending est retirée de la projection conservative, même si un autre signal de cette slot avait déjà été settled.
Cela empêche une frontier terminale d'avancer à travers du travail abandonné lors d'un stop, d'un source fault, d'une erreur hydration ou d'un gap de continuité prouvé.
## Arithmetic transactionnelle
Les mutations de `RawTransactionIngestProcessingFrontier` sont durcies :
```text
observe_pending : overflow vérifié avant mutation de l'entrée
observe_settled : overflow vérifié avant mutation de l'entrée
settle_pending : settled.checked_add avant décrément pending
```
Un `source.frontier_slot_settled_counter_exhausted` laisse donc le pending et la projection inchangés ; aucun état partiellement muté n'est publié.
## Duplicate storm et backpressure
Le canari source crée une petite borne déterministe et injecte plusieurs signaux identiques :
```text
hydration in-flight -> abort/join puis 0 tâche et 0 pending
3 signaux identiques -> 1 clé coalescée, 3 pending
4e signal -> source.hydration_pending_saturated
aucune tâche HTTP supplémentaire
abort -> 0 pending, 0 tâche, frontier inchangée au dernier niveau sûr
```
La borne productive existante reste inchangée et aucun canal unbounded n'est ajouté.
## Retry ownership
Le Worker conserve exactement un appel `get_transaction_observed` dans son chemin hydration et n'ajoute aucun `sleep`/`interval` de retry. Les retries réseau, backoff, rate-limit et reroutage restent exclusivement dans Transport.
Les tests runtime existants continuent de qualifier Store lent, source failure, stop pendant drain, saturation centrale, drain timeout et counter exhaustion snapshot.
## Fichiers modifiés
```text
Cargo.toml
crates/ksp-worker-raw-transaction-ingest-lib/src/runtime_resources.rs
crates/ksp-worker-raw-transaction-ingest-lib/tests/hardening.rs
crates/ksp-worker-raw-transaction-ingest-lib/unit_tests/runtime_resources.rs
docs/validation/029-V0_3_12_YELLOWSTONE_HYDRATION_CONTINUITY.md
```
## Fichier ajouté
```text
deltas/0.3.12/pre.009.md
```
## Fichiers supprimés
```text
aucun
```
## Non-changements
`pre.009` n'ajoute ni ne modifie :
```text
API publique Worker
snapshot public
Store/Config/Backfill
Common RAW
Transport replay semantics
from_slot / ReplayInfo
repair/failover
retry Worker
checkpoint durable
```
## Validations exécutées dans l'environnement d'assemblage
```text
python3 scripts/audit_rust_workspace_rules.py
python3 scripts/audit_markdown_tables.py README.md RULES.md ROADMAP.md CHANGELOG.md docs prompts crates deltas
canaris statiques pre.009 no-orphan/retry ownership
contrôle des versions de fichiers modifiés
contrôle diff exact contre pre.008-fix.002
unzip -t du delta final
reproduction du delta sur la base exacte
```
`cargo`, `rustc` et `rustfmt` ne sont pas disponibles dans l'environnement d'assemblage. Aucun gate Cargo local n'est revendiqué.
## Gate opérateur requis
```bash
cargo fmt --all
python3 scripts/audit_rust_workspace_rules.py
python3 scripts/audit_markdown_tables.py README.md RULES.md ROADMAP.md CHANGELOG.md docs prompts crates deltas
cargo check --workspace
cargo clippy --workspace --all-targets --all-features -- -D warnings
cargo test -p ksp-onchain-transport-lib
cargo test -p ksp-worker-raw-transaction-ingest-lib
cargo tree -p ksp-worker-raw-transaction-ingest-lib --edges normal
cargo tree -p ksp-worker-raw-transaction-ingest-lib -e features
cargo tree --duplicates
```
## Suite
`pre.010` reste consacré à la completeness/security cross-layer : fixtures Legacy/V0, dependency firewall, public API exact, redaction, release completeness et scanners, sans nouveau scope fonctionnel.

View File

@@ -1,5 +1,5 @@
<!-- file: docs/validation/029-V0_3_12_YELLOWSTONE_HYDRATION_CONTINUITY.md -->
<!-- version: 18 -->
<!-- version: 19 -->
# Validation v0.3.12 — Yellowstone + hydration HTTP + continuité de run du Worker RawTransaction
@@ -1705,3 +1705,104 @@ history remediation remains outside this crate
```
Le correctif ne modifie aucun comportement reconnect/replay, aucune ownership de `from_slot`/`ReplayInfo`, aucune processing frontier, hydration, admission, persistence ou API publique.
## 73. Gate opérateur `pre.008-fix.002`
Le gate opérateur communiqué pour `0.3.12-pre.8.fix.2` est entièrement vert :
```text
General Rust rule audit : clean
Rust export completeness audit : 0 candidate(s)
KSP workspace Rust rule audit : clean
Markdown table audit : clean (340 tables, 817 fichiers)
cargo check --workspace : PASS
cargo clippy --workspace --all-targets --all-features -- -D warnings : PASS
ksp-onchain-transport-lib : 388 unit PASS, 51 public_api PASS, 43 release_completeness PASS, doc-tests PASS
ksp-worker-raw-transaction-ingest-lib : 63 unit PASS, 9 dependency_boundary PASS, 15 hardening PASS, 9 public_api PASS, 3 release_completeness PASS
```
Les smokes réseau restent `ignored` et operator-only. `pre.008-fix.002` devient la base autoritaire de `pre.009`.
## 74. No-orphan et frontier conservative `pre.009`
Le cleanup terminal du source task conserve l'ordre suivant :
```text
stop/fault source
-> abort_all des hydrations privées
-> join de toutes les tâches privées
-> clear de la coalescence/pending source
-> abandon des pending de processing frontier
-> Closing
-> close session Transport
-> Closed ou Failed
```
L'abandon d'un pending n'est jamais transformé en `settled`. `discard_all_pending` supprime de la frontier toute slot qui possédait encore du travail pending, même si d'autres signaux de cette même slot avaient déjà été settled. La dernière `processing_frontier_slot` publiée reste donc conservative et ne peut pas avancer à travers du travail abandonné pendant stop/fault.
Aucun nouvel état public ou compteur n'est ajouté.
## 75. Hardening races/retry/backpressure `pre.009`
Les nouveaux canaris déterministes couvrent :
```text
duplicate storm bornée par max_pending_signals
hydration réellement in-flight -> abort/join puis projection pending nettoyée
même network/signature/commitment -> une seule entrée de coalescence
saturation -> source.hydration_pending_saturated, aucun drop silencieux
abort -> pending_signal_count = 0, map vide, JoinSet vide
abort -> hydration_pending = 0 sans faux settlement de la slot abandonnée
settle_pending overflow -> aucune mutation partielle du pending/frontier
un seul appel Worker get_transaction_observed
aucun timer sleep/interval de retry Worker
```
Le retry HTTP reste exclusivement propriété de `ksp-onchain-transport-lib`. Le Worker ne superpose donc aucun retry loop au retry Transport.
Les canaris runtime déjà présents depuis le socle `0.3.11` restent les preuves de :
```text
Store lent -> backpressure observable sans drop ni admission unbounded
source failure -> compteur source_failure et terminal fault stable
stop pendant drain -> une erreur Store déjà observée n'est pas masquée
drain timeout -> abort/join de tous les source/persistence tasks possédés
counter exhaustion snapshot -> ERROR_CODE_RAW_TRANSACTION_INGEST_COUNTER_EXHAUSTED sans wrap
```
Les fixtures hydration existantes restent les preuves de `getTransaction null -> Missing` et des erreurs de validation/hydration terminales sans payload distant.
## 76. Non-claims et gate `pre.009`
`pre.009` ne modifie pas :
```text
Transport reconnect/from_slot/ReplayInfo
politique proven retention gap -> source fault
provenance composite
format Common RAW
Store API/backend
processing frontier publique
API publique Worker
Config ou Backfill
```
Aucun retry missing futur, failover multi-provider, repair automatique, checkpoint durable ou exactly-once n'est introduit.
Gate opérateur requis :
```bash
cargo fmt --all
python3 scripts/audit_rust_workspace_rules.py
python3 scripts/audit_markdown_tables.py README.md RULES.md ROADMAP.md CHANGELOG.md docs prompts crates deltas
cargo check --workspace
cargo clippy --workspace --all-targets --all-features -- -D warnings
cargo test -p ksp-onchain-transport-lib
cargo test -p ksp-worker-raw-transaction-ingest-lib
cargo tree -p ksp-worker-raw-transaction-ingest-lib --edges normal
cargo tree -p ksp-worker-raw-transaction-ingest-lib -e features
cargo tree --duplicates
```
Dans l'environnement d'assemblage, `cargo`, `rustc` et `rustfmt` ne sont pas installés ; aucun gate Cargo local n'est revendiqué.