v0.3.14-pre.010

crates/ksp-worker-raw-transaction-ingest-lib/src/runtime_resources.rs

@@ -1,5 +1,5 @@
 // file: crates/ksp-worker-raw-transaction-ingest-lib/src/runtime_resources.rs
-// version: 38
+// version: 39
 
 use sha2::Digest; // rust-rules: trait-import
 
@@ -18,6 +18,7 @@ pub const MIN_RAW_TRANSACTION_INGEST_HTTP_POLL_INTERVAL: std::time::Duration = s
 /// Minimum number of confirmed blocks accepted during one HTTP live polling cycle.
 pub const MIN_RAW_TRANSACTION_INGEST_HTTP_POLL_MAX_BLOCKS_PER_CYCLE: u16 = 1;
 
+const MAX_RAW_TRANSACTION_INGEST_REPAIR_BURST: usize = 1;
 const RAW_TRANSACTION_INGEST_HELIUS_TRANSACTION_FILTER_FINGERPRINT_DOMAIN: &[u8] = b"ksp.raw_transaction_ingest.helius_transaction.filter.v1\0";
 const RAW_TRANSACTION_INGEST_HELIUS_TRANSACTION_HTTP_PROTOCOL: &str = "helius_ws_http";
 const RAW_TRANSACTION_INGEST_HELIUS_TRANSACTION_HTTP_SOURCE_KEY_DOMAIN: &[u8] = b"ksp.raw_transaction_ingest.helius_transaction_http.source_key.v1\0";
@@ -36,6 +37,197 @@ const RAW_TRANSACTION_INGEST_YELLOWSTONE_FILTER_FINGERPRINT_DOMAIN: &[u8] = b"ks
 const RAW_TRANSACTION_INGEST_YELLOWSTONE_HTTP_PROTOCOL: &str = "yellowstone_http";
 const RAW_TRANSACTION_INGEST_YELLOWSTONE_HTTP_SOURCE_KEY_DOMAIN: &[u8] = b"ksp.raw_transaction_ingest.yellowstone_http.source_key.v1\0";
 
+#[derive(Clone, Copy, Debug, Eq, PartialEq)]
+enum RawTransactionIngestTrafficClass {
+    Nominal,
+    Repair,
+}
+
+struct RawTransactionIngestFairTurnState {
+    active: bool,
+    last_granted: std::option::Option<RawTransactionIngestTrafficClass>,
+    next_ticket: u64,
+    waiters: std::collections::VecDeque<(u64, RawTransactionIngestTrafficClass)>,
+}
+
+struct RawTransactionIngestFairTurnGate {
+    max_waiters: usize,
+    notify: tokio::sync::Notify,
+    state: std::sync::Mutex<RawTransactionIngestFairTurnState>,
+}
+
+struct RawTransactionIngestFairWaiterGuard {
+    armed: bool,
+    gate: std::sync::Arc<RawTransactionIngestFairTurnGate>,
+    ticket: u64,
+}
+
+struct RawTransactionIngestFairTurnGuard {
+    gate: std::sync::Arc<RawTransactionIngestFairTurnGate>,
+}
+
+impl RawTransactionIngestFairTurnGate {
+    fn new(max_waiters: usize) -> Self {
+        return Self {
+            max_waiters,
+            notify: tokio::sync::Notify::new(),
+            state: std::sync::Mutex::new(RawTransactionIngestFairTurnState {
+                active: false,
+                last_granted: std::option::Option::None,
+                next_ticket: 1,
+                waiters: std::collections::VecDeque::new(),
+            }),
+        };
+    }
+
+    async fn acquire(self: std::sync::Arc<Self>, class: RawTransactionIngestTrafficClass) -> ksp_core_lib::Result<RawTransactionIngestFairTurnGuard> {
+        let mut waiter = match RawTransactionIngestFairWaiterGuard::register(std::sync::Arc::clone(&self), class) {
+            std::result::Result::Ok(value) => value,
+            std::result::Result::Err(error) => return std::result::Result::Err(error),
+        };
+        loop {
+            let notify_gate = std::sync::Arc::clone(&self);
+            let notified = notify_gate.notify.notified();
+            let granted = {
+                let mut state = match self.state.lock() {
+                    std::result::Result::Ok(value) => value,
+                    std::result::Result::Err(poisoned) => poisoned.into_inner(),
+                };
+                if state.active || fair_turn_selected_ticket(&state) != std::option::Option::Some(waiter.ticket) {
+                    false
+                } else {
+                    let position = state.waiters.iter().position(|(ticket, _)| return *ticket == waiter.ticket);
+                    let position = match position {
+                        std::option::Option::Some(value) => value,
+                        std::option::Option::None => return std::result::Result::Err(crate::runtime_error("source.fairness_waiter_missing")),
+                    };
+                    let removed = state.waiters.remove(position);
+                    let removed = match removed {
+                        std::option::Option::Some(value) => value,
+                        std::option::Option::None => return std::result::Result::Err(crate::runtime_error("source.fairness_waiter_missing")),
+                    };
+                    if removed.1 != class {
+                        return std::result::Result::Err(crate::runtime_error("source.fairness_waiter_class_mismatch"));
+                    }
+                    state.active = true;
+                    state.last_granted = std::option::Option::Some(class);
+                    true
+                }
+            };
+            if granted {
+                waiter.armed = false;
+                std::mem::drop(notified);
+                std::mem::drop(notify_gate);
+                return std::result::Result::Ok(RawTransactionIngestFairTurnGuard { gate: self });
+            }
+            notified.await;
+        }
+    }
+}
+
+impl RawTransactionIngestFairWaiterGuard {
+    fn register(gate: std::sync::Arc<RawTransactionIngestFairTurnGate>, class: RawTransactionIngestTrafficClass) -> ksp_core_lib::Result<Self> {
+        let ticket = {
+            let mut state = match gate.state.lock() {
+                std::result::Result::Ok(value) => value,
+                std::result::Result::Err(poisoned) => poisoned.into_inner(),
+            };
+            if gate.max_waiters == 0 || state.waiters.len() >= gate.max_waiters {
+                return std::result::Result::Err(crate::runtime_error("source.fairness_waiter_capacity_exceeded"));
+            }
+            let ticket = state.next_ticket;
+            state.next_ticket = match state.next_ticket.checked_add(1) {
+                std::option::Option::Some(value) => value,
+                std::option::Option::None => return std::result::Result::Err(crate::runtime_error("source.fairness_ticket_exhausted")),
+            };
+            state.waiters.push_back((ticket, class));
+            ticket
+        };
+        gate.notify.notify_waiters();
+        return std::result::Result::Ok(Self { armed: true, gate, ticket });
+    }
+}
+
+impl std::ops::Drop for RawTransactionIngestFairWaiterGuard {
+    fn drop(&mut self) {
+        if !self.armed {
+            return;
+        }
+        let mut state = match self.gate.state.lock() {
+            std::result::Result::Ok(value) => value,
+            std::result::Result::Err(poisoned) => poisoned.into_inner(),
+        };
+        if let std::option::Option::Some(position) = state.waiters.iter().position(|(ticket, _)| return *ticket == self.ticket) {
+            let _removed = state.waiters.remove(position);
+        }
+        std::mem::drop(state);
+        self.gate.notify.notify_waiters();
+        return;
+    }
+}
+
+impl std::ops::Drop for RawTransactionIngestFairTurnGuard {
+    fn drop(&mut self) {
+        let mut state = match self.gate.state.lock() {
+            std::result::Result::Ok(value) => value,
+            std::result::Result::Err(poisoned) => poisoned.into_inner(),
+        };
+        state.active = false;
+        std::mem::drop(state);
+        self.gate.notify.notify_waiters();
+        return;
+    }
+}
+
+fn fair_turn_selected_ticket(state: &RawTransactionIngestFairTurnState) -> std::option::Option<u64> {
+    let nominal = state.waiters.iter().find(|(_, class)| return *class == RawTransactionIngestTrafficClass::Nominal);
+    let repair = state.waiters.iter().find(|(_, class)| return *class == RawTransactionIngestTrafficClass::Repair);
+    let selected = match (nominal, repair) {
+        (std::option::Option::Some(nominal), std::option::Option::Some(repair)) => match state.last_granted {
+            std::option::Option::Some(RawTransactionIngestTrafficClass::Nominal) => repair,
+            std::option::Option::Some(RawTransactionIngestTrafficClass::Repair) | std::option::Option::None => nominal,
+        },
+        (std::option::Option::Some(nominal), std::option::Option::None) => nominal,
+        (std::option::Option::None, std::option::Option::Some(repair)) => repair,
+        (std::option::Option::None, std::option::Option::None) => return std::option::Option::None,
+    };
+    return std::option::Option::Some(selected.0);
+}
+
+fn validate_repair_fairness_contract(admission_capacity: usize, persistence_concurrency: usize) -> ksp_core_lib::Result<()> {
+    if admission_capacity == 0
+        || persistence_concurrency == 0
+        || repair_block_fetch_limit(persistence_concurrency) == 0
+        || MAX_RAW_TRANSACTION_INGEST_REPAIR_BURST != 1
+        || !repair_fairness_catalog_is_complete()
+    {
+        return std::result::Result::Err(crate::runtime_error("runtime_resources.repair_fairness_invalid"));
+    }
+    return std::result::Result::Ok(());
+}
+
+fn repair_block_fetch_limit(existing_capacity: usize) -> usize {
+    return existing_capacity.min(4);
+}
+
+fn repair_fairness_catalog_is_complete() -> bool {
+    let mut state = RawTransactionIngestFairTurnState {
+        active: false,
+        last_granted: std::option::Option::None,
+        next_ticket: 3,
+        waiters: std::collections::VecDeque::from([(1, RawTransactionIngestTrafficClass::Nominal), (2, RawTransactionIngestTrafficClass::Repair)]),
+    };
+    if fair_turn_selected_ticket(&state) != std::option::Option::Some(1) {
+        return false;
+    }
+    state.last_granted = std::option::Option::Some(RawTransactionIngestTrafficClass::Nominal);
+    if fair_turn_selected_ticket(&state) != std::option::Option::Some(2) {
+        return false;
+    }
+    state.last_granted = std::option::Option::Some(RawTransactionIngestTrafficClass::Repair);
+    return fair_turn_selected_ticket(&state) == std::option::Option::Some(1);
+}
+
 #[derive(Clone, Copy, Debug, Eq, PartialEq)]
 enum RawTransactionIngestHttpDiscoveryStrategy {
     ClosedRange,
@@ -2336,6 +2528,9 @@ impl crate::RawTransactionIngestRuntimeResources {
         {
             return std::result::Result::Err(error);
         }
+        if let std::result::Result::Err(error) = validate_repair_fairness_contract(settings.admission_queue_capacity(), settings.persistence_concurrency()) {
+            return std::result::Result::Err(error);
+        }
         let hydration_pending_budget = settings.admission_queue_capacity();
         let hydration_in_flight_budget = settings.persistence_concurrency();
         let inventory = std::sync::Arc::new(std::sync::Mutex::new(RawTransactionIngestSourceInventory::new(source_keys)));
@@ -4263,6 +4458,7 @@ enum RawTransactionIngestSharedHydrationResult {
 }
 
 struct RawTransactionIngestGlobalHydrationRegistry {
+    hydration_fairness: std::sync::Arc<RawTransactionIngestFairTurnGate>,
     hydration_permits: std::sync::Arc<tokio::sync::Semaphore>,
     max_pending: usize,
     pending: std::sync::Mutex<
@@ -4276,12 +4472,26 @@ struct RawTransactionIngestGlobalHydrationRegistry {
 impl RawTransactionIngestGlobalHydrationRegistry {
     fn new(max_pending: usize, max_in_flight: usize) -> Self {
         return Self {
+            hydration_fairness: std::sync::Arc::new(RawTransactionIngestFairTurnGate::new(max_pending)),
             hydration_permits: std::sync::Arc::new(tokio::sync::Semaphore::new(max_in_flight)),
             max_pending,
             pending: std::sync::Mutex::new(std::collections::BTreeMap::new()),
         };
     }
 
+    async fn acquire_hydration_permit(&self, class: RawTransactionIngestTrafficClass) -> ksp_core_lib::Result<tokio::sync::OwnedSemaphorePermit> {
+        let turn = match std::sync::Arc::clone(&self.hydration_fairness).acquire(class).await {
+            std::result::Result::Ok(value) => value,
+            std::result::Result::Err(error) => return std::result::Result::Err(error),
+        };
+        let permit = std::sync::Arc::clone(&self.hydration_permits).acquire_owned().await;
+        std::mem::drop(turn);
+        return match permit {
+            std::result::Result::Ok(value) => std::result::Result::Ok(value),
+            std::result::Result::Err(_) => std::result::Result::Err(crate::runtime_error("source.global_hydration_permit_closed")),
+        };
+    }
+
     fn subscribe_or_lead(
         &self,
         key: &RawTransactionIngestHydrationKey,
@@ -4599,10 +4809,9 @@ async fn fetch_hydration_shared(
     };
     if leader {
         let mut leader_guard = RawTransactionIngestHydrationLeaderGuard::new(std::sync::Arc::clone(&global_registry), key.clone());
-        let permit = std::sync::Arc::clone(&global_registry.hydration_permits).acquire_owned().await;
-        let _permit = match permit {
+        let _permit = match global_registry.acquire_hydration_permit(RawTransactionIngestTrafficClass::Nominal).await {
             std::result::Result::Ok(value) => value,
-            std::result::Result::Err(_) => return std::result::Result::Err(crate::runtime_error("source.global_hydration_permit_closed")),
+            std::result::Result::Err(error) => return std::result::Result::Err(error),
         };
         let fetched = fetch_hydration(http_pool, hydration_role, expected_network, key.clone(), commitment).await;
         return match fetched {

crates/ksp-worker-raw-transaction-ingest-lib/tests/hardening.rs

@@ -1,7 +1,7 @@
 // file: crates/ksp-worker-raw-transaction-ingest-lib/tests/hardening.rs
-// version: 33
+// version: 34
 
-//! External public, security, redaction and release-boundary hardening canaries through `v0.3.14-pre.009`.
+//! External public, security, redaction and release-boundary hardening canaries through `v0.3.14-pre.010`.
 
 fn network(value: &'static str) -> std::option::Option<ksp_store_lib::RawNetworkId> {
     let result = ksp_store_lib::RawNetworkId::new(value);
@@ -1234,3 +1234,32 @@ fn v0_3_14_pre_009_health_policy_is_present_future_coverage_gated_and_source_neu
     assert!(!snapshot.contains("WorkerHealth::Faulted"), "Faulted must remain a Worker lifecycle state rather than a new health enum variant");
     return;
 }
+
+#[test]
+fn v0_3_14_pre_010_repair_fairness_shares_existing_bounds_without_second_pipeline() {
+    let continuity = include_str!("../src/continuity.rs");
+    let resources = include_str!("../src/runtime_resources.rs");
+    let admission = include_str!("../src/admission.rs");
+    let root = include_str!("../src/lib.rs");
+    for required in [
+        "MAX_RAW_TRANSACTION_INGEST_REPAIR_BURST: usize = 1",
+        "RawTransactionIngestTrafficClass",
+        "RawTransactionIngestFairTurnGate",
+        "RawTransactionIngestTrafficClass::Nominal",
+        "RawTransactionIngestTrafficClass::Repair",
+        "acquire_hydration_permit",
+        "validate_repair_fairness_contract",
+        "repair_block_fetch_limit",
+    ] {
+        assert!(resources.contains(required) || continuity.contains(required), "required pre.010 fairness guard missing: {required}");
+    }
+    assert!(continuity.contains("MAX_RAW_TRANSACTION_INGEST_REPAIR_BLOCK_FETCH_IN_FLIGHT: usize = 4"));
+    assert!(continuity.contains("MAX_RAW_TRANSACTION_INGEST_REPAIR_DISCOVERY_WINDOW_SLOTS: u64 = 512"));
+    assert!(admission.contains("tokio::sync::mpsc::channel(capacity)"));
+    assert!(!resources.contains("tokio::sync::mpsc::channel("), "pre.010 created a second admission pipeline");
+    assert_eq!(resources.matches("struct RawTransactionIngestGlobalHydrationRegistry").count(), 1);
+    assert!(!resources.contains("RepairHydrationRegistry"));
+    assert!(!root.contains("RawTransactionIngestTrafficClass"));
+    assert!(!root.contains("RawTransactionIngestFairTurnGate"));
+    return;
+}

crates/ksp-worker-raw-transaction-ingest-lib/tests/release_completeness.rs

@@ -1,7 +1,7 @@
 // file: crates/ksp-worker-raw-transaction-ingest-lib/tests/release_completeness.rs
-// version: 28
+// version: 29
 
-//! Release-completeness canaries through the `v0.3.14-pre.009` multi-source health-policy tranche.
+//! Release-completeness canaries through the `v0.3.14-pre.010` repair backpressure/fairness tranche.
 
 #[test]
 fn pre_010_production_module_inventory_is_exact() -> std::io::Result<()> {
@@ -367,3 +367,22 @@ fn v0_3_14_pre_009_health_policy_canaries_are_present_without_public_source_iden
     assert!(!root.contains("pub use self::continuity::RawTransactionIngestTargetCoverage"));
     return;
 }
+
+#[test]
+fn v0_3_14_pre_010_repair_fairness_canaries_are_present_without_public_or_pipeline_growth() {
+    let hardening = include_str!("hardening.rs");
+    let resource_tests = include_str!("../unit_tests/runtime_resources.rs");
+    let root = include_str!("../src/lib.rs");
+    for required in [
+        "v0_3_14_pre_010_fair_scheduler_alternates_nominal_and_repair_with_bounded_burst",
+        "v0_3_14_pre_010_fair_waiter_bound_is_checked_and_released_without_reserved_capacity",
+        "v0_3_14_pre_010_nominal_and_repair_hydration_share_one_capacity_one_permit_pool",
+        "v0_3_14_pre_010_repair_only_progresses_when_existing_capacity_is_one",
+    ] {
+        assert!(resource_tests.contains(required), "required pre.010 fairness canary missing: {required}");
+    }
+    assert!(hardening.contains("v0_3_14_pre_010_repair_fairness_shares_existing_bounds_without_second_pipeline"));
+    assert!(!root.contains("RawTransactionIngestTrafficClass"));
+    assert!(!root.contains("RawTransactionIngestFairTurnGate"));
+    return;
+}

crates/ksp-worker-raw-transaction-ingest-lib/unit_tests/runtime_resources.rs

@@ -1,5 +1,5 @@
 // file: crates/ksp-worker-raw-transaction-ingest-lib/unit_tests/runtime_resources.rs
-// version: 31
+// version: 32
 
 fn grpc_endpoint(cluster: &str) -> std::option::Option<ksp_onchain_transport_lib::YellowstoneGrpcEndpointSettings> {
     return grpc_endpoint_with_identity(cluster, "yellowstone-fixture", "fixture-provider");
@@ -4142,3 +4142,82 @@ fn v0_3_14_pre_009_inventory_health_projection_tracks_reconciled_coverage() {
     assert!(reconciled.future_target_coverage());
     return;
 }
+
+#[test]
+fn v0_3_14_pre_010_fair_scheduler_alternates_nominal_and_repair_with_bounded_burst() {
+    assert_eq!(super::MAX_RAW_TRANSACTION_INGEST_REPAIR_BURST, 1);
+    let mut state = super::RawTransactionIngestFairTurnState {
+        active: false,
+        last_granted: std::option::Option::None,
+        next_ticket: 3,
+        waiters: std::collections::VecDeque::from([
+            (1, super::RawTransactionIngestTrafficClass::Nominal),
+            (2, super::RawTransactionIngestTrafficClass::Repair),
+        ]),
+    };
+    assert_eq!(super::fair_turn_selected_ticket(&state), std::option::Option::Some(1));
+    state.last_granted = std::option::Option::Some(super::RawTransactionIngestTrafficClass::Nominal);
+    assert_eq!(super::fair_turn_selected_ticket(&state), std::option::Option::Some(2));
+    state.last_granted = std::option::Option::Some(super::RawTransactionIngestTrafficClass::Repair);
+    assert_eq!(super::fair_turn_selected_ticket(&state), std::option::Option::Some(1));
+    let _removed = state.waiters.pop_front();
+    assert_eq!(super::fair_turn_selected_ticket(&state), std::option::Option::Some(2));
+    return;
+}
+
+#[test]
+fn v0_3_14_pre_010_fair_waiter_bound_is_checked_and_released_without_reserved_capacity() {
+    assert!(super::validate_repair_fairness_contract(1, 1).is_ok());
+    assert_eq!(super::repair_block_fetch_limit(1), 1);
+    assert_eq!(super::repair_block_fetch_limit(4), 4);
+    assert_eq!(super::repair_block_fetch_limit(64), 4);
+    let gate = std::sync::Arc::new(super::RawTransactionIngestFairTurnGate::new(1));
+    let first = super::RawTransactionIngestFairWaiterGuard::register(std::sync::Arc::clone(&gate), super::RawTransactionIngestTrafficClass::Repair);
+    let first = match first {
+        std::result::Result::Ok(value) => value,
+        std::result::Result::Err(_) => return,
+    };
+    assert!(super::RawTransactionIngestFairWaiterGuard::register(std::sync::Arc::clone(&gate), super::RawTransactionIngestTrafficClass::Nominal,).is_err());
+    std::mem::drop(first);
+    assert!(super::RawTransactionIngestFairWaiterGuard::register(gate, super::RawTransactionIngestTrafficClass::Nominal).is_ok());
+    return;
+}
+
+#[tokio::test]
+async fn v0_3_14_pre_010_nominal_and_repair_hydration_share_one_capacity_one_permit_pool() {
+    let registry = std::sync::Arc::new(super::RawTransactionIngestGlobalHydrationRegistry::new(2, 1));
+    let nominal = registry.acquire_hydration_permit(super::RawTransactionIngestTrafficClass::Nominal).await;
+    let nominal = match nominal {
+        std::result::Result::Ok(value) => value,
+        std::result::Result::Err(_) => return,
+    };
+    let repair_registry = std::sync::Arc::clone(&registry);
+    let repair = tokio::spawn(async move {
+        return repair_registry.acquire_hydration_permit(super::RawTransactionIngestTrafficClass::Repair).await;
+    });
+    tokio::task::yield_now().await;
+    assert!(!repair.is_finished());
+    std::mem::drop(nominal);
+    let repair = match repair.await {
+        std::result::Result::Ok(std::result::Result::Ok(value)) => value,
+        std::result::Result::Ok(std::result::Result::Err(_)) | std::result::Result::Err(_) => return,
+    };
+    assert_eq!(registry.hydration_permits.available_permits(), 0);
+    std::mem::drop(repair);
+    assert_eq!(registry.hydration_permits.available_permits(), 1);
+    return;
+}
+
+#[tokio::test]
+async fn v0_3_14_pre_010_repair_only_progresses_when_existing_capacity_is_one() {
+    let registry = super::RawTransactionIngestGlobalHydrationRegistry::new(1, 1);
+    let permit = registry.acquire_hydration_permit(super::RawTransactionIngestTrafficClass::Repair).await;
+    let permit = match permit {
+        std::result::Result::Ok(value) => value,
+        std::result::Result::Err(_) => return,
+    };
+    assert_eq!(registry.hydration_permits.available_permits(), 0);
+    std::mem::drop(permit);
+    assert_eq!(registry.hydration_permits.available_permits(), 1);
+    return;
+}