v0.3.14-pre.009

crates/ksp-worker-raw-transaction-ingest-lib/src/runtime_resources.rs

@@ -1,5 +1,5 @@
 // file: crates/ksp-worker-raw-transaction-ingest-lib/src/runtime_resources.rs
-// version: 36
+// version: 37
 
 use sha2::Digest; // rust-rules: trait-import
 
@@ -104,6 +104,7 @@ struct RawTransactionIngestSourceInventory {
 #[derive(Clone)]
 struct RawTransactionIngestSourceInventoryPublisher {
     aggregate_sender: tokio::sync::watch::Sender<crate::RawTransactionIngestProcessingFrontierProjection>,
+    continuity_contracts: std::sync::Arc<std::sync::Mutex<crate::RawTransactionIngestContinuityContracts>>,
     entry_index: usize,
     inventory: std::sync::Arc<std::sync::Mutex<RawTransactionIngestSourceInventory>>,
     source_key: [u8; 32],
@@ -361,7 +362,7 @@ impl RawTransactionIngestSourceInventory {
         return Self { source_keys, source_projections };
     }
 
-    fn supervisor_state(&self) -> ksp_core_lib::Result<(std::vec::Vec<[u8; 32]>, std::option::Option<u64>)> {
+    fn active_source_keys(&self) -> ksp_core_lib::Result<std::vec::Vec<[u8; 32]>> {
         if self.source_keys.len() != self.source_projections.len() {
             return std::result::Result::Err(crate::runtime_error("source.inventory_shape_mismatch"));
         }
@@ -371,6 +372,27 @@ impl RawTransactionIngestSourceInventory {
                 active_source_keys.push(*source_key);
             }
         }
+        return std::result::Result::Ok(active_source_keys);
+    }
+
+    fn failed_source_keys(&self) -> ksp_core_lib::Result<std::vec::Vec<[u8; 32]>> {
+        if self.source_keys.len() != self.source_projections.len() {
+            return std::result::Result::Err(crate::runtime_error("source.inventory_shape_mismatch"));
+        }
+        let mut failed_source_keys = std::vec::Vec::new();
+        for (source_key, projection) in self.source_keys.iter().zip(self.source_projections.iter()) {
+            if projection.source_state() == std::option::Option::Some(crate::RawTransactionIngestSourceState::Failed) {
+                failed_source_keys.push(*source_key);
+            }
+        }
+        return std::result::Result::Ok(failed_source_keys);
+    }
+
+    fn supervisor_state(&self) -> ksp_core_lib::Result<(std::vec::Vec<[u8; 32]>, std::option::Option<u64>)> {
+        let active_source_keys = match self.active_source_keys() {
+            std::result::Result::Ok(value) => value,
+            std::result::Result::Err(error) => return std::result::Result::Err(error),
+        };
         let aggregate = match self.aggregate() {
             std::result::Result::Ok(value) => value,
             std::result::Result::Err(error) => return std::result::Result::Err(error),
@@ -400,9 +422,49 @@ impl RawTransactionIngestSourceInventory {
     }
 }
 
+fn source_inventory_health_projection(
+    inventory: &std::sync::Arc<std::sync::Mutex<RawTransactionIngestSourceInventory>>,
+    continuity_contracts: &std::sync::Arc<std::sync::Mutex<crate::RawTransactionIngestContinuityContracts>>,
+) -> ksp_core_lib::Result<crate::RawTransactionIngestProcessingFrontierProjection> {
+    let (aggregate, active_source_keys, failed_source_keys) = {
+        let inventory = match inventory.lock() {
+            std::result::Result::Ok(value) => value,
+            std::result::Result::Err(poisoned) => poisoned.into_inner(),
+        };
+        let aggregate = match inventory.aggregate() {
+            std::result::Result::Ok(value) => value,
+            std::result::Result::Err(error) => return std::result::Result::Err(error),
+        };
+        let active_source_keys = match inventory.active_source_keys() {
+            std::result::Result::Ok(value) => value,
+            std::result::Result::Err(error) => return std::result::Result::Err(error),
+        };
+        let failed_source_keys = match inventory.failed_source_keys() {
+            std::result::Result::Ok(value) => value,
+            std::result::Result::Err(error) => return std::result::Result::Err(error),
+        };
+        (aggregate, active_source_keys, failed_source_keys)
+    };
+    let (continuity_frontier_slot, continuity_has_open_gaps, future_target_coverage, failed_source_losses_reconciled) = {
+        let mut contracts = match continuity_contracts.lock() {
+            std::result::Result::Ok(value) => value,
+            std::result::Result::Err(poisoned) => poisoned.into_inner(),
+        };
+        match contracts.health_projection(active_source_keys.as_slice(), failed_source_keys.as_slice(), aggregate.processing_frontier_slot()) {
+            std::result::Result::Ok(value) => value,
+            std::result::Result::Err(error) => return std::result::Result::Err(error),
+        }
+    };
+    return std::result::Result::Ok(
+        aggregate
+            .with_continuity_health(continuity_frontier_slot, continuity_has_open_gaps, future_target_coverage)
+            .with_failed_source_losses_reconciled(failed_source_losses_reconciled),
+    );
+}
+
 impl RawTransactionIngestSourceInventoryPublisher {
     fn publish(&self, projection: crate::RawTransactionIngestProcessingFrontierProjection) -> ksp_core_lib::Result<()> {
-        let aggregate = {
+        {
             let mut inventory = match self.inventory.lock() {
                 std::result::Result::Ok(value) => value,
                 std::result::Result::Err(poisoned) => poisoned.into_inner(),
@@ -410,10 +472,10 @@ impl RawTransactionIngestSourceInventoryPublisher {
             if let std::result::Result::Err(error) = inventory.update(self.entry_index, self.source_key, projection) {
                 return std::result::Result::Err(error);
             }
-            match inventory.aggregate() {
-                std::result::Result::Ok(value) => value,
-                std::result::Result::Err(error) => return std::result::Result::Err(error),
-            }
+        }
+        let aggregate = match source_inventory_health_projection(&self.inventory, &self.continuity_contracts) {
+            std::result::Result::Ok(value) => value,
+            std::result::Result::Err(error) => return std::result::Result::Err(error),
         };
         self.aggregate_sender.send_replace(aggregate);
         return std::result::Result::Ok(());
@@ -1481,6 +1543,7 @@ impl crate::RawTransactionIngestHttpBlockPollingSource {
                         fault = std::option::Option::Some(error);
                         break;
                     }
+                    processing_frontier.publish();
                     next_scan_slot = match proven_end_slot.checked_add(1) {
                         std::option::Option::Some(value) => value,
                         std::option::Option::None => {
@@ -2295,6 +2358,7 @@ impl crate::RawTransactionIngestRuntimeResources {
             };
             let publisher = RawTransactionIngestSourceInventoryPublisher {
                 aggregate_sender: processing_frontier_sender.clone(),
+                continuity_contracts: std::sync::Arc::clone(&continuity_contracts),
                 entry_index,
                 inventory: std::sync::Arc::clone(&inventory),
                 source_key: source.source_key(),
@@ -2321,6 +2385,7 @@ impl crate::RawTransactionIngestRuntimeResources {
             children,
             std::sync::Arc::clone(&continuity_contracts),
             std::sync::Arc::clone(&inventory),
+            processing_frontier_sender,
         )
         .await;
         let validation = {
@@ -2418,6 +2483,7 @@ async fn supervise_live_source_tasks(
     mut children: tokio::task::JoinSet<([u8; 32], ksp_core_lib::Result<()>)>,
     continuity_contracts: std::sync::Arc<std::sync::Mutex<crate::RawTransactionIngestContinuityContracts>>,
     inventory: std::sync::Arc<std::sync::Mutex<RawTransactionIngestSourceInventory>>,
+    processing_frontier_sender: tokio::sync::watch::Sender<crate::RawTransactionIngestProcessingFrontierProjection>,
 ) -> ksp_core_lib::Result<()> {
     loop {
         if *stop_receiver.borrow() {
@@ -2502,6 +2568,14 @@ async fn supervise_live_source_tasks(
         };
         match decision {
             crate::RawTransactionIngestSourceLossDecision::Continue => {
+                let aggregate = match source_inventory_health_projection(&inventory, &continuity_contracts) {
+                    std::result::Result::Ok(value) => value,
+                    std::result::Result::Err(error) => {
+                        source_stop_sender.send_replace(true);
+                        return drain_live_source_tasks(&mut children, std::option::Option::Some(error)).await;
+                    },
+                };
+                processing_frontier_sender.send_replace(aggregate);
                 continue;
             },
             crate::RawTransactionIngestSourceLossDecision::Fault => {