v0.3.6-pre.008

crates/ksp-job-backfill-lib/src/execution.rs

@@ -0,0 +1,272 @@
+// file: crates/ksp-job-backfill-lib/src/execution.rs
+// version: 1
+
+use futures_util::StreamExt; // rust-rules: trait-import
+
+/// Bounded result of one concurrent Backfill candidate execution pass.
+///
+/// This is intentionally smaller than the concrete latest-value snapshot planned for `pre.009`.
+/// It exposes only the admission/frontier/checkpoint facts required to prove safe resumption.
+#[derive(Clone, Debug, Eq, PartialEq)]
+pub struct BackfillExecutionBatch {
+    candidate_count: usize,
+    admitted_count: usize,
+    finished_count: usize,
+    durable_count: usize,
+    hole_count: usize,
+    maximum_in_flight: usize,
+    local_contiguous_completed: usize,
+    discovery_partial: bool,
+    checkpoint: crate::BackfillCheckpoint,
+    failure_code: std::option::Option<ksp_core_lib::ErrorCode>,
+}
+
+impl BackfillExecutionBatch {
+    /// Returns the number of candidates present in the bounded discovery result.
+    #[must_use]
+    pub const fn candidate_count(&self) -> usize {
+        return self.candidate_count;
+    }
+
+    /// Returns the number of candidates admitted into hydration during this execution pass.
+    #[must_use]
+    pub const fn admitted_count(&self) -> usize {
+        return self.admitted_count;
+    }
+
+    /// Returns the number of admitted candidates whose hydration/persistence future reached a known outcome.
+    #[must_use]
+    pub const fn finished_count(&self) -> usize {
+        return self.finished_count;
+    }
+
+    /// Returns the number of candidates whose Store outcome is durable for checkpoint advancement.
+    #[must_use]
+    pub const fn durable_count(&self) -> usize {
+        return self.durable_count;
+    }
+
+    /// Returns the number of known candidate outcomes that intentionally block the contiguous frontier.
+    #[must_use]
+    pub const fn hole_count(&self) -> usize {
+        return self.hole_count;
+    }
+
+    /// Returns the greatest number of candidate futures simultaneously in flight.
+    #[must_use]
+    pub const fn maximum_in_flight(&self) -> usize {
+        return self.maximum_in_flight;
+    }
+
+    /// Returns the durable contiguous prefix completed inside the current discovery result.
+    #[must_use]
+    pub const fn local_contiguous_completed(&self) -> usize {
+        return self.local_contiguous_completed;
+    }
+
+    /// Returns the safe caller-owned checkpoint after draining all work already admitted.
+    #[must_use]
+    pub const fn checkpoint(&self) -> &crate::BackfillCheckpoint {
+        return &self.checkpoint;
+    }
+
+    /// Returns the first stable fatal error code that stopped new admissions, when one occurred.
+    #[must_use]
+    pub const fn failure_code(&self) -> std::option::Option<ksp_core_lib::ErrorCode> {
+        return self.failure_code;
+    }
+
+    /// Returns whether discovery or candidate outcomes left the pass incomplete.
+    #[must_use]
+    pub const fn is_partial(&self) -> bool {
+        return self.discovery_partial || self.hole_count != 0 || self.failure_code.is_some();
+    }
+}
+
+/// Executes one bounded discovered candidate set with request-owned hydration concurrency.
+///
+/// At most `request.hydration_concurrency()` candidate futures are kept in flight. Candidate
+/// completions may arrive out of order, but the checkpoint advances only across the contiguous
+/// durable prefix. `Missing` remains a non-fatal hole. Conflict or any Transport/conversion/Store
+/// error stops new admissions immediately; work already admitted is drained before this function
+/// returns so a submitted Store operation is never silently abandoned by the batch coordinator.
+pub async fn execute_backfill_discovery(
+    transport: &ksp_onchain_transport_lib::HttpTransportPool,
+    store: &ksp_store_lib::Store,
+    request: &crate::BackfillRequest,
+    discovery: &crate::BackfillDiscovery,
+) -> ksp_core_lib::Result<BackfillExecutionBatch> {
+    let validation = crate::validate_discovery_identity(request, discovery);
+    if let std::result::Result::Err(error) = validation {
+        return std::result::Result::Err(error);
+    }
+    let processor = RuntimeCandidateProcessor { transport, store, request };
+    return execute_with_processor(&processor, request, discovery).await;
+}
+
+type CandidateProcessFuture<'a> =
+    std::pin::Pin<std::boxed::Box<dyn std::future::Future<Output = ksp_core_lib::Result<crate::BackfillPersistenceOutcome>> + 'a>>;
+
+trait CandidateProcessor {
+    fn process<'a>(&'a self, candidate: &'a crate::BackfillCandidate) -> CandidateProcessFuture<'a>;
+}
+
+struct RuntimeCandidateProcessor<'a> {
+    transport: &'a ksp_onchain_transport_lib::HttpTransportPool,
+    store: &'a ksp_store_lib::Store,
+    request: &'a crate::BackfillRequest,
+}
+
+impl CandidateProcessor for RuntimeCandidateProcessor<'_> {
+    fn process<'a>(&'a self, candidate: &'a crate::BackfillCandidate) -> CandidateProcessFuture<'a> {
+        return std::boxed::Box::pin(async move {
+            let received_at = match current_raw_timestamp() {
+                std::result::Result::Ok(value) => value,
+                std::result::Result::Err(error) => return std::result::Result::Err(error),
+            };
+            let hydration = crate::hydrate_backfill_candidate(self.transport, self.request, candidate, received_at).await;
+            let hydration = match hydration {
+                std::result::Result::Ok(value) => value,
+                std::result::Result::Err(error) => return std::result::Result::Err(error),
+            };
+            return crate::persist_backfill_hydration(self.store, hydration).await;
+        });
+    }
+}
+
+async fn execute_with_processor<P>(
+    processor: &P,
+    request: &crate::BackfillRequest,
+    discovery: &crate::BackfillDiscovery,
+) -> ksp_core_lib::Result<BackfillExecutionBatch>
+where
+    P: CandidateProcessor,
+{
+    let validation = crate::validate_discovery_identity(request, discovery);
+    if let std::result::Result::Err(error) = validation {
+        return std::result::Result::Err(error);
+    }
+    let resume_offset = match crate::execution_resume_offset(request, discovery) {
+        std::result::Result::Ok(value) => value,
+        std::result::Result::Err(error) => return std::result::Result::Err(error),
+    };
+    let mut frontier = crate::CompletionFrontier::new(discovery.candidates().len());
+    if matches!(request.scope().kind(), crate::BackfillScopeKind::AfterAddress | crate::BackfillScopeKind::ExplicitSignatures) {
+        let seeded = frontier.seed_prefix(resume_offset);
+        if let std::result::Result::Err(error) = seeded {
+            return std::result::Result::Err(error);
+        }
+    }
+    let mut next_index = resume_offset;
+    let mut admitted_count = 0_usize;
+    let mut finished_count = 0_usize;
+    let mut durable_count = 0_usize;
+    let mut hole_count = 0_usize;
+    let mut maximum_in_flight = 0_usize;
+    let mut failure_code = std::option::Option::<ksp_core_lib::ErrorCode>::None;
+    let mut in_flight = futures_util::stream::FuturesUnordered::new();
+    loop {
+        while failure_code.is_none() && next_index < discovery.candidates().len() && in_flight.len() < request.hydration_concurrency() {
+            let index = next_index;
+            let candidate = &discovery.candidates()[index];
+            let future = processor.process(candidate);
+            in_flight.push(async move {
+                return (index, future.await);
+            });
+            next_index += 1;
+            admitted_count = match checked_increment(admitted_count, "admitted_count") {
+                std::result::Result::Ok(value) => value,
+                std::result::Result::Err(error) => return std::result::Result::Err(error),
+            };
+            maximum_in_flight = std::cmp::max(maximum_in_flight, in_flight.len());
+        }
+        let completed = in_flight.next().await;
+        let (index, result) = match completed {
+            std::option::Option::Some(completed) => completed,
+            std::option::Option::None => break,
+        };
+        finished_count = match checked_increment(finished_count, "finished_count") {
+            std::result::Result::Ok(value) => value,
+            std::result::Result::Err(error) => return std::result::Result::Err(error),
+        };
+        match result {
+            std::result::Result::Ok(outcome) => {
+                if persistence_advances_frontier(&outcome) {
+                    let marked = frontier.mark_durable(index);
+                    if let std::result::Result::Err(error) = marked {
+                        return std::result::Result::Err(error);
+                    }
+                    durable_count = match checked_increment(durable_count, "durable_count") {
+                        std::result::Result::Ok(value) => value,
+                        std::result::Result::Err(error) => return std::result::Result::Err(error),
+                    };
+                } else {
+                    hole_count = match checked_increment(hole_count, "hole_count") {
+                        std::result::Result::Ok(value) => value,
+                        std::result::Result::Err(error) => return std::result::Result::Err(error),
+                    };
+                    if outcome.entity() == crate::BackfillEntityPersistence::Conflict && failure_code.is_none() {
+                        failure_code = std::option::Option::Some(ksp_store_lib::ERROR_CODE_RAW_CONFLICT);
+                    }
+                }
+            },
+            std::result::Result::Err(error) => {
+                hole_count = match checked_increment(hole_count, "hole_count") {
+                    std::result::Result::Ok(value) => value,
+                    std::result::Result::Err(increment_error) => return std::result::Result::Err(increment_error),
+                };
+                if failure_code.is_none() {
+                    failure_code = std::option::Option::Some(error.code());
+                }
+            },
+        }
+    }
+    let checkpoint = match crate::checkpoint_from_frontier(request, discovery, &frontier) {
+        std::result::Result::Ok(value) => value,
+        std::result::Result::Err(error) => return std::result::Result::Err(error),
+    };
+    return std::result::Result::Ok(BackfillExecutionBatch {
+        candidate_count: discovery.candidates().len(),
+        admitted_count,
+        finished_count,
+        durable_count,
+        hole_count,
+        maximum_in_flight,
+        local_contiguous_completed: frontier.contiguous_completed(),
+        discovery_partial: discovery.is_partial(),
+        checkpoint,
+        failure_code,
+    });
+}
+
+fn persistence_advances_frontier(outcome: &crate::BackfillPersistenceOutcome) -> bool {
+    return matches!(
+        outcome.entity(),
+        crate::BackfillEntityPersistence::Inserted | crate::BackfillEntityPersistence::AlreadyPresent | crate::BackfillEntityPersistence::SkippedPurged
+    );
+}
+
+fn checked_increment(value: usize, field: &'static str) -> ksp_core_lib::Result<usize> {
+    return value.checked_add(1).ok_or_else(|| return execution_error(field));
+}
+
+fn current_raw_timestamp() -> ksp_core_lib::Result<ksp_store_lib::RawTimestamp> {
+    let duration = std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH);
+    let duration = match duration {
+        std::result::Result::Ok(duration) => duration,
+        std::result::Result::Err(_) => return std::result::Result::Err(execution_error("clock.before_epoch")),
+    };
+    let millis = match u64::try_from(duration.as_millis()) {
+        std::result::Result::Ok(value) => value,
+        std::result::Result::Err(_) => return std::result::Result::Err(execution_error("clock.millis")),
+    };
+    return ksp_store_lib::RawTimestamp::from_unix_millis(millis);
+}
+
+fn execution_error(field: &'static str) -> ksp_core_lib::Error {
+    return ksp_core_lib::Error::new(crate::ERROR_CODE_BACKFILL_EXECUTION_INVALID, "invalid bounded Backfill execution state").with_context("field", field);
+}
+
+#[cfg(test)]
+#[path = "../unit_tests/execution.rs"]
+mod tests;