v0.3.6-pre.009

crates/ksp-job-backfill-lib/src/discovery.rs

@@ -1,5 +1,5 @@
 // file: crates/ksp-job-backfill-lib/src/discovery.rs
-// version: 3
+// version: 4
 
 /// Network-scoped identity of one discovered transaction candidate before canonical signature decoding.
 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
@@ -217,7 +217,7 @@ pub async fn discover_backfill_candidates(
         max_candidates = request.max_candidates(),
         "starting bounded Backfill candidate discovery"
     );
-    let result = discover_with_source(transport, request).await;
+    let result = discover_with_source(transport, request, std::option::Option::None).await;
     if let std::result::Result::Ok(discovery) = &result {
         ksp_logging_lib::debug!(
             target: crate::TRACING_TARGET,
@@ -234,14 +234,30 @@ pub async fn discover_backfill_candidates(
     return result;
 }
 
-async fn discover_with_source<S>(source: &S, request: &crate::BackfillRequest) -> ksp_core_lib::Result<BackfillDiscovery>
+/// Discovers one bounded candidate set while allowing cooperative cancellation of in-flight RPC futures.
+pub(crate) async fn discover_backfill_candidates_cancellable(
+    transport: &ksp_onchain_transport_lib::HttpTransportPool,
+    request: &crate::BackfillRequest,
+    cancellation: &crate::BackfillCancellationSignal,
+) -> ksp_core_lib::Result<BackfillDiscovery> {
+    return discover_with_source(transport, request, std::option::Option::Some(cancellation)).await;
+}
+
+async fn discover_with_source<S>(
+    source: &S,
+    request: &crate::BackfillRequest,
+    cancellation: std::option::Option<&crate::BackfillCancellationSignal>,
+) -> ksp_core_lib::Result<BackfillDiscovery>
 where
     S: SignaturePageSource,
 {
+    if cancellation.is_some_and(crate::BackfillCancellationSignal::is_requested) {
+        return std::result::Result::Err(cancelled_error());
+    }
     return match request.scope().kind() {
         crate::BackfillScopeKind::ExplicitSignatures => discover_explicit(request),
-        crate::BackfillScopeKind::LatestAddress | crate::BackfillScopeKind::BeforeAddress => discover_older(source, request).await,
-        crate::BackfillScopeKind::AfterAddress => discover_after(source, request).await,
+        crate::BackfillScopeKind::LatestAddress | crate::BackfillScopeKind::BeforeAddress => discover_older(source, request, cancellation).await,
+        crate::BackfillScopeKind::AfterAddress => discover_after(source, request, cancellation).await,
     };
 }
 
@@ -264,7 +280,11 @@ fn discover_explicit(request: &crate::BackfillRequest) -> ksp_core_lib::Result<B
     ));
 }
 
-async fn discover_older<S>(source: &S, request: &crate::BackfillRequest) -> ksp_core_lib::Result<BackfillDiscovery>
+async fn discover_older<S>(
+    source: &S,
+    request: &crate::BackfillRequest,
+    cancellation: std::option::Option<&crate::BackfillCancellationSignal>,
+) -> ksp_core_lib::Result<BackfillDiscovery>
 where
     S: SignaturePageSource,
 {
@@ -298,7 +318,7 @@ where
             std::option::Option::Some(request.commitment().transport()),
             request.min_context_slot(),
         );
-        let page_result = source.fetch_signature_page(request.role(), address, config).await;
+        let page_result = fetch_signature_page(source, request.role(), address, config, cancellation).await;
         let page = match page_result {
             std::result::Result::Ok(page) => page,
             std::result::Result::Err(error) => return std::result::Result::Err(error),
@@ -347,7 +367,11 @@ where
     return std::result::Result::Ok(BackfillDiscovery::new(request.network().clone(), request.scope_fingerprint(), candidates, pages_fetched, boundary));
 }
 
-async fn discover_after<S>(source: &S, request: &crate::BackfillRequest) -> ksp_core_lib::Result<BackfillDiscovery>
+async fn discover_after<S>(
+    source: &S,
+    request: &crate::BackfillRequest,
+    cancellation: std::option::Option<&crate::BackfillCancellationSignal>,
+) -> ksp_core_lib::Result<BackfillDiscovery>
 where
     S: SignaturePageSource,
 {
@@ -375,7 +399,7 @@ where
             std::option::Option::Some(request.commitment().transport()),
             request.min_context_slot(),
         );
-        let page_result = source.fetch_signature_page(request.role(), address, config).await;
+        let page_result = fetch_signature_page(source, request.role(), address, config, cancellation).await;
         let page = match page_result {
             std::result::Result::Ok(page) => page,
             std::result::Result::Err(error) => return std::result::Result::Err(error),
@@ -430,6 +454,27 @@ where
     ));
 }
 
+async fn fetch_signature_page<S>(
+    source: &S,
+    role: &ksp_onchain_transport_lib::HttpRoleName,
+    address: &ksp_core_lib::Pubkey,
+    config: ksp_onchain_transport_lib::SolanaSignaturesForAddressConfig,
+    cancellation: std::option::Option<&crate::BackfillCancellationSignal>,
+) -> ksp_core_lib::Result<std::vec::Vec<SignaturePageEntry>>
+where
+    S: SignaturePageSource,
+{
+    let future = source.fetch_signature_page(role, address, config);
+    return match cancellation {
+        std::option::Option::Some(signal) => signal.run_cancellable(future).await,
+        std::option::Option::None => future.await,
+    };
+}
+
+fn cancelled_error() -> ksp_core_lib::Error {
+    return ksp_core_lib::Error::new(crate::ERROR_CODE_BACKFILL_CANCELLED, "Backfill discovery cancelled before RPC completion");
+}
+
 fn validated_signature(value: &str) -> ksp_core_lib::Result<crate::BackfillSignature> {
     return crate::BackfillSignature::new(value.to_owned());
 }

crates/ksp-job-backfill-lib/src/error.rs

@@ -1,5 +1,5 @@
 // file: crates/ksp-job-backfill-lib/src/error.rs
-// version: 4
+// version: 5
 
 /// Error code used when one Backfill checkpoint/frontier is incompatible with the current Job or semantic scope.
 pub const ERROR_CODE_BACKFILL_CHECKPOINT_INVALID: ksp_core_lib::ErrorCode = ksp_core_lib::ErrorCode::new("job_backfill", "checkpoint_invalid");
@@ -15,5 +15,10 @@ pub const ERROR_CODE_BACKFILL_PERSISTENCE_INVALID: ksp_core_lib::ErrorCode = ksp
 pub const ERROR_CODE_BACKFILL_RAW_CONVERSION_INVALID: ksp_core_lib::ErrorCode = ksp_core_lib::ErrorCode::new("job_backfill", "raw_conversion_invalid");
 /// Error code used when one Backfill request violates its bounded admission contract.
 pub const ERROR_CODE_BACKFILL_REQUEST_INVALID: ksp_core_lib::ErrorCode = ksp_core_lib::ErrorCode::new("job_backfill", "request_invalid");
+/// Error code used when the concrete Backfill runtime reaches an impossible lifecycle or notification state.
+pub const ERROR_CODE_BACKFILL_RUNTIME_INVALID: ksp_core_lib::ErrorCode = ksp_core_lib::ErrorCode::new("job_backfill", "runtime_invalid");
 /// Error code used when one transaction signature text violates the bounded Base58-shape contract.
 pub const ERROR_CODE_BACKFILL_SIGNATURE_INVALID: ksp_core_lib::ErrorCode = ksp_core_lib::ErrorCode::new("job_backfill", "signature_invalid");
+
+/// Error code used internally when cooperative cancellation wins before durable Store submission.
+pub(crate) const ERROR_CODE_BACKFILL_CANCELLED: ksp_core_lib::ErrorCode = ksp_core_lib::ErrorCode::new("job_backfill", "cancelled");

crates/ksp-job-backfill-lib/src/execution.rs

@@ -1,22 +1,28 @@
 // file: crates/ksp-job-backfill-lib/src/execution.rs
-// version: 1
+// version: 2
 
 use futures_util::StreamExt; // rust-rules: trait-import
 
 /// Bounded result of one concurrent Backfill candidate execution pass.
-///
-/// This is intentionally smaller than the concrete latest-value snapshot planned for `pre.009`.
-/// It exposes only the admission/frontier/checkpoint facts required to prove safe resumption.
 #[derive(Clone, Debug, Eq, PartialEq)]
 pub struct BackfillExecutionBatch {
     candidate_count: usize,
     admitted_count: usize,
     finished_count: usize,
     durable_count: usize,
+    inserted_count: usize,
+    already_present_count: usize,
+    purged_count: usize,
+    missing_count: usize,
+    conflict_count: usize,
+    observation_inserted_count: usize,
+    observation_already_present_count: usize,
+    cancelled_count: usize,
     hole_count: usize,
     maximum_in_flight: usize,
     local_contiguous_completed: usize,
     discovery_partial: bool,
+    cancelled: bool,
     checkpoint: crate::BackfillCheckpoint,
     failure_code: std::option::Option<ksp_core_lib::ErrorCode>,
 }
@@ -46,6 +52,54 @@ impl BackfillExecutionBatch {
         return self.durable_count;
     }
 
+    /// Returns the number of newly inserted canonical RAW entities.
+    #[must_use]
+    pub const fn inserted_count(&self) -> usize {
+        return self.inserted_count;
+    }
+
+    /// Returns the number of canonical RAW entities already durable.
+    #[must_use]
+    pub const fn already_present_count(&self) -> usize {
+        return self.already_present_count;
+    }
+
+    /// Returns the number of purge tombstones respected by normal persistence.
+    #[must_use]
+    pub const fn purged_count(&self) -> usize {
+        return self.purged_count;
+    }
+
+    /// Returns the number of candidates whose `getTransaction` result was missing.
+    #[must_use]
+    pub const fn missing_count(&self) -> usize {
+        return self.missing_count;
+    }
+
+    /// Returns the number of Store content conflicts.
+    #[must_use]
+    pub const fn conflict_count(&self) -> usize {
+        return self.conflict_count;
+    }
+
+    /// Returns the number of newly inserted acquisition observations.
+    #[must_use]
+    pub const fn observation_inserted_count(&self) -> usize {
+        return self.observation_inserted_count;
+    }
+
+    /// Returns the number of acquisition observations already durable.
+    #[must_use]
+    pub const fn observation_already_present_count(&self) -> usize {
+        return self.observation_already_present_count;
+    }
+
+    /// Returns the number of admitted candidates cancelled before Store submission.
+    #[must_use]
+    pub const fn cancelled_count(&self) -> usize {
+        return self.cancelled_count;
+    }
+
     /// Returns the number of known candidate outcomes that intentionally block the contiguous frontier.
     #[must_use]
     pub const fn hole_count(&self) -> usize {
@@ -76,20 +130,114 @@ impl BackfillExecutionBatch {
         return self.failure_code;
     }
 
+    /// Returns whether cooperative cancellation stopped admissions or cancelled an admitted pre-Store operation.
+    #[must_use]
+    pub const fn was_cancelled(&self) -> bool {
+        return self.cancelled;
+    }
+
     /// Returns whether discovery or candidate outcomes left the pass incomplete.
     #[must_use]
     pub const fn is_partial(&self) -> bool {
-        return self.discovery_partial || self.hole_count != 0 || self.failure_code.is_some();
+        return self.discovery_partial || self.hole_count != 0 || self.failure_code.is_some() || self.cancelled;
+    }
+}
+
+/// Internal latest-value execution facts published after each known candidate completion.
+#[derive(Clone, Debug, Eq, PartialEq)]
+pub(crate) struct BackfillExecutionProgress {
+    admitted_count: usize,
+    finished_count: usize,
+    inserted_count: usize,
+    already_present_count: usize,
+    purged_count: usize,
+    missing_count: usize,
+    conflict_count: usize,
+    observation_inserted_count: usize,
+    observation_already_present_count: usize,
+    cancelled_count: usize,
+    hole_count: usize,
+    maximum_in_flight: usize,
+    contiguous_completed: usize,
+    checkpoint: crate::BackfillCheckpoint,
+}
+
+impl BackfillExecutionProgress {
+    /// Returns the cumulative admission count.
+    pub(crate) const fn admitted_count(&self) -> usize {
+        return self.admitted_count;
+    }
+
+    /// Returns the cumulative finished count.
+    pub(crate) const fn finished_count(&self) -> usize {
+        return self.finished_count;
+    }
+
+    /// Returns the cumulative inserted entity count.
+    pub(crate) const fn inserted_count(&self) -> usize {
+        return self.inserted_count;
+    }
+
+    /// Returns the cumulative already-present entity count.
+    pub(crate) const fn already_present_count(&self) -> usize {
+        return self.already_present_count;
+    }
+
+    /// Returns the cumulative purged entity count.
+    pub(crate) const fn purged_count(&self) -> usize {
+        return self.purged_count;
+    }
+
+    /// Returns the cumulative missing count.
+    pub(crate) const fn missing_count(&self) -> usize {
+        return self.missing_count;
+    }
+
+    /// Returns the cumulative conflict count.
+    pub(crate) const fn conflict_count(&self) -> usize {
+        return self.conflict_count;
+    }
+
+    /// Returns the cumulative inserted observation count.
+    pub(crate) const fn observation_inserted_count(&self) -> usize {
+        return self.observation_inserted_count;
+    }
+
+    /// Returns the cumulative already-present observation count.
+    pub(crate) const fn observation_already_present_count(&self) -> usize {
+        return self.observation_already_present_count;
+    }
+
+    /// Returns the cumulative cancelled candidate count.
+    pub(crate) const fn cancelled_count(&self) -> usize {
+        return self.cancelled_count;
+    }
+
+    /// Returns the cumulative hole count.
+    pub(crate) const fn hole_count(&self) -> usize {
+        return self.hole_count;
+    }
+
+    /// Returns the maximum observed in-flight count.
+    pub(crate) const fn maximum_in_flight(&self) -> usize {
+        return self.maximum_in_flight;
+    }
+
+    /// Returns the cumulative safe contiguous prefix represented by the progress checkpoint.
+    pub(crate) const fn contiguous_completed(&self) -> usize {
+        return self.contiguous_completed;
+    }
+
+    /// Returns the safe checkpoint proven at this progress position.
+    pub(crate) const fn checkpoint(&self) -> &crate::BackfillCheckpoint {
+        return &self.checkpoint;
     }
 }
 
 /// Executes one bounded discovered candidate set with request-owned hydration concurrency.
 ///
-/// At most `request.hydration_concurrency()` candidate futures are kept in flight. Candidate
-/// completions may arrive out of order, but the checkpoint advances only across the contiguous
-/// durable prefix. `Missing` remains a non-fatal hole. Conflict or any Transport/conversion/Store
-/// error stops new admissions immediately; work already admitted is drained before this function
-/// returns so a submitted Store operation is never silently abandoned by the batch coordinator.
+/// This compatibility entry point has no external cancellation handle. Transport owns retries and
+/// endpoint selection; Store submissions are always awaited to completion.
 pub async fn execute_backfill_discovery(
     transport: &ksp_onchain_transport_lib::HttpTransportPool,
     store: &ksp_store_lib::Store,
@@ -100,8 +248,21 @@ pub async fn execute_backfill_discovery(
     if let std::result::Result::Err(error) = validation {
         return std::result::Result::Err(error);
     }
-    let processor = RuntimeCandidateProcessor { transport, store, request };
-    return execute_with_processor(&processor, request, discovery).await;
+    let processor = RuntimeCandidateProcessor { transport, store, request, cancellation: std::option::Option::None };
+    return execute_with_processor(&processor, request, discovery, std::option::Option::None, std::option::Option::None).await;
+}
+
+/// Executes one bounded discovery with cooperative cancellation and concrete progress publication.
+pub(crate) async fn execute_backfill_discovery_cancellable(
+    transport: &ksp_onchain_transport_lib::HttpTransportPool,
+    store: &ksp_store_lib::Store,
+    request: &crate::BackfillRequest,
+    discovery: &crate::BackfillDiscovery,
+    cancellation: &crate::BackfillCancellationSignal,
+    publisher: &crate::BackfillRuntimePublisher,
+) -> ksp_core_lib::Result<BackfillExecutionBatch> {
+    let processor = RuntimeCandidateProcessor { transport, store, request, cancellation: std::option::Option::Some(cancellation) };
+    return execute_with_processor(&processor, request, discovery, std::option::Option::Some(cancellation), std::option::Option::Some(publisher)).await;
 }
 
 type CandidateProcessFuture<'a> =
@@ -115,6 +276,7 @@ struct RuntimeCandidateProcessor<'a> {
     transport: &'a ksp_onchain_transport_lib::HttpTransportPool,
     store: &'a ksp_store_lib::Store,
     request: &'a crate::BackfillRequest,
+    cancellation: std::option::Option<&'a crate::BackfillCancellationSignal>,
 }
 
 impl CandidateProcessor for RuntimeCandidateProcessor<'_> {
@@ -124,20 +286,31 @@ impl CandidateProcessor for RuntimeCandidateProcessor<'_> {
                 std::result::Result::Ok(value) => value,
                 std::result::Result::Err(error) => return std::result::Result::Err(error),
             };
-            let hydration = crate::hydrate_backfill_candidate(self.transport, self.request, candidate, received_at).await;
+            let hydration_future = crate::hydrate_backfill_candidate(self.transport, self.request, candidate, received_at);
+            let hydration = match self.cancellation {
+                std::option::Option::Some(signal) => signal.run_cancellable(hydration_future).await,
+                std::option::Option::None => hydration_future.await,
+            };
             let hydration = match hydration {
                 std::result::Result::Ok(value) => value,
                 std::result::Result::Err(error) => return std::result::Result::Err(error),
             };
+            if self.cancellation.is_some_and(crate::BackfillCancellationSignal::is_requested) {
+                return std::result::Result::Err(cancelled_error());
+            }
+            // Deliberately not cancellable once Store submission begins: a submitted durable write is drained.
             return crate::persist_backfill_hydration(self.store, hydration).await;
         });
     }
 }
 
+#[allow(clippy::too_many_lines)]
 async fn execute_with_processor<P>(
     processor: &P,
     request: &crate::BackfillRequest,
     discovery: &crate::BackfillDiscovery,
+    cancellation: std::option::Option<&crate::BackfillCancellationSignal>,
+    publisher: std::option::Option<&crate::BackfillRuntimePublisher>,
 ) -> ksp_core_lib::Result<BackfillExecutionBatch>
 where
     P: CandidateProcessor,
@@ -161,12 +334,53 @@ where
     let mut admitted_count = 0_usize;
     let mut finished_count = 0_usize;
     let mut durable_count = 0_usize;
+    let mut inserted_count = 0_usize;
+    let mut already_present_count = 0_usize;
+    let mut purged_count = 0_usize;
+    let mut missing_count = 0_usize;
+    let mut conflict_count = 0_usize;
+    let mut observation_inserted_count = 0_usize;
+    let mut observation_already_present_count = 0_usize;
+    let mut cancelled_count = 0_usize;
     let mut hole_count = 0_usize;
     let mut maximum_in_flight = 0_usize;
     let mut failure_code = std::option::Option::<ksp_core_lib::ErrorCode>::None;
+    let mut cancellation_requested = false;
     let mut in_flight = futures_util::stream::FuturesUnordered::new();
     loop {
-        while failure_code.is_none() && next_index < discovery.candidates().len() && in_flight.len() < request.hydration_concurrency() {
+        if !cancellation_requested && cancellation.is_some_and(crate::BackfillCancellationSignal::is_requested) {
+            cancellation_requested = true;
+            let progress = progress_from_state(
+                request,
+                discovery,
+                &frontier,
+                admitted_count,
+                finished_count,
+                inserted_count,
+                already_present_count,
+                purged_count,
+                missing_count,
+                conflict_count,
+                observation_inserted_count,
+                observation_already_present_count,
+                cancelled_count,
+                hole_count,
+                maximum_in_flight,
+            );
+            let progress = match progress {
+                std::result::Result::Ok(value) => value,
+                std::result::Result::Err(error) => return std::result::Result::Err(error),
+            };
+            let published = publish_progress(publisher, &progress, true, true);
+            if let std::result::Result::Err(error) = published {
+                return std::result::Result::Err(error);
+            }
+        }
+        while failure_code.is_none()
+            && !cancellation_requested
+            && next_index < discovery.candidates().len()
+            && in_flight.len() < request.hydration_concurrency()
+        {
             let index = next_index;
             let candidate = &discovery.candidates()[index];
             let future = processor.process(candidate);
@@ -191,6 +405,19 @@ where
         };
         match result {
             std::result::Result::Ok(outcome) => {
+                let classified = classify_persistence(
+                    &outcome,
+                    &mut inserted_count,
+                    &mut already_present_count,
+                    &mut purged_count,
+                    &mut missing_count,
+                    &mut conflict_count,
+                    &mut observation_inserted_count,
+                    &mut observation_already_present_count,
+                );
+                if let std::result::Result::Err(error) = classified {
+                    return std::result::Result::Err(error);
+                }
                 if persistence_advances_frontier(&outcome) {
                     let marked = frontier.mark_durable(index);
                     if let std::result::Result::Err(error) = marked {
@@ -215,11 +442,43 @@ where
                     std::result::Result::Ok(value) => value,
                     std::result::Result::Err(increment_error) => return std::result::Result::Err(increment_error),
                 };
-                if failure_code.is_none() {
+                if error.code() == crate::ERROR_CODE_BACKFILL_CANCELLED {
+                    cancelled_count = match checked_increment(cancelled_count, "cancelled_count") {
+                        std::result::Result::Ok(value) => value,
+                        std::result::Result::Err(increment_error) => return std::result::Result::Err(increment_error),
+                    };
+                    cancellation_requested = true;
+                } else if failure_code.is_none() {
                     failure_code = std::option::Option::Some(error.code());
                 }
             },
         }
+        let progress = progress_from_state(
+            request,
+            discovery,
+            &frontier,
+            admitted_count,
+            finished_count,
+            inserted_count,
+            already_present_count,
+            purged_count,
+            missing_count,
+            conflict_count,
+            observation_inserted_count,
+            observation_already_present_count,
+            cancelled_count,
+            hole_count,
+            maximum_in_flight,
+        );
+        let progress = match progress {
+            std::result::Result::Ok(value) => value,
+            std::result::Result::Err(error) => return std::result::Result::Err(error),
+        };
+        let draining = cancellation_requested || failure_code.is_some();
+        let published = publish_progress(publisher, &progress, cancellation_requested, draining);
+        if let std::result::Result::Err(error) = published {
+            return std::result::Result::Err(error);
+        }
     }
     let checkpoint = match crate::checkpoint_from_frontier(request, discovery, &frontier) {
         std::result::Result::Ok(value) => value,
@@ -230,15 +489,140 @@ where
         admitted_count,
         finished_count,
         durable_count,
+        inserted_count,
+        already_present_count,
+        purged_count,
+        missing_count,
+        conflict_count,
+        observation_inserted_count,
+        observation_already_present_count,
+        cancelled_count,
         hole_count,
         maximum_in_flight,
         local_contiguous_completed: frontier.contiguous_completed(),
         discovery_partial: discovery.is_partial(),
+        cancelled: cancellation_requested,
         checkpoint,
         failure_code,
     });
 }
 
+#[allow(clippy::too_many_arguments)]
+fn progress_from_state(
+    request: &crate::BackfillRequest,
+    discovery: &crate::BackfillDiscovery,
+    frontier: &crate::CompletionFrontier,
+    admitted_count: usize,
+    finished_count: usize,
+    inserted_count: usize,
+    already_present_count: usize,
+    purged_count: usize,
+    missing_count: usize,
+    conflict_count: usize,
+    observation_inserted_count: usize,
+    observation_already_present_count: usize,
+    cancelled_count: usize,
+    hole_count: usize,
+    maximum_in_flight: usize,
+) -> ksp_core_lib::Result<BackfillExecutionProgress> {
+    let checkpoint = match crate::checkpoint_from_frontier(request, discovery, frontier) {
+        std::result::Result::Ok(value) => value,
+        std::result::Result::Err(error) => return std::result::Result::Err(error),
+    };
+    return std::result::Result::Ok(BackfillExecutionProgress {
+        admitted_count,
+        finished_count,
+        inserted_count,
+        already_present_count,
+        purged_count,
+        missing_count,
+        conflict_count,
+        observation_inserted_count,
+        observation_already_present_count,
+        cancelled_count,
+        hole_count,
+        maximum_in_flight,
+        contiguous_completed: checkpoint.completed_prefix(),
+        checkpoint,
+    });
+}
+
+fn publish_progress(
+    publisher: std::option::Option<&crate::BackfillRuntimePublisher>,
+    progress: &BackfillExecutionProgress,
+    cancelling: bool,
+    draining: bool,
+) -> ksp_core_lib::Result<()> {
+    if let std::option::Option::Some(publisher) = publisher {
+        let published = publisher.publish_execution_progress(progress, cancelling, draining);
+        if let std::result::Result::Err(error) = published {
+            return std::result::Result::Err(error);
+        }
+    }
+    return std::result::Result::Ok(());
+}
+
+#[allow(clippy::too_many_arguments)]
+fn classify_persistence(
+    outcome: &crate::BackfillPersistenceOutcome,
+    inserted_count: &mut usize,
+    already_present_count: &mut usize,
+    purged_count: &mut usize,
+    missing_count: &mut usize,
+    conflict_count: &mut usize,
+    observation_inserted_count: &mut usize,
+    observation_already_present_count: &mut usize,
+) -> ksp_core_lib::Result<()> {
+    match outcome.entity() {
+        crate::BackfillEntityPersistence::Inserted => {
+            *inserted_count = match checked_increment(*inserted_count, "inserted_count") {
+                std::result::Result::Ok(value) => value,
+                std::result::Result::Err(error) => return std::result::Result::Err(error),
+            };
+        },
+        crate::BackfillEntityPersistence::AlreadyPresent => {
+            *already_present_count = match checked_increment(*already_present_count, "already_present_count") {
+                std::result::Result::Ok(value) => value,
+                std::result::Result::Err(error) => return std::result::Result::Err(error),
+            };
+        },
+        crate::BackfillEntityPersistence::SkippedPurged => {
+            *purged_count = match checked_increment(*purged_count, "purged_count") {
+                std::result::Result::Ok(value) => value,
+                std::result::Result::Err(error) => return std::result::Result::Err(error),
+            };
+        },
+        crate::BackfillEntityPersistence::Missing => {
+            *missing_count = match checked_increment(*missing_count, "missing_count") {
+                std::result::Result::Ok(value) => value,
+                std::result::Result::Err(error) => return std::result::Result::Err(error),
+            };
+        },
+        crate::BackfillEntityPersistence::Conflict => {
+            *conflict_count = match checked_increment(*conflict_count, "conflict_count") {
+                std::result::Result::Ok(value) => value,
+                std::result::Result::Err(error) => return std::result::Result::Err(error),
+            };
+        },
+    }
+    match outcome.observation() {
+        crate::BackfillObservationPersistence::Inserted => {
+            *observation_inserted_count = match checked_increment(*observation_inserted_count, "observation_inserted_count") {
+                std::result::Result::Ok(value) => value,
+                std::result::Result::Err(error) => return std::result::Result::Err(error),
+            };
+        },
+        crate::BackfillObservationPersistence::AlreadyPresent => {
+            *observation_already_present_count = match checked_increment(*observation_already_present_count, "observation_already_present_count") {
+                std::result::Result::Ok(value) => value,
+                std::result::Result::Err(error) => return std::result::Result::Err(error),
+            };
+        },
+        crate::BackfillObservationPersistence::NotRecorded | crate::BackfillObservationPersistence::NotApplicable => {},
+    }
+    return std::result::Result::Ok(());
+}
+
 fn persistence_advances_frontier(outcome: &crate::BackfillPersistenceOutcome) -> bool {
     return matches!(
         outcome.entity(),
@@ -263,6 +647,10 @@ fn current_raw_timestamp() -> ksp_core_lib::Result<ksp_store_lib::RawTimestamp>
     return ksp_store_lib::RawTimestamp::from_unix_millis(millis);
 }
 
+fn cancelled_error() -> ksp_core_lib::Error {
+    return ksp_core_lib::Error::new(crate::ERROR_CODE_BACKFILL_CANCELLED, "Backfill candidate cancelled before durable Store submission");
+}
+
 fn execution_error(field: &'static str) -> ksp_core_lib::Error {
     return ksp_core_lib::Error::new(crate::ERROR_CODE_BACKFILL_EXECUTION_INVALID, "invalid bounded Backfill execution state").with_context("field", field);
 }

crates/ksp-job-backfill-lib/src/lib.rs

@@ -1,5 +1,5 @@
 // file: crates/ksp-job-backfill-lib/src/lib.rs
-// version: 4
+// version: 5
 
 #![warn(missing_docs)]
 #![deny(unreachable_pub)]
@@ -11,8 +11,8 @@
 //! `getSignaturesForAddress` pagination and canonical RAW v1 conversion through observed
 //! `getTransaction`. Transport retains provider/endpoint selection and retry; Store retains
 //! durable idempotence through the atomic Store facade. This tranche also owns bounded concurrent
-//! candidate execution and caller-owned contiguous checkpoints. Cancellation and concrete latest-value
-//! snapshots are added by later v0.3.6 tranches.
+//! candidate execution, caller-owned contiguous checkpoints, cooperative cancellation and concrete
+//! latest-value snapshots for external listeners.
 
 mod checkpoint;
 mod constants;
@@ -22,6 +22,7 @@ mod error;
 mod execution;
 mod persistence;
 mod request;
+mod runtime;
 
 /// Opaque caller-owned checkpoint for one controlled Backfill resumption.
 pub use self::checkpoint::BackfillCheckpoint;
@@ -59,6 +60,8 @@ pub use self::error::ERROR_CODE_BACKFILL_PERSISTENCE_INVALID;
 pub use self::error::ERROR_CODE_BACKFILL_RAW_CONVERSION_INVALID;
 /// Error code used when one Backfill request violates its bounded admission contract.
 pub use self::error::ERROR_CODE_BACKFILL_REQUEST_INVALID;
+/// Error code used when the concrete Backfill runtime reaches an impossible lifecycle or notification state.
+pub use self::error::ERROR_CODE_BACKFILL_RUNTIME_INVALID;
 /// Error code used when one transaction signature text violates the bounded Base58-shape contract.
 pub use self::error::ERROR_CODE_BACKFILL_SIGNATURE_INVALID;
 /// Bounded result of one concurrent Backfill candidate execution pass.
@@ -97,6 +100,18 @@ pub use self::request::MAX_BACKFILL_PAGES;
 pub use self::request::MAX_BACKFILL_SIGNATURE_TEXT_BYTES;
 /// Minimum Base58 text length possible for one canonical 64-byte Solana signature.
 pub use self::request::MIN_BACKFILL_SIGNATURE_TEXT_BYTES;
+/// Stable Job kind code used by the concrete historical RAW transaction Backfill runtime.
+pub use self::runtime::BACKFILL_JOB_KIND_CODE;
+/// Cloneable external control handle for one concrete Backfill runtime.
+pub use self::runtime::BackfillJobHandle;
+/// Current concrete phase of one historical RAW transaction Backfill Job.
+pub use self::runtime::BackfillJobPhase;
+/// Concrete single-run Backfill coordinator paired with a cloneable control/snapshot handle.
+pub use self::runtime::BackfillJobRuntime;
+/// Complete safe latest-value snapshot of one concrete historical RAW transaction Backfill Job.
+pub use self::runtime::BackfillJobSnapshot;
+/// Cloneable runtime-neutral-facing latest-value source for concrete Backfill snapshots.
+pub use self::runtime::BackfillSnapshotSource;
 
 /// Internal contiguous completion frontier used by bounded execution.
 pub(crate) use self::checkpoint::CompletionFrontier;
@@ -114,3 +129,19 @@ pub(crate) use self::checkpoint::validate_request_checkpoint;
 pub(crate) use self::constants::TRACING_TARGET;
 /// Exact private Base58 decoder shared by the public signature wrapper and hydration path.
 pub(crate) use self::conversion::decode_backfill_signature;
+/// Internal cancellable discovery path used by the concrete runtime.
+pub(crate) use self::discovery::discover_backfill_candidates_cancellable;
+/// Internal cooperative cancellation code used to distinguish cancellation from failure.
+pub(crate) use self::error::ERROR_CODE_BACKFILL_CANCELLED;
+/// Internal execution progress facts fed into the concrete latest-value source.
+pub(crate) use self::execution::BackfillExecutionProgress;
+/// Internal cancellable execution path used by the concrete runtime.
+pub(crate) use self::execution::execute_backfill_discovery_cancellable;
+/// Internal concrete cancellation signal shared by discovery and execution.
+pub(crate) use self::runtime::BackfillCancellationSignal;
+/// Internal atomic terminal/cancellation arbitration shared by runtime tests and coordinator.
+pub(crate) use self::runtime::BackfillRuntimeControl;
+/// Internal execution progress publisher feeding the latest-value snapshot source.
+pub(crate) use self::runtime::BackfillRuntimePublisher;
+/// Internal terminal race result used by coordinator canaries.
+pub(crate) use self::runtime::TerminalClaim;

crates/ksp-job-backfill-lib/src/runtime.rs

@@ -0,0 +1,715 @@
+// file: crates/ksp-job-backfill-lib/src/runtime.rs
+// version: 2
+
+/// Stable Job kind code used by the concrete historical RAW transaction Backfill runtime.
+pub const BACKFILL_JOB_KIND_CODE: &str = "solana.raw_transaction.backfill";
+
+const CONTROL_ACTIVE: u8 = 0;
+const CONTROL_CANCELLATION_REQUESTED: u8 = 1;
+const CONTROL_CANCELLED: u8 = 3;
+const CONTROL_COMPLETED: u8 = 2;
+const CONTROL_FAILED: u8 = 4;
+
+/// Current concrete phase of one historical RAW transaction Backfill Job.
+#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)]
+#[non_exhaustive]
+pub enum BackfillJobPhase {
+    /// Runtime was created but execution has not started.
+    Created,
+    /// Candidate discovery is in progress.
+    Discovering,
+    /// Candidate hydration and persistence are being admitted and completed.
+    Executing,
+    /// Cancellation or a fatal result stopped admission while already submitted work is draining.
+    Draining,
+    /// No more work can be admitted and the Job is terminal.
+    Finished,
+}
+
+impl BackfillJobPhase {
+    /// Returns the stable safe code for this concrete runtime phase.
+    #[must_use]
+    pub const fn code(self) -> &'static str {
+        return match self {
+            Self::Created => "created",
+            Self::Discovering => "discovering",
+            Self::Executing => "executing",
+            Self::Draining => "draining",
+            Self::Finished => "finished",
+        };
+    }
+}
+
+/// Complete safe latest-value snapshot of one concrete historical RAW transaction Backfill Job.
+#[derive(Clone, Debug, Eq, PartialEq)]
+pub struct BackfillJobSnapshot {
+    phase: BackfillJobPhase,
+    scope_kind: crate::BackfillScopeKind,
+    discovery_boundary: std::option::Option<crate::BackfillDiscoveryBoundary>,
+    candidates_selected: usize,
+    candidates_admitted: usize,
+    candidates_finished: usize,
+    entities_inserted: usize,
+    entities_existing: usize,
+    entities_purged: usize,
+    missing: usize,
+    conflicts: usize,
+    observations_inserted: usize,
+    observations_existing: usize,
+    cancelled_candidates: usize,
+    holes: usize,
+    maximum_in_flight: usize,
+    contiguous_completed: usize,
+    checkpoint: std::option::Option<crate::BackfillCheckpoint>,
+    failure_code: std::option::Option<ksp_core_lib::ErrorCode>,
+}
+
+impl BackfillJobSnapshot {
+    fn initial(request: &crate::BackfillRequest) -> Self {
+        return Self {
+            phase: BackfillJobPhase::Created,
+            scope_kind: request.scope().kind(),
+            discovery_boundary: std::option::Option::None,
+            candidates_selected: 0,
+            candidates_admitted: 0,
+            candidates_finished: 0,
+            entities_inserted: 0,
+            entities_existing: 0,
+            entities_purged: 0,
+            missing: 0,
+            conflicts: 0,
+            observations_inserted: 0,
+            observations_existing: 0,
+            cancelled_candidates: 0,
+            holes: 0,
+            maximum_in_flight: 0,
+            contiguous_completed: request.checkpoint().map_or(0, crate::BackfillCheckpoint::completed_prefix),
+            checkpoint: request.checkpoint().cloned(),
+            failure_code: std::option::Option::None,
+        };
+    }
+
+    /// Returns the concrete execution phase represented by this snapshot.
+    #[must_use]
+    pub const fn phase(&self) -> BackfillJobPhase {
+        return self.phase;
+    }
+
+    /// Returns the safe scope category without rendering address or signature payloads.
+    #[must_use]
+    pub const fn scope_kind(&self) -> crate::BackfillScopeKind {
+        return self.scope_kind;
+    }
+
+    /// Returns the discovery boundary once candidate selection has completed.
+    #[must_use]
+    pub const fn discovery_boundary(&self) -> std::option::Option<crate::BackfillDiscoveryBoundary> {
+        return self.discovery_boundary;
+    }
+
+    /// Returns the bounded candidate count selected by discovery.
+    #[must_use]
+    pub const fn candidates_selected(&self) -> usize {
+        return self.candidates_selected;
+    }
+
+    /// Returns the number of candidates admitted into hydration.
+    #[must_use]
+    pub const fn candidates_admitted(&self) -> usize {
+        return self.candidates_admitted;
+    }
+
+    /// Returns the number of admitted candidates that reached a known coordinator result.
+    #[must_use]
+    pub const fn candidates_finished(&self) -> usize {
+        return self.candidates_finished;
+    }
+
+    /// Returns the number of newly inserted canonical RAW transactions.
+    #[must_use]
+    pub const fn entities_inserted(&self) -> usize {
+        return self.entities_inserted;
+    }
+
+    /// Returns the number of identical canonical RAW transactions already present.
+    #[must_use]
+    pub const fn entities_existing(&self) -> usize {
+        return self.entities_existing;
+    }
+
+    /// Returns the number of durable purge tombstones respected by normal Backfill persistence.
+    #[must_use]
+    pub const fn entities_purged(&self) -> usize {
+        return self.entities_purged;
+    }
+
+    /// Returns the number of `getTransaction = null` candidates.
+    #[must_use]
+    pub const fn missing(&self) -> usize {
+        return self.missing;
+    }
+
+    /// Returns the number of Store content conflicts observed by this run.
+    #[must_use]
+    pub const fn conflicts(&self) -> usize {
+        return self.conflicts;
+    }
+
+    /// Returns the number of newly inserted acquisition observations.
+    #[must_use]
+    pub const fn observations_inserted(&self) -> usize {
+        return self.observations_inserted;
+    }
+
+    /// Returns the number of acquisition observations already durable.
+    #[must_use]
+    pub const fn observations_existing(&self) -> usize {
+        return self.observations_existing;
+    }
+
+    /// Returns the number of admitted candidates cancelled before Store submission.
+    #[must_use]
+    pub const fn cancelled_candidates(&self) -> usize {
+        return self.cancelled_candidates;
+    }
+
+    /// Returns the number of known candidate outcomes blocking the contiguous frontier.
+    #[must_use]
+    pub const fn holes(&self) -> usize {
+        return self.holes;
+    }
+
+    /// Returns the greatest observed concurrent candidate count.
+    #[must_use]
+    pub const fn maximum_in_flight(&self) -> usize {
+        return self.maximum_in_flight;
+    }
+
+    /// Returns the cumulative safe contiguous completion prefix represented by the current checkpoint.
+    #[must_use]
+    pub const fn contiguous_completed(&self) -> usize {
+        return self.contiguous_completed;
+    }
+
+    /// Returns the latest caller-owned safe checkpoint, when one exists.
+    #[must_use]
+    pub const fn checkpoint(&self) -> std::option::Option<&crate::BackfillCheckpoint> {
+        return self.checkpoint.as_ref();
+    }
+
+    /// Returns the stable fatal error code retained by a failed Job, when one exists.
+    #[must_use]
+    pub const fn failure_code(&self) -> std::option::Option<ksp_core_lib::ErrorCode> {
+        return self.failure_code;
+    }
+}
+
+/// Cloneable runtime-neutral-facing latest-value source for concrete Backfill snapshots.
+#[derive(Clone)]
+pub struct BackfillSnapshotSource {
+    receiver: tokio::sync::watch::Receiver<ksp_job_api::JobNotification<BackfillJobSnapshot>>,
+}
+
+impl ksp_job_api::JobSnapshotSource for BackfillSnapshotSource {
+    type Snapshot = BackfillJobSnapshot;
+
+    fn current(&self) -> ksp_job_api::JobNotification<Self::Snapshot> {
+        return self.receiver.borrow().clone();
+    }
+
+    fn wait_for_change(&self, observed: ksp_job_api::JobNotificationSequence) -> ksp_job_api::JobSnapshotFuture<'_, Self::Snapshot> {
+        let mut receiver = self.receiver.clone();
+        return std::boxed::Box::pin(async move {
+            loop {
+                let current = receiver.borrow().clone();
+                if current.sequence().is_after(observed) || current.state().is_terminal() {
+                    return current;
+                }
+                let changed = receiver.changed().await;
+                if changed.is_err() {
+                    return receiver.borrow().clone();
+                }
+            }
+        });
+    }
+}
+
+impl std::fmt::Debug for BackfillSnapshotSource {
+    fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        let current = self.receiver.borrow();
+        return formatter.debug_struct("BackfillSnapshotSource").field("sequence", &current.sequence()).field("state", &current.state()).finish();
+    }
+}
+
+/// Cloneable external control handle for one concrete Backfill runtime.
+#[derive(Clone)]
+pub struct BackfillJobHandle {
+    control: BackfillRuntimeControl,
+    snapshots: BackfillSnapshotSource,
+}
+
+impl BackfillJobHandle {
+    /// Requests cooperative cancellation and returns `true` only when accepted before terminal publication.
+    #[must_use]
+    pub fn cancel(&self) -> bool {
+        return self.control.request_cancellation();
+    }
+
+    /// Returns an independent latest-value snapshot source for one listener.
+    #[must_use]
+    pub fn snapshots(&self) -> BackfillSnapshotSource {
+        return self.snapshots.clone();
+    }
+
+    /// Returns whether cooperative cancellation has been accepted for this non-terminal Job.
+    #[must_use]
+    pub fn is_cancellation_requested(&self) -> bool {
+        return self.control.is_cancellation_requested();
+    }
+}
+
+impl std::fmt::Debug for BackfillJobHandle {
+    fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        return formatter
+            .debug_struct("BackfillJobHandle")
+            .field("cancellation_requested", &self.is_cancellation_requested())
+            .field("snapshot", &<BackfillSnapshotSource as ksp_job_api::JobSnapshotSource>::current(&self.snapshots))
+            .finish();
+    }
+}
+
+/// Concrete single-run Backfill coordinator paired with a cloneable control/snapshot handle.
+pub struct BackfillJobRuntime {
+    request: crate::BackfillRequest,
+    control: BackfillRuntimeControl,
+    cancellation: BackfillCancellationSignal,
+    publisher: BackfillRuntimePublisher,
+    handle: BackfillJobHandle,
+}
+
+impl BackfillJobRuntime {
+    /// Creates one runtime in `Created` state and its stable latest-value channel.
+    pub fn new(request: crate::BackfillRequest) -> ksp_core_lib::Result<Self> {
+        let kind = match ksp_job_api::JobKindCode::new(BACKFILL_JOB_KIND_CODE) {
+            std::result::Result::Ok(value) => value,
+            std::result::Result::Err(error) => return std::result::Result::Err(error),
+        };
+        let initial_snapshot = BackfillJobSnapshot::initial(&request);
+        let initial = ksp_job_api::JobNotification::new(
+            request.job_id().clone(),
+            kind.clone(),
+            ksp_job_api::JobNotificationSequence::initial(),
+            ksp_job_api::JobState::Created,
+            initial_snapshot,
+        );
+        let (snapshot_sender, snapshot_receiver) = tokio::sync::watch::channel(initial);
+        let (cancel_sender, cancel_receiver) = tokio::sync::watch::channel(false);
+        let control = BackfillRuntimeControl::new(cancel_sender);
+        let cancellation = BackfillCancellationSignal::new(control.token(), cancel_receiver);
+        let snapshots = BackfillSnapshotSource { receiver: snapshot_receiver };
+        let handle = BackfillJobHandle { control: control.clone(), snapshots: snapshots.clone() };
+        let publisher = BackfillRuntimePublisher { sender: snapshot_sender, id: request.job_id().clone(), kind };
+        return std::result::Result::Ok(Self { request, control, cancellation, publisher, handle });
+    }
+
+    /// Returns a cloneable control and latest-value observation handle before the runtime is moved into execution.
+    #[must_use]
+    pub fn handle(&self) -> BackfillJobHandle {
+        return self.handle.clone();
+    }
+
+    /// Runs discovery, bounded execution, cooperative cancellation and terminal snapshot publication.
+    pub async fn run(
+        self,
+        transport: &ksp_onchain_transport_lib::HttpTransportPool,
+        store: &ksp_store_lib::Store,
+    ) -> ksp_core_lib::Result<BackfillJobSnapshot> {
+        if self.cancellation.is_requested() {
+            let claimed = self.control.claim_normal_terminal();
+            if claimed != TerminalClaim::Cancelled {
+                return std::result::Result::Err(runtime_error("terminal.pre_start"));
+            }
+            return self.publisher.publish_cancelled_from_created();
+        }
+        let started = self.publisher.publish_running(BackfillJobPhase::Discovering);
+        if let std::result::Result::Err(error) = started {
+            self.control.claim_failed();
+            return std::result::Result::Err(error);
+        }
+        let discovery = crate::discover_backfill_candidates_cancellable(transport, &self.request, &self.cancellation).await;
+        let discovery = match discovery {
+            std::result::Result::Ok(value) => value,
+            std::result::Result::Err(error) => {
+                if error.code() == crate::ERROR_CODE_BACKFILL_CANCELLED {
+                    return self.finish_cancelled();
+                }
+                self.control.claim_failed();
+                let published = self.publisher.publish_failed(error.code());
+                if let std::result::Result::Err(publish_error) = published {
+                    return std::result::Result::Err(publish_error);
+                }
+                return std::result::Result::Err(error);
+            },
+        };
+        let published = self.publisher.publish_discovery(&discovery);
+        if let std::result::Result::Err(error) = published {
+            self.control.claim_failed();
+            return std::result::Result::Err(error);
+        }
+        let batch = crate::execute_backfill_discovery_cancellable(transport, store, &self.request, &discovery, &self.cancellation, &self.publisher).await;
+        let batch = match batch {
+            std::result::Result::Ok(value) => value,
+            std::result::Result::Err(error) => {
+                self.control.claim_failed();
+                let published = self.publisher.publish_failed(error.code());
+                if let std::result::Result::Err(publish_error) = published {
+                    return std::result::Result::Err(publish_error);
+                }
+                return std::result::Result::Err(error);
+            },
+        };
+        if let std::option::Option::Some(code) = batch.failure_code() {
+            self.control.claim_failed();
+            let published = self.publisher.publish_batch_terminal(&batch, ksp_job_api::JobState::Failed, std::option::Option::Some(code));
+            if let std::result::Result::Err(error) = published {
+                return std::result::Result::Err(error);
+            }
+            return std::result::Result::Err(ksp_core_lib::Error::new(code, "Backfill execution reached a fatal candidate result"));
+        }
+        let terminal = self.control.claim_normal_terminal();
+        return match terminal {
+            TerminalClaim::Cancelled => {
+                let cancelling = self.publisher.publish_batch_cancelling(&batch);
+                if let std::result::Result::Err(error) = cancelling {
+                    return std::result::Result::Err(error);
+                }
+                self.publisher.publish_batch_terminal(&batch, ksp_job_api::JobState::Cancelled, std::option::Option::None)
+            },
+            TerminalClaim::Completed => {
+                let completion = if batch.is_partial() { ksp_job_api::JobCompletion::Partial } else { ksp_job_api::JobCompletion::Complete };
+                self.publisher.publish_batch_terminal(&batch, ksp_job_api::JobState::Completed(completion), std::option::Option::None)
+            },
+            TerminalClaim::Failed => std::result::Result::Err(runtime_error("terminal.failed")),
+        };
+    }
+
+    fn finish_cancelled(&self) -> ksp_core_lib::Result<BackfillJobSnapshot> {
+        let terminal = self.control.claim_normal_terminal();
+        if terminal != TerminalClaim::Cancelled {
+            return std::result::Result::Err(runtime_error("terminal.cancelled"));
+        }
+        let cancelling = self.publisher.publish_cancelling();
+        if let std::result::Result::Err(error) = cancelling {
+            return std::result::Result::Err(error);
+        }
+        return self.publisher.publish_terminal(ksp_job_api::JobState::Cancelled, std::option::Option::None);
+    }
+}
+
+/// Internal atomic terminal/cancellation arbitration shared by runtime and external handle.
+#[derive(Clone)]
+pub(crate) struct BackfillRuntimeControl {
+    state: std::sync::Arc<std::sync::atomic::AtomicU8>,
+    token: ksp_job_api::JobCancellationToken,
+    cancel_sender: tokio::sync::watch::Sender<bool>,
+}
+
+impl BackfillRuntimeControl {
+    /// Creates one active control state paired with the cancellation wake channel.
+    pub(crate) fn new(cancel_sender: tokio::sync::watch::Sender<bool>) -> Self {
+        return Self {
+            state: std::sync::Arc::new(std::sync::atomic::AtomicU8::new(CONTROL_ACTIVE)),
+            token: ksp_job_api::JobCancellationToken::new(),
+            cancel_sender,
+        };
+    }
+
+    /// Returns the runtime-neutral cancellation token mirrored by this control.
+    pub(crate) fn token(&self) -> ksp_job_api::JobCancellationToken {
+        return self.token.clone();
+    }
+
+    /// Atomically accepts the first pre-terminal cancellation request.
+    pub(crate) fn request_cancellation(&self) -> bool {
+        let accepted = self
+            .state
+            .compare_exchange(CONTROL_ACTIVE, CONTROL_CANCELLATION_REQUESTED, std::sync::atomic::Ordering::AcqRel, std::sync::atomic::Ordering::Acquire)
+            .is_ok();
+        if accepted && self.token.cancel() {
+            let _previous = self.cancel_sender.send_replace(true);
+        }
+        return accepted;
+    }
+
+    /// Returns whether cooperative cancellation was accepted.
+    pub(crate) fn is_cancellation_requested(&self) -> bool {
+        return self.token.is_cancellation_requested();
+    }
+
+    /// Atomically resolves the completion-versus-cancellation terminal race.
+    pub(crate) fn claim_normal_terminal(&self) -> TerminalClaim {
+        let completed =
+            self.state.compare_exchange(CONTROL_ACTIVE, CONTROL_COMPLETED, std::sync::atomic::Ordering::AcqRel, std::sync::atomic::Ordering::Acquire);
+        if completed.is_ok() {
+            return TerminalClaim::Completed;
+        }
+        let cancelled = self.state.compare_exchange(
+            CONTROL_CANCELLATION_REQUESTED,
+            CONTROL_CANCELLED,
+            std::sync::atomic::Ordering::AcqRel,
+            std::sync::atomic::Ordering::Acquire,
+        );
+        if cancelled.is_ok() {
+            return TerminalClaim::Cancelled;
+        }
+        return TerminalClaim::Failed;
+    }
+
+    /// Marks a non-terminal control as failed, overriding a pending cancellation request.
+    pub(crate) fn claim_failed(&self) {
+        loop {
+            let state = self.state.load(std::sync::atomic::Ordering::Acquire);
+            if matches!(state, CONTROL_COMPLETED | CONTROL_CANCELLED | CONTROL_FAILED) {
+                return;
+            }
+            let changed = self.state.compare_exchange(state, CONTROL_FAILED, std::sync::atomic::Ordering::AcqRel, std::sync::atomic::Ordering::Acquire);
+            if changed.is_ok() {
+                return;
+            }
+        }
+    }
+}
+
+/// Internal wakeable cancellation signal used only around abandonable pre-Store futures.
+#[derive(Clone)]
+pub(crate) struct BackfillCancellationSignal {
+    token: ksp_job_api::JobCancellationToken,
+    receiver: tokio::sync::watch::Receiver<bool>,
+}
+
+impl BackfillCancellationSignal {
+    /// Creates one signal from the runtime-neutral token and Tokio wake receiver.
+    pub(crate) fn new(token: ksp_job_api::JobCancellationToken, receiver: tokio::sync::watch::Receiver<bool>) -> Self {
+        return Self { token, receiver };
+    }
+
+    /// Returns whether cancellation has been requested.
+    pub(crate) fn is_requested(&self) -> bool {
+        return self.token.is_cancellation_requested();
+    }
+
+    /// Runs one abandonable operation until it completes or cancellation wins.
+    pub(crate) async fn run_cancellable<F, T>(&self, operation: F) -> ksp_core_lib::Result<T>
+    where
+        F: std::future::Future<Output = ksp_core_lib::Result<T>>,
+    {
+        if self.is_requested() {
+            return std::result::Result::Err(cancelled_error());
+        }
+        let mut receiver = self.receiver.clone();
+        return tokio::select! {
+            biased;
+            _ = wait_for_cancellation(&self.token, &mut receiver) => std::result::Result::Err(cancelled_error()),
+            result = operation => result,
+        };
+    }
+}
+
+async fn wait_for_cancellation(token: &ksp_job_api::JobCancellationToken, receiver: &mut tokio::sync::watch::Receiver<bool>) {
+    loop {
+        if token.is_cancellation_requested() || *receiver.borrow() {
+            return;
+        }
+        let changed = receiver.changed().await;
+        if changed.is_err() {
+            return;
+        }
+    }
+}
+
+/// Internal result of atomically claiming a normal terminal state.
+#[derive(Clone, Copy, Debug, Eq, PartialEq)]
+pub(crate) enum TerminalClaim {
+    Completed,
+    Cancelled,
+    Failed,
+}
+
+/// Internal latest-value publisher owning the concrete Backfill notification stream.
+#[derive(Clone)]
+pub(crate) struct BackfillRuntimePublisher {
+    sender: tokio::sync::watch::Sender<ksp_job_api::JobNotification<BackfillJobSnapshot>>,
+    id: ksp_job_api::JobId,
+    kind: ksp_job_api::JobKindCode,
+}
+
+impl BackfillRuntimePublisher {
+    /// Publishes one non-terminal running phase.
+    pub(crate) fn publish_running(&self, phase: BackfillJobPhase) -> ksp_core_lib::Result<BackfillJobSnapshot> {
+        return self.publish_with(ksp_job_api::JobState::Running, |snapshot| {
+            snapshot.phase = phase;
+        });
+    }
+
+    /// Publishes the complete bounded discovery result as the current execution snapshot.
+    pub(crate) fn publish_discovery(&self, discovery: &crate::BackfillDiscovery) -> ksp_core_lib::Result<BackfillJobSnapshot> {
+        return self.publish_with(ksp_job_api::JobState::Running, |snapshot| {
+            snapshot.phase = BackfillJobPhase::Executing;
+            snapshot.discovery_boundary = std::option::Option::Some(discovery.boundary());
+            snapshot.candidates_selected = discovery.candidates().len();
+        });
+    }
+
+    /// Publishes one coalescable execution progress value.
+    pub(crate) fn publish_execution_progress(
+        &self,
+        progress: &crate::BackfillExecutionProgress,
+        cancelling: bool,
+        draining: bool,
+    ) -> ksp_core_lib::Result<BackfillJobSnapshot> {
+        let state = if cancelling { ksp_job_api::JobState::Cancelling } else { ksp_job_api::JobState::Running };
+        return self.publish_with(state, |snapshot| {
+            snapshot.phase = if draining { BackfillJobPhase::Draining } else { BackfillJobPhase::Executing };
+            apply_progress(snapshot, progress);
+        });
+    }
+
+    /// Publishes cancellation observation before terminal cancellation.
+    pub(crate) fn publish_cancelling(&self) -> ksp_core_lib::Result<BackfillJobSnapshot> {
+        return self.publish_with(ksp_job_api::JobState::Cancelling, |snapshot| {
+            snapshot.phase = BackfillJobPhase::Draining;
+        });
+    }
+
+    /// Publishes the drained batch state while cancellation is terminalizing.
+    pub(crate) fn publish_batch_cancelling(&self, batch: &crate::BackfillExecutionBatch) -> ksp_core_lib::Result<BackfillJobSnapshot> {
+        return self.publish_with(ksp_job_api::JobState::Cancelling, |snapshot| {
+            snapshot.phase = BackfillJobPhase::Draining;
+            apply_batch(snapshot, batch);
+        });
+    }
+
+    /// Publishes one terminal state together with the fully drained execution batch.
+    pub(crate) fn publish_batch_terminal(
+        &self,
+        batch: &crate::BackfillExecutionBatch,
+        state: ksp_job_api::JobState,
+        failure_code: std::option::Option<ksp_core_lib::ErrorCode>,
+    ) -> ksp_core_lib::Result<BackfillJobSnapshot> {
+        return self.publish_with(state, |snapshot| {
+            snapshot.phase = BackfillJobPhase::Finished;
+            snapshot.failure_code = failure_code;
+            apply_batch(snapshot, batch);
+        });
+    }
+
+    /// Publishes a terminal failure before a batch exists.
+    pub(crate) fn publish_failed(&self, code: ksp_core_lib::ErrorCode) -> ksp_core_lib::Result<BackfillJobSnapshot> {
+        return self.publish_terminal(ksp_job_api::JobState::Failed, std::option::Option::Some(code));
+    }
+
+    /// Publishes a terminal state without a completed execution batch.
+    pub(crate) fn publish_terminal(
+        &self,
+        state: ksp_job_api::JobState,
+        failure_code: std::option::Option<ksp_core_lib::ErrorCode>,
+    ) -> ksp_core_lib::Result<BackfillJobSnapshot> {
+        return self.publish_with(state, |snapshot| {
+            snapshot.phase = BackfillJobPhase::Finished;
+            snapshot.failure_code = failure_code;
+        });
+    }
+
+    /// Publishes direct Created-to-Cancelled termination before execution starts.
+    pub(crate) fn publish_cancelled_from_created(&self) -> ksp_core_lib::Result<BackfillJobSnapshot> {
+        return self.publish_with(ksp_job_api::JobState::Cancelled, |snapshot| {
+            snapshot.phase = BackfillJobPhase::Finished;
+        });
+    }
+
+    fn publish_with<F>(&self, state: ksp_job_api::JobState, update: F) -> ksp_core_lib::Result<BackfillJobSnapshot>
+    where
+        F: FnOnce(&mut BackfillJobSnapshot),
+    {
+        let current = self.sender.borrow().clone();
+        if current.state().is_terminal() {
+            return std::result::Result::Err(runtime_error("notification.terminal"));
+        }
+        if !valid_snapshot_transition(current.state(), state) {
+            return std::result::Result::Err(runtime_error("notification.transition"));
+        }
+        let sequence = match current.sequence().next() {
+            std::result::Result::Ok(value) => value,
+            std::result::Result::Err(error) => return std::result::Result::Err(error),
+        };
+        let mut snapshot = current.snapshot().clone();
+        update(&mut snapshot);
+        let notification = ksp_job_api::JobNotification::new(self.id.clone(), self.kind.clone(), sequence, state, snapshot.clone());
+        let _previous = self.sender.send_replace(notification);
+        return std::result::Result::Ok(snapshot);
+    }
+}
+
+fn valid_snapshot_transition(source: ksp_job_api::JobState, target: ksp_job_api::JobState) -> bool {
+    if source.is_terminal() {
+        return false;
+    }
+    return matches!(
+        (source, target),
+        (ksp_job_api::JobState::Created, ksp_job_api::JobState::Running)
+            | (ksp_job_api::JobState::Created, ksp_job_api::JobState::Cancelled)
+            | (ksp_job_api::JobState::Running, ksp_job_api::JobState::Running)
+            | (ksp_job_api::JobState::Running, ksp_job_api::JobState::Cancelling)
+            | (ksp_job_api::JobState::Running, ksp_job_api::JobState::Completed(_))
+            | (ksp_job_api::JobState::Running, ksp_job_api::JobState::Failed)
+            | (ksp_job_api::JobState::Cancelling, ksp_job_api::JobState::Cancelling)
+            | (ksp_job_api::JobState::Cancelling, ksp_job_api::JobState::Cancelled)
+            | (ksp_job_api::JobState::Cancelling, ksp_job_api::JobState::Failed)
+    );
+}
+
+fn apply_progress(snapshot: &mut BackfillJobSnapshot, progress: &crate::BackfillExecutionProgress) {
+    snapshot.candidates_admitted = progress.admitted_count();
+    snapshot.candidates_finished = progress.finished_count();
+    snapshot.entities_inserted = progress.inserted_count();
+    snapshot.entities_existing = progress.already_present_count();
+    snapshot.entities_purged = progress.purged_count();
+    snapshot.missing = progress.missing_count();
+    snapshot.conflicts = progress.conflict_count();
+    snapshot.observations_inserted = progress.observation_inserted_count();
+    snapshot.observations_existing = progress.observation_already_present_count();
+    snapshot.cancelled_candidates = progress.cancelled_count();
+    snapshot.holes = progress.hole_count();
+    snapshot.maximum_in_flight = progress.maximum_in_flight();
+    snapshot.contiguous_completed = progress.contiguous_completed();
+    snapshot.checkpoint = std::option::Option::Some(progress.checkpoint().clone());
+}
+
+fn apply_batch(snapshot: &mut BackfillJobSnapshot, batch: &crate::BackfillExecutionBatch) {
+    snapshot.candidates_admitted = batch.admitted_count();
+    snapshot.candidates_finished = batch.finished_count();
+    snapshot.entities_inserted = batch.inserted_count();
+    snapshot.entities_existing = batch.already_present_count();
+    snapshot.entities_purged = batch.purged_count();
+    snapshot.missing = batch.missing_count();
+    snapshot.conflicts = batch.conflict_count();
+    snapshot.observations_inserted = batch.observation_inserted_count();
+    snapshot.observations_existing = batch.observation_already_present_count();
+    snapshot.cancelled_candidates = batch.cancelled_count();
+    snapshot.holes = batch.hole_count();
+    snapshot.maximum_in_flight = batch.maximum_in_flight();
+    snapshot.contiguous_completed = batch.checkpoint().completed_prefix();
+    snapshot.checkpoint = std::option::Option::Some(batch.checkpoint().clone());
+}
+
+fn cancelled_error() -> ksp_core_lib::Error {
+    return ksp_core_lib::Error::new(crate::ERROR_CODE_BACKFILL_CANCELLED, "Backfill operation cancelled before durable Store submission");
+}
+
+fn runtime_error(field: &'static str) -> ksp_core_lib::Error {
+    return ksp_core_lib::Error::new(crate::ERROR_CODE_BACKFILL_RUNTIME_INVALID, "invalid concrete Backfill runtime state").with_context("field", field);
+}
+
+#[cfg(test)]
+#[path = "../unit_tests/runtime.rs"]
+mod tests;