khadhroony-solana-project/crates/ksp-job-backfill-lib/src/runtime.rs

// file: crates/ksp-job-backfill-lib/src/runtime.rs
// version: 3

/// Stable Job kind code used by the concrete historical RAW transaction Backfill runtime.
pub const BACKFILL_JOB_KIND_CODE: &str = "solana.raw_transaction.backfill";

const CONTROL_ACTIVE: u8 = 0;
const CONTROL_CANCELLATION_REQUESTED: u8 = 1;
const CONTROL_CANCELLED: u8 = 3;
const CONTROL_COMPLETED: u8 = 2;
const CONTROL_FAILED: u8 = 4;

/// Current concrete phase of one historical RAW transaction Backfill Job.
#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)]
#[non_exhaustive]
pub enum BackfillJobPhase {
    /// Runtime was created but execution has not started.
    Created,
    /// Candidate discovery is in progress.
    Discovering,
    /// Candidate hydration and persistence are being admitted and completed.
    Executing,
    /// Cancellation or a fatal result stopped admission while already submitted work is draining.
    Draining,
    /// No more work can be admitted and the Job is terminal.
    Finished,
}

impl crate::BackfillJobPhase {
    /// Returns the stable safe code for this concrete runtime phase.
    #[must_use]
    pub const fn code(self) -> &'static str {
        return match self {
            Self::Created => "created",
            Self::Discovering => "discovering",
            Self::Executing => "executing",
            Self::Draining => "draining",
            Self::Finished => "finished",
        };
    }
}

/// Complete safe latest-value snapshot of one concrete historical RAW transaction Backfill Job.
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct BackfillJobSnapshot {
    phase: crate::BackfillJobPhase,
    scope_kind: crate::BackfillScopeKind,
    discovery_boundary: std::option::Option<crate::BackfillDiscoveryBoundary>,
    candidates_selected: usize,
    candidates_admitted: usize,
    candidates_finished: usize,
    entities_inserted: usize,
    entities_existing: usize,
    entities_purged: usize,
    missing: usize,
    conflicts: usize,
    observations_inserted: usize,
    observations_existing: usize,
    cancelled_candidates: usize,
    holes: usize,
    maximum_in_flight: usize,
    contiguous_completed: usize,
    checkpoint: std::option::Option<crate::BackfillCheckpoint>,
    failure_code: std::option::Option<ksp_core_lib::ErrorCode>,
}

impl crate::BackfillJobSnapshot {
    fn initial(request: &crate::BackfillRequest) -> Self {
        return Self {
            phase: crate::BackfillJobPhase::Created,
            scope_kind: request.scope().kind(),
            discovery_boundary: std::option::Option::None,
            candidates_selected: 0,
            candidates_admitted: 0,
            candidates_finished: 0,
            entities_inserted: 0,
            entities_existing: 0,
            entities_purged: 0,
            missing: 0,
            conflicts: 0,
            observations_inserted: 0,
            observations_existing: 0,
            cancelled_candidates: 0,
            holes: 0,
            maximum_in_flight: 0,
            contiguous_completed: request.checkpoint().map_or(0, crate::BackfillCheckpoint::completed_prefix),
            checkpoint: request.checkpoint().cloned(),
            failure_code: std::option::Option::None,
        };
    }

    /// Returns the concrete execution phase represented by this snapshot.
    #[must_use]
    pub const fn phase(&self) -> crate::BackfillJobPhase {
        return self.phase;
    }

    /// Returns the safe scope category without rendering address or signature payloads.
    #[must_use]
    pub const fn scope_kind(&self) -> crate::BackfillScopeKind {
        return self.scope_kind;
    }

    /// Returns the discovery boundary once candidate selection has completed.
    #[must_use]
    pub const fn discovery_boundary(&self) -> std::option::Option<crate::BackfillDiscoveryBoundary> {
        return self.discovery_boundary;
    }

    /// Returns the bounded candidate count selected by discovery.
    #[must_use]
    pub const fn candidates_selected(&self) -> usize {
        return self.candidates_selected;
    }

    /// Returns the number of candidates admitted into hydration.
    #[must_use]
    pub const fn candidates_admitted(&self) -> usize {
        return self.candidates_admitted;
    }

    /// Returns the number of admitted candidates that reached a known coordinator result.
    #[must_use]
    pub const fn candidates_finished(&self) -> usize {
        return self.candidates_finished;
    }

    /// Returns the number of newly inserted canonical RAW transactions.
    #[must_use]
    pub const fn entities_inserted(&self) -> usize {
        return self.entities_inserted;
    }

    /// Returns the number of identical canonical RAW transactions already present.
    #[must_use]
    pub const fn entities_existing(&self) -> usize {
        return self.entities_existing;
    }

    /// Returns the number of durable purge tombstones respected by normal Backfill persistence.
    #[must_use]
    pub const fn entities_purged(&self) -> usize {
        return self.entities_purged;
    }

    /// Returns the number of `getTransaction = null` candidates.
    #[must_use]
    pub const fn missing(&self) -> usize {
        return self.missing;
    }

    /// Returns the number of Store content conflicts observed by this run.
    #[must_use]
    pub const fn conflicts(&self) -> usize {
        return self.conflicts;
    }

    /// Returns the number of newly inserted acquisition observations.
    #[must_use]
    pub const fn observations_inserted(&self) -> usize {
        return self.observations_inserted;
    }

    /// Returns the number of acquisition observations already durable.
    #[must_use]
    pub const fn observations_existing(&self) -> usize {
        return self.observations_existing;
    }

    /// Returns the number of admitted candidates cancelled before Store submission.
    #[must_use]
    pub const fn cancelled_candidates(&self) -> usize {
        return self.cancelled_candidates;
    }

    /// Returns the number of known candidate outcomes blocking the contiguous frontier.
    #[must_use]
    pub const fn holes(&self) -> usize {
        return self.holes;
    }

    /// Returns the greatest observed concurrent candidate count.
    #[must_use]
    pub const fn maximum_in_flight(&self) -> usize {
        return self.maximum_in_flight;
    }

    /// Returns the cumulative safe contiguous completion prefix represented by the current checkpoint.
    #[must_use]
    pub const fn contiguous_completed(&self) -> usize {
        return self.contiguous_completed;
    }

    /// Returns the latest caller-owned safe checkpoint, when one exists.
    #[must_use]
    pub const fn checkpoint(&self) -> std::option::Option<&crate::BackfillCheckpoint> {
        return self.checkpoint.as_ref();
    }

    /// Returns the stable fatal error code retained by a failed Job, when one exists.
    #[must_use]
    pub const fn failure_code(&self) -> std::option::Option<ksp_core_lib::ErrorCode> {
        return self.failure_code;
    }
}

/// Cloneable runtime-neutral-facing latest-value source for concrete Backfill snapshots.
#[derive(Clone)]
pub struct BackfillSnapshotSource {
    receiver: tokio::sync::watch::Receiver<ksp_job_api::JobNotification<crate::BackfillJobSnapshot>>,
}

impl ksp_job_api::JobSnapshotSource for crate::BackfillSnapshotSource {
    type Snapshot = crate::BackfillJobSnapshot;

    fn current(&self) -> ksp_job_api::JobNotification<Self::Snapshot> {
        return self.receiver.borrow().clone();
    }

    fn wait_for_change(&self, observed: ksp_job_api::JobNotificationSequence) -> ksp_job_api::JobSnapshotFuture<'_, Self::Snapshot> {
        let mut receiver = self.receiver.clone();
        return std::boxed::Box::pin(async move {
            loop {
                let current = receiver.borrow().clone();
                if current.sequence().is_after(observed) || current.state().is_terminal() {
                    return current;
                }
                let changed = receiver.changed().await;
                if changed.is_err() {
                    return receiver.borrow().clone();
                }
            }
        });
    }
}

impl std::fmt::Debug for crate::BackfillSnapshotSource {
    fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
        let current = self.receiver.borrow();
        return formatter.debug_struct("BackfillSnapshotSource").field("sequence", &current.sequence()).field("state", &current.state()).finish();
    }
}

/// Cloneable external control handle for one concrete Backfill runtime.
#[derive(Clone)]
pub struct BackfillJobHandle {
    control: BackfillRuntimeControl,
    snapshots: crate::BackfillSnapshotSource,
}

impl crate::BackfillJobHandle {
    /// Requests cooperative cancellation and returns `true` only when accepted before terminal publication.
    #[must_use]
    pub fn cancel(&self) -> bool {
        return self.control.request_cancellation();
    }

    /// Returns an independent latest-value snapshot source for one listener.
    #[must_use]
    pub fn snapshots(&self) -> crate::BackfillSnapshotSource {
        return self.snapshots.clone();
    }

    /// Returns whether cooperative cancellation has been accepted for this non-terminal Job.
    #[must_use]
    pub fn is_cancellation_requested(&self) -> bool {
        return self.control.is_cancellation_requested();
    }
}

impl std::fmt::Debug for crate::BackfillJobHandle {
    fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
        return formatter
            .debug_struct("BackfillJobHandle")
            .field("cancellation_requested", &self.is_cancellation_requested())
            .field("snapshot", &<crate::BackfillSnapshotSource as ksp_job_api::JobSnapshotSource>::current(&self.snapshots))
            .finish();
    }
}

/// Concrete single-run Backfill coordinator paired with a cloneable control/snapshot handle.
pub struct BackfillJobRuntime {
    request: crate::BackfillRequest,
    control: BackfillRuntimeControl,
    cancellation: crate::BackfillCancellationSignal,
    publisher: crate::BackfillRuntimePublisher,
    handle: crate::BackfillJobHandle,
}

impl crate::BackfillJobRuntime {
    /// Creates one runtime in `Created` state and its stable latest-value channel.
    pub fn new(request: crate::BackfillRequest) -> ksp_core_lib::Result<Self> {
        let kind = match ksp_job_api::JobKindCode::new(crate::BACKFILL_JOB_KIND_CODE) {
            std::result::Result::Ok(value) => value,
            std::result::Result::Err(error) => return std::result::Result::Err(error),
        };
        let initial_snapshot = crate::BackfillJobSnapshot::initial(&request);
        let initial = ksp_job_api::JobNotification::new(
            request.job_id().clone(),
            kind.clone(),
            ksp_job_api::JobNotificationSequence::initial(),
            ksp_job_api::JobState::Created,
            initial_snapshot,
        );
        let (snapshot_sender, snapshot_receiver) = tokio::sync::watch::channel(initial);
        let (cancel_sender, cancel_receiver) = tokio::sync::watch::channel(false);
        let control = BackfillRuntimeControl::new(cancel_sender);
        let cancellation = crate::BackfillCancellationSignal::new(control.token(), cancel_receiver);
        let snapshots = crate::BackfillSnapshotSource { receiver: snapshot_receiver };
        let handle = crate::BackfillJobHandle { control: control.clone(), snapshots: snapshots.clone() };
        let publisher = crate::BackfillRuntimePublisher { sender: snapshot_sender, id: request.job_id().clone(), kind };
        return std::result::Result::Ok(Self { request, control, cancellation, publisher, handle });
    }

    /// Returns a cloneable control and latest-value observation handle before the runtime is moved into execution.
    #[must_use]
    pub fn handle(&self) -> crate::BackfillJobHandle {
        return self.handle.clone();
    }

    /// Runs discovery, bounded execution, cooperative cancellation and terminal snapshot publication.
    pub async fn run(
        self,
        transport: &ksp_onchain_transport_lib::HttpTransportPool,
        store: &ksp_store_lib::Store,
    ) -> ksp_core_lib::Result<crate::BackfillJobSnapshot> {
        if self.cancellation.is_requested() {
            let claimed = self.control.claim_normal_terminal();
            if claimed != TerminalClaim::Cancelled {
                return std::result::Result::Err(runtime_error("terminal.pre_start"));
            }
            return self.publisher.publish_cancelled_from_created();
        }
        let started = self.publisher.publish_running(crate::BackfillJobPhase::Discovering);
        if let std::result::Result::Err(error) = started {
            self.control.claim_failed();
            return std::result::Result::Err(error);
        }
        let discovery = crate::discover_backfill_candidates_cancellable(transport, &self.request, &self.cancellation).await;
        let discovery = match discovery {
            std::result::Result::Ok(value) => value,
            std::result::Result::Err(error) => {
                if error.code() == crate::ERROR_CODE_BACKFILL_CANCELLED {
                    return self.finish_cancelled();
                }
                self.control.claim_failed();
                let published = self.publisher.publish_failed(error.code());
                if let std::result::Result::Err(publish_error) = published {
                    return std::result::Result::Err(publish_error);
                }
                return std::result::Result::Err(error);
            },
        };
        let published = self.publisher.publish_discovery(&discovery);
        if let std::result::Result::Err(error) = published {
            self.control.claim_failed();
            return std::result::Result::Err(error);
        }
        let batch = crate::execute_backfill_discovery_cancellable(transport, store, &self.request, &discovery, &self.cancellation, &self.publisher).await;
        let batch = match batch {
            std::result::Result::Ok(value) => value,
            std::result::Result::Err(error) => {
                self.control.claim_failed();
                let published = self.publisher.publish_failed(error.code());
                if let std::result::Result::Err(publish_error) = published {
                    return std::result::Result::Err(publish_error);
                }
                return std::result::Result::Err(error);
            },
        };
        if let std::option::Option::Some(code) = batch.failure_code() {
            self.control.claim_failed();
            let published = self.publisher.publish_batch_terminal(&batch, ksp_job_api::JobState::Failed, std::option::Option::Some(code));
            if let std::result::Result::Err(error) = published {
                return std::result::Result::Err(error);
            }
            return std::result::Result::Err(ksp_core_lib::Error::new(code, "Backfill execution reached a fatal candidate result"));
        }
        let terminal = self.control.claim_normal_terminal();
        return match terminal {
            TerminalClaim::Cancelled => {
                let cancelling = self.publisher.publish_batch_cancelling(&batch);
                if let std::result::Result::Err(error) = cancelling {
                    return std::result::Result::Err(error);
                }
                self.publisher.publish_batch_terminal(&batch, ksp_job_api::JobState::Cancelled, std::option::Option::None)
            },
            TerminalClaim::Completed => {
                let completion = if batch.is_partial() { ksp_job_api::JobCompletion::Partial } else { ksp_job_api::JobCompletion::Complete };
                self.publisher.publish_batch_terminal(&batch, ksp_job_api::JobState::Completed(completion), std::option::Option::None)
            },
            TerminalClaim::Failed => std::result::Result::Err(runtime_error("terminal.failed")),
        };
    }

    fn finish_cancelled(&self) -> ksp_core_lib::Result<crate::BackfillJobSnapshot> {
        let terminal = self.control.claim_normal_terminal();
        if terminal != TerminalClaim::Cancelled {
            return std::result::Result::Err(runtime_error("terminal.cancelled"));
        }
        let cancelling = self.publisher.publish_cancelling();
        if let std::result::Result::Err(error) = cancelling {
            return std::result::Result::Err(error);
        }
        return self.publisher.publish_terminal(ksp_job_api::JobState::Cancelled, std::option::Option::None);
    }
}

/// Internal atomic terminal/cancellation arbitration shared by runtime and external handle.
#[derive(Clone)]
struct BackfillRuntimeControl {
    state: std::sync::Arc<std::sync::atomic::AtomicU8>,
    token: ksp_job_api::JobCancellationToken,
    cancel_sender: tokio::sync::watch::Sender<bool>,
}

impl BackfillRuntimeControl {
    /// Creates one active control state paired with the cancellation wake channel.
    fn new(cancel_sender: tokio::sync::watch::Sender<bool>) -> Self {
        return Self {
            state: std::sync::Arc::new(std::sync::atomic::AtomicU8::new(CONTROL_ACTIVE)),
            token: ksp_job_api::JobCancellationToken::new(),
            cancel_sender,
        };
    }

    /// Returns the runtime-neutral cancellation token mirrored by this control.
    fn token(&self) -> ksp_job_api::JobCancellationToken {
        return self.token.clone();
    }

    /// Atomically accepts the first pre-terminal cancellation request.
    fn request_cancellation(&self) -> bool {
        let accepted = self
            .state
            .compare_exchange(CONTROL_ACTIVE, CONTROL_CANCELLATION_REQUESTED, std::sync::atomic::Ordering::AcqRel, std::sync::atomic::Ordering::Acquire)
            .is_ok();
        if accepted && self.token.cancel() {
            let _previous = self.cancel_sender.send_replace(true);
        }
        return accepted;
    }

    /// Returns whether cooperative cancellation was accepted.
    fn is_cancellation_requested(&self) -> bool {
        return self.token.is_cancellation_requested();
    }

    /// Atomically resolves the completion-versus-cancellation terminal race.
    fn claim_normal_terminal(&self) -> TerminalClaim {
        let completed =
            self.state.compare_exchange(CONTROL_ACTIVE, CONTROL_COMPLETED, std::sync::atomic::Ordering::AcqRel, std::sync::atomic::Ordering::Acquire);
        if completed.is_ok() {
            return TerminalClaim::Completed;
        }
        let cancelled = self.state.compare_exchange(
            CONTROL_CANCELLATION_REQUESTED,
            CONTROL_CANCELLED,
            std::sync::atomic::Ordering::AcqRel,
            std::sync::atomic::Ordering::Acquire,
        );
        if cancelled.is_ok() {
            return TerminalClaim::Cancelled;
        }
        return TerminalClaim::Failed;
    }

    /// Marks a non-terminal control as failed, overriding a pending cancellation request.
    fn claim_failed(&self) {
        loop {
            let state = self.state.load(std::sync::atomic::Ordering::Acquire);
            if matches!(state, CONTROL_COMPLETED | CONTROL_CANCELLED | CONTROL_FAILED) {
                return;
            }
            let changed = self.state.compare_exchange(state, CONTROL_FAILED, std::sync::atomic::Ordering::AcqRel, std::sync::atomic::Ordering::Acquire);
            if changed.is_ok() {
                return;
            }
        }
    }
}

/// Internal wakeable cancellation signal used only around abandonable pre-Store futures.
#[derive(Clone)]
pub(crate) struct BackfillCancellationSignal {
    token: ksp_job_api::JobCancellationToken,
    receiver: tokio::sync::watch::Receiver<bool>,
}

impl crate::BackfillCancellationSignal {
    /// Creates one signal from the runtime-neutral token and Tokio wake receiver.
    pub(crate) fn new(token: ksp_job_api::JobCancellationToken, receiver: tokio::sync::watch::Receiver<bool>) -> Self {
        return Self { token, receiver };
    }

    /// Returns whether cancellation has been requested.
    pub(crate) fn is_requested(&self) -> bool {
        return self.token.is_cancellation_requested();
    }

    /// Runs one abandonable operation until it completes or cancellation wins.
    pub(crate) async fn run_cancellable<F, T>(&self, operation: F) -> ksp_core_lib::Result<T>
    where
        F: std::future::Future<Output = ksp_core_lib::Result<T>>,
    {
        if self.is_requested() {
            return std::result::Result::Err(cancelled_error());
        }
        let mut receiver = self.receiver.clone();
        return tokio::select! {
            biased;
            _ = wait_for_cancellation(&self.token, &mut receiver) => std::result::Result::Err(cancelled_error()),
            result = operation => result,
        };
    }
}

async fn wait_for_cancellation(token: &ksp_job_api::JobCancellationToken, receiver: &mut tokio::sync::watch::Receiver<bool>) {
    loop {
        if token.is_cancellation_requested() || *receiver.borrow() {
            return;
        }
        let changed = receiver.changed().await;
        if changed.is_err() {
            return;
        }
    }
}

/// Internal result of atomically claiming a normal terminal state.
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
enum TerminalClaim {
    Completed,
    Cancelled,
    Failed,
}

/// Internal latest-value publisher owning the concrete Backfill notification stream.
#[derive(Clone)]
pub(crate) struct BackfillRuntimePublisher {
    sender: tokio::sync::watch::Sender<ksp_job_api::JobNotification<crate::BackfillJobSnapshot>>,
    id: ksp_job_api::JobId,
    kind: ksp_job_api::JobKindCode,
}

impl crate::BackfillRuntimePublisher {
    /// Publishes one non-terminal running phase.
    pub(crate) fn publish_running(&self, phase: crate::BackfillJobPhase) -> ksp_core_lib::Result<crate::BackfillJobSnapshot> {
        return self.publish_with(ksp_job_api::JobState::Running, |snapshot| {
            snapshot.phase = phase;
        });
    }

    /// Publishes the complete bounded discovery result as the current execution snapshot.
    pub(crate) fn publish_discovery(&self, discovery: &crate::BackfillDiscovery) -> ksp_core_lib::Result<crate::BackfillJobSnapshot> {
        return self.publish_with(ksp_job_api::JobState::Running, |snapshot| {
            snapshot.phase = crate::BackfillJobPhase::Executing;
            snapshot.discovery_boundary = std::option::Option::Some(discovery.boundary());
            snapshot.candidates_selected = discovery.candidates().len();
        });
    }

    /// Publishes one coalescable execution progress value.
    pub(crate) fn publish_execution_progress(
        &self,
        progress: &crate::BackfillExecutionProgress,
        cancelling: bool,
        draining: bool,
    ) -> ksp_core_lib::Result<crate::BackfillJobSnapshot> {
        let state = if cancelling { ksp_job_api::JobState::Cancelling } else { ksp_job_api::JobState::Running };
        return self.publish_with(state, |snapshot| {
            snapshot.phase = if draining { crate::BackfillJobPhase::Draining } else { crate::BackfillJobPhase::Executing };
            apply_progress(snapshot, progress);
        });
    }

    /// Publishes cancellation observation before terminal cancellation.
    pub(crate) fn publish_cancelling(&self) -> ksp_core_lib::Result<crate::BackfillJobSnapshot> {
        return self.publish_with(ksp_job_api::JobState::Cancelling, |snapshot| {
            snapshot.phase = crate::BackfillJobPhase::Draining;
        });
    }

    /// Publishes the drained batch state while cancellation is terminalizing.
    pub(crate) fn publish_batch_cancelling(&self, batch: &crate::BackfillExecutionBatch) -> ksp_core_lib::Result<crate::BackfillJobSnapshot> {
        return self.publish_with(ksp_job_api::JobState::Cancelling, |snapshot| {
            snapshot.phase = crate::BackfillJobPhase::Draining;
            apply_batch(snapshot, batch);
        });
    }

    /// Publishes one terminal state together with the fully drained execution batch.
    pub(crate) fn publish_batch_terminal(
        &self,
        batch: &crate::BackfillExecutionBatch,
        state: ksp_job_api::JobState,
        failure_code: std::option::Option<ksp_core_lib::ErrorCode>,
    ) -> ksp_core_lib::Result<crate::BackfillJobSnapshot> {
        return self.publish_with(state, |snapshot| {
            snapshot.phase = crate::BackfillJobPhase::Finished;
            snapshot.failure_code = failure_code;
            apply_batch(snapshot, batch);
        });
    }

    /// Publishes a terminal failure before a batch exists.
    pub(crate) fn publish_failed(&self, code: ksp_core_lib::ErrorCode) -> ksp_core_lib::Result<crate::BackfillJobSnapshot> {
        return self.publish_terminal(ksp_job_api::JobState::Failed, std::option::Option::Some(code));
    }

    /// Publishes a terminal state without a completed execution batch.
    pub(crate) fn publish_terminal(
        &self,
        state: ksp_job_api::JobState,
        failure_code: std::option::Option<ksp_core_lib::ErrorCode>,
    ) -> ksp_core_lib::Result<crate::BackfillJobSnapshot> {
        return self.publish_with(state, |snapshot| {
            snapshot.phase = crate::BackfillJobPhase::Finished;
            snapshot.failure_code = failure_code;
        });
    }

    /// Publishes direct Created-to-Cancelled termination before execution starts.
    pub(crate) fn publish_cancelled_from_created(&self) -> ksp_core_lib::Result<crate::BackfillJobSnapshot> {
        return self.publish_with(ksp_job_api::JobState::Cancelled, |snapshot| {
            snapshot.phase = crate::BackfillJobPhase::Finished;
        });
    }

    fn publish_with<F>(&self, state: ksp_job_api::JobState, update: F) -> ksp_core_lib::Result<crate::BackfillJobSnapshot>
    where
        F: FnOnce(&mut crate::BackfillJobSnapshot),
    {
        let current = self.sender.borrow().clone();
        if current.state().is_terminal() {
            return std::result::Result::Err(runtime_error("notification.terminal"));
        }
        if !valid_snapshot_transition(current.state(), state) {
            return std::result::Result::Err(runtime_error("notification.transition"));
        }
        let sequence = match current.sequence().next() {
            std::result::Result::Ok(value) => value,
            std::result::Result::Err(error) => return std::result::Result::Err(error),
        };
        let mut snapshot = current.snapshot().clone();
        update(&mut snapshot);
        let notification = ksp_job_api::JobNotification::new(self.id.clone(), self.kind.clone(), sequence, state, snapshot.clone());
        let _previous = self.sender.send_replace(notification);
        return std::result::Result::Ok(snapshot);
    }
}

fn valid_snapshot_transition(source: ksp_job_api::JobState, target: ksp_job_api::JobState) -> bool {
    if source.is_terminal() {
        return false;
    }
    return matches!(
        (source, target),
        (ksp_job_api::JobState::Created, ksp_job_api::JobState::Running)
            | (ksp_job_api::JobState::Created, ksp_job_api::JobState::Cancelled)
            | (ksp_job_api::JobState::Running, ksp_job_api::JobState::Running)
            | (ksp_job_api::JobState::Running, ksp_job_api::JobState::Cancelling)
            | (ksp_job_api::JobState::Running, ksp_job_api::JobState::Completed(_))
            | (ksp_job_api::JobState::Running, ksp_job_api::JobState::Failed)
            | (ksp_job_api::JobState::Cancelling, ksp_job_api::JobState::Cancelling)
            | (ksp_job_api::JobState::Cancelling, ksp_job_api::JobState::Cancelled)
            | (ksp_job_api::JobState::Cancelling, ksp_job_api::JobState::Failed)
    );
}

fn apply_progress(snapshot: &mut crate::BackfillJobSnapshot, progress: &crate::BackfillExecutionProgress) {
    snapshot.candidates_admitted = progress.admitted_count();
    snapshot.candidates_finished = progress.finished_count();
    snapshot.entities_inserted = progress.inserted_count();
    snapshot.entities_existing = progress.already_present_count();
    snapshot.entities_purged = progress.purged_count();
    snapshot.missing = progress.missing_count();
    snapshot.conflicts = progress.conflict_count();
    snapshot.observations_inserted = progress.observation_inserted_count();
    snapshot.observations_existing = progress.observation_already_present_count();
    snapshot.cancelled_candidates = progress.cancelled_count();
    snapshot.holes = progress.hole_count();
    snapshot.maximum_in_flight = progress.maximum_in_flight();
    snapshot.contiguous_completed = progress.contiguous_completed();
    snapshot.checkpoint = std::option::Option::Some(progress.checkpoint().clone());
}

fn apply_batch(snapshot: &mut crate::BackfillJobSnapshot, batch: &crate::BackfillExecutionBatch) {
    snapshot.candidates_admitted = batch.admitted_count();
    snapshot.candidates_finished = batch.finished_count();
    snapshot.entities_inserted = batch.inserted_count();
    snapshot.entities_existing = batch.already_present_count();
    snapshot.entities_purged = batch.purged_count();
    snapshot.missing = batch.missing_count();
    snapshot.conflicts = batch.conflict_count();
    snapshot.observations_inserted = batch.observation_inserted_count();
    snapshot.observations_existing = batch.observation_already_present_count();
    snapshot.cancelled_candidates = batch.cancelled_count();
    snapshot.holes = batch.hole_count();
    snapshot.maximum_in_flight = batch.maximum_in_flight();
    snapshot.contiguous_completed = batch.checkpoint().completed_prefix();
    snapshot.checkpoint = std::option::Option::Some(batch.checkpoint().clone());
}

fn cancelled_error() -> ksp_core_lib::Error {
    return ksp_core_lib::Error::new(crate::ERROR_CODE_BACKFILL_CANCELLED, "Backfill operation cancelled before durable Store submission");
}

fn runtime_error(field: &'static str) -> ksp_core_lib::Error {
    return ksp_core_lib::Error::new(crate::ERROR_CODE_BACKFILL_RUNTIME_INVALID, "invalid concrete Backfill runtime state").with_context("field", field);
}

#[cfg(test)]
#[path = "../unit_tests/runtime.rs"]
mod tests;