v0.3.6-pre.008

This commit is contained in:
2026-09-01 15:14:57 +02:00
parent 138395ac35
commit 75cbf9df09
18 changed files with 1483 additions and 104 deletions

View File

@@ -1,12 +1,12 @@
# file: Cargo.toml # file: Cargo.toml
# version: 405 # version: 406
[workspace] [workspace]
resolver = "3" resolver = "3"
members = ["crates/ksp-app-config-desk", "crates/ksp-app-solprices-desk", "crates/ksp-app-wallet-desk", "crates/ksp-config-lib", "crates/ksp-core-lib", "crates/ksp-interface-lib", "crates/ksp-job-api", "crates/ksp-job-backfill-lib", "crates/ksp-logging-lib", "crates/ksp-offchain-transport-lib", "crates/ksp-onchain-transport-lib", "crates/ksp-program-api", "crates/ksp-store-api", "crates/ksp-store-lib", "crates/ksp-store-postgres-lib", "crates/ksp-wallet-lib"] members = ["crates/ksp-app-config-desk", "crates/ksp-app-solprices-desk", "crates/ksp-app-wallet-desk", "crates/ksp-config-lib", "crates/ksp-core-lib", "crates/ksp-interface-lib", "crates/ksp-job-api", "crates/ksp-job-backfill-lib", "crates/ksp-logging-lib", "crates/ksp-offchain-transport-lib", "crates/ksp-onchain-transport-lib", "crates/ksp-program-api", "crates/ksp-store-api", "crates/ksp-store-lib", "crates/ksp-store-postgres-lib", "crates/ksp-wallet-lib"]
[workspace.package] [workspace.package]
version = "0.3.6-pre.7" version = "0.3.6-pre.8"
edition = "2024" edition = "2024"
license = "MIT" license = "MIT"
repository = "https://git.sasedev.com/Sasedev/khadhroony-solana-project" repository = "https://git.sasedev.com/Sasedev/khadhroony-solana-project"

View File

@@ -1,5 +1,5 @@
# file: crates/ksp-job-backfill-lib/Cargo.toml # file: crates/ksp-job-backfill-lib/Cargo.toml
# version: 2 # version: 3
[package] [package]
name = "ksp-job-backfill-lib" name = "ksp-job-backfill-lib"
@@ -8,6 +8,7 @@ edition.workspace = true
repository.workspace = true repository.workspace = true
[dependencies] [dependencies]
futures-util = { workspace = true, features = ["std"] }
ksp-core-lib = { path = "../ksp-core-lib" } ksp-core-lib = { path = "../ksp-core-lib" }
ksp-job-api = { path = "../ksp-job-api" } ksp-job-api = { path = "../ksp-job-api" }
ksp-logging-lib = { path = "../ksp-logging-lib" } ksp-logging-lib = { path = "../ksp-logging-lib" }

View File

@@ -0,0 +1,218 @@
// file: crates/ksp-job-backfill-lib/src/checkpoint.rs
// version: 1
/// Opaque caller-owned checkpoint for one controlled Backfill resumption.
///
/// The checkpoint carries no payload, endpoint, provider, URL or secret. Persistence of this
/// value is deliberately external to Store in v0.3.6; the Backfill library only validates and
/// consumes checkpoints supplied back by its caller.
#[derive(Clone, Eq, PartialEq)]
pub struct BackfillCheckpoint {
job_id: ksp_job_api::JobId,
scope_fingerprint: crate::BackfillScopeFingerprint,
completed_prefix: usize,
resume_before: std::option::Option<crate::BackfillSignature>,
}
impl BackfillCheckpoint {
/// Returns the logical Job identity that owns this checkpoint.
#[must_use]
pub const fn job_id(&self) -> &ksp_job_api::JobId {
return &self.job_id;
}
/// Returns the semantic scope fingerprint bound to this checkpoint.
#[must_use]
pub const fn scope_fingerprint(&self) -> crate::BackfillScopeFingerprint {
return self.scope_fingerprint;
}
/// Returns the number of candidates proven durable in one contiguous prefix.
#[must_use]
pub const fn completed_prefix(&self) -> usize {
return self.completed_prefix;
}
/// Creates one internally proven checkpoint.
pub(crate) fn new(
job_id: ksp_job_api::JobId,
scope_fingerprint: crate::BackfillScopeFingerprint,
completed_prefix: usize,
resume_before: std::option::Option<crate::BackfillSignature>,
) -> Self {
return Self { job_id, scope_fingerprint, completed_prefix, resume_before };
}
/// Returns the internal exclusive `before` cursor used only by controlled Before resumption.
pub(crate) const fn resume_before(&self) -> std::option::Option<&crate::BackfillSignature> {
return self.resume_before.as_ref();
}
}
impl std::fmt::Debug for BackfillCheckpoint {
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
return formatter
.debug_struct("BackfillCheckpoint")
.field("job_id", &self.job_id)
.field("scope_fingerprint", &self.scope_fingerprint)
.field("completed_prefix", &self.completed_prefix)
.field("has_resume_before", &self.resume_before.is_some())
.finish();
}
}
/// Private bounded bitmap tracking durable completions and their contiguous prefix.
pub(crate) struct CompletionFrontier {
durable: std::vec::Vec<bool>,
contiguous_completed: usize,
}
impl CompletionFrontier {
/// Creates one empty frontier for the exact bounded candidate count.
pub(crate) fn new(candidate_count: usize) -> Self {
return Self { durable: vec![false; candidate_count], contiguous_completed: 0 };
}
/// Seeds a previously proven replay prefix before processing the remaining candidates.
pub(crate) fn seed_prefix(&mut self, completed_prefix: usize) -> ksp_core_lib::Result<()> {
if completed_prefix > self.durable.len() {
return std::result::Result::Err(checkpoint_error("checkpoint.completed_prefix"));
}
for index in 0..completed_prefix {
self.durable[index] = true;
}
self.contiguous_completed = completed_prefix;
return std::result::Result::Ok(());
}
/// Marks one candidate durable and advances only through the now-contiguous prefix.
pub(crate) fn mark_durable(&mut self, index: usize) -> ksp_core_lib::Result<()> {
let slot = match self.durable.get_mut(index) {
std::option::Option::Some(slot) => slot,
std::option::Option::None => return std::result::Result::Err(checkpoint_error("frontier.index")),
};
*slot = true;
while self.contiguous_completed < self.durable.len() && self.durable[self.contiguous_completed] {
self.contiguous_completed += 1;
}
return std::result::Result::Ok(());
}
/// Returns the number of durable candidates in the contiguous prefix.
pub(crate) const fn contiguous_completed(&self) -> usize {
return self.contiguous_completed;
}
}
/// Validates one checkpoint against the Job, semantic scope and scope-specific cursor shape.
pub(crate) fn validate_request_checkpoint(
request_job_id: &ksp_job_api::JobId,
scope_fingerprint: crate::BackfillScopeFingerprint,
scope_kind: crate::BackfillScopeKind,
checkpoint: &BackfillCheckpoint,
) -> ksp_core_lib::Result<()> {
if checkpoint.job_id() != request_job_id {
return std::result::Result::Err(checkpoint_error("checkpoint.job_id"));
}
if checkpoint.scope_fingerprint() != scope_fingerprint {
return std::result::Result::Err(checkpoint_error("checkpoint.scope_fingerprint"));
}
if scope_kind != crate::BackfillScopeKind::BeforeAddress && checkpoint.resume_before().is_some() {
return std::result::Result::Err(checkpoint_error("checkpoint.resume_before"));
}
return std::result::Result::Ok(());
}
/// Resolves the exclusive Before cursor from a validated checkpoint or the original scope anchor.
pub(crate) fn resume_before_cursor(request: &crate::BackfillRequest) -> std::option::Option<std::string::String> {
if request.scope().kind() != crate::BackfillScopeKind::BeforeAddress {
return std::option::Option::None;
}
if let std::option::Option::Some(checkpoint) = request.checkpoint()
&& let std::option::Option::Some(cursor) = checkpoint.resume_before()
{
return std::option::Option::Some(cursor.as_str().to_owned());
}
return request.scope().anchor().map(|anchor| return anchor.as_str().to_owned());
}
/// Resolves the replay prefix skipped only by After and Explicit execution.
pub(crate) fn execution_resume_offset(request: &crate::BackfillRequest, discovery: &crate::BackfillDiscovery) -> ksp_core_lib::Result<usize> {
let checkpoint = match request.checkpoint() {
std::option::Option::Some(checkpoint) => checkpoint,
std::option::Option::None => return std::result::Result::Ok(0),
};
return match request.scope().kind() {
crate::BackfillScopeKind::AfterAddress | crate::BackfillScopeKind::ExplicitSignatures => {
if checkpoint.completed_prefix() > discovery.candidates().len() {
return std::result::Result::Err(checkpoint_error("checkpoint.completed_prefix"));
}
std::result::Result::Ok(checkpoint.completed_prefix())
},
crate::BackfillScopeKind::LatestAddress | crate::BackfillScopeKind::BeforeAddress => std::result::Result::Ok(0),
};
}
/// Builds the safe next checkpoint from one drained contiguous completion frontier.
pub(crate) fn checkpoint_from_frontier(
request: &crate::BackfillRequest,
discovery: &crate::BackfillDiscovery,
frontier: &CompletionFrontier,
) -> ksp_core_lib::Result<BackfillCheckpoint> {
let validation = validate_discovery_identity(request, discovery);
if let std::result::Result::Err(error) = validation {
return std::result::Result::Err(error);
}
let local_prefix = frontier.contiguous_completed();
if request.scope().kind() == crate::BackfillScopeKind::AfterAddress && discovery.boundary() == crate::BackfillDiscoveryBoundary::AfterAnchorNotReached {
if let std::option::Option::Some(checkpoint) = request.checkpoint() {
return std::result::Result::Ok(checkpoint.clone());
}
return std::result::Result::Ok(BackfillCheckpoint::new(request.job_id().clone(), request.scope_fingerprint(), 0, std::option::Option::None));
}
let (completed_prefix, resume_before) = match request.scope().kind() {
crate::BackfillScopeKind::LatestAddress => (local_prefix, std::option::Option::None),
crate::BackfillScopeKind::BeforeAddress => {
let previous = request.checkpoint().map_or(0, BackfillCheckpoint::completed_prefix);
let completed_prefix = match previous.checked_add(local_prefix) {
std::option::Option::Some(value) => value,
std::option::Option::None => return std::result::Result::Err(checkpoint_error("checkpoint.completed_prefix")),
};
let resume_before = if local_prefix == 0 {
match request.checkpoint() {
std::option::Option::Some(checkpoint) => checkpoint.resume_before().cloned(),
std::option::Option::None => request.scope().anchor().cloned(),
}
} else {
let index = local_prefix - 1;
let candidate = match discovery.candidates().get(index) {
std::option::Option::Some(candidate) => candidate,
std::option::Option::None => return std::result::Result::Err(checkpoint_error("frontier.index")),
};
std::option::Option::Some(candidate.identity().signature().clone())
};
(completed_prefix, resume_before)
},
crate::BackfillScopeKind::AfterAddress | crate::BackfillScopeKind::ExplicitSignatures => (local_prefix, std::option::Option::None),
};
return std::result::Result::Ok(BackfillCheckpoint::new(request.job_id().clone(), request.scope_fingerprint(), completed_prefix, resume_before));
}
/// Validates that one discovery result belongs to the request network and semantic scope.
pub(crate) fn validate_discovery_identity(request: &crate::BackfillRequest, discovery: &crate::BackfillDiscovery) -> ksp_core_lib::Result<()> {
if discovery.network() != request.network() {
return std::result::Result::Err(checkpoint_error("discovery.network"));
}
if discovery.scope_fingerprint() != request.scope_fingerprint() {
return std::result::Result::Err(checkpoint_error("discovery.scope_fingerprint"));
}
return std::result::Result::Ok(());
}
fn checkpoint_error(field: &'static str) -> ksp_core_lib::Error {
return ksp_core_lib::Error::new(crate::ERROR_CODE_BACKFILL_CHECKPOINT_INVALID, "invalid Backfill checkpoint/frontier state").with_context("field", field);
}
#[cfg(test)]
#[path = "../unit_tests/checkpoint.rs"]
mod tests;

View File

@@ -1,5 +1,5 @@
// file: crates/ksp-job-backfill-lib/src/discovery.rs // file: crates/ksp-job-backfill-lib/src/discovery.rs
// version: 2 // version: 3
/// Network-scoped identity of one discovered transaction candidate before canonical signature decoding. /// Network-scoped identity of one discovered transaction candidate before canonical signature decoding.
#[derive(Clone, Debug, Eq, Hash, PartialEq)] #[derive(Clone, Debug, Eq, Hash, PartialEq)]
@@ -100,6 +100,17 @@ pub struct BackfillDiscovery {
} }
impl BackfillDiscovery { impl BackfillDiscovery {
/// Creates one internally validated bounded discovery value.
pub(crate) fn new(
network: ksp_store_lib::RawNetworkId,
scope_fingerprint: crate::BackfillScopeFingerprint,
candidates: std::vec::Vec<BackfillCandidate>,
pages_fetched: usize,
boundary: BackfillDiscoveryBoundary,
) -> Self {
return Self { network, scope_fingerprint, candidates, pages_fetched, boundary };
}
/// Returns the logical network shared by every candidate identity. /// Returns the logical network shared by every candidate identity.
#[must_use] #[must_use]
pub const fn network(&self) -> &ksp_store_lib::RawNetworkId { pub const fn network(&self) -> &ksp_store_lib::RawNetworkId {
@@ -244,13 +255,13 @@ fn discover_explicit(request: &crate::BackfillRequest) -> ksp_core_lib::Result<B
let identity = BackfillCandidateIdentity::new(request.network().clone(), signature.clone()); let identity = BackfillCandidateIdentity::new(request.network().clone(), signature.clone());
candidates.push(BackfillCandidate::new(identity, std::option::Option::None)); candidates.push(BackfillCandidate::new(identity, std::option::Option::None));
} }
return std::result::Result::Ok(BackfillDiscovery { return std::result::Result::Ok(BackfillDiscovery::new(
network: request.network().clone(), request.network().clone(),
scope_fingerprint: request.scope_fingerprint(), request.scope_fingerprint(),
candidates, candidates,
pages_fetched: 0, 0,
boundary: BackfillDiscoveryBoundary::ExplicitInput, BackfillDiscoveryBoundary::ExplicitInput,
}); ));
} }
async fn discover_older<S>(source: &S, request: &crate::BackfillRequest) -> ksp_core_lib::Result<BackfillDiscovery> async fn discover_older<S>(source: &S, request: &crate::BackfillRequest) -> ksp_core_lib::Result<BackfillDiscovery>
@@ -262,7 +273,7 @@ where
std::option::Option::None => return std::result::Result::Err(discovery_invalid("scope.address")), std::option::Option::None => return std::result::Result::Err(discovery_invalid("scope.address")),
}; };
let mut before = match request.scope().kind() { let mut before = match request.scope().kind() {
crate::BackfillScopeKind::BeforeAddress => request.scope().anchor().map(|value| return value.as_str().to_owned()), crate::BackfillScopeKind::BeforeAddress => crate::resume_before_cursor(request),
crate::BackfillScopeKind::LatestAddress => std::option::Option::None, crate::BackfillScopeKind::LatestAddress => std::option::Option::None,
crate::BackfillScopeKind::AfterAddress | crate::BackfillScopeKind::ExplicitSignatures => { crate::BackfillScopeKind::AfterAddress | crate::BackfillScopeKind::ExplicitSignatures => {
return std::result::Result::Err(discovery_invalid("scope.kind")); return std::result::Result::Err(discovery_invalid("scope.kind"));
@@ -333,13 +344,7 @@ where
} }
before = std::option::Option::Some(next_before.as_str().to_owned()); before = std::option::Option::Some(next_before.as_str().to_owned());
}; };
return std::result::Result::Ok(BackfillDiscovery { return std::result::Result::Ok(BackfillDiscovery::new(request.network().clone(), request.scope_fingerprint(), candidates, pages_fetched, boundary));
network: request.network().clone(),
scope_fingerprint: request.scope_fingerprint(),
candidates,
pages_fetched,
boundary,
});
} }
async fn discover_after<S>(source: &S, request: &crate::BackfillRequest) -> ksp_core_lib::Result<BackfillDiscovery> async fn discover_after<S>(source: &S, request: &crate::BackfillRequest) -> ksp_core_lib::Result<BackfillDiscovery>
@@ -416,13 +421,13 @@ where
} }
before = std::option::Option::Some(next_before.as_str().to_owned()); before = std::option::Option::Some(next_before.as_str().to_owned());
}; };
return std::result::Result::Ok(BackfillDiscovery { return std::result::Result::Ok(BackfillDiscovery::new(
network: request.network().clone(), request.network().clone(),
scope_fingerprint: request.scope_fingerprint(), request.scope_fingerprint(),
candidates: nearest.into_iter().collect(), nearest.into_iter().collect(),
pages_fetched, pages_fetched,
boundary, boundary,
}); ));
} }
fn validated_signature(value: &str) -> ksp_core_lib::Result<crate::BackfillSignature> { fn validated_signature(value: &str) -> ksp_core_lib::Result<crate::BackfillSignature> {

View File

@@ -1,10 +1,14 @@
// file: crates/ksp-job-backfill-lib/src/error.rs // file: crates/ksp-job-backfill-lib/src/error.rs
// version: 3 // version: 4
/// Error code used when one Backfill checkpoint/frontier is incompatible with the current Job or semantic scope.
pub const ERROR_CODE_BACKFILL_CHECKPOINT_INVALID: ksp_core_lib::ErrorCode = ksp_core_lib::ErrorCode::new("job_backfill", "checkpoint_invalid");
/// Error code used when a signature page violates a bounded discovery invariant. /// Error code used when a signature page violates a bounded discovery invariant.
pub const ERROR_CODE_BACKFILL_DISCOVERY_INVALID: ksp_core_lib::ErrorCode = ksp_core_lib::ErrorCode::new("job_backfill", "discovery_invalid"); pub const ERROR_CODE_BACKFILL_DISCOVERY_INVALID: ksp_core_lib::ErrorCode = ksp_core_lib::ErrorCode::new("job_backfill", "discovery_invalid");
/// Error code used when paginated discovery cannot advance its exclusive RPC cursor safely. /// Error code used when paginated discovery cannot advance its exclusive RPC cursor safely.
pub const ERROR_CODE_BACKFILL_DISCOVERY_STALLED: ksp_core_lib::ErrorCode = ksp_core_lib::ErrorCode::new("job_backfill", "discovery_stalled"); pub const ERROR_CODE_BACKFILL_DISCOVERY_STALLED: ksp_core_lib::ErrorCode = ksp_core_lib::ErrorCode::new("job_backfill", "discovery_stalled");
/// Error code used when bounded concurrent candidate execution violates its internal admission or clock invariants.
pub const ERROR_CODE_BACKFILL_EXECUTION_INVALID: ksp_core_lib::ErrorCode = ksp_core_lib::ErrorCode::new("job_backfill", "execution_invalid");
/// Error code used when Store persistence returns an impossible Backfill state or targets a different network. /// Error code used when Store persistence returns an impossible Backfill state or targets a different network.
pub const ERROR_CODE_BACKFILL_PERSISTENCE_INVALID: ksp_core_lib::ErrorCode = ksp_core_lib::ErrorCode::new("job_backfill", "persistence_invalid"); pub const ERROR_CODE_BACKFILL_PERSISTENCE_INVALID: ksp_core_lib::ErrorCode = ksp_core_lib::ErrorCode::new("job_backfill", "persistence_invalid");
/// Error code used when deterministic Transport-to-RAW conversion violates the v1 contract. /// Error code used when deterministic Transport-to-RAW conversion violates the v1 contract.

View File

@@ -0,0 +1,272 @@
// file: crates/ksp-job-backfill-lib/src/execution.rs
// version: 1
use futures_util::StreamExt; // rust-rules: trait-import
/// Bounded result of one concurrent Backfill candidate execution pass.
///
/// This is intentionally smaller than the concrete latest-value snapshot planned for `pre.009`.
/// It exposes only the admission/frontier/checkpoint facts required to prove safe resumption.
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct BackfillExecutionBatch {
candidate_count: usize,
admitted_count: usize,
finished_count: usize,
durable_count: usize,
hole_count: usize,
maximum_in_flight: usize,
local_contiguous_completed: usize,
discovery_partial: bool,
checkpoint: crate::BackfillCheckpoint,
failure_code: std::option::Option<ksp_core_lib::ErrorCode>,
}
impl BackfillExecutionBatch {
/// Returns the number of candidates present in the bounded discovery result.
#[must_use]
pub const fn candidate_count(&self) -> usize {
return self.candidate_count;
}
/// Returns the number of candidates admitted into hydration during this execution pass.
#[must_use]
pub const fn admitted_count(&self) -> usize {
return self.admitted_count;
}
/// Returns the number of admitted candidates whose hydration/persistence future reached a known outcome.
#[must_use]
pub const fn finished_count(&self) -> usize {
return self.finished_count;
}
/// Returns the number of candidates whose Store outcome is durable for checkpoint advancement.
#[must_use]
pub const fn durable_count(&self) -> usize {
return self.durable_count;
}
/// Returns the number of known candidate outcomes that intentionally block the contiguous frontier.
#[must_use]
pub const fn hole_count(&self) -> usize {
return self.hole_count;
}
/// Returns the greatest number of candidate futures simultaneously in flight.
#[must_use]
pub const fn maximum_in_flight(&self) -> usize {
return self.maximum_in_flight;
}
/// Returns the durable contiguous prefix completed inside the current discovery result.
#[must_use]
pub const fn local_contiguous_completed(&self) -> usize {
return self.local_contiguous_completed;
}
/// Returns the safe caller-owned checkpoint after draining all work already admitted.
#[must_use]
pub const fn checkpoint(&self) -> &crate::BackfillCheckpoint {
return &self.checkpoint;
}
/// Returns the first stable fatal error code that stopped new admissions, when one occurred.
#[must_use]
pub const fn failure_code(&self) -> std::option::Option<ksp_core_lib::ErrorCode> {
return self.failure_code;
}
/// Returns whether discovery or candidate outcomes left the pass incomplete.
#[must_use]
pub const fn is_partial(&self) -> bool {
return self.discovery_partial || self.hole_count != 0 || self.failure_code.is_some();
}
}
/// Executes one bounded discovered candidate set with request-owned hydration concurrency.
///
/// At most `request.hydration_concurrency()` candidate futures are kept in flight. Candidate
/// completions may arrive out of order, but the checkpoint advances only across the contiguous
/// durable prefix. `Missing` remains a non-fatal hole. Conflict or any Transport/conversion/Store
/// error stops new admissions immediately; work already admitted is drained before this function
/// returns so a submitted Store operation is never silently abandoned by the batch coordinator.
pub async fn execute_backfill_discovery(
transport: &ksp_onchain_transport_lib::HttpTransportPool,
store: &ksp_store_lib::Store,
request: &crate::BackfillRequest,
discovery: &crate::BackfillDiscovery,
) -> ksp_core_lib::Result<BackfillExecutionBatch> {
let validation = crate::validate_discovery_identity(request, discovery);
if let std::result::Result::Err(error) = validation {
return std::result::Result::Err(error);
}
let processor = RuntimeCandidateProcessor { transport, store, request };
return execute_with_processor(&processor, request, discovery).await;
}
type CandidateProcessFuture<'a> =
std::pin::Pin<std::boxed::Box<dyn std::future::Future<Output = ksp_core_lib::Result<crate::BackfillPersistenceOutcome>> + 'a>>;
trait CandidateProcessor {
fn process<'a>(&'a self, candidate: &'a crate::BackfillCandidate) -> CandidateProcessFuture<'a>;
}
struct RuntimeCandidateProcessor<'a> {
transport: &'a ksp_onchain_transport_lib::HttpTransportPool,
store: &'a ksp_store_lib::Store,
request: &'a crate::BackfillRequest,
}
impl CandidateProcessor for RuntimeCandidateProcessor<'_> {
fn process<'a>(&'a self, candidate: &'a crate::BackfillCandidate) -> CandidateProcessFuture<'a> {
return std::boxed::Box::pin(async move {
let received_at = match current_raw_timestamp() {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let hydration = crate::hydrate_backfill_candidate(self.transport, self.request, candidate, received_at).await;
let hydration = match hydration {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
return crate::persist_backfill_hydration(self.store, hydration).await;
});
}
}
async fn execute_with_processor<P>(
processor: &P,
request: &crate::BackfillRequest,
discovery: &crate::BackfillDiscovery,
) -> ksp_core_lib::Result<BackfillExecutionBatch>
where
P: CandidateProcessor,
{
let validation = crate::validate_discovery_identity(request, discovery);
if let std::result::Result::Err(error) = validation {
return std::result::Result::Err(error);
}
let resume_offset = match crate::execution_resume_offset(request, discovery) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let mut frontier = crate::CompletionFrontier::new(discovery.candidates().len());
if matches!(request.scope().kind(), crate::BackfillScopeKind::AfterAddress | crate::BackfillScopeKind::ExplicitSignatures) {
let seeded = frontier.seed_prefix(resume_offset);
if let std::result::Result::Err(error) = seeded {
return std::result::Result::Err(error);
}
}
let mut next_index = resume_offset;
let mut admitted_count = 0_usize;
let mut finished_count = 0_usize;
let mut durable_count = 0_usize;
let mut hole_count = 0_usize;
let mut maximum_in_flight = 0_usize;
let mut failure_code = std::option::Option::<ksp_core_lib::ErrorCode>::None;
let mut in_flight = futures_util::stream::FuturesUnordered::new();
loop {
while failure_code.is_none() && next_index < discovery.candidates().len() && in_flight.len() < request.hydration_concurrency() {
let index = next_index;
let candidate = &discovery.candidates()[index];
let future = processor.process(candidate);
in_flight.push(async move {
return (index, future.await);
});
next_index += 1;
admitted_count = match checked_increment(admitted_count, "admitted_count") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
maximum_in_flight = std::cmp::max(maximum_in_flight, in_flight.len());
}
let completed = in_flight.next().await;
let (index, result) = match completed {
std::option::Option::Some(completed) => completed,
std::option::Option::None => break,
};
finished_count = match checked_increment(finished_count, "finished_count") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
match result {
std::result::Result::Ok(outcome) => {
if persistence_advances_frontier(&outcome) {
let marked = frontier.mark_durable(index);
if let std::result::Result::Err(error) = marked {
return std::result::Result::Err(error);
}
durable_count = match checked_increment(durable_count, "durable_count") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
} else {
hole_count = match checked_increment(hole_count, "hole_count") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
if outcome.entity() == crate::BackfillEntityPersistence::Conflict && failure_code.is_none() {
failure_code = std::option::Option::Some(ksp_store_lib::ERROR_CODE_RAW_CONFLICT);
}
}
},
std::result::Result::Err(error) => {
hole_count = match checked_increment(hole_count, "hole_count") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(increment_error) => return std::result::Result::Err(increment_error),
};
if failure_code.is_none() {
failure_code = std::option::Option::Some(error.code());
}
},
}
}
let checkpoint = match crate::checkpoint_from_frontier(request, discovery, &frontier) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
return std::result::Result::Ok(BackfillExecutionBatch {
candidate_count: discovery.candidates().len(),
admitted_count,
finished_count,
durable_count,
hole_count,
maximum_in_flight,
local_contiguous_completed: frontier.contiguous_completed(),
discovery_partial: discovery.is_partial(),
checkpoint,
failure_code,
});
}
fn persistence_advances_frontier(outcome: &crate::BackfillPersistenceOutcome) -> bool {
return matches!(
outcome.entity(),
crate::BackfillEntityPersistence::Inserted | crate::BackfillEntityPersistence::AlreadyPresent | crate::BackfillEntityPersistence::SkippedPurged
);
}
fn checked_increment(value: usize, field: &'static str) -> ksp_core_lib::Result<usize> {
return value.checked_add(1).ok_or_else(|| return execution_error(field));
}
fn current_raw_timestamp() -> ksp_core_lib::Result<ksp_store_lib::RawTimestamp> {
let duration = std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH);
let duration = match duration {
std::result::Result::Ok(duration) => duration,
std::result::Result::Err(_) => return std::result::Result::Err(execution_error("clock.before_epoch")),
};
let millis = match u64::try_from(duration.as_millis()) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return std::result::Result::Err(execution_error("clock.millis")),
};
return ksp_store_lib::RawTimestamp::from_unix_millis(millis);
}
fn execution_error(field: &'static str) -> ksp_core_lib::Error {
return ksp_core_lib::Error::new(crate::ERROR_CODE_BACKFILL_EXECUTION_INVALID, "invalid bounded Backfill execution state").with_context("field", field);
}
#[cfg(test)]
#[path = "../unit_tests/execution.rs"]
mod tests;

View File

@@ -1,5 +1,5 @@
// file: crates/ksp-job-backfill-lib/src/lib.rs // file: crates/ksp-job-backfill-lib/src/lib.rs
// version: 3 // version: 4
#![warn(missing_docs)] #![warn(missing_docs)]
#![deny(unreachable_pub)] #![deny(unreachable_pub)]
@@ -10,16 +10,21 @@
//! This tranche owns explicit admission, network-scoped candidate identity, deterministic //! This tranche owns explicit admission, network-scoped candidate identity, deterministic
//! `getSignaturesForAddress` pagination and canonical RAW v1 conversion through observed //! `getSignaturesForAddress` pagination and canonical RAW v1 conversion through observed
//! `getTransaction`. Transport retains provider/endpoint selection and retry; Store retains //! `getTransaction`. Transport retains provider/endpoint selection and retry; Store retains
//! durable idempotence through the atomic Store facade. Concurrency, checkpointing, cancellation //! durable idempotence through the atomic Store facade. This tranche also owns bounded concurrent
//! and concrete latest-value snapshots are added by later v0.3.6 tranches. //! candidate execution and caller-owned contiguous checkpoints. Cancellation and concrete latest-value
//! snapshots are added by later v0.3.6 tranches.
mod checkpoint;
mod constants; mod constants;
mod conversion; mod conversion;
mod discovery; mod discovery;
mod error; mod error;
mod execution;
mod persistence; mod persistence;
mod request; mod request;
/// Opaque caller-owned checkpoint for one controlled Backfill resumption.
pub use self::checkpoint::BackfillCheckpoint;
/// Result of hydrating one deterministic candidate through observed `getTransaction`. /// Result of hydrating one deterministic candidate through observed `getTransaction`.
pub use self::conversion::BackfillHydrationOutcome; pub use self::conversion::BackfillHydrationOutcome;
/// Complete in-memory RAW transaction acquisition ready for later Store persistence. /// Complete in-memory RAW transaction acquisition ready for later Store persistence.
@@ -40,10 +45,14 @@ pub use self::discovery::BackfillDiscovery;
pub use self::discovery::BackfillDiscoveryBoundary; pub use self::discovery::BackfillDiscoveryBoundary;
/// Discovers one bounded deterministic candidate set through the typed KSP Transport wrapper. /// Discovers one bounded deterministic candidate set through the typed KSP Transport wrapper.
pub use self::discovery::discover_backfill_candidates; pub use self::discovery::discover_backfill_candidates;
/// Error code used when one checkpoint/frontier is incompatible with the current Job or semantic scope.
pub use self::error::ERROR_CODE_BACKFILL_CHECKPOINT_INVALID;
/// Error code used when a signature page violates a bounded discovery invariant. /// Error code used when a signature page violates a bounded discovery invariant.
pub use self::error::ERROR_CODE_BACKFILL_DISCOVERY_INVALID; pub use self::error::ERROR_CODE_BACKFILL_DISCOVERY_INVALID;
/// Error code used when paginated discovery cannot advance its exclusive RPC cursor safely. /// Error code used when paginated discovery cannot advance its exclusive RPC cursor safely.
pub use self::error::ERROR_CODE_BACKFILL_DISCOVERY_STALLED; pub use self::error::ERROR_CODE_BACKFILL_DISCOVERY_STALLED;
/// Error code used when bounded concurrent execution violates an internal admission or clock invariant.
pub use self::error::ERROR_CODE_BACKFILL_EXECUTION_INVALID;
/// Error code used when Store persistence returns an impossible Backfill state or targets a different network. /// Error code used when Store persistence returns an impossible Backfill state or targets a different network.
pub use self::error::ERROR_CODE_BACKFILL_PERSISTENCE_INVALID; pub use self::error::ERROR_CODE_BACKFILL_PERSISTENCE_INVALID;
/// Error code used when deterministic Transport-to-RAW conversion violates the v1 contract. /// Error code used when deterministic Transport-to-RAW conversion violates the v1 contract.
@@ -52,6 +61,10 @@ pub use self::error::ERROR_CODE_BACKFILL_RAW_CONVERSION_INVALID;
pub use self::error::ERROR_CODE_BACKFILL_REQUEST_INVALID; pub use self::error::ERROR_CODE_BACKFILL_REQUEST_INVALID;
/// Error code used when one transaction signature text violates the bounded Base58-shape contract. /// Error code used when one transaction signature text violates the bounded Base58-shape contract.
pub use self::error::ERROR_CODE_BACKFILL_SIGNATURE_INVALID; pub use self::error::ERROR_CODE_BACKFILL_SIGNATURE_INVALID;
/// Bounded result of one concurrent Backfill candidate execution pass.
pub use self::execution::BackfillExecutionBatch;
/// Executes one bounded discovered candidate set with request-owned hydration concurrency.
pub use self::execution::execute_backfill_discovery;
/// Canonical entity disposition produced by one Backfill Store persistence attempt. /// Canonical entity disposition produced by one Backfill Store persistence attempt.
pub use self::persistence::BackfillEntityPersistence; pub use self::persistence::BackfillEntityPersistence;
/// Observation disposition produced by one Backfill Store persistence attempt. /// Observation disposition produced by one Backfill Store persistence attempt.
@@ -85,6 +98,18 @@ pub use self::request::MAX_BACKFILL_SIGNATURE_TEXT_BYTES;
/// Minimum Base58 text length possible for one canonical 64-byte Solana signature. /// Minimum Base58 text length possible for one canonical 64-byte Solana signature.
pub use self::request::MIN_BACKFILL_SIGNATURE_TEXT_BYTES; pub use self::request::MIN_BACKFILL_SIGNATURE_TEXT_BYTES;
/// Internal contiguous completion frontier used by bounded execution.
pub(crate) use self::checkpoint::CompletionFrontier;
/// Builds one safe caller-owned checkpoint from the current contiguous frontier.
pub(crate) use self::checkpoint::checkpoint_from_frontier;
/// Resolves the candidate offset skipped by one validated replay checkpoint.
pub(crate) use self::checkpoint::execution_resume_offset;
/// Resolves the internal exclusive Before cursor for discovery resumption.
pub(crate) use self::checkpoint::resume_before_cursor;
/// Validates that one discovery result belongs to the current request identity.
pub(crate) use self::checkpoint::validate_discovery_identity;
/// Validates one caller-owned checkpoint against Job and semantic scope identity.
pub(crate) use self::checkpoint::validate_request_checkpoint;
/// Owning tracing target used by the concrete Backfill runtime. /// Owning tracing target used by the concrete Backfill runtime.
pub(crate) use self::constants::TRACING_TARGET; pub(crate) use self::constants::TRACING_TARGET;
/// Exact private Base58 decoder shared by the public signature wrapper and hydration path. /// Exact private Base58 decoder shared by the public signature wrapper and hydration path.

View File

@@ -1,5 +1,5 @@
// file: crates/ksp-job-backfill-lib/src/persistence.rs // file: crates/ksp-job-backfill-lib/src/persistence.rs
// version: 1 // version: 2
/// Canonical entity disposition produced by one Backfill Store persistence attempt. /// Canonical entity disposition produced by one Backfill Store persistence attempt.
#[non_exhaustive] #[non_exhaustive]
@@ -43,6 +43,15 @@ pub struct BackfillPersistenceOutcome {
} }
impl BackfillPersistenceOutcome { impl BackfillPersistenceOutcome {
/// Creates one internally classified persistence result.
pub(crate) fn new(
reference: ksp_store_lib::RawTransactionReference,
entity: BackfillEntityPersistence,
observation: BackfillObservationPersistence,
) -> Self {
return Self { reference, entity, observation };
}
/// Returns the network-scoped canonical transaction identity classified by this result. /// Returns the network-scoped canonical transaction identity classified by this result.
#[must_use] #[must_use]
pub fn reference(&self) -> &ksp_store_lib::RawTransactionReference { pub fn reference(&self) -> &ksp_store_lib::RawTransactionReference {
@@ -112,11 +121,11 @@ where
return std::result::Result::Err(persistence_error("store.network")); return std::result::Result::Err(persistence_error("store.network"));
} }
return match hydration { return match hydration {
crate::BackfillHydrationOutcome::Missing(_) => std::result::Result::Ok(BackfillPersistenceOutcome { crate::BackfillHydrationOutcome::Missing(_) => std::result::Result::Ok(BackfillPersistenceOutcome::new(
reference, reference,
entity: BackfillEntityPersistence::Missing, BackfillEntityPersistence::Missing,
observation: BackfillObservationPersistence::NotApplicable, BackfillObservationPersistence::NotApplicable,
}), )),
crate::BackfillHydrationOutcome::Available(acquisition) => { crate::BackfillHydrationOutcome::Available(acquisition) => {
let (transaction, observation) = acquisition.into_parts(); let (transaction, observation) = acquisition.into_parts();
persist_available_with_port(port, reference, transaction, observation).await persist_available_with_port(port, reference, transaction, observation).await
@@ -144,11 +153,11 @@ where
std::result::Result::Ok(outcome) => map_store_outcome(reference, outcome), std::result::Result::Ok(outcome) => map_store_outcome(reference, outcome),
std::result::Result::Err(error) => { std::result::Result::Err(error) => {
if error.code() == ksp_store_lib::ERROR_CODE_RAW_CONFLICT { if error.code() == ksp_store_lib::ERROR_CODE_RAW_CONFLICT {
return std::result::Result::Ok(BackfillPersistenceOutcome { return std::result::Result::Ok(BackfillPersistenceOutcome::new(
reference, reference,
entity: BackfillEntityPersistence::Conflict, BackfillEntityPersistence::Conflict,
observation: BackfillObservationPersistence::NotRecorded, BackfillObservationPersistence::NotRecorded,
}); ));
} }
std::result::Result::Err(error) std::result::Result::Err(error)
}, },
@@ -162,32 +171,32 @@ fn map_store_outcome(
let entity = outcome.entity(); let entity = outcome.entity();
let observation = outcome.observation(); let observation = outcome.observation();
if entity == ksp_store_lib::RawEntityWriteOutcome::Inserted && observation == ksp_store_lib::RawObservationWriteOutcome::Inserted { if entity == ksp_store_lib::RawEntityWriteOutcome::Inserted && observation == ksp_store_lib::RawObservationWriteOutcome::Inserted {
return std::result::Result::Ok(BackfillPersistenceOutcome { return std::result::Result::Ok(BackfillPersistenceOutcome::new(
reference, reference,
entity: BackfillEntityPersistence::Inserted, BackfillEntityPersistence::Inserted,
observation: BackfillObservationPersistence::Inserted, BackfillObservationPersistence::Inserted,
}); ));
} }
if entity == ksp_store_lib::RawEntityWriteOutcome::AlreadyPresent && observation == ksp_store_lib::RawObservationWriteOutcome::Inserted { if entity == ksp_store_lib::RawEntityWriteOutcome::AlreadyPresent && observation == ksp_store_lib::RawObservationWriteOutcome::Inserted {
return std::result::Result::Ok(BackfillPersistenceOutcome { return std::result::Result::Ok(BackfillPersistenceOutcome::new(
reference, reference,
entity: BackfillEntityPersistence::AlreadyPresent, BackfillEntityPersistence::AlreadyPresent,
observation: BackfillObservationPersistence::Inserted, BackfillObservationPersistence::Inserted,
}); ));
} }
if entity == ksp_store_lib::RawEntityWriteOutcome::AlreadyPresent && observation == ksp_store_lib::RawObservationWriteOutcome::AlreadyPresent { if entity == ksp_store_lib::RawEntityWriteOutcome::AlreadyPresent && observation == ksp_store_lib::RawObservationWriteOutcome::AlreadyPresent {
return std::result::Result::Ok(BackfillPersistenceOutcome { return std::result::Result::Ok(BackfillPersistenceOutcome::new(
reference, reference,
entity: BackfillEntityPersistence::AlreadyPresent, BackfillEntityPersistence::AlreadyPresent,
observation: BackfillObservationPersistence::AlreadyPresent, BackfillObservationPersistence::AlreadyPresent,
}); ));
} }
if entity == ksp_store_lib::RawEntityWriteOutcome::SkippedPurged && observation == ksp_store_lib::RawObservationWriteOutcome::NotRecorded { if entity == ksp_store_lib::RawEntityWriteOutcome::SkippedPurged && observation == ksp_store_lib::RawObservationWriteOutcome::NotRecorded {
return std::result::Result::Ok(BackfillPersistenceOutcome { return std::result::Result::Ok(BackfillPersistenceOutcome::new(
reference, reference,
entity: BackfillEntityPersistence::SkippedPurged, BackfillEntityPersistence::SkippedPurged,
observation: BackfillObservationPersistence::NotRecorded, BackfillObservationPersistence::NotRecorded,
}); ));
} }
return std::result::Result::Err(persistence_error("store.outcome")); return std::result::Result::Err(persistence_error("store.outcome"));
} }

View File

@@ -1,5 +1,5 @@
// file: crates/ksp-job-backfill-lib/src/request.rs // file: crates/ksp-job-backfill-lib/src/request.rs
// version: 2 // version: 3
use sha2::Digest; // rust-rules: trait-import use sha2::Digest; // rust-rules: trait-import
@@ -251,6 +251,7 @@ pub struct BackfillRequest {
hydration_concurrency: usize, hydration_concurrency: usize,
min_context_slot: std::option::Option<u64>, min_context_slot: std::option::Option<u64>,
scope_fingerprint: BackfillScopeFingerprint, scope_fingerprint: BackfillScopeFingerprint,
checkpoint: std::option::Option<crate::BackfillCheckpoint>,
} }
impl BackfillRequest { impl BackfillRequest {
@@ -308,6 +309,7 @@ impl BackfillRequest {
hydration_concurrency, hydration_concurrency,
min_context_slot, min_context_slot,
scope_fingerprint, scope_fingerprint,
checkpoint: std::option::Option::None,
}); });
} }
@@ -379,6 +381,22 @@ impl BackfillRequest {
pub const fn scope_fingerprint(&self) -> BackfillScopeFingerprint { pub const fn scope_fingerprint(&self) -> BackfillScopeFingerprint {
return self.scope_fingerprint; return self.scope_fingerprint;
} }
/// Attaches one caller-owned checkpoint after validating Job and semantic scope identity.
pub fn with_checkpoint(mut self, checkpoint: crate::BackfillCheckpoint) -> ksp_core_lib::Result<Self> {
let validation = crate::validate_request_checkpoint(&self.job_id, self.scope_fingerprint, self.scope.kind(), &checkpoint);
if let std::result::Result::Err(error) = validation {
return std::result::Result::Err(error);
}
self.checkpoint = std::option::Option::Some(checkpoint);
return std::result::Result::Ok(self);
}
/// Returns the validated optional checkpoint supplied for controlled resumption.
#[must_use]
pub const fn checkpoint(&self) -> std::option::Option<&crate::BackfillCheckpoint> {
return self.checkpoint.as_ref();
}
} }
impl std::fmt::Debug for BackfillRequest { impl std::fmt::Debug for BackfillRequest {
@@ -396,6 +414,7 @@ impl std::fmt::Debug for BackfillRequest {
.field("hydration_concurrency", &self.hydration_concurrency) .field("hydration_concurrency", &self.hydration_concurrency)
.field("min_context_slot", &self.min_context_slot) .field("min_context_slot", &self.min_context_slot)
.field("scope_fingerprint", &self.scope_fingerprint) .field("scope_fingerprint", &self.scope_fingerprint)
.field("has_checkpoint", &self.checkpoint.is_some())
.finish(); .finish();
} }
} }

View File

@@ -1,12 +1,13 @@
// file: crates/ksp-job-backfill-lib/tests/dependency_boundary.rs // file: crates/ksp-job-backfill-lib/tests/dependency_boundary.rs
// version: 3 // version: 4
//! Dependency firewall canaries through Backfill Store persistence. //! Dependency firewall canaries through bounded Backfill execution/checkpointing.
#[test] #[test]
fn pre_007_manifest_uses_only_planned_ksp_edges_and_backend_neutral_store() { fn pre_008_manifest_uses_only_planned_ksp_edges_and_private_futures_runtime() {
let manifest = include_str!("../Cargo.toml"); let manifest = include_str!("../Cargo.toml");
for required in [ for required in [
"futures-util = { workspace = true, features = [\"std\"] }",
"ksp-core-lib = { path = \"../ksp-core-lib\" }", "ksp-core-lib = { path = \"../ksp-core-lib\" }",
"ksp-job-api = { path = \"../ksp-job-api\" }", "ksp-job-api = { path = \"../ksp-job-api\" }",
"ksp-logging-lib = { path = \"../ksp-logging-lib\" }", "ksp-logging-lib = { path = \"../ksp-logging-lib\" }",
@@ -32,22 +33,26 @@ fn pre_007_manifest_uses_only_planned_ksp_edges_and_backend_neutral_store() {
] { ] {
assert!(!manifest.contains(forbidden), "forbidden Backfill dependency present: {forbidden}"); assert!(!manifest.contains(forbidden), "forbidden Backfill dependency present: {forbidden}");
} }
let normal_dependencies = match manifest.split("[dev-dependencies]").next() {
std::option::Option::Some(value) => value,
std::option::Option::None => manifest,
};
assert!(!normal_dependencies.contains("tokio ="), "Tokio must not become a public/normal Backfill dependency in pre.008");
return; return;
} }
#[test] #[test]
fn pre_007_production_sources_keep_transport_and_store_in_their_owned_layers() { fn pre_008_production_sources_keep_transport_store_and_scheduler_ownership_separate() {
let non_conversion_sources = [ let neutral_sources = [
include_str!("../src/checkpoint.rs"),
include_str!("../src/constants.rs"), include_str!("../src/constants.rs"),
include_str!("../src/discovery.rs"), include_str!("../src/discovery.rs"),
include_str!("../src/error.rs"), include_str!("../src/error.rs"),
include_str!("../src/lib.rs"), include_str!("../src/lib.rs"),
include_str!("../src/request.rs"), include_str!("../src/request.rs"),
]; ];
for source in non_conversion_sources { for source in neutral_sources {
for forbidden in for forbidden in ["ksp_config_lib::", "ksp_interface_lib::", "ksp_store_api::", "ksp_store_postgres_lib::", "reqwest::", "std::env", "tonic::"] {
["ksp_config_lib::", "ksp_interface_lib::", "ksp_store_api::", "ksp_store_postgres_lib::", "reqwest::", "serde_json::", "std::env", "tonic::"]
{
assert!(!source.contains(forbidden), "forbidden concrete Backfill path detected: {forbidden}"); assert!(!source.contains(forbidden), "forbidden concrete Backfill path detected: {forbidden}");
} }
} }
@@ -59,23 +64,24 @@ fn pre_007_production_sources_keep_transport_and_store_in_their_owned_layers() {
assert!(conversion.contains("BackfillHydrationOutcome::Missing(reference)")); assert!(conversion.contains("BackfillHydrationOutcome::Missing(reference)"));
assert!(!conversion.contains("execute_standard_rpc")); assert!(!conversion.contains("execute_standard_rpc"));
assert!(!conversion.contains("persist_raw_transaction_acquisition")); assert!(!conversion.contains("persist_raw_transaction_acquisition"));
for forbidden in ["ksp_config_lib::", "ksp_interface_lib::", "ksp_store_api::", "ksp_store_postgres_lib::", "reqwest::", "std::env", "tonic::"] {
assert!(!conversion.contains(forbidden), "forbidden RAW conversion path detected: {forbidden}");
}
let persistence = include_str!("../src/persistence.rs"); let persistence = include_str!("../src/persistence.rs");
assert!(persistence.contains("persist_raw_transaction_acquisition")); assert!(persistence.contains("persist_raw_transaction_acquisition"));
assert!(persistence.contains("RawTransactionAcquisitionMode::Normal")); assert!(persistence.contains("RawTransactionAcquisitionMode::Normal"));
assert!(persistence.contains("ERROR_CODE_RAW_CONFLICT")); assert!(persistence.contains("ERROR_CODE_RAW_CONFLICT"));
assert!(!persistence.contains("record_raw_transaction_observation")); assert!(!persistence.contains("record_raw_transaction_observation"));
assert!(!persistence.contains("RawTransactionAcquisitionMode::ForceRehydrate")); assert!(!persistence.contains("RawTransactionAcquisitionMode::ForceRehydrate"));
for forbidden in ["ksp_config_lib::", "ksp_interface_lib::", "ksp_store_api::", "ksp_store_postgres_lib::", "reqwest::", "std::env", "tonic::"] {
assert!(!persistence.contains(forbidden), "forbidden Store persistence path detected: {forbidden}");
}
let discovery = include_str!("../src/discovery.rs"); let discovery = include_str!("../src/discovery.rs");
assert!(discovery.contains("get_signatures_for_address")); assert!(discovery.contains("get_signatures_for_address"));
assert!(!discovery.contains("execute_standard_rpc")); assert!(!discovery.contains("execute_standard_rpc"));
assert!(!discovery.contains("retry")); assert!(!discovery.contains("retry"));
assert!(!discovery.contains("endpoint_name")); assert!(!discovery.contains("endpoint_name"));
assert!(!discovery.contains("HttpEndpoint")); let execution = include_str!("../src/execution.rs");
assert!(execution.contains("FuturesUnordered"));
assert!(execution.contains("hydrate_backfill_candidate"));
assert!(execution.contains("persist_backfill_hydration"));
assert!(execution.contains("request.hydration_concurrency()"));
for forbidden in ["tokio::spawn", "tokio::time", "retry", "endpoint_name", "provider()", "ForceRehydrate", "get_raw_transaction"] {
assert!(!execution.contains(forbidden), "forbidden scheduler/policy ownership detected: {forbidden}");
}
return; return;
} }

View File

@@ -1,7 +1,7 @@
// file: crates/ksp-job-backfill-lib/tests/public_api.rs // file: crates/ksp-job-backfill-lib/tests/public_api.rs
// version: 3 // version: 4
//! Public API canaries for bounded Backfill discovery, RAW v1 conversion and Store persistence. //! Public API canaries through bounded Backfill execution and caller-owned checkpoints.
#[test] #[test]
fn pre_005_request_scope_and_discovery_contracts_are_available_from_crate_root() { fn pre_005_request_scope_and_discovery_contracts_are_available_from_crate_root() {
@@ -96,3 +96,13 @@ fn pre_007_store_persistence_contract_is_available_from_crate_root() {
assert_eq!(ksp_job_backfill_lib::ERROR_CODE_BACKFILL_PERSISTENCE_INVALID, ksp_core_lib::ErrorCode::new("job_backfill", "persistence_invalid")); assert_eq!(ksp_job_backfill_lib::ERROR_CODE_BACKFILL_PERSISTENCE_INVALID, ksp_core_lib::ErrorCode::new("job_backfill", "persistence_invalid"));
return; return;
} }
#[test]
fn pre_008_checkpoint_and_bounded_execution_contracts_are_available_from_crate_root() {
let _execute = ksp_job_backfill_lib::execute_backfill_discovery;
let _checkpoint: std::option::Option<ksp_job_backfill_lib::BackfillCheckpoint> = std::option::Option::None;
let _batch: std::option::Option<ksp_job_backfill_lib::BackfillExecutionBatch> = std::option::Option::None;
assert_eq!(ksp_job_backfill_lib::ERROR_CODE_BACKFILL_CHECKPOINT_INVALID, ksp_core_lib::ErrorCode::new("job_backfill", "checkpoint_invalid"));
assert_eq!(ksp_job_backfill_lib::ERROR_CODE_BACKFILL_EXECUTION_INVALID, ksp_core_lib::ErrorCode::new("job_backfill", "execution_invalid"));
return;
}

View File

@@ -1,10 +1,10 @@
// file: crates/ksp-job-backfill-lib/tests/release_completeness.rs // file: crates/ksp-job-backfill-lib/tests/release_completeness.rs
// version: 3 // version: 4
//! Completeness canaries through the `pre.007` Backfill Store persistence tranche. //! Completeness canaries through the `pre.008` bounded execution/checkpoint tranche.
#[test] #[test]
fn pre_007_production_module_inventory_is_exact() -> std::io::Result<()> { fn pre_008_production_module_inventory_is_exact() -> std::io::Result<()> {
let source_root = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("src"); let source_root = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("src");
let entries = match std::fs::read_dir(source_root) { let entries = match std::fs::read_dir(source_root) {
std::result::Result::Ok(value) => value, std::result::Result::Ok(value) => value,
@@ -32,12 +32,15 @@ fn pre_007_production_module_inventory_is_exact() -> std::io::Result<()> {
} }
} }
names.sort_unstable(); names.sort_unstable();
assert_eq!(names, std::vec!["constants.rs", "conversion.rs", "discovery.rs", "error.rs", "lib.rs", "persistence.rs", "request.rs"]); assert_eq!(
names,
std::vec!["checkpoint.rs", "constants.rs", "conversion.rs", "discovery.rs", "error.rs", "execution.rs", "lib.rs", "persistence.rs", "request.rs"]
);
return std::result::Result::Ok(()); return std::result::Result::Ok(());
} }
#[test] #[test]
fn pre_007_surface_adds_store_persistence_without_checkpoint_runtime() { fn pre_008_surface_adds_bounded_execution_and_checkpoint_without_pre_009_runtime() {
let root = include_str!("../src/lib.rs"); let root = include_str!("../src/lib.rs");
for required in [ for required in [
"BackfillCandidate", "BackfillCandidate",
@@ -53,19 +56,20 @@ fn pre_007_surface_adds_store_persistence_without_checkpoint_runtime() {
"BackfillRawAcquisition", "BackfillRawAcquisition",
"BackfillHydrationOutcome", "BackfillHydrationOutcome",
"hydrate_backfill_candidate", "hydrate_backfill_candidate",
"RAW_TRANSACTION_FORMAT_ID",
"RAW_TRANSACTION_FORMAT_VERSION",
"ERROR_CODE_BACKFILL_RAW_CONVERSION_INVALID",
"BackfillPersistenceOutcome", "BackfillPersistenceOutcome",
"BackfillEntityPersistence", "BackfillEntityPersistence",
"BackfillObservationPersistence", "BackfillObservationPersistence",
"persist_backfill_hydration", "persist_backfill_hydration",
"ERROR_CODE_BACKFILL_PERSISTENCE_INVALID", "BackfillCheckpoint",
"BackfillExecutionBatch",
"execute_backfill_discovery",
"ERROR_CODE_BACKFILL_CHECKPOINT_INVALID",
"ERROR_CODE_BACKFILL_EXECUTION_INVALID",
] { ] {
assert!(root.contains(required), "required pre.006 public contract missing: {required}"); assert!(root.contains(required), "required pre.008 public contract missing: {required}");
} }
for forbidden in ["BackfillCheckpoint", "BackfillJobHandle", "JobSnapshotSource"] { for forbidden in ["BackfillJobHandle", "BackfillSnapshot", "JobSnapshotSource", "tokio::", "FuturesUnordered"] {
assert!(!root.contains(forbidden), "later Backfill tranche leaked into pre.007: {forbidden}"); assert!(!root.contains(forbidden), "pre.009 or runtime implementation detail leaked into public root: {forbidden}");
} }
assert!(!root.contains("pub mod ")); assert!(!root.contains("pub mod "));
return; return;

View File

@@ -0,0 +1,168 @@
// file: crates/ksp-job-backfill-lib/unit_tests/checkpoint.rs
// version: 1
fn signature(character: char) -> std::option::Option<crate::BackfillSignature> {
return crate::BackfillSignature::new(character.to_string().repeat(crate::MIN_BACKFILL_SIGNATURE_TEXT_BYTES)).ok();
}
fn request(scope: crate::BackfillScope) -> std::option::Option<crate::BackfillRequest> {
let job_id = match ksp_job_api::JobId::new("backfill:checkpoint-test") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return std::option::Option::None,
};
let network = match ksp_store_lib::RawNetworkId::new("devnet") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return std::option::Option::None,
};
return crate::BackfillRequest::new(
job_id,
network,
ksp_onchain_transport_lib::HttpRoleName::new("history"),
crate::BackfillCommitment::Confirmed,
scope,
100,
10,
10,
4,
std::option::Option::None,
)
.ok();
}
fn candidate(request: &crate::BackfillRequest, character: char) -> std::option::Option<crate::BackfillCandidate> {
let signature = match signature(character) {
std::option::Option::Some(value) => value,
std::option::Option::None => return std::option::Option::None,
};
let identity = crate::BackfillCandidateIdentity::new(request.network().clone(), signature);
return std::option::Option::Some(crate::BackfillCandidate::new(identity, std::option::Option::None));
}
fn discovery(
request: &crate::BackfillRequest,
characters: &[char],
boundary: crate::BackfillDiscoveryBoundary,
) -> std::option::Option<crate::BackfillDiscovery> {
let mut candidates = std::vec::Vec::with_capacity(characters.len());
for character in characters {
let value = match candidate(request, *character) {
std::option::Option::Some(value) => value,
std::option::Option::None => return std::option::Option::None,
};
candidates.push(value);
}
return std::option::Option::Some(crate::BackfillDiscovery::new(request.network().clone(), request.scope_fingerprint(), candidates, 1, boundary));
}
#[test]
fn pre_008_frontier_advances_only_across_contiguous_durable_results() {
let mut frontier = crate::CompletionFrontier::new(4);
assert!(frontier.mark_durable(1).is_ok());
assert_eq!(frontier.contiguous_completed(), 0);
assert!(frontier.mark_durable(0).is_ok());
assert_eq!(frontier.contiguous_completed(), 2);
assert!(frontier.mark_durable(3).is_ok());
assert_eq!(frontier.contiguous_completed(), 2);
assert!(frontier.mark_durable(2).is_ok());
assert_eq!(frontier.contiguous_completed(), 4);
return;
}
#[test]
fn pre_008_checkpoint_is_bound_to_job_and_scope_fingerprint() {
let scope = crate::BackfillScope::latest_address(ksp_core_lib::Pubkey::new_from_array([1_u8; 32]));
let request = match request(scope) {
std::option::Option::Some(value) => value,
std::option::Option::None => return,
};
let other_job = match ksp_job_api::JobId::new("backfill:other") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return,
};
let wrong_job = crate::BackfillCheckpoint::new(other_job, request.scope_fingerprint(), 0, std::option::Option::None);
let result = request.clone().with_checkpoint(wrong_job);
assert!(result.is_err());
if let std::result::Result::Err(error) = result {
assert_eq!(error.code(), crate::ERROR_CODE_BACKFILL_CHECKPOINT_INVALID);
}
let other_scope = crate::BackfillScope::latest_address(ksp_core_lib::Pubkey::new_from_array([2_u8; 32]));
let other_request = match request(other_scope) {
std::option::Option::Some(value) => value,
std::option::Option::None => return,
};
let wrong_scope = crate::BackfillCheckpoint::new(request.job_id().clone(), other_request.scope_fingerprint(), 0, std::option::Option::None);
assert!(request.with_checkpoint(wrong_scope).is_err());
return;
}
#[test]
fn pre_008_before_checkpoint_is_cumulative_and_tracks_last_contiguous_candidate() {
let anchor = match signature('8') {
std::option::Option::Some(value) => value,
std::option::Option::None => return,
};
let scope = crate::BackfillScope::before_address(ksp_core_lib::Pubkey::new_from_array([3_u8; 32]), anchor);
let request = match request(scope) {
std::option::Option::Some(value) => value,
std::option::Option::None => return,
};
let first_discovery = match discovery(&request, &['7', '6', '5'], crate::BackfillDiscoveryBoundary::RpcBoundary) {
std::option::Option::Some(value) => value,
std::option::Option::None => return,
};
let mut first_frontier = crate::CompletionFrontier::new(3);
assert!(first_frontier.mark_durable(1).is_ok());
assert_eq!(first_frontier.contiguous_completed(), 0);
assert!(first_frontier.mark_durable(0).is_ok());
let first = match crate::checkpoint_from_frontier(&request, &first_discovery, &first_frontier) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return,
};
assert_eq!(first.completed_prefix(), 2);
let expected_first = "6".repeat(64);
assert_eq!(first.resume_before().map(crate::BackfillSignature::as_str), std::option::Option::Some(expected_first.as_str()));
let resumed = match request.with_checkpoint(first) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return,
};
let second_discovery = match discovery(&resumed, &['5', '4'], crate::BackfillDiscoveryBoundary::RpcBoundary) {
std::option::Option::Some(value) => value,
std::option::Option::None => return,
};
let mut second_frontier = crate::CompletionFrontier::new(2);
assert!(second_frontier.mark_durable(0).is_ok());
let second = match crate::checkpoint_from_frontier(&resumed, &second_discovery, &second_frontier) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return,
};
assert_eq!(second.completed_prefix(), 3);
let expected_second = "5".repeat(64);
assert_eq!(second.resume_before().map(crate::BackfillSignature::as_str), std::option::Option::Some(expected_second.as_str()));
return;
}
#[test]
fn pre_008_after_anchor_not_reached_never_advances_checkpoint() {
let anchor = match signature('1') {
std::option::Option::Some(value) => value,
std::option::Option::None => return,
};
let scope = crate::BackfillScope::after_address(ksp_core_lib::Pubkey::new_from_array([4_u8; 32]), anchor);
let request = match request(scope) {
std::option::Option::Some(value) => value,
std::option::Option::None => return,
};
let partial = match discovery(&request, &['7', '6', '5'], crate::BackfillDiscoveryBoundary::AfterAnchorNotReached) {
std::option::Option::Some(value) => value,
std::option::Option::None => return,
};
let mut frontier = crate::CompletionFrontier::new(3);
assert!(frontier.mark_durable(0).is_ok());
assert!(frontier.mark_durable(1).is_ok());
let checkpoint = match crate::checkpoint_from_frontier(&request, &partial, &frontier) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return,
};
assert_eq!(checkpoint.completed_prefix(), 0);
return;
}

View File

@@ -1,5 +1,5 @@
// file: crates/ksp-job-backfill-lib/unit_tests/discovery.rs // file: crates/ksp-job-backfill-lib/unit_tests/discovery.rs
// version: 1 // version: 2
#[derive(Clone, Debug, Eq, PartialEq)] #[derive(Clone, Debug, Eq, PartialEq)]
struct PageCall { struct PageCall {
@@ -305,3 +305,53 @@ async fn pre_005_explicit_scope_never_calls_transport_and_preserves_network_scop
assert!(source.calls().is_empty()); assert!(source.calls().is_empty());
return; return;
} }
#[tokio::test]
async fn pre_008_before_resume_uses_checkpoint_cursor_instead_of_original_anchor() {
let anchor = match signature('8') {
std::option::Option::Some(value) => value,
std::option::Option::None => return,
};
let resume = match signature('5') {
std::option::Option::Some(value) => value,
std::option::Option::None => return,
};
let scope = crate::BackfillScope::before_address(ksp_core_lib::Pubkey::new_from_array([8_u8; 32]), anchor);
let request = match request(scope, 2, 2, 4) {
std::option::Option::Some(value) => value,
std::option::Option::None => return,
};
let checkpoint = crate::BackfillCheckpoint::new(request.job_id().clone(), request.scope_fingerprint(), 2, std::option::Option::Some(resume.clone()));
let request = match request.with_checkpoint(checkpoint) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return,
};
let source = FakeSource::new(std::vec![std::vec![page_entry('4', 40)]]);
let result = super::discover_with_source(&source, &request).await;
assert!(result.is_ok());
let calls = source.calls();
assert_eq!(calls.len(), 1);
assert_eq!(calls[0].before.as_deref(), std::option::Option::Some(resume.as_str()));
return;
}
#[tokio::test]
async fn pre_008_latest_resume_restarts_from_current_latest_without_rpc_cursor() {
let scope = crate::BackfillScope::latest_address(ksp_core_lib::Pubkey::new_from_array([9_u8; 32]));
let request = match request(scope, 2, 2, 4) {
std::option::Option::Some(value) => value,
std::option::Option::None => return,
};
let checkpoint = crate::BackfillCheckpoint::new(request.job_id().clone(), request.scope_fingerprint(), 2, std::option::Option::None);
let request = match request.with_checkpoint(checkpoint) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return,
};
let source = FakeSource::new(std::vec![std::vec![page_entry('7', 70)]]);
let result = super::discover_with_source(&source, &request).await;
assert!(result.is_ok());
let calls = source.calls();
assert_eq!(calls.len(), 1);
assert_eq!(calls[0].before, std::option::Option::None);
return;
}

View File

@@ -0,0 +1,276 @@
// file: crates/ksp-job-backfill-lib/unit_tests/execution.rs
// version: 1
#[derive(Clone, Copy)]
enum FakeDisposition {
Durable,
Missing,
Conflict,
Failure,
}
#[derive(Clone, Copy)]
struct FakePlan {
pending_polls: usize,
disposition: FakeDisposition,
}
struct FakeProcessor {
plans: std::vec::Vec<FakePlan>,
active: std::sync::atomic::AtomicUsize,
maximum: std::sync::atomic::AtomicUsize,
calls: std::sync::atomic::AtomicUsize,
}
impl FakeProcessor {
fn new(plans: std::vec::Vec<FakePlan>) -> Self {
return Self {
plans,
active: std::sync::atomic::AtomicUsize::new(0),
maximum: std::sync::atomic::AtomicUsize::new(0),
calls: std::sync::atomic::AtomicUsize::new(0),
};
}
fn maximum(&self) -> usize {
return self.maximum.load(std::sync::atomic::Ordering::SeqCst);
}
fn calls(&self) -> usize {
return self.calls.load(std::sync::atomic::Ordering::SeqCst);
}
}
impl super::CandidateProcessor for FakeProcessor {
fn process<'a>(&'a self, candidate: &'a crate::BackfillCandidate) -> super::CandidateProcessFuture<'a> {
self.calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
let first_byte = candidate.identity().signature().as_str().as_bytes()[0];
let plan_index = usize::from(first_byte.saturating_sub(b'1'));
let plan = match self.plans.get(plan_index) {
std::option::Option::Some(plan) => *plan,
std::option::Option::None => FakePlan { pending_polls: 0, disposition: FakeDisposition::Failure },
};
return std::boxed::Box::pin(async move {
let mut pending_polls = plan.pending_polls;
let mut started = false;
std::future::poll_fn(|context| {
if !started {
started = true;
let active = self.active.fetch_add(1, std::sync::atomic::Ordering::SeqCst) + 1;
self.maximum.fetch_max(active, std::sync::atomic::Ordering::SeqCst);
}
if pending_polls != 0 {
pending_polls -= 1;
context.waker().wake_by_ref();
return std::task::Poll::Pending;
}
self.active.fetch_sub(1, std::sync::atomic::Ordering::SeqCst);
return std::task::Poll::Ready(());
})
.await;
return planned_outcome(candidate, plan.disposition);
});
}
}
fn planned_outcome(candidate: &crate::BackfillCandidate, disposition: FakeDisposition) -> ksp_core_lib::Result<crate::BackfillPersistenceOutcome> {
if matches!(disposition, FakeDisposition::Failure) {
return std::result::Result::Err(ksp_core_lib::Error::new(crate::ERROR_CODE_BACKFILL_RAW_CONVERSION_INVALID, "planned candidate failure"));
}
let fill = candidate.identity().signature().as_str().as_bytes()[0];
let reference =
ksp_store_lib::RawTransactionReference::new(candidate.identity().network().clone(), ksp_store_lib::RawTransactionSignature::new([fill; 64]));
let (entity, observation) = match disposition {
FakeDisposition::Durable => (crate::BackfillEntityPersistence::AlreadyPresent, crate::BackfillObservationPersistence::AlreadyPresent),
FakeDisposition::Missing => (crate::BackfillEntityPersistence::Missing, crate::BackfillObservationPersistence::NotApplicable),
FakeDisposition::Conflict => (crate::BackfillEntityPersistence::Conflict, crate::BackfillObservationPersistence::NotRecorded),
FakeDisposition::Failure => return std::result::Result::Err(super::execution_error("test.disposition")),
};
return std::result::Result::Ok(crate::BackfillPersistenceOutcome::new(reference, entity, observation));
}
fn signature(character: char) -> std::option::Option<crate::BackfillSignature> {
return crate::BackfillSignature::new(character.to_string().repeat(crate::MIN_BACKFILL_SIGNATURE_TEXT_BYTES)).ok();
}
fn explicit_request(characters: &[char], concurrency: usize) -> std::option::Option<crate::BackfillRequest> {
let mut signatures = std::vec::Vec::with_capacity(characters.len());
for character in characters {
let value = match signature(*character) {
std::option::Option::Some(value) => value,
std::option::Option::None => return std::option::Option::None,
};
signatures.push(value);
}
let scope = match crate::BackfillScope::explicit_signatures(signatures) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return std::option::Option::None,
};
let job_id = match ksp_job_api::JobId::new("backfill:execution-test") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return std::option::Option::None,
};
let network = match ksp_store_lib::RawNetworkId::new("devnet") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return std::option::Option::None,
};
return crate::BackfillRequest::new(
job_id,
network,
ksp_onchain_transport_lib::HttpRoleName::new("history"),
crate::BackfillCommitment::Confirmed,
scope,
100,
10,
characters.len(),
concurrency,
std::option::Option::None,
)
.ok();
}
fn discovery(request: &crate::BackfillRequest, characters: &[char]) -> std::option::Option<crate::BackfillDiscovery> {
let mut candidates = std::vec::Vec::with_capacity(characters.len());
for character in characters {
let signature = match signature(*character) {
std::option::Option::Some(value) => value,
std::option::Option::None => return std::option::Option::None,
};
let identity = crate::BackfillCandidateIdentity::new(request.network().clone(), signature);
candidates.push(crate::BackfillCandidate::new(identity, std::option::Option::None));
}
return std::option::Option::Some(crate::BackfillDiscovery::new(
request.network().clone(),
request.scope_fingerprint(),
candidates,
0,
crate::BackfillDiscoveryBoundary::ExplicitInput,
));
}
#[tokio::test]
async fn pre_008_execution_is_bounded_and_reconciles_out_of_order_durable_completions() {
let request = match explicit_request(&['1', '2', '3', '4'], 2) {
std::option::Option::Some(value) => value,
std::option::Option::None => return,
};
let discovery = match discovery(&request, &['1', '2', '3', '4']) {
std::option::Option::Some(value) => value,
std::option::Option::None => return,
};
let processor = FakeProcessor::new(std::vec![
FakePlan { pending_polls: 6, disposition: FakeDisposition::Durable },
FakePlan { pending_polls: 0, disposition: FakeDisposition::Durable },
FakePlan { pending_polls: 2, disposition: FakeDisposition::Durable },
FakePlan { pending_polls: 0, disposition: FakeDisposition::Durable },
]);
let result = super::execute_with_processor(&processor, &request, &discovery).await;
let batch = match result {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return,
};
assert_eq!(batch.admitted_count(), 4);
assert_eq!(batch.finished_count(), 4);
assert_eq!(batch.durable_count(), 4);
assert_eq!(batch.hole_count(), 0);
assert_eq!(batch.maximum_in_flight(), 2);
assert_eq!(processor.maximum(), 2);
assert_eq!(batch.local_contiguous_completed(), 4);
assert_eq!(batch.checkpoint().completed_prefix(), 4);
assert!(!batch.is_partial());
return;
}
#[tokio::test]
async fn pre_008_missing_is_non_fatal_but_blocks_frontier_while_later_candidates_continue() {
let request = match explicit_request(&['1', '2', '3'], 2) {
std::option::Option::Some(value) => value,
std::option::Option::None => return,
};
let discovery = match discovery(&request, &['1', '2', '3']) {
std::option::Option::Some(value) => value,
std::option::Option::None => return,
};
let processor = FakeProcessor::new(std::vec![
FakePlan { pending_polls: 1, disposition: FakeDisposition::Durable },
FakePlan { pending_polls: 0, disposition: FakeDisposition::Missing },
FakePlan { pending_polls: 0, disposition: FakeDisposition::Durable },
]);
let batch = match super::execute_with_processor(&processor, &request, &discovery).await {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return,
};
assert_eq!(batch.admitted_count(), 3);
assert_eq!(batch.finished_count(), 3);
assert_eq!(batch.durable_count(), 2);
assert_eq!(batch.hole_count(), 1);
assert_eq!(batch.local_contiguous_completed(), 1);
assert_eq!(batch.checkpoint().completed_prefix(), 1);
assert_eq!(batch.failure_code(), std::option::Option::None);
assert!(batch.is_partial());
return;
}
#[tokio::test]
async fn pre_008_conflict_stops_new_admissions_and_drains_already_in_flight_work() {
let request = match explicit_request(&['1', '2', '3', '4'], 2) {
std::option::Option::Some(value) => value,
std::option::Option::None => return,
};
let discovery = match discovery(&request, &['1', '2', '3', '4']) {
std::option::Option::Some(value) => value,
std::option::Option::None => return,
};
let processor = FakeProcessor::new(std::vec![
FakePlan { pending_polls: 6, disposition: FakeDisposition::Durable },
FakePlan { pending_polls: 0, disposition: FakeDisposition::Conflict },
FakePlan { pending_polls: 0, disposition: FakeDisposition::Durable },
FakePlan { pending_polls: 0, disposition: FakeDisposition::Durable },
]);
let batch = match super::execute_with_processor(&processor, &request, &discovery).await {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return,
};
assert_eq!(processor.calls(), 2);
assert_eq!(batch.admitted_count(), 2);
assert_eq!(batch.finished_count(), 2);
assert_eq!(batch.durable_count(), 1);
assert_eq!(batch.hole_count(), 1);
assert_eq!(batch.local_contiguous_completed(), 1);
assert_eq!(batch.failure_code(), std::option::Option::Some(ksp_store_lib::ERROR_CODE_RAW_CONFLICT));
assert!(batch.is_partial());
return;
}
#[tokio::test]
async fn pre_008_explicit_resume_skips_only_the_checkpointed_contiguous_prefix() {
let request = match explicit_request(&['1', '2', '3', '4'], 2) {
std::option::Option::Some(value) => value,
std::option::Option::None => return,
};
let checkpoint = crate::BackfillCheckpoint::new(request.job_id().clone(), request.scope_fingerprint(), 2, std::option::Option::None);
let request = match request.with_checkpoint(checkpoint) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return,
};
let discovery = match discovery(&request, &['1', '2', '3', '4']) {
std::option::Option::Some(value) => value,
std::option::Option::None => return,
};
let processor = FakeProcessor::new(std::vec![
FakePlan { pending_polls: 0, disposition: FakeDisposition::Failure },
FakePlan { pending_polls: 0, disposition: FakeDisposition::Failure },
FakePlan { pending_polls: 0, disposition: FakeDisposition::Durable },
FakePlan { pending_polls: 0, disposition: FakeDisposition::Durable },
]);
let batch = match super::execute_with_processor(&processor, &request, &discovery).await {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return,
};
assert_eq!(processor.calls(), 2);
assert_eq!(batch.admitted_count(), 2);
assert_eq!(batch.local_contiguous_completed(), 4);
assert_eq!(batch.checkpoint().completed_prefix(), 4);
assert_eq!(batch.failure_code(), std::option::Option::None);
return;
}

306
deltas/0.3.6/pre.008.md Normal file
View File

@@ -0,0 +1,306 @@
<!-- file: deltas/0.3.6/pre.008.md -->
<!-- version: 1 -->
# Delta `0.3.6-pre.008` — concurrence bornée, frontier contiguë et checkpoint Backfill
## Base requise
```text
0.3.6-pre.007 appliquée
workspace.package.version = 0.3.6-pre.7
```
Le gate opérateur fourni pour cette base confirme :
```text
cargo fmt --all PASS
python3 scripts/audit_rust_workspace_rules.py PASS / clean
python3 scripts/audit_markdown_tables.py ... PASS / clean (264 tables / 152 fichiers)
cargo check --workspace PASS
cargo clippy --workspace --all-targets PASS / aucun warning
cargo test -p ksp-job-backfill-lib PASS
unitaires 29 PASS
dependency_boundary 2 PASS
public_api 4 PASS
release_completeness 2 PASS
cargo tree -p ksp-job-backfill-lib --edges normal exécuté
cargo tree -p ksp-job-backfill-lib -e features exécuté
```
## Objectif
Matérialiser exclusivement `pre.008` du plan 027 : exécuter les candidats découverts avec une concurrence métier bornée, réconcilier les fins hors ordre par une frontier contiguë et produire un checkpoint sûr pour reprise contrôlée.
La tranche n'ouvre pas encore l'annulation coopérative, le snapshot latest-value concret, les états terminaux complets ou les courses d'annulation de `pre.009`.
## Checkpoint caller-owned
La tranche ajoute le type public opaque :
```text
BackfillCheckpoint
```
Il contient uniquement :
```text
JobId
BackfillScopeFingerprint
completed_prefix
resume_before privé si scope BeforeAddress
```
Il ne contient ni payload RAW, ni URL, ni provider, ni endpoint, ni credential, ni liste de signatures.
`BackfillRequest::with_checkpoint` valide obligatoirement :
```text
checkpoint.job_id == request.job_id
checkpoint.scope_fingerprint == request.scope_fingerprint
```
Un checkpoint incompatible est rejeté par :
```text
ERROR_CODE_BACKFILL_CHECKPOINT_INVALID
```
Aucune table Store, migration ou persistance durable de Job n'est créée. Le checkpoint est caller-owned et qualifie uniquement une reprise contrôlée ; la v0.3.6 ne revendique pas une reprise crash-safe multi-processus.
## Sémantique de reprise par scope
### LatestAddress
La reprise repart volontairement sans curseur RPC depuis la vue latest courante. L'idempotence Store absorbe les candidats déjà durables. Le préfixe historique du checkpoint n'est jamais utilisé comme offset de la nouvelle vue latest.
### BeforeAddress
Le checkpoint conserve comme curseur exclusif le dernier candidat du préfixe contigu durable. Si aucun candidat nouveau n'est durable, le curseur précédent — ou l'ancre initiale — est conservé. Le compteur `completed_prefix` est cumulatif entre reprises contrôlées.
### AfterAddress
Le scope borné est rejoué et seul le préfixe contigu validé est ignoré. Si la découverte s'arrête sur :
```text
AfterAnchorNotReached
```
aucun progrès nouveau de checkpoint n'est publié, même si certaines hydrations observées ont fini durablement ; le checkpoint précédent est conservé lorsqu'il existe.
### ExplicitSignatures
La liste dédupliquée et fingerprintée est rejouée et seul son préfixe contigu déjà validé est ignoré.
## Frontier contiguë
Chaque candidat conserve son index stable de découverte. Une bitmap privée bornée à `max_candidates <= 10_000` suit les résultats durables.
La frontier avance uniquement pour :
```text
Inserted
AlreadyPresent
SkippedPurged
```
Les résultats suivants créent un trou et ne sont jamais marqués durables :
```text
Missing
Conflict
erreur Transport
conversion invalide
erreur Store
```
Une completion hors ordre ne saute jamais un index incomplet. Les tests reproduisent explicitement :
```text
index 1 durable -> frontier 0
index 0 durable -> frontier 2
index 3 durable -> frontier 2
index 2 durable -> frontier 4
```
## Exécution concurrente bornée
La tranche ajoute :
```text
execute_backfill_discovery
BackfillExecutionBatch
```
L'implémentation utilise un `FuturesUnordered` privé et ne garde jamais plus de :
```text
request.hydration_concurrency()
```
futures candidates en vol.
Aucun `tokio::spawn`, timer Job, retry Job, pacing Job ou choix d'endpoint n'est introduit. Transport reste propriétaire de sa politique réseau et Store de sa persistance.
Le batch expose uniquement des faits bornés utiles à la preuve de frontier : candidats, admis, terminés, durables, trous, maximum en vol, préfixe contigu local, checkpoint et premier code d'échec fatal.
## Missing, conflit et erreurs
`Missing` est un trou non fatal : les admissions suivantes continuent, mais la frontier reste bloquée tant que ce trou existe.
Un `Conflict` ou toute erreur Transport/conversion/Store finale :
1. devient le premier `failure_code` stable du batch ;
2. arrête immédiatement les nouvelles admissions ;
3. laisse finir toutes les futures déjà admises ;
4. produit ensuite le checkpoint correspondant au dernier préfixe réellement durable.
Cette phase ne publie pas encore l'état Job terminal `Failed`/`Partial`; cette projection appartient au snapshot/runtime de `pre.009`.
## Horodatage d'acquisition
Le chemin public d'exécution produit le `RawTimestamp` de réception à partir de `SystemTime` immédiatement avant l'hydratation. Une horloge avant epoch ou un timestamp non représentable devient une erreur structurée :
```text
ERROR_CODE_BACKFILL_EXECUTION_INVALID
```
Aucun clock/provider type n'entre dans l'API publique.
## Dépendances
La crate ajoute comme dépendance normale :
```text
futures-util = { workspace = true, features = ["std"] }
```
Cette dépendance existe déjà dans `[workspace.dependencies]`; aucune nouvelle version externe n'est introduite.
Tokio reste exclusivement en `dev-dependencies` pour les tests async et aucun type Tokio/Futures n'est exposé par la crate root.
## Tests matérialisés
La tranche ajoute **10 tests unitaires**, portant le total à **39 unitaires** :
- frontier hors ordre strictement contiguë ;
- checkpoint rejeté sur mauvais `JobId` ou fingerprint ;
- checkpoint Before cumulatif avec curseur du dernier candidat contigu ;
- After sans ancre atteinte n'avance jamais le checkpoint ;
- Before reprend au curseur checkpoint plutôt qu'à l'ancre initiale ;
- Latest reprend sans curseur depuis la vue courante ;
- concurrence observée strictement bornée malgré fins hors ordre ;
- Missing non fatal mais bloquant pour la frontier ;
- conflit arrêtant les admissions et drainant le travail déjà en vol ;
- reprise Explicit sautant uniquement le préfixe checkpointé.
Une canarie publique supplémentaire porte les canaries d'intégration à **9** :
```text
dependency_boundary 2
public_api 5
release_completeness 2
```
Les canaries verrouillent également l'inventaire exact des modules, `futures-util` comme détail privé normal, l'absence de Tokio normal, l'absence de retry/pacing/endpoint selection Job et la fermeture des surfaces `BackfillSnapshot`/`BackfillJobHandle` de `pre.009`.
## Fichiers ajoutés
```text
crates/ksp-job-backfill-lib/src/checkpoint.rs
crates/ksp-job-backfill-lib/src/execution.rs
crates/ksp-job-backfill-lib/unit_tests/checkpoint.rs
crates/ksp-job-backfill-lib/unit_tests/execution.rs
deltas/0.3.6/pre.008.md
```
## Fichiers modifiés
```text
Cargo.toml
crates/ksp-job-backfill-lib/Cargo.toml
crates/ksp-job-backfill-lib/src/discovery.rs
crates/ksp-job-backfill-lib/src/error.rs
crates/ksp-job-backfill-lib/src/lib.rs
crates/ksp-job-backfill-lib/src/persistence.rs
crates/ksp-job-backfill-lib/src/request.rs
crates/ksp-job-backfill-lib/tests/dependency_boundary.rs
crates/ksp-job-backfill-lib/tests/public_api.rs
crates/ksp-job-backfill-lib/tests/release_completeness.rs
crates/ksp-job-backfill-lib/unit_tests/discovery.rs
docs/plans/027-V0_3_6_JOB_API_BACKFILL_PLAN.md
docs/validation/023-V0_3_6_JOB_API_BACKFILL.md
```
## Versions d'en-tête
Les fichiers Rust/TOML modifiés incrémentent tous leur version d'en-tête exactement d'une unité :
```text
Cargo.toml 405 -> 406
crates/ksp-job-backfill-lib/Cargo.toml 2 -> 3
crates/ksp-job-backfill-lib/src/discovery.rs 2 -> 3
crates/ksp-job-backfill-lib/src/error.rs 3 -> 4
crates/ksp-job-backfill-lib/src/lib.rs 3 -> 4
crates/ksp-job-backfill-lib/src/persistence.rs 1 -> 2
crates/ksp-job-backfill-lib/src/request.rs 2 -> 3
crates/ksp-job-backfill-lib/tests/dependency_boundary.rs 3 -> 4
crates/ksp-job-backfill-lib/tests/public_api.rs 3 -> 4
crates/ksp-job-backfill-lib/tests/release_completeness.rs 3 -> 4
crates/ksp-job-backfill-lib/unit_tests/discovery.rs 1 -> 2
```
Les quatre nouveaux fichiers Rust commencent à `version: 1`. Le plan 027 et la validation 023 passent de `13` à `14`. Le présent delta commence à `version: 1`.
## Version workspace
La tranche modifie du Rust :
```text
workspace.package.version = 0.3.6-pre.8
delivery = 0.3.6-pre.008
commit = v0.3.6-pre.008
```
Aucun tag prerelease.
## Fichiers supprimés
Aucun.
## Frontières conservées
- aucune persistence durable de checkpoint dans Store ;
- aucune table ou migration Job ;
- aucun retry/pacing/timer Job ;
- aucune sélection provider/endpoint Job ;
- aucun `ForceRehydrate` ;
- aucun `tokio::spawn` ;
- aucun type Tokio/Futures public ;
- aucune annulation coopérative de `pre.009` ;
- aucun `BackfillSnapshot` latest-value concret ;
- aucune application ou Worker ;
- aucun changement README/USAGE/CHANGELOG/ROADMAP/prompt.
## Validations exécutées dans l'environnement d'assemblage
```text
python3 scripts/audit_rust_workspace_rules.py
python3 scripts/audit_markdown_tables.py README.md RULES.md ROADMAP.md CHANGELOG.md docs prompts crates deltas/0.3.6
```
L'environnement d'assemblage ne fournit ni `cargo`, ni `rustc`, ni `rustfmt`; aucun résultat Rust de `pre.008` n'est donc revendiqué localement.
## Gate opérateur demandé
```bash
cargo fmt --all
python3 scripts/audit_rust_workspace_rules.py
python3 scripts/audit_markdown_tables.py README.md RULES.md ROADMAP.md CHANGELOG.md docs prompts crates deltas/0.3.6
cargo check --workspace
cargo clippy --workspace --all-targets
cargo test -p ksp-job-backfill-lib
cargo tree -p ksp-job-backfill-lib --edges normal
cargo tree -p ksp-job-backfill-lib -e features
```
Résultat attendu après gate : **39 unitaires + 9 canaries**, sans warning. Un échec reste dans la lane `pre.008-fix.001`; un gate vert ouvre `pre.009`.

View File

@@ -1,5 +1,5 @@
<!-- file: docs/plans/027-V0_3_6_JOB_API_BACKFILL_PLAN.md --> <!-- file: docs/plans/027-V0_3_6_JOB_API_BACKFILL_PLAN.md -->
<!-- version: 13 --> <!-- version: 14 -->
# Plan v0.3.6 — Job API et premier backfill RAW # Plan v0.3.6 — Job API et premier backfill RAW
@@ -472,19 +472,23 @@ Le gate opérateur de `pre.006-fix.001` confirme les audits, `cargo check`, les
### `pre.007` — Persistance Store et idempotence ### `pre.007` — Persistance Store et idempotence
**Statut : matérialisé ; gate opérateur à rejouer.** **Statut : alisé ; gate opérateur vert.**
Budget cible : **15-20 min**. Entrée : RAW v1 figé et gate `pre.006-fix.002` vert. La tranche ajoute `persist_backfill_hydration` et une projection Backfill explicite des outcomes Store sans créer de second contrat de persistance. Une acquisition disponible est consommée par l'unique capacité atomique `RawTransactionWrite::persist_raw_transaction_acquisition` via `ksp-store-lib`, toujours en `RawTransactionAcquisitionMode::Normal`; aucune prélecture Store, écriture d'observation séparée ou `ForceRehydrate` n'est introduite. Budget cible : **15-20 min**. Entrée : RAW v1 figé et gate `pre.006-fix.002` vert. La tranche ajoute `persist_backfill_hydration` et une projection Backfill explicite des outcomes Store sans créer de second contrat de persistance. Une acquisition disponible est consommée par l'unique capacité atomique `RawTransactionWrite::persist_raw_transaction_acquisition` via `ksp-store-lib`, toujours en `RawTransactionAcquisitionMode::Normal`; aucune prélecture Store, écriture d'observation séparée ou `ForceRehydrate` n'est introduite.
Le réseau du Store est comparé à l'identité `(RawNetworkId, signature)` avant toute écriture. `Missing` n'effectue aucune écriture. Les outcomes Store valides sont projetés en dispositions entité/observation distinctes : `Inserted/Inserted`, `AlreadyPresent/Inserted`, `AlreadyPresent/AlreadyPresent` et `SkippedPurged/NotRecorded`. `ERROR_CODE_RAW_CONFLICT` devient un outcome Backfill `Conflict/NotRecorded`, distinct d'un succès idempotent ; les autres erreurs Store restent des erreurs. Les combinaisons impossibles en mode normal, dont `Rehydrated`, sont rejetées par `ERROR_CODE_BACKFILL_PERSISTENCE_INVALID`. Le réseau du Store est comparé à l'identité `(RawNetworkId, signature)` avant toute écriture. `Missing` n'effectue aucune écriture. Les outcomes Store valides sont projetés en dispositions entité/observation distinctes : `Inserted/Inserted`, `AlreadyPresent/Inserted`, `AlreadyPresent/AlreadyPresent` et `SkippedPurged/NotRecorded`. `ERROR_CODE_RAW_CONFLICT` devient un outcome Backfill `Conflict/NotRecorded`, distinct d'un succès idempotent ; les autres erreurs Store restent des erreurs. Les combinaisons impossibles en mode normal, dont `Rehydrated`, sont rejetées par `ERROR_CODE_BACKFILL_PERSISTENCE_INVALID`.
La tranche matérialise 29 tests unitaires et 8 canaries d'intégration au total. Les doubles Store restent privés et déterministes ; ils couvrent insert, nouvelle observation sur entité existante, relance idempotente, purge normale, conflit, erreur Store, mismatch réseau et incohérence de référence. La frontière contiguë, la concurrence et le checkpoint restent fermés jusqu'à `pre.008`. La tranche matérialise 29 tests unitaires et 8 canaries d'intégration au total. Les doubles Store restent privés et déterministes ; ils couvrent insert, nouvelle observation sur entité existante, relance idempotente, purge normale, conflit, erreur Store, mismatch réseau et incohérence de référence. Le gate opérateur est intégralement vert : audits Rust/Markdown, `cargo check`, Clippy sans warning, 29 unitaires, 8 canaries et arbres Cargo passent sur `0.3.6-pre.7`. La frontière contiguë, la concurrence et le checkpoint restent fermés jusqu'à `pre.008`.
### `pre.008` — Concurrence, frontier et checkpoint ### `pre.008` — Concurrence, frontier et checkpoint
**Statut : planifié.** **Statut : matérialisé ; gate opérateur à rejouer.**
Budget cible : **15-20 min**. Entrée : résultats Store typés. Ajouter admissions bornées, réconciliation hors ordre et reprise. Sortie : frontière contiguë, trous, bornes et checkpoints des quatre scopes prouvés. Budget cible : **15-20 min**. Entrée : résultats Store typés et gate `pre.007` vert. La tranche ajoute `BackfillCheckpoint`, `BackfillExecutionBatch` et `execute_backfill_discovery`. Le checkpoint est opaque, caller-owned et validé contre `JobId` + fingerprint de scope ; aucune table ou persistance durable de Job n'est créée. `BackfillRequest::with_checkpoint` reste additif afin de préserver les constructeurs existants.
L'exécution utilise un `FuturesUnordered` privé et maintient au plus `hydration_concurrency` candidats actifs, sans `tokio::spawn`, temporisation Job, retry Job ni file non bornée. Les résultats Store durables `Inserted`, `AlreadyPresent` et `SkippedPurged` avancent une bitmap bornée de complétion ; `Missing`, conflit ou erreur laissent un trou. Un conflit ou toute erreur Transport/conversion/Store arrête immédiatement les nouvelles admissions, puis le batch draine les futures déjà admises avant de publier son checkpoint.
La reprise suit les quatre sémantiques figées : Latest repart de la vue courante et s'appuie sur l'idempotence Store ; Before reprend via le dernier curseur contigu et cumule le préfixe durable ; After et Explicit rejouent le scope et sautent uniquement le préfixe validé. Si une découverte After finit sur `AfterAnchorNotReached`, aucun progrès nouveau de checkpoint n'est publié ; le checkpoint précédent est conservé s'il existe. La tranche matérialise 39 tests unitaires et 9 canaries d'intégration. L'annulation coopérative, le snapshot latest-value complet et les courses terminales restent fermés jusqu'à `pre.009`.
### `pre.009` — Annulation et snapshots concrets ### `pre.009` — Annulation et snapshots concrets

View File

@@ -1,5 +1,5 @@
<!-- file: docs/validation/023-V0_3_6_JOB_API_BACKFILL.md --> <!-- file: docs/validation/023-V0_3_6_JOB_API_BACKFILL.md -->
<!-- version: 13 --> <!-- version: 14 -->
# Validation v0.3.6 — Job API et premier backfill RAW # Validation v0.3.6 — Job API et premier backfill RAW
@@ -113,7 +113,7 @@ Aucune entrée absolue, traversée, avec séparateur inversé ou lien symbolique
- [X] Nombre de candidats entre 1 et 10 000. - [X] Nombre de candidats entre 1 et 10 000.
- [X] Concurrence d'hydratation entre 1 et 64. - [X] Concurrence d'hydratation entre 1 et 64.
- [X] `min_context_slot` transmis sans en changer le sens. - [X] `min_context_slot` transmis sans en changer le sens.
- [ ] Checkpoint rejeté si son JobId ou son fingerprint de scope diffère. - [ ] Checkpoint rejeté si son JobId ou son fingerprint de scope diffère ; test `pre.008` matérialisé, gate opérateur à rejouer.
- [X] Aucun défaut d'application silencieux dans la bibliothèque. - [X] Aucun défaut d'application silencieux dans la bibliothèque.
## 8. Pagination, ordre et déduplication ## 8. Pagination, ordre et déduplication
@@ -129,13 +129,13 @@ Aucune entrée absolue, traversée, avec séparateur inversé ou lien symbolique
## 9. Acquisition Transport et retry ## 9. Acquisition Transport et retry
- [ ] Les wrappers typés officiels restent l'unique chemin RPC. - [X] Les wrappers typés officiels restent l'unique chemin RPC, confirmé par audit de source jusqu'à `pre.008`.
- [X] Retour observé additif de `getTransaction` matérialisé et gate opérateur vert. - [X] Retour observé additif de `getTransaction` matérialisé et gate opérateur vert.
- [X] Canarie déterministe : un `429` sur le premier endpoint reroute et rapporte le second provider/endpoint victorieux ; gate opérateur vert. - [X] Canarie déterministe : un `429` sur le premier endpoint reroute et rapporte le second provider/endpoint victorieux ; gate opérateur vert.
- [X] `HttpObservedValue<T>` ne contient que valeur typée, endpoint sûr et provider ; URL, headers et body HTTP brut restent privés. - [X] `HttpObservedValue<T>` ne contient que valeur typée, endpoint sûr et provider ; URL, headers et body HTTP brut restent privés.
- [ ] Aucun retry, pacing ou sélection d'endpoint dans Job. - [X] Aucun retry, pacing ou sélection d'endpoint dans Job, confirmé par audit de source `pre.008`.
- [ ] Aucune détection d'erreur par chaîne. - [X] Aucune détection d'erreur par chaîne dans la composition Backfill `pre.008`.
- [ ] Une erreur Transport finale arrête les admissions avec un code Job sûr. - [ ] Une erreur Transport finale arrête les admissions avec un code Job sûr ; exécuteur et test `pre.008` matérialisés, gate opérateur à rejouer.
## 10. RAW v1 et provenance ## 10. RAW v1 et provenance
@@ -161,20 +161,22 @@ Aucune entrée absolue, traversée, avec séparateur inversé ou lien symbolique
## 11. Persistance Store ## 11. Persistance Store
`pre.007` matérialise la composition Store normale sans élargir les responsabilités. Le chemin public consomme un `BackfillHydrationOutcome`, contrôle d'abord le réseau logique du Store, puis utilise exclusivement l'écriture atomique transaction + observation de `ksp-store-lib`. `Missing` ne déclenche aucune écriture. Un conflit de contenu Store est converti en outcome Backfill explicite afin que les tranches de frontier/snapshot puissent le compter puis le traiter comme trou/échec, sans le confondre avec l'idempotence. Les tests Rust de cette tranche sont matérialisés mais restent à exécuter dans le gate opérateur. `pre.007` matérialise la composition Store normale sans élargir les responsabilités. Le chemin public consomme un `BackfillHydrationOutcome`, contrôle d'abord le réseau logique du Store, puis utilise exclusivement l'écriture atomique transaction + observation de `ksp-store-lib`. `Missing` ne déclenche aucune écriture. Un conflit de contenu Store est converti en outcome Backfill explicite afin que les tranches de frontier/snapshot puissent le compter puis le traiter comme trou/échec, sans le confondre avec l'idempotence. Le gate opérateur `pre.007` est intégralement vert : audits, `cargo check`, Clippy sans warning, 29 unitaires, 8 canaries et arbres Cargo passent.
- [X] Dépendance unique vers `ksp-store-lib`, jamais vers backend ou API directement, confirmée par audit de manifeste/source. - [X] Dépendance unique vers `ksp-store-lib`, jamais vers backend ou API directement, confirmée par audit de manifeste/source.
- [X] Écriture atomique transaction plus observation composée en mode `Normal` dans l'unique chemin de persistance. - [X] Écriture atomique transaction plus observation composée en mode `Normal` dans l'unique chemin de persistance.
- [X] Aucune prélecture de présence avant hydratation ou persistance. - [X] Aucune prélecture de présence avant hydratation ou persistance.
- [ ] Entités insérée, déjà présente et purgée distinguées au gate Rust `pre.007`. - [X] Entités insérée, déjà présente et purgée distinguées au gate Rust `pre.007`.
- [ ] Observations insérée, déjà présente et non enregistrée distinguées au gate Rust `pre.007`. - [X] Observations insérée, déjà présente et non enregistrée distinguées au gate Rust `pre.007`.
- [ ] `getTransaction = null` ne produit aucune écriture fabriquée au gate Rust `pre.007`. - [X] `getTransaction = null` ne produit aucune écriture fabriquée au gate Rust `pre.007`.
- [ ] Conflit Store reste un outcome terminal visible, distinct d'un succès idempotent, au gate Rust `pre.007`. - [X] Conflit Store reste un outcome terminal visible, distinct d'un succès idempotent, au gate Rust `pre.007`.
- [ ] Même JobId et même endpoint donnent une relance idempotente au gate Rust `pre.007`. - [X] Même JobId et même endpoint donnent une relance idempotente au gate Rust `pre.007`.
- [X] `ForceRehydrate` absent du chemin normal `pre.007` par audit de source. - [X] `ForceRehydrate` absent du chemin normal `pre.007` par audit de source.
## 12. Concurrence, frontière et checkpoint ## 12. Concurrence, frontière et checkpoint
`pre.008` matérialise un checkpoint opaque caller-owned, sans table Store ni promesse de reprise crash-safe. `BackfillRequest::with_checkpoint` valide `JobId` et fingerprint avant usage. L'exécuteur utilise une collection `FuturesUnordered` strictement bornée par `hydration_concurrency`, réconcilie les fins hors ordre par index stable et draine les futures déjà admises après la première erreur fatale. Les tests couvrent frontier contiguë, Missing, conflit, reprise Before/Latest/Explicit et protection After lorsque l'ancre n'est pas atteinte ; ils restent à exécuter dans le gate opérateur.
- [ ] Nombre d'hydratations en vol borné par la requête. - [ ] Nombre d'hydratations en vol borné par la requête.
- [ ] Aucune file de candidats ou notifications non bornée. - [ ] Aucune file de candidats ou notifications non bornée.
- [ ] Résultats hors ordre réconciliés par index stable. - [ ] Résultats hors ordre réconciliés par index stable.
@@ -202,10 +204,10 @@ Aucune entrée absolue, traversée, avec séparateur inversé ou lien symbolique
- [X] `ksp-job-api` dépend exactement de Core et n'a aucune feature, dev-dependency ou build-dependency. - [X] `ksp-job-api` dépend exactement de Core et n'a aucune feature, dev-dependency ou build-dependency.
- [X] `ksp-job-backfill-lib` ne dépend pas de Config, App, Wallet, Program, Interface ou backend Store ; `ksp-store-lib` est consommée avec `default-features = false`. - [X] `ksp-job-backfill-lib` ne dépend pas de Config, App, Wallet, Program, Interface ou backend Store ; `ksp-store-lib` est consommée avec `default-features = false`.
- [ ] Aucun retour de dépendance vers Job depuis les couches basses. - [ ] Aucun retour de dépendance vers Job depuis les couches basses.
- [X] Versions/features auditées : aucune nouvelle version externe ; `sha2` et `tokio` réutilisent les entrées workspace existantes, Tokio étant dev-only en `pre.005`. - [X] Versions/features auditées : aucune nouvelle version externe ; `sha2`, `futures-util` et `tokio` réutilisent les entrées workspace existantes. `futures-util` devient un détail normal privé de `pre.008`, Tokio reste dev-only.
- [X] `pre.005` matérialise 11 tests unitaires et 6 canaries dintégration ; le gate opérateur les exécute tous avec succès. - [X] `pre.005` matérialise 11 tests unitaires et 6 canaries dintégration ; le gate opérateur les exécute tous avec succès.
- [X] `pre.006` matérialise 19 tests unitaires et 7 canaries dintégration ; le gate opérateur les exécute tous avec succès, avec uniquement le warning Clippy corrigé par `pre.006-fix.001`. - [X] `pre.006` matérialise 19 tests unitaires et 7 canaries dintégration ; le gate opérateur les exécute tous avec succès, avec uniquement le warning Clippy corrigé par `pre.006-fix.001`.
- [X] `cargo clippy --workspace --all-targets` warning-free sur `pre.006-fix.002`. - [X] `cargo clippy --workspace --all-targets` warning-free sur `pre.007` ; le gate `pre.008` reste à rejouer.
- [ ] `cargo fmt --all -- --check` vert. - [ ] `cargo fmt --all -- --check` vert.
- [X] `scripts/audit_rust_workspace_rules.py` vert sur `pre.005` dans l'environnement d'assemblage et dans le gate opérateur. - [X] `scripts/audit_rust_workspace_rules.py` vert sur `pre.005` dans l'environnement d'assemblage et dans le gate opérateur.
- [X] `scripts/audit_markdown_tables.py` vert sur `pre.005` dans le gate opérateur (264 tables / 147 fichiers, delta inclus). - [X] `scripts/audit_markdown_tables.py` vert sur `pre.005` dans le gate opérateur (264 tables / 147 fichiers, delta inclus).
@@ -217,7 +219,7 @@ Aucune entrée absolue, traversée, avec séparateur inversé ou lien symbolique
## 15. Preuves d'intégration et fermeture ## 15. Preuves d'intégration et fermeture
- [X] Fake Transport et fake Store déterministes sans backend direct matérialisés ; exécution du fake Store `pre.007` à confirmer au gate opérateur. - [X] Fake Transport et fake Store déterministes sans backend direct exécutés avec succès au gate `pre.007`; fake processor concurrent `pre.008` matérialisé, gate à rejouer.
- [ ] Vertical découverte, hydratation, conversion, persistance et observation couvert. - [ ] Vertical découverte, hydratation, conversion, persistance et observation couvert.
- [ ] Smoke Devnet plus PostgreSQL configuré exécuté si l'environnement explicite est disponible. - [ ] Smoke Devnet plus PostgreSQL configuré exécuté si l'environnement explicite est disponible.
- [ ] Aucun endpoint payant, credential ou donnée sensible requis par les tests normaux. - [ ] Aucun endpoint payant, credential ou donnée sensible requis par les tests normaux.