From 4fdeab4d6792a90afd4114c42424614a62b3fc2e Mon Sep 17 00:00:00 2001 From: SinuS Von SifriduS Date: Tue, 1 Sep 2026 13:00:21 +0200 Subject: [PATCH] v0.3.6-pre.005 --- Cargo.toml | 6 +- crates/ksp-job-backfill-lib/Cargo.toml | 22 + crates/ksp-job-backfill-lib/src/constants.rs | 7 + crates/ksp-job-backfill-lib/src/discovery.rs | 443 +++++++++++++++++ crates/ksp-job-backfill-lib/src/error.rs | 11 + crates/ksp-job-backfill-lib/src/lib.rs | 64 +++ crates/ksp-job-backfill-lib/src/request.rs | 466 ++++++++++++++++++ .../tests/dependency_boundary.rs | 66 +++ .../ksp-job-backfill-lib/tests/public_api.rs | 66 +++ .../tests/release_completeness.rs | 61 +++ .../unit_tests/discovery.rs | 311 ++++++++++++ .../unit_tests/request.rs | 244 +++++++++ deltas/0.3.6/pre.005.md | 286 +++++++++++ docs/architecture/004-COMPONENT_INVENTORY.md | 76 +-- docs/architecture/005-DEPENDENCY_GRAPH.md | 14 +- .../009-ACQUISITION_WORKERS_AND_JOBS.md | 14 +- .../plans/027-V0_3_6_JOB_API_BACKFILL_PLAN.md | 20 +- .../validation/023-V0_3_6_JOB_API_BACKFILL.md | 25 +- 18 files changed, 2132 insertions(+), 70 deletions(-) create mode 100644 crates/ksp-job-backfill-lib/Cargo.toml create mode 100644 crates/ksp-job-backfill-lib/src/constants.rs create mode 100644 crates/ksp-job-backfill-lib/src/discovery.rs create mode 100644 crates/ksp-job-backfill-lib/src/error.rs create mode 100644 crates/ksp-job-backfill-lib/src/lib.rs create mode 100644 crates/ksp-job-backfill-lib/src/request.rs create mode 100644 crates/ksp-job-backfill-lib/tests/dependency_boundary.rs create mode 100644 crates/ksp-job-backfill-lib/tests/public_api.rs create mode 100644 crates/ksp-job-backfill-lib/tests/release_completeness.rs create mode 100644 crates/ksp-job-backfill-lib/unit_tests/discovery.rs create mode 100644 crates/ksp-job-backfill-lib/unit_tests/request.rs create mode 100644 deltas/0.3.6/pre.005.md diff --git a/Cargo.toml b/Cargo.toml index 1b2e502..6906bc2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,12 +1,12 @@ # file: Cargo.toml -# version: 399 +# version: 400 [workspace] resolver = "3" -members = ["crates/ksp-app-config-desk", "crates/ksp-app-solprices-desk", "crates/ksp-app-wallet-desk", "crates/ksp-config-lib", "crates/ksp-core-lib", "crates/ksp-interface-lib", "crates/ksp-job-api", "crates/ksp-logging-lib", "crates/ksp-offchain-transport-lib", "crates/ksp-onchain-transport-lib", "crates/ksp-program-api", "crates/ksp-store-api", "crates/ksp-store-lib", "crates/ksp-store-postgres-lib", "crates/ksp-wallet-lib"] +members = ["crates/ksp-app-config-desk", "crates/ksp-app-solprices-desk", "crates/ksp-app-wallet-desk", "crates/ksp-config-lib", "crates/ksp-core-lib", "crates/ksp-interface-lib", "crates/ksp-job-api", "crates/ksp-job-backfill-lib", "crates/ksp-logging-lib", "crates/ksp-offchain-transport-lib", "crates/ksp-onchain-transport-lib", "crates/ksp-program-api", "crates/ksp-store-api", "crates/ksp-store-lib", "crates/ksp-store-postgres-lib", "crates/ksp-wallet-lib"] [workspace.package] -version = "0.3.6-pre.4" +version = "0.3.6-pre.5" edition = "2024" license = "MIT" repository = "https://git.sasedev.com/Sasedev/khadhroony-solana-project" diff --git a/crates/ksp-job-backfill-lib/Cargo.toml b/crates/ksp-job-backfill-lib/Cargo.toml new file mode 100644 index 0000000..0bf602e --- /dev/null +++ b/crates/ksp-job-backfill-lib/Cargo.toml @@ -0,0 +1,22 @@ +# file: crates/ksp-job-backfill-lib/Cargo.toml +# version: 1 + +[package] +name = "ksp-job-backfill-lib" +version.workspace = true +edition.workspace = true +repository.workspace = true + +[dependencies] +ksp-core-lib = { path = "../ksp-core-lib" } +ksp-job-api = { path = "../ksp-job-api" } +ksp-logging-lib = { path = "../ksp-logging-lib" } +ksp-onchain-transport-lib = { path = "../ksp-onchain-transport-lib" } +ksp-store-lib = { path = "../ksp-store-lib", default-features = false } +sha2.workspace = true + +[dev-dependencies] +tokio = { workspace = true, features = ["macros", "rt-multi-thread"] } + +[lints] +workspace = true diff --git a/crates/ksp-job-backfill-lib/src/constants.rs b/crates/ksp-job-backfill-lib/src/constants.rs new file mode 100644 index 0000000..9f5e5dd --- /dev/null +++ b/crates/ksp-job-backfill-lib/src/constants.rs @@ -0,0 +1,7 @@ +// file: crates/ksp-job-backfill-lib/src/constants.rs +// version: 1 + +//! Backfill runtime-owned constants. + +/// Owning tracing target for the concrete bounded RAW backfill runtime. +pub(crate) const TRACING_TARGET: &str = "ksp-job-backfill-lib"; diff --git a/crates/ksp-job-backfill-lib/src/discovery.rs b/crates/ksp-job-backfill-lib/src/discovery.rs new file mode 100644 index 0000000..dfd872e --- /dev/null +++ b/crates/ksp-job-backfill-lib/src/discovery.rs @@ -0,0 +1,443 @@ +// file: crates/ksp-job-backfill-lib/src/discovery.rs +// version: 1 + +/// Network-scoped identity of one discovered transaction candidate before canonical signature decoding. +#[derive(Clone, Debug, Eq, Hash, PartialEq)] +pub struct BackfillCandidateIdentity { + network: ksp_store_lib::RawNetworkId, + signature: crate::BackfillSignature, +} + +impl BackfillCandidateIdentity { + /// Creates one candidate identity from its logical network and encoded transaction signature. + #[must_use] + pub fn new(network: ksp_store_lib::RawNetworkId, signature: crate::BackfillSignature) -> Self { + return Self { network, signature }; + } + + /// Returns the logical network that scopes signature uniqueness. + #[must_use] + pub const fn network(&self) -> &ksp_store_lib::RawNetworkId { + return &self.network; + } + + /// Returns the validated encoded transaction signature. + #[must_use] + pub const fn signature(&self) -> &crate::BackfillSignature { + return &self.signature; + } +} + +/// One deterministic transaction candidate produced by bounded discovery. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct BackfillCandidate { + identity: BackfillCandidateIdentity, + discovered_slot: std::option::Option, +} + +impl BackfillCandidate { + /// Creates one candidate from a network-scoped identity and optional discovery slot. + #[must_use] + pub fn new(identity: BackfillCandidateIdentity, discovered_slot: std::option::Option) -> Self { + return Self { identity, discovered_slot }; + } + + /// Returns the network-scoped candidate identity. + #[must_use] + pub const fn identity(&self) -> &BackfillCandidateIdentity { + return &self.identity; + } + + /// Returns the slot reported by address discovery, when one was available. + #[must_use] + pub const fn discovered_slot(&self) -> std::option::Option { + return self.discovered_slot; + } +} + +/// Reason bounded candidate discovery stopped. +#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)] +pub enum BackfillDiscoveryBoundary { + /// Explicit signatures required no RPC pagination. + ExplicitInput, + /// The caller's requested candidate limit was satisfied. + CandidateLimit, + /// The RPC returned a short or empty page, reaching the bounded remote history boundary. + RpcBoundary, + /// The maximum page count was exhausted before the requested address window completed. + PageLimit, + /// An `AfterAddress` scan exhausted its page bound before the exclusive anchor boundary was reached. + AfterAnchorNotReached, +} + +impl BackfillDiscoveryBoundary { + /// Returns the stable diagnostic code for this boundary. + #[must_use] + pub const fn code(self) -> &'static str { + return match self { + Self::ExplicitInput => "explicit_input", + Self::CandidateLimit => "candidate_limit", + Self::RpcBoundary => "rpc_boundary", + Self::PageLimit => "page_limit", + Self::AfterAnchorNotReached => "after_anchor_not_reached", + }; + } + + /// Returns whether the discovery result is partial and must not advance a future checkpoint beyond the unresolved gap. + #[must_use] + pub const fn is_partial(self) -> bool { + return matches!(self, Self::PageLimit | Self::AfterAnchorNotReached); + } +} + +/// Complete bounded output of one candidate discovery pass. +pub struct BackfillDiscovery { + network: ksp_store_lib::RawNetworkId, + scope_fingerprint: crate::BackfillScopeFingerprint, + candidates: std::vec::Vec, + pages_fetched: usize, + boundary: BackfillDiscoveryBoundary, +} + +impl BackfillDiscovery { + /// Returns the logical network shared by every candidate identity. + #[must_use] + pub const fn network(&self) -> &ksp_store_lib::RawNetworkId { + return &self.network; + } + + /// Returns the semantic scope fingerprint copied from the validated request. + #[must_use] + pub const fn scope_fingerprint(&self) -> crate::BackfillScopeFingerprint { + return self.scope_fingerprint; + } + + /// Returns discovered candidates in deterministic processing order. + #[must_use] + pub fn candidates(&self) -> &[BackfillCandidate] { + return self.candidates.as_slice(); + } + + /// Returns the number of `getSignaturesForAddress` pages fetched by this pass. + #[must_use] + pub const fn pages_fetched(&self) -> usize { + return self.pages_fetched; + } + + /// Returns the reason discovery stopped. + #[must_use] + pub const fn boundary(&self) -> BackfillDiscoveryBoundary { + return self.boundary; + } + + /// Returns whether the bounded pass stopped before a complete requested discovery window was proven. + #[must_use] + pub const fn is_partial(&self) -> bool { + return self.boundary.is_partial(); + } +} + +impl std::fmt::Debug for BackfillDiscovery { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + return formatter + .debug_struct("BackfillDiscovery") + .field("network", &self.network) + .field("scope_fingerprint", &self.scope_fingerprint) + .field("candidate_count", &self.candidates.len()) + .field("pages_fetched", &self.pages_fetched) + .field("boundary", &self.boundary) + .finish(); + } +} + +struct SignaturePageEntry { + signature: std::string::String, + slot: u64, +} + +type SignaturePageFuture<'a> = std::pin::Pin>> + 'a>>; + +trait SignaturePageSource { + fn fetch_signature_page<'a>( + &'a self, + role: &'a ksp_onchain_transport_lib::HttpRoleName, + address: &'a ksp_core_lib::Pubkey, + config: ksp_onchain_transport_lib::SolanaSignaturesForAddressConfig, + ) -> SignaturePageFuture<'a>; +} + +impl SignaturePageSource for ksp_onchain_transport_lib::HttpTransportPool { + fn fetch_signature_page<'a>( + &'a self, + role: &'a ksp_onchain_transport_lib::HttpRoleName, + address: &'a ksp_core_lib::Pubkey, + config: ksp_onchain_transport_lib::SolanaSignaturesForAddressConfig, + ) -> SignaturePageFuture<'a> { + return std::boxed::Box::pin(async move { + let result = self.get_signatures_for_address(role, address, std::option::Option::Some(&config)).await; + let infos = match result { + std::result::Result::Ok(infos) => infos, + std::result::Result::Err(error) => return std::result::Result::Err(error), + }; + let mut entries = std::vec::Vec::with_capacity(infos.len()); + for info in infos { + entries.push(SignaturePageEntry { signature: info.signature().to_owned(), slot: info.slot() }); + } + return std::result::Result::Ok(entries); + }); + } +} + +/// Discovers one bounded deterministic candidate set using only the typed KSP Transport wrapper. +/// +/// Provider, endpoint and protocol selection remain entirely owned by Transport. Candidate identity +/// is scoped only by the request network plus transaction signature. +pub async fn discover_backfill_candidates( + transport: &ksp_onchain_transport_lib::HttpTransportPool, + request: &crate::BackfillRequest, +) -> ksp_core_lib::Result { + ksp_logging_lib::debug!( + target: crate::TRACING_TARGET, + job_id = request.job_id().as_str(), + network = request.network().as_str(), + scope = request.scope().kind().code(), + page_size = request.page_size(), + max_pages = request.max_pages(), + max_candidates = request.max_candidates(), + "starting bounded Backfill candidate discovery" + ); + let result = discover_with_source(transport, request).await; + if let std::result::Result::Ok(discovery) = &result { + ksp_logging_lib::debug!( + target: crate::TRACING_TARGET, + job_id = request.job_id().as_str(), + network = request.network().as_str(), + scope = request.scope().kind().code(), + pages_fetched = discovery.pages_fetched(), + candidate_count = discovery.candidates().len(), + boundary = discovery.boundary().code(), + partial = discovery.is_partial(), + "completed bounded Backfill candidate discovery" + ); + } + return result; +} + +async fn discover_with_source(source: &S, request: &crate::BackfillRequest) -> ksp_core_lib::Result +where + S: SignaturePageSource, +{ + return match request.scope().kind() { + crate::BackfillScopeKind::ExplicitSignatures => discover_explicit(request), + crate::BackfillScopeKind::LatestAddress | crate::BackfillScopeKind::BeforeAddress => discover_older(source, request).await, + crate::BackfillScopeKind::AfterAddress => discover_after(source, request).await, + }; +} + +fn discover_explicit(request: &crate::BackfillRequest) -> ksp_core_lib::Result { + let signatures = match request.scope().signatures() { + std::option::Option::Some(signatures) => signatures, + std::option::Option::None => return std::result::Result::Err(discovery_invalid("scope.signatures")), + }; + let mut candidates = std::vec::Vec::with_capacity(signatures.len()); + for signature in signatures { + let identity = BackfillCandidateIdentity::new(request.network().clone(), signature.clone()); + candidates.push(BackfillCandidate::new(identity, std::option::Option::None)); + } + return std::result::Result::Ok(BackfillDiscovery { + network: request.network().clone(), + scope_fingerprint: request.scope_fingerprint(), + candidates, + pages_fetched: 0, + boundary: BackfillDiscoveryBoundary::ExplicitInput, + }); +} + +async fn discover_older(source: &S, request: &crate::BackfillRequest) -> ksp_core_lib::Result +where + S: SignaturePageSource, +{ + let address = match request.scope().address() { + std::option::Option::Some(address) => address, + std::option::Option::None => return std::result::Result::Err(discovery_invalid("scope.address")), + }; + let mut before = match request.scope().kind() { + crate::BackfillScopeKind::BeforeAddress => request.scope().anchor().map(|value| value.as_str().to_owned()), + crate::BackfillScopeKind::LatestAddress => std::option::Option::None, + crate::BackfillScopeKind::AfterAddress | crate::BackfillScopeKind::ExplicitSignatures => { + return std::result::Result::Err(discovery_invalid("scope.kind")); + }, + }; + let mut candidates = std::vec::Vec::with_capacity(request.max_candidates()); + let mut seen = std::collections::HashSet::::with_capacity(request.max_candidates()); + let mut pages_fetched = 0_usize; + let boundary = loop { + if candidates.len() >= request.max_candidates() { + break BackfillDiscoveryBoundary::CandidateLimit; + } + if pages_fetched >= request.max_pages() { + break BackfillDiscoveryBoundary::PageLimit; + } + let remaining = request.max_candidates() - candidates.len(); + let page_limit = std::cmp::min(request.page_size(), remaining); + let config = ksp_onchain_transport_lib::SolanaSignaturesForAddressConfig::new( + before.clone(), + std::option::Option::None, + std::option::Option::Some(page_limit), + std::option::Option::Some(request.commitment().transport()), + request.min_context_slot(), + ); + let page_result = source.fetch_signature_page(request.role(), address, config).await; + let page = match page_result { + std::result::Result::Ok(page) => page, + std::result::Result::Err(error) => return std::result::Result::Err(error), + }; + pages_fetched += 1; + if page.len() > page_limit { + return std::result::Result::Err(discovery_invalid("page.length")); + } + let page_len = page.len(); + let next_before = match page.last() { + std::option::Option::Some(entry) => { + let signature_result = validated_signature(entry.signature.as_str()); + match signature_result { + std::result::Result::Ok(signature) => std::option::Option::Some(signature), + std::result::Result::Err(error) => return std::result::Result::Err(error), + } + }, + std::option::Option::None => std::option::Option::None, + }; + for entry in page { + let signature_result = validated_signature(entry.signature.as_str()); + let signature = match signature_result { + std::result::Result::Ok(signature) => signature, + std::result::Result::Err(error) => return std::result::Result::Err(error), + }; + if seen.insert(signature.clone()) { + let identity = BackfillCandidateIdentity::new(request.network().clone(), signature); + candidates.push(BackfillCandidate::new(identity, std::option::Option::Some(entry.slot))); + if candidates.len() >= request.max_candidates() { + break; + } + } + } + if page_len < page_limit { + break BackfillDiscoveryBoundary::RpcBoundary; + } + let next_before = match next_before { + std::option::Option::Some(next_before) => next_before, + std::option::Option::None => break BackfillDiscoveryBoundary::RpcBoundary, + }; + if before.as_deref() == std::option::Option::Some(next_before.as_str()) { + return std::result::Result::Err(discovery_stalled()); + } + before = std::option::Option::Some(next_before.as_str().to_owned()); + }; + return std::result::Result::Ok(BackfillDiscovery { + network: request.network().clone(), + scope_fingerprint: request.scope_fingerprint(), + candidates, + pages_fetched, + boundary, + }); +} + +async fn discover_after(source: &S, request: &crate::BackfillRequest) -> ksp_core_lib::Result +where + S: SignaturePageSource, +{ + let address = match request.scope().address() { + std::option::Option::Some(address) => address, + std::option::Option::None => return std::result::Result::Err(discovery_invalid("scope.address")), + }; + let anchor = match request.scope().anchor() { + std::option::Option::Some(anchor) => anchor, + std::option::Option::None => return std::result::Result::Err(discovery_invalid("scope.anchor")), + }; + let until = anchor.as_str().to_owned(); + let mut before = std::option::Option::::None; + let mut nearest = std::collections::VecDeque::::with_capacity(request.max_candidates()); + let mut seen = std::collections::HashSet::::new(); + let mut pages_fetched = 0_usize; + let boundary = loop { + if pages_fetched >= request.max_pages() { + break BackfillDiscoveryBoundary::AfterAnchorNotReached; + } + let config = ksp_onchain_transport_lib::SolanaSignaturesForAddressConfig::new( + before.clone(), + std::option::Option::Some(until.clone()), + std::option::Option::Some(request.page_size()), + std::option::Option::Some(request.commitment().transport()), + request.min_context_slot(), + ); + let page_result = source.fetch_signature_page(request.role(), address, config).await; + let page = match page_result { + std::result::Result::Ok(page) => page, + std::result::Result::Err(error) => return std::result::Result::Err(error), + }; + pages_fetched += 1; + if page.len() > request.page_size() { + return std::result::Result::Err(discovery_invalid("page.length")); + } + let page_len = page.len(); + let next_before = match page.last() { + std::option::Option::Some(entry) => { + let signature_result = validated_signature(entry.signature.as_str()); + match signature_result { + std::result::Result::Ok(signature) => std::option::Option::Some(signature), + std::result::Result::Err(error) => return std::result::Result::Err(error), + } + }, + std::option::Option::None => std::option::Option::None, + }; + for entry in page { + let signature_result = validated_signature(entry.signature.as_str()); + let signature = match signature_result { + std::result::Result::Ok(signature) => signature, + std::result::Result::Err(error) => return std::result::Result::Err(error), + }; + if seen.insert(signature.clone()) { + let identity = BackfillCandidateIdentity::new(request.network().clone(), signature); + nearest.push_back(BackfillCandidate::new(identity, std::option::Option::Some(entry.slot))); + if nearest.len() > request.max_candidates() { + nearest.pop_front(); + } + } + } + if page_len < request.page_size() { + break BackfillDiscoveryBoundary::RpcBoundary; + } + let next_before = match next_before { + std::option::Option::Some(next_before) => next_before, + std::option::Option::None => break BackfillDiscoveryBoundary::RpcBoundary, + }; + if before.as_deref() == std::option::Option::Some(next_before.as_str()) { + return std::result::Result::Err(discovery_stalled()); + } + before = std::option::Option::Some(next_before.as_str().to_owned()); + }; + return std::result::Result::Ok(BackfillDiscovery { + network: request.network().clone(), + scope_fingerprint: request.scope_fingerprint(), + candidates: nearest.into_iter().collect(), + pages_fetched, + boundary, + }); +} + +fn validated_signature(value: &str) -> ksp_core_lib::Result { + return crate::BackfillSignature::new(value.to_owned()); +} + +fn discovery_invalid(field: &'static str) -> ksp_core_lib::Error { + return ksp_core_lib::Error::new(crate::ERROR_CODE_BACKFILL_DISCOVERY_INVALID, "invalid bounded Backfill discovery state").with_context("field", field); +} + +fn discovery_stalled() -> ksp_core_lib::Error { + return ksp_core_lib::Error::new(crate::ERROR_CODE_BACKFILL_DISCOVERY_STALLED, "Backfill discovery cursor did not advance") + .with_context("field", "before"); +} + +#[cfg(test)] +#[path = "../unit_tests/discovery.rs"] +mod tests; diff --git a/crates/ksp-job-backfill-lib/src/error.rs b/crates/ksp-job-backfill-lib/src/error.rs new file mode 100644 index 0000000..a1a9d77 --- /dev/null +++ b/crates/ksp-job-backfill-lib/src/error.rs @@ -0,0 +1,11 @@ +// file: crates/ksp-job-backfill-lib/src/error.rs +// version: 1 + +/// Error code used when a signature page violates a bounded discovery invariant. +pub const ERROR_CODE_BACKFILL_DISCOVERY_INVALID: ksp_core_lib::ErrorCode = ksp_core_lib::ErrorCode::new("job_backfill", "discovery_invalid"); +/// Error code used when paginated discovery cannot advance its exclusive RPC cursor safely. +pub const ERROR_CODE_BACKFILL_DISCOVERY_STALLED: ksp_core_lib::ErrorCode = ksp_core_lib::ErrorCode::new("job_backfill", "discovery_stalled"); +/// Error code used when one Backfill request violates its bounded admission contract. +pub const ERROR_CODE_BACKFILL_REQUEST_INVALID: ksp_core_lib::ErrorCode = ksp_core_lib::ErrorCode::new("job_backfill", "request_invalid"); +/// Error code used when one transaction signature text violates the bounded Base58-shape contract. +pub const ERROR_CODE_BACKFILL_SIGNATURE_INVALID: ksp_core_lib::ErrorCode = ksp_core_lib::ErrorCode::new("job_backfill", "signature_invalid"); diff --git a/crates/ksp-job-backfill-lib/src/lib.rs b/crates/ksp-job-backfill-lib/src/lib.rs new file mode 100644 index 0000000..f01480c --- /dev/null +++ b/crates/ksp-job-backfill-lib/src/lib.rs @@ -0,0 +1,64 @@ +// file: crates/ksp-job-backfill-lib/src/lib.rs +// version: 1 + +#![warn(missing_docs)] +#![deny(unreachable_pub)] +#![forbid(unsafe_code)] + +//! Concrete bounded historical RAW transaction Backfill foundation. +//! +//! This tranche owns explicit admission, network-scoped candidate identity and deterministic +//! `getSignaturesForAddress` pagination. Transport retains provider/endpoint selection and retry; +//! Store retains durable idempotence and persistence. RAW conversion, persistence, concurrency, +//! checkpointing, cancellation and concrete latest-value snapshots are added by later v0.3.6 tranches. + +mod constants; +mod discovery; +mod error; +mod request; + +/// One deterministic transaction candidate produced by bounded discovery. +pub use self::discovery::BackfillCandidate; +/// Network-scoped identity of one discovered transaction candidate before canonical signature decoding. +pub use self::discovery::BackfillCandidateIdentity; +/// Complete bounded output of one candidate discovery pass. +pub use self::discovery::BackfillDiscovery; +/// Reason bounded candidate discovery stopped. +pub use self::discovery::BackfillDiscoveryBoundary; +/// Discovers one bounded deterministic candidate set through the typed KSP Transport wrapper. +pub use self::discovery::discover_backfill_candidates; +/// Error code used when a signature page violates a bounded discovery invariant. +pub use self::error::ERROR_CODE_BACKFILL_DISCOVERY_INVALID; +/// Error code used when paginated discovery cannot advance its exclusive RPC cursor safely. +pub use self::error::ERROR_CODE_BACKFILL_DISCOVERY_STALLED; +/// Error code used when one Backfill request violates its bounded admission contract. +pub use self::error::ERROR_CODE_BACKFILL_REQUEST_INVALID; +/// Error code used when one transaction signature text violates the bounded Base58-shape contract. +pub use self::error::ERROR_CODE_BACKFILL_SIGNATURE_INVALID; +/// Commitment levels intentionally admitted by the historical Backfill vertical. +pub use self::request::BackfillCommitment; +/// Fully explicit bounded request for one historical transaction Backfill Job. +pub use self::request::BackfillRequest; +/// Validated bounded discovery scope for one historical Backfill Job. +pub use self::request::BackfillScope; +/// Opaque deterministic fingerprint of one semantic Backfill scope. +pub use self::request::BackfillScopeFingerprint; +/// Stable category of one bounded Backfill discovery scope. +pub use self::request::BackfillScopeKind; +/// Bounded Base58-shaped transaction signature text used before canonical 64-byte decoding. +pub use self::request::BackfillSignature; +/// Maximum number of transaction candidates admitted by one bounded Backfill Job. +pub use self::request::MAX_BACKFILL_CANDIDATES; +/// Maximum number of concurrent transaction hydrations admitted by one Backfill request. +pub use self::request::MAX_BACKFILL_HYDRATION_CONCURRENCY; +/// Maximum number of `getSignaturesForAddress` pages admitted by one address Backfill request. +pub use self::request::MAX_BACKFILL_PAGES; +/// Maximum page size admitted for one `getSignaturesForAddress` request. +pub use self::request::MAX_BACKFILL_PAGE_SIZE; +/// Maximum Base58 text length possible for one canonical 64-byte Solana signature. +pub use self::request::MAX_BACKFILL_SIGNATURE_TEXT_BYTES; +/// Minimum Base58 text length possible for one canonical 64-byte Solana signature. +pub use self::request::MIN_BACKFILL_SIGNATURE_TEXT_BYTES; + +/// Owning tracing target used by the concrete Backfill runtime. +pub(crate) use self::constants::TRACING_TARGET; diff --git a/crates/ksp-job-backfill-lib/src/request.rs b/crates/ksp-job-backfill-lib/src/request.rs new file mode 100644 index 0000000..5431b22 --- /dev/null +++ b/crates/ksp-job-backfill-lib/src/request.rs @@ -0,0 +1,466 @@ +// file: crates/ksp-job-backfill-lib/src/request.rs +// version: 1 + +use sha2::Digest; // rust-rules: trait-import + +/// Maximum number of transaction candidates admitted by one bounded Backfill Job. +pub const MAX_BACKFILL_CANDIDATES: usize = 10_000; +/// Maximum number of concurrent transaction hydrations admitted by one Backfill request. +pub const MAX_BACKFILL_HYDRATION_CONCURRENCY: usize = 64; +/// Maximum number of `getSignaturesForAddress` pages admitted by one address Backfill request. +pub const MAX_BACKFILL_PAGES: usize = 10_000; +/// Maximum page size admitted for one `getSignaturesForAddress` request. +pub const MAX_BACKFILL_PAGE_SIZE: usize = 1_000; +/// Maximum Base58 text length possible for one canonical 64-byte Solana signature. +pub const MAX_BACKFILL_SIGNATURE_TEXT_BYTES: usize = 88; +/// Minimum Base58 text length possible for one canonical 64-byte Solana signature. +pub const MIN_BACKFILL_SIGNATURE_TEXT_BYTES: usize = 64; + +/// Commitment levels intentionally admitted by the historical Backfill vertical. +#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)] +pub enum BackfillCommitment { + /// Read history at Solana `confirmed` commitment. + Confirmed, + /// Read history at Solana `finalized` commitment. + Finalized, +} + +impl BackfillCommitment { + /// Returns the stable Backfill commitment code. + #[must_use] + pub const fn code(self) -> &'static str { + return match self { + Self::Confirmed => "confirmed", + Self::Finalized => "finalized", + }; + } + + /// Maps the narrowed Backfill commitment to the Transport-owned Solana commitment. + #[must_use] + pub(crate) const fn transport(self) -> ksp_onchain_transport_lib::SolanaCommitment { + return match self { + Self::Confirmed => ksp_onchain_transport_lib::SolanaCommitment::Confirmed, + Self::Finalized => ksp_onchain_transport_lib::SolanaCommitment::Finalized, + }; + } +} + +/// Bounded Base58-shaped transaction signature text used before canonical 64-byte decoding. +/// +/// This type deliberately validates only the encoded shape required by discovery. Exact decoding +/// to the Store-owned 64-byte signature is introduced by the RAW conversion tranche. +#[derive(Clone, Eq, Hash, Ord, PartialEq, PartialOrd)] +pub struct BackfillSignature(std::string::String); + +impl BackfillSignature { + /// Creates one bounded Base58-shaped signature text. + pub fn new(value: impl std::convert::Into) -> ksp_core_lib::Result { + let value = value.into(); + if !valid_signature_text(value.as_str()) { + return std::result::Result::Err(signature_error()); + } + return std::result::Result::Ok(Self(value)); + } + + /// Returns the validated encoded signature text. + #[must_use] + pub fn as_str(&self) -> &str { + return self.0.as_str(); + } +} + +impl std::fmt::Debug for BackfillSignature { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + return formatter.write_str("BackfillSignature(..)"); + } +} + +/// Stable category of one bounded Backfill discovery scope. +#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)] +pub enum BackfillScopeKind { + /// Read the current newest address window. + LatestAddress, + /// Read address history older than one exclusive anchor. + BeforeAddress, + /// Read the bounded newer window closest to one exclusive anchor. + AfterAddress, + /// Hydrate an explicit bounded signature list without address discovery. + ExplicitSignatures, +} + +impl BackfillScopeKind { + /// Returns the stable scope code used by diagnostics and scope fingerprinting. + #[must_use] + pub const fn code(self) -> &'static str { + return match self { + Self::LatestAddress => "latest_address", + Self::BeforeAddress => "before_address", + Self::AfterAddress => "after_address", + Self::ExplicitSignatures => "explicit_signatures", + }; + } +} + +#[derive(Clone, Eq, PartialEq)] +enum BackfillScopeValue { + LatestAddress { address: ksp_core_lib::Pubkey }, + BeforeAddress { address: ksp_core_lib::Pubkey, anchor: BackfillSignature }, + AfterAddress { address: ksp_core_lib::Pubkey, anchor: BackfillSignature }, + ExplicitSignatures { signatures: std::vec::Vec }, +} + +/// Validated bounded discovery scope for one historical Backfill Job. +#[derive(Clone, Eq, PartialEq)] +pub struct BackfillScope { + value: BackfillScopeValue, +} + +impl BackfillScope { + /// Creates a scope starting at the newest known history for one address. + #[must_use] + pub const fn latest_address(address: ksp_core_lib::Pubkey) -> Self { + return Self { value: BackfillScopeValue::LatestAddress { address } }; + } + + /// Creates a scope reading history older than one exclusive address anchor. + #[must_use] + pub fn before_address(address: ksp_core_lib::Pubkey, anchor: BackfillSignature) -> Self { + return Self { value: BackfillScopeValue::BeforeAddress { address, anchor } }; + } + + /// Creates a scope reading the bounded newer history closest to one exclusive address anchor. + #[must_use] + pub fn after_address(address: ksp_core_lib::Pubkey, anchor: BackfillSignature) -> Self { + return Self { value: BackfillScopeValue::AfterAddress { address, anchor } }; + } + + /// Creates an explicit signature scope with stable first-occurrence deduplication. + pub fn explicit_signatures(signatures: std::vec::Vec) -> ksp_core_lib::Result { + if signatures.is_empty() || signatures.len() > crate::MAX_BACKFILL_CANDIDATES { + return std::result::Result::Err(request_error("scope.signatures")); + } + let mut unique = std::vec::Vec::with_capacity(signatures.len()); + let mut seen = std::collections::HashSet::with_capacity(signatures.len()); + for signature in signatures { + if seen.insert(signature.clone()) { + unique.push(signature); + } + } + if unique.is_empty() { + return std::result::Result::Err(request_error("scope.signatures")); + } + return std::result::Result::Ok(Self { value: BackfillScopeValue::ExplicitSignatures { signatures: unique } }); + } + + /// Returns the stable category of this scope. + #[must_use] + pub const fn kind(&self) -> BackfillScopeKind { + return match &self.value { + BackfillScopeValue::LatestAddress { .. } => BackfillScopeKind::LatestAddress, + BackfillScopeValue::BeforeAddress { .. } => BackfillScopeKind::BeforeAddress, + BackfillScopeValue::AfterAddress { .. } => BackfillScopeKind::AfterAddress, + BackfillScopeValue::ExplicitSignatures { .. } => BackfillScopeKind::ExplicitSignatures, + }; + } + + /// Returns the address used by address scopes. + #[must_use] + pub const fn address(&self) -> std::option::Option<&ksp_core_lib::Pubkey> { + return match &self.value { + BackfillScopeValue::LatestAddress { address } + | BackfillScopeValue::BeforeAddress { address, .. } + | BackfillScopeValue::AfterAddress { address, .. } => std::option::Option::Some(address), + BackfillScopeValue::ExplicitSignatures { .. } => std::option::Option::None, + }; + } + + /// Returns the exclusive anchor used by before/after scopes. + #[must_use] + pub const fn anchor(&self) -> std::option::Option<&BackfillSignature> { + return match &self.value { + BackfillScopeValue::BeforeAddress { anchor, .. } | BackfillScopeValue::AfterAddress { anchor, .. } => std::option::Option::Some(anchor), + BackfillScopeValue::LatestAddress { .. } | BackfillScopeValue::ExplicitSignatures { .. } => std::option::Option::None, + }; + } + + /// Returns the stable deduplicated explicit signature list, when this is an explicit scope. + #[must_use] + pub fn signatures(&self) -> std::option::Option<&[BackfillSignature]> { + return match &self.value { + BackfillScopeValue::ExplicitSignatures { signatures } => std::option::Option::Some(signatures.as_slice()), + BackfillScopeValue::LatestAddress { .. } | BackfillScopeValue::BeforeAddress { .. } | BackfillScopeValue::AfterAddress { .. } => { + std::option::Option::None + }, + }; + } +} + +impl std::fmt::Debug for BackfillScope { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let mut debug = formatter.debug_struct("BackfillScope"); + debug.field("kind", &self.kind()); + match &self.value { + BackfillScopeValue::LatestAddress { address } => { + debug.field("address", address); + }, + BackfillScopeValue::BeforeAddress { address, .. } | BackfillScopeValue::AfterAddress { address, .. } => { + debug.field("address", address).field("anchor", &""); + }, + BackfillScopeValue::ExplicitSignatures { signatures } => { + debug.field("signature_count", &signatures.len()); + }, + } + return debug.finish(); + } +} + +/// Opaque deterministic fingerprint of one semantic Backfill scope. +#[derive(Clone, Copy, Eq, Hash, PartialEq)] +pub struct BackfillScopeFingerprint([u8; 32]); + +impl BackfillScopeFingerprint { + /// Returns the exact deterministic fingerprint bytes. + #[must_use] + pub const fn as_bytes(&self) -> &[u8; 32] { + return &self.0; + } +} + +impl std::fmt::Debug for BackfillScopeFingerprint { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + return formatter.write_str("BackfillScopeFingerprint(..)"); + } +} + +/// Fully explicit bounded request for one historical transaction Backfill Job. +#[derive(Clone, Eq, PartialEq)] +pub struct BackfillRequest { + job_id: ksp_job_api::JobId, + network: ksp_store_lib::RawNetworkId, + role: ksp_onchain_transport_lib::HttpRoleName, + commitment: BackfillCommitment, + scope: BackfillScope, + page_size: usize, + max_pages: usize, + max_candidates: usize, + hydration_concurrency: usize, + min_context_slot: std::option::Option, + scope_fingerprint: BackfillScopeFingerprint, +} + +impl BackfillRequest { + /// Creates and validates one fully explicit bounded Backfill request. + #[allow(clippy::too_many_arguments)] + pub fn new( + job_id: ksp_job_api::JobId, + network: ksp_store_lib::RawNetworkId, + role: ksp_onchain_transport_lib::HttpRoleName, + commitment: BackfillCommitment, + scope: BackfillScope, + page_size: usize, + max_pages: usize, + max_candidates: usize, + hydration_concurrency: usize, + min_context_slot: std::option::Option, + ) -> ksp_core_lib::Result { + if page_size == 0 || page_size > crate::MAX_BACKFILL_PAGE_SIZE { + return std::result::Result::Err(request_error("page_size")); + } + if max_pages == 0 || max_pages > crate::MAX_BACKFILL_PAGES { + return std::result::Result::Err(request_error("max_pages")); + } + if max_candidates == 0 || max_candidates > crate::MAX_BACKFILL_CANDIDATES { + return std::result::Result::Err(request_error("max_candidates")); + } + if hydration_concurrency == 0 || hydration_concurrency > crate::MAX_BACKFILL_HYDRATION_CONCURRENCY { + return std::result::Result::Err(request_error("hydration_concurrency")); + } + if role.as_str().is_empty() || role.as_str().trim() != role.as_str() { + return std::result::Result::Err(request_error("role")); + } + if scope.kind() == BackfillScopeKind::ExplicitSignatures { + if min_context_slot.is_some() { + return std::result::Result::Err(request_error("min_context_slot")); + } + let signature_count = match scope.signatures() { + std::option::Option::Some(signatures) => signatures.len(), + std::option::Option::None => return std::result::Result::Err(request_error("scope.signatures")), + }; + if signature_count > max_candidates { + return std::result::Result::Err(request_error("max_candidates")); + } + } + let scope_fingerprint = fingerprint_scope(&network, commitment, &scope, page_size, max_pages, max_candidates, min_context_slot); + return std::result::Result::Ok(Self { + job_id, + network, + role, + commitment, + scope, + page_size, + max_pages, + max_candidates, + hydration_concurrency, + min_context_slot, + scope_fingerprint, + }); + } + + /// Returns the caller-owned logical Job identity. + #[must_use] + pub const fn job_id(&self) -> &ksp_job_api::JobId { + return &self.job_id; + } + + /// Returns the Store-scoped logical network identity for every candidate in this Job. + #[must_use] + pub const fn network(&self) -> &ksp_store_lib::RawNetworkId { + return &self.network; + } + + /// Returns the logical HTTP role used only for Transport selection. + #[must_use] + pub const fn role(&self) -> &ksp_onchain_transport_lib::HttpRoleName { + return &self.role; + } + + /// Returns the narrowed commitment used by discovery and hydration. + #[must_use] + pub const fn commitment(&self) -> BackfillCommitment { + return self.commitment; + } + + /// Returns the validated discovery scope. + #[must_use] + pub const fn scope(&self) -> &BackfillScope { + return &self.scope; + } + + /// Returns the per-request signature page size. + #[must_use] + pub const fn page_size(&self) -> usize { + return self.page_size; + } + + /// Returns the maximum number of address pages admitted by this Job. + #[must_use] + pub const fn max_pages(&self) -> usize { + return self.max_pages; + } + + /// Returns the maximum number of transaction candidates admitted by this Job. + #[must_use] + pub const fn max_candidates(&self) -> usize { + return self.max_candidates; + } + + /// Returns the future hydration concurrency bound. + #[must_use] + pub const fn hydration_concurrency(&self) -> usize { + return self.hydration_concurrency; + } + + /// Returns the optional Solana minimum context slot used by address discovery. + #[must_use] + pub const fn min_context_slot(&self) -> std::option::Option { + return self.min_context_slot; + } + + /// Returns the deterministic semantic scope fingerprint. + /// + /// Transport role, provider, endpoint and protocol are deliberately excluded. They describe + /// acquisition provenance, not transaction or scope identity. + #[must_use] + pub const fn scope_fingerprint(&self) -> BackfillScopeFingerprint { + return self.scope_fingerprint; + } +} + +impl std::fmt::Debug for BackfillRequest { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + return formatter + .debug_struct("BackfillRequest") + .field("job_id", &self.job_id) + .field("network", &self.network) + .field("role", &self.role) + .field("commitment", &self.commitment) + .field("scope", &self.scope) + .field("page_size", &self.page_size) + .field("max_pages", &self.max_pages) + .field("max_candidates", &self.max_candidates) + .field("hydration_concurrency", &self.hydration_concurrency) + .field("min_context_slot", &self.min_context_slot) + .field("scope_fingerprint", &self.scope_fingerprint) + .finish(); + } +} + +fn fingerprint_scope( + network: &ksp_store_lib::RawNetworkId, + commitment: BackfillCommitment, + scope: &BackfillScope, + page_size: usize, + max_pages: usize, + max_candidates: usize, + min_context_slot: std::option::Option, +) -> BackfillScopeFingerprint { + let mut hasher = sha2::Sha256::new(); + hasher.update(b"ksp.job.backfill.scope.v1\0"); + hash_bytes(&mut hasher, network.as_str().as_bytes()); + hash_bytes(&mut hasher, commitment.code().as_bytes()); + hash_bytes(&mut hasher, scope.kind().code().as_bytes()); + if let std::option::Option::Some(address) = scope.address() { + hash_bytes(&mut hasher, address.as_ref()); + } + if let std::option::Option::Some(anchor) = scope.anchor() { + hash_bytes(&mut hasher, anchor.as_str().as_bytes()); + } + if let std::option::Option::Some(signatures) = scope.signatures() { + for signature in signatures { + hash_bytes(&mut hasher, signature.as_str().as_bytes()); + } + } + hash_u64(&mut hasher, page_size as u64); + hash_u64(&mut hasher, max_pages as u64); + hash_u64(&mut hasher, max_candidates as u64); + match min_context_slot { + std::option::Option::Some(slot) => { + hasher.update([1_u8]); + hash_u64(&mut hasher, slot); + }, + std::option::Option::None => hasher.update([0_u8]), + } + let bytes: [u8; 32] = hasher.finalize().into(); + return BackfillScopeFingerprint(bytes); +} + +fn hash_bytes(hasher: &mut sha2::Sha256, value: &[u8]) { + hash_u64(hasher, value.len() as u64); + hasher.update(value); +} + +fn hash_u64(hasher: &mut sha2::Sha256, value: u64) { + hasher.update(value.to_be_bytes()); +} + +fn request_error(field: &'static str) -> ksp_core_lib::Error { + return ksp_core_lib::Error::new(crate::ERROR_CODE_BACKFILL_REQUEST_INVALID, "invalid bounded Backfill request").with_context("field", field); +} + +fn signature_error() -> ksp_core_lib::Error { + return ksp_core_lib::Error::new(crate::ERROR_CODE_BACKFILL_SIGNATURE_INVALID, "invalid bounded Backfill transaction signature") + .with_context("field", "signature"); +} + +fn valid_signature_text(value: &str) -> bool { + if value.len() < crate::MIN_BACKFILL_SIGNATURE_TEXT_BYTES || value.len() > crate::MAX_BACKFILL_SIGNATURE_TEXT_BYTES { + return false; + } + return value.bytes().all(|byte| { + return matches!(byte, b'1'..=b'9' | b'A'..=b'H' | b'J'..=b'N' | b'P'..=b'Z' | b'a'..=b'k' | b'm'..=b'z'); + }); +} + +#[cfg(test)] +#[path = "../unit_tests/request.rs"] +mod tests; diff --git a/crates/ksp-job-backfill-lib/tests/dependency_boundary.rs b/crates/ksp-job-backfill-lib/tests/dependency_boundary.rs new file mode 100644 index 0000000..a26d842 --- /dev/null +++ b/crates/ksp-job-backfill-lib/tests/dependency_boundary.rs @@ -0,0 +1,66 @@ +// file: crates/ksp-job-backfill-lib/tests/dependency_boundary.rs +// version: 1 + +//! Dependency firewall canaries for the concrete Backfill foundation. + +#[test] +fn pre_005_manifest_uses_only_planned_ksp_edges_and_backend_neutral_store() { + let manifest = include_str!("../Cargo.toml"); + for required in [ + "ksp-core-lib = { path = \"../ksp-core-lib\" }", + "ksp-job-api = { path = \"../ksp-job-api\" }", + "ksp-logging-lib = { path = \"../ksp-logging-lib\" }", + "ksp-onchain-transport-lib = { path = \"../ksp-onchain-transport-lib\" }", + "ksp-store-lib = { path = \"../ksp-store-lib\", default-features = false }", + "sha2.workspace = true", + ] { + assert!(manifest.contains(required), "required Backfill dependency missing: {required}"); + } + for forbidden in [ + "ksp-config-lib", + "ksp-interface-lib", + "ksp-program-api", + "ksp-store-api", + "ksp-store-postgres-lib", + "ksp-wallet-lib", + "solana-", + "reqwest", + "serde", + "tonic", + ] { + assert!(!manifest.contains(forbidden), "forbidden Backfill dependency present: {forbidden}"); + } + return; +} + +#[test] +fn pre_005_production_sources_keep_transport_and_store_in_their_owned_layers() { + let sources = [ + include_str!("../src/constants.rs"), + include_str!("../src/discovery.rs"), + include_str!("../src/error.rs"), + include_str!("../src/lib.rs"), + include_str!("../src/request.rs"), + ]; + for source in sources { + for forbidden in [ + "ksp_config_lib::", + "ksp_interface_lib::", + "ksp_store_api::", + "ksp_store_postgres_lib::", + "reqwest::", + "serde_json::", + "std::env", + "tonic::", + ] { + assert!(!source.contains(forbidden), "forbidden concrete Backfill path detected: {forbidden}"); + } + } + let discovery = include_str!("../src/discovery.rs"); + assert!(discovery.contains("get_signatures_for_address")); + assert!(!discovery.contains("execute_standard_rpc")); + assert!(!discovery.contains("retry")); + assert!(!discovery.contains("endpoint_name")); + assert!(!discovery.contains("HttpEndpoint")); + return; +} diff --git a/crates/ksp-job-backfill-lib/tests/public_api.rs b/crates/ksp-job-backfill-lib/tests/public_api.rs new file mode 100644 index 0000000..649c8c2 --- /dev/null +++ b/crates/ksp-job-backfill-lib/tests/public_api.rs @@ -0,0 +1,66 @@ +// file: crates/ksp-job-backfill-lib/tests/public_api.rs +// version: 1 + +//! Public API canaries for the bounded Backfill foundation. + +#[test] +fn pre_005_request_scope_and_discovery_contracts_are_available_from_crate_root() { + let _discover = ksp_job_backfill_lib::discover_backfill_candidates; + let address = ksp_core_lib::Pubkey::new_from_array([21_u8; 32]); + let scope = ksp_job_backfill_lib::BackfillScope::latest_address(address); + let network = ksp_store_lib::RawNetworkId::new("devnet"); + assert!(network.is_ok()); + let network = match network { + std::result::Result::Ok(value) => value, + std::result::Result::Err(_) => return, + }; + let job_id = ksp_job_api::JobId::new("backfill:public-api"); + assert!(job_id.is_ok()); + let job_id = match job_id { + std::result::Result::Ok(value) => value, + std::result::Result::Err(_) => return, + }; + let request = ksp_job_backfill_lib::BackfillRequest::new( + job_id, + network, + ksp_onchain_transport_lib::HttpRoleName::new("history"), + ksp_job_backfill_lib::BackfillCommitment::Confirmed, + scope, + 100, + 10, + 500, + 8, + std::option::Option::Some(100), + ); + assert!(request.is_ok()); + let request = match request { + std::result::Result::Ok(value) => value, + std::result::Result::Err(_) => return, + }; + assert_eq!(request.scope().kind(), ksp_job_backfill_lib::BackfillScopeKind::LatestAddress); + assert_eq!(request.page_size(), 100); + return; +} + +#[test] +fn pre_005_candidate_identity_is_network_plus_signature_not_transport_source() { + let signature = ksp_job_backfill_lib::BackfillSignature::new("1".repeat(ksp_job_backfill_lib::MIN_BACKFILL_SIGNATURE_TEXT_BYTES)); + let signature = match signature { + std::result::Result::Ok(value) => value, + std::result::Result::Err(_) => return, + }; + let mainnet = ksp_store_lib::RawNetworkId::new("mainnet"); + let devnet = ksp_store_lib::RawNetworkId::new("devnet"); + let mainnet = match mainnet { + std::result::Result::Ok(value) => value, + std::result::Result::Err(_) => return, + }; + let devnet = match devnet { + std::result::Result::Ok(value) => value, + std::result::Result::Err(_) => return, + }; + let mainnet_identity = ksp_job_backfill_lib::BackfillCandidateIdentity::new(mainnet, signature.clone()); + let devnet_identity = ksp_job_backfill_lib::BackfillCandidateIdentity::new(devnet, signature); + assert_ne!(mainnet_identity, devnet_identity); + return; +} diff --git a/crates/ksp-job-backfill-lib/tests/release_completeness.rs b/crates/ksp-job-backfill-lib/tests/release_completeness.rs new file mode 100644 index 0000000..5f6fab6 --- /dev/null +++ b/crates/ksp-job-backfill-lib/tests/release_completeness.rs @@ -0,0 +1,61 @@ +// file: crates/ksp-job-backfill-lib/tests/release_completeness.rs +// version: 1 + +//! Completeness canaries for the `pre.005` Backfill foundation. + +#[test] +fn pre_005_production_module_inventory_is_exact() -> std::io::Result<()> { + let source_root = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("src"); + let entries = match std::fs::read_dir(source_root) { + std::result::Result::Ok(value) => value, + std::result::Result::Err(error) => return std::result::Result::Err(error), + }; + let mut names = std::vec::Vec::new(); + for entry in entries { + let entry = match entry { + std::result::Result::Ok(value) => value, + std::result::Result::Err(error) => return std::result::Result::Err(error), + }; + let file_type = match entry.file_type() { + std::result::Result::Ok(value) => value, + std::result::Result::Err(error) => return std::result::Result::Err(error), + }; + if !file_type.is_file() { + continue; + } + let name = match entry.file_name().into_string() { + std::result::Result::Ok(value) => value, + std::result::Result::Err(_) => continue, + }; + if name.ends_with(".rs") { + names.push(name); + } + } + names.sort_unstable(); + assert_eq!(names, std::vec!["constants.rs", "discovery.rs", "error.rs", "lib.rs", "request.rs"]); + return std::result::Result::Ok(()); +} + +#[test] +fn pre_005_surface_is_discovery_only_without_raw_persistence_or_checkpoint_runtime() { + let root = include_str!("../src/lib.rs"); + for required in [ + "BackfillCandidate", + "BackfillCandidateIdentity", + "BackfillCommitment", + "BackfillDiscovery", + "BackfillDiscoveryBoundary", + "BackfillRequest", + "BackfillScope", + "BackfillScopeFingerprint", + "BackfillSignature", + "discover_backfill_candidates", + ] { + assert!(root.contains(required), "required pre.005 public contract missing: {required}"); + } + for forbidden in ["RawTransactionObservation", "persist_raw_transaction_acquisition", "BackfillCheckpoint", "BackfillJobHandle", "JobSnapshotSource"] { + assert!(!root.contains(forbidden), "later Backfill tranche leaked into pre.005: {forbidden}"); + } + assert!(!root.contains("pub mod ")); + return; +} diff --git a/crates/ksp-job-backfill-lib/unit_tests/discovery.rs b/crates/ksp-job-backfill-lib/unit_tests/discovery.rs new file mode 100644 index 0000000..ccca8ee --- /dev/null +++ b/crates/ksp-job-backfill-lib/unit_tests/discovery.rs @@ -0,0 +1,311 @@ +// file: crates/ksp-job-backfill-lib/unit_tests/discovery.rs +// version: 1 + +#[derive(Clone, Debug, Eq, PartialEq)] +struct PageCall { + before: std::option::Option, + until: std::option::Option, + limit: std::option::Option, + commitment: std::option::Option, + min_context_slot: std::option::Option, +} + +struct FakeSource { + pages: std::sync::Mutex>>, + calls: std::sync::Mutex>, +} + +impl FakeSource { + fn new(pages: std::vec::Vec>) -> Self { + return Self { pages: std::sync::Mutex::new(pages.into()), calls: std::sync::Mutex::new(std::vec::Vec::new()) }; + } + + fn calls(&self) -> std::vec::Vec { + let guard = self.calls.lock(); + return match guard { + std::result::Result::Ok(value) => value.clone(), + std::result::Result::Err(_) => std::vec::Vec::new(), + }; + } +} + +impl super::SignaturePageSource for FakeSource { + fn fetch_signature_page<'a>( + &'a self, + _role: &'a ksp_onchain_transport_lib::HttpRoleName, + _address: &'a ksp_core_lib::Pubkey, + config: ksp_onchain_transport_lib::SolanaSignaturesForAddressConfig, + ) -> super::SignaturePageFuture<'a> { + let call = PageCall { + before: config.before().map(str::to_owned), + until: config.until().map(str::to_owned), + limit: config.limit(), + commitment: config.commitment(), + min_context_slot: config.min_context_slot(), + }; + let calls_result = self.calls.lock(); + match calls_result { + std::result::Result::Ok(mut calls) => calls.push(call), + std::result::Result::Err(_) => { + return std::boxed::Box::pin(async { + return std::result::Result::Err(ksp_core_lib::Error::new( + crate::ERROR_CODE_BACKFILL_DISCOVERY_INVALID, + "test call recorder lock poisoned", + )); + }); + }, + } + let pages_result = self.pages.lock(); + let page = match pages_result { + std::result::Result::Ok(mut pages) => match pages.pop_front() { + std::option::Option::Some(value) => value, + std::option::Option::None => std::vec::Vec::new(), + }, + std::result::Result::Err(_) => { + return std::boxed::Box::pin(async { + return std::result::Result::Err(ksp_core_lib::Error::new( + crate::ERROR_CODE_BACKFILL_DISCOVERY_INVALID, + "test page queue lock poisoned", + )); + }); + }, + }; + return std::boxed::Box::pin(async move { return std::result::Result::Ok(page); }); + } +} + +fn page_entry(character: char, slot: u64) -> super::SignaturePageEntry { + return super::SignaturePageEntry { signature: character.to_string().repeat(crate::MIN_BACKFILL_SIGNATURE_TEXT_BYTES), slot }; +} + +fn signature(character: char) -> std::option::Option { + return match crate::BackfillSignature::new(character.to_string().repeat(crate::MIN_BACKFILL_SIGNATURE_TEXT_BYTES)) { + std::result::Result::Ok(value) => std::option::Option::Some(value), + std::result::Result::Err(_) => std::option::Option::None, + }; +} + +fn request(scope: crate::BackfillScope, page_size: usize, max_pages: usize, max_candidates: usize) -> std::option::Option { + let job_id = match ksp_job_api::JobId::new("backfill:discovery-test") { + std::result::Result::Ok(value) => value, + std::result::Result::Err(_) => return std::option::Option::None, + }; + let network = match ksp_store_lib::RawNetworkId::new("devnet") { + std::result::Result::Ok(value) => value, + std::result::Result::Err(_) => return std::option::Option::None, + }; + let result = crate::BackfillRequest::new( + job_id, + network, + ksp_onchain_transport_lib::HttpRoleName::new("history"), + crate::BackfillCommitment::Confirmed, + scope, + page_size, + max_pages, + max_candidates, + 1, + std::option::Option::Some(42), + ); + return match result { + std::result::Result::Ok(value) => std::option::Option::Some(value), + std::result::Result::Err(_) => std::option::Option::None, + }; +} + +fn candidate_signatures(discovery: &crate::BackfillDiscovery) -> std::vec::Vec { + let mut signatures = std::vec::Vec::with_capacity(discovery.candidates().len()); + for candidate in discovery.candidates() { + signatures.push(candidate.identity().signature().as_str().to_owned()); + } + return signatures; +} + +#[tokio::test] +async fn pre_005_latest_paginates_newest_first_and_deduplicates_pages_stably() { + let scope = crate::BackfillScope::latest_address(ksp_core_lib::Pubkey::new_from_array([1_u8; 32])); + let request = match request(scope, 3, 4, 10) { + std::option::Option::Some(value) => value, + std::option::Option::None => return, + }; + let source = FakeSource::new(std::vec![ + std::vec![page_entry('6', 60), page_entry('5', 50), page_entry('5', 50)], + std::vec![page_entry('4', 40), page_entry('3', 30)], + ]); + let result = super::discover_with_source(&source, &request).await; + assert!(result.is_ok()); + let discovery = match result { + std::result::Result::Ok(value) => value, + std::result::Result::Err(_) => return, + }; + assert_eq!(candidate_signatures(&discovery), std::vec!["6".repeat(64), "5".repeat(64), "4".repeat(64), "3".repeat(64)]); + assert_eq!(discovery.pages_fetched(), 2); + assert_eq!(discovery.boundary(), crate::BackfillDiscoveryBoundary::RpcBoundary); + assert!(!discovery.is_partial()); + let calls = source.calls(); + assert_eq!(calls.len(), 2); + assert_eq!(calls[0].before, std::option::Option::None); + let expected_cursor = "5".repeat(64); + assert_eq!(calls[1].before.as_deref(), std::option::Option::Some(expected_cursor.as_str())); + assert_eq!(calls[0].until, std::option::Option::None); + assert_eq!(calls[0].limit, std::option::Option::Some(3)); + assert_eq!(calls[0].commitment, std::option::Option::Some(ksp_onchain_transport_lib::SolanaCommitment::Confirmed)); + assert_eq!(calls[0].min_context_slot, std::option::Option::Some(42)); + return; +} + +#[tokio::test] +async fn pre_005_before_uses_exclusive_anchor_then_advances_rpc_cursor() { + let anchor = match signature('7') { + std::option::Option::Some(value) => value, + std::option::Option::None => return, + }; + let scope = crate::BackfillScope::before_address(ksp_core_lib::Pubkey::new_from_array([2_u8; 32]), anchor.clone()); + let request = match request(scope, 2, 3, 5) { + std::option::Option::Some(value) => value, + std::option::Option::None => return, + }; + let source = FakeSource::new(std::vec![std::vec![page_entry('6', 60), page_entry('5', 50)], std::vec![page_entry('4', 40)]]); + let result = super::discover_with_source(&source, &request).await; + let discovery = match result { + std::result::Result::Ok(value) => value, + std::result::Result::Err(_) => return, + }; + assert_eq!(candidate_signatures(&discovery), std::vec!["6".repeat(64), "5".repeat(64), "4".repeat(64)]); + let calls = source.calls(); + assert_eq!(calls.len(), 2); + assert_eq!(calls[0].before.as_deref(), std::option::Option::Some(anchor.as_str())); + let expected_cursor = "5".repeat(64); + assert_eq!(calls[1].before.as_deref(), std::option::Option::Some(expected_cursor.as_str())); + assert_eq!(calls[0].until, std::option::Option::None); + return; +} + +#[tokio::test] +async fn pre_005_after_keeps_only_nearest_newer_window_and_preserves_rpc_order() { + let anchor = match signature('1') { + std::option::Option::Some(value) => value, + std::option::Option::None => return, + }; + let scope = crate::BackfillScope::after_address(ksp_core_lib::Pubkey::new_from_array([3_u8; 32]), anchor.clone()); + let request = match request(scope, 3, 3, 3) { + std::option::Option::Some(value) => value, + std::option::Option::None => return, + }; + let source = FakeSource::new(std::vec![ + std::vec![page_entry('7', 70), page_entry('6', 60), page_entry('5', 50)], + std::vec![page_entry('4', 40), page_entry('3', 30)], + ]); + let result = super::discover_with_source(&source, &request).await; + let discovery = match result { + std::result::Result::Ok(value) => value, + std::result::Result::Err(_) => return, + }; + assert_eq!(candidate_signatures(&discovery), std::vec!["5".repeat(64), "4".repeat(64), "3".repeat(64)]); + assert_eq!(discovery.boundary(), crate::BackfillDiscoveryBoundary::RpcBoundary); + assert!(!discovery.is_partial()); + let calls = source.calls(); + assert_eq!(calls.len(), 2); + assert_eq!(calls[0].until.as_deref(), std::option::Option::Some(anchor.as_str())); + assert_eq!(calls[1].until.as_deref(), std::option::Option::Some(anchor.as_str())); + let expected_cursor = "5".repeat(64); + assert_eq!(calls[1].before.as_deref(), std::option::Option::Some(expected_cursor.as_str())); + return; +} + +#[tokio::test] +async fn pre_005_after_page_bound_is_partial_and_does_not_claim_anchor_completion() { + let anchor = match signature('1') { + std::option::Option::Some(value) => value, + std::option::Option::None => return, + }; + let scope = crate::BackfillScope::after_address(ksp_core_lib::Pubkey::new_from_array([4_u8; 32]), anchor); + let request = match request(scope, 2, 2, 3) { + std::option::Option::Some(value) => value, + std::option::Option::None => return, + }; + let source = FakeSource::new(std::vec![ + std::vec![page_entry('7', 70), page_entry('6', 60)], + std::vec![page_entry('5', 50), page_entry('4', 40)], + ]); + let result = super::discover_with_source(&source, &request).await; + let discovery = match result { + std::result::Result::Ok(value) => value, + std::result::Result::Err(_) => return, + }; + assert_eq!(candidate_signatures(&discovery), std::vec!["6".repeat(64), "5".repeat(64), "4".repeat(64)]); + assert_eq!(discovery.boundary(), crate::BackfillDiscoveryBoundary::AfterAnchorNotReached); + assert!(discovery.is_partial()); + assert_eq!(discovery.pages_fetched(), 2); + return; +} + +#[tokio::test] +async fn pre_005_latest_page_bound_is_partial_when_full_pages_leave_more_history_possible() { + let scope = crate::BackfillScope::latest_address(ksp_core_lib::Pubkey::new_from_array([5_u8; 32])); + let request = match request(scope, 2, 1, 5) { + std::option::Option::Some(value) => value, + std::option::Option::None => return, + }; + let source = FakeSource::new(std::vec![std::vec![page_entry('7', 70), page_entry('6', 60)]]); + let result = super::discover_with_source(&source, &request).await; + let discovery = match result { + std::result::Result::Ok(value) => value, + std::result::Result::Err(_) => return, + }; + assert_eq!(discovery.boundary(), crate::BackfillDiscoveryBoundary::PageLimit); + assert!(discovery.is_partial()); + return; +} + +#[tokio::test] +async fn pre_005_explicit_scope_never_calls_transport_and_preserves_network_scoped_identity() { + let first = match signature('2') { + std::option::Option::Some(value) => value, + std::option::Option::None => return, + }; + let second = match signature('3') { + std::option::Option::Some(value) => value, + std::option::Option::None => return, + }; + let scope_result = crate::BackfillScope::explicit_signatures(std::vec![first.clone(), second.clone(), first]); + let scope = match scope_result { + std::result::Result::Ok(value) => value, + std::result::Result::Err(_) => return, + }; + let job_id = match ksp_job_api::JobId::new("backfill:explicit") { + std::result::Result::Ok(value) => value, + std::result::Result::Err(_) => return, + }; + let network = match ksp_store_lib::RawNetworkId::new("synthetic") { + std::result::Result::Ok(value) => value, + std::result::Result::Err(_) => return, + }; + let request_result = crate::BackfillRequest::new( + job_id, + network.clone(), + ksp_onchain_transport_lib::HttpRoleName::new("unused"), + crate::BackfillCommitment::Finalized, + scope, + 100, + 10, + 10, + 1, + std::option::Option::None, + ); + let request = match request_result { + std::result::Result::Ok(value) => value, + std::result::Result::Err(_) => return, + }; + let source = FakeSource::new(std::vec::Vec::new()); + let result = super::discover_with_source(&source, &request).await; + let discovery = match result { + std::result::Result::Ok(value) => value, + std::result::Result::Err(_) => return, + }; + assert_eq!(discovery.boundary(), crate::BackfillDiscoveryBoundary::ExplicitInput); + assert_eq!(discovery.pages_fetched(), 0); + assert_eq!(discovery.candidates().len(), 2); + assert_eq!(discovery.candidates()[0].identity().network(), &network); + assert!(source.calls().is_empty()); + return; +} diff --git a/crates/ksp-job-backfill-lib/unit_tests/request.rs b/crates/ksp-job-backfill-lib/unit_tests/request.rs new file mode 100644 index 0000000..87a8d8f --- /dev/null +++ b/crates/ksp-job-backfill-lib/unit_tests/request.rs @@ -0,0 +1,244 @@ +// file: crates/ksp-job-backfill-lib/unit_tests/request.rs +// version: 1 + +fn signature(character: char) -> std::option::Option { + return match crate::BackfillSignature::new(character.to_string().repeat(crate::MIN_BACKFILL_SIGNATURE_TEXT_BYTES)) { + std::result::Result::Ok(value) => std::option::Option::Some(value), + std::result::Result::Err(_) => std::option::Option::None, + }; +} + +fn network(value: &str) -> std::option::Option { + return match ksp_store_lib::RawNetworkId::new(value) { + std::result::Result::Ok(value) => std::option::Option::Some(value), + std::result::Result::Err(_) => std::option::Option::None, + }; +} + +fn job_id(value: &str) -> std::option::Option { + return match ksp_job_api::JobId::new(value) { + std::result::Result::Ok(value) => std::option::Option::Some(value), + std::result::Result::Err(_) => std::option::Option::None, + }; +} + +#[test] +fn pre_005_signature_shape_is_bounded_and_debug_redacted() { + let minimum = crate::BackfillSignature::new("1".repeat(crate::MIN_BACKFILL_SIGNATURE_TEXT_BYTES)); + assert!(minimum.is_ok()); + let maximum = crate::BackfillSignature::new("z".repeat(crate::MAX_BACKFILL_SIGNATURE_TEXT_BYTES)); + assert!(maximum.is_ok()); + assert!(crate::BackfillSignature::new("1".repeat(crate::MIN_BACKFILL_SIGNATURE_TEXT_BYTES - 1)).is_err()); + assert!(crate::BackfillSignature::new("1".repeat(crate::MAX_BACKFILL_SIGNATURE_TEXT_BYTES + 1)).is_err()); + assert!(crate::BackfillSignature::new(format!("{}0", "1".repeat(crate::MIN_BACKFILL_SIGNATURE_TEXT_BYTES - 1))).is_err()); + let minimum = match minimum { + std::result::Result::Ok(value) => value, + std::result::Result::Err(_) => return, + }; + let debug = format!("{minimum:?}"); + assert_eq!(debug, "BackfillSignature(..)"); + assert!(!debug.contains(minimum.as_str())); + return; +} + +#[test] +fn pre_005_explicit_scope_deduplicates_at_first_occurrence_without_reordering() { + let first = match signature('1') { + std::option::Option::Some(value) => value, + std::option::Option::None => return, + }; + let second = match signature('2') { + std::option::Option::Some(value) => value, + std::option::Option::None => return, + }; + let scope_result = crate::BackfillScope::explicit_signatures(std::vec![first.clone(), second.clone(), first.clone(), second.clone()]); + assert!(scope_result.is_ok()); + let scope = match scope_result { + std::result::Result::Ok(value) => value, + std::result::Result::Err(_) => return, + }; + let signatures = match scope.signatures() { + std::option::Option::Some(value) => value, + std::option::Option::None => return, + }; + assert_eq!(signatures, &[first, second]); + let debug = format!("{scope:?}"); + assert!(debug.contains("signature_count")); + assert!(!debug.contains(signatures[0].as_str())); + return; +} + +#[test] +fn pre_005_request_bounds_are_exact_and_explicit_context_is_rejected() { + let network = match network("devnet") { + std::option::Option::Some(value) => value, + std::option::Option::None => return, + }; + let job_id = match job_id("backfill:test") { + std::option::Option::Some(value) => value, + std::option::Option::None => return, + }; + let role = ksp_onchain_transport_lib::HttpRoleName::new("history"); + let address = ksp_core_lib::Pubkey::new_from_array([7_u8; 32]); + let scope = crate::BackfillScope::latest_address(address); + let valid = crate::BackfillRequest::new( + job_id.clone(), + network.clone(), + role.clone(), + crate::BackfillCommitment::Confirmed, + scope.clone(), + crate::MAX_BACKFILL_PAGE_SIZE, + crate::MAX_BACKFILL_PAGES, + crate::MAX_BACKFILL_CANDIDATES, + crate::MAX_BACKFILL_HYDRATION_CONCURRENCY, + std::option::Option::Some(1), + ); + assert!(valid.is_ok()); + for (page_size, max_pages, max_candidates, concurrency) in [ + (0, 1, 1, 1), + (crate::MAX_BACKFILL_PAGE_SIZE + 1, 1, 1, 1), + (1, 0, 1, 1), + (1, crate::MAX_BACKFILL_PAGES + 1, 1, 1), + (1, 1, 0, 1), + (1, 1, crate::MAX_BACKFILL_CANDIDATES + 1, 1), + (1, 1, 1, 0), + (1, 1, 1, crate::MAX_BACKFILL_HYDRATION_CONCURRENCY + 1), + ] { + let result = crate::BackfillRequest::new( + job_id.clone(), + network.clone(), + role.clone(), + crate::BackfillCommitment::Finalized, + scope.clone(), + page_size, + max_pages, + max_candidates, + concurrency, + std::option::Option::None, + ); + assert!(result.is_err()); + } + let explicit_signature = match signature('3') { + std::option::Option::Some(value) => value, + std::option::Option::None => return, + }; + let explicit_scope = crate::BackfillScope::explicit_signatures(std::vec![explicit_signature]); + let explicit_scope = match explicit_scope { + std::result::Result::Ok(value) => value, + std::result::Result::Err(_) => return, + }; + let explicit_with_context = crate::BackfillRequest::new( + job_id, + network, + role, + crate::BackfillCommitment::Confirmed, + explicit_scope, + 1, + 1, + 1, + 1, + std::option::Option::Some(10), + ); + assert!(explicit_with_context.is_err()); + return; +} + +#[test] +fn pre_005_scope_fingerprint_is_network_semantic_and_transport_source_independent() { + let signature = match signature('4') { + std::option::Option::Some(value) => value, + std::option::Option::None => return, + }; + let scope_result = crate::BackfillScope::explicit_signatures(std::vec![signature]); + let scope = match scope_result { + std::result::Result::Ok(value) => value, + std::result::Result::Err(_) => return, + }; + let mainnet = match network("mainnet") { + std::option::Option::Some(value) => value, + std::option::Option::None => return, + }; + let devnet = match network("devnet") { + std::option::Option::Some(value) => value, + std::option::Option::None => return, + }; + let first_job = match job_id("backfill:first") { + std::option::Option::Some(value) => value, + std::option::Option::None => return, + }; + let second_job = match job_id("backfill:second") { + std::option::Option::Some(value) => value, + std::option::Option::None => return, + }; + let first = crate::BackfillRequest::new( + first_job, + devnet.clone(), + ksp_onchain_transport_lib::HttpRoleName::new("primary-http"), + crate::BackfillCommitment::Confirmed, + scope.clone(), + 100, + 5, + 10, + 1, + std::option::Option::None, + ); + let second = crate::BackfillRequest::new( + second_job, + devnet, + ksp_onchain_transport_lib::HttpRoleName::new("fallback-http"), + crate::BackfillCommitment::Confirmed, + scope.clone(), + 100, + 5, + 10, + 64, + std::option::Option::None, + ); + let other_network = crate::BackfillRequest::new( + match job_id("backfill:third") { + std::option::Option::Some(value) => value, + std::option::Option::None => return, + }, + mainnet, + ksp_onchain_transport_lib::HttpRoleName::new("primary-http"), + crate::BackfillCommitment::Confirmed, + scope, + 100, + 5, + 10, + 1, + std::option::Option::None, + ); + let first = match first { + std::result::Result::Ok(value) => value, + std::result::Result::Err(_) => return, + }; + let second = match second { + std::result::Result::Ok(value) => value, + std::result::Result::Err(_) => return, + }; + let other_network = match other_network { + std::result::Result::Ok(value) => value, + std::result::Result::Err(_) => return, + }; + assert_eq!(first.scope_fingerprint(), second.scope_fingerprint()); + assert_ne!(first.scope_fingerprint(), other_network.scope_fingerprint()); + return; +} + +#[test] +fn pre_005_scope_kind_and_anchor_are_distinct_semantics() { + let anchor = match signature('5') { + std::option::Option::Some(value) => value, + std::option::Option::None => return, + }; + let address = ksp_core_lib::Pubkey::new_from_array([9_u8; 32]); + let before = crate::BackfillScope::before_address(address, anchor.clone()); + let after = crate::BackfillScope::after_address(address, anchor); + assert_eq!(before.kind(), crate::BackfillScopeKind::BeforeAddress); + assert_eq!(after.kind(), crate::BackfillScopeKind::AfterAddress); + assert!(before.anchor().is_some()); + assert!(after.anchor().is_some()); + assert_ne!(before, after); + return; +} diff --git a/deltas/0.3.6/pre.005.md b/deltas/0.3.6/pre.005.md new file mode 100644 index 0000000..7f08a13 --- /dev/null +++ b/deltas/0.3.6/pre.005.md @@ -0,0 +1,286 @@ + + + +# Delta `0.3.6-pre.005` — fondation Backfill et découverte bornée + +## Base requise + +```text +0.3.6-pre.004 appliquée +workspace.package.version = 0.3.6-pre.4 +``` + +Le gate opérateur fourni pour cette base confirme : + +```text +cargo fmt --all PASS +python3 scripts/audit_rust_workspace_rules.py PASS / clean +python3 scripts/audit_markdown_tables.py ... PASS / clean (264 tables / 146 fichiers) +cargo check --workspace PASS +cargo clippy --workspace --all-targets PASS +cargo test -p ksp-onchain-transport-lib PASS + unitaires 385 PASS + public_api 50 PASS + release_completeness 43 PASS + doctests 4 PASS +cargo tree -p ksp-onchain-transport-lib --edges normal exécuté +cargo tree -p ksp-onchain-transport-lib -e features exécuté +``` + +Les smokes réseau restent opt-in et ignorés dans le gate normal, conformément à leur contrat. + +## Objectif + +Matérialiser exclusivement la tranche `pre.005` du plan 027 : créer `ksp-job-backfill-lib` et figer la requête, les scopes, l'identité des candidats, les bornes ainsi que la découverte paginée déterministe avant toute hydratation `getTransaction`, conversion RAW, persistance Store, checkpoint ou runtime de Job. + +Cette tranche résout explicitement un invariant d'identité qui ne doit jamais dépendre de la topologie d'acquisition : une transaction/signature est identifiée logiquement par son réseau et sa signature, pas par le provider, l'endpoint ou le protocole qui l'a retrouvée. + +## Identité réseau contre provenance Transport + +Le contrat de découverte introduit : + +```text +BackfillCandidateIdentity = (RawNetworkId, BackfillSignature) +``` + +`RawNetworkId` est un identifiant logique ouvert réexporté par `ksp-store-lib`. Des namespaces comme `mainnet`, `devnet`, `testnet`, `localnet`, `synthetic` ou un futur réseau nommé restent donc distincts sans imposer un enum fermé dans Job. + +En conséquence : + +- la même signature observée sur deux réseaux différents représente deux identités distinctes ; +- la même signature acquise plusieurs fois sur le même réseau via HTTP, WebSocket ou gRPC converge vers la même identité transactionnelle ; +- rôle Transport, provider, endpoint et protocole décrivent la sélection ou la provenance d'acquisition ; ils ne participent jamais à l'identité transactionnelle ; +- `JobId` et la future concurrence d'hydratation ne participent pas non plus au fingerprint sémantique d'un scope. + +Le Store API possède déjà `RawTransactionReference { network, signature }`. Le backend PostgreSQL actuel lie en outre un Store physique à un seul `RawNetworkId` via `ksp_store_identity`; une clé physique locale basée sur la signature y reste donc dans un namespace réseau unique. Tout futur backend hébergeant plusieurs réseaux dans le même namespace physique devra préserver l'identité logique `(network, signature)` par sa clé, sa partition ou un mécanisme équivalent. + +## Crate `ksp-job-backfill-lib` + +La nouvelle crate est une bibliothèque comportementale indépendante, sans exécutable ni Worker. Ses dépendances normales sont limitées à : + +```text +ksp-core-lib +ksp-job-api +ksp-logging-lib +ksp-onchain-transport-lib +ksp-store-lib (default-features = false) +sha2 (workspace) +``` + +Tokio est uniquement une `dev-dependency` avec les features de test nécessaires aux canaries async. Aucun backend Store n'est activé par défaut par Backfill. + +La crate ne dépend pas directement de Config, Interface, Program, Wallet, Store API, Store PostgreSQL, reqwest, tonic ou d'une crate protocolaire Solana. + +## Requête bornée + +`BackfillRequest` impose explicitement : + +- `JobId` ; +- `RawNetworkId` ; +- rôle HTTP Transport ; +- engagement `Confirmed` ou `Finalized` ; +- un des quatre scopes ; +- page size `1..=1000` ; +- max pages `1..=10000` ; +- max candidates `1..=10000` ; +- future concurrence d'hydratation `1..=64` ; +- `min_context_slot` optionnel uniquement pour les scopes adresse. + +Le rôle HTTP sert uniquement à la sélection Transport. Il est volontairement exclu du fingerprint du scope avec provider, endpoint et protocole. + +## Scopes + +Quatre scopes sont matérialisés : + +- `LatestAddress(address)` : fenêtre courante depuis le plus récent ; +- `BeforeAddress(address, anchor)` : historique plus ancien que l'ancre exclusive ; +- `AfterAddress(address, anchor)` : fenêtre plus récente la plus proche d'une ancre exclusive ; +- `ExplicitSignatures(signatures)` : liste explicite bornée sans découverte adresse. + +Les signatures explicites sont dédupliquées selon la première occurrence et conservent leur ordre d'entrée. + +`BackfillSignature` valide dans cette tranche uniquement la forme texte Base58 bornée compatible avec une signature Solana de 64 octets. Le décodage Base58 exact vers la signature RAW canonique de 64 octets reste volontairement réservé à `pre.006`; `pre.005` ne prétend donc pas encore valider le contenu binaire canonique. + +## Fingerprint de scope + +`BackfillScopeFingerprint` utilise SHA-256 avec séparation de domaine et couvre les paramètres qui changent réellement la sémantique de découverte : + +- réseau ; +- commitment ; +- type de scope ; +- adresse ou liste explicite ; +- direction/ancre ; +- page size ; +- max pages ; +- max candidates ; +- `min_context_slot`. + +Il exclut intentionnellement : + +- `JobId` ; +- rôle HTTP ; +- provider ; +- endpoint ; +- protocole ; +- concurrence d'hydratation. + +Changer de source d'acquisition ne change donc pas l'identité du scope ni celle des transactions candidates. + +## Découverte paginée + +La voie de production appelle uniquement le wrapper typé existant : + +```text +HttpTransportPool::get_signatures_for_address +``` + +Aucun client HTTP, retry loop, pacing, cooldown, sélection d'endpoint ou politique fournisseur n'entre dans Job. + +Les règles figées sont : + +- Latest et Before utilisent le curseur `before` et conservent l'ordre RPC du plus récent au plus ancien ; +- Before démarre avec l'ancre exclusive fournie ; +- After utilise `until = anchor`, avance par `before` et ne conserve que les `max_candidates` les plus proches de l'ancre parmi les signatures plus récentes ; +- la déduplication est stable entre pages ; +- une page fournisseur plus grande que la limite demandée est rejetée ; +- un curseur qui n'avance pas est rejeté ; +- une page courte ou vide matérialise `RpcBoundary` ; +- atteindre la limite de candidats matérialise `CandidateLimit` ; +- épuiser `max_pages` en Latest/Before matérialise `PageLimit` et un résultat partiel ; +- épuiser `max_pages` en After avant preuve de la frontière matérialise `AfterAnchorNotReached` et un résultat partiel. + +La borne After est volontairement conservative : un Job ne prétend jamais avoir rejoint l'ancre si la limite de pages empêche de le prouver. + +## Tests matérialisés + +La tranche ajoute **11 tests unitaires** : + +- validation et redaction de signature ; +- déduplication explicite stable ; +- bornes exactes de requête ; +- fingerprint indépendant de JobId/rôle/concurrence mais distinct par réseau ; +- distinction sémantique Before/After ; +- Latest paginé avec déduplication inter-pages ; +- Before avec ancre exclusive et progression du curseur ; +- After avec fenêtre la plus proche de l'ancre ; +- After borné partiel ; +- Latest borné partiel ; +- explicite sans appel Transport et avec identité réseau. + +Elle ajoute **6 canaries d'intégration** : + +- deux canaries de dépendances/firewalls ; +- deux canaries d'API publique, dont l'identité `(network, signature)` ; +- deux canaries de complétude qui empêchent l'ouverture anticipée de RAW/persistance/checkpoint/runtime. + +Les doubles de page sont privés aux unit tests ; aucune abstraction fake n'entre dans l'API publique. + +## Documentation courante réconciliée + +Les documents d'architecture actifs sont alignés avec le nom désormais matérialisé `ksp-job-backfill-lib` et son graphe réel sans Config : + +```text +docs/architecture/004-COMPONENT_INVENTORY.md +docs/architecture/005-DEPENDENCY_GRAPH.md +docs/architecture/009-ACQUISITION_WORKERS_AND_JOBS.md +``` + +Les anciens plans historiques ne sont pas réécrits. README, USAGE, CHANGELOG et ROADMAP restent fermés jusqu'aux tranches prévues de réconciliation/fermeture. + +## Fichiers ajoutés + +```text +crates/ksp-job-backfill-lib/Cargo.toml +crates/ksp-job-backfill-lib/src/constants.rs +crates/ksp-job-backfill-lib/src/discovery.rs +crates/ksp-job-backfill-lib/src/error.rs +crates/ksp-job-backfill-lib/src/lib.rs +crates/ksp-job-backfill-lib/src/request.rs +crates/ksp-job-backfill-lib/tests/dependency_boundary.rs +crates/ksp-job-backfill-lib/tests/public_api.rs +crates/ksp-job-backfill-lib/tests/release_completeness.rs +crates/ksp-job-backfill-lib/unit_tests/discovery.rs +crates/ksp-job-backfill-lib/unit_tests/request.rs +deltas/0.3.6/pre.005.md +``` + +## Fichiers modifiés + +```text +Cargo.toml +docs/architecture/004-COMPONENT_INVENTORY.md +docs/architecture/005-DEPENDENCY_GRAPH.md +docs/architecture/009-ACQUISITION_WORKERS_AND_JOBS.md +docs/plans/027-V0_3_6_JOB_API_BACKFILL_PLAN.md +docs/validation/023-V0_3_6_JOB_API_BACKFILL.md +``` + +Mécanique Cargo : + +```text +header version: 399 -> 400 +workspace.package.version: 0.3.6-pre.4 -> 0.3.6-pre.5 +workspace members: + crates/ksp-job-backfill-lib +workspace dependencies versionnées: inchangées +``` + +## Fichiers supprimés + +Aucun. + +## Frontières conservées + +- aucun code kbot3 n'est copié ; kbot3 reste uniquement une référence fonctionnelle ; +- aucune hydratation `getTransaction` n'est encore implémentée dans Job ; +- aucune conversion RAW v1 n'est ouverte ; +- aucune écriture Store, observation d'acquisition ou politique `ForceRehydrate` n'est ouverte ; +- aucun checkpoint, frontier contiguë, runtime Job ou notification concrète n'est ouvert ; +- aucune lecture Config/env n'est introduite ; +- aucun endpoint/provider/protocole ne devient une clé transactionnelle ; +- aucun backend Store n'est imposé ; +- aucun README/USAGE/CHANGELOG/ROADMAP n'est rouvert. + +## Validations exécutées dans l'environnement d'assemblage + +```text +python3 scripts/audit_rust_workspace_rules.py +-> General Rust rule audit: clean +-> Rust export completeness audit: 0 candidate(s) +-> KSP workspace Rust rule audit: clean + +python3 scripts/audit_markdown_tables.py README.md RULES.md ROADMAP.md CHANGELOG.md docs prompts crates deltas/0.3.6 +-> Markdown table audit: clean (264 tables / 147 fichiers, présent delta inclus) + +python3 -m unittest scripts/tests/test_audit_markdown_tables.py +-> 5 tests / OK +``` + +Le premier audit Markdown après réconciliation de l'inventaire a détecté uniquement l'élargissement mécanique requis par `ksp-job-backfill-lib`; la table a été réalignée sur le contenu le plus large avant le second passage propre. + +L'environnement d'assemblage ne fournit ni `cargo`, ni `rustc`, ni `rustfmt`. Les 11 unitaires et 6 canaries Rust sont matérialisés mais ne sont pas annoncés comme exécutés localement. + +## Gate opérateur demandé + +```bash +cargo fmt --all +python3 scripts/audit_rust_workspace_rules.py +python3 scripts/audit_markdown_tables.py README.md RULES.md ROADMAP.md CHANGELOG.md docs prompts crates deltas/0.3.6 +cargo check --workspace +cargo clippy --workspace --all-targets +cargo test -p ksp-job-backfill-lib +cargo tree -p ksp-job-backfill-lib --edges normal +cargo tree -p ksp-job-backfill-lib -e features +``` + +Le gate doit notamment confirmer : + +- 11 unitaires + 6 canaries ; +- `ksp-store-lib` sans activation implicite du backend PostgreSQL ; +- aucun accès direct à Store API/backend, Config ou dépendance protocolaire ; +- pagination Latest/Before/After et explicite déterministe ; +- résultat partiel conservateur aux bornes ; +- identité candidate scellée par réseau + signature, indépendamment du rôle/provider/endpoint/protocole. + +## Questions ouvertes + +Aucune question ne bloque `pre.006` après un gate opérateur vert de `pre.005`. diff --git a/docs/architecture/004-COMPONENT_INVENTORY.md b/docs/architecture/004-COMPONENT_INVENTORY.md index 8541bcb..c430d37 100644 --- a/docs/architecture/004-COMPONENT_INVENTORY.md +++ b/docs/architecture/004-COMPONENT_INVENTORY.md @@ -1,5 +1,5 @@ - + # Inventaire initial des composants KSP @@ -17,43 +17,43 @@ Ce document maintient l'inventaire synthétique des composants retenus ou presse ## Inventaire synthétique -| Domaine | Composant | Type | Statut | Première cible actuelle | Mission | -|-------------------------|------------------------------------------|--------------------|--------------|---------------------------------|---------------------------------------------------------------------------------------------| -| Core | `ksp-core-lib` | lib | Stable | `0.1.1` | Error/Result, Program IDs et primitives fondamentales | -| Logging | `ksp-logging-lib` | lib | Stable | `0.1.2` | façade unique tracing KSP | -| Config | `ksp-config-lib` | lib | Stable | `0.1.3` | documents, profils, env et persistence Config | -| Config Desk | `ksp-app-config-desk` | app | Stable | `0.1.4` | validation/management Config | -| On-chain HTTP | `ksp-onchain-transport-lib` | lib | Stable | `0.2.1`–`0.2.4` | HTTP standard complet : 52/52 current + 14/14 historical | -| Wallet | `ksp-wallet-lib` | lib | Stable | `0.2.5` | `.kspwallet`, VIEW/OWNER, secrets, signature, import/export | -| Wallet Desk | `ksp-app-wallet-desk` | app | Stable | `0.2.6` | Wallet + Config + balance HTTP + projection SOL/USD auxiliaire | -| Wallet V2 | `ksp-wallet-lib` | lib | Stable | `0.2.6` | wire/runtime V2 + API default/versionnée + migration explicite | -| Standard WS | `ksp-onchain-transport-lib` | lib | Stable | `0.2.7` | WebSocket Solana 18/18, sessions/subscriptions bornées | -| Helius WS | `ksp-onchain-transport-lib` | lib | Stable | `0.2.8` | LaserStream WS : 7 standard + transaction, actor partagé | -| Yellowstone | `ksp-onchain-transport-lib` | lib | Stable | `0.2.9` | client gRPC standard/provider-neutral | -| Off-chain price | `ksp-offchain-transport-lib` | lib | Stable | `0.2.11` | prix SOL/USD multi-provider, limits et availability | -| SOL Prices Desk | `ksp-app-solprices-desk` | app | Stable | `0.2.12` | HID provider-agnostic pour visualisation/refresh prix | -| Interface passive | `ksp-interface-lib` | lib | Stable | `0.2.13` | façade wire + contrats passifs partagés, dont événements acquisition provider-neutral | -| Program API | `ksp-program-api` | API | Stable | `0.2.14` | contrats extensibles Program | -| Program impl. | `ksp-program-lib` | lib | Retenu | vertical slices ultérieurs | implementations Program officielles | -| Program extension | `ksp-program--lib` | lib externe | À la demande | dès besoin | implementation externe de `ksp-program-api` | -| Store API | `ksp-store-api` | API | Retenu | `0.3.1` | RAW transaction/account, observations, queries, outcomes, rétention et capabilities backend | -| Store runtime | `ksp-store-lib` | lib | Retenu | `0.3.2`–`0.3.4` | fondation puis conformance RAW par slices, dispatch features/config | -| Store PostgreSQL | `ksp-store-postgres-lib` | lib | Retenu | `0.3.2`–`0.3.4` | fondation, RawTransaction puis RawAccountState/complétude | -| Job lifecycle | `ksp-job-api` | API | Retenu | `0.3.6` | lifecycle des jobs terminables | -| Backfill | `ksp-job-backfill` | job/lib à préciser | Retenu | `0.3.6` | acquisition historique vers RAW via `ksp-store-lib` | -| Backfill Desk | nom à fixer | app | Retenu | `0.3.7` | contrôle/inspection du backfill RAW | -| Worker lifecycle | `ksp-worker-api` | API | Retenu | fin couche RAW | lifecycle des services continus | -| RAW worker | `ksp-worker-raw-retriever` ou nom révisé | worker | Retenu | fin couche RAW | acquisition live vers RAW | -| CORE processor | nom à fixer | processor/lib | Retenu | couche CORE | normalisation Solana générique RAW -> CORE | -| CORE worker | nom à fixer | worker | Retenu | fin couche CORE | backlog RAW -> CORE continu | -| Materializer API | `ksp-materializer-api` | API | Retenu | premier groupe DECODE | contrats extensibles matérialisation | -| Materializer impl. | `ksp-materializer-lib` | lib | Retenu | premier groupe DECODE | implementations officielles communes | -| Execution policy | `ksp-execution-policy-api` | API | Retenu | premier vrai besoin execution | décision/safety multi-contexte | -| Execution orchestration | `ksp-execution-lib` | lib | Retenu | premier vrai cycle execution | Program + policy + Wallet + transport | -| Scenarios | `ksp-scenario--lib` | lib | Retenu | vertical slices | validation métier/devnet par groupe | -| Scenario API | `ksp-scenario-api` | API | Non retenu | — | norme souple avant trait commun | -| Market Desk | `ksp-app-market-desk` | app | Pressenti | après Meteora/Raydium/Pump/Orca | tokens, pools, trades, liquidity, price, OHLC | -| Trading Intelligence | noms à définir | libs/jobs | Pressenti | après données stables | features/signaux/anomalies/ML | +| Domaine | Composant | Type | Statut | Première cible actuelle | Mission | +|-------------------------|------------------------------------------|---------------|--------------|---------------------------------|---------------------------------------------------------------------------------------------| +| Core | `ksp-core-lib` | lib | Stable | `0.1.1` | Error/Result, Program IDs et primitives fondamentales | +| Logging | `ksp-logging-lib` | lib | Stable | `0.1.2` | façade unique tracing KSP | +| Config | `ksp-config-lib` | lib | Stable | `0.1.3` | documents, profils, env et persistence Config | +| Config Desk | `ksp-app-config-desk` | app | Stable | `0.1.4` | validation/management Config | +| On-chain HTTP | `ksp-onchain-transport-lib` | lib | Stable | `0.2.1`–`0.2.4` | HTTP standard complet : 52/52 current + 14/14 historical | +| Wallet | `ksp-wallet-lib` | lib | Stable | `0.2.5` | `.kspwallet`, VIEW/OWNER, secrets, signature, import/export | +| Wallet Desk | `ksp-app-wallet-desk` | app | Stable | `0.2.6` | Wallet + Config + balance HTTP + projection SOL/USD auxiliaire | +| Wallet V2 | `ksp-wallet-lib` | lib | Stable | `0.2.6` | wire/runtime V2 + API default/versionnée + migration explicite | +| Standard WS | `ksp-onchain-transport-lib` | lib | Stable | `0.2.7` | WebSocket Solana 18/18, sessions/subscriptions bornées | +| Helius WS | `ksp-onchain-transport-lib` | lib | Stable | `0.2.8` | LaserStream WS : 7 standard + transaction, actor partagé | +| Yellowstone | `ksp-onchain-transport-lib` | lib | Stable | `0.2.9` | client gRPC standard/provider-neutral | +| Off-chain price | `ksp-offchain-transport-lib` | lib | Stable | `0.2.11` | prix SOL/USD multi-provider, limits et availability | +| SOL Prices Desk | `ksp-app-solprices-desk` | app | Stable | `0.2.12` | HID provider-agnostic pour visualisation/refresh prix | +| Interface passive | `ksp-interface-lib` | lib | Stable | `0.2.13` | façade wire + contrats passifs partagés, dont événements acquisition provider-neutral | +| Program API | `ksp-program-api` | API | Stable | `0.2.14` | contrats extensibles Program | +| Program impl. | `ksp-program-lib` | lib | Retenu | vertical slices ultérieurs | implementations Program officielles | +| Program extension | `ksp-program--lib` | lib externe | À la demande | dès besoin | implementation externe de `ksp-program-api` | +| Store API | `ksp-store-api` | API | Retenu | `0.3.1` | RAW transaction/account, observations, queries, outcomes, rétention et capabilities backend | +| Store runtime | `ksp-store-lib` | lib | Retenu | `0.3.2`–`0.3.4` | fondation puis conformance RAW par slices, dispatch features/config | +| Store PostgreSQL | `ksp-store-postgres-lib` | lib | Retenu | `0.3.2`–`0.3.4` | fondation, RawTransaction puis RawAccountState/complétude | +| Job lifecycle | `ksp-job-api` | API | Retenu | `0.3.6` | lifecycle des jobs terminables | +| Backfill | `ksp-job-backfill-lib` | lib | Retenu | `0.3.6` | Job borné : découverte/hydratation historique vers RAW via Transport + `ksp-store-lib` | +| Backfill Desk | nom à fixer | app | Retenu | `0.3.7` | contrôle/inspection du backfill RAW | +| Worker lifecycle | `ksp-worker-api` | API | Retenu | fin couche RAW | lifecycle des services continus | +| RAW worker | `ksp-worker-raw-retriever` ou nom révisé | worker | Retenu | fin couche RAW | acquisition live vers RAW | +| CORE processor | nom à fixer | processor/lib | Retenu | couche CORE | normalisation Solana générique RAW -> CORE | +| CORE worker | nom à fixer | worker | Retenu | fin couche CORE | backlog RAW -> CORE continu | +| Materializer API | `ksp-materializer-api` | API | Retenu | premier groupe DECODE | contrats extensibles matérialisation | +| Materializer impl. | `ksp-materializer-lib` | lib | Retenu | premier groupe DECODE | implementations officielles communes | +| Execution policy | `ksp-execution-policy-api` | API | Retenu | premier vrai besoin execution | décision/safety multi-contexte | +| Execution orchestration | `ksp-execution-lib` | lib | Retenu | premier vrai cycle execution | Program + policy + Wallet + transport | +| Scenarios | `ksp-scenario--lib` | lib | Retenu | vertical slices | validation métier/devnet par groupe | +| Scenario API | `ksp-scenario-api` | API | Non retenu | — | norme souple avant trait commun | +| Market Desk | `ksp-app-market-desk` | app | Pressenti | après Meteora/Raydium/Pump/Orca | tokens, pools, trades, liquidity, price, OHLC | +| Trading Intelligence | noms à définir | libs/jobs | Pressenti | après données stables | features/signaux/anomalies/ML | ## Contrats séparés retenus diff --git a/docs/architecture/005-DEPENDENCY_GRAPH.md b/docs/architecture/005-DEPENDENCY_GRAPH.md index d0bef04..4594b01 100644 --- a/docs/architecture/005-DEPENDENCY_GRAPH.md +++ b/docs/architecture/005-DEPENDENCY_GRAPH.md @@ -1,5 +1,5 @@ - + # Graphe de dépendances KSP @@ -367,18 +367,18 @@ ksp-job-api -> ksp-core-lib ``` -Premier job pressenti : +Premier job concret : ```text -ksp-job-backfill +ksp-job-backfill-lib -> ksp-job-api - -> ksp-onchain-transport-lib - -> ksp-store-lib - -> ksp-config-lib + -> ksp-core-lib -> ksp-logging-lib + -> ksp-onchain-transport-lib + -> ksp-store-lib # façade backend-neutral ; dépendance sans feature backend imposée ``` -Il remplit RAW et ne décode rien. +Il remplit RAW et ne décode aucun programme. Il ne dépend pas de Config : l'application supérieure construit explicitement Transport, Store et la requête Job. Le réseau appartient au scope/à l'identité durable ; rôle, provider, endpoint et protocole restent des choix ou provenances d'acquisition et ne deviennent jamais une clé de transaction. ## Workers diff --git a/docs/architecture/009-ACQUISITION_WORKERS_AND_JOBS.md b/docs/architecture/009-ACQUISITION_WORKERS_AND_JOBS.md index e077666..d6dc6ae 100644 --- a/docs/architecture/009-ACQUISITION_WORKERS_AND_JOBS.md +++ b/docs/architecture/009-ACQUISITION_WORKERS_AND_JOBS.md @@ -1,5 +1,5 @@ - + # Acquisition, workers, jobs et pipelines spécialisés @@ -59,7 +59,7 @@ Une crate spécialisée `ksp-pipeline-raw-ingestion-lib` peut être introduite l Elle ne choisit pas le provider réseau et ne pilote pas le range historique. -### `ksp-job-backfill` +### `ksp-job-backfill-lib` Le premier backfill historique appartient à la couche RAW : @@ -80,6 +80,8 @@ Le job : - utilise `ksp-job-api` pour son lifecycle ; - gère scope/range/pagination/checkpoint ; +- porte explicitement le réseau logique du Store dans le scope et dans l'identité de chaque transaction candidate ; +- traite rôle HTTP, provider, endpoint et protocole comme sélection/provenance d'acquisition, jamais comme identité transactionnelle ; - n'effectue aucun décodage Program ; - n'écrit pas directement des faits CORE/DECODE/SPECIALIZED. @@ -339,12 +341,12 @@ Les événements utiles comprennent notamment : ### RAW backfill ```text -ksp-job-backfill +ksp-job-backfill-lib -> ksp-job-api - -> ksp-onchain-transport-lib - -> ksp-store-lib # façade Store ; backend sélectionné par feature + Config - -> ksp-config-lib # orchestration/config, pas ownership transport + -> ksp-core-lib -> ksp-logging-lib + -> ksp-onchain-transport-lib + -> ksp-store-lib # façade Store ; aucun backend imposé par la crate Job ``` ### RAW worker diff --git a/docs/plans/027-V0_3_6_JOB_API_BACKFILL_PLAN.md b/docs/plans/027-V0_3_6_JOB_API_BACKFILL_PLAN.md index de3f693..0c96cc5 100644 --- a/docs/plans/027-V0_3_6_JOB_API_BACKFILL_PLAN.md +++ b/docs/plans/027-V0_3_6_JOB_API_BACKFILL_PLAN.md @@ -1,5 +1,5 @@ - + # Plan v0.3.6 — Job API et premier backfill RAW @@ -226,7 +226,11 @@ Chaque transaction disponible produit : - une `RawAcquisitionProvenance` avec fournisseur, protocole Solana HTTP JSON-RPC, méthode `getTransaction`, endpoint sûr, engagement, instant de réception et `JobId` comme session de capture ; - une `RawTransactionObservation` dont la clé déterministe est un SHA-256 domain-separated du `JobId`, du fingerprint de scope, de la signature, du fournisseur, de l'endpoint, de l'engagement et de la version du contrat d'acquisition. -Une reprise du même Job par le même endpoint retrouve donc l'observation. Un autre endpoint constitue une nouvelle acquisition légitime. Une différence de payload canonique pour la même référence reste un conflit Store, jamais un skip. +L'identité transactionnelle logique est donc strictement `(RawNetworkId, signature)` : `mainnet`, `devnet`, `testnet`, `localnet`, `synthetic` ou tout autre réseau logique validé constituent des namespaces distincts. Le rôle Transport, le provider, l'endpoint et le protocole HTTP/WS/gRPC n'entrent jamais dans cette identité. Plusieurs acquisitions du même réseau et de la même signature convergent vers la même transaction canonique, quelle que soit leur voie d'acquisition ; elles peuvent en revanche produire des observations de provenance distinctes. + +Le backend PostgreSQL actuel lie un Store physique à exactement un `RawNetworkId` via `ksp_store_identity`. Son index physique peut donc rester local au namespace réseau. Tout backend futur capable d'héberger plusieurs réseaux dans un même namespace physique doit inclure le réseau dans sa clé, sa partition ou un mécanisme équivalent garantissant la même identité logique `(network, signature)`. + +Le fingerprint de scope inclut le réseau et les paramètres sémantiques, mais exclut rôle HTTP, provider, endpoint et protocole. Une reprise du même Job par le même endpoint retrouve donc l'observation. Un autre endpoint constitue une nouvelle acquisition légitime. Une différence de payload canonique pour la même référence reste un conflit Store, jamais un skip. ## 13. Persistance et idempotence @@ -422,17 +426,21 @@ Le gate opérateur de `pre.003` confirme `cargo fmt`, les audits Rust/Markdown e ### `pre.004` — Provenance Transport observée -**Statut : matérialisé ; gate opérateur à rejouer.** +**Statut : clôturé ; gate opérateur vert.** Budget cible : **15-20 min**. Entrée : besoin de provenance confirmé par le plan et gate `pre.003-fix.001` vert. La tranche ajoute `HttpObservedValue` comme enveloppe typée ne conservant que la valeur, le nom sûr de l'endpoint victorieux et son provider. `HttpTransportPool::get_transaction_observed` reprend exactement les validations et le moteur `execute_standard_rpc` existants ; le moteur commun possède désormais une voie interne observée et l'API historique continue à ne retourner que la valeur. -Les tests couvrent le succès direct `null`, la surface publique et surtout un retry `429` entre deux endpoints de même priorité : le résultat observé rapporte le second endpoint/provider qui a réellement produit la réponse, jamais le candidat initial. URL, headers et body HTTP brut restent absents du contrat, et le `Debug` de l'enveloppe ne rend pas la valeur typée. Sortie attendue après gate : routage/retry/provenance Transport verts sans nouvelle dépendance ni changement de politique. +Les tests couvrent le succès direct `null`, la surface publique et surtout un retry `429` entre deux endpoints de même priorité : le résultat observé rapporte le second endpoint/provider qui a réellement produit la réponse, jamais le candidat initial. URL, headers et body HTTP brut restent absents du contrat, et le `Debug` de l'enveloppe ne rend pas la valeur typée. Le gate opérateur est vert : audits, `cargo check`, Clippy, 385 unitaires Transport, 50 canaries publiques, 43 canaries de complétude, doctests et arbres Cargo passent sans nouvelle dépendance/feature Transport. ### `pre.005` — Fondation Backfill et découverte -**Statut : planifié.** +**Statut : matérialisé ; gate opérateur à rejouer.** -Budget cible : **15-20 min**. Entrée : Transport observé stable. Créer la crate, les requêtes, scopes, validations, candidats et pagination. Sortie : fixtures latest, before, after et explicite, avec déduplication stable. +Budget cible : **15-20 min**. Entrée : Transport observé stable et gate `pre.004` vert. La tranche crée `ksp-job-backfill-lib` avec dépendances directes Job API, Core, Logging, Onchain Transport, Store façade sans feature backend imposée et SHA-256 déjà centralisé au workspace ; Tokio reste uniquement une dev-dependency pour les tests async de cette tranche. + +La requête impose explicitement `JobId`, `RawNetworkId`, rôle HTTP, engagement `Confirmed|Finalized`, scope, page size `1..=1000`, pages `1..=10000`, candidats `1..=10000`, future concurrence d'hydratation `1..=64` et `min_context_slot` seulement pour les scopes adresse. Les quatre scopes sont matérialisés. Les signatures explicites sont bornées, Base58-shaped et dédupliquées à première occurrence ; le décodage exact vers 64 octets reste volontairement `pre.006`. + +`BackfillCandidateIdentity` est explicitement `(RawNetworkId, BackfillSignature)` ; rôle/provider/endpoint/protocole sont exclus du fingerprint de scope et ne peuvent donc pas devenir une identité transactionnelle. La découverte réelle appelle uniquement `HttpTransportPool::get_signatures_for_address`, sans client, retry, pacing ou endpoint policy dans Job. Latest/Before paginent par `before`, After conserve la fenêtre plus récente la plus proche de l'ancre via `until`, et les limites de pages produisent un résultat partiel observable plutôt qu'une fausse complétion. Les doubles déterministes restent privés aux tests. La tranche matérialise 11 tests unitaires et 6 canaries d’intégration ; leur exécution Cargo reste une preuve opérateur. Sortie attendue après gate : latest, before, after et explicite, déduplication stable, frontières partielles et firewall de dépendances verts. ### `pre.006` — Conversion RAW v1 et provenance diff --git a/docs/validation/023-V0_3_6_JOB_API_BACKFILL.md b/docs/validation/023-V0_3_6_JOB_API_BACKFILL.md index 5c3c6d7..f6f8edd 100644 --- a/docs/validation/023-V0_3_6_JOB_API_BACKFILL.md +++ b/docs/validation/023-V0_3_6_JOB_API_BACKFILL.md @@ -1,5 +1,5 @@ - + # Validation v0.3.6 — Job API et premier backfill RAW @@ -55,15 +55,15 @@ Aucune entrée absolue, traversée, avec séparateur inversé ou lien symbolique - [X] Les arbres Cargo normal/features de ce gate confirment que `ksp-job-api` dépend toujours uniquement de `ksp-core-lib` et n'ouvre aucune feature. - [X] `pre.003-fix.001` remplace uniquement la construction via le re-export racine par le constructeur privé directement visible dans le module `notification`, sans changer l'API publique ni le nombre de tests. - [X] Gate opérateur de `pre.003-fix.001` : `cargo fmt`, audits Rust/Markdown, `cargo check`, Clippy, treize unitaires + quatorze canaries `ksp-job-api` et arbres Cargo normal/features verts. -- [X] `pre.004` matérialise une voie observée additive de `getTransaction` sans modifier le client, les settings, le routage ni la politique de retry Transport. -- [X] Le workspace compte 14 crates et aucune crate Job au point de départ. +- [X] `pre.004` matérialise une voie observée additive de `getTransaction` sans modifier le client, les settings, le routage ni la politique de retry Transport ; gate opérateur vert. +- [X] Le workspace comptait 14 crates avant `pre.005`; `ksp-job-backfill-lib` devient la 15e crate membre sans créer d'exécutable ni de Worker. ## 4. Autorités et cohérence documentaire - [X] `RULES.md` et la hiérarchie documentaire ont été appliqués. - [X] Les chemins canoniques réels sous `docs/rules/` ont été retenus lorsque le prompt utilisait un chemin abrégé. - [X] Le nom fixé est `ksp-job-backfill-lib`. -- [X] Les occurrences historiques de `ksp-job-backfill` sont identifiées pour réconciliation avant fermeture. +- [X] Les documents d'architecture courants sont réconciliés sur `ksp-job-backfill-lib`; les anciens plans historiques restent historiques et ne sont pas réécrits. - [X] `ksp-job-api` et `ksp-job-backfill-lib` sont obligatoires en parallèle dans la release finale. - [X] Worker API, application et pipeline RAW partagé sont explicitement hors v0.3.6. - [ ] Architecture, README, roadmap et contrats finaux réconciliés avant publication. @@ -97,6 +97,8 @@ Aucune entrée absolue, traversée, avec séparateur inversé ou lien symbolique ## 7. Admission et portées +`pre.005` matérialise les contrats et leurs tests, mais les critères comportementaux ci-dessous restent ouverts jusqu'au gate Cargo opérateur. L'identité candidate est auditée statiquement comme `(RawNetworkId, signature)` ; rôle HTTP, provider, endpoint et protocole ne participent ni à cette identité ni au fingerprint sémantique du scope. + - [ ] `LatestAddress`, `BeforeAddress`, `AfterAddress` et `ExplicitSignatures` couverts. - [ ] Adresse validée par le type Core existant. - [ ] Signature Base58 décodée vers exactement 64 octets. @@ -123,8 +125,8 @@ Aucune entrée absolue, traversée, avec séparateur inversé ou lien symbolique ## 9. Acquisition Transport et retry - [ ] Les wrappers typés officiels restent l'unique chemin RPC. -- [X] Retour observé additif de `getTransaction` matérialisé ; gate opérateur à rejouer. -- [X] Canarie déterministe : un `429` sur le premier endpoint reroute et rapporte le second provider/endpoint victorieux ; gate opérateur à rejouer. +- [X] Retour observé additif de `getTransaction` matérialisé et gate opérateur vert. +- [X] Canarie déterministe : un `429` sur le premier endpoint reroute et rapporte le second provider/endpoint victorieux ; gate opérateur vert. - [X] `HttpObservedValue` ne contient que valeur typée, endpoint sûr et provider ; URL, headers et body HTTP brut restent privés. - [ ] Aucun retry, pacing ou sélection d'endpoint dans Job. - [ ] Aucune détection d'erreur par chaîne. @@ -132,6 +134,8 @@ Aucune entrée absolue, traversée, avec séparateur inversé ou lien symbolique ## 10. RAW v1 et provenance +- [X] Identité logique future auditée : transaction/signature scoping par `RawNetworkId`, jamais par rôle/provider/endpoint/protocole. +- [X] Le Store PostgreSQL actuel est mono-réseau via `ksp_store_identity`; un futur backend multi-réseaux devra préserver `(network, signature)` par clé/partition équivalente. - [ ] Format `ksp.solana.raw_transaction`, version `1`, figé. - [ ] `getTransaction` demandé en `base64` avec version maximale `0`. - [ ] Bytes JSON compacts déterministes et golden tests stables. @@ -183,12 +187,13 @@ Aucune entrée absolue, traversée, avec séparateur inversé ou lien symbolique ## 14. Firewalls et qualité Rust - [X] `ksp-job-api` dépend exactement de Core et n'a aucune feature, dev-dependency ou build-dependency. -- [ ] `ksp-job-backfill-lib` ne dépend pas de Config, App, Wallet, Program, Interface ou backend Store. +- [X] `ksp-job-backfill-lib` ne dépend pas de Config, App, Wallet, Program, Interface ou backend Store ; `ksp-store-lib` est consommée avec `default-features = false`. - [ ] Aucun retour de dépendance vers Job depuis les couches basses. -- [ ] Versions et features des nouvelles dépendances auditées avant ajout. +- [X] Versions/features auditées : aucune nouvelle version externe ; `sha2` et `tokio` réutilisent les entrées workspace existantes, Tokio étant dev-only en `pre.005`. +- [X] `pre.005` matérialise 11 tests unitaires et 6 canaries d’intégration ; leur exécution Cargo reste ouverte jusqu’au gate opérateur. - [ ] `cargo fmt --all -- --check` vert. -- [X] `scripts/audit_rust_workspace_rules.py` vert sur `pre.003` dans l'environnement d'assemblage. -- [X] `scripts/audit_markdown_tables.py` vert sur `pre.003` dans l'environnement d'assemblage (264 tables / 144 fichiers). +- [X] `scripts/audit_rust_workspace_rules.py` vert sur `pre.005` dans l'environnement d'assemblage. +- [X] `scripts/audit_markdown_tables.py` vert sur `pre.005` dans l'environnement d'assemblage (264 tables / 147 fichiers, delta inclus). - [X] Contrats de fichiers de `FILE_CONTRACTS.md` vérifiés sur chaque ajout de `pre.001`. - [ ] `cargo check --workspace` vert. - [ ] `cargo clippy --workspace --all-targets --all-features -- -D warnings` vert.