v0.3.6-pre.005
This commit is contained in:
22
crates/ksp-job-backfill-lib/Cargo.toml
Normal file
22
crates/ksp-job-backfill-lib/Cargo.toml
Normal file
@@ -0,0 +1,22 @@
|
||||
# file: crates/ksp-job-backfill-lib/Cargo.toml
|
||||
# version: 1
|
||||
|
||||
[package]
|
||||
name = "ksp-job-backfill-lib"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
repository.workspace = true
|
||||
|
||||
[dependencies]
|
||||
ksp-core-lib = { path = "../ksp-core-lib" }
|
||||
ksp-job-api = { path = "../ksp-job-api" }
|
||||
ksp-logging-lib = { path = "../ksp-logging-lib" }
|
||||
ksp-onchain-transport-lib = { path = "../ksp-onchain-transport-lib" }
|
||||
ksp-store-lib = { path = "../ksp-store-lib", default-features = false }
|
||||
sha2.workspace = true
|
||||
|
||||
[dev-dependencies]
|
||||
tokio = { workspace = true, features = ["macros", "rt-multi-thread"] }
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
7
crates/ksp-job-backfill-lib/src/constants.rs
Normal file
7
crates/ksp-job-backfill-lib/src/constants.rs
Normal file
@@ -0,0 +1,7 @@
|
||||
// file: crates/ksp-job-backfill-lib/src/constants.rs
|
||||
// version: 1
|
||||
|
||||
//! Backfill runtime-owned constants.
|
||||
|
||||
/// Owning tracing target for the concrete bounded RAW backfill runtime.
|
||||
pub(crate) const TRACING_TARGET: &str = "ksp-job-backfill-lib";
|
||||
443
crates/ksp-job-backfill-lib/src/discovery.rs
Normal file
443
crates/ksp-job-backfill-lib/src/discovery.rs
Normal file
@@ -0,0 +1,443 @@
|
||||
// file: crates/ksp-job-backfill-lib/src/discovery.rs
|
||||
// version: 1
|
||||
|
||||
/// Network-scoped identity of one discovered transaction candidate before canonical signature decoding.
|
||||
#[derive(Clone, Debug, Eq, Hash, PartialEq)]
|
||||
pub struct BackfillCandidateIdentity {
|
||||
network: ksp_store_lib::RawNetworkId,
|
||||
signature: crate::BackfillSignature,
|
||||
}
|
||||
|
||||
impl BackfillCandidateIdentity {
|
||||
/// Creates one candidate identity from its logical network and encoded transaction signature.
|
||||
#[must_use]
|
||||
pub fn new(network: ksp_store_lib::RawNetworkId, signature: crate::BackfillSignature) -> Self {
|
||||
return Self { network, signature };
|
||||
}
|
||||
|
||||
/// Returns the logical network that scopes signature uniqueness.
|
||||
#[must_use]
|
||||
pub const fn network(&self) -> &ksp_store_lib::RawNetworkId {
|
||||
return &self.network;
|
||||
}
|
||||
|
||||
/// Returns the validated encoded transaction signature.
|
||||
#[must_use]
|
||||
pub const fn signature(&self) -> &crate::BackfillSignature {
|
||||
return &self.signature;
|
||||
}
|
||||
}
|
||||
|
||||
/// One deterministic transaction candidate produced by bounded discovery.
|
||||
#[derive(Clone, Debug, Eq, PartialEq)]
|
||||
pub struct BackfillCandidate {
|
||||
identity: BackfillCandidateIdentity,
|
||||
discovered_slot: std::option::Option<u64>,
|
||||
}
|
||||
|
||||
impl BackfillCandidate {
|
||||
/// Creates one candidate from a network-scoped identity and optional discovery slot.
|
||||
#[must_use]
|
||||
pub fn new(identity: BackfillCandidateIdentity, discovered_slot: std::option::Option<u64>) -> Self {
|
||||
return Self { identity, discovered_slot };
|
||||
}
|
||||
|
||||
/// Returns the network-scoped candidate identity.
|
||||
#[must_use]
|
||||
pub const fn identity(&self) -> &BackfillCandidateIdentity {
|
||||
return &self.identity;
|
||||
}
|
||||
|
||||
/// Returns the slot reported by address discovery, when one was available.
|
||||
#[must_use]
|
||||
pub const fn discovered_slot(&self) -> std::option::Option<u64> {
|
||||
return self.discovered_slot;
|
||||
}
|
||||
}
|
||||
|
||||
/// Reason bounded candidate discovery stopped.
|
||||
#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)]
|
||||
pub enum BackfillDiscoveryBoundary {
|
||||
/// Explicit signatures required no RPC pagination.
|
||||
ExplicitInput,
|
||||
/// The caller's requested candidate limit was satisfied.
|
||||
CandidateLimit,
|
||||
/// The RPC returned a short or empty page, reaching the bounded remote history boundary.
|
||||
RpcBoundary,
|
||||
/// The maximum page count was exhausted before the requested address window completed.
|
||||
PageLimit,
|
||||
/// An `AfterAddress` scan exhausted its page bound before the exclusive anchor boundary was reached.
|
||||
AfterAnchorNotReached,
|
||||
}
|
||||
|
||||
impl BackfillDiscoveryBoundary {
|
||||
/// Returns the stable diagnostic code for this boundary.
|
||||
#[must_use]
|
||||
pub const fn code(self) -> &'static str {
|
||||
return match self {
|
||||
Self::ExplicitInput => "explicit_input",
|
||||
Self::CandidateLimit => "candidate_limit",
|
||||
Self::RpcBoundary => "rpc_boundary",
|
||||
Self::PageLimit => "page_limit",
|
||||
Self::AfterAnchorNotReached => "after_anchor_not_reached",
|
||||
};
|
||||
}
|
||||
|
||||
/// Returns whether the discovery result is partial and must not advance a future checkpoint beyond the unresolved gap.
|
||||
#[must_use]
|
||||
pub const fn is_partial(self) -> bool {
|
||||
return matches!(self, Self::PageLimit | Self::AfterAnchorNotReached);
|
||||
}
|
||||
}
|
||||
|
||||
/// Complete bounded output of one candidate discovery pass.
|
||||
pub struct BackfillDiscovery {
|
||||
network: ksp_store_lib::RawNetworkId,
|
||||
scope_fingerprint: crate::BackfillScopeFingerprint,
|
||||
candidates: std::vec::Vec<BackfillCandidate>,
|
||||
pages_fetched: usize,
|
||||
boundary: BackfillDiscoveryBoundary,
|
||||
}
|
||||
|
||||
impl BackfillDiscovery {
|
||||
/// Returns the logical network shared by every candidate identity.
|
||||
#[must_use]
|
||||
pub const fn network(&self) -> &ksp_store_lib::RawNetworkId {
|
||||
return &self.network;
|
||||
}
|
||||
|
||||
/// Returns the semantic scope fingerprint copied from the validated request.
|
||||
#[must_use]
|
||||
pub const fn scope_fingerprint(&self) -> crate::BackfillScopeFingerprint {
|
||||
return self.scope_fingerprint;
|
||||
}
|
||||
|
||||
/// Returns discovered candidates in deterministic processing order.
|
||||
#[must_use]
|
||||
pub fn candidates(&self) -> &[BackfillCandidate] {
|
||||
return self.candidates.as_slice();
|
||||
}
|
||||
|
||||
/// Returns the number of `getSignaturesForAddress` pages fetched by this pass.
|
||||
#[must_use]
|
||||
pub const fn pages_fetched(&self) -> usize {
|
||||
return self.pages_fetched;
|
||||
}
|
||||
|
||||
/// Returns the reason discovery stopped.
|
||||
#[must_use]
|
||||
pub const fn boundary(&self) -> BackfillDiscoveryBoundary {
|
||||
return self.boundary;
|
||||
}
|
||||
|
||||
/// Returns whether the bounded pass stopped before a complete requested discovery window was proven.
|
||||
#[must_use]
|
||||
pub const fn is_partial(&self) -> bool {
|
||||
return self.boundary.is_partial();
|
||||
}
|
||||
}
|
||||
|
||||
impl std::fmt::Debug for BackfillDiscovery {
|
||||
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
return formatter
|
||||
.debug_struct("BackfillDiscovery")
|
||||
.field("network", &self.network)
|
||||
.field("scope_fingerprint", &self.scope_fingerprint)
|
||||
.field("candidate_count", &self.candidates.len())
|
||||
.field("pages_fetched", &self.pages_fetched)
|
||||
.field("boundary", &self.boundary)
|
||||
.finish();
|
||||
}
|
||||
}
|
||||
|
||||
struct SignaturePageEntry {
|
||||
signature: std::string::String,
|
||||
slot: u64,
|
||||
}
|
||||
|
||||
type SignaturePageFuture<'a> = std::pin::Pin<std::boxed::Box<dyn std::future::Future<Output = ksp_core_lib::Result<std::vec::Vec<SignaturePageEntry>>> + 'a>>;
|
||||
|
||||
trait SignaturePageSource {
|
||||
fn fetch_signature_page<'a>(
|
||||
&'a self,
|
||||
role: &'a ksp_onchain_transport_lib::HttpRoleName,
|
||||
address: &'a ksp_core_lib::Pubkey,
|
||||
config: ksp_onchain_transport_lib::SolanaSignaturesForAddressConfig,
|
||||
) -> SignaturePageFuture<'a>;
|
||||
}
|
||||
|
||||
impl SignaturePageSource for ksp_onchain_transport_lib::HttpTransportPool {
|
||||
fn fetch_signature_page<'a>(
|
||||
&'a self,
|
||||
role: &'a ksp_onchain_transport_lib::HttpRoleName,
|
||||
address: &'a ksp_core_lib::Pubkey,
|
||||
config: ksp_onchain_transport_lib::SolanaSignaturesForAddressConfig,
|
||||
) -> SignaturePageFuture<'a> {
|
||||
return std::boxed::Box::pin(async move {
|
||||
let result = self.get_signatures_for_address(role, address, std::option::Option::Some(&config)).await;
|
||||
let infos = match result {
|
||||
std::result::Result::Ok(infos) => infos,
|
||||
std::result::Result::Err(error) => return std::result::Result::Err(error),
|
||||
};
|
||||
let mut entries = std::vec::Vec::with_capacity(infos.len());
|
||||
for info in infos {
|
||||
entries.push(SignaturePageEntry { signature: info.signature().to_owned(), slot: info.slot() });
|
||||
}
|
||||
return std::result::Result::Ok(entries);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
/// Discovers one bounded deterministic candidate set using only the typed KSP Transport wrapper.
|
||||
///
|
||||
/// Provider, endpoint and protocol selection remain entirely owned by Transport. Candidate identity
|
||||
/// is scoped only by the request network plus transaction signature.
|
||||
pub async fn discover_backfill_candidates(
|
||||
transport: &ksp_onchain_transport_lib::HttpTransportPool,
|
||||
request: &crate::BackfillRequest,
|
||||
) -> ksp_core_lib::Result<BackfillDiscovery> {
|
||||
ksp_logging_lib::debug!(
|
||||
target: crate::TRACING_TARGET,
|
||||
job_id = request.job_id().as_str(),
|
||||
network = request.network().as_str(),
|
||||
scope = request.scope().kind().code(),
|
||||
page_size = request.page_size(),
|
||||
max_pages = request.max_pages(),
|
||||
max_candidates = request.max_candidates(),
|
||||
"starting bounded Backfill candidate discovery"
|
||||
);
|
||||
let result = discover_with_source(transport, request).await;
|
||||
if let std::result::Result::Ok(discovery) = &result {
|
||||
ksp_logging_lib::debug!(
|
||||
target: crate::TRACING_TARGET,
|
||||
job_id = request.job_id().as_str(),
|
||||
network = request.network().as_str(),
|
||||
scope = request.scope().kind().code(),
|
||||
pages_fetched = discovery.pages_fetched(),
|
||||
candidate_count = discovery.candidates().len(),
|
||||
boundary = discovery.boundary().code(),
|
||||
partial = discovery.is_partial(),
|
||||
"completed bounded Backfill candidate discovery"
|
||||
);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
async fn discover_with_source<S>(source: &S, request: &crate::BackfillRequest) -> ksp_core_lib::Result<BackfillDiscovery>
|
||||
where
|
||||
S: SignaturePageSource,
|
||||
{
|
||||
return match request.scope().kind() {
|
||||
crate::BackfillScopeKind::ExplicitSignatures => discover_explicit(request),
|
||||
crate::BackfillScopeKind::LatestAddress | crate::BackfillScopeKind::BeforeAddress => discover_older(source, request).await,
|
||||
crate::BackfillScopeKind::AfterAddress => discover_after(source, request).await,
|
||||
};
|
||||
}
|
||||
|
||||
fn discover_explicit(request: &crate::BackfillRequest) -> ksp_core_lib::Result<BackfillDiscovery> {
|
||||
let signatures = match request.scope().signatures() {
|
||||
std::option::Option::Some(signatures) => signatures,
|
||||
std::option::Option::None => return std::result::Result::Err(discovery_invalid("scope.signatures")),
|
||||
};
|
||||
let mut candidates = std::vec::Vec::with_capacity(signatures.len());
|
||||
for signature in signatures {
|
||||
let identity = BackfillCandidateIdentity::new(request.network().clone(), signature.clone());
|
||||
candidates.push(BackfillCandidate::new(identity, std::option::Option::None));
|
||||
}
|
||||
return std::result::Result::Ok(BackfillDiscovery {
|
||||
network: request.network().clone(),
|
||||
scope_fingerprint: request.scope_fingerprint(),
|
||||
candidates,
|
||||
pages_fetched: 0,
|
||||
boundary: BackfillDiscoveryBoundary::ExplicitInput,
|
||||
});
|
||||
}
|
||||
|
||||
async fn discover_older<S>(source: &S, request: &crate::BackfillRequest) -> ksp_core_lib::Result<BackfillDiscovery>
|
||||
where
|
||||
S: SignaturePageSource,
|
||||
{
|
||||
let address = match request.scope().address() {
|
||||
std::option::Option::Some(address) => address,
|
||||
std::option::Option::None => return std::result::Result::Err(discovery_invalid("scope.address")),
|
||||
};
|
||||
let mut before = match request.scope().kind() {
|
||||
crate::BackfillScopeKind::BeforeAddress => request.scope().anchor().map(|value| value.as_str().to_owned()),
|
||||
crate::BackfillScopeKind::LatestAddress => std::option::Option::None,
|
||||
crate::BackfillScopeKind::AfterAddress | crate::BackfillScopeKind::ExplicitSignatures => {
|
||||
return std::result::Result::Err(discovery_invalid("scope.kind"));
|
||||
},
|
||||
};
|
||||
let mut candidates = std::vec::Vec::with_capacity(request.max_candidates());
|
||||
let mut seen = std::collections::HashSet::<crate::BackfillSignature>::with_capacity(request.max_candidates());
|
||||
let mut pages_fetched = 0_usize;
|
||||
let boundary = loop {
|
||||
if candidates.len() >= request.max_candidates() {
|
||||
break BackfillDiscoveryBoundary::CandidateLimit;
|
||||
}
|
||||
if pages_fetched >= request.max_pages() {
|
||||
break BackfillDiscoveryBoundary::PageLimit;
|
||||
}
|
||||
let remaining = request.max_candidates() - candidates.len();
|
||||
let page_limit = std::cmp::min(request.page_size(), remaining);
|
||||
let config = ksp_onchain_transport_lib::SolanaSignaturesForAddressConfig::new(
|
||||
before.clone(),
|
||||
std::option::Option::None,
|
||||
std::option::Option::Some(page_limit),
|
||||
std::option::Option::Some(request.commitment().transport()),
|
||||
request.min_context_slot(),
|
||||
);
|
||||
let page_result = source.fetch_signature_page(request.role(), address, config).await;
|
||||
let page = match page_result {
|
||||
std::result::Result::Ok(page) => page,
|
||||
std::result::Result::Err(error) => return std::result::Result::Err(error),
|
||||
};
|
||||
pages_fetched += 1;
|
||||
if page.len() > page_limit {
|
||||
return std::result::Result::Err(discovery_invalid("page.length"));
|
||||
}
|
||||
let page_len = page.len();
|
||||
let next_before = match page.last() {
|
||||
std::option::Option::Some(entry) => {
|
||||
let signature_result = validated_signature(entry.signature.as_str());
|
||||
match signature_result {
|
||||
std::result::Result::Ok(signature) => std::option::Option::Some(signature),
|
||||
std::result::Result::Err(error) => return std::result::Result::Err(error),
|
||||
}
|
||||
},
|
||||
std::option::Option::None => std::option::Option::None,
|
||||
};
|
||||
for entry in page {
|
||||
let signature_result = validated_signature(entry.signature.as_str());
|
||||
let signature = match signature_result {
|
||||
std::result::Result::Ok(signature) => signature,
|
||||
std::result::Result::Err(error) => return std::result::Result::Err(error),
|
||||
};
|
||||
if seen.insert(signature.clone()) {
|
||||
let identity = BackfillCandidateIdentity::new(request.network().clone(), signature);
|
||||
candidates.push(BackfillCandidate::new(identity, std::option::Option::Some(entry.slot)));
|
||||
if candidates.len() >= request.max_candidates() {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
if page_len < page_limit {
|
||||
break BackfillDiscoveryBoundary::RpcBoundary;
|
||||
}
|
||||
let next_before = match next_before {
|
||||
std::option::Option::Some(next_before) => next_before,
|
||||
std::option::Option::None => break BackfillDiscoveryBoundary::RpcBoundary,
|
||||
};
|
||||
if before.as_deref() == std::option::Option::Some(next_before.as_str()) {
|
||||
return std::result::Result::Err(discovery_stalled());
|
||||
}
|
||||
before = std::option::Option::Some(next_before.as_str().to_owned());
|
||||
};
|
||||
return std::result::Result::Ok(BackfillDiscovery {
|
||||
network: request.network().clone(),
|
||||
scope_fingerprint: request.scope_fingerprint(),
|
||||
candidates,
|
||||
pages_fetched,
|
||||
boundary,
|
||||
});
|
||||
}
|
||||
|
||||
async fn discover_after<S>(source: &S, request: &crate::BackfillRequest) -> ksp_core_lib::Result<BackfillDiscovery>
|
||||
where
|
||||
S: SignaturePageSource,
|
||||
{
|
||||
let address = match request.scope().address() {
|
||||
std::option::Option::Some(address) => address,
|
||||
std::option::Option::None => return std::result::Result::Err(discovery_invalid("scope.address")),
|
||||
};
|
||||
let anchor = match request.scope().anchor() {
|
||||
std::option::Option::Some(anchor) => anchor,
|
||||
std::option::Option::None => return std::result::Result::Err(discovery_invalid("scope.anchor")),
|
||||
};
|
||||
let until = anchor.as_str().to_owned();
|
||||
let mut before = std::option::Option::<std::string::String>::None;
|
||||
let mut nearest = std::collections::VecDeque::<BackfillCandidate>::with_capacity(request.max_candidates());
|
||||
let mut seen = std::collections::HashSet::<crate::BackfillSignature>::new();
|
||||
let mut pages_fetched = 0_usize;
|
||||
let boundary = loop {
|
||||
if pages_fetched >= request.max_pages() {
|
||||
break BackfillDiscoveryBoundary::AfterAnchorNotReached;
|
||||
}
|
||||
let config = ksp_onchain_transport_lib::SolanaSignaturesForAddressConfig::new(
|
||||
before.clone(),
|
||||
std::option::Option::Some(until.clone()),
|
||||
std::option::Option::Some(request.page_size()),
|
||||
std::option::Option::Some(request.commitment().transport()),
|
||||
request.min_context_slot(),
|
||||
);
|
||||
let page_result = source.fetch_signature_page(request.role(), address, config).await;
|
||||
let page = match page_result {
|
||||
std::result::Result::Ok(page) => page,
|
||||
std::result::Result::Err(error) => return std::result::Result::Err(error),
|
||||
};
|
||||
pages_fetched += 1;
|
||||
if page.len() > request.page_size() {
|
||||
return std::result::Result::Err(discovery_invalid("page.length"));
|
||||
}
|
||||
let page_len = page.len();
|
||||
let next_before = match page.last() {
|
||||
std::option::Option::Some(entry) => {
|
||||
let signature_result = validated_signature(entry.signature.as_str());
|
||||
match signature_result {
|
||||
std::result::Result::Ok(signature) => std::option::Option::Some(signature),
|
||||
std::result::Result::Err(error) => return std::result::Result::Err(error),
|
||||
}
|
||||
},
|
||||
std::option::Option::None => std::option::Option::None,
|
||||
};
|
||||
for entry in page {
|
||||
let signature_result = validated_signature(entry.signature.as_str());
|
||||
let signature = match signature_result {
|
||||
std::result::Result::Ok(signature) => signature,
|
||||
std::result::Result::Err(error) => return std::result::Result::Err(error),
|
||||
};
|
||||
if seen.insert(signature.clone()) {
|
||||
let identity = BackfillCandidateIdentity::new(request.network().clone(), signature);
|
||||
nearest.push_back(BackfillCandidate::new(identity, std::option::Option::Some(entry.slot)));
|
||||
if nearest.len() > request.max_candidates() {
|
||||
nearest.pop_front();
|
||||
}
|
||||
}
|
||||
}
|
||||
if page_len < request.page_size() {
|
||||
break BackfillDiscoveryBoundary::RpcBoundary;
|
||||
}
|
||||
let next_before = match next_before {
|
||||
std::option::Option::Some(next_before) => next_before,
|
||||
std::option::Option::None => break BackfillDiscoveryBoundary::RpcBoundary,
|
||||
};
|
||||
if before.as_deref() == std::option::Option::Some(next_before.as_str()) {
|
||||
return std::result::Result::Err(discovery_stalled());
|
||||
}
|
||||
before = std::option::Option::Some(next_before.as_str().to_owned());
|
||||
};
|
||||
return std::result::Result::Ok(BackfillDiscovery {
|
||||
network: request.network().clone(),
|
||||
scope_fingerprint: request.scope_fingerprint(),
|
||||
candidates: nearest.into_iter().collect(),
|
||||
pages_fetched,
|
||||
boundary,
|
||||
});
|
||||
}
|
||||
|
||||
fn validated_signature(value: &str) -> ksp_core_lib::Result<crate::BackfillSignature> {
|
||||
return crate::BackfillSignature::new(value.to_owned());
|
||||
}
|
||||
|
||||
fn discovery_invalid(field: &'static str) -> ksp_core_lib::Error {
|
||||
return ksp_core_lib::Error::new(crate::ERROR_CODE_BACKFILL_DISCOVERY_INVALID, "invalid bounded Backfill discovery state").with_context("field", field);
|
||||
}
|
||||
|
||||
fn discovery_stalled() -> ksp_core_lib::Error {
|
||||
return ksp_core_lib::Error::new(crate::ERROR_CODE_BACKFILL_DISCOVERY_STALLED, "Backfill discovery cursor did not advance")
|
||||
.with_context("field", "before");
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[path = "../unit_tests/discovery.rs"]
|
||||
mod tests;
|
||||
11
crates/ksp-job-backfill-lib/src/error.rs
Normal file
11
crates/ksp-job-backfill-lib/src/error.rs
Normal file
@@ -0,0 +1,11 @@
|
||||
// file: crates/ksp-job-backfill-lib/src/error.rs
|
||||
// version: 1
|
||||
|
||||
/// Error code used when a signature page violates a bounded discovery invariant.
|
||||
pub const ERROR_CODE_BACKFILL_DISCOVERY_INVALID: ksp_core_lib::ErrorCode = ksp_core_lib::ErrorCode::new("job_backfill", "discovery_invalid");
|
||||
/// Error code used when paginated discovery cannot advance its exclusive RPC cursor safely.
|
||||
pub const ERROR_CODE_BACKFILL_DISCOVERY_STALLED: ksp_core_lib::ErrorCode = ksp_core_lib::ErrorCode::new("job_backfill", "discovery_stalled");
|
||||
/// Error code used when one Backfill request violates its bounded admission contract.
|
||||
pub const ERROR_CODE_BACKFILL_REQUEST_INVALID: ksp_core_lib::ErrorCode = ksp_core_lib::ErrorCode::new("job_backfill", "request_invalid");
|
||||
/// Error code used when one transaction signature text violates the bounded Base58-shape contract.
|
||||
pub const ERROR_CODE_BACKFILL_SIGNATURE_INVALID: ksp_core_lib::ErrorCode = ksp_core_lib::ErrorCode::new("job_backfill", "signature_invalid");
|
||||
64
crates/ksp-job-backfill-lib/src/lib.rs
Normal file
64
crates/ksp-job-backfill-lib/src/lib.rs
Normal file
@@ -0,0 +1,64 @@
|
||||
// file: crates/ksp-job-backfill-lib/src/lib.rs
|
||||
// version: 1
|
||||
|
||||
#![warn(missing_docs)]
|
||||
#![deny(unreachable_pub)]
|
||||
#![forbid(unsafe_code)]
|
||||
|
||||
//! Concrete bounded historical RAW transaction Backfill foundation.
|
||||
//!
|
||||
//! This tranche owns explicit admission, network-scoped candidate identity and deterministic
|
||||
//! `getSignaturesForAddress` pagination. Transport retains provider/endpoint selection and retry;
|
||||
//! Store retains durable idempotence and persistence. RAW conversion, persistence, concurrency,
|
||||
//! checkpointing, cancellation and concrete latest-value snapshots are added by later v0.3.6 tranches.
|
||||
|
||||
mod constants;
|
||||
mod discovery;
|
||||
mod error;
|
||||
mod request;
|
||||
|
||||
/// One deterministic transaction candidate produced by bounded discovery.
|
||||
pub use self::discovery::BackfillCandidate;
|
||||
/// Network-scoped identity of one discovered transaction candidate before canonical signature decoding.
|
||||
pub use self::discovery::BackfillCandidateIdentity;
|
||||
/// Complete bounded output of one candidate discovery pass.
|
||||
pub use self::discovery::BackfillDiscovery;
|
||||
/// Reason bounded candidate discovery stopped.
|
||||
pub use self::discovery::BackfillDiscoveryBoundary;
|
||||
/// Discovers one bounded deterministic candidate set through the typed KSP Transport wrapper.
|
||||
pub use self::discovery::discover_backfill_candidates;
|
||||
/// Error code used when a signature page violates a bounded discovery invariant.
|
||||
pub use self::error::ERROR_CODE_BACKFILL_DISCOVERY_INVALID;
|
||||
/// Error code used when paginated discovery cannot advance its exclusive RPC cursor safely.
|
||||
pub use self::error::ERROR_CODE_BACKFILL_DISCOVERY_STALLED;
|
||||
/// Error code used when one Backfill request violates its bounded admission contract.
|
||||
pub use self::error::ERROR_CODE_BACKFILL_REQUEST_INVALID;
|
||||
/// Error code used when one transaction signature text violates the bounded Base58-shape contract.
|
||||
pub use self::error::ERROR_CODE_BACKFILL_SIGNATURE_INVALID;
|
||||
/// Commitment levels intentionally admitted by the historical Backfill vertical.
|
||||
pub use self::request::BackfillCommitment;
|
||||
/// Fully explicit bounded request for one historical transaction Backfill Job.
|
||||
pub use self::request::BackfillRequest;
|
||||
/// Validated bounded discovery scope for one historical Backfill Job.
|
||||
pub use self::request::BackfillScope;
|
||||
/// Opaque deterministic fingerprint of one semantic Backfill scope.
|
||||
pub use self::request::BackfillScopeFingerprint;
|
||||
/// Stable category of one bounded Backfill discovery scope.
|
||||
pub use self::request::BackfillScopeKind;
|
||||
/// Bounded Base58-shaped transaction signature text used before canonical 64-byte decoding.
|
||||
pub use self::request::BackfillSignature;
|
||||
/// Maximum number of transaction candidates admitted by one bounded Backfill Job.
|
||||
pub use self::request::MAX_BACKFILL_CANDIDATES;
|
||||
/// Maximum number of concurrent transaction hydrations admitted by one Backfill request.
|
||||
pub use self::request::MAX_BACKFILL_HYDRATION_CONCURRENCY;
|
||||
/// Maximum number of `getSignaturesForAddress` pages admitted by one address Backfill request.
|
||||
pub use self::request::MAX_BACKFILL_PAGES;
|
||||
/// Maximum page size admitted for one `getSignaturesForAddress` request.
|
||||
pub use self::request::MAX_BACKFILL_PAGE_SIZE;
|
||||
/// Maximum Base58 text length possible for one canonical 64-byte Solana signature.
|
||||
pub use self::request::MAX_BACKFILL_SIGNATURE_TEXT_BYTES;
|
||||
/// Minimum Base58 text length possible for one canonical 64-byte Solana signature.
|
||||
pub use self::request::MIN_BACKFILL_SIGNATURE_TEXT_BYTES;
|
||||
|
||||
/// Owning tracing target used by the concrete Backfill runtime.
|
||||
pub(crate) use self::constants::TRACING_TARGET;
|
||||
466
crates/ksp-job-backfill-lib/src/request.rs
Normal file
466
crates/ksp-job-backfill-lib/src/request.rs
Normal file
@@ -0,0 +1,466 @@
|
||||
// file: crates/ksp-job-backfill-lib/src/request.rs
|
||||
// version: 1
|
||||
|
||||
use sha2::Digest; // rust-rules: trait-import
|
||||
|
||||
/// Maximum number of transaction candidates admitted by one bounded Backfill Job.
|
||||
pub const MAX_BACKFILL_CANDIDATES: usize = 10_000;
|
||||
/// Maximum number of concurrent transaction hydrations admitted by one Backfill request.
|
||||
pub const MAX_BACKFILL_HYDRATION_CONCURRENCY: usize = 64;
|
||||
/// Maximum number of `getSignaturesForAddress` pages admitted by one address Backfill request.
|
||||
pub const MAX_BACKFILL_PAGES: usize = 10_000;
|
||||
/// Maximum page size admitted for one `getSignaturesForAddress` request.
|
||||
pub const MAX_BACKFILL_PAGE_SIZE: usize = 1_000;
|
||||
/// Maximum Base58 text length possible for one canonical 64-byte Solana signature.
|
||||
pub const MAX_BACKFILL_SIGNATURE_TEXT_BYTES: usize = 88;
|
||||
/// Minimum Base58 text length possible for one canonical 64-byte Solana signature.
|
||||
pub const MIN_BACKFILL_SIGNATURE_TEXT_BYTES: usize = 64;
|
||||
|
||||
/// Commitment levels intentionally admitted by the historical Backfill vertical.
|
||||
#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)]
|
||||
pub enum BackfillCommitment {
|
||||
/// Read history at Solana `confirmed` commitment.
|
||||
Confirmed,
|
||||
/// Read history at Solana `finalized` commitment.
|
||||
Finalized,
|
||||
}
|
||||
|
||||
impl BackfillCommitment {
|
||||
/// Returns the stable Backfill commitment code.
|
||||
#[must_use]
|
||||
pub const fn code(self) -> &'static str {
|
||||
return match self {
|
||||
Self::Confirmed => "confirmed",
|
||||
Self::Finalized => "finalized",
|
||||
};
|
||||
}
|
||||
|
||||
/// Maps the narrowed Backfill commitment to the Transport-owned Solana commitment.
|
||||
#[must_use]
|
||||
pub(crate) const fn transport(self) -> ksp_onchain_transport_lib::SolanaCommitment {
|
||||
return match self {
|
||||
Self::Confirmed => ksp_onchain_transport_lib::SolanaCommitment::Confirmed,
|
||||
Self::Finalized => ksp_onchain_transport_lib::SolanaCommitment::Finalized,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/// Bounded Base58-shaped transaction signature text used before canonical 64-byte decoding.
|
||||
///
|
||||
/// This type deliberately validates only the encoded shape required by discovery. Exact decoding
|
||||
/// to the Store-owned 64-byte signature is introduced by the RAW conversion tranche.
|
||||
#[derive(Clone, Eq, Hash, Ord, PartialEq, PartialOrd)]
|
||||
pub struct BackfillSignature(std::string::String);
|
||||
|
||||
impl BackfillSignature {
|
||||
/// Creates one bounded Base58-shaped signature text.
|
||||
pub fn new(value: impl std::convert::Into<std::string::String>) -> ksp_core_lib::Result<Self> {
|
||||
let value = value.into();
|
||||
if !valid_signature_text(value.as_str()) {
|
||||
return std::result::Result::Err(signature_error());
|
||||
}
|
||||
return std::result::Result::Ok(Self(value));
|
||||
}
|
||||
|
||||
/// Returns the validated encoded signature text.
|
||||
#[must_use]
|
||||
pub fn as_str(&self) -> &str {
|
||||
return self.0.as_str();
|
||||
}
|
||||
}
|
||||
|
||||
impl std::fmt::Debug for BackfillSignature {
|
||||
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
return formatter.write_str("BackfillSignature(..)");
|
||||
}
|
||||
}
|
||||
|
||||
/// Stable category of one bounded Backfill discovery scope.
|
||||
#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)]
|
||||
pub enum BackfillScopeKind {
|
||||
/// Read the current newest address window.
|
||||
LatestAddress,
|
||||
/// Read address history older than one exclusive anchor.
|
||||
BeforeAddress,
|
||||
/// Read the bounded newer window closest to one exclusive anchor.
|
||||
AfterAddress,
|
||||
/// Hydrate an explicit bounded signature list without address discovery.
|
||||
ExplicitSignatures,
|
||||
}
|
||||
|
||||
impl BackfillScopeKind {
|
||||
/// Returns the stable scope code used by diagnostics and scope fingerprinting.
|
||||
#[must_use]
|
||||
pub const fn code(self) -> &'static str {
|
||||
return match self {
|
||||
Self::LatestAddress => "latest_address",
|
||||
Self::BeforeAddress => "before_address",
|
||||
Self::AfterAddress => "after_address",
|
||||
Self::ExplicitSignatures => "explicit_signatures",
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Eq, PartialEq)]
|
||||
enum BackfillScopeValue {
|
||||
LatestAddress { address: ksp_core_lib::Pubkey },
|
||||
BeforeAddress { address: ksp_core_lib::Pubkey, anchor: BackfillSignature },
|
||||
AfterAddress { address: ksp_core_lib::Pubkey, anchor: BackfillSignature },
|
||||
ExplicitSignatures { signatures: std::vec::Vec<BackfillSignature> },
|
||||
}
|
||||
|
||||
/// Validated bounded discovery scope for one historical Backfill Job.
|
||||
#[derive(Clone, Eq, PartialEq)]
|
||||
pub struct BackfillScope {
|
||||
value: BackfillScopeValue,
|
||||
}
|
||||
|
||||
impl BackfillScope {
|
||||
/// Creates a scope starting at the newest known history for one address.
|
||||
#[must_use]
|
||||
pub const fn latest_address(address: ksp_core_lib::Pubkey) -> Self {
|
||||
return Self { value: BackfillScopeValue::LatestAddress { address } };
|
||||
}
|
||||
|
||||
/// Creates a scope reading history older than one exclusive address anchor.
|
||||
#[must_use]
|
||||
pub fn before_address(address: ksp_core_lib::Pubkey, anchor: BackfillSignature) -> Self {
|
||||
return Self { value: BackfillScopeValue::BeforeAddress { address, anchor } };
|
||||
}
|
||||
|
||||
/// Creates a scope reading the bounded newer history closest to one exclusive address anchor.
|
||||
#[must_use]
|
||||
pub fn after_address(address: ksp_core_lib::Pubkey, anchor: BackfillSignature) -> Self {
|
||||
return Self { value: BackfillScopeValue::AfterAddress { address, anchor } };
|
||||
}
|
||||
|
||||
/// Creates an explicit signature scope with stable first-occurrence deduplication.
|
||||
pub fn explicit_signatures(signatures: std::vec::Vec<BackfillSignature>) -> ksp_core_lib::Result<Self> {
|
||||
if signatures.is_empty() || signatures.len() > crate::MAX_BACKFILL_CANDIDATES {
|
||||
return std::result::Result::Err(request_error("scope.signatures"));
|
||||
}
|
||||
let mut unique = std::vec::Vec::with_capacity(signatures.len());
|
||||
let mut seen = std::collections::HashSet::with_capacity(signatures.len());
|
||||
for signature in signatures {
|
||||
if seen.insert(signature.clone()) {
|
||||
unique.push(signature);
|
||||
}
|
||||
}
|
||||
if unique.is_empty() {
|
||||
return std::result::Result::Err(request_error("scope.signatures"));
|
||||
}
|
||||
return std::result::Result::Ok(Self { value: BackfillScopeValue::ExplicitSignatures { signatures: unique } });
|
||||
}
|
||||
|
||||
/// Returns the stable category of this scope.
|
||||
#[must_use]
|
||||
pub const fn kind(&self) -> BackfillScopeKind {
|
||||
return match &self.value {
|
||||
BackfillScopeValue::LatestAddress { .. } => BackfillScopeKind::LatestAddress,
|
||||
BackfillScopeValue::BeforeAddress { .. } => BackfillScopeKind::BeforeAddress,
|
||||
BackfillScopeValue::AfterAddress { .. } => BackfillScopeKind::AfterAddress,
|
||||
BackfillScopeValue::ExplicitSignatures { .. } => BackfillScopeKind::ExplicitSignatures,
|
||||
};
|
||||
}
|
||||
|
||||
/// Returns the address used by address scopes.
|
||||
#[must_use]
|
||||
pub const fn address(&self) -> std::option::Option<&ksp_core_lib::Pubkey> {
|
||||
return match &self.value {
|
||||
BackfillScopeValue::LatestAddress { address }
|
||||
| BackfillScopeValue::BeforeAddress { address, .. }
|
||||
| BackfillScopeValue::AfterAddress { address, .. } => std::option::Option::Some(address),
|
||||
BackfillScopeValue::ExplicitSignatures { .. } => std::option::Option::None,
|
||||
};
|
||||
}
|
||||
|
||||
/// Returns the exclusive anchor used by before/after scopes.
|
||||
#[must_use]
|
||||
pub const fn anchor(&self) -> std::option::Option<&BackfillSignature> {
|
||||
return match &self.value {
|
||||
BackfillScopeValue::BeforeAddress { anchor, .. } | BackfillScopeValue::AfterAddress { anchor, .. } => std::option::Option::Some(anchor),
|
||||
BackfillScopeValue::LatestAddress { .. } | BackfillScopeValue::ExplicitSignatures { .. } => std::option::Option::None,
|
||||
};
|
||||
}
|
||||
|
||||
/// Returns the stable deduplicated explicit signature list, when this is an explicit scope.
|
||||
#[must_use]
|
||||
pub fn signatures(&self) -> std::option::Option<&[BackfillSignature]> {
|
||||
return match &self.value {
|
||||
BackfillScopeValue::ExplicitSignatures { signatures } => std::option::Option::Some(signatures.as_slice()),
|
||||
BackfillScopeValue::LatestAddress { .. } | BackfillScopeValue::BeforeAddress { .. } | BackfillScopeValue::AfterAddress { .. } => {
|
||||
std::option::Option::None
|
||||
},
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
impl std::fmt::Debug for BackfillScope {
|
||||
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
let mut debug = formatter.debug_struct("BackfillScope");
|
||||
debug.field("kind", &self.kind());
|
||||
match &self.value {
|
||||
BackfillScopeValue::LatestAddress { address } => {
|
||||
debug.field("address", address);
|
||||
},
|
||||
BackfillScopeValue::BeforeAddress { address, .. } | BackfillScopeValue::AfterAddress { address, .. } => {
|
||||
debug.field("address", address).field("anchor", &"<redacted>");
|
||||
},
|
||||
BackfillScopeValue::ExplicitSignatures { signatures } => {
|
||||
debug.field("signature_count", &signatures.len());
|
||||
},
|
||||
}
|
||||
return debug.finish();
|
||||
}
|
||||
}
|
||||
|
||||
/// Opaque deterministic fingerprint of one semantic Backfill scope.
|
||||
#[derive(Clone, Copy, Eq, Hash, PartialEq)]
|
||||
pub struct BackfillScopeFingerprint([u8; 32]);
|
||||
|
||||
impl BackfillScopeFingerprint {
|
||||
/// Returns the exact deterministic fingerprint bytes.
|
||||
#[must_use]
|
||||
pub const fn as_bytes(&self) -> &[u8; 32] {
|
||||
return &self.0;
|
||||
}
|
||||
}
|
||||
|
||||
impl std::fmt::Debug for BackfillScopeFingerprint {
|
||||
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
return formatter.write_str("BackfillScopeFingerprint(..)");
|
||||
}
|
||||
}
|
||||
|
||||
/// Fully explicit bounded request for one historical transaction Backfill Job.
|
||||
#[derive(Clone, Eq, PartialEq)]
|
||||
pub struct BackfillRequest {
|
||||
job_id: ksp_job_api::JobId,
|
||||
network: ksp_store_lib::RawNetworkId,
|
||||
role: ksp_onchain_transport_lib::HttpRoleName,
|
||||
commitment: BackfillCommitment,
|
||||
scope: BackfillScope,
|
||||
page_size: usize,
|
||||
max_pages: usize,
|
||||
max_candidates: usize,
|
||||
hydration_concurrency: usize,
|
||||
min_context_slot: std::option::Option<u64>,
|
||||
scope_fingerprint: BackfillScopeFingerprint,
|
||||
}
|
||||
|
||||
impl BackfillRequest {
|
||||
/// Creates and validates one fully explicit bounded Backfill request.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub fn new(
|
||||
job_id: ksp_job_api::JobId,
|
||||
network: ksp_store_lib::RawNetworkId,
|
||||
role: ksp_onchain_transport_lib::HttpRoleName,
|
||||
commitment: BackfillCommitment,
|
||||
scope: BackfillScope,
|
||||
page_size: usize,
|
||||
max_pages: usize,
|
||||
max_candidates: usize,
|
||||
hydration_concurrency: usize,
|
||||
min_context_slot: std::option::Option<u64>,
|
||||
) -> ksp_core_lib::Result<Self> {
|
||||
if page_size == 0 || page_size > crate::MAX_BACKFILL_PAGE_SIZE {
|
||||
return std::result::Result::Err(request_error("page_size"));
|
||||
}
|
||||
if max_pages == 0 || max_pages > crate::MAX_BACKFILL_PAGES {
|
||||
return std::result::Result::Err(request_error("max_pages"));
|
||||
}
|
||||
if max_candidates == 0 || max_candidates > crate::MAX_BACKFILL_CANDIDATES {
|
||||
return std::result::Result::Err(request_error("max_candidates"));
|
||||
}
|
||||
if hydration_concurrency == 0 || hydration_concurrency > crate::MAX_BACKFILL_HYDRATION_CONCURRENCY {
|
||||
return std::result::Result::Err(request_error("hydration_concurrency"));
|
||||
}
|
||||
if role.as_str().is_empty() || role.as_str().trim() != role.as_str() {
|
||||
return std::result::Result::Err(request_error("role"));
|
||||
}
|
||||
if scope.kind() == BackfillScopeKind::ExplicitSignatures {
|
||||
if min_context_slot.is_some() {
|
||||
return std::result::Result::Err(request_error("min_context_slot"));
|
||||
}
|
||||
let signature_count = match scope.signatures() {
|
||||
std::option::Option::Some(signatures) => signatures.len(),
|
||||
std::option::Option::None => return std::result::Result::Err(request_error("scope.signatures")),
|
||||
};
|
||||
if signature_count > max_candidates {
|
||||
return std::result::Result::Err(request_error("max_candidates"));
|
||||
}
|
||||
}
|
||||
let scope_fingerprint = fingerprint_scope(&network, commitment, &scope, page_size, max_pages, max_candidates, min_context_slot);
|
||||
return std::result::Result::Ok(Self {
|
||||
job_id,
|
||||
network,
|
||||
role,
|
||||
commitment,
|
||||
scope,
|
||||
page_size,
|
||||
max_pages,
|
||||
max_candidates,
|
||||
hydration_concurrency,
|
||||
min_context_slot,
|
||||
scope_fingerprint,
|
||||
});
|
||||
}
|
||||
|
||||
/// Returns the caller-owned logical Job identity.
|
||||
#[must_use]
|
||||
pub const fn job_id(&self) -> &ksp_job_api::JobId {
|
||||
return &self.job_id;
|
||||
}
|
||||
|
||||
/// Returns the Store-scoped logical network identity for every candidate in this Job.
|
||||
#[must_use]
|
||||
pub const fn network(&self) -> &ksp_store_lib::RawNetworkId {
|
||||
return &self.network;
|
||||
}
|
||||
|
||||
/// Returns the logical HTTP role used only for Transport selection.
|
||||
#[must_use]
|
||||
pub const fn role(&self) -> &ksp_onchain_transport_lib::HttpRoleName {
|
||||
return &self.role;
|
||||
}
|
||||
|
||||
/// Returns the narrowed commitment used by discovery and hydration.
|
||||
#[must_use]
|
||||
pub const fn commitment(&self) -> BackfillCommitment {
|
||||
return self.commitment;
|
||||
}
|
||||
|
||||
/// Returns the validated discovery scope.
|
||||
#[must_use]
|
||||
pub const fn scope(&self) -> &BackfillScope {
|
||||
return &self.scope;
|
||||
}
|
||||
|
||||
/// Returns the per-request signature page size.
|
||||
#[must_use]
|
||||
pub const fn page_size(&self) -> usize {
|
||||
return self.page_size;
|
||||
}
|
||||
|
||||
/// Returns the maximum number of address pages admitted by this Job.
|
||||
#[must_use]
|
||||
pub const fn max_pages(&self) -> usize {
|
||||
return self.max_pages;
|
||||
}
|
||||
|
||||
/// Returns the maximum number of transaction candidates admitted by this Job.
|
||||
#[must_use]
|
||||
pub const fn max_candidates(&self) -> usize {
|
||||
return self.max_candidates;
|
||||
}
|
||||
|
||||
/// Returns the future hydration concurrency bound.
|
||||
#[must_use]
|
||||
pub const fn hydration_concurrency(&self) -> usize {
|
||||
return self.hydration_concurrency;
|
||||
}
|
||||
|
||||
/// Returns the optional Solana minimum context slot used by address discovery.
|
||||
#[must_use]
|
||||
pub const fn min_context_slot(&self) -> std::option::Option<u64> {
|
||||
return self.min_context_slot;
|
||||
}
|
||||
|
||||
/// Returns the deterministic semantic scope fingerprint.
|
||||
///
|
||||
/// Transport role, provider, endpoint and protocol are deliberately excluded. They describe
|
||||
/// acquisition provenance, not transaction or scope identity.
|
||||
#[must_use]
|
||||
pub const fn scope_fingerprint(&self) -> BackfillScopeFingerprint {
|
||||
return self.scope_fingerprint;
|
||||
}
|
||||
}
|
||||
|
||||
impl std::fmt::Debug for BackfillRequest {
|
||||
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
return formatter
|
||||
.debug_struct("BackfillRequest")
|
||||
.field("job_id", &self.job_id)
|
||||
.field("network", &self.network)
|
||||
.field("role", &self.role)
|
||||
.field("commitment", &self.commitment)
|
||||
.field("scope", &self.scope)
|
||||
.field("page_size", &self.page_size)
|
||||
.field("max_pages", &self.max_pages)
|
||||
.field("max_candidates", &self.max_candidates)
|
||||
.field("hydration_concurrency", &self.hydration_concurrency)
|
||||
.field("min_context_slot", &self.min_context_slot)
|
||||
.field("scope_fingerprint", &self.scope_fingerprint)
|
||||
.finish();
|
||||
}
|
||||
}
|
||||
|
||||
fn fingerprint_scope(
|
||||
network: &ksp_store_lib::RawNetworkId,
|
||||
commitment: BackfillCommitment,
|
||||
scope: &BackfillScope,
|
||||
page_size: usize,
|
||||
max_pages: usize,
|
||||
max_candidates: usize,
|
||||
min_context_slot: std::option::Option<u64>,
|
||||
) -> BackfillScopeFingerprint {
|
||||
let mut hasher = sha2::Sha256::new();
|
||||
hasher.update(b"ksp.job.backfill.scope.v1\0");
|
||||
hash_bytes(&mut hasher, network.as_str().as_bytes());
|
||||
hash_bytes(&mut hasher, commitment.code().as_bytes());
|
||||
hash_bytes(&mut hasher, scope.kind().code().as_bytes());
|
||||
if let std::option::Option::Some(address) = scope.address() {
|
||||
hash_bytes(&mut hasher, address.as_ref());
|
||||
}
|
||||
if let std::option::Option::Some(anchor) = scope.anchor() {
|
||||
hash_bytes(&mut hasher, anchor.as_str().as_bytes());
|
||||
}
|
||||
if let std::option::Option::Some(signatures) = scope.signatures() {
|
||||
for signature in signatures {
|
||||
hash_bytes(&mut hasher, signature.as_str().as_bytes());
|
||||
}
|
||||
}
|
||||
hash_u64(&mut hasher, page_size as u64);
|
||||
hash_u64(&mut hasher, max_pages as u64);
|
||||
hash_u64(&mut hasher, max_candidates as u64);
|
||||
match min_context_slot {
|
||||
std::option::Option::Some(slot) => {
|
||||
hasher.update([1_u8]);
|
||||
hash_u64(&mut hasher, slot);
|
||||
},
|
||||
std::option::Option::None => hasher.update([0_u8]),
|
||||
}
|
||||
let bytes: [u8; 32] = hasher.finalize().into();
|
||||
return BackfillScopeFingerprint(bytes);
|
||||
}
|
||||
|
||||
fn hash_bytes(hasher: &mut sha2::Sha256, value: &[u8]) {
|
||||
hash_u64(hasher, value.len() as u64);
|
||||
hasher.update(value);
|
||||
}
|
||||
|
||||
fn hash_u64(hasher: &mut sha2::Sha256, value: u64) {
|
||||
hasher.update(value.to_be_bytes());
|
||||
}
|
||||
|
||||
fn request_error(field: &'static str) -> ksp_core_lib::Error {
|
||||
return ksp_core_lib::Error::new(crate::ERROR_CODE_BACKFILL_REQUEST_INVALID, "invalid bounded Backfill request").with_context("field", field);
|
||||
}
|
||||
|
||||
fn signature_error() -> ksp_core_lib::Error {
|
||||
return ksp_core_lib::Error::new(crate::ERROR_CODE_BACKFILL_SIGNATURE_INVALID, "invalid bounded Backfill transaction signature")
|
||||
.with_context("field", "signature");
|
||||
}
|
||||
|
||||
fn valid_signature_text(value: &str) -> bool {
|
||||
if value.len() < crate::MIN_BACKFILL_SIGNATURE_TEXT_BYTES || value.len() > crate::MAX_BACKFILL_SIGNATURE_TEXT_BYTES {
|
||||
return false;
|
||||
}
|
||||
return value.bytes().all(|byte| {
|
||||
return matches!(byte, b'1'..=b'9' | b'A'..=b'H' | b'J'..=b'N' | b'P'..=b'Z' | b'a'..=b'k' | b'm'..=b'z');
|
||||
});
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[path = "../unit_tests/request.rs"]
|
||||
mod tests;
|
||||
66
crates/ksp-job-backfill-lib/tests/dependency_boundary.rs
Normal file
66
crates/ksp-job-backfill-lib/tests/dependency_boundary.rs
Normal file
@@ -0,0 +1,66 @@
|
||||
// file: crates/ksp-job-backfill-lib/tests/dependency_boundary.rs
|
||||
// version: 1
|
||||
|
||||
//! Dependency firewall canaries for the concrete Backfill foundation.
|
||||
|
||||
#[test]
|
||||
fn pre_005_manifest_uses_only_planned_ksp_edges_and_backend_neutral_store() {
|
||||
let manifest = include_str!("../Cargo.toml");
|
||||
for required in [
|
||||
"ksp-core-lib = { path = \"../ksp-core-lib\" }",
|
||||
"ksp-job-api = { path = \"../ksp-job-api\" }",
|
||||
"ksp-logging-lib = { path = \"../ksp-logging-lib\" }",
|
||||
"ksp-onchain-transport-lib = { path = \"../ksp-onchain-transport-lib\" }",
|
||||
"ksp-store-lib = { path = \"../ksp-store-lib\", default-features = false }",
|
||||
"sha2.workspace = true",
|
||||
] {
|
||||
assert!(manifest.contains(required), "required Backfill dependency missing: {required}");
|
||||
}
|
||||
for forbidden in [
|
||||
"ksp-config-lib",
|
||||
"ksp-interface-lib",
|
||||
"ksp-program-api",
|
||||
"ksp-store-api",
|
||||
"ksp-store-postgres-lib",
|
||||
"ksp-wallet-lib",
|
||||
"solana-",
|
||||
"reqwest",
|
||||
"serde",
|
||||
"tonic",
|
||||
] {
|
||||
assert!(!manifest.contains(forbidden), "forbidden Backfill dependency present: {forbidden}");
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn pre_005_production_sources_keep_transport_and_store_in_their_owned_layers() {
|
||||
let sources = [
|
||||
include_str!("../src/constants.rs"),
|
||||
include_str!("../src/discovery.rs"),
|
||||
include_str!("../src/error.rs"),
|
||||
include_str!("../src/lib.rs"),
|
||||
include_str!("../src/request.rs"),
|
||||
];
|
||||
for source in sources {
|
||||
for forbidden in [
|
||||
"ksp_config_lib::",
|
||||
"ksp_interface_lib::",
|
||||
"ksp_store_api::",
|
||||
"ksp_store_postgres_lib::",
|
||||
"reqwest::",
|
||||
"serde_json::",
|
||||
"std::env",
|
||||
"tonic::",
|
||||
] {
|
||||
assert!(!source.contains(forbidden), "forbidden concrete Backfill path detected: {forbidden}");
|
||||
}
|
||||
}
|
||||
let discovery = include_str!("../src/discovery.rs");
|
||||
assert!(discovery.contains("get_signatures_for_address"));
|
||||
assert!(!discovery.contains("execute_standard_rpc"));
|
||||
assert!(!discovery.contains("retry"));
|
||||
assert!(!discovery.contains("endpoint_name"));
|
||||
assert!(!discovery.contains("HttpEndpoint"));
|
||||
return;
|
||||
}
|
||||
66
crates/ksp-job-backfill-lib/tests/public_api.rs
Normal file
66
crates/ksp-job-backfill-lib/tests/public_api.rs
Normal file
@@ -0,0 +1,66 @@
|
||||
// file: crates/ksp-job-backfill-lib/tests/public_api.rs
|
||||
// version: 1
|
||||
|
||||
//! Public API canaries for the bounded Backfill foundation.
|
||||
|
||||
#[test]
|
||||
fn pre_005_request_scope_and_discovery_contracts_are_available_from_crate_root() {
|
||||
let _discover = ksp_job_backfill_lib::discover_backfill_candidates;
|
||||
let address = ksp_core_lib::Pubkey::new_from_array([21_u8; 32]);
|
||||
let scope = ksp_job_backfill_lib::BackfillScope::latest_address(address);
|
||||
let network = ksp_store_lib::RawNetworkId::new("devnet");
|
||||
assert!(network.is_ok());
|
||||
let network = match network {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(_) => return,
|
||||
};
|
||||
let job_id = ksp_job_api::JobId::new("backfill:public-api");
|
||||
assert!(job_id.is_ok());
|
||||
let job_id = match job_id {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(_) => return,
|
||||
};
|
||||
let request = ksp_job_backfill_lib::BackfillRequest::new(
|
||||
job_id,
|
||||
network,
|
||||
ksp_onchain_transport_lib::HttpRoleName::new("history"),
|
||||
ksp_job_backfill_lib::BackfillCommitment::Confirmed,
|
||||
scope,
|
||||
100,
|
||||
10,
|
||||
500,
|
||||
8,
|
||||
std::option::Option::Some(100),
|
||||
);
|
||||
assert!(request.is_ok());
|
||||
let request = match request {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(_) => return,
|
||||
};
|
||||
assert_eq!(request.scope().kind(), ksp_job_backfill_lib::BackfillScopeKind::LatestAddress);
|
||||
assert_eq!(request.page_size(), 100);
|
||||
return;
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn pre_005_candidate_identity_is_network_plus_signature_not_transport_source() {
|
||||
let signature = ksp_job_backfill_lib::BackfillSignature::new("1".repeat(ksp_job_backfill_lib::MIN_BACKFILL_SIGNATURE_TEXT_BYTES));
|
||||
let signature = match signature {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(_) => return,
|
||||
};
|
||||
let mainnet = ksp_store_lib::RawNetworkId::new("mainnet");
|
||||
let devnet = ksp_store_lib::RawNetworkId::new("devnet");
|
||||
let mainnet = match mainnet {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(_) => return,
|
||||
};
|
||||
let devnet = match devnet {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(_) => return,
|
||||
};
|
||||
let mainnet_identity = ksp_job_backfill_lib::BackfillCandidateIdentity::new(mainnet, signature.clone());
|
||||
let devnet_identity = ksp_job_backfill_lib::BackfillCandidateIdentity::new(devnet, signature);
|
||||
assert_ne!(mainnet_identity, devnet_identity);
|
||||
return;
|
||||
}
|
||||
61
crates/ksp-job-backfill-lib/tests/release_completeness.rs
Normal file
61
crates/ksp-job-backfill-lib/tests/release_completeness.rs
Normal file
@@ -0,0 +1,61 @@
|
||||
// file: crates/ksp-job-backfill-lib/tests/release_completeness.rs
|
||||
// version: 1
|
||||
|
||||
//! Completeness canaries for the `pre.005` Backfill foundation.
|
||||
|
||||
#[test]
|
||||
fn pre_005_production_module_inventory_is_exact() -> std::io::Result<()> {
|
||||
let source_root = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("src");
|
||||
let entries = match std::fs::read_dir(source_root) {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(error) => return std::result::Result::Err(error),
|
||||
};
|
||||
let mut names = std::vec::Vec::new();
|
||||
for entry in entries {
|
||||
let entry = match entry {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(error) => return std::result::Result::Err(error),
|
||||
};
|
||||
let file_type = match entry.file_type() {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(error) => return std::result::Result::Err(error),
|
||||
};
|
||||
if !file_type.is_file() {
|
||||
continue;
|
||||
}
|
||||
let name = match entry.file_name().into_string() {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(_) => continue,
|
||||
};
|
||||
if name.ends_with(".rs") {
|
||||
names.push(name);
|
||||
}
|
||||
}
|
||||
names.sort_unstable();
|
||||
assert_eq!(names, std::vec!["constants.rs", "discovery.rs", "error.rs", "lib.rs", "request.rs"]);
|
||||
return std::result::Result::Ok(());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn pre_005_surface_is_discovery_only_without_raw_persistence_or_checkpoint_runtime() {
|
||||
let root = include_str!("../src/lib.rs");
|
||||
for required in [
|
||||
"BackfillCandidate",
|
||||
"BackfillCandidateIdentity",
|
||||
"BackfillCommitment",
|
||||
"BackfillDiscovery",
|
||||
"BackfillDiscoveryBoundary",
|
||||
"BackfillRequest",
|
||||
"BackfillScope",
|
||||
"BackfillScopeFingerprint",
|
||||
"BackfillSignature",
|
||||
"discover_backfill_candidates",
|
||||
] {
|
||||
assert!(root.contains(required), "required pre.005 public contract missing: {required}");
|
||||
}
|
||||
for forbidden in ["RawTransactionObservation", "persist_raw_transaction_acquisition", "BackfillCheckpoint", "BackfillJobHandle", "JobSnapshotSource"] {
|
||||
assert!(!root.contains(forbidden), "later Backfill tranche leaked into pre.005: {forbidden}");
|
||||
}
|
||||
assert!(!root.contains("pub mod "));
|
||||
return;
|
||||
}
|
||||
311
crates/ksp-job-backfill-lib/unit_tests/discovery.rs
Normal file
311
crates/ksp-job-backfill-lib/unit_tests/discovery.rs
Normal file
@@ -0,0 +1,311 @@
|
||||
// file: crates/ksp-job-backfill-lib/unit_tests/discovery.rs
|
||||
// version: 1
|
||||
|
||||
#[derive(Clone, Debug, Eq, PartialEq)]
|
||||
struct PageCall {
|
||||
before: std::option::Option<std::string::String>,
|
||||
until: std::option::Option<std::string::String>,
|
||||
limit: std::option::Option<usize>,
|
||||
commitment: std::option::Option<ksp_onchain_transport_lib::SolanaCommitment>,
|
||||
min_context_slot: std::option::Option<u64>,
|
||||
}
|
||||
|
||||
struct FakeSource {
|
||||
pages: std::sync::Mutex<std::collections::VecDeque<std::vec::Vec<super::SignaturePageEntry>>>,
|
||||
calls: std::sync::Mutex<std::vec::Vec<PageCall>>,
|
||||
}
|
||||
|
||||
impl FakeSource {
|
||||
fn new(pages: std::vec::Vec<std::vec::Vec<super::SignaturePageEntry>>) -> Self {
|
||||
return Self { pages: std::sync::Mutex::new(pages.into()), calls: std::sync::Mutex::new(std::vec::Vec::new()) };
|
||||
}
|
||||
|
||||
fn calls(&self) -> std::vec::Vec<PageCall> {
|
||||
let guard = self.calls.lock();
|
||||
return match guard {
|
||||
std::result::Result::Ok(value) => value.clone(),
|
||||
std::result::Result::Err(_) => std::vec::Vec::new(),
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
impl super::SignaturePageSource for FakeSource {
|
||||
fn fetch_signature_page<'a>(
|
||||
&'a self,
|
||||
_role: &'a ksp_onchain_transport_lib::HttpRoleName,
|
||||
_address: &'a ksp_core_lib::Pubkey,
|
||||
config: ksp_onchain_transport_lib::SolanaSignaturesForAddressConfig,
|
||||
) -> super::SignaturePageFuture<'a> {
|
||||
let call = PageCall {
|
||||
before: config.before().map(str::to_owned),
|
||||
until: config.until().map(str::to_owned),
|
||||
limit: config.limit(),
|
||||
commitment: config.commitment(),
|
||||
min_context_slot: config.min_context_slot(),
|
||||
};
|
||||
let calls_result = self.calls.lock();
|
||||
match calls_result {
|
||||
std::result::Result::Ok(mut calls) => calls.push(call),
|
||||
std::result::Result::Err(_) => {
|
||||
return std::boxed::Box::pin(async {
|
||||
return std::result::Result::Err(ksp_core_lib::Error::new(
|
||||
crate::ERROR_CODE_BACKFILL_DISCOVERY_INVALID,
|
||||
"test call recorder lock poisoned",
|
||||
));
|
||||
});
|
||||
},
|
||||
}
|
||||
let pages_result = self.pages.lock();
|
||||
let page = match pages_result {
|
||||
std::result::Result::Ok(mut pages) => match pages.pop_front() {
|
||||
std::option::Option::Some(value) => value,
|
||||
std::option::Option::None => std::vec::Vec::new(),
|
||||
},
|
||||
std::result::Result::Err(_) => {
|
||||
return std::boxed::Box::pin(async {
|
||||
return std::result::Result::Err(ksp_core_lib::Error::new(
|
||||
crate::ERROR_CODE_BACKFILL_DISCOVERY_INVALID,
|
||||
"test page queue lock poisoned",
|
||||
));
|
||||
});
|
||||
},
|
||||
};
|
||||
return std::boxed::Box::pin(async move { return std::result::Result::Ok(page); });
|
||||
}
|
||||
}
|
||||
|
||||
fn page_entry(character: char, slot: u64) -> super::SignaturePageEntry {
|
||||
return super::SignaturePageEntry { signature: character.to_string().repeat(crate::MIN_BACKFILL_SIGNATURE_TEXT_BYTES), slot };
|
||||
}
|
||||
|
||||
fn signature(character: char) -> std::option::Option<crate::BackfillSignature> {
|
||||
return match crate::BackfillSignature::new(character.to_string().repeat(crate::MIN_BACKFILL_SIGNATURE_TEXT_BYTES)) {
|
||||
std::result::Result::Ok(value) => std::option::Option::Some(value),
|
||||
std::result::Result::Err(_) => std::option::Option::None,
|
||||
};
|
||||
}
|
||||
|
||||
fn request(scope: crate::BackfillScope, page_size: usize, max_pages: usize, max_candidates: usize) -> std::option::Option<crate::BackfillRequest> {
|
||||
let job_id = match ksp_job_api::JobId::new("backfill:discovery-test") {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(_) => return std::option::Option::None,
|
||||
};
|
||||
let network = match ksp_store_lib::RawNetworkId::new("devnet") {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(_) => return std::option::Option::None,
|
||||
};
|
||||
let result = crate::BackfillRequest::new(
|
||||
job_id,
|
||||
network,
|
||||
ksp_onchain_transport_lib::HttpRoleName::new("history"),
|
||||
crate::BackfillCommitment::Confirmed,
|
||||
scope,
|
||||
page_size,
|
||||
max_pages,
|
||||
max_candidates,
|
||||
1,
|
||||
std::option::Option::Some(42),
|
||||
);
|
||||
return match result {
|
||||
std::result::Result::Ok(value) => std::option::Option::Some(value),
|
||||
std::result::Result::Err(_) => std::option::Option::None,
|
||||
};
|
||||
}
|
||||
|
||||
fn candidate_signatures(discovery: &crate::BackfillDiscovery) -> std::vec::Vec<std::string::String> {
|
||||
let mut signatures = std::vec::Vec::with_capacity(discovery.candidates().len());
|
||||
for candidate in discovery.candidates() {
|
||||
signatures.push(candidate.identity().signature().as_str().to_owned());
|
||||
}
|
||||
return signatures;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn pre_005_latest_paginates_newest_first_and_deduplicates_pages_stably() {
|
||||
let scope = crate::BackfillScope::latest_address(ksp_core_lib::Pubkey::new_from_array([1_u8; 32]));
|
||||
let request = match request(scope, 3, 4, 10) {
|
||||
std::option::Option::Some(value) => value,
|
||||
std::option::Option::None => return,
|
||||
};
|
||||
let source = FakeSource::new(std::vec![
|
||||
std::vec![page_entry('6', 60), page_entry('5', 50), page_entry('5', 50)],
|
||||
std::vec![page_entry('4', 40), page_entry('3', 30)],
|
||||
]);
|
||||
let result = super::discover_with_source(&source, &request).await;
|
||||
assert!(result.is_ok());
|
||||
let discovery = match result {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(_) => return,
|
||||
};
|
||||
assert_eq!(candidate_signatures(&discovery), std::vec!["6".repeat(64), "5".repeat(64), "4".repeat(64), "3".repeat(64)]);
|
||||
assert_eq!(discovery.pages_fetched(), 2);
|
||||
assert_eq!(discovery.boundary(), crate::BackfillDiscoveryBoundary::RpcBoundary);
|
||||
assert!(!discovery.is_partial());
|
||||
let calls = source.calls();
|
||||
assert_eq!(calls.len(), 2);
|
||||
assert_eq!(calls[0].before, std::option::Option::None);
|
||||
let expected_cursor = "5".repeat(64);
|
||||
assert_eq!(calls[1].before.as_deref(), std::option::Option::Some(expected_cursor.as_str()));
|
||||
assert_eq!(calls[0].until, std::option::Option::None);
|
||||
assert_eq!(calls[0].limit, std::option::Option::Some(3));
|
||||
assert_eq!(calls[0].commitment, std::option::Option::Some(ksp_onchain_transport_lib::SolanaCommitment::Confirmed));
|
||||
assert_eq!(calls[0].min_context_slot, std::option::Option::Some(42));
|
||||
return;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn pre_005_before_uses_exclusive_anchor_then_advances_rpc_cursor() {
|
||||
let anchor = match signature('7') {
|
||||
std::option::Option::Some(value) => value,
|
||||
std::option::Option::None => return,
|
||||
};
|
||||
let scope = crate::BackfillScope::before_address(ksp_core_lib::Pubkey::new_from_array([2_u8; 32]), anchor.clone());
|
||||
let request = match request(scope, 2, 3, 5) {
|
||||
std::option::Option::Some(value) => value,
|
||||
std::option::Option::None => return,
|
||||
};
|
||||
let source = FakeSource::new(std::vec![std::vec![page_entry('6', 60), page_entry('5', 50)], std::vec![page_entry('4', 40)]]);
|
||||
let result = super::discover_with_source(&source, &request).await;
|
||||
let discovery = match result {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(_) => return,
|
||||
};
|
||||
assert_eq!(candidate_signatures(&discovery), std::vec!["6".repeat(64), "5".repeat(64), "4".repeat(64)]);
|
||||
let calls = source.calls();
|
||||
assert_eq!(calls.len(), 2);
|
||||
assert_eq!(calls[0].before.as_deref(), std::option::Option::Some(anchor.as_str()));
|
||||
let expected_cursor = "5".repeat(64);
|
||||
assert_eq!(calls[1].before.as_deref(), std::option::Option::Some(expected_cursor.as_str()));
|
||||
assert_eq!(calls[0].until, std::option::Option::None);
|
||||
return;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn pre_005_after_keeps_only_nearest_newer_window_and_preserves_rpc_order() {
|
||||
let anchor = match signature('1') {
|
||||
std::option::Option::Some(value) => value,
|
||||
std::option::Option::None => return,
|
||||
};
|
||||
let scope = crate::BackfillScope::after_address(ksp_core_lib::Pubkey::new_from_array([3_u8; 32]), anchor.clone());
|
||||
let request = match request(scope, 3, 3, 3) {
|
||||
std::option::Option::Some(value) => value,
|
||||
std::option::Option::None => return,
|
||||
};
|
||||
let source = FakeSource::new(std::vec![
|
||||
std::vec![page_entry('7', 70), page_entry('6', 60), page_entry('5', 50)],
|
||||
std::vec![page_entry('4', 40), page_entry('3', 30)],
|
||||
]);
|
||||
let result = super::discover_with_source(&source, &request).await;
|
||||
let discovery = match result {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(_) => return,
|
||||
};
|
||||
assert_eq!(candidate_signatures(&discovery), std::vec!["5".repeat(64), "4".repeat(64), "3".repeat(64)]);
|
||||
assert_eq!(discovery.boundary(), crate::BackfillDiscoveryBoundary::RpcBoundary);
|
||||
assert!(!discovery.is_partial());
|
||||
let calls = source.calls();
|
||||
assert_eq!(calls.len(), 2);
|
||||
assert_eq!(calls[0].until.as_deref(), std::option::Option::Some(anchor.as_str()));
|
||||
assert_eq!(calls[1].until.as_deref(), std::option::Option::Some(anchor.as_str()));
|
||||
let expected_cursor = "5".repeat(64);
|
||||
assert_eq!(calls[1].before.as_deref(), std::option::Option::Some(expected_cursor.as_str()));
|
||||
return;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn pre_005_after_page_bound_is_partial_and_does_not_claim_anchor_completion() {
|
||||
let anchor = match signature('1') {
|
||||
std::option::Option::Some(value) => value,
|
||||
std::option::Option::None => return,
|
||||
};
|
||||
let scope = crate::BackfillScope::after_address(ksp_core_lib::Pubkey::new_from_array([4_u8; 32]), anchor);
|
||||
let request = match request(scope, 2, 2, 3) {
|
||||
std::option::Option::Some(value) => value,
|
||||
std::option::Option::None => return,
|
||||
};
|
||||
let source = FakeSource::new(std::vec![
|
||||
std::vec![page_entry('7', 70), page_entry('6', 60)],
|
||||
std::vec![page_entry('5', 50), page_entry('4', 40)],
|
||||
]);
|
||||
let result = super::discover_with_source(&source, &request).await;
|
||||
let discovery = match result {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(_) => return,
|
||||
};
|
||||
assert_eq!(candidate_signatures(&discovery), std::vec!["6".repeat(64), "5".repeat(64), "4".repeat(64)]);
|
||||
assert_eq!(discovery.boundary(), crate::BackfillDiscoveryBoundary::AfterAnchorNotReached);
|
||||
assert!(discovery.is_partial());
|
||||
assert_eq!(discovery.pages_fetched(), 2);
|
||||
return;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn pre_005_latest_page_bound_is_partial_when_full_pages_leave_more_history_possible() {
|
||||
let scope = crate::BackfillScope::latest_address(ksp_core_lib::Pubkey::new_from_array([5_u8; 32]));
|
||||
let request = match request(scope, 2, 1, 5) {
|
||||
std::option::Option::Some(value) => value,
|
||||
std::option::Option::None => return,
|
||||
};
|
||||
let source = FakeSource::new(std::vec![std::vec![page_entry('7', 70), page_entry('6', 60)]]);
|
||||
let result = super::discover_with_source(&source, &request).await;
|
||||
let discovery = match result {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(_) => return,
|
||||
};
|
||||
assert_eq!(discovery.boundary(), crate::BackfillDiscoveryBoundary::PageLimit);
|
||||
assert!(discovery.is_partial());
|
||||
return;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn pre_005_explicit_scope_never_calls_transport_and_preserves_network_scoped_identity() {
|
||||
let first = match signature('2') {
|
||||
std::option::Option::Some(value) => value,
|
||||
std::option::Option::None => return,
|
||||
};
|
||||
let second = match signature('3') {
|
||||
std::option::Option::Some(value) => value,
|
||||
std::option::Option::None => return,
|
||||
};
|
||||
let scope_result = crate::BackfillScope::explicit_signatures(std::vec![first.clone(), second.clone(), first]);
|
||||
let scope = match scope_result {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(_) => return,
|
||||
};
|
||||
let job_id = match ksp_job_api::JobId::new("backfill:explicit") {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(_) => return,
|
||||
};
|
||||
let network = match ksp_store_lib::RawNetworkId::new("synthetic") {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(_) => return,
|
||||
};
|
||||
let request_result = crate::BackfillRequest::new(
|
||||
job_id,
|
||||
network.clone(),
|
||||
ksp_onchain_transport_lib::HttpRoleName::new("unused"),
|
||||
crate::BackfillCommitment::Finalized,
|
||||
scope,
|
||||
100,
|
||||
10,
|
||||
10,
|
||||
1,
|
||||
std::option::Option::None,
|
||||
);
|
||||
let request = match request_result {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(_) => return,
|
||||
};
|
||||
let source = FakeSource::new(std::vec::Vec::new());
|
||||
let result = super::discover_with_source(&source, &request).await;
|
||||
let discovery = match result {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(_) => return,
|
||||
};
|
||||
assert_eq!(discovery.boundary(), crate::BackfillDiscoveryBoundary::ExplicitInput);
|
||||
assert_eq!(discovery.pages_fetched(), 0);
|
||||
assert_eq!(discovery.candidates().len(), 2);
|
||||
assert_eq!(discovery.candidates()[0].identity().network(), &network);
|
||||
assert!(source.calls().is_empty());
|
||||
return;
|
||||
}
|
||||
244
crates/ksp-job-backfill-lib/unit_tests/request.rs
Normal file
244
crates/ksp-job-backfill-lib/unit_tests/request.rs
Normal file
@@ -0,0 +1,244 @@
|
||||
// file: crates/ksp-job-backfill-lib/unit_tests/request.rs
|
||||
// version: 1
|
||||
|
||||
fn signature(character: char) -> std::option::Option<crate::BackfillSignature> {
|
||||
return match crate::BackfillSignature::new(character.to_string().repeat(crate::MIN_BACKFILL_SIGNATURE_TEXT_BYTES)) {
|
||||
std::result::Result::Ok(value) => std::option::Option::Some(value),
|
||||
std::result::Result::Err(_) => std::option::Option::None,
|
||||
};
|
||||
}
|
||||
|
||||
fn network(value: &str) -> std::option::Option<ksp_store_lib::RawNetworkId> {
|
||||
return match ksp_store_lib::RawNetworkId::new(value) {
|
||||
std::result::Result::Ok(value) => std::option::Option::Some(value),
|
||||
std::result::Result::Err(_) => std::option::Option::None,
|
||||
};
|
||||
}
|
||||
|
||||
fn job_id(value: &str) -> std::option::Option<ksp_job_api::JobId> {
|
||||
return match ksp_job_api::JobId::new(value) {
|
||||
std::result::Result::Ok(value) => std::option::Option::Some(value),
|
||||
std::result::Result::Err(_) => std::option::Option::None,
|
||||
};
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn pre_005_signature_shape_is_bounded_and_debug_redacted() {
|
||||
let minimum = crate::BackfillSignature::new("1".repeat(crate::MIN_BACKFILL_SIGNATURE_TEXT_BYTES));
|
||||
assert!(minimum.is_ok());
|
||||
let maximum = crate::BackfillSignature::new("z".repeat(crate::MAX_BACKFILL_SIGNATURE_TEXT_BYTES));
|
||||
assert!(maximum.is_ok());
|
||||
assert!(crate::BackfillSignature::new("1".repeat(crate::MIN_BACKFILL_SIGNATURE_TEXT_BYTES - 1)).is_err());
|
||||
assert!(crate::BackfillSignature::new("1".repeat(crate::MAX_BACKFILL_SIGNATURE_TEXT_BYTES + 1)).is_err());
|
||||
assert!(crate::BackfillSignature::new(format!("{}0", "1".repeat(crate::MIN_BACKFILL_SIGNATURE_TEXT_BYTES - 1))).is_err());
|
||||
let minimum = match minimum {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(_) => return,
|
||||
};
|
||||
let debug = format!("{minimum:?}");
|
||||
assert_eq!(debug, "BackfillSignature(..)");
|
||||
assert!(!debug.contains(minimum.as_str()));
|
||||
return;
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn pre_005_explicit_scope_deduplicates_at_first_occurrence_without_reordering() {
|
||||
let first = match signature('1') {
|
||||
std::option::Option::Some(value) => value,
|
||||
std::option::Option::None => return,
|
||||
};
|
||||
let second = match signature('2') {
|
||||
std::option::Option::Some(value) => value,
|
||||
std::option::Option::None => return,
|
||||
};
|
||||
let scope_result = crate::BackfillScope::explicit_signatures(std::vec![first.clone(), second.clone(), first.clone(), second.clone()]);
|
||||
assert!(scope_result.is_ok());
|
||||
let scope = match scope_result {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(_) => return,
|
||||
};
|
||||
let signatures = match scope.signatures() {
|
||||
std::option::Option::Some(value) => value,
|
||||
std::option::Option::None => return,
|
||||
};
|
||||
assert_eq!(signatures, &[first, second]);
|
||||
let debug = format!("{scope:?}");
|
||||
assert!(debug.contains("signature_count"));
|
||||
assert!(!debug.contains(signatures[0].as_str()));
|
||||
return;
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn pre_005_request_bounds_are_exact_and_explicit_context_is_rejected() {
|
||||
let network = match network("devnet") {
|
||||
std::option::Option::Some(value) => value,
|
||||
std::option::Option::None => return,
|
||||
};
|
||||
let job_id = match job_id("backfill:test") {
|
||||
std::option::Option::Some(value) => value,
|
||||
std::option::Option::None => return,
|
||||
};
|
||||
let role = ksp_onchain_transport_lib::HttpRoleName::new("history");
|
||||
let address = ksp_core_lib::Pubkey::new_from_array([7_u8; 32]);
|
||||
let scope = crate::BackfillScope::latest_address(address);
|
||||
let valid = crate::BackfillRequest::new(
|
||||
job_id.clone(),
|
||||
network.clone(),
|
||||
role.clone(),
|
||||
crate::BackfillCommitment::Confirmed,
|
||||
scope.clone(),
|
||||
crate::MAX_BACKFILL_PAGE_SIZE,
|
||||
crate::MAX_BACKFILL_PAGES,
|
||||
crate::MAX_BACKFILL_CANDIDATES,
|
||||
crate::MAX_BACKFILL_HYDRATION_CONCURRENCY,
|
||||
std::option::Option::Some(1),
|
||||
);
|
||||
assert!(valid.is_ok());
|
||||
for (page_size, max_pages, max_candidates, concurrency) in [
|
||||
(0, 1, 1, 1),
|
||||
(crate::MAX_BACKFILL_PAGE_SIZE + 1, 1, 1, 1),
|
||||
(1, 0, 1, 1),
|
||||
(1, crate::MAX_BACKFILL_PAGES + 1, 1, 1),
|
||||
(1, 1, 0, 1),
|
||||
(1, 1, crate::MAX_BACKFILL_CANDIDATES + 1, 1),
|
||||
(1, 1, 1, 0),
|
||||
(1, 1, 1, crate::MAX_BACKFILL_HYDRATION_CONCURRENCY + 1),
|
||||
] {
|
||||
let result = crate::BackfillRequest::new(
|
||||
job_id.clone(),
|
||||
network.clone(),
|
||||
role.clone(),
|
||||
crate::BackfillCommitment::Finalized,
|
||||
scope.clone(),
|
||||
page_size,
|
||||
max_pages,
|
||||
max_candidates,
|
||||
concurrency,
|
||||
std::option::Option::None,
|
||||
);
|
||||
assert!(result.is_err());
|
||||
}
|
||||
let explicit_signature = match signature('3') {
|
||||
std::option::Option::Some(value) => value,
|
||||
std::option::Option::None => return,
|
||||
};
|
||||
let explicit_scope = crate::BackfillScope::explicit_signatures(std::vec![explicit_signature]);
|
||||
let explicit_scope = match explicit_scope {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(_) => return,
|
||||
};
|
||||
let explicit_with_context = crate::BackfillRequest::new(
|
||||
job_id,
|
||||
network,
|
||||
role,
|
||||
crate::BackfillCommitment::Confirmed,
|
||||
explicit_scope,
|
||||
1,
|
||||
1,
|
||||
1,
|
||||
1,
|
||||
std::option::Option::Some(10),
|
||||
);
|
||||
assert!(explicit_with_context.is_err());
|
||||
return;
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn pre_005_scope_fingerprint_is_network_semantic_and_transport_source_independent() {
|
||||
let signature = match signature('4') {
|
||||
std::option::Option::Some(value) => value,
|
||||
std::option::Option::None => return,
|
||||
};
|
||||
let scope_result = crate::BackfillScope::explicit_signatures(std::vec![signature]);
|
||||
let scope = match scope_result {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(_) => return,
|
||||
};
|
||||
let mainnet = match network("mainnet") {
|
||||
std::option::Option::Some(value) => value,
|
||||
std::option::Option::None => return,
|
||||
};
|
||||
let devnet = match network("devnet") {
|
||||
std::option::Option::Some(value) => value,
|
||||
std::option::Option::None => return,
|
||||
};
|
||||
let first_job = match job_id("backfill:first") {
|
||||
std::option::Option::Some(value) => value,
|
||||
std::option::Option::None => return,
|
||||
};
|
||||
let second_job = match job_id("backfill:second") {
|
||||
std::option::Option::Some(value) => value,
|
||||
std::option::Option::None => return,
|
||||
};
|
||||
let first = crate::BackfillRequest::new(
|
||||
first_job,
|
||||
devnet.clone(),
|
||||
ksp_onchain_transport_lib::HttpRoleName::new("primary-http"),
|
||||
crate::BackfillCommitment::Confirmed,
|
||||
scope.clone(),
|
||||
100,
|
||||
5,
|
||||
10,
|
||||
1,
|
||||
std::option::Option::None,
|
||||
);
|
||||
let second = crate::BackfillRequest::new(
|
||||
second_job,
|
||||
devnet,
|
||||
ksp_onchain_transport_lib::HttpRoleName::new("fallback-http"),
|
||||
crate::BackfillCommitment::Confirmed,
|
||||
scope.clone(),
|
||||
100,
|
||||
5,
|
||||
10,
|
||||
64,
|
||||
std::option::Option::None,
|
||||
);
|
||||
let other_network = crate::BackfillRequest::new(
|
||||
match job_id("backfill:third") {
|
||||
std::option::Option::Some(value) => value,
|
||||
std::option::Option::None => return,
|
||||
},
|
||||
mainnet,
|
||||
ksp_onchain_transport_lib::HttpRoleName::new("primary-http"),
|
||||
crate::BackfillCommitment::Confirmed,
|
||||
scope,
|
||||
100,
|
||||
5,
|
||||
10,
|
||||
1,
|
||||
std::option::Option::None,
|
||||
);
|
||||
let first = match first {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(_) => return,
|
||||
};
|
||||
let second = match second {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(_) => return,
|
||||
};
|
||||
let other_network = match other_network {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(_) => return,
|
||||
};
|
||||
assert_eq!(first.scope_fingerprint(), second.scope_fingerprint());
|
||||
assert_ne!(first.scope_fingerprint(), other_network.scope_fingerprint());
|
||||
return;
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn pre_005_scope_kind_and_anchor_are_distinct_semantics() {
|
||||
let anchor = match signature('5') {
|
||||
std::option::Option::Some(value) => value,
|
||||
std::option::Option::None => return,
|
||||
};
|
||||
let address = ksp_core_lib::Pubkey::new_from_array([9_u8; 32]);
|
||||
let before = crate::BackfillScope::before_address(address, anchor.clone());
|
||||
let after = crate::BackfillScope::after_address(address, anchor);
|
||||
assert_eq!(before.kind(), crate::BackfillScopeKind::BeforeAddress);
|
||||
assert_eq!(after.kind(), crate::BackfillScopeKind::AfterAddress);
|
||||
assert!(before.anchor().is_some());
|
||||
assert!(after.anchor().is_some());
|
||||
assert_ne!(before, after);
|
||||
return;
|
||||
}
|
||||
Reference in New Issue
Block a user