v0.3.13-pre.002

This commit is contained in:
2026-09-10 09:50:42 +02:00
parent ee0359efd5
commit 793178b345
15 changed files with 976 additions and 41 deletions

View File

@@ -1,5 +1,5 @@
// file: crates/ksp-onchain-transport-lib/src/grpc_subscribe.rs
// version: 8
// version: 9
const MAX_GRPC_BLOCKHASH_TEXT_LENGTH_BYTES: usize = 128;
const MAX_GRPC_BLOCK_VECTOR_COUNT: usize = 65_536;
@@ -2529,13 +2529,34 @@ impl YellowstoneSubscribeEntryFilter {
}
}
/// Opaque deterministic identity material for one complete Yellowstone Subscribe request.
///
/// The encoded bytes remain private. `Hash` feeds those canonical bytes to a caller-provided hasher while `Debug` exposes only their length.
#[derive(Clone, Eq, PartialEq)]
pub struct YellowstoneSubscribeRequestIdentity {
bytes: std::vec::Vec<u8>,
}
impl std::hash::Hash for crate::YellowstoneSubscribeRequestIdentity {
fn hash<H: std::hash::Hasher>(&self, state: &mut H) {
state.write(&(self.bytes.len() as u64).to_be_bytes());
state.write(self.bytes.as_slice());
return;
}
}
impl std::fmt::Debug for crate::YellowstoneSubscribeRequestIdentity {
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
return formatter.debug_struct("YellowstoneSubscribeRequestIdentity").field("byte_len", &self.bytes.len()).finish();
}
}
/// Provider-neutral standard Yellowstone subscribe request owned by KSP.
///
/// The seven upstream maps are represented independently and retain named empty entries. An entirely empty map is the logical KSP representation of no active
/// filter in that family; protobuf map encoding does not distinguish an omitted map from an empty map. Filter-group names are globally unique across all seven
/// maps so the names echoed by `SubscribeUpdate.filters` remain unambiguous. `Debug` exposes only counts and common scalar options, never filter names or
/// future
/// filter payloads.
/// future filter payloads.
#[derive(Clone, Eq, PartialEq)]
pub struct YellowstoneSubscribeRequest {
accounts: std::collections::BTreeMap<crate::YellowstoneSubscribeFilterName, crate::YellowstoneSubscribeAccountFilter>,
@@ -2551,7 +2572,7 @@ pub struct YellowstoneSubscribeRequest {
from_slot: std::option::Option<u64>,
}
impl YellowstoneSubscribeRequest {
impl crate::YellowstoneSubscribeRequest {
/// Creates an empty subscribe request. Empty requests are valid because later bidi lifecycle code uses request mutations to clear filters or carry ping
/// state.
#[must_use]
@@ -2762,6 +2783,40 @@ impl YellowstoneSubscribeRequest {
return self.from_slot;
}
/// Builds one opaque deterministic identity for the complete logical Subscribe request.
///
/// The identity preserves filter-family separation, globally sorted filter names, exact filter wire payloads and common request options. Its Debug surface
/// exposes only the encoded byte length. Callers may hash the opaque value but cannot recover the underlying identity bytes through this API.
pub fn identity(&self) -> ksp_core_lib::Result<crate::YellowstoneSubscribeRequestIdentity> {
let validation = self.validate();
if let std::result::Result::Err(error) = validation {
return std::result::Result::Err(error);
}
let mut bytes = std::vec::Vec::new();
append_subscribe_identity_map(&mut bytes, b"accounts", &self.accounts, |filter| return filter.to_wire());
append_subscribe_identity_map(&mut bytes, b"blocks", &self.blocks, |filter| return filter.to_wire());
append_subscribe_identity_map(&mut bytes, b"blocks_meta", &self.blocks_meta, |filter| return filter.to_wire());
append_subscribe_identity_map(&mut bytes, b"entry", &self.entry, |filter| return filter.to_wire());
append_subscribe_identity_map(&mut bytes, b"slots", &self.slots, |filter| return filter.to_wire());
append_subscribe_identity_map(&mut bytes, b"transactions", &self.transactions, |filter| return filter.to_wire());
append_subscribe_identity_map(&mut bytes, b"transactions_status", &self.transactions_status, |filter| return filter.to_wire());
let mut common = match self.to_wire() {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
common.accounts.clear();
common.blocks.clear();
common.blocks_meta.clear();
common.entry.clear();
common.slots.clear();
common.transactions.clear();
common.transactions_status.clear();
let common = yellowstone_grpc_proto::prost::Message::encode_to_vec(&common);
append_subscribe_identity_component(&mut bytes, b"common");
append_subscribe_identity_component(&mut bytes, common.as_slice());
return std::result::Result::Ok(crate::YellowstoneSubscribeRequestIdentity { bytes });
}
/// Validates all deterministic common subscribe-request bounds before any network I/O.
pub fn validate(&self) -> ksp_core_lib::Result<()> {
if self.total_filter_count() > MAX_GRPC_SUBSCRIBE_FILTER_GROUP_COUNT {
@@ -3710,13 +3765,38 @@ fn commitment_to_wire(commitment: std::option::Option<crate::SolanaCommitment>)
});
}
impl std::default::Default for YellowstoneSubscribeRequest {
fn append_subscribe_identity_component(output: &mut std::vec::Vec<u8>, value: &[u8]) {
output.extend_from_slice(&(value.len() as u64).to_be_bytes());
output.extend_from_slice(value);
return;
}
fn append_subscribe_identity_map<V, W, F>(
output: &mut std::vec::Vec<u8>,
family: &[u8],
values: &std::collections::BTreeMap<crate::YellowstoneSubscribeFilterName, V>,
mut to_wire: F,
) where
W: yellowstone_grpc_proto::prost::Message,
F: FnMut(&V) -> W,
{
append_subscribe_identity_component(output, family);
output.extend_from_slice(&(values.len() as u64).to_be_bytes());
for (name, filter) in values {
append_subscribe_identity_component(output, name.as_str().as_bytes());
let wire = yellowstone_grpc_proto::prost::Message::encode_to_vec(&to_wire(filter));
append_subscribe_identity_component(output, wire.as_slice());
}
return;
}
impl std::default::Default for crate::YellowstoneSubscribeRequest {
fn default() -> Self {
return Self::new();
}
}
impl std::fmt::Debug for YellowstoneSubscribeRequest {
impl std::fmt::Debug for crate::YellowstoneSubscribeRequest {
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
return formatter
.debug_struct("YellowstoneSubscribeRequest")

View File

@@ -1,5 +1,5 @@
// file: crates/ksp-onchain-transport-lib/src/lib.rs
// version: 46
// version: 47
#![warn(missing_docs)]
#![deny(unreachable_pub)]
@@ -215,6 +215,8 @@ pub use self::grpc_subscribe::YellowstoneSubscribePingUpdate;
pub use self::grpc_subscribe::YellowstoneSubscribePongUpdate;
/// Provider-neutral standard Yellowstone Subscribe request.
pub use self::grpc_subscribe::YellowstoneSubscribeRequest;
/// Opaque deterministic identity for one complete Yellowstone Subscribe request.
pub use self::grpc_subscribe::YellowstoneSubscribeRequestIdentity;
/// Complete slot-family filter group for standard Yellowstone Subscribe.
pub use self::grpc_subscribe::YellowstoneSubscribeSlotFilter;
/// Complete transaction-family filter shared by transactions and transaction-status maps.

View File

@@ -1,5 +1,5 @@
// file: crates/ksp-onchain-transport-lib/tests/public_api.rs
// version: 51
// version: 52
//! Integration tests for the public `ksp-onchain-transport-lib` consumer contract.
@@ -1032,3 +1032,15 @@ fn public_v0_3_10_pre_004_observed_get_block_surface_is_available_from_crate_roo
let _ = method;
return;
}
#[test]
fn public_v0_3_13_pre_002_yellowstone_subscribe_identity_is_opaque_and_available_from_crate_root() {
let request = ksp_onchain_transport_lib::YellowstoneSubscribeRequest::new();
let identity = request.identity().expect("empty validated request identity must build");
let _identity_type = std::any::type_name::<ksp_onchain_transport_lib::YellowstoneSubscribeRequestIdentity>();
let debug = std::format!("{identity:?}");
assert!(debug.contains("YellowstoneSubscribeRequestIdentity"));
assert!(debug.contains("byte_len"));
assert!(!debug.contains("bytes:"));
return;
}

View File

@@ -1,5 +1,5 @@
// file: crates/ksp-onchain-transport-lib/tests/release_completeness.rs
// version: 43
// version: 44
//! Release-level completeness canaries for staged HTTP and WebSocket Transport coverage.
@@ -1339,3 +1339,18 @@ fn release_v0_2_9_pre_010_adds_bounded_reconnect_replay_and_conservative_continu
let _snapshot = std::any::type_name::<ksp_onchain_transport_lib::YellowstoneGrpcSubscribeSnapshot>();
let _state = ksp_onchain_transport_lib::YellowstoneGrpcSubscribeState::Reconnecting;
}
#[test]
fn release_v0_3_13_pre_002_yellowstone_subscribe_identity_remains_opaque_and_dependency_neutral() {
let source = include_str!("../src/grpc_subscribe.rs");
let root = include_str!("../src/lib.rs");
let identity_start = source.find("pub struct YellowstoneSubscribeRequestIdentity").expect("identity struct must remain present");
let request_start = source.find("/// Provider-neutral standard Yellowstone subscribe request").expect("request contract marker must remain present");
let identity_surface = &source[identity_start..request_start];
assert!(source.contains("pub fn identity(&self)"));
assert!(root.contains("pub use self::grpc_subscribe::YellowstoneSubscribeRequestIdentity;"));
assert!(!identity_surface.contains("pub fn bytes("));
assert!(!identity_surface.contains("pub fn as_bytes("));
assert!(!root.contains("pub use yellowstone_grpc_proto"));
return;
}

View File

@@ -1,5 +1,5 @@
// file: crates/ksp-onchain-transport-lib/unit_tests/grpc_subscribe.rs
// version: 5
// version: 6
fn filter_name(value: &str) -> crate::YellowstoneSubscribeFilterName {
return crate::YellowstoneSubscribeFilterName::new(value).expect("fixture filter name must validate");
@@ -138,6 +138,37 @@ fn yellowstone_subscribe_common_bounds_reject_before_wire_conversion() {
assert!(filters.insert_account_filter(excess_name, crate::YellowstoneSubscribeAccountFilter::new()).is_err());
}
#[test]
fn yellowstone_subscribe_request_identity_is_order_stable_exact_and_debug_redacted() {
let mut first = crate::YellowstoneSubscribeRequest::new();
let mut second = crate::YellowstoneSubscribeRequest::new();
let mut changed = crate::YellowstoneSubscribeRequest::new();
let mut filter_a = crate::YellowstoneSubscribeTransactionFilter::new();
filter_a.set_failed(std::option::Option::Some(false));
let mut filter_b = crate::YellowstoneSubscribeTransactionFilter::new();
filter_b.set_vote(std::option::Option::Some(false));
assert!(first.insert_transaction_filter(filter_name("identity-filter-beta-canary"), filter_b.clone()).is_ok());
assert!(first.insert_transaction_filter(filter_name("identity-filter-alpha-canary"), filter_a.clone()).is_ok());
assert!(second.insert_transaction_filter(filter_name("identity-filter-alpha-canary"), filter_a.clone()).is_ok());
assert!(second.insert_transaction_filter(filter_name("identity-filter-beta-canary"), filter_b.clone()).is_ok());
filter_a.set_failed(std::option::Option::Some(true));
assert!(changed.insert_transaction_filter(filter_name("identity-filter-alpha-canary"), filter_a).is_ok());
assert!(changed.insert_transaction_filter(filter_name("identity-filter-beta-canary"), filter_b).is_ok());
first.set_commitment(std::option::Option::Some(crate::SolanaCommitment::Confirmed));
second.set_commitment(std::option::Option::Some(crate::SolanaCommitment::Confirmed));
changed.set_commitment(std::option::Option::Some(crate::SolanaCommitment::Confirmed));
let first_identity = first.identity().expect("first request identity must build");
let second_identity = second.identity().expect("second request identity must build");
let changed_identity = changed.identity().expect("changed request identity must build");
assert_eq!(first_identity, second_identity);
assert_ne!(first_identity, changed_identity);
let debug = std::format!("{first_identity:?}");
assert!(debug.contains("byte_len"));
assert!(!debug.contains("identity-filter-alpha-canary"));
assert!(!debug.contains("identity-filter-beta-canary"));
return;
}
#[test]
fn yellowstone_subscribe_debug_omits_filter_names_and_future_payloads() {
let mut request = crate::YellowstoneSubscribeRequest::new();

View File

@@ -1,5 +1,5 @@
// file: crates/ksp-worker-raw-transaction-ingest-lib/src/lib.rs
// version: 17
// version: 18
#![warn(missing_docs)]
#![deny(unreachable_pub)]
@@ -10,8 +10,9 @@
//! This tranche owns the concrete Worker family identity, validated technical settings
//! and the caller-runtime-owned lifecycle with private child-task supervision. This tranche also
//! owns bounded source-neutral admission, common RAW canonicalization/assembly and backend-neutral
//! Store persistence in normal mode plus concrete latest-value snapshots projected onto Worker API;
//! the validated Yellowstone/HTTP runtime-resource contract now drives one productive supervised source task.
//! Store persistence in normal mode plus concrete latest-value snapshots projected onto Worker API. `pre.002` adds a bounded 1..32 caller-composed live-source
//! aggregate with deterministic logical source identities while deliberately keeping productive multi-source activation gated until the supervisor tranche; the
//! validated Yellowstone/HTTP runtime-resource contract still drives the only productive supervised source task.
//! Transaction/TransactionStatus/Block updates feed bounded HTTP `getTransaction` hydration and the existing central
//! admission path; BlockMeta/Slot remain continuity-only signals. Hydration is coalesced by network/signature/commitment
//! under bounded in-flight and pending budgets. A bounded run-local processing frontier projects hydration pending,
@@ -43,6 +44,8 @@ pub use self::error::ERROR_CODE_RAW_TRANSACTION_INGEST_SOURCE_FAILED;
pub use self::error::ERROR_CODE_RAW_TRANSACTION_INGEST_STORE_FAILED;
/// Stable Worker kind code used by the continuous RAW transaction ingest vertical.
pub use self::identity::RAW_TRANSACTION_INGEST_WORKER_KIND_CODE;
/// Maximum number of logical live sources accepted by one runtime-resource aggregate.
pub use self::runtime_resources::MAX_RAW_TRANSACTION_INGEST_LIVE_SOURCES;
/// Cloneable external control handle for one continuous RAW transaction ingest Worker.
pub use self::runtime::RawTransactionIngestHandle;
/// Runtime-neutral boxed future resolving after one RAW transaction ingest Worker has fully reached a terminal lifecycle state.

View File

@@ -1,5 +1,5 @@
// file: crates/ksp-worker-raw-transaction-ingest-lib/src/runtime.rs
// version: 11
// version: 12
type PersistencePort = std::sync::Arc<dyn crate::RawTransactionIngestPersistencePort + 'static>;
type PersistenceTasks = tokio::task::JoinSet<ksp_core_lib::Result<crate::RawTransactionIngestPersistenceOutcome>>;
@@ -97,7 +97,10 @@ impl crate::RawTransactionIngestWorker {
return start_foundation(settings, runtime, std::option::Option::Some(store));
}
/// Starts one Worker with caller-composed runtime resources and one supervised productive Yellowstone source task.
/// Starts one Worker with caller-composed runtime resources.
///
/// `pre.002` accepts a validated multi-source aggregate but intentionally activates only the existing single-Yellowstone supervisor path; aggregates with
/// more than one source fail closed before any source task is spawned.
pub fn start_with_runtime_resources(
settings: crate::RawTransactionIngestSettings,
store: std::sync::Arc<ksp_store_lib::Store>,
@@ -114,7 +117,10 @@ impl crate::RawTransactionIngestWorker {
if let std::result::Result::Err(error) = runtime_resources.validate_network(settings.network()) {
return std::result::Result::Err(error);
}
let source = runtime_resources.into_yellowstone_source();
let source = match runtime_resources.into_yellowstone_source() {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let source_settings = settings.clone();
let (processing_frontier_sender, processing_frontier_receiver) =
tokio::sync::watch::channel(crate::RawTransactionIngestProcessingFrontierProjection::empty());

View File

@@ -1,8 +1,12 @@
// file: crates/ksp-worker-raw-transaction-ingest-lib/src/runtime_resources.rs
// version: 12
// version: 13
use sha2::Digest; // rust-rules: trait-import
/// Maximum number of logical live sources accepted in one RAW transaction ingest runtime-resource aggregate.
pub const MAX_RAW_TRANSACTION_INGEST_LIVE_SOURCES: usize = 32;
const RAW_TRANSACTION_INGEST_LIVE_SOURCE_KEY_DOMAIN: &[u8] = b"ksp.raw_transaction_ingest.live_source.source_key.v1\0";
const RAW_TRANSACTION_INGEST_YELLOWSTONE_FILTER_FINGERPRINT_DOMAIN: &[u8] = b"ksp.raw_transaction_ingest.yellowstone.filters.v1\0";
const RAW_TRANSACTION_INGEST_YELLOWSTONE_HTTP_PROTOCOL: &str = "yellowstone_http";
const RAW_TRANSACTION_INGEST_YELLOWSTONE_HTTP_SOURCE_KEY_DOMAIN: &[u8] = b"ksp.raw_transaction_ingest.yellowstone_http.source_key.v1\0";
@@ -14,6 +18,24 @@ enum RawTransactionIngestSourceFamily {
TransactionStatus,
}
enum RawTransactionIngestLiveSource {
Yellowstone(crate::RawTransactionIngestYellowstoneSource),
}
impl RawTransactionIngestLiveSource {
fn network(&self) -> &ksp_store_lib::RawNetworkId {
return match self {
Self::Yellowstone(source) => &source.network,
};
}
fn source_key(&self) -> [u8; 32] {
return match self {
Self::Yellowstone(source) => source.source_key,
};
}
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
enum RawTransactionIngestContinuityFamily {
BlockMeta,
@@ -310,6 +332,7 @@ pub struct RawTransactionIngestYellowstoneSource {
hydration_role: ksp_onchain_transport_lib::HttpRoleName,
network: ksp_store_lib::RawNetworkId,
route: RawTransactionIngestSourceRoute,
source_key: [u8; 32],
}
impl crate::RawTransactionIngestYellowstoneSource {
@@ -358,7 +381,12 @@ impl crate::RawTransactionIngestYellowstoneSource {
if compatible_http_routes == 0 {
return std::result::Result::Err(crate::runtime_error("runtime_resources.hydration_role_unsupported"));
}
return std::result::Result::Ok(Self { yellowstone_channel, subscribe_request, http_pool, hydration_role, network, route });
let request_identity = match subscribe_request.identity() {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return std::result::Result::Err(crate::runtime_error("runtime_resources.yellowstone_identity_invalid")),
};
let source_key = yellowstone_live_source_key(&network, &route, subscribe_request.commitment(), &request_identity);
return std::result::Result::Ok(Self { yellowstone_channel, subscribe_request, http_pool, hydration_role, network, route, source_key });
}
/// Runs the productive Yellowstone source task until cooperative stop or one safe terminal source failure.
@@ -510,45 +538,138 @@ impl std::fmt::Debug for crate::RawTransactionIngestYellowstoneSource {
.field("has_from_slot", &self.subscribe_request.from_slot().is_some())
.field("hydration_role", &self.hydration_role.as_str())
.field("http_endpoint_count", &http_snapshot.endpoint_count())
.field("source_key_bytes", &self.source_key.len())
.finish();
}
}
/// Caller-composed runtime resources accepted by the continuous RAW transaction ingest Worker.
///
/// This first runtime-resource contract owns exactly one Yellowstone source. It deliberately does not expose a provider enum, source collection, callback,
/// enqueue surface or lower-layer client escape hatch.
/// The aggregate owns a bounded collection of capability-specific live-source contracts. `pre.002` validates composition and stable logical identities but
/// deliberately keeps productive multi-source supervision gated until the dedicated supervisor tranche.
pub struct RawTransactionIngestRuntimeResources {
yellowstone_source: crate::RawTransactionIngestYellowstoneSource,
sources: std::vec::Vec<RawTransactionIngestLiveSource>,
}
impl crate::RawTransactionIngestRuntimeResources {
/// Owns the validated productive Yellowstone source contract consumed by `start_with_runtime_resources`.
/// Creates one runtime-resource aggregate from the already productive Yellowstone source contract.
#[must_use]
pub fn new(yellowstone_source: crate::RawTransactionIngestYellowstoneSource) -> Self {
return Self { yellowstone_source };
return Self { sources: std::vec![RawTransactionIngestLiveSource::Yellowstone(yellowstone_source)] };
}
/// Validates that caller-owned Worker settings target the same logical network as the composed Yellowstone/HTTP source.
/// Returns the number of validated logical live sources currently owned by this aggregate.
#[must_use]
pub fn source_count(&self) -> usize {
return self.sources.len();
}
/// Adds one validated Yellowstone source while preserving the global 1..32 bound, one-network invariant and unique logical source identity.
pub fn try_push_yellowstone_source(&mut self, yellowstone_source: crate::RawTransactionIngestYellowstoneSource) -> ksp_core_lib::Result<()> {
if self.sources.len() >= crate::MAX_RAW_TRANSACTION_INGEST_LIVE_SOURCES {
return std::result::Result::Err(crate::runtime_error("runtime_resources.source_count_exceeded"));
}
let candidate = RawTransactionIngestLiveSource::Yellowstone(yellowstone_source);
let expected_network = match self.sources.first() {
std::option::Option::Some(source) => source.network(),
std::option::Option::None => return std::result::Result::Err(crate::runtime_error("runtime_resources.source_collection_empty")),
};
if candidate.network() != expected_network {
return std::result::Result::Err(crate::runtime_error("runtime_resources.source_network_mismatch"));
}
let source_key = candidate.source_key();
if self.sources.iter().any(|source| return source.source_key() == source_key) {
return std::result::Result::Err(crate::runtime_error("runtime_resources.duplicate_source_identity"));
}
self.sources.push(candidate);
return std::result::Result::Ok(());
}
/// Validates that caller-owned Worker settings target the same logical network as every composed live source.
pub(crate) fn validate_network(&self, network: &ksp_store_lib::RawNetworkId) -> ksp_core_lib::Result<()> {
if &self.yellowstone_source.network != network {
return std::result::Result::Err(crate::runtime_error("runtime_resources.worker_network_mismatch"));
if self.sources.is_empty() || self.sources.len() > crate::MAX_RAW_TRANSACTION_INGEST_LIVE_SOURCES {
return std::result::Result::Err(crate::runtime_error("runtime_resources.source_collection_invalid"));
}
let mut source_keys = std::collections::BTreeSet::new();
for source in &self.sources {
if source.network() != network {
return std::result::Result::Err(crate::runtime_error("runtime_resources.worker_network_mismatch"));
}
if !source_keys.insert(source.source_key()) {
return std::result::Result::Err(crate::runtime_error("runtime_resources.duplicate_source_identity"));
}
}
return std::result::Result::Ok(());
}
/// Consumes the runtime-resource aggregate into its single validated Yellowstone source for private supervisor wiring.
pub(crate) fn into_yellowstone_source(self) -> crate::RawTransactionIngestYellowstoneSource {
return self.yellowstone_source;
/// Consumes the aggregate into the single Yellowstone source supported by the current productive supervisor wiring.
///
/// Multi-source activation is intentionally deferred to the dedicated supervisor/source-inventory tranche; a collection larger than one therefore fails
/// closed rather than silently dropping sources or starting only a subset.
pub(crate) fn into_yellowstone_source(self) -> ksp_core_lib::Result<crate::RawTransactionIngestYellowstoneSource> {
if self.sources.is_empty() {
return std::result::Result::Err(crate::runtime_error("runtime_resources.source_collection_empty"));
}
if self.sources.len() != 1 {
return std::result::Result::Err(crate::runtime_error("runtime_resources.multi_source_activation_pending"));
}
let mut sources = self.sources.into_iter();
return match sources.next() {
std::option::Option::Some(RawTransactionIngestLiveSource::Yellowstone(source)) => std::result::Result::Ok(source),
std::option::Option::None => std::result::Result::Err(crate::runtime_error("runtime_resources.source_collection_empty")),
};
}
}
impl std::fmt::Debug for crate::RawTransactionIngestRuntimeResources {
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
return formatter.debug_struct("RawTransactionIngestRuntimeResources").field("yellowstone_source", &self.yellowstone_source).finish();
return formatter.debug_struct("RawTransactionIngestRuntimeResources").field("source_count", &self.sources.len()).finish();
}
}
struct RawTransactionIngestSourceKeyHashWriter<'a> {
hasher: &'a mut sha2::Sha256,
}
impl std::hash::Hasher for RawTransactionIngestSourceKeyHashWriter<'_> {
fn finish(&self) -> u64 {
return 0;
}
fn write(&mut self, bytes: &[u8]) {
self.hasher.update(bytes);
return;
}
}
fn yellowstone_live_source_key(
network: &ksp_store_lib::RawNetworkId,
route: &RawTransactionIngestSourceRoute,
commitment: std::option::Option<ksp_onchain_transport_lib::SolanaCommitment>,
request_identity: &ksp_onchain_transport_lib::YellowstoneSubscribeRequestIdentity,
) -> [u8; 32] {
let mut hasher = sha2::Sha256::new();
hasher.update(RAW_TRANSACTION_INGEST_LIVE_SOURCE_KEY_DOMAIN);
hash_live_source_key_component(&mut hasher, b"yellowstone");
hash_live_source_key_component(&mut hasher, network.as_str().as_bytes());
hash_live_source_key_component(&mut hasher, route.provider.as_str().as_bytes());
hash_live_source_key_component(&mut hasher, route.endpoint_id.as_str().as_bytes());
let commitment = match commitment {
std::option::Option::Some(value) => value.as_str(),
std::option::Option::None => "none",
};
hash_live_source_key_component(&mut hasher, commitment.as_bytes());
let mut writer = RawTransactionIngestSourceKeyHashWriter { hasher: &mut hasher };
std::hash::Hash::hash(request_identity, &mut writer);
return hasher.finalize().into();
}
fn hash_live_source_key_component(hasher: &mut sha2::Sha256, value: &[u8]) {
hasher.update((value.len() as u64).to_be_bytes());
hasher.update(value);
return;
}
fn compatible_http_route_count(
pool: &ksp_onchain_transport_lib::HttpTransportPool,
hydration_role: &ksp_onchain_transport_lib::HttpRoleName,

View File

@@ -1,5 +1,5 @@
// file: crates/ksp-worker-raw-transaction-ingest-lib/tests/public_api.rs
// version: 11
// version: 12
//! External public-surface proofs for the RAW transaction ingest Worker foundation.
@@ -182,6 +182,21 @@ fn v0_3_12_pre_002_runtime_resource_types_are_consumable_without_client_escape_h
return;
}
#[test]
fn v0_3_13_pre_002_multi_source_runtime_resource_surface_is_bounded_and_source_neutral() {
let _source_count: fn(&ksp_worker_raw_transaction_ingest_lib::RawTransactionIngestRuntimeResources) -> usize =
ksp_worker_raw_transaction_ingest_lib::RawTransactionIngestRuntimeResources::source_count;
let _push_yellowstone: fn(
&mut ksp_worker_raw_transaction_ingest_lib::RawTransactionIngestRuntimeResources,
ksp_worker_raw_transaction_ingest_lib::RawTransactionIngestYellowstoneSource,
) -> ksp_core_lib::Result<()> = ksp_worker_raw_transaction_ingest_lib::RawTransactionIngestRuntimeResources::try_push_yellowstone_source;
assert_eq!(ksp_worker_raw_transaction_ingest_lib::MAX_RAW_TRANSACTION_INGEST_LIVE_SOURCES, 32);
let root = include_str!("../src/lib.rs");
assert!(!root.contains("RawTransactionIngestLiveSource"));
assert!(!root.contains("source_key"));
return;
}
#[test]
fn v0_3_12_pre_007_processing_frontier_snapshot_getters_are_public_and_processing_only() {
let _hydration_pending: fn(&ksp_worker_raw_transaction_ingest_lib::RawTransactionIngestSnapshot) -> usize =

View File

@@ -1,5 +1,5 @@
// file: crates/ksp-worker-raw-transaction-ingest-lib/tests/release_completeness.rs
// version: 9
// version: 10
//! Release-completeness canaries through the `pre.010` public/release/security hardening tranche.
@@ -70,6 +70,7 @@ fn pre_010_public_root_export_inventory_is_exact() {
"ERROR_CODE_RAW_TRANSACTION_INGEST_SOURCE_FAILED",
"ERROR_CODE_RAW_TRANSACTION_INGEST_STORE_FAILED",
"MAX_RAW_TRANSACTION_INGEST_ADMISSION_QUEUE_CAPACITY",
"MAX_RAW_TRANSACTION_INGEST_LIVE_SOURCES",
"MAX_RAW_TRANSACTION_INGEST_PERSISTENCE_CONCURRENCY",
"MAX_RAW_TRANSACTION_INGEST_SHUTDOWN_DRAIN_TIMEOUT",
"MIN_RAW_TRANSACTION_INGEST_ADMISSION_QUEUE_CAPACITY",
@@ -131,6 +132,7 @@ fn pre_010_external_hardening_suite_is_present_and_scoped() {
assert!(public_api.contains("pre_008_snapshot_surface_and_common_projection_are_public_and_stable"));
assert!(public_api.contains("pre_009_source_and_drain_timeout_error_codes_are_public_and_stable"));
assert!(public_api.contains("v0_3_12_pre_002_runtime_resource_types_are_consumable_without_client_escape_hatch"));
assert!(public_api.contains("v0_3_13_pre_002_multi_source_runtime_resource_surface_is_bounded_and_source_neutral"));
assert!(public_api.contains("v0_3_12_pre_007_processing_frontier_snapshot_getters_are_public_and_processing_only"));
assert!(public_api.contains("pre_008_snapshot_surface_and_common_projection_are_public_and_stable"));
return;

View File

@@ -1,5 +1,5 @@
// file: crates/ksp-worker-raw-transaction-ingest-lib/unit_tests/runtime_resources.rs
// version: 11
// version: 12
fn grpc_endpoint(cluster: &str) -> std::option::Option<ksp_onchain_transport_lib::YellowstoneGrpcEndpointSettings> {
return grpc_endpoint_with_identity(cluster, "yellowstone-fixture", "fixture-provider");
@@ -191,6 +191,51 @@ fn filter_names(values: &[&str]) -> std::option::Option<std::vec::Vec<ksp_onchai
return std::option::Option::Some(filters);
}
fn source_with_identity(
cluster: &str,
endpoint_name: &str,
filter_name: &str,
failed: std::option::Option<bool>,
) -> std::option::Option<crate::RawTransactionIngestYellowstoneSource> {
return source_with_identity_and_hydration_role(cluster, endpoint_name, filter_name, failed, "hydration");
}
fn source_with_identity_and_hydration_role(
cluster: &str,
endpoint_name: &str,
filter_name: &str,
failed: std::option::Option<bool>,
hydration_role: &str,
) -> std::option::Option<crate::RawTransactionIngestYellowstoneSource> {
let endpoint = match grpc_endpoint_with_identity(cluster, endpoint_name, "fixture-provider") {
std::option::Option::Some(value) => value,
std::option::Option::None => return std::option::Option::None,
};
let channel = match ksp_onchain_transport_lib::YellowstoneGrpcChannel::prepare(&endpoint) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return std::option::Option::None,
};
let name = match ksp_onchain_transport_lib::YellowstoneSubscribeFilterName::new(filter_name) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return std::option::Option::None,
};
let mut filter = ksp_onchain_transport_lib::YellowstoneSubscribeTransactionFilter::new();
filter.set_failed(failed);
let mut request = ksp_onchain_transport_lib::YellowstoneSubscribeRequest::new();
if request.insert_transaction_filter(name, filter).is_err() {
return std::option::Option::None;
}
request.set_commitment(std::option::Option::Some(ksp_onchain_transport_lib::SolanaCommitment::Confirmed));
let pool = match http_pool(cluster, hydration_role, "get_transaction") {
std::option::Option::Some(value) => value,
std::option::Option::None => return std::option::Option::None,
};
return match crate::RawTransactionIngestYellowstoneSource::new(channel, request, pool, ksp_onchain_transport_lib::HttpRoleName::new(hydration_role)) {
std::result::Result::Ok(value) => std::option::Option::Some(value),
std::result::Result::Err(_) => std::option::Option::None,
};
}
fn signal_source() -> std::option::Option<crate::RawTransactionIngestYellowstoneSource> {
let endpoint = match grpc_endpoint("devnet") {
std::option::Option::Some(value) => value,
@@ -544,9 +589,11 @@ async fn pre_002_runtime_resource_debug_is_safe_and_exposes_no_client_inner() {
let resources = crate::RawTransactionIngestRuntimeResources::new(source);
let debug = std::format!("{resources:?}");
assert!(debug.contains("RawTransactionIngestRuntimeResources"));
assert!(debug.contains("devnet"));
assert!(debug.contains("blocks_meta_filter_count"));
assert!(debug.contains("slot_filter_count"));
assert!(debug.contains("source_count"));
assert!(debug.contains("1"));
assert!(!debug.contains("devnet"));
assert!(!debug.contains("blocks_meta_filter_count"));
assert!(!debug.contains("slot_filter_count"));
assert!(!debug.contains("GRPC-SECRET-CANARY"));
assert!(!debug.contains("HTTP-SECRET-CANARY"));
assert!(!debug.contains("127.0.0.1"));
@@ -591,6 +638,116 @@ async fn pre_002_runtime_resources_reject_worker_network_mismatch_without_starti
return;
}
#[tokio::test(flavor = "current_thread")]
async fn v0_3_13_pre_002_live_source_key_is_stable_and_request_sensitive() {
let first = match source_with_identity("devnet", "source-a", "tx-a", std::option::Option::Some(false)) {
std::option::Option::Some(value) => value,
std::option::Option::None => return,
};
let same = match source_with_identity("devnet", "source-a", "tx-a", std::option::Option::Some(false)) {
std::option::Option::Some(value) => value,
std::option::Option::None => return,
};
let changed_filter = match source_with_identity("devnet", "source-a", "tx-a", std::option::Option::Some(true)) {
std::option::Option::Some(value) => value,
std::option::Option::None => return,
};
let changed_endpoint = match source_with_identity("devnet", "source-b", "tx-a", std::option::Option::Some(false)) {
std::option::Option::Some(value) => value,
std::option::Option::None => return,
};
let changed_hydration_role = match source_with_identity_and_hydration_role("devnet", "source-a", "tx-a", std::option::Option::Some(false), "secondary") {
std::option::Option::Some(value) => value,
std::option::Option::None => return,
};
assert_eq!(first.source_key, same.source_key);
assert_eq!(first.source_key, changed_hydration_role.source_key);
assert_ne!(first.source_key, changed_filter.source_key);
assert_ne!(first.source_key, changed_endpoint.source_key);
return;
}
#[tokio::test(flavor = "current_thread")]
async fn v0_3_13_pre_002_runtime_resources_accept_exact_bound_and_reject_excess_transactionally() {
let first = match source_with_identity("devnet", "source-00", "tx", std::option::Option::None) {
std::option::Option::Some(value) => value,
std::option::Option::None => return,
};
let mut resources = crate::RawTransactionIngestRuntimeResources::new(first);
for index in 1..crate::MAX_RAW_TRANSACTION_INGEST_LIVE_SOURCES {
let endpoint_name = std::format!("source-{index:02}");
let source = match source_with_identity("devnet", endpoint_name.as_str(), "tx", std::option::Option::None) {
std::option::Option::Some(value) => value,
std::option::Option::None => return,
};
assert!(resources.try_push_yellowstone_source(source).is_ok());
}
assert_eq!(resources.source_count(), crate::MAX_RAW_TRANSACTION_INGEST_LIVE_SOURCES);
let excess = match source_with_identity("devnet", "source-excess", "tx", std::option::Option::None) {
std::option::Option::Some(value) => value,
std::option::Option::None => return,
};
let error = match resources.try_push_yellowstone_source(excess) {
std::result::Result::Ok(()) => return,
std::result::Result::Err(error) => error,
};
assert_eq!(error.code(), crate::ERROR_CODE_RAW_TRANSACTION_INGEST_RUNTIME_INVALID);
assert!(error.context().iter().any(|context| return context.value() == "runtime_resources.source_count_exceeded"));
assert_eq!(resources.source_count(), crate::MAX_RAW_TRANSACTION_INGEST_LIVE_SOURCES);
return;
}
#[tokio::test(flavor = "current_thread")]
async fn v0_3_13_pre_002_runtime_resources_reject_duplicate_and_cross_network_source_before_activation() {
let first = match source_with_identity("devnet", "source-a", "tx", std::option::Option::None) {
std::option::Option::Some(value) => value,
std::option::Option::None => return,
};
let duplicate = match source_with_identity("devnet", "source-a", "tx", std::option::Option::None) {
std::option::Option::Some(value) => value,
std::option::Option::None => return,
};
let other_network = match source_with_identity("mainnet", "source-b", "tx", std::option::Option::None) {
std::option::Option::Some(value) => value,
std::option::Option::None => return,
};
let mut resources = crate::RawTransactionIngestRuntimeResources::new(first);
let duplicate_error = match resources.try_push_yellowstone_source(duplicate) {
std::result::Result::Ok(()) => return,
std::result::Result::Err(error) => error,
};
assert!(duplicate_error.context().iter().any(|context| return context.value() == "runtime_resources.duplicate_source_identity"));
assert_eq!(resources.source_count(), 1);
let network_error = match resources.try_push_yellowstone_source(other_network) {
std::result::Result::Ok(()) => return,
std::result::Result::Err(error) => error,
};
assert!(network_error.context().iter().any(|context| return context.value() == "runtime_resources.source_network_mismatch"));
assert_eq!(resources.source_count(), 1);
return;
}
#[tokio::test(flavor = "current_thread")]
async fn v0_3_13_pre_002_multi_source_activation_fails_closed_until_supervisor_tranche() {
let first = match source_with_identity("devnet", "source-a", "tx", std::option::Option::None) {
std::option::Option::Some(value) => value,
std::option::Option::None => return,
};
let second = match source_with_identity("devnet", "source-b", "tx", std::option::Option::None) {
std::option::Option::Some(value) => value,
std::option::Option::None => return,
};
let mut resources = crate::RawTransactionIngestRuntimeResources::new(first);
assert!(resources.try_push_yellowstone_source(second).is_ok());
let error = match resources.into_yellowstone_source() {
std::result::Result::Ok(_) => return,
std::result::Result::Err(error) => error,
};
assert_eq!(error.code(), crate::ERROR_CODE_RAW_TRANSACTION_INGEST_RUNTIME_INVALID);
assert!(error.context().iter().any(|context| return context.value() == "runtime_resources.multi_source_activation_pending"));
return;
}
#[tokio::test(flavor = "current_thread")]
async fn pre_003_transaction_signal_fixture_preserves_exact_source_neutral_identity() {
let source = match signal_source() {