v0.3.13-pre.002
This commit is contained in:
@@ -1,5 +1,5 @@
|
||||
// file: crates/ksp-onchain-transport-lib/src/grpc_subscribe.rs
|
||||
// version: 8
|
||||
// version: 9
|
||||
|
||||
const MAX_GRPC_BLOCKHASH_TEXT_LENGTH_BYTES: usize = 128;
|
||||
const MAX_GRPC_BLOCK_VECTOR_COUNT: usize = 65_536;
|
||||
@@ -2529,13 +2529,34 @@ impl YellowstoneSubscribeEntryFilter {
|
||||
}
|
||||
}
|
||||
|
||||
/// Opaque deterministic identity material for one complete Yellowstone Subscribe request.
|
||||
///
|
||||
/// The encoded bytes remain private. `Hash` feeds those canonical bytes to a caller-provided hasher while `Debug` exposes only their length.
|
||||
#[derive(Clone, Eq, PartialEq)]
|
||||
pub struct YellowstoneSubscribeRequestIdentity {
|
||||
bytes: std::vec::Vec<u8>,
|
||||
}
|
||||
|
||||
impl std::hash::Hash for crate::YellowstoneSubscribeRequestIdentity {
|
||||
fn hash<H: std::hash::Hasher>(&self, state: &mut H) {
|
||||
state.write(&(self.bytes.len() as u64).to_be_bytes());
|
||||
state.write(self.bytes.as_slice());
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
impl std::fmt::Debug for crate::YellowstoneSubscribeRequestIdentity {
|
||||
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
return formatter.debug_struct("YellowstoneSubscribeRequestIdentity").field("byte_len", &self.bytes.len()).finish();
|
||||
}
|
||||
}
|
||||
|
||||
/// Provider-neutral standard Yellowstone subscribe request owned by KSP.
|
||||
///
|
||||
/// The seven upstream maps are represented independently and retain named empty entries. An entirely empty map is the logical KSP representation of no active
|
||||
/// filter in that family; protobuf map encoding does not distinguish an omitted map from an empty map. Filter-group names are globally unique across all seven
|
||||
/// maps so the names echoed by `SubscribeUpdate.filters` remain unambiguous. `Debug` exposes only counts and common scalar options, never filter names or
|
||||
/// future
|
||||
/// filter payloads.
|
||||
/// future filter payloads.
|
||||
#[derive(Clone, Eq, PartialEq)]
|
||||
pub struct YellowstoneSubscribeRequest {
|
||||
accounts: std::collections::BTreeMap<crate::YellowstoneSubscribeFilterName, crate::YellowstoneSubscribeAccountFilter>,
|
||||
@@ -2551,7 +2572,7 @@ pub struct YellowstoneSubscribeRequest {
|
||||
from_slot: std::option::Option<u64>,
|
||||
}
|
||||
|
||||
impl YellowstoneSubscribeRequest {
|
||||
impl crate::YellowstoneSubscribeRequest {
|
||||
/// Creates an empty subscribe request. Empty requests are valid because later bidi lifecycle code uses request mutations to clear filters or carry ping
|
||||
/// state.
|
||||
#[must_use]
|
||||
@@ -2762,6 +2783,40 @@ impl YellowstoneSubscribeRequest {
|
||||
return self.from_slot;
|
||||
}
|
||||
|
||||
/// Builds one opaque deterministic identity for the complete logical Subscribe request.
|
||||
///
|
||||
/// The identity preserves filter-family separation, globally sorted filter names, exact filter wire payloads and common request options. Its Debug surface
|
||||
/// exposes only the encoded byte length. Callers may hash the opaque value but cannot recover the underlying identity bytes through this API.
|
||||
pub fn identity(&self) -> ksp_core_lib::Result<crate::YellowstoneSubscribeRequestIdentity> {
|
||||
let validation = self.validate();
|
||||
if let std::result::Result::Err(error) = validation {
|
||||
return std::result::Result::Err(error);
|
||||
}
|
||||
let mut bytes = std::vec::Vec::new();
|
||||
append_subscribe_identity_map(&mut bytes, b"accounts", &self.accounts, |filter| return filter.to_wire());
|
||||
append_subscribe_identity_map(&mut bytes, b"blocks", &self.blocks, |filter| return filter.to_wire());
|
||||
append_subscribe_identity_map(&mut bytes, b"blocks_meta", &self.blocks_meta, |filter| return filter.to_wire());
|
||||
append_subscribe_identity_map(&mut bytes, b"entry", &self.entry, |filter| return filter.to_wire());
|
||||
append_subscribe_identity_map(&mut bytes, b"slots", &self.slots, |filter| return filter.to_wire());
|
||||
append_subscribe_identity_map(&mut bytes, b"transactions", &self.transactions, |filter| return filter.to_wire());
|
||||
append_subscribe_identity_map(&mut bytes, b"transactions_status", &self.transactions_status, |filter| return filter.to_wire());
|
||||
let mut common = match self.to_wire() {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(error) => return std::result::Result::Err(error),
|
||||
};
|
||||
common.accounts.clear();
|
||||
common.blocks.clear();
|
||||
common.blocks_meta.clear();
|
||||
common.entry.clear();
|
||||
common.slots.clear();
|
||||
common.transactions.clear();
|
||||
common.transactions_status.clear();
|
||||
let common = yellowstone_grpc_proto::prost::Message::encode_to_vec(&common);
|
||||
append_subscribe_identity_component(&mut bytes, b"common");
|
||||
append_subscribe_identity_component(&mut bytes, common.as_slice());
|
||||
return std::result::Result::Ok(crate::YellowstoneSubscribeRequestIdentity { bytes });
|
||||
}
|
||||
|
||||
/// Validates all deterministic common subscribe-request bounds before any network I/O.
|
||||
pub fn validate(&self) -> ksp_core_lib::Result<()> {
|
||||
if self.total_filter_count() > MAX_GRPC_SUBSCRIBE_FILTER_GROUP_COUNT {
|
||||
@@ -3710,13 +3765,38 @@ fn commitment_to_wire(commitment: std::option::Option<crate::SolanaCommitment>)
|
||||
});
|
||||
}
|
||||
|
||||
impl std::default::Default for YellowstoneSubscribeRequest {
|
||||
fn append_subscribe_identity_component(output: &mut std::vec::Vec<u8>, value: &[u8]) {
|
||||
output.extend_from_slice(&(value.len() as u64).to_be_bytes());
|
||||
output.extend_from_slice(value);
|
||||
return;
|
||||
}
|
||||
|
||||
fn append_subscribe_identity_map<V, W, F>(
|
||||
output: &mut std::vec::Vec<u8>,
|
||||
family: &[u8],
|
||||
values: &std::collections::BTreeMap<crate::YellowstoneSubscribeFilterName, V>,
|
||||
mut to_wire: F,
|
||||
) where
|
||||
W: yellowstone_grpc_proto::prost::Message,
|
||||
F: FnMut(&V) -> W,
|
||||
{
|
||||
append_subscribe_identity_component(output, family);
|
||||
output.extend_from_slice(&(values.len() as u64).to_be_bytes());
|
||||
for (name, filter) in values {
|
||||
append_subscribe_identity_component(output, name.as_str().as_bytes());
|
||||
let wire = yellowstone_grpc_proto::prost::Message::encode_to_vec(&to_wire(filter));
|
||||
append_subscribe_identity_component(output, wire.as_slice());
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
impl std::default::Default for crate::YellowstoneSubscribeRequest {
|
||||
fn default() -> Self {
|
||||
return Self::new();
|
||||
}
|
||||
}
|
||||
|
||||
impl std::fmt::Debug for YellowstoneSubscribeRequest {
|
||||
impl std::fmt::Debug for crate::YellowstoneSubscribeRequest {
|
||||
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
return formatter
|
||||
.debug_struct("YellowstoneSubscribeRequest")
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
// file: crates/ksp-onchain-transport-lib/src/lib.rs
|
||||
// version: 46
|
||||
// version: 47
|
||||
|
||||
#![warn(missing_docs)]
|
||||
#![deny(unreachable_pub)]
|
||||
@@ -215,6 +215,8 @@ pub use self::grpc_subscribe::YellowstoneSubscribePingUpdate;
|
||||
pub use self::grpc_subscribe::YellowstoneSubscribePongUpdate;
|
||||
/// Provider-neutral standard Yellowstone Subscribe request.
|
||||
pub use self::grpc_subscribe::YellowstoneSubscribeRequest;
|
||||
/// Opaque deterministic identity for one complete Yellowstone Subscribe request.
|
||||
pub use self::grpc_subscribe::YellowstoneSubscribeRequestIdentity;
|
||||
/// Complete slot-family filter group for standard Yellowstone Subscribe.
|
||||
pub use self::grpc_subscribe::YellowstoneSubscribeSlotFilter;
|
||||
/// Complete transaction-family filter shared by transactions and transaction-status maps.
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
// file: crates/ksp-onchain-transport-lib/tests/public_api.rs
|
||||
// version: 51
|
||||
// version: 52
|
||||
|
||||
//! Integration tests for the public `ksp-onchain-transport-lib` consumer contract.
|
||||
|
||||
@@ -1032,3 +1032,15 @@ fn public_v0_3_10_pre_004_observed_get_block_surface_is_available_from_crate_roo
|
||||
let _ = method;
|
||||
return;
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn public_v0_3_13_pre_002_yellowstone_subscribe_identity_is_opaque_and_available_from_crate_root() {
|
||||
let request = ksp_onchain_transport_lib::YellowstoneSubscribeRequest::new();
|
||||
let identity = request.identity().expect("empty validated request identity must build");
|
||||
let _identity_type = std::any::type_name::<ksp_onchain_transport_lib::YellowstoneSubscribeRequestIdentity>();
|
||||
let debug = std::format!("{identity:?}");
|
||||
assert!(debug.contains("YellowstoneSubscribeRequestIdentity"));
|
||||
assert!(debug.contains("byte_len"));
|
||||
assert!(!debug.contains("bytes:"));
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
// file: crates/ksp-onchain-transport-lib/tests/release_completeness.rs
|
||||
// version: 43
|
||||
// version: 44
|
||||
|
||||
//! Release-level completeness canaries for staged HTTP and WebSocket Transport coverage.
|
||||
|
||||
@@ -1339,3 +1339,18 @@ fn release_v0_2_9_pre_010_adds_bounded_reconnect_replay_and_conservative_continu
|
||||
let _snapshot = std::any::type_name::<ksp_onchain_transport_lib::YellowstoneGrpcSubscribeSnapshot>();
|
||||
let _state = ksp_onchain_transport_lib::YellowstoneGrpcSubscribeState::Reconnecting;
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn release_v0_3_13_pre_002_yellowstone_subscribe_identity_remains_opaque_and_dependency_neutral() {
|
||||
let source = include_str!("../src/grpc_subscribe.rs");
|
||||
let root = include_str!("../src/lib.rs");
|
||||
let identity_start = source.find("pub struct YellowstoneSubscribeRequestIdentity").expect("identity struct must remain present");
|
||||
let request_start = source.find("/// Provider-neutral standard Yellowstone subscribe request").expect("request contract marker must remain present");
|
||||
let identity_surface = &source[identity_start..request_start];
|
||||
assert!(source.contains("pub fn identity(&self)"));
|
||||
assert!(root.contains("pub use self::grpc_subscribe::YellowstoneSubscribeRequestIdentity;"));
|
||||
assert!(!identity_surface.contains("pub fn bytes("));
|
||||
assert!(!identity_surface.contains("pub fn as_bytes("));
|
||||
assert!(!root.contains("pub use yellowstone_grpc_proto"));
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
// file: crates/ksp-onchain-transport-lib/unit_tests/grpc_subscribe.rs
|
||||
// version: 5
|
||||
// version: 6
|
||||
|
||||
fn filter_name(value: &str) -> crate::YellowstoneSubscribeFilterName {
|
||||
return crate::YellowstoneSubscribeFilterName::new(value).expect("fixture filter name must validate");
|
||||
@@ -138,6 +138,37 @@ fn yellowstone_subscribe_common_bounds_reject_before_wire_conversion() {
|
||||
assert!(filters.insert_account_filter(excess_name, crate::YellowstoneSubscribeAccountFilter::new()).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn yellowstone_subscribe_request_identity_is_order_stable_exact_and_debug_redacted() {
|
||||
let mut first = crate::YellowstoneSubscribeRequest::new();
|
||||
let mut second = crate::YellowstoneSubscribeRequest::new();
|
||||
let mut changed = crate::YellowstoneSubscribeRequest::new();
|
||||
let mut filter_a = crate::YellowstoneSubscribeTransactionFilter::new();
|
||||
filter_a.set_failed(std::option::Option::Some(false));
|
||||
let mut filter_b = crate::YellowstoneSubscribeTransactionFilter::new();
|
||||
filter_b.set_vote(std::option::Option::Some(false));
|
||||
assert!(first.insert_transaction_filter(filter_name("identity-filter-beta-canary"), filter_b.clone()).is_ok());
|
||||
assert!(first.insert_transaction_filter(filter_name("identity-filter-alpha-canary"), filter_a.clone()).is_ok());
|
||||
assert!(second.insert_transaction_filter(filter_name("identity-filter-alpha-canary"), filter_a.clone()).is_ok());
|
||||
assert!(second.insert_transaction_filter(filter_name("identity-filter-beta-canary"), filter_b.clone()).is_ok());
|
||||
filter_a.set_failed(std::option::Option::Some(true));
|
||||
assert!(changed.insert_transaction_filter(filter_name("identity-filter-alpha-canary"), filter_a).is_ok());
|
||||
assert!(changed.insert_transaction_filter(filter_name("identity-filter-beta-canary"), filter_b).is_ok());
|
||||
first.set_commitment(std::option::Option::Some(crate::SolanaCommitment::Confirmed));
|
||||
second.set_commitment(std::option::Option::Some(crate::SolanaCommitment::Confirmed));
|
||||
changed.set_commitment(std::option::Option::Some(crate::SolanaCommitment::Confirmed));
|
||||
let first_identity = first.identity().expect("first request identity must build");
|
||||
let second_identity = second.identity().expect("second request identity must build");
|
||||
let changed_identity = changed.identity().expect("changed request identity must build");
|
||||
assert_eq!(first_identity, second_identity);
|
||||
assert_ne!(first_identity, changed_identity);
|
||||
let debug = std::format!("{first_identity:?}");
|
||||
assert!(debug.contains("byte_len"));
|
||||
assert!(!debug.contains("identity-filter-alpha-canary"));
|
||||
assert!(!debug.contains("identity-filter-beta-canary"));
|
||||
return;
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn yellowstone_subscribe_debug_omits_filter_names_and_future_payloads() {
|
||||
let mut request = crate::YellowstoneSubscribeRequest::new();
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
// file: crates/ksp-worker-raw-transaction-ingest-lib/src/lib.rs
|
||||
// version: 17
|
||||
// version: 18
|
||||
|
||||
#![warn(missing_docs)]
|
||||
#![deny(unreachable_pub)]
|
||||
@@ -10,8 +10,9 @@
|
||||
//! This tranche owns the concrete Worker family identity, validated technical settings
|
||||
//! and the caller-runtime-owned lifecycle with private child-task supervision. This tranche also
|
||||
//! owns bounded source-neutral admission, common RAW canonicalization/assembly and backend-neutral
|
||||
//! Store persistence in normal mode plus concrete latest-value snapshots projected onto Worker API;
|
||||
//! the validated Yellowstone/HTTP runtime-resource contract now drives one productive supervised source task.
|
||||
//! Store persistence in normal mode plus concrete latest-value snapshots projected onto Worker API. `pre.002` adds a bounded 1..32 caller-composed live-source
|
||||
//! aggregate with deterministic logical source identities while deliberately keeping productive multi-source activation gated until the supervisor tranche; the
|
||||
//! validated Yellowstone/HTTP runtime-resource contract still drives the only productive supervised source task.
|
||||
//! Transaction/TransactionStatus/Block updates feed bounded HTTP `getTransaction` hydration and the existing central
|
||||
//! admission path; BlockMeta/Slot remain continuity-only signals. Hydration is coalesced by network/signature/commitment
|
||||
//! under bounded in-flight and pending budgets. A bounded run-local processing frontier projects hydration pending,
|
||||
@@ -43,6 +44,8 @@ pub use self::error::ERROR_CODE_RAW_TRANSACTION_INGEST_SOURCE_FAILED;
|
||||
pub use self::error::ERROR_CODE_RAW_TRANSACTION_INGEST_STORE_FAILED;
|
||||
/// Stable Worker kind code used by the continuous RAW transaction ingest vertical.
|
||||
pub use self::identity::RAW_TRANSACTION_INGEST_WORKER_KIND_CODE;
|
||||
/// Maximum number of logical live sources accepted by one runtime-resource aggregate.
|
||||
pub use self::runtime_resources::MAX_RAW_TRANSACTION_INGEST_LIVE_SOURCES;
|
||||
/// Cloneable external control handle for one continuous RAW transaction ingest Worker.
|
||||
pub use self::runtime::RawTransactionIngestHandle;
|
||||
/// Runtime-neutral boxed future resolving after one RAW transaction ingest Worker has fully reached a terminal lifecycle state.
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
// file: crates/ksp-worker-raw-transaction-ingest-lib/src/runtime.rs
|
||||
// version: 11
|
||||
// version: 12
|
||||
|
||||
type PersistencePort = std::sync::Arc<dyn crate::RawTransactionIngestPersistencePort + 'static>;
|
||||
type PersistenceTasks = tokio::task::JoinSet<ksp_core_lib::Result<crate::RawTransactionIngestPersistenceOutcome>>;
|
||||
@@ -97,7 +97,10 @@ impl crate::RawTransactionIngestWorker {
|
||||
return start_foundation(settings, runtime, std::option::Option::Some(store));
|
||||
}
|
||||
|
||||
/// Starts one Worker with caller-composed runtime resources and one supervised productive Yellowstone source task.
|
||||
/// Starts one Worker with caller-composed runtime resources.
|
||||
///
|
||||
/// `pre.002` accepts a validated multi-source aggregate but intentionally activates only the existing single-Yellowstone supervisor path; aggregates with
|
||||
/// more than one source fail closed before any source task is spawned.
|
||||
pub fn start_with_runtime_resources(
|
||||
settings: crate::RawTransactionIngestSettings,
|
||||
store: std::sync::Arc<ksp_store_lib::Store>,
|
||||
@@ -114,7 +117,10 @@ impl crate::RawTransactionIngestWorker {
|
||||
if let std::result::Result::Err(error) = runtime_resources.validate_network(settings.network()) {
|
||||
return std::result::Result::Err(error);
|
||||
}
|
||||
let source = runtime_resources.into_yellowstone_source();
|
||||
let source = match runtime_resources.into_yellowstone_source() {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(error) => return std::result::Result::Err(error),
|
||||
};
|
||||
let source_settings = settings.clone();
|
||||
let (processing_frontier_sender, processing_frontier_receiver) =
|
||||
tokio::sync::watch::channel(crate::RawTransactionIngestProcessingFrontierProjection::empty());
|
||||
|
||||
@@ -1,8 +1,12 @@
|
||||
// file: crates/ksp-worker-raw-transaction-ingest-lib/src/runtime_resources.rs
|
||||
// version: 12
|
||||
// version: 13
|
||||
|
||||
use sha2::Digest; // rust-rules: trait-import
|
||||
|
||||
/// Maximum number of logical live sources accepted in one RAW transaction ingest runtime-resource aggregate.
|
||||
pub const MAX_RAW_TRANSACTION_INGEST_LIVE_SOURCES: usize = 32;
|
||||
|
||||
const RAW_TRANSACTION_INGEST_LIVE_SOURCE_KEY_DOMAIN: &[u8] = b"ksp.raw_transaction_ingest.live_source.source_key.v1\0";
|
||||
const RAW_TRANSACTION_INGEST_YELLOWSTONE_FILTER_FINGERPRINT_DOMAIN: &[u8] = b"ksp.raw_transaction_ingest.yellowstone.filters.v1\0";
|
||||
const RAW_TRANSACTION_INGEST_YELLOWSTONE_HTTP_PROTOCOL: &str = "yellowstone_http";
|
||||
const RAW_TRANSACTION_INGEST_YELLOWSTONE_HTTP_SOURCE_KEY_DOMAIN: &[u8] = b"ksp.raw_transaction_ingest.yellowstone_http.source_key.v1\0";
|
||||
@@ -14,6 +18,24 @@ enum RawTransactionIngestSourceFamily {
|
||||
TransactionStatus,
|
||||
}
|
||||
|
||||
enum RawTransactionIngestLiveSource {
|
||||
Yellowstone(crate::RawTransactionIngestYellowstoneSource),
|
||||
}
|
||||
|
||||
impl RawTransactionIngestLiveSource {
|
||||
fn network(&self) -> &ksp_store_lib::RawNetworkId {
|
||||
return match self {
|
||||
Self::Yellowstone(source) => &source.network,
|
||||
};
|
||||
}
|
||||
|
||||
fn source_key(&self) -> [u8; 32] {
|
||||
return match self {
|
||||
Self::Yellowstone(source) => source.source_key,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
enum RawTransactionIngestContinuityFamily {
|
||||
BlockMeta,
|
||||
@@ -310,6 +332,7 @@ pub struct RawTransactionIngestYellowstoneSource {
|
||||
hydration_role: ksp_onchain_transport_lib::HttpRoleName,
|
||||
network: ksp_store_lib::RawNetworkId,
|
||||
route: RawTransactionIngestSourceRoute,
|
||||
source_key: [u8; 32],
|
||||
}
|
||||
|
||||
impl crate::RawTransactionIngestYellowstoneSource {
|
||||
@@ -358,7 +381,12 @@ impl crate::RawTransactionIngestYellowstoneSource {
|
||||
if compatible_http_routes == 0 {
|
||||
return std::result::Result::Err(crate::runtime_error("runtime_resources.hydration_role_unsupported"));
|
||||
}
|
||||
return std::result::Result::Ok(Self { yellowstone_channel, subscribe_request, http_pool, hydration_role, network, route });
|
||||
let request_identity = match subscribe_request.identity() {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(_) => return std::result::Result::Err(crate::runtime_error("runtime_resources.yellowstone_identity_invalid")),
|
||||
};
|
||||
let source_key = yellowstone_live_source_key(&network, &route, subscribe_request.commitment(), &request_identity);
|
||||
return std::result::Result::Ok(Self { yellowstone_channel, subscribe_request, http_pool, hydration_role, network, route, source_key });
|
||||
}
|
||||
|
||||
/// Runs the productive Yellowstone source task until cooperative stop or one safe terminal source failure.
|
||||
@@ -510,45 +538,138 @@ impl std::fmt::Debug for crate::RawTransactionIngestYellowstoneSource {
|
||||
.field("has_from_slot", &self.subscribe_request.from_slot().is_some())
|
||||
.field("hydration_role", &self.hydration_role.as_str())
|
||||
.field("http_endpoint_count", &http_snapshot.endpoint_count())
|
||||
.field("source_key_bytes", &self.source_key.len())
|
||||
.finish();
|
||||
}
|
||||
}
|
||||
|
||||
/// Caller-composed runtime resources accepted by the continuous RAW transaction ingest Worker.
|
||||
///
|
||||
/// This first runtime-resource contract owns exactly one Yellowstone source. It deliberately does not expose a provider enum, source collection, callback,
|
||||
/// enqueue surface or lower-layer client escape hatch.
|
||||
/// The aggregate owns a bounded collection of capability-specific live-source contracts. `pre.002` validates composition and stable logical identities but
|
||||
/// deliberately keeps productive multi-source supervision gated until the dedicated supervisor tranche.
|
||||
pub struct RawTransactionIngestRuntimeResources {
|
||||
yellowstone_source: crate::RawTransactionIngestYellowstoneSource,
|
||||
sources: std::vec::Vec<RawTransactionIngestLiveSource>,
|
||||
}
|
||||
|
||||
impl crate::RawTransactionIngestRuntimeResources {
|
||||
/// Owns the validated productive Yellowstone source contract consumed by `start_with_runtime_resources`.
|
||||
/// Creates one runtime-resource aggregate from the already productive Yellowstone source contract.
|
||||
#[must_use]
|
||||
pub fn new(yellowstone_source: crate::RawTransactionIngestYellowstoneSource) -> Self {
|
||||
return Self { yellowstone_source };
|
||||
return Self { sources: std::vec![RawTransactionIngestLiveSource::Yellowstone(yellowstone_source)] };
|
||||
}
|
||||
|
||||
/// Validates that caller-owned Worker settings target the same logical network as the composed Yellowstone/HTTP source.
|
||||
/// Returns the number of validated logical live sources currently owned by this aggregate.
|
||||
#[must_use]
|
||||
pub fn source_count(&self) -> usize {
|
||||
return self.sources.len();
|
||||
}
|
||||
|
||||
/// Adds one validated Yellowstone source while preserving the global 1..32 bound, one-network invariant and unique logical source identity.
|
||||
pub fn try_push_yellowstone_source(&mut self, yellowstone_source: crate::RawTransactionIngestYellowstoneSource) -> ksp_core_lib::Result<()> {
|
||||
if self.sources.len() >= crate::MAX_RAW_TRANSACTION_INGEST_LIVE_SOURCES {
|
||||
return std::result::Result::Err(crate::runtime_error("runtime_resources.source_count_exceeded"));
|
||||
}
|
||||
let candidate = RawTransactionIngestLiveSource::Yellowstone(yellowstone_source);
|
||||
let expected_network = match self.sources.first() {
|
||||
std::option::Option::Some(source) => source.network(),
|
||||
std::option::Option::None => return std::result::Result::Err(crate::runtime_error("runtime_resources.source_collection_empty")),
|
||||
};
|
||||
if candidate.network() != expected_network {
|
||||
return std::result::Result::Err(crate::runtime_error("runtime_resources.source_network_mismatch"));
|
||||
}
|
||||
let source_key = candidate.source_key();
|
||||
if self.sources.iter().any(|source| return source.source_key() == source_key) {
|
||||
return std::result::Result::Err(crate::runtime_error("runtime_resources.duplicate_source_identity"));
|
||||
}
|
||||
self.sources.push(candidate);
|
||||
return std::result::Result::Ok(());
|
||||
}
|
||||
|
||||
/// Validates that caller-owned Worker settings target the same logical network as every composed live source.
|
||||
pub(crate) fn validate_network(&self, network: &ksp_store_lib::RawNetworkId) -> ksp_core_lib::Result<()> {
|
||||
if &self.yellowstone_source.network != network {
|
||||
return std::result::Result::Err(crate::runtime_error("runtime_resources.worker_network_mismatch"));
|
||||
if self.sources.is_empty() || self.sources.len() > crate::MAX_RAW_TRANSACTION_INGEST_LIVE_SOURCES {
|
||||
return std::result::Result::Err(crate::runtime_error("runtime_resources.source_collection_invalid"));
|
||||
}
|
||||
let mut source_keys = std::collections::BTreeSet::new();
|
||||
for source in &self.sources {
|
||||
if source.network() != network {
|
||||
return std::result::Result::Err(crate::runtime_error("runtime_resources.worker_network_mismatch"));
|
||||
}
|
||||
if !source_keys.insert(source.source_key()) {
|
||||
return std::result::Result::Err(crate::runtime_error("runtime_resources.duplicate_source_identity"));
|
||||
}
|
||||
}
|
||||
return std::result::Result::Ok(());
|
||||
}
|
||||
|
||||
/// Consumes the runtime-resource aggregate into its single validated Yellowstone source for private supervisor wiring.
|
||||
pub(crate) fn into_yellowstone_source(self) -> crate::RawTransactionIngestYellowstoneSource {
|
||||
return self.yellowstone_source;
|
||||
/// Consumes the aggregate into the single Yellowstone source supported by the current productive supervisor wiring.
|
||||
///
|
||||
/// Multi-source activation is intentionally deferred to the dedicated supervisor/source-inventory tranche; a collection larger than one therefore fails
|
||||
/// closed rather than silently dropping sources or starting only a subset.
|
||||
pub(crate) fn into_yellowstone_source(self) -> ksp_core_lib::Result<crate::RawTransactionIngestYellowstoneSource> {
|
||||
if self.sources.is_empty() {
|
||||
return std::result::Result::Err(crate::runtime_error("runtime_resources.source_collection_empty"));
|
||||
}
|
||||
if self.sources.len() != 1 {
|
||||
return std::result::Result::Err(crate::runtime_error("runtime_resources.multi_source_activation_pending"));
|
||||
}
|
||||
let mut sources = self.sources.into_iter();
|
||||
return match sources.next() {
|
||||
std::option::Option::Some(RawTransactionIngestLiveSource::Yellowstone(source)) => std::result::Result::Ok(source),
|
||||
std::option::Option::None => std::result::Result::Err(crate::runtime_error("runtime_resources.source_collection_empty")),
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
impl std::fmt::Debug for crate::RawTransactionIngestRuntimeResources {
|
||||
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
return formatter.debug_struct("RawTransactionIngestRuntimeResources").field("yellowstone_source", &self.yellowstone_source).finish();
|
||||
return formatter.debug_struct("RawTransactionIngestRuntimeResources").field("source_count", &self.sources.len()).finish();
|
||||
}
|
||||
}
|
||||
|
||||
struct RawTransactionIngestSourceKeyHashWriter<'a> {
|
||||
hasher: &'a mut sha2::Sha256,
|
||||
}
|
||||
|
||||
impl std::hash::Hasher for RawTransactionIngestSourceKeyHashWriter<'_> {
|
||||
fn finish(&self) -> u64 {
|
||||
return 0;
|
||||
}
|
||||
|
||||
fn write(&mut self, bytes: &[u8]) {
|
||||
self.hasher.update(bytes);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
fn yellowstone_live_source_key(
|
||||
network: &ksp_store_lib::RawNetworkId,
|
||||
route: &RawTransactionIngestSourceRoute,
|
||||
commitment: std::option::Option<ksp_onchain_transport_lib::SolanaCommitment>,
|
||||
request_identity: &ksp_onchain_transport_lib::YellowstoneSubscribeRequestIdentity,
|
||||
) -> [u8; 32] {
|
||||
let mut hasher = sha2::Sha256::new();
|
||||
hasher.update(RAW_TRANSACTION_INGEST_LIVE_SOURCE_KEY_DOMAIN);
|
||||
hash_live_source_key_component(&mut hasher, b"yellowstone");
|
||||
hash_live_source_key_component(&mut hasher, network.as_str().as_bytes());
|
||||
hash_live_source_key_component(&mut hasher, route.provider.as_str().as_bytes());
|
||||
hash_live_source_key_component(&mut hasher, route.endpoint_id.as_str().as_bytes());
|
||||
let commitment = match commitment {
|
||||
std::option::Option::Some(value) => value.as_str(),
|
||||
std::option::Option::None => "none",
|
||||
};
|
||||
hash_live_source_key_component(&mut hasher, commitment.as_bytes());
|
||||
let mut writer = RawTransactionIngestSourceKeyHashWriter { hasher: &mut hasher };
|
||||
std::hash::Hash::hash(request_identity, &mut writer);
|
||||
return hasher.finalize().into();
|
||||
}
|
||||
|
||||
fn hash_live_source_key_component(hasher: &mut sha2::Sha256, value: &[u8]) {
|
||||
hasher.update((value.len() as u64).to_be_bytes());
|
||||
hasher.update(value);
|
||||
return;
|
||||
}
|
||||
|
||||
fn compatible_http_route_count(
|
||||
pool: &ksp_onchain_transport_lib::HttpTransportPool,
|
||||
hydration_role: &ksp_onchain_transport_lib::HttpRoleName,
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
// file: crates/ksp-worker-raw-transaction-ingest-lib/tests/public_api.rs
|
||||
// version: 11
|
||||
// version: 12
|
||||
|
||||
//! External public-surface proofs for the RAW transaction ingest Worker foundation.
|
||||
|
||||
@@ -182,6 +182,21 @@ fn v0_3_12_pre_002_runtime_resource_types_are_consumable_without_client_escape_h
|
||||
return;
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn v0_3_13_pre_002_multi_source_runtime_resource_surface_is_bounded_and_source_neutral() {
|
||||
let _source_count: fn(&ksp_worker_raw_transaction_ingest_lib::RawTransactionIngestRuntimeResources) -> usize =
|
||||
ksp_worker_raw_transaction_ingest_lib::RawTransactionIngestRuntimeResources::source_count;
|
||||
let _push_yellowstone: fn(
|
||||
&mut ksp_worker_raw_transaction_ingest_lib::RawTransactionIngestRuntimeResources,
|
||||
ksp_worker_raw_transaction_ingest_lib::RawTransactionIngestYellowstoneSource,
|
||||
) -> ksp_core_lib::Result<()> = ksp_worker_raw_transaction_ingest_lib::RawTransactionIngestRuntimeResources::try_push_yellowstone_source;
|
||||
assert_eq!(ksp_worker_raw_transaction_ingest_lib::MAX_RAW_TRANSACTION_INGEST_LIVE_SOURCES, 32);
|
||||
let root = include_str!("../src/lib.rs");
|
||||
assert!(!root.contains("RawTransactionIngestLiveSource"));
|
||||
assert!(!root.contains("source_key"));
|
||||
return;
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn v0_3_12_pre_007_processing_frontier_snapshot_getters_are_public_and_processing_only() {
|
||||
let _hydration_pending: fn(&ksp_worker_raw_transaction_ingest_lib::RawTransactionIngestSnapshot) -> usize =
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
// file: crates/ksp-worker-raw-transaction-ingest-lib/tests/release_completeness.rs
|
||||
// version: 9
|
||||
// version: 10
|
||||
|
||||
//! Release-completeness canaries through the `pre.010` public/release/security hardening tranche.
|
||||
|
||||
@@ -70,6 +70,7 @@ fn pre_010_public_root_export_inventory_is_exact() {
|
||||
"ERROR_CODE_RAW_TRANSACTION_INGEST_SOURCE_FAILED",
|
||||
"ERROR_CODE_RAW_TRANSACTION_INGEST_STORE_FAILED",
|
||||
"MAX_RAW_TRANSACTION_INGEST_ADMISSION_QUEUE_CAPACITY",
|
||||
"MAX_RAW_TRANSACTION_INGEST_LIVE_SOURCES",
|
||||
"MAX_RAW_TRANSACTION_INGEST_PERSISTENCE_CONCURRENCY",
|
||||
"MAX_RAW_TRANSACTION_INGEST_SHUTDOWN_DRAIN_TIMEOUT",
|
||||
"MIN_RAW_TRANSACTION_INGEST_ADMISSION_QUEUE_CAPACITY",
|
||||
@@ -131,6 +132,7 @@ fn pre_010_external_hardening_suite_is_present_and_scoped() {
|
||||
assert!(public_api.contains("pre_008_snapshot_surface_and_common_projection_are_public_and_stable"));
|
||||
assert!(public_api.contains("pre_009_source_and_drain_timeout_error_codes_are_public_and_stable"));
|
||||
assert!(public_api.contains("v0_3_12_pre_002_runtime_resource_types_are_consumable_without_client_escape_hatch"));
|
||||
assert!(public_api.contains("v0_3_13_pre_002_multi_source_runtime_resource_surface_is_bounded_and_source_neutral"));
|
||||
assert!(public_api.contains("v0_3_12_pre_007_processing_frontier_snapshot_getters_are_public_and_processing_only"));
|
||||
assert!(public_api.contains("pre_008_snapshot_surface_and_common_projection_are_public_and_stable"));
|
||||
return;
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
// file: crates/ksp-worker-raw-transaction-ingest-lib/unit_tests/runtime_resources.rs
|
||||
// version: 11
|
||||
// version: 12
|
||||
|
||||
fn grpc_endpoint(cluster: &str) -> std::option::Option<ksp_onchain_transport_lib::YellowstoneGrpcEndpointSettings> {
|
||||
return grpc_endpoint_with_identity(cluster, "yellowstone-fixture", "fixture-provider");
|
||||
@@ -191,6 +191,51 @@ fn filter_names(values: &[&str]) -> std::option::Option<std::vec::Vec<ksp_onchai
|
||||
return std::option::Option::Some(filters);
|
||||
}
|
||||
|
||||
fn source_with_identity(
|
||||
cluster: &str,
|
||||
endpoint_name: &str,
|
||||
filter_name: &str,
|
||||
failed: std::option::Option<bool>,
|
||||
) -> std::option::Option<crate::RawTransactionIngestYellowstoneSource> {
|
||||
return source_with_identity_and_hydration_role(cluster, endpoint_name, filter_name, failed, "hydration");
|
||||
}
|
||||
|
||||
fn source_with_identity_and_hydration_role(
|
||||
cluster: &str,
|
||||
endpoint_name: &str,
|
||||
filter_name: &str,
|
||||
failed: std::option::Option<bool>,
|
||||
hydration_role: &str,
|
||||
) -> std::option::Option<crate::RawTransactionIngestYellowstoneSource> {
|
||||
let endpoint = match grpc_endpoint_with_identity(cluster, endpoint_name, "fixture-provider") {
|
||||
std::option::Option::Some(value) => value,
|
||||
std::option::Option::None => return std::option::Option::None,
|
||||
};
|
||||
let channel = match ksp_onchain_transport_lib::YellowstoneGrpcChannel::prepare(&endpoint) {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(_) => return std::option::Option::None,
|
||||
};
|
||||
let name = match ksp_onchain_transport_lib::YellowstoneSubscribeFilterName::new(filter_name) {
|
||||
std::result::Result::Ok(value) => value,
|
||||
std::result::Result::Err(_) => return std::option::Option::None,
|
||||
};
|
||||
let mut filter = ksp_onchain_transport_lib::YellowstoneSubscribeTransactionFilter::new();
|
||||
filter.set_failed(failed);
|
||||
let mut request = ksp_onchain_transport_lib::YellowstoneSubscribeRequest::new();
|
||||
if request.insert_transaction_filter(name, filter).is_err() {
|
||||
return std::option::Option::None;
|
||||
}
|
||||
request.set_commitment(std::option::Option::Some(ksp_onchain_transport_lib::SolanaCommitment::Confirmed));
|
||||
let pool = match http_pool(cluster, hydration_role, "get_transaction") {
|
||||
std::option::Option::Some(value) => value,
|
||||
std::option::Option::None => return std::option::Option::None,
|
||||
};
|
||||
return match crate::RawTransactionIngestYellowstoneSource::new(channel, request, pool, ksp_onchain_transport_lib::HttpRoleName::new(hydration_role)) {
|
||||
std::result::Result::Ok(value) => std::option::Option::Some(value),
|
||||
std::result::Result::Err(_) => std::option::Option::None,
|
||||
};
|
||||
}
|
||||
|
||||
fn signal_source() -> std::option::Option<crate::RawTransactionIngestYellowstoneSource> {
|
||||
let endpoint = match grpc_endpoint("devnet") {
|
||||
std::option::Option::Some(value) => value,
|
||||
@@ -544,9 +589,11 @@ async fn pre_002_runtime_resource_debug_is_safe_and_exposes_no_client_inner() {
|
||||
let resources = crate::RawTransactionIngestRuntimeResources::new(source);
|
||||
let debug = std::format!("{resources:?}");
|
||||
assert!(debug.contains("RawTransactionIngestRuntimeResources"));
|
||||
assert!(debug.contains("devnet"));
|
||||
assert!(debug.contains("blocks_meta_filter_count"));
|
||||
assert!(debug.contains("slot_filter_count"));
|
||||
assert!(debug.contains("source_count"));
|
||||
assert!(debug.contains("1"));
|
||||
assert!(!debug.contains("devnet"));
|
||||
assert!(!debug.contains("blocks_meta_filter_count"));
|
||||
assert!(!debug.contains("slot_filter_count"));
|
||||
assert!(!debug.contains("GRPC-SECRET-CANARY"));
|
||||
assert!(!debug.contains("HTTP-SECRET-CANARY"));
|
||||
assert!(!debug.contains("127.0.0.1"));
|
||||
@@ -591,6 +638,116 @@ async fn pre_002_runtime_resources_reject_worker_network_mismatch_without_starti
|
||||
return;
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "current_thread")]
|
||||
async fn v0_3_13_pre_002_live_source_key_is_stable_and_request_sensitive() {
|
||||
let first = match source_with_identity("devnet", "source-a", "tx-a", std::option::Option::Some(false)) {
|
||||
std::option::Option::Some(value) => value,
|
||||
std::option::Option::None => return,
|
||||
};
|
||||
let same = match source_with_identity("devnet", "source-a", "tx-a", std::option::Option::Some(false)) {
|
||||
std::option::Option::Some(value) => value,
|
||||
std::option::Option::None => return,
|
||||
};
|
||||
let changed_filter = match source_with_identity("devnet", "source-a", "tx-a", std::option::Option::Some(true)) {
|
||||
std::option::Option::Some(value) => value,
|
||||
std::option::Option::None => return,
|
||||
};
|
||||
let changed_endpoint = match source_with_identity("devnet", "source-b", "tx-a", std::option::Option::Some(false)) {
|
||||
std::option::Option::Some(value) => value,
|
||||
std::option::Option::None => return,
|
||||
};
|
||||
let changed_hydration_role = match source_with_identity_and_hydration_role("devnet", "source-a", "tx-a", std::option::Option::Some(false), "secondary") {
|
||||
std::option::Option::Some(value) => value,
|
||||
std::option::Option::None => return,
|
||||
};
|
||||
assert_eq!(first.source_key, same.source_key);
|
||||
assert_eq!(first.source_key, changed_hydration_role.source_key);
|
||||
assert_ne!(first.source_key, changed_filter.source_key);
|
||||
assert_ne!(first.source_key, changed_endpoint.source_key);
|
||||
return;
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "current_thread")]
|
||||
async fn v0_3_13_pre_002_runtime_resources_accept_exact_bound_and_reject_excess_transactionally() {
|
||||
let first = match source_with_identity("devnet", "source-00", "tx", std::option::Option::None) {
|
||||
std::option::Option::Some(value) => value,
|
||||
std::option::Option::None => return,
|
||||
};
|
||||
let mut resources = crate::RawTransactionIngestRuntimeResources::new(first);
|
||||
for index in 1..crate::MAX_RAW_TRANSACTION_INGEST_LIVE_SOURCES {
|
||||
let endpoint_name = std::format!("source-{index:02}");
|
||||
let source = match source_with_identity("devnet", endpoint_name.as_str(), "tx", std::option::Option::None) {
|
||||
std::option::Option::Some(value) => value,
|
||||
std::option::Option::None => return,
|
||||
};
|
||||
assert!(resources.try_push_yellowstone_source(source).is_ok());
|
||||
}
|
||||
assert_eq!(resources.source_count(), crate::MAX_RAW_TRANSACTION_INGEST_LIVE_SOURCES);
|
||||
let excess = match source_with_identity("devnet", "source-excess", "tx", std::option::Option::None) {
|
||||
std::option::Option::Some(value) => value,
|
||||
std::option::Option::None => return,
|
||||
};
|
||||
let error = match resources.try_push_yellowstone_source(excess) {
|
||||
std::result::Result::Ok(()) => return,
|
||||
std::result::Result::Err(error) => error,
|
||||
};
|
||||
assert_eq!(error.code(), crate::ERROR_CODE_RAW_TRANSACTION_INGEST_RUNTIME_INVALID);
|
||||
assert!(error.context().iter().any(|context| return context.value() == "runtime_resources.source_count_exceeded"));
|
||||
assert_eq!(resources.source_count(), crate::MAX_RAW_TRANSACTION_INGEST_LIVE_SOURCES);
|
||||
return;
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "current_thread")]
|
||||
async fn v0_3_13_pre_002_runtime_resources_reject_duplicate_and_cross_network_source_before_activation() {
|
||||
let first = match source_with_identity("devnet", "source-a", "tx", std::option::Option::None) {
|
||||
std::option::Option::Some(value) => value,
|
||||
std::option::Option::None => return,
|
||||
};
|
||||
let duplicate = match source_with_identity("devnet", "source-a", "tx", std::option::Option::None) {
|
||||
std::option::Option::Some(value) => value,
|
||||
std::option::Option::None => return,
|
||||
};
|
||||
let other_network = match source_with_identity("mainnet", "source-b", "tx", std::option::Option::None) {
|
||||
std::option::Option::Some(value) => value,
|
||||
std::option::Option::None => return,
|
||||
};
|
||||
let mut resources = crate::RawTransactionIngestRuntimeResources::new(first);
|
||||
let duplicate_error = match resources.try_push_yellowstone_source(duplicate) {
|
||||
std::result::Result::Ok(()) => return,
|
||||
std::result::Result::Err(error) => error,
|
||||
};
|
||||
assert!(duplicate_error.context().iter().any(|context| return context.value() == "runtime_resources.duplicate_source_identity"));
|
||||
assert_eq!(resources.source_count(), 1);
|
||||
let network_error = match resources.try_push_yellowstone_source(other_network) {
|
||||
std::result::Result::Ok(()) => return,
|
||||
std::result::Result::Err(error) => error,
|
||||
};
|
||||
assert!(network_error.context().iter().any(|context| return context.value() == "runtime_resources.source_network_mismatch"));
|
||||
assert_eq!(resources.source_count(), 1);
|
||||
return;
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "current_thread")]
|
||||
async fn v0_3_13_pre_002_multi_source_activation_fails_closed_until_supervisor_tranche() {
|
||||
let first = match source_with_identity("devnet", "source-a", "tx", std::option::Option::None) {
|
||||
std::option::Option::Some(value) => value,
|
||||
std::option::Option::None => return,
|
||||
};
|
||||
let second = match source_with_identity("devnet", "source-b", "tx", std::option::Option::None) {
|
||||
std::option::Option::Some(value) => value,
|
||||
std::option::Option::None => return,
|
||||
};
|
||||
let mut resources = crate::RawTransactionIngestRuntimeResources::new(first);
|
||||
assert!(resources.try_push_yellowstone_source(second).is_ok());
|
||||
let error = match resources.into_yellowstone_source() {
|
||||
std::result::Result::Ok(_) => return,
|
||||
std::result::Result::Err(error) => error,
|
||||
};
|
||||
assert_eq!(error.code(), crate::ERROR_CODE_RAW_TRANSACTION_INGEST_RUNTIME_INVALID);
|
||||
assert!(error.context().iter().any(|context| return context.value() == "runtime_resources.multi_source_activation_pending"));
|
||||
return;
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "current_thread")]
|
||||
async fn pre_003_transaction_signal_fixture_preserves_exact_source_neutral_identity() {
|
||||
let source = match signal_source() {
|
||||
|
||||
Reference in New Issue
Block a user