From f7c57f21c57c53105507aa4e3ee6ce71e86796e5 Mon Sep 17 00:00:00 2001 From: SinuS Von SifriduS Date: Mon, 7 Sep 2026 18:58:48 +0200 Subject: [PATCH] v0.3.10-pre.006 --- Cargo.toml | 4 +- crates/ksp-job-backfill-lib/Cargo.toml | 4 +- .../tests/dependency_boundary.rs | 34 +- .../ksp-job-backfill-lib/tests/hardening.rs | 4 +- .../tests/yellowstone_raw_parity.rs | 465 +++++++++++++++ crates/ksp-raw-transaction-lib/src/lib.rs | 21 +- .../ksp-raw-transaction-lib/src/signature.rs | 155 ++++- crates/ksp-raw-transaction-lib/src/wire.rs | 545 ++++++++++++++++++ .../tests/dependency_boundary.rs | 4 +- .../tests/public_api.rs | 23 +- .../tests/release_completeness.rs | 4 +- .../unit_tests/signature.rs | 21 +- .../unit_tests/wire.rs | 122 ++++ deltas/0.3.10/pre.006.md | 206 +++++++ .../011-RAW_TRANSACTION_ACQUISITION.md | 132 +++-- ...031-V0_3_10_RAW_TRANSACTION_INGEST_PLAN.md | 16 +- .../027-V0_3_10_RAW_TRANSACTION_INGEST.md | 49 +- 17 files changed, 1748 insertions(+), 61 deletions(-) create mode 100644 crates/ksp-job-backfill-lib/tests/yellowstone_raw_parity.rs create mode 100644 crates/ksp-raw-transaction-lib/src/wire.rs create mode 100644 crates/ksp-raw-transaction-lib/unit_tests/wire.rs create mode 100644 deltas/0.3.10/pre.006.md diff --git a/Cargo.toml b/Cargo.toml index ed75d5e..6750cd2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,12 +1,12 @@ # file: Cargo.toml -# version: 498 +# version: 499 [workspace] resolver = "3" members = ["crates/ksp-app-backfill-desk", "crates/ksp-app-config-desk", "crates/ksp-app-solprices-desk", "crates/ksp-app-store-desk", "crates/ksp-app-wallet-desk", "crates/ksp-config-lib", "crates/ksp-core-lib", "crates/ksp-interface-lib", "crates/ksp-job-api", "crates/ksp-job-backfill-lib", "crates/ksp-logging-lib", "crates/ksp-offchain-transport-lib", "crates/ksp-onchain-transport-lib", "crates/ksp-program-api", "crates/ksp-raw-transaction-lib", "crates/ksp-store-api", "crates/ksp-store-lib", "crates/ksp-store-postgres-lib", "crates/ksp-wallet-lib", "crates/ksp-worker-api"] [workspace.package] -version = "0.3.10-pre.5.fix.2" +version = "0.3.10-pre.6" edition = "2024" license = "MIT" repository = "https://git.sasedev.com/Sasedev/khadhroony-solana-project" diff --git a/crates/ksp-job-backfill-lib/Cargo.toml b/crates/ksp-job-backfill-lib/Cargo.toml index cbc02c8..79ea5c5 100644 --- a/crates/ksp-job-backfill-lib/Cargo.toml +++ b/crates/ksp-job-backfill-lib/Cargo.toml @@ -1,5 +1,5 @@ # file: crates/ksp-job-backfill-lib/Cargo.toml -# version: 6 +# version: 7 [package] name = "ksp-job-backfill-lib" @@ -22,6 +22,8 @@ tokio = { workspace = true, features = ["macros", "sync"] } [dev-dependencies] tokio = { workspace = true, features = ["macros", "rt-multi-thread"] } tokio-tungstenite = { workspace = true, features = ["handshake"] } +tonic = { workspace = true, features = ["codegen", "server"] } +yellowstone-grpc-proto = { workspace = true, features = ["tonic"] } [lints] workspace = true diff --git a/crates/ksp-job-backfill-lib/tests/dependency_boundary.rs b/crates/ksp-job-backfill-lib/tests/dependency_boundary.rs index e099133..bcd240f 100644 --- a/crates/ksp-job-backfill-lib/tests/dependency_boundary.rs +++ b/crates/ksp-job-backfill-lib/tests/dependency_boundary.rs @@ -1,11 +1,15 @@ // file: crates/ksp-job-backfill-lib/tests/dependency_boundary.rs -// version: 8 +// version: 9 //! Dependency firewall canaries through the concrete cancellation and latest-value runtime tranche. #[test] fn pre_003_manifest_adds_common_raw_edge_without_crossing_store_or_runtime_boundaries() { let manifest = include_str!("../Cargo.toml"); + let production_manifest = match manifest.split_once("[dev-dependencies]") { + std::option::Option::Some((production, _)) => production, + std::option::Option::None => manifest, + }; for required in [ "futures-util = { workspace = true, features = [\"std\"] }", "ksp-core-lib = { path = \"../ksp-core-lib\" }", @@ -18,7 +22,7 @@ fn pre_003_manifest_adds_common_raw_edge_without_crossing_store_or_runtime_bound "sha2.workspace = true", "tokio = { workspace = true, features = [\"macros\", \"sync\"] }", ] { - assert!(manifest.contains(required), "required Backfill dependency missing: {required}"); + assert!(production_manifest.contains(required), "required Backfill dependency missing: {required}"); } for forbidden in [ "ksp-config-lib", @@ -33,7 +37,7 @@ fn pre_003_manifest_adds_common_raw_edge_without_crossing_store_or_runtime_bound "serde = {", "tonic", ] { - assert!(!manifest.contains(forbidden), "forbidden Backfill dependency present: {forbidden}"); + assert!(!production_manifest.contains(forbidden), "forbidden Backfill dependency present: {forbidden}"); } let root = include_str!("../src/lib.rs"); assert!(!root.contains("tokio::"), "Tokio implementation types must not leak through the public crate root"); @@ -59,6 +63,30 @@ fn pre_005_websocket_parity_dependency_stays_test_only() { return; } +#[test] +fn pre_006_yellowstone_parity_dependencies_stay_test_only() { + let manifest = include_str!("../Cargo.toml"); + let parts = manifest.splitn(2, "[dev-dependencies]").collect::>(); + let (dependencies, dev_dependencies) = match parts.as_slice() { + [dependencies, dev_dependencies] => (*dependencies, *dev_dependencies), + _ => { + assert_eq!(parts.len(), 2, "Backfill manifest must retain exactly one dev-dependencies section"); + return; + }, + }; + assert!(!dependencies.contains("tonic"), "Yellowstone parity server must not become a production Backfill dependency"); + assert!(!dependencies.contains("yellowstone-grpc-proto"), "Yellowstone protobuf must not become a production Backfill dependency"); + assert!( + dev_dependencies.contains("tonic = { workspace = true, features = [\"codegen\", \"server\"] }"), + "Yellowstone parity server must remain an explicit test-only dependency", + ); + assert!( + dev_dependencies.contains("yellowstone-grpc-proto = { workspace = true, features = [\"tonic\"] }"), + "Yellowstone protobuf must remain an explicit test-only dependency", + ); + return; +} + #[test] fn pre_009_production_sources_keep_transport_store_and_scheduler_ownership_separate() { let neutral_sources = [ diff --git a/crates/ksp-job-backfill-lib/tests/hardening.rs b/crates/ksp-job-backfill-lib/tests/hardening.rs index aba448a..2b584a4 100644 --- a/crates/ksp-job-backfill-lib/tests/hardening.rs +++ b/crates/ksp-job-backfill-lib/tests/hardening.rs @@ -1,5 +1,5 @@ // file: crates/ksp-job-backfill-lib/tests/hardening.rs -// version: 6 +// version: 7 //! Adversarial, security, visibility and external-boundary hardening canaries for `pre.010`. @@ -275,7 +275,7 @@ fn pre_010_manifest_dependency_surface_is_exact_and_backend_neutral() { "tokio", ]) ); - assert_eq!(dev, std::collections::BTreeSet::from(["tokio", "tokio-tungstenite"])); + assert_eq!(dev, std::collections::BTreeSet::from(["tokio", "tokio-tungstenite", "tonic", "yellowstone-grpc-proto"])); assert!(manifest.contains("ksp-store-lib = { path = \"../ksp-store-lib\", default-features = false }")); assert!(manifest.contains("tokio = { workspace = true, features = [\"macros\", \"sync\"] }")); assert!(!manifest.contains("ksp-store-postgres-lib")); diff --git a/crates/ksp-job-backfill-lib/tests/yellowstone_raw_parity.rs b/crates/ksp-job-backfill-lib/tests/yellowstone_raw_parity.rs new file mode 100644 index 0000000..a9b4fa0 --- /dev/null +++ b/crates/ksp-job-backfill-lib/tests/yellowstone_raw_parity.rs @@ -0,0 +1,465 @@ +// file: crates/ksp-job-backfill-lib/tests/yellowstone_raw_parity.rs +// version: 1 + +struct FixtureStream { + receiver: tokio::sync::mpsc::Receiver, +} + +impl FixtureStream { + fn new(receiver: tokio::sync::mpsc::Receiver) -> Self { + return Self { receiver }; + } +} + +impl futures_util::Stream for FixtureStream { + type Item = T; + + fn poll_next(self: std::pin::Pin<&mut Self>, context: &mut std::task::Context<'_>) -> std::task::Poll> { + return self.get_mut().receiver.poll_recv(context); + } +} + +#[derive(Clone)] +struct FixtureGeyser; + +#[allow(clippy::implicit_return)] // tonic::async_trait generates async wrapper tails outside the authored fixture bodies. +#[tonic::async_trait] +impl yellowstone_grpc_proto::geyser::geyser_server::Geyser for FixtureGeyser { + type SubscribeStream = FixtureStream>; + + async fn subscribe( + &self, + request: tonic::Request>, + ) -> std::result::Result, tonic::Status> { + let mut inbound = request.into_inner(); + let (sender, receiver) = tokio::sync::mpsc::channel(4); + tokio::spawn(async move { + let initial = inbound.message().await; + if !matches!(initial, std::result::Result::Ok(std::option::Option::Some(_))) { + return; + } + if sender.send(std::result::Result::Ok(transaction_update())).await.is_err() { + return; + } + let _ = sender.send(std::result::Result::Ok(block_update())).await; + }); + return std::result::Result::Ok(tonic::Response::new(FixtureStream::new(receiver))); + } + + type SubscribeDeshredStream = futures_util::stream::Empty>; + + async fn subscribe_deshred( + &self, + _request: tonic::Request>, + ) -> std::result::Result, tonic::Status> { + return std::result::Result::Err(tonic::Status::unimplemented("deshred is outside the pre.006 fixture")); + } + + type SubscribeGossipStream = futures_util::stream::Empty>; + + async fn subscribe_gossip( + &self, + _request: tonic::Request, + ) -> std::result::Result, tonic::Status> { + return std::result::Result::Err(tonic::Status::unimplemented("gossip is outside the pre.006 fixture")); + } + + async fn subscribe_replay_info( + &self, + _request: tonic::Request, + ) -> std::result::Result, tonic::Status> { + return std::result::Result::Err(tonic::Status::unimplemented("replay info is outside the pre.006 fixture")); + } + + async fn ping( + &self, + _request: tonic::Request, + ) -> std::result::Result, tonic::Status> { + return std::result::Result::Err(tonic::Status::unimplemented("unary is outside the pre.006 fixture")); + } + + async fn get_latest_blockhash( + &self, + _request: tonic::Request, + ) -> std::result::Result, tonic::Status> { + return std::result::Result::Err(tonic::Status::unimplemented("unary is outside the pre.006 fixture")); + } + + async fn get_block_height( + &self, + _request: tonic::Request, + ) -> std::result::Result, tonic::Status> { + return std::result::Result::Err(tonic::Status::unimplemented("unary is outside the pre.006 fixture")); + } + + async fn get_slot( + &self, + _request: tonic::Request, + ) -> std::result::Result, tonic::Status> { + return std::result::Result::Err(tonic::Status::unimplemented("unary is outside the pre.006 fixture")); + } + + async fn is_blockhash_valid( + &self, + _request: tonic::Request, + ) -> std::result::Result, tonic::Status> { + return std::result::Result::Err(tonic::Status::unimplemented("unary is outside the pre.006 fixture")); + } + + async fn get_version( + &self, + _request: tonic::Request, + ) -> std::result::Result, tonic::Status> { + return std::result::Result::Err(tonic::Status::unimplemented("unary is outside the pre.006 fixture")); + } +} + +struct FixtureServer { + endpoint_url: std::string::String, + shutdown: std::option::Option>, + task: tokio::task::JoinHandle<()>, +} + +impl FixtureServer { + async fn start() -> std::result::Result { + let bind_address = match "127.0.0.1:0".parse::() { + std::result::Result::Ok(value) => value, + std::result::Result::Err(_) => return std::result::Result::Err("fixture bind address must parse"), + }; + let incoming = match tonic::transport::server::TcpIncoming::bind(bind_address) { + std::result::Result::Ok(value) => value, + std::result::Result::Err(_) => return std::result::Result::Err("fixture gRPC listener must bind"), + }; + let local_address = match incoming.local_addr() { + std::result::Result::Ok(value) => value, + std::result::Result::Err(_) => return std::result::Result::Err("fixture gRPC listener must expose local address"), + }; + let (shutdown, shutdown_receiver) = tokio::sync::oneshot::channel(); + let task = tokio::spawn(async move { + let service = yellowstone_grpc_proto::geyser::geyser_server::GeyserServer::new(FixtureGeyser); + let result = tonic::transport::Server::builder() + .serve_with_incoming_shutdown(service, incoming, async move { + let _ = shutdown_receiver.await; + }) + .await; + assert!(result.is_ok()); + }); + return std::result::Result::Ok(Self { endpoint_url: format!("http://{local_address}"), shutdown: std::option::Option::Some(shutdown), task }); + } + + async fn stop(mut self) { + if let std::option::Option::Some(shutdown) = self.shutdown.take() { + let _ = shutdown.send(()); + } + let result = self.task.await; + assert!(result.is_ok()); + } +} + +fn transaction_info() -> yellowstone_grpc_proto::geyser::SubscribeUpdateTransactionInfo { + return yellowstone_grpc_proto::geyser::SubscribeUpdateTransactionInfo { + signature: vec![9_u8; 64], + is_vote: false, + transaction: std::option::Option::Some(yellowstone_grpc_proto::solana::storage::confirmed_block::Transaction { + signatures: vec![vec![9_u8; 64]], + message: std::option::Option::Some(yellowstone_grpc_proto::solana::storage::confirmed_block::Message { + header: std::option::Option::Some(yellowstone_grpc_proto::solana::storage::confirmed_block::MessageHeader { + num_required_signatures: 1, + num_readonly_signed_accounts: 0, + num_readonly_unsigned_accounts: 1, + }), + account_keys: vec![vec![1_u8; 32], vec![2_u8; 32]], + recent_blockhash: vec![3_u8; 32], + instructions: vec![yellowstone_grpc_proto::solana::storage::confirmed_block::CompiledInstruction { + program_id_index: 1, + accounts: vec![0_u8], + data: vec![4_u8, 5, 6], + }], + versioned: true, + address_table_lookups: vec![], + config: std::option::Option::Some(yellowstone_grpc_proto::solana::storage::confirmed_block::TransactionConfig { + priority_fee: std::option::Option::Some(7), + compute_unit_limit: std::option::Option::Some(8), + loaded_accounts_data_size_limit: std::option::Option::Some(9), + heap_size: std::option::Option::Some(32_768), + }), + }), + }), + meta: std::option::Option::Some(yellowstone_grpc_proto::solana::storage::confirmed_block::TransactionStatusMeta { + fee: 5_000, + pre_balances: vec![100, 200], + post_balances: vec![90, 210], + compute_units_consumed: std::option::Option::Some(123), + cost_units: std::option::Option::Some(456), + ..std::default::Default::default() + }), + index: 3, + }; +} + +fn transaction_update() -> yellowstone_grpc_proto::geyser::SubscribeUpdate { + return yellowstone_grpc_proto::geyser::SubscribeUpdate { + filters: vec!["transactions-pre-006".to_owned()], + update_oneof: std::option::Option::Some(yellowstone_grpc_proto::geyser::subscribe_update::UpdateOneof::Transaction( + yellowstone_grpc_proto::geyser::SubscribeUpdateTransaction { transaction: std::option::Option::Some(transaction_info()), slot: 42 }, + )), + created_at: std::option::Option::None, + }; +} + +fn block_update() -> yellowstone_grpc_proto::geyser::SubscribeUpdate { + return yellowstone_grpc_proto::geyser::SubscribeUpdate { + filters: vec!["blocks-pre-006".to_owned()], + update_oneof: std::option::Option::Some(yellowstone_grpc_proto::geyser::subscribe_update::UpdateOneof::Block( + yellowstone_grpc_proto::geyser::SubscribeUpdateBlock { + slot: 42, + blockhash: ksp_core_lib::Pubkey::new_from_array([21_u8; 32]).to_string(), + rewards: std::option::Option::None, + block_time: std::option::Option::Some(yellowstone_grpc_proto::solana::storage::confirmed_block::UnixTimestamp { timestamp: 1_787_104_000 }), + block_height: std::option::Option::Some(yellowstone_grpc_proto::solana::storage::confirmed_block::BlockHeight { block_height: 41 }), + transactions: vec![transaction_info()], + parent_slot: 41, + parent_blockhash: ksp_core_lib::Pubkey::new_from_array([22_u8; 32]).to_string(), + executed_transaction_count: 1, + updated_account_count: 0, + accounts: vec![], + entries_count: 0, + entries: vec![], + }, + )), + created_at: std::option::Option::None, + }; +} + +fn fixture_settings(url: &str) -> ksp_core_lib::Result { + let endpoint_url = match ksp_onchain_transport_lib::YellowstoneGrpcEndpointUrl::parse(url) { + std::result::Result::Ok(value) => value, + std::result::Result::Err(error) => return std::result::Result::Err(error), + }; + return std::result::Result::Ok(ksp_onchain_transport_lib::YellowstoneGrpcEndpointSettings::new( + "pre-006-fixture", + true, + ksp_onchain_transport_lib::YellowstoneGrpcProviderName::new("fixture-provider"), + ksp_onchain_transport_lib::YellowstoneGrpcClusterName::new("devnet"), + endpoint_url, + ksp_onchain_transport_lib::YellowstoneGrpcSessionSettings::default(), + )); +} + +fn project_wire( + transaction: &ksp_onchain_transport_lib::YellowstoneStoredTransaction, +) -> ksp_core_lib::Result { + let message = transaction.message(); + let header = message.header(); + let required = match u8::try_from(header.num_required_signatures()) { + std::result::Result::Ok(value) => value, + std::result::Result::Err(_) => return std::result::Result::Err(projection_error("num_required_signatures")), + }; + let readonly_signed = match u8::try_from(header.num_readonly_signed_accounts()) { + std::result::Result::Ok(value) => value, + std::result::Result::Err(_) => return std::result::Result::Err(projection_error("num_readonly_signed_accounts")), + }; + let readonly_unsigned = match u8::try_from(header.num_readonly_unsigned_accounts()) { + std::result::Result::Ok(value) => value, + std::result::Result::Err(_) => return std::result::Result::Err(projection_error("num_readonly_unsigned_accounts")), + }; + let version = if message.config().is_some() { + ksp_raw_transaction_lib::RawSolanaMessageVersion::V1 + } else if message.versioned() { + ksp_raw_transaction_lib::RawSolanaMessageVersion::V0 + } else { + ksp_raw_transaction_lib::RawSolanaMessageVersion::Legacy + }; + let signatures = transaction.signatures().iter().map(|value| return *value.as_bytes()).collect(); + let account_keys = message.account_keys().iter().map(solana_pubkey_bytes).collect(); + let mut instructions = std::vec::Vec::with_capacity(message.instructions().len()); + for instruction in message.instructions() { + let program_id_index = match u8::try_from(instruction.program_id_index()) { + std::result::Result::Ok(value) => value, + std::result::Result::Err(_) => return std::result::Result::Err(projection_error("program_id_index")), + }; + instructions.push(ksp_raw_transaction_lib::RawSolanaCompiledInstruction::new( + program_id_index, + instruction.accounts().to_vec(), + instruction.data().to_vec(), + )); + } + let address_table_lookups = message + .address_table_lookups() + .iter() + .map(|lookup| { + return ksp_raw_transaction_lib::RawSolanaAddressTableLookup::new( + lookup.account_key().to_bytes(), + lookup.writable_indexes().to_vec(), + lookup.readonly_indexes().to_vec(), + ); + }) + .collect(); + let config = message.config().map(|value| { + return ksp_raw_transaction_lib::RawSolanaTransactionConfig::new( + value.priority_fee(), + value.compute_unit_limit(), + value.loaded_accounts_data_size_limit(), + value.heap_size(), + ); + }); + let raw_message = ksp_raw_transaction_lib::RawSolanaTransactionMessage::new( + version, + ksp_raw_transaction_lib::RawSolanaMessageHeader::new(required, readonly_signed, readonly_unsigned), + account_keys, + *message.recent_blockhash().as_bytes(), + instructions, + address_table_lookups, + config, + ); + return std::result::Result::Ok(ksp_raw_transaction_lib::RawSolanaTransactionWire::new(signatures, raw_message)); +} + +fn solana_pubkey_bytes(value: &ksp_core_lib::Pubkey) -> [u8; 32] { + return value.to_bytes(); +} + +fn projection_error(field: &'static str) -> ksp_core_lib::Error { + return ksp_core_lib::Error::new("KSP-RAW-TRANSACTION-PRE-006-PROJECTION", "Yellowstone fixture field does not fit common wire") + .with_context("field", field); +} + +fn expected_v1_wire() -> ksp_raw_transaction_lib::RawSolanaTransactionWire { + let message = ksp_raw_transaction_lib::RawSolanaTransactionMessage::new( + ksp_raw_transaction_lib::RawSolanaMessageVersion::V1, + ksp_raw_transaction_lib::RawSolanaMessageHeader::new(1, 0, 1), + vec![[1_u8; 32], [2_u8; 32]], + [3_u8; 32], + vec![ksp_raw_transaction_lib::RawSolanaCompiledInstruction::new(1, vec![0_u8], vec![4_u8, 5, 6])], + vec![], + std::option::Option::Some(ksp_raw_transaction_lib::RawSolanaTransactionConfig::new( + std::option::Option::Some(7), + std::option::Option::Some(8), + std::option::Option::Some(9), + std::option::Option::Some(32_768), + )), + ); + return ksp_raw_transaction_lib::RawSolanaTransactionWire::new(vec![[9_u8; 64]], message); +} + +fn fail_test(message: std::string::String) { + assert!(message.is_empty(), "{message}"); +} + +#[tokio::test] +async fn pre_006_yellowstone_transaction_and_block_preserve_exact_v1_wire_and_block_time() { + let server = match FixtureServer::start().await { + std::result::Result::Ok(value) => value, + std::result::Result::Err(error) => { + assert!(error.is_empty(), "fixture server start failed: {error}"); + return; + }, + }; + let settings = match fixture_settings(server.endpoint_url.as_str()) { + std::result::Result::Ok(value) => value, + std::result::Result::Err(error) => { + fail_test(format!("fixture settings failed: {}", error.code())); + return; + }, + }; + let channel = match ksp_onchain_transport_lib::YellowstoneGrpcChannel::connect(&settings).await { + std::result::Result::Ok(value) => value, + std::result::Result::Err(error) => { + fail_test(format!("fixture channel connect failed: {}", error.code())); + return; + }, + }; + let request = ksp_onchain_transport_lib::YellowstoneSubscribeRequest::new(); + let mut session = match channel.open_standard_subscribe(request).await { + std::result::Result::Ok(value) => value, + std::result::Result::Err(error) => { + fail_test(format!("fixture subscribe failed: {}", error.code())); + return; + }, + }; + let expected = match ksp_raw_transaction_lib::serialize_solana_transaction_wire(&expected_v1_wire()) { + std::result::Result::Ok(value) => value, + std::result::Result::Err(error) => { + fail_test(format!("expected V1 wire serialization failed: {}", error.code())); + return; + }, + }; + let transaction = match session.next_update().await { + std::result::Result::Ok(std::option::Option::Some(value)) => value, + std::result::Result::Ok(std::option::Option::None) => { + fail_test("transaction update must be present".to_owned()); + return; + }, + std::result::Result::Err(error) => { + fail_test(format!("transaction update decode failed: {}", error.code())); + return; + }, + }; + let transaction = match transaction { + ksp_onchain_transport_lib::YellowstoneSubscribeUpdate::Transaction(value) => value, + _ => { + fail_test("first Yellowstone fixture update must be a transaction".to_owned()); + return; + }, + }; + assert_eq!(transaction.slot(), 42); + assert_eq!(transaction.transaction().index(), 3); + assert_eq!(transaction.transaction().meta().fee(), 5_000); + let transaction_wire = match project_wire(transaction.transaction().transaction()) { + std::result::Result::Ok(value) => value, + std::result::Result::Err(error) => { + fail_test(format!("transaction projection failed: {}", error.code())); + return; + }, + }; + let transaction_bytes = match ksp_raw_transaction_lib::serialize_solana_transaction_wire(&transaction_wire) { + std::result::Result::Ok(value) => value, + std::result::Result::Err(error) => { + fail_test(format!("transaction V1 wire serialization failed: {}", error.code())); + return; + }, + }; + assert_eq!(transaction_bytes, expected); + let block = match session.next_update().await { + std::result::Result::Ok(std::option::Option::Some(value)) => value, + std::result::Result::Ok(std::option::Option::None) => { + fail_test("block update must be present".to_owned()); + return; + }, + std::result::Result::Err(error) => { + fail_test(format!("block update decode failed: {}", error.code())); + return; + }, + }; + let block = match block { + ksp_onchain_transport_lib::YellowstoneSubscribeUpdate::Block(value) => value, + _ => { + fail_test("second Yellowstone fixture update must be a block".to_owned()); + return; + }, + }; + assert_eq!(block.slot(), 42); + assert_eq!(block.block_time(), std::option::Option::Some(1_787_104_000)); + assert_eq!(block.transactions().len(), 1); + assert_eq!(block.transactions()[0].index(), 3); + let block_wire = match project_wire(block.transactions()[0].transaction()) { + std::result::Result::Ok(value) => value, + std::result::Result::Err(error) => { + fail_test(format!("block transaction projection failed: {}", error.code())); + return; + }, + }; + let block_bytes = match ksp_raw_transaction_lib::serialize_solana_transaction_wire(&block_wire) { + std::result::Result::Ok(value) => value, + std::result::Result::Err(error) => { + fail_test(format!("block V1 wire serialization failed: {}", error.code())); + return; + }, + }; + assert_eq!(block_bytes, expected); + assert_eq!(block_bytes, transaction_bytes); + assert!(transaction.transaction().meta().error().is_none()); + assert_eq!(block.transactions()[0].meta().fee(), transaction.transaction().meta().fee()); + assert!(session.close().await.is_ok()); + server.stop().await; +} diff --git a/crates/ksp-raw-transaction-lib/src/lib.rs b/crates/ksp-raw-transaction-lib/src/lib.rs index c68b1fd..5c36a8f 100644 --- a/crates/ksp-raw-transaction-lib/src/lib.rs +++ b/crates/ksp-raw-transaction-lib/src/lib.rs @@ -1,5 +1,5 @@ // file: crates/ksp-raw-transaction-lib/src/lib.rs -// version: 2 +// version: 3 #![warn(missing_docs)] #![deny(unreachable_pub)] @@ -16,6 +16,7 @@ mod acquisition; mod canonical; mod error; mod signature; +mod wire; /// Complete source-neutral RAW transaction acquisition containing one canonical entity and one producer-owned observation. pub use self::acquisition::RawTransactionAcquisition; @@ -47,6 +48,24 @@ pub use self::signature::MIN_RAW_TRANSACTION_SIGNATURE_TEXT_BYTES; pub use self::signature::extract_raw_transaction_signature_from_binary_base64; /// Parses one bounded Base58 Solana transaction signature to exactly 64 canonical bytes. pub use self::signature::parse_raw_transaction_signature; +/// One source-neutral v0 address-table lookup. +pub use self::wire::RawSolanaAddressTableLookup; +/// One source-neutral compiled Solana instruction. +pub use self::wire::RawSolanaCompiledInstruction; +/// Source-neutral Solana message header. +pub use self::wire::RawSolanaMessageHeader; +/// Source-neutral Solana transaction message version. +pub use self::wire::RawSolanaMessageVersion; +/// Optional source-neutral Transaction V1 inline configuration. +pub use self::wire::RawSolanaTransactionConfig; +/// Complete source-neutral Solana transaction message. +pub use self::wire::RawSolanaTransactionMessage; +/// Complete source-neutral Solana transaction wire material. +pub use self::wire::RawSolanaTransactionWire; +/// Serializes one source-neutral Solana transaction to exact canonical wire bytes. +pub use self::wire::serialize_solana_transaction_wire; +/// Serializes one source-neutral Solana transaction to canonical padded standard Base64. +pub use self::wire::serialize_solana_transaction_wire_base64; /// Creates a safe canonicalization error without copying source payload material. pub(crate) use self::error::canonicalization_error; diff --git a/crates/ksp-raw-transaction-lib/src/signature.rs b/crates/ksp-raw-transaction-lib/src/signature.rs index 5bb3089..c9e3e22 100644 --- a/crates/ksp-raw-transaction-lib/src/signature.rs +++ b/crates/ksp-raw-transaction-lib/src/signature.rs @@ -1,5 +1,5 @@ // file: crates/ksp-raw-transaction-lib/src/signature.rs -// version: 2 +// version: 3 use base64::Engine; // rust-rules: trait-import @@ -68,7 +68,14 @@ pub fn extract_raw_transaction_signature_from_binary_base64(value: &str) -> ksp_ if base64::engine::general_purpose::STANDARD.encode(decoded.as_slice()) != value { return std::result::Result::Err(crate::signature_error()); } - let prefix = decode_signature_count(decoded.as_slice()); + if decoded.first() == std::option::Option::Some(&0x81) { + return extract_v1_signature(decoded.as_slice()); + } + return extract_legacy_or_v0_signature(decoded.as_slice()); +} + +fn extract_legacy_or_v0_signature(decoded: &[u8]) -> ksp_core_lib::Result { + let prefix = decode_signature_count(decoded); let (signature_count, prefix_len) = match prefix { std::result::Result::Ok(prefix) => prefix, std::result::Result::Err(error) => return std::result::Result::Err(error), @@ -87,11 +94,151 @@ pub fn extract_raw_transaction_signature_from_binary_base64(value: &str) -> ksp_ if message_offset >= decoded.len() { return std::result::Result::Err(crate::signature_error()); } - let first_end = match prefix_len.checked_add(64) { + return copy_signature(decoded, prefix_len); +} + +fn extract_v1_signature(decoded: &[u8]) -> ksp_core_lib::Result { + if decoded.len() > 4_096 || decoded.len() < 42 { + return std::result::Result::Err(crate::signature_error()); + } + let required_signatures = match decoded.get(1) { + std::option::Option::Some(value) => usize::from(*value), + std::option::Option::None => return std::result::Result::Err(crate::signature_error()), + }; + let readonly_signed = match decoded.get(2) { + std::option::Option::Some(value) => usize::from(*value), + std::option::Option::None => return std::result::Result::Err(crate::signature_error()), + }; + let readonly_unsigned = match decoded.get(3) { + std::option::Option::Some(value) => usize::from(*value), + std::option::Option::None => return std::result::Result::Err(crate::signature_error()), + }; + if required_signatures == 0 || required_signatures > 12 || readonly_signed >= required_signatures { + return std::result::Result::Err(crate::signature_error()); + } + let mask_bytes = match decoded.get(4..8) { + std::option::Option::Some(value) => value, + std::option::Option::None => return std::result::Result::Err(crate::signature_error()), + }; + let mask = u32::from_le_bytes([mask_bytes[0], mask_bytes[1], mask_bytes[2], mask_bytes[3]]); + if mask & !0x1f != 0 || matches!(mask & 0b11, 1 | 2) { + return std::result::Result::Err(crate::signature_error()); + } + let instruction_count = match decoded.get(40) { + std::option::Option::Some(value) => usize::from(*value), + std::option::Option::None => return std::result::Result::Err(crate::signature_error()), + }; + let address_count = match decoded.get(41) { + std::option::Option::Some(value) => usize::from(*value), + std::option::Option::None => return std::result::Result::Err(crate::signature_error()), + }; + if instruction_count > 64 || address_count > 64 || address_count < required_signatures + readonly_unsigned { + return std::result::Result::Err(crate::signature_error()); + } + let addresses_len = match address_count.checked_mul(32) { + std::option::Option::Some(value) => value, + std::option::Option::None => return std::result::Result::Err(crate::signature_error()), + }; + let mut offset = match 42_usize.checked_add(addresses_len) { + std::option::Option::Some(value) => value, + std::option::Option::None => return std::result::Result::Err(crate::signature_error()), + }; + let config_len = config_value_length(mask); + offset = match offset.checked_add(config_len) { + std::option::Option::Some(value) => value, + std::option::Option::None => return std::result::Result::Err(crate::signature_error()), + }; + let headers_len = match instruction_count.checked_mul(4) { + std::option::Option::Some(value) => value, + std::option::Option::None => return std::result::Result::Err(crate::signature_error()), + }; + let headers_end = match offset.checked_add(headers_len) { + std::option::Option::Some(value) => value, + std::option::Option::None => return std::result::Result::Err(crate::signature_error()), + }; + if headers_end > decoded.len() { + return std::result::Result::Err(crate::signature_error()); + } + let mut payload_len = 0_usize; + for index in 0..instruction_count { + let header_offset = offset + (index * 4); + let program_index = usize::from(decoded[header_offset]); + let account_index_count = usize::from(decoded[header_offset + 1]); + let data_len = u16::from_le_bytes([decoded[header_offset + 2], decoded[header_offset + 3]]); + if program_index >= address_count { + return std::result::Result::Err(crate::signature_error()); + } + payload_len = match payload_len.checked_add(account_index_count).and_then(|value| return value.checked_add(usize::from(data_len))) { + std::option::Option::Some(value) => value, + std::option::Option::None => return std::result::Result::Err(crate::signature_error()), + }; + } + let payload_end = match headers_end.checked_add(payload_len) { + std::option::Option::Some(value) => value, + std::option::Option::None => return std::result::Result::Err(crate::signature_error()), + }; + let payload = match decoded.get(headers_end..payload_end) { + std::option::Option::Some(value) => value, + std::option::Option::None => return std::result::Result::Err(crate::signature_error()), + }; + let mut payload_offset = 0_usize; + for index in 0..instruction_count { + let header_offset = offset + (index * 4); + let account_index_count = usize::from(decoded[header_offset + 1]); + let data_len = usize::from(u16::from_le_bytes([decoded[header_offset + 2], decoded[header_offset + 3]])); + let account_end = match payload_offset.checked_add(account_index_count) { + std::option::Option::Some(value) => value, + std::option::Option::None => return std::result::Result::Err(crate::signature_error()), + }; + let account_indexes = match payload.get(payload_offset..account_end) { + std::option::Option::Some(value) => value, + std::option::Option::None => return std::result::Result::Err(crate::signature_error()), + }; + if account_indexes.iter().any(|value| return usize::from(*value) >= address_count) { + return std::result::Result::Err(crate::signature_error()); + } + payload_offset = match account_end.checked_add(data_len) { + std::option::Option::Some(value) => value, + std::option::Option::None => return std::result::Result::Err(crate::signature_error()), + }; + } + let signatures_len = match required_signatures.checked_mul(64) { + std::option::Option::Some(value) => value, + std::option::Option::None => return std::result::Result::Err(crate::signature_error()), + }; + let expected_end = match payload_end.checked_add(signatures_len) { + std::option::Option::Some(value) => value, + std::option::Option::None => return std::result::Result::Err(crate::signature_error()), + }; + if expected_end != decoded.len() { + return std::result::Result::Err(crate::signature_error()); + } + return copy_signature(decoded, payload_end); +} + +fn config_value_length(mask: u32) -> usize { + let mut length = 0_usize; + if mask & 0b11 == 0b11 { + length += 8; + } + if mask & (1_u32 << 2) != 0 { + length += 4; + } + if mask & (1_u32 << 3) != 0 { + length += 4; + } + if mask & (1_u32 << 4) != 0 { + length += 4; + } + return length; +} + +fn copy_signature(decoded: &[u8], offset: usize) -> ksp_core_lib::Result { + let first_end = match offset.checked_add(64) { std::option::Option::Some(end) => end, std::option::Option::None => return std::result::Result::Err(crate::signature_error()), }; - let first = match decoded.get(prefix_len..first_end) { + let first = match decoded.get(offset..first_end) { std::option::Option::Some(first) => first, std::option::Option::None => return std::result::Result::Err(crate::signature_error()), }; diff --git a/crates/ksp-raw-transaction-lib/src/wire.rs b/crates/ksp-raw-transaction-lib/src/wire.rs new file mode 100644 index 0000000..e87ced0 --- /dev/null +++ b/crates/ksp-raw-transaction-lib/src/wire.rs @@ -0,0 +1,545 @@ +// file: crates/ksp-raw-transaction-lib/src/wire.rs +// version: 2 + +use base64::Engine; // rust-rules: trait-import + +const MAX_LEGACY_SHORT_VECTOR_VALUE: usize = u16::MAX as usize; +const MAX_V1_ADDRESS_COUNT: usize = 64; +const MAX_V1_INSTRUCTION_COUNT: usize = 64; +const MAX_V1_SIGNATURE_COUNT: usize = 12; +const MAX_V1_TRANSACTION_BYTES: usize = 4_096; + +/// Source-neutral Solana transaction message version used by the exact wire serializer. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum RawSolanaMessageVersion { + /// Legacy Solana message without a version prefix. + Legacy, + /// Versioned transaction message v0, prefixed with `0x80`. + V0, + /// Transaction V1 / SIMD-0385 message, prefixed with `0x81`. + V1, +} + +/// Exact three-byte Solana message header. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct RawSolanaMessageHeader { + num_required_signatures: u8, + num_readonly_signed_accounts: u8, + num_readonly_unsigned_accounts: u8, +} + +impl crate::RawSolanaMessageHeader { + /// Creates an exact Solana message header. + #[must_use] + pub const fn new(num_required_signatures: u8, num_readonly_signed_accounts: u8, num_readonly_unsigned_accounts: u8) -> Self { + return Self { num_required_signatures, num_readonly_signed_accounts, num_readonly_unsigned_accounts }; + } + + /// Returns the required signature count. + #[must_use] + pub const fn num_required_signatures(self) -> u8 { + return self.num_required_signatures; + } + + /// Returns the readonly signed-account count. + #[must_use] + pub const fn num_readonly_signed_accounts(self) -> u8 { + return self.num_readonly_signed_accounts; + } + + /// Returns the readonly unsigned-account count. + #[must_use] + pub const fn num_readonly_unsigned_accounts(self) -> u8 { + return self.num_readonly_unsigned_accounts; + } +} + +/// One source-neutral compiled Solana instruction. +#[derive(Clone, Eq, PartialEq)] +pub struct RawSolanaCompiledInstruction { + program_id_index: u8, + accounts: std::vec::Vec, + data: std::vec::Vec, +} + +impl crate::RawSolanaCompiledInstruction { + /// Creates one compiled instruction while preserving exact ordered account indexes and data bytes. + #[must_use] + pub fn new(program_id_index: u8, accounts: std::vec::Vec, data: std::vec::Vec) -> Self { + return Self { program_id_index, accounts, data }; + } + + /// Returns the program account index. + #[must_use] + pub const fn program_id_index(&self) -> u8 { + return self.program_id_index; + } + + /// Returns ordered account indexes. + #[must_use] + pub fn accounts(&self) -> &[u8] { + return self.accounts.as_slice(); + } + + /// Returns exact instruction data bytes. + #[must_use] + pub fn data(&self) -> &[u8] { + return self.data.as_slice(); + } +} + +impl std::fmt::Debug for crate::RawSolanaCompiledInstruction { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + return formatter + .debug_struct("RawSolanaCompiledInstruction") + .field("program_id_index", &self.program_id_index) + .field("account_index_count", &self.accounts.len()) + .field("data_length", &self.data.len()) + .finish(); + } +} + +/// One source-neutral v0 address-table lookup. +#[derive(Clone, Eq, PartialEq)] +pub struct RawSolanaAddressTableLookup { + account_key: [u8; 32], + writable_indexes: std::vec::Vec, + readonly_indexes: std::vec::Vec, +} + +impl crate::RawSolanaAddressTableLookup { + /// Creates one exact v0 address-table lookup. + #[must_use] + pub fn new(account_key: [u8; 32], writable_indexes: std::vec::Vec, readonly_indexes: std::vec::Vec) -> Self { + return Self { account_key, writable_indexes, readonly_indexes }; + } + + /// Returns the lookup-table account key bytes. + #[must_use] + pub const fn account_key(&self) -> &[u8; 32] { + return &self.account_key; + } + + /// Returns ordered writable lookup indexes. + #[must_use] + pub fn writable_indexes(&self) -> &[u8] { + return self.writable_indexes.as_slice(); + } + + /// Returns ordered readonly lookup indexes. + #[must_use] + pub fn readonly_indexes(&self) -> &[u8] { + return self.readonly_indexes.as_slice(); + } +} + +impl std::fmt::Debug for crate::RawSolanaAddressTableLookup { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + return formatter + .debug_struct("RawSolanaAddressTableLookup") + .field("writable_index_count", &self.writable_indexes.len()) + .field("readonly_index_count", &self.readonly_indexes.len()) + .finish_non_exhaustive(); + } +} + +/// Optional inline budget configuration carried by Solana Transaction V1. +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] +pub struct RawSolanaTransactionConfig { + priority_fee: std::option::Option, + compute_unit_limit: std::option::Option, + loaded_accounts_data_size_limit: std::option::Option, + heap_size: std::option::Option, +} + +impl crate::RawSolanaTransactionConfig { + /// Creates an explicit Transaction V1 configuration, including the meaningful all-`None` case. + #[must_use] + pub const fn new( + priority_fee: std::option::Option, + compute_unit_limit: std::option::Option, + loaded_accounts_data_size_limit: std::option::Option, + heap_size: std::option::Option, + ) -> Self { + return Self { priority_fee, compute_unit_limit, loaded_accounts_data_size_limit, heap_size }; + } + + /// Returns the optional priority fee. + #[must_use] + pub const fn priority_fee(self) -> std::option::Option { + return self.priority_fee; + } + + /// Returns the optional compute-unit limit. + #[must_use] + pub const fn compute_unit_limit(self) -> std::option::Option { + return self.compute_unit_limit; + } + + /// Returns the optional loaded-account-data-size limit. + #[must_use] + pub const fn loaded_accounts_data_size_limit(self) -> std::option::Option { + return self.loaded_accounts_data_size_limit; + } + + /// Returns the optional heap-size override. + #[must_use] + pub const fn heap_size(self) -> std::option::Option { + return self.heap_size; + } +} + +/// Complete source-neutral Solana transaction message required for exact wire serialization. +#[derive(Clone, Eq, PartialEq)] +pub struct RawSolanaTransactionMessage { + version: crate::RawSolanaMessageVersion, + header: crate::RawSolanaMessageHeader, + account_keys: std::vec::Vec<[u8; 32]>, + recent_blockhash: [u8; 32], + instructions: std::vec::Vec, + address_table_lookups: std::vec::Vec, + config: std::option::Option, +} + +impl crate::RawSolanaTransactionMessage { + /// Creates one exact source-neutral message. + #[must_use] + pub fn new( + version: crate::RawSolanaMessageVersion, + header: crate::RawSolanaMessageHeader, + account_keys: std::vec::Vec<[u8; 32]>, + recent_blockhash: [u8; 32], + instructions: std::vec::Vec, + address_table_lookups: std::vec::Vec, + config: std::option::Option, + ) -> Self { + return Self { version, header, account_keys, recent_blockhash, instructions, address_table_lookups, config }; + } + + /// Returns the message version. + #[must_use] + pub const fn version(&self) -> crate::RawSolanaMessageVersion { + return self.version; + } + + /// Returns the message header. + #[must_use] + pub const fn header(&self) -> crate::RawSolanaMessageHeader { + return self.header; + } + + /// Returns ordered static account key bytes. + #[must_use] + pub fn account_keys(&self) -> &[[u8; 32]] { + return self.account_keys.as_slice(); + } + + /// Returns the recent blockhash bytes. + #[must_use] + pub const fn recent_blockhash(&self) -> &[u8; 32] { + return &self.recent_blockhash; + } + + /// Returns ordered compiled instructions. + #[must_use] + pub fn instructions(&self) -> &[crate::RawSolanaCompiledInstruction] { + return self.instructions.as_slice(); + } + + /// Returns ordered v0 address-table lookups. + #[must_use] + pub fn address_table_lookups(&self) -> &[crate::RawSolanaAddressTableLookup] { + return self.address_table_lookups.as_slice(); + } + + /// Returns optional V1 inline configuration. + #[must_use] + pub const fn config(&self) -> std::option::Option { + return self.config; + } +} + +impl std::fmt::Debug for crate::RawSolanaTransactionMessage { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + return formatter + .debug_struct("RawSolanaTransactionMessage") + .field("version", &self.version) + .field("header", &self.header) + .field("account_key_count", &self.account_keys.len()) + .field("instruction_count", &self.instructions.len()) + .field("address_table_lookup_count", &self.address_table_lookups.len()) + .field("has_config", &self.config.is_some()) + .finish(); + } +} + +/// Complete source-neutral Solana transaction wire material. +#[derive(Clone, Eq, PartialEq)] +pub struct RawSolanaTransactionWire { + signatures: std::vec::Vec<[u8; 64]>, + message: crate::RawSolanaTransactionMessage, +} + +impl crate::RawSolanaTransactionWire { + /// Creates one complete transaction wire. + #[must_use] + pub fn new(signatures: std::vec::Vec<[u8; 64]>, message: crate::RawSolanaTransactionMessage) -> Self { + return Self { signatures, message }; + } + + /// Returns ordered signature bytes. + #[must_use] + pub fn signatures(&self) -> &[[u8; 64]] { + return self.signatures.as_slice(); + } + + /// Returns the exact source-neutral message. + #[must_use] + pub const fn message(&self) -> &crate::RawSolanaTransactionMessage { + return &self.message; + } +} + +impl std::fmt::Debug for crate::RawSolanaTransactionWire { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + return formatter.debug_struct("RawSolanaTransactionWire").field("signature_count", &self.signatures.len()).field("message", &self.message).finish(); + } +} + +/// Serializes one source-neutral Solana transaction to exact canonical wire bytes. +pub fn serialize_solana_transaction_wire(transaction: &crate::RawSolanaTransactionWire) -> ksp_core_lib::Result> { + if let std::result::Result::Err(error) = validate_transaction(transaction) { + return std::result::Result::Err(error); + } + let mut output = std::vec::Vec::new(); + match transaction.message().version() { + crate::RawSolanaMessageVersion::Legacy | crate::RawSolanaMessageVersion::V0 => { + if let std::result::Result::Err(error) = encode_short_vec(transaction.signatures().len(), &mut output) { + return std::result::Result::Err(error); + } + for signature in transaction.signatures() { + output.extend_from_slice(signature); + } + if transaction.message().version() == crate::RawSolanaMessageVersion::V0 { + output.push(0x80); + } + if let std::result::Result::Err(error) = encode_legacy_or_v0_message(transaction.message(), &mut output) { + return std::result::Result::Err(error); + } + }, + crate::RawSolanaMessageVersion::V1 => { + if let std::result::Result::Err(error) = encode_v1_message(transaction, &mut output) { + return std::result::Result::Err(error); + } + }, + } + if output.len() > ksp_store_api::MAX_RAW_PAYLOAD_BYTES { + return std::result::Result::Err(crate::material_error("wire.bytes").with_context("actual_len", output.len().to_string())); + } + if transaction.message().version() == crate::RawSolanaMessageVersion::V1 && output.len() > MAX_V1_TRANSACTION_BYTES { + return std::result::Result::Err(crate::material_error("wire.bytes").with_context("actual_len", output.len().to_string())); + } + return std::result::Result::Ok(output); +} + +/// Serializes one source-neutral Solana transaction and returns canonical padded standard Base64. +pub fn serialize_solana_transaction_wire_base64(transaction: &crate::RawSolanaTransactionWire) -> ksp_core_lib::Result { + let bytes = match crate::serialize_solana_transaction_wire(transaction) { + std::result::Result::Ok(value) => value, + std::result::Result::Err(error) => return std::result::Result::Err(error), + }; + return std::result::Result::Ok(base64::engine::general_purpose::STANDARD.encode(bytes)); +} + +fn validate_transaction(transaction: &crate::RawSolanaTransactionWire) -> ksp_core_lib::Result<()> { + let message = transaction.message(); + let required_signatures = usize::from(message.header().num_required_signatures()); + if required_signatures == 0 || transaction.signatures().len() != required_signatures { + return std::result::Result::Err(crate::material_error("wire")); + } + if usize::from(message.header().num_readonly_signed_accounts()) >= required_signatures { + return std::result::Result::Err(crate::material_error("wire")); + } + if message.account_keys().len() < required_signatures + usize::from(message.header().num_readonly_unsigned_accounts()) { + return std::result::Result::Err(crate::material_error("wire")); + } + let loaded_count = message.address_table_lookups().iter().fold(0_usize, |count, lookup| { + return count.saturating_add(lookup.writable_indexes().len()).saturating_add(lookup.readonly_indexes().len()); + }); + let account_count = message.account_keys().len().saturating_add(loaded_count); + for instruction in message.instructions() { + if usize::from(instruction.program_id_index()) >= account_count { + return std::result::Result::Err(crate::material_error("wire")); + } + for account_index in instruction.accounts() { + if usize::from(*account_index) >= account_count { + return std::result::Result::Err(crate::material_error("wire")); + } + } + } + match message.version() { + crate::RawSolanaMessageVersion::Legacy => { + if !message.address_table_lookups().is_empty() || message.config().is_some() { + return std::result::Result::Err(crate::material_error("wire")); + } + }, + crate::RawSolanaMessageVersion::V0 => { + if message.config().is_some() { + return std::result::Result::Err(crate::material_error("wire")); + } + }, + crate::RawSolanaMessageVersion::V1 => { + if message.config().is_none() || !message.address_table_lookups().is_empty() { + return std::result::Result::Err(crate::material_error("wire")); + } + if transaction.signatures().len() > MAX_V1_SIGNATURE_COUNT + || message.account_keys().len() > MAX_V1_ADDRESS_COUNT + || message.instructions().len() > MAX_V1_INSTRUCTION_COUNT + { + return std::result::Result::Err(crate::material_error("wire")); + } + for (index, account) in message.account_keys().iter().enumerate() { + if message.account_keys()[..index].contains(account) { + return std::result::Result::Err(crate::material_error("wire")); + } + } + for instruction in message.instructions() { + if instruction.accounts().len() > usize::from(u8::MAX) || instruction.data().len() > usize::from(u16::MAX) { + return std::result::Result::Err(crate::material_error("wire")); + } + } + if let std::option::Option::Some(heap_size) = message.config().and_then(crate::RawSolanaTransactionConfig::heap_size) { + if !(32_768..=262_144).contains(&heap_size) || heap_size % 1_024 != 0 { + return std::result::Result::Err(crate::material_error("wire")); + } + } + }, + } + return std::result::Result::Ok(()); +} + +fn encode_legacy_or_v0_message(message: &crate::RawSolanaTransactionMessage, output: &mut std::vec::Vec) -> ksp_core_lib::Result<()> { + encode_header(message.header(), output); + if let std::result::Result::Err(error) = encode_short_vec(message.account_keys().len(), output) { + return std::result::Result::Err(error); + } + for account_key in message.account_keys() { + output.extend_from_slice(account_key); + } + output.extend_from_slice(message.recent_blockhash()); + if let std::result::Result::Err(error) = encode_short_vec(message.instructions().len(), output) { + return std::result::Result::Err(error); + } + for instruction in message.instructions() { + output.push(instruction.program_id_index()); + if let std::result::Result::Err(error) = encode_short_vec(instruction.accounts().len(), output) { + return std::result::Result::Err(error); + } + output.extend_from_slice(instruction.accounts()); + if let std::result::Result::Err(error) = encode_short_vec(instruction.data().len(), output) { + return std::result::Result::Err(error); + } + output.extend_from_slice(instruction.data()); + } + if message.version() == crate::RawSolanaMessageVersion::V0 { + if let std::result::Result::Err(error) = encode_short_vec(message.address_table_lookups().len(), output) { + return std::result::Result::Err(error); + } + for lookup in message.address_table_lookups() { + output.extend_from_slice(lookup.account_key()); + if let std::result::Result::Err(error) = encode_short_vec(lookup.writable_indexes().len(), output) { + return std::result::Result::Err(error); + } + output.extend_from_slice(lookup.writable_indexes()); + if let std::result::Result::Err(error) = encode_short_vec(lookup.readonly_indexes().len(), output) { + return std::result::Result::Err(error); + } + output.extend_from_slice(lookup.readonly_indexes()); + } + } + return std::result::Result::Ok(()); +} + +fn encode_v1_message(transaction: &crate::RawSolanaTransactionWire, output: &mut std::vec::Vec) -> ksp_core_lib::Result<()> { + let message = transaction.message(); + let config = match message.config() { + std::option::Option::Some(value) => value, + std::option::Option::None => return std::result::Result::Err(crate::material_error("wire")), + }; + output.push(0x81); + encode_header(message.header(), output); + let mut mask = 0_u32; + if config.priority_fee().is_some() { + mask |= 0b11; + } + if config.compute_unit_limit().is_some() { + mask |= 1_u32 << 2; + } + if config.loaded_accounts_data_size_limit().is_some() { + mask |= 1_u32 << 3; + } + if config.heap_size().is_some() { + mask |= 1_u32 << 4; + } + output.extend_from_slice(&mask.to_le_bytes()); + output.extend_from_slice(message.recent_blockhash()); + output.push(message.instructions().len() as u8); + output.push(message.account_keys().len() as u8); + for account_key in message.account_keys() { + output.extend_from_slice(account_key); + } + if let std::option::Option::Some(value) = config.priority_fee() { + output.extend_from_slice(&value.to_le_bytes()); + } + if let std::option::Option::Some(value) = config.compute_unit_limit() { + output.extend_from_slice(&value.to_le_bytes()); + } + if let std::option::Option::Some(value) = config.loaded_accounts_data_size_limit() { + output.extend_from_slice(&value.to_le_bytes()); + } + if let std::option::Option::Some(value) = config.heap_size() { + output.extend_from_slice(&value.to_le_bytes()); + } + for instruction in message.instructions() { + output.push(instruction.program_id_index()); + output.push(instruction.accounts().len() as u8); + output.extend_from_slice(&(instruction.data().len() as u16).to_le_bytes()); + } + for instruction in message.instructions() { + output.extend_from_slice(instruction.accounts()); + output.extend_from_slice(instruction.data()); + } + for signature in transaction.signatures() { + output.extend_from_slice(signature); + } + return std::result::Result::Ok(()); +} + +fn encode_header(header: crate::RawSolanaMessageHeader, output: &mut std::vec::Vec) { + output.push(header.num_required_signatures()); + output.push(header.num_readonly_signed_accounts()); + output.push(header.num_readonly_unsigned_accounts()); +} + +fn encode_short_vec(value: usize, output: &mut std::vec::Vec) -> ksp_core_lib::Result<()> { + if value > MAX_LEGACY_SHORT_VECTOR_VALUE { + return std::result::Result::Err(crate::material_error("wire")); + } + let mut remaining = value; + loop { + let mut byte = (remaining & 0x7f) as u8; + remaining >>= 7; + if remaining != 0 { + byte |= 0x80; + } + output.push(byte); + if remaining == 0 { + break; + } + } + return std::result::Result::Ok(()); +} + +#[cfg(test)] +#[path = "../unit_tests/wire.rs"] +mod tests; diff --git a/crates/ksp-raw-transaction-lib/tests/dependency_boundary.rs b/crates/ksp-raw-transaction-lib/tests/dependency_boundary.rs index 78126aa..ef22748 100644 --- a/crates/ksp-raw-transaction-lib/tests/dependency_boundary.rs +++ b/crates/ksp-raw-transaction-lib/tests/dependency_boundary.rs @@ -1,5 +1,5 @@ // file: crates/ksp-raw-transaction-lib/tests/dependency_boundary.rs -// version: 3 +// version: 4 //! Dependency-boundary canaries for the common RAW Transaction foundation. @@ -10,7 +10,7 @@ fn manifest() -> std::string::String { fn production_sources() -> std::string::String { let root = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("src"); let mut combined = std::string::String::new(); - for name in ["acquisition.rs", "canonical.rs", "error.rs", "lib.rs", "signature.rs"] { + for name in ["acquisition.rs", "canonical.rs", "error.rs", "lib.rs", "signature.rs", "wire.rs"] { let source = std::fs::read_to_string(root.join(name)).unwrap_or_default(); combined.push_str(source.as_str()); } diff --git a/crates/ksp-raw-transaction-lib/tests/public_api.rs b/crates/ksp-raw-transaction-lib/tests/public_api.rs index c04fa80..7ae22ce 100644 --- a/crates/ksp-raw-transaction-lib/tests/public_api.rs +++ b/crates/ksp-raw-transaction-lib/tests/public_api.rs @@ -1,5 +1,5 @@ // file: crates/ksp-raw-transaction-lib/tests/public_api.rs -// version: 3 +// version: 4 //! Integration canaries for the public common RAW Transaction crate-root surface. @@ -99,3 +99,24 @@ fn pre_004_embedded_signature_and_block_material_contract_are_available_from_cra } return; } + +#[test] +fn pre_006_source_neutral_wire_contract_is_available_from_crate_root() { + let message = ksp_raw_transaction_lib::RawSolanaTransactionMessage::new( + ksp_raw_transaction_lib::RawSolanaMessageVersion::V1, + ksp_raw_transaction_lib::RawSolanaMessageHeader::new(1, 0, 0), + vec![[1_u8; 32]], + [2_u8; 32], + vec![ksp_raw_transaction_lib::RawSolanaCompiledInstruction::new(0, vec![0], vec![3])], + vec![], + std::option::Option::Some(ksp_raw_transaction_lib::RawSolanaTransactionConfig::default()), + ); + let wire = ksp_raw_transaction_lib::RawSolanaTransactionWire::new(vec![[4_u8; 64]], message); + let encoded = ksp_raw_transaction_lib::serialize_solana_transaction_wire_base64(&wire); + assert!(encoded.is_ok()); + if let std::result::Result::Ok(encoded) = encoded { + assert!(encoded.starts_with("gQ")); + assert!(ksp_raw_transaction_lib::extract_raw_transaction_signature_from_binary_base64(encoded.as_str()).is_ok()); + } + return; +} diff --git a/crates/ksp-raw-transaction-lib/tests/release_completeness.rs b/crates/ksp-raw-transaction-lib/tests/release_completeness.rs index 721ba34..457783d 100644 --- a/crates/ksp-raw-transaction-lib/tests/release_completeness.rs +++ b/crates/ksp-raw-transaction-lib/tests/release_completeness.rs @@ -1,5 +1,5 @@ // file: crates/ksp-raw-transaction-lib/tests/release_completeness.rs -// version: 2 +// version: 3 //! Release-completeness canaries for the common RAW Transaction foundation. @@ -22,7 +22,7 @@ fn pre_002_production_module_inventory_is_exact() { names.push(name.to_owned()); } names.sort(); - let expected = ["acquisition.rs", "canonical.rs", "error.rs", "lib.rs", "signature.rs"]; + let expected = ["acquisition.rs", "canonical.rs", "error.rs", "lib.rs", "signature.rs", "wire.rs"]; assert_eq!(names.len(), expected.len()); for (actual, expected_name) in names.iter().zip(expected.iter()) { assert_eq!(actual.as_str(), *expected_name); diff --git a/crates/ksp-raw-transaction-lib/unit_tests/signature.rs b/crates/ksp-raw-transaction-lib/unit_tests/signature.rs index d990416..a66d5e1 100644 --- a/crates/ksp-raw-transaction-lib/unit_tests/signature.rs +++ b/crates/ksp-raw-transaction-lib/unit_tests/signature.rs @@ -1,5 +1,7 @@ // file: crates/ksp-raw-transaction-lib/unit_tests/signature.rs -// version: 2 +// version: 3 + +use base64::Engine; // rust-rules: trait-import #[test] fn pre_002_signature_parser_accepts_exact_sixty_four_zero_bytes() { @@ -60,3 +62,20 @@ fn pre_004_embedded_signature_rejects_invalid_base64_noncanonical_short_vec_and_ } return; } + +#[test] +fn pre_006_embedded_signature_extracts_terminal_v1_signature_and_rejects_trailing_bytes() { + let encoded = "gQEAAR8AAABRUVFRUVFRUVFRUVFRUVFRUVFRUVFRUVFRUVFRUVFRUQECQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQgcAAAAAAAAACAAAAAkAAAAAgAAAAQEDAAABAgOqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqq"; + let signature = crate::extract_raw_transaction_signature_from_binary_base64(encoded); + assert!(signature.is_ok()); + if let std::result::Result::Ok(signature) = signature { + assert_eq!(signature.as_bytes(), &[0xaa_u8; 64]); + } + let decoded = base64::engine::general_purpose::STANDARD.decode(encoded); + assert!(decoded.is_ok()); + if let std::result::Result::Ok(mut decoded) = decoded { + decoded.push(0); + let trailing = base64::engine::general_purpose::STANDARD.encode(decoded); + assert!(crate::extract_raw_transaction_signature_from_binary_base64(trailing.as_str()).is_err()); + } +} diff --git a/crates/ksp-raw-transaction-lib/unit_tests/wire.rs b/crates/ksp-raw-transaction-lib/unit_tests/wire.rs new file mode 100644 index 0000000..88fd968 --- /dev/null +++ b/crates/ksp-raw-transaction-lib/unit_tests/wire.rs @@ -0,0 +1,122 @@ +// file: crates/ksp-raw-transaction-lib/unit_tests/wire.rs +// version: 1 + +const LEGACY_GOLDEN_BASE64: &str = "AREREREREREREREREREREREREREREREREREREREREREREREREREREREREREREREREREREREREREREREREREREREBAAECISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIjExMTExMTExMTExMTExMTExMTExMTExMTExMTExMTExAQEBAAKquw=="; +const V0_GOLDEN_BASE64: &str = "ARISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhISEhKAAQAAASMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIBAAEBAcwBJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQBAAA="; +const V1_EMPTY_CONFIG_GOLDEN_BASE64: &str = "gQEAAAAAAABSUlJSUlJSUlJSUlJSUlJSUlJSUlJSUlJSUlJSUlJSUgABQ0NDQ0NDQ0NDQ0NDQ0NDQ0NDQ0NDQ0NDQ0NDQ0NDQ0Orq6urq6urq6urq6urq6urq6urq6urq6urq6urq6urq6urq6urq6urq6urq6urq6urq6urq6urq6urq6urq6ur"; +const V1_GOLDEN_BASE64: &str = "gQEAAR8AAABRUVFRUVFRUVFRUVFRUVFRUVFRUVFRUVFRUVFRUVFRUQECQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQgcAAAAAAAAACAAAAAkAAAAAgAAAAQEDAAABAgOqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqq"; + +fn legacy_wire() -> crate::RawSolanaTransactionWire { + let message = crate::RawSolanaTransactionMessage::new( + crate::RawSolanaMessageVersion::Legacy, + crate::RawSolanaMessageHeader::new(1, 0, 1), + vec![[0x21_u8; 32], [0x22_u8; 32]], + [0x31_u8; 32], + vec![crate::RawSolanaCompiledInstruction::new(1, vec![0], vec![0xaa, 0xbb])], + vec![], + std::option::Option::None, + ); + return crate::RawSolanaTransactionWire::new(vec![[0x11_u8; 64]], message); +} + +fn v0_wire() -> crate::RawSolanaTransactionWire { + let message = crate::RawSolanaTransactionMessage::new( + crate::RawSolanaMessageVersion::V0, + crate::RawSolanaMessageHeader::new(1, 0, 0), + vec![[0x23_u8; 32]], + [0x32_u8; 32], + vec![crate::RawSolanaCompiledInstruction::new(0, vec![1], vec![0xcc])], + vec![crate::RawSolanaAddressTableLookup::new([0x24_u8; 32], vec![0], vec![])], + std::option::Option::None, + ); + return crate::RawSolanaTransactionWire::new(vec![[0x12_u8; 64]], message); +} + +fn v1_wire() -> crate::RawSolanaTransactionWire { + let message = crate::RawSolanaTransactionMessage::new( + crate::RawSolanaMessageVersion::V1, + crate::RawSolanaMessageHeader::new(1, 0, 1), + vec![[0x41_u8; 32], [0x42_u8; 32]], + [0x51_u8; 32], + vec![crate::RawSolanaCompiledInstruction::new(1, vec![0], vec![1, 2, 3])], + vec![], + std::option::Option::Some(crate::RawSolanaTransactionConfig::new( + std::option::Option::Some(7), + std::option::Option::Some(8), + std::option::Option::Some(9), + std::option::Option::Some(32_768), + )), + ); + return crate::RawSolanaTransactionWire::new(vec![[0xaa_u8; 64]], message); +} + +fn v1_empty_config_wire() -> crate::RawSolanaTransactionWire { + let message = crate::RawSolanaTransactionMessage::new( + crate::RawSolanaMessageVersion::V1, + crate::RawSolanaMessageHeader::new(1, 0, 0), + vec![[0x43_u8; 32]], + [0x52_u8; 32], + vec![], + vec![], + std::option::Option::Some(crate::RawSolanaTransactionConfig::default()), + ); + return crate::RawSolanaTransactionWire::new(vec![[0xab_u8; 64]], message); +} + +#[test] +fn pre_006_legacy_v0_and_v1_wire_goldens_are_exact() { + let legacy = crate::serialize_solana_transaction_wire_base64(&legacy_wire()); + let v0 = crate::serialize_solana_transaction_wire_base64(&v0_wire()); + let v1 = crate::serialize_solana_transaction_wire_base64(&v1_wire()); + let v1_empty = crate::serialize_solana_transaction_wire_base64(&v1_empty_config_wire()); + assert_eq!(legacy.as_deref(), std::result::Result::Ok(LEGACY_GOLDEN_BASE64)); + assert_eq!(v0.as_deref(), std::result::Result::Ok(V0_GOLDEN_BASE64)); + assert_eq!(v1.as_deref(), std::result::Result::Ok(V1_GOLDEN_BASE64)); + assert_eq!(v1_empty.as_deref(), std::result::Result::Ok(V1_EMPTY_CONFIG_GOLDEN_BASE64)); + let v1_bytes = crate::serialize_solana_transaction_wire(&v1_wire()); + assert!(v1_bytes.is_ok()); + if let std::result::Result::Ok(v1_bytes) = v1_bytes { + assert_eq!(v1_bytes.len(), 198); + assert_eq!(v1_bytes.first(), std::option::Option::Some(&0x81)); + assert_eq!(v1_bytes.get(v1_bytes.len() - 64..), std::option::Option::Some([0xaa_u8; 64].as_slice())); + } +} + +#[test] +fn pre_006_v1_structural_guards_reject_alt_duplicates_and_missing_config() { + let mut missing_config = v1_wire(); + missing_config.message.config = std::option::Option::None; + assert!(crate::serialize_solana_transaction_wire(&missing_config).is_err()); + let mut with_lookup = v1_wire(); + with_lookup.message.address_table_lookups.push(crate::RawSolanaAddressTableLookup::new([9_u8; 32], vec![], vec![])); + assert!(crate::serialize_solana_transaction_wire(&with_lookup).is_err()); + let mut duplicate = v1_wire(); + duplicate.message.account_keys[1] = duplicate.message.account_keys[0]; + assert!(crate::serialize_solana_transaction_wire(&duplicate).is_err()); +} + +#[test] +fn pre_006_v1_program_index_zero_is_valid_and_bounds_remain_enforced() { + let message = crate::RawSolanaTransactionMessage::new( + crate::RawSolanaMessageVersion::V1, + crate::RawSolanaMessageHeader::new(1, 0, 0), + vec![[1_u8; 32]], + [2_u8; 32], + vec![crate::RawSolanaCompiledInstruction::new(0, vec![0], vec![])], + vec![], + std::option::Option::Some(crate::RawSolanaTransactionConfig::default()), + ); + let wire = crate::RawSolanaTransactionWire::new(vec![[3_u8; 64]], message); + assert!(crate::serialize_solana_transaction_wire(&wire).is_ok()); + let too_many_addresses = crate::RawSolanaTransactionMessage::new( + crate::RawSolanaMessageVersion::V1, + crate::RawSolanaMessageHeader::new(1, 0, 0), + (0_u8..65_u8).map(|value| return [value; 32]).collect(), + [2_u8; 32], + vec![], + vec![], + std::option::Option::Some(crate::RawSolanaTransactionConfig::default()), + ); + let wire = crate::RawSolanaTransactionWire::new(vec![[3_u8; 64]], too_many_addresses); + assert!(crate::serialize_solana_transaction_wire(&wire).is_err()); +} diff --git a/deltas/0.3.10/pre.006.md b/deltas/0.3.10/pre.006.md new file mode 100644 index 0000000..65db425 --- /dev/null +++ b/deltas/0.3.10/pre.006.md @@ -0,0 +1,206 @@ + + + +# Delta `0.3.10-pre.006` — Transaction V1 + parité Yellowstone transaction/block + +## Base requise + +```text +livraison : 0.3.10-pre.005-fix.002 +workspace.package.version = 0.3.10-pre.5.fix.2 +``` + +Le gate opérateur communiqué le 2026-09-06 ferme cette baseline : audits statiques propres, `cargo check --workspace` PASS, Clippy strict PASS, common RAW PASS, 387 tests Transport avec suites publiques/release/doctests PASS, et suite Backfill complète PASS, y compris hardening et les deux canaris WS/Helius. + +## Objectif + +Fermer avant le Worker concret : + +```text +Transaction V1 / SIMD-0385 dans la common source-neutral +Yellowstone Transaction -> transaction wire exact +Yellowstone Block -> même transaction wire + block_time +qualification RAW-direct vs hydration +``` + +Aucun adapter productif `Transport DTO -> common` n'est ajouté. `TR-C2` reste Worker-owned et `pre.007` reste le premier point où la crate Worker concrète peut apparaître. + +## Version + +```text +workspace.package.version = 0.3.10-pre.6 +Cargo.toml header = 499 +workspace members = 20 +``` + +## Common wire Legacy/V0/V1 + +`ksp-raw-transaction-lib` ajoute un modèle source-neutral : + +```text +RawSolanaMessageVersion +RawSolanaMessageHeader +RawSolanaCompiledInstruction +RawSolanaAddressTableLookup +RawSolanaTransactionConfig +RawSolanaTransactionMessage +RawSolanaTransactionWire +``` + +et deux serializers : + +```text +serialize_solana_transaction_wire +serialize_solana_transaction_wire_base64 +``` + +Legacy et V0 conservent le layout signatures-first existant. V1 suit SIMD-0385 : `0x81`, header, config mask, lifetime, compteurs, adresses, valeurs config, headers/payloads d'instructions et signatures terminales. + +Les gardes V1 verrouillent notamment : + +```text +transaction <= 4096 bytes +1..=12 signatures +<= 64 adresses +<= 64 instructions +num_readonly_signed < num_required_signatures +indexes programme/comptes bornés +aucune Address Lookup Table +aucune adresse dupliquée +config présente, y compris vide +aucun trailing byte après les signatures +``` + +Les goldens couvrent Legacy, V0, V1 configuration complète et V1 configuration vide. + +## Extraction de signature V1 + +`extract_raw_transaction_signature_from_binary_base64` distingue désormais : + +```text +premier octet == 0x81 -> parser V1 et signatures terminales +sinon -> parser short-vector signatures-first Legacy/V0 +``` + +Le parser V1 parcourt structurellement mask/config, adresses, headers, payloads et signatures et exige la fin exacte du wire. + +## Parité Yellowstone + +`crates/ksp-job-backfill-lib/tests/yellowstone_raw_parity.rs` monte un serveur Geyser local déterministe, ouvre le vrai `YellowstoneGrpcChannel::open_standard_subscribe` puis reçoit : + +```text +SubscribeUpdateTransaction V1 +SubscribeUpdateBlock V1 portant la même transaction +``` + +La projection vers `RawSolanaTransactionWire` est strictement test-only. Le canari exige le même wire V1 exact pour les deux updates et vérifie slot, transaction index et `block_time` côté Block. + +## Qualification RAW + +Le gate ferme volontairement une qualification partielle : + +```text +Yellowstone transaction wire = direct/fiable +Yellowstone Transaction RAW = hydration HTTP obligatoire : block_time absent +Yellowstone Block RAW complet= hydration HTTP obligatoire : meta JSON byte-identical non prouvée +``` + +Aucun champ manquant n'est inventé et aucune projection protobuf -> JSON ad hoc n'est introduite pour forcer un hash RAW identique. + +## Dependency firewall + +Le Backfill ajoute uniquement en dev : + +```text +tonic = { workspace = true, features = ["codegen", "server"] } +yellowstone-grpc-proto = { workspace = true, features = ["tonic"] } +``` + +`dependency_boundary.rs` verrouille leur absence des dépendances production et `hardening.rs` recalibre le set dev exact. Les dépendances normales restent inchangées. + +Il n'existe toujours aucun edge productif : + +```text +ksp-onchain-transport-lib -> ksp-raw-transaction-lib +ksp-raw-transaction-lib -> ksp-onchain-transport-lib +``` + +## Fraîcheur externe revalidée + +Revalidation le 2026-09-07 : + +```text +SIMD-0385 : Transaction V1, status Review, version byte 129, signatures terminales, sans ALT +Solana Transaction V1 examples : capacité Yellowstone avec Message.config ; activation cluster encore feature-gated/pre-release +``` + +Sources : + +```text +https://github.com/solana-foundation/solana-improvement-documents/blob/main/proposals/0385-transaction-v1.md +https://github.com/solana-foundation/transaction-v1-examples +https://github.com/rpcpool/yellowstone-grpc/blob/master/yellowstone-grpc-proto/proto/solana-storage.proto +``` + +KSP implémente la capacité de lecture/canonicalisation V1 sans déclarer son activation sur Mainnet ou un autre cluster. + +## Fichiers ajoutés + +```text +crates/ksp-job-backfill-lib/tests/yellowstone_raw_parity.rs +crates/ksp-raw-transaction-lib/src/wire.rs +crates/ksp-raw-transaction-lib/unit_tests/wire.rs +deltas/0.3.10/pre.006.md +``` + +## Fichiers modifiés + +```text +Cargo.toml +crates/ksp-job-backfill-lib/Cargo.toml +crates/ksp-job-backfill-lib/tests/dependency_boundary.rs +crates/ksp-job-backfill-lib/tests/hardening.rs +crates/ksp-raw-transaction-lib/src/lib.rs +crates/ksp-raw-transaction-lib/src/signature.rs +crates/ksp-raw-transaction-lib/tests/dependency_boundary.rs +crates/ksp-raw-transaction-lib/tests/public_api.rs +crates/ksp-raw-transaction-lib/tests/release_completeness.rs +crates/ksp-raw-transaction-lib/unit_tests/signature.rs +docs/architecture/011-RAW_TRANSACTION_ACQUISITION.md +docs/plans/031-V0_3_10_RAW_TRANSACTION_INGEST_PLAN.md +docs/validation/027-V0_3_10_RAW_TRANSACTION_INGEST.md +``` + +## Fichiers supprimés + +Aucun. + +## Gate opérateur demandé + +```bash +cargo fmt --all +python3 scripts/audit_rust_workspace_rules.py +python3 scripts/audit_markdown_tables.py README.md RULES.md ROADMAP.md CHANGELOG.md docs prompts crates deltas +cargo check --workspace +cargo clippy --workspace --all-targets --all-features -- -D warnings +cargo test -p ksp-raw-transaction-lib +cargo test -p ksp-onchain-transport-lib +cargo test -p ksp-job-backfill-lib +cargo tree -p ksp-job-backfill-lib --edges normal +cargo tree -p ksp-job-backfill-lib --edges dev +cargo tree -p ksp-job-backfill-lib -e features +``` + +Les arbres Cargo sont requis car `pre.006` ajoute deux dev-dependencies test-only. + +## Décisions prises + +```text +common sérialise Legacy/V0/V1 sans dépendre de Transport +V1 est reconnu par 0x81 et possède des signatures terminales +Yellowstone Transaction/Block conservent un transaction wire V1 exact sur canari +Yellowstone RAW complet reste hydraté par HTTP tant que TR-C4 meta/block_time complet n'est pas prouvé +aucun RAW v2 automatique +aucun Worker concret avant pre.007 +TR-C2 reste Worker-owned +``` diff --git a/docs/architecture/011-RAW_TRANSACTION_ACQUISITION.md b/docs/architecture/011-RAW_TRANSACTION_ACQUISITION.md index 2d691d6..54c3783 100644 --- a/docs/architecture/011-RAW_TRANSACTION_ACQUISITION.md +++ b/docs/architecture/011-RAW_TRANSACTION_ACQUISITION.md @@ -1,5 +1,5 @@ - + # Acquisition et alimentation `RawTransaction` @@ -665,6 +665,54 @@ aucun edge Transport <-> ksp-raw-transaction-lib La migration doit préserver exactement les golden bytes/hash RAW v1 déjà prouvés. Aucun RAW v2 n'est justifié. +### 13.1 Transaction wire Legacy/V0/V1 + +`pre.006` matérialise dans `ksp-raw-transaction-lib` un modèle wire Solana source-neutral, indépendant de Transport et des crates runtime Solana. Il sérialise exactement les formes Legacy, V0 et V1 puis expose aussi la représentation Base64. + +Pour V1, le contrat suit SIMD-0385 tel qu'audité le 7 septembre 2026 : + +```text +0x81 +LegacyHeader +TransactionConfigMask u32 LE +LifetimeSpecifier [32] +NumInstructions u8 +NumAddresses u8 +Addresses +ConfigValues +InstructionHeaders +InstructionPayloads +Signatures terminales +``` + +Les limites common verrouillées sont : transaction `<= 4096` octets, `<= 12` signatures, `<= 64` adresses, `<= 64` instructions, indexes bornés, aucune ALT en V1 et aucune donnée après les signatures. La présence de `Message.config` Yellowstone distingue V1 ; `versioned=true` sans config reste V0, et `versioned=false` sans config reste Legacy. + +L'extracteur de signature Base64 common reconnaît désormais V1 au premier octet `0x81` et parcourt structurellement le message jusqu'aux signatures terminales au lieu d'appliquer le short-vector signatures-first de Legacy/V0. + +Le format V1 est encore documenté upstream comme pre-release/feature-gated ; KSP implémente donc la capacité de lecture/canonicalisation sans déclarer son activation sur un cluster particulier. + +### 13.2 Qualification Yellowstone `pre.006` + +Un canari cross-layer test-only dans Backfill monte un serveur Geyser local, ouvre le vrai `YellowstoneGrpcChannel::open_standard_subscribe`, reçoit successivement une update `Transaction` V1 et une update `Block` contenant la même transaction, puis projette les DTO Transport vers le modèle wire common uniquement dans le test. + +Le gate prouve : + +```text +Transaction update -> slot/index + transaction wire V1 exact +Block update -> slot/index + même transaction wire V1 exact + block_time +Transport -> aucun edge production vers ksp-raw-transaction-lib +Backfill -> tonic/yellowstone-grpc-proto strictement dev-only +``` + +Il ne qualifie pas un RAW-direct complet Yellowstone : une update `Transaction` ne porte pas `block_time`, et la représentation protobuf Yellowstone de la meta n'est pas démontrée byte-identical avec la projection JSON HTTP servant au RAW v1 actuel. La décision `TR-C4` reste donc conservative : + +```text +Yellowstone Transaction/Block -> signal structuré + transaction wire fidèle +RAW v1 complet -> hydration HTTP avant persistence +``` + +Aucun adapter productif n'est ajouté en `pre.006`. Conformément à `TR-C2`, l'adapter `Transport DTO -> common` demeure réservé au Worker de `pre.007`. + ## 14. Handoff `0.3.10` : Worker live ### 14.1 Contrat fonctionnel @@ -805,7 +853,7 @@ Aucun code, DTO, Config, endpoint, secret, dépendance ou convention de version ## 18. Sources externes auditées -Sources consultées le **4 septembre 2026**. Cette liste consolide les anciens audits A/B dans un seul registre documentaire. +Sources consultées le **4 septembre 2026** et revalidées pour Transaction V1 le **7 septembre 2026**. Cette liste consolide les anciens audits A/B dans un seul registre documentaire. ### 18.1 Solana standard @@ -828,43 +876,46 @@ Sources consultées le **4 septembre 2026**. Cette liste consolide les anciens a ### 18.2 Yellowstone et providers -| Source | URL | -|----------------------------------|---------------------------------------------------------------------------------------------------| -| Yellowstone proto | https://github.com/rpcpool/yellowstone-grpc/blob/master/yellowstone-grpc-proto/proto/geyser.proto | -| Yellowstone README | https://github.com/rpcpool/yellowstone-grpc/blob/master/README.md | -| Yellowstone changelog | https://github.com/rpcpool/yellowstone-grpc/blob/master/CHANGELOG.md | -| Helius data streaming | https://www.helius.dev/docs/data-streaming | -| Helius LaserStream | https://www.helius.dev/blog/introducing-laserstream | -| Helius WebSockets | https://www.helius.dev/blog/laserstream-websockets | -| Helius enhanced WebSockets | https://www.helius.dev/blog/introducing-next-generation-enhanced-websockets | -| Helius getTransactionsForAddress | https://www.helius.dev/blog/introducing-gettransactionsforaddress | -| Helius preprocessed transactions | https://www.helius.dev/docs/shred-delivery/preprocessed-transactions | -| PublicNode Solana | https://solana.publicnode.com/ | -| OrbitFlare Yellowstone | https://docs.orbitflare.com/data-streaming/yellowstone | -| OrbitFlare RPC | https://orbitflare.com/products/rpc-nodes | -| OrbitFlare gRPC | https://orbitflare.com/products/solana-grpc | -| QuickNode Yellowstone | https://www.quicknode.com/guides/solana-development/tooling/solana-grpc/solana-grpc | -| QuickNode gRPC plans | https://www.quicknode.com/blog/solana-grpc-is-now-included-with-scale-and-business-plans | -| Alchemy Solana gRPC | https://www.alchemy.com/solana-grpc | -| Alchemy Yellowstone quickstart | https://www.alchemy.com/docs/reference/yellowstone-grpc-quickstart | -| Alchemy historical replay | https://www.alchemy.com/docs/reference/yellowstone-grpc-historical-replay | -| Alchemy compute/bandwidth | https://www.alchemy.com/docs/reference/compute-unit-costs | -| Chainstack Yellowstone | https://chainstack.com/yellowstone-grpc-more-streams-same-price/ | -| Shyft Yellowstone | https://shyft.to/solana-yellowstone-grpc | -| Shyft pricing | https://shyft.to/solana-rpc-grpc-pricing | -| Shyft RabbitStream | https://shyft.to/solana-shreds-rabbitstream | -| Triton pricing | https://triton.one/pricing | -| Triton Yellowstone | https://blog.triton.one/complete-guide-to-solana-streaming-and-yellowstone-grpc/ | -| Triton Fumarole | https://blog.triton.one/introducing-yellowstone-fumarole/ | -| Triton Deshred | https://blog.triton.one/deshred-transactions-the-fastest-path-to-solana-data/ | -| dRPC Yellowstone | https://drpc.org/docs/solana-yellowstone-geyser-grpc | -| Ankr Solana | https://www.ankr.com/docs/rpc-service/chains/chains-list/s-t/ | -| Ankr pricing | https://www.ankr.com/docs/rpc-service/pricing/ | -| GetBlock pricing | https://getblock.io/pricing-new/ | -| bloXroute pricing | https://bloxroute.com/pricing/ | -| Jito ShredStream | https://docs.jito.wtf/lowlatencytxnfeed/ | -| DoubleZero Edge | https://docs.malbeclabs.com/Edge%20Subscriber%20Connection/ | -| Old Faithful | https://github.com/rpcpool/yellowstone-faithful | +| Source | URL | +|----------------------------------|--------------------------------------------------------------------------------------------------------------| +| Yellowstone proto | https://github.com/rpcpool/yellowstone-grpc/blob/master/yellowstone-grpc-proto/proto/geyser.proto | +| Yellowstone storage proto | https://github.com/rpcpool/yellowstone-grpc/blob/master/yellowstone-grpc-proto/proto/solana-storage.proto | +| SIMD-0385 Transaction V1 | https://github.com/solana-foundation/solana-improvement-documents/blob/main/proposals/0385-transaction-v1.md | +| Transaction V1 examples | https://github.com/solana-foundation/transaction-v1-examples | +| Yellowstone README | https://github.com/rpcpool/yellowstone-grpc/blob/master/README.md | +| Yellowstone changelog | https://github.com/rpcpool/yellowstone-grpc/blob/master/CHANGELOG.md | +| Helius data streaming | https://www.helius.dev/docs/data-streaming | +| Helius LaserStream | https://www.helius.dev/blog/introducing-laserstream | +| Helius WebSockets | https://www.helius.dev/blog/laserstream-websockets | +| Helius enhanced WebSockets | https://www.helius.dev/blog/introducing-next-generation-enhanced-websockets | +| Helius getTransactionsForAddress | https://www.helius.dev/blog/introducing-gettransactionsforaddress | +| Helius preprocessed transactions | https://www.helius.dev/docs/shred-delivery/preprocessed-transactions | +| PublicNode Solana | https://solana.publicnode.com/ | +| OrbitFlare Yellowstone | https://docs.orbitflare.com/data-streaming/yellowstone | +| OrbitFlare RPC | https://orbitflare.com/products/rpc-nodes | +| OrbitFlare gRPC | https://orbitflare.com/products/solana-grpc | +| QuickNode Yellowstone | https://www.quicknode.com/guides/solana-development/tooling/solana-grpc/solana-grpc | +| QuickNode gRPC plans | https://www.quicknode.com/blog/solana-grpc-is-now-included-with-scale-and-business-plans | +| Alchemy Solana gRPC | https://www.alchemy.com/solana-grpc | +| Alchemy Yellowstone quickstart | https://www.alchemy.com/docs/reference/yellowstone-grpc-quickstart | +| Alchemy historical replay | https://www.alchemy.com/docs/reference/yellowstone-grpc-historical-replay | +| Alchemy compute/bandwidth | https://www.alchemy.com/docs/reference/compute-unit-costs | +| Chainstack Yellowstone | https://chainstack.com/yellowstone-grpc-more-streams-same-price/ | +| Shyft Yellowstone | https://shyft.to/solana-yellowstone-grpc | +| Shyft pricing | https://shyft.to/solana-rpc-grpc-pricing | +| Shyft RabbitStream | https://shyft.to/solana-shreds-rabbitstream | +| Triton pricing | https://triton.one/pricing | +| Triton Yellowstone | https://blog.triton.one/complete-guide-to-solana-streaming-and-yellowstone-grpc/ | +| Triton Fumarole | https://blog.triton.one/introducing-yellowstone-fumarole/ | +| Triton Deshred | https://blog.triton.one/deshred-transactions-the-fastest-path-to-solana-data/ | +| dRPC Yellowstone | https://drpc.org/docs/solana-yellowstone-geyser-grpc | +| Ankr Solana | https://www.ankr.com/docs/rpc-service/chains/chains-list/s-t/ | +| Ankr pricing | https://www.ankr.com/docs/rpc-service/pricing/ | +| GetBlock pricing | https://getblock.io/pricing-new/ | +| bloXroute pricing | https://bloxroute.com/pricing/ | +| Jito ShredStream | https://docs.jito.wtf/lowlatencytxnfeed/ | +| DoubleZero Edge | https://docs.malbeclabs.com/Edge%20Subscriber%20Connection/ | +| Old Faithful | https://github.com/rpcpool/yellowstone-faithful | Les URLs sont documentaires. Aucun endpoint runtime, token ou secret de compte n'est ajouté au dépôt par cet audit. @@ -883,6 +934,9 @@ support vs preuve = axes séparés ; NON PROUVÉ n'implique pas REJET provider model = capabilities/configuration, jamais enum provider fermé network identity = mainnet canonique ; mainnet-beta alias legacy/externe RAW canonicalisation = lower-layer commune, sans edge Job <-> Worker +Transaction V1 = wire source-neutral Legacy/V0/V1 dans common ; activation cluster non supposée +Yellowstone pre.006 = transaction wire V1 qualifié ; RAW complet via hydration HTTP +TR-C2 = adapter productif Transport DTO -> common réservé au Worker pre.007 0.3.10 = Worker live multi-source + adaptations communes nécessaires 0.3.12 = Job Backfill multi-stratégie historique ``` diff --git a/docs/plans/031-V0_3_10_RAW_TRANSACTION_INGEST_PLAN.md b/docs/plans/031-V0_3_10_RAW_TRANSACTION_INGEST_PLAN.md index ef7f8f5..b12dd34 100644 --- a/docs/plans/031-V0_3_10_RAW_TRANSACTION_INGEST_PLAN.md +++ b/docs/plans/031-V0_3_10_RAW_TRANSACTION_INGEST_PLAN.md @@ -1,5 +1,5 @@ - + # Plan v0.3.10 — RAW Transaction commune + Worker d’ingestion multi-source @@ -1080,7 +1080,19 @@ Le second rejeu opérateur de `pre.005-fix.001` révèle ensuite deux gardes de ### `pre.006` — parité Yellowstone transactions/blocks -Fermer les adapters structurés Yellowstone transaction/block, Transaction V1, meta et cross-source parity. Aucune persistance directe si le gate byte-identical échoue. +**Statut : réalisé, en attente du gate opérateur Rust.** + +La common RAW gagne un modèle transaction wire source-neutral Legacy/V0/V1 et des serializers exacts bytes/Base64. Les goldens couvrent Legacy, V0, V1 avec configuration complète et V1 avec configuration présente mais vide. L'extracteur de première signature Base64 traite désormais le layout V1 `0x81` à signatures terminales et conserve le chemin signatures-first Legacy/V0. + +Le modèle V1 verrouille les contraintes SIMD-0385 utiles à l'ingestion : 4096 octets maximum, 12 signatures, 64 adresses, 64 instructions, indexes bornés, aucune ALT, configuration inline et absence de trailing bytes après les signatures. La common reste Transport/Config/Job/Worker/runtime-free. + +Un canari `yellowstone_raw_parity.rs` monte un Geyser Tonic local et utilise le vrai chemin public Transport `YellowstoneGrpcChannel -> open_standard_subscribe -> YellowstoneSubscribeUpdate`. Une `Transaction` V1 puis un `Block` V1 portant la même transaction sont projetés test-only vers la common et doivent sérialiser exactement le même wire V1. Le canari vérifie aussi slot, transaction index et `block_time` côté Block. + +La qualification RAW-direct complète est refusée : l'update Transaction n'a pas `block_time`, et la meta Yellowstone protobuf n'est pas prouvée byte-identical avec la meta JSON HTTP du RAW v1 actuel. Le futur Worker doit donc utiliser Yellowstone comme signal live/transaction-wire riche puis hydrater par HTTP avant persistence RAW, tant qu'un gate ultérieur ne prouve pas `TR-C4`. + +`tonic` et `yellowstone-grpc-proto` sont ajoutés uniquement en `dev-dependencies` du Backfill pour le serveur déterministe ; les canaris dependency/hardening verrouillent leur absence du graphe productif. Aucun adapter productif n'est placé en Backfill ou Transport : `TR-C2` reste réservé au Worker `pre.007`. + +La fraicheur upstream a été revalidée le 7 septembre 2026 : SIMD-0385 décrit encore V1 comme `Review`/pre-release et les exemples Solana signalent une activation cluster feature-gated. KSP implémente donc le décodage/canonicalisation sans supposer l'activation Mainnet. ### `pre.007` — Worker foundation/runtime diff --git a/docs/validation/027-V0_3_10_RAW_TRANSACTION_INGEST.md b/docs/validation/027-V0_3_10_RAW_TRANSACTION_INGEST.md index fe441d8..b7292a1 100644 --- a/docs/validation/027-V0_3_10_RAW_TRANSACTION_INGEST.md +++ b/docs/validation/027-V0_3_10_RAW_TRANSACTION_INGEST.md @@ -1,5 +1,5 @@ - + # Validation v0.3.10 — RAW Transaction commune + Worker d’ingestion @@ -942,3 +942,50 @@ KSP workspace Rust rule audit: clean Markdown table audit: clean (339 table(s), 761 file(s)) ``` +### 13.10 Gate d'assemblage `pre.006` — Transaction V1 et Yellowstone + +Le rejeu opérateur de `pre.005-fix.002` communiqué le 6 septembre 2026 est intégralement vert : audits statiques, `cargo check --workspace`, Clippy strict, common RAW, 387 tests Transport avec suites publiques/release/doctests, et suite Backfill complète incluant hardening et parité WS/Helius. `pre.006` part donc de cette baseline fermée. + +La tranche ajoute dans `ksp-raw-transaction-lib` : + +```text +RawSolanaMessageVersion Legacy/V0/V1 +RawSolanaMessageHeader +RawSolanaCompiledInstruction +RawSolanaAddressTableLookup +RawSolanaTransactionConfig +RawSolanaTransactionMessage +RawSolanaTransactionWire +serialize_solana_transaction_wire +serialize_solana_transaction_wire_base64 +``` + +Les tests common imposent des goldens Base64 exacts Legacy/V0/V1, la configuration V1 vide significative, les limites structurelles V1, l'interdiction des ALT et l'extraction de signature depuis les signatures terminales. + +Le canari Backfill `tests/yellowstone_raw_parity.rs` utilise un serveur Geyser local et le vrai canal public Yellowstone Transport. Il exige : + +```text +Transaction V1 : slot = 42, index = 3, wire = golden common +Block V1 : slot = 42, index = 3, block_time présent, wire identique +meta : conservée comme DTO Yellowstone, sans prétendre à l'identité JSON HTTP +``` + +Décision de qualification : + +```text +Yellowstone transaction wire = QUALIFIÉ +Yellowstone Transaction RAW = HYDRATION HTTP (block_time absent) +Yellowstone Block RAW complet = HYDRATION HTTP (meta JSON byte-identical non prouvée) +TR-C2 adapter productif = différé au Worker pre.007 +``` + +Le manifest Backfill ajoute uniquement en dev : + +```text +tonic = { workspace = true, features = ["codegen", "server"] } +yellowstone-grpc-proto = { workspace = true, features = ["tonic"] } +``` + +`tests/dependency_boundary.rs` vérifie qu'elles ne figurent pas avant `[dev-dependencies]`, et le canari hardening attend exactement `{tokio, tokio-tungstenite, tonic, yellowstone-grpc-proto}` en dev tout en gardant l'inventaire normal inchangé. + +L'environnement d'assemblage ne fournit ni `cargo`, ni `rustc`, ni `rustfmt`. Le gate Cargo/Clippy/tests reste donc opérateur. Les audits Python KSP et Markdown sont exécutés après fermeture du delta et leurs résultats sont enregistrés dans `deltas/0.3.10/pre.006.md`.