// file: crates/ksp-job-backfill-lib/tests/yellowstone_raw_parity.rs // version: 3 //! Deterministic local Yellowstone gRPC parity canary for source-neutral RAW transaction v1 projection. struct FixtureStream { receiver: tokio::sync::mpsc::Receiver, } impl FixtureStream { fn new(receiver: tokio::sync::mpsc::Receiver) -> Self { return Self { receiver }; } } impl futures_util::Stream for FixtureStream { type Item = T; fn poll_next(self: std::pin::Pin<&mut Self>, context: &mut std::task::Context<'_>) -> std::task::Poll> { return self.get_mut().receiver.poll_recv(context); } } #[derive(Clone)] struct FixtureGeyser; #[allow(clippy::implicit_return)] // tonic::async_trait generates async wrapper tails outside the authored fixture bodies. #[tonic::async_trait] impl yellowstone_grpc_proto::geyser::geyser_server::Geyser for FixtureGeyser { type SubscribeStream = FixtureStream>; async fn subscribe( &self, request: tonic::Request>, ) -> std::result::Result, tonic::Status> { let mut inbound = request.into_inner(); let (sender, receiver) = tokio::sync::mpsc::channel(4); tokio::spawn(async move { let initial = inbound.message().await; if !matches!(initial, std::result::Result::Ok(std::option::Option::Some(_))) { return; } if sender.send(std::result::Result::Ok(transaction_update())).await.is_err() { return; } let _ = sender.send(std::result::Result::Ok(block_update())).await; }); return std::result::Result::Ok(tonic::Response::new(FixtureStream::new(receiver))); } type SubscribeDeshredStream = futures_util::stream::Empty>; async fn subscribe_deshred( &self, _request: tonic::Request>, ) -> std::result::Result, tonic::Status> { return std::result::Result::Err(tonic::Status::unimplemented("deshred is outside the pre.006 fixture")); } type SubscribeGossipStream = futures_util::stream::Empty>; async fn subscribe_gossip( &self, _request: tonic::Request, ) -> std::result::Result, tonic::Status> { return std::result::Result::Err(tonic::Status::unimplemented("gossip is outside the pre.006 fixture")); } async fn subscribe_replay_info( &self, _request: tonic::Request, ) -> std::result::Result, tonic::Status> { return std::result::Result::Err(tonic::Status::unimplemented("replay info is outside the pre.006 fixture")); } async fn ping( &self, _request: tonic::Request, ) -> std::result::Result, tonic::Status> { return std::result::Result::Err(tonic::Status::unimplemented("unary is outside the pre.006 fixture")); } async fn get_latest_blockhash( &self, _request: tonic::Request, ) -> std::result::Result, tonic::Status> { return std::result::Result::Err(tonic::Status::unimplemented("unary is outside the pre.006 fixture")); } async fn get_block_height( &self, _request: tonic::Request, ) -> std::result::Result, tonic::Status> { return std::result::Result::Err(tonic::Status::unimplemented("unary is outside the pre.006 fixture")); } async fn get_slot( &self, _request: tonic::Request, ) -> std::result::Result, tonic::Status> { return std::result::Result::Err(tonic::Status::unimplemented("unary is outside the pre.006 fixture")); } async fn is_blockhash_valid( &self, _request: tonic::Request, ) -> std::result::Result, tonic::Status> { return std::result::Result::Err(tonic::Status::unimplemented("unary is outside the pre.006 fixture")); } async fn get_version( &self, _request: tonic::Request, ) -> std::result::Result, tonic::Status> { return std::result::Result::Err(tonic::Status::unimplemented("unary is outside the pre.006 fixture")); } } struct FixtureServer { endpoint_url: std::string::String, shutdown: std::option::Option>, task: tokio::task::JoinHandle<()>, } impl FixtureServer { async fn start() -> std::result::Result { let bind_address = match "127.0.0.1:0".parse::() { std::result::Result::Ok(value) => value, std::result::Result::Err(_) => return std::result::Result::Err("fixture bind address must parse"), }; let incoming = match tonic::transport::server::TcpIncoming::bind(bind_address) { std::result::Result::Ok(value) => value, std::result::Result::Err(_) => return std::result::Result::Err("fixture gRPC listener must bind"), }; let local_address = match incoming.local_addr() { std::result::Result::Ok(value) => value, std::result::Result::Err(_) => return std::result::Result::Err("fixture gRPC listener must expose local address"), }; let (shutdown, shutdown_receiver) = tokio::sync::oneshot::channel(); let task = tokio::spawn(async move { let service = yellowstone_grpc_proto::geyser::geyser_server::GeyserServer::new(FixtureGeyser); let result = tonic::transport::Server::builder() .serve_with_incoming_shutdown(service, incoming, async move { let _ = shutdown_receiver.await; }) .await; assert!(result.is_ok()); }); return std::result::Result::Ok(Self { endpoint_url: format!("http://{local_address}"), shutdown: std::option::Option::Some(shutdown), task }); } async fn stop(mut self) { if let std::option::Option::Some(shutdown) = self.shutdown.take() { let _ = shutdown.send(()); } let result = self.task.await; assert!(result.is_ok()); } } fn transaction_info() -> yellowstone_grpc_proto::geyser::SubscribeUpdateTransactionInfo { return yellowstone_grpc_proto::geyser::SubscribeUpdateTransactionInfo { signature: vec![9_u8; 64], is_vote: false, transaction: std::option::Option::Some(yellowstone_grpc_proto::solana::storage::confirmed_block::Transaction { signatures: vec![vec![9_u8; 64]], message: std::option::Option::Some(yellowstone_grpc_proto::solana::storage::confirmed_block::Message { header: std::option::Option::Some(yellowstone_grpc_proto::solana::storage::confirmed_block::MessageHeader { num_required_signatures: 1, num_readonly_signed_accounts: 0, num_readonly_unsigned_accounts: 1, }), account_keys: vec![vec![1_u8; 32], vec![2_u8; 32]], recent_blockhash: vec![3_u8; 32], instructions: vec![yellowstone_grpc_proto::solana::storage::confirmed_block::CompiledInstruction { program_id_index: 1, accounts: vec![0_u8], data: vec![4_u8, 5, 6], }], versioned: true, address_table_lookups: vec![], config: std::option::Option::Some(yellowstone_grpc_proto::solana::storage::confirmed_block::TransactionConfig { priority_fee: std::option::Option::Some(7), compute_unit_limit: std::option::Option::Some(8), loaded_accounts_data_size_limit: std::option::Option::Some(9), heap_size: std::option::Option::Some(32_768), }), }), }), meta: std::option::Option::Some(yellowstone_grpc_proto::solana::storage::confirmed_block::TransactionStatusMeta { fee: 5_000, pre_balances: vec![100, 200], post_balances: vec![90, 210], compute_units_consumed: std::option::Option::Some(123), cost_units: std::option::Option::Some(456), ..std::default::Default::default() }), index: 3, }; } fn transaction_update() -> yellowstone_grpc_proto::geyser::SubscribeUpdate { return yellowstone_grpc_proto::geyser::SubscribeUpdate { filters: vec!["transactions-pre-006".to_owned()], update_oneof: std::option::Option::Some(yellowstone_grpc_proto::geyser::subscribe_update::UpdateOneof::Transaction( yellowstone_grpc_proto::geyser::SubscribeUpdateTransaction { transaction: std::option::Option::Some(transaction_info()), slot: 42 }, )), created_at: std::option::Option::None, }; } fn block_update() -> yellowstone_grpc_proto::geyser::SubscribeUpdate { return yellowstone_grpc_proto::geyser::SubscribeUpdate { filters: vec!["blocks-pre-006".to_owned()], update_oneof: std::option::Option::Some(yellowstone_grpc_proto::geyser::subscribe_update::UpdateOneof::Block( yellowstone_grpc_proto::geyser::SubscribeUpdateBlock { slot: 42, blockhash: ksp_core_lib::Pubkey::new_from_array([21_u8; 32]).to_string(), rewards: std::option::Option::None, block_time: std::option::Option::Some(yellowstone_grpc_proto::solana::storage::confirmed_block::UnixTimestamp { timestamp: 1_787_104_000 }), block_height: std::option::Option::Some(yellowstone_grpc_proto::solana::storage::confirmed_block::BlockHeight { block_height: 41 }), transactions: vec![transaction_info()], parent_slot: 41, parent_blockhash: ksp_core_lib::Pubkey::new_from_array([22_u8; 32]).to_string(), executed_transaction_count: 1, updated_account_count: 0, accounts: vec![], entries_count: 0, entries: vec![], }, )), created_at: std::option::Option::None, }; } fn fixture_settings(url: &str) -> ksp_core_lib::Result { let endpoint_url = match ksp_onchain_transport_lib::YellowstoneGrpcEndpointUrl::parse(url) { std::result::Result::Ok(value) => value, std::result::Result::Err(error) => return std::result::Result::Err(error), }; return std::result::Result::Ok(ksp_onchain_transport_lib::YellowstoneGrpcEndpointSettings::new( "pre-006-fixture", true, ksp_onchain_transport_lib::YellowstoneGrpcProviderName::new("fixture-provider"), ksp_onchain_transport_lib::YellowstoneGrpcClusterName::new("devnet"), endpoint_url, ksp_onchain_transport_lib::YellowstoneGrpcSessionSettings::default(), )); } fn project_wire( transaction: &ksp_onchain_transport_lib::YellowstoneStoredTransaction, ) -> ksp_core_lib::Result { let message = transaction.message(); let header = message.header(); let required = match u8::try_from(header.num_required_signatures()) { std::result::Result::Ok(value) => value, std::result::Result::Err(_) => return std::result::Result::Err(projection_error("num_required_signatures")), }; let readonly_signed = match u8::try_from(header.num_readonly_signed_accounts()) { std::result::Result::Ok(value) => value, std::result::Result::Err(_) => return std::result::Result::Err(projection_error("num_readonly_signed_accounts")), }; let readonly_unsigned = match u8::try_from(header.num_readonly_unsigned_accounts()) { std::result::Result::Ok(value) => value, std::result::Result::Err(_) => return std::result::Result::Err(projection_error("num_readonly_unsigned_accounts")), }; let version = if message.config().is_some() { ksp_raw_transaction_lib::RawSolanaMessageVersion::V1 } else if message.versioned() { ksp_raw_transaction_lib::RawSolanaMessageVersion::V0 } else { ksp_raw_transaction_lib::RawSolanaMessageVersion::Legacy }; let signatures = transaction.signatures().iter().map(|value| return *value.as_bytes()).collect(); let account_keys = message.account_keys().iter().map(solana_pubkey_bytes).collect(); let mut instructions = std::vec::Vec::with_capacity(message.instructions().len()); for instruction in message.instructions() { let program_id_index = match u8::try_from(instruction.program_id_index()) { std::result::Result::Ok(value) => value, std::result::Result::Err(_) => return std::result::Result::Err(projection_error("program_id_index")), }; instructions.push(ksp_raw_transaction_lib::RawSolanaCompiledInstruction::new( program_id_index, instruction.accounts().to_vec(), instruction.data().to_vec(), )); } let address_table_lookups = message .address_table_lookups() .iter() .map(|lookup| { return ksp_raw_transaction_lib::RawSolanaAddressTableLookup::new( lookup.account_key().to_bytes(), lookup.writable_indexes().to_vec(), lookup.readonly_indexes().to_vec(), ); }) .collect(); let config = message.config().map(|value| { return ksp_raw_transaction_lib::RawSolanaTransactionConfig::new( value.priority_fee(), value.compute_unit_limit(), value.loaded_accounts_data_size_limit(), value.heap_size(), ); }); let raw_message = ksp_raw_transaction_lib::RawSolanaTransactionMessage::new( version, ksp_raw_transaction_lib::RawSolanaMessageHeader::new(required, readonly_signed, readonly_unsigned), account_keys, *message.recent_blockhash().as_bytes(), instructions, address_table_lookups, config, ); return std::result::Result::Ok(ksp_raw_transaction_lib::RawSolanaTransactionWire::new(signatures, raw_message)); } fn solana_pubkey_bytes(value: &ksp_core_lib::Pubkey) -> [u8; 32] { return value.to_bytes(); } fn projection_error(field: &'static str) -> ksp_core_lib::Error { return ksp_core_lib::Error::new(ksp_core_lib::ErrorCode::new("test", "yellowstone_raw_projection"), "Yellowstone fixture field does not fit common wire") .with_context("field", field); } fn error_code_text(error: &ksp_core_lib::Error) -> std::string::String { return format!("{}.{}", error.code().domain(), error.code().code()); } fn expected_v1_wire() -> ksp_raw_transaction_lib::RawSolanaTransactionWire { let message = ksp_raw_transaction_lib::RawSolanaTransactionMessage::new( ksp_raw_transaction_lib::RawSolanaMessageVersion::V1, ksp_raw_transaction_lib::RawSolanaMessageHeader::new(1, 0, 1), vec![[1_u8; 32], [2_u8; 32]], [3_u8; 32], vec![ksp_raw_transaction_lib::RawSolanaCompiledInstruction::new(1, vec![0_u8], vec![4_u8, 5, 6])], vec![], std::option::Option::Some(ksp_raw_transaction_lib::RawSolanaTransactionConfig::new( std::option::Option::Some(7), std::option::Option::Some(8), std::option::Option::Some(9), std::option::Option::Some(32_768), )), ); return ksp_raw_transaction_lib::RawSolanaTransactionWire::new(vec![[9_u8; 64]], message); } fn fail_test(message: std::string::String) { assert!(message.is_empty(), "{message}"); } #[tokio::test] async fn pre_006_yellowstone_transaction_and_block_preserve_exact_v1_wire_and_block_time() { let server = match FixtureServer::start().await { std::result::Result::Ok(value) => value, std::result::Result::Err(error) => { assert!(error.is_empty(), "fixture server start failed: {error}"); return; }, }; let settings = match fixture_settings(server.endpoint_url.as_str()) { std::result::Result::Ok(value) => value, std::result::Result::Err(error) => { fail_test(format!("fixture settings failed: {}", error_code_text(&error))); return; }, }; let channel = match ksp_onchain_transport_lib::YellowstoneGrpcChannel::connect(&settings).await { std::result::Result::Ok(value) => value, std::result::Result::Err(error) => { fail_test(format!("fixture channel connect failed: {}", error_code_text(&error))); return; }, }; let request = ksp_onchain_transport_lib::YellowstoneSubscribeRequest::new(); let mut session = match channel.open_standard_subscribe(request).await { std::result::Result::Ok(value) => value, std::result::Result::Err(error) => { fail_test(format!("fixture subscribe failed: {}", error_code_text(&error))); return; }, }; let expected = match ksp_raw_transaction_lib::serialize_solana_transaction_wire(&expected_v1_wire()) { std::result::Result::Ok(value) => value, std::result::Result::Err(error) => { fail_test(format!("expected V1 wire serialization failed: {}", error_code_text(&error))); return; }, }; let transaction = match session.next_update().await { std::result::Result::Ok(std::option::Option::Some(value)) => value, std::result::Result::Ok(std::option::Option::None) => { fail_test("transaction update must be present".to_owned()); return; }, std::result::Result::Err(error) => { fail_test(format!("transaction update decode failed: {}", error_code_text(&error))); return; }, }; let transaction = match transaction { ksp_onchain_transport_lib::YellowstoneSubscribeUpdate::Transaction(value) => value, _ => { fail_test("first Yellowstone fixture update must be a transaction".to_owned()); return; }, }; assert_eq!(transaction.slot(), 42); assert_eq!(transaction.transaction().index(), 3); assert_eq!(transaction.transaction().meta().fee(), 5_000); let transaction_wire = match project_wire(transaction.transaction().transaction()) { std::result::Result::Ok(value) => value, std::result::Result::Err(error) => { fail_test(format!("transaction projection failed: {}", error_code_text(&error))); return; }, }; let transaction_bytes = match ksp_raw_transaction_lib::serialize_solana_transaction_wire(&transaction_wire) { std::result::Result::Ok(value) => value, std::result::Result::Err(error) => { fail_test(format!("transaction V1 wire serialization failed: {}", error_code_text(&error))); return; }, }; assert_eq!(transaction_bytes, expected); let block = match session.next_update().await { std::result::Result::Ok(std::option::Option::Some(value)) => value, std::result::Result::Ok(std::option::Option::None) => { fail_test("block update must be present".to_owned()); return; }, std::result::Result::Err(error) => { fail_test(format!("block update decode failed: {}", error_code_text(&error))); return; }, }; let block = match block { ksp_onchain_transport_lib::YellowstoneSubscribeUpdate::Block(value) => value, _ => { fail_test("second Yellowstone fixture update must be a block".to_owned()); return; }, }; assert_eq!(block.slot(), 42); assert_eq!(block.block_time(), std::option::Option::Some(1_787_104_000)); assert_eq!(block.transactions().len(), 1); assert_eq!(block.transactions()[0].index(), 3); let block_wire = match project_wire(block.transactions()[0].transaction()) { std::result::Result::Ok(value) => value, std::result::Result::Err(error) => { fail_test(format!("block transaction projection failed: {}", error_code_text(&error))); return; }, }; let block_bytes = match ksp_raw_transaction_lib::serialize_solana_transaction_wire(&block_wire) { std::result::Result::Ok(value) => value, std::result::Result::Err(error) => { fail_test(format!("block V1 wire serialization failed: {}", error_code_text(&error))); return; }, }; assert_eq!(block_bytes, expected); assert_eq!(block_bytes, transaction_bytes); assert!(transaction.transaction().meta().error().is_none()); assert_eq!(block.transactions()[0].meta().fee(), transaction.transaction().meta().fee()); assert!(session.close().await.is_ok()); server.stop().await; }