// file: crates/ksp-app-raw-transaction-ingest-desk/src/route_inventory.rs // version: 6 //! Config-only logical route inventory for Raw Transaction Ingest Desk. struct CompositeProfileCatalog { default_profile: String, profile_ids: std::vec::Vec, } #[derive(Default)] struct RouteCapabilities { has_get_transaction: bool, has_http_block_scan: bool, has_yellowstone: bool, has_standard_logs: bool, has_standard_block: bool, has_helius_transaction: bool, helius_source_declared: bool, helius_source_missing_secret: bool, yellowstone_has_get_block: bool, yellowstone_source_missing_secret: bool, } impl RouteCapabilities { fn absorb_transport(&mut self, transport: &ksp_config_lib::ResolvedTransportConfig, network: &str) { self.has_get_transaction |= http_role_supports_methods(transport.http_settings(), &["getTransaction"]); self.has_http_block_scan |= http_role_supports_methods(transport.http_settings(), &["getSlot", "getBlocksWithLimit", "getBlock"]); self.has_yellowstone |= has_yellowstone_grpc(transport, network); self.has_standard_logs |= has_ws_capability( transport, network, ksp_onchain_transport_lib::WsProtocolKind::SolanaStandard, ksp_onchain_transport_lib::WsSubscriptionKind::Logs, ); self.has_standard_block |= has_ws_capability( transport, network, ksp_onchain_transport_lib::WsProtocolKind::SolanaStandard, ksp_onchain_transport_lib::WsSubscriptionKind::Block, ); self.has_helius_transaction |= has_ws_capability( transport, network, ksp_onchain_transport_lib::WsProtocolKind::HeliusLaserStream, ksp_onchain_transport_lib::WsSubscriptionKind::HeliusTransaction, ); } } /// Rebuilds the complete safe route inventory from validated Config without opening any network, Store or Worker resource. pub(crate) fn build_route_inventory(management: &ksp_config_lib::ConfigManagement, generation: u32) -> ksp_core_lib::Result { let environment = ksp_config_lib::ConfigEnvironment::load(); let environment = match environment { std::result::Result::Ok(value) => value, std::result::Result::Err(error) => return std::result::Result::Err(error), }; return build_route_inventory_with_environment(management, &environment, generation); } /// Rebuilds the complete safe route inventory from one already captured Config environment snapshot. pub(crate) fn build_route_inventory_with_environment( management: &ksp_config_lib::ConfigManagement, environment: &ksp_config_lib::ConfigEnvironment, generation: u32, ) -> ksp_core_lib::Result { let catalog = composite_profile_catalog(management); let catalog = match catalog { std::result::Result::Ok(value) => value, std::result::Result::Err(error) => return std::result::Result::Err(error), }; let mut profiles = std::vec::Vec::with_capacity(catalog.profile_ids.len()); for profile_id in &catalog.profile_ids { profiles.push(build_profile_inventory(management, environment, profile_id.as_str(), profile_id == &catalog.default_profile)); } ksp_logging_lib::debug!( target: crate::TRACING_TARGET, domain = crate::TRACING_DOMAIN_INVENTORY, generation, profile_count = profiles.len(), "rebuilt Raw Transaction Ingest Desk network-centric Config-only route inventory" ); return std::result::Result::Ok(crate::RawIngestRouteInventoryDto { default_profile: catalog.default_profile, generation, profiles }); } fn composite_profile_catalog(management: &ksp_config_lib::ConfigManagement) -> ksp_core_lib::Result { let file_id = ksp_config_lib::ConfigFileId::new(ksp_config_lib::FILE_ID_COMPOSITE_KSP_APP_RAW_TRANSACTION_INGEST_DESK); let file_id = match file_id { std::result::Result::Ok(value) => value, std::result::Result::Err(error) => return std::result::Result::Err(error), }; let document = management.engine().load_validated_document(&file_id); let document = match document { std::result::Result::Ok(value) => value, std::result::Result::Err(error) => return std::result::Result::Err(error), }; let root = document.value().as_object(); let root = match root { std::option::Option::Some(value) => value, std::option::Option::None => return inventory_invalid("validated Raw Transaction Ingest Desk composite root is not an object"), }; let default_profile = root.get("default_profile").and_then(|value| return value.as_str()); let default_profile = match default_profile { std::option::Option::Some(value) if !value.is_empty() => value.to_owned(), _ => return inventory_invalid("validated Raw Transaction Ingest Desk composite has no readable default profile"), }; let profile_values = root.get("profiles").and_then(|value| return value.as_array()); let profile_values = match profile_values { std::option::Option::Some(value) if !value.is_empty() => value, _ => return inventory_invalid("validated Raw Transaction Ingest Desk composite has no profile inventory"), }; let mut profile_ids = std::vec::Vec::with_capacity(profile_values.len()); let mut seen = std::collections::BTreeSet::::new(); for profile in profile_values { let profile_id = profile.get("profile_id").and_then(|value| return value.as_str()); let profile_id = match profile_id { std::option::Option::Some(value) if !value.is_empty() => value, _ => return inventory_invalid("validated Raw Transaction Ingest Desk composite contains an unreadable profile identifier"), }; if !seen.insert(profile_id.to_owned()) { return inventory_invalid("validated Raw Transaction Ingest Desk composite contains duplicate profile identifiers"); } profile_ids.push(profile_id.to_owned()); } if !seen.contains(default_profile.as_str()) { return inventory_invalid("validated Raw Transaction Ingest Desk composite default profile is absent from the profile inventory"); } return std::result::Result::Ok(CompositeProfileCatalog { default_profile, profile_ids }); } fn build_profile_inventory( management: &ksp_config_lib::ConfigManagement, environment: &ksp_config_lib::ConfigEnvironment, profile_id: &str, is_default: bool, ) -> crate::RawIngestProfileInventoryDto { let composite = crate::load_raw_transaction_ingest_desk_composite_profile(management, std::option::Option::Some(profile_id)); let composite = match composite { std::result::Result::Ok(value) => value, std::result::Result::Err(_) => { return unavailable_profile(profile_id, is_default, std::option::Option::None, crate::RawIngestRouteUnavailableReason::ProfileUnresolved); }, }; let store_profile = crate::required_composite_component_profile(&composite, crate::COMPOSITE_COMPONENT_ID_STORE, ksp_config_lib::FILE_ID_STD_STORE); let store_profile = match store_profile { std::result::Result::Ok(value) => value, std::result::Result::Err(_) => { return unavailable_profile(profile_id, is_default, std::option::Option::None, crate::RawIngestRouteUnavailableReason::ProfileUnresolved); }, }; let store = management.engine().resolve_store_config_profile(&store_profile, environment); let store = match store { std::result::Result::Ok(value) => value, std::result::Result::Err(error) => { let reason = resolution_error_reason(&error); return unavailable_profile(profile_id, is_default, std::option::Option::None, reason); }, }; let network = store.settings().network().as_str().to_owned(); let base_transport_profile = crate::required_composite_component_profile(&composite, crate::COMPOSITE_COMPONENT_ID_TRANSPORT, ksp_config_lib::FILE_ID_STD_TRANSPORT); let base_transport_profile = match base_transport_profile { std::result::Result::Ok(value) => value, std::result::Result::Err(_) => { return unavailable_profile( profile_id, is_default, std::option::Option::Some(network.as_str()), crate::RawIngestRouteUnavailableReason::ProfileUnresolved, ); }, }; let base_transport = management.engine().resolve_transport_config_profile(&base_transport_profile, environment); let base_transport = match base_transport { std::result::Result::Ok(value) => value, std::result::Result::Err(error) => { return unavailable_profile(profile_id, is_default, std::option::Option::Some(network.as_str()), resolution_error_reason(&error)); }, }; if !transport_matches_network(&base_transport, network.as_str()) { return unavailable_profile( profile_id, is_default, std::option::Option::Some(network.as_str()), crate::RawIngestRouteUnavailableReason::NetworkMismatch, ); } let mut capabilities = RouteCapabilities::default(); capabilities.absorb_transport(&base_transport, network.as_str()); for (component_id, component) in composite.components() { if component_id == crate::COMPOSITE_COMPONENT_ID_TRANSPORT { continue; } if !component_id.starts_with(crate::COMPOSITE_COMPONENT_ID_TRANSPORT_PREFIX) { continue; } if component_id != crate::COMPOSITE_COMPONENT_ID_TRANSPORT_HELIUS && component_id != crate::COMPOSITE_COMPONENT_ID_TRANSPORT_YELLOWSTONE { return unavailable_profile( profile_id, is_default, std::option::Option::Some(network.as_str()), crate::RawIngestRouteUnavailableReason::ProfileUnresolved, ); } if component_id == crate::COMPOSITE_COMPONENT_ID_TRANSPORT_HELIUS { capabilities.helius_source_declared = true; } if component.resolved().file_id().as_str() != ksp_config_lib::FILE_ID_STD_TRANSPORT { return unavailable_profile( profile_id, is_default, std::option::Option::Some(network.as_str()), crate::RawIngestRouteUnavailableReason::ProfileUnresolved, ); } let resolved = management.engine().resolve_transport_config_profile(component.resolved(), environment); let resolved = match resolved { std::result::Result::Ok(value) => value, std::result::Result::Err(error) if error.code() == ksp_config_lib::ERROR_CODE_ENVIRONMENT_VARIABLE_MISSING => { mark_optional_source_missing_secret(&mut capabilities, component_id.as_str()); continue; }, std::result::Result::Err(_) => { return unavailable_profile( profile_id, is_default, std::option::Option::Some(network.as_str()), crate::RawIngestRouteUnavailableReason::ProfileUnresolved, ); }, }; if !transport_matches_network(&resolved, network.as_str()) { return unavailable_profile( profile_id, is_default, std::option::Option::Some(network.as_str()), crate::RawIngestRouteUnavailableReason::NetworkMismatch, ); } if component_id == crate::COMPOSITE_COMPONENT_ID_TRANSPORT_YELLOWSTONE { capabilities.yellowstone_has_get_block = http_role_supports_methods(resolved.http_settings(), &["getBlock"]); } capabilities.absorb_transport(&resolved, network.as_str()); } let routes = build_configured_profile_routes(&capabilities, network.as_str()); return crate::RawIngestProfileInventoryDto { is_default, network: std::option::Option::Some(network), profile_id: profile_id.to_owned(), routes }; } fn mark_optional_source_missing_secret(capabilities: &mut RouteCapabilities, component_id: &str) { if component_id == crate::COMPOSITE_COMPONENT_ID_TRANSPORT_HELIUS { capabilities.helius_source_missing_secret = true; } if component_id == crate::COMPOSITE_COMPONENT_ID_TRANSPORT_YELLOWSTONE { capabilities.yellowstone_source_missing_secret = true; } } fn build_configured_profile_routes(capabilities: &RouteCapabilities, network: &str) -> std::vec::Vec { let yellowstone_reason = if capabilities.yellowstone_source_missing_secret { crate::RawIngestRouteUnavailableReason::MissingRequiredSecret } else { crate::RawIngestRouteUnavailableReason::MissingYellowstoneGrpc }; let helius_reason = if capabilities.helius_source_missing_secret { crate::RawIngestRouteUnavailableReason::MissingRequiredSecret } else { crate::RawIngestRouteUnavailableReason::MissingHeliusTransactionCapability }; let mut routes = vec![ route_with_requirements( crate::RawIngestRouteId::YellowstoneHydrated, network, &[ (capabilities.has_yellowstone, yellowstone_reason), (capabilities.yellowstone_has_get_block, crate::RawIngestRouteUnavailableReason::MissingHttpGetBlock), ], ), route_with_requirements( crate::RawIngestRouteId::StandardLogsHydrated, network, &[ (capabilities.has_standard_logs, crate::RawIngestRouteUnavailableReason::MissingWsLogsCapability), (capabilities.has_get_transaction, crate::RawIngestRouteUnavailableReason::MissingHttpGetTransaction), ], ), route_with_requirements( crate::RawIngestRouteId::StandardBlockDirect, network, &[(capabilities.has_standard_block, crate::RawIngestRouteUnavailableReason::MissingWsBlockCapability)], ), ]; if capabilities.helius_source_declared { routes.push(route_with_requirements( crate::RawIngestRouteId::HeliusTransactionHydrated, network, &[ (capabilities.has_helius_transaction, helius_reason), (capabilities.has_get_transaction, crate::RawIngestRouteUnavailableReason::MissingHttpGetTransaction), ], )); } routes.push(route_with_requirements( crate::RawIngestRouteId::HttpBlockPolling, network, &[(capabilities.has_http_block_scan, crate::RawIngestRouteUnavailableReason::MissingHttpBlockScan)], )); return routes; } fn route_with_requirements( route_id: crate::RawIngestRouteId, network: &str, requirements: &[(bool, crate::RawIngestRouteUnavailableReason)], ) -> crate::RawIngestRouteDto { for (available, reason) in requirements.iter().copied() { if !available { return crate::RawIngestRouteDto::unavailable(route_id, std::option::Option::Some(network), reason); } } return crate::RawIngestRouteDto::configured(route_id, network); } fn unavailable_profile( profile_id: &str, is_default: bool, network: std::option::Option<&str>, reason: crate::RawIngestRouteUnavailableReason, ) -> crate::RawIngestProfileInventoryDto { let routes = crate::RawIngestRouteId::all() .into_iter() .map(|route_id| return crate::RawIngestRouteDto::unavailable(route_id, network, reason)) .collect::>(); return crate::RawIngestProfileInventoryDto { is_default, network: network.map(str::to_owned), profile_id: profile_id.to_owned(), routes }; } fn resolution_error_reason(error: &ksp_core_lib::Error) -> crate::RawIngestRouteUnavailableReason { if error.code() == ksp_config_lib::ERROR_CODE_ENVIRONMENT_VARIABLE_MISSING { return crate::RawIngestRouteUnavailableReason::MissingRequiredSecret; } return crate::RawIngestRouteUnavailableReason::ProfileUnresolved; } fn transport_matches_network(transport: &ksp_config_lib::ResolvedTransportConfig, network: &str) -> bool { let networks = enabled_transport_networks(transport); return networks.len() == 1 && networks.contains(network); } fn enabled_transport_networks(transport: &ksp_config_lib::ResolvedTransportConfig) -> std::collections::BTreeSet { let mut networks = std::collections::BTreeSet::::new(); for endpoint in transport.http_settings().endpoints() { if endpoint.enabled() { networks.insert(endpoint.cluster().as_str().to_owned()); } } if let std::option::Option::Some(settings) = transport.ws_settings() { for endpoint in settings.endpoints() { if endpoint.enabled() { networks.insert(endpoint.cluster().as_str().to_owned()); } } } if let std::option::Option::Some(settings) = transport.grpc_settings() { for endpoint in settings.endpoints() { if endpoint.enabled() { networks.insert(endpoint.cluster().as_str().to_owned()); } } } return networks; } fn has_ws_capability( transport: &ksp_config_lib::ResolvedTransportConfig, network: &str, protocol: ksp_onchain_transport_lib::WsProtocolKind, capability: ksp_onchain_transport_lib::WsSubscriptionKind, ) -> bool { let settings = match transport.ws_settings() { std::option::Option::Some(value) => value, std::option::Option::None => return false, }; for endpoint in settings.endpoints() { if endpoint.enabled() && endpoint.cluster().as_str() == network && endpoint.protocol() == protocol && endpoint.supports_subscription(capability) { return true; } } return false; } fn has_yellowstone_grpc(transport: &ksp_config_lib::ResolvedTransportConfig, network: &str) -> bool { let settings = match transport.grpc_settings() { std::option::Option::Some(value) => value, std::option::Option::None => return false, }; return settings.endpoints().iter().any(|endpoint| return endpoint.enabled() && endpoint.cluster().as_str() == network); } fn http_role_supports_methods(settings: &ksp_onchain_transport_lib::HttpTransportSettings, methods: &[&'static str]) -> bool { let mut request_kinds = std::vec::Vec::<&'static str>::with_capacity(methods.len()); for method_name in methods { let descriptor = ksp_onchain_transport_lib::find_http_rpc_method(method_name); let descriptor = match descriptor { std::option::Option::Some(value) => value, std::option::Option::None => return false, }; request_kinds.push(descriptor.request_kind()); } let mut role_names = std::collections::BTreeSet::::new(); for endpoint in settings.endpoints() { if !endpoint.enabled() { continue; } for role in endpoint.roles() { if role.enabled() { role_names.insert(role.role().as_str().to_owned()); } } } for role_name in role_names { let mut complete = true; for request_kind in &request_kinds { if !http_role_supports_request_kind(settings, role_name.as_str(), request_kind) { complete = false; break; } } if complete { return true; } } return false; } fn http_role_supports_request_kind(settings: &ksp_onchain_transport_lib::HttpTransportSettings, role_name: &str, request_kind: &str) -> bool { for endpoint in settings.endpoints() { if !endpoint.enabled() { continue; } for role in endpoint.roles() { if !role.enabled() || role.role().as_str() != role_name { continue; } for capability in role.request_kinds() { if capability.is_wildcard() || capability.as_str() == request_kind { return true; } } } } return false; } fn inventory_invalid(message: &'static str) -> ksp_core_lib::Result { return std::result::Result::Err(ksp_core_lib::Error::new(crate::ERROR_CODE_ROUTE_INVENTORY_INVALID, message)); } #[cfg(test)] #[path = "../unit_tests/route_inventory.rs"] mod tests;