Sasedev/khadhroony-solana-project
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

v0.3.15-pre.009

Browse Source
This commit is contained in:
SinuS von SifriduS
2026-09-14 11:38:55 +02:00
parent d3eea1aa12
commit 11105fac28
26 changed files with 1194 additions and 293 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
Cargo.toml
View File

@@ -1,12 +1,12 @@
# file: Cargo.toml # file: Cargo.toml
# version: 613 # version: 614
[workspace] [workspace]
resolver = "3" resolver = "3"
members = ["crates/ksp-app-backfill-desk", "crates/ksp-app-config-desk", "crates/ksp-app-raw-transaction-ingest-desk", "crates/ksp-app-solprices-desk", "crates/ksp-app-store-desk", "crates/ksp-app-wallet-desk", "crates/ksp-config-lib", "crates/ksp-core-lib", "crates/ksp-interface-lib", "crates/ksp-job-api", "crates/ksp-job-backfill-lib", "crates/ksp-logging-lib", "crates/ksp-offchain-transport-lib", "crates/ksp-onchain-transport-lib", "crates/ksp-program-api", "crates/ksp-raw-transaction-lib", "crates/ksp-store-api", "crates/ksp-store-lib", "crates/ksp-store-postgres-lib", "crates/ksp-wallet-lib", "crates/ksp-worker-api", "crates/ksp-worker-raw-transaction-ingest-lib"] members = ["crates/ksp-app-backfill-desk", "crates/ksp-app-config-desk", "crates/ksp-app-raw-transaction-ingest-desk", "crates/ksp-app-solprices-desk", "crates/ksp-app-store-desk", "crates/ksp-app-wallet-desk", "crates/ksp-config-lib", "crates/ksp-core-lib", "crates/ksp-interface-lib", "crates/ksp-job-api", "crates/ksp-job-backfill-lib", "crates/ksp-logging-lib", "crates/ksp-offchain-transport-lib", "crates/ksp-onchain-transport-lib", "crates/ksp-program-api", "crates/ksp-raw-transaction-lib", "crates/ksp-store-api", "crates/ksp-store-lib", "crates/ksp-store-postgres-lib", "crates/ksp-wallet-lib", "crates/ksp-worker-api", "crates/ksp-worker-raw-transaction-ingest-lib"]
[workspace.package] [workspace.package]
version = "0.3.15-pre.8.fix.4" version = "0.3.15-pre.9"
edition = "2024" edition = "2024"
license = "MIT" license = "MIT"
repository = "https://git.sasedev.com/Sasedev/khadhroony-solana-project" repository = "https://git.sasedev.com/Sasedev/khadhroony-solana-project"

26
config/std.transport.json
View File

@@ -437,6 +437,32 @@
} }
} }
] ]
},
{
"name": "solana_mainnet_public_repair",
"enabled": true,
"provider": "solana-public",
"cluster": "mainnet",
"url": "${KSP_PUBLIC_SOLANA_MAINNET_HTTP_URL:-https://api.mainnet.solana.com}",
"connect_timeout_ms": 5000,
"request_timeout_ms": 15000,
"max_idle_connections_per_host": 8,
"roles": [
{
"role": "default",
"enabled": true,
"request_kinds": [
"*"
],
"priority": 100,
"limits": {
"requests_per_second": 5,
"burst_capacity": 10,
"max_concurrent_requests": 8,
"pause_after_rate_limit_ms": 1000
}
}
]
} }
], ],
"ws_endpoints": [ "ws_endpoints": [

6
crates/ksp-app-raw-transaction-ingest-desk/README.md
View File

@@ -1,5 +1,5 @@
<!-- file: crates/ksp-app-raw-transaction-ingest-desk/README.md --> <!-- file: crates/ksp-app-raw-transaction-ingest-desk/README.md -->
<!-- version: 9 --> <!-- version: 10 -->
# `ksp-app-raw-transaction-ingest-desk` # `ksp-app-raw-transaction-ingest-desk`
@@ -29,7 +29,7 @@ L'inventaire de routes est calculé côté Rust à partir du composite Config va
Au Start, le backend revalide une sélection logique par `profile_id + route_id + inventory_generation + commitment`. Il recharge Config, reproouve le réseau Store/Transport, reconstruit le pool HTTP ou l'endpoint WS/gRPC exact requis, ouvre le Store via `ksp-store-lib`, exige un health `Ready` puis lance le Worker exact via `RawTransactionIngestWorker::start_with_runtime_resources`. Les ressources physiques et handles restent exclusivement côté Rust. Au Start, le backend revalide une sélection logique par `profile_id + route_id + inventory_generation + commitment`. Il recharge Config, reproouve le réseau Store/Transport, reconstruit le pool HTTP ou l'endpoint WS/gRPC exact requis, ouvre le Store via `ksp-store-lib`, exige un health `Ready` puis lance le Worker exact via `RawTransactionIngestWorker::start_with_runtime_resources`. Les ressources physiques et handles restent exclusivement côté Rust.
Le runtime est actuellement mono-route : un seul Start peut posséder le slot applicatif. Le handle Worker est installé avant l'accusé Start. Stop demande un arrêt coopératif, attend le terminal Worker, puis le monitor backend reprend l'unique `Arc<Store>` et exécute `Store::close()` avant de libérer le slot. Si le Worker est déjà devenu terminal et que son Store a déjà été fermé, un Stop tardif retourne idempotemment le dernier état terminal sûr au lieu d'échouer comme runtime inactif. Ce cache terminal ne contient ni handle ni ressource physique et est effacé dès la réservation du Start suivant. Le partage d'un Store entre plusieurs Workers reste une responsabilité de la tranche multi-route suivante. Le runtime accepte désormais plusieurs routes simultanées sur un même réseau. Chaque route possède son Worker indépendant ; deux Starts de la même identité logique `profile_id + route_id` sont refusés, et un Start d'un autre réseau est refusé tant que le Store partagé reste ouvert. Le premier Worker ouvre et valide le Store, les suivants réutilisent le même `Arc<Store>`, puis la dernière route terminale déclenche seule la fermeture explicite du Store. Un fault d'une route ne stoppe pas les Workers siblings. Stop est ciblé par route et reste idempotent lorsqu'un terminal sûr a déjà été retenu.
Les profils applicatifs sont réseau-centriques (`devnet`, `mainnet`, `testnet`). Un profil peut agréger plusieurs profils Transport du même réseau : le provider reste une source de capability et ne devient jamais une identité réseau ni un choix de profil applicatif. Les profils applicatifs sont réseau-centriques (`devnet`, `mainnet`, `testnet`). Un profil peut agréger plusieurs profils Transport du même réseau : le provider reste une source de capability et ne devient jamais une identité réseau ni un choix de profil applicatif.
Une route explicitement liée à un provider n’est projetée pour un réseau que si le composite déclare une source Transport de ce provider. Ainsi, la route Helius Transaction existe sur Devnet/Mainnet mais n’est pas présentée sur Testnet tant qu’Helius ne fournit pas de source Testnet configurée. Une route explicitement liée à un provider n’est projetée pour un réseau que si le composite déclare une source Transport de ce provider. Ainsi, la route Helius Transaction existe sur Devnet/Mainnet mais n’est pas présentée sur Testnet tant qu’Helius ne fournit pas de source Testnet configurée.
@@ -50,7 +50,7 @@ Les profils standard publics committés déclarent `Block` en plus de `Logs`; `s
Une route `Configured` est uniquement **composable depuis Config**. La disponibilité réelle du Store et du Worker est revalidée lors du Start avant publication de l'accusé runtime. Les états spontanés après Start ne sont pas encore streamés vers le frontend ; l'observabilité continue est ajoutée par la tranche monitoring dédiée. Une route `Configured` est uniquement **composable depuis Config**. La disponibilité réelle du Store et du Worker est revalidée lors du Start avant publication de l'accusé runtime. Les états spontanés après Start ne sont pas encore streamés vers le frontend ; l'observabilité continue est ajoutée par la tranche monitoring dédiée.
Pour `yellowstone-hydrated`, le Start utilise désormais le HTTP du **même profil Transport Yellowstone** que le gRPC. Sur Mainnet, le profil `publicnode_mainnet` hydrate donc via `https://solana-rpc.publicnode.com` au lieu de retomber sur `api.mainnet.solana.com`. Le profil PublicNode ne réutilise plus la limite locale Solana-public de `5 req/s`; il conserve une concurrence locale bornée et laisse les réponses `429` piloter le cooldown Transport. Cette correction ne supprime pas encore le coût structurel d'une hydration `getTransaction` par transaction : la réduction de cette dépendance, le pool HTTP multi-provider et le multi-route appartiennent à `pre.009`. Pour `yellowstone-hydrated`, la stratégie Mainnet n'effectue plus un `getTransaction` par notification transactionnelle. Le Desk construit un abonnement Yellowstone Block léger ; chaque bloc observé déclenche une reconciliation HTTP `getBlock` Full/Base64 sur le même réseau. Le profil `publicnode_mainnet` expose un pool HTTP logique `default` contenant PublicNode et le RPC public Solana comme fallback de même priorité. Le pool conserve les limites propres à chaque endpoint, son health/cooldown et son round-robin interne. La reconciliation Yellowstone tolère un décalage bref entre le gRPC et les RPC HTTP grâce à un retry borné et interruptible par Stop. Le chemin protobuf -> RAW direct reste différé tant que la canonicalisation exacte du meta n'est pas prouvée pour toutes les formes supportées.
## Ports de développement ## Ports de développement

13
crates/ksp-app-raw-transaction-ingest-desk/USAGE.md
View File

@@ -1,5 +1,5 @@
<!-- file: crates/ksp-app-raw-transaction-ingest-desk/USAGE.md --> <!-- file: crates/ksp-app-raw-transaction-ingest-desk/USAGE.md -->
<!-- version: 8 --> <!-- version: 9 -->
# Utilisation de `ksp-app-raw-transaction-ingest-desk` # Utilisation de `ksp-app-raw-transaction-ingest-desk`
@@ -36,15 +36,15 @@ Une route provider-specific dont aucune source n’est déclarée pour le résea
Les URLs, tokens, metadata secrètes, handles Transport/Store/Worker, URI Store, source keys et payloads RAW restent backend-only. Les URLs, tokens, metadata secrètes, handles Transport/Store/Worker, URI Store, source keys et payloads RAW restent backend-only.
### Start/Stop mono-route ### Start/Stop multi-route
Pour une route `Configured`, choisir `confirmed` ou `finalized` puis utiliser `Start`. Le backend revalide la génération d'inventaire et la composition Config, ouvre le Store correspondant, exige son état `Ready`, construit le Worker exact et installe son handle avant de renvoyer l'accusé runtime. Pour une route `Configured`, choisir `confirmed` ou `finalized` puis utiliser `Start`. Le backend revalide la génération d'inventaire et la composition Config, ouvre le Store correspondant lorsqu'aucun Store compatible n'est encore actif, exige son état `Ready`, construit le Worker exact puis installe son handle avant de renvoyer l'accusé runtime.
Un seul Worker peut être actif ou en démarrage dans cette tranche. Un second Start est refusé tant que le Worker précédent n'est pas terminal et que son Store n'a pas été explicitement fermé. `Stop` est coopératif : la commande attend le terminal Worker et la fermeture Store avant de rendre le slot à nouveau disponible. Si le Worker est déjà terminal et nettoyé avant le clic Stop, la commande retourne idempotemment le dernier état terminal sûr ; ce comportement permet de resynchroniser le contrôle `pre.008` sans introduire le monitoring continu réservé à une tranche ultérieure. Plusieurs routes peuvent être actives simultanément lorsqu'elles appartiennent au même réseau logique. Chaque route conserve son Worker indépendant et son Stop ciblé. Le Store est partagé entre ces Workers ; le dernier Worker terminal déclenche la fermeture explicite du Store. Une route `Faulted` ne force pas l'arrêt des autres routes. Un Start d'un autre réseau est refusé tant que le Store partagé du réseau courant reste ouvert.
Le frontend n'effectue aucun polling runtime continu dans cette tranche. Une terminaison spontanée est nettoyée côté backend ; sa projection temps réel vers l'UI appartient au monitoring ultérieur. Le frontend ne fournit pas encore de streaming continu des snapshots runtime : une terminaison spontanée est nettoyée côté backend et un Stop tardif peut resynchroniser la route avec son dernier terminal sûr.
Sur Mainnet, `yellowstone-hydrated` compose le gRPC et l'HTTP depuis la source `transport.yellowstone` déclarée par le composite. Avec le profil committé `publicnode_mainnet`, les notifications Yellowstone PublicNode sont donc hydratées par le RPC HTTP PublicNode du même profil. Ce mode reste une route hydratée : il peut encore être limité par le débit `getTransaction`. La combinaison de plusieurs endpoints HTTP/providers et le chemin Yellowstone direct sans hydration systématique sont volontairement reportés à `pre.009`. Sur Mainnet, `yellowstone-hydrated` utilise un abonnement Yellowstone Block comme signal de temps réel puis reconcile chaque slot via `getBlock` Full/Base64. Cette stratégie évite l'hydration `getTransaction` unitaire par transaction. Le profil PublicNode peut exposer plusieurs endpoints HTTP sous le même rôle logique ; Transport distribue alors les requêtes selon ses règles de priorité, disponibilité, concurrence et cooldown. La conversion directe du payload Yellowstone vers le RAW canonique reste réservée à une évolution ultérieure tant que la parité complète du meta n'est pas prouvée.
## 3. Raisons d'indisponibilité ## 3. Raisons d'indisponibilité
@@ -53,6 +53,7 @@ Les reasons exposées sont des codes applicatifs bornés, par exemple :
```text ```text
profile_unresolved profile_unresolved
network_mismatch network_mismatch
missing_http_get_block
missing_http_get_transaction missing_http_get_transaction
missing_http_block_scan missing_http_block_scan
missing_ws_logs_capability missing_ws_logs_capability

11
crates/ksp-app-raw-transaction-ingest-desk/frontend/main.html
View File

@@ -1,5 +1,5 @@
<!-- file: crates/ksp-app-raw-transaction-ingest-desk/frontend/main.html --> <!-- file: crates/ksp-app-raw-transaction-ingest-desk/frontend/main.html -->
<!-- version: 5 --> <!-- version: 6 -->
<!DOCTYPE html> <!DOCTYPE html>
<html lang="fr"> <html lang="fr">
@@ -51,7 +51,7 @@
<div class="d-flex align-items-start justify-content-between gap-3 flex-wrap mb-4"> <div class="d-flex align-items-start justify-content-between gap-3 flex-wrap mb-4">
<div> <div>
<h1 class="h3 mb-1">Routes live</h1> <h1 class="h3 mb-1">Routes live</h1>
<p class="text-body-secondary mb-0">Inventaire Config et runtime mono-route Store + Worker avec Start/Stop backend-owned.</p> <p class="text-body-secondary mb-0">Inventaire Config et runtime multi-route avec un Worker indépendant par route et Store partagé same-network.</p>
</div> </div>
<button id="refreshRouteInventory" class="btn btn-outline-primary btn-sm" type="button"> <button id="refreshRouteInventory" class="btn btn-outline-primary btn-sm" type="button">
<i class="fa-solid fa-rotate me-2" aria-hidden="true"></i>Refresh <i class="fa-solid fa-rotate me-2" aria-hidden="true"></i>Refresh
@@ -79,20 +79,17 @@
</div> </div>
</div> </div>
<div class="alert alert-info" role="status"> <div class="alert alert-info" role="status">
<strong>Configured</strong> signifie que Config prouve la composabilité. Start revalide Config, ouvre le Store puis lance exactement un Worker. pre.008 reste volontairement mono-route. <strong>Configured</strong> signifie que Config prouve la composabilité. Start revalide Config puis lance un Worker indépendant. Les routes du même réseau partagent le Store ; chaque fault reste isolé à sa route.
</div> </div>
<div class="card border-primary-subtle mb-4"> <div class="card border-primary-subtle mb-4">
<div class="card-body d-flex align-items-center justify-content-between gap-3 flex-wrap"> <div class="card-body d-flex align-items-center justify-content-between gap-3 flex-wrap">
<div> <div>
<div class="small text-body-secondary">Runtime mono-route</div> <div class="small text-body-secondary">Runtime multi-route / Store partagé</div>
<div class="d-flex flex-wrap gap-2 mt-1"> <div class="d-flex flex-wrap gap-2 mt-1">
<span id="routeRuntimeState" class="badge text-bg-secondary">idle</span> <span id="routeRuntimeState" class="badge text-bg-secondary">idle</span>
<span id="routeRuntimeIdentity" class="app-route-code small text-body-secondary">aucun Worker actif</span> <span id="routeRuntimeIdentity" class="app-route-code small text-body-secondary">aucun Worker actif</span>
</div> </div>
</div> </div>
<button id="stopRoute" class="btn btn-outline-danger btn-sm" type="button" disabled>
<i class="fa-solid fa-stop me-2" aria-hidden="true"></i>Stop
</button>
</div> </div>
</div> </div>
<div class="small text-body-secondary mb-3">Réseau affiché : <span id="routeSelectedProfile" class="app-route-code">—</span></div> <div class="small text-body-secondary mb-3">Réseau affiché : <span id="routeSelectedProfile" class="app-route-code">—</span></div>

121
crates/ksp-app-raw-transaction-ingest-desk/frontend/ts/main.ts
View File

@@ -1,5 +1,5 @@
// file: crates/ksp-app-raw-transaction-ingest-desk/frontend/ts/main.ts // file: crates/ksp-app-raw-transaction-ingest-desk/frontend/ts/main.ts
// version: 5 // version: 6
import "bootstrap"; import "bootstrap";
import ResizeObserver from "resize-observer-polyfill"; import ResizeObserver from "resize-observer-polyfill";
@@ -10,6 +10,7 @@ import type { RawIngestRouteId } from "./bindings/ksp_app_raw_transaction_ingest
import type { RawIngestRouteInventoryDto } from "./bindings/ksp_app_raw_transaction_ingest_desk/dto_route/RawIngestRouteInventoryDto.ts"; import type { RawIngestRouteInventoryDto } from "./bindings/ksp_app_raw_transaction_ingest_desk/dto_route/RawIngestRouteInventoryDto.ts";
import type { RawIngestRouteRuntimeDto } from "./bindings/ksp_app_raw_transaction_ingest_desk/dto_route/RawIngestRouteRuntimeDto.ts"; import type { RawIngestRouteRuntimeDto } from "./bindings/ksp_app_raw_transaction_ingest_desk/dto_route/RawIngestRouteRuntimeDto.ts";
import type { RawIngestRouteStartRequestDto } from "./bindings/ksp_app_raw_transaction_ingest_desk/dto_route/RawIngestRouteStartRequestDto.ts"; import type { RawIngestRouteStartRequestDto } from "./bindings/ksp_app_raw_transaction_ingest_desk/dto_route/RawIngestRouteStartRequestDto.ts";
import type { RawIngestRouteStopRequestDto } from "./bindings/ksp_app_raw_transaction_ingest_desk/dto_route/RawIngestRouteStopRequestDto.ts";
import { frontendDebug, frontendInfo, frontendTrace, frontendWarn, installFrontendConsoleBridge } from "./frontend_log"; import { frontendDebug, frontendInfo, frontendTrace, frontendWarn, installFrontendConsoleBridge } from "./frontend_log";
import { invokeKsp } from "./invoke"; import { invokeKsp } from "./invoke";
@@ -23,8 +24,8 @@ const viewTitles: Record<ViewId, string> = {
diagnostics: "Diagnostics", diagnostics: "Diagnostics",
}; };
let activeRuntime: RawIngestRouteRuntimeDto | null = null; const activeRuntimes = new Map<string, RawIngestRouteRuntimeDto>();
let lastRuntime: RawIngestRouteRuntimeDto | null = null; const lastRuntimes = new Map<string, RawIngestRouteRuntimeDto>();
let routeInventory: RawIngestRouteInventoryDto | null = null; let routeInventory: RawIngestRouteInventoryDto | null = null;
let selectedProfileId: string | null = null; let selectedProfileId: string | null = null;
@@ -88,6 +89,7 @@ function reasonLabel(reason: string | null): string {
const labels: Record<string, string> = { const labels: Record<string, string> = {
profile_unresolved: "profil Config non résolu", profile_unresolved: "profil Config non résolu",
network_mismatch: "réseau Transport/Store incohérent", network_mismatch: "réseau Transport/Store incohérent",
missing_http_get_block: "HTTP getBlock indisponible",
missing_http_get_transaction: "HTTP getTransaction indisponible", missing_http_get_transaction: "HTTP getTransaction indisponible",
missing_http_block_scan: "HTTP block scan incomplet", missing_http_block_scan: "HTTP block scan incomplet",
missing_ws_logs_capability: "capability WS Logs absente", missing_ws_logs_capability: "capability WS Logs absente",
@@ -102,6 +104,10 @@ function reasonLabel(reason: string | null): string {
return labels[reason] ?? reason; return labels[reason] ?? reason;
} }
function runtimeKey(profileId: string, routeId: RawIngestRouteId): string {
return `${profileId}::${routeId}`;
}
function renderSelectedProfile(): void { function renderSelectedProfile(): void {
if (routeInventory === null) { if (routeInventory === null) {
return; return;
@@ -118,6 +124,9 @@ function renderSelectedProfile(): void {
if (routeRoot) { if (routeRoot) {
routeRoot.replaceChildren(); routeRoot.replaceChildren();
for (const route of profile.routes) { for (const route of profile.routes) {
const key = runtimeKey(profile.profileId, route.routeId);
const runtime = activeRuntimes.get(key) ?? lastRuntimes.get(key) ?? null;
const active = runtime !== null && runtimeIsActive(runtime);
const column = document.createElement("div"); const column = document.createElement("div");
column.className = "col-12 col-xl-6 app-route-column"; column.className = "col-12 col-xl-6 app-route-column";
const card = document.createElement("div"); const card = document.createElement("div");
@@ -133,8 +142,9 @@ function renderSelectedProfile(): void {
const meta = document.createElement("div"); const meta = document.createElement("div");
meta.className = "d-flex flex-wrap gap-2 mt-3 app-route-meta"; meta.className = "d-flex flex-wrap gap-2 mt-3 app-route-meta";
const state = document.createElement("span"); const state = document.createElement("span");
state.className = route.selectable ? "badge text-bg-success" : "badge text-bg-secondary"; const projectedState = runtime?.state ?? route.state;
state.textContent = route.state; state.className = projectedState === "faulted" ? "badge text-bg-danger" : active ? "badge text-bg-primary" : route.selectable ? "badge text-bg-success" : "badge text-bg-secondary";
state.textContent = projectedState;
const network = document.createElement("span"); const network = document.createElement("span");
network.className = "badge text-bg-light border text-dark"; network.className = "badge text-bg-light border text-dark";
network.textContent = route.network ?? "network unresolved"; network.textContent = route.network ?? "network unresolved";
@@ -145,17 +155,29 @@ function renderSelectedProfile(): void {
body.append(title, code, meta); body.append(title, code, meta);
if (route.selectable) { if (route.selectable) {
const actions = document.createElement("div"); const actions = document.createElement("div");
actions.className = "mt-3"; actions.className = "d-flex gap-2 mt-3";
const start = document.createElement("button"); const start = document.createElement("button");
start.className = "btn btn-primary btn-sm"; start.className = "btn btn-primary btn-sm";
start.type = "button"; start.type = "button";
start.textContent = "Start"; start.textContent = "Start";
start.disabled = activeRuntime !== null; start.disabled = active;
start.dataset.routeId = route.routeId; start.dataset.routeId = route.routeId;
start.addEventListener("click", () => { start.addEventListener("click", () => {
void startRoute(route.routeId).catch(() => frontendWarn("main", "Raw Transaction Ingest Desk route Start failed")); void startRoute(route.routeId).catch(() => frontendWarn("main", "Raw Transaction Ingest Desk route Start failed"));
}); });
actions.append(start); actions.append(start);
if (active && runtime !== null) {
const stop = document.createElement("button");
stop.className = "btn btn-outline-danger btn-sm";
stop.type = "button";
stop.textContent = "Stop";
stop.disabled = runtime.state === "stopping";
stop.dataset.routeId = route.routeId;
stop.addEventListener("click", () => {
void stopRoute(runtime).catch(() => frontendWarn("main", "Raw Transaction Ingest Desk route Stop failed"));
});
actions.append(stop);
}
body.append(actions); body.append(actions);
} }
card.append(body); card.append(body);
@@ -164,6 +186,7 @@ function renderSelectedProfile(): void {
} }
} }
frontendTrace("main", "Raw Transaction Ingest Desk Config route inventory profile rendered", { frontendTrace("main", "Raw Transaction Ingest Desk Config route inventory profile rendered", {
activeRouteCount: activeRuntimes.size,
profileId: profile.profileId, profileId: profile.profileId,
routeCount: profile.routes.length, routeCount: profile.routes.length,
selectableCount: profile.routes.filter(route => route.selectable).length, selectableCount: profile.routes.filter(route => route.selectable).length,
@@ -179,33 +202,28 @@ function runtimeIsActive(runtime: RawIngestRouteRuntimeDto): boolean {
return runtime.state === "starting" || runtime.state === "running" || runtime.state === "stopping"; return runtime.state === "starting" || runtime.state === "running" || runtime.state === "stopping";
} }
function renderMonoRouteRuntime(): void { function renderMultiRouteRuntime(): void {
const projected = activeRuntime ?? lastRuntime; const active = Array.from(activeRuntimes.values());
const state = document.querySelector<HTMLElement>("#routeRuntimeState"); const state = document.querySelector<HTMLElement>("#routeRuntimeState");
if (state) { if (state) {
state.textContent = projected?.state ?? "idle"; state.textContent = active.length === 0 ? "idle" : `${active.length} active`;
state.className = projected?.state === "faulted" ? "badge text-bg-danger" : activeRuntime ? "badge text-bg-primary" : "badge text-bg-secondary"; state.className = active.length === 0 ? "badge text-bg-secondary" : "badge text-bg-primary";
}
text(
"#routeRuntimeIdentity",
projected ? `${projected.profileId} / ${projected.routeId} / ${projected.commitment}` : "aucun Worker actif",
);
const stop = document.querySelector<HTMLButtonElement>("#stopRoute");
if (stop) {
stop.disabled = activeRuntime === null || activeRuntime.state === "stopping";
} }
const identities = active.map(runtime => `${runtime.routeId}/${runtime.commitment}`).join(", ");
text("#routeRuntimeIdentity", identities.length === 0 ? "aucun Worker actif" : identities);
const profile = document.querySelector<HTMLSelectElement>("#routeProfile"); const profile = document.querySelector<HTMLSelectElement>("#routeProfile");
if (profile) { if (profile) {
profile.disabled = activeRuntime !== null; profile.disabled = active.length > 0;
}
const commitment = document.querySelector<HTMLSelectElement>("#routeCommitment");
if (commitment) {
commitment.disabled = activeRuntime !== null;
} }
} }
async function startRoute(routeId: RawIngestRouteId): Promise<void> { async function startRoute(routeId: RawIngestRouteId): Promise<void> {
if (routeInventory === null || selectedProfileId === null || activeRuntime !== null) { if (routeInventory === null || selectedProfileId === null) {
return;
}
const key = runtimeKey(selectedProfileId, routeId);
const existing = activeRuntimes.get(key);
if (existing !== undefined && runtimeIsActive(existing)) {
return; return;
} }
const request: RawIngestRouteStartRequestDto = { const request: RawIngestRouteStartRequestDto = {
@@ -214,37 +232,49 @@ async function startRoute(routeId: RawIngestRouteId): Promise<void> {
profileId: selectedProfileId, profileId: selectedProfileId,
routeId, routeId,
}; };
frontendDebug("main", "Raw Transaction Ingest Desk mono-route Start requested", { frontendDebug("main", "Raw Transaction Ingest Desk route Start requested", {
commitment: request.commitment, commitment: request.commitment,
profileId: request.profileId, profileId: request.profileId,
routeId: request.routeId, routeId: request.routeId,
}); });
const response = await invokeKsp<RawIngestRouteRuntimeDto>("main", "start_route", { request }); const response = await invokeKsp<RawIngestRouteRuntimeDto>("main", "start_route", { request });
lastRuntime = response; lastRuntimes.set(key, response);
activeRuntime = runtimeIsActive(response) ? response : null; if (runtimeIsActive(response)) {
renderMonoRouteRuntime(); activeRuntimes.set(key, response);
} else {
activeRuntimes.delete(key);
}
renderMultiRouteRuntime();
renderSelectedProfile(); renderSelectedProfile();
frontendInfo("main", "Raw Transaction Ingest Desk mono-route Start acknowledged", { frontendInfo("main", "Raw Transaction Ingest Desk route Start acknowledged", {
activeRouteCount: activeRuntimes.size,
profileId: response.profileId, profileId: response.profileId,
routeId: response.routeId, routeId: response.routeId,
state: response.state, state: response.state,
}); });
} }
async function stopRoute(): Promise<void> { async function stopRoute(runtime: RawIngestRouteRuntimeDto): Promise<void> {
if (activeRuntime === null) { const key = runtimeKey(runtime.profileId, runtime.routeId);
return; const request: RawIngestRouteStopRequestDto = {
} profileId: runtime.profileId,
frontendDebug("main", "Raw Transaction Ingest Desk mono-route Stop requested", { routeId: runtime.routeId,
profileId: activeRuntime.profileId, };
routeId: activeRuntime.routeId, frontendDebug("main", "Raw Transaction Ingest Desk targeted route Stop requested", {
profileId: request.profileId,
routeId: request.routeId,
}); });
const response = await invokeKsp<RawIngestRouteRuntimeDto>("main", "stop_route"); const response = await invokeKsp<RawIngestRouteRuntimeDto>("main", "stop_route", { request });
lastRuntime = response; lastRuntimes.set(key, response);
activeRuntime = runtimeIsActive(response) ? response : null; if (runtimeIsActive(response)) {
renderMonoRouteRuntime(); activeRuntimes.set(key, response);
} else {
activeRuntimes.delete(key);
}
renderMultiRouteRuntime();
renderSelectedProfile(); renderSelectedProfile();
frontendInfo("main", "Raw Transaction Ingest Desk mono-route Stop completed", { frontendInfo("main", "Raw Transaction Ingest Desk targeted route Stop completed", {
activeRouteCount: activeRuntimes.size,
profileId: response.profileId, profileId: response.profileId,
routeId: response.routeId, routeId: response.routeId,
state: response.state, state: response.state,
@@ -271,7 +301,7 @@ function renderRouteInventory(inventory: RawIngestRouteInventoryDto): void {
text("#runtimeInventoryGeneration", inventory.generation.toString()); text("#runtimeInventoryGeneration", inventory.generation.toString());
text("#runtimeInventoryDefaultProfile", inventory.defaultProfile); text("#runtimeInventoryDefaultProfile", inventory.defaultProfile);
renderSelectedProfile(); renderSelectedProfile();
renderMonoRouteRuntime(); renderMultiRouteRuntime();
frontendTrace("main", "Raw Transaction Ingest Desk Config route inventory rendered", { frontendTrace("main", "Raw Transaction Ingest Desk Config route inventory rendered", {
generation: inventory.generation, generation: inventory.generation,
profileCount: inventory.profiles.length, profileCount: inventory.profiles.length,
@@ -323,9 +353,6 @@ function bindRefreshControls(): void {
document.querySelector<HTMLButtonElement>("#refreshRuntimeStatus")?.addEventListener("click", () => { document.querySelector<HTMLButtonElement>("#refreshRuntimeStatus")?.addEventListener("click", () => {
void refreshRuntimeStatus().catch(() => frontendWarn("main", "Raw Transaction Ingest Desk runtime status refresh failed")); void refreshRuntimeStatus().catch(() => frontendWarn("main", "Raw Transaction Ingest Desk runtime status refresh failed"));
}); });
document.querySelector<HTMLButtonElement>("#stopRoute")?.addEventListener("click", () => {
void stopRoute().catch(() => frontendWarn("main", "Raw Transaction Ingest Desk route Stop failed"));
});
document.querySelector<HTMLSelectElement>("#routeProfile")?.addEventListener("change", event => { document.querySelector<HTMLSelectElement>("#routeProfile")?.addEventListener("change", event => {
const selector = event.currentTarget; const selector = event.currentTarget;
if (!(selector instanceof HTMLSelectElement)) { if (!(selector instanceof HTMLSelectElement)) {
@@ -345,7 +372,7 @@ async function initializeMain(): Promise<void> {
activateView("routes", "startup"); activateView("routes", "startup");
await refreshRouteInventory(); await refreshRouteInventory();
await refreshRuntimeStatus(); await refreshRuntimeStatus();
frontendInfo("main", "Raw Transaction Ingest Desk mono-route runtime frontend ready"); frontendInfo("main", "Raw Transaction Ingest Desk multi-route shared-Store runtime frontend ready");
} }
void initializeMain().catch(() => frontendWarn("main", "Raw Transaction Ingest Desk frontend initialization failed")); void initializeMain().catch(() => frontendWarn("main", "Raw Transaction Ingest Desk frontend initialization failed"));

12
crates/ksp-app-raw-transaction-ingest-desk/src/app_state.rs
View File

@@ -1,5 +1,5 @@
// file: crates/ksp-app-raw-transaction-ingest-desk/src/app_state.rs // file: crates/ksp-app-raw-transaction-ingest-desk/src/app_state.rs
// version: 5 // version: 6
//! Shared backend state owned by the Raw Transaction Ingest Desk Tauri application. //! Shared backend state owned by the Raw Transaction Ingest Desk Tauri application.
@@ -115,7 +115,7 @@ impl crate::AppState {
return crate::validate_route_start(&self.config_management, generation, request); return crate::validate_route_start(&self.config_management, generation, request);
} }
/// Starts one real mono-route Store + Worker runtime after repeating the complete Start-time revalidation. /// Starts one independent route Worker after repeating complete Start-time revalidation, sharing Store only on an exact network match.
pub(crate) async fn start_route(&self, request: &crate::RawIngestRouteStartRequestDto) -> ksp_core_lib::Result<crate::RouteRuntimeLaunch> { pub(crate) async fn start_route(&self, request: &crate::RawIngestRouteStartRequestDto) -> ksp_core_lib::Result<crate::RouteRuntimeLaunch> {
let generation = { let generation = {
let generation = self.inventory_generation.lock(); let generation = self.inventory_generation.lock();
@@ -137,9 +137,9 @@ impl crate::AppState {
return crate::start_route_runtime(std::sync::Arc::clone(&self.route_runtime), prepared).await; return crate::start_route_runtime(std::sync::Arc::clone(&self.route_runtime), prepared).await;
} }
/// Requests cooperative Stop and waits until the Worker is terminal and Store cleanup has released the mono-route slot. /// Requests cooperative Stop for one exact logical route and waits until its Worker cleanup is complete.
pub(crate) async fn stop_route(&self) -> ksp_core_lib::Result<crate::RawIngestRouteRuntimeDto> { pub(crate) async fn stop_route(&self, request: &crate::RawIngestRouteStopRequestDto) -> ksp_core_lib::Result<crate::RawIngestRouteRuntimeDto> {
return self.route_runtime.stop_and_wait().await; return self.route_runtime.stop_and_wait(request).await;
} }
/// Builds the safe scaffold status exposed by the shell. /// Builds the safe scaffold status exposed by the shell.
@@ -175,7 +175,7 @@ impl crate::AppState {
application_version: env!("CARGO_PKG_VERSION").to_owned(), application_version: env!("CARGO_PKG_VERSION").to_owned(),
config_document_count: document_count, config_document_count: document_count,
fallback_logging_active: runtime.fallback_active, fallback_logging_active: runtime.fallback_active,
shell_phase: "pre.008-mono-route-runtime".to_owned(), shell_phase: "pre.009-multi-route-mainnet".to_owned(),
startup_diagnostic: runtime.startup_diagnostic.clone(), startup_diagnostic: runtime.startup_diagnostic.clone(),
}); });
} }

4
crates/ksp-app-raw-transaction-ingest-desk/src/constants.rs
View File

@@ -1,5 +1,5 @@
// file: crates/ksp-app-raw-transaction-ingest-desk/src/constants.rs // file: crates/ksp-app-raw-transaction-ingest-desk/src/constants.rs
// version: 5 // version: 6
//! Application-owned tracing targets and Config component identifiers. //! Application-owned tracing targets and Config component identifiers.
@@ -21,7 +21,7 @@ pub(crate) const TRACING_DOMAIN_BOOTSTRAP: &str = "raw_transaction_ingest.bootst
pub(crate) const TRACING_DOMAIN_FRONTEND: &str = "frontend"; pub(crate) const TRACING_DOMAIN_FRONTEND: &str = "frontend";
/// Structured domain used while rebuilding Config-only route inventory. /// Structured domain used while rebuilding Config-only route inventory.
pub(crate) const TRACING_DOMAIN_INVENTORY: &str = "raw_transaction_ingest.inventory"; pub(crate) const TRACING_DOMAIN_INVENTORY: &str = "raw_transaction_ingest.inventory";
/// Structured domain used by mono-route Store and Worker lifecycle operations. /// Structured domain used by multi-route shared-Store and Worker lifecycle operations.
pub(crate) const TRACING_DOMAIN_ROUTE_RUNTIME: &str = "raw_transaction_ingest.route_runtime"; pub(crate) const TRACING_DOMAIN_ROUTE_RUNTIME: &str = "raw_transaction_ingest.route_runtime";
/// Structured domain used while revalidating future Start requests and reconstructing route resources. /// Structured domain used while revalidating future Start requests and reconstructing route resources.
pub(crate) const TRACING_DOMAIN_ROUTE_START: &str = "raw_transaction_ingest.route_start"; pub(crate) const TRACING_DOMAIN_ROUTE_START: &str = "raw_transaction_ingest.route_start";

23
crates/ksp-app-raw-transaction-ingest-desk/src/dto_route.rs
View File

@@ -1,5 +1,5 @@
// file: crates/ksp-app-raw-transaction-ingest-desk/src/dto_route.rs // file: crates/ksp-app-raw-transaction-ingest-desk/src/dto_route.rs
// version: 5 // version: 6
//! Safe route-identity, Config-composability and lifecycle DTOs for Raw Transaction Ingest Desk. //! Safe route-identity, Config-composability and lifecycle DTOs for Raw Transaction Ingest Desk.
@@ -10,7 +10,7 @@ use ts_rs::TS; // rust-rules: trait-import
#[serde(rename_all = "kebab-case")] #[serde(rename_all = "kebab-case")]
#[ts(export, export_to = "../frontend/ts/bindings/ksp_app_raw_transaction_ingest_desk/dto_route/RawIngestRouteId.ts")] #[ts(export, export_to = "../frontend/ts/bindings/ksp_app_raw_transaction_ingest_desk/dto_route/RawIngestRouteId.ts")]
pub(crate) enum RawIngestRouteId { pub(crate) enum RawIngestRouteId {
/// Yellowstone Subscribe plus same-network HTTP hydration. /// Yellowstone block subscription plus same-network HTTP block reconciliation.
YellowstoneHydrated, YellowstoneHydrated,
/// Standard Solana logs subscription plus same-network HTTP hydration. /// Standard Solana logs subscription plus same-network HTTP hydration.
StandardLogsHydrated, StandardLogsHydrated,
@@ -45,7 +45,7 @@ impl crate::RawIngestRouteId {
#[must_use] #[must_use]
pub(crate) const fn label(self) -> &'static str { pub(crate) const fn label(self) -> &'static str {
return match self { return match self {
Self::YellowstoneHydrated => "Yellowstone + HTTP hydration", Self::YellowstoneHydrated => "Yellowstone Block + HTTP block reconciliation",
Self::StandardLogsHydrated => "Standard Logs + HTTP hydration", Self::StandardLogsHydrated => "Standard Logs + HTTP hydration",
Self::StandardBlockDirect => "Standard Block direct", Self::StandardBlockDirect => "Standard Block direct",
Self::HeliusTransactionHydrated => "Helius Transaction + HTTP hydration", Self::HeliusTransactionHydrated => "Helius Transaction + HTTP hydration",
@@ -112,6 +112,17 @@ pub(crate) struct RawIngestRouteStartRequestDto {
pub(crate) route_id: crate::RawIngestRouteId, pub(crate) route_id: crate::RawIngestRouteId,
} }
/// Safe targeted Stop request containing only the logical Worker route identity.
#[derive(Clone, Debug, serde::Deserialize, TS)]
#[serde(rename_all = "camelCase")]
#[ts(export, export_to = "../frontend/ts/bindings/ksp_app_raw_transaction_ingest_desk/dto_route/RawIngestRouteStopRequestDto.ts")]
pub(crate) struct RawIngestRouteStopRequestDto {
/// Safe logical network-profile identifier that owns the Worker route.
pub(crate) profile_id: String,
/// Stable logical route identifier to stop.
pub(crate) route_id: crate::RawIngestRouteId,
}
/// Safe acknowledgement that exact Start-time Transport resources were reconstructed and validated without Worker launch. /// Safe acknowledgement that exact Start-time Transport resources were reconstructed and validated without Worker launch.
#[derive(Clone, Debug, serde::Serialize, TS)] #[derive(Clone, Debug, serde::Serialize, TS)]
#[serde(rename_all = "camelCase")] #[serde(rename_all = "camelCase")]
@@ -129,7 +140,7 @@ pub(crate) struct RawIngestRouteStartValidationDto {
pub(crate) route_id: crate::RawIngestRouteId, pub(crate) route_id: crate::RawIngestRouteId,
} }
/// Safe mono-route runtime acknowledgement containing only logical identity and lifecycle state. /// Safe per-route runtime acknowledgement containing only logical identity and lifecycle state.
#[derive(Clone, Debug, serde::Serialize, TS)] #[derive(Clone, Debug, serde::Serialize, TS)]
#[serde(rename_all = "camelCase")] #[serde(rename_all = "camelCase")]
#[ts(export, export_to = "../frontend/ts/bindings/ksp_app_raw_transaction_ingest_desk/dto_route/RawIngestRouteRuntimeDto.ts")] #[ts(export, export_to = "../frontend/ts/bindings/ksp_app_raw_transaction_ingest_desk/dto_route/RawIngestRouteRuntimeDto.ts")]
@@ -142,7 +153,7 @@ pub(crate) struct RawIngestRouteRuntimeDto {
pub(crate) network: String, pub(crate) network: String,
/// Safe logical network-profile identifier. /// Safe logical network-profile identifier.
pub(crate) profile_id: String, pub(crate) profile_id: String,
/// Exact logical route owned by the mono-route Worker. /// Exact logical route owned by the independent Worker.
pub(crate) route_id: crate::RawIngestRouteId, pub(crate) route_id: crate::RawIngestRouteId,
/// Safe application-owned projection of current Worker lifecycle. /// Safe application-owned projection of current Worker lifecycle.
pub(crate) state: crate::RawIngestRouteState, pub(crate) state: crate::RawIngestRouteState,
@@ -174,6 +185,8 @@ pub(crate) enum RawIngestRouteUnavailableReason {
ProfileUnresolved, ProfileUnresolved,
/// Transport and Store do not prove one identical logical network. /// Transport and Store do not prove one identical logical network.
NetworkMismatch, NetworkMismatch,
/// No HTTP role can satisfy `getBlock` for same-network block reconciliation.
MissingHttpGetBlock,
/// No HTTP role can satisfy `getTransaction` for same-network hydration. /// No HTTP role can satisfy `getTransaction` for same-network hydration.
MissingHttpGetTransaction, MissingHttpGetTransaction,
/// No HTTP role can satisfy the complete block-polling method set. /// No HTTP role can satisfy the complete block-polling method set.

11
crates/ksp-app-raw-transaction-ingest-desk/src/errors.rs
View File

@@ -1,5 +1,5 @@
// file: crates/ksp-app-raw-transaction-ingest-desk/src/errors.rs // file: crates/ksp-app-raw-transaction-ingest-desk/src/errors.rs
// version: 4 // version: 5
//! Stable Raw Transaction Ingest Desk application error codes. //! Stable Raw Transaction Ingest Desk application error codes.
@@ -26,11 +26,14 @@ pub(crate) const ERROR_CODE_ROUTE_INVENTORY_INVALID: ksp_core_lib::ErrorCode =
/// Exact Start-time Transport resources cannot be reconstructed safely. /// Exact Start-time Transport resources cannot be reconstructed safely.
pub(crate) const ERROR_CODE_ROUTE_START_PREPARATION_FAILED: ksp_core_lib::ErrorCode = pub(crate) const ERROR_CODE_ROUTE_START_PREPARATION_FAILED: ksp_core_lib::ErrorCode =
ksp_core_lib::ErrorCode::new("raw_transaction_ingest_desk", "route_start_preparation_failed"); ksp_core_lib::ErrorCode::new("raw_transaction_ingest_desk", "route_start_preparation_failed");
/// A mono-route Worker runtime already owns the application slot. /// The selected logical route is already starting/active or shared Store lifecycle is temporarily exclusive.
pub(crate) const ERROR_CODE_ROUTE_RUNTIME_ACTIVE: ksp_core_lib::ErrorCode = ksp_core_lib::ErrorCode::new("raw_transaction_ingest_desk", "route_runtime_active"); pub(crate) const ERROR_CODE_ROUTE_RUNTIME_ACTIVE: ksp_core_lib::ErrorCode = ksp_core_lib::ErrorCode::new("raw_transaction_ingest_desk", "route_runtime_active");
/// No mono-route Worker runtime is currently active. /// No selected route Worker runtime is currently active.
pub(crate) const ERROR_CODE_ROUTE_RUNTIME_NOT_ACTIVE: ksp_core_lib::ErrorCode = pub(crate) const ERROR_CODE_ROUTE_RUNTIME_NOT_ACTIVE: ksp_core_lib::ErrorCode =
ksp_core_lib::ErrorCode::new("raw_transaction_ingest_desk", "route_runtime_not_active"); ksp_core_lib::ErrorCode::new("raw_transaction_ingest_desk", "route_runtime_not_active");
/// A selected route cannot share the currently open Store because its network differs.
pub(crate) const ERROR_CODE_ROUTE_RUNTIME_NETWORK_MISMATCH: ksp_core_lib::ErrorCode =
ksp_core_lib::ErrorCode::new("raw_transaction_ingest_desk", "route_runtime_network_mismatch");
/// Store or Worker startup failed after Start-time route revalidation. /// Store or Worker startup failed after Start-time route revalidation.
pub(crate) const ERROR_CODE_ROUTE_RUNTIME_START_FAILED: ksp_core_lib::ErrorCode = pub(crate) const ERROR_CODE_ROUTE_RUNTIME_START_FAILED: ksp_core_lib::ErrorCode =
ksp_core_lib::ErrorCode::new("raw_transaction_ingest_desk", "route_runtime_start_failed"); ksp_core_lib::ErrorCode::new("raw_transaction_ingest_desk", "route_runtime_start_failed");
@@ -40,7 +43,7 @@ pub(crate) const ERROR_CODE_ROUTE_RUNTIME_STORE_NOT_READY: ksp_core_lib::ErrorCo
/// The Store could not be reclaimed or explicitly closed after Worker termination. /// The Store could not be reclaimed or explicitly closed after Worker termination.
pub(crate) const ERROR_CODE_ROUTE_RUNTIME_STORE_SHUTDOWN_FAILED: ksp_core_lib::ErrorCode = pub(crate) const ERROR_CODE_ROUTE_RUNTIME_STORE_SHUTDOWN_FAILED: ksp_core_lib::ErrorCode =
ksp_core_lib::ErrorCode::new("raw_transaction_ingest_desk", "route_runtime_store_shutdown_failed"); ksp_core_lib::ErrorCode::new("raw_transaction_ingest_desk", "route_runtime_store_shutdown_failed");
/// The mono-route runtime reservation/handle state changed unexpectedly. /// The multi-route runtime reservation/handle state changed unexpectedly.
pub(crate) const ERROR_CODE_ROUTE_RUNTIME_STATE_INVALID: ksp_core_lib::ErrorCode = pub(crate) const ERROR_CODE_ROUTE_RUNTIME_STATE_INVALID: ksp_core_lib::ErrorCode =
ksp_core_lib::ErrorCode::new("raw_transaction_ingest_desk", "route_runtime_state_invalid"); ksp_core_lib::ErrorCode::new("raw_transaction_ingest_desk", "route_runtime_state_invalid");
/// Start request references an inventory generation that is no longer current. /// Start request references an inventory generation that is no longer current.

20
crates/ksp-app-raw-transaction-ingest-desk/src/lib.rs
View File

@@ -1,5 +1,5 @@
// file: crates/ksp-app-raw-transaction-ingest-desk/src/lib.rs // file: crates/ksp-app-raw-transaction-ingest-desk/src/lib.rs
// version: 5 // version: 6
//! Tauri desktop application scaffold for composing and supervising KSP live RAW transaction ingest routes. //! Tauri desktop application scaffold for composing and supervising KSP live RAW transaction ingest routes.
@@ -58,7 +58,7 @@ pub(crate) use self::constants::TRACING_DOMAIN_BOOTSTRAP;
pub(crate) use self::constants::TRACING_DOMAIN_FRONTEND; pub(crate) use self::constants::TRACING_DOMAIN_FRONTEND;
/// Structured domain used while rebuilding Config-only route inventory. /// Structured domain used while rebuilding Config-only route inventory.
pub(crate) use self::constants::TRACING_DOMAIN_INVENTORY; pub(crate) use self::constants::TRACING_DOMAIN_INVENTORY;
/// Structured domain used by mono-route Store and Worker lifecycle operations. /// Structured domain used by multi-route shared-Store and Worker lifecycle operations.
pub(crate) use self::constants::TRACING_DOMAIN_ROUTE_RUNTIME; pub(crate) use self::constants::TRACING_DOMAIN_ROUTE_RUNTIME;
/// Structured domain used while revalidating future Start requests. /// Structured domain used while revalidating future Start requests.
pub(crate) use self::constants::TRACING_DOMAIN_ROUTE_START; pub(crate) use self::constants::TRACING_DOMAIN_ROUTE_START;
@@ -92,7 +92,7 @@ pub(crate) use self::dto_route::RawIngestRouteFoundationDto;
pub(crate) use self::dto_route::RawIngestRouteId; pub(crate) use self::dto_route::RawIngestRouteId;
/// Complete deterministic Config-only route inventory. /// Complete deterministic Config-only route inventory.
pub(crate) use self::dto_route::RawIngestRouteInventoryDto; pub(crate) use self::dto_route::RawIngestRouteInventoryDto;
/// Safe mono-route runtime acknowledgement containing only logical identity and lifecycle. /// Safe per-route runtime acknowledgement containing only logical identity and lifecycle.
pub(crate) use self::dto_route::RawIngestRouteRuntimeDto; pub(crate) use self::dto_route::RawIngestRouteRuntimeDto;
/// Safe future Start request containing only logical route selection and inventory generation. /// Safe future Start request containing only logical route selection and inventory generation.
pub(crate) use self::dto_route::RawIngestRouteStartRequestDto; pub(crate) use self::dto_route::RawIngestRouteStartRequestDto;
@@ -100,6 +100,8 @@ pub(crate) use self::dto_route::RawIngestRouteStartRequestDto;
pub(crate) use self::dto_route::RawIngestRouteStartValidationDto; pub(crate) use self::dto_route::RawIngestRouteStartValidationDto;
/// Stable route lifecycle states reserved for runtime projections. /// Stable route lifecycle states reserved for runtime projections.
pub(crate) use self::dto_route::RawIngestRouteState; pub(crate) use self::dto_route::RawIngestRouteState;
/// Safe targeted Stop request containing only logical route identity.
pub(crate) use self::dto_route::RawIngestRouteStopRequestDto;
/// Safe reason explaining why one logical route is not composable from Config. /// Safe reason explaining why one logical route is not composable from Config.
pub(crate) use self::dto_route::RawIngestRouteUnavailableReason; pub(crate) use self::dto_route::RawIngestRouteUnavailableReason;
/// Shared Raw Transaction Ingest Desk application state is internally inconsistent. /// Shared Raw Transaction Ingest Desk application state is internally inconsistent.
@@ -116,13 +118,15 @@ pub(crate) use self::errors::ERROR_CODE_FRONTEND_LOG_TARGET_INVALID;
pub(crate) use self::errors::ERROR_CODE_LOGGING_BOOTSTRAP_FAILED; pub(crate) use self::errors::ERROR_CODE_LOGGING_BOOTSTRAP_FAILED;
/// Config-only route inventory cannot be projected safely. /// Config-only route inventory cannot be projected safely.
pub(crate) use self::errors::ERROR_CODE_ROUTE_INVENTORY_INVALID; pub(crate) use self::errors::ERROR_CODE_ROUTE_INVENTORY_INVALID;
/// A mono-route Worker runtime already owns the application slot. /// A Worker runtime already owns the same logical route in the shared application runtime.
pub(crate) use self::errors::ERROR_CODE_ROUTE_RUNTIME_ACTIVE; pub(crate) use self::errors::ERROR_CODE_ROUTE_RUNTIME_ACTIVE;
/// No mono-route Worker runtime is currently active. /// Selected route cannot share the currently open Store because its network differs.
pub(crate) use self::errors::ERROR_CODE_ROUTE_RUNTIME_NETWORK_MISMATCH;
/// No selected route Worker runtime is currently active.
pub(crate) use self::errors::ERROR_CODE_ROUTE_RUNTIME_NOT_ACTIVE; pub(crate) use self::errors::ERROR_CODE_ROUTE_RUNTIME_NOT_ACTIVE;
/// Store or Worker startup failed after Start-time route revalidation. /// Store or Worker startup failed after Start-time route revalidation.
pub(crate) use self::errors::ERROR_CODE_ROUTE_RUNTIME_START_FAILED; pub(crate) use self::errors::ERROR_CODE_ROUTE_RUNTIME_START_FAILED;
/// The mono-route runtime reservation/handle state changed unexpectedly. /// The multi-route runtime reservation or handle state changed unexpectedly.
pub(crate) use self::errors::ERROR_CODE_ROUTE_RUNTIME_STATE_INVALID; pub(crate) use self::errors::ERROR_CODE_ROUTE_RUNTIME_STATE_INVALID;
/// The opened Store did not prove Ready before Worker launch. /// The opened Store did not prove Ready before Worker launch.
pub(crate) use self::errors::ERROR_CODE_ROUTE_RUNTIME_STORE_NOT_READY; pub(crate) use self::errors::ERROR_CODE_ROUTE_RUNTIME_STORE_NOT_READY;
@@ -154,9 +158,9 @@ pub(crate) use self::logging_runtime::launch_identity;
pub(crate) use self::route_inventory::build_route_inventory; pub(crate) use self::route_inventory::build_route_inventory;
/// Rebuilds the route inventory from one already captured Config environment snapshot. /// Rebuilds the route inventory from one already captured Config environment snapshot.
pub(crate) use self::route_inventory::build_route_inventory_with_environment; pub(crate) use self::route_inventory::build_route_inventory_with_environment;
/// Launch ownership for one mono-route Worker and its caller-owned Store. /// Launch ownership for one independent route Worker sharing the application Store when eligible.
pub(crate) use self::route_runtime::RouteRuntimeLaunch; pub(crate) use self::route_runtime::RouteRuntimeLaunch;
/// Shared mono-route runtime state admitting at most one starting or active Worker. /// Shared multi-route runtime state with one independent Worker per route and one same-network Store.
pub(crate) use self::route_runtime::RouteRuntimeState; pub(crate) use self::route_runtime::RouteRuntimeState;
/// Opens Store and starts one exact route Worker after Start-time revalidation. /// Opens Store and starts one exact route Worker after Start-time revalidation.
pub(crate) use self::route_runtime::start_route_runtime; pub(crate) use self::route_runtime::start_route_runtime;

8
crates/ksp-app-raw-transaction-ingest-desk/src/route_inventory.rs
View File

@@ -1,5 +1,5 @@
// file: crates/ksp-app-raw-transaction-ingest-desk/src/route_inventory.rs // file: crates/ksp-app-raw-transaction-ingest-desk/src/route_inventory.rs
// version: 5 // version: 6
//! Config-only logical route inventory for Raw Transaction Ingest Desk. //! Config-only logical route inventory for Raw Transaction Ingest Desk.
@@ -18,6 +18,7 @@ struct RouteCapabilities {
has_helius_transaction: bool, has_helius_transaction: bool,
helius_source_declared: bool, helius_source_declared: bool,
helius_source_missing_secret: bool, helius_source_missing_secret: bool,
yellowstone_has_get_block: bool,
yellowstone_source_missing_secret: bool, yellowstone_source_missing_secret: bool,
} }
@@ -236,6 +237,9 @@ fn build_profile_inventory(
crate::RawIngestRouteUnavailableReason::NetworkMismatch, crate::RawIngestRouteUnavailableReason::NetworkMismatch,
); );
} }
if component_id == crate::COMPOSITE_COMPONENT_ID_TRANSPORT_YELLOWSTONE {
capabilities.yellowstone_has_get_block = http_role_supports_methods(resolved.http_settings(), &["getBlock"]);
}
capabilities.absorb_transport(&resolved, network.as_str()); capabilities.absorb_transport(&resolved, network.as_str());
} }
let routes = build_configured_profile_routes(&capabilities, network.as_str()); let routes = build_configured_profile_routes(&capabilities, network.as_str());
@@ -268,7 +272,7 @@ fn build_configured_profile_routes(capabilities: &RouteCapabilities, network: &s
network, network,
&[ &[
(capabilities.has_yellowstone, yellowstone_reason), (capabilities.has_yellowstone, yellowstone_reason),
(capabilities.has_get_transaction, crate::RawIngestRouteUnavailableReason::MissingHttpGetTransaction), (capabilities.yellowstone_has_get_block, crate::RawIngestRouteUnavailableReason::MissingHttpGetBlock),
], ],
), ),
route_with_requirements( route_with_requirements(

549
crates/ksp-app-raw-transaction-ingest-desk/src/route_runtime.rs
View File

@@ -1,29 +1,41 @@
// file: crates/ksp-app-raw-transaction-ingest-desk/src/route_runtime.rs // file: crates/ksp-app-raw-transaction-ingest-desk/src/route_runtime.rs
// version: 4 // version: 5
//! Mono-route Store and Worker lifecycle owned by Raw Transaction Ingest Desk. //! Multi-route Store and independent Worker lifecycle owned by Raw Transaction Ingest Desk.
const ROUTE_STOP_CLEANUP_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(2); const ROUTE_STOP_CLEANUP_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(2);
const STORE_RECLAIM_POLL_INTERVAL: std::time::Duration = std::time::Duration::from_millis(5); const STORE_RECLAIM_POLL_INTERVAL: std::time::Duration = std::time::Duration::from_millis(5);
const STORE_RECLAIM_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(1); const STORE_RECLAIM_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(1);
struct RouteRuntimeInner { struct RouteRuntimeInner {
last_terminal: std::option::Option<crate::RawIngestRouteRuntimeDto>, last_terminals: std::vec::Vec<crate::RawIngestRouteRuntimeDto>,
network: std::option::Option<String>,
next_sequence: u64, next_sequence: u64,
slot: RouteRuntimeSlot, routes: std::vec::Vec<RouteRuntimeSlot>,
store: std::option::Option<std::sync::Arc<ksp_store_lib::Store>>,
store_closing_token: std::option::Option<u64>,
store_opening_token: std::option::Option<u64>,
} }
/// Shared mono-route runtime state. `pre.008` admits at most one active or starting Worker. /// Shared multi-route runtime state. Each route owns an independent Worker while same-network routes share one Store.
pub(crate) struct RouteRuntimeState { pub(crate) struct RouteRuntimeState {
inner: std::sync::Mutex<RouteRuntimeInner>, inner: std::sync::Mutex<RouteRuntimeInner>,
} }
impl crate::RouteRuntimeState { impl crate::RouteRuntimeState {
/// Creates an idle mono-route runtime state. /// Creates an idle multi-route runtime state without Store or Worker ownership.
#[must_use] #[must_use]
pub(crate) fn new() -> Self { pub(crate) fn new() -> Self {
return Self { return Self {
inner: std::sync::Mutex::new(RouteRuntimeInner { last_terminal: std::option::Option::None, next_sequence: 0, slot: RouteRuntimeSlot::Idle }), inner: std::sync::Mutex::new(RouteRuntimeInner {
last_terminals: std::vec::Vec::new(),
network: std::option::Option::None,
next_sequence: 0,
routes: std::vec::Vec::new(),
store: std::option::Option::None,
store_closing_token: std::option::Option::None,
store_opening_token: std::option::Option::None,
}),
}; };
} }
@@ -33,14 +45,34 @@ impl crate::RouteRuntimeState {
std::result::Result::Ok(value) => value, std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return std::result::Result::Err(runtime_lock_error()), std::result::Result::Err(_) => return std::result::Result::Err(runtime_lock_error()),
}; };
if !matches!(inner.slot, RouteRuntimeSlot::Idle) { if inner.store_closing_token.is_some() {
return std::result::Result::Err(ksp_core_lib::Error::new( return std::result::Result::Err(ksp_core_lib::Error::new(
crate::ERROR_CODE_ROUTE_RUNTIME_ACTIVE, crate::ERROR_CODE_ROUTE_RUNTIME_ACTIVE,
"Raw Transaction Ingest Desk already owns one mono-route Worker runtime", "Raw Transaction Ingest Desk Store is still closing after the previous route set",
)); ));
} }
let sequence = inner.next_sequence.checked_add(1); if inner.routes.iter().any(|slot| return slot.matches_identity(prepared.profile_id.as_str(), prepared.route_id)) {
let sequence = match sequence { return std::result::Result::Err(ksp_core_lib::Error::new(
crate::ERROR_CODE_ROUTE_RUNTIME_ACTIVE,
"Raw Transaction Ingest Desk already owns one Worker for the selected logical route",
));
}
if let std::option::Option::Some(network) = inner.network.as_deref()
&& network != prepared.network.as_str()
{
return std::result::Result::Err(ksp_core_lib::Error::new(
crate::ERROR_CODE_ROUTE_RUNTIME_NETWORK_MISMATCH,
"Raw Transaction Ingest Desk cannot share one Store across different logical networks",
));
}
let needs_store_open = inner.store.is_none();
if needs_store_open && inner.store_opening_token.is_some() {
return std::result::Result::Err(ksp_core_lib::Error::new(
crate::ERROR_CODE_ROUTE_RUNTIME_ACTIVE,
"Raw Transaction Ingest Desk same-network Store is still opening",
));
}
let sequence = match inner.next_sequence.checked_add(1) {
std::option::Option::Some(value) => value, std::option::Option::Some(value) => value,
std::option::Option::None => { std::option::Option::None => {
return std::result::Result::Err(ksp_core_lib::Error::new( return std::result::Result::Err(ksp_core_lib::Error::new(
@@ -66,10 +98,55 @@ impl crate::RouteRuntimeState {
profile_id: prepared.profile_id.clone(), profile_id: prepared.profile_id.clone(),
route_id: prepared.route_id, route_id: prepared.route_id,
}; };
inner.last_terminal = std::option::Option::None; if inner.network.is_none() {
inner.last_terminals.clear();
inner.network = std::option::Option::Some(prepared.network.clone());
}
inner.last_terminals.retain(|terminal| return !identity.matches_dto(terminal));
inner.next_sequence = sequence; inner.next_sequence = sequence;
inner.slot = RouteRuntimeSlot::Starting { token: sequence }; if needs_store_open {
return std::result::Result::Ok(RouteRuntimeReservation { identity, token: sequence, worker_id }); inner.store_opening_token = std::option::Option::Some(sequence);
}
inner.routes.push(RouteRuntimeSlot::Starting { identity: identity.clone(), token: sequence });
return std::result::Result::Ok(RouteRuntimeReservation { identity, needs_store_open, token: sequence, worker_id });
}
fn install_store(&self, token: u64, store: std::sync::Arc<ksp_store_lib::Store>) -> ksp_core_lib::Result<()> {
let inner = self.inner.lock();
let mut inner = match inner {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return std::result::Result::Err(runtime_lock_error()),
};
if inner.store_opening_token != std::option::Option::Some(token) || inner.store.is_some() {
return std::result::Result::Err(ksp_core_lib::Error::new(
crate::ERROR_CODE_ROUTE_RUNTIME_STATE_INVALID,
"Raw Transaction Ingest Desk shared Store reservation changed before installation",
));
}
inner.store = std::option::Option::Some(store);
inner.store_opening_token = std::option::Option::None;
return std::result::Result::Ok(());
}
fn shared_store(&self, expected_network: &str) -> ksp_core_lib::Result<std::sync::Arc<ksp_store_lib::Store>> {
let inner = self.inner.lock();
let inner = match inner {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return std::result::Result::Err(runtime_lock_error()),
};
if inner.network.as_deref() != std::option::Option::Some(expected_network) {
return std::result::Result::Err(ksp_core_lib::Error::new(
crate::ERROR_CODE_ROUTE_RUNTIME_NETWORK_MISMATCH,
"Raw Transaction Ingest Desk shared Store network no longer matches the selected route",
));
}
return match &inner.store {
std::option::Option::Some(store) => std::result::Result::Ok(std::sync::Arc::clone(store)),
std::option::Option::None => std::result::Result::Err(ksp_core_lib::Error::new(
crate::ERROR_CODE_ROUTE_RUNTIME_STATE_INVALID,
"Raw Transaction Ingest Desk shared Store is not installed",
)),
};
} }
fn activate( fn activate(
@@ -82,117 +159,226 @@ impl crate::RouteRuntimeState {
std::result::Result::Ok(value) => value, std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return std::result::Result::Err(runtime_lock_error()), std::result::Result::Err(_) => return std::result::Result::Err(runtime_lock_error()),
}; };
match &inner.slot { let position = inner.routes.iter().position(|slot| return slot.starting_token() == std::option::Option::Some(reservation.token));
RouteRuntimeSlot::Starting { token } if *token == reservation.token => {}, let position = match position {
RouteRuntimeSlot::Idle | RouteRuntimeSlot::Starting { .. } | RouteRuntimeSlot::Active { .. } => { std::option::Option::Some(value) => value,
std::option::Option::None => {
return std::result::Result::Err(ksp_core_lib::Error::new( return std::result::Result::Err(ksp_core_lib::Error::new(
crate::ERROR_CODE_ROUTE_RUNTIME_STATE_INVALID, crate::ERROR_CODE_ROUTE_RUNTIME_STATE_INVALID,
"Raw Transaction Ingest Desk mono-route runtime reservation changed before Worker activation", "Raw Transaction Ingest Desk route reservation changed before Worker activation",
)); ));
}, },
} };
let state = project_worker_state(handle.snapshot_source().current().worker_snapshot().state()); let state = project_worker_state(handle.snapshot_source().current().worker_snapshot().state());
inner.slot = RouteRuntimeSlot::Active { handle, identity: reservation.identity.clone(), token: reservation.token }; inner.routes[position] = RouteRuntimeSlot::Active { handle, identity: reservation.identity.clone(), token: reservation.token };
return std::result::Result::Ok(reservation.identity.dto(state)); return std::result::Result::Ok(reservation.identity.dto(state));
} }
fn rollback(&self, token: u64) { fn rollback(&self, token: u64) -> ksp_core_lib::Result<std::option::Option<std::sync::Arc<ksp_store_lib::Store>>> {
let inner = self.inner.lock(); let inner = self.inner.lock();
let mut inner = match inner { let mut inner = match inner {
std::result::Result::Ok(value) => value, std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return, std::result::Result::Err(_) => return std::result::Result::Err(runtime_lock_error()),
}; };
if matches!(&inner.slot, RouteRuntimeSlot::Starting { token: current, .. } if *current == token) { inner.routes.retain(|slot| return slot.token() != token);
inner.slot = RouteRuntimeSlot::Idle; if inner.store_opening_token == std::option::Option::Some(token) {
inner.store_opening_token = std::option::Option::None;
} }
} if inner.routes.is_empty() {
let store = inner.store.take();
fn finish(&self, token: u64, terminal: crate::RawIngestRouteRuntimeDto) { if store.is_some() {
let inner = self.inner.lock(); inner.store_closing_token = std::option::Option::Some(token);
let mut inner = match inner { } else {
std::result::Result::Ok(value) => value, inner.network = std::option::Option::None;
std::result::Result::Err(_) => return,
};
if matches!(&inner.slot, RouteRuntimeSlot::Active { token: current, .. } if *current == token) {
inner.last_terminal = std::option::Option::Some(terminal);
inner.slot = RouteRuntimeSlot::Idle;
}
}
/// Requests cooperative Stop, waits for terminal Worker state and returns after the terminal monitor has released the mono-route slot.
pub(crate) async fn stop_and_wait(&self) -> ksp_core_lib::Result<crate::RawIngestRouteRuntimeDto> {
let active = {
let inner = self.inner.lock();
let inner = match inner {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return std::result::Result::Err(runtime_lock_error()),
};
match &inner.slot {
RouteRuntimeSlot::Active { handle, identity, .. } => std::result::Result::Ok((handle.clone(), identity.clone())),
RouteRuntimeSlot::Idle => match &inner.last_terminal {
std::option::Option::Some(terminal) => return std::result::Result::Ok(terminal.clone()),
std::option::Option::None => std::result::Result::Err(ksp_core_lib::Error::new(
crate::ERROR_CODE_ROUTE_RUNTIME_NOT_ACTIVE,
"Raw Transaction Ingest Desk has no active mono-route Worker to stop",
)),
},
RouteRuntimeSlot::Starting { .. } => std::result::Result::Err(ksp_core_lib::Error::new(
crate::ERROR_CODE_ROUTE_RUNTIME_NOT_ACTIVE,
"Raw Transaction Ingest Desk mono-route Worker is still starting and cannot be stopped yet",
)),
} }
return std::result::Result::Ok(store);
}
return std::result::Result::Ok(std::option::Option::None);
}
fn finish(&self, token: u64, terminal: crate::RawIngestRouteRuntimeDto) -> ksp_core_lib::Result<std::option::Option<std::sync::Arc<ksp_store_lib::Store>>> {
let inner = self.inner.lock();
let mut inner = match inner {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return std::result::Result::Err(runtime_lock_error()),
}; };
let (handle, identity) = match active { let before = inner.routes.len();
inner.routes.retain(|slot| return slot.token() != token);
if inner.routes.len() == before {
return std::result::Result::Ok(std::option::Option::None);
}
inner.last_terminals.retain(|previous| return previous.profile_id != terminal.profile_id || previous.route_id != terminal.route_id);
inner.last_terminals.push(terminal);
if inner.routes.is_empty() {
let store = inner.store.take();
if store.is_some() {
inner.store_closing_token = std::option::Option::Some(token);
} else {
inner.network = std::option::Option::None;
}
return std::result::Result::Ok(store);
}
return std::result::Result::Ok(std::option::Option::None);
}
fn complete_store_close(&self, token: u64) {
let inner = self.inner.lock();
let mut inner = match inner {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return,
};
if inner.store_closing_token == std::option::Option::Some(token) {
inner.store_closing_token = std::option::Option::None;
if inner.routes.is_empty() && inner.store.is_none() && inner.store_opening_token.is_none() {
inner.network = std::option::Option::None;
}
}
}
/// Requests cooperative Stop for one exact logical route and waits for its Worker cleanup.
pub(crate) async fn stop_and_wait(&self, request: &crate::RawIngestRouteStopRequestDto) -> ksp_core_lib::Result<crate::RawIngestRouteRuntimeDto> {
let target = self.stop_target(request);
let target = match target {
std::result::Result::Ok(value) => value, std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error), std::result::Result::Err(error) => return std::result::Result::Err(error),
}; };
let (handle, identity, token) = match target {
RouteStopTarget::Active { handle, identity, token } => (handle, identity, token),
RouteStopTarget::Terminal(value) => return std::result::Result::Ok(value),
RouteStopTarget::TerminalClosing => return self.wait_terminal_store_cleanup(request).await,
};
let _first_stop_request = handle.request_stop(); let _first_stop_request = handle.request_stop();
let terminal = handle.wait_terminal().await; if let std::result::Result::Err(error) = handle.wait_terminal().await {
let terminal = match terminal { return std::result::Result::Err(
ksp_core_lib::Error::new(crate::ERROR_CODE_ROUTE_RUNTIME_STATE_INVALID, "Cannot observe terminal Raw Transaction Ingest Desk Worker state")
.with_source(error),
);
}
return self.wait_route_cleanup(token, &identity).await;
}
fn stop_target(&self, request: &crate::RawIngestRouteStopRequestDto) -> ksp_core_lib::Result<RouteStopTarget> {
let inner = self.inner.lock();
let inner = match inner {
std::result::Result::Ok(value) => value, std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => { std::result::Result::Err(_) => return std::result::Result::Err(runtime_lock_error()),
return std::result::Result::Err( };
ksp_core_lib::Error::new(crate::ERROR_CODE_ROUTE_RUNTIME_STATE_INVALID, "Cannot observe terminal Raw Transaction Ingest Desk Worker state") let slot = inner.routes.iter().find(|slot| return slot.matches_identity(request.profile_id.as_str(), request.route_id));
.with_source(error), return match slot {
); std::option::Option::Some(RouteRuntimeSlot::Active { handle, identity, token }) => {
std::result::Result::Ok(RouteStopTarget::Active { handle: handle.clone(), identity: identity.clone(), token: *token })
},
std::option::Option::Some(RouteRuntimeSlot::Starting { .. }) => std::result::Result::Err(ksp_core_lib::Error::new(
crate::ERROR_CODE_ROUTE_RUNTIME_NOT_ACTIVE,
"Raw Transaction Ingest Desk route Worker is still starting and cannot be stopped yet",
)),
std::option::Option::None => {
let terminal =
inner.last_terminals.iter().find(|terminal| return terminal.profile_id == request.profile_id && terminal.route_id == request.route_id);
match terminal {
std::option::Option::Some(value) if inner.store_closing_token.is_none() => {
std::result::Result::Ok(RouteStopTarget::Terminal(value.clone()))
},
std::option::Option::Some(_) => std::result::Result::Ok(RouteStopTarget::TerminalClosing),
std::option::Option::None => std::result::Result::Err(ksp_core_lib::Error::new(
crate::ERROR_CODE_ROUTE_RUNTIME_NOT_ACTIVE,
"Raw Transaction Ingest Desk has no active Worker for the selected logical route",
)),
}
}, },
}; };
}
async fn wait_route_cleanup(&self, token: u64, identity: &RouteRuntimeIdentity) -> ksp_core_lib::Result<crate::RawIngestRouteRuntimeDto> {
let started = std::time::Instant::now(); let started = std::time::Instant::now();
loop { loop {
let idle = self.is_idle(); let cleanup = self.route_cleanup_complete(token, identity);
let idle = match idle { let cleanup = match cleanup {
std::result::Result::Ok(value) => value, std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error), std::result::Result::Err(error) => return std::result::Result::Err(error),
}; };
if idle { if let std::option::Option::Some(value) = cleanup {
return std::result::Result::Ok(identity.dto(project_worker_state(terminal))); return std::result::Result::Ok(value);
} }
if started.elapsed() >= ROUTE_STOP_CLEANUP_TIMEOUT { if started.elapsed() >= ROUTE_STOP_CLEANUP_TIMEOUT {
return std::result::Result::Err(ksp_core_lib::Error::new( return std::result::Result::Err(ksp_core_lib::Error::new(
crate::ERROR_CODE_ROUTE_RUNTIME_STORE_SHUTDOWN_FAILED, crate::ERROR_CODE_ROUTE_RUNTIME_STORE_SHUTDOWN_FAILED,
"Raw Transaction Ingest Desk terminal cleanup did not release the mono-route slot", "Raw Transaction Ingest Desk terminal cleanup did not release the selected route",
)); ));
} }
tokio::time::sleep(STORE_RECLAIM_POLL_INTERVAL).await; tokio::time::sleep(STORE_RECLAIM_POLL_INTERVAL).await;
} }
} }
fn is_idle(&self) -> ksp_core_lib::Result<bool> { async fn wait_terminal_store_cleanup(&self, request: &crate::RawIngestRouteStopRequestDto) -> ksp_core_lib::Result<crate::RawIngestRouteRuntimeDto> {
let started = std::time::Instant::now();
loop {
let terminal = self.late_terminal_after_store_cleanup(request);
let terminal = match terminal {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
if let std::option::Option::Some(value) = terminal {
return std::result::Result::Ok(value);
}
if started.elapsed() >= ROUTE_STOP_CLEANUP_TIMEOUT {
return std::result::Result::Err(ksp_core_lib::Error::new(
crate::ERROR_CODE_ROUTE_RUNTIME_STORE_SHUTDOWN_FAILED,
"Raw Transaction Ingest Desk final shared Store cleanup did not complete for the selected terminal route",
));
}
tokio::time::sleep(STORE_RECLAIM_POLL_INTERVAL).await;
}
}
fn late_terminal_after_store_cleanup(
&self,
request: &crate::RawIngestRouteStopRequestDto,
) -> ksp_core_lib::Result<std::option::Option<crate::RawIngestRouteRuntimeDto>> {
let inner = self.inner.lock(); let inner = self.inner.lock();
let inner = match inner { let inner = match inner {
std::result::Result::Ok(value) => value, std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return std::result::Result::Err(runtime_lock_error()), std::result::Result::Err(_) => return std::result::Result::Err(runtime_lock_error()),
}; };
return std::result::Result::Ok(matches!(inner.slot, RouteRuntimeSlot::Idle)); let terminal = inner.last_terminals.iter().find(|terminal| return terminal.profile_id == request.profile_id && terminal.route_id == request.route_id);
let terminal = match terminal {
std::option::Option::Some(value) => value,
std::option::Option::None => {
return std::result::Result::Err(ksp_core_lib::Error::new(
crate::ERROR_CODE_ROUTE_RUNTIME_NOT_ACTIVE,
"Raw Transaction Ingest Desk terminal route is no longer retained",
));
},
};
if inner.store_closing_token.is_some() {
return std::result::Result::Ok(std::option::Option::None);
}
return std::result::Result::Ok(std::option::Option::Some(terminal.clone()));
}
fn route_cleanup_complete(
&self,
token: u64,
identity: &RouteRuntimeIdentity,
) -> ksp_core_lib::Result<std::option::Option<crate::RawIngestRouteRuntimeDto>> {
let inner = self.inner.lock();
let inner = match inner {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return std::result::Result::Err(runtime_lock_error()),
};
if inner.routes.iter().any(|slot| return slot.token() == token) || inner.store_closing_token == std::option::Option::Some(token) {
return std::result::Result::Ok(std::option::Option::None);
}
let terminal = inner.last_terminals.iter().find(|value| return identity.matches_dto(value));
return std::result::Result::Ok(terminal.cloned());
} }
} }
/// Launch ownership returned only after Store readiness, Worker creation and handle installation have succeeded. /// Launch ownership returned only after shared Store readiness, Worker creation and route-handle installation have succeeded.
pub(crate) struct RouteRuntimeLaunch { pub(crate) struct RouteRuntimeLaunch {
acknowledgement: crate::RawIngestRouteRuntimeDto, acknowledgement: crate::RawIngestRouteRuntimeDto,
handle: ksp_worker_raw_transaction_ingest_lib::RawTransactionIngestHandle, handle: ksp_worker_raw_transaction_ingest_lib::RawTransactionIngestHandle,
runtime_state: std::sync::Arc<crate::RouteRuntimeState>, runtime_state: std::sync::Arc<crate::RouteRuntimeState>,
store: std::sync::Arc<ksp_store_lib::Store>,
token: u64, token: u64,
} }
@@ -203,7 +389,7 @@ impl crate::RouteRuntimeLaunch {
return self.acknowledgement.clone(); return self.acknowledgement.clone();
} }
/// Waits for terminal Worker state, explicitly closes Store after the Worker releases its `Arc`, then frees the mono-route slot. /// Waits for this Worker terminal state, removes only its route and closes Store only after the final same-network route terminates.
pub(crate) async fn monitor(self) { pub(crate) async fn monitor(self) {
let terminal = self.handle.wait_terminal().await; let terminal = self.handle.wait_terminal().await;
let terminal_state = match terminal { let terminal_state = match terminal {
@@ -212,7 +398,7 @@ impl crate::RouteRuntimeLaunch {
target: crate::TRACING_TARGET, target: crate::TRACING_TARGET,
domain = crate::TRACING_DOMAIN_ROUTE_RUNTIME, domain = crate::TRACING_DOMAIN_ROUTE_RUNTIME,
worker_state = state.code(), worker_state = state.code(),
"Raw Transaction Ingest Desk mono-route Worker reached terminal state" "Raw Transaction Ingest Desk route Worker reached terminal state"
); );
project_worker_state(state) project_worker_state(state)
}, },
@@ -227,64 +413,99 @@ impl crate::RouteRuntimeLaunch {
crate::RawIngestRouteState::Faulted crate::RawIngestRouteState::Faulted
}, },
}; };
let close = close_store_arc(self.store).await;
if let std::result::Result::Err(error) = close {
ksp_logging_lib::warn!(
target: crate::TRACING_TARGET,
domain = crate::TRACING_DOMAIN_ROUTE_RUNTIME,
error_domain = error.code().domain(),
error_code = error.code().code(),
"Raw Transaction Ingest Desk Store shutdown failed after terminal Worker"
);
}
let mut terminal = self.acknowledgement; let mut terminal = self.acknowledgement;
terminal.state = terminal_state; terminal.state = terminal_state;
self.runtime_state.finish(self.token, terminal); let store = self.runtime_state.finish(self.token, terminal);
let store = match store {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => {
ksp_logging_lib::warn!(
target: crate::TRACING_TARGET,
domain = crate::TRACING_DOMAIN_ROUTE_RUNTIME,
error_domain = error.code().domain(),
error_code = error.code().code(),
"Raw Transaction Ingest Desk route cleanup state update failed"
);
return;
},
};
if let std::option::Option::Some(store) = store {
let close = close_store_arc(store).await;
if let std::result::Result::Err(error) = close {
ksp_logging_lib::warn!(
target: crate::TRACING_TARGET,
domain = crate::TRACING_DOMAIN_ROUTE_RUNTIME,
error_domain = error.code().domain(),
error_code = error.code().code(),
"Raw Transaction Ingest Desk shared Store shutdown failed after final Worker"
);
}
self.runtime_state.complete_store_close(self.token);
}
} }
} }
/// Opens one Store, starts one exact Worker route and atomically installs its handle in the mono-route application slot. /// Starts one exact route Worker and reuses the same ready Store for any already-active same-network route.
pub(crate) async fn start_route_runtime( pub(crate) async fn start_route_runtime(
runtime_state: std::sync::Arc<crate::RouteRuntimeState>, runtime_state: std::sync::Arc<crate::RouteRuntimeState>,
prepared: crate::PreparedRouteStart, prepared: crate::PreparedRouteStart,
) -> ksp_core_lib::Result<crate::RouteRuntimeLaunch> { ) -> ksp_core_lib::Result<crate::RouteRuntimeLaunch> {
let reservation = runtime_state.reserve(&prepared); let reservation = match runtime_state.reserve(&prepared) {
let reservation = match reservation {
std::result::Result::Ok(value) => value, std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error), std::result::Result::Err(error) => return std::result::Result::Err(error),
}; };
let store = ksp_store_lib::Store::open(prepared.store_settings).await; let store = if reservation.needs_store_open {
let store = match store { let store = ksp_store_lib::Store::open(prepared.store_settings).await;
std::result::Result::Ok(value) => value, let store = match store {
std::result::Result::Err(error) => { std::result::Result::Ok(value) => value,
runtime_state.rollback(reservation.token); std::result::Result::Err(error) => {
return std::result::Result::Err( let _rollback = runtime_state.rollback(reservation.token);
ksp_core_lib::Error::new(crate::ERROR_CODE_ROUTE_RUNTIME_START_FAILED, "Cannot open Raw Transaction Ingest Desk Store runtime") return std::result::Result::Err(
ksp_core_lib::Error::new(crate::ERROR_CODE_ROUTE_RUNTIME_START_FAILED, "Cannot open Raw Transaction Ingest Desk shared Store runtime")
.with_source(error),
);
},
};
let health = store.health().await;
if health.state() != ksp_store_lib::StoreHealthState::Ready {
let close = store.close().await;
let _rollback = runtime_state.rollback(reservation.token);
if let std::result::Result::Err(error) = close {
return std::result::Result::Err(
ksp_core_lib::Error::new(
crate::ERROR_CODE_ROUTE_RUNTIME_STORE_SHUTDOWN_FAILED,
"Cannot close non-ready Raw Transaction Ingest Desk shared Store runtime",
)
.with_source(error), .with_source(error),
); );
}, }
}; return std::result::Result::Err(ksp_core_lib::Error::new(
let health = store.health().await; crate::ERROR_CODE_ROUTE_RUNTIME_STORE_NOT_READY,
if health.state() != ksp_store_lib::StoreHealthState::Ready { "Raw Transaction Ingest Desk shared Store readiness probe did not prove Ready",
let close = store.close().await; ));
runtime_state.rollback(reservation.token);
if let std::result::Result::Err(error) = close {
return std::result::Result::Err(
ksp_core_lib::Error::new(
crate::ERROR_CODE_ROUTE_RUNTIME_STORE_SHUTDOWN_FAILED,
"Cannot close non-ready Raw Transaction Ingest Desk Store runtime",
)
.with_source(error),
);
} }
return std::result::Result::Err(ksp_core_lib::Error::new( let store = std::sync::Arc::new(store);
crate::ERROR_CODE_ROUTE_RUNTIME_STORE_NOT_READY, if let std::result::Result::Err(error) = runtime_state.install_store(reservation.token, std::sync::Arc::clone(&store)) {
"Raw Transaction Ingest Desk Store readiness probe did not prove Ready", let close = close_store_arc(store).await;
)); let _rollback = runtime_state.rollback(reservation.token);
} if let std::result::Result::Err(close_error) = close {
return std::result::Result::Err(close_error);
}
return std::result::Result::Err(error);
}
store
} else {
std::mem::drop(prepared.store_settings);
match runtime_state.shared_store(prepared.network.as_str()) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => {
let _rollback = runtime_state.rollback(reservation.token);
return std::result::Result::Err(error);
},
}
};
let network = store.runtime_snapshot().network().clone(); let network = store.runtime_snapshot().network().clone();
let settings = ksp_worker_raw_transaction_ingest_lib::RawTransactionIngestSettings::with_defaults(network, reservation.worker_id.clone()); let settings = ksp_worker_raw_transaction_ingest_lib::RawTransactionIngestSettings::with_defaults(network, reservation.worker_id.clone());
let store = std::sync::Arc::new(store);
let handle = ksp_worker_raw_transaction_ingest_lib::RawTransactionIngestWorker::start_with_runtime_resources( let handle = ksp_worker_raw_transaction_ingest_lib::RawTransactionIngestWorker::start_with_runtime_resources(
settings, settings,
std::sync::Arc::clone(&store), std::sync::Arc::clone(&store),
@@ -293,13 +514,21 @@ pub(crate) async fn start_route_runtime(
let handle = match handle { let handle = match handle {
std::result::Result::Ok(value) => value, std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => { std::result::Result::Err(error) => {
let close = close_store_arc(store).await; let cleanup_store = runtime_state.rollback(reservation.token);
runtime_state.rollback(reservation.token); let cleanup_store = match cleanup_store {
if let std::result::Result::Err(close_error) = close { std::result::Result::Ok(value) => value,
return std::result::Result::Err(close_error); std::result::Result::Err(rollback_error) => return std::result::Result::Err(rollback_error),
};
std::mem::drop(store);
if let std::option::Option::Some(value) = cleanup_store {
let close = close_store_arc(value).await;
runtime_state.complete_store_close(reservation.token);
if let std::result::Result::Err(close_error) = close {
return std::result::Result::Err(close_error);
}
} }
return std::result::Result::Err( return std::result::Result::Err(
ksp_core_lib::Error::new(crate::ERROR_CODE_ROUTE_RUNTIME_START_FAILED, "Cannot start Raw Transaction Ingest Desk Worker runtime") ksp_core_lib::Error::new(crate::ERROR_CODE_ROUTE_RUNTIME_START_FAILED, "Cannot start Raw Transaction Ingest Desk route Worker runtime")
.with_source(error), .with_source(error),
); );
}, },
@@ -310,14 +539,23 @@ pub(crate) async fn start_route_runtime(
std::result::Result::Err(error) => { std::result::Result::Err(error) => {
let _stop_requested = handle.request_stop(); let _stop_requested = handle.request_stop();
let _terminal = handle.wait_terminal().await; let _terminal = handle.wait_terminal().await;
let close = close_store_arc(store).await; let cleanup_store = runtime_state.rollback(reservation.token);
runtime_state.rollback(reservation.token); let cleanup_store = match cleanup_store {
if let std::result::Result::Err(close_error) = close { std::result::Result::Ok(value) => value,
return std::result::Result::Err(close_error); std::result::Result::Err(rollback_error) => return std::result::Result::Err(rollback_error),
};
std::mem::drop(store);
if let std::option::Option::Some(value) = cleanup_store {
let close = close_store_arc(value).await;
runtime_state.complete_store_close(reservation.token);
if let std::result::Result::Err(close_error) = close {
return std::result::Result::Err(close_error);
}
} }
return std::result::Result::Err(error); return std::result::Result::Err(error);
}, },
}; };
std::mem::drop(store);
ksp_logging_lib::info!( ksp_logging_lib::info!(
target: crate::TRACING_TARGET, target: crate::TRACING_TARGET,
domain = crate::TRACING_DOMAIN_ROUTE_RUNTIME, domain = crate::TRACING_DOMAIN_ROUTE_RUNTIME,
@@ -325,9 +563,9 @@ pub(crate) async fn start_route_runtime(
network = reservation.identity.network.as_str(), network = reservation.identity.network.as_str(),
route_id = reservation.identity.route_id.as_str(), route_id = reservation.identity.route_id.as_str(),
commitment = reservation.identity.commitment.as_str(), commitment = reservation.identity.commitment.as_str(),
"Raw Transaction Ingest Desk mono-route Worker handle installed" "Raw Transaction Ingest Desk independent route Worker handle installed on shared Store"
); );
return std::result::Result::Ok(crate::RouteRuntimeLaunch { acknowledgement, handle, runtime_state, store, token: reservation.token }); return std::result::Result::Ok(crate::RouteRuntimeLaunch { acknowledgement, handle, runtime_state, token: reservation.token });
} }
#[derive(Clone)] #[derive(Clone)]
@@ -350,17 +588,32 @@ impl RouteRuntimeIdentity {
state, state,
}; };
} }
fn matches_dto(&self, value: &crate::RawIngestRouteRuntimeDto) -> bool {
return self.profile_id == value.profile_id && self.route_id == value.route_id;
}
}
enum RouteStopTarget {
Active {
handle: ksp_worker_raw_transaction_ingest_lib::RawTransactionIngestHandle,
identity: RouteRuntimeIdentity,
token: u64,
},
Terminal(crate::RawIngestRouteRuntimeDto),
TerminalClosing,
} }
struct RouteRuntimeReservation { struct RouteRuntimeReservation {
identity: RouteRuntimeIdentity, identity: RouteRuntimeIdentity,
needs_store_open: bool,
token: u64, token: u64,
worker_id: ksp_worker_api::WorkerId, worker_id: ksp_worker_api::WorkerId,
} }
enum RouteRuntimeSlot { enum RouteRuntimeSlot {
Idle,
Starting { Starting {
identity: RouteRuntimeIdentity,
token: u64, token: u64,
}, },
Active { Active {
@@ -370,6 +623,28 @@ enum RouteRuntimeSlot {
}, },
} }
impl RouteRuntimeSlot {
fn matches_identity(&self, profile_id: &str, route_id: crate::RawIngestRouteId) -> bool {
let identity = match self {
Self::Starting { identity, .. } | Self::Active { identity, .. } => identity,
};
return identity.profile_id == profile_id && identity.route_id == route_id;
}
fn starting_token(&self) -> std::option::Option<u64> {
return match self {
Self::Starting { token, .. } => std::option::Option::Some(*token),
Self::Active { .. } => std::option::Option::None,
};
}
fn token(&self) -> u64 {
return match self {
Self::Starting { token, .. } | Self::Active { token, .. } => *token,
};
}
}
async fn close_store_arc(store: std::sync::Arc<ksp_store_lib::Store>) -> ksp_core_lib::Result<()> { async fn close_store_arc(store: std::sync::Arc<ksp_store_lib::Store>) -> ksp_core_lib::Result<()> {
let started = std::time::Instant::now(); let started = std::time::Instant::now();
let mut store = store; let mut store = store;
@@ -382,20 +657,18 @@ async fn close_store_arc(store: std::sync::Arc<ksp_store_lib::Store>) -> ksp_cor
std::result::Result::Err(error) => std::result::Result::Err( std::result::Result::Err(error) => std::result::Result::Err(
ksp_core_lib::Error::new( ksp_core_lib::Error::new(
crate::ERROR_CODE_ROUTE_RUNTIME_STORE_SHUTDOWN_FAILED, crate::ERROR_CODE_ROUTE_RUNTIME_STORE_SHUTDOWN_FAILED,
"Cannot close Raw Transaction Ingest Desk Store runtime", "Cannot close Raw Transaction Ingest Desk shared Store runtime",
) )
.with_source(error), .with_source(error),
), ),
}; };
}, },
std::result::Result::Err(shared) => { std::result::Result::Err(shared) => store = shared,
store = shared;
},
} }
if started.elapsed() >= STORE_RECLAIM_TIMEOUT { if started.elapsed() >= STORE_RECLAIM_TIMEOUT {
return std::result::Result::Err(ksp_core_lib::Error::new( return std::result::Result::Err(ksp_core_lib::Error::new(
crate::ERROR_CODE_ROUTE_RUNTIME_STORE_SHUTDOWN_FAILED, crate::ERROR_CODE_ROUTE_RUNTIME_STORE_SHUTDOWN_FAILED,
"Raw Transaction Ingest Desk Store remained shared after terminal Worker", "Raw Transaction Ingest Desk shared Store remained referenced after final Worker",
)); ));
} }
tokio::time::sleep(STORE_RECLAIM_POLL_INTERVAL).await; tokio::time::sleep(STORE_RECLAIM_POLL_INTERVAL).await;
@@ -414,7 +687,7 @@ fn project_worker_state(state: ksp_worker_api::WorkerState) -> crate::RawIngestR
} }
fn runtime_lock_error() -> ksp_core_lib::Error { fn runtime_lock_error() -> ksp_core_lib::Error {
return ksp_core_lib::Error::new(crate::ERROR_CODE_APP_STATE_LOCK_FAILED, "Raw Transaction Ingest Desk mono-route runtime state lock is poisoned"); return ksp_core_lib::Error::new(crate::ERROR_CODE_APP_STATE_LOCK_FAILED, "Raw Transaction Ingest Desk multi-route runtime state lock is poisoned");
} }
#[cfg(test)] #[cfg(test)]

12
crates/ksp-app-raw-transaction-ingest-desk/src/route_start.rs
View File

@@ -1,5 +1,5 @@
// file: crates/ksp-app-raw-transaction-ingest-desk/src/route_start.rs // file: crates/ksp-app-raw-transaction-ingest-desk/src/route_start.rs
// version: 3 // version: 4
//! Start-time Config revalidation and exact Transport/Store resource reconstruction for route validation and runtime launch. //! Start-time Config revalidation and exact Transport/Store resource reconstruction for route validation and runtime launch.
@@ -278,18 +278,22 @@ fn prepare_yellowstone(
std::result::Result::Ok(value) => value, std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error), std::result::Result::Err(error) => return std::result::Result::Err(error),
}; };
let filter_name = ksp_onchain_transport_lib::YellowstoneSubscribeFilterName::new("raw-ingest-all-transactions"); let filter_name = ksp_onchain_transport_lib::YellowstoneSubscribeFilterName::new("raw-ingest-blocks");
let filter_name = match filter_name { let filter_name = match filter_name {
std::result::Result::Ok(value) => value, std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error), std::result::Result::Err(error) => return std::result::Result::Err(error),
}; };
let mut block_filter = ksp_onchain_transport_lib::YellowstoneSubscribeBlockFilter::new();
block_filter.set_include_transactions(std::option::Option::Some(false));
block_filter.set_include_accounts(std::option::Option::Some(false));
block_filter.set_include_entries(std::option::Option::Some(false));
let mut request = ksp_onchain_transport_lib::YellowstoneSubscribeRequest::new(); let mut request = ksp_onchain_transport_lib::YellowstoneSubscribeRequest::new();
let inserted = request.insert_transaction_filter(filter_name, ksp_onchain_transport_lib::YellowstoneSubscribeTransactionFilter::new()); let inserted = request.insert_block_filter(filter_name, block_filter);
if let std::result::Result::Err(error) = inserted { if let std::result::Result::Err(error) = inserted {
return std::result::Result::Err(error); return std::result::Result::Err(error);
} }
request.set_commitment(std::option::Option::Some(commitment)); request.set_commitment(std::option::Option::Some(commitment));
let role = single_http_role(transport.http_settings(), network, &["getTransaction"]); let role = single_http_role(transport.http_settings(), network, &["getBlock"]);
let role = match role { let role = match role {
std::result::Result::Ok(value) => value, std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error), std::result::Result::Err(error) => return std::result::Result::Err(error),

9
crates/ksp-app-raw-transaction-ingest-desk/src/tauri.rs
View File

@@ -1,5 +1,5 @@
// file: crates/ksp-app-raw-transaction-ingest-desk/src/tauri.rs // file: crates/ksp-app-raw-transaction-ingest-desk/src/tauri.rs
// version: 5 // version: 6
//! Tauri runtime assembly for the KSP Raw Transaction Ingest desktop application. //! Tauri runtime assembly for the KSP Raw Transaction Ingest desktop application.
@@ -164,8 +164,11 @@ async fn start_route(
} }
#[tauri::command] #[tauri::command]
async fn stop_route(state: tauri::State<'_, crate::AppState>) -> std::result::Result<crate::RawIngestRouteRuntimeDto, crate::CommandErrorDto> { async fn stop_route(
let result = state.stop_route().await; request: crate::RawIngestRouteStopRequestDto,
state: tauri::State<'_, crate::AppState>,
) -> std::result::Result<crate::RawIngestRouteRuntimeDto, crate::CommandErrorDto> {
let result = state.stop_route(&request).await;
return match result { return match result {
std::result::Result::Ok(value) => std::result::Result::Ok(value), std::result::Result::Ok(value) => std::result::Result::Ok(value),
std::result::Result::Err(error) => std::result::Result::Err(project_command_error("stop_route", crate::TRACING_DOMAIN_ROUTE_RUNTIME, &error)), std::result::Result::Err(error) => std::result::Result::Err(project_command_error("stop_route", crate::TRACING_DOMAIN_ROUTE_RUNTIME, &error)),

17
crates/ksp-app-raw-transaction-ingest-desk/tests/desktop_contract.rs
View File

@@ -1,5 +1,5 @@
// file: crates/ksp-app-raw-transaction-ingest-desk/tests/desktop_contract.rs // file: crates/ksp-app-raw-transaction-ingest-desk/tests/desktop_contract.rs
// version: 5 // version: 6
//! Desktop scaffold contract canaries for Raw Transaction Ingest Desk. //! Desktop scaffold contract canaries for Raw Transaction Ingest Desk.
@@ -186,17 +186,18 @@ fn pre_007_start_preflight_is_backend_owned_generation_bound_and_does_not_launch
} }
#[test] #[test]
fn pre_008_start_stop_commands_and_frontend_controls_are_mono_route_and_backend_owned() { fn pre_009_start_stop_controls_are_multi_route_targeted_and_backend_owned() {
let tauri = read_text(app_root().join("src/tauri.rs").as_path()); let tauri = read_text(app_root().join("src/tauri.rs").as_path());
for required in ["start_route", "stop_route", "launch.monitor().await"] { for required in ["start_route", "stop_route", "RawIngestRouteStopRequestDto", "launch.monitor().await"] {
assert!(tauri.contains(required), "missing pre.008 Tauri runtime marker {required}"); assert!(tauri.contains(required), "missing pre.009 Tauri runtime marker {required}");
} }
let html = read_text(app_root().join("frontend/main.html").as_path()); let html = read_text(app_root().join("frontend/main.html").as_path());
for required in ["id=\"routeCommitment\"", "id=\"routeRuntimeState\"", "id=\"routeRuntimeIdentity\"", "id=\"stopRoute\""] { for required in ["id=\"routeCommitment\"", "id=\"routeRuntimeState\"", "id=\"routeRuntimeIdentity\"", "Runtime multi-route / Store partagé"] {
assert!(html.contains(required), "missing pre.008 frontend runtime marker {required}"); assert!(html.contains(required), "missing pre.009 frontend runtime marker {required}");
} }
assert!(!html.contains("id=\"stopRoute\""));
let main = read_text(app_root().join("frontend/ts/main.ts").as_path()); let main = read_text(app_root().join("frontend/ts/main.ts").as_path());
for required in ["mono-route Start requested", "mono-route Stop requested", "start_route", "stop_route"] { for required in ["activeRuntimes", "targeted route Stop requested", "start_route", "stop_route", "RawIngestRouteStopRequestDto"] {
assert!(main.contains(required), "missing pre.008 frontend control marker {required}"); assert!(main.contains(required), "missing pre.009 frontend control marker {required}");
} }
} }

45
crates/ksp-app-raw-transaction-ingest-desk/tests/release_completeness.rs
View File

@@ -1,5 +1,5 @@
// file: crates/ksp-app-raw-transaction-ingest-desk/tests/release_completeness.rs // file: crates/ksp-app-raw-transaction-ingest-desk/tests/release_completeness.rs
// version: 7 // version: 8
//! Release-completeness canaries for Raw Transaction Ingest Desk Start-resource reconstruction. //! Release-completeness canaries for Raw Transaction Ingest Desk Start-resource reconstruction.
@@ -92,23 +92,35 @@ fn pre_007_start_revalidation_reuses_inventory_and_requires_profile_network_iden
} }
#[test] #[test]
fn pre_008_mono_route_slot_blocks_second_start_until_terminal_store_cleanup() { fn pre_009_multi_route_runtime_keeps_independent_workers_on_one_same_network_store() {
let runtime = read_text(app_root().join("src/route_runtime.rs").as_path()); let runtime = read_text(app_root().join("src/route_runtime.rs").as_path());
for required in [ for required in [
"RouteRuntimeSlot::Starting", "routes: std::vec::Vec<RouteRuntimeSlot>",
"RouteRuntimeSlot::Active", "store: std::option::Option<std::sync::Arc<ksp_store_lib::Store>>",
"ERROR_CODE_ROUTE_RUNTIME_ACTIVE", "ERROR_CODE_ROUTE_RUNTIME_NETWORK_MISMATCH",
"inner.last_terminal = std::option::Option::None", "already owns one Worker for the selected logical route",
"std::option::Option::Some(terminal) => return std::result::Result::Ok(terminal.clone())", "shared_store(prepared.network.as_str())",
"self.runtime_state.finish(self.token, terminal)", "self.runtime_state.finish(self.token, terminal)",
"Arc::try_unwrap", "Arc::try_unwrap",
] { ] {
assert!(runtime.contains(required), "missing mono-route ownership marker {required}"); assert!(runtime.contains(required), "missing pre.009 shared-Store marker {required}");
} }
assert!(!runtime.contains("RouteRuntimeSlot::Idle"));
} }
#[test] #[test]
fn pre_008_fix_004_yellowstone_hydration_uses_its_declared_transport_source_not_base_transport() { fn pre_009_targeted_stop_is_route_scoped_and_late_terminal_safe() {
let dto = read_text(app_root().join("src/dto_route.rs").as_path());
assert!(dto.contains("RawIngestRouteStopRequestDto"));
let runtime = read_text(app_root().join("src/route_runtime.rs").as_path());
assert!(runtime.contains("stop_and_wait(&self, request: &crate::RawIngestRouteStopRequestDto)"));
assert!(runtime.contains("terminal.profile_id == request.profile_id && terminal.route_id == request.route_id"));
let tauri = read_text(app_root().join("src/tauri.rs").as_path());
assert!(tauri.contains("request: crate::RawIngestRouteStopRequestDto"));
}
#[test]
fn pre_009_yellowstone_uses_block_subscription_and_one_get_block_path_instead_of_per_transaction_hydration() {
let route_start = read_text(app_root().join("src/route_start.rs").as_path()); let route_start = read_text(app_root().join("src/route_start.rs").as_path());
let start = route_start.find("fn prepare_yellowstone("); let start = route_start.find("fn prepare_yellowstone(");
let end = route_start.find("fn prepare_http_polling("); let end = route_start.find("fn prepare_http_polling(");
@@ -118,9 +130,14 @@ fn pre_008_fix_004_yellowstone_hydration_uses_its_declared_transport_source_not_
(std::option::Option::Some(start), std::option::Option::Some(end)) if start < end => &route_start[start..end], (std::option::Option::Some(start), std::option::Option::Some(end)) if start < end => &route_start[start..end],
_ => "", _ => "",
}; };
assert!(yellowstone.contains("resolve_optional_transport")); assert!(yellowstone.contains("insert_block_filter"));
assert!(yellowstone.contains("COMPOSITE_COMPONENT_ID_TRANSPORT_YELLOWSTONE")); assert!(yellowstone.contains("set_include_transactions(std::option::Option::Some(false))"));
assert!(yellowstone.contains("single_http_role(transport.http_settings(), network, &[\"getTransaction\"])")); assert!(yellowstone.contains("single_http_role(transport.http_settings(), network, &[\"getBlock\"])"));
assert!(yellowstone.contains("HttpTransportPool::new(transport.http_settings().clone())")); assert!(!yellowstone.contains("insert_transaction_filter"));
assert!(!yellowstone.contains("base_transport")); let workspace = read_text(app_root().join("../ksp-worker-raw-transaction-ingest-lib/src/runtime_resources.rs").as_path());
assert!(workspace.contains("RawTransactionIngestYellowstoneMode::BlockHydration"));
assert!(workspace.contains("fetch_yellowstone_block_ingresses"));
assert!(workspace.contains("get_block_observed"));
assert!(workspace.contains("RAW_TRANSACTION_INGEST_YELLOWSTONE_BLOCK_RECONCILIATION_ATTEMPTS"));
assert!(workspace.contains("stop_receiver.changed()"));
} }

4
crates/ksp-app-raw-transaction-ingest-desk/unit_tests/dto_route.rs
View File

@@ -1,5 +1,5 @@
// file: crates/ksp-app-raw-transaction-ingest-desk/unit_tests/dto_route.rs // file: crates/ksp-app-raw-transaction-ingest-desk/unit_tests/dto_route.rs
// version: 3 // version: 4
#[test] #[test]
fn pre_005_route_ids_are_exact_and_stably_ordered() { fn pre_005_route_ids_are_exact_and_stably_ordered() {
@@ -31,6 +31,7 @@ fn pre_006_unavailable_reason_codes_are_exact_and_safe() {
let values = [ let values = [
crate::RawIngestRouteUnavailableReason::ProfileUnresolved, crate::RawIngestRouteUnavailableReason::ProfileUnresolved,
crate::RawIngestRouteUnavailableReason::NetworkMismatch, crate::RawIngestRouteUnavailableReason::NetworkMismatch,
crate::RawIngestRouteUnavailableReason::MissingHttpGetBlock,
crate::RawIngestRouteUnavailableReason::MissingHttpGetTransaction, crate::RawIngestRouteUnavailableReason::MissingHttpGetTransaction,
crate::RawIngestRouteUnavailableReason::MissingHttpBlockScan, crate::RawIngestRouteUnavailableReason::MissingHttpBlockScan,
crate::RawIngestRouteUnavailableReason::MissingWsLogsCapability, crate::RawIngestRouteUnavailableReason::MissingWsLogsCapability,
@@ -47,6 +48,7 @@ fn pre_006_unavailable_reason_codes_are_exact_and_safe() {
serde_json::json!([ serde_json::json!([
"profile_unresolved", "profile_unresolved",
"network_mismatch", "network_mismatch",
"missing_http_get_block",
"missing_http_get_transaction", "missing_http_get_transaction",
"missing_http_block_scan", "missing_http_block_scan",
"missing_ws_logs_capability", "missing_ws_logs_capability",

64
crates/ksp-app-raw-transaction-ingest-desk/unit_tests/route_runtime.rs
View File

@@ -1,12 +1,64 @@
// file: crates/ksp-app-raw-transaction-ingest-desk/unit_tests/route_runtime.rs // file: crates/ksp-app-raw-transaction-ingest-desk/unit_tests/route_runtime.rs
// version: 1 // version: 2
#[test] #[test]
fn pre_008_mono_route_runtime_starts_idle_without_worker_or_store_material() { fn pre_009_multi_route_runtime_starts_without_workers_or_shared_store_material() {
let state = crate::RouteRuntimeState::new(); let state = crate::RouteRuntimeState::new();
let idle = state.is_idle(); let inner = state.inner.lock();
assert!(idle.is_ok()); assert!(inner.is_ok());
if let std::result::Result::Ok(idle) = idle { if let std::result::Result::Ok(inner) = inner {
assert!(idle); assert!(inner.routes.is_empty());
assert!(inner.store.is_none());
assert!(inner.network.is_none());
assert!(inner.store_opening_token.is_none());
assert!(inner.store_closing_token.is_none());
} }
} }
#[test]
fn pre_009_late_targeted_stop_returns_only_the_matching_terminal_route() {
let state = crate::RouteRuntimeState::new();
let terminal = crate::RawIngestRouteRuntimeDto {
commitment: crate::RawIngestCommitment::Confirmed,
inventory_generation: 9,
network: "mainnet".to_owned(),
profile_id: "mainnet".to_owned(),
route_id: crate::RawIngestRouteId::YellowstoneHydrated,
state: crate::RawIngestRouteState::Faulted,
};
let inner = state.inner.lock();
assert!(inner.is_ok());
if let std::result::Result::Ok(mut inner) = inner {
inner.last_terminals.push(terminal.clone());
}
let request = crate::RawIngestRouteStopRequestDto { profile_id: "mainnet".to_owned(), route_id: crate::RawIngestRouteId::YellowstoneHydrated };
let target = state.stop_target(&request);
assert!(target.is_ok());
assert!(matches!(&target, std::result::Result::Ok(super::RouteStopTarget::Terminal(_))));
if let std::result::Result::Ok(super::RouteStopTarget::Terminal(value)) = target {
assert_eq!(value.profile_id, terminal.profile_id);
assert_eq!(value.route_id, terminal.route_id);
assert_eq!(value.state, crate::RawIngestRouteState::Faulted);
}
}
#[test]
fn pre_009_late_targeted_stop_waits_while_final_shared_store_is_closing() {
let state = crate::RouteRuntimeState::new();
let inner = state.inner.lock();
assert!(inner.is_ok());
if let std::result::Result::Ok(mut inner) = inner {
inner.last_terminals.push(crate::RawIngestRouteRuntimeDto {
commitment: crate::RawIngestCommitment::Finalized,
inventory_generation: 11,
network: "mainnet".to_owned(),
profile_id: "mainnet".to_owned(),
route_id: crate::RawIngestRouteId::HttpBlockPolling,
state: crate::RawIngestRouteState::Stopped,
});
inner.store_closing_token = std::option::Option::Some(77);
}
let request = crate::RawIngestRouteStopRequestDto { profile_id: "mainnet".to_owned(), route_id: crate::RawIngestRouteId::HttpBlockPolling };
let target = state.stop_target(&request);
assert!(matches!(target, std::result::Result::Ok(super::RouteStopTarget::TerminalClosing)));
}

26
crates/ksp-config-lib/unit_tests/transport.rs
View File

@@ -1,5 +1,5 @@
// file: crates/ksp-config-lib/unit_tests/transport.rs // file: crates/ksp-config-lib/unit_tests/transport.rs
// version: 17 // version: 18
#[test] #[test]
fn fixture_transport_profile_maps_complete_runtime_contract() { fn fixture_transport_profile_maps_complete_runtime_contract() {
@@ -274,17 +274,25 @@ fn committed_v3_publicnode_profiles_map_provider_neutral_yellowstone_grpc_with_n
}; };
assert_eq!(resolved.profile_id(), profile_id); assert_eq!(resolved.profile_id(), profile_id);
if profile_id == "publicnode_mainnet" { if profile_id == "publicnode_mainnet" {
assert_eq!(resolved.settings().endpoints().len(), 1); assert_eq!(resolved.settings().endpoints().len(), 2);
let http = &resolved.settings().endpoints()[0]; let publicnode = &resolved.settings().endpoints()[0];
assert_eq!(http.name(), "publicnode_solana_mainnet_rpc"); assert_eq!(publicnode.name(), "publicnode_solana_mainnet_rpc");
assert_eq!(http.provider().as_str(), "publicnode"); assert_eq!(publicnode.provider().as_str(), "publicnode");
assert_eq!(http.cluster().as_str(), "mainnet"); assert_eq!(publicnode.cluster().as_str(), "mainnet");
assert_eq!(http.url().as_str(), "https://solana-rpc.publicnode.com"); assert_eq!(publicnode.url().as_str(), "https://solana-rpc.publicnode.com");
assert_eq!(http.roles().len(), 1); assert_eq!(publicnode.roles().len(), 1);
let limits = http.roles()[0].limits(); let limits = publicnode.roles()[0].limits();
assert!(limits.requests_per_second().is_none()); assert!(limits.requests_per_second().is_none());
assert!(limits.burst_capacity().is_none()); assert!(limits.burst_capacity().is_none());
assert_eq!(limits.max_concurrent_requests().map(std::num::NonZeroU32::get), std::option::Option::Some(8)); assert_eq!(limits.max_concurrent_requests().map(std::num::NonZeroU32::get), std::option::Option::Some(8));
let solana_public = &resolved.settings().endpoints()[1];
assert_eq!(solana_public.name(), "solana_mainnet_public_repair");
assert_eq!(solana_public.provider().as_str(), "solana-public");
assert_eq!(solana_public.cluster().as_str(), "mainnet");
assert_eq!(solana_public.roles()[0].role().as_str(), "default");
let solana_limits = solana_public.roles()[0].limits();
assert_eq!(solana_limits.requests_per_second().map(std::num::NonZeroU32::get), std::option::Option::Some(5));
assert_eq!(solana_limits.burst_capacity().map(std::num::NonZeroU32::get), std::option::Option::Some(10));
} }
assert!(resolved.ws_settings().is_some()); assert!(resolved.ws_settings().is_some());
let grpc = resolved.grpc_settings(); let grpc = resolved.grpc_settings();

280
crates/ksp-worker-raw-transaction-ingest-lib/src/runtime_resources.rs
View File

@@ -1,5 +1,5 @@
// file: crates/ksp-worker-raw-transaction-ingest-lib/src/runtime_resources.rs // file: crates/ksp-worker-raw-transaction-ingest-lib/src/runtime_resources.rs
// version: 44 // version: 45
use sha2::Digest; // rust-rules: trait-import use sha2::Digest; // rust-rules: trait-import
@@ -32,6 +32,9 @@ const RAW_TRANSACTION_INGEST_STANDARD_BLOCK_PROTOCOL: &str = "solana_ws";
const RAW_TRANSACTION_INGEST_STANDARD_LOGS_FILTER_FINGERPRINT_DOMAIN: &[u8] = b"ksp.raw_transaction_ingest.standard_logs.filter.v1\0"; const RAW_TRANSACTION_INGEST_STANDARD_LOGS_FILTER_FINGERPRINT_DOMAIN: &[u8] = b"ksp.raw_transaction_ingest.standard_logs.filter.v1\0";
const RAW_TRANSACTION_INGEST_STANDARD_LOGS_HTTP_PROTOCOL: &str = "solana_ws_http"; const RAW_TRANSACTION_INGEST_STANDARD_LOGS_HTTP_PROTOCOL: &str = "solana_ws_http";
const RAW_TRANSACTION_INGEST_STANDARD_LOGS_HTTP_SOURCE_KEY_DOMAIN: &[u8] = b"ksp.raw_transaction_ingest.standard_logs_http.source_key.v1\0"; const RAW_TRANSACTION_INGEST_STANDARD_LOGS_HTTP_SOURCE_KEY_DOMAIN: &[u8] = b"ksp.raw_transaction_ingest.standard_logs_http.source_key.v1\0";
const RAW_TRANSACTION_INGEST_YELLOWSTONE_BLOCK_HTTP_ACQUISITION_METHOD: &str = "block_subscribe_get_block";
const RAW_TRANSACTION_INGEST_YELLOWSTONE_BLOCK_RECONCILIATION_ATTEMPTS: u8 = 4;
const RAW_TRANSACTION_INGEST_YELLOWSTONE_BLOCK_RECONCILIATION_DELAY: std::time::Duration = std::time::Duration::from_millis(100);
const RAW_TRANSACTION_INGEST_YELLOWSTONE_COVERAGE_SCOPE_FINGERPRINT_DOMAIN: &[u8] = b"ksp.raw_transaction_ingest.yellowstone.coverage_scope.v1\0"; const RAW_TRANSACTION_INGEST_YELLOWSTONE_COVERAGE_SCOPE_FINGERPRINT_DOMAIN: &[u8] = b"ksp.raw_transaction_ingest.yellowstone.coverage_scope.v1\0";
const RAW_TRANSACTION_INGEST_YELLOWSTONE_FILTER_FINGERPRINT_DOMAIN: &[u8] = b"ksp.raw_transaction_ingest.yellowstone.filters.v1\0"; const RAW_TRANSACTION_INGEST_YELLOWSTONE_FILTER_FINGERPRINT_DOMAIN: &[u8] = b"ksp.raw_transaction_ingest.yellowstone.filters.v1\0";
const RAW_TRANSACTION_INGEST_YELLOWSTONE_HTTP_PROTOCOL: &str = "yellowstone_http"; const RAW_TRANSACTION_INGEST_YELLOWSTONE_HTTP_PROTOCOL: &str = "yellowstone_http";
@@ -371,10 +374,15 @@ impl RawTransactionIngestLiveSource {
std::result::Result::Ok(value) => value, std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error), std::result::Result::Err(error) => return std::result::Result::Err(error),
}; };
let known_reference_hydration =
match http_role_supports_rpc_method(&source.http_pool, &source.hydration_role, "getTransaction", source.network.as_str()) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let reference_bearing = let reference_bearing =
source.subscribe_request.transaction_filter_count() > 0 || source.subscribe_request.transaction_status_filter_count() > 0; source.subscribe_request.transaction_filter_count() > 0 || source.subscribe_request.transaction_status_filter_count() > 0;
let live_block_material = source.subscribe_request.block_filter_count() > 0; let live_block_material = source.subscribe_request.block_filter_count() > 0;
(reference_bearing, live_block_material, true, http_block_scan, true) (reference_bearing, live_block_material, true, http_block_scan, known_reference_hydration)
}, },
}; };
let block_material = live_block_material || http_block_scan; let block_material = live_block_material || http_block_scan;
@@ -404,7 +412,11 @@ impl RawTransactionIngestLiveSource {
} }
fn uses_hydration(&self) -> bool { fn uses_hydration(&self) -> bool {
return matches!(self, Self::HeliusTransaction(_) | Self::StandardLogs(_) | Self::Yellowstone(_)); return match self {
Self::HeliusTransaction(_) | Self::StandardLogs(_) => true,
Self::Yellowstone(source) => source.uses_transaction_hydration(),
Self::HttpBlockPolling(_) | Self::StandardBlock(_) => false,
};
} }
async fn run( async fn run(
@@ -975,6 +987,12 @@ struct RawTransactionIngestHydrationContext {
source_key_domain: &'static [u8], source_key_domain: &'static [u8],
} }
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
enum RawTransactionIngestYellowstoneMode {
TransactionHydration,
BlockHydration,
}
/// Validated Yellowstone plus HTTP runtime source owned by the continuous RAW transaction ingest Worker. /// Validated Yellowstone plus HTTP runtime source owned by the continuous RAW transaction ingest Worker.
/// ///
/// The Transport-owned channel, subscribe request, HTTP pool and hydration role remain private. Construction validates deterministic source-composition /// The Transport-owned channel, subscribe request, HTTP pool and hydration role remain private. Construction validates deterministic source-composition
@@ -984,6 +1002,7 @@ pub struct RawTransactionIngestYellowstoneSource {
subscribe_request: ksp_onchain_transport_lib::YellowstoneSubscribeRequest, subscribe_request: ksp_onchain_transport_lib::YellowstoneSubscribeRequest,
http_pool: ksp_onchain_transport_lib::HttpTransportPool, http_pool: ksp_onchain_transport_lib::HttpTransportPool,
hydration_role: ksp_onchain_transport_lib::HttpRoleName, hydration_role: ksp_onchain_transport_lib::HttpRoleName,
mode: RawTransactionIngestYellowstoneMode,
network: ksp_store_lib::RawNetworkId, network: ksp_store_lib::RawNetworkId,
route: RawTransactionIngestSourceRoute, route: RawTransactionIngestSourceRoute,
source_key: [u8; 32], source_key: [u8; 32],
@@ -1023,7 +1042,19 @@ impl crate::RawTransactionIngestYellowstoneSource {
std::result::Result::Err(_) => return std::result::Result::Err(crate::runtime_error("runtime_resources.yellowstone_endpoint_unrepresentable")), std::result::Result::Err(_) => return std::result::Result::Err(crate::runtime_error("runtime_resources.yellowstone_endpoint_unrepresentable")),
}; };
let route = RawTransactionIngestSourceRoute { endpoint_id, provider }; let route = RawTransactionIngestSourceRoute { endpoint_id, provider };
let method = match ksp_onchain_transport_lib::find_http_rpc_method("getTransaction") { let mode = if subscribe_request.block_filter_count() > 0
&& subscribe_request.transaction_filter_count() == 0
&& subscribe_request.transaction_status_filter_count() == 0
{
RawTransactionIngestYellowstoneMode::BlockHydration
} else {
RawTransactionIngestYellowstoneMode::TransactionHydration
};
let method_name = match mode {
RawTransactionIngestYellowstoneMode::TransactionHydration => "getTransaction",
RawTransactionIngestYellowstoneMode::BlockHydration => "getBlock",
};
let method = match ksp_onchain_transport_lib::find_http_rpc_method(method_name) {
std::option::Option::Some(value) => value, std::option::Option::Some(value) => value,
std::option::Option::None => return std::result::Result::Err(crate::runtime_error("runtime_resources.hydration_method_missing")), std::option::Option::None => return std::result::Result::Err(crate::runtime_error("runtime_resources.hydration_method_missing")),
}; };
@@ -1040,7 +1071,11 @@ impl crate::RawTransactionIngestYellowstoneSource {
std::result::Result::Err(_) => return std::result::Result::Err(crate::runtime_error("runtime_resources.yellowstone_identity_invalid")), std::result::Result::Err(_) => return std::result::Result::Err(crate::runtime_error("runtime_resources.yellowstone_identity_invalid")),
}; };
let source_key = yellowstone_live_source_key(&network, &route, subscribe_request.commitment(), &request_identity); let source_key = yellowstone_live_source_key(&network, &route, subscribe_request.commitment(), &request_identity);
return std::result::Result::Ok(Self { yellowstone_channel, subscribe_request, http_pool, hydration_role, network, route, source_key }); return std::result::Result::Ok(Self { yellowstone_channel, subscribe_request, http_pool, hydration_role, mode, network, route, source_key });
}
fn uses_transaction_hydration(&self) -> bool {
return self.mode == RawTransactionIngestYellowstoneMode::TransactionHydration;
} }
fn hydration_context(&self) -> RawTransactionIngestHydrationContext { fn hydration_context(&self) -> RawTransactionIngestHydrationContext {
@@ -1070,6 +1105,9 @@ impl crate::RawTransactionIngestYellowstoneSource {
processing_frontier_sender: tokio::sync::watch::Sender<crate::RawTransactionIngestProcessingFrontierProjection>, processing_frontier_sender: tokio::sync::watch::Sender<crate::RawTransactionIngestProcessingFrontierProjection>,
shared: RawTransactionIngestSourceRuntimeShared, shared: RawTransactionIngestSourceRuntimeShared,
) -> ksp_core_lib::Result<()> { ) -> ksp_core_lib::Result<()> {
if self.mode == RawTransactionIngestYellowstoneMode::BlockHydration {
return self.run_block_hydration(settings, stop_receiver, admission_sender, processing_frontier_sender).await;
}
let RawTransactionIngestSourceRuntimeShared { continuity_contracts, global_hydration_registry, hydration_in_flight_limit, hydration_pending_limit } = let RawTransactionIngestSourceRuntimeShared { continuity_contracts, global_hydration_registry, hydration_in_flight_limit, hydration_pending_limit } =
shared; shared;
let opened = tokio::select! { let opened = tokio::select! {
@@ -1197,6 +1235,119 @@ impl crate::RawTransactionIngestYellowstoneSource {
}, },
}; };
} }
async fn run_block_hydration(
self,
settings: crate::RawTransactionIngestSettings,
mut stop_receiver: tokio::sync::watch::Receiver<bool>,
admission_sender: tokio::sync::mpsc::Sender<crate::RawTransactionIngress>,
processing_frontier_sender: tokio::sync::watch::Sender<crate::RawTransactionIngestProcessingFrontierProjection>,
) -> ksp_core_lib::Result<()> {
let opened = tokio::select! {
biased;
_ = stop_receiver.changed() => {
return std::result::Result::Ok(());
}
result = self.yellowstone_channel.open_standard_subscribe(self.subscribe_request.clone()) => result,
};
let mut session = match opened {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(source_transport_error(error.code())),
};
let mut processing_frontier = RawTransactionIngestProcessingFrontierReporter::new(processing_frontier_sender);
let snapshot_source = session.snapshot_source();
if let std::result::Result::Err(error) = processing_frontier.observe_session_snapshot(snapshot_source.current()) {
processing_frontier.set_source_state(crate::RawTransactionIngestSourceState::Failed);
let _closed = session.close().await;
return std::result::Result::Err(error);
}
let mut session_snapshot_source = std::option::Option::Some(snapshot_source);
let mut fault = std::option::Option::None;
loop {
if *stop_receiver.borrow() {
break;
}
tokio::select! {
biased;
_ = stop_receiver.changed() => {
break;
}
source_snapshot = wait_yellowstone_session_snapshot(&mut session_snapshot_source) => {
match source_snapshot {
std::option::Option::Some(snapshot) => {
if let std::result::Result::Err(error) = processing_frontier.observe_session_snapshot(snapshot) {
fault = std::option::Option::Some(error);
break;
}
},
std::option::Option::None => session_snapshot_source = std::option::Option::None,
}
}
update = session.next_update() => {
let update = match update {
std::result::Result::Ok(std::option::Option::Some(value)) => value,
std::result::Result::Ok(std::option::Option::None) => {
fault = std::option::Option::Some(crate::runtime_error("source.session_closed"));
break;
},
std::result::Result::Err(error) => {
fault = std::option::Option::Some(source_transport_error(error.code()));
break;
},
};
if let ksp_onchain_transport_lib::YellowstoneSubscribeUpdate::Block(value) = update {
let fetched = fetch_yellowstone_block_ingresses(&self, &settings, value.slot(), &mut stop_receiver).await;
let ingresses = match fetched {
std::result::Result::Ok(std::option::Option::Some(value)) => value,
std::result::Result::Ok(std::option::Option::None) => break,
std::result::Result::Err(error) => {
fault = std::option::Option::Some(error);
break;
},
};
for ingress in ingresses {
let sent = tokio::select! {
biased;
_ = stop_receiver.changed() => {
break;
}
result = admission_sender.send(ingress) => result,
};
if sent.is_err() {
if !*stop_receiver.borrow() {
fault = std::option::Option::Some(crate::runtime_error("source.admission_closed"));
}
break;
}
}
if fault.is_some() || *stop_receiver.borrow() {
break;
}
if let std::result::Result::Err(error) = processing_frontier.observe_settled(value.slot()) {
fault = std::option::Option::Some(error);
break;
}
}
}
}
}
processing_frontier.set_source_state(crate::RawTransactionIngestSourceState::Closing);
let closed = session.close().await;
if let std::option::Option::Some(error) = fault {
processing_frontier.set_source_state(crate::RawTransactionIngestSourceState::Failed);
return std::result::Result::Err(error);
}
return match closed {
std::result::Result::Ok(()) => {
processing_frontier.set_source_state(crate::RawTransactionIngestSourceState::Closed);
std::result::Result::Ok(())
},
std::result::Result::Err(error) => {
processing_frontier.set_source_state(crate::RawTransactionIngestSourceState::Failed);
std::result::Result::Err(source_transport_error(error.code()))
},
};
}
} }
/// Validated Helius `transactionSubscribe` plus HTTP hydration source owned by the continuous RAW transaction ingest Worker. /// Validated Helius `transactionSubscribe` plus HTTP hydration source owned by the continuous RAW transaction ingest Worker.
@@ -3198,6 +3349,125 @@ fn validate_http_block_polling_discovery(next_scan_slot: u64, discovered: &[u64]
return std::result::Result::Ok(()); return std::result::Result::Ok(());
} }
async fn fetch_yellowstone_block_ingresses(
source: &crate::RawTransactionIngestYellowstoneSource,
settings: &crate::RawTransactionIngestSettings,
slot: u64,
stop_receiver: &mut tokio::sync::watch::Receiver<bool>,
) -> ksp_core_lib::Result<std::option::Option<std::vec::Vec<crate::RawTransactionIngress>>> {
let commitment = match source.subscribe_request.commitment() {
std::option::Option::Some(value) => value,
std::option::Option::None => return std::result::Result::Err(crate::runtime_error("source.yellowstone_commitment_missing")),
};
let config = ksp_onchain_transport_lib::SolanaGetBlockConfig::new(
std::option::Option::Some(commitment),
std::option::Option::Some(ksp_onchain_transport_lib::SolanaTransactionEncoding::Base64),
std::option::Option::Some(ksp_onchain_transport_lib::SolanaTransactionDetails::Full),
std::option::Option::Some(1),
std::option::Option::Some(false),
);
let mut attempt = 0_u8;
let observed = loop {
attempt = attempt.saturating_add(1);
let request = tokio::select! {
biased;
_ = stop_receiver.changed() => {
return std::result::Result::Ok(std::option::Option::None);
}
result = source.http_pool.get_block_observed(&source.hydration_role, slot, std::option::Option::Some(&config)) => result,
};
match request {
std::result::Result::Ok(value) if value.value().is_some() => break value,
std::result::Result::Ok(_) if attempt < RAW_TRANSACTION_INGEST_YELLOWSTONE_BLOCK_RECONCILIATION_ATTEMPTS => {},
std::result::Result::Ok(_) => {
return std::result::Result::Err(crate::runtime_error("source.yellowstone_block_not_available"));
},
std::result::Result::Err(_) if attempt < RAW_TRANSACTION_INGEST_YELLOWSTONE_BLOCK_RECONCILIATION_ATTEMPTS => {},
std::result::Result::Err(error) => return std::result::Result::Err(source_transport_error(error.code())),
}
tokio::select! {
biased;
_ = stop_receiver.changed() => {
return std::result::Result::Ok(std::option::Option::None);
}
_ = tokio::time::sleep(RAW_TRANSACTION_INGEST_YELLOWSTONE_BLOCK_RECONCILIATION_DELAY) => {},
}
};
let block = match observed.value() {
std::option::Option::Some(value) => value,
std::option::Option::None => return std::result::Result::Err(crate::runtime_error("source.yellowstone_block_not_available")),
};
let received_at = match current_raw_timestamp() {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let transactions = match block.transactions() {
ksp_onchain_transport_lib::SolanaWireField::Value(value) => value,
ksp_onchain_transport_lib::SolanaWireField::Omitted | ksp_onchain_transport_lib::SolanaWireField::Null => {
return std::result::Result::Err(crate::runtime_error("source.yellowstone_block_transactions_missing"));
},
};
let provenance = build_yellowstone_block_provenance(source, settings, observed.endpoint_name(), observed.provider(), received_at);
let provenance = match provenance {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let mut ingresses = std::vec::Vec::with_capacity(transactions.len());
for (position, transaction) in transactions.iter().enumerate() {
let material = build_http_block_polling_material_from_transaction(&source.network, slot, block.block_time(), transaction, position);
let material = match material {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
ingresses.push(crate::RawTransactionIngress {
material,
network: source.network.clone(),
provenance: provenance.clone(),
source_key: source.source_key,
});
}
return std::result::Result::Ok(std::option::Option::Some(ingresses));
}
fn build_yellowstone_block_provenance(
source: &crate::RawTransactionIngestYellowstoneSource,
settings: &crate::RawTransactionIngestSettings,
endpoint_name: &str,
provider_name: &ksp_onchain_transport_lib::HttpProviderName,
received_at: ksp_store_lib::RawTimestamp,
) -> ksp_core_lib::Result<ksp_store_lib::RawAcquisitionProvenance> {
let (provider, endpoint_id) = match composite_provenance_codes(&source.route, "ys", provider_name.as_str(), endpoint_name) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let protocol = match ksp_store_lib::RawProvenanceCode::new(RAW_TRANSACTION_INGEST_YELLOWSTONE_HTTP_PROTOCOL) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return std::result::Result::Err(crate::runtime_error("source.yellowstone_block_protocol_unrepresentable")),
};
let acquisition_method = match ksp_store_lib::RawProvenanceCode::new(RAW_TRANSACTION_INGEST_YELLOWSTONE_BLOCK_HTTP_ACQUISITION_METHOD) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return std::result::Result::Err(crate::runtime_error("source.yellowstone_block_method_unrepresentable")),
};
let capture_session = match ksp_store_lib::RawProvenanceCode::new(settings.worker_id().as_str()) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return std::result::Result::Err(crate::runtime_error("source.yellowstone_block_capture_session_unrepresentable")),
};
let commitment = match source.subscribe_request.commitment() {
std::option::Option::Some(value) => value,
std::option::Option::None => return std::result::Result::Err(crate::runtime_error("source.yellowstone_commitment_missing")),
};
let commitment = match ksp_store_lib::RawProvenanceCode::new(commitment.as_str()) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return std::result::Result::Err(crate::runtime_error("source.yellowstone_block_commitment_unrepresentable")),
};
return std::result::Result::Ok(
ksp_store_lib::RawAcquisitionProvenance::new(provider, protocol, acquisition_method, ksp_store_lib::RawAcquisitionOrigin::Live, received_at)
.with_endpoint_id(endpoint_id)
.with_commitment(commitment)
.with_capture_session_id(capture_session),
);
}
fn project_http_block_polling_ingresses( fn project_http_block_polling_ingresses(
source: &crate::RawTransactionIngestHttpBlockPollingSource, source: &crate::RawTransactionIngestHttpBlockPollingSource,
settings: &crate::RawTransactionIngestSettings, settings: &crate::RawTransactionIngestSettings,

75
crates/ksp-worker-raw-transaction-ingest-lib/unit_tests/runtime_resources.rs
View File

@@ -1,5 +1,5 @@
// file: crates/ksp-worker-raw-transaction-ingest-lib/unit_tests/runtime_resources.rs // file: crates/ksp-worker-raw-transaction-ingest-lib/unit_tests/runtime_resources.rs
// version: 35 // version: 36
fn grpc_endpoint(cluster: &str) -> std::option::Option<ksp_onchain_transport_lib::YellowstoneGrpcEndpointSettings> { fn grpc_endpoint(cluster: &str) -> std::option::Option<ksp_onchain_transport_lib::YellowstoneGrpcEndpointSettings> {
return grpc_endpoint_with_identity(cluster, "yellowstone-fixture", "fixture-provider"); return grpc_endpoint_with_identity(cluster, "yellowstone-fixture", "fixture-provider");
@@ -359,6 +359,25 @@ fn transaction_request(
return std::option::Option::Some(request); return std::option::Option::Some(request);
} }
fn block_request(
commitment: std::option::Option<ksp_onchain_transport_lib::SolanaCommitment>,
) -> std::option::Option<ksp_onchain_transport_lib::YellowstoneSubscribeRequest> {
let name = match ksp_onchain_transport_lib::YellowstoneSubscribeFilterName::new("block-fixture") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return std::option::Option::None,
};
let mut filter = ksp_onchain_transport_lib::YellowstoneSubscribeBlockFilter::new();
filter.set_include_transactions(std::option::Option::Some(false));
filter.set_include_accounts(std::option::Option::Some(false));
filter.set_include_entries(std::option::Option::Some(false));
let mut request = ksp_onchain_transport_lib::YellowstoneSubscribeRequest::new();
if request.insert_block_filter(name, filter).is_err() {
return std::option::Option::None;
}
request.set_commitment(commitment);
return std::option::Option::Some(request);
}
struct SignalViewFixture { struct SignalViewFixture {
created_at: std::option::Option<ksp_onchain_transport_lib::YellowstoneUpdateTimestamp>, created_at: std::option::Option<ksp_onchain_transport_lib::YellowstoneUpdateTimestamp>,
family: super::RawTransactionIngestSourceFamily, family: super::RawTransactionIngestSourceFamily,
@@ -836,6 +855,60 @@ async fn pre_002_source_accepts_matching_confirmed_transaction_and_get_transacti
return; return;
} }
#[tokio::test(flavor = "current_thread")]
async fn pre_009_yellowstone_block_mode_uses_get_block_without_transaction_hydration() {
let endpoint = match grpc_endpoint("devnet") {
std::option::Option::Some(value) => value,
std::option::Option::None => return,
};
let channel = match ksp_onchain_transport_lib::YellowstoneGrpcChannel::prepare(&endpoint) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return,
};
let request = match block_request(std::option::Option::Some(ksp_onchain_transport_lib::SolanaCommitment::Confirmed)) {
std::option::Option::Some(value) => value,
std::option::Option::None => return,
};
let pool = match http_pool("devnet", "hydration", "get_block") {
std::option::Option::Some(value) => value,
std::option::Option::None => return,
};
let source = crate::RawTransactionIngestYellowstoneSource::new(channel, request, pool, ksp_onchain_transport_lib::HttpRoleName::new("hydration"));
let source = match source {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return,
};
assert_eq!(source.mode, super::RawTransactionIngestYellowstoneMode::BlockHydration);
assert!(!source.uses_transaction_hydration());
}
#[tokio::test(flavor = "current_thread")]
async fn pre_009_yellowstone_block_mode_rejects_get_transaction_only_http_role() {
let endpoint = match grpc_endpoint("devnet") {
std::option::Option::Some(value) => value,
std::option::Option::None => return,
};
let channel = match ksp_onchain_transport_lib::YellowstoneGrpcChannel::prepare(&endpoint) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return,
};
let request = match block_request(std::option::Option::Some(ksp_onchain_transport_lib::SolanaCommitment::Confirmed)) {
std::option::Option::Some(value) => value,
std::option::Option::None => return,
};
let pool = match http_pool("devnet", "hydration", "get_transaction") {
std::option::Option::Some(value) => value,
std::option::Option::None => return,
};
let source = crate::RawTransactionIngestYellowstoneSource::new(channel, request, pool, ksp_onchain_transport_lib::HttpRoleName::new("hydration"));
let error = match source {
std::result::Result::Ok(_) => return,
std::result::Result::Err(error) => error,
};
assert_eq!(error.code(), crate::ERROR_CODE_RAW_TRANSACTION_INGEST_RUNTIME_INVALID);
assert!(error.context().iter().any(|context| return context.value() == "runtime_resources.hydration_role_unsupported"));
}
#[tokio::test(flavor = "current_thread")] #[tokio::test(flavor = "current_thread")]
async fn pre_002_source_rejects_processed_and_implicit_commitment() { async fn pre_002_source_rejects_processed_and_implicit_commitment() {
for commitment in [std::option::Option::Some(ksp_onchain_transport_lib::SolanaCommitment::Processed), std::option::Option::None] { for commitment in [std::option::Option::Some(ksp_onchain_transport_lib::SolanaCommitment::Processed), std::option::Option::None] {

111
deltas/0.3.15/pre.009.md Normal file
View File

@@ -0,0 +1,111 @@
<!-- file: deltas/0.3.15/pre.009.md -->
<!-- version: 1 -->
# Delta `0.3.15-pre.009`
## Base
```text
base archive : ksp-general-0.3.15-pre.008-fix.004.zip
base SHA-256 : 206ad2a2a423e8a14a6644877faf0b69b8435e7558aeba6ed3dafff0f16f73a6
base version : 0.3.15-pre.8.fix.4
```
Le gate opérateur de la base passe les audits Rust/Markdown, `cargo check --workspace`, Clippy strict, les tests Raw Transaction Ingest Desk et le workspace all-targets/all-features. Les limites live restantes sont traitées comme des propriétés de stratégie/provider et non comme un défaut du lifecycle mono-route de `pre.008`.
## Objectif
Matérialiser le runtime multi-route Mainnet : un Worker indépendant par route, un Store partagé uniquement entre routes du même réseau, Stop ciblé, isolation des faults et réduction du goulet Yellowstone observé avec l'hydration `getTransaction` unitaire.
## Runtime multi-route et Store partagé
Le backend conserve la règle architecturale :
```text
1 route logique = 1 Worker indépendant
N routes same-network = N Workers + 1 Arc<Store> partagé
```
Le premier Start réserve la route, ouvre le Store et exige `Ready`. Les Starts suivants du même réseau réutilisent le Store déjà ouvert ; un Start d'un autre réseau est refusé tant que le Store courant reste possédé. Deux Workers de la même identité logique `profile_id + route_id` ne peuvent pas coexister.
Chaque monitor retire uniquement sa route terminale. Tant qu'un sibling reste actif, le Store reste ouvert. La dernière route terminale reprend l'unique Store, attend la libération des clones Worker, exécute `Store::close()` puis libère l'identité réseau. Aucun gap, TargetCoverage, processing frontier ou handle Worker n'est partagé entre routes.
`stop_route` reçoit désormais une identité logique ciblée `profile_id + route_id`. Un terminal tardif reste idempotent ; si la dernière route a déjà terminé mais que le Store final se ferme encore, Stop attend la fin de ce cleanup sans conserver de `MutexGuard` à travers un `await`.
## Yellowstone Mainnet : block trigger + reconciliation HTTP
La route stable `yellowstone-hydrated` conserve son identifiant IPC pour compatibilité mais change de stratégie productive :
```text
AVANT
Yellowstone Transaction/Status/Block
-> une référence transactionnelle
-> getTransaction
-> RAW
PRE.009
Yellowstone Block léger
-> slot
-> getBlock Full/Base64
-> toutes les transactions Legacy/V0/V1 du bloc
-> RAW
```
Le filtre Yellowstone Block n'embarque pas les transactions dans le payload gRPC ; il sert de trigger temps réel. Le Worker distingue explicitement `TransactionHydration` et `BlockHydration`. Le mode block n'entre plus dans la registry globale de `getTransaction`.
La reconciliation `getBlock` est interruptible par Stop et retente de façon bornée lorsqu'un bloc n'est pas encore visible ou qu'un endpoint du pool échoue, afin d'absorber le décalage bref entre disponibilité Yellowstone et disponibilité HTTP. Aucune canonicalisation directe protobuf -> RAW n'est revendiquée dans cette tranche.
## Pool HTTP Mainnet multi-provider
Le profil `publicnode_mainnet` expose deux endpoints HTTP sous le même rôle logique `default` :
```text
PublicNode Mainnet RPC
Solana public Mainnet repair
```
PublicNode conserve sa concurrence locale sans quota numérique inventé ; Solana public conserve sa limite locale `5 req/s`, burst `10`. `HttpTransportPool` garde la sélection par rôle/capability, priorité, health, cooldown et fallback/round-robin entre endpoints éligibles.
Cette tranche prouve le modèle multi-provider sans prétendre qu'un nombre arbitraire d'URLs gratuites garantit une capacité additive stable. D'autres endpoints pourront être ajoutés par Config lorsque leurs capabilities et limites seront connues.
## Frontend
La vue Routes conserve le profil réseau sélectionné et permet plusieurs Starts simultanés sur ses routes `Configured`. Chaque carte possède son Stop ciblé lorsque sa route est active. Le bandeau runtime projette uniquement le nombre de routes actives et leurs identités logiques sûres.
Le monitoring continu des terminaisons spontanées reste `pre.010`; jusqu'alors, le frontend peut conserver localement le dernier accusé actif jusqu'à un Stop ciblé/resynchronisation.
## Frontières conservées
```text
Config : profils, secrets, endpoints et pool multi-provider
Transport : HTTP pool, health/cooldown, WS/gRPC et retry réseau
Worker : stratégie de source, block reconciliation, admission et persistence
Store : idempotence (network, signature) + observations/provenances
Desk : composition de routes, ownership Workers et Store partagé same-network
```
Aucun partage de gap/frontier entre Workers, aucun Job Backfill automatique, aucun endpoint/secret/URI Store/handle/payload RAW sur IPC.
## Version
```text
workspace.package.version : 0.3.15-pre.9
root Cargo header counter : 614
```
## Gate opérateur attendu
```bash
cargo fmt --all
cargo fmt --all -- --check
python3 scripts/audit_rust_workspace_rules.py
python3 scripts/audit_markdown_tables.py README.md RULES.md ROADMAP.md CHANGELOG.md docs prompts crates deltas/0.3.15
cargo check --workspace
cargo clippy --workspace --all-targets --all-features -- -D warnings
cargo test -p ksp-app-raw-transaction-ingest-desk --all-targets --all-features
cargo test -p ksp-worker-raw-transaction-ingest-lib --all-targets --all-features
cargo test --workspace --all-targets --all-features
(cd crates/ksp-app-raw-transaction-ingest-desk && cargo tauri dev)
```
Aucun script npm applicatif n'est exécuté directement par l'opérateur. `cargo tauri build` reste réservé au gate final prévu par le plan `0.3.15`.

12
docs/architecture/011-RAW_TRANSACTION_ACQUISITION.md
View File

@@ -1,5 +1,5 @@
<!-- file: docs/architecture/011-RAW_TRANSACTION_ACQUISITION.md --> <!-- file: docs/architecture/011-RAW_TRANSACTION_ACQUISITION.md -->
<!-- version: 15 --> <!-- version: 16 -->
# Acquisition et alimentation `RawTransaction` # Acquisition et alimentation `RawTransaction`
@@ -164,7 +164,7 @@ Le caller ne lui fournit pas une signature, un `program_id`, une plage historiqu
Deux entrées publiques coexistent : `start` conserve la fondation sans source productive, tandis que `start_with_runtime_resources` lance la collection de ressources live validée. Les cinq familles productives matérialisées sont : Deux entrées publiques coexistent : `start` conserve la fondation sans source productive, tandis que `start_with_runtime_resources` lance la collection de ressources live validée. Les cinq familles productives matérialisées sont :
```text ```text
Yellowstone Transaction / TransactionStatus / Block Yellowstone Transaction / TransactionStatus
Standard WS logsSubscribe Standard WS logsSubscribe
Helius transactionSubscribe Helius transactionSubscribe
-> signaux transactionnels source-neutral -> signaux transactionnels source-neutral
@@ -172,6 +172,12 @@ Helius transactionSubscribe
-> HTTP getTransaction observed -> HTTP getTransaction observed
-> Common RAW -> Common RAW
Yellowstone Block (mode de reconciliation caller-selected)
-> trigger slot temps réel
-> HTTP getBlock observed Full/Base64
-> toutes les transactions Legacy|V0|V1 du bloc
-> Common RAW
Standard WS blockSubscribe Full/Base64 Legacy|V0|V1 Standard WS blockSubscribe Full/Base64 Legacy|V0|V1
HTTP live block polling getSlot -> getBlocksWithLimit -> getBlock observed HTTP live block polling getSlot -> getBlocksWithLimit -> getBlock observed
-> qualification RAW directe Legacy|V0|V1 -> qualification RAW directe Legacy|V0|V1
@@ -297,7 +303,7 @@ HTTP getBlock full/base64 Legacy/V0/V1
provider stream compatible full transaction après parité prouvée provider stream compatible full transaction après parité prouvée
``` ```
Dans le Worker V1 actuel, Yellowstone reste volontairement traité comme signal puis hydraté par HTTP `getTransaction` afin de conserver un seul chemin de canonicalisation productif pour cette famille. Les voies directes effectivement matérialisées dans le Worker sont Standard Block et HTTP Block Polling. Le Worker conserve le mode Yellowstone transaction/status + `getTransaction` pour les compositions qui le demandent, mais la route Desk Mainnet utilise désormais Yellowstone Block comme trigger et `getBlock` comme chemin de reconciliation. Cela réduit le chemin nominal d'environ une requête HTTP par transaction à une requête par bloc tout en réutilisant la canonicalisation block déjà qualifiée. La projection protobuf Yellowstone -> RAW directe reste non revendiquée tant que la parité complète du meta n'est pas prouvée. Les voies RAW-directes existantes restent Standard Block et HTTP Block Polling.
Chemin logique : Chemin logique :

12
docs/plans/036-V0_3_15_RAW_TRANSACTION_INGEST_DESK_PLAN.md
View File

@@ -1,5 +1,5 @@
<!-- file: docs/plans/036-V0_3_15_RAW_TRANSACTION_INGEST_DESK_PLAN.md --> <!-- file: docs/plans/036-V0_3_15_RAW_TRANSACTION_INGEST_DESK_PLAN.md -->
<!-- version: 12 --> <!-- version: 13 -->
# Plan v0.3.15 — Raw Transaction Ingest Desk # Plan v0.3.15 — Raw Transaction Ingest Desk
@@ -238,7 +238,7 @@ Le frontend reçoit uniquement la projection sûre de ce modèle. Les ressources
| Route V1 | Network | Required capability | Optional capability | Config evidence | Worker resource | Composable aujourd’hui ? | Missing contract | Live-testable ? | | Route V1 | Network | Required capability | Optional capability | Config evidence | Worker resource | Composable aujourd’hui ? | Missing contract | Live-testable ? |
|-------------------------------|------------------|----------------------------------------------------------------|-------------------------------------------------|---------------------------------|-----------------------------------------------|-----------------------------------------------------------------------------------|------------------------------------------------|--------------------------------------| |-------------------------------|------------------|----------------------------------------------------------------|-------------------------------------------------|---------------------------------|-----------------------------------------------|-----------------------------------------------------------------------------------|------------------------------------------------|--------------------------------------|
| `yellowstone-hydrated` | réseau du profil | Yellowstone gRPC Subscribe + HTTP `getTransaction` même réseau | scan HTTP repair si rôle compatible | gRPC + HTTP role | `RawTransactionIngestYellowstoneSource` | oui sur profil complet et secrets résolus | aucun contrat essentiel | provider-gated selon profil | | `yellowstone-hydrated` | réseau du profil | Yellowstone gRPC Block Subscribe + HTTP `getBlock` même réseau | reconciliation HTTP bloc multi-provider | gRPC + HTTP role | `RawTransactionIngestYellowstoneSource` | oui sur profil complet et secrets résolus | aucun contrat essentiel | provider-gated selon profil |
| `standard-logs-hydrated` | réseau du profil | WS Solana standard `logsSubscribe` + HTTP `getTransaction` | scan HTTP repair si rôle compatible | `Logs` + HTTP role | `RawTransactionIngestStandardLogsSource` | oui côté Config sur profils déclarant `Logs` ; enforcement Worker en `pre.004` | enforcement Worker minimal | oui, notamment Devnet keyless | | `standard-logs-hydrated` | réseau du profil | WS Solana standard `logsSubscribe` + HTTP `getTransaction` | scan HTTP repair si rôle compatible | `Logs` + HTTP role | `RawTransactionIngestStandardLogsSource` | oui côté Config sur profils déclarant `Logs` ; enforcement Worker en `pre.004` | enforcement Worker minimal | oui, notamment Devnet keyless |
| `standard-block-direct` | réseau du profil | WS Solana standard `blockSubscribe` explicitement déclaré | aucune hydration obligatoire | WS capability `Block` | `RawTransactionIngestStandardBlockSource` | oui sur les profils publics standard qui déclarent explicitement `Block` | enforcement Worker minimal + endpoint qualifié | seulement endpoint prouvé compatible | | `standard-block-direct` | réseau du profil | WS Solana standard `blockSubscribe` explicitement déclaré | aucune hydration obligatoire | WS capability `Block` | `RawTransactionIngestStandardBlockSource` | oui sur les profils publics standard qui déclarent explicitement `Block` | enforcement Worker minimal + endpoint qualifié | seulement endpoint prouvé compatible |
| `helius-transaction-hydrated` | réseau du profil | WS Helius `transactionSubscribe` + HTTP `getTransaction` | scan HTTP repair si rôle compatible | `HeliusTransaction` + HTTP role | `RawTransactionIngestHeliusTransactionSource` | oui côté Config sur `helius_*` si secret résolu ; enforcement Worker en `pre.004` | enforcement Worker minimal | provider-gated Helius | | `helius-transaction-hydrated` | réseau du profil | WS Helius `transactionSubscribe` + HTTP `getTransaction` | scan HTTP repair si rôle compatible | `HeliusTransaction` + HTTP role | `RawTransactionIngestHeliusTransactionSource` | oui côté Config sur `helius_*` si secret résolu ; enforcement Worker en `pre.004` | enforcement Worker minimal | provider-gated Helius |
@@ -795,15 +795,17 @@ Le slot reste occupé pendant le drain terminal et la fermeture Store ; aucun se
Le test live Mainnet a montré que `yellowstone-hydrated` ouvrait bien le gRPC PublicNode mais reconstruisait son pool `getTransaction` à partir du Transport de base `mainnet_public`, donc `api.mainnet.solana.com`. Le fix impose que le gRPC Yellowstone et son hydration HTTP proviennent du même composant `transport.yellowstone`. Le profil `publicnode_mainnet` utilise désormais `https://solana-rpc.publicnode.com` pour son endpoint HTTP et n'hérite plus de la limite locale Solana-public `5 req/s`; la concurrence reste bornée et les `429` continuent à déclencher le cooldown Transport. Le test live Mainnet a montré que `yellowstone-hydrated` ouvrait bien le gRPC PublicNode mais reconstruisait son pool `getTransaction` à partir du Transport de base `mainnet_public`, donc `api.mainnet.solana.com`. Le fix impose que le gRPC Yellowstone et son hydration HTTP proviennent du même composant `transport.yellowstone`. Le profil `publicnode_mainnet` utilise désormais `https://solana-rpc.publicnode.com` pour son endpoint HTTP et n'hérite plus de la limite locale Solana-public `5 req/s`; la concurrence reste bornée et les `429` continuent à déclencher le cooldown Transport.
Ce fix ne prétend pas rendre viable le firehose Mainnet complet par hydration unitaire. La suppression de l'hydration systématique quand Yellowstone porte déjà le matériau transactionnel, le pool HTTP multi-provider, le multi-route, la déduplication cross-route et le repair HTTP par bloc sont explicitement déplacés vers `pre.009`. Ce fix ne prétend pas rendre viable le firehose Mainnet complet par hydration unitaire. `pre.009` remplace finalement ce chemin nominal par Yellowstone Block -> `getBlock` multi-provider : l'hydratation `getTransaction` systématique disparaît de la route Desk, tandis que la projection protobuf -> RAW directe reste différée jusqu'à preuve de parité canonique complète.
Budget cible : une tranche runtime mono-route. Budget cible : une tranche runtime mono-route.
### `pre.009` — optimisation Mainnet, multi-route et Store partagé ### `pre.009` — optimisation Mainnet, multi-route et Store partagé
Permettre plusieurs routes simultanées avec un Worker indépendant par route, un même Store partagé lorsque le réseau est strictement identique et une isolation des faults entre routes. La tranche doit également traiter le goulet observé sur Mainnet : privilégier un chemin Yellowstone direct lorsque le matériau RAW peut être reconstruit sans `getTransaction`, réserver HTTP à la réparation/hydration réellement nécessaire, et introduire un pool HTTP multi-provider dont les quotas, health, cooldown et capabilities restent endpoint-owned. Matérialiser plusieurs routes simultanées avec **un Worker indépendant par route**, un même `Arc<Store>` partagé uniquement lorsque le réseau est strictement identique, Stop ciblé et isolation des faults. Le premier Start same-network ouvre/valide Store ; les suivants le réutilisent ; la dernière route terminale seule ferme Store. Aucun gap, frontier ou handle Worker n'est partagé entre routes.
Le Store conserve l'idempotence `(network, signature)` et les observations/provenances permettent à plusieurs routes de confirmer la même transaction sans créer une seconde entité RAW. Le repair HTTP doit privilégier le niveau bloc lorsqu'il permet de récupérer plusieurs transactions par requête. Pour Mainnet, remplacer le chemin Yellowstone nominal `transaction -> getTransaction` par `Yellowstone Block -> getBlock Full/Base64`. L'abonnement Block sert de trigger temps réel sans transporter le payload transactionnel ; la reconciliation HTTP récupère toutes les transactions du bloc en une requête, avec retry borné/interruptible afin d'absorber le décalage court entre disponibilité gRPC et HTTP. Cette tranche ne revendique pas encore une canonicalisation protobuf -> RAW directe.
Le profil `publicnode_mainnet` expose un pool HTTP de même rôle contenant PublicNode et Solana public. Les limites, health, cooldown et concurrence restent endpoint-owned et `HttpTransportPool` conserve son routage/fallback. Store assure toujours l'idempotence `(network, signature)` et conserve les observations/provenances distinctes lorsque plusieurs routes voient la même transaction.
Budget cible : une tranche runtime/throughput Mainnet + multi-route. Budget cible : une tranche runtime/throughput Mainnet + multi-route.

12
docs/validation/032-V0_3_15_RAW_TRANSACTION_INGEST_DESK.md
View File

@@ -1,5 +1,5 @@
<!-- file: docs/validation/032-V0_3_15_RAW_TRANSACTION_INGEST_DESK.md --> <!-- file: docs/validation/032-V0_3_15_RAW_TRANSACTION_INGEST_DESK.md -->
<!-- version: 13 --> <!-- version: 14 -->
# Validation v0.3.15 — Raw Transaction Ingest Desk # Validation v0.3.15 — Raw Transaction Ingest Desk
@@ -174,7 +174,7 @@ Plusieurs Workers peuvent recevoir le même `Arc<Store>` ; leurs lifecycle/gaps/
| Route V1 | Network | Required capability | Optional capability | Config evidence | Worker resource | Composable aujourd’hui ? | Missing contract | Live-testable ? | | Route V1 | Network | Required capability | Optional capability | Config evidence | Worker resource | Composable aujourd’hui ? | Missing contract | Live-testable ? |
|-------------------------------|------------------|----------------------------------------------------------------|-------------------------------------------------|-----------------------|-----------------------------------------------|---------------------------------------------------------|--------------------------------------------------|-----------------------------------------------| |-------------------------------|------------------|----------------------------------------------------------------|-------------------------------------------------|-----------------------|-----------------------------------------------|---------------------------------------------------------|--------------------------------------------------|-----------------------------------------------|
| `yellowstone-hydrated` | réseau du profil | Yellowstone gRPC Subscribe + HTTP `getTransaction` même réseau | scan HTTP repair si rôle compatible | gRPC + HTTP role | `RawTransactionIngestYellowstoneSource` | oui sur profil complet et secrets résolus | aucun contrat essentiel | provider-gated selon profil | | `yellowstone-hydrated` | réseau du profil | Yellowstone gRPC Block Subscribe + HTTP `getBlock` même réseau | reconciliation HTTP bloc multi-provider | gRPC + HTTP role | `RawTransactionIngestYellowstoneSource` | oui sur profil complet et secrets résolus | aucun contrat essentiel | provider-gated selon profil |
| `standard-logs-hydrated` | réseau du profil | WS Solana standard `logsSubscribe` + HTTP `getTransaction` | scan HTTP repair si rôle compatible | WS + HTTP role | `RawTransactionIngestStandardLogsSource` | non prouvable strictement avant capability WS explicite | Transport + Config + enforcement Worker minimaux | oui, notamment Devnet keyless après extension | | `standard-logs-hydrated` | réseau du profil | WS Solana standard `logsSubscribe` + HTTP `getTransaction` | scan HTTP repair si rôle compatible | WS + HTTP role | `RawTransactionIngestStandardLogsSource` | non prouvable strictement avant capability WS explicite | Transport + Config + enforcement Worker minimaux | oui, notamment Devnet keyless après extension |
| `standard-block-direct` | réseau du profil | WS Solana standard `blockSubscribe` explicitement déclaré | aucune hydration obligatoire | WS capability Block | `RawTransactionIngestStandardBlockSource` | non prouvable aujourd’hui | Transport + Config + enforcement Worker minimaux | seulement endpoint prouvé compatible | | `standard-block-direct` | réseau du profil | WS Solana standard `blockSubscribe` explicitement déclaré | aucune hydration obligatoire | WS capability Block | `RawTransactionIngestStandardBlockSource` | non prouvable aujourd’hui | Transport + Config + enforcement Worker minimaux | seulement endpoint prouvé compatible |
| `helius-transaction-hydrated` | réseau du profil | WS Helius `transactionSubscribe` + HTTP `getTransaction` | scan HTTP repair si rôle compatible | Helius WS + HTTP role | `RawTransactionIngestHeliusTransactionSource` | non dans les profils committed actuels | Config data/composite + capability WS explicite | provider-gated Helius | | `helius-transaction-hydrated` | réseau du profil | WS Helius `transactionSubscribe` + HTTP `getTransaction` | scan HTTP repair si rôle compatible | Helius WS + HTTP role | `RawTransactionIngestHeliusTransactionSource` | non dans les profils committed actuels | Config data/composite + capability WS explicite | provider-gated Helius |
@@ -419,11 +419,15 @@ Le parcours live `pre.008-fix.003` prouve que le channel Yellowstone PublicNode
Le fix corrige la composition : `prepare_yellowstone` prend désormais son role/pool HTTP dans le même `ResolvedTransportConfig` optionnel qui fournit le gRPC. Le profil `publicnode_mainnet` expose `publicnode_solana_mainnet_rpc` sur `https://solana-rpc.publicnode.com`, provider `publicnode`, sans recopier la limite locale `5 req/s` du profil Solana-public. La concurrence locale reste bornée à 8 et le cooldown Transport sur `429` reste actif. Le fix corrige la composition : `prepare_yellowstone` prend désormais son role/pool HTTP dans le même `ResolvedTransportConfig` optionnel qui fournit le gRPC. Le profil `publicnode_mainnet` expose `publicnode_solana_mainnet_rpc` sur `https://solana-rpc.publicnode.com`, provider `publicnode`, sans recopier la limite locale `5 req/s` du profil Solana-public. La concurrence locale reste bornée à 8 et le cooldown Transport sur `429` reste actif.
Non-claim : ce changement réduit une erreur de composition mais ne garantit pas qu'une hydration HTTP unitaire puisse suivre le firehose Mainnet. L'optimisation structurelle Yellowstone direct + pool HTTP multi-provider + repair bloc est reportée à `pre.009`. Non-claim : ce changement réduit une erreur de composition mais ne garantit pas qu'une hydration HTTP unitaire puisse suivre le firehose Mainnet. `pre.009` choisit finalement Yellowstone Block -> `getBlock` multi-provider comme optimisation sûre ; la projection protobuf -> RAW directe reste non revendiquée.
### `pre.009` — optimisation Mainnet et runtime multi-route ### `pre.009` — optimisation Mainnet et runtime multi-route
Un Worker par route, même Store si réseau identique, isolation des faults, chemin Yellowstone direct lorsque possible, pool HTTP multi-provider et repair par bloc pour éviter que `getTransaction` unitaire soit le chemin nominal du temps réel. La tranche matérialise un Worker indépendant par route, Stop ciblé, Store partagé uniquement sur identité réseau exacte et fermeture Store par la dernière route. Les faults restent route-local et aucune TargetCoverage/gap/frontier n'est partagée entre Workers.
La route Yellowstone bascule sur un filtre Block sans transactions embarquées puis reconcile le slot par `getBlock` Full/Base64. Le Worker distingue explicitement son ancien mode transaction-hydration du mode block-reconciliation ; ce dernier n'entre plus dans la registry d'hydration `getTransaction`. La récupération bloc est interruptible par Stop et retente de façon bornée lorsque le bloc n'est pas encore visible ou qu'un endpoint du pool échoue.
Le profil `publicnode_mainnet` contient deux endpoints HTTP de même rôle : PublicNode et Solana public repair. La validation doit prouver que Config conserve leurs limites séparées et que la route exige désormais `getBlock`, pas `getTransaction`. Le direct protobuf -> RAW n'est pas revendiqué.
### `pre.010` — monitoring ### `pre.010` — monitoring