102 Commits

Author SHA1 Message Date
dd48ed9140 v0.3.6-rel.001 2026-09-01 23:43:43 +02:00
5a65c28898 v0.3.6-pre.013 2026-09-01 23:38:36 +02:00
42374115c7 v0.3.6-pre.012 2026-09-01 17:42:24 +02:00
9506df9487 v0.3.6-pre.011 2026-09-01 17:18:12 +02:00
7f8ce987a5 v0.3.6-pre.010 2026-09-01 16:55:14 +02:00
8dbd86c297 v0.3.6-pre.009-fix.002 2026-09-01 16:30:51 +02:00
75b2d7e7f1 v0.3.6-pre.009-fix.001 2026-09-01 16:17:22 +02:00
b861a1e3b8 v0.3.6-pre.009 2026-09-01 16:04:58 +02:00
d89aee9910 v0.3.6-pre.008-fix.001 2026-09-01 15:22:54 +02:00
75cbf9df09 v0.3.6-pre.008 2026-09-01 15:14:57 +02:00
138395ac35 v0.3.6-pre.007 2026-09-01 14:47:23 +02:00
be0032e76b v0.3.6-pre.006-fix.002 2026-09-01 14:28:13 +02:00
8096e6f07b v0.3.6-pre.006-fix.001 2026-09-01 14:24:22 +02:00
032919fa07 v0.3.6-pre.006 2026-09-01 14:08:53 +02:00
ea5f756d17 v0.3.6-pre.005-fix.001 2026-09-01 13:42:34 +02:00
4fdeab4d67 v0.3.6-pre.005 2026-09-01 13:00:21 +02:00
478548b7f4 v0.3.6-pre.004 2026-09-01 12:57:25 +02:00
20cf700f22 v0.3.6-pre.003-fix.001 2026-09-01 12:05:28 +02:00
f92c22d55a v0.3.6-pre.003 2026-09-01 11:58:38 +02:00
cb670952e8 v0.3.6-pre.002-fix.001 2026-09-01 10:55:35 +02:00
a25d8dd55a v0.3.6-pre.002 2026-09-01 10:45:39 +02:00
ecfc30a94b v0.3.6-pre.001-fix.001 2026-09-01 10:43:09 +02:00
6f2d0a092a v0.3.6-pre.001 2026-09-01 09:35:02 +02:00
3de0488cab v0.3.5-rel.001 2026-08-31 14:28:12 +02:00
2460162243 v0.3.5-pre.008-fix.001 2026-08-31 14:27:15 +02:00
1b9104d64b v0.3.5-pre.008 2026-08-31 14:01:23 +02:00
80be36bfdd v0.3.5-pre.007 2026-08-31 13:07:56 +02:00
e8d2382ac3 v0.3.5-pre.006 2026-08-31 11:43:40 +02:00
749e53edc2 v0.3.5-pre.005-fix.001 2026-08-31 11:23:40 +02:00
7377a6f6cd v0.3.5-pre.005 2026-08-31 11:17:09 +02:00
45601ce386 v0.3.5-pre.004 2026-08-31 11:14:44 +02:00
7d490eb5d8 v0.3.5-pre.003 2026-08-31 11:01:45 +02:00
ca018095bf v0.3.5-pre.002-fix.001 2026-08-31 10:44:51 +02:00
0dd722ffca v0.3.5-pre.002 2026-08-31 10:36:10 +02:00
75e8030b07 v0.3.5-pre.001-fix.001 2026-08-31 10:21:47 +02:00
c24e4c48f8 v0.3.5-pre.001 2026-08-31 09:45:27 +02:00
c7cfb9557d v0.3.4-rel.001 2026-08-31 00:52:56 +02:00
c27428d6c0 v0.3.4-pre.013 2026-08-31 00:46:52 +02:00
dfa01b1381 v0.3.4-pre.012 2026-08-31 00:33:51 +02:00
5fc7f65b2c v0.3.4-pre.011 2026-08-30 23:56:11 +02:00
bbf804b7fc v0.3.4-pre.010 2026-08-30 23:46:37 +02:00
13687ec2fe v0.3.4-pre.009 2026-08-30 23:39:43 +02:00
b756da5149 v0.3.4-pre.008-fix.001 2026-08-30 23:25:12 +02:00
04234926ce v0.3.4-pre.008 2026-08-30 23:15:48 +02:00
bfc69f5631 v0.3.4-pre.007-fix.001 2026-08-30 22:54:53 +02:00
902b5fba99 v0.3.4-pre.007 2026-08-30 22:47:33 +02:00
fccb7d876c v0.3.4-pre.006 2026-08-30 22:31:02 +02:00
50d4142797 v0.3.4-pre.005 2026-08-30 22:17:47 +02:00
eb6dbc31e8 v0.3.4-pre.004-fix.001 2026-08-30 22:06:24 +02:00
fb8b585aa2 v0.3.4-pre.004 2026-08-30 21:57:23 +02:00
dad18c16d5 v0.3.4-pre.003-fix.001 2026-08-30 21:45:30 +02:00
a704a5722e v0.3.4-pre.003 2026-08-30 21:37:02 +02:00
dfa3723a7f v0.3.4-pre.002-fix.001 2026-08-30 21:17:38 +02:00
6bd593f467 v0.3.4-pre.002 2026-08-30 21:11:23 +02:00
03ad0ba063 v0.3.4-pre.001-fix.001 2026-08-30 19:29:05 +02:00
d95b1095e5 v0.3.4-pre.001 2026-08-30 19:21:15 +02:00
bcce89f076 v0.3.3-rel.001 2026-08-30 18:26:36 +02:00
66d5b03495 v0.3.3-pre.013 2026-08-30 18:03:06 +02:00
7af751c888 v0.3.3-pre.012-fix.001 2026-08-30 17:46:49 +02:00
9f2d5ea704 v0.3.3-pre.012 2026-08-30 17:02:44 +02:00
8428b75b96 v0.3.3-pre.011 2026-08-30 16:37:28 +02:00
282b616a0e v0.3.3-pre.010-fix.001 2026-08-30 16:07:51 +02:00
12eaf7a3de v0.3.3-pre.010 2026-08-30 16:06:20 +02:00
f28e4d87f0 v0.3.3-pre.009-fix.006 2026-08-30 15:53:07 +02:00
ba2e52f841 v0.3.3-pre.009-fix.005 2026-08-30 15:46:03 +02:00
5f1938c7c3 v0.3.3-pre.009-fix.004 2026-08-30 15:05:58 +02:00
5e2f959c01 v0.3.3-pre.009-fix.003 2026-08-30 14:56:51 +02:00
8da04717b6 v0.3.3-pre.009-fix.002 2026-08-30 14:48:41 +02:00
c094828704 v0.3.3-pre.009-fix.001 2026-08-30 14:43:00 +02:00
ce658e58f7 v0.3.3-pre.009 2026-08-30 14:29:33 +02:00
7f0a642972 v0.3.3-pre.008-fix.001 2026-08-30 14:10:43 +02:00
84ab2b3651 v0.3.3-pre.008 2026-08-30 14:10:10 +02:00
1f0b202135 v0.3.3-pre.007-fix.001 2026-08-30 13:43:58 +02:00
aa56a12846 v0.3.3-pre.007 2026-08-30 13:37:22 +02:00
10996f11f7 v0.3.3-pre.006-fix.001 2026-08-30 13:13:01 +02:00
1352a61d70 v0.3.3-pre.006 2026-08-30 13:06:33 +02:00
66a3926adb v0.3.3-pre.005 2026-08-30 12:42:50 +02:00
917e602a87 v0.3.3-pre.004 2026-08-30 11:00:07 +02:00
56fadb364a v0.3.3-pre.003-fix.003 2026-08-30 10:36:22 +02:00
8fa1c8ce8d v0.3.3-pre.003-fix.002 2026-08-30 10:26:38 +02:00
c17e78c6a8 v0.3.3-pre.003-fix.001 2026-08-30 10:21:48 +02:00
61bf7ba468 v0.3.3-pre.003 2026-08-30 09:22:53 +02:00
9712c7e1f7 v0.3.3-pre.002 2026-08-30 08:41:26 +02:00
a560df80ce v0.3.3-pre.001 2026-08-30 07:59:00 +02:00
bb56574824 v0.3.2-rel.001 2026-08-30 06:13:07 +02:00
8363c8bdc9 v0.3.2-pre.012 2026-08-30 06:08:18 +02:00
b375f263dc v0.3.2-pre.011 2026-08-30 05:55:29 +02:00
e1fe419028 v0.3.2-pre.010 2026-08-29 23:44:19 +02:00
9ec4f26874 v0.3.2-pre.009 2026-08-29 22:00:20 +02:00
80eea82398 v0.3.2-pre.008 2026-08-29 21:54:00 +02:00
e39cf656b2 v0.3.2-pre.007 2026-08-29 21:40:57 +02:00
c4f56d9e85 v0.3.2-pre.006 2026-08-29 19:53:00 +02:00
c1dbaad88d v0.3.2-pre.005-fix.001 2026-08-29 19:35:35 +02:00
d93184d41d v0.3.2-pre.005 2026-08-29 19:33:43 +02:00
de3cec6a23 v0.3.2-pre.004-fix.001 2026-08-29 19:04:58 +02:00
0a4cddafc4 v0.3.2-pre.004-fix.001 2026-08-29 19:04:47 +02:00
6d2b1401aa v0.3.2-pre.004 2026-08-29 18:42:37 +02:00
0a8fb5dd1b v0.3.2-pre.003-fix.001 2026-08-29 17:47:22 +02:00
a8c90107b5 v0.3.2-pre.003 2026-08-29 17:43:52 +02:00
ced653bfc0 v0.3.2-pre.002-fix.001 2026-08-29 17:42:14 +02:00
153014be9d v0.3.2-pre.002 2026-08-29 16:48:16 +02:00
8d4b3b67fb v0.3.2-pre.001 2026-08-29 16:45:14 +02:00
352 changed files with 58821 additions and 619 deletions

View File

@@ -1,10 +1,19 @@
# file: .env.example
# version: 10
# version: 13
# KSP Logging root directory. Used by config/std.logging.json for relative log output paths.
# The current Config document fallback is "logs" when neither the process environment nor .env defines this variable.
KSP_LOGS_DIRECTORY=logs
# PostgreSQL URI for the Devnet Store target.
KSP_SECRET_STORE_DEVNET_POSTGRES_URI=postgresql://...
# PostgreSQL URI for the Mainnet Store target.
KSP_SECRET_STORE_MAINNET_POSTGRES_URI=postgresql://...
# PostgreSQL URI for the Testnet Store target.
KSP_SECRET_STORE_TESTNET_POSTGRES_URI=postgresql://...
# KSP Wallet root directory. Used by config/std.wallet.json before an optional profile subdirectory is appended.
# The committed Wallet document falls back to "wallets" when neither the process environment nor .env defines this variable.
KSP_WALLETS_DIRECTORY=wallets

View File

@@ -1,8 +1,68 @@
<!-- file: CHANGELOG.md -->
<!-- version: 19 -->
<!-- version: 26 -->
# Changelog KSP
## 0.3.6 — Job API + backfill RAW historique observable — 2026-09-01
`0.3.6` introduit `ksp-job-api` comme contrat passif et runtime-neutral pour les traitements bornés/terminables : identité `JobId`/`JobKindCode`, lifecycle explicite, annulation coopérative partagée, snapshots typés et notifications latest-value consommables par plusieurs listeners sans transformer l'API en scheduler, runtime Tokio ou bus d'événements. La surface reste Core-only et sert immédiatement au premier consumer concret `ksp-job-backfill-lib`.
`ksp-job-backfill-lib` ferme une première verticale historique `RawTransaction` sur les abstractions KSP existantes. Les scopes `LatestAddress`, `BeforeAddress`, `AfterAddress` et `ExplicitSignatures` sont bornés par pages/candidats/concurrence, dédupliqués de manière stable et liés à un fingerprint sémantique réseau/scope qui exclut volontairement provider, endpoint, rôle Transport et `JobId`. La découverte utilise les wrappers typés de `ksp-onchain-transport-lib`; l'hydratation passe exclusivement par `get_transaction_observed`, conserve la provenance du provider/endpoint gagnant et transforme la réponse en RAW v1 canonique avec hash SHA-256 exact, sans SDK provider ni modèle wire dupliqué.
La persistance passe uniquement par `ksp-store-lib` et son écriture atomique `RawTransaction + RawTransactionObservation` en mode normal. Les résultats distinguent insertion, présence idempotente, tombstone purgé, transaction manquante et conflit de contenu ; aucun read-before-write, `ForceRehydrate`, SQL, backend PostgreSQL direct ou décision de rétention n'est déplacé dans Job. Le réseau Store est vérifié avant écriture et l'identité logique de transaction reste `(RawNetworkId, Signature)` indépendamment de la source d'acquisition.
Le runtime Backfill ajoute concurrence bornée, réconciliation des terminaisons hors ordre, frontier strictement contiguë et checkpoint caller-owned. Les reprises `Latest`, `Before`, `After` et `ExplicitSignatures` ne sautent que le préfixe durable réellement prouvé. `BackfillJobRuntime` expose un `BackfillJobHandle` clonable avec annulation coopérative et `BackfillSnapshotSource` latest-value ; les opérations pré-Store peuvent être abandonnées, tandis qu'une persistance déjà soumise est toujours drainée avant la décision terminale. Les snapshots conservent phase, compteurs, holes/conflicts/missing, maximum in-flight, frontier/checkpoint et code d'échec sans projeter payload RAW, secret, URL ou état runtime interne.
Le hardening externe verrouille les codes d'erreur, bornes adversariales, redaction `Debug`, indépendance du fingerprint, absence de fuite Tokio/Futures/backend dans la façade, dépendances backend-neutral et discipline d'import/visibilité crate-root. La réconciliation documentaire finale ajoute les README/USAGE durables de Job API et Backfill et aligne les architectures Jobs/Workers/Apps sans transformer les guides d'utilisation en journal de release.
Les gates de clôture passent `cargo fmt --all -- --check`, audits Rust/Markdown, `cargo check --workspace`, Clippy workspace `--all-targets --all-features -- -D warnings`, `cargo test --workspace --all-targets --all-features`, tests ciblés Job/Backfill et graphes Cargo. Les smokes réseau/Tauri déjà explicitement opt-in restent ignorés par défaut ; aucune preuve non exécutée n'est déclarée PASS.
## 0.3.5 — Interface acquisition events partagés — 2026-08-31
`0.3.5` étend `ksp-interface-lib` avec deux familles passives d'acquisition réellement partagées, sans transformer Interface en runtime, en event bus ou en seconde couche RAW. `SlotLifecycleEvent` expose un `slot` et un `SlotLifecycleStage` non exhaustif limité à `Processed`, `FirstShredReceived`, `Completed`, `CreatedBank`, `Dead`, `OptimisticallyConfirmed` et `Rooted`. La normalisation conserve la sémantique commune : les notifications Solana `optimisticConfirmation` et Yellowstone `Confirmed` convergent vers `OptimisticallyConfirmed`, tandis que Solana `root` et Yellowstone `Finalized` convergent vers `Rooted`; les différences d'ordre, de complétude, de replay et de transport restent la responsabilité du producteur/Transport.
La seconde famille matérialise le fait minimal d'exécution transactionnelle partagé par plusieurs sources. `TransactionSignature` possède exactement 64 octets et un `Debug` redacted, `TransactionExecutionOutcome` distingue seulement `Succeeded` et `Failed`, et `TransactionExecutionEvent` transporte uniquement `slot + signature + outcome`. Les logs, erreurs provider détaillées, commitments, indexes, timestamps, filtres et payloads complets restent Transport-owned. Les snapshots HTTP `getSignatureStatuses`, les transitions one-shot `signatureSubscribe`, un éventuel `TransactionLogEvent`, les votes et les entrées Yellowstone ne sont pas fusionnés artificiellement dans ce contrat.
La frontière d'ownership reste stricte : les DTOs wire/provider demeurent dans `ksp-onchain-transport-lib`, les événements passifs provider-neutral sont Interface-owned, et les modèles persistants/replayables `RawTransaction` / `RawAccountState` restent `ksp-store-api`. `ksp-interface-lib` conserve exactement `ksp-core-lib` comme seule dépendance normale, sans feature propre, dev/build dependency, serde, codec, logging ou runtime. Les canaris publics, external consumer, inventaires exacts et hardening vérifient également qu'aucun second RAW, metadata source ou payload hostile n'entre dans la surface Interface.
Les gates de clôture passent audits Rust/Markdown, `cargo check --workspace`, Clippy all-targets, tests ciblés Interface/Program API, `cargo test --workspace` et graphes Cargo. `cargo tree -p ksp-interface-lib --edges normal` confirme le chemin `ksp-interface-lib -> ksp-core-lib -> solana-pubkey -> solana-address`; le graphe features ne montre aucune feature propre Interface et les doublons éventuels restent ceux du workspace global. La documentation durable a été réconciliée et `TransactionLogEvent` est conservé comme idée différée soumise à un nouveau gate consumer/bornes.
`prompts/025-V0_3_6_START_PROMPT.md` ouvre `0.3.6` sur le développement parallèle de `ksp-job-api` et `ksp-job-backfill-lib`. La release doit reprendre fonctionnellement le backfill historique kbot3 sans en copier le code : audit obligatoire de l'archive historique, `RawTransaction` par adresse via `getSignaturesForAddress` + `getTransaction`, directions/anchors et déduplication, hydratation/persistence via les façades KSP, frontier/checkpoint/reprise, cancellation/concurrency bornées, idempotence et distinctions missing/conflit. `ksp-job-api` doit en parallèle stabiliser lifecycle/progress/outcome et un contrat de notifications/listeners borné pour qu'une couche supérieure puisse visualiser l'état sans parser les logs. Le ROADMAP enchaîne ensuite l'app de backfill/inspection `0.3.7`, `ksp-worker-api` `0.3.8`, `ksp-worker-live-transactions-retriever-lib` `0.3.9` puis une application de monitoring/visualisation Jobs + Workers `0.3.10`, avec reprise du même pattern d'observabilité côté Worker sans fusionner les sémantiques Job/Worker/Store.
## 0.3.4 — Store/PostgreSQL RawAccountState + complétude RAW — 2026-08-31
`0.3.4` complète la seconde vertical slice RAW physique sur le couple `ksp-store-lib` / `ksp-store-postgres-lib` et ferme la conformance PostgreSQL des **10 capabilities** backend-agnostic de `ksp-store-api` : les six capabilities `RawTransaction*` acquises en `0.3.3` restent intactes et les quatre capabilities `RawAccountStateRead`, `RawAccountStateWrite`, `RawAccountObservationRead` et `RawAccountObservationWrite` sont désormais implémentées par `PostgresBackend` puis dispatchées par la façade `Store`. La séparation reste stricte : les consommateurs ordinaires passent par `ksp-store-lib`, le backend PostgreSQL conserve SQL/driver/pool/TLS/migrations privés, et la façade reste compilable/testable sans backend via `--no-default-features`.
La migration additive V002 introduit `ksp_raw_account_states` et `ksp_raw_account_observations` au-dessus de V000/V001 sans modifier leurs bytes. L'identité canonique account est `(pubkey, slot, state_hash)` : plusieurs états d'un même compte dans un même slot restent représentables lorsque `state_hash` diffère, tandis qu'une collision sur la référence complète déclenche une comparaison exacte de `lamports`, `owner`, `executable`, `rent_epoch` et `data` avant de conclure à l'idempotence ou à `store_api.raw_conflict`. Les `u64` physiques utilisent `NUMERIC(20,0)`, les clés/hashes/signatures utilisent des `BYTEA` de largeur contrainte, et les bytes account restent complets jusqu'à la borne KSP de 16 MiB. V002 contient exactement 32 ressources gérées et son checksum final est `ff21605ed45f7ab4c0f92bbb692700b4118a9488b04d50a31d259ac59bdb550e`; V000 et V001 restent respectivement `d29068b8c13b9dc0cc9ef6aaadd0fa12d41e0fe4c56541a1118c4bfc846a1450` et `31488cda2f08f3f46c4cdbdbb6c18c243662fada02eac4487040c8735d72cc51`.
L'acquisition `RawAccountState + RawAccountObservation` est transactionnelle : `INSERT ... ON CONFLICT DO NOTHING`, lecture/verrouillage du canonical en collision, comparaison exacte et rollback complet lorsque l'observation diverge. Une observation supplémentaire vérifie la référence existante sous transaction et ne crée jamais implicitement son state. Les métadonnées Yellowstone `is_startup`, `transaction_signature` et `write_version` restent optionnelles et observation-only ; aucune FK transaction n'est inventée. La navigation account utilise une keyset `(slot, pubkey, state_hash)` ASC/DESC, avec filtre pubkey optionnel et cursor KSPA V1 opaque de 109 octets lié au réseau, au filtre, au range, à la direction et à la famille afin d'empêcher les replays cross-query/cross-family. Aucun `OFFSET`, plafond métier de batch, index owner/provider/time ou lifecycle destructif account n'est introduit.
Les gates de clôture valident audits Rust/Markdown, `cargo check --workspace`, Clippy all-targets, tests ciblés Store/API/PostgreSQL/Config, `cargo test --workspace`, façade sans feature PostgreSQL et graphes Cargo. Le live `postgres_raw_account_live` passe sur **PostgreSQL 17** avec bootstrap V000/V001/V002, round-trip des bytes et du domaine `u64`, idempotence/conflit, états distincts même pubkey+slot, observations/metadata optionnelles, pagination/cursors, concurrence, annulation/rollback et coexistence `RawTransaction`. La réconciliation documentaire finale est ensuite validée sans réouvrir code ni migrations.
`prompts/024-V0_3_5_START_PROMPT.md` ouvre `0.3.5` sur `ksp-interface-lib` uniquement. Cette release doit réauditer les surfaces d'acquisition actuelles et matérialiser seulement les modèles passifs/event-only réellement partagés, sans recopier `RawTransaction`/`RawAccountState`, sans créer un event bus et sans déplacer les DTOs provider/transport. Les candidats logs, slot/root/slotsUpdates, transaction status et vote sont traités par matrice sémantique ; une famille reste reportée si la convergence ou le consumer réel n'est pas démontré. L'archive historique kbot3 n'est pas requise pour ce gate : les sources de vérité sont la base KSP stable et les contrats officiels actuels des transports concernés.
## 0.3.3 — Store/PostgreSQL RawTransaction vertical slice — 2026-08-30
`0.3.3` complète la première vertical slice RAW physique sur le couple `ksp-store-lib` / `ksp-store-postgres-lib` sans modifier les contrats backend-agnostic acquis dans `ksp-store-api`. `PostgresBackend` et la façade `Store` implémentent désormais les six capabilities `RawTransactionRead`, `RawTransactionWrite`, `RawTransactionObservationRead`, `RawTransactionObservationWrite`, `RawTransactionRetentionRead` et `RawTransactionRetentionWrite`. Une base PostgreSQL reste liée à un unique `RawNetworkId` par `ksp_store_identity`; le mauvais réseau est refusé avant I/O, les slots `u64` sont conservés exactement en `NUMERIC(20,0)`, et la migration logique V001 reste découpée en ressources tables/contraintes/indexes avec vérification de compatibilité du schéma effectif.
La persistence canonique utilise une transaction PostgreSQL unique pour `RawTransaction + observation`, tente l'insert sous contrainte d'unicité puis compare le contenu réel sous verrou en cas de collision : contenu identique -> idempotence, divergence -> `store_api.raw_conflict`. Les observations supplémentaires conservent leur provenance complète et ne créent jamais implicitement un canonical absent. La lecture reconstruit `Full` depuis le payload chaud et `Archived` depuis la relation archive ; `Purged` reste un tombstone minimal. La navigation est une keyset stricte `(slot, signature)` ASC/DESC sans `OFFSET`, avec cursor V1 opaque de 109 octets lié au réseau, à la direction, aux bornes et à la dernière clé. Aucun plafond métier de batch n'est introduit : seule la borne physique PostgreSQL de `LIMIT requested + 1` est appliquée.
La rétention physique supportée est `Full -> Archived -> Purged`, sérialisée par `FOR UPDATE`, avec outcomes `Applied`, `AlreadyAtTarget` et `ExpectedStateMismatch`. Le mode normal respecte le tombstone après purge ; `ForceRehydrate` reste explicite et atomique. `Compacted` demeure volontairement non supporté physiquement et retourne `store.postgres_retention_compaction_unsupported` plutôt que de présenter TOAST ou une compression implicite comme contrat KSP. Les erreurs backend sont réduites à des classes/phases statiques puis projetées vers des codes Store/API stables, sans URI, SQL, SQLSTATE, bind ni texte serveur.
La preuve PostgreSQL réelle a conduit à durcir l'introspection de schéma PostgreSQL 17 : canonicalisation ciblée des CHECK numériques reconstruits par le catalogue, conservation des littéraux texte, restauration des helpers de classification de schéma et distinction d'un drift d'une migration déjà enregistrée lorsque `schema_autoupdate=false`. Les ressources SQL V000/V001 et leurs checksums sont restés inchangés pendant ces corrections (`V000 d29068b8c13b9dc0cc9ef6aaadd0fa12d41e0fe4c56541a1118c4bfc846a1450`, `V001 31488cda2f08f3f46c4cdbdbb6c18c243662fada02eac4487040c8735d72cc51`).
Le gate technique final `pre.011` passe audits Rust/Markdown, `cargo check --workspace`, Clippy all-targets, les tests ciblés Store/API/PostgreSQL/Config, les tests et checks façade avec `--no-default-features`, `cargo test --workspace` et les graphes Cargo. Le live `postgres_raw_transaction_live` est ensuite rejoué avec succès sur **PostgreSQL 17**, couvrant bootstrap/drift-repair, atomicité, concurrence identique/divergente, rollback sur collision et annulation, pagination/cursor, rétention/races, ForceRehydrate et réouverture durable. Aucun build Tauri supplémentaire n'est requis : `0.3.3` ne change ni resources applicatives ni packaging desktop. `RawAccountState` PostgreSQL et la complétude RAW restent réservés à `0.3.4`. `prompts/023-V0_3_4_START_PROMPT.md` ouvre cette slice suivante sur les quatre capabilities `RawAccount*`, une migration additive au-dessus de V000/V001, puis la conformance finale des dix capabilities RAW ; l'archive historique kbot3 y reste une source de comparaison ciblée account/observation, jamais une architecture à recopier.
## 0.3.2 — Store/PostgreSQL runtime foundation — 2026-08-30
`0.3.2` introduit ensemble `ksp-store-lib` et `ksp-store-postgres-lib` comme fondation runtime/backend PostgreSQL au-dessus des contrats backend-agnostic de `ksp-store-api`, sans encore implémenter de capability RAW métier. La façade `Store` conserve un backend connu `Postgres` activé par défaut via Cargo feature, reste compilable avec `--no-default-features`, lie chaque instance à exactement un `RawNetworkId` et n'expose aucun `Pool`, `Client`, `Row`, `Statement`, driver ou SQL physique. `ksp-config-lib` possède désormais `std.store` avec trois targets nommés `devnet`, `mainnet` et `testnet`, chacun associé à un réseau explicite et à une URI PostgreSQL Secret indépendante ; Store/backend ne lisent directement ni `.env`, ni `KSP_*`, ni `PG*`, ni `.pgpass`.
Le backend de référence utilise `tokio-postgres 0.7.18`, `deadpool-postgres 0.14.2`, `tokio-postgres-rustls 0.14.0`, Rustls 0.23 avec roots système/AWS-LC et des modes TLS KSP limités à `Disabled` et `VerifyFull`. Les URI sont parsées puis normalisées sous la policy typée KSP, le pool et les deadlines connect/wait/create/recycle/shutdown sont bornés, et une ouverture réussie exige une connexion physique puis un bootstrap valide. Le moteur de migrations privé commence par `V000__bootstrap.sql`, enregistre version/nom/SHA-256 dans `ksp_store_schema_migrations`, sérialise les runners par advisory transaction lock borné et refuse checksum mismatch ou schema plus récent sans down automatique. La façade fournit également `runtime_snapshot()` et `health().await` avec une projection portable/redacted de readiness et des compteurs pool sûrs.
Les canaris de hardening verrouillent 84 exports crate-root côté façade, la frontière backend/Config, l'absence de bypass environnement, la redaction des URI/erreurs, la matrice de features et l'absence de persistence `RawTransaction`/`RawAccountState`. Le gate technique final a été rejoué après `cargo clean` : audits Rust/Markdown, `cargo check --workspace`, Clippy, tests ciblés de toutes les crates, `cargo test --workspace`, graphes Cargo et les trois builds Tauri Linux passent. Le smoke PostgreSQL réel passe sur **PostgreSQL 17** et prouve bootstrap initial/idempotent/concurrent, mismatch/recovery checksum, rollback transactionnel, health `Ready` et fermeture bornée ; la policy de fondation reste PostgreSQL >= 15 sans maximum KSP artificiel.
`prompts/022-V0_3_3_START_PROMPT.md` ouvre ensuite uniquement la vertical slice PostgreSQL `RawTransaction` complète sur les mêmes crates et la même fondation : six capabilities transaction/observation/rétention, atomicité, idempotence/conflit, get/list cursorisé, tombstone et `ForceRehydrate`. `RawAccountState` PostgreSQL et la complétude RAW restent réservés à `0.3.4`. L'archive historique `khadhroony-bot3_v0.5.3-pre.005-fix010.zip` reste obligatoire au `pre.001` de `0.3.3`, mais seulement pour réauditer l'ancien schéma/repository RAW transaction comme source d'héritage, jamais comme autorité KSP.
## 0.3.1 — Store API RAW foundation — 2026-08-29
`0.3.1` introduit `ksp-store-api` comme contrat backend-agnostic de persistence N1 RAW, sans runtime Store ni backend physique. La release stabilise deux familles réellement convergentes : `RawTransaction` avec payload canonique opaque/versionné, identité réseau+signature et observations dacquisition séparées, puis `RawAccountState` avec bytes complets, identité réseau+pubkey+slot+hash et observations pouvant conserver les enrichissements Yellowstone sans les confondre avec létat canonique. `TransactionStatusObservation` reste reporté faute de convergence sémantique suffisante entre snapshot HTTP, transition WebSocket et update Yellowstone ; `logsSubscribe`, slot/root/slotsUpdates et vote restent event-only candidats, `RawBlock` reste une idée conditionnelle et Yellowstone `Entry` reste rejeté de la taxonomie active.

View File

@@ -1,12 +1,12 @@
# file: Cargo.toml
# version: 331
# version: 415
[workspace]
resolver = "3"
members = ["crates/ksp-app-config-desk", "crates/ksp-app-solprices-desk", "crates/ksp-app-wallet-desk", "crates/ksp-config-lib", "crates/ksp-core-lib", "crates/ksp-interface-lib", "crates/ksp-logging-lib", "crates/ksp-offchain-transport-lib", "crates/ksp-onchain-transport-lib", "crates/ksp-program-api", "crates/ksp-store-api", "crates/ksp-wallet-lib"]
members = ["crates/ksp-app-config-desk", "crates/ksp-app-solprices-desk", "crates/ksp-app-wallet-desk", "crates/ksp-config-lib", "crates/ksp-core-lib", "crates/ksp-interface-lib", "crates/ksp-job-api", "crates/ksp-job-backfill-lib", "crates/ksp-logging-lib", "crates/ksp-offchain-transport-lib", "crates/ksp-onchain-transport-lib", "crates/ksp-program-api", "crates/ksp-store-api", "crates/ksp-store-lib", "crates/ksp-store-postgres-lib", "crates/ksp-wallet-lib"]
[workspace.package]
version = "0.3.1"
version = "0.3.6"
edition = "2024"
license = "MIT"
repository = "https://git.sasedev.com/Sasedev/khadhroony-solana-project"
@@ -14,20 +14,24 @@ authors = ["SinuS von SifriduS <sinus@sasedev.net>"]
publish = false
[workspace.dependencies]
argon2 = { version = "^0.5", default-features = false }
argon2 = { version = "^0.6", default-features = false }
base64 = { version = "^0.23" }
chacha20poly1305 = { version = "^0.11", default-features = false }
chrono = { version = "^0.4", default-features = false }
deadpool-postgres = { version = "^0.14", default-features = false }
directories = { version = "^6.0" }
ed25519-dalek = { version = "^3.0", default-features = false }
fs2 = { version = "^0.4" }
futures-util = { version = "^0.3", default-features = false }
getrandom = { version = "^0.4", default-features = false }
http = { version = "^1.5", default-features = false }
jsonschema = { version = "^0.51", default-features = false }
jsonschema = { version = "^0.52", default-features = false }
reqwest = { version = "^0.13", default-features = false }
rustls = { version = "^0.23", default-features = false }
rustls-native-certs = { version = "^0.8", default-features = false }
serde = { version = "^1.0" }
serde_json = { version = "^1.0" }
sha2 = { version = "^0.11", default-features = false }
solana-keypair = { version = "^3.1", default-features = false }
solana-pubkey = { version = "^4.3", default-features = false }
tauri = { version = "^2.11" }
@@ -39,6 +43,8 @@ tracing = { version = "^0.1", default-features = false }
tracing-subscriber = { version = "^0.3", default-features = false }
tracing-appender = { version = "^0.2", default-features = false }
tokio = { version = "^1.53", default-features = false }
tokio-postgres = { version = "^0.7", default-features = false }
tokio-postgres-rustls = { version = "^0.14", default-features = false }
tokio-tungstenite = { version = "^0.30", default-features = false }
tonic = { version = "^0.14", default-features = false }
tonic-prost = { version = "^0.14", default-features = false }

View File

@@ -1,5 +1,5 @@
<!-- file: README.md -->
<!-- version: 7 -->
<!-- version: 8 -->
# Khadhroony Solana Project
@@ -47,6 +47,12 @@ Les besoins du trading constituent une priorité produit à court terme mais ne
- Aucun `rust-toolchain.toml` n'est utilisé.
- Les environnements et contraintes des démonstrations doivent être visibles dans leur nomenclature lorsqu'ils ne sont pas sélectionnables.
## Fondations opérationnelles actuelles
La couche RAW dispose d'une façade Store backend-neutral et d'un premier job historique concret. `ksp-job-api` porte les contrats passifs communs des jobs bornés ; `ksp-job-backfill-lib` compose Transport observé et Store pour découvrir, hydrater et persister des transactions historiques RAW avec concurrence bornée, checkpoint contigu caller-owned, annulation coopérative et snapshots latest-value.
Ces contrats restent distincts des futurs workers continus : un job borné n'est ni un service worker ni un pipeline générique imposé aux autres couches.
## Points d'entrée
- [`RULES.md`](RULES.md) — index des règles normatives ;

View File

@@ -1,5 +1,5 @@
<!-- file: ROADMAP.md -->
<!-- version: 94 -->
<!-- version: 101 -->
# Roadmap KSP
@@ -94,26 +94,36 @@ RAW -> STRUCTURAL -> DECODED -> DOMAIN
## 0.3.x — RAW / acquisition persistée
- [X] `0.3.1``ksp-store-api` stable : modèles N1 RAW backend-agnostic `RawTransaction` et `RawAccountState` avec observations, provenance, payload/hash/timestamps bornés, 10 capabilities object-safe, queries cursorisées sans plafond métier arbitraire, outcomes idempotence/conflit et lifecycle logique rétention/tombstone/force-rehydrate ; aucun backend physique, Config, runtime Store, notification dédiée ni surface STRUCTURAL/DECODED/DOMAIN.
- [ ] `0.3.2` Introduire ensemble `ksp-store-lib` et `ksp-store-postgres-lib` pour la **fondation runtime/backend PostgreSQL uniquement** : façade Store, feature `postgres` par défaut, dispatch des backends compilés, Config `std.store`/secrets, connexion/pool/TLS à réauditer, bootstrap/migrations privés et health/readiness seulement si un contrat portable est réellement justifié. Aucun schéma `RawTransaction`/`RawAccountState` nest ajouté dans cette slice.
- [ ] `0.3.3`Étendre le même couple `ksp-store-lib` + `ksp-store-postgres-lib` avec la vertical slice PostgreSQL `RawTransaction` complète : persistence/observation atomiques, get/list cursorisé, idempotence/conflit, rétention/tombstone/force-rehydrate, concurrence et rollback validés sur PostgreSQL réel.
- [ ] `0.3.4`Étendre le même couple avec `RawAccountState` + observation, puis fermer la complétude/conformance RAW cross-family, les indexes/migrations physiques nécessaires et le hardening PostgreSQL final.
- [ ] `0.3.5` Étendre `ksp-interface-lib` uniquement avec les modèles passifs/events réellement partagés par les premiers consumers dacquisition, sans dupliquer les modèles persistants de `ksp-store-api`.
- [ ] `0.3.6`Introduire `ksp-job-api` et un premier job de backfill historique concret consommant `ksp-store-lib`, avec policy/batch-size/progression possédés par le job et non par Store.
- [ ] `0.3.7` — Introduire une application spécialisée de backfill/inspection RAW.
- [ ] Compléter ensuite la couche RAW avec le worker/service live, son contrôle et les outils dexploitation réellement nécessaires avant de passer à la couche de normalisation générique suivante.
- [X] `0.3.2``ksp-store-lib` + `ksp-store-postgres-lib` stables comme fondation runtime/backend PostgreSQL : feature `postgres` par défaut, `Store` lié à un unique `RawNetworkId`, Config `std.store` avec targets/bases `devnet`/`mainnet`/`testnet`, pool Deadpool borné, `tokio-postgres`, TLS Rustls `Disabled`/`VerifyFull`, moteur de migrations privé `V000` + SHA-256/advisory lock, health/readiness portable et close borné. Gate complet + PostgreSQL réel major 17 verts ; aucune table/capability `RawTransaction`/`RawAccountState` métier n'est encore ajoutée.
- [X] `0.3.3`Vertical slice PostgreSQL `RawTransaction` complète sur `ksp-store-lib` + `ksp-store-postgres-lib` : six capabilities transaction/observation/rétention, V001 physique liée à un réseau, acquisition canonical+observation atomique, idempotence/conflit, get/list keyset cursorisé, archive/purge/tombstone/ForceRehydrate, hardening des erreurs et du schéma, concurrence et rollback validés sur PostgreSQL 17.
- [X] `0.3.4`Vertical slice PostgreSQL `RawAccountState` complète sur `ksp-store-lib` + `ksp-store-postgres-lib` : quatre capabilities account ajoutées aux six transaction pour une conformance RAW 10/10, V002 additive de 32 ressources au-dessus de V000/V001 immuables, state+observation atomiques, idempotence/conflit exacts, metadata Yellowstone observation-only, get/list keyset `(slot,pubkey,state_hash)` avec cursor KSPA anti-replay, hardening cross-family et live validé sur PostgreSQL 17 sans rétention destructive account.
- [X] `0.3.5``ksp-interface-lib` étendu avec deux familles passives réellement partagées : `SlotLifecycleEvent` (`Processed`, `FirstShredReceived`, `Completed`, `CreatedBank`, `Dead`, `OptimisticallyConfirmed`, `Rooted`) et `TransactionExecutionEvent` (`slot + TransactionSignature[64] + Succeeded/Failed`). Interface reste Core-only, provider-neutral, sans serde/codec/runtime/event bus et sans duplication de `RawTransaction`/`RawAccountState`; les DTOs riches restent Transport-owned et les candidats non convergents restent différés.
- [X] `0.3.6``ksp-job-api` + `ksp-job-backfill-lib` stables pour la première verticale historique `RawTransaction` : lifecycle/cancellation/latest-value runtime-neutral, scopes `Latest`/`Before`/`After`/signatures explicites, découverte/hydratation Transport observée, RAW v1 canonique, persistance Store atomique/idempotente, concurrence bornée, frontier contiguë, checkpoint/reprise caller-owned, snapshots sûrs et hardening externe sans dépendance backend/provider directe.
- [ ] `0.3.7` — Introduire `ksp-app-backfill-desk`, application Tauri spécialisée de contrôle et monitoring de `ksp-job-backfill-lib`, composée via Config : sélection de composition/profil, lancement/annulation, scopes/anchors, limites, progression latest-value, compteurs, frontier/checkpoint/reprise et outcome terminal. Elle ne devient ni navigateur Store généraliste, ni scheduler, ni propriétaire des retries/providers/backends.
- [ ] `0.3.8` — Introduire `ksp-app-store-desk` V1, application Tauri backend-agnostique de diagnostic, consultation et requêtage du Store via `ksp-store-lib` + Config : health/capabilities, `RawTransaction`, `RawAccountState`, observations, rétention/tombstones et pagination sûre. Cette application évoluera ensuite avec les couches STRUCTURAL, DECODED, processing/materialization et DOMAIN réellement ajoutées au Store, sans SQL ni backend physique dans l'UI.
- [ ] `0.3.9` — Introduire `ksp-worker-api` comme API générique de lifecycle/health/progression pour services continus, en reprenant le pattern latest-value stabilisé par Job tout en gardant les sémantiques Worker distinctes des jobs terminables et des wake-ups Store post-commit.
- [ ] `0.3.10` — Introduire `ksp-worker-raw-transaction-ingest-lib` pour l'acquisition continue de `RawTransaction` via les surfaces live de `ksp-onchain-transport-lib`, persistance atomique par `ksp-store-lib`, reprise/backpressure/idempotence et notifications `ksp-worker-api`, sans decode Program ni dépendance backend/provider directe.
- [ ] `0.3.11` — Introduire `ksp-app-raw-transaction-ingest-desk`, application Tauri spécialisée de contrôle et monitoring du worker live : lifecycle, health, rates, backpressure, reconnect/recovery et compteurs sûrs ; la consultation détaillée des données persistées reste la responsabilité de `ksp-app-store-desk`.
### TODO/IDEAS — applications spécialisées et control plane
- [ ] **TODO** — faire évoluer `ksp-app-store-desk` avec chaque nouvelle couche réellement persistée : RAW d'abord, puis STRUCTURAL, DECODED, journal de processing/materialization et projections DOMAIN selon les contrats effectivement disponibles. Les vues avancées restent backend-agnostiques et passent uniquement par `ksp-store-lib`.
- [ ] **TODO** — différer une future `ksp-app-control-desk` jusqu'à ce que KSP dispose au minimum d'un niveau N3/D3 de processing/materialization exploitable et de plusieurs decoders réels. Cette application sera une surface **end-user simplifiée** : démarrer/arrêter les flux autorisés, lancer une recherche courante, voir l'état global et les erreurs importantes. Elle ne recopiera pas le diagnostic détaillé, les réglages avancés, les tables complètes ni toutes les fonctions des `ksp-app-*-desk` spécialisées, qui resteront les outils d'administration, développement et investigation approfondie.
### TODO/IDEAS — taxonomie N1, processing et rétention
- [ ] **TODO** — maintenir la matrice dadmission HTTP/WS/gRPC/provider lors de toute nouvelle famille N1 : plusieurs sources ne convergent vers un même struct que si elles satisfont la même sémantique sans perte.
- [X] `RawAccountState` + observation — contrat commun stabilisé en `0.3.1` avec bytes complets + slot, provenance séparée et enrichissements source-specific optionnels ; la persistence PostgreSQL physique reste réservée à `0.3.4`.
- [ ] **TODO**`TransactionStatusObservation` : réauditer `signatureSubscribe`, `getSignatureStatuses`, Yellowstone TransactionStatus et extensions provider lorsquun consumer réel apparaît ; ne pas fusionner snapshot, transition et update dans un modèle Option-soup.
- [ ] **TODO** — logs realtime : conserver `logMessages` dans `RawTransaction` jusquà la décomposition STRUCTURAL ; traiter `logsSubscribe` comme event-only candidat et décider son contrat passif dans `ksp-interface-lib`, sans table Store par défaut. Le format canonique dun wake-up « donnée persistée disponible » reste distinct et appartient à `ksp-store-api` conformément à `KSP-NOTIFY-*`, mais ne sera matérialisé quavec un publisher/consumer réel.
- [ ] **TODO** slot/root/slotsUpdates et vote : ne créer un modèle passif commun que si un consumer realtime réel et une sémantique cross-ledger/provider justifient le contrat ; aucune persistence Store par défaut.
- [X] `RawAccountState` + observation — contrat commun stabilisé en `0.3.1` avec bytes complets + slot, provenance séparée et enrichissements source-specific optionnels ; la persistence PostgreSQL physique est complétée en `0.3.4` avec les quatre capabilities account et la conformance RAW 10/10.
- [ ] **TODO**statut/commitment transactionnel restant : `0.3.5` couvre uniquement le fait passif dexécution `slot + signature + outcome`; réauditer séparément `signatureSubscribe` et `getSignatureStatuses` lorsquun consumer de commitment/snapshot réel apparaît, sans fusionner snapshot, transition et execution update dans un modèle Option-soup.
- [ ] **IDEA** — logs realtime enrichis : `logsSubscribe` alimente déjà la projection minimale `TransactionExecutionEvent`, mais un éventuel `TransactionLogEvent` portant les lignes de log reste différé dans `docs/IDEAS.md` jusquà démonstration dun consumer et de bornes explicites. `logMessages` reste dans `RawTransaction` jusquà STRUCTURAL ; le wake-up post-commit reste distinct et Store API-owned conformément à `KSP-NOTIFY-*`.
- [X] slot/root/slotsUpdates `0.3.5` stabilise `SlotLifecycleEvent` pour lintersection réellement partagée, sans persistence Store par défaut et sans promettre lordre/complétude du flux.
- [ ] **TODO** — vote realtime : reste hors Interface tant quaucun consumer transversal et aucune sémantique provider-neutral suffisamment précise ne justifient un contrat partagé.
- [ ] **IDEA**`RawBlock` : ne rouvrir que si une information block-level non reconstructible devient nécessaire ; `getBlock` doit dabord être traité comme source de `RawTransaction`, pas comme invitation à recopier le ledger en blocs.
- [ ] **REJET ACTUEL** — Yellowstone `Entry` : trop bas niveau et aucune destination replay/decomposition/event métier justifiant un modèle KSP nest identifiée.
- [ ] **TODO** — processing ledger : reprendre lidée kbot2/kbot3 `stage + processor identity/version + input identity/hash + terminal status`, sans faire dun `processed: bool` la preuve durable unique ; prévoir force replay/version upgrades lorsque les processors seront ouverts.
- [X] lifecycle RAW logique — `RawRetentionState`, tombstone minimal, normal-skip et force-rehydrate sont stabilisés en `0.3.1` pour `RawTransaction`.
- [ ] **TODO** rétention physique : définir plus tard compression/archive backend, critères déligibilité fondés sur les preuves de processing et maintenance worker/job ; Store applique une transition demandée mais ne décide pas seul quun RAW peut être purgé.
- [X] rétention physique `RawTransaction` PostgreSQL — `0.3.3` matérialise `Full -> Archived -> Purged`, tombstone et ForceRehydrate atomiques ; `Compacted` reste explicitement unsupported tant quaucune représentation compactée réelle nexiste.
- [ ] **TODO** — policy de rétention/compaction : définir les critères déligibilité fondés sur les preuves de processing et la maintenance worker/job ; Store applique une transition demandée mais ne décide pas seul quun RAW peut être archivé/purgé, et la compaction physique ne sera ajoutée quavec un besoin réel.
- [X] frontière `ksp-interface-lib` / `ksp-store-api` — ownership documenté et canaris de non-duplication stabilisés en `0.3.1`; les events passifs non persistés restent Interface, les modèles persistants/replayables restent Store API.
- [ ] **IDEA** — réauditer la structure de processing/decode/materialization historique kbot2/kbot3 lors de louverture de N2/N3 ; conserver lisolation instruction/CPI et les statuts terminal/versionnés, sans reprendre automatiquement le schéma SQL historique.
@@ -123,7 +133,8 @@ RAW -> STRUCTURAL -> DECODED -> DOMAIN
- [ ] Implémenter en priorité `RawTransaction -> STRUCTURAL` sans decoder Program : transaction/message, comptes/références, instructions top-level, CPI/inner instructions, logs/meta/balances/return data et relations structurelles.
- [ ] Vérifier avant extension si d'autres familles N1 possèdent une vraie décomposition STRUCTURAL utile ; ne pas créer de niveau vide par convention.
- [ ] Ajouter replay/backfill RAW -> STRUCTURAL avec processing versionné.
- [ ] Ajouter worker/service STRUCTURAL et l'application de contrôle/inspection utile.
- [ ] Ajouter le worker/service STRUCTURAL utile sans le coupler à un worker RAW concret.
- [ ] Faire évoluer `ksp-app-store-desk` avec des vues/requêtes STRUCTURAL lorsque cette couche est réellement persistée ; ne pas créer une seconde application de browsing des mêmes données.
## Séries DECODED/DOMAIN/EXECUTION — progression verticale

View File

@@ -0,0 +1,81 @@
{
"format_version": 2,
"default_profile": "devnet",
"profiles": [
{
"profile_id": "devnet",
"network": "devnet",
"backend": "postgres",
"postgres": {
"connection_uri": "${KSP_SECRET_STORE_DEVNET_POSTGRES_URI:-postgresql://localhost/ksp_devnet}",
"pool": {
"max_connections": 8,
"connect_timeout_ms": 10000,
"wait_timeout_ms": 5000,
"create_timeout_ms": 10000,
"recycle_timeout_ms": 5000
},
"tls": {
"mode": "verify_full"
},
"bootstrap": {
"schema_autocreate": true,
"schema_autoupdate": true,
"migration_timeout_ms": 30000,
"migration_lock_timeout_ms": 10000
},
"shutdown_timeout_ms": 5000
}
},
{
"profile_id": "mainnet",
"network": "mainnet-beta",
"backend": "postgres",
"postgres": {
"connection_uri": "${KSP_SECRET_STORE_MAINNET_POSTGRES_URI:-postgresql://localhost/ksp_mainnet}",
"pool": {
"max_connections": 8,
"connect_timeout_ms": 10000,
"wait_timeout_ms": 5000,
"create_timeout_ms": 10000,
"recycle_timeout_ms": 5000
},
"tls": {
"mode": "verify_full"
},
"bootstrap": {
"schema_autocreate": true,
"schema_autoupdate": true,
"migration_timeout_ms": 30000,
"migration_lock_timeout_ms": 10000
},
"shutdown_timeout_ms": 5000
}
},
{
"profile_id": "testnet",
"network": "testnet",
"backend": "postgres",
"postgres": {
"connection_uri": "${KSP_SECRET_STORE_TESTNET_POSTGRES_URI:-postgresql://localhost/ksp_testnet}",
"pool": {
"max_connections": 8,
"connect_timeout_ms": 10000,
"wait_timeout_ms": 5000,
"create_timeout_ms": 10000,
"recycle_timeout_ms": 5000
},
"tls": {
"mode": "verify_full"
},
"bootstrap": {
"schema_autocreate": true,
"schema_autoupdate": true,
"migration_timeout_ms": 30000,
"migration_lock_timeout_ms": 10000
},
"shutdown_timeout_ms": 5000
}
}
]
}

View File

@@ -0,0 +1,277 @@
{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"$id": "urn:ksp:schema:std.store:v2",
"title": "KSP standard Store configuration",
"type": "object",
"additionalProperties": false,
"required": [
"format_version",
"default_profile",
"profiles"
],
"properties": {
"format_version": {
"enum": [
1,
2
]
},
"default_profile": {
"$ref": "#/$defs/profileId"
},
"profiles": {
"type": "array",
"minItems": 1
}
},
"oneOf": [
{
"properties": {
"format_version": {
"const": 1
},
"profiles": {
"items": {
"$ref": "#/$defs/profileV1"
}
}
}
},
{
"properties": {
"format_version": {
"const": 2
},
"profiles": {
"items": {
"$ref": "#/$defs/profileV2"
}
}
}
}
],
"$defs": {
"profileId": {
"type": "string",
"pattern": "^[a-z0-9][a-z0-9._-]*$"
},
"networkId": {
"type": "string",
"minLength": 1,
"maxLength": 128,
"pattern": "^[A-Za-z0-9_.:-]+$"
},
"duration100To60000": {
"type": "integer",
"minimum": 100,
"maximum": 60000
},
"profileV1": {
"type": "object",
"additionalProperties": false,
"required": [
"profile_id",
"network",
"backend",
"postgres"
],
"properties": {
"profile_id": {
"$ref": "#/$defs/profileId"
},
"network": {
"$ref": "#/$defs/networkId"
},
"backend": {
"const": "postgres"
},
"postgres": {
"$ref": "#/$defs/postgresV1"
}
}
},
"profileV2": {
"type": "object",
"additionalProperties": false,
"required": [
"profile_id",
"network",
"backend",
"postgres"
],
"properties": {
"profile_id": {
"$ref": "#/$defs/profileId"
},
"network": {
"$ref": "#/$defs/networkId"
},
"backend": {
"const": "postgres"
},
"postgres": {
"$ref": "#/$defs/postgresV2"
}
}
},
"postgresV1": {
"type": "object",
"additionalProperties": false,
"required": [
"connection_uri",
"pool",
"tls",
"bootstrap",
"shutdown_timeout_ms"
],
"properties": {
"connection_uri": {
"type": "string",
"minLength": 1
},
"pool": {
"$ref": "#/$defs/pool"
},
"tls": {
"$ref": "#/$defs/tls"
},
"bootstrap": {
"$ref": "#/$defs/bootstrapV1"
},
"shutdown_timeout_ms": {
"type": "integer",
"minimum": 100,
"maximum": 30000
}
}
},
"postgresV2": {
"type": "object",
"additionalProperties": false,
"required": [
"connection_uri",
"pool",
"tls",
"bootstrap",
"shutdown_timeout_ms"
],
"properties": {
"connection_uri": {
"type": "string",
"minLength": 1
},
"pool": {
"$ref": "#/$defs/pool"
},
"tls": {
"$ref": "#/$defs/tls"
},
"bootstrap": {
"$ref": "#/$defs/bootstrapV2"
},
"shutdown_timeout_ms": {
"type": "integer",
"minimum": 100,
"maximum": 30000
}
}
},
"pool": {
"type": "object",
"additionalProperties": false,
"required": [
"max_connections",
"connect_timeout_ms",
"wait_timeout_ms",
"create_timeout_ms",
"recycle_timeout_ms"
],
"properties": {
"max_connections": {
"type": "integer",
"minimum": 1,
"maximum": 64
},
"connect_timeout_ms": {
"$ref": "#/$defs/duration100To60000"
},
"wait_timeout_ms": {
"$ref": "#/$defs/duration100To60000"
},
"create_timeout_ms": {
"$ref": "#/$defs/duration100To60000"
},
"recycle_timeout_ms": {
"$ref": "#/$defs/duration100To60000"
}
}
},
"tls": {
"type": "object",
"additionalProperties": false,
"required": [
"mode"
],
"properties": {
"mode": {
"enum": [
"disabled",
"verify_full"
]
}
}
},
"bootstrapV1": {
"type": "object",
"additionalProperties": false,
"required": [
"auto_migrate",
"migration_timeout_ms",
"migration_lock_timeout_ms"
],
"properties": {
"auto_migrate": {
"type": "boolean"
},
"migration_timeout_ms": {
"type": "integer",
"minimum": 1000,
"maximum": 300000
},
"migration_lock_timeout_ms": {
"type": "integer",
"minimum": 100,
"maximum": 120000
}
}
},
"bootstrapV2": {
"type": "object",
"additionalProperties": false,
"required": [
"schema_autocreate",
"schema_autoupdate",
"migration_timeout_ms",
"migration_lock_timeout_ms"
],
"properties": {
"schema_autocreate": {
"type": "boolean"
},
"schema_autoupdate": {
"type": "boolean"
},
"migration_timeout_ms": {
"type": "integer",
"minimum": 1000,
"maximum": 300000
},
"migration_lock_timeout_ms": {
"type": "integer",
"minimum": 100,
"maximum": 120000
}
}
}
}
}

81
config/std.store.json Normal file
View File

@@ -0,0 +1,81 @@
{
"format_version": 2,
"default_profile": "devnet",
"profiles": [
{
"profile_id": "devnet",
"network": "devnet",
"backend": "postgres",
"postgres": {
"connection_uri": "${KSP_SECRET_STORE_DEVNET_POSTGRES_URI:-postgresql://localhost/ksp_devnet}",
"pool": {
"max_connections": 8,
"connect_timeout_ms": 10000,
"wait_timeout_ms": 5000,
"create_timeout_ms": 10000,
"recycle_timeout_ms": 5000
},
"tls": {
"mode": "verify_full"
},
"bootstrap": {
"schema_autocreate": true,
"schema_autoupdate": true,
"migration_timeout_ms": 30000,
"migration_lock_timeout_ms": 10000
},
"shutdown_timeout_ms": 5000
}
},
{
"profile_id": "mainnet",
"network": "mainnet-beta",
"backend": "postgres",
"postgres": {
"connection_uri": "${KSP_SECRET_STORE_MAINNET_POSTGRES_URI:-postgresql://localhost/ksp_mainnet}",
"pool": {
"max_connections": 8,
"connect_timeout_ms": 10000,
"wait_timeout_ms": 5000,
"create_timeout_ms": 10000,
"recycle_timeout_ms": 5000
},
"tls": {
"mode": "verify_full"
},
"bootstrap": {
"schema_autocreate": true,
"schema_autoupdate": true,
"migration_timeout_ms": 30000,
"migration_lock_timeout_ms": 10000
},
"shutdown_timeout_ms": 5000
}
},
{
"profile_id": "testnet",
"network": "testnet",
"backend": "postgres",
"postgres": {
"connection_uri": "${KSP_SECRET_STORE_TESTNET_POSTGRES_URI:-postgresql://localhost/ksp_testnet}",
"pool": {
"max_connections": 8,
"connect_timeout_ms": 10000,
"wait_timeout_ms": 5000,
"create_timeout_ms": 10000,
"recycle_timeout_ms": 5000
},
"tls": {
"mode": "verify_full"
},
"bootstrap": {
"schema_autocreate": true,
"schema_autoupdate": true,
"migration_timeout_ms": 30000,
"migration_lock_timeout_ms": 10000
},
"shutdown_timeout_ms": 5000
}
}
]
}

View File

@@ -56,11 +56,13 @@
"../../config/composite.ksp-app-wallet-desk.json": "config/composite.ksp-app-wallet-desk.json",
"../../config/std.logging.json": "config/std.logging.json",
"../../config/std.offchain_transport.json": "config/std.offchain_transport.json",
"../../config/std.store.json": "config/std.store.json",
"../../config/std.transport.json": "config/std.transport.json",
"../../config/std.wallet.json": "config/std.wallet.json",
"../../config/schemas/composite.schema.json": "config/schemas/composite.schema.json",
"../../config/schemas/std.logging.schema.json": "config/schemas/std.logging.schema.json",
"../../config/schemas/std.offchain_transport.schema.json": "config/schemas/std.offchain_transport.schema.json",
"../../config/schemas/std.store.schema.json": "config/schemas/std.store.schema.json",
"../../config/schemas/std.transport.schema.json": "config/schemas/std.transport.schema.json",
"../../config/schemas/std.wallet.schema.json": "config/schemas/std.wallet.schema.json"
}

View File

@@ -1,5 +1,5 @@
// file: crates/ksp-app-config-desk/tests/desktop_contract.rs
// version: 10
// version: 11
//! Desktop build/shell contract audits for Config Desk.
@@ -110,7 +110,7 @@ fn pre_018_packaged_runtime_bundles_config_resources_and_activates_shared_writab
let resources = tauri.pointer("/bundle/resources").and_then(serde_json::Value::as_object);
assert!(resources.is_some(), "packaged Config resources map must exist");
if let std::option::Option::Some(resources) = resources {
assert_eq!(resources.len(), 11);
assert_eq!(resources.len(), 13);
assert_eq!(
resources.get("../../config/composite.ksp-app-solprices-desk.json").and_then(serde_json::Value::as_str),
std::option::Option::Some("config/composite.ksp-app-solprices-desk.json"),
@@ -124,6 +124,11 @@ fn pre_018_packaged_runtime_bundles_config_resources_and_activates_shared_writab
resources.get("../../config/schemas/std.offchain_transport.schema.json").and_then(serde_json::Value::as_str),
std::option::Option::Some("config/schemas/std.offchain_transport.schema.json"),
);
assert_eq!(resources.get("../../config/std.store.json").and_then(serde_json::Value::as_str), std::option::Option::Some("config/std.store.json"),);
assert_eq!(
resources.get("../../config/schemas/std.store.schema.json").and_then(serde_json::Value::as_str),
std::option::Option::Some("config/schemas/std.store.schema.json"),
);
assert_eq!(
resources.get("../../config/schemas/std.wallet.schema.json").and_then(serde_json::Value::as_str),
std::option::Option::Some("config/schemas/std.wallet.schema.json"),

View File

@@ -1,5 +1,5 @@
// file: crates/ksp-app-config-desk/unit_tests/profiles.rs
// version: 7
// version: 8
#[test]
fn profile_inventory_exposes_registered_profile_documents() {
@@ -15,6 +15,9 @@ fn profile_inventory_exposes_registered_profile_documents() {
assert!(inventory.iter().any(|document| -> bool {
return document.file_id == ksp_config_lib::FILE_ID_STD_TRANSPORT;
}));
assert!(inventory.iter().any(|document| -> bool {
return document.file_id == ksp_config_lib::FILE_ID_STD_STORE;
}));
assert!(inventory.iter().any(|document| -> bool {
return document.file_id == ksp_config_lib::FILE_ID_STD_WALLET;
}));

View File

@@ -56,11 +56,13 @@
"../../config/composite.ksp-app-wallet-desk.json": "config/composite.ksp-app-wallet-desk.json",
"../../config/std.logging.json": "config/std.logging.json",
"../../config/std.offchain_transport.json": "config/std.offchain_transport.json",
"../../config/std.store.json": "config/std.store.json",
"../../config/std.transport.json": "config/std.transport.json",
"../../config/std.wallet.json": "config/std.wallet.json",
"../../config/schemas/composite.schema.json": "config/schemas/composite.schema.json",
"../../config/schemas/std.logging.schema.json": "config/schemas/std.logging.schema.json",
"../../config/schemas/std.offchain_transport.schema.json": "config/schemas/std.offchain_transport.schema.json",
"../../config/schemas/std.store.schema.json": "config/schemas/std.store.schema.json",
"../../config/schemas/std.transport.schema.json": "config/schemas/std.transport.schema.json",
"../../config/schemas/std.wallet.schema.json": "config/schemas/std.wallet.schema.json"
}

View File

@@ -1,5 +1,5 @@
// file: crates/ksp-app-solprices-desk/tests/desktop_contract.rs
// version: 9
// version: 10
//! Desktop scaffold, shared-template and Config packaging contract audits for SOL Prices Desk `0.2.12`.
@@ -85,19 +85,21 @@ fn pre_002_package_is_mixed_lib_bin_and_frontend_is_scaffold_only() {
}
#[test]
fn pre_003_packaging_contains_current_eleven_config_resources() {
fn pre_004_packaging_contains_current_thirteen_config_resources() {
let root = app_root();
let tauri = read_json(root.join("tauri.conf.json").as_path());
let resources = tauri.pointer("/bundle/resources").and_then(serde_json::Value::as_object);
assert!(resources.is_some());
if let std::option::Option::Some(resources) = resources {
assert_eq!(resources.len(), 11);
assert_eq!(resources.len(), 13);
assert_eq!(
resources.get("../../config/composite.ksp-app-solprices-desk.json").and_then(serde_json::Value::as_str),
std::option::Option::Some("config/composite.ksp-app-solprices-desk.json"),
);
assert!(resources.contains_key("../../config/std.offchain_transport.json"));
assert!(resources.contains_key("../../config/schemas/std.offchain_transport.schema.json"));
assert!(resources.contains_key("../../config/std.store.json"));
assert!(resources.contains_key("../../config/schemas/std.store.schema.json"));
}
let tauri_source = read_text(root.join("src/tauri.rs").as_path());
assert!(tauri_source.contains("ksp_config_lib::prepare_packaged_runtime"));

View File

@@ -56,11 +56,13 @@
"../../config/composite.ksp-app-wallet-desk.json": "config/composite.ksp-app-wallet-desk.json",
"../../config/std.logging.json": "config/std.logging.json",
"../../config/std.offchain_transport.json": "config/std.offchain_transport.json",
"../../config/std.store.json": "config/std.store.json",
"../../config/std.transport.json": "config/std.transport.json",
"../../config/std.wallet.json": "config/std.wallet.json",
"../../config/schemas/composite.schema.json": "config/schemas/composite.schema.json",
"../../config/schemas/std.logging.schema.json": "config/schemas/std.logging.schema.json",
"../../config/schemas/std.offchain_transport.schema.json": "config/schemas/std.offchain_transport.schema.json",
"../../config/schemas/std.store.schema.json": "config/schemas/std.store.schema.json",
"../../config/schemas/std.transport.schema.json": "config/schemas/std.transport.schema.json",
"../../config/schemas/std.wallet.schema.json": "config/schemas/std.wallet.schema.json"
}

View File

@@ -1,5 +1,5 @@
// file: crates/ksp-app-wallet-desk/tests/desktop_contract.rs
// version: 30
// version: 31
//! Desktop build, shell and Config-status contract audits for Wallet Desk.
@@ -432,7 +432,7 @@ fn pre_018_packaged_runtime_bundles_config_resources_and_keeps_wallet_desk_versi
let resources = tauri.pointer("/bundle/resources").and_then(serde_json::Value::as_object);
assert!(resources.is_some(), "packaged Wallet Desk Config resources map must exist");
if let std::option::Option::Some(resources) = resources {
assert_eq!(resources.len(), 11);
assert_eq!(resources.len(), 13);
assert_eq!(
resources.get("../../config/composite.ksp-app-solprices-desk.json").and_then(serde_json::Value::as_str),
std::option::Option::Some("config/composite.ksp-app-solprices-desk.json"),
@@ -449,6 +449,11 @@ fn pre_018_packaged_runtime_bundles_config_resources_and_keeps_wallet_desk_versi
resources.get("../../config/std.offchain_transport.json").and_then(serde_json::Value::as_str),
std::option::Option::Some("config/std.offchain_transport.json"),
);
assert_eq!(resources.get("../../config/std.store.json").and_then(serde_json::Value::as_str), std::option::Option::Some("config/std.store.json"),);
assert_eq!(
resources.get("../../config/schemas/std.store.schema.json").and_then(serde_json::Value::as_str),
std::option::Option::Some("config/schemas/std.store.schema.json"),
);
assert_eq!(
resources.get("../../config/schemas/std.offchain_transport.schema.json").and_then(serde_json::Value::as_str),
std::option::Option::Some("config/schemas/std.offchain_transport.schema.json"),

View File

@@ -1,5 +1,5 @@
// file: crates/ksp-app-wallet-desk/tests/release_compliance.rs
// version: 9
// version: 10
//! Release-wide deterministic compliance canaries for Wallet Desk.
@@ -199,7 +199,7 @@ fn packaged_resources_include_only_registered_config_sources_and_schemas() {
std::option::Option::Some(value) => value,
std::option::Option::None => return,
};
assert_eq!(resources.len(), 11);
assert_eq!(resources.len(), 13);
for (source, destination) in resources {
let destination = destination.as_str();
assert!(destination.is_some(), "resource destination must be textual");

View File

@@ -1,5 +1,5 @@
# file: crates/ksp-config-lib/Cargo.toml
# version: 9
# version: 10
[package]
name = "ksp-config-lib"
@@ -14,6 +14,7 @@ ksp-core-lib = { path = "../ksp-core-lib" }
ksp-logging-lib = { path = "../ksp-logging-lib" }
ksp-offchain-transport-lib = { path = "../ksp-offchain-transport-lib" }
ksp-onchain-transport-lib = { path = "../ksp-onchain-transport-lib" }
ksp-store-lib = { path = "../ksp-store-lib", default-features = false }
serde = { workspace = true, features = ["derive"] }
serde_json.workspace = true

View File

@@ -1,5 +1,5 @@
<!-- file: crates/ksp-config-lib/README.md -->
<!-- version: 10 -->
<!-- version: 11 -->
# ksp-config-lib
@@ -25,6 +25,7 @@ La crate centralise les documents JSON, leurs schemas, les profils et compositio
- l'adapter du document Logging effectif vers `ksp_logging_lib::LoggingSettings` ;
- l'adapter du document Transport V1/V2/V3 vers `HttpTransportSettings`, `WsTransportSettings` et, en V3, `YellowstoneGrpcTransportSettings`, y compris redaction/provenance des URLs `KSP_SECRET_*` ;
- l'adapter de `cfg.std.offchain_transport` vers `ksp_offchain_transport_lib::MarketPriceService`, avec contrôle de provenance des credentials/public fields et sans rendre les limites provider configurables ;
- l'adapter de `cfg.std.store` vers `ksp_store_lib::StoreSettings`, avec sélection d'un target nommé, réseau explicite et URI PostgreSQL à provenance `Secret` ;
- la surface de management pour inspecter et réparer les sources Config enregistrées, modifier `std.logging.json`, consulter les rapports d'environnement, révéler explicitement une valeur réelle et modifier `.env` ;
- les écritures atomiques JSON/`.env` et la protection des permissions `.env` ;
- les audits workspace empêchant les bypass d'ownership Config et les oublis dans `.env.example`.
@@ -34,14 +35,17 @@ La crate centralise les documents JSON, leurs schemas, les profils et compositio
Le registre par défaut connaît :
```text
cfg.composite.ksp-app-wallet-desk -> config/composite.ksp-app-wallet-desk.json
cfg.composite.ksp-app-solprices-desk -> config/composite.ksp-app-solprices-desk.json
cfg.composite.ksp-app-wallet-desk -> config/composite.ksp-app-wallet-desk.json
cfg.std.logging -> config/std.logging.json
cfg.std.offchain_transport -> config/std.offchain_transport.json
cfg.std.store -> config/std.store.json
cfg.std.transport -> config/std.transport.json
cfg.std.wallet -> config/std.wallet.json
schema.composite -> config/schemas/composite.schema.json
schema.std.logging -> config/schemas/std.logging.schema.json
schema.std.offchain_transport -> config/schemas/std.offchain_transport.schema.json
schema.std.store -> config/schemas/std.store.schema.json
schema.std.transport -> config/schemas/std.transport.schema.json
schema.std.wallet -> config/schemas/std.wallet.schema.json
```
@@ -50,7 +54,7 @@ schema.std.wallet -> config/schemas/std.wallet.schema.json
`ConfigManagement::read_source()` permet d'inspecter le texte brut d'un document Config enregistré même lorsque ce document est invalide. `save_source_candidate()` complète cette frontière : le candidat brut est parsé, validé contre son schema et les invariants sémantiques KSP, puis persisté atomiquement uniquement après validation complète. Le `file_id` doit appartenir au registre et désigner un document Config ; aucun path arbitraire n'est accepté.
`config/examples/composite.example.json` conserve lexemple générique. `config/composite.ksp-app-wallet-desk.json` est le premier composite runtime concret : il sélectionne Logging, Transport et Wallet par `file_id`, sans dépendre de leurs filenames physiques.
`config/examples/composite.example.json` conserve lexemple générique. Les composites runtime committed restent possédés par Config : `config/composite.ksp-app-solprices-desk.json` sélectionne Logging + Off-chain Transport, tandis que `config/composite.ksp-app-wallet-desk.json` sélectionne Logging + Off-chain Transport + On-chain Transport + Wallet. Tous référencent leurs documents par `file_id`, sans dépendre de filenames physiques.
Le fichier local d'environnement est :
@@ -68,11 +72,11 @@ Les autres crates et applications KSP ne doivent pas :
- parser ou écrire directement `.env` ;
- ouvrir directement les documents Config connus par leur filename physique ;
- réimplémenter la sélection de profils, les compositions ou les placeholders ;
- reconstruire elles-mêmes la configuration Logging, On-chain Transport, Off-chain Transport ou Wallet depuis le JSON.
- reconstruire elles-mêmes la configuration Logging, On-chain Transport, Off-chain Transport, Store ou Wallet depuis le JSON.
`ksp-config-lib` dépend de `ksp-core-lib` pour `Error`/`Result`, de `ksp-logging-lib` pour les événements Config utiles et le contrat `LoggingSettings`, de `ksp-onchain-transport-lib` pour construire le contrat runtime On-chain Transport et de `ksp-offchain-transport-lib` pour construire le service market-price dans la direction Config -> Transport. Le document Wallet reste un contrat de chemins/profils Config et nintroduit aucune dépendance Config -> `ksp-wallet-lib`.
`ksp-config-lib` dépend de `ksp-core-lib` pour `Error`/`Result`, de `ksp-logging-lib` pour les événements Config utiles et le contrat `LoggingSettings`, de `ksp-onchain-transport-lib` pour construire le contrat runtime On-chain Transport et de `ksp-offchain-transport-lib` pour construire le service market-price dans la direction Config -> Transport et de `ksp-store-lib` avec `default-features = false` pour construire les settings Store dans la direction Config -> Store sans forcer un backend physique. Le document Wallet reste un contrat de chemins/profils Config et nintroduit aucune dépendance Config -> `ksp-wallet-lib`.
La dépendance inverse est interdite : `ksp-core-lib`, `ksp-logging-lib`, `ksp-onchain-transport-lib` et `ksp-offchain-transport-lib` ne dépendent pas de Config.
La dépendance inverse est interdite : `ksp-core-lib`, `ksp-logging-lib`, `ksp-onchain-transport-lib`, `ksp-offchain-transport-lib` et les crates Store ne dépendent pas de Config.
Config ne possède pas le `LoggingGuard`. L'application ou le service qui orchestre le runtime construit la configuration effective puis possède le lifecycle `ksp_logging_lib::initialize/reinitialize`.
@@ -84,7 +88,7 @@ Un secret reste accessible au runtime ou au management lorsqu'un consumer autori
Les méthodes `reveal_*` constituent un opt-in explicite au réel. L'authentification/autorisation de l'utilisateur humain appartient à l'application appelante et les valeurs retournées par ces méthodes ne doivent jamais être journalisées.
Le document Logging refuse les valeurs de sensibilité `Secret` dans sa configuration effective. Le document Transport accepte les valeurs secrètes pour les URLs HTTP/WebSocket et, en V3, pour `grpc_endpoints[].secret_metadata[]` : la valeur réelle est transmise au runtime légitime, tandis que la projection sûre et les `Debug` restent redacted. Les metadata gRPC publiques et secrètes sont séparées et leur provenance Config est contrôlée avant mapping. `std.offchain_transport` exige une provenance `Secret` pour les API keys effectives et une provenance `Public` pour la paire DexScreener lorsqu'elle vient de l'environnement ; il ne permet ni URL provider arbitraire ni override de rate limit. `std.wallet` refuse également toute sensibilité `Secret` pour `wallets_directory`/`wallets_subdirectory`; les passwords Wallet restent un autre flux Config et ne sont jamais stockés dans ce JSON.
Le document Logging refuse les valeurs de sensibilité `Secret` dans sa configuration effective. Le document Transport accepte les valeurs secrètes pour les URLs HTTP/WebSocket et, en V3, pour `grpc_endpoints[].secret_metadata[]` : la valeur réelle est transmise au runtime légitime, tandis que la projection sûre et les `Debug` restent redacted. Les metadata gRPC publiques et secrètes sont séparées et leur provenance Config est contrôlée avant mapping. `std.offchain_transport` exige une provenance `Secret` pour les API keys effectives et une provenance `Public` pour la paire DexScreener lorsqu'elle vient de l'environnement ; il ne permet ni URL provider arbitraire ni override de rate limit. `std.store` exige une provenance `Secret` pour chaque URI PostgreSQL effective et conserve des targets réseau-spécifiques indépendants (`devnet`, `mainnet`, `testnet`) sans exposer l'URI dans les projections sûres. `std.wallet` refuse également toute sensibilité `Secret` pour `wallets_directory`/`wallets_subdirectory`; les passwords Wallet restent un autre flux Config et ne sont jamais stockés dans ce JSON.
## Documentation
@@ -94,6 +98,8 @@ Le document Logging refuse les valeurs de sensibilité `Secret` dans sa configur
- [`../../config/std.logging.json`](../../config/std.logging.json) — document standard Logging ;
- [`../../config/std.transport.json`](../../config/std.transport.json) — document standard Transport V3 HTTP + WebSocket + Yellowstone gRPC, avec lecture backward des V1/V2 ;
- [`../../config/std.offchain_transport.json`](../../config/std.offchain_transport.json) — document standard Off-chain Transport V1, actuellement limité au domaine `market_price` SOL/USD ;
- [`../../config/std.store.json`](../../config/std.store.json) — targets Store PostgreSQL Devnet/Mainnet/Testnet et settings runtime bornés ;
- [`../../config/std.wallet.json`](../../config/std.wallet.json) — racine Wallet globale et sous-répertoire optionnel par profil ;
- [`../../config/composite.ksp-app-wallet-desk.json`](../../config/composite.ksp-app-wallet-desk.json) — composition Logging/Transport/Wallet de Wallet Desk ;
- [`../../config/composite.ksp-app-solprices-desk.json`](../../config/composite.ksp-app-solprices-desk.json) — composition Logging/Off-chain Transport de SOL Prices Desk ;
- [`../../config/composite.ksp-app-wallet-desk.json`](../../config/composite.ksp-app-wallet-desk.json) — composition Logging/Off-chain Transport/On-chain Transport/Wallet de Wallet Desk ;
- [`../../.env.example`](../../.env.example) — inventaire versionné des variables d'environnement runtime.

View File

@@ -1,5 +1,5 @@
<!-- file: crates/ksp-config-lib/USAGE.md -->
<!-- version: 13 -->
<!-- version: 14 -->
# Utilisation de ksp-config-lib
@@ -28,8 +28,11 @@ Les arguments compris par Config sont :
```text
--cfgpath=/path/to/config
--schemapath=/path/to/schemas
--filemap=cfg.composite.ksp-app-solprices-desk=my-solprices-desk.json
--filemap=cfg.composite.ksp-app-wallet-desk=my-wallet-desk.json
--filemap=cfg.std.logging=my-logging.json
--filemap=cfg.std.offchain_transport=my-offchain-transport.json
--filemap=cfg.std.store=my-store.json
--filemap=cfg.std.transport=my-transport.json
--filemap=cfg.std.wallet=my-wallet.json
```
@@ -212,7 +215,56 @@ Config ne permet pas de fournir `base_url`, `endpoint_url`, `rate_limit` ou `req
ksp-config-lib -> ksp-offchain-transport-lib
```
Off-chain Transport ne lit ni `.env`, ni `KSP_*`, ni les documents Config. Une application telle que la future `ksp-app-solprices-desk` peut recevoir le service déjà composé puis utiliser uniquement `registry()`, `refresh`, `refresh_many` et `refresh_all`.
Off-chain Transport ne lit ni `.env`, ni `KSP_*`, ni les documents Config. `ksp-app-solprices-desk` reçoit le service déjà composé puis utilise uniquement la surface provider-neutral `registry()`, `refresh`, `refresh_many` et `refresh_all`.
### 4.4 Construire le Store depuis Config
`cfg.std.store` définit des targets nommés. Chaque target sélectionne exactement un réseau logique, un backend et une URI PostgreSQL distincte. Config résout les secrets puis construit le contrat backend-neutral `ksp_store_lib::StoreSettings` sans activer la feature PostgreSQL du consumer :
```rust
let store_config = match engine.load_resolved_store_config(
std::option::Option::Some("devnet"),
&environment,
) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let target_id = store_config.target_id();
let network = store_config.settings().network();
let _ = (target_id, network);
let store_settings = store_config.into_settings();
let store = match ksp_store_lib::Store::open(store_settings).await {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let _health = store.health().await;
let closed = store.close().await;
if let std::result::Result::Err(error) = closed {
return std::result::Result::Err(error);
}
```
Targets committed :
```text
devnet -> network devnet -> KSP_SECRET_STORE_DEVNET_POSTGRES_URI
mainnet -> network mainnet-beta -> KSP_SECRET_STORE_MAINNET_POSTGRES_URI
testnet -> network testnet -> KSP_SECRET_STORE_TESTNET_POSTGRES_URI
```
`default_profile = "devnet"` choisit un seul target. La sélection d'un autre target se fait par le `profile_id` explicite ; `ksp-store-lib` ne multiplexe pas plusieurs bases ou réseaux dans une même instance.
Chaque `connection_uri` doit provenir d'un placeholder `KSP_SECRET_*`/`KSPB_SECRET_*`. Une URI littérale ou issue d'une variable non secrète est rejetée par l'adapter effectif. La valeur réelle est transmise au runtime Store, mais `ResolvedStoreConfig`, `StoreSettings` et les projections sûres ne l'affichent pas.
La direction de dépendance reste :
```text
ksp-config-lib -> ksp-store-lib (default-features = false)
ksp-store-lib -X-> ksp-config-lib
ksp-store-postgres-lib -X-> ksp-config-lib
```
## 5. Profils et composites
@@ -237,7 +289,7 @@ let component = match composite.component("wallet") {
let wallet = engine.resolve_wallet_config_profile(component.resolved(), &environment);
```
La même forme existe pour Logging via `resolve_logging_config_profile`. Le composite concret `cfg.composite.ksp-app-wallet-desk` référence actuellement `logging`, `transport` et `wallet`; Wallet Desk valide ces trois frontières au bootstrap.
La même forme existe pour Logging via `resolve_logging_config_profile`. Le composite `cfg.composite.ksp-app-solprices-desk` référence `logging` et `offchain_transport`. Le composite `cfg.composite.ksp-app-wallet-desk` référence `logging`, `offchain_transport`, `transport` et `wallet`; chaque application valide ses frontières de composition au bootstrap.
## 6. Management de `std.logging.json`

View File

@@ -1,5 +1,5 @@
// file: crates/ksp-config-lib/src/lib.rs
// version: 20
// version: 21
#![warn(missing_docs)]
#![deny(unreachable_pub)]
@@ -9,7 +9,7 @@
//!
//! The `0.1.3` surface owns bootstrap roots, the logical file registry, JSON/JSON Schema validation, standard-document profiles, generic composites and
//! KSP/KSPB environment resolution through process + `.env` + fallback precedence. Resolved values preserve real/safe representations, sensitivity and
//! provenance. Standard Logging, on-chain Transport (HTTP/WebSocket/Yellowstone gRPC) and Wallet documents map explicitly to their runtime consumer
//! provenance. Standard Logging, on-chain Transport (HTTP/WebSocket/Yellowstone gRPC), Store and Wallet documents map explicitly to their runtime consumer
//! contracts, while the management surface provides typed Logging mutation, safe environment reports, explicit privileged reveal calls and atomic
//! JSON/`.env` persistence.
@@ -27,6 +27,7 @@ mod persistence;
mod profile;
mod registry;
mod sensitivity;
mod store;
mod transport;
mod wallet;
@@ -164,6 +165,10 @@ pub use self::registry::DEFAULT_STD_LOGGING_SCHEMA_FILENAME;
pub use self::registry::DEFAULT_STD_OFFCHAIN_TRANSPORT_FILENAME;
/// Default physical filename for the standard Off-chain Transport JSON Schema document.
pub use self::registry::DEFAULT_STD_OFFCHAIN_TRANSPORT_SCHEMA_FILENAME;
/// Default physical filename for the standard Store configuration document.
pub use self::registry::DEFAULT_STD_STORE_FILENAME;
/// Default physical filename for the standard Store JSON Schema document.
pub use self::registry::DEFAULT_STD_STORE_SCHEMA_FILENAME;
/// Default physical filename for the standard HTTP + WebSocket + Yellowstone gRPC Transport configuration document.
pub use self::registry::DEFAULT_STD_TRANSPORT_FILENAME;
/// Default physical filename for the standard HTTP + WebSocket + Yellowstone gRPC Transport JSON Schema document.
@@ -182,6 +187,8 @@ pub use self::registry::FILE_ID_SCHEMA_COMPOSITE;
pub use self::registry::FILE_ID_SCHEMA_STD_LOGGING;
/// Logical file identifier for the standard Off-chain Transport JSON Schema document.
pub use self::registry::FILE_ID_SCHEMA_STD_OFFCHAIN_TRANSPORT;
/// Logical file identifier for the standard Store JSON Schema document.
pub use self::registry::FILE_ID_SCHEMA_STD_STORE;
/// Logical file identifier for the standard HTTP + WebSocket + Yellowstone gRPC Transport JSON Schema document.
pub use self::registry::FILE_ID_SCHEMA_STD_TRANSPORT;
/// Logical file identifier for the standard Wallet JSON Schema document.
@@ -190,6 +197,8 @@ pub use self::registry::FILE_ID_SCHEMA_STD_WALLET;
pub use self::registry::FILE_ID_STD_LOGGING;
/// Logical file identifier for the standard Off-chain Transport configuration document.
pub use self::registry::FILE_ID_STD_OFFCHAIN_TRANSPORT;
/// Logical file identifier for the standard Store configuration document.
pub use self::registry::FILE_ID_STD_STORE;
/// Logical file identifier for the standard HTTP + WebSocket + Yellowstone gRPC Transport configuration document.
pub use self::registry::FILE_ID_STD_TRANSPORT;
/// Logical file identifier for the standard Wallet configuration document.
@@ -204,6 +213,8 @@ pub use self::sensitivity::REDACTED_CONFIG_VALUE;
pub use self::sensitivity::ResolvedConfigJson;
/// One resolved Config string preserving real/safe representations and provenance.
pub use self::sensitivity::ResolvedConfigText;
/// Effective standard Store configuration mapped to backend-neutral Store settings.
pub use self::store::ResolvedStoreConfig;
/// Effective standard Transport configuration mapped to HTTP plus optional WebSocket and Yellowstone gRPC runtime settings.
pub use self::transport::ResolvedTransportConfig;
/// Effective standard Wallet configuration resolved to validated filesystem roots.

View File

@@ -1,5 +1,5 @@
// file: crates/ksp-config-lib/src/registry.rs
// version: 12
// version: 13
/// Bootstrap argument used to replace a known Config filename mapping.
pub const ARG_FILE_MAP: &str = "--filemap";
@@ -17,6 +17,10 @@ pub const DEFAULT_STD_LOGGING_SCHEMA_FILENAME: &str = "std.logging.schema.json";
pub const DEFAULT_STD_OFFCHAIN_TRANSPORT_FILENAME: &str = "std.offchain_transport.json";
/// Default physical filename for the standard Off-chain Transport JSON Schema document.
pub const DEFAULT_STD_OFFCHAIN_TRANSPORT_SCHEMA_FILENAME: &str = "std.offchain_transport.schema.json";
/// Default physical filename for the standard Store configuration document.
pub const DEFAULT_STD_STORE_FILENAME: &str = "std.store.json";
/// Default physical filename for the standard Store JSON Schema document.
pub const DEFAULT_STD_STORE_SCHEMA_FILENAME: &str = "std.store.schema.json";
/// Default physical filename for the standard HTTP + WebSocket + Yellowstone gRPC Transport configuration document.
pub const DEFAULT_STD_TRANSPORT_FILENAME: &str = "std.transport.json";
/// Default physical filename for the standard HTTP + WebSocket + Yellowstone gRPC Transport JSON Schema document.
@@ -35,6 +39,8 @@ pub const FILE_ID_SCHEMA_COMPOSITE: &str = "schema.composite";
pub const FILE_ID_SCHEMA_STD_LOGGING: &str = "schema.std.logging";
/// Logical file identifier for the standard Off-chain Transport JSON Schema document.
pub const FILE_ID_SCHEMA_STD_OFFCHAIN_TRANSPORT: &str = "schema.std.offchain_transport";
/// Logical file identifier for the standard Store JSON Schema document.
pub const FILE_ID_SCHEMA_STD_STORE: &str = "schema.std.store";
/// Logical file identifier for the standard HTTP + WebSocket + Yellowstone gRPC Transport JSON Schema document.
pub const FILE_ID_SCHEMA_STD_TRANSPORT: &str = "schema.std.transport";
/// Logical file identifier for the standard Wallet JSON Schema document.
@@ -43,6 +49,8 @@ pub const FILE_ID_SCHEMA_STD_WALLET: &str = "schema.std.wallet";
pub const FILE_ID_STD_LOGGING: &str = "cfg.std.logging";
/// Logical file identifier for the standard Off-chain Transport configuration document.
pub const FILE_ID_STD_OFFCHAIN_TRANSPORT: &str = "cfg.std.offchain_transport";
/// Logical file identifier for the standard Store configuration document.
pub const FILE_ID_STD_STORE: &str = "cfg.std.store";
/// Logical file identifier for the standard HTTP + WebSocket + Yellowstone gRPC Transport configuration document.
pub const FILE_ID_STD_TRANSPORT: &str = "cfg.std.transport";
/// Logical file identifier for the standard Wallet configuration document.
@@ -214,6 +222,22 @@ impl ConfigFileRegistry {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let store = ConfigFileDescriptor::new(
FILE_ID_STD_STORE,
ConfigFileKind::Config,
DEFAULT_STD_STORE_FILENAME,
std::option::Option::Some(FILE_ID_SCHEMA_STD_STORE),
);
let store = match store {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let store_schema =
ConfigFileDescriptor::new(FILE_ID_SCHEMA_STD_STORE, ConfigFileKind::Schema, DEFAULT_STD_STORE_SCHEMA_FILENAME, std::option::Option::None);
let store_schema = match store_schema {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let transport = ConfigFileDescriptor::new(
FILE_ID_STD_TRANSPORT,
ConfigFileKind::Config,
@@ -254,6 +278,8 @@ impl ConfigFileRegistry {
logging_schema,
offchain_transport,
offchain_transport_schema,
store,
store_schema,
transport,
transport_schema,
wallet,

View File

@@ -0,0 +1,305 @@
// file: crates/ksp-config-lib/src/store.rs
// version: 3
/// Effective standard Store configuration mapped to `ksp_store_lib::StoreSettings`.
pub struct ResolvedStoreConfig {
effective: crate::ResolvedConfigJson,
file_id: crate::ConfigFileId,
profile_id: String,
selection_source: crate::ConfigProfileSelectionSource,
settings: ksp_store_lib::StoreSettings,
source_path: std::path::PathBuf,
}
impl ResolvedStoreConfig {
/// Returns the detailed environment-resolved Config view.
#[must_use]
pub const fn effective(&self) -> &crate::ResolvedConfigJson {
return &self.effective;
}
/// Returns the logical Config file identifier used by this runtime configuration.
#[must_use]
pub const fn file_id(&self) -> &crate::ConfigFileId {
return &self.file_id;
}
/// Returns the selected standard Store profile identifier.
///
/// For `std.store`, the profile identifier is also the stable named Store target identifier.
#[must_use]
pub fn profile_id(&self) -> &str {
return self.profile_id.as_str();
}
/// Returns the selected named Store target identifier.
#[must_use]
pub fn target_id(&self) -> &str {
return self.profile_id.as_str();
}
/// Returns the source that selected the standard Store profile.
#[must_use]
pub const fn selection_source(&self) -> crate::ConfigProfileSelectionSource {
return self.selection_source;
}
/// Borrows the backend-neutral Store settings without exposing the connection URI.
#[must_use]
pub const fn settings(&self) -> &ksp_store_lib::StoreSettings {
return &self.settings;
}
/// Consumes the resolved Config and returns the Store-owned runtime settings.
#[must_use]
pub fn into_settings(self) -> ksp_store_lib::StoreSettings {
return self.settings;
}
/// Returns the physical source Config document path.
#[must_use]
pub fn source_path(&self) -> &std::path::Path {
return self.source_path.as_path();
}
}
impl std::fmt::Debug for ResolvedStoreConfig {
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
return formatter
.debug_struct("ResolvedStoreConfig")
.field("effective", &self.effective)
.field("file_id", &self.file_id)
.field("profile_id", &self.profile_id)
.field("selection_source", &self.selection_source)
.field("settings", &self.settings)
.field("source_path", &self.source_path)
.finish();
}
}
impl crate::ConfigDocumentEngine {
/// Loads `std.store`, resolves one profile/environment and maps it to backend-neutral Store settings.
pub fn load_resolved_store_config(
&self,
requested_profile: std::option::Option<&str>,
environment: &crate::ConfigEnvironment,
) -> ksp_core_lib::Result<ResolvedStoreConfig> {
let file_id = crate::ConfigFileId::new(crate::FILE_ID_STD_STORE);
let file_id = match file_id {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let profile = self.load_resolved_profile(&file_id, requested_profile);
let profile = match profile {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
return resolve_store_profile(&profile, environment);
}
/// Maps an already resolved `cfg.std.store` profile while preserving its selection provenance.
pub fn resolve_store_config_profile(
&self,
profile: &crate::ResolvedConfigProfile,
environment: &crate::ConfigEnvironment,
) -> ksp_core_lib::Result<ResolvedStoreConfig> {
if profile.file_id().as_str() != crate::FILE_ID_STD_STORE {
return std::result::Result::Err(effective_error(profile, "resolved Config profile does not reference the standard Store document"));
}
let descriptor = self.registry().descriptor(profile.file_id());
if let std::result::Result::Err(error) = descriptor {
return std::result::Result::Err(error);
}
return resolve_store_profile(profile, environment);
}
}
#[derive(serde::Deserialize)]
#[serde(deny_unknown_fields)]
struct EffectiveStoreSource {
backend: String,
format_version: u32,
network: String,
postgres: EffectivePostgresSource,
profile_id: String,
}
#[derive(serde::Deserialize)]
#[serde(deny_unknown_fields)]
struct EffectivePostgresSource {
bootstrap: EffectivePostgresBootstrapSource,
connection_uri: String,
pool: EffectivePostgresPoolSource,
shutdown_timeout_ms: u64,
tls: EffectivePostgresTlsSource,
}
#[derive(serde::Deserialize)]
#[serde(deny_unknown_fields)]
struct EffectivePostgresPoolSource {
connect_timeout_ms: u64,
create_timeout_ms: u64,
max_connections: u32,
recycle_timeout_ms: u64,
wait_timeout_ms: u64,
}
#[derive(serde::Deserialize)]
#[serde(deny_unknown_fields)]
struct EffectivePostgresTlsSource {
mode: String,
}
#[derive(serde::Deserialize)]
#[serde(deny_unknown_fields)]
struct EffectivePostgresBootstrapSource {
auto_migrate: std::option::Option<bool>,
migration_lock_timeout_ms: u64,
migration_timeout_ms: u64,
schema_autocreate: std::option::Option<bool>,
schema_autoupdate: std::option::Option<bool>,
}
fn resolve_store_profile(profile: &crate::ResolvedConfigProfile, environment: &crate::ConfigEnvironment) -> ksp_core_lib::Result<ResolvedStoreConfig> {
ksp_logging_lib::trace!(target: crate::TRACING_TARGET, profile_id = profile.profile_id(), "mapping standard Store Config profile");
let effective = profile.resolve_effective_environment_detailed(environment);
let effective = match effective {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let provenance = validate_connection_uri_provenance(&effective, profile);
if let std::result::Result::Err(error) = provenance {
return std::result::Result::Err(error);
}
let source = serde_json::from_value::<EffectiveStoreSource>(effective.value().clone());
let source = match source {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => {
return std::result::Result::Err(
effective_error(profile, "effective Store Config cannot be decoded into the runtime adapter contract").with_source(error),
);
},
};
if source.format_version != 1 && source.format_version != 2 {
return std::result::Result::Err(effective_error(profile, "effective Store format_version is unsupported"));
}
if source.profile_id != profile.profile_id() {
return std::result::Result::Err(effective_error(profile, "effective Store profile_id does not match the selected profile"));
}
if source.backend != "postgres" {
return std::result::Result::Err(effective_error(profile, "effective Store backend is unsupported").with_context("backend", source.backend));
}
let tls_mode = match source.postgres.tls.mode.as_str() {
"disabled" => ksp_store_lib::PostgresTlsMode::Disabled,
"verify_full" => ksp_store_lib::PostgresTlsMode::VerifyFull,
_ => return std::result::Result::Err(effective_error(profile, "effective Store PostgreSQL TLS mode is unsupported")),
};
let pool = ksp_store_lib::PostgresPoolSettings::new(
source.postgres.pool.max_connections,
std::time::Duration::from_millis(source.postgres.pool.connect_timeout_ms),
std::time::Duration::from_millis(source.postgres.pool.wait_timeout_ms),
std::time::Duration::from_millis(source.postgres.pool.create_timeout_ms),
std::time::Duration::from_millis(source.postgres.pool.recycle_timeout_ms),
);
let bootstrap = match source.format_version {
1 => {
let auto_migrate =
match (source.postgres.bootstrap.auto_migrate, source.postgres.bootstrap.schema_autocreate, source.postgres.bootstrap.schema_autoupdate) {
(std::option::Option::Some(value), std::option::Option::None, std::option::Option::None) => value,
_ => return std::result::Result::Err(effective_error(profile, "effective Store V1 bootstrap policy is invalid")),
};
ksp_store_lib::PostgresBootstrapSettings::new(
auto_migrate,
std::time::Duration::from_millis(source.postgres.bootstrap.migration_timeout_ms),
std::time::Duration::from_millis(source.postgres.bootstrap.migration_lock_timeout_ms),
)
},
2 => {
let (schema_autocreate, schema_autoupdate) =
match (source.postgres.bootstrap.auto_migrate, source.postgres.bootstrap.schema_autocreate, source.postgres.bootstrap.schema_autoupdate) {
(std::option::Option::None, std::option::Option::Some(autocreate), std::option::Option::Some(autoupdate)) => (autocreate, autoupdate),
_ => return std::result::Result::Err(effective_error(profile, "effective Store V2 bootstrap policy is invalid")),
};
ksp_store_lib::PostgresBootstrapSettings::with_schema_policy(
schema_autocreate,
schema_autoupdate,
std::time::Duration::from_millis(source.postgres.bootstrap.migration_timeout_ms),
std::time::Duration::from_millis(source.postgres.bootstrap.migration_lock_timeout_ms),
)
},
_ => return std::result::Result::Err(effective_error(profile, "effective Store format_version is unsupported")),
};
let network = ksp_store_lib::RawNetworkId::new(source.network);
let network = match network {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return std::result::Result::Err(effective_error(profile, "effective Store network identifier is invalid")),
};
let postgres = ksp_store_lib::PostgresStoreSettings::new(source.postgres.connection_uri, pool, tls_mode, bootstrap);
let settings = ksp_store_lib::StoreSettings::new(
network,
ksp_store_lib::StoreBackendSettings::Postgres(postgres),
std::time::Duration::from_millis(source.postgres.shutdown_timeout_ms),
);
if let std::result::Result::Err(error) = settings.validate() {
return std::result::Result::Err(store_contract_error(profile, &error));
}
ksp_logging_lib::debug!(
target: crate::TRACING_TARGET,
profile_id = profile.profile_id(),
network = settings.network().as_str(),
backend = settings.backend_kind().code(),
"mapped standard Store Config to Store settings"
);
return std::result::Result::Ok(ResolvedStoreConfig {
effective,
file_id: profile.file_id().clone(),
profile_id: profile.profile_id().to_owned(),
selection_source: profile.selection_source(),
settings,
source_path: profile.path().to_path_buf(),
});
}
fn validate_connection_uri_provenance(effective: &crate::ResolvedConfigJson, profile: &crate::ResolvedConfigProfile) -> ksp_core_lib::Result<()> {
let provenance = match effective.provenance_at("/postgres/connection_uri") {
std::option::Option::Some(value) => value,
std::option::Option::None => return std::result::Result::Err(effective_error(profile, "Store PostgreSQL connection URI provenance is unavailable")),
};
let mut has_secret_environment = false;
for item in provenance {
let variable_name = match item.variable_name() {
std::option::Option::Some(value) => value,
std::option::Option::None => continue,
};
let sensitivity = crate::ConfigSensitivity::from_variable_name(variable_name);
let sensitivity = match sensitivity {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
if !sensitivity.is_secret() {
return std::result::Result::Err(effective_error(profile, "Store PostgreSQL connection URI may reference only secret environment variables"));
}
has_secret_environment = true;
}
if !has_secret_environment {
return std::result::Result::Err(effective_error(profile, "Store PostgreSQL connection URI requires secret environment provenance"));
}
return std::result::Result::Ok(());
}
fn store_contract_error(profile: &crate::ResolvedConfigProfile, error: &ksp_core_lib::Error) -> ksp_core_lib::Error {
return effective_error(profile, "effective Store settings fail the Store runtime contract")
.with_context("store_error_domain", error.code().domain())
.with_context("store_error_code", error.code().code());
}
fn effective_error(profile: &crate::ResolvedConfigProfile, reason: &'static str) -> ksp_core_lib::Error {
return ksp_core_lib::Error::new(crate::ERROR_CODE_EFFECTIVE_CONFIG_INVALID, "effective Config cannot be mapped to the requested runtime contract")
.with_context("file_id", profile.file_id().as_str())
.with_context("profile_id", profile.profile_id())
.with_context("reason", reason);
}
#[cfg(test)]
#[path = "../unit_tests/store.rs"]
mod tests;

View File

@@ -1,5 +1,5 @@
// file: crates/ksp-config-lib/tests/ownership.rs
// version: 8
// version: 9
//! Workspace ownership audits for KSP application configuration boundaries.
@@ -280,6 +280,49 @@ fn workspace_crates_do_not_hardcode_config_managed_physical_files() {
}
}
#[test]
fn store_config_adapter_does_not_force_backend_feature_or_reverse_dependency() {
let root = workspace_root();
let config_manifest_path = root.join("crates/ksp-config-lib/Cargo.toml");
let config_manifest = std::fs::read_to_string(config_manifest_path.as_path());
assert!(config_manifest.is_ok(), "unable to read {}", config_manifest_path.display());
if let std::result::Result::Ok(config_manifest) = config_manifest {
assert!(
config_manifest.contains("ksp-store-lib = { path = \"../ksp-store-lib\", default-features = false }"),
"Config -> Store dependency must not force a physical backend feature"
);
}
for crate_name in ["ksp-store-lib", "ksp-store-postgres-lib"] {
let manifest_path = root.join("crates").join(crate_name).join("Cargo.toml");
let manifest = std::fs::read_to_string(manifest_path.as_path());
assert!(manifest.is_ok(), "unable to read {}", manifest_path.display());
if let std::result::Result::Ok(manifest) = manifest {
assert!(!manifest.contains("ksp-config-lib"), "{} must not depend back on Config", manifest_path.display());
}
}
}
#[test]
fn store_runtime_sources_do_not_bypass_config_for_environment_or_libpq_files() {
let root = workspace_root();
for crate_name in ["ksp-store-lib", "ksp-store-postgres-lib"] {
let source_root = root.join("crates").join(crate_name).join("src");
let mut rust_files = std::vec::Vec::new();
collect_rust_files(source_root.as_path(), &mut rust_files);
for rust_file in rust_files {
let source = std::fs::read_to_string(rust_file.as_path());
assert!(source.is_ok(), "unable to read {}", rust_file.display());
let source = match source {
std::result::Result::Ok(value) => non_comment_source(value.as_str()),
std::result::Result::Err(_) => continue,
};
for forbidden in ["std::env", "dotenv", "KSP_", "KSPB_", "\"PG", ".pgpass"] {
assert!(!source.contains(forbidden), "{} bypasses Config through forbidden Store environment/libpq token {forbidden}", rust_file.display());
}
}
}
}
#[test]
fn environment_name_scanner_ignores_namespace_labels_but_keeps_concrete_names() {
let source = r#"

View File

@@ -1,8 +1,8 @@
// file: crates/ksp-config-lib/tests/public_api.rs
// version: 25
// version: 26
//! Integration tests for the public `ksp-config-lib` bootstrap, registry, JSON/profile/composite, environment-resolution, sensitivity,
//! Logging/Transport adapters and management contracts.
//! Logging/Transport/Store adapters and management contracts.
#[test]
fn bootstrap_contract_is_available_from_crate_root() {
@@ -80,19 +80,21 @@ fn registry_descriptor_inventory_is_available_from_crate_root() {
assert!(registry.is_ok(), "public registry should remain constructible: {registry:?}");
if let std::result::Result::Ok(registry) = registry {
let descriptors: std::vec::Vec<&ksp_config_lib::ConfigFileDescriptor> = registry.descriptors().collect();
assert_eq!(descriptors.len(), 11);
assert_eq!(descriptors.len(), 13);
assert_eq!(descriptors[0].file_id().as_str(), ksp_config_lib::FILE_ID_COMPOSITE_KSP_APP_SOLPRICES_DESK);
assert_eq!(descriptors[1].file_id().as_str(), ksp_config_lib::FILE_ID_COMPOSITE_KSP_APP_WALLET_DESK);
assert_eq!(descriptors[2].file_id().as_str(), ksp_config_lib::FILE_ID_STD_LOGGING);
assert_eq!(descriptors[3].file_id().as_str(), ksp_config_lib::FILE_ID_STD_OFFCHAIN_TRANSPORT);
assert_eq!(descriptors[4].file_id().as_str(), ksp_config_lib::FILE_ID_STD_TRANSPORT);
assert_eq!(descriptors[5].file_id().as_str(), ksp_config_lib::FILE_ID_STD_WALLET);
assert_eq!(descriptors[6].file_id().as_str(), ksp_config_lib::FILE_ID_SCHEMA_COMPOSITE);
assert_eq!(descriptors[7].file_id().as_str(), ksp_config_lib::FILE_ID_SCHEMA_STD_LOGGING);
assert_eq!(descriptors[8].file_id().as_str(), ksp_config_lib::FILE_ID_SCHEMA_STD_OFFCHAIN_TRANSPORT);
assert_eq!(descriptors[9].file_id().as_str(), ksp_config_lib::FILE_ID_SCHEMA_STD_TRANSPORT);
assert_eq!(descriptors[10].file_id().as_str(), ksp_config_lib::FILE_ID_SCHEMA_STD_WALLET);
let schema_file_id = descriptors[5].schema_file_id();
assert_eq!(descriptors[4].file_id().as_str(), ksp_config_lib::FILE_ID_STD_STORE);
assert_eq!(descriptors[5].file_id().as_str(), ksp_config_lib::FILE_ID_STD_TRANSPORT);
assert_eq!(descriptors[6].file_id().as_str(), ksp_config_lib::FILE_ID_STD_WALLET);
assert_eq!(descriptors[7].file_id().as_str(), ksp_config_lib::FILE_ID_SCHEMA_COMPOSITE);
assert_eq!(descriptors[8].file_id().as_str(), ksp_config_lib::FILE_ID_SCHEMA_STD_LOGGING);
assert_eq!(descriptors[9].file_id().as_str(), ksp_config_lib::FILE_ID_SCHEMA_STD_OFFCHAIN_TRANSPORT);
assert_eq!(descriptors[10].file_id().as_str(), ksp_config_lib::FILE_ID_SCHEMA_STD_STORE);
assert_eq!(descriptors[11].file_id().as_str(), ksp_config_lib::FILE_ID_SCHEMA_STD_TRANSPORT);
assert_eq!(descriptors[12].file_id().as_str(), ksp_config_lib::FILE_ID_SCHEMA_STD_WALLET);
let schema_file_id = descriptors[6].schema_file_id();
assert!(schema_file_id.is_some(), "public Wallet descriptor should preserve schema association");
if let std::option::Option::Some(schema_file_id) = schema_file_id {
assert_eq!(schema_file_id.as_str(), ksp_config_lib::FILE_ID_SCHEMA_STD_WALLET);
@@ -211,6 +213,18 @@ fn logging_adapter_contract_is_available_from_crate_root() {
assert!(std::mem::size_of::<ksp_config_lib::ResolvedLoggingConfig>() > 0);
}
#[test]
fn store_adapter_contract_is_available_from_crate_root() {
let adapter = ksp_config_lib::ConfigDocumentEngine::load_resolved_store_config;
let composite_adapter = ksp_config_lib::ConfigDocumentEngine::resolve_store_config_profile;
let _ = (adapter, composite_adapter);
assert_eq!(ksp_config_lib::FILE_ID_STD_STORE, "cfg.std.store");
assert_eq!(ksp_config_lib::FILE_ID_SCHEMA_STD_STORE, "schema.std.store");
assert_eq!(ksp_config_lib::DEFAULT_STD_STORE_FILENAME, "std.store.json");
assert_eq!(ksp_config_lib::DEFAULT_STD_STORE_SCHEMA_FILENAME, "std.store.schema.json");
assert!(std::mem::size_of::<ksp_config_lib::ResolvedStoreConfig>() > 0);
}
#[test]
fn management_contracts_are_available_from_crate_root() {
let workspace = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../..");

View File

@@ -0,0 +1,81 @@
{
"format_version": 2,
"default_profile": "devnet",
"profiles": [
{
"profile_id": "devnet",
"network": "devnet",
"backend": "postgres",
"postgres": {
"connection_uri": "${KSP_SECRET_STORE_DEVNET_POSTGRES_URI:-postgresql://localhost/ksp_devnet}",
"pool": {
"max_connections": 8,
"connect_timeout_ms": 10000,
"wait_timeout_ms": 5000,
"create_timeout_ms": 10000,
"recycle_timeout_ms": 5000
},
"tls": {
"mode": "verify_full"
},
"bootstrap": {
"schema_autocreate": true,
"schema_autoupdate": true,
"migration_timeout_ms": 30000,
"migration_lock_timeout_ms": 10000
},
"shutdown_timeout_ms": 5000
}
},
{
"profile_id": "mainnet",
"network": "mainnet-beta",
"backend": "postgres",
"postgres": {
"connection_uri": "${KSP_SECRET_STORE_MAINNET_POSTGRES_URI:-postgresql://localhost/ksp_mainnet}",
"pool": {
"max_connections": 8,
"connect_timeout_ms": 10000,
"wait_timeout_ms": 5000,
"create_timeout_ms": 10000,
"recycle_timeout_ms": 5000
},
"tls": {
"mode": "verify_full"
},
"bootstrap": {
"schema_autocreate": true,
"schema_autoupdate": true,
"migration_timeout_ms": 30000,
"migration_lock_timeout_ms": 10000
},
"shutdown_timeout_ms": 5000
}
},
{
"profile_id": "testnet",
"network": "testnet",
"backend": "postgres",
"postgres": {
"connection_uri": "${KSP_SECRET_STORE_TESTNET_POSTGRES_URI:-postgresql://localhost/ksp_testnet}",
"pool": {
"max_connections": 8,
"connect_timeout_ms": 10000,
"wait_timeout_ms": 5000,
"create_timeout_ms": 10000,
"recycle_timeout_ms": 5000
},
"tls": {
"mode": "verify_full"
},
"bootstrap": {
"schema_autocreate": true,
"schema_autoupdate": true,
"migration_timeout_ms": 30000,
"migration_lock_timeout_ms": 10000
},
"shutdown_timeout_ms": 5000
}
}
]
}

View File

@@ -1,5 +1,5 @@
// file: crates/ksp-config-lib/unit_tests/registry.rs
// version: 10
// version: 11
#[test]
fn descriptors_expose_complete_registry_in_deterministic_file_id_order() {
@@ -7,7 +7,7 @@ fn descriptors_expose_complete_registry_in_deterministic_file_id_order() {
assert!(registry.is_ok(), "default registry should be valid: {registry:?}");
if let std::result::Result::Ok(registry) = registry {
let descriptors: std::vec::Vec<&crate::ConfigFileDescriptor> = registry.descriptors().collect();
assert_eq!(descriptors.len(), 11);
assert_eq!(descriptors.len(), 13);
assert_eq!(descriptors[0].file_id().as_str(), crate::FILE_ID_COMPOSITE_KSP_APP_SOLPRICES_DESK);
assert_eq!(descriptors[0].filename(), std::path::Path::new(crate::DEFAULT_COMPOSITE_KSP_APP_SOLPRICES_DESK_FILENAME));
assert_eq!(descriptors[0].schema_file_id().map(crate::ConfigFileId::as_str), std::option::Option::Some(crate::FILE_ID_SCHEMA_COMPOSITE));
@@ -15,17 +15,19 @@ fn descriptors_expose_complete_registry_in_deterministic_file_id_order() {
assert_eq!(descriptors[1].filename(), std::path::Path::new(crate::DEFAULT_COMPOSITE_KSP_APP_WALLET_DESK_FILENAME));
assert_eq!(descriptors[2].file_id().as_str(), crate::FILE_ID_STD_LOGGING);
assert_eq!(descriptors[3].file_id().as_str(), crate::FILE_ID_STD_OFFCHAIN_TRANSPORT);
assert_eq!(descriptors[4].file_id().as_str(), crate::FILE_ID_STD_TRANSPORT);
assert_eq!(descriptors[5].file_id().as_str(), crate::FILE_ID_STD_WALLET);
assert_eq!(descriptors[5].filename(), std::path::Path::new(crate::DEFAULT_STD_WALLET_FILENAME));
assert_eq!(descriptors[5].schema_file_id().map(crate::ConfigFileId::as_str), std::option::Option::Some(crate::FILE_ID_SCHEMA_STD_WALLET));
assert_eq!(descriptors[6].file_id().as_str(), crate::FILE_ID_SCHEMA_COMPOSITE);
assert_eq!(descriptors[7].file_id().as_str(), crate::FILE_ID_SCHEMA_STD_LOGGING);
assert_eq!(descriptors[8].file_id().as_str(), crate::FILE_ID_SCHEMA_STD_OFFCHAIN_TRANSPORT);
assert_eq!(descriptors[9].file_id().as_str(), crate::FILE_ID_SCHEMA_STD_TRANSPORT);
assert_eq!(descriptors[10].file_id().as_str(), crate::FILE_ID_SCHEMA_STD_WALLET);
assert!(descriptors[0..6].iter().all(|descriptor| return descriptor.kind() == crate::ConfigFileKind::Config));
assert!(descriptors[6..11].iter().all(|descriptor| return descriptor.kind() == crate::ConfigFileKind::Schema));
assert_eq!(descriptors[4].file_id().as_str(), crate::FILE_ID_STD_STORE);
assert_eq!(descriptors[5].file_id().as_str(), crate::FILE_ID_STD_TRANSPORT);
assert_eq!(descriptors[6].file_id().as_str(), crate::FILE_ID_STD_WALLET);
assert_eq!(descriptors[6].filename(), std::path::Path::new(crate::DEFAULT_STD_WALLET_FILENAME));
assert_eq!(descriptors[6].schema_file_id().map(crate::ConfigFileId::as_str), std::option::Option::Some(crate::FILE_ID_SCHEMA_STD_WALLET));
assert_eq!(descriptors[7].file_id().as_str(), crate::FILE_ID_SCHEMA_COMPOSITE);
assert_eq!(descriptors[8].file_id().as_str(), crate::FILE_ID_SCHEMA_STD_LOGGING);
assert_eq!(descriptors[9].file_id().as_str(), crate::FILE_ID_SCHEMA_STD_OFFCHAIN_TRANSPORT);
assert_eq!(descriptors[10].file_id().as_str(), crate::FILE_ID_SCHEMA_STD_STORE);
assert_eq!(descriptors[11].file_id().as_str(), crate::FILE_ID_SCHEMA_STD_TRANSPORT);
assert_eq!(descriptors[12].file_id().as_str(), crate::FILE_ID_SCHEMA_STD_WALLET);
assert!(descriptors[0..7].iter().all(|descriptor| return descriptor.kind() == crate::ConfigFileKind::Config));
assert!(descriptors[7..13].iter().all(|descriptor| return descriptor.kind() == crate::ConfigFileKind::Schema));
}
}
@@ -114,6 +116,27 @@ fn defaults_register_offchain_transport_document_and_schema_with_distinct_roots(
}
}
#[test]
fn defaults_register_store_document_and_schema_with_distinct_roots() {
let registry = crate::ConfigFileRegistry::defaults();
assert!(registry.is_ok());
if let std::result::Result::Ok(registry) = registry {
let config_id = crate::ConfigFileId::new(crate::FILE_ID_STD_STORE);
let schema_id = crate::ConfigFileId::new(crate::FILE_ID_SCHEMA_STD_STORE);
if let (std::result::Result::Ok(config_id), std::result::Result::Ok(schema_id)) = (config_id, schema_id) {
let config = registry.descriptor(&config_id);
let schema = registry.descriptor(&schema_id);
if let (std::result::Result::Ok(config), std::result::Result::Ok(schema)) = (config, schema) {
assert_eq!(config.kind(), crate::ConfigFileKind::Config);
assert_eq!(config.filename(), std::path::Path::new(crate::DEFAULT_STD_STORE_FILENAME));
assert_eq!(config.schema_file_id(), std::option::Option::Some(&schema_id));
assert_eq!(schema.kind(), crate::ConfigFileKind::Schema);
assert_eq!(schema.filename(), std::path::Path::new(crate::DEFAULT_STD_STORE_SCHEMA_FILENAME));
}
}
}
}
#[test]
fn defaults_register_transport_document_and_schema_with_distinct_roots() {
let registry = crate::ConfigFileRegistry::defaults();

View File

@@ -0,0 +1,285 @@
// file: crates/ksp-config-lib/unit_tests/store.rs
// version: 3
#[test]
fn committed_store_profile_maps_exact_runtime_settings_and_secret_fallback() {
let engine = committed_engine();
let engine = match engine {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return,
};
let environment = crate::ConfigEnvironment::from_maps(std::collections::BTreeMap::new(), std::collections::BTreeMap::new());
let resolved = engine.load_resolved_store_config(std::option::Option::None, &environment);
assert!(resolved.is_ok(), "committed Store profile should map without opening PostgreSQL: {resolved:?}");
if let std::result::Result::Ok(resolved) = resolved {
assert_eq!(resolved.file_id().as_str(), crate::FILE_ID_STD_STORE);
assert_eq!(resolved.profile_id(), "devnet");
assert_eq!(resolved.target_id(), "devnet");
assert_eq!(resolved.selection_source(), crate::ConfigProfileSelectionSource::DefaultProfile);
assert_eq!(resolved.settings().backend_kind(), ksp_store_lib::StoreBackendKind::Postgres);
assert_eq!(resolved.settings().network().as_str(), "devnet");
assert_eq!(resolved.settings().shutdown_timeout(), std::time::Duration::from_millis(5_000));
let postgres = match resolved.settings().backend() {
ksp_store_lib::StoreBackendSettings::Postgres(postgres) => std::option::Option::Some(postgres),
_ => std::option::Option::None,
};
assert!(postgres.is_some(), "pre.004 fixture should map to the PostgreSQL Store backend");
if let std::option::Option::Some(postgres) = postgres {
assert_eq!(postgres.pool().max_connections(), 8);
assert_eq!(postgres.pool().connect_timeout(), std::time::Duration::from_millis(10_000));
assert_eq!(postgres.pool().wait_timeout(), std::time::Duration::from_millis(5_000));
assert_eq!(postgres.pool().create_timeout(), std::time::Duration::from_millis(10_000));
assert_eq!(postgres.pool().recycle_timeout(), std::time::Duration::from_millis(5_000));
assert_eq!(postgres.tls_mode(), ksp_store_lib::PostgresTlsMode::VerifyFull);
assert!(postgres.bootstrap().auto_migrate());
assert!(postgres.bootstrap().schema_autocreate());
assert!(postgres.bootstrap().schema_autoupdate());
assert_eq!(postgres.bootstrap().migration_timeout(), std::time::Duration::from_millis(30_000));
assert_eq!(postgres.bootstrap().migration_lock_timeout(), std::time::Duration::from_millis(10_000));
}
assert!(resolved.effective().sensitivity().is_secret());
let safe = resolved.effective().safe_value().to_string();
assert!(!safe.contains("postgresql://localhost/ksp_devnet"));
assert!(safe.contains(crate::REDACTED_CONFIG_VALUE));
let debug = format!("{resolved:?}");
assert!(!debug.contains("postgresql://localhost/ksp_devnet"));
}
}
#[test]
fn process_store_uri_wins_and_remains_redacted_in_safe_views() {
let engine = committed_engine();
let engine = match engine {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return,
};
let canary = "postgresql://secret-user:secret-pass@db.example/ksp_devnet";
let mut process = std::collections::BTreeMap::<String, String>::new();
process.insert("KSP_SECRET_STORE_DEVNET_POSTGRES_URI".to_owned(), canary.to_owned());
let environment = crate::ConfigEnvironment::from_maps(process, std::collections::BTreeMap::new());
let resolved = engine.load_resolved_store_config(std::option::Option::None, &environment);
assert!(resolved.is_ok(), "secret process Store URI should map: {resolved:?}");
if let std::result::Result::Ok(resolved) = resolved {
assert_eq!(resolved.effective().value().pointer("/postgres/connection_uri").and_then(serde_json::Value::as_str), std::option::Option::Some(canary));
assert!(!resolved.effective().safe_value().to_string().contains(canary));
assert!(!format!("{resolved:?}").contains(canary));
let provenance = resolved.effective().provenance_at("/postgres/connection_uri");
assert!(provenance.is_some());
if let std::option::Option::Some(provenance) = provenance {
assert!(provenance.iter().any(|item| return item.environment_source() == std::option::Option::Some(crate::ConfigEnvironmentSource::Process)));
}
}
}
#[test]
fn literal_or_nonsecret_store_uri_is_rejected_by_effective_adapter() {
for value in ["postgresql://literal.invalid/ksp", "${KSP_PUBLIC_STORE_POSTGRES_URI:-postgresql://public.invalid/ksp}"] {
let fixture = tempfile::tempdir();
assert!(fixture.is_ok());
let fixture = match fixture {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return,
};
let source = committed_document_value();
let mut source = match source {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return,
};
let profiles = source.get_mut("profiles").and_then(serde_json::Value::as_array_mut);
if let std::option::Option::Some(profiles) = profiles
&& let std::option::Option::Some(profile) = profiles.first_mut()
{
profile["postgres"]["connection_uri"] = serde_json::Value::String(value.to_owned());
}
let engine = fixture_engine_with_document(fixture.path(), &source);
assert!(engine.is_ok());
let engine = match engine {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return,
};
let environment = crate::ConfigEnvironment::from_maps(std::collections::BTreeMap::new(), std::collections::BTreeMap::new());
let resolved = engine.load_resolved_store_config(std::option::Option::None, &environment);
assert!(resolved.is_err(), "Store URI without secret provenance must be rejected");
if let std::result::Result::Err(error) = resolved {
assert_eq!(error.code(), crate::ERROR_CODE_EFFECTIVE_CONFIG_INVALID);
assert!(!format!("{error:?}").contains("literal.invalid"));
assert!(!format!("{error:?}").contains("public.invalid"));
}
}
}
#[test]
fn named_store_targets_select_one_network_and_database_without_runtime_multiplexing() {
let engine = committed_engine();
let engine = match engine {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return,
};
let mut process = std::collections::BTreeMap::<String, String>::new();
process.insert("KSP_SECRET_STORE_DEVNET_POSTGRES_URI".to_owned(), "postgresql://devnet.invalid/ksp_devnet".to_owned());
process.insert("KSP_SECRET_STORE_MAINNET_POSTGRES_URI".to_owned(), "postgresql://mainnet.invalid/ksp_mainnet".to_owned());
process.insert("KSP_SECRET_STORE_TESTNET_POSTGRES_URI".to_owned(), "postgresql://testnet.invalid/ksp_testnet".to_owned());
let environment = crate::ConfigEnvironment::from_maps(process, std::collections::BTreeMap::new());
for (target_id, network, expected_uri) in [
("devnet", "devnet", "postgresql://devnet.invalid/ksp_devnet"),
("mainnet", "mainnet-beta", "postgresql://mainnet.invalid/ksp_mainnet"),
("testnet", "testnet", "postgresql://testnet.invalid/ksp_testnet"),
] {
let resolved = engine.load_resolved_store_config(std::option::Option::Some(target_id), &environment);
assert!(resolved.is_ok(), "named Store target should resolve independently: {target_id}: {resolved:?}");
if let std::result::Result::Ok(resolved) = resolved {
assert_eq!(resolved.target_id(), target_id);
assert_eq!(resolved.profile_id(), target_id);
assert_eq!(resolved.selection_source(), crate::ConfigProfileSelectionSource::Explicit);
assert_eq!(resolved.settings().network().as_str(), network);
assert_eq!(
resolved.effective().value().pointer("/postgres/connection_uri").and_then(serde_json::Value::as_str),
std::option::Option::Some(expected_uri),
);
assert!(!resolved.effective().safe_value().to_string().contains(expected_uri));
}
}
return;
}
#[test]
fn pre_003_fix_001_v1_auto_migrate_remains_backward_readable_and_maps_both_schema_policies() {
for auto_migrate in [false, true] {
let fixture = tempfile::tempdir();
assert!(fixture.is_ok());
let fixture = match fixture {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return,
};
let source = committed_document_value();
let mut source = match source {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return,
};
source["format_version"] = serde_json::Value::from(1);
let profiles = source.get_mut("profiles").and_then(serde_json::Value::as_array_mut);
if let std::option::Option::Some(profiles) = profiles {
for profile in profiles {
let bootstrap = profile.pointer_mut("/postgres/bootstrap").and_then(serde_json::Value::as_object_mut);
if let std::option::Option::Some(bootstrap) = bootstrap {
bootstrap.remove("schema_autocreate");
bootstrap.remove("schema_autoupdate");
bootstrap.insert("auto_migrate".to_owned(), serde_json::Value::Bool(auto_migrate));
}
}
}
let engine = fixture_engine_with_document(fixture.path(), &source);
assert!(engine.is_ok());
let engine = match engine {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return,
};
let environment = crate::ConfigEnvironment::from_maps(std::collections::BTreeMap::new(), std::collections::BTreeMap::new());
let resolved = engine.load_resolved_store_config(std::option::Option::None, &environment);
assert!(resolved.is_ok(), "Store Config V1 compatibility mapping failed: {resolved:?}");
if let std::result::Result::Ok(resolved) = resolved
&& let ksp_store_lib::StoreBackendSettings::Postgres(postgres) = resolved.settings().backend()
{
assert_eq!(postgres.bootstrap().schema_autocreate(), auto_migrate);
assert_eq!(postgres.bootstrap().schema_autoupdate(), auto_migrate);
assert_eq!(postgres.bootstrap().auto_migrate(), auto_migrate);
}
}
return;
}
#[test]
fn pre_003_fix_001_v2_schema_creation_and_update_policies_map_independently() {
let fixture = tempfile::tempdir();
assert!(fixture.is_ok());
let fixture = match fixture {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return,
};
let source = committed_document_value();
let mut source = match source {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return,
};
let profiles = source.get_mut("profiles").and_then(serde_json::Value::as_array_mut);
if let std::option::Option::Some(profiles) = profiles
&& let std::option::Option::Some(profile) = profiles.first_mut()
{
profile["postgres"]["bootstrap"]["schema_autocreate"] = serde_json::Value::Bool(false);
profile["postgres"]["bootstrap"]["schema_autoupdate"] = serde_json::Value::Bool(true);
}
let engine = fixture_engine_with_document(fixture.path(), &source);
assert!(engine.is_ok());
let engine = match engine {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return,
};
let environment = crate::ConfigEnvironment::from_maps(std::collections::BTreeMap::new(), std::collections::BTreeMap::new());
let resolved = engine.load_resolved_store_config(std::option::Option::None, &environment);
assert!(resolved.is_ok(), "Store Config V2 split schema policy mapping failed: {resolved:?}");
if let std::result::Result::Ok(resolved) = resolved
&& let ksp_store_lib::StoreBackendSettings::Postgres(postgres) = resolved.settings().backend()
{
assert!(!postgres.bootstrap().schema_autocreate());
assert!(postgres.bootstrap().schema_autoupdate());
}
return;
}
fn committed_engine() -> ksp_core_lib::Result<crate::ConfigDocumentEngine> {
let workspace = workspace_root();
let bootstrap = crate::ConfigBootstrapOptions::from_paths(workspace.join("config"), workspace.join("config/schemas"));
let bootstrap = match bootstrap {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let registry = crate::ConfigFileRegistry::defaults();
let registry = match registry {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
return std::result::Result::Ok(crate::ConfigDocumentEngine::new(bootstrap, registry));
}
fn fixture_engine_with_document(root: &std::path::Path, document: &serde_json::Value) -> ksp_core_lib::Result<crate::ConfigDocumentEngine> {
let config_root = root.join("config");
if let std::result::Result::Err(error) = std::fs::create_dir_all(config_root.as_path()) {
return std::result::Result::Err(
ksp_core_lib::Error::new(crate::ERROR_CODE_JSON_FILE_READ_FAILED, "test Config root cannot be created").with_source(error),
);
}
let bytes = serde_json::to_vec_pretty(document);
let bytes = match bytes {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => {
return std::result::Result::Err(
ksp_core_lib::Error::new(crate::ERROR_CODE_JSON_SYNTAX_INVALID, "test Store Config cannot be encoded").with_source(error),
);
},
};
let path = config_root.join(crate::DEFAULT_STD_STORE_FILENAME);
if let std::result::Result::Err(error) = std::fs::write(path.as_path(), bytes) {
return std::result::Result::Err(
ksp_core_lib::Error::new(crate::ERROR_CODE_JSON_FILE_READ_FAILED, "test Store Config cannot be written").with_source(error),
);
}
let bootstrap = crate::ConfigBootstrapOptions::from_paths(config_root, workspace_root().join("config/schemas"));
let bootstrap = match bootstrap {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let registry = crate::ConfigFileRegistry::defaults();
let registry = match registry {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
return std::result::Result::Ok(crate::ConfigDocumentEngine::new(bootstrap, registry));
}
fn committed_document_value() -> std::result::Result<serde_json::Value, serde_json::Error> {
return serde_json::from_str(include_str!("../../../config/std.store.json"));
}
fn workspace_root() -> std::path::PathBuf {
return std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../..");
}

View File

@@ -1,9 +1,11 @@
<!-- file: crates/ksp-interface-lib/README.md -->
<!-- version: 2 -->
<!-- version: 3 -->
# ksp-interface-lib
`ksp-interface-lib` est la façade wire officielle KSP destinée aux contrats passifs partagés par les implémentations Program Solana officielles ou externes. La crate expose uniquement des structures de représentation/admission ; elle ne possède ni transport, ni exécution, ni persistence, ni comportement métier Program.
`ksp-interface-lib` possède les contrats passifs KSP qui doivent être partagés entre plusieurs composants sans imposer leur runtime d'origine. Sa surface couvre actuellement deux familles distinctes : les contrats wire Program génériques et un petit ensemble de faits d'acquisition provider-neutral dont la sémantique commune a été démontrée.
La crate reste une façade de représentation. Elle ne possède ni transport réseau, ni event bus, ni worker/job, ni persistence, ni exécution, ni comportement métier Program.
## Ownership
@@ -15,9 +17,9 @@ Error / ErrorCode / Result
Program IDs fondamentaux
```
`Pubkey` est réexporté depuis le crate-root Interface afin qu'un consumer wire n'introduise aucun wrapper d'identité parallèle. Les Program IDs restent possédés et répertoriés par Core.
`Pubkey` est réexporté depuis le crate-root Interface afin qu'un consumer n'introduise aucun wrapper d'identité parallèle. Les Program IDs restent possédés et répertoriés par Core.
La dependency direction candidate `0.2.13` reste strictement :
Le graphe normal reste strictement :
```text
ksp-interface-lib
@@ -25,9 +27,11 @@ ksp-interface-lib
└── solana-pubkey
```
## Surface publique `0.2.13`
La crate ne possède aucune feature Cargo, aucune `dev-dependency` et aucune `build-dependency` runtime propre.
La façade crate-root expose exactement :
## Surface publique
La façade crate-root expose :
```text
Pubkey
@@ -36,10 +40,17 @@ MAX_PROGRAM_INSTRUCTION_ACCOUNTS
ProgramInstruction
MAX_PROGRAM_INSTRUCTION_DATA_LEN
ERROR_CODE_PROGRAM_INSTRUCTION_LIMIT_EXCEEDED
SlotLifecycleStage
SlotLifecycleEvent
TransactionSignature
TransactionExecutionOutcome
TransactionExecutionEvent
```
Aucun module interne n'est public.
## Contrats Program passifs
### `ProgramAccountMeta`
`ProgramAccountMeta` représente un compte ordonné d'une instruction avec :
@@ -68,16 +79,14 @@ data: Vec<u8>
Les cas vides sont valides et une `program_id` inconnue du registry KSP reste admissible.
## Bornes d'admission
Interface applique deux limites locales :
### Bornes d'admission
| Limite | Valeur |
|------------------------------------|----------|
| `MAX_PROGRAM_INSTRUCTION_ACCOUNTS` | `255` |
| `MAX_PROGRAM_INSTRUCTION_DATA_LEN` | `10_240` |
Ces valeurs sont des **bornes d'admission Interface**. Elles ne constituent pas une garantie qu'une instruction donnée tient dans toutes les contraintes d'une transaction Solana top-level. La limite CPI de comptes uniques n'est notamment pas transformée en règle artificielle sur la liste d'account metas.
Ces valeurs sont des **bornes d'admission Interface**. Elles ne constituent pas une garantie qu'une instruction donnée respecte à elle seule toutes les contraintes d'une transaction Solana complète.
Un dépassement utilise le code commun :
@@ -89,21 +98,74 @@ Le contexte d'erreur est limité aux métadonnées sûres `field`, `actual_len`
Le `Debug` de `ProgramInstruction` est volontairement borné : il affiche `program_id`, `account_count` et `data_len`, jamais les accounts complets ni les octets du payload.
## Faits passifs d'acquisition
Les événements Interface sont des **projections provider-neutral supplémentaires**. Ils ne remplacent jamais les DTOs riches de `ksp-onchain-transport-lib` et ne deviennent jamais la source de vérité durable d'un backlog.
La conversion depuis un DTO HTTP/WS/gRPC/provider appartient à la composition ou au consumer qui connaît les deux contrats. `ksp-interface-lib` ne dépend donc pas de Transport.
### `SlotLifecycleEvent`
`SlotLifecycleEvent` conserve exactement :
```text
slot: u64
stage: SlotLifecycleStage
```
Les stages actuellement représentés sont :
```text
Processed
FirstShredReceived
Completed
CreatedBank
Dead
OptimisticallyConfirmed
Rooted
```
`SlotLifecycleStage` est `#[non_exhaustive]` afin qu'un consumer externe traite explicitement l'évolution future de l'enum.
Parent, timestamp, diagnostics de slot mort, source/provider, filter/session metadata et autres détails Transport ne sont pas copiés dans ce contrat minimal.
### `TransactionExecutionEvent`
`TransactionExecutionEvent` conserve exactement :
```text
slot: u64
signature: TransactionSignature
outcome: TransactionExecutionOutcome
```
`TransactionSignature` contient exactement les 64 bytes canoniques déjà décodés d'une signature Solana. Son `Debug` ne rend pas les bytes.
`TransactionExecutionOutcome` distingue uniquement :
```text
Succeeded
Failed
```
L'enum est `#[non_exhaustive]`. Le contrat ne contient aucun log, payload RAW, erreur provider, commitment, provenance, network id, source metadata ni détail d'exécution arbitraire. Une source ambiguë ou insuffisante doit rester dans son owner Transport au lieu de forcer un événement Interface.
## Codecs et runtime
La foundation `0.2.13` n'ajoute aucun codec par réflexe :
La crate n'ajoute aucun codec ou runtime par réflexe :
```text
serde / serde_json absents
borsh absent
wincode absent
bincode absent
solana-instruction absent
borsh absent tant qu'aucun wire réel ne le requiert
wincode absent tant qu'aucun wire réel ne le requiert
bincode interdit pour les codecs wire KSP
transport/runtime réseau absent
logging runtime absent
```
Des codecs/layouts/discriminants spécifiques pourront être ajoutés ultérieurement uniquement lorsqu'un vertical Program réel en démontre le besoin et que leur ownership wire appartient bien à Interface.
Des codecs/layouts/discriminants spécifiques peuvent être ajoutés uniquement lorsqu'un protocole réel en démontre le besoin et que leur ownership wire appartient bien à Interface.
La crate ne produit aucun événement runtime. Elle ne dépend donc pas de `ksp-logging-lib` et ne possède ni `constants.rs` ni `TRACING_TARGET`. Si un futur comportement Interface exige réellement du logging, le runtime devra passer par la façade Logging KSP plutôt que par une dépendance directe à Tracing.
Les événements passifs ne constituent pas un comportement runtime. La crate ne dépend donc pas de `ksp-logging-lib` et ne possède ni `constants.rs` ni `TRACING_TARGET`.
## Frontières
@@ -112,20 +174,20 @@ La crate ne produit aucun événement runtime. Elle ne dépend donc pas de `ksp-
```text
RPC / WebSocket / gRPC
provider DTOs Transport
wallet / signature
sessions / reconnect / backpressure
Config / environnement
persistence / Store
persistence / Store / cursor / retention
notifications post-commit Store
worker / job / scheduler / event bus
Program decoding / recognition / proofs
execution policy / signers
transaction replay / CPI path / runtime logs
lifecycle réseau
RawTransaction / RawAccountState
```
La foundation Program API est reportée à `0.2.14`. Les wires génériques d'acquisition/CORE restent reportés à `0.3.2+`.
Les données persistantes/replayables restent dans `ksp-store-api`. Les notifications post-commit de données persistées restent un contrat Store API lorsqu'un publisher/consumer réel les justifie. Les faits réseau riches restent Transport-owned.
## Références
- [Usage public](USAGE.md)
- [Plan `0.2.13`](../../docs/plans/020-V0_2_13_INTERFACE_PLAN.md)
- [Validation `0.2.13`](../../docs/validation/016-V0_2_13_INTERFACE.md)
- [Architecture Wire + Program](../../docs/architecture/006-WIRE_AND_PROGRAM.md)
- [Architecture Acquisition/Workers/Jobs](../../docs/architecture/009-ACQUISITION_WORKERS_AND_JOBS.md)

View File

@@ -1,9 +1,9 @@
<!-- file: crates/ksp-interface-lib/USAGE.md -->
<!-- version: 2 -->
<!-- version: 3 -->
# Usage de ksp-interface-lib
Cette page décrit la façade publique matérialisée par `0.2.13`. Les modules internes ne font pas partie du contrat consommable : utiliser uniquement les exports du crate-root.
Cette page décrit l'utilisation durable de la façade publique. Les modules internes ne font pas partie du contrat consommable : utiliser uniquement les exports du crate-root.
## Construire des account metas
@@ -52,9 +52,9 @@ match result {
L'ordre et les doublons des accounts sont conservés. Les octets `data` restent opaques : `ProgramInstruction` ne les sérialise, désérialise ni interprète.
Les `Vec` fournis à `try_new` sont consommés par la structure après validation des bornes ; aucun clone ou reformatage interne n'est requis par le contrat actuel.
Les `Vec` fournis à `try_new` sont consommés par la structure après validation des bornes ; aucun clone ou reformatage interne n'est requis par le contrat.
## Bornes
## Respecter les bornes Program
Les limites publiques sont :
@@ -63,9 +63,9 @@ assert_eq!(ksp_interface_lib::MAX_PROGRAM_INSTRUCTION_ACCOUNTS, 255);
assert_eq!(ksp_interface_lib::MAX_PROGRAM_INSTRUCTION_DATA_LEN, 10_240);
```
`255` account metas et `10_240` bytes de data sont admis. `256` account metas ou `10_241` bytes sont refusés par `try_new` avant création d'un `ProgramInstruction` valide.
`255` account metas et `10_240` bytes de data sont admis. `256` account metas ou `10_241` bytes sont refusés par `ProgramInstruction::try_new`.
Ces limites sont des bornes locales Interface et ne promettent pas qu'une instruction admise respecte à elle seule toutes les contraintes de taille/account-set d'une transaction Solana complète.
Ces limites sont locales à Interface et ne promettent pas qu'une instruction admise respecte à elle seule toutes les contraintes d'une transaction Solana complète.
## Observer une erreur de limite
@@ -92,31 +92,108 @@ maximum_len
Le contenu du payload et les account metas arbitraires ne sont pas projetés dans le diagnostic.
## Debug borné
## Construire un événement de lifecycle de slot
Le `Debug` de `ProgramInstruction` contient uniquement :
Un producer/composer qui a déjà établi la correspondance sémantique avec son DTO Transport peut construire le fait passif partagé :
```text
program_id
account_count
data_len
```rust
let event = ksp_interface_lib::SlotLifecycleEvent::new(
42,
ksp_interface_lib::SlotLifecycleStage::Processed,
);
assert_eq!(event.slot(), 42);
assert_eq!(event.stage(), ksp_interface_lib::SlotLifecycleStage::Processed);
```
Il ne faut donc pas attendre de ce rendu une sérialisation wire ou un dump du payload.
Les stages actuellement disponibles sont :
```text
Processed
FirstShredReceived
Completed
CreatedBank
Dead
OptimisticallyConfirmed
Rooted
```
Un consumer externe doit traiter `SlotLifecycleStage` comme une enum évolutive `#[non_exhaustive]` et prévoir un fallback dans ses `match`.
Le type ne contient volontairement ni timestamp, ni parent, ni diagnostic, ni source/provider, ni identifiant de subscription.
## Construire un événement d'exécution de transaction
Une signature doit d'abord être disponible sous sa forme canonique de 64 bytes :
```rust
let signature = ksp_interface_lib::TransactionSignature::new([7_u8; 64]);
let event = ksp_interface_lib::TransactionExecutionEvent::new(
123,
signature,
ksp_interface_lib::TransactionExecutionOutcome::Succeeded,
);
assert_eq!(event.slot(), 123);
assert_eq!(event.signature().as_bytes(), &[7_u8; 64]);
assert_eq!(
event.outcome(),
ksp_interface_lib::TransactionExecutionOutcome::Succeeded,
);
```
`TransactionExecutionOutcome` distingue uniquement `Succeeded` et `Failed` et reste `#[non_exhaustive]`.
Le `Debug` de `TransactionSignature` et de `TransactionExecutionEvent` n'affiche pas les bytes de signature.
## Convertir depuis Transport
Ne pas ajouter `ksp-onchain-transport-lib` comme dépendance de `ksp-interface-lib` pour fournir des `From<TransportDto>`.
La conversion appartient au composant qui connaît les deux côtés :
```text
Transport DTO riche
|
| conversion explicite dans composition/consumer
v
Interface event passif minimal
```
Ne construire un événement Interface que si la source fournit suffisamment d'information pour le fait commun exact. Si l'état est ambigu, conserver le DTO dans son owner Transport ou déclencher une hydratation adaptée ; ne pas inventer de valeur par défaut.
## Ne pas utiliser les événements comme stockage durable
`SlotLifecycleEvent` et `TransactionExecutionEvent` sont des faits passifs, pas des modèles RAW replayables.
Ils ne remplacent pas :
```text
RawTransaction
RawTransactionObservation
RawAccountState
RawAccountObservation
Store backlog / cursor / retention
```
Les consumers qui ont besoin de reprise après crash ou de replay doivent s'appuyer sur `ksp-store-lib`/`ksp-store-api` selon leur responsabilité, pas sur un event Interface en mémoire.
## Dépendances à ne pas ajouter côté consumer
Un consumer de la façade Interface n'a pas besoin d'ajouter un SDK Program Solana uniquement pour reconstruire `ProgramInstruction`. La crate utilise le `Pubkey` canonique partagé avec Core et conserve son propre contrat passif.
Un consumer de la façade Interface n'a pas besoin d'ajouter un SDK Program Solana uniquement pour reconstruire `ProgramInstruction`, ni un runtime Transport pour manipuler les événements passifs déjà normalisés.
La foundation ne fournit volontairement pas :
La crate ne fournit volontairement pas :
```text
serde générique
Borsh / Wincode générique
solana-instruction interop automatique
transport réseau
converters provider automatiques
Program decoder/preparer
signing/execution
persistence/event bus
```
Ces surfaces doivent être introduites dans leur owner respectif lorsqu'un cas réel le justifie, pas comme dépendances implicites d'un consumer Interface.
Ces surfaces doivent être introduites dans leur owner respectif lorsqu'un cas réel le justifie.

View File

@@ -1,20 +1,22 @@
// file: crates/ksp-interface-lib/src/lib.rs
// version: 3
// version: 5
#![warn(missing_docs)]
#![deny(unreachable_pub)]
#![forbid(unsafe_code)]
//! Passive wire contracts shared by KSP Program implementations.
//! Passive contracts shared across KSP component boundaries.
//!
//! The foundation reuses the canonical Solana [`Pubkey`] owned by
//! `ksp-core-lib` and exposes only bounded, passive Program-facing structures.
//! Runtime, transport, persistence and Program behavior remain outside this
//! crate.
//! The crate reuses canonical Solana primitives owned by `ksp-core-lib` and
//! exposes only bounded, passive Program-facing and provider-neutral
//! acquisition structures. Runtime, transport, persistence and Program
//! behavior remain outside this crate.
mod error;
mod program_account_meta;
mod program_instruction;
mod slot_lifecycle;
mod transaction_execution;
/// Error code used when an Interface-owned Program instruction admission limit is exceeded.
pub use self::error::ERROR_CODE_PROGRAM_INSTRUCTION_LIMIT_EXCEEDED;
@@ -26,5 +28,15 @@ pub use self::program_account_meta::ProgramAccountMeta;
pub use self::program_instruction::MAX_PROGRAM_INSTRUCTION_DATA_LEN;
/// Passive, bounded Program instruction wire contract.
pub use self::program_instruction::ProgramInstruction;
/// Passive provider-neutral occurrence of one slot lifecycle stage.
pub use self::slot_lifecycle::SlotLifecycleEvent;
/// Provider-neutral stage in the lifecycle of an observed Solana slot.
pub use self::slot_lifecycle::SlotLifecycleStage;
/// Passive provider-neutral observation of one transaction execution result.
pub use self::transaction_execution::TransactionExecutionEvent;
/// Provider-neutral outcome of one observed transaction execution.
pub use self::transaction_execution::TransactionExecutionOutcome;
/// Canonical 64-byte Solana transaction signature used by passive Interface events.
pub use self::transaction_execution::TransactionSignature;
/// Canonical Solana account address primitive owned by `ksp-core-lib`.
pub use ksp_core_lib::Pubkey;

View File

@@ -0,0 +1,53 @@
// file: crates/ksp-interface-lib/src/slot_lifecycle.rs
// version: 1
/// Provider-neutral stage in the lifecycle of an observed Solana slot.
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
#[non_exhaustive]
pub enum SlotLifecycleStage {
/// The slot has been processed.
Processed,
/// The first shred for the slot has been received.
FirstShredReceived,
/// Slot ingestion has completed.
Completed,
/// A bank has been created for the slot.
CreatedBank,
/// The slot has been marked dead.
Dead,
/// The slot has reached optimistic confirmation.
OptimisticallyConfirmed,
/// The slot has become rooted.
Rooted,
}
/// Passive provider-neutral occurrence of one slot lifecycle stage.
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub struct SlotLifecycleEvent {
slot: u64,
stage: SlotLifecycleStage,
}
impl SlotLifecycleEvent {
/// Creates a lifecycle event for `slot` and `stage`.
#[must_use]
pub const fn new(slot: u64, stage: SlotLifecycleStage) -> Self {
return Self { slot, stage };
}
/// Returns the observed slot exactly as supplied.
#[must_use]
pub const fn slot(&self) -> u64 {
return self.slot;
}
/// Returns the provider-neutral lifecycle stage.
#[must_use]
pub const fn stage(&self) -> SlotLifecycleStage {
return self.stage;
}
}
#[cfg(test)]
#[path = "../unit_tests/slot_lifecycle.rs"]
mod tests;

View File

@@ -0,0 +1,85 @@
// file: crates/ksp-interface-lib/src/transaction_execution.rs
// version: 1
/// Canonical 64-byte Solana transaction signature used by passive Interface events.
#[derive(Clone, Copy, Eq, Hash, Ord, PartialEq, PartialOrd)]
pub struct TransactionSignature([u8; 64]);
impl TransactionSignature {
/// Creates one transaction signature from already-decoded canonical bytes.
#[must_use]
pub const fn new(bytes: [u8; 64]) -> Self {
return Self(bytes);
}
/// Returns the exact canonical signature bytes.
#[must_use]
pub const fn as_bytes(&self) -> &[u8; 64] {
return &self.0;
}
}
impl std::fmt::Debug for TransactionSignature {
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
return formatter.write_str("TransactionSignature(..)");
}
}
/// Provider-neutral outcome of one observed transaction execution.
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
#[non_exhaustive]
pub enum TransactionExecutionOutcome {
/// The observed transaction execution completed without a transaction error.
Succeeded,
/// The observed transaction execution completed with a transaction error.
Failed,
}
/// Passive provider-neutral observation of one transaction execution result.
#[derive(Clone, Copy, Eq, PartialEq)]
pub struct TransactionExecutionEvent {
slot: u64,
signature: TransactionSignature,
outcome: TransactionExecutionOutcome,
}
impl TransactionExecutionEvent {
/// Creates one transaction execution event from its common provider-neutral fields.
#[must_use]
pub const fn new(slot: u64, signature: TransactionSignature, outcome: TransactionExecutionOutcome) -> Self {
return Self { slot, signature, outcome };
}
/// Returns the containing slot exactly as observed by the producer.
#[must_use]
pub const fn slot(&self) -> u64 {
return self.slot;
}
/// Returns the canonical transaction signature.
#[must_use]
pub const fn signature(&self) -> TransactionSignature {
return self.signature;
}
/// Returns the normalized execution outcome.
#[must_use]
pub const fn outcome(&self) -> TransactionExecutionOutcome {
return self.outcome;
}
}
impl std::fmt::Debug for TransactionExecutionEvent {
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
return formatter
.debug_struct("TransactionExecutionEvent")
.field("slot", &self.slot)
.field("signature", &"<omitted>")
.field("outcome", &self.outcome)
.finish();
}
}
#[cfg(test)]
#[path = "../unit_tests/transaction_execution.rs"]
mod tests;

View File

@@ -1,5 +1,5 @@
// file: crates/ksp-interface-lib/tests/dependency_boundary.rs
// version: 5
// version: 8
//! Dependency and passive-surface canaries for the Interface foundation.
@@ -91,6 +91,8 @@ fn pre_005_all_production_sources_preserve_the_dependency_firewall() {
include_str!("../src/lib.rs"),
include_str!("../src/program_account_meta.rs"),
include_str!("../src/program_instruction.rs"),
include_str!("../src/slot_lifecycle.rs"),
include_str!("../src/transaction_execution.rs"),
];
for source in production_sources {
for forbidden in [
@@ -129,3 +131,52 @@ fn pre_005_instruction_source_has_no_narrowing_cast_or_hidden_codec_entry_point(
}
return;
}
#[test]
fn v0_3_5_pre_004_acquisition_sources_remain_provider_neutral_and_runtime_free() {
let acquisition_sources = [include_str!("../src/slot_lifecycle.rs"), include_str!("../src/transaction_execution.rs")];
for source in acquisition_sources {
for forbidden in [
"borsh::",
"bincode::",
"ksp_config_lib::",
"ksp_logging_lib::",
"ksp_onchain_transport_lib::",
"ksp_store_api::",
"ksp_store_lib::",
"reqwest::",
"serde::",
"serde_json::",
"solana_instruction::",
"tauri::",
"tokio::",
"tonic::",
concat!("tracing", "::"),
"wincode::",
] {
assert!(!source.contains(forbidden), "forbidden acquisition dependency path detected: {forbidden}");
}
}
return;
}
#[test]
fn v0_3_5_pre_005_manifest_keeps_exact_core_only_graph_without_features_or_hidden_dependency_sections() {
let manifest = include_str!("../Cargo.toml");
assert!(!manifest.contains("[features]"));
assert!(!manifest.contains("[dev-dependencies]"));
assert!(!manifest.contains("[build-dependencies]"));
assert_eq!(manifest.matches("[dependencies]").count(), 1);
let dependencies_tail = manifest.split("[dependencies]").nth(1);
assert!(dependencies_tail.is_some(), "Interface dependencies section must exist");
let dependencies_tail = match dependencies_tail {
std::option::Option::Some(value) => value,
std::option::Option::None => return,
};
let dependencies = match dependencies_tail.split("[lints]").next() {
std::option::Option::Some(value) => value,
std::option::Option::None => return,
};
assert_eq!(manifest_dependency_names(dependencies), std::vec!["ksp-core-lib"]);
return;
}

View File

@@ -1,12 +1,22 @@
// file: crates/ksp-interface-lib/tests/external_consumer.rs
// version: 1
// version: 2
//! Downstream-style consumer canary for the public Interface facade.
//! Downstream-style consumer canaries for the public Interface facade.
fn consume_instruction(instruction: ksp_interface_lib::ProgramInstruction) -> (ksp_interface_lib::Pubkey, usize, usize) {
return (*instruction.program_id(), instruction.accounts().len(), instruction.data().len());
}
fn consume_slot_lifecycle(event: ksp_interface_lib::SlotLifecycleEvent) -> (u64, ksp_interface_lib::SlotLifecycleStage) {
return (event.slot(), event.stage());
}
fn consume_transaction_execution(
event: ksp_interface_lib::TransactionExecutionEvent,
) -> (u64, ksp_interface_lib::TransactionSignature, ksp_interface_lib::TransactionExecutionOutcome) {
return (event.slot(), event.signature(), event.outcome());
}
#[test]
fn pre_005_external_consumer_uses_only_the_crate_root_facade() {
let program_id = ksp_interface_lib::Pubkey::new_from_array([0xA1_u8; 32]);
@@ -25,3 +35,13 @@ fn pre_005_external_consumer_uses_only_the_crate_root_facade() {
assert_eq!(ksp_interface_lib::ERROR_CODE_PROGRAM_INSTRUCTION_LIMIT_EXCEEDED.domain(), "interface");
return;
}
#[test]
fn v0_3_5_pre_004_external_consumer_uses_only_crate_root_for_both_acquisition_families() {
let lifecycle = ksp_interface_lib::SlotLifecycleEvent::new(u64::MAX, ksp_interface_lib::SlotLifecycleStage::OptimisticallyConfirmed);
assert_eq!(consume_slot_lifecycle(lifecycle), (u64::MAX, ksp_interface_lib::SlotLifecycleStage::OptimisticallyConfirmed));
let signature = ksp_interface_lib::TransactionSignature::new([0xA3_u8; 64]);
let execution = ksp_interface_lib::TransactionExecutionEvent::new(u64::MAX - 1, signature, ksp_interface_lib::TransactionExecutionOutcome::Failed);
assert_eq!(consume_transaction_execution(execution), (u64::MAX - 1, signature, ksp_interface_lib::TransactionExecutionOutcome::Failed));
return;
}

View File

@@ -1,75 +1,75 @@
// file: crates/ksp-interface-lib/tests/release_completeness.rs
// version: 1
// version: 5
//! Release-level completeness canaries for the `0.2.13` Interface foundation.
//! Release-completeness canaries for the public `ksp-interface-lib` surface.
#[test]
fn pre_005_exact_crate_root_export_inventory_is_stable() {
let crate_root = include_str!("../src/lib.rs");
let mut actual = std::vec::Vec::new();
for line in crate_root.lines() {
let trimmed = line.trim();
if trimmed.starts_with("pub use ") {
actual.push(trimmed);
}
}
actual.sort_unstable();
let mut expected = std::vec![
"pub use self::error::ERROR_CODE_PROGRAM_INSTRUCTION_LIMIT_EXCEEDED;",
"pub use self::program_account_meta::MAX_PROGRAM_INSTRUCTION_ACCOUNTS;",
"pub use self::program_account_meta::ProgramAccountMeta;",
"pub use self::program_instruction::MAX_PROGRAM_INSTRUCTION_DATA_LEN;",
"pub use self::program_instruction::ProgramInstruction;",
"pub use ksp_core_lib::Pubkey;",
];
expected.sort_unstable();
assert_eq!(actual, expected);
assert!(!crate_root.contains("pub mod "));
return;
fn crate_root_source() -> std::result::Result<std::string::String, std::io::Error> {
let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("src/lib.rs");
return std::fs::read_to_string(path);
}
#[test]
fn pre_005_production_module_inventory_contains_no_second_wire_domain() -> std::io::Result<()> {
let source_root = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("src");
let entries = match std::fs::read_dir(source_root) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let mut names = std::vec::Vec::new();
for entry in entries {
let entry = match entry {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let file_type = match entry.file_type() {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
if !file_type.is_file() {
continue;
}
let name = match entry.file_name().into_string() {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => continue,
};
if name.ends_with(".rs") {
names.push(name);
}
}
names.sort_unstable();
assert_eq!(names, std::vec!["error.rs", "lib.rs", "program_account_meta.rs", "program_instruction.rs"]);
return std::result::Result::Ok(());
}
#[test]
fn pre_005_foundation_has_one_error_code_and_two_bounded_passive_types() {
fn v0_3_5_pre_004_foundation_keeps_exactly_two_passive_acquisition_families() {
assert_eq!(ksp_interface_lib::ERROR_CODE_PROGRAM_INSTRUCTION_LIMIT_EXCEEDED.domain(), "interface");
assert_eq!(ksp_interface_lib::ERROR_CODE_PROGRAM_INSTRUCTION_LIMIT_EXCEEDED.code(), "program_instruction_limit_exceeded");
assert_eq!(ksp_interface_lib::MAX_PROGRAM_INSTRUCTION_ACCOUNTS, 255);
assert_eq!(ksp_interface_lib::MAX_PROGRAM_INSTRUCTION_DATA_LEN, 10_240);
let program_id = ksp_interface_lib::Pubkey::new_from_array([0xB1_u8; 32]);
let account = ksp_interface_lib::ProgramAccountMeta::readonly(ksp_interface_lib::Pubkey::new_from_array([0xB2_u8; 32]), false);
let instruction = ksp_interface_lib::ProgramInstruction::try_new(program_id, std::vec![account], std::vec![0xB3_u8]);
assert!(instruction.is_ok());
let lifecycle = ksp_interface_lib::SlotLifecycleEvent::new(u64::MAX, ksp_interface_lib::SlotLifecycleStage::Rooted);
assert_eq!(lifecycle.slot(), u64::MAX);
assert_eq!(lifecycle.stage(), ksp_interface_lib::SlotLifecycleStage::Rooted);
let signature = ksp_interface_lib::TransactionSignature::new([0xB4_u8; 64]);
let execution = ksp_interface_lib::TransactionExecutionEvent::new(u64::MAX, signature, ksp_interface_lib::TransactionExecutionOutcome::Succeeded);
assert_eq!(execution.slot(), u64::MAX);
assert_eq!(execution.signature(), signature);
assert_eq!(execution.outcome(), ksp_interface_lib::TransactionExecutionOutcome::Succeeded);
return;
}
#[test]
fn v0_3_5_pre_004_exact_crate_root_export_inventory_is_stable() -> std::result::Result<(), std::io::Error> {
let source = match crate_root_source() {
Ok(source) => source,
Err(error) => return Err(error),
};
let public_use_count = source.lines().filter(|line| return line.starts_with("pub use ")).count();
assert_eq!(public_use_count, 11);
assert!(source.contains("pub use self::error::ERROR_CODE_PROGRAM_INSTRUCTION_LIMIT_EXCEEDED;"));
assert!(source.contains("pub use self::program_account_meta::MAX_PROGRAM_INSTRUCTION_ACCOUNTS;"));
assert!(source.contains("pub use self::program_account_meta::ProgramAccountMeta;"));
assert!(source.contains("pub use self::program_instruction::MAX_PROGRAM_INSTRUCTION_DATA_LEN;"));
assert!(source.contains("pub use self::program_instruction::ProgramInstruction;"));
assert!(source.contains("pub use self::slot_lifecycle::SlotLifecycleEvent;"));
assert!(source.contains("pub use self::slot_lifecycle::SlotLifecycleStage;"));
assert!(source.contains("pub use self::transaction_execution::TransactionExecutionEvent;"));
assert!(source.contains("pub use self::transaction_execution::TransactionExecutionOutcome;"));
assert!(source.contains("pub use self::transaction_execution::TransactionSignature;"));
assert!(source.contains("pub use ksp_core_lib::Pubkey;"));
return Ok(());
}
#[test]
fn v0_3_5_pre_004_production_module_inventory_is_exact() -> std::result::Result<(), std::io::Error> {
let source = match crate_root_source() {
Ok(source) => source,
Err(error) => return Err(error),
};
let modules = source
.lines()
.filter_map(|line| return line.strip_prefix("mod ").and_then(|module| return module.strip_suffix(';')))
.collect::<std::vec::Vec<_>>();
assert_eq!(modules, std::vec!["error", "program_account_meta", "program_instruction", "slot_lifecycle", "transaction_execution"]);
assert!(!source.contains("serde"));
assert!(!source.contains("tracing"));
assert!(!source.contains("ksp_onchain_transport"));
assert!(!source.contains("ksp_store"));
return Ok(());
}
#[test]
fn v0_3_5_pre_004_evolutive_acquisition_enums_remain_non_exhaustive() {
let lifecycle_source = include_str!("../src/slot_lifecycle.rs");
let execution_source = include_str!("../src/transaction_execution.rs");
assert!(lifecycle_source.contains("#[non_exhaustive]\npub enum SlotLifecycleStage"));
assert!(execution_source.contains("#[non_exhaustive]\npub enum TransactionExecutionOutcome"));
return;
}

View File

@@ -0,0 +1,100 @@
// file: crates/ksp-interface-lib/tests/security_hardening.rs
// version: 1
//! Adversarial and ownership-hardening canaries for passive Interface acquisition events.
const HOSTILE_MARKER: &str = "INTERFACE-ACQUISITION-SECRET-CANARY";
#[test]
fn v0_3_5_pre_005_signature_and_event_debug_never_render_signature_bytes() {
let mut bytes = [0x41_u8; 64];
bytes[..HOSTILE_MARKER.len()].copy_from_slice(HOSTILE_MARKER.as_bytes());
let signature = ksp_interface_lib::TransactionSignature::new(bytes);
assert_eq!(signature.as_bytes(), &bytes);
let signature_debug = std::format!("{signature:?}");
assert_eq!(signature_debug, "TransactionSignature(..)");
assert!(!signature_debug.contains(HOSTILE_MARKER));
let event = ksp_interface_lib::TransactionExecutionEvent::new(u64::MAX, signature, ksp_interface_lib::TransactionExecutionOutcome::Failed);
let event_debug = std::format!("{event:?}");
assert!(event_debug.contains("TransactionExecutionEvent"));
assert!(event_debug.contains("18446744073709551615"));
assert!(event_debug.contains("Failed"));
assert!(!event_debug.contains(HOSTILE_MARKER));
return;
}
#[test]
fn v0_3_5_pre_005_acquisition_structs_keep_exact_minimal_fields_without_payload_or_source_metadata() {
let lifecycle_source = include_str!("../src/slot_lifecycle.rs");
let execution_source = include_str!("../src/transaction_execution.rs");
assert!(lifecycle_source.contains("pub struct SlotLifecycleEvent {\n slot: u64,\n stage: SlotLifecycleStage,\n}"));
assert!(execution_source.contains("pub struct TransactionSignature([u8; 64]);"));
assert!(execution_source.contains(
"pub struct TransactionExecutionEvent {\n slot: u64,\n signature: TransactionSignature,\n outcome: TransactionExecutionOutcome,\n}"
));
for source in [lifecycle_source, execution_source] {
for forbidden in [
"std::vec::Vec<",
"std::string::String",
"Box<",
"serde_json::Value",
"timestamp:",
"provider_id:",
"endpoint:",
"filter:",
"network:",
"payload:",
] {
assert!(!source.contains(forbidden), "forbidden acquisition field/surface detected: {forbidden}");
}
}
return;
}
#[test]
fn v0_3_5_pre_005_interface_exports_no_second_raw_or_deferred_event_family() {
let crate_root = include_str!("../src/lib.rs");
for forbidden in [
"RawTransaction",
"RawAccount",
"RawPayload",
"RawPageCursor",
"RawRetention",
"RawObservation",
"TransactionLogEvent",
"TransactionCommitmentEvent",
"LedgerEntryEvent",
"VoteEvent",
"BlockEvent",
] {
assert!(!crate_root.contains(forbidden), "forbidden Interface export detected: {forbidden}");
}
return;
}
#[test]
fn v0_3_5_pre_005_program_instruction_foundation_remains_bounded_and_unchanged() {
assert_eq!(ksp_interface_lib::MAX_PROGRAM_INSTRUCTION_ACCOUNTS, 255);
assert_eq!(ksp_interface_lib::MAX_PROGRAM_INSTRUCTION_DATA_LEN, 10_240);
let program_id = ksp_interface_lib::Pubkey::new_from_array([0xD1_u8; 32]);
let account = ksp_interface_lib::ProgramAccountMeta::readonly(ksp_interface_lib::Pubkey::new_from_array([0xD2_u8; 32]), false);
let admitted = ksp_interface_lib::ProgramInstruction::try_new(
program_id,
std::vec![account; ksp_interface_lib::MAX_PROGRAM_INSTRUCTION_ACCOUNTS],
std::vec![0x5A_u8; ksp_interface_lib::MAX_PROGRAM_INSTRUCTION_DATA_LEN],
);
assert!(admitted.is_ok());
let admitted = match admitted {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return,
};
assert_eq!(admitted.accounts().len(), ksp_interface_lib::MAX_PROGRAM_INSTRUCTION_ACCOUNTS);
assert_eq!(admitted.data().len(), ksp_interface_lib::MAX_PROGRAM_INSTRUCTION_DATA_LEN);
let too_many_accounts =
ksp_interface_lib::ProgramInstruction::try_new(program_id, std::vec![account; ksp_interface_lib::MAX_PROGRAM_INSTRUCTION_ACCOUNTS + 1], std::vec![]);
assert!(too_many_accounts.is_err());
let oversized_data =
ksp_interface_lib::ProgramInstruction::try_new(program_id, std::vec![], std::vec![0x5A_u8; ksp_interface_lib::MAX_PROGRAM_INSTRUCTION_DATA_LEN + 1]);
assert!(oversized_data.is_err());
return;
}

View File

@@ -0,0 +1,29 @@
// file: crates/ksp-interface-lib/tests/slot_lifecycle_public_api.rs
// version: 2
//! Public API canaries for the shared slot lifecycle event contract.
#[test]
fn public_v0_3_5_pre_002_slot_lifecycle_contract_is_available_from_crate_root() {
let event = ksp_interface_lib::SlotLifecycleEvent::new(u64::MAX, ksp_interface_lib::SlotLifecycleStage::Rooted);
assert_eq!(event.slot(), u64::MAX);
assert_eq!(event.stage(), ksp_interface_lib::SlotLifecycleStage::Rooted);
}
#[test]
fn public_v0_3_5_pre_002_slot_lifecycle_stage_remains_downstream_evolvable() {
fn stage_label(stage: ksp_interface_lib::SlotLifecycleStage) -> &'static str {
return match stage {
ksp_interface_lib::SlotLifecycleStage::Processed => "processed",
ksp_interface_lib::SlotLifecycleStage::FirstShredReceived => "first_shred_received",
ksp_interface_lib::SlotLifecycleStage::Completed => "completed",
ksp_interface_lib::SlotLifecycleStage::CreatedBank => "created_bank",
ksp_interface_lib::SlotLifecycleStage::Dead => "dead",
ksp_interface_lib::SlotLifecycleStage::OptimisticallyConfirmed => "optimistically_confirmed",
ksp_interface_lib::SlotLifecycleStage::Rooted => "rooted",
_ => "future",
};
}
assert_eq!(stage_label(ksp_interface_lib::SlotLifecycleStage::Processed), "processed");
assert_eq!(stage_label(ksp_interface_lib::SlotLifecycleStage::Rooted), "rooted");
}

View File

@@ -0,0 +1,28 @@
// file: crates/ksp-interface-lib/tests/transaction_execution_public_api.rs
// version: 1
//! Public API canaries for the provider-neutral transaction execution contract.
#[test]
fn public_v0_3_5_pre_003_transaction_execution_contract_is_available_from_crate_root() {
let signature = ksp_interface_lib::TransactionSignature::new([0xC7_u8; 64]);
let event = ksp_interface_lib::TransactionExecutionEvent::new(u64::MAX, signature, ksp_interface_lib::TransactionExecutionOutcome::Succeeded);
assert_eq!(signature.as_bytes(), &[0xC7_u8; 64]);
assert_eq!(event.slot(), u64::MAX);
assert_eq!(event.signature(), signature);
assert_eq!(event.outcome(), ksp_interface_lib::TransactionExecutionOutcome::Succeeded);
}
fn outcome_label(outcome: ksp_interface_lib::TransactionExecutionOutcome) -> &'static str {
return match outcome {
ksp_interface_lib::TransactionExecutionOutcome::Succeeded => "succeeded",
ksp_interface_lib::TransactionExecutionOutcome::Failed => "failed",
_ => "future",
};
}
#[test]
fn public_v0_3_5_pre_003_transaction_execution_outcome_remains_downstream_evolvable() {
assert_eq!(outcome_label(ksp_interface_lib::TransactionExecutionOutcome::Succeeded), "succeeded");
assert_eq!(outcome_label(ksp_interface_lib::TransactionExecutionOutcome::Failed), "failed");
}

View File

@@ -0,0 +1,45 @@
// file: crates/ksp-interface-lib/unit_tests/slot_lifecycle.rs
// version: 2
#[test]
fn slot_lifecycle_stages_are_distinct_copy_and_complete_for_the_admitted_family() {
fn assert_copy<T: Copy>() {}
assert_copy::<crate::SlotLifecycleStage>();
let stages = [
crate::SlotLifecycleStage::Processed,
crate::SlotLifecycleStage::FirstShredReceived,
crate::SlotLifecycleStage::Completed,
crate::SlotLifecycleStage::CreatedBank,
crate::SlotLifecycleStage::Dead,
crate::SlotLifecycleStage::OptimisticallyConfirmed,
crate::SlotLifecycleStage::Rooted,
];
for (index, stage) in stages.iter().enumerate() {
for other in stages.iter().skip(index + 1) {
assert_ne!(stage, other);
}
}
}
#[test]
fn slot_lifecycle_event_preserves_full_u64_slot_and_stage() {
fn assert_copy<T: Copy>() {}
assert_copy::<crate::SlotLifecycleEvent>();
let event = crate::SlotLifecycleEvent::new(u64::MAX, crate::SlotLifecycleStage::Rooted);
assert_eq!(event.slot(), u64::MAX);
assert_eq!(event.stage(), crate::SlotLifecycleStage::Rooted);
let copied = event;
assert_eq!(event, copied);
}
#[test]
fn slot_lifecycle_debug_is_bounded_and_contains_only_shared_fields() {
let event = crate::SlotLifecycleEvent::new(42, crate::SlotLifecycleStage::OptimisticallyConfirmed);
let debug = std::format!("{event:?}");
assert!(debug.len() <= 128);
assert!(debug.contains("slot: 42"));
assert!(debug.contains("OptimisticallyConfirmed"));
assert!(!debug.contains("provider"));
assert!(!debug.contains("yellowstone"));
assert!(!debug.contains("websocket"));
}

View File

@@ -0,0 +1,28 @@
// file: crates/ksp-interface-lib/unit_tests/transaction_execution.rs
// version: 1
#[test]
fn transaction_signature_preserves_exact_bytes_and_redacts_debug() {
let signature = crate::TransactionSignature::new([0xA5_u8; 64]);
assert_eq!(signature.as_bytes(), &[0xA5_u8; 64]);
assert_eq!(std::format!("{signature:?}"), "TransactionSignature(..)");
}
#[test]
fn transaction_execution_outcomes_are_distinct_and_copy() {
let succeeded = crate::TransactionExecutionOutcome::Succeeded;
let failed = crate::TransactionExecutionOutcome::Failed;
assert_ne!(succeeded, failed);
let copied = succeeded;
assert_eq!(copied, crate::TransactionExecutionOutcome::Succeeded);
}
#[test]
fn transaction_execution_event_preserves_full_slot_signature_and_outcome() {
let signature = crate::TransactionSignature::new([0xB6_u8; 64]);
let event = crate::TransactionExecutionEvent::new(u64::MAX, signature, crate::TransactionExecutionOutcome::Failed);
assert_eq!(event.slot(), u64::MAX);
assert_eq!(event.signature(), signature);
assert_eq!(event.outcome(), crate::TransactionExecutionOutcome::Failed);
assert_eq!(std::format!("{event:?}"), "TransactionExecutionEvent { slot: 18446744073709551615, signature: \"<omitted>\", outcome: Failed }");
}

View File

@@ -0,0 +1,14 @@
# file: crates/ksp-job-api/Cargo.toml
# version: 1
[package]
name = "ksp-job-api"
version.workspace = true
edition.workspace = true
repository.workspace = true
[dependencies]
ksp-core-lib = { path = "../ksp-core-lib" }
[lints]
workspace = true

View File

@@ -0,0 +1,72 @@
<!-- file: crates/ksp-job-api/README.md -->
<!-- version: 1 -->
# ksp-job-api
`ksp-job-api` fournit les contrats passifs et runtime-neutral communs aux jobs bornés de KSP.
La crate possède l'identité logique d'un job, son lifecycle terminable, l'intention d'annulation coopérative et le contrat latest-value utilisé pour observer un snapshot complet. Elle ne possède aucun runtime concret, aucune politique métier de backfill, aucun Transport, aucun Store et aucun Worker.
## Responsabilités
La façade crate-root expose :
- `JobId` et `JobKindCode`, bornés et validés ;
- `JobState` et `JobCompletion` ;
- `JobLifecycle`, propriétaire des transitions admises ;
- `JobCancellationToken`, cloneable et idempotent ;
- `JobNotificationSequence`, strictement monotone et sans wrap silencieux ;
- `JobNotification<S>`, valeur observable complète à une position donnée ;
- `JobSnapshotSource`, contrat runtime-neutral de lecture courante et attente d'une valeur plus récente ;
- les codes d'erreur Job et les types `Error`/`Result` communs de Core.
## Lifecycle
Le lifecycle admis reste explicitement borné :
```text
Created
| | +----------------------> Cancelled
v
Running
| | +----> Cancelling -----> Cancelled
| | | | +---------> Failed
| +-------------> Completed(Complete|Partial)
|
+------------------------> Completed(Complete|Partial)
+------------------------> Failed
```
Tout état terminal est immuable. Une transition invalide retourne `ERROR_CODE_JOB_TRANSITION_INVALID` sans modifier l'état source.
## Observation latest-value
`JobSnapshotSource` n'impose ni callback, ni queue d'événements, ni runtime async particulier. Un listener peut :
1. lire la valeur complète courante ;
2. mémoriser sa `JobNotificationSequence` ;
3. attendre une valeur plus récente ;
4. recevoir directement la dernière valeur complète, même si plusieurs mises à jour intermédiaires ont été coalescées.
Le snapshot concret appartient au job qui implémente la source. `ksp-job-api` ne connaît pas son contenu.
## Annulation
`JobCancellationToken` représente uniquement une intention coopérative partagée. Il ne tue pas une tâche, n'annule pas une I/O par lui-même et ne décide pas du résultat terminal. Le runtime concret reste responsable d'observer le token aux frontières sûres et de publier son état final.
## Firewall
La dépendance normale est volontairement minimale :
```text
ksp-job-api
-> ksp-core-lib
```
La crate ne dépend pas de Tokio, Futures, Transport, Store, Config, Logging, serde ni d'une crate de job concret.
## Documentation
- [`USAGE.md`](USAGE.md) — utilisation durable des contrats Job ;
- [`../../docs/architecture/003-COMPONENT_CONTRACTS.md`](../../docs/architecture/003-COMPONENT_CONTRACTS.md) — contrats de composants ;
- [`../../docs/architecture/009-ACQUISITION_WORKERS_AND_JOBS.md`](../../docs/architecture/009-ACQUISITION_WORKERS_AND_JOBS.md) — séparation workers/jobs et premier backfill RAW.

141
crates/ksp-job-api/USAGE.md Normal file
View File

@@ -0,0 +1,141 @@
<!-- file: crates/ksp-job-api/USAGE.md -->
<!-- version: 1 -->
# Utilisation de ksp-job-api
Cette page décrit la façade publique durable de `ksp-job-api`. Les consumers utilisent uniquement les exports du crate-root.
## Construire une identité de job
```rust
fn job_identity() -> ksp_job_api::Result<(ksp_job_api::JobId, ksp_job_api::JobKindCode)> {
let id = match ksp_job_api::JobId::new("raw-backfill-mainnet-0001") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let kind = match ksp_job_api::JobKindCode::new("raw_transaction_backfill") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
return std::result::Result::Ok((id, kind));
}
```
`JobId` identifie un job logique et ses reprises contrôlées. `JobKindCode` identifie une famille de jobs. Les deux sont bornés et utilisent un alphabet sûr ; leur `Debug` n'est pas une surface destinée à transporter un payload métier.
## Piloter un lifecycle passif
```rust
fn lifecycle(id: ksp_job_api::JobId, kind: ksp_job_api::JobKindCode) -> ksp_job_api::Result<ksp_job_api::JobLifecycle> {
let mut lifecycle = ksp_job_api::JobLifecycle::new(id, kind);
if let std::result::Result::Err(error) = lifecycle.start() {
return std::result::Result::Err(error);
}
if let std::result::Result::Err(error) = lifecycle.complete(ksp_job_api::JobCompletion::Complete) {
return std::result::Result::Err(error);
}
return std::result::Result::Ok(lifecycle);
}
```
Le producer ne doit pas forcer un état directement. Il utilise les opérations de `JobLifecycle`, qui refusent les transitions hors matrice.
Pour une annulation observée pendant l'exécution :
```rust
if let std::result::Result::Err(error) = lifecycle.mark_cancelling() {
return std::result::Result::Err(error);
}
if let std::result::Result::Err(error) = lifecycle.mark_cancelled() {
return std::result::Result::Err(error);
}
```
## Partager une intention d'annulation
```rust
let token = ksp_job_api::JobCancellationToken::new();
let listener = token.clone();
assert!(!listener.is_cancellation_requested());
assert!(token.request_cancellation());
assert!(listener.is_cancellation_requested());
assert!(!token.request_cancellation());
```
Le premier appel qui change l'état retourne `true`. Les demandes suivantes sont idempotentes et retournent `false`.
Le token ne doit pas être interprété comme une primitive de kill : le runtime concret décide où l'annulation peut interrompre l'admission ou une attente et quelles opérations déjà engagées doivent être drainées.
## Publier une valeur latest-value
Un producer concret peut construire une notification complète :
```rust
#[derive(Clone)]
struct Snapshot {
completed: usize,
}
let id = ksp_job_api::JobId::new("job-0001")?;
let kind = ksp_job_api::JobKindCode::new("example")?;
let sequence = ksp_job_api::JobNotificationSequence::initial();
let notification = ksp_job_api::JobNotification::new(
id,
kind,
sequence,
ksp_job_api::JobState::Running,
Snapshot { completed: 0 },
);
assert_eq!(notification.sequence().value(), 0);
assert_eq!(notification.state(), ksp_job_api::JobState::Running);
```
Le `Debug` de `JobNotification<S>` masque volontairement le snapshot. Le type concret `S` doit lui-même rester sûr à exposer lorsque le consumer accède explicitement à `snapshot()`.
## Implémenter une source de snapshots
Une implémentation concrète possède son runtime et expose seulement le contrat `JobSnapshotSource` :
```rust
async fn observe<S>(source: &S)
where
S: ksp_job_api::JobSnapshotSource,
{
let current = source.current();
let observed = current.sequence();
let newer = source.wait_for_change(observed).await;
assert!(newer.sequence().is_after(observed));
}
```
`wait_for_change` retourne la dernière valeur complète connue après coalescence éventuelle. Un consumer ne doit donc pas supposer qu'il recevra chaque mise à jour intermédiaire.
## Faire avancer une séquence
```rust
let first = ksp_job_api::JobNotificationSequence::initial();
let second = match first.next() {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
assert!(second.is_after(first));
```
L'épuisement de `u64` est une erreur explicite ; la séquence ne wrappe jamais silencieusement.
## Frontières à respecter
Ne pas ajouter à `ksp-job-api` :
```text
Tokio/Futures runtime concret
Transport ou Store
Config/Logging
DTO métier d'un job précis
scheduler, worker ou control plane
persistence de checkpoint
```
Ces responsabilités appartiennent aux crates concrètes et à la composition supérieure.

View File

@@ -0,0 +1,44 @@
// file: crates/ksp-job-api/src/cancellation.rs
// version: 1
/// Runtime-neutral cloneable token carrying cooperative cancellation intent.
#[derive(Clone)]
pub struct JobCancellationToken {
requested: std::sync::Arc<std::sync::atomic::AtomicBool>,
}
impl JobCancellationToken {
/// Creates a token with no cancellation request.
#[must_use]
pub fn new() -> Self {
return Self { requested: std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)) };
}
/// Requests cancellation and returns `true` only for the first request shared by all clones.
#[must_use]
pub fn cancel(&self) -> bool {
return !self.requested.swap(true, std::sync::atomic::Ordering::AcqRel);
}
/// Reports whether cancellation has been requested through any clone.
#[must_use]
pub fn is_cancellation_requested(&self) -> bool {
return self.requested.load(std::sync::atomic::Ordering::Acquire);
}
}
impl std::default::Default for JobCancellationToken {
fn default() -> Self {
return Self::new();
}
}
impl std::fmt::Debug for JobCancellationToken {
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
return formatter.debug_struct("JobCancellationToken").field("cancellation_requested", &self.is_cancellation_requested()).finish();
}
}
#[cfg(test)]
#[path = "../unit_tests/cancellation.rs"]
mod tests;

View File

@@ -0,0 +1,12 @@
// file: crates/ksp-job-api/src/error.rs
// version: 2
/// Error code used when a Job identifier violates its bounded safe-code contract.
pub const ERROR_CODE_JOB_ID_INVALID: ksp_core_lib::ErrorCode = ksp_core_lib::ErrorCode::new("job_api", "job_id_invalid");
/// Error code used when a Job kind code violates its bounded safe-code contract.
pub const ERROR_CODE_JOB_KIND_INVALID: ksp_core_lib::ErrorCode = ksp_core_lib::ErrorCode::new("job_api", "job_kind_invalid");
/// Error code used when a Job notification sequence cannot advance without wrapping.
pub const ERROR_CODE_JOB_NOTIFICATION_SEQUENCE_EXHAUSTED: ksp_core_lib::ErrorCode =
ksp_core_lib::ErrorCode::new("job_api", "job_notification_sequence_exhausted");
/// Error code used when a requested Job lifecycle transition is not allowed.
pub const ERROR_CODE_JOB_TRANSITION_INVALID: ksp_core_lib::ErrorCode = ksp_core_lib::ErrorCode::new("job_api", "job_transition_invalid");

View File

@@ -0,0 +1,76 @@
// file: crates/ksp-job-api/src/identity.rs
// version: 1
/// Maximum UTF-8 byte length admitted for one Job identifier.
pub const MAX_JOB_ID_BYTES: usize = 128;
/// Maximum UTF-8 byte length admitted for one Job kind code.
pub const MAX_JOB_KIND_CODE_BYTES: usize = 128;
/// Bounded caller-supplied identity of one logical Job and its controlled resumptions.
#[derive(Clone, Eq, Hash, Ord, PartialEq, PartialOrd)]
pub struct JobId(std::string::String);
impl JobId {
/// Creates one non-empty Job identifier using the KSP safe-code alphabet.
pub fn new(value: impl std::convert::Into<std::string::String>) -> crate::Result<Self> {
let value = value.into();
if !valid_job_code(value.as_str(), crate::MAX_JOB_ID_BYTES) {
return std::result::Result::Err(identity_error(crate::ERROR_CODE_JOB_ID_INVALID, "job_id"));
}
return std::result::Result::Ok(Self(value));
}
/// Returns the validated Job identifier.
#[must_use]
pub fn as_str(&self) -> &str {
return self.0.as_str();
}
}
impl std::fmt::Debug for JobId {
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
return formatter.write_str("JobId(..)");
}
}
/// Bounded stable code identifying one concrete family of Jobs.
#[derive(Clone, Eq, Hash, Ord, PartialEq, PartialOrd)]
pub struct JobKindCode(std::string::String);
impl JobKindCode {
/// Creates one non-empty Job kind using the KSP safe-code alphabet.
pub fn new(value: impl std::convert::Into<std::string::String>) -> crate::Result<Self> {
let value = value.into();
if !valid_job_code(value.as_str(), crate::MAX_JOB_KIND_CODE_BYTES) {
return std::result::Result::Err(identity_error(crate::ERROR_CODE_JOB_KIND_INVALID, "job_kind"));
}
return std::result::Result::Ok(Self(value));
}
/// Returns the validated stable Job kind code.
#[must_use]
pub fn as_str(&self) -> &str {
return self.0.as_str();
}
}
impl std::fmt::Debug for JobKindCode {
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
return formatter.debug_tuple("JobKindCode").field(&self.0).finish();
}
}
fn identity_error(code: crate::ErrorCode, field: &'static str) -> crate::Error {
return crate::Error::new(code, "invalid bounded Job identity").with_context("field", field);
}
fn valid_job_code(value: &str, maximum_len: usize) -> bool {
if value.is_empty() || value.len() > maximum_len {
return false;
}
return value.bytes().all(|byte| return byte.is_ascii_alphanumeric() || matches!(byte, b'_' | b'-' | b'.' | b':'));
}
#[cfg(test)]
#[path = "../unit_tests/identity.rs"]
mod tests;

View File

@@ -0,0 +1,60 @@
// file: crates/ksp-job-api/src/lib.rs
// version: 2
#![warn(missing_docs)]
#![deny(unreachable_pub)]
#![forbid(unsafe_code)]
//! Passive runtime-neutral lifecycle contracts for bounded KSP Jobs.
//!
//! This foundation owns validated Job identity, explicit lifecycle transitions
//! cooperative cancellation intent and runtime-neutral latest-value observation.
//! Concrete Job behavior, runtime spawning, Transport, Store and Worker contracts
//! remain outside this crate.
mod cancellation;
mod error;
mod identity;
mod lifecycle;
mod notification;
/// Runtime-neutral cloneable token carrying cooperative cancellation intent.
pub use self::cancellation::JobCancellationToken;
/// Error code used when a Job identifier violates its bounded safe-code contract.
pub use self::error::ERROR_CODE_JOB_ID_INVALID;
/// Error code used when a Job kind code violates its bounded safe-code contract.
pub use self::error::ERROR_CODE_JOB_KIND_INVALID;
/// Error code used when a Job notification sequence cannot advance without wrapping.
pub use self::error::ERROR_CODE_JOB_NOTIFICATION_SEQUENCE_EXHAUSTED;
/// Error code used when a requested Job lifecycle transition is not allowed.
pub use self::error::ERROR_CODE_JOB_TRANSITION_INVALID;
/// Bounded caller-supplied identity of one logical Job and its controlled resumptions.
pub use self::identity::JobId;
/// Bounded stable code identifying one concrete family of Jobs.
pub use self::identity::JobKindCode;
/// Maximum UTF-8 byte length admitted for one Job identifier.
pub use self::identity::MAX_JOB_ID_BYTES;
/// Maximum UTF-8 byte length admitted for one Job kind code.
pub use self::identity::MAX_JOB_KIND_CODE_BYTES;
/// Completion classification of a Job that reached its normal terminal state.
pub use self::lifecycle::JobCompletion;
/// Passive owner of one Job identity and its validated lifecycle state.
pub use self::lifecycle::JobLifecycle;
/// Current lifecycle state of one bounded Job.
pub use self::lifecycle::JobState;
/// Latest complete observable value for one Job at a monotone sequence position.
pub use self::notification::JobNotification;
/// Monotone sequence attached to one latest-value Job notification stream.
pub use self::notification::JobNotificationSequence;
/// Runtime-neutral future returned while observing a latest-value Job snapshot source.
pub use self::notification::JobSnapshotFuture;
/// Runtime-neutral read and change-wait contract for one latest-value Job snapshot stream.
pub use self::notification::JobSnapshotSource;
/// Common KSP error type used by Job-facing contracts.
pub use ksp_core_lib::Error;
/// Stable structured code identifying a KSP error category and condition.
pub use ksp_core_lib::ErrorCode;
/// Structured contextual field attached to a KSP error.
pub use ksp_core_lib::ErrorContext;
/// Common KSP result alias using [`Error`].
pub use ksp_core_lib::Result;

View File

@@ -0,0 +1,168 @@
// file: crates/ksp-job-api/src/lifecycle.rs
// version: 1
/// Completion classification of a Job that reached its normal terminal state.
#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)]
#[non_exhaustive]
pub enum JobCompletion {
/// Every selected candidate or unit of work reached a durable complete result.
Complete,
/// The Job terminated normally while retaining one or more explicitly observable gaps.
Partial,
}
impl JobCompletion {
/// Returns the stable safe code for this completion classification.
#[must_use]
pub const fn code(&self) -> &'static str {
return match self {
Self::Complete => "complete",
Self::Partial => "partial",
};
}
}
/// Current lifecycle state of one bounded Job.
#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)]
#[non_exhaustive]
pub enum JobState {
/// The Job was admitted but has not started producing work.
Created,
/// The Job is actively producing or draining normal work.
Running,
/// Cancellation was observed and new work must no longer be admitted.
Cancelling,
/// The Job reached a normal terminal state with an explicit completion classification.
Completed(crate::JobCompletion),
/// The Job reached its cooperative cancellation terminal state.
Cancelled,
/// The Job reached a terminal failure state.
Failed,
}
impl JobState {
/// Returns the stable safe lifecycle code without rendering Job data.
#[must_use]
pub const fn code(&self) -> &'static str {
return match self {
Self::Created => "created",
Self::Running => "running",
Self::Cancelling => "cancelling",
Self::Completed(_) => "completed",
Self::Cancelled => "cancelled",
Self::Failed => "failed",
};
}
/// Returns the normal completion classification when the state is [`Self::Completed`].
#[must_use]
pub const fn completion(&self) -> std::option::Option<crate::JobCompletion> {
return match self {
Self::Completed(completion) => std::option::Option::Some(*completion),
_ => std::option::Option::None,
};
}
/// Reports whether no later lifecycle transition is permitted.
#[must_use]
pub const fn is_terminal(&self) -> bool {
return matches!(self, Self::Completed(_) | Self::Cancelled | Self::Failed);
}
}
/// Passive owner of one Job identity and its validated lifecycle state.
#[derive(Eq, PartialEq)]
pub struct JobLifecycle {
id: crate::JobId,
kind: crate::JobKindCode,
state: crate::JobState,
}
impl JobLifecycle {
/// Creates one lifecycle in [`JobState::Created`] state.
#[must_use]
pub const fn new(id: crate::JobId, kind: crate::JobKindCode) -> Self {
return Self { id, kind, state: crate::JobState::Created };
}
/// Returns the logical Job identity.
#[must_use]
pub const fn id(&self) -> &crate::JobId {
return &self.id;
}
/// Returns the stable Job family code.
#[must_use]
pub const fn kind(&self) -> &crate::JobKindCode {
return &self.kind;
}
/// Returns the current lifecycle state.
#[must_use]
pub const fn state(&self) -> crate::JobState {
return self.state;
}
/// Transitions a newly created Job into its running state.
pub fn start(&mut self) -> crate::Result<()> {
return self.transition(crate::JobState::Running);
}
/// Records that a running Job observed cooperative cancellation intent.
pub fn mark_cancelling(&mut self) -> crate::Result<()> {
return self.transition(crate::JobState::Cancelling);
}
/// Completes a running or cancelling Job with its explicit normal outcome.
pub fn complete(&mut self, completion: crate::JobCompletion) -> crate::Result<()> {
return self.transition(crate::JobState::Completed(completion));
}
/// Marks a created or cancelling Job as cooperatively cancelled.
pub fn mark_cancelled(&mut self) -> crate::Result<()> {
return self.transition(crate::JobState::Cancelled);
}
/// Marks a running or cancelling Job as failed.
pub fn fail(&mut self) -> crate::Result<()> {
return self.transition(crate::JobState::Failed);
}
fn transition(&mut self, target: crate::JobState) -> crate::Result<()> {
if !allowed_transition(self.state, target) {
return std::result::Result::Err(transition_error(self.state, target));
}
self.state = target;
return std::result::Result::Ok(());
}
}
impl std::fmt::Debug for JobLifecycle {
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
return formatter.debug_struct("JobLifecycle").field("id", &self.id).field("kind", &self.kind).field("state", &self.state).finish();
}
}
fn allowed_transition(source: crate::JobState, target: crate::JobState) -> bool {
return matches!(
(source, target),
(crate::JobState::Created, crate::JobState::Running)
| (crate::JobState::Created, crate::JobState::Cancelled)
| (crate::JobState::Running, crate::JobState::Cancelling)
| (crate::JobState::Running, crate::JobState::Completed(_))
| (crate::JobState::Running, crate::JobState::Failed)
| (crate::JobState::Cancelling, crate::JobState::Completed(_))
| (crate::JobState::Cancelling, crate::JobState::Cancelled)
| (crate::JobState::Cancelling, crate::JobState::Failed)
);
}
fn transition_error(source: crate::JobState, target: crate::JobState) -> crate::Error {
return crate::Error::new(crate::ERROR_CODE_JOB_TRANSITION_INVALID, "invalid Job lifecycle transition")
.with_context("source_state", source.code())
.with_context("target_state", target.code());
}
#[cfg(test)]
#[path = "../unit_tests/lifecycle.rs"]
mod tests;

View File

@@ -0,0 +1,126 @@
// file: crates/ksp-job-api/src/notification.rs
// version: 2
/// Monotone sequence attached to one latest-value Job notification stream.
#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)]
pub struct JobNotificationSequence(u64);
impl JobNotificationSequence {
/// Creates the initial sequence position for one Job snapshot stream.
#[must_use]
pub const fn initial() -> Self {
return Self(0);
}
/// Returns the opaque numeric position carried by this sequence.
#[must_use]
pub const fn value(&self) -> u64 {
return self.0;
}
/// Advances the sequence exactly once or reports exhaustion without wrapping.
pub fn next(&self) -> crate::Result<Self> {
let next = match self.0.checked_add(1) {
std::option::Option::Some(value) => value,
std::option::Option::None => {
return std::result::Result::Err(
crate::Error::new(crate::ERROR_CODE_JOB_NOTIFICATION_SEQUENCE_EXHAUSTED, "Job notification sequence exhausted")
.with_context("sequence", self.0.to_string()),
);
},
};
return std::result::Result::Ok(Self(next));
}
/// Reports whether this sequence is strictly newer than an observed sequence.
#[must_use]
pub const fn is_after(&self, observed: Self) -> bool {
return self.0 > observed.0;
}
}
/// Latest complete observable value for one Job at a monotone sequence position.
#[derive(Clone, Eq, PartialEq)]
pub struct JobNotification<S> {
id: crate::JobId,
kind: crate::JobKindCode,
sequence: crate::JobNotificationSequence,
snapshot: S,
state: crate::JobState,
}
impl<S> JobNotification<S> {
/// Creates one immutable latest-value notification from an already validated Job identity and snapshot.
#[must_use]
pub const fn new(id: crate::JobId, kind: crate::JobKindCode, sequence: crate::JobNotificationSequence, state: crate::JobState, snapshot: S) -> Self {
return Self { id, kind, sequence, snapshot, state };
}
/// Returns the logical Job identity.
#[must_use]
pub const fn id(&self) -> &crate::JobId {
return &self.id;
}
/// Returns the stable Job family code.
#[must_use]
pub const fn kind(&self) -> &crate::JobKindCode {
return &self.kind;
}
/// Returns the monotone sequence of this latest value.
#[must_use]
pub const fn sequence(&self) -> crate::JobNotificationSequence {
return self.sequence;
}
/// Returns the complete safe snapshot owned by the concrete Job contract.
#[must_use]
pub const fn snapshot(&self) -> &S {
return &self.snapshot;
}
/// Returns the lifecycle state represented by this snapshot.
#[must_use]
pub const fn state(&self) -> crate::JobState {
return self.state;
}
}
impl<S> std::fmt::Debug for JobNotification<S> {
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
return formatter
.debug_struct("JobNotification")
.field("id", &self.id)
.field("kind", &self.kind)
.field("sequence", &self.sequence)
.field("state", &self.state)
.field("snapshot", &"<redacted>")
.finish();
}
}
/// Runtime-neutral future returned while observing a latest-value Job snapshot source.
pub type JobSnapshotFuture<'a, S> = std::pin::Pin<std::boxed::Box<dyn std::future::Future<Output = crate::JobNotification<S>> + std::marker::Send + 'a>>;
/// Runtime-neutral read and change-wait contract for one latest-value Job snapshot stream.
pub trait JobSnapshotSource: std::marker::Send + std::marker::Sync {
/// Complete snapshot type retained by the concrete source.
type Snapshot: std::clone::Clone + std::marker::Send + std::marker::Sync + 'static;
/// Returns the complete current value without requiring replay of prior notifications.
#[must_use]
fn current(&self) -> crate::JobNotification<Self::Snapshot>;
/// Waits for a value newer than `observed`, returning the complete current snapshot after coalescing any intermediate updates.
fn wait_for_change(&self, observed: crate::JobNotificationSequence) -> crate::JobSnapshotFuture<'_, Self::Snapshot>;
}
#[cfg(test)]
fn exhausted_notification_sequence() -> crate::JobNotificationSequence {
return JobNotificationSequence(u64::MAX);
}
#[cfg(test)]
#[path = "../unit_tests/notification.rs"]
mod tests;

View File

@@ -0,0 +1,82 @@
// file: crates/ksp-job-api/tests/dependency_boundary.rs
// version: 2
//! Dependency and runtime-neutrality canaries for the Job API foundation.
#[test]
fn pre_002_manifest_has_exact_core_only_dependency_graph() {
let manifest = include_str!("../Cargo.toml");
assert!(!manifest.contains("[features]"));
assert!(!manifest.contains("[dev-dependencies]"));
assert!(!manifest.contains("[build-dependencies]"));
assert_eq!(manifest.matches("[dependencies]").count(), 1);
let dependencies_tail = manifest.split("[dependencies]").nth(1);
assert!(dependencies_tail.is_some());
let dependencies_tail = match dependencies_tail {
std::option::Option::Some(value) => value,
std::option::Option::None => return,
};
let dependencies = match dependencies_tail.split("[lints]").next() {
std::option::Option::Some(value) => value,
std::option::Option::None => return,
};
assert_eq!(manifest_dependency_names(dependencies), std::vec!["ksp-core-lib"]);
return;
}
#[test]
fn pre_003_production_sources_forbid_runtime_domain_and_wire_dependencies() {
let sources = [
include_str!("../src/cancellation.rs"),
include_str!("../src/error.rs"),
include_str!("../src/identity.rs"),
include_str!("../src/lib.rs"),
include_str!("../src/lifecycle.rs"),
include_str!("../src/notification.rs"),
];
for source in sources {
for forbidden in [
"ksp_config_lib::",
"ksp_interface_lib::",
"ksp_logging_lib::",
"ksp_offchain_transport_lib::",
"ksp_onchain_transport_lib::",
"ksp_store_api::",
"ksp_store_lib::",
"ksp_worker",
"reqwest::",
"serde::",
"serde_json::",
"solana_",
"tauri::",
"tokio::",
"tonic::",
concat!("tracing", "::"),
] {
assert!(!source.contains(forbidden), "forbidden Job API dependency path detected: {forbidden}");
}
}
return;
}
fn manifest_dependency_names(section: &str) -> std::vec::Vec<&str> {
let mut names = std::vec::Vec::new();
for line in section.lines() {
let content = match line.split('#').next() {
std::option::Option::Some(value) => value.trim(),
std::option::Option::None => continue,
};
if content.is_empty() {
continue;
}
let name = match content.split('=').next() {
std::option::Option::Some(value) => value.trim().trim_end_matches(".workspace"),
std::option::Option::None => continue,
};
if !name.is_empty() {
names.push(name);
}
}
names.sort_unstable();
return names;
}

View File

@@ -0,0 +1,148 @@
// file: crates/ksp-job-api/tests/notifications.rs
// version: 1
//! External-consumer canaries for latest-value Job observation.
#[derive(Clone, Debug, Eq, PartialEq)]
struct TestSnapshot {
completed: u64,
}
#[derive(Clone)]
struct TestSnapshotSource {
current: std::sync::Arc<ksp_job_api::JobNotification<TestSnapshot>>,
}
impl TestSnapshotSource {
fn new(current: ksp_job_api::JobNotification<TestSnapshot>) -> Self {
return Self { current: std::sync::Arc::new(current) };
}
}
impl ksp_job_api::JobSnapshotSource for TestSnapshotSource {
type Snapshot = TestSnapshot;
fn current(&self) -> ksp_job_api::JobNotification<Self::Snapshot> {
return self.current.as_ref().clone();
}
fn wait_for_change(&self, observed: ksp_job_api::JobNotificationSequence) -> ksp_job_api::JobSnapshotFuture<'_, Self::Snapshot> {
let current = ksp_job_api::JobSnapshotSource::current(self);
assert!(current.sequence().is_after(observed));
return std::boxed::Box::pin(std::future::ready(current));
}
}
struct TestWake;
impl std::task::Wake for TestWake {
fn wake(self: std::sync::Arc<Self>) {
return;
}
}
fn poll_ready<S>(mut future: ksp_job_api::JobSnapshotFuture<'_, S>) -> std::option::Option<ksp_job_api::JobNotification<S>> {
let waker = std::task::Waker::from(std::sync::Arc::new(TestWake));
let mut context = std::task::Context::from_waker(&waker);
return match std::future::Future::poll(future.as_mut(), &mut context) {
std::task::Poll::Ready(value) => std::option::Option::Some(value),
std::task::Poll::Pending => std::option::Option::None,
};
}
fn notification_sequence(value: u64) -> std::option::Option<ksp_job_api::JobNotificationSequence> {
let mut sequence = ksp_job_api::JobNotificationSequence::initial();
for _ in 0..value {
sequence = match sequence.next() {
std::result::Result::Ok(next) => next,
std::result::Result::Err(_) => return std::option::Option::None,
};
}
return std::option::Option::Some(sequence);
}
fn notification(sequence_value: u64, state: ksp_job_api::JobState, completed: u64) -> std::option::Option<ksp_job_api::JobNotification<TestSnapshot>> {
let id = match ksp_job_api::JobId::new("external-job-001") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return std::option::Option::None,
};
let kind = match ksp_job_api::JobKindCode::new("backfill_raw") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return std::option::Option::None,
};
let sequence = match notification_sequence(sequence_value) {
std::option::Option::Some(value) => value,
std::option::Option::None => return std::option::Option::None,
};
return std::option::Option::Some(ksp_job_api::JobNotification::new(id, kind, sequence, state, TestSnapshot { completed }));
}
#[test]
fn pre_003_external_notification_contract_is_consumable_from_crate_root() {
let notification = notification(3, ksp_job_api::JobState::Running, 2);
assert!(notification.is_some());
let notification = match notification {
std::option::Option::Some(value) => value,
std::option::Option::None => return,
};
assert_eq!(notification.id().as_str(), "external-job-001");
assert_eq!(notification.kind().as_str(), "backfill_raw");
assert_eq!(notification.sequence().value(), 3);
assert_eq!(notification.state(), ksp_job_api::JobState::Running);
assert_eq!(notification.snapshot().completed, 2);
return;
}
#[test]
fn pre_003_slow_and_independent_listeners_resynchronize_to_latest_value() {
let latest = notification(8, ksp_job_api::JobState::Running, 7);
assert!(latest.is_some());
let source = match latest {
std::option::Option::Some(value) => TestSnapshotSource::new(value),
std::option::Option::None => return,
};
let listener_a = source.clone();
let listener_b = source.clone();
let observed_a = match notification_sequence(2) {
std::option::Option::Some(value) => value,
std::option::Option::None => return,
};
let observed_b = match notification_sequence(6) {
std::option::Option::Some(value) => value,
std::option::Option::None => return,
};
let changed_a = poll_ready(ksp_job_api::JobSnapshotSource::wait_for_change(&listener_a, observed_a));
let changed_b = poll_ready(ksp_job_api::JobSnapshotSource::wait_for_change(&listener_b, observed_b));
assert!(changed_a.is_some());
assert!(changed_b.is_some());
let changed_a = match changed_a {
std::option::Option::Some(value) => value,
std::option::Option::None => return,
};
let changed_b = match changed_b {
std::option::Option::Some(value) => value,
std::option::Option::None => return,
};
assert_eq!(changed_a.sequence().value(), 8);
assert_eq!(changed_b.sequence().value(), 8);
assert_eq!(changed_a.snapshot().completed, 7);
assert_eq!(changed_b.snapshot().completed, 7);
return;
}
#[test]
fn pre_003_terminal_snapshot_remains_readable_from_shared_source() {
let terminal = notification(9, ksp_job_api::JobState::Completed(ksp_job_api::JobCompletion::Partial), 8);
assert!(terminal.is_some());
let source = match terminal {
std::option::Option::Some(value) => TestSnapshotSource::new(value),
std::option::Option::None => return,
};
let cloned = source.clone();
let current = ksp_job_api::JobSnapshotSource::current(&cloned);
assert_eq!(current.sequence().value(), 9);
assert_eq!(current.state(), ksp_job_api::JobState::Completed(ksp_job_api::JobCompletion::Partial));
assert!(current.state().is_terminal());
assert_eq!(current.snapshot().completed, 8);
return;
}

View File

@@ -0,0 +1,53 @@
// file: crates/ksp-job-api/tests/public_api.rs
// version: 1
//! External-consumer canaries for the public Job API foundation.
#[test]
fn pre_002_identity_lifecycle_and_cancellation_are_consumable_from_crate_root() {
let id = ksp_job_api::JobId::new("campaign-2026.09.01:001");
let kind = ksp_job_api::JobKindCode::new("backfill_raw");
assert!(id.is_ok());
assert!(kind.is_ok());
let id = match id {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return,
};
let kind = match kind {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return,
};
let mut lifecycle = ksp_job_api::JobLifecycle::new(id.clone(), kind.clone());
assert_eq!(lifecycle.id(), &id);
assert_eq!(lifecycle.kind(), &kind);
assert_eq!(lifecycle.state(), ksp_job_api::JobState::Created);
assert!(lifecycle.start().is_ok());
assert!(lifecycle.complete(ksp_job_api::JobCompletion::Partial).is_ok());
assert_eq!(lifecycle.state().completion(), std::option::Option::Some(ksp_job_api::JobCompletion::Partial));
let token = ksp_job_api::JobCancellationToken::new();
let clone = token.clone();
assert!(token.cancel());
assert!(clone.is_cancellation_requested());
assert!(!clone.cancel());
return;
}
#[test]
fn pre_002_error_codes_are_stable_and_core_owned() {
let codes: [ksp_core_lib::ErrorCode; 3] =
[ksp_job_api::ERROR_CODE_JOB_ID_INVALID, ksp_job_api::ERROR_CODE_JOB_KIND_INVALID, ksp_job_api::ERROR_CODE_JOB_TRANSITION_INVALID];
assert_eq!(codes[0].domain(), "job_api");
assert_eq!(codes[0].code(), "job_id_invalid");
assert_eq!(codes[1].domain(), "job_api");
assert_eq!(codes[1].code(), "job_kind_invalid");
assert_eq!(codes[2].domain(), "job_api");
assert_eq!(codes[2].code(), "job_transition_invalid");
return;
}
#[test]
fn pre_002_public_bounds_are_exact() {
assert_eq!(ksp_job_api::MAX_JOB_ID_BYTES, 128);
assert_eq!(ksp_job_api::MAX_JOB_KIND_CODE_BYTES, 128);
return;
}

View File

@@ -0,0 +1,101 @@
// file: crates/ksp-job-api/tests/release_completeness.rs
// version: 2
//! Completeness canaries for the current Job API foundation.
#[test]
fn pre_003_crate_root_export_inventory_is_exact() {
let crate_root = include_str!("../src/lib.rs");
let mut actual = std::vec::Vec::new();
for line in crate_root.lines() {
let trimmed = line.trim();
if trimmed.starts_with("pub use ") {
actual.push(trimmed);
}
}
actual.sort_unstable();
let mut expected = std::vec![
"pub use self::cancellation::JobCancellationToken;",
"pub use self::error::ERROR_CODE_JOB_ID_INVALID;",
"pub use self::error::ERROR_CODE_JOB_KIND_INVALID;",
"pub use self::error::ERROR_CODE_JOB_NOTIFICATION_SEQUENCE_EXHAUSTED;",
"pub use self::error::ERROR_CODE_JOB_TRANSITION_INVALID;",
"pub use self::identity::JobId;",
"pub use self::identity::JobKindCode;",
"pub use self::identity::MAX_JOB_ID_BYTES;",
"pub use self::identity::MAX_JOB_KIND_CODE_BYTES;",
"pub use self::lifecycle::JobCompletion;",
"pub use self::lifecycle::JobLifecycle;",
"pub use self::lifecycle::JobState;",
"pub use self::notification::JobNotification;",
"pub use self::notification::JobNotificationSequence;",
"pub use self::notification::JobSnapshotFuture;",
"pub use self::notification::JobSnapshotSource;",
"pub use ksp_core_lib::Error;",
"pub use ksp_core_lib::ErrorCode;",
"pub use ksp_core_lib::ErrorContext;",
"pub use ksp_core_lib::Result;",
];
expected.sort_unstable();
assert_eq!(actual, expected);
assert!(!crate_root.contains("pub mod "));
return;
}
#[test]
fn pre_003_production_module_inventory_is_exact() -> std::io::Result<()> {
let source_root = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("src");
let entries = match std::fs::read_dir(source_root) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let mut names = std::vec::Vec::new();
for entry in entries {
let entry = match entry {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let file_type = match entry.file_type() {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
if !file_type.is_file() {
continue;
}
let name = match entry.file_name().into_string() {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => continue,
};
if name.ends_with(".rs") {
names.push(name);
}
}
names.sort_unstable();
assert_eq!(names, std::vec!["cancellation.rs", "error.rs", "identity.rs", "lib.rs", "lifecycle.rs", "notification.rs"]);
return std::result::Result::Ok(());
}
#[test]
fn pre_003_surface_opens_only_generic_notifications_without_backfill_or_worker_contracts() {
let sources = [
include_str!("../src/cancellation.rs"),
include_str!("../src/identity.rs"),
include_str!("../src/lib.rs"),
include_str!("../src/lifecycle.rs"),
include_str!("../src/notification.rs"),
];
for source in sources {
for forbidden in ["BackfillRequest", "WorkerControl", "WorkerState", "spawn(", "RawTransaction", "provider", "endpoint"] {
assert!(!source.contains(forbidden), "future or domain-specific Job contract leaked early: {forbidden}");
}
}
let notification_source = include_str!("../src/notification.rs");
assert!(notification_source.contains("pub struct JobNotification<S>"));
assert!(notification_source.contains("pub trait JobSnapshotSource"));
assert!(notification_source.contains("std::future::Future"));
assert!(!notification_source.contains("tokio::"));
let lifecycle_source = include_str!("../src/lifecycle.rs");
assert!(lifecycle_source.contains("#[derive(Eq, PartialEq)]\npub struct JobLifecycle"));
assert!(!lifecycle_source.contains("#[derive(Clone, Eq, PartialEq)]\npub struct JobLifecycle"));
return;
}

View File

@@ -0,0 +1,72 @@
// file: crates/ksp-job-api/tests/security_hardening.rs
// version: 1
//! Adversarial lifecycle, cancellation and redaction canaries.
const HOSTILE_MARKER: &str = "JOB-IDENTITY-SECRET-CANARY";
#[test]
fn pre_002_job_id_and_lifecycle_debug_redact_caller_identity() {
let id = ksp_job_api::JobId::new(HOSTILE_MARKER);
let kind = ksp_job_api::JobKindCode::new("backfill_raw");
assert!(id.is_ok());
assert!(kind.is_ok());
let id = match id {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return,
};
let kind = match kind {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return,
};
assert_eq!(std::format!("{id:?}"), "JobId(..)");
let lifecycle = ksp_job_api::JobLifecycle::new(id, kind);
let debug = std::format!("{lifecycle:?}");
assert!(debug.contains("JobLifecycle"));
assert!(debug.contains("backfill_raw"));
assert!(debug.contains("Created"));
assert!(!debug.contains(HOSTILE_MARKER));
return;
}
#[test]
fn pre_002_terminal_states_are_immutable_under_all_public_mutators() {
let id = ksp_job_api::JobId::new("terminal-job");
let kind = ksp_job_api::JobKindCode::new("test_job");
assert!(id.is_ok());
assert!(kind.is_ok());
let id = match id {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return,
};
let kind = match kind {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return,
};
let mut lifecycle = ksp_job_api::JobLifecycle::new(id, kind);
assert!(lifecycle.start().is_ok());
assert!(lifecycle.complete(ksp_job_api::JobCompletion::Complete).is_ok());
assert!(lifecycle.start().is_err());
assert!(lifecycle.mark_cancelling().is_err());
assert!(lifecycle.complete(ksp_job_api::JobCompletion::Partial).is_err());
assert!(lifecycle.mark_cancelled().is_err());
assert!(lifecycle.fail().is_err());
assert_eq!(lifecycle.state(), ksp_job_api::JobState::Completed(ksp_job_api::JobCompletion::Complete));
return;
}
#[test]
fn pre_002_cancellation_token_is_send_sync_and_cross_thread_visible() {
fn require_send_sync<T: std::marker::Send + std::marker::Sync>() {}
require_send_sync::<ksp_job_api::JobCancellationToken>();
let token = ksp_job_api::JobCancellationToken::new();
let worker_token = token.clone();
let thread = std::thread::spawn(move || return worker_token.cancel());
let request = match thread.join() {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return,
};
assert!(request);
assert!(token.is_cancellation_requested());
return;
}

View File

@@ -0,0 +1,22 @@
// file: crates/ksp-job-api/unit_tests/cancellation.rs
// version: 1
#[test]
fn pre_002_cancellation_is_shared_and_idempotent() {
let token = crate::JobCancellationToken::new();
let clone = token.clone();
assert!(!token.is_cancellation_requested());
assert!(clone.cancel());
assert!(token.is_cancellation_requested());
assert!(!token.cancel());
assert!(!clone.cancel());
return;
}
#[test]
fn pre_002_default_token_starts_without_cancellation() {
let token = crate::JobCancellationToken::default();
assert!(!token.is_cancellation_requested());
assert_eq!(std::format!("{token:?}"), "JobCancellationToken { cancellation_requested: false }");
return;
}

View File

@@ -0,0 +1,57 @@
// file: crates/ksp-job-api/unit_tests/identity.rs
// version: 1
#[test]
fn pre_002_identity_accepts_safe_codes_at_exact_bounds() {
let job_id = crate::JobId::new("a".repeat(crate::MAX_JOB_ID_BYTES));
let kind = crate::JobKindCode::new("backfill.raw:solana-1");
assert!(job_id.is_ok());
assert!(kind.is_ok());
let job_id = match job_id {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return,
};
let kind = match kind {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return,
};
assert_eq!(job_id.as_str().len(), crate::MAX_JOB_ID_BYTES);
assert_eq!(kind.as_str(), "backfill.raw:solana-1");
return;
}
#[test]
fn pre_002_identity_rejects_empty_oversized_and_unsafe_values() {
for value in [std::string::String::new(), "a".repeat(crate::MAX_JOB_ID_BYTES + 1), "unsafe/value".to_string(), "unicode-é".to_string()] {
let rejected = crate::JobId::new(value);
assert!(rejected.is_err());
let error = match rejected {
std::result::Result::Err(value) => value,
std::result::Result::Ok(_) => continue,
};
assert_eq!(error.code(), crate::ERROR_CODE_JOB_ID_INVALID);
assert_eq!(error.context().len(), 1);
assert_eq!(error.context()[0].key(), "field");
assert_eq!(error.context()[0].value(), "job_id");
}
for value in [std::string::String::new(), "b".repeat(crate::MAX_JOB_KIND_CODE_BYTES + 1), "backfill raw".to_string()] {
let rejected = crate::JobKindCode::new(value);
assert!(rejected.is_err());
let error = match rejected {
std::result::Result::Err(value) => value,
std::result::Result::Ok(_) => continue,
};
assert_eq!(error.code(), crate::ERROR_CODE_JOB_KIND_INVALID);
assert_eq!(error.context()[0].value(), "job_kind");
}
return;
}
#[test]
fn pre_002_private_validator_uses_exact_safe_code_alphabet() {
assert!(super::valid_job_code("AZaz09_.:-", 10));
for rejected in ["slash/value", "space value", "line\nbreak", "é"] {
assert!(!super::valid_job_code(rejected, crate::MAX_JOB_ID_BYTES));
}
return;
}

View File

@@ -0,0 +1,153 @@
// file: crates/ksp-job-api/unit_tests/lifecycle.rs
// version: 2
fn new_lifecycle() -> std::option::Option<crate::JobLifecycle> {
let id = match crate::JobId::new("job-001") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return std::option::Option::None,
};
let kind = match crate::JobKindCode::new("backfill_raw") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return std::option::Option::None,
};
return std::option::Option::Some(crate::JobLifecycle::new(id, kind));
}
#[test]
fn pre_002_lifecycle_accepts_every_planned_terminal_path() {
for completion in [crate::JobCompletion::Complete, crate::JobCompletion::Partial] {
let lifecycle = new_lifecycle();
assert!(lifecycle.is_some());
let mut lifecycle = match lifecycle {
std::option::Option::Some(value) => value,
std::option::Option::None => continue,
};
assert!(lifecycle.start().is_ok());
assert!(lifecycle.complete(completion).is_ok());
assert_eq!(lifecycle.state(), crate::JobState::Completed(completion));
assert!(lifecycle.state().is_terminal());
assert_eq!(lifecycle.state().completion(), std::option::Option::Some(completion));
}
let lifecycle = new_lifecycle();
assert!(lifecycle.is_some());
let mut lifecycle = match lifecycle {
std::option::Option::Some(value) => value,
std::option::Option::None => return,
};
assert!(lifecycle.start().is_ok());
assert!(lifecycle.fail().is_ok());
assert_eq!(lifecycle.state(), crate::JobState::Failed);
let lifecycle = new_lifecycle();
assert!(lifecycle.is_some());
let mut lifecycle = match lifecycle {
std::option::Option::Some(value) => value,
std::option::Option::None => return,
};
assert!(lifecycle.mark_cancelled().is_ok());
assert_eq!(lifecycle.state(), crate::JobState::Cancelled);
return;
}
#[test]
fn pre_002_cancelling_allows_cancel_complete_or_fail() {
for terminal in [crate::JobState::Cancelled, crate::JobState::Completed(crate::JobCompletion::Complete), crate::JobState::Failed] {
let lifecycle = new_lifecycle();
assert!(lifecycle.is_some());
let mut lifecycle = match lifecycle {
std::option::Option::Some(value) => value,
std::option::Option::None => continue,
};
assert!(lifecycle.start().is_ok());
assert!(lifecycle.mark_cancelling().is_ok());
let result = match terminal {
crate::JobState::Cancelled => lifecycle.mark_cancelled(),
crate::JobState::Completed(completion) => lifecycle.complete(completion),
crate::JobState::Failed => lifecycle.fail(),
_ => return,
};
assert!(result.is_ok());
assert_eq!(lifecycle.state(), terminal);
}
return;
}
#[test]
fn pre_002_invalid_transition_preserves_source_state_and_reports_safe_context() {
let lifecycle = new_lifecycle();
assert!(lifecycle.is_some());
let mut lifecycle = match lifecycle {
std::option::Option::Some(value) => value,
std::option::Option::None => return,
};
let rejected = lifecycle.complete(crate::JobCompletion::Complete);
assert!(rejected.is_err());
assert_eq!(lifecycle.state(), crate::JobState::Created);
let error = match rejected {
std::result::Result::Err(value) => value,
std::result::Result::Ok(_) => return,
};
assert_eq!(error.code(), crate::ERROR_CODE_JOB_TRANSITION_INVALID);
assert_eq!(error.context().len(), 2);
assert_eq!(error.context()[0].key(), "source_state");
assert_eq!(error.context()[0].value(), "created");
assert_eq!(error.context()[1].key(), "target_state");
assert_eq!(error.context()[1].value(), "completed");
return;
}
#[test]
fn pre_002_every_terminal_state_rejects_later_mutation() {
let terminals = [crate::JobState::Completed(crate::JobCompletion::Complete), crate::JobState::Cancelled, crate::JobState::Failed];
for terminal in terminals {
let lifecycle = new_lifecycle();
assert!(lifecycle.is_some());
let mut lifecycle = match lifecycle {
std::option::Option::Some(value) => value,
std::option::Option::None => continue,
};
if terminal == crate::JobState::Cancelled {
assert!(lifecycle.mark_cancelled().is_ok());
} else {
assert!(lifecycle.start().is_ok());
let terminal_result = match terminal {
crate::JobState::Completed(completion) => lifecycle.complete(completion),
crate::JobState::Failed => lifecycle.fail(),
_ => return,
};
assert!(terminal_result.is_ok());
}
let rejected = lifecycle.start();
assert!(rejected.is_err());
assert_eq!(lifecycle.state(), terminal);
}
return;
}
#[test]
fn pre_002_transition_matrix_is_exact() {
let states = [
crate::JobState::Created,
crate::JobState::Running,
crate::JobState::Cancelling,
crate::JobState::Completed(crate::JobCompletion::Complete),
crate::JobState::Cancelled,
crate::JobState::Failed,
];
for source in states {
for target in states {
let expected = matches!(
(source, target),
(crate::JobState::Created, crate::JobState::Running)
| (crate::JobState::Created, crate::JobState::Cancelled)
| (crate::JobState::Running, crate::JobState::Cancelling)
| (crate::JobState::Running, crate::JobState::Completed(_))
| (crate::JobState::Running, crate::JobState::Failed)
| (crate::JobState::Cancelling, crate::JobState::Completed(_))
| (crate::JobState::Cancelling, crate::JobState::Cancelled)
| (crate::JobState::Cancelling, crate::JobState::Failed)
);
assert_eq!(super::allowed_transition(source, target), expected, "unexpected transition matrix result for {source:?} -> {target:?}");
}
}
return;
}

View File

@@ -0,0 +1,81 @@
// file: crates/ksp-job-api/unit_tests/notification.rs
// version: 1
fn sequence(value: u64) -> crate::JobNotificationSequence {
let mut sequence = crate::JobNotificationSequence::initial();
for _ in 0..value {
sequence = match sequence.next() {
std::result::Result::Ok(next) => next,
std::result::Result::Err(_) => return sequence,
};
}
return sequence;
}
#[test]
fn pre_003_notification_sequence_advances_strictly_and_orders_positions() {
let mut first = crate::JobNotificationSequence::initial();
for _ in 0..41 {
let next = first.next();
assert!(next.is_ok());
first = match next {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return,
};
}
let second = first.next();
assert!(second.is_ok());
let second = match second {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return,
};
assert_eq!(first.value(), 41);
assert_eq!(second.value(), 42);
assert!(second.is_after(first));
assert!(!first.is_after(second));
assert!(second > first);
return;
}
#[test]
fn pre_003_notification_sequence_exhaustion_is_explicit_and_non_wrapping() {
let exhausted = super::exhausted_notification_sequence().next();
assert!(exhausted.is_err());
let error = match exhausted {
std::result::Result::Err(value) => value,
std::result::Result::Ok(_) => return,
};
assert_eq!(error.code(), crate::ERROR_CODE_JOB_NOTIFICATION_SEQUENCE_EXHAUSTED);
assert_eq!(error.context().len(), 1);
assert_eq!(error.context()[0].key(), "sequence");
assert_eq!(error.context()[0].value(), u64::MAX.to_string());
return;
}
#[test]
fn pre_003_notification_preserves_complete_value_and_redacts_snapshot_debug() {
let id = crate::JobId::new("job-notify-001");
let kind = crate::JobKindCode::new("backfill_raw");
assert!(id.is_ok());
assert!(kind.is_ok());
let id = match id {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return,
};
let kind = match kind {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return,
};
let notification =
crate::JobNotification::new(id.clone(), kind.clone(), sequence(7), crate::JobState::Running, "RAW-PAYLOAD-MUST-NOT-APPEAR-IN-DEBUG".to_string());
assert_eq!(notification.id(), &id);
assert_eq!(notification.kind(), &kind);
assert_eq!(notification.sequence().value(), 7);
assert_eq!(notification.state(), crate::JobState::Running);
assert_eq!(notification.snapshot(), "RAW-PAYLOAD-MUST-NOT-APPEAR-IN-DEBUG");
let debug = std::format!("{notification:?}");
assert!(debug.contains("JobNotification"));
assert!(debug.contains("<redacted>"));
assert!(!debug.contains("RAW-PAYLOAD-MUST-NOT-APPEAR-IN-DEBUG"));
return;
}

View File

@@ -0,0 +1,25 @@
# file: crates/ksp-job-backfill-lib/Cargo.toml
# version: 4
[package]
name = "ksp-job-backfill-lib"
version.workspace = true
edition.workspace = true
repository.workspace = true
[dependencies]
futures-util = { workspace = true, features = ["std"] }
ksp-core-lib = { path = "../ksp-core-lib" }
ksp-job-api = { path = "../ksp-job-api" }
ksp-logging-lib = { path = "../ksp-logging-lib" }
ksp-onchain-transport-lib = { path = "../ksp-onchain-transport-lib" }
ksp-store-lib = { path = "../ksp-store-lib", default-features = false }
serde_json.workspace = true
sha2.workspace = true
tokio = { workspace = true, features = ["macros", "sync"] }
[dev-dependencies]
tokio = { workspace = true, features = ["macros", "rt-multi-thread"] }
[lints]
workspace = true

View File

@@ -0,0 +1,123 @@
<!-- file: crates/ksp-job-backfill-lib/README.md -->
<!-- version: 1 -->
# ksp-job-backfill-lib
`ksp-job-backfill-lib` implémente le premier job historique concret de KSP : un backfill borné de transactions Solana vers la couche RAW durable.
La crate compose les contrats Job, Transport observé et Store backend-neutral sans posséder leurs politiques internes. Elle couvre l'admission, la découverte historique, l'hydratation `getTransaction`, la conversion RAW v1, la persistance atomique, la concurrence bornée, la frontier contiguë, le checkpoint caller-owned, l'annulation coopérative et les snapshots latest-value.
## Identité
L'identité logique d'une transaction reste :
```text
(RawNetworkId, RawTransactionSignature)
```
Le rôle HTTP, le provider, l'endpoint et le protocole ne font pas partie de cette identité. Ils décrivent la sélection et/ou la provenance d'acquisition.
Le fingerprint d'un scope est déterministe sur sa sémantique de découverte et son réseau ; il n'intègre pas `JobId`, provider, endpoint, protocole ou rôle Transport.
## Scopes
Quatre scopes sont exposés :
```text
LatestAddress
BeforeAddress
AfterAddress
ExplicitSignatures
```
La requête borne explicitement page size, nombre de pages, nombre de candidats et concurrence d'hydratation. Les signatures explicites sont dédupliquées de façon stable à la première occurrence.
## Hydratation et RAW v1
L'hydratation passe uniquement par la voie observée de `ksp-onchain-transport-lib`.
Un résultat disponible produit en mémoire :
- un `RawTransaction` canonique ;
- une `RawTransactionObservation` liée à la même référence logique ;
- une provenance contenant le provider et l'endpoint réellement gagnants ;
- un payload JSON canonique `ksp.solana.raw_transaction`, version `1` ;
- un digest SHA-256 des bytes canoniques.
Un `getTransaction = null` devient `BackfillHydrationOutcome::Missing` et ne fabrique ni payload ni provenance.
## Persistance
La persistance utilise exclusivement `ksp-store-lib` avec `default-features = false` côté dépendance de crate :
```text
ksp-job-backfill-lib
-> ksp-store-lib
-X-> backend imposé par le job
```
Le chemin d'écriture est l'acquisition atomique `RawTransaction + RawTransactionObservation` en mode normal. La crate distingue insertion, déjà présent, tombstone purgé, observation nouvelle/existante, missing et conflit. Un conflit de contenu n'est jamais converti en succès idempotent.
## Concurrence et checkpoint
L'exécution maintient au plus `hydration_concurrency` candidats actifs. Les terminaisons hors ordre sont réconciliées par index stable ; le checkpoint n'avance que sur un préfixe **contigu** d'outcomes durablement sûrs.
`BackfillCheckpoint` est opaque et caller-owned. Il lie :
```text
JobId
scope fingerprint
completed contiguous prefix
private Before resume cursor
```
La crate ne persiste pas elle-même ce checkpoint. Une reprise crash-safe durable nécessite donc que le caller choisisse explicitement où conserver le checkpoint retourné.
## Runtime et annulation
`BackfillJobRuntime` est un coordinator single-run. `BackfillJobHandle` peut être cloné pour :
- demander une annulation coopérative ;
- lire l'état de cette demande ;
- obtenir une source latest-value indépendante.
L'annulation arrête l'admission de nouveau travail et peut interrompre certaines attentes pré-Store. Une persistance Store déjà soumise est toujours drainée avant publication terminale.
Les snapshots exposent uniquement des compteurs, phases, boundary, checkpoint et code d'échec sûrs ; ils ne contiennent ni payload RAW, ni URL/credential Transport.
## Dépendances
Les dépendances normales sont :
```text
ksp-core-lib
ksp-job-api
ksp-logging-lib
ksp-onchain-transport-lib
ksp-store-lib (default-features = false)
futures-util
serde_json
sha2
tokio (macros + sync, détail runtime privé)
```
La crate ne dépend pas de Config, `ksp-store-api` directement, `ksp-store-postgres-lib` ni d'un SDK provider.
## Hors périmètre
La crate ne possède pas :
- application desktop ou CLI ;
- Worker API / worker live ;
- retry, pacing ou sélection d'endpoint Transport ;
- backend Store concret ;
- decoding Program ou matérialisation CORE/DECODE/SPECIALIZED ;
- table dédiée de checkpoint ;
- control plane Job générique.
## Documentation
- [`USAGE.md`](USAGE.md) — construction des scopes/requêtes, runtime, snapshots et reprise ;
- [`../ksp-job-api/README.md`](../ksp-job-api/README.md) — contrats Job runtime-neutral ;
- [`../../docs/architecture/009-ACQUISITION_WORKERS_AND_JOBS.md`](../../docs/architecture/009-ACQUISITION_WORKERS_AND_JOBS.md) — architecture durable acquisition/jobs ;
- [`../../docs/architecture/008-DATA_MATERIALIZATION_AND_STORE.md`](../../docs/architecture/008-DATA_MATERIALIZATION_AND_STORE.md) — frontière RAW/Store.

View File

@@ -0,0 +1,206 @@
<!-- file: crates/ksp-job-backfill-lib/USAGE.md -->
<!-- version: 2 -->
# Utilisation de ksp-job-backfill-lib
Cette page décrit l'utilisation durable de la façade publique de `ksp-job-backfill-lib`. Elle suppose qu'un caller a déjà construit un `HttpTransportPool` et un `Store` compatibles avec le même réseau logique.
## Construire une signature et un scope
```rust
let anchor = match ksp_job_backfill_lib::BackfillSignature::new(
"1111111111111111111111111111111111111111111111111111111111111111",
) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let address = ksp_core_lib::Pubkey::new_from_array([7_u8; 32]);
let scope = ksp_job_backfill_lib::BackfillScope::before_address(address, anchor);
```
Les autres formes sont :
```rust
let latest = ksp_job_backfill_lib::BackfillScope::latest_address(address);
let before = ksp_job_backfill_lib::BackfillScope::before_address(address, anchor.clone());
let after = ksp_job_backfill_lib::BackfillScope::after_address(address, anchor.clone());
let explicit = ksp_job_backfill_lib::BackfillScope::explicit_signatures(vec![anchor]);
```
`ExplicitSignatures` déduplique la liste en conservant la première occurrence. Les scopes address utilisent `getSignaturesForAddress` ; le scope explicite n'effectue aucune découverte address.
## Construire une requête bornée
```rust
fn request(scope: ksp_job_backfill_lib::BackfillScope) -> ksp_core_lib::Result<ksp_job_backfill_lib::BackfillRequest> {
let job_id = match ksp_job_api::JobId::new("raw-backfill-0001") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let network = match ksp_store_lib::RawNetworkId::new("mainnet") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let role = ksp_onchain_transport_lib::HttpRoleName::new("historical");
return ksp_job_backfill_lib::BackfillRequest::new(
job_id,
network,
role,
ksp_job_backfill_lib::BackfillCommitment::Finalized,
scope,
500,
20,
5_000,
16,
std::option::Option::None,
);
}
```
Bornes publiques :
```text
page_size 1 ..= 1_000
max_pages 1 ..= 10_000
max_candidates 1 ..= 10_000
hydration_concurrency 1 ..= 64
```
Le `min_context_slot` n'est pas admis pour `ExplicitSignatures`.
## Comprendre le fingerprint
```rust
let fingerprint = request.scope_fingerprint();
let bytes: &[u8; 32] = fingerprint.as_bytes();
```
Le fingerprint représente le scope sémantique et le réseau. Il ne change pas uniquement parce que le rôle Transport, le provider ou l'endpoint d'acquisition change.
Ne pas utiliser le fingerprint comme identité de transaction : l'identité durable reste `(network, signature)`.
## Exécuter le runtime complet
```rust
async fn run_backfill(
request: ksp_job_backfill_lib::BackfillRequest,
transport: &ksp_onchain_transport_lib::HttpTransportPool,
store: &ksp_store_lib::Store,
) -> ksp_core_lib::Result<ksp_job_backfill_lib::BackfillJobSnapshot> {
let runtime = match ksp_job_backfill_lib::BackfillJobRuntime::new(request) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
return runtime.run(transport, store).await;
}
```
Le Store et la requête doivent cibler le même `RawNetworkId`. Un mismatch est rejeté avant l'écriture.
## Observer la progression
Obtenir le handle avant de déplacer le runtime dans `run` :
```rust
let runtime = ksp_job_backfill_lib::BackfillJobRuntime::new(request)?;
let handle = runtime.handle();
let snapshots = handle.snapshots();
let current = ksp_job_api::JobSnapshotSource::current(&snapshots);
let observed = current.sequence();
let newer = ksp_job_api::JobSnapshotSource::wait_for_change(&snapshots, observed).await;
assert!(newer.sequence().is_after(observed));
```
Chaque listener peut cloner sa propre `BackfillSnapshotSource`. La source est latest-value : plusieurs mises à jour intermédiaires peuvent être coalescées, mais la valeur retournée est toujours un snapshot complet.
Les compteurs publics du snapshot couvrent notamment :
```text
candidates_selected / admitted / finished
entities_inserted / existing / purged
observations_inserted / existing
missing / conflicts / holes
maximum_in_flight
contiguous_completed
checkpoint
failure_code
```
## Demander une annulation
```rust
let accepted = handle.cancel();
if accepted {
assert!(handle.is_cancellation_requested());
}
```
L'annulation est coopérative. Elle peut stopper de nouvelles admissions et certaines attentes avant persistance. Une écriture Store déjà soumise est drainée ; le caller ne doit donc pas supposer qu'une demande d'annulation rend immédiatement toutes les opérations in-flight inexistantes.
Une demande faite après publication terminale est rejetée (`false`).
## Reprendre avec un checkpoint
Le snapshot ou le batch d'exécution peut fournir un `BackfillCheckpoint` sûr. Pour une reprise contrôlée :
```rust
let resumed = match request.with_checkpoint(checkpoint) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
```
Le checkpoint doit appartenir au même `JobId` et au même fingerprint de scope.
Sémantique de reprise :
```text
LatestAddress repart du latest courant ; frontier contiguë conservée dans le nouveau run
BeforeAddress reprend avec le dernier cursor before prouvé
AfterAddress rejoue le scope et saute seulement le préfixe contigu prouvé
ExplicitSignatures rejoue la liste et saute seulement le préfixe contigu prouvé
```
`BackfillCheckpoint` n'est pas persisté automatiquement. Si le caller exige une reprise après crash/process restart, il doit stocker ce checkpoint dans une surface durable appropriée puis le réinjecter explicitement.
## Utiliser les primitives séparément
La façade expose aussi les étapes pour des compositions/tests spécialisés :
```text
discover_backfill_candidates
hydrate_backfill_candidate
persist_backfill_hydration
execute_backfill_discovery
```
`hydrate_backfill_candidate` ne persiste rien. `persist_backfill_hydration` n'effectue aucun appel Transport. `execute_backfill_discovery` combine hydratation/persistance sur un résultat de découverte déjà validé.
Pour un flux applicatif normal qui veut lifecycle + snapshots + annulation, préférer `BackfillJobRuntime::run`.
## Interpréter les outcomes de persistance
Les outcomes distinguent explicitement :
```text
entity: Inserted | AlreadyPresent | SkippedPurged | Conflict
observation: Inserted | AlreadyPresent | NotRecorded | NotApplicable
```
`Missing` ne provoque aucune écriture Store. Un conflit de contenu reste observable comme conflit et ne doit pas être traité comme une relance idempotente réussie.
## Frontières à respecter
Le caller ne doit pas :
- pré-lire le Store pour décider s'il faut hydrater une transaction ;
- appeler directement `ksp-store-postgres-lib` depuis le job ;
- ajouter une politique de retry/pacing qui concurrence Transport ;
- utiliser provider/endpoint comme identité transactionnelle ;
- inventer une provenance pour `getTransaction = null` ;
- interpréter un checkpoint caller-owned comme une garantie de persistence crash-safe automatique ;
- utiliser le job RAW comme decoder Program ou processor CORE.

View File

@@ -0,0 +1,218 @@
// file: crates/ksp-job-backfill-lib/src/checkpoint.rs
// version: 2
/// Opaque caller-owned checkpoint for one controlled Backfill resumption.
///
/// The checkpoint carries no payload, endpoint, provider, URL or secret. Persistence of this
/// value is deliberately external to Store in v0.3.6; the Backfill library only validates and
/// consumes checkpoints supplied back by its caller.
#[derive(Clone, Eq, PartialEq)]
pub struct BackfillCheckpoint {
job_id: ksp_job_api::JobId,
scope_fingerprint: crate::BackfillScopeFingerprint,
completed_prefix: usize,
resume_before: std::option::Option<crate::BackfillSignature>,
}
impl crate::BackfillCheckpoint {
/// Returns the logical Job identity that owns this checkpoint.
#[must_use]
pub const fn job_id(&self) -> &ksp_job_api::JobId {
return &self.job_id;
}
/// Returns the semantic scope fingerprint bound to this checkpoint.
#[must_use]
pub const fn scope_fingerprint(&self) -> crate::BackfillScopeFingerprint {
return self.scope_fingerprint;
}
/// Returns the number of candidates proven durable in one contiguous prefix.
#[must_use]
pub const fn completed_prefix(&self) -> usize {
return self.completed_prefix;
}
/// Creates one internally proven checkpoint.
pub(crate) fn new(
job_id: ksp_job_api::JobId,
scope_fingerprint: crate::BackfillScopeFingerprint,
completed_prefix: usize,
resume_before: std::option::Option<crate::BackfillSignature>,
) -> Self {
return Self { job_id, scope_fingerprint, completed_prefix, resume_before };
}
/// Returns the internal exclusive `before` cursor used only by controlled Before resumption.
pub(crate) const fn resume_before(&self) -> std::option::Option<&crate::BackfillSignature> {
return self.resume_before.as_ref();
}
}
impl std::fmt::Debug for crate::BackfillCheckpoint {
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
return formatter
.debug_struct("BackfillCheckpoint")
.field("job_id", &self.job_id)
.field("scope_fingerprint", &self.scope_fingerprint)
.field("completed_prefix", &self.completed_prefix)
.field("has_resume_before", &self.resume_before.is_some())
.finish();
}
}
/// Private bounded bitmap tracking durable completions and their contiguous prefix.
pub(crate) struct CompletionFrontier {
durable: std::vec::Vec<bool>,
contiguous_completed: usize,
}
impl crate::CompletionFrontier {
/// Creates one empty frontier for the exact bounded candidate count.
pub(crate) fn new(candidate_count: usize) -> Self {
return Self { durable: vec![false; candidate_count], contiguous_completed: 0 };
}
/// Seeds a previously proven replay prefix before processing the remaining candidates.
pub(crate) fn seed_prefix(&mut self, completed_prefix: usize) -> ksp_core_lib::Result<()> {
if completed_prefix > self.durable.len() {
return std::result::Result::Err(checkpoint_error("checkpoint.completed_prefix"));
}
for index in 0..completed_prefix {
self.durable[index] = true;
}
self.contiguous_completed = completed_prefix;
return std::result::Result::Ok(());
}
/// Marks one candidate durable and advances only through the now-contiguous prefix.
pub(crate) fn mark_durable(&mut self, index: usize) -> ksp_core_lib::Result<()> {
let slot = match self.durable.get_mut(index) {
std::option::Option::Some(slot) => slot,
std::option::Option::None => return std::result::Result::Err(checkpoint_error("frontier.index")),
};
*slot = true;
while self.contiguous_completed < self.durable.len() && self.durable[self.contiguous_completed] {
self.contiguous_completed += 1;
}
return std::result::Result::Ok(());
}
/// Returns the number of durable candidates in the contiguous prefix.
pub(crate) const fn contiguous_completed(&self) -> usize {
return self.contiguous_completed;
}
}
/// Validates one checkpoint against the Job, semantic scope and scope-specific cursor shape.
pub(crate) fn validate_request_checkpoint(
request_job_id: &ksp_job_api::JobId,
scope_fingerprint: crate::BackfillScopeFingerprint,
scope_kind: crate::BackfillScopeKind,
checkpoint: &crate::BackfillCheckpoint,
) -> ksp_core_lib::Result<()> {
if checkpoint.job_id() != request_job_id {
return std::result::Result::Err(checkpoint_error("checkpoint.job_id"));
}
if checkpoint.scope_fingerprint() != scope_fingerprint {
return std::result::Result::Err(checkpoint_error("checkpoint.scope_fingerprint"));
}
if scope_kind != crate::BackfillScopeKind::BeforeAddress && checkpoint.resume_before().is_some() {
return std::result::Result::Err(checkpoint_error("checkpoint.resume_before"));
}
return std::result::Result::Ok(());
}
/// Resolves the exclusive Before cursor from a validated checkpoint or the original scope anchor.
pub(crate) fn resume_before_cursor(request: &crate::BackfillRequest) -> std::option::Option<std::string::String> {
if request.scope().kind() != crate::BackfillScopeKind::BeforeAddress {
return std::option::Option::None;
}
if let std::option::Option::Some(checkpoint) = request.checkpoint()
&& let std::option::Option::Some(cursor) = checkpoint.resume_before()
{
return std::option::Option::Some(cursor.as_str().to_owned());
}
return request.scope().anchor().map(|anchor| return anchor.as_str().to_owned());
}
/// Resolves the replay prefix skipped only by After and Explicit execution.
pub(crate) fn execution_resume_offset(request: &crate::BackfillRequest, discovery: &crate::BackfillDiscovery) -> ksp_core_lib::Result<usize> {
let checkpoint = match request.checkpoint() {
std::option::Option::Some(checkpoint) => checkpoint,
std::option::Option::None => return std::result::Result::Ok(0),
};
return match request.scope().kind() {
crate::BackfillScopeKind::AfterAddress | crate::BackfillScopeKind::ExplicitSignatures => {
if checkpoint.completed_prefix() > discovery.candidates().len() {
return std::result::Result::Err(checkpoint_error("checkpoint.completed_prefix"));
}
std::result::Result::Ok(checkpoint.completed_prefix())
},
crate::BackfillScopeKind::LatestAddress | crate::BackfillScopeKind::BeforeAddress => std::result::Result::Ok(0),
};
}
/// Builds the safe next checkpoint from one drained contiguous completion frontier.
pub(crate) fn checkpoint_from_frontier(
request: &crate::BackfillRequest,
discovery: &crate::BackfillDiscovery,
frontier: &crate::CompletionFrontier,
) -> ksp_core_lib::Result<crate::BackfillCheckpoint> {
let validation = crate::validate_discovery_identity(request, discovery);
if let std::result::Result::Err(error) = validation {
return std::result::Result::Err(error);
}
let local_prefix = frontier.contiguous_completed();
if request.scope().kind() == crate::BackfillScopeKind::AfterAddress && discovery.boundary() == crate::BackfillDiscoveryBoundary::AfterAnchorNotReached {
if let std::option::Option::Some(checkpoint) = request.checkpoint() {
return std::result::Result::Ok(checkpoint.clone());
}
return std::result::Result::Ok(crate::BackfillCheckpoint::new(request.job_id().clone(), request.scope_fingerprint(), 0, std::option::Option::None));
}
let (completed_prefix, resume_before) = match request.scope().kind() {
crate::BackfillScopeKind::LatestAddress => (local_prefix, std::option::Option::None),
crate::BackfillScopeKind::BeforeAddress => {
let previous = request.checkpoint().map_or(0, crate::BackfillCheckpoint::completed_prefix);
let completed_prefix = match previous.checked_add(local_prefix) {
std::option::Option::Some(value) => value,
std::option::Option::None => return std::result::Result::Err(checkpoint_error("checkpoint.completed_prefix")),
};
let resume_before = if local_prefix == 0 {
match request.checkpoint() {
std::option::Option::Some(checkpoint) => checkpoint.resume_before().cloned(),
std::option::Option::None => request.scope().anchor().cloned(),
}
} else {
let index = local_prefix - 1;
let candidate = match discovery.candidates().get(index) {
std::option::Option::Some(candidate) => candidate,
std::option::Option::None => return std::result::Result::Err(checkpoint_error("frontier.index")),
};
std::option::Option::Some(candidate.identity().signature().clone())
};
(completed_prefix, resume_before)
},
crate::BackfillScopeKind::AfterAddress | crate::BackfillScopeKind::ExplicitSignatures => (local_prefix, std::option::Option::None),
};
return std::result::Result::Ok(crate::BackfillCheckpoint::new(request.job_id().clone(), request.scope_fingerprint(), completed_prefix, resume_before));
}
/// Validates that one discovery result belongs to the request network and semantic scope.
pub(crate) fn validate_discovery_identity(request: &crate::BackfillRequest, discovery: &crate::BackfillDiscovery) -> ksp_core_lib::Result<()> {
if discovery.network() != request.network() {
return std::result::Result::Err(checkpoint_error("discovery.network"));
}
if discovery.scope_fingerprint() != request.scope_fingerprint() {
return std::result::Result::Err(checkpoint_error("discovery.scope_fingerprint"));
}
return std::result::Result::Ok(());
}
fn checkpoint_error(field: &'static str) -> ksp_core_lib::Error {
return ksp_core_lib::Error::new(crate::ERROR_CODE_BACKFILL_CHECKPOINT_INVALID, "invalid Backfill checkpoint/frontier state").with_context("field", field);
}
#[cfg(test)]
#[path = "../unit_tests/checkpoint.rs"]
mod tests;

View File

@@ -0,0 +1,7 @@
// file: crates/ksp-job-backfill-lib/src/constants.rs
// version: 1
//! Backfill runtime-owned constants.
/// Owning tracing target for the concrete bounded RAW backfill runtime.
pub(crate) const TRACING_TARGET: &str = "ksp-job-backfill-lib";

View File

@@ -0,0 +1,485 @@
// file: crates/ksp-job-backfill-lib/src/conversion.rs
// version: 4
use sha2::Digest; // rust-rules: trait-import
/// KSP-owned source-independent RAW transaction format identifier produced by this Backfill vertical.
pub const RAW_TRANSACTION_FORMAT_ID: &str = "ksp.solana.raw_transaction";
/// Initial KSP-owned RAW transaction format version produced by this Backfill vertical.
pub const RAW_TRANSACTION_FORMAT_VERSION: u32 = 1;
const RAW_TRANSACTION_METHOD_CODE: &str = "getTransaction";
const RAW_TRANSACTION_OBSERVATION_CONTRACT_VERSION: u32 = 1;
const RAW_TRANSACTION_PROTOCOL_CODE: &str = "solana.http.json_rpc";
/// Complete in-memory RAW transaction acquisition ready for the later Store persistence tranche.
#[derive(Debug)]
pub struct BackfillRawAcquisition {
inner: Box<BackfillRawAcquisitionInner>,
}
#[derive(Debug)]
struct BackfillRawAcquisitionInner {
transaction: ksp_store_lib::RawTransaction,
observation: ksp_store_lib::RawTransactionObservation,
}
impl crate::BackfillRawAcquisition {
/// Returns the canonical RAW transaction produced from the typed Transport response.
#[must_use]
pub const fn transaction(&self) -> &ksp_store_lib::RawTransaction {
return &self.inner.transaction;
}
/// Returns the acquisition observation whose provenance records the actual successful endpoint.
#[must_use]
pub const fn observation(&self) -> &ksp_store_lib::RawTransactionObservation {
return &self.inner.observation;
}
/// Consumes the in-memory acquisition into the canonical transaction and its observation.
#[must_use]
pub fn into_parts(self) -> (ksp_store_lib::RawTransaction, ksp_store_lib::RawTransactionObservation) {
let inner = *self.inner;
return (inner.transaction, inner.observation);
}
}
/// Result of hydrating one deterministic Backfill candidate through observed `getTransaction`.
#[derive(Debug)]
pub enum BackfillHydrationOutcome {
/// The RPC returned one complete transaction and conversion produced canonical RAW plus provenance.
Available(crate::BackfillRawAcquisition),
/// The RPC returned JSON `null`; only the canonical transaction identity exists and no provenance is fabricated.
Missing(ksp_store_lib::RawTransactionReference),
}
impl crate::BackfillHydrationOutcome {
/// Returns the network-scoped transaction identity represented by this hydration outcome.
#[must_use]
pub fn reference(&self) -> &ksp_store_lib::RawTransactionReference {
return match self {
Self::Available(acquisition) => acquisition.transaction().reference(),
Self::Missing(reference) => reference,
};
}
/// Returns whether `getTransaction` returned JSON `null`.
#[must_use]
pub const fn is_missing(&self) -> bool {
return matches!(self, Self::Missing(_));
}
}
/// Hydrates one candidate with the typed observed Transport path and converts it to canonical RAW v1.
///
/// The caller supplies the local receipt timestamp because wall-clock ownership remains outside this
/// pure conversion tranche. Transport retains endpoint selection and retry. This function never
/// persists to Store; persistence begins in `pre.007`.
pub async fn hydrate_backfill_candidate(
transport: &ksp_onchain_transport_lib::HttpTransportPool,
request: &crate::BackfillRequest,
candidate: &crate::BackfillCandidate,
received_at: ksp_store_lib::RawTimestamp,
) -> ksp_core_lib::Result<crate::BackfillHydrationOutcome> {
let reference = canonical_reference(request, candidate);
let reference = match reference {
std::result::Result::Ok(reference) => reference,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let config = ksp_onchain_transport_lib::SolanaGetTransactionConfig::new(
std::option::Option::Some(request.commitment().transport()),
std::option::Option::Some(ksp_onchain_transport_lib::SolanaTransactionEncoding::Base64),
std::option::Option::Some(0),
);
let observed = transport.get_transaction_observed(request.role(), candidate.identity().signature().as_str(), std::option::Option::Some(&config)).await;
let observed = match observed {
std::result::Result::Ok(observed) => observed,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let provider = observed.provider().as_str().to_owned();
let endpoint = observed.endpoint_name().to_owned();
let transaction = observed.into_value();
let transaction = match transaction {
std::option::Option::Some(transaction) => transaction,
std::option::Option::None => return std::result::Result::Ok(crate::BackfillHydrationOutcome::Missing(reference)),
};
let fields = CanonicalTransactionFields {
slot: transaction.slot(),
block_time: transaction.block_time(),
transaction: transaction.transaction(),
meta: transaction.meta(),
version: transaction.version(),
transaction_index: transaction.transaction_index(),
};
let acquisition = convert_available_fields(request, reference, fields, provider.as_str(), endpoint.as_str(), received_at);
return match acquisition {
std::result::Result::Ok(acquisition) => std::result::Result::Ok(crate::BackfillHydrationOutcome::Available(acquisition)),
std::result::Result::Err(error) => std::result::Result::Err(error),
};
}
/// Decodes one validated Base58 signature to exactly 64 canonical bytes without a Solana SDK dependency.
pub(crate) fn decode_backfill_signature(signature: &crate::BackfillSignature) -> ksp_core_lib::Result<ksp_store_lib::RawTransactionSignature> {
let text = signature.as_str().as_bytes();
let mut decoded = [0_u8; 64];
let mut leading_zeroes = 0_usize;
for byte in text {
if *byte != b'1' {
break;
}
leading_zeroes += 1;
}
for byte in text {
let digit = match base58_digit(*byte) {
std::option::Option::Some(digit) => digit,
std::option::Option::None => return std::result::Result::Err(conversion_error("signature")),
};
let mut carry = u32::from(digit);
for output in decoded.iter_mut().rev() {
let value = (u32::from(*output) * 58) + carry;
*output = (value & 0xff) as u8;
carry = value >> 8;
}
if carry != 0 {
return std::result::Result::Err(conversion_error("signature"));
}
}
let significant_len = match decoded.iter().position(|byte| return *byte != 0) {
std::option::Option::Some(index) => decoded.len() - index,
std::option::Option::None => 0,
};
if leading_zeroes + significant_len != decoded.len() {
return std::result::Result::Err(conversion_error("signature"));
}
return std::result::Result::Ok(ksp_store_lib::RawTransactionSignature::new(decoded));
}
struct CanonicalTransactionFields<'a> {
slot: u64,
block_time: std::option::Option<i64>,
transaction: &'a ksp_onchain_transport_lib::SolanaEncodedTransaction,
meta: &'a ksp_onchain_transport_lib::SolanaWireField<serde_json::Value>,
version: &'a ksp_onchain_transport_lib::SolanaWireField<ksp_onchain_transport_lib::SolanaTransactionVersion>,
transaction_index: &'a ksp_onchain_transport_lib::SolanaWireField<u32>,
}
fn canonical_reference(request: &crate::BackfillRequest, candidate: &crate::BackfillCandidate) -> ksp_core_lib::Result<ksp_store_lib::RawTransactionReference> {
if candidate.identity().network() != request.network() {
return std::result::Result::Err(conversion_error("candidate.network"));
}
let signature = crate::decode_backfill_signature(candidate.identity().signature());
let signature = match signature {
std::result::Result::Ok(signature) => signature,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
return std::result::Result::Ok(ksp_store_lib::RawTransactionReference::new(candidate.identity().network().clone(), signature));
}
fn convert_available_fields(
request: &crate::BackfillRequest,
reference: ksp_store_lib::RawTransactionReference,
fields: CanonicalTransactionFields<'_>,
provider: &str,
endpoint: &str,
received_at: ksp_store_lib::RawTimestamp,
) -> ksp_core_lib::Result<crate::BackfillRawAcquisition> {
let block_time = convert_block_time(fields.block_time);
let block_time = match block_time {
std::result::Result::Ok(block_time) => block_time,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let bytes = canonical_payload_bytes(&fields);
let bytes = match bytes {
std::result::Result::Ok(bytes) => bytes,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let hash: [u8; 32] = sha2::Sha256::digest(bytes.as_slice()).into();
let format_id = ksp_store_lib::RawFormatId::new(crate::RAW_TRANSACTION_FORMAT_ID);
let format_id = match format_id {
std::result::Result::Ok(format_id) => format_id,
std::result::Result::Err(_) => return std::result::Result::Err(conversion_error("payload.format_id")),
};
let payload = ksp_store_lib::RawPayload::try_new(
format_id,
crate::RAW_TRANSACTION_FORMAT_VERSION,
bytes.into_boxed_slice(),
ksp_store_lib::RawContentHash::new(hash),
);
let payload = match payload {
std::result::Result::Ok(payload) => payload,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let provenance = build_provenance(request, provider, endpoint, received_at);
let provenance = match provenance {
std::result::Result::Ok(provenance) => provenance,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let observation_key = observation_key(request, &reference, provider, endpoint);
let transaction = ksp_store_lib::RawTransaction::new(reference.clone(), fields.slot, block_time, payload);
let observation = ksp_store_lib::RawTransactionObservation::new(observation_key, reference, provenance);
return std::result::Result::Ok(crate::BackfillRawAcquisition { inner: Box::new(BackfillRawAcquisitionInner { transaction, observation }) });
}
fn convert_block_time(value: std::option::Option<i64>) -> ksp_core_lib::Result<std::option::Option<ksp_store_lib::RawTimestamp>> {
let seconds = match value {
std::option::Option::Some(seconds) => seconds,
std::option::Option::None => return std::result::Result::Ok(std::option::Option::None),
};
let seconds = match u64::try_from(seconds) {
std::result::Result::Ok(seconds) => seconds,
std::result::Result::Err(_) => return std::result::Result::Err(conversion_error("block_time")),
};
let millis = match seconds.checked_mul(1_000) {
std::option::Option::Some(millis) => millis,
std::option::Option::None => return std::result::Result::Err(conversion_error("block_time")),
};
let timestamp = ksp_store_lib::RawTimestamp::from_unix_millis(millis);
return match timestamp {
std::result::Result::Ok(timestamp) => std::result::Result::Ok(std::option::Option::Some(timestamp)),
std::result::Result::Err(_) => std::result::Result::Err(conversion_error("block_time")),
};
}
fn canonical_payload_bytes(fields: &CanonicalTransactionFields<'_>) -> ksp_core_lib::Result<std::vec::Vec<u8>> {
let (transaction_data, transaction_encoding) = match fields.transaction {
ksp_onchain_transport_lib::SolanaEncodedTransaction::Binary { data, encoding } => {
if *encoding != ksp_onchain_transport_lib::SolanaTransactionBinaryEncoding::Base64 {
return std::result::Result::Err(conversion_error("transaction.encoding"));
}
(data.as_str(), "base64")
},
ksp_onchain_transport_lib::SolanaEncodedTransaction::LegacyBinary(_) | ksp_onchain_transport_lib::SolanaEncodedTransaction::Json(_) => {
return std::result::Result::Err(conversion_error("transaction.encoding"));
},
};
let mut bytes = std::vec::Vec::new();
bytes.extend_from_slice(b"{\"transaction\":[");
if let std::result::Result::Err(error) = append_json_string(&mut bytes, transaction_data) {
return std::result::Result::Err(error);
}
bytes.push(b',');
if let std::result::Result::Err(error) = append_json_string(&mut bytes, transaction_encoding) {
return std::result::Result::Err(error);
}
bytes.push(b']');
if let std::result::Result::Err(error) = append_wire_value(&mut bytes, "meta", fields.meta, |output, value| return append_canonical_json(output, value)) {
return std::result::Result::Err(error);
}
let version_result = append_wire_value(&mut bytes, "version", fields.version, |output, value| {
return match value {
ksp_onchain_transport_lib::SolanaTransactionVersion::Legacy => append_json_string(output, "legacy"),
ksp_onchain_transport_lib::SolanaTransactionVersion::Number(number) => {
output.extend_from_slice(number.to_string().as_bytes());
std::result::Result::Ok(())
},
};
});
if let std::result::Result::Err(error) = version_result {
return std::result::Result::Err(error);
}
let transaction_index_result = append_wire_value(&mut bytes, "transactionIndex", fields.transaction_index, |output, value| {
output.extend_from_slice(value.to_string().as_bytes());
return std::result::Result::Ok(());
});
if let std::result::Result::Err(error) = transaction_index_result {
return std::result::Result::Err(error);
}
bytes.push(b'}');
return std::result::Result::Ok(bytes);
}
fn append_wire_value<T, F>(
output: &mut std::vec::Vec<u8>,
key: &str,
field: &ksp_onchain_transport_lib::SolanaWireField<T>,
mut append_value: F,
) -> ksp_core_lib::Result<()>
where
F: FnMut(&mut std::vec::Vec<u8>, &T) -> ksp_core_lib::Result<()>,
{
return match field {
ksp_onchain_transport_lib::SolanaWireField::Omitted => std::result::Result::Ok(()),
ksp_onchain_transport_lib::SolanaWireField::Null => {
output.push(b',');
let key_result = append_json_string(output, key);
if let std::result::Result::Err(error) = key_result {
return std::result::Result::Err(error);
}
output.extend_from_slice(b":null");
std::result::Result::Ok(())
},
ksp_onchain_transport_lib::SolanaWireField::Value(value) => {
output.push(b',');
let key_result = append_json_string(output, key);
if let std::result::Result::Err(error) = key_result {
return std::result::Result::Err(error);
}
output.push(b':');
append_value(output, value)
},
};
}
fn append_canonical_json(output: &mut std::vec::Vec<u8>, value: &serde_json::Value) -> ksp_core_lib::Result<()> {
return match value {
serde_json::Value::Null => {
output.extend_from_slice(b"null");
std::result::Result::Ok(())
},
serde_json::Value::Bool(value) => {
if *value {
output.extend_from_slice(b"true");
} else {
output.extend_from_slice(b"false");
}
std::result::Result::Ok(())
},
serde_json::Value::Number(value) => {
output.extend_from_slice(value.to_string().as_bytes());
std::result::Result::Ok(())
},
serde_json::Value::String(value) => append_json_string(output, value.as_str()),
serde_json::Value::Array(values) => {
output.push(b'[');
for (index, item) in values.iter().enumerate() {
if index != 0 {
output.push(b',');
}
let item_result = append_canonical_json(output, item);
if let std::result::Result::Err(error) = item_result {
return std::result::Result::Err(error);
}
}
output.push(b']');
std::result::Result::Ok(())
},
serde_json::Value::Object(values) => {
output.push(b'{');
let mut keys = values.keys().collect::<std::vec::Vec<_>>();
keys.sort_unstable();
for (index, key) in keys.iter().enumerate() {
if index != 0 {
output.push(b',');
}
let key_result = append_json_string(output, key.as_str());
if let std::result::Result::Err(error) = key_result {
return std::result::Result::Err(error);
}
output.push(b':');
let item = values.get(key.as_str());
let item = match item {
std::option::Option::Some(item) => item,
std::option::Option::None => return std::result::Result::Err(conversion_error("payload.meta")),
};
let item_result = append_canonical_json(output, item);
if let std::result::Result::Err(error) = item_result {
return std::result::Result::Err(error);
}
}
output.push(b'}');
std::result::Result::Ok(())
},
};
}
fn append_json_string(output: &mut std::vec::Vec<u8>, value: &str) -> ksp_core_lib::Result<()> {
let encoded = serde_json::to_vec(value);
return match encoded {
std::result::Result::Ok(encoded) => {
output.extend_from_slice(encoded.as_slice());
std::result::Result::Ok(())
},
std::result::Result::Err(_) => std::result::Result::Err(conversion_error("payload.json")),
};
}
fn build_provenance(
request: &crate::BackfillRequest,
provider: &str,
endpoint: &str,
received_at: ksp_store_lib::RawTimestamp,
) -> ksp_core_lib::Result<ksp_store_lib::RawAcquisitionProvenance> {
let provider = match provenance_code(provider, "provenance.provider") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let protocol = match provenance_code(RAW_TRANSACTION_PROTOCOL_CODE, "provenance.protocol") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let method = match provenance_code(RAW_TRANSACTION_METHOD_CODE, "provenance.method") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let endpoint = match provenance_code(endpoint, "provenance.endpoint") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let commitment = match provenance_code(request.commitment().code(), "provenance.commitment") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let capture_session = match provenance_code(request.job_id().as_str(), "provenance.capture_session") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let provenance = ksp_store_lib::RawAcquisitionProvenance::new(provider, protocol, method, ksp_store_lib::RawAcquisitionOrigin::Backfill, received_at)
.with_endpoint_id(endpoint)
.with_commitment(commitment)
.with_capture_session_id(capture_session);
return std::result::Result::Ok(provenance);
}
fn provenance_code(value: &str, field: &'static str) -> ksp_core_lib::Result<ksp_store_lib::RawProvenanceCode> {
let code = ksp_store_lib::RawProvenanceCode::new(value);
return match code {
std::result::Result::Ok(code) => std::result::Result::Ok(code),
std::result::Result::Err(_) => std::result::Result::Err(conversion_error(field)),
};
}
fn observation_key(
request: &crate::BackfillRequest,
reference: &ksp_store_lib::RawTransactionReference,
provider: &str,
endpoint: &str,
) -> ksp_store_lib::RawObservationKey {
let mut hasher = sha2::Sha256::new();
hasher.update(b"ksp.job.backfill.raw_transaction_observation.v1\0");
hash_bytes(&mut hasher, request.job_id().as_str().as_bytes());
hash_bytes(&mut hasher, request.scope_fingerprint().as_bytes());
hash_bytes(&mut hasher, reference.signature().as_bytes());
hash_bytes(&mut hasher, provider.as_bytes());
hash_bytes(&mut hasher, endpoint.as_bytes());
hash_bytes(&mut hasher, request.commitment().code().as_bytes());
hasher.update(RAW_TRANSACTION_OBSERVATION_CONTRACT_VERSION.to_be_bytes());
let bytes: [u8; 32] = hasher.finalize().into();
return ksp_store_lib::RawObservationKey::new(bytes);
}
fn hash_bytes(hasher: &mut sha2::Sha256, value: &[u8]) {
hasher.update((value.len() as u64).to_be_bytes());
hasher.update(value);
}
fn base58_digit(byte: u8) -> std::option::Option<u8> {
return match byte {
b'1'..=b'9' => std::option::Option::Some(byte - b'1'),
b'A'..=b'H' => std::option::Option::Some((byte - b'A') + 9),
b'J'..=b'N' => std::option::Option::Some((byte - b'J') + 17),
b'P'..=b'Z' => std::option::Option::Some((byte - b'P') + 22),
b'a'..=b'k' => std::option::Option::Some((byte - b'a') + 33),
b'm'..=b'z' => std::option::Option::Some((byte - b'm') + 44),
_ => std::option::Option::None,
};
}
fn conversion_error(field: &'static str) -> ksp_core_lib::Error {
return ksp_core_lib::Error::new(crate::ERROR_CODE_BACKFILL_RAW_CONVERSION_INVALID, "invalid deterministic Backfill RAW conversion")
.with_context("field", field);
}
#[cfg(test)]
#[path = "../unit_tests/conversion.rs"]
mod tests;

View File

@@ -0,0 +1,492 @@
// file: crates/ksp-job-backfill-lib/src/discovery.rs
// version: 5
/// Network-scoped identity of one discovered transaction candidate before canonical signature decoding.
#[derive(Clone, Debug, Eq, Hash, PartialEq)]
pub struct BackfillCandidateIdentity {
network: ksp_store_lib::RawNetworkId,
signature: crate::BackfillSignature,
}
impl crate::BackfillCandidateIdentity {
/// Creates one candidate identity from its logical network and encoded transaction signature.
#[must_use]
pub fn new(network: ksp_store_lib::RawNetworkId, signature: crate::BackfillSignature) -> Self {
return Self { network, signature };
}
/// Returns the logical network that scopes signature uniqueness.
#[must_use]
pub const fn network(&self) -> &ksp_store_lib::RawNetworkId {
return &self.network;
}
/// Returns the validated encoded transaction signature.
#[must_use]
pub const fn signature(&self) -> &crate::BackfillSignature {
return &self.signature;
}
}
/// One deterministic transaction candidate produced by bounded discovery.
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct BackfillCandidate {
identity: crate::BackfillCandidateIdentity,
discovered_slot: std::option::Option<u64>,
}
impl crate::BackfillCandidate {
/// Creates one candidate from a network-scoped identity and optional discovery slot.
#[must_use]
pub fn new(identity: crate::BackfillCandidateIdentity, discovered_slot: std::option::Option<u64>) -> Self {
return Self { identity, discovered_slot };
}
/// Returns the network-scoped candidate identity.
#[must_use]
pub const fn identity(&self) -> &crate::BackfillCandidateIdentity {
return &self.identity;
}
/// Returns the slot reported by address discovery, when one was available.
#[must_use]
pub const fn discovered_slot(&self) -> std::option::Option<u64> {
return self.discovered_slot;
}
}
/// Reason bounded candidate discovery stopped.
#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)]
pub enum BackfillDiscoveryBoundary {
/// Explicit signatures required no RPC pagination.
ExplicitInput,
/// The caller's requested candidate limit was satisfied.
CandidateLimit,
/// The RPC returned a short or empty page, reaching the bounded remote history boundary.
RpcBoundary,
/// The maximum page count was exhausted before the requested address window completed.
PageLimit,
/// An `AfterAddress` scan exhausted its page bound before the exclusive anchor boundary was reached.
AfterAnchorNotReached,
}
impl crate::BackfillDiscoveryBoundary {
/// Returns the stable diagnostic code for this boundary.
#[must_use]
pub const fn code(self) -> &'static str {
return match self {
Self::ExplicitInput => "explicit_input",
Self::CandidateLimit => "candidate_limit",
Self::RpcBoundary => "rpc_boundary",
Self::PageLimit => "page_limit",
Self::AfterAnchorNotReached => "after_anchor_not_reached",
};
}
/// Returns whether the discovery result is partial and must not advance a future checkpoint beyond the unresolved gap.
#[must_use]
pub const fn is_partial(self) -> bool {
return matches!(self, Self::PageLimit | Self::AfterAnchorNotReached);
}
}
/// Complete bounded output of one candidate discovery pass.
pub struct BackfillDiscovery {
network: ksp_store_lib::RawNetworkId,
scope_fingerprint: crate::BackfillScopeFingerprint,
candidates: std::vec::Vec<crate::BackfillCandidate>,
pages_fetched: usize,
boundary: crate::BackfillDiscoveryBoundary,
}
impl crate::BackfillDiscovery {
/// Creates one internally validated bounded discovery value.
pub(crate) fn new(
network: ksp_store_lib::RawNetworkId,
scope_fingerprint: crate::BackfillScopeFingerprint,
candidates: std::vec::Vec<crate::BackfillCandidate>,
pages_fetched: usize,
boundary: crate::BackfillDiscoveryBoundary,
) -> Self {
return Self { network, scope_fingerprint, candidates, pages_fetched, boundary };
}
/// Returns the logical network shared by every candidate identity.
#[must_use]
pub const fn network(&self) -> &ksp_store_lib::RawNetworkId {
return &self.network;
}
/// Returns the semantic scope fingerprint copied from the validated request.
#[must_use]
pub const fn scope_fingerprint(&self) -> crate::BackfillScopeFingerprint {
return self.scope_fingerprint;
}
/// Returns discovered candidates in deterministic processing order.
#[must_use]
pub fn candidates(&self) -> &[crate::BackfillCandidate] {
return self.candidates.as_slice();
}
/// Returns the number of `getSignaturesForAddress` pages fetched by this pass.
#[must_use]
pub const fn pages_fetched(&self) -> usize {
return self.pages_fetched;
}
/// Returns the reason discovery stopped.
#[must_use]
pub const fn boundary(&self) -> crate::BackfillDiscoveryBoundary {
return self.boundary;
}
/// Returns whether the bounded pass stopped before a complete requested discovery window was proven.
#[must_use]
pub const fn is_partial(&self) -> bool {
return self.boundary.is_partial();
}
}
impl std::fmt::Debug for crate::BackfillDiscovery {
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
return formatter
.debug_struct("BackfillDiscovery")
.field("network", &self.network)
.field("scope_fingerprint", &self.scope_fingerprint)
.field("candidate_count", &self.candidates.len())
.field("pages_fetched", &self.pages_fetched)
.field("boundary", &self.boundary)
.finish();
}
}
struct SignaturePageEntry {
signature: std::string::String,
slot: u64,
}
type SignaturePageFuture<'a> = std::pin::Pin<std::boxed::Box<dyn std::future::Future<Output = ksp_core_lib::Result<std::vec::Vec<SignaturePageEntry>>> + 'a>>;
trait SignaturePageSource {
fn fetch_signature_page<'a>(
&'a self,
role: &'a ksp_onchain_transport_lib::HttpRoleName,
address: &'a ksp_core_lib::Pubkey,
config: ksp_onchain_transport_lib::SolanaSignaturesForAddressConfig,
) -> SignaturePageFuture<'a>;
}
impl SignaturePageSource for ksp_onchain_transport_lib::HttpTransportPool {
fn fetch_signature_page<'a>(
&'a self,
role: &'a ksp_onchain_transport_lib::HttpRoleName,
address: &'a ksp_core_lib::Pubkey,
config: ksp_onchain_transport_lib::SolanaSignaturesForAddressConfig,
) -> SignaturePageFuture<'a> {
return std::boxed::Box::pin(async move {
let result = self.get_signatures_for_address(role, address, std::option::Option::Some(&config)).await;
let infos = match result {
std::result::Result::Ok(infos) => infos,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let mut entries = std::vec::Vec::with_capacity(infos.len());
for info in infos {
entries.push(SignaturePageEntry { signature: info.signature().to_owned(), slot: info.slot() });
}
return std::result::Result::Ok(entries);
});
}
}
/// Discovers one bounded deterministic candidate set using only the typed KSP Transport wrapper.
///
/// Provider, endpoint and protocol selection remain entirely owned by Transport. Candidate identity
/// is scoped only by the request network plus transaction signature.
pub async fn discover_backfill_candidates(
transport: &ksp_onchain_transport_lib::HttpTransportPool,
request: &crate::BackfillRequest,
) -> ksp_core_lib::Result<crate::BackfillDiscovery> {
ksp_logging_lib::debug!(
target: crate::TRACING_TARGET,
job_id = request.job_id().as_str(),
network = request.network().as_str(),
scope = request.scope().kind().code(),
page_size = request.page_size(),
max_pages = request.max_pages(),
max_candidates = request.max_candidates(),
"starting bounded Backfill candidate discovery"
);
let result = discover_with_source(transport, request, std::option::Option::None).await;
if let std::result::Result::Ok(discovery) = &result {
ksp_logging_lib::debug!(
target: crate::TRACING_TARGET,
job_id = request.job_id().as_str(),
network = request.network().as_str(),
scope = request.scope().kind().code(),
pages_fetched = discovery.pages_fetched(),
candidate_count = discovery.candidates().len(),
boundary = discovery.boundary().code(),
partial = discovery.is_partial(),
"completed bounded Backfill candidate discovery"
);
}
return result;
}
/// Discovers one bounded candidate set while allowing cooperative cancellation of in-flight RPC futures.
pub(crate) async fn discover_backfill_candidates_cancellable(
transport: &ksp_onchain_transport_lib::HttpTransportPool,
request: &crate::BackfillRequest,
cancellation: &crate::BackfillCancellationSignal,
) -> ksp_core_lib::Result<crate::BackfillDiscovery> {
return discover_with_source(transport, request, std::option::Option::Some(cancellation)).await;
}
async fn discover_with_source<S>(
source: &S,
request: &crate::BackfillRequest,
cancellation: std::option::Option<&crate::BackfillCancellationSignal>,
) -> ksp_core_lib::Result<crate::BackfillDiscovery>
where
S: SignaturePageSource,
{
if cancellation.is_some_and(crate::BackfillCancellationSignal::is_requested) {
return std::result::Result::Err(cancelled_error());
}
return match request.scope().kind() {
crate::BackfillScopeKind::ExplicitSignatures => discover_explicit(request),
crate::BackfillScopeKind::LatestAddress | crate::BackfillScopeKind::BeforeAddress => discover_older(source, request, cancellation).await,
crate::BackfillScopeKind::AfterAddress => discover_after(source, request, cancellation).await,
};
}
fn discover_explicit(request: &crate::BackfillRequest) -> ksp_core_lib::Result<crate::BackfillDiscovery> {
let signatures = match request.scope().signatures() {
std::option::Option::Some(signatures) => signatures,
std::option::Option::None => return std::result::Result::Err(discovery_invalid("scope.signatures")),
};
let mut candidates = std::vec::Vec::with_capacity(signatures.len());
for signature in signatures {
let identity = crate::BackfillCandidateIdentity::new(request.network().clone(), signature.clone());
candidates.push(crate::BackfillCandidate::new(identity, std::option::Option::None));
}
return std::result::Result::Ok(crate::BackfillDiscovery::new(
request.network().clone(),
request.scope_fingerprint(),
candidates,
0,
crate::BackfillDiscoveryBoundary::ExplicitInput,
));
}
async fn discover_older<S>(
source: &S,
request: &crate::BackfillRequest,
cancellation: std::option::Option<&crate::BackfillCancellationSignal>,
) -> ksp_core_lib::Result<crate::BackfillDiscovery>
where
S: SignaturePageSource,
{
let address = match request.scope().address() {
std::option::Option::Some(address) => address,
std::option::Option::None => return std::result::Result::Err(discovery_invalid("scope.address")),
};
let mut before = match request.scope().kind() {
crate::BackfillScopeKind::BeforeAddress => crate::resume_before_cursor(request),
crate::BackfillScopeKind::LatestAddress => std::option::Option::None,
crate::BackfillScopeKind::AfterAddress | crate::BackfillScopeKind::ExplicitSignatures => {
return std::result::Result::Err(discovery_invalid("scope.kind"));
},
};
let mut candidates = std::vec::Vec::with_capacity(request.max_candidates());
let mut seen = std::collections::HashSet::<crate::BackfillSignature>::with_capacity(request.max_candidates());
let mut pages_fetched = 0_usize;
let boundary = loop {
if candidates.len() >= request.max_candidates() {
break crate::BackfillDiscoveryBoundary::CandidateLimit;
}
if pages_fetched >= request.max_pages() {
break crate::BackfillDiscoveryBoundary::PageLimit;
}
let remaining = request.max_candidates() - candidates.len();
let page_limit = std::cmp::min(request.page_size(), remaining);
let config = ksp_onchain_transport_lib::SolanaSignaturesForAddressConfig::new(
before.clone(),
std::option::Option::None,
std::option::Option::Some(page_limit),
std::option::Option::Some(request.commitment().transport()),
request.min_context_slot(),
);
let page_result = fetch_signature_page(source, request.role(), address, config, cancellation).await;
let page = match page_result {
std::result::Result::Ok(page) => page,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
pages_fetched += 1;
if page.len() > page_limit {
return std::result::Result::Err(discovery_invalid("page.length"));
}
let page_len = page.len();
let next_before = match page.last() {
std::option::Option::Some(entry) => {
let signature_result = validated_signature(entry.signature.as_str());
match signature_result {
std::result::Result::Ok(signature) => std::option::Option::Some(signature),
std::result::Result::Err(error) => return std::result::Result::Err(error),
}
},
std::option::Option::None => std::option::Option::None,
};
for entry in page {
let signature_result = validated_signature(entry.signature.as_str());
let signature = match signature_result {
std::result::Result::Ok(signature) => signature,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
if seen.insert(signature.clone()) {
let identity = crate::BackfillCandidateIdentity::new(request.network().clone(), signature);
candidates.push(crate::BackfillCandidate::new(identity, std::option::Option::Some(entry.slot)));
if candidates.len() >= request.max_candidates() {
break;
}
}
}
if page_len < page_limit {
break crate::BackfillDiscoveryBoundary::RpcBoundary;
}
let next_before = match next_before {
std::option::Option::Some(next_before) => next_before,
std::option::Option::None => break crate::BackfillDiscoveryBoundary::RpcBoundary,
};
if before.as_deref() == std::option::Option::Some(next_before.as_str()) {
return std::result::Result::Err(discovery_stalled());
}
before = std::option::Option::Some(next_before.as_str().to_owned());
};
return std::result::Result::Ok(crate::BackfillDiscovery::new(request.network().clone(), request.scope_fingerprint(), candidates, pages_fetched, boundary));
}
async fn discover_after<S>(
source: &S,
request: &crate::BackfillRequest,
cancellation: std::option::Option<&crate::BackfillCancellationSignal>,
) -> ksp_core_lib::Result<crate::BackfillDiscovery>
where
S: SignaturePageSource,
{
let address = match request.scope().address() {
std::option::Option::Some(address) => address,
std::option::Option::None => return std::result::Result::Err(discovery_invalid("scope.address")),
};
let anchor = match request.scope().anchor() {
std::option::Option::Some(anchor) => anchor,
std::option::Option::None => return std::result::Result::Err(discovery_invalid("scope.anchor")),
};
let until = anchor.as_str().to_owned();
let mut before = std::option::Option::<std::string::String>::None;
let mut nearest = std::collections::VecDeque::<crate::BackfillCandidate>::with_capacity(request.max_candidates());
let mut seen = std::collections::HashSet::<crate::BackfillSignature>::new();
let mut pages_fetched = 0_usize;
let boundary = loop {
if pages_fetched >= request.max_pages() {
break crate::BackfillDiscoveryBoundary::AfterAnchorNotReached;
}
let config = ksp_onchain_transport_lib::SolanaSignaturesForAddressConfig::new(
before.clone(),
std::option::Option::Some(until.clone()),
std::option::Option::Some(request.page_size()),
std::option::Option::Some(request.commitment().transport()),
request.min_context_slot(),
);
let page_result = fetch_signature_page(source, request.role(), address, config, cancellation).await;
let page = match page_result {
std::result::Result::Ok(page) => page,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
pages_fetched += 1;
if page.len() > request.page_size() {
return std::result::Result::Err(discovery_invalid("page.length"));
}
let page_len = page.len();
let next_before = match page.last() {
std::option::Option::Some(entry) => {
let signature_result = validated_signature(entry.signature.as_str());
match signature_result {
std::result::Result::Ok(signature) => std::option::Option::Some(signature),
std::result::Result::Err(error) => return std::result::Result::Err(error),
}
},
std::option::Option::None => std::option::Option::None,
};
for entry in page {
let signature_result = validated_signature(entry.signature.as_str());
let signature = match signature_result {
std::result::Result::Ok(signature) => signature,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
if seen.insert(signature.clone()) {
let identity = crate::BackfillCandidateIdentity::new(request.network().clone(), signature);
nearest.push_back(crate::BackfillCandidate::new(identity, std::option::Option::Some(entry.slot)));
if nearest.len() > request.max_candidates() {
nearest.pop_front();
}
}
}
if page_len < request.page_size() {
break crate::BackfillDiscoveryBoundary::RpcBoundary;
}
let next_before = match next_before {
std::option::Option::Some(next_before) => next_before,
std::option::Option::None => break crate::BackfillDiscoveryBoundary::RpcBoundary,
};
if before.as_deref() == std::option::Option::Some(next_before.as_str()) {
return std::result::Result::Err(discovery_stalled());
}
before = std::option::Option::Some(next_before.as_str().to_owned());
};
return std::result::Result::Ok(crate::BackfillDiscovery::new(
request.network().clone(),
request.scope_fingerprint(),
nearest.into_iter().collect(),
pages_fetched,
boundary,
));
}
async fn fetch_signature_page<S>(
source: &S,
role: &ksp_onchain_transport_lib::HttpRoleName,
address: &ksp_core_lib::Pubkey,
config: ksp_onchain_transport_lib::SolanaSignaturesForAddressConfig,
cancellation: std::option::Option<&crate::BackfillCancellationSignal>,
) -> ksp_core_lib::Result<std::vec::Vec<SignaturePageEntry>>
where
S: SignaturePageSource,
{
let future = source.fetch_signature_page(role, address, config);
return match cancellation {
std::option::Option::Some(signal) => signal.run_cancellable(future).await,
std::option::Option::None => future.await,
};
}
fn cancelled_error() -> ksp_core_lib::Error {
return ksp_core_lib::Error::new(crate::ERROR_CODE_BACKFILL_CANCELLED, "Backfill discovery cancelled before RPC completion");
}
fn validated_signature(value: &str) -> ksp_core_lib::Result<crate::BackfillSignature> {
return crate::BackfillSignature::new(value.to_owned());
}
fn discovery_invalid(field: &'static str) -> ksp_core_lib::Error {
return ksp_core_lib::Error::new(crate::ERROR_CODE_BACKFILL_DISCOVERY_INVALID, "invalid bounded Backfill discovery state").with_context("field", field);
}
fn discovery_stalled() -> ksp_core_lib::Error {
return ksp_core_lib::Error::new(crate::ERROR_CODE_BACKFILL_DISCOVERY_STALLED, "Backfill discovery cursor did not advance").with_context("field", "before");
}
#[cfg(test)]
#[path = "../unit_tests/discovery.rs"]
mod tests;

View File

@@ -0,0 +1,24 @@
// file: crates/ksp-job-backfill-lib/src/error.rs
// version: 5
/// Error code used when one Backfill checkpoint/frontier is incompatible with the current Job or semantic scope.
pub const ERROR_CODE_BACKFILL_CHECKPOINT_INVALID: ksp_core_lib::ErrorCode = ksp_core_lib::ErrorCode::new("job_backfill", "checkpoint_invalid");
/// Error code used when a signature page violates a bounded discovery invariant.
pub const ERROR_CODE_BACKFILL_DISCOVERY_INVALID: ksp_core_lib::ErrorCode = ksp_core_lib::ErrorCode::new("job_backfill", "discovery_invalid");
/// Error code used when paginated discovery cannot advance its exclusive RPC cursor safely.
pub const ERROR_CODE_BACKFILL_DISCOVERY_STALLED: ksp_core_lib::ErrorCode = ksp_core_lib::ErrorCode::new("job_backfill", "discovery_stalled");
/// Error code used when bounded concurrent candidate execution violates its internal admission or clock invariants.
pub const ERROR_CODE_BACKFILL_EXECUTION_INVALID: ksp_core_lib::ErrorCode = ksp_core_lib::ErrorCode::new("job_backfill", "execution_invalid");
/// Error code used when Store persistence returns an impossible Backfill state or targets a different network.
pub const ERROR_CODE_BACKFILL_PERSISTENCE_INVALID: ksp_core_lib::ErrorCode = ksp_core_lib::ErrorCode::new("job_backfill", "persistence_invalid");
/// Error code used when deterministic Transport-to-RAW conversion violates the v1 contract.
pub const ERROR_CODE_BACKFILL_RAW_CONVERSION_INVALID: ksp_core_lib::ErrorCode = ksp_core_lib::ErrorCode::new("job_backfill", "raw_conversion_invalid");
/// Error code used when one Backfill request violates its bounded admission contract.
pub const ERROR_CODE_BACKFILL_REQUEST_INVALID: ksp_core_lib::ErrorCode = ksp_core_lib::ErrorCode::new("job_backfill", "request_invalid");
/// Error code used when the concrete Backfill runtime reaches an impossible lifecycle or notification state.
pub const ERROR_CODE_BACKFILL_RUNTIME_INVALID: ksp_core_lib::ErrorCode = ksp_core_lib::ErrorCode::new("job_backfill", "runtime_invalid");
/// Error code used when one transaction signature text violates the bounded Base58-shape contract.
pub const ERROR_CODE_BACKFILL_SIGNATURE_INVALID: ksp_core_lib::ErrorCode = ksp_core_lib::ErrorCode::new("job_backfill", "signature_invalid");
/// Error code used internally when cooperative cancellation wins before durable Store submission.
pub(crate) const ERROR_CODE_BACKFILL_CANCELLED: ksp_core_lib::ErrorCode = ksp_core_lib::ErrorCode::new("job_backfill", "cancelled");

View File

@@ -0,0 +1,660 @@
// file: crates/ksp-job-backfill-lib/src/execution.rs
// version: 3
use futures_util::StreamExt; // rust-rules: trait-import
/// Bounded result of one concurrent Backfill candidate execution pass.
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct BackfillExecutionBatch {
candidate_count: usize,
admitted_count: usize,
finished_count: usize,
durable_count: usize,
inserted_count: usize,
already_present_count: usize,
purged_count: usize,
missing_count: usize,
conflict_count: usize,
observation_inserted_count: usize,
observation_already_present_count: usize,
cancelled_count: usize,
hole_count: usize,
maximum_in_flight: usize,
local_contiguous_completed: usize,
discovery_partial: bool,
cancelled: bool,
checkpoint: crate::BackfillCheckpoint,
failure_code: std::option::Option<ksp_core_lib::ErrorCode>,
}
impl crate::BackfillExecutionBatch {
/// Returns the number of candidates present in the bounded discovery result.
#[must_use]
pub const fn candidate_count(&self) -> usize {
return self.candidate_count;
}
/// Returns the number of candidates admitted into hydration during this execution pass.
#[must_use]
pub const fn admitted_count(&self) -> usize {
return self.admitted_count;
}
/// Returns the number of admitted candidates whose hydration/persistence future reached a known outcome.
#[must_use]
pub const fn finished_count(&self) -> usize {
return self.finished_count;
}
/// Returns the number of candidates whose Store outcome is durable for checkpoint advancement.
#[must_use]
pub const fn durable_count(&self) -> usize {
return self.durable_count;
}
/// Returns the number of newly inserted canonical RAW entities.
#[must_use]
pub const fn inserted_count(&self) -> usize {
return self.inserted_count;
}
/// Returns the number of canonical RAW entities already durable.
#[must_use]
pub const fn already_present_count(&self) -> usize {
return self.already_present_count;
}
/// Returns the number of purge tombstones respected by normal persistence.
#[must_use]
pub const fn purged_count(&self) -> usize {
return self.purged_count;
}
/// Returns the number of candidates whose `getTransaction` result was missing.
#[must_use]
pub const fn missing_count(&self) -> usize {
return self.missing_count;
}
/// Returns the number of Store content conflicts.
#[must_use]
pub const fn conflict_count(&self) -> usize {
return self.conflict_count;
}
/// Returns the number of newly inserted acquisition observations.
#[must_use]
pub const fn observation_inserted_count(&self) -> usize {
return self.observation_inserted_count;
}
/// Returns the number of acquisition observations already durable.
#[must_use]
pub const fn observation_already_present_count(&self) -> usize {
return self.observation_already_present_count;
}
/// Returns the number of admitted candidates cancelled before Store submission.
#[must_use]
pub const fn cancelled_count(&self) -> usize {
return self.cancelled_count;
}
/// Returns the number of known candidate outcomes that intentionally block the contiguous frontier.
#[must_use]
pub const fn hole_count(&self) -> usize {
return self.hole_count;
}
/// Returns the greatest number of candidate futures simultaneously in flight.
#[must_use]
pub const fn maximum_in_flight(&self) -> usize {
return self.maximum_in_flight;
}
/// Returns the durable contiguous prefix completed inside the current discovery result.
#[must_use]
pub const fn local_contiguous_completed(&self) -> usize {
return self.local_contiguous_completed;
}
/// Returns the safe caller-owned checkpoint after draining all work already admitted.
#[must_use]
pub const fn checkpoint(&self) -> &crate::BackfillCheckpoint {
return &self.checkpoint;
}
/// Returns the first stable fatal error code that stopped new admissions, when one occurred.
#[must_use]
pub const fn failure_code(&self) -> std::option::Option<ksp_core_lib::ErrorCode> {
return self.failure_code;
}
/// Returns whether cooperative cancellation stopped admissions or cancelled an admitted pre-Store operation.
#[must_use]
pub const fn was_cancelled(&self) -> bool {
return self.cancelled;
}
/// Returns whether discovery or candidate outcomes left the pass incomplete.
#[must_use]
pub const fn is_partial(&self) -> bool {
return self.discovery_partial || self.hole_count != 0 || self.failure_code.is_some() || self.cancelled;
}
}
/// Internal latest-value execution facts published after each known candidate completion.
#[derive(Clone, Debug, Eq, PartialEq)]
pub(crate) struct BackfillExecutionProgress {
admitted_count: usize,
finished_count: usize,
inserted_count: usize,
already_present_count: usize,
purged_count: usize,
missing_count: usize,
conflict_count: usize,
observation_inserted_count: usize,
observation_already_present_count: usize,
cancelled_count: usize,
hole_count: usize,
maximum_in_flight: usize,
contiguous_completed: usize,
checkpoint: crate::BackfillCheckpoint,
}
impl crate::BackfillExecutionProgress {
/// Returns the cumulative admission count.
pub(crate) const fn admitted_count(&self) -> usize {
return self.admitted_count;
}
/// Returns the cumulative finished count.
pub(crate) const fn finished_count(&self) -> usize {
return self.finished_count;
}
/// Returns the cumulative inserted entity count.
pub(crate) const fn inserted_count(&self) -> usize {
return self.inserted_count;
}
/// Returns the cumulative already-present entity count.
pub(crate) const fn already_present_count(&self) -> usize {
return self.already_present_count;
}
/// Returns the cumulative purged entity count.
pub(crate) const fn purged_count(&self) -> usize {
return self.purged_count;
}
/// Returns the cumulative missing count.
pub(crate) const fn missing_count(&self) -> usize {
return self.missing_count;
}
/// Returns the cumulative conflict count.
pub(crate) const fn conflict_count(&self) -> usize {
return self.conflict_count;
}
/// Returns the cumulative inserted observation count.
pub(crate) const fn observation_inserted_count(&self) -> usize {
return self.observation_inserted_count;
}
/// Returns the cumulative already-present observation count.
pub(crate) const fn observation_already_present_count(&self) -> usize {
return self.observation_already_present_count;
}
/// Returns the cumulative cancelled candidate count.
pub(crate) const fn cancelled_count(&self) -> usize {
return self.cancelled_count;
}
/// Returns the cumulative hole count.
pub(crate) const fn hole_count(&self) -> usize {
return self.hole_count;
}
/// Returns the maximum observed in-flight count.
pub(crate) const fn maximum_in_flight(&self) -> usize {
return self.maximum_in_flight;
}
/// Returns the cumulative safe contiguous prefix represented by the progress checkpoint.
pub(crate) const fn contiguous_completed(&self) -> usize {
return self.contiguous_completed;
}
/// Returns the safe checkpoint proven at this progress position.
pub(crate) const fn checkpoint(&self) -> &crate::BackfillCheckpoint {
return &self.checkpoint;
}
}
/// Executes one bounded discovered candidate set with request-owned hydration concurrency.
///
/// This compatibility entry point has no external cancellation handle. Transport owns retries and
/// endpoint selection; Store submissions are always awaited to completion.
pub async fn execute_backfill_discovery(
transport: &ksp_onchain_transport_lib::HttpTransportPool,
store: &ksp_store_lib::Store,
request: &crate::BackfillRequest,
discovery: &crate::BackfillDiscovery,
) -> ksp_core_lib::Result<crate::BackfillExecutionBatch> {
let validation = crate::validate_discovery_identity(request, discovery);
if let std::result::Result::Err(error) = validation {
return std::result::Result::Err(error);
}
let processor = RuntimeCandidateProcessor { transport, store, request, cancellation: std::option::Option::None };
return execute_with_processor(&processor, request, discovery, std::option::Option::None, std::option::Option::None).await;
}
/// Executes one bounded discovery with cooperative cancellation and concrete progress publication.
pub(crate) async fn execute_backfill_discovery_cancellable(
transport: &ksp_onchain_transport_lib::HttpTransportPool,
store: &ksp_store_lib::Store,
request: &crate::BackfillRequest,
discovery: &crate::BackfillDiscovery,
cancellation: &crate::BackfillCancellationSignal,
publisher: &crate::BackfillRuntimePublisher,
) -> ksp_core_lib::Result<crate::BackfillExecutionBatch> {
let processor = RuntimeCandidateProcessor { transport, store, request, cancellation: std::option::Option::Some(cancellation) };
return execute_with_processor(&processor, request, discovery, std::option::Option::Some(cancellation), std::option::Option::Some(publisher)).await;
}
type CandidateProcessFuture<'a> =
std::pin::Pin<std::boxed::Box<dyn std::future::Future<Output = ksp_core_lib::Result<crate::BackfillPersistenceOutcome>> + 'a>>;
trait CandidateProcessor {
fn process<'a>(&'a self, candidate: &'a crate::BackfillCandidate) -> CandidateProcessFuture<'a>;
}
struct RuntimeCandidateProcessor<'a> {
transport: &'a ksp_onchain_transport_lib::HttpTransportPool,
store: &'a ksp_store_lib::Store,
request: &'a crate::BackfillRequest,
cancellation: std::option::Option<&'a crate::BackfillCancellationSignal>,
}
impl CandidateProcessor for RuntimeCandidateProcessor<'_> {
fn process<'a>(&'a self, candidate: &'a crate::BackfillCandidate) -> CandidateProcessFuture<'a> {
return std::boxed::Box::pin(async move {
let received_at = match current_raw_timestamp() {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let hydration_future = crate::hydrate_backfill_candidate(self.transport, self.request, candidate, received_at);
let hydration = match self.cancellation {
std::option::Option::Some(signal) => signal.run_cancellable(hydration_future).await,
std::option::Option::None => hydration_future.await,
};
let hydration = match hydration {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
if self.cancellation.is_some_and(crate::BackfillCancellationSignal::is_requested) {
return std::result::Result::Err(cancelled_error());
}
// Deliberately not cancellable once Store submission begins: a submitted durable write is drained.
return crate::persist_backfill_hydration(self.store, hydration).await;
});
}
}
#[allow(clippy::too_many_lines)]
async fn execute_with_processor<P>(
processor: &P,
request: &crate::BackfillRequest,
discovery: &crate::BackfillDiscovery,
cancellation: std::option::Option<&crate::BackfillCancellationSignal>,
publisher: std::option::Option<&crate::BackfillRuntimePublisher>,
) -> ksp_core_lib::Result<crate::BackfillExecutionBatch>
where
P: CandidateProcessor,
{
let validation = crate::validate_discovery_identity(request, discovery);
if let std::result::Result::Err(error) = validation {
return std::result::Result::Err(error);
}
let resume_offset = match crate::execution_resume_offset(request, discovery) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let mut frontier = crate::CompletionFrontier::new(discovery.candidates().len());
if matches!(request.scope().kind(), crate::BackfillScopeKind::AfterAddress | crate::BackfillScopeKind::ExplicitSignatures) {
let seeded = frontier.seed_prefix(resume_offset);
if let std::result::Result::Err(error) = seeded {
return std::result::Result::Err(error);
}
}
let mut next_index = resume_offset;
let mut admitted_count = 0_usize;
let mut finished_count = 0_usize;
let mut durable_count = 0_usize;
let mut inserted_count = 0_usize;
let mut already_present_count = 0_usize;
let mut purged_count = 0_usize;
let mut missing_count = 0_usize;
let mut conflict_count = 0_usize;
let mut observation_inserted_count = 0_usize;
let mut observation_already_present_count = 0_usize;
let mut cancelled_count = 0_usize;
let mut hole_count = 0_usize;
let mut maximum_in_flight = 0_usize;
let mut failure_code = std::option::Option::<ksp_core_lib::ErrorCode>::None;
let mut cancellation_requested = false;
let mut in_flight = futures_util::stream::FuturesUnordered::new();
loop {
if !cancellation_requested && cancellation.is_some_and(crate::BackfillCancellationSignal::is_requested) {
cancellation_requested = true;
let progress = progress_from_state(
request,
discovery,
&frontier,
admitted_count,
finished_count,
inserted_count,
already_present_count,
purged_count,
missing_count,
conflict_count,
observation_inserted_count,
observation_already_present_count,
cancelled_count,
hole_count,
maximum_in_flight,
);
let progress = match progress {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let published = publish_progress(publisher, &progress, true, true);
if let std::result::Result::Err(error) = published {
return std::result::Result::Err(error);
}
}
while failure_code.is_none()
&& !cancellation_requested
&& next_index < discovery.candidates().len()
&& in_flight.len() < request.hydration_concurrency()
{
let index = next_index;
let candidate = &discovery.candidates()[index];
let future = processor.process(candidate);
in_flight.push(async move {
return (index, future.await);
});
next_index += 1;
admitted_count = match checked_increment(admitted_count, "admitted_count") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
maximum_in_flight = std::cmp::max(maximum_in_flight, in_flight.len());
}
let completed = in_flight.next().await;
let (index, result) = match completed {
std::option::Option::Some(completed) => completed,
std::option::Option::None => break,
};
finished_count = match checked_increment(finished_count, "finished_count") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
match result {
std::result::Result::Ok(outcome) => {
let classified = classify_persistence(
&outcome,
&mut inserted_count,
&mut already_present_count,
&mut purged_count,
&mut missing_count,
&mut conflict_count,
&mut observation_inserted_count,
&mut observation_already_present_count,
);
if let std::result::Result::Err(error) = classified {
return std::result::Result::Err(error);
}
if persistence_advances_frontier(&outcome) {
let marked = frontier.mark_durable(index);
if let std::result::Result::Err(error) = marked {
return std::result::Result::Err(error);
}
durable_count = match checked_increment(durable_count, "durable_count") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
} else {
hole_count = match checked_increment(hole_count, "hole_count") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
if outcome.entity() == crate::BackfillEntityPersistence::Conflict && failure_code.is_none() {
failure_code = std::option::Option::Some(ksp_store_lib::ERROR_CODE_RAW_CONFLICT);
}
}
},
std::result::Result::Err(error) => {
hole_count = match checked_increment(hole_count, "hole_count") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(increment_error) => return std::result::Result::Err(increment_error),
};
if error.code() == crate::ERROR_CODE_BACKFILL_CANCELLED {
cancelled_count = match checked_increment(cancelled_count, "cancelled_count") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(increment_error) => return std::result::Result::Err(increment_error),
};
cancellation_requested = true;
} else if failure_code.is_none() {
failure_code = std::option::Option::Some(error.code());
}
},
}
let progress = progress_from_state(
request,
discovery,
&frontier,
admitted_count,
finished_count,
inserted_count,
already_present_count,
purged_count,
missing_count,
conflict_count,
observation_inserted_count,
observation_already_present_count,
cancelled_count,
hole_count,
maximum_in_flight,
);
let progress = match progress {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let draining = cancellation_requested || failure_code.is_some();
let published = publish_progress(publisher, &progress, cancellation_requested, draining);
if let std::result::Result::Err(error) = published {
return std::result::Result::Err(error);
}
}
let checkpoint = match crate::checkpoint_from_frontier(request, discovery, &frontier) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
return std::result::Result::Ok(crate::BackfillExecutionBatch {
candidate_count: discovery.candidates().len(),
admitted_count,
finished_count,
durable_count,
inserted_count,
already_present_count,
purged_count,
missing_count,
conflict_count,
observation_inserted_count,
observation_already_present_count,
cancelled_count,
hole_count,
maximum_in_flight,
local_contiguous_completed: frontier.contiguous_completed(),
discovery_partial: discovery.is_partial(),
cancelled: cancellation_requested,
checkpoint,
failure_code,
});
}
#[allow(clippy::too_many_arguments)]
fn progress_from_state(
request: &crate::BackfillRequest,
discovery: &crate::BackfillDiscovery,
frontier: &crate::CompletionFrontier,
admitted_count: usize,
finished_count: usize,
inserted_count: usize,
already_present_count: usize,
purged_count: usize,
missing_count: usize,
conflict_count: usize,
observation_inserted_count: usize,
observation_already_present_count: usize,
cancelled_count: usize,
hole_count: usize,
maximum_in_flight: usize,
) -> ksp_core_lib::Result<crate::BackfillExecutionProgress> {
let checkpoint = match crate::checkpoint_from_frontier(request, discovery, frontier) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
return std::result::Result::Ok(crate::BackfillExecutionProgress {
admitted_count,
finished_count,
inserted_count,
already_present_count,
purged_count,
missing_count,
conflict_count,
observation_inserted_count,
observation_already_present_count,
cancelled_count,
hole_count,
maximum_in_flight,
contiguous_completed: checkpoint.completed_prefix(),
checkpoint,
});
}
fn publish_progress(
publisher: std::option::Option<&crate::BackfillRuntimePublisher>,
progress: &crate::BackfillExecutionProgress,
cancelling: bool,
draining: bool,
) -> ksp_core_lib::Result<()> {
if let std::option::Option::Some(publisher) = publisher {
let published = publisher.publish_execution_progress(progress, cancelling, draining);
if let std::result::Result::Err(error) = published {
return std::result::Result::Err(error);
}
}
return std::result::Result::Ok(());
}
#[allow(clippy::too_many_arguments)]
fn classify_persistence(
outcome: &crate::BackfillPersistenceOutcome,
inserted_count: &mut usize,
already_present_count: &mut usize,
purged_count: &mut usize,
missing_count: &mut usize,
conflict_count: &mut usize,
observation_inserted_count: &mut usize,
observation_already_present_count: &mut usize,
) -> ksp_core_lib::Result<()> {
match outcome.entity() {
crate::BackfillEntityPersistence::Inserted => {
*inserted_count = match checked_increment(*inserted_count, "inserted_count") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
},
crate::BackfillEntityPersistence::AlreadyPresent => {
*already_present_count = match checked_increment(*already_present_count, "already_present_count") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
},
crate::BackfillEntityPersistence::SkippedPurged => {
*purged_count = match checked_increment(*purged_count, "purged_count") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
},
crate::BackfillEntityPersistence::Missing => {
*missing_count = match checked_increment(*missing_count, "missing_count") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
},
crate::BackfillEntityPersistence::Conflict => {
*conflict_count = match checked_increment(*conflict_count, "conflict_count") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
},
}
match outcome.observation() {
crate::BackfillObservationPersistence::Inserted => {
*observation_inserted_count = match checked_increment(*observation_inserted_count, "observation_inserted_count") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
},
crate::BackfillObservationPersistence::AlreadyPresent => {
*observation_already_present_count = match checked_increment(*observation_already_present_count, "observation_already_present_count") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
},
crate::BackfillObservationPersistence::NotRecorded | crate::BackfillObservationPersistence::NotApplicable => {},
}
return std::result::Result::Ok(());
}
fn persistence_advances_frontier(outcome: &crate::BackfillPersistenceOutcome) -> bool {
return matches!(
outcome.entity(),
crate::BackfillEntityPersistence::Inserted | crate::BackfillEntityPersistence::AlreadyPresent | crate::BackfillEntityPersistence::SkippedPurged
);
}
fn checked_increment(value: usize, field: &'static str) -> ksp_core_lib::Result<usize> {
return value.checked_add(1).ok_or_else(|| return execution_error(field));
}
fn current_raw_timestamp() -> ksp_core_lib::Result<ksp_store_lib::RawTimestamp> {
let duration = std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH);
let duration = match duration {
std::result::Result::Ok(duration) => duration,
std::result::Result::Err(_) => return std::result::Result::Err(execution_error("clock.before_epoch")),
};
let millis = match u64::try_from(duration.as_millis()) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return std::result::Result::Err(execution_error("clock.millis")),
};
return ksp_store_lib::RawTimestamp::from_unix_millis(millis);
}
fn cancelled_error() -> ksp_core_lib::Error {
return ksp_core_lib::Error::new(crate::ERROR_CODE_BACKFILL_CANCELLED, "Backfill candidate cancelled before durable Store submission");
}
fn execution_error(field: &'static str) -> ksp_core_lib::Error {
return ksp_core_lib::Error::new(crate::ERROR_CODE_BACKFILL_EXECUTION_INVALID, "invalid bounded Backfill execution state").with_context("field", field);
}
#[cfg(test)]
#[path = "../unit_tests/execution.rs"]
mod tests;

View File

@@ -0,0 +1,143 @@
// file: crates/ksp-job-backfill-lib/src/lib.rs
// version: 6
#![warn(missing_docs)]
#![deny(unreachable_pub)]
#![forbid(unsafe_code)]
//! Concrete bounded historical RAW transaction Backfill foundation.
//!
//! This tranche owns explicit admission, network-scoped candidate identity, deterministic
//! `getSignaturesForAddress` pagination and canonical RAW v1 conversion through observed
//! `getTransaction`. Transport retains provider/endpoint selection and retry; Store retains
//! durable idempotence through the atomic Store facade. This tranche also owns bounded concurrent
//! candidate execution, caller-owned contiguous checkpoints, cooperative cancellation and concrete
//! latest-value snapshots for external listeners.
mod checkpoint;
mod constants;
mod conversion;
mod discovery;
mod error;
mod execution;
mod persistence;
mod request;
mod runtime;
/// Opaque caller-owned checkpoint for one controlled Backfill resumption.
pub use self::checkpoint::BackfillCheckpoint;
/// Result of hydrating one deterministic candidate through observed `getTransaction`.
pub use self::conversion::BackfillHydrationOutcome;
/// Complete in-memory RAW transaction acquisition ready for later Store persistence.
pub use self::conversion::BackfillRawAcquisition;
/// KSP-owned source-independent RAW transaction format identifier produced by this Backfill vertical.
pub use self::conversion::RAW_TRANSACTION_FORMAT_ID;
/// Initial KSP-owned RAW transaction format version produced by this Backfill vertical.
pub use self::conversion::RAW_TRANSACTION_FORMAT_VERSION;
/// Hydrates one candidate through observed Transport and converts a non-null response to canonical RAW v1.
pub use self::conversion::hydrate_backfill_candidate;
/// One deterministic transaction candidate produced by bounded discovery.
pub use self::discovery::BackfillCandidate;
/// Network-scoped identity of one discovered transaction candidate before canonical signature decoding.
pub use self::discovery::BackfillCandidateIdentity;
/// Complete bounded output of one candidate discovery pass.
pub use self::discovery::BackfillDiscovery;
/// Reason bounded candidate discovery stopped.
pub use self::discovery::BackfillDiscoveryBoundary;
/// Discovers one bounded deterministic candidate set through the typed KSP Transport wrapper.
pub use self::discovery::discover_backfill_candidates;
/// Error code used when one checkpoint/frontier is incompatible with the current Job or semantic scope.
pub use self::error::ERROR_CODE_BACKFILL_CHECKPOINT_INVALID;
/// Error code used when a signature page violates a bounded discovery invariant.
pub use self::error::ERROR_CODE_BACKFILL_DISCOVERY_INVALID;
/// Error code used when paginated discovery cannot advance its exclusive RPC cursor safely.
pub use self::error::ERROR_CODE_BACKFILL_DISCOVERY_STALLED;
/// Error code used when bounded concurrent execution violates an internal admission or clock invariant.
pub use self::error::ERROR_CODE_BACKFILL_EXECUTION_INVALID;
/// Error code used when Store persistence returns an impossible Backfill state or targets a different network.
pub use self::error::ERROR_CODE_BACKFILL_PERSISTENCE_INVALID;
/// Error code used when deterministic Transport-to-RAW conversion violates the v1 contract.
pub use self::error::ERROR_CODE_BACKFILL_RAW_CONVERSION_INVALID;
/// Error code used when one Backfill request violates its bounded admission contract.
pub use self::error::ERROR_CODE_BACKFILL_REQUEST_INVALID;
/// Error code used when the concrete Backfill runtime reaches an impossible lifecycle or notification state.
pub use self::error::ERROR_CODE_BACKFILL_RUNTIME_INVALID;
/// Error code used when one transaction signature text violates the bounded Base58-shape contract.
pub use self::error::ERROR_CODE_BACKFILL_SIGNATURE_INVALID;
/// Bounded result of one concurrent Backfill candidate execution pass.
pub use self::execution::BackfillExecutionBatch;
/// Executes one bounded discovered candidate set with request-owned hydration concurrency.
pub use self::execution::execute_backfill_discovery;
/// Canonical entity disposition produced by one Backfill Store persistence attempt.
pub use self::persistence::BackfillEntityPersistence;
/// Observation disposition produced by one Backfill Store persistence attempt.
pub use self::persistence::BackfillObservationPersistence;
/// Stable Backfill projection of one hydration persistence result.
pub use self::persistence::BackfillPersistenceOutcome;
/// Persists one hydrated result through the backend-neutral atomic Store contract.
pub use self::persistence::persist_backfill_hydration;
/// Commitment levels intentionally admitted by the historical Backfill vertical.
pub use self::request::BackfillCommitment;
/// Fully explicit bounded request for one historical transaction Backfill Job.
pub use self::request::BackfillRequest;
/// Validated bounded discovery scope for one historical Backfill Job.
pub use self::request::BackfillScope;
/// Opaque deterministic fingerprint of one semantic Backfill scope.
pub use self::request::BackfillScopeFingerprint;
/// Stable category of one bounded Backfill discovery scope.
pub use self::request::BackfillScopeKind;
/// Bounded Base58-shaped transaction signature text with exact RAW signature conversion.
pub use self::request::BackfillSignature;
/// Maximum number of transaction candidates admitted by one bounded Backfill Job.
pub use self::request::MAX_BACKFILL_CANDIDATES;
/// Maximum number of concurrent transaction hydrations admitted by one Backfill request.
pub use self::request::MAX_BACKFILL_HYDRATION_CONCURRENCY;
/// Maximum page size admitted for one `getSignaturesForAddress` request.
pub use self::request::MAX_BACKFILL_PAGE_SIZE;
/// Maximum number of `getSignaturesForAddress` pages admitted by one address Backfill request.
pub use self::request::MAX_BACKFILL_PAGES;
/// Maximum Base58 text length possible for one canonical 64-byte Solana signature.
pub use self::request::MAX_BACKFILL_SIGNATURE_TEXT_BYTES;
/// Minimum Base58 text length possible for one canonical 64-byte Solana signature.
pub use self::request::MIN_BACKFILL_SIGNATURE_TEXT_BYTES;
/// Stable Job kind code used by the concrete historical RAW transaction Backfill runtime.
pub use self::runtime::BACKFILL_JOB_KIND_CODE;
/// Cloneable external control handle for one concrete Backfill runtime.
pub use self::runtime::BackfillJobHandle;
/// Current concrete phase of one historical RAW transaction Backfill Job.
pub use self::runtime::BackfillJobPhase;
/// Concrete single-run Backfill coordinator paired with a cloneable control/snapshot handle.
pub use self::runtime::BackfillJobRuntime;
/// Complete safe latest-value snapshot of one concrete historical RAW transaction Backfill Job.
pub use self::runtime::BackfillJobSnapshot;
/// Cloneable runtime-neutral-facing latest-value source for concrete Backfill snapshots.
pub use self::runtime::BackfillSnapshotSource;
/// Internal contiguous completion frontier used by bounded execution.
pub(crate) use self::checkpoint::CompletionFrontier;
/// Builds one safe caller-owned checkpoint from the current contiguous frontier.
pub(crate) use self::checkpoint::checkpoint_from_frontier;
/// Resolves the candidate offset skipped by one validated replay checkpoint.
pub(crate) use self::checkpoint::execution_resume_offset;
/// Resolves the internal exclusive Before cursor for discovery resumption.
pub(crate) use self::checkpoint::resume_before_cursor;
/// Validates that one discovery result belongs to the current request identity.
pub(crate) use self::checkpoint::validate_discovery_identity;
/// Validates one caller-owned checkpoint against Job and semantic scope identity.
pub(crate) use self::checkpoint::validate_request_checkpoint;
/// Owning tracing target used by the concrete Backfill runtime.
pub(crate) use self::constants::TRACING_TARGET;
/// Exact private Base58 decoder shared by the public signature wrapper and hydration path.
pub(crate) use self::conversion::decode_backfill_signature;
/// Internal cancellable discovery path used by the concrete runtime.
pub(crate) use self::discovery::discover_backfill_candidates_cancellable;
/// Internal cooperative cancellation code used to distinguish cancellation from failure.
pub(crate) use self::error::ERROR_CODE_BACKFILL_CANCELLED;
/// Internal execution progress facts fed into the concrete latest-value source.
pub(crate) use self::execution::BackfillExecutionProgress;
/// Internal cancellable execution path used by the concrete runtime.
pub(crate) use self::execution::execute_backfill_discovery_cancellable;
/// Internal concrete cancellation signal shared by discovery and execution.
pub(crate) use self::runtime::BackfillCancellationSignal;
/// Internal execution progress publisher feeding the latest-value snapshot source.
pub(crate) use self::runtime::BackfillRuntimePublisher;

View File

@@ -0,0 +1,210 @@
// file: crates/ksp-job-backfill-lib/src/persistence.rs
// version: 3
/// Canonical entity disposition produced by one Backfill Store persistence attempt.
#[non_exhaustive]
#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)]
pub enum BackfillEntityPersistence {
/// The canonical RAW transaction was inserted for the first time.
Inserted,
/// Identical canonical RAW transaction content was already durable.
AlreadyPresent,
/// A durable purge tombstone prevented normal Backfill rehydration.
SkippedPurged,
/// `getTransaction` returned JSON `null`, so no canonical Store write was attempted.
Missing,
/// Store reported divergent canonical content for the same network-scoped transaction identity.
Conflict,
}
/// Observation disposition produced by one Backfill Store persistence attempt.
#[non_exhaustive]
#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)]
pub enum BackfillObservationPersistence {
/// The deterministic acquisition observation was inserted for the first time.
Inserted,
/// The same deterministic acquisition observation was already durable.
AlreadyPresent,
/// Store intentionally recorded no observation because normal persistence skipped a purged entity or detected a conflict.
NotRecorded,
/// No observation existed because hydration returned `Missing` before persistence.
NotApplicable,
}
/// Stable Backfill projection of one hydration persistence result.
///
/// The canonical transaction identity remains `(network, signature)`. Provider, endpoint and
/// transport details can affect the persisted observation but never the transaction identity.
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct BackfillPersistenceOutcome {
reference: ksp_store_lib::RawTransactionReference,
entity: crate::BackfillEntityPersistence,
observation: crate::BackfillObservationPersistence,
}
impl crate::BackfillPersistenceOutcome {
/// Creates one internally classified persistence result.
pub(crate) fn new(
reference: ksp_store_lib::RawTransactionReference,
entity: crate::BackfillEntityPersistence,
observation: crate::BackfillObservationPersistence,
) -> Self {
return Self { reference, entity, observation };
}
/// Returns the network-scoped canonical transaction identity classified by this result.
#[must_use]
pub fn reference(&self) -> &ksp_store_lib::RawTransactionReference {
return &self.reference;
}
/// Returns the canonical transaction persistence disposition.
#[must_use]
pub const fn entity(&self) -> crate::BackfillEntityPersistence {
return self.entity;
}
/// Returns the acquisition-observation persistence disposition.
#[must_use]
pub const fn observation(&self) -> crate::BackfillObservationPersistence {
return self.observation;
}
}
/// Persists one hydrated Backfill result through the backend-neutral Store facade.
///
/// Available acquisitions use the existing atomic transaction-plus-observation Store contract in
/// `Normal` mode. A durable purge is therefore respected and never rehydrated implicitly.
/// `Missing` performs no Store write. Stable Store content conflicts are projected explicitly as
/// [`BackfillEntityPersistence::Conflict`] rather than being silently treated as idempotent skips.
/// Other Store failures remain errors.
pub async fn persist_backfill_hydration(
store: &ksp_store_lib::Store,
hydration: crate::BackfillHydrationOutcome,
) -> ksp_core_lib::Result<crate::BackfillPersistenceOutcome> {
return persist_hydration_with_port(store, hydration).await;
}
trait RawTransactionPersistencePort: std::marker::Send + std::marker::Sync {
fn network_matches(&self, network: &ksp_store_lib::RawNetworkId) -> bool;
fn persist_acquisition<'a>(
&'a self,
transaction: ksp_store_lib::RawTransaction,
observation: ksp_store_lib::RawTransactionObservation,
mode: ksp_store_lib::RawTransactionAcquisitionMode,
) -> ksp_store_lib::StoreApiFuture<'a, ksp_store_lib::Result<ksp_store_lib::RawAcquisitionWriteOutcome>>;
}
impl RawTransactionPersistencePort for ksp_store_lib::Store {
fn network_matches(&self, network: &ksp_store_lib::RawNetworkId) -> bool {
let snapshot = self.runtime_snapshot();
return snapshot.network() == network;
}
fn persist_acquisition<'a>(
&'a self,
transaction: ksp_store_lib::RawTransaction,
observation: ksp_store_lib::RawTransactionObservation,
mode: ksp_store_lib::RawTransactionAcquisitionMode,
) -> ksp_store_lib::StoreApiFuture<'a, ksp_store_lib::Result<ksp_store_lib::RawAcquisitionWriteOutcome>> {
return ksp_store_lib::RawTransactionWrite::persist_raw_transaction_acquisition(self, transaction, observation, mode);
}
}
async fn persist_hydration_with_port<P>(port: &P, hydration: crate::BackfillHydrationOutcome) -> ksp_core_lib::Result<crate::BackfillPersistenceOutcome>
where
P: RawTransactionPersistencePort,
{
let reference = hydration.reference().clone();
if !port.network_matches(reference.network()) {
return std::result::Result::Err(persistence_error("store.network"));
}
return match hydration {
crate::BackfillHydrationOutcome::Missing(_) => std::result::Result::Ok(crate::BackfillPersistenceOutcome::new(
reference,
crate::BackfillEntityPersistence::Missing,
crate::BackfillObservationPersistence::NotApplicable,
)),
crate::BackfillHydrationOutcome::Available(acquisition) => {
let (transaction, observation) = acquisition.into_parts();
persist_available_with_port(port, reference, transaction, observation).await
},
};
}
async fn persist_available_with_port<P>(
port: &P,
reference: ksp_store_lib::RawTransactionReference,
transaction: ksp_store_lib::RawTransaction,
observation: ksp_store_lib::RawTransactionObservation,
) -> ksp_core_lib::Result<crate::BackfillPersistenceOutcome>
where
P: RawTransactionPersistencePort,
{
if !port.network_matches(reference.network()) {
return std::result::Result::Err(persistence_error("store.network"));
}
if transaction.reference() != &reference || observation.transaction() != &reference {
return std::result::Result::Err(persistence_error("acquisition.reference"));
}
let result = port.persist_acquisition(transaction, observation, ksp_store_lib::RawTransactionAcquisitionMode::Normal).await;
return match result {
std::result::Result::Ok(outcome) => map_store_outcome(reference, outcome),
std::result::Result::Err(error) => {
if error.code() == ksp_store_lib::ERROR_CODE_RAW_CONFLICT {
return std::result::Result::Ok(crate::BackfillPersistenceOutcome::new(
reference,
crate::BackfillEntityPersistence::Conflict,
crate::BackfillObservationPersistence::NotRecorded,
));
}
std::result::Result::Err(error)
},
};
}
fn map_store_outcome(
reference: ksp_store_lib::RawTransactionReference,
outcome: ksp_store_lib::RawAcquisitionWriteOutcome,
) -> ksp_core_lib::Result<crate::BackfillPersistenceOutcome> {
let entity = outcome.entity();
let observation = outcome.observation();
if entity == ksp_store_lib::RawEntityWriteOutcome::Inserted && observation == ksp_store_lib::RawObservationWriteOutcome::Inserted {
return std::result::Result::Ok(crate::BackfillPersistenceOutcome::new(
reference,
crate::BackfillEntityPersistence::Inserted,
crate::BackfillObservationPersistence::Inserted,
));
}
if entity == ksp_store_lib::RawEntityWriteOutcome::AlreadyPresent && observation == ksp_store_lib::RawObservationWriteOutcome::Inserted {
return std::result::Result::Ok(crate::BackfillPersistenceOutcome::new(
reference,
crate::BackfillEntityPersistence::AlreadyPresent,
crate::BackfillObservationPersistence::Inserted,
));
}
if entity == ksp_store_lib::RawEntityWriteOutcome::AlreadyPresent && observation == ksp_store_lib::RawObservationWriteOutcome::AlreadyPresent {
return std::result::Result::Ok(crate::BackfillPersistenceOutcome::new(
reference,
crate::BackfillEntityPersistence::AlreadyPresent,
crate::BackfillObservationPersistence::AlreadyPresent,
));
}
if entity == ksp_store_lib::RawEntityWriteOutcome::SkippedPurged && observation == ksp_store_lib::RawObservationWriteOutcome::NotRecorded {
return std::result::Result::Ok(crate::BackfillPersistenceOutcome::new(
reference,
crate::BackfillEntityPersistence::SkippedPurged,
crate::BackfillObservationPersistence::NotRecorded,
));
}
return std::result::Result::Err(persistence_error("store.outcome"));
}
fn persistence_error(field: &'static str) -> ksp_core_lib::Error {
return ksp_core_lib::Error::new(crate::ERROR_CODE_BACKFILL_PERSISTENCE_INVALID, "invalid Backfill Store persistence state").with_context("field", field);
}
#[cfg(test)]
#[path = "../unit_tests/persistence.rs"]
mod tests;

View File

@@ -0,0 +1,494 @@
// file: crates/ksp-job-backfill-lib/src/request.rs
// version: 5
use sha2::Digest; // rust-rules: trait-import
/// Maximum number of transaction candidates admitted by one bounded Backfill Job.
pub const MAX_BACKFILL_CANDIDATES: usize = 10_000;
/// Maximum number of concurrent transaction hydrations admitted by one Backfill request.
pub const MAX_BACKFILL_HYDRATION_CONCURRENCY: usize = 64;
/// Maximum number of `getSignaturesForAddress` pages admitted by one address Backfill request.
pub const MAX_BACKFILL_PAGES: usize = 10_000;
/// Maximum page size admitted for one `getSignaturesForAddress` request.
pub const MAX_BACKFILL_PAGE_SIZE: usize = 1_000;
/// Maximum Base58 text length possible for one canonical 64-byte Solana signature.
pub const MAX_BACKFILL_SIGNATURE_TEXT_BYTES: usize = 88;
/// Minimum Base58 text length possible for one canonical 64-byte Solana signature.
pub const MIN_BACKFILL_SIGNATURE_TEXT_BYTES: usize = 64;
/// Commitment levels intentionally admitted by the historical Backfill vertical.
#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)]
pub enum BackfillCommitment {
/// Read history at Solana `confirmed` commitment.
Confirmed,
/// Read history at Solana `finalized` commitment.
Finalized,
}
impl crate::BackfillCommitment {
/// Returns the stable Backfill commitment code.
#[must_use]
pub const fn code(self) -> &'static str {
return match self {
Self::Confirmed => "confirmed",
Self::Finalized => "finalized",
};
}
/// Maps the narrowed Backfill commitment to the Transport-owned Solana commitment.
#[must_use]
pub(crate) const fn transport(self) -> ksp_onchain_transport_lib::SolanaCommitment {
return match self {
Self::Confirmed => ksp_onchain_transport_lib::SolanaCommitment::Confirmed,
Self::Finalized => ksp_onchain_transport_lib::SolanaCommitment::Finalized,
};
}
}
/// Bounded Base58-shaped transaction signature text used by discovery and RAW conversion.
///
/// Construction validates the encoded shape required by discovery. Exact conversion to the
/// Store-owned 64-byte signature is available through [`Self::to_raw_transaction_signature`].
#[derive(Clone, Eq, Hash, Ord, PartialEq, PartialOrd)]
pub struct BackfillSignature(std::string::String);
impl crate::BackfillSignature {
/// Creates one bounded Base58-shaped signature text.
pub fn new(value: impl std::convert::Into<std::string::String>) -> ksp_core_lib::Result<Self> {
let value = value.into();
if !valid_signature_text(value.as_str()) {
return std::result::Result::Err(signature_error());
}
return std::result::Result::Ok(Self(value));
}
/// Returns the validated encoded signature text.
#[must_use]
pub fn as_str(&self) -> &str {
return self.0.as_str();
}
/// Decodes this Base58 text to the exact Store-owned 64-byte Solana signature.
pub fn to_raw_transaction_signature(&self) -> ksp_core_lib::Result<ksp_store_lib::RawTransactionSignature> {
return crate::decode_backfill_signature(self);
}
}
impl std::fmt::Debug for crate::BackfillSignature {
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
return formatter.write_str("BackfillSignature(..)");
}
}
/// Stable category of one bounded Backfill discovery scope.
#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)]
pub enum BackfillScopeKind {
/// Read the current newest address window.
LatestAddress,
/// Read address history older than one exclusive anchor.
BeforeAddress,
/// Read the bounded newer window closest to one exclusive anchor.
AfterAddress,
/// Hydrate an explicit bounded signature list without address discovery.
ExplicitSignatures,
}
impl crate::BackfillScopeKind {
/// Returns the stable scope code used by diagnostics and scope fingerprinting.
#[must_use]
pub const fn code(self) -> &'static str {
return match self {
Self::LatestAddress => "latest_address",
Self::BeforeAddress => "before_address",
Self::AfterAddress => "after_address",
Self::ExplicitSignatures => "explicit_signatures",
};
}
}
#[derive(Clone, Eq, PartialEq)]
enum BackfillScopeValue {
LatestAddress { address: ksp_core_lib::Pubkey },
BeforeAddress { address: ksp_core_lib::Pubkey, anchor: crate::BackfillSignature },
AfterAddress { address: ksp_core_lib::Pubkey, anchor: crate::BackfillSignature },
ExplicitSignatures { signatures: std::vec::Vec<crate::BackfillSignature> },
}
/// Validated bounded discovery scope for one historical Backfill Job.
#[derive(Clone, Eq, PartialEq)]
pub struct BackfillScope {
value: BackfillScopeValue,
}
impl crate::BackfillScope {
/// Creates a scope starting at the newest known history for one address.
#[must_use]
pub const fn latest_address(address: ksp_core_lib::Pubkey) -> Self {
return Self { value: BackfillScopeValue::LatestAddress { address } };
}
/// Creates a scope reading history older than one exclusive address anchor.
#[must_use]
pub fn before_address(address: ksp_core_lib::Pubkey, anchor: crate::BackfillSignature) -> Self {
return Self { value: BackfillScopeValue::BeforeAddress { address, anchor } };
}
/// Creates a scope reading the bounded newer history closest to one exclusive address anchor.
#[must_use]
pub fn after_address(address: ksp_core_lib::Pubkey, anchor: crate::BackfillSignature) -> Self {
return Self { value: BackfillScopeValue::AfterAddress { address, anchor } };
}
/// Creates an explicit signature scope with stable first-occurrence deduplication.
pub fn explicit_signatures(signatures: std::vec::Vec<crate::BackfillSignature>) -> ksp_core_lib::Result<Self> {
if signatures.is_empty() || signatures.len() > crate::MAX_BACKFILL_CANDIDATES {
return std::result::Result::Err(request_error("scope.signatures"));
}
let mut unique = std::vec::Vec::with_capacity(signatures.len());
let mut seen = std::collections::HashSet::with_capacity(signatures.len());
for signature in signatures {
if seen.insert(signature.clone()) {
unique.push(signature);
}
}
if unique.is_empty() {
return std::result::Result::Err(request_error("scope.signatures"));
}
return std::result::Result::Ok(Self { value: BackfillScopeValue::ExplicitSignatures { signatures: unique } });
}
/// Returns the stable category of this scope.
#[must_use]
pub const fn kind(&self) -> crate::BackfillScopeKind {
return match &self.value {
BackfillScopeValue::LatestAddress { .. } => crate::BackfillScopeKind::LatestAddress,
BackfillScopeValue::BeforeAddress { .. } => crate::BackfillScopeKind::BeforeAddress,
BackfillScopeValue::AfterAddress { .. } => crate::BackfillScopeKind::AfterAddress,
BackfillScopeValue::ExplicitSignatures { .. } => crate::BackfillScopeKind::ExplicitSignatures,
};
}
/// Returns the address used by address scopes.
#[must_use]
pub const fn address(&self) -> std::option::Option<&ksp_core_lib::Pubkey> {
return match &self.value {
BackfillScopeValue::LatestAddress { address }
| BackfillScopeValue::BeforeAddress { address, .. }
| BackfillScopeValue::AfterAddress { address, .. } => std::option::Option::Some(address),
BackfillScopeValue::ExplicitSignatures { .. } => std::option::Option::None,
};
}
/// Returns the exclusive anchor used by before/after scopes.
#[must_use]
pub const fn anchor(&self) -> std::option::Option<&crate::BackfillSignature> {
return match &self.value {
BackfillScopeValue::BeforeAddress { anchor, .. } | BackfillScopeValue::AfterAddress { anchor, .. } => std::option::Option::Some(anchor),
BackfillScopeValue::LatestAddress { .. } | BackfillScopeValue::ExplicitSignatures { .. } => std::option::Option::None,
};
}
/// Returns the stable deduplicated explicit signature list, when this is an explicit scope.
#[must_use]
pub fn signatures(&self) -> std::option::Option<&[crate::BackfillSignature]> {
return match &self.value {
BackfillScopeValue::ExplicitSignatures { signatures } => std::option::Option::Some(signatures.as_slice()),
BackfillScopeValue::LatestAddress { .. } | BackfillScopeValue::BeforeAddress { .. } | BackfillScopeValue::AfterAddress { .. } => {
std::option::Option::None
},
};
}
}
impl std::fmt::Debug for crate::BackfillScope {
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
let mut debug = formatter.debug_struct("BackfillScope");
debug.field("kind", &self.kind());
match &self.value {
BackfillScopeValue::LatestAddress { address } => {
debug.field("address", address);
},
BackfillScopeValue::BeforeAddress { address, .. } | BackfillScopeValue::AfterAddress { address, .. } => {
debug.field("address", address).field("anchor", &"<redacted>");
},
BackfillScopeValue::ExplicitSignatures { signatures } => {
debug.field("signature_count", &signatures.len());
},
}
return debug.finish();
}
}
/// Opaque deterministic fingerprint of one semantic Backfill scope.
#[derive(Clone, Copy, Eq, Hash, PartialEq)]
pub struct BackfillScopeFingerprint([u8; 32]);
impl crate::BackfillScopeFingerprint {
const fn from_bytes(bytes: [u8; 32]) -> Self {
return Self(bytes);
}
/// Returns the exact deterministic fingerprint bytes.
#[must_use]
pub const fn as_bytes(&self) -> &[u8; 32] {
return &self.0;
}
}
impl std::fmt::Debug for crate::BackfillScopeFingerprint {
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
return formatter.write_str("BackfillScopeFingerprint(..)");
}
}
/// Fully explicit bounded request for one historical transaction Backfill Job.
#[derive(Clone, Eq, PartialEq)]
pub struct BackfillRequest {
job_id: ksp_job_api::JobId,
network: ksp_store_lib::RawNetworkId,
role: ksp_onchain_transport_lib::HttpRoleName,
commitment: crate::BackfillCommitment,
scope: crate::BackfillScope,
page_size: usize,
max_pages: usize,
max_candidates: usize,
hydration_concurrency: usize,
min_context_slot: std::option::Option<u64>,
scope_fingerprint: crate::BackfillScopeFingerprint,
checkpoint: std::option::Option<crate::BackfillCheckpoint>,
}
impl crate::BackfillRequest {
/// Creates and validates one fully explicit bounded Backfill request.
#[allow(clippy::too_many_arguments)]
pub fn new(
job_id: ksp_job_api::JobId,
network: ksp_store_lib::RawNetworkId,
role: ksp_onchain_transport_lib::HttpRoleName,
commitment: crate::BackfillCommitment,
scope: crate::BackfillScope,
page_size: usize,
max_pages: usize,
max_candidates: usize,
hydration_concurrency: usize,
min_context_slot: std::option::Option<u64>,
) -> ksp_core_lib::Result<Self> {
if page_size == 0 || page_size > crate::MAX_BACKFILL_PAGE_SIZE {
return std::result::Result::Err(request_error("page_size"));
}
if max_pages == 0 || max_pages > crate::MAX_BACKFILL_PAGES {
return std::result::Result::Err(request_error("max_pages"));
}
if max_candidates == 0 || max_candidates > crate::MAX_BACKFILL_CANDIDATES {
return std::result::Result::Err(request_error("max_candidates"));
}
if hydration_concurrency == 0 || hydration_concurrency > crate::MAX_BACKFILL_HYDRATION_CONCURRENCY {
return std::result::Result::Err(request_error("hydration_concurrency"));
}
if role.as_str().is_empty() || role.as_str().trim() != role.as_str() {
return std::result::Result::Err(request_error("role"));
}
if scope.kind() == crate::BackfillScopeKind::ExplicitSignatures {
if min_context_slot.is_some() {
return std::result::Result::Err(request_error("min_context_slot"));
}
let signature_count = match scope.signatures() {
std::option::Option::Some(signatures) => signatures.len(),
std::option::Option::None => return std::result::Result::Err(request_error("scope.signatures")),
};
if signature_count > max_candidates {
return std::result::Result::Err(request_error("max_candidates"));
}
}
let scope_fingerprint = fingerprint_scope(&network, commitment, &scope, page_size, max_pages, max_candidates, min_context_slot);
return std::result::Result::Ok(Self {
job_id,
network,
role,
commitment,
scope,
page_size,
max_pages,
max_candidates,
hydration_concurrency,
min_context_slot,
scope_fingerprint,
checkpoint: std::option::Option::None,
});
}
/// Returns the caller-owned logical Job identity.
#[must_use]
pub const fn job_id(&self) -> &ksp_job_api::JobId {
return &self.job_id;
}
/// Returns the Store-scoped logical network identity for every candidate in this Job.
#[must_use]
pub const fn network(&self) -> &ksp_store_lib::RawNetworkId {
return &self.network;
}
/// Returns the logical HTTP role used only for Transport selection.
#[must_use]
pub const fn role(&self) -> &ksp_onchain_transport_lib::HttpRoleName {
return &self.role;
}
/// Returns the narrowed commitment used by discovery and hydration.
#[must_use]
pub const fn commitment(&self) -> crate::BackfillCommitment {
return self.commitment;
}
/// Returns the validated discovery scope.
#[must_use]
pub const fn scope(&self) -> &crate::BackfillScope {
return &self.scope;
}
/// Returns the per-request signature page size.
#[must_use]
pub const fn page_size(&self) -> usize {
return self.page_size;
}
/// Returns the maximum number of address pages admitted by this Job.
#[must_use]
pub const fn max_pages(&self) -> usize {
return self.max_pages;
}
/// Returns the maximum number of transaction candidates admitted by this Job.
#[must_use]
pub const fn max_candidates(&self) -> usize {
return self.max_candidates;
}
/// Returns the future hydration concurrency bound.
#[must_use]
pub const fn hydration_concurrency(&self) -> usize {
return self.hydration_concurrency;
}
/// Returns the optional Solana minimum context slot used by address discovery.
#[must_use]
pub const fn min_context_slot(&self) -> std::option::Option<u64> {
return self.min_context_slot;
}
/// Returns the deterministic semantic scope fingerprint.
///
/// Transport role, provider, endpoint and protocol are deliberately excluded. They describe
/// acquisition provenance, not transaction or scope identity.
#[must_use]
pub const fn scope_fingerprint(&self) -> crate::BackfillScopeFingerprint {
return self.scope_fingerprint;
}
/// Attaches one caller-owned checkpoint after validating Job and semantic scope identity.
pub fn with_checkpoint(mut self, checkpoint: crate::BackfillCheckpoint) -> ksp_core_lib::Result<Self> {
let validation = crate::validate_request_checkpoint(&self.job_id, self.scope_fingerprint, self.scope.kind(), &checkpoint);
if let std::result::Result::Err(error) = validation {
return std::result::Result::Err(error);
}
self.checkpoint = std::option::Option::Some(checkpoint);
return std::result::Result::Ok(self);
}
/// Returns the validated optional checkpoint supplied for controlled resumption.
#[must_use]
pub const fn checkpoint(&self) -> std::option::Option<&crate::BackfillCheckpoint> {
return self.checkpoint.as_ref();
}
}
impl std::fmt::Debug for crate::BackfillRequest {
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
return formatter
.debug_struct("BackfillRequest")
.field("job_id", &self.job_id)
.field("network", &self.network)
.field("role", &self.role)
.field("commitment", &self.commitment)
.field("scope", &self.scope)
.field("page_size", &self.page_size)
.field("max_pages", &self.max_pages)
.field("max_candidates", &self.max_candidates)
.field("hydration_concurrency", &self.hydration_concurrency)
.field("min_context_slot", &self.min_context_slot)
.field("scope_fingerprint", &self.scope_fingerprint)
.field("has_checkpoint", &self.checkpoint.is_some())
.finish();
}
}
fn fingerprint_scope(
network: &ksp_store_lib::RawNetworkId,
commitment: crate::BackfillCommitment,
scope: &crate::BackfillScope,
page_size: usize,
max_pages: usize,
max_candidates: usize,
min_context_slot: std::option::Option<u64>,
) -> crate::BackfillScopeFingerprint {
let mut hasher = sha2::Sha256::new();
hasher.update(b"ksp.job.backfill.scope.v1\0");
hash_bytes(&mut hasher, network.as_str().as_bytes());
hash_bytes(&mut hasher, commitment.code().as_bytes());
hash_bytes(&mut hasher, scope.kind().code().as_bytes());
if let std::option::Option::Some(address) = scope.address() {
hash_bytes(&mut hasher, address.as_ref());
}
if let std::option::Option::Some(anchor) = scope.anchor() {
hash_bytes(&mut hasher, anchor.as_str().as_bytes());
}
if let std::option::Option::Some(signatures) = scope.signatures() {
for signature in signatures {
hash_bytes(&mut hasher, signature.as_str().as_bytes());
}
}
hash_u64(&mut hasher, page_size as u64);
hash_u64(&mut hasher, max_pages as u64);
hash_u64(&mut hasher, max_candidates as u64);
match min_context_slot {
std::option::Option::Some(slot) => {
hasher.update([1_u8]);
hash_u64(&mut hasher, slot);
},
std::option::Option::None => hasher.update([0_u8]),
}
let bytes: [u8; 32] = hasher.finalize().into();
return crate::BackfillScopeFingerprint::from_bytes(bytes);
}
fn hash_bytes(hasher: &mut sha2::Sha256, value: &[u8]) {
hash_u64(hasher, value.len() as u64);
hasher.update(value);
}
fn hash_u64(hasher: &mut sha2::Sha256, value: u64) {
hasher.update(value.to_be_bytes());
}
fn request_error(field: &'static str) -> ksp_core_lib::Error {
return ksp_core_lib::Error::new(crate::ERROR_CODE_BACKFILL_REQUEST_INVALID, "invalid bounded Backfill request").with_context("field", field);
}
fn signature_error() -> ksp_core_lib::Error {
return ksp_core_lib::Error::new(crate::ERROR_CODE_BACKFILL_SIGNATURE_INVALID, "invalid bounded Backfill transaction signature")
.with_context("field", "signature");
}
fn valid_signature_text(value: &str) -> bool {
if value.len() < crate::MIN_BACKFILL_SIGNATURE_TEXT_BYTES || value.len() > crate::MAX_BACKFILL_SIGNATURE_TEXT_BYTES {
return false;
}
return value.bytes().all(|byte| {
return matches!(byte, b'1'..=b'9' | b'A'..=b'H' | b'J'..=b'N' | b'P'..=b'Z' | b'a'..=b'k' | b'm'..=b'z');
});
}
#[cfg(test)]
#[path = "../unit_tests/request.rs"]
mod tests;

View File

@@ -0,0 +1,715 @@
// file: crates/ksp-job-backfill-lib/src/runtime.rs
// version: 3
/// Stable Job kind code used by the concrete historical RAW transaction Backfill runtime.
pub const BACKFILL_JOB_KIND_CODE: &str = "solana.raw_transaction.backfill";
const CONTROL_ACTIVE: u8 = 0;
const CONTROL_CANCELLATION_REQUESTED: u8 = 1;
const CONTROL_CANCELLED: u8 = 3;
const CONTROL_COMPLETED: u8 = 2;
const CONTROL_FAILED: u8 = 4;
/// Current concrete phase of one historical RAW transaction Backfill Job.
#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)]
#[non_exhaustive]
pub enum BackfillJobPhase {
/// Runtime was created but execution has not started.
Created,
/// Candidate discovery is in progress.
Discovering,
/// Candidate hydration and persistence are being admitted and completed.
Executing,
/// Cancellation or a fatal result stopped admission while already submitted work is draining.
Draining,
/// No more work can be admitted and the Job is terminal.
Finished,
}
impl crate::BackfillJobPhase {
/// Returns the stable safe code for this concrete runtime phase.
#[must_use]
pub const fn code(self) -> &'static str {
return match self {
Self::Created => "created",
Self::Discovering => "discovering",
Self::Executing => "executing",
Self::Draining => "draining",
Self::Finished => "finished",
};
}
}
/// Complete safe latest-value snapshot of one concrete historical RAW transaction Backfill Job.
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct BackfillJobSnapshot {
phase: crate::BackfillJobPhase,
scope_kind: crate::BackfillScopeKind,
discovery_boundary: std::option::Option<crate::BackfillDiscoveryBoundary>,
candidates_selected: usize,
candidates_admitted: usize,
candidates_finished: usize,
entities_inserted: usize,
entities_existing: usize,
entities_purged: usize,
missing: usize,
conflicts: usize,
observations_inserted: usize,
observations_existing: usize,
cancelled_candidates: usize,
holes: usize,
maximum_in_flight: usize,
contiguous_completed: usize,
checkpoint: std::option::Option<crate::BackfillCheckpoint>,
failure_code: std::option::Option<ksp_core_lib::ErrorCode>,
}
impl crate::BackfillJobSnapshot {
fn initial(request: &crate::BackfillRequest) -> Self {
return Self {
phase: crate::BackfillJobPhase::Created,
scope_kind: request.scope().kind(),
discovery_boundary: std::option::Option::None,
candidates_selected: 0,
candidates_admitted: 0,
candidates_finished: 0,
entities_inserted: 0,
entities_existing: 0,
entities_purged: 0,
missing: 0,
conflicts: 0,
observations_inserted: 0,
observations_existing: 0,
cancelled_candidates: 0,
holes: 0,
maximum_in_flight: 0,
contiguous_completed: request.checkpoint().map_or(0, crate::BackfillCheckpoint::completed_prefix),
checkpoint: request.checkpoint().cloned(),
failure_code: std::option::Option::None,
};
}
/// Returns the concrete execution phase represented by this snapshot.
#[must_use]
pub const fn phase(&self) -> crate::BackfillJobPhase {
return self.phase;
}
/// Returns the safe scope category without rendering address or signature payloads.
#[must_use]
pub const fn scope_kind(&self) -> crate::BackfillScopeKind {
return self.scope_kind;
}
/// Returns the discovery boundary once candidate selection has completed.
#[must_use]
pub const fn discovery_boundary(&self) -> std::option::Option<crate::BackfillDiscoveryBoundary> {
return self.discovery_boundary;
}
/// Returns the bounded candidate count selected by discovery.
#[must_use]
pub const fn candidates_selected(&self) -> usize {
return self.candidates_selected;
}
/// Returns the number of candidates admitted into hydration.
#[must_use]
pub const fn candidates_admitted(&self) -> usize {
return self.candidates_admitted;
}
/// Returns the number of admitted candidates that reached a known coordinator result.
#[must_use]
pub const fn candidates_finished(&self) -> usize {
return self.candidates_finished;
}
/// Returns the number of newly inserted canonical RAW transactions.
#[must_use]
pub const fn entities_inserted(&self) -> usize {
return self.entities_inserted;
}
/// Returns the number of identical canonical RAW transactions already present.
#[must_use]
pub const fn entities_existing(&self) -> usize {
return self.entities_existing;
}
/// Returns the number of durable purge tombstones respected by normal Backfill persistence.
#[must_use]
pub const fn entities_purged(&self) -> usize {
return self.entities_purged;
}
/// Returns the number of `getTransaction = null` candidates.
#[must_use]
pub const fn missing(&self) -> usize {
return self.missing;
}
/// Returns the number of Store content conflicts observed by this run.
#[must_use]
pub const fn conflicts(&self) -> usize {
return self.conflicts;
}
/// Returns the number of newly inserted acquisition observations.
#[must_use]
pub const fn observations_inserted(&self) -> usize {
return self.observations_inserted;
}
/// Returns the number of acquisition observations already durable.
#[must_use]
pub const fn observations_existing(&self) -> usize {
return self.observations_existing;
}
/// Returns the number of admitted candidates cancelled before Store submission.
#[must_use]
pub const fn cancelled_candidates(&self) -> usize {
return self.cancelled_candidates;
}
/// Returns the number of known candidate outcomes blocking the contiguous frontier.
#[must_use]
pub const fn holes(&self) -> usize {
return self.holes;
}
/// Returns the greatest observed concurrent candidate count.
#[must_use]
pub const fn maximum_in_flight(&self) -> usize {
return self.maximum_in_flight;
}
/// Returns the cumulative safe contiguous completion prefix represented by the current checkpoint.
#[must_use]
pub const fn contiguous_completed(&self) -> usize {
return self.contiguous_completed;
}
/// Returns the latest caller-owned safe checkpoint, when one exists.
#[must_use]
pub const fn checkpoint(&self) -> std::option::Option<&crate::BackfillCheckpoint> {
return self.checkpoint.as_ref();
}
/// Returns the stable fatal error code retained by a failed Job, when one exists.
#[must_use]
pub const fn failure_code(&self) -> std::option::Option<ksp_core_lib::ErrorCode> {
return self.failure_code;
}
}
/// Cloneable runtime-neutral-facing latest-value source for concrete Backfill snapshots.
#[derive(Clone)]
pub struct BackfillSnapshotSource {
receiver: tokio::sync::watch::Receiver<ksp_job_api::JobNotification<crate::BackfillJobSnapshot>>,
}
impl ksp_job_api::JobSnapshotSource for crate::BackfillSnapshotSource {
type Snapshot = crate::BackfillJobSnapshot;
fn current(&self) -> ksp_job_api::JobNotification<Self::Snapshot> {
return self.receiver.borrow().clone();
}
fn wait_for_change(&self, observed: ksp_job_api::JobNotificationSequence) -> ksp_job_api::JobSnapshotFuture<'_, Self::Snapshot> {
let mut receiver = self.receiver.clone();
return std::boxed::Box::pin(async move {
loop {
let current = receiver.borrow().clone();
if current.sequence().is_after(observed) || current.state().is_terminal() {
return current;
}
let changed = receiver.changed().await;
if changed.is_err() {
return receiver.borrow().clone();
}
}
});
}
}
impl std::fmt::Debug for crate::BackfillSnapshotSource {
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
let current = self.receiver.borrow();
return formatter.debug_struct("BackfillSnapshotSource").field("sequence", &current.sequence()).field("state", &current.state()).finish();
}
}
/// Cloneable external control handle for one concrete Backfill runtime.
#[derive(Clone)]
pub struct BackfillJobHandle {
control: BackfillRuntimeControl,
snapshots: crate::BackfillSnapshotSource,
}
impl crate::BackfillJobHandle {
/// Requests cooperative cancellation and returns `true` only when accepted before terminal publication.
#[must_use]
pub fn cancel(&self) -> bool {
return self.control.request_cancellation();
}
/// Returns an independent latest-value snapshot source for one listener.
#[must_use]
pub fn snapshots(&self) -> crate::BackfillSnapshotSource {
return self.snapshots.clone();
}
/// Returns whether cooperative cancellation has been accepted for this non-terminal Job.
#[must_use]
pub fn is_cancellation_requested(&self) -> bool {
return self.control.is_cancellation_requested();
}
}
impl std::fmt::Debug for crate::BackfillJobHandle {
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
return formatter
.debug_struct("BackfillJobHandle")
.field("cancellation_requested", &self.is_cancellation_requested())
.field("snapshot", &<crate::BackfillSnapshotSource as ksp_job_api::JobSnapshotSource>::current(&self.snapshots))
.finish();
}
}
/// Concrete single-run Backfill coordinator paired with a cloneable control/snapshot handle.
pub struct BackfillJobRuntime {
request: crate::BackfillRequest,
control: BackfillRuntimeControl,
cancellation: crate::BackfillCancellationSignal,
publisher: crate::BackfillRuntimePublisher,
handle: crate::BackfillJobHandle,
}
impl crate::BackfillJobRuntime {
/// Creates one runtime in `Created` state and its stable latest-value channel.
pub fn new(request: crate::BackfillRequest) -> ksp_core_lib::Result<Self> {
let kind = match ksp_job_api::JobKindCode::new(crate::BACKFILL_JOB_KIND_CODE) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let initial_snapshot = crate::BackfillJobSnapshot::initial(&request);
let initial = ksp_job_api::JobNotification::new(
request.job_id().clone(),
kind.clone(),
ksp_job_api::JobNotificationSequence::initial(),
ksp_job_api::JobState::Created,
initial_snapshot,
);
let (snapshot_sender, snapshot_receiver) = tokio::sync::watch::channel(initial);
let (cancel_sender, cancel_receiver) = tokio::sync::watch::channel(false);
let control = BackfillRuntimeControl::new(cancel_sender);
let cancellation = crate::BackfillCancellationSignal::new(control.token(), cancel_receiver);
let snapshots = crate::BackfillSnapshotSource { receiver: snapshot_receiver };
let handle = crate::BackfillJobHandle { control: control.clone(), snapshots: snapshots.clone() };
let publisher = crate::BackfillRuntimePublisher { sender: snapshot_sender, id: request.job_id().clone(), kind };
return std::result::Result::Ok(Self { request, control, cancellation, publisher, handle });
}
/// Returns a cloneable control and latest-value observation handle before the runtime is moved into execution.
#[must_use]
pub fn handle(&self) -> crate::BackfillJobHandle {
return self.handle.clone();
}
/// Runs discovery, bounded execution, cooperative cancellation and terminal snapshot publication.
pub async fn run(
self,
transport: &ksp_onchain_transport_lib::HttpTransportPool,
store: &ksp_store_lib::Store,
) -> ksp_core_lib::Result<crate::BackfillJobSnapshot> {
if self.cancellation.is_requested() {
let claimed = self.control.claim_normal_terminal();
if claimed != TerminalClaim::Cancelled {
return std::result::Result::Err(runtime_error("terminal.pre_start"));
}
return self.publisher.publish_cancelled_from_created();
}
let started = self.publisher.publish_running(crate::BackfillJobPhase::Discovering);
if let std::result::Result::Err(error) = started {
self.control.claim_failed();
return std::result::Result::Err(error);
}
let discovery = crate::discover_backfill_candidates_cancellable(transport, &self.request, &self.cancellation).await;
let discovery = match discovery {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => {
if error.code() == crate::ERROR_CODE_BACKFILL_CANCELLED {
return self.finish_cancelled();
}
self.control.claim_failed();
let published = self.publisher.publish_failed(error.code());
if let std::result::Result::Err(publish_error) = published {
return std::result::Result::Err(publish_error);
}
return std::result::Result::Err(error);
},
};
let published = self.publisher.publish_discovery(&discovery);
if let std::result::Result::Err(error) = published {
self.control.claim_failed();
return std::result::Result::Err(error);
}
let batch = crate::execute_backfill_discovery_cancellable(transport, store, &self.request, &discovery, &self.cancellation, &self.publisher).await;
let batch = match batch {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => {
self.control.claim_failed();
let published = self.publisher.publish_failed(error.code());
if let std::result::Result::Err(publish_error) = published {
return std::result::Result::Err(publish_error);
}
return std::result::Result::Err(error);
},
};
if let std::option::Option::Some(code) = batch.failure_code() {
self.control.claim_failed();
let published = self.publisher.publish_batch_terminal(&batch, ksp_job_api::JobState::Failed, std::option::Option::Some(code));
if let std::result::Result::Err(error) = published {
return std::result::Result::Err(error);
}
return std::result::Result::Err(ksp_core_lib::Error::new(code, "Backfill execution reached a fatal candidate result"));
}
let terminal = self.control.claim_normal_terminal();
return match terminal {
TerminalClaim::Cancelled => {
let cancelling = self.publisher.publish_batch_cancelling(&batch);
if let std::result::Result::Err(error) = cancelling {
return std::result::Result::Err(error);
}
self.publisher.publish_batch_terminal(&batch, ksp_job_api::JobState::Cancelled, std::option::Option::None)
},
TerminalClaim::Completed => {
let completion = if batch.is_partial() { ksp_job_api::JobCompletion::Partial } else { ksp_job_api::JobCompletion::Complete };
self.publisher.publish_batch_terminal(&batch, ksp_job_api::JobState::Completed(completion), std::option::Option::None)
},
TerminalClaim::Failed => std::result::Result::Err(runtime_error("terminal.failed")),
};
}
fn finish_cancelled(&self) -> ksp_core_lib::Result<crate::BackfillJobSnapshot> {
let terminal = self.control.claim_normal_terminal();
if terminal != TerminalClaim::Cancelled {
return std::result::Result::Err(runtime_error("terminal.cancelled"));
}
let cancelling = self.publisher.publish_cancelling();
if let std::result::Result::Err(error) = cancelling {
return std::result::Result::Err(error);
}
return self.publisher.publish_terminal(ksp_job_api::JobState::Cancelled, std::option::Option::None);
}
}
/// Internal atomic terminal/cancellation arbitration shared by runtime and external handle.
#[derive(Clone)]
struct BackfillRuntimeControl {
state: std::sync::Arc<std::sync::atomic::AtomicU8>,
token: ksp_job_api::JobCancellationToken,
cancel_sender: tokio::sync::watch::Sender<bool>,
}
impl BackfillRuntimeControl {
/// Creates one active control state paired with the cancellation wake channel.
fn new(cancel_sender: tokio::sync::watch::Sender<bool>) -> Self {
return Self {
state: std::sync::Arc::new(std::sync::atomic::AtomicU8::new(CONTROL_ACTIVE)),
token: ksp_job_api::JobCancellationToken::new(),
cancel_sender,
};
}
/// Returns the runtime-neutral cancellation token mirrored by this control.
fn token(&self) -> ksp_job_api::JobCancellationToken {
return self.token.clone();
}
/// Atomically accepts the first pre-terminal cancellation request.
fn request_cancellation(&self) -> bool {
let accepted = self
.state
.compare_exchange(CONTROL_ACTIVE, CONTROL_CANCELLATION_REQUESTED, std::sync::atomic::Ordering::AcqRel, std::sync::atomic::Ordering::Acquire)
.is_ok();
if accepted && self.token.cancel() {
let _previous = self.cancel_sender.send_replace(true);
}
return accepted;
}
/// Returns whether cooperative cancellation was accepted.
fn is_cancellation_requested(&self) -> bool {
return self.token.is_cancellation_requested();
}
/// Atomically resolves the completion-versus-cancellation terminal race.
fn claim_normal_terminal(&self) -> TerminalClaim {
let completed =
self.state.compare_exchange(CONTROL_ACTIVE, CONTROL_COMPLETED, std::sync::atomic::Ordering::AcqRel, std::sync::atomic::Ordering::Acquire);
if completed.is_ok() {
return TerminalClaim::Completed;
}
let cancelled = self.state.compare_exchange(
CONTROL_CANCELLATION_REQUESTED,
CONTROL_CANCELLED,
std::sync::atomic::Ordering::AcqRel,
std::sync::atomic::Ordering::Acquire,
);
if cancelled.is_ok() {
return TerminalClaim::Cancelled;
}
return TerminalClaim::Failed;
}
/// Marks a non-terminal control as failed, overriding a pending cancellation request.
fn claim_failed(&self) {
loop {
let state = self.state.load(std::sync::atomic::Ordering::Acquire);
if matches!(state, CONTROL_COMPLETED | CONTROL_CANCELLED | CONTROL_FAILED) {
return;
}
let changed = self.state.compare_exchange(state, CONTROL_FAILED, std::sync::atomic::Ordering::AcqRel, std::sync::atomic::Ordering::Acquire);
if changed.is_ok() {
return;
}
}
}
}
/// Internal wakeable cancellation signal used only around abandonable pre-Store futures.
#[derive(Clone)]
pub(crate) struct BackfillCancellationSignal {
token: ksp_job_api::JobCancellationToken,
receiver: tokio::sync::watch::Receiver<bool>,
}
impl crate::BackfillCancellationSignal {
/// Creates one signal from the runtime-neutral token and Tokio wake receiver.
pub(crate) fn new(token: ksp_job_api::JobCancellationToken, receiver: tokio::sync::watch::Receiver<bool>) -> Self {
return Self { token, receiver };
}
/// Returns whether cancellation has been requested.
pub(crate) fn is_requested(&self) -> bool {
return self.token.is_cancellation_requested();
}
/// Runs one abandonable operation until it completes or cancellation wins.
pub(crate) async fn run_cancellable<F, T>(&self, operation: F) -> ksp_core_lib::Result<T>
where
F: std::future::Future<Output = ksp_core_lib::Result<T>>,
{
if self.is_requested() {
return std::result::Result::Err(cancelled_error());
}
let mut receiver = self.receiver.clone();
return tokio::select! {
biased;
_ = wait_for_cancellation(&self.token, &mut receiver) => std::result::Result::Err(cancelled_error()),
result = operation => result,
};
}
}
async fn wait_for_cancellation(token: &ksp_job_api::JobCancellationToken, receiver: &mut tokio::sync::watch::Receiver<bool>) {
loop {
if token.is_cancellation_requested() || *receiver.borrow() {
return;
}
let changed = receiver.changed().await;
if changed.is_err() {
return;
}
}
}
/// Internal result of atomically claiming a normal terminal state.
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
enum TerminalClaim {
Completed,
Cancelled,
Failed,
}
/// Internal latest-value publisher owning the concrete Backfill notification stream.
#[derive(Clone)]
pub(crate) struct BackfillRuntimePublisher {
sender: tokio::sync::watch::Sender<ksp_job_api::JobNotification<crate::BackfillJobSnapshot>>,
id: ksp_job_api::JobId,
kind: ksp_job_api::JobKindCode,
}
impl crate::BackfillRuntimePublisher {
/// Publishes one non-terminal running phase.
pub(crate) fn publish_running(&self, phase: crate::BackfillJobPhase) -> ksp_core_lib::Result<crate::BackfillJobSnapshot> {
return self.publish_with(ksp_job_api::JobState::Running, |snapshot| {
snapshot.phase = phase;
});
}
/// Publishes the complete bounded discovery result as the current execution snapshot.
pub(crate) fn publish_discovery(&self, discovery: &crate::BackfillDiscovery) -> ksp_core_lib::Result<crate::BackfillJobSnapshot> {
return self.publish_with(ksp_job_api::JobState::Running, |snapshot| {
snapshot.phase = crate::BackfillJobPhase::Executing;
snapshot.discovery_boundary = std::option::Option::Some(discovery.boundary());
snapshot.candidates_selected = discovery.candidates().len();
});
}
/// Publishes one coalescable execution progress value.
pub(crate) fn publish_execution_progress(
&self,
progress: &crate::BackfillExecutionProgress,
cancelling: bool,
draining: bool,
) -> ksp_core_lib::Result<crate::BackfillJobSnapshot> {
let state = if cancelling { ksp_job_api::JobState::Cancelling } else { ksp_job_api::JobState::Running };
return self.publish_with(state, |snapshot| {
snapshot.phase = if draining { crate::BackfillJobPhase::Draining } else { crate::BackfillJobPhase::Executing };
apply_progress(snapshot, progress);
});
}
/// Publishes cancellation observation before terminal cancellation.
pub(crate) fn publish_cancelling(&self) -> ksp_core_lib::Result<crate::BackfillJobSnapshot> {
return self.publish_with(ksp_job_api::JobState::Cancelling, |snapshot| {
snapshot.phase = crate::BackfillJobPhase::Draining;
});
}
/// Publishes the drained batch state while cancellation is terminalizing.
pub(crate) fn publish_batch_cancelling(&self, batch: &crate::BackfillExecutionBatch) -> ksp_core_lib::Result<crate::BackfillJobSnapshot> {
return self.publish_with(ksp_job_api::JobState::Cancelling, |snapshot| {
snapshot.phase = crate::BackfillJobPhase::Draining;
apply_batch(snapshot, batch);
});
}
/// Publishes one terminal state together with the fully drained execution batch.
pub(crate) fn publish_batch_terminal(
&self,
batch: &crate::BackfillExecutionBatch,
state: ksp_job_api::JobState,
failure_code: std::option::Option<ksp_core_lib::ErrorCode>,
) -> ksp_core_lib::Result<crate::BackfillJobSnapshot> {
return self.publish_with(state, |snapshot| {
snapshot.phase = crate::BackfillJobPhase::Finished;
snapshot.failure_code = failure_code;
apply_batch(snapshot, batch);
});
}
/// Publishes a terminal failure before a batch exists.
pub(crate) fn publish_failed(&self, code: ksp_core_lib::ErrorCode) -> ksp_core_lib::Result<crate::BackfillJobSnapshot> {
return self.publish_terminal(ksp_job_api::JobState::Failed, std::option::Option::Some(code));
}
/// Publishes a terminal state without a completed execution batch.
pub(crate) fn publish_terminal(
&self,
state: ksp_job_api::JobState,
failure_code: std::option::Option<ksp_core_lib::ErrorCode>,
) -> ksp_core_lib::Result<crate::BackfillJobSnapshot> {
return self.publish_with(state, |snapshot| {
snapshot.phase = crate::BackfillJobPhase::Finished;
snapshot.failure_code = failure_code;
});
}
/// Publishes direct Created-to-Cancelled termination before execution starts.
pub(crate) fn publish_cancelled_from_created(&self) -> ksp_core_lib::Result<crate::BackfillJobSnapshot> {
return self.publish_with(ksp_job_api::JobState::Cancelled, |snapshot| {
snapshot.phase = crate::BackfillJobPhase::Finished;
});
}
fn publish_with<F>(&self, state: ksp_job_api::JobState, update: F) -> ksp_core_lib::Result<crate::BackfillJobSnapshot>
where
F: FnOnce(&mut crate::BackfillJobSnapshot),
{
let current = self.sender.borrow().clone();
if current.state().is_terminal() {
return std::result::Result::Err(runtime_error("notification.terminal"));
}
if !valid_snapshot_transition(current.state(), state) {
return std::result::Result::Err(runtime_error("notification.transition"));
}
let sequence = match current.sequence().next() {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let mut snapshot = current.snapshot().clone();
update(&mut snapshot);
let notification = ksp_job_api::JobNotification::new(self.id.clone(), self.kind.clone(), sequence, state, snapshot.clone());
let _previous = self.sender.send_replace(notification);
return std::result::Result::Ok(snapshot);
}
}
fn valid_snapshot_transition(source: ksp_job_api::JobState, target: ksp_job_api::JobState) -> bool {
if source.is_terminal() {
return false;
}
return matches!(
(source, target),
(ksp_job_api::JobState::Created, ksp_job_api::JobState::Running)
| (ksp_job_api::JobState::Created, ksp_job_api::JobState::Cancelled)
| (ksp_job_api::JobState::Running, ksp_job_api::JobState::Running)
| (ksp_job_api::JobState::Running, ksp_job_api::JobState::Cancelling)
| (ksp_job_api::JobState::Running, ksp_job_api::JobState::Completed(_))
| (ksp_job_api::JobState::Running, ksp_job_api::JobState::Failed)
| (ksp_job_api::JobState::Cancelling, ksp_job_api::JobState::Cancelling)
| (ksp_job_api::JobState::Cancelling, ksp_job_api::JobState::Cancelled)
| (ksp_job_api::JobState::Cancelling, ksp_job_api::JobState::Failed)
);
}
fn apply_progress(snapshot: &mut crate::BackfillJobSnapshot, progress: &crate::BackfillExecutionProgress) {
snapshot.candidates_admitted = progress.admitted_count();
snapshot.candidates_finished = progress.finished_count();
snapshot.entities_inserted = progress.inserted_count();
snapshot.entities_existing = progress.already_present_count();
snapshot.entities_purged = progress.purged_count();
snapshot.missing = progress.missing_count();
snapshot.conflicts = progress.conflict_count();
snapshot.observations_inserted = progress.observation_inserted_count();
snapshot.observations_existing = progress.observation_already_present_count();
snapshot.cancelled_candidates = progress.cancelled_count();
snapshot.holes = progress.hole_count();
snapshot.maximum_in_flight = progress.maximum_in_flight();
snapshot.contiguous_completed = progress.contiguous_completed();
snapshot.checkpoint = std::option::Option::Some(progress.checkpoint().clone());
}
fn apply_batch(snapshot: &mut crate::BackfillJobSnapshot, batch: &crate::BackfillExecutionBatch) {
snapshot.candidates_admitted = batch.admitted_count();
snapshot.candidates_finished = batch.finished_count();
snapshot.entities_inserted = batch.inserted_count();
snapshot.entities_existing = batch.already_present_count();
snapshot.entities_purged = batch.purged_count();
snapshot.missing = batch.missing_count();
snapshot.conflicts = batch.conflict_count();
snapshot.observations_inserted = batch.observation_inserted_count();
snapshot.observations_existing = batch.observation_already_present_count();
snapshot.cancelled_candidates = batch.cancelled_count();
snapshot.holes = batch.hole_count();
snapshot.maximum_in_flight = batch.maximum_in_flight();
snapshot.contiguous_completed = batch.checkpoint().completed_prefix();
snapshot.checkpoint = std::option::Option::Some(batch.checkpoint().clone());
}
fn cancelled_error() -> ksp_core_lib::Error {
return ksp_core_lib::Error::new(crate::ERROR_CODE_BACKFILL_CANCELLED, "Backfill operation cancelled before durable Store submission");
}
fn runtime_error(field: &'static str) -> ksp_core_lib::Error {
return ksp_core_lib::Error::new(crate::ERROR_CODE_BACKFILL_RUNTIME_INVALID, "invalid concrete Backfill runtime state").with_context("field", field);
}
#[cfg(test)]
#[path = "../unit_tests/runtime.rs"]
mod tests;

View File

@@ -0,0 +1,93 @@
// file: crates/ksp-job-backfill-lib/tests/dependency_boundary.rs
// version: 6
//! Dependency firewall canaries through the concrete cancellation and latest-value runtime tranche.
#[test]
fn pre_009_manifest_uses_only_planned_ksp_edges_and_private_tokio_runtime() {
let manifest = include_str!("../Cargo.toml");
for required in [
"futures-util = { workspace = true, features = [\"std\"] }",
"ksp-core-lib = { path = \"../ksp-core-lib\" }",
"ksp-job-api = { path = \"../ksp-job-api\" }",
"ksp-logging-lib = { path = \"../ksp-logging-lib\" }",
"ksp-onchain-transport-lib = { path = \"../ksp-onchain-transport-lib\" }",
"ksp-store-lib = { path = \"../ksp-store-lib\", default-features = false }",
"serde_json.workspace = true",
"sha2.workspace = true",
"tokio = { workspace = true, features = [\"macros\", \"sync\"] }",
] {
assert!(manifest.contains(required), "required Backfill dependency missing: {required}");
}
for forbidden in [
"ksp-config-lib",
"ksp-interface-lib",
"ksp-program-api",
"ksp-store-api",
"ksp-store-postgres-lib",
"ksp-wallet-lib",
"solana-",
"reqwest",
"serde.workspace = true",
"serde = {",
"tonic",
] {
assert!(!manifest.contains(forbidden), "forbidden Backfill dependency present: {forbidden}");
}
let root = include_str!("../src/lib.rs");
assert!(!root.contains("tokio::"), "Tokio implementation types must not leak through the public crate root");
return;
}
#[test]
fn pre_009_production_sources_keep_transport_store_and_scheduler_ownership_separate() {
let neutral_sources = [
include_str!("../src/checkpoint.rs"),
include_str!("../src/constants.rs"),
include_str!("../src/discovery.rs"),
include_str!("../src/error.rs"),
include_str!("../src/lib.rs"),
include_str!("../src/request.rs"),
include_str!("../src/runtime.rs"),
];
for source in neutral_sources {
for forbidden in ["ksp_config_lib::", "ksp_interface_lib::", "ksp_store_api::", "ksp_store_postgres_lib::", "reqwest::", "std::env", "tonic::"] {
assert!(!source.contains(forbidden), "forbidden concrete Backfill path detected: {forbidden}");
}
}
let conversion = include_str!("../src/conversion.rs");
assert!(conversion.contains("serde_json::"));
assert!(conversion.contains("get_transaction_observed"));
assert!(conversion.contains("SolanaTransactionEncoding::Base64"));
assert!(conversion.contains("std::option::Option::Some(0)"));
assert!(conversion.contains("BackfillHydrationOutcome::Missing(reference)"));
assert!(!conversion.contains("execute_standard_rpc"));
assert!(!conversion.contains("persist_raw_transaction_acquisition"));
let persistence = include_str!("../src/persistence.rs");
assert!(persistence.contains("persist_raw_transaction_acquisition"));
assert!(persistence.contains("RawTransactionAcquisitionMode::Normal"));
assert!(persistence.contains("ERROR_CODE_RAW_CONFLICT"));
assert!(!persistence.contains("record_raw_transaction_observation"));
assert!(!persistence.contains("RawTransactionAcquisitionMode::ForceRehydrate"));
let discovery = include_str!("../src/discovery.rs");
assert!(discovery.contains("get_signatures_for_address"));
assert!(!discovery.contains("execute_standard_rpc"));
assert!(!discovery.contains("retry"));
assert!(!discovery.contains("endpoint_name"));
let execution = include_str!("../src/execution.rs");
assert!(execution.contains("FuturesUnordered"));
assert!(execution.contains("hydrate_backfill_candidate"));
assert!(execution.contains("persist_backfill_hydration"));
assert!(execution.contains("request.hydration_concurrency()"));
for forbidden in ["tokio::spawn", "tokio::time", "retry", "endpoint_name", "provider()", "ForceRehydrate", "get_raw_transaction"] {
assert!(!execution.contains(forbidden), "forbidden scheduler/policy ownership detected: {forbidden}");
}
let runtime = include_str!("../src/runtime.rs");
assert!(runtime.contains("tokio::sync::watch"));
assert!(runtime.contains("JobSnapshotSource"));
assert!(runtime.contains("JobCancellationToken"));
assert!(!runtime.contains("tokio::spawn"));
assert!(!runtime.contains("tokio::time"));
assert!(!runtime.contains("reqwest::"));
return;
}

View File

@@ -0,0 +1,350 @@
// file: crates/ksp-job-backfill-lib/tests/hardening.rs
// version: 4
//! Adversarial, security, visibility and external-boundary hardening canaries for `pre.010`.
fn signature_text() -> std::string::String {
return "2".repeat(ksp_job_backfill_lib::MIN_BACKFILL_SIGNATURE_TEXT_BYTES);
}
fn signature() -> std::option::Option<ksp_job_backfill_lib::BackfillSignature> {
let result = ksp_job_backfill_lib::BackfillSignature::new(signature_text());
return match result {
std::result::Result::Ok(value) => std::option::Option::Some(value),
std::result::Result::Err(error) => {
assert_eq!(error.code(), ksp_job_backfill_lib::ERROR_CODE_BACKFILL_SIGNATURE_INVALID);
std::option::Option::None
},
};
}
fn network(value: &'static str) -> std::option::Option<ksp_store_lib::RawNetworkId> {
let result = ksp_store_lib::RawNetworkId::new(value);
return match result {
std::result::Result::Ok(value) => std::option::Option::Some(value),
std::result::Result::Err(error) => {
assert_eq!(error.code().domain(), "store_api");
std::option::Option::None
},
};
}
fn job_id(value: &'static str) -> std::option::Option<ksp_job_api::JobId> {
let result = ksp_job_api::JobId::new(value);
return match result {
std::result::Result::Ok(value) => std::option::Option::Some(value),
std::result::Result::Err(error) => {
assert_eq!(error.code().domain(), "job_api");
std::option::Option::None
},
};
}
fn explicit_scope() -> std::option::Option<ksp_job_backfill_lib::BackfillScope> {
let signature = match signature() {
std::option::Option::Some(value) => value,
std::option::Option::None => return std::option::Option::None,
};
let result = ksp_job_backfill_lib::BackfillScope::explicit_signatures(vec![signature]);
return match result {
std::result::Result::Ok(value) => std::option::Option::Some(value),
std::result::Result::Err(error) => {
assert_eq!(error.code(), ksp_job_backfill_lib::ERROR_CODE_BACKFILL_REQUEST_INVALID);
std::option::Option::None
},
};
}
fn request(
id: &'static str,
network_name: &'static str,
role_name: &'static str,
scope: ksp_job_backfill_lib::BackfillScope,
) -> std::option::Option<ksp_job_backfill_lib::BackfillRequest> {
let id = match job_id(id) {
std::option::Option::Some(value) => value,
std::option::Option::None => return std::option::Option::None,
};
let network = match network(network_name) {
std::option::Option::Some(value) => value,
std::option::Option::None => return std::option::Option::None,
};
let result = ksp_job_backfill_lib::BackfillRequest::new(
id,
network,
ksp_onchain_transport_lib::HttpRoleName::new(role_name),
ksp_job_backfill_lib::BackfillCommitment::Confirmed,
scope,
100,
10,
500,
8,
std::option::Option::None,
);
return match result {
std::result::Result::Ok(value) => std::option::Option::Some(value),
std::result::Result::Err(error) => {
assert_eq!(error.code(), ksp_job_backfill_lib::ERROR_CODE_BACKFILL_REQUEST_INVALID);
std::option::Option::None
},
};
}
#[test]
fn pre_010_external_error_codes_are_stable_unique_and_domain_scoped() {
let codes = [
(ksp_job_backfill_lib::ERROR_CODE_BACKFILL_CHECKPOINT_INVALID, "checkpoint_invalid"),
(ksp_job_backfill_lib::ERROR_CODE_BACKFILL_DISCOVERY_INVALID, "discovery_invalid"),
(ksp_job_backfill_lib::ERROR_CODE_BACKFILL_DISCOVERY_STALLED, "discovery_stalled"),
(ksp_job_backfill_lib::ERROR_CODE_BACKFILL_EXECUTION_INVALID, "execution_invalid"),
(ksp_job_backfill_lib::ERROR_CODE_BACKFILL_PERSISTENCE_INVALID, "persistence_invalid"),
(ksp_job_backfill_lib::ERROR_CODE_BACKFILL_RAW_CONVERSION_INVALID, "raw_conversion_invalid"),
(ksp_job_backfill_lib::ERROR_CODE_BACKFILL_REQUEST_INVALID, "request_invalid"),
(ksp_job_backfill_lib::ERROR_CODE_BACKFILL_RUNTIME_INVALID, "runtime_invalid"),
(ksp_job_backfill_lib::ERROR_CODE_BACKFILL_SIGNATURE_INVALID, "signature_invalid"),
];
for (index, (code, expected)) in codes.iter().enumerate() {
assert_eq!(code.domain(), "job_backfill");
assert_eq!(code.code(), *expected);
for (other_index, (other, _)) in codes.iter().enumerate() {
if index != other_index {
assert_ne!(code, other);
}
}
}
return;
}
#[test]
fn pre_010_external_request_adversarial_bounds_return_stable_codes() {
let invalid_signature = ksp_job_backfill_lib::BackfillSignature::new("0".repeat(ksp_job_backfill_lib::MIN_BACKFILL_SIGNATURE_TEXT_BYTES));
let error = match invalid_signature {
std::result::Result::Ok(_) => return,
std::result::Result::Err(error) => error,
};
assert_eq!(error.code(), ksp_job_backfill_lib::ERROR_CODE_BACKFILL_SIGNATURE_INVALID);
let id = match job_id("backfill:hardening:bounds") {
std::option::Option::Some(value) => value,
std::option::Option::None => return,
};
let network = match network("devnet") {
std::option::Option::Some(value) => value,
std::option::Option::None => return,
};
let scope = match explicit_scope() {
std::option::Option::Some(value) => value,
std::option::Option::None => return,
};
let result = ksp_job_backfill_lib::BackfillRequest::new(
id,
network,
ksp_onchain_transport_lib::HttpRoleName::new("history"),
ksp_job_backfill_lib::BackfillCommitment::Finalized,
scope,
0,
1,
1,
1,
std::option::Option::None,
);
let error = match result {
std::result::Result::Ok(_) => return,
std::result::Result::Err(error) => error,
};
assert_eq!(error.code(), ksp_job_backfill_lib::ERROR_CODE_BACKFILL_REQUEST_INVALID);
assert!(error.context().iter().any(|context| return context.key() == "field" && context.value() == "page_size"));
return;
}
#[test]
fn pre_010_scope_fingerprint_is_network_sensitive_and_transport_source_independent() {
let scope = match explicit_scope() {
std::option::Option::Some(value) => value,
std::option::Option::None => return,
};
let first = match request("backfill:hardening:fingerprint:a", "devnet", "history-primary", scope.clone()) {
std::option::Option::Some(value) => value,
std::option::Option::None => return,
};
let second = match request("backfill:hardening:fingerprint:b", "devnet", "history-secondary", scope.clone()) {
std::option::Option::Some(value) => value,
std::option::Option::None => return,
};
let other_network = match request("backfill:hardening:fingerprint:c", "mainnet", "history-primary", scope) {
std::option::Option::Some(value) => value,
std::option::Option::None => return,
};
assert_eq!(first.scope_fingerprint(), second.scope_fingerprint());
assert_ne!(first.scope_fingerprint(), other_network.scope_fingerprint());
return;
}
#[test]
fn pre_010_debug_surfaces_redact_signature_fingerprint_and_runtime_snapshot_payloads() {
let raw_signature = signature_text();
let scope = match explicit_scope() {
std::option::Option::Some(value) => value,
std::option::Option::None => return,
};
let backfill_request = match request("backfill:hardening:debug", "devnet", "history", scope) {
std::option::Option::Some(value) => value,
std::option::Option::None => return,
};
let request_debug = format!("{backfill_request:?}");
assert!(!request_debug.contains(raw_signature.as_str()));
assert!(request_debug.contains("BackfillScopeFingerprint(..)"));
assert!(request_debug.contains("signature_count"));
let fingerprint_debug = format!("{:?}", backfill_request.scope_fingerprint());
assert_eq!(fingerprint_debug, "BackfillScopeFingerprint(..)");
let runtime = ksp_job_backfill_lib::BackfillJobRuntime::new(backfill_request);
let runtime = match runtime {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return,
};
let handle_debug = format!("{:?}", runtime.handle());
assert!(!handle_debug.contains(raw_signature.as_str()));
assert!(handle_debug.contains("snapshot"));
assert!(handle_debug.contains("<redacted>"));
return;
}
#[test]
fn pre_010_external_runtime_handle_starts_created_and_cancellation_is_idempotent() {
let scope = match explicit_scope() {
std::option::Option::Some(value) => value,
std::option::Option::None => return,
};
let backfill_request = match request("backfill:hardening:control", "devnet", "history", scope) {
std::option::Option::Some(value) => value,
std::option::Option::None => return,
};
let runtime = ksp_job_backfill_lib::BackfillJobRuntime::new(backfill_request);
let runtime = match runtime {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return,
};
let handle = runtime.handle();
let source = handle.snapshots();
let current = <ksp_job_backfill_lib::BackfillSnapshotSource as ksp_job_api::JobSnapshotSource>::current(&source);
assert_eq!(current.state(), ksp_job_api::JobState::Created);
assert_eq!(current.snapshot().phase(), ksp_job_backfill_lib::BackfillJobPhase::Created);
assert!(!handle.is_cancellation_requested());
assert!(handle.cancel());
assert!(handle.is_cancellation_requested());
assert!(!handle.cancel());
return;
}
#[test]
fn pre_010_manifest_dependency_surface_is_exact_and_backend_neutral() {
let manifest = include_str!("../Cargo.toml");
let mut section = "";
let mut normal = std::collections::BTreeSet::new();
let mut dev = std::collections::BTreeSet::new();
for line in manifest.lines() {
let trimmed = line.trim();
if trimmed.starts_with('[') && trimmed.ends_with(']') {
section = trimmed;
continue;
}
if trimmed.is_empty() || trimmed.starts_with('#') {
continue;
}
let name = match trimmed.split_once('=') {
std::option::Option::Some((name, _)) => name.trim(),
std::option::Option::None => continue,
};
if section == "[dependencies]" {
normal.insert(name);
} else if section == "[dev-dependencies]" {
dev.insert(name);
}
}
assert_eq!(
normal,
std::collections::BTreeSet::from([
"futures-util",
"ksp-core-lib",
"ksp-job-api",
"ksp-logging-lib",
"ksp-onchain-transport-lib",
"ksp-store-lib",
"serde_json.workspace",
"sha2.workspace",
"tokio",
])
);
assert_eq!(dev, std::collections::BTreeSet::from(["tokio"]));
assert!(manifest.contains("ksp-store-lib = { path = \"../ksp-store-lib\", default-features = false }"));
assert!(manifest.contains("tokio = { workspace = true, features = [\"macros\", \"sync\"] }"));
assert!(!manifest.contains("ksp-store-postgres-lib"));
assert!(!manifest.contains("ksp-store-api"));
return;
}
#[test]
fn pre_010_source_visibility_contract_uses_crate_root_for_shared_items() {
let root = include_str!("../src/lib.rs");
assert!(!root.contains("BackfillRuntimeControl"));
assert!(!root.contains("TerminalClaim"));
let source_contracts: [(&str, &[&str]); 3] = [
(include_str!("../src/checkpoint.rs"), &["BackfillCheckpoint", "CompletionFrontier"]),
(
include_str!("../src/request.rs"),
&["BackfillCommitment", "BackfillSignature", "BackfillScopeKind", "BackfillScope", "BackfillScopeFingerprint", "BackfillRequest"],
),
(include_str!("../src/runtime.rs"), &["BackfillJobPhase", "BackfillJobSnapshot", "BackfillJobHandle", "BackfillJobRuntime"]),
];
for (source, symbols) in source_contracts {
for symbol in symbols {
let required = format!("impl crate::{symbol}");
assert!(source.contains(required.as_str()), "shared item must use crate-root impl path: {symbol}");
let forbidden = format!("impl {symbol}");
assert!(!source.contains(forbidden.as_str()), "shared item must not use bare impl path: {symbol}");
}
}
let request = include_str!("../src/request.rs");
assert!(request.contains("crate::BackfillScopeFingerprint::from_bytes(bytes)"));
assert!(!request.contains("crate::BackfillScopeFingerprint(bytes)"));
assert!(!request.contains("return BackfillScopeFingerprint(bytes)"));
for (module, source) in [
("checkpoint", include_str!("../src/checkpoint.rs")),
("conversion", include_str!("../src/conversion.rs")),
("discovery", include_str!("../src/discovery.rs")),
("execution", include_str!("../src/execution.rs")),
("persistence", include_str!("../src/persistence.rs")),
("request", include_str!("../src/request.rs")),
("runtime", include_str!("../src/runtime.rs")),
] {
let forbidden = format!("crate::{module}::");
assert!(!source.contains(forbidden.as_str()), "internal module path bypasses crate-root façade: {forbidden}");
}
return;
}
#[test]
fn pre_010_lower_layers_have_no_dependency_return_to_job() {
for manifest in [
include_str!("../../ksp-core-lib/Cargo.toml"),
include_str!("../../ksp-logging-lib/Cargo.toml"),
include_str!("../../ksp-onchain-transport-lib/Cargo.toml"),
include_str!("../../ksp-store-api/Cargo.toml"),
include_str!("../../ksp-store-lib/Cargo.toml"),
include_str!("../../ksp-store-postgres-lib/Cargo.toml"),
] {
assert!(!manifest.contains("ksp-job-api"));
assert!(!manifest.contains("ksp-job-backfill-lib"));
}
return;
}
#[test]
fn pre_010_public_root_exposes_no_runtime_or_backend_implementation_types() {
let root = include_str!("../src/lib.rs");
for forbidden in
["pub mod ", "tokio::", "FuturesUnordered", "serde_json::", "sha2::", "ksp_store_api::", "ksp_store_postgres_lib::", "reqwest::", "tonic::"]
{
assert!(!root.contains(forbidden), "implementation/backend detail leaked into public root: {forbidden}");
}
return;
}

View File

@@ -0,0 +1,125 @@
// file: crates/ksp-job-backfill-lib/tests/public_api.rs
// version: 7
//! Public API canaries through the concrete Backfill cancellation and latest-value runtime tranche.
#[test]
fn pre_005_request_scope_and_discovery_contracts_are_available_from_crate_root() {
let _discover = ksp_job_backfill_lib::discover_backfill_candidates;
let address = ksp_core_lib::Pubkey::new_from_array([21_u8; 32]);
let scope = ksp_job_backfill_lib::BackfillScope::latest_address(address);
let network = ksp_store_lib::RawNetworkId::new("devnet");
assert!(network.is_ok());
let network = match network {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return,
};
let job_id = ksp_job_api::JobId::new("backfill:public-api");
assert!(job_id.is_ok());
let job_id = match job_id {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return,
};
let request = ksp_job_backfill_lib::BackfillRequest::new(
job_id,
network,
ksp_onchain_transport_lib::HttpRoleName::new("history"),
ksp_job_backfill_lib::BackfillCommitment::Confirmed,
scope,
100,
10,
500,
8,
std::option::Option::Some(100),
);
assert!(request.is_ok());
let request = match request {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return,
};
assert_eq!(request.scope().kind(), ksp_job_backfill_lib::BackfillScopeKind::LatestAddress);
assert_eq!(request.page_size(), 100);
return;
}
#[test]
fn pre_005_candidate_identity_is_network_plus_signature_not_transport_source() {
let signature = ksp_job_backfill_lib::BackfillSignature::new("1".repeat(ksp_job_backfill_lib::MIN_BACKFILL_SIGNATURE_TEXT_BYTES));
let signature = match signature {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return,
};
let mainnet = ksp_store_lib::RawNetworkId::new("mainnet");
let devnet = ksp_store_lib::RawNetworkId::new("devnet");
let mainnet = match mainnet {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return,
};
let devnet = match devnet {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return,
};
let mainnet_identity = ksp_job_backfill_lib::BackfillCandidateIdentity::new(mainnet, signature.clone());
let devnet_identity = ksp_job_backfill_lib::BackfillCandidateIdentity::new(devnet, signature);
assert_ne!(mainnet_identity, devnet_identity);
return;
}
#[test]
fn pre_006_raw_conversion_contract_is_available_from_crate_root() {
assert_eq!(ksp_job_backfill_lib::RAW_TRANSACTION_FORMAT_ID, "ksp.solana.raw_transaction");
assert_eq!(ksp_job_backfill_lib::RAW_TRANSACTION_FORMAT_VERSION, 1);
let signature = ksp_job_backfill_lib::BackfillSignature::new("1".repeat(64));
assert!(signature.is_ok());
let signature = match signature {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return,
};
let decoded = signature.to_raw_transaction_signature();
assert!(decoded.is_ok());
if let std::result::Result::Ok(decoded) = decoded {
assert_eq!(decoded.as_bytes(), &[0_u8; 64]);
}
let _hydrate = ksp_job_backfill_lib::hydrate_backfill_candidate;
let _outcome: std::option::Option<ksp_job_backfill_lib::BackfillHydrationOutcome> = std::option::Option::None;
let _acquisition: std::option::Option<ksp_job_backfill_lib::BackfillRawAcquisition> = std::option::Option::None;
assert_eq!(ksp_job_backfill_lib::ERROR_CODE_BACKFILL_RAW_CONVERSION_INVALID, ksp_core_lib::ErrorCode::new("job_backfill", "raw_conversion_invalid"));
return;
}
#[test]
fn pre_007_store_persistence_contract_is_available_from_crate_root() {
let _persist = ksp_job_backfill_lib::persist_backfill_hydration;
let _outcome: std::option::Option<ksp_job_backfill_lib::BackfillPersistenceOutcome> = std::option::Option::None;
let _entity = ksp_job_backfill_lib::BackfillEntityPersistence::AlreadyPresent;
let _observation = ksp_job_backfill_lib::BackfillObservationPersistence::AlreadyPresent;
assert_eq!(ksp_job_backfill_lib::ERROR_CODE_BACKFILL_PERSISTENCE_INVALID, ksp_core_lib::ErrorCode::new("job_backfill", "persistence_invalid"));
return;
}
#[test]
fn pre_008_checkpoint_and_bounded_execution_contracts_are_available_from_crate_root() {
let _execute = ksp_job_backfill_lib::execute_backfill_discovery;
let _checkpoint: std::option::Option<ksp_job_backfill_lib::BackfillCheckpoint> = std::option::Option::None;
let _batch: std::option::Option<ksp_job_backfill_lib::BackfillExecutionBatch> = std::option::Option::None;
assert_eq!(ksp_job_backfill_lib::ERROR_CODE_BACKFILL_CHECKPOINT_INVALID, ksp_core_lib::ErrorCode::new("job_backfill", "checkpoint_invalid"));
assert_eq!(ksp_job_backfill_lib::ERROR_CODE_BACKFILL_EXECUTION_INVALID, ksp_core_lib::ErrorCode::new("job_backfill", "execution_invalid"));
return;
}
#[test]
fn pre_009_concrete_runtime_snapshot_and_control_contracts_are_available_from_crate_root() {
fn assert_source<T>(_: std::marker::PhantomData<T>)
where
T: ksp_job_api::JobSnapshotSource<Snapshot = ksp_job_backfill_lib::BackfillJobSnapshot>,
{
return;
}
assert_source(std::marker::PhantomData::<ksp_job_backfill_lib::BackfillSnapshotSource>);
let _runtime_new = ksp_job_backfill_lib::BackfillJobRuntime::new;
let _handle: std::option::Option<ksp_job_backfill_lib::BackfillJobHandle> = std::option::Option::None;
let _phase = ksp_job_backfill_lib::BackfillJobPhase::Discovering;
assert_eq!(ksp_job_backfill_lib::BACKFILL_JOB_KIND_CODE, "solana.raw_transaction.backfill");
assert_eq!(ksp_job_backfill_lib::ERROR_CODE_BACKFILL_RUNTIME_INVALID, ksp_core_lib::ErrorCode::new("job_backfill", "runtime_invalid"));
return;
}

View File

@@ -0,0 +1,113 @@
// file: crates/ksp-job-backfill-lib/tests/release_completeness.rs
// version: 9
//! Completeness canaries through the `pre.010` hardening and external-canary tranche.
#[test]
fn pre_010_production_module_inventory_is_exact() -> std::io::Result<()> {
let source_root = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("src");
let entries = match std::fs::read_dir(source_root) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let mut names = std::vec::Vec::new();
for entry in entries {
let entry = match entry {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let file_type = match entry.file_type() {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
if !file_type.is_file() {
continue;
}
let name = match entry.file_name().into_string() {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => continue,
};
if name.ends_with(".rs") {
names.push(name);
}
}
names.sort_unstable();
assert_eq!(
names,
std::vec![
"checkpoint.rs",
"constants.rs",
"conversion.rs",
"discovery.rs",
"error.rs",
"execution.rs",
"lib.rs",
"persistence.rs",
"request.rs",
"runtime.rs",
]
);
return std::result::Result::Ok(());
}
#[test]
fn pre_010_surface_closes_concrete_cancellation_and_latest_value_runtime() {
let root = include_str!("../src/lib.rs");
for required in [
"BackfillCandidate",
"BackfillCandidateIdentity",
"BackfillCommitment",
"BackfillDiscovery",
"BackfillDiscoveryBoundary",
"BackfillRequest",
"BackfillScope",
"BackfillScopeFingerprint",
"BackfillSignature",
"discover_backfill_candidates",
"BackfillRawAcquisition",
"BackfillHydrationOutcome",
"hydrate_backfill_candidate",
"BackfillPersistenceOutcome",
"BackfillEntityPersistence",
"BackfillObservationPersistence",
"persist_backfill_hydration",
"BackfillCheckpoint",
"BackfillExecutionBatch",
"execute_backfill_discovery",
"ERROR_CODE_BACKFILL_CHECKPOINT_INVALID",
"ERROR_CODE_BACKFILL_EXECUTION_INVALID",
"BackfillJobHandle",
"BackfillJobPhase",
"BackfillJobRuntime",
"BackfillJobSnapshot",
"BackfillSnapshotSource",
"BACKFILL_JOB_KIND_CODE",
"ERROR_CODE_BACKFILL_RUNTIME_INVALID",
] {
assert!(root.contains(required), "required pre.010 public contract missing: {required}");
}
for forbidden in ["tokio::", "FuturesUnordered", "watch::Receiver", "watch::Sender"] {
assert!(!root.contains(forbidden), "runtime implementation detail leaked into public root: {forbidden}");
}
assert!(!root.contains("pub mod "));
return;
}
#[test]
fn pre_010_external_hardening_suite_is_present_and_scoped() {
let hardening = include_str!("hardening.rs");
for required in [
"pre_010_external_error_codes_are_stable_unique_and_domain_scoped",
"pre_010_external_request_adversarial_bounds_return_stable_codes",
"pre_010_scope_fingerprint_is_network_sensitive_and_transport_source_independent",
"pre_010_debug_surfaces_redact_signature_fingerprint_and_runtime_snapshot_payloads",
"pre_010_external_runtime_handle_starts_created_and_cancellation_is_idempotent",
"pre_010_manifest_dependency_surface_is_exact_and_backend_neutral",
"pre_010_source_visibility_contract_uses_crate_root_for_shared_items",
"pre_010_lower_layers_have_no_dependency_return_to_job",
"pre_010_public_root_exposes_no_runtime_or_backend_implementation_types",
] {
assert!(hardening.contains(required), "required pre.010 hardening canary missing: {required}");
}
return;
}

View File

@@ -0,0 +1,168 @@
// file: crates/ksp-job-backfill-lib/unit_tests/checkpoint.rs
// version: 2
fn signature(character: char) -> std::option::Option<crate::BackfillSignature> {
return crate::BackfillSignature::new(character.to_string().repeat(crate::MIN_BACKFILL_SIGNATURE_TEXT_BYTES)).ok();
}
fn request(scope: crate::BackfillScope) -> std::option::Option<crate::BackfillRequest> {
let job_id = match ksp_job_api::JobId::new("backfill:checkpoint-test") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return std::option::Option::None,
};
let network = match ksp_store_lib::RawNetworkId::new("devnet") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return std::option::Option::None,
};
return crate::BackfillRequest::new(
job_id,
network,
ksp_onchain_transport_lib::HttpRoleName::new("history"),
crate::BackfillCommitment::Confirmed,
scope,
100,
10,
10,
4,
std::option::Option::None,
)
.ok();
}
fn candidate(request: &crate::BackfillRequest, character: char) -> std::option::Option<crate::BackfillCandidate> {
let signature = match signature(character) {
std::option::Option::Some(value) => value,
std::option::Option::None => return std::option::Option::None,
};
let identity = crate::BackfillCandidateIdentity::new(request.network().clone(), signature);
return std::option::Option::Some(crate::BackfillCandidate::new(identity, std::option::Option::None));
}
fn discovery(
request: &crate::BackfillRequest,
characters: &[char],
boundary: crate::BackfillDiscoveryBoundary,
) -> std::option::Option<crate::BackfillDiscovery> {
let mut candidates = std::vec::Vec::with_capacity(characters.len());
for character in characters {
let value = match candidate(request, *character) {
std::option::Option::Some(value) => value,
std::option::Option::None => return std::option::Option::None,
};
candidates.push(value);
}
return std::option::Option::Some(crate::BackfillDiscovery::new(request.network().clone(), request.scope_fingerprint(), candidates, 1, boundary));
}
#[test]
fn pre_008_frontier_advances_only_across_contiguous_durable_results() {
let mut frontier = crate::CompletionFrontier::new(4);
assert!(frontier.mark_durable(1).is_ok());
assert_eq!(frontier.contiguous_completed(), 0);
assert!(frontier.mark_durable(0).is_ok());
assert_eq!(frontier.contiguous_completed(), 2);
assert!(frontier.mark_durable(3).is_ok());
assert_eq!(frontier.contiguous_completed(), 2);
assert!(frontier.mark_durable(2).is_ok());
assert_eq!(frontier.contiguous_completed(), 4);
return;
}
#[test]
fn pre_008_checkpoint_is_bound_to_job_and_scope_fingerprint() {
let scope = crate::BackfillScope::latest_address(ksp_core_lib::Pubkey::new_from_array([1_u8; 32]));
let primary_request = match request(scope) {
std::option::Option::Some(value) => value,
std::option::Option::None => return,
};
let other_job = match ksp_job_api::JobId::new("backfill:other") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return,
};
let wrong_job = crate::BackfillCheckpoint::new(other_job, primary_request.scope_fingerprint(), 0, std::option::Option::None);
let result = primary_request.clone().with_checkpoint(wrong_job);
assert!(result.is_err());
if let std::result::Result::Err(error) = result {
assert_eq!(error.code(), crate::ERROR_CODE_BACKFILL_CHECKPOINT_INVALID);
}
let other_scope = crate::BackfillScope::latest_address(ksp_core_lib::Pubkey::new_from_array([2_u8; 32]));
let other_request = match request(other_scope) {
std::option::Option::Some(value) => value,
std::option::Option::None => return,
};
let wrong_scope = crate::BackfillCheckpoint::new(primary_request.job_id().clone(), other_request.scope_fingerprint(), 0, std::option::Option::None);
assert!(primary_request.with_checkpoint(wrong_scope).is_err());
return;
}
#[test]
fn pre_008_before_checkpoint_is_cumulative_and_tracks_last_contiguous_candidate() {
let anchor = match signature('8') {
std::option::Option::Some(value) => value,
std::option::Option::None => return,
};
let scope = crate::BackfillScope::before_address(ksp_core_lib::Pubkey::new_from_array([3_u8; 32]), anchor);
let request = match request(scope) {
std::option::Option::Some(value) => value,
std::option::Option::None => return,
};
let first_discovery = match discovery(&request, &['7', '6', '5'], crate::BackfillDiscoveryBoundary::RpcBoundary) {
std::option::Option::Some(value) => value,
std::option::Option::None => return,
};
let mut first_frontier = crate::CompletionFrontier::new(3);
assert!(first_frontier.mark_durable(1).is_ok());
assert_eq!(first_frontier.contiguous_completed(), 0);
assert!(first_frontier.mark_durable(0).is_ok());
let first = match crate::checkpoint_from_frontier(&request, &first_discovery, &first_frontier) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return,
};
assert_eq!(first.completed_prefix(), 2);
let expected_first = "6".repeat(64);
assert_eq!(first.resume_before().map(crate::BackfillSignature::as_str), std::option::Option::Some(expected_first.as_str()));
let resumed = match request.with_checkpoint(first) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return,
};
let second_discovery = match discovery(&resumed, &['5', '4'], crate::BackfillDiscoveryBoundary::RpcBoundary) {
std::option::Option::Some(value) => value,
std::option::Option::None => return,
};
let mut second_frontier = crate::CompletionFrontier::new(2);
assert!(second_frontier.mark_durable(0).is_ok());
let second = match crate::checkpoint_from_frontier(&resumed, &second_discovery, &second_frontier) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return,
};
assert_eq!(second.completed_prefix(), 3);
let expected_second = "5".repeat(64);
assert_eq!(second.resume_before().map(crate::BackfillSignature::as_str), std::option::Option::Some(expected_second.as_str()));
return;
}
#[test]
fn pre_008_after_anchor_not_reached_never_advances_checkpoint() {
let anchor = match signature('1') {
std::option::Option::Some(value) => value,
std::option::Option::None => return,
};
let scope = crate::BackfillScope::after_address(ksp_core_lib::Pubkey::new_from_array([4_u8; 32]), anchor);
let request = match request(scope) {
std::option::Option::Some(value) => value,
std::option::Option::None => return,
};
let partial = match discovery(&request, &['7', '6', '5'], crate::BackfillDiscoveryBoundary::AfterAnchorNotReached) {
std::option::Option::Some(value) => value,
std::option::Option::None => return,
};
let mut frontier = crate::CompletionFrontier::new(3);
assert!(frontier.mark_durable(0).is_ok());
assert!(frontier.mark_durable(1).is_ok());
let checkpoint = match crate::checkpoint_from_frontier(&request, &partial, &frontier) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return,
};
assert_eq!(checkpoint.completed_prefix(), 0);
return;
}

View File

@@ -0,0 +1,301 @@
// file: crates/ksp-job-backfill-lib/unit_tests/conversion.rs
// version: 2
fn signature_text() -> std::option::Option<crate::BackfillSignature> {
return match crate::BackfillSignature::new("1".repeat(64)) {
std::result::Result::Ok(value) => std::option::Option::Some(value),
std::result::Result::Err(_) => std::option::Option::None,
};
}
fn request(network: &str, signature: crate::BackfillSignature) -> std::option::Option<crate::BackfillRequest> {
let network = match ksp_store_lib::RawNetworkId::new(network) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return std::option::Option::None,
};
let job_id = match ksp_job_api::JobId::new("backfill:pre006") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return std::option::Option::None,
};
let scope = match crate::BackfillScope::explicit_signatures(std::vec![signature]) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return std::option::Option::None,
};
return match crate::BackfillRequest::new(
job_id,
network,
ksp_onchain_transport_lib::HttpRoleName::new("history"),
crate::BackfillCommitment::Finalized,
scope,
100,
10,
100,
4,
std::option::Option::None,
) {
std::result::Result::Ok(value) => std::option::Option::Some(value),
std::result::Result::Err(_) => std::option::Option::None,
};
}
fn candidate(network: &str, signature: crate::BackfillSignature) -> std::option::Option<crate::BackfillCandidate> {
let network = match ksp_store_lib::RawNetworkId::new(network) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return std::option::Option::None,
};
return std::option::Option::Some(crate::BackfillCandidate::new(crate::BackfillCandidateIdentity::new(network, signature), std::option::Option::Some(42)));
}
fn received_at() -> std::option::Option<ksp_store_lib::RawTimestamp> {
return match ksp_store_lib::RawTimestamp::from_unix_millis(1_700_000_001_000) {
std::result::Result::Ok(value) => std::option::Option::Some(value),
std::result::Result::Err(_) => std::option::Option::None,
};
}
fn fields<'a>(
transaction: &'a ksp_onchain_transport_lib::SolanaEncodedTransaction,
meta: &'a ksp_onchain_transport_lib::SolanaWireField<serde_json::Value>,
version: &'a ksp_onchain_transport_lib::SolanaWireField<ksp_onchain_transport_lib::SolanaTransactionVersion>,
transaction_index: &'a ksp_onchain_transport_lib::SolanaWireField<u32>,
block_time: std::option::Option<i64>,
) -> super::CanonicalTransactionFields<'a> {
return super::CanonicalTransactionFields { slot: 123, block_time, transaction, meta, version, transaction_index };
}
#[test]
fn pre_006_signature_decoding_requires_exactly_sixty_four_bytes() {
let signature = match signature_text() {
std::option::Option::Some(value) => value,
std::option::Option::None => return,
};
let decoded = signature.to_raw_transaction_signature();
assert!(decoded.is_ok());
if let std::result::Result::Ok(decoded) = decoded {
assert_eq!(decoded.as_bytes(), &[0_u8; 64]);
}
let too_many_zeroes = crate::BackfillSignature::new("1".repeat(65));
assert!(too_many_zeroes.is_ok());
if let std::result::Result::Ok(too_many_zeroes) = too_many_zeroes {
assert!(too_many_zeroes.to_raw_transaction_signature().is_err());
}
let overflow = crate::BackfillSignature::new("z".repeat(88));
assert!(overflow.is_ok());
if let std::result::Result::Ok(overflow) = overflow {
assert!(overflow.to_raw_transaction_signature().is_err());
}
return;
}
#[test]
fn pre_006_canonical_payload_golden_bytes_hash_and_provenance_are_exact() {
let signature = match signature_text() {
std::option::Option::Some(value) => value,
std::option::Option::None => return,
};
let request = match request("devnet", signature.clone()) {
std::option::Option::Some(value) => value,
std::option::Option::None => return,
};
let candidate = match candidate("devnet", signature) {
std::option::Option::Some(value) => value,
std::option::Option::None => return,
};
let reference = match super::canonical_reference(&request, &candidate) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return,
};
let transaction = ksp_onchain_transport_lib::SolanaEncodedTransaction::Binary {
data: "AQID".to_owned(),
encoding: ksp_onchain_transport_lib::SolanaTransactionBinaryEncoding::Base64,
};
let meta = ksp_onchain_transport_lib::SolanaWireField::Value(serde_json::json!({"z": 1, "a": {"y": true, "x": null}}));
let version = ksp_onchain_transport_lib::SolanaWireField::Value(ksp_onchain_transport_lib::SolanaTransactionVersion::Legacy);
let transaction_index = ksp_onchain_transport_lib::SolanaWireField::Value(7_u32);
let received_at = match received_at() {
std::option::Option::Some(value) => value,
std::option::Option::None => return,
};
let acquisition = super::convert_available_fields(
&request,
reference,
fields(&transaction, &meta, &version, &transaction_index, std::option::Option::Some(1_700_000_000)),
"publicnode",
"devnet-primary",
received_at,
);
assert!(acquisition.is_ok());
let acquisition = match acquisition {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return,
};
let expected = b"{\"transaction\":[\"AQID\",\"base64\"],\"meta\":{\"a\":{\"x\":null,\"y\":true},\"z\":1},\"version\":\"legacy\",\"transactionIndex\":7}";
assert_eq!(acquisition.transaction().payload().bytes(), expected);
assert_eq!(acquisition.transaction().payload().byte_len(), 112);
assert_eq!(acquisition.transaction().payload().format_id().as_str(), crate::RAW_TRANSACTION_FORMAT_ID);
assert_eq!(acquisition.transaction().payload().format_version(), crate::RAW_TRANSACTION_FORMAT_VERSION);
assert_eq!(
acquisition.transaction().payload().content_hash().as_bytes(),
&[
34, 7, 146, 210, 177, 93, 38, 47, 218, 36, 44, 178, 32, 119, 78, 233, 221, 239, 254, 191, 4, 220, 250, 218, 188, 248, 239, 118, 169, 177, 167, 195
]
);
assert_eq!(acquisition.transaction().slot(), 123);
assert_eq!(acquisition.transaction().block_time().map(|value| return value.unix_millis()), std::option::Option::Some(1_700_000_000_000));
assert_eq!(acquisition.transaction().reference().network().as_str(), "devnet");
assert_eq!(acquisition.transaction().reference().signature().as_bytes(), &[0_u8; 64]);
let provenance = acquisition.observation().provenance();
assert_eq!(provenance.provider().as_str(), "publicnode");
assert_eq!(provenance.protocol().as_str(), "solana.http.json_rpc");
assert_eq!(provenance.acquisition_method().as_str(), "getTransaction");
assert_eq!(provenance.endpoint_id().map(ksp_store_lib::RawProvenanceCode::as_str), std::option::Option::Some("devnet-primary"));
assert_eq!(provenance.commitment().map(ksp_store_lib::RawProvenanceCode::as_str), std::option::Option::Some("finalized"));
assert_eq!(provenance.capture_session_id().map(ksp_store_lib::RawProvenanceCode::as_str), std::option::Option::Some("backfill:pre006"));
assert_eq!(provenance.received_at(), received_at);
assert!(provenance.source_payload_hash().is_none());
assert!(provenance.source_payload_size_bytes().is_none());
return;
}
#[test]
fn pre_006_wire_omission_and_null_produce_distinct_canonical_bytes() {
let transaction = ksp_onchain_transport_lib::SolanaEncodedTransaction::Binary {
data: "AQID".to_owned(),
encoding: ksp_onchain_transport_lib::SolanaTransactionBinaryEncoding::Base64,
};
let meta_omitted = ksp_onchain_transport_lib::SolanaWireField::Omitted;
let meta_null = ksp_onchain_transport_lib::SolanaWireField::Null;
let version_omitted = ksp_onchain_transport_lib::SolanaWireField::Omitted;
let version_null = ksp_onchain_transport_lib::SolanaWireField::Null;
let transaction_index_omitted = ksp_onchain_transport_lib::SolanaWireField::Omitted;
let transaction_index_null = ksp_onchain_transport_lib::SolanaWireField::Null;
let omitted = super::canonical_payload_bytes(&fields(&transaction, &meta_omitted, &version_omitted, &transaction_index_omitted, std::option::Option::None));
assert!(omitted.is_ok());
let omitted = match omitted {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return,
};
assert_eq!(omitted, b"{\"transaction\":[\"AQID\",\"base64\"]}");
let nulls = super::canonical_payload_bytes(&fields(&transaction, &meta_null, &version_null, &transaction_index_null, std::option::Option::None));
assert!(nulls.is_ok());
let nulls = match nulls {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return,
};
assert_eq!(nulls, b"{\"transaction\":[\"AQID\",\"base64\"],\"meta\":null,\"version\":null,\"transactionIndex\":null}");
assert_ne!(omitted, nulls);
return;
}
#[test]
fn pre_006_non_base64_transaction_shapes_are_rejected() {
let base58 = ksp_onchain_transport_lib::SolanaEncodedTransaction::Binary {
data: "1111".to_owned(),
encoding: ksp_onchain_transport_lib::SolanaTransactionBinaryEncoding::Base58,
};
let legacy = ksp_onchain_transport_lib::SolanaEncodedTransaction::LegacyBinary("1111".to_owned());
let json = ksp_onchain_transport_lib::SolanaEncodedTransaction::Json(serde_json::json!({"message": {}}));
let meta = ksp_onchain_transport_lib::SolanaWireField::Omitted;
let version = ksp_onchain_transport_lib::SolanaWireField::Omitted;
let transaction_index = ksp_onchain_transport_lib::SolanaWireField::Omitted;
for transaction in [&base58, &legacy, &json] {
let result = super::canonical_payload_bytes(&fields(transaction, &meta, &version, &transaction_index, std::option::Option::None));
assert!(result.is_err());
if let std::result::Result::Err(error) = result {
assert_eq!(error.code(), crate::ERROR_CODE_BACKFILL_RAW_CONVERSION_INVALID);
}
}
return;
}
#[test]
fn pre_006_negative_and_unrepresentable_block_times_are_terminal_conversion_errors() {
let negative = super::convert_block_time(std::option::Option::Some(-1));
assert!(negative.is_err());
let oversized = super::convert_block_time(std::option::Option::Some(i64::MAX));
assert!(oversized.is_err());
let absent = super::convert_block_time(std::option::Option::None);
assert!(matches!(absent, std::result::Result::Ok(std::option::Option::None)));
return;
}
#[test]
fn pre_006_observation_key_is_deterministic_and_endpoint_specific() {
let signature = match signature_text() {
std::option::Option::Some(value) => value,
std::option::Option::None => return,
};
let request = match request("devnet", signature.clone()) {
std::option::Option::Some(value) => value,
std::option::Option::None => return,
};
let candidate = match candidate("devnet", signature) {
std::option::Option::Some(value) => value,
std::option::Option::None => return,
};
let reference = match super::canonical_reference(&request, &candidate) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return,
};
let first = super::observation_key(&request, &reference, "provider", "endpoint-a");
let same = super::observation_key(&request, &reference, "provider", "endpoint-a");
let other_endpoint = super::observation_key(&request, &reference, "provider", "endpoint-b");
let other_provider = super::observation_key(&request, &reference, "provider-2", "endpoint-a");
assert_eq!(first, same);
assert_ne!(first, other_endpoint);
assert_ne!(first, other_provider);
return;
}
#[test]
fn pre_006_candidate_network_mismatch_is_rejected_before_transport() {
let signature = match signature_text() {
std::option::Option::Some(value) => value,
std::option::Option::None => return,
};
let request = match request("devnet", signature.clone()) {
std::option::Option::Some(value) => value,
std::option::Option::None => return,
};
let candidate = match candidate("mainnet", signature) {
std::option::Option::Some(value) => value,
std::option::Option::None => return,
};
let reference = super::canonical_reference(&request, &candidate);
assert!(reference.is_err());
if let std::result::Result::Err(error) = reference {
assert_eq!(error.code(), crate::ERROR_CODE_BACKFILL_RAW_CONVERSION_INVALID);
}
return;
}
#[test]
fn pre_006_missing_outcome_contains_only_network_scoped_reference() {
let signature = match signature_text() {
std::option::Option::Some(value) => value,
std::option::Option::None => return,
};
let request = match request("devnet", signature.clone()) {
std::option::Option::Some(value) => value,
std::option::Option::None => return,
};
let candidate = match candidate("devnet", signature) {
std::option::Option::Some(value) => value,
std::option::Option::None => return,
};
let reference = match super::canonical_reference(&request, &candidate) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return,
};
let outcome = crate::BackfillHydrationOutcome::Missing(reference);
assert!(outcome.is_missing());
assert_eq!(outcome.reference().network().as_str(), "devnet");
assert_eq!(outcome.reference().signature().as_bytes(), &[0_u8; 64]);
return;
}
#[test]
fn pre_006_fix_001_raw_acquisition_uses_one_private_indirection() {
assert_eq!(std::mem::size_of::<crate::BackfillRawAcquisition>(), std::mem::size_of::<usize>(),);
return;
}

View File

@@ -0,0 +1,412 @@
// file: crates/ksp-job-backfill-lib/unit_tests/discovery.rs
// version: 6
#[derive(Clone, Debug, Eq, PartialEq)]
struct PageCall {
before: std::option::Option<std::string::String>,
until: std::option::Option<std::string::String>,
limit: std::option::Option<usize>,
commitment: std::option::Option<ksp_onchain_transport_lib::SolanaCommitment>,
min_context_slot: std::option::Option<u64>,
}
struct FakeSource {
pages: std::sync::Mutex<std::collections::VecDeque<std::vec::Vec<super::SignaturePageEntry>>>,
calls: std::sync::Mutex<std::vec::Vec<PageCall>>,
}
struct PendingSource {
calls: std::sync::atomic::AtomicUsize,
}
impl PendingSource {
fn new() -> Self {
return Self { calls: std::sync::atomic::AtomicUsize::new(0) };
}
fn calls(&self) -> usize {
return self.calls.load(std::sync::atomic::Ordering::Acquire);
}
}
impl super::SignaturePageSource for PendingSource {
fn fetch_signature_page<'a>(
&'a self,
_role: &'a ksp_onchain_transport_lib::HttpRoleName,
_address: &'a ksp_core_lib::Pubkey,
_config: ksp_onchain_transport_lib::SolanaSignaturesForAddressConfig,
) -> super::SignaturePageFuture<'a> {
self.calls.fetch_add(1, std::sync::atomic::Ordering::AcqRel);
return std::boxed::Box::pin(async {
return std::future::pending::<ksp_core_lib::Result<std::vec::Vec<super::SignaturePageEntry>>>().await;
});
}
}
impl FakeSource {
fn new(pages: std::vec::Vec<std::vec::Vec<super::SignaturePageEntry>>) -> Self {
return Self { pages: std::sync::Mutex::new(pages.into()), calls: std::sync::Mutex::new(std::vec::Vec::new()) };
}
fn calls(&self) -> std::vec::Vec<PageCall> {
let guard = self.calls.lock();
return match guard {
std::result::Result::Ok(value) => value.clone(),
std::result::Result::Err(_) => std::vec::Vec::new(),
};
}
}
impl super::SignaturePageSource for FakeSource {
fn fetch_signature_page<'a>(
&'a self,
_role: &'a ksp_onchain_transport_lib::HttpRoleName,
_address: &'a ksp_core_lib::Pubkey,
config: ksp_onchain_transport_lib::SolanaSignaturesForAddressConfig,
) -> super::SignaturePageFuture<'a> {
let call = PageCall {
before: config.before().map(str::to_owned),
until: config.until().map(str::to_owned),
limit: config.limit(),
commitment: config.commitment(),
min_context_slot: config.min_context_slot(),
};
let calls_result = self.calls.lock();
match calls_result {
std::result::Result::Ok(mut calls) => calls.push(call),
std::result::Result::Err(_) => {
return std::boxed::Box::pin(async {
return std::result::Result::Err(ksp_core_lib::Error::new(
crate::ERROR_CODE_BACKFILL_DISCOVERY_INVALID,
"test call recorder lock poisoned",
));
});
},
}
let pages_result = self.pages.lock();
let page = match pages_result {
std::result::Result::Ok(mut pages) => match pages.pop_front() {
std::option::Option::Some(value) => value,
std::option::Option::None => std::vec::Vec::new(),
},
std::result::Result::Err(_) => {
return std::boxed::Box::pin(async {
return std::result::Result::Err(ksp_core_lib::Error::new(crate::ERROR_CODE_BACKFILL_DISCOVERY_INVALID, "test page queue lock poisoned"));
});
},
};
return std::boxed::Box::pin(async move {
return std::result::Result::Ok(page);
});
}
}
fn page_entry(character: char, slot: u64) -> super::SignaturePageEntry {
return super::SignaturePageEntry { signature: character.to_string().repeat(crate::MIN_BACKFILL_SIGNATURE_TEXT_BYTES), slot };
}
fn signature(character: char) -> std::option::Option<crate::BackfillSignature> {
return match crate::BackfillSignature::new(character.to_string().repeat(crate::MIN_BACKFILL_SIGNATURE_TEXT_BYTES)) {
std::result::Result::Ok(value) => std::option::Option::Some(value),
std::result::Result::Err(_) => std::option::Option::None,
};
}
fn request(scope: crate::BackfillScope, page_size: usize, max_pages: usize, max_candidates: usize) -> std::option::Option<crate::BackfillRequest> {
let job_id = match ksp_job_api::JobId::new("backfill:discovery-test") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return std::option::Option::None,
};
let network = match ksp_store_lib::RawNetworkId::new("devnet") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return std::option::Option::None,
};
let result = crate::BackfillRequest::new(
job_id,
network,
ksp_onchain_transport_lib::HttpRoleName::new("history"),
crate::BackfillCommitment::Confirmed,
scope,
page_size,
max_pages,
max_candidates,
1,
std::option::Option::Some(42),
);
return match result {
std::result::Result::Ok(value) => std::option::Option::Some(value),
std::result::Result::Err(_) => std::option::Option::None,
};
}
fn candidate_signatures(discovery: &crate::BackfillDiscovery) -> std::vec::Vec<std::string::String> {
let mut signatures = std::vec::Vec::with_capacity(discovery.candidates().len());
for candidate in discovery.candidates() {
signatures.push(candidate.identity().signature().as_str().to_owned());
}
return signatures;
}
#[tokio::test]
async fn pre_005_latest_paginates_newest_first_and_deduplicates_pages_stably() {
let scope = crate::BackfillScope::latest_address(ksp_core_lib::Pubkey::new_from_array([1_u8; 32]));
let request = match request(scope, 3, 4, 10) {
std::option::Option::Some(value) => value,
std::option::Option::None => return,
};
let source = FakeSource::new(std::vec![
std::vec![page_entry('6', 60), page_entry('5', 50), page_entry('5', 50)],
std::vec![page_entry('4', 40), page_entry('3', 30)],
]);
let result = super::discover_with_source(&source, &request, std::option::Option::None).await;
assert!(result.is_ok());
let discovery = match result {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return,
};
assert_eq!(candidate_signatures(&discovery), std::vec!["6".repeat(64), "5".repeat(64), "4".repeat(64), "3".repeat(64)]);
assert_eq!(discovery.pages_fetched(), 2);
assert_eq!(discovery.boundary(), crate::BackfillDiscoveryBoundary::RpcBoundary);
assert!(!discovery.is_partial());
let calls = source.calls();
assert_eq!(calls.len(), 2);
assert_eq!(calls[0].before, std::option::Option::None);
let expected_cursor = "5".repeat(64);
assert_eq!(calls[1].before.as_deref(), std::option::Option::Some(expected_cursor.as_str()));
assert_eq!(calls[0].until, std::option::Option::None);
assert_eq!(calls[0].limit, std::option::Option::Some(3));
assert_eq!(calls[0].commitment, std::option::Option::Some(ksp_onchain_transport_lib::SolanaCommitment::Confirmed));
assert_eq!(calls[0].min_context_slot, std::option::Option::Some(42));
return;
}
#[tokio::test]
async fn pre_005_before_uses_exclusive_anchor_then_advances_rpc_cursor() {
let anchor = match signature('7') {
std::option::Option::Some(value) => value,
std::option::Option::None => return,
};
let scope = crate::BackfillScope::before_address(ksp_core_lib::Pubkey::new_from_array([2_u8; 32]), anchor.clone());
let request = match request(scope, 2, 3, 5) {
std::option::Option::Some(value) => value,
std::option::Option::None => return,
};
let source = FakeSource::new(std::vec![std::vec![page_entry('6', 60), page_entry('5', 50)], std::vec![page_entry('4', 40)]]);
let result = super::discover_with_source(&source, &request, std::option::Option::None).await;
let discovery = match result {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return,
};
assert_eq!(candidate_signatures(&discovery), std::vec!["6".repeat(64), "5".repeat(64), "4".repeat(64)]);
let calls = source.calls();
assert_eq!(calls.len(), 2);
assert_eq!(calls[0].before.as_deref(), std::option::Option::Some(anchor.as_str()));
let expected_cursor = "5".repeat(64);
assert_eq!(calls[1].before.as_deref(), std::option::Option::Some(expected_cursor.as_str()));
assert_eq!(calls[0].until, std::option::Option::None);
return;
}
#[tokio::test]
async fn pre_005_after_keeps_only_nearest_newer_window_and_preserves_rpc_order() {
let anchor = match signature('1') {
std::option::Option::Some(value) => value,
std::option::Option::None => return,
};
let scope = crate::BackfillScope::after_address(ksp_core_lib::Pubkey::new_from_array([3_u8; 32]), anchor.clone());
let request = match request(scope, 3, 3, 3) {
std::option::Option::Some(value) => value,
std::option::Option::None => return,
};
let source = FakeSource::new(std::vec![
std::vec![page_entry('7', 70), page_entry('6', 60), page_entry('5', 50)],
std::vec![page_entry('4', 40), page_entry('3', 30)],
]);
let result = super::discover_with_source(&source, &request, std::option::Option::None).await;
let discovery = match result {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return,
};
assert_eq!(candidate_signatures(&discovery), std::vec!["5".repeat(64), "4".repeat(64), "3".repeat(64)]);
assert_eq!(discovery.boundary(), crate::BackfillDiscoveryBoundary::RpcBoundary);
assert!(!discovery.is_partial());
let calls = source.calls();
assert_eq!(calls.len(), 2);
assert_eq!(calls[0].until.as_deref(), std::option::Option::Some(anchor.as_str()));
assert_eq!(calls[1].until.as_deref(), std::option::Option::Some(anchor.as_str()));
let expected_cursor = "5".repeat(64);
assert_eq!(calls[1].before.as_deref(), std::option::Option::Some(expected_cursor.as_str()));
return;
}
#[tokio::test]
async fn pre_005_after_page_bound_is_partial_and_does_not_claim_anchor_completion() {
let anchor = match signature('1') {
std::option::Option::Some(value) => value,
std::option::Option::None => return,
};
let scope = crate::BackfillScope::after_address(ksp_core_lib::Pubkey::new_from_array([4_u8; 32]), anchor);
let request = match request(scope, 2, 2, 3) {
std::option::Option::Some(value) => value,
std::option::Option::None => return,
};
let source = FakeSource::new(std::vec![std::vec![page_entry('7', 70), page_entry('6', 60)], std::vec![page_entry('5', 50), page_entry('4', 40)],]);
let result = super::discover_with_source(&source, &request, std::option::Option::None).await;
let discovery = match result {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return,
};
assert_eq!(candidate_signatures(&discovery), std::vec!["6".repeat(64), "5".repeat(64), "4".repeat(64)]);
assert_eq!(discovery.boundary(), crate::BackfillDiscoveryBoundary::AfterAnchorNotReached);
assert!(discovery.is_partial());
assert_eq!(discovery.pages_fetched(), 2);
return;
}
#[tokio::test]
async fn pre_005_latest_page_bound_is_partial_when_full_pages_leave_more_history_possible() {
let scope = crate::BackfillScope::latest_address(ksp_core_lib::Pubkey::new_from_array([5_u8; 32]));
let request = match request(scope, 2, 1, 5) {
std::option::Option::Some(value) => value,
std::option::Option::None => return,
};
let source = FakeSource::new(std::vec![std::vec![page_entry('7', 70), page_entry('6', 60)]]);
let result = super::discover_with_source(&source, &request, std::option::Option::None).await;
let discovery = match result {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return,
};
assert_eq!(discovery.boundary(), crate::BackfillDiscoveryBoundary::PageLimit);
assert!(discovery.is_partial());
return;
}
#[tokio::test]
async fn pre_005_explicit_scope_never_calls_transport_and_preserves_network_scoped_identity() {
let first = match signature('2') {
std::option::Option::Some(value) => value,
std::option::Option::None => return,
};
let second = match signature('3') {
std::option::Option::Some(value) => value,
std::option::Option::None => return,
};
let scope_result = crate::BackfillScope::explicit_signatures(std::vec![first.clone(), second.clone(), first]);
let scope = match scope_result {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return,
};
let job_id = match ksp_job_api::JobId::new("backfill:explicit") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return,
};
let network = match ksp_store_lib::RawNetworkId::new("synthetic") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return,
};
let request_result = crate::BackfillRequest::new(
job_id,
network.clone(),
ksp_onchain_transport_lib::HttpRoleName::new("unused"),
crate::BackfillCommitment::Finalized,
scope,
100,
10,
10,
1,
std::option::Option::None,
);
let request = match request_result {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return,
};
let source = FakeSource::new(std::vec::Vec::new());
let result = super::discover_with_source(&source, &request, std::option::Option::None).await;
let discovery = match result {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return,
};
assert_eq!(discovery.boundary(), crate::BackfillDiscoveryBoundary::ExplicitInput);
assert_eq!(discovery.pages_fetched(), 0);
assert_eq!(discovery.candidates().len(), 2);
assert_eq!(discovery.candidates()[0].identity().network(), &network);
assert!(source.calls().is_empty());
return;
}
#[tokio::test]
async fn pre_008_before_resume_uses_checkpoint_cursor_instead_of_original_anchor() {
let anchor = match signature('8') {
std::option::Option::Some(value) => value,
std::option::Option::None => return,
};
let resume = match signature('5') {
std::option::Option::Some(value) => value,
std::option::Option::None => return,
};
let scope = crate::BackfillScope::before_address(ksp_core_lib::Pubkey::new_from_array([8_u8; 32]), anchor);
let request = match request(scope, 2, 2, 4) {
std::option::Option::Some(value) => value,
std::option::Option::None => return,
};
let checkpoint = crate::BackfillCheckpoint::new(request.job_id().clone(), request.scope_fingerprint(), 2, std::option::Option::Some(resume.clone()));
let request = match request.with_checkpoint(checkpoint) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return,
};
let source = FakeSource::new(std::vec![std::vec![page_entry('4', 40)]]);
let result = super::discover_with_source(&source, &request, std::option::Option::None).await;
assert!(result.is_ok());
let calls = source.calls();
assert_eq!(calls.len(), 1);
assert_eq!(calls[0].before.as_deref(), std::option::Option::Some(resume.as_str()));
return;
}
#[tokio::test]
async fn pre_008_latest_resume_restarts_from_current_latest_without_rpc_cursor() {
let scope = crate::BackfillScope::latest_address(ksp_core_lib::Pubkey::new_from_array([9_u8; 32]));
let request = match request(scope, 2, 2, 4) {
std::option::Option::Some(value) => value,
std::option::Option::None => return,
};
let checkpoint = crate::BackfillCheckpoint::new(request.job_id().clone(), request.scope_fingerprint(), 2, std::option::Option::None);
let request = match request.with_checkpoint(checkpoint) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return,
};
let source = FakeSource::new(std::vec![std::vec![page_entry('7', 70)]]);
let result = super::discover_with_source(&source, &request, std::option::Option::None).await;
assert!(result.is_ok());
let calls = source.calls();
assert_eq!(calls.len(), 1);
assert_eq!(calls[0].before, std::option::Option::None);
return;
}
#[tokio::test]
async fn pre_009_discovery_rpc_wait_is_cancelled_cooperatively() {
let scope = crate::BackfillScope::latest_address(ksp_core_lib::Pubkey::new_from_array([9_u8; 32]));
let request = match request(scope, 2, 2, 5) {
std::option::Option::Some(value) => value,
std::option::Option::None => return,
};
let source = PendingSource::new();
let token = ksp_job_api::JobCancellationToken::new();
let (cancel_sender, cancel_receiver) = tokio::sync::watch::channel(false);
let cancellation = crate::BackfillCancellationSignal::new(token.clone(), cancel_receiver);
let discovery = super::discover_with_source(&source, &request, std::option::Option::Some(&cancellation));
let cancel = async {
tokio::task::yield_now().await;
assert!(token.cancel());
assert!(cancel_sender.send(true).is_ok());
};
let (result, ()) = tokio::join!(discovery, cancel);
let error = match result {
std::result::Result::Ok(_) => return,
std::result::Result::Err(error) => error,
};
assert_eq!(error.code(), crate::ERROR_CODE_BACKFILL_CANCELLED);
assert_eq!(source.calls(), 1);
return;
}

View File

@@ -0,0 +1,321 @@
// file: crates/ksp-job-backfill-lib/unit_tests/execution.rs
// version: 3
#[derive(Clone, Copy)]
enum FakeDisposition {
Durable,
Missing,
Conflict,
Failure,
}
#[derive(Clone, Copy)]
struct FakePlan {
pending_polls: usize,
disposition: FakeDisposition,
}
struct FakeProcessor {
plans: std::vec::Vec<FakePlan>,
active: std::sync::atomic::AtomicUsize,
maximum: std::sync::atomic::AtomicUsize,
calls: std::sync::atomic::AtomicUsize,
}
impl FakeProcessor {
fn new(plans: std::vec::Vec<FakePlan>) -> Self {
return Self {
plans,
active: std::sync::atomic::AtomicUsize::new(0),
maximum: std::sync::atomic::AtomicUsize::new(0),
calls: std::sync::atomic::AtomicUsize::new(0),
};
}
fn maximum(&self) -> usize {
return self.maximum.load(std::sync::atomic::Ordering::SeqCst);
}
fn calls(&self) -> usize {
return self.calls.load(std::sync::atomic::Ordering::SeqCst);
}
}
impl super::CandidateProcessor for FakeProcessor {
fn process<'a>(&'a self, candidate: &'a crate::BackfillCandidate) -> super::CandidateProcessFuture<'a> {
self.calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
let first_byte = candidate.identity().signature().as_str().as_bytes()[0];
let plan_index = usize::from(first_byte.saturating_sub(b'1'));
let plan = match self.plans.get(plan_index) {
std::option::Option::Some(plan) => *plan,
std::option::Option::None => FakePlan { pending_polls: 0, disposition: FakeDisposition::Failure },
};
return std::boxed::Box::pin(async move {
let mut pending_polls = plan.pending_polls;
let mut started = false;
std::future::poll_fn(|context| {
if !started {
started = true;
let active = self.active.fetch_add(1, std::sync::atomic::Ordering::SeqCst) + 1;
self.maximum.fetch_max(active, std::sync::atomic::Ordering::SeqCst);
}
if pending_polls != 0 {
pending_polls -= 1;
context.waker().wake_by_ref();
return std::task::Poll::Pending;
}
self.active.fetch_sub(1, std::sync::atomic::Ordering::SeqCst);
return std::task::Poll::Ready(());
})
.await;
return planned_outcome(candidate, plan.disposition);
});
}
}
fn planned_outcome(candidate: &crate::BackfillCandidate, disposition: FakeDisposition) -> ksp_core_lib::Result<crate::BackfillPersistenceOutcome> {
if matches!(disposition, FakeDisposition::Failure) {
return std::result::Result::Err(ksp_core_lib::Error::new(crate::ERROR_CODE_BACKFILL_RAW_CONVERSION_INVALID, "planned candidate failure"));
}
let fill = candidate.identity().signature().as_str().as_bytes()[0];
let reference =
ksp_store_lib::RawTransactionReference::new(candidate.identity().network().clone(), ksp_store_lib::RawTransactionSignature::new([fill; 64]));
let (entity, observation) = match disposition {
FakeDisposition::Durable => (crate::BackfillEntityPersistence::AlreadyPresent, crate::BackfillObservationPersistence::AlreadyPresent),
FakeDisposition::Missing => (crate::BackfillEntityPersistence::Missing, crate::BackfillObservationPersistence::NotApplicable),
FakeDisposition::Conflict => (crate::BackfillEntityPersistence::Conflict, crate::BackfillObservationPersistence::NotRecorded),
FakeDisposition::Failure => return std::result::Result::Err(super::execution_error("test.disposition")),
};
return std::result::Result::Ok(crate::BackfillPersistenceOutcome::new(reference, entity, observation));
}
fn signature(character: char) -> std::option::Option<crate::BackfillSignature> {
return crate::BackfillSignature::new(character.to_string().repeat(crate::MIN_BACKFILL_SIGNATURE_TEXT_BYTES)).ok();
}
fn explicit_request(characters: &[char], concurrency: usize) -> std::option::Option<crate::BackfillRequest> {
let mut signatures = std::vec::Vec::with_capacity(characters.len());
for character in characters {
let value = match signature(*character) {
std::option::Option::Some(value) => value,
std::option::Option::None => return std::option::Option::None,
};
signatures.push(value);
}
let scope = match crate::BackfillScope::explicit_signatures(signatures) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return std::option::Option::None,
};
let job_id = match ksp_job_api::JobId::new("backfill:execution-test") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return std::option::Option::None,
};
let network = match ksp_store_lib::RawNetworkId::new("devnet") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return std::option::Option::None,
};
return crate::BackfillRequest::new(
job_id,
network,
ksp_onchain_transport_lib::HttpRoleName::new("history"),
crate::BackfillCommitment::Confirmed,
scope,
100,
10,
characters.len(),
concurrency,
std::option::Option::None,
)
.ok();
}
fn discovery(request: &crate::BackfillRequest, characters: &[char]) -> std::option::Option<crate::BackfillDiscovery> {
let mut candidates = std::vec::Vec::with_capacity(characters.len());
for character in characters {
let signature = match signature(*character) {
std::option::Option::Some(value) => value,
std::option::Option::None => return std::option::Option::None,
};
let identity = crate::BackfillCandidateIdentity::new(request.network().clone(), signature);
candidates.push(crate::BackfillCandidate::new(identity, std::option::Option::None));
}
return std::option::Option::Some(crate::BackfillDiscovery::new(
request.network().clone(),
request.scope_fingerprint(),
candidates,
0,
crate::BackfillDiscoveryBoundary::ExplicitInput,
));
}
#[tokio::test]
async fn pre_008_execution_is_bounded_and_reconciles_out_of_order_durable_completions() {
let request = match explicit_request(&['1', '2', '3', '4'], 2) {
std::option::Option::Some(value) => value,
std::option::Option::None => return,
};
let discovery = match discovery(&request, &['1', '2', '3', '4']) {
std::option::Option::Some(value) => value,
std::option::Option::None => return,
};
let processor = FakeProcessor::new(std::vec![
FakePlan { pending_polls: 6, disposition: FakeDisposition::Durable },
FakePlan { pending_polls: 0, disposition: FakeDisposition::Durable },
FakePlan { pending_polls: 2, disposition: FakeDisposition::Durable },
FakePlan { pending_polls: 0, disposition: FakeDisposition::Durable },
]);
let result = super::execute_with_processor(&processor, &request, &discovery, std::option::Option::None, std::option::Option::None).await;
let batch = match result {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return,
};
assert_eq!(batch.admitted_count(), 4);
assert_eq!(batch.finished_count(), 4);
assert_eq!(batch.durable_count(), 4);
assert_eq!(batch.hole_count(), 0);
assert_eq!(batch.maximum_in_flight(), 2);
assert_eq!(processor.maximum(), 2);
assert_eq!(batch.local_contiguous_completed(), 4);
assert_eq!(batch.checkpoint().completed_prefix(), 4);
assert!(!batch.is_partial());
return;
}
#[tokio::test]
async fn pre_008_missing_is_non_fatal_but_blocks_frontier_while_later_candidates_continue() {
let request = match explicit_request(&['1', '2', '3'], 2) {
std::option::Option::Some(value) => value,
std::option::Option::None => return,
};
let discovery = match discovery(&request, &['1', '2', '3']) {
std::option::Option::Some(value) => value,
std::option::Option::None => return,
};
let processor = FakeProcessor::new(std::vec![
FakePlan { pending_polls: 1, disposition: FakeDisposition::Durable },
FakePlan { pending_polls: 0, disposition: FakeDisposition::Missing },
FakePlan { pending_polls: 0, disposition: FakeDisposition::Durable },
]);
let batch = match super::execute_with_processor(&processor, &request, &discovery, std::option::Option::None, std::option::Option::None).await {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return,
};
assert_eq!(batch.admitted_count(), 3);
assert_eq!(batch.finished_count(), 3);
assert_eq!(batch.durable_count(), 2);
assert_eq!(batch.hole_count(), 1);
assert_eq!(batch.local_contiguous_completed(), 1);
assert_eq!(batch.checkpoint().completed_prefix(), 1);
assert_eq!(batch.failure_code(), std::option::Option::None);
assert!(batch.is_partial());
return;
}
#[tokio::test]
async fn pre_008_conflict_stops_new_admissions_and_drains_already_in_flight_work() {
let request = match explicit_request(&['1', '2', '3', '4'], 2) {
std::option::Option::Some(value) => value,
std::option::Option::None => return,
};
let discovery = match discovery(&request, &['1', '2', '3', '4']) {
std::option::Option::Some(value) => value,
std::option::Option::None => return,
};
let processor = FakeProcessor::new(std::vec![
FakePlan { pending_polls: 6, disposition: FakeDisposition::Durable },
FakePlan { pending_polls: 0, disposition: FakeDisposition::Conflict },
FakePlan { pending_polls: 0, disposition: FakeDisposition::Durable },
FakePlan { pending_polls: 0, disposition: FakeDisposition::Durable },
]);
let batch = match super::execute_with_processor(&processor, &request, &discovery, std::option::Option::None, std::option::Option::None).await {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return,
};
assert_eq!(processor.calls(), 2);
assert_eq!(batch.admitted_count(), 2);
assert_eq!(batch.finished_count(), 2);
assert_eq!(batch.durable_count(), 1);
assert_eq!(batch.hole_count(), 1);
assert_eq!(batch.local_contiguous_completed(), 1);
assert_eq!(batch.failure_code(), std::option::Option::Some(ksp_store_lib::ERROR_CODE_RAW_CONFLICT));
assert!(batch.is_partial());
return;
}
#[tokio::test]
async fn pre_008_explicit_resume_skips_only_the_checkpointed_contiguous_prefix() {
let request = match explicit_request(&['1', '2', '3', '4'], 2) {
std::option::Option::Some(value) => value,
std::option::Option::None => return,
};
let checkpoint = crate::BackfillCheckpoint::new(request.job_id().clone(), request.scope_fingerprint(), 2, std::option::Option::None);
let request = match request.with_checkpoint(checkpoint) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return,
};
let discovery = match discovery(&request, &['1', '2', '3', '4']) {
std::option::Option::Some(value) => value,
std::option::Option::None => return,
};
let processor = FakeProcessor::new(std::vec![
FakePlan { pending_polls: 0, disposition: FakeDisposition::Failure },
FakePlan { pending_polls: 0, disposition: FakeDisposition::Failure },
FakePlan { pending_polls: 0, disposition: FakeDisposition::Durable },
FakePlan { pending_polls: 0, disposition: FakeDisposition::Durable },
]);
let batch = match super::execute_with_processor(&processor, &request, &discovery, std::option::Option::None, std::option::Option::None).await {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return,
};
assert_eq!(processor.calls(), 2);
assert_eq!(batch.admitted_count(), 2);
assert_eq!(batch.local_contiguous_completed(), 4);
assert_eq!(batch.checkpoint().completed_prefix(), 4);
assert_eq!(batch.failure_code(), std::option::Option::None);
return;
}
#[tokio::test]
async fn pre_009_cancellation_stops_admission_and_drains_already_admitted_candidate_work() {
let request = match explicit_request(&['1', '2', '3', '4'], 2) {
std::option::Option::Some(value) => value,
std::option::Option::None => return,
};
let discovery = match discovery(&request, &['1', '2', '3', '4']) {
std::option::Option::Some(value) => value,
std::option::Option::None => return,
};
let processor = FakeProcessor::new(std::vec![
FakePlan { pending_polls: 12, disposition: FakeDisposition::Durable },
FakePlan { pending_polls: 12, disposition: FakeDisposition::Durable },
FakePlan { pending_polls: 0, disposition: FakeDisposition::Durable },
FakePlan { pending_polls: 0, disposition: FakeDisposition::Durable },
]);
let token = ksp_job_api::JobCancellationToken::new();
let (cancel_sender, cancel_receiver) = tokio::sync::watch::channel(false);
let signal = crate::BackfillCancellationSignal::new(token.clone(), cancel_receiver);
let execution = super::execute_with_processor(&processor, &request, &discovery, std::option::Option::Some(&signal), std::option::Option::None);
let cancellation = async {
loop {
if processor.calls() >= 2 {
break;
}
tokio::task::yield_now().await;
}
assert!(token.cancel());
assert!(cancel_sender.send(true).is_ok());
};
let (result, ()) = tokio::join!(execution, cancellation);
let batch = match result {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return,
};
assert_eq!(processor.calls(), 2);
assert_eq!(batch.admitted_count(), 2);
assert_eq!(batch.finished_count(), 2);
assert_eq!(batch.durable_count(), 2);
assert_eq!(batch.cancelled_count(), 0);
assert!(batch.was_cancelled());
assert_eq!(batch.checkpoint().completed_prefix(), 2);
return;
}

View File

@@ -0,0 +1,348 @@
// file: crates/ksp-job-backfill-lib/unit_tests/persistence.rs
// version: 1
#[derive(Clone, Copy)]
enum FakeResponse {
Outcome(ksp_store_lib::RawAcquisitionWriteOutcome),
Conflict,
Failure,
}
struct FakePersistencePort {
network: ksp_store_lib::RawNetworkId,
responses: std::sync::Mutex<std::collections::VecDeque<FakeResponse>>,
calls: std::sync::atomic::AtomicUsize,
normal_mode_only: std::sync::atomic::AtomicBool,
}
impl FakePersistencePort {
fn new(network: ksp_store_lib::RawNetworkId, responses: &[FakeResponse]) -> Self {
return Self {
network,
responses: std::sync::Mutex::new(responses.iter().copied().collect()),
calls: std::sync::atomic::AtomicUsize::new(0),
normal_mode_only: std::sync::atomic::AtomicBool::new(true),
};
}
fn calls(&self) -> usize {
return self.calls.load(std::sync::atomic::Ordering::SeqCst);
}
fn used_only_normal_mode(&self) -> bool {
return self.normal_mode_only.load(std::sync::atomic::Ordering::SeqCst);
}
}
impl super::RawTransactionPersistencePort for FakePersistencePort {
fn network_matches(&self, network: &ksp_store_lib::RawNetworkId) -> bool {
return &self.network == network;
}
fn persist_acquisition<'a>(
&'a self,
transaction: ksp_store_lib::RawTransaction,
observation: ksp_store_lib::RawTransactionObservation,
mode: ksp_store_lib::RawTransactionAcquisitionMode,
) -> ksp_store_lib::StoreApiFuture<'a, ksp_store_lib::Result<ksp_store_lib::RawAcquisitionWriteOutcome>> {
self.calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
if mode != ksp_store_lib::RawTransactionAcquisitionMode::Normal {
self.normal_mode_only.store(false, std::sync::atomic::Ordering::SeqCst);
}
let result = if transaction.reference() != observation.transaction() {
std::result::Result::Err(ksp_core_lib::Error::new(
ksp_core_lib::ErrorCode::new("test", "reference_mismatch"),
"fake persistence reference mismatch",
))
} else {
let response = match self.responses.lock() {
std::result::Result::Ok(mut responses) => responses.pop_front(),
std::result::Result::Err(_) => std::option::Option::None,
};
match response {
std::option::Option::Some(FakeResponse::Outcome(outcome)) => std::result::Result::Ok(outcome),
std::option::Option::Some(FakeResponse::Conflict) => {
std::result::Result::Err(ksp_core_lib::Error::new(ksp_store_lib::ERROR_CODE_RAW_CONFLICT, "fake canonical content conflict"))
},
std::option::Option::Some(FakeResponse::Failure) | std::option::Option::None => {
std::result::Result::Err(ksp_core_lib::Error::new(ksp_core_lib::ErrorCode::new("test", "store_failure"), "fake Store failure"))
},
}
};
return std::boxed::Box::pin(async move {
return result;
});
}
}
fn raw_network(value: &str) -> std::option::Option<ksp_store_lib::RawNetworkId> {
return match ksp_store_lib::RawNetworkId::new(value) {
std::result::Result::Ok(network) => std::option::Option::Some(network),
std::result::Result::Err(_) => std::option::Option::None,
};
}
fn raw_reference(network: &str, signature_byte: u8) -> std::option::Option<ksp_store_lib::RawTransactionReference> {
let network = match raw_network(network) {
std::option::Option::Some(network) => network,
std::option::Option::None => return std::option::Option::None,
};
return std::option::Option::Some(ksp_store_lib::RawTransactionReference::new(network, ksp_store_lib::RawTransactionSignature::new([signature_byte; 64])));
}
fn raw_acquisition_parts(
network: &str,
signature_byte: u8,
observation_byte: u8,
) -> std::option::Option<(ksp_store_lib::RawTransactionReference, ksp_store_lib::RawTransaction, ksp_store_lib::RawTransactionObservation)> {
let reference = match raw_reference(network, signature_byte) {
std::option::Option::Some(reference) => reference,
std::option::Option::None => return std::option::Option::None,
};
let format_id = match ksp_store_lib::RawFormatId::new(crate::RAW_TRANSACTION_FORMAT_ID) {
std::result::Result::Ok(format_id) => format_id,
std::result::Result::Err(_) => return std::option::Option::None,
};
let payload = ksp_store_lib::RawPayload::try_new(
format_id,
crate::RAW_TRANSACTION_FORMAT_VERSION,
std::vec![signature_byte].into_boxed_slice(),
ksp_store_lib::RawContentHash::new([signature_byte; 32]),
);
let payload = match payload {
std::result::Result::Ok(payload) => payload,
std::result::Result::Err(_) => return std::option::Option::None,
};
let received_at = match ksp_store_lib::RawTimestamp::from_unix_millis(1_700_000_000_000) {
std::result::Result::Ok(received_at) => received_at,
std::result::Result::Err(_) => return std::option::Option::None,
};
let provider = match ksp_store_lib::RawProvenanceCode::new("provider") {
std::result::Result::Ok(provider) => provider,
std::result::Result::Err(_) => return std::option::Option::None,
};
let protocol = match ksp_store_lib::RawProvenanceCode::new("solana.http.json_rpc") {
std::result::Result::Ok(protocol) => protocol,
std::result::Result::Err(_) => return std::option::Option::None,
};
let method = match ksp_store_lib::RawProvenanceCode::new("getTransaction") {
std::result::Result::Ok(method) => method,
std::result::Result::Err(_) => return std::option::Option::None,
};
let provenance = ksp_store_lib::RawAcquisitionProvenance::new(provider, protocol, method, ksp_store_lib::RawAcquisitionOrigin::Backfill, received_at);
let transaction = ksp_store_lib::RawTransaction::new(reference.clone(), 42, std::option::Option::None, payload);
let observation =
ksp_store_lib::RawTransactionObservation::new(ksp_store_lib::RawObservationKey::new([observation_byte; 32]), reference.clone(), provenance);
return std::option::Option::Some((reference, transaction, observation));
}
fn store_outcome(
entity: ksp_store_lib::RawEntityWriteOutcome,
observation: ksp_store_lib::RawObservationWriteOutcome,
) -> ksp_store_lib::RawAcquisitionWriteOutcome {
return ksp_store_lib::RawAcquisitionWriteOutcome::new(entity, observation);
}
#[tokio::test]
async fn pre_007_missing_skips_store_and_preserves_network_scoped_identity() {
let reference = match raw_reference("devnet", 1) {
std::option::Option::Some(reference) => reference,
std::option::Option::None => return,
};
let network = match raw_network("devnet") {
std::option::Option::Some(network) => network,
std::option::Option::None => return,
};
let port = FakePersistencePort::new(network, &[]);
let result = super::persist_hydration_with_port(&port, crate::BackfillHydrationOutcome::Missing(reference.clone())).await;
assert!(result.is_ok());
if let std::result::Result::Ok(result) = result {
assert_eq!(result.reference(), &reference);
assert_eq!(result.entity(), crate::BackfillEntityPersistence::Missing);
assert_eq!(result.observation(), crate::BackfillObservationPersistence::NotApplicable);
}
assert_eq!(port.calls(), 0);
return;
}
#[tokio::test]
async fn pre_007_store_network_mismatch_is_rejected_before_any_write() {
let reference = match raw_reference("devnet", 2) {
std::option::Option::Some(reference) => reference,
std::option::Option::None => return,
};
let network = match raw_network("mainnet") {
std::option::Option::Some(network) => network,
std::option::Option::None => return,
};
let port = FakePersistencePort::new(network, &[]);
let result = super::persist_hydration_with_port(&port, crate::BackfillHydrationOutcome::Missing(reference)).await;
assert!(result.is_err());
if let std::result::Result::Err(error) = result {
assert_eq!(error.code(), crate::ERROR_CODE_BACKFILL_PERSISTENCE_INVALID);
}
assert_eq!(port.calls(), 0);
return;
}
#[tokio::test]
async fn pre_007_atomic_insert_maps_entity_and_observation_without_second_write() {
let (reference, transaction, observation) = match raw_acquisition_parts("devnet", 3, 13) {
std::option::Option::Some(parts) => parts,
std::option::Option::None => return,
};
let network = match raw_network("devnet") {
std::option::Option::Some(network) => network,
std::option::Option::None => return,
};
let response = store_outcome(ksp_store_lib::RawEntityWriteOutcome::Inserted, ksp_store_lib::RawObservationWriteOutcome::Inserted);
let port = FakePersistencePort::new(network, &[FakeResponse::Outcome(response)]);
let result = super::persist_available_with_port(&port, reference.clone(), transaction, observation).await;
assert!(result.is_ok());
if let std::result::Result::Ok(result) = result {
assert_eq!(result.reference(), &reference);
assert_eq!(result.entity(), crate::BackfillEntityPersistence::Inserted);
assert_eq!(result.observation(), crate::BackfillObservationPersistence::Inserted);
}
assert_eq!(port.calls(), 1);
assert!(port.used_only_normal_mode());
return;
}
#[tokio::test]
async fn pre_007_existing_entity_distinguishes_new_from_idempotent_observation() {
let network = match raw_network("devnet") {
std::option::Option::Some(network) => network,
std::option::Option::None => return,
};
let first = store_outcome(ksp_store_lib::RawEntityWriteOutcome::AlreadyPresent, ksp_store_lib::RawObservationWriteOutcome::Inserted);
let second = store_outcome(ksp_store_lib::RawEntityWriteOutcome::AlreadyPresent, ksp_store_lib::RawObservationWriteOutcome::AlreadyPresent);
let port = FakePersistencePort::new(network, &[FakeResponse::Outcome(first), FakeResponse::Outcome(second)]);
let (reference, transaction, observation) = match raw_acquisition_parts("devnet", 4, 14) {
std::option::Option::Some(parts) => parts,
std::option::Option::None => return,
};
let result = super::persist_available_with_port(&port, reference, transaction, observation).await;
assert!(result.is_ok());
if let std::result::Result::Ok(result) = result {
assert_eq!(result.entity(), crate::BackfillEntityPersistence::AlreadyPresent);
assert_eq!(result.observation(), crate::BackfillObservationPersistence::Inserted);
}
let (reference, transaction, observation) = match raw_acquisition_parts("devnet", 4, 14) {
std::option::Option::Some(parts) => parts,
std::option::Option::None => return,
};
let rerun = super::persist_available_with_port(&port, reference, transaction, observation).await;
assert!(rerun.is_ok());
if let std::result::Result::Ok(rerun) = rerun {
assert_eq!(rerun.entity(), crate::BackfillEntityPersistence::AlreadyPresent);
assert_eq!(rerun.observation(), crate::BackfillObservationPersistence::AlreadyPresent);
}
assert_eq!(port.calls(), 2);
assert!(port.used_only_normal_mode());
return;
}
#[tokio::test]
async fn pre_007_normal_backfill_respects_purged_tombstone_without_observation() {
let (reference, transaction, observation) = match raw_acquisition_parts("devnet", 5, 15) {
std::option::Option::Some(parts) => parts,
std::option::Option::None => return,
};
let network = match raw_network("devnet") {
std::option::Option::Some(network) => network,
std::option::Option::None => return,
};
let response = store_outcome(ksp_store_lib::RawEntityWriteOutcome::SkippedPurged, ksp_store_lib::RawObservationWriteOutcome::NotRecorded);
let port = FakePersistencePort::new(network, &[FakeResponse::Outcome(response)]);
let result = super::persist_available_with_port(&port, reference, transaction, observation).await;
assert!(result.is_ok());
if let std::result::Result::Ok(result) = result {
assert_eq!(result.entity(), crate::BackfillEntityPersistence::SkippedPurged);
assert_eq!(result.observation(), crate::BackfillObservationPersistence::NotRecorded);
}
assert!(port.used_only_normal_mode());
return;
}
#[tokio::test]
async fn pre_007_store_content_conflict_is_explicit_and_not_idempotent_success() {
let (reference, transaction, observation) = match raw_acquisition_parts("devnet", 6, 16) {
std::option::Option::Some(parts) => parts,
std::option::Option::None => return,
};
let network = match raw_network("devnet") {
std::option::Option::Some(network) => network,
std::option::Option::None => return,
};
let port = FakePersistencePort::new(network, &[FakeResponse::Conflict]);
let result = super::persist_available_with_port(&port, reference, transaction, observation).await;
assert!(result.is_ok());
if let std::result::Result::Ok(result) = result {
assert_eq!(result.entity(), crate::BackfillEntityPersistence::Conflict);
assert_eq!(result.observation(), crate::BackfillObservationPersistence::NotRecorded);
assert_ne!(result.entity(), crate::BackfillEntityPersistence::AlreadyPresent);
}
return;
}
#[tokio::test]
async fn pre_007_non_conflict_store_failure_propagates_unchanged() {
let (reference, transaction, observation) = match raw_acquisition_parts("devnet", 7, 17) {
std::option::Option::Some(parts) => parts,
std::option::Option::None => return,
};
let network = match raw_network("devnet") {
std::option::Option::Some(network) => network,
std::option::Option::None => return,
};
let port = FakePersistencePort::new(network, &[FakeResponse::Failure]);
let result = super::persist_available_with_port(&port, reference, transaction, observation).await;
assert!(result.is_err());
if let std::result::Result::Err(error) = result {
assert_eq!(error.code(), ksp_core_lib::ErrorCode::new("test", "store_failure"));
}
return;
}
#[test]
fn pre_007_normal_mode_rejects_impossible_store_outcome_combinations() {
let reference = match raw_reference("devnet", 8) {
std::option::Option::Some(reference) => reference,
std::option::Option::None => return,
};
let impossible = [
store_outcome(ksp_store_lib::RawEntityWriteOutcome::Rehydrated, ksp_store_lib::RawObservationWriteOutcome::Inserted),
store_outcome(ksp_store_lib::RawEntityWriteOutcome::Inserted, ksp_store_lib::RawObservationWriteOutcome::AlreadyPresent),
store_outcome(ksp_store_lib::RawEntityWriteOutcome::SkippedPurged, ksp_store_lib::RawObservationWriteOutcome::Inserted),
];
for outcome in impossible {
let result = super::map_store_outcome(reference.clone(), outcome);
assert!(result.is_err());
if let std::result::Result::Err(error) = result {
assert_eq!(error.code(), crate::ERROR_CODE_BACKFILL_PERSISTENCE_INVALID);
}
}
return;
}
#[tokio::test]
async fn pre_007_mismatched_transaction_observation_reference_is_rejected_before_store() {
let (reference, transaction, _) = match raw_acquisition_parts("devnet", 9, 19) {
std::option::Option::Some(parts) => parts,
std::option::Option::None => return,
};
let (_, _, observation) = match raw_acquisition_parts("devnet", 10, 20) {
std::option::Option::Some(parts) => parts,
std::option::Option::None => return,
};
let network = match raw_network("devnet") {
std::option::Option::Some(network) => network,
std::option::Option::None => return,
};
let port = FakePersistencePort::new(network, &[]);
let result = super::persist_available_with_port(&port, reference, transaction, observation).await;
assert!(result.is_err());
assert_eq!(port.calls(), 0);
return;
}

View File

@@ -0,0 +1,234 @@
// file: crates/ksp-job-backfill-lib/unit_tests/request.rs
// version: 1
fn signature(character: char) -> std::option::Option<crate::BackfillSignature> {
return match crate::BackfillSignature::new(character.to_string().repeat(crate::MIN_BACKFILL_SIGNATURE_TEXT_BYTES)) {
std::result::Result::Ok(value) => std::option::Option::Some(value),
std::result::Result::Err(_) => std::option::Option::None,
};
}
fn network(value: &str) -> std::option::Option<ksp_store_lib::RawNetworkId> {
return match ksp_store_lib::RawNetworkId::new(value) {
std::result::Result::Ok(value) => std::option::Option::Some(value),
std::result::Result::Err(_) => std::option::Option::None,
};
}
fn job_id(value: &str) -> std::option::Option<ksp_job_api::JobId> {
return match ksp_job_api::JobId::new(value) {
std::result::Result::Ok(value) => std::option::Option::Some(value),
std::result::Result::Err(_) => std::option::Option::None,
};
}
#[test]
fn pre_005_signature_shape_is_bounded_and_debug_redacted() {
let minimum = crate::BackfillSignature::new("1".repeat(crate::MIN_BACKFILL_SIGNATURE_TEXT_BYTES));
assert!(minimum.is_ok());
let maximum = crate::BackfillSignature::new("z".repeat(crate::MAX_BACKFILL_SIGNATURE_TEXT_BYTES));
assert!(maximum.is_ok());
assert!(crate::BackfillSignature::new("1".repeat(crate::MIN_BACKFILL_SIGNATURE_TEXT_BYTES - 1)).is_err());
assert!(crate::BackfillSignature::new("1".repeat(crate::MAX_BACKFILL_SIGNATURE_TEXT_BYTES + 1)).is_err());
assert!(crate::BackfillSignature::new(format!("{}0", "1".repeat(crate::MIN_BACKFILL_SIGNATURE_TEXT_BYTES - 1))).is_err());
let minimum = match minimum {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return,
};
let debug = format!("{minimum:?}");
assert_eq!(debug, "BackfillSignature(..)");
assert!(!debug.contains(minimum.as_str()));
return;
}
#[test]
fn pre_005_explicit_scope_deduplicates_at_first_occurrence_without_reordering() {
let first = match signature('1') {
std::option::Option::Some(value) => value,
std::option::Option::None => return,
};
let second = match signature('2') {
std::option::Option::Some(value) => value,
std::option::Option::None => return,
};
let scope_result = crate::BackfillScope::explicit_signatures(std::vec![first.clone(), second.clone(), first.clone(), second.clone()]);
assert!(scope_result.is_ok());
let scope = match scope_result {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return,
};
let signatures = match scope.signatures() {
std::option::Option::Some(value) => value,
std::option::Option::None => return,
};
assert_eq!(signatures, &[first, second]);
let debug = format!("{scope:?}");
assert!(debug.contains("signature_count"));
assert!(!debug.contains(signatures[0].as_str()));
return;
}
#[test]
fn pre_005_request_bounds_are_exact_and_explicit_context_is_rejected() {
let network = match network("devnet") {
std::option::Option::Some(value) => value,
std::option::Option::None => return,
};
let job_id = match job_id("backfill:test") {
std::option::Option::Some(value) => value,
std::option::Option::None => return,
};
let role = ksp_onchain_transport_lib::HttpRoleName::new("history");
let address = ksp_core_lib::Pubkey::new_from_array([7_u8; 32]);
let scope = crate::BackfillScope::latest_address(address);
let valid = crate::BackfillRequest::new(
job_id.clone(),
network.clone(),
role.clone(),
crate::BackfillCommitment::Confirmed,
scope.clone(),
crate::MAX_BACKFILL_PAGE_SIZE,
crate::MAX_BACKFILL_PAGES,
crate::MAX_BACKFILL_CANDIDATES,
crate::MAX_BACKFILL_HYDRATION_CONCURRENCY,
std::option::Option::Some(1),
);
assert!(valid.is_ok());
for (page_size, max_pages, max_candidates, concurrency) in [
(0, 1, 1, 1),
(crate::MAX_BACKFILL_PAGE_SIZE + 1, 1, 1, 1),
(1, 0, 1, 1),
(1, crate::MAX_BACKFILL_PAGES + 1, 1, 1),
(1, 1, 0, 1),
(1, 1, crate::MAX_BACKFILL_CANDIDATES + 1, 1),
(1, 1, 1, 0),
(1, 1, 1, crate::MAX_BACKFILL_HYDRATION_CONCURRENCY + 1),
] {
let result = crate::BackfillRequest::new(
job_id.clone(),
network.clone(),
role.clone(),
crate::BackfillCommitment::Finalized,
scope.clone(),
page_size,
max_pages,
max_candidates,
concurrency,
std::option::Option::None,
);
assert!(result.is_err());
}
let explicit_signature = match signature('3') {
std::option::Option::Some(value) => value,
std::option::Option::None => return,
};
let explicit_scope = crate::BackfillScope::explicit_signatures(std::vec![explicit_signature]);
let explicit_scope = match explicit_scope {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return,
};
let explicit_with_context =
crate::BackfillRequest::new(job_id, network, role, crate::BackfillCommitment::Confirmed, explicit_scope, 1, 1, 1, 1, std::option::Option::Some(10));
assert!(explicit_with_context.is_err());
return;
}
#[test]
fn pre_005_scope_fingerprint_is_network_semantic_and_transport_source_independent() {
let signature = match signature('4') {
std::option::Option::Some(value) => value,
std::option::Option::None => return,
};
let scope_result = crate::BackfillScope::explicit_signatures(std::vec![signature]);
let scope = match scope_result {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return,
};
let mainnet = match network("mainnet") {
std::option::Option::Some(value) => value,
std::option::Option::None => return,
};
let devnet = match network("devnet") {
std::option::Option::Some(value) => value,
std::option::Option::None => return,
};
let first_job = match job_id("backfill:first") {
std::option::Option::Some(value) => value,
std::option::Option::None => return,
};
let second_job = match job_id("backfill:second") {
std::option::Option::Some(value) => value,
std::option::Option::None => return,
};
let first = crate::BackfillRequest::new(
first_job,
devnet.clone(),
ksp_onchain_transport_lib::HttpRoleName::new("primary-http"),
crate::BackfillCommitment::Confirmed,
scope.clone(),
100,
5,
10,
1,
std::option::Option::None,
);
let second = crate::BackfillRequest::new(
second_job,
devnet,
ksp_onchain_transport_lib::HttpRoleName::new("fallback-http"),
crate::BackfillCommitment::Confirmed,
scope.clone(),
100,
5,
10,
64,
std::option::Option::None,
);
let other_network = crate::BackfillRequest::new(
match job_id("backfill:third") {
std::option::Option::Some(value) => value,
std::option::Option::None => return,
},
mainnet,
ksp_onchain_transport_lib::HttpRoleName::new("primary-http"),
crate::BackfillCommitment::Confirmed,
scope,
100,
5,
10,
1,
std::option::Option::None,
);
let first = match first {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return,
};
let second = match second {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return,
};
let other_network = match other_network {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return,
};
assert_eq!(first.scope_fingerprint(), second.scope_fingerprint());
assert_ne!(first.scope_fingerprint(), other_network.scope_fingerprint());
return;
}
#[test]
fn pre_005_scope_kind_and_anchor_are_distinct_semantics() {
let anchor = match signature('5') {
std::option::Option::Some(value) => value,
std::option::Option::None => return,
};
let address = ksp_core_lib::Pubkey::new_from_array([9_u8; 32]);
let before = crate::BackfillScope::before_address(address, anchor.clone());
let after = crate::BackfillScope::after_address(address, anchor);
assert_eq!(before.kind(), crate::BackfillScopeKind::BeforeAddress);
assert_eq!(after.kind(), crate::BackfillScopeKind::AfterAddress);
assert!(before.anchor().is_some());
assert!(after.anchor().is_some());
assert_ne!(before, after);
return;
}

View File

@@ -0,0 +1,158 @@
// file: crates/ksp-job-backfill-lib/unit_tests/runtime.rs
// version: 3
use ksp_job_api::JobSnapshotSource; // rust-rules: trait-import
fn request() -> std::option::Option<crate::BackfillRequest> {
let signature = match crate::BackfillSignature::new("1".repeat(crate::MIN_BACKFILL_SIGNATURE_TEXT_BYTES)) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return std::option::Option::None,
};
let scope = match crate::BackfillScope::explicit_signatures(std::vec![signature]) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return std::option::Option::None,
};
let job_id = match ksp_job_api::JobId::new("backfill:runtime-test") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return std::option::Option::None,
};
let network = match ksp_store_lib::RawNetworkId::new("devnet") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return std::option::Option::None,
};
return crate::BackfillRequest::new(
job_id,
network,
ksp_onchain_transport_lib::HttpRoleName::new("history"),
crate::BackfillCommitment::Confirmed,
scope,
100,
10,
1,
1,
std::option::Option::None,
)
.ok();
}
#[tokio::test]
async fn pre_009_latest_value_source_coalesces_progress_for_slow_independent_listeners() {
let request = match request() {
std::option::Option::Some(value) => value,
std::option::Option::None => return,
};
let runtime = match crate::BackfillJobRuntime::new(request) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return,
};
let handle = runtime.handle();
let listener_a = handle.snapshots();
let listener_b = handle.snapshots();
let initial = listener_a.current();
assert_eq!(initial.sequence().value(), 0);
assert_eq!(initial.state(), ksp_job_api::JobState::Created);
let first = runtime.publisher.publish_running(crate::BackfillJobPhase::Discovering);
assert!(first.is_ok());
let second = runtime.publisher.publish_running(crate::BackfillJobPhase::Executing);
assert!(second.is_ok());
let coalesced = listener_a.wait_for_change(initial.sequence()).await;
assert_eq!(coalesced.sequence().value(), 2);
assert_eq!(coalesced.snapshot().phase(), crate::BackfillJobPhase::Executing);
let listener_b_current = listener_b.current();
assert_eq!(listener_b_current.sequence().value(), 2);
assert_eq!(listener_b_current.snapshot().phase(), crate::BackfillJobPhase::Executing);
return;
}
#[tokio::test]
async fn pre_009_terminal_snapshot_is_retained_and_late_cancellation_is_rejected() {
let request = match request() {
std::option::Option::Some(value) => value,
std::option::Option::None => return,
};
let runtime = match crate::BackfillJobRuntime::new(request) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return,
};
let handle = runtime.handle();
let listener = handle.snapshots();
let started = runtime.publisher.publish_running(crate::BackfillJobPhase::Discovering);
assert!(started.is_ok());
assert_eq!(runtime.control.claim_normal_terminal(), super::TerminalClaim::Completed);
let terminal = runtime.publisher.publish_terminal(ksp_job_api::JobState::Completed(ksp_job_api::JobCompletion::Complete), std::option::Option::None);
assert!(terminal.is_ok());
assert!(!handle.cancel());
let current = listener.current();
assert!(current.state().is_terminal());
assert_eq!(current.snapshot().phase(), crate::BackfillJobPhase::Finished);
let retained = listener.wait_for_change(current.sequence()).await;
assert_eq!(retained.sequence(), current.sequence());
assert_eq!(retained.state(), current.state());
return;
}
#[test]
fn pre_009_terminal_race_is_first_decision_wins_for_cancellation_vs_completion() {
let (cancel_sender, _) = tokio::sync::watch::channel(false);
let cancellation_first = super::BackfillRuntimeControl::new(cancel_sender);
assert!(cancellation_first.request_cancellation());
assert_eq!(cancellation_first.claim_normal_terminal(), super::TerminalClaim::Cancelled);
assert!(!cancellation_first.request_cancellation());
let (cancel_sender, _) = tokio::sync::watch::channel(false);
let completion_first = super::BackfillRuntimeControl::new(cancel_sender);
assert_eq!(completion_first.claim_normal_terminal(), super::TerminalClaim::Completed);
assert!(!completion_first.request_cancellation());
return;
}
#[test]
fn pre_009_fatal_failure_overrides_pending_cancellation_before_terminal_publication() {
let (cancel_sender, _) = tokio::sync::watch::channel(false);
let control = super::BackfillRuntimeControl::new(cancel_sender);
assert!(control.request_cancellation());
control.claim_failed();
assert_eq!(control.claim_normal_terminal(), super::TerminalClaim::Failed);
assert!(!control.request_cancellation());
return;
}
#[tokio::test]
async fn pre_009_long_running_pre_store_future_is_cancelled_cooperatively() {
let (cancel_sender, cancel_receiver) = tokio::sync::watch::channel(false);
let control = super::BackfillRuntimeControl::new(cancel_sender);
let signal = crate::BackfillCancellationSignal::new(control.token(), cancel_receiver);
let operation = std::future::pending::<ksp_core_lib::Result<usize>>();
let wait = signal.run_cancellable(operation);
let cancel = async {
tokio::task::yield_now().await;
assert!(control.request_cancellation());
};
let (result, ()) = tokio::join!(wait, cancel);
let error = match result {
std::result::Result::Ok(_) => return,
std::result::Result::Err(error) => error,
};
assert_eq!(error.code(), crate::ERROR_CODE_BACKFILL_CANCELLED);
return;
}
#[test]
fn pre_009_snapshot_debug_and_public_shape_do_not_include_transport_or_raw_payloads() {
let request = match request() {
std::option::Option::Some(value) => value,
std::option::Option::None => return,
};
let runtime = match crate::BackfillJobRuntime::new(request) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return,
};
let snapshot = runtime.handle().snapshots().current();
let debug = format!("{:?}", snapshot.snapshot());
for forbidden in ["http://", "https://", "endpoint", "provider", "raw_payload", "transaction_data"] {
assert!(!debug.contains(forbidden), "unsafe snapshot diagnostic leaked: {forbidden}");
}
assert_eq!(snapshot.snapshot().scope_kind(), crate::BackfillScopeKind::ExplicitSignatures);
assert_eq!(snapshot.snapshot().candidates_selected(), 0);
assert!(snapshot.snapshot().failure_code().is_none());
return;
}

View File

@@ -1,5 +1,5 @@
// file: crates/ksp-onchain-transport-lib/src/http_executor.rs
// version: 4
// version: 7
const HTTP_BAD_GATEWAY: u16 = 502;
const HTTP_GATEWAY_TIMEOUT: u16 = 504;
@@ -8,6 +8,63 @@ const HTTP_REQUEST_TIMEOUT: u16 = 408;
const HTTP_SERVICE_UNAVAILABLE: u16 = 503;
const HTTP_TOO_MANY_REQUESTS: u16 = 429;
/// Typed value returned by an observed HTTP RPC path together with the safe identity of the endpoint that produced the successful response.
///
/// Endpoint URLs, headers and raw HTTP bodies are intentionally absent.
#[derive(Clone, PartialEq)]
pub struct HttpObservedValue<T> {
value: T,
endpoint_name: std::string::String,
provider: crate::HttpProviderName,
}
impl<T> HttpObservedValue<T> {
/// Returns the typed RPC value.
#[must_use]
pub const fn value(&self) -> &T {
return &self.value;
}
/// Returns the safe configured identity of the endpoint that produced the successful response.
#[must_use]
pub fn endpoint_name(&self) -> &str {
return self.endpoint_name.as_str();
}
/// Returns the safe provider descriptor attached to the successful endpoint.
#[must_use]
pub const fn provider(&self) -> &crate::HttpProviderName {
return &self.provider;
}
/// Consumes the observation and returns only the typed value.
#[must_use]
pub fn into_value(self) -> T {
return self.value;
}
/// Builds an observed value from a successful Transport attempt and its safe routing identity.
pub(crate) fn new(value: T, endpoint_name: std::string::String, provider: crate::HttpProviderName) -> Self {
return Self { value, endpoint_name, provider };
}
/// Consumes the observation into its typed value and safe routing identity for crate-internal typed decoding.
pub(crate) fn into_parts(self) -> (T, std::string::String, crate::HttpProviderName) {
return (self.value, self.endpoint_name, self.provider);
}
}
impl<T> std::fmt::Debug for HttpObservedValue<T> {
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
return formatter
.debug_struct("HttpObservedValue")
.field("endpoint_name", &self.endpoint_name)
.field("provider", &self.provider)
.field("value", &"<available>")
.finish();
}
}
impl crate::HttpTransportPool {
/// Executes one audited standard Solana HTTP JSON-RPC method through KSP routing, admission and bounded retry policy.
///
@@ -19,6 +76,33 @@ impl crate::HttpTransportPool {
method: &crate::HttpRpcMethodDescriptor,
params: std::vec::Vec<serde_json::Value>,
) -> ksp_core_lib::Result<serde_json::Value> {
return self.execute_standard_rpc_with(role, method, params, |value, _permit| return value).await;
}
/// Executes one audited standard Solana HTTP JSON-RPC method and retains only safe routing identity for the successful attempt.
pub(crate) async fn execute_standard_rpc_observed(
&self,
role: &crate::HttpRoleName,
method: &crate::HttpRpcMethodDescriptor,
params: std::vec::Vec<serde_json::Value>,
) -> ksp_core_lib::Result<crate::HttpObservedValue<serde_json::Value>> {
return self
.execute_standard_rpc_with(role, method, params, |value, permit| {
return crate::HttpObservedValue::new(value, permit.selection().endpoint_name().to_owned(), permit.client().provider().clone());
})
.await;
}
async fn execute_standard_rpc_with<T, F>(
&self,
role: &crate::HttpRoleName,
method: &crate::HttpRpcMethodDescriptor,
params: std::vec::Vec<serde_json::Value>,
on_success: F,
) -> ksp_core_lib::Result<T>
where
F: std::ops::FnOnce(serde_json::Value, &crate::HttpRequestPermit) -> T,
{
let support = method.ensure_runtime_supported();
if let std::result::Result::Err(error) = support {
return std::result::Result::Err(error);
@@ -156,7 +240,12 @@ impl crate::HttpTransportPool {
http_status = status,
"completed Solana HTTP JSON-RPC request"
);
return parsed.into_result();
let value = parsed.into_result();
let value = match value {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
return std::result::Result::Ok(on_success(value, &permit));
}
}
}
@@ -203,14 +292,11 @@ async fn wait_retry_delay(delay: std::time::Duration, deadline: std::time::Insta
return std::time::Instant::now() < deadline;
}
fn execution_timeout(method: &crate::HttpRpcMethodDescriptor, message: &str) -> ksp_core_lib::Result<serde_json::Value> {
fn execution_timeout<T>(method: &crate::HttpRpcMethodDescriptor, message: &str) -> ksp_core_lib::Result<T> {
return std::result::Result::Err(ksp_core_lib::Error::new(crate::ERROR_CODE_TIMEOUT, message).with_context("rpc_method", method.method()));
}
fn rate_limited_error(
method: &crate::HttpRpcMethodDescriptor,
provider_retry_after: std::option::Option<std::time::Duration>,
) -> ksp_core_lib::Result<serde_json::Value> {
fn rate_limited_error<T>(method: &crate::HttpRpcMethodDescriptor, provider_retry_after: std::option::Option<std::time::Duration>) -> ksp_core_lib::Result<T> {
let mut error = ksp_core_lib::Error::new(crate::ERROR_CODE_RATE_LIMITED, "Solana HTTP endpoint rate-limited the JSON-RPC request")
.with_context("rpc_method", method.method());
if let std::option::Option::Some(delay) = provider_retry_after {
@@ -219,7 +305,7 @@ fn rate_limited_error(
return std::result::Result::Err(error);
}
fn http_status_error(method: &crate::HttpRpcMethodDescriptor, status: u16) -> ksp_core_lib::Result<serde_json::Value> {
fn http_status_error<T>(method: &crate::HttpRpcMethodDescriptor, status: u16) -> ksp_core_lib::Result<T> {
return std::result::Result::Err(
ksp_core_lib::Error::new(crate::ERROR_CODE_HTTP_REQUEST_FAILED, "Solana HTTP endpoint returned an unsuccessful status")
.with_context("rpc_method", method.method())

View File

@@ -1,5 +1,5 @@
// file: crates/ksp-onchain-transport-lib/src/lib.rs
// version: 44
// version: 45
#![warn(missing_docs)]
#![deny(unreachable_pub)]
@@ -271,6 +271,8 @@ pub use self::http_client::HttpEndpointClient;
pub use self::http_client::HttpEndpointRoleSnapshot;
/// Safe metadata snapshot for one logical HTTP endpoint.
pub use self::http_client::HttpEndpointSnapshot;
/// Typed RPC value paired with the safe identity of the HTTP endpoint that produced the successful response.
pub use self::http_executor::HttpObservedValue;
/// Result of one logical endpoint selection.
pub use self::http_pool::HttpEndpointSelection;
/// Runtime admission permit for one HTTP request.

View File

@@ -1,5 +1,5 @@
// file: crates/ksp-onchain-transport-lib/src/rpc_transactions.rs
// version: 10
// version: 11
const MAX_RECENT_PRIORITIZATION_FEE_ACCOUNTS: usize = 128;
const MAX_SIGNATURES_FOR_ADDRESS_LIMIT: usize = 1_000;
@@ -1249,25 +1249,42 @@ impl crate::HttpTransportPool {
signature: &str,
config: std::option::Option<&crate::SolanaGetTransactionConfig>,
) -> ksp_core_lib::Result<std::option::Option<crate::SolanaConfirmedTransaction>> {
if let std::option::Option::Some(config) = config
&& config.commitment() == std::option::Option::Some(crate::SolanaCommitment::Processed)
{
return std::result::Result::Err(
ksp_core_lib::Error::new(
crate::ERROR_CODE_INVALID_RPC_PARAMETERS,
"getTransaction commitment must be confirmed or finalized when explicitly provided",
)
.with_context("rpc_method", "getTransaction")
.with_context("commitment", "processed"),
);
}
let mut params = std::vec![serde_json::Value::String(signature.to_owned())];
if let std::option::Option::Some(config) = config {
params.push((*config).to_json_value());
}
let params = get_transaction_params(signature, config);
let params = match params {
std::result::Result::Ok(params) => params,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
return self.execute_get_transaction(role, params).await;
}
/// Executes the current object-form `getTransaction` request and reports the safe identity of the endpoint that produced the successful response.
///
/// Routing, admission, timeout and retry behavior are identical to [`Self::get_transaction`]. The returned observation never contains an endpoint URL,
/// HTTP headers or a raw HTTP body.
pub async fn get_transaction_observed(
&self,
role: &crate::HttpRoleName,
signature: &str,
config: std::option::Option<&crate::SolanaGetTransactionConfig>,
) -> ksp_core_lib::Result<crate::HttpObservedValue<std::option::Option<crate::SolanaConfirmedTransaction>>> {
let params = get_transaction_params(signature, config);
let params = match params {
std::result::Result::Ok(params) => params,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let observed = self.execute_transaction_rpc_observed("getTransaction", role, params).await;
let observed = match observed {
std::result::Result::Ok(observed) => observed,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let (value, endpoint_name, provider) = observed.into_parts();
let transaction = decode_get_transaction(value);
return match transaction {
std::result::Result::Ok(transaction) => std::result::Result::Ok(crate::HttpObservedValue::new(transaction, endpoint_name, provider)),
std::result::Result::Err(error) => std::result::Result::Err(error),
};
}
/// Executes the deprecated bare-encoding `getTransaction` request form retained by Solana RPC for backwards compatibility.
#[deprecated(note = "use HttpTransportPool::get_transaction with SolanaGetTransactionConfig; the bare encoding request form is deprecated")]
pub async fn get_transaction_legacy(
@@ -1297,14 +1314,7 @@ impl crate::HttpTransportPool {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
if value.is_null() {
return std::result::Result::Ok(std::option::Option::None);
}
let transaction = crate::SolanaConfirmedTransaction::decode_wire("getTransaction", value);
return match transaction {
std::result::Result::Ok(transaction) => std::result::Result::Ok(std::option::Option::Some(transaction)),
std::result::Result::Err(error) => std::result::Result::Err(error),
};
return decode_get_transaction(value);
}
/// Executes typed `requestAirdrop` through the common KSP HTTP transport path.
@@ -1466,6 +1476,54 @@ impl crate::HttpTransportPool {
};
return self.execute_standard_rpc(role, method, params).await;
}
async fn execute_transaction_rpc_observed(
&self,
method_name: &'static str,
role: &crate::HttpRoleName,
params: std::vec::Vec<serde_json::Value>,
) -> ksp_core_lib::Result<crate::HttpObservedValue<serde_json::Value>> {
let method = transaction_descriptor(method_name);
let method = match method {
std::result::Result::Ok(method) => method,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
return self.execute_standard_rpc_observed(role, method, params).await;
}
}
fn get_transaction_params(
signature: &str,
config: std::option::Option<&crate::SolanaGetTransactionConfig>,
) -> ksp_core_lib::Result<std::vec::Vec<serde_json::Value>> {
if let std::option::Option::Some(config) = config
&& config.commitment() == std::option::Option::Some(crate::SolanaCommitment::Processed)
{
return std::result::Result::Err(
ksp_core_lib::Error::new(
crate::ERROR_CODE_INVALID_RPC_PARAMETERS,
"getTransaction commitment must be confirmed or finalized when explicitly provided",
)
.with_context("rpc_method", "getTransaction")
.with_context("commitment", "processed"),
);
}
let mut params = std::vec![serde_json::Value::String(signature.to_owned())];
if let std::option::Option::Some(config) = config {
params.push((*config).to_json_value());
}
return std::result::Result::Ok(params);
}
fn decode_get_transaction(value: serde_json::Value) -> ksp_core_lib::Result<std::option::Option<crate::SolanaConfirmedTransaction>> {
if value.is_null() {
return std::result::Result::Ok(std::option::Option::None);
}
let transaction = crate::SolanaConfirmedTransaction::decode_wire("getTransaction", value);
return match transaction {
std::result::Result::Ok(transaction) => std::result::Result::Ok(std::option::Option::Some(transaction)),
std::result::Result::Err(error) => std::result::Result::Err(error),
};
}
#[derive(serde::Deserialize)]

View File

@@ -1,5 +1,5 @@
// file: crates/ksp-onchain-transport-lib/tests/public_api.rs
// version: 48
// version: 49
//! Integration tests for the public `ksp-onchain-transport-lib` consumer contract.
@@ -199,6 +199,15 @@ fn public_pre_002_shared_rpc_types_are_constructible_from_crate_root() {
assert_eq!(vote.vote_pubkey(), std::option::Option::Some(&pubkey));
}
#[test]
fn public_v0_3_6_pre_004_observed_get_transaction_surface_is_available_from_crate_root() {
let _get_transaction_observed = ksp_onchain_transport_lib::HttpTransportPool::get_transaction_observed;
let observed: std::option::Option<
ksp_onchain_transport_lib::HttpObservedValue<std::option::Option<ksp_onchain_transport_lib::SolanaConfirmedTransaction>>,
> = std::option::Option::None;
assert!(observed.is_none());
}
#[test]
fn public_pre_003_account_wrappers_are_available_from_crate_root() {
let _get_account_info = ksp_onchain_transport_lib::HttpTransportPool::get_account_info;

View File

@@ -1,5 +1,5 @@
// file: crates/ksp-onchain-transport-lib/unit_tests/rpc_transactions.rs
// version: 10
// version: 11
#[test]
fn transaction_encoding_strings_match_current_and_legacy_wire_labels() {
@@ -966,6 +966,80 @@ async fn typed_get_transaction_preserves_raw_json_meta_version_and_transaction_i
);
}
#[tokio::test(flavor = "current_thread")]
async fn typed_get_transaction_observed_reports_actual_winner_after_retry_reroute() {
let (first_url, first_handle) = serve_transaction_status_and_count("429 Too Many Requests");
let (winner_url, winner_handle) = serve_transaction_once(include_str!("../fixtures/http/get_transaction.base64.json"));
let urls = [(first_url.as_str(), "first-endpoint", "first-provider"), (winner_url.as_str(), "winner-endpoint", "winner-provider")];
let mut endpoints = std::vec::Vec::with_capacity(urls.len());
for (url, endpoint_name, provider) in urls {
let role = crate::HttpEndpointRoleSettings::new(
crate::HttpRoleName::new("default"),
true,
std::vec![crate::HttpRequestKind::wildcard()],
10,
crate::HttpRoleLimits::new(std::option::Option::None, std::option::Option::None, std::option::Option::None, std::option::Option::None),
);
endpoints.push(crate::HttpEndpointSettings::new(
endpoint_name,
true,
crate::HttpProviderName::new(provider),
crate::HttpClusterName::new("local"),
crate::HttpEndpointUrl::parse(url).expect("fixture URL must parse"),
std::time::Duration::from_millis(100),
std::time::Duration::from_secs(1),
std::option::Option::Some(1),
std::vec![role],
));
}
let pool = crate::HttpTransportPool::new(crate::HttpTransportSettings::new(
endpoints,
crate::HttpRetrySettings::new(1, std::time::Duration::from_millis(1), std::time::Duration::from_millis(2)),
))
.expect("observed fixture pool must build");
let config = crate::SolanaGetTransactionConfig::new(
std::option::Option::Some(crate::SolanaCommitment::Confirmed),
std::option::Option::Some(crate::SolanaTransactionEncoding::Base64),
std::option::Option::Some(0),
);
let observed = pool
.get_transaction_observed(&crate::HttpRoleName::new("default"), "fixture-signature", std::option::Option::Some(&config))
.await
.expect("retry-safe observed getTransaction must succeed on the second endpoint");
assert_eq!(observed.endpoint_name(), "winner-endpoint");
assert_eq!(observed.provider().as_str(), "winner-provider");
let transaction = observed.value().as_ref().expect("winning response must contain a transaction");
assert_eq!(transaction.slot(), 431_000_062);
assert!(matches!(
transaction.transaction(),
crate::SolanaEncodedTransaction::Binary { encoding: crate::SolanaTransactionBinaryEncoding::Base64, .. }
));
let (first_count, first_request) = first_handle.join().expect("first fixture server must join");
assert_eq!(first_count, 1);
assert_eq!(transaction_request_body(first_request.as_str())["method"], serde_json::json!("getTransaction"));
let winner_request = winner_handle.join().expect("winner fixture server must join");
assert_eq!(transaction_request_body(winner_request.as_str())["method"], serde_json::json!("getTransaction"));
}
#[tokio::test(flavor = "current_thread")]
async fn typed_get_transaction_observed_preserves_null_and_redacts_typed_value_debug() {
let (url, handle) = serve_transaction_once(include_str!("../fixtures/http/get_transaction.null.json"));
let pool = transaction_pool_for_url(url.as_str());
let observed = pool
.get_transaction_observed(&crate::HttpRoleName::new("default"), "fixture-signature", std::option::Option::None)
.await
.expect("observed null getTransaction must succeed");
assert!(observed.value().is_none());
assert_eq!(observed.endpoint_name(), "fixture-0");
assert_eq!(observed.provider().as_str(), "fixture");
let rendered = format!("{observed:?}");
assert!(rendered.contains("fixture-0"));
assert!(rendered.contains("fixture"));
assert!(rendered.contains("<available>"));
assert!(!rendered.contains("fixture-signature"));
handle.join().expect("fixture server must join");
}
#[tokio::test(flavor = "current_thread")]
async fn typed_get_transaction_preserves_unsupported_version_rpc_error() {
let (url, handle) = serve_transaction_once(include_str!("../fixtures/http/get_transaction.error_unsupported_version.json"));

View File

@@ -1,5 +1,5 @@
// file: crates/ksp-program-api/src/program_instruction_decode.rs
// version: 1
// version: 2
/// Recognition strength reported by one Program instruction implementation.
///
@@ -38,10 +38,10 @@ pub enum ProgramInstructionDecodeOutcome<Decoded> {
impl<Decoded> std::fmt::Debug for ProgramInstructionDecodeOutcome<Decoded> {
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
Self::Decoded(_) => return formatter.write_str("Decoded"),
Self::Unsupported => return formatter.write_str("Unsupported"),
}
return match self {
Self::Decoded(_) => formatter.write_str("Decoded"),
Self::Unsupported => formatter.write_str("Unsupported"),
};
}
}

View File

@@ -0,0 +1,20 @@
# file: crates/ksp-store-lib/Cargo.toml
# version: 2
[package]
name = "ksp-store-lib"
version.workspace = true
edition.workspace = true
repository.workspace = true
[features]
default = ["postgres"]
postgres = ["dep:ksp-store-postgres-lib"]
[dependencies]
ksp-logging-lib = { path = "../ksp-logging-lib" }
ksp-store-api = { path = "../ksp-store-api" }
ksp-store-postgres-lib = { path = "../ksp-store-postgres-lib", optional = true }
[lints]
workspace = true

View File

@@ -0,0 +1,115 @@
<!-- file: crates/ksp-store-lib/README.md -->
<!-- version: 5 -->
# ksp-store-lib
`ksp-store-lib` est la façade runtime Store commune de KSP.
Elle expose aux consumers une surface backend-neutral, réexporte les contrats RAW de `ksp-store-api`, sélectionne uniquement les backends compilés et masque leurs objets physiques. Le backend PostgreSQL officiel est activé par défaut via la feature `postgres` et reste implémenté dans `ksp-store-postgres-lib`.
## Responsabilités
`ksp-store-lib` possède :
- `StoreSettings`, avec un réseau logique unique, un backend sélectionné et un timeout de fermeture borné ;
- les settings PostgreSQL publics KSP-owned : pool, TLS, bootstrap/migrations et URI sensible ;
- la feature `postgres` par défaut et le comportement explicite `backend_not_compiled` lorsque PostgreSQL est sélectionné sans cette feature ;
- `Store::open`, qui ne retourne une instance qu'après validation, ouverture physique du backend compilé et bootstrap/history réussis ;
- `Store::runtime_snapshot()` pour les compteurs runtime sûrs sans I/O ;
- `Store::health().await` pour la readiness portable et bornée ;
- `Store::close(self).await` pour la fermeture explicite bornée ;
- le mapping des erreurs backend vers des codes Store stables sans exposer les erreurs physiques ;
- les dix capabilities RAW de `ksp-store-api` dispatchées vers le backend compilé : six `RawTransaction*` et quatre `RawAccount*` ;
- une validation réseau backend-neutral avant dispatch pour toutes les opérations qui portent explicitement un réseau ;
- les réexports crate-root de `ksp-store-api` nécessaires aux consumers ordinaires.
## Une instance = un réseau
Une instance `Store` représente exactement :
```text
1 Store = 1 RawNetworkId + 1 backend physique sélectionné
```
Le runtime Store n'est pas un multiplexeur multi-database ou multi-réseau. La sélection d'un target nommé appartient à Config. Le document `std.store` peut donc définir plusieurs targets indépendants, mais un appel à `Store::open` reçoit les settings d'un seul target.
Cette séparation permet d'utiliser des bases PostgreSQL distinctes par réseau tout en conservant le réseau dans l'identité logique des données RAW.
## PostgreSQL
Avec la feature par défaut :
```text
ksp-store-lib
-> ksp-store-api
-> ksp-logging-lib
-> ksp-store-postgres-lib
```
`ksp-store-lib` ne réexporte aucun type `tokio-postgres`, Deadpool ou Rustls.
Les modes TLS publics sont volontairement limités à :
```text
Disabled
VerifyFull
```
`VerifyFull` impose TLS avec vérification de la chaîne et de l'identité serveur. La policy typée Store prime sur les paramètres TLS présents dans l'URI.
## Config et secrets
Store ne lit ni `.env`, ni variables `KSP_*` / `KSPB_*`, ni variables/fichiers implicites libpq (`PG*`, `.pgpass`, fichiers TLS PostgreSQL).
`ksp-config-lib` possède `std.store`, la résolution des secrets et la sélection du target. Il construit ensuite un `StoreSettings` backend-neutral. L'URI PostgreSQL reste nécessaire au runtime mais n'a aucun getter public dans `ksp-store-lib` et son `Debug` est redacted.
## Surface RAW
`Store` implémente exactement dix capabilities RAW backend-neutral :
```text
RawTransactionRead
RawTransactionWrite
RawTransactionObservationRead
RawTransactionObservationWrite
RawTransactionRetentionRead
RawTransactionRetentionWrite
RawAccountStateRead
RawAccountStateWrite
RawAccountObservationRead
RawAccountObservationWrite
```
Pour `RawTransaction`, la façade fournit la lecture canonique et des observations, l'acquisition atomique transaction+observation, l'ajout idempotent d'observations, la pagination keyset et les transitions de rétention demandées par le caller.
Pour `RawAccountState`, elle fournit :
- lecture d'un état complet par référence durable `(network, pubkey, slot, state_hash)` ;
- pagination keyset des références dans l'ordre total `(slot, pubkey, state_hash)`, avec filtre pubkey optionnel ;
- écriture atomique état+observation avec idempotence exacte et conflit sur contenu divergent ;
- lecture et ajout d'observations account ;
- validation réseau avant dispatch pour les opérations dont l'input porte explicitement un réseau.
Les cursors restent opaques et propres à leur famille. Store ne leur attribue aucune sémantique de batch, priorité ou scheduling.
## Hors périmètre
Sont volontairement hors de cette surface :
- rétention, archivage, purge, delete ou compaction de `RawAccountState` ;
- batch-size, priorité, backlog ou policy de worker/job ;
- transport d'acquisition, Program decoding et materialization ;
- exposition publique de SQL, pool, client, row, statement ou transaction PostgreSQL.
## Documentation
- [`USAGE.md`](USAGE.md) — guide pratique de construction, lifecycle et capabilities Store ;
- [`../ksp-store-postgres-lib/README.md`](../ksp-store-postgres-lib/README.md) — responsabilité du backend PostgreSQL physique ;
- [`../../config/std.store.json`](../../config/std.store.json) — targets Store committed ;
- [`../../docs/architecture/008-DATA_MATERIALIZATION_AND_STORE.md`](../../docs/architecture/008-DATA_MATERIALIZATION_AND_STORE.md) — architecture durable Store ;
- [`../../docs/plans/023-V0_3_2_STORE_POSTGRES_FOUNDATION_PLAN.md`](../../docs/plans/023-V0_3_2_STORE_POSTGRES_FOUNDATION_PLAN.md) — plan de fondation ;
- [`../../docs/validation/019-V0_3_2_STORE_POSTGRES_FOUNDATION.md`](../../docs/validation/019-V0_3_2_STORE_POSTGRES_FOUNDATION.md) — validation de fondation ;
- [`../../docs/plans/024-V0_3_3_STORE_POSTGRES_RAW_TRANSACTION_PLAN.md`](../../docs/plans/024-V0_3_3_STORE_POSTGRES_RAW_TRANSACTION_PLAN.md) — plan `RawTransaction` ;
- [`../../docs/validation/020-V0_3_3_STORE_POSTGRES_RAW_TRANSACTION.md`](../../docs/validation/020-V0_3_3_STORE_POSTGRES_RAW_TRANSACTION.md) — validation `RawTransaction` ;
- [`../../docs/plans/025-V0_3_4_STORE_POSTGRES_RAW_ACCOUNT_PLAN.md`](../../docs/plans/025-V0_3_4_STORE_POSTGRES_RAW_ACCOUNT_PLAN.md) — design `RawAccountState` et complétude RAW ;
- [`../../docs/validation/021-V0_3_4_STORE_POSTGRES_RAW_ACCOUNT.md`](../../docs/validation/021-V0_3_4_STORE_POSTGRES_RAW_ACCOUNT.md) — validation `RawAccountState` et conformance 10/10.

View File

@@ -0,0 +1,384 @@
<!-- file: crates/ksp-store-lib/USAGE.md -->
<!-- version: 5 -->
# Utilisation de ksp-store-lib
## 1. Dépendance et backend compilé
Le consumer runtime dépend de la façade commune :
```toml
[dependencies]
ksp-store-lib = { path = "../ksp-store-lib" }
```
La feature par défaut compile le backend PostgreSQL :
```text
postgres
```
Pour compiler la façade sans backend physique :
```toml
ksp-store-lib = { path = "../ksp-store-lib", default-features = false }
```
Dans ce mode, les settings PostgreSQL restent représentables, mais `Store::open` retourne `ERROR_CODE_BACKEND_NOT_COMPILED` avant toute I/O si PostgreSQL est sélectionné.
Un consumer applicatif ordinaire ne dépend pas directement de `ksp-store-postgres-lib`.
## 2. Obtenir les settings depuis Config
Le chemin applicatif recommandé passe par `ksp-config-lib`, propriétaire de `std.store`, de la résolution `.env` et des secrets.
```rust
fn resolve_store_settings(
engine: &ksp_config_lib::ConfigDocumentEngine,
environment: &ksp_config_lib::ConfigEnvironment,
target: std::option::Option<&str>,
) -> ksp_core_lib::Result<ksp_store_lib::StoreSettings> {
let resolved = engine.load_resolved_store_config(target, environment);
let resolved = match resolved {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
return std::result::Result::Ok(resolved.into_settings());
}
```
Chaque `StoreSettings` sélectionne exactement un réseau logique et un backend physique. La sélection d'un target nommé appartient à Config ; `Store` ne route pas automatiquement entre plusieurs targets.
## 3. Construire des settings programmatiquement
La construction directe est utile pour les tests et outils qui ne passent pas par Config.
```rust
fn programmatic_store_settings(connection_uri: std::string::String) -> ksp_store_lib::Result<ksp_store_lib::StoreSettings> {
let network = ksp_store_lib::RawNetworkId::new("devnet");
let network = match network {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let postgres = ksp_store_lib::PostgresStoreSettings::new(
connection_uri,
ksp_store_lib::PostgresPoolSettings::default(),
ksp_store_lib::PostgresTlsMode::VerifyFull,
ksp_store_lib::PostgresBootstrapSettings::default(),
);
let settings = ksp_store_lib::StoreSettings::with_default_shutdown(
network,
ksp_store_lib::StoreBackendSettings::Postgres(postgres),
);
if let std::result::Result::Err(error) = settings.validate() {
return std::result::Result::Err(error);
}
return std::result::Result::Ok(settings);
}
```
`PostgresStoreSettings` ne fournit aucun getter public de l'URI. Son `Debug` remplace cette valeur par `<redacted>`.
## 4. Ouvrir, sonder et fermer un Store
`Store::open` est async et ne retourne un succès qu'après validation des settings, ouverture du backend compilé et bootstrap requis.
```rust
async fn use_store(settings: ksp_store_lib::StoreSettings) -> ksp_store_lib::Result<()> {
let store = ksp_store_lib::Store::open(settings).await;
let store = match store {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let runtime = store.runtime_snapshot();
let _network = runtime.network();
let _capacity = runtime.pool_capacity();
let _size = runtime.pool_size();
let _available = runtime.pool_available();
let _waiting = runtime.pool_waiting();
let health = store.health().await;
match health.state() {
ksp_store_lib::StoreHealthState::Ready => {}
ksp_store_lib::StoreHealthState::NotReady => {
let _safe_error_code = health.last_error_code();
}
_ => {}
}
return store.close().await;
}
```
`Store::close(self)` consomme l'instance. Une fermeture explicite ne peut donc pas être suivie d'une nouvelle opération via la même valeur.
## 5. Lire une transaction RAW
Importer le trait correspondant suffit pour utiliser la façade :
```rust
use ksp_store_lib::RawTransactionRead;
async fn read_transaction(
store: &ksp_store_lib::Store,
reference: &ksp_store_lib::RawTransactionReference,
) -> ksp_store_lib::Result<std::option::Option<ksp_store_lib::RawTransaction>> {
return store.get_raw_transaction(reference).await;
}
```
Une transaction absente retourne `None`. Une transaction `Purged` retourne également `None` pour le payload canonique ; son tombstone reste accessible via la capability de rétention.
## 6. Paginer les références de transactions
La pagination est keyset et utilise un cursor opaque. Le consumer ne doit pas interpréter ses bytes.
```rust
use ksp_store_lib::RawTransactionRead;
async fn first_transaction_page(
store: &ksp_store_lib::Store,
network: ksp_store_lib::RawNetworkId,
) -> ksp_store_lib::Result<ksp_store_lib::RawPage<ksp_store_lib::RawTransactionReference>> {
let limit = ksp_store_lib::RawPageLimit::new(100);
let limit = match limit {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let slots = ksp_store_lib::RawSlotRange::new(std::option::Option::None, std::option::Option::None);
let slots = match slots {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let query = ksp_store_lib::RawTransactionQuery::new(
network,
slots,
ksp_store_lib::RawSortDirection::Ascending,
ksp_store_lib::RawPageRequest::first(limit),
);
return store.list_raw_transactions(&query).await;
}
```
Pour continuer, recopier le cursor retourné par `RawPage::next_cursor()` dans `RawPageRequest::after`. Le réseau, la direction et les bornes de slots doivent rester identiques à ceux de la query ayant produit le cursor.
## 7. Persister une acquisition canonique
La transaction canonique et son observation initiale forment une seule opération atomique.
```rust
use ksp_store_lib::RawTransactionWrite;
async fn persist_acquisition(
store: &ksp_store_lib::Store,
transaction: ksp_store_lib::RawTransaction,
observation: ksp_store_lib::RawTransactionObservation,
) -> ksp_store_lib::Result<ksp_store_lib::RawAcquisitionWriteOutcome> {
return store
.persist_raw_transaction_acquisition(
transaction,
observation,
ksp_store_lib::RawTransactionAcquisitionMode::Normal,
)
.await;
}
```
Le mode `Normal` respecte un tombstone `Purged`. `ForceRehydrate` doit être choisi explicitement lorsqu'un caller veut restaurer un payload purgé et que l'identité retenue est compatible.
Un contenu divergent sous la même identité produit `ERROR_CODE_RAW_CONFLICT`; Store ne remplace jamais silencieusement le contenu gagnant.
## 8. Lire et ajouter une observation
Une observation supplémentaire référence une transaction canonique déjà durable.
```rust
use ksp_store_lib::RawTransactionObservationRead;
use ksp_store_lib::RawTransactionObservationWrite;
async fn use_observation(
store: &ksp_store_lib::Store,
key: &ksp_store_lib::RawObservationKey,
observation: ksp_store_lib::RawTransactionObservation,
) -> ksp_store_lib::Result<ksp_store_lib::RawObservationWriteOutcome> {
let existing = store.get_raw_transaction_observation(key).await;
if let std::result::Result::Err(error) = existing {
return std::result::Result::Err(error);
}
return store.record_raw_transaction_observation(observation).await;
}
```
`record_raw_transaction_observation` ne crée pas implicitement le canonique. Une référence absente est signalée par `ERROR_CODE_RAW_REFERENCE_NOT_FOUND`.
## 9. Lire la rétention et le tombstone
```rust
use ksp_store_lib::RawTransactionRetentionRead;
async fn read_retention(
store: &ksp_store_lib::Store,
reference: &ksp_store_lib::RawTransactionReference,
) -> ksp_store_lib::Result<std::option::Option<ksp_store_lib::RawRetentionState>> {
let tombstone = store.get_raw_transaction_tombstone(reference).await;
if let std::result::Result::Err(error) = tombstone {
return std::result::Result::Err(error);
}
return store.get_raw_transaction_retention_state(reference).await;
}
```
Le tombstone minimal est utile uniquement lorsque le payload a été purgé ; il ne remplace pas le modèle canonique lorsqu'un payload est encore disponible.
## 10. Appliquer une transition de rétention
La policy qui décide qu'une transition est autorisée appartient au caller. Store applique uniquement la transition demandée de manière atomique.
```rust
use ksp_store_lib::RawTransactionRetentionWrite;
async fn archive_transaction(
store: &ksp_store_lib::Store,
reference: ksp_store_lib::RawTransactionReference,
) -> ksp_store_lib::Result<ksp_store_lib::RawRetentionWriteOutcome> {
let transition = ksp_store_lib::RawTransactionRetentionTransition::try_new(
reference,
ksp_store_lib::RawRetentionState::Full,
ksp_store_lib::RawRetentionState::Archived,
);
let transition = match transition {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
return store.transition_raw_transaction_retention(transition).await;
}
```
Le backend PostgreSQL supporte physiquement `Full -> Archived -> Purged`. Une transition impliquant `Compacted` est rejetée par ce backend tant qu'aucune représentation compactée réelle n'est implémentée.
## 11. Lire et paginer les états account RAW
Importer `RawAccountStateRead` donne accès à la lecture par référence et à la navigation déterministe.
```rust
use ksp_store_lib::RawAccountStateRead;
async fn read_account_state(
store: &ksp_store_lib::Store,
reference: &ksp_store_lib::RawAccountStateReference,
) -> ksp_store_lib::Result<std::option::Option<ksp_store_lib::RawAccountState>> {
return store.get_raw_account_state(reference).await;
}
```
Pour une première page :
```rust
use ksp_store_lib::RawAccountStateRead;
async fn first_account_page(
store: &ksp_store_lib::Store,
network: ksp_store_lib::RawNetworkId,
pubkey: std::option::Option<ksp_store_lib::Pubkey>,
) -> ksp_store_lib::Result<ksp_store_lib::RawPage<ksp_store_lib::RawAccountStateReference>> {
let limit = ksp_store_lib::RawPageLimit::new(100);
let limit = match limit {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let slots = ksp_store_lib::RawSlotRange::new(std::option::Option::None, std::option::Option::None);
let slots = match slots {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let query = ksp_store_lib::RawAccountStateQuery::new(
network,
pubkey,
slots,
ksp_store_lib::RawSortDirection::Ascending,
ksp_store_lib::RawPageRequest::first(limit),
);
return store.list_raw_account_states(&query).await;
}
```
La continuation utilise uniquement `RawPage::next_cursor()` puis `RawPageRequest::after`. Le cursor account est opaque et lié au réseau, au filtre pubkey, à la direction et aux bornes de slots de la query qui l'a produit.
## 12. Persister une acquisition account
L'état canonique et son observation initiale sont une seule opération atomique.
```rust
use ksp_store_lib::RawAccountStateWrite;
async fn persist_account_acquisition(
store: &ksp_store_lib::Store,
state: ksp_store_lib::RawAccountState,
observation: ksp_store_lib::RawAccountObservation,
) -> ksp_store_lib::Result<ksp_store_lib::RawAcquisitionWriteOutcome> {
return store.persist_raw_account_acquisition(state, observation).await;
}
```
`state.reference()` et `observation.account()` doivent désigner exactement le même état et le réseau du `Store`. Une répétition byte-identique est idempotente ; un contenu divergent sous la même identité retourne `ERROR_CODE_RAW_CONFLICT`.
## 13. Lire et ajouter une observation account
```rust
use ksp_store_lib::RawAccountObservationRead;
use ksp_store_lib::RawAccountObservationWrite;
async fn use_account_observation(
store: &ksp_store_lib::Store,
key: &ksp_store_lib::RawObservationKey,
observation: ksp_store_lib::RawAccountObservation,
) -> ksp_store_lib::Result<ksp_store_lib::RawObservationWriteOutcome> {
let existing = store.get_raw_account_observation(key).await;
if let std::result::Result::Err(error) = existing {
return std::result::Result::Err(error);
}
return store.record_raw_account_observation(observation).await;
}
```
`record_raw_account_observation` ne crée jamais implicitement l'état canonique ; une référence absente retourne `ERROR_CODE_RAW_REFERENCE_NOT_FOUND`. `RawObservationKey` ne contient pas de réseau : la lecture par clé reste liée au backend mono-réseau déjà ouvert.
## 14. Diagnostics et erreurs
Les snapshots et erreurs de façade n'exposent ni URI, host, user, database, SQL, handle backend, valeur de bind ni texte d'erreur PostgreSQL.
Les codes Store utiles incluent notamment :
```text
store.wrong_network
store.raw_reference_not_found
store.postgres_read_failed
store.postgres_write_failed
store.postgres_data_invalid
store.postgres_page_limit_unsupported
store.postgres_retention_compaction_unsupported
```
Les conflits et queries invalides utilisent les codes backend-neutral `store_api.raw_conflict` et `store_api.raw_query_invalid`.
## 15. Limites de la façade
La façade ne fournit pas d'accès public au SQL, au pool, aux clients ou transactions PostgreSQL. Elle dispatch les dix capabilities RAW de l'API commune, mais ne fournit aucune capability de rétention, archivage, purge, delete ou compaction account.
La taille de page est une primitive de navigation. Les décisions de batch, priorité, backlog et scheduling appartiennent aux workers/jobs, pas à Store.

View File

@@ -0,0 +1,7 @@
// file: crates/ksp-store-lib/src/constants.rs
// version: 2
//! Store facade-owned constants.
/// Owning tracing target reserved for events emitted by the common Store runtime facade.
pub(crate) const TRACING_TARGET: &str = "ksp-store-lib";

View File

@@ -0,0 +1,44 @@
// file: crates/ksp-store-lib/src/error.rs
// version: 6
/// Error code reserved for operations attempted after a Store backend has entered its closed state.
pub const ERROR_CODE_BACKEND_CLOSED: ksp_store_api::ErrorCode = ksp_store_api::ErrorCode::new("store", "backend_closed");
/// Error code used when a known Store backend was selected but its Cargo feature is not compiled.
pub const ERROR_CODE_BACKEND_NOT_COMPILED: ksp_store_api::ErrorCode = ksp_store_api::ErrorCode::new("store", "backend_not_compiled");
/// Error code used when a compiled Store backend cannot complete its bounded opening lifecycle.
pub const ERROR_CODE_BACKEND_OPEN_FAILED: ksp_store_api::ErrorCode = ksp_store_api::ErrorCode::new("store", "backend_open_failed");
/// Error code used when the PostgreSQL backend rejects or cannot normalize its physical connection configuration.
pub const ERROR_CODE_POSTGRES_CONFIG_INVALID: ksp_store_api::ErrorCode = ksp_store_api::ErrorCode::new("store", "postgres_config_invalid");
/// Error code used when PostgreSQL physical connection establishment fails without exposing remote or credential details.
pub const ERROR_CODE_POSTGRES_CONNECT_FAILED: ksp_store_api::ErrorCode = ksp_store_api::ErrorCode::new("store", "postgres_connect_failed");
/// Error code used when PostgreSQL returns persisted RAW data incompatible with the stable Store contract.
pub const ERROR_CODE_POSTGRES_DATA_INVALID: ksp_store_api::ErrorCode = ksp_store_api::ErrorCode::new("store", "postgres_data_invalid");
/// Error code used when a lightweight PostgreSQL health/readiness probe fails safely.
pub const ERROR_CODE_POSTGRES_HEALTH_FAILED: ksp_store_api::ErrorCode = ksp_store_api::ErrorCode::new("store", "postgres_health_failed");
/// Error code used when PostgreSQL migration/bootstrap execution fails without exposing server text or SQL.
pub const ERROR_CODE_POSTGRES_MIGRATION_FAILED: ksp_store_api::ErrorCode = ksp_store_api::ErrorCode::new("store", "postgres_migration_failed");
/// Error code used when persisted PostgreSQL migration history diverges from the embedded immutable KSP history.
pub const ERROR_CODE_POSTGRES_MIGRATION_MISMATCH: ksp_store_api::ErrorCode = ksp_store_api::ErrorCode::new("store", "postgres_migration_mismatch");
/// Error code used when the requested RAW page size exceeds the exact PostgreSQL LIMIT representation boundary.
pub const ERROR_CODE_POSTGRES_PAGE_LIMIT_UNSUPPORTED: ksp_store_api::ErrorCode = ksp_store_api::ErrorCode::new("store", "postgres_page_limit_unsupported");
/// Error code used when a bounded PostgreSQL pool wait, create or recycle operation reaches its deadline.
pub const ERROR_CODE_POSTGRES_POOL_TIMEOUT: ksp_store_api::ErrorCode = ksp_store_api::ErrorCode::new("store", "postgres_pool_timeout");
/// Error code used when a PostgreSQL RAW read fails without exposing SQL, bind values or server text.
pub const ERROR_CODE_POSTGRES_READ_FAILED: ksp_store_api::ErrorCode = ksp_store_api::ErrorCode::new("store", "postgres_read_failed");
/// Error code used when PostgreSQL cannot represent a requested RAW retention compaction state.
pub const ERROR_CODE_POSTGRES_RETENTION_COMPACTION_UNSUPPORTED: ksp_store_api::ErrorCode =
ksp_store_api::ErrorCode::new("store", "postgres_retention_compaction_unsupported");
/// Error code used when PostgreSQL history contains a migration newer than this Store runtime understands.
pub const ERROR_CODE_POSTGRES_SCHEMA_NEWER: ksp_store_api::ErrorCode = ksp_store_api::ErrorCode::new("store", "postgres_schema_newer");
/// Error code used when verified PostgreSQL TLS setup or negotiation cannot be completed safely.
pub const ERROR_CODE_POSTGRES_TLS_FAILED: ksp_store_api::ErrorCode = ksp_store_api::ErrorCode::new("store", "postgres_tls_failed");
/// Error code used when a PostgreSQL RAW write fails without exposing SQL, bind values or server text.
pub const ERROR_CODE_POSTGRES_WRITE_FAILED: ksp_store_api::ErrorCode = ksp_store_api::ErrorCode::new("store", "postgres_write_failed");
/// Error code used when a RAW write requires a canonical reference that is not durable.
pub const ERROR_CODE_RAW_REFERENCE_NOT_FOUND: ksp_store_api::ErrorCode = ksp_store_api::ErrorCode::new("store", "raw_reference_not_found");
/// Error code used when backend-neutral Store settings violate runtime bounds or invariants.
pub const ERROR_CODE_SETTINGS_INVALID: ksp_store_api::ErrorCode = ksp_store_api::ErrorCode::new("store", "settings_invalid");
/// Error code used when a Store cannot complete its explicit shutdown inside the configured bound.
pub const ERROR_CODE_SHUTDOWN_TIMEOUT: ksp_store_api::ErrorCode = ksp_store_api::ErrorCode::new("store", "shutdown_timeout");
/// Error code used when a network-scoped Store operation targets a network different from the opened Store binding.
pub const ERROR_CODE_WRONG_NETWORK: ksp_store_api::ErrorCode = ksp_store_api::ErrorCode::new("store", "wrong_network");

View File

@@ -0,0 +1,132 @@
// file: crates/ksp-store-lib/src/health.rs
// version: 1
/// Portable Store health state independent from the selected physical backend.
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
#[non_exhaustive]
pub enum StoreHealthState {
/// The selected Store backend answered the bounded readiness probe and its migration foundation is current.
Ready,
/// The Store instance exists but its latest bounded readiness probe could not prove readiness.
NotReady,
}
/// Safe synchronous Store runtime snapshot without performing backend I/O.
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct StoreRuntimeSnapshot {
backend_kind: crate::StoreBackendKind,
network: ksp_store_api::RawNetworkId,
pool_available: u32,
pool_capacity: u32,
pool_size: u32,
pool_waiting: u32,
}
impl StoreRuntimeSnapshot {
/// Creates one portable runtime projection from backend-owned safe counters.
#[must_use]
pub(crate) fn new(
backend_kind: crate::StoreBackendKind,
network: ksp_store_api::RawNetworkId,
pool_capacity: u32,
pool_size: u32,
pool_available: u32,
pool_waiting: u32,
) -> Self {
return Self { backend_kind, network, pool_available, pool_capacity, pool_size, pool_waiting };
}
/// Returns the selected backend identity.
#[must_use]
pub const fn backend_kind(&self) -> crate::StoreBackendKind {
return self.backend_kind;
}
/// Returns the one logical network bound to this Store instance.
#[must_use]
pub const fn network(&self) -> &ksp_store_api::RawNetworkId {
return &self.network;
}
/// Returns the number of currently available pooled backend objects.
#[must_use]
pub const fn pool_available(&self) -> u32 {
return self.pool_available;
}
/// Returns the configured maximum pooled backend object count.
#[must_use]
pub const fn pool_capacity(&self) -> u32 {
return self.pool_capacity;
}
/// Returns the current pooled backend object count.
#[must_use]
pub const fn pool_size(&self) -> u32 {
return self.pool_size;
}
/// Returns the number of tasks currently waiting for a pooled backend object.
#[must_use]
pub const fn pool_waiting(&self) -> u32 {
return self.pool_waiting;
}
}
/// Portable Store readiness projection containing only safe runtime and migration diagnostics.
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct StoreHealthSnapshot {
last_error_code: std::option::Option<ksp_store_api::ErrorCode>,
migration_version: std::option::Option<u64>,
pending_migration_count: u32,
runtime: StoreRuntimeSnapshot,
state: StoreHealthState,
}
impl StoreHealthSnapshot {
/// Creates one safe health projection from already classified backend diagnostics.
#[must_use]
pub(crate) fn new(
state: StoreHealthState,
runtime: StoreRuntimeSnapshot,
migration_version: std::option::Option<u64>,
pending_migration_count: u32,
last_error_code: std::option::Option<ksp_store_api::ErrorCode>,
) -> Self {
return Self { last_error_code, migration_version, pending_migration_count, runtime, state };
}
/// Returns the latest safe error code when readiness could not be proven.
#[must_use]
pub const fn last_error_code(&self) -> std::option::Option<ksp_store_api::ErrorCode> {
return self.last_error_code;
}
/// Returns the migration version observed by the readiness probe when available.
#[must_use]
pub const fn migration_version(&self) -> std::option::Option<u64> {
return self.migration_version;
}
/// Returns the number of embedded migrations newer than the observed applied version.
#[must_use]
pub const fn pending_migration_count(&self) -> u32 {
return self.pending_migration_count;
}
/// Returns the safe synchronous runtime projection captured for this health probe.
#[must_use]
pub const fn runtime(&self) -> &StoreRuntimeSnapshot {
return &self.runtime;
}
/// Returns whether this probe proved the Store ready.
#[must_use]
pub const fn state(&self) -> StoreHealthState {
return self.state;
}
}
#[cfg(test)]
#[path = "../unit_tests/health.rs"]
mod tests;

View File

@@ -0,0 +1,214 @@
// file: crates/ksp-store-lib/src/lib.rs
// version: 9
#![warn(missing_docs)]
#![deny(unreachable_pub)]
#![forbid(unsafe_code)]
//! Common backend-neutral Store runtime facade for KSP.
//!
//! The runtime facade owns backend selection, lifecycle, safe diagnostics and
//! backend-neutral capability dispatch. `0.3.3-pre.008` completed the six PostgreSQL
//! `RawTransaction` capabilities. `0.3.4-pre.008` adds the four `RawAccount*` capabilities
//! on both the physical backend and this common facade, completing the RAW inventory at
//! ten capabilities without exposing physical types.
//!
//! The default `postgres` feature compiles the official PostgreSQL backend as
//! an optional implementation dependency. No backend implementation type is
//! part of this crate's public surface.
mod constants;
mod error;
mod health;
mod settings;
mod store;
/// Error code reserved for operations attempted after a Store backend is closed.
pub use self::error::ERROR_CODE_BACKEND_CLOSED;
/// Error code used when a known Store backend is selected without its compiled feature.
pub use self::error::ERROR_CODE_BACKEND_NOT_COMPILED;
/// Error code used when a compiled Store backend cannot complete opening.
pub use self::error::ERROR_CODE_BACKEND_OPEN_FAILED;
/// Error code used when PostgreSQL physical configuration is malformed or unsupported.
pub use self::error::ERROR_CODE_POSTGRES_CONFIG_INVALID;
/// Error code used when PostgreSQL physical connection establishment fails.
pub use self::error::ERROR_CODE_POSTGRES_CONNECT_FAILED;
/// Error code used when PostgreSQL returns persisted RAW data incompatible with the Store contract.
pub use self::error::ERROR_CODE_POSTGRES_DATA_INVALID;
/// Error code used when a lightweight PostgreSQL health/readiness probe fails safely.
pub use self::error::ERROR_CODE_POSTGRES_HEALTH_FAILED;
/// Error code used when PostgreSQL migration/bootstrap execution fails safely.
pub use self::error::ERROR_CODE_POSTGRES_MIGRATION_FAILED;
/// Error code used when PostgreSQL migration history diverges from the embedded immutable KSP history.
pub use self::error::ERROR_CODE_POSTGRES_MIGRATION_MISMATCH;
/// Error code used when a requested RAW page size exceeds PostgreSQL's exact physical LIMIT boundary.
pub use self::error::ERROR_CODE_POSTGRES_PAGE_LIMIT_UNSUPPORTED;
/// Error code used when a bounded PostgreSQL pool operation reaches its deadline.
pub use self::error::ERROR_CODE_POSTGRES_POOL_TIMEOUT;
/// Error code used when a PostgreSQL RAW read statement fails safely.
pub use self::error::ERROR_CODE_POSTGRES_READ_FAILED;
/// Error code used when PostgreSQL cannot represent a requested RAW retention compaction state.
pub use self::error::ERROR_CODE_POSTGRES_RETENTION_COMPACTION_UNSUPPORTED;
/// Error code used when PostgreSQL schema history is newer than this Store runtime.
pub use self::error::ERROR_CODE_POSTGRES_SCHEMA_NEWER;
/// Error code used when PostgreSQL verified TLS setup or negotiation fails.
pub use self::error::ERROR_CODE_POSTGRES_TLS_FAILED;
/// Error code used when a PostgreSQL RAW write statement or transaction fails safely.
pub use self::error::ERROR_CODE_POSTGRES_WRITE_FAILED;
/// Error code used when a RAW write requires a canonical reference that is not durable.
pub use self::error::ERROR_CODE_RAW_REFERENCE_NOT_FOUND;
/// Error code used when Store settings violate backend-neutral bounds or invariants.
pub use self::error::ERROR_CODE_SETTINGS_INVALID;
/// Error code used when explicit Store shutdown exceeds its configured deadline.
pub use self::error::ERROR_CODE_SHUTDOWN_TIMEOUT;
/// Error code used when a network-scoped operation targets a network different from the Store binding.
pub use self::error::ERROR_CODE_WRONG_NETWORK;
/// Portable Store health/readiness projection containing only safe diagnostics.
pub use self::health::StoreHealthSnapshot;
/// Portable Store health state independent from physical backend types.
pub use self::health::StoreHealthState;
/// Safe synchronous Store runtime snapshot containing backend-neutral pool counters.
pub use self::health::StoreRuntimeSnapshot;
/// Bounded PostgreSQL bootstrap and migration settings owned by the Store facade.
pub use self::settings::PostgresBootstrapSettings;
/// Bounded PostgreSQL connection-pool settings owned by the Store facade.
pub use self::settings::PostgresPoolSettings;
/// PostgreSQL settings owned by the Store facade without exposing backend implementation types.
pub use self::settings::PostgresStoreSettings;
/// TLS policy accepted by the backend-neutral PostgreSQL settings surface.
pub use self::settings::PostgresTlsMode;
/// Backend identity understood independently from compiled Cargo features.
pub use self::settings::StoreBackendKind;
/// Backend-specific settings selected through the common Store facade.
pub use self::settings::StoreBackendSettings;
/// Complete backend-neutral settings consumed by the common Store runtime facade.
pub use self::settings::StoreSettings;
/// Opaque common Store runtime facade with consuming async shutdown.
pub use self::store::Store;
/// Error code used when a RAW write collides with divergent content for the same logical identity.
pub use ksp_store_api::ERROR_CODE_RAW_CONFLICT;
/// Error code used when a RAW Store model violates one of its backend-agnostic invariants.
pub use ksp_store_api::ERROR_CODE_RAW_MODEL_INVALID;
/// Error code used when a KSP-owned RAW persistence payload violates its format or admission contract.
pub use ksp_store_api::ERROR_CODE_RAW_PAYLOAD_INVALID;
/// Error code used when acquisition provenance is malformed, unsafe or internally inconsistent.
pub use ksp_store_api::ERROR_CODE_RAW_PROVENANCE_INVALID;
/// Error code used when one RAW query or cursor violates backend-agnostic query invariants.
pub use ksp_store_api::ERROR_CODE_RAW_QUERY_INVALID;
/// Error code used when a RAW retention transition violates the logical lifecycle contract.
pub use ksp_store_api::ERROR_CODE_RAW_RETENTION_INVALID;
/// Common KSP error type used by Store-facing contracts.
pub use ksp_store_api::Error;
/// Stable structured code identifying a KSP error category and condition.
pub use ksp_store_api::ErrorCode;
/// Structured contextual field attached to a KSP error.
pub use ksp_store_api::ErrorContext;
/// Maximum complete RAW account-data length admitted by the Store API.
pub use ksp_store_api::MAX_RAW_ACCOUNT_DATA_BYTES;
/// Maximum UTF-8 byte length accepted for one safe logical RAW/provenance code.
pub use ksp_store_api::MAX_RAW_CODE_BYTES;
/// Maximum opaque query cursor length admitted by the Store API.
pub use ksp_store_api::MAX_RAW_PAGE_CURSOR_BYTES;
/// Maximum KSP-owned canonical RAW payload admitted by the Store API.
pub use ksp_store_api::MAX_RAW_PAYLOAD_BYTES;
/// Maximum source-wire payload size recorded as acquisition metadata.
pub use ksp_store_api::MAX_RAW_SOURCE_PAYLOAD_BYTES;
/// Maximum supported Unix millisecond timestamp.
pub use ksp_store_api::MAX_RAW_UNIX_MILLIS;
/// Canonical Solana account address primitive shared by persistent models.
pub use ksp_store_api::Pubkey;
/// Persistable acquisition observation linked to one complete canonical RAW account state.
pub use ksp_store_api::RawAccountObservation;
/// Read capability for persisted RAW account-state observations.
pub use ksp_store_api::RawAccountObservationRead;
/// Write capability for additional observations of already persisted RAW account states.
pub use ksp_store_api::RawAccountObservationWrite;
/// Canonical complete N1 RAW account state independent from acquisition transport.
pub use ksp_store_api::RawAccountState;
/// Backend-independent list query for complete canonical RAW account states.
pub use ksp_store_api::RawAccountStateQuery;
/// Read capability for complete canonical RAW account states.
pub use ksp_store_api::RawAccountStateRead;
/// Durable backend-independent identity of one canonical RAW account state.
pub use ksp_store_api::RawAccountStateReference;
/// Write capability for complete canonical RAW account-state acquisitions.
pub use ksp_store_api::RawAccountStateWrite;
/// Origin category describing why one acquisition was performed.
pub use ksp_store_api::RawAcquisitionOrigin;
/// Safe source-independent acquisition provenance attached to one persisted observation.
pub use ksp_store_api::RawAcquisitionProvenance;
/// Combined outcome of one atomic canonical RAW entity plus observation acquisition.
pub use ksp_store_api::RawAcquisitionWriteOutcome;
/// Fixed-size digest identifying canonical or source bytes without retaining them.
pub use ksp_store_api::RawContentHash;
/// Outcome for one canonical RAW entity in an idempotent persistence operation.
pub use ksp_store_api::RawEntityWriteOutcome;
/// Bounded identifier of one KSP-owned source-independent RAW persistence format.
pub use ksp_store_api::RawFormatId;
/// Bounded logical network/cluster identifier used in backend-independent Store identities.
pub use ksp_store_api::RawNetworkId;
/// Stable deterministic idempotence key for one persisted acquisition observation.
pub use ksp_store_api::RawObservationKey;
/// Outcome for one deterministic acquisition observation write.
pub use ksp_store_api::RawObservationWriteOutcome;
/// One deterministic page of backend-independent Store results.
pub use ksp_store_api::RawPage;
/// Opaque backend-owned cursor returned by one deterministic Store query.
pub use ksp_store_api::RawPageCursor;
/// Caller-requested page size without an arbitrary KSP policy ceiling.
pub use ksp_store_api::RawPageLimit;
/// Opaque-cursor page request used by backend-independent list operations.
pub use ksp_store_api::RawPageRequest;
/// Bounded source-independent KSP RAW persistence payload.
pub use ksp_store_api::RawPayload;
/// Bounded logical code used by acquisition provenance fields.
pub use ksp_store_api::RawProvenanceCode;
/// Logical availability state of one canonical RAW payload.
pub use ksp_store_api::RawRetentionState;
/// Outcome of one atomic RAW retention transition.
pub use ksp_store_api::RawRetentionWriteOutcome;
/// Optional inclusive Solana slot bounds for one Store query.
pub use ksp_store_api::RawSlotRange;
/// Deterministic traversal direction for Store list queries.
pub use ksp_store_api::RawSortDirection;
/// Bounded UTC timestamp represented as whole milliseconds since Unix epoch.
pub use ksp_store_api::RawTimestamp;
/// Canonical source-independent N1 RAW transaction persisted by Store backends.
pub use ksp_store_api::RawTransaction;
/// Explicit write mode for canonical RAW transaction acquisitions.
pub use ksp_store_api::RawTransactionAcquisitionMode;
/// Persistable acquisition observation linked to one canonical RAW transaction.
pub use ksp_store_api::RawTransactionObservation;
/// Read capability for persisted RAW transaction observations.
pub use ksp_store_api::RawTransactionObservationRead;
/// Write capability for additional observations of already persisted RAW transactions.
pub use ksp_store_api::RawTransactionObservationWrite;
/// Backend-independent list query for canonical RAW transactions.
pub use ksp_store_api::RawTransactionQuery;
/// Read capability for canonical RAW transactions.
pub use ksp_store_api::RawTransactionRead;
/// Durable backend-independent identity of one canonical RAW transaction.
pub use ksp_store_api::RawTransactionReference;
/// Read capability for canonical RAW transaction retention metadata.
pub use ksp_store_api::RawTransactionRetentionRead;
/// Requested compare-and-transition operation for one RAW transaction retention state.
pub use ksp_store_api::RawTransactionRetentionTransition;
/// Write capability for policy-authorized RAW transaction retention transitions.
pub use ksp_store_api::RawTransactionRetentionWrite;
/// Canonical 64-byte Solana transaction signature used by Store identities.
pub use ksp_store_api::RawTransactionSignature;
/// Minimal durable identity retained after a canonical RAW transaction payload is purged.
pub use ksp_store_api::RawTransactionTombstone;
/// Write capability for canonical RAW transaction acquisitions.
pub use ksp_store_api::RawTransactionWrite;
/// Common KSP result alias using [`Error`].
pub use ksp_store_api::Result;
/// Boxed async operation returned by object-safe Store capability contracts.
pub use ksp_store_api::StoreApiFuture;
/// Crate-owned tracing target reserved for Store runtime behavior.
pub(crate) use self::constants::TRACING_TARGET;
// Keep the mandatory crate-owned tracing target part of the compiled scaffold
// without inventing runtime logging before the first behavioral log site.
const _: &str = crate::TRACING_TARGET;

View File

@@ -0,0 +1,423 @@
// file: crates/ksp-store-lib/src/settings.rs
// version: 4
const DEFAULT_CONNECT_TIMEOUT_MS: u64 = 10_000;
const DEFAULT_MAX_CONNECTIONS: u32 = 8;
const DEFAULT_MIGRATION_LOCK_TIMEOUT_MS: u64 = 10_000;
const DEFAULT_MIGRATION_TIMEOUT_MS: u64 = 30_000;
const DEFAULT_POOL_CREATE_TIMEOUT_MS: u64 = 10_000;
const DEFAULT_POOL_RECYCLE_TIMEOUT_MS: u64 = 5_000;
const DEFAULT_POOL_WAIT_TIMEOUT_MS: u64 = 5_000;
const DEFAULT_SHUTDOWN_TIMEOUT_MS: u64 = 5_000;
const MAX_CONNECTIONS: u32 = 64;
const MAX_CONNECT_TIMEOUT_MS: u64 = 60_000;
const MAX_MIGRATION_LOCK_TIMEOUT_MS: u64 = 120_000;
const MAX_MIGRATION_TIMEOUT_MS: u64 = 300_000;
const MAX_POOL_CREATE_TIMEOUT_MS: u64 = 60_000;
const MAX_POOL_RECYCLE_TIMEOUT_MS: u64 = 60_000;
const MAX_POOL_WAIT_TIMEOUT_MS: u64 = 60_000;
const MAX_SHUTDOWN_TIMEOUT_MS: u64 = 30_000;
const MIN_CONNECTIONS: u32 = 1;
const MIN_CONNECT_TIMEOUT_MS: u64 = 100;
const MIN_MIGRATION_LOCK_TIMEOUT_MS: u64 = 100;
const MIN_MIGRATION_TIMEOUT_MS: u64 = 1_000;
const MIN_POOL_CREATE_TIMEOUT_MS: u64 = 100;
const MIN_POOL_RECYCLE_TIMEOUT_MS: u64 = 100;
const MIN_POOL_WAIT_TIMEOUT_MS: u64 = 100;
const MIN_SHUTDOWN_TIMEOUT_MS: u64 = 100;
/// Backend identity understood by the common Store runtime independently from compiled Cargo features.
#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)]
#[non_exhaustive]
pub enum StoreBackendKind {
/// Official PostgreSQL Store backend.
Postgres,
}
impl StoreBackendKind {
/// Returns the stable safe backend code used in diagnostics and configuration mapping.
#[must_use]
pub const fn code(&self) -> &'static str {
return match self {
Self::Postgres => "postgres",
};
}
}
/// TLS policy accepted by the backend-neutral PostgreSQL settings surface.
#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)]
#[non_exhaustive]
pub enum PostgresTlsMode {
/// Connect without TLS.
Disabled,
/// Require TLS and verify both the certificate chain and requested server identity.
VerifyFull,
}
/// Bounded PostgreSQL connection-pool settings owned by the Store facade.
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub struct PostgresPoolSettings {
connect_timeout: std::time::Duration,
create_timeout: std::time::Duration,
max_connections: u32,
recycle_timeout: std::time::Duration,
wait_timeout: std::time::Duration,
}
impl PostgresPoolSettings {
/// Creates explicit PostgreSQL pool bounds without performing any I/O.
#[must_use]
pub const fn new(
max_connections: u32,
connect_timeout: std::time::Duration,
wait_timeout: std::time::Duration,
create_timeout: std::time::Duration,
recycle_timeout: std::time::Duration,
) -> Self {
return Self { connect_timeout, create_timeout, max_connections, recycle_timeout, wait_timeout };
}
/// Returns the timeout for establishing one physical PostgreSQL connection.
#[must_use]
pub const fn connect_timeout(&self) -> std::time::Duration {
return self.connect_timeout;
}
/// Returns the timeout for creating one pooled PostgreSQL object.
#[must_use]
pub const fn create_timeout(&self) -> std::time::Duration {
return self.create_timeout;
}
/// Returns the maximum number of physical PostgreSQL connections owned by the pool.
#[must_use]
pub const fn max_connections(&self) -> u32 {
return self.max_connections;
}
/// Returns the timeout for recycling one pooled PostgreSQL object.
#[must_use]
pub const fn recycle_timeout(&self) -> std::time::Duration {
return self.recycle_timeout;
}
/// Returns the maximum time one acquisition can wait for pool capacity.
#[must_use]
pub const fn wait_timeout(&self) -> std::time::Duration {
return self.wait_timeout;
}
/// Validates all pool bounds without opening a connection.
pub fn validate(&self) -> ksp_store_api::Result<()> {
if self.max_connections < MIN_CONNECTIONS || self.max_connections > MAX_CONNECTIONS {
return std::result::Result::Err(
ksp_store_api::Error::new(crate::ERROR_CODE_SETTINGS_INVALID, "PostgreSQL pool connection bound is invalid")
.with_context("field", "postgres.pool.max_connections")
.with_context("minimum", MIN_CONNECTIONS.to_string())
.with_context("maximum", MAX_CONNECTIONS.to_string()),
);
}
let connect_validation = validate_duration("postgres.pool.connect_timeout", self.connect_timeout, MIN_CONNECT_TIMEOUT_MS, MAX_CONNECT_TIMEOUT_MS);
if let std::result::Result::Err(error) = connect_validation {
return std::result::Result::Err(error);
}
let wait_validation = validate_duration("postgres.pool.wait_timeout", self.wait_timeout, MIN_POOL_WAIT_TIMEOUT_MS, MAX_POOL_WAIT_TIMEOUT_MS);
if let std::result::Result::Err(error) = wait_validation {
return std::result::Result::Err(error);
}
let create_validation = validate_duration("postgres.pool.create_timeout", self.create_timeout, MIN_POOL_CREATE_TIMEOUT_MS, MAX_POOL_CREATE_TIMEOUT_MS);
if let std::result::Result::Err(error) = create_validation {
return std::result::Result::Err(error);
}
let recycle_validation =
validate_duration("postgres.pool.recycle_timeout", self.recycle_timeout, MIN_POOL_RECYCLE_TIMEOUT_MS, MAX_POOL_RECYCLE_TIMEOUT_MS);
if let std::result::Result::Err(error) = recycle_validation {
return std::result::Result::Err(error);
}
return std::result::Result::Ok(());
}
}
impl std::default::Default for PostgresPoolSettings {
fn default() -> Self {
return Self::new(
DEFAULT_MAX_CONNECTIONS,
std::time::Duration::from_millis(DEFAULT_CONNECT_TIMEOUT_MS),
std::time::Duration::from_millis(DEFAULT_POOL_WAIT_TIMEOUT_MS),
std::time::Duration::from_millis(DEFAULT_POOL_CREATE_TIMEOUT_MS),
std::time::Duration::from_millis(DEFAULT_POOL_RECYCLE_TIMEOUT_MS),
);
}
}
/// Bounded PostgreSQL bootstrap settings owned by the Store facade.
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub struct PostgresBootstrapSettings {
migration_lock_timeout: std::time::Duration,
migration_timeout: std::time::Duration,
schema_autocreate: bool,
schema_autoupdate: bool,
}
impl PostgresBootstrapSettings {
/// Creates bootstrap settings using the legacy single migration switch for source compatibility.
///
/// The supplied value is mapped to both schema auto-creation and schema auto-update. New code should prefer
/// [`Self::with_schema_policy`] when these policies need to differ.
#[must_use]
pub const fn new(auto_migrate: bool, migration_timeout: std::time::Duration, migration_lock_timeout: std::time::Duration) -> Self {
return Self::with_schema_policy(auto_migrate, auto_migrate, migration_timeout, migration_lock_timeout);
}
/// Creates explicit schema creation/update policy and migration deadlines.
#[must_use]
pub const fn with_schema_policy(
schema_autocreate: bool,
schema_autoupdate: bool,
migration_timeout: std::time::Duration,
migration_lock_timeout: std::time::Duration,
) -> Self {
return Self { migration_lock_timeout, migration_timeout, schema_autocreate, schema_autoupdate };
}
/// Returns the legacy pending-migration switch, mapped to the schema auto-update policy.
#[must_use]
pub const fn auto_migrate(&self) -> bool {
return self.schema_autoupdate;
}
/// Returns whether an absent KSP-managed schema may be created or adopted during Store opening.
#[must_use]
pub const fn schema_autocreate(&self) -> bool {
return self.schema_autocreate;
}
/// Returns whether pending migrations and safe additive schema repairs may be applied during Store opening.
#[must_use]
pub const fn schema_autoupdate(&self) -> bool {
return self.schema_autoupdate;
}
/// Returns the bounded wait allowed for the private PostgreSQL migration lock.
#[must_use]
pub const fn migration_lock_timeout(&self) -> std::time::Duration {
return self.migration_lock_timeout;
}
/// Returns the bounded duration allowed for one migration/bootstrap run.
#[must_use]
pub const fn migration_timeout(&self) -> std::time::Duration {
return self.migration_timeout;
}
/// Validates bootstrap and migration deadlines without contacting PostgreSQL.
pub fn validate(&self) -> ksp_store_api::Result<()> {
let migration_validation =
validate_duration("postgres.bootstrap.migration_timeout", self.migration_timeout, MIN_MIGRATION_TIMEOUT_MS, MAX_MIGRATION_TIMEOUT_MS);
if let std::result::Result::Err(error) = migration_validation {
return std::result::Result::Err(error);
}
let lock_validation = validate_duration(
"postgres.bootstrap.migration_lock_timeout",
self.migration_lock_timeout,
MIN_MIGRATION_LOCK_TIMEOUT_MS,
MAX_MIGRATION_LOCK_TIMEOUT_MS,
);
if let std::result::Result::Err(error) = lock_validation {
return std::result::Result::Err(error);
}
return std::result::Result::Ok(());
}
}
impl std::default::Default for PostgresBootstrapSettings {
fn default() -> Self {
return Self::with_schema_policy(
true,
true,
std::time::Duration::from_millis(DEFAULT_MIGRATION_TIMEOUT_MS),
std::time::Duration::from_millis(DEFAULT_MIGRATION_LOCK_TIMEOUT_MS),
);
}
}
/// PostgreSQL settings owned by the Store facade and independent from Config or backend implementation types.
pub struct PostgresStoreSettings {
bootstrap: PostgresBootstrapSettings,
connection_uri: std::string::String,
pool: PostgresPoolSettings,
tls_mode: PostgresTlsMode,
}
impl PostgresStoreSettings {
/// Creates PostgreSQL Store settings from an explicitly supplied sensitive connection URI and typed runtime bounds.
#[must_use]
pub fn new(
connection_uri: impl std::convert::Into<std::string::String>,
pool: PostgresPoolSettings,
tls_mode: PostgresTlsMode,
bootstrap: PostgresBootstrapSettings,
) -> Self {
return Self { bootstrap, connection_uri: connection_uri.into(), pool, tls_mode };
}
/// Returns the PostgreSQL bootstrap settings without exposing the sensitive connection URI.
#[must_use]
pub const fn bootstrap(&self) -> PostgresBootstrapSettings {
return self.bootstrap;
}
/// Returns the sensitive PostgreSQL connection URI only to the compiled backend bridge.
#[cfg(feature = "postgres")]
#[must_use]
pub(crate) fn connection_uri(&self) -> &str {
return self.connection_uri.as_str();
}
/// Returns the PostgreSQL pool settings without exposing the sensitive connection URI.
#[must_use]
pub const fn pool(&self) -> PostgresPoolSettings {
return self.pool;
}
/// Returns the selected PostgreSQL TLS policy without exposing the sensitive connection URI.
#[must_use]
pub const fn tls_mode(&self) -> PostgresTlsMode {
return self.tls_mode;
}
/// Validates backend-neutral PostgreSQL settings without parsing the URI or performing I/O.
pub fn validate(&self) -> ksp_store_api::Result<()> {
if self.connection_uri.is_empty() {
return std::result::Result::Err(
ksp_store_api::Error::new(crate::ERROR_CODE_SETTINGS_INVALID, "PostgreSQL connection URI is required")
.with_context("field", "postgres.connection_uri"),
);
}
let pool_validation = self.pool.validate();
if let std::result::Result::Err(error) = pool_validation {
return std::result::Result::Err(error);
}
let bootstrap_validation = self.bootstrap.validate();
if let std::result::Result::Err(error) = bootstrap_validation {
return std::result::Result::Err(error);
}
return std::result::Result::Ok(());
}
}
impl std::fmt::Debug for PostgresStoreSettings {
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
return formatter
.debug_struct("PostgresStoreSettings")
.field("connection_uri", &"<redacted>")
.field("pool", &self.pool)
.field("tls_mode", &self.tls_mode)
.field("bootstrap", &self.bootstrap)
.finish();
}
}
/// Backend-specific settings selected through the common Store facade.
#[derive(Debug)]
#[non_exhaustive]
pub enum StoreBackendSettings {
/// Settings for the known PostgreSQL backend, whether or not its Cargo feature is compiled.
Postgres(PostgresStoreSettings),
}
impl StoreBackendSettings {
/// Returns the stable backend identity represented by these settings.
#[must_use]
pub const fn kind(&self) -> StoreBackendKind {
return match self {
Self::Postgres(_) => StoreBackendKind::Postgres,
};
}
/// Validates backend-specific settings without performing I/O.
pub fn validate(&self) -> ksp_store_api::Result<()> {
return match self {
Self::Postgres(settings) => settings.validate(),
};
}
}
/// Complete backend-neutral settings consumed by the common Store runtime facade.
#[derive(Debug)]
pub struct StoreSettings {
backend: StoreBackendSettings,
network: ksp_store_api::RawNetworkId,
shutdown_timeout: std::time::Duration,
}
impl StoreSettings {
/// Creates complete Store runtime settings for one explicit logical network, backend and shutdown bound.
#[must_use]
pub fn new(network: ksp_store_api::RawNetworkId, backend: StoreBackendSettings, shutdown_timeout: std::time::Duration) -> Self {
return Self { backend, network, shutdown_timeout };
}
/// Returns the selected backend settings.
#[must_use]
pub const fn backend(&self) -> &StoreBackendSettings {
return &self.backend;
}
/// Returns the selected stable backend identity.
#[must_use]
pub const fn backend_kind(&self) -> StoreBackendKind {
return self.backend.kind();
}
/// Returns the single logical network bound to this Store instance.
#[must_use]
pub const fn network(&self) -> &ksp_store_api::RawNetworkId {
return &self.network;
}
/// Returns the maximum duration allowed for explicit Store shutdown.
#[must_use]
pub const fn shutdown_timeout(&self) -> std::time::Duration {
return self.shutdown_timeout;
}
/// Validates all backend-neutral Store settings before any backend I/O can start.
pub fn validate(&self) -> ksp_store_api::Result<()> {
let shutdown_validation = validate_duration("shutdown_timeout", self.shutdown_timeout, MIN_SHUTDOWN_TIMEOUT_MS, MAX_SHUTDOWN_TIMEOUT_MS);
if let std::result::Result::Err(error) = shutdown_validation {
return std::result::Result::Err(error);
}
let backend_validation = self.backend.validate();
if let std::result::Result::Err(error) = backend_validation {
return std::result::Result::Err(error);
}
return std::result::Result::Ok(());
}
}
impl StoreSettings {
/// Creates settings using the common default shutdown bound while keeping network and backend construction explicit.
#[must_use]
pub fn with_default_shutdown(network: ksp_store_api::RawNetworkId, backend: StoreBackendSettings) -> Self {
return Self::new(network, backend, std::time::Duration::from_millis(DEFAULT_SHUTDOWN_TIMEOUT_MS));
}
}
fn validate_duration(field: &'static str, value: std::time::Duration, minimum_ms: u64, maximum_ms: u64) -> ksp_store_api::Result<()> {
let minimum = std::time::Duration::from_millis(minimum_ms);
let maximum = std::time::Duration::from_millis(maximum_ms);
if value < minimum || value > maximum {
return std::result::Result::Err(
ksp_store_api::Error::new(crate::ERROR_CODE_SETTINGS_INVALID, "Store runtime duration is outside the supported resource bound")
.with_context("field", field)
.with_context("minimum_ms", minimum_ms.to_string())
.with_context("maximum_ms", maximum_ms.to_string()),
);
}
return std::result::Result::Ok(());
}
#[cfg(test)]
#[path = "../unit_tests/settings.rs"]
mod tests;

View File

@@ -0,0 +1,651 @@
// file: crates/ksp-store-lib/src/store.rs
// version: 7
/// Opaque common Store runtime facade.
///
/// A successful value is returned only after the selected compiled backend has completed its bounded physical opening path. PostgreSQL pool, client, TLS and
/// driver types remain private to the backend crate.
pub struct Store {
backend_kind: crate::StoreBackendKind,
network: ksp_store_api::RawNetworkId,
#[cfg(feature = "postgres")]
runtime: StoreRuntime,
shutdown_timeout: std::time::Duration,
}
impl Store {
/// Validates settings, selects the requested backend and opens one ready Store instance for exactly one logical network.
///
/// A known backend whose Cargo feature is absent is rejected before any I/O. A successful PostgreSQL result proves that one physical pooled connection has
/// been established under the typed TLS and timeout policy.
pub async fn open(settings: crate::StoreSettings) -> ksp_store_api::Result<Self> {
let validation = settings.validate();
if let std::result::Result::Err(error) = validation {
return std::result::Result::Err(error);
}
let backend_kind = settings.backend_kind();
let network = settings.network().clone();
let shutdown_timeout = settings.shutdown_timeout();
ksp_logging_lib::debug!(
target: crate::TRACING_TARGET,
backend = backend_kind.code(),
network = network.as_str(),
"opening Store runtime"
);
return match settings.backend() {
crate::StoreBackendSettings::Postgres(postgres) => open_postgres(backend_kind, network, shutdown_timeout, postgres).await,
};
}
/// Returns a safe synchronous runtime snapshot without performing backend I/O.
#[must_use]
pub fn runtime_snapshot(&self) -> crate::StoreRuntimeSnapshot {
#[cfg(feature = "postgres")]
{
return match &self.runtime {
StoreRuntime::Postgres(backend) => map_postgres_runtime_snapshot(backend.runtime_snapshot(), self.backend_kind, self.network.clone()),
};
}
#[cfg(not(feature = "postgres"))]
{
return crate::StoreRuntimeSnapshot::new(self.backend_kind, self.network.clone(), 0, 0, 0, 0);
}
}
/// Runs the selected backend's lightweight bounded readiness probe and returns only portable redacted diagnostics.
pub async fn health(&self) -> crate::StoreHealthSnapshot {
#[cfg(feature = "postgres")]
{
return match &self.runtime {
StoreRuntime::Postgres(backend) => {
let snapshot = backend.health().await;
map_postgres_health_snapshot(snapshot, self.backend_kind, self.network.clone())
},
};
}
#[cfg(not(feature = "postgres"))]
{
return crate::StoreHealthSnapshot::new(
crate::StoreHealthState::NotReady,
self.runtime_snapshot(),
std::option::Option::None,
0,
std::option::Option::Some(crate::ERROR_CODE_BACKEND_NOT_COMPILED),
);
}
}
/// Explicitly closes the Store runtime, consumes its facade handle and applies the configured bounded shutdown deadline.
pub async fn close(self) -> ksp_store_api::Result<()> {
let backend_kind = self.backend_kind;
#[cfg(feature = "postgres")]
{
let network = self.network;
let shutdown_timeout = self.shutdown_timeout;
let result = match self.runtime {
StoreRuntime::Postgres(backend) => backend.close(shutdown_timeout).await,
};
return match result {
std::result::Result::Ok(()) => {
ksp_logging_lib::debug!(
target: crate::TRACING_TARGET,
backend = backend_kind.code(),
network = network.as_str(),
"Store runtime closed"
);
std::result::Result::Ok(())
},
std::result::Result::Err(error) => std::result::Result::Err(map_postgres_error(error, backend_kind, network.as_str())),
};
}
#[cfg(not(feature = "postgres"))]
{
return std::result::Result::Err(unavailable_runtime_error(backend_kind));
}
}
}
impl std::fmt::Debug for Store {
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
return formatter
.debug_struct("Store")
.field("backend_kind", &self.backend_kind)
.field("network", &self.network)
.field("shutdown_timeout", &self.shutdown_timeout)
.finish_non_exhaustive();
}
}
impl ksp_store_api::RawAccountObservationRead for Store {
fn get_raw_account_observation<'a>(
&'a self,
observation_key: &'a ksp_store_api::RawObservationKey,
) -> ksp_store_api::StoreApiFuture<'a, ksp_store_api::Result<std::option::Option<ksp_store_api::RawAccountObservation>>> {
return std::boxed::Box::pin(async move {
#[cfg(feature = "postgres")]
{
return match &self.runtime {
StoreRuntime::Postgres(backend) => {
let result = backend.get_raw_account_observation(observation_key).await;
result.map_err(|error| return map_postgres_error(error, self.backend_kind, self.network.as_str()))
},
};
}
#[cfg(not(feature = "postgres"))]
{
let _ = observation_key;
return std::result::Result::Err(unavailable_runtime_error(self.backend_kind));
}
});
}
}
impl ksp_store_api::RawAccountObservationWrite for Store {
fn record_raw_account_observation<'a>(
&'a self,
observation: ksp_store_api::RawAccountObservation,
) -> ksp_store_api::StoreApiFuture<'a, ksp_store_api::Result<ksp_store_api::RawObservationWriteOutcome>> {
let network_check = validate_operation_network(&self.network, observation.account().network(), self.backend_kind);
if let std::result::Result::Err(error) = network_check {
return std::boxed::Box::pin(async move {
return std::result::Result::Err(error);
});
}
return std::boxed::Box::pin(async move {
#[cfg(feature = "postgres")]
{
return match &self.runtime {
StoreRuntime::Postgres(backend) => {
let result = backend.record_raw_account_observation(observation).await;
result.map_err(|error| return map_postgres_error(error, self.backend_kind, self.network.as_str()))
},
};
}
#[cfg(not(feature = "postgres"))]
{
let _ = observation;
return std::result::Result::Err(unavailable_runtime_error(self.backend_kind));
}
});
}
}
impl ksp_store_api::RawAccountStateRead for Store {
fn get_raw_account_state<'a>(
&'a self,
reference: &'a ksp_store_api::RawAccountStateReference,
) -> ksp_store_api::StoreApiFuture<'a, ksp_store_api::Result<std::option::Option<ksp_store_api::RawAccountState>>> {
let network_check = validate_operation_network(&self.network, reference.network(), self.backend_kind);
if let std::result::Result::Err(error) = network_check {
return std::boxed::Box::pin(async move {
return std::result::Result::Err(error);
});
}
return std::boxed::Box::pin(async move {
#[cfg(feature = "postgres")]
{
return match &self.runtime {
StoreRuntime::Postgres(backend) => {
let result = backend.get_raw_account_state(reference).await;
result.map_err(|error| return map_postgres_error(error, self.backend_kind, self.network.as_str()))
},
};
}
#[cfg(not(feature = "postgres"))]
{
let _ = reference;
return std::result::Result::Err(unavailable_runtime_error(self.backend_kind));
}
});
}
fn list_raw_account_states<'a>(
&'a self,
query: &'a ksp_store_api::RawAccountStateQuery,
) -> ksp_store_api::StoreApiFuture<'a, ksp_store_api::Result<ksp_store_api::RawPage<ksp_store_api::RawAccountStateReference>>> {
let network_check = validate_operation_network(&self.network, query.network(), self.backend_kind);
if let std::result::Result::Err(error) = network_check {
return std::boxed::Box::pin(async move {
return std::result::Result::Err(error);
});
}
return std::boxed::Box::pin(async move {
#[cfg(feature = "postgres")]
{
return match &self.runtime {
StoreRuntime::Postgres(backend) => {
let result = backend.list_raw_account_states(query).await;
result.map_err(|error| return map_postgres_error(error, self.backend_kind, self.network.as_str()))
},
};
}
#[cfg(not(feature = "postgres"))]
{
let _ = query;
return std::result::Result::Err(unavailable_runtime_error(self.backend_kind));
}
});
}
}
impl ksp_store_api::RawAccountStateWrite for Store {
fn persist_raw_account_acquisition<'a>(
&'a self,
state: ksp_store_api::RawAccountState,
observation: ksp_store_api::RawAccountObservation,
) -> ksp_store_api::StoreApiFuture<'a, ksp_store_api::Result<ksp_store_api::RawAcquisitionWriteOutcome>> {
let state_network = validate_operation_network(&self.network, state.reference().network(), self.backend_kind);
if let std::result::Result::Err(error) = state_network {
return std::boxed::Box::pin(async move {
return std::result::Result::Err(error);
});
}
let observation_network = validate_operation_network(&self.network, observation.account().network(), self.backend_kind);
if let std::result::Result::Err(error) = observation_network {
return std::boxed::Box::pin(async move {
return std::result::Result::Err(error);
});
}
return std::boxed::Box::pin(async move {
#[cfg(feature = "postgres")]
{
return match &self.runtime {
StoreRuntime::Postgres(backend) => {
let result = backend.persist_raw_account_acquisition(state, observation).await;
result.map_err(|error| return map_postgres_error(error, self.backend_kind, self.network.as_str()))
},
};
}
#[cfg(not(feature = "postgres"))]
{
let _ = state;
let _ = observation;
return std::result::Result::Err(unavailable_runtime_error(self.backend_kind));
}
});
}
}
impl ksp_store_api::RawTransactionRead for Store {
fn get_raw_transaction<'a>(
&'a self,
reference: &'a ksp_store_api::RawTransactionReference,
) -> ksp_store_api::StoreApiFuture<'a, ksp_store_api::Result<std::option::Option<ksp_store_api::RawTransaction>>> {
let network_check = validate_operation_network(&self.network, reference.network(), self.backend_kind);
if let std::result::Result::Err(error) = network_check {
return std::boxed::Box::pin(async move {
return std::result::Result::Err(error);
});
}
return std::boxed::Box::pin(async move {
#[cfg(feature = "postgres")]
{
return match &self.runtime {
StoreRuntime::Postgres(backend) => {
let result = backend.get_raw_transaction(reference).await;
result.map_err(|error| return map_postgres_error(error, self.backend_kind, self.network.as_str()))
},
};
}
#[cfg(not(feature = "postgres"))]
{
let _ = reference;
return std::result::Result::Err(unavailable_runtime_error(self.backend_kind));
}
});
}
fn list_raw_transactions<'a>(
&'a self,
query: &'a ksp_store_api::RawTransactionQuery,
) -> ksp_store_api::StoreApiFuture<'a, ksp_store_api::Result<ksp_store_api::RawPage<ksp_store_api::RawTransactionReference>>> {
let network_check = validate_operation_network(&self.network, query.network(), self.backend_kind);
if let std::result::Result::Err(error) = network_check {
return std::boxed::Box::pin(async move {
return std::result::Result::Err(error);
});
}
return std::boxed::Box::pin(async move {
#[cfg(feature = "postgres")]
{
return match &self.runtime {
StoreRuntime::Postgres(backend) => {
let result = backend.list_raw_transactions(query).await;
result.map_err(|error| return map_postgres_error(error, self.backend_kind, self.network.as_str()))
},
};
}
#[cfg(not(feature = "postgres"))]
{
let _ = query;
return std::result::Result::Err(unavailable_runtime_error(self.backend_kind));
}
});
}
}
impl ksp_store_api::RawTransactionWrite for Store {
fn persist_raw_transaction_acquisition<'a>(
&'a self,
transaction: ksp_store_api::RawTransaction,
observation: ksp_store_api::RawTransactionObservation,
mode: ksp_store_api::RawTransactionAcquisitionMode,
) -> ksp_store_api::StoreApiFuture<'a, ksp_store_api::Result<ksp_store_api::RawAcquisitionWriteOutcome>> {
let transaction_network = validate_operation_network(&self.network, transaction.reference().network(), self.backend_kind);
if let std::result::Result::Err(error) = transaction_network {
return std::boxed::Box::pin(async move {
return std::result::Result::Err(error);
});
}
let observation_network = validate_operation_network(&self.network, observation.transaction().network(), self.backend_kind);
if let std::result::Result::Err(error) = observation_network {
return std::boxed::Box::pin(async move {
return std::result::Result::Err(error);
});
}
return std::boxed::Box::pin(async move {
#[cfg(feature = "postgres")]
{
return match &self.runtime {
StoreRuntime::Postgres(backend) => {
let result = backend.persist_raw_transaction_acquisition(transaction, observation, mode).await;
result.map_err(|error| return map_postgres_error(error, self.backend_kind, self.network.as_str()))
},
};
}
#[cfg(not(feature = "postgres"))]
{
let _ = transaction;
let _ = observation;
let _ = mode;
return std::result::Result::Err(unavailable_runtime_error(self.backend_kind));
}
});
}
}
impl ksp_store_api::RawTransactionObservationRead for Store {
fn get_raw_transaction_observation<'a>(
&'a self,
observation_key: &'a ksp_store_api::RawObservationKey,
) -> ksp_store_api::StoreApiFuture<'a, ksp_store_api::Result<std::option::Option<ksp_store_api::RawTransactionObservation>>> {
return std::boxed::Box::pin(async move {
#[cfg(feature = "postgres")]
{
return match &self.runtime {
StoreRuntime::Postgres(backend) => {
let result = backend.get_raw_transaction_observation(observation_key).await;
result.map_err(|error| return map_postgres_error(error, self.backend_kind, self.network.as_str()))
},
};
}
#[cfg(not(feature = "postgres"))]
{
let _ = observation_key;
return std::result::Result::Err(unavailable_runtime_error(self.backend_kind));
}
});
}
}
impl ksp_store_api::RawTransactionObservationWrite for Store {
fn record_raw_transaction_observation<'a>(
&'a self,
observation: ksp_store_api::RawTransactionObservation,
) -> ksp_store_api::StoreApiFuture<'a, ksp_store_api::Result<ksp_store_api::RawObservationWriteOutcome>> {
let network_check = validate_operation_network(&self.network, observation.transaction().network(), self.backend_kind);
if let std::result::Result::Err(error) = network_check {
return std::boxed::Box::pin(async move {
return std::result::Result::Err(error);
});
}
return std::boxed::Box::pin(async move {
#[cfg(feature = "postgres")]
{
return match &self.runtime {
StoreRuntime::Postgres(backend) => {
let result = backend.record_raw_transaction_observation(observation).await;
result.map_err(|error| return map_postgres_error(error, self.backend_kind, self.network.as_str()))
},
};
}
#[cfg(not(feature = "postgres"))]
{
let _ = observation;
return std::result::Result::Err(unavailable_runtime_error(self.backend_kind));
}
});
}
}
impl ksp_store_api::RawTransactionRetentionRead for Store {
fn get_raw_transaction_retention_state<'a>(
&'a self,
reference: &'a ksp_store_api::RawTransactionReference,
) -> ksp_store_api::StoreApiFuture<'a, ksp_store_api::Result<std::option::Option<ksp_store_api::RawRetentionState>>> {
let network_check = validate_operation_network(&self.network, reference.network(), self.backend_kind);
if let std::result::Result::Err(error) = network_check {
return std::boxed::Box::pin(async move {
return std::result::Result::Err(error);
});
}
return std::boxed::Box::pin(async move {
#[cfg(feature = "postgres")]
{
return match &self.runtime {
StoreRuntime::Postgres(backend) => {
let result = backend.get_raw_transaction_retention_state(reference).await;
result.map_err(|error| return map_postgres_error(error, self.backend_kind, self.network.as_str()))
},
};
}
#[cfg(not(feature = "postgres"))]
{
let _ = reference;
return std::result::Result::Err(unavailable_runtime_error(self.backend_kind));
}
});
}
fn get_raw_transaction_tombstone<'a>(
&'a self,
reference: &'a ksp_store_api::RawTransactionReference,
) -> ksp_store_api::StoreApiFuture<'a, ksp_store_api::Result<std::option::Option<ksp_store_api::RawTransactionTombstone>>> {
let network_check = validate_operation_network(&self.network, reference.network(), self.backend_kind);
if let std::result::Result::Err(error) = network_check {
return std::boxed::Box::pin(async move {
return std::result::Result::Err(error);
});
}
return std::boxed::Box::pin(async move {
#[cfg(feature = "postgres")]
{
return match &self.runtime {
StoreRuntime::Postgres(backend) => {
let result = backend.get_raw_transaction_tombstone(reference).await;
result.map_err(|error| return map_postgres_error(error, self.backend_kind, self.network.as_str()))
},
};
}
#[cfg(not(feature = "postgres"))]
{
let _ = reference;
return std::result::Result::Err(unavailable_runtime_error(self.backend_kind));
}
});
}
}
impl ksp_store_api::RawTransactionRetentionWrite for Store {
fn transition_raw_transaction_retention<'a>(
&'a self,
transition: ksp_store_api::RawTransactionRetentionTransition,
) -> ksp_store_api::StoreApiFuture<'a, ksp_store_api::Result<ksp_store_api::RawRetentionWriteOutcome>> {
let network_check = validate_operation_network(&self.network, transition.reference().network(), self.backend_kind);
if let std::result::Result::Err(error) = network_check {
return std::boxed::Box::pin(async move {
return std::result::Result::Err(error);
});
}
return std::boxed::Box::pin(async move {
#[cfg(feature = "postgres")]
{
return match &self.runtime {
StoreRuntime::Postgres(backend) => {
let result = backend.transition_raw_transaction_retention(transition).await;
result.map_err(|error| return map_postgres_error(error, self.backend_kind, self.network.as_str()))
},
};
}
#[cfg(not(feature = "postgres"))]
{
let _ = transition;
return std::result::Result::Err(unavailable_runtime_error(self.backend_kind));
}
});
}
}
fn validate_operation_network(
store_network: &ksp_store_api::RawNetworkId,
operation_network: &ksp_store_api::RawNetworkId,
backend_kind: crate::StoreBackendKind,
) -> ksp_store_api::Result<()> {
if store_network != operation_network {
return std::result::Result::Err(
ksp_store_api::Error::new(crate::ERROR_CODE_WRONG_NETWORK, "Store operation targeted a different logical network")
.with_context("backend", backend_kind.code())
.with_context("network", store_network.as_str()),
);
}
return std::result::Result::Ok(());
}
#[cfg(feature = "postgres")]
enum StoreRuntime {
Postgres(ksp_store_postgres_lib::PostgresBackend),
}
#[cfg(feature = "postgres")]
async fn open_postgres(
backend_kind: crate::StoreBackendKind,
network: ksp_store_api::RawNetworkId,
shutdown_timeout: std::time::Duration,
settings: &crate::PostgresStoreSettings,
) -> ksp_store_api::Result<Store> {
let bootstrap = settings.bootstrap();
let pool = settings.pool();
let tls_mode = match settings.tls_mode() {
crate::PostgresTlsMode::Disabled => ksp_store_postgres_lib::PostgresBackendTlsMode::Disabled,
crate::PostgresTlsMode::VerifyFull => ksp_store_postgres_lib::PostgresBackendTlsMode::VerifyFull,
};
let backend_settings = ksp_store_postgres_lib::PostgresBackendSettings::with_schema_policy(
network.clone(),
settings.connection_uri(),
pool.max_connections(),
pool.connect_timeout(),
pool.wait_timeout(),
pool.create_timeout(),
pool.recycle_timeout(),
tls_mode,
bootstrap.schema_autocreate(),
bootstrap.schema_autoupdate(),
bootstrap.migration_timeout(),
bootstrap.migration_lock_timeout(),
);
let opened = ksp_store_postgres_lib::PostgresBackend::open(backend_settings).await;
return match opened {
std::result::Result::Ok(backend) => {
ksp_logging_lib::debug!(
target: crate::TRACING_TARGET,
backend = backend_kind.code(),
network = network.as_str(),
"Store backend is physically ready"
);
std::result::Result::Ok(Store { backend_kind, network, runtime: StoreRuntime::Postgres(backend), shutdown_timeout })
},
std::result::Result::Err(error) => std::result::Result::Err(map_postgres_error(error, backend_kind, network.as_str())),
};
}
#[cfg(not(feature = "postgres"))]
async fn open_postgres(
backend_kind: crate::StoreBackendKind,
_network: ksp_store_api::RawNetworkId,
_shutdown_timeout: std::time::Duration,
_settings: &crate::PostgresStoreSettings,
) -> ksp_store_api::Result<Store> {
return std::result::Result::Err(unavailable_runtime_error(backend_kind));
}
#[cfg(feature = "postgres")]
fn map_postgres_error(error: ksp_store_postgres_lib::PostgresBackendError, backend_kind: crate::StoreBackendKind, network: &str) -> ksp_store_api::Error {
let code = postgres_error_code(error.kind());
return ksp_store_api::Error::new(code, "PostgreSQL Store backend operation failed")
.with_context("backend", backend_kind.code())
.with_context("network", network)
.with_context("phase", error.phase());
}
#[cfg(feature = "postgres")]
fn map_postgres_runtime_snapshot(
snapshot: ksp_store_postgres_lib::PostgresBackendRuntimeSnapshot,
backend_kind: crate::StoreBackendKind,
network: ksp_store_api::RawNetworkId,
) -> crate::StoreRuntimeSnapshot {
return crate::StoreRuntimeSnapshot::new(
backend_kind,
network,
snapshot.pool_capacity(),
snapshot.pool_size(),
snapshot.pool_available(),
snapshot.pool_waiting(),
);
}
#[cfg(feature = "postgres")]
fn map_postgres_health_snapshot(
snapshot: ksp_store_postgres_lib::PostgresBackendHealthSnapshot,
backend_kind: crate::StoreBackendKind,
network: ksp_store_api::RawNetworkId,
) -> crate::StoreHealthSnapshot {
let state = if snapshot.is_ready() { crate::StoreHealthState::Ready } else { crate::StoreHealthState::NotReady };
let error_code = snapshot.error_kind().map(|kind| return postgres_error_code(kind));
let runtime = map_postgres_runtime_snapshot(snapshot.runtime().clone(), backend_kind, network);
return crate::StoreHealthSnapshot::new(state, runtime, snapshot.migration_version(), snapshot.pending_migration_count(), error_code);
}
#[cfg(feature = "postgres")]
fn postgres_error_code(kind: ksp_store_postgres_lib::PostgresBackendErrorKind) -> ksp_store_api::ErrorCode {
return match kind {
ksp_store_postgres_lib::PostgresBackendErrorKind::ConfigInvalid => crate::ERROR_CODE_POSTGRES_CONFIG_INVALID,
ksp_store_postgres_lib::PostgresBackendErrorKind::ConnectFailed => crate::ERROR_CODE_POSTGRES_CONNECT_FAILED,
ksp_store_postgres_lib::PostgresBackendErrorKind::Conflict => ksp_store_api::ERROR_CODE_RAW_CONFLICT,
ksp_store_postgres_lib::PostgresBackendErrorKind::DataInvalid => crate::ERROR_CODE_POSTGRES_DATA_INVALID,
ksp_store_postgres_lib::PostgresBackendErrorKind::HealthFailed => crate::ERROR_CODE_POSTGRES_HEALTH_FAILED,
ksp_store_postgres_lib::PostgresBackendErrorKind::PoolTimeout => crate::ERROR_CODE_POSTGRES_POOL_TIMEOUT,
ksp_store_postgres_lib::PostgresBackendErrorKind::MigrationFailed => crate::ERROR_CODE_POSTGRES_MIGRATION_FAILED,
ksp_store_postgres_lib::PostgresBackendErrorKind::MigrationMismatch => crate::ERROR_CODE_POSTGRES_MIGRATION_MISMATCH,
ksp_store_postgres_lib::PostgresBackendErrorKind::PageLimitUnsupported => crate::ERROR_CODE_POSTGRES_PAGE_LIMIT_UNSUPPORTED,
ksp_store_postgres_lib::PostgresBackendErrorKind::QueryInvalid => ksp_store_api::ERROR_CODE_RAW_QUERY_INVALID,
ksp_store_postgres_lib::PostgresBackendErrorKind::ReadFailed => crate::ERROR_CODE_POSTGRES_READ_FAILED,
ksp_store_postgres_lib::PostgresBackendErrorKind::ReferenceNotFound => crate::ERROR_CODE_RAW_REFERENCE_NOT_FOUND,
ksp_store_postgres_lib::PostgresBackendErrorKind::RetentionCompactionUnsupported => crate::ERROR_CODE_POSTGRES_RETENTION_COMPACTION_UNSUPPORTED,
ksp_store_postgres_lib::PostgresBackendErrorKind::SchemaNewer => crate::ERROR_CODE_POSTGRES_SCHEMA_NEWER,
ksp_store_postgres_lib::PostgresBackendErrorKind::ShutdownTimeout => crate::ERROR_CODE_SHUTDOWN_TIMEOUT,
ksp_store_postgres_lib::PostgresBackendErrorKind::TlsFailed => crate::ERROR_CODE_POSTGRES_TLS_FAILED,
ksp_store_postgres_lib::PostgresBackendErrorKind::WriteFailed => crate::ERROR_CODE_POSTGRES_WRITE_FAILED,
ksp_store_postgres_lib::PostgresBackendErrorKind::WrongNetwork => crate::ERROR_CODE_WRONG_NETWORK,
_ => crate::ERROR_CODE_BACKEND_OPEN_FAILED,
};
}
#[cfg(not(feature = "postgres"))]
fn unavailable_runtime_error(backend_kind: crate::StoreBackendKind) -> ksp_store_api::Error {
return ksp_store_api::Error::new(crate::ERROR_CODE_BACKEND_NOT_COMPILED, "Selected Store backend is not compiled")
.with_context("backend", backend_kind.code());
}
#[cfg(test)]
#[path = "../unit_tests/store.rs"]
mod tests;

View File

@@ -0,0 +1,87 @@
// file: crates/ksp-store-lib/tests/dependency_boundary.rs
// version: 8
#![warn(missing_docs)]
#![deny(unreachable_pub)]
#![forbid(unsafe_code)]
//! Cargo-feature and dependency-boundary canaries for the common Store runtime facade.
#[test]
fn pre_005_manifest_keeps_backend_physical_dependencies_out_of_facade() {
let manifest = include_str!("../Cargo.toml");
assert!(manifest.contains("default = [\"postgres\"]"));
assert!(manifest.contains("postgres = [\"dep:ksp-store-postgres-lib\"]"));
assert!(manifest.contains("ksp-logging-lib = { path = \"../ksp-logging-lib\" }"));
assert!(manifest.contains("ksp-store-api = { path = \"../ksp-store-api\" }"));
assert!(manifest.contains("ksp-store-postgres-lib = { path = \"../ksp-store-postgres-lib\", optional = true }"));
for forbidden in [
"ksp-config-lib",
"ksp-materializer",
"ksp-program",
"ksp-onchain-transport-lib",
"ksp-offchain-transport-lib",
"tokio-postgres",
"deadpool-postgres",
"rustls",
] {
assert!(!manifest.contains(forbidden), "forbidden Store facade dependency detected: {forbidden}");
}
return;
}
#[test]
fn pre_005_facade_exposes_no_physical_postgres_types_or_environment_bypass() {
let crate_root = include_str!("../src/lib.rs");
assert!(crate_root.contains("pub use self::settings::StoreSettings;"));
assert!(crate_root.contains("pub use self::health::StoreHealthSnapshot;"));
assert!(crate_root.contains("pub use self::health::StoreRuntimeSnapshot;"));
assert!(crate_root.contains("pub use self::store::Store;"));
assert!(crate_root.contains("pub use ksp_store_api::RawTransaction;"));
assert!(crate_root.contains("const _: &str = crate::TRACING_TARGET;"));
for forbidden in [
"pub mod ",
"pub use ksp_store_postgres_lib",
"tokio_postgres",
"deadpool_postgres",
"rustls::",
"deadpool::managed::Pool",
"tokio_postgres::Client",
"tokio_postgres::Row",
"tokio_postgres::Statement",
] {
assert!(!crate_root.contains(forbidden), "forbidden physical backend facade surface detected: {forbidden}");
}
let production = format!("{}\n{}\n{}", include_str!("../src/health.rs"), include_str!("../src/settings.rs"), include_str!("../src/store.rs"));
for forbidden in ["ksp_config_lib", "std::env", "dotenv", "PGHOST", "PGPORT", "PGUSER", "PGPASSWORD", ".pgpass", "tokio_postgres", "deadpool_postgres"] {
assert!(!production.contains(forbidden), "forbidden Store facade ownership bypass detected: {forbidden}");
}
return;
}
#[test]
fn pre_008_facade_dispatches_exact_ten_raw_capabilities_without_physical_leak() {
let store = include_str!("../src/store.rs");
for required in [
"impl ksp_store_api::RawAccountObservationRead for Store",
"impl ksp_store_api::RawAccountObservationWrite for Store",
"impl ksp_store_api::RawAccountStateRead for Store",
"impl ksp_store_api::RawAccountStateWrite for Store",
"impl ksp_store_api::RawTransactionObservationRead for Store",
"impl ksp_store_api::RawTransactionObservationWrite for Store",
"impl ksp_store_api::RawTransactionRead for Store",
"impl ksp_store_api::RawTransactionRetentionRead for Store",
"impl ksp_store_api::RawTransactionRetentionWrite for Store",
"impl ksp_store_api::RawTransactionWrite for Store",
"validate_operation_network",
"StoreRuntime::Postgres(backend)",
"map_postgres_error",
] {
assert!(store.contains(required), "missing pre.008 Store capability dispatch contract: {required}");
}
assert_eq!(store.matches("impl ksp_store_api::Raw").count(), 10);
for forbidden in ["tokio_postgres::", "deadpool_postgres::", "CREATE TABLE", "INSERT INTO", "UPDATE ksp_", "DELETE FROM"] {
assert!(!store.contains(forbidden), "pre.008 facade leaked physical backend material: {forbidden}");
}
return;
}

View File

@@ -0,0 +1,55 @@
// file: crates/ksp-store-lib/tests/feature_mismatch.rs
// version: 4
#![warn(missing_docs)]
#![deny(unreachable_pub)]
#![forbid(unsafe_code)]
//! Feature-selection and pre-I/O PostgreSQL failure canaries for the common Store facade.
fn poll_ready<T>(future: impl std::future::Future<Output = T>) -> T {
let mut future = std::boxed::Box::pin(future);
let waker = std::task::Waker::noop();
let mut context = std::task::Context::from_waker(waker);
return match std::future::Future::poll(future.as_mut(), &mut context) {
std::task::Poll::Ready(value) => value,
std::task::Poll::Pending => panic!("Store pre-I/O feature/config rejection unexpectedly became pending"),
};
}
fn valid_network() -> ksp_store_lib::RawNetworkId {
return match ksp_store_lib::RawNetworkId::new("devnet") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => panic!("valid test network rejected: {error:?}"),
};
}
fn settings(connection_uri: &str) -> ksp_store_lib::StoreSettings {
let postgres = ksp_store_lib::PostgresStoreSettings::new(
connection_uri,
ksp_store_lib::PostgresPoolSettings::default(),
ksp_store_lib::PostgresTlsMode::Disabled,
ksp_store_lib::PostgresBootstrapSettings::default(),
);
return ksp_store_lib::StoreSettings::with_default_shutdown(valid_network(), ksp_store_lib::StoreBackendSettings::Postgres(postgres));
}
#[cfg(not(feature = "postgres"))]
#[test]
fn pre_005_known_postgres_without_feature_returns_stable_error_before_io() {
let result = poll_ready(ksp_store_lib::Store::open(settings("postgresql://operator-supplied-sensitive-value@localhost/ksp")));
let error = result.err();
assert_eq!(error.map(|value| return value.code()), std::option::Option::Some(ksp_store_lib::ERROR_CODE_BACKEND_NOT_COMPILED));
return;
}
#[cfg(feature = "postgres")]
#[test]
fn pre_005_compiled_postgres_rejects_malformed_uri_without_secret_leak_before_io() {
let secret_canary = "not-a-postgresql-uri-secret-canary";
let result = poll_ready(ksp_store_lib::Store::open(settings(secret_canary)));
let error = result.err();
assert_eq!(error.as_ref().map(|value| return value.code()), std::option::Option::Some(ksp_store_lib::ERROR_CODE_POSTGRES_CONFIG_INVALID));
assert!(!format!("{error:?}").contains(secret_canary));
return;
}

View File

@@ -0,0 +1,323 @@
// file: crates/ksp-store-lib/tests/hardening_completeness.rs
// version: 6
#![warn(missing_docs)]
#![deny(unreachable_pub)]
#![forbid(unsafe_code)]
//! Hardening, exact-surface and feature-matrix canaries for the common Store facade.
const SECRET_CANARY: &str = "KSP-STORE-SECRET-CANARY-PRE009";
fn poll_ready<T>(future: impl std::future::Future<Output = T>) -> T {
let mut future = std::boxed::Box::pin(future);
let waker = std::task::Waker::noop();
let mut context = std::task::Context::from_waker(waker);
return match std::future::Future::poll(future.as_mut(), &mut context) {
std::task::Poll::Ready(value) => value,
std::task::Poll::Pending => panic!("Store hardening pre-I/O rejection unexpectedly became pending"),
};
}
fn valid_network() -> ksp_store_lib::RawNetworkId {
return match ksp_store_lib::RawNetworkId::new("devnet") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => panic!("valid hardening network rejected: {error:?}"),
};
}
fn hostile_settings(connection_uri: &str) -> ksp_store_lib::StoreSettings {
let postgres = ksp_store_lib::PostgresStoreSettings::new(
connection_uri,
ksp_store_lib::PostgresPoolSettings::default(),
ksp_store_lib::PostgresTlsMode::VerifyFull,
ksp_store_lib::PostgresBootstrapSettings::default(),
);
return ksp_store_lib::StoreSettings::with_default_shutdown(valid_network(), ksp_store_lib::StoreBackendSettings::Postgres(postgres));
}
fn public_reexport_names(source: &str) -> std::vec::Vec<&str> {
let mut names = std::vec::Vec::new();
for line in source.lines() {
let trimmed = line.trim();
if !trimmed.starts_with("pub use ") || !trimmed.ends_with(';') {
continue;
}
let without_semicolon = trimmed.trim_end_matches(';');
let name = match without_semicolon.rsplit("::").next() {
std::option::Option::Some(value) => value,
std::option::Option::None => continue,
};
names.push(name);
}
names.sort_unstable();
return names;
}
fn manifest_dependency_names(source: &str) -> std::vec::Vec<&str> {
let dependencies_tail = match source.split("[dependencies]").nth(1) {
std::option::Option::Some(value) => value,
std::option::Option::None => return std::vec::Vec::new(),
};
let dependencies = match dependencies_tail.split("[lints]").next() {
std::option::Option::Some(value) => value,
std::option::Option::None => return std::vec::Vec::new(),
};
let mut names = std::vec::Vec::new();
for line in dependencies.lines() {
let content = match line.split('#').next() {
std::option::Option::Some(value) => value.trim(),
std::option::Option::None => continue,
};
if content.is_empty() {
continue;
}
let key = match content.split('=').next() {
std::option::Option::Some(value) => value.trim().trim_end_matches(".workspace"),
std::option::Option::None => continue,
};
if !key.is_empty() {
names.push(key);
}
}
names.sort_unstable();
return names;
}
fn raw_capability_trait_names<'a>(source: &'a str, implementor: &str) -> std::vec::Vec<&'a str> {
let mut names = std::vec::Vec::new();
for line in source.lines() {
let trimmed = line.trim();
if !trimmed.starts_with("impl ksp_store_api::Raw") || !trimmed.contains(implementor) {
continue;
}
let trait_tail = match trimmed.strip_prefix("impl ksp_store_api::") {
std::option::Option::Some(value) => value,
std::option::Option::None => continue,
};
let trait_name = match trait_tail.split(" for ").next() {
std::option::Option::Some(value) => value,
std::option::Option::None => continue,
};
names.push(trait_name);
}
names.sort_unstable();
return names;
}
#[test]
fn pre_009_facade_modules_and_crate_root_exports_are_exact() {
let crate_root = include_str!("../src/lib.rs");
for required in ["mod constants;", "mod error;", "mod health;", "mod settings;", "mod store;"] {
assert!(crate_root.contains(required), "missing Store facade module: {required}");
}
assert!(!crate_root.contains("pub mod "));
let actual = public_reexport_names(crate_root);
let mut expected = [
"ERROR_CODE_BACKEND_CLOSED",
"ERROR_CODE_BACKEND_NOT_COMPILED",
"ERROR_CODE_BACKEND_OPEN_FAILED",
"ERROR_CODE_POSTGRES_CONFIG_INVALID",
"ERROR_CODE_POSTGRES_CONNECT_FAILED",
"ERROR_CODE_POSTGRES_DATA_INVALID",
"ERROR_CODE_POSTGRES_HEALTH_FAILED",
"ERROR_CODE_POSTGRES_MIGRATION_FAILED",
"ERROR_CODE_POSTGRES_MIGRATION_MISMATCH",
"ERROR_CODE_POSTGRES_PAGE_LIMIT_UNSUPPORTED",
"ERROR_CODE_POSTGRES_POOL_TIMEOUT",
"ERROR_CODE_POSTGRES_READ_FAILED",
"ERROR_CODE_POSTGRES_RETENTION_COMPACTION_UNSUPPORTED",
"ERROR_CODE_POSTGRES_SCHEMA_NEWER",
"ERROR_CODE_POSTGRES_TLS_FAILED",
"ERROR_CODE_POSTGRES_WRITE_FAILED",
"ERROR_CODE_RAW_CONFLICT",
"ERROR_CODE_RAW_MODEL_INVALID",
"ERROR_CODE_RAW_PAYLOAD_INVALID",
"ERROR_CODE_RAW_PROVENANCE_INVALID",
"ERROR_CODE_RAW_QUERY_INVALID",
"ERROR_CODE_RAW_REFERENCE_NOT_FOUND",
"ERROR_CODE_RAW_RETENTION_INVALID",
"ERROR_CODE_SETTINGS_INVALID",
"ERROR_CODE_SHUTDOWN_TIMEOUT",
"ERROR_CODE_WRONG_NETWORK",
"Error",
"ErrorCode",
"ErrorContext",
"MAX_RAW_ACCOUNT_DATA_BYTES",
"MAX_RAW_CODE_BYTES",
"MAX_RAW_PAGE_CURSOR_BYTES",
"MAX_RAW_PAYLOAD_BYTES",
"MAX_RAW_SOURCE_PAYLOAD_BYTES",
"MAX_RAW_UNIX_MILLIS",
"PostgresBootstrapSettings",
"PostgresPoolSettings",
"PostgresStoreSettings",
"PostgresTlsMode",
"Pubkey",
"RawAccountObservation",
"RawAccountObservationRead",
"RawAccountObservationWrite",
"RawAccountState",
"RawAccountStateQuery",
"RawAccountStateRead",
"RawAccountStateReference",
"RawAccountStateWrite",
"RawAcquisitionOrigin",
"RawAcquisitionProvenance",
"RawAcquisitionWriteOutcome",
"RawContentHash",
"RawEntityWriteOutcome",
"RawFormatId",
"RawNetworkId",
"RawObservationKey",
"RawObservationWriteOutcome",
"RawPage",
"RawPageCursor",
"RawPageLimit",
"RawPageRequest",
"RawPayload",
"RawProvenanceCode",
"RawRetentionState",
"RawRetentionWriteOutcome",
"RawSlotRange",
"RawSortDirection",
"RawTimestamp",
"RawTransaction",
"RawTransactionAcquisitionMode",
"RawTransactionObservation",
"RawTransactionObservationRead",
"RawTransactionObservationWrite",
"RawTransactionQuery",
"RawTransactionRead",
"RawTransactionReference",
"RawTransactionRetentionRead",
"RawTransactionRetentionTransition",
"RawTransactionRetentionWrite",
"RawTransactionSignature",
"RawTransactionTombstone",
"RawTransactionWrite",
"Result",
"Store",
"StoreApiFuture",
"StoreBackendKind",
"StoreBackendSettings",
"StoreHealthSnapshot",
"StoreHealthState",
"StoreRuntimeSnapshot",
"StoreSettings",
];
expected.sort_unstable();
assert_eq!(actual.as_slice(), expected.as_slice());
assert_eq!(actual.len(), 91);
return;
}
#[test]
fn pre_009_facade_manifest_and_feature_contract_are_exact() {
let manifest = include_str!("../Cargo.toml");
assert!(manifest.contains("default = [\"postgres\"]"));
assert!(manifest.contains("postgres = [\"dep:ksp-store-postgres-lib\"]"));
let actual = manifest_dependency_names(manifest);
let expected = ["ksp-logging-lib", "ksp-store-api", "ksp-store-postgres-lib"];
assert_eq!(actual.as_slice(), expected.as_slice());
for forbidden in ["tokio-postgres", "deadpool-postgres", "rustls", "sqlx", "ksp-config-lib", "ksp-onchain-transport-lib", "ksp-offchain-transport-lib"] {
assert!(!manifest.contains(forbidden), "forbidden Store facade dependency detected: {forbidden}");
}
return;
}
#[test]
fn pre_009_secret_canary_never_crosses_settings_or_pre_io_error_debug() {
let malformed = std::format!("not-a-postgresql-uri-{SECRET_CANARY}");
let settings = hostile_settings(malformed.as_str());
let rendered = std::format!("{settings:?}");
assert!(!rendered.contains(SECRET_CANARY));
assert!(rendered.contains("<redacted>"));
let result = poll_ready(ksp_store_lib::Store::open(settings));
let error = match result {
std::result::Result::Err(value) => value,
std::result::Result::Ok(_) => panic!("hostile Store settings unexpectedly opened"),
};
#[cfg(feature = "postgres")]
assert_eq!(error.code(), ksp_store_lib::ERROR_CODE_POSTGRES_CONFIG_INVALID);
#[cfg(not(feature = "postgres"))]
assert_eq!(error.code(), ksp_store_lib::ERROR_CODE_BACKEND_NOT_COMPILED);
assert!(!std::format!("{error}").contains(SECRET_CANARY));
assert!(!std::format!("{error:?}").contains(SECRET_CANARY));
return;
}
#[test]
fn pre_009_facade_production_sources_keep_config_env_physical_sql_and_backend_handles_out() {
let production = std::format!(
"{}
{}
{}
{}",
include_str!("../src/error.rs"),
include_str!("../src/health.rs"),
include_str!("../src/settings.rs"),
include_str!("../src/store.rs")
);
for forbidden in [
"ksp_config_lib",
"std::env",
"dotenv",
"PGHOST",
"PGPORT",
"PGUSER",
"PGPASSWORD",
".pgpass",
"tokio_postgres::Client",
"tokio_postgres::Row",
"tokio_postgres::Statement",
"deadpool_postgres::Pool",
"sqlx::",
"CREATE TABLE",
"INSERT INTO",
"UPDATE ",
"DELETE FROM",
] {
assert!(!production.contains(forbidden), "forbidden facade ownership/runtime material detected: {forbidden}");
}
return;
}
#[test]
fn pre_010_facade_raw_capability_inventory_is_exactly_ten() {
let store = include_str!("../src/store.rs");
let capability_impls = [
"impl ksp_store_api::RawAccountObservationRead for Store",
"impl ksp_store_api::RawAccountObservationWrite for Store",
"impl ksp_store_api::RawAccountStateRead for Store",
"impl ksp_store_api::RawAccountStateWrite for Store",
"impl ksp_store_api::RawTransactionObservationRead for Store",
"impl ksp_store_api::RawTransactionObservationWrite for Store",
"impl ksp_store_api::RawTransactionRead for Store",
"impl ksp_store_api::RawTransactionRetentionRead for Store",
"impl ksp_store_api::RawTransactionRetentionWrite for Store",
"impl ksp_store_api::RawTransactionWrite for Store",
];
for implementation in capability_impls {
assert_eq!(store.matches(implementation).count(), 1, "unexpected Store capability implementation inventory: {implementation}");
}
assert_eq!(store.matches("impl ksp_store_api::Raw").count(), 10);
assert_eq!(store.matches("validate_operation_network(").count(), 14);
return;
}
#[test]
fn pre_010_facade_and_backend_raw_capability_sets_match_exactly_without_account_retention() {
let store = include_str!("../src/store.rs");
let backend = include_str!("../../ksp-store-postgres-lib/src/runtime.rs");
let store_traits = raw_capability_trait_names(store, " for Store");
let backend_traits = raw_capability_trait_names(backend, " for PostgresBackend");
assert_eq!(store_traits.len(), 10);
assert_eq!(backend_traits.len(), 10);
assert_eq!(store_traits, backend_traits);
for forbidden in ["RawAccountRetentionRead", "RawAccountRetentionWrite", "RawAccountDelete", "RawAccountCompaction"] {
assert!(!store_traits.contains(&forbidden), "unexpected account capability added to Store: {forbidden}");
assert!(!backend_traits.contains(&forbidden), "unexpected account capability added to PostgreSQL backend: {forbidden}");
}
return;
}

Some files were not shown because too many files have changed in this diff Show More