v0.5.3-pre.004

This commit is contained in:
2026-08-12 14:31:48 +02:00
parent 400ced4832
commit b6583bd9fb
47 changed files with 2978 additions and 291 deletions

View File

@@ -1,5 +1,5 @@
# file: Cargo.toml # file: Cargo.toml
# version: 72 # version: 73
[workspace] [workspace]
resolver = "3" resolver = "3"
@@ -19,7 +19,7 @@ members = [
] ]
[workspace.package] [workspace.package]
version = "0.5.3-pre.3" version = "0.5.3-pre.4"
edition = "2024" edition = "2024"
license = "MIT" license = "MIT"
repository = "https://git.sasedev.com/Sasedev/khadhroony-bot3" repository = "https://git.sasedev.com/Sasedev/khadhroony-bot3"
@@ -89,7 +89,7 @@ spl-token-metadata-interface = { version = "^1.0", features = ["serde", "serde-t
spl-token-group-interface = { version = "^0.7", features = [] } spl-token-group-interface = { version = "^0.7", features = [] }
spl-token-interface = { version = "^3.0", features = [] } spl-token-interface = { version = "^3.0", features = [] }
spl-token-2022-interface = { version = "^3.1", features = ["serde"] } spl-token-2022-interface = { version = "^3.1", features = ["serde"] }
sqlx = { version = "^0.9", features = ["bigdecimal", "chrono", "default", "derive", "json", "mysql", "postgres", "runtime-tokio", "sqlite", "tls-rustls", "uuid"] } sqlx = { version = "^0.9", features = ["all-databases", "bigdecimal", "chrono", "default", "derive", "json", "runtime-tokio", "tls-rustls", "uuid"] }
tauri = { version = "^2.11", features = ["default", "tray-icon"] } tauri = { version = "^2.11", features = ["default", "tray-icon"] }
tauri-build = { version = "^2.6", features = [] } tauri-build = { version = "^2.6", features = [] }
tauri-plugin-tracing = { version = "^0.3", default-features = false, features = [] } tauri-plugin-tracing = { version = "^0.3", default-features = false, features = [] }

View File

@@ -1,5 +1,5 @@
<!-- file: docs/architecture/STORAGE_ARCHITECTURE.md --> <!-- file: docs/architecture/STORAGE_ARCHITECTURE.md -->
<!-- version: 8 --> <!-- version: 9 -->
# Architecture du stockage # Architecture du stockage
@@ -148,9 +148,31 @@ Principes :
- `DROP`/`TRUNCATE` séparés de l'auto-init ; - `DROP`/`TRUNCATE` séparés de l'auto-init ;
- refus explicite d'un schéma contenant encore des objets `kb_sol_*`. - refus explicite d'un schéma contenant encore des objets `kb_sol_*`.
Le baseline `pre.003` contient **236 ressources SQL atomiques** et **75 index attendus**. Ces nombres décrivent l'implémentation PostgreSQL actuelle, pas une API publique gelée. Après la revue repositories/index de `pre.004`, le baseline contient **240 ressources SQL atomiques** et **79 index attendus** (63 explicites + 16 index de clés primaires). Ces nombres décrivent l'implémentation PostgreSQL actuelle, pas une API publique gelée.
## 7. Propriétés attendues ## 7. Replay, invalidation et pagination (`pre.004`)
`pre.004` ne modifie pas les 16 tables ; il ferme les chemins de repository au-dessus delles.
### 7.1 Instructions top-level et CPI
Les deux catégories restent physiquement séparées. La lecture de replay construit une union logique avec un scope explicite et un ordre stable `(slot, signature, scope_rank, instruction_path)`. Le curseur de page reprend cette clé ; aucun `OFFSET` nappartient au contrat durable.
### 7.2 Invalidation descendante
Un remplacement N2 Core dune signature invalide dans la même transaction ses descendants N3 decode/coverage/materialization et les ledgers concernés. Un remplacement decode invalide ses sorties de matérialisation pour lidentité exacte decoder name/version/input. Les préfixes de ledger sont comparés littéralement, sans `LIKE`, afin que `_` ne puisse pas agir comme wildcard.
Le lifecycle instructionnel est ensuite recomposé depuis les descendants réellement courants. Une ancienne matérialisation produite par une version antérieure du decoder ne suffit pas à déclarer le nouvel input matérialisé.
### 7.3 Account observation -> Core account state
Le flux N1 -> N2 réservé en `pre.003` devient opérationnel. La sélection est bornée et version-aware ; la présence dun ancien Core state ne masque pas une nouvelle version du normalizer. `lamports` et `rent_epoch` conservent tout le domaine `u64` via `NUMERIC(20,0)` et les conversions vers `BIGINT` échouent explicitement hors domaine signé.
### 7.4 Bornes et instrumentation
Les lectures interactives publiques sont plafonnées à 500 lignes. Les campagnes de processing Core/decode/account-state utilisent des batches distincts plafonnés à 1 000. Les opérations PostgreSQL réécrites tracent `action`, durée, lignes et outcome sous `ks-store`/`ks-store.pg`, sans SQL brut ni valeur bindée.
## 8. Propriétés attendues
- initialisation idempotente ; - initialisation idempotente ;
- pagination bornée ; - pagination bornée ;
@@ -164,6 +186,6 @@ Le baseline `pre.003` contient **236 ressources SQL atomiques** et **75 index at
Les index non uniques restent des optimisations physiques évolutives. Les noms/colonnes/types/nullabilités/PK/FK/uniques/checks/sémantiques du baseline N1-N3 seront figés séparément lors de la tranche de gel. Les index non uniques restent des optimisations physiques évolutives. Les noms/colonnes/types/nullabilités/PK/FK/uniques/checks/sémantiques du baseline N1-N3 seront figés séparément lors de la tranche de gel.
## 8. Données de test ## 9. Données de test
Les fixtures privées, bases locales et preuves temporaires ne font pas partie des livraisons. Les matrices contractuelles partagées restent sous `test-fixtures/contract-matrices/` lorsqu'elles sont nécessaires aux tests. Les fixtures privées, bases locales et preuves temporaires ne font pas partie des livraisons. Les matrices contractuelles partagées restent sous `test-fixtures/contract-matrices/` lorsqu'elles sont nécessaires aux tests.

View File

@@ -1,5 +1,5 @@
<!-- file: docs/guides/POSTGRES_STORAGE.md --> <!-- file: docs/guides/POSTGRES_STORAGE.md -->
<!-- version: 6 --> <!-- version: 7 -->
# Guide PostgreSQL et contrats de stockage # Guide PostgreSQL et contrats de stockage
@@ -79,12 +79,27 @@ Le corpus `pre.003` contient actuellement :
- 59 créations d'index explicites ; - 59 créations d'index explicites ;
- 16 ressources `TRUNCATE` ; - 16 ressources `TRUNCATE` ;
- 16 ressources `DROP` ; - 16 ressources `DROP` ;
- soit **236 ressources SQL**. - quatre index supplémentaires justifiés par le replay/invalidation `pre.004` ;
- soit **240 ressources SQL**.
PostgreSQL attend **75 index** au total dans son diagnostic : 59 index explicites plus les 16 index de clé primaire créés par PostgreSQL. PostgreSQL attend désormais **79 index** au total dans son diagnostic : 63 index explicites plus les 16 index de clé primaire créés par PostgreSQL.
Les index non uniques restent des optimisations physiques et ne font pas partie du gel logique N1-N3. Les index non uniques restent des optimisations physiques et ne font pas partie du gel logique N1-N3.
## Repositories `pre.004`
La tranche `pre.004` conserve les 16 tables et ferme les comportements suivants :
- replay logique commun top-level/CPI avec scope explicite et pagination keyset ;
- lifecycle symétrique top-level/CPI ;
- invalidation N2 Core -> N3 lors dun remplacement de signature ;
- invalidation decode -> matérialisation pour lidentité decoder exacte ;
- normalisation versionnée `k_sol_obs_account_observations` -> `k_sol_core_account_states` ;
- limites interactives à 500 et batches de processing à 1 000 ;
- traces fonctionnelles `TRACE ks-store` avec durée/lignes/outcome, sans bind values.
Les quatre nouveaux index couvrent lordre de replay CPI et les recherches dinvalidation/descendants dans coverage/materialization. Leur utilité finale doit encore être confirmée par des plans PostgreSQL réels avant le gel ; les index non uniques restent une optimisation physique.
## Statut du schéma ## Statut du schéma
La façade n'utilise plus `_sqlx_migrations` comme preuve du baseline actuel. Le statut est dérivé du contrat réellement observé : La façade n'utilise plus `_sqlx_migrations` comme preuve du baseline actuel. Le statut est dérivé du contrat réellement observé :

View File

@@ -1,5 +1,5 @@
<!-- file: docs/plans/V0_5_3_KS_STORE_NORMALIZATION_PLAN.md --> <!-- file: docs/plans/V0_5_3_KS_STORE_NORMALIZATION_PLAN.md -->
<!-- version: 8 --> <!-- version: 9 -->
# Plan `0.5.3` — audit et normalisation de `ks-store` # Plan `0.5.3` — audit et normalisation de `ks-store`
@@ -1609,3 +1609,73 @@ Les autres metadata transactionnelles auditées (`fee`, rewards, compute units,
- les projections N4 metadata/trading. - les projections N4 metadata/trading.
Ces points restent dans les tranches suivantes et peuvent décaler le premier candidat de clôture au-delà de `pre.007`. Ces points restent dans les tranches suivantes et peuvent décaler le premier candidat de clôture au-delà de `pre.007`.
## 34. État réalisé de `0.5.3-pre.004`
`pre.004` ferme la tranche repositories/replay sans ajouter de table et sans déclarer encore le gel formel N1/N2/N3.
### 34.1 Replay top-level + CPI
Les instructions top-level et CPI restent dans leurs deux tables physiques. Les repositories construisent désormais une lecture logique commune avec `CoreInstructionScope::{TopLevel, Inner}`. Le replay contextuel N2 -> N3 fonctionne sur les deux scopes ; le lifecycle est mis à jour dans la table physique correspondant au chemin dinstruction.
Lordre de page durable est :
```text
(slot, signature, scope_rank, instruction_path)
```
`PageRequest` abandonne `offset` et utilise un curseur borné. Les lectures interactives sont plafonnées à 500 lignes.
### 34.2 Batches de processing
Les campagnes Core extraction, decode et account-state utilisent des limites propres, plafonnées à 1 000 entrées par batch. Elles ne réutilisent pas la limite interactive de 500 et aucune surface publique ne conserve lancien plafond 100 000.
### 34.3 Invalidation descendante
Le remplacement dun graphe Core invalide, dans la même transaction, les descendants decode/coverage/materialization et les ledgers de la signature avant reconstruction N2.
Le remplacement dun résultat decode invalide les matérialisations appartenant à lidentité exacte `(decoder_name, decoder_version, decode_input_key)`. Les clés de ledger sont comparées par préfixe littéral avec `LEFT(...)=prefix`, jamais avec `LIKE`.
Après persistence, le lifecycle instructionnel est recalculé depuis les descendants courants. Une sortie produite par une ancienne version du decoder ne peut donc pas faire apparaître la nouvelle version comme matérialisée.
### 34.4 Account observation -> account state
Le couple réservé en `pre.003` devient opérationnel :
```text
k_sol_obs_account_observations
-> account_state_normalization ledger
-> k_sol_core_account_states
```
La sélection transporte le nom/version du normalizer ; une nouvelle version peut rejouer les observations même si un Core state dune version précédente existe. Le hash dinput assure lidempotence de la version courante. Les valeurs `lamports`/`rent_epoch` conservent le domaine `u64` complet via `NUMERIC(20,0)`.
### 34.5 Index et ressources SQL
Aucune table ni contrainte logique nest ajoutée. Quatre index non uniques sont ajoutés pour les nouveaux chemins :
- ordre de replay des CPI par slot/signature/path ;
- lookup coverage par signature/path ;
- descendants materialized outputs par decode input ;
- descendants materialized outputs par signature/input.
Le corpus passe de 236 à **240 ressources SQL**, avec **63 index explicites / 79 index attendus** en comptant les 16 clés primaires. Les index restent ajustables avant gel sur la base des plans `EXPLAIN`.
### 34.6 Instrumentation PostgreSQL
Les opérations de repository réécrites émettent des événements `TRACE` sous le target canonique `ks-store` avec :
```text
backend="postgres"
domain="ks-store.pg"
action
elapsed_ms
rows
outcome
```
Le statement logging automatique SQLx reste désactivé. Les traces normales ne publient ni SQL brut, ni valeurs bindées, ni secret de connexion.
### 34.7 Reste avant gel
La validation PostgreSQL réelle des nouveaux chemins, les plans `EXPLAIN`, la présentation desktop finale et le snapshot machine-readable restent dans les tranches suivantes. `pre.004` stabilise les repositories candidats ; il ne déclare pas encore N1/N2/N3 frozen.

View File

@@ -1,8 +1,13 @@
<!-- file: kb-app-demo-desktop/CHANGELOG.md --> <!-- file: kb-app-demo-desktop/CHANGELOG.md -->
<!-- version: 65 --> <!-- version: 66 -->
# CHANGELOG — kb-app-demo-desktop # CHANGELOG — kb-app-demo-desktop
## `0.5.3-pre.004`
- aligne la limite maximale de la fenêtre technique de candidats replay Store sur la nouvelle borne interactive backend-agnostique de 500 lignes ;
- ne réalise aucune refonte fonctionnelle de lUI Store : celle-ci reste réservée à la tranche consommateurs `pre.005`.
## `0.5.3-pre.003` ## `0.5.3-pre.003`
- adapte les vues de replay/journaux aux contrats N3 `MaterializedOutput*` et au journal `k_sol_mat_outputs` sans réintroduire de dépendance PostgreSQL dans les DTO desktop ; - adapte les vues de replay/journaux aux contrats N3 `MaterializedOutput*` et au journal `k_sol_mat_outputs` sans réintroduire de dépendance PostgreSQL dans les DTO desktop ;

View File

@@ -1,5 +1,5 @@
<!-- file: kb-app-demo-desktop/frontend/demo_store_replay_candidates.html --> <!-- file: kb-app-demo-desktop/frontend/demo_store_replay_candidates.html -->
<!-- version: 16 --> <!-- version: 17 -->
<!DOCTYPE html> <!DOCTYPE html>
<html lang="fr"> <html lang="fr">
@@ -131,7 +131,7 @@
</div> </div>
<div class="col-6 col-md-3 col-xl-1"> <div class="col-6 col-md-3 col-xl-1">
<label for="transactionLimitInput" class="form-label">Limite</label> <label for="transactionLimitInput" class="form-label">Limite</label>
<input id="transactionLimitInput" class="form-control" type="number" min="1" max="100000" step="1" value="500" /> <input id="transactionLimitInput" class="form-control" type="number" min="1" max="500" step="1" value="500" />
</div> </div>
<div class="col-6 col-md-3 col-xl-2"> <div class="col-6 col-md-3 col-xl-2">
<div class="form-check form-switch mb-2"> <div class="form-check form-switch mb-2">
@@ -206,7 +206,7 @@
</div> </div>
<div class="col-6 col-lg-1"> <div class="col-6 col-lg-1">
<label for="programLimitInput" class="form-label">Limite</label> <label for="programLimitInput" class="form-label">Limite</label>
<input id="programLimitInput" class="form-control" type="number" min="1" max="100000" step="1" value="500" /> <input id="programLimitInput" class="form-control" type="number" min="1" max="500" step="1" value="500" />
</div> </div>
<div class="col-12 col-lg-2"> <div class="col-12 col-lg-2">
<div class="d-grid gap-2"> <div class="d-grid gap-2">
@@ -264,7 +264,7 @@
</div> </div>
<div class="col-6 col-lg-2"> <div class="col-6 col-lg-2">
<label for="mintLimitInput" class="form-label">Limite</label> <label for="mintLimitInput" class="form-label">Limite</label>
<input id="mintLimitInput" class="form-control" type="number" min="1" max="100000" step="1" value="500" /> <input id="mintLimitInput" class="form-control" type="number" min="1" max="500" step="1" value="500" />
</div> </div>
<div class="col-12 col-lg-3"> <div class="col-12 col-lg-3">
<div class="d-grid d-sm-flex gap-2"> <div class="d-grid d-sm-flex gap-2">
@@ -322,7 +322,7 @@
</div> </div>
<div class="col-6 col-lg-2"> <div class="col-6 col-lg-2">
<label for="ownerLimitInput" class="form-label">Limite</label> <label for="ownerLimitInput" class="form-label">Limite</label>
<input id="ownerLimitInput" class="form-control" type="number" min="1" max="100000" step="1" value="500" /> <input id="ownerLimitInput" class="form-control" type="number" min="1" max="500" step="1" value="500" />
</div> </div>
<div class="col-12 col-lg-3"> <div class="col-12 col-lg-3">
<div class="d-grid d-sm-flex gap-2"> <div class="d-grid d-sm-flex gap-2">
@@ -380,7 +380,7 @@
</div> </div>
<div class="col-6 col-lg-2"> <div class="col-6 col-lg-2">
<label for="accountLimitInput" class="form-label">Limite</label> <label for="accountLimitInput" class="form-label">Limite</label>
<input id="accountLimitInput" class="form-control" type="number" min="1" max="100000" step="1" value="500" /> <input id="accountLimitInput" class="form-control" type="number" min="1" max="500" step="1" value="500" />
</div> </div>
<div class="col-12 col-lg-3"> <div class="col-12 col-lg-3">
<div class="d-grid d-sm-flex gap-2"> <div class="d-grid d-sm-flex gap-2">

View File

@@ -1,5 +1,5 @@
// file: kb-app-demo-desktop/frontend/ts/demo_store_replay_candidates.ts // file: kb-app-demo-desktop/frontend/ts/demo_store_replay_candidates.ts
// version: 11 // version: 12
import * as bootstrap from "bootstrap"; import * as bootstrap from "bootstrap";
import "simplebar"; import "simplebar";
@@ -94,7 +94,7 @@ let mintTable: Api<DemoStoreReplayEntityRow> | null = null;
let ownerTable: Api<DemoStoreReplayEntityRow> | null = null; let ownerTable: Api<DemoStoreReplayEntityRow> | null = null;
let accountTable: Api<DemoStoreReplayEntityRow> | null = null; let accountTable: Api<DemoStoreReplayEntityRow> | null = null;
let busyOperationCount = 0; let busyOperationCount = 0;
let maximumReplayCandidateLimit = 100_000; let maximumReplayCandidateLimit = 500;
function element<T extends HTMLElement>(selector: string): T { function element<T extends HTMLElement>(selector: string): T {
const value = document.querySelector<T>(selector); const value = document.querySelector<T>(selector);

View File

@@ -1,8 +1,15 @@
<!-- file: ks-pipeline/CHANGELOG.md --> <!-- file: ks-pipeline/CHANGELOG.md -->
<!-- version: 37 --> <!-- version: 38 -->
# CHANGELOG — ks-pipeline # CHANGELOG — ks-pipeline
## `0.5.3-pre.004`
- ajoute lorchestrateur générique `account_state_normalization` pour le replay N1 observation de compte -> N2 état Core, avec identité/version de normalizer, input hash, force replay et persistance déchec bornée ;
- force la sélection effective à inclure les inputs déjà courants uniquement lorsquun force replay est explicitement demandé ;
- borne les campagnes Core extraction et decode à 1 000 entrées conformément aux nouveaux contrats de processing `ks-store` ;
- consomme le replay Core top-level+CPI et ses contrats cursorisés sans introduire de sémantique Anchor/IDL dans le pipeline générique.
## `0.5.3-pre.003` ## `0.5.3-pre.003`
- corrige le premier delta `pre.003` en utilisant le nom canonique exporté `MdCanonicalInnerInstructionGroup` pour la reconstruction des chemins CPI. - corrige le premier delta `pre.003` en utilisant le nom canonique exporté `MdCanonicalInnerInstructionGroup` pour la reconstruction des chemins CPI.

View File

@@ -0,0 +1,405 @@
// file: ks-pipeline/src/account_state_normalization.rs
// version: 1
//! Generic Solana account-observation to Core account-state normalization pipeline.
use sha2::Digest; // rust-rules: trait-import
/// Stable processor name used by generic account-state normalization.
pub const ACCOUNT_STATE_NORMALIZER_NAME: &str = "account_observation_to_core";
/// Current generic account-state normalizer implementation version.
pub const ACCOUNT_STATE_NORMALIZER_VERSION: &str = "1";
/// Bounded generic account-state normalization request.
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct AccountStateNormalizationRequest {
/// Source observation selection.
pub selection: ks_store::AccountStateSelectionFilter,
/// Forces replacement even when the same processor version and input hash already succeeded.
pub force_replay: bool,
}
impl crate::AccountStateNormalizationRequest {
/// Builds a request bound to the active generic account-state normalizer identity.
pub fn new(
account_keys: std::vec::Vec<std::string::String>,
min_slot: std::option::Option<u64>,
max_slot: std::option::Option<u64>,
force_replay: bool,
limit: u32,
) -> ks_core::Result<Self> {
let selection_result = ks_store::AccountStateSelectionFilter::new(
crate::ACCOUNT_STATE_NORMALIZER_NAME,
crate::ACCOUNT_STATE_NORMALIZER_VERSION,
account_keys,
min_slot,
max_slot,
force_replay,
limit,
);
let selection = match selection_result {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
return std::result::Result::Ok(Self { selection, force_replay });
}
}
/// Summary returned by one generic account-state normalization campaign.
#[derive(Clone, Debug, Default, Eq, PartialEq)]
pub struct AccountStateNormalizationSummary {
/// Complete observations selected from N1.
pub selected: u64,
/// Core account states inserted or replaced in N2.
pub normalized: u64,
/// Inputs skipped because the same processor version and hash are already current.
pub skipped: u64,
/// Inputs that failed normalization or persistence.
pub failed: u64,
}
/// Executes one bounded replayable N1 account-observation to N2 Core account-state campaign.
pub async fn execute_account_state_normalization<S>(
store: &S,
request: &crate::AccountStateNormalizationRequest,
) -> ks_core::Result<crate::AccountStateNormalizationSummary>
where
S: ks_store::AccountStateStore + Sync,
{
if request.selection.processor_name != crate::ACCOUNT_STATE_NORMALIZER_NAME
|| request.selection.processor_version != crate::ACCOUNT_STATE_NORMALIZER_VERSION
{
return std::result::Result::Err(ks_core::Error::invalid_state(
"account-state selection processor identity does not match the active normalizer",
));
}
let mut effective_selection = request.selection.clone();
effective_selection.include_current = request.force_replay;
let observations_result =
store.list_account_observations_for_normalization(&effective_selection).await;
let observations = match observations_result {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let mut summary = crate::AccountStateNormalizationSummary {
selected: match u64::try_from(observations.len()) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => {
return std::result::Result::Err(ks_core::Error::invalid_state(format!(
"account-state selection length conversion failed: {error}"
)));
},
},
..crate::AccountStateNormalizationSummary::default()
};
tracing::info!(target: crate::TRACING_TARGET, action = "execute_account_state_normalization", selected = summary.selected, force_replay = request.force_replay, processor_name = crate::ACCOUNT_STATE_NORMALIZER_NAME, processor_version = crate::ACCOUNT_STATE_NORMALIZER_VERSION, "account-state normalization campaign started");
for observation in &observations {
let identity_result = ledger_identity_from_observation(observation);
let identity = match identity_result {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
if !request.force_replay {
let current_result = store.is_account_state_current(&identity).await;
match current_result {
std::result::Result::Ok(true) => {
summary.skipped += 1;
continue;
},
std::result::Result::Ok(false) => {},
std::result::Result::Err(error) => return std::result::Result::Err(error),
}
}
let state_result = core_state_from_observation(observation);
let state = match state_result {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => {
let failure_result =
persist_normalization_failure(store, observation.id, identity, &error).await;
if let std::result::Result::Err(persistence_error) = failure_result {
return std::result::Result::Err(persistence_error);
}
summary.failed += 1;
continue;
},
};
let bundle =
ks_store::AccountStatePersistenceBundle { ledger_identity: identity.clone(), state };
let persistence_result = store.persist_account_state(&bundle, request.force_replay).await;
match persistence_result {
std::result::Result::Ok(outcome) => {
if outcome.inserted_count > 0 || outcome.updated_count > 0 {
summary.normalized += 1;
} else {
summary.skipped += 1;
}
},
std::result::Result::Err(error) => {
let failure_result =
persist_normalization_failure(store, observation.id, identity, &error).await;
if let std::result::Result::Err(persistence_error) = failure_result {
return std::result::Result::Err(persistence_error);
}
summary.failed += 1;
},
}
}
tracing::info!(target: crate::TRACING_TARGET, action = "execute_account_state_normalization", selected = summary.selected, normalized = summary.normalized, skipped = summary.skipped, failed = summary.failed, force_replay = request.force_replay, processor_name = crate::ACCOUNT_STATE_NORMALIZER_NAME, processor_version = crate::ACCOUNT_STATE_NORMALIZER_VERSION, "account-state normalization campaign completed");
return std::result::Result::Ok(summary);
}
fn ledger_identity_from_observation(
observation: &ks_store::AccountObservationRow,
) -> ks_core::Result<ks_store::ProcessingLedgerIdentity> {
let hash = canonical_account_observation_hash(observation);
return ks_store::ProcessingLedgerIdentity::new(
ks_store::ACCOUNT_STATE_NORMALIZATION_STAGE,
crate::ACCOUNT_STATE_NORMALIZER_NAME,
crate::ACCOUNT_STATE_NORMALIZER_VERSION,
observation.observation_key.clone(),
hash,
);
}
fn canonical_account_observation_hash(
observation: &ks_store::AccountObservationRow,
) -> std::string::String {
let owner = match observation.owner.as_deref() {
std::option::Option::Some(value) => value,
std::option::Option::None => "",
};
let lamports = match observation.lamports.as_deref() {
std::option::Option::Some(value) => value,
std::option::Option::None => "",
};
let rent_epoch = match observation.rent_epoch.as_deref() {
std::option::Option::Some(value) => value,
std::option::Option::None => "",
};
let data_hash = match observation.data_hash.as_deref() {
std::option::Option::Some(value) => value,
std::option::Option::None => "",
};
let space = match observation.space {
std::option::Option::Some(value) => value,
std::option::Option::None => 0_i64,
};
let executable = match observation.executable {
std::option::Option::Some(value) => value,
std::option::Option::None => false,
};
let mut hasher = sha2::Sha256::new();
for value in [
observation.observation_key.as_str(),
observation.account_key.as_str(),
owner,
lamports,
rent_epoch,
data_hash,
] {
hasher.update(value.as_bytes());
hasher.update([0_u8]);
}
hasher.update(observation.context_slot.to_le_bytes());
hasher.update(space.to_le_bytes());
hasher.update([u8::from(executable)]);
let digest = hasher.finalize();
let mut output = std::string::String::with_capacity(64);
for byte in digest {
output.push_str(format!("{byte:02x}").as_str());
}
return output;
}
fn core_state_from_observation(
observation: &ks_store::AccountObservationRow,
) -> ks_core::Result<ks_store::CoreAccountStateInsert> {
let slot_result = u64::try_from(observation.context_slot);
let slot = match slot_result {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => {
return std::result::Result::Err(ks_core::Error::invalid_state(format!(
"account observation context slot is negative: {error}"
)));
},
};
let owner = match &observation.owner {
std::option::Option::Some(value) => value.clone(),
std::option::Option::None => {
return std::result::Result::Err(ks_core::Error::invalid_state(
"account observation owner is missing",
));
},
};
let lamports_result = parse_u64_text(observation.lamports.as_deref(), "lamports");
let lamports = match lamports_result {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let executable = match observation.executable {
std::option::Option::Some(value) => value,
std::option::Option::None => {
return std::result::Result::Err(ks_core::Error::invalid_state(
"account observation executable flag is missing",
));
},
};
let rent_epoch_result = parse_u64_text(observation.rent_epoch.as_deref(), "rent epoch");
let rent_epoch = match rent_epoch_result {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let space_sql = match observation.space {
std::option::Option::Some(value) => value,
std::option::Option::None => {
return std::result::Result::Err(ks_core::Error::invalid_state(
"account observation space is missing",
));
},
};
let space_result = u64::try_from(space_sql);
let space = match space_result {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => {
return std::result::Result::Err(ks_core::Error::invalid_state(format!(
"account observation space is negative: {error}"
)));
},
};
let data_base64 = match &observation.data_base64 {
std::option::Option::Some(value) => value.clone(),
std::option::Option::None => {
return std::result::Result::Err(ks_core::Error::invalid_state(
"account observation data is missing",
));
},
};
let data_hash = match &observation.data_hash {
std::option::Option::Some(value) => value.clone(),
std::option::Option::None => {
return std::result::Result::Err(ks_core::Error::invalid_state(
"account observation data hash is missing",
));
},
};
let state = ks_store::CoreAccountStateInsert {
source_observation_id: observation.id,
account_key: observation.account_key.clone(),
slot,
owner,
lamports,
executable,
rent_epoch,
space,
data_base64,
data_hash,
};
let validation_result = state.validate();
if let std::result::Result::Err(error) = validation_result {
return std::result::Result::Err(error);
}
return std::result::Result::Ok(state);
}
fn parse_u64_text(value: std::option::Option<&str>, label: &str) -> ks_core::Result<u64> {
let text = match value {
std::option::Option::Some(value) => value,
std::option::Option::None => {
return std::result::Result::Err(ks_core::Error::invalid_state(format!(
"account observation {label} is missing"
)));
},
};
let parse_result = text.parse::<u64>();
return match parse_result {
std::result::Result::Ok(value) => std::result::Result::Ok(value),
std::result::Result::Err(error) => std::result::Result::Err(ks_core::Error::invalid_state(
format!("account observation {label} is invalid: {error}"),
)),
};
}
async fn persist_normalization_failure<S>(
store: &S,
source_observation_id: i64,
identity: ks_store::ProcessingLedgerIdentity,
error: &ks_core::Error,
) -> ks_core::Result<()>
where
S: ks_store::AccountStateStore + Sync,
{
let message = bounded_error_message(error.to_string().as_str());
let failure = ks_store::AccountStateNormalizationFailure {
source_observation_id,
ledger_identity: identity,
error_code: "account_state_normalization_failed".to_string(),
error_message: message,
};
let persistence_result = store.mark_account_state_failed(&failure).await;
return match persistence_result {
std::result::Result::Ok(_outcome) => std::result::Result::Ok(()),
std::result::Result::Err(persistence_error) => std::result::Result::Err(persistence_error),
};
}
fn bounded_error_message(value: &str) -> std::string::String {
const MAX_ERROR_MESSAGE_CHARS: usize = 512;
return value.chars().take(MAX_ERROR_MESSAGE_CHARS).collect();
}
#[cfg(test)]
mod tests {
#[test]
fn request_constructor_binds_active_processor_identity() {
let request_result = crate::AccountStateNormalizationRequest::new(
std::vec::Vec::new(),
std::option::Option::None,
std::option::Option::None,
true,
10,
);
let request = match request_result {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => panic!("unexpected request error: {error}"),
};
assert_eq!(request.selection.processor_name, crate::ACCOUNT_STATE_NORMALIZER_NAME);
assert_eq!(request.selection.processor_version, crate::ACCOUNT_STATE_NORMALIZER_VERSION);
assert!(request.selection.include_current);
assert!(request.force_replay);
}
#[test]
fn request_constructor_excludes_current_inputs_without_force_replay() {
let request_result = crate::AccountStateNormalizationRequest::new(
std::vec::Vec::new(),
std::option::Option::None,
std::option::Option::None,
false,
10,
);
let request = match request_result {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => panic!("unexpected request error: {error}"),
};
assert!(!request.selection.include_current);
assert!(!request.force_replay);
}
#[test]
fn unsigned_decimal_parser_accepts_full_u64_range() {
assert_eq!(
super::parse_u64_text(std::option::Option::Some("18446744073709551615"), "lamports"),
std::result::Result::Ok(u64::MAX)
);
}
#[test]
fn unsigned_decimal_parser_rejects_negative_values() {
assert!(super::parse_u64_text(std::option::Option::Some("-1"), "lamports").is_err());
}
#[test]
fn error_messages_are_bounded() {
assert_eq!(super::bounded_error_message("x".repeat(600).as_str()).len(), 512);
}
}

View File

@@ -1,5 +1,5 @@
// file: ks-pipeline/src/core_extraction.rs // file: ks-pipeline/src/core_extraction.rs
// version: 13 // version: 14
//! Canonical Solana transaction to normalized core extraction pipeline. //! Canonical Solana transaction to normalized core extraction pipeline.
@@ -51,10 +51,11 @@ pub struct CoreExtractionRequest {
impl CoreExtractionRequest { impl CoreExtractionRequest {
/// Validates campaign bounds and source values. /// Validates campaign bounds and source values.
pub fn validate(&self) -> ks_core::Result<()> { pub fn validate(&self) -> ks_core::Result<()> {
if self.limit == 0 { if self.limit == 0 || self.limit > ks_store::MAX_CORE_EXTRACTION_SELECTION_ROWS {
return std::result::Result::Err(ks_core::Error::config( return std::result::Result::Err(ks_core::Error::config(format!(
"core extraction limit must be greater than zero", "core extraction limit must be between 1 and {}",
)); ks_store::MAX_CORE_EXTRACTION_SELECTION_ROWS
)));
} }
if self.max_concurrent_extractions == 0 { if self.max_concurrent_extractions == 0 {
return std::result::Result::Err(ks_core::Error::config( return std::result::Result::Err(ks_core::Error::config(

View File

@@ -1,5 +1,5 @@
// file: ks-pipeline/src/decode_replay.rs // file: ks-pipeline/src/decode_replay.rs
// version: 4 // version: 5
//! Common contextual instruction decode and optional materialization pipeline. //! Common contextual instruction decode and optional materialization pipeline.
@@ -59,6 +59,12 @@ impl DecodeReplayRequest {
"decode replay concurrency must be greater than zero", "decode replay concurrency must be greater than zero",
)); ));
} }
if self.selection.limit == 0 || self.selection.limit > ks_store::MAX_DECODE_SELECTION_ROWS {
return std::result::Result::Err(ks_core::Error::config(format!(
"decode replay selection limit must be between 1 and {}",
ks_store::MAX_DECODE_SELECTION_ROWS
)));
}
if self.force_replay if self.force_replay
&& self.selection.signatures.is_empty() && self.selection.signatures.is_empty()
&& !self.force_replay_all_matching && !self.force_replay_all_matching

View File

@@ -1,5 +1,5 @@
// file: ks-pipeline/src/lib.rs // file: ks-pipeline/src/lib.rs
// version: 22 // version: 23
#![forbid(unsafe_code)] #![forbid(unsafe_code)]
#![deny(unreachable_pub)] #![deny(unreachable_pub)]
@@ -7,6 +7,7 @@
//! Pipeline orchestration boundary. //! Pipeline orchestration boundary.
mod account_state_normalization;
mod backfill; mod backfill;
mod constants; mod constants;
mod core_extraction; mod core_extraction;
@@ -30,6 +31,16 @@ mod spl_token_2022_proof_orchestration;
mod spl_token_2022_stateful; mod spl_token_2022_stateful;
mod spl_token_stateful; mod spl_token_stateful;
/// Stable processor name used by generic account-state normalization.
pub use self::account_state_normalization::ACCOUNT_STATE_NORMALIZER_NAME;
/// Current generic account-state normalizer implementation version.
pub use self::account_state_normalization::ACCOUNT_STATE_NORMALIZER_VERSION;
/// Bounded generic account-state normalization request.
pub use self::account_state_normalization::AccountStateNormalizationRequest;
/// Summary returned by one generic account-state normalization campaign.
pub use self::account_state_normalization::AccountStateNormalizationSummary;
/// Executes one bounded replayable N1 account-observation to N2 Core account-state campaign.
pub use self::account_state_normalization::execute_account_state_normalization;
/// Address category used by one targeted backfill campaign. /// Address category used by one targeted backfill campaign.
pub use self::backfill::BackfillAddressKind; pub use self::backfill::BackfillAddressKind;
/// Chronological direction relative to one anchor signature. /// Chronological direction relative to one anchor signature.

View File

@@ -1,8 +1,46 @@
<!-- file: ks-store/CHANGELOG.md --> <!-- file: ks-store/CHANGELOG.md -->
<!-- version: 29 --> <!-- version: 33 -->
# CHANGELOG — ks-store # CHANGELOG — ks-store
## `0.5.3-pre.004`
### Fix `pre.004-delta-fix-003`
- corrige deux canaris de source auto-référentiels : leurs assertions négatives analysaient aussi le bloc `#[cfg(test)]` qui contenait lui-même la chaîne interdite ;
- limite désormais les vérifications de source à la partie production précédant `#[cfg(test)]` dans `account_state_queries.rs` et `invalidation_queries.rs` ;
- ne modifie aucune requête PostgreSQL ni aucun comportement runtime.
### Fix `pre.004-delta-fix-002`
- ajoute les annotations explicites `std::string::String` sur `origin_text` et `status_text` dans le mapper PostgreSQL des observations de compte ;
- corrige les deux erreurs `E0282` d'inférence générique `read_row<T>()` apparues après la suppression de l'opérateur `?`, sans modifier le contrat ni le comportement du mapper.
### Fix `pre.004-delta-fix-001`
- remplace les 53 utilisations accidentelles de l'opérateur `?` introduites dans les nouveaux mappers PostgreSQL account-state/Core replay par une propagation explicite `match` conforme aux règles du workspace ;
- ne modifie ni les contrats publics, ni le SQL, ni les règles de replay/pagination/invalidation de `pre.004`.
### Repositories et replay
- rend le replay N2 -> N3 symétrique pour les instructions top-level et CPI via une union logique `CoreInstructionScope::{TopLevel, Inner}` sans fusionner les deux tables physiques ;
- remplace la pagination `OFFSET` du contrat Core par un curseur stable `(slot, signature, scope_rank, instruction_path)` et borne les lectures interactives à 500 lignes ;
- borne séparément les batches Core extraction, decode et account-state à 1 000 entrées afin quaucune API publique ne conserve les anciennes limites massives ;
- rend opérationnel le flux générique N1 `account observation` -> N2 `account state` avec sélection version-aware, idempotence par processing ledger, force replay et conservation du domaine `u64` pour lamports/rent epoch.
### Invalidation et lifecycle
- invalide les descendants N3 decode/coverage/materialization dune signature dans la même transaction lorsquun graphe Core est remplacé ;
- invalide les sorties de matérialisation dun decode remplacé pour lidentité exacte decoder name/version/input et utilise des préfixes de ledger littéraux au lieu de `LIKE` ;
- recalcule le lifecycle top-level/CPI depuis les descendants réellement courants et empêche une matérialisation dune ancienne version de decoder de rendre la nouvelle version artificiellement `materialized`.
### Index, bornes et observabilité
- ajoute quatre index non uniques dédiés à lordre de replay CPI et aux recherches de descendants coverage/materialization ; le corpus passe à 240 ressources SQL, 63 index explicites et 79 index attendus avec les PK ;
- ajoute les canaris de conversion `u64`/`BIGINT` et les limites de batch ;
- instrumente les opérations PostgreSQL réécrites sous `target: crate::TRACING_TARGET`, `backend="postgres"`, `domain="ks-store.pg"` avec `action`, durée, lignes et résultat, sans SQL brut ni valeur bindée.
## `0.5.3-pre.003` ## `0.5.3-pre.003`
### Fix `pre.003-delta-fix-005` ### Fix `pre.003-delta-fix-005`

View File

@@ -1,5 +1,5 @@
<!-- file: ks-store/README.md --> <!-- file: ks-store/README.md -->
<!-- version: 7 --> <!-- version: 8 -->
# ks-store # ks-store
@@ -11,10 +11,10 @@ La crate expose :
- `Store` et `StoreOpenOptions` comme façade d'ouverture et handle persistant backend-agnostique ; - `Store` et `StoreOpenOptions` comme façade d'ouverture et handle persistant backend-agnostique ;
- les DTO et entités N1 raw/acquisition, N2 Core, N3 decode/materialization et ledger ; - les DTO et entités N1 raw/acquisition, N2 Core, N3 decode/materialization et ledger ;
- les traits async `StoreHealthStore`, `RawTransactionStore`, `CoreTransactionStore`, `CoreExtractionStore` et `DecodePipelineStore` ; - les traits async `StoreHealthStore`, `RawTransactionStore`, `CoreTransactionStore`, `AccountStateStore`, `CoreExtractionStore` et `DecodePipelineStore` ;
- les filtres et résultats de replay sans préfixe backend ; - les filtres et résultats de replay sans préfixe backend ;
- des diagnostics et résumés runtime/init sanitisés ne révélant ni secret ni nom physique d'objet ; - des diagnostics et résumés runtime/init sanitisés ne révélant ni secret ni nom physique d'objet ;
- des transactions atomiques pour extraction, décodage et matérialisation. - des transactions atomiques pour normalisation détat de compte, extraction Core, décodage et matérialisation.
## Frontière backend ## Frontière backend
@@ -40,7 +40,20 @@ Le baseline candidat au gel comprend **16 tables N1-N3** :
Les ressources PostgreSQL sont rangées sous `migrations/postgres/` et suivent la règle **une instruction SQL par fichier**. L'orchestrateur privé applique tables, contraintes et index dans une transaction unique et refuse explicitement un schéma historique contenant encore des objets `kb_sol_*`. Les ressources PostgreSQL sont rangées sous `migrations/postgres/` et suivent la règle **une instruction SQL par fichier**. L'orchestrateur privé applique tables, contraintes et index dans une transaction unique et refuse explicitement un schéma historique contenant encore des objets `kb_sol_*`.
Le baseline comporte actuellement **236 ressources SQL atomiques** et **75 index attendus**, comptés sans exposer leurs noms dans l'API publique. Le baseline conserve **16 tables** et comporte désormais **240 ressources SQL atomiques** et **79 index attendus** après la revue des chemins de replay `pre.004`, comptés sans exposer leurs noms dans l'API publique.
## Repositories et replay `0.5.3-pre.004`
`pre.004` ferme la symétrie opérationnelle des repositories sans ajouter de table :
- les instructions top-level et CPI restent séparées physiquement mais sont lues par une union logique commune pour le replay ;
- le lifecycle decode/materialization est recalculé pour les deux scopes et tient compte de lidentité/version du decoder courant ;
- le remplacement dun graphe Core invalide ses descendants N3 dans la même transaction ; un remplacement decode invalide les sorties de matérialisation de ce decoder/input ;
- `k_sol_obs_account_observations` -> `k_sol_core_account_states` devient un replay N1 -> N2 opérationnel, versionné par processing ledger ;
- les pages interactives utilisent un curseur stable et sont bornées à 500 lignes ; les batches de processing Core/decode/account-state sont séparément bornés à 1 000 ;
- les opérations PostgreSQL réécrites émettent des traces `TRACE` sous `ks-store` avec durée/résultat/nombre de lignes, sans SQL brut ni valeur bindée.
Le gel formel N1/N2/N3 nest pas encore déclaré : il reste conditionné au snapshot et aux validations de réconciliation prévus dans la tranche dédiée.
## Readiness des futurs décodeurs ## Readiness des futurs décodeurs

View File

@@ -1,15 +1,16 @@
<!-- file: ks-store/TODO.md --> <!-- file: ks-store/TODO.md -->
<!-- version: 14 --> <!-- version: 15 -->
# TODO — ks-store # TODO — ks-store
## Série `0.5.x` ## Série `0.5.x`
- [ ] `0.5.3` — finaliser en `pre.004` les repositories/replay au-dessus du baseline de 16 tables : sélection autonome des CPI, lecture logique top-level+CPI, invalidation descendante, replay N1 -> N2 / N2 -> N3 / frontière N3 -> N4 et ingestion/replay du nouveau couple observation/état de compte. - [x] `0.5.3-pre.004` repositories/replay : lecture logique top-level+CPI, lifecycle symétrique, invalidation descendante Core/decode, replay account observation -> account state et frontière N3 -> N4 maintenue via `k_sol_mat_outputs`.
- [ ] `0.5.3` — finaliser en `pre.004` pagination/limites, conversions BIGINT et plans/index justifiés par les requêtes réelles ; les 75 index du baseline `pre.003` restent optimisables avant gel. - [x] `0.5.3-pre.004` — remplace loffset public par pagination cursorisée Core, borne les lectures interactives à 500 et les batches de processing à 1 000, ajoute les canaris BIGINT/u64 et quatre index dédiés aux nouveaux chemins.
- [ ] `0.5.3` — confirmer les 79 index attendus par plans `EXPLAIN` sur dataset PostgreSQL représentatif avant gel ; les index non uniques restent une optimisation physique ajustable.
- [ ] `0.5.3` — finaliser en `pre.005` la présentation fonctionnelle des surfaces `demo_store_*`, `demo_config` et le splash autour des modèles/compteurs du nouveau baseline sans noms d'objets physiques. - [ ] `0.5.3` — finaliser en `pre.005` la présentation fonctionnelle des surfaces `demo_store_*`, `demo_config` et le splash autour des modèles/compteurs du nouveau baseline sans noms d'objets physiques.
- [ ] `0.5.3` — produire le snapshot machine-readable du baseline N1/N2/N3 réellement stabilisé, ajouter les canaris de gel/anti-`kb_sol_*`/anti-backend leak/anti-secret, confirmer le canari future-decoder sur transactions + comptes et documenter la politique N4 plus souple. - [ ] `0.5.3` — produire le snapshot machine-readable du baseline N1/N2/N3 réellement stabilisé, ajouter les canaris de gel/anti-`kb_sol_*`/anti-backend leak/anti-secret, confirmer le canari future-decoder sur transactions + comptes et documenter la politique N4 plus souple.
- [ ] `0.5.3` — valider PostgreSQL réel sur base vide/recréée : 16 tables, index attendus, réouverture idempotente, refus de l'ancien namespace, rollback et roundtrips raw/Core/decode/materialization. - [ ] `0.5.3` — valider PostgreSQL réel sur base vide/recréée : 16 tables, 79 index attendus, réouverture idempotente, refus de l'ancien namespace, rollback, roundtrips raw/Core/decode/materialization/account-state, CPI et invalidation descendante.
- [ ] `0.5.x` — concevoir les projections N4 metadata : asset/token metadata commune Metaplex + Token-2022 et program metadata séparée pour SPM, toutes deux reconstructibles depuis `k_sol_mat_outputs`. - [ ] `0.5.x` — concevoir les projections N4 metadata : asset/token metadata commune Metaplex + Token-2022 et program metadata séparée pour SPM, toutes deux reconstructibles depuis `k_sol_mat_outputs`.
## Plus tard ## Plus tard

View File

@@ -1,5 +1,5 @@
<!-- file: ks-store/USAGE.md --> <!-- file: ks-store/USAGE.md -->
<!-- version: 6 --> <!-- version: 7 -->
# Utilisation de ks-store # Utilisation de ks-store
@@ -50,7 +50,7 @@ ks-store/migrations/postgres/
Chaque fichier contient une seule instruction SQL. Les fonctions privées du backend l'embarquent par `include_str!`; le DDL n'est pas dupliqué dans des chaînes Rust. Chaque fichier contient une seule instruction SQL. Les fonctions privées du backend l'embarquent par `include_str!`; le DDL n'est pas dupliqué dans des chaînes Rust.
Le baseline courant compte 236 ressources SQL atomiques et 75 index attendus. Le baseline conserve 16 tables et compte désormais 240 ressources SQL atomiques et 79 index attendus après la revue des chemins de replay `pre.004`.
## Résumés sûrs ## Résumés sûrs
@@ -93,12 +93,13 @@ Les traits publics effectivement implémentés sont :
- `StoreHealthStore` ; - `StoreHealthStore` ;
- `RawTransactionStore` ; - `RawTransactionStore` ;
- `CoreTransactionStore` ; - `CoreTransactionStore` ;
- `AccountStateStore` ;
- `CoreExtractionStore` ; - `CoreExtractionStore` ;
- `DecodePipelineStore`. - `DecodePipelineStore`.
`Store` implémente ces traits et délègue au backend privé. `Store` implémente ces traits et délègue au backend privé.
Le schéma `pre.003` réserve aussi les contrats génériques d'observation/état de comptes nécessaires au gel N1/N2. Leur ingestion/replay opérationnel est raccordé dans la tranche repositories/replay suivante ; leur présence dans le baseline évite une modification structurelle ultérieure pour un futur décodeur de comptes. Le couple générique observation/état de compte est opérationnel en `pre.004` : `RawTransactionStore` persiste les observations N1, puis `AccountStateStore` sélectionne et normalise de manière versionnée/idempotente vers N2. Une nouvelle version du normalizer nest pas masquée par la présence dun ancien état Core : le processing ledger porte lidentité/version/hash du traitement.
## Core et replay N2 -> N3 ## Core et replay N2 -> N3
@@ -112,7 +113,7 @@ Le replay d'instruction Core transporte désormais :
Le contrat de replay Core est en version 3. Les anciens constructeurs restent utilisables pour les tests/sources synthétiques ; `ks-store` enrichit les inputs issus du Core persistant avec le contexte disponible. Le contrat de replay Core est en version 3. Les anciens constructeurs restent utilisables pour les tests/sources synthétiques ; `ks-store` enrichit les inputs issus du Core persistant avec le contexte disponible.
Les instructions top-level et CPI restent deux tables physiques. La sélection logique commune et le replay autonome des CPI sont finalisés dans la tranche repositories/replay. Les instructions top-level et CPI restent deux tables physiques, mais la lecture/replay utilise une union logique commune. `CoreInstructionScope` distingue explicitement `TopLevel` et `Inner`, et le lifecycle est mis à jour dans la table physique correspondant au chemin sélectionné.
## Matérialisation N3 ## Matérialisation N3
@@ -147,7 +148,10 @@ Les contrats publics utilisent `ReplayTransaction*`, `ReplayProgram*` et `Replay
```rust ```rust
let first_page = ks_store::PageRequest::first_page(); let first_page = ks_store::PageRequest::first_page();
let next_page = match ks_store::PageRequest::new(250, 250) { let next_page = match ks_store::PageRequest::after(
250,
cursor,
) {
Ok(value) => value, Ok(value) => value,
Err(error) => return Err(error), Err(error) => return Err(error),
}; };
@@ -155,30 +159,32 @@ assert_eq!(first_page.limit, ks_store::DEFAULT_PAGE_SIZE);
assert!(next_page.limit <= ks_store::MAX_PAGE_SIZE); assert!(next_page.limit <= ks_store::MAX_PAGE_SIZE);
``` ```
La refonte cursorisée et la suppression des gros offsets comme contrat durable appartiennent à la tranche repositories/replay. `PageRequest` nexpose plus doffset durable. Pour le replay Core, le curseur encode la clé stable `(slot, signature, scope, instruction_path)` ; la requête demande `limit + 1` afin de produire `PageSlice.next_cursor` sans doublon ni saut sur un dataset figé. Les lectures interactives sont bornées à 500 lignes. Les batches de processing Core/decode/account-state utilisent leurs propres limites, plafonnées à 1 000.
## Transactions atomiques ## Transactions atomiques
Les bundles `CoreExtractionBundle`, `DecodePersistenceBundle` et `MaterializationPersistenceBundle` regroupent les écritures qui doivent réussir ou être annulées ensemble. Les consommateurs ne doivent pas reproduire manuellement ces transactions avec des écritures SQL isolées. Les bundles `AccountStatePersistenceBundle`, `CoreExtractionBundle`, `DecodePersistenceBundle` et `MaterializationPersistenceBundle` regroupent les écritures qui doivent réussir ou être annulées ensemble. Les consommateurs ne doivent pas reproduire manuellement ces transactions avec des écritures SQL isolées.
## Tests PostgreSQL réels ## Tests PostgreSQL réels
Les tests opt-in utilisent `KS_SECRET_POSTGRES_TEST_URL`. Aucun credential réel ne doit être enregistré dans les fixtures ou logs. Les erreurs d'ouverture de connexion sont volontairement sanitisées. Les tests opt-in utilisent `KS_SECRET_POSTGRES_TEST_URL`. Aucun credential réel ne doit être enregistré dans les fixtures ou logs. Les erreurs d'ouverture de connexion sont volontairement sanitisées.
Pour `pre.003`, les validations PostgreSQL à exécuter après compilation sont particulièrement importantes : Pour `pre.004`, les validations PostgreSQL à exécuter après compilation sont particulièrement importantes :
- init sur base vide ; - init sur base vide ;
- réouverture idempotente ; - réouverture idempotente ;
- présence des 16 tables et 75 index attendus ; - présence des 16 tables et 79 index attendus ;
- refus d'un schéma historique `kb_sol_*` ; - refus d'un schéma historique `kb_sol_*` ;
- roundtrip raw/Core/decode/materialization ; - roundtrip raw/Core/decode/materialization ;
- rollback transactionnel Core/decode. - roundtrip account observation -> account state, y compris nouvelle version de normalizer et force replay ;
- replay top-level + CPI paginé sur plusieurs pages ;
- invalidation descendante lors dun remplacement Core/decode ;
- rollback transactionnel account-state/Core/decode.
## Travaux encore ouverts après `pre.003` ## Travaux encore ouverts après `pre.004`
- sélection/replay autonome des CPI et lecture logique top-level+CPI ; - validation PostgreSQL réelle des nouveaux chemins account-state, CPI, invalidation et pagination sur dataset représentatif ;
- invalidation descendante N1 -> N2 -> N3 -> N4 ; - revue finale des plans `EXPLAIN` avant de considérer les 79 index comme suffisants ;
- replay opérationnel du nouveau flux d'observations/états de comptes ; - snapshot machine-readable et gel formel N1/N2/N3 dans la tranche de réconciliation technique ;
- pagination cursorisée et revue finale des index par plans réels ; - refonte desktop fonctionnelle finale autour du nouveau baseline en `pre.005` ;
- snapshot machine-readable et gel formel en tranche de réconciliation technique ; - projections N4 spécialisées, hors `0.5.3`.
- refonte desktop fonctionnelle finale autour du nouveau baseline.

View File

@@ -0,0 +1,5 @@
-- file: migrations/postgres/schema/indexes/create_index_if_not_exists_ix_k_sol_core_inner_instructions_slot.sql
-- version: 1
-- Create PostgreSQL index ix_k_sol_core_inner_instructions_slot when absent.
CREATE INDEX IF NOT EXISTS ix_k_sol_core_inner_instructions_slot ON k_sol_core_inner_instructions (slot, signature, instruction_path);

View File

@@ -0,0 +1,5 @@
-- file: migrations/postgres/schema/indexes/create_index_if_not_exists_ix_k_sol_decode_coverage_observations_signature.sql
-- version: 1
-- Create PostgreSQL index ix_k_sol_decode_coverage_observations_signature when absent.
CREATE INDEX IF NOT EXISTS ix_k_sol_decode_coverage_observations_signature ON k_sol_decode_coverage_observations (signature, instruction_path);

View File

@@ -0,0 +1,5 @@
-- file: migrations/postgres/schema/indexes/create_index_if_not_exists_ix_k_sol_mat_outputs_signature.sql
-- version: 1
-- Create PostgreSQL index ix_k_sol_mat_outputs_signature when absent.
CREATE INDEX IF NOT EXISTS ix_k_sol_mat_outputs_signature ON k_sol_mat_outputs (signature, source_decode_input_key);

View File

@@ -0,0 +1,5 @@
-- file: migrations/postgres/schema/indexes/create_index_if_not_exists_ix_k_sol_mat_outputs_source_decode_input.sql
-- version: 1
-- Create PostgreSQL index ix_k_sol_mat_outputs_source_decode_input when absent.
CREATE INDEX IF NOT EXISTS ix_k_sol_mat_outputs_source_decode_input ON k_sol_mat_outputs (source_decode_input_key, updated_at DESC, id DESC);

View File

@@ -1,5 +1,5 @@
// file: ks-store/src/contracts.rs // file: ks-store/src/contracts.rs
// version: 7 // version: 8
//! Backend-neutral storage contracts used by pipeline crates. //! Backend-neutral storage contracts used by pipeline crates.
@@ -11,8 +11,16 @@ mod pagination;
mod replay; mod replay;
mod repository; mod repository;
/// Stable processing stage used for generic account-state normalization.
pub use self::dto::ACCOUNT_STATE_NORMALIZATION_STAGE;
/// Generic account acquisition observation insert contract. /// Generic account acquisition observation insert contract.
pub use self::dto::AccountObservationInsert; pub use self::dto::AccountObservationInsert;
/// Failed generic account-state normalization attempt.
pub use self::dto::AccountStateNormalizationFailure;
/// Atomic generic account-state persistence bundle.
pub use self::dto::AccountStatePersistenceBundle;
/// Bounded generic account-state selection filter.
pub use self::dto::AccountStateSelectionFilter;
/// Transaction acquisition observation origin. /// Transaction acquisition observation origin.
pub use self::dto::AcquisitionObservationOrigin; pub use self::dto::AcquisitionObservationOrigin;
/// Generic account acquisition observation status. /// Generic account acquisition observation status.
@@ -43,6 +51,8 @@ pub use self::dto::CoreInstructionLifecycleMark;
pub use self::dto::CoreInstructionProcessingState; pub use self::dto::CoreInstructionProcessingState;
/// Core instruction replay filter contract. /// Core instruction replay filter contract.
pub use self::dto::CoreInstructionReplayFilter; pub use self::dto::CoreInstructionReplayFilter;
/// Logical Core instruction scope.
pub use self::dto::CoreInstructionScope;
/// Core log insert contract. /// Core log insert contract.
pub use self::dto::CoreLogInsert; pub use self::dto::CoreLogInsert;
/// Core return-data insert contract. /// Core return-data insert contract.
@@ -61,11 +71,18 @@ pub use self::dto::DecodeFailure;
pub use self::dto::DecodeObservationInsert; pub use self::dto::DecodeObservationInsert;
/// Atomic persistence bundle for one decoder and one contextual input. /// Atomic persistence bundle for one decoder and one contextual input.
pub use self::dto::DecodePersistenceBundle; pub use self::dto::DecodePersistenceBundle;
/// Source-neutral external decode schema provenance.
pub use self::dto::DecodeSchemaProvenance; pub use self::dto::DecodeSchemaProvenance;
/// Bounded contextual instruction selection filter for decode campaigns. /// Bounded contextual instruction selection filter for decode campaigns.
pub use self::dto::DecodeSelectionFilter; pub use self::dto::DecodeSelectionFilter;
/// Insert or upsert result contract returned by repositories. /// Insert or upsert result contract returned by repositories.
pub use self::dto::InsertOutcome; pub use self::dto::InsertOutcome;
/// Maximum number of account observations selected by one normalization query.
pub use self::dto::MAX_ACCOUNT_STATE_SELECTION_ROWS;
/// Maximum canonical transactions selected by one Core extraction batch.
pub use self::dto::MAX_CORE_EXTRACTION_SELECTION_ROWS;
/// Maximum contextual inputs selected by one decode processing batch.
pub use self::dto::MAX_DECODE_SELECTION_ROWS;
/// Maximum number of materialized rows returned by one bounded query. /// Maximum number of materialized rows returned by one bounded query.
pub use self::dto::MAX_MATERIALIZED_OUTPUT_QUERY_ROWS; pub use self::dto::MAX_MATERIALIZED_OUTPUT_QUERY_ROWS;
/// Atomic persistence bundle for one materializer and one decoded observation. /// Atomic persistence bundle for one materializer and one decoded observation.
@@ -116,7 +133,6 @@ pub use self::dto::StoreResourceStatistics;
pub use self::dto::StoreRuntimeSummary; pub use self::dto::StoreRuntimeSummary;
/// Transaction acquisition observation insert contract. /// Transaction acquisition observation insert contract.
pub use self::dto::TransactionObservationInsert; pub use self::dto::TransactionObservationInsert;
/// Transaction acquisition observation status.
/// Generic account acquisition observation SQL-like row contract. /// Generic account acquisition observation SQL-like row contract.
pub use self::entity::AccountObservationRow; pub use self::entity::AccountObservationRow;
/// Core account key SQL-like row contract. /// Core account key SQL-like row contract.
@@ -127,6 +143,8 @@ pub use self::entity::CoreAccountStateRow;
pub use self::entity::CoreBalanceChangeRow; pub use self::entity::CoreBalanceChangeRow;
/// Core inner instruction SQL-like row contract. /// Core inner instruction SQL-like row contract.
pub use self::entity::CoreInnerInstructionRow; pub use self::entity::CoreInnerInstructionRow;
/// Logical top-level or CPI instruction SQL-like replay row.
pub use self::entity::CoreInstructionReplayRow;
/// Core instruction SQL-like row contract. /// Core instruction SQL-like row contract.
pub use self::entity::CoreInstructionRow; pub use self::entity::CoreInstructionRow;
/// Core log SQL-like row contract. /// Core log SQL-like row contract.
@@ -147,10 +165,14 @@ pub use self::health::StoreHealthSnapshot;
pub use self::health::StoreHealthStatus; pub use self::health::StoreHealthStatus;
/// Default repository page size. /// Default repository page size.
pub use self::pagination::DEFAULT_PAGE_SIZE; pub use self::pagination::DEFAULT_PAGE_SIZE;
/// Maximum encoded cursor size.
pub use self::pagination::MAX_PAGE_CURSOR_LENGTH;
/// Maximum repository page size. /// Maximum repository page size.
pub use self::pagination::MAX_PAGE_SIZE; pub use self::pagination::MAX_PAGE_SIZE;
/// Page request contract for repository list operations. /// Page request contract for repository list operations.
pub use self::pagination::PageRequest; pub use self::pagination::PageRequest;
/// One bounded page and its continuation cursor.
pub use self::pagination::PageSlice;
/// Sort direction for repository list operations. /// Sort direction for repository list operations.
pub use self::pagination::SortDirection; pub use self::pagination::SortDirection;
/// Maximum number of rows returned by one replay candidate query. /// Maximum number of rows returned by one replay candidate query.
@@ -171,6 +193,8 @@ pub use self::replay::ReplayProgramSummary;
pub use self::replay::ReplayTransactionCandidate; pub use self::replay::ReplayTransactionCandidate;
/// Bounded read-only filter for transaction replay candidates. /// Bounded read-only filter for transaction replay candidates.
pub use self::replay::ReplayTransactionFilter; pub use self::replay::ReplayTransactionFilter;
/// Generic account observation to Core account-state storage behavior.
pub use self::repository::AccountStateStore;
/// Canonical transaction to core extraction storage behavior. /// Canonical transaction to core extraction storage behavior.
pub use self::repository::CoreExtractionStore; pub use self::repository::CoreExtractionStore;
/// Core Solana storage behavior. /// Core Solana storage behavior.

View File

@@ -1,8 +1,9 @@
// file: ks-store/src/contracts/dto.rs // file: ks-store/src/contracts/dto.rs
// version: 7 // version: 8
//! Backend-neutral DTO exports for storage repository contracts. //! Backend-neutral DTO exports for storage repository contracts.
mod account_state;
mod core; mod core;
mod core_extraction; mod core_extraction;
mod decode; mod decode;
@@ -10,6 +11,16 @@ mod outcome;
mod raw; mod raw;
mod store; mod store;
/// Stable processing stage used for generic account-state normalization.
pub use self::account_state::ACCOUNT_STATE_NORMALIZATION_STAGE;
/// Failed generic account-state normalization attempt.
pub use self::account_state::AccountStateNormalizationFailure;
/// Atomic generic account-state persistence bundle.
pub use self::account_state::AccountStatePersistenceBundle;
/// Bounded account-state normalization selection filter.
pub use self::account_state::AccountStateSelectionFilter;
/// Maximum number of account observations selected by one normalization query.
pub use self::account_state::MAX_ACCOUNT_STATE_SELECTION_ROWS;
/// Core account key insert contract. /// Core account key insert contract.
pub use self::core::CoreAccountKeyInsert; pub use self::core::CoreAccountKeyInsert;
/// Core account key source category. /// Core account key source category.
@@ -30,6 +41,8 @@ pub use self::core::CoreInstructionLifecycleMark;
pub use self::core::CoreInstructionProcessingState; pub use self::core::CoreInstructionProcessingState;
/// Core instruction replay filter contract. /// Core instruction replay filter contract.
pub use self::core::CoreInstructionReplayFilter; pub use self::core::CoreInstructionReplayFilter;
/// Logical scope of one replayable Core instruction.
pub use self::core::CoreInstructionScope;
/// Core log insert contract. /// Core log insert contract.
pub use self::core::CoreLogInsert; pub use self::core::CoreLogInsert;
/// Core return-data insert contract. /// Core return-data insert contract.
@@ -42,6 +55,8 @@ pub use self::core_extraction::CoreExtractionBundle;
pub use self::core_extraction::CoreExtractionFailure; pub use self::core_extraction::CoreExtractionFailure;
/// Bounded canonical transaction selection filter for core extraction. /// Bounded canonical transaction selection filter for core extraction.
pub use self::core_extraction::CoreExtractionSelectionFilter; pub use self::core_extraction::CoreExtractionSelectionFilter;
/// Maximum canonical transactions selected by one Core extraction batch.
pub use self::core_extraction::MAX_CORE_EXTRACTION_SELECTION_ROWS;
/// Stable processing ledger identity. /// Stable processing ledger identity.
pub use self::core_extraction::ProcessingLedgerIdentity; pub use self::core_extraction::ProcessingLedgerIdentity;
/// Stable processing ledger status. /// Stable processing ledger status.
@@ -61,6 +76,8 @@ pub use self::decode::DecodePersistenceBundle;
pub use self::decode::DecodeSchemaProvenance; pub use self::decode::DecodeSchemaProvenance;
/// Bounded contextual instruction selection filter for decode campaigns. /// Bounded contextual instruction selection filter for decode campaigns.
pub use self::decode::DecodeSelectionFilter; pub use self::decode::DecodeSelectionFilter;
/// Maximum contextual inputs selected by one decode processing batch.
pub use self::decode::MAX_DECODE_SELECTION_ROWS;
/// Maximum number of materialized rows returned by one bounded query. /// Maximum number of materialized rows returned by one bounded query.
pub use self::decode::MAX_MATERIALIZED_OUTPUT_QUERY_ROWS; pub use self::decode::MAX_MATERIALIZED_OUTPUT_QUERY_ROWS;
/// Atomic persistence bundle for one materializer and one decoded observation. /// Atomic persistence bundle for one materializer and one decoded observation.

View File

@@ -0,0 +1,200 @@
// file: ks-store/src/contracts/dto/account_state.rs
// version: 1
//! Generic account-observation to Core account-state replay contracts.
/// Stable processing stage used for generic account-state normalization.
pub const ACCOUNT_STATE_NORMALIZATION_STAGE: &str = "account_state_normalization";
/// Maximum number of account observations selected by one normalization query.
pub const MAX_ACCOUNT_STATE_SELECTION_ROWS: u32 = 1000;
/// Bounded selection filter for replayable account observations.
#[derive(Clone, Debug, Eq, PartialEq, serde::Deserialize, serde::Serialize)]
pub struct AccountStateSelectionFilter {
/// Processor name whose current succeeded inputs may be excluded.
pub processor_name: std::string::String,
/// Processor version whose current succeeded inputs may be excluded.
pub processor_version: std::string::String,
/// Optional exact account keys.
pub account_keys: std::vec::Vec<std::string::String>,
/// Optional inclusive minimum context slot.
pub min_slot: std::option::Option<u64>,
/// Optional inclusive maximum context slot.
pub max_slot: std::option::Option<u64>,
/// Whether observations already succeeded for the selected processor identity may be selected again.
pub include_current: bool,
/// Maximum selected observations.
pub limit: u32,
}
impl crate::AccountStateSelectionFilter {
/// Builds a bounded account-state normalization selection.
pub fn new(
processor_name: impl std::convert::Into<std::string::String>,
processor_version: impl std::convert::Into<std::string::String>,
account_keys: std::vec::Vec<std::string::String>,
min_slot: std::option::Option<u64>,
max_slot: std::option::Option<u64>,
include_current: bool,
limit: u32,
) -> ks_core::Result<Self> {
let processor_name_value = processor_name.into();
let processor_version_value = processor_version.into();
if processor_name_value.trim().is_empty() || processor_version_value.trim().is_empty() {
return std::result::Result::Err(ks_core::Error::db(
"account-state selection processor identity must not be empty",
));
}
if limit == 0 || limit > crate::MAX_ACCOUNT_STATE_SELECTION_ROWS {
return std::result::Result::Err(ks_core::Error::db(format!(
"account-state selection limit must be between 1 and {}",
crate::MAX_ACCOUNT_STATE_SELECTION_ROWS
)));
}
if let (std::option::Option::Some(minimum), std::option::Option::Some(maximum)) =
(min_slot, max_slot)
{
if minimum > maximum {
return std::result::Result::Err(ks_core::Error::db(
"account-state selection minimum slot must not exceed maximum slot",
));
}
}
for account_key in &account_keys {
if account_key.trim().is_empty() {
return std::result::Result::Err(ks_core::Error::db(
"account-state selection account keys must not be empty",
));
}
}
return std::result::Result::Ok(Self {
processor_name: processor_name_value,
processor_version: processor_version_value,
account_keys,
min_slot,
max_slot,
include_current,
limit,
});
}
/// Builds a default pending account-state selection.
pub fn pending(
processor_name: impl std::convert::Into<std::string::String>,
processor_version: impl std::convert::Into<std::string::String>,
limit: u32,
) -> ks_core::Result<Self> {
return Self::new(
processor_name,
processor_version,
std::vec::Vec::new(),
std::option::Option::None,
std::option::Option::None,
false,
limit,
);
}
}
/// Atomic persistence bundle for one generic account observation normalized to Core state.
#[derive(Clone, Debug, PartialEq, serde::Deserialize, serde::Serialize)]
pub struct AccountStatePersistenceBundle {
/// Processing ledger identity.
pub ledger_identity: crate::ProcessingLedgerIdentity,
/// Canonical Core account state produced from the observation.
pub state: crate::CoreAccountStateInsert,
}
impl crate::AccountStatePersistenceBundle {
/// Validates the account-state normalization identity and lineage.
pub fn validate(&self) -> ks_core::Result<()> {
if self.ledger_identity.stage != crate::ACCOUNT_STATE_NORMALIZATION_STAGE {
return std::result::Result::Err(ks_core::Error::db(
"account-state persistence stage is invalid",
));
}
let state_result = self.state.validate();
if let std::result::Result::Err(error) = state_result {
return std::result::Result::Err(error);
}
return std::result::Result::Ok(());
}
}
/// Failed generic account-state normalization attempt.
#[derive(Clone, Debug, Eq, PartialEq, serde::Deserialize, serde::Serialize)]
pub struct AccountStateNormalizationFailure {
/// Source raw account observation technical id.
pub source_observation_id: i64,
/// Processing ledger identity.
pub ledger_identity: crate::ProcessingLedgerIdentity,
/// Stable machine-readable error code.
pub error_code: std::string::String,
/// Bounded diagnostic message.
pub error_message: std::string::String,
}
impl crate::AccountStateNormalizationFailure {
/// Validates a failure record before persistence.
pub fn validate(&self) -> ks_core::Result<()> {
if self.source_observation_id <= 0
|| self.ledger_identity.stage != crate::ACCOUNT_STATE_NORMALIZATION_STAGE
|| self.error_code.trim().is_empty()
|| self.error_message.trim().is_empty()
{
return std::result::Result::Err(ks_core::Error::db(
"account-state normalization failure is invalid",
));
}
return std::result::Result::Ok(());
}
}
#[cfg(test)]
mod tests {
#[test]
fn account_state_selection_is_bounded() {
assert!(crate::AccountStateSelectionFilter::pending("normalizer", "1", 0).is_err());
assert!(
crate::AccountStateSelectionFilter::pending(
"normalizer",
"1",
crate::MAX_ACCOUNT_STATE_SELECTION_ROWS + 1,
)
.is_err()
);
}
#[test]
fn account_state_selection_requires_processor_identity() {
assert!(crate::AccountStateSelectionFilter::pending("", "1", 10).is_err());
assert!(crate::AccountStateSelectionFilter::pending("normalizer", "", 10).is_err());
}
#[test]
fn account_state_bundle_requires_the_canonical_stage() {
let identity = crate::ProcessingLedgerIdentity {
stage: "other".to_string(),
processor_name: "normalizer".to_string(),
processor_version: "1".to_string(),
input_key: "observation".to_string(),
input_hash: "hash".to_string(),
};
let bundle = crate::AccountStatePersistenceBundle {
ledger_identity: identity,
state: crate::CoreAccountStateInsert {
source_observation_id: 1,
account_key: "account".to_string(),
slot: 1,
owner: "owner".to_string(),
lamports: 0,
executable: false,
rent_epoch: 0,
space: 0,
data_base64: std::string::String::new(),
data_hash: "hash".to_string(),
},
};
assert!(bundle.validate().is_err());
}
}

View File

@@ -1,8 +1,51 @@
// file: ks-store/src/contracts/dto/core.rs // file: ks-store/src/contracts/dto/core.rs
// version: 3 // version: 4
//! Core Solana storage DTOs. //! Core Solana storage DTOs.
/// Logical scope of one Core instruction persisted in the top-level or CPI table.
#[derive(Clone, Copy, Debug, Eq, PartialEq, serde::Deserialize, serde::Serialize)]
pub enum CoreInstructionScope {
/// Transaction message top-level instruction.
TopLevel,
/// Inner/CPI instruction emitted during program execution.
Inner,
}
impl crate::CoreInstructionScope {
/// Returns the stable storage code used by backend-neutral query contracts.
pub fn as_code(self) -> &'static str {
return match self {
Self::TopLevel => "top_level",
Self::Inner => "inner",
};
}
/// Resolves a stable scope code returned by a storage adapter.
pub fn from_code(value: &str) -> ks_core::Result<Self> {
return match value {
"top_level" => std::result::Result::Ok(Self::TopLevel),
"inner" => std::result::Result::Ok(Self::Inner),
_unknown => {
std::result::Result::Err(ks_core::Error::db("unknown core instruction scope"))
},
};
}
/// Infers the scope from the canonical instruction path convention.
pub fn from_instruction_path(instruction_path: &str) -> ks_core::Result<Self> {
if instruction_path.trim().is_empty() {
return std::result::Result::Err(ks_core::Error::db(
"core instruction path must not be empty",
));
}
if instruction_path.contains('/') {
return std::result::Result::Ok(Self::Inner);
}
return std::result::Result::Ok(Self::TopLevel);
}
}
/// Processing state for one normalized core instruction. /// Processing state for one normalized core instruction.
#[derive(Clone, Copy, Debug, Eq, PartialEq, serde::Deserialize, serde::Serialize)] #[derive(Clone, Copy, Debug, Eq, PartialEq, serde::Deserialize, serde::Serialize)]
pub enum CoreInstructionProcessingState { pub enum CoreInstructionProcessingState {
@@ -590,6 +633,8 @@ pub struct CoreInstructionReplayFilter {
pub processing_state: std::option::Option<crate::CoreInstructionProcessingState>, pub processing_state: std::option::Option<crate::CoreInstructionProcessingState>,
/// Optional program id filter. /// Optional program id filter.
pub program_id: std::option::Option<std::string::String>, pub program_id: std::option::Option<std::string::String>,
/// Optional logical instruction scope.
pub scope: std::option::Option<crate::CoreInstructionScope>,
/// Optional inclusive minimum slot. /// Optional inclusive minimum slot.
pub min_slot: std::option::Option<u64>, pub min_slot: std::option::Option<u64>,
/// Optional inclusive maximum slot. /// Optional inclusive maximum slot.
@@ -601,6 +646,7 @@ impl crate::CoreInstructionReplayFilter {
pub fn new( pub fn new(
processing_state: std::option::Option<crate::CoreInstructionProcessingState>, processing_state: std::option::Option<crate::CoreInstructionProcessingState>,
program_id: std::option::Option<std::string::String>, program_id: std::option::Option<std::string::String>,
scope: std::option::Option<crate::CoreInstructionScope>,
min_slot: std::option::Option<u64>, min_slot: std::option::Option<u64>,
max_slot: std::option::Option<u64>, max_slot: std::option::Option<u64>,
) -> ks_core::Result<Self> { ) -> ks_core::Result<Self> {
@@ -625,6 +671,7 @@ impl crate::CoreInstructionReplayFilter {
return std::result::Result::Ok(Self { return std::result::Result::Ok(Self {
processing_state, processing_state,
program_id, program_id,
scope,
min_slot, min_slot,
max_slot, max_slot,
}); });
@@ -637,6 +684,7 @@ impl crate::CoreInstructionReplayFilter {
crate::CoreInstructionProcessingState::Pending, crate::CoreInstructionProcessingState::Pending,
), ),
program_id: std::option::Option::None, program_id: std::option::Option::None,
scope: std::option::Option::None,
min_slot: std::option::Option::None, min_slot: std::option::Option::None,
max_slot: std::option::Option::None, max_slot: std::option::Option::None,
}; };
@@ -1035,13 +1083,27 @@ mod tests {
std::option::Option::Some(" ".to_string()), std::option::Option::Some(" ".to_string()),
std::option::Option::None, std::option::Option::None,
std::option::Option::None, std::option::Option::None,
std::option::Option::None,
); );
assert!(result.is_err()); assert!(result.is_err());
} }
#[test]
fn instruction_scope_is_inferred_from_canonical_path() {
assert_eq!(
crate::CoreInstructionScope::from_instruction_path("2"),
std::result::Result::Ok(crate::CoreInstructionScope::TopLevel)
);
assert_eq!(
crate::CoreInstructionScope::from_instruction_path("2/1"),
std::result::Result::Ok(crate::CoreInstructionScope::Inner)
);
}
#[test] #[test]
fn replay_filter_rejects_inverted_slots() { fn replay_filter_rejects_inverted_slots() {
let result = crate::CoreInstructionReplayFilter::new( let result = crate::CoreInstructionReplayFilter::new(
std::option::Option::None,
std::option::Option::None, std::option::Option::None,
std::option::Option::None, std::option::Option::None,
std::option::Option::Some(10), std::option::Option::Some(10),

View File

@@ -1,8 +1,11 @@
// file: ks-store/src/contracts/dto/core_extraction.rs // file: ks-store/src/contracts/dto/core_extraction.rs
// version: 3 // version: 4
//! Canonical transaction to core extraction storage contracts. //! Canonical transaction to core extraction storage contracts.
/// Maximum canonical transactions selected by one Core extraction batch.
pub const MAX_CORE_EXTRACTION_SELECTION_ROWS: u32 = 1000;
/// Stable processing status for one extraction ledger entry. /// Stable processing status for one extraction ledger entry.
#[derive(Clone, Copy, Debug, Eq, PartialEq, serde::Deserialize, serde::Serialize)] #[derive(Clone, Copy, Debug, Eq, PartialEq, serde::Deserialize, serde::Serialize)]
pub enum ProcessingLedgerStatus { pub enum ProcessingLedgerStatus {
@@ -41,10 +44,11 @@ impl crate::CoreExtractionSelectionFilter {
program_id: std::option::Option<std::string::String>, program_id: std::option::Option<std::string::String>,
limit: u32, limit: u32,
) -> ks_core::Result<Self> { ) -> ks_core::Result<Self> {
if limit == 0 { if limit == 0 || limit > crate::MAX_CORE_EXTRACTION_SELECTION_ROWS {
return std::result::Result::Err(ks_core::Error::db( return std::result::Result::Err(ks_core::Error::db(format!(
"core extraction selection limit must be greater than zero", "core extraction selection limit must be between 1 and {}",
)); crate::MAX_CORE_EXTRACTION_SELECTION_ROWS
)));
} }
if let (std::option::Option::Some(minimum), std::option::Option::Some(maximum)) = if let (std::option::Option::Some(minimum), std::option::Option::Some(maximum)) =
(min_slot, max_slot) (min_slot, max_slot)
@@ -271,6 +275,14 @@ mod tests {
assert!(result.is_err()); assert!(result.is_err());
} }
#[test]
fn selection_filter_rejects_limit_above_processing_bound() {
let result = crate::CoreExtractionSelectionFilter::pending(
crate::MAX_CORE_EXTRACTION_SELECTION_ROWS + 1,
);
assert!(result.is_err());
}
#[test] #[test]
fn selection_filter_rejects_inverted_slots() { fn selection_filter_rejects_inverted_slots() {
let result = crate::CoreExtractionSelectionFilter::new( let result = crate::CoreExtractionSelectionFilter::new(

View File

@@ -1,8 +1,10 @@
// file: ks-store/src/contracts/dto/decode.rs // file: ks-store/src/contracts/dto/decode.rs
// version: 6 // version: 7
//! Backend-neutral decode, coverage and materialization persistence DTOs. //! Backend-neutral decode, coverage and materialization persistence DTOs.
/// Maximum contextual inputs selected by one decode processing batch.
pub const MAX_DECODE_SELECTION_ROWS: u32 = 1000;
/// Maximum number of materialized rows returned by one bounded query. /// Maximum number of materialized rows returned by one bounded query.
pub const MAX_MATERIALIZED_OUTPUT_QUERY_ROWS: u32 = 500; pub const MAX_MATERIALIZED_OUTPUT_QUERY_ROWS: u32 = 500;
@@ -111,10 +113,11 @@ impl crate::DecodeSelectionFilter {
incomplete_signatures: bool, incomplete_signatures: bool,
limit: u32, limit: u32,
) -> ks_core::Result<Self> { ) -> ks_core::Result<Self> {
if limit == 0 { if limit == 0 || limit > crate::MAX_DECODE_SELECTION_ROWS {
return std::result::Result::Err(ks_core::Error::db( return std::result::Result::Err(ks_core::Error::db(format!(
"decode selection limit must be greater than zero", "decode selection limit must be between 1 and {}",
)); crate::MAX_DECODE_SELECTION_ROWS
)));
} }
if min_slot.is_some() && max_slot.is_some() && min_slot > max_slot { if min_slot.is_some() && max_slot.is_some() && min_slot > max_slot {
return std::result::Result::Err(ks_core::Error::db( return std::result::Result::Err(ks_core::Error::db(
@@ -634,6 +637,13 @@ mod tests {
assert!(crate::DecodeSelectionFilter::actionable(0).is_err()); assert!(crate::DecodeSelectionFilter::actionable(0).is_err());
} }
#[test]
fn actionable_filter_rejects_limit_above_processing_bound() {
assert!(
crate::DecodeSelectionFilter::actionable(crate::MAX_DECODE_SELECTION_ROWS + 1).is_err()
);
}
#[test] #[test]
fn incomplete_signature_filter_preserves_signature_limit_semantics() { fn incomplete_signature_filter_preserves_signature_limit_semantics() {
let result = crate::DecodeSelectionFilter::new( let result = crate::DecodeSelectionFilter::new(

View File

@@ -1,5 +1,5 @@
// file: ks-store/src/contracts/entity.rs // file: ks-store/src/contracts/entity.rs
// version: 6 // version: 7
//! Backend-neutral persisted entity exports for storage adapters. //! Backend-neutral persisted entity exports for storage adapters.
@@ -14,6 +14,8 @@ pub use self::core::CoreAccountStateRow;
pub use self::core::CoreBalanceChangeRow; pub use self::core::CoreBalanceChangeRow;
/// Core inner instruction persisted row contract. /// Core inner instruction persisted row contract.
pub use self::core::CoreInnerInstructionRow; pub use self::core::CoreInnerInstructionRow;
/// Logical Core instruction replay row spanning top-level and inner/CPI storage.
pub use self::core::CoreInstructionReplayRow;
/// Core instruction persisted row contract. /// Core instruction persisted row contract.
pub use self::core::CoreInstructionRow; pub use self::core::CoreInstructionRow;
/// Core log persisted row contract. /// Core log persisted row contract.

View File

@@ -1,5 +1,5 @@
// file: ks-store/src/contracts/entity/core.rs // file: ks-store/src/contracts/entity/core.rs
// version: 4 // version: 5
//! Core Solana persisted entities. //! Core Solana persisted entities.
@@ -82,6 +82,41 @@ pub struct CoreAccountStateRow {
pub created_at: chrono::DateTime<chrono::Utc>, pub created_at: chrono::DateTime<chrono::Utc>,
} }
/// Logical Core instruction row spanning top-level and inner/CPI storage.
#[derive(Clone, Debug, PartialEq, serde::Deserialize, serde::Serialize)]
pub struct CoreInstructionReplayRow {
/// Technical primary key inside the physical instruction scope.
pub id: i64,
/// Parent core transaction technical key.
pub transaction_id: i64,
/// Transaction signature.
pub signature: std::string::String,
/// Transaction slot stored as SQL `BIGINT`.
pub slot: i64,
/// Logical top-level or inner/CPI scope.
pub scope: crate::CoreInstructionScope,
/// Optional immediate parent instruction path for inner/CPI rows.
pub parent_instruction_path: std::option::Option<std::string::String>,
/// Stable instruction path.
pub instruction_path: std::string::String,
/// Program id.
pub program_id: std::string::String,
/// Optional runtime invocation stack height.
pub stack_height: std::option::Option<i32>,
/// Instruction accounts as JSON.
pub accounts_json: serde_json::Value,
/// Optional instruction payload retained in the hot store.
pub payload_json: std::option::Option<serde_json::Value>,
/// Optional deterministic payload digest.
pub payload_json_hash: std::option::Option<std::string::String>,
/// Current processing state.
pub processing_state: crate::CoreInstructionProcessingState,
/// Insert timestamp.
pub created_at: chrono::DateTime<chrono::Utc>,
/// Last lifecycle update timestamp.
pub updated_at: chrono::DateTime<chrono::Utc>,
}
/// Core instruction persisted row contract. /// Core instruction persisted row contract.
#[derive(Clone, Debug, PartialEq, serde::Deserialize, serde::Serialize)] #[derive(Clone, Debug, PartialEq, serde::Deserialize, serde::Serialize)]
pub struct CoreInstructionRow { pub struct CoreInstructionRow {

View File

@@ -1,12 +1,14 @@
// file: ks-store/src/contracts/pagination.rs // file: ks-store/src/contracts/pagination.rs
// version: 3 // version: 4
//! Backend-neutral pagination and sorting contracts for repository operations. //! Backend-neutral pagination contracts for repository operations.
/// Default page size for repository list operations. /// Default page size for repository list operations.
pub const DEFAULT_PAGE_SIZE: u16 = 100; pub const DEFAULT_PAGE_SIZE: u16 = 100;
/// Maximum page size for repository list operations. /// Maximum page size for repository list operations.
pub const MAX_PAGE_SIZE: u16 = 1000; pub const MAX_PAGE_SIZE: u16 = 500;
/// Maximum serialized opaque cursor length accepted by repository list operations.
pub const MAX_PAGE_CURSOR_LENGTH: usize = 512;
/// Sort direction for repository list operations. /// Sort direction for repository list operations.
#[derive(Clone, Copy, Debug, Eq, PartialEq, serde::Deserialize, serde::Serialize)] #[derive(Clone, Copy, Debug, Eq, PartialEq, serde::Deserialize, serde::Serialize)]
@@ -17,18 +19,21 @@ pub enum SortDirection {
Desc, Desc,
} }
/// Page request contract for repository list operations. /// Stable cursor request contract for repository list operations.
#[derive(Clone, Copy, Debug, Eq, PartialEq, serde::Deserialize, serde::Serialize)] #[derive(Clone, Debug, Eq, PartialEq, serde::Deserialize, serde::Serialize)]
pub struct PageRequest { pub struct PageRequest {
/// Maximum number of rows to return. /// Maximum number of rows to return.
pub limit: u16, pub limit: u16,
/// Zero-based row offset. /// Optional opaque cursor returned by the previous page.
pub offset: u64, pub cursor: std::option::Option<std::string::String>,
} }
impl crate::PageRequest { impl crate::PageRequest {
/// Builds a page request after minimal bounds validation. /// Builds a page request after validating the limit and opaque cursor bounds.
pub fn new(limit: u16, offset: u64) -> ks_core::Result<Self> { pub fn new(
limit: u16,
cursor: std::option::Option<std::string::String>,
) -> ks_core::Result<Self> {
if limit == 0 { if limit == 0 {
return std::result::Result::Err(ks_core::Error::db( return std::result::Result::Err(ks_core::Error::db(
"page limit must be greater than zero", "page limit must be greater than zero",
@@ -39,36 +44,95 @@ impl crate::PageRequest {
"page limit exceeds maximum page size", "page limit exceeds maximum page size",
)); ));
} }
return std::result::Result::Ok(Self { limit, offset }); let cursor_value = cursor.and_then(|value| {
let trimmed = value.trim();
if trimmed.is_empty() {
return std::option::Option::None;
}
return std::option::Option::Some(trimmed.to_string());
});
if cursor_value
.as_deref()
.is_some_and(|value| return value.len() > crate::MAX_PAGE_CURSOR_LENGTH)
{
return std::result::Result::Err(ks_core::Error::db(
"page cursor exceeds maximum encoded length",
));
}
return std::result::Result::Ok(Self { limit, cursor: cursor_value });
} }
/// Builds the default first page request. /// Builds the default first page request.
pub fn first_page() -> Self { pub fn first_page() -> Self {
return Self { return Self {
limit: crate::DEFAULT_PAGE_SIZE, limit: crate::DEFAULT_PAGE_SIZE,
offset: 0, cursor: std::option::Option::None,
}; };
} }
/// Builds a request continuing after an opaque cursor returned by a previous page.
pub fn after(
limit: u16,
cursor: impl std::convert::Into<std::string::String>,
) -> ks_core::Result<Self> {
return Self::new(limit, std::option::Option::Some(cursor.into()));
}
}
/// One bounded page of rows and its continuation cursor.
#[derive(Clone, Debug, PartialEq, serde::Deserialize, serde::Serialize)]
pub struct PageSlice<T> {
/// Rows returned for this page.
pub rows: std::vec::Vec<T>,
/// Opaque cursor for the next page when additional rows exist.
pub next_cursor: std::option::Option<std::string::String>,
}
impl<T> crate::PageSlice<T> {
/// Builds one page from already bounded rows and an optional continuation cursor.
pub fn new(
rows: std::vec::Vec<T>,
next_cursor: std::option::Option<std::string::String>,
) -> Self {
return Self { rows, next_cursor };
}
/// Returns true when this page has no continuation cursor.
pub fn is_last_page(&self) -> bool {
return self.next_cursor.is_none();
}
} }
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
#[test] #[test]
fn page_request_rejects_zero_limit() { fn page_request_rejects_zero_limit() {
let result = crate::PageRequest::new(0, 0); let result = crate::PageRequest::new(0, std::option::Option::None);
assert!(result.is_err()); assert!(result.is_err());
} }
#[test] #[test]
fn page_request_rejects_limit_above_maximum() { fn page_request_rejects_limit_above_maximum() {
let result = crate::PageRequest::new(crate::MAX_PAGE_SIZE + 1, 0); let result = crate::PageRequest::new(crate::MAX_PAGE_SIZE + 1, std::option::Option::None);
assert!(result.is_err()); assert!(result.is_err());
} }
#[test] #[test]
fn first_page_uses_default_limit() { fn page_request_rejects_oversized_cursor() {
let result = crate::PageRequest::after(10, "x".repeat(crate::MAX_PAGE_CURSOR_LENGTH + 1));
assert!(result.is_err());
}
#[test]
fn first_page_uses_default_limit_without_cursor() {
let request = crate::PageRequest::first_page(); let request = crate::PageRequest::first_page();
assert_eq!(request.limit, crate::DEFAULT_PAGE_SIZE); assert_eq!(request.limit, crate::DEFAULT_PAGE_SIZE);
assert_eq!(request.offset, 0); assert!(request.cursor.is_none());
}
#[test]
fn page_slice_reports_last_page_from_cursor_presence() {
let page = crate::PageSlice::new(vec![1_u32], std::option::Option::None);
assert!(page.is_last_page());
} }
} }

View File

@@ -1,10 +1,10 @@
// file: ks-store/src/contracts/replay.rs // file: ks-store/src/contracts/replay.rs
// version: 6 // version: 7
//! Backend-agnostic replay candidate filters and read models. //! Backend-agnostic replay candidate filters and read models.
/// Maximum number of rows returned by one replay candidate query. /// Maximum number of rows returned by one replay candidate query.
pub const MAX_REPLAY_CANDIDATE_ROWS: u32 = 100_000; pub const MAX_REPLAY_CANDIDATE_ROWS: u32 = 500;
/// Program occurrence scope used while filtering replay candidates. /// Program occurrence scope used while filtering replay candidates.
#[derive(Clone, Copy, Debug, Eq, PartialEq)] #[derive(Clone, Copy, Debug, Eq, PartialEq)]

View File

@@ -1,5 +1,5 @@
// file: ks-store/src/contracts/repository.rs // file: ks-store/src/contracts/repository.rs
// version: 4 // version: 5
//! Storage trait definitions shared by concrete stores. //! Storage trait definitions shared by concrete stores.
@@ -29,6 +29,9 @@ pub trait RawTransactionStore {
async fn has_transaction_observation_key(&self, observation_key: &str) async fn has_transaction_observation_key(&self, observation_key: &str)
-> ks_core::Result<bool>; -> ks_core::Result<bool>;
/// Returns true when a generic account observation key is already stored.
async fn has_account_observation_key(&self, observation_key: &str) -> ks_core::Result<bool>;
/// Stores one canonical source-independent transaction payload. /// Stores one canonical source-independent transaction payload.
async fn insert_raw_transaction( async fn insert_raw_transaction(
&self, &self,
@@ -41,6 +44,12 @@ pub trait RawTransactionStore {
input: &crate::TransactionObservationInsert, input: &crate::TransactionObservationInsert,
) -> ks_core::Result<crate::InsertOutcome>; ) -> ks_core::Result<crate::InsertOutcome>;
/// Stores one generic replayable account acquisition observation.
async fn insert_account_observation(
&self,
input: &crate::AccountObservationInsert,
) -> ks_core::Result<crate::InsertOutcome>;
/// Updates canonical raw transaction retention and processing metadata. /// Updates canonical raw transaction retention and processing metadata.
async fn mark_raw_payload_lifecycle( async fn mark_raw_payload_lifecycle(
&self, &self,
@@ -92,14 +101,14 @@ pub trait CoreTransactionStore {
&self, &self,
filter: &crate::CoreInstructionReplayFilter, filter: &crate::CoreInstructionReplayFilter,
page_request: &crate::PageRequest, page_request: &crate::PageRequest,
) -> ks_core::Result<std::vec::Vec<crate::CoreInstructionRow>>; ) -> ks_core::Result<crate::PageSlice<crate::CoreInstructionReplayRow>>;
/// Lists replay inputs with instruction context, logs, balances and account keys. /// Lists replay inputs with instruction context, logs, balances and account keys.
async fn list_core_instruction_replay_inputs( async fn list_core_instruction_replay_inputs(
&self, &self,
filter: &crate::CoreInstructionReplayFilter, filter: &crate::CoreInstructionReplayFilter,
page_request: &crate::PageRequest, page_request: &crate::PageRequest,
) -> ks_core::Result<std::vec::Vec<crate::MdCoreInstructionReplayInput>>; ) -> ks_core::Result<crate::PageSlice<crate::MdCoreInstructionReplayInput>>;
/// Updates one normalized core instruction lifecycle state. /// Updates one normalized core instruction lifecycle state.
async fn mark_core_instruction_lifecycle( async fn mark_core_instruction_lifecycle(
@@ -108,6 +117,35 @@ pub trait CoreTransactionStore {
) -> ks_core::Result<crate::InsertOutcome>; ) -> ks_core::Result<crate::InsertOutcome>;
} }
/// Generic account observation to Core account-state normalization storage behavior.
#[async_trait::async_trait]
pub trait AccountStateStore {
/// Lists complete account observations selected for first normalization or replay.
async fn list_account_observations_for_normalization(
&self,
filter: &crate::AccountStateSelectionFilter,
) -> ks_core::Result<std::vec::Vec<crate::AccountObservationRow>>;
/// Returns true when the same account-state normalizer version succeeded for the same input hash.
async fn is_account_state_current(
&self,
identity: &crate::ProcessingLedgerIdentity,
) -> ks_core::Result<bool>;
/// Atomically persists one Core account state and its processing ledger result.
async fn persist_account_state(
&self,
bundle: &crate::AccountStatePersistenceBundle,
force_replay: bool,
) -> ks_core::Result<crate::InsertOutcome>;
/// Persists one failed account-state normalization attempt.
async fn mark_account_state_failed(
&self,
failure: &crate::AccountStateNormalizationFailure,
) -> ks_core::Result<crate::InsertOutcome>;
}
/// Canonical transaction to core extraction storage behavior. /// Canonical transaction to core extraction storage behavior.
#[async_trait::async_trait] #[async_trait::async_trait]
pub trait CoreExtractionStore { pub trait CoreExtractionStore {

View File

@@ -1,5 +1,5 @@
// file: ks-store/src/lib.rs // file: ks-store/src/lib.rs
// version: 12 // version: 13
//! Backend-agnostic Solana storage contracts and store facade. //! Backend-agnostic Solana storage contracts and store facade.
#![warn(missing_docs)] #![warn(missing_docs)]
@@ -67,10 +67,14 @@ pub(crate) use self::postgres::decode_store_schema_statements;
pub(crate) use self::postgres::decode_store_table_diagnostic_specs; pub(crate) use self::postgres::decode_store_table_diagnostic_specs;
/// Crate-internal expected PostgreSQL index inventory shared through the crate-root facade. /// Crate-internal expected PostgreSQL index inventory shared through the crate-root facade.
pub(crate) use self::postgres::expected_postgres_index_names; pub(crate) use self::postgres::expected_postgres_index_names;
/// Crate-internal account-observation key lookup shared through the crate-root facade.
pub(crate) use self::postgres::has_account_observation_key;
/// Crate-internal storage symbol `has_raw_transaction_signature` shared through the crate-root facade. /// Crate-internal storage symbol `has_raw_transaction_signature` shared through the crate-root facade.
pub(crate) use self::postgres::has_raw_transaction_signature; pub(crate) use self::postgres::has_raw_transaction_signature;
/// Crate-internal storage symbol `has_transaction_observation_key` shared through the crate-root facade. /// Crate-internal storage symbol `has_transaction_observation_key` shared through the crate-root facade.
pub(crate) use self::postgres::has_transaction_observation_key; pub(crate) use self::postgres::has_transaction_observation_key;
/// Crate-internal account-observation insert shared through the crate-root facade.
pub(crate) use self::postgres::insert_account_observation;
/// Crate-internal storage symbol `insert_core_account_keys` shared through the crate-root facade. /// Crate-internal storage symbol `insert_core_account_keys` shared through the crate-root facade.
pub(crate) use self::postgres::insert_core_account_keys; pub(crate) use self::postgres::insert_core_account_keys;
/// Crate-internal storage symbol `insert_core_balance_changes` shared through the crate-root facade. /// Crate-internal storage symbol `insert_core_balance_changes` shared through the crate-root facade.
@@ -87,10 +91,18 @@ pub(crate) use self::postgres::insert_core_transaction;
pub(crate) use self::postgres::insert_raw_transaction; pub(crate) use self::postgres::insert_raw_transaction;
/// Crate-internal storage symbol `insert_transaction_observation` shared through the crate-root facade. /// Crate-internal storage symbol `insert_transaction_observation` shared through the crate-root facade.
pub(crate) use self::postgres::insert_transaction_observation; pub(crate) use self::postgres::insert_transaction_observation;
/// Crate-internal Core descendant invalidation shared through the crate-root facade.
pub(crate) use self::postgres::invalidate_core_descendants_in_transaction;
/// Crate-internal decode descendant invalidation shared through the crate-root facade.
pub(crate) use self::postgres::invalidate_decode_descendants_in_transaction;
/// Crate-internal account-state currentness lookup shared through the crate-root facade.
pub(crate) use self::postgres::is_account_state_current;
/// Crate-internal storage symbol `is_core_extraction_current` shared through the crate-root facade. /// Crate-internal storage symbol `is_core_extraction_current` shared through the crate-root facade.
pub(crate) use self::postgres::is_core_extraction_current; pub(crate) use self::postgres::is_core_extraction_current;
/// Crate-internal storage symbol `is_decode_current` shared through the crate-root facade. /// Crate-internal storage symbol `is_decode_current` shared through the crate-root facade.
pub(crate) use self::postgres::is_decode_current; pub(crate) use self::postgres::is_decode_current;
/// Crate-internal account-observation normalization selection shared through the crate-root facade.
pub(crate) use self::postgres::list_account_observations_for_normalization;
/// Crate-internal storage symbol `list_core_instruction_replay_inputs` shared through the crate-root facade. /// Crate-internal storage symbol `list_core_instruction_replay_inputs` shared through the crate-root facade.
pub(crate) use self::postgres::list_core_instruction_replay_inputs; pub(crate) use self::postgres::list_core_instruction_replay_inputs;
/// Crate-internal storage symbol `list_core_instructions_for_replay` shared through the crate-root facade. /// Crate-internal storage symbol `list_core_instructions_for_replay` shared through the crate-root facade.
@@ -119,10 +131,14 @@ pub(crate) use self::postgres::load_expected_index_counts;
pub(crate) use self::postgres::load_server_version; pub(crate) use self::postgres::load_server_version;
/// Crate-internal storage symbol `load_table_statistics` shared through the crate-root facade. /// Crate-internal storage symbol `load_table_statistics` shared through the crate-root facade.
pub(crate) use self::postgres::load_table_statistics; pub(crate) use self::postgres::load_table_statistics;
/// Crate-internal account-state failure persistence shared through the crate-root facade.
pub(crate) use self::postgres::mark_account_state_failed;
/// Crate-internal storage symbol `mark_core_extraction_failed` shared through the crate-root facade. /// Crate-internal storage symbol `mark_core_extraction_failed` shared through the crate-root facade.
pub(crate) use self::postgres::mark_core_extraction_failed; pub(crate) use self::postgres::mark_core_extraction_failed;
/// Crate-internal storage symbol `mark_decode_failed` shared through the crate-root facade. /// Crate-internal storage symbol `mark_decode_failed` shared through the crate-root facade.
pub(crate) use self::postgres::mark_decode_failed; pub(crate) use self::postgres::mark_decode_failed;
/// Crate-internal account-state persistence shared through the crate-root facade.
pub(crate) use self::postgres::persist_account_state;
/// Crate-internal storage symbol `persist_core_extraction` shared through the crate-root facade. /// Crate-internal storage symbol `persist_core_extraction` shared through the crate-root facade.
pub(crate) use self::postgres::persist_core_extraction; pub(crate) use self::postgres::persist_core_extraction;
/// Crate-internal storage symbol `persist_decode_coverage_declarations` shared through the crate-root facade. /// Crate-internal storage symbol `persist_decode_coverage_declarations` shared through the crate-root facade.
@@ -138,6 +154,8 @@ pub(crate) use self::postgres::postgres_test_guard;
pub(crate) use self::postgres::raw_store_schema_statements; pub(crate) use self::postgres::raw_store_schema_statements;
/// Crate-internal storage symbol `raw_store_table_diagnostic_specs` shared through the crate-root facade. /// Crate-internal storage symbol `raw_store_table_diagnostic_specs` shared through the crate-root facade.
pub(crate) use self::postgres::raw_store_table_diagnostic_specs; pub(crate) use self::postgres::raw_store_table_diagnostic_specs;
/// Crate-internal instruction lifecycle recomputation shared through the crate-root facade.
pub(crate) use self::postgres::recompute_instruction_lifecycle_in_transaction;
/// Crate-internal storage symbol `run_health_check` shared through the crate-root facade. /// Crate-internal storage symbol `run_health_check` shared through the crate-root facade.
pub(crate) use self::postgres::run_health_check; pub(crate) use self::postgres::run_health_check;
/// Crate-internal storage symbol `table_exists` shared through the crate-root facade. /// Crate-internal storage symbol `table_exists` shared through the crate-root facade.
@@ -183,10 +201,20 @@ pub(crate) use self::postgres::validate_postgres_sql_resources;
/// Transitional logical schema contract identifier used until the `0.5.3-pre.3` rebuild. /// Transitional logical schema contract identifier used until the `0.5.3-pre.3` rebuild.
pub use self::constants::STORE_SCHEMA_CONTRACT_VERSION; pub use self::constants::STORE_SCHEMA_CONTRACT_VERSION;
/// Account-state normalization processing-stage identifier.
pub use self::contracts::ACCOUNT_STATE_NORMALIZATION_STAGE;
/// Generic account acquisition observation insert contract. /// Generic account acquisition observation insert contract.
pub use self::contracts::AccountObservationInsert; pub use self::contracts::AccountObservationInsert;
/// Generic account acquisition observation persisted row contract. /// Generic account acquisition observation persisted row contract.
pub use self::contracts::AccountObservationRow; pub use self::contracts::AccountObservationRow;
/// Failed generic account-state normalization attempt.
pub use self::contracts::AccountStateNormalizationFailure;
/// Atomic persistence bundle for one generic Core account state.
pub use self::contracts::AccountStatePersistenceBundle;
/// Bounded generic account-observation selection for Core normalization.
pub use self::contracts::AccountStateSelectionFilter;
/// Generic account observation to Core account-state storage behavior.
pub use self::contracts::AccountStateStore;
/// Transaction acquisition observation origin. /// Transaction acquisition observation origin.
pub use self::contracts::AcquisitionObservationOrigin; pub use self::contracts::AcquisitionObservationOrigin;
/// Generic account acquisition observation status. /// Generic account acquisition observation status.
@@ -227,8 +255,12 @@ pub use self::contracts::CoreInstructionLifecycleMark;
pub use self::contracts::CoreInstructionProcessingState; pub use self::contracts::CoreInstructionProcessingState;
/// Core instruction replay filter contract. /// Core instruction replay filter contract.
pub use self::contracts::CoreInstructionReplayFilter; pub use self::contracts::CoreInstructionReplayFilter;
/// Logical top-level or CPI instruction row selected for replay.
pub use self::contracts::CoreInstructionReplayRow;
/// Core instruction persisted row contract. /// Core instruction persisted row contract.
pub use self::contracts::CoreInstructionRow; pub use self::contracts::CoreInstructionRow;
/// Logical scope of one replayable Core instruction.
pub use self::contracts::CoreInstructionScope;
/// Core log insert contract. /// Core log insert contract.
pub use self::contracts::CoreLogInsert; pub use self::contracts::CoreLogInsert;
/// Core log persisted row contract. /// Core log persisted row contract.
@@ -265,8 +297,16 @@ pub use self::contracts::DecodeSchemaProvenance;
pub use self::contracts::DecodeSelectionFilter; pub use self::contracts::DecodeSelectionFilter;
/// Insert or upsert result contract returned by repositories. /// Insert or upsert result contract returned by repositories.
pub use self::contracts::InsertOutcome; pub use self::contracts::InsertOutcome;
/// Maximum number of account observations selected by one normalization query.
pub use self::contracts::MAX_ACCOUNT_STATE_SELECTION_ROWS;
/// Maximum canonical transactions selected by one Core extraction batch.
pub use self::contracts::MAX_CORE_EXTRACTION_SELECTION_ROWS;
/// Maximum contextual inputs selected by one decode processing batch.
pub use self::contracts::MAX_DECODE_SELECTION_ROWS;
/// Maximum number of materialized rows returned by one bounded query. /// Maximum number of materialized rows returned by one bounded query.
pub use self::contracts::MAX_MATERIALIZED_OUTPUT_QUERY_ROWS; pub use self::contracts::MAX_MATERIALIZED_OUTPUT_QUERY_ROWS;
/// Maximum encoded opaque page-cursor length.
pub use self::contracts::MAX_PAGE_CURSOR_LENGTH;
/// Maximum repository page size. /// Maximum repository page size.
pub use self::contracts::MAX_PAGE_SIZE; pub use self::contracts::MAX_PAGE_SIZE;
/// Maximum number of rows returned by one replay candidate query. /// Maximum number of rows returned by one replay candidate query.
@@ -281,6 +321,8 @@ pub use self::contracts::MaterializedOutputInsert;
pub use self::contracts::MaterializedOutputQueryRow; pub use self::contracts::MaterializedOutputQueryRow;
/// Page request contract for repository list operations. /// Page request contract for repository list operations.
pub use self::contracts::PageRequest; pub use self::contracts::PageRequest;
/// One bounded repository page and its optional continuation cursor.
pub use self::contracts::PageSlice;
/// Stable processing ledger identity. /// Stable processing ledger identity.
pub use self::contracts::ProcessingLedgerIdentity; pub use self::contracts::ProcessingLedgerIdentity;
/// Stable processing ledger status. /// Stable processing ledger status.

View File

@@ -1,5 +1,5 @@
// file: ks-store/src/postgres.rs // file: ks-store/src/postgres.rs
// version: 13 // version: 14
//! Private PostgreSQL implementation of the backend-agnostic store contracts. //! Private PostgreSQL implementation of the backend-agnostic store contracts.
@@ -54,8 +54,10 @@ pub(crate) use self::migrations::table_stats_k_sol_ops_processing_ledger_sql;
pub(crate) use self::migrations::table_stats_k_sol_raw_transactions_sql; pub(crate) use self::migrations::table_stats_k_sol_raw_transactions_sql;
pub(crate) use self::migrations::validate_postgres_sql_resources; pub(crate) use self::migrations::validate_postgres_sql_resources;
pub(crate) use self::query::apply_store_schema; pub(crate) use self::query::apply_store_schema;
pub(crate) use self::query::has_account_observation_key;
pub(crate) use self::query::has_raw_transaction_signature; pub(crate) use self::query::has_raw_transaction_signature;
pub(crate) use self::query::has_transaction_observation_key; pub(crate) use self::query::has_transaction_observation_key;
pub(crate) use self::query::insert_account_observation;
pub(crate) use self::query::insert_core_account_keys; pub(crate) use self::query::insert_core_account_keys;
pub(crate) use self::query::insert_core_balance_changes; pub(crate) use self::query::insert_core_balance_changes;
pub(crate) use self::query::insert_core_inner_instructions; pub(crate) use self::query::insert_core_inner_instructions;
@@ -64,8 +66,12 @@ pub(crate) use self::query::insert_core_logs;
pub(crate) use self::query::insert_core_transaction; pub(crate) use self::query::insert_core_transaction;
pub(crate) use self::query::insert_raw_transaction; pub(crate) use self::query::insert_raw_transaction;
pub(crate) use self::query::insert_transaction_observation; pub(crate) use self::query::insert_transaction_observation;
pub(crate) use self::query::invalidate_core_descendants_in_transaction;
pub(crate) use self::query::invalidate_decode_descendants_in_transaction;
pub(crate) use self::query::is_account_state_current;
pub(crate) use self::query::is_core_extraction_current; pub(crate) use self::query::is_core_extraction_current;
pub(crate) use self::query::is_decode_current; pub(crate) use self::query::is_decode_current;
pub(crate) use self::query::list_account_observations_for_normalization;
pub(crate) use self::query::list_core_instruction_replay_inputs; pub(crate) use self::query::list_core_instruction_replay_inputs;
pub(crate) use self::query::list_core_instructions_for_replay; pub(crate) use self::query::list_core_instructions_for_replay;
pub(crate) use self::query::list_decode_coverage_summary; pub(crate) use self::query::list_decode_coverage_summary;
@@ -80,12 +86,15 @@ pub(crate) use self::query::load_current_schema;
pub(crate) use self::query::load_expected_index_counts; pub(crate) use self::query::load_expected_index_counts;
pub(crate) use self::query::load_server_version; pub(crate) use self::query::load_server_version;
pub(crate) use self::query::load_table_statistics; pub(crate) use self::query::load_table_statistics;
pub(crate) use self::query::mark_account_state_failed;
pub(crate) use self::query::mark_core_extraction_failed; pub(crate) use self::query::mark_core_extraction_failed;
pub(crate) use self::query::mark_decode_failed; pub(crate) use self::query::mark_decode_failed;
pub(crate) use self::query::persist_account_state;
pub(crate) use self::query::persist_core_extraction; pub(crate) use self::query::persist_core_extraction;
pub(crate) use self::query::persist_decode_coverage_declarations; pub(crate) use self::query::persist_decode_coverage_declarations;
pub(crate) use self::query::persist_decode_result; pub(crate) use self::query::persist_decode_result;
pub(crate) use self::query::persist_materialization_result; pub(crate) use self::query::persist_materialization_result;
pub(crate) use self::query::recompute_instruction_lifecycle_in_transaction;
pub(crate) use self::query::run_health_check; pub(crate) use self::query::run_health_check;
pub(crate) use self::query::table_exists; pub(crate) use self::query::table_exists;
pub(crate) use self::query::update_core_instruction_lifecycle; pub(crate) use self::query::update_core_instruction_lifecycle;

View File

@@ -1,5 +1,5 @@
// file: ks-store/src/postgres/migrations.rs // file: ks-store/src/postgres/migrations.rs
// version: 10 // version: 11
//! PostgreSQL schema resources and migration conventions for the storage backend. //! PostgreSQL schema resources and migration conventions for the storage backend.
@@ -83,6 +83,7 @@ const EXPECTED_POSTGRES_INDEX_NAMES: &[&str] = &[
"ix_k_sol_core_inner_instructions_parent", "ix_k_sol_core_inner_instructions_parent",
"ix_k_sol_core_inner_instructions_processing", "ix_k_sol_core_inner_instructions_processing",
"ix_k_sol_core_inner_instructions_program", "ix_k_sol_core_inner_instructions_program",
"ix_k_sol_core_inner_instructions_slot",
"ix_k_sol_core_instructions_processing", "ix_k_sol_core_instructions_processing",
"ix_k_sol_core_instructions_program", "ix_k_sol_core_instructions_program",
"ix_k_sol_core_instructions_slot", "ix_k_sol_core_instructions_slot",
@@ -94,10 +95,13 @@ const EXPECTED_POSTGRES_INDEX_NAMES: &[&str] = &[
"ix_k_sol_decode_coverage_declarations_program", "ix_k_sol_decode_coverage_declarations_program",
"ix_k_sol_decode_coverage_observations_entry", "ix_k_sol_decode_coverage_observations_entry",
"ix_k_sol_decode_coverage_observations_program_status", "ix_k_sol_decode_coverage_observations_program_status",
"ix_k_sol_decode_coverage_observations_signature",
"ix_k_sol_decode_events_family_commit", "ix_k_sol_decode_events_family_commit",
"ix_k_sol_decode_events_program_surface", "ix_k_sol_decode_events_program_surface",
"ix_k_sol_decode_events_signature_path", "ix_k_sol_decode_events_signature_path",
"ix_k_sol_mat_outputs_processor_family_slot", "ix_k_sol_mat_outputs_processor_family_slot",
"ix_k_sol_mat_outputs_signature",
"ix_k_sol_mat_outputs_source_decode_input",
"ix_k_sol_mat_outputs_signature_family", "ix_k_sol_mat_outputs_signature_family",
"ix_k_sol_obs_account_observations_account_slot", "ix_k_sol_obs_account_observations_account_slot",
"ix_k_sol_obs_account_observations_provider_method", "ix_k_sol_obs_account_observations_provider_method",
@@ -1277,6 +1281,12 @@ fn create_pg_index_if_not_exists_ix_k_sol_core_inner_instructions_program() -> &
); );
} }
fn create_pg_index_if_not_exists_ix_k_sol_core_inner_instructions_slot() -> &'static str {
return include_str!(
"../../migrations/postgres/schema/indexes/create_index_if_not_exists_ix_k_sol_core_inner_instructions_slot.sql"
);
}
fn create_pg_index_if_not_exists_ix_k_sol_core_instructions_processing() -> &'static str { fn create_pg_index_if_not_exists_ix_k_sol_core_instructions_processing() -> &'static str {
return include_str!( return include_str!(
"../../migrations/postgres/schema/indexes/create_index_if_not_exists_ix_k_sol_core_instructions_processing.sql" "../../migrations/postgres/schema/indexes/create_index_if_not_exists_ix_k_sol_core_instructions_processing.sql"
@@ -1344,6 +1354,12 @@ fn create_pg_index_if_not_exists_ix_k_sol_decode_coverage_observations_program_s
); );
} }
fn create_pg_index_if_not_exists_ix_k_sol_decode_coverage_observations_signature() -> &'static str {
return include_str!(
"../../migrations/postgres/schema/indexes/create_index_if_not_exists_ix_k_sol_decode_coverage_observations_signature.sql"
);
}
fn create_pg_index_if_not_exists_ix_k_sol_decode_events_family_commit() -> &'static str { fn create_pg_index_if_not_exists_ix_k_sol_decode_events_family_commit() -> &'static str {
return include_str!( return include_str!(
"../../migrations/postgres/schema/indexes/create_index_if_not_exists_ix_k_sol_decode_events_family_commit.sql" "../../migrations/postgres/schema/indexes/create_index_if_not_exists_ix_k_sol_decode_events_family_commit.sql"
@@ -1374,6 +1390,18 @@ fn create_pg_index_if_not_exists_ix_k_sol_mat_outputs_signature_family() -> &'st
); );
} }
fn create_pg_index_if_not_exists_ix_k_sol_mat_outputs_signature() -> &'static str {
return include_str!(
"../../migrations/postgres/schema/indexes/create_index_if_not_exists_ix_k_sol_mat_outputs_signature.sql"
);
}
fn create_pg_index_if_not_exists_ix_k_sol_mat_outputs_source_decode_input() -> &'static str {
return include_str!(
"../../migrations/postgres/schema/indexes/create_index_if_not_exists_ix_k_sol_mat_outputs_source_decode_input.sql"
);
}
fn create_pg_index_if_not_exists_ix_k_sol_obs_account_observations_account_slot() -> &'static str { fn create_pg_index_if_not_exists_ix_k_sol_obs_account_observations_account_slot() -> &'static str {
return include_str!( return include_str!(
"../../migrations/postgres/schema/indexes/create_index_if_not_exists_ix_k_sol_obs_account_observations_account_slot.sql" "../../migrations/postgres/schema/indexes/create_index_if_not_exists_ix_k_sol_obs_account_observations_account_slot.sql"
@@ -1942,6 +1970,7 @@ pub(crate) fn core_store_schema_statements() -> std::vec::Vec<&'static str> {
create_pg_index_if_not_exists_ix_k_sol_core_inner_instructions_parent(), create_pg_index_if_not_exists_ix_k_sol_core_inner_instructions_parent(),
create_pg_index_if_not_exists_ix_k_sol_core_inner_instructions_processing(), create_pg_index_if_not_exists_ix_k_sol_core_inner_instructions_processing(),
create_pg_index_if_not_exists_ix_k_sol_core_inner_instructions_program(), create_pg_index_if_not_exists_ix_k_sol_core_inner_instructions_program(),
create_pg_index_if_not_exists_ix_k_sol_core_inner_instructions_slot(),
create_pg_index_if_not_exists_ix_k_sol_core_instructions_processing(), create_pg_index_if_not_exists_ix_k_sol_core_instructions_processing(),
create_pg_index_if_not_exists_ix_k_sol_core_instructions_program(), create_pg_index_if_not_exists_ix_k_sol_core_instructions_program(),
create_pg_index_if_not_exists_ix_k_sol_core_instructions_slot(), create_pg_index_if_not_exists_ix_k_sol_core_instructions_slot(),
@@ -1985,11 +2014,14 @@ pub(crate) fn decode_store_schema_statements() -> std::vec::Vec<&'static str> {
create_pg_index_if_not_exists_ix_k_sol_decode_coverage_declarations_program(), create_pg_index_if_not_exists_ix_k_sol_decode_coverage_declarations_program(),
create_pg_index_if_not_exists_ix_k_sol_decode_coverage_observations_entry(), create_pg_index_if_not_exists_ix_k_sol_decode_coverage_observations_entry(),
create_pg_index_if_not_exists_ix_k_sol_decode_coverage_observations_program_status(), create_pg_index_if_not_exists_ix_k_sol_decode_coverage_observations_program_status(),
create_pg_index_if_not_exists_ix_k_sol_decode_coverage_observations_signature(),
create_pg_index_if_not_exists_ix_k_sol_decode_events_family_commit(), create_pg_index_if_not_exists_ix_k_sol_decode_events_family_commit(),
create_pg_index_if_not_exists_ix_k_sol_decode_events_program_surface(), create_pg_index_if_not_exists_ix_k_sol_decode_events_program_surface(),
create_pg_index_if_not_exists_ix_k_sol_decode_events_signature_path(), create_pg_index_if_not_exists_ix_k_sol_decode_events_signature_path(),
create_pg_index_if_not_exists_ix_k_sol_mat_outputs_processor_family_slot(), create_pg_index_if_not_exists_ix_k_sol_mat_outputs_processor_family_slot(),
create_pg_index_if_not_exists_ix_k_sol_mat_outputs_signature(),
create_pg_index_if_not_exists_ix_k_sol_mat_outputs_signature_family(), create_pg_index_if_not_exists_ix_k_sol_mat_outputs_signature_family(),
create_pg_index_if_not_exists_ix_k_sol_mat_outputs_source_decode_input(),
create_pg_index_if_not_exists_ux_k_sol_decode_coverage_declarations_identity(), create_pg_index_if_not_exists_ux_k_sol_decode_coverage_declarations_identity(),
create_pg_index_if_not_exists_ux_k_sol_decode_coverage_observations_identity(), create_pg_index_if_not_exists_ux_k_sol_decode_coverage_observations_identity(),
create_pg_index_if_not_exists_ux_k_sol_decode_events_processor_input_event(), create_pg_index_if_not_exists_ux_k_sol_decode_events_processor_input_event(),
@@ -2208,12 +2240,12 @@ mod tests {
} }
#[test] #[test]
fn candidate_baseline_inventory_has_sixteen_tables_and_seventy_five_indexes() { fn candidate_baseline_inventory_has_sixteen_tables_and_seventy_nine_indexes() {
let table_count = super::RAW_STORE_TABLE_NAMES.len() let table_count = super::RAW_STORE_TABLE_NAMES.len()
+ super::CORE_STORE_TABLE_NAMES.len() + super::CORE_STORE_TABLE_NAMES.len()
+ super::DECODE_STORE_TABLE_NAMES.len(); + super::DECODE_STORE_TABLE_NAMES.len();
assert_eq!(table_count, 16); assert_eq!(table_count, 16);
assert_eq!(crate::expected_postgres_index_names().len(), 75); assert_eq!(crate::expected_postgres_index_names().len(), 79);
} }
#[test] #[test]
@@ -2232,7 +2264,7 @@ mod tests {
} }
let maintenance_count = super::truncate_store_schema_statements().len() let maintenance_count = super::truncate_store_schema_statements().len()
+ super::drop_store_schema_statements().len(); + super::drop_store_schema_statements().len();
assert_eq!(schema_count + maintenance_count, 236); assert_eq!(schema_count + maintenance_count, 240);
} }
#[test] #[test]

View File

@@ -1,17 +1,23 @@
// file: ks-store/src/postgres/query.rs // file: ks-store/src/postgres/query.rs
// version: 5 // version: 6
//! PostgreSQL query modules. //! PostgreSQL query modules.
mod account_state_queries;
mod core_extraction_queries; mod core_extraction_queries;
mod core_queries; mod core_queries;
mod decode_pipeline_queries; mod decode_pipeline_queries;
mod health_queries; mod health_queries;
mod invalidation_queries;
mod raw_queries; mod raw_queries;
mod replay_candidate_queries; mod replay_candidate_queries;
mod schema_queries; mod schema_queries;
mod table_diagnostics_queries; mod table_diagnostics_queries;
pub(crate) use self::account_state_queries::is_account_state_current;
pub(crate) use self::account_state_queries::list_account_observations_for_normalization;
pub(crate) use self::account_state_queries::mark_account_state_failed;
pub(crate) use self::account_state_queries::persist_account_state;
pub(crate) use self::core_extraction_queries::is_core_extraction_current; pub(crate) use self::core_extraction_queries::is_core_extraction_current;
pub(crate) use self::core_extraction_queries::list_raw_transactions_for_core_extraction; pub(crate) use self::core_extraction_queries::list_raw_transactions_for_core_extraction;
pub(crate) use self::core_extraction_queries::mark_core_extraction_failed; pub(crate) use self::core_extraction_queries::mark_core_extraction_failed;
@@ -37,8 +43,13 @@ pub(crate) use self::decode_pipeline_queries::persist_materialization_result;
pub(crate) use self::health_queries::load_current_schema; pub(crate) use self::health_queries::load_current_schema;
pub(crate) use self::health_queries::load_server_version; pub(crate) use self::health_queries::load_server_version;
pub(crate) use self::health_queries::run_health_check; pub(crate) use self::health_queries::run_health_check;
pub(crate) use self::invalidation_queries::invalidate_core_descendants_in_transaction;
pub(crate) use self::invalidation_queries::invalidate_decode_descendants_in_transaction;
pub(crate) use self::invalidation_queries::recompute_instruction_lifecycle_in_transaction;
pub(crate) use self::raw_queries::has_account_observation_key;
pub(crate) use self::raw_queries::has_raw_transaction_signature; pub(crate) use self::raw_queries::has_raw_transaction_signature;
pub(crate) use self::raw_queries::has_transaction_observation_key; pub(crate) use self::raw_queries::has_transaction_observation_key;
pub(crate) use self::raw_queries::insert_account_observation;
pub(crate) use self::raw_queries::insert_raw_transaction; pub(crate) use self::raw_queries::insert_raw_transaction;
pub(crate) use self::raw_queries::insert_transaction_observation; pub(crate) use self::raw_queries::insert_transaction_observation;
pub(crate) use self::raw_queries::update_raw_payload_lifecycle; pub(crate) use self::raw_queries::update_raw_payload_lifecycle;

View File

@@ -0,0 +1,708 @@
// file: ks-store/src/postgres/query/account_state_queries.rs
// version: 4
//! PostgreSQL queries for generic account-observation to Core account-state normalization.
use sqlx::Row; // rust-rules: trait-import
pub(crate) async fn list_account_observations_for_normalization(
pool: &sqlx::PgPool,
filter: &crate::AccountStateSelectionFilter,
) -> ks_core::Result<std::vec::Vec<crate::AccountObservationRow>> {
let min_slot_result = optional_sql_bigint_from_u64(filter.min_slot);
let min_slot = match min_slot_result {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let max_slot_result = optional_sql_bigint_from_u64(filter.max_slot);
let max_slot = match max_slot_result {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let started_at = std::time::Instant::now();
let query_result = sqlx::query(
"SELECT o.id, o.observation_key, o.account_key, o.context_slot, o.owner, o.lamports::TEXT AS lamports, o.executable, o.rent_epoch::TEXT AS rent_epoch, o.space, o.data_base64, o.data_hash, o.provider, o.endpoint_code, o.protocol, o.acquisition_method, o.commitment, o.capture_session_id, o.filter_code, o.origin, o.detected_at, o.received_at, o.normalized_at, o.persisted_at, o.payload_size_bytes, o.source_payload_hash, o.status, o.error_code, o.error_message FROM k_sol_obs_account_observations o WHERE o.owner IS NOT NULL AND o.lamports IS NOT NULL AND o.executable IS NOT NULL AND o.rent_epoch IS NOT NULL AND o.space IS NOT NULL AND o.data_base64 IS NOT NULL AND o.data_hash IS NOT NULL AND (cardinality($1::TEXT[]) = 0 OR o.account_key = ANY($1)) AND ($2::BIGINT IS NULL OR o.context_slot >= $2) AND ($3::BIGINT IS NULL OR o.context_slot <= $3) AND ($4 OR NOT EXISTS (SELECT 1 FROM k_sol_ops_processing_ledger l WHERE l.stage = 'account_state_normalization' AND l.processor_name = $5 AND l.processor_version = $6 AND l.input_key = o.observation_key AND l.status = 'succeeded')) ORDER BY o.context_slot ASC, o.id ASC LIMIT $7",
)
.bind(&filter.account_keys)
.bind(min_slot)
.bind(max_slot)
.bind(filter.include_current)
.bind(filter.processor_name.as_str())
.bind(filter.processor_version.as_str())
.bind(i64::from(filter.limit))
.fetch_all(pool)
.await;
let rows = match query_result {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => {
trace_query_failure("list_account_observations", started_at, &error);
return std::result::Result::Err(ks_core::Error::db(format!(
"postgres account observation normalization query failed: {error}"
)));
},
};
let mut output = std::vec::Vec::with_capacity(rows.len());
for row in rows {
let mapped_result = map_account_observation_row(&row);
match mapped_result {
std::result::Result::Ok(value) => output.push(value),
std::result::Result::Err(error) => return std::result::Result::Err(error),
}
}
trace_query_success("list_account_observations", started_at, output.len());
return std::result::Result::Ok(output);
}
pub(crate) async fn is_account_state_current(
pool: &sqlx::PgPool,
identity: &crate::ProcessingLedgerIdentity,
) -> ks_core::Result<bool> {
if identity.stage != crate::ACCOUNT_STATE_NORMALIZATION_STAGE {
return std::result::Result::Err(ks_core::Error::db(
"account-state current check stage is invalid",
));
}
let started_at = std::time::Instant::now();
let query_result = sqlx::query_scalar::<sqlx::Postgres, bool>(
"SELECT EXISTS(SELECT 1 FROM k_sol_ops_processing_ledger WHERE stage = $1 AND processor_name = $2 AND processor_version = $3 AND input_key = $4 AND input_hash = $5 AND status = 'succeeded')",
)
.bind(identity.stage.as_str())
.bind(identity.processor_name.as_str())
.bind(identity.processor_version.as_str())
.bind(identity.input_key.as_str())
.bind(identity.input_hash.as_str())
.fetch_one(pool)
.await;
return match query_result {
std::result::Result::Ok(value) => {
trace_query_success(
"is_account_state_current",
started_at,
if value { 1_usize } else { 0_usize },
);
std::result::Result::Ok(value)
},
std::result::Result::Err(error) => {
trace_query_failure("is_account_state_current", started_at, &error);
std::result::Result::Err(ks_core::Error::db(format!(
"postgres account-state ledger current check failed: {error}"
)))
},
};
}
pub(crate) async fn persist_account_state(
pool: &sqlx::PgPool,
bundle: &crate::AccountStatePersistenceBundle,
force_replay: bool,
) -> ks_core::Result<crate::InsertOutcome> {
let validation_result = bundle.validate();
if let std::result::Result::Err(error) = validation_result {
return std::result::Result::Err(error);
}
if !force_replay {
let current_result = crate::is_account_state_current(pool, &bundle.ledger_identity).await;
match current_result {
std::result::Result::Ok(true) => {
return std::result::Result::Ok(crate::InsertOutcome::new(0, 0, 1));
},
std::result::Result::Ok(false) => {},
std::result::Result::Err(error) => return std::result::Result::Err(error),
}
}
let started_at = std::time::Instant::now();
let transaction_result = pool.begin().await;
let mut transaction = match transaction_result {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => {
return std::result::Result::Err(ks_core::Error::db(format!(
"postgres account-state transaction begin failed: {error}"
)));
},
};
let source_result =
load_source_observation(&mut transaction, bundle.state.source_observation_id).await;
let source = match source_result {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let lineage_result = validate_source_lineage(&source, bundle);
if let std::result::Result::Err(error) = lineage_result {
return std::result::Result::Err(error);
}
let existed_result = sqlx::query_scalar::<sqlx::Postgres, bool>(
"SELECT EXISTS(SELECT 1 FROM k_sol_core_account_states WHERE source_observation_id = $1)",
)
.bind(bundle.state.source_observation_id)
.fetch_one(&mut *transaction)
.await;
let existed = match existed_result {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => {
return std::result::Result::Err(ks_core::Error::db(format!(
"postgres account-state existing row query failed: {error}"
)));
},
};
let slot_result = sql_bigint_from_u64(
bundle.state.slot,
"account-state slot does not fit into PostgreSQL BIGINT",
);
let slot = match slot_result {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let space_result = sql_bigint_from_u64(
bundle.state.space,
"account-state space does not fit into PostgreSQL BIGINT",
);
let space = match space_result {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let lamports = bundle.state.lamports.to_string();
let rent_epoch = bundle.state.rent_epoch.to_string();
let upsert_result = sqlx::query(
"INSERT INTO k_sol_core_account_states (source_observation_id, account_key, slot, owner, lamports, executable, rent_epoch, space, data_base64, data_hash) VALUES ($1, $2, $3, $4, $5::NUMERIC, $6, $7::NUMERIC, $8, $9, $10) ON CONFLICT (source_observation_id) DO UPDATE SET account_key = EXCLUDED.account_key, slot = EXCLUDED.slot, owner = EXCLUDED.owner, lamports = EXCLUDED.lamports, executable = EXCLUDED.executable, rent_epoch = EXCLUDED.rent_epoch, space = EXCLUDED.space, data_base64 = EXCLUDED.data_base64, data_hash = EXCLUDED.data_hash",
)
.bind(bundle.state.source_observation_id)
.bind(bundle.state.account_key.as_str())
.bind(slot)
.bind(bundle.state.owner.as_str())
.bind(lamports.as_str())
.bind(bundle.state.executable)
.bind(rent_epoch.as_str())
.bind(space)
.bind(bundle.state.data_base64.as_str())
.bind(bundle.state.data_hash.as_str())
.execute(&mut *transaction)
.await;
if let std::result::Result::Err(error) = upsert_result {
return std::result::Result::Err(ks_core::Error::db(format!(
"postgres account-state upsert failed: {error}"
)));
}
let observation_result = sqlx::query(
"UPDATE k_sol_obs_account_observations SET status = 'normalized', normalized_at = NOW(), error_code = NULL, error_message = NULL WHERE id = $1",
)
.bind(bundle.state.source_observation_id)
.execute(&mut *transaction)
.await;
if let std::result::Result::Err(error) = observation_result {
return std::result::Result::Err(ks_core::Error::db(format!(
"postgres account observation normalization update failed: {error}"
)));
}
let ledger_result = upsert_ledger_terminal(
&mut transaction,
&bundle.ledger_identity,
"succeeded",
std::option::Option::None,
std::option::Option::None,
)
.await;
if let std::result::Result::Err(error) = ledger_result {
return std::result::Result::Err(error);
}
let commit_result = transaction.commit().await;
if let std::result::Result::Err(error) = commit_result {
return std::result::Result::Err(ks_core::Error::db(format!(
"postgres account-state transaction commit failed: {error}"
)));
}
let outcome = if existed {
crate::InsertOutcome::new(0, 1, 0)
} else {
crate::InsertOutcome::new(1, 0, 0)
};
trace_query_success("persist_account_state", started_at, 1);
return std::result::Result::Ok(outcome);
}
pub(crate) async fn mark_account_state_failed(
pool: &sqlx::PgPool,
failure: &crate::AccountStateNormalizationFailure,
) -> ks_core::Result<crate::InsertOutcome> {
let validation_result = failure.validate();
if let std::result::Result::Err(error) = validation_result {
return std::result::Result::Err(error);
}
let started_at = std::time::Instant::now();
let transaction_result = pool.begin().await;
let mut transaction = match transaction_result {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => {
return std::result::Result::Err(ks_core::Error::db(format!(
"postgres account-state failure transaction begin failed: {error}"
)));
},
};
let update_result = sqlx::query(
"UPDATE k_sol_obs_account_observations SET status = 'failed', error_code = $1, error_message = $2 WHERE id = $3",
)
.bind(failure.error_code.as_str())
.bind(failure.error_message.as_str())
.bind(failure.source_observation_id)
.execute(&mut *transaction)
.await;
let rows_affected = match update_result {
std::result::Result::Ok(value) => value.rows_affected(),
std::result::Result::Err(error) => {
return std::result::Result::Err(ks_core::Error::db(format!(
"postgres account observation failure update failed: {error}"
)));
},
};
if rows_affected == 0 {
return std::result::Result::Err(ks_core::Error::db(
"account-state normalization source observation does not exist",
));
}
let ledger_result = upsert_ledger_terminal(
&mut transaction,
&failure.ledger_identity,
"failed",
std::option::Option::Some(failure.error_code.as_str()),
std::option::Option::Some(failure.error_message.as_str()),
)
.await;
if let std::result::Result::Err(error) = ledger_result {
return std::result::Result::Err(error);
}
let commit_result = transaction.commit().await;
if let std::result::Result::Err(error) = commit_result {
return std::result::Result::Err(ks_core::Error::db(format!(
"postgres account-state failure transaction commit failed: {error}"
)));
}
trace_query_success("mark_account_state_failed", started_at, 1);
return std::result::Result::Ok(crate::InsertOutcome::new(0, 1, 0));
}
struct SourceAccountObservation {
observation_key: std::string::String,
account_key: std::string::String,
context_slot: i64,
owner: std::string::String,
lamports: std::string::String,
executable: bool,
rent_epoch: std::string::String,
space: i64,
data_base64: std::string::String,
data_hash: std::string::String,
}
async fn load_source_observation(
transaction: &mut sqlx::Transaction<'_, sqlx::Postgres>,
source_observation_id: i64,
) -> ks_core::Result<SourceAccountObservation> {
let query_result = sqlx::query(
"SELECT observation_key, account_key, context_slot, owner, lamports::TEXT AS lamports, executable, rent_epoch::TEXT AS rent_epoch, space, data_base64, data_hash FROM k_sol_obs_account_observations WHERE id = $1 AND owner IS NOT NULL AND lamports IS NOT NULL AND executable IS NOT NULL AND rent_epoch IS NOT NULL AND space IS NOT NULL AND data_base64 IS NOT NULL AND data_hash IS NOT NULL LIMIT 1",
)
.bind(source_observation_id)
.fetch_optional(&mut **transaction)
.await;
let row = match query_result {
std::result::Result::Ok(std::option::Option::Some(value)) => value,
std::result::Result::Ok(std::option::Option::None) => {
return std::result::Result::Err(ks_core::Error::db(
"account-state normalization requires one complete source observation",
));
},
std::result::Result::Err(error) => {
return std::result::Result::Err(ks_core::Error::db(format!(
"postgres account-state source observation query failed: {error}"
)));
},
};
let observation_key = match read_row(&row, "observation_key") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let account_key = match read_row(&row, "account_key") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let context_slot = match read_row(&row, "context_slot") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let owner = match read_row(&row, "owner") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let lamports = match read_row(&row, "lamports") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let executable = match read_row(&row, "executable") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let rent_epoch = match read_row(&row, "rent_epoch") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let space = match read_row(&row, "space") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let data_base64 = match read_row(&row, "data_base64") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let data_hash = match read_row(&row, "data_hash") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
return std::result::Result::Ok(SourceAccountObservation {
observation_key,
account_key,
context_slot,
owner,
lamports,
executable,
rent_epoch,
space,
data_base64,
data_hash,
});
}
fn validate_source_lineage(
source: &SourceAccountObservation,
bundle: &crate::AccountStatePersistenceBundle,
) -> ks_core::Result<()> {
let slot_result = u64::try_from(source.context_slot);
let source_slot = match slot_result {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => {
return std::result::Result::Err(ks_core::Error::db(format!(
"account observation context slot is negative: {error}"
)));
},
};
let space_result = u64::try_from(source.space);
let source_space = match space_result {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => {
return std::result::Result::Err(ks_core::Error::db(format!(
"account observation space is negative: {error}"
)));
},
};
if source.observation_key != bundle.ledger_identity.input_key
|| source.account_key != bundle.state.account_key
|| source_slot != bundle.state.slot
|| source.owner != bundle.state.owner
|| source.lamports != bundle.state.lamports.to_string()
|| source.executable != bundle.state.executable
|| source.rent_epoch != bundle.state.rent_epoch.to_string()
|| source_space != bundle.state.space
|| source.data_base64 != bundle.state.data_base64
|| source.data_hash != bundle.state.data_hash
{
return std::result::Result::Err(ks_core::Error::db(
"account-state persistence bundle does not match its source observation",
));
}
return std::result::Result::Ok(());
}
async fn upsert_ledger_terminal(
transaction: &mut sqlx::Transaction<'_, sqlx::Postgres>,
identity: &crate::ProcessingLedgerIdentity,
status: &str,
error_code: std::option::Option<&str>,
error_message: std::option::Option<&str>,
) -> ks_core::Result<()> {
let query_result = sqlx::query(
"INSERT INTO k_sol_ops_processing_ledger (stage, processor_name, processor_version, input_key, input_hash, status, attempt_count, started_at, finished_at, error_code, error_message) VALUES ($1, $2, $3, $4, $5, $6, 1, NOW(), NOW(), $7, $8) ON CONFLICT (stage, processor_name, processor_version, input_key) DO UPDATE SET input_hash = EXCLUDED.input_hash, status = EXCLUDED.status, attempt_count = k_sol_ops_processing_ledger.attempt_count + 1, started_at = NOW(), finished_at = NOW(), error_code = EXCLUDED.error_code, error_message = EXCLUDED.error_message, updated_at = NOW()",
)
.bind(identity.stage.as_str())
.bind(identity.processor_name.as_str())
.bind(identity.processor_version.as_str())
.bind(identity.input_key.as_str())
.bind(identity.input_hash.as_str())
.bind(status)
.bind(error_code)
.bind(error_message)
.execute(&mut **transaction)
.await;
if let std::result::Result::Err(error) = query_result {
return std::result::Result::Err(ks_core::Error::db(format!(
"postgres account-state processing ledger upsert failed: {error}"
)));
}
return std::result::Result::Ok(());
}
fn map_account_observation_row(
row: &sqlx::postgres::PgRow,
) -> ks_core::Result<crate::AccountObservationRow> {
let origin_text: std::string::String = match read_row(row, "origin") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let status_text: std::string::String = match read_row(row, "status") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let id = match read_row(row, "id") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let observation_key = match read_row(row, "observation_key") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let account_key = match read_row(row, "account_key") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let context_slot = match read_row(row, "context_slot") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let owner = match read_row(row, "owner") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let lamports = match read_row(row, "lamports") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let executable = match read_row(row, "executable") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let rent_epoch = match read_row(row, "rent_epoch") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let space = match read_row(row, "space") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let data_base64 = match read_row(row, "data_base64") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let data_hash = match read_row(row, "data_hash") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let provider = match read_row(row, "provider") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let endpoint_code = match read_row(row, "endpoint_code") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let protocol = match read_row(row, "protocol") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let acquisition_method = match read_row(row, "acquisition_method") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let commitment = match read_row(row, "commitment") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let capture_session_id = match read_row(row, "capture_session_id") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let filter_code = match read_row(row, "filter_code") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let detected_at = match read_row(row, "detected_at") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let received_at = match read_row(row, "received_at") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let normalized_at = match read_row(row, "normalized_at") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let persisted_at = match read_row(row, "persisted_at") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let payload_size_bytes = match read_row(row, "payload_size_bytes") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let source_payload_hash = match read_row(row, "source_payload_hash") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let error_code = match read_row(row, "error_code") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let error_message = match read_row(row, "error_message") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let origin = match observation_origin_from_sql(origin_text.as_str()) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let status = match observation_status_from_sql(status_text.as_str()) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
return std::result::Result::Ok(crate::AccountObservationRow {
id,
observation_key,
account_key,
context_slot,
owner,
lamports,
executable,
rent_epoch,
space,
data_base64,
data_hash,
provider,
endpoint_code,
protocol,
acquisition_method,
commitment,
capture_session_id,
filter_code,
origin,
detected_at,
received_at,
normalized_at,
persisted_at,
payload_size_bytes,
source_payload_hash,
status,
error_code,
error_message,
});
}
fn observation_origin_from_sql(
value: &str,
) -> ks_core::Result<crate::AcquisitionObservationOrigin> {
return match value {
"live" => std::result::Result::Ok(crate::AcquisitionObservationOrigin::Live),
"backfill" => std::result::Result::Ok(crate::AcquisitionObservationOrigin::Backfill),
"replay" => std::result::Result::Ok(crate::AcquisitionObservationOrigin::Replay),
"repair" => std::result::Result::Ok(crate::AcquisitionObservationOrigin::Repair),
"migration" => std::result::Result::Ok(crate::AcquisitionObservationOrigin::Migration),
_unknown => {
std::result::Result::Err(ks_core::Error::db("unknown account observation origin"))
},
};
}
fn observation_status_from_sql(
value: &str,
) -> ks_core::Result<crate::AcquisitionObservationStatus> {
return match value {
"detected" => std::result::Result::Ok(crate::AcquisitionObservationStatus::Detected),
"received" => std::result::Result::Ok(crate::AcquisitionObservationStatus::Received),
"normalized" => std::result::Result::Ok(crate::AcquisitionObservationStatus::Normalized),
"persisted" => std::result::Result::Ok(crate::AcquisitionObservationStatus::Persisted),
"failed" => std::result::Result::Ok(crate::AcquisitionObservationStatus::Failed),
"missing" => std::result::Result::Ok(crate::AcquisitionObservationStatus::Missing),
_unknown => {
std::result::Result::Err(ks_core::Error::db("unknown account observation status"))
},
};
}
fn optional_sql_bigint_from_u64(
value: std::option::Option<u64>,
) -> ks_core::Result<std::option::Option<i64>> {
return match value {
std::option::Option::Some(raw) => {
let converted_result = i64::try_from(raw);
match converted_result {
std::result::Result::Ok(converted) => {
std::result::Result::Ok(std::option::Option::Some(converted))
},
std::result::Result::Err(error) => std::result::Result::Err(ks_core::Error::db(
format!("unsigned value does not fit into PostgreSQL BIGINT: {error}"),
)),
}
},
std::option::Option::None => std::result::Result::Ok(std::option::Option::None),
};
}
fn sql_bigint_from_u64(value: u64, message: &str) -> ks_core::Result<i64> {
let result = i64::try_from(value);
return match result {
std::result::Result::Ok(converted) => std::result::Result::Ok(converted),
std::result::Result::Err(error) => {
std::result::Result::Err(ks_core::Error::db(format!("{message}: {error}")))
},
};
}
fn read_row<'row, T>(row: &'row sqlx::postgres::PgRow, column: &str) -> ks_core::Result<T>
where
T: sqlx::Decode<'row, sqlx::Postgres> + sqlx::Type<sqlx::Postgres>,
{
let value_result = row.try_get::<T, &str>(column);
return match value_result {
std::result::Result::Ok(value) => std::result::Result::Ok(value),
std::result::Result::Err(error) => std::result::Result::Err(ks_core::Error::db(format!(
"postgres account-state row read failed for {column}: {error}"
))),
};
}
fn trace_query_success(action: &str, started_at: std::time::Instant, rows: usize) {
let elapsed_ms = started_at.elapsed().as_secs_f64() * 1000.0;
tracing::trace!(target: crate::TRACING_TARGET, backend = "postgres", domain = "ks-store.pg", action = action, elapsed_ms, rows, outcome = "success", "PostgreSQL store operation completed");
}
fn trace_query_failure(action: &str, started_at: std::time::Instant, error: &sqlx::Error) {
let elapsed_ms = started_at.elapsed().as_secs_f64() * 1000.0;
tracing::trace!(target: crate::TRACING_TARGET, backend = "postgres", domain = "ks-store.pg", action = action, elapsed_ms, error = %error, outcome = "error", "PostgreSQL store operation failed");
}
#[cfg(test)]
mod tests {
#[test]
fn account_state_selection_is_processor_version_aware() {
let source = include_str!("account_state_queries.rs");
let tests_marker = source.find("#[cfg(test)]");
let production_source = match tests_marker {
std::option::Option::Some(index) => &source[..index],
std::option::Option::None => source,
};
assert!(production_source.contains("l.processor_name = $5"));
assert!(production_source.contains("l.processor_version = $6"));
assert!(production_source.contains("l.input_key = o.observation_key"));
assert!(!production_source.contains("NOT EXISTS (SELECT 1 FROM k_sol_core_account_states"));
}
#[test]
fn account_state_bigint_conversion_rejects_unsigned_overflow() {
assert!(super::sql_bigint_from_u64(u64::MAX, "overflow").is_err());
assert!(super::optional_sql_bigint_from_u64(std::option::Option::Some(u64::MAX)).is_err());
}
#[test]
fn account_state_status_mapping_is_closed() {
assert_eq!(
super::observation_status_from_sql("normalized"),
std::result::Result::Ok(crate::AcquisitionObservationStatus::Normalized)
);
assert!(super::observation_status_from_sql("other").is_err());
}
}

View File

@@ -1,5 +1,5 @@
// file: ks-store/src/postgres/query/core_extraction_queries.rs // file: ks-store/src/postgres/query/core_extraction_queries.rs
// version: 8 // version: 9
//! PostgreSQL queries for atomic canonical transaction to core extraction. //! PostgreSQL queries for atomic canonical transaction to core extraction.
@@ -22,6 +22,7 @@ pub(crate) async fn list_raw_transactions_for_core_extraction(
let processing_state = filter.processing_state.map(raw_processing_state_to_sql); let processing_state = filter.processing_state.map(raw_processing_state_to_sql);
let limit = i64::from(filter.limit); let limit = i64::from(filter.limit);
let signatures_are_empty = filter.signatures.is_empty(); let signatures_are_empty = filter.signatures.is_empty();
let started_at = std::time::Instant::now();
let query_result = sqlx::query( let query_result = sqlx::query(
"SELECT raw.id, raw.signature, raw.slot, raw.block_time, raw.canonical_json, raw.canonical_json_hash, raw.canonical_format_version, raw.retention_state, raw.processing_state, raw.created_at, raw.updated_at FROM k_sol_raw_transactions raw WHERE ($1 OR raw.signature = ANY($2)) AND ($3::BIGINT IS NULL OR raw.slot >= $3) AND ($4::BIGINT IS NULL OR raw.slot <= $4) AND ($5::TEXT IS NULL OR raw.processing_state = $5) AND ($6::TEXT IS NULL OR EXISTS (SELECT 1 FROM k_sol_core_instructions instruction WHERE instruction.signature = raw.signature AND instruction.program_id = $6)) ORDER BY raw.slot ASC, raw.signature ASC LIMIT $7", "SELECT raw.id, raw.signature, raw.slot, raw.block_time, raw.canonical_json, raw.canonical_json_hash, raw.canonical_format_version, raw.retention_state, raw.processing_state, raw.created_at, raw.updated_at FROM k_sol_raw_transactions raw WHERE ($1 OR raw.signature = ANY($2)) AND ($3::BIGINT IS NULL OR raw.slot >= $3) AND ($4::BIGINT IS NULL OR raw.slot <= $4) AND ($5::TEXT IS NULL OR raw.processing_state = $5) AND ($6::TEXT IS NULL OR EXISTS (SELECT 1 FROM k_sol_core_instructions instruction WHERE instruction.signature = raw.signature AND instruction.program_id = $6)) ORDER BY raw.slot ASC, raw.signature ASC LIMIT $7",
) )
@@ -37,6 +38,7 @@ pub(crate) async fn list_raw_transactions_for_core_extraction(
let rows = match query_result { let rows = match query_result {
std::result::Result::Ok(value) => value, std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => { std::result::Result::Err(error) => {
trace_query_failure("list_raw_transactions_for_core_extraction", started_at, &error);
return std::result::Result::Err(ks_core::Error::db(format!( return std::result::Result::Err(ks_core::Error::db(format!(
"postgres core extraction raw selection failed: {error}" "postgres core extraction raw selection failed: {error}"
))); )));
@@ -51,6 +53,7 @@ pub(crate) async fn list_raw_transactions_for_core_extraction(
}; };
output.push(mapped); output.push(mapped);
} }
trace_query_success("list_raw_transactions_for_core_extraction", started_at, output.len());
return std::result::Result::Ok(output); return std::result::Result::Ok(output);
} }
@@ -58,6 +61,7 @@ pub(crate) async fn is_core_extraction_current(
pool: &sqlx::PgPool, pool: &sqlx::PgPool,
identity: &crate::ProcessingLedgerIdentity, identity: &crate::ProcessingLedgerIdentity,
) -> ks_core::Result<bool> { ) -> ks_core::Result<bool> {
let started_at = std::time::Instant::now();
let query_result = sqlx::query_scalar::<sqlx::Postgres, bool>( let query_result = sqlx::query_scalar::<sqlx::Postgres, bool>(
"SELECT EXISTS(SELECT 1 FROM k_sol_ops_processing_ledger WHERE stage = $1 AND processor_name = $2 AND processor_version = $3 AND input_key = $4 AND input_hash = $5 AND status = 'succeeded')", "SELECT EXISTS(SELECT 1 FROM k_sol_ops_processing_ledger WHERE stage = $1 AND processor_name = $2 AND processor_version = $3 AND input_key = $4 AND input_hash = $5 AND status = 'succeeded')",
) )
@@ -69,10 +73,20 @@ pub(crate) async fn is_core_extraction_current(
.fetch_one(pool) .fetch_one(pool)
.await; .await;
return match query_result { return match query_result {
std::result::Result::Ok(value) => std::result::Result::Ok(value), std::result::Result::Ok(value) => {
std::result::Result::Err(error) => std::result::Result::Err(ks_core::Error::db(format!( trace_query_success(
"is_core_extraction_current",
started_at,
if value { 1_usize } else { 0_usize },
);
std::result::Result::Ok(value)
},
std::result::Result::Err(error) => {
trace_query_failure("is_core_extraction_current", started_at, &error);
std::result::Result::Err(ks_core::Error::db(format!(
"postgres core extraction ledger lookup failed: {error}" "postgres core extraction ledger lookup failed: {error}"
))), )))
},
}; };
} }
@@ -81,6 +95,7 @@ pub(crate) async fn persist_core_extraction(
bundle: &crate::CoreExtractionBundle, bundle: &crate::CoreExtractionBundle,
_force_replay: bool, _force_replay: bool,
) -> ks_core::Result<crate::InsertOutcome> { ) -> ks_core::Result<crate::InsertOutcome> {
let started_at = std::time::Instant::now();
let validation_result = bundle.validate(); let validation_result = bundle.validate();
if let std::result::Result::Err(error) = validation_result { if let std::result::Result::Err(error) = validation_result {
return std::result::Result::Err(error); return std::result::Result::Err(error);
@@ -109,6 +124,14 @@ pub(crate) async fn persist_core_extraction(
}, },
}; };
if existing.is_some() { if existing.is_some() {
let invalidation_result = crate::invalidate_core_descendants_in_transaction(
&mut transaction,
bundle.transaction.signature.as_str(),
)
.await;
if let std::result::Result::Err(error) = invalidation_result {
return std::result::Result::Err(error);
}
let delete_result = sqlx::query("DELETE FROM k_sol_core_transactions WHERE signature = $1") let delete_result = sqlx::query("DELETE FROM k_sol_core_transactions WHERE signature = $1")
.bind(bundle.transaction.signature.as_str()) .bind(bundle.transaction.signature.as_str())
.execute(&mut *transaction) .execute(&mut *transaction)
@@ -202,17 +225,18 @@ pub(crate) async fn persist_core_extraction(
std::result::Result::Ok(value) => value, std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error), std::result::Result::Err(error) => return std::result::Result::Err(error),
}; };
return std::result::Result::Ok(crate::InsertOutcome::new( let outcome =
inserted_count, crate::InsertOutcome::new(inserted_count, if existing.is_some() { 1 } else { 0 }, 0);
if existing.is_some() { 1 } else { 0 }, let elapsed_ms = started_at.elapsed().as_secs_f64() * 1000.0;
0, tracing::trace!(target: crate::TRACING_TARGET, backend = "postgres", domain = "ks-store.pg", action = "persist_core_extraction", elapsed_ms, rows = inserted_count, replaced = existing.is_some(), outcome = "success", "PostgreSQL store operation completed");
)); return std::result::Result::Ok(outcome);
} }
pub(crate) async fn mark_core_extraction_failed( pub(crate) async fn mark_core_extraction_failed(
pool: &sqlx::PgPool, pool: &sqlx::PgPool,
failure: &crate::CoreExtractionFailure, failure: &crate::CoreExtractionFailure,
) -> ks_core::Result<crate::InsertOutcome> { ) -> ks_core::Result<crate::InsertOutcome> {
let started_at = std::time::Instant::now();
let transaction_result = pool.begin().await; let transaction_result = pool.begin().await;
let mut transaction = match transaction_result { let mut transaction = match transaction_result {
std::result::Result::Ok(value) => value, std::result::Result::Ok(value) => value,
@@ -251,6 +275,7 @@ pub(crate) async fn mark_core_extraction_failed(
"postgres core extraction failure commit failed: {error}" "postgres core extraction failure commit failed: {error}"
))); )));
} }
trace_query_success("mark_core_extraction_failed", started_at, 1);
return std::result::Result::Ok(crate::InsertOutcome::new(0, 1, 0)); return std::result::Result::Ok(crate::InsertOutcome::new(0, 1, 0));
} }
@@ -803,6 +828,16 @@ fn balance_change_kind_to_sql(value: crate::CoreBalanceChangeKind) -> &'static s
}; };
} }
fn trace_query_success(action: &str, started_at: std::time::Instant, rows: usize) {
let elapsed_ms = started_at.elapsed().as_secs_f64() * 1000.0;
tracing::trace!(target: crate::TRACING_TARGET, backend = "postgres", domain = "ks-store.pg", action = action, elapsed_ms, rows, outcome = "success", "PostgreSQL store operation completed");
}
fn trace_query_failure(action: &str, started_at: std::time::Instant, error: &sqlx::Error) {
let elapsed_ms = started_at.elapsed().as_secs_f64() * 1000.0;
tracing::trace!(target: crate::TRACING_TARGET, backend = "postgres", domain = "ks-store.pg", action = action, elapsed_ms, error = %error, outcome = "error", "PostgreSQL store operation failed");
}
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
#[test] #[test]

View File

@@ -1,12 +1,22 @@
// file: ks-store/src/postgres/query/core_queries.rs // file: ks-store/src/postgres/query/core_queries.rs
// version: 9 // version: 11
//! PostgreSQL queries for normalized Solana core storage. //! PostgreSQL queries for normalized Solana core storage.
use sqlx::Row; // rust-rules: trait-import use sqlx::Row; // rust-rules: trait-import
const TOP_LEVEL_INSTRUCTIONS_CONTEXT_SQL: &str = "SELECT COALESCE(jsonb_agg(jsonb_build_object('instructionIndex', instruction_path::BIGINT, 'instructionPath', instruction_path, 'programId', program_id, 'stackHeight', stack_height, 'payloadJson', payload_json, 'payloadHash', payload_json_hash) ORDER BY instruction_path::BIGINT, instruction_path ASC), '[]'::jsonb) FROM k_sol_core_instructions WHERE signature = $1 AND instruction_path ~ '^[0-9]+$'"; const TOP_LEVEL_INSTRUCTIONS_CONTEXT_SQL: &str = "SELECT COALESCE(jsonb_agg(jsonb_build_object('instructionIndex', instruction_path::BIGINT, 'instructionPath', instruction_path, 'programId', program_id, 'stackHeight', stack_height, 'payloadJson', payload_json, 'payloadHash', payload_json_hash) ORDER BY instruction_path::BIGINT, instruction_path ASC), '[]'::jsonb) FROM k_sol_core_instructions WHERE signature = $1 AND instruction_path ~ '^[0-9]+$'";
const INCOMPLETE_DECODE_INPUTS_SQL: &str = "WITH incomplete AS (SELECT signature, MIN(slot) AS first_slot FROM k_sol_core_instructions WHERE (cardinality($1::text[]) = 0 OR signature = ANY($1)) AND ($3::bigint IS NULL OR slot >= $3) AND ($4::bigint IS NULL OR slot <= $4) AND (cardinality($5::text[]) = 0 OR program_id = ANY($5)) AND (processing_state = ANY($2) OR (processing_state = 'ignored' AND lifecycle_reason = 'unsupported')) GROUP BY signature ORDER BY MIN(slot), signature LIMIT $7) SELECT i.id, i.transaction_id, i.signature, i.slot, i.instruction_path, i.program_id, i.stack_height, i.accounts_json, i.payload_json, i.payload_json_hash, i.processing_state, i.created_at, i.updated_at FROM k_sol_core_instructions i JOIN incomplete s ON s.signature = i.signature WHERE (cardinality($5::text[]) = 0 OR i.program_id = ANY($5)) AND (cardinality($6::text[]) = 0 OR i.instruction_path = ANY($6)) ORDER BY i.slot ASC, i.signature ASC, i.instruction_path ASC"; const CORE_INSTRUCTION_REPLAY_SQL: &str = "WITH instruction_rows AS (SELECT id, transaction_id, signature, slot, 'top_level'::TEXT AS scope_code, 0::INTEGER AS scope_rank, NULL::TEXT AS parent_instruction_path, instruction_path, program_id, stack_height, accounts_json, payload_json, payload_json_hash, processing_state, lifecycle_reason, created_at, updated_at FROM k_sol_core_instructions UNION ALL SELECT id, transaction_id, signature, slot, 'inner'::TEXT AS scope_code, 1::INTEGER AS scope_rank, parent_instruction_path, instruction_path, program_id, stack_height, accounts_json, payload_json, payload_json_hash, processing_state, lifecycle_reason, created_at, updated_at FROM k_sol_core_inner_instructions) SELECT id, transaction_id, signature, slot, scope_code, scope_rank, parent_instruction_path, instruction_path, program_id, stack_height, accounts_json, payload_json, payload_json_hash, processing_state, created_at, updated_at FROM instruction_rows WHERE ($1::TEXT IS NULL OR processing_state = $1) AND ($2::TEXT IS NULL OR program_id = $2) AND ($3::TEXT IS NULL OR scope_code = $3) AND ($4::BIGINT IS NULL OR slot >= $4) AND ($5::BIGINT IS NULL OR slot <= $5) AND ($6::BIGINT IS NULL OR (slot, signature, scope_rank, instruction_path) > ($6, $7, $8, $9)) ORDER BY slot ASC, signature ASC, scope_rank ASC, instruction_path ASC LIMIT $10";
const DECODE_INPUTS_SQL: &str = "WITH instruction_rows AS (SELECT id, transaction_id, signature, slot, 'top_level'::TEXT AS scope_code, 0::INTEGER AS scope_rank, NULL::TEXT AS parent_instruction_path, instruction_path, program_id, stack_height, accounts_json, payload_json, payload_json_hash, processing_state, lifecycle_reason, created_at, updated_at FROM k_sol_core_instructions UNION ALL SELECT id, transaction_id, signature, slot, 'inner'::TEXT AS scope_code, 1::INTEGER AS scope_rank, parent_instruction_path, instruction_path, program_id, stack_height, accounts_json, payload_json, payload_json_hash, processing_state, lifecycle_reason, created_at, updated_at FROM k_sol_core_inner_instructions) SELECT id, transaction_id, signature, slot, scope_code, scope_rank, parent_instruction_path, instruction_path, program_id, stack_height, accounts_json, payload_json, payload_json_hash, processing_state, created_at, updated_at FROM instruction_rows WHERE (cardinality($1::TEXT[]) = 0 OR signature = ANY($1)) AND (cardinality($2::TEXT[]) = 0 OR processing_state = ANY($2)) AND ($3::BIGINT IS NULL OR slot >= $3) AND ($4::BIGINT IS NULL OR slot <= $4) AND (cardinality($5::TEXT[]) = 0 OR program_id = ANY($5)) AND (cardinality($6::TEXT[]) = 0 OR instruction_path = ANY($6)) ORDER BY slot ASC, signature ASC, scope_rank ASC, instruction_path ASC LIMIT $7";
const INCOMPLETE_DECODE_INPUTS_SQL: &str = "WITH instruction_rows AS (SELECT id, transaction_id, signature, slot, 'top_level'::TEXT AS scope_code, 0::INTEGER AS scope_rank, NULL::TEXT AS parent_instruction_path, instruction_path, program_id, stack_height, accounts_json, payload_json, payload_json_hash, processing_state, lifecycle_reason, created_at, updated_at FROM k_sol_core_instructions UNION ALL SELECT id, transaction_id, signature, slot, 'inner'::TEXT AS scope_code, 1::INTEGER AS scope_rank, parent_instruction_path, instruction_path, program_id, stack_height, accounts_json, payload_json, payload_json_hash, processing_state, lifecycle_reason, created_at, updated_at FROM k_sol_core_inner_instructions), incomplete AS (SELECT signature, MIN(slot) AS first_slot FROM instruction_rows WHERE (cardinality($1::TEXT[]) = 0 OR signature = ANY($1)) AND ($3::BIGINT IS NULL OR slot >= $3) AND ($4::BIGINT IS NULL OR slot <= $4) AND (cardinality($5::TEXT[]) = 0 OR program_id = ANY($5)) AND (processing_state = ANY($2) OR (processing_state = 'ignored' AND lifecycle_reason = 'unsupported')) GROUP BY signature ORDER BY MIN(slot), signature LIMIT $7) SELECT i.id, i.transaction_id, i.signature, i.slot, i.scope_code, i.scope_rank, i.parent_instruction_path, i.instruction_path, i.program_id, i.stack_height, i.accounts_json, i.payload_json, i.payload_json_hash, i.processing_state, i.created_at, i.updated_at FROM instruction_rows i JOIN incomplete s ON s.signature = i.signature WHERE (cardinality($5::TEXT[]) = 0 OR i.program_id = ANY($5)) AND (cardinality($6::TEXT[]) = 0 OR i.instruction_path = ANY($6)) ORDER BY i.slot ASC, i.signature ASC, i.scope_rank ASC, i.instruction_path ASC";
#[derive(Clone, Debug, serde::Deserialize, serde::Serialize)]
struct CoreInstructionPageCursor {
slot: i64,
signature: std::string::String,
scope_rank: i32,
instruction_path: std::string::String,
}
pub(crate) async fn insert_core_transaction( pub(crate) async fn insert_core_transaction(
pool: &sqlx::PgPool, pool: &sqlx::PgPool,
@@ -279,7 +289,7 @@ pub(crate) async fn list_core_instructions_for_replay(
pool: &sqlx::PgPool, pool: &sqlx::PgPool,
filter: &crate::CoreInstructionReplayFilter, filter: &crate::CoreInstructionReplayFilter,
page_request: &crate::PageRequest, page_request: &crate::PageRequest,
) -> ks_core::Result<std::vec::Vec<crate::CoreInstructionRow>> { ) -> ks_core::Result<crate::PageSlice<crate::CoreInstructionReplayRow>> {
let min_slot_result = optional_sql_slot_from_u64(filter.min_slot); let min_slot_result = optional_sql_slot_from_u64(filter.min_slot);
let min_slot = match min_slot_result { let min_slot = match min_slot_result {
std::result::Result::Ok(value) => value, std::result::Result::Ok(value) => value,
@@ -290,63 +300,84 @@ pub(crate) async fn list_core_instructions_for_replay(
std::result::Result::Ok(value) => value, std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error), std::result::Result::Err(error) => return std::result::Result::Err(error),
}; };
let offset_result = let cursor_result = decode_core_instruction_cursor(page_request.cursor.as_deref());
sql_i64_from_u64(page_request.offset, "page offset does not fit into SQL BIGINT"); let cursor = match cursor_result {
let offset = match offset_result {
std::result::Result::Ok(value) => value, std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error), std::result::Result::Err(error) => return std::result::Result::Err(error),
}; };
let processing_state = filter.processing_state.map(instruction_processing_state_to_sql); let processing_state = filter.processing_state.map(instruction_processing_state_to_sql);
let query_result = sqlx::query( let scope_code = filter.scope.map(crate::CoreInstructionScope::as_code);
"SELECT id, transaction_id, signature, slot, instruction_path, program_id, stack_height, accounts_json, payload_json, payload_json_hash, processing_state, created_at, updated_at FROM k_sol_core_instructions WHERE ($1::TEXT IS NULL OR processing_state = $1) AND ($2::TEXT IS NULL OR program_id = $2) AND ($3::BIGINT IS NULL OR slot >= $3) AND ($4::BIGINT IS NULL OR slot <= $4) ORDER BY slot ASC, id ASC LIMIT $5 OFFSET $6", let query_limit = i64::from(page_request.limit) + 1;
) let started_at = std::time::Instant::now();
let query_result = sqlx::query(CORE_INSTRUCTION_REPLAY_SQL)
.bind(processing_state) .bind(processing_state)
.bind(filter.program_id.as_deref()) .bind(filter.program_id.as_deref())
.bind(scope_code)
.bind(min_slot) .bind(min_slot)
.bind(max_slot) .bind(max_slot)
.bind(i64::from(page_request.limit)) .bind(cursor.as_ref().map(|value| return value.slot))
.bind(offset) .bind(cursor.as_ref().map(|value| return value.signature.as_str()))
.bind(cursor.as_ref().map(|value| return value.scope_rank))
.bind(cursor.as_ref().map(|value| return value.instruction_path.as_str()))
.bind(query_limit)
.fetch_all(pool) .fetch_all(pool)
.await; .await;
let rows = match query_result { let rows = match query_result {
std::result::Result::Ok(value) => value, std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => { std::result::Result::Err(error) => {
trace_query_failure("list_core_instruction_page", started_at, &error);
return std::result::Result::Err(ks_core::Error::db(format!( return std::result::Result::Err(ks_core::Error::db(format!(
"postgres core instruction replay query failed: {error}" "postgres core instruction replay query failed: {error}"
))); )));
}, },
}; };
let mut output = std::vec::Vec::with_capacity(rows.len()); let mut output = std::vec::Vec::with_capacity(rows.len().min(usize::from(page_request.limit)));
for row in rows { for row in rows.iter().take(usize::from(page_request.limit)) {
let mapped_result = row_to_core_instruction_row(&row); let mapped_result = row_to_core_instruction_replay_row(row);
match mapped_result { match mapped_result {
std::result::Result::Ok(value) => output.push(value), std::result::Result::Ok(value) => output.push(value),
std::result::Result::Err(error) => return std::result::Result::Err(error), std::result::Result::Err(error) => return std::result::Result::Err(error),
} }
} }
return std::result::Result::Ok(output); let next_cursor = if rows.len() > usize::from(page_request.limit) {
let last = output.last();
match last {
std::option::Option::Some(value) => {
let cursor_result = encode_core_instruction_cursor(value);
match cursor_result {
std::result::Result::Ok(encoded) => std::option::Option::Some(encoded),
std::result::Result::Err(error) => return std::result::Result::Err(error),
}
},
std::option::Option::None => std::option::Option::None,
}
} else {
std::option::Option::None
};
trace_query_success("list_core_instruction_page", started_at, output.len());
return std::result::Result::Ok(crate::PageSlice::new(output, next_cursor));
} }
pub(crate) async fn list_core_instruction_replay_inputs( pub(crate) async fn list_core_instruction_replay_inputs(
pool: &sqlx::PgPool, pool: &sqlx::PgPool,
filter: &crate::CoreInstructionReplayFilter, filter: &crate::CoreInstructionReplayFilter,
page_request: &crate::PageRequest, page_request: &crate::PageRequest,
) -> ks_core::Result<std::vec::Vec<crate::MdCoreInstructionReplayInput>> { ) -> ks_core::Result<crate::PageSlice<crate::MdCoreInstructionReplayInput>> {
let instructions_result = let instructions_result =
crate::list_core_instructions_for_replay(pool, filter, page_request).await; crate::list_core_instructions_for_replay(pool, filter, page_request).await;
let instructions = match instructions_result { let instructions = match instructions_result {
std::result::Result::Ok(value) => value, std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error), std::result::Result::Err(error) => return std::result::Result::Err(error),
}; };
let mut output = std::vec::Vec::with_capacity(instructions.len()); let mut output = std::vec::Vec::with_capacity(instructions.rows.len());
for instruction in instructions { for instruction in &instructions.rows {
let input_result = load_replay_input_for_instruction(pool, &instruction).await; let input_result = load_replay_input_for_instruction(pool, instruction).await;
match input_result { match input_result {
std::result::Result::Ok(value) => output.push(value), std::result::Result::Ok(value) => output.push(value),
std::result::Result::Err(error) => return std::result::Result::Err(error), std::result::Result::Err(error) => return std::result::Result::Err(error),
} }
} }
return std::result::Result::Ok(output); return std::result::Result::Ok(crate::PageSlice::new(output, instructions.next_cursor));
} }
pub(crate) async fn list_decode_replay_inputs( pub(crate) async fn list_decode_replay_inputs(
@@ -367,10 +398,9 @@ pub(crate) async fn list_decode_replay_inputs(
let processing_states = filter let processing_states = filter
.processing_states .processing_states
.iter() .iter()
.map(|state| { .map(|state| return instruction_processing_state_to_sql(*state).to_string())
return instruction_processing_state_to_sql(*state).to_string();
})
.collect::<std::vec::Vec<_>>(); .collect::<std::vec::Vec<_>>();
let started_at = std::time::Instant::now();
let rows_result = if filter.incomplete_signatures { let rows_result = if filter.incomplete_signatures {
sqlx::query(INCOMPLETE_DECODE_INPUTS_SQL) sqlx::query(INCOMPLETE_DECODE_INPUTS_SQL)
.bind(&filter.signatures) .bind(&filter.signatures)
@@ -383,9 +413,7 @@ pub(crate) async fn list_decode_replay_inputs(
.fetch_all(pool) .fetch_all(pool)
.await .await
} else { } else {
sqlx::query( sqlx::query(DECODE_INPUTS_SQL)
"SELECT id, transaction_id, signature, slot, instruction_path, program_id, stack_height, accounts_json, payload_json, payload_json_hash, processing_state, created_at, updated_at FROM k_sol_core_instructions WHERE (cardinality($1::text[]) = 0 OR signature = ANY($1)) AND (cardinality($2::text[]) = 0 OR processing_state = ANY($2)) AND ($3::bigint IS NULL OR slot >= $3) AND ($4::bigint IS NULL OR slot <= $4) AND (cardinality($5::text[]) = 0 OR program_id = ANY($5)) AND (cardinality($6::text[]) = 0 OR instruction_path = ANY($6)) ORDER BY slot ASC, signature ASC, instruction_path ASC LIMIT $7",
)
.bind(&filter.signatures) .bind(&filter.signatures)
.bind(&processing_states) .bind(&processing_states)
.bind(min_slot) .bind(min_slot)
@@ -399,6 +427,7 @@ pub(crate) async fn list_decode_replay_inputs(
let rows = match rows_result { let rows = match rows_result {
std::result::Result::Ok(value) => value, std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => { std::result::Result::Err(error) => {
trace_query_failure("list_decode_replay_inputs", started_at, &error);
return std::result::Result::Err(ks_core::Error::db(format!( return std::result::Result::Err(ks_core::Error::db(format!(
"postgres decode replay input selection failed: {error}" "postgres decode replay input selection failed: {error}"
))); )));
@@ -406,7 +435,7 @@ pub(crate) async fn list_decode_replay_inputs(
}; };
let mut output = std::vec::Vec::with_capacity(rows.len()); let mut output = std::vec::Vec::with_capacity(rows.len());
for row in rows { for row in rows {
let instruction_result = row_to_core_instruction_row(&row); let instruction_result = row_to_core_instruction_replay_row(&row);
let instruction = match instruction_result { let instruction = match instruction_result {
std::result::Result::Ok(value) => value, std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error), std::result::Result::Err(error) => return std::result::Result::Err(error),
@@ -417,6 +446,7 @@ pub(crate) async fn list_decode_replay_inputs(
std::result::Result::Err(error) => return std::result::Result::Err(error), std::result::Result::Err(error) => return std::result::Result::Err(error),
} }
} }
trace_query_success("list_decode_replay_inputs", started_at, output.len());
return std::result::Result::Ok(output); return std::result::Result::Ok(output);
} }
@@ -425,9 +455,22 @@ pub(crate) async fn update_core_instruction_lifecycle(
mark: &crate::CoreInstructionLifecycleMark, mark: &crate::CoreInstructionLifecycleMark,
) -> ks_core::Result<crate::InsertOutcome> { ) -> ks_core::Result<crate::InsertOutcome> {
let processing_state = instruction_processing_state_to_sql(mark.processing_state); let processing_state = instruction_processing_state_to_sql(mark.processing_state);
let query_result = sqlx::query( let scope_result =
"UPDATE k_sol_core_instructions SET processing_state = $1, processor_name = $2, processor_version = $3, lifecycle_reason = $4, updated_at = NOW() WHERE signature = $5 AND instruction_path = $6", crate::CoreInstructionScope::from_instruction_path(mark.instruction_path.as_str());
) let scope = match scope_result {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let sql = match scope {
crate::CoreInstructionScope::TopLevel => {
"UPDATE k_sol_core_instructions SET processing_state = $1, processor_name = $2, processor_version = $3, lifecycle_reason = $4, updated_at = NOW() WHERE signature = $5 AND instruction_path = $6"
},
crate::CoreInstructionScope::Inner => {
"UPDATE k_sol_core_inner_instructions SET processing_state = $1, processor_name = $2, processor_version = $3, lifecycle_reason = $4, updated_at = NOW() WHERE signature = $5 AND instruction_path = $6"
},
};
let started_at = std::time::Instant::now();
let query_result = sqlx::query(sql)
.bind(processing_state) .bind(processing_state)
.bind(mark.processor_name.as_deref()) .bind(mark.processor_name.as_deref())
.bind(mark.processor_version.as_deref()) .bind(mark.processor_version.as_deref())
@@ -436,10 +479,24 @@ pub(crate) async fn update_core_instruction_lifecycle(
.bind(mark.instruction_path.as_str()) .bind(mark.instruction_path.as_str())
.execute(pool) .execute(pool)
.await; .await;
return outcome_from_update_result( let outcome_result = outcome_from_update_result(
query_result, query_result,
"postgres core instruction lifecycle update failed", "postgres core instruction lifecycle update failed",
); );
return match outcome_result {
std::result::Result::Ok(outcome) => {
trace_query_success(
"update_core_instruction_lifecycle",
started_at,
match usize::try_from(outcome.updated_count) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => usize::MAX,
},
);
std::result::Result::Ok(outcome)
},
std::result::Result::Err(error) => std::result::Result::Err(error),
};
} }
fn instruction_processing_state_to_sql( fn instruction_processing_state_to_sql(
@@ -608,21 +665,24 @@ fn outcome_from_update_result(
}; };
} }
fn row_to_core_instruction_row( fn row_to_core_instruction_replay_row(
row: &sqlx::postgres::PgRow, row: &sqlx::postgres::PgRow,
) -> ks_core::Result<crate::CoreInstructionRow> { ) -> ks_core::Result<crate::CoreInstructionReplayRow> {
let state_text_result: std::result::Result<std::string::String, sqlx::Error> = let state_text: std::string::String = match read_row_value(row, "processing_state") {
row.try_get("processing_state");
let state_text = match state_text_result {
std::result::Result::Ok(value) => value, std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => { std::result::Result::Err(error) => return std::result::Result::Err(error),
return std::result::Result::Err(ks_core::Error::db(format!(
"postgres core instruction processing state read failed: {error}"
)));
},
}; };
let state_result = instruction_processing_state_from_sql(state_text.as_str()); let processing_state_result = instruction_processing_state_from_sql(state_text.as_str());
let processing_state = match state_result { let processing_state = match processing_state_result {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let scope_text: std::string::String = match read_row_value(row, "scope_code") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let scope_result = crate::CoreInstructionScope::from_code(scope_text.as_str());
let scope = match scope_result {
std::result::Result::Ok(value) => value, std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error), std::result::Result::Err(error) => return std::result::Result::Err(error),
}; };
@@ -642,6 +702,10 @@ fn row_to_core_instruction_row(
std::result::Result::Ok(value) => value, std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error), std::result::Result::Err(error) => return std::result::Result::Err(error),
}; };
let parent_instruction_path = match read_row_value(row, "parent_instruction_path") {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let instruction_path = match read_row_value(row, "instruction_path") { let instruction_path = match read_row_value(row, "instruction_path") {
std::result::Result::Ok(value) => value, std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error), std::result::Result::Err(error) => return std::result::Result::Err(error),
@@ -674,11 +738,13 @@ fn row_to_core_instruction_row(
std::result::Result::Ok(value) => value, std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error), std::result::Result::Err(error) => return std::result::Result::Err(error),
}; };
return std::result::Result::Ok(crate::CoreInstructionRow { return std::result::Result::Ok(crate::CoreInstructionReplayRow {
id, id,
transaction_id, transaction_id,
signature, signature,
slot, slot,
scope,
parent_instruction_path,
instruction_path, instruction_path,
program_id, program_id,
stack_height, stack_height,
@@ -706,7 +772,7 @@ where
async fn load_replay_input_for_instruction( async fn load_replay_input_for_instruction(
pool: &sqlx::PgPool, pool: &sqlx::PgPool,
instruction: &crate::CoreInstructionRow, instruction: &crate::CoreInstructionReplayRow,
) -> ks_core::Result<crate::MdCoreInstructionReplayInput> { ) -> ks_core::Result<crate::MdCoreInstructionReplayInput> {
let transaction_context_result = let transaction_context_result =
sqlx::query_as::<sqlx::Postgres, (bool, std::option::Option<serde_json::Value>, std::option::Option<i64>)>( sqlx::query_as::<sqlx::Postgres, (bool, std::option::Option<serde_json::Value>, std::option::Option<i64>)>(
@@ -798,7 +864,7 @@ async fn load_replay_input_for_instruction(
}; };
return input.with_core_context( return input.with_core_context(
transaction_context.2, transaction_context.2,
std::option::Option::None, instruction.parent_instruction_path.clone(),
stack_height, stack_height,
return_data, return_data,
); );
@@ -922,8 +988,120 @@ async fn load_json_aggregate(
}; };
} }
fn decode_core_instruction_cursor(
cursor: std::option::Option<&str>,
) -> ks_core::Result<std::option::Option<CoreInstructionPageCursor>> {
let raw = match cursor {
std::option::Option::Some(value) => value,
std::option::Option::None => return std::result::Result::Ok(std::option::Option::None),
};
let decode_result = serde_json::from_str::<CoreInstructionPageCursor>(raw);
let decoded = match decode_result {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_error) => {
return std::result::Result::Err(ks_core::Error::db(
"core instruction page cursor is invalid",
));
},
};
if decoded.slot < 0
|| decoded.signature.trim().is_empty()
|| decoded.instruction_path.trim().is_empty()
|| !matches!(decoded.scope_rank, 0 | 1)
{
return std::result::Result::Err(ks_core::Error::db(
"core instruction page cursor values are invalid",
));
}
return std::result::Result::Ok(std::option::Option::Some(decoded));
}
fn encode_core_instruction_cursor(
row: &crate::CoreInstructionReplayRow,
) -> ks_core::Result<std::string::String> {
let scope_rank = match row.scope {
crate::CoreInstructionScope::TopLevel => 0,
crate::CoreInstructionScope::Inner => 1,
};
let cursor = CoreInstructionPageCursor {
slot: row.slot,
signature: row.signature.clone(),
scope_rank,
instruction_path: row.instruction_path.clone(),
};
let encode_result = serde_json::to_string(&cursor);
return match encode_result {
std::result::Result::Ok(value) if value.len() <= crate::MAX_PAGE_CURSOR_LENGTH => {
std::result::Result::Ok(value)
},
std::result::Result::Ok(_value) => std::result::Result::Err(ks_core::Error::db(
"core instruction page cursor exceeds maximum encoded length",
)),
std::result::Result::Err(error) => std::result::Result::Err(ks_core::Error::db(format!(
"core instruction page cursor serialization failed: {error}"
))),
};
}
fn trace_query_success(action: &str, started_at: std::time::Instant, rows: usize) {
let elapsed_ms = started_at.elapsed().as_secs_f64() * 1000.0;
tracing::trace!(target: crate::TRACING_TARGET, backend = "postgres", domain = "ks-store.pg", action = action, elapsed_ms, rows, outcome = "success", "PostgreSQL store operation completed");
}
fn trace_query_failure(action: &str, started_at: std::time::Instant, error: &sqlx::Error) {
let elapsed_ms = started_at.elapsed().as_secs_f64() * 1000.0;
tracing::trace!(target: crate::TRACING_TARGET, backend = "postgres", domain = "ks-store.pg", action = action, elapsed_ms, error = %error, outcome = "error", "PostgreSQL store operation failed");
}
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
#[test]
fn core_instruction_replay_query_is_union_based_and_offset_free() {
assert!(super::CORE_INSTRUCTION_REPLAY_SQL.contains("k_sol_core_instructions"));
assert!(super::CORE_INSTRUCTION_REPLAY_SQL.contains("k_sol_core_inner_instructions"));
assert!(super::CORE_INSTRUCTION_REPLAY_SQL.contains("scope_code"));
assert!(!super::CORE_INSTRUCTION_REPLAY_SQL.to_ascii_uppercase().contains(" OFFSET "));
}
#[test]
fn core_instruction_cursor_round_trips_top_level_and_inner_order_fields() {
let row = crate::CoreInstructionReplayRow {
id: 1,
transaction_id: 2,
signature: "signature".to_string(),
slot: 3,
scope: crate::CoreInstructionScope::Inner,
parent_instruction_path: std::option::Option::Some("0".to_string()),
instruction_path: "0/1".to_string(),
program_id: "program".to_string(),
stack_height: std::option::Option::Some(2),
accounts_json: serde_json::json!([]),
payload_json: std::option::Option::None,
payload_json_hash: std::option::Option::None,
processing_state: crate::CoreInstructionProcessingState::Pending,
created_at: chrono::Utc::now(),
updated_at: chrono::Utc::now(),
};
let encoded_result = super::encode_core_instruction_cursor(&row);
let encoded = match encoded_result {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => panic!("unexpected cursor encode error: {error}"),
};
let decoded_result =
super::decode_core_instruction_cursor(std::option::Option::Some(encoded.as_str()));
let decoded = match decoded_result {
std::result::Result::Ok(std::option::Option::Some(value)) => value,
std::result::Result::Ok(std::option::Option::None) => {
panic!("cursor unexpectedly absent")
},
std::result::Result::Err(error) => panic!("unexpected cursor decode error: {error}"),
};
assert_eq!(decoded.slot, 3);
assert_eq!(decoded.signature, "signature");
assert_eq!(decoded.scope_rank, 1);
assert_eq!(decoded.instruction_path, "0/1");
}
#[test] #[test]
fn incomplete_decode_input_query_projects_every_core_instruction_row_column() { fn incomplete_decode_input_query_projects_every_core_instruction_row_column() {
for column in [ for column in [
@@ -931,6 +1109,8 @@ mod tests {
"i.transaction_id", "i.transaction_id",
"i.signature", "i.signature",
"i.slot", "i.slot",
"i.scope_code",
"i.parent_instruction_path",
"i.instruction_path", "i.instruction_path",
"i.program_id", "i.program_id",
"i.stack_height", "i.stack_height",
@@ -1061,7 +1241,7 @@ mod tests {
panic!("unexpected replay input query error: {error}") panic!("unexpected replay input query error: {error}")
}, },
}; };
let target = replay_inputs.iter().find(|input| { let target = replay_inputs.rows.iter().find(|input| {
return input.signature == signature && input.instruction_path == "2"; return input.signature == signature && input.instruction_path == "2";
}); });
let target = match target { let target = match target {

View File

@@ -1,5 +1,5 @@
// file: ks-store/src/postgres/query/decode_pipeline_queries.rs // file: ks-store/src/postgres/query/decode_pipeline_queries.rs
// version: 11 // version: 12
//! PostgreSQL queries for contextual decode, coverage and materialization persistence. //! PostgreSQL queries for contextual decode, coverage and materialization persistence.
@@ -26,36 +26,20 @@ pub(crate) async fn list_decode_inputs(
pool: &sqlx::PgPool, pool: &sqlx::PgPool,
filter: &crate::DecodeSelectionFilter, filter: &crate::DecodeSelectionFilter,
) -> ks_core::Result<std::vec::Vec<crate::MdCoreInstructionReplayInput>> { ) -> ks_core::Result<std::vec::Vec<crate::MdCoreInstructionReplayInput>> {
tracing::debug!(target: crate::TRACING_TARGET, backend = "postgres", domain = "ks-store.pg", action = "list_decode_inputs", signature_count = filter.signatures.len(), signature_sample = ?filter.signatures.iter().take(5).map(std::string::String::as_str).collect::<std::vec::Vec<_>>(), processing_states = ?filter.processing_states, min_slot = ?filter.min_slot, max_slot = ?filter.max_slot, program_ids = ?filter.program_ids, instruction_paths = ?filter.instruction_paths, limit = filter.limit, "query PostgreSQL contextual decode inputs"); return crate::list_decode_replay_inputs(pool, filter).await;
let result = crate::list_decode_replay_inputs(pool, filter).await;
return match result {
std::result::Result::Ok(inputs) => {
let selected_input_keys = inputs
.iter()
.take(10)
.map(|input| return input.replay_input_key.as_str())
.collect::<std::vec::Vec<_>>();
tracing::debug!(target: crate::TRACING_TARGET, backend = "postgres", domain = "ks-store.pg", action = "list_decode_inputs", selected_count = inputs.len(), input_key_sample = ?selected_input_keys, "PostgreSQL contextual decode inputs selected");
std::result::Result::Ok(inputs)
},
std::result::Result::Err(error) => {
tracing::error!(target: crate::TRACING_TARGET, backend = "postgres", domain = "ks-store.pg", action = "list_decode_inputs", error = %error, "PostgreSQL contextual decode input query failed");
std::result::Result::Err(error)
},
};
} }
pub(crate) async fn list_materialized_outputs( pub(crate) async fn list_materialized_outputs(
pool: &sqlx::PgPool, pool: &sqlx::PgPool,
filter: &crate::MaterializedOutputFilter, filter: &crate::MaterializedOutputFilter,
) -> ks_core::Result<std::vec::Vec<crate::MaterializedOutputQueryRow>> { ) -> ks_core::Result<std::vec::Vec<crate::MaterializedOutputQueryRow>> {
tracing::debug!(target: crate::TRACING_TARGET, backend = "postgres", domain = "ks-store.pg", action = "list_materialized_outputs", processor_name = ?filter.processor_name, materialized_family = ?filter.materialized_family, signature_contains = ?filter.signature_contains, limit = filter.limit, "query bounded PostgreSQL materialized outputs");
if filter.limit == 0 || filter.limit > crate::MAX_MATERIALIZED_OUTPUT_QUERY_ROWS { if filter.limit == 0 || filter.limit > crate::MAX_MATERIALIZED_OUTPUT_QUERY_ROWS {
return std::result::Result::Err(ks_core::Error::db(format!( return std::result::Result::Err(ks_core::Error::db(format!(
"materialized output query limit must be between 1 and {}", "materialized output query limit must be between 1 and {}",
crate::MAX_MATERIALIZED_OUTPUT_QUERY_ROWS crate::MAX_MATERIALIZED_OUTPUT_QUERY_ROWS
))); )));
} }
let started_at = std::time::Instant::now();
let query_result = sqlx::query_as::<sqlx::Postgres, MaterializedOutputDatabaseRow>( let query_result = sqlx::query_as::<sqlx::Postgres, MaterializedOutputDatabaseRow>(
"SELECT processor_name, processor_version, input_key, output_key, source_event_key, source_decoder_name, source_decoder_version, signature, slot, materialized_family, payload_jsonb AS payload_json, created_at::text AS created_at, updated_at::text AS updated_at FROM k_sol_mat_outputs WHERE ($1::text IS NULL OR processor_name = $1) AND ($2::text IS NULL OR materialized_family = $2) AND ($3::text IS NULL OR POSITION(LOWER($3) IN LOWER(signature)) > 0) ORDER BY slot DESC, id DESC LIMIT $4", "SELECT processor_name, processor_version, input_key, output_key, source_event_key, source_decoder_name, source_decoder_version, signature, slot, materialized_family, payload_jsonb AS payload_json, created_at::text AS created_at, updated_at::text AS updated_at FROM k_sol_mat_outputs WHERE ($1::text IS NULL OR processor_name = $1) AND ($2::text IS NULL OR materialized_family = $2) AND ($3::text IS NULL OR POSITION(LOWER($3) IN LOWER(signature)) > 0) ORDER BY slot DESC, id DESC LIMIT $4",
) )
@@ -68,6 +52,7 @@ pub(crate) async fn list_materialized_outputs(
let rows = match query_result { let rows = match query_result {
std::result::Result::Ok(value) => value, std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => { std::result::Result::Err(error) => {
trace_query_failure("list_materialized_outputs", started_at, &error);
return std::result::Result::Err(ks_core::Error::db(format!( return std::result::Result::Err(ks_core::Error::db(format!(
"postgres materialized output query failed: {error}" "postgres materialized output query failed: {error}"
))); )));
@@ -100,7 +85,7 @@ pub(crate) async fn list_materialized_outputs(
updated_at: row.updated_at, updated_at: row.updated_at,
}); });
} }
tracing::debug!(target: crate::TRACING_TARGET, backend = "postgres", domain = "ks-store.pg", action = "list_materialized_outputs", row_count = output.len(), "bounded PostgreSQL materialized outputs loaded"); trace_query_success("list_materialized_outputs", started_at, output.len());
return std::result::Result::Ok(output); return std::result::Result::Ok(output);
} }
@@ -108,7 +93,7 @@ pub(crate) async fn is_decode_current(
pool: &sqlx::PgPool, pool: &sqlx::PgPool,
identity: &crate::ProcessingLedgerIdentity, identity: &crate::ProcessingLedgerIdentity,
) -> ks_core::Result<bool> { ) -> ks_core::Result<bool> {
tracing::debug!(target: crate::TRACING_TARGET, backend = "postgres", domain = "ks-store.pg", action = "is_decode_current", stage = %identity.stage, processor_name = %identity.processor_name, processor_version = %identity.processor_version, input_key = %identity.input_key, input_hash = %identity.input_hash, "query PostgreSQL processing ledger current state"); let started_at = std::time::Instant::now();
let query_result = sqlx::query_scalar::<sqlx::Postgres, bool>( let query_result = sqlx::query_scalar::<sqlx::Postgres, bool>(
"SELECT EXISTS(SELECT 1 FROM k_sol_ops_processing_ledger WHERE stage = $1 AND processor_name = $2 AND processor_version = $3 AND input_key = $4 AND input_hash = $5 AND status = 'succeeded')", "SELECT EXISTS(SELECT 1 FROM k_sol_ops_processing_ledger WHERE stage = $1 AND processor_name = $2 AND processor_version = $3 AND input_key = $4 AND input_hash = $5 AND status = 'succeeded')",
) )
@@ -121,11 +106,15 @@ pub(crate) async fn is_decode_current(
.await; .await;
return match query_result { return match query_result {
std::result::Result::Ok(value) => { std::result::Result::Ok(value) => {
tracing::debug!(target: crate::TRACING_TARGET, backend = "postgres", domain = "ks-store.pg", action = "is_decode_current", stage = %identity.stage, processor_name = %identity.processor_name, processor_version = %identity.processor_version, input_key = %identity.input_key, input_hash = %identity.input_hash, current = value, "PostgreSQL processing ledger current state loaded"); trace_query_success(
"is_decode_current",
started_at,
if value { 1_usize } else { 0_usize },
);
std::result::Result::Ok(value) std::result::Result::Ok(value)
}, },
std::result::Result::Err(error) => { std::result::Result::Err(error) => {
tracing::error!(target: crate::TRACING_TARGET, backend = "postgres", domain = "ks-store.pg", action = "is_decode_current", stage = %identity.stage, processor_name = %identity.processor_name, processor_version = %identity.processor_version, input_key = %identity.input_key, error = %error, "PostgreSQL processing ledger current check failed"); trace_query_failure("is_decode_current", started_at, &error);
std::result::Result::Err(ks_core::Error::db(format!( std::result::Result::Err(ks_core::Error::db(format!(
"postgres decode ledger current check failed: {error}" "postgres decode ledger current check failed: {error}"
))) )))
@@ -138,16 +127,11 @@ pub(crate) async fn persist_decode_coverage_declarations(
declarations: &[crate::DecodeCoverageDeclarationInsert], declarations: &[crate::DecodeCoverageDeclarationInsert],
) -> ks_core::Result<crate::InsertOutcome> { ) -> ks_core::Result<crate::InsertOutcome> {
if declarations.is_empty() { if declarations.is_empty() {
tracing::debug!(target: crate::TRACING_TARGET, backend = "postgres", domain = "ks-store.pg", action = "persist_coverage_declarations", declaration_count = 0_usize, "skip empty PostgreSQL decode coverage declarations");
return std::result::Result::Ok(crate::InsertOutcome::new(0, 0, 0)); return std::result::Result::Ok(crate::InsertOutcome::new(0, 0, 0));
} }
let started_at = std::time::Instant::now();
let processor_name = declarations[0].processor_name.as_str(); let processor_name = declarations[0].processor_name.as_str();
let processor_version = declarations[0].processor_version.as_str(); let processor_version = declarations[0].processor_version.as_str();
let program_ids = declarations
.iter()
.map(|entry| return entry.program_id.as_str())
.collect::<std::vec::Vec<_>>();
tracing::debug!(target: crate::TRACING_TARGET, backend = "postgres", domain = "ks-store.pg", action = "persist_coverage_declarations", processor_name = %processor_name, processor_version = %processor_version, declaration_count = declarations.len(), program_ids = ?program_ids, "persist PostgreSQL decode coverage declarations");
if declarations.iter().any(|entry| { if declarations.iter().any(|entry| {
return entry.processor_name != processor_name return entry.processor_name != processor_name
|| entry.processor_version != processor_version || entry.processor_version != processor_version
@@ -306,16 +290,21 @@ pub(crate) async fn persist_decode_coverage_declarations(
))); )));
} }
let outcome = crate::InsertOutcome::new(inserted_count, updated_count, skipped_count); let outcome = crate::InsertOutcome::new(inserted_count, updated_count, skipped_count);
tracing::debug!(target: crate::TRACING_TARGET, backend = "postgres", domain = "ks-store.pg", action = "persist_coverage_declarations", processor_name = %processor_name, processor_version = %processor_version, outcome = ?outcome, "PostgreSQL decode coverage declarations persisted"); let row_count = inserted_count.saturating_add(updated_count).saturating_add(skipped_count);
let trace_rows = match usize::try_from(row_count) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_error) => usize::MAX,
};
trace_query_success("persist_coverage_declarations", started_at, trace_rows);
return std::result::Result::Ok(outcome); return std::result::Result::Ok(outcome);
} }
pub(crate) async fn persist_decode_result( pub(crate) async fn persist_decode_result(
pool: &sqlx::PgPool, pool: &sqlx::PgPool,
bundle: &crate::DecodePersistenceBundle, bundle: &crate::DecodePersistenceBundle,
force_replay: bool, _force_replay: bool,
) -> ks_core::Result<crate::InsertOutcome> { ) -> ks_core::Result<crate::InsertOutcome> {
tracing::debug!(target: crate::TRACING_TARGET, backend = "postgres", domain = "ks-store.pg", action = "persist_decode_result", signature = %bundle.signature, instruction_path = %bundle.instruction_path, stage = %bundle.ledger_identity.stage, processor_name = %bundle.ledger_identity.processor_name, processor_version = %bundle.ledger_identity.processor_version, input_key = %bundle.ledger_identity.input_key, input_hash = %bundle.ledger_identity.input_hash, status = %bundle.status, observation_count = bundle.observations.len(), coverage_program_id = %bundle.coverage.program_id, force_replay, "persist PostgreSQL contextual decode result"); let started_at = std::time::Instant::now();
let validation_result = bundle.validate(); let validation_result = bundle.validate();
if let std::result::Result::Err(error) = validation_result { if let std::result::Result::Err(error) = validation_result {
return std::result::Result::Err(error); return std::result::Result::Err(error);
@@ -329,6 +318,16 @@ pub(crate) async fn persist_decode_result(
))); )));
}, },
}; };
let invalidation_result = crate::invalidate_decode_descendants_in_transaction(
&mut transaction,
bundle.ledger_identity.processor_name.as_str(),
bundle.ledger_identity.processor_version.as_str(),
bundle.ledger_identity.input_key.as_str(),
)
.await;
if let std::result::Result::Err(error) = invalidation_result {
return std::result::Result::Err(error);
}
let delete_events_result = sqlx::query( let delete_events_result = sqlx::query(
"DELETE FROM k_sol_decode_events WHERE processor_name = $1 AND processor_version = $2 AND input_key = $3", "DELETE FROM k_sol_decode_events WHERE processor_name = $1 AND processor_version = $2 AND input_key = $3",
) )
@@ -438,27 +437,14 @@ pub(crate) async fn persist_decode_result(
if let std::result::Result::Err(error) = materialized_count_result { if let std::result::Result::Err(error) = materialized_count_result {
return std::result::Result::Err(error); return std::result::Result::Err(error);
} }
let lifecycle_state = match bundle.status.as_str() { let lifecycle_result = crate::recompute_instruction_lifecycle_in_transaction(
"decoded" => "decoded", &mut transaction,
"ignored" | "unsupported" => "ignored", bundle.signature.as_str(),
"failed" => "failed", bundle.instruction_path.as_str(),
_unknown => "decoded",
};
let lifecycle_result = sqlx::query(
"UPDATE k_sol_core_instructions SET processing_state = CASE WHEN $1 = 'decoded' AND processing_state <> 'materialized' THEN 'decoded' WHEN $1 IN ('ignored', 'failed') AND processing_state IN ('pending', 'failed', 'replay_requested', 'ignored') THEN $1 ELSE processing_state END, processor_name = CASE WHEN $1 = 'decoded' AND processing_state <> 'materialized' THEN $2 WHEN $1 IN ('ignored', 'failed') AND processing_state IN ('pending', 'failed', 'replay_requested', 'ignored') THEN $2 ELSE processor_name END, processor_version = CASE WHEN $1 = 'decoded' AND processing_state <> 'materialized' THEN $3 WHEN $1 IN ('ignored', 'failed') AND processing_state IN ('pending', 'failed', 'replay_requested', 'ignored') THEN $3 ELSE processor_version END, lifecycle_reason = CASE WHEN $1 = 'decoded' AND processing_state <> 'materialized' THEN $4 WHEN $1 IN ('ignored', 'failed') AND processing_state IN ('pending', 'failed', 'replay_requested', 'ignored') THEN $4 ELSE lifecycle_reason END, updated_at = NOW() WHERE signature = $5 AND instruction_path = $6",
) )
.bind(lifecycle_state)
.bind(bundle.ledger_identity.processor_name.as_str())
.bind(bundle.ledger_identity.processor_version.as_str())
.bind(bundle.status.as_str())
.bind(bundle.signature.as_str())
.bind(bundle.instruction_path.as_str())
.execute(&mut *transaction)
.await; .await;
if let std::result::Result::Err(error) = lifecycle_result { if let std::result::Result::Err(error) = lifecycle_result {
return std::result::Result::Err(ks_core::Error::db(format!( return std::result::Result::Err(error);
"postgres decode instruction lifecycle update failed: {error}"
)));
} }
let ledger_status = if bundle.status == "failed" { "failed" } else { "succeeded" }; let ledger_status = if bundle.status == "failed" { "failed" } else { "succeeded" };
let ledger_result = crate::postgres::query::decode_pipeline_queries::upsert_ledger_terminal( let ledger_result = crate::postgres::query::decode_pipeline_queries::upsert_ledger_terminal(
@@ -488,7 +474,8 @@ pub(crate) async fn persist_decode_result(
}, },
}; };
let outcome = crate::InsertOutcome::new(count, 1, 0); let outcome = crate::InsertOutcome::new(count, 1, 0);
tracing::debug!(target: crate::TRACING_TARGET, backend = "postgres", domain = "ks-store.pg", action = "persist_decode_result", signature = %bundle.signature, instruction_path = %bundle.instruction_path, processor_name = %bundle.ledger_identity.processor_name, processor_version = %bundle.ledger_identity.processor_version, status = %bundle.status, outcome = ?outcome, committed = true, "PostgreSQL contextual decode result persisted"); let elapsed_ms = started_at.elapsed().as_secs_f64() * 1000.0;
tracing::trace!(target: crate::TRACING_TARGET, backend = "postgres", domain = "ks-store.pg", action = "persist_decode_result", elapsed_ms, rows = count, outcome = "success", "PostgreSQL store operation completed");
return std::result::Result::Ok(outcome); return std::result::Result::Ok(outcome);
} }
@@ -496,7 +483,7 @@ pub(crate) async fn mark_decode_failed(
pool: &sqlx::PgPool, pool: &sqlx::PgPool,
failure: &crate::DecodeFailure, failure: &crate::DecodeFailure,
) -> ks_core::Result<crate::InsertOutcome> { ) -> ks_core::Result<crate::InsertOutcome> {
tracing::error!(target: crate::TRACING_TARGET, backend = "postgres", domain = "ks-store.pg", action = "mark_decode_failed", signature = %failure.signature, instruction_path = %failure.instruction_path, processor_name = %failure.ledger_identity.processor_name, processor_version = %failure.ledger_identity.processor_version, input_key = %failure.ledger_identity.input_key, input_hash = %failure.ledger_identity.input_hash, error_code = %failure.error_code, error_message = %failure.error_message, "persist PostgreSQL contextual decode failure"); let started_at = std::time::Instant::now();
if failure.ledger_identity.stage != "instruction_decode" if failure.ledger_identity.stage != "instruction_decode"
|| failure.signature.trim().is_empty() || failure.signature.trim().is_empty()
|| failure.instruction_path.trim().is_empty() || failure.instruction_path.trim().is_empty()
@@ -514,21 +501,32 @@ pub(crate) async fn mark_decode_failed(
))); )));
}, },
}; };
let lifecycle_result = sqlx::query( let invalidation_result = crate::invalidate_decode_descendants_in_transaction(
"UPDATE k_sol_core_instructions SET processing_state = CASE WHEN processing_state IN ('pending', 'failed', 'replay_requested', 'ignored') THEN 'failed' ELSE processing_state END, processor_name = CASE WHEN processing_state IN ('pending', 'failed', 'replay_requested', 'ignored') THEN $1 ELSE processor_name END, processor_version = CASE WHEN processing_state IN ('pending', 'failed', 'replay_requested', 'ignored') THEN $2 ELSE processor_version END, lifecycle_reason = CASE WHEN processing_state IN ('pending', 'failed', 'replay_requested', 'ignored') THEN $3 ELSE lifecycle_reason END, updated_at = NOW() WHERE signature = $4 AND instruction_path = $5", &mut transaction,
failure.ledger_identity.processor_name.as_str(),
failure.ledger_identity.processor_version.as_str(),
failure.ledger_identity.input_key.as_str(),
) )
.await;
if let std::result::Result::Err(error) = invalidation_result {
return std::result::Result::Err(error);
}
for sql in [
"DELETE FROM k_sol_decode_events WHERE processor_name = $1 AND processor_version = $2 AND input_key = $3",
"DELETE FROM k_sol_decode_coverage_observations WHERE processor_name = $1 AND processor_version = $2 AND input_key = $3",
] {
let delete_result = sqlx::query(sql)
.bind(failure.ledger_identity.processor_name.as_str()) .bind(failure.ledger_identity.processor_name.as_str())
.bind(failure.ledger_identity.processor_version.as_str()) .bind(failure.ledger_identity.processor_version.as_str())
.bind(failure.error_code.as_str()) .bind(failure.ledger_identity.input_key.as_str())
.bind(failure.signature.as_str())
.bind(failure.instruction_path.as_str())
.execute(&mut *transaction) .execute(&mut *transaction)
.await; .await;
if let std::result::Result::Err(error) = lifecycle_result { if let std::result::Result::Err(error) = delete_result {
return std::result::Result::Err(ks_core::Error::db(format!( return std::result::Result::Err(ks_core::Error::db(format!(
"postgres decode failure lifecycle update failed: {error}" "postgres failed decode replacement cleanup failed: {error}"
))); )));
} }
}
let ledger_result = crate::postgres::query::decode_pipeline_queries::upsert_ledger_terminal( let ledger_result = crate::postgres::query::decode_pipeline_queries::upsert_ledger_terminal(
&mut transaction, &mut transaction,
&failure.ledger_identity, &failure.ledger_identity,
@@ -540,6 +538,15 @@ pub(crate) async fn mark_decode_failed(
if let std::result::Result::Err(error) = ledger_result { if let std::result::Result::Err(error) = ledger_result {
return std::result::Result::Err(error); return std::result::Result::Err(error);
} }
let lifecycle_result = crate::recompute_instruction_lifecycle_in_transaction(
&mut transaction,
failure.signature.as_str(),
failure.instruction_path.as_str(),
)
.await;
if let std::result::Result::Err(error) = lifecycle_result {
return std::result::Result::Err(error);
}
let commit_result = transaction.commit().await; let commit_result = transaction.commit().await;
if let std::result::Result::Err(error) = commit_result { if let std::result::Result::Err(error) = commit_result {
return std::result::Result::Err(ks_core::Error::db(format!( return std::result::Result::Err(ks_core::Error::db(format!(
@@ -547,33 +554,16 @@ pub(crate) async fn mark_decode_failed(
))); )));
} }
let outcome = crate::InsertOutcome::new(0, 1, 0); let outcome = crate::InsertOutcome::new(0, 1, 0);
tracing::debug!(target: crate::TRACING_TARGET, backend = "postgres", domain = "ks-store.pg", action = "mark_decode_failed", signature = %failure.signature, instruction_path = %failure.instruction_path, processor_name = %failure.ledger_identity.processor_name, processor_version = %failure.ledger_identity.processor_version, outcome = ?outcome, committed = true, "PostgreSQL contextual decode failure persisted"); trace_query_success("mark_decode_failed", started_at, 1);
return std::result::Result::Ok(outcome); return std::result::Result::Ok(outcome);
} }
pub(crate) async fn persist_materialization_result( pub(crate) async fn persist_materialization_result(
pool: &sqlx::PgPool, pool: &sqlx::PgPool,
bundle: &crate::MaterializationPersistenceBundle, bundle: &crate::MaterializationPersistenceBundle,
force_replay: bool, _force_replay: bool,
) -> ks_core::Result<crate::InsertOutcome> { ) -> ks_core::Result<crate::InsertOutcome> {
if bundle.status == "failed" { let started_at = std::time::Instant::now();
tracing::error!(
target: crate::TRACING_TARGET, backend = "postgres", domain = "ks-store.pg",
action = "persist_materialization_failure",
signature = %bundle.signature,
instruction_path = %bundle.instruction_path,
processor_name = %bundle.ledger_identity.processor_name,
processor_version = %bundle.ledger_identity.processor_version,
input_key = %bundle.ledger_identity.input_key,
input_hash = %bundle.ledger_identity.input_hash,
source_decoder_name = %bundle.source_decoder_name,
source_decoder_version = %bundle.source_decoder_version,
error_code = ?bundle.error_code,
error_message = ?bundle.error_message,
"persist PostgreSQL materialization failure"
);
}
tracing::debug!(target: crate::TRACING_TARGET, backend = "postgres", domain = "ks-store.pg", action = "persist_materialization_result", signature = %bundle.signature, instruction_path = %bundle.instruction_path, stage = %bundle.ledger_identity.stage, processor_name = %bundle.ledger_identity.processor_name, processor_version = %bundle.ledger_identity.processor_version, input_key = %bundle.ledger_identity.input_key, input_hash = %bundle.ledger_identity.input_hash, source_decoder_name = %bundle.source_decoder_name, source_decoder_version = %bundle.source_decoder_version, source_decode_input_key = %bundle.source_decode_input_key, status = %bundle.status, output_count = bundle.outputs.len(), force_replay, "persist PostgreSQL materialization result");
let validation_result = bundle.validate(); let validation_result = bundle.validate();
if let std::result::Result::Err(error) = validation_result { if let std::result::Result::Err(error) = validation_result {
return std::result::Result::Err(error); return std::result::Result::Err(error);
@@ -646,23 +636,6 @@ pub(crate) async fn persist_materialization_result(
if let std::result::Result::Err(error) = ledger_result { if let std::result::Result::Err(error) = ledger_result {
return std::result::Result::Err(error); return std::result::Result::Err(error);
} }
if !bundle.outputs.is_empty() {
let lifecycle_result = sqlx::query(
"UPDATE k_sol_core_instructions SET processing_state = 'materialized', processor_name = $1, processor_version = $2, lifecycle_reason = $3, updated_at = NOW() WHERE signature = $4 AND instruction_path = $5",
)
.bind(bundle.ledger_identity.processor_name.as_str())
.bind(bundle.ledger_identity.processor_version.as_str())
.bind(bundle.status.as_str())
.bind(bundle.signature.as_str())
.bind(bundle.instruction_path.as_str())
.execute(&mut *transaction)
.await;
if let std::result::Result::Err(error) = lifecycle_result {
return std::result::Result::Err(ks_core::Error::db(format!(
"postgres materialization lifecycle update failed: {error}"
)));
}
}
let coverage_result = let coverage_result =
crate::postgres::query::decode_pipeline_queries::refresh_materialized_coverage_count( crate::postgres::query::decode_pipeline_queries::refresh_materialized_coverage_count(
&mut transaction, &mut transaction,
@@ -674,6 +647,15 @@ pub(crate) async fn persist_materialization_result(
if let std::result::Result::Err(error) = coverage_result { if let std::result::Result::Err(error) = coverage_result {
return std::result::Result::Err(error); return std::result::Result::Err(error);
} }
let lifecycle_result = crate::recompute_instruction_lifecycle_in_transaction(
&mut transaction,
bundle.signature.as_str(),
bundle.instruction_path.as_str(),
)
.await;
if let std::result::Result::Err(error) = lifecycle_result {
return std::result::Result::Err(error);
}
let commit_result = transaction.commit().await; let commit_result = transaction.commit().await;
if let std::result::Result::Err(error) = commit_result { if let std::result::Result::Err(error) = commit_result {
return std::result::Result::Err(ks_core::Error::db(format!( return std::result::Result::Err(ks_core::Error::db(format!(
@@ -690,7 +672,8 @@ pub(crate) async fn persist_materialization_result(
}, },
}; };
let outcome = crate::InsertOutcome::new(count, 1, 0); let outcome = crate::InsertOutcome::new(count, 1, 0);
tracing::debug!(target: crate::TRACING_TARGET, backend = "postgres", domain = "ks-store.pg", action = "persist_materialization_result", signature = %bundle.signature, instruction_path = %bundle.instruction_path, processor_name = %bundle.ledger_identity.processor_name, processor_version = %bundle.ledger_identity.processor_version, status = %bundle.status, outcome = ?outcome, committed = true, "PostgreSQL materialization result persisted"); let elapsed_ms = started_at.elapsed().as_secs_f64() * 1000.0;
tracing::trace!(target: crate::TRACING_TARGET, backend = "postgres", domain = "ks-store.pg", action = "persist_materialization_result", elapsed_ms, rows = count, outcome = "success", "PostgreSQL store operation completed");
return std::result::Result::Ok(outcome); return std::result::Result::Ok(outcome);
} }
@@ -700,12 +683,12 @@ pub(crate) async fn list_decode_coverage_summary(
processor_version: std::option::Option<&str>, processor_version: std::option::Option<&str>,
limit: u32, limit: u32,
) -> ks_core::Result<std::vec::Vec<crate::DecodeCoverageSummaryRow>> { ) -> ks_core::Result<std::vec::Vec<crate::DecodeCoverageSummaryRow>> {
tracing::debug!(target: crate::TRACING_TARGET, backend = "postgres", domain = "ks-store.pg", action = "list_decode_coverage_summary", processor_name = ?processor_name, processor_version = ?processor_version, limit, "query PostgreSQL decode coverage summary");
if limit == 0 { if limit == 0 {
return std::result::Result::Err(ks_core::Error::db( return std::result::Result::Err(ks_core::Error::db(
"decode coverage summary limit must be greater than zero", "decode coverage summary limit must be greater than zero",
)); ));
} }
let started_at = std::time::Instant::now();
let query_result = sqlx::query( let query_result = sqlx::query(
"WITH declared AS (SELECT processor_name, processor_version, program_id, surface_code, entry_code, COUNT(*)::BIGINT AS declared_count FROM k_sol_decode_coverage_declarations WHERE ($1::text IS NULL OR processor_name = $1) AND ($2::text IS NULL OR processor_version = $2) GROUP BY processor_name, processor_version, program_id, surface_code, entry_code), observed AS (SELECT processor_name, processor_version, program_id, surface_code, COALESCE(entry_code, 'unknown') AS entry_code, COUNT(*)::BIGINT AS observed_count, COUNT(*) FILTER (WHERE recognized)::BIGINT AS recognized_count, COALESCE(SUM(decoded_count), 0)::BIGINT AS decoded_count, COALESCE(SUM(materialized_count), 0)::BIGINT AS materialized_count, COALESCE(SUM(error_count), 0)::BIGINT AS error_count, COUNT(*) FILTER (WHERE status = 'unsupported' OR entry_code IS NULL)::BIGINT AS unknown_count, COUNT(*) FILTER (WHERE NOT transaction_failed)::BIGINT AS successful_transaction_count, COUNT(*) FILTER (WHERE transaction_failed)::BIGINT AS failed_transaction_count FROM k_sol_decode_coverage_observations WHERE ($1::text IS NULL OR processor_name = $1) AND ($2::text IS NULL OR processor_version = $2) GROUP BY processor_name, processor_version, program_id, surface_code, COALESCE(entry_code, 'unknown')) SELECT COALESCE(d.processor_name, o.processor_name) AS processor_name, COALESCE(d.processor_version, o.processor_version) AS processor_version, COALESCE(d.program_id, o.program_id) AS program_id, COALESCE(d.surface_code, o.surface_code) AS surface_code, COALESCE(d.entry_code, o.entry_code) AS entry_code, COALESCE(d.declared_count, 0)::BIGINT AS declared_count, COALESCE(o.observed_count, 0)::BIGINT AS observed_count, COALESCE(o.recognized_count, 0)::BIGINT AS recognized_count, COALESCE(o.decoded_count, 0)::BIGINT AS decoded_count, COALESCE(o.materialized_count, 0)::BIGINT AS materialized_count, COALESCE(o.error_count, 0)::BIGINT AS error_count, COALESCE(o.unknown_count, 0)::BIGINT AS unknown_count, COALESCE(o.successful_transaction_count, 0)::BIGINT AS successful_transaction_count, COALESCE(o.failed_transaction_count, 0)::BIGINT AS failed_transaction_count FROM declared d FULL OUTER JOIN observed o ON d.processor_name = o.processor_name AND d.processor_version = o.processor_version AND d.program_id = o.program_id AND COALESCE(d.surface_code, '') = COALESCE(o.surface_code, '') AND d.entry_code = o.entry_code ORDER BY observed_count DESC, processor_name, program_id, entry_code LIMIT $3", "WITH declared AS (SELECT processor_name, processor_version, program_id, surface_code, entry_code, COUNT(*)::BIGINT AS declared_count FROM k_sol_decode_coverage_declarations WHERE ($1::text IS NULL OR processor_name = $1) AND ($2::text IS NULL OR processor_version = $2) GROUP BY processor_name, processor_version, program_id, surface_code, entry_code), observed AS (SELECT processor_name, processor_version, program_id, surface_code, COALESCE(entry_code, 'unknown') AS entry_code, COUNT(*)::BIGINT AS observed_count, COUNT(*) FILTER (WHERE recognized)::BIGINT AS recognized_count, COALESCE(SUM(decoded_count), 0)::BIGINT AS decoded_count, COALESCE(SUM(materialized_count), 0)::BIGINT AS materialized_count, COALESCE(SUM(error_count), 0)::BIGINT AS error_count, COUNT(*) FILTER (WHERE status = 'unsupported' OR entry_code IS NULL)::BIGINT AS unknown_count, COUNT(*) FILTER (WHERE NOT transaction_failed)::BIGINT AS successful_transaction_count, COUNT(*) FILTER (WHERE transaction_failed)::BIGINT AS failed_transaction_count FROM k_sol_decode_coverage_observations WHERE ($1::text IS NULL OR processor_name = $1) AND ($2::text IS NULL OR processor_version = $2) GROUP BY processor_name, processor_version, program_id, surface_code, COALESCE(entry_code, 'unknown')) SELECT COALESCE(d.processor_name, o.processor_name) AS processor_name, COALESCE(d.processor_version, o.processor_version) AS processor_version, COALESCE(d.program_id, o.program_id) AS program_id, COALESCE(d.surface_code, o.surface_code) AS surface_code, COALESCE(d.entry_code, o.entry_code) AS entry_code, COALESCE(d.declared_count, 0)::BIGINT AS declared_count, COALESCE(o.observed_count, 0)::BIGINT AS observed_count, COALESCE(o.recognized_count, 0)::BIGINT AS recognized_count, COALESCE(o.decoded_count, 0)::BIGINT AS decoded_count, COALESCE(o.materialized_count, 0)::BIGINT AS materialized_count, COALESCE(o.error_count, 0)::BIGINT AS error_count, COALESCE(o.unknown_count, 0)::BIGINT AS unknown_count, COALESCE(o.successful_transaction_count, 0)::BIGINT AS successful_transaction_count, COALESCE(o.failed_transaction_count, 0)::BIGINT AS failed_transaction_count FROM declared d FULL OUTER JOIN observed o ON d.processor_name = o.processor_name AND d.processor_version = o.processor_version AND d.program_id = o.program_id AND COALESCE(d.surface_code, '') = COALESCE(o.surface_code, '') AND d.entry_code = o.entry_code ORDER BY observed_count DESC, processor_name, program_id, entry_code LIMIT $3",
) )
@@ -717,6 +700,7 @@ pub(crate) async fn list_decode_coverage_summary(
let rows = match query_result { let rows = match query_result {
std::result::Result::Ok(value) => value, std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => { std::result::Result::Err(error) => {
trace_query_failure("list_decode_coverage_summary", started_at, &error);
return std::result::Result::Err(ks_core::Error::db(format!( return std::result::Result::Err(ks_core::Error::db(format!(
"postgres decode coverage summary query failed: {error}" "postgres decode coverage summary query failed: {error}"
))); )));
@@ -731,7 +715,7 @@ pub(crate) async fn list_decode_coverage_summary(
std::result::Result::Err(error) => return std::result::Result::Err(error), std::result::Result::Err(error) => return std::result::Result::Err(error),
} }
} }
tracing::debug!(target: crate::TRACING_TARGET, backend = "postgres", domain = "ks-store.pg", action = "list_decode_coverage_summary", processor_name = ?processor_name, processor_version = ?processor_version, row_count = output.len(), "PostgreSQL decode coverage summary loaded"); trace_query_success("list_decode_coverage_summary", started_at, output.len());
return std::result::Result::Ok(output); return std::result::Result::Ok(output);
} }
@@ -975,6 +959,16 @@ fn map_error<T>(column: &str, error: sqlx::Error) -> ks_core::Result<T> {
))); )));
} }
fn trace_query_success(action: &str, started_at: std::time::Instant, rows: usize) {
let elapsed_ms = started_at.elapsed().as_secs_f64() * 1000.0;
tracing::trace!(target: crate::TRACING_TARGET, backend = "postgres", domain = "ks-store.pg", action = action, elapsed_ms, rows, outcome = "success", "PostgreSQL store operation completed");
}
fn trace_query_failure(action: &str, started_at: std::time::Instant, error: &sqlx::Error) {
let elapsed_ms = started_at.elapsed().as_secs_f64() * 1000.0;
tracing::trace!(target: crate::TRACING_TARGET, backend = "postgres", domain = "ks-store.pg", action = action, elapsed_ms, error = %error, outcome = "error", "PostgreSQL store operation failed");
}
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
fn result_or_panic<T, E>(result: std::result::Result<T, E>) -> T fn result_or_panic<T, E>(result: std::result::Result<T, E>) -> T

View File

@@ -0,0 +1,275 @@
// file: ks-store/src/postgres/query/invalidation_queries.rs
// version: 2
//! PostgreSQL descendant invalidation and aggregate instruction lifecycle helpers.
pub(crate) async fn invalidate_core_descendants_in_transaction(
transaction: &mut sqlx::Transaction<'_, sqlx::Postgres>,
signature: &str,
) -> ks_core::Result<u64> {
let input_prefix = format!("{signature}:");
let mut deleted = 0_u64;
for (sql, stage) in [
(
"DELETE FROM k_sol_ops_processing_ledger WHERE stage = $1 AND LEFT(input_key, LENGTH($2)) = $2",
"event_materialization",
),
(
"DELETE FROM k_sol_ops_processing_ledger WHERE stage = $1 AND LEFT(input_key, LENGTH($2)) = $2",
"instruction_decode",
),
] {
let result = sqlx::query(sql)
.bind(stage)
.bind(input_prefix.as_str())
.execute(&mut **transaction)
.await;
match result {
std::result::Result::Ok(value) => {
deleted = deleted.saturating_add(value.rows_affected())
},
std::result::Result::Err(error) => {
return std::result::Result::Err(ks_core::Error::db(format!(
"postgres descendant ledger invalidation failed: {error}"
)));
},
}
}
for sql in [
"DELETE FROM k_sol_mat_outputs WHERE signature = $1",
"DELETE FROM k_sol_decode_events WHERE signature = $1",
"DELETE FROM k_sol_decode_coverage_observations WHERE signature = $1",
] {
let result = sqlx::query(sql).bind(signature).execute(&mut **transaction).await;
match result {
std::result::Result::Ok(value) => {
deleted = deleted.saturating_add(value.rows_affected())
},
std::result::Result::Err(error) => {
return std::result::Result::Err(ks_core::Error::db(format!(
"postgres descendant output invalidation failed: {error}"
)));
},
}
}
return std::result::Result::Ok(deleted);
}
pub(crate) async fn invalidate_decode_descendants_in_transaction(
transaction: &mut sqlx::Transaction<'_, sqlx::Postgres>,
decoder_name: &str,
decoder_version: &str,
decode_input_key: &str,
) -> ks_core::Result<u64> {
let materialization_prefix = format!("{decode_input_key}:{decoder_name}:{decoder_version}:");
let ledger_result = sqlx::query(
"DELETE FROM k_sol_ops_processing_ledger WHERE stage = 'event_materialization' AND LEFT(input_key, LENGTH($1)) = $1",
)
.bind(materialization_prefix.as_str())
.execute(&mut **transaction)
.await;
let ledger_deleted = match ledger_result {
std::result::Result::Ok(value) => value.rows_affected(),
std::result::Result::Err(error) => {
return std::result::Result::Err(ks_core::Error::db(format!(
"postgres materialization ledger invalidation failed: {error}"
)));
},
};
let output_result = sqlx::query(
"DELETE FROM k_sol_mat_outputs WHERE source_decoder_name = $1 AND source_decoder_version = $2 AND source_decode_input_key = $3",
)
.bind(decoder_name)
.bind(decoder_version)
.bind(decode_input_key)
.execute(&mut **transaction)
.await;
let output_deleted = match output_result {
std::result::Result::Ok(value) => value.rows_affected(),
std::result::Result::Err(error) => {
return std::result::Result::Err(ks_core::Error::db(format!(
"postgres materialization output invalidation failed: {error}"
)));
},
};
return std::result::Result::Ok(ledger_deleted.saturating_add(output_deleted));
}
pub(crate) async fn recompute_instruction_lifecycle_in_transaction(
transaction: &mut sqlx::Transaction<'_, sqlx::Postgres>,
signature: &str,
instruction_path: &str,
) -> ks_core::Result<u64> {
let decode_input_key = format!("{signature}:{instruction_path}");
let decode_result = sqlx::query_as::<
sqlx::Postgres,
(std::string::String, std::string::String, std::string::String),
>(
"SELECT processor_name, processor_version, status FROM k_sol_decode_coverage_observations WHERE input_key = $1 ORDER BY updated_at DESC, id DESC LIMIT 1",
)
.bind(decode_input_key.as_str())
.fetch_optional(&mut **transaction)
.await;
let decode = match decode_result {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => {
return std::result::Result::Err(ks_core::Error::db(format!(
"postgres decode lifecycle lookup failed: {error}"
)));
},
};
if let std::option::Option::Some((decoder_name, decoder_version, status)) = decode {
let materialized_result = sqlx::query_as::<
sqlx::Postgres,
(std::string::String, std::string::String),
>(
"SELECT processor_name, processor_version FROM k_sol_mat_outputs WHERE source_decode_input_key = $1 AND source_decoder_name = $2 AND source_decoder_version = $3 ORDER BY updated_at DESC, id DESC LIMIT 1",
)
.bind(decode_input_key.as_str())
.bind(decoder_name.as_str())
.bind(decoder_version.as_str())
.fetch_optional(&mut **transaction)
.await;
let materialized = match materialized_result {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => {
return std::result::Result::Err(ks_core::Error::db(format!(
"postgres materialized lifecycle lookup failed: {error}"
)));
},
};
if let std::option::Option::Some((processor_name, processor_version)) = materialized {
return update_instruction_lifecycle_in_transaction(
transaction,
signature,
instruction_path,
"materialized",
std::option::Option::Some(processor_name.as_str()),
std::option::Option::Some(processor_version.as_str()),
std::option::Option::Some("materialized"),
)
.await;
}
let processing_state = match status.as_str() {
"decoded" => "decoded",
"ignored" | "unsupported" => "ignored",
"failed" => "failed",
_unknown => "decoded",
};
return update_instruction_lifecycle_in_transaction(
transaction,
signature,
instruction_path,
processing_state,
std::option::Option::Some(decoder_name.as_str()),
std::option::Option::Some(decoder_version.as_str()),
std::option::Option::Some(status.as_str()),
)
.await;
}
let ledger_result = sqlx::query_as::<
sqlx::Postgres,
(std::string::String, std::string::String, std::string::String),
>(
"SELECT processor_name, processor_version, status FROM k_sol_ops_processing_ledger WHERE stage = 'instruction_decode' AND input_key = $1 ORDER BY updated_at DESC, id DESC LIMIT 1",
)
.bind(decode_input_key.as_str())
.fetch_optional(&mut **transaction)
.await;
let ledger = match ledger_result {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => {
return std::result::Result::Err(ks_core::Error::db(format!(
"postgres decode ledger lifecycle lookup failed: {error}"
)));
},
};
if let std::option::Option::Some((processor_name, processor_version, status)) = ledger {
let processing_state = if status == "failed" { "failed" } else { "decoded" };
return update_instruction_lifecycle_in_transaction(
transaction,
signature,
instruction_path,
processing_state,
std::option::Option::Some(processor_name.as_str()),
std::option::Option::Some(processor_version.as_str()),
std::option::Option::Some(status.as_str()),
)
.await;
}
return update_instruction_lifecycle_in_transaction(
transaction,
signature,
instruction_path,
"pending",
std::option::Option::None,
std::option::Option::None,
std::option::Option::Some("descendants_invalidated"),
)
.await;
}
async fn update_instruction_lifecycle_in_transaction(
transaction: &mut sqlx::Transaction<'_, sqlx::Postgres>,
signature: &str,
instruction_path: &str,
processing_state: &str,
processor_name: std::option::Option<&str>,
processor_version: std::option::Option<&str>,
reason: std::option::Option<&str>,
) -> ks_core::Result<u64> {
let scope_result = crate::CoreInstructionScope::from_instruction_path(instruction_path);
let scope = match scope_result {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let sql = match scope {
crate::CoreInstructionScope::TopLevel => {
"UPDATE k_sol_core_instructions SET processing_state = $1, processor_name = $2, processor_version = $3, lifecycle_reason = $4, updated_at = NOW() WHERE signature = $5 AND instruction_path = $6"
},
crate::CoreInstructionScope::Inner => {
"UPDATE k_sol_core_inner_instructions SET processing_state = $1, processor_name = $2, processor_version = $3, lifecycle_reason = $4, updated_at = NOW() WHERE signature = $5 AND instruction_path = $6"
},
};
let result = sqlx::query(sql)
.bind(processing_state)
.bind(processor_name)
.bind(processor_version)
.bind(reason)
.bind(signature)
.bind(instruction_path)
.execute(&mut **transaction)
.await;
return match result {
std::result::Result::Ok(value) => std::result::Result::Ok(value.rows_affected()),
std::result::Result::Err(error) => std::result::Result::Err(ks_core::Error::db(format!(
"postgres aggregate instruction lifecycle update failed: {error}"
))),
};
}
#[cfg(test)]
mod tests {
#[test]
fn lifecycle_materialization_is_scoped_to_the_current_decoder_identity() {
let source = include_str!("invalidation_queries.rs");
assert!(source.contains("source_decode_input_key = $1 AND source_decoder_name = $2 AND source_decoder_version = $3"));
}
#[test]
fn descendant_invalidation_stages_are_exact() {
let source = include_str!("invalidation_queries.rs");
let tests_marker = source.find("#[cfg(test)]");
let production_source = match tests_marker {
std::option::Option::Some(index) => &source[..index],
std::option::Option::None => source,
};
assert!(production_source.contains("stage = 'event_materialization'"));
assert!(production_source.contains("\"instruction_decode\""));
assert!(production_source.contains("k_sol_mat_outputs"));
assert!(production_source.contains("k_sol_decode_events"));
assert!(production_source.contains("k_sol_decode_coverage_observations"));
assert!(production_source.contains("LEFT(input_key, LENGTH("));
assert!(!production_source.contains("input_key LIKE"));
}
}

View File

@@ -1,5 +1,5 @@
// file: ks-store/src/postgres/query/raw_queries.rs // file: ks-store/src/postgres/query/raw_queries.rs
// version: 8 // version: 9
//! PostgreSQL canonical transaction and acquisition observation SQL queries. //! PostgreSQL canonical transaction and acquisition observation SQL queries.
@@ -49,6 +49,40 @@ pub(crate) async fn has_transaction_observation_key(
}; };
} }
pub(crate) async fn has_account_observation_key(
pool: &sqlx::PgPool,
observation_key: &str,
) -> ks_core::Result<bool> {
let validation_result =
validate_required_text(observation_key, "account observation key must not be empty");
if let std::result::Result::Err(error) = validation_result {
return std::result::Result::Err(error);
}
let started_at = std::time::Instant::now();
let query_result = sqlx::query_scalar::<sqlx::Postgres, bool>(
"SELECT EXISTS(SELECT 1 FROM k_sol_obs_account_observations WHERE observation_key = $1)",
)
.bind(observation_key)
.fetch_one(pool)
.await;
return match query_result {
std::result::Result::Ok(value) => {
trace_query_success(
"has_account_observation_key",
started_at,
if value { 1_usize } else { 0_usize },
);
std::result::Result::Ok(value)
},
std::result::Result::Err(error) => {
trace_query_failure("has_account_observation_key", started_at, &error);
std::result::Result::Err(ks_core::Error::db(format!(
"postgres account observation lookup failed: {error}"
)))
},
};
}
pub(crate) async fn insert_raw_transaction( pub(crate) async fn insert_raw_transaction(
pool: &sqlx::PgPool, pool: &sqlx::PgPool,
input: &crate::RawTransactionInsert, input: &crate::RawTransactionInsert,
@@ -145,6 +179,83 @@ pub(crate) async fn insert_transaction_observation(
}; };
} }
pub(crate) async fn insert_account_observation(
pool: &sqlx::PgPool,
input: &crate::AccountObservationInsert,
) -> ks_core::Result<crate::InsertOutcome> {
let validation_result = input.validate();
if let std::result::Result::Err(error) = validation_result {
return std::result::Result::Err(error);
}
let context_slot_result = sql_slot_from_u64(input.context_slot);
let context_slot = match context_slot_result {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let space_result = optional_sql_bigint_from_u64(input.space);
let space = match space_result {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let payload_size_result = optional_sql_bigint_from_u64(input.payload_size_bytes);
let payload_size_bytes = match payload_size_result {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let lamports = input.lamports.map(|value| return value.to_string());
let rent_epoch = input.rent_epoch.map(|value| return value.to_string());
let origin = transaction_observation_origin_to_sql(input.origin);
let status = transaction_observation_status_to_sql(input.status);
let started_at = std::time::Instant::now();
let query_result = sqlx::query_scalar::<sqlx::Postgres, i64>(
"INSERT INTO k_sol_obs_account_observations (observation_key, account_key, context_slot, owner, lamports, executable, rent_epoch, space, data_base64, data_hash, provider, endpoint_code, protocol, acquisition_method, commitment, capture_session_id, filter_code, origin, detected_at, received_at, normalized_at, payload_size_bytes, source_payload_hash, status, error_code, error_message) VALUES ($1, $2, $3, $4, $5::NUMERIC, $6, $7::NUMERIC, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26) ON CONFLICT (observation_key) DO NOTHING RETURNING id",
)
.bind(input.observation_key.as_str())
.bind(input.account_key.as_str())
.bind(context_slot)
.bind(input.owner.as_deref())
.bind(lamports.as_deref())
.bind(input.executable)
.bind(rent_epoch.as_deref())
.bind(space)
.bind(input.data_base64.as_deref())
.bind(input.data_hash.as_deref())
.bind(input.provider.as_str())
.bind(input.endpoint_code.as_deref())
.bind(input.protocol.as_str())
.bind(input.acquisition_method.as_str())
.bind(input.commitment.as_deref())
.bind(input.capture_session_id.as_deref())
.bind(input.filter_code.as_deref())
.bind(origin)
.bind(input.detected_at.as_ref())
.bind(input.received_at)
.bind(input.normalized_at.as_ref())
.bind(payload_size_bytes)
.bind(input.source_payload_hash.as_deref())
.bind(status)
.bind(input.error_code.as_deref())
.bind(input.error_message.as_deref())
.fetch_optional(pool)
.await;
return match query_result {
std::result::Result::Ok(std::option::Option::Some(_id)) => {
trace_query_success("insert_account_observation", started_at, 1);
std::result::Result::Ok(crate::InsertOutcome::new(1, 0, 0))
},
std::result::Result::Ok(std::option::Option::None) => {
trace_query_success("insert_account_observation", started_at, 0);
std::result::Result::Ok(crate::InsertOutcome::new(0, 0, 1))
},
std::result::Result::Err(error) => {
trace_query_failure("insert_account_observation", started_at, &error);
std::result::Result::Err(ks_core::Error::db(format!(
"postgres account observation insert failed: {error}"
)))
},
};
}
pub(crate) async fn update_raw_payload_lifecycle( pub(crate) async fn update_raw_payload_lifecycle(
pool: &sqlx::PgPool, pool: &sqlx::PgPool,
mark: &crate::RawPayloadLifecycleMark, mark: &crate::RawPayloadLifecycleMark,
@@ -282,6 +393,16 @@ fn outcome_from_update_result(
}; };
} }
fn trace_query_success(action: &str, started_at: std::time::Instant, rows: usize) {
let elapsed_ms = started_at.elapsed().as_secs_f64() * 1000.0;
tracing::trace!(target: crate::TRACING_TARGET, backend = "postgres", domain = "ks-store.pg", action = action, elapsed_ms, rows, outcome = "success", "PostgreSQL store operation completed");
}
fn trace_query_failure(action: &str, started_at: std::time::Instant, error: &sqlx::Error) {
let elapsed_ms = started_at.elapsed().as_secs_f64() * 1000.0;
tracing::trace!(target: crate::TRACING_TARGET, backend = "postgres", domain = "ks-store.pg", action = action, elapsed_ms, error = %error, outcome = "error", "PostgreSQL store operation failed");
}
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
#[test] #[test]
@@ -313,6 +434,12 @@ mod tests {
assert_eq!(value, "normalized"); assert_eq!(value, "normalized");
} }
#[test]
fn optional_bigint_rejects_values_above_bigint() {
let result = super::optional_sql_bigint_from_u64(std::option::Option::Some(u64::MAX));
assert!(result.is_err());
}
#[test] #[test]
fn sql_slot_rejects_values_above_bigint() { fn sql_slot_rejects_values_above_bigint() {
let result = super::sql_slot_from_u64(u64::MAX); let result = super::sql_slot_from_u64(u64::MAX);

View File

@@ -1,5 +1,5 @@
// file: ks-store/src/postgres/query/replay_candidate_queries.rs // file: ks-store/src/postgres/query/replay_candidate_queries.rs
// version: 7 // version: 8
//! Read-only PostgreSQL queries for replay candidate discovery. //! Read-only PostgreSQL queries for replay candidate discovery.
@@ -67,6 +67,7 @@ pub(crate) async fn list_replay_transaction_candidates(
} else { } else {
crate::postgres::query::replay_candidate_queries::transaction_candidate_sql_asc() crate::postgres::query::replay_candidate_queries::transaction_candidate_sql_asc()
}; };
let started_at = std::time::Instant::now();
let query_result = sqlx::query_as::< let query_result = sqlx::query_as::<
sqlx::Postgres, sqlx::Postgres,
crate::postgres::query::replay_candidate_queries::ReplayTransactionCandidateRow, crate::postgres::query::replay_candidate_queries::ReplayTransactionCandidateRow,
@@ -86,6 +87,7 @@ pub(crate) async fn list_replay_transaction_candidates(
let rows = match query_result { let rows = match query_result {
std::result::Result::Ok(value) => value, std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => { std::result::Result::Err(error) => {
trace_query_failure("list_replay_transaction_candidates", started_at, &error);
return std::result::Result::Err(ks_core::Error::db(format!( return std::result::Result::Err(ks_core::Error::db(format!(
"postgres replay transaction candidate query failed: {error}" "postgres replay transaction candidate query failed: {error}"
))); )));
@@ -110,6 +112,7 @@ pub(crate) async fn list_replay_transaction_candidates(
updated_at: row.updated_at, updated_at: row.updated_at,
}); });
} }
trace_query_success("list_replay_transaction_candidates", started_at, output.len());
return std::result::Result::Ok(output); return std::result::Result::Ok(output);
} }
@@ -117,6 +120,7 @@ pub(crate) async fn list_replay_program_summaries(
pool: &sqlx::PgPool, pool: &sqlx::PgPool,
filter: &crate::ReplayProgramFilter, filter: &crate::ReplayProgramFilter,
) -> ks_core::Result<std::vec::Vec<crate::ReplayProgramSummary>> { ) -> ks_core::Result<std::vec::Vec<crate::ReplayProgramSummary>> {
let started_at = std::time::Instant::now();
let query_result = sqlx::query_as::<sqlx::Postgres, crate::postgres::query::replay_candidate_queries::ReplayProgramSummaryRow>( let query_result = sqlx::query_as::<sqlx::Postgres, crate::postgres::query::replay_candidate_queries::ReplayProgramSummaryRow>(
r#"WITH occurrences AS ( r#"WITH occurrences AS (
SELECT program_id, signature, slot, 'top_level'::TEXT AS scope FROM k_sol_core_instructions SELECT program_id, signature, slot, 'top_level'::TEXT AS scope FROM k_sol_core_instructions
@@ -145,6 +149,7 @@ pub(crate) async fn list_replay_program_summaries(
let rows = match query_result { let rows = match query_result {
std::result::Result::Ok(value) => value, std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => { std::result::Result::Err(error) => {
trace_query_failure("list_replay_program_summaries", started_at, &error);
return std::result::Result::Err(ks_core::Error::db(format!( return std::result::Result::Err(ks_core::Error::db(format!(
"postgres replay program summary query failed: {error}" "postgres replay program summary query failed: {error}"
))); )));
@@ -162,6 +167,7 @@ pub(crate) async fn list_replay_program_summaries(
max_slot: row.max_slot, max_slot: row.max_slot,
}); });
} }
trace_query_success("list_replay_program_summaries", started_at, output.len());
return std::result::Result::Ok(output); return std::result::Result::Ok(output);
} }
@@ -170,6 +176,7 @@ pub(crate) async fn list_replay_entity_summaries(
filter: &crate::ReplayEntityFilter, filter: &crate::ReplayEntityFilter,
) -> ks_core::Result<std::vec::Vec<crate::ReplayEntitySummary>> { ) -> ks_core::Result<std::vec::Vec<crate::ReplayEntitySummary>> {
let entity_kind = filter.entity_kind.as_sql(); let entity_kind = filter.entity_kind.as_sql();
let started_at = std::time::Instant::now();
let query_result = sqlx::query_as::< let query_result = sqlx::query_as::<
sqlx::Postgres, sqlx::Postgres,
crate::postgres::query::replay_candidate_queries::ReplayEntitySummaryRow, crate::postgres::query::replay_candidate_queries::ReplayEntitySummaryRow,
@@ -207,6 +214,7 @@ pub(crate) async fn list_replay_entity_summaries(
let rows = match query_result { let rows = match query_result {
std::result::Result::Ok(value) => value, std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => { std::result::Result::Err(error) => {
trace_query_failure("list_replay_entity_summaries", started_at, &error);
return std::result::Result::Err(ks_core::Error::db(format!( return std::result::Result::Err(ks_core::Error::db(format!(
"postgres replay entity summary query failed: {error}" "postgres replay entity summary query failed: {error}"
))); )));
@@ -223,6 +231,7 @@ pub(crate) async fn list_replay_entity_summaries(
max_slot: row.max_slot, max_slot: row.max_slot,
}); });
} }
trace_query_success("list_replay_entity_summaries", started_at, output.len());
return std::result::Result::Ok(output); return std::result::Result::Ok(output);
} }
@@ -392,6 +401,16 @@ fn optional_sql_bigint(
}; };
} }
fn trace_query_success(action: &str, started_at: std::time::Instant, rows: usize) {
let elapsed_ms = started_at.elapsed().as_secs_f64() * 1000.0;
tracing::trace!(target: crate::TRACING_TARGET, backend = "postgres", domain = "ks-store.pg", action = action, elapsed_ms, rows, outcome = "success", "PostgreSQL store operation completed");
}
fn trace_query_failure(action: &str, started_at: std::time::Instant, error: &sqlx::Error) {
let elapsed_ms = started_at.elapsed().as_secs_f64() * 1000.0;
tracing::trace!(target: crate::TRACING_TARGET, backend = "postgres", domain = "ks-store.pg", action = action, elapsed_ms, error = %error, outcome = "error", "PostgreSQL store operation failed");
}
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
#[tokio::test] #[tokio::test]

View File

@@ -1,8 +1,9 @@
// file: ks-store/src/postgres/repository.rs // file: ks-store/src/postgres/repository.rs
// version: 2 // version: 3
//! PostgreSQL repository implementations. //! PostgreSQL repository implementations.
mod account_state_repository;
mod core_extraction_repository; mod core_extraction_repository;
mod core_transaction_repository; mod core_transaction_repository;
mod decode_pipeline_repository; mod decode_pipeline_repository;

View File

@@ -0,0 +1,52 @@
// file: ks-store/src/postgres/repository/account_state_repository.rs
// version: 1
//! PostgreSQL generic account observation to Core account-state repository.
#[async_trait::async_trait]
impl crate::AccountStateStore for crate::PostgresStore {
#[expect(
clippy::implicit_return,
reason = "async_trait expansion triggers implicit_return on generated async trait methods."
)]
async fn list_account_observations_for_normalization(
&self,
filter: &crate::AccountStateSelectionFilter,
) -> ks_core::Result<std::vec::Vec<crate::AccountObservationRow>> {
return crate::list_account_observations_for_normalization(self.pool(), filter).await;
}
#[expect(
clippy::implicit_return,
reason = "async_trait expansion triggers implicit_return on generated async trait methods."
)]
async fn is_account_state_current(
&self,
identity: &crate::ProcessingLedgerIdentity,
) -> ks_core::Result<bool> {
return crate::is_account_state_current(self.pool(), identity).await;
}
#[expect(
clippy::implicit_return,
reason = "async_trait expansion triggers implicit_return on generated async trait methods."
)]
async fn persist_account_state(
&self,
bundle: &crate::AccountStatePersistenceBundle,
force_replay: bool,
) -> ks_core::Result<crate::InsertOutcome> {
return crate::persist_account_state(self.pool(), bundle, force_replay).await;
}
#[expect(
clippy::implicit_return,
reason = "async_trait expansion triggers implicit_return on generated async trait methods."
)]
async fn mark_account_state_failed(
&self,
failure: &crate::AccountStateNormalizationFailure,
) -> ks_core::Result<crate::InsertOutcome> {
return crate::mark_account_state_failed(self.pool(), failure).await;
}
}

View File

@@ -1,5 +1,5 @@
// file: ks-store/src/postgres/repository/core_transaction_repository.rs // file: ks-store/src/postgres/repository/core_transaction_repository.rs
// version: 3 // version: 4
//! PostgreSQL core Solana repository implementation. //! PostgreSQL core Solana repository implementation.
@@ -79,7 +79,7 @@ impl crate::CoreTransactionStore for crate::PostgresStore {
&self, &self,
filter: &crate::CoreInstructionReplayFilter, filter: &crate::CoreInstructionReplayFilter,
page_request: &crate::PageRequest, page_request: &crate::PageRequest,
) -> ks_core::Result<std::vec::Vec<crate::CoreInstructionRow>> { ) -> ks_core::Result<crate::PageSlice<crate::CoreInstructionReplayRow>> {
return crate::list_core_instructions_for_replay(self.pool(), filter, page_request).await; return crate::list_core_instructions_for_replay(self.pool(), filter, page_request).await;
} }
@@ -91,7 +91,7 @@ impl crate::CoreTransactionStore for crate::PostgresStore {
&self, &self,
filter: &crate::CoreInstructionReplayFilter, filter: &crate::CoreInstructionReplayFilter,
page_request: &crate::PageRequest, page_request: &crate::PageRequest,
) -> ks_core::Result<std::vec::Vec<crate::MdCoreInstructionReplayInput>> { ) -> ks_core::Result<crate::PageSlice<crate::MdCoreInstructionReplayInput>> {
return crate::list_core_instruction_replay_inputs(self.pool(), filter, page_request).await; return crate::list_core_instruction_replay_inputs(self.pool(), filter, page_request).await;
} }

View File

@@ -1,5 +1,5 @@
// file: ks-store/src/postgres/repository/raw_transaction_repository.rs // file: ks-store/src/postgres/repository/raw_transaction_repository.rs
// version: 3 // version: 4
//! PostgreSQL canonical transaction and acquisition observation repository implementation. //! PostgreSQL canonical transaction and acquisition observation repository implementation.
@@ -27,6 +27,14 @@ impl crate::RawTransactionStore for crate::PostgresStore {
return crate::has_transaction_observation_key(self.pool(), observation_key).await; return crate::has_transaction_observation_key(self.pool(), observation_key).await;
} }
#[expect(
clippy::implicit_return,
reason = "async_trait expansion triggers implicit_return on generated async trait methods."
)]
async fn has_account_observation_key(&self, observation_key: &str) -> ks_core::Result<bool> {
return crate::has_account_observation_key(self.pool(), observation_key).await;
}
#[expect( #[expect(
clippy::implicit_return, clippy::implicit_return,
reason = "async_trait expansion triggers implicit_return on generated async trait methods." reason = "async_trait expansion triggers implicit_return on generated async trait methods."
@@ -49,6 +57,17 @@ impl crate::RawTransactionStore for crate::PostgresStore {
return crate::insert_transaction_observation(self.pool(), input).await; return crate::insert_transaction_observation(self.pool(), input).await;
} }
#[expect(
clippy::implicit_return,
reason = "async_trait expansion triggers implicit_return on generated async trait methods."
)]
async fn insert_account_observation(
&self,
input: &crate::AccountObservationInsert,
) -> ks_core::Result<crate::InsertOutcome> {
return crate::insert_account_observation(self.pool(), input).await;
}
#[expect( #[expect(
clippy::implicit_return, clippy::implicit_return,
reason = "async_trait expansion triggers implicit_return on generated async trait methods." reason = "async_trait expansion triggers implicit_return on generated async trait methods."

View File

@@ -1,5 +1,5 @@
// file: ks-store/src/store.rs // file: ks-store/src/store.rs
// version: 10 // version: 11
//! Backend-agnostic store facade and connection ownership. //! Backend-agnostic store facade and connection ownership.
@@ -531,6 +531,20 @@ impl crate::RawTransactionStore for crate::Store {
}; };
} }
#[expect(
clippy::implicit_return,
reason = "async_trait expansion triggers implicit_return on generated async trait methods."
)]
async fn has_account_observation_key(&self, observation_key: &str) -> ks_core::Result<bool> {
return match &self.backend {
StoreBackend::Disabled => std::result::Result::Err(disabled_store_error()),
StoreBackend::Postgres(store) => {
crate::RawTransactionStore::has_account_observation_key(store, observation_key)
.await
},
};
}
#[expect( #[expect(
clippy::implicit_return, clippy::implicit_return,
reason = "async_trait expansion triggers implicit_return on generated async trait methods." reason = "async_trait expansion triggers implicit_return on generated async trait methods."
@@ -563,6 +577,22 @@ impl crate::RawTransactionStore for crate::Store {
}; };
} }
#[expect(
clippy::implicit_return,
reason = "async_trait expansion triggers implicit_return on generated async trait methods."
)]
async fn insert_account_observation(
&self,
input: &crate::AccountObservationInsert,
) -> ks_core::Result<crate::InsertOutcome> {
return match &self.backend {
StoreBackend::Disabled => std::result::Result::Err(disabled_store_error()),
StoreBackend::Postgres(store) => {
crate::RawTransactionStore::insert_account_observation(store, input).await
},
};
}
#[expect( #[expect(
clippy::implicit_return, clippy::implicit_return,
reason = "async_trait expansion triggers implicit_return on generated async trait methods." reason = "async_trait expansion triggers implicit_return on generated async trait methods."
@@ -686,7 +716,7 @@ impl crate::CoreTransactionStore for crate::Store {
&self, &self,
filter: &crate::CoreInstructionReplayFilter, filter: &crate::CoreInstructionReplayFilter,
page_request: &crate::PageRequest, page_request: &crate::PageRequest,
) -> ks_core::Result<std::vec::Vec<crate::CoreInstructionRow>> { ) -> ks_core::Result<crate::PageSlice<crate::CoreInstructionReplayRow>> {
return match &self.backend { return match &self.backend {
StoreBackend::Disabled => std::result::Result::Err(disabled_store_error()), StoreBackend::Disabled => std::result::Result::Err(disabled_store_error()),
StoreBackend::Postgres(store) => { StoreBackend::Postgres(store) => {
@@ -708,7 +738,7 @@ impl crate::CoreTransactionStore for crate::Store {
&self, &self,
filter: &crate::CoreInstructionReplayFilter, filter: &crate::CoreInstructionReplayFilter,
page_request: &crate::PageRequest, page_request: &crate::PageRequest,
) -> ks_core::Result<std::vec::Vec<crate::MdCoreInstructionReplayInput>> { ) -> ks_core::Result<crate::PageSlice<crate::MdCoreInstructionReplayInput>> {
return match &self.backend { return match &self.backend {
StoreBackend::Disabled => std::result::Result::Err(disabled_store_error()), StoreBackend::Disabled => std::result::Result::Err(disabled_store_error()),
StoreBackend::Postgres(store) => { StoreBackend::Postgres(store) => {
@@ -739,6 +769,75 @@ impl crate::CoreTransactionStore for crate::Store {
} }
} }
#[async_trait::async_trait]
impl crate::AccountStateStore for crate::Store {
#[expect(
clippy::implicit_return,
reason = "async_trait expansion triggers implicit_return on generated async trait methods."
)]
async fn list_account_observations_for_normalization(
&self,
filter: &crate::AccountStateSelectionFilter,
) -> ks_core::Result<std::vec::Vec<crate::AccountObservationRow>> {
return match &self.backend {
StoreBackend::Disabled => std::result::Result::Err(disabled_store_error()),
StoreBackend::Postgres(store) => {
crate::AccountStateStore::list_account_observations_for_normalization(store, filter)
.await
},
};
}
#[expect(
clippy::implicit_return,
reason = "async_trait expansion triggers implicit_return on generated async trait methods."
)]
async fn is_account_state_current(
&self,
identity: &crate::ProcessingLedgerIdentity,
) -> ks_core::Result<bool> {
return match &self.backend {
StoreBackend::Disabled => std::result::Result::Err(disabled_store_error()),
StoreBackend::Postgres(store) => {
crate::AccountStateStore::is_account_state_current(store, identity).await
},
};
}
#[expect(
clippy::implicit_return,
reason = "async_trait expansion triggers implicit_return on generated async trait methods."
)]
async fn persist_account_state(
&self,
bundle: &crate::AccountStatePersistenceBundle,
force_replay: bool,
) -> ks_core::Result<crate::InsertOutcome> {
return match &self.backend {
StoreBackend::Disabled => std::result::Result::Err(disabled_store_error()),
StoreBackend::Postgres(store) => {
crate::AccountStateStore::persist_account_state(store, bundle, force_replay).await
},
};
}
#[expect(
clippy::implicit_return,
reason = "async_trait expansion triggers implicit_return on generated async trait methods."
)]
async fn mark_account_state_failed(
&self,
failure: &crate::AccountStateNormalizationFailure,
) -> ks_core::Result<crate::InsertOutcome> {
return match &self.backend {
StoreBackend::Disabled => std::result::Result::Err(disabled_store_error()),
StoreBackend::Postgres(store) => {
crate::AccountStateStore::mark_account_state_failed(store, failure).await
},
};
}
}
#[async_trait::async_trait] #[async_trait::async_trait]
impl crate::CoreExtractionStore for crate::Store { impl crate::CoreExtractionStore for crate::Store {
#[expect( #[expect(