From bfdb2e69ae0e082e658c8b3421adcdf800035161 Mon Sep 17 00:00:00 2001 From: SinuS Von SifriduS Date: Tue, 9 Jun 2026 10:13:03 +0200 Subject: [PATCH] 0.7.51 --- CHANGELOG.md | 4 +- Cargo.toml | 2 +- README.md | 76 +- ROADMAP.md | 29 +- docs/DB_EVENT_MODEL_REVIEW.md | 32 +- docs/DEX_DECODER_MATRIX.md | 20 +- docs/DEX_EVENT_COVERAGE_MATRIX.md | 27 +- docs/VALIDATION_STATUS_0_7_51.md | 42 + docs/VALIDATION_STATUS_0_7_51_FINAL.md | 47 ++ docs/VALIDATION_STATUS_0_7_51_MAX_DECODER.md | 56 ++ ...hadhroony-bobobot_0.7.51-raydium-amm-v4.md | 219 +++++- ...hadhroony-bobobot_0.7.52-raydium-stable.md | 626 +++++++++++++++ .../RAYDIUM_AMM_V4_EVENT_COVERAGE_REPORT.md | 141 ++++ docs/reports/RAYDIUM_POOL_V4_DECISION_NOTE.md | 56 ++ kb_demo_app/package.json | 2 +- kb_demo_app/tauri.conf.json | 2 +- kb_lib/src/db.rs | 14 + kb_lib/src/db/dtos.rs | 9 +- kb_lib/src/db/dtos/instruction_observation.rs | 37 +- kb_lib/src/db/dtos/launch_event.rs | 41 + ...ogram_instruction_discriminator_summary.rs | 29 + kb_lib/src/db/queries.rs | 13 + kb_lib/src/db/queries/dex_decoded_event.rs | 318 ++++++++ .../src/db/queries/instruction_observation.rs | 215 +++++ kb_lib/src/db/queries/launch_event.rs | 139 ++++ kb_lib/src/db/queries/pool_admin_event.rs | 29 + .../queries/program_instruction_diagnostic.rs | 37 +- kb_lib/src/dex.rs | 1 + kb_lib/src/dex/raydium_amm_v4.rs | 740 +++++++++++++++++- kb_lib/src/dex_decode.rs | 698 ++++++++++++----- kb_lib/src/dex_detection_route.rs | 11 +- kb_lib/src/dex_event_classification.rs | 77 ++ kb_lib/src/dex_event_coverage.rs | 100 +++ kb_lib/src/instruction_observation_index.rs | 249 ++---- kb_lib/src/lib.rs | 32 + kb_lib/src/local_pipeline_diagnostics.rs | 530 ++----------- kb_lib/src/local_pipeline_replay.rs | 24 +- kb_lib/src/non_trade_event_materialization.rs | 289 +++---- .../SQL_VALIDATION_RAYDIUM_AMM_V4_0_7_51.sql | 323 ++++++++ ...DIUM_AMM_V4_0_7_51_PRE2_REPLAY_CLEANUP.sql | 117 +++ ...7_51_PRE3_MATERIALIZATION_EXPLANATIONS.sql | 156 ++++ 41 files changed, 4485 insertions(+), 1124 deletions(-) create mode 100644 docs/VALIDATION_STATUS_0_7_51.md create mode 100644 docs/VALIDATION_STATUS_0_7_51_FINAL.md create mode 100644 docs/VALIDATION_STATUS_0_7_51_MAX_DECODER.md create mode 100644 docs/prompts/PROMPT_REPRISE_khadhroony-bobobot_0.7.52-raydium-stable.md create mode 100644 docs/reports/RAYDIUM_AMM_V4_EVENT_COVERAGE_REPORT.md create mode 100644 docs/reports/RAYDIUM_POOL_V4_DECISION_NOTE.md create mode 100644 kb_lib/src/db/dtos/launch_event.rs create mode 100644 kb_lib/src/db/queries/launch_event.rs create mode 100644 validation_sql/SQL_VALIDATION_RAYDIUM_AMM_V4_0_7_51.sql create mode 100644 validation_sql/SQL_VALIDATION_RAYDIUM_AMM_V4_0_7_51_PRE2_REPLAY_CLEANUP.sql create mode 100644 validation_sql/SQL_VALIDATION_RAYDIUM_AMM_V4_0_7_51_PRE3_MATERIALIZATION_EXPLANATIONS.sql diff --git a/CHANGELOG.md b/CHANGELOG.md index c35bfef..96863a5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -82,5 +82,5 @@ 0.7.49 - Raydium CLMM event coverage clôturé : 45 entrées listées, 33 instructions locales observées/décodées, 25 entrées matérialisées, ajout `k_sol_orderbook_events`, matérialisation des limit orders, liquidity, fees, rewards, admin/config et lifecycle prouvés par corpus, préparation audit-only des 11 Anchor Program-data events non observés, nettoyage des `raydium_clmm.instruction_audit` et `upstream_git.instruction_match` redondants, validation des invariants failed transaction / non-swap / trade-candle. 0.7.50-pre3 - Raydium Launchpad self-CPI/pool catalog correction : ajout du preset Demo3 `raydium_launchpad`, décodage direct des self-CPI Launchpad `trade_event` et `pool_create_event`, correction des indices `initialize*` (`pool_state=5`, `base_mint=6`, `quote_mint=7`) et routage des `initialize*` vers la matérialisation catalogue pool/pair Launchpad sans promotion trade/candle. 0.7.50 - Raydium Launchpad event coverage bootstrap : normalisation locale canonique vers `raydium_launchpad`, ajout de `RAYDIUM_LAUNCHPAD_PROGRAM_ID`, synchronisation des entrées Carbon Launchpad dans le registre upstream, fallback audit/mapped decoder pour discriminants Launchpad, enrichissement audit Anchor self-CPI, maintien conservatoire en `decoded_events_only`, rapport Launchpad et SQL de validation dédiés. -0.7.50-pre-r2 - Clôture CPMM/CLMM post-Launchpad : ajout des entrées Carbon `cpi_event` pour `raydium_cpmm` et `raydium_clmm`, ajout de `raydium_clmm.update_dynamic_fee_config`, normalisation des Program-data events CLMM, ajout de la table `k_sol_token_account_events` et de la matérialisation `create_support_mint_associated`, reclassement des familles ambiguës (`cpi_transport`, `liquidity_calculation`, `liquidity_change`, `position_open`, `pool_create`, `admin_config`, `account_create`, `idl_management`), codage du discriminant CPMM `40f4bc78a7e9690a` comme `raydium_cpmm.anchor_idl_instruction` decoded-only après inspection Solscan, et contexte de secours pour matérialisation liquidity CLMM via événements frères quand possible. -0.7.50-final - Clôture Raydium Launchpad et recheck Raydium : correction FK-safe du cleanup `raydium_cpmm.instruction_audit` pour le discriminant `40f4bc78a7e9690a`, conservation de `raydium_cpmm.anchor_idl_instruction` en decoded-only, suppression attendue des decoded events CPMM sans ligne coverage, documentation finale Launchpad/CPMM/CLMM et prompt de reprise `0.7.51 raydium_amm_v4`. +0.7.50-pre-r2 - Clôture CPMM/CLMM post-Launchpad : ajout des entrées Carbon `cpi_event` pour `raydium_cpmm` et `raydium_clmm`, ajout de `raydium_clmm.update_dynamic_fee_config`, normalisation des Program-data events CLMM, ajout de la table `k_sol_token_account_events` et de la matérialisation `create_support_mint_associated`, reclassement des familles ambiguës (`cpi_transport`, `liquidity_calculation`, `liquidity_change`, `position_open`, `pool_create`, `admin_config`, `account_create`, `idl_management`), codage du discriminant CPMM `40f4bc78a7e9690a` comme `raydium_cpmm.anchor_idl_instruction` decoded-only après inspection Solscan, et contexte de secours pour matérialisation liquidity CLMM via événements frères quand possible. +0.7.51 - Raydium AMM v4 event coverage clôturé : decoder maximal local pour tous les discriminants officiels AMM v4 `00..11`, spécialisation des swaps `swap_base_in/out` et `swap_base_in/out_v2`, suppression durable du `raydium_amm_v4.swap` legacy, index AMM v4 en discriminant 1 octet, matérialisation validée des swaps, liquidity, lifecycle, fees, admin/config et side effects orderbook, `pre_initialize` conservé comme lifecycle audit deprecated/partial, `simulate_info` decoded-only, reset replay renforcé par `protocol_name`, validation des invariants failed/non-swap/single-target/unexplained gaps et maintien de `raydium_pool_v4` en audit conditionnel sans decoder autonome. diff --git a/Cargo.toml b/Cargo.toml index 61620e9..42afbe8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,7 +8,7 @@ members = [ ] [workspace.package] -version = "0.7.50" +version = "0.7.51" edition = "2024" license = "MIT" repository = "https://git.sasedev.com/Sasedev/khadhroony-bobobot" diff --git a/README.md b/README.md index 65948ee..094b3f1 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,55 @@ # khadhroony-bobobot +## État final validé `0.7.51` — `raydium_amm_v4` + +La tranche `0.7.51 raydium_amm_v4` est clôturable côté `kb_lib` après validation locale du decoder maximal AMM v4. + +Points verrouillés : + +- `raydium_amm_v4` est le code canonique local ; +- program id canonique : `675kPX9MHTjS2zt1qfr1NYHuzeLXfQM9H24wFSUt1Mp8` ; +- tous les discriminants officiels AMM v4 `00..11` sont reconnus et observés localement ; +- les swaps sont spécialisés par discriminant : `swap_base_in`, `swap_base_out`, `swap_base_in_v2`, `swap_base_out_v2` ; +- le `event_kind` legacy `raydium_amm_v4.swap` est interdit et doit rester absent ; +- les discriminants AMM v4 sont indexés sur 1 octet, jamais comme discriminants Anchor 8 octets ; +- `pre_initialize` est conservé pour les scans historiques et matérialisé comme lifecycle audit minimal, sans création de pair exploitable ; +- `simulate_info` reste `decoded_events_only` ; +- `monitor_step`, `migrate_to_open_book` et `admin_cancel_orders` sont des side effects orderbook AMM v4 ; +- `raydium_pool_v4` reste une source d'audit/comparaison et ne devient pas un decoder autonome sans program id + corpus local. + +Validation locale finale rapportée : + +```text +cargo test -p kb_lib -> 405 passed / 0 failed +cargo clippy -p kb_lib --all-targets -- -D warnings -> OK +``` + +Dernier replay local : + +```text +195 replayed +0 decode skipped +195 ledger upserts +70 unsafe ledger rows +168 trades +7 liquidity +15 lifecycle +0 tokenAccount +668 candle upserts +instructionObservations = 2599 +resetDeleted = 1578 +catalog = 61 tokens / 65 pools / 65 pairs +``` + +Livrables `0.7.51` : + +- `docs/reports/RAYDIUM_AMM_V4_EVENT_COVERAGE_REPORT.md` ; +- `docs/reports/RAYDIUM_POOL_V4_DECISION_NOTE.md` ; +- `validation_sql/SQL_VALIDATION_RAYDIUM_AMM_V4_0_7_51.sql` ; +- `docs/VALIDATION_STATUS_0_7_51_FINAL.md`. + + `khadhroony-bobobot` est un workspace Rust destiné à la détection, au décodage, à l’analyse et, à terme, au trading semi-automatisé de tokens Solana. Ce document reflète le point de reprise `0.7.43-E5C` et l’état de consolidation atteint après `0.7.45` pour `meteora_dlmm`. La version Cargo a évolué ensuite à `0.7.46` côté workspace. Le lot Meteora initialement ouvert en bloc a été redécoupé : `meteora_dlmm` est traité séparément, puis la suite reprend `meteora_damm_v1`, `meteora_damm_v2` et `meteora_dbc` un par un. @@ -29,7 +78,7 @@ non-swap CLMM avec trade_count > 0 = 0 Les 11 Anchor / `Program data` events CLMM restent listés en `upstream_git_unverified` car aucun corpus local ne les observe encore. Le code est préparé pour les accueillir en audit-only lorsqu’ils apparaîtront dans un corpus local, sans créer de trade/candle par défaut. -La tranche fonctionnelle ouverte est `0.7.50-pre-r2`, dédiée à la clôture Raydium Launchpad puis à la re-vérification CPMM/CLMM, avant `0.7.51 raydium_amm_v4` et `0.7.52 raydium_stable`. `raydium_pool_v4` reste un audit conditionnel `0.7.53` et ne doit pas être promu sans confirmation de program id/rôle/corpus. +La tranche `0.7.51 raydium_amm_v4` est maintenant validée côté `kb_lib`. La suite de roadmap reprend avec `0.7.52 raydium_stable`, tandis que `raydium_pool_v4` reste un audit conditionnel ultérieur et ne doit pas être promu sans confirmation de program id/rôle/corpus. ## Organisation documentaire @@ -582,27 +631,4 @@ Cette tranche complète la clôture Raydium en ajoutant `cpi_event` pour CPMM/CL Le discriminant CPMM `40f4bc78a7e9690a` est désormais codé comme `raydium_cpmm.anchor_idl_instruction` : les signatures inspectées correspondent aux instructions Anchor `IdlCreateAccount` / `IdlCloseAccount`, donc il reste `decoded_events_only` et ne matérialise aucune table métier. -Rapport de clôture : `docs/reports/RAYDIUM_CPMM_CLMM_RECHECK_REPORT_0_7_50_PRE_R2.md`. - -## Note 0.7.50-final — Launchpad closure and Raydium recheck cleanup - -The final `0.7.50` cleanup keeps the Raydium CPMM discriminator `40f4bc78a7e9690a` as `raydium_cpmm.anchor_idl_instruction` decoded-only and removes stale `raydium_cpmm.instruction_audit` duplicates in an FK-safe way by unlinking `k_sol_instruction_observations.decoded_event_id` before deletion. - -Expected post-replay checks: - -```text -raydium_cpmm.instruction_audit = 0 -raydium_cpmm decoded events missing coverage row = 0 -``` - -Validation helper: - -```text -validation_sql/SQL_VALIDATION_RAYDIUM_CPMM_AUDIT_CLEANUP_0_7_50_FINAL.sql -``` - -Next-session handoff: - -```text -docs/prompts/PROMPT_REPRISE_khadhroony-bobobot_0.7.51-raydium-amm-v4.md -``` +Rapport de clôture : `docs/reports/RAYDIUM_CPMM_CLMM_RECHECK_REPORT_0_7_50_PRE_R2.md`. \ No newline at end of file diff --git a/ROADMAP.md b/ROADMAP.md index 251f47d..6f130ae 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -35,7 +35,7 @@ Règles de planification : | `0.7.48` | `raydium_cpmm` | Clôturé : instructions/events CPMM, lifecycle, fees, admin/config, deposit/withdraw, `lp_change_event`, invariants trade/candle. | | `0.7.49` | `raydium_clmm` | Clôturé : 33 instructions observées/décodées, orderbook CLMM, liquidity/fee/reward/admin/lifecycle, fallbacks upstream nettoyés, 11 Program-data events préparés mais non observés. | | `0.7.50` | `raydium_launchpad` | Bootstrap ouvert : surface LaunchLab/Launchpad, discriminants Carbon/IDL, fallback audit, SQL de validation, aucune matérialisation métier sans corpus. | -| `0.7.51` | `raydium_amm_v4` | Reprendre AMM v4 legacy au même niveau de couverture que CPMM/CLMM : swaps, pool lifecycle, liquidity, fees/admin, side effects documentés. | +| `0.7.51` | `raydium_amm_v4` | Clôturé : decoder maximal `00..11`, swaps spécialisés, lifecycle/liquidity/fees/admin/orderbook, `pre_initialize` audit, `simulate_info` decoded-only, cleanup legacy/fallback. | | `0.7.52` | `raydium_stable` | Reprendre Raydium Stable : program ids/IDL, swaps stables, pool lifecycle, liquidity, fees/admin, invariants pricing/candles. | | `0.7.53` | `raydium_pool_v4` | Audit / program-id decision seulement : confirmer program id, rôle exact et corpus avant toute promotion métier. | | `0.7.54` | `pump_swap` | Couvrir `buy/sell` et tous les events auxiliaires disponibles : fees, cashback, volume accumulator, admin/config. | @@ -1308,7 +1308,7 @@ Rapport associé : `docs/reports/RAYDIUM_CPMM_CLMM_RECHECK_REPORT_0_7_50_PRE_R2. ### 6.083. Version `0.7.51` — `raydium_amm_v4` event coverage Objectif : hisser AMM v4 legacy au niveau de couverture CPMM/CLMM. -À faire : revisiter swaps, initialize/pool lifecycle, add/remove liquidity, fees/admin/config, side effects SPL, failed transaction safety, fallback upstream et validation SQL dédiée. +Réalisé : decoder maximal AMM v4 pour tous les discriminants officiels `00..11`, spécialisation des swaps (`swap_base_in`, `swap_base_out`, `swap_base_in_v2`, `swap_base_out_v2`), suppression du legacy `raydium_amm_v4.swap`, observation locale de tous les discriminants, matérialisation validée des familles trade, liquidity, lifecycle, fee, admin/config et orderbook, `pre_initialize` conservé comme lifecycle audit deprecated/partial, `simulate_info` conservé en decoded-only, gaps successful non matérialisés expliqués, et validation des invariants failed/non-swap/single-target. ### 6.084. Version `0.7.52` — `raydium_stable` event coverage Objectif : reprendre Raydium Stable comme troisième tranche Raydium post-CLMM. @@ -1651,6 +1651,27 @@ La tranche CPMM reconnaît désormais tous les discriminants instruction-level l La suite après `0.7.49 raydium_clmm` reprend en `0.7.50-pre-r2` par la clôture Launchpad et la re-vérification CPMM/CLMM, puis `0.7.51 raydium_amm_v4`, `0.7.52 raydium_stable` et `0.7.53 raydium_pool_v4` uniquement comme audit conditionnel, en gardant la même discipline : sources Git/IDL + Solscan pour accélérer la découverte, mais corpus local obligatoire avant toute promotion métier. -### Note `0.7.50-final` — clôture Launchpad + recheck Raydium +## Clôture `0.7.51` — Raydium AMM v4 -`0.7.50` se clôture avec `raydium_launchpad` et la re-vérification CPMM/CLMM. Le dernier correctif cible le cleanup FK-safe des anciens `raydium_cpmm.instruction_audit` `40f4bc78a7e9690a`, maintenant remplacés par `raydium_cpmm.anchor_idl_instruction` decoded-only. La suite planifiée reste `0.7.51 raydium_amm_v4`, puis `0.7.52 raydium_stable`, avec découverte accélérée par Solscan `instruction=` et validation obligatoire par corpus local. +La tranche `0.7.51 raydium_amm_v4` est clôturable côté code et corpus local AMM v4. + +Résultats de validation : + +- `cargo test -p kb_lib` : `405 passed`, `0 failed` ; +- `cargo clippy -p kb_lib --all-targets -- -D warnings` : OK ; +- dernier replay : `195 replayed`, `0 decode skipped`, `168 trades`, `7 liquidity`, `15 lifecycle`, `668 candle upserts`, `instructionObservations=2599`, `resetDeleted=1578` ; +- tous les discriminants AMM v4 `00..11` sont observés localement ; +- `raydium_amm_v4.swap` legacy, decoded sans coverage, instruction observations 8 octets, non-swap trade, failed tx trade et multi-target materialization sont vides ; +- `pre_initialize` est matérialisé comme lifecycle audit minimal pour les transactions successful, sans création de pair exploitable ; +- `migrate_to_open_book` est orderbook-only ; +- `simulate_info` reste decoded-only. + +Décision `raydium_pool_v4` : ne pas ouvrir de decoder autonome dans cette tranche. La roadmap conserve uniquement une entrée conditionnelle : + +- même program id/layout compatible AMM v4 : intégrer dans `raydium_amm_v4` ; +- autre program id/surface strategy/wrapper/farm/lending : future tranche dédiée après corpus ; +- IDL ambiguë sans corpus : rester en audit roadmap. + +Le rapport de décision est `docs/reports/RAYDIUM_POOL_V4_DECISION_NOTE.md`. + +La suite reprend `0.7.52 raydium_stable`, après rechecks CPMM/CLMM/Launchpad si la base de validation les contient. diff --git a/docs/DB_EVENT_MODEL_REVIEW.md b/docs/DB_EVENT_MODEL_REVIEW.md index 3a2bbaf..b4292c9 100644 --- a/docs/DB_EVENT_MODEL_REVIEW.md +++ b/docs/DB_EVENT_MODEL_REVIEW.md @@ -338,10 +338,34 @@ La re-vérification Raydium CLMM introduit une table dédiée `k_sol_token_accou Cette table permet de suivre les événements Token-2022/ATA significatifs sans les confondre avec les trades ou les liquidités. -## Note `0.7.50-final` — FK-safe cleanup for instruction observations +## Note `0.7.51` — impact AMM v4 sur le modèle DB -`k_sol_instruction_observations` is a technical index table. When a legacy `*.instruction_audit` decoded event is replaced by a local specialized event, existing observation rows can still point to the old decoded event id on already-created SQLite databases. +Aucune nouvelle table n'est ajoutée pour `raydium_amm_v4`. -The final 0.7.50 cleanup therefore unlinks `k_sol_instruction_observations.decoded_event_id` before deleting replaced CPMM instruction-audit rows. New databases define the `decoded_event_id` foreign key with `ON DELETE SET NULL` for the same reason. +Décisions DB maintenues : -This is not a business-table promotion. It only keeps the technical observation index consistent with decoded event cleanup. +- `k_sol_instruction_observations` reste une table technique d'indexation instruction/discriminant ; +- AMM v4 utilise des discriminants d'un octet, donc l'index technique doit conserver `09`, `0b`, `10`, `11`, etc. sans les convertir en discriminants Anchor huit octets ; +- le refresh de `k_sol_instruction_observations` reconstruit les observations par transaction avant upsert, afin de supprimer les restes historiques en huit octets après changement de stratégie d'indexation ; +- les side effects SPL Token / Token-2022 restent transversaux et ne doivent pas être promus comme events directs `raydium_amm_v4.*` ; +- les side effects Serum/OpenBook de AMM v4 doivent être documentés comme contexte orderbook, pas comme trades OpenBook autonomes ; +- `raydium_pool_v4` ne justifie aucune table ni aucun decoder séparé sans corpus local. + +Le modèle actuel suffit pour ouvrir la tranche : decoded events + coverage entries + instruction observations + matérialisations existantes trade/liquidity/lifecycle/fee/admin/orderbook. Une extension future orderbook/vault/token-account ne doit être ajoutée qu'après preuves multi-DEX. + + + +### Clôture `0.7.51` — AMM v4 et modèle DB + +La validation finale AMM v4 confirme qu'aucune nouvelle table n'est requise pour `raydium_amm_v4`. + +Règles validées : + +- chaque decoded event AMM v4 matérialisé cible au plus une table métier principale ; +- `migrate_to_open_book`, `monitor_step` et `admin_cancel_orders` alimentent `k_sol_orderbook_events` uniquement ; +- `pre_initialize` alimente `k_sol_pool_lifecycle_events` comme audit deprecated/partial, sans création de paire exploitable ; +- `simulate_info` reste dans `k_sol_dex_decoded_events` uniquement ; +- les `deposit` / `withdraw` sans pool/pair catalogue ou sans deltas exploitables restent decoded-only expliqués ; +- les side effects SPL Token / Token-2022 restent transversaux. + +Contrôle final AMM v4 : le SQL `materialized_target_count > 1` doit rester vide. diff --git a/docs/DEX_DECODER_MATRIX.md b/docs/DEX_DECODER_MATRIX.md index 319f1fa..7d9d6c9 100644 --- a/docs/DEX_DECODER_MATRIX.md +++ b/docs/DEX_DECODER_MATRIX.md @@ -31,7 +31,7 @@ Cette matrice complète `kb_lib/src/dex_support_matrix.rs`. Elle documente **ce | 1 | `raydium_cpmm` | `supported / 0.7.50-pre-r2 closure recheck` | Couverture CPMM clôturée : swaps, lifecycle, fees, admin/config, deposit/withdraw, `lp_change_event`, `swap_event` decoded-only, `cpi_event` transport Carbon et `anchor_idl_instruction` Solscan/manual pour `40f4bc78a7e9690a`. | Ne pas promouvoir `anchor_idl_instruction` : c'est de la gestion Anchor IDL, pas un événement AMM métier. | | 2 | `raydium_clmm` | `supported / 0.7.50-pre-r2 closure recheck` | Couverture CLMM complétée : `cpi_event`, `update_dynamic_fee_config`, Program-data events locaux, `create_support_mint_associated` vers `k_sol_token_account_events`, familles sans `unknown`, router/swap Program-data en decoded-only. | Rejouer la base CLMM et confirmer que les seuls résidus sont `decoded_events_only`, transactions failed ou absence prouvée de contexte pool/pair. | | 3 | `raydium_launchpad` | `bootstrap / 0.7.50` | Surface canonique normalisée, 1 entrée programme + 26 discriminants Carbon/IDL listés, fallback audit/mapped decoder, SQL dédié. | Créer DB neuve, backfill par discriminant, replay forcé, promouvoir seulement après corpus local. | -| 4 | `raydium_amm_v4` | `supported / 0.7.51 planned` | Swaps AMM v4 legacy matérialisés. | Reprendre AMM v4 au niveau CPMM/CLMM : pool lifecycle, liquidity, fees/admin, side effects, fallback cleanup. | +| 4 | `raydium_amm_v4` | `supported / 0.7.51 closed` | Decoder maximal AMM v4 `00..11`, swaps spécialisés, lifecycle/liquidity/fees/admin/orderbook validés. | Rechecks CPMM/CLMM/Launchpad puis `raydium_stable`. | | 5 | `raydium_stable_swap` | `planned / 0.7.52` | Entrée conservée. | Reprendre Stable séparément : swaps stables, pool lifecycle, liquidity, fees/admin, montants/prix exploitables. | | 6 | `raydium_pool_v4` | `to_verify / 0.7.53 conditional audit` | IDL annexe mentionnée par fnzero, non présente dans l'archive locale, pas de program id/rôle confirmé ici. | Ne pas promouvoir tant que program id distinct, rôle exact et corpus exploitable ne sont pas confirmés. | | 7 | `pump_swap` | `supported / 0.7.54 planned` | `buy`/`sell` décodés et matérialisés ; trade/candle OK. | Ajouter tous les events Carbon/Solana Streamer : cashback, fee, volume accumulator, admin/config ; conserver les non-trades hors candles. | @@ -249,6 +249,20 @@ La clôture `0.7.50-pre-r2` complète les tranches `0.7.48` et `0.7.49` sans rou - Les Program-data events CLMM reçoivent des `local_event_kind` et familles explicites. - `create_support_mint_associated` introduit une cible métier spécialisée : `k_sol_token_account_events`. -## Note `0.7.50-final` — Raydium CPMM post-Launchpad recheck +## Note `0.7.51` — `raydium_amm_v4` + +| Champ | Décision `0.7.51` | +|---|---| +| Code local | `raydium_amm_v4` | +| Program id canonique | `675kPX9MHTjS2zt1qfr1NYHuzeLXfQM9H24wFSUt1Mp8` | +| Statut | `supported / 0.7.51 closed` ; max-decoder local `00..11` validé | +| Sources principales | Carbon `raydium-amm-v4-decoder`, Pinax `src/raydium/amm`, fnzero `raydium_amm_v4.json`, Solscan Program IDL | +| Swaps | `swap_base_in`, `swap_base_out`, `swap_base_in_v2`, `swap_base_out_v2` | +| Pool lifecycle | `initialize`, `initialize2`, `pre_initialize` | +| Liquidity | `deposit`, `withdraw` | +| Fees/admin/orderbook side effects | `withdraw_pnl`, `withdraw_srm`, `set_params`, `monitor_step`, `admin_cancel_orders`, `migrate_to_open_book`, config account ops | +| SPL Token / Token-2022 side effects | transversaux, non promus comme `raydium_amm_v4.*` directs | +| `raydium_pool_v4` | audit comparatif uniquement ; pas de decoder autonome sans program id et corpus local | + +La tranche a été validée sur base SQLite dédiée : tous les discriminants `00..11` sont observés localement. Les gaps de matérialisation restants sont expliqués par decoded-only, transaction failed ou absence de catalogue/deltas exploitables. -`raydium_cpmm` remains closed. The final post-Launchpad cleanup removes the three legacy `raydium_cpmm.instruction_audit` rows for discriminator `40f4bc78a7e9690a` after they have been replaced by the local decoded-only `raydium_cpmm.anchor_idl_instruction` entry. This does not promote the discriminator to a trade, liquidity, fee, admin or lifecycle event. diff --git a/docs/DEX_EVENT_COVERAGE_MATRIX.md b/docs/DEX_EVENT_COVERAGE_MATRIX.md index c32a8cc..ebd10ca 100644 --- a/docs/DEX_EVENT_COVERAGE_MATRIX.md +++ b/docs/DEX_EVENT_COVERAGE_MATRIX.md @@ -170,19 +170,22 @@ Validation attendue après replay : aucune entrée CPMM/CLMM ne doit rester en ` Rapport associé : `docs/reports/RAYDIUM_CPMM_CLMM_RECHECK_REPORT_0_7_50_PRE_R2.md`. -## Note `0.7.50-final` — CPMM audit cleanup after Launchpad recheck +## `0.7.51` — `raydium_amm_v4` -The `raydium_cpmm` recheck identified a legacy residual `raydium_cpmm.instruction_audit` row for discriminator `40f4bc78a7e9690a`. The discriminator is locally mapped as `raydium_cpmm.anchor_idl_instruction` and belongs to Anchor IDL management, not to AMM business activity. +Sources inventoriées : Carbon `raydium-amm-v4-decoder`, Pinax `src/raydium/amm`, fnzero `raydium_amm_v4.json`, Solscan Program IDL. `raydium_pool_v4` est comparé mais non promu. -Final rule: +Validation locale finale : tous les discriminants AMM v4 officiels `00..11` sont observés ; `instruction_audit`, fallback upstream, decoded sans coverage, observations 8 octets, non-swap trade, failed tx trade, gaps inexpliqués et multi-target materialization sont vides. -| Decoder | Entry | Family | DB target | Decision | +| Famille | Entrées AMM v4 | Statut `0.7.51-final` | Cible DB | Règle | |---|---|---|---|---| -| `raydium_cpmm` | `anchor_idl_instruction` / `40f4bc78a7e9690a` | `idl_management` | `k_sol_dex_decoded_events_only` | Keep decoded-only; remove legacy `raydium_cpmm.instruction_audit` duplicates after replay/coverage refresh. | - -Expected residual checks after final replay: - -```text -raydium_cpmm.instruction_audit = 0 -raydium_cpmm decoded events missing coverage row = 0 -``` +| `swap` | `swap_base_in`, `swap_base_out`, `swap_base_in_v2`, `swap_base_out_v2` | observed/materialized partiel expliqué | `k_sol_trade_events` | Trade/candle seulement si tx successful + montants vault fiables ; sinon `skipTradeReason`. | +| `pool_create` | `initialize`, `initialize2_pool` | observed/materialized | `k_sol_pool_lifecycle_events` | Alimente lifecycle et catalogue seulement quand les mints/pool sont prouvés. | +| `pool_create` deprecated | `pre_initialize` | observed/materialized audit minimal | `k_sol_pool_lifecycle_events` | Lifecycle deprecated/partial ; ne crée pas de pair exploitable sans mints. | +| `liquidity_add` | `deposit` | observed/materialized partiel expliqué | `k_sol_liquidity_events` | Jamais trade/candle ; pools absents du catalogue restent decoded-only expliqués. | +| `liquidity_remove` | `withdraw` | observed/materialized partiel expliqué | `k_sol_liquidity_events` | Jamais trade/candle ; deltas/catalogue manquants doivent être explicités. | +| `fee` | `withdraw_pnl`, `withdraw_srm` | observed/materialized partiel expliqué | `k_sol_fee_events` | Jamais trade/candle. | +| `admin/config` | `set_params`, `create_config_account`, `update_config_account` | observed/materialized | `k_sol_pool_admin_events` | Preuve métier par corpus uniquement. | +| `orderbook side effects` | `monitor_step`, `migrate_to_open_book`, `admin_cancel_orders` | observed/materialized | `k_sol_orderbook_events` | Side effects OpenBook/Serum ; pas de trade OpenBook autonome, pas de lifecycle en double. | +| `cpi/informational` | `simulate_info` | observed decoded-only | `k_sol_dex_decoded_events_only` | Audit technique uniquement. | +| `token side effects` | SPL Token / Token-2022 inner instructions | transversal | decoded-only actuellement | Ne pas promouvoir comme AMM v4 direct. | +| `unknown/unmapped audit` | residual `raydium_amm_v4.instruction_audit` | vide | decoded-only si futur inconnu | Tout residual doit être expliqué avant promotion. | diff --git a/docs/VALIDATION_STATUS_0_7_51.md b/docs/VALIDATION_STATUS_0_7_51.md new file mode 100644 index 0000000..6b3cac4 --- /dev/null +++ b/docs/VALIDATION_STATUS_0_7_51.md @@ -0,0 +1,42 @@ + + +# Validation status — `0.7.51 raydium_amm_v4` + +## Commandes demandées + +```bash +cargo fmt +cargo test -p kb_lib +cargo clippy -p kb_lib --all-targets -- -D warnings +``` + +## Résultat dans le sandbox + +Non exécuté : `cargo`, `rustc` et `rustfmt` ne sont pas disponibles dans l'environnement de génération. + +```text +cargo fmt -> cargo: command not found +``` + +## Contrôles statiques effectués + +- Extraction et modification de l'archive `0.7.50-raydium-launchpad-final`. +- Vérification de l'équilibre basique `{}` et `()` sur les fichiers Rust modifiés. +- Vérification des occurrences ajoutées : aucun `unwrap` / `expect` ajouté dans les nouveaux blocs AMM v4. +- Création des livrables docs + SQL demandés. + +## Validation locale requise + +Appliquer le delta, puis exécuter localement : + +```bash +cargo fmt +cargo test -p kb_lib +cargo clippy -p kb_lib --all-targets -- -D warnings +``` + +Ensuite créer une base SQLite vide dédiée `0.7.51`, constituer le corpus Demo3/Demo2 AMM v4, replay avec `forceDexDecode=yes`, puis exécuter : + +```text +validation_sql/SQL_VALIDATION_RAYDIUM_AMM_V4_0_7_51.sql +``` diff --git a/docs/VALIDATION_STATUS_0_7_51_FINAL.md b/docs/VALIDATION_STATUS_0_7_51_FINAL.md new file mode 100644 index 0000000..5314465 --- /dev/null +++ b/docs/VALIDATION_STATUS_0_7_51_FINAL.md @@ -0,0 +1,47 @@ + + +# Validation Status — `0.7.51 raydium_amm_v4 final` + +## Rust + +```text +cargo test -p kb_lib -> 405 passed / 0 failed +cargo clippy -p kb_lib --all-targets -- -D warnings -> OK +``` + +## Replay final + +```text +195 replayed +0 decode skipped +195 ledger upserts +70 unsafe ledger rows +168 trades +7 liquidity +15 lifecycle +0 tokenAccount +668 candle upserts +instructionObservations = 2599 +resetDeleted = 1578 +catalog = 61 tokens / 65 pools / 65 pairs +``` + +## SQL blocking checks + +Résultat attendu et rapporté : `vide` pour les contrôles suivants. + +- `raydium_amm_v4.swap` legacy ; +- decoded events AMM v4 sans coverage entry ; +- observations AMM v4 en discriminant plus long qu'un octet ; +- non-swap AMM v4 avec trade ; +- failed tx AMM v4 avec trade ; +- successful non-materialized AMM v4 sans raison explicite ; +- matérialisation multi-target AMM v4. + +## Points validés + +- Tous les discriminants officiels AMM v4 `00..11` sont observés localement. +- `pre_initialize` : `decoded_success_count=7`, `lifecycle_count=7`. +- `migrate_to_open_book` : orderbook-only (`lifecycle_count=0`, `orderbook_count=6`). +- `simulate_info` : decoded-only. +- `raydium_pool_v4` : audit-only / décision conditionnelle, sans decoder local. diff --git a/docs/VALIDATION_STATUS_0_7_51_MAX_DECODER.md b/docs/VALIDATION_STATUS_0_7_51_MAX_DECODER.md new file mode 100644 index 0000000..a40e933 --- /dev/null +++ b/docs/VALIDATION_STATUS_0_7_51_MAX_DECODER.md @@ -0,0 +1,56 @@ +# file: VALIDATION_STATUS_0_7_51_MAX_DECODER.md + +# Validation status — `0.7.51 raydium_amm_v4 max-decoder` + +## Scope + +Delta incrémental après le premier patch `0.7.51 raydium_amm_v4`. + +Objectifs couverts : + +- correction du test `swap_base_in` avec une payload `0x09` ; +- ajout d'un test dédié `swap_base_in_v2` avec une payload `0x10` ; +- suppression de la route métier legacy `raydium_amm_v4.swap` ; +- reconnaissance locale maximale des discriminants AMM v4 officiels `00..11` ; +- conservation des instructions dépréciées comme events decoded-only ou non-trade matérialisables si corpus successful ; +- reclassement orderbook/fee/admin/liquidity/lifecycle pour les non-swaps AMM v4 ; +- reconstruction des observations techniques par transaction avant upsert ; +- extension du SQL de validation AMM v4. + +## Sandbox validation + +Non exécutée dans l'environnement de génération : `cargo`, `rustc` et `rustfmt` ne sont pas disponibles. + +## Validation locale obligatoire + +```bash +cargo fmt +cargo test -p kb_lib +cargo clippy -p kb_lib --all-targets -- -D warnings +``` + +## Replay local attendu + +Après application du delta, relancer le replay dédié `0.7.51` avec : + +```text +skipDexDecode = no +forceDexDecode = yes +deferInstructionObservations = yes +``` + +Puis exécuter : + +```text +validation_sql/SQL_VALIDATION_RAYDIUM_AMM_V4_0_7_51.sql +``` + +Les contrôles bloquants attendus vides sont : + +- `raydium_amm_v4.instruction_audit` résiduel ; +- `upstream_git.instruction_match` localement couvert ; +- `raydium_amm_v4.swap` legacy ; +- decoded AMM v4 sans coverage entry ; +- observations AMM v4 avec `length(discriminator_hex) > 2` ; +- non-swap AMM v4 avec `trade_count > 0` ; +- failed tx matérialisée en trade. diff --git a/docs/prompts/PROMPT_REPRISE_khadhroony-bobobot_0.7.51-raydium-amm-v4.md b/docs/prompts/PROMPT_REPRISE_khadhroony-bobobot_0.7.51-raydium-amm-v4.md index ab999bb..d1f7285 100644 --- a/docs/prompts/PROMPT_REPRISE_khadhroony-bobobot_0.7.51-raydium-amm-v4.md +++ b/docs/prompts/PROMPT_REPRISE_khadhroony-bobobot_0.7.51-raydium-amm-v4.md @@ -8,7 +8,9 @@ Reprise du projet `khadhroony-bobobot` après clôture de `0.7.50 raydium_launch Utiliser la dernière archive complète du workspace intégrant les deltas validés jusqu'à : +```text 0.7.50-raydium-launchpad-final +``` Joindre aussi les docs et SQL de validation à jour : @@ -20,6 +22,7 @@ docs/DEX_DECODER_MATRIX.md docs/DEX_EVENT_COVERAGE_MATRIX.md docs/DB_EVENT_MODEL_REVIEW.md docs/reports/RAYDIUM_LAUNCHPAD_EVENT_COVERAGE_REPORT.md +docs/reports/RAYDIUM_CPMM_EVENT_COVERAGE_REPORT.md validation_sql/SQL_VALIDATION_RAYDIUM_LAUNCHPAD_0_7_50.sql validation_sql/SQL_VALIDATION_RAYDIUM_CPMM_AUDIT_CLEANUP_0_7_50_FINAL.sql validation_sql/SQL_VALIDATION_RAYDIUM_CPMM_0_7_50_RECHECK.sql @@ -30,20 +33,21 @@ validation_sql/SQL_VALIDATION_RAYDIUM_CLMM_0_7_50_RECHECK.sql `0.7.50` a clôturé `raydium_launchpad` et consolidé les rechecks CPMM/CLMM. -Dernier replay local rapporté avant clôture : +Dernier replay local rapporté après cleanup final CPMM : ```text -1103 replayed +1124 replayed 0 decode skipped 1124 ledger upserts -542 unsafe ledger rows +539 unsafe ledger rows 561 trades 50 liquidity 13 lifecycle 0 tokenAccount 2224 candle upserts -instructionObservations = 7013 +instructionObservations = 7010 resetDeleted = 1182 +catalog = 37 tokens / 40 pools / 40 pairs ``` Points de clôture à préserver : @@ -53,13 +57,16 @@ raydium_launchpad : surface canonique, program id LanMV9sAd7wArD4vJFi2qDdfnVhFxY Launchpad trade_event matérialisé seulement quand corpus + successful tx le prouvent Launchpad initialize* fournit le catalogue pool/pair, pas de faux trade/candle CPMM 40f4bc78a7e9690a est raydium_cpmm.anchor_idl_instruction decoded-only -CPMM residual raydium_cpmm.instruction_audit 40f4bc78a7e9690a doit être nettoyé après replay final +CPMM residual raydium_cpmm.instruction_audit 40f4bc78a7e9690a = vide après replay final +CPMM decoded event without coverage entry = vide après replay final +CPMM upstream_git.instruction_match fallback résiduel = vide +CPMM non-swap materialization gap hors failed tx = vide CLMM residual instruction_audit / upstream fallback doivent rester vides k_sol_instruction_observations reste une table technique, pas une table métier Solscan instruction= est une aide de découverte, pas une preuve métier ``` -Requête CPMM post-fix obligatoire : +Requêtes CPMM post-fix obligatoires avant d'ouvrir `0.7.51` : ```sql SELECT @@ -73,7 +80,38 @@ GROUP BY discriminator_hex ORDER BY audit_count DESC, discriminator_hex; ``` -Cette requête doit être vide après replay `forceDexDecode=yes`. +```sql +SELECT + de.event_kind, + COUNT(*) AS decoded_count +FROM k_sol_dex_decoded_events de +LEFT JOIN k_sol_dex_event_coverage_entries ce + ON ce.decoder_code = 'raydium_cpmm' + AND ce.local_event_kind = de.event_kind +WHERE de.protocol_name = 'raydium_cpmm' + AND ce.id IS NULL +GROUP BY de.event_kind +ORDER BY decoded_count DESC, de.event_kind; +``` + +Ces deux requêtes doivent être vides après replay `forceDexDecode=yes`. + +## Décision de reprise + +Ouvrir une nouvelle tranche : + +```text +0.7.51 raydium_amm_v4 +``` + +Ne pas commencer par `raydium_pool_v4` comme nouveau decoder autonome tant que son program id et son rôle métier ne sont pas prouvés localement. + +`raydium_pool_v4` doit être traité dans `0.7.51` comme une **source à auditer / comparer** avec `raydium_amm_v4`, pas comme version déjà décidée. La roadmap peut conserver une entrée de décision `raydium_pool_v4 audit / program-id decision`, mais cette entrée doit être reformulée comme décision conditionnelle : + +```text +si raydium_pool_v4 correspond au même program id AMM v4 ou à un layout alternatif -> intégrer à raydium_amm_v4 +si raydium_pool_v4 correspond à un autre program id / strategy / farm / pool wrapper -> créer une tranche dédiée seulement après corpus local +``` ## Objectif `0.7.51` — `raydium_amm_v4` @@ -84,6 +122,7 @@ swaps pool lifecycle / pool_create add_liquidity / remove_liquidity fees / admin/config +open_orders / target_orders / serum/openbook side effects documentés side effects SPL Token / Token-2022 documentés mais non promus comme raydium_amm_v4.* directs fallback instruction_audit nettoyé quand une entrée locale spécialisée couvre l'instruction coverage entries synchronisées et rafraîchies @@ -108,18 +147,75 @@ https://solscan.io/account/675kPX9MHTjS2zt1qfr1NYHuzeLXfQM9H24wFSUt1Mp8#programI https://solscan.io/account/675kPX9MHTjS2zt1qfr1NYHuzeLXfQM9H24wFSUt1Mp8?instruction=&hide_spam=true&hide_failed=true&show_related=false&sort=desc ``` +## Nouvelle base de travail + +Démarrer `0.7.51` sur une base SQLite vide dédiée. + +Avant le replay de validation complet, prévoir un corpus initial construit volontairement : + +```text +1. Demo3 program_id = 675kPX9MHTjS2zt1qfr1NYHuzeLXfQM9H24wFSUt1Mp8 +2. Solscan Program IDL + instruction= +3. backfill Demo2 de signatures contenant des instructions AMM v4 variées +4. backfill de pools AMM v4 quand Demo3/Solscan fournit un AMM/pool account fiable +``` + +Ne pas interpréter l'absence de résultat Solscan comme absence on-chain définitive. + ## Sources Git/IDL à utiliser systématiquement +Sources globales : + ```text https://github.com/sevenlabs-hq/carbon/tree/main/decoders https://github.com/0xfnzero/solana-streamer https://github.com/0xfnzero/sol-parser-sdk/tree/main/idl +https://github.com/0xfnzero/sol-parser-sdk/tree/main/idls https://github.com/pinax-network/substreams-solana-idls/tree/main/src https://github.com/hodlwarden/solana-tx-parser/tree/main/src https://docs.vybenetwork.com/docs/available-dexs-amms ``` -Pour AMM v4, vérifier aussi les IDL/JSON Raydium legacy présents dans fnzero, notamment les fichiers autour de `raydium_amm_v4` / `raydium_pool_v4`, sans promouvoir `raydium_pool_v4` tant que son program id et son rôle métier ne sont pas prouvés localement. +Sources spécifiques `raydium_amm_v4` à vérifier en priorité : + +```text +https://github.com/sevenlabs-hq/carbon/tree/main/decoders/raydium-amm-v4-decoder +https://github.com/pinax-network/substreams-solana-idls/tree/main/src/raydium/amm +https://github.com/0xfnzero/sol-parser-sdk/blob/main/idl/raydium_amm_v4.json +https://github.com/0xfnzero/sol-parser-sdk/blob/main/idls/raydium_amm_v4.json +https://github.com/0xfnzero/sol-parser-sdk/blob/main/idl/raydium_pool_v4.json +https://github.com/0xfnzero/sol-parser-sdk/blob/main/idls/raydium_pool_v4.json +https://solscan.io/account/675kPX9MHTjS2zt1qfr1NYHuzeLXfQM9H24wFSUt1Mp8#programIdl +``` + +## Vérification obligatoire `raydium_pool_v4` + +Avant de coder une tranche séparée `raydium_pool_v4`, faire une vérification explicite : + +```text +1. comparer idl/raydium_pool_v4.json et idls/raydium_pool_v4.json +2. comparer idl/raydium_amm_v4.json et idls/raydium_amm_v4.json +3. chercher si raydium_pool_v4 contient un program id explicite +4. comparer les instructions communes : initialize, initialize2, deposit, withdraw, swapBaseIn, swapBaseOut, monitorStep, admin/config +5. vérifier si raydium_pool_v4 décrit : + - le même Raydium AMM v4 program id 675kPX... + - un layout alternatif d'instruction + - un wrapper strategy / pool / farming / lending + - une ancienne ABI non directement liée au program id 675kPX... +6. ne pas promouvoir `raydium_pool_v4` sans corpus local : + - k_sol_instruction_observations + - decoded events locaux + - coverage local_event_kind + - absence de fallback upstream +``` + +Décision attendue dans `0.7.51` : + +```text +Option A : raydium_pool_v4 = alias/source complémentaire de raydium_amm_v4 -> intégrer ses discriminants/layouts dans raydium_amm_v4 et supprimer la version roadmap autonome. +Option B : raydium_pool_v4 = autre program id / autre surface -> conserver une future version dédiée avec program id prouvé. +Option C : raydium_pool_v4 = IDL ambiguë / strategy wrapper sans corpus -> garder en audit roadmap, pas de decoder local. +``` ## Règles fixes @@ -150,10 +246,11 @@ instruction_audit et upstream_git.instruction_match doivent être nettoyés quan 1. Créer une nouvelle base SQLite dédiée `0.7.51`. 2. Inventorier Carbon/fnzero/Pinax/Solscan Program IDL pour `raydium_amm_v4`. -3. Synchroniser `k_sol_dex_event_coverage_entries` avec `decoder_code = raydium_amm_v4`. -4. Utiliser Solscan `instruction=` pour obtenir rapidement des signatures non failed. -5. Backfill Demo2 signature/pool. -6. Replay local avec : +3. Auditer `raydium_pool_v4` avant de décider si la roadmap garde une tranche dédiée. +4. Synchroniser `k_sol_dex_event_coverage_entries` avec `decoder_code = raydium_amm_v4`. +5. Utiliser Solscan `instruction=` pour obtenir rapidement des signatures non failed. +6. Backfill Demo2 signature/pool sur corpus varié. +7. Replay local avec : ```text skipDexDecode = no @@ -161,7 +258,7 @@ forceDexDecode = yes deferInstructionObservations = yes ``` -7. Vérifier : +8. Vérifier : ```text coverage listed/observed/materialized @@ -170,6 +267,101 @@ residual upstream_git.instruction_match failed tx materialization = 0 non-trade trade_count = 0 trade/candle only for swap events validés +raydium_pool_v4 decision documented +``` + +## SQL de contrôle minimal `0.7.51` + +Coverage AMM v4 : + +```sql +SELECT + entry_name, + entry_kind, + event_family, + expected_db_target, + proof_status, + local_event_kind, + discriminator_hex, + observed_count, + materialized_count, + trade_count +FROM k_sol_dex_event_coverage_entries +WHERE decoder_code = 'raydium_amm_v4' +ORDER BY entry_kind, entry_name, discriminator_hex; +``` + +Instruction observations : + +```sql +SELECT + instruction_name, + discriminator_hex, + COUNT(*) AS observed_count, + COUNT(DISTINCT signature) AS tx_count +FROM k_sol_instruction_observations +WHERE decoder_code = 'raydium_amm_v4' +GROUP BY instruction_name, discriminator_hex +ORDER BY observed_count DESC, instruction_name, discriminator_hex; +``` + +Residual audit : + +```sql +SELECT + json_extract(payload_json, '$.discriminatorHex') AS discriminator_hex, + COUNT(*) AS audit_count, + COUNT(DISTINCT transaction_id) AS tx_count +FROM k_sol_dex_decoded_events +WHERE protocol_name = 'raydium_amm_v4' + AND event_kind = 'raydium_amm_v4.instruction_audit' +GROUP BY discriminator_hex +ORDER BY audit_count DESC, discriminator_hex; +``` + +Fallback upstream : + +```sql +SELECT + json_extract(ug.payload_json, '$.upstreamDecoderCode') AS upstream_decoder_code, + json_extract(ug.payload_json, '$.upstreamEntryName') AS entry_name, + json_extract(ug.payload_json, '$.upstreamDiscriminatorHex') AS discriminator_hex, + json_extract(ug.payload_json, '$.upstreamSourceRepo') AS source_repo, + COUNT(*) AS fallback_count, + COUNT(DISTINCT ug.transaction_id) AS tx_count +FROM k_sol_dex_decoded_events ug +JOIN k_sol_dex_event_coverage_entries ce + ON ce.decoder_code = json_extract(ug.payload_json, '$.upstreamDecoderCode') + AND ce.entry_name = json_extract(ug.payload_json, '$.upstreamEntryName') + AND ce.discriminator_hex = json_extract(ug.payload_json, '$.upstreamDiscriminatorHex') + AND ce.local_event_kind IS NOT NULL + AND ce.local_event_kind <> '' +WHERE ug.protocol_name = 'upstream_git' + AND ug.event_kind = 'upstream_git.instruction_match' + AND json_extract(ug.payload_json, '$.upstreamDecoderCode') = 'raydium_amm_v4' +GROUP BY upstream_decoder_code, entry_name, discriminator_hex, source_repo +ORDER BY fallback_count DESC, entry_name; +``` + +Non-swap safety : + +```sql +SELECT + de.event_kind, + ce.event_family, + COUNT(*) AS decoded_count, + COUNT(te.id) AS trade_count +FROM k_sol_dex_decoded_events de +LEFT JOIN k_sol_dex_event_coverage_entries ce + ON ce.decoder_code = 'raydium_amm_v4' + AND ce.local_event_kind = de.event_kind +LEFT JOIN k_sol_trade_events te + ON te.decoded_event_id = de.id +WHERE de.protocol_name = 'raydium_amm_v4' +GROUP BY de.event_kind, ce.event_family +HAVING ce.event_family <> 'swap' + AND COUNT(te.id) > 0 +ORDER BY trade_count DESC, de.event_kind; ``` ## Livrables attendus @@ -181,6 +373,7 @@ docs/DEX_DECODER_MATRIX.md docs/DEX_EVENT_COVERAGE_MATRIX.md docs/DB_EVENT_MODEL_REVIEW.md docs/reports/RAYDIUM_AMM_V4_EVENT_COVERAGE_REPORT.md +docs/reports/RAYDIUM_POOL_V4_DECISION_NOTE.md validation_sql/SQL_VALIDATION_RAYDIUM_AMM_V4_0_7_51.sql ``` diff --git a/docs/prompts/PROMPT_REPRISE_khadhroony-bobobot_0.7.52-raydium-stable.md b/docs/prompts/PROMPT_REPRISE_khadhroony-bobobot_0.7.52-raydium-stable.md new file mode 100644 index 0000000..413ac5e --- /dev/null +++ b/docs/prompts/PROMPT_REPRISE_khadhroony-bobobot_0.7.52-raydium-stable.md @@ -0,0 +1,626 @@ + + +# Prompt de reprise — `khadhroony-bobobot` — `0.7.52 raydium_stable_swap` + +Reprise du projet `khadhroony-bobobot` après clôture de `0.7.51 raydium_amm_v4`. + +## Archive de départ + +Utiliser la dernière archive complète du workspace intégrant les deltas validés jusqu’à : + +```text +0.7.51-raydium-amm-v4-final +``` + +Joindre aussi les docs et SQL de validation à jour : + +```text +README.md +ROADMAP.md +CHANGELOG.md +docs/DEX_DECODER_MATRIX.md +docs/DEX_EVENT_COVERAGE_MATRIX.md +docs/DB_EVENT_MODEL_REVIEW.md +docs/reports/RAYDIUM_AMM_V4_EVENT_COVERAGE_REPORT.md +docs/reports/RAYDIUM_POOL_V4_DECISION_NOTE.md +docs/VALIDATION_STATUS_0_7_51_FINAL.md +validation_sql/SQL_VALIDATION_RAYDIUM_AMM_V4_0_7_51.sql +``` + +## État validé avant reprise + +`0.7.51` a clôturé `raydium_amm_v4`. + +Validation locale finale rapportée : + +```text +cargo test -p kb_lib +405 passed / 0 failed + +cargo clippy -p kb_lib --all-targets -- -D warnings +OK +``` + +Dernier replay local `0.7.51` : + +```text +195 replayed +0 decode skipped +195 ledger upserts +70 unsafe ledger rows +168 trades +7 liquidity +15 lifecycle +0 tokenAccount +668 candle upserts +instructionObservations = 2599 +resetDeleted = 1578 +catalog = 61 tokens / 65 pools / 65 pairs +``` + +Points de clôture AMM v4 à préserver : + +```text +raydium_amm_v4.swap legacy = vide +decoded without coverage entry = vide +instruction_observations > 1 octet = vide +non-swap -> trade = vide +failed tx -> trade = vide +unexplained successful non-materialized events = vide +multi-target materialization = vide +pre_initialize lifecycle audit = 7 / 7 +migrate_to_open_book = orderbook only +simulate_info = decoded-only +raydium_pool_v4 = audit-only / pas de decoder autonome +``` + +## Décision de reprise + +Ouvrir une nouvelle tranche : + +```text +0.7.52 raydium_stable_swap +``` + +Code local canonique : + +```text +raydium_stable_swap +``` + +Program id canonique à utiliser comme hypothèse de départ : + +```text +5quBtoiQqxF9Jv6KYKctB59NT3gtJD2Y65kdnB1Uev3h +``` + +Important : upstream Git/IDL/Solscan est un indice, pas une preuve métier. Le program id doit être confirmé par corpus local via `k_sol_instruction_observations`, decoded events, coverage entries et absence de fallback upstream. + +## Nouvelle base de travail + +Démarrer `0.7.52` sur une base SQLite vide dédiée. + +Avant le replay de validation complet, construire volontairement un corpus initial : + +```text +1. Demo3 program_id = 5quBtoiQqxF9Jv6KYKctB59NT3gtJD2Y65kdnB1Uev3h +2. Solscan non filtré + essais instruction= +3. backfill Demo2 de signatures contenant des instructions stable swap variées +4. backfill de pools stable swap quand Demo3/Solscan fournit un AMM/pool account fiable +``` + +Ne pas interpréter l’absence de résultat Solscan comme absence on-chain définitive. + +## Note Solscan importante + +Pour `raydium_stable_swap`, il semble que Solscan ne dispose pas d’un Program IDL exploitable sur : + +```text +https://solscan.io/account/5quBtoiQqxF9Jv6KYKctB59NT3gtJD2Y65kdnB1Uev3h#programIdl +``` + +Donc le filtrage Solscan par instruction peut ne pas fonctionner avec les discriminants 8 octets Carbon/Pinax. + +Il faut tester deux approches : + +```text +1. Liens exploratoires courts : instruction=00, 01, 02, ... +2. Liens discriminants upstream 8 octets : instruction= +``` + +Solscan est une aide de découverte uniquement. La preuve métier reste locale : signatures backfillées, decoded events, instruction observations et coverage DB. + +## Sources Git/IDL à utiliser systématiquement + +Sources globales : + +```text +https://github.com/sevenlabs-hq/carbon/tree/main/decoders +https://github.com/0xfnzero/solana-streamer +https://github.com/0xfnzero/sol-parser-sdk/tree/main/idl +https://github.com/0xfnzero/sol-parser-sdk/tree/main/idls +https://github.com/pinax-network/substreams-solana-idls/tree/main/src +https://github.com/hodlwarden/solana-tx-parser/tree/main/src +https://docs.vybenetwork.com/docs/available-dexs-amms +``` + +Sources spécifiques `raydium_stable_swap` à vérifier en priorité : + +```text +https://github.com/sevenlabs-hq/carbon/tree/main/decoders/raydium-stable-swap-decoder +https://github.com/pinax-network/substreams-solana-idls/tree/main/src/raydium/stable +https://github.com/pinax-network/substreams-solana-idls/tree/main/src/raydium/stable/idl.json +https://github.com/pinax-network/substreams-solana-idls/tree/main/src/raydium/stable/instructions.rs +https://github.com/pinax-network/substreams-solana-idls/tree/main/src/raydium/stable/events.rs +https://github.com/pinax-network/substreams-solana-idls/tree/main/src/raydium/stable/accounts.rs +``` + +## Solscan — liens exploratoires + +Base non filtrée : + +```text +https://solscan.io/account/5quBtoiQqxF9Jv6KYKctB59NT3gtJD2Y65kdnB1Uev3h?hide_spam=false&hide_failed=false&show_related=true&sort=desc +``` + +Base non filtrée sans related : + +```text +https://solscan.io/account/5quBtoiQqxF9Jv6KYKctB59NT3gtJD2Y65kdnB1Uev3h?hide_spam=false&hide_failed=false&show_related=false&sort=desc +``` + +### Essais courts `instruction=00..11` + +Ces liens sont exploratoires. Ils ne prouvent pas que le program utilise des discriminants 1 octet ; ils servent seulement à tester le comportement Solscan quand aucun IDL n’est présent. + +```text +instruction=00 +https://solscan.io/account/5quBtoiQqxF9Jv6KYKctB59NT3gtJD2Y65kdnB1Uev3h?instruction=00&hide_spam=false&hide_failed=false&show_related=false&sort=desc + +instruction=01 +https://solscan.io/account/5quBtoiQqxF9Jv6KYKctB59NT3gtJD2Y65kdnB1Uev3h?instruction=01&hide_spam=false&hide_failed=false&show_related=false&sort=desc + +instruction=02 +https://solscan.io/account/5quBtoiQqxF9Jv6KYKctB59NT3gtJD2Y65kdnB1Uev3h?instruction=02&hide_spam=false&hide_failed=false&show_related=false&sort=desc + +instruction=03 +https://solscan.io/account/5quBtoiQqxF9Jv6KYKctB59NT3gtJD2Y65kdnB1Uev3h?instruction=03&hide_spam=false&hide_failed=false&show_related=false&sort=desc + +instruction=04 +https://solscan.io/account/5quBtoiQqxF9Jv6KYKctB59NT3gtJD2Y65kdnB1Uev3h?instruction=04&hide_spam=false&hide_failed=false&show_related=false&sort=desc + +instruction=05 +https://solscan.io/account/5quBtoiQqxF9Jv6KYKctB59NT3gtJD2Y65kdnB1Uev3h?instruction=05&hide_spam=false&hide_failed=false&show_related=false&sort=desc + +instruction=06 +https://solscan.io/account/5quBtoiQqxF9Jv6KYKctB59NT3gtJD2Y65kdnB1Uev3h?instruction=06&hide_spam=false&hide_failed=false&show_related=false&sort=desc + +instruction=07 +https://solscan.io/account/5quBtoiQqxF9Jv6KYKctB59NT3gtJD2Y65kdnB1Uev3h?instruction=07&hide_spam=false&hide_failed=false&show_related=false&sort=desc + +instruction=08 +https://solscan.io/account/5quBtoiQqxF9Jv6KYKctB59NT3gtJD2Y65kdnB1Uev3h?instruction=08&hide_spam=false&hide_failed=false&show_related=false&sort=desc + +instruction=09 +https://solscan.io/account/5quBtoiQqxF9Jv6KYKctB59NT3gtJD2Y65kdnB1Uev3h?instruction=09&hide_spam=false&hide_failed=false&show_related=false&sort=desc + +instruction=0a +https://solscan.io/account/5quBtoiQqxF9Jv6KYKctB59NT3gtJD2Y65kdnB1Uev3h?instruction=0a&hide_spam=false&hide_failed=false&show_related=false&sort=desc + +instruction=0b +https://solscan.io/account/5quBtoiQqxF9Jv6KYKctB59NT3gtJD2Y65kdnB1Uev3h?instruction=0b&hide_spam=false&hide_failed=false&show_related=false&sort=desc + +instruction=0c +https://solscan.io/account/5quBtoiQqxF9Jv6KYKctB59NT3gtJD2Y65kdnB1Uev3h?instruction=0c&hide_spam=false&hide_failed=false&show_related=false&sort=desc + +instruction=0d +https://solscan.io/account/5quBtoiQqxF9Jv6KYKctB59NT3gtJD2Y65kdnB1Uev3h?instruction=0d&hide_spam=false&hide_failed=false&show_related=false&sort=desc + +instruction=0e +https://solscan.io/account/5quBtoiQqxF9Jv6KYKctB59NT3gtJD2Y65kdnB1Uev3h?instruction=0e&hide_spam=false&hide_failed=false&show_related=false&sort=desc + +instruction=0f +https://solscan.io/account/5quBtoiQqxF9Jv6KYKctB59NT3gtJD2Y65kdnB1Uev3h?instruction=0f&hide_spam=false&hide_failed=false&show_related=false&sort=desc + +instruction=10 +https://solscan.io/account/5quBtoiQqxF9Jv6KYKctB59NT3gtJD2Y65kdnB1Uev3h?instruction=10&hide_spam=false&hide_failed=false&show_related=false&sort=desc + +instruction=11 +https://solscan.io/account/5quBtoiQqxF9Jv6KYKctB59NT3gtJD2Y65kdnB1Uev3h?instruction=11&hide_spam=false&hide_failed=false&show_related=false&sort=desc +``` + +### Essais discriminants 8 octets Carbon/Pinax + +À tester aussi, mais ne pas bloquer si Solscan ne filtre rien. + +```text +initialize / afaf6d1f0d989bed +https://solscan.io/account/5quBtoiQqxF9Jv6KYKctB59NT3gtJD2Y65kdnB1Uev3h?instruction=afaf6d1f0d989bed&hide_spam=false&hide_failed=false&show_related=false&sort=desc + +pre_initialize / ff5c572dc6acec02 +https://solscan.io/account/5quBtoiQqxF9Jv6KYKctB59NT3gtJD2Y65kdnB1Uev3h?instruction=ff5c572dc6acec02&hide_spam=false&hide_failed=false&show_related=false&sort=desc + +deposit / f223c68952e1f2b6 +https://solscan.io/account/5quBtoiQqxF9Jv6KYKctB59NT3gtJD2Y65kdnB1Uev3h?instruction=f223c68952e1f2b6&hide_spam=false&hide_failed=false&show_related=false&sort=desc + +withdraw / b712469c946da122 +https://solscan.io/account/5quBtoiQqxF9Jv6KYKctB59NT3gtJD2Y65kdnB1Uev3h?instruction=b712469c946da122&hide_spam=false&hide_failed=false&show_related=false&sort=desc + +swap_base_in / 2aec48a2f2182754 +https://solscan.io/account/5quBtoiQqxF9Jv6KYKctB59NT3gtJD2Y65kdnB1Uev3h?instruction=2aec48a2f2182754&hide_spam=false&hide_failed=false&show_related=false&sort=desc + +swap_base_out / a3d29bd0af92d596 +https://solscan.io/account/5quBtoiQqxF9Jv6KYKctB59NT3gtJD2Y65kdnB1Uev3h?instruction=a3d29bd0af92d596&hide_spam=false&hide_failed=false&show_related=false&sort=desc +``` + +## Instructions/discriminants de départ à couvrir + +À partir de Carbon stable swap, à vérifier contre Pinax : + +```text +initialize afaf6d1f0d989bed pool_create / k_sol_pool_lifecycle_events +pre_initialize ff5c572dc6acec02 pool_create deprecated/partial / k_sol_pool_lifecycle_events si pool context suffisant +deposit f223c68952e1f2b6 liquidity_add / k_sol_liquidity_events +withdraw b712469c946da122 liquidity_remove / k_sol_liquidity_events +swap_base_in 2aec48a2f2182754 swap / k_sol_trade_events +swap_base_out a3d29bd0af92d596 swap / k_sol_trade_events +``` + +Si Pinax expose des discriminants numériques ou une ABI non Anchor, ne pas forcer les discriminants 8 octets. Le decoder local doit suivre le layout prouvé par corpus local. + +## Objectif `0.7.52` — `raydium_stable_swap` + +Reprendre Raydium Stable Swap au même niveau de couverture que CPMM/CLMM/AMM v4 : + +```text +initialize / pre_initialize +pool lifecycle / pool_create +deposit / withdraw +swap_base_in / swap_base_out +fees / admin/config si présents dans IDL/events/accounts +OpenBook/Serum side effects documentés si présents +side effects SPL Token / Token-2022 documentés mais non promus comme raydium_stable_swap.* directs +fallback instruction_audit nettoyé quand une entrée locale spécialisée couvre l’instruction +coverage entries synchronisées et rafraîchies +decoded-only explicitement expliqué quand la matérialisation métier est impossible +``` + +## Règles fixes + +```text +Rust 2024 +pas de mod.rs +fichiers Rust avec // file: ... +pas de anyhow +pas de thiserror +pas de ? / unwrap / expect dans kb_lib applicatif +match / if let Err / let Err = ... else +rustdoc sur API publique +re-exports db.rs puis lib.rs si DB modifiée +``` + +## Invariants métier + +```text +non-trade event = jamais trade/candle +failed transaction = audit-only / jamais matérialisée métier +upstream Git/IDL/Solscan = indice, pas preuve métier +program id upstream non promu sans corpus local +side effects SPL Token / Token-2022 restent transversaux sauf preuve multi-DEX et décision DB +instruction_audit et upstream_git.instruction_match doivent être nettoyés quand une entrée locale spécialisée couvre le discriminant +observed_count ne doit pas obligatoirement égaler materialized_count +règle de clôture : observed_count = materialized_count + decoded_only_explained_count + failed_count +``` + +## Workflow conseillé + +1. Créer une nouvelle base SQLite dédiée `0.7.52`. +2. Inventorier Carbon + Pinax pour `raydium_stable_swap`. +3. Vérifier explicitement le program id `5quBtoiQqxF9Jv6KYKctB59NT3gtJD2Y65kdnB1Uev3h`. +4. Vérifier si les discriminants sont 8 octets Anchor-like, 1 octet, ou autre layout. +5. Synchroniser `k_sol_dex_event_coverage_entries` avec `decoder_code = raydium_stable_swap`. +6. Utiliser Solscan seulement comme aide exploratoire ; si le filtre instruction échoue, utiliser Demo3 program_id + signatures récentes/non filtrées. +7. Backfill Demo2 signature/pool sur corpus varié. +8. Replay local avec : + +```text +skipDexDecode = no +forceDexDecode = yes +deferInstructionObservations = yes +``` + +9. Vérifier : + +```text +coverage listed/observed/materialized +residual instruction_audit +residual upstream_git.instruction_match +decoded without coverage entry +failed tx materialization = 0 +non-trade trade_count = 0 +single-target materialization +trade/candle only for swap events validés +decoded-only explanations +``` + +## SQL de contrôle minimal `0.7.52` + +Coverage stable swap : + +```sql +SELECT + entry_name, + entry_kind, + event_family, + expected_db_target, + proof_status, + local_event_kind, + discriminator_hex, + observed_count, + materialized_count, + trade_count +FROM k_sol_dex_event_coverage_entries +WHERE decoder_code = 'raydium_stable_swap' +ORDER BY entry_kind, entry_name, discriminator_hex; +``` + +Instruction observations : + +```sql +SELECT + instruction_name, + discriminator_hex, + COUNT(*) AS observed_count, + COUNT(DISTINCT signature) AS tx_count +FROM k_sol_instruction_observations +WHERE decoder_code = 'raydium_stable_swap' +GROUP BY instruction_name, discriminator_hex +ORDER BY observed_count DESC, instruction_name, discriminator_hex; +``` + +Residual audit : + +```sql +SELECT + json_extract(payload_json, '$.discriminatorHex') AS discriminator_hex, + COUNT(*) AS audit_count, + COUNT(DISTINCT transaction_id) AS tx_count +FROM k_sol_dex_decoded_events +WHERE protocol_name = 'raydium_stable_swap' + AND event_kind = 'raydium_stable_swap.instruction_audit' +GROUP BY discriminator_hex +ORDER BY audit_count DESC, discriminator_hex; +``` + +Fallback upstream : + +```sql +SELECT + json_extract(ug.payload_json, '$.upstreamDecoderCode') AS upstream_decoder_code, + json_extract(ug.payload_json, '$.upstreamEntryName') AS entry_name, + json_extract(ug.payload_json, '$.upstreamDiscriminatorHex') AS discriminator_hex, + json_extract(ug.payload_json, '$.upstreamSourceRepo') AS source_repo, + COUNT(*) AS fallback_count, + COUNT(DISTINCT ug.transaction_id) AS tx_count +FROM k_sol_dex_decoded_events ug +JOIN k_sol_dex_event_coverage_entries ce + ON ce.decoder_code = json_extract(ug.payload_json, '$.upstreamDecoderCode') + AND ce.entry_name = json_extract(ug.payload_json, '$.upstreamEntryName') + AND ce.discriminator_hex = json_extract(ug.payload_json, '$.upstreamDiscriminatorHex') + AND ce.local_event_kind IS NOT NULL + AND ce.local_event_kind <> '' +WHERE ug.protocol_name = 'upstream_git' + AND ug.event_kind = 'upstream_git.instruction_match' + AND json_extract(ug.payload_json, '$.upstreamDecoderCode') = 'raydium_stable_swap' +GROUP BY upstream_decoder_code, entry_name, discriminator_hex, source_repo +ORDER BY fallback_count DESC, entry_name; +``` + +Non-swap safety : + +```sql +SELECT + de.event_kind, + ce.event_family, + COUNT(*) AS decoded_count, + COUNT(te.id) AS trade_count +FROM k_sol_dex_decoded_events de +LEFT JOIN k_sol_dex_event_coverage_entries ce + ON ce.decoder_code = 'raydium_stable_swap' + AND ce.local_event_kind = de.event_kind +LEFT JOIN k_sol_trade_events te + ON te.decoded_event_id = de.id +WHERE de.protocol_name = 'raydium_stable_swap' +GROUP BY de.event_kind, ce.event_family +HAVING ce.event_family <> 'swap' + AND COUNT(te.id) > 0 +ORDER BY trade_count DESC, de.event_kind; +``` + +Failed tx safety : + +```sql +SELECT + de.event_kind, + COUNT(*) AS decoded_failed_count, + COUNT(te.id) AS trade_count +FROM k_sol_dex_decoded_events de +JOIN k_sol_chain_transactions tx + ON tx.id = de.transaction_id +LEFT JOIN k_sol_trade_events te + ON te.decoded_event_id = de.id +WHERE de.protocol_name = 'raydium_stable_swap' + AND tx.err_json IS NOT NULL + AND tx.err_json <> '' + AND tx.err_json <> 'null' +GROUP BY de.event_kind +HAVING COUNT(te.id) > 0 +ORDER BY trade_count DESC, de.event_kind; +``` + +Decoded without coverage : + +```sql +SELECT + de.event_kind, + COUNT(*) AS decoded_count +FROM k_sol_dex_decoded_events de +LEFT JOIN k_sol_dex_event_coverage_entries ce + ON ce.decoder_code = 'raydium_stable_swap' + AND ce.local_event_kind = de.event_kind +WHERE de.protocol_name = 'raydium_stable_swap' + AND ce.id IS NULL +GROUP BY de.event_kind +ORDER BY decoded_count DESC, de.event_kind; +``` + +Multi-target materialization : + +```sql +SELECT + de.event_kind, + COUNT(DISTINCT de.id) AS decoded_count, + COUNT(DISTINCT te.id) AS trade_count, + COUNT(DISTINCT le.id) AS liquidity_count, + COUNT(DISTINCT pe.id) AS lifecycle_count, + COUNT(DISTINCT fe.id) AS fee_count, + COUNT(DISTINCT ae.id) AS admin_count, + COUNT(DISTINCT oe.id) AS orderbook_count, + ( + CASE WHEN COUNT(DISTINCT te.id) > 0 THEN 1 ELSE 0 END + + CASE WHEN COUNT(DISTINCT le.id) > 0 THEN 1 ELSE 0 END + + CASE WHEN COUNT(DISTINCT pe.id) > 0 THEN 1 ELSE 0 END + + CASE WHEN COUNT(DISTINCT fe.id) > 0 THEN 1 ELSE 0 END + + CASE WHEN COUNT(DISTINCT ae.id) > 0 THEN 1 ELSE 0 END + + CASE WHEN COUNT(DISTINCT oe.id) > 0 THEN 1 ELSE 0 END + ) AS materialized_target_count +FROM k_sol_dex_decoded_events de +LEFT JOIN k_sol_trade_events te + ON te.decoded_event_id = de.id +LEFT JOIN k_sol_liquidity_events le + ON le.decoded_event_id = de.id +LEFT JOIN k_sol_pool_lifecycle_events pe + ON pe.decoded_event_id = de.id +LEFT JOIN k_sol_fee_events fe + ON fe.decoded_event_id = de.id +LEFT JOIN k_sol_pool_admin_events ae + ON ae.decoded_event_id = de.id +LEFT JOIN k_sol_orderbook_events oe + ON oe.decoded_event_id = de.id +WHERE de.protocol_name = 'raydium_stable_swap' +GROUP BY de.event_kind +HAVING materialized_target_count > 1 +ORDER BY materialized_target_count DESC, de.event_kind; +``` + +Unexplained successful non-materialized events : + +```sql +SELECT + de.event_kind, + COUNT(*) AS unexplained_count +FROM k_sol_dex_decoded_events de +JOIN k_sol_chain_transactions tx + ON tx.id = de.transaction_id +LEFT JOIN k_sol_trade_events te + ON te.decoded_event_id = de.id +LEFT JOIN k_sol_liquidity_events le + ON le.decoded_event_id = de.id +LEFT JOIN k_sol_pool_lifecycle_events pe + ON pe.decoded_event_id = de.id +LEFT JOIN k_sol_fee_events fe + ON fe.decoded_event_id = de.id +LEFT JOIN k_sol_pool_admin_events ae + ON ae.decoded_event_id = de.id +LEFT JOIN k_sol_orderbook_events oe + ON oe.decoded_event_id = de.id +LEFT JOIN k_sol_token_account_events tae + ON tae.decoded_event_id = de.id +WHERE de.protocol_name = 'raydium_stable_swap' + AND ( + tx.err_json IS NULL + OR tx.err_json = '' + OR tx.err_json = 'null' + ) + AND te.id IS NULL + AND le.id IS NULL + AND pe.id IS NULL + AND fe.id IS NULL + AND ae.id IS NULL + AND oe.id IS NULL + AND tae.id IS NULL + AND COALESCE(TRIM(json_extract(de.payload_json, '$.skipTradeReason')), '') = '' + AND COALESCE(TRIM(json_extract(de.payload_json, '$.skipLiquidityReason')), '') = '' + AND COALESCE(TRIM(json_extract(de.payload_json, '$.skipLifecycleReason')), '') = '' + AND COALESCE(TRIM(json_extract(de.payload_json, '$.skipCatalogReason')), '') = '' +GROUP BY de.event_kind +ORDER BY unexplained_count DESC, de.event_kind; +``` + +Materialization summary : + +```sql +SELECT + de.event_kind, + COUNT(DISTINCT de.id) AS decoded_count, + COUNT(DISTINCT te.id) AS trade_count, + COUNT(DISTINCT le.id) AS liquidity_count, + COUNT(DISTINCT pe.id) AS lifecycle_count, + COUNT(DISTINCT fe.id) AS fee_count, + COUNT(DISTINCT ae.id) AS admin_count, + COUNT(DISTINCT oe.id) AS orderbook_count +FROM k_sol_dex_decoded_events de +LEFT JOIN k_sol_trade_events te + ON te.decoded_event_id = de.id +LEFT JOIN k_sol_liquidity_events le + ON le.decoded_event_id = de.id +LEFT JOIN k_sol_pool_lifecycle_events pe + ON pe.decoded_event_id = de.id +LEFT JOIN k_sol_fee_events fe + ON fe.decoded_event_id = de.id +LEFT JOIN k_sol_pool_admin_events ae + ON ae.decoded_event_id = de.id +LEFT JOIN k_sol_orderbook_events oe + ON oe.decoded_event_id = de.id +WHERE de.protocol_name = 'raydium_stable_swap' +GROUP BY de.event_kind +ORDER BY de.event_kind; +``` + +## Livrables attendus + +```text +archive delta fichiers modifiés/ajoutés +README.md / ROADMAP.md / CHANGELOG.md mis à jour +docs/DEX_DECODER_MATRIX.md +docs/DEX_EVENT_COVERAGE_MATRIX.md +docs/DB_EVENT_MODEL_REVIEW.md +docs/reports/RAYDIUM_STABLE_SWAP_EVENT_COVERAGE_REPORT.md +validation_sql/SQL_VALIDATION_RAYDIUM_STABLE_SWAP_0_7_52.sql +``` + +Validation finale locale : + +```bash +cargo fmt +cargo test -p kb_lib +cargo clippy -p kb_lib --all-targets -- -D warnings +``` + +## Critères de clôture `0.7.52` + +```text +tous les discriminants stable swap connus sont listés en coverage +tous les discriminants stable swap connus sont observés localement ou explicitement marqués mapped_unverified +instruction_audit résiduel vide pour les discriminants couverts +fallback upstream_git.instruction_match résiduel vide pour les discriminants couverts +decoded without coverage vide +non-swap -> trade vide +failed tx -> trade vide +multi-target materialization vide +successful decoded-only events expliqués par skip*Reason +trade/candle uniquement pour swaps avec montants fiables +deposit/withdraw uniquement vers liquidity +initialize/pre_initialize uniquement vers lifecycle +simulate/transport éventuel reste decoded-only sauf preuve métier +``` diff --git a/docs/reports/RAYDIUM_AMM_V4_EVENT_COVERAGE_REPORT.md b/docs/reports/RAYDIUM_AMM_V4_EVENT_COVERAGE_REPORT.md new file mode 100644 index 0000000..24c8c20 --- /dev/null +++ b/docs/reports/RAYDIUM_AMM_V4_EVENT_COVERAGE_REPORT.md @@ -0,0 +1,141 @@ + + +# Raydium AMM v4 Event Coverage Report — `0.7.51-final` + +## Scope + +Tranche : `0.7.51 raydium_amm_v4`. + +Program id canonique local : + +```text +675kPX9MHTjS2zt1qfr1NYHuzeLXfQM9H24wFSUt1Mp8 +``` + +Code local canonique : + +```text +raydium_amm_v4 +``` + +Cette tranche reprend AMM v4 legacy après `0.7.50 raydium_launchpad` et les rechecks CPMM/CLMM. Les sources Git/IDL/Solscan restent des indices ; les statuts observé/matérialisé proviennent du corpus local et du replay forcé. + +## Sources inventoriées + +Sources utilisées comme indices de coverage : + +- Carbon : `decoders/raydium-amm-v4-decoder` ; +- Pinax : `src/raydium/amm` ; +- fnzero `sol-parser-sdk` : `idl/raydium_amm_v4.json` et `idls/raydium_amm_v4.json` ; +- fnzero `sol-parser-sdk` : `idl/raydium_pool_v4.json` et `idls/raydium_pool_v4.json`, audit comparatif uniquement ; +- Solscan Program IDL et recherche `instruction=` pour `675kPX9MHTjS2zt1qfr1NYHuzeLXfQM9H24wFSUt1Mp8`. + +## Validation Rust et replay final + +Validation locale rapportée : + +```text +cargo test -p kb_lib -> 405 passed / 0 failed +cargo clippy -p kb_lib --all-targets -- -D warnings -> OK +``` + +Replay local final : + +```text +195 replayed +0 decode skipped +195 ledger upserts +70 unsafe ledger rows +168 trades +7 liquidity +15 lifecycle +0 tokenAccount +668 candle upserts +instructionObservations = 2599 +resetDeleted = 1578 +catalog = 61 tokens / 65 pools / 65 pairs +``` + +## Coverage finale par discriminant + +| Discriminant | Entrée | Famille | Local event kind | Cible DB | Observed | Materialized | Trade | +|---|---|---|---|---|---:|---:|---:| +| `00` | `initialize` | `pool_create` | `raydium_amm_v4.initialize` | `k_sol_pool_lifecycle_events` | 4 | 4 | 0 | +| `01` | `initialize2` | `pool_create` | `raydium_amm_v4.initialize2_pool` | `k_sol_pool_lifecycle_events` | 8 | 8 | 0 | +| `02` | `monitor_step` | `order_place` | `raydium_amm_v4.monitor_step` | `k_sol_orderbook_events` | 20 | 20 | 0 | +| `03` | `deposit` | `liquidity_add` | `raydium_amm_v4.deposit` | `k_sol_liquidity_events` | 10 | 5 | 0 | +| `04` | `withdraw` | `liquidity_remove` | `raydium_amm_v4.withdraw` | `k_sol_liquidity_events` | 3 | 2 | 0 | +| `05` | `migrate_to_open_book` | `order_place` | `raydium_amm_v4.migrate_to_open_book` | `k_sol_orderbook_events` | 6 | 6 | 0 | +| `06` | `set_params` | `admin_config` | `raydium_amm_v4.set_params` | `k_sol_pool_admin_events` | 1 | 1 | 0 | +| `07` | `withdraw_pnl` | `fee` | `raydium_amm_v4.withdraw_pnl` | `k_sol_fee_events` | 1 | 1 | 0 | +| `08` | `withdraw_srm` | `fee` | `raydium_amm_v4.withdraw_srm` | `k_sol_fee_events` | 2 | 1 | 0 | +| `09` | `swap_base_in` | `swap` | `raydium_amm_v4.swap_base_in` | `k_sol_trade_events` | 76 | 66 | 66 | +| `0a` | `pre_initialize` | `pool_create` | `raydium_amm_v4.pre_initialize` | `k_sol_pool_lifecycle_events` | 8 | 7 | 0 | +| `0b` | `swap_base_out` | `swap` | `raydium_amm_v4.swap_base_out` | `k_sol_trade_events` | 3 | 1 | 1 | +| `0c` | `simulate_info` | `cpi_transport` | `raydium_amm_v4.simulate_info` | `k_sol_dex_decoded_events_only` | 6 | 0 | 0 | +| `0d` | `admin_cancel_orders` | `order_cancel` | `raydium_amm_v4.admin_cancel_orders` | `k_sol_orderbook_events` | 22 | 22 | 0 | +| `0e` | `create_config_account` | `admin_config` | `raydium_amm_v4.create_config_account` | `k_sol_pool_admin_events` | 50 | 50 | 0 | +| `0f` | `update_config_account` | `admin_config` | `raydium_amm_v4.update_config_account` | `k_sol_pool_admin_events` | 6 | 6 | 0 | +| `10` | `swap_base_in_v2` | `swap` | `raydium_amm_v4.swap_base_in_v2` | `k_sol_trade_events` | 35 | 35 | 35 | +| `11` | `swap_base_out_v2` | `swap` | `raydium_amm_v4.swap_base_out_v2` | `k_sol_trade_events` | 7 | 7 | 7 | + +Toutes les entrées ont `proof_status=upstream_git_local_corpus_materialized`, sauf `simulate_info`, qui reste volontairement `upstream_git_local_corpus_observed` / decoded-only. + +## Matérialisation métier finale + +| Event kind | Decoded | Trade | Liquidity | Lifecycle | Fee | Admin | Orderbook | +|---|---:|---:|---:|---:|---:|---:|---:| +| `raydium_amm_v4.admin_cancel_orders` | 22 | 0 | 0 | 0 | 0 | 0 | 22 | +| `raydium_amm_v4.create_config_account` | 50 | 0 | 0 | 0 | 0 | 50 | 0 | +| `raydium_amm_v4.deposit` | 10 | 0 | 5 | 0 | 0 | 0 | 0 | +| `raydium_amm_v4.initialize2_pool` | 8 | 0 | 0 | 8 | 0 | 0 | 0 | +| `raydium_amm_v4.migrate_to_open_book` | 6 | 0 | 0 | 0 | 0 | 0 | 6 | +| `raydium_amm_v4.monitor_step` | 20 | 0 | 0 | 0 | 0 | 0 | 20 | +| `raydium_amm_v4.pre_initialize` | 8 | 0 | 0 | 7 | 0 | 0 | 0 | +| `raydium_amm_v4.set_params` | 1 | 0 | 0 | 0 | 0 | 1 | 0 | +| `raydium_amm_v4.simulate_info` | 6 | 0 | 0 | 0 | 0 | 0 | 0 | +| `raydium_amm_v4.swap_base_in` | 76 | 66 | 0 | 0 | 0 | 0 | 0 | +| `raydium_amm_v4.swap_base_in_v2` | 35 | 35 | 0 | 0 | 0 | 0 | 0 | +| `raydium_amm_v4.swap_base_out` | 3 | 1 | 0 | 0 | 0 | 0 | 0 | +| `raydium_amm_v4.swap_base_out_v2` | 7 | 7 | 0 | 0 | 0 | 0 | 0 | +| `raydium_amm_v4.update_config_account` | 6 | 0 | 0 | 0 | 0 | 6 | 0 | +| `raydium_amm_v4.withdraw` | 3 | 0 | 2 | 0 | 0 | 0 | 0 | +| `raydium_amm_v4.withdraw_pnl` | 1 | 0 | 0 | 0 | 1 | 0 | 0 | +| `raydium_amm_v4.withdraw_srm` | 2 | 0 | 0 | 0 | 1 | 0 | 0 | + +## Invariants validés + +Les requêtes finales donnent `vide` pour : + +- `raydium_amm_v4.swap` legacy ; +- decoded AMM v4 sans coverage entry ; +- observations AMM v4 dont `length(discriminator_hex) > 2` ; +- non-swap AMM v4 avec trade ; +- transaction failed AMM v4 avec trade ; +- event successful non matérialisé sans raison explicite ; +- event AMM v4 matérialisé vers plus d'une table métier principale. + +## Gaps expliqués + +Les écarts `observed_count > materialized_count` sont acceptés uniquement s'ils sont expliqués : + +- `swap_base_in` / `swap_base_out` : decoded-only lorsque les deltas vault ou montants exploitables sont absents ; +- `deposit` / `withdraw` : non matérialisés lorsque le pool/pair catalogue ou les deltas nécessaires sont absents ; +- `withdraw_srm` : non matérialisé si le contexte fee exploitable est absent ; +- `pre_initialize` : 1 transaction failed ; les 7 transactions successful sont matérialisées en lifecycle audit minimal ; +- `simulate_info` : decoded-only assumé. + +Pour `deposit`, le corpus final montre 5 événements matérialisés sur le pool catalogué `2dRNngAm729NzLbb1pzgHtfHvPqR4XHFmFyYK78EfEeX` / pair `FIDA/RAY`, et 5 événements decoded-only sur des pools absents du catalogue local. + +## Décisions spécifiques + +- `raydium_amm_v4.swap` est définitivement interdit : les swaps doivent rester spécialisés. +- `pre_initialize` est conservé pour les scans historiques, matérialisé comme lifecycle audit deprecated/partial, sans pair exploitable. +- `migrate_to_open_book`, `monitor_step` et `admin_cancel_orders` sont des side effects orderbook AMM v4, pas des trades OpenBook autonomes. +- `simulate_info` reste `k_sol_dex_decoded_events_only`. +- Les side effects SPL Token / Token-2022 restent transversaux. +- `raydium_pool_v4` n'est pas promu en decoder autonome dans `0.7.51`. + +## Clôture + +`0.7.51 raydium_amm_v4` est clôturable côté `kb_lib` sous réserve de conserver les rechecks CPMM/CLMM/Launchpad dans la validation globale de workspace lorsque la base utilisée les contient. diff --git a/docs/reports/RAYDIUM_POOL_V4_DECISION_NOTE.md b/docs/reports/RAYDIUM_POOL_V4_DECISION_NOTE.md new file mode 100644 index 0000000..7e0a59a --- /dev/null +++ b/docs/reports/RAYDIUM_POOL_V4_DECISION_NOTE.md @@ -0,0 +1,56 @@ + + +# Raydium Pool v4 Decision Note — `0.7.51` + +## Décision courte + +`raydium_pool_v4` ne doit pas être ouvert comme decoder autonome dans `0.7.51`. + +Statut retenu : + +```text +Option C — IDL ambiguë / strategy-pool wrapper sans corpus local suffisant. +``` + +Conséquence : `raydium_pool_v4` reste une source d'audit/comparaison pour `raydium_amm_v4`. Toute promotion en tranche dédiée exige un program id prouvé localement, des observations dans `k_sol_instruction_observations`, des decoded events locaux et une absence de fallback upstream inexpliqué. + +## Comparaison synthétique + +| Source | Nom / rôle constaté | Indices structurants | Décision locale | +|---|---|---|---| +| `idl/raydium_amm_v4.json` | `raydium_amm` | Instructions AMM v4 legacy : `initialize`, `initialize2`, `deposit`, `withdraw`, `swapBaseIn`, `swapBaseOut`, `monitorStep`, `setParams`. | Source principale pour `raydium_amm_v4`. | +| `idls/raydium_amm_v4.json` | `raydium_amm` | Variante parallèle à comparer ligne à ligne avec `idl/`. | Source principale complémentaire. | +| `idl/raydium_pool_v4.json` | `Raydium Liquidity Pool V4` | Entrées orientées stratégie/pool wrapper : `initializeStrategy`, comptes `strategyState`, `strategyAuthority`, `lendingProgramId`; contient aussi des noms comme `swapBaseIn`. | Audit uniquement. Ne pas promouvoir. | +| `idls/raydium_pool_v4.json` | `Raydium Liquidity Pool V4` | Même famille de signaux que `idl/raydium_pool_v4.json` à vérifier localement. | Audit uniquement. | + +## Raisons de non-promotion + +1. `raydium_pool_v4` ne prouve pas encore un program id local autonome dans le workspace. +2. La présence de noms communs (`swapBaseIn`, comptes OpenBook) ne suffit pas à conclure que la surface est le program id AMM v4 canonique `675kPX...`. +3. Les entrées `initializeStrategy`, `strategyState`, `strategyAuthority` et `lendingProgramId` indiquent un rôle potentiellement distinct : strategy, wrapper, pool manager, lending ou ancienne ABI composite. +4. Aucune ligne locale `k_sol_instruction_observations` / `k_sol_dex_decoded_events` / `k_sol_dex_event_coverage_entries.local_event_kind` ne justifie une tranche autonome au moment de l'ouverture `0.7.51`. + +## Règle de décision future + +- Si `raydium_pool_v4` correspond finalement au même program id AMM v4 ou à un layout alternatif compatible, intégrer les discriminants/layouts validés dans `raydium_amm_v4`. +- Si `raydium_pool_v4` correspond à un autre program id / wrapper / strategy / lending surface, créer une tranche dédiée seulement après corpus local. +- Si l'IDL reste ambiguë, conserver l'entrée en roadmap comme audit conditionnel sans decoder runtime. + +## SQL local attendu avant toute promotion + +Une future promotion doit au minimum montrer : + +```sql +SELECT + decoder_code, + instruction_name, + discriminator_hex, + COUNT(*) AS observed_count, + COUNT(DISTINCT signature) AS tx_count +FROM k_sol_instruction_observations +WHERE decoder_code IN ('raydium_amm_v4', 'raydium_pool_v4') +GROUP BY decoder_code, instruction_name, discriminator_hex +ORDER BY decoder_code, observed_count DESC; +``` + +Puis une preuve de decoded events locaux, de coverage entries mappées et d'absence de fallback upstream résiduel. diff --git a/kb_demo_app/package.json b/kb_demo_app/package.json index 62a83df..35516d1 100644 --- a/kb_demo_app/package.json +++ b/kb_demo_app/package.json @@ -1,7 +1,7 @@ { "name": "kb-demo-app", "private": true, - "version": "0.7.50", + "version": "0.7.51", "type": "module", "scripts": { "dev": "vite", diff --git a/kb_demo_app/tauri.conf.json b/kb_demo_app/tauri.conf.json index 1579680..05d0d5a 100644 --- a/kb_demo_app/tauri.conf.json +++ b/kb_demo_app/tauri.conf.json @@ -1,7 +1,7 @@ { "$schema": "https://schema.tauri.app/config/2", "productName": "kb-demo-app", - "version": "0.7.50", + "version": "0.7.51", "identifier": "com.sasedev.kb-demo-app", "build": { "beforeDevCommand": "npm run dev", diff --git a/kb_lib/src/db.rs b/kb_lib/src/db.rs index 38b8d92..2e84bab 100644 --- a/kb_lib/src/db.rs +++ b/kb_lib/src/db.rs @@ -155,8 +155,13 @@ pub use queries::query_db_runtime_events_list_recent; pub use queries::query_dex_decode_replay_ledger_get_by_signature; pub use queries::query_dex_decode_replay_ledger_get_by_transaction; pub use queries::query_dex_decode_replay_ledger_upsert; +pub use queries::query_dex_decoded_events_cleanup_raydium_launchpad_anchor_self_cpi_audits; pub use queries::query_dex_decoded_events_delete_by_key; +pub use queries::query_dex_decoded_events_delete_instruction_audit_by_discriminator; +pub use queries::query_dex_decoded_events_delete_local_replay_scope_by_transaction_id; pub use queries::query_dex_decoded_events_delete_locally_covered_upstream_instruction_matches; +pub use queries::query_dex_decoded_events_delete_raydium_clmm_instruction_audit_by_discriminator; +pub use queries::query_dex_decoded_events_delete_raydium_launchpad_anchor_self_cpi_audit; pub use queries::query_dex_decoded_events_delete_replaced_raydium_cpmm_instruction_audits; pub use queries::query_dex_decoded_events_delete_meteora_dlmm_anchor_swap_instruction_audits; pub use queries::query_dex_decoded_events_delete_related_instruction_audit; @@ -165,6 +170,7 @@ pub use queries::query_dex_decoded_events_get_by_key; pub use queries::query_dex_decoded_events_get_latest_pump_fun_create_payload_by_mint; pub use queries::query_dex_decoded_events_list_by_transaction_id; pub use queries::query_dex_decoded_events_upsert; +pub use queries::query_dex_decoded_events_update_payload_json_by_id; pub use queries::query_dex_event_coverage_entries_delete_by_decoder; pub use queries::query_dex_event_coverage_entries_list_by_decoder; pub use queries::query_dex_event_coverage_entries_list_summary_by_decoder; @@ -177,6 +183,11 @@ pub use queries::query_dexs_upsert; pub use queries::query_fee_events_get_by_decoded_event_id; pub use queries::query_fee_events_list_recent; pub use queries::query_fee_events_upsert; +pub use queries::query_instruction_observation_source_rows_list_by_signature; +pub use queries::query_instruction_observation_source_rows_list_recent; +pub use queries::query_instruction_observation_source_rows_list_replay_window; +pub use queries::query_instruction_observations_delete_by_transaction_ids; +pub use dtos::InstructionObservationSourceRow; pub use queries::query_instruction_observations_list_by_filter; pub use queries::query_instruction_observations_upsert; pub use queries::query_known_http_endpoints_get; @@ -188,6 +199,8 @@ pub use queries::query_known_ws_endpoints_upsert; pub use queries::query_launch_attributions_get_by_decoded_event_id; pub use queries::query_launch_attributions_list_by_pool_id; pub use queries::query_launch_attributions_upsert; +pub use queries::query_launch_events_upsert; +pub use dtos::LaunchEventUpsertInput; pub use queries::query_launch_surface_keys_get_by_match; pub use queries::query_launch_surface_keys_list_by_surface_id; pub use queries::query_launch_surface_keys_upsert; @@ -237,6 +250,7 @@ pub use queries::query_pairs_get_by_pool_id; pub use queries::query_pairs_list; pub use queries::query_pairs_update_symbol; pub use queries::query_pairs_upsert; +pub use queries::query_pool_admin_events_delete_by_decoded_event_id; pub use queries::query_pool_admin_events_get_by_decoded_event_id; pub use queries::query_pool_admin_events_list_recent; pub use queries::query_pool_admin_events_upsert; diff --git a/kb_lib/src/db/dtos.rs b/kb_lib/src/db/dtos.rs index f718063..c7b776b 100644 --- a/kb_lib/src/db/dtos.rs +++ b/kb_lib/src/db/dtos.rs @@ -13,10 +13,11 @@ mod dex_decode_replay_ledger; mod dex_decoded_event; mod dex_event_coverage_entry; mod fee_event; +mod instruction_observation; mod known_http_endpoint; mod known_ws_endpoint; -mod instruction_observation; mod launch_attribution; +mod launch_event; mod launch_surface; mod launch_surface_key; mod liquidity_event; @@ -72,6 +73,8 @@ pub(crate) use local_pipeline_diagnostics::LocalPipelineDiagnosticCountersRow; pub(crate) use local_pipeline_diagnostics::LocalPoolOriginDiagnosticSampleRow; pub(crate) use local_pipeline_diagnostics::LocalRaydiumProgramInstructionDiagnosticSummaryRow; pub(crate) use local_pipeline_diagnostics::LocalTokenMetadataGapDiagnosticSampleRow; +pub(crate) use program_instruction_discriminator_summary::ProgramInstructionDiscriminatorSummaryAccumulator; +pub(crate) use program_instruction_discriminator_summary::ProgramInstructionDiscriminatorSummaryKey; pub use analysis_signal::AnalysisSignalDto; pub use chain_instruction::ChainInstructionDto; @@ -85,10 +88,12 @@ pub use dex_decoded_event::DexDecodedEventDto; pub use dex_event_coverage_entry::DexEventCoverageEntryDto; pub use dex_event_coverage_entry::DexEventCoverageSummaryDto; pub use fee_event::FeeEventDto; +pub use instruction_observation::InstructionObservationDto; +pub use instruction_observation::InstructionObservationSourceRow; pub use known_http_endpoint::KnownHttpEndpointDto; pub use known_ws_endpoint::KnownWsEndpointDto; -pub use instruction_observation::InstructionObservationDto; pub use launch_attribution::LaunchAttributionDto; +pub use launch_event::LaunchEventUpsertInput; pub use launch_surface::LaunchSurfaceDto; pub use launch_surface_key::LaunchSurfaceKeyDto; pub use liquidity_event::LiquidityEventDto; diff --git a/kb_lib/src/db/dtos/instruction_observation.rs b/kb_lib/src/db/dtos/instruction_observation.rs index 7ca3e1f..9bcc685 100644 --- a/kb_lib/src/db/dtos/instruction_observation.rs +++ b/kb_lib/src/db/dtos/instruction_observation.rs @@ -31,7 +31,7 @@ pub struct InstructionObservationDto { pub program_id: std::string::String, /// Local decoder code when resolved. pub decoder_code: std::option::Option, - /// First eight instruction-data bytes as lower-hex. + /// Instruction discriminator bytes as lower-hex; AMM v4 uses one byte, Anchor-style programs use eight bytes. pub discriminator_hex: std::option::Option, /// Known local instruction name when resolved. pub instruction_name: std::option::Option, @@ -51,6 +51,41 @@ pub struct InstructionObservationDto { pub updated_at: chrono::DateTime, } +/// Raw source row used to rebuild the technical instruction-observation index. +#[derive(Debug, Clone, sqlx::FromRow)] +pub struct InstructionObservationSourceRow { + /// Internal transaction id. + pub transaction_id: i64, + /// Transaction signature. + pub signature: std::string::String, + /// Optional slot number. + pub slot: std::option::Option, + /// Optional block time. + pub block_time: std::option::Option, + /// Optional transaction error JSON. + pub err_json: std::option::Option, + /// Internal instruction id. + pub instruction_id: i64, + /// Optional parent instruction id for CPI rows. + pub parent_instruction_id: std::option::Option, + /// Outer instruction index. + pub instruction_index: i64, + /// Optional inner instruction index. + pub inner_instruction_index: std::option::Option, + /// Program id, when available. + pub program_id: std::option::Option, + /// Instruction account list JSON. + pub accounts_json: std::string::String, + /// Optional instruction data JSON. + pub data_json: std::option::Option, + /// Optional pool account resolved from a decoded event on the same instruction. + pub pool_account: std::option::Option, + /// Optional decoded event kind on the same instruction. + pub decoded_event_kind: std::option::Option, + /// Optional decoded event id on the same instruction. + pub decoded_event_id: std::option::Option, +} + impl InstructionObservationDto { /// Creates a new instruction observation DTO. #[allow(clippy::too_many_arguments)] diff --git a/kb_lib/src/db/dtos/launch_event.rs b/kb_lib/src/db/dtos/launch_event.rs new file mode 100644 index 0000000..46136f9 --- /dev/null +++ b/kb_lib/src/db/dtos/launch_event.rs @@ -0,0 +1,41 @@ +// file: kb_lib/src/db/dtos/launch_event.rs + +//! Launch event DTO helpers. + +/// Input used to upsert one launch event row. +#[derive(Debug, Clone)] +pub struct LaunchEventUpsertInput { + /// Source transaction id. + pub transaction_id: i64, + /// Source decoded event id. + pub decoded_event_id: i64, + /// Optional DEX id. + pub dex_id: std::option::Option, + /// Optional pool id. + pub pool_id: std::option::Option, + /// Optional pair id. + pub pair_id: std::option::Option, + /// Transaction signature. + pub signature: std::string::String, + /// Optional slot number stored as i64. + pub slot: std::option::Option, + /// Protocol code. + pub protocol_name: std::string::String, + /// Program id. + pub program_id: std::string::String, + /// Decoded event kind. + pub event_kind: std::string::String, + /// Optional pool account. + pub pool_account: std::option::Option, + /// Optional actor wallet. + pub actor_wallet: std::option::Option, + /// Launch event role. + pub event_role: std::string::String, + /// Optional related account. + pub related_account: std::option::Option, + /// Optional related mint. + pub related_mint: std::option::Option, + /// Raw decoded payload JSON. + pub payload_json: serde_json::Value, +} + diff --git a/kb_lib/src/db/dtos/program_instruction_discriminator_summary.rs b/kb_lib/src/db/dtos/program_instruction_discriminator_summary.rs index 220f51b..050d489 100644 --- a/kb_lib/src/db/dtos/program_instruction_discriminator_summary.rs +++ b/kb_lib/src/db/dtos/program_instruction_discriminator_summary.rs @@ -44,3 +44,32 @@ pub struct ProgramInstructionDiscriminatorSummaryDto { /// Accounts JSON preview from the latest row. pub latest_accounts_json_preview: std::option::Option, } + +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] +pub(crate) struct ProgramInstructionDiscriminatorSummaryKey { + pub(crate) program_id: std::string::String, + pub(crate) discriminator_hex: std::option::Option, + pub(crate) accounts_count: u64, + pub(crate) stack_height: std::option::Option, + pub(crate) is_inner_instruction: bool, +} + +#[derive(Debug, Clone)] +pub(crate) struct ProgramInstructionDiscriminatorSummaryAccumulator { + pub(crate) key: ProgramInstructionDiscriminatorSummaryKey, + pub(crate) known_instruction_name: std::option::Option, + pub(crate) occurrence_count: u64, + pub(crate) decoded_event_count: u64, + pub(crate) transaction_signatures: std::collections::BTreeSet, + pub(crate) latest_slot: std::option::Option, + pub(crate) latest_signature: std::string::String, + pub(crate) latest_instruction_id: i64, + pub(crate) latest_instruction_index: u32, + pub(crate) latest_inner_instruction_index: std::option::Option, + pub(crate) latest_parsed_type: std::option::Option, + pub(crate) latest_decoded_event_kind: std::option::Option, + pub(crate) latest_data_json_preview: std::option::Option, + pub(crate) latest_accounts_json_preview: std::option::Option, +} + + diff --git a/kb_lib/src/db/queries.rs b/kb_lib/src/db/queries.rs index 69f955f..fb2e245 100644 --- a/kb_lib/src/db/queries.rs +++ b/kb_lib/src/db/queries.rs @@ -17,6 +17,7 @@ mod instruction_observation; mod known_http_endpoint; mod known_ws_endpoint; mod launch_attribution; +mod launch_event; mod launch_surface; mod launch_surface_key; mod liquidity_event; @@ -73,15 +74,21 @@ pub use dex::query_dexs_upsert; pub use dex_decode_replay_ledger::query_dex_decode_replay_ledger_get_by_signature; pub use dex_decode_replay_ledger::query_dex_decode_replay_ledger_get_by_transaction; pub use dex_decode_replay_ledger::query_dex_decode_replay_ledger_upsert; +pub use dex_decoded_event::query_dex_decoded_events_cleanup_raydium_launchpad_anchor_self_cpi_audits; pub use dex_decoded_event::query_dex_decoded_events_delete_by_key; +pub use dex_decoded_event::query_dex_decoded_events_delete_instruction_audit_by_discriminator; +pub use dex_decoded_event::query_dex_decoded_events_delete_local_replay_scope_by_transaction_id; pub use dex_decoded_event::query_dex_decoded_events_delete_locally_covered_upstream_instruction_matches; pub use dex_decoded_event::query_dex_decoded_events_delete_meteora_dlmm_anchor_swap_instruction_audits; +pub use dex_decoded_event::query_dex_decoded_events_delete_raydium_clmm_instruction_audit_by_discriminator; +pub use dex_decoded_event::query_dex_decoded_events_delete_raydium_launchpad_anchor_self_cpi_audit; pub use dex_decoded_event::query_dex_decoded_events_delete_related_instruction_audit; pub use dex_decoded_event::query_dex_decoded_events_delete_replaced_raydium_clmm_instruction_audits; pub use dex_decoded_event::query_dex_decoded_events_delete_replaced_raydium_cpmm_instruction_audits; pub use dex_decoded_event::query_dex_decoded_events_get_by_key; pub use dex_decoded_event::query_dex_decoded_events_get_latest_pump_fun_create_payload_by_mint; pub use dex_decoded_event::query_dex_decoded_events_list_by_transaction_id; +pub use dex_decoded_event::query_dex_decoded_events_update_payload_json_by_id; pub use dex_decoded_event::query_dex_decoded_events_upsert; pub use dex_event_coverage_entry::query_dex_event_coverage_entries_delete_by_decoder; pub use dex_event_coverage_entry::query_dex_event_coverage_entries_list_by_decoder; @@ -92,6 +99,10 @@ pub use dex_event_coverage_entry::query_dex_event_coverage_entries_upsert; pub use fee_event::query_fee_events_get_by_decoded_event_id; pub use fee_event::query_fee_events_list_recent; pub use fee_event::query_fee_events_upsert; +pub use instruction_observation::query_instruction_observation_source_rows_list_by_signature; +pub use instruction_observation::query_instruction_observation_source_rows_list_recent; +pub use instruction_observation::query_instruction_observation_source_rows_list_replay_window; +pub use instruction_observation::query_instruction_observations_delete_by_transaction_ids; pub use instruction_observation::query_instruction_observations_list_by_filter; pub use instruction_observation::query_instruction_observations_upsert; pub use known_http_endpoint::query_known_http_endpoints_get; @@ -103,6 +114,7 @@ pub use known_ws_endpoint::query_known_ws_endpoints_upsert; pub use launch_attribution::query_launch_attributions_get_by_decoded_event_id; pub use launch_attribution::query_launch_attributions_list_by_pool_id; pub use launch_attribution::query_launch_attributions_upsert; +pub use launch_event::query_launch_events_upsert; pub use launch_surface::query_launch_surfaces_get_by_code; pub use launch_surface::query_launch_surfaces_list; pub use launch_surface::query_launch_surfaces_upsert; @@ -155,6 +167,7 @@ pub use pair_metric::query_pair_metrics_upsert; pub use pool::query_pools_get_by_address; pub use pool::query_pools_list; pub use pool::query_pools_upsert; +pub use pool_admin_event::query_pool_admin_events_delete_by_decoded_event_id; pub use pool_admin_event::query_pool_admin_events_get_by_decoded_event_id; pub use pool_admin_event::query_pool_admin_events_list_recent; pub use pool_admin_event::query_pool_admin_events_upsert; diff --git a/kb_lib/src/db/queries/dex_decoded_event.rs b/kb_lib/src/db/queries/dex_decoded_event.rs index aaf1488..cfbea4c 100644 --- a/kb_lib/src/db/queries/dex_decoded_event.rs +++ b/kb_lib/src/db/queries/dex_decoded_event.rs @@ -89,6 +89,135 @@ LIMIT 1 } } +/// Deletes all locally decoded DEX rows and linked materialization for one transaction. +/// +/// This reset is intentionally scoped by `protocol_name`, not by current coverage entries. +/// It removes legacy event kinds that no longer exist in the decoder, such as the old +/// `raydium_amm_v4.swap` row that was replaced by specialized AMM v4 swap variants. +pub async fn query_dex_decoded_events_delete_local_replay_scope_by_transaction_id( + database: &crate::Database, + transaction_id: i64, +) -> Result { + match database.connection() { + crate::DatabaseConnection::Sqlite(pool) => { + let statements = [ + ( + "k_sol_instruction_observations decoded links", + r#" +UPDATE k_sol_instruction_observations +SET decoded_event_id = NULL +WHERE decoded_event_id IN ( + SELECT id + FROM k_sol_dex_decoded_events + WHERE transaction_id = ? + AND protocol_name IN ( + 'raydium_amm_v4', 'raydium_cpmm', 'raydium_clmm', 'raydium_launchpad', + 'pump_fun', 'pump_swap', 'meteora_dbc', 'meteora_dlmm', 'meteora_damm_v1', + 'meteora_damm_v2', 'orca_whirlpools', 'fluxbeam', 'dexlab', 'openbook_v2', + 'phoenix_v1' + ) +) + "#, + ), + ( + "k_sol_launch_attributions", + "DELETE FROM k_sol_launch_attributions WHERE transaction_id = ?", + ), + ( + "k_sol_launch_events", + "DELETE FROM k_sol_launch_events WHERE transaction_id = ?", + ), + ("k_sol_trade_events", "DELETE FROM k_sol_trade_events WHERE transaction_id = ?"), + ( + "k_sol_liquidity_events", + "DELETE FROM k_sol_liquidity_events WHERE transaction_id = ?", + ), + ( + "k_sol_pool_lifecycle_events", + "DELETE FROM k_sol_pool_lifecycle_events WHERE transaction_id = ?", + ), + ("k_sol_fee_events", "DELETE FROM k_sol_fee_events WHERE transaction_id = ?"), + ( + "k_sol_reward_events", + "DELETE FROM k_sol_reward_events WHERE transaction_id = ?", + ), + ( + "k_sol_pool_admin_events", + "DELETE FROM k_sol_pool_admin_events WHERE transaction_id = ?", + ), + ( + "k_sol_orderbook_events", + "DELETE FROM k_sol_orderbook_events WHERE transaction_id = ?", + ), + ( + "k_sol_token_account_events", + "DELETE FROM k_sol_token_account_events WHERE transaction_id = ?", + ), + ( + "k_sol_dex_decoded_events", + r#" +DELETE FROM k_sol_dex_decoded_events +WHERE transaction_id = ? + AND protocol_name IN ( + 'raydium_amm_v4', 'raydium_cpmm', 'raydium_clmm', 'raydium_launchpad', + 'pump_fun', 'pump_swap', 'meteora_dbc', 'meteora_dlmm', 'meteora_damm_v1', + 'meteora_damm_v2', 'orca_whirlpools', 'fluxbeam', 'dexlab', 'openbook_v2', + 'phoenix_v1' + ) + "#, + ), + ]; + let mut deleted_count = 0_u64; + for (scope_name, statement) in statements { + let query_result = sqlx::query(statement).bind(transaction_id).execute(pool).await; + let result = match query_result { + Ok(result) => result, + Err(error) => { + return Err(crate::Error::Db(format!( + "cannot delete local DEX replay scope '{}' for transaction id '{}' on sqlite: {}", + scope_name, transaction_id, error + ))); + }, + }; + deleted_count = deleted_count.saturating_add(result.rows_affected()); + } + return Ok(deleted_count); + }, + } +} + +/// Updates the persisted payload of one decoded DEX event row. +pub async fn query_dex_decoded_events_update_payload_json_by_id( + database: &crate::Database, + decoded_event_id: i64, + payload_json: &str, +) -> Result { + match database.connection() { + crate::DatabaseConnection::Sqlite(pool) => { + let query_result = sqlx::query( + r#" +UPDATE k_sol_dex_decoded_events +SET payload_json = ? +WHERE id = ? + "#, + ) + .bind(payload_json) + .bind(decoded_event_id) + .execute(pool) + .await; + match query_result { + Ok(result) => return Ok(result.rows_affected()), + Err(error) => { + return Err(crate::Error::Db(format!( + "cannot update k_sol_dex_decoded_events payload_json by id '{}' on sqlite: {}", + decoded_event_id, error + ))); + }, + } + }, + } +} + /// Deletes one decoded DEX event row by its natural key. pub async fn query_dex_decoded_events_delete_by_key( database: &crate::Database, @@ -698,6 +827,195 @@ LIMIT 1 } } +/// Deletes a Raydium CLMM instruction-audit row by discriminator for one transaction. +pub async fn query_dex_decoded_events_delete_raydium_clmm_instruction_audit_by_discriminator( + database: &crate::Database, + transaction_id: i64, + discriminator_hex: &str, +) -> Result { + match database.connection() { + crate::DatabaseConnection::Sqlite(pool) => { + let delete_result = sqlx::query( + r#" +DELETE FROM k_sol_dex_decoded_events +WHERE transaction_id = ? + AND protocol_name = 'raydium_clmm' + AND event_kind = 'raydium_clmm.instruction_audit' + AND ( + json_extract(payload_json, '$.discriminatorHex') = ? + OR json_extract(payload_json, '$.discriminator_hex') = ? + OR json_extract(payload_json, '$.instructionDiscriminatorHex') = ? + OR json_extract(payload_json, '$.instruction_discriminator_hex') = ? + OR json_extract(payload_json, '$.anchorEventDiscriminatorHex') = ? + OR json_extract(payload_json, '$.anchor_event_discriminator_hex') = ? + ) + "#, + ) + .bind(transaction_id) + .bind(discriminator_hex.to_string()) + .bind(discriminator_hex.to_string()) + .bind(discriminator_hex.to_string()) + .bind(discriminator_hex.to_string()) + .bind(discriminator_hex.to_string()) + .bind(discriminator_hex.to_string()) + .execute(pool) + .await; + match delete_result { + Ok(result) => return Ok(result.rows_affected()), + Err(error) => { + return Err(crate::Error::Db(format!( + "cannot delete Raydium CLMM residual instruction audit '{}': {}", + discriminator_hex, error + ))); + }, + } + }, + } +} + +/// Deletes a protocol instruction-audit row by discriminator for one transaction. +pub async fn query_dex_decoded_events_delete_instruction_audit_by_discriminator( + database: &crate::Database, + transaction_id: i64, + protocol_name: &str, + audit_event_kind: &str, + discriminator_hex: &str, +) -> Result { + match database.connection() { + crate::DatabaseConnection::Sqlite(pool) => { + let delete_result = sqlx::query( + r#" +DELETE FROM k_sol_dex_decoded_events +WHERE transaction_id = ? + AND protocol_name = ? + AND event_kind = ? + AND ( + json_extract(payload_json, '$.discriminatorHex') = ? + OR json_extract(payload_json, '$.discriminator_hex') = ? + OR json_extract(payload_json, '$.instructionDiscriminatorHex') = ? + OR json_extract(payload_json, '$.instruction_discriminator_hex') = ? + OR json_extract(payload_json, '$.anchorEventDiscriminatorHex') = ? + OR json_extract(payload_json, '$.anchor_event_discriminator_hex') = ? + OR instr(lower(COALESCE(payload_json, '')), lower(?)) > 0 + ) + "#, + ) + .bind(transaction_id) + .bind(protocol_name.to_string()) + .bind(audit_event_kind.to_string()) + .bind(discriminator_hex.to_string()) + .bind(discriminator_hex.to_string()) + .bind(discriminator_hex.to_string()) + .bind(discriminator_hex.to_string()) + .bind(discriminator_hex.to_string()) + .bind(discriminator_hex.to_string()) + .bind(discriminator_hex.to_string()) + .execute(pool) + .await; + match delete_result { + Ok(result) => return Ok(result.rows_affected()), + Err(error) => { + return Err(crate::Error::Db(format!( + "cannot delete replaced instruction audit by discriminator on sqlite: {}", + error + ))); + }, + } + }, + } +} + +/// Deletes a Raydium Launchpad self-CPI audit row replaced by a direct decoded event. +pub async fn query_dex_decoded_events_delete_raydium_launchpad_anchor_self_cpi_audit( + database: &crate::Database, + transaction_id: i64, + anchor_self_cpi_log_selector_hex: &str, + anchor_event_discriminator_hex: &str, +) -> Result { + match database.connection() { + crate::DatabaseConnection::Sqlite(pool) => { + let delete_result = sqlx::query( + r#" +DELETE FROM k_sol_dex_decoded_events +WHERE transaction_id = ? + AND protocol_name = 'raydium_launchpad' + AND event_kind = 'raydium_launchpad.instruction_audit' + AND json_extract(payload_json, '$.anchorSelfCpiLog') = 1 + AND json_extract(payload_json, '$.anchorSelfCpiLogSelectorHex') = ? + AND json_extract(payload_json, '$.anchorEventDiscriminatorHex') = ? + "#, + ) + .bind(transaction_id) + .bind(anchor_self_cpi_log_selector_hex.to_string()) + .bind(anchor_event_discriminator_hex.to_string()) + .execute(pool) + .await; + match delete_result { + Ok(result) => return Ok(result.rows_affected()), + Err(error) => { + return Err(crate::Error::Db(format!( + "cannot delete replaced Raydium Launchpad self-CPI instruction audit on sqlite: {}", + error + ))); + }, + } + }, + } +} + +/// Deletes Raydium Launchpad self-CPI audit rows once their direct decoded rows exist. +pub async fn query_dex_decoded_events_cleanup_raydium_launchpad_anchor_self_cpi_audits( + database: &crate::Database, + transaction_id: i64, + anchor_self_cpi_log_selector_hex: &str, +) -> Result { + match database.connection() { + crate::DatabaseConnection::Sqlite(pool) => { + let delete_result = sqlx::query( + r#" +DELETE FROM k_sol_dex_decoded_events +WHERE id IN ( + SELECT audit.id + FROM k_sol_dex_decoded_events audit + WHERE audit.transaction_id = ? + AND audit.protocol_name = 'raydium_launchpad' + AND audit.event_kind = 'raydium_launchpad.instruction_audit' + AND json_extract(audit.payload_json, '$.anchorSelfCpiLog') = 1 + AND json_extract(audit.payload_json, '$.anchorSelfCpiLogSelectorHex') = ? + AND EXISTS ( + SELECT 1 + FROM k_sol_dex_decoded_events direct + WHERE direct.transaction_id = audit.transaction_id + AND direct.protocol_name = 'raydium_launchpad' + AND direct.event_kind IN ( + 'raydium_launchpad.trade_event', + 'raydium_launchpad.pool_create_event', + 'raydium_launchpad.claim_vested_event', + 'raydium_launchpad.create_vesting_event' + ) + AND json_extract(direct.payload_json, '$.anchorEventDiscriminatorHex') = + json_extract(audit.payload_json, '$.anchorEventDiscriminatorHex') + ) +) + "#, + ) + .bind(transaction_id) + .bind(anchor_self_cpi_log_selector_hex.to_string()) + .execute(pool) + .await; + match delete_result { + Ok(result) => return Ok(result.rows_affected()), + Err(error) => { + return Err(crate::Error::Db(format!( + "cannot cleanup replaced Raydium Launchpad self-CPI instruction audits on sqlite: {}", + error + ))); + }, + } + }, + } +} + #[cfg(test)] mod tests { async fn make_database() -> crate::Database { diff --git a/kb_lib/src/db/queries/instruction_observation.rs b/kb_lib/src/db/queries/instruction_observation.rs index ec5f0ea..10cb231 100644 --- a/kb_lib/src/db/queries/instruction_observation.rs +++ b/kb_lib/src/db/queries/instruction_observation.rs @@ -171,3 +171,218 @@ LIMIT ? }, } } + +/// Deletes instruction observations for a set of transaction ids before rebuilding the technical index. +pub async fn query_instruction_observations_delete_by_transaction_ids( + database: &crate::Database, + transaction_ids: &[i64], +) -> Result { + if transaction_ids.is_empty() { + return Ok(0); + } + match database.connection() { + crate::DatabaseConnection::Sqlite(pool) => { + let mut deleted_count: u64 = 0; + for transaction_id_chunk in transaction_ids.chunks(900) { + let mut query_builder = sqlx::QueryBuilder::::new( + "DELETE FROM k_sol_instruction_observations WHERE transaction_id IN (", + ); + let mut separated = query_builder.separated(", "); + for transaction_id in transaction_id_chunk { + separated.push_bind(*transaction_id); + } + separated.push_unseparated(")"); + let query = query_builder.build(); + let query_result = query.execute(pool).await; + match query_result { + Ok(query_result) => { + deleted_count = deleted_count.saturating_add(query_result.rows_affected()); + }, + Err(error) => { + return Err(crate::Error::Db(format!( + "cannot delete k_sol_instruction_observations by transaction ids on sqlite: {}", + error + ))); + }, + } + } + return Ok(deleted_count); + }, + } +} + +/// Lists instruction-observation source rows for one transaction signature. +pub async fn query_instruction_observation_source_rows_list_by_signature( + database: &crate::Database, + signature: &str, +) -> Result, crate::Error> { + match database.connection() { + crate::DatabaseConnection::Sqlite(pool) => { + let query_result = + sqlx::query_as::( + r#" +SELECT + tx.id AS transaction_id, + tx.signature AS signature, + tx.slot AS slot, + tx.block_time_unix AS block_time, + tx.err_json AS err_json, + ins.id AS instruction_id, + ins.parent_instruction_id AS parent_instruction_id, + ins.instruction_index AS instruction_index, + ins.inner_instruction_index AS inner_instruction_index, + ins.program_id AS program_id, + ins.accounts_json AS accounts_json, + ins.data_json AS data_json, + de.pool_account AS pool_account, + de.event_kind AS decoded_event_kind, + de.id AS decoded_event_id +FROM k_sol_chain_instructions ins +JOIN k_sol_chain_transactions tx + ON tx.id = ins.transaction_id +LEFT JOIN k_sol_dex_decoded_events de + ON de.transaction_id = tx.id + AND de.instruction_id = ins.id +WHERE tx.signature = ? +ORDER BY ins.instruction_index ASC, ins.inner_instruction_index ASC, ins.id ASC + "#, + ) + .bind(signature.to_string()) + .fetch_all(pool) + .await; + match query_result { + Ok(rows) => return Ok(rows), + Err(error) => { + return Err(crate::Error::Db(format!( + "cannot list instruction observation source rows for signature '{}': {}", + signature, error + ))); + }, + } + }, + } +} + +/// Lists instruction-observation source rows for the local replay window. +pub async fn query_instruction_observation_source_rows_list_replay_window( + database: &crate::Database, + limit: std::option::Option, +) -> Result, crate::Error> { + let effective_limit = match limit { + Some(limit) => { + if limit <= 0 { + 10_000 + } else { + limit + } + }, + None => 10_000, + }; + match database.connection() { + crate::DatabaseConnection::Sqlite(pool) => { + let query_result = + sqlx::query_as::( + r#" +WITH replay_transactions AS ( + SELECT id + FROM k_sol_chain_transactions + ORDER BY id ASC + LIMIT ? +) +SELECT + tx.id AS transaction_id, + tx.signature AS signature, + tx.slot AS slot, + tx.block_time_unix AS block_time, + tx.err_json AS err_json, + ins.id AS instruction_id, + ins.parent_instruction_id AS parent_instruction_id, + ins.instruction_index AS instruction_index, + ins.inner_instruction_index AS inner_instruction_index, + ins.program_id AS program_id, + ins.accounts_json AS accounts_json, + ins.data_json AS data_json, + de.pool_account AS pool_account, + de.event_kind AS decoded_event_kind, + de.id AS decoded_event_id +FROM k_sol_chain_instructions ins +JOIN replay_transactions replay_tx + ON replay_tx.id = ins.transaction_id +JOIN k_sol_chain_transactions tx + ON tx.id = ins.transaction_id +LEFT JOIN k_sol_dex_decoded_events de + ON de.transaction_id = tx.id + AND de.instruction_id = ins.id +ORDER BY tx.id ASC, ins.instruction_index ASC, ins.inner_instruction_index ASC, ins.id ASC + "#, + ) + .bind(effective_limit) + .fetch_all(pool) + .await; + match query_result { + Ok(rows) => return Ok(rows), + Err(error) => { + return Err(crate::Error::Db(format!( + "cannot list instruction observation source rows for replay window: {}", + error + ))); + }, + } + }, + } +} + +/// Lists recent instruction-observation source rows. +pub async fn query_instruction_observation_source_rows_list_recent( + database: &crate::Database, + limit: u32, +) -> Result, crate::Error> { + if limit == 0 { + return Ok(std::vec::Vec::new()); + } + match database.connection() { + crate::DatabaseConnection::Sqlite(pool) => { + let query_result = + sqlx::query_as::( + r#" +SELECT + tx.id AS transaction_id, + tx.signature AS signature, + tx.slot AS slot, + tx.block_time_unix AS block_time, + tx.err_json AS err_json, + ins.id AS instruction_id, + ins.parent_instruction_id AS parent_instruction_id, + ins.instruction_index AS instruction_index, + ins.inner_instruction_index AS inner_instruction_index, + ins.program_id AS program_id, + ins.accounts_json AS accounts_json, + ins.data_json AS data_json, + de.pool_account AS pool_account, + de.event_kind AS decoded_event_kind, + de.id AS decoded_event_id +FROM k_sol_chain_instructions ins +JOIN k_sol_chain_transactions tx + ON tx.id = ins.transaction_id +LEFT JOIN k_sol_dex_decoded_events de + ON de.transaction_id = tx.id + AND de.instruction_id = ins.id +ORDER BY ins.id DESC +LIMIT ? + "#, + ) + .bind(i64::from(limit)) + .fetch_all(pool) + .await; + match query_result { + Ok(rows) => return Ok(rows), + Err(error) => { + return Err(crate::Error::Db(format!( + "cannot list recent instruction observation source rows: {}", + error + ))); + }, + } + }, + } +} diff --git a/kb_lib/src/db/queries/launch_event.rs b/kb_lib/src/db/queries/launch_event.rs new file mode 100644 index 0000000..ee7968b --- /dev/null +++ b/kb_lib/src/db/queries/launch_event.rs @@ -0,0 +1,139 @@ +// file: kb_lib/src/db/queries/launch_event.rs + +//! Queries for `k_sol_launch_events`. + +/// Inserts or updates one launch event by decoded event id. +pub async fn query_launch_events_upsert( + database: &crate::Database, + input: &crate::LaunchEventUpsertInput, +) -> Result { + match database.connection() { + crate::DatabaseConnection::Sqlite(pool) => { + let existing_result = sqlx::query_scalar::( + r#" +SELECT id +FROM k_sol_launch_events +WHERE decoded_event_id = ? +LIMIT 1 + "#, + ) + .bind(input.decoded_event_id) + .fetch_optional(pool) + .await; + let existing_id = match existing_result { + Ok(existing_id) => existing_id, + Err(error) => { + return Err(crate::Error::Db(format!( + "cannot fetch k_sol_launch_events id for decoded_event_id '{}' on sqlite: {}", + input.decoded_event_id, error + ))); + }, + }; + if let Some(existing_id) = existing_id { + let update_result = sqlx::query( + r#" +UPDATE k_sol_launch_events +SET + transaction_id = ?, + dex_id = ?, + pool_id = ?, + pair_id = ?, + signature = ?, + slot = ?, + protocol_name = ?, + program_id = ?, + event_kind = ?, + pool_account = ?, + actor_wallet = ?, + event_role = ?, + related_account = ?, + related_mint = ?, + payload_json = ?, + executed_at = ? +WHERE id = ? + "#, + ) + .bind(input.transaction_id) + .bind(input.dex_id) + .bind(input.pool_id) + .bind(input.pair_id) + .bind(input.signature.clone()) + .bind(input.slot) + .bind(input.protocol_name.clone()) + .bind(input.program_id.clone()) + .bind(input.event_kind.clone()) + .bind(input.pool_account.clone()) + .bind(input.actor_wallet.clone()) + .bind(input.event_role.clone()) + .bind(input.related_account.clone()) + .bind(input.related_mint.clone()) + .bind(input.payload_json.clone()) + .bind(chrono::Utc::now().to_rfc3339()) + .bind(existing_id) + .execute(pool) + .await; + if let Err(error) = update_result { + return Err(crate::Error::Db(format!( + "cannot update k_sol_launch_events id '{}' on sqlite: {}", + existing_id, error + ))); + } + return Ok(existing_id); + } + let insert_result = sqlx::query( + r#" +INSERT INTO k_sol_launch_events ( + transaction_id, + decoded_event_id, + dex_id, + pool_id, + pair_id, + signature, + slot, + protocol_name, + program_id, + event_kind, + pool_account, + actor_wallet, + event_role, + related_account, + related_mint, + payload_json, + executed_at, + created_at +) +VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + "#, + ) + .bind(input.transaction_id) + .bind(input.decoded_event_id) + .bind(input.dex_id) + .bind(input.pool_id) + .bind(input.pair_id) + .bind(input.signature.clone()) + .bind(input.slot) + .bind(input.protocol_name.clone()) + .bind(input.program_id.clone()) + .bind(input.event_kind.clone()) + .bind(input.pool_account.clone()) + .bind(input.actor_wallet.clone()) + .bind(input.event_role.clone()) + .bind(input.related_account.clone()) + .bind(input.related_mint.clone()) + .bind(input.payload_json.clone()) + .bind(chrono::Utc::now().to_rfc3339()) + .bind(chrono::Utc::now().to_rfc3339()) + .execute(pool) + .await; + match insert_result { + Ok(result) => return Ok(result.last_insert_rowid()), + Err(error) => { + return Err(crate::Error::Db(format!( + "cannot insert k_sol_launch_events on sqlite: {}", + error + ))); + }, + } + }, + } +} diff --git a/kb_lib/src/db/queries/pool_admin_event.rs b/kb_lib/src/db/queries/pool_admin_event.rs index 504ede4..cf91d8e 100644 --- a/kb_lib/src/db/queries/pool_admin_event.rs +++ b/kb_lib/src/db/queries/pool_admin_event.rs @@ -291,3 +291,32 @@ LIMIT ? }, } } + +/// Deletes a stale pool admin materialization by decoded event id. +pub async fn query_pool_admin_events_delete_by_decoded_event_id( + database: &crate::Database, + decoded_event_id: i64, +) -> Result { + match database.connection() { + crate::DatabaseConnection::Sqlite(pool) => { + let delete_result = sqlx::query( + r#" +DELETE FROM k_sol_pool_admin_events +WHERE decoded_event_id = ? + "#, + ) + .bind(decoded_event_id) + .execute(pool) + .await; + match delete_result { + Ok(delete_result) => return Ok(delete_result.rows_affected()), + Err(error) => { + return Err(crate::Error::Db(format!( + "cannot delete k_sol_pool_admin_events for decoded_event_id '{}' on sqlite: {}", + decoded_event_id, error + ))); + }, + } + }, + } +} diff --git a/kb_lib/src/db/queries/program_instruction_diagnostic.rs b/kb_lib/src/db/queries/program_instruction_diagnostic.rs index c2a09f8..98435d9 100644 --- a/kb_lib/src/db/queries/program_instruction_diagnostic.rs +++ b/kb_lib/src/db/queries/program_instruction_diagnostic.rs @@ -73,39 +73,12 @@ LIMIT ? } } -#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] -struct ProgramInstructionDiscriminatorSummaryKey { - program_id: std::string::String, - discriminator_hex: std::option::Option, - accounts_count: u64, - stack_height: std::option::Option, - is_inner_instruction: bool, -} - -#[derive(Debug, Clone)] -struct ProgramInstructionDiscriminatorSummaryAccumulator { - key: ProgramInstructionDiscriminatorSummaryKey, - known_instruction_name: std::option::Option, - occurrence_count: u64, - decoded_event_count: u64, - transaction_signatures: std::collections::BTreeSet, - latest_slot: std::option::Option, - latest_signature: std::string::String, - latest_instruction_id: i64, - latest_instruction_index: u32, - latest_inner_instruction_index: std::option::Option, - latest_parsed_type: std::option::Option, - latest_decoded_event_kind: std::option::Option, - latest_data_json_preview: std::option::Option, - latest_accounts_json_preview: std::option::Option, -} - fn build_program_instruction_discriminator_summaries( rows: std::vec::Vec, ) -> Result, crate::Error> { let mut grouped = std::collections::BTreeMap::< - ProgramInstructionDiscriminatorSummaryKey, - ProgramInstructionDiscriminatorSummaryAccumulator, + crate::db::dtos::ProgramInstructionDiscriminatorSummaryKey, + crate::db::dtos::ProgramInstructionDiscriminatorSummaryAccumulator, >::new(); for row in rows { let summary_row_result = build_summary_row_from_discriminator_entity(row); @@ -189,7 +162,7 @@ fn build_program_instruction_discriminator_summaries( fn build_summary_row_from_discriminator_entity( row: crate::ProgramInstructionDiscriminatorRowEntity, -) -> Result { +) -> Result { let program_id = match row.program_id.clone() { Some(program_id) => program_id, None => "unknown".to_string(), @@ -252,14 +225,14 @@ fn build_summary_row_from_discriminator_entity( let mut transaction_signatures = std::collections::BTreeSet::new(); transaction_signatures.insert(row.signature.clone()); let decoded_event_count = if row.decoded_event_id.is_some() { 1_u64 } else { 0_u64 }; - let key = ProgramInstructionDiscriminatorSummaryKey { + let key = crate::db::dtos::ProgramInstructionDiscriminatorSummaryKey { program_id, discriminator_hex, accounts_count, stack_height, is_inner_instruction: row.parent_instruction_id.is_some(), }; - return Ok(ProgramInstructionDiscriminatorSummaryAccumulator { + return Ok(crate::db::dtos::ProgramInstructionDiscriminatorSummaryAccumulator { key, known_instruction_name, occurrence_count: 1, diff --git a/kb_lib/src/dex.rs b/kb_lib/src/dex.rs index 496c9bc..5cc9f67 100644 --- a/kb_lib/src/dex.rs +++ b/kb_lib/src/dex.rs @@ -70,6 +70,7 @@ pub use pump_swap::PumpSwapTradeDecoded; pub use raydium_amm_v4::RaydiumAmmV4DecodedEvent; pub use raydium_amm_v4::RaydiumAmmV4Decoder; pub use raydium_amm_v4::RaydiumAmmV4Initialize2PoolDecoded; +pub use raydium_amm_v4::RaydiumAmmV4InstructionDecoded; pub use raydium_amm_v4::RaydiumAmmV4SwapDecoded; pub use raydium_clmm::RaydiumClmmCollectProtocolFeeDecoded; pub use raydium_clmm::RaydiumClmmCreatePoolDecoded; diff --git a/kb_lib/src/dex/raydium_amm_v4.rs b/kb_lib/src/dex/raydium_amm_v4.rs index 702c4df..ffec570 100644 --- a/kb_lib/src/dex/raydium_amm_v4.rs +++ b/kb_lib/src/dex/raydium_amm_v4.rs @@ -40,6 +40,12 @@ pub struct RaydiumAmmV4SwapDecoded { pub signature: std::string::String, /// Program id. pub program_id: std::string::String, + /// Local decoded event kind derived from the AMM v4 instruction discriminator. + pub event_kind: std::string::String, + /// Upstream instruction name derived from the AMM v4 instruction discriminator. + pub instruction_name: std::string::String, + /// One-byte Raydium AMM v4 instruction discriminator in hex form. + pub discriminator_hex: std::string::String, /// AMM pool/state account. pub pool_account: std::string::String, /// Raydium AMM authority account. @@ -78,6 +84,49 @@ pub struct RaydiumAmmV4SwapDecoded { pub payload_json: serde_json::Value, } +/// Decoded Raydium AmmV4 non-swap or decoded-only instruction event. +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +pub struct RaydiumAmmV4InstructionDecoded { + /// Parent transaction id. + pub transaction_id: i64, + /// Parent instruction id. + pub instruction_id: i64, + /// Transaction signature. + pub signature: std::string::String, + /// Program id. + pub program_id: std::string::String, + /// Local decoded event kind derived from the AMM v4 instruction discriminator. + pub event_kind: std::string::String, + /// Upstream instruction name derived from the AMM v4 instruction discriminator. + pub instruction_name: std::string::String, + /// One-byte Raydium AMM v4 instruction discriminator in hex form. + pub discriminator_hex: std::string::String, + /// Optional AMM pool/state account. + pub pool_account: std::option::Option, + /// Optional Raydium AMM authority account. + pub authority: std::option::Option, + /// Optional AMM open-orders account. + pub open_orders: std::option::Option, + /// Optional AMM target-orders account. + pub target_orders: std::option::Option, + /// Optional market program account. + pub market_program: std::option::Option, + /// Optional market account. + pub market_account: std::option::Option, + /// Optional LP mint account. + pub lp_mint: std::option::Option, + /// Optional token A mint after best-effort account layout extraction. + pub token_a_mint: std::option::Option, + /// Optional token B mint after best-effort account layout extraction. + pub token_b_mint: std::option::Option, + /// Optional AMM coin/base vault account. + pub base_vault: std::option::Option, + /// Optional AMM pc/quote vault account. + pub quote_vault: std::option::Option, + /// Decoded payload. + pub payload_json: serde_json::Value, +} + /// Decoded Raydium AmmV4 event. #[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] pub enum RaydiumAmmV4DecodedEvent { @@ -85,6 +134,8 @@ pub enum RaydiumAmmV4DecodedEvent { Initialize2Pool(std::boxed::Box), /// Swap event decoded from a direct or inner Raydium AMM v4 instruction. Swap(std::boxed::Box), + /// Known Raydium AMM v4 instruction decoded without direct trade materialization. + Instruction(std::boxed::Box), } /// Raydium AmmV4 decoder. @@ -182,11 +233,27 @@ impl RaydiumAmmV4Decoder { token_balances.as_slice(), ); match swap_result { - Ok(Some(swap)) => decoded_events - .push(crate::RaydiumAmmV4DecodedEvent::Swap(std::boxed::Box::new(swap))), + Ok(Some(swap)) => { + decoded_events + .push(crate::RaydiumAmmV4DecodedEvent::Swap(std::boxed::Box::new(swap))); + continue; + }, Ok(None) => {}, Err(error) => return Err(error), } + let instruction_event = decode_known_instruction_event( + transaction, + transaction_id, + instruction, + instruction_id, + program_id, + accounts.as_slice(), + ); + if let Some(instruction_event) = instruction_event { + decoded_events.push(crate::RaydiumAmmV4DecodedEvent::Instruction( + std::boxed::Box::new(instruction_event), + )); + } } return Ok(decoded_events); } @@ -209,9 +276,15 @@ fn decode_initialize2_event( let token_a_mint = extract_account(accounts, 8); let token_b_mint = extract_account(accounts, 9); let market_account = extract_account(accounts, 16); + let data_base58 = parse_optional_data_json_as_base58(instruction.data_json.as_deref()); + let discriminator_hex = raydium_amm_v4_instruction_discriminator_hex(data_base58.as_deref()); let payload_json = serde_json::json!({ "decoder": "raydium_amm_v4", "eventKind": "initialize2_pool", + "instructionName": "initialize2", + "upstreamInstructionName": "initialize2", + "discriminatorHex": discriminator_hex, + "instructionDiscriminatorHex": discriminator_hex, "signature": transaction.signature, "instructionId": instruction_id, "instructionIndex": instruction.instruction_index, @@ -251,6 +324,11 @@ fn decode_swap_event( if accounts.len() < 8 { return Ok(None); } + let data_base58 = parse_optional_data_json_as_base58(instruction.data_json.as_deref()); + let swap_instruction = match raydium_amm_v4_swap_instruction(data_base58.as_deref()) { + Some(swap_instruction) => swap_instruction, + None => return Ok(None), + }; let pool_account = match extract_account(accounts, 1) { Some(pool_account) => pool_account, None => return Ok(None), @@ -294,11 +372,14 @@ fn decode_swap_event( normalized_pair.quote_vault.as_str(), ); let parent_program = parent_program_id_for_instruction(instruction, transaction_instructions); - let data_base58 = parse_optional_data_json_as_base58(instruction.data_json.as_deref()); let trade_candidate = base_amount_raw.is_some() && quote_amount_raw.is_some(); let payload_json = serde_json::json!({ "decoder": "raydium_amm_v4", - "eventKind": "swap", + "eventKind": swap_instruction.event_kind, + "instructionName": swap_instruction.instruction_name, + "upstreamInstructionName": swap_instruction.instruction_name, + "discriminatorHex": swap_instruction.discriminator_hex, + "instructionDiscriminatorHex": swap_instruction.discriminator_hex, "signature": transaction.signature, "instructionId": instruction_id, "instructionIndex": instruction.instruction_index, @@ -337,6 +418,9 @@ fn decode_swap_event( instruction_id, signature: transaction.signature.clone(), program_id: program_id.to_string(), + event_kind: swap_instruction.event_kind.to_string(), + instruction_name: swap_instruction.instruction_name.to_string(), + discriminator_hex: swap_instruction.discriminator_hex.to_string(), pool_account, authority, token_a_mint: normalized_pair.base_mint, @@ -358,6 +442,439 @@ fn decode_swap_event( })); } +fn decode_known_instruction_event( + transaction: &crate::ChainTransactionDto, + transaction_id: i64, + instruction: &crate::ChainInstructionDto, + instruction_id: i64, + program_id: &str, + accounts: &[std::string::String], +) -> std::option::Option { + let data_base58 = parse_optional_data_json_as_base58(instruction.data_json.as_deref()); + let data_bytes = raydium_amm_v4_instruction_data_bytes(data_base58.as_deref()); + let identity = match raydium_amm_v4_instruction_identity(data_base58.as_deref()) { + Some(identity) => identity, + None => return None, + }; + let account_layout = raydium_amm_v4_account_layout(identity.discriminator_hex); + let pool_account = extract_account_by_index(accounts, account_layout.pool_account_index); + let authority = extract_account_by_index(accounts, account_layout.authority_index); + let open_orders = extract_account_by_index(accounts, account_layout.open_orders_index); + let target_orders = extract_account_by_index(accounts, account_layout.target_orders_index); + let market_program = extract_account_by_index(accounts, account_layout.market_program_index); + let market_account = extract_account_by_index(accounts, account_layout.market_account_index); + let lp_mint = extract_account_by_index(accounts, account_layout.lp_mint_index); + let token_a_mint = extract_account_by_index(accounts, account_layout.token_a_mint_index); + let token_b_mint = extract_account_by_index(accounts, account_layout.token_b_mint_index); + let base_vault = extract_account_by_index(accounts, account_layout.base_vault_index); + let quote_vault = extract_account_by_index(accounts, account_layout.quote_vault_index); + let mut payload_json = serde_json::json!({ + "decoder": "raydium_amm_v4", + "eventKind": identity.event_kind, + "instructionName": identity.instruction_name, + "upstreamInstructionName": identity.instruction_name, + "discriminatorHex": identity.discriminator_hex, + "instructionDiscriminatorHex": identity.discriminator_hex, + "signature": transaction.signature, + "instructionId": instruction_id, + "instructionIndex": instruction.instruction_index, + "innerInstructionIndex": instruction.inner_instruction_index, + "innerInstruction": instruction.inner_instruction_index.is_some(), + "parentInstructionId": instruction.parent_instruction_id, + "programId": program_id, + "accounts": accounts, + "data": data_base58, + "instructionDataLength": data_bytes.as_ref().map(std::vec::Vec::len), + "poolAccount": pool_account.clone(), + "authority": authority.clone(), + "openOrders": open_orders.clone(), + "targetOrders": target_orders.clone(), + "marketProgram": market_program.clone(), + "marketAccount": market_account.clone(), + "lpMint": lp_mint.clone(), + "tokenAMint": token_a_mint.clone(), + "tokenBMint": token_b_mint.clone(), + "baseVault": base_vault.clone(), + "quoteVault": quote_vault.clone(), + "tradeCandidate": false, + "candleCandidate": false, + "skipTradeReason": "decoded_only_instruction", + "skipCandleReason": "decoded_only_instruction" + }); + enrich_known_instruction_payload(&mut payload_json, identity.discriminator_hex, data_bytes.as_deref()); + return Some(crate::RaydiumAmmV4InstructionDecoded { + transaction_id, + instruction_id, + signature: transaction.signature.clone(), + program_id: program_id.to_string(), + event_kind: identity.event_kind.to_string(), + instruction_name: identity.instruction_name.to_string(), + discriminator_hex: identity.discriminator_hex.to_string(), + pool_account, + authority, + open_orders, + target_orders, + market_program, + market_account, + lp_mint, + token_a_mint, + token_b_mint, + base_vault, + quote_vault, + payload_json, + }); +} + +#[derive(Clone, Copy)] +struct RaydiumAmmV4AccountLayout { + pool_account_index: std::option::Option, + authority_index: std::option::Option, + open_orders_index: std::option::Option, + target_orders_index: std::option::Option, + market_program_index: std::option::Option, + market_account_index: std::option::Option, + lp_mint_index: std::option::Option, + token_a_mint_index: std::option::Option, + token_b_mint_index: std::option::Option, + base_vault_index: std::option::Option, + quote_vault_index: std::option::Option, +} + +fn raydium_amm_v4_empty_account_layout() -> RaydiumAmmV4AccountLayout { + return RaydiumAmmV4AccountLayout { + pool_account_index: None, + authority_index: None, + open_orders_index: None, + target_orders_index: None, + market_program_index: None, + market_account_index: None, + lp_mint_index: None, + token_a_mint_index: None, + token_b_mint_index: None, + base_vault_index: None, + quote_vault_index: None, + }; +} + +fn raydium_amm_v4_account_layout(discriminator_hex: &str) -> RaydiumAmmV4AccountLayout { + let mut layout = raydium_amm_v4_empty_account_layout(); + match discriminator_hex { + "00" => { + layout.pool_account_index = Some(3); + }, + "01" => { + layout.pool_account_index = Some(4); + layout.authority_index = Some(5); + layout.open_orders_index = Some(6); + layout.lp_mint_index = Some(7); + layout.token_a_mint_index = Some(8); + layout.token_b_mint_index = Some(9); + layout.base_vault_index = Some(10); + layout.quote_vault_index = Some(11); + layout.target_orders_index = Some(12); + layout.market_program_index = Some(15); + layout.market_account_index = Some(16); + }, + "02" => { + layout.pool_account_index = Some(3); + layout.authority_index = Some(4); + layout.open_orders_index = Some(5); + layout.target_orders_index = Some(6); + layout.base_vault_index = Some(7); + layout.quote_vault_index = Some(8); + layout.market_program_index = Some(9); + layout.market_account_index = Some(10); + }, + "03" => { + layout.pool_account_index = Some(1); + layout.authority_index = Some(2); + layout.open_orders_index = Some(3); + layout.target_orders_index = Some(4); + layout.lp_mint_index = Some(5); + layout.base_vault_index = Some(6); + layout.quote_vault_index = Some(7); + layout.market_account_index = Some(8); + }, + "04" => { + layout.pool_account_index = Some(1); + layout.authority_index = Some(2); + layout.open_orders_index = Some(3); + layout.target_orders_index = Some(4); + layout.lp_mint_index = Some(5); + layout.base_vault_index = Some(6); + layout.quote_vault_index = Some(7); + layout.market_program_index = Some(8); + layout.market_account_index = Some(9); + }, + "05" => { + layout.pool_account_index = Some(3); + layout.authority_index = Some(4); + layout.open_orders_index = Some(5); + layout.base_vault_index = Some(6); + layout.quote_vault_index = Some(7); + layout.target_orders_index = Some(8); + layout.market_program_index = Some(9); + layout.market_account_index = Some(10); + }, + "06" => { + layout.pool_account_index = Some(1); + layout.authority_index = Some(2); + layout.open_orders_index = Some(3); + layout.target_orders_index = Some(4); + layout.base_vault_index = Some(5); + layout.quote_vault_index = Some(6); + layout.market_program_index = Some(7); + layout.market_account_index = Some(8); + }, + "07" => { + layout.pool_account_index = Some(1); + layout.authority_index = Some(3); + layout.open_orders_index = Some(4); + layout.base_vault_index = Some(5); + layout.quote_vault_index = Some(6); + layout.target_orders_index = Some(10); + layout.market_program_index = Some(11); + layout.market_account_index = Some(12); + }, + "08" => { + layout.pool_account_index = Some(1); + layout.authority_index = Some(3); + }, + "09" | "0b" => { + layout.pool_account_index = Some(1); + layout.authority_index = Some(2); + layout.open_orders_index = Some(3); + layout.target_orders_index = Some(4); + layout.base_vault_index = Some(5); + layout.quote_vault_index = Some(6); + layout.market_program_index = Some(7); + layout.market_account_index = Some(8); + }, + "0a" => { + layout.pool_account_index = Some(4); + }, + "0c" => { + layout.pool_account_index = Some(1); + }, + "0d" => { + layout.pool_account_index = Some(1); + }, + "10" | "11" => { + layout.pool_account_index = Some(1); + layout.authority_index = Some(2); + layout.base_vault_index = Some(3); + layout.quote_vault_index = Some(4); + }, + _ => {}, + } + return layout; +} + +fn extract_account_by_index( + accounts: &[std::string::String], + index: std::option::Option, +) -> std::option::Option { + let index = match index { + Some(index) => index, + None => return None, + }; + return extract_account(accounts, index); +} + +fn raydium_amm_v4_instruction_data_bytes( + data_base58: std::option::Option<&str>, +) -> std::option::Option> { + let data_base58 = match data_base58 { + Some(data_base58) => data_base58, + None => return None, + }; + let bytes_result = bs58::decode(data_base58).into_vec(); + match bytes_result { + Ok(bytes) => return Some(bytes), + Err(_) => return None, + } +} + +fn enrich_known_instruction_payload( + payload_json: &mut serde_json::Value, + discriminator_hex: &str, + data: std::option::Option<&[u8]>, +) { + let data = match data { + Some(data) => data, + None => return, + }; + match discriminator_hex { + "00" => { + insert_u8(payload_json, "nonce", data, 1); + insert_u64_string(payload_json, "openTime", data, 2); + }, + "01" => { + insert_u8(payload_json, "nonce", data, 1); + insert_u64_string(payload_json, "openTime", data, 2); + insert_u64_string(payload_json, "initPcAmount", data, 10); + insert_u64_string(payload_json, "initCoinAmount", data, 18); + insert_u64_string(payload_json, "tokenAAmount", data, 18); + insert_u64_string(payload_json, "tokenBAmount", data, 10); + }, + "02" => { + insert_u16(payload_json, "planOrderLimit", data, 1); + insert_u16(payload_json, "placeOrderLimit", data, 3); + insert_u16(payload_json, "cancelOrderLimit", data, 5); + }, + "03" => { + insert_u64_string(payload_json, "maxCoinAmount", data, 1); + insert_u64_string(payload_json, "maxPcAmount", data, 9); + insert_u64_string(payload_json, "baseSide", data, 17); + insert_u64_string(payload_json, "otherAmountMin", data, 25); + insert_u64_string(payload_json, "tokenAAmount", data, 1); + insert_u64_string(payload_json, "tokenBAmount", data, 9); + }, + "04" => { + insert_u64_string(payload_json, "lpAmountRaw", data, 1); + insert_u64_string(payload_json, "liquidity", data, 1); + insert_u64_string(payload_json, "minCoinAmount", data, 9); + insert_u64_string(payload_json, "minPcAmount", data, 17); + }, + "06" => { + insert_u8(payload_json, "configParam", data, 1); + insert_u64_string(payload_json, "configValue", data, 2); + insert_fixed_hex(payload_json, "configPubkeyHex", data, 2, 32); + insert_u64_string(payload_json, "lastOrderNumerator", data, 2); + insert_u64_string(payload_json, "lastOrderDenominator", data, 10); + }, + "08" => { + insert_u64_string(payload_json, "amountRaw", data, 1); + }, + "09" | "10" => { + insert_u64_string(payload_json, "amountIn", data, 1); + insert_u64_string(payload_json, "minimumAmountOut", data, 9); + }, + "0a" => { + insert_u8(payload_json, "nonce", data, 1); + }, + "0b" | "11" => { + insert_u64_string(payload_json, "maxAmountIn", data, 1); + insert_u64_string(payload_json, "amountOut", data, 9); + }, + "0c" => { + insert_u8(payload_json, "simulateParam", data, 1); + insert_u64_string(payload_json, "amountIn", data, 2); + insert_u64_string(payload_json, "minimumAmountOut", data, 10); + insert_u64_string(payload_json, "maxAmountIn", data, 2); + insert_u64_string(payload_json, "amountOut", data, 10); + }, + "0d" => { + insert_u16(payload_json, "orderCancelLimit", data, 1); + }, + "0f" => { + insert_u8(payload_json, "configParam", data, 1); + insert_fixed_hex(payload_json, "configOwnerHex", data, 2, 32); + insert_u64_string(payload_json, "createPoolFee", data, 2); + }, + _ => {}, + } +} + +fn insert_u8(payload_json: &mut serde_json::Value, key: &str, data: &[u8], offset: usize) { + let value = match read_u8(data, offset) { + Some(value) => value, + None => return, + }; + insert_json_value( + payload_json, + key, + serde_json::Value::Number(serde_json::Number::from(value as u64)), + ); +} + +fn insert_u16(payload_json: &mut serde_json::Value, key: &str, data: &[u8], offset: usize) { + let value = match read_u16_le(data, offset) { + Some(value) => value, + None => return, + }; + insert_json_value( + payload_json, + key, + serde_json::Value::Number(serde_json::Number::from(value as u64)), + ); +} + +fn insert_u64_string(payload_json: &mut serde_json::Value, key: &str, data: &[u8], offset: usize) { + let value = match read_u64_le(data, offset) { + Some(value) => value, + None => return, + }; + insert_json_value( + payload_json, + key, + serde_json::Value::String(value.to_string()), + ); +} + +fn insert_fixed_hex( + payload_json: &mut serde_json::Value, + key: &str, + data: &[u8], + offset: usize, + len: usize, +) { + if data.len() < offset + len { + return; + } + let slice = &data[offset..offset + len]; + insert_json_value(payload_json, key, serde_json::Value::String(bytes_to_hex(slice))); +} + +fn insert_json_value(payload_json: &mut serde_json::Value, key: &str, value: serde_json::Value) { + let object_option = payload_json.as_object_mut(); + let object = match object_option { + Some(object) => object, + None => return, + }; + object.insert(key.to_string(), value); +} + +fn read_u8(data: &[u8], offset: usize) -> std::option::Option { + if data.len() < offset + 1 { + return None; + } + return Some(data[offset]); +} + +fn read_u16_le(data: &[u8], offset: usize) -> std::option::Option { + if data.len() < offset + 2 { + return None; + } + let bytes = [data[offset], data[offset + 1]]; + return Some(u16::from_le_bytes(bytes)); +} + +fn read_u64_le(data: &[u8], offset: usize) -> std::option::Option { + if data.len() < offset + 8 { + return None; + } + let bytes = [ + data[offset], + data[offset + 1], + data[offset + 2], + data[offset + 3], + data[offset + 4], + data[offset + 5], + data[offset + 6], + data[offset + 7], + ]; + return Some(u64::from_le_bytes(bytes)); +} + +fn bytes_to_hex(bytes: &[u8]) -> std::string::String { + let mut output = std::string::String::new(); + for byte in bytes { + output.push_str(format!("{byte:02x}").as_str()); + } + return output; +} + #[derive(Debug, Clone)] struct TokenBalanceRecord { account_address: std::option::Option, @@ -401,6 +918,183 @@ struct NormalizedVaultPair { quote_mint: std::string::String, } +#[derive(Clone, Copy)] +struct RaydiumAmmV4InstructionIdentity { + instruction_name: &'static str, + event_kind: &'static str, + discriminator_hex: &'static str, +} + +fn raydium_amm_v4_swap_instruction( + data_base58: std::option::Option<&str>, +) -> std::option::Option { + let identity = match raydium_amm_v4_instruction_identity(data_base58) { + Some(identity) => identity, + None => return None, + }; + match identity.discriminator_hex { + "09" | "0b" | "10" | "11" => return Some(identity), + _ => return None, + } +} + +fn raydium_amm_v4_instruction_identity( + data_base58: std::option::Option<&str>, +) -> std::option::Option { + let discriminator_hex = match raydium_amm_v4_instruction_discriminator_hex(data_base58) { + Some(discriminator_hex) => discriminator_hex, + None => return None, + }; + match discriminator_hex.as_str() { + "00" => { + return Some(RaydiumAmmV4InstructionIdentity { + instruction_name: "initialize", + event_kind: "raydium_amm_v4.initialize", + discriminator_hex: "00", + }); + }, + "01" => { + return Some(RaydiumAmmV4InstructionIdentity { + instruction_name: "initialize2", + event_kind: "raydium_amm_v4.initialize2_pool", + discriminator_hex: "01", + }); + }, + "02" => { + return Some(RaydiumAmmV4InstructionIdentity { + instruction_name: "monitor_step", + event_kind: "raydium_amm_v4.monitor_step", + discriminator_hex: "02", + }); + }, + "03" => { + return Some(RaydiumAmmV4InstructionIdentity { + instruction_name: "deposit", + event_kind: "raydium_amm_v4.deposit", + discriminator_hex: "03", + }); + }, + "04" => { + return Some(RaydiumAmmV4InstructionIdentity { + instruction_name: "withdraw", + event_kind: "raydium_amm_v4.withdraw", + discriminator_hex: "04", + }); + }, + "05" => { + return Some(RaydiumAmmV4InstructionIdentity { + instruction_name: "migrate_to_open_book", + event_kind: "raydium_amm_v4.migrate_to_open_book", + discriminator_hex: "05", + }); + }, + "06" => { + return Some(RaydiumAmmV4InstructionIdentity { + instruction_name: "set_params", + event_kind: "raydium_amm_v4.set_params", + discriminator_hex: "06", + }); + }, + "07" => { + return Some(RaydiumAmmV4InstructionIdentity { + instruction_name: "withdraw_pnl", + event_kind: "raydium_amm_v4.withdraw_pnl", + discriminator_hex: "07", + }); + }, + "08" => { + return Some(RaydiumAmmV4InstructionIdentity { + instruction_name: "withdraw_srm", + event_kind: "raydium_amm_v4.withdraw_srm", + discriminator_hex: "08", + }); + }, + "09" => { + return Some(RaydiumAmmV4InstructionIdentity { + instruction_name: "swap_base_in", + event_kind: "raydium_amm_v4.swap_base_in", + discriminator_hex: "09", + }); + }, + "0a" => { + return Some(RaydiumAmmV4InstructionIdentity { + instruction_name: "pre_initialize", + event_kind: "raydium_amm_v4.pre_initialize", + discriminator_hex: "0a", + }); + }, + "0b" => { + return Some(RaydiumAmmV4InstructionIdentity { + instruction_name: "swap_base_out", + event_kind: "raydium_amm_v4.swap_base_out", + discriminator_hex: "0b", + }); + }, + "0c" => { + return Some(RaydiumAmmV4InstructionIdentity { + instruction_name: "simulate_info", + event_kind: "raydium_amm_v4.simulate_info", + discriminator_hex: "0c", + }); + }, + "0d" => { + return Some(RaydiumAmmV4InstructionIdentity { + instruction_name: "admin_cancel_orders", + event_kind: "raydium_amm_v4.admin_cancel_orders", + discriminator_hex: "0d", + }); + }, + "0e" => { + return Some(RaydiumAmmV4InstructionIdentity { + instruction_name: "create_config_account", + event_kind: "raydium_amm_v4.create_config_account", + discriminator_hex: "0e", + }); + }, + "0f" => { + return Some(RaydiumAmmV4InstructionIdentity { + instruction_name: "update_config_account", + event_kind: "raydium_amm_v4.update_config_account", + discriminator_hex: "0f", + }); + }, + "10" => { + return Some(RaydiumAmmV4InstructionIdentity { + instruction_name: "swap_base_in_v2", + event_kind: "raydium_amm_v4.swap_base_in_v2", + discriminator_hex: "10", + }); + }, + "11" => { + return Some(RaydiumAmmV4InstructionIdentity { + instruction_name: "swap_base_out_v2", + event_kind: "raydium_amm_v4.swap_base_out_v2", + discriminator_hex: "11", + }); + }, + _ => return None, + } +} + +fn raydium_amm_v4_instruction_discriminator_hex( + data_base58: std::option::Option<&str>, +) -> std::option::Option { + let data_base58 = match data_base58 { + Some(data_base58) => data_base58, + None => return None, + }; + let bytes_result = bs58::decode(data_base58).into_vec(); + let bytes = match bytes_result { + Ok(bytes) => bytes, + Err(_) => return None, + }; + let first_byte = match bytes.first() { + Some(first_byte) => first_byte, + None => return None, + }; + return Some(format!("{first_byte:02x}")); +} + fn parse_transaction_meta_value( transaction: &crate::ChainTransactionDto, transaction_json: &serde_json::Value, @@ -1209,6 +1903,14 @@ mod tests { } fn make_swap_instruction() -> crate::ChainInstructionDto { + return make_swap_instruction_with_data("63SfuT4qF7xK35jRTGqxuUT"); + } + + fn make_swap_v2_instruction() -> crate::ChainInstructionDto { + return make_swap_instruction_with_data("9rj8cBJgMm4L1xvzfy5AUsy"); + } + + fn make_swap_instruction_with_data(data_base58: &str) -> crate::ChainInstructionDto { let mut dto = crate::ChainInstructionDto::new( 100, Some(55), @@ -1228,7 +1930,7 @@ mod tests { "UserQuote111" ]) .to_string(), - Some(serde_json::json!("9rj8cBJgMm4L1xvzfy5AUsy").to_string()), + Some(serde_json::json!(data_base58).to_string()), None, None, ); @@ -1305,6 +2007,9 @@ mod tests { assert_eq!(event.transaction_id, 100); assert_eq!(event.instruction_id, 200); assert_eq!(event.pool_account, "Pool111".to_string()); + assert_eq!(event.event_kind, "raydium_amm_v4.swap_base_in".to_string()); + assert_eq!(event.instruction_name, "swap_base_in".to_string()); + assert_eq!(event.discriminator_hex, "09".to_string()); assert_eq!(event.token_a_mint, "BaseMint111".to_string()); assert_eq!(event.token_b_mint, crate::WSOL_MINT_ID.to_string()); assert_eq!(event.base_vault, "BaseVault111".to_string()); @@ -1320,4 +2025,29 @@ mod tests { _ => panic!("expected swap event"), } } + #[test] + fn raydium_amm_v4_swap_base_in_v2_is_decoded_from_inner_instruction_and_vault_deltas() { + let decoder = crate::RaydiumAmmV4Decoder::new(); + let transaction = make_swap_transaction(); + let instructions = vec![make_swap_v2_instruction()]; + let decoded_result = decoder.decode_transaction(&transaction, &instructions); + let decoded = match decoded_result { + Ok(decoded) => decoded, + Err(error) => panic!("decode must succeed: {}", error), + }; + assert_eq!(decoded.len(), 1); + match &decoded[0] { + crate::RaydiumAmmV4DecodedEvent::Swap(event) => { + assert_eq!(event.event_kind, "raydium_amm_v4.swap_base_in_v2".to_string()); + assert_eq!(event.instruction_name, "swap_base_in_v2".to_string()); + assert_eq!(event.discriminator_hex, "10".to_string()); + assert_eq!(event.pool_account, "Pool111".to_string()); + assert_eq!(event.base_amount_raw, Some("100".to_string())); + assert_eq!(event.quote_amount_raw, Some("100".to_string())); + assert_eq!(event.trade_side, Some("BuyBase".to_string())); + }, + _ => panic!("expected swap event"), + } + } + } diff --git a/kb_lib/src/dex_decode.rs b/kb_lib/src/dex_decode.rs index c5ed817..2c41ea0 100644 --- a/kb_lib/src/dex_decode.rs +++ b/kb_lib/src/dex_decode.rs @@ -523,43 +523,16 @@ impl DexDecodeService { transaction_id: i64, discriminator_hex: &str, ) -> Result<(), crate::Error> { - match self.database.connection() { - crate::DatabaseConnection::Sqlite(pool) => { - let delete_result = sqlx::query( - r#" -DELETE FROM k_sol_dex_decoded_events -WHERE transaction_id = ? - AND protocol_name = 'raydium_clmm' - AND event_kind = 'raydium_clmm.instruction_audit' - AND ( - json_extract(payload_json, '$.discriminatorHex') = ? - OR json_extract(payload_json, '$.discriminator_hex') = ? - OR json_extract(payload_json, '$.instructionDiscriminatorHex') = ? - OR json_extract(payload_json, '$.instruction_discriminator_hex') = ? - OR json_extract(payload_json, '$.anchorEventDiscriminatorHex') = ? - OR json_extract(payload_json, '$.anchor_event_discriminator_hex') = ? - ) - "#, - ) - .bind(transaction_id) - .bind(discriminator_hex.to_string()) - .bind(discriminator_hex.to_string()) - .bind(discriminator_hex.to_string()) - .bind(discriminator_hex.to_string()) - .bind(discriminator_hex.to_string()) - .bind(discriminator_hex.to_string()) - .execute(pool) - .await; - match delete_result { - Ok(_) => return Ok(()), - Err(error) => { - return Err(crate::Error::Db(format!( - "cannot delete Raydium CLMM residual instruction audit '{}': {}", - discriminator_hex, error - ))); - }, - } - }, + let delete_result = + crate::query_dex_decoded_events_delete_raydium_clmm_instruction_audit_by_discriminator( + self.database.as_ref(), + transaction_id, + discriminator_hex, + ) + .await; + match delete_result { + Ok(_) => return Ok(()), + Err(error) => return Err(error), } } @@ -706,88 +679,18 @@ WHERE transaction_id = ? Some(audit_event_kind) => audit_event_kind, None => return Ok(()), }; - match self.database.connection() { - crate::DatabaseConnection::Sqlite(pool) => { - let unlink_result = sqlx::query( - r#" -UPDATE k_sol_instruction_observations -SET decoded_event_id = NULL -WHERE decoded_event_id IN ( - SELECT id - FROM k_sol_dex_decoded_events - WHERE transaction_id = ? - AND protocol_name = ? - AND event_kind = ? - AND ( - json_extract(payload_json, '$.discriminatorHex') = ? - OR json_extract(payload_json, '$.discriminator_hex') = ? - OR json_extract(payload_json, '$.instructionDiscriminatorHex') = ? - OR json_extract(payload_json, '$.instruction_discriminator_hex') = ? - OR json_extract(payload_json, '$.anchorEventDiscriminatorHex') = ? - OR json_extract(payload_json, '$.anchor_event_discriminator_hex') = ? - OR instr(lower(COALESCE(payload_json, '')), lower(?)) > 0 - ) -) - "#, - ) - .bind(transaction_id) - .bind(protocol_name.to_string()) - .bind(audit_event_kind.to_string()) - .bind(discriminator_hex.to_string()) - .bind(discriminator_hex.to_string()) - .bind(discriminator_hex.to_string()) - .bind(discriminator_hex.to_string()) - .bind(discriminator_hex.to_string()) - .bind(discriminator_hex.to_string()) - .bind(discriminator_hex.to_string()) - .execute(pool) - .await; - if let Err(error) = unlink_result { - return Err(crate::Error::Db(format!( - "cannot unlink replaced instruction audit observation by discriminator on sqlite: {}", - error - ))); - } - - let delete_result = sqlx::query( - r#" -DELETE FROM k_sol_dex_decoded_events -WHERE transaction_id = ? - AND protocol_name = ? - AND event_kind = ? - AND ( - json_extract(payload_json, '$.discriminatorHex') = ? - OR json_extract(payload_json, '$.discriminator_hex') = ? - OR json_extract(payload_json, '$.instructionDiscriminatorHex') = ? - OR json_extract(payload_json, '$.instruction_discriminator_hex') = ? - OR json_extract(payload_json, '$.anchorEventDiscriminatorHex') = ? - OR json_extract(payload_json, '$.anchor_event_discriminator_hex') = ? - OR instr(lower(COALESCE(payload_json, '')), lower(?)) > 0 - ) - "#, - ) - .bind(transaction_id) - .bind(protocol_name.to_string()) - .bind(audit_event_kind.to_string()) - .bind(discriminator_hex.to_string()) - .bind(discriminator_hex.to_string()) - .bind(discriminator_hex.to_string()) - .bind(discriminator_hex.to_string()) - .bind(discriminator_hex.to_string()) - .bind(discriminator_hex.to_string()) - .bind(discriminator_hex.to_string()) - .execute(pool) - .await; - match delete_result { - Ok(_) => return Ok(()), - Err(error) => { - return Err(crate::Error::Db(format!( - "cannot delete replaced instruction audit by discriminator on sqlite: {}", - error - ))); - }, - } - }, + let delete_result = + crate::query_dex_decoded_events_delete_instruction_audit_by_discriminator( + self.database.as_ref(), + transaction_id, + protocol_name, + audit_event_kind, + discriminator_hex, + ) + .await; + match delete_result { + Ok(_) => return Ok(()), + Err(error) => return Err(error), } } @@ -797,34 +700,17 @@ WHERE transaction_id = ? _instruction_id: i64, anchor_event_discriminator_hex: &str, ) -> Result<(), crate::Error> { - match self.database.connection() { - crate::DatabaseConnection::Sqlite(pool) => { - let delete_result = sqlx::query( - r#" -DELETE FROM k_sol_dex_decoded_events -WHERE transaction_id = ? - AND protocol_name = 'raydium_launchpad' - AND event_kind = 'raydium_launchpad.instruction_audit' - AND json_extract(payload_json, '$.anchorSelfCpiLog') = 1 - AND json_extract(payload_json, '$.anchorSelfCpiLogSelectorHex') = ? - AND json_extract(payload_json, '$.anchorEventDiscriminatorHex') = ? - "#, - ) - .bind(transaction_id) - .bind(METEORA_ANCHOR_SELF_CPI_LOG_SELECTOR_HEX.to_string()) - .bind(anchor_event_discriminator_hex.to_string()) - .execute(pool) - .await; - match delete_result { - Ok(_) => return Ok(()), - Err(error) => { - return Err(crate::Error::Db(format!( - "cannot delete replaced Raydium Launchpad self-CPI instruction audit on sqlite: {}", - error - ))); - }, - } - }, + let delete_result = + crate::query_dex_decoded_events_delete_raydium_launchpad_anchor_self_cpi_audit( + self.database.as_ref(), + transaction_id, + METEORA_ANCHOR_SELF_CPI_LOG_SELECTOR_HEX, + anchor_event_discriminator_hex, + ) + .await; + match delete_result { + Ok(_) => return Ok(()), + Err(error) => return Err(error), } } @@ -836,50 +722,16 @@ WHERE transaction_id = ? Some(transaction_id) => transaction_id, None => return Ok(()), }; - match self.database.connection() { - crate::DatabaseConnection::Sqlite(pool) => { - let delete_result = sqlx::query( - r#" -DELETE FROM k_sol_dex_decoded_events -WHERE id IN ( - SELECT audit.id - FROM k_sol_dex_decoded_events audit - WHERE audit.transaction_id = ? - AND audit.protocol_name = 'raydium_launchpad' - AND audit.event_kind = 'raydium_launchpad.instruction_audit' - AND json_extract(audit.payload_json, '$.anchorSelfCpiLog') = 1 - AND json_extract(audit.payload_json, '$.anchorSelfCpiLogSelectorHex') = ? - AND EXISTS ( - SELECT 1 - FROM k_sol_dex_decoded_events direct - WHERE direct.transaction_id = audit.transaction_id - AND direct.protocol_name = 'raydium_launchpad' - AND direct.event_kind IN ( - 'raydium_launchpad.trade_event', - 'raydium_launchpad.pool_create_event', - 'raydium_launchpad.claim_vested_event', - 'raydium_launchpad.create_vesting_event' + let cleanup_result = + crate::query_dex_decoded_events_cleanup_raydium_launchpad_anchor_self_cpi_audits( + self.database.as_ref(), + transaction_id, + METEORA_ANCHOR_SELF_CPI_LOG_SELECTOR_HEX, ) - AND json_extract(direct.payload_json, '$.anchorEventDiscriminatorHex') = - json_extract(audit.payload_json, '$.anchorEventDiscriminatorHex') - ) -) - "#, - ) - .bind(transaction_id) - .bind(METEORA_ANCHOR_SELF_CPI_LOG_SELECTOR_HEX.to_string()) - .execute(pool) - .await; - match delete_result { - Ok(_) => return Ok(()), - Err(error) => { - return Err(crate::Error::Db(format!( - "cannot cleanup replaced Raydium Launchpad self-CPI instruction audits for signature '{}': {}", - transaction.signature, error - ))); - }, - } - }, + .await; + match cleanup_result { + Ok(_) => return Ok(()), + Err(error) => return Err(error), } } @@ -1518,7 +1370,7 @@ WHERE id IN ( event.instruction_id, "raydium_amm_v4", event.program_id.clone(), - "raydium_amm_v4.swap", + event.event_kind.as_str(), Some(event.pool_account.clone()), None, Some(event.token_a_mint.clone()), @@ -1528,6 +1380,24 @@ WHERE id IN ( ) .await; }, + crate::RaydiumAmmV4DecodedEvent::Instruction(event) => { + return self + .materialize_named_dex_event( + transaction, + event.transaction_id, + event.instruction_id, + "raydium_amm_v4", + event.program_id.clone(), + event.event_kind.as_str(), + event.pool_account.clone(), + event.market_account.clone(), + event.token_a_mint.clone(), + event.token_b_mint.clone(), + event.lp_mint.clone(), + event.payload_json.clone(), + ) + .await; + }, } } @@ -1978,7 +1848,11 @@ WHERE id IN ( let accounts = parse_instruction_accounts_vec(instruction.accounts_json.as_str()); let data_base58 = parse_instruction_data_base58(instruction.data_json.as_deref()); let data_bytes = instruction_data_bytes_from_base58(data_base58.as_deref()); - let discriminator_hex = discriminator_hex_from_bytes(data_bytes.as_deref(), 0); + let discriminator_hex = raydium_instruction_discriminator_hex( + audit_spec.protocol_name, + data_bytes.as_deref(), + 0, + ); let anchor_event_spec = raydium_launchpad_anchor_self_cpi_event_spec( audit_spec.protocol_name, data_bytes.as_deref(), @@ -2598,6 +2472,17 @@ enum RaydiumMappedNonTradeAmountLayout { CpmmPoolStatus, CpmmWithdraw, LaunchpadInitialize, + AmmV4Initialize, + AmmV4Initialize2, + AmmV4MonitorStep, + AmmV4Deposit, + AmmV4Withdraw, + AmmV4SetParams, + AmmV4WithdrawSrm, + AmmV4PreInitialize, + AmmV4SimulateInfo, + AmmV4AdminCancelOrders, + AmmV4UpdateConfigAccount, } fn raydium_instruction_audit_spec( @@ -2649,6 +2534,9 @@ fn raydium_mapped_non_trade_instruction_spec( account_count, ); } + if protocol_name == "raydium_amm_v4" { + return raydium_amm_v4_mapped_non_trade_instruction_spec(discriminator_hex, account_count); + } if protocol_name == "raydium_clmm" { if discriminator_hex == "e445a52e51cb9a1d" { return Some(RaydiumMappedNonTradeInstructionSpec { @@ -3173,6 +3061,198 @@ fn raydium_mapped_non_trade_instruction_spec( return None; } +fn raydium_amm_v4_mapped_non_trade_instruction_spec( + discriminator_hex: &str, + account_count: usize, +) -> std::option::Option { + match discriminator_hex { + "00" => { + if account_count >= 4 { + return Some(RaydiumMappedNonTradeInstructionSpec { + instruction_name: "initialize", + event_kind: "raydium_amm_v4.initialize", + pool_account_index: Some(3), + token_a_mint_index: None, + token_b_mint_index: None, + lp_mint_index: None, + amount_layout: RaydiumMappedNonTradeAmountLayout::AmmV4Initialize, + }); + } + }, + "01" => { + if account_count >= 10 { + return Some(RaydiumMappedNonTradeInstructionSpec { + instruction_name: "initialize2", + event_kind: "raydium_amm_v4.initialize2_pool", + pool_account_index: Some(4), + token_a_mint_index: Some(8), + token_b_mint_index: Some(9), + lp_mint_index: Some(7), + amount_layout: RaydiumMappedNonTradeAmountLayout::AmmV4Initialize2, + }); + } + }, + "02" => { + if account_count >= 4 { + return Some(RaydiumMappedNonTradeInstructionSpec { + instruction_name: "monitor_step", + event_kind: "raydium_amm_v4.monitor_step", + pool_account_index: Some(3), + token_a_mint_index: None, + token_b_mint_index: None, + lp_mint_index: None, + amount_layout: RaydiumMappedNonTradeAmountLayout::AmmV4MonitorStep, + }); + } + }, + "03" => { + if account_count >= 8 { + return Some(RaydiumMappedNonTradeInstructionSpec { + instruction_name: "deposit", + event_kind: "raydium_amm_v4.deposit", + pool_account_index: Some(1), + token_a_mint_index: None, + token_b_mint_index: None, + lp_mint_index: Some(5), + amount_layout: RaydiumMappedNonTradeAmountLayout::AmmV4Deposit, + }); + } + }, + "04" => { + if account_count >= 8 { + return Some(RaydiumMappedNonTradeInstructionSpec { + instruction_name: "withdraw", + event_kind: "raydium_amm_v4.withdraw", + pool_account_index: Some(1), + token_a_mint_index: None, + token_b_mint_index: None, + lp_mint_index: Some(5), + amount_layout: RaydiumMappedNonTradeAmountLayout::AmmV4Withdraw, + }); + } + }, + "05" => { + if account_count >= 4 { + return Some(RaydiumMappedNonTradeInstructionSpec { + instruction_name: "migrate_to_open_book", + event_kind: "raydium_amm_v4.migrate_to_open_book", + pool_account_index: Some(3), + token_a_mint_index: None, + token_b_mint_index: None, + lp_mint_index: None, + amount_layout: RaydiumMappedNonTradeAmountLayout::None, + }); + } + }, + "06" => { + if account_count >= 2 { + return Some(RaydiumMappedNonTradeInstructionSpec { + instruction_name: "set_params", + event_kind: "raydium_amm_v4.set_params", + pool_account_index: Some(1), + token_a_mint_index: None, + token_b_mint_index: None, + lp_mint_index: None, + amount_layout: RaydiumMappedNonTradeAmountLayout::AmmV4SetParams, + }); + } + }, + "07" => { + if account_count >= 2 { + return Some(RaydiumMappedNonTradeInstructionSpec { + instruction_name: "withdraw_pnl", + event_kind: "raydium_amm_v4.withdraw_pnl", + pool_account_index: Some(1), + token_a_mint_index: None, + token_b_mint_index: None, + lp_mint_index: None, + amount_layout: RaydiumMappedNonTradeAmountLayout::None, + }); + } + }, + "08" => { + if account_count >= 2 { + return Some(RaydiumMappedNonTradeInstructionSpec { + instruction_name: "withdraw_srm", + event_kind: "raydium_amm_v4.withdraw_srm", + pool_account_index: Some(1), + token_a_mint_index: None, + token_b_mint_index: None, + lp_mint_index: None, + amount_layout: RaydiumMappedNonTradeAmountLayout::AmmV4WithdrawSrm, + }); + } + }, + "0a" => { + if account_count >= 5 { + return Some(RaydiumMappedNonTradeInstructionSpec { + instruction_name: "pre_initialize", + event_kind: "raydium_amm_v4.pre_initialize", + pool_account_index: Some(4), + token_a_mint_index: None, + token_b_mint_index: None, + lp_mint_index: None, + amount_layout: RaydiumMappedNonTradeAmountLayout::AmmV4PreInitialize, + }); + } + }, + "0c" => { + if account_count >= 2 { + return Some(RaydiumMappedNonTradeInstructionSpec { + instruction_name: "simulate_info", + event_kind: "raydium_amm_v4.simulate_info", + pool_account_index: Some(1), + token_a_mint_index: None, + token_b_mint_index: None, + lp_mint_index: None, + amount_layout: RaydiumMappedNonTradeAmountLayout::AmmV4SimulateInfo, + }); + } + }, + "0d" => { + if account_count >= 2 { + return Some(RaydiumMappedNonTradeInstructionSpec { + instruction_name: "admin_cancel_orders", + event_kind: "raydium_amm_v4.admin_cancel_orders", + pool_account_index: Some(1), + token_a_mint_index: None, + token_b_mint_index: None, + lp_mint_index: None, + amount_layout: RaydiumMappedNonTradeAmountLayout::AmmV4AdminCancelOrders, + }); + } + }, + "0e" => { + if account_count >= 1 { + return Some(RaydiumMappedNonTradeInstructionSpec { + instruction_name: "create_config_account", + event_kind: "raydium_amm_v4.create_config_account", + pool_account_index: None, + token_a_mint_index: None, + token_b_mint_index: None, + lp_mint_index: None, + amount_layout: RaydiumMappedNonTradeAmountLayout::None, + }); + } + }, + "0f" => { + if account_count >= 1 { + return Some(RaydiumMappedNonTradeInstructionSpec { + instruction_name: "update_config_account", + event_kind: "raydium_amm_v4.update_config_account", + pool_account_index: None, + token_a_mint_index: None, + token_b_mint_index: None, + lp_mint_index: None, + amount_layout: RaydiumMappedNonTradeAmountLayout::AmmV4UpdateConfigAccount, + }); + } + }, + _ => {}, + } + return None; +} + fn raydium_launchpad_mapped_non_trade_instruction_spec( discriminator_hex: &str, account_count: usize, @@ -3497,6 +3577,189 @@ fn insert_raydium_mapped_amounts( ); } }, + RaydiumMappedNonTradeAmountLayout::AmmV4Initialize => { + if let Some(nonce) = read_u8_from_bytes(data, 1) { + object.insert( + "nonce".to_string(), + serde_json::Value::Number(serde_json::Number::from(nonce as u64)), + ); + } + if let Some(open_time) = read_u64_le_from_bytes(data, 2) { + object.insert("openTime".to_string(), serde_json::Value::String(open_time.to_string())); + } + }, + RaydiumMappedNonTradeAmountLayout::AmmV4Initialize2 => { + if let Some(nonce) = read_u8_from_bytes(data, 1) { + object.insert( + "nonce".to_string(), + serde_json::Value::Number(serde_json::Number::from(nonce as u64)), + ); + } + if let Some(open_time) = read_u64_le_from_bytes(data, 2) { + object.insert("openTime".to_string(), serde_json::Value::String(open_time.to_string())); + } + if let Some(init_pc_amount) = read_u64_le_from_bytes(data, 10) { + object.insert( + "initPcAmount".to_string(), + serde_json::Value::String(init_pc_amount.to_string()), + ); + object.insert( + "tokenBAmount".to_string(), + serde_json::Value::String(init_pc_amount.to_string()), + ); + } + if let Some(init_coin_amount) = read_u64_le_from_bytes(data, 18) { + object.insert( + "initCoinAmount".to_string(), + serde_json::Value::String(init_coin_amount.to_string()), + ); + object.insert( + "tokenAAmount".to_string(), + serde_json::Value::String(init_coin_amount.to_string()), + ); + } + }, + RaydiumMappedNonTradeAmountLayout::AmmV4MonitorStep => { + if let Some(plan_order_limit) = read_u16_le_from_bytes(data, 1) { + object.insert( + "planOrderLimit".to_string(), + serde_json::Value::Number(serde_json::Number::from(plan_order_limit as u64)), + ); + } + if let Some(place_order_limit) = read_u16_le_from_bytes(data, 3) { + object.insert( + "placeOrderLimit".to_string(), + serde_json::Value::Number(serde_json::Number::from(place_order_limit as u64)), + ); + } + if let Some(cancel_order_limit) = read_u16_le_from_bytes(data, 5) { + object.insert( + "cancelOrderLimit".to_string(), + serde_json::Value::Number(serde_json::Number::from(cancel_order_limit as u64)), + ); + } + }, + RaydiumMappedNonTradeAmountLayout::AmmV4Deposit => { + if let Some(max_coin_amount) = read_u64_le_from_bytes(data, 1) { + object.insert( + "maxCoinAmount".to_string(), + serde_json::Value::String(max_coin_amount.to_string()), + ); + object.insert( + "tokenAAmount".to_string(), + serde_json::Value::String(max_coin_amount.to_string()), + ); + } + if let Some(max_pc_amount) = read_u64_le_from_bytes(data, 9) { + object.insert( + "maxPcAmount".to_string(), + serde_json::Value::String(max_pc_amount.to_string()), + ); + object.insert( + "tokenBAmount".to_string(), + serde_json::Value::String(max_pc_amount.to_string()), + ); + } + if let Some(base_side) = read_u64_le_from_bytes(data, 17) { + object.insert("baseSide".to_string(), serde_json::Value::String(base_side.to_string())); + } + if let Some(other_amount_min) = read_u64_le_from_bytes(data, 25) { + object.insert( + "otherAmountMin".to_string(), + serde_json::Value::String(other_amount_min.to_string()), + ); + } + }, + RaydiumMappedNonTradeAmountLayout::AmmV4Withdraw => { + if let Some(lp_amount) = read_u64_le_from_bytes(data, 1) { + object.insert("lpAmountRaw".to_string(), serde_json::Value::String(lp_amount.to_string())); + object.insert("liquidity".to_string(), serde_json::Value::String(lp_amount.to_string())); + } + if let Some(min_coin_amount) = read_u64_le_from_bytes(data, 9) { + object.insert( + "minCoinAmount".to_string(), + serde_json::Value::String(min_coin_amount.to_string()), + ); + } + if let Some(min_pc_amount) = read_u64_le_from_bytes(data, 17) { + object.insert( + "minPcAmount".to_string(), + serde_json::Value::String(min_pc_amount.to_string()), + ); + } + }, + RaydiumMappedNonTradeAmountLayout::AmmV4SetParams => { + if let Some(param) = read_u8_from_bytes(data, 1) { + object.insert( + "configParam".to_string(), + serde_json::Value::Number(serde_json::Number::from(param as u64)), + ); + } + if let Some(value) = read_u64_le_from_bytes(data, 2) { + object.insert("configValue".to_string(), serde_json::Value::String(value.to_string())); + } + if let Some(last_order_denominator) = read_u64_le_from_bytes(data, 10) { + object.insert( + "lastOrderDenominator".to_string(), + serde_json::Value::String(last_order_denominator.to_string()), + ); + } + }, + RaydiumMappedNonTradeAmountLayout::AmmV4WithdrawSrm => { + if let Some(amount) = read_u64_le_from_bytes(data, 1) { + object.insert("amountRaw".to_string(), serde_json::Value::String(amount.to_string())); + } + }, + RaydiumMappedNonTradeAmountLayout::AmmV4PreInitialize => { + object.insert("deprecatedInstruction".to_string(), serde_json::Value::Bool(true)); + object.insert("partialLifecycle".to_string(), serde_json::Value::Bool(true)); + object.insert( + "skipCatalogReason".to_string(), + serde_json::Value::String("missing_token_mints".to_string()), + ); + if let Some(nonce) = read_u8_from_bytes(data, 1) { + object.insert( + "nonce".to_string(), + serde_json::Value::Number(serde_json::Number::from(nonce as u64)), + ); + } + }, + RaydiumMappedNonTradeAmountLayout::AmmV4SimulateInfo => { + if let Some(param) = read_u8_from_bytes(data, 1) { + object.insert( + "simulateParam".to_string(), + serde_json::Value::Number(serde_json::Number::from(param as u64)), + ); + } + if let Some(amount_in) = read_u64_le_from_bytes(data, 2) { + object.insert("amountIn".to_string(), serde_json::Value::String(amount_in.to_string())); + } + if let Some(amount_out) = read_u64_le_from_bytes(data, 10) { + object.insert("amountOutOrMinimumAmountOut".to_string(), serde_json::Value::String(amount_out.to_string())); + } + }, + RaydiumMappedNonTradeAmountLayout::AmmV4AdminCancelOrders => { + if let Some(limit) = read_u16_le_from_bytes(data, 1) { + object.insert( + "orderCancelLimit".to_string(), + serde_json::Value::Number(serde_json::Number::from(limit as u64)), + ); + } + }, + RaydiumMappedNonTradeAmountLayout::AmmV4UpdateConfigAccount => { + if let Some(param) = read_u8_from_bytes(data, 1) { + object.insert( + "configParam".to_string(), + serde_json::Value::Number(serde_json::Number::from(param as u64)), + ); + } + if let Some(create_pool_fee) = read_u64_le_from_bytes(data, 2) { + object.insert( + "createPoolFee".to_string(), + serde_json::Value::String(create_pool_fee.to_string()), + ); + } + }, RaydiumMappedNonTradeAmountLayout::LaunchpadInitialize => { object.insert( "poolKindHint".to_string(), @@ -3555,6 +3818,19 @@ fn read_u8_from_bytes(data: &[u8], offset: usize) -> std::option::Option { return Some(data[offset]); } +fn read_u16_le_from_bytes(data: &[u8], offset: usize) -> std::option::Option { + if data.len() < offset + 2 { + return None; + } + let mut bytes = [0_u8; 2]; + let mut index = 0_usize; + while index < 2 { + bytes[index] = data[offset + index]; + index += 1; + } + return Some(u16::from_le_bytes(bytes)); +} + fn read_i32_le_from_bytes(data: &[u8], offset: usize) -> std::option::Option { if data.len() < offset + 4 { return None; @@ -3701,7 +3977,7 @@ fn build_meteora_instruction_audit_payload( }; let data_base58 = parse_instruction_data_base58(instruction.data_json.as_deref()); let data_bytes = instruction_data_bytes_from_base58(data_base58.as_deref()); - let discriminator_hex = discriminator_hex_from_bytes(data_bytes.as_deref(), 0); + let discriminator_hex = raydium_instruction_discriminator_hex(protocol_name, data_bytes.as_deref(), 0); let anchor_self_cpi_log = discriminator_hex.as_deref() == Some(METEORA_ANCHOR_SELF_CPI_LOG_SELECTOR_HEX); let anchor_event_discriminator_hex = if anchor_self_cpi_log { @@ -4239,7 +4515,7 @@ fn build_raydium_instruction_audit_payload( }; let data_base58 = parse_instruction_data_base58(instruction.data_json.as_deref()); let data_bytes = instruction_data_bytes_from_base58(data_base58.as_deref()); - let discriminator_hex = discriminator_hex_from_bytes(data_bytes.as_deref(), 0); + let discriminator_hex = raydium_instruction_discriminator_hex(protocol_name, data_bytes.as_deref(), 0); let anchor_self_cpi_log = discriminator_hex.as_deref() == Some(METEORA_ANCHOR_SELF_CPI_LOG_SELECTOR_HEX); let anchor_event_discriminator_hex = if anchor_self_cpi_log { @@ -4438,6 +4714,40 @@ fn discriminator_hex_from_base58( return discriminator_hex_from_bytes(bytes.as_deref(), 0); } +fn raydium_instruction_discriminator_hex( + protocol_name: &str, + bytes: std::option::Option<&[u8]>, + offset: usize, +) -> std::option::Option { + if protocol_name == "raydium_amm_v4" { + return discriminator_hex_from_bytes_with_len(bytes, offset, 1); + } + return discriminator_hex_from_bytes(bytes, offset); +} + +fn discriminator_hex_from_bytes_with_len( + bytes: std::option::Option<&[u8]>, + offset: usize, + length: usize, +) -> std::option::Option { + let bytes = match bytes { + Some(bytes) => bytes, + None => return None, + }; + if bytes.len() < offset + length { + return None; + } + let mut text = std::string::String::new(); + let mut index = offset; + let end = offset + length; + while index < end { + let byte = bytes[index]; + text.push_str(format!("{byte:02x}").as_str()); + index += 1; + } + return Some(text); +} + fn discriminator_hex_from_bytes( bytes: std::option::Option<&[u8]>, offset: usize, diff --git a/kb_lib/src/dex_detection_route.rs b/kb_lib/src/dex_detection_route.rs index f37900b..ca26640 100644 --- a/kb_lib/src/dex_detection_route.rs +++ b/kb_lib/src/dex_detection_route.rs @@ -49,7 +49,16 @@ pub(crate) fn dex_detection_route( crate::dex_detection_route::DexDetectionRoute::RaydiumAmmV4Initialize2Pool, ); }, - ("raydium_amm_v4", "raydium_amm_v4.swap") => { + ("raydium_amm_v4", "raydium_amm_v4.swap_base_in") => { + return Some(crate::dex_detection_route::DexDetectionRoute::RaydiumAmmV4Trade); + }, + ("raydium_amm_v4", "raydium_amm_v4.swap_base_out") => { + return Some(crate::dex_detection_route::DexDetectionRoute::RaydiumAmmV4Trade); + }, + ("raydium_amm_v4", "raydium_amm_v4.swap_base_in_v2") => { + return Some(crate::dex_detection_route::DexDetectionRoute::RaydiumAmmV4Trade); + }, + ("raydium_amm_v4", "raydium_amm_v4.swap_base_out_v2") => { return Some(crate::dex_detection_route::DexDetectionRoute::RaydiumAmmV4Trade); }, ("raydium_cpmm", "raydium_cpmm.swap_base_input") => { diff --git a/kb_lib/src/dex_event_classification.rs b/kb_lib/src/dex_event_classification.rs index d323bff..bbfd837 100644 --- a/kb_lib/src/dex_event_classification.rs +++ b/kb_lib/src/dex_event_classification.rs @@ -357,6 +357,12 @@ pub fn is_dex_candle_candidate_event_kind(event_kind: &str) -> bool { /// Returns true for liquidity lifecycle changes that must not become candles. pub fn is_dex_liquidity_event_kind(event_kind: &str) -> bool { + if event_kind.contains(".withdraw_pnl") { + return false; + } + if event_kind.contains(".withdraw_srm") { + return false; + } if event_kind.contains(".deposit") { return true; } @@ -418,6 +424,12 @@ pub fn is_dex_liquidity_add_event_kind(event_kind: &str) -> bool { /// Returns true for liquidity remove-like DEX events. pub fn is_dex_liquidity_remove_event_kind(event_kind: &str) -> bool { + if event_kind.contains(".withdraw_pnl") { + return false; + } + if event_kind.contains(".withdraw_srm") { + return false; + } if event_kind.contains(".withdraw") { return true; } @@ -487,6 +499,12 @@ pub fn is_dex_fee_event_kind(event_kind: &str) -> bool { if event_kind.contains("withdraw_protocol_fees") { return true; } + if event_kind.contains(".withdraw_pnl") { + return true; + } + if event_kind.contains(".withdraw_srm") { + return true; + } if event_kind.contains("partner_claim_fee") { return true; } @@ -506,6 +524,15 @@ pub fn is_dex_reward_event_kind(event_kind: &str) -> bool { /// Returns true for orderbook or limit-order events that must not become candles. pub fn is_dex_orderbook_event_kind(event_kind: &str) -> bool { + if event_kind.contains(".monitor_step") { + return true; + } + if event_kind.contains(".migrate_to_open_book") { + return true; + } + if event_kind.contains(".admin_cancel_orders") { + return true; + } if event_kind.contains(".order_place") { return true; } @@ -538,6 +565,9 @@ pub fn is_dex_orderbook_event_kind(event_kind: &str) -> bool { /// Returns true for pool, pair, launch, mint, burn or migration lifecycle events. pub fn is_dex_pool_lifecycle_event_kind(event_kind: &str) -> bool { + if event_kind == "raydium_amm_v4.pre_initialize" { + return true; + } if event_kind.contains(".create_lock_escrow") { return true; } @@ -621,6 +651,12 @@ pub fn is_dex_token_burn_event_kind(event_kind: &str) -> bool { /// Returns true for launch-surface or pool migration events. pub fn is_dex_migration_event_kind(event_kind: &str) -> bool { + if event_kind == "raydium_amm_v4.migrate_to_open_book" { + return false; + } + if event_kind.contains(".migrate_to_open_book") { + return false; + } if event_kind.contains(".migrate") { return true; } @@ -632,6 +668,9 @@ pub fn is_dex_migration_event_kind(event_kind: &str) -> bool { /// Returns true for pool creation or initialization events. pub fn is_dex_pool_creation_event_kind(event_kind: &str) -> bool { + if event_kind == "raydium_amm_v4.pre_initialize" { + return true; + } if event_kind.contains("amm_config") { return false; } @@ -711,6 +750,15 @@ pub fn is_dex_token_account_close_event_kind(event_kind: &str) -> bool { /// Returns true for admin, configuration or permission changes. pub fn is_dex_admin_event_kind(event_kind: &str) -> bool { + if event_kind.contains(".admin_cancel_orders") { + return false; + } + if event_kind.contains(".monitor_step") { + return false; + } + if event_kind.contains(".migrate_to_open_book") { + return false; + } if event_kind.contains(".close_platform_global_access") { return true; } @@ -1114,6 +1162,10 @@ mod tests { super::classify_dex_event_category_code("raydium_cpmm.initialize"), "pool_lifecycle" ); + assert_eq!( + super::classify_dex_event_category_code("raydium_amm_v4.pre_initialize"), + "pool_lifecycle" + ); } #[test] @@ -1122,7 +1174,32 @@ mod tests { super::classify_dex_event_lifecycle_kind_code("raydium_cpmm.initialize"), "pool_creation" ); + assert_eq!( + super::classify_dex_event_lifecycle_kind_code("raydium_amm_v4.pre_initialize"), + "pool_creation" + ); assert_eq!(super::classify_dex_event_lifecycle_kind_code("pump_fun.create"), "launch"); + assert_eq!( + crate::dex_event_classification::classify_dex_event_category_code( + "raydium_amm_v4.migrate_to_open_book", + ), + "unknown", + ); + assert_eq!( + crate::dex_event_classification::classify_dex_event_lifecycle_kind_code( + "raydium_amm_v4.migrate_to_open_book", + ), + "unknown", + ); + assert!(crate::dex_event_classification::is_dex_orderbook_event_kind( + "raydium_amm_v4.migrate_to_open_book", + )); + assert!(!crate::dex_event_classification::is_dex_pool_lifecycle_event_kind( + "raydium_amm_v4.migrate_to_open_book", + )); + assert!(!crate::dex_event_classification::is_dex_migration_event_kind( + "raydium_amm_v4.migrate_to_open_book", + )); assert_eq!( super::classify_dex_event_lifecycle_kind_code("meteora_dbc.migrate"), "migration" diff --git a/kb_lib/src/dex_event_coverage.rs b/kb_lib/src/dex_event_coverage.rs index cd55efe..558eab8 100644 --- a/kb_lib/src/dex_event_coverage.rs +++ b/kb_lib/src/dex_event_coverage.rs @@ -220,6 +220,46 @@ fn infer_expected_db_target_for_entry( { return Some(crate::DexEventCoverageEntryDto::DB_TARGET_DECODED_EVENTS_ONLY.to_string()); } + if decoder_code == "raydium_amm_v4" { + if entry_name == "swap_base_in" + || entry_name == "swap_base_out" + || entry_name == "swap_base_in_v2" + || entry_name == "swap_base_out_v2" + { + return Some(crate::DexEventCoverageEntryDto::DB_TARGET_TRADE_EVENTS.to_string()); + } + if entry_name == "initialize" + || entry_name == "initialize2" + || entry_name == "pre_initialize" + { + return Some( + crate::DexEventCoverageEntryDto::DB_TARGET_POOL_LIFECYCLE_EVENTS.to_string(), + ); + } + if entry_name == "deposit" || entry_name == "withdraw" { + return Some(crate::DexEventCoverageEntryDto::DB_TARGET_LIQUIDITY_EVENTS.to_string()); + } + if entry_name == "withdraw_pnl" || entry_name == "withdraw_srm" { + return Some(crate::DexEventCoverageEntryDto::DB_TARGET_FEE_EVENTS.to_string()); + } + if entry_name == "admin_cancel_orders" || entry_name == "migrate_to_open_book" { + return Some(crate::DexEventCoverageEntryDto::DB_TARGET_ORDERBOOK_EVENTS.to_string()); + } + if entry_name == "monitor_step" { + return Some(crate::DexEventCoverageEntryDto::DB_TARGET_ORDERBOOK_EVENTS.to_string()); + } + if entry_name == "create_config_account" + || entry_name == "update_config_account" + || entry_name == "set_params" + { + return Some(crate::DexEventCoverageEntryDto::DB_TARGET_POOL_ADMIN_EVENTS.to_string()); + } + if entry_name == "simulate_info" { + return Some( + crate::DexEventCoverageEntryDto::DB_TARGET_DECODED_EVENTS_ONLY.to_string(), + ); + } + } if decoder_code == "raydium_clmm" { if entry_name == "initialize_reward" { return Some(crate::DexEventCoverageEntryDto::DB_TARGET_REWARD_EVENTS.to_string()); @@ -356,6 +396,9 @@ fn infer_event_family_for_entry( if decoder_code == "raydium_launchpad" { return infer_raydium_launchpad_event_family(entry_name, entry_kind); } + if decoder_code == "raydium_amm_v4" { + return infer_raydium_amm_v4_event_family(entry_name, entry_kind); + } if decoder_code == "raydium_clmm" { return infer_raydium_clmm_event_family(entry_name, entry_kind); } @@ -365,6 +408,36 @@ fn infer_event_family_for_entry( return infer_event_family(entry_name, entry_kind); } +fn infer_raydium_amm_v4_event_family( + entry_name: &str, + entry_kind: &str, +) -> std::option::Option { + if entry_kind == crate::ENTRY_KIND_PROGRAM { + return None; + } + match entry_name { + "swap_base_in" => return Some("swap".to_string()), + "swap_base_out" => return Some("swap".to_string()), + "swap_base_in_v2" => return Some("swap".to_string()), + "swap_base_out_v2" => return Some("swap".to_string()), + "initialize" => return Some("pool_create".to_string()), + "initialize2" => return Some("pool_create".to_string()), + "pre_initialize" => return Some("pool_create".to_string()), + "deposit" => return Some("liquidity_add".to_string()), + "withdraw" => return Some("liquidity_remove".to_string()), + "withdraw_pnl" => return Some("fee".to_string()), + "withdraw_srm" => return Some("fee".to_string()), + "admin_cancel_orders" => return Some("order_cancel".to_string()), + "migrate_to_open_book" => return Some("order_place".to_string()), + "create_config_account" => return Some("admin_config".to_string()), + "update_config_account" => return Some("admin_config".to_string()), + "set_params" => return Some("admin_config".to_string()), + "monitor_step" => return Some("order_place".to_string()), + "simulate_info" => return Some("cpi_transport".to_string()), + _ => return infer_event_family(entry_name, entry_kind), + } +} + fn infer_raydium_cpmm_event_family( entry_name: &str, entry_kind: &str, @@ -628,10 +701,37 @@ fn raydium_launchpad_local_entry_is_known(entry_name: &str) -> bool { } } +fn raydium_amm_v4_local_event_kind(entry_name: &str) -> std::option::Option { + match entry_name { + "swap_base_in" => return Some("raydium_amm_v4.swap_base_in".to_string()), + "swap_base_out" => return Some("raydium_amm_v4.swap_base_out".to_string()), + "swap_base_in_v2" => return Some("raydium_amm_v4.swap_base_in_v2".to_string()), + "swap_base_out_v2" => return Some("raydium_amm_v4.swap_base_out_v2".to_string()), + "initialize" => return Some("raydium_amm_v4.initialize".to_string()), + "initialize2" => return Some("raydium_amm_v4.initialize2_pool".to_string()), + "pre_initialize" => return Some("raydium_amm_v4.pre_initialize".to_string()), + "deposit" => return Some("raydium_amm_v4.deposit".to_string()), + "withdraw" => return Some("raydium_amm_v4.withdraw".to_string()), + "withdraw_pnl" => return Some("raydium_amm_v4.withdraw_pnl".to_string()), + "withdraw_srm" => return Some("raydium_amm_v4.withdraw_srm".to_string()), + "admin_cancel_orders" => return Some("raydium_amm_v4.admin_cancel_orders".to_string()), + "migrate_to_open_book" => return Some("raydium_amm_v4.migrate_to_open_book".to_string()), + "create_config_account" => return Some("raydium_amm_v4.create_config_account".to_string()), + "update_config_account" => return Some("raydium_amm_v4.update_config_account".to_string()), + "set_params" => return Some("raydium_amm_v4.set_params".to_string()), + "monitor_step" => return Some("raydium_amm_v4.monitor_step".to_string()), + "simulate_info" => return Some("raydium_amm_v4.simulate_info".to_string()), + _ => return None, + } +} + pub(crate) fn known_local_event_kind( decoder_code: &str, entry_name: &str, ) -> std::option::Option { + if decoder_code == "raydium_amm_v4" { + return raydium_amm_v4_local_event_kind(entry_name); + } if decoder_code == "raydium_launchpad" && raydium_launchpad_local_entry_is_known(entry_name) { return Some(format!("raydium_launchpad.{}", entry_name)); } diff --git a/kb_lib/src/instruction_observation_index.rs b/kb_lib/src/instruction_observation_index.rs index c398a2d..3431904 100644 --- a/kb_lib/src/instruction_observation_index.rs +++ b/kb_lib/src/instruction_observation_index.rs @@ -6,25 +6,6 @@ //! aid used to find local corpus evidence by program, decoder, instruction //! discriminator and instruction name. -#[derive(Debug, Clone, sqlx::FromRow)] -struct InstructionObservationSourceRow { - transaction_id: i64, - signature: std::string::String, - slot: std::option::Option, - block_time: std::option::Option, - err_json: std::option::Option, - instruction_id: i64, - parent_instruction_id: std::option::Option, - instruction_index: i64, - inner_instruction_index: std::option::Option, - program_id: std::option::Option, - accounts_json: std::string::String, - data_json: std::option::Option, - pool_account: std::option::Option, - decoded_event_kind: std::option::Option, - decoded_event_id: std::option::Option, -} - /// Result of refreshing the instruction-observation index. #[derive(Debug, Clone, Default, serde::Serialize, serde::Deserialize)] #[serde(rename_all = "camelCase")] @@ -88,8 +69,23 @@ impl InstructionObservationIndexService { async fn upsert_source_rows( &self, - rows: std::vec::Vec, + rows: std::vec::Vec, ) -> Result { + let mut transaction_ids = std::vec::Vec::::new(); + for row in &rows { + if transaction_ids.contains(&row.transaction_id) { + continue; + } + transaction_ids.push(row.transaction_id); + } + let delete_result = crate::query_instruction_observations_delete_by_transaction_ids( + self.database.as_ref(), + transaction_ids.as_slice(), + ) + .await; + if let Err(error) = delete_result { + return Err(error); + } let mut result = crate::InstructionObservationIndexRefreshResult::default(); for row in rows { result.scanned_instruction_count += 1; @@ -111,182 +107,46 @@ impl InstructionObservationIndexService { async fn list_source_rows_by_signature( &self, signature: &str, - ) -> Result, crate::Error> { - match self.database.connection() { - crate::DatabaseConnection::Sqlite(pool) => { - let query_result = sqlx::query_as::( - r#" -SELECT - tx.id AS transaction_id, - tx.signature AS signature, - tx.slot AS slot, - tx.block_time_unix AS block_time, - tx.err_json AS err_json, - ins.id AS instruction_id, - ins.parent_instruction_id AS parent_instruction_id, - ins.instruction_index AS instruction_index, - ins.inner_instruction_index AS inner_instruction_index, - ins.program_id AS program_id, - ins.accounts_json AS accounts_json, - ins.data_json AS data_json, - de.pool_account AS pool_account, - de.event_kind AS decoded_event_kind, - de.id AS decoded_event_id -FROM k_sol_chain_instructions ins -JOIN k_sol_chain_transactions tx - ON tx.id = ins.transaction_id -LEFT JOIN k_sol_dex_decoded_events de - ON de.transaction_id = tx.id - AND de.instruction_id = ins.id -WHERE tx.signature = ? -ORDER BY ins.instruction_index ASC, ins.inner_instruction_index ASC, ins.id ASC - "#, - ) - .bind(signature.to_string()) - .fetch_all(pool) - .await; - match query_result { - Ok(rows) => return Ok(rows), - Err(error) => { - return Err(crate::Error::Db(format!( - "cannot list instruction observation source rows for signature '{}': {}", - signature, error - ))); - }, - } - }, - } + ) -> Result, crate::Error> { + return crate::query_instruction_observation_source_rows_list_by_signature( + self.database.as_ref(), + signature, + ) + .await; } async fn list_replay_window_source_rows( &self, limit: std::option::Option, - ) -> Result, crate::Error> { - let effective_limit = match limit { - Some(limit) => { - if limit <= 0 { - 10_000 - } else { - limit - } - }, - None => 10_000, - }; - match self.database.connection() { - crate::DatabaseConnection::Sqlite(pool) => { - let query_result = sqlx::query_as::( - r#" -WITH replay_transactions AS ( - SELECT id - FROM k_sol_chain_transactions - ORDER BY id ASC - LIMIT ? -) -SELECT - tx.id AS transaction_id, - tx.signature AS signature, - tx.slot AS slot, - tx.block_time_unix AS block_time, - tx.err_json AS err_json, - ins.id AS instruction_id, - ins.parent_instruction_id AS parent_instruction_id, - ins.instruction_index AS instruction_index, - ins.inner_instruction_index AS inner_instruction_index, - ins.program_id AS program_id, - ins.accounts_json AS accounts_json, - ins.data_json AS data_json, - de.pool_account AS pool_account, - de.event_kind AS decoded_event_kind, - de.id AS decoded_event_id -FROM k_sol_chain_instructions ins -JOIN replay_transactions replay_tx - ON replay_tx.id = ins.transaction_id -JOIN k_sol_chain_transactions tx - ON tx.id = ins.transaction_id -LEFT JOIN k_sol_dex_decoded_events de - ON de.transaction_id = tx.id - AND de.instruction_id = ins.id -ORDER BY tx.id ASC, ins.instruction_index ASC, ins.inner_instruction_index ASC, ins.id ASC - "#, - ) - .bind(effective_limit) - .fetch_all(pool) - .await; - match query_result { - Ok(rows) => return Ok(rows), - Err(error) => { - return Err(crate::Error::Db(format!( - "cannot list instruction observation source rows for replay window: {}", - error - ))); - }, - } - }, - } + ) -> Result, crate::Error> { + return crate::query_instruction_observation_source_rows_list_replay_window( + self.database.as_ref(), + limit, + ) + .await; } async fn list_recent_source_rows( &self, limit: u32, - ) -> Result, crate::Error> { - if limit == 0 { - return Ok(std::vec::Vec::new()); - } - match self.database.connection() { - crate::DatabaseConnection::Sqlite(pool) => { - let query_result = sqlx::query_as::( - r#" -SELECT - tx.id AS transaction_id, - tx.signature AS signature, - tx.slot AS slot, - tx.block_time_unix AS block_time, - tx.err_json AS err_json, - ins.id AS instruction_id, - ins.parent_instruction_id AS parent_instruction_id, - ins.instruction_index AS instruction_index, - ins.inner_instruction_index AS inner_instruction_index, - ins.program_id AS program_id, - ins.accounts_json AS accounts_json, - ins.data_json AS data_json, - de.pool_account AS pool_account, - de.event_kind AS decoded_event_kind, - de.id AS decoded_event_id -FROM k_sol_chain_instructions ins -JOIN k_sol_chain_transactions tx - ON tx.id = ins.transaction_id -LEFT JOIN k_sol_dex_decoded_events de - ON de.transaction_id = tx.id - AND de.instruction_id = ins.id -ORDER BY ins.id DESC -LIMIT ? - "#, - ) - .bind(i64::from(limit)) - .fetch_all(pool) - .await; - match query_result { - Ok(rows) => return Ok(rows), - Err(error) => { - return Err(crate::Error::Db(format!( - "cannot list recent instruction observation source rows: {}", - error - ))); - }, - } - }, - } + ) -> Result, crate::Error> { + return crate::query_instruction_observation_source_rows_list_recent( + self.database.as_ref(), + limit, + ) + .await; } } fn build_instruction_observation_dto( - row: InstructionObservationSourceRow, + row: crate::InstructionObservationSourceRow, ) -> std::option::Option { let program_id = match row.program_id.clone() { Some(program_id) => program_id, None => return None, }; - let discriminator_hex = discriminator_hex_from_data_json(row.data_json.as_ref()); + let discriminator_hex = + discriminator_hex_from_data_json(row.data_json.as_ref(), program_id.as_str()); let decoder_code = resolve_decoder_code(program_id.as_str()); let instruction_name = resolve_instruction_name( program_id.as_str(), @@ -340,6 +200,31 @@ fn resolve_instruction_name( Some(discriminator_hex) => discriminator_hex, None => return None, }; + if program_id == crate::RAYDIUM_AMM_V4_PROGRAM_ID || decoder_code == Some("raydium_amm_v4") { + let name = match discriminator_hex { + "00" => "raydium_amm_v4.initialize", + "01" => "raydium_amm_v4.initialize2_pool", + "02" => "raydium_amm_v4.monitor_step", + "03" => "raydium_amm_v4.deposit", + "04" => "raydium_amm_v4.withdraw", + "05" => "raydium_amm_v4.migrate_to_open_book", + "06" => "raydium_amm_v4.set_params", + "07" => "raydium_amm_v4.withdraw_pnl", + "08" => "raydium_amm_v4.withdraw_srm", + "09" => "raydium_amm_v4.swap_base_in", + "0a" => "raydium_amm_v4.pre_initialize", + "0b" => "raydium_amm_v4.swap_base_out", + "0c" => "raydium_amm_v4.simulate_info", + "0d" => "raydium_amm_v4.admin_cancel_orders", + "0e" => "raydium_amm_v4.create_config_account", + "0f" => "raydium_amm_v4.update_config_account", + "10" => "raydium_amm_v4.swap_base_in_v2", + "11" => "raydium_amm_v4.swap_base_out_v2", + _ => return None, + }; + return Some(name.to_string()); + } + if program_id == crate::RAYDIUM_CPMM_PROGRAM_ID || decoder_code == Some("raydium_cpmm") { let name = match discriminator_hex { "9c5420764587467b" => "raydium_cpmm.close_permission_pda", @@ -418,15 +303,21 @@ fn resolve_instruction_name( fn discriminator_hex_from_data_json( data_json: std::option::Option<&std::string::String>, + program_id: &str, ) -> std::option::Option { let decoded = match decode_data_json_as_bytes(data_json) { Some(decoded) => decoded, None => return None, }; - if decoded.len() < 8 { + let discriminator_len = if program_id == crate::RAYDIUM_AMM_V4_PROGRAM_ID { + 1_usize + } else { + 8_usize + }; + if decoded.len() < discriminator_len { return None; } - return Some(bytes_to_hex(&decoded[0..8])); + return Some(bytes_to_hex(&decoded[0..discriminator_len])); } fn decode_data_json_as_bytes( diff --git a/kb_lib/src/lib.rs b/kb_lib/src/lib.rs index 8a27f8a..f215b9d 100644 --- a/kb_lib/src/lib.rs +++ b/kb_lib/src/lib.rs @@ -750,14 +750,24 @@ pub use db::query_dex_decode_replay_ledger_get_by_signature; pub use db::query_dex_decode_replay_ledger_get_by_transaction; /// Inserts or updates one DEX decode replay ledger row. pub use db::query_dex_decode_replay_ledger_upsert; +/// Cleans Raydium Launchpad self-CPI audit rows replaced by direct decoded rows. +pub use db::query_dex_decoded_events_cleanup_raydium_launchpad_anchor_self_cpi_audits; /// Deletes one decoded DEX event row by its natural key. pub use db::query_dex_decoded_events_delete_by_key; +/// Deletes an instruction-audit row by discriminator for one protocol. +pub use db::query_dex_decoded_events_delete_instruction_audit_by_discriminator; +/// Deletes local DEX decoded rows and linked materialization rows for one replayed transaction. +pub use db::query_dex_decoded_events_delete_local_replay_scope_by_transaction_id; /// Deletes upstream registry instruction-match rows already covered by specialized local decoders. pub use db::query_dex_decoded_events_delete_locally_covered_upstream_instruction_matches; /// Deletes Meteora DLMM Anchor self-CPI swap audit rows already covered by decoded swaps. pub use db::query_dex_decoded_events_delete_meteora_dlmm_anchor_swap_instruction_audits; /// Deletes decoded DEX instruction audit rows related to one decoded instruction. pub use db::query_dex_decoded_events_delete_related_instruction_audit; +/// Deletes one Raydium CLMM instruction-audit row by discriminator. +pub use db::query_dex_decoded_events_delete_raydium_clmm_instruction_audit_by_discriminator; +/// Deletes one Raydium Launchpad self-CPI audit row by discriminator. +pub use db::query_dex_decoded_events_delete_raydium_launchpad_anchor_self_cpi_audit; /// Deletes Raydium CLMM instruction-audit rows for locally mapped CLMM instructions. pub use db::query_dex_decoded_events_delete_replaced_raydium_clmm_instruction_audits; /// Deletes Raydium CPMM instruction-audit rows already covered by local named rows. @@ -770,6 +780,8 @@ pub use db::query_dex_decoded_events_get_latest_pump_fun_create_payload_by_mint; pub use db::query_dex_decoded_events_list_by_transaction_id; /// Inserts or updates one decoded DEX event row. pub use db::query_dex_decoded_events_upsert; +/// Updates the persisted payload of one decoded DEX event row. +pub use db::query_dex_decoded_events_update_payload_json_by_id; /// Deletes DEX event coverage entries for one decoder. pub use db::query_dex_event_coverage_entries_delete_by_decoder; /// Lists DEX event coverage entries for one decoder. @@ -795,8 +807,20 @@ pub use db::query_fee_events_list_recent; /// Inserts or updates one normalized fee event row. pub use db::query_fee_events_upsert; /// Inserts one on-chain observation row and returns its numeric id. +/// Lists instruction-observation source rows for one transaction signature. +pub use db::query_instruction_observation_source_rows_list_by_signature; +/// Lists recent instruction-observation source rows. +pub use db::query_instruction_observation_source_rows_list_recent; +/// Lists instruction-observation source rows for the local replay window. +pub use db::query_instruction_observation_source_rows_list_replay_window; +/// Deletes instruction observations for a set of transaction ids before rebuilding the technical index. +pub use db::query_instruction_observations_delete_by_transaction_ids; +/// Lists instruction observations by optional filters. pub use db::query_instruction_observations_list_by_filter; +/// Upserts one instruction observation row. pub use db::query_instruction_observations_upsert; +/// Raw source row used to rebuild the technical instruction-observation index. +pub use db::InstructionObservationSourceRow; /// Reads one known HTTP endpoint by name. pub use db::query_known_http_endpoints_get; /// Lists all known HTTP endpoints. @@ -815,6 +839,10 @@ pub use db::query_launch_attributions_get_by_decoded_event_id; pub use db::query_launch_attributions_list_by_pool_id; /// Inserts or updates one launch attribution row and returns its stable internal id. pub use db::query_launch_attributions_upsert; +/// Inserts or updates one launch event row. +pub use db::query_launch_events_upsert; +/// Input used to upsert one launch event row. +pub use db::LaunchEventUpsertInput; /// Returns one launch-surface matching key identified by its kind and value, if it exists. pub use db::query_launch_surface_keys_get_by_match; /// Lists all launch-surface matching keys attached to one launch surface id. @@ -913,6 +941,8 @@ pub use db::query_pairs_list; pub use db::query_pairs_update_symbol; /// Inserts or updates one normalized pair row by pool id. pub use db::query_pairs_upsert; +/// Deletes one stale pool administration event by decoded-event id. +pub use db::query_pool_admin_events_delete_by_decoded_event_id; /// Returns one pool administration event by decoded-event id. pub use db::query_pool_admin_events_get_by_decoded_event_id; /// Lists recent pool administration events ordered from newest to oldest. @@ -1161,6 +1191,8 @@ pub use dex::RaydiumAmmV4DecodedEvent; pub use dex::RaydiumAmmV4Decoder; /// Decoded Raydium AmmV4 initialize2 pool event. pub use dex::RaydiumAmmV4Initialize2PoolDecoded; +/// Decoded Raydium AmmV4 non-swap or decoded-only instruction event. +pub use dex::RaydiumAmmV4InstructionDecoded; /// Decoded Raydium AMM v4 swap event. pub use dex::RaydiumAmmV4SwapDecoded; /// Decoded Raydium CLMM collect_protocol_fee instruction. diff --git a/kb_lib/src/local_pipeline_diagnostics.rs b/kb_lib/src/local_pipeline_diagnostics.rs index 566fd75..f542f47 100644 --- a/kb_lib/src/local_pipeline_diagnostics.rs +++ b/kb_lib/src/local_pipeline_diagnostics.rs @@ -25,7 +25,7 @@ impl LocalPipelineDiagnosticsService { pub async fn diagnose_for_validation( &self, ) -> Result { - let counters_result = query_lightweight_validation_counters(self.database.as_ref()).await; + let counters_result = crate::query_local_pipeline_diagnostic_get_counters(self.database.as_ref()).await; let counters = match counters_result { Ok(counters) => counters, Err(error) => return Err(error), @@ -364,475 +364,21 @@ impl LocalPipelineDiagnosticsService { } } -async fn query_lightweight_validation_counters( - database: &crate::Database, -) -> Result { - match database.connection() { - crate::DatabaseConnection::Sqlite(pool) => { - let transaction_count = { - let counter_result = query_validation_i64( - pool, - "SELECT COUNT(*) FROM k_sol_chain_transactions", - "transaction_count", - ) - .await; - match counter_result { - Ok(value) => value, - Err(error) => return Err(error), - } - }; - let ok_transaction_count = { - let counter_result = query_validation_i64( - pool, - "SELECT COUNT(*) FROM k_sol_chain_transactions WHERE err_json IS NULL", - "ok_transaction_count", - ) - .await; - match counter_result { - Ok(value) => value, - Err(error) => return Err(error), - } - }; - let failed_transaction_count = { - let counter_result = query_validation_i64( - pool, - "SELECT COUNT(*) FROM k_sol_chain_transactions WHERE err_json IS NOT NULL", - "failed_transaction_count", - ) - .await; - match counter_result { - Ok(value) => value, - Err(error) => return Err(error), - } - }; - let decoded_event_count = { - let counter_result = query_validation_i64( - pool, - "SELECT COUNT(*) FROM k_sol_dex_decoded_events", - "decoded_event_count", - ) - .await; - match counter_result { - Ok(value) => value, - Err(error) => return Err(error), - } - }; - let decoded_trade_candidate_count = { - let counter_result = query_validation_i64(pool, "SELECT COUNT(*) FROM k_sol_dex_decoded_events WHERE json_extract(payload_json, '$.tradeCandidate') = 1", "decoded_trade_candidate_count").await; - match counter_result { - Ok(value) => value, - Err(error) => return Err(error), - } - }; - let decoded_candle_candidate_count = { - let counter_result = query_validation_i64(pool, "SELECT COUNT(*) FROM k_sol_dex_decoded_events WHERE json_extract(payload_json, '$.candleCandidate') = 1", "decoded_candle_candidate_count").await; - match counter_result { - Ok(value) => value, - Err(error) => return Err(error), - } - }; - let decoded_non_trade_useful_event_count = { - let counter_result = query_validation_i64(pool, "SELECT COUNT(*) FROM k_sol_dex_decoded_events WHERE COALESCE(json_extract(payload_json, '$.nonTradeUseful'), 0) = 1 OR COALESCE(json_extract(payload_json, '$.eventActionability'), '') = 'non_trade_useful'", "decoded_non_trade_useful_event_count").await; - match counter_result { - Ok(value) => value, - Err(error) => return Err(error), - } - }; - let decoded_non_actionable_trade_event_count = { - let counter_result = query_validation_i64(pool, "SELECT COUNT(*) FROM k_sol_dex_decoded_events WHERE COALESCE(json_extract(payload_json, '$.eventActionability'), '') = 'non_actionable_trade' OR (COALESCE(json_extract(payload_json, '$.eventActionability'), '') = '' AND COALESCE(json_extract(payload_json, '$.eventCategory'), '') = 'trade' AND COALESCE(json_extract(payload_json, '$.tradeCandidate'), 0) = 0 AND COALESCE(json_extract(payload_json, '$.transactionFailed'), 0) = 0)", "decoded_non_actionable_trade_event_count").await; - match counter_result { - Ok(value) => value, - Err(error) => return Err(error), - } - }; - let decoded_unknown_event_count = { - let counter_result = query_validation_i64(pool, "SELECT COUNT(*) FROM k_sol_dex_decoded_events WHERE COALESCE(json_extract(payload_json, '$.eventCategory'), 'unknown') = 'unknown'", "decoded_unknown_event_count").await; - match counter_result { - Ok(value) => value, - Err(error) => return Err(error), - } - }; - let liquidity_event_count = { - let counter_result = query_validation_i64( - pool, - "SELECT COUNT(*) FROM k_sol_liquidity_events", - "liquidity_event_count", - ) - .await; - match counter_result { - Ok(value) => value, - Err(error) => return Err(error), - } - }; - let pool_lifecycle_event_count = { - let counter_result = query_validation_i64( - pool, - "SELECT COUNT(*) FROM k_sol_pool_lifecycle_events", - "pool_lifecycle_event_count", - ) - .await; - match counter_result { - Ok(value) => value, - Err(error) => return Err(error), - } - }; - let fee_event_count = { - let counter_result = query_validation_i64( - pool, - "SELECT COUNT(*) FROM k_sol_fee_events", - "fee_event_count", - ) - .await; - match counter_result { - Ok(value) => value, - Err(error) => return Err(error), - } - }; - let reward_event_count = { - let counter_result = query_validation_i64( - pool, - "SELECT COUNT(*) FROM k_sol_reward_events", - "reward_event_count", - ) - .await; - match counter_result { - Ok(value) => value, - Err(error) => return Err(error), - } - }; - let pool_admin_event_count = { - let counter_result = query_validation_i64( - pool, - "SELECT COUNT(*) FROM k_sol_pool_admin_events", - "pool_admin_event_count", - ) - .await; - match counter_result { - Ok(value) => value, - Err(error) => return Err(error), - } - }; - let missing_trade_event_count = { - let counter_result = query_validation_i64(pool, "SELECT COUNT(*) FROM k_sol_dex_decoded_events dde WHERE json_extract(dde.payload_json, '$.tradeCandidate') = 1 AND NOT EXISTS (SELECT 1 FROM k_sol_trade_events te WHERE te.decoded_event_id = dde.id)", "missing_trade_event_count").await; - match counter_result { - Ok(value) => value, - Err(error) => return Err(error), - } - }; - let decoded_trade_candidate_without_trade_event_count = missing_trade_event_count; - let decoded_trade_candidate_without_trade_event_on_ok_transaction_count = { - let counter_result = query_validation_i64(pool, "SELECT COUNT(*) FROM k_sol_dex_decoded_events dde JOIN k_sol_chain_transactions ct ON ct.id = dde.transaction_id WHERE json_extract(dde.payload_json, '$.tradeCandidate') = 1 AND NOT EXISTS (SELECT 1 FROM k_sol_trade_events te WHERE te.decoded_event_id = dde.id) AND ct.err_json IS NULL AND dde.pool_account IS NOT NULL AND dde.token_a_mint IS NOT NULL AND dde.token_b_mint IS NOT NULL AND EXISTS (SELECT 1 FROM k_sol_pools p JOIN k_sol_pairs pair ON pair.pool_id = p.id WHERE p.address = dde.pool_account)", "decoded_trade_candidate_without_trade_event_on_ok_transaction_count").await; - match counter_result { - Ok(value) => value, - Err(error) => return Err(error), - } - }; - let decoded_trade_candidate_without_trade_event_on_failed_transaction_count = { - let counter_result = query_validation_i64(pool, "SELECT COUNT(*) FROM k_sol_dex_decoded_events dde JOIN k_sol_chain_transactions ct ON ct.id = dde.transaction_id WHERE json_extract(dde.payload_json, '$.tradeCandidate') = 1 AND NOT EXISTS (SELECT 1 FROM k_sol_trade_events te WHERE te.decoded_event_id = dde.id) AND ct.err_json IS NOT NULL", "decoded_trade_candidate_without_trade_event_on_failed_transaction_count").await; - match counter_result { - Ok(value) => value, - Err(error) => return Err(error), - } - }; - let actionable_missing_trade_event_count = - decoded_trade_candidate_without_trade_event_on_ok_transaction_count; - let ignored_failed_transaction_trade_candidate_count = - decoded_trade_candidate_without_trade_event_on_failed_transaction_count; - let decoded_trade_candidate_without_amount_payload_count = { - let counter_result = query_validation_i64(pool, "SELECT COUNT(*) FROM k_sol_dex_decoded_events dde WHERE json_extract(dde.payload_json, '$.tradeCandidate') = 1 AND NOT EXISTS (SELECT 1 FROM k_sol_trade_events te WHERE te.decoded_event_id = dde.id) AND ((json_extract(dde.payload_json, '$.baseAmountRaw') IS NULL AND json_extract(dde.payload_json, '$.base_amount_raw') IS NULL) OR (json_extract(dde.payload_json, '$.quoteAmountRaw') IS NULL AND json_extract(dde.payload_json, '$.quote_amount_raw') IS NULL))", "decoded_trade_candidate_without_amount_payload_count").await; - match counter_result { - Ok(value) => value, - Err(error) => return Err(error), - } - }; - let trade_event_count = { - let counter_result = query_validation_i64( - pool, - "SELECT COUNT(*) FROM k_sol_trade_events", - "trade_event_count", - ) - .await; - match counter_result { - Ok(value) => value, - Err(error) => return Err(error), - } - }; - let invalid_trade_event_count = { - let counter_result = query_validation_i64(pool, "SELECT COUNT(*) FROM k_sol_trade_events WHERE base_amount_raw IS NULL OR quote_amount_raw IS NULL OR price_quote_per_base IS NULL OR CAST(base_amount_raw AS INTEGER) <= 0 OR CAST(quote_amount_raw AS INTEGER) <= 0 OR price_quote_per_base <= 0", "invalid_trade_event_count").await; - match counter_result { - Ok(value) => value, - Err(error) => return Err(error), - } - }; - let pair_candle_count = { - let counter_result = query_validation_i64( - pool, - "SELECT COUNT(*) FROM k_sol_pair_candles", - "pair_candle_count", - ) - .await; - match counter_result { - Ok(value) => value, - Err(error) => return Err(error), - } - }; - let duplicate_decoded_event_trade_count = { - let counter_result = query_validation_i64(pool, "SELECT COUNT(*) FROM (SELECT decoded_event_id FROM k_sol_trade_events WHERE decoded_event_id IS NOT NULL GROUP BY decoded_event_id HAVING COUNT(*) > 1)", "duplicate_decoded_event_trade_count").await; - match counter_result { - Ok(value) => value, - Err(error) => return Err(error), - } - }; - let multi_trade_signature_pair_count = { - let counter_result = query_validation_i64(pool, "SELECT COUNT(*) FROM (SELECT signature, pair_id FROM k_sol_trade_events GROUP BY signature, pair_id HAVING COUNT(*) > 1)", "multi_trade_signature_pair_count").await; - match counter_result { - Ok(value) => value, - Err(error) => return Err(error), - } - }; - let duplicate_candle_bucket_count = { - let counter_result = query_validation_i64(pool, "SELECT COUNT(*) FROM (SELECT pair_id, timeframe_seconds, bucket_start_unix FROM k_sol_pair_candles GROUP BY pair_id, timeframe_seconds, bucket_start_unix HAVING COUNT(*) > 1)", "duplicate_candle_bucket_count").await; - match counter_result { - Ok(value) => value, - Err(error) => return Err(error), - } - }; - let token_count = { - let counter_result = - query_validation_i64(pool, "SELECT COUNT(*) FROM k_sol_tokens", "token_count") - .await; - match counter_result { - Ok(value) => value, - Err(error) => return Err(error), - } - }; - let token_metadata_missing_count = { - let counter_result = query_validation_i64(pool, "SELECT COUNT(*) FROM k_sol_tokens WHERE symbol IS NULL OR TRIM(symbol) = '' OR name IS NULL OR TRIM(name) = ''", "token_metadata_missing_count").await; - match counter_result { - Ok(value) => value, - Err(error) => return Err(error), - } - }; - let tradable_token_metadata_missing_count = { - let counter_result = query_validation_i64(pool, "SELECT COUNT(DISTINCT token.id) FROM k_sol_tokens token JOIN (SELECT pair.base_token_id AS token_id FROM k_sol_pairs pair JOIN k_sol_trade_events te ON te.pair_id = pair.id UNION SELECT pair.quote_token_id AS token_id FROM k_sol_pairs pair JOIN k_sol_trade_events te ON te.pair_id = pair.id) tradable_pair_token ON tradable_pair_token.token_id = token.id WHERE token.symbol IS NULL OR TRIM(token.symbol) = '' OR token.name IS NULL OR TRIM(token.name) = ''", "tradable_token_metadata_missing_count").await; - match counter_result { - Ok(value) => value, - Err(error) => return Err(error), - } - }; - let quote_token_metadata_missing_count = { - let counter_result = query_validation_i64(pool, "SELECT COUNT(DISTINCT quote_token.id) FROM k_sol_pairs pair JOIN k_sol_tokens quote_token ON quote_token.id = pair.quote_token_id WHERE quote_token.symbol IS NULL OR TRIM(quote_token.symbol) = '' OR quote_token.name IS NULL OR TRIM(quote_token.name) = ''", "quote_token_metadata_missing_count").await; - match counter_result { - Ok(value) => value, - Err(error) => return Err(error), - } - }; - let pair_symbol_fallback_count = { - let counter_result = query_validation_i64(pool, "SELECT COUNT(*) FROM k_sol_pairs pair JOIN k_sol_tokens base_token ON base_token.id = pair.base_token_id JOIN k_sol_tokens quote_token ON quote_token.id = pair.quote_token_id WHERE pair.symbol IS NULL OR TRIM(pair.symbol) = '' OR pair.symbol = base_token.mint || '/' || quote_token.mint OR instr(pair.symbol, base_token.mint) > 0 OR instr(pair.symbol, quote_token.mint) > 0", "pair_symbol_fallback_count").await; - match counter_result { - Ok(value) => value, - Err(error) => return Err(error), - } - }; - let pair_symbol_resolved_count = { - let counter_result = query_validation_i64(pool, "SELECT COUNT(*) FROM k_sol_pairs pair JOIN k_sol_tokens base_token ON base_token.id = pair.base_token_id JOIN k_sol_tokens quote_token ON quote_token.id = pair.quote_token_id WHERE pair.symbol IS NOT NULL AND TRIM(pair.symbol) != '' AND pair.symbol != base_token.mint || '/' || quote_token.mint AND instr(pair.symbol, base_token.mint) = 0 AND instr(pair.symbol, quote_token.mint) = 0", "pair_symbol_resolved_count").await; - match counter_result { - Ok(value) => value, - Err(error) => return Err(error), - } - }; - let wsol_quote_pair_count = { - let counter_result = query_validation_i64(pool, "SELECT COUNT(*) FROM k_sol_pairs pair JOIN k_sol_tokens quote_token ON quote_token.id = pair.quote_token_id WHERE quote_token.mint = 'So11111111111111111111111111111111111111112'", "wsol_quote_pair_count").await; - match counter_result { - Ok(value) => value, - Err(error) => return Err(error), - } - }; - let stable_quote_pair_count = { - let counter_result = query_validation_i64(pool, "SELECT COUNT(*) FROM k_sol_pairs pair JOIN k_sol_tokens quote_token ON quote_token.id = pair.quote_token_id WHERE quote_token.mint IN ('EPjFWdd5AufqSSqeM2qN1xzybapC8G4wEGGkZwyTDt1v', 'Es9vMFrzaCERmJfrF4H2FYD4KCoNkY11McCe8BenwNYB', 'USD1ttGY1N17NEEHLmELoaybftRBUSErhqYiQzvEmuB', 'JuprjznTrTSp2UFa3ZBUFgwdAmtZCq4MQCwysN55USD')", "stable_quote_pair_count").await; - match counter_result { - Ok(value) => value, - Err(error) => return Err(error), - } - }; - let pool_count = { - let counter_result = - query_validation_i64(pool, "SELECT COUNT(*) FROM k_sol_pools", "pool_count") - .await; - match counter_result { - Ok(value) => value, - Err(error) => return Err(error), - } - }; - let pair_count = { - let counter_result = - query_validation_i64(pool, "SELECT COUNT(*) FROM k_sol_pairs", "pair_count") - .await; - match counter_result { - Ok(value) => value, - Err(error) => return Err(error), - } - }; - let literal_pair_without_trade_count = { - let counter_result = query_validation_i64(pool, "SELECT COUNT(*) FROM k_sol_pairs pair WHERE NOT EXISTS (SELECT 1 FROM k_sol_trade_events te WHERE te.pair_id = pair.id)", "literal_pair_without_trade_count").await; - match counter_result { - Ok(value) => value, - Err(error) => return Err(error), - } - }; - let literal_pair_without_candle_count = { - let counter_result = query_validation_i64(pool, "SELECT COUNT(*) FROM k_sol_pairs pair WHERE NOT EXISTS (SELECT 1 FROM k_sol_pair_candles pc WHERE pc.pair_id = pair.id)", "literal_pair_without_candle_count").await; - match counter_result { - Ok(value) => value, - Err(error) => return Err(error), - } - }; - let trade_materialized_pair_count = { - let counter_result = query_validation_i64( - pool, - "SELECT COUNT(DISTINCT pair_id) FROM k_sol_trade_events", - "trade_materialized_pair_count", - ) - .await; - match counter_result { - Ok(value) => value, - Err(error) => return Err(error), - } - }; - let candle_materialized_pair_count = { - let counter_result = query_validation_i64( - pool, - "SELECT COUNT(DISTINCT pair_id) FROM k_sol_pair_candles", - "candle_materialized_pair_count", - ) - .await; - match counter_result { - Ok(value) => value, - Err(error) => return Err(error), - } - }; - let actionable_pair_count = { - let counter_result = query_validation_i64(pool, "SELECT COUNT(*) FROM k_sol_pairs pair JOIN k_sol_pools p ON p.id = pair.pool_id WHERE EXISTS (SELECT 1 FROM k_sol_dex_decoded_events dde JOIN k_sol_chain_transactions ct ON ct.id = dde.transaction_id WHERE dde.pool_account = p.address AND json_extract(dde.payload_json, '$.tradeCandidate') = 1 AND ct.err_json IS NULL)", "actionable_pair_count").await; - match counter_result { - Ok(value) => value, - Err(error) => return Err(error), - } - }; - let candle_bucket_timeframe_count = { - let counter_result = query_validation_i64( - pool, - "SELECT COUNT(DISTINCT timeframe_seconds) FROM k_sol_pair_candles", - "candle_bucket_timeframe_count", - ) - .await; - match counter_result { - Ok(value) => value, - Err(error) => return Err(error), - } - }; - let non_actionable_pair_count = { - let counter_result = query_validation_i64(pool, "SELECT COUNT(*) FROM k_sol_pairs pair JOIN k_sol_pools p ON p.id = pair.pool_id WHERE EXISTS (SELECT 1 FROM k_sol_dex_decoded_events dde WHERE dde.pool_account = p.address AND json_extract(dde.payload_json, '$.tradeCandidate') = 1 AND NOT EXISTS (SELECT 1 FROM k_sol_trade_events te WHERE te.decoded_event_id = dde.id)) AND NOT EXISTS (SELECT 1 FROM k_sol_dex_decoded_events dde JOIN k_sol_chain_transactions ct ON ct.id = dde.transaction_id WHERE dde.pool_account = p.address AND json_extract(dde.payload_json, '$.tradeCandidate') = 1 AND ct.err_json IS NULL AND NOT EXISTS (SELECT 1 FROM k_sol_trade_events te WHERE te.decoded_event_id = dde.id))", "non_actionable_pair_count").await; - match counter_result { - Ok(value) => value, - Err(error) => return Err(error), - } - }; - let blocking_pair_without_trade_count = { - let counter_result = query_validation_i64(pool, "SELECT COUNT(*) FROM k_sol_pairs pair JOIN k_sol_pools p ON p.id = pair.pool_id WHERE EXISTS (SELECT 1 FROM k_sol_dex_decoded_events dde JOIN k_sol_chain_transactions ct ON ct.id = dde.transaction_id WHERE dde.pool_account = p.address AND json_extract(dde.payload_json, '$.tradeCandidate') = 1 AND ct.err_json IS NULL) AND NOT EXISTS (SELECT 1 FROM k_sol_trade_events te WHERE te.pair_id = pair.id)", "blocking_pair_without_trade_count").await; - match counter_result { - Ok(value) => value, - Err(error) => return Err(error), - } - }; - let blocking_pair_without_candle_count = { - let counter_result = query_validation_i64(pool, "SELECT COUNT(*) FROM k_sol_pairs pair JOIN k_sol_pools p ON p.id = pair.pool_id WHERE EXISTS (SELECT 1 FROM k_sol_dex_decoded_events dde JOIN k_sol_chain_transactions ct ON ct.id = dde.transaction_id WHERE dde.pool_account = p.address AND json_extract(dde.payload_json, '$.candleCandidate') = 1 AND ct.err_json IS NULL) AND NOT EXISTS (SELECT 1 FROM k_sol_pair_candles pc WHERE pc.pair_id = pair.id)", "blocking_pair_without_candle_count").await; - match counter_result { - Ok(value) => value, - Err(error) => return Err(error), - } - }; - return Ok(crate::LocalPipelineDiagnosticCountersDto { - transaction_count, - ok_transaction_count, - failed_transaction_count, - decoded_event_count, - decoded_trade_candidate_count, - decoded_candle_candidate_count, - decoded_non_trade_useful_event_count, - decoded_non_actionable_trade_event_count, - decoded_unknown_event_count, - liquidity_event_count, - pool_lifecycle_event_count, - fee_event_count, - reward_event_count, - pool_admin_event_count, - missing_trade_event_count, - decoded_trade_candidate_without_trade_event_count, - decoded_trade_candidate_without_trade_event_on_ok_transaction_count, - decoded_trade_candidate_without_trade_event_on_failed_transaction_count, - actionable_missing_trade_event_count, - ignored_failed_transaction_trade_candidate_count, - decoded_trade_candidate_without_amount_payload_count, - trade_event_count, - invalid_trade_event_count, - pair_candle_count, - duplicate_decoded_event_trade_count, - multi_trade_signature_pair_count, - duplicate_candle_bucket_count, - token_count, - token_metadata_missing_count, - tradable_token_metadata_missing_count, - quote_token_metadata_missing_count, - pair_symbol_fallback_count, - pair_symbol_resolved_count, - wsol_quote_pair_count, - stable_quote_pair_count, - pool_count, - pair_count, - literal_pair_without_trade_count, - literal_pair_without_candle_count, - trade_materialized_pair_count, - candle_materialized_pair_count, - actionable_pair_count, - candle_bucket_timeframe_count, - non_actionable_pair_count, - blocking_pair_without_trade_count, - blocking_pair_without_candle_count, - pair_without_trade_count: blocking_pair_without_trade_count, - pair_without_candle_count: blocking_pair_without_candle_count, - }); - }, - } -} - -async fn query_validation_i64( - pool: &sqlx::Pool, - sql: &'static str, - counter_name: &str, -) -> Result { - let result = sqlx::query_scalar::(sql).fetch_one(pool).await; - match result { - Ok(value) => return Ok(value), - Err(error) => { - return Err(crate::Error::Db(format!( - "cannot read local pipeline validation counter '{}' on sqlite: {}", - counter_name, error - ))); - }, - } -} - async fn load_event_coverage_summaries( database: &crate::Database, ) -> Result, crate::Error> { - let coverage_service = - crate::DexEventCoverageService::new(std::sync::Arc::new(database.clone())); - let refresh_result = coverage_service.refresh_local_counts(None).await; - let refresh_result = match refresh_result { - Ok(refresh_result) => refresh_result, + let refresh_result = + crate::query_dex_event_coverage_entries_refresh_local_counts(database).await; + match refresh_result { + Ok(_) => {}, Err(error) => return Err(error), - }; - return Ok(refresh_result.summaries); + } + let summaries_result = + crate::query_dex_event_coverage_entries_list_summary_by_decoder(database).await; + match summaries_result { + Ok(summaries) => return Ok(summaries), + Err(error) => return Err(error), + } } #[derive(Debug, Clone)] @@ -871,23 +417,41 @@ fn aggregate_event_coverage_summaries( upstream_git_local_corpus_materialized_entry_count: 0, }; for summary in summaries { - aggregate.listed_entry_count += summary.listed_entry_count; - aggregate.decoded_entry_count += summary.decoded_entry_count; - aggregate.observed_entry_count += summary.observed_entry_count; - aggregate.materialized_entry_count += summary.materialized_entry_count; - aggregate.total_observed_count += summary.total_observed_count; - aggregate.total_materialized_count += summary.total_materialized_count; - aggregate.trade_count += summary.trade_count; - aggregate.audit_only_entry_count += summary.audit_only_entry_count; - aggregate.missing_db_target_entry_count += summary.missing_db_target_entry_count; - aggregate.upstream_git_unverified_entry_count += - summary.upstream_git_unverified_entry_count; - aggregate.upstream_git_mapped_unverified_entry_count += - summary.upstream_git_mapped_unverified_entry_count; - aggregate.upstream_git_local_corpus_observed_entry_count += - summary.upstream_git_local_corpus_observed_entry_count; - aggregate.upstream_git_local_corpus_materialized_entry_count += - summary.upstream_git_local_corpus_materialized_entry_count; + aggregate.listed_entry_count = + aggregate.listed_entry_count.saturating_add(summary.listed_entry_count); + aggregate.decoded_entry_count = + aggregate.decoded_entry_count.saturating_add(summary.decoded_entry_count); + aggregate.observed_entry_count = aggregate + .observed_entry_count + .saturating_add(summary.observed_entry_count); + aggregate.materialized_entry_count = aggregate + .materialized_entry_count + .saturating_add(summary.materialized_entry_count); + aggregate.total_observed_count = aggregate + .total_observed_count + .saturating_add(summary.total_observed_count); + aggregate.total_materialized_count = aggregate + .total_materialized_count + .saturating_add(summary.total_materialized_count); + aggregate.trade_count = aggregate.trade_count.saturating_add(summary.trade_count); + aggregate.audit_only_entry_count = aggregate + .audit_only_entry_count + .saturating_add(summary.audit_only_entry_count); + aggregate.missing_db_target_entry_count = aggregate + .missing_db_target_entry_count + .saturating_add(summary.missing_db_target_entry_count); + aggregate.upstream_git_unverified_entry_count = aggregate + .upstream_git_unverified_entry_count + .saturating_add(summary.upstream_git_unverified_entry_count); + aggregate.upstream_git_mapped_unverified_entry_count = aggregate + .upstream_git_mapped_unverified_entry_count + .saturating_add(summary.upstream_git_mapped_unverified_entry_count); + aggregate.upstream_git_local_corpus_observed_entry_count = aggregate + .upstream_git_local_corpus_observed_entry_count + .saturating_add(summary.upstream_git_local_corpus_observed_entry_count); + aggregate.upstream_git_local_corpus_materialized_entry_count = aggregate + .upstream_git_local_corpus_materialized_entry_count + .saturating_add(summary.upstream_git_local_corpus_materialized_entry_count); } return aggregate; } diff --git a/kb_lib/src/local_pipeline_replay.rs b/kb_lib/src/local_pipeline_replay.rs index 2a50c87..fd26185 100644 --- a/kb_lib/src/local_pipeline_replay.rs +++ b/kb_lib/src/local_pipeline_replay.rs @@ -7,7 +7,7 @@ //! deterministic local pipeline over their signatures. const LOCAL_PIPELINE_DEX_DECODER_SCOPE: &str = "dex_decode.local_pipeline"; -const LOCAL_PIPELINE_DEX_DECODER_VERSION: &str = "dex_decode.v0.7.46.damm_v1_events1"; +const LOCAL_PIPELINE_DEX_DECODER_VERSION: &str = "dex_decode.v0.7.51.raydium_amm_v4_max_decoder"; fn default_skip_certified_dex_decode() -> bool { return true; @@ -280,6 +280,28 @@ impl LocalPipelineReplayService { ); }, None => { + let replay_scope_delete_result = + crate::query_dex_decoded_events_delete_local_replay_scope_by_transaction_id( + self.database.as_ref(), + transaction_id, + ) + .await; + match replay_scope_delete_result { + Ok(deleted_count) => { + result.reset_market_materialization_deleted_count = result + .reset_market_materialization_deleted_count + .saturating_add(deleted_count); + if deleted_count > 0 { + tracing::debug!( + signature = %signature, + transaction_id, + deleted_count, + "local pipeline replay deleted stale local DEX replay scope before decode" + ); + } + }, + Err(error) => return Err(error), + } let decode_result = dex_decode.decode_transaction_by_signature(signature.as_str()).await; match decode_result { diff --git a/kb_lib/src/non_trade_event_materialization.rs b/kb_lib/src/non_trade_event_materialization.rs index c531c9c..c13d109 100644 --- a/kb_lib/src/non_trade_event_materialization.rs +++ b/kb_lib/src/non_trade_event_materialization.rs @@ -482,38 +482,24 @@ impl NonTradeEventMaterializationService { Some(decoded_event_id) => decoded_event_id, None => return Ok(()), }; - match self.database.connection() { - crate::DatabaseConnection::Sqlite(pool) => { - let delete_result = sqlx::query( - r#" -DELETE FROM k_sol_pool_admin_events -WHERE decoded_event_id = ? - "#, - ) - .bind(decoded_event_id) - .execute(pool) - .await; - let delete_result = match delete_result { - Ok(delete_result) => delete_result, - Err(error) => { - return Err(crate::Error::Db(format!( - "cannot delete stale k_sol_pool_admin_events for lifecycle decoded_event_id '{}' on sqlite: {}", - decoded_event_id, error - ))); - }, - }; - let deleted_count = delete_result.rows_affected(); - if deleted_count > 0 { - tracing::debug!( - decoded_event_id = decoded_event_id, - event_kind = %decoded_event.event_kind, - deleted_count = deleted_count, - "removed stale pool admin materialization for lifecycle event" - ); - } - return Ok(()); - }, + let delete_result = crate::query_pool_admin_events_delete_by_decoded_event_id( + self.database.as_ref(), + decoded_event_id, + ) + .await; + let deleted_count = match delete_result { + Ok(deleted_count) => deleted_count, + Err(error) => return Err(error), + }; + if deleted_count > 0 { + tracing::debug!( + decoded_event_id = decoded_event_id, + event_kind = %decoded_event.event_kind, + deleted_count = deleted_count, + "removed stale pool admin materialization for lifecycle event" + ); } + return Ok(()); } async fn materialize_pool_admin_event( @@ -712,132 +698,28 @@ WHERE decoded_event_id = ? }, None => None, }; - match self.database.connection() { - crate::DatabaseConnection::Sqlite(pool) => { - let existing_result = sqlx::query_scalar::( - r#" -SELECT id -FROM k_sol_launch_events -WHERE decoded_event_id = ? -LIMIT 1 - "#, - ) - .bind(decoded_event_id) - .fetch_optional(pool) - .await; - let existing_id = match existing_result { - Ok(existing_id) => existing_id, - Err(error) => { - return Err(crate::Error::Db(format!( - "cannot fetch k_sol_launch_events id for decoded_event_id '{}' on sqlite: {}", - decoded_event_id, error - ))); - }, - }; - if let Some(existing_id) = existing_id { - let update_result = sqlx::query( - r#" -UPDATE k_sol_launch_events -SET - transaction_id = ?, - dex_id = ?, - pool_id = ?, - pair_id = ?, - signature = ?, - slot = ?, - protocol_name = ?, - program_id = ?, - event_kind = ?, - pool_account = ?, - actor_wallet = ?, - event_role = ?, - related_account = ?, - related_mint = ?, - payload_json = ?, - executed_at = ? -WHERE id = ? - "#, - ) - .bind(transaction_id) - .bind(context.dex_id) - .bind(context.pool_id) - .bind(context.pair_id) - .bind(transaction.signature.clone()) - .bind(slot_i64) - .bind(decoded_event.protocol_name.clone()) - .bind(decoded_event.program_id.clone()) - .bind(decoded_event.event_kind.clone()) - .bind(decoded_event.pool_account.clone()) - .bind(actor_wallet.clone()) - .bind(event_role.clone()) - .bind(related_account.clone()) - .bind(related_mint.clone()) - .bind(decoded_event.payload_json.clone()) - .bind(chrono::Utc::now().to_rfc3339()) - .bind(existing_id) - .execute(pool) - .await; - if let Err(error) = update_result { - return Err(crate::Error::Db(format!( - "cannot update k_sol_launch_events id '{}' on sqlite: {}", - existing_id, error - ))); - } - return Ok(true); - } - let insert_result = sqlx::query( - r#" -INSERT INTO k_sol_launch_events ( - transaction_id, - decoded_event_id, - dex_id, - pool_id, - pair_id, - signature, - slot, - protocol_name, - program_id, - event_kind, - pool_account, - actor_wallet, - event_role, - related_account, - related_mint, - payload_json, - executed_at, - created_at -) -VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) - "#, - ) - .bind(transaction_id) - .bind(decoded_event_id) - .bind(context.dex_id) - .bind(context.pool_id) - .bind(context.pair_id) - .bind(transaction.signature.clone()) - .bind(slot_i64) - .bind(decoded_event.protocol_name.clone()) - .bind(decoded_event.program_id.clone()) - .bind(decoded_event.event_kind.clone()) - .bind(decoded_event.pool_account.clone()) - .bind(actor_wallet) - .bind(event_role) - .bind(related_account) - .bind(related_mint) - .bind(decoded_event.payload_json.clone()) - .bind(chrono::Utc::now().to_rfc3339()) - .bind(chrono::Utc::now().to_rfc3339()) - .execute(pool) - .await; - if let Err(error) = insert_result { - return Err(crate::Error::Db(format!( - "cannot insert k_sol_launch_events on sqlite: {}", - error - ))); - } - return Ok(true); - }, + let input = crate::LaunchEventUpsertInput { + transaction_id, + decoded_event_id, + dex_id: context.dex_id, + pool_id: context.pool_id, + pair_id: context.pair_id, + signature: transaction.signature.clone(), + slot: slot_i64, + protocol_name: decoded_event.protocol_name.clone(), + program_id: decoded_event.program_id.clone(), + event_kind: decoded_event.event_kind.clone(), + pool_account: decoded_event.pool_account.clone(), + actor_wallet, + event_role, + related_account, + related_mint, + payload_json: payload.clone(), + }; + let upsert_result = crate::query_launch_events_upsert(self.database.as_ref(), &input).await; + match upsert_result { + Ok(_) => return Ok(true), + Err(error) => return Err(error), } } @@ -861,15 +743,51 @@ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) }; let dex_id = match context.dex_id { Some(dex_id) => dex_id, - None => return Ok(false), + None => { + let annotate_result = self + .annotate_decoded_event_payload( + decoded_event, + "skipLiquidityReason", + "missing_dex_catalog", + ) + .await; + if let Err(error) = annotate_result { + return Err(error); + } + return Ok(false); + }, }; let pool_id = match context.pool_id { Some(pool_id) => pool_id, - None => return Ok(false), + None => { + let annotate_result = self + .annotate_decoded_event_payload( + decoded_event, + "skipLiquidityReason", + "missing_pool_catalog", + ) + .await; + if let Err(error) = annotate_result { + return Err(error); + } + return Ok(false); + }, }; let pair = match context.pair { Some(pair) => pair, - None => return Ok(false), + None => { + let annotate_result = self + .annotate_decoded_event_payload( + decoded_event, + "skipLiquidityReason", + "missing_pair_catalog", + ) + .await; + if let Err(error) = annotate_result { + return Err(error); + } + return Ok(false); + }, }; let pair_id = match pair.id { Some(pair_id) => Some(pair_id), @@ -1087,6 +1005,57 @@ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) } } + async fn annotate_decoded_event_payload( + &self, + decoded_event: &crate::DexDecodedEventDto, + reason_key: &str, + reason_value: &str, + ) -> Result<(), crate::Error> { + let decoded_event_id = match decoded_event.id { + Some(decoded_event_id) => decoded_event_id, + None => return Ok(()), + }; + let payload_result = serde_json::from_str::( + decoded_event.payload_json.as_str(), + ); + let mut object = match payload_result { + Ok(serde_json::Value::Object(object)) => object, + Ok(other) => { + let mut object = serde_json::Map::new(); + object.insert("rawPayload".to_string(), other); + object + }, + Err(_) => serde_json::Map::new(), + }; + let existing_reason = match object.get(reason_key).and_then(serde_json::Value::as_str) { + Some(existing_reason) => existing_reason.trim().to_string(), + None => std::string::String::new(), + }; + if existing_reason.is_empty() { + object.insert( + reason_key.to_string(), + serde_json::Value::String(reason_value.to_string()), + ); + } + if reason_key == "skipLiquidityReason" { + object.insert( + "skipCatalogReason".to_string(), + serde_json::Value::String(reason_value.to_string()), + ); + } + let payload_json = serde_json::Value::Object(object).to_string(); + let update_result = crate::query_dex_decoded_events_update_payload_json_by_id( + self.database.as_ref(), + decoded_event_id, + payload_json.as_str(), + ) + .await; + match update_result { + Ok(_) => return Ok(()), + Err(error) => return Err(error), + } + } + async fn resolve_liquidity_context( &self, transaction: &crate::ChainTransactionDto, diff --git a/validation_sql/SQL_VALIDATION_RAYDIUM_AMM_V4_0_7_51.sql b/validation_sql/SQL_VALIDATION_RAYDIUM_AMM_V4_0_7_51.sql new file mode 100644 index 0000000..7bbdd15 --- /dev/null +++ b/validation_sql/SQL_VALIDATION_RAYDIUM_AMM_V4_0_7_51.sql @@ -0,0 +1,323 @@ +-- file: validation_sql/SQL_VALIDATION_RAYDIUM_AMM_V4_0_7_51.sql +-- Validation finale de la tranche 0.7.51 raydium_amm_v4. +-- À exécuter après replay forceDexDecode=yes / deferInstructionObservations=yes. + +-- 01. Coverage AMM v4 finale. +SELECT + entry_name, + entry_kind, + event_family, + expected_db_target, + proof_status, + local_event_kind, + discriminator_hex, + observed_count, + materialized_count, + trade_count +FROM k_sol_dex_event_coverage_entries +WHERE decoder_code = 'raydium_amm_v4' +ORDER BY entry_kind, entry_name, discriminator_hex; + +-- 02. Instruction observations AMM v4. +SELECT + instruction_name, + discriminator_hex, + COUNT(*) AS observed_count, + COUNT(DISTINCT signature) AS tx_count +FROM k_sol_instruction_observations +WHERE decoder_code = 'raydium_amm_v4' +GROUP BY instruction_name, discriminator_hex +ORDER BY observed_count DESC, instruction_name, discriminator_hex; + +-- 03. Legacy swap générique interdit. +SELECT + de.event_kind, + COUNT(*) AS decoded_count, + COUNT(te.id) AS trade_count +FROM k_sol_dex_decoded_events de +LEFT JOIN k_sol_trade_events te + ON te.decoded_event_id = de.id +WHERE de.protocol_name = 'raydium_amm_v4' + AND de.event_kind = 'raydium_amm_v4.swap' +GROUP BY de.event_kind; + +-- 04. Aucun decoded AMM v4 sans coverage entry. +SELECT + de.event_kind, + COUNT(*) AS decoded_count +FROM k_sol_dex_decoded_events de +LEFT JOIN k_sol_dex_event_coverage_entries ce + ON ce.decoder_code = 'raydium_amm_v4' + AND ce.local_event_kind = de.event_kind +WHERE de.protocol_name = 'raydium_amm_v4' + AND ce.id IS NULL +GROUP BY de.event_kind +ORDER BY decoded_count DESC, de.event_kind; + +-- 05. Les observations AMM v4 doivent rester en discriminant 1 octet. +SELECT + discriminator_hex, + COUNT(*) AS observation_count, + COUNT(DISTINCT signature) AS tx_count +FROM k_sol_instruction_observations +WHERE decoder_code = 'raydium_amm_v4' + AND discriminator_hex IS NOT NULL + AND length(discriminator_hex) > 2 +GROUP BY discriminator_hex +ORDER BY observation_count DESC, discriminator_hex; + +-- 06. Tous les discriminants officiels AMM v4 doivent être observés. +WITH expected(discriminator_hex, instruction_name) AS ( + VALUES + ('00', 'raydium_amm_v4.initialize'), + ('01', 'raydium_amm_v4.initialize2_pool'), + ('02', 'raydium_amm_v4.monitor_step'), + ('03', 'raydium_amm_v4.deposit'), + ('04', 'raydium_amm_v4.withdraw'), + ('05', 'raydium_amm_v4.migrate_to_open_book'), + ('06', 'raydium_amm_v4.set_params'), + ('07', 'raydium_amm_v4.withdraw_pnl'), + ('08', 'raydium_amm_v4.withdraw_srm'), + ('09', 'raydium_amm_v4.swap_base_in'), + ('0a', 'raydium_amm_v4.pre_initialize'), + ('0b', 'raydium_amm_v4.swap_base_out'), + ('0c', 'raydium_amm_v4.simulate_info'), + ('0d', 'raydium_amm_v4.admin_cancel_orders'), + ('0e', 'raydium_amm_v4.create_config_account'), + ('0f', 'raydium_amm_v4.update_config_account'), + ('10', 'raydium_amm_v4.swap_base_in_v2'), + ('11', 'raydium_amm_v4.swap_base_out_v2') +) +SELECT + expected.discriminator_hex, + expected.instruction_name +FROM expected +LEFT JOIN k_sol_instruction_observations io + ON io.decoder_code = 'raydium_amm_v4' + AND io.discriminator_hex = expected.discriminator_hex +WHERE io.id IS NULL +ORDER BY expected.discriminator_hex; + +-- 07. Residual instruction_audit. +SELECT + json_extract(payload_json, '$.discriminatorHex') AS discriminator_hex, + COUNT(*) AS audit_count, + COUNT(DISTINCT transaction_id) AS tx_count +FROM k_sol_dex_decoded_events +WHERE protocol_name = 'raydium_amm_v4' + AND event_kind = 'raydium_amm_v4.instruction_audit' +GROUP BY discriminator_hex +ORDER BY audit_count DESC, discriminator_hex; + +-- 08. Fallback upstream localement couvert. +SELECT + json_extract(ug.payload_json, '$.upstreamDecoderCode') AS upstream_decoder_code, + json_extract(ug.payload_json, '$.upstreamEntryName') AS entry_name, + json_extract(ug.payload_json, '$.upstreamDiscriminatorHex') AS discriminator_hex, + json_extract(ug.payload_json, '$.upstreamSourceRepo') AS source_repo, + COUNT(*) AS fallback_count, + COUNT(DISTINCT ug.transaction_id) AS tx_count +FROM k_sol_dex_decoded_events ug +JOIN k_sol_dex_event_coverage_entries ce + ON ce.decoder_code = json_extract(ug.payload_json, '$.upstreamDecoderCode') + AND ce.entry_name = json_extract(ug.payload_json, '$.upstreamEntryName') + AND ce.discriminator_hex = json_extract(ug.payload_json, '$.upstreamDiscriminatorHex') + AND ce.local_event_kind IS NOT NULL + AND ce.local_event_kind <> '' +WHERE ug.protocol_name = 'upstream_git' + AND ug.event_kind = 'upstream_git.instruction_match' + AND json_extract(ug.payload_json, '$.upstreamDecoderCode') = 'raydium_amm_v4' +GROUP BY upstream_decoder_code, entry_name, discriminator_hex, source_repo +ORDER BY fallback_count DESC, entry_name; + +-- 09. Non-swap safety : aucun non-swap ne doit produire de trade. +SELECT + de.event_kind, + ce.event_family, + COUNT(*) AS decoded_count, + COUNT(te.id) AS trade_count +FROM k_sol_dex_decoded_events de +LEFT JOIN k_sol_dex_event_coverage_entries ce + ON ce.decoder_code = 'raydium_amm_v4' + AND ce.local_event_kind = de.event_kind +LEFT JOIN k_sol_trade_events te + ON te.decoded_event_id = de.id +WHERE de.protocol_name = 'raydium_amm_v4' +GROUP BY de.event_kind, ce.event_family +HAVING ce.event_family <> 'swap' + AND COUNT(te.id) > 0 +ORDER BY trade_count DESC, de.event_kind; + +-- 10. Failed transaction safety : aucune failed tx ne doit être matérialisée en trade. +SELECT + de.event_kind, + COUNT(*) AS decoded_failed_count, + COUNT(te.id) AS trade_count +FROM k_sol_dex_decoded_events de +JOIN k_sol_chain_transactions tx + ON tx.id = de.transaction_id +LEFT JOIN k_sol_trade_events te + ON te.decoded_event_id = de.id +WHERE de.protocol_name = 'raydium_amm_v4' + AND tx.err_json IS NOT NULL + AND tx.err_json <> '' + AND tx.err_json <> 'null' +GROUP BY de.event_kind +HAVING COUNT(te.id) > 0 +ORDER BY trade_count DESC, de.event_kind; + +-- 11. Aucun successful non-materialized sans raison explicite. +SELECT + de.event_kind, + COUNT(*) AS unexplained_count +FROM k_sol_dex_decoded_events de +JOIN k_sol_chain_transactions tx + ON tx.id = de.transaction_id +LEFT JOIN k_sol_trade_events te + ON te.decoded_event_id = de.id +LEFT JOIN k_sol_liquidity_events le + ON le.decoded_event_id = de.id +LEFT JOIN k_sol_pool_lifecycle_events pe + ON pe.decoded_event_id = de.id +LEFT JOIN k_sol_fee_events fe + ON fe.decoded_event_id = de.id +LEFT JOIN k_sol_pool_admin_events ae + ON ae.decoded_event_id = de.id +LEFT JOIN k_sol_orderbook_events oe + ON oe.decoded_event_id = de.id +LEFT JOIN k_sol_token_account_events tae + ON tae.decoded_event_id = de.id +WHERE de.protocol_name = 'raydium_amm_v4' + AND ( + tx.err_json IS NULL + OR tx.err_json = '' + OR tx.err_json = 'null' + ) + AND te.id IS NULL + AND le.id IS NULL + AND pe.id IS NULL + AND fe.id IS NULL + AND ae.id IS NULL + AND oe.id IS NULL + AND tae.id IS NULL + AND COALESCE(TRIM(json_extract(de.payload_json, '$.skipTradeReason')), '') = '' + AND COALESCE(TRIM(json_extract(de.payload_json, '$.skipLiquidityReason')), '') = '' + AND COALESCE(TRIM(json_extract(de.payload_json, '$.skipLifecycleReason')), '') = '' + AND COALESCE(TRIM(json_extract(de.payload_json, '$.skipCatalogReason')), '') = '' + AND de.event_kind NOT IN ( + 'raydium_amm_v4.simulate_info' + ) +GROUP BY de.event_kind +ORDER BY unexplained_count DESC, de.event_kind; + +-- 12. pre_initialize lifecycle audit minimal. +SELECT + COUNT(*) AS decoded_success_count, + COUNT(pe.id) AS lifecycle_count +FROM k_sol_dex_decoded_events de +JOIN k_sol_chain_transactions tx + ON tx.id = de.transaction_id +LEFT JOIN k_sol_pool_lifecycle_events pe + ON pe.decoded_event_id = de.id +WHERE de.protocol_name = 'raydium_amm_v4' + AND de.event_kind = 'raydium_amm_v4.pre_initialize' + AND ( + tx.err_json IS NULL + OR tx.err_json = '' + OR tx.err_json = 'null' + ); + +-- 13. Dépôts : contexte catalogue pool/pair. +SELECT + de.pool_account, + COUNT(DISTINCT de.id) AS decoded_count, + COUNT(DISTINCT le.id) AS liquidity_count, + COUNT(DISTINCT p.id) AS pool_catalog_count, + COUNT(DISTINCT pair.id) AS pair_catalog_count, + GROUP_CONCAT(DISTINCT pair.id) AS pair_ids +FROM k_sol_dex_decoded_events de +LEFT JOIN k_sol_liquidity_events le + ON le.decoded_event_id = de.id +LEFT JOIN k_sol_pools p + ON p.address = de.pool_account +LEFT JOIN k_sol_pairs pair + ON pair.pool_id = p.id +WHERE de.protocol_name = 'raydium_amm_v4' + AND de.event_kind = 'raydium_amm_v4.deposit' +GROUP BY de.pool_account +ORDER BY decoded_count DESC, de.pool_account; + +-- 14. Matérialisation single-target : aucun event AMM v4 ne doit alimenter deux tables métier principales. +SELECT + de.event_kind, + COUNT(DISTINCT de.id) AS decoded_count, + COUNT(DISTINCT te.id) AS trade_count, + COUNT(DISTINCT le.id) AS liquidity_count, + COUNT(DISTINCT pe.id) AS lifecycle_count, + COUNT(DISTINCT fe.id) AS fee_count, + COUNT(DISTINCT ae.id) AS admin_count, + COUNT(DISTINCT oe.id) AS orderbook_count, + ( + CASE WHEN COUNT(DISTINCT te.id) > 0 THEN 1 ELSE 0 END + + CASE WHEN COUNT(DISTINCT le.id) > 0 THEN 1 ELSE 0 END + + CASE WHEN COUNT(DISTINCT pe.id) > 0 THEN 1 ELSE 0 END + + CASE WHEN COUNT(DISTINCT fe.id) > 0 THEN 1 ELSE 0 END + + CASE WHEN COUNT(DISTINCT ae.id) > 0 THEN 1 ELSE 0 END + + CASE WHEN COUNT(DISTINCT oe.id) > 0 THEN 1 ELSE 0 END + ) AS materialized_target_count +FROM k_sol_dex_decoded_events de +LEFT JOIN k_sol_trade_events te + ON te.decoded_event_id = de.id +LEFT JOIN k_sol_liquidity_events le + ON le.decoded_event_id = de.id +LEFT JOIN k_sol_pool_lifecycle_events pe + ON pe.decoded_event_id = de.id +LEFT JOIN k_sol_fee_events fe + ON fe.decoded_event_id = de.id +LEFT JOIN k_sol_pool_admin_events ae + ON ae.decoded_event_id = de.id +LEFT JOIN k_sol_orderbook_events oe + ON oe.decoded_event_id = de.id +WHERE de.protocol_name = 'raydium_amm_v4' +GROUP BY de.event_kind +HAVING materialized_target_count > 1 +ORDER BY materialized_target_count DESC, de.event_kind; + +-- 15. Résumé par table métier. +SELECT + de.event_kind, + COUNT(DISTINCT de.id) AS decoded_count, + COUNT(DISTINCT te.id) AS trade_count, + COUNT(DISTINCT le.id) AS liquidity_count, + COUNT(DISTINCT pe.id) AS lifecycle_count, + COUNT(DISTINCT fe.id) AS fee_count, + COUNT(DISTINCT ae.id) AS admin_count, + COUNT(DISTINCT oe.id) AS orderbook_count +FROM k_sol_dex_decoded_events de +LEFT JOIN k_sol_trade_events te + ON te.decoded_event_id = de.id +LEFT JOIN k_sol_liquidity_events le + ON le.decoded_event_id = de.id +LEFT JOIN k_sol_pool_lifecycle_events pe + ON pe.decoded_event_id = de.id +LEFT JOIN k_sol_fee_events fe + ON fe.decoded_event_id = de.id +LEFT JOIN k_sol_pool_admin_events ae + ON ae.decoded_event_id = de.id +LEFT JOIN k_sol_orderbook_events oe + ON oe.decoded_event_id = de.id +WHERE de.protocol_name = 'raydium_amm_v4' +GROUP BY de.event_kind +ORDER BY de.event_kind; + +-- 16. Décision raydium_pool_v4 : aucune promotion runtime attendue en 0.7.51. +SELECT + decoder_code, + entry_name, + local_event_kind, + proof_status, + observed_count, + materialized_count +FROM k_sol_dex_event_coverage_entries +WHERE decoder_code = 'raydium_pool_v4' +ORDER BY entry_name; diff --git a/validation_sql/SQL_VALIDATION_RAYDIUM_AMM_V4_0_7_51_PRE2_REPLAY_CLEANUP.sql b/validation_sql/SQL_VALIDATION_RAYDIUM_AMM_V4_0_7_51_PRE2_REPLAY_CLEANUP.sql new file mode 100644 index 0000000..bd9723b --- /dev/null +++ b/validation_sql/SQL_VALIDATION_RAYDIUM_AMM_V4_0_7_51_PRE2_REPLAY_CLEANUP.sql @@ -0,0 +1,117 @@ +-- file: validation_sql/SQL_VALIDATION_RAYDIUM_AMM_V4_0_7_51_PRE2_REPLAY_CLEANUP.sql +-- Raydium AMM v4 replay cleanup checks after 0.7.51-pre.2. + +-- 1. Legacy generic AMM v4 swap must be gone. +SELECT + de.event_kind, + COUNT(*) AS decoded_count, + COUNT(te.id) AS trade_count +FROM k_sol_dex_decoded_events de +LEFT JOIN k_sol_trade_events te + ON te.decoded_event_id = de.id +WHERE de.protocol_name = 'raydium_amm_v4' + AND de.event_kind = 'raydium_amm_v4.swap' +GROUP BY de.event_kind; + +-- 2. All AMM v4 decoded event kinds must have coverage entries. +SELECT + de.event_kind, + COUNT(*) AS decoded_count +FROM k_sol_dex_decoded_events de +LEFT JOIN k_sol_dex_event_coverage_entries ce + ON ce.decoder_code = 'raydium_amm_v4' + AND ce.local_event_kind = de.event_kind +WHERE de.protocol_name = 'raydium_amm_v4' + AND ce.id IS NULL +GROUP BY de.event_kind +ORDER BY decoded_count DESC, de.event_kind; + +-- 3. AMM v4 instruction observations must use 1-byte discriminators. +SELECT + discriminator_hex, + COUNT(*) AS observation_count, + COUNT(DISTINCT signature) AS tx_count +FROM k_sol_instruction_observations +WHERE decoder_code = 'raydium_amm_v4' + AND discriminator_hex IS NOT NULL + AND length(discriminator_hex) > 2 +GROUP BY discriminator_hex +ORDER BY observation_count DESC, discriminator_hex; + +-- 4. Non-swap AMM v4 events must never materialize trades. +SELECT + de.event_kind, + ce.event_family, + COUNT(*) AS decoded_count, + COUNT(te.id) AS trade_count +FROM k_sol_dex_decoded_events de +LEFT JOIN k_sol_dex_event_coverage_entries ce + ON ce.decoder_code = 'raydium_amm_v4' + AND ce.local_event_kind = de.event_kind +LEFT JOIN k_sol_trade_events te + ON te.decoded_event_id = de.id +WHERE de.protocol_name = 'raydium_amm_v4' +GROUP BY de.event_kind, ce.event_family +HAVING ce.event_family <> 'swap' + AND COUNT(te.id) > 0 +ORDER BY trade_count DESC, de.event_kind; + +-- 5. Failed AMM v4 transactions must never materialize trades. +SELECT + de.event_kind, + COUNT(*) AS decoded_failed_count, + COUNT(te.id) AS trade_count +FROM k_sol_dex_decoded_events de +JOIN k_sol_chain_transactions tx + ON tx.id = de.transaction_id +LEFT JOIN k_sol_trade_events te + ON te.decoded_event_id = de.id +WHERE de.protocol_name = 'raydium_amm_v4' + AND tx.err_json IS NOT NULL + AND tx.err_json <> '' + AND tx.err_json <> 'null' +GROUP BY de.event_kind +HAVING COUNT(te.id) > 0 +ORDER BY trade_count DESC, de.event_kind; + +-- 6. AMM v4 successful swaps missing trade materialization. +SELECT + de.event_kind, + COUNT(*) AS decoded_success_count, + COUNT(te.id) AS trade_count +FROM k_sol_dex_decoded_events de +JOIN k_sol_chain_transactions tx + ON tx.id = de.transaction_id +LEFT JOIN k_sol_trade_events te + ON te.decoded_event_id = de.id +WHERE de.protocol_name = 'raydium_amm_v4' + AND de.event_kind IN ( + 'raydium_amm_v4.swap_base_in', + 'raydium_amm_v4.swap_base_in_v2', + 'raydium_amm_v4.swap_base_out', + 'raydium_amm_v4.swap_base_out_v2' + ) + AND ( + tx.err_json IS NULL + OR tx.err_json = '' + OR tx.err_json = 'null' + ) +GROUP BY de.event_kind +HAVING COUNT(te.id) < COUNT(*) +ORDER BY de.event_kind; + +-- 7. AMM v4 coverage table after refresh. +SELECT + entry_name, + entry_kind, + event_family, + expected_db_target, + proof_status, + local_event_kind, + discriminator_hex, + observed_count, + materialized_count, + trade_count +FROM k_sol_dex_event_coverage_entries +WHERE decoder_code = 'raydium_amm_v4' +ORDER BY entry_kind, entry_name, discriminator_hex; diff --git a/validation_sql/SQL_VALIDATION_RAYDIUM_AMM_V4_0_7_51_PRE3_MATERIALIZATION_EXPLANATIONS.sql b/validation_sql/SQL_VALIDATION_RAYDIUM_AMM_V4_0_7_51_PRE3_MATERIALIZATION_EXPLANATIONS.sql new file mode 100644 index 0000000..97c433f --- /dev/null +++ b/validation_sql/SQL_VALIDATION_RAYDIUM_AMM_V4_0_7_51_PRE3_MATERIALIZATION_EXPLANATIONS.sql @@ -0,0 +1,156 @@ +-- file: validation_sql/SQL_VALIDATION_RAYDIUM_AMM_V4_0_7_51_PRE3_MATERIALIZATION_EXPLANATIONS.sql +-- Raydium AMM v4 0.7.51-pre.3 materialization explanation checks. + +-- 1. AMM v4 legacy generic swap must stay absent. +SELECT + de.event_kind, + COUNT(*) AS decoded_count, + COUNT(te.id) AS trade_count +FROM k_sol_dex_decoded_events de +LEFT JOIN k_sol_trade_events te + ON te.decoded_event_id = de.id +WHERE de.protocol_name = 'raydium_amm_v4' + AND de.event_kind = 'raydium_amm_v4.swap' +GROUP BY de.event_kind; + +-- 2. AMM v4 decoded events must all have coverage entries. +SELECT + de.event_kind, + COUNT(*) AS decoded_count +FROM k_sol_dex_decoded_events de +LEFT JOIN k_sol_dex_event_coverage_entries ce + ON ce.decoder_code = 'raydium_amm_v4' + AND ce.local_event_kind = de.event_kind +WHERE de.protocol_name = 'raydium_amm_v4' + AND ce.id IS NULL +GROUP BY de.event_kind +ORDER BY decoded_count DESC, de.event_kind; + +-- 3. AMM v4 observations must use one-byte discriminators. +SELECT + discriminator_hex, + COUNT(*) AS observation_count, + COUNT(DISTINCT signature) AS tx_count +FROM k_sol_instruction_observations +WHERE decoder_code = 'raydium_amm_v4' + AND discriminator_hex IS NOT NULL + AND length(discriminator_hex) > 2 +GROUP BY discriminator_hex +ORDER BY observation_count DESC, discriminator_hex; + +-- 4. Non-swap AMM v4 events must never materialize as trades. +SELECT + de.event_kind, + ce.event_family, + COUNT(*) AS decoded_count, + COUNT(te.id) AS trade_count +FROM k_sol_dex_decoded_events de +LEFT JOIN k_sol_dex_event_coverage_entries ce + ON ce.decoder_code = 'raydium_amm_v4' + AND ce.local_event_kind = de.event_kind +LEFT JOIN k_sol_trade_events te + ON te.decoded_event_id = de.id +WHERE de.protocol_name = 'raydium_amm_v4' +GROUP BY de.event_kind, ce.event_family +HAVING ce.event_family <> 'swap' + AND COUNT(te.id) > 0 +ORDER BY trade_count DESC, de.event_kind; + +-- 5. Failed transactions must not materialize AMM v4 trades. +SELECT + de.event_kind, + COUNT(*) AS decoded_failed_count, + COUNT(te.id) AS trade_count +FROM k_sol_dex_decoded_events de +JOIN k_sol_chain_transactions tx + ON tx.id = de.transaction_id +LEFT JOIN k_sol_trade_events te + ON te.decoded_event_id = de.id +WHERE de.protocol_name = 'raydium_amm_v4' + AND tx.err_json IS NOT NULL + AND tx.err_json <> '' + AND tx.err_json <> 'null' +GROUP BY de.event_kind +HAVING COUNT(te.id) > 0 +ORDER BY trade_count DESC, de.event_kind; + +-- 6. Successful AMM v4 decoded events that are not materialized must carry an explanation. +SELECT + de.event_kind, + COUNT(*) AS unexplained_count +FROM k_sol_dex_decoded_events de +JOIN k_sol_chain_transactions tx + ON tx.id = de.transaction_id +LEFT JOIN k_sol_trade_events te + ON te.decoded_event_id = de.id +LEFT JOIN k_sol_liquidity_events le + ON le.decoded_event_id = de.id +LEFT JOIN k_sol_pool_lifecycle_events pe + ON pe.decoded_event_id = de.id +LEFT JOIN k_sol_fee_events fe + ON fe.decoded_event_id = de.id +LEFT JOIN k_sol_pool_admin_events ae + ON ae.decoded_event_id = de.id +LEFT JOIN k_sol_orderbook_events oe + ON oe.decoded_event_id = de.id +LEFT JOIN k_sol_token_account_events tae + ON tae.decoded_event_id = de.id +WHERE de.protocol_name = 'raydium_amm_v4' + AND ( + tx.err_json IS NULL + OR tx.err_json = '' + OR tx.err_json = 'null' + ) + AND te.id IS NULL + AND le.id IS NULL + AND pe.id IS NULL + AND fe.id IS NULL + AND ae.id IS NULL + AND oe.id IS NULL + AND tae.id IS NULL + AND COALESCE(TRIM(json_extract(de.payload_json, '$.skipTradeReason')), '') = '' + AND COALESCE(TRIM(json_extract(de.payload_json, '$.skipLiquidityReason')), '') = '' + AND COALESCE(TRIM(json_extract(de.payload_json, '$.skipLifecycleReason')), '') = '' + AND COALESCE(TRIM(json_extract(de.payload_json, '$.skipCatalogReason')), '') = '' + AND de.event_kind NOT IN ( + 'raydium_amm_v4.simulate_info' + ) +GROUP BY de.event_kind +ORDER BY unexplained_count DESC, de.event_kind; + +-- 7. pre_initialize should materialize successful legacy/deprecated instructions as lifecycle audit rows. +SELECT + COUNT(*) AS decoded_success_count, + COUNT(pe.id) AS lifecycle_count +FROM k_sol_dex_decoded_events de +JOIN k_sol_chain_transactions tx + ON tx.id = de.transaction_id +LEFT JOIN k_sol_pool_lifecycle_events pe + ON pe.decoded_event_id = de.id +WHERE de.protocol_name = 'raydium_amm_v4' + AND de.event_kind = 'raydium_amm_v4.pre_initialize' + AND ( + tx.err_json IS NULL + OR tx.err_json = '' + OR tx.err_json = 'null' + ); + +-- 8. Deposit context by real catalog keys. +SELECT + de.pool_account, + COUNT(DISTINCT de.id) AS decoded_count, + COUNT(DISTINCT le.id) AS liquidity_count, + COUNT(DISTINCT p.id) AS pool_catalog_count, + COUNT(DISTINCT pair.id) AS pair_catalog_count, + GROUP_CONCAT(DISTINCT pair.id) AS pair_ids +FROM k_sol_dex_decoded_events de +LEFT JOIN k_sol_liquidity_events le + ON le.decoded_event_id = de.id +LEFT JOIN k_sol_pools p + ON p.address = de.pool_account +LEFT JOIN k_sol_pairs pair + ON pair.pool_id = p.id +WHERE de.protocol_name = 'raydium_amm_v4' + AND de.event_kind = 'raydium_amm_v4.deposit' +GROUP BY de.pool_account +ORDER BY decoded_count DESC, de.pool_account;