From 7f130dba6b54fc56b3ecff3512b40321c86565f5 Mon Sep 17 00:00:00 2001 From: SinuS Von SifriduS Date: Mon, 11 May 2026 11:02:47 +0200 Subject: [PATCH] 0.7.28 --- CHANGELOG.md | 1 + Cargo.toml | 2 +- README.md | 463 +-- ROADMAP.md | 344 +- kb_demo_app/frontend/demo_pipeline2.html | 47 +- ...ipeline2ProtocolCandidateSummaryPayload.ts | 10 + ...ipeline2ProtocolCandidateSummaryRequest.ts | 10 + kb_demo_app/frontend/ts/demo_pipeline2.ts | 49 + kb_demo_app/package.json | 2 +- kb_demo_app/src/demo_pipeline2.rs | 64 + kb_demo_app/src/lib.rs | 1 + kb_demo_app/tauri.conf.json | 2 +- kb_lib/src/constants.rs | 149 +- kb_lib/src/db.rs | 16 + kb_lib/src/db/dtos.rs | 6 + kb_lib/src/db/dtos/protocol_candidate.rs | 114 + .../src/db/dtos/protocol_candidate_summary.rs | 96 + .../src/db/dtos/transaction_classification.rs | 130 + kb_lib/src/db/entities.rs | 6 + kb_lib/src/db/entities/protocol_candidate.rs | 32 + .../db/entities/protocol_candidate_summary.rs | 30 + .../db/entities/transaction_classification.rs | 32 + kb_lib/src/db/queries.rs | 12 + kb_lib/src/db/queries/protocol_candidate.rs | 337 ++ .../db/queries/transaction_classification.rs | 263 ++ kb_lib/src/db/schema.rs | 498 +++ kb_lib/src/detect/solana_ws.rs | 18 +- kb_lib/src/dex_catalog.rs | 295 ++ kb_lib/src/dex_decode.rs | 2884 ++++--------- kb_lib/src/dex_decode_context.rs | 55 + .../src/dex_decoded_event_materialization.rs | 140 + kb_lib/src/dex_detect.rs | 3633 ++--------------- kb_lib/src/dex_detection_route.rs | 131 + kb_lib/src/dex_event_classification.rs | 509 +++ kb_lib/src/dex_pool_materialization.rs | 662 +++ kb_lib/src/lib.rs | 199 +- kb_lib/src/local_pipeline_replay.rs | 22 + kb_lib/src/protocol_candidate_recording.rs | 530 +++ kb_lib/src/token_backfill.rs | 11 + kb_lib/src/trade_aggregation.rs | 2855 +------------ kb_lib/src/trade_aggregation_context.rs | 216 + kb_lib/src/trade_amount_resolution.rs | 650 +++ kb_lib/src/trade_event_materialization.rs | 230 ++ kb_lib/src/trade_metric_update.rs | 279 ++ kb_lib/src/trade_pump_swap_amounts.rs | 1233 ++++++ kb_lib/src/trade_side_resolution.rs | 118 + kb_lib/src/trade_solana_amounts.rs | 913 +++++ kb_lib/src/transaction_classification.rs | 466 +++ kb_lib/src/tx_resolution.rs | 17 + 49 files changed, 10301 insertions(+), 8481 deletions(-) create mode 100644 kb_demo_app/frontend/ts/bindings/DemoPipeline2ProtocolCandidateSummaryPayload.ts create mode 100644 kb_demo_app/frontend/ts/bindings/DemoPipeline2ProtocolCandidateSummaryRequest.ts create mode 100644 kb_lib/src/db/dtos/protocol_candidate.rs create mode 100644 kb_lib/src/db/dtos/protocol_candidate_summary.rs create mode 100644 kb_lib/src/db/dtos/transaction_classification.rs create mode 100644 kb_lib/src/db/entities/protocol_candidate.rs create mode 100644 kb_lib/src/db/entities/protocol_candidate_summary.rs create mode 100644 kb_lib/src/db/entities/transaction_classification.rs create mode 100644 kb_lib/src/db/queries/protocol_candidate.rs create mode 100644 kb_lib/src/db/queries/transaction_classification.rs create mode 100644 kb_lib/src/dex_catalog.rs create mode 100644 kb_lib/src/dex_decode_context.rs create mode 100644 kb_lib/src/dex_decoded_event_materialization.rs create mode 100644 kb_lib/src/dex_detection_route.rs create mode 100644 kb_lib/src/dex_event_classification.rs create mode 100644 kb_lib/src/dex_pool_materialization.rs create mode 100644 kb_lib/src/protocol_candidate_recording.rs create mode 100644 kb_lib/src/trade_aggregation_context.rs create mode 100644 kb_lib/src/trade_amount_resolution.rs create mode 100644 kb_lib/src/trade_event_materialization.rs create mode 100644 kb_lib/src/trade_metric_update.rs create mode 100644 kb_lib/src/trade_pump_swap_amounts.rs create mode 100644 kb_lib/src/trade_side_resolution.rs create mode 100644 kb_lib/src/trade_solana_amounts.rs create mode 100644 kb_lib/src/transaction_classification.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 823120b..77287c9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -58,3 +58,4 @@ 0.7.25 - Enrichissement metadata des tokens, avec résolution locale limitée à SOL / WSOL, résolution des autres mints via comptes on-chain, Token-2022, Metaplex ou payloads DEX, et conservation explicite des cas non résolus 0.7.26 - Diagnostics locaux du pipeline persisté, correction de l’agrégation instruction-scoped des swaps Raydium, clarification des compteurs de replay/upsert, et validation qu’aucun trade candidate issu d’une transaction OK n’est perdu 0.7.27 - Validation multi-DEX et non-régression du pipeline sur Pump.fun, PumpSwap, Raydium CPMM et Raydium CLMM, avec corpus de tests, diagnostics de référence et garanties sur les événements non pricés +0.7.28 - nettoyer la couche DEX avant d’ajouter de nouveaux protocoles, sans modifier le transport HTTP/WS déjà stabilisé. diff --git a/Cargo.toml b/Cargo.toml index e0b3c15..a429edd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,7 +8,7 @@ members = [ ] [workspace.package] -version = "0.7.27" +version = "0.7.28" edition = "2024" license = "MIT" repository = "https://git.sasedev.com/Sasedev/khadhroony-bobobot" diff --git a/README.md b/README.md index 3d75785..36b0ca0 100644 --- a/README.md +++ b/README.md @@ -2,256 +2,259 @@ # khadhroony-bobobot -Projet personnel Rust de détection, d’analyse de patterns et de trading/swap semi-automatisé de tokens et meme-tokens sur la blockchain Solana. +`khadhroony-bobobot` est un workspace Rust destiné à la détection, au décodage, à l’analyse et, à terme, au trading semi-automatisé de tokens Solana. + +Le README précédent décrivait surtout l’état `0.3.1`. Ce fichier reflète l’état de reprise autour de `0.7.27` : le socle transport HTTP/WS, la résolution transactionnelle, le modèle SQLite, plusieurs connecteurs DEX, les candles, les signaux analytiques et l’application de démonstration existent déjà. ## 1. Objectif -L’objectif du projet est de construire une application capable de : +L’objectif opérationnel est de construire progressivement une application capable de : -- détecter la création de tokens et de paires sur Solana, -- suivre leur évolution technique et de marché, -- collecter des informations utiles au filtrage, -- analyser des patterns statistiques ou comportementaux, -- préparer ensuite la gestion de wallets et les opérations de swap/trading. +- détecter l’apparition de nouveaux tokens, pools, paires et listings sur Solana ; +- identifier la première source de mint ou de lancement, même lorsque le token migre ensuite vers un autre DEX ; +- décoder les transactions pertinentes des DEX et launch surfaces ciblés ; +- séparer les swaps/candles des événements utiles seulement à l’analyse : liquidité, cycle de vie de pool, fees, rewards, administration, wallets observés ; +- produire des métriques exploitables : prix, volume, candles/OHLCV, activité, bursts, déséquilibres buy/sell, signaux analytiques ; +- préparer ensuite des règles déterministes de filtrage, d’achat, de vente, de stop-loss et de trailing stop ; +- conserver une traçabilité locale suffisante pour rejouer, diagnostiquer et améliorer les décodeurs. -Les cibles observées incluent notamment les DEX et protocoles tels que : +Le but court terme n’est pas encore le live trading. Le but court terme est de fiabiliser le décodage multi-DEX et la matérialisation des objets métier nécessaires au trading. -- Pump.fun -- PumpSwap -- Raydium -- Meteora -- Bags -- FluxBeam -- LaunchLab / LaunchBeam -- Heaven -- DexLab -- Moonit -- Zora +## 2. Workspace -La liste exacte pourra évoluer au fil du projet. +Le workspace contient deux crates principales. -## 2. Architecture générale +| Crate | Rôle | +|---|---| +| `kb_lib` | Bibliothèque métier : configuration, tracing, clients réseau, pool HTTP, manager WS, résolution transactionnelle, décodage DEX, détection métier, persistance SQLite, backfill, metadata, candles, signaux analytiques. | +| `kb_demo_app` | Application Tauri V2 de démonstration et d’inspection : fenêtres `Demo Ws`, `Demo Ws Manager`, `Demo Http`, `Demo Pipeline`, `Demo Pipeline 2`, graphiques candles et commandes de backfill/replay. | -Le workspace Rust est organisé autour de deux sous-crates principales : +La logique métier doit rester dans `kb_lib`. `kb_demo_app` doit rester une façade UI/Tauri et ne doit pas récupérer de logique Solana ou DEX profonde. -- `kb_lib` -- `kb_demo_app` +## 3. État actuel autour de `0.7.27` -### `kb_lib` +### 3.1. Socle stabilisé à ne pas refactorer maintenant -`kb_lib` porte la logique métier et technique : +Ces éléments fonctionnent et ne sont pas bloquants pour les DEX. Ils ne doivent pas être remaniés dans la phase immédiate : -- configuration JSON, -- tracing, -- constantes Solana, -- clients réseau HTTP / WebSocket, -- gestion future de gRPC Yellowstone, -- persistance, -- analyse de patterns, -- gestion des wallets, -- logique de détection et de filtrage. +- `ws_client.rs` ; +- `ws_manager.rs` ; +- `http_client.rs` ; +- `http_pool.rs` ; +- couches JSON-RPC WS/HTTP déjà stabilisées ; +- orchestration réseau utilisée par les fenêtres de démonstration. + +Ils pourront être améliorés plus tard, mais la priorité actuelle est le décodage DEX, les événements métier et les tables d’analyse. + +### 3.2. Pipeline métier existant + +Le pipeline `0.7.x` couvre déjà les étapes suivantes : -### `kb_demo_app` +1. réception d’observations via RPC WS ou backfill HTTP ; +2. résolution des transactions via HTTP RPC ; +3. projection transactionnelle normalisée en base ; +4. décodage DEX dans `k_sol_dex_decoded_events` ; +5. détection métier vers tokens, pools, paires, listings, origins et wallets observés ; +6. matérialisation des trades exploitables ; +7. agrégation pair metrics ; +8. génération candles/OHLCV ; +9. signaux analytiques simples ; +10. inspection via l’application de démonstration. + +### 3.3. Connecteurs validés manuellement via l’application de démo -`kb_demo_app` est l’application Demo Tauri V2 avec frontend TypeScript. +Les connecteurs suivants ont déjà été testés via l’application de démonstration et doivent être verrouillés par corpus/replay avant d’ajouter de nouveaux DEX : -Son rôle est de : +- `pump_fun` ; +- `pump_swap` ; +- `raydium_cpmm` ; +- `raydium_clmm`. -- afficher l’interface utilisateur, -- exposer les commandes Tauri, -- déléguer la logique à `kb_lib`, -- afficher les états, logs et événements remontés par la bibliothèque. +### 3.4. Connecteurs déjà présents mais à consolider par corpus + +Les modules suivants existent ou sont partiellement représentés dans le code, mais doivent être consolidés par corpus local, invariants et documentation : -Le crate expose une bibliothèque interne `kb_demo_app_lib` afin de limiter le couplage avec `main.rs` et de préparer une évolution mobile ultérieure. +- `meteora_dbc` ; +- `meteora_damm_v1` ; +- `meteora_damm_v2` ; +- `orca_whirlpools` ; +- `fluxbeam` ; +- `dexlab` ; +- `raydium_amm_v4` legacy ; +- launch origins déjà amorcées : `meteora_fun_launch`, `bags`, `moonit`. -## 3. Contraintes techniques +## 4. Matrice DEX et launch surfaces -Le projet suit des contraintes strictes. +La distinction importante est la suivante : -### Contraintes de style et de structure +- un **DEX effectif** permet de détecter et décoder des swaps, pools, liquidité et candles ; +- une **launch surface** peut être la première source de mint ou de lancement, même si le token migre ensuite vers Raydium, Meteora ou un autre AMM ; +- pour le trading, la première source de mint est une information de filtrage et de ciblage aussi importante que le DEX final. -- Rust 2024. -- Pas de `mod.rs`. -- Les fichiers Rust commencent par une entête `// file: ...`. -- Les fichiers `lib.rs` et `main.rs` activent `#![deny(unreachable_pub)]` et `#![warn(missing_docs)]`. -- Les éléments publics sont documentés. -- Les expositions publiques passent par la racine des crates. - -### Contraintes de code - -- pas de `anyhow`, -- pas de `thiserror`, -- pas de `?` dans le code applicatif, -- pas de `unwrap` ni `expect` dans le code applicatif, -- utilisation préférée de `match`, `if let Err`, `let Err = ... else`. - -Les tests unitaires peuvent utiliser `?`, `unwrap` et `expect` si nécessaire. - -### Contraintes d’import - -- pas de `use` sur les types/fonctions des crates externes, -- seuls les imports de traits sont tolérés, -- les appels doivent utiliser les chemins complets, par exemple `std::string::String` ou `tokio::sync::mpsc::Sender`. - -## 4. Configuration - -L’application utilisera un fichier `config.json`. - -La configuration devra à terme permettre de définir : - -- les endpoints HTTP, -- les endpoints WebSocket, -- leur nom logique, -- les rôles affectés à chaque endpoint, -- les limitations de débit, -- les délais et timeouts, -- les répertoires locaux, -- la base SQLite, -- le futur répertoire des wallets Solana, -- les paramètres de tracing, -- la politique de reconnexion. - -Chaque endpoint devra pouvoir être identifié et affecté à une tâche spécifique, par exemple : - -- réception des notifications de slots, -- réception des program subscriptions, -- réception des logs, -- exécution des requêtes HTTP, -- endpoint de secours. - -Cela permet de répartir la charge ou d’adapter le provider selon son niveau de service, ses limitations ou l’usage d’une API key. - -## 5. Tracing et logs - -Le tracing sera centralisé dans `kb_lib`. - -Le système devra supporter : - -- sortie console, -- sortie fichier, -- niveau configurable, -- format de message configurable, -- format temporel configurable, -- ANSI console activable/désactivable, -- comportement spécifique pour les tests. - -## 6. Clients réseau - -## 6.1. `WsClient` - -`kb_lib` devra contenir un `WsClient` asynchrone basé sur `tokio-tungstenite`. - -Exigences initiales : - -- client duplicable, -- connexion à plusieurs serveurs WS RPC, -- identifiants de requêtes incrémentaux par instance, -- flux de lecture et flux d’écriture séparés, -- séparation des réponses RPC et des notifications, -- registre `subscribe` / `unsubscribe`, -- tentative d’unsubscribe avant fermeture, -- timeout pour ne pas bloquer le disconnect. - -Le client ne devra pas s’appuyer sur `solana-pubsub-client`, même si son comportement fonctionnel peut s’en inspirer. - -## 6.2. `HttpClient` - -`kb_lib` devra contenir un `HttpClient` asynchrone basé sur `reqwest`. - -Exigences initiales : - -- client duplicable, -- connexion à plusieurs endpoints HTTP RPC, -- limites de requêtes configurables, -- profils par endpoint, -- adaptation à des providers publics ou privés. - -Le client ne devra pas s’appuyer sur le `RpcClient` officiel de `solana-client`. - -## 6.3. `GrpcClient` - -Le support `GrpcClient` basé sur `yellowstone-grpc-client` et `yellowstone-grpc-proto` est prévu dans une phase ultérieure. - -## 7. Types et RPC - -Le projet cherchera à réutiliser autant que possible les types officiels fournis par l’écosystème Solana, notamment pour les payloads et surtout pour le parsing des réponses. - -Objectif : - -- limiter l’invention de structures approximatives, -- réutiliser les types des crates officielles lorsque cela est pertinent, -- encapsuler si besoin ces types dans une couche propre au projet. - -Le projet pourra embarquer son propre générateur de requêtes JSON-RPC 2.0 afin d’encapsuler proprement les appels HTTP et WS. - -## 8. Base de données - -Le stockage initial se fera dans SQLite. - -Cette première étape doit permettre : - -- de conserver l’historique observé, -- de stocker des événements et états techniques, -- de préparer l’analyse. - -Une migration vers PostgreSQL pourra être envisagée plus tard lorsque l’application aura stabilisé ses besoins. - -## 9. Frontend - -L’application Tauri démarrera avec une interface volontairement simple. - -### UI minimale prévue - -- un bouton ou toggle de connexion, -- un bouton d’arrêt si nécessaire, -- une zone de texte scrollable et en lecture seule, -- affichage des messages reçus depuis `kb_lib`. - -Cette première UI servira surtout à valider : - -- l’initialisation, -- le tracing, -- la délégation vers `kb_lib`, -- la remontée d’événements depuis les clients réseau. - -## 10. Constantes Solana - -Le projet devra réutiliser autant que possible les Program IDs et identifiants officiels fournis par les crates officielles au lieu de les recopier à la main. - -Exemples : - -- SPL Token -- SPL Token-2022 -- Associated Token Account -- Wrapped SOL mint -- System Program -- Compute Budget Program - -## 11. Génération de bindings TypeScript - -Les structures partagées entre Rust et le frontend devront être générées avec `ts-rs`. - -Le flux prévu est : - -```bash -cargo test export_bindings +### 4.1. Matrice de travail + +| Code cible | Type | Statut actuel | Prochaine action | +|---|---:|---|---| +| `pump_fun` | Launch + bonding curve | testé via démo | verrouiller corpus, invariants et documentation | +| `pump_swap` | AMM / swap | testé via démo | verrouiller corpus, invariants et candles | +| `raydium_cpmm` | AMM | testé via démo | verrouiller corpus, swaps et candles | +| `raydium_clmm` | CLMM | testé via démo | verrouiller corpus, swaps et candles | +| `raydium_launchlab` / `raydium_launchpad` | Launch surface + migration | manquant | ajouter comme origine de mint/lancement et migration vers Raydium | +| `raydium_amm_v4` | AMM legacy | présent, à isoler | traiter après les autres Raydium avec corpus dédié | +| `meteora_dbc` | Launch / bonding curve | présent, à consolider | corpus, lifecycle, migration et swaps exploitables | +| `meteora_damm_v1` | AMM legacy | présent, à consolider | corpus et séparation swaps/liquidité/events | +| `meteora_damm_v2` | AMM | présent, à consolider | corpus et séparation swaps/liquidité/events | +| `meteora_dlmm` | DLMM | manquant | ajouter à la matrice, puis corpus avant décodage | +| `orca_whirlpools` | CLMM | présent, à consolider | corpus fiable et validation des instructions utiles | +| `fluxbeam` | DEX | présent, à consolider | corpus fiable avant validation | +| `dexlab` | DEX | présent, à consolider | corpus fiable avant validation | +| `bags` | Launch surface / attribution | amorcé | conserver comme origine de lancement, relier à Meteora si prouvé | +| `letsbonk` / `bonk_fun` | Launch surface | manquant | ajouter comme origine LaunchLab/Raydium, pas comme AMM autonome tant que non prouvé | +| `boop_fun` | Launch surface | manquant | ajouter comme origine de mint/lancement et migration | +| `moonshot` / `moonit` | Launch surface | amorcé partiellement | remplacer les heuristiques faibles par corpus et règles prouvées | +| `believe` | Launch surface | manquant | ajouter comme origine associée à Meteora DBC si les comptes l’attestent | +| `heaven` | Launch + AMM candidat | manquant | ajouter corpus et déterminer séparation launch/swap | +| `zora_solana` | À vérifier | écarté maintenant | ne pas intégrer avant preuve de programme Solana pertinent | + +## 5. Base de données + +SQLite reste le stockage local initial. + +Organisation actuelle à conserver : + +- `kb_lib/src/db/schema.rs` crée les tables et index ; +- chaque table/index est créée dans une fonction dédiée ; +- les requêtes sont sous `kb_lib/src/db/queries/` ; +- les entités persistées sont sous `kb_lib/src/db/entities/` ; +- les DTO applicatifs sont sous `kb_lib/src/db/dtos/`. + +`schema.rs` n’est donc pas un fichier métier à splitter immédiatement. Il reste acceptable tant qu’il garde uniquement la responsabilité de création de schéma. + +### 5.1. Tables existantes importantes + +Le modèle actuel contient déjà notamment : + +- transactions et instructions Solana normalisées ; +- DEX connus ; +- événements DEX décodés ; +- tokens, pools, pool tokens, paires, listings ; +- launch surfaces et attributions ; +- pool origins ; +- swaps et trade events ; +- liquidity events ; +- wallets, participations, holdings ; +- candles ; +- metrics et analytic signals ; +- diagnostics locaux. + +### 5.2. Tables futures prioritaires + +Avant d’étendre trop agressivement les DEX, le modèle doit prévoir les cas non directement tradables : + +| Table cible | Rôle | +|---|---| +| `k_sol_transaction_classifications` | classifier les transactions connues, inconnues, partielles, échouées, non-DEX, DEX-candidates, launch-candidates. | +| `k_sol_protocol_candidates` | conserver les programmes ou patterns suspects/récurrents qui ne correspondent pas encore à un DEX connu. | +| `k_sol_pool_lifecycle_events` | matérialiser initialize/create/migrate/open/close/status events. | +| `k_sol_fee_events` | conserver fees, creator fees, protocol fees, fund fees. | +| `k_sol_reward_events` | conserver reward params, init rewards, collect rewards. | +| `k_sol_pool_admin_events` | conserver changements de config, authority, pause/resume, paramètres de pool. | + +`k_sol_liquidity_events` existe déjà et doit être stabilisée/étendue plutôt que recréée sans nécessité. + +## 6. Politique de refactor actuelle + +Le code et la documentation sont vivants. Les refactors agressifs sont acceptables lorsque cela rend le pipeline plus propre et plus durable, à condition de respecter ces limites : + +- ne pas casser les fonctionnalités déjà validées ; +- ne pas toucher pour le moment aux clients et managers réseau stabilisés ; +- faire des étapes courtes, testables et rejouables ; +- conserver les invariants de replay local ; +- ne pas transformer un événement non price-action en trade/candle ; +- documenter les nouveaux types publics avec une rustdoc utile mais pas surchargée ; +- laisser `local_pipeline_diagnostics` servir d’outil temporaire de validation tant que les DEX ne sont pas stabilisés. + +Les fichiers à surveiller en priorité sont : + +| Fichier | Action recommandée | +|---|---| +| `kb_lib/src/dex_decode.rs` | extraire classification, catégories d’événements et enrichissement commun. | +| `kb_lib/src/dex_detect.rs` | extraire helpers communs pool/pair/listing/origin/wallets et isoler les handlers par famille. | +| `kb_lib/src/trade_aggregation.rs` | isoler extraction de montants, normalisation trade et pricing. | +| `kb_lib/src/dex/*.rs` | homogénéiser les contrats de décodeurs sans forcer un gros trait prématuré. | + +## 7. Contraintes de code + +Contraintes maintenues : + +- Rust 2024 ; +- pas de `mod.rs` ; +- fichiers Rust avec entête `// file: ...` ; +- fichiers `.toml` avec entête `# file: ...` ; +- exposition centralisée via `lib.rs` ; +- `#![deny(unreachable_pub)]` et `#![warn(missing_docs)]` dans les racines concernées ; +- pas de `anyhow` ; +- pas de `thiserror` ; +- pas de `?`, `unwrap`, `expect` dans le code applicatif ; +- usage privilégié de `match`, `if let Err`, `let Err = ... else` ; +- imports externes limités, sauf traits lorsque nécessaire ; +- tests unitaires et tests de replay maintenus. + +Les tests peuvent rester plus souples lorsque cela clarifie le test. + +## 8. Priorité immédiate + +La reprise doit suivre cet ordre : + +1. finir/verrouiller `0.7.27` sur `pump_fun`, `pump_swap`, `raydium_cpmm`, `raydium_clmm` ; +2. démarrer `0.7.28` par un refactor commun DEX sans toucher au transport ; +3. ajouter la matrice DEX documentée et les corpus de validation ; +4. ajouter les tables de classification des transactions inconnues et protocol candidates ; +5. matérialiser les événements non-trade : lifecycle, liquidité, fees, rewards, admin ; +6. consolider Meteora, y compris `meteora_dlmm` dans la matrice ; +7. ajouter les launch surfaces manquantes comme origines de mint : LaunchLab/Launchpad, LetsBonk/Bonk.fun, Boop.fun, Moonshot/Moonit, Believe ; +8. traiter Heaven ; +9. consolider Orca/FluxBeam/DexLab ; +10. isoler Raydium AMM v4 legacy ; +11. effectuer une validation DEX v1 consolidée ; +12. reprendre ensuite l’UI analytique et les vues token/pair/pool. + +## 9. Fichiers utiles pour reprendre dans une nouvelle session + +Pour reprendre rapidement le codage dans une nouvelle session, fournir au minimum : + +- `README.md` ; +- `ROADMAP.md` ; +- `CHANGELOG.md` ; +- `Cargo.toml` racine ; +- `kb_lib/Cargo.toml` ; +- `kb_lib/src/lib.rs` ; +- `kb_lib/src/constants.rs` ; +- `kb_lib/src/dex.rs` ; +- `kb_lib/src/dex/*.rs` ; +- `kb_lib/src/dex_decode.rs` ; +- `kb_lib/src/dex_detect.rs` ; +- `kb_lib/src/trade_aggregation.rs` ; +- `kb_lib/src/pair_candle_aggregation.rs` ; +- `kb_lib/src/local_pipeline_replay.rs` ; +- `kb_lib/src/local_pipeline_validation.rs` ; +- `kb_lib/src/local_pipeline_diagnostics.rs` ; +- `kb_lib/src/db/schema.rs` ; +- `kb_lib/src/db.rs` ; +- `kb_lib/src/db/entities.rs` et `kb_lib/src/db/entities/*` ; +- `kb_lib/src/db/dtos.rs` et `kb_lib/src/db/dtos/*` ; +- `kb_lib/src/db/queries.rs` et `kb_lib/src/db/queries/*`. + +Ajouter `kb_demo_app/src/demo_pipeline*.rs` seulement si la tâche concerne l’UI ou les diagnostics affichés. + +## 10. Prompt court de reprise + +```text +Je reprends le workspace Rust khadhroony-bobobot autour de la version 0.7.27. +Objectif actuel : finaliser le pipeline DEX Solana avant trading. +Ne pas toucher pour le moment à ws_client/ws_manager/http_client/http_pool : ils fonctionnent et sont non bloquants. +Priorité : refactor DEX commun à partir de 0.7.28, matrice DEX, transactions inconnues/protocol candidates, événements non-trade, puis ajout/consolidation des DEX et launch surfaces. +Respecter les contraintes : Rust 2024, pas de mod.rs, pas de anyhow/thiserror, pas de ?/unwrap/expect dans le code applicatif, rustdoc utile sur l’API publique. +Les connecteurs à verrouiller avant extension sont pump_fun, pump_swap, raydium_cpmm et raydium_clmm. +Les launch surfaces sont importantes comme première source de mint, même si le token migre ensuite vers Raydium/Meteora/autre. ``` - -Exemple déjà présent : - -- `kb_demo_app/src/splash.rs` -- `kb_demo_app/frontend/ts/bindings/SplashOrder.ts` - -## 12. État du projet - -voir `CHANGELOG.md` - -## 13. Feuille de route - -La feuille de route détaillée est disponible dans : - -- `ROADMAP.md` - -## 14. Licence - -Licence MIT. diff --git a/ROADMAP.md b/ROADMAP.md index 4b87cf2..008a748 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -749,9 +749,7 @@ Réalisé : - seuls les trade candidates issus de transactions échouées restent ignorés. ### 6.059. Version `0.7.27` — Validation multi-DEX des connecteurs déjà branchés -Objectif : verrouiller la non-régression du pipeline actuel avant d’ajouter de nouveaux DEX ou d’ouvrir la phase d’analyse `0.8.x`. - -À faire : +Réalisé : - rejouer des bases neuves de test pour `pump_fun`, `pump_swap`, `raydium_cpmm` et `raydium_clmm`, - ne pas ajouter de nouveau DEX dans cette version ; cette version sert uniquement à valider les connecteurs déjà branchés, @@ -765,60 +763,147 @@ Objectif : verrouiller la non-régression du pipeline actuel avant d’ajouter d - conserver la tolérance aux événements DEX partiels tout en refusant les trades sans montant ou prix exploitable, - valider que les transactions échouées restent traçables dans les événements décodés sans produire de `k_sol_trade_events`. -### 6.060. Version `0.7.28` — Matérialisation des événements liquidité et cycle de vie pool -Objectif : exploiter les événements non buy/sell utiles à l’analyse et au trading semi-automatique sans les mélanger avec les trades/candles. +### 6.060. Version `0.7.28` — Refactor DEX commun et préparation extension +Réalisé : + +- ne pas toucher à `ws_client.rs`, `ws_manager.rs`, `http_client.rs`, `http_pool.rs` ni aux couches JSON-RPC déjà stabilisées, +- extraire depuis `dex_decode.rs` les catégories communes d’événements : trade, candle candidate, liquidity candidate, fee candidate, reward candidate, admin candidate, pool lifecycle candidate, +- créer une représentation interne documentée pour les familles d’événements DEX afin d’éviter les chaînes dispersées dans plusieurs fichiers, +- clarifier la différence entre événement décodé, événement actionnable, trade candidate, candle candidate et événement conservé seulement pour analyse, +- simplifier `dex_decode.rs` en gardant son rôle de service de persistance-orchestration, +- simplifier `dex_detect.rs` en extrayant les helpers communs pool/pair/listing/origin/wallet quand cela réduit la duplication, +- homogénéiser les contrats des modules `kb_lib/src/dex/*.rs` sans imposer trop tôt un trait générique lourd, +- vérifier la rustdoc publique : utile, courte, orientée responsabilité ; supprimer la documentation redondante ou trop chargée, +- conserver les tests verts et ajouter des tests de non-régression sur les catégories d’événements existantes. + +Contraintes : + +- refactor agressif autorisé si le résultat est plus propre, +- chaque changement doit rester rejouable et testable, +- aucun changement de comportement métier volontaire dans cette version, +- aucun événement non price-action ne doit devenir un trade ou une candle. + +### 6.061. Version `0.7.29` — Matrice DEX, corpus et documentation de support +Objectif : rendre explicite ce qui est validé, présent, partiel, manquant ou volontairement ignoré. À faire : -- stabiliser ou ajouter la table `k_sol_liquidity_events`, -- stabiliser ou ajouter la table `k_sol_pool_lifecycle_events`, -- matérialiser les événements de type `increase_liquidity`, `decrease_liquidity`, `add_liquidity`, `remove_liquidity`, `open_position`, `close_position`, `initialize`, `create_pool`, `migrate` et assimilés, -- rattacher chaque événement métier à `dex_id`, `pool_id`, `pair_id`, `transaction_id`, `decoded_event_id`, `signature` et `slot`, -- conserver le `payload_json` source pour audit et extension future, -- alimenter les diagnostics locaux avec les compteurs liquidité et cycle de vie, -- garantir qu’un événement de liquidité ou de cycle de vie ne produit jamais de candle directement, -- préparer les signaux futurs liés à la liquidité initiale, à l’ajout/retrait brutal de liquidité, aux migrations et aux changements de statut de pool. +- ajouter et maintenir une matrice DEX dans `README.md` et `ROADMAP.md`, +- distinguer clairement `DEX effectif`, `launch surface`, `pool origin`, `launch origin` et `migration target`, +- documenter que les launch surfaces sont importantes comme première source de mint/lancement même lorsqu’un token migre ensuite vers un autre DEX, +- constituer une liste de corpus par DEX/surface : signatures, pools, token mints, résultat attendu, +- indiquer pour chaque protocole : statut, source de preuve, type d’événements couverts, tables alimentées, limites connues, +- retirer `zora_solana` du phasage actif tant qu’aucun programme Solana pertinent n’est prouvé, +- ajouter `meteora_dlmm` à la matrice comme variante Meteora manquante à couvrir plus tard, +- préparer les diagnostics SQL de référence par protocole et par table métier. -### 6.061. Version `0.7.29` — Matérialisation des événements fees, rewards et administration -Objectif : conserver les événements non price-action utiles au risque, à l’analyse économique et à la traçabilité opérationnelle des pools. +Matrice cible initiale : + +| Code cible | Type | Statut | Objectif immédiat | +|---|---:|---|---| +| `pump_fun` | launch + bonding curve | testé via démo | verrouiller corpus et invariants | +| `pump_swap` | AMM / swap | testé via démo | verrouiller trades/candles | +| `raydium_cpmm` | AMM | testé via démo | verrouiller trades/candles | +| `raydium_clmm` | CLMM | testé via démo | verrouiller trades/candles | +| `raydium_launchlab` / `raydium_launchpad` | launch surface | manquant | détecter mint, launch, migration | +| `raydium_amm_v4` | AMM legacy | présent, à isoler | corpus dédié après autres DEX | +| `meteora_dbc` | launch / bonding curve | présent, à consolider | lifecycle, migration, swaps utiles | +| `meteora_damm_v1` | AMM legacy | présent, à consolider | corpus et séparation events | +| `meteora_damm_v2` | AMM | présent, à consolider | corpus et séparation events | +| `meteora_dlmm` | DLMM | manquant | ajouter corpus avant décodeur | +| `orca_whirlpools` | CLMM | présent, à consolider | validation par corpus | +| `fluxbeam` | DEX | présent, à consolider | validation par corpus | +| `dexlab` | DEX | présent, à consolider | validation par corpus | +| `bags` | launch surface | amorcé | attribution fiable, migration si prouvée | +| `letsbonk` / `bonk_fun` | launch surface | manquant | origine LaunchLab/Raydium | +| `boop_fun` | launch surface | manquant | origine mint/lancement/migration | +| `moonshot` / `moonit` | launch surface | amorcé partiellement | corpus, éviter heuristique faible | +| `believe` | launch surface | manquant | origine Meteora DBC si comptes probants | +| `heaven` | launch + AMM candidat | manquant | corpus et séparation launch/swap | + +### 6.062. Version `0.7.30` — Transactions inconnues et protocol candidates +Objectif : ne plus perdre les transactions utiles qui ne correspondent pas encore à un DEX connu. À faire : -- ajouter ou stabiliser `k_sol_fee_events`, -- ajouter ou stabiliser `k_sol_reward_events`, -- ajouter ou stabiliser `k_sol_pool_admin_events`, +- ajouter `k_sol_transaction_classifications`, +- ajouter `k_sol_protocol_candidates`, +- classifier les transactions résolues en catégories : known dex, known launch surface, unknown program, non-dex, failed transaction, partial decode, ignored technical transaction, +- conserver les `program_id`, comptes, signatures, préfixes de `data`, logs et indices d’instructions utiles à l’analyse, +- créer des requêtes de diagnostic pour repérer les programmes inconnus fréquents, +- permettre de promouvoir plus tard un protocol candidate vers un vrai DEX/surface sans perdre l’historique, +- garantir que ces tables n’alimentent jamais directement les trades/candles. + +### 6.063. Version `0.7.31` — Événements non-trade v1 : liquidité et cycle de vie pool +Objectif : exploiter les événements utiles à l’analyse et au trading semi-automatique sans les mélanger avec les swaps/candles. + +À faire : + +- stabiliser et étendre `k_sol_liquidity_events` au lieu de la recréer inutilement, +- ajouter `k_sol_pool_lifecycle_events`, +- matérialiser les événements `initialize`, `create_pool`, `migrate`, `open_position`, `close_position`, `increase_liquidity`, `decrease_liquidity`, `add_liquidity`, `remove_liquidity` et assimilés, +- rattacher chaque événement à `dex_id`, `pool_id`, `pair_id`, `transaction_id`, `decoded_event_id`, `signature` et `slot` lorsque les informations existent, +- conserver le `payload_json` source pour audit, +- alimenter les diagnostics locaux avec les compteurs liquidité/lifecycle, +- garantir qu’un événement de liquidité ou de cycle de vie ne produit jamais de candle directement. + +### 6.064. Version `0.7.32` — Événements non-trade v2 : fees, rewards et administration +Objectif : conserver les événements utiles au risque, au scoring, à l’économie du pool et à la traçabilité opérationnelle. + +À faire : + +- ajouter `k_sol_fee_events`, +- ajouter `k_sol_reward_events`, +- ajouter `k_sol_pool_admin_events`, - matérialiser les événements `collect_protocol_fee`, `collect_fund_fee`, `collect_creator_fee`, `collect_fee` et assimilés, - matérialiser les événements `set_reward_params`, `initialize_reward`, `collect_reward`, `update_reward_infos` et assimilés, - matérialiser les événements `set_config`, `update_config`, `set_authority`, `set_fee_rate`, `pause`, `resume` et assimilés, -- rattacher chaque événement à la transaction, au decoded event, au pool, à la paire et aux wallets observés lorsque les comptes sont disponibles, -- documenter clairement que ces événements sont utiles à l’analyse et au scoring mais ne sont ni des trades ni des candles. +- rattacher ces événements aux transactions, decoded events, pools, paires et wallets observés lorsque les comptes le permettent, +- documenter clairement que ces événements ne sont ni des trades ni des candles. -### 6.062. Version `0.7.30` — Meteora : DBC / DAMM v1 / DAMM v2 -Objectif : ajouter ou stabiliser les connecteurs Meteora avec une séparation claire entre swaps, liquidité, fees, rewards et événements pool. +### 6.065. Version `0.7.33` — Meteora : DBC / DAMM v1 / DAMM v2 / DLMM +Objectif : consolider Meteora comme famille multi-programmes au lieu de traiter chaque variante comme un cas isolé incomplet. À faire : -- vérifier les programmes et discriminants réellement utilisés pour `Meteora DBC`, `Meteora DAMM v1` et `Meteora DAMM v2`, -- constituer un petit corpus local de pools et signatures fiables pour chaque variante, -- décoder les créations de pool, swaps et événements de liquidité exploitables, -- alimenter `k_sol_dex_decoded_events`, les tables métier pool/pair/listing et les nouvelles tables d’événements non-trade, -- vérifier l’idempotence du replay local sur un corpus mixte Meteora, -- documenter les limites connues des variantes insuffisamment couvertes par le corpus. +- vérifier les programmes et discriminants réellement utilisés pour `Meteora DBC`, `Meteora DAMM v1`, `Meteora DAMM v2` et `Meteora DLMM`, +- ajouter `meteora_dlmm` à la couverture cible seulement après corpus fiable, +- constituer un corpus local par variante, +- décoder les créations de pool, swaps, liquidités et événements lifecycle exploitables, +- identifier les cas où `DBC` sert de launch origin avant migration vers un AMM, +- alimenter `k_sol_dex_decoded_events`, les tables pool/pair/listing, les origins et les tables non-trade, +- vérifier l’idempotence du replay local sur un corpus Meteora mixte, +- documenter les limites connues des variantes insuffisamment couvertes. -### 6.063. Version `0.7.31` — Launch DEX : LaunchLab / Fun Launch / Bags / Moonit -Objectif : couvrir les surfaces de lancement et de migration de tokens sans confondre origine de lancement et protocole DEX final. +### 6.066. Version `0.7.34` — Launch surfaces : LaunchLab, LetsBonk, Bags, Moonshot/Moonit, Boop.fun, Believe +Objectif : détecter la première source de mint/lancement des tokens même lorsque le swap final se fait ailleurs. À faire : -- ajouter ou stabiliser les mappings `LaunchLab`, `Fun Launch`, `Bags` et `Moonit`, -- distinguer clairement launch origin, pool origin et DEX effectif, -- détecter les créations de token, pools initiaux, migrations et listings dérivés, +- ajouter ou stabiliser `raydium_launchlab` / `raydium_launchpad`, +- ajouter `letsbonk` / `bonk_fun` comme surface d’origine rattachée à LaunchLab/Raydium si le corpus le prouve, +- ajouter `boop_fun` comme surface d’origine et suivre ses migrations, +- consolider `moonshot` / `moonit` avec corpus au lieu de simples heuristiques faibles, +- consolider `bags` comme surface d’origine, notamment lorsque le token passe par Meteora DBC/DAMM, +- ajouter `believe` comme surface d’origine associée à Meteora DBC si les comptes/authorities le prouvent, +- distinguer `launch_origin`, `pool_origin`, `dex_effective` et `migration_target`, - rattacher les launch origins aux pools et paires lorsque les comptes permettent un matching fiable, -- enrichir les diagnostics pour exposer les objets détectés par surface de lancement, -- éviter les heuristiques trop larges lorsqu’un suffixe de mint ou un label externe ne suffit pas à prouver l’origine. +- exposer les origins dans les diagnostics et l’UI d’inspection. -### 6.064. Version `0.7.32` — Orca / FluxBeam / DexLab : corpus et validation ciblée -Objectif : ajouter les connecteurs restants à partir de corpus locaux vérifiables et garder les décodeurs heuristiques isolés tant qu’ils ne sont pas prouvés. +### 6.067. Version `0.7.35` — Heaven : corpus, launch et AMM +Objectif : ajouter Heaven sans le classer trop tôt comme simple DEX ou simple launchpad. + +À faire : + +- vérifier le ou les programmes Heaven réellement observés sur mainnet, +- constituer un corpus local de mints, pools et swaps Heaven, +- séparer les événements de lancement des événements de swap, +- ajouter les decoded events, launch origins, pool/pair/listing et trade events seulement lorsque les instructions sont prouvées, +- documenter les limites si le corpus ne permet pas encore de matérialiser tous les événements, +- vérifier que Heaven ne crée pas de candles invalides en cas d’événement de launch non pricé. + +### 6.068. Version `0.7.36` — Orca / FluxBeam / DexLab : corpus et validation ciblée +Objectif : consolider les connecteurs déjà présents à partir de corpus locaux vérifiables. À faire : @@ -829,32 +914,33 @@ Objectif : ajouter les connecteurs restants à partir de corpus locaux vérifiab - marquer explicitement les variantes partiellement supportées ou heuristiques, - rejouer les corpus plusieurs fois pour vérifier l’idempotence et l’absence de trades/candles invalides. -### 6.065. Version `0.7.33` — Validation DEX v1 consolidée -Objectif : rejouer tous les DEX supportés et valider les invariants du pipeline complet avant de traiter Raydium AMM v4 legacy séparément. - -À faire : - -- rejouer des bases neuves couvrant tous les connecteurs DEX supportés hors `raydium_amm_v4` legacy, -- vérifier les compteurs globaux et par DEX : decoded events, trade events, liquidity events, lifecycle events, fee events, reward events, admin events, candles et analytic signals, -- contrôler que chaque famille d’événements alimente uniquement les tables métier prévues, -- vérifier les diagnostics bloquants et les samples d’anomalie, -- documenter les corpus utilisés pour chaque DEX, -- conserver une matrice de support par DEX, variante, instruction et type d’événement. - -### 6.066. Version `0.7.34` — Raydium AMM v4 legacy : corpus et validation ciblée -Objectif : traiter le vrai Raydium AMM v4 historique après les autres DEX, afin de l’isoler correctement de `raydium_cpmm`, `raydium_clmm` et des labels Raydium génériques. +### 6.069. Version `0.7.37` — Raydium AMM v4 legacy : corpus et validation ciblée +Objectif : traiter le vrai Raydium AMM v4 historique après les autres Raydium, afin de l’isoler de `raydium_cpmm`, `raydium_clmm` et des labels Raydium génériques. À faire : - rechercher des pools réellement rattachés au programme `675kPX9MHTjS2zt1qfr1NYHuzeLXfQM9H24wFSUt1Mp8`, -- constituer un petit corpus local de signatures/pools AMM v4 fiables pour les tests, -- vérifier que les adresses issues de Dexscreener ou d’autres explorateurs ne sont pas seulement catégorisées globalement comme `Raydium`, +- constituer un petit corpus local de signatures/pools AMM v4 fiables, +- vérifier que les adresses issues d’explorateurs ne sont pas seulement catégorisées globalement comme `Raydium`, - ajouter des requêtes de diagnostic par `program_id`, `accounts_json` et préfixe `data_json`, -- valider la prise en charge `initialize2` et identifier les instructions de swap AMM v4 à supporter si elles apparaissent dans les transactions testées, -- renommer et stabiliser les fonctions internes autour de `raydium_amm_v4` afin d’éviter l’ambiguïté avec `raydium_cpmm` et `raydium_clmm`, -- documenter les limites connues si le corpus AMM v4 reste trop faible. +- valider `initialize2` et identifier les instructions de swap AMM v4 à supporter si elles apparaissent dans le corpus, +- renommer/stabiliser les fonctions internes autour de `raydium_amm_v4` pour éviter l’ambiguïté avec `raydium_cpmm` et `raydium_clmm`, +- documenter les limites connues si le corpus AMM v4 reste faible. -### 6.067. Version `0.7.35` — `kb_demo_app` : overlays analytiques +### 6.070. Version `0.7.38` — Validation DEX v1 consolidée +Objectif : rejouer tous les DEX et launch surfaces supportés et valider les invariants du pipeline complet. + +À faire : + +- rejouer des bases neuves couvrant tous les connecteurs DEX supportés, +- vérifier les compteurs globaux et par DEX : decoded events, trade events, liquidity events, lifecycle events, fee events, reward events, admin events, candles et analytic signals, +- contrôler que chaque famille d’événements alimente uniquement les tables métier prévues, +- vérifier les diagnostics bloquants et les samples d’anomalie, +- documenter les corpus utilisés pour chaque DEX/surface, +- conserver une matrice de support par DEX, variante, instruction et type d’événement, +- verrouiller les invariants avant d’ouvrir l’analyse `0.8.x`. + +### 6.071. Version `0.7.39` — `kb_demo_app` : overlays analytiques Objectif : rendre visibles les signaux analytiques directement sur les graphes et vues de marché. À faire : @@ -863,9 +949,9 @@ Objectif : rendre visibles les signaux analytiques directement sur les graphes e - ajouter des marqueurs pour `first_trade_seen`, `trade_burst_60s`, `buy_sell_imbalance_60s`, `price_jump_up_60s`, `price_jump_down_60s` et `volume_spike_60s`, - permettre le filtrage par type de signal et par sévérité, - afficher un panneau latéral listant les signaux liés à une paire et à un timeframe, -- préparer l’extension future vers des indicateurs Ichimoku, Kumo, projections ABCD et égalités temps/prix sans les mélanger au pipeline de décodage DEX. +- préparer l’extension future vers Ichimoku, Kumo, projections ABCD et égalités temps/prix sans les mélanger au pipeline de décodage DEX. -### 6.068. Version `0.7.36` — `kb_demo_app` : vues consolidées token / pair / pool +### 6.072. Version `0.7.40` — `kb_demo_app` : vues consolidées token / pair / pool Objectif : fournir une lecture métier plus confortable du modèle `0.7.x`. À faire : @@ -873,11 +959,11 @@ Objectif : fournir une lecture métier plus confortable du modèle `0.7.x`. - ajouter une fiche token avec mint, programme token, metadata, pools, paires et historique de découverte, - ajouter une fiche paire avec base/quote, DEX, pool, métriques, candles, signaux et derniers trades, - ajouter une fiche pool avec composition, vaults, origine, première signature vue, programme DEX et statut de décodage, -- relier dans l’UI les launch origins, pool origins, wallets observés, holdings observés, événements de liquidité, événements lifecycle, fees, rewards, admin, candles et analytic signals, +- relier dans l’UI les launch origins, pool origins, wallets observés, holdings observés, événements de liquidité, lifecycle, fees, rewards, admin, candles et analytic signals, - préparer une navigation transversale entre objets techniques et objets métier, - rendre explicites les cas `tradeCount = null`, `lastPriceQuotePerBase = null`, tokens non enrichis et événements conservés uniquement pour analyse. -### 6.069. Version `0.7.37` — Finition UI `0.7.x` +### 6.073. Version `0.7.41` — Finition UI `0.7.x` Objectif : stabiliser la couche desktop de validation avant l’ouverture de `0.8.x`. À faire : @@ -886,41 +972,52 @@ Objectif : stabiliser la couche desktop de validation avant l’ouverture de `0. - améliorer la navigation, les filtres et la pagination, - ajouter les derniers raffinements de confort et de lisibilité, - préparer une base UI suffisamment stable pour la future phase d’analyse et filtrage `0.8.x`, -- vérifier que les commandes Tauri restent de simples façades vers `kb_lib` et ne récupèrent pas de logique métier. +- vérifier que les commandes Tauri restent de simples façades vers `kb_lib`. -### 6.070. Version `0.7.x` — Couverture DEX v1 -Objectif : structurer les connecteurs DEX autour d’un pipeline complet de résolution, décodage et normalisation métier. +### 6.074. Version `0.7.x` — Couverture DEX v1 +Objectif : structurer les connecteurs DEX autour d’un pipeline complet de résolution, décodage, normalisation métier et classification des événements non-trade. -Protocoles cibles : +Protocoles et surfaces cibles : -- Pump.fun -- PumpSwap -- Raydium CPMM -- Raydium CLMM -- Meteora DBC -- Meteora DAMM v2 -- Meteora DAMM v1 -- LaunchLab / Fun Launch -- Bags -- Moonit -- Orca -- FluxBeam -- DexLab -- Raydium AMM v4 legacy +- Pump.fun, +- PumpSwap, +- Raydium CPMM, +- Raydium CLMM, +- Raydium LaunchLab / Launchpad, +- Raydium AMM v4 legacy, +- Meteora DBC, +- Meteora DAMM v1, +- Meteora DAMM v2, +- Meteora DLMM, +- Orca Whirlpools, +- FluxBeam, +- DexLab, +- Bags, +- LetsBonk / Bonk.fun, +- Boop.fun, +- Moonshot / Moonit, +- Believe, +- Heaven. + +Hors périmètre immédiat : + +- `zora_solana`, tant qu’aucun programme Solana pertinent et exploitable n’est prouvé. Résultat attendu : - identification fiable des programmes et versions, - résolution des signatures pertinentes, - décodage des transactions utiles, -- création d’objets métier riches pour tokens, pools, paires, listings, participants et holdings observés, +- conservation des transactions inconnues ou candidates sans perte d’information, +- création d’objets métier riches pour tokens, pools, paires, listings, participants, origins et holdings observés, +- distinction claire entre première source de mint, launch origin, pool origin, DEX effectif et migration target, - enrichissement metadata des tokens découverts, -- séparation claire entre événements candle/trade et événements utiles seulement à l’analyse, aux frais, à la liquidité, aux rewards, à l’administration ou au cycle de vie des pools, +- séparation stricte entre événements candle/trade et événements utiles seulement à l’analyse, - matérialisation progressive des événements non-trade dans des tables métier dédiées, - préparation d’une détection temps réel hybride et d’un backfill ciblé compatible avec les mêmes objets métier, -- préparation d’agrégats DEX plus riches, de candles / OHLCV et d’une UI d’inspection du pipeline `0.7.x`. +- préparation d’agrégats DEX plus riches, de candles/OHLCV et d’une UI d’inspection du pipeline `0.7.x`. -### 6.071. Version `0.8.x` — Analyse et filtrage +### 6.075. Version `0.8.x` — Analyse et filtrage Objectif : transformer les événements bruts en signaux exploitables. À faire : @@ -935,7 +1032,7 @@ Objectif : transformer les événements bruts en signaux exploitables. - outils de sélection manuelle de points ABC et projection d’un point D selon des règles temps/prix explicites, - séparation stricte entre signaux analytiques observés, projections hypothétiques et décisions de trading. -### 6.072. Version `1.x.y` — Wallets et swap préparatoire +### 6.076. Version `1.x.y` — Wallets et swap préparatoire Objectif : préparer la couche d’action. À faire : @@ -946,7 +1043,7 @@ Objectif : préparer la couche d’action. - préparation d’ordres et de swaps, - simulation et garde-fous. -### 6.073. Version `2.x.y` — Trading semi-automatisé +### 6.077. Version `2.x.y` — Trading semi-automatisé Objectif : brancher l’analyse à l’action tout en gardant des garde-fous explicites. À faire : @@ -957,7 +1054,7 @@ Objectif : brancher l’analyse à l’action tout en gardant des garde-fous exp - confirmations explicites ou semi-automatiques, - journaux d’exécution. -### 6.074. Version `3.x.y` — Yellowstone gRPC +### 6.078. Version `3.x.y` — Yellowstone gRPC Objectif : ajouter le connecteur gRPC dédié. À faire : @@ -970,41 +1067,59 @@ Objectif : ajouter le connecteur gRPC dédié. ## 7. Organisation des modules ciblés ### 7.1. `kb_lib` -Modules cibles à court terme : +Modules stables à ne pas remanier dans la phase immédiate : -- `error.rs` -- `config.rs` -- `tracing.rs` -- `constants.rs` -- `types.rs` - `ws_client.rs` - `ws_manager.rs` - `http_client.rs` - `http_pool.rs` - `json_rpc_ws.rs` - `solana_pubsub_ws.rs` -- `detect.rs` + +Modules ciblés par le refactor et la consolidation DEX : + +- `dex.rs` +- `dex/*.rs` - `dex_decode.rs` - `dex_detect.rs` - `trade_aggregation.rs` - `pair_candle_aggregation.rs` - `pair_analytic_signal.rs` +- `launch_origin.rs` +- `pool_origin.rs` +- `wallet_observation.rs` +- `wallet_holding_observation.rs` - `token_metadata.rs` - `local_pipeline_replay.rs` +- `local_pipeline_validation.rs` - `local_pipeline_diagnostics.rs` -- `db/entities/*` -- `db/dtos/*` -- `db/queries/*` -### 7.2. `kb_demo_app` +`local_pipeline_diagnostics.rs` est volontairement conservé comme outil temporaire de validation. Il pourra devenir obsolète ou être remplacé lorsque les tests DEX seront stabilisés. Il n’est pas prioritaire de le refactorer maintenant. + +### 7.2. Base de données + +Organisation de la couche DB à conserver : + +- `db/schema.rs` : création des tables et index uniquement ; chaque table ou index reste dans une fonction dédiée, +- `db/entities/*` : entités proches des lignes persistées, +- `db/dtos/*` : DTOs applicatifs, +- `db/queries/*` : requêtes SQL regroupées par table ou usage, +- `db/queries/local_pipeline_diagnostics.rs` : requêtes de diagnostic local, utiles pendant la validation DEX. + +`schema.rs` peut rester long tant qu’il reste strictement un fichier de schéma. Le split prioritaire concerne plutôt les responsabilités métier dans `dex_decode.rs`, `dex_detect.rs` et `trade_aggregation.rs`. + +### 7.3. `kb_demo_app` Responsabilités cibles : - lancement Tauri, - commandes UI, - affichage des états et messages, - réception des événements venant de `kb_lib`, -- persistance future des préférences UI, -- fenêtres de démonstration / diagnostic isolées. +- fenêtres de démonstration / diagnostic isolées, +- inspection du pipeline persisté, +- affichage candles et futurs overlays analytiques. + +`kb_demo_app` ne doit pas contenir de logique métier DEX profonde. ## 8. Ligne de conduite sur le `WsClient` Le `WsClient` doit être conçu en plusieurs couches : @@ -1048,7 +1163,7 @@ Le projet doit maintenir au minimum : - un `README.md` global, - un `ROADMAP.md` global, - un `CHANGELOG.md` global, -- des `README.md` et `TODO.md` par crate à mesure de l’évolution, +- des `README.md` et `TODO.md` par crate à mesure de l’évolution (surtout en version 1.0), - des tests unitaires robustes, - les bindings TS générés via `cargo test export_bindings` lorsque les types partagés évoluent. @@ -1056,16 +1171,21 @@ Le projet doit maintenir au minimum : La priorité immédiate est désormais la suivante : -1. terminer la validation `0.7.27` sur les connecteurs déjà branchés : `pump_fun`, `pump_swap`, `raydium_cpmm` et `raydium_clmm`, sans ajouter de nouveau DEX dans cette étape, +1. terminer la validation `0.7.27` sur `pump_fun`, `pump_swap`, `raydium_cpmm` et `raydium_clmm`, sans ajouter de nouveau DEX dans cette étape, 2. vérifier sur bases neuves et après replay local les invariants bloquants du pipeline : `diagnosticsClean = true`, `blockingIssueCount = 0`, aucun trade candidate exploitable perdu, aucun trade event invalide, aucun doublon réel par `decoded_event_id`, aucune candle dupliquée par bucket, -3. documenter les requêtes SQL de diagnostic de référence et les résultats attendus pour les tables clés du pipeline, -4. matérialiser ensuite les événements non buy/sell utiles au trading et à l’analyse : liquidité, cycle de vie des pools, fees, rewards et administration, -5. garantir que ces événements non-trade restent séparés des `k_sol_trade_events` et des candles tout en restant rattachés aux transactions, decoded events, pools, pairs et wallets observés, -6. ajouter ou stabiliser les autres DEX par lots vérifiables : Meteora, surfaces de lancement, Orca, FluxBeam et DexLab, -7. traiter `raydium_amm_v4` legacy seulement après les autres DEX, avec un corpus dédié prouvant le programme `675kPX9MHTjS2zt1qfr1NYHuzeLXfQM9H24wFSUt1Mp8`, -8. effectuer une validation DEX v1 consolidée sur tous les connecteurs supportés avant de considérer la couche DEX `0.7.x` comme stable, -9. ajouter ensuite les overlays des signaux analytiques sur les candles, -10. consolider les vues métier `token / pair / pool` dans `kb_demo_app`, y compris les événements liquidité, lifecycle, fees, rewards et admin, -11. stabiliser l’ergonomie, les filtres, la pagination et la navigation de l’UI d’inspection, -12. préparer ensuite l’ouverture de `0.8.x` pour l’analyse, les filtres, les patterns et les projections graphiques, -13. préparer enfin Yellowstone gRPC comme extension de capacité, et non comme remplacement du socle HTTP / WS existant. +3. démarrer `0.7.28` par le refactor DEX commun, sans toucher aux clients HTTP/WS ni aux managers réseau stabilisés, +4. ajouter la matrice DEX et launch surfaces, avec statut, corpus, limites et prochaine action pour chaque protocole, +5. ajouter les tables de classification des transactions inconnues et des protocol candidates afin de ne plus perdre les transactions utiles non encore décodables, +6. matérialiser ensuite les événements non-trade : liquidité, cycle de vie des pools, fees, rewards et administration, +7. garantir que ces événements non-trade restent séparés des `k_sol_trade_events` et des candles tout en restant rattachés aux transactions, decoded events, pools, pairs et wallets observés, +8. consolider Meteora, y compris `meteora_dlmm`, après corpus fiable, +9. ajouter les launch surfaces manquantes comme premières sources de mint/lancement : Raydium LaunchLab/Launchpad, LetsBonk/Bonk.fun, Boop.fun, Moonshot/Moonit, Believe et Bags, +10. traiter Heaven avec séparation launch/swap, +11. consolider Orca, FluxBeam et DexLab sur corpus, +12. traiter `raydium_amm_v4` legacy seulement après les autres Raydium, avec corpus dédié prouvant le programme `675kPX9MHTjS2zt1qfr1NYHuzeLXfQM9H24wFSUt1Mp8`, +13. effectuer une validation DEX v1 consolidée sur tous les connecteurs supportés avant de considérer la couche DEX `0.7.x` comme stable, +14. ajouter ensuite les overlays des signaux analytiques sur les candles, +15. consolider les vues métier `token / pair / pool` dans `kb_demo_app`, y compris les événements liquidité, lifecycle, fees, rewards et admin, +16. stabiliser l’ergonomie, les filtres, la pagination et la navigation de l’UI d’inspection, +17. préparer ensuite l’ouverture de `0.8.x` pour l’analyse, les filtres, les patterns et les projections graphiques, +18. préparer enfin Yellowstone gRPC comme extension de capacité, et non comme remplacement du socle HTTP / WS existant. diff --git a/kb_demo_app/frontend/demo_pipeline2.html b/kb_demo_app/frontend/demo_pipeline2.html index 2f9d7ff..a8f3a01 100644 --- a/kb_demo_app/frontend/demo_pipeline2.html +++ b/kb_demo_app/frontend/demo_pipeline2.html @@ -29,11 +29,11 @@

-

-
+
+
+

+ +

+
+
+

+ Résume les programmes candidats par priorité : transaction_count, occurrence_count, dernier slot et dernière signature. +

+ +
+ + +
+ +
+ +
+
+
+
+

+
+

+ +

+
+
+ +
+
+
+

-

-
+
diff --git a/kb_demo_app/frontend/ts/bindings/DemoPipeline2ProtocolCandidateSummaryPayload.ts b/kb_demo_app/frontend/ts/bindings/DemoPipeline2ProtocolCandidateSummaryPayload.ts new file mode 100644 index 0000000..9a73c0c --- /dev/null +++ b/kb_demo_app/frontend/ts/bindings/DemoPipeline2ProtocolCandidateSummaryPayload.ts @@ -0,0 +1,10 @@ +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. + +/** + * Response payload for protocol candidate summary diagnostics. + */ +export type DemoPipeline2ProtocolCandidateSummaryPayload = { +/** + * Pretty JSON summary rows. + */ +summariesJson: string, }; diff --git a/kb_demo_app/frontend/ts/bindings/DemoPipeline2ProtocolCandidateSummaryRequest.ts b/kb_demo_app/frontend/ts/bindings/DemoPipeline2ProtocolCandidateSummaryRequest.ts new file mode 100644 index 0000000..2c1a802 --- /dev/null +++ b/kb_demo_app/frontend/ts/bindings/DemoPipeline2ProtocolCandidateSummaryRequest.ts @@ -0,0 +1,10 @@ +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. + +/** + * Request payload for protocol candidate summary diagnostics. + */ +export type DemoPipeline2ProtocolCandidateSummaryRequest = { +/** + * Maximum number of summary rows to return. + */ +limit: number, }; diff --git a/kb_demo_app/frontend/ts/demo_pipeline2.ts b/kb_demo_app/frontend/ts/demo_pipeline2.ts index a961997..9e5ab3c 100644 --- a/kb_demo_app/frontend/ts/demo_pipeline2.ts +++ b/kb_demo_app/frontend/ts/demo_pipeline2.ts @@ -15,6 +15,8 @@ import type { DemoPipeline2PairCandlesRequest } from "./bindings/DemoPipeline2Pa import type { DemoPipeline2PairCandlesPayload } from "./bindings/DemoPipeline2PairCandlesPayload.ts"; import type { DemoPipeline2LocalDiagnosticsPayload } from "./bindings/DemoPipeline2LocalDiagnosticsPayload.ts"; import type { DemoPipeline2LocalValidationPayload } from "./bindings/DemoPipeline2LocalValidationPayload.ts"; +import { DemoPipeline2ProtocolCandidateSummaryRequest } from './bindings/DemoPipeline2ProtocolCandidateSummaryRequest.ts'; +import { DemoPipeline2ProtocolCandidateSummaryPayload } from './bindings/DemoPipeline2ProtocolCandidateSummaryPayload.ts'; (window as Window & typeof globalThis & { bootstrap?: typeof bootstrap }).bootstrap = bootstrap; (window as Window & typeof globalThis & { ResizeObserver?: typeof ResizeObserver }).ResizeObserver = ResizeObserver; @@ -354,6 +356,9 @@ document.addEventListener("DOMContentLoaded", async () => { const diagnoseLocalPipelineButton = document.querySelector("#demoPipeline2DiagnoseLocalPipelineButton"); const validateLocalPipelineButton = document.querySelector("#demoPipeline2ValidateLocalPipelineButton"); + const protocolCandidateLimitInput = document.querySelector("#demoPipeline2ProtocolCandidateLimitInput"); + const refreshProtocolCandidatesButton = document.querySelector("#demoPipeline2RefreshProtocolCandidatesButton"); + const pairSelect = document.querySelector("#demoPipeline2PairSelect"); const timeframeSelect = document.querySelector("#demoPipeline2TimeframeSelect"); const customTimeframeInput = document.querySelector("#demoPipeline2CustomTimeframeInput"); @@ -366,6 +371,8 @@ document.addEventListener("DOMContentLoaded", async () => { const localDiagnosticsTextarea = document.querySelector("#demoPipeline2LocalDiagnosticsTextarea"); const localValidationTextarea = document.querySelector("#demoPipeline2LocalValidationTextarea"); + const protocolCandidateSummariesTextarea = document.querySelector("#demoPipeline2ProtocolCandidateSummariesTextarea"); + const clearLogButton = document.querySelector("#demoPipeline2ClearLogButton"); const logTextarea = document.querySelector("#demoPipeline2LogTextarea"); @@ -388,6 +395,8 @@ document.addEventListener("DOMContentLoaded", async () => { !replayLocalPipelineButton || !diagnoseLocalPipelineButton || !validateLocalPipelineButton || + !protocolCandidateLimitInput || + !refreshProtocolCandidatesButton || !pairSelect || !timeframeSelect || !customTimeframeInput || @@ -396,6 +405,7 @@ document.addEventListener("DOMContentLoaded", async () => { !backfillSummaryTextarea || !localDiagnosticsTextarea || !localValidationTextarea || + !protocolCandidateSummariesTextarea || !chartElement || !chartMeta || !clearLogButton || @@ -644,6 +654,45 @@ document.addEventListener("DOMContentLoaded", async () => { } }); + refreshProtocolCandidatesButton.addEventListener("click", async () => { + const limit = readPositiveIntegerInput( + protocolCandidateLimitInput, + logTextarea, + "protocolCandidateSummaryLimit", + ); + if (limit === undefined) { + return; + } + + appendLogLine( + logTextarea, + `[ui] loading protocol candidate summaries with limit '${limit.toString()}'`, + ); + + const request: DemoPipeline2ProtocolCandidateSummaryRequest = { + limit, + }; + + try { + const payload = await invoke( + "demo_pipeline2_get_protocol_candidate_summaries", + { request }, + ); + + protocolCandidateSummariesTextarea.value = payload.summariesJson; + + appendLogLine( + logTextarea, + "[ui] protocol candidate summaries loaded", + ); + } catch (error) { + appendLogLine( + logTextarea, + `[ui] protocol candidate summary error: ${String(error)}`, + ); + } + }); + loadCandlesButton.addEventListener("click", async () => { const pairIdText = pairSelect.value.trim(); if (pairIdText === "") { diff --git a/kb_demo_app/package.json b/kb_demo_app/package.json index d3e1010..c106621 100644 --- a/kb_demo_app/package.json +++ b/kb_demo_app/package.json @@ -1,7 +1,7 @@ { "name": "kb-demo-app", "private": true, - "version": "0.7.27", + "version": "0.7.28", "type": "module", "scripts": { "dev": "vite", diff --git a/kb_demo_app/src/demo_pipeline2.rs b/kb_demo_app/src/demo_pipeline2.rs index 74e6380..d3f2083 100644 --- a/kb_demo_app/src/demo_pipeline2.rs +++ b/kb_demo_app/src/demo_pipeline2.rs @@ -10,6 +10,34 @@ use tauri::Manager; use ts_rs::TS; + +/// Request payload for protocol candidate summary diagnostics. +#[derive(Clone, Debug, serde::Deserialize, ts_rs::TS)] +#[ts( + export, + export_to = "../frontend/ts/bindings/DemoPipeline2ProtocolCandidateSummaryRequest.ts" +)] +#[serde(rename_all = "camelCase")] +pub(crate) struct DemoPipeline2ProtocolCandidateSummaryRequest { + /// Maximum number of summary rows to return. + pub limit: u32, +} + +/// Response payload for protocol candidate summary diagnostics. +#[derive(Clone, Debug, serde::Serialize, ts_rs::TS)] +#[ts( + export, + export_to = "../frontend/ts/bindings/DemoPipeline2ProtocolCandidateSummaryPayload.ts" +)] +#[serde(rename_all = "camelCase")] +pub(crate) struct DemoPipeline2ProtocolCandidateSummaryPayload { + /// Pretty JSON summary rows. + pub summaries_json: std::string::String, +} + + + + /// Local diagnostics payload returned to the UI. #[derive(Clone, Debug, serde::Serialize, TS)] #[ts( @@ -690,6 +718,42 @@ pub(crate) struct DemoPipeline2PairCandlesPayload { pub candles_json: std::string::String, } +/// Lists protocol candidate summaries ordered by investigation priority. +#[tauri::command] +pub(crate) async fn demo_pipeline2_get_protocol_candidate_summaries( + state: tauri::State<'_, crate::AppState>, + request: DemoPipeline2ProtocolCandidateSummaryRequest, +) -> Result { + if request.limit == 0 { + return Err("protocol candidate summary limit must be > 0".to_string()); + } + let summaries_result = kb_lib::query_protocol_candidate_summaries_list_by_priority( + state.database.as_ref(), + request.limit, + ) + .await; + let summaries = match summaries_result { + Ok(summaries) => summaries, + Err(error) => { + return Err(format!( + "cannot list protocol candidate summaries with limit '{}': {}", + request.limit, error + )); + }, + }; + let summaries_json_result = serde_json::to_string_pretty(&summaries); + let summaries_json = match summaries_json_result { + Ok(summaries_json) => summaries_json, + Err(error) => { + return Err(format!( + "cannot serialize protocol candidate summaries: {}", + error + )); + }, + }; + return Ok(DemoPipeline2ProtocolCandidateSummaryPayload { summaries_json }); +} + /// Runs local pipeline diagnostics from persisted data only. #[tauri::command] pub(crate) async fn demo_pipeline2_diagnose_local_pipeline( diff --git a/kb_demo_app/src/lib.rs b/kb_demo_app/src/lib.rs index 02d9c23..45d151d 100644 --- a/kb_demo_app/src/lib.rs +++ b/kb_demo_app/src/lib.rs @@ -153,6 +153,7 @@ pub async fn run() -> Result<(), kb_lib::Error> { crate::demo_pipeline2::demo_pipeline2_replay_local_pipeline, crate::demo_pipeline2::demo_pipeline2_diagnose_local_pipeline, crate::demo_pipeline2::demo_pipeline2_validate_local_pipeline, + crate::demo_pipeline2::demo_pipeline2_get_protocol_candidate_summaries, ]); tauri_builder = tauri_builder.plugin(tracing_builder.build::()); tauri_builder = tauri_builder.setup(|app| { diff --git a/kb_demo_app/tauri.conf.json b/kb_demo_app/tauri.conf.json index 048d32c..5e77a71 100644 --- a/kb_demo_app/tauri.conf.json +++ b/kb_demo_app/tauri.conf.json @@ -1,7 +1,7 @@ { "$schema": "https://schema.tauri.app/config/2", "productName": "kb-demo-app", - "version": "0.7.27", + "version": "0.7.28", "identifier": "com.sasedev.kb-demo-app", "build": { "beforeDevCommand": "npm run dev", diff --git a/kb_lib/src/constants.rs b/kb_lib/src/constants.rs index 3c4aab2..7e686df 100644 --- a/kb_lib/src/constants.rs +++ b/kb_lib/src/constants.rs @@ -14,17 +14,134 @@ pub const SPL_TOKEN_2022_PROGRAM_ID: &str = "TokenzQdBNbLqP5VEhdkAS6EPFLC1PHnBqC /// @see solana_sdk::pubkey::Pubkey = spl_associated_token_account_interface::program::ID pub const ASSOCIATED_TOKEN_PROGRAM_ID: &str = "ATokenGPvbdGVxr1b2hvZbsiqW5xWH25efTNsLJA8knL"; -/// Wrapped SOL mint identifier. ("So11111111111111111111111111111111111111112"). -/// @see solana_sdk::pubkey::Pubkey = spl_token_interface::native_mint::ID -pub const WSOL_MINT_ID: &str = "So11111111111111111111111111111111111111112"; +/// Address Lookup Table program identifier. ("AddressLookupTab1e1111111111111111111111111"). +/// @see solana_sdk_ids::address_lookup_table::ID +pub const ADDRESS_LOOKUP_TABLE_PROGRAM_ID: &str = "AddressLookupTab1e1111111111111111111111111"; + +/// BPF Loader program identifier. ("BPFLoader1111111111111111111111111111111111"). +/// @see solana_sdk_ids::bpf_loader_deprecated::ID +pub const BPF_LOADER_DEPRECATED_PROGRAM_ID: &str = "BPFLoader1111111111111111111111111111111111"; + +/// BPF Loader program identifier. ("BPFLoaderUpgradeab1e11111111111111111111111"). +/// @see solana_sdk_ids::bpf_loader_upgradeable::ID +pub const BPF_LOADER_UPGRADEABLE_PROGRAM_ID: &str = "BPFLoaderUpgradeab1e11111111111111111111111"; + +/// Compute Budget program identifier. ("ComputeBudget111111111111111111111111111111"). +/// @see solana_sdk_ids::compute_budget::ID +pub const COMPUTE_BUDGET_PROGRAM_ID: &str = "ComputeBudget111111111111111111111111111111"; + +/// Config program identifier. ("Config1111111111111111111111111111111111111"). +/// @see solana_sdk::pubkey::Pubkey = solana_sdk_ids::config::ID +pub const CONFIG_PROGRAM_ID: &str = "Config1111111111111111111111111111111111111"; + +/// ED25519 program identifier. ("Ed25519SigVerify111111111111111111111111111"). +/// @see solana_sdk::pubkey::Pubkey = solana_sdk_ids::ed25519_program::ID +pub const ED25519_PROGRAM_ID: &str = "Ed25519SigVerify111111111111111111111111111"; + +/// Feature program identifier. ("Feature111111111111111111111111111111111111"). +/// @see solana_sdk::pubkey::Pubkey = solana_sdk_ids::feature::ID +pub const FEATURE_PROGRAM_ID: &str = "Feature111111111111111111111111111111111111"; + +/// Incinerator program identifier. ("1nc1nerator11111111111111111111111111111111"). +/// @see solana_sdk::pubkey::Pubkey = solana_sdk_ids::incinerator::ID +pub const INCINERATOR_PROGRAM_ID: &str = "1nc1nerator11111111111111111111111111111111"; + +/// Loader V4 program identifier. ("LoaderV411111111111111111111111111111111111"). +/// @see solana_sdk::pubkey::Pubkey = solana_sdk_ids::loader_v4::ID +pub const LOADER_V4_PROGRAM_ID: &str = "LoaderV411111111111111111111111111111111111"; + +/// Native Loader program identifier. ("NativeLoader1111111111111111111111111111111"). +/// @see solana_sdk::pubkey::Pubkey = solana_sdk_ids::native_loader::ID +pub const NATIVE_LOADER_PROGRAM_ID: &str = "NativeLoader1111111111111111111111111111111"; + +/// Secp256k1 program identifier. ("KeccakSecp256k11111111111111111111111111111"). +/// @see solana_sdk::pubkey::Pubkey = solana_sdk_ids::secp256k1_program::ID +pub const SECP256K1_PROGRAM_ID: &str = "KeccakSecp256k11111111111111111111111111111"; + +/// Secp256r1 program identifier. ("Secp256r1SigVerify1111111111111111111111111"). +/// @see solana_sdk::pubkey::Pubkey = solana_sdk_ids::secp256r1_program::ID +pub const SECP256R1_PROGRAM_ID: &str = "Secp256r1SigVerify1111111111111111111111111"; + +/// Stake program identifier. ("Stake11111111111111111111111111111111111111"). +/// @see solana_sdk::pubkey::Pubkey = solana_sdk_ids::stake::ID +pub const STAKE_PROGRAM_ID: &str = "Stake11111111111111111111111111111111111111"; + +/// Stake Config program identifier. ("StakeConfig11111111111111111111111111111111"). +/// @see solana_sdk::pubkey::Pubkey = solana_sdk_ids::stake::config::ID +pub const STAKE_CONFIG_PROGRAM_ID: &str = "StakeConfig11111111111111111111111111111111"; /// System program identifier. ("11111111111111111111111111111111"). /// @see solana_sdk::pubkey::Pubkey = solana_sdk_ids::system_program::ID pub const SYSTEM_PROGRAM_ID: &str = "11111111111111111111111111111111"; -/// Compute Budget program identifier. ("ComputeBudget111111111111111111111111111111"). -/// @see solana_sdk_ids::compute_budget::ID -pub const COMPUTE_BUDGET_PROGRAM_ID: &str = "ComputeBudget111111111111111111111111111111"; +/// Vote program identifier. ("Vote111111111111111111111111111111111111111"). +/// @see solana_sdk::pubkey::Pubkey = solana_sdk_ids::vote::ID +pub const VOTE_PROGRAM_ID: &str = "Vote111111111111111111111111111111111111111"; + +/// Sysvar program identifier. ("Sysvar1111111111111111111111111111111111111"). +/// @see solana_sdk::pubkey::Pubkey = solana_sdk_ids::sysvar::ID +pub const SYSVAR_PROGRAM_ID: &str = "Sysvar1111111111111111111111111111111111111"; + +/// Sysvar Clock program identifier. ("SysvarC1ock11111111111111111111111111111111"). +/// @see solana_sdk::pubkey::Pubkey = solana_sdk_ids::sysvar::clock::ID +pub const SYSVAR_CLOCK_PROGRAM_ID: &str = "SysvarC1ock11111111111111111111111111111111"; + +/// Sysvar Epoch Rewards program identifier. ("SysvarEpochRewards1111111111111111111111111"). +/// @see solana_sdk::pubkey::Pubkey = solana_sdk_ids::sysvar::epoch_rewards::ID +pub const SYSVAR_EPOCH_REWARDS_PROGRAM_ID: &str = "SysvarEpochRewards1111111111111111111111111"; + +/// Sysvar Epoch Schedule program identifier. ("SysvarEpochSchedu1e111111111111111111111111"). +/// @see solana_sdk::pubkey::Pubkey = solana_sdk_ids::sysvar::epoch_schedule::ID +pub const SYSVAR_EPOCH_SCHEDULE_PROGRAM_ID: &str = "SysvarEpochSchedu1e111111111111111111111111"; + +/// Sysvar Fees program identifier. ("SysvarFees111111111111111111111111111111111"). +/// @see solana_sdk::pubkey::Pubkey = solana_sdk_ids::sysvar::fees::ID +pub const SYSVAR_FEES_PROGRAM_ID: &str = "SysvarFees111111111111111111111111111111111"; + +/// Sysvar Instructions program identifier. ("Sysvar1nstructions1111111111111111111111111"). +/// @see solana_sdk::pubkey::Pubkey = solana_sdk_ids::sysvar::instructions::ID +pub const SYSVAR_INSTRUCTIONS_PROGRAM_ID: &str = "Sysvar1nstructions1111111111111111111111111"; + +/// Sysvar Last Restart Slot program identifier. ("SysvarLastRestartS1ot1111111111111111111111"). +/// @see solana_sdk::pubkey::Pubkey = solana_sdk_ids::sysvar::last_restart_slot::ID +pub const SYSVAR_LAST_RESTART_SLOT_PROGRAM_ID: &str = "SysvarLastRestartS1ot1111111111111111111111"; + +/// Sysvar Recent Blockhashes program identifier. ("SysvarRecentB1ockHashes11111111111111111111"). +/// @see solana_sdk::pubkey::Pubkey = solana_sdk_ids::sysvar::recent_blockhashes::ID +pub const SYSVAR_RECENT_BLOCKHASHES_PROGRAM_ID: &str = + "SysvarRecentB1ockHashes11111111111111111111"; + +/// Sysvar Rent program identifier. ("SysvarRent111111111111111111111111111111111"). +/// @see solana_sdk::pubkey::Pubkey = solana_sdk_ids::sysvar::rent::ID +pub const SYSVAR_RENT_PROGRAM_ID: &str = "SysvarRent111111111111111111111111111111111"; + +/// Sysvar Rewards program identifier. ("SysvarRewards111111111111111111111111111111"). +/// @see solana_sdk::pubkey::Pubkey = solana_sdk_ids::sysvar::rewards::ID +pub const SYSVAR_REWARDS_PROGRAM_ID: &str = "SysvarRewards111111111111111111111111111111"; + +/// Sysvar Slot Hashes program identifier. ("SysvarS1otHashes111111111111111111111111111"). +/// @see solana_sdk::pubkey::Pubkey = solana_sdk_ids::sysvar::slot_hashes::ID +pub const SYSVAR_SLOT_HASHES_PROGRAM_ID: &str = "SysvarS1otHashes111111111111111111111111111"; + +/// Sysvar Slot History program identifier. ("SysvarS1otHistory11111111111111111111111111"). +/// @see solana_sdk::pubkey::Pubkey = solana_sdk_ids::sysvar::slot_history::ID +pub const SYSVAR_SLOT_HISTORY_PROGRAM_ID: &str = "SysvarS1otHistory11111111111111111111111111"; + +/// Sysvar Stake History program identifier. ("SysvarStakeHistory1111111111111111111111111"). +/// @see solana_sdk::pubkey::Pubkey = solana_sdk_ids::sysvar::stake_history::ID +pub const SYSVAR_STAKE_HISTORY_PROGRAM_ID: &str = "SysvarStakeHistory1111111111111111111111111"; + +/// Zk Token Proof program identifier. ("ZkTokenProof1111111111111111111111111111111"). +/// @see solana_sdk::pubkey::Pubkey = solana_sdk_ids::zk_token_proof_program::ID +pub const ZK_TOKEN_PROOF_PROGRAM_ID: &str = "ZkTokenProof1111111111111111111111111111111"; + +/// Zk El Gamal Proof program identifier. ("ZkE1Gama1Proof11111111111111111111111111111"). +/// @see solana_sdk::pubkey::Pubkey = solana_sdk_ids::zk_elgamal_proof_program::ID +pub const ZK_ELGAMAL_PROOF_PROGRAM_ID: &str = "ZkE1Gama1Proof11111111111111111111111111111"; + +/// Wrapped SOL mint identifier. ("So11111111111111111111111111111111111111112"). +/// @see solana_sdk::pubkey::Pubkey = spl_token_interface::native_mint::ID +pub const WSOL_MINT_ID: &str = "So11111111111111111111111111111111111111112"; /// DexLab Swap/Pool program id. ("DSwpgjMvXhtGn6BsbqmacdBZyfLj6jSWf3HJpdJtmg6N"). pub const DEXLAB_PROGRAM_ID: &str = "DSwpgjMvXhtGn6BsbqmacdBZyfLj6jSWf3HJpdJtmg6N"; @@ -41,6 +158,11 @@ pub const METEORA_DAMM_V2_PROGRAM_ID: &str = "cpamdpZCGKUy5JxQXB4dcpGPiikHawvSWA /// Meteora DBC program id. ("dbcij3LWUppWqq96dh6gJWwBifmcGfLSB5D4DuSMaqN"). pub const METEORA_DBC_PROGRAM_ID: &str = "dbcij3LWUppWqq96dh6gJWwBifmcGfLSB5D4DuSMaqN"; +/// Meteora DLMM program id. ("LBUZKhRxPF3XUpBCjp4YzTKgLccjZhTSDM9YuVaPwxo"). +/// +/// DLMM = Dynamic Liquidity Market Maker. +pub const METEORA_DLMM_PROGRAM_ID: &str = "LBUZKhRxPF3XUpBCjp4YzTKgLccjZhTSDM9YuVaPwxo"; + /// Orca Whirlpools program id. ("whirLbMiicVdio4qvUfM5KAg6Ct8VwpYzGff3uctyCc"). pub const ORCA_WHIRLPOOLS_PROGRAM_ID: &str = "whirLbMiicVdio4qvUfM5KAg6Ct8VwpYzGff3uctyCc"; @@ -58,3 +180,18 @@ pub const RAYDIUM_CLMM_PROGRAM_ID: &str = "CAMMCzo5YL8w4VFF8KVHrK22GGUsp5VTaW7gr /// Raydium CPMM mainnet program id. ("CPMMoo8L3F4NbTegBCKVNunggL7H1ZpdTHKxQB5qKP1C"). pub const RAYDIUM_CPMM_PROGRAM_ID: &str = "CPMMoo8L3F4NbTegBCKVNunggL7H1ZpdTHKxQB5qKP1C"; + +/// Raydium LaunchLab program id. ("LanMV9sAd7wArD4vJFi2qDdfnVhFxYSUg6eADduJ3uj"). +pub const RAYDIUM_LAUNCHLAB_PROGRAM_ID: &str = "LanMV9sAd7wArD4vJFi2qDdfnVhFxYSUg6eADduJ3uj"; + +/// Raydium AMM routing program id. ("routeUGWgWzqBWFcrCfv8tritsqukccJPu3q5GPP3xS"). +pub const RAYDIUM_AMM_ROUTING_PROGRAM_ID: &str = "routeUGWgWzqBWFcrCfv8tritsqukccJPu3q5GPP3xS"; + +/// Raydium Stable Swap AMM program id, deprecated. ("5quBtoiQqxF9Jv6KYKctB59NT3gtJD2Y65kdnB1Uev3h"). +pub const RAYDIUM_STABLE_SWAP_AMM_PROGRAM_ID: &str = "5quBtoiQqxF9Jv6KYKctB59NT3gtJD2Y65kdnB1Uev3h"; + +/// Known Solana arbitrage/sandwich bot program id observed in local corpus. +/// +/// This is not treated as a DEX program. It is used only to tag protocol +/// candidates with `candidate_surface = "arbitrage_bot"`. +pub const ARBITRAGE_BOT_6MWVT_PROGRAM_ID: &str = "6MWVTis8rmmk6Vt9zmAJJbmb3VuLpzoQ1aHH4N6wQEGh"; diff --git a/kb_lib/src/db.rs b/kb_lib/src/db.rs index e2e7b94..b68063a 100644 --- a/kb_lib/src/db.rs +++ b/kb_lib/src/db.rs @@ -50,11 +50,14 @@ pub use dtos::PoolDto; pub use dtos::PoolListingDto; pub use dtos::PoolOriginDto; pub use dtos::PoolTokenDto; +pub use dtos::ProtocolCandidateDto; +pub use dtos::ProtocolCandidateSummaryDto; pub use dtos::SwapDto; pub use dtos::TokenBurnEventDto; pub use dtos::TokenDto; pub use dtos::TokenMintEventDto; pub use dtos::TradeEventDto; +pub use dtos::TransactionClassificationDto; pub use dtos::WalletDto; pub use dtos::WalletHoldingDto; pub use dtos::WalletParticipationDto; @@ -82,11 +85,14 @@ pub use entities::PoolEntity; pub use entities::PoolListingEntity; pub use entities::PoolOriginEntity; pub use entities::PoolTokenEntity; +pub use entities::ProtocolCandidateEntity; +pub use entities::ProtocolCandidateSummaryEntity; pub use entities::SwapEntity; pub use entities::TokenBurnEventEntity; pub use entities::TokenEntity; pub use entities::TokenMintEventEntity; pub use entities::TradeEventEntity; +pub use entities::TransactionClassificationEntity; pub use entities::WalletEntity; pub use entities::WalletHoldingEntity; pub use entities::WalletParticipationEntity; @@ -172,6 +178,12 @@ pub use queries::query_pool_tokens_upsert; pub use queries::query_pools_get_by_address; pub use queries::query_pools_list; pub use queries::query_pools_upsert; +pub use queries::query_protocol_candidate_summaries_list_by_priority; +pub use queries::query_protocol_candidates_delete_by_transaction_id; +pub use queries::query_protocol_candidates_insert; +pub use queries::query_protocol_candidates_list_by_program_id; +pub use queries::query_protocol_candidates_list_by_transaction_id; +pub use queries::query_protocol_candidates_list_recent; pub use queries::query_swaps_list_recent; pub use queries::query_swaps_upsert; pub use queries::query_token_burn_events_list_recent; @@ -187,6 +199,10 @@ pub use queries::query_trade_events_get_by_decoded_event_id; pub use queries::query_trade_events_list_by_pair_id; pub use queries::query_trade_events_list_by_transaction_id; pub use queries::query_trade_events_upsert; +pub use queries::query_transaction_classifications_get_by_signature; +pub use queries::query_transaction_classifications_get_by_transaction_id; +pub use queries::query_transaction_classifications_list_recent; +pub use queries::query_transaction_classifications_upsert; pub use queries::query_wallet_holdings_get_by_wallet_and_token; pub use queries::query_wallet_holdings_list_by_wallet_id; pub use queries::query_wallet_holdings_upsert; diff --git a/kb_lib/src/db/dtos.rs b/kb_lib/src/db/dtos.rs index 0262d7d..e2b5677 100644 --- a/kb_lib/src/db/dtos.rs +++ b/kb_lib/src/db/dtos.rs @@ -27,11 +27,14 @@ mod pool; mod pool_listing; mod pool_origin; mod pool_token; +mod protocol_candidate; +mod protocol_candidate_summary; mod swap; mod token; mod token_burn_event; mod token_mint_event; mod trade_event; +mod transaction_classification; mod wallet; mod wallet_holding; mod wallet_participation; @@ -82,11 +85,14 @@ pub use pool::PoolDto; pub use pool_listing::PoolListingDto; pub use pool_origin::PoolOriginDto; pub use pool_token::PoolTokenDto; +pub use protocol_candidate::ProtocolCandidateDto; +pub use protocol_candidate_summary::ProtocolCandidateSummaryDto; pub use swap::SwapDto; pub use token::TokenDto; pub use token_burn_event::TokenBurnEventDto; pub use token_mint_event::TokenMintEventDto; pub use trade_event::TradeEventDto; +pub use transaction_classification::TransactionClassificationDto; pub use wallet::WalletDto; pub use wallet_holding::WalletHoldingDto; pub use wallet_participation::WalletParticipationDto; diff --git a/kb_lib/src/db/dtos/protocol_candidate.rs b/kb_lib/src/db/dtos/protocol_candidate.rs new file mode 100644 index 0000000..856aaf0 --- /dev/null +++ b/kb_lib/src/db/dtos/protocol_candidate.rs @@ -0,0 +1,114 @@ +// file: kb_lib/src/db/dtos/protocol_candidate.rs + +//! Protocol candidate DTO. + +/// Application-facing protocol candidate DTO. +/// +/// A protocol candidate records a program/instruction that should be inspected +/// later because it may correspond to an unsupported DEX, launch surface, +/// migration path or protocol-specific non-trade event. +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +pub struct ProtocolCandidateDto { + /// Optional numeric primary key. + pub id: std::option::Option, + /// Related chain transaction id. + pub transaction_id: i64, + /// Optional related chain instruction id. + pub instruction_id: std::option::Option, + /// Transaction signature. + pub signature: std::string::String, + /// Optional Solana slot. + pub slot: std::option::Option, + /// Program id observed in the transaction instruction. + pub program_id: std::string::String, + /// Optional program name hint from parsed transaction data. + pub program_name_hint: std::option::Option, + /// Optional candidate protocol code. + pub candidate_protocol: std::option::Option, + /// Optional candidate surface code. + pub candidate_surface: std::option::Option, + /// Human-readable reason. + pub reason: std::string::String, + /// Serialized JSON evidence. + pub evidence_json: std::string::String, + /// Creation timestamp. + pub created_at: chrono::DateTime, +} + +impl ProtocolCandidateDto { + /// Creates a protocol candidate DTO. + #[allow(clippy::too_many_arguments)] + pub fn new( + transaction_id: i64, + instruction_id: std::option::Option, + signature: std::string::String, + slot: std::option::Option, + program_id: std::string::String, + program_name_hint: std::option::Option, + candidate_protocol: std::option::Option, + candidate_surface: std::option::Option, + reason: std::string::String, + evidence_json: std::string::String, + ) -> Self { + return Self { + id: None, + transaction_id, + instruction_id, + signature, + slot, + program_id, + program_name_hint, + candidate_protocol, + candidate_surface, + reason, + evidence_json, + created_at: chrono::Utc::now(), + }; + } +} + +impl TryFrom for ProtocolCandidateDto { + type Error = crate::Error; + + fn try_from(entity: crate::ProtocolCandidateEntity) -> Result { + let slot = match entity.slot { + Some(slot) => { + let slot_result = u64::try_from(slot); + match slot_result { + Ok(slot) => Some(slot), + Err(error) => { + return Err(crate::Error::Db(format!( + "cannot convert protocol candidate slot '{}' to u64: {}", + slot, error + ))); + }, + } + }, + None => None, + }; + let created_at_result = chrono::DateTime::parse_from_rfc3339(&entity.created_at); + let created_at = match created_at_result { + Ok(created_at) => created_at.with_timezone(&chrono::Utc), + Err(error) => { + return Err(crate::Error::Db(format!( + "cannot parse protocol candidate created_at '{}': {}", + entity.created_at, error + ))); + }, + }; + return Ok(Self { + id: Some(entity.id), + transaction_id: entity.transaction_id, + instruction_id: entity.instruction_id, + signature: entity.signature, + slot, + program_id: entity.program_id, + program_name_hint: entity.program_name_hint, + candidate_protocol: entity.candidate_protocol, + candidate_surface: entity.candidate_surface, + reason: entity.reason, + evidence_json: entity.evidence_json, + created_at, + }); + } +} diff --git a/kb_lib/src/db/dtos/protocol_candidate_summary.rs b/kb_lib/src/db/dtos/protocol_candidate_summary.rs new file mode 100644 index 0000000..b8bdc22 --- /dev/null +++ b/kb_lib/src/db/dtos/protocol_candidate_summary.rs @@ -0,0 +1,96 @@ +// file: kb_lib/src/db/dtos/protocol_candidate_summary.rs + +//! Protocol candidate summary DTO. + +/// Aggregated protocol candidate diagnostic row. +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +pub struct ProtocolCandidateSummaryDto { + /// Program id observed in protocol candidates. + pub program_id: std::string::String, + /// Optional program name hint. + pub program_name_hint: std::option::Option, + /// Optional candidate protocol. + pub candidate_protocol: std::option::Option, + /// Optional candidate surface. + pub candidate_surface: std::option::Option, + /// Candidate reason. + pub reason: std::string::String, + /// Number of candidate rows. + pub occurrence_count: u64, + /// Number of distinct transactions. + pub transaction_count: u64, + /// Latest observed slot. + pub last_slot: std::option::Option, + /// Latest candidate row id in this group. + pub latest_candidate_id: i64, + /// Latest signature in this group. + pub latest_signature: std::string::String, + /// Latest candidate creation timestamp. + pub latest_created_at: chrono::DateTime, +} + +impl TryFrom for ProtocolCandidateSummaryDto { + type Error = crate::Error; + + fn try_from(entity: crate::ProtocolCandidateSummaryEntity) -> Result { + let occurrence_count_result = u64::try_from(entity.occurrence_count); + let occurrence_count = match occurrence_count_result { + Ok(occurrence_count) => occurrence_count, + Err(error) => { + return Err(crate::Error::Db(format!( + "cannot convert protocol candidate occurrence_count '{}' to u64: {}", + entity.occurrence_count, error + ))); + }, + }; + let transaction_count_result = u64::try_from(entity.transaction_count); + let transaction_count = match transaction_count_result { + Ok(transaction_count) => transaction_count, + Err(error) => { + return Err(crate::Error::Db(format!( + "cannot convert protocol candidate transaction_count '{}' to u64: {}", + entity.transaction_count, error + ))); + }, + }; + let last_slot = match entity.last_slot { + Some(last_slot) => { + let slot_result = u64::try_from(last_slot); + match slot_result { + Ok(slot) => Some(slot), + Err(error) => { + return Err(crate::Error::Db(format!( + "cannot convert protocol candidate last_slot '{}' to u64: {}", + last_slot, error + ))); + }, + } + }, + None => None, + }; + let latest_created_at_result = + chrono::DateTime::parse_from_rfc3339(entity.latest_created_at.as_str()); + let latest_created_at = match latest_created_at_result { + Ok(latest_created_at) => latest_created_at.with_timezone(&chrono::Utc), + Err(error) => { + return Err(crate::Error::Db(format!( + "cannot parse protocol candidate latest_created_at '{}': {}", + entity.latest_created_at, error + ))); + }, + }; + return Ok(Self { + program_id: entity.program_id, + program_name_hint: entity.program_name_hint, + candidate_protocol: entity.candidate_protocol, + candidate_surface: entity.candidate_surface, + reason: entity.reason, + occurrence_count, + transaction_count, + last_slot, + latest_candidate_id: entity.latest_candidate_id, + latest_signature: entity.latest_signature, + latest_created_at, + }); + } +} diff --git a/kb_lib/src/db/dtos/transaction_classification.rs b/kb_lib/src/db/dtos/transaction_classification.rs new file mode 100644 index 0000000..9661852 --- /dev/null +++ b/kb_lib/src/db/dtos/transaction_classification.rs @@ -0,0 +1,130 @@ +// file: kb_lib/src/db/dtos/transaction_classification.rs + +//! Transaction classification DTO. + +/// Application-facing transaction classification DTO. +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +pub struct TransactionClassificationDto { + /// Optional numeric primary key. + pub id: std::option::Option, + /// Related chain transaction id. + pub transaction_id: i64, + /// Transaction signature. + pub signature: std::string::String, + /// Optional Solana slot. + pub slot: std::option::Option, + /// Stable classification kind. + pub classification_kind: std::string::String, + /// Optional primary protocol name. + pub primary_protocol: std::option::Option, + /// Optional primary program id. + pub primary_program_id: std::option::Option, + /// Confidence level from 0 to 100. + pub confidence_level: i16, + /// Human-readable reason. + pub reason: std::string::String, + /// Serialized JSON evidence. + pub evidence_json: std::string::String, + /// Creation timestamp. + pub created_at: chrono::DateTime, + /// Update timestamp. + pub updated_at: chrono::DateTime, +} + +impl TransactionClassificationDto { + /// Creates a new transaction classification DTO. + #[allow(clippy::too_many_arguments)] + pub fn new( + transaction_id: i64, + signature: std::string::String, + slot: std::option::Option, + classification_kind: std::string::String, + primary_protocol: std::option::Option, + primary_program_id: std::option::Option, + confidence_level: i16, + reason: std::string::String, + evidence_json: std::string::String, + ) -> Self { + let now = chrono::Utc::now(); + return Self { + id: None, + transaction_id, + signature, + slot, + classification_kind, + primary_protocol, + primary_program_id, + confidence_level, + reason, + evidence_json, + created_at: now, + updated_at: now, + }; + } +} + +impl TryFrom for TransactionClassificationDto { + type Error = crate::Error; + + fn try_from(entity: crate::TransactionClassificationEntity) -> Result { + let slot = match entity.slot { + Some(slot) => { + let slot_result = u64::try_from(slot); + match slot_result { + Ok(slot) => Some(slot), + Err(error) => { + return Err(crate::Error::Db(format!( + "cannot convert transaction classification slot '{}' to u64: {}", + slot, error + ))); + }, + } + }, + None => None, + }; + let created_at_result = chrono::DateTime::parse_from_rfc3339(&entity.created_at); + let created_at = match created_at_result { + Ok(created_at) => created_at.with_timezone(&chrono::Utc), + Err(error) => { + return Err(crate::Error::Db(format!( + "cannot parse transaction classification created_at '{}': {}", + entity.created_at, error + ))); + }, + }; + let updated_at_result = chrono::DateTime::parse_from_rfc3339(&entity.updated_at); + let updated_at = match updated_at_result { + Ok(updated_at) => updated_at.with_timezone(&chrono::Utc), + Err(error) => { + return Err(crate::Error::Db(format!( + "cannot parse transaction classification updated_at '{}': {}", + entity.updated_at, error + ))); + }, + }; + let confidence_level_result = i16::try_from(entity.confidence_level); + let confidence_level = match confidence_level_result { + Ok(confidence_level) => confidence_level, + Err(error) => { + return Err(crate::Error::Db(format!( + "cannot convert transaction classification confidence_level '{}' to i16: {}", + entity.confidence_level, error + ))); + }, + }; + return Ok(Self { + id: Some(entity.id), + transaction_id: entity.transaction_id, + signature: entity.signature, + slot, + classification_kind: entity.classification_kind, + primary_protocol: entity.primary_protocol, + primary_program_id: entity.primary_program_id, + confidence_level, + reason: entity.reason, + evidence_json: entity.evidence_json, + created_at, + updated_at, + }); + } +} diff --git a/kb_lib/src/db/entities.rs b/kb_lib/src/db/entities.rs index 0e10997..bd39c94 100644 --- a/kb_lib/src/db/entities.rs +++ b/kb_lib/src/db/entities.rs @@ -28,11 +28,14 @@ mod pool; mod pool_listing; mod pool_origin; mod pool_token; +mod protocol_candidate; +mod protocol_candidate_summary; mod swap; mod token; mod token_burn_event; mod token_mint_event; mod trade_event; +mod transaction_classification; mod wallet; mod wallet_holding; mod wallet_participation; @@ -61,11 +64,14 @@ pub use pool::PoolEntity; pub use pool_listing::PoolListingEntity; pub use pool_origin::PoolOriginEntity; pub use pool_token::PoolTokenEntity; +pub use protocol_candidate::ProtocolCandidateEntity; +pub use protocol_candidate_summary::ProtocolCandidateSummaryEntity; pub use swap::SwapEntity; pub use token::TokenEntity; pub use token_burn_event::TokenBurnEventEntity; pub use token_mint_event::TokenMintEventEntity; pub use trade_event::TradeEventEntity; +pub use transaction_classification::TransactionClassificationEntity; pub use wallet::WalletEntity; pub use wallet_holding::WalletHoldingEntity; pub use wallet_participation::WalletParticipationEntity; diff --git a/kb_lib/src/db/entities/protocol_candidate.rs b/kb_lib/src/db/entities/protocol_candidate.rs new file mode 100644 index 0000000..c30eaee --- /dev/null +++ b/kb_lib/src/db/entities/protocol_candidate.rs @@ -0,0 +1,32 @@ +// file: kb_lib/src/db/entities/protocol_candidate.rs + +//! Protocol candidate entity. + +/// Persisted protocol candidate row. +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, sqlx::FromRow)] +pub struct ProtocolCandidateEntity { + /// Numeric primary key. + pub id: i64, + /// Related chain transaction id. + pub transaction_id: i64, + /// Optional related chain instruction id. + pub instruction_id: std::option::Option, + /// Transaction signature. + pub signature: std::string::String, + /// Optional Solana slot. + pub slot: std::option::Option, + /// Program id observed in the transaction instruction. + pub program_id: std::string::String, + /// Optional program name hint from parsed transaction data. + pub program_name_hint: std::option::Option, + /// Optional candidate protocol code. + pub candidate_protocol: std::option::Option, + /// Optional candidate surface code. + pub candidate_surface: std::option::Option, + /// Human-readable reason. + pub reason: std::string::String, + /// Serialized JSON evidence. + pub evidence_json: std::string::String, + /// Creation timestamp encoded as RFC3339 UTC text. + pub created_at: std::string::String, +} diff --git a/kb_lib/src/db/entities/protocol_candidate_summary.rs b/kb_lib/src/db/entities/protocol_candidate_summary.rs new file mode 100644 index 0000000..5717436 --- /dev/null +++ b/kb_lib/src/db/entities/protocol_candidate_summary.rs @@ -0,0 +1,30 @@ +// file: kb_lib/src/db/entities/protocol_candidate_summary.rs + +//! Protocol candidate summary entity. + +/// Aggregated protocol candidate diagnostic row. +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, sqlx::FromRow)] +pub struct ProtocolCandidateSummaryEntity { + /// Program id observed in protocol candidates. + pub program_id: std::string::String, + /// Optional program name hint. + pub program_name_hint: std::option::Option, + /// Optional candidate protocol. + pub candidate_protocol: std::option::Option, + /// Optional candidate surface. + pub candidate_surface: std::option::Option, + /// Candidate reason. + pub reason: std::string::String, + /// Number of candidate rows. + pub occurrence_count: i64, + /// Number of distinct transactions. + pub transaction_count: i64, + /// Latest observed slot. + pub last_slot: std::option::Option, + /// Latest candidate row id in this group. + pub latest_candidate_id: i64, + /// Latest signature in this group. + pub latest_signature: std::string::String, + /// Latest candidate creation timestamp encoded as RFC3339 UTC text. + pub latest_created_at: std::string::String, +} diff --git a/kb_lib/src/db/entities/transaction_classification.rs b/kb_lib/src/db/entities/transaction_classification.rs new file mode 100644 index 0000000..3c18aa9 --- /dev/null +++ b/kb_lib/src/db/entities/transaction_classification.rs @@ -0,0 +1,32 @@ +// file: kb_lib/src/db/entities/transaction_classification.rs + +//! Transaction classification entity. + +/// Persisted transaction classification row. +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, sqlx::FromRow)] +pub struct TransactionClassificationEntity { + /// Numeric primary key. + pub id: i64, + /// Related chain transaction id. + pub transaction_id: i64, + /// Transaction signature. + pub signature: std::string::String, + /// Optional Solana slot. + pub slot: std::option::Option, + /// Stable classification kind. + pub classification_kind: std::string::String, + /// Optional primary protocol name. + pub primary_protocol: std::option::Option, + /// Optional primary program id. + pub primary_program_id: std::option::Option, + /// Confidence level from 0 to 100. + pub confidence_level: i64, + /// Human-readable reason. + pub reason: std::string::String, + /// Serialized JSON evidence. + pub evidence_json: std::string::String, + /// Creation timestamp encoded as RFC3339 UTC text. + pub created_at: std::string::String, + /// Update timestamp encoded as RFC3339 UTC text. + pub updated_at: std::string::String, +} diff --git a/kb_lib/src/db/queries.rs b/kb_lib/src/db/queries.rs index d5e054f..557d3df 100644 --- a/kb_lib/src/db/queries.rs +++ b/kb_lib/src/db/queries.rs @@ -27,11 +27,13 @@ mod pool; mod pool_listing; mod pool_origin; mod pool_token; +mod protocol_candidate; mod swap; mod token; mod token_burn_event; mod token_mint_event; mod trade_event; +mod transaction_classification; mod wallet; mod wallet_holding; mod wallet_participation; @@ -118,6 +120,12 @@ pub use pool_origin::query_pool_origins_list; pub use pool_origin::query_pool_origins_upsert; pub use pool_token::query_pool_tokens_list_by_pool_id; pub use pool_token::query_pool_tokens_upsert; +pub use protocol_candidate::query_protocol_candidate_summaries_list_by_priority; +pub use protocol_candidate::query_protocol_candidates_delete_by_transaction_id; +pub use protocol_candidate::query_protocol_candidates_insert; +pub use protocol_candidate::query_protocol_candidates_list_by_program_id; +pub use protocol_candidate::query_protocol_candidates_list_by_transaction_id; +pub use protocol_candidate::query_protocol_candidates_list_recent; pub use swap::query_swaps_list_recent; pub use swap::query_swaps_upsert; pub use token::query_tokens_get_by_id; @@ -133,6 +141,10 @@ pub use trade_event::query_trade_events_get_by_decoded_event_id; pub use trade_event::query_trade_events_list_by_pair_id; pub use trade_event::query_trade_events_list_by_transaction_id; pub use trade_event::query_trade_events_upsert; +pub use transaction_classification::query_transaction_classifications_get_by_signature; +pub use transaction_classification::query_transaction_classifications_get_by_transaction_id; +pub use transaction_classification::query_transaction_classifications_list_recent; +pub use transaction_classification::query_transaction_classifications_upsert; pub use wallet::query_wallets_get_by_address; pub use wallet::query_wallets_list; pub use wallet::query_wallets_upsert; diff --git a/kb_lib/src/db/queries/protocol_candidate.rs b/kb_lib/src/db/queries/protocol_candidate.rs new file mode 100644 index 0000000..b62afa6 --- /dev/null +++ b/kb_lib/src/db/queries/protocol_candidate.rs @@ -0,0 +1,337 @@ +// file: kb_lib/src/db/queries/protocol_candidate.rs + +//! Queries for `k_sol_protocol_candidates`. + +/// Inserts one protocol candidate row. +pub async fn query_protocol_candidates_insert( + database: &crate::Database, + dto: &crate::ProtocolCandidateDto, +) -> Result { + let slot_i64 = match dto.slot { + Some(slot) => { + let slot_result = i64::try_from(slot); + match slot_result { + Ok(slot) => Some(slot), + Err(error) => { + return Err(crate::Error::Db(format!( + "cannot convert protocol candidate slot '{}' to i64: {}", + slot, error + ))); + }, + } + }, + None => None, + }; + match database.connection() { + crate::DatabaseConnection::Sqlite(pool) => { + let query_result = sqlx::query( + r#" +INSERT INTO k_sol_protocol_candidates ( + transaction_id, + instruction_id, + signature, + slot, + program_id, + program_name_hint, + candidate_protocol, + candidate_surface, + reason, + evidence_json, + created_at +) +VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + "#, + ) + .bind(dto.transaction_id) + .bind(dto.instruction_id) + .bind(dto.signature.clone()) + .bind(slot_i64) + .bind(dto.program_id.clone()) + .bind(dto.program_name_hint.clone()) + .bind(dto.candidate_protocol.clone()) + .bind(dto.candidate_surface.clone()) + .bind(dto.reason.clone()) + .bind(dto.evidence_json.clone()) + .bind(dto.created_at.to_rfc3339()) + .execute(pool) + .await; + match query_result { + Ok(query_result) => return Ok(query_result.last_insert_rowid()), + Err(error) => { + return Err(crate::Error::Db(format!( + "cannot insert k_sol_protocol_candidates on sqlite: {}", + error + ))); + }, + } + }, + } +} + +/// Deletes protocol candidates for one transaction. +/// +/// This is useful before recomputing candidates for a replayed transaction. +pub async fn query_protocol_candidates_delete_by_transaction_id( + database: &crate::Database, + transaction_id: i64, +) -> Result { + match database.connection() { + crate::DatabaseConnection::Sqlite(pool) => { + let query_result = sqlx::query( + r#" +DELETE FROM k_sol_protocol_candidates +WHERE transaction_id = ? + "#, + ) + .bind(transaction_id) + .execute(pool) + .await; + match query_result { + Ok(query_result) => return Ok(query_result.rows_affected()), + Err(error) => { + return Err(crate::Error::Db(format!( + "cannot delete k_sol_protocol_candidates for transaction_id '{}' on sqlite: {}", + transaction_id, error + ))); + }, + } + }, + } +} + +/// Lists protocol candidates for one transaction. +pub async fn query_protocol_candidates_list_by_transaction_id( + database: &crate::Database, + transaction_id: i64, +) -> Result, crate::Error> { + match database.connection() { + crate::DatabaseConnection::Sqlite(pool) => { + let query_result = sqlx::query_as::( + r#" +SELECT + id, + transaction_id, + instruction_id, + signature, + slot, + program_id, + program_name_hint, + candidate_protocol, + candidate_surface, + reason, + evidence_json, + created_at +FROM k_sol_protocol_candidates +WHERE transaction_id = ? +ORDER BY id ASC + "#, + ) + .bind(transaction_id) + .fetch_all(pool) + .await; + let entities = match query_result { + Ok(entities) => entities, + Err(error) => { + return Err(crate::Error::Db(format!( + "cannot list k_sol_protocol_candidates for transaction_id '{}' on sqlite: {}", + transaction_id, error + ))); + }, + }; + return protocol_candidate_entities_to_dtos(entities); + }, + } +} + +/// Lists protocol candidates for one program id. +pub async fn query_protocol_candidates_list_by_program_id( + database: &crate::Database, + program_id: &str, + limit: u32, +) -> Result, crate::Error> { + if limit == 0 { + return Ok(std::vec::Vec::new()); + } + match database.connection() { + crate::DatabaseConnection::Sqlite(pool) => { + let query_result = sqlx::query_as::( + r#" +SELECT + id, + transaction_id, + instruction_id, + signature, + slot, + program_id, + program_name_hint, + candidate_protocol, + candidate_surface, + reason, + evidence_json, + created_at +FROM k_sol_protocol_candidates +WHERE program_id = ? +ORDER BY id DESC +LIMIT ? + "#, + ) + .bind(program_id.to_string()) + .bind(i64::from(limit)) + .fetch_all(pool) + .await; + let entities = match query_result { + Ok(entities) => entities, + Err(error) => { + return Err(crate::Error::Db(format!( + "cannot list k_sol_protocol_candidates for program_id '{}' on sqlite: {}", + program_id, error + ))); + }, + }; + return protocol_candidate_entities_to_dtos(entities); + }, + } +} + +/// Lists recent protocol candidates ordered from newest to oldest. +pub async fn query_protocol_candidates_list_recent( + database: &crate::Database, + limit: u32, +) -> Result, crate::Error> { + if limit == 0 { + return Ok(std::vec::Vec::new()); + } + match database.connection() { + crate::DatabaseConnection::Sqlite(pool) => { + let query_result = sqlx::query_as::( + r#" +SELECT + id, + transaction_id, + instruction_id, + signature, + slot, + program_id, + program_name_hint, + candidate_protocol, + candidate_surface, + reason, + evidence_json, + created_at +FROM k_sol_protocol_candidates +ORDER BY id DESC +LIMIT ? + "#, + ) + .bind(i64::from(limit)) + .fetch_all(pool) + .await; + let entities = match query_result { + Ok(entities) => entities, + Err(error) => { + return Err(crate::Error::Db(format!( + "cannot list recent k_sol_protocol_candidates on sqlite: {}", + error + ))); + }, + }; + return protocol_candidate_entities_to_dtos(entities); + }, + } +} + +fn protocol_candidate_entities_to_dtos( + entities: std::vec::Vec, +) -> Result, crate::Error> { + let mut dtos = std::vec::Vec::new(); + for entity in entities { + let dto_result = crate::ProtocolCandidateDto::try_from(entity); + let dto = match dto_result { + Ok(dto) => dto, + Err(error) => return Err(error), + }; + dtos.push(dto); + } + return Ok(dtos); +} + +/// Lists protocol candidate summaries ordered by investigation priority. +pub async fn query_protocol_candidate_summaries_list_by_priority( + database: &crate::Database, + limit: u32, +) -> Result, crate::Error> { + if limit == 0 { + return Ok(std::vec::Vec::new()); + } + match database.connection() { + crate::DatabaseConnection::Sqlite(pool) => { + let query_result = + sqlx::query_as::( + r#" +WITH grouped AS ( + SELECT + program_id, + program_name_hint, + candidate_protocol, + candidate_surface, + reason, + COUNT(*) AS occurrence_count, + COUNT(DISTINCT signature) AS transaction_count, + MAX(slot) AS last_slot, + MAX(id) AS latest_candidate_id + FROM k_sol_protocol_candidates + GROUP BY + program_id, + program_name_hint, + candidate_protocol, + candidate_surface, + reason +) +SELECT + grouped.program_id, + grouped.program_name_hint, + grouped.candidate_protocol, + grouped.candidate_surface, + grouped.reason, + grouped.occurrence_count, + grouped.transaction_count, + grouped.last_slot, + grouped.latest_candidate_id, + latest.signature AS latest_signature, + latest.created_at AS latest_created_at +FROM grouped +JOIN k_sol_protocol_candidates latest + ON latest.id = grouped.latest_candidate_id +ORDER BY + grouped.transaction_count DESC, + grouped.occurrence_count DESC, + grouped.last_slot DESC, + grouped.latest_candidate_id DESC +LIMIT ? + "#, + ) + .bind(i64::from(limit)) + .fetch_all(pool) + .await; + let entities = match query_result { + Ok(entities) => entities, + Err(error) => { + return Err(crate::Error::Db(format!( + "cannot list k_sol_protocol_candidates summaries on sqlite: {}", + error + ))); + }, + }; + let mut dtos = std::vec::Vec::new(); + for entity in entities { + let dto_result = crate::ProtocolCandidateSummaryDto::try_from(entity); + let dto = match dto_result { + Ok(dto) => dto, + Err(error) => return Err(error), + }; + dtos.push(dto); + } + return Ok(dtos); + }, + } +} diff --git a/kb_lib/src/db/queries/transaction_classification.rs b/kb_lib/src/db/queries/transaction_classification.rs new file mode 100644 index 0000000..5e8b3df --- /dev/null +++ b/kb_lib/src/db/queries/transaction_classification.rs @@ -0,0 +1,263 @@ +// file: kb_lib/src/db/queries/transaction_classification.rs + +//! Queries for `k_sol_transaction_classifications`. + +/// Inserts or updates one transaction classification row. +pub async fn query_transaction_classifications_upsert( + database: &crate::Database, + dto: &crate::TransactionClassificationDto, +) -> Result { + let slot_i64 = match dto.slot { + Some(slot) => { + let slot_result = i64::try_from(slot); + match slot_result { + Ok(slot) => Some(slot), + Err(error) => { + return Err(crate::Error::Db(format!( + "cannot convert transaction classification slot '{}' to i64: {}", + slot, error + ))); + }, + } + }, + None => None, + }; + match database.connection() { + crate::DatabaseConnection::Sqlite(pool) => { + let query_result = sqlx::query( + r#" +INSERT INTO k_sol_transaction_classifications ( + transaction_id, + signature, + slot, + classification_kind, + primary_protocol, + primary_program_id, + confidence_level, + reason, + evidence_json, + created_at, + updated_at +) +VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) +ON CONFLICT(transaction_id) DO UPDATE SET + signature = excluded.signature, + slot = excluded.slot, + classification_kind = excluded.classification_kind, + primary_protocol = excluded.primary_protocol, + primary_program_id = excluded.primary_program_id, + confidence_level = excluded.confidence_level, + reason = excluded.reason, + evidence_json = excluded.evidence_json, + updated_at = excluded.updated_at + "#, + ) + .bind(dto.transaction_id) + .bind(dto.signature.clone()) + .bind(slot_i64) + .bind(dto.classification_kind.clone()) + .bind(dto.primary_protocol.clone()) + .bind(dto.primary_program_id.clone()) + .bind(i64::from(dto.confidence_level)) + .bind(dto.reason.clone()) + .bind(dto.evidence_json.clone()) + .bind(dto.created_at.to_rfc3339()) + .bind(dto.updated_at.to_rfc3339()) + .execute(pool) + .await; + if let Err(error) = query_result { + return Err(crate::Error::Db(format!( + "cannot upsert k_sol_transaction_classifications on sqlite: {}", + error + ))); + } + + let id_result = sqlx::query_scalar::( + r#" +SELECT id +FROM k_sol_transaction_classifications +WHERE transaction_id = ? +LIMIT 1 + "#, + ) + .bind(dto.transaction_id) + .fetch_one(pool) + .await; + match id_result { + Ok(id) => return Ok(id), + Err(error) => { + return Err(crate::Error::Db(format!( + "cannot fetch k_sol_transaction_classifications id for transaction_id '{}' on sqlite: {}", + dto.transaction_id, error + ))); + }, + } + }, + } +} + +/// Reads one transaction classification by transaction id. +pub async fn query_transaction_classifications_get_by_transaction_id( + database: &crate::Database, + transaction_id: i64, +) -> Result, crate::Error> { + match database.connection() { + crate::DatabaseConnection::Sqlite(pool) => { + let query_result = + sqlx::query_as::( + r#" +SELECT + id, + transaction_id, + signature, + slot, + classification_kind, + primary_protocol, + primary_program_id, + confidence_level, + reason, + evidence_json, + created_at, + updated_at +FROM k_sol_transaction_classifications +WHERE transaction_id = ? +LIMIT 1 + "#, + ) + .bind(transaction_id) + .fetch_optional(pool) + .await; + let entity_option = match query_result { + Ok(entity_option) => entity_option, + Err(error) => { + return Err(crate::Error::Db(format!( + "cannot fetch k_sol_transaction_classifications for transaction_id '{}' on sqlite: {}", + transaction_id, error + ))); + }, + }; + match entity_option { + Some(entity) => { + let dto_result = crate::TransactionClassificationDto::try_from(entity); + match dto_result { + Ok(dto) => return Ok(Some(dto)), + Err(error) => return Err(error), + } + }, + None => return Ok(None), + } + }, + } +} + +/// Reads one transaction classification by signature. +pub async fn query_transaction_classifications_get_by_signature( + database: &crate::Database, + signature: &str, +) -> Result, crate::Error> { + match database.connection() { + crate::DatabaseConnection::Sqlite(pool) => { + let query_result = + sqlx::query_as::( + r#" +SELECT + id, + transaction_id, + signature, + slot, + classification_kind, + primary_protocol, + primary_program_id, + confidence_level, + reason, + evidence_json, + created_at, + updated_at +FROM k_sol_transaction_classifications +WHERE signature = ? +LIMIT 1 + "#, + ) + .bind(signature.to_string()) + .fetch_optional(pool) + .await; + let entity_option = match query_result { + Ok(entity_option) => entity_option, + Err(error) => { + return Err(crate::Error::Db(format!( + "cannot fetch k_sol_transaction_classifications for signature '{}' on sqlite: {}", + signature, error + ))); + }, + }; + match entity_option { + Some(entity) => { + let dto_result = crate::TransactionClassificationDto::try_from(entity); + match dto_result { + Ok(dto) => return Ok(Some(dto)), + Err(error) => return Err(error), + } + }, + None => return Ok(None), + } + }, + } +} + +/// Lists recent transaction classifications ordered from newest to oldest. +pub async fn query_transaction_classifications_list_recent( + database: &crate::Database, + limit: u32, +) -> Result, crate::Error> { + if limit == 0 { + return Ok(std::vec::Vec::new()); + } + match database.connection() { + crate::DatabaseConnection::Sqlite(pool) => { + let query_result = + sqlx::query_as::( + r#" +SELECT + id, + transaction_id, + signature, + slot, + classification_kind, + primary_protocol, + primary_program_id, + confidence_level, + reason, + evidence_json, + created_at, + updated_at +FROM k_sol_transaction_classifications +ORDER BY id DESC +LIMIT ? + "#, + ) + .bind(i64::from(limit)) + .fetch_all(pool) + .await; + let entities = match query_result { + Ok(entities) => entities, + Err(error) => { + return Err(crate::Error::Db(format!( + "cannot list k_sol_transaction_classifications on sqlite: {}", + error + ))); + }, + }; + let mut dtos = std::vec::Vec::new(); + for entity in entities { + let dto_result = crate::TransactionClassificationDto::try_from(entity); + let dto = match dto_result { + Ok(dto) => dto, + Err(error) => return Err(error), + }; + + dtos.push(dto); + } + return Ok(dtos); + }, + } +} diff --git a/kb_lib/src/db/schema.rs b/kb_lib/src/db/schema.rs index 84efdb1..4d5f3bc 100644 --- a/kb_lib/src/db/schema.rs +++ b/kb_lib/src/db/schema.rs @@ -230,6 +230,94 @@ pub(crate) async fn ensure_schema(database: &crate::Database) -> Result<(), crat if let Err(error) = result { return Err(error); } + let result = create_tbl_transaction_classifications(pool).await; + if let Err(error) = result { + return Err(error); + } + let result = create_uix_transaction_classifications_transaction_id(pool).await; + if let Err(error) = result { + return Err(error); + } + let result = create_idx_transaction_classifications_kind(pool).await; + if let Err(error) = result { + return Err(error); + } + let result = create_tbl_protocol_candidates(pool).await; + if let Err(error) = result { + return Err(error); + } + let result = create_idx_protocol_candidates_transaction_id(pool).await; + if let Err(error) = result { + return Err(error); + } + let result = create_idx_protocol_candidates_program_id(pool).await; + if let Err(error) = result { + return Err(error); + } + let result = create_tbl_pool_lifecycle_events(pool).await; + if let Err(error) = result { + return Err(error); + } + let result = create_idx_pool_lifecycle_events_transaction_id(pool).await; + if let Err(error) = result { + return Err(error); + } + let result = create_idx_pool_lifecycle_events_pool_id(pool).await; + if let Err(error) = result { + return Err(error); + } + let result = create_uix_pool_lifecycle_events_decoded_event_id(pool).await; + if let Err(error) = result { + return Err(error); + } + let result = create_tbl_fee_events(pool).await; + if let Err(error) = result { + return Err(error); + } + let result = create_idx_fee_events_transaction_id(pool).await; + if let Err(error) = result { + return Err(error); + } + let result = create_idx_fee_events_pool_id(pool).await; + if let Err(error) = result { + return Err(error); + } + let result = create_uix_fee_events_decoded_event_id(pool).await; + if let Err(error) = result { + return Err(error); + } + let result = create_tbl_reward_events(pool).await; + if let Err(error) = result { + return Err(error); + } + let result = create_idx_reward_events_transaction_id(pool).await; + if let Err(error) = result { + return Err(error); + } + let result = create_idx_reward_events_pool_id(pool).await; + if let Err(error) = result { + return Err(error); + } + let result = create_uix_reward_events_decoded_event_id(pool).await; + if let Err(error) = result { + return Err(error); + } + let result = create_tbl_pool_admin_events(pool).await; + if let Err(error) = result { + return Err(error); + } + let result = create_idx_pool_admin_events_transaction_id(pool).await; + if let Err(error) = result { + return Err(error); + } + let result = create_idx_pool_admin_events_pool_id(pool).await; + if let Err(error) = result { + return Err(error); + } + let result = create_uix_pool_admin_events_decoded_event_id(pool).await; + if let Err(error) = result { + return Err(error); + } let result = create_tbl_launch_surfaces(pool).await; if let Err(error) = result { return Err(error); @@ -1878,3 +1966,413 @@ ON k_sol_pair_analytic_signals(pair_id) ) .await; } + +/// Creates `k_sol_transaction_classifications`. +async fn create_tbl_transaction_classifications( + pool: &sqlx::SqlitePool, +) -> Result<(), crate::Error> { + return execute_sqlite_schema_statement( + pool, + "create_tbl_transaction_classifications", + r#" +CREATE TABLE IF NOT EXISTS k_sol_transaction_classifications ( + id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, + transaction_id INTEGER NOT NULL, + signature TEXT NOT NULL, + slot INTEGER NULL, + classification_kind TEXT NOT NULL, + primary_protocol TEXT NULL, + primary_program_id TEXT NULL, + confidence_level INTEGER NOT NULL, + reason TEXT NOT NULL, + evidence_json TEXT NOT NULL, + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL +) + "#, + ) + .await; +} + +/// Creates unique index on `k_sol_transaction_classifications(transaction_id)`. +async fn create_uix_transaction_classifications_transaction_id( + pool: &sqlx::SqlitePool, +) -> Result<(), crate::Error> { + return execute_sqlite_schema_statement( + pool, + "create_uix_transaction_classifications_transaction_id", + r#" +CREATE UNIQUE INDEX IF NOT EXISTS uix_transaction_classifications_transaction_id +ON k_sol_transaction_classifications (transaction_id) + "#, + ) + .await; +} + +/// Creates index on `k_sol_transaction_classifications(classification_kind)`. +async fn create_idx_transaction_classifications_kind( + pool: &sqlx::SqlitePool, +) -> Result<(), crate::Error> { + return execute_sqlite_schema_statement( + pool, + "create_idx_transaction_classifications_kind", + r#" +CREATE INDEX IF NOT EXISTS idx_transaction_classifications_kind +ON k_sol_transaction_classifications (classification_kind) + "#, + ) + .await; +} + +/// Creates `k_sol_protocol_candidates`. +async fn create_tbl_protocol_candidates(pool: &sqlx::SqlitePool) -> Result<(), crate::Error> { + return execute_sqlite_schema_statement( + pool, + "create_tbl_protocol_candidates", + r#" +CREATE TABLE IF NOT EXISTS k_sol_protocol_candidates ( + id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, + transaction_id INTEGER NOT NULL, + instruction_id INTEGER NULL, + signature TEXT NOT NULL, + slot INTEGER NULL, + program_id TEXT NOT NULL, + program_name_hint TEXT NULL, + candidate_protocol TEXT NULL, + candidate_surface TEXT NULL, + reason TEXT NOT NULL, + evidence_json TEXT NOT NULL, + created_at TEXT NOT NULL +) + "#, + ) + .await; +} + +/// Creates index on `k_sol_protocol_candidates(transaction_id)`. +async fn create_idx_protocol_candidates_transaction_id( + pool: &sqlx::SqlitePool, +) -> Result<(), crate::Error> { + return execute_sqlite_schema_statement( + pool, + "create_idx_protocol_candidates_transaction_id", + r#" +CREATE INDEX IF NOT EXISTS idx_protocol_candidates_transaction_id +ON k_sol_protocol_candidates (transaction_id) + "#, + ) + .await; +} + +/// Creates index on `k_sol_protocol_candidates(program_id)`. +async fn create_idx_protocol_candidates_program_id( + pool: &sqlx::SqlitePool, +) -> Result<(), crate::Error> { + return execute_sqlite_schema_statement( + pool, + "create_idx_protocol_candidates_program_id", + r#" +CREATE INDEX IF NOT EXISTS idx_protocol_candidates_program_id +ON k_sol_protocol_candidates (program_id) + "#, + ) + .await; +} + +/// Creates `k_sol_pool_lifecycle_events`. +async fn create_tbl_pool_lifecycle_events(pool: &sqlx::SqlitePool) -> Result<(), crate::Error> { + return execute_sqlite_schema_statement( + pool, + "create_tbl_pool_lifecycle_events", + r#" +CREATE TABLE IF NOT EXISTS k_sol_pool_lifecycle_events ( + id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, + transaction_id INTEGER NOT NULL, + decoded_event_id INTEGER NULL, + dex_id INTEGER NULL, + pool_id INTEGER NULL, + pair_id INTEGER NULL, + signature TEXT NOT NULL, + slot INTEGER NULL, + protocol_name TEXT NOT NULL, + program_id TEXT NOT NULL, + event_kind TEXT NOT NULL, + pool_account TEXT NULL, + token_a_mint TEXT NULL, + token_b_mint TEXT NULL, + payload_json TEXT NOT NULL, + executed_at TEXT NOT NULL, + created_at TEXT NOT NULL +) + "#, + ) + .await; +} + +/// Creates index on `k_sol_pool_lifecycle_events(transaction_id)`. +async fn create_idx_pool_lifecycle_events_transaction_id( + pool: &sqlx::SqlitePool, +) -> Result<(), crate::Error> { + return execute_sqlite_schema_statement( + pool, + "create_idx_pool_lifecycle_events_transaction_id", + r#" +CREATE INDEX IF NOT EXISTS idx_pool_lifecycle_events_transaction_id +ON k_sol_pool_lifecycle_events (transaction_id) + "#, + ) + .await; +} + +/// Creates index on `k_sol_pool_lifecycle_events(pool_id)`. +async fn create_idx_pool_lifecycle_events_pool_id( + pool: &sqlx::SqlitePool, +) -> Result<(), crate::Error> { + return execute_sqlite_schema_statement( + pool, + "create_idx_pool_lifecycle_events_pool_id", + r#" +CREATE INDEX IF NOT EXISTS idx_pool_lifecycle_events_pool_id +ON k_sol_pool_lifecycle_events (pool_id) + "#, + ) + .await; +} + +/// Creates partial unique index on `k_sol_pool_lifecycle_events(decoded_event_id)`. +async fn create_uix_pool_lifecycle_events_decoded_event_id( + pool: &sqlx::SqlitePool, +) -> Result<(), crate::Error> { + return execute_sqlite_schema_statement( + pool, + "create_uix_pool_lifecycle_events_decoded_event_id", + r#" +CREATE UNIQUE INDEX IF NOT EXISTS uix_pool_lifecycle_events_decoded_event_id +ON k_sol_pool_lifecycle_events (decoded_event_id) +WHERE decoded_event_id IS NOT NULL + "#, + ) + .await; +} + +/// Creates `k_sol_fee_events`. +async fn create_tbl_fee_events(pool: &sqlx::SqlitePool) -> Result<(), crate::Error> { + return execute_sqlite_schema_statement( + pool, + "create_tbl_fee_events", + r#" +CREATE TABLE IF NOT EXISTS k_sol_fee_events ( + id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, + transaction_id INTEGER NOT NULL, + decoded_event_id INTEGER NULL, + dex_id INTEGER NULL, + pool_id INTEGER NULL, + pair_id INTEGER NULL, + signature TEXT NOT NULL, + slot INTEGER NULL, + protocol_name TEXT NOT NULL, + program_id TEXT NOT NULL, + event_kind TEXT NOT NULL, + pool_account TEXT NULL, + actor_wallet TEXT NULL, + fee_token_mint TEXT NULL, + fee_amount_raw TEXT NULL, + payload_json TEXT NOT NULL, + executed_at TEXT NOT NULL, + created_at TEXT NOT NULL +) + "#, + ) + .await; +} + +/// Creates index on `k_sol_fee_events(transaction_id)`. +async fn create_idx_fee_events_transaction_id(pool: &sqlx::SqlitePool) -> Result<(), crate::Error> { + return execute_sqlite_schema_statement( + pool, + "create_idx_fee_events_transaction_id", + r#" +CREATE INDEX IF NOT EXISTS idx_fee_events_transaction_id +ON k_sol_fee_events (transaction_id) + "#, + ) + .await; +} + +/// Creates index on `k_sol_fee_events(pool_id)`. +async fn create_idx_fee_events_pool_id(pool: &sqlx::SqlitePool) -> Result<(), crate::Error> { + return execute_sqlite_schema_statement( + pool, + "create_idx_fee_events_pool_id", + r#" +CREATE INDEX IF NOT EXISTS idx_fee_events_pool_id +ON k_sol_fee_events (pool_id) + "#, + ) + .await; +} + +/// Creates partial unique index on `k_sol_fee_events(decoded_event_id)`. +async fn create_uix_fee_events_decoded_event_id( + pool: &sqlx::SqlitePool, +) -> Result<(), crate::Error> { + return execute_sqlite_schema_statement( + pool, + "create_uix_fee_events_decoded_event_id", + r#" +CREATE UNIQUE INDEX IF NOT EXISTS uix_fee_events_decoded_event_id +ON k_sol_fee_events (decoded_event_id) +WHERE decoded_event_id IS NOT NULL + "#, + ) + .await; +} + +/// Creates `k_sol_reward_events`. +async fn create_tbl_reward_events(pool: &sqlx::SqlitePool) -> Result<(), crate::Error> { + return execute_sqlite_schema_statement( + pool, + "create_tbl_reward_events", + r#" +CREATE TABLE IF NOT EXISTS k_sol_reward_events ( + id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, + transaction_id INTEGER NOT NULL, + decoded_event_id INTEGER NULL, + dex_id INTEGER NULL, + pool_id INTEGER NULL, + pair_id INTEGER NULL, + signature TEXT NOT NULL, + slot INTEGER NULL, + protocol_name TEXT NOT NULL, + program_id TEXT NOT NULL, + event_kind TEXT NOT NULL, + pool_account TEXT NULL, + actor_wallet TEXT NULL, + reward_token_mint TEXT NULL, + reward_amount_raw TEXT NULL, + payload_json TEXT NOT NULL, + executed_at TEXT NOT NULL, + created_at TEXT NOT NULL +) + "#, + ) + .await; +} + +/// Creates index on `k_sol_reward_events(transaction_id)`. +async fn create_idx_reward_events_transaction_id( + pool: &sqlx::SqlitePool, +) -> Result<(), crate::Error> { + return execute_sqlite_schema_statement( + pool, + "create_idx_reward_events_transaction_id", + r#" +CREATE INDEX IF NOT EXISTS idx_reward_events_transaction_id +ON k_sol_reward_events (transaction_id) + "#, + ) + .await; +} + +/// Creates index on `k_sol_reward_events(pool_id)`. +async fn create_idx_reward_events_pool_id(pool: &sqlx::SqlitePool) -> Result<(), crate::Error> { + return execute_sqlite_schema_statement( + pool, + "create_idx_reward_events_pool_id", + r#" +CREATE INDEX IF NOT EXISTS idx_reward_events_pool_id +ON k_sol_reward_events (pool_id) + "#, + ) + .await; +} + +/// Creates partial unique index on `k_sol_reward_events(decoded_event_id)`. +async fn create_uix_reward_events_decoded_event_id( + pool: &sqlx::SqlitePool, +) -> Result<(), crate::Error> { + return execute_sqlite_schema_statement( + pool, + "create_uix_reward_events_decoded_event_id", + r#" +CREATE UNIQUE INDEX IF NOT EXISTS uix_reward_events_decoded_event_id +ON k_sol_reward_events (decoded_event_id) +WHERE decoded_event_id IS NOT NULL + "#, + ) + .await; +} + +/// Creates `k_sol_pool_admin_events`. +async fn create_tbl_pool_admin_events(pool: &sqlx::SqlitePool) -> Result<(), crate::Error> { + return execute_sqlite_schema_statement( + pool, + "create_tbl_pool_admin_events", + r#" +CREATE TABLE IF NOT EXISTS k_sol_pool_admin_events ( + id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, + transaction_id INTEGER NOT NULL, + decoded_event_id INTEGER NULL, + dex_id INTEGER NULL, + pool_id INTEGER NULL, + pair_id INTEGER NULL, + signature TEXT NOT NULL, + slot INTEGER NULL, + protocol_name TEXT NOT NULL, + program_id TEXT NOT NULL, + event_kind TEXT NOT NULL, + pool_account TEXT NULL, + actor_wallet TEXT NULL, + admin_action TEXT NULL, + payload_json TEXT NOT NULL, + executed_at TEXT NOT NULL, + created_at TEXT NOT NULL +) + "#, + ) + .await; +} + +/// Creates index on `k_sol_pool_admin_events(transaction_id)`. +async fn create_idx_pool_admin_events_transaction_id( + pool: &sqlx::SqlitePool, +) -> Result<(), crate::Error> { + return execute_sqlite_schema_statement( + pool, + "create_idx_pool_admin_events_transaction_id", + r#" +CREATE INDEX IF NOT EXISTS idx_pool_admin_events_transaction_id +ON k_sol_pool_admin_events (transaction_id) + "#, + ) + .await; +} + +/// Creates index on `k_sol_pool_admin_events(pool_id)`. +async fn create_idx_pool_admin_events_pool_id(pool: &sqlx::SqlitePool) -> Result<(), crate::Error> { + return execute_sqlite_schema_statement( + pool, + "create_idx_pool_admin_events_pool_id", + r#" +CREATE INDEX IF NOT EXISTS idx_pool_admin_events_pool_id +ON k_sol_pool_admin_events (pool_id) + "#, + ) + .await; +} + +/// Creates partial unique index on `k_sol_pool_admin_events(decoded_event_id)`. +async fn create_uix_pool_admin_events_decoded_event_id( + pool: &sqlx::SqlitePool, +) -> Result<(), crate::Error> { + return execute_sqlite_schema_statement( + pool, + "create_uix_pool_admin_events_decoded_event_id", + r#" +CREATE UNIQUE INDEX IF NOT EXISTS uix_pool_admin_events_decoded_event_id +ON k_sol_pool_admin_events (decoded_event_id) +WHERE decoded_event_id IS NOT NULL + "#, + ) + .await; +} diff --git a/kb_lib/src/detect/solana_ws.rs b/kb_lib/src/detect/solana_ws.rs index 69a9364..643e587 100644 --- a/kb_lib/src/detect/solana_ws.rs +++ b/kb_lib/src/detect/solana_ws.rs @@ -155,8 +155,8 @@ impl SolanaWsDetectionService { Some(token_program) => token_program, None => return Ok(None), }; - if token_program != crate::SPL_TOKEN_PROGRAM_ID.to_string() - && token_program != crate::SPL_TOKEN_2022_PROGRAM_ID.to_string() + if token_program.as_str() != crate::SPL_TOKEN_PROGRAM_ID + && token_program.as_str() != crate::SPL_TOKEN_2022_PROGRAM_ID { return Ok(None); } @@ -181,7 +181,7 @@ impl SolanaWsDetectionService { let slot = extract_slot_from_result(notification.method.as_str(), ¬ification.params.result); let payload = build_notification_payload(notification); - let is_quote_token = mint == crate::WSOL_MINT_ID.to_string(); + let is_quote_token = mint.as_str() == crate::WSOL_MINT_ID; let input = crate::DetectionTokenCandidateInput::new( mint, None, @@ -230,8 +230,8 @@ impl SolanaWsDetectionService { Some(owner) => owner, None => return Ok(None), }; - if owner == crate::SPL_TOKEN_PROGRAM_ID.to_string() - || owner == crate::SPL_TOKEN_2022_PROGRAM_ID.to_string() + if owner.as_str() == crate::SPL_TOKEN_PROGRAM_ID + || owner.as_str() == crate::SPL_TOKEN_2022_PROGRAM_ID { return Ok(None); } @@ -603,10 +603,10 @@ fn build_signal_kind_for_notification( } let owner_option = extract_account_owner(account_value); if let Some(owner) = owner_option { - if owner == crate::SPL_TOKEN_PROGRAM_ID.to_string() { + if owner.as_str() == crate::SPL_TOKEN_PROGRAM_ID { return "signal.account_notification.spl_token".to_string(); } - if owner == crate::SPL_TOKEN_2022_PROGRAM_ID.to_string() { + if owner.as_str() == crate::SPL_TOKEN_2022_PROGRAM_ID { return "signal.account_notification.spl_token_2022".to_string(); } } @@ -650,10 +650,10 @@ fn build_signal_kind_for_notification( Some(owner) => owner, None => return "signal.program_notification.generic".to_string(), }; - if owner == crate::SPL_TOKEN_PROGRAM_ID.to_string() { + if owner.as_str() == crate::SPL_TOKEN_PROGRAM_ID { return "signal.program_notification.spl_token".to_string(); } - if owner == crate::SPL_TOKEN_2022_PROGRAM_ID.to_string() { + if owner.as_str() == crate::SPL_TOKEN_2022_PROGRAM_ID { return "signal.program_notification.spl_token_2022".to_string(); } return "signal.program_notification.generic".to_string(); diff --git a/kb_lib/src/dex_catalog.rs b/kb_lib/src/dex_catalog.rs new file mode 100644 index 0000000..ee204e7 --- /dev/null +++ b/kb_lib/src/dex_catalog.rs @@ -0,0 +1,295 @@ +// file: kb_lib/src/dex_catalog.rs + +//! Internal DEX catalog and persistence helpers. +//! +//! This module centralizes known DEX metadata used by detection, +//! decoding and future launch-surface integrations. +//! +//! It does not decode instructions and does not classify events. + +/// Static metadata for one known DEX entry. +#[derive(Debug, Copy, Clone, Eq, PartialEq)] +pub(crate) struct DexCatalogItem { + /// Stable internal DEX code. + pub(crate) code: &'static str, + /// Human-readable DEX name. + pub(crate) name: &'static str, + /// Main Solana program id. + pub(crate) program_id: std::option::Option<&'static str>, + /// Optional router program id. + pub(crate) router_program_id: std::option::Option<&'static str>, + /// Whether this DEX is currently enabled for detection. + pub(crate) is_enabled: bool, +} + +/// Returns metadata for one known DEX or launch-backed swap surface. +pub(crate) fn dex_catalog_item_by_code( + code: &str, +) -> std::option::Option { + match code { + "raydium" => { + return Some(crate::dex_catalog::DexCatalogItem { + code: "raydium", + name: "Raydium AMM v4", + program_id: Some(crate::RAYDIUM_AMM_V4_PROGRAM_ID), + router_program_id: None, + is_enabled: true, + }); + }, + "raydium_cpmm" => { + return Some(crate::dex_catalog::DexCatalogItem { + code: "raydium_cpmm", + name: "Raydium CPMM", + program_id: Some(crate::RAYDIUM_CPMM_PROGRAM_ID), + router_program_id: None, + is_enabled: true, + }); + }, + "raydium_clmm" => { + return Some(crate::dex_catalog::DexCatalogItem { + code: "raydium_clmm", + name: "Raydium CLMM", + program_id: Some(crate::RAYDIUM_CLMM_PROGRAM_ID), + router_program_id: None, + is_enabled: true, + }); + }, + "pump_fun" => { + return Some(crate::dex_catalog::DexCatalogItem { + code: "pump_fun", + name: "Pump.fun", + program_id: Some(crate::PUMP_FUN_PROGRAM_ID), + router_program_id: None, + is_enabled: true, + }); + }, + "pump_swap" => { + return Some(crate::dex_catalog::DexCatalogItem { + code: "pump_swap", + name: "PumpSwap", + program_id: Some(crate::PUMP_SWAP_PROGRAM_ID), + router_program_id: None, + is_enabled: true, + }); + }, + "meteora_dbc" => { + return Some(crate::dex_catalog::DexCatalogItem { + code: "meteora_dbc", + name: "Meteora DBC", + program_id: Some(crate::METEORA_DBC_PROGRAM_ID), + router_program_id: None, + is_enabled: true, + }); + }, + "meteora_damm_v1" => { + return Some(crate::dex_catalog::DexCatalogItem { + code: "meteora_damm_v1", + name: "Meteora DAMM v1", + program_id: Some(crate::METEORA_DAMM_V1_PROGRAM_ID), + router_program_id: None, + is_enabled: true, + }); + }, + "meteora_damm_v2" => { + return Some(crate::dex_catalog::DexCatalogItem { + code: "meteora_damm_v2", + name: "Meteora DAMM v2", + program_id: Some(crate::METEORA_DAMM_V2_PROGRAM_ID), + router_program_id: None, + is_enabled: true, + }); + }, + "orca_whirlpools" => { + return Some(crate::dex_catalog::DexCatalogItem { + code: "orca_whirlpools", + name: "Orca Whirlpools", + program_id: Some(crate::ORCA_WHIRLPOOLS_PROGRAM_ID), + router_program_id: None, + is_enabled: true, + }); + }, + "fluxbeam" => { + return Some(crate::dex_catalog::DexCatalogItem { + code: "fluxbeam", + name: "FluxBeam", + program_id: Some(crate::FLUXBEAM_PROGRAM_ID), + router_program_id: None, + is_enabled: true, + }); + }, + "dexlab" => { + return Some(crate::dex_catalog::DexCatalogItem { + code: "dexlab", + name: "DexLab Swap/Pool", + program_id: Some(crate::DEXLAB_PROGRAM_ID), + router_program_id: None, + is_enabled: true, + }); + }, + // Planned launch/swap surfaces. + // + // These entries are intentionally present before decoder support so that + // the roadmap can evolve without duplicating DEX metadata later. + // + // Program ids should be filled only after validation against live + // transactions and official or otherwise trustworthy references. + "raydium_launchlab" => { + return Some(crate::dex_catalog::DexCatalogItem { + code: "raydium_launchlab", + name: "Raydium LaunchLab", + program_id: None, + router_program_id: None, + is_enabled: false, + }); + }, + "letsbonk" => { + return Some(crate::dex_catalog::DexCatalogItem { + code: "letsbonk", + name: "LetsBonk / Bonk.fun", + program_id: None, + router_program_id: None, + is_enabled: false, + }); + }, + "boop_fun" => { + return Some(crate::dex_catalog::DexCatalogItem { + code: "boop_fun", + name: "Boop.fun", + program_id: None, + router_program_id: None, + is_enabled: false, + }); + }, + "moonshot" => { + return Some(crate::dex_catalog::DexCatalogItem { + code: "moonshot", + name: "Moonshot", + program_id: None, + router_program_id: None, + is_enabled: false, + }); + }, + "believe" => { + return Some(crate::dex_catalog::DexCatalogItem { + code: "believe", + name: "Believe", + program_id: None, + router_program_id: None, + is_enabled: false, + }); + }, + "heaven" => { + return Some(crate::dex_catalog::DexCatalogItem { + code: "heaven", + name: "Heaven", + program_id: None, + router_program_id: None, + is_enabled: false, + }); + }, + _ => return None, + } +} + +/// Ensures that one known DEX exists in storage and returns its internal id. +pub(crate) async fn ensure_known_dex( + database: &crate::Database, + code: &str, +) -> Result { + let dex_result = crate::query_dexs_get_by_code(database, code).await; + let dex_option = match dex_result { + Ok(dex_option) => dex_option, + Err(error) => return Err(error), + }; + match dex_option { + Some(dex) => match dex.id { + Some(dex_id) => return Ok(dex_id), + None => { + return Err(crate::Error::InvalidState(format!("{} dex has no internal id", code))); + }, + }, + None => { + let catalog_item_option = crate::dex_catalog::dex_catalog_item_by_code(code); + let catalog_item = match catalog_item_option { + Some(catalog_item) => catalog_item, + None => { + return Err(crate::Error::InvalidState(format!( + "unknown dex catalog code '{}'", + code + ))); + }, + }; + let program_id = match catalog_item.program_id { + Some(program_id) => Some(program_id.to_string()), + None => None, + }; + let router_program_id = match catalog_item.router_program_id { + Some(router_program_id) => Some(router_program_id.to_string()), + None => None, + }; + let dex_dto = crate::DexDto::new( + catalog_item.code.to_string(), + catalog_item.name.to_string(), + program_id, + router_program_id, + catalog_item.is_enabled, + ); + return crate::query_dexs_upsert(database, &dex_dto).await; + }, + } +} + +#[cfg(test)] +mod tests { + #[test] + fn known_active_dexes_are_available_from_catalog() { + let codes = [ + "raydium", + "raydium_cpmm", + "raydium_clmm", + "pump_fun", + "pump_swap", + "meteora_dbc", + "meteora_damm_v1", + "meteora_damm_v2", + "orca_whirlpools", + "fluxbeam", + "dexlab", + ]; + + for code in codes { + let item_option = crate::dex_catalog::dex_catalog_item_by_code(code); + let item = match item_option { + Some(item) => item, + None => { + panic!("expected known dex catalog item for '{}'", code); + }, + }; + assert_eq!(item.code, code); + assert!(item.is_enabled); + assert!(item.program_id.is_some()); + } + } + + #[test] + fn planned_launch_surfaces_are_present_but_disabled() { + let codes = ["raydium_launchlab", "letsbonk", "boop_fun", "moonshot", "believe", "heaven"]; + for code in codes { + let item_option = crate::dex_catalog::dex_catalog_item_by_code(code); + let item = match item_option { + Some(item) => item, + None => { + panic!("expected planned launch surface catalog item for '{}'", code); + }, + }; + assert_eq!(item.code, code); + assert!(!item.is_enabled); + } + } + + #[test] + fn unknown_dex_code_is_not_silently_accepted() { + let item_option = crate::dex_catalog::dex_catalog_item_by_code("zora_solana"); + assert!(item_option.is_none()); + } +} diff --git a/kb_lib/src/dex_decode.rs b/kb_lib/src/dex_decode.rs index da8503b..5bbd0f8 100644 --- a/kb_lib/src/dex_decode.rs +++ b/kb_lib/src/dex_decode.rs @@ -37,233 +37,145 @@ impl DexDecodeService { }; } - async fn decode_and_persist_raydium_clmm_events( - &self, - transaction: &crate::ChainTransactionDto, - instructions: &[crate::ChainInstructionDto], - ) -> Result, crate::Error> { - let mut persisted = std::vec::Vec::new(); - for instruction in instructions { - let program_id = match instruction.program_id.as_ref() { - Some(program_id) => program_id, - None => continue, - }; - if program_id.as_str() != crate::RAYDIUM_CLMM_PROGRAM_ID { - continue; - } - let data_json = match instruction.data_json.as_ref() { - Some(data_json) => data_json, - None => continue, - }; - let decoded_events = crate::decode_raydium_clmm_instruction( - instruction.accounts_json.as_str(), - data_json.as_str(), - ); - for decoded_event in &decoded_events { - let persist_result = - self.persist_raydium_clmm_event(transaction, instruction, decoded_event).await; - let persisted_event = match persist_result { - Ok(persisted_event) => persisted_event, - Err(error) => return Err(error), - }; - persisted.push(persisted_event); - } - } - return Ok(persisted); - } - /// Decodes one projected transaction and persists the decoded events. pub async fn decode_transaction_by_signature( &self, signature: &str, ) -> Result, crate::Error> { - let transaction_result = - crate::query_chain_transactions_get_by_signature(self.database.as_ref(), signature) - .await; - let transaction_option = match transaction_result { - Ok(transaction_option) => transaction_option, - Err(error) => return Err(error), - }; - let transaction = match transaction_option { - Some(transaction) => transaction, - None => { - return Err(crate::Error::InvalidState(format!( - "cannot decode unknown chain transaction '{}'", - signature - ))); - }, - }; - let transaction_id_option = transaction.id; - let transaction_id = match transaction_id_option { - Some(transaction_id) => transaction_id, - None => { - return Err(crate::Error::InvalidState(format!( - "chain transaction '{}' has no internal id", - signature - ))); - }, - }; - let instructions_result = crate::query_chain_instructions_list_by_transaction_id( + let context_result = crate::dex_decode_context::load_dex_decode_transaction_context( self.database.as_ref(), - transaction_id, + signature, ) .await; - let instructions = match instructions_result { - Ok(instructions) => instructions, + let context = match context_result { + Ok(context) => context, Err(error) => return Err(error), }; + let transaction = context.transaction; + let instructions = context.instructions; let mut persisted = std::vec::Vec::new(); - let raydium_decoded_result = - self.raydium_amm_v4_decoder.decode_transaction(&transaction, &instructions); - let raydium_decoded = match raydium_decoded_result { - Ok(raydium_decoded) => raydium_decoded, - Err(error) => return Err(error), - }; - for decoded_event in &raydium_decoded { - let persist_result = - self.persist_raydium_amm_v4_event(&transaction, decoded_event).await; - let persisted_event = match persist_result { - Ok(persisted_event) => persisted_event, - Err(error) => return Err(error), - }; - persisted.push(persisted_event); + let append_result = append_persisted_events_result( + &mut persisted, + self.decode_and_persist_raydium_amm_v4_events(&transaction, &instructions).await, + ); + if let Err(error) = append_result { + return Err(error); } - let raydium_cpmm_persisted_result = - self.decode_and_persist_raydium_cpmm_events(&transaction, &instructions).await; - let raydium_cpmm_persisted = match raydium_cpmm_persisted_result { - Ok(raydium_cpmm_persisted) => raydium_cpmm_persisted, - Err(error) => return Err(error), - }; - for persisted_event in raydium_cpmm_persisted { - persisted.push(persisted_event); + let append_result = append_persisted_events_result( + &mut persisted, + self.decode_and_persist_raydium_cpmm_events(&transaction, &instructions).await, + ); + if let Err(error) = append_result { + return Err(error); } - let raydium_clmm_persisted_result = - self.decode_and_persist_raydium_clmm_events(&transaction, &instructions).await; - let raydium_clmm_persisted = match raydium_clmm_persisted_result { - Ok(raydium_clmm_persisted) => raydium_clmm_persisted, - Err(error) => return Err(error), - }; - for persisted_event in raydium_clmm_persisted { - persisted.push(persisted_event); + let append_result = append_persisted_events_result( + &mut persisted, + self.decode_and_persist_raydium_clmm_events(&transaction, &instructions).await, + ); + if let Err(error) = append_result { + return Err(error); } - let pump_fun_decoded_result = - self.pump_fun_decoder.decode_transaction(&transaction, &instructions); - let pump_fun_decoded = match pump_fun_decoded_result { - Ok(pump_fun_decoded) => pump_fun_decoded, - Err(error) => return Err(error), - }; - for decoded_event in &pump_fun_decoded { - let persist_result = self.persist_pump_fun_event(&transaction, decoded_event).await; - let persisted_event = match persist_result { - Ok(persisted_event) => persisted_event, - Err(error) => return Err(error), - }; - persisted.push(persisted_event); + let append_result = append_persisted_events_result( + &mut persisted, + self.decode_and_persist_pump_fun_events(&transaction, &instructions).await, + ); + if let Err(error) = append_result { + return Err(error); } - let pump_swap_decoded_result = - self.pump_swap_decoder.decode_transaction(&transaction, &instructions); - let pump_swap_decoded = match pump_swap_decoded_result { - Ok(pump_swap_decoded) => pump_swap_decoded, - Err(error) => return Err(error), - }; - for decoded_event in &pump_swap_decoded { - let persist_result = self.persist_pump_swap_event(&transaction, decoded_event).await; - let persisted_event = match persist_result { - Ok(persisted_event) => persisted_event, - Err(error) => return Err(error), - }; - persisted.push(persisted_event); + let append_result = append_persisted_events_result( + &mut persisted, + self.decode_and_persist_pump_swap_events(&transaction, &instructions).await, + ); + if let Err(error) = append_result { + return Err(error); } - let meteora_dbc_decoded_result = - self.meteora_dbc_decoder.decode_transaction(&transaction, &instructions); - let meteora_dbc_decoded = match meteora_dbc_decoded_result { - Ok(meteora_dbc_decoded) => meteora_dbc_decoded, - Err(error) => return Err(error), - }; - for decoded_event in &meteora_dbc_decoded { - let persist_result = self.persist_meteora_dbc_event(&transaction, decoded_event).await; - let persisted_event = match persist_result { - Ok(persisted_event) => persisted_event, - Err(error) => return Err(error), - }; - persisted.push(persisted_event); + let append_result = append_persisted_events_result( + &mut persisted, + self.decode_and_persist_meteora_dbc_events(&transaction, &instructions).await, + ); + if let Err(error) = append_result { + return Err(error); } - let meteora_damm_v2_decoded_result = - self.meteora_damm_v2_decoder.decode_transaction(&transaction, &instructions); - let meteora_damm_v2_decoded = match meteora_damm_v2_decoded_result { - Ok(meteora_damm_v2_decoded) => meteora_damm_v2_decoded, - Err(error) => return Err(error), - }; - for decoded_event in &meteora_damm_v2_decoded { - let persist_result = - self.persist_meteora_damm_v2_event(&transaction, decoded_event).await; - let persisted_event = match persist_result { - Ok(persisted_event) => persisted_event, - Err(error) => return Err(error), - }; - persisted.push(persisted_event); + let append_result = append_persisted_events_result( + &mut persisted, + self.decode_and_persist_meteora_damm_v2_events(&transaction, &instructions) + .await, + ); + if let Err(error) = append_result { + return Err(error); } - let meteora_damm_v1_decoded_result = - self.meteora_damm_v1_decoder.decode_transaction(&transaction, &instructions); - let meteora_damm_v1_decoded = match meteora_damm_v1_decoded_result { - Ok(meteora_damm_v1_decoded) => meteora_damm_v1_decoded, - Err(error) => return Err(error), - }; - for decoded_event in &meteora_damm_v1_decoded { - let persist_result = - self.persist_meteora_damm_v1_event(&transaction, decoded_event).await; - let persisted_event = match persist_result { - Ok(persisted_event) => persisted_event, - Err(error) => return Err(error), - }; - persisted.push(persisted_event); + let append_result = append_persisted_events_result( + &mut persisted, + self.decode_and_persist_meteora_damm_v1_events(&transaction, &instructions) + .await, + ); + if let Err(error) = append_result { + return Err(error); } - let orca_whirlpools_decoded_result = - self.orca_whirlpools_decoder.decode_transaction(&transaction, &instructions); - let orca_whirlpools_decoded = match orca_whirlpools_decoded_result { - Ok(orca_whirlpools_decoded) => orca_whirlpools_decoded, - Err(error) => return Err(error), - }; - for decoded_event in &orca_whirlpools_decoded { - let persist_result = - self.persist_orca_whirlpools_event(&transaction, decoded_event).await; - let persisted_event = match persist_result { - Ok(persisted_event) => persisted_event, - Err(error) => return Err(error), - }; - persisted.push(persisted_event); + let append_result = append_persisted_events_result( + &mut persisted, + self.decode_and_persist_orca_whirlpools_events(&transaction, &instructions) + .await, + ); + if let Err(error) = append_result { + return Err(error); } - let fluxbeam_decoded_result = - self.fluxbeam_decoder.decode_transaction(&transaction, &instructions); - let fluxbeam_decoded = match fluxbeam_decoded_result { - Ok(fluxbeam_decoded) => fluxbeam_decoded, - Err(error) => return Err(error), - }; - for decoded_event in &fluxbeam_decoded { - let persist_result = self.persist_fluxbeam_event(&transaction, decoded_event).await; - let persisted_event = match persist_result { - Ok(persisted_event) => persisted_event, - Err(error) => return Err(error), - }; - persisted.push(persisted_event); + let append_result = append_persisted_events_result( + &mut persisted, + self.decode_and_persist_fluxbeam_events(&transaction, &instructions).await, + ); + if let Err(error) = append_result { + return Err(error); } - let dexlab_decoded_result = - self.dexlab_decoder.decode_transaction(&transaction, &instructions); - let dexlab_decoded = match dexlab_decoded_result { - Ok(dexlab_decoded) => dexlab_decoded, - Err(error) => return Err(error), - }; - for decoded_event in &dexlab_decoded { - let persist_result = self.persist_dexlab_event(&transaction, decoded_event).await; - let persisted_event = match persist_result { - Ok(persisted_event) => persisted_event, - Err(error) => return Err(error), - }; - persisted.push(persisted_event); + let append_result = append_persisted_events_result( + &mut persisted, + self.decode_and_persist_dexlab_events(&transaction, &instructions).await, + ); + if let Err(error) = append_result { + return Err(error); } return Ok(persisted); } + async fn materialize_named_dex_event( + &self, + transaction: &crate::ChainTransactionDto, + transaction_id: i64, + instruction_id: i64, + protocol_name: &str, + program_id: std::string::String, + event_kind: &str, + pool_account: std::option::Option, + market_account: std::option::Option, + token_a_mint: std::option::Option, + token_b_mint: std::option::Option, + lp_mint: std::option::Option, + payload_json: serde_json::Value, + ) -> Result { + let input = crate::dex_decoded_event_materialization::DexDecodedEventMaterializationInput { + database: self.database.as_ref(), + persistence: &self.persistence, + transaction, + transaction_id, + instruction_id: Some(instruction_id), + protocol_name: protocol_name.to_string(), + program_id, + event_kind: event_kind.to_string(), + pool_account, + market_account, + token_a_mint, + token_b_mint, + lp_mint, + enrichment_payload_json: payload_json.clone(), + observation_payload_json: payload_json, + observation_kind: format!("dex.{event_kind}"), + signal_kind: format!("signal.dex.{event_kind}"), + missing_after_upsert_message: "decoded event disappeared after upsert".to_string(), + }; + return crate::dex_decoded_event_materialization::materialize_dex_decoded_event(input) + .await; + } + async fn persist_dexlab_event( &self, transaction: &crate::ChainTransactionDto, @@ -271,190 +183,40 @@ impl DexDecodeService { ) -> Result { match decoded_event { crate::DexlabDecodedEvent::CreatePool(event) => { - let payload_json_result = enrich_and_serialize_dex_decoded_payload( - "dexlab", - "dexlab.create_pool", - event.payload_json.clone(), - ); - let payload_json = match payload_json_result { - Ok(payload_json) => payload_json, - Err(error) => return Err(error), - }; - let existing_result = crate::query_dex_decoded_events_get_by_key( - self.database.as_ref(), - event.transaction_id, - Some(event.instruction_id), - "dexlab.create_pool", - ) - .await; - let existing_option = match existing_result { - Ok(existing_option) => existing_option, - Err(error) => return Err(error), - }; - let already_present = existing_option.is_some(); - let dto = crate::DexDecodedEventDto::new( - event.transaction_id, - Some(event.instruction_id), - "dexlab".to_string(), - event.program_id.clone(), - "dexlab.create_pool".to_string(), - event.pool_account.clone(), - None, - event.token_a_mint.clone(), - event.token_b_mint.clone(), - None, - payload_json, - ); - let upsert_result = - crate::query_dex_decoded_events_upsert(self.database.as_ref(), &dto).await; - if let Err(error) = upsert_result { - return Err(error); - } - let fetched_result = crate::query_dex_decoded_events_get_by_key( - self.database.as_ref(), - event.transaction_id, - Some(event.instruction_id), - "dexlab.create_pool", - ) - .await; - let fetched_option = match fetched_result { - Ok(fetched_option) => fetched_option, - Err(error) => return Err(error), - }; - let fetched = match fetched_option { - Some(fetched) => fetched, - None => { - return Err(crate::Error::InvalidState( - "decoded event disappeared after upsert".to_string(), - )); - }, - }; - if !already_present { - let payload_value = event.payload_json.clone(); - let observation_result = self - .persistence - .record_observation(&crate::DetectionObservationInput::new( - "dex.dexlab.create_pool".to_string(), - crate::ObservationSourceKind::HttpRpc, - transaction.source_endpoint_name.clone(), - transaction.signature.clone(), - transaction.slot, - payload_value.clone(), - )) - .await; - let observation_id = match observation_result { - Ok(observation_id) => observation_id, - Err(error) => return Err(error), - }; - let signal_result = self - .persistence - .record_signal(&crate::DetectionSignalInput::new( - "signal.dex.dexlab.create_pool".to_string(), - crate::AnalysisSignalSeverity::Low, - transaction.signature.clone(), - Some(observation_id), - None, - payload_value, - )) - .await; - if let Err(error) = signal_result { - return Err(error); - } - } - return Ok(fetched); + return self + .materialize_named_dex_event( + transaction, + event.transaction_id, + event.instruction_id, + "dexlab", + event.program_id.clone(), + "dexlab.create_pool", + event.pool_account.clone(), + None, + event.token_a_mint.clone(), + event.token_b_mint.clone(), + None, + event.payload_json.clone(), + ) + .await; }, crate::DexlabDecodedEvent::Swap(event) => { - let payload_json_result = enrich_and_serialize_dex_decoded_payload( - "dexlab", - "dexlab.swap", - event.payload_json.clone(), - ); - let payload_json = match payload_json_result { - Ok(payload_json) => payload_json, - Err(error) => return Err(error), - }; - let existing_result = crate::query_dex_decoded_events_get_by_key( - self.database.as_ref(), - event.transaction_id, - Some(event.instruction_id), - "dexlab.swap", - ) - .await; - let existing_option = match existing_result { - Ok(existing_option) => existing_option, - Err(error) => return Err(error), - }; - let already_present = existing_option.is_some(); - let dto = crate::DexDecodedEventDto::new( - event.transaction_id, - Some(event.instruction_id), - "dexlab".to_string(), - event.program_id.clone(), - "dexlab.swap".to_string(), - event.pool_account.clone(), - None, - event.token_a_mint.clone(), - event.token_b_mint.clone(), - None, - payload_json, - ); - let upsert_result = - crate::query_dex_decoded_events_upsert(self.database.as_ref(), &dto).await; - if let Err(error) = upsert_result { - return Err(error); - } - let fetched_result = crate::query_dex_decoded_events_get_by_key( - self.database.as_ref(), - event.transaction_id, - Some(event.instruction_id), - "dexlab.swap", - ) - .await; - let fetched_option = match fetched_result { - Ok(fetched_option) => fetched_option, - Err(error) => return Err(error), - }; - let fetched = match fetched_option { - Some(fetched) => fetched, - None => { - return Err(crate::Error::InvalidState( - "decoded event disappeared after upsert".to_string(), - )); - }, - }; - if !already_present { - let payload_value = event.payload_json.clone(); - let observation_result = self - .persistence - .record_observation(&crate::DetectionObservationInput::new( - "dex.dexlab.swap".to_string(), - crate::ObservationSourceKind::HttpRpc, - transaction.source_endpoint_name.clone(), - transaction.signature.clone(), - transaction.slot, - payload_value.clone(), - )) - .await; - let observation_id = match observation_result { - Ok(observation_id) => observation_id, - Err(error) => return Err(error), - }; - let signal_result = self - .persistence - .record_signal(&crate::DetectionSignalInput::new( - "signal.dex.dexlab.swap".to_string(), - crate::AnalysisSignalSeverity::Low, - transaction.signature.clone(), - Some(observation_id), - None, - payload_value, - )) - .await; - if let Err(error) = signal_result { - return Err(error); - } - } - return Ok(fetched); + return self + .materialize_named_dex_event( + transaction, + event.transaction_id, + event.instruction_id, + "dexlab", + event.program_id.clone(), + "dexlab.swap", + event.pool_account.clone(), + None, + event.token_a_mint.clone(), + event.token_b_mint.clone(), + None, + event.payload_json.clone(), + ) + .await; }, } } @@ -466,190 +228,40 @@ impl DexDecodeService { ) -> Result { match decoded_event { crate::FluxbeamDecodedEvent::CreatePool(event) => { - let payload_json_result = enrich_and_serialize_dex_decoded_payload( - "fluxbeam", - "fluxbeam.create_pool", - event.payload_json.clone(), - ); - let payload_json = match payload_json_result { - Ok(payload_json) => payload_json, - Err(error) => return Err(error), - }; - let existing_result = crate::query_dex_decoded_events_get_by_key( - self.database.as_ref(), - event.transaction_id, - Some(event.instruction_id), - "fluxbeam.create_pool", - ) - .await; - let existing_option = match existing_result { - Ok(existing_option) => existing_option, - Err(error) => return Err(error), - }; - let already_present = existing_option.is_some(); - let dto = crate::DexDecodedEventDto::new( - event.transaction_id, - Some(event.instruction_id), - "fluxbeam".to_string(), - event.program_id.clone(), - "fluxbeam.create_pool".to_string(), - event.pool_account.clone(), - None, - event.token_a_mint.clone(), - event.token_b_mint.clone(), - event.lp_mint.clone(), - payload_json, - ); - let upsert_result = - crate::query_dex_decoded_events_upsert(self.database.as_ref(), &dto).await; - if let Err(error) = upsert_result { - return Err(error); - } - let fetched_result = crate::query_dex_decoded_events_get_by_key( - self.database.as_ref(), - event.transaction_id, - Some(event.instruction_id), - "fluxbeam.create_pool", - ) - .await; - let fetched_option = match fetched_result { - Ok(fetched_option) => fetched_option, - Err(error) => return Err(error), - }; - let fetched = match fetched_option { - Some(fetched) => fetched, - None => { - return Err(crate::Error::InvalidState( - "decoded event disappeared after upsert".to_string(), - )); - }, - }; - if !already_present { - let payload_value = event.payload_json.clone(); - let observation_result = self - .persistence - .record_observation(&crate::DetectionObservationInput::new( - "dex.fluxbeam.create_pool".to_string(), - crate::ObservationSourceKind::HttpRpc, - transaction.source_endpoint_name.clone(), - transaction.signature.clone(), - transaction.slot, - payload_value.clone(), - )) - .await; - let observation_id = match observation_result { - Ok(observation_id) => observation_id, - Err(error) => return Err(error), - }; - let signal_result = self - .persistence - .record_signal(&crate::DetectionSignalInput::new( - "signal.dex.fluxbeam.create_pool".to_string(), - crate::AnalysisSignalSeverity::Low, - transaction.signature.clone(), - Some(observation_id), - None, - payload_value, - )) - .await; - if let Err(error) = signal_result { - return Err(error); - } - } - return Ok(fetched); + return self + .materialize_named_dex_event( + transaction, + event.transaction_id, + event.instruction_id, + "fluxbeam", + event.program_id.clone(), + "fluxbeam.create_pool", + event.pool_account.clone(), + None, + event.token_a_mint.clone(), + event.token_b_mint.clone(), + event.lp_mint.clone(), + event.payload_json.clone(), + ) + .await; }, crate::FluxbeamDecodedEvent::Swap(event) => { - let payload_json_result = enrich_and_serialize_dex_decoded_payload( - "fluxbeam", - "fluxbeam.swap", - event.payload_json.clone(), - ); - let payload_json = match payload_json_result { - Ok(payload_json) => payload_json, - Err(error) => return Err(error), - }; - let existing_result = crate::query_dex_decoded_events_get_by_key( - self.database.as_ref(), - event.transaction_id, - Some(event.instruction_id), - "fluxbeam.swap", - ) - .await; - let existing_option = match existing_result { - Ok(existing_option) => existing_option, - Err(error) => return Err(error), - }; - let already_present = existing_option.is_some(); - let dto = crate::DexDecodedEventDto::new( - event.transaction_id, - Some(event.instruction_id), - "fluxbeam".to_string(), - event.program_id.clone(), - "fluxbeam.swap".to_string(), - event.pool_account.clone(), - None, - event.token_a_mint.clone(), - event.token_b_mint.clone(), - None, - payload_json, - ); - let upsert_result = - crate::query_dex_decoded_events_upsert(self.database.as_ref(), &dto).await; - if let Err(error) = upsert_result { - return Err(error); - } - let fetched_result = crate::query_dex_decoded_events_get_by_key( - self.database.as_ref(), - event.transaction_id, - Some(event.instruction_id), - "fluxbeam.swap", - ) - .await; - let fetched_option = match fetched_result { - Ok(fetched_option) => fetched_option, - Err(error) => return Err(error), - }; - let fetched = match fetched_option { - Some(fetched) => fetched, - None => { - return Err(crate::Error::InvalidState( - "decoded event disappeared after upsert".to_string(), - )); - }, - }; - if !already_present { - let payload_value = event.payload_json.clone(); - let observation_result = self - .persistence - .record_observation(&crate::DetectionObservationInput::new( - "dex.fluxbeam.swap".to_string(), - crate::ObservationSourceKind::HttpRpc, - transaction.source_endpoint_name.clone(), - transaction.signature.clone(), - transaction.slot, - payload_value.clone(), - )) - .await; - let observation_id = match observation_result { - Ok(observation_id) => observation_id, - Err(error) => return Err(error), - }; - let signal_result = self - .persistence - .record_signal(&crate::DetectionSignalInput::new( - "signal.dex.fluxbeam.swap".to_string(), - crate::AnalysisSignalSeverity::Low, - transaction.signature.clone(), - Some(observation_id), - None, - payload_value, - )) - .await; - if let Err(error) = signal_result { - return Err(error); - } - } - return Ok(fetched); + return self + .materialize_named_dex_event( + transaction, + event.transaction_id, + event.instruction_id, + "fluxbeam", + event.program_id.clone(), + "fluxbeam.swap", + event.pool_account.clone(), + None, + event.token_a_mint.clone(), + event.token_b_mint.clone(), + None, + event.payload_json.clone(), + ) + .await; }, } } @@ -661,190 +273,40 @@ impl DexDecodeService { ) -> Result { match decoded_event { crate::OrcaWhirlpoolsDecodedEvent::CreatePool(event) => { - let payload_json_result = enrich_and_serialize_dex_decoded_payload( - "orca_whirlpools", - "orca_whirlpools.create_pool", - event.payload_json.clone(), - ); - let payload_json = match payload_json_result { - Ok(payload_json) => payload_json, - Err(error) => return Err(error), - }; - let existing_result = crate::query_dex_decoded_events_get_by_key( - self.database.as_ref(), - event.transaction_id, - Some(event.instruction_id), - "orca_whirlpools.create_pool", - ) - .await; - let existing_option = match existing_result { - Ok(existing_option) => existing_option, - Err(error) => return Err(error), - }; - let already_present = existing_option.is_some(); - let dto = crate::DexDecodedEventDto::new( - event.transaction_id, - Some(event.instruction_id), - "orca_whirlpools".to_string(), - event.program_id.clone(), - "orca_whirlpools.create_pool".to_string(), - event.pool_account.clone(), - None, - event.token_a_mint.clone(), - event.token_b_mint.clone(), - event.config_account.clone(), - payload_json, - ); - let upsert_result = - crate::query_dex_decoded_events_upsert(self.database.as_ref(), &dto).await; - if let Err(error) = upsert_result { - return Err(error); - } - let fetched_result = crate::query_dex_decoded_events_get_by_key( - self.database.as_ref(), - event.transaction_id, - Some(event.instruction_id), - "orca_whirlpools.create_pool", - ) - .await; - let fetched_option = match fetched_result { - Ok(fetched_option) => fetched_option, - Err(error) => return Err(error), - }; - let fetched = match fetched_option { - Some(fetched) => fetched, - None => { - return Err(crate::Error::InvalidState( - "decoded event disappeared after upsert".to_string(), - )); - }, - }; - if !already_present { - let payload_value = event.payload_json.clone(); - let observation_result = self - .persistence - .record_observation(&crate::DetectionObservationInput::new( - "dex.orca_whirlpools.create_pool".to_string(), - crate::ObservationSourceKind::HttpRpc, - transaction.source_endpoint_name.clone(), - transaction.signature.clone(), - transaction.slot, - payload_value.clone(), - )) - .await; - let observation_id = match observation_result { - Ok(observation_id) => observation_id, - Err(error) => return Err(error), - }; - let signal_result = self - .persistence - .record_signal(&crate::DetectionSignalInput::new( - "signal.dex.orca_whirlpools.create_pool".to_string(), - crate::AnalysisSignalSeverity::Low, - transaction.signature.clone(), - Some(observation_id), - None, - payload_value, - )) - .await; - if let Err(error) = signal_result { - return Err(error); - } - } - return Ok(fetched); + return self + .materialize_named_dex_event( + transaction, + event.transaction_id, + event.instruction_id, + "orca_whirlpools", + event.program_id.clone(), + "orca_whirlpools.create_pool", + event.pool_account.clone(), + None, + event.token_a_mint.clone(), + event.token_b_mint.clone(), + event.config_account.clone(), + event.payload_json.clone(), + ) + .await; }, crate::OrcaWhirlpoolsDecodedEvent::Swap(event) => { - let payload_json_result = enrich_and_serialize_dex_decoded_payload( - "orca_whirlpools", - "orca_whirlpools.swap", - event.payload_json.clone(), - ); - let payload_json = match payload_json_result { - Ok(payload_json) => payload_json, - Err(error) => return Err(error), - }; - let existing_result = crate::query_dex_decoded_events_get_by_key( - self.database.as_ref(), - event.transaction_id, - Some(event.instruction_id), - "orca_whirlpools.swap", - ) - .await; - let existing_option = match existing_result { - Ok(existing_option) => existing_option, - Err(error) => return Err(error), - }; - let already_present = existing_option.is_some(); - let dto = crate::DexDecodedEventDto::new( - event.transaction_id, - Some(event.instruction_id), - "orca_whirlpools".to_string(), - event.program_id.clone(), - "orca_whirlpools.swap".to_string(), - event.pool_account.clone(), - None, - event.token_a_mint.clone(), - event.token_b_mint.clone(), - None, - payload_json, - ); - let upsert_result = - crate::query_dex_decoded_events_upsert(self.database.as_ref(), &dto).await; - if let Err(error) = upsert_result { - return Err(error); - } - let fetched_result = crate::query_dex_decoded_events_get_by_key( - self.database.as_ref(), - event.transaction_id, - Some(event.instruction_id), - "orca_whirlpools.swap", - ) - .await; - let fetched_option = match fetched_result { - Ok(fetched_option) => fetched_option, - Err(error) => return Err(error), - }; - let fetched = match fetched_option { - Some(fetched) => fetched, - None => { - return Err(crate::Error::InvalidState( - "decoded event disappeared after upsert".to_string(), - )); - }, - }; - if !already_present { - let payload_value = event.payload_json.clone(); - let observation_result = self - .persistence - .record_observation(&crate::DetectionObservationInput::new( - "dex.orca_whirlpools.swap".to_string(), - crate::ObservationSourceKind::HttpRpc, - transaction.source_endpoint_name.clone(), - transaction.signature.clone(), - transaction.slot, - payload_value.clone(), - )) - .await; - let observation_id = match observation_result { - Ok(observation_id) => observation_id, - Err(error) => return Err(error), - }; - let signal_result = self - .persistence - .record_signal(&crate::DetectionSignalInput::new( - "signal.dex.orca_whirlpools.swap".to_string(), - crate::AnalysisSignalSeverity::Low, - transaction.signature.clone(), - Some(observation_id), - None, - payload_value, - )) - .await; - if let Err(error) = signal_result { - return Err(error); - } - } - return Ok(fetched); + return self + .materialize_named_dex_event( + transaction, + event.transaction_id, + event.instruction_id, + "orca_whirlpools", + event.program_id.clone(), + "orca_whirlpools.swap", + event.pool_account.clone(), + None, + event.token_a_mint.clone(), + event.token_b_mint.clone(), + None, + event.payload_json.clone(), + ) + .await; }, } } @@ -856,190 +318,40 @@ impl DexDecodeService { ) -> Result { match decoded_event { crate::MeteoraDammV1DecodedEvent::CreatePool(event) => { - let payload_json_result = enrich_and_serialize_dex_decoded_payload( - "meteora_damm_v1", - "meteora_damm_v1.create_pool", - event.payload_json.clone(), - ); - let payload_json = match payload_json_result { - Ok(payload_json) => payload_json, - Err(error) => return Err(error), - }; - let existing_result = crate::query_dex_decoded_events_get_by_key( - self.database.as_ref(), - event.transaction_id, - Some(event.instruction_id), - "meteora_damm_v1.create_pool", - ) - .await; - let existing_option = match existing_result { - Ok(existing_option) => existing_option, - Err(error) => return Err(error), - }; - let already_present = existing_option.is_some(); - let dto = crate::DexDecodedEventDto::new( - event.transaction_id, - Some(event.instruction_id), - "meteora_damm_v1".to_string(), - event.program_id.clone(), - "meteora_damm_v1.create_pool".to_string(), - event.pool_account.clone(), - None, - event.token_a_mint.clone(), - event.token_b_mint.clone(), - event.config_account.clone(), - payload_json, - ); - let upsert_result = - crate::query_dex_decoded_events_upsert(self.database.as_ref(), &dto).await; - if let Err(error) = upsert_result { - return Err(error); - } - let fetched_result = crate::query_dex_decoded_events_get_by_key( - self.database.as_ref(), - event.transaction_id, - Some(event.instruction_id), - "meteora_damm_v1.create_pool", - ) - .await; - let fetched_option = match fetched_result { - Ok(fetched_option) => fetched_option, - Err(error) => return Err(error), - }; - let fetched = match fetched_option { - Some(fetched) => fetched, - None => { - return Err(crate::Error::InvalidState( - "decoded event disappeared after upsert".to_string(), - )); - }, - }; - if !already_present { - let payload_value = event.payload_json.clone(); - let observation_result = self - .persistence - .record_observation(&crate::DetectionObservationInput::new( - "dex.meteora_damm_v1.create_pool".to_string(), - crate::ObservationSourceKind::HttpRpc, - transaction.source_endpoint_name.clone(), - transaction.signature.clone(), - transaction.slot, - payload_value.clone(), - )) - .await; - let observation_id = match observation_result { - Ok(observation_id) => observation_id, - Err(error) => return Err(error), - }; - let signal_result = self - .persistence - .record_signal(&crate::DetectionSignalInput::new( - "signal.dex.meteora_damm_v1.create_pool".to_string(), - crate::AnalysisSignalSeverity::Low, - transaction.signature.clone(), - Some(observation_id), - None, - payload_value, - )) - .await; - if let Err(error) = signal_result { - return Err(error); - } - } - return Ok(fetched); + return self + .materialize_named_dex_event( + transaction, + event.transaction_id, + event.instruction_id, + "meteora_damm_v1", + event.program_id.clone(), + "meteora_damm_v1.create_pool", + event.pool_account.clone(), + None, + event.token_a_mint.clone(), + event.token_b_mint.clone(), + event.config_account.clone(), + event.payload_json.clone(), + ) + .await; }, crate::MeteoraDammV1DecodedEvent::Swap(event) => { - let payload_json_result = enrich_and_serialize_dex_decoded_payload( - "meteora_damm_v1", - "meteora_damm_v1.swap", - event.payload_json.clone(), - ); - let payload_json = match payload_json_result { - Ok(payload_json) => payload_json, - Err(error) => return Err(error), - }; - let existing_result = crate::query_dex_decoded_events_get_by_key( - self.database.as_ref(), - event.transaction_id, - Some(event.instruction_id), - "meteora_damm_v1.swap", - ) - .await; - let existing_option = match existing_result { - Ok(existing_option) => existing_option, - Err(error) => return Err(error), - }; - let already_present = existing_option.is_some(); - let dto = crate::DexDecodedEventDto::new( - event.transaction_id, - Some(event.instruction_id), - "meteora_damm_v1".to_string(), - event.program_id.clone(), - "meteora_damm_v1.swap".to_string(), - event.pool_account.clone(), - None, - event.token_a_mint.clone(), - event.token_b_mint.clone(), - None, - payload_json, - ); - let upsert_result = - crate::query_dex_decoded_events_upsert(self.database.as_ref(), &dto).await; - if let Err(error) = upsert_result { - return Err(error); - } - let fetched_result = crate::query_dex_decoded_events_get_by_key( - self.database.as_ref(), - event.transaction_id, - Some(event.instruction_id), - "meteora_damm_v1.swap", - ) - .await; - let fetched_option = match fetched_result { - Ok(fetched_option) => fetched_option, - Err(error) => return Err(error), - }; - let fetched = match fetched_option { - Some(fetched) => fetched, - None => { - return Err(crate::Error::InvalidState( - "decoded event disappeared after upsert".to_string(), - )); - }, - }; - if !already_present { - let payload_value = event.payload_json.clone(); - let observation_result = self - .persistence - .record_observation(&crate::DetectionObservationInput::new( - "dex.meteora_damm_v1.swap".to_string(), - crate::ObservationSourceKind::HttpRpc, - transaction.source_endpoint_name.clone(), - transaction.signature.clone(), - transaction.slot, - payload_value.clone(), - )) - .await; - let observation_id = match observation_result { - Ok(observation_id) => observation_id, - Err(error) => return Err(error), - }; - let signal_result = self - .persistence - .record_signal(&crate::DetectionSignalInput::new( - "signal.dex.meteora_damm_v1.swap".to_string(), - crate::AnalysisSignalSeverity::Low, - transaction.signature.clone(), - Some(observation_id), - None, - payload_value, - )) - .await; - if let Err(error) = signal_result { - return Err(error); - } - } - return Ok(fetched); + return self + .materialize_named_dex_event( + transaction, + event.transaction_id, + event.instruction_id, + "meteora_damm_v1", + event.program_id.clone(), + "meteora_damm_v1.swap", + event.pool_account.clone(), + None, + event.token_a_mint.clone(), + event.token_b_mint.clone(), + None, + event.payload_json.clone(), + ) + .await; }, } } @@ -1051,190 +363,40 @@ impl DexDecodeService { ) -> Result { match decoded_event { crate::MeteoraDammV2DecodedEvent::CreatePool(event) => { - let payload_json_result = enrich_and_serialize_dex_decoded_payload( - "meteora_damm_v2", - "meteora_damm_v2.create_pool", - event.payload_json.clone(), - ); - let payload_json = match payload_json_result { - Ok(payload_json) => payload_json, - Err(error) => return Err(error), - }; - let existing_result = crate::query_dex_decoded_events_get_by_key( - self.database.as_ref(), - event.transaction_id, - Some(event.instruction_id), - "meteora_damm_v2.create_pool", - ) - .await; - let existing_option = match existing_result { - Ok(existing_option) => existing_option, - Err(error) => return Err(error), - }; - let already_present = existing_option.is_some(); - let dto = crate::DexDecodedEventDto::new( - event.transaction_id, - Some(event.instruction_id), - "meteora_damm_v2".to_string(), - event.program_id.clone(), - "meteora_damm_v2.create_pool".to_string(), - event.pool_account.clone(), - None, - event.token_a_mint.clone(), - event.token_b_mint.clone(), - event.config_account.clone(), - payload_json, - ); - let upsert_result = - crate::query_dex_decoded_events_upsert(self.database.as_ref(), &dto).await; - if let Err(error) = upsert_result { - return Err(error); - } - let fetched_result = crate::query_dex_decoded_events_get_by_key( - self.database.as_ref(), - event.transaction_id, - Some(event.instruction_id), - "meteora_damm_v2.create_pool", - ) - .await; - let fetched_option = match fetched_result { - Ok(fetched_option) => fetched_option, - Err(error) => return Err(error), - }; - let fetched = match fetched_option { - Some(fetched) => fetched, - None => { - return Err(crate::Error::InvalidState( - "decoded event disappeared after upsert".to_string(), - )); - }, - }; - if !already_present { - let payload_value = event.payload_json.clone(); - let observation_result = self - .persistence - .record_observation(&crate::DetectionObservationInput::new( - "dex.meteora_damm_v2.create_pool".to_string(), - crate::ObservationSourceKind::HttpRpc, - transaction.source_endpoint_name.clone(), - transaction.signature.clone(), - transaction.slot, - payload_value.clone(), - )) - .await; - let observation_id = match observation_result { - Ok(observation_id) => observation_id, - Err(error) => return Err(error), - }; - let signal_result = self - .persistence - .record_signal(&crate::DetectionSignalInput::new( - "signal.dex.meteora_damm_v2.create_pool".to_string(), - crate::AnalysisSignalSeverity::Low, - transaction.signature.clone(), - Some(observation_id), - None, - payload_value, - )) - .await; - if let Err(error) = signal_result { - return Err(error); - } - } - return Ok(fetched); + return self + .materialize_named_dex_event( + transaction, + event.transaction_id, + event.instruction_id, + "meteora_damm_v2", + event.program_id.clone(), + "meteora_damm_v2.create_pool", + event.pool_account.clone(), + None, + event.token_a_mint.clone(), + event.token_b_mint.clone(), + event.config_account.clone(), + event.payload_json.clone(), + ) + .await; }, crate::MeteoraDammV2DecodedEvent::Swap(event) => { - let payload_json_result = enrich_and_serialize_dex_decoded_payload( - "meteora_damm_v2", - "meteora_damm_v2.swap", - event.payload_json.clone(), - ); - let payload_json = match payload_json_result { - Ok(payload_json) => payload_json, - Err(error) => return Err(error), - }; - let existing_result = crate::query_dex_decoded_events_get_by_key( - self.database.as_ref(), - event.transaction_id, - Some(event.instruction_id), - "meteora_damm_v2.swap", - ) - .await; - let existing_option = match existing_result { - Ok(existing_option) => existing_option, - Err(error) => return Err(error), - }; - let already_present = existing_option.is_some(); - let dto = crate::DexDecodedEventDto::new( - event.transaction_id, - Some(event.instruction_id), - "meteora_damm_v2".to_string(), - event.program_id.clone(), - "meteora_damm_v2.swap".to_string(), - event.pool_account.clone(), - None, - event.token_a_mint.clone(), - event.token_b_mint.clone(), - None, - payload_json, - ); - let upsert_result = - crate::query_dex_decoded_events_upsert(self.database.as_ref(), &dto).await; - if let Err(error) = upsert_result { - return Err(error); - } - let fetched_result = crate::query_dex_decoded_events_get_by_key( - self.database.as_ref(), - event.transaction_id, - Some(event.instruction_id), - "meteora_damm_v2.swap", - ) - .await; - let fetched_option = match fetched_result { - Ok(fetched_option) => fetched_option, - Err(error) => return Err(error), - }; - let fetched = match fetched_option { - Some(fetched) => fetched, - None => { - return Err(crate::Error::InvalidState( - "decoded event disappeared after upsert".to_string(), - )); - }, - }; - if !already_present { - let payload_value = event.payload_json.clone(); - let observation_result = self - .persistence - .record_observation(&crate::DetectionObservationInput::new( - "dex.meteora_damm_v2.swap".to_string(), - crate::ObservationSourceKind::HttpRpc, - transaction.source_endpoint_name.clone(), - transaction.signature.clone(), - transaction.slot, - payload_value.clone(), - )) - .await; - let observation_id = match observation_result { - Ok(observation_id) => observation_id, - Err(error) => return Err(error), - }; - let signal_result = self - .persistence - .record_signal(&crate::DetectionSignalInput::new( - "signal.dex.meteora_damm_v2.swap".to_string(), - crate::AnalysisSignalSeverity::Low, - transaction.signature.clone(), - Some(observation_id), - None, - payload_value, - )) - .await; - if let Err(error) = signal_result { - return Err(error); - } - } - return Ok(fetched); + return self + .materialize_named_dex_event( + transaction, + event.transaction_id, + event.instruction_id, + "meteora_damm_v2", + event.program_id.clone(), + "meteora_damm_v2.swap", + event.pool_account.clone(), + None, + event.token_a_mint.clone(), + event.token_b_mint.clone(), + None, + event.payload_json.clone(), + ) + .await; }, } } @@ -1246,190 +408,40 @@ impl DexDecodeService { ) -> Result { match decoded_event { crate::MeteoraDbcDecodedEvent::CreatePool(event) => { - let payload_json_result = enrich_and_serialize_dex_decoded_payload( - "meteora_dbc", - "meteora_dbc.create_pool", - event.payload_json.clone(), - ); - let payload_json = match payload_json_result { - Ok(payload_json) => payload_json, - Err(error) => return Err(error), - }; - let existing_result = crate::query_dex_decoded_events_get_by_key( - self.database.as_ref(), - event.transaction_id, - Some(event.instruction_id), - "meteora_dbc.create_pool", - ) - .await; - let existing_option = match existing_result { - Ok(existing_option) => existing_option, - Err(error) => return Err(error), - }; - let already_present = existing_option.is_some(); - let dto = crate::DexDecodedEventDto::new( - event.transaction_id, - Some(event.instruction_id), - "meteora_dbc".to_string(), - event.program_id.clone(), - "meteora_dbc.create_pool".to_string(), - event.pool_account.clone(), - None, - event.token_a_mint.clone(), - event.token_b_mint.clone(), - event.config_account.clone(), - payload_json, - ); - let upsert_result = - crate::query_dex_decoded_events_upsert(self.database.as_ref(), &dto).await; - if let Err(error) = upsert_result { - return Err(error); - } - let fetched_result = crate::query_dex_decoded_events_get_by_key( - self.database.as_ref(), - event.transaction_id, - Some(event.instruction_id), - "meteora_dbc.create_pool", - ) - .await; - let fetched_option = match fetched_result { - Ok(fetched_option) => fetched_option, - Err(error) => return Err(error), - }; - let fetched = match fetched_option { - Some(fetched) => fetched, - None => { - return Err(crate::Error::InvalidState( - "decoded event disappeared after upsert".to_string(), - )); - }, - }; - if !already_present { - let payload_value = event.payload_json.clone(); - let observation_result = self - .persistence - .record_observation(&crate::DetectionObservationInput::new( - "dex.meteora_dbc.create_pool".to_string(), - crate::ObservationSourceKind::HttpRpc, - transaction.source_endpoint_name.clone(), - transaction.signature.clone(), - transaction.slot, - payload_value.clone(), - )) - .await; - let observation_id = match observation_result { - Ok(observation_id) => observation_id, - Err(error) => return Err(error), - }; - let signal_result = self - .persistence - .record_signal(&crate::DetectionSignalInput::new( - "signal.dex.meteora_dbc.create_pool".to_string(), - crate::AnalysisSignalSeverity::Low, - transaction.signature.clone(), - Some(observation_id), - None, - payload_value, - )) - .await; - if let Err(error) = signal_result { - return Err(error); - } - } - return Ok(fetched); + return self + .materialize_named_dex_event( + transaction, + event.transaction_id, + event.instruction_id, + "meteora_dbc", + event.program_id.clone(), + "meteora_dbc.create_pool", + event.pool_account.clone(), + None, + event.token_a_mint.clone(), + event.token_b_mint.clone(), + event.config_account.clone(), + event.payload_json.clone(), + ) + .await; }, crate::MeteoraDbcDecodedEvent::Swap(event) => { - let payload_json_result = enrich_and_serialize_dex_decoded_payload( - "meteora_dbc", - "meteora_dbc.swap", - event.payload_json.clone(), - ); - let payload_json = match payload_json_result { - Ok(payload_json) => payload_json, - Err(error) => return Err(error), - }; - let existing_result = crate::query_dex_decoded_events_get_by_key( - self.database.as_ref(), - event.transaction_id, - Some(event.instruction_id), - "meteora_dbc.swap", - ) - .await; - let existing_option = match existing_result { - Ok(existing_option) => existing_option, - Err(error) => return Err(error), - }; - let already_present = existing_option.is_some(); - let dto = crate::DexDecodedEventDto::new( - event.transaction_id, - Some(event.instruction_id), - "meteora_dbc".to_string(), - event.program_id.clone(), - "meteora_dbc.swap".to_string(), - event.pool_account.clone(), - None, - event.token_a_mint.clone(), - event.token_b_mint.clone(), - None, - payload_json, - ); - let upsert_result = - crate::query_dex_decoded_events_upsert(self.database.as_ref(), &dto).await; - if let Err(error) = upsert_result { - return Err(error); - } - let fetched_result = crate::query_dex_decoded_events_get_by_key( - self.database.as_ref(), - event.transaction_id, - Some(event.instruction_id), - "meteora_dbc.swap", - ) - .await; - let fetched_option = match fetched_result { - Ok(fetched_option) => fetched_option, - Err(error) => return Err(error), - }; - let fetched = match fetched_option { - Some(fetched) => fetched, - None => { - return Err(crate::Error::InvalidState( - "decoded event disappeared after upsert".to_string(), - )); - }, - }; - if !already_present { - let payload_value = event.payload_json.clone(); - let observation_result = self - .persistence - .record_observation(&crate::DetectionObservationInput::new( - "dex.meteora_dbc.swap".to_string(), - crate::ObservationSourceKind::HttpRpc, - transaction.source_endpoint_name.clone(), - transaction.signature.clone(), - transaction.slot, - payload_value.clone(), - )) - .await; - let observation_id = match observation_result { - Ok(observation_id) => observation_id, - Err(error) => return Err(error), - }; - let signal_result = self - .persistence - .record_signal(&crate::DetectionSignalInput::new( - "signal.dex.meteora_dbc.swap".to_string(), - crate::AnalysisSignalSeverity::Low, - transaction.signature.clone(), - Some(observation_id), - None, - payload_value, - )) - .await; - if let Err(error) = signal_result { - return Err(error); - } - } - return Ok(fetched); + return self + .materialize_named_dex_event( + transaction, + event.transaction_id, + event.instruction_id, + "meteora_dbc", + event.program_id.clone(), + "meteora_dbc.swap", + event.pool_account.clone(), + None, + event.token_a_mint.clone(), + event.token_b_mint.clone(), + None, + event.payload_json.clone(), + ) + .await; }, } } @@ -1441,136 +453,26 @@ impl DexDecodeService { ) -> Result { match decoded_event { crate::RaydiumAmmV4DecodedEvent::Initialize2Pool(event) => { - let payload_json_result = enrich_and_serialize_dex_decoded_payload( - "raydium_amm_v4", - "raydium_amm_v4.initialize2_pool", - event.payload_json.clone(), - ); - let payload_json = match payload_json_result { - Ok(payload_json) => payload_json, - Err(error) => return Err(error), - }; - let existing_result = crate::query_dex_decoded_events_get_by_key( - self.database.as_ref(), - event.transaction_id, - Some(event.instruction_id), - "raydium_amm_v4.initialize2_pool", - ) - .await; - let existing_option = match existing_result { - Ok(existing_option) => existing_option, - Err(error) => return Err(error), - }; - let already_present = existing_option.is_some(); - let dto = crate::DexDecodedEventDto::new( - event.transaction_id, - Some(event.instruction_id), - "raydium_amm_v4".to_string(), - event.program_id.clone(), - "raydium_amm_v4.initialize2_pool".to_string(), - event.pool_account.clone(), - event.lp_mint.clone(), - event.token_a_mint.clone(), - event.token_b_mint.clone(), - event.market_account.clone(), - payload_json, - ); - let upsert_result = - crate::query_dex_decoded_events_upsert(self.database.as_ref(), &dto).await; - if let Err(error) = upsert_result { - return Err(error); - } - let fetched_result = crate::query_dex_decoded_events_get_by_key( - self.database.as_ref(), - event.transaction_id, - Some(event.instruction_id), - "raydium_amm_v4.initialize2_pool", - ) - .await; - let fetched_option = match fetched_result { - Ok(fetched_option) => fetched_option, - Err(error) => return Err(error), - }; - let fetched = match fetched_option { - Some(fetched) => fetched, - None => { - return Err(crate::Error::InvalidState( - "decoded event disappeared after upsert".to_string(), - )); - }, - }; - if !already_present { - let payload_value = event.payload_json.clone(); - let observation_result = self - .persistence - .record_observation(&crate::DetectionObservationInput::new( - "dex.raydium_amm_v4.initialize2_pool".to_string(), - crate::ObservationSourceKind::HttpRpc, - transaction.source_endpoint_name.clone(), - transaction.signature.clone(), - transaction.slot, - payload_value.clone(), - )) - .await; - let observation_id = match observation_result { - Ok(observation_id) => observation_id, - Err(error) => return Err(error), - }; - let signal_result = self - .persistence - .record_signal(&crate::DetectionSignalInput::new( - "signal.dex.raydium_amm_v4.initialize2_pool".to_string(), - crate::AnalysisSignalSeverity::Low, - transaction.signature.clone(), - Some(observation_id), - None, - payload_value, - )) - .await; - if let Err(error) = signal_result { - return Err(error); - } - } - return Ok(fetched); + return self + .materialize_named_dex_event( + transaction, + event.transaction_id, + event.instruction_id, + "raydium_amm_v4", + event.program_id.clone(), + "raydium_amm_v4.initialize2_pool", + event.pool_account.clone(), + event.market_account.clone(), + event.token_a_mint.clone(), + event.token_b_mint.clone(), + event.lp_mint.clone(), + event.payload_json.clone(), + ) + .await; }, } } - async fn decode_and_persist_raydium_cpmm_events( - &self, - transaction: &crate::ChainTransactionDto, - instructions: &[crate::ChainInstructionDto], - ) -> Result, crate::Error> { - let mut persisted = std::vec::Vec::new(); - for instruction in instructions { - let program_id = match instruction.program_id.as_ref() { - Some(program_id) => program_id, - None => continue, - }; - if program_id.as_str() != crate::RAYDIUM_CPMM_PROGRAM_ID { - continue; - } - let data_json = match instruction.data_json.as_ref() { - Some(data_json) => data_json, - None => continue, - }; - let decoded_events = crate::decode_raydium_cpmm_instruction( - instruction.accounts_json.as_str(), - data_json.as_str(), - ); - for decoded_event in &decoded_events { - let persist_result = - self.persist_raydium_cpmm_event(transaction, instruction, decoded_event).await; - let persisted_event = match persist_result { - Ok(persisted_event) => persisted_event, - Err(error) => return Err(error), - }; - persisted.push(persisted_event); - } - } - return Ok(persisted); - } - async fn persist_raydium_clmm_event( &self, transaction: &crate::ChainTransactionDto, @@ -1604,107 +506,31 @@ impl DexDecodeService { )); }, }; - let payload_json_result = enrich_serialized_dex_decoded_payload( + let payload_value_result = enriched_raydium_payload_value( "raydium_clmm", event_kind.as_str(), raw_payload_json.as_str(), ); - let payload_json = match payload_json_result { - Ok(payload_json) => payload_json, + let payload_value = match payload_value_result { + Ok(payload_value) => payload_value, Err(error) => return Err(error), }; - let existing_result = crate::query_dex_decoded_events_get_by_key( - self.database.as_ref(), - transaction_id, - Some(instruction_id), - event_kind.as_str(), - ) - .await; - let existing_option = match existing_result { - Ok(existing_option) => existing_option, - Err(error) => return Err(error), - }; - let already_present = existing_option.is_some(); - let dto = crate::DexDecodedEventDto::new( - transaction_id, - Some(instruction_id), - "raydium_clmm".to_string(), - crate::RAYDIUM_CLMM_PROGRAM_ID.to_string(), - event_kind.clone(), - Some(decoded_event.pool_account().to_string()), - None, - Some(decoded_event.base_mint().to_string()), - Some(decoded_event.quote_mint().to_string()), - None, - payload_json.clone(), - ); - let upsert_result = - crate::query_dex_decoded_events_upsert(self.database.as_ref(), &dto).await; - if let Err(error) = upsert_result { - return Err(error); - } - let fetched_result = crate::query_dex_decoded_events_get_by_key( - self.database.as_ref(), - transaction_id, - Some(instruction_id), - event_kind.as_str(), - ) - .await; - let fetched_option = match fetched_result { - Ok(fetched_option) => fetched_option, - Err(error) => return Err(error), - }; - let fetched = match fetched_option { - Some(fetched) => fetched, - None => { - return Err(crate::Error::InvalidState( - "decoded raydium clmm event disappeared after upsert".to_string(), - )); - }, - }; - if !already_present { - let payload_value_result = - serde_json::from_str::(payload_json.as_str()); - let payload_value = match payload_value_result { - Ok(payload_value) => payload_value, - Err(error) => { - return Err(crate::Error::Json(format!( - "cannot parse raydium clmm payload after serialization: {}", - error - ))); - }, - }; - let observation_result = self - .persistence - .record_observation(&crate::DetectionObservationInput::new( - format!("dex.{}", event_kind), - crate::ObservationSourceKind::HttpRpc, - transaction.source_endpoint_name.clone(), - transaction.signature.clone(), - transaction.slot, - payload_value.clone(), - )) - .await; - let observation_id = match observation_result { - Ok(observation_id) => observation_id, - Err(error) => return Err(error), - }; - let signal_result = self - .persistence - .record_signal(&crate::DetectionSignalInput::new( - format!("signal.dex.{}", event_kind), - crate::AnalysisSignalSeverity::Low, - transaction.signature.clone(), - Some(observation_id), - None, - payload_value, - )) - .await; - if let Err(error) = signal_result { - return Err(error); - } - } - return Ok(fetched); + return self + .materialize_named_dex_event( + transaction, + transaction_id, + instruction_id, + "raydium_clmm", + crate::RAYDIUM_CLMM_PROGRAM_ID.to_string(), + event_kind.as_str(), + Some(decoded_event.pool_account().to_string()), + None, + Some(decoded_event.base_mint().to_string()), + Some(decoded_event.quote_mint().to_string()), + None, + payload_value, + ) + .await; } async fn persist_raydium_cpmm_event( @@ -1740,107 +566,31 @@ impl DexDecodeService { )); }, }; - let payload_json_result = enrich_serialized_dex_decoded_payload( + let payload_value_result = enriched_raydium_payload_value( "raydium_cpmm", event_kind.as_str(), raw_payload_json.as_str(), ); - let payload_json = match payload_json_result { - Ok(payload_json) => payload_json, + let payload_value = match payload_value_result { + Ok(payload_value) => payload_value, Err(error) => return Err(error), }; - let existing_result = crate::query_dex_decoded_events_get_by_key( - self.database.as_ref(), - transaction_id, - Some(instruction_id), - event_kind.as_str(), - ) - .await; - let existing_option = match existing_result { - Ok(existing_option) => existing_option, - Err(error) => return Err(error), - }; - let already_present = existing_option.is_some(); - let dto = crate::DexDecodedEventDto::new( - transaction_id, - Some(instruction_id), - "raydium_cpmm".to_string(), - crate::RAYDIUM_CPMM_PROGRAM_ID.to_string(), - event_kind.clone(), - Some(decoded_event.pool_account().to_string()), - None, - Some(decoded_event.base_mint().to_string()), - Some(decoded_event.quote_mint().to_string()), - None, - payload_json.clone(), - ); - let upsert_result = - crate::query_dex_decoded_events_upsert(self.database.as_ref(), &dto).await; - if let Err(error) = upsert_result { - return Err(error); - } - let fetched_result = crate::query_dex_decoded_events_get_by_key( - self.database.as_ref(), - transaction_id, - Some(instruction_id), - event_kind.as_str(), - ) - .await; - let fetched_option = match fetched_result { - Ok(fetched_option) => fetched_option, - Err(error) => return Err(error), - }; - let fetched = match fetched_option { - Some(fetched) => fetched, - None => { - return Err(crate::Error::InvalidState( - "decoded raydium cpmm event disappeared after upsert".to_string(), - )); - }, - }; - if !already_present { - let payload_value_result = - serde_json::from_str::(payload_json.as_str()); - let payload_value = match payload_value_result { - Ok(payload_value) => payload_value, - Err(error) => { - return Err(crate::Error::Json(format!( - "cannot parse raydium cpmm payload after serialization: {}", - error - ))); - }, - }; - let observation_result = self - .persistence - .record_observation(&crate::DetectionObservationInput::new( - format!("dex.{}", event_kind), - crate::ObservationSourceKind::HttpRpc, - transaction.source_endpoint_name.clone(), - transaction.signature.clone(), - transaction.slot, - payload_value.clone(), - )) - .await; - let observation_id = match observation_result { - Ok(observation_id) => observation_id, - Err(error) => return Err(error), - }; - let signal_result = self - .persistence - .record_signal(&crate::DetectionSignalInput::new( - format!("signal.dex.{}", event_kind), - crate::AnalysisSignalSeverity::Low, - transaction.signature.clone(), - Some(observation_id), - None, - payload_value, - )) - .await; - if let Err(error) = signal_result { - return Err(error); - } - } - return Ok(fetched); + return self + .materialize_named_dex_event( + transaction, + transaction_id, + instruction_id, + "raydium_cpmm", + crate::RAYDIUM_CPMM_PROGRAM_ID.to_string(), + event_kind.as_str(), + Some(decoded_event.pool_account().to_string()), + None, + Some(decoded_event.base_mint().to_string()), + Some(decoded_event.quote_mint().to_string()), + None, + payload_value, + ) + .await; } async fn persist_pump_fun_event( @@ -1850,97 +600,22 @@ impl DexDecodeService { ) -> Result { match decoded_event { crate::PumpFunDecodedEvent::CreateV2Token(event) => { - let payload_json_result = enrich_and_serialize_dex_decoded_payload( - "pump_fun", - "pump_fun.create_v2_token", - event.payload_json.clone(), - ); - let payload_json = match payload_json_result { - Ok(payload_json) => payload_json, - Err(error) => return Err(error), - }; - let existing_result = crate::query_dex_decoded_events_get_by_key( - self.database.as_ref(), - event.transaction_id, - Some(event.instruction_id), - "pump_fun.create_v2_token", - ) - .await; - let existing_option = match existing_result { - Ok(existing_option) => existing_option, - Err(error) => return Err(error), - }; - let already_present = existing_option.is_some(); - let dto = crate::DexDecodedEventDto::new( - event.transaction_id, - Some(event.instruction_id), - "pump_fun".to_string(), - event.program_id.clone(), - "pump_fun.create_v2_token".to_string(), - event.bonding_curve.clone(), - None, - event.mint.clone(), - Some(crate::WSOL_MINT_ID.to_string()), - event.associated_bonding_curve.clone(), - payload_json, - ); - let upsert_result = - crate::query_dex_decoded_events_upsert(self.database.as_ref(), &dto).await; - if let Err(error) = upsert_result { - return Err(error); - } - let fetched_result = crate::query_dex_decoded_events_get_by_key( - self.database.as_ref(), - event.transaction_id, - Some(event.instruction_id), - "pump_fun.create_v2_token", - ) - .await; - let fetched_option = match fetched_result { - Ok(fetched_option) => fetched_option, - Err(error) => return Err(error), - }; - let fetched = match fetched_option { - Some(fetched) => fetched, - None => { - return Err(crate::Error::InvalidState( - "decoded event disappeared after upsert".to_string(), - )); - }, - }; - if !already_present { - let payload_value = event.payload_json.clone(); - let observation_result = self - .persistence - .record_observation(&crate::DetectionObservationInput::new( - "dex.pump_fun.create_v2_token".to_string(), - crate::ObservationSourceKind::HttpRpc, - transaction.source_endpoint_name.clone(), - transaction.signature.clone(), - transaction.slot, - payload_value.clone(), - )) - .await; - let observation_id = match observation_result { - Ok(observation_id) => observation_id, - Err(error) => return Err(error), - }; - let signal_result = self - .persistence - .record_signal(&crate::DetectionSignalInput::new( - "signal.dex.pump_fun.create_v2_token".to_string(), - crate::AnalysisSignalSeverity::Low, - transaction.signature.clone(), - Some(observation_id), - None, - payload_value, - )) - .await; - if let Err(error) = signal_result { - return Err(error); - } - } - return Ok(fetched); + return self + .materialize_named_dex_event( + transaction, + event.transaction_id, + event.instruction_id, + "pump_fun", + event.program_id.clone(), + "pump_fun.create_v2_token", + event.bonding_curve.clone(), + None, + event.mint.clone(), + Some(crate::WSOL_MINT_ID.to_string()), + event.associated_bonding_curve.clone(), + event.payload_json.clone(), + ) + .await; }, crate::PumpFunDecodedEvent::BuyTrade(event) => { return self @@ -1975,97 +650,29 @@ impl DexDecodeService { signal_kind: &str, observation_kind: &str, ) -> Result { - let payload_json_result = enrich_and_serialize_dex_decoded_payload( - "pump_fun", - event_kind, - event.payload_json.clone(), - ); - let payload_json = match payload_json_result { - Ok(payload_json) => payload_json, - Err(error) => return Err(error), + let input = crate::dex_decoded_event_materialization::DexDecodedEventMaterializationInput { + database: self.database.as_ref(), + persistence: &self.persistence, + transaction, + transaction_id: event.transaction_id, + instruction_id: Some(event.instruction_id), + protocol_name: "pump_fun".to_string(), + program_id: event.program_id.clone(), + event_kind: event_kind.to_string(), + pool_account: event.bonding_curve.clone(), + market_account: None, + token_a_mint: event.mint.clone(), + token_b_mint: Some(crate::WSOL_MINT_ID.to_string()), + lp_mint: event.associated_bonding_curve.clone(), + enrichment_payload_json: event.payload_json.clone(), + observation_payload_json: event.payload_json.clone(), + observation_kind: observation_kind.to_string(), + signal_kind: signal_kind.to_string(), + missing_after_upsert_message: "decoded pump.fun trade event disappeared after upsert" + .to_string(), }; - let existing_result = crate::query_dex_decoded_events_get_by_key( - self.database.as_ref(), - event.transaction_id, - Some(event.instruction_id), - event_kind, - ) - .await; - let existing_option = match existing_result { - Ok(existing_option) => existing_option, - Err(error) => return Err(error), - }; - let already_present = existing_option.is_some(); - let dto = crate::DexDecodedEventDto::new( - event.transaction_id, - Some(event.instruction_id), - "pump_fun".to_string(), - event.program_id.clone(), - event_kind.to_string(), - event.bonding_curve.clone(), - None, - event.mint.clone(), - Some(crate::WSOL_MINT_ID.to_string()), - event.associated_bonding_curve.clone(), - payload_json, - ); - let upsert_result = - crate::query_dex_decoded_events_upsert(self.database.as_ref(), &dto).await; - if let Err(error) = upsert_result { - return Err(error); - } - let fetched_result = crate::query_dex_decoded_events_get_by_key( - self.database.as_ref(), - event.transaction_id, - Some(event.instruction_id), - event_kind, - ) - .await; - let fetched_option = match fetched_result { - Ok(fetched_option) => fetched_option, - Err(error) => return Err(error), - }; - let fetched = match fetched_option { - Some(fetched) => fetched, - None => { - return Err(crate::Error::InvalidState( - "decoded pump.fun trade event disappeared after upsert".to_string(), - )); - }, - }; - if !already_present { - let payload_value = event.payload_json.clone(); - let observation_result = self - .persistence - .record_observation(&crate::DetectionObservationInput::new( - observation_kind.to_string(), - crate::ObservationSourceKind::HttpRpc, - transaction.source_endpoint_name.clone(), - transaction.signature.clone(), - transaction.slot, - payload_value.clone(), - )) - .await; - let observation_id = match observation_result { - Ok(observation_id) => observation_id, - Err(error) => return Err(error), - }; - let signal_result = self - .persistence - .record_signal(&crate::DetectionSignalInput::new( - signal_kind.to_string(), - crate::AnalysisSignalSeverity::Low, - transaction.signature.clone(), - Some(observation_id), - None, - payload_value, - )) - .await; - if let Err(error) = signal_result { - return Err(error); - } - } - return Ok(fetched); + return crate::dex_decoded_event_materialization::materialize_dex_decoded_event(input) + .await; } async fn persist_pump_swap_event( @@ -2107,326 +714,346 @@ impl DexDecodeService { signal_kind: &str, observation_kind: &str, ) -> Result { - let payload_value = prepare_pump_swap_trade_payload_for_classification(event); - let payload_json_result = - enrich_and_serialize_dex_decoded_payload("pump_swap", event_kind, payload_value); - let payload_json = match payload_json_result { - Ok(payload_json) => payload_json, - Err(error) => return Err(error), + let enrichment_payload_json = prepare_pump_swap_trade_payload_for_classification(event); + let input = crate::dex_decoded_event_materialization::DexDecodedEventMaterializationInput { + database: self.database.as_ref(), + persistence: &self.persistence, + transaction, + transaction_id: event.transaction_id, + instruction_id: Some(event.instruction_id), + protocol_name: "pump_swap".to_string(), + program_id: event.program_id.clone(), + event_kind: event_kind.to_string(), + pool_account: event.pool_account.clone(), + market_account: None, + token_a_mint: event.token_a_mint.clone(), + token_b_mint: event.token_b_mint.clone(), + lp_mint: event.pool_v2.clone(), + enrichment_payload_json, + observation_payload_json: event.payload_json.clone(), + observation_kind: observation_kind.to_string(), + signal_kind: signal_kind.to_string(), + missing_after_upsert_message: "decoded event disappeared after upsert".to_string(), }; - let existing_result = crate::query_dex_decoded_events_get_by_key( - self.database.as_ref(), - event.transaction_id, - Some(event.instruction_id), - event_kind, - ) - .await; - let existing_option = match existing_result { - Ok(existing_option) => existing_option, - Err(error) => return Err(error), - }; - let already_present = existing_option.is_some(); - let dto = crate::DexDecodedEventDto::new( - event.transaction_id, - Some(event.instruction_id), - "pump_swap".to_string(), - event.program_id.clone(), - event_kind.to_string(), - event.pool_account.clone(), - None, - event.token_a_mint.clone(), - event.token_b_mint.clone(), - event.pool_v2.clone(), - payload_json, - ); - let upsert_result = - crate::query_dex_decoded_events_upsert(self.database.as_ref(), &dto).await; - if let Err(error) = upsert_result { - return Err(error); - } - let fetched_result = crate::query_dex_decoded_events_get_by_key( - self.database.as_ref(), - event.transaction_id, - Some(event.instruction_id), - event_kind, - ) - .await; - let fetched_option = match fetched_result { - Ok(fetched_option) => fetched_option, - Err(error) => return Err(error), - }; - let fetched = match fetched_option { - Some(fetched) => fetched, - None => { - return Err(crate::Error::InvalidState( - "decoded event disappeared after upsert".to_string(), - )); - }, - }; - if !already_present { - let payload_value = event.payload_json.clone(); - let observation_result = self - .persistence - .record_observation(&crate::DetectionObservationInput::new( - observation_kind.to_string(), - crate::ObservationSourceKind::HttpRpc, - transaction.source_endpoint_name.clone(), - transaction.signature.clone(), - transaction.slot, - payload_value.clone(), - )) - .await; - let observation_id = match observation_result { - Ok(observation_id) => observation_id, - Err(error) => return Err(error), + return crate::dex_decoded_event_materialization::materialize_dex_decoded_event(input) + .await; + } + + async fn decode_and_persist_raydium_cpmm_events( + &self, + transaction: &crate::ChainTransactionDto, + instructions: &[crate::ChainInstructionDto], + ) -> Result, crate::Error> { + let mut persisted = std::vec::Vec::new(); + for instruction in instructions { + let program_id = match instruction.program_id.as_ref() { + Some(program_id) => program_id, + None => continue, }; - let signal_result = self - .persistence - .record_signal(&crate::DetectionSignalInput::new( - signal_kind.to_string(), - crate::AnalysisSignalSeverity::Low, - transaction.signature.clone(), - Some(observation_id), - None, - payload_value, - )) - .await; - if let Err(error) = signal_result { - return Err(error); + if program_id.as_str() != crate::RAYDIUM_CPMM_PROGRAM_ID { + continue; + } + let data_json = match instruction.data_json.as_ref() { + Some(data_json) => data_json, + None => continue, + }; + let decoded_events = crate::decode_raydium_cpmm_instruction( + instruction.accounts_json.as_str(), + data_json.as_str(), + ); + for decoded_event in &decoded_events { + let persist_result = + self.persist_raydium_cpmm_event(transaction, instruction, decoded_event).await; + let persisted_event = match persist_result { + Ok(persisted_event) => persisted_event, + Err(error) => return Err(error), + }; + persisted.push(persisted_event); } } - return Ok(fetched); + return Ok(persisted); + } + + async fn decode_and_persist_raydium_clmm_events( + &self, + transaction: &crate::ChainTransactionDto, + instructions: &[crate::ChainInstructionDto], + ) -> Result, crate::Error> { + let mut persisted = std::vec::Vec::new(); + for instruction in instructions { + let program_id = match instruction.program_id.as_ref() { + Some(program_id) => program_id, + None => continue, + }; + if program_id.as_str() != crate::RAYDIUM_CLMM_PROGRAM_ID { + continue; + } + let data_json = match instruction.data_json.as_ref() { + Some(data_json) => data_json, + None => continue, + }; + let decoded_events = crate::decode_raydium_clmm_instruction( + instruction.accounts_json.as_str(), + data_json.as_str(), + ); + for decoded_event in &decoded_events { + let persist_result = + self.persist_raydium_clmm_event(transaction, instruction, decoded_event).await; + let persisted_event = match persist_result { + Ok(persisted_event) => persisted_event, + Err(error) => return Err(error), + }; + persisted.push(persisted_event); + } + } + return Ok(persisted); + } + + async fn decode_and_persist_raydium_amm_v4_events( + &self, + transaction: &crate::ChainTransactionDto, + instructions: &[crate::ChainInstructionDto], + ) -> Result, crate::Error> { + let decoded_result = + self.raydium_amm_v4_decoder.decode_transaction(transaction, instructions); + let decoded_events = match decoded_result { + Ok(decoded_events) => decoded_events, + Err(error) => return Err(error), + }; + let mut persisted = std::vec::Vec::new(); + for decoded_event in &decoded_events { + let persist_result = + self.persist_raydium_amm_v4_event(transaction, decoded_event).await; + let persisted_event = match persist_result { + Ok(persisted_event) => persisted_event, + Err(error) => return Err(error), + }; + persisted.push(persisted_event); + } + return Ok(persisted); + } + + async fn decode_and_persist_pump_fun_events( + &self, + transaction: &crate::ChainTransactionDto, + instructions: &[crate::ChainInstructionDto], + ) -> Result, crate::Error> { + let decoded_result = self.pump_fun_decoder.decode_transaction(transaction, instructions); + let decoded_events = match decoded_result { + Ok(decoded_events) => decoded_events, + Err(error) => return Err(error), + }; + let mut persisted = std::vec::Vec::new(); + for decoded_event in &decoded_events { + let persist_result = self.persist_pump_fun_event(transaction, decoded_event).await; + let persisted_event = match persist_result { + Ok(persisted_event) => persisted_event, + Err(error) => return Err(error), + }; + persisted.push(persisted_event); + } + return Ok(persisted); + } + + async fn decode_and_persist_pump_swap_events( + &self, + transaction: &crate::ChainTransactionDto, + instructions: &[crate::ChainInstructionDto], + ) -> Result, crate::Error> { + let decoded_result = self.pump_swap_decoder.decode_transaction(transaction, instructions); + let decoded_events = match decoded_result { + Ok(decoded_events) => decoded_events, + Err(error) => return Err(error), + }; + let mut persisted = std::vec::Vec::new(); + for decoded_event in &decoded_events { + let persist_result = self.persist_pump_swap_event(transaction, decoded_event).await; + let persisted_event = match persist_result { + Ok(persisted_event) => persisted_event, + Err(error) => return Err(error), + }; + + persisted.push(persisted_event); + } + return Ok(persisted); + } + + async fn decode_and_persist_meteora_dbc_events( + &self, + transaction: &crate::ChainTransactionDto, + instructions: &[crate::ChainInstructionDto], + ) -> Result, crate::Error> { + let decoded_result = self.meteora_dbc_decoder.decode_transaction(transaction, instructions); + let decoded_events = match decoded_result { + Ok(decoded_events) => decoded_events, + Err(error) => return Err(error), + }; + let mut persisted = std::vec::Vec::new(); + for decoded_event in &decoded_events { + let persist_result = self.persist_meteora_dbc_event(transaction, decoded_event).await; + let persisted_event = match persist_result { + Ok(persisted_event) => persisted_event, + Err(error) => return Err(error), + }; + persisted.push(persisted_event); + } + return Ok(persisted); + } + + async fn decode_and_persist_meteora_damm_v1_events( + &self, + transaction: &crate::ChainTransactionDto, + instructions: &[crate::ChainInstructionDto], + ) -> Result, crate::Error> { + let decoded_result = + self.meteora_damm_v1_decoder.decode_transaction(transaction, instructions); + let decoded_events = match decoded_result { + Ok(decoded_events) => decoded_events, + Err(error) => return Err(error), + }; + let mut persisted = std::vec::Vec::new(); + for decoded_event in &decoded_events { + let persist_result = + self.persist_meteora_damm_v1_event(transaction, decoded_event).await; + let persisted_event = match persist_result { + Ok(persisted_event) => persisted_event, + Err(error) => return Err(error), + }; + persisted.push(persisted_event); + } + return Ok(persisted); + } + + async fn decode_and_persist_meteora_damm_v2_events( + &self, + transaction: &crate::ChainTransactionDto, + instructions: &[crate::ChainInstructionDto], + ) -> Result, crate::Error> { + let decoded_result = + self.meteora_damm_v2_decoder.decode_transaction(transaction, instructions); + let decoded_events = match decoded_result { + Ok(decoded_events) => decoded_events, + Err(error) => return Err(error), + }; + let mut persisted = std::vec::Vec::new(); + for decoded_event in &decoded_events { + let persist_result = + self.persist_meteora_damm_v2_event(transaction, decoded_event).await; + let persisted_event = match persist_result { + Ok(persisted_event) => persisted_event, + Err(error) => return Err(error), + }; + persisted.push(persisted_event); + } + return Ok(persisted); + } + + async fn decode_and_persist_orca_whirlpools_events( + &self, + transaction: &crate::ChainTransactionDto, + instructions: &[crate::ChainInstructionDto], + ) -> Result, crate::Error> { + let decoded_result = + self.orca_whirlpools_decoder.decode_transaction(transaction, instructions); + let decoded_events = match decoded_result { + Ok(decoded_events) => decoded_events, + Err(error) => return Err(error), + }; + let mut persisted = std::vec::Vec::new(); + for decoded_event in &decoded_events { + let persist_result = + self.persist_orca_whirlpools_event(transaction, decoded_event).await; + let persisted_event = match persist_result { + Ok(persisted_event) => persisted_event, + Err(error) => return Err(error), + }; + persisted.push(persisted_event); + } + return Ok(persisted); + } + + async fn decode_and_persist_fluxbeam_events( + &self, + transaction: &crate::ChainTransactionDto, + instructions: &[crate::ChainInstructionDto], + ) -> Result, crate::Error> { + let decoded_result = self.fluxbeam_decoder.decode_transaction(transaction, instructions); + let decoded_events = match decoded_result { + Ok(decoded_events) => decoded_events, + Err(error) => return Err(error), + }; + let mut persisted = std::vec::Vec::new(); + for decoded_event in &decoded_events { + let persist_result = self.persist_fluxbeam_event(transaction, decoded_event).await; + let persisted_event = match persist_result { + Ok(persisted_event) => persisted_event, + Err(error) => return Err(error), + }; + persisted.push(persisted_event); + } + return Ok(persisted); + } + + async fn decode_and_persist_dexlab_events( + &self, + transaction: &crate::ChainTransactionDto, + instructions: &[crate::ChainInstructionDto], + ) -> Result, crate::Error> { + let decoded_result = self.dexlab_decoder.decode_transaction(transaction, instructions); + let decoded_events = match decoded_result { + Ok(decoded_events) => decoded_events, + Err(error) => return Err(error), + }; + let mut persisted = std::vec::Vec::new(); + for decoded_event in &decoded_events { + let persist_result = self.persist_dexlab_event(transaction, decoded_event).await; + let persisted_event = match persist_result { + Ok(persisted_event) => persisted_event, + Err(error) => return Err(error), + }; + persisted.push(persisted_event); + } + return Ok(persisted); } } -// Classifies a DEX event kind into a stable business category. -fn classify_dex_event_category(event_kind: &str) -> &'static str { - if is_dex_reward_event_kind(event_kind) { - return "reward"; +fn append_persisted_events( + target: &mut std::vec::Vec, + source: std::vec::Vec, +) { + for persisted_event in source { + target.push(persisted_event); } - if is_dex_fee_event_kind(event_kind) { - return "fee"; - } - if is_dex_liquidity_event_kind(event_kind) { - return "liquidity"; - } - if is_dex_pool_lifecycle_event_kind(event_kind) { - return "pool_lifecycle"; - } - if is_dex_admin_event_kind(event_kind) { - return "admin"; - } - if is_dex_trade_event_kind(event_kind) { - return "trade"; - } - return "unknown"; } -// Returns true when the event kind represents a swap-like event. -fn is_dex_trade_event_kind(event_kind: &str) -> bool { - if event_kind.ends_with(".buy") { - return true; - } - if event_kind.ends_with(".sell") { - return true; - } - if event_kind.ends_with(".swap") { - return true; - } - if event_kind.contains(".swap_") { - return true; - } - return false; -} - -// Returns true when the event kind can directly produce a candle candidate. -fn is_dex_candle_candidate_event_kind(event_kind: &str) -> bool { - if event_kind.contains("router") { - return false; - } - if event_kind.contains("route") { - return false; - } - return is_dex_trade_event_kind(event_kind); -} - -// Returns true for liquidity lifecycle changes that must not become candles. -fn is_dex_liquidity_event_kind(event_kind: &str) -> bool { - if event_kind.contains(".deposit") { - return true; - } - if event_kind.contains(".withdraw") { - return true; - } - if event_kind.contains(".increase_liquidity") { - return true; - } - if event_kind.contains(".decrease_liquidity") { - return true; - } - if event_kind.contains(".open_position") { - return true; - } - if event_kind.contains(".close_position") { - return true; - } - return false; -} - -// Returns true for fee collection events. -fn is_dex_fee_event_kind(event_kind: &str) -> bool { - if event_kind.contains("collect_creator_fee") { - return true; - } - if event_kind.contains("collect_protocol_fee") { - return true; - } - if event_kind.contains("collect_fund_fee") { - return true; - } - if event_kind.contains("collect_fee") { - return true; - } - return false; -} - -// Returns true for reward/incentive events. -fn is_dex_reward_event_kind(event_kind: &str) -> bool { - if event_kind.contains("reward") { - return true; - } - if event_kind.contains("emission") { - return true; - } - return false; -} - -// Returns true for pool creation / initialization / migration events. -fn is_dex_pool_lifecycle_event_kind(event_kind: &str) -> bool { - if event_kind.contains(".initialize") { - return true; - } - if event_kind.contains(".initialize_with_permission") { - return true; - } - if event_kind.contains(".create_pool") { - return true; - } - if event_kind.contains(".create_v2_token") { - return true; - } - if event_kind.contains(".migrate") { - return true; - } - return false; -} - -// Returns true for admin/config/permission changes. -fn is_dex_admin_event_kind(event_kind: &str) -> bool { - if event_kind.contains("admin") { - return true; - } - if event_kind.contains("config") { - return true; - } - if event_kind.contains("permission") { - return true; - } - if event_kind.contains("set_") { - return true; - } - if event_kind.contains("update_") { - return true; - } - return false; -} - -// Enriches a decoded payload with non-destructive classification metadata. -fn enrich_dex_decoded_payload( - protocol_name: &str, - event_kind: &str, - payload_json: serde_json::Value, -) -> serde_json::Value { - let event_category = classify_dex_event_category(event_kind); - let trade_candidate = is_dex_trade_event_kind(event_kind); - let candle_candidate = is_dex_candle_candidate_event_kind(event_kind); - let mut object = match payload_json { - serde_json::Value::Object(object) => object, - other => { - let mut object = serde_json::Map::new(); - object.insert("rawPayload".to_owned(), other); - object - }, +fn append_persisted_events_result( + target: &mut std::vec::Vec, + source_result: Result, crate::Error>, +) -> Result<(), crate::Error> { + let source = match source_result { + Ok(source) => source, + Err(error) => return Err(error), }; - json_insert_string_if_missing(&mut object, "protocolName", protocol_name); - json_insert_string_if_missing(&mut object, "eventKind", event_kind); - json_insert_string_if_missing(&mut object, "eventCategory", event_category); - json_insert_bool_if_missing(&mut object, "tradeCandidate", trade_candidate); - json_insert_bool_if_missing(&mut object, "candleCandidate", candle_candidate); - json_insert_i64_if_missing(&mut object, "eventClassificationVersion", 1); - if !trade_candidate { - json_insert_string_if_missing(&mut object, "skipTradeReason", "non_trade_event"); - } else if !candle_candidate { - json_insert_string_if_missing( - &mut object, - "skipCandleReason", - "route_or_multihop_event_requires_leg_resolution", - ); - } - return serde_json::Value::Object(object); + append_persisted_events(target, source); + return Ok(()); } -// Inserts a string JSON property without overriding existing decoded data. -fn json_insert_string_if_missing( - object: &mut serde_json::Map, - key: &str, - value: &str, -) { - if object.contains_key(key) { - return; - } - object.insert(key.to_owned(), serde_json::Value::String(value.to_owned())); -} - -// Inserts a bool JSON property without overriding existing decoded data. -fn json_insert_bool_if_missing( - object: &mut serde_json::Map, - key: &str, - value: bool, -) { - if object.contains_key(key) { - return; - } - object.insert(key.to_owned(), serde_json::Value::Bool(value)); -} - -// Inserts an i64 JSON property without overriding existing decoded data. -fn json_insert_i64_if_missing( - object: &mut serde_json::Map, - key: &str, - value: i64, -) { - if object.contains_key(key) { - return; - } - object.insert(key.to_owned(), serde_json::Value::Number(serde_json::Number::from(value))); -} -fn enrich_and_serialize_dex_decoded_payload( +fn enriched_raydium_payload_value( protocol_name: &str, event_kind: &str, - payload_json: serde_json::Value, -) -> Result { - let enriched_payload = enrich_dex_decoded_payload(protocol_name, event_kind, payload_json); - let payload_json_result = serde_json::to_string(&enriched_payload); - match payload_json_result { - Ok(payload_json) => return Ok(payload_json), + raw_payload_json: &str, +) -> Result { + let payload_value_result = serde_json::from_str::(raw_payload_json); + let payload_value = match payload_value_result { + Ok(payload_value) => payload_value, Err(error) => { return Err(crate::Error::Json(format!( - "cannot serialize enriched decoded payload for '{}': {}", - event_kind, error + "cannot parse decoded {} payload for '{}': {}", + protocol_name, event_kind, error ))); }, - } + }; + return Ok(crate::enrich_dex_decoded_payload(protocol_name, event_kind, payload_value)); } // Marks incomplete PumpSwap decoded trades as non-materializable candidates before generic @@ -2459,24 +1086,6 @@ fn prepare_pump_swap_trade_payload_for_classification( return serde_json::Value::Object(object); } -fn enrich_serialized_dex_decoded_payload( - protocol_name: &str, - event_kind: &str, - payload_json: &str, -) -> Result { - let payload_value_result = serde_json::from_str::(payload_json); - let payload_value = match payload_value_result { - Ok(payload_value) => payload_value, - Err(error) => { - return Err(crate::Error::Json(format!( - "cannot parse decoded payload for '{}': {}", - event_kind, error - ))); - }, - }; - return enrich_and_serialize_dex_decoded_payload(protocol_name, event_kind, payload_value); -} - #[cfg(test)] mod tests { @@ -3185,32 +1794,53 @@ mod tests { #[test] fn classifies_swap_events_as_trade_candidates() { - assert_eq!(super::classify_dex_event_category("raydium_cpmm.swap_base_input"), "trade"); - assert_eq!(super::classify_dex_event_category("raydium_cpmm.swap_base_output"), "trade"); - assert_eq!(super::classify_dex_event_category("raydium_clmm.swap"), "trade"); - assert_eq!(super::classify_dex_event_category("raydium_clmm.swap_v2"), "trade"); - assert_eq!(super::classify_dex_event_category("pump_fun.buy"), "trade"); - assert!(super::is_dex_trade_event_kind("raydium_cpmm.swap_base_input")); - assert!(super::is_dex_candle_candidate_event_kind("raydium_cpmm.swap_base_input")); + assert_eq!( + crate::classify_dex_event_category_code("raydium_cpmm.swap_base_input"), + "trade" + ); + assert_eq!( + crate::classify_dex_event_category_code("raydium_cpmm.swap_base_output"), + "trade" + ); + assert_eq!(crate::classify_dex_event_category_code("raydium_clmm.swap"), "trade"); + assert_eq!(crate::classify_dex_event_category_code("raydium_clmm.swap_v2"), "trade"); + assert_eq!(crate::classify_dex_event_category_code("pump_fun.buy"), "trade"); + assert!(crate::is_dex_trade_event_kind("raydium_cpmm.swap_base_input")); + assert!(crate::is_dex_candle_candidate_event_kind("raydium_cpmm.swap_base_input")); } #[test] fn classifies_router_swap_as_trade_but_not_direct_candle_candidate() { - assert_eq!(super::classify_dex_event_category("raydium_clmm.swap_router_base_in"), "trade"); - assert!(super::is_dex_trade_event_kind("raydium_clmm.swap_router_base_in")); - assert!(!super::is_dex_candle_candidate_event_kind("raydium_clmm.swap_router_base_in")); + assert_eq!( + crate::classify_dex_event_category_code("raydium_clmm.swap_router_base_in"), + "trade" + ); + assert!(crate::is_dex_trade_event_kind("raydium_clmm.swap_router_base_in")); + assert!(!crate::is_dex_candle_candidate_event_kind("raydium_clmm.swap_router_base_in")); } #[test] fn classifies_fee_reward_liquidity_and_lifecycle_events() { - assert_eq!(super::classify_dex_event_category("raydium_cpmm.collect_creator_fee"), "fee"); - assert_eq!(super::classify_dex_event_category("raydium_clmm.collect_protocol_fee"), "fee"); - assert_eq!(super::classify_dex_event_category("raydium_clmm.set_reward_params"), "reward"); assert_eq!( - super::classify_dex_event_category("raydium_clmm.increase_liquidity_v2"), + crate::classify_dex_event_category_code("raydium_cpmm.collect_creator_fee"), + "fee" + ); + assert_eq!( + crate::classify_dex_event_category_code("raydium_clmm.collect_protocol_fee"), + "fee" + ); + assert_eq!( + crate::classify_dex_event_category_code("raydium_clmm.set_reward_params"), + "reward" + ); + assert_eq!( + crate::classify_dex_event_category_code("raydium_clmm.increase_liquidity_v2"), "liquidity" ); - assert_eq!(super::classify_dex_event_category("raydium_cpmm.initialize"), "pool_lifecycle"); + assert_eq!( + crate::classify_dex_event_category_code("raydium_cpmm.initialize"), + "pool_lifecycle" + ); } #[test] @@ -3219,7 +1849,7 @@ mod tests { "eventCategory": "custom", "amountIn": "10" }); - let enriched_payload = super::enrich_dex_decoded_payload( + let enriched_payload = crate::enrich_dex_decoded_payload( "raydium_cpmm", "raydium_cpmm.swap_base_input", payload_json, @@ -3250,7 +1880,7 @@ mod tests { #[test] fn enriches_non_object_payload_as_raw_payload() { let payload_json = serde_json::Value::String("raw".to_owned()); - let enriched_payload = super::enrich_dex_decoded_payload( + let enriched_payload = crate::enrich_dex_decoded_payload( "raydium_clmm", "raydium_clmm.collect_protocol_fee", payload_json, diff --git a/kb_lib/src/dex_decode_context.rs b/kb_lib/src/dex_decode_context.rs new file mode 100644 index 0000000..b6d2cdb --- /dev/null +++ b/kb_lib/src/dex_decode_context.rs @@ -0,0 +1,55 @@ +// file: kb_lib/src/dex_decode_context.rs + +//! Transaction context loading for DEX decoding. +//! +//! This module loads the persisted transaction and projected instructions +//! required by `DexDecodeService`. + +/// Transaction context required by DEX decoding. +pub(crate) struct DexDecodeTransactionContext { + /// Persisted chain transaction. + pub(crate) transaction: crate::ChainTransactionDto, + /// Projected transaction instructions. + pub(crate) instructions: std::vec::Vec, +} + +/// Loads the transaction and its projected instructions for DEX decoding. +pub(crate) async fn load_dex_decode_transaction_context( + database: &crate::Database, + signature: &str, +) -> Result { + let transaction_result = + crate::query_chain_transactions_get_by_signature(database, signature).await; + let transaction_option = match transaction_result { + Ok(transaction_option) => transaction_option, + Err(error) => return Err(error), + }; + let transaction = match transaction_option { + Some(transaction) => transaction, + None => { + return Err(crate::Error::InvalidState(format!( + "cannot decode unknown chain transaction '{}'", + signature + ))); + }, + }; + let transaction_id = match transaction.id { + Some(transaction_id) => transaction_id, + None => { + return Err(crate::Error::InvalidState(format!( + "chain transaction '{}' has no internal id", + signature + ))); + }, + }; + let instructions_result = + crate::query_chain_instructions_list_by_transaction_id(database, transaction_id).await; + let instructions = match instructions_result { + Ok(instructions) => instructions, + Err(error) => return Err(error), + }; + return Ok(crate::dex_decode_context::DexDecodeTransactionContext { + transaction, + instructions, + }); +} diff --git a/kb_lib/src/dex_decoded_event_materialization.rs b/kb_lib/src/dex_decoded_event_materialization.rs new file mode 100644 index 0000000..6401a69 --- /dev/null +++ b/kb_lib/src/dex_decoded_event_materialization.rs @@ -0,0 +1,140 @@ +// file: kb_lib/src/dex_decoded_event_materialization.rs + +//! Decoded DEX event materialization helpers. +//! +//! This module centralizes persistence of decoded DEX events: +//! payload enrichment, upsert, fetch-after-upsert, observation recording +//! and signal recording. + +/// Input required to persist one decoded DEX event. +pub(crate) struct DexDecodedEventMaterializationInput<'a> { + /// Database connection. + pub(crate) database: &'a crate::Database, + /// Detection persistence service. + pub(crate) persistence: &'a crate::DetectionPersistenceService, + /// Parent transaction. + pub(crate) transaction: &'a crate::ChainTransactionDto, + /// Internal transaction id. + pub(crate) transaction_id: i64, + /// Optional internal instruction id. + pub(crate) instruction_id: std::option::Option, + /// Stable protocol name. + pub(crate) protocol_name: std::string::String, + /// Program id that produced the event. + pub(crate) program_id: std::string::String, + /// Stable decoded event kind. + pub(crate) event_kind: std::string::String, + /// Optional pool account. + pub(crate) pool_account: std::option::Option, + /// Optional market account. + pub(crate) market_account: std::option::Option, + /// Optional token A mint. + pub(crate) token_a_mint: std::option::Option, + /// Optional token B mint. + pub(crate) token_b_mint: std::option::Option, + /// Optional LP mint or protocol-specific secondary mint. + pub(crate) lp_mint: std::option::Option, + /// Payload used for classification enrichment and DB storage. + pub(crate) enrichment_payload_json: serde_json::Value, + /// Payload recorded in the detection observation. + pub(crate) observation_payload_json: serde_json::Value, + /// Detection observation kind. + pub(crate) observation_kind: std::string::String, + /// Detection signal kind. + pub(crate) signal_kind: std::string::String, + /// Diagnostic message emitted when fetch-after-upsert fails. + pub(crate) missing_after_upsert_message: std::string::String, +} + +/// Persists one decoded DEX event and records its first-seen observation/signal. +pub(crate) async fn materialize_dex_decoded_event( + input: crate::dex_decoded_event_materialization::DexDecodedEventMaterializationInput<'_>, +) -> Result { + let payload_json_result = crate::enrich_and_serialize_dex_decoded_payload( + input.protocol_name.as_str(), + input.event_kind.as_str(), + input.enrichment_payload_json, + ); + let payload_json = match payload_json_result { + Ok(payload_json) => payload_json, + Err(error) => return Err(error), + }; + let existing_result = crate::query_dex_decoded_events_get_by_key( + input.database, + input.transaction_id, + input.instruction_id, + input.event_kind.as_str(), + ) + .await; + let existing_option = match existing_result { + Ok(existing_option) => existing_option, + Err(error) => return Err(error), + }; + let already_present = existing_option.is_some(); + let dto = crate::DexDecodedEventDto::new( + input.transaction_id, + input.instruction_id, + input.protocol_name, + input.program_id, + input.event_kind.clone(), + input.pool_account, + input.market_account, + input.token_a_mint, + input.token_b_mint, + input.lp_mint, + payload_json, + ); + let upsert_result = crate::query_dex_decoded_events_upsert(input.database, &dto).await; + if let Err(error) = upsert_result { + return Err(error); + } + let fetched_result = crate::query_dex_decoded_events_get_by_key( + input.database, + input.transaction_id, + input.instruction_id, + input.event_kind.as_str(), + ) + .await; + let fetched_option = match fetched_result { + Ok(fetched_option) => fetched_option, + Err(error) => return Err(error), + }; + let fetched = match fetched_option { + Some(fetched) => fetched, + None => { + return Err(crate::Error::InvalidState(input.missing_after_upsert_message)); + }, + }; + if !already_present { + let observation_result = input + .persistence + .record_observation(&crate::DetectionObservationInput::new( + input.observation_kind, + crate::ObservationSourceKind::HttpRpc, + input.transaction.source_endpoint_name.clone(), + input.transaction.signature.clone(), + input.transaction.slot, + input.observation_payload_json.clone(), + )) + .await; + let observation_id = match observation_result { + Ok(observation_id) => observation_id, + Err(error) => return Err(error), + }; + let signal_result = input + .persistence + .record_signal(&crate::DetectionSignalInput::new( + input.signal_kind, + crate::AnalysisSignalSeverity::Low, + input.transaction.signature.clone(), + Some(observation_id), + None, + input.observation_payload_json, + )) + .await; + if let Err(error) = signal_result { + return Err(error); + } + } + return Ok(fetched); +} diff --git a/kb_lib/src/dex_detect.rs b/kb_lib/src/dex_detect.rs index dc11355..ceef9db 100644 --- a/kb_lib/src/dex_detect.rs +++ b/kb_lib/src/dex_detect.rs @@ -79,231 +79,63 @@ impl DexDetectService { }; let mut detection_results = std::vec::Vec::new(); for decoded_event in &decoded_events { - if decoded_event.protocol_name == "raydium_amm_v4" - && decoded_event.event_kind == "raydium_amm_v4.initialize2_pool" - { - let detect_result = - self.detect_raydium_initialize2_pool(&transaction, decoded_event).await; - let detect_result = match detect_result { - Ok(detect_result) => detect_result, - Err(error) => return Err(error), - }; - detection_results.push(detect_result); - } - if decoded_event.protocol_name == "raydium_cpmm" - && (decoded_event.event_kind == "raydium_cpmm.swap_base_input" - || decoded_event.event_kind == "raydium_cpmm.swap_base_output") - { - let detect_result = - self.detect_raydium_cpmm_trade(&transaction, decoded_event).await; - let detect_result = match detect_result { - Ok(detect_result) => detect_result, - Err(error) => return Err(error), - }; - detection_results.push(detect_result); - } - if decoded_event.protocol_name == "raydium_clmm" - && decoded_event.event_kind == "raydium_clmm.swap_v2" - { - let detect_result = - self.detect_raydium_clmm_trade(&transaction, decoded_event).await; - let detect_result = match detect_result { - Ok(detect_result) => detect_result, - Err(error) => return Err(error), - }; - detection_results.push(detect_result); - } - if decoded_event.protocol_name == "pump_fun" - && decoded_event.event_kind == "pump_fun.create_v2_token" - { - let detect_result = - self.detect_pump_fun_create_v2_token(&transaction, decoded_event).await; - let detect_result = match detect_result { - Ok(detect_result) => detect_result, - Err(error) => return Err(error), - }; - detection_results.push(detect_result); - } - if decoded_event.protocol_name == "pump_fun" - && decoded_event.event_kind == "pump_fun.buy" - { - let detect_result = self.detect_pump_fun_trade(&transaction, decoded_event).await; - let detect_result = match detect_result { - Ok(detect_result) => detect_result, - Err(error) => return Err(error), - }; - detection_results.push(detect_result); - } - if decoded_event.protocol_name == "pump_fun" - && decoded_event.event_kind == "pump_fun.sell" - { - let detect_result = self.detect_pump_fun_trade(&transaction, decoded_event).await; - let detect_result = match detect_result { - Ok(detect_result) => detect_result, - Err(error) => return Err(error), - }; - detection_results.push(detect_result); - } - if decoded_event.protocol_name == "pump_swap" - && (decoded_event.event_kind == "pump_swap.buy" - || decoded_event.event_kind == "pump_swap.sell") - && (decoded_event.pool_account.is_none() - || decoded_event.token_a_mint.is_none() - || decoded_event.token_b_mint.is_none()) - { - tracing::trace!( - decoded_event_id = ?decoded_event.id, - event_kind = %decoded_event.event_kind, - pool_account = ?decoded_event.pool_account, - "skipping incomplete pump_swap decoded event during detection" - ); - continue; - } - if decoded_event.protocol_name == "pump_swap" - && decoded_event.event_kind == "pump_swap.buy" - { - let detect_result = self.detect_pump_swap_trade(&transaction, decoded_event).await; - let detect_result = match detect_result { - Ok(detect_result) => detect_result, - Err(error) => return Err(error), - }; - detection_results.push(detect_result); - } - if decoded_event.protocol_name == "pump_swap" - && decoded_event.event_kind == "pump_swap.sell" - { - let detect_result = self.detect_pump_swap_trade(&transaction, decoded_event).await; - let detect_result = match detect_result { - Ok(detect_result) => detect_result, - Err(error) => return Err(error), - }; - detection_results.push(detect_result); - } - if decoded_event.protocol_name == "meteora_dbc" - && decoded_event.event_kind == "meteora_dbc.create_pool" - { - let detect_result = self.detect_meteora_dbc_pool(&transaction, decoded_event).await; - let detect_result = match detect_result { - Ok(detect_result) => detect_result, - Err(error) => return Err(error), - }; - detection_results.push(detect_result); - } - if decoded_event.protocol_name == "meteora_dbc" - && decoded_event.event_kind == "meteora_dbc.swap" - { - let detect_result = self.detect_meteora_dbc_pool(&transaction, decoded_event).await; - let detect_result = match detect_result { - Ok(detect_result) => detect_result, - Err(error) => return Err(error), - }; - detection_results.push(detect_result); - } - if decoded_event.protocol_name == "meteora_damm_v2" - && decoded_event.event_kind == "meteora_damm_v2.create_pool" - { - let detect_result = - self.detect_meteora_damm_v2_pool(&transaction, decoded_event).await; - let detect_result = match detect_result { - Ok(detect_result) => detect_result, - Err(error) => return Err(error), - }; - detection_results.push(detect_result); - } - if decoded_event.protocol_name == "meteora_damm_v2" - && decoded_event.event_kind == "meteora_damm_v2.swap" - { - let detect_result = - self.detect_meteora_damm_v2_pool(&transaction, decoded_event).await; - let detect_result = match detect_result { - Ok(detect_result) => detect_result, - Err(error) => return Err(error), - }; - detection_results.push(detect_result); - } - if decoded_event.protocol_name == "meteora_damm_v1" - && decoded_event.event_kind == "meteora_damm_v1.create_pool" - { - let detect_result = - self.detect_meteora_damm_v1_pool(&transaction, decoded_event).await; - let detect_result = match detect_result { - Ok(detect_result) => detect_result, - Err(error) => return Err(error), - }; - detection_results.push(detect_result); - } - if decoded_event.protocol_name == "meteora_damm_v1" - && decoded_event.event_kind == "meteora_damm_v1.swap" - { - let detect_result = - self.detect_meteora_damm_v1_pool(&transaction, decoded_event).await; - let detect_result = match detect_result { - Ok(detect_result) => detect_result, - Err(error) => return Err(error), - }; - detection_results.push(detect_result); - } - if decoded_event.protocol_name == "orca_whirlpools" - && decoded_event.event_kind == "orca_whirlpools.create_pool" - { - let detect_result = - self.detect_orca_whirlpools_pool(&transaction, decoded_event).await; - let detect_result = match detect_result { - Ok(detect_result) => detect_result, - Err(error) => return Err(error), - }; - detection_results.push(detect_result); - } - if decoded_event.protocol_name == "orca_whirlpools" - && decoded_event.event_kind == "orca_whirlpools.swap" - { - let detect_result = - self.detect_orca_whirlpools_pool(&transaction, decoded_event).await; - let detect_result = match detect_result { - Ok(detect_result) => detect_result, - Err(error) => return Err(error), - }; - detection_results.push(detect_result); - } - if decoded_event.protocol_name == "fluxbeam" - && decoded_event.event_kind == "fluxbeam.create_pool" - { - let detect_result = self.detect_fluxbeam_pool(&transaction, decoded_event).await; - let detect_result = match detect_result { - Ok(detect_result) => detect_result, - Err(error) => return Err(error), - }; - detection_results.push(detect_result); - } - if decoded_event.protocol_name == "fluxbeam" - && decoded_event.event_kind == "fluxbeam.swap" - { - let detect_result = self.detect_fluxbeam_pool(&transaction, decoded_event).await; - let detect_result = match detect_result { - Ok(detect_result) => detect_result, - Err(error) => return Err(error), - }; - detection_results.push(detect_result); - } - if decoded_event.protocol_name == "dexlab" - && decoded_event.event_kind == "dexlab.create_pool" - { - let detect_result = self.detect_dexlab_pool(&transaction, decoded_event).await; - let detect_result = match detect_result { - Ok(detect_result) => detect_result, - Err(error) => return Err(error), - }; - detection_results.push(detect_result); - } - if decoded_event.protocol_name == "dexlab" && decoded_event.event_kind == "dexlab.swap" - { - let detect_result = self.detect_dexlab_pool(&transaction, decoded_event).await; - let detect_result = match detect_result { - Ok(detect_result) => detect_result, - Err(error) => return Err(error), - }; - detection_results.push(detect_result); - } + let route_option = crate::dex_detection_route::dex_detection_route(decoded_event); + let route = match route_option { + Some(route) => route, + None => continue, + }; + let detect_result = match route { + crate::dex_detection_route::DexDetectionRoute::RaydiumAmmV4Initialize2Pool => { + self.detect_raydium_initialize2_pool(&transaction, decoded_event).await + }, + crate::dex_detection_route::DexDetectionRoute::RaydiumCpmmTrade => { + self.detect_raydium_cpmm_trade(&transaction, decoded_event).await + }, + crate::dex_detection_route::DexDetectionRoute::RaydiumClmmTrade => { + self.detect_raydium_clmm_trade(&transaction, decoded_event).await + }, + crate::dex_detection_route::DexDetectionRoute::PumpFunCreateV2Token => { + self.detect_pump_fun_create_v2_token(&transaction, decoded_event).await + }, + crate::dex_detection_route::DexDetectionRoute::PumpFunTrade => { + self.detect_pump_fun_trade(&transaction, decoded_event).await + }, + crate::dex_detection_route::DexDetectionRoute::PumpSwapTrade => { + self.detect_pump_swap_trade(&transaction, decoded_event).await + }, + crate::dex_detection_route::DexDetectionRoute::SkipIncompletePumpSwapTrade => { + tracing::trace!( + decoded_event_id = ?decoded_event.id, + event_kind = %decoded_event.event_kind, + pool_account = ?decoded_event.pool_account, + "skipping incomplete pump_swap decoded event during detection" + ); + continue; + }, + crate::dex_detection_route::DexDetectionRoute::MeteoraDbcPool => { + self.detect_meteora_dbc_pool(&transaction, decoded_event).await + }, + crate::dex_detection_route::DexDetectionRoute::MeteoraDammV2Pool => { + self.detect_meteora_damm_v2_pool(&transaction, decoded_event).await + }, + crate::dex_detection_route::DexDetectionRoute::MeteoraDammV1Pool => { + self.detect_meteora_damm_v1_pool(&transaction, decoded_event).await + }, + crate::dex_detection_route::DexDetectionRoute::OrcaWhirlpoolsPool => { + self.detect_orca_whirlpools_pool(&transaction, decoded_event).await + }, + crate::dex_detection_route::DexDetectionRoute::FluxbeamPool => { + self.detect_fluxbeam_pool(&transaction, decoded_event).await + }, + crate::dex_detection_route::DexDetectionRoute::DexlabPool => { + self.detect_dexlab_pool(&transaction, decoded_event).await + }, + }; + let detect_result = match detect_result { + Ok(detect_result) => detect_result, + Err(error) => return Err(error), + }; + detection_results.push(detect_result); } return Ok(detection_results); } @@ -313,263 +145,51 @@ impl DexDetectService { transaction: &crate::ChainTransactionDto, decoded_event: &crate::DexDecodedEventDto, ) -> Result { - let decoded_event_id_option = decoded_event.id; - let decoded_event_id = match decoded_event_id_option { - Some(decoded_event_id) => decoded_event_id, - None => { - return Err(crate::Error::InvalidState( - "decoded dex event has no internal id".to_string(), - )); - }, - }; - let dex_id_result = self.ensure_raydium_dex().await; + let dex_id_result = + crate::dex_catalog::ensure_known_dex(self.database.as_ref(), "raydium").await; let dex_id = match dex_id_result { Ok(dex_id) => dex_id, Err(error) => return Err(error), }; - let pool_address_option = decoded_event.pool_account.clone(); - let pool_address = match pool_address_option { - Some(pool_address) => pool_address, - None => { - return Err(crate::Error::InvalidState(format!( - "decoded event '{}' has no pool_account", - decoded_event_id - ))); - }, - }; - let token_a_mint_option = decoded_event.token_a_mint.clone(); - let token_a_mint = match token_a_mint_option { - Some(token_a_mint) => token_a_mint, - None => { - return Err(crate::Error::InvalidState(format!( - "decoded event '{}' has no token_a_mint", - decoded_event_id - ))); - }, - }; - let token_b_mint_option = decoded_event.token_b_mint.clone(); - let token_b_mint = match token_b_mint_option { - Some(token_b_mint) => token_b_mint, - None => { - return Err(crate::Error::InvalidState(format!( - "decoded event '{}' has no token_b_mint", - decoded_event_id - ))); - }, - }; - let lp_mint = decoded_event.lp_mint.clone(); - let base_is_token_a = choose_base_quote_order(token_a_mint.as_str(), token_b_mint.as_str()); - let base_mint = if base_is_token_a { token_a_mint.clone() } else { token_b_mint.clone() }; - let quote_mint = if base_is_token_a { token_b_mint.clone() } else { token_a_mint.clone() }; - let base_token_id_result = self.ensure_token(base_mint.as_str()).await; - let base_token_id = match base_token_id_result { - Ok(base_token_id) => base_token_id, - Err(error) => return Err(error), - }; - let quote_token_id_result = self.ensure_token(quote_mint.as_str()).await; - let quote_token_id = match quote_token_id_result { - Ok(quote_token_id) => quote_token_id, - Err(error) => return Err(error), - }; - let lp_token_id = match lp_mint.clone() { - Some(lp_mint) => { - let lp_token_id_result = self.ensure_token(lp_mint.as_str()).await; - match lp_token_id_result { - Ok(lp_token_id) => Some(lp_token_id), - Err(error) => return Err(error), - } - }, - None => None, - }; - let existing_pool_result = - crate::query_pools_get_by_address(self.database.as_ref(), pool_address.as_str()).await; - let existing_pool_option = match existing_pool_result { - Ok(existing_pool_option) => existing_pool_option, - Err(error) => return Err(error), - }; - let created_pool = existing_pool_option.is_none(); - let pool_id = match existing_pool_option { - Some(pool) => { - let pool_id_option = pool.id; - match pool_id_option { - Some(pool_id) => pool_id, - None => { - return Err(crate::Error::InvalidState(format!( - "pool '{}' has no internal id", - pool.address - ))); - }, - } - }, - None => { - let pool_dto = crate::PoolDto::new( - dex_id, - pool_address.clone(), - crate::PoolKind::Amm, - crate::PoolStatus::Active, - ); - let upsert_result = - crate::query_pools_upsert(self.database.as_ref(), &pool_dto).await; - match upsert_result { - Ok(pool_id) => pool_id, - Err(error) => return Err(error), - } - }, - }; - let existing_pair_result = - crate::query_pairs_get_by_pool_id(self.database.as_ref(), pool_id).await; - let existing_pair_option = match existing_pair_result { - Ok(existing_pair_option) => existing_pair_option, - Err(error) => return Err(error), - }; - let created_pair = existing_pair_option.is_none(); - let pair_symbol = build_pair_symbol(base_mint.as_str(), quote_mint.as_str()); - let pair_id = match existing_pair_option { - Some(pair) => { - let pair_id_option = pair.id; - match pair_id_option { - Some(pair_id) => pair_id, - None => { - return Err(crate::Error::InvalidState(format!( - "pair for pool '{}' has no internal id", - pool_id - ))); - }, - } - }, - None => { - let pair_dto = crate::PairDto::new( - dex_id, - pool_id, - base_token_id, - quote_token_id, - pair_symbol, - ); - let upsert_result = - crate::query_pairs_upsert(self.database.as_ref(), &pair_dto).await; - match upsert_result { - Ok(pair_id) => pair_id, - Err(error) => return Err(error), - } - }, - }; - let upsert_base_pool_token_result = crate::query_pool_tokens_upsert( - self.database.as_ref(), - &crate::PoolTokenDto::new( - pool_id, - base_token_id, - crate::PoolTokenRole::Base, + let input_result = + crate::dex_pool_materialization::DexPoolMaterializationInput::from_decoded_event( + decoded_event, + dex_id, + crate::PoolKind::Amm, + crate::PoolStatus::Active, + crate::dex_pool_materialization::DexPoolTokenOrder::ChooseBaseQuoteFromTokenAB, None, - Some(0), - ), - ) - .await; - if let Err(error) = upsert_base_pool_token_result { - return Err(error); - } - let upsert_quote_pool_token_result = crate::query_pool_tokens_upsert( - self.database.as_ref(), - &crate::PoolTokenDto::new( - pool_id, - quote_token_id, - crate::PoolTokenRole::Quote, None, - Some(1), - ), - ) - .await; - if let Err(error) = upsert_quote_pool_token_result { - return Err(error); - } - if let Some(lp_token_id) = lp_token_id { - let upsert_lp_pool_token_result = crate::query_pool_tokens_upsert( - self.database.as_ref(), - &crate::PoolTokenDto::new( - pool_id, - lp_token_id, - crate::PoolTokenRole::LpMint, - None, - None, - ), - ) - .await; - if let Err(error) = upsert_lp_pool_token_result { - return Err(error); - } - } - let existing_listing_result = - crate::query_pool_listings_get_by_pool_id(self.database.as_ref(), pool_id).await; - let existing_listing_option = match existing_listing_result { - Ok(existing_listing_option) => existing_listing_option, + transaction.source_endpoint_name.clone(), + ); + let input = match input_result { + Ok(input) => input, Err(error) => return Err(error), }; - let created_listing = existing_listing_option.is_none(); - let pool_listing_id = match existing_listing_option { - Some(pool_listing) => pool_listing.id, - None => { - let listing_id_result = self - .upsert_pool_listing_from_decoded_event(dex_id, pool_id, pair_id, transaction) - .await; - match listing_id_result { - Ok(listing_id) => Some(listing_id), - Err(error) => return Err(error), - } - }, + let detection_result = + crate::dex_pool_materialization::materialize_dex_pool(self.database.as_ref(), &input) + .await; + let detection_result = match detection_result { + Ok(detection_result) => detection_result, + Err(error) => return Err(error), }; let payload_value_result = parse_payload_json(decoded_event.payload_json.as_str()); let payload_value = match payload_value_result { Ok(payload_value) => payload_value, Err(error) => return Err(error), }; - if created_pool { - let signal_result = self - .record_detection_signal( - transaction, - "signal.dex.new_pool", - crate::AnalysisSignalSeverity::Low, - payload_value.clone(), - ) - .await; - if let Err(error) = signal_result { - return Err(error); - } + let signal_result = self + .record_pool_detection_signals( + transaction, + "signal.dex", + &detection_result, + payload_value, + ) + .await; + if let Err(error) = signal_result { + return Err(error); } - if created_pair { - let signal_result = self - .record_detection_signal( - transaction, - "signal.dex.new_pair", - crate::AnalysisSignalSeverity::Low, - payload_value.clone(), - ) - .await; - if let Err(error) = signal_result { - return Err(error); - } - } - if created_listing { - let signal_result = self - .record_detection_signal( - transaction, - "signal.dex.first_listing_seen", - crate::AnalysisSignalSeverity::Low, - payload_value, - ) - .await; - if let Err(error) = signal_result { - return Err(error); - } - } - return Ok(crate::DexPoolDetectionResult { - decoded_event_id, - dex_id, - pool_id, - pair_id, - pool_listing_id, - created_pool, - created_pair, - created_listing, - }); + return Ok(detection_result); } async fn detect_pump_fun_create_v2_token( @@ -577,228 +197,60 @@ impl DexDetectService { transaction: &crate::ChainTransactionDto, decoded_event: &crate::DexDecodedEventDto, ) -> Result { - let decoded_event_id_option = decoded_event.id; - let decoded_event_id = match decoded_event_id_option { - Some(decoded_event_id) => decoded_event_id, - None => { - return Err(crate::Error::InvalidState( - "decoded dex event has no internal id".to_string(), - )); - }, - }; - let dex_id_result = self.ensure_pump_fun_dex().await; + let dex_id_result = + crate::dex_catalog::ensure_known_dex(self.database.as_ref(), "pump_fun").await; let dex_id = match dex_id_result { Ok(dex_id) => dex_id, Err(error) => return Err(error), }; - let pool_address_option = decoded_event.pool_account.clone(); - let pool_address = match pool_address_option { - Some(pool_address) => pool_address, - None => { - return Err(crate::Error::InvalidState(format!( - "decoded event '{}' has no pool_account", - decoded_event_id - ))); - }, - }; - let token_mint_option = decoded_event.token_a_mint.clone(); - let token_mint = match token_mint_option { - Some(token_mint) => token_mint, - None => { - return Err(crate::Error::InvalidState(format!( - "decoded event '{}' has no token_a_mint", - decoded_event_id - ))); - }, - }; - let quote_mint = crate::WSOL_MINT_ID.to_string(); - let base_is_token_a = choose_base_quote_order(token_mint.as_str(), quote_mint.as_str()); - let base_mint = if base_is_token_a { token_mint.clone() } else { quote_mint.clone() }; - let quote_mint_ordered = - if base_is_token_a { quote_mint.clone() } else { token_mint.clone() }; - let base_token_id_result = self.ensure_token(base_mint.as_str()).await; - let base_token_id = match base_token_id_result { - Ok(base_token_id) => base_token_id, + let token_mint_result = + crate::dex_pool_materialization::required_token_a_mint(decoded_event); + let token_mint = match token_mint_result { + Ok(token_mint) => token_mint, Err(error) => return Err(error), }; - let quote_token_id_result = self.ensure_token(quote_mint_ordered.as_str()).await; - let quote_token_id = match quote_token_id_result { - Ok(quote_token_id) => quote_token_id, - Err(error) => return Err(error), - }; - let existing_pool_result = - crate::query_pools_get_by_address(self.database.as_ref(), pool_address.as_str()).await; - let existing_pool_option = match existing_pool_result { - Ok(existing_pool_option) => existing_pool_option, - Err(error) => return Err(error), - }; - let created_pool = existing_pool_option.is_none(); - let pool_id = match existing_pool_option { - Some(pool) => { - let pool_id_option = pool.id; - match pool_id_option { - Some(pool_id) => pool_id, - None => { - return Err(crate::Error::InvalidState(format!( - "pool '{}' has no internal id", - pool.address - ))); - }, - } - }, - None => { - let pool_dto = crate::PoolDto::new( - dex_id, - pool_address.clone(), - crate::PoolKind::BondingCurve, - crate::PoolStatus::Pending, - ); - let upsert_result = - crate::query_pools_upsert(self.database.as_ref(), &pool_dto).await; - match upsert_result { - Ok(pool_id) => pool_id, - Err(error) => return Err(error), - } - }, - }; - let existing_pair_result = - crate::query_pairs_get_by_pool_id(self.database.as_ref(), pool_id).await; - let existing_pair_option = match existing_pair_result { - Ok(existing_pair_option) => existing_pair_option, - Err(error) => return Err(error), - }; - let created_pair = existing_pair_option.is_none(); - let pair_symbol = build_pair_symbol(base_mint.as_str(), quote_mint_ordered.as_str()); - let pair_id = match existing_pair_option { - Some(pair) => { - let pair_id_option = pair.id; - match pair_id_option { - Some(pair_id) => pair_id, - None => { - return Err(crate::Error::InvalidState(format!( - "pair for pool '{}' has no internal id", - pool_id - ))); - }, - } - }, - None => { - let pair_dto = crate::PairDto::new( - dex_id, - pool_id, - base_token_id, - quote_token_id, - pair_symbol, - ); - let upsert_result = - crate::query_pairs_upsert(self.database.as_ref(), &pair_dto).await; - match upsert_result { - Ok(pair_id) => pair_id, - Err(error) => return Err(error), - } - }, - }; - let upsert_base_pool_token_result = crate::query_pool_tokens_upsert( - self.database.as_ref(), - &crate::PoolTokenDto::new( - pool_id, - base_token_id, - crate::PoolTokenRole::Base, + let input_result = + crate::dex_pool_materialization::DexPoolMaterializationInput::from_decoded_event_with_mints( + decoded_event, + dex_id, + token_mint, + crate::WSOL_MINT_ID.to_string(), + decoded_event.lp_mint.clone(), + crate::PoolKind::BondingCurve, + crate::PoolStatus::Pending, + crate::dex_pool_materialization::DexPoolTokenOrder::ChooseBaseQuoteFromTokenAB, None, - Some(0), - ), - ) - .await; - if let Err(error) = upsert_base_pool_token_result { - return Err(error); - } - let upsert_quote_pool_token_result = crate::query_pool_tokens_upsert( - self.database.as_ref(), - &crate::PoolTokenDto::new( - pool_id, - quote_token_id, - crate::PoolTokenRole::Quote, None, - Some(1), - ), - ) - .await; - if let Err(error) = upsert_quote_pool_token_result { - return Err(error); - } - let existing_listing_result = - crate::query_pool_listings_get_by_pool_id(self.database.as_ref(), pool_id).await; - let existing_listing_option = match existing_listing_result { - Ok(existing_listing_option) => existing_listing_option, + transaction.source_endpoint_name.clone(), + ); + let input = match input_result { + Ok(input) => input, Err(error) => return Err(error), }; - let created_listing = existing_listing_option.is_none(); - let pool_listing_id = match existing_listing_option { - Some(pool_listing) => pool_listing.id, - None => { - let listing_id_result = self - .upsert_pool_listing_from_decoded_event(dex_id, pool_id, pair_id, transaction) - .await; - match listing_id_result { - Ok(listing_id) => Some(listing_id), - Err(error) => return Err(error), - } - }, + let detection_result = + crate::dex_pool_materialization::materialize_dex_pool(self.database.as_ref(), &input) + .await; + let detection_result = match detection_result { + Ok(detection_result) => detection_result, + Err(error) => return Err(error), }; let payload_value_result = parse_payload_json(decoded_event.payload_json.as_str()); let payload_value = match payload_value_result { Ok(payload_value) => payload_value, Err(error) => return Err(error), }; - if created_pool { - let signal_result = self - .record_detection_signal( - transaction, - "signal.dex.pump_fun.new_pool", - crate::AnalysisSignalSeverity::Low, - payload_value.clone(), - ) - .await; - if let Err(error) = signal_result { - return Err(error); - } + let signal_result = self + .record_pool_detection_signals( + transaction, + "signal.dex.pump_fun", + &detection_result, + payload_value, + ) + .await; + if let Err(error) = signal_result { + return Err(error); } - if created_pair { - let signal_result = self - .record_detection_signal( - transaction, - "signal.dex.pump_fun.new_pair", - crate::AnalysisSignalSeverity::Low, - payload_value.clone(), - ) - .await; - if let Err(error) = signal_result { - return Err(error); - } - } - if created_listing { - let signal_result = self - .record_detection_signal( - transaction, - "signal.dex.pump_fun.first_listing_seen", - crate::AnalysisSignalSeverity::Low, - payload_value, - ) - .await; - if let Err(error) = signal_result { - return Err(error); - } - } - return Ok(crate::DexPoolDetectionResult { - decoded_event_id, - dex_id, - pool_id, - pair_id, - pool_listing_id, - created_pool, - created_pair, - created_listing, - }); + return Ok(detection_result); } async fn detect_pump_fun_trade( @@ -806,58 +258,12 @@ impl DexDetectService { transaction: &crate::ChainTransactionDto, decoded_event: &crate::DexDecodedEventDto, ) -> Result { - let decoded_event_id_option = decoded_event.id; - let decoded_event_id = match decoded_event_id_option { - Some(decoded_event_id) => decoded_event_id, - None => { - return Err(crate::Error::InvalidState( - "decoded dex event has no internal id".to_string(), - )); - }, - }; - let dex_id_result = self.ensure_pump_fun_dex().await; + let dex_id_result = + crate::dex_catalog::ensure_known_dex(self.database.as_ref(), "pump_fun").await; let dex_id = match dex_id_result { Ok(dex_id) => dex_id, Err(error) => return Err(error), }; - let pool_address_option = decoded_event.pool_account.clone(); - let pool_address = match pool_address_option { - Some(pool_address) => pool_address, - None => { - return Err(crate::Error::InvalidState(format!( - "decoded event '{}' has no pool_account", - decoded_event_id - ))); - }, - }; - let token_a_mint_option = decoded_event.token_a_mint.clone(); - let token_a_mint = match token_a_mint_option { - Some(token_a_mint) => token_a_mint, - None => { - return Err(crate::Error::InvalidState(format!( - "decoded event '{}' has no token_a_mint", - decoded_event_id - ))); - }, - }; - let token_b_mint_option = decoded_event.token_b_mint.clone(); - let token_b_mint = match token_b_mint_option { - Some(token_b_mint) => token_b_mint, - None => { - return Err(crate::Error::InvalidState(format!( - "decoded event '{}' has no token_b_mint", - decoded_event_id - ))); - }, - }; - let base_is_token_a = choose_base_quote_order(token_a_mint.as_str(), token_b_mint.as_str()); - let base_mint = if base_is_token_a { token_a_mint.clone() } else { token_b_mint.clone() }; - let quote_mint = if base_is_token_a { token_b_mint.clone() } else { token_a_mint.clone() }; - let base_token_id_result = self.ensure_token(base_mint.as_str()).await; - let base_token_id = match base_token_id_result { - Ok(base_token_id) => base_token_id, - Err(error) => return Err(error), - }; let payload_value_result = parse_payload_json(decoded_event.payload_json.as_str()); let payload_value = match payload_value_result { Ok(payload_value) => payload_value, @@ -866,168 +272,40 @@ impl DexDetectService { let vault_addresses = extract_pump_fun_vault_addresses(&payload_value); let token_a_vault_address = vault_addresses.0; let token_b_vault_address = vault_addresses.1; - - let base_vault_address = if base_is_token_a { - token_a_vault_address.clone() - } else { - token_b_vault_address.clone() - }; - let quote_vault_address = if base_is_token_a { - token_b_vault_address.clone() - } else { - token_a_vault_address.clone() - }; - let quote_token_id_result = self.ensure_token(quote_mint.as_str()).await; - let quote_token_id = match quote_token_id_result { - Ok(quote_token_id) => quote_token_id, + let input_result = + crate::dex_pool_materialization::DexPoolMaterializationInput::from_decoded_event( + decoded_event, + dex_id, + crate::PoolKind::BondingCurve, + crate::PoolStatus::Active, + crate::dex_pool_materialization::DexPoolTokenOrder::ChooseBaseQuoteFromTokenAB, + token_a_vault_address, + token_b_vault_address, + transaction.source_endpoint_name.clone(), + ); + let input = match input_result { + Ok(input) => input, Err(error) => return Err(error), }; - let existing_pool_result = - crate::query_pools_get_by_address(self.database.as_ref(), pool_address.as_str()).await; - let existing_pool_option = match existing_pool_result { - Ok(existing_pool_option) => existing_pool_option, + let detection_result = + crate::dex_pool_materialization::materialize_dex_pool(self.database.as_ref(), &input) + .await; + let detection_result = match detection_result { + Ok(detection_result) => detection_result, Err(error) => return Err(error), }; - let created_pool = existing_pool_option.is_none(); - let pool_id = match existing_pool_option { - Some(pool) => { - let pool_id_option = pool.id; - match pool_id_option { - Some(pool_id) => pool_id, - None => { - return Err(crate::Error::InvalidState(format!( - "pool '{}' has no internal id", - pool.address - ))); - }, - } - }, - None => { - let pool_dto = crate::PoolDto::new( - dex_id, - pool_address.clone(), - crate::PoolKind::BondingCurve, - crate::PoolStatus::Active, - ); - let upsert_result = - crate::query_pools_upsert(self.database.as_ref(), &pool_dto).await; - match upsert_result { - Ok(pool_id) => pool_id, - Err(error) => return Err(error), - } - }, - }; - let existing_pair_result = - crate::query_pairs_get_by_pool_id(self.database.as_ref(), pool_id).await; - let existing_pair_option = match existing_pair_result { - Ok(existing_pair_option) => existing_pair_option, - Err(error) => return Err(error), - }; - let created_pair = existing_pair_option.is_none(); - let pair_symbol = build_pair_symbol(base_mint.as_str(), quote_mint.as_str()); - let pair_dto = - crate::PairDto::new(dex_id, pool_id, base_token_id, quote_token_id, pair_symbol); - let pair_id_result = crate::query_pairs_upsert(self.database.as_ref(), &pair_dto).await; - let pair_id = match pair_id_result { - Ok(pair_id) => pair_id, - Err(error) => return Err(error), - }; - let upsert_base_pool_token_result = crate::query_pool_tokens_upsert( - self.database.as_ref(), - &crate::PoolTokenDto::new( - pool_id, - base_token_id, - crate::PoolTokenRole::Base, - base_vault_address, - Some(0), - ), - ) - .await; - if let Err(error) = upsert_base_pool_token_result { + let signal_result = self + .record_pool_detection_signals( + transaction, + "signal.dex.pump_fun", + &detection_result, + payload_value, + ) + .await; + if let Err(error) = signal_result { return Err(error); } - let upsert_quote_pool_token_result = crate::query_pool_tokens_upsert( - self.database.as_ref(), - &crate::PoolTokenDto::new( - pool_id, - quote_token_id, - crate::PoolTokenRole::Quote, - quote_vault_address, - Some(1), - ), - ) - .await; - if let Err(error) = upsert_quote_pool_token_result { - return Err(error); - } - let existing_listing_result = - crate::query_pool_listings_get_by_pool_id(self.database.as_ref(), pool_id).await; - let existing_listing_option = match existing_listing_result { - Ok(existing_listing_option) => existing_listing_option, - Err(error) => return Err(error), - }; - let created_listing = existing_listing_option.is_none(); - let pool_listing_id = match existing_listing_option { - Some(pool_listing) => pool_listing.id, - None => { - let listing_id_result = self - .upsert_pool_listing_from_decoded_event(dex_id, pool_id, pair_id, transaction) - .await; - match listing_id_result { - Ok(listing_id) => Some(listing_id), - Err(error) => return Err(error), - } - }, - }; - if created_pool { - let signal_result = self - .record_detection_signal( - transaction, - "signal.dex.pump_fun.new_pool", - crate::AnalysisSignalSeverity::Low, - payload_value.clone(), - ) - .await; - if let Err(error) = signal_result { - return Err(error); - } - } - if created_pair { - let signal_result = self - .record_detection_signal( - transaction, - "signal.dex.pump_fun.new_pair", - crate::AnalysisSignalSeverity::Low, - payload_value.clone(), - ) - .await; - if let Err(error) = signal_result { - return Err(error); - } - } - if created_listing { - let signal_result = self - .record_detection_signal( - transaction, - "signal.dex.pump_fun.first_listing_seen", - crate::AnalysisSignalSeverity::Low, - payload_value, - ) - .await; - if let Err(error) = signal_result { - return Err(error); - } - } - return Ok(crate::DexPoolDetectionResult { - decoded_event_id, - dex_id, - pool_id, - pair_id, - pool_listing_id, - created_pool, - created_pair, - created_listing, - }); + return Ok(detection_result); } async fn detect_pump_swap_trade( @@ -1035,58 +313,12 @@ impl DexDetectService { transaction: &crate::ChainTransactionDto, decoded_event: &crate::DexDecodedEventDto, ) -> Result { - let decoded_event_id_option = decoded_event.id; - let decoded_event_id = match decoded_event_id_option { - Some(decoded_event_id) => decoded_event_id, - None => { - return Err(crate::Error::InvalidState( - "decoded dex event has no internal id".to_string(), - )); - }, - }; - let dex_id_result = self.ensure_pump_swap_dex().await; + let dex_id_result = + crate::dex_catalog::ensure_known_dex(self.database.as_ref(), "pump_swap").await; let dex_id = match dex_id_result { Ok(dex_id) => dex_id, Err(error) => return Err(error), }; - let pool_address_option = decoded_event.pool_account.clone(); - let pool_address = match pool_address_option { - Some(pool_address) => pool_address, - None => { - return Err(crate::Error::InvalidState(format!( - "decoded event '{}' has no pool_account", - decoded_event_id - ))); - }, - }; - let token_a_mint_option = decoded_event.token_a_mint.clone(); - let token_a_mint = match token_a_mint_option { - Some(token_a_mint) => token_a_mint, - None => { - return Err(crate::Error::InvalidState(format!( - "decoded event '{}' has no token_a_mint", - decoded_event_id - ))); - }, - }; - let token_b_mint_option = decoded_event.token_b_mint.clone(); - let token_b_mint = match token_b_mint_option { - Some(token_b_mint) => token_b_mint, - None => { - return Err(crate::Error::InvalidState(format!( - "decoded event '{}' has no token_b_mint", - decoded_event_id - ))); - }, - }; - let base_is_token_a = choose_base_quote_order(token_a_mint.as_str(), token_b_mint.as_str()); - let base_mint = if base_is_token_a { token_a_mint.clone() } else { token_b_mint.clone() }; - let quote_mint = if base_is_token_a { token_b_mint.clone() } else { token_a_mint.clone() }; - let base_token_id_result = self.ensure_token(base_mint.as_str()).await; - let base_token_id = match base_token_id_result { - Ok(base_token_id) => base_token_id, - Err(error) => return Err(error), - }; let payload_value_result = parse_payload_json(decoded_event.payload_json.as_str()); let payload_value = match payload_value_result { Ok(payload_value) => payload_value, @@ -1095,168 +327,40 @@ impl DexDetectService { let vault_addresses = extract_pump_swap_vault_addresses(&payload_value); let token_a_vault_address = vault_addresses.0; let token_b_vault_address = vault_addresses.1; - - let base_vault_address = if base_is_token_a { - token_a_vault_address.clone() - } else { - token_b_vault_address.clone() - }; - let quote_vault_address = if base_is_token_a { - token_b_vault_address.clone() - } else { - token_a_vault_address.clone() - }; - let quote_token_id_result = self.ensure_token(quote_mint.as_str()).await; - let quote_token_id = match quote_token_id_result { - Ok(quote_token_id) => quote_token_id, + let input_result = + crate::dex_pool_materialization::DexPoolMaterializationInput::from_decoded_event( + decoded_event, + dex_id, + crate::PoolKind::Amm, + crate::PoolStatus::Active, + crate::dex_pool_materialization::DexPoolTokenOrder::ChooseBaseQuoteFromTokenAB, + token_a_vault_address, + token_b_vault_address, + transaction.source_endpoint_name.clone(), + ); + let input = match input_result { + Ok(input) => input, Err(error) => return Err(error), }; - let existing_pool_result = - crate::query_pools_get_by_address(self.database.as_ref(), pool_address.as_str()).await; - let existing_pool_option = match existing_pool_result { - Ok(existing_pool_option) => existing_pool_option, + let detection_result = + crate::dex_pool_materialization::materialize_dex_pool(self.database.as_ref(), &input) + .await; + let detection_result = match detection_result { + Ok(detection_result) => detection_result, Err(error) => return Err(error), }; - let created_pool = existing_pool_option.is_none(); - let pool_id = match existing_pool_option { - Some(pool) => { - let pool_id_option = pool.id; - match pool_id_option { - Some(pool_id) => pool_id, - None => { - return Err(crate::Error::InvalidState(format!( - "pool '{}' has no internal id", - pool.address - ))); - }, - } - }, - None => { - let pool_dto = crate::PoolDto::new( - dex_id, - pool_address.clone(), - crate::PoolKind::Amm, - crate::PoolStatus::Active, - ); - let upsert_result = - crate::query_pools_upsert(self.database.as_ref(), &pool_dto).await; - match upsert_result { - Ok(pool_id) => pool_id, - Err(error) => return Err(error), - } - }, - }; - let existing_pair_result = - crate::query_pairs_get_by_pool_id(self.database.as_ref(), pool_id).await; - let existing_pair_option = match existing_pair_result { - Ok(existing_pair_option) => existing_pair_option, - Err(error) => return Err(error), - }; - let created_pair = existing_pair_option.is_none(); - let pair_symbol = build_pair_symbol(base_mint.as_str(), quote_mint.as_str()); - let pair_dto = - crate::PairDto::new(dex_id, pool_id, base_token_id, quote_token_id, pair_symbol); - let pair_id_result = crate::query_pairs_upsert(self.database.as_ref(), &pair_dto).await; - let pair_id = match pair_id_result { - Ok(pair_id) => pair_id, - Err(error) => return Err(error), - }; - let upsert_base_pool_token_result = crate::query_pool_tokens_upsert( - self.database.as_ref(), - &crate::PoolTokenDto::new( - pool_id, - base_token_id, - crate::PoolTokenRole::Base, - base_vault_address, - Some(0), - ), - ) - .await; - if let Err(error) = upsert_base_pool_token_result { + let signal_result = self + .record_pool_detection_signals( + transaction, + "signal.dex.pump_swap", + &detection_result, + payload_value, + ) + .await; + if let Err(error) = signal_result { return Err(error); } - let upsert_quote_pool_token_result = crate::query_pool_tokens_upsert( - self.database.as_ref(), - &crate::PoolTokenDto::new( - pool_id, - quote_token_id, - crate::PoolTokenRole::Quote, - quote_vault_address, - Some(1), - ), - ) - .await; - if let Err(error) = upsert_quote_pool_token_result { - return Err(error); - } - let existing_listing_result = - crate::query_pool_listings_get_by_pool_id(self.database.as_ref(), pool_id).await; - let existing_listing_option = match existing_listing_result { - Ok(existing_listing_option) => existing_listing_option, - Err(error) => return Err(error), - }; - let created_listing = existing_listing_option.is_none(); - let pool_listing_id = match existing_listing_option { - Some(pool_listing) => pool_listing.id, - None => { - let listing_id_result = self - .upsert_pool_listing_from_decoded_event(dex_id, pool_id, pair_id, transaction) - .await; - match listing_id_result { - Ok(listing_id) => Some(listing_id), - Err(error) => return Err(error), - } - }, - }; - if created_pool { - let signal_result = self - .record_detection_signal( - transaction, - "signal.dex.pump_swap.new_pool", - crate::AnalysisSignalSeverity::Low, - payload_value.clone(), - ) - .await; - if let Err(error) = signal_result { - return Err(error); - } - } - if created_pair { - let signal_result = self - .record_detection_signal( - transaction, - "signal.dex.pump_swap.new_pair", - crate::AnalysisSignalSeverity::Low, - payload_value.clone(), - ) - .await; - if let Err(error) = signal_result { - return Err(error); - } - } - if created_listing { - let signal_result = self - .record_detection_signal( - transaction, - "signal.dex.pump_swap.first_listing_seen", - crate::AnalysisSignalSeverity::Low, - payload_value, - ) - .await; - if let Err(error) = signal_result { - return Err(error); - } - } - return Ok(crate::DexPoolDetectionResult { - decoded_event_id, - dex_id, - pool_id, - pair_id, - pool_listing_id, - created_pool, - created_pair, - created_listing, - }); + return Ok(detection_result); } async fn detect_meteora_dbc_pool( @@ -1264,236 +368,16 @@ impl DexDetectService { transaction: &crate::ChainTransactionDto, decoded_event: &crate::DexDecodedEventDto, ) -> Result { - let decoded_event_id_option = decoded_event.id; - let decoded_event_id = match decoded_event_id_option { - Some(decoded_event_id) => decoded_event_id, - None => { - return Err(crate::Error::InvalidState( - "decoded dex event has no internal id".to_string(), - )); - }, - }; - let dex_id_result = self.ensure_meteora_dbc_dex().await; - let dex_id = match dex_id_result { - Ok(dex_id) => dex_id, - Err(error) => return Err(error), - }; - let pool_address_option = decoded_event.pool_account.clone(); - let pool_address = match pool_address_option { - Some(pool_address) => pool_address, - None => { - return Err(crate::Error::InvalidState(format!( - "decoded event '{}' has no pool_account", - decoded_event_id - ))); - }, - }; - let token_a_mint_option = decoded_event.token_a_mint.clone(); - let token_a_mint = match token_a_mint_option { - Some(token_a_mint) => token_a_mint, - None => { - return Err(crate::Error::InvalidState(format!( - "decoded event '{}' has no token_a_mint", - decoded_event_id - ))); - }, - }; - let token_b_mint_option = decoded_event.token_b_mint.clone(); - let token_b_mint = match token_b_mint_option { - Some(token_b_mint) => token_b_mint, - None => { - return Err(crate::Error::InvalidState(format!( - "decoded event '{}' has no token_b_mint", - decoded_event_id - ))); - }, - }; - let base_is_token_a = choose_base_quote_order(token_a_mint.as_str(), token_b_mint.as_str()); - let base_mint = if base_is_token_a { token_a_mint.clone() } else { token_b_mint.clone() }; - let quote_mint = if base_is_token_a { token_b_mint.clone() } else { token_a_mint.clone() }; - let base_token_id_result = self.ensure_token(base_mint.as_str()).await; - let base_token_id = match base_token_id_result { - Ok(base_token_id) => base_token_id, - Err(error) => return Err(error), - }; - let quote_token_id_result = self.ensure_token(quote_mint.as_str()).await; - let quote_token_id = match quote_token_id_result { - Ok(quote_token_id) => quote_token_id, - Err(error) => return Err(error), - }; - let existing_pool_result = - crate::query_pools_get_by_address(self.database.as_ref(), pool_address.as_str()).await; - let existing_pool_option = match existing_pool_result { - Ok(existing_pool_option) => existing_pool_option, - Err(error) => return Err(error), - }; - let created_pool = existing_pool_option.is_none(); - let pool_id = match existing_pool_option { - Some(pool) => { - let pool_id_option = pool.id; - match pool_id_option { - Some(pool_id) => pool_id, - None => { - return Err(crate::Error::InvalidState(format!( - "pool '{}' has no internal id", - pool.address - ))); - }, - } - }, - None => { - let pool_dto = crate::PoolDto::new( - dex_id, - pool_address.clone(), - crate::PoolKind::BondingCurve, - crate::PoolStatus::Pending, - ); - let upsert_result = - crate::query_pools_upsert(self.database.as_ref(), &pool_dto).await; - match upsert_result { - Ok(pool_id) => pool_id, - Err(error) => return Err(error), - } - }, - }; - let existing_pair_result = - crate::query_pairs_get_by_pool_id(self.database.as_ref(), pool_id).await; - let existing_pair_option = match existing_pair_result { - Ok(existing_pair_option) => existing_pair_option, - Err(error) => return Err(error), - }; - let created_pair = existing_pair_option.is_none(); - let pair_symbol = build_pair_symbol(base_mint.as_str(), quote_mint.as_str()); - let pair_id = match existing_pair_option { - Some(pair) => { - let pair_id_option = pair.id; - match pair_id_option { - Some(pair_id) => pair_id, - None => { - return Err(crate::Error::InvalidState(format!( - "pair for pool '{}' has no internal id", - pool_id - ))); - }, - } - }, - None => { - let pair_dto = crate::PairDto::new( - dex_id, - pool_id, - base_token_id, - quote_token_id, - pair_symbol, - ); - let upsert_result = - crate::query_pairs_upsert(self.database.as_ref(), &pair_dto).await; - match upsert_result { - Ok(pair_id) => pair_id, - Err(error) => return Err(error), - } - }, - }; - let upsert_base_pool_token_result = crate::query_pool_tokens_upsert( - self.database.as_ref(), - &crate::PoolTokenDto::new( - pool_id, - base_token_id, - crate::PoolTokenRole::Base, - None, - Some(0), - ), - ) - .await; - if let Err(error) = upsert_base_pool_token_result { - return Err(error); - } - let upsert_quote_pool_token_result = crate::query_pool_tokens_upsert( - self.database.as_ref(), - &crate::PoolTokenDto::new( - pool_id, - quote_token_id, - crate::PoolTokenRole::Quote, - None, - Some(1), - ), - ) - .await; - if let Err(error) = upsert_quote_pool_token_result { - return Err(error); - } - let existing_listing_result = - crate::query_pool_listings_get_by_pool_id(self.database.as_ref(), pool_id).await; - let existing_listing_option = match existing_listing_result { - Ok(existing_listing_option) => existing_listing_option, - Err(error) => return Err(error), - }; - let created_listing = existing_listing_option.is_none(); - let pool_listing_id = match existing_listing_option { - Some(pool_listing) => pool_listing.id, - None => { - let listing_id_result = self - .upsert_pool_listing_from_decoded_event(dex_id, pool_id, pair_id, transaction) - .await; - match listing_id_result { - Ok(listing_id) => Some(listing_id), - Err(error) => return Err(error), - } - }, - }; - let payload_value_result = parse_payload_json(decoded_event.payload_json.as_str()); - let payload_value = match payload_value_result { - Ok(payload_value) => payload_value, - Err(error) => return Err(error), - }; - if created_pool { - let signal_result = self - .record_detection_signal( - transaction, - "signal.dex.meteora_dbc.new_pool", - crate::AnalysisSignalSeverity::Low, - payload_value.clone(), - ) - .await; - if let Err(error) = signal_result { - return Err(error); - } - } - if created_pair { - let signal_result = self - .record_detection_signal( - transaction, - "signal.dex.meteora_dbc.new_pair", - crate::AnalysisSignalSeverity::Low, - payload_value.clone(), - ) - .await; - if let Err(error) = signal_result { - return Err(error); - } - } - if created_listing { - let signal_result = self - .record_detection_signal( - transaction, - "signal.dex.meteora_dbc.first_listing_seen", - crate::AnalysisSignalSeverity::Low, - payload_value, - ) - .await; - if let Err(error) = signal_result { - return Err(error); - } - } - return Ok(crate::DexPoolDetectionResult { - decoded_event_id, - dex_id, - pool_id, - pair_id, - pool_listing_id, - created_pool, - created_pair, - created_listing, - }); + return self + .detect_materialized_pool_from_decoded_event( + transaction, + decoded_event, + "meteora_dbc", + crate::PoolKind::BondingCurve, + crate::PoolStatus::Pending, + "signal.dex.meteora_dbc", + ) + .await; } async fn detect_meteora_damm_v2_pool( @@ -1501,236 +385,16 @@ impl DexDetectService { transaction: &crate::ChainTransactionDto, decoded_event: &crate::DexDecodedEventDto, ) -> Result { - let decoded_event_id_option = decoded_event.id; - let decoded_event_id = match decoded_event_id_option { - Some(decoded_event_id) => decoded_event_id, - None => { - return Err(crate::Error::InvalidState( - "decoded dex event has no internal id".to_string(), - )); - }, - }; - let dex_id_result = self.ensure_meteora_damm_v2_dex().await; - let dex_id = match dex_id_result { - Ok(dex_id) => dex_id, - Err(error) => return Err(error), - }; - let pool_address_option = decoded_event.pool_account.clone(); - let pool_address = match pool_address_option { - Some(pool_address) => pool_address, - None => { - return Err(crate::Error::InvalidState(format!( - "decoded event '{}' has no pool_account", - decoded_event_id - ))); - }, - }; - let token_a_mint_option = decoded_event.token_a_mint.clone(); - let token_a_mint = match token_a_mint_option { - Some(token_a_mint) => token_a_mint, - None => { - return Err(crate::Error::InvalidState(format!( - "decoded event '{}' has no token_a_mint", - decoded_event_id - ))); - }, - }; - let token_b_mint_option = decoded_event.token_b_mint.clone(); - let token_b_mint = match token_b_mint_option { - Some(token_b_mint) => token_b_mint, - None => { - return Err(crate::Error::InvalidState(format!( - "decoded event '{}' has no token_b_mint", - decoded_event_id - ))); - }, - }; - let base_is_token_a = choose_base_quote_order(token_a_mint.as_str(), token_b_mint.as_str()); - let base_mint = if base_is_token_a { token_a_mint.clone() } else { token_b_mint.clone() }; - let quote_mint = if base_is_token_a { token_b_mint.clone() } else { token_a_mint.clone() }; - let base_token_id_result = self.ensure_token(base_mint.as_str()).await; - let base_token_id = match base_token_id_result { - Ok(base_token_id) => base_token_id, - Err(error) => return Err(error), - }; - let quote_token_id_result = self.ensure_token(quote_mint.as_str()).await; - let quote_token_id = match quote_token_id_result { - Ok(quote_token_id) => quote_token_id, - Err(error) => return Err(error), - }; - let existing_pool_result = - crate::query_pools_get_by_address(self.database.as_ref(), pool_address.as_str()).await; - let existing_pool_option = match existing_pool_result { - Ok(existing_pool_option) => existing_pool_option, - Err(error) => return Err(error), - }; - let created_pool = existing_pool_option.is_none(); - let pool_id = match existing_pool_option { - Some(pool) => { - let pool_id_option = pool.id; - match pool_id_option { - Some(pool_id) => pool_id, - None => { - return Err(crate::Error::InvalidState(format!( - "pool '{}' has no internal id", - pool.address - ))); - }, - } - }, - None => { - let pool_dto = crate::PoolDto::new( - dex_id, - pool_address.clone(), - crate::PoolKind::Amm, - crate::PoolStatus::Active, - ); - let upsert_result = - crate::query_pools_upsert(self.database.as_ref(), &pool_dto).await; - match upsert_result { - Ok(pool_id) => pool_id, - Err(error) => return Err(error), - } - }, - }; - let existing_pair_result = - crate::query_pairs_get_by_pool_id(self.database.as_ref(), pool_id).await; - let existing_pair_option = match existing_pair_result { - Ok(existing_pair_option) => existing_pair_option, - Err(error) => return Err(error), - }; - let created_pair = existing_pair_option.is_none(); - let pair_symbol = build_pair_symbol(base_mint.as_str(), quote_mint.as_str()); - let pair_id = match existing_pair_option { - Some(pair) => { - let pair_id_option = pair.id; - match pair_id_option { - Some(pair_id) => pair_id, - None => { - return Err(crate::Error::InvalidState(format!( - "pair for pool '{}' has no internal id", - pool_id - ))); - }, - } - }, - None => { - let pair_dto = crate::PairDto::new( - dex_id, - pool_id, - base_token_id, - quote_token_id, - pair_symbol, - ); - let upsert_result = - crate::query_pairs_upsert(self.database.as_ref(), &pair_dto).await; - match upsert_result { - Ok(pair_id) => pair_id, - Err(error) => return Err(error), - } - }, - }; - let upsert_base_pool_token_result = crate::query_pool_tokens_upsert( - self.database.as_ref(), - &crate::PoolTokenDto::new( - pool_id, - base_token_id, - crate::PoolTokenRole::Base, - None, - Some(0), - ), - ) - .await; - if let Err(error) = upsert_base_pool_token_result { - return Err(error); - } - let upsert_quote_pool_token_result = crate::query_pool_tokens_upsert( - self.database.as_ref(), - &crate::PoolTokenDto::new( - pool_id, - quote_token_id, - crate::PoolTokenRole::Quote, - None, - Some(1), - ), - ) - .await; - if let Err(error) = upsert_quote_pool_token_result { - return Err(error); - } - let existing_listing_result = - crate::query_pool_listings_get_by_pool_id(self.database.as_ref(), pool_id).await; - let existing_listing_option = match existing_listing_result { - Ok(existing_listing_option) => existing_listing_option, - Err(error) => return Err(error), - }; - let created_listing = existing_listing_option.is_none(); - let pool_listing_id = match existing_listing_option { - Some(pool_listing) => pool_listing.id, - None => { - let listing_id_result = self - .upsert_pool_listing_from_decoded_event(dex_id, pool_id, pair_id, transaction) - .await; - match listing_id_result { - Ok(listing_id) => Some(listing_id), - Err(error) => return Err(error), - } - }, - }; - let payload_value_result = parse_payload_json(decoded_event.payload_json.as_str()); - let payload_value = match payload_value_result { - Ok(payload_value) => payload_value, - Err(error) => return Err(error), - }; - if created_pool { - let signal_result = self - .record_detection_signal( - transaction, - "signal.dex.meteora_damm_v2.new_pool", - crate::AnalysisSignalSeverity::Low, - payload_value.clone(), - ) - .await; - if let Err(error) = signal_result { - return Err(error); - } - } - if created_pair { - let signal_result = self - .record_detection_signal( - transaction, - "signal.dex.meteora_damm_v2.new_pair", - crate::AnalysisSignalSeverity::Low, - payload_value.clone(), - ) - .await; - if let Err(error) = signal_result { - return Err(error); - } - } - if created_listing { - let signal_result = self - .record_detection_signal( - transaction, - "signal.dex.meteora_damm_v2.first_listing_seen", - crate::AnalysisSignalSeverity::Low, - payload_value, - ) - .await; - if let Err(error) = signal_result { - return Err(error); - } - } - return Ok(crate::DexPoolDetectionResult { - decoded_event_id, - dex_id, - pool_id, - pair_id, - pool_listing_id, - created_pool, - created_pair, - created_listing, - }); + return self + .detect_materialized_pool_from_decoded_event( + transaction, + decoded_event, + "meteora_damm_v2", + crate::PoolKind::Amm, + crate::PoolStatus::Active, + "signal.dex.meteora_damm_v2", + ) + .await; } async fn detect_meteora_damm_v1_pool( @@ -1738,236 +402,16 @@ impl DexDetectService { transaction: &crate::ChainTransactionDto, decoded_event: &crate::DexDecodedEventDto, ) -> Result { - let decoded_event_id_option = decoded_event.id; - let decoded_event_id = match decoded_event_id_option { - Some(decoded_event_id) => decoded_event_id, - None => { - return Err(crate::Error::InvalidState( - "decoded dex event has no internal id".to_string(), - )); - }, - }; - let dex_id_result = self.ensure_meteora_damm_v1_dex().await; - let dex_id = match dex_id_result { - Ok(dex_id) => dex_id, - Err(error) => return Err(error), - }; - let pool_address_option = decoded_event.pool_account.clone(); - let pool_address = match pool_address_option { - Some(pool_address) => pool_address, - None => { - return Err(crate::Error::InvalidState(format!( - "decoded event '{}' has no pool_account", - decoded_event_id - ))); - }, - }; - let token_a_mint_option = decoded_event.token_a_mint.clone(); - let token_a_mint = match token_a_mint_option { - Some(token_a_mint) => token_a_mint, - None => { - return Err(crate::Error::InvalidState(format!( - "decoded event '{}' has no token_a_mint", - decoded_event_id - ))); - }, - }; - let token_b_mint_option = decoded_event.token_b_mint.clone(); - let token_b_mint = match token_b_mint_option { - Some(token_b_mint) => token_b_mint, - None => { - return Err(crate::Error::InvalidState(format!( - "decoded event '{}' has no token_b_mint", - decoded_event_id - ))); - }, - }; - let base_is_token_a = choose_base_quote_order(token_a_mint.as_str(), token_b_mint.as_str()); - let base_mint = if base_is_token_a { token_a_mint.clone() } else { token_b_mint.clone() }; - let quote_mint = if base_is_token_a { token_b_mint.clone() } else { token_a_mint.clone() }; - let base_token_id_result = self.ensure_token(base_mint.as_str()).await; - let base_token_id = match base_token_id_result { - Ok(base_token_id) => base_token_id, - Err(error) => return Err(error), - }; - let quote_token_id_result = self.ensure_token(quote_mint.as_str()).await; - let quote_token_id = match quote_token_id_result { - Ok(quote_token_id) => quote_token_id, - Err(error) => return Err(error), - }; - let existing_pool_result = - crate::query_pools_get_by_address(self.database.as_ref(), pool_address.as_str()).await; - let existing_pool_option = match existing_pool_result { - Ok(existing_pool_option) => existing_pool_option, - Err(error) => return Err(error), - }; - let created_pool = existing_pool_option.is_none(); - let pool_id = match existing_pool_option { - Some(pool) => { - let pool_id_option = pool.id; - match pool_id_option { - Some(pool_id) => pool_id, - None => { - return Err(crate::Error::InvalidState(format!( - "pool '{}' has no internal id", - pool.address - ))); - }, - } - }, - None => { - let pool_dto = crate::PoolDto::new( - dex_id, - pool_address.clone(), - crate::PoolKind::Amm, - crate::PoolStatus::Active, - ); - let upsert_result = - crate::query_pools_upsert(self.database.as_ref(), &pool_dto).await; - match upsert_result { - Ok(pool_id) => pool_id, - Err(error) => return Err(error), - } - }, - }; - let existing_pair_result = - crate::query_pairs_get_by_pool_id(self.database.as_ref(), pool_id).await; - let existing_pair_option = match existing_pair_result { - Ok(existing_pair_option) => existing_pair_option, - Err(error) => return Err(error), - }; - let created_pair = existing_pair_option.is_none(); - let pair_symbol = build_pair_symbol(base_mint.as_str(), quote_mint.as_str()); - let pair_id = match existing_pair_option { - Some(pair) => { - let pair_id_option = pair.id; - match pair_id_option { - Some(pair_id) => pair_id, - None => { - return Err(crate::Error::InvalidState(format!( - "pair for pool '{}' has no internal id", - pool_id - ))); - }, - } - }, - None => { - let pair_dto = crate::PairDto::new( - dex_id, - pool_id, - base_token_id, - quote_token_id, - pair_symbol, - ); - let upsert_result = - crate::query_pairs_upsert(self.database.as_ref(), &pair_dto).await; - match upsert_result { - Ok(pair_id) => pair_id, - Err(error) => return Err(error), - } - }, - }; - let upsert_base_pool_token_result = crate::query_pool_tokens_upsert( - self.database.as_ref(), - &crate::PoolTokenDto::new( - pool_id, - base_token_id, - crate::PoolTokenRole::Base, - None, - Some(0), - ), - ) - .await; - if let Err(error) = upsert_base_pool_token_result { - return Err(error); - } - let upsert_quote_pool_token_result = crate::query_pool_tokens_upsert( - self.database.as_ref(), - &crate::PoolTokenDto::new( - pool_id, - quote_token_id, - crate::PoolTokenRole::Quote, - None, - Some(1), - ), - ) - .await; - if let Err(error) = upsert_quote_pool_token_result { - return Err(error); - } - let existing_listing_result = - crate::query_pool_listings_get_by_pool_id(self.database.as_ref(), pool_id).await; - let existing_listing_option = match existing_listing_result { - Ok(existing_listing_option) => existing_listing_option, - Err(error) => return Err(error), - }; - let created_listing = existing_listing_option.is_none(); - let pool_listing_id = match existing_listing_option { - Some(pool_listing) => pool_listing.id, - None => { - let listing_id_result = self - .upsert_pool_listing_from_decoded_event(dex_id, pool_id, pair_id, transaction) - .await; - match listing_id_result { - Ok(listing_id) => Some(listing_id), - Err(error) => return Err(error), - } - }, - }; - let payload_value_result = parse_payload_json(decoded_event.payload_json.as_str()); - let payload_value = match payload_value_result { - Ok(payload_value) => payload_value, - Err(error) => return Err(error), - }; - if created_pool { - let signal_result = self - .record_detection_signal( - transaction, - "signal.dex.meteora_damm_v1.new_pool", - crate::AnalysisSignalSeverity::Low, - payload_value.clone(), - ) - .await; - if let Err(error) = signal_result { - return Err(error); - } - } - if created_pair { - let signal_result = self - .record_detection_signal( - transaction, - "signal.dex.meteora_damm_v1.new_pair", - crate::AnalysisSignalSeverity::Low, - payload_value.clone(), - ) - .await; - if let Err(error) = signal_result { - return Err(error); - } - } - if created_listing { - let signal_result = self - .record_detection_signal( - transaction, - "signal.dex.meteora_damm_v1.first_listing_seen", - crate::AnalysisSignalSeverity::Low, - payload_value, - ) - .await; - if let Err(error) = signal_result { - return Err(error); - } - } - return Ok(crate::DexPoolDetectionResult { - decoded_event_id, - dex_id, - pool_id, - pair_id, - pool_listing_id, - created_pool, - created_pair, - created_listing, - }); + return self + .detect_materialized_pool_from_decoded_event( + transaction, + decoded_event, + "meteora_damm_v1", + crate::PoolKind::Amm, + crate::PoolStatus::Active, + "signal.dex.meteora_damm_v1", + ) + .await; } async fn detect_orca_whirlpools_pool( @@ -1975,236 +419,16 @@ impl DexDetectService { transaction: &crate::ChainTransactionDto, decoded_event: &crate::DexDecodedEventDto, ) -> Result { - let decoded_event_id_option = decoded_event.id; - let decoded_event_id = match decoded_event_id_option { - Some(decoded_event_id) => decoded_event_id, - None => { - return Err(crate::Error::InvalidState( - "decoded dex event has no internal id".to_string(), - )); - }, - }; - let dex_id_result = self.ensure_orca_whirlpools_dex().await; - let dex_id = match dex_id_result { - Ok(dex_id) => dex_id, - Err(error) => return Err(error), - }; - let pool_address_option = decoded_event.pool_account.clone(); - let pool_address = match pool_address_option { - Some(pool_address) => pool_address, - None => { - return Err(crate::Error::InvalidState(format!( - "decoded event '{}' has no pool_account", - decoded_event_id - ))); - }, - }; - let token_a_mint_option = decoded_event.token_a_mint.clone(); - let token_a_mint = match token_a_mint_option { - Some(token_a_mint) => token_a_mint, - None => { - return Err(crate::Error::InvalidState(format!( - "decoded event '{}' has no token_a_mint", - decoded_event_id - ))); - }, - }; - let token_b_mint_option = decoded_event.token_b_mint.clone(); - let token_b_mint = match token_b_mint_option { - Some(token_b_mint) => token_b_mint, - None => { - return Err(crate::Error::InvalidState(format!( - "decoded event '{}' has no token_b_mint", - decoded_event_id - ))); - }, - }; - let base_is_token_a = choose_base_quote_order(token_a_mint.as_str(), token_b_mint.as_str()); - let base_mint = if base_is_token_a { token_a_mint.clone() } else { token_b_mint.clone() }; - let quote_mint = if base_is_token_a { token_b_mint.clone() } else { token_a_mint.clone() }; - let base_token_id_result = self.ensure_token(base_mint.as_str()).await; - let base_token_id = match base_token_id_result { - Ok(base_token_id) => base_token_id, - Err(error) => return Err(error), - }; - let quote_token_id_result = self.ensure_token(quote_mint.as_str()).await; - let quote_token_id = match quote_token_id_result { - Ok(quote_token_id) => quote_token_id, - Err(error) => return Err(error), - }; - let existing_pool_result = - crate::query_pools_get_by_address(self.database.as_ref(), pool_address.as_str()).await; - let existing_pool_option = match existing_pool_result { - Ok(existing_pool_option) => existing_pool_option, - Err(error) => return Err(error), - }; - let created_pool = existing_pool_option.is_none(); - let pool_id = match existing_pool_option { - Some(pool) => { - let pool_id_option = pool.id; - match pool_id_option { - Some(pool_id) => pool_id, - None => { - return Err(crate::Error::InvalidState(format!( - "pool '{}' has no internal id", - pool.address - ))); - }, - } - }, - None => { - let pool_dto = crate::PoolDto::new( - dex_id, - pool_address.clone(), - crate::PoolKind::Clmm, - crate::PoolStatus::Active, - ); - let upsert_result = - crate::query_pools_upsert(self.database.as_ref(), &pool_dto).await; - match upsert_result { - Ok(pool_id) => pool_id, - Err(error) => return Err(error), - } - }, - }; - let existing_pair_result = - crate::query_pairs_get_by_pool_id(self.database.as_ref(), pool_id).await; - let existing_pair_option = match existing_pair_result { - Ok(existing_pair_option) => existing_pair_option, - Err(error) => return Err(error), - }; - let created_pair = existing_pair_option.is_none(); - let pair_symbol = build_pair_symbol(base_mint.as_str(), quote_mint.as_str()); - let pair_id = match existing_pair_option { - Some(pair) => { - let pair_id_option = pair.id; - match pair_id_option { - Some(pair_id) => pair_id, - None => { - return Err(crate::Error::InvalidState(format!( - "pair for pool '{}' has no internal id", - pool_id - ))); - }, - } - }, - None => { - let pair_dto = crate::PairDto::new( - dex_id, - pool_id, - base_token_id, - quote_token_id, - pair_symbol, - ); - let upsert_result = - crate::query_pairs_upsert(self.database.as_ref(), &pair_dto).await; - match upsert_result { - Ok(pair_id) => pair_id, - Err(error) => return Err(error), - } - }, - }; - let upsert_base_pool_token_result = crate::query_pool_tokens_upsert( - self.database.as_ref(), - &crate::PoolTokenDto::new( - pool_id, - base_token_id, - crate::PoolTokenRole::Base, - None, - Some(0), - ), - ) - .await; - if let Err(error) = upsert_base_pool_token_result { - return Err(error); - } - let upsert_quote_pool_token_result = crate::query_pool_tokens_upsert( - self.database.as_ref(), - &crate::PoolTokenDto::new( - pool_id, - quote_token_id, - crate::PoolTokenRole::Quote, - None, - Some(1), - ), - ) - .await; - if let Err(error) = upsert_quote_pool_token_result { - return Err(error); - } - let existing_listing_result = - crate::query_pool_listings_get_by_pool_id(self.database.as_ref(), pool_id).await; - let existing_listing_option = match existing_listing_result { - Ok(existing_listing_option) => existing_listing_option, - Err(error) => return Err(error), - }; - let created_listing = existing_listing_option.is_none(); - let pool_listing_id = match existing_listing_option { - Some(pool_listing) => pool_listing.id, - None => { - let listing_id_result = self - .upsert_pool_listing_from_decoded_event(dex_id, pool_id, pair_id, transaction) - .await; - match listing_id_result { - Ok(listing_id) => Some(listing_id), - Err(error) => return Err(error), - } - }, - }; - let payload_value_result = parse_payload_json(decoded_event.payload_json.as_str()); - let payload_value = match payload_value_result { - Ok(payload_value) => payload_value, - Err(error) => return Err(error), - }; - if created_pool { - let signal_result = self - .record_detection_signal( - transaction, - "signal.dex.orca_whirlpools.new_pool", - crate::AnalysisSignalSeverity::Low, - payload_value.clone(), - ) - .await; - if let Err(error) = signal_result { - return Err(error); - } - } - if created_pair { - let signal_result = self - .record_detection_signal( - transaction, - "signal.dex.orca_whirlpools.new_pair", - crate::AnalysisSignalSeverity::Low, - payload_value.clone(), - ) - .await; - if let Err(error) = signal_result { - return Err(error); - } - } - if created_listing { - let signal_result = self - .record_detection_signal( - transaction, - "signal.dex.orca_whirlpools.first_listing_seen", - crate::AnalysisSignalSeverity::Low, - payload_value, - ) - .await; - if let Err(error) = signal_result { - return Err(error); - } - } - return Ok(crate::DexPoolDetectionResult { - decoded_event_id, - dex_id, - pool_id, - pair_id, - pool_listing_id, - created_pool, - created_pair, - created_listing, - }); + return self + .detect_materialized_pool_from_decoded_event( + transaction, + decoded_event, + "orca_whirlpools", + crate::PoolKind::Clmm, + crate::PoolStatus::Active, + "signal.dex.orca_whirlpools", + ) + .await; } async fn detect_fluxbeam_pool( @@ -2212,236 +436,16 @@ impl DexDetectService { transaction: &crate::ChainTransactionDto, decoded_event: &crate::DexDecodedEventDto, ) -> Result { - let decoded_event_id_option = decoded_event.id; - let decoded_event_id = match decoded_event_id_option { - Some(decoded_event_id) => decoded_event_id, - None => { - return Err(crate::Error::InvalidState( - "decoded dex event has no internal id".to_string(), - )); - }, - }; - let dex_id_result = self.ensure_fluxbeam_dex().await; - let dex_id = match dex_id_result { - Ok(dex_id) => dex_id, - Err(error) => return Err(error), - }; - let pool_address_option = decoded_event.pool_account.clone(); - let pool_address = match pool_address_option { - Some(pool_address) => pool_address, - None => { - return Err(crate::Error::InvalidState(format!( - "decoded event '{}' has no pool_account", - decoded_event_id - ))); - }, - }; - let token_a_mint_option = decoded_event.token_a_mint.clone(); - let token_a_mint = match token_a_mint_option { - Some(token_a_mint) => token_a_mint, - None => { - return Err(crate::Error::InvalidState(format!( - "decoded event '{}' has no token_a_mint", - decoded_event_id - ))); - }, - }; - let token_b_mint_option = decoded_event.token_b_mint.clone(); - let token_b_mint = match token_b_mint_option { - Some(token_b_mint) => token_b_mint, - None => { - return Err(crate::Error::InvalidState(format!( - "decoded event '{}' has no token_b_mint", - decoded_event_id - ))); - }, - }; - let base_is_token_a = choose_base_quote_order(token_a_mint.as_str(), token_b_mint.as_str()); - let base_mint = if base_is_token_a { token_a_mint.clone() } else { token_b_mint.clone() }; - let quote_mint = if base_is_token_a { token_b_mint.clone() } else { token_a_mint.clone() }; - let base_token_id_result = self.ensure_token(base_mint.as_str()).await; - let base_token_id = match base_token_id_result { - Ok(base_token_id) => base_token_id, - Err(error) => return Err(error), - }; - let quote_token_id_result = self.ensure_token(quote_mint.as_str()).await; - let quote_token_id = match quote_token_id_result { - Ok(quote_token_id) => quote_token_id, - Err(error) => return Err(error), - }; - let existing_pool_result = - crate::query_pools_get_by_address(self.database.as_ref(), pool_address.as_str()).await; - let existing_pool_option = match existing_pool_result { - Ok(existing_pool_option) => existing_pool_option, - Err(error) => return Err(error), - }; - let created_pool = existing_pool_option.is_none(); - let pool_id = match existing_pool_option { - Some(pool) => { - let pool_id_option = pool.id; - match pool_id_option { - Some(pool_id) => pool_id, - None => { - return Err(crate::Error::InvalidState(format!( - "pool '{}' has no internal id", - pool.address - ))); - }, - } - }, - None => { - let pool_dto = crate::PoolDto::new( - dex_id, - pool_address.clone(), - crate::PoolKind::Amm, - crate::PoolStatus::Active, - ); - let upsert_result = - crate::query_pools_upsert(self.database.as_ref(), &pool_dto).await; - match upsert_result { - Ok(pool_id) => pool_id, - Err(error) => return Err(error), - } - }, - }; - let existing_pair_result = - crate::query_pairs_get_by_pool_id(self.database.as_ref(), pool_id).await; - let existing_pair_option = match existing_pair_result { - Ok(existing_pair_option) => existing_pair_option, - Err(error) => return Err(error), - }; - let created_pair = existing_pair_option.is_none(); - let pair_symbol = build_pair_symbol(base_mint.as_str(), quote_mint.as_str()); - let pair_id = match existing_pair_option { - Some(pair) => { - let pair_id_option = pair.id; - match pair_id_option { - Some(pair_id) => pair_id, - None => { - return Err(crate::Error::InvalidState(format!( - "pair for pool '{}' has no internal id", - pool_id - ))); - }, - } - }, - None => { - let pair_dto = crate::PairDto::new( - dex_id, - pool_id, - base_token_id, - quote_token_id, - pair_symbol, - ); - let upsert_result = - crate::query_pairs_upsert(self.database.as_ref(), &pair_dto).await; - match upsert_result { - Ok(pair_id) => pair_id, - Err(error) => return Err(error), - } - }, - }; - let upsert_base_pool_token_result = crate::query_pool_tokens_upsert( - self.database.as_ref(), - &crate::PoolTokenDto::new( - pool_id, - base_token_id, - crate::PoolTokenRole::Base, - None, - Some(0), - ), - ) - .await; - if let Err(error) = upsert_base_pool_token_result { - return Err(error); - } - let upsert_quote_pool_token_result = crate::query_pool_tokens_upsert( - self.database.as_ref(), - &crate::PoolTokenDto::new( - pool_id, - quote_token_id, - crate::PoolTokenRole::Quote, - None, - Some(1), - ), - ) - .await; - if let Err(error) = upsert_quote_pool_token_result { - return Err(error); - } - let existing_listing_result = - crate::query_pool_listings_get_by_pool_id(self.database.as_ref(), pool_id).await; - let existing_listing_option = match existing_listing_result { - Ok(existing_listing_option) => existing_listing_option, - Err(error) => return Err(error), - }; - let created_listing = existing_listing_option.is_none(); - let pool_listing_id = match existing_listing_option { - Some(pool_listing) => pool_listing.id, - None => { - let listing_id_result = self - .upsert_pool_listing_from_decoded_event(dex_id, pool_id, pair_id, transaction) - .await; - match listing_id_result { - Ok(listing_id) => Some(listing_id), - Err(error) => return Err(error), - } - }, - }; - let payload_value_result = parse_payload_json(decoded_event.payload_json.as_str()); - let payload_value = match payload_value_result { - Ok(payload_value) => payload_value, - Err(error) => return Err(error), - }; - if created_pool { - let signal_result = self - .record_detection_signal( - transaction, - "signal.dex.fluxbeam.new_pool", - crate::AnalysisSignalSeverity::Low, - payload_value.clone(), - ) - .await; - if let Err(error) = signal_result { - return Err(error); - } - } - if created_pair { - let signal_result = self - .record_detection_signal( - transaction, - "signal.dex.fluxbeam.new_pair", - crate::AnalysisSignalSeverity::Low, - payload_value.clone(), - ) - .await; - if let Err(error) = signal_result { - return Err(error); - } - } - if created_listing { - let signal_result = self - .record_detection_signal( - transaction, - "signal.dex.fluxbeam.first_listing_seen", - crate::AnalysisSignalSeverity::Low, - payload_value, - ) - .await; - if let Err(error) = signal_result { - return Err(error); - } - } - return Ok(crate::DexPoolDetectionResult { - decoded_event_id, - dex_id, - pool_id, - pair_id, - pool_listing_id, - created_pool, - created_pair, - created_listing, - }); + return self + .detect_materialized_pool_from_decoded_event( + transaction, + decoded_event, + "fluxbeam", + crate::PoolKind::Amm, + crate::PoolStatus::Active, + "signal.dex.fluxbeam", + ) + .await; } async fn detect_dexlab_pool( @@ -2449,236 +453,16 @@ impl DexDetectService { transaction: &crate::ChainTransactionDto, decoded_event: &crate::DexDecodedEventDto, ) -> Result { - let decoded_event_id_option = decoded_event.id; - let decoded_event_id = match decoded_event_id_option { - Some(decoded_event_id) => decoded_event_id, - None => { - return Err(crate::Error::InvalidState( - "decoded dex event has no internal id".to_string(), - )); - }, - }; - let dex_id_result = self.ensure_dexlab_dex().await; - let dex_id = match dex_id_result { - Ok(dex_id) => dex_id, - Err(error) => return Err(error), - }; - let pool_address_option = decoded_event.pool_account.clone(); - let pool_address = match pool_address_option { - Some(pool_address) => pool_address, - None => { - return Err(crate::Error::InvalidState(format!( - "decoded event '{}' has no pool_account", - decoded_event_id - ))); - }, - }; - let token_a_mint_option = decoded_event.token_a_mint.clone(); - let token_a_mint = match token_a_mint_option { - Some(token_a_mint) => token_a_mint, - None => { - return Err(crate::Error::InvalidState(format!( - "decoded event '{}' has no token_a_mint", - decoded_event_id - ))); - }, - }; - let token_b_mint_option = decoded_event.token_b_mint.clone(); - let token_b_mint = match token_b_mint_option { - Some(token_b_mint) => token_b_mint, - None => { - return Err(crate::Error::InvalidState(format!( - "decoded event '{}' has no token_b_mint", - decoded_event_id - ))); - }, - }; - let base_is_token_a = choose_base_quote_order(token_a_mint.as_str(), token_b_mint.as_str()); - let base_mint = if base_is_token_a { token_a_mint.clone() } else { token_b_mint.clone() }; - let quote_mint = if base_is_token_a { token_b_mint.clone() } else { token_a_mint.clone() }; - let base_token_id_result = self.ensure_token(base_mint.as_str()).await; - let base_token_id = match base_token_id_result { - Ok(base_token_id) => base_token_id, - Err(error) => return Err(error), - }; - let quote_token_id_result = self.ensure_token(quote_mint.as_str()).await; - let quote_token_id = match quote_token_id_result { - Ok(quote_token_id) => quote_token_id, - Err(error) => return Err(error), - }; - let existing_pool_result = - crate::query_pools_get_by_address(self.database.as_ref(), pool_address.as_str()).await; - let existing_pool_option = match existing_pool_result { - Ok(existing_pool_option) => existing_pool_option, - Err(error) => return Err(error), - }; - let created_pool = existing_pool_option.is_none(); - let pool_id = match existing_pool_option { - Some(pool) => { - let pool_id_option = pool.id; - match pool_id_option { - Some(pool_id) => pool_id, - None => { - return Err(crate::Error::InvalidState(format!( - "pool '{}' has no internal id", - pool.address - ))); - }, - } - }, - None => { - let pool_dto = crate::PoolDto::new( - dex_id, - pool_address.clone(), - crate::PoolKind::Amm, - crate::PoolStatus::Active, - ); - let upsert_result = - crate::query_pools_upsert(self.database.as_ref(), &pool_dto).await; - match upsert_result { - Ok(pool_id) => pool_id, - Err(error) => return Err(error), - } - }, - }; - let existing_pair_result = - crate::query_pairs_get_by_pool_id(self.database.as_ref(), pool_id).await; - let existing_pair_option = match existing_pair_result { - Ok(existing_pair_option) => existing_pair_option, - Err(error) => return Err(error), - }; - let created_pair = existing_pair_option.is_none(); - let pair_symbol = build_pair_symbol(base_mint.as_str(), quote_mint.as_str()); - let pair_id = match existing_pair_option { - Some(pair) => { - let pair_id_option = pair.id; - match pair_id_option { - Some(pair_id) => pair_id, - None => { - return Err(crate::Error::InvalidState(format!( - "pair for pool '{}' has no internal id", - pool_id - ))); - }, - } - }, - None => { - let pair_dto = crate::PairDto::new( - dex_id, - pool_id, - base_token_id, - quote_token_id, - pair_symbol, - ); - let upsert_result = - crate::query_pairs_upsert(self.database.as_ref(), &pair_dto).await; - match upsert_result { - Ok(pair_id) => pair_id, - Err(error) => return Err(error), - } - }, - }; - let upsert_base_pool_token_result = crate::query_pool_tokens_upsert( - self.database.as_ref(), - &crate::PoolTokenDto::new( - pool_id, - base_token_id, - crate::PoolTokenRole::Base, - None, - Some(0), - ), - ) - .await; - if let Err(error) = upsert_base_pool_token_result { - return Err(error); - } - let upsert_quote_pool_token_result = crate::query_pool_tokens_upsert( - self.database.as_ref(), - &crate::PoolTokenDto::new( - pool_id, - quote_token_id, - crate::PoolTokenRole::Quote, - None, - Some(1), - ), - ) - .await; - if let Err(error) = upsert_quote_pool_token_result { - return Err(error); - } - let existing_listing_result = - crate::query_pool_listings_get_by_pool_id(self.database.as_ref(), pool_id).await; - let existing_listing_option = match existing_listing_result { - Ok(existing_listing_option) => existing_listing_option, - Err(error) => return Err(error), - }; - let created_listing = existing_listing_option.is_none(); - let pool_listing_id = match existing_listing_option { - Some(pool_listing) => pool_listing.id, - None => { - let listing_id_result = self - .upsert_pool_listing_from_decoded_event(dex_id, pool_id, pair_id, transaction) - .await; - match listing_id_result { - Ok(listing_id) => Some(listing_id), - Err(error) => return Err(error), - } - }, - }; - let payload_value_result = parse_payload_json(decoded_event.payload_json.as_str()); - let payload_value = match payload_value_result { - Ok(payload_value) => payload_value, - Err(error) => return Err(error), - }; - if created_pool { - let signal_result = self - .record_detection_signal( - transaction, - "signal.dex.dexlab.new_pool", - crate::AnalysisSignalSeverity::Low, - payload_value.clone(), - ) - .await; - if let Err(error) = signal_result { - return Err(error); - } - } - if created_pair { - let signal_result = self - .record_detection_signal( - transaction, - "signal.dex.dexlab.new_pair", - crate::AnalysisSignalSeverity::Low, - payload_value.clone(), - ) - .await; - if let Err(error) = signal_result { - return Err(error); - } - } - if created_listing { - let signal_result = self - .record_detection_signal( - transaction, - "signal.dex.dexlab.first_listing_seen", - crate::AnalysisSignalSeverity::Low, - payload_value, - ) - .await; - if let Err(error) = signal_result { - return Err(error); - } - } - return Ok(crate::DexPoolDetectionResult { - decoded_event_id, - dex_id, - pool_id, - pair_id, - pool_listing_id, - created_pool, - created_pair, - created_listing, - }); + return self + .detect_materialized_pool_from_decoded_event( + transaction, + decoded_event, + "dexlab", + crate::PoolKind::Amm, + crate::PoolStatus::Active, + "signal.dex.dexlab", + ) + .await; } async fn detect_raydium_clmm_trade( @@ -2686,46 +470,12 @@ impl DexDetectService { transaction: &crate::ChainTransactionDto, decoded_event: &crate::DexDecodedEventDto, ) -> Result { - let decoded_event_id = match decoded_event.id { - Some(decoded_event_id) => decoded_event_id, - None => { - return Err(crate::Error::InvalidState( - "decoded dex event has no internal id".to_string(), - )); - }, - }; - let dex_id_result = self.ensure_raydium_clmm_dex().await; + let dex_id_result = + crate::dex_catalog::ensure_known_dex(self.database.as_ref(), "raydium_clmm").await; let dex_id = match dex_id_result { Ok(dex_id) => dex_id, Err(error) => return Err(error), }; - let pool_address = match decoded_event.pool_account.clone() { - Some(pool_address) => pool_address, - None => { - return Err(crate::Error::InvalidState(format!( - "decoded event '{}' has no pool_account", - decoded_event_id - ))); - }, - }; - let base_mint = match decoded_event.token_a_mint.clone() { - Some(base_mint) => base_mint, - None => { - return Err(crate::Error::InvalidState(format!( - "decoded event '{}' has no token_a_mint", - decoded_event_id - ))); - }, - }; - let quote_mint = match decoded_event.token_b_mint.clone() { - Some(quote_mint) => quote_mint, - None => { - return Err(crate::Error::InvalidState(format!( - "decoded event '{}' has no token_b_mint", - decoded_event_id - ))); - }, - }; let payload_value_result = parse_payload_json(decoded_event.payload_json.as_str()); let payload_value = match payload_value_result { Ok(payload_value) => payload_value, @@ -2733,161 +483,40 @@ impl DexDetectService { }; let base_vault_address = extract_payload_string_field(&payload_value, "base_vault"); let quote_vault_address = extract_payload_string_field(&payload_value, "quote_vault"); - let base_token_id_result = self.ensure_token(base_mint.as_str()).await; - let base_token_id = match base_token_id_result { - Ok(base_token_id) => base_token_id, - Err(error) => return Err(error), - }; - let quote_token_id_result = self.ensure_token(quote_mint.as_str()).await; - let quote_token_id = match quote_token_id_result { - Ok(quote_token_id) => quote_token_id, - Err(error) => return Err(error), - }; - let existing_pool_result = - crate::query_pools_get_by_address(self.database.as_ref(), pool_address.as_str()).await; - let existing_pool_option = match existing_pool_result { - Ok(existing_pool_option) => existing_pool_option, - Err(error) => return Err(error), - }; - let created_pool = existing_pool_option.is_none(); - let pool_id = match existing_pool_option { - Some(pool) => match pool.id { - Some(pool_id) => pool_id, - None => { - return Err(crate::Error::InvalidState(format!( - "pool '{}' has no internal id", - pool.address - ))); - }, - }, - None => { - let pool_dto = crate::PoolDto::new( - dex_id, - pool_address.clone(), - crate::PoolKind::Clmm, - crate::PoolStatus::Active, - ); - let upsert_result = - crate::query_pools_upsert(self.database.as_ref(), &pool_dto).await; - match upsert_result { - Ok(pool_id) => pool_id, - Err(error) => return Err(error), - } - }, - }; - let existing_pair_result = - crate::query_pairs_get_by_pool_id(self.database.as_ref(), pool_id).await; - let existing_pair_option = match existing_pair_result { - Ok(existing_pair_option) => existing_pair_option, - Err(error) => return Err(error), - }; - - let created_pair = existing_pair_option.is_none(); - let pair_symbol = build_pair_symbol(base_mint.as_str(), quote_mint.as_str()); - - let pair_dto = - crate::PairDto::new(dex_id, pool_id, base_token_id, quote_token_id, pair_symbol); - let pair_id_result = crate::query_pairs_upsert(self.database.as_ref(), &pair_dto).await; - let pair_id = match pair_id_result { - Ok(pair_id) => pair_id, - Err(error) => return Err(error), - }; - let upsert_base_pool_token_result = crate::query_pool_tokens_upsert( - self.database.as_ref(), - &crate::PoolTokenDto::new( - pool_id, - base_token_id, - crate::PoolTokenRole::Base, + let input_result = + crate::dex_pool_materialization::DexPoolMaterializationInput::from_decoded_event( + decoded_event, + dex_id, + crate::PoolKind::Clmm, + crate::PoolStatus::Active, + crate::dex_pool_materialization::DexPoolTokenOrder::AlreadyBaseQuote, base_vault_address, - Some(0), - ), - ) - .await; - if let Err(error) = upsert_base_pool_token_result { - return Err(error); - } - let upsert_quote_pool_token_result = crate::query_pool_tokens_upsert( - self.database.as_ref(), - &crate::PoolTokenDto::new( - pool_id, - quote_token_id, - crate::PoolTokenRole::Quote, quote_vault_address, - Some(1), - ), - ) - .await; - if let Err(error) = upsert_quote_pool_token_result { - return Err(error); - } - let existing_listing_result = - crate::query_pool_listings_get_by_pool_id(self.database.as_ref(), pool_id).await; - let existing_listing_option = match existing_listing_result { - Ok(existing_listing_option) => existing_listing_option, + transaction.source_endpoint_name.clone(), + ); + let input = match input_result { + Ok(input) => input, Err(error) => return Err(error), }; - let created_listing = existing_listing_option.is_none(); - let pool_listing_id = match existing_listing_option { - Some(pool_listing) => pool_listing.id, - None => { - let listing_id_result = self - .upsert_pool_listing_from_decoded_event(dex_id, pool_id, pair_id, transaction) - .await; - match listing_id_result { - Ok(listing_id) => Some(listing_id), - Err(error) => return Err(error), - } - }, + let detection_result = + crate::dex_pool_materialization::materialize_dex_pool(self.database.as_ref(), &input) + .await; + let detection_result = match detection_result { + Ok(detection_result) => detection_result, + Err(error) => return Err(error), }; - if created_pool { - let signal_result = self - .record_detection_signal( - transaction, - "signal.dex.raydium_clmm.new_pool", - crate::AnalysisSignalSeverity::Low, - payload_value.clone(), - ) - .await; - if let Err(error) = signal_result { - return Err(error); - } + let signal_result = self + .record_pool_detection_signals( + transaction, + "signal.dex.raydium_clmm", + &detection_result, + payload_value, + ) + .await; + if let Err(error) = signal_result { + return Err(error); } - if created_pair { - let signal_result = self - .record_detection_signal( - transaction, - "signal.dex.raydium_clmm.new_pair", - crate::AnalysisSignalSeverity::Low, - payload_value.clone(), - ) - .await; - if let Err(error) = signal_result { - return Err(error); - } - } - if created_listing { - let signal_result = self - .record_detection_signal( - transaction, - "signal.dex.raydium_clmm.first_listing_seen", - crate::AnalysisSignalSeverity::Low, - payload_value, - ) - .await; - if let Err(error) = signal_result { - return Err(error); - } - } - return Ok(crate::DexPoolDetectionResult { - decoded_event_id, - dex_id, - pool_id, - pair_id, - pool_listing_id, - created_pool, - created_pair, - created_listing, - }); + return Ok(detection_result); } async fn detect_raydium_cpmm_trade( @@ -2895,46 +524,12 @@ impl DexDetectService { transaction: &crate::ChainTransactionDto, decoded_event: &crate::DexDecodedEventDto, ) -> Result { - let decoded_event_id = match decoded_event.id { - Some(decoded_event_id) => decoded_event_id, - None => { - return Err(crate::Error::InvalidState( - "decoded dex event has no internal id".to_string(), - )); - }, - }; - let dex_id_result = self.ensure_raydium_cpmm_dex().await; + let dex_id_result = + crate::dex_catalog::ensure_known_dex(self.database.as_ref(), "raydium_cpmm").await; let dex_id = match dex_id_result { Ok(dex_id) => dex_id, Err(error) => return Err(error), }; - let pool_address = match decoded_event.pool_account.clone() { - Some(pool_address) => pool_address, - None => { - return Err(crate::Error::InvalidState(format!( - "decoded event '{}' has no pool_account", - decoded_event_id - ))); - }, - }; - let base_mint = match decoded_event.token_a_mint.clone() { - Some(base_mint) => base_mint, - None => { - return Err(crate::Error::InvalidState(format!( - "decoded event '{}' has no token_a_mint", - decoded_event_id - ))); - }, - }; - let quote_mint = match decoded_event.token_b_mint.clone() { - Some(quote_mint) => quote_mint, - None => { - return Err(crate::Error::InvalidState(format!( - "decoded event '{}' has no token_b_mint", - decoded_event_id - ))); - }, - }; let payload_value_result = parse_payload_json(decoded_event.payload_json.as_str()); let payload_value = match payload_value_result { Ok(payload_value) => payload_value, @@ -2942,522 +537,96 @@ impl DexDetectService { }; let base_vault_address = extract_payload_string_field(&payload_value, "base_vault"); let quote_vault_address = extract_payload_string_field(&payload_value, "quote_vault"); - let base_token_id_result = self.ensure_token(base_mint.as_str()).await; - let base_token_id = match base_token_id_result { - Ok(base_token_id) => base_token_id, - Err(error) => return Err(error), - }; - let quote_token_id_result = self.ensure_token(quote_mint.as_str()).await; - let quote_token_id = match quote_token_id_result { - Ok(quote_token_id) => quote_token_id, - Err(error) => return Err(error), - }; - let existing_pool_result = - crate::query_pools_get_by_address(self.database.as_ref(), pool_address.as_str()).await; - let existing_pool_option = match existing_pool_result { - Ok(existing_pool_option) => existing_pool_option, - Err(error) => return Err(error), - }; - let created_pool = existing_pool_option.is_none(); - let pool_id = match existing_pool_option { - Some(pool) => match pool.id { - Some(pool_id) => pool_id, - None => { - return Err(crate::Error::InvalidState(format!( - "pool '{}' has no internal id", - pool.address - ))); - }, - }, - None => { - let pool_dto = crate::PoolDto::new( - dex_id, - pool_address.clone(), - crate::PoolKind::Amm, - crate::PoolStatus::Active, - ); - let upsert_result = - crate::query_pools_upsert(self.database.as_ref(), &pool_dto).await; - match upsert_result { - Ok(pool_id) => pool_id, - Err(error) => return Err(error), - } - }, - }; - let existing_pair_result = - crate::query_pairs_get_by_pool_id(self.database.as_ref(), pool_id).await; - let existing_pair_option = match existing_pair_result { - Ok(existing_pair_option) => existing_pair_option, - Err(error) => return Err(error), - }; - let created_pair = existing_pair_option.is_none(); - let pair_symbol = build_pair_symbol(base_mint.as_str(), quote_mint.as_str()); - let pair_dto = - crate::PairDto::new(dex_id, pool_id, base_token_id, quote_token_id, pair_symbol); - let pair_id_result = crate::query_pairs_upsert(self.database.as_ref(), &pair_dto).await; - let pair_id = match pair_id_result { - Ok(pair_id) => pair_id, - Err(error) => return Err(error), - }; - let upsert_base_pool_token_result = crate::query_pool_tokens_upsert( - self.database.as_ref(), - &crate::PoolTokenDto::new( - pool_id, - base_token_id, - crate::PoolTokenRole::Base, + let input_result = + crate::dex_pool_materialization::DexPoolMaterializationInput::from_decoded_event( + decoded_event, + dex_id, + crate::PoolKind::Amm, + crate::PoolStatus::Active, + crate::dex_pool_materialization::DexPoolTokenOrder::AlreadyBaseQuote, base_vault_address, - Some(0), - ), - ) - .await; - if let Err(error) = upsert_base_pool_token_result { - return Err(error); - } - let upsert_quote_pool_token_result = crate::query_pool_tokens_upsert( - self.database.as_ref(), - &crate::PoolTokenDto::new( - pool_id, - quote_token_id, - crate::PoolTokenRole::Quote, quote_vault_address, - Some(1), - ), - ) - .await; - if let Err(error) = upsert_quote_pool_token_result { + transaction.source_endpoint_name.clone(), + ); + let input = match input_result { + Ok(input) => input, + Err(error) => return Err(error), + }; + let detection_result = + crate::dex_pool_materialization::materialize_dex_pool(self.database.as_ref(), &input) + .await; + let detection_result = match detection_result { + Ok(detection_result) => detection_result, + Err(error) => return Err(error), + }; + let signal_result = self + .record_pool_detection_signals( + transaction, + "signal.dex.raydium_cpmm", + &detection_result, + payload_value, + ) + .await; + if let Err(error) = signal_result { return Err(error); } - let existing_listing_result = - crate::query_pool_listings_get_by_pool_id(self.database.as_ref(), pool_id).await; - let existing_listing_option = match existing_listing_result { - Ok(existing_listing_option) => existing_listing_option, - Err(error) => return Err(error), - }; - let created_listing = existing_listing_option.is_none(); - let pool_listing_id = match existing_listing_option { - Some(pool_listing) => pool_listing.id, - None => { - let listing_id_result = self - .upsert_pool_listing_from_decoded_event(dex_id, pool_id, pair_id, transaction) - .await; - match listing_id_result { - Ok(listing_id) => Some(listing_id), - Err(error) => return Err(error), - } - }, - }; - if created_pool { - let signal_result = self - .record_detection_signal( - transaction, - "signal.dex.raydium_cpmm.new_pool", - crate::AnalysisSignalSeverity::Low, - payload_value.clone(), - ) - .await; - if let Err(error) = signal_result { - return Err(error); - } - } - if created_pair { - let signal_result = self - .record_detection_signal( - transaction, - "signal.dex.raydium_cpmm.new_pair", - crate::AnalysisSignalSeverity::Low, - payload_value.clone(), - ) - .await; - if let Err(error) = signal_result { - return Err(error); - } - } - if created_listing { - let signal_result = self - .record_detection_signal( - transaction, - "signal.dex.raydium_cpmm.first_listing_seen", - crate::AnalysisSignalSeverity::Low, - payload_value, - ) - .await; - if let Err(error) = signal_result { - return Err(error); - } - } - return Ok(crate::DexPoolDetectionResult { - decoded_event_id, - dex_id, - pool_id, - pair_id, - pool_listing_id, - created_pool, - created_pair, - created_listing, - }); + return Ok(detection_result); } - async fn ensure_raydium_cpmm_dex(&self) -> Result { - let dex_result = - crate::query_dexs_get_by_code(self.database.as_ref(), "raydium_cpmm").await; - let dex_option = match dex_result { - Ok(dex_option) => dex_option, - Err(error) => return Err(error), - }; - match dex_option { - Some(dex) => match dex.id { - Some(dex_id) => return Ok(dex_id), - None => { - return Err(crate::Error::InvalidState( - "raydium_cpmm dex has no internal id".to_string(), - )); - }, - }, - None => { - let dex_dto = crate::DexDto::new( - "raydium_cpmm".to_string(), - "Raydium CPMM".to_string(), - Some(crate::RAYDIUM_CPMM_PROGRAM_ID.to_string()), - None, - true, - ); - return crate::query_dexs_upsert(self.database.as_ref(), &dex_dto).await; - }, - } - } - - async fn ensure_raydium_clmm_dex(&self) -> Result { - let dex_result = - crate::query_dexs_get_by_code(self.database.as_ref(), "raydium_clmm").await; - let dex_option = match dex_result { - Ok(dex_option) => dex_option, - Err(error) => return Err(error), - }; - match dex_option { - Some(dex) => match dex.id { - Some(dex_id) => return Ok(dex_id), - None => { - return Err(crate::Error::InvalidState( - "raydium_clmm dex has no internal id".to_string(), - )); - }, - }, - None => { - let dex_dto = crate::DexDto::new( - "raydium_clmm".to_string(), - "Raydium CLMM".to_string(), - Some(crate::RAYDIUM_CLMM_PROGRAM_ID.to_string()), - None, - true, - ); - return crate::query_dexs_upsert(self.database.as_ref(), &dex_dto).await; - }, - } - } - - async fn ensure_dexlab_dex(&self) -> Result { - let dex_result = crate::query_dexs_get_by_code(self.database.as_ref(), "dexlab").await; - let dex_option = match dex_result { - Ok(dex_option) => dex_option, - Err(error) => return Err(error), - }; - match dex_option { - Some(dex) => match dex.id { - Some(dex_id) => return Ok(dex_id), - None => { - return Err(crate::Error::InvalidState( - "dexlab dex has no internal id".to_string(), - )); - }, - }, - None => { - let dex_dto = crate::DexDto::new( - "dexlab".to_string(), - "DexLab Swap/Pool".to_string(), - Some(crate::DEXLAB_PROGRAM_ID.to_string()), - None, - true, - ); - return crate::query_dexs_upsert(self.database.as_ref(), &dex_dto).await; - }, - } - } - - async fn ensure_fluxbeam_dex(&self) -> Result { - let dex_result = crate::query_dexs_get_by_code(self.database.as_ref(), "fluxbeam").await; - let dex_option = match dex_result { - Ok(dex_option) => dex_option, - Err(error) => return Err(error), - }; - match dex_option { - Some(dex) => match dex.id { - Some(dex_id) => return Ok(dex_id), - None => { - return Err(crate::Error::InvalidState( - "fluxbeam dex has no internal id".to_string(), - )); - }, - }, - None => { - let dex_dto = crate::DexDto::new( - "fluxbeam".to_string(), - "FluxBeam".to_string(), - Some(crate::FLUXBEAM_PROGRAM_ID.to_string()), - None, - true, - ); - return crate::query_dexs_upsert(self.database.as_ref(), &dex_dto).await; - }, - } - } - - async fn ensure_orca_whirlpools_dex(&self) -> Result { - let dex_result = - crate::query_dexs_get_by_code(self.database.as_ref(), "orca_whirlpools").await; - let dex_option = match dex_result { - Ok(dex_option) => dex_option, - Err(error) => return Err(error), - }; - match dex_option { - Some(dex) => match dex.id { - Some(dex_id) => return Ok(dex_id), - None => { - return Err(crate::Error::InvalidState( - "orca_whirlpools dex has no internal id".to_string(), - )); - }, - }, - None => { - let dex_dto = crate::DexDto::new( - "orca_whirlpools".to_string(), - "Orca Whirlpools".to_string(), - Some(crate::ORCA_WHIRLPOOLS_PROGRAM_ID.to_string()), - None, - true, - ); - return crate::query_dexs_upsert(self.database.as_ref(), &dex_dto).await; - }, - } - } - - async fn ensure_meteora_damm_v1_dex(&self) -> Result { - let dex_result = - crate::query_dexs_get_by_code(self.database.as_ref(), "meteora_damm_v1").await; - let dex_option = match dex_result { - Ok(dex_option) => dex_option, - Err(error) => return Err(error), - }; - match dex_option { - Some(dex) => match dex.id { - Some(dex_id) => return Ok(dex_id), - None => { - return Err(crate::Error::InvalidState( - "meteora_damm_v1 dex has no internal id".to_string(), - )); - }, - }, - None => { - let dex_dto = crate::DexDto::new( - "meteora_damm_v1".to_string(), - "Meteora DAMM v1".to_string(), - Some(crate::METEORA_DAMM_V1_PROGRAM_ID.to_string()), - None, - true, - ); - return crate::query_dexs_upsert(self.database.as_ref(), &dex_dto).await; - }, - } - } - - async fn ensure_meteora_damm_v2_dex(&self) -> Result { - let dex_result = - crate::query_dexs_get_by_code(self.database.as_ref(), "meteora_damm_v2").await; - let dex_option = match dex_result { - Ok(dex_option) => dex_option, - Err(error) => return Err(error), - }; - match dex_option { - Some(dex) => match dex.id { - Some(dex_id) => return Ok(dex_id), - None => { - return Err(crate::Error::InvalidState( - "meteora_damm_v2 dex has no internal id".to_string(), - )); - }, - }, - None => { - let dex_dto = crate::DexDto::new( - "meteora_damm_v2".to_string(), - "Meteora DAMM v2".to_string(), - Some(crate::METEORA_DAMM_V2_PROGRAM_ID.to_string()), - None, - true, - ); - return crate::query_dexs_upsert(self.database.as_ref(), &dex_dto).await; - }, - } - } - - async fn ensure_meteora_dbc_dex(&self) -> Result { - let dex_result = crate::query_dexs_get_by_code(self.database.as_ref(), "meteora_dbc").await; - let dex_option = match dex_result { - Ok(dex_option) => dex_option, - Err(error) => return Err(error), - }; - match dex_option { - Some(dex) => match dex.id { - Some(dex_id) => return Ok(dex_id), - None => { - return Err(crate::Error::InvalidState( - "meteora_dbc dex has no internal id".to_string(), - )); - }, - }, - None => { - let dex_dto = crate::DexDto::new( - "meteora_dbc".to_string(), - "Meteora DBC".to_string(), - Some(crate::METEORA_DBC_PROGRAM_ID.to_string()), - None, - true, - ); - return crate::query_dexs_upsert(self.database.as_ref(), &dex_dto).await; - }, - } - } - - async fn ensure_pump_swap_dex(&self) -> Result { - let dex_result = crate::query_dexs_get_by_code(self.database.as_ref(), "pump_swap").await; - let dex_option = match dex_result { - Ok(dex_option) => dex_option, - Err(error) => return Err(error), - }; - match dex_option { - Some(dex) => match dex.id { - Some(dex_id) => return Ok(dex_id), - None => { - return Err(crate::Error::InvalidState( - "pump_swap dex has no internal id".to_string(), - )); - }, - }, - None => { - let dex_dto = crate::DexDto::new( - "pump_swap".to_string(), - "PumpSwap".to_string(), - Some(crate::PUMP_SWAP_PROGRAM_ID.to_string()), - None, - true, - ); - return crate::query_dexs_upsert(self.database.as_ref(), &dex_dto).await; - }, - } - } - - async fn ensure_pump_fun_dex(&self) -> Result { - let dex_result = crate::query_dexs_get_by_code(self.database.as_ref(), "pump_fun").await; - let dex_option = match dex_result { - Ok(dex_option) => dex_option, - Err(error) => return Err(error), - }; - match dex_option { - Some(dex) => match dex.id { - Some(dex_id) => return Ok(dex_id), - None => { - return Err(crate::Error::InvalidState( - "pump_fun dex has no internal id".to_string(), - )); - }, - }, - None => { - let dex_dto = crate::DexDto::new( - "pump_fun".to_string(), - "Pump.fun".to_string(), - Some(crate::PUMP_FUN_PROGRAM_ID.to_string()), - None, - true, - ); - return crate::query_dexs_upsert(self.database.as_ref(), &dex_dto).await; - }, - } - } - - async fn ensure_raydium_dex(&self) -> Result { - let dex_result = crate::query_dexs_get_by_code(self.database.as_ref(), "raydium").await; - let dex_option = match dex_result { - Ok(dex_option) => dex_option, - Err(error) => return Err(error), - }; - match dex_option { - Some(dex) => match dex.id { - Some(dex_id) => return Ok(dex_id), - None => { - return Err(crate::Error::InvalidState( - "raydium dex has no internal id".to_string(), - )); - }, - }, - None => { - let dex_dto = crate::DexDto::new( - "raydium".to_string(), - "Raydium".to_string(), - Some(crate::RAYDIUM_AMM_V4_PROGRAM_ID.to_string()), - None, - true, - ); - return crate::query_dexs_upsert(self.database.as_ref(), &dex_dto).await; - }, - } - } - - async fn ensure_token(&self, mint: &str) -> Result { - let token_result = crate::query_tokens_get_by_mint(self.database.as_ref(), mint).await; - let token_option = match token_result { - Ok(token_option) => token_option, - Err(error) => return Err(error), - }; - match token_option { - Some(token) => match token.id { - Some(token_id) => return Ok(token_id), - None => { - return Err(crate::Error::InvalidState(format!( - "token '{}' has no internal id", - mint - ))); - }, - }, - None => { - let token_dto = crate::TokenDto::new( - mint.to_string(), - None, - None, - None, - crate::SPL_TOKEN_PROGRAM_ID.to_string(), - is_quote_mint(mint), - ); - return crate::query_tokens_upsert(self.database.as_ref(), &token_dto).await; - }, - } - } - - async fn upsert_pool_listing_from_decoded_event( + async fn detect_materialized_pool_from_decoded_event( &self, - dex_id: i64, - pool_id: i64, - pair_id: i64, transaction: &crate::ChainTransactionDto, - ) -> Result { - let listing_dto = crate::PoolListingDto::new( - dex_id, - pool_id, - Some(pair_id), - crate::ObservationSourceKind::Dex, - transaction.source_endpoint_name.clone(), - None, - None, - None, - ); - return crate::query_pool_listings_upsert(self.database.as_ref(), &listing_dto).await; + decoded_event: &crate::DexDecodedEventDto, + dex_code: &str, + pool_kind: crate::PoolKind, + pool_status: crate::PoolStatus, + signal_prefix: &str, + ) -> Result { + let dex_id_result = + crate::dex_catalog::ensure_known_dex(self.database.as_ref(), dex_code).await; + let dex_id = match dex_id_result { + Ok(dex_id) => dex_id, + Err(error) => return Err(error), + }; + let input_result = + crate::dex_pool_materialization::DexPoolMaterializationInput::from_decoded_event( + decoded_event, + dex_id, + pool_kind, + pool_status, + crate::dex_pool_materialization::DexPoolTokenOrder::ChooseBaseQuoteFromTokenAB, + None, + None, + transaction.source_endpoint_name.clone(), + ); + let input = match input_result { + Ok(input) => input, + Err(error) => return Err(error), + }; + let detection_result = + crate::dex_pool_materialization::materialize_dex_pool(self.database.as_ref(), &input) + .await; + let detection_result = match detection_result { + Ok(detection_result) => detection_result, + Err(error) => return Err(error), + }; + let payload_value_result = parse_payload_json(decoded_event.payload_json.as_str()); + let payload_value = match payload_value_result { + Ok(payload_value) => payload_value, + Err(error) => return Err(error), + }; + let signal_result = self + .record_pool_detection_signals( + transaction, + signal_prefix, + &detection_result, + payload_value, + ) + .await; + if let Err(error) = signal_result { + return Err(error); + } + return Ok(detection_result); } async fn record_detection_signal( @@ -3494,44 +663,58 @@ impl DexDetectService { )) .await; } -} -fn is_quote_mint(mint: &str) -> bool { - return mint == crate::WSOL_MINT_ID; -} - -fn choose_base_quote_order(token_a_mint: &str, token_b_mint: &str) -> bool { - let token_a_is_quote = is_quote_mint(token_a_mint); - let token_b_is_quote = is_quote_mint(token_b_mint); - if token_a_is_quote && !token_b_is_quote { - return false; + async fn record_pool_detection_signals( + &self, + transaction: &crate::ChainTransactionDto, + signal_prefix: &str, + detection_result: &crate::DexPoolDetectionResult, + payload: serde_json::Value, + ) -> Result<(), crate::Error> { + if detection_result.created_pool { + let signal_kind = format!("{signal_prefix}.new_pool"); + let signal_result = self + .record_detection_signal( + transaction, + signal_kind.as_str(), + crate::AnalysisSignalSeverity::Low, + payload.clone(), + ) + .await; + if let Err(error) = signal_result { + return Err(error); + } + } + if detection_result.created_pair { + let signal_kind = format!("{signal_prefix}.new_pair"); + let signal_result = self + .record_detection_signal( + transaction, + signal_kind.as_str(), + crate::AnalysisSignalSeverity::Low, + payload.clone(), + ) + .await; + if let Err(error) = signal_result { + return Err(error); + } + } + if detection_result.created_listing { + let signal_kind = format!("{signal_prefix}.first_listing_seen"); + let signal_result = self + .record_detection_signal( + transaction, + signal_kind.as_str(), + crate::AnalysisSignalSeverity::Low, + payload, + ) + .await; + if let Err(error) = signal_result { + return Err(error); + } + } + return Ok(()); } - if token_b_is_quote && !token_a_is_quote { - return true; - } - return true; -} - -fn build_pair_symbol( - base_mint: &str, - quote_mint: &str, -) -> std::option::Option { - let base_symbol = symbol_hint_from_mint(base_mint); - let quote_symbol = symbol_hint_from_mint(quote_mint); - - match (base_symbol, quote_symbol) { - (Some(base_symbol), Some(quote_symbol)) => { - return Some(format!("{base_symbol}/{quote_symbol}")); - }, - _ => return None, - } -} - -fn symbol_hint_from_mint(symbol_hint_from_mint: &str) -> std::option::Option { - if symbol_hint_from_mint == crate::WSOL_MINT_ID { - return Some("WSOL".to_string()); - } - return None; } fn parse_payload_json(payload_json: &str) -> Result { diff --git a/kb_lib/src/dex_detection_route.rs b/kb_lib/src/dex_detection_route.rs new file mode 100644 index 0000000..42f7ec3 --- /dev/null +++ b/kb_lib/src/dex_detection_route.rs @@ -0,0 +1,131 @@ +// file: kb_lib/src/dex_detection_route.rs + +//! Routing from decoded DEX events to business-level detection handlers. + +/// Internal route selected for one decoded DEX event. +#[derive(Debug, Copy, Clone, Eq, PartialEq)] +pub(crate) enum DexDetectionRoute { + /// Raydium AMM v4 initialize2 pool route. + RaydiumAmmV4Initialize2Pool, + /// Raydium CPMM trade route. + RaydiumCpmmTrade, + /// Raydium CLMM trade route. + RaydiumClmmTrade, + /// Pump.fun create token route. + PumpFunCreateV2Token, + /// Pump.fun trade route. + PumpFunTrade, + /// PumpSwap trade route. + PumpSwapTrade, + /// Incomplete PumpSwap event route. + SkipIncompletePumpSwapTrade, + /// Meteora DBC pool route. + MeteoraDbcPool, + /// Meteora DAMM v2 pool route. + MeteoraDammV2Pool, + /// Meteora DAMM v1 pool route. + MeteoraDammV1Pool, + /// Orca Whirlpools pool route. + OrcaWhirlpoolsPool, + /// FluxBeam pool route. + FluxbeamPool, + /// DexLab pool route. + DexlabPool, +} + +/// Selects the business-level detection route for one decoded DEX event. +pub(crate) fn dex_detection_route( + decoded_event: &crate::DexDecodedEventDto, +) -> std::option::Option { + match (decoded_event.protocol_name.as_str(), decoded_event.event_kind.as_str()) { + ("raydium_amm_v4", "raydium_amm_v4.initialize2_pool") => { + return Some( + crate::dex_detection_route::DexDetectionRoute::RaydiumAmmV4Initialize2Pool, + ); + }, + ("raydium_cpmm", "raydium_cpmm.swap_base_input") => { + return Some(crate::dex_detection_route::DexDetectionRoute::RaydiumCpmmTrade); + }, + ("raydium_cpmm", "raydium_cpmm.swap_base_output") => { + return Some(crate::dex_detection_route::DexDetectionRoute::RaydiumCpmmTrade); + }, + ("raydium_clmm", "raydium_clmm.swap_v2") => { + return Some(crate::dex_detection_route::DexDetectionRoute::RaydiumClmmTrade); + }, + ("pump_fun", "pump_fun.create_v2_token") => { + return Some(crate::dex_detection_route::DexDetectionRoute::PumpFunCreateV2Token); + }, + ("pump_fun", "pump_fun.buy") => { + return Some(crate::dex_detection_route::DexDetectionRoute::PumpFunTrade); + }, + ("pump_fun", "pump_fun.sell") => { + return Some(crate::dex_detection_route::DexDetectionRoute::PumpFunTrade); + }, + ("pump_swap", "pump_swap.buy") => { + if crate::dex_detection_route::is_incomplete_pump_swap_decoded_event(decoded_event) { + return Some( + crate::dex_detection_route::DexDetectionRoute::SkipIncompletePumpSwapTrade, + ); + } + return Some(crate::dex_detection_route::DexDetectionRoute::PumpSwapTrade); + }, + ("pump_swap", "pump_swap.sell") => { + if crate::dex_detection_route::is_incomplete_pump_swap_decoded_event(decoded_event) { + return Some( + crate::dex_detection_route::DexDetectionRoute::SkipIncompletePumpSwapTrade, + ); + } + return Some(crate::dex_detection_route::DexDetectionRoute::PumpSwapTrade); + }, + ("meteora_dbc", "meteora_dbc.create_pool") => { + return Some(crate::dex_detection_route::DexDetectionRoute::MeteoraDbcPool); + }, + ("meteora_dbc", "meteora_dbc.swap") => { + return Some(crate::dex_detection_route::DexDetectionRoute::MeteoraDbcPool); + }, + ("meteora_damm_v2", "meteora_damm_v2.create_pool") => { + return Some(crate::dex_detection_route::DexDetectionRoute::MeteoraDammV2Pool); + }, + ("meteora_damm_v2", "meteora_damm_v2.swap") => { + return Some(crate::dex_detection_route::DexDetectionRoute::MeteoraDammV2Pool); + }, + ("meteora_damm_v1", "meteora_damm_v1.create_pool") => { + return Some(crate::dex_detection_route::DexDetectionRoute::MeteoraDammV1Pool); + }, + ("meteora_damm_v1", "meteora_damm_v1.swap") => { + return Some(crate::dex_detection_route::DexDetectionRoute::MeteoraDammV1Pool); + }, + ("orca_whirlpools", "orca_whirlpools.create_pool") => { + return Some(crate::dex_detection_route::DexDetectionRoute::OrcaWhirlpoolsPool); + }, + ("orca_whirlpools", "orca_whirlpools.swap") => { + return Some(crate::dex_detection_route::DexDetectionRoute::OrcaWhirlpoolsPool); + }, + ("fluxbeam", "fluxbeam.create_pool") => { + return Some(crate::dex_detection_route::DexDetectionRoute::FluxbeamPool); + }, + ("fluxbeam", "fluxbeam.swap") => { + return Some(crate::dex_detection_route::DexDetectionRoute::FluxbeamPool); + }, + ("dexlab", "dexlab.create_pool") => { + return Some(crate::dex_detection_route::DexDetectionRoute::DexlabPool); + }, + ("dexlab", "dexlab.swap") => { + return Some(crate::dex_detection_route::DexDetectionRoute::DexlabPool); + }, + _ => return None, + } +} + +fn is_incomplete_pump_swap_decoded_event(decoded_event: &crate::DexDecodedEventDto) -> bool { + if decoded_event.pool_account.is_none() { + return true; + } + if decoded_event.token_a_mint.is_none() { + return true; + } + if decoded_event.token_b_mint.is_none() { + return true; + } + return false; +} diff --git a/kb_lib/src/dex_event_classification.rs b/kb_lib/src/dex_event_classification.rs new file mode 100644 index 0000000..2da9dca --- /dev/null +++ b/kb_lib/src/dex_event_classification.rs @@ -0,0 +1,509 @@ +// file: kb_lib/src/dex_event_classification.rs + +//! Shared DEX event classification and decoded-payload enrichment. +//! +//! This module contains deterministic helpers used by DEX decoding, +//! trade aggregation and future non-trade event materialization. +//! +//! It intentionally does not decode protocol instructions and does not +//! perform database access. + +/// Stable business category assigned to one decoded DEX event kind. +#[derive(Debug, Copy, Clone, Eq, PartialEq, serde::Serialize, serde::Deserialize)] +pub enum DexEventCategory { + /// Swap-like event that can potentially become a normalized trade. + Trade, + /// Liquidity deposit, withdraw, position open or position close event. + Liquidity, + /// Fee collection event. + Fee, + /// Reward or emission event. + Reward, + /// Pool creation, initialization or migration event. + PoolLifecycle, + /// Protocol administration, configuration or permission update event. + Admin, + /// Event kind that is not classified yet. + Unknown, +} + +impl DexEventCategory { + /// Returns the stable string code persisted inside decoded payload metadata. + pub fn as_str(self) -> &'static str { + match self { + Self::Trade => return "trade", + Self::Liquidity => return "liquidity", + Self::Fee => return "fee", + Self::Reward => return "reward", + Self::PoolLifecycle => return "pool_lifecycle", + Self::Admin => return "admin", + Self::Unknown => return "unknown", + } + } +} + +/// Classifies a DEX event kind into a stable business category. +pub fn classify_dex_event_category(event_kind: &str) -> DexEventCategory { + if is_dex_reward_event_kind(event_kind) { + return DexEventCategory::Reward; + } + if is_dex_fee_event_kind(event_kind) { + return DexEventCategory::Fee; + } + if is_dex_liquidity_event_kind(event_kind) { + return DexEventCategory::Liquidity; + } + if is_dex_pool_lifecycle_event_kind(event_kind) { + return DexEventCategory::PoolLifecycle; + } + if is_dex_admin_event_kind(event_kind) { + return DexEventCategory::Admin; + } + if is_dex_trade_event_kind(event_kind) { + return DexEventCategory::Trade; + } + return DexEventCategory::Unknown; +} + +/// Classifies a DEX event kind and returns the persisted category code. +pub fn classify_dex_event_category_code(event_kind: &str) -> &'static str { + return classify_dex_event_category(event_kind).as_str(); +} + +/// Returns true when the event kind represents a swap-like event. +pub fn is_dex_trade_event_kind(event_kind: &str) -> bool { + if event_kind.ends_with(".buy") { + return true; + } + if event_kind.ends_with(".sell") { + return true; + } + if event_kind.ends_with(".swap") { + return true; + } + if event_kind.contains(".swap_") { + return true; + } + if event_kind.ends_with(".exact_input") { + return true; + } + if event_kind.ends_with(".exact_output") { + return true; + } + return false; +} + +/// Returns true when the event kind can directly produce a candle candidate. +pub fn is_dex_candle_candidate_event_kind(event_kind: &str) -> bool { + if event_kind.contains("router") { + return false; + } + if event_kind.contains("route") { + return false; + } + return is_dex_trade_event_kind(event_kind); +} + +/// Returns true for liquidity lifecycle changes that must not become candles. +pub fn is_dex_liquidity_event_kind(event_kind: &str) -> bool { + if event_kind.contains(".deposit") { + return true; + } + if event_kind.contains(".withdraw") { + return true; + } + if event_kind.contains(".increase_liquidity") { + return true; + } + if event_kind.contains(".decrease_liquidity") { + return true; + } + if event_kind.contains(".open_position") { + return true; + } + if event_kind.contains(".close_position") { + return true; + } + return false; +} + +/// Returns true for fee collection events. +pub fn is_dex_fee_event_kind(event_kind: &str) -> bool { + if event_kind.contains("collect_creator_fee") { + return true; + } + if event_kind.contains("collect_protocol_fee") { + return true; + } + if event_kind.contains("collect_fund_fee") { + return true; + } + if event_kind.contains("collect_fee") { + return true; + } + return false; +} + +/// Returns true for reward or incentive events. +pub fn is_dex_reward_event_kind(event_kind: &str) -> bool { + if event_kind.contains("reward") { + return true; + } + if event_kind.contains("emission") { + return true; + } + return false; +} + +/// Returns true for pool creation, initialization or migration events. +pub fn is_dex_pool_lifecycle_event_kind(event_kind: &str) -> bool { + if event_kind.contains(".initialize") { + return true; + } + if event_kind.contains(".initialize_with_permission") { + return true; + } + if event_kind.contains(".create_pool") { + return true; + } + if event_kind.contains(".create_v2_token") { + return true; + } + if event_kind.contains(".migrate") { + return true; + } + return false; +} + +/// Returns true for admin, configuration or permission changes. +pub fn is_dex_admin_event_kind(event_kind: &str) -> bool { + if event_kind.contains("admin") { + return true; + } + if event_kind.contains("config") { + return true; + } + if event_kind.contains("permission") { + return true; + } + if event_kind.contains("set_") { + return true; + } + if event_kind.contains("update_") { + return true; + } + return false; +} + +/// Returns true when a decoded payload is marked as a trade candidate. +/// +/// Explicit payload metadata wins over event-kind inference. This allows +/// incomplete decoded events, such as incomplete PumpSwap trades, to opt out. +pub fn is_decoded_event_trade_candidate(event_kind: &str, payload: &serde_json::Value) -> bool { + let trade_candidate_option = + extract_top_level_bool_by_candidate_keys(payload, &["tradeCandidate", "trade_candidate"]); + if let Some(trade_candidate) = trade_candidate_option { + return trade_candidate; + } + let event_category_option = + extract_string_by_candidate_keys(payload, &["eventCategory", "event_category"]); + if let Some(event_category) = event_category_option { + return event_category.as_str() == DexEventCategory::Trade.as_str(); + } + return is_dex_trade_event_kind(event_kind); +} + +/// Returns true when a decoded payload can be materialized as a candle candidate. +pub fn is_decoded_event_candle_candidate(event_kind: &str, payload: &serde_json::Value) -> bool { + let candle_candidate_option = + extract_top_level_bool_by_candidate_keys(payload, &["candleCandidate", "candle_candidate"]); + if let Some(candle_candidate) = candle_candidate_option { + return candle_candidate; + } + if !is_decoded_event_trade_candidate(event_kind, payload) { + return false; + } + return is_dex_candle_candidate_event_kind(event_kind); +} + +/// Enriches a decoded payload with non-destructive classification metadata. +pub fn enrich_dex_decoded_payload( + protocol_name: &str, + event_kind: &str, + payload_json: serde_json::Value, +) -> serde_json::Value { + let event_category = classify_dex_event_category_code(event_kind); + let trade_candidate = is_dex_trade_event_kind(event_kind); + let candle_candidate = is_dex_candle_candidate_event_kind(event_kind); + let mut object = match payload_json { + serde_json::Value::Object(object) => object, + other => { + let mut object = serde_json::Map::new(); + object.insert("rawPayload".to_owned(), other); + object + }, + }; + json_insert_string_if_missing(&mut object, "protocolName", protocol_name); + json_insert_string_if_missing(&mut object, "eventKind", event_kind); + json_insert_string_if_missing(&mut object, "eventCategory", event_category); + json_insert_bool_if_missing(&mut object, "tradeCandidate", trade_candidate); + json_insert_bool_if_missing(&mut object, "candleCandidate", candle_candidate); + json_insert_i64_if_missing(&mut object, "eventClassificationVersion", 1); + if !trade_candidate { + json_insert_string_if_missing(&mut object, "skipTradeReason", "non_trade_event"); + } else if !candle_candidate { + json_insert_string_if_missing( + &mut object, + "skipCandleReason", + "route_or_multihop_event_requires_leg_resolution", + ); + } + return serde_json::Value::Object(object); +} + +/// Enriches a decoded payload and serializes it as JSON. +pub fn enrich_and_serialize_dex_decoded_payload( + protocol_name: &str, + event_kind: &str, + payload_json: serde_json::Value, +) -> Result { + let enriched_payload = enrich_dex_decoded_payload(protocol_name, event_kind, payload_json); + let payload_json_result = serde_json::to_string(&enriched_payload); + match payload_json_result { + Ok(payload_json) => return Ok(payload_json), + Err(error) => { + return Err(crate::Error::Json(format!( + "cannot serialize enriched decoded payload for '{}': {}", + event_kind, error + ))); + }, + } +} + +/// Parses, enriches and serializes a decoded payload. +pub fn enrich_serialized_dex_decoded_payload( + protocol_name: &str, + event_kind: &str, + payload_json: &str, +) -> Result { + let payload_value_result = serde_json::from_str::(payload_json); + let payload_value = match payload_value_result { + Ok(payload_value) => payload_value, + Err(error) => { + return Err(crate::Error::Json(format!( + "cannot parse decoded payload for '{}': {}", + event_kind, error + ))); + }, + }; + return enrich_and_serialize_dex_decoded_payload(protocol_name, event_kind, payload_value); +} + +fn json_insert_string_if_missing( + object: &mut serde_json::Map, + key: &str, + value: &str, +) { + if object.contains_key(key) { + return; + } + object.insert(key.to_owned(), serde_json::Value::String(value.to_owned())); +} + +fn json_insert_bool_if_missing( + object: &mut serde_json::Map, + key: &str, + value: bool, +) { + if object.contains_key(key) { + return; + } + object.insert(key.to_owned(), serde_json::Value::Bool(value)); +} + +fn json_insert_i64_if_missing( + object: &mut serde_json::Map, + key: &str, + value: i64, +) { + if object.contains_key(key) { + return; + } + object.insert(key.to_owned(), serde_json::Value::Number(serde_json::Number::from(value))); +} + +fn extract_top_level_bool_by_candidate_keys( + payload: &serde_json::Value, + candidate_keys: &[&str], +) -> std::option::Option { + let object = match payload.as_object() { + Some(object) => object, + None => return None, + }; + for candidate_key in candidate_keys { + let value_option = object.get(*candidate_key); + let value = match value_option { + Some(value) => value, + None => continue, + }; + if let Some(value_bool) = value.as_bool() { + return Some(value_bool); + } + if let Some(value_i64) = value.as_i64() { + return Some(value_i64 != 0); + } + if let Some(value_u64) = value.as_u64() { + return Some(value_u64 != 0); + } + if let Some(value_text) = value.as_str() { + let normalized = value_text.trim().to_ascii_lowercase(); + if normalized.as_str() == "true" { + return Some(true); + } + if normalized.as_str() == "false" { + return Some(false); + } + if normalized.as_str() == "1" { + return Some(true); + } + if normalized.as_str() == "0" { + return Some(false); + } + } + } + return None; +} + +fn extract_string_by_candidate_keys( + value: &serde_json::Value, + candidate_keys: &[&str], +) -> std::option::Option { + if let Some(object) = value.as_object() { + for candidate_key in candidate_keys { + let direct_option = object.get(*candidate_key); + if let Some(direct) = direct_option { + let direct_text_option = direct.as_str(); + if let Some(direct_text) = direct_text_option { + return Some(direct_text.to_string()); + } + } + } + for nested_value in object.values() { + let nested_result = extract_string_by_candidate_keys(nested_value, candidate_keys); + if nested_result.is_some() { + return nested_result; + } + } + return None; + } + if let Some(array) = value.as_array() { + for nested_value in array { + let nested_result = extract_string_by_candidate_keys(nested_value, candidate_keys); + if nested_result.is_some() { + return nested_result; + } + } + } + + return None; +} + +#[cfg(test)] +mod tests { + #[test] + fn classifies_swap_events_as_trade_candidates() { + assert_eq!( + super::classify_dex_event_category_code("raydium_cpmm.swap_base_input"), + "trade" + ); + assert_eq!( + super::classify_dex_event_category_code("raydium_cpmm.swap_base_output"), + "trade" + ); + assert_eq!(super::classify_dex_event_category_code("raydium_clmm.swap"), "trade"); + assert_eq!(super::classify_dex_event_category_code("raydium_clmm.swap_v2"), "trade"); + assert_eq!(super::classify_dex_event_category_code("raydium_clmm.exact_output"), "trade"); + assert_eq!(super::classify_dex_event_category_code("pump_fun.buy"), "trade"); + assert!(super::is_dex_trade_event_kind("raydium_cpmm.swap_base_input")); + assert!(super::is_dex_candle_candidate_event_kind("raydium_cpmm.swap_base_input")); + } + + #[test] + fn classifies_router_swap_as_trade_but_not_direct_candle_candidate() { + assert_eq!( + super::classify_dex_event_category_code("raydium_clmm.swap_router_base_in"), + "trade" + ); + assert!(super::is_dex_trade_event_kind("raydium_clmm.swap_router_base_in")); + assert!(!super::is_dex_candle_candidate_event_kind("raydium_clmm.swap_router_base_in")); + } + + #[test] + fn classifies_fee_reward_liquidity_and_lifecycle_events() { + assert_eq!( + super::classify_dex_event_category_code("raydium_cpmm.collect_creator_fee"), + "fee" + ); + assert_eq!( + super::classify_dex_event_category_code("raydium_clmm.collect_protocol_fee"), + "fee" + ); + assert_eq!( + super::classify_dex_event_category_code("raydium_clmm.set_reward_params"), + "reward" + ); + assert_eq!( + super::classify_dex_event_category_code("raydium_clmm.increase_liquidity_v2"), + "liquidity" + ); + assert_eq!( + super::classify_dex_event_category_code("raydium_cpmm.initialize"), + "pool_lifecycle" + ); + } + + #[test] + fn enriched_payload_keeps_existing_fields() { + let payload_json = serde_json::json!({ + "eventCategory": "custom", + "amountIn": "10" + }); + let enriched_payload = super::enrich_dex_decoded_payload( + "raydium_cpmm", + "raydium_cpmm.swap_base_input", + payload_json, + ); + + let object_option = enriched_payload.as_object(); + let object = match object_option { + Some(object) => object, + None => { + panic!("expected enriched payload object"); + }, + }; + assert_eq!( + object.get("eventCategory"), + Some(&serde_json::Value::String("custom".to_owned())) + ); + assert_eq!( + object.get("protocolName"), + Some(&serde_json::Value::String("raydium_cpmm".to_owned())) + ); + assert_eq!( + object.get("eventKind"), + Some(&serde_json::Value::String("raydium_cpmm.swap_base_input".to_owned())) + ); + assert_eq!(object.get("tradeCandidate"), Some(&serde_json::Value::Bool(true))); + assert_eq!(object.get("candleCandidate"), Some(&serde_json::Value::Bool(true))); + } + + #[test] + fn decoded_event_payload_candidate_flags_win_over_event_kind() { + let payload_json = serde_json::json!({ + "tradeCandidate": false, + "candleCandidate": false + }); + assert!(!super::is_decoded_event_trade_candidate("pump_swap.buy", &payload_json)); + assert!(!super::is_decoded_event_candle_candidate("pump_swap.buy", &payload_json)); + } +} diff --git a/kb_lib/src/dex_pool_materialization.rs b/kb_lib/src/dex_pool_materialization.rs new file mode 100644 index 0000000..ee862f1 --- /dev/null +++ b/kb_lib/src/dex_pool_materialization.rs @@ -0,0 +1,662 @@ +// file: kb_lib/src/dex_pool_materialization.rs + +//! Shared DEX pool materialization helpers. +//! +//! This module persists normalized pool, pair, pool token and pool listing +//! records from decoded DEX events. +//! +//! It intentionally does not decode instructions and does not record analysis +//! signals. Detection services remain responsible for deciding which signals +//! to emit. + +/// Token ordering strategy used when materializing a decoded DEX pool. +#[derive(Debug, Copy, Clone, Eq, PartialEq)] +pub(crate) enum DexPoolTokenOrder { + /// `token_a_mint` is already the base token and `token_b_mint` is already the quote token. + AlreadyBaseQuote, + /// Base/quote order should be chosen from token A/B using known quote-token hints. + ChooseBaseQuoteFromTokenAB, +} + +/// Input required to materialize a normalized DEX pool. +#[derive(Debug, Clone)] +pub(crate) struct DexPoolMaterializationInput { + /// Parent decoded event id. + pub(crate) decoded_event_id: i64, + /// Internal DEX id. + pub(crate) dex_id: i64, + /// Pool account address. + pub(crate) pool_address: std::string::String, + /// Token A mint, or already-base mint depending on `token_order`. + pub(crate) token_a_mint: std::string::String, + /// Token B mint, or already-quote mint depending on `token_order`. + pub(crate) token_b_mint: std::string::String, + /// Optional LP mint. + pub(crate) lp_mint: std::option::Option, + /// Optional token A vault address, or base vault when `token_order` is `AlreadyBaseQuote`. + pub(crate) token_a_vault_address: std::option::Option, + /// Optional token B vault address, or quote vault when `token_order` is `AlreadyBaseQuote`. + pub(crate) token_b_vault_address: std::option::Option, + /// Pool kind to persist. + pub(crate) pool_kind: crate::PoolKind, + /// Pool status to persist. + pub(crate) pool_status: crate::PoolStatus, + /// Token ordering strategy. + pub(crate) token_order: crate::dex_pool_materialization::DexPoolTokenOrder, + /// Listing source kind. + pub(crate) listing_source_kind: crate::ObservationSourceKind, + /// Optional source endpoint logical name. + pub(crate) source_endpoint_name: std::option::Option, +} + +impl DexPoolMaterializationInput { + /// Creates a materialization input from a decoded event requiring pool and two token mints. + pub(crate) fn from_decoded_event( + decoded_event: &crate::DexDecodedEventDto, + dex_id: i64, + pool_kind: crate::PoolKind, + pool_status: crate::PoolStatus, + token_order: crate::dex_pool_materialization::DexPoolTokenOrder, + token_a_vault_address: std::option::Option, + token_b_vault_address: std::option::Option, + source_endpoint_name: std::option::Option, + ) -> Result { + let decoded_event_id_result = + crate::dex_pool_materialization::required_decoded_event_id(decoded_event); + let decoded_event_id = match decoded_event_id_result { + Ok(decoded_event_id) => decoded_event_id, + Err(error) => return Err(error), + }; + let pool_address_result = + crate::dex_pool_materialization::required_pool_account(decoded_event); + let pool_address = match pool_address_result { + Ok(pool_address) => pool_address, + Err(error) => return Err(error), + }; + let token_a_mint_result = + crate::dex_pool_materialization::required_token_a_mint(decoded_event); + let token_a_mint = match token_a_mint_result { + Ok(token_a_mint) => token_a_mint, + Err(error) => return Err(error), + }; + let token_b_mint_result = + crate::dex_pool_materialization::required_token_b_mint(decoded_event); + let token_b_mint = match token_b_mint_result { + Ok(token_b_mint) => token_b_mint, + Err(error) => return Err(error), + }; + return Ok(Self { + decoded_event_id, + dex_id, + pool_address, + token_a_mint, + token_b_mint, + lp_mint: decoded_event.lp_mint.clone(), + token_a_vault_address, + token_b_vault_address, + pool_kind, + pool_status, + token_order, + listing_source_kind: crate::ObservationSourceKind::Dex, + source_endpoint_name, + }); + } + + /// Creates a materialization input from a decoded event and explicit token mints. + /// + /// This is used by launch or bonding-curve events where the decoded event + /// may expose only the launched token mint and the quote mint is inferred + /// by the detector. + pub(crate) fn from_decoded_event_with_mints( + decoded_event: &crate::DexDecodedEventDto, + dex_id: i64, + token_a_mint: std::string::String, + token_b_mint: std::string::String, + lp_mint: std::option::Option, + pool_kind: crate::PoolKind, + pool_status: crate::PoolStatus, + token_order: crate::dex_pool_materialization::DexPoolTokenOrder, + token_a_vault_address: std::option::Option, + token_b_vault_address: std::option::Option, + source_endpoint_name: std::option::Option, + ) -> Result { + let decoded_event_id_result = + crate::dex_pool_materialization::required_decoded_event_id(decoded_event); + let decoded_event_id = match decoded_event_id_result { + Ok(decoded_event_id) => decoded_event_id, + Err(error) => return Err(error), + }; + let pool_address_result = + crate::dex_pool_materialization::required_pool_account(decoded_event); + let pool_address = match pool_address_result { + Ok(pool_address) => pool_address, + Err(error) => return Err(error), + }; + return Ok(Self { + decoded_event_id, + dex_id, + pool_address, + token_a_mint, + token_b_mint, + lp_mint, + token_a_vault_address, + token_b_vault_address, + pool_kind, + pool_status, + token_order, + listing_source_kind: crate::ObservationSourceKind::Dex, + source_endpoint_name, + }); + } +} + +/// Returns the decoded event id or fails with a stable diagnostic. +pub(crate) fn required_decoded_event_id( + decoded_event: &crate::DexDecodedEventDto, +) -> Result { + match decoded_event.id { + Some(decoded_event_id) => return Ok(decoded_event_id), + None => { + return Err(crate::Error::InvalidState( + "decoded dex event has no internal id".to_string(), + )); + }, + } +} + +/// Returns the pool account or fails with a stable diagnostic. +pub(crate) fn required_pool_account( + decoded_event: &crate::DexDecodedEventDto, +) -> Result { + let decoded_event_id_result = + crate::dex_pool_materialization::required_decoded_event_id(decoded_event); + let decoded_event_id = match decoded_event_id_result { + Ok(decoded_event_id) => decoded_event_id, + Err(error) => return Err(error), + }; + match decoded_event.pool_account.clone() { + Some(pool_account) => return Ok(pool_account), + None => { + return Err(crate::Error::InvalidState(format!( + "decoded event '{}' has no pool_account", + decoded_event_id + ))); + }, + } +} + +/// Returns token A mint or fails with a stable diagnostic. +pub(crate) fn required_token_a_mint( + decoded_event: &crate::DexDecodedEventDto, +) -> Result { + let decoded_event_id_result = + crate::dex_pool_materialization::required_decoded_event_id(decoded_event); + let decoded_event_id = match decoded_event_id_result { + Ok(decoded_event_id) => decoded_event_id, + Err(error) => return Err(error), + }; + match decoded_event.token_a_mint.clone() { + Some(token_a_mint) => return Ok(token_a_mint), + None => { + return Err(crate::Error::InvalidState(format!( + "decoded event '{}' has no token_a_mint", + decoded_event_id + ))); + }, + } +} + +/// Returns token B mint or fails with a stable diagnostic. +pub(crate) fn required_token_b_mint( + decoded_event: &crate::DexDecodedEventDto, +) -> Result { + let decoded_event_id_result = + crate::dex_pool_materialization::required_decoded_event_id(decoded_event); + let decoded_event_id = match decoded_event_id_result { + Ok(decoded_event_id) => decoded_event_id, + Err(error) => return Err(error), + }; + match decoded_event.token_b_mint.clone() { + Some(token_b_mint) => return Ok(token_b_mint), + None => { + return Err(crate::Error::InvalidState(format!( + "decoded event '{}' has no token_b_mint", + decoded_event_id + ))); + }, + } +} + +/// Persists pool, pair, pool tokens and listing, returning the materialized ids. +pub(crate) async fn materialize_dex_pool( + database: &crate::Database, + input: &crate::dex_pool_materialization::DexPoolMaterializationInput, +) -> Result { + let ordered_tokens = crate::dex_pool_materialization::ordered_pool_tokens_from_input(input); + let base_token_id_result = + crate::dex_pool_materialization::ensure_token(database, ordered_tokens.base_mint.as_str()) + .await; + let base_token_id = match base_token_id_result { + Ok(base_token_id) => base_token_id, + Err(error) => return Err(error), + }; + let quote_token_id_result = + crate::dex_pool_materialization::ensure_token(database, ordered_tokens.quote_mint.as_str()) + .await; + let quote_token_id = match quote_token_id_result { + Ok(quote_token_id) => quote_token_id, + Err(error) => return Err(error), + }; + let lp_token_id = match input.lp_mint.clone() { + Some(lp_mint) => { + let lp_token_id_result = + crate::dex_pool_materialization::ensure_token(database, lp_mint.as_str()).await; + match lp_token_id_result { + Ok(lp_token_id) => Some(lp_token_id), + Err(error) => return Err(error), + } + }, + None => None, + }; + let pool_result = crate::dex_pool_materialization::ensure_pool(database, input).await; + let pool_materialization = match pool_result { + Ok(pool_materialization) => pool_materialization, + Err(error) => return Err(error), + }; + let pair_result = crate::dex_pool_materialization::ensure_pair( + database, + input.dex_id, + pool_materialization.pool_id, + base_token_id, + quote_token_id, + ordered_tokens.base_mint.as_str(), + ordered_tokens.quote_mint.as_str(), + ) + .await; + let pair_materialization = match pair_result { + Ok(pair_materialization) => pair_materialization, + Err(error) => return Err(error), + }; + let upsert_base_pool_token_result = crate::query_pool_tokens_upsert( + database, + &crate::PoolTokenDto::new( + pool_materialization.pool_id, + base_token_id, + crate::PoolTokenRole::Base, + ordered_tokens.base_vault_address, + Some(0), + ), + ) + .await; + if let Err(error) = upsert_base_pool_token_result { + return Err(error); + } + let upsert_quote_pool_token_result = crate::query_pool_tokens_upsert( + database, + &crate::PoolTokenDto::new( + pool_materialization.pool_id, + quote_token_id, + crate::PoolTokenRole::Quote, + ordered_tokens.quote_vault_address, + Some(1), + ), + ) + .await; + if let Err(error) = upsert_quote_pool_token_result { + return Err(error); + } + if let Some(lp_token_id) = lp_token_id { + let upsert_lp_pool_token_result = crate::query_pool_tokens_upsert( + database, + &crate::PoolTokenDto::new( + pool_materialization.pool_id, + lp_token_id, + crate::PoolTokenRole::LpMint, + None, + None, + ), + ) + .await; + if let Err(error) = upsert_lp_pool_token_result { + return Err(error); + } + } + let listing_result = crate::dex_pool_materialization::ensure_pool_listing( + database, + input, + pool_materialization.pool_id, + pair_materialization.pair_id, + ) + .await; + let listing_materialization = match listing_result { + Ok(listing_materialization) => listing_materialization, + Err(error) => return Err(error), + }; + return Ok(crate::DexPoolDetectionResult { + decoded_event_id: input.decoded_event_id, + dex_id: input.dex_id, + pool_id: pool_materialization.pool_id, + pair_id: pair_materialization.pair_id, + pool_listing_id: listing_materialization.pool_listing_id, + created_pool: pool_materialization.created_pool, + created_pair: pair_materialization.created_pair, + created_listing: listing_materialization.created_listing, + }); +} + +#[derive(Debug, Clone)] +struct OrderedPoolTokens { + base_mint: std::string::String, + quote_mint: std::string::String, + base_vault_address: std::option::Option, + quote_vault_address: std::option::Option, +} + +fn ordered_pool_tokens_from_input( + input: &crate::dex_pool_materialization::DexPoolMaterializationInput, +) -> OrderedPoolTokens { + match input.token_order { + crate::dex_pool_materialization::DexPoolTokenOrder::AlreadyBaseQuote => { + return OrderedPoolTokens { + base_mint: input.token_a_mint.clone(), + quote_mint: input.token_b_mint.clone(), + base_vault_address: input.token_a_vault_address.clone(), + quote_vault_address: input.token_b_vault_address.clone(), + }; + }, + crate::dex_pool_materialization::DexPoolTokenOrder::ChooseBaseQuoteFromTokenAB => { + let base_is_token_a = crate::dex_pool_materialization::choose_base_quote_order( + input.token_a_mint.as_str(), + input.token_b_mint.as_str(), + ); + let base_mint = if base_is_token_a { + input.token_a_mint.clone() + } else { + input.token_b_mint.clone() + }; + let quote_mint = if base_is_token_a { + input.token_b_mint.clone() + } else { + input.token_a_mint.clone() + }; + let base_vault_address = if base_is_token_a { + input.token_a_vault_address.clone() + } else { + input.token_b_vault_address.clone() + }; + let quote_vault_address = if base_is_token_a { + input.token_b_vault_address.clone() + } else { + input.token_a_vault_address.clone() + }; + return OrderedPoolTokens { + base_mint, + quote_mint, + base_vault_address, + quote_vault_address, + }; + }, + } +} + +async fn ensure_pool( + database: &crate::Database, + input: &crate::dex_pool_materialization::DexPoolMaterializationInput, +) -> Result { + let existing_pool_result = + crate::query_pools_get_by_address(database, input.pool_address.as_str()).await; + let existing_pool_option = match existing_pool_result { + Ok(existing_pool_option) => existing_pool_option, + Err(error) => return Err(error), + }; + let created_pool = existing_pool_option.is_none(); + let pool_id = match existing_pool_option { + Some(pool) => match pool.id { + Some(pool_id) => pool_id, + None => { + return Err(crate::Error::InvalidState(format!( + "pool '{}' has no internal id", + pool.address + ))); + }, + }, + None => { + let pool_dto = crate::PoolDto::new( + input.dex_id, + input.pool_address.clone(), + input.pool_kind, + input.pool_status, + ); + let upsert_result = crate::query_pools_upsert(database, &pool_dto).await; + match upsert_result { + Ok(pool_id) => pool_id, + Err(error) => return Err(error), + } + }, + }; + return Ok(PoolMaterialization { pool_id, created_pool }); +} + +async fn ensure_pair( + database: &crate::Database, + dex_id: i64, + pool_id: i64, + base_token_id: i64, + quote_token_id: i64, + base_mint: &str, + quote_mint: &str, +) -> Result { + let existing_pair_result = crate::query_pairs_get_by_pool_id(database, pool_id).await; + let existing_pair_option = match existing_pair_result { + Ok(existing_pair_option) => existing_pair_option, + Err(error) => return Err(error), + }; + let created_pair = existing_pair_option.is_none(); + let pair_symbol = crate::dex_pool_materialization::build_pair_symbol(base_mint, quote_mint); + let pair_id = match existing_pair_option { + Some(pair) => match pair.id { + Some(pair_id) => pair_id, + None => { + return Err(crate::Error::InvalidState(format!( + "pair for pool '{}' has no internal id", + pool_id + ))); + }, + }, + None => { + let pair_dto = + crate::PairDto::new(dex_id, pool_id, base_token_id, quote_token_id, pair_symbol); + let upsert_result = crate::query_pairs_upsert(database, &pair_dto).await; + match upsert_result { + Ok(pair_id) => pair_id, + Err(error) => return Err(error), + } + }, + }; + return Ok(PairMaterialization { pair_id, created_pair }); +} + +async fn ensure_pool_listing( + database: &crate::Database, + input: &crate::dex_pool_materialization::DexPoolMaterializationInput, + pool_id: i64, + pair_id: i64, +) -> Result { + let existing_listing_result = + crate::query_pool_listings_get_by_pool_id(database, pool_id).await; + let existing_listing_option = match existing_listing_result { + Ok(existing_listing_option) => existing_listing_option, + Err(error) => return Err(error), + }; + let created_listing = existing_listing_option.is_none(); + let pool_listing_id = match existing_listing_option { + Some(pool_listing) => pool_listing.id, + None => { + let listing_dto = crate::PoolListingDto::new( + input.dex_id, + pool_id, + Some(pair_id), + input.listing_source_kind, + input.source_endpoint_name.clone(), + None, + None, + None, + ); + let upsert_result = crate::query_pool_listings_upsert(database, &listing_dto).await; + match upsert_result { + Ok(listing_id) => Some(listing_id), + Err(error) => return Err(error), + } + }, + }; + return Ok(ListingMaterialization { pool_listing_id, created_listing }); +} + +async fn ensure_token(database: &crate::Database, mint: &str) -> Result { + let token_result = crate::query_tokens_get_by_mint(database, mint).await; + let token_option = match token_result { + Ok(token_option) => token_option, + Err(error) => return Err(error), + }; + match token_option { + Some(token) => match token.id { + Some(token_id) => return Ok(token_id), + None => { + return Err(crate::Error::InvalidState(format!( + "token '{}' has no internal id", + mint + ))); + }, + }, + None => { + let token_dto = crate::TokenDto::new( + mint.to_string(), + None, + None, + None, + crate::SPL_TOKEN_PROGRAM_ID.to_string(), + crate::dex_pool_materialization::is_quote_mint(mint), + ); + return crate::query_tokens_upsert(database, &token_dto).await; + }, + } +} + +#[derive(Debug, Clone)] +struct PoolMaterialization { + pool_id: i64, + created_pool: bool, +} + +#[derive(Debug, Clone)] +struct PairMaterialization { + pair_id: i64, + created_pair: bool, +} + +#[derive(Debug, Clone)] +struct ListingMaterialization { + pool_listing_id: std::option::Option, + created_listing: bool, +} + +fn is_quote_mint(mint: &str) -> bool { + return mint == crate::WSOL_MINT_ID; +} + +fn choose_base_quote_order(token_a_mint: &str, token_b_mint: &str) -> bool { + let token_a_is_quote = crate::dex_pool_materialization::is_quote_mint(token_a_mint); + let token_b_is_quote = crate::dex_pool_materialization::is_quote_mint(token_b_mint); + if token_a_is_quote && !token_b_is_quote { + return false; + } + if token_b_is_quote && !token_a_is_quote { + return true; + } + return true; +} + +fn build_pair_symbol( + base_mint: &str, + quote_mint: &str, +) -> std::option::Option { + let base_symbol = crate::dex_pool_materialization::symbol_hint_from_mint(base_mint); + let quote_symbol = crate::dex_pool_materialization::symbol_hint_from_mint(quote_mint); + match (base_symbol, quote_symbol) { + (Some(base_symbol), Some(quote_symbol)) => { + return Some(format!("{base_symbol}/{quote_symbol}")); + }, + _ => return None, + } +} + +fn symbol_hint_from_mint(mint: &str) -> std::option::Option { + if mint == crate::WSOL_MINT_ID { + return Some("WSOL".to_string()); + } + return None; +} + +#[cfg(test)] +mod tests { + #[test] + fn quote_token_is_moved_to_quote_side_when_order_is_chosen() { + let input = crate::dex_pool_materialization::DexPoolMaterializationInput { + decoded_event_id: 1, + dex_id: 2, + pool_address: "Pool111".to_string(), + token_a_mint: crate::WSOL_MINT_ID.to_string(), + token_b_mint: "TokenB111".to_string(), + lp_mint: None, + token_a_vault_address: Some("VaultA111".to_string()), + token_b_vault_address: Some("VaultB111".to_string()), + pool_kind: crate::PoolKind::Amm, + pool_status: crate::PoolStatus::Active, + token_order: + crate::dex_pool_materialization::DexPoolTokenOrder::ChooseBaseQuoteFromTokenAB, + listing_source_kind: crate::ObservationSourceKind::Dex, + source_endpoint_name: Some("test".to_string()), + }; + let ordered = super::ordered_pool_tokens_from_input(&input); + assert_eq!(ordered.base_mint, "TokenB111"); + assert_eq!(ordered.quote_mint, crate::WSOL_MINT_ID); + assert_eq!(ordered.base_vault_address, Some("VaultB111".to_string())); + assert_eq!(ordered.quote_vault_address, Some("VaultA111".to_string())); + } + + #[test] + fn already_base_quote_order_is_preserved() { + let input = crate::dex_pool_materialization::DexPoolMaterializationInput { + decoded_event_id: 1, + dex_id: 2, + pool_address: "Pool111".to_string(), + token_a_mint: crate::WSOL_MINT_ID.to_string(), + token_b_mint: "TokenB111".to_string(), + lp_mint: None, + token_a_vault_address: Some("BaseVault111".to_string()), + token_b_vault_address: Some("QuoteVault111".to_string()), + pool_kind: crate::PoolKind::Amm, + pool_status: crate::PoolStatus::Active, + token_order: crate::dex_pool_materialization::DexPoolTokenOrder::AlreadyBaseQuote, + listing_source_kind: crate::ObservationSourceKind::Dex, + source_endpoint_name: Some("test".to_string()), + }; + let ordered = super::ordered_pool_tokens_from_input(&input); + assert_eq!(ordered.base_mint, crate::WSOL_MINT_ID); + assert_eq!(ordered.quote_mint, "TokenB111"); + assert_eq!(ordered.base_vault_address, Some("BaseVault111".to_string())); + assert_eq!(ordered.quote_vault_address, Some("QuoteVault111".to_string())); + } + + #[test] + fn pair_symbol_is_none_when_only_one_symbol_is_known() { + let pair_symbol = super::build_pair_symbol(crate::WSOL_MINT_ID, "TokenB111"); + assert_eq!(pair_symbol, None); + } + + #[test] + fn wsol_symbol_hint_is_known() { + let symbol = super::symbol_hint_from_mint(crate::WSOL_MINT_ID); + assert_eq!(symbol, Some("WSOL".to_string())); + } +} diff --git a/kb_lib/src/lib.rs b/kb_lib/src/lib.rs index 9fe9bc4..ebfb38a 100644 --- a/kb_lib/src/lib.rs +++ b/kb_lib/src/lib.rs @@ -23,10 +23,22 @@ mod db; mod detect; /// DEX-specific transaction decoders. mod dex; +/// Internal known DEX catalog. +mod dex_catalog; /// Persistence-oriented DEX decoding service. mod dex_decode; +/// Transaction context loading for DEX decoding. +mod dex_decode_context; +/// Decoded DEX event materialization helpers. +mod dex_decoded_event_materialization; /// Business-level detection built from decoded DEX events. mod dex_detect; +/// Decoded DEX event to business-detection routing. +mod dex_detection_route; +/// Shared DEX event classification and decoded-payload enrichment helpers. +mod dex_event_classification; +/// Shared DEX pool materialization helpers. +mod dex_pool_materialization; /// Shared error type for `kb_lib`. mod error; /// Generic asynchronous HTTP JSON-RPC client. @@ -55,6 +67,8 @@ mod pair_candle_query; mod pair_symbol; /// Cross-DEX pool-origin recording service. mod pool_origin; +/// Protocol candidate recording. +mod protocol_candidate_recording; /// Typed Solana WebSocket PubSub helpers built on top of the generic JSON-RPC transport. mod solana_pubsub_ws; /// Historical token backfill service. @@ -65,6 +79,22 @@ mod token_metadata; mod tracing; /// Cross-DEX trade aggregation service. mod trade_aggregation; +/// Database context loading for trade aggregation. +mod trade_aggregation_context; +/// Trade amount resolution orchestration. +mod trade_amount_resolution; +/// Trade-event materialization. +mod trade_event_materialization; +/// Trade metric update and pricing helpers. +mod trade_metric_update; +/// PumpSwap trade amount recovery helpers. +mod trade_pump_swap_amounts; +/// Trade-side resolution helpers. +mod trade_side_resolution; +/// Solana transaction/meta trade amount extraction helpers. +mod trade_solana_amounts; +/// Transaction classification service. +mod transaction_classification; /// Projection of resolved transactions into normalized internal DB tables. mod tx_model; /// Transaction resolution pipeline. @@ -104,46 +134,143 @@ pub use config::SolanaConfig; pub use config::SqliteDatabaseConfig; /// WebSocket endpoint configuration. pub use config::WsEndpointConfig; +/// Address Lookup Table program identifier. ("AddressLookupTab1e1111111111111111111111111"). +/// @see solana_sdk::pubkey::Pubkey = solana_sdk_ids::address_lookup_table::ID +pub use constants::ADDRESS_LOOKUP_TABLE_PROGRAM_ID; +/// Arbitrage Bot (6MWVT) / Arbitrage or Sandwich Bot. ("6MWVTis8rmmk6Vt9zmAJJbmb3VuLpzoQ1aHH4N6wQEGh"). +pub use constants::ARBITRAGE_BOT_6MWVT_PROGRAM_ID; /// Associated Token Account program identifier. ("ATokenGPvbdGVxr1b2hvZbsiqW5xWH25efTNsLJA8knL"). /// @see solana_sdk::pubkey::Pubkey = spl_associated_token_account_interface::program::ID pub use constants::ASSOCIATED_TOKEN_PROGRAM_ID; +/// BPF Loader program identifier. ("BPFLoader1111111111111111111111111111111111"). +/// @see solana_sdk::pubkey::Pubkey = solana_sdk_ids::bpf_loader_deprecated::ID +pub use constants::BPF_LOADER_DEPRECATED_PROGRAM_ID; +/// BPF Loader program identifier. ("BPFLoaderUpgradeab1e11111111111111111111111"). +/// @see solana_sdk::pubkey::Pubkey = solana_sdk_ids::bpf_loader_upgradeable::ID +pub use constants::BPF_LOADER_UPGRADEABLE_PROGRAM_ID; /// Compute Budget program identifier. ("ComputeBudget111111111111111111111111111111"). -/// @see solana_sdk_ids::compute_budget::ID +/// @see solana_sdk::pubkey::Pubkey = solana_sdk_ids::compute_budget::ID pub use constants::COMPUTE_BUDGET_PROGRAM_ID; +/// Config program identifier. ("Config1111111111111111111111111111111111111"). +/// @see solana_sdk::pubkey::Pubkey = solana_sdk_ids::config::ID +pub use constants::CONFIG_PROGRAM_ID; /// DexLab Swap/Pool program id. ("DSwpgjMvXhtGn6BsbqmacdBZyfLj6jSWf3HJpdJtmg6N"). pub use constants::DEXLAB_PROGRAM_ID; +/// ED25519 program identifier. ("Ed25519SigVerify111111111111111111111111111"). +/// @see solana_sdk::pubkey::Pubkey = solana_sdk_ids::ed25519_program::ID +pub use constants::ED25519_PROGRAM_ID; +/// Feature program identifier. ("Feature111111111111111111111111111111111111"). +/// @see solana_sdk::pubkey::Pubkey = solana_sdk_ids::feature::ID +pub use constants::FEATURE_PROGRAM_ID; /// FluxBeam program id. ("FLUXubRmkEi2q6K3Y9kBPg9248ggaZVsoSFhtJHSrm1X"). pub use constants::FLUXBEAM_PROGRAM_ID; +/// Incinerator program identifier. ("1nc1nerator11111111111111111111111111111111"). +/// @see solana_sdk::pubkey::Pubkey = solana_sdk_ids::incinerator::ID +pub use constants::INCINERATOR_PROGRAM_ID; +/// Loader V4 program identifier. ("LoaderV411111111111111111111111111111111111"). +/// @see solana_sdk::pubkey::Pubkey = solana_sdk_ids::loader_v4::ID +pub use constants::LOADER_V4_PROGRAM_ID; /// Meteora DAMM v1 program id. ("Eo7WjKq67rjJQSZxS6z3YkapzY3eMj6Xy8X5EQVn5UaB"). pub use constants::METEORA_DAMM_V1_PROGRAM_ID; /// Meteora DAMM v2 program id. ("cpamdpZCGKUy5JxQXB4dcpGPiikHawvSWAd6mEn1sGG"). pub use constants::METEORA_DAMM_V2_PROGRAM_ID; /// Meteora DBC program id. ("dbcij3LWUppWqq96dh6gJWwBifmcGfLSB5D4DuSMaqN"). pub use constants::METEORA_DBC_PROGRAM_ID; +/// Meteora DLMM program id. ("LBUZKhRxPF3XUpBCjp4YzTKgLccjZhTSDM9YuVaPwxo"). +pub use constants::METEORA_DLMM_PROGRAM_ID; +/// Native Loader program identifier. ("NativeLoader1111111111111111111111111111111"). +/// @see solana_sdk::pubkey::Pubkey = solana_sdk_ids::native_loader::ID +pub use constants::NATIVE_LOADER_PROGRAM_ID; /// Orca Whirlpools program id. ("whirLbMiicVdio4qvUfM5KAg6Ct8VwpYzGff3uctyCc"). pub use constants::ORCA_WHIRLPOOLS_PROGRAM_ID; /// Pump.fun program id. ("6EF8rrecthR5Dkzon8Nwu78hRvfCKubJ14M5uBEwF6P"). pub use constants::PUMP_FUN_PROGRAM_ID; /// PumpSwap / PumpAMM program id. ("pAMMBay6oceH9fJKBRHGP5D4bD4sWpmSwMn52FMfXEA"). pub use constants::PUMP_SWAP_PROGRAM_ID; +/// Raydium AMM routing program id. ("routeUGWgWzqBWFcrCfv8tritsqukccJPu3q5GPP3xS"). +pub use constants::RAYDIUM_AMM_ROUTING_PROGRAM_ID; /// Raydium AmmV4 program id. ("675kPX9MHTjS2zt1qfr1NYHuzeLXfQM9H24wFSUt1Mp8"). pub use constants::RAYDIUM_AMM_V4_PROGRAM_ID; /// Raydium CLMM program id. ("CAMMCzo5YL8w4VFF8KVHrK22GGUsp5VTaW7grrKgrWqK"). pub use constants::RAYDIUM_CLMM_PROGRAM_ID; /// Raydium CPMM mainnet program id. ("CPMMoo8L3F4NbTegBCKVNunggL7H1ZpdTHKxQB5qKP1C"). pub use constants::RAYDIUM_CPMM_PROGRAM_ID; +/// Raydium LaunchLab program id. ("LanMV9sAd7wArD4vJFi2qDdfnVhFxYSUg6eADduJ3uj"). +pub use constants::RAYDIUM_LAUNCHLAB_PROGRAM_ID; +/// Raydium Stable Swap AMM program id, deprecated. ("5quBtoiQqxF9Jv6KYKctB59NT3gtJD2Y65kdnB1Uev3h"). +pub use constants::RAYDIUM_STABLE_SWAP_AMM_PROGRAM_ID; +/// Secp256k1 program identifier. ("KeccakSecp256k11111111111111111111111111111"). +/// @see solana_sdk::pubkey::Pubkey = solana_sdk_ids::secp256k1_program::ID +pub use constants::SECP256K1_PROGRAM_ID; +/// Secp256r1 program identifier. ("Secp256r1SigVerify1111111111111111111111111"). +/// @see solana_sdk::pubkey::Pubkey = solana_sdk_ids::secp256r1_program::ID +pub use constants::SECP256R1_PROGRAM_ID; /// SPL Token-2022 program identifier. ("TokenzQdBNbLqP5VEhdkAS6EPFLC1PHnBqCXEpPxuEb"). /// @see solana_sdk::pubkey::Pubkey = spl_token_2022_interface::ID pub use constants::SPL_TOKEN_2022_PROGRAM_ID; /// SPL Token program identifier. ("TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA"). /// @see solana_sdk::pubkey::Pubkey = spl_token_interface::ID pub use constants::SPL_TOKEN_PROGRAM_ID; +/// Stake Config program identifier. ("StakeConfig11111111111111111111111111111111"). +/// @see solana_sdk::pubkey::Pubkey = solana_sdk_ids::stake::config::ID +pub use constants::STAKE_CONFIG_PROGRAM_ID; +/// Stake program identifier. ("Stake11111111111111111111111111111111111111"). +/// @see solana_sdk::pubkey::Pubkey = solana_sdk_ids::stake::ID +pub use constants::STAKE_PROGRAM_ID; /// System program identifier. ("11111111111111111111111111111111"). /// @see solana_sdk::pubkey::Pubkey = solana_sdk_ids::system_program::ID pub use constants::SYSTEM_PROGRAM_ID; +/// Sysvar Clock program identifier. ("SysvarC1ock11111111111111111111111111111111"). +/// @see solana_sdk::pubkey::Pubkey = solana_sdk_ids::sysvar::clock::ID +pub use constants::SYSVAR_CLOCK_PROGRAM_ID; +/// Sysvar Epoch Rewards program identifier. ("SysvarEpochRewards1111111111111111111111111"). +/// @see solana_sdk::pubkey::Pubkey = solana_sdk_ids::sysvar::epoch_rewards::ID +pub use constants::SYSVAR_EPOCH_REWARDS_PROGRAM_ID; +/// Sysvar Epoch Schedule program identifier. ("SysvarEpochSchedu1e111111111111111111111111"). +/// @see solana_sdk::pubkey::Pubkey = solana_sdk_ids::sysvar::epoch_schedule::ID +pub use constants::SYSVAR_EPOCH_SCHEDULE_PROGRAM_ID; +/// Sysvar Fees program identifier. ("SysvarFees111111111111111111111111111111111"). +/// @see solana_sdk::pubkey::Pubkey = solana_sdk_ids::sysvar::fees::ID +pub use constants::SYSVAR_FEES_PROGRAM_ID; +/// Sysvar Instructions program identifier. ("Sysvar1nstructions1111111111111111111111111"). +/// @see solana_sdk::pubkey::Pubkey = solana_sdk_ids::sysvar::instructions::ID +pub use constants::SYSVAR_INSTRUCTIONS_PROGRAM_ID; +/// Sysvar Last Restart Slot program identifier. ("SysvarLastRestartS1ot1111111111111111111111"). +/// @see solana_sdk::pubkey::Pubkey = solana_sdk_ids::sysvar::last_restart_slot::ID +pub use constants::SYSVAR_LAST_RESTART_SLOT_PROGRAM_ID; +/// Sysvar program identifier. ("Sysvar1111111111111111111111111111111111111"). +/// @see solana_sdk::pubkey::Pubkey = solana_sdk_ids::sysvar::ID +pub use constants::SYSVAR_PROGRAM_ID; +/// Sysvar Recent Blockhashes program identifier. ("SysvarRecentB1ockHashes11111111111111111111"). +/// @see solana_sdk::pubkey::Pubkey = solana_sdk_ids::sysvar::recent_blockhashes::ID +pub use constants::SYSVAR_RECENT_BLOCKHASHES_PROGRAM_ID; +/// Sysvar Rent program identifier. ("SysvarRent111111111111111111111111111111111"). +/// @see solana_sdk::pubkey::Pubkey = solana_sdk_ids::sysvar::rent::ID +pub use constants::SYSVAR_RENT_PROGRAM_ID; +/// Sysvar Rewards program identifier. ("SysvarRewards111111111111111111111111111111"). +/// @see solana_sdk::pubkey::Pubkey = solana_sdk_ids::sysvar::rewards::ID +pub use constants::SYSVAR_REWARDS_PROGRAM_ID; +/// Sysvar Slot Hashes program identifier. ("SysvarS1otHashes111111111111111111111111111"). +/// @see solana_sdk::pubkey::Pubkey = solana_sdk_ids::sysvar::slot_hashes::ID +pub use constants::SYSVAR_SLOT_HASHES_PROGRAM_ID; +/// Sysvar Slot History program identifier. ("SysvarS1otHistory11111111111111111111111111"). +/// @see solana_sdk::pubkey::Pubkey = solana_sdk_ids::sysvar::slot_history::ID +pub use constants::SYSVAR_SLOT_HISTORY_PROGRAM_ID; +/// Sysvar Stake History program identifier. ("SysvarStakeHistory1111111111111111111111111"). +/// @see solana_sdk::pubkey::Pubkey = solana_sdk_ids::sysvar::stake_history::ID +pub use constants::SYSVAR_STAKE_HISTORY_PROGRAM_ID; +/// Vote program identifier. ("Vote111111111111111111111111111111111111111"). +/// @see solana_sdk::pubkey::Pubkey = solana_sdk_ids::vote::ID +pub use constants::VOTE_PROGRAM_ID; /// Wrapped SOL mint identifier. ("So11111111111111111111111111111111111111112"). /// @see solana_sdk::pubkey::Pubkey = spl_token_interface::native_mint::ID pub use constants::WSOL_MINT_ID; +/// Zk El Gamal Proof program identifier. ("ZkE1Gama1Proof11111111111111111111111111111"). +/// @see solana_sdk::pubkey::Pubkey = solana_sdk_ids::zk_elgamal_proof_program::ID +pub use constants::ZK_ELGAMAL_PROOF_PROGRAM_ID; +/// Zk Token Proof program identifier. ("ZkTokenProof1111111111111111111111111111111"). +/// @see solana_sdk::pubkey::Pubkey = solana_sdk_ids::zk_token_proof_program::ID +pub use constants::ZK_TOKEN_PROOF_PROGRAM_ID; /// Application-facing analysis signal DTO. pub use db::AnalysisSignalDto; /// Persisted analysis signal row. @@ -284,6 +411,18 @@ pub use db::PoolTokenDto; pub use db::PoolTokenEntity; /// Role of one token inside a normalized pool. pub use db::PoolTokenRole; +/// Application-facing protocol candidate DTO. +/// +/// A protocol candidate records a program/instruction that should be inspected +/// later because it may correspond to an unsupported DEX, launch surface, +/// migration path or protocol-specific non-trade event. +pub use db::ProtocolCandidateDto; +/// Persisted protocol candidate row. +pub use db::ProtocolCandidateEntity; +/// Aggregated protocol candidate diagnostic row. +pub use db::ProtocolCandidateSummaryDto; +/// Aggregated protocol candidate diagnostic row. +pub use db::ProtocolCandidateSummaryEntity; /// Application-facing normalized swap DTO. pub use db::SwapDto; /// Persisted normalized swap row. @@ -306,6 +445,10 @@ pub use db::TokenMintEventEntity; pub use db::TradeEventDto; /// Persisted trade-event row. pub use db::TradeEventEntity; +/// Application-facing transaction classification DTO. +pub use db::TransactionClassificationDto; +/// Persisted transaction classification row. +pub use db::TransactionClassificationEntity; /// Application-facing wallet DTO. pub use db::WalletDto; /// Persisted wallet row. @@ -482,6 +625,20 @@ pub use db::query_pools_get_by_address; pub use db::query_pools_list; /// Inserts or updates one normalized pool row by address. pub use db::query_pools_upsert; +/// Lists protocol candidate summaries ordered by investigation priority. +pub use db::query_protocol_candidate_summaries_list_by_priority; +/// Deletes protocol candidates for one transaction. +/// +/// This is useful before recomputing candidates for a replayed transaction. +pub use db::query_protocol_candidates_delete_by_transaction_id; +/// Inserts one protocol candidate row. +pub use db::query_protocol_candidates_insert; +/// Lists protocol candidates for one program id. +pub use db::query_protocol_candidates_list_by_program_id; +/// Lists protocol candidates for one transaction. +pub use db::query_protocol_candidates_list_by_transaction_id; +/// Lists recent protocol candidates ordered from newest to oldest. +pub use db::query_protocol_candidates_list_recent; /// Lists recent swaps ordered from newest to oldest. pub use db::query_swaps_list_recent; /// Inserts or updates one normalized swap row. @@ -512,6 +669,14 @@ pub use db::query_trade_events_list_by_pair_id; pub use db::query_trade_events_list_by_transaction_id; /// Inserts or updates one trade-event row and returns its stable internal id. pub use db::query_trade_events_upsert; +/// Reads one transaction classification by signature. +pub use db::query_transaction_classifications_get_by_signature; +/// Reads one transaction classification by transaction id. +pub use db::query_transaction_classifications_get_by_transaction_id; +/// Lists recent transaction classifications ordered from newest to oldest. +pub use db::query_transaction_classifications_list_recent; +/// Inserts or updates one transaction classification row. +pub use db::query_transaction_classifications_upsert; /// Returns one wallet-holding row identified by `(wallet_id, token_id)`, if it exists. pub use db::query_wallet_holdings_get_by_wallet_and_token; /// Lists wallet-holding rows for one wallet id. @@ -644,6 +809,36 @@ pub use dex_decode::DexDecodeService; pub use dex_detect::DexDetectService; /// Result of one business-level DEX pool detection. pub use dex_detect::DexPoolDetectionResult; +/// Stable DEX event business category. +pub use dex_event_classification::DexEventCategory; +/// Classifies a DEX event kind into a stable category. +pub use dex_event_classification::classify_dex_event_category; +/// Classifies a DEX event kind and returns its persisted category code. +pub use dex_event_classification::classify_dex_event_category_code; +/// Enriches and serializes a decoded DEX payload. +pub use dex_event_classification::enrich_and_serialize_dex_decoded_payload; +/// Enriches a decoded DEX payload with classification metadata. +pub use dex_event_classification::enrich_dex_decoded_payload; +/// Parses, enriches and serializes a decoded DEX payload. +pub use dex_event_classification::enrich_serialized_dex_decoded_payload; +/// Returns true when a decoded payload is a candle candidate. +pub use dex_event_classification::is_decoded_event_candle_candidate; +/// Returns true when a decoded payload is a trade candidate. +pub use dex_event_classification::is_decoded_event_trade_candidate; +/// Returns true for admin/config/permission DEX events. +pub use dex_event_classification::is_dex_admin_event_kind; +/// Returns true when a DEX event kind can directly feed candle materialization. +pub use dex_event_classification::is_dex_candle_candidate_event_kind; +/// Returns true for fee collection DEX events. +pub use dex_event_classification::is_dex_fee_event_kind; +/// Returns true for liquidity lifecycle DEX events. +pub use dex_event_classification::is_dex_liquidity_event_kind; +/// Returns true for pool lifecycle DEX events. +pub use dex_event_classification::is_dex_pool_lifecycle_event_kind; +/// Returns true for reward or emission DEX events. +pub use dex_event_classification::is_dex_reward_event_kind; +/// Returns true for swap-like DEX events. +pub use dex_event_classification::is_dex_trade_event_kind; /// Global error type used by the `kb_lib` crate. /// /// The project intentionally avoids `anyhow` and `thiserror`, so this @@ -793,6 +988,8 @@ pub use tracing::init_tracing; pub use trade_aggregation::TradeAggregationResult; /// Trade-aggregation service. pub use trade_aggregation::TradeAggregationService; +/// Service used to classify projected Solana transactions. +pub use transaction_classification::TransactionClassificationService; /// Service projecting resolved transaction JSON into internal chain tables. pub use tx_model::TransactionModelService; /// Result of one transaction resolution pass. diff --git a/kb_lib/src/local_pipeline_replay.rs b/kb_lib/src/local_pipeline_replay.rs index a78d73a..80cfbd1 100644 --- a/kb_lib/src/local_pipeline_replay.rs +++ b/kb_lib/src/local_pipeline_replay.rs @@ -63,6 +63,10 @@ pub struct LocalPipelineReplayResult { /// This is a replay write/result counter, not the number of distinct rows /// currently persisted in the analytic signal table. pub analytic_signal_upsert_count: usize, + /// Total transaction classification rows upserted during replay. + pub transaction_classification_count: usize, + /// Number of transactions that produced a classification error. + pub transaction_classification_error_count: usize, /// Number of token metadata rows updated after replay. pub token_metadata_updated_count: usize, /// Number of pair symbols updated after replay. @@ -122,6 +126,8 @@ impl LocalPipelineReplayService { let pair_candle_aggregation = crate::PairCandleAggregationService::new(self.database.clone()); let pair_analytic_signal = crate::PairAnalyticSignalService::new(self.database.clone()); + let transaction_classification = + crate::TransactionClassificationService::new(self.database.clone()); let mut result = LocalPipelineReplayResult { selected_transaction_count: signatures.len(), ..Default::default() @@ -209,6 +215,22 @@ impl LocalPipelineReplayService { ); }, } + let classification_result = transaction_classification + .classify_transaction_by_signature(signature.as_str()) + .await; + match classification_result { + Ok(_) => { + result.transaction_classification_count += 1; + }, + Err(error) => { + result.transaction_classification_error_count += 1; + tracing::warn!( + signature = %signature, + error = %error, + "local pipeline replay transaction classification step failed" + ); + }, + } result.replayed_transaction_count += 1; } if config.refresh_missing_token_metadata { diff --git a/kb_lib/src/protocol_candidate_recording.rs b/kb_lib/src/protocol_candidate_recording.rs new file mode 100644 index 0000000..67d4777 --- /dev/null +++ b/kb_lib/src/protocol_candidate_recording.rs @@ -0,0 +1,530 @@ +// file: kb_lib/src/protocol_candidate_recording.rs + +//! Protocol candidate recording. +//! +//! This module records candidate protocol/program instructions for transactions +//! that were not fully decoded by the current DEX decoders. + +/// Input used to record protocol candidates for one classified transaction. +pub(crate) struct ProtocolCandidateRecordingInput<'a> { + /// Database connection. + pub(crate) database: &'a crate::Database, + /// Persisted transaction. + pub(crate) transaction: &'a crate::ChainTransactionDto, + /// Internal transaction id. + pub(crate) transaction_id: i64, + /// Projected instructions for the transaction. + pub(crate) instructions: &'a [crate::ChainInstructionDto], + /// Persisted classification kind. + pub(crate) classification_kind: &'a str, +} + +/// Records protocol candidates for one classified transaction. +/// +/// Existing candidates for the same transaction are deleted first so replay is +/// deterministic. +pub(crate) async fn record_protocol_candidates_for_classification( + input: crate::protocol_candidate_recording::ProtocolCandidateRecordingInput<'_>, +) -> Result { + let delete_result = crate::query_protocol_candidates_delete_by_transaction_id( + input.database, + input.transaction_id, + ) + .await; + if let Err(error) = delete_result { + return Err(error); + } + let candidate_specs = + crate::protocol_candidate_recording::build_protocol_candidate_specs_for_classification( + input.transaction, + input.transaction_id, + input.instructions, + input.classification_kind, + ); + let mut inserted_count = 0_usize; + for candidate_spec in candidate_specs { + let dto = crate::ProtocolCandidateDto::new( + input.transaction_id, + candidate_spec.instruction_id, + input.transaction.signature.clone(), + input.transaction.slot, + candidate_spec.program_id, + candidate_spec.program_name_hint, + candidate_spec.candidate_protocol, + candidate_spec.candidate_surface, + candidate_spec.reason, + candidate_spec.evidence_json, + ); + let insert_result = crate::query_protocol_candidates_insert(input.database, &dto).await; + match insert_result { + Ok(_) => { + inserted_count += 1; + }, + Err(error) => return Err(error), + } + } + return Ok(inserted_count); +} + +struct ProtocolCandidateSpec { + instruction_id: std::option::Option, + program_id: std::string::String, + program_name_hint: std::option::Option, + candidate_protocol: std::option::Option, + candidate_surface: std::option::Option, + reason: std::string::String, + evidence_json: std::string::String, +} + +fn build_protocol_candidate_specs_for_classification( + transaction: &crate::ChainTransactionDto, + transaction_id: i64, + instructions: &[crate::ChainInstructionDto], + classification_kind: &str, +) -> std::vec::Vec { + if classification_kind == "known_dex_program_unclassified" { + return build_known_dex_program_candidate_specs(transaction, transaction_id, instructions); + } + if classification_kind == "unknown_or_unclassified" { + return build_unknown_program_candidate_specs(transaction, transaction_id, instructions); + } + return std::vec::Vec::new(); +} + +fn build_known_dex_program_candidate_specs( + transaction: &crate::ChainTransactionDto, + transaction_id: i64, + instructions: &[crate::ChainInstructionDto], +) -> std::vec::Vec { + let mut specs = std::vec::Vec::new(); + for instruction in instructions { + let program_id = match instruction.program_id.clone() { + Some(program_id) => program_id, + None => continue, + }; + let known_protocol = known_dex_protocol_name(program_id.as_str()); + let known_protocol = match known_protocol { + Some(known_protocol) => known_protocol, + None => continue, + }; + let evidence_json = build_instruction_evidence_json( + transaction, + transaction_id, + instruction, + "known_dex_program_without_decoded_event", + ); + specs.push(ProtocolCandidateSpec { + instruction_id: instruction.id, + program_id, + program_name_hint: instruction.program_name.clone(), + candidate_protocol: Some(known_protocol.to_string()), + candidate_surface: None, + reason: "known DEX program instruction did not produce a decoded DEX event".to_string(), + evidence_json, + }); + } + return specs; +} + +fn build_unknown_program_candidate_specs( + transaction: &crate::ChainTransactionDto, + transaction_id: i64, + instructions: &[crate::ChainInstructionDto], +) -> std::vec::Vec { + let mut specs = std::vec::Vec::new(); + for instruction in instructions { + let program_id = match instruction.program_id.clone() { + Some(program_id) => program_id, + None => continue, + }; + if should_ignore_program_id(program_id.as_str()) { + continue; + } + let surface_hint = infer_candidate_surface(program_id.as_str(), instruction); + let evidence_json = build_instruction_evidence_json( + transaction, + transaction_id, + instruction, + "unknown_or_unclassified_program_instruction", + ); + specs.push(ProtocolCandidateSpec { + instruction_id: instruction.id, + program_id, + program_name_hint: instruction.program_name.clone(), + candidate_protocol: None, + candidate_surface: surface_hint, + reason: "transaction has no decoded DEX event and includes a non-ignored program instruction".to_string(), + evidence_json, + }); + } + return specs; +} + +fn build_instruction_evidence_json( + transaction: &crate::ChainTransactionDto, + transaction_id: i64, + instruction: &crate::ChainInstructionDto, + reason_code: &str, +) -> std::string::String { + let evidence_value = serde_json::json!({ + "reasonCode": reason_code, + "transactionId": transaction_id, + "signature": transaction.signature, + "slot": transaction.slot, + "instructionId": instruction.id, + "parentInstructionId": instruction.parent_instruction_id, + "instructionIndex": instruction.instruction_index, + "programId": instruction.program_id, + "programName": instruction.program_name, + "stackHeight": instruction.stack_height, + "parsedType": instruction.parsed_type + }); + let evidence_json_result = serde_json::to_string(&evidence_value); + match evidence_json_result { + Ok(evidence_json) => return evidence_json, + Err(error) => { + return format!( + "{{\"reasonCode\":\"evidence_serialization_failed\",\"error\":\"{}\"}}", + error + ); + }, + } +} + +fn known_dex_protocol_name(program_id: &str) -> std::option::Option<&'static str> { + if program_id == crate::RAYDIUM_AMM_V4_PROGRAM_ID { + return Some("raydium_amm_v4"); + } + if program_id == crate::RAYDIUM_CPMM_PROGRAM_ID { + return Some("raydium_cpmm"); + } + if program_id == crate::RAYDIUM_CLMM_PROGRAM_ID { + return Some("raydium_clmm"); + } + if program_id == crate::RAYDIUM_LAUNCHLAB_PROGRAM_ID { + return Some("raydium_launchlab"); + } + if program_id == crate::RAYDIUM_AMM_ROUTING_PROGRAM_ID { + return Some("raydium_router"); + } + if program_id == crate::RAYDIUM_STABLE_SWAP_AMM_PROGRAM_ID { + return Some("raydium_stable_swap"); + } + if program_id == crate::PUMP_FUN_PROGRAM_ID { + return Some("pump_fun"); + } + if program_id == crate::PUMP_SWAP_PROGRAM_ID { + return Some("pump_swap"); + } + if program_id == crate::METEORA_DBC_PROGRAM_ID { + return Some("meteora_dbc"); + } + if program_id == crate::METEORA_DLMM_PROGRAM_ID { + return Some("meteora_dlmm"); + } + if program_id == crate::METEORA_DAMM_V1_PROGRAM_ID { + return Some("meteora_damm_v1"); + } + if program_id == crate::METEORA_DAMM_V2_PROGRAM_ID { + return Some("meteora_damm_v2"); + } + if program_id == crate::ORCA_WHIRLPOOLS_PROGRAM_ID { + return Some("orca_whirlpools"); + } + if program_id == crate::FLUXBEAM_PROGRAM_ID { + return Some("fluxbeam"); + } + if program_id == crate::DEXLAB_PROGRAM_ID { + return Some("dexlab"); + } + return None; +} + +fn should_ignore_program_id(program_id: &str) -> bool { + if program_id == crate::SYSTEM_PROGRAM_ID { + return true; + } + if program_id == crate::SPL_TOKEN_PROGRAM_ID { + return true; + } + if program_id == crate::SPL_TOKEN_2022_PROGRAM_ID { + return true; + } + if program_id == crate::ASSOCIATED_TOKEN_PROGRAM_ID { + return true; + } + if program_id == crate::COMPUTE_BUDGET_PROGRAM_ID { + return true; + } + if program_id == crate::ADDRESS_LOOKUP_TABLE_PROGRAM_ID { + return true; + } + if program_id == crate::BPF_LOADER_DEPRECATED_PROGRAM_ID { + return true; + } + if program_id == crate::BPF_LOADER_UPGRADEABLE_PROGRAM_ID { + return true; + } + if program_id == crate::LOADER_V4_PROGRAM_ID { + return true; + } + if program_id == crate::NATIVE_LOADER_PROGRAM_ID { + return true; + } + if program_id == crate::CONFIG_PROGRAM_ID { + return true; + } + if program_id == crate::VOTE_PROGRAM_ID { + return true; + } + if program_id == crate::STAKE_PROGRAM_ID { + return true; + } + if program_id == crate::STAKE_CONFIG_PROGRAM_ID { + return true; + } + if program_id == crate::ED25519_PROGRAM_ID { + return true; + } + if program_id == crate::SECP256K1_PROGRAM_ID { + return true; + } + if program_id == crate::SECP256R1_PROGRAM_ID { + return true; + } + if program_id == crate::ZK_TOKEN_PROOF_PROGRAM_ID { + return true; + } + if program_id == crate::ZK_ELGAMAL_PROOF_PROGRAM_ID { + return true; + } + if program_id == crate::SYSVAR_PROGRAM_ID { + return true; + } + if program_id == crate::SYSVAR_CLOCK_PROGRAM_ID { + return true; + } + if program_id == crate::SYSVAR_RENT_PROGRAM_ID { + return true; + } + if program_id == crate::SYSVAR_INSTRUCTIONS_PROGRAM_ID { + return true; + } + if program_id == crate::SYSVAR_EPOCH_REWARDS_PROGRAM_ID { + return true; + } + if program_id == crate::SYSVAR_EPOCH_SCHEDULE_PROGRAM_ID { + return true; + } + if program_id == crate::SYSVAR_FEES_PROGRAM_ID { + return true; + } + if program_id == crate::SYSVAR_LAST_RESTART_SLOT_PROGRAM_ID { + return true; + } + if program_id == crate::SYSVAR_RECENT_BLOCKHASHES_PROGRAM_ID { + return true; + } + if program_id == crate::SYSVAR_REWARDS_PROGRAM_ID { + return true; + } + if program_id == crate::SYSVAR_SLOT_HASHES_PROGRAM_ID { + return true; + } + if program_id == crate::SYSVAR_SLOT_HISTORY_PROGRAM_ID { + return true; + } + if program_id == crate::SYSVAR_STAKE_HISTORY_PROGRAM_ID { + return true; + } + return false; +} + +fn infer_candidate_surface( + program_id: &str, + instruction: &crate::ChainInstructionDto, +) -> std::option::Option { + if program_id == crate::ARBITRAGE_BOT_6MWVT_PROGRAM_ID { + return Some("arbitrage_bot".to_string()); + } + if is_known_launch_surface_program_id(program_id) { + return Some("launch_surface".to_string()); + } + if let Some(program_name) = instruction.program_name.as_deref() { + let normalized = program_name.to_ascii_lowercase(); + if normalized.contains("arbitrage") { + return Some("arbitrage_bot".to_string()); + } + if normalized.contains("sandwich") { + return Some("arbitrage_bot".to_string()); + } + if normalized.contains("launch") { + return Some("launch_surface".to_string()); + } + if normalized.contains("meteora") { + return Some("meteora_related".to_string()); + } + if normalized.contains("raydium") { + return Some("raydium_related".to_string()); + } + if normalized.contains("pump") { + return Some("pump_related".to_string()); + } + if normalized.contains("swap") { + return Some("swap_related".to_string()); + } + } + return None; +} + +fn is_known_launch_surface_program_id(_program_id: &str) -> bool { + // Filled in later after program ids are verified from live corpus and + // official or sufficiently reliable references. + return false; +} + +#[cfg(test)] +mod tests { + #[test] + fn associated_token_program_is_ignored() { + let transaction = test_transaction(); + let instructions = vec![test_instruction( + 0, + Some(crate::ASSOCIATED_TOKEN_PROGRAM_ID.to_string()), + Some("spl-associated-token-account".to_string()), + )]; + let specs = super::build_protocol_candidate_specs_for_classification( + &transaction, + 1, + &instructions, + "unknown_or_unclassified", + ); + assert_eq!(specs.len(), 0); + } + + #[test] + fn meteora_dlmm_is_known_dex_protocol() { + let transaction = test_transaction(); + let instructions = vec![test_instruction( + 0, + Some(crate::METEORA_DLMM_PROGRAM_ID.to_string()), + Some("Meteora DLMM".to_string()), + )]; + let specs = super::build_protocol_candidate_specs_for_classification( + &transaction, + 1, + &instructions, + "known_dex_program_unclassified", + ); + assert_eq!(specs.len(), 1); + assert_eq!(specs[0].program_id, crate::METEORA_DLMM_PROGRAM_ID); + assert_eq!(specs[0].candidate_protocol, Some("meteora_dlmm".to_string())); + } + + #[test] + fn known_arbitrage_bot_gets_surface_hint() { + let transaction = test_transaction(); + let instructions = vec![test_instruction( + 0, + Some(crate::ARBITRAGE_BOT_6MWVT_PROGRAM_ID.to_string()), + None, + )]; + let specs = super::build_protocol_candidate_specs_for_classification( + &transaction, + 1, + &instructions, + "unknown_or_unclassified", + ); + assert_eq!(specs.len(), 1); + assert_eq!(specs[0].candidate_surface, Some("arbitrage_bot".to_string())); + } + + fn test_instruction( + instruction_index: u32, + program_id: std::option::Option, + program_name: std::option::Option, + ) -> crate::ChainInstructionDto { + return crate::ChainInstructionDto::new( + 1, + None, + instruction_index, + None, + program_id, + program_name, + None, + "[]".to_string(), + None, + None, + Some(serde_json::json!({}).to_string()), + ); + } + + fn test_transaction() -> crate::ChainTransactionDto { + let mut transaction = crate::ChainTransactionDto::new( + "signature_1".to_string(), + Some(123), + None, + Some("test".to_string()), + None, + None, + None, + serde_json::json!({}).to_string(), + ); + transaction.id = Some(1); + return transaction; + } + + #[test] + fn known_dex_candidate_is_built_for_unclassified_known_program() { + let transaction = test_transaction(); + let instructions = vec![test_instruction( + 0, + Some(crate::METEORA_DAMM_V2_PROGRAM_ID.to_string()), + Some("Meteora".to_string()), + )]; + let specs = super::build_protocol_candidate_specs_for_classification( + &transaction, + 1, + &instructions, + "known_dex_program_unclassified", + ); + assert_eq!(specs.len(), 1); + assert_eq!(specs[0].program_id, crate::METEORA_DAMM_V2_PROGRAM_ID); + assert_eq!(specs[0].candidate_protocol, Some("meteora_damm_v2".to_string())); + } + + #[test] + fn ignored_program_is_not_recorded_as_unknown_candidate() { + let transaction = test_transaction(); + let instructions = vec![test_instruction( + 0, + Some(crate::SPL_TOKEN_PROGRAM_ID.to_string()), + Some("spl-token".to_string()), + )]; + let specs = super::build_protocol_candidate_specs_for_classification( + &transaction, + 1, + &instructions, + "unknown_or_unclassified", + ); + assert_eq!(specs.len(), 0); + } + + #[test] + fn unknown_non_ignored_program_is_recorded() { + let transaction = test_transaction(); + let instructions = vec![test_instruction( + 0, + Some("UnknownProgram111111111111111111111111111111111".to_string()), + Some("unknown swap program".to_string()), + )]; + let specs = super::build_protocol_candidate_specs_for_classification( + &transaction, + 1, + &instructions, + "unknown_or_unclassified", + ); + assert_eq!(specs.len(), 1); + assert_eq!(specs[0].candidate_surface, Some("swap_related".to_string())); + } +} diff --git a/kb_lib/src/token_backfill.rs b/kb_lib/src/token_backfill.rs index 8429b2e..a0faea6 100644 --- a/kb_lib/src/token_backfill.rs +++ b/kb_lib/src/token_backfill.rs @@ -82,6 +82,7 @@ pub struct TokenBackfillService { wallet_observation_service: crate::WalletObservationService, trade_aggregation_service: crate::TradeAggregationService, pair_candle_aggregation_service: crate::PairCandleAggregationService, + transaction_classification_service: crate::TransactionClassificationService, token_metadata_service: crate::TokenMetadataBackfillService, } @@ -102,6 +103,8 @@ impl TokenBackfillService { let trade_aggregation_service = crate::TradeAggregationService::new(database.clone()); let pair_candle_aggregation_service = crate::PairCandleAggregationService::new(database.clone()); + let transaction_classification_service = + crate::TransactionClassificationService::new(database.clone()); let token_metadata_service = crate::TokenMetadataBackfillService::new( http_pool.clone(), database.clone(), @@ -120,6 +123,7 @@ impl TokenBackfillService { wallet_observation_service, trade_aggregation_service, pair_candle_aggregation_service, + transaction_classification_service, token_metadata_service, }; } @@ -436,6 +440,13 @@ impl TokenBackfillService { Ok(pair_candle_aggregations) => pair_candle_aggregations, Err(error) => return Err(error), }; + let transaction_classification_result = self + .transaction_classification_service + .classify_transaction_by_signature(signature.as_str()) + .await; + if let Err(error) = transaction_classification_result { + return Err(error); + } return Ok(TokenBackfillSignatureResult { resolved_transaction_count: 1, missing_transaction_count: 0, diff --git a/kb_lib/src/trade_aggregation.rs b/kb_lib/src/trade_aggregation.rs index 631c87e..c6c8518 100644 --- a/kb_lib/src/trade_aggregation.rs +++ b/kb_lib/src/trade_aggregation.rs @@ -17,32 +17,6 @@ pub struct TradeAggregationResult { pub created_trade_event: bool, } -type ExtractedTradeAmounts = ( - std::option::Option, - std::option::Option, - std::option::Option, -); - -#[derive(Debug, Clone)] -struct PumpSwapPoolBalanceDeltaAmounts { - base_amount_raw: std::string::String, - quote_amount_raw: std::string::String, - price_quote_per_base: f64, -} - -#[derive(Debug, Clone)] -enum PumpSwapPoolBalanceDeltaResolution { - Matched(PumpSwapPoolBalanceDeltaAmounts), - DirectionMismatch, - MissingData, -} - -#[derive(Debug, Clone)] -struct TokenBalanceRawAmount { - raw_amount: i128, - decimals: std::option::Option, -} - /// Trade-aggregation service. #[derive(Debug, Clone)] pub struct TradeAggregationService { @@ -62,129 +36,49 @@ impl TradeAggregationService { &self, signature: &str, ) -> Result, crate::Error> { - let transaction_result = - crate::query_chain_transactions_get_by_signature(self.database.as_ref(), signature) - .await; - let transaction_option = match transaction_result { - Ok(transaction_option) => transaction_option, - Err(error) => return Err(error), - }; - let transaction = match transaction_option { - Some(transaction) => transaction, - None => { - return Err(crate::Error::InvalidState(format!( - "cannot aggregate trades for unknown transaction '{}'", - signature - ))); - }, - }; - let transaction_id = match transaction.id { - Some(transaction_id) => transaction_id, - None => { - return Err(crate::Error::InvalidState(format!( - "transaction '{}' has no internal id", - signature - ))); - }, - }; - let decoded_events_result = crate::query_dex_decoded_events_list_by_transaction_id( - self.database.as_ref(), - transaction_id, - ) - .await; - let decoded_events = match decoded_events_result { - Ok(decoded_events) => decoded_events, - Err(error) => return Err(error), - }; - let mut results = std::vec::Vec::new(); - for decoded_event in &decoded_events { - if !is_trade_event_kind(decoded_event.event_kind.as_str()) { - continue; - } - let decoded_event_id = match decoded_event.id { - Some(decoded_event_id) => decoded_event_id, - None => { - return Err(crate::Error::InvalidState( - "decoded event has no internal id".to_string(), - )); - }, - }; - let existing_trade_result = crate::query_trade_events_get_by_decoded_event_id( + let transaction_context = + crate::trade_aggregation_context::load_trade_aggregation_transaction_context( self.database.as_ref(), - decoded_event_id, + signature, ) .await; - let existing_trade_option = match existing_trade_result { - Ok(existing_trade_option) => existing_trade_option, + let transaction_context = match transaction_context { + Ok(transaction_context) => transaction_context, + Err(error) => return Err(error), + }; + let transaction = transaction_context.transaction; + let transaction_id = transaction_context.transaction_id; + let decoded_events = transaction_context.decoded_events; + let mut results = std::vec::Vec::new(); + for decoded_event in &decoded_events { + if !crate::is_dex_trade_event_kind(decoded_event.event_kind.as_str()) { + continue; + } + let event_context = + crate::trade_aggregation_context::load_trade_aggregation_decoded_event_context( + self.database.as_ref(), + decoded_event, + ) + .await; + let event_context = match event_context { + Ok(Some(event_context)) => event_context, + Ok(None) => continue, Err(error) => return Err(error), }; - let pool_address = match decoded_event.pool_account.clone() { - Some(pool_address) => pool_address, - None => continue, - }; - let pool_result = - crate::query_pools_get_by_address(self.database.as_ref(), pool_address.as_str()) - .await; - let pool_option = match pool_result { - Ok(pool_option) => pool_option, - Err(error) => return Err(error), - }; - let pool = match pool_option { - Some(pool) => pool, - None => continue, - }; - let pool_id = match pool.id { - Some(pool_id) => pool_id, - None => { - return Err(crate::Error::InvalidState(format!( - "pool '{}' has no internal id", - pool.address - ))); - }, - }; - let pair_result = - crate::query_pairs_get_by_pool_id(self.database.as_ref(), pool_id).await; - let pair_option = match pair_result { - Ok(pair_option) => pair_option, - Err(error) => return Err(error), - }; - let pair = match pair_option { - Some(pair) => pair, - None => continue, - }; - let pair_id = match pair.id { - Some(pair_id) => pair_id, - None => { - return Err(crate::Error::InvalidState(format!( - "pair for pool '{}' has no internal id", - pool_id - ))); - }, - }; - let base_token_result = - crate::query_tokens_get_by_id(self.database.as_ref(), pair.base_token_id).await; - let (base_token_mint, base_token_decimals) = match base_token_result { - Ok(Some(token)) => (Some(token.mint), token.decimals), - Ok(None) => (None, None), - Err(error) => return Err(error), - }; - let quote_token_result = - crate::query_tokens_get_by_id(self.database.as_ref(), pair.quote_token_id).await; - let (quote_token_mint, quote_token_decimals) = match quote_token_result { - Ok(Some(token)) => (Some(token.mint), token.decimals), - Ok(None) => (None, None), - Err(error) => return Err(error), - }; - let pool_tokens_result = - crate::query_pool_tokens_list_by_pool_id(self.database.as_ref(), pool_id).await; - let pool_tokens = match pool_tokens_result { - Ok(pool_tokens) => pool_tokens, - Err(error) => return Err(error), - }; - let base_vault_address = - find_pool_token_vault_address_by_token_id(&pool_tokens, pair.base_token_id); - let quote_vault_address = - find_pool_token_vault_address_by_token_id(&pool_tokens, pair.quote_token_id); + + let decoded_event_id = event_context.decoded_event_id; + let existing_trade_option = event_context.existing_trade_event; + let pool_address = event_context.pool_address; + let pool = event_context.pool; + let pool_id = event_context.pool_id; + let pair = event_context.pair; + let pair_id = event_context.pair_id; + let base_token_mint = event_context.base_token_mint; + let base_token_decimals = event_context.base_token_decimals; + let quote_token_mint = event_context.quote_token_mint; + let quote_token_decimals = event_context.quote_token_decimals; + let base_vault_address = event_context.base_vault_address; + let quote_vault_address = event_context.quote_vault_address; let payload_result = serde_json::from_str::(decoded_event.payload_json.as_str()); let payload = match payload_result { @@ -200,7 +94,8 @@ impl TradeAggregationService { continue; }, }; - if !is_decoded_event_trade_candidate(decoded_event.event_kind.as_str(), &payload) { + if !crate::is_decoded_event_trade_candidate(decoded_event.event_kind.as_str(), &payload) + { tracing::debug!( event_kind = %decoded_event.event_kind, pool_account = ?decoded_event.pool_account, @@ -209,7 +104,10 @@ impl TradeAggregationService { ); continue; } - if !is_decoded_event_candle_candidate(decoded_event.event_kind.as_str(), &payload) { + if !crate::is_decoded_event_candle_candidate( + decoded_event.event_kind.as_str(), + &payload, + ) { tracing::debug!( event_kind = %decoded_event.event_kind, pool_account = ?decoded_event.pool_account, @@ -218,416 +116,32 @@ impl TradeAggregationService { ); continue; } - let trade_side = extract_trade_side(decoded_event.event_kind.as_str(), &payload); - let mut base_amount_raw = extract_amount_string( - &payload, - &["baseAmountRaw", "base_amount_raw", "baseAmount", "amountBase", "amountInBase"], - ); - let mut quote_amount_raw = extract_amount_string( - &payload, - &[ - "quoteAmountRaw", - "quote_amount_raw", - "quoteAmount", - "amountQuote", - "amountOutQuote", - ], - ); - let mut price_quote_per_base = None; - if decoded_event.event_kind.starts_with("pump_swap.") - && (base_amount_raw.is_none() - || quote_amount_raw.is_none() - || price_quote_per_base.is_none()) - { - let pool_owner_result = - match (base_token_mint.as_deref(), quote_token_mint.as_deref()) { - (Some(base_mint), Some(quote_mint)) => { - resolve_pump_swap_trade_amounts_from_pool_balance_deltas( - transaction.meta_json.as_deref(), - pool_address.as_str(), - base_mint, - quote_mint, - decoded_event.event_kind.as_str(), - base_token_decimals, - quote_token_decimals, - ) - }, - _ => Ok(PumpSwapPoolBalanceDeltaResolution::MissingData), - }; - let pool_owner_resolution = match pool_owner_result { - Ok(pool_owner_resolution) => pool_owner_resolution, - Err(error) => return Err(error), - }; - let pool_owner_resolution_label = match &pool_owner_resolution { - PumpSwapPoolBalanceDeltaResolution::Matched(_) => "matched", - PumpSwapPoolBalanceDeltaResolution::DirectionMismatch => "direction_mismatch", - PumpSwapPoolBalanceDeltaResolution::MissingData => "missing_data", - }; - tracing::debug!( - event_kind = %decoded_event.event_kind, - pool_account = ?decoded_event.pool_account, - decoded_event_id = ?decoded_event.id, - transaction_signature = %transaction.signature, - base_mint = ?base_token_mint, - quote_mint = ?quote_token_mint, - pool_owner_resolution = %pool_owner_resolution_label, - "pump_swap pool-owner delta resolution result" - ); - match pool_owner_resolution { - PumpSwapPoolBalanceDeltaResolution::Matched(amounts) => { - base_amount_raw = Some(amounts.base_amount_raw); - quote_amount_raw = Some(amounts.quote_amount_raw); - price_quote_per_base = Some(amounts.price_quote_per_base); - tracing::debug!( - event_kind = %decoded_event.event_kind, - pool_account = ?decoded_event.pool_account, - decoded_event_id = ?decoded_event.id, - base_mint = ?base_token_mint, - quote_mint = ?quote_token_mint, - base_amount_raw = ?base_amount_raw, - quote_amount_raw = ?quote_amount_raw, - price_quote_per_base = ?price_quote_per_base, - "pump_swap trade amounts recovered from pool-owner token balance deltas" - ); - }, - PumpSwapPoolBalanceDeltaResolution::DirectionMismatch => { - tracing::debug!( - event_kind = %decoded_event.event_kind, - pool_account = ?decoded_event.pool_account, - decoded_event_id = ?decoded_event.id, - transaction_signature = %transaction.signature, - "pump_swap pool-owner full-transaction delta direction mismatch; continuing with instruction-scoped fallbacks" - ); - }, - PumpSwapPoolBalanceDeltaResolution::MissingData => {}, - } - let decoded_instruction_index = match decoded_event.instruction_id { - Some(instruction_id) => { - let instruction_result = crate::query_chain_instructions_get_by_id( - self.database.as_ref(), - instruction_id, - ) - .await; - let instruction_option = match instruction_result { - Ok(instruction_option) => instruction_option, - Err(error) => return Err(error), - }; - match instruction_option { - Some(instruction) => Some(instruction.instruction_index), - None => None, - } - }, - None => None, - }; - let payload_user_base_token_account = extract_string_by_candidate_keys( - &payload, - &["userBaseTokenAccount", "user_base_token_account"], - ); - let payload_user_quote_token_account = extract_string_by_candidate_keys( - &payload, - &["userQuoteTokenAccount", "user_quote_token_account"], - ); - let payload_pool_base_token_account = extract_string_by_candidate_keys( - &payload, - &["poolBaseTokenAccount", "pool_base_token_account"], - ); - let payload_pool_quote_token_account = extract_string_by_candidate_keys( - &payload, - &["poolQuoteTokenAccount", "pool_quote_token_account"], - ); - let effective_base_vault_address = match base_vault_address.as_deref() { - Some(base_vault_address) => Some(base_vault_address), - None => payload_pool_base_token_account.as_deref(), - }; - let effective_quote_vault_address = match quote_vault_address.as_deref() { - Some(quote_vault_address) => Some(quote_vault_address), - None => payload_pool_quote_token_account.as_deref(), - }; - let ( - input_vault_address, - output_vault_address, - input_token_account, - output_token_account, - ) = if decoded_event.event_kind.ends_with(".buy") { - ( - effective_quote_vault_address, - effective_base_vault_address, - payload_user_quote_token_account.as_deref(), - payload_user_base_token_account.as_deref(), - ) - } else if decoded_event.event_kind.ends_with(".sell") { - ( - effective_base_vault_address, - effective_quote_vault_address, - payload_user_base_token_account.as_deref(), - payload_user_quote_token_account.as_deref(), - ) - } else { - (None, None, None, None) - }; - let inferred_result = extract_trade_amounts_from_instruction_token_transfers( - transaction.meta_json.as_deref(), - decoded_instruction_index, - input_vault_address, - output_vault_address, - input_token_account, - output_token_account, - effective_base_vault_address, - effective_quote_vault_address, - ); - let inferred = match inferred_result { - Ok(inferred) => inferred, - Err(error) => return Err(error), - }; - if base_amount_raw.is_none() { - base_amount_raw = inferred.0; - } - if quote_amount_raw.is_none() { - quote_amount_raw = inferred.1; - } - if price_quote_per_base.is_none() { - price_quote_per_base = inferred.2; - } - if base_amount_raw.is_none() || quote_amount_raw.is_none() { - let fallback_result = extract_trade_amounts_from_vault_balance_deltas( - transaction.transaction_json.as_str(), - transaction.meta_json.as_deref(), - effective_base_vault_address, - effective_quote_vault_address, - ); - let fallback = match fallback_result { - Ok(fallback) => fallback, - Err(error) => return Err(error), - }; - if base_amount_raw.is_none() { - base_amount_raw = fallback.0; - } - if quote_amount_raw.is_none() { - quote_amount_raw = fallback.1; - } - if price_quote_per_base.is_none() { - price_quote_per_base = fallback.2; - } - } - if base_amount_raw.is_none() - || quote_amount_raw.is_none() - || price_quote_per_base.is_none() - { - let transaction_value_result = build_transaction_value_with_meta_json( - transaction.transaction_json.as_str(), - transaction.meta_json.as_deref(), - ); - let transaction_value = match transaction_value_result { - Ok(transaction_value) => transaction_value, - Err(error) => return Err(error), - }; - let fallback_amounts = - match (base_token_mint.as_deref(), quote_token_mint.as_deref()) { - (Some(base_mint), Some(quote_mint)) => { - try_build_pump_swap_trade_amounts_from_token_balance_deltas( - &transaction_value, - base_mint, - quote_mint, - ) - }, - _ => None, - }; - if let Some(fallback_amounts) = fallback_amounts { - if base_amount_raw.is_none() { - base_amount_raw = convert_ui_amount_to_raw_string( - fallback_amounts.base_amount, - base_token_decimals, - ); - } - if quote_amount_raw.is_none() { - quote_amount_raw = convert_ui_amount_to_raw_string( - fallback_amounts.quote_amount, - quote_token_decimals, - ); - } - if price_quote_per_base.is_none() { - price_quote_per_base = Some(fallback_amounts.price_quote_per_base); - } - tracing::debug!( - event_kind = %decoded_event.event_kind, - pool_account = ?decoded_event.pool_account, - decoded_event_id = ?decoded_event.id, - base_mint = ?base_token_mint, - quote_mint = ?quote_token_mint, - base_amount_raw = ?base_amount_raw, - quote_amount_raw = ?quote_amount_raw, - price_quote_per_base = ?price_quote_per_base, - "pump_swap trade amounts recovered from token balance deltas" - ); - } - } - } - if decoded_event.event_kind.starts_with("pump_fun.") - && (base_amount_raw.is_none() - || quote_amount_raw.is_none() - || price_quote_per_base.is_none()) - { - let inferred_result = extract_pump_fun_amounts_from_transaction( - transaction.transaction_json.as_str(), - transaction.meta_json.as_deref(), - base_vault_address.as_deref(), - quote_vault_address.as_deref(), - ); - let inferred = match inferred_result { - Ok(inferred) => inferred, - Err(error) => return Err(error), - }; - if base_amount_raw.is_none() { - base_amount_raw = inferred.0; - } - if quote_amount_raw.is_none() { - quote_amount_raw = inferred.1; - } - if price_quote_per_base.is_none() { - price_quote_per_base = inferred.2; - } - } - if (decoded_event.event_kind.starts_with("raydium_cpmm.") - || decoded_event.event_kind.starts_with("raydium_clmm.")) - && (base_amount_raw.is_none() - || quote_amount_raw.is_none() - || price_quote_per_base.is_none()) - { - let decoded_instruction_index = match decoded_event.instruction_id { - Some(instruction_id) => { - let instruction_result = crate::query_chain_instructions_get_by_id( - self.database.as_ref(), - instruction_id, - ) - .await; - let instruction_option = match instruction_result { - Ok(instruction_option) => instruction_option, - Err(error) => return Err(error), - }; - match instruction_option { - Some(instruction) => Some(instruction.instruction_index), - None => None, - } - }, - None => None, - }; - let payload_input_vault_address = - extract_string_by_candidate_keys(&payload, &["inputVault", "input_vault"]); - let payload_output_vault_address = - extract_string_by_candidate_keys(&payload, &["outputVault", "output_vault"]); - let payload_input_token_account = extract_string_by_candidate_keys( - &payload, - &["inputTokenAccount", "input_token_account"], - ); - let payload_output_token_account = extract_string_by_candidate_keys( - &payload, - &["outputTokenAccount", "output_token_account"], - ); - let payload_base_vault_address = - extract_string_by_candidate_keys(&payload, &["baseVault", "base_vault"]); - let payload_quote_vault_address = - extract_string_by_candidate_keys(&payload, &["quoteVault", "quote_vault"]); - let effective_base_vault_address = match base_vault_address.as_deref() { - Some(base_vault_address) => Some(base_vault_address), - None => payload_base_vault_address.as_deref(), - }; - let effective_quote_vault_address = match quote_vault_address.as_deref() { - Some(quote_vault_address) => Some(quote_vault_address), - None => payload_quote_vault_address.as_deref(), - }; - let inferred_result = extract_trade_amounts_from_instruction_token_transfers( - transaction.meta_json.as_deref(), - decoded_instruction_index, - payload_input_vault_address.as_deref(), - payload_output_vault_address.as_deref(), - payload_input_token_account.as_deref(), - payload_output_token_account.as_deref(), - effective_base_vault_address, - effective_quote_vault_address, - ); - let inferred = match inferred_result { - Ok(inferred) => inferred, - Err(error) => return Err(error), - }; - if base_amount_raw.is_none() { - base_amount_raw = inferred.0; - } - if quote_amount_raw.is_none() { - quote_amount_raw = inferred.1; - } - if price_quote_per_base.is_none() { - price_quote_per_base = inferred.2; - } - } - if decoded_event.event_kind.starts_with("raydium_cpmm.") - && (base_amount_raw.is_none() || quote_amount_raw.is_none()) - { - let inferred_result = extract_trade_amounts_from_vault_balance_deltas( - transaction.transaction_json.as_str(), - transaction.meta_json.as_deref(), - base_vault_address.as_deref(), - quote_vault_address.as_deref(), - ); - let inferred = match inferred_result { - Ok(inferred) => inferred, - Err(error) => return Err(error), - }; - if base_amount_raw.is_none() { - base_amount_raw = inferred.0; - } - if quote_amount_raw.is_none() { - quote_amount_raw = inferred.1; - } - if price_quote_per_base.is_none() { - price_quote_per_base = inferred.2; - } - } - if decoded_event.event_kind.starts_with("raydium_clmm.") - && (base_amount_raw.is_none() || quote_amount_raw.is_none()) - { - let inferred_result = extract_trade_amounts_from_vault_balance_deltas( - transaction.transaction_json.as_str(), - transaction.meta_json.as_deref(), - base_vault_address.as_deref(), - quote_vault_address.as_deref(), - ); - let inferred = match inferred_result { - Ok(inferred) => inferred, - Err(error) => return Err(error), - }; - if base_amount_raw.is_none() { - base_amount_raw = inferred.0; - } - if quote_amount_raw.is_none() { - quote_amount_raw = inferred.1; - } - if price_quote_per_base.is_none() { - price_quote_per_base = inferred.2; - } - } - if price_quote_per_base.is_none() { - price_quote_per_base = compute_price_quote_per_base_from_raw_amounts_with_decimals( - base_amount_raw.as_deref(), - quote_amount_raw.as_deref(), - base_token_decimals, - quote_token_decimals, - ); - } - if price_quote_per_base.is_none() { - price_quote_per_base = compute_price_quote_per_base_with_decimals( - transaction.meta_json.as_deref(), - transaction.transaction_json.as_str(), - base_vault_address.as_deref(), - quote_vault_address.as_deref(), - ); - } - if price_quote_per_base.is_none() { - price_quote_per_base = compute_price_quote_per_base_from_raw_amounts( - base_amount_raw.as_deref(), - quote_amount_raw.as_deref(), - ); - } - if !is_priced_trade_event( + let trade_side = crate::trade_side_resolution::extract_trade_side( + decoded_event.event_kind.as_str(), + &payload, ); + let amount_input = crate::trade_amount_resolution::TradeAmountResolutionInput { + database: self.database.as_ref(), + transaction: &transaction, + decoded_event, + payload: &payload, + pool_address: pool_address.as_str(), + base_token_mint: base_token_mint.as_deref(), + quote_token_mint: quote_token_mint.as_deref(), + base_token_decimals, + quote_token_decimals, + base_vault_address: base_vault_address.as_deref(), + quote_vault_address: quote_vault_address.as_deref(), + }; + let amount_resolution = + crate::trade_amount_resolution::resolve_trade_amounts(&amount_input).await; + let amount_resolution = match amount_resolution { + Ok(amount_resolution) => amount_resolution, + Err(error) => return Err(error), + }; + let base_amount_raw = amount_resolution.base_amount_raw.clone(); + let quote_amount_raw = amount_resolution.quote_amount_raw.clone(); + let price_quote_per_base = amount_resolution.price_quote_per_base; + if !crate::trade_metric_update::is_priced_trade_event( base_amount_raw.as_deref(), quote_amount_raw.as_deref(), price_quote_per_base, @@ -644,1803 +158,35 @@ impl TradeAggregationService { ); continue; } - let slot_i64 = convert_slot_to_i64(transaction.slot); - let created_trade_event = existing_trade_option.is_none(); - let trade_event_dto = crate::TradeEventDto::new( - pool.dex_id, - pool_id, - pair_id, - transaction_id, - decoded_event_id, - transaction.signature.clone(), - slot_i64, - trade_side, - pair.base_token_id, - pair.quote_token_id, - base_amount_raw.clone(), - quote_amount_raw.clone(), - price_quote_per_base, - crate::ObservationSourceKind::Dex, - transaction.source_endpoint_name.clone(), - decoded_event.payload_json.clone(), - ); - tracing::debug!( - event_kind = %decoded_event.event_kind, - pool_account = ?decoded_event.pool_account, - decoded_event_id = ?decoded_event.id, - created_trade_event = created_trade_event, - "trade aggregation candidate" - ); - let trade_event_id_result = - crate::query_trade_events_upsert(self.database.as_ref(), &trade_event_dto).await; - let trade_event_id = match trade_event_id_result { - Ok(trade_event_id) => trade_event_id, - Err(error) => return Err(error), - }; - let pair_metric_result = - crate::query_pair_metrics_get_by_pair_id(self.database.as_ref(), pair_id).await; - let pair_metric_option = match pair_metric_result { - Ok(pair_metric_option) => pair_metric_option, - Err(error) => return Err(error), - }; - let pair_metric_id = if let Some(existing_metric) = pair_metric_option { - let existing_metric_id = match existing_metric.id { - Some(existing_metric_id) => existing_metric_id, - None => { - return Err(crate::Error::InvalidState( - "pair metric has no internal id".to_string(), - )); - }, - }; - if created_trade_event { - let mut updated_metric = existing_metric.clone(); - apply_trade_to_pair_metric( - &mut updated_metric, - slot_i64, - transaction.signature.clone(), - trade_side, - base_amount_raw.clone(), - quote_amount_raw.clone(), - price_quote_per_base, - ); - let upsert_result = - crate::query_pair_metrics_upsert(self.database.as_ref(), &updated_metric) - .await; - if let Err(error) = upsert_result { - return Err(error); - } - } - existing_metric_id - } else { - let mut new_metric = crate::PairMetricDto::new(pair_id); - apply_trade_to_pair_metric( - &mut new_metric, - slot_i64, - transaction.signature.clone(), + let materialization_input = + crate::trade_event_materialization::TradeEventMaterializationInput { + database: self.database.as_ref(), + persistence: &self.persistence, + transaction: &transaction, + transaction_id, + decoded_event, + decoded_event_id, + existing_trade_event: existing_trade_option, + pool: &pool, + pool_id, + pair: &pair, + pair_id, trade_side, - base_amount_raw.clone(), - quote_amount_raw.clone(), - price_quote_per_base, - ); - let upsert_result = - crate::query_pair_metrics_upsert(self.database.as_ref(), &new_metric).await; - match upsert_result { - Ok(pair_metric_id) => pair_metric_id, - Err(error) => return Err(error), - } - }; - if created_trade_event { - let payload = serde_json::json!({ - "pairId": pair_id, - "poolId": pool_id, - "tradeEventId": trade_event_id, - "tradeSide": format!("{:?}", trade_side), - "baseAmountRaw": base_amount_raw, - "quoteAmountRaw": quote_amount_raw, - "priceQuotePerBase": price_quote_per_base, - "transactionSignature": transaction.signature - }); - let observation_result = self - .persistence - .record_observation(&crate::DetectionObservationInput::new( - "dex.trade_aggregation".to_string(), - crate::ObservationSourceKind::Dex, - transaction.source_endpoint_name.clone(), - transaction.signature.clone(), - transaction.slot, - payload.clone(), - )) - .await; - let observation_id = match observation_result { - Ok(observation_id) => observation_id, - Err(error) => return Err(error), + amount_resolution: &amount_resolution, }; - let signal_result = self - .persistence - .record_signal(&crate::DetectionSignalInput::new( - "signal.dex.trade_aggregation.recorded".to_string(), - crate::AnalysisSignalSeverity::Low, - transaction.signature.clone(), - Some(observation_id), - None, - payload, - )) + let materialization_result = + crate::trade_event_materialization::materialize_trade_event(materialization_input) .await; - if let Err(error) = signal_result { - return Err(error); - } - } - results.push(crate::TradeAggregationResult { - trade_event_id, - pair_metric_id, - pair_id, - pool_id, - created_trade_event, - }); + let materialization_result = match materialization_result { + Ok(materialization_result) => materialization_result, + Err(error) => return Err(error), + }; + results.push(materialization_result); } return Ok(results); } } -fn is_decoded_event_trade_candidate(event_kind: &str, payload: &serde_json::Value) -> bool { - let trade_candidate_option = - extract_top_level_bool_by_candidate_keys(payload, &["tradeCandidate", "trade_candidate"]); - if let Some(trade_candidate) = trade_candidate_option { - return trade_candidate; - } - let event_category_option = - extract_string_by_candidate_keys(payload, &["eventCategory", "event_category"]); - if let Some(event_category) = event_category_option { - return event_category.as_str() == "trade"; - } - return is_trade_event_kind(event_kind); -} - -fn is_decoded_event_candle_candidate(event_kind: &str, payload: &serde_json::Value) -> bool { - let candle_candidate_option = - extract_top_level_bool_by_candidate_keys(payload, &["candleCandidate", "candle_candidate"]); - if let Some(candle_candidate) = candle_candidate_option { - return candle_candidate; - } - if !is_decoded_event_trade_candidate(event_kind, payload) { - return false; - } - return is_trade_event_kind(event_kind); -} - -fn extract_top_level_bool_by_candidate_keys( - payload: &serde_json::Value, - candidate_keys: &[&str], -) -> std::option::Option { - let object = match payload.as_object() { - Some(object) => object, - None => return None, - }; - for candidate_key in candidate_keys { - let value_option = object.get(*candidate_key); - let value = match value_option { - Some(value) => value, - None => continue, - }; - if let Some(value_bool) = value.as_bool() { - return Some(value_bool); - } - if let Some(value_i64) = value.as_i64() { - return Some(value_i64 != 0); - } - if let Some(value_u64) = value.as_u64() { - return Some(value_u64 != 0); - } - if let Some(value_text) = value.as_str() { - let normalized = value_text.trim().to_ascii_lowercase(); - if normalized.as_str() == "true" { - return Some(true); - } - if normalized.as_str() == "false" { - return Some(false); - } - if normalized.as_str() == "1" { - return Some(true); - } - if normalized.as_str() == "0" { - return Some(false); - } - } - } - return None; -} - -fn is_priced_trade_event( - base_amount_raw: std::option::Option<&str>, - quote_amount_raw: std::option::Option<&str>, - price_quote_per_base: std::option::Option, -) -> bool { - let base_amount_raw = match base_amount_raw { - Some(base_amount_raw) => base_amount_raw.trim(), - None => return false, - }; - if base_amount_raw.is_empty() { - return false; - } - let base_amount_result = base_amount_raw.parse::(); - let base_amount = match base_amount_result { - Ok(base_amount) => base_amount, - Err(_) => return false, - }; - if base_amount <= 0 { - return false; - } - let quote_amount_raw = match quote_amount_raw { - Some(quote_amount_raw) => quote_amount_raw.trim(), - None => return false, - }; - if quote_amount_raw.is_empty() { - return false; - } - let quote_amount_result = quote_amount_raw.parse::(); - let quote_amount = match quote_amount_result { - Ok(quote_amount) => quote_amount, - Err(_) => return false, - }; - if quote_amount <= 0 { - return false; - } - let price = match price_quote_per_base { - Some(price) => price, - None => return false, - }; - if !price.is_finite() { - return false; - } - return price > 0.0; -} - -fn is_trade_event_kind(is_trade_event_kind: &str) -> bool { - if is_trade_event_kind.ends_with(".swap") { - return true; - } - if is_trade_event_kind.ends_with(".buy") { - return true; - } - if is_trade_event_kind.ends_with(".sell") { - return true; - } - if is_trade_event_kind == "raydium_cpmm.swap_base_input" { - return true; - } - if is_trade_event_kind == "raydium_cpmm.swap_base_output" { - return true; - } - if is_trade_event_kind == "raydium_clmm.swap_v2" { - return true; - } - if is_trade_event_kind == "raydium_clmm.swap_router_base_in" { - return true; - } - if is_trade_event_kind == "raydium_clmm.swap_router_base_out" { - return true; - } - if is_trade_event_kind == "raydium_clmm.exact_output" { - return true; - } - return false; -} - -fn convert_slot_to_i64(slot: std::option::Option) -> std::option::Option { - match slot { - Some(slot) => match i64::try_from(slot) { - Ok(slot) => return Some(slot), - Err(_) => return None, - }, - None => return None, - } -} - -fn extract_trade_side(event_kind: &str, payload: &serde_json::Value) -> crate::SwapTradeSide { - let trade_side_option = extract_string_by_candidate_keys(payload, &["tradeSide", "trade_side"]); - match trade_side_option.as_deref() { - Some("BuyBase") => return crate::SwapTradeSide::BuyBase, - Some("buy") => return crate::SwapTradeSide::BuyBase, - Some("BUY") => return crate::SwapTradeSide::BuyBase, - Some("SellBase") => return crate::SwapTradeSide::SellBase, - Some("sell") => return crate::SwapTradeSide::SellBase, - Some("SELL") => return crate::SwapTradeSide::SellBase, - _ => {}, - } - if event_kind.ends_with(".buy") { - return crate::SwapTradeSide::BuyBase; - } - if event_kind.ends_with(".sell") { - return crate::SwapTradeSide::SellBase; - } - return crate::SwapTradeSide::Unknown; -} - -fn extract_amount_string( - payload: &serde_json::Value, - candidate_keys: &[&str], -) -> std::option::Option { - return extract_scalar_as_string_by_candidate_keys(payload, candidate_keys); -} - -fn apply_trade_to_pair_metric( - metric: &mut crate::PairMetricDto, - slot: std::option::Option, - signature: std::string::String, - trade_side: crate::SwapTradeSide, - base_amount_raw: std::option::Option, - quote_amount_raw: std::option::Option, - price_quote_per_base: std::option::Option, -) { - metric.trade_count += 1; - if trade_side == crate::SwapTradeSide::BuyBase { - metric.buy_count += 1; - } - if trade_side == crate::SwapTradeSide::SellBase { - metric.sell_count += 1; - } - if metric.first_slot.is_none() { - metric.first_slot = slot; - } - if metric.first_signature.is_none() { - metric.first_signature = Some(signature.clone()); - } - metric.last_slot = slot; - metric.last_signature = Some(signature); - metric.cumulative_base_amount_raw = - add_raw_amounts(metric.cumulative_base_amount_raw.clone(), base_amount_raw); - metric.cumulative_quote_amount_raw = - add_raw_amounts(metric.cumulative_quote_amount_raw.clone(), quote_amount_raw); - if price_quote_per_base.is_some() { - metric.last_price_quote_per_base = price_quote_per_base; - } - metric.updated_at = chrono::Utc::now(); -} - -fn add_raw_amounts( - left: std::option::Option, - right: std::option::Option, -) -> std::option::Option { - match (left, right) { - (None, None) => return None, - (Some(left), None) => return Some(left), - (None, Some(right)) => return Some(right), - (Some(left), Some(right)) => { - let left_value_result = left.parse::(); - let left_value = match left_value_result { - Ok(left_value) => left_value, - Err(_) => return Some(left), - }; - let right_value_result = right.parse::(); - let right_value = match right_value_result { - Ok(right_value) => right_value, - Err(_) => return Some(left), - }; - return Some((left_value + right_value).to_string()); - }, - } -} - -fn extract_string_by_candidate_keys( - value: &serde_json::Value, - candidate_keys: &[&str], -) -> std::option::Option { - if let Some(object) = value.as_object() { - for candidate_key in candidate_keys { - let direct_option = object.get(*candidate_key); - if let Some(direct) = direct_option { - let direct_text_option = direct.as_str(); - if let Some(direct_text) = direct_text_option { - return Some(direct_text.to_string()); - } - } - } - for nested_value in object.values() { - let nested_result = extract_string_by_candidate_keys(nested_value, candidate_keys); - if nested_result.is_some() { - return nested_result; - } - } - return None; - } - if let Some(array) = value.as_array() { - for nested_value in array { - let nested_result = extract_string_by_candidate_keys(nested_value, candidate_keys); - if nested_result.is_some() { - return nested_result; - } - } - } - return None; -} - -fn extract_scalar_as_string_by_candidate_keys( - value: &serde_json::Value, - candidate_keys: &[&str], -) -> std::option::Option { - if let Some(object) = value.as_object() { - for candidate_key in candidate_keys { - let direct_option = object.get(*candidate_key); - if let Some(direct) = direct_option { - if let Some(text) = direct.as_str() { - return Some(text.to_string()); - } - if let Some(number) = direct.as_i64() { - return Some(number.to_string()); - } - if let Some(number) = direct.as_u64() { - return Some(number.to_string()); - } - if let Some(number) = direct.as_f64() { - return Some(number.to_string()); - } - } - } - for nested_value in object.values() { - let nested_result = - extract_scalar_as_string_by_candidate_keys(nested_value, candidate_keys); - if nested_result.is_some() { - return nested_result; - } - } - return None; - } - if let Some(array) = value.as_array() { - for nested_value in array { - let nested_result = - extract_scalar_as_string_by_candidate_keys(nested_value, candidate_keys); - if nested_result.is_some() { - return nested_result; - } - } - } - return None; -} - -fn find_pool_token_vault_address_by_token_id( - pool_tokens: &[crate::PoolTokenDto], - token_id: i64, -) -> std::option::Option { - for pool_token in pool_tokens { - if pool_token.token_id != token_id { - continue; - } - let vault_address_option = pool_token.vault_address.clone(); - let vault_address = match vault_address_option { - Some(vault_address) => vault_address.trim().to_string(), - None => continue, - }; - if vault_address.is_empty() { - continue; - } - return Some(vault_address); - } - return None; -} - -fn extract_trade_amounts_from_vault_balance_deltas( - transaction_json: &str, - meta_json: std::option::Option<&str>, - base_vault_address: std::option::Option<&str>, - quote_vault_address: std::option::Option<&str>, -) -> Result { - let meta_json = match meta_json { - Some(meta_json) => meta_json, - None => return Ok((None, None, None)), - }; - let transaction_value_result = serde_json::from_str::(transaction_json); - let transaction_value = match transaction_value_result { - Ok(transaction_value) => transaction_value, - Err(error) => { - return Err(crate::Error::Json(format!( - "cannot parse transaction_json for pump_swap amount extraction: {}", - error - ))); - }, - }; - let meta_value_result = serde_json::from_str::(meta_json); - let meta_value = match meta_value_result { - Ok(meta_value) => meta_value, - Err(error) => { - return Err(crate::Error::Json(format!( - "cannot parse meta_json for pump_swap amount extraction: {}", - error - ))); - }, - }; - let account_keys_result = extract_transaction_account_keys(&transaction_value); - let account_keys = match account_keys_result { - Ok(account_keys) => account_keys, - Err(error) => return Err(error), - }; - let pre_balances_result = - extract_token_balance_map(&meta_value, &account_keys, "preTokenBalances"); - let pre_balances = match pre_balances_result { - Ok(pre_balances) => pre_balances, - Err(error) => return Err(error), - }; - let post_balances_result = - extract_token_balance_map(&meta_value, &account_keys, "postTokenBalances"); - let post_balances = match post_balances_result { - Ok(post_balances) => post_balances, - Err(error) => return Err(error), - }; - let mut base_amount_raw = None; - let mut quote_amount_raw = None; - let mut price_quote_per_base = None; - if let Some(base_vault_address) = base_vault_address { - let base_pre = pre_balances.get(base_vault_address); - let base_post = post_balances.get(base_vault_address); - let base_pre_raw = base_pre.map(|value| return value.0.clone()); - let base_post_raw = base_post.map(|value| return value.0.clone()); - base_amount_raw = compute_amount_delta_abs(base_pre_raw, base_post_raw); - let base_pre_ui = base_pre.and_then(|value| return value.1); - let base_post_ui = base_post.and_then(|value| return value.1); - let base_delta_ui = compute_ui_delta_abs(base_pre_ui, base_post_ui); - if let Some(quote_vault_address) = quote_vault_address { - let quote_pre = pre_balances.get(quote_vault_address); - let quote_post = post_balances.get(quote_vault_address); - let quote_pre_raw = quote_pre.map(|value| return value.0.clone()); - let quote_post_raw = quote_post.map(|value| return value.0.clone()); - quote_amount_raw = compute_amount_delta_abs(quote_pre_raw, quote_post_raw); - let quote_pre_ui = quote_pre.and_then(|value| return value.1); - let quote_post_ui = quote_post.and_then(|value| return value.1); - let quote_delta_ui = compute_ui_delta_abs(quote_pre_ui, quote_post_ui); - if let (Some(base_delta_ui), Some(quote_delta_ui)) = (base_delta_ui, quote_delta_ui) { - if base_delta_ui > 0.0 { - price_quote_per_base = Some(quote_delta_ui / base_delta_ui); - } - } - } - } - return Ok((base_amount_raw, quote_amount_raw, price_quote_per_base)); -} - -fn extract_trade_amounts_from_instruction_token_transfers( - meta_json: std::option::Option<&str>, - instruction_index: std::option::Option, - input_vault_address: std::option::Option<&str>, - output_vault_address: std::option::Option<&str>, - input_token_account: std::option::Option<&str>, - output_token_account: std::option::Option<&str>, - base_vault_address: std::option::Option<&str>, - quote_vault_address: std::option::Option<&str>, -) -> Result { - let meta_json = match meta_json { - Some(meta_json) => meta_json, - None => return Ok((None, None, None)), - }; - let instruction_index = match instruction_index { - Some(instruction_index) => u64::from(instruction_index), - None => return Ok((None, None, None)), - }; - let input_vault_address = match input_vault_address { - Some(input_vault_address) => input_vault_address.trim(), - None => return Ok((None, None, None)), - }; - let output_vault_address = match output_vault_address { - Some(output_vault_address) => output_vault_address.trim(), - None => return Ok((None, None, None)), - }; - let input_token_account = match input_token_account { - Some(input_token_account) => input_token_account.trim(), - None => return Ok((None, None, None)), - }; - let output_token_account = match output_token_account { - Some(output_token_account) => output_token_account.trim(), - None => return Ok((None, None, None)), - }; - let base_vault_address = match base_vault_address { - Some(base_vault_address) => base_vault_address.trim(), - None => return Ok((None, None, None)), - }; - let quote_vault_address = match quote_vault_address { - Some(quote_vault_address) => quote_vault_address.trim(), - None => return Ok((None, None, None)), - }; - if input_vault_address.is_empty() - || output_vault_address.is_empty() - || input_token_account.is_empty() - || output_token_account.is_empty() - || base_vault_address.is_empty() - || quote_vault_address.is_empty() - { - return Ok((None, None, None)); - } - let meta_value_result = serde_json::from_str::(meta_json); - let meta_value = match meta_value_result { - Ok(meta_value) => meta_value, - Err(error) => { - return Err(crate::Error::Json(format!( - "cannot parse meta_json for instruction-scoped token transfer amount extraction: {}", - error - ))); - }, - }; - let inner_groups_option = - meta_value.get("innerInstructions").and_then(|value| return value.as_array()); - let inner_groups = match inner_groups_option { - Some(inner_groups) => inner_groups, - None => return Ok((None, None, None)), - }; - let mut input_amount_raw = None; - let mut output_amount_raw = None; - for inner_group in inner_groups { - let group_index_option = inner_group.get("index").and_then(|value| return value.as_u64()); - let group_index = match group_index_option { - Some(group_index) => group_index, - None => continue, - }; - if group_index != instruction_index { - continue; - } - let instructions_option = - inner_group.get("instructions").and_then(|value| return value.as_array()); - let instructions = match instructions_option { - Some(instructions) => instructions, - None => continue, - }; - for instruction in instructions { - if !is_spl_token_transfer_instruction(instruction) { - continue; - } - let parsed_option = instruction.get("parsed"); - let parsed = match parsed_option { - Some(parsed) => parsed, - None => continue, - }; - let info_option = parsed.get("info"); - let info = match info_option { - Some(info) => info, - None => continue, - }; - let source_option = extract_string_by_candidate_keys(info, &["source"]); - let source = match source_option { - Some(source) => source, - None => continue, - }; - let destination_option = extract_string_by_candidate_keys(info, &["destination"]); - let destination = match destination_option { - Some(destination) => destination, - None => continue, - }; - let amount_option = extract_scalar_as_string_by_candidate_keys(info, &["amount"]); - let amount = match amount_option { - Some(amount) => amount, - None => continue, - }; - if input_amount_raw.is_none() - && account_equals(source.as_str(), input_token_account) - && account_equals(destination.as_str(), input_vault_address) - { - input_amount_raw = Some(amount.clone()); - continue; - } - if output_amount_raw.is_none() - && account_equals(source.as_str(), output_vault_address) - && account_equals(destination.as_str(), output_token_account) - { - output_amount_raw = Some(amount); - continue; - } - } - } - if input_amount_raw.is_none() && output_amount_raw.is_none() { - return Ok((None, None, None)); - } - if account_equals(input_vault_address, base_vault_address) - && account_equals(output_vault_address, quote_vault_address) - { - return Ok((input_amount_raw, output_amount_raw, None)); - } - if account_equals(input_vault_address, quote_vault_address) - && account_equals(output_vault_address, base_vault_address) - { - return Ok((output_amount_raw, input_amount_raw, None)); - } - return Ok((None, None, None)); -} - -fn is_spl_token_transfer_instruction(instruction: &serde_json::Value) -> bool { - let program_id_option = instruction.get("programId").and_then(|value| return value.as_str()); - if let Some(program_id) = program_id_option { - let spl_token_program_id = crate::SPL_TOKEN_PROGRAM_ID.to_string(); - let spl_token_2022_program_id = crate::SPL_TOKEN_2022_PROGRAM_ID.to_string(); - if program_id != spl_token_program_id.as_str() - && program_id != spl_token_2022_program_id.as_str() - { - return false; - } - } - let parsed_type_option = instruction - .get("parsed") - .and_then(|parsed| return parsed.get("type")) - .and_then(|value| return value.as_str()); - match parsed_type_option { - Some("transfer") => return true, - Some("transferChecked") => return true, - _ => return false, - } -} - -fn account_equals(left: &str, account_equals: &str) -> bool { - let left = left.trim(); - let right = account_equals.trim(); - if left.is_empty() || right.is_empty() { - return false; - } - return left == right; -} - -fn extract_pump_fun_amounts_from_transaction( - transaction_json: &str, - meta_json: std::option::Option<&str>, - base_vault_address: std::option::Option<&str>, - quote_native_address: std::option::Option<&str>, -) -> Result { - let meta_json = match meta_json { - Some(meta_json) => meta_json, - None => return Ok((None, None, None)), - }; - let transaction_value_result = serde_json::from_str::(transaction_json); - let transaction_value = match transaction_value_result { - Ok(transaction_value) => transaction_value, - Err(error) => { - return Err(crate::Error::Json(format!( - "cannot parse transaction_json for pump_fun amount extraction: {}", - error - ))); - }, - }; - let meta_value_result = serde_json::from_str::(meta_json); - let meta_value = match meta_value_result { - Ok(meta_value) => meta_value, - Err(error) => { - return Err(crate::Error::Json(format!( - "cannot parse meta_json for pump_fun amount extraction: {}", - error - ))); - }, - }; - let account_keys_result = extract_transaction_account_keys(&transaction_value); - let account_keys = match account_keys_result { - Ok(account_keys) => account_keys, - Err(error) => return Err(error), - }; - let pre_balances_result = - extract_token_balance_map(&meta_value, &account_keys, "preTokenBalances"); - let pre_balances = match pre_balances_result { - Ok(pre_balances) => pre_balances, - Err(error) => return Err(error), - }; - let post_balances_result = - extract_token_balance_map(&meta_value, &account_keys, "postTokenBalances"); - let post_balances = match post_balances_result { - Ok(post_balances) => post_balances, - Err(error) => return Err(error), - }; - let mut base_amount_raw = None; - let mut quote_amount_raw = None; - let mut price_quote_per_base = None; - let mut base_delta_ui = None; - if let Some(base_vault_address) = base_vault_address { - let base_pre = pre_balances.get(base_vault_address); - let base_post = post_balances.get(base_vault_address); - let base_pre_raw = base_pre.map(|value| return value.0.clone()); - let base_post_raw = base_post.map(|value| return value.0.clone()); - base_amount_raw = compute_amount_delta_abs(base_pre_raw, base_post_raw); - let base_pre_ui = base_pre.and_then(|value| return value.1); - let base_post_ui = base_post.and_then(|value| return value.1); - base_delta_ui = compute_ui_delta_abs(base_pre_ui, base_post_ui); - } - if let Some(quote_native_address) = quote_native_address { - let quote_delta_result = extract_native_balance_delta_by_address( - &meta_value, - &account_keys, - quote_native_address, - ); - let quote_delta = match quote_delta_result { - Ok(quote_delta) => quote_delta, - Err(error) => return Err(error), - }; - if let Some(quote_delta_lamports) = quote_delta { - quote_amount_raw = Some(quote_delta_lamports.to_string()); - let quote_delta_ui = quote_delta_lamports as f64 / 1_000_000_000.0; - if let Some(base_delta_ui) = base_delta_ui { - if base_delta_ui > 0.0 { - price_quote_per_base = Some(quote_delta_ui / base_delta_ui); - } - } - } - } - return Ok((base_amount_raw, quote_amount_raw, price_quote_per_base)); -} - -fn extract_native_balance_delta_by_address( - meta_value: &serde_json::Value, - account_keys: &[std::string::String], - address: &str, -) -> Result, crate::Error> { - let mut account_index = None; - for (index, account_key) in account_keys.iter().enumerate() { - if account_key.as_str() == address { - account_index = Some(index); - break; - } - } - let account_index = match account_index { - Some(account_index) => account_index, - None => return Ok(None), - }; - let pre_balances_option = - meta_value.get("preBalances").and_then(|value| return value.as_array()); - let post_balances_option = - meta_value.get("postBalances").and_then(|value| return value.as_array()); - let pre_balances = match pre_balances_option { - Some(pre_balances) => pre_balances, - None => return Ok(None), - }; - let post_balances = match post_balances_option { - Some(post_balances) => post_balances, - None => return Ok(None), - }; - if account_index >= pre_balances.len() || account_index >= post_balances.len() { - return Ok(None); - } - let pre_balance_option = pre_balances[account_index].as_u64(); - let post_balance_option = post_balances[account_index].as_u64(); - let pre_balance = match pre_balance_option { - Some(pre_balance) => pre_balance, - None => return Ok(None), - }; - let post_balance = match post_balance_option { - Some(post_balance) => post_balance, - None => return Ok(None), - }; - if post_balance >= pre_balance { - return Ok(Some(post_balance - pre_balance)); - } - return Ok(Some(pre_balance - post_balance)); -} - -fn extract_transaction_account_keys( - transaction_value: &serde_json::Value, -) -> Result, crate::Error> { - let candidate_arrays = [ - transaction_value - .get("message") - .and_then(|value| return value.get("accountKeys")), - transaction_value - .get("transaction") - .and_then(|value| return value.get("message")) - .and_then(|value| return value.get("accountKeys")), - transaction_value.get("accountKeys"), - ]; - for candidate_array_option in candidate_arrays { - let candidate_array = match candidate_array_option { - Some(candidate_array) => candidate_array, - None => continue, - }; - let array = match candidate_array.as_array() { - Some(array) => array, - None => continue, - }; - let mut account_keys = std::vec::Vec::new(); - for item in array { - if let Some(value) = item.as_str() { - account_keys.push(value.to_string()); - continue; - } - let pubkey_option = item.get("pubkey").and_then(|value| return value.as_str()); - if let Some(pubkey) = pubkey_option { - account_keys.push(pubkey.to_string()); - continue; - } - } - if !account_keys.is_empty() { - return Ok(account_keys); - } - } - return Err(crate::Error::Json( - "cannot extract accountKeys from transaction_json".to_string(), - )); -} - -fn extract_token_balance_map( - meta_value: &serde_json::Value, - account_keys: &[std::string::String], - field_name: &str, -) -> Result< - std::collections::BTreeMap< - std::string::String, - (std::string::String, std::option::Option), - >, - crate::Error, -> { - let mut result = std::collections::BTreeMap::< - std::string::String, - (std::string::String, std::option::Option), - >::new(); - let balances_option = meta_value.get(field_name).and_then(|value| return value.as_array()); - let balances = match balances_option { - Some(balances) => balances, - None => return Ok(result), - }; - for balance in balances { - let account_index_option = - balance.get("accountIndex").and_then(|value| return value.as_u64()); - let account_index = match account_index_option { - Some(account_index) => account_index as usize, - None => continue, - }; - if account_index >= account_keys.len() { - continue; - } - let account_address = account_keys[account_index].clone(); - let ui_token_amount_option = balance.get("uiTokenAmount"); - let ui_token_amount = match ui_token_amount_option { - Some(ui_token_amount) => ui_token_amount, - None => continue, - }; - let raw_amount_option = - ui_token_amount.get("amount").and_then(|value| return value.as_str()); - let raw_amount = match raw_amount_option { - Some(raw_amount) => raw_amount.to_string(), - None => continue, - }; - let ui_amount_string_option = - ui_token_amount.get("uiAmountString").and_then(|value| return value.as_str()); - let ui_amount = match ui_amount_string_option { - Some(ui_amount_string) => { - let parse_result = ui_amount_string.parse::(); - match parse_result { - Ok(ui_amount) => Some(ui_amount), - Err(_) => None, - } - }, - None => None, - }; - result.insert(account_address, (raw_amount, ui_amount)); - } - return Ok(result); -} - -fn compute_amount_delta_abs( - pre_amount: std::option::Option, - post_amount: std::option::Option, -) -> std::option::Option { - let pre_amount = match pre_amount { - Some(pre_amount) => pre_amount, - None => "0".to_string(), - }; - let post_amount = match post_amount { - Some(post_amount) => post_amount, - None => "0".to_string(), - }; - let pre_value_result = pre_amount.parse::(); - let pre_value = match pre_value_result { - Ok(pre_value) => pre_value, - Err(_) => return None, - }; - let post_value_result = post_amount.parse::(); - let post_value = match post_value_result { - Ok(post_value) => post_value, - Err(_) => return None, - }; - let delta = if post_value >= pre_value { - post_value - pre_value - } else { - pre_value - post_value - }; - return Some(delta.to_string()); -} - -fn compute_ui_delta_abs( - pre_amount: std::option::Option, - post_amount: std::option::Option, -) -> std::option::Option { - let pre_amount = match pre_amount { - Some(pre_amount) => pre_amount, - None => 0.0, - }; - let post_amount = match post_amount { - Some(post_amount) => post_amount, - None => 0.0, - }; - let delta = if post_amount >= pre_amount { - post_amount - pre_amount - } else { - pre_amount - post_amount - }; - return Some(delta); -} - -fn compute_price_quote_per_base_from_raw_amounts_with_decimals( - base_amount_raw: std::option::Option<&str>, - quote_amount_raw: std::option::Option<&str>, - base_decimals: std::option::Option, - quote_decimals: std::option::Option, -) -> std::option::Option { - let base_decimals = match base_decimals { - Some(base_decimals) => base_decimals, - None => return None, - }; - let quote_decimals = match quote_decimals { - Some(quote_decimals) => quote_decimals, - None => return None, - }; - let base_amount_raw = match base_amount_raw { - Some(base_amount_raw) => base_amount_raw.trim(), - None => return None, - }; - let quote_amount_raw = match quote_amount_raw { - Some(quote_amount_raw) => quote_amount_raw.trim(), - None => return None, - }; - if base_amount_raw.is_empty() || quote_amount_raw.is_empty() { - return None; - } - let base_amount_result = base_amount_raw.parse::(); - let base_amount = match base_amount_result { - Ok(base_amount) => base_amount, - Err(_) => return None, - }; - let quote_amount_result = quote_amount_raw.parse::(); - let quote_amount = match quote_amount_result { - Ok(quote_amount) => quote_amount, - Err(_) => return None, - }; - if base_amount <= 0.0 || quote_amount <= 0.0 { - return None; - } - let base_scale = 10_f64.powi(i32::from(base_decimals)); - let quote_scale = 10_f64.powi(i32::from(quote_decimals)); - if base_scale <= 0.0 || quote_scale <= 0.0 { - return None; - } - let base_ui_amount = base_amount / base_scale; - let quote_ui_amount = quote_amount / quote_scale; - if base_ui_amount <= 0.0 || quote_ui_amount <= 0.0 { - return None; - } - return Some(quote_ui_amount / base_ui_amount); -} - -fn compute_price_quote_per_base_from_raw_amounts( - base_amount_raw: std::option::Option<&str>, - quote_amount_raw: std::option::Option<&str>, -) -> std::option::Option { - let base_amount_raw = match base_amount_raw { - Some(base_amount_raw) => base_amount_raw.trim(), - None => return None, - }; - let quote_amount_raw = match quote_amount_raw { - Some(quote_amount_raw) => quote_amount_raw.trim(), - None => return None, - }; - if base_amount_raw.is_empty() || quote_amount_raw.is_empty() { - return None; - } - let base_amount_result = base_amount_raw.parse::(); - let base_amount = match base_amount_result { - Ok(base_amount) => base_amount, - Err(_) => return None, - }; - let quote_amount_result = quote_amount_raw.parse::(); - let quote_amount = match quote_amount_result { - Ok(quote_amount) => quote_amount, - Err(_) => return None, - }; - if base_amount <= 0.0 { - return None; - } - return Some(quote_amount / base_amount); -} - -fn compute_price_quote_per_base_with_decimals( - meta_json: std::option::Option<&str>, - transaction_json: &str, - base_vault_address: std::option::Option<&str>, - quote_vault_address: std::option::Option<&str>, -) -> std::option::Option { - let inferred_result = extract_trade_amounts_from_vault_balance_deltas( - transaction_json, - meta_json, - base_vault_address, - quote_vault_address, - ); - let inferred = match inferred_result { - Ok(inferred) => inferred, - Err(_) => return None, - }; - return inferred.2; -} - -fn build_transaction_value_with_meta_json( - transaction_json: &str, - meta_json: std::option::Option<&str>, -) -> Result { - let transaction_value_result = serde_json::from_str::(transaction_json); - let mut transaction_value = match transaction_value_result { - Ok(transaction_value) => transaction_value, - Err(error) => { - return Err(crate::Error::Json(format!( - "cannot parse transaction_json for pump_swap token-balance fallback: {}", - error - ))); - }, - }; - let meta_json = match meta_json { - Some(meta_json) => meta_json, - None => return Ok(transaction_value), - }; - let meta_value_result = serde_json::from_str::(meta_json); - let meta_value = match meta_value_result { - Ok(meta_value) => meta_value, - Err(error) => { - return Err(crate::Error::Json(format!( - "cannot parse meta_json for pump_swap token-balance fallback: {}", - error - ))); - }, - }; - match transaction_value.as_object_mut() { - Some(object) => { - object.insert("meta".to_string(), meta_value); - return Ok(transaction_value); - }, - None => {}, - } - return Ok(serde_json::json!({ - "transaction": transaction_value, - "meta": meta_value - })); -} - -fn convert_ui_amount_to_raw_string( - amount: f64, - decimals: std::option::Option, -) -> std::option::Option { - if !amount.is_finite() { - return None; - } - if amount <= 0.0_f64 { - return None; - } - let decimals = match decimals { - Some(decimals) => decimals, - None => return None, - }; - let scale = 10_f64.powi(i32::from(decimals)); - if !scale.is_finite() { - return None; - } - if scale <= 0.0_f64 { - return None; - } - let raw_amount = amount * scale; - if !raw_amount.is_finite() { - return None; - } - if raw_amount <= 0.0_f64 { - return None; - } - let rounded = raw_amount.round(); - if rounded <= 0.0_f64 { - return None; - } - return Some(format!("{:.0}", rounded)); -} - -// Private PumpSwap fallback amount extracted from transaction token balance deltas. -#[derive(Clone, Debug, PartialEq)] -struct PumpSwapBalanceDeltaTradeAmounts { - base_amount: f64, - quote_amount: f64, - price_quote_per_base: f64, -} - -// Private key used to match pre/post SPL token balances. -#[derive(Clone, Debug, Eq, Ord, PartialEq, PartialOrd)] -struct TokenBalanceDeltaKey { - account_index: i64, - mint: std::string::String, - owner: std::string::String, -} - -// Private accumulator used while reading pre/post token balances. -#[derive(Clone, Debug)] -struct TokenBalanceDeltaAccumulator { - pre_amount: f64, - post_amount: f64, -} - -fn try_build_pump_swap_trade_amounts_from_token_balance_deltas( - transaction_json: &serde_json::Value, - base_mint: &str, - quote_mint: &str, -) -> std::option::Option { - if base_mint.is_empty() { - return std::option::Option::None; - } - if quote_mint.is_empty() { - return std::option::Option::None; - } - if base_mint == quote_mint { - return std::option::Option::None; - } - let meta_json = match find_transaction_meta_json(transaction_json) { - std::option::Option::Some(value) => value, - std::option::Option::None => { - return std::option::Option::None; - }, - }; - let pre_token_balances = match json_array_field(meta_json, "preTokenBalances") { - std::option::Option::Some(value) => value, - std::option::Option::None => { - return std::option::Option::None; - }, - }; - let post_token_balances = match json_array_field(meta_json, "postTokenBalances") { - std::option::Option::Some(value) => value, - std::option::Option::None => { - return std::option::Option::None; - }, - }; - let mut deltas: std::collections::BTreeMap = - std::collections::BTreeMap::new(); - apply_token_balance_side_to_delta_map( - &mut deltas, - pre_token_balances, - base_mint, - quote_mint, - true, - ); - apply_token_balance_side_to_delta_map( - &mut deltas, - post_token_balances, - base_mint, - quote_mint, - false, - ); - let mut base_amount = 0.0_f64; - let mut quote_amount = 0.0_f64; - for (key, accumulator) in &deltas { - let delta_abs = (accumulator.post_amount - accumulator.pre_amount).abs(); - if !delta_abs.is_finite() { - continue; - } - if delta_abs <= 0.0_f64 { - continue; - } - if key.mint == base_mint { - if delta_abs > base_amount { - base_amount = delta_abs; - } - } else if key.mint == quote_mint && delta_abs > quote_amount { - quote_amount = delta_abs; - } - } - if !base_amount.is_finite() { - return std::option::Option::None; - } - if !quote_amount.is_finite() { - return std::option::Option::None; - } - if base_amount <= 0.0_f64 { - return std::option::Option::None; - } - if quote_amount <= 0.0_f64 { - return std::option::Option::None; - } - let price_quote_per_base = quote_amount / base_amount; - if !price_quote_per_base.is_finite() { - return std::option::Option::None; - } - if price_quote_per_base <= 0.0_f64 { - return std::option::Option::None; - } - return std::option::Option::Some(PumpSwapBalanceDeltaTradeAmounts { - base_amount, - quote_amount, - price_quote_per_base, - }); -} - -fn apply_token_balance_side_to_delta_map( - deltas: &mut std::collections::BTreeMap, - token_balances: &[serde_json::Value], - base_mint: &str, - quote_mint: &str, - is_pre: bool, -) { - for token_balance in token_balances { - let mint = match json_string_field(token_balance, "mint") { - std::option::Option::Some(value) => value, - std::option::Option::None => { - continue; - }, - }; - if mint != base_mint && mint != quote_mint { - continue; - } - let account_index = match json_i64_field(token_balance, "accountIndex") { - std::option::Option::Some(value) => value, - std::option::Option::None => { - continue; - }, - }; - let owner = match json_string_field(token_balance, "owner") { - std::option::Option::Some(value) => value, - std::option::Option::None => std::string::String::new(), - }; - let ui_amount = match token_balance_ui_amount(token_balance) { - std::option::Option::Some(value) => value, - std::option::Option::None => { - continue; - }, - }; - if !ui_amount.is_finite() { - continue; - } - let key = TokenBalanceDeltaKey { account_index, mint, owner }; - let entry = deltas.entry(key).or_insert(TokenBalanceDeltaAccumulator { - pre_amount: 0.0_f64, - post_amount: 0.0_f64, - }); - if is_pre { - entry.pre_amount = ui_amount; - } else { - entry.post_amount = ui_amount; - } - } -} - -fn token_balance_ui_amount(token_balance: &serde_json::Value) -> std::option::Option { - let ui_token_amount = match json_object_field(token_balance, "uiTokenAmount") { - std::option::Option::Some(value) => value, - std::option::Option::None => { - return std::option::Option::None; - }, - }; - match json_f64_field(ui_token_amount, "uiAmount") { - std::option::Option::Some(value) => { - return std::option::Option::Some(value); - }, - std::option::Option::None => {}, - } - match json_string_field(ui_token_amount, "uiAmountString") { - std::option::Option::Some(value) => match value.parse::() { - std::result::Result::Ok(parsed) => { - return std::option::Option::Some(parsed); - }, - std::result::Result::Err(_) => {}, - }, - std::option::Option::None => {}, - } - let raw_amount = match json_string_field(ui_token_amount, "amount") { - std::option::Option::Some(value) => value, - std::option::Option::None => { - return std::option::Option::None; - }, - }; - let decimals = match json_i64_field(ui_token_amount, "decimals") { - std::option::Option::Some(value) => value, - std::option::Option::None => { - return std::option::Option::None; - }, - }; - if decimals < 0_i64 { - return std::option::Option::None; - } - if decimals > 18_i64 { - return std::option::Option::None; - } - let raw_amount_f64 = match raw_amount.parse::() { - std::result::Result::Ok(value) => value, - std::result::Result::Err(_) => { - return std::option::Option::None; - }, - }; - let divisor = 10_f64.powi(decimals as i32); - if divisor <= 0.0_f64 { - return std::option::Option::None; - } - return std::option::Option::Some(raw_amount_f64 / divisor); -} - -fn find_transaction_meta_json( - transaction_json: &serde_json::Value, -) -> std::option::Option<&serde_json::Value> { - match json_object_field(transaction_json, "meta") { - std::option::Option::Some(value) => { - return std::option::Option::Some(value); - }, - std::option::Option::None => {}, - } - match json_object_field(transaction_json, "transaction") { - std::option::Option::Some(transaction_value) => { - match json_object_field(transaction_value, "meta") { - std::option::Option::Some(value) => { - return std::option::Option::Some(value); - }, - std::option::Option::None => {}, - } - }, - std::option::Option::None => {}, - } - match json_object_field(transaction_json, "result") { - std::option::Option::Some(result_value) => { - match json_object_field(result_value, "meta") { - std::option::Option::Some(value) => { - return std::option::Option::Some(value); - }, - std::option::Option::None => {}, - } - match json_object_field(result_value, "transaction") { - std::option::Option::Some(transaction_value) => { - match json_object_field(transaction_value, "meta") { - std::option::Option::Some(value) => { - return std::option::Option::Some(value); - }, - std::option::Option::None => {}, - } - }, - std::option::Option::None => {}, - } - }, - std::option::Option::None => {}, - } - return std::option::Option::None; -} - -fn json_object_field<'a>( - json_object_field: &'a serde_json::Value, - field_name: &str, -) -> std::option::Option<&'a serde_json::Value> { - match json_object_field { - serde_json::Value::Object(object) => match object.get(field_name) { - std::option::Option::Some(field_value) => { - return std::option::Option::Some(field_value); - }, - std::option::Option::None => return std::option::Option::None, - }, - _ => return std::option::Option::None, - } -} - -fn json_array_field<'a>( - value: &'a serde_json::Value, - field_name: &str, -) -> std::option::Option<&'a [serde_json::Value]> { - let field_value = match json_object_field(value, field_name) { - std::option::Option::Some(found) => found, - std::option::Option::None => { - return std::option::Option::None; - }, - }; - match field_value { - serde_json::Value::Array(values) => return std::option::Option::Some(values.as_slice()), - _ => return std::option::Option::None, - } -} - -fn json_string_field( - value: &serde_json::Value, - field_name: &str, -) -> std::option::Option { - let field_value = match json_object_field(value, field_name) { - std::option::Option::Some(found) => found, - std::option::Option::None => { - return std::option::Option::None; - }, - }; - match field_value { - serde_json::Value::String(text) => return std::option::Option::Some(text.clone()), - serde_json::Value::Number(number) => return std::option::Option::Some(number.to_string()), - _ => return std::option::Option::None, - } -} - -fn json_i64_field(value: &serde_json::Value, field_name: &str) -> std::option::Option { - let field_value = match json_object_field(value, field_name) { - std::option::Option::Some(found) => found, - std::option::Option::None => { - return std::option::Option::None; - }, - }; - match field_value { - serde_json::Value::Number(number) => return number.as_i64(), - serde_json::Value::String(text) => match text.parse::() { - std::result::Result::Ok(parsed) => return std::option::Option::Some(parsed), - std::result::Result::Err(_) => return std::option::Option::None, - }, - _ => return std::option::Option::None, - } -} - -fn json_f64_field(value: &serde_json::Value, field_name: &str) -> std::option::Option { - let field_value = match json_object_field(value, field_name) { - std::option::Option::Some(found) => found, - std::option::Option::None => { - return std::option::Option::None; - }, - }; - match field_value { - serde_json::Value::Number(number) => return number.as_f64(), - serde_json::Value::String(text) => match text.parse::() { - std::result::Result::Ok(parsed) => return std::option::Option::Some(parsed), - std::result::Result::Err(_) => return std::option::Option::None, - }, - _ => return std::option::Option::None, - } -} - -fn resolve_pump_swap_trade_amounts_from_pool_balance_deltas( - meta_json: std::option::Option<&str>, - pool_owner: &str, - base_mint: &str, - quote_mint: &str, - event_kind: &str, - base_token_decimals: std::option::Option, - quote_token_decimals: std::option::Option, -) -> Result { - let meta_json = match meta_json { - Some(meta_json) => meta_json, - None => return Ok(PumpSwapPoolBalanceDeltaResolution::MissingData), - }; - let pool_owner = pool_owner.trim(); - let base_mint = base_mint.trim(); - let quote_mint = quote_mint.trim(); - if pool_owner.is_empty() || base_mint.is_empty() || quote_mint.is_empty() { - return Ok(PumpSwapPoolBalanceDeltaResolution::MissingData); - } - if base_mint == quote_mint { - return Ok(PumpSwapPoolBalanceDeltaResolution::MissingData); - } - let meta_value_result = serde_json::from_str::(meta_json); - let meta_value = match meta_value_result { - Ok(meta_value) => meta_value, - Err(error) => { - return Err(crate::Error::Json(format!( - "cannot parse meta_json for pump_swap pool-owner balance delta extraction: {}", - error - ))); - }, - }; - let base_pre = sum_token_balance_raw_amount_by_owner_and_mint( - &meta_value, - "preTokenBalances", - pool_owner, - base_mint, - ); - let base_post = sum_token_balance_raw_amount_by_owner_and_mint( - &meta_value, - "postTokenBalances", - pool_owner, - base_mint, - ); - let quote_pre = sum_token_balance_raw_amount_by_owner_and_mint( - &meta_value, - "preTokenBalances", - pool_owner, - quote_mint, - ); - let quote_post = sum_token_balance_raw_amount_by_owner_and_mint( - &meta_value, - "postTokenBalances", - pool_owner, - quote_mint, - ); - if base_pre.is_none() && base_post.is_none() { - return Ok(PumpSwapPoolBalanceDeltaResolution::MissingData); - } - if quote_pre.is_none() && quote_post.is_none() { - return Ok(PumpSwapPoolBalanceDeltaResolution::MissingData); - } - let base_pre_raw = token_balance_raw_amount_value(base_pre.as_ref()); - let base_post_raw = token_balance_raw_amount_value(base_post.as_ref()); - let quote_pre_raw = token_balance_raw_amount_value(quote_pre.as_ref()); - let quote_post_raw = token_balance_raw_amount_value(quote_post.as_ref()); - let base_delta = base_post_raw - base_pre_raw; - let quote_delta = quote_post_raw - quote_pre_raw; - if base_delta == 0 || quote_delta == 0 { - return Ok(PumpSwapPoolBalanceDeltaResolution::MissingData); - } - let pool_delta_side = if base_delta < 0 && quote_delta > 0 { - crate::SwapTradeSide::BuyBase - } else if base_delta > 0 && quote_delta < 0 { - crate::SwapTradeSide::SellBase - } else { - return Ok(PumpSwapPoolBalanceDeltaResolution::MissingData); - }; - let event_side = trade_side_from_event_kind(event_kind); - if event_side != pool_delta_side { - return Ok(PumpSwapPoolBalanceDeltaResolution::DirectionMismatch); - } - let base_amount_abs = i128_abs(base_delta); - let quote_amount_abs = i128_abs(quote_delta); - if base_amount_abs <= 0 || quote_amount_abs <= 0 { - return Ok(PumpSwapPoolBalanceDeltaResolution::MissingData); - } - let base_decimals = first_some_u8( - base_token_decimals, - first_token_balance_decimals(base_pre.as_ref(), base_post.as_ref()), - ); - let quote_decimals = first_some_u8( - quote_token_decimals, - first_token_balance_decimals(quote_pre.as_ref(), quote_post.as_ref()), - ); - let price_quote_per_base = compute_price_quote_per_base_from_i128_raw_amounts_with_decimals( - base_amount_abs, - quote_amount_abs, - base_decimals, - quote_decimals, - ); - let price_quote_per_base = match price_quote_per_base { - Some(price_quote_per_base) => price_quote_per_base, - None => return Ok(PumpSwapPoolBalanceDeltaResolution::MissingData), - }; - return Ok(PumpSwapPoolBalanceDeltaResolution::Matched(PumpSwapPoolBalanceDeltaAmounts { - base_amount_raw: base_amount_abs.to_string(), - quote_amount_raw: quote_amount_abs.to_string(), - price_quote_per_base, - })); -} - -fn sum_token_balance_raw_amount_by_owner_and_mint( - meta_value: &serde_json::Value, - field_name: &str, - owner: &str, - mint: &str, -) -> std::option::Option { - let balances_value = match json_array_field(meta_value, field_name) { - std::option::Option::Some(balances_value) => balances_value, - std::option::Option::None => return std::option::Option::None, - }; - let mut total = 0_i128; - let mut found = false; - let mut decimals = std::option::Option::None; - for balance in balances_value { - let balance_mint = match json_string_field(balance, "mint") { - std::option::Option::Some(balance_mint) => balance_mint, - std::option::Option::None => continue, - }; - if balance_mint != mint { - continue; - } - let balance_owner = match json_string_field(balance, "owner") { - std::option::Option::Some(balance_owner) => balance_owner, - std::option::Option::None => continue, - }; - if balance_owner != owner { - continue; - } - let ui_token_amount = match json_object_field(balance, "uiTokenAmount") { - std::option::Option::Some(ui_token_amount) => ui_token_amount, - std::option::Option::None => continue, - }; - let amount_text = match json_string_field(ui_token_amount, "amount") { - std::option::Option::Some(amount_text) => amount_text, - std::option::Option::None => continue, - }; - let amount_result = amount_text.parse::(); - let amount = match amount_result { - std::result::Result::Ok(amount) => amount, - std::result::Result::Err(_) => continue, - }; - total += amount; - found = true; - if decimals.is_none() { - let decimals_i64 = match json_i64_field(ui_token_amount, "decimals") { - std::option::Option::Some(decimals_i64) => decimals_i64, - std::option::Option::None => continue, - }; - let decimals_u8 = match u8::try_from(decimals_i64) { - std::result::Result::Ok(decimals_u8) => decimals_u8, - std::result::Result::Err(_) => continue, - }; - decimals = std::option::Option::Some(decimals_u8); - } - } - if !found { - return std::option::Option::None; - } - return std::option::Option::Some(TokenBalanceRawAmount { raw_amount: total, decimals }); -} - -fn token_balance_raw_amount_value(value: std::option::Option<&TokenBalanceRawAmount>) -> i128 { - match value { - std::option::Option::Some(value) => return value.raw_amount, - std::option::Option::None => return 0_i128, - } -} - -fn first_token_balance_decimals( - left: std::option::Option<&TokenBalanceRawAmount>, - right: std::option::Option<&TokenBalanceRawAmount>, -) -> std::option::Option { - if let std::option::Option::Some(left) = left { - if left.decimals.is_some() { - return left.decimals; - } - } - if let std::option::Option::Some(right) = right { - return right.decimals; - } - return std::option::Option::None; -} - -fn first_some_u8( - left: std::option::Option, - right: std::option::Option, -) -> std::option::Option { - match left { - std::option::Option::Some(left) => return std::option::Option::Some(left), - std::option::Option::None => return right, - } -} - -fn trade_side_from_event_kind(event_kind: &str) -> crate::SwapTradeSide { - if event_kind.ends_with(".buy") { - return crate::SwapTradeSide::BuyBase; - } - if event_kind.ends_with(".sell") { - return crate::SwapTradeSide::SellBase; - } - return crate::SwapTradeSide::Unknown; -} - -fn i128_abs(i128_abs: i128) -> i128 { - if i128_abs < 0_i128 { - return -i128_abs; - } - return i128_abs; -} - -fn compute_price_quote_per_base_from_i128_raw_amounts_with_decimals( - base_amount_raw: i128, - quote_amount_raw: i128, - base_decimals: std::option::Option, - quote_decimals: std::option::Option, -) -> std::option::Option { - if base_amount_raw <= 0_i128 || quote_amount_raw <= 0_i128 { - return std::option::Option::None; - } - let base_decimals = match base_decimals { - std::option::Option::Some(base_decimals) => base_decimals, - std::option::Option::None => return std::option::Option::None, - }; - let quote_decimals = match quote_decimals { - std::option::Option::Some(quote_decimals) => quote_decimals, - std::option::Option::None => return std::option::Option::None, - }; - let base_scale = 10_f64.powi(i32::from(base_decimals)); - let quote_scale = 10_f64.powi(i32::from(quote_decimals)); - if !base_scale.is_finite() || !quote_scale.is_finite() { - return std::option::Option::None; - } - if base_scale <= 0.0_f64 || quote_scale <= 0.0_f64 { - return std::option::Option::None; - } - let base_amount = base_amount_raw as f64 / base_scale; - let quote_amount = quote_amount_raw as f64 / quote_scale; - if !base_amount.is_finite() || !quote_amount.is_finite() { - return std::option::Option::None; - } - if base_amount <= 0.0_f64 || quote_amount <= 0.0_f64 { - return std::option::Option::None; - } - let price_quote_per_base = quote_amount / base_amount; - if !price_quote_per_base.is_finite() { - return std::option::Option::None; - } - if price_quote_per_base <= 0.0_f64 { - return std::option::Option::None; - } - return std::option::Option::Some(price_quote_per_base); -} - #[cfg(test)] mod tests { async fn make_database() -> std::sync::Arc { @@ -2624,423 +370,4 @@ mod tests { }; assert_eq!(pair_metric.trade_count, 1); } - - #[test] - fn is_priced_trade_event_rejects_unpriced_values() { - let result = super::is_priced_trade_event(None, Some("2500"), Some(2.5)); - assert!(!result); - let result = super::is_priced_trade_event(Some("1000"), None, Some(2.5)); - assert!(!result); - let result = super::is_priced_trade_event(Some("1000"), Some("2500"), None); - assert!(!result); - let result = super::is_priced_trade_event(Some("0"), Some("2500"), Some(2.5)); - assert!(!result); - let result = super::is_priced_trade_event(Some("1000"), Some("0"), Some(2.5)); - assert!(!result); - let result = super::is_priced_trade_event(Some("-1"), Some("2500"), Some(2.5)); - assert!(!result); - let result = super::is_priced_trade_event(Some("1000"), Some("-1"), Some(2.5)); - assert!(!result); - let result = super::is_priced_trade_event(Some("abc"), Some("2500"), Some(2.5)); - assert!(!result); - let result = super::is_priced_trade_event(Some("1000"), Some("abc"), Some(2.5)); - assert!(!result); - let result = super::is_priced_trade_event(Some("1000"), Some("2500"), Some(0.0)); - assert!(!result); - let result = super::is_priced_trade_event(Some("1000"), Some("2500"), Some(f64::NAN)); - assert!(!result); - } - - #[test] - fn pump_swap_balance_delta_amounts_are_recovered_from_meta() { - let transaction_json = serde_json::json!({ - "meta": { - "preTokenBalances": [ - { - "accountIndex": 1, - "mint": "BASE111", - "owner": "owner_a", - "uiTokenAmount": { - "uiAmount": 10.0, - "uiAmountString": "10", - "decimals": 6, - "amount": "10000000" - } - }, - { - "accountIndex": 2, - "mint": "QUOTE111", - "owner": "owner_b", - "uiTokenAmount": { - "uiAmount": 2.0, - "uiAmountString": "2", - "decimals": 9, - "amount": "2000000000" - } - } - ], - "postTokenBalances": [ - { - "accountIndex": 1, - "mint": "BASE111", - "owner": "owner_a", - "uiTokenAmount": { - "uiAmount": 15.0, - "uiAmountString": "15", - "decimals": 6, - "amount": "15000000" - } - }, - { - "accountIndex": 2, - "mint": "QUOTE111", - "owner": "owner_b", - "uiTokenAmount": { - "uiAmount": 1.5, - "uiAmountString": "1.5", - "decimals": 9, - "amount": "1500000000" - } - } - ] - } - }); - let amounts = match super::try_build_pump_swap_trade_amounts_from_token_balance_deltas( - &transaction_json, - "BASE111", - "QUOTE111", - ) { - std::option::Option::Some(value) => value, - std::option::Option::None => { - panic!("expected pump_swap balance delta amounts"); - }, - }; - assert_eq!(amounts.base_amount, 5.0_f64); - assert_eq!(amounts.quote_amount, 0.5_f64); - assert_eq!(amounts.price_quote_per_base, 0.1_f64); - } - - #[test] - fn pump_swap_balance_delta_amounts_use_raw_amount_when_ui_amount_is_null() { - let transaction_json = serde_json::json!({ - "meta": { - "preTokenBalances": [ - { - "accountIndex": 1, - "mint": "BASE111", - "owner": "owner_a", - "uiTokenAmount": { - "uiAmount": null, - "decimals": 6, - "amount": "10000000" - } - }, - { - "accountIndex": 2, - "mint": "QUOTE111", - "owner": "owner_b", - "uiTokenAmount": { - "uiAmount": null, - "decimals": 9, - "amount": "2000000000" - } - } - ], - "postTokenBalances": [ - { - "accountIndex": 1, - "mint": "BASE111", - "owner": "owner_a", - "uiTokenAmount": { - "uiAmount": null, - "decimals": 6, - "amount": "15000000" - } - }, - { - "accountIndex": 2, - "mint": "QUOTE111", - "owner": "owner_b", - "uiTokenAmount": { - "uiAmount": null, - "decimals": 9, - "amount": "1500000000" - } - } - ] - } - }); - let amounts = match super::try_build_pump_swap_trade_amounts_from_token_balance_deltas( - &transaction_json, - "BASE111", - "QUOTE111", - ) { - std::option::Option::Some(value) => value, - std::option::Option::None => { - panic!("expected pump_swap balance delta amounts"); - }, - }; - assert_eq!(amounts.base_amount, 5.0_f64); - assert_eq!(amounts.quote_amount, 0.5_f64); - assert_eq!(amounts.price_quote_per_base, 0.1_f64); - } - - #[test] - fn pump_swap_balance_delta_amounts_return_none_when_base_delta_is_zero() { - let transaction_json = serde_json::json!({ - "meta": { - "preTokenBalances": [ - { - "accountIndex": 1, - "mint": "BASE111", - "owner": "owner_a", - "uiTokenAmount": { - "uiAmount": 10.0, - "uiAmountString": "10", - "decimals": 6, - "amount": "10000000" - } - }, - { - "accountIndex": 2, - "mint": "QUOTE111", - "owner": "owner_b", - "uiTokenAmount": { - "uiAmount": 2.0, - "uiAmountString": "2", - "decimals": 9, - "amount": "2000000000" - } - } - ], - "postTokenBalances": [ - { - "accountIndex": 1, - "mint": "BASE111", - "owner": "owner_a", - "uiTokenAmount": { - "uiAmount": 10.0, - "uiAmountString": "10", - "decimals": 6, - "amount": "10000000" - } - }, - { - "accountIndex": 2, - "mint": "QUOTE111", - "owner": "owner_b", - "uiTokenAmount": { - "uiAmount": 1.5, - "uiAmountString": "1.5", - "decimals": 9, - "amount": "1500000000" - } - } - ] - } - }); - let amounts = super::try_build_pump_swap_trade_amounts_from_token_balance_deltas( - &transaction_json, - "BASE111", - "QUOTE111", - ); - assert_eq!(amounts, std::option::Option::None); - } - - #[test] - fn pump_swap_pool_owner_delta_accepts_buy_direction() { - let meta_json = serde_json::json!({ - "preTokenBalances": [ - { - "accountIndex": 1, - "mint": "BASE111", - "owner": "Pool111", - "uiTokenAmount": { - "amount": "7967771862", - "decimals": 9 - } - }, - { - "accountIndex": 2, - "mint": "QUOTE111", - "owner": "Pool111", - "uiTokenAmount": { - "amount": "8970607", - "decimals": 9 - } - } - ], - "postTokenBalances": [ - { - "accountIndex": 1, - "mint": "BASE111", - "owner": "Pool111", - "uiTokenAmount": { - "amount": "7573340642", - "decimals": 9 - } - }, - { - "accountIndex": 2, - "mint": "QUOTE111", - "owner": "Pool111", - "uiTokenAmount": { - "amount": "9438980", - "decimals": 9 - } - } - ] - }); - let meta_json_text = meta_json.to_string(); - let result = super::resolve_pump_swap_trade_amounts_from_pool_balance_deltas( - Some(meta_json_text.as_str()), - "Pool111", - "BASE111", - "QUOTE111", - "pump_swap.buy", - Some(9), - Some(9), - ); - let resolution = match result { - Ok(resolution) => resolution, - Err(error) => panic!("pump_swap delta resolution must succeed: {}", error), - }; - match resolution { - super::PumpSwapPoolBalanceDeltaResolution::Matched(amounts) => { - assert_eq!(amounts.base_amount_raw, "394431220"); - assert_eq!(amounts.quote_amount_raw, "468373"); - assert!(amounts.price_quote_per_base > 0.00118); - assert!(amounts.price_quote_per_base < 0.00119); - }, - _ => panic!("expected matched pump_swap buy delta"), - } - } - - #[test] - fn pump_swap_pool_owner_delta_rejects_wrong_direction() { - let meta_json = serde_json::json!({ - "preTokenBalances": [ - { - "accountIndex": 1, - "mint": "BASE111", - "owner": "Pool111", - "uiTokenAmount": { - "amount": "7967771862", - "decimals": 9 - } - }, - { - "accountIndex": 2, - "mint": "QUOTE111", - "owner": "Pool111", - "uiTokenAmount": { - "amount": "8970607", - "decimals": 9 - } - } - ], - "postTokenBalances": [ - { - "accountIndex": 1, - "mint": "BASE111", - "owner": "Pool111", - "uiTokenAmount": { - "amount": "7573340642", - "decimals": 9 - } - }, - { - "accountIndex": 2, - "mint": "QUOTE111", - "owner": "Pool111", - "uiTokenAmount": { - "amount": "9438980", - "decimals": 9 - } - } - ] - }); - let meta_json_text = meta_json.to_string(); - let result = super::resolve_pump_swap_trade_amounts_from_pool_balance_deltas( - Some(meta_json_text.as_str()), - "Pool111", - "BASE111", - "QUOTE111", - "pump_swap.sell", - Some(9), - Some(9), - ); - let resolution = match result { - Ok(resolution) => resolution, - Err(error) => panic!("pump_swap delta resolution must succeed: {}", error), - }; - match resolution { - super::PumpSwapPoolBalanceDeltaResolution::DirectionMismatch => {}, - _ => panic!("expected pump_swap direction mismatch"), - } - } - - #[test] - fn pump_swap_pool_owner_delta_ignores_non_pool_owner() { - let meta_json = serde_json::json!({ - "preTokenBalances": [ - { - "accountIndex": 1, - "mint": "BASE111", - "owner": "OtherOwner111", - "uiTokenAmount": { - "amount": "1000", - "decimals": 6 - } - }, - { - "accountIndex": 2, - "mint": "QUOTE111", - "owner": "OtherOwner111", - "uiTokenAmount": { - "amount": "2000", - "decimals": 6 - } - } - ], - "postTokenBalances": [ - { - "accountIndex": 1, - "mint": "BASE111", - "owner": "OtherOwner111", - "uiTokenAmount": { - "amount": "900", - "decimals": 6 - } - }, - { - "accountIndex": 2, - "mint": "QUOTE111", - "owner": "OtherOwner111", - "uiTokenAmount": { - "amount": "2100", - "decimals": 6 - } - } - ] - }); - let meta_json_text = meta_json.to_string(); - let result = super::resolve_pump_swap_trade_amounts_from_pool_balance_deltas( - Some(meta_json_text.as_str()), - "Pool111", - "BASE111", - "QUOTE111", - "pump_swap.buy", - Some(6), - Some(6), - ); - let resolution = match result { - Ok(resolution) => resolution, - Err(error) => panic!("pump_swap delta resolution must succeed: {}", error), - }; - match resolution { - super::PumpSwapPoolBalanceDeltaResolution::MissingData => {}, - _ => panic!("expected missing data for non-pool owner"), - } - } } diff --git a/kb_lib/src/trade_aggregation_context.rs b/kb_lib/src/trade_aggregation_context.rs new file mode 100644 index 0000000..901d5d6 --- /dev/null +++ b/kb_lib/src/trade_aggregation_context.rs @@ -0,0 +1,216 @@ +// file: kb_lib/src/trade_aggregation_context.rs + +//! Database context loading for trade aggregation. +//! +//! This module resolves the normalized database context required by +//! `TradeAggregationService`: transaction, decoded events, pool, pair, +//! base/quote token metadata and pool token vault addresses. + +/// Transaction-level context used by trade aggregation. +pub(crate) struct TradeAggregationTransactionContext { + /// Persisted transaction row. + pub(crate) transaction: crate::ChainTransactionDto, + /// Internal transaction id. + pub(crate) transaction_id: i64, + /// Decoded DEX events attached to the transaction. + pub(crate) decoded_events: std::vec::Vec, +} + +/// Decoded-event-level context used by trade aggregation. +pub(crate) struct TradeAggregationDecodedEventContext { + /// Internal decoded event id. + pub(crate) decoded_event_id: i64, + /// Existing trade event, when this decoded event was already materialized. + pub(crate) existing_trade_event: std::option::Option, + /// Pool account address from the decoded event. + pub(crate) pool_address: std::string::String, + /// Persisted pool row. + pub(crate) pool: crate::PoolDto, + /// Internal pool id. + pub(crate) pool_id: i64, + /// Persisted pair row. + pub(crate) pair: crate::PairDto, + /// Internal pair id. + pub(crate) pair_id: i64, + /// Base token mint, when the token row exists. + pub(crate) base_token_mint: std::option::Option, + /// Base token decimals, when known. + pub(crate) base_token_decimals: std::option::Option, + /// Quote token mint, when the token row exists. + pub(crate) quote_token_mint: std::option::Option, + /// Quote token decimals, when known. + pub(crate) quote_token_decimals: std::option::Option, + /// Base token vault address, when known. + pub(crate) base_vault_address: std::option::Option, + /// Quote token vault address, when known. + pub(crate) quote_vault_address: std::option::Option, +} + +/// Loads a transaction and its decoded DEX events from one signature. +pub(crate) async fn load_trade_aggregation_transaction_context( + database: &crate::Database, + signature: &str, +) -> Result { + let transaction_result = + crate::query_chain_transactions_get_by_signature(database, signature).await; + let transaction_option = match transaction_result { + Ok(transaction_option) => transaction_option, + Err(error) => return Err(error), + }; + let transaction = match transaction_option { + Some(transaction) => transaction, + None => { + return Err(crate::Error::InvalidState(format!( + "cannot aggregate trades for unknown transaction '{}'", + signature + ))); + }, + }; + let transaction_id = match transaction.id { + Some(transaction_id) => transaction_id, + None => { + return Err(crate::Error::InvalidState(format!( + "transaction '{}' has no internal id", + signature + ))); + }, + }; + let decoded_events_result = + crate::query_dex_decoded_events_list_by_transaction_id(database, transaction_id).await; + let decoded_events = match decoded_events_result { + Ok(decoded_events) => decoded_events, + Err(error) => return Err(error), + }; + return Ok(crate::trade_aggregation_context::TradeAggregationTransactionContext { + transaction, + transaction_id, + decoded_events, + }); +} + +/// Loads the normalized DB context for one decoded event. +/// +/// Returns `Ok(None)` when the decoded event is not materializable yet: +/// missing pool account, pool row or pair row. +pub(crate) async fn load_trade_aggregation_decoded_event_context( + database: &crate::Database, + decoded_event: &crate::DexDecodedEventDto, +) -> Result< + std::option::Option, + crate::Error, +> { + let decoded_event_id = match decoded_event.id { + Some(decoded_event_id) => decoded_event_id, + None => { + return Err(crate::Error::InvalidState("decoded event has no internal id".to_string())); + }, + }; + let existing_trade_result = + crate::query_trade_events_get_by_decoded_event_id(database, decoded_event_id).await; + let existing_trade_event = match existing_trade_result { + Ok(existing_trade_event) => existing_trade_event, + Err(error) => return Err(error), + }; + let pool_address = match decoded_event.pool_account.clone() { + Some(pool_address) => pool_address, + None => return Ok(None), + }; + let pool_result = crate::query_pools_get_by_address(database, pool_address.as_str()).await; + let pool_option = match pool_result { + Ok(pool_option) => pool_option, + Err(error) => return Err(error), + }; + let pool = match pool_option { + Some(pool) => pool, + None => return Ok(None), + }; + let pool_id = match pool.id { + Some(pool_id) => pool_id, + None => { + return Err(crate::Error::InvalidState(format!( + "pool '{}' has no internal id", + pool.address + ))); + }, + }; + let pair_result = crate::query_pairs_get_by_pool_id(database, pool_id).await; + let pair_option = match pair_result { + Ok(pair_option) => pair_option, + Err(error) => return Err(error), + }; + let pair = match pair_option { + Some(pair) => pair, + None => return Ok(None), + }; + let pair_id = match pair.id { + Some(pair_id) => pair_id, + None => { + return Err(crate::Error::InvalidState(format!( + "pair for pool '{}' has no internal id", + pool_id + ))); + }, + }; + let base_token_result = crate::query_tokens_get_by_id(database, pair.base_token_id).await; + let (base_token_mint, base_token_decimals) = match base_token_result { + Ok(Some(token)) => (Some(token.mint), token.decimals), + Ok(None) => (None, None), + Err(error) => return Err(error), + }; + let quote_token_result = crate::query_tokens_get_by_id(database, pair.quote_token_id).await; + let (quote_token_mint, quote_token_decimals) = match quote_token_result { + Ok(Some(token)) => (Some(token.mint), token.decimals), + Ok(None) => (None, None), + Err(error) => return Err(error), + }; + let pool_tokens_result = crate::query_pool_tokens_list_by_pool_id(database, pool_id).await; + let pool_tokens = match pool_tokens_result { + Ok(pool_tokens) => pool_tokens, + Err(error) => return Err(error), + }; + let base_vault_address = + crate::trade_aggregation_context::find_pool_token_vault_address_by_token_id( + &pool_tokens, + pair.base_token_id, + ); + let quote_vault_address = + crate::trade_aggregation_context::find_pool_token_vault_address_by_token_id( + &pool_tokens, + pair.quote_token_id, + ); + return Ok(Some(crate::trade_aggregation_context::TradeAggregationDecodedEventContext { + decoded_event_id, + existing_trade_event, + pool_address, + pool, + pool_id, + pair, + pair_id, + base_token_mint, + base_token_decimals, + quote_token_mint, + quote_token_decimals, + base_vault_address, + quote_vault_address, + })); +} + +fn find_pool_token_vault_address_by_token_id( + pool_tokens: &[crate::PoolTokenDto], + token_id: i64, +) -> std::option::Option { + for pool_token in pool_tokens { + if pool_token.token_id != token_id { + continue; + } + let vault_address = match pool_token.vault_address.clone() { + Some(vault_address) => vault_address.trim().to_string(), + None => continue, + }; + if vault_address.is_empty() { + continue; + } + return Some(vault_address); + } + return None; +} diff --git a/kb_lib/src/trade_amount_resolution.rs b/kb_lib/src/trade_amount_resolution.rs new file mode 100644 index 0000000..070b952 --- /dev/null +++ b/kb_lib/src/trade_amount_resolution.rs @@ -0,0 +1,650 @@ +// file: kb_lib/src/trade_amount_resolution.rs + +//! Trade amount resolution orchestration. +//! +//! This module resolves base/quote raw amounts and quote/base price for one +//! decoded trade candidate by applying protocol-specific and generic fallback +//! strategies in deterministic order. + +/// Input context required to resolve trade amounts. +pub(crate) struct TradeAmountResolutionInput<'a> { + /// Database connection. + pub(crate) database: &'a crate::Database, + /// Persisted transaction row. + pub(crate) transaction: &'a crate::ChainTransactionDto, + /// Decoded DEX event row. + pub(crate) decoded_event: &'a crate::DexDecodedEventDto, + /// Decoded event payload. + pub(crate) payload: &'a serde_json::Value, + /// Pool account address. + pub(crate) pool_address: &'a str, + /// Base token mint, when known. + pub(crate) base_token_mint: std::option::Option<&'a str>, + /// Quote token mint, when known. + pub(crate) quote_token_mint: std::option::Option<&'a str>, + /// Base token decimals, when known. + pub(crate) base_token_decimals: std::option::Option, + /// Quote token decimals, when known. + pub(crate) quote_token_decimals: std::option::Option, + /// Base token vault address, when known. + pub(crate) base_vault_address: std::option::Option<&'a str>, + /// Quote token vault address, when known. + pub(crate) quote_vault_address: std::option::Option<&'a str>, +} + +/// Resolved raw trade amounts and quote/base price. +#[derive(Debug, Clone)] +pub(crate) struct TradeAmountResolution { + /// Base amount in raw token units. + pub(crate) base_amount_raw: std::option::Option, + /// Quote amount in raw token units. + pub(crate) quote_amount_raw: std::option::Option, + /// Quote/base price. + pub(crate) price_quote_per_base: std::option::Option, +} + +/// Resolves trade amounts from payload and protocol-specific fallbacks. +pub(crate) async fn resolve_trade_amounts( + input: &crate::trade_amount_resolution::TradeAmountResolutionInput<'_>, +) -> Result { + let mut base_amount_raw = crate::trade_amount_resolution::extract_amount_string( + input.payload, + &["baseAmountRaw", "base_amount_raw", "baseAmount", "amountBase", "amountInBase"], + ); + let mut quote_amount_raw = crate::trade_amount_resolution::extract_amount_string( + input.payload, + &[ + "quoteAmountRaw", + "quote_amount_raw", + "quoteAmount", + "amountQuote", + "amountOutQuote", + ], + ); + let mut price_quote_per_base = None; + if input.decoded_event.event_kind.starts_with("pump_swap.") + && (base_amount_raw.is_none() + || quote_amount_raw.is_none() + || price_quote_per_base.is_none()) + { + let resolution_result = crate::trade_amount_resolution::apply_pump_swap_amount_fallbacks( + input, + &mut base_amount_raw, + &mut quote_amount_raw, + &mut price_quote_per_base, + ) + .await; + if let Err(error) = resolution_result { + return Err(error); + } + } + if input.decoded_event.event_kind.starts_with("pump_fun.") + && (base_amount_raw.is_none() + || quote_amount_raw.is_none() + || price_quote_per_base.is_none()) + { + let resolution_result = crate::trade_amount_resolution::apply_pump_fun_amount_fallback( + input, + &mut base_amount_raw, + &mut quote_amount_raw, + &mut price_quote_per_base, + ); + if let Err(error) = resolution_result { + return Err(error); + } + } + if (input.decoded_event.event_kind.starts_with("raydium_cpmm.") + || input.decoded_event.event_kind.starts_with("raydium_clmm.")) + && (base_amount_raw.is_none() + || quote_amount_raw.is_none() + || price_quote_per_base.is_none()) + { + let resolution_result = + crate::trade_amount_resolution::apply_raydium_instruction_amount_fallback( + input, + &mut base_amount_raw, + &mut quote_amount_raw, + &mut price_quote_per_base, + ) + .await; + if let Err(error) = resolution_result { + return Err(error); + } + } + if input.decoded_event.event_kind.starts_with("raydium_cpmm.") + && (base_amount_raw.is_none() || quote_amount_raw.is_none()) + { + let resolution_result = crate::trade_amount_resolution::apply_vault_balance_delta_fallback( + input, + input.base_vault_address, + input.quote_vault_address, + &mut base_amount_raw, + &mut quote_amount_raw, + &mut price_quote_per_base, + ); + if let Err(error) = resolution_result { + return Err(error); + } + } + if input.decoded_event.event_kind.starts_with("raydium_clmm.") + && (base_amount_raw.is_none() || quote_amount_raw.is_none()) + { + let resolution_result = crate::trade_amount_resolution::apply_vault_balance_delta_fallback( + input, + input.base_vault_address, + input.quote_vault_address, + &mut base_amount_raw, + &mut quote_amount_raw, + &mut price_quote_per_base, + ); + if let Err(error) = resolution_result { + return Err(error); + } + } + if price_quote_per_base.is_none() { + price_quote_per_base = + crate::trade_metric_update::compute_price_quote_per_base_from_raw_amounts_with_decimals( + base_amount_raw.as_deref(), + quote_amount_raw.as_deref(), + input.base_token_decimals, + input.quote_token_decimals, + ); + } + if price_quote_per_base.is_none() { + price_quote_per_base = + crate::trade_solana_amounts::compute_price_quote_per_base_with_decimals( + input.transaction.meta_json.as_deref(), + input.transaction.transaction_json.as_str(), + input.base_vault_address, + input.quote_vault_address, + ); + } + if price_quote_per_base.is_none() { + price_quote_per_base = + crate::trade_metric_update::compute_price_quote_per_base_from_raw_amounts( + base_amount_raw.as_deref(), + quote_amount_raw.as_deref(), + ); + } + return Ok(crate::trade_amount_resolution::TradeAmountResolution { + base_amount_raw, + quote_amount_raw, + price_quote_per_base, + }); +} + +async fn apply_pump_swap_amount_fallbacks( + input: &crate::trade_amount_resolution::TradeAmountResolutionInput<'_>, + base_amount_raw: &mut std::option::Option, + quote_amount_raw: &mut std::option::Option, + price_quote_per_base: &mut std::option::Option, +) -> Result<(), crate::Error> { + let pool_owner_result = match (input.base_token_mint, input.quote_token_mint) { + (Some(base_mint), Some(quote_mint)) => { + crate::trade_pump_swap_amounts::resolve_pump_swap_trade_amounts_from_pool_balance_deltas( + input.transaction.meta_json.as_deref(), + input.pool_address, + base_mint, + quote_mint, + input.decoded_event.event_kind.as_str(), + input.base_token_decimals, + input.quote_token_decimals, + ) + }, + _ => Ok(crate::trade_pump_swap_amounts::PumpSwapPoolBalanceDeltaResolution::MissingData), + }; + let pool_owner_resolution = match pool_owner_result { + Ok(pool_owner_resolution) => pool_owner_resolution, + Err(error) => return Err(error), + }; + let pool_owner_resolution_label = pool_owner_resolution.as_label(); + tracing::debug!( + event_kind = %input.decoded_event.event_kind, + pool_account = ?input.decoded_event.pool_account, + decoded_event_id = ?input.decoded_event.id, + transaction_signature = %input.transaction.signature, + base_mint = ?input.base_token_mint, + quote_mint = ?input.quote_token_mint, + pool_owner_resolution = %pool_owner_resolution_label, + "pump_swap pool-owner delta resolution result" + ); + match pool_owner_resolution { + crate::trade_pump_swap_amounts::PumpSwapPoolBalanceDeltaResolution::Matched(amounts) => { + *base_amount_raw = Some(amounts.base_amount_raw); + *quote_amount_raw = Some(amounts.quote_amount_raw); + *price_quote_per_base = Some(amounts.price_quote_per_base); + tracing::debug!( + event_kind = %input.decoded_event.event_kind, + pool_account = ?input.decoded_event.pool_account, + decoded_event_id = ?input.decoded_event.id, + base_mint = ?input.base_token_mint, + quote_mint = ?input.quote_token_mint, + base_amount_raw = ?base_amount_raw, + quote_amount_raw = ?quote_amount_raw, + price_quote_per_base = ?price_quote_per_base, + "pump_swap trade amounts recovered from pool-owner token balance deltas" + ); + }, + crate::trade_pump_swap_amounts::PumpSwapPoolBalanceDeltaResolution::DirectionMismatch => { + tracing::debug!( + event_kind = %input.decoded_event.event_kind, + pool_account = ?input.decoded_event.pool_account, + decoded_event_id = ?input.decoded_event.id, + transaction_signature = %input.transaction.signature, + "pump_swap pool-owner full-transaction delta direction mismatch; continuing with instruction-scoped fallbacks" + ); + }, + crate::trade_pump_swap_amounts::PumpSwapPoolBalanceDeltaResolution::MissingData => {}, + } + let decoded_instruction_index_result = + crate::trade_amount_resolution::load_decoded_instruction_index( + input.database, + input.decoded_event, + ) + .await; + let decoded_instruction_index = match decoded_instruction_index_result { + Ok(decoded_instruction_index) => decoded_instruction_index, + Err(error) => return Err(error), + }; + let payload_user_base_token_account = + crate::trade_amount_resolution::extract_string_by_candidate_keys( + input.payload, + &["userBaseTokenAccount", "user_base_token_account"], + ); + let payload_user_quote_token_account = + crate::trade_amount_resolution::extract_string_by_candidate_keys( + input.payload, + &["userQuoteTokenAccount", "user_quote_token_account"], + ); + let payload_pool_base_token_account = + crate::trade_amount_resolution::extract_string_by_candidate_keys( + input.payload, + &["poolBaseTokenAccount", "pool_base_token_account"], + ); + let payload_pool_quote_token_account = + crate::trade_amount_resolution::extract_string_by_candidate_keys( + input.payload, + &["poolQuoteTokenAccount", "pool_quote_token_account"], + ); + let effective_base_vault_address = match input.base_vault_address { + Some(base_vault_address) => Some(base_vault_address), + None => payload_pool_base_token_account.as_deref(), + }; + let effective_quote_vault_address = match input.quote_vault_address { + Some(quote_vault_address) => Some(quote_vault_address), + None => payload_pool_quote_token_account.as_deref(), + }; + let (input_vault_address, output_vault_address, input_token_account, output_token_account) = + if input.decoded_event.event_kind.ends_with(".buy") { + ( + effective_quote_vault_address, + effective_base_vault_address, + payload_user_quote_token_account.as_deref(), + payload_user_base_token_account.as_deref(), + ) + } else if input.decoded_event.event_kind.ends_with(".sell") { + ( + effective_base_vault_address, + effective_quote_vault_address, + payload_user_base_token_account.as_deref(), + payload_user_quote_token_account.as_deref(), + ) + } else { + (None, None, None, None) + }; + let inferred_result = + crate::trade_solana_amounts::extract_trade_amounts_from_instruction_token_transfers( + input.transaction.meta_json.as_deref(), + decoded_instruction_index, + input_vault_address, + output_vault_address, + input_token_account, + output_token_account, + effective_base_vault_address, + effective_quote_vault_address, + ); + let inferred = match inferred_result { + Ok(inferred) => inferred, + Err(error) => return Err(error), + }; + if base_amount_raw.is_none() { + *base_amount_raw = inferred.0; + } + if quote_amount_raw.is_none() { + *quote_amount_raw = inferred.1; + } + if price_quote_per_base.is_none() { + *price_quote_per_base = inferred.2; + } + if base_amount_raw.is_none() || quote_amount_raw.is_none() { + let fallback_result = + crate::trade_solana_amounts::extract_trade_amounts_from_vault_balance_deltas( + input.transaction.transaction_json.as_str(), + input.transaction.meta_json.as_deref(), + effective_base_vault_address, + effective_quote_vault_address, + ); + let fallback = match fallback_result { + Ok(fallback) => fallback, + Err(error) => return Err(error), + }; + if base_amount_raw.is_none() { + *base_amount_raw = fallback.0; + } + if quote_amount_raw.is_none() { + *quote_amount_raw = fallback.1; + } + if price_quote_per_base.is_none() { + *price_quote_per_base = fallback.2; + } + } + + if base_amount_raw.is_none() || quote_amount_raw.is_none() || price_quote_per_base.is_none() { + let transaction_value_result = + crate::trade_pump_swap_amounts::build_transaction_value_with_meta_json( + input.transaction.transaction_json.as_str(), + input.transaction.meta_json.as_deref(), + ); + let transaction_value = match transaction_value_result { + Ok(transaction_value) => transaction_value, + Err(error) => return Err(error), + }; + let fallback_amounts = match (input.base_token_mint, input.quote_token_mint) { + (Some(base_mint), Some(quote_mint)) => { + crate::trade_pump_swap_amounts::try_build_pump_swap_trade_amounts_from_token_balance_deltas( + &transaction_value, + base_mint, + quote_mint, + ) + }, + _ => None, + }; + if let Some(fallback_amounts) = fallback_amounts { + if base_amount_raw.is_none() { + *base_amount_raw = crate::trade_pump_swap_amounts::convert_ui_amount_to_raw_string( + fallback_amounts.base_amount, + input.base_token_decimals, + ); + } + if quote_amount_raw.is_none() { + *quote_amount_raw = crate::trade_pump_swap_amounts::convert_ui_amount_to_raw_string( + fallback_amounts.quote_amount, + input.quote_token_decimals, + ); + } + if price_quote_per_base.is_none() { + *price_quote_per_base = Some(fallback_amounts.price_quote_per_base); + } + tracing::debug!( + event_kind = %input.decoded_event.event_kind, + pool_account = ?input.decoded_event.pool_account, + decoded_event_id = ?input.decoded_event.id, + base_mint = ?input.base_token_mint, + quote_mint = ?input.quote_token_mint, + base_amount_raw = ?base_amount_raw, + quote_amount_raw = ?quote_amount_raw, + price_quote_per_base = ?price_quote_per_base, + "pump_swap trade amounts recovered from token balance deltas" + ); + } + } + return Ok(()); +} + +fn apply_pump_fun_amount_fallback( + input: &crate::trade_amount_resolution::TradeAmountResolutionInput<'_>, + base_amount_raw: &mut std::option::Option, + quote_amount_raw: &mut std::option::Option, + price_quote_per_base: &mut std::option::Option, +) -> Result<(), crate::Error> { + let inferred_result = crate::trade_solana_amounts::extract_pump_fun_amounts_from_transaction( + input.transaction.transaction_json.as_str(), + input.transaction.meta_json.as_deref(), + input.base_vault_address, + input.quote_vault_address, + ); + let inferred = match inferred_result { + Ok(inferred) => inferred, + Err(error) => return Err(error), + }; + if base_amount_raw.is_none() { + *base_amount_raw = inferred.0; + } + if quote_amount_raw.is_none() { + *quote_amount_raw = inferred.1; + } + if price_quote_per_base.is_none() { + *price_quote_per_base = inferred.2; + } + return Ok(()); +} + +async fn apply_raydium_instruction_amount_fallback( + input: &crate::trade_amount_resolution::TradeAmountResolutionInput<'_>, + base_amount_raw: &mut std::option::Option, + quote_amount_raw: &mut std::option::Option, + price_quote_per_base: &mut std::option::Option, +) -> Result<(), crate::Error> { + let decoded_instruction_index_result = + crate::trade_amount_resolution::load_decoded_instruction_index( + input.database, + input.decoded_event, + ) + .await; + let decoded_instruction_index = match decoded_instruction_index_result { + Ok(decoded_instruction_index) => decoded_instruction_index, + Err(error) => return Err(error), + }; + let payload_input_vault_address = + crate::trade_amount_resolution::extract_string_by_candidate_keys( + input.payload, + &["inputVault", "input_vault"], + ); + let payload_output_vault_address = + crate::trade_amount_resolution::extract_string_by_candidate_keys( + input.payload, + &["outputVault", "output_vault"], + ); + let payload_input_token_account = + crate::trade_amount_resolution::extract_string_by_candidate_keys( + input.payload, + &["inputTokenAccount", "input_token_account"], + ); + let payload_output_token_account = + crate::trade_amount_resolution::extract_string_by_candidate_keys( + input.payload, + &["outputTokenAccount", "output_token_account"], + ); + let payload_base_vault_address = + crate::trade_amount_resolution::extract_string_by_candidate_keys( + input.payload, + &["baseVault", "base_vault"], + ); + let payload_quote_vault_address = + crate::trade_amount_resolution::extract_string_by_candidate_keys( + input.payload, + &["quoteVault", "quote_vault"], + ); + let effective_base_vault_address = match input.base_vault_address { + Some(base_vault_address) => Some(base_vault_address), + None => payload_base_vault_address.as_deref(), + }; + let effective_quote_vault_address = match input.quote_vault_address { + Some(quote_vault_address) => Some(quote_vault_address), + None => payload_quote_vault_address.as_deref(), + }; + let inferred_result = + crate::trade_solana_amounts::extract_trade_amounts_from_instruction_token_transfers( + input.transaction.meta_json.as_deref(), + decoded_instruction_index, + payload_input_vault_address.as_deref(), + payload_output_vault_address.as_deref(), + payload_input_token_account.as_deref(), + payload_output_token_account.as_deref(), + effective_base_vault_address, + effective_quote_vault_address, + ); + let inferred = match inferred_result { + Ok(inferred) => inferred, + Err(error) => return Err(error), + }; + if base_amount_raw.is_none() { + *base_amount_raw = inferred.0; + } + if quote_amount_raw.is_none() { + *quote_amount_raw = inferred.1; + } + if price_quote_per_base.is_none() { + *price_quote_per_base = inferred.2; + } + return Ok(()); +} + +fn apply_vault_balance_delta_fallback( + input: &crate::trade_amount_resolution::TradeAmountResolutionInput<'_>, + base_vault_address: std::option::Option<&str>, + quote_vault_address: std::option::Option<&str>, + base_amount_raw: &mut std::option::Option, + quote_amount_raw: &mut std::option::Option, + price_quote_per_base: &mut std::option::Option, +) -> Result<(), crate::Error> { + let inferred_result = + crate::trade_solana_amounts::extract_trade_amounts_from_vault_balance_deltas( + input.transaction.transaction_json.as_str(), + input.transaction.meta_json.as_deref(), + base_vault_address, + quote_vault_address, + ); + let inferred = match inferred_result { + Ok(inferred) => inferred, + Err(error) => return Err(error), + }; + if base_amount_raw.is_none() { + *base_amount_raw = inferred.0; + } + if quote_amount_raw.is_none() { + *quote_amount_raw = inferred.1; + } + if price_quote_per_base.is_none() { + *price_quote_per_base = inferred.2; + } + return Ok(()); +} + +async fn load_decoded_instruction_index( + database: &crate::Database, + decoded_event: &crate::DexDecodedEventDto, +) -> Result, crate::Error> { + let instruction_id = match decoded_event.instruction_id { + Some(instruction_id) => instruction_id, + None => return Ok(None), + }; + let instruction_result = + crate::query_chain_instructions_get_by_id(database, instruction_id).await; + let instruction_option = match instruction_result { + Ok(instruction_option) => instruction_option, + Err(error) => return Err(error), + }; + match instruction_option { + Some(instruction) => return Ok(Some(instruction.instruction_index)), + None => return Ok(None), + } +} + +fn extract_amount_string( + payload: &serde_json::Value, + candidate_keys: &[&str], +) -> std::option::Option { + return crate::trade_amount_resolution::extract_scalar_as_string_by_candidate_keys( + payload, + candidate_keys, + ); +} + +fn extract_string_by_candidate_keys( + value: &serde_json::Value, + candidate_keys: &[&str], +) -> std::option::Option { + if let Some(object) = value.as_object() { + for candidate_key in candidate_keys { + let direct_option = object.get(*candidate_key); + if let Some(direct) = direct_option { + let direct_text_option = direct.as_str(); + if let Some(direct_text) = direct_text_option { + return Some(direct_text.to_string()); + } + } + } + for nested_value in object.values() { + let nested_result = crate::trade_amount_resolution::extract_string_by_candidate_keys( + nested_value, + candidate_keys, + ); + if nested_result.is_some() { + return nested_result; + } + } + return None; + } + if let Some(array) = value.as_array() { + for nested_value in array { + let nested_result = crate::trade_amount_resolution::extract_string_by_candidate_keys( + nested_value, + candidate_keys, + ); + if nested_result.is_some() { + return nested_result; + } + } + } + return None; +} + +fn extract_scalar_as_string_by_candidate_keys( + value: &serde_json::Value, + candidate_keys: &[&str], +) -> std::option::Option { + if let Some(object) = value.as_object() { + for candidate_key in candidate_keys { + let direct_option = object.get(*candidate_key); + if let Some(direct) = direct_option { + if let Some(text) = direct.as_str() { + return Some(text.to_string()); + } + if let Some(number) = direct.as_i64() { + return Some(number.to_string()); + } + if let Some(number) = direct.as_u64() { + return Some(number.to_string()); + } + if let Some(number) = direct.as_f64() { + return Some(number.to_string()); + } + } + } + for nested_value in object.values() { + let nested_result = + crate::trade_amount_resolution::extract_scalar_as_string_by_candidate_keys( + nested_value, + candidate_keys, + ); + if nested_result.is_some() { + return nested_result; + } + } + return None; + } + if let Some(array) = value.as_array() { + for nested_value in array { + let nested_result = + crate::trade_amount_resolution::extract_scalar_as_string_by_candidate_keys( + nested_value, + candidate_keys, + ); + if nested_result.is_some() { + return nested_result; + } + } + } + return None; +} diff --git a/kb_lib/src/trade_event_materialization.rs b/kb_lib/src/trade_event_materialization.rs new file mode 100644 index 0000000..ecebc39 --- /dev/null +++ b/kb_lib/src/trade_event_materialization.rs @@ -0,0 +1,230 @@ +// file: kb_lib/src/trade_event_materialization.rs + +//! Trade-event materialization. +//! +//! This module persists normalized trade events, updates pair metrics and +//! records the corresponding detection observation/signal. + +/// Input required to materialize one normalized trade event. +pub(crate) struct TradeEventMaterializationInput<'a> { + /// Database connection. + pub(crate) database: &'a crate::Database, + /// Detection persistence service used for observations and signals. + pub(crate) persistence: &'a crate::DetectionPersistenceService, + /// Persisted transaction row. + pub(crate) transaction: &'a crate::ChainTransactionDto, + /// Internal transaction id. + pub(crate) transaction_id: i64, + /// Decoded DEX event row. + pub(crate) decoded_event: &'a crate::DexDecodedEventDto, + /// Internal decoded event id. + pub(crate) decoded_event_id: i64, + /// Existing trade event, when the decoded event was already materialized. + pub(crate) existing_trade_event: std::option::Option, + /// Persisted pool row. + pub(crate) pool: &'a crate::PoolDto, + /// Internal pool id. + pub(crate) pool_id: i64, + /// Persisted pair row. + pub(crate) pair: &'a crate::PairDto, + /// Internal pair id. + pub(crate) pair_id: i64, + /// Trade side. + pub(crate) trade_side: crate::SwapTradeSide, + /// Resolved trade amounts. + pub(crate) amount_resolution: &'a crate::trade_amount_resolution::TradeAmountResolution, +} + +/// Persists one normalized trade event and updates pair-level metrics. +pub(crate) async fn materialize_trade_event( + input: crate::trade_event_materialization::TradeEventMaterializationInput<'_>, +) -> Result { + let base_amount_raw = input.amount_resolution.base_amount_raw.clone(); + let quote_amount_raw = input.amount_resolution.quote_amount_raw.clone(); + let price_quote_per_base = input.amount_resolution.price_quote_per_base; + let slot_i64 = crate::trade_metric_update::convert_slot_to_i64(input.transaction.slot); + let created_trade_event = input.existing_trade_event.is_none(); + let trade_event_dto = crate::TradeEventDto::new( + input.pool.dex_id, + input.pool_id, + input.pair_id, + input.transaction_id, + input.decoded_event_id, + input.transaction.signature.clone(), + slot_i64, + input.trade_side, + input.pair.base_token_id, + input.pair.quote_token_id, + base_amount_raw.clone(), + quote_amount_raw.clone(), + price_quote_per_base, + crate::ObservationSourceKind::Dex, + input.transaction.source_endpoint_name.clone(), + input.decoded_event.payload_json.clone(), + ); + tracing::debug!( + event_kind = %input.decoded_event.event_kind, + pool_account = ?input.decoded_event.pool_account, + decoded_event_id = ?input.decoded_event.id, + created_trade_event = created_trade_event, + "trade aggregation candidate" + ); + let trade_event_id_result = + crate::query_trade_events_upsert(input.database, &trade_event_dto).await; + let trade_event_id = match trade_event_id_result { + Ok(trade_event_id) => trade_event_id, + Err(error) => return Err(error), + }; + let pair_metric_id_result = crate::trade_event_materialization::upsert_pair_metric_for_trade( + input.database, + input.pair_id, + slot_i64, + input.transaction.signature.clone(), + input.trade_side, + base_amount_raw.clone(), + quote_amount_raw.clone(), + price_quote_per_base, + created_trade_event, + ) + .await; + let pair_metric_id = match pair_metric_id_result { + Ok(pair_metric_id) => pair_metric_id, + Err(error) => return Err(error), + }; + if created_trade_event { + let observation_result = + crate::trade_event_materialization::record_trade_aggregation_observation_and_signal( + input.persistence, + input.transaction, + input.pair_id, + input.pool_id, + trade_event_id, + input.trade_side, + base_amount_raw, + quote_amount_raw, + price_quote_per_base, + ) + .await; + if let Err(error) = observation_result { + return Err(error); + } + } + return Ok(crate::TradeAggregationResult { + trade_event_id, + pair_metric_id, + pair_id: input.pair_id, + pool_id: input.pool_id, + created_trade_event, + }); +} + +async fn upsert_pair_metric_for_trade( + database: &crate::Database, + pair_id: i64, + slot_i64: std::option::Option, + signature: std::string::String, + trade_side: crate::SwapTradeSide, + base_amount_raw: std::option::Option, + quote_amount_raw: std::option::Option, + price_quote_per_base: std::option::Option, + created_trade_event: bool, +) -> Result { + let pair_metric_result = crate::query_pair_metrics_get_by_pair_id(database, pair_id).await; + let pair_metric_option = match pair_metric_result { + Ok(pair_metric_option) => pair_metric_option, + Err(error) => return Err(error), + }; + if let Some(existing_metric) = pair_metric_option { + let existing_metric_id = match existing_metric.id { + Some(existing_metric_id) => existing_metric_id, + None => { + return Err(crate::Error::InvalidState( + "pair metric has no internal id".to_string(), + )); + }, + }; + if created_trade_event { + let mut updated_metric = existing_metric.clone(); + crate::trade_metric_update::apply_trade_to_pair_metric( + &mut updated_metric, + slot_i64, + signature, + trade_side, + base_amount_raw, + quote_amount_raw, + price_quote_per_base, + ); + let upsert_result = crate::query_pair_metrics_upsert(database, &updated_metric).await; + if let Err(error) = upsert_result { + return Err(error); + } + } + return Ok(existing_metric_id); + } + let mut new_metric = crate::PairMetricDto::new(pair_id); + crate::trade_metric_update::apply_trade_to_pair_metric( + &mut new_metric, + slot_i64, + signature, + trade_side, + base_amount_raw, + quote_amount_raw, + price_quote_per_base, + ); + let upsert_result = crate::query_pair_metrics_upsert(database, &new_metric).await; + match upsert_result { + Ok(pair_metric_id) => return Ok(pair_metric_id), + Err(error) => return Err(error), + } +} + +async fn record_trade_aggregation_observation_and_signal( + persistence: &crate::DetectionPersistenceService, + transaction: &crate::ChainTransactionDto, + pair_id: i64, + pool_id: i64, + trade_event_id: i64, + trade_side: crate::SwapTradeSide, + base_amount_raw: std::option::Option, + quote_amount_raw: std::option::Option, + price_quote_per_base: std::option::Option, +) -> Result<(), crate::Error> { + let payload = serde_json::json!({ + "pairId": pair_id, + "poolId": pool_id, + "tradeEventId": trade_event_id, + "tradeSide": format!("{:?}", trade_side), + "baseAmountRaw": base_amount_raw, + "quoteAmountRaw": quote_amount_raw, + "priceQuotePerBase": price_quote_per_base, + "transactionSignature": transaction.signature + }); + let observation_result = persistence + .record_observation(&crate::DetectionObservationInput::new( + "dex.trade_aggregation".to_string(), + crate::ObservationSourceKind::Dex, + transaction.source_endpoint_name.clone(), + transaction.signature.clone(), + transaction.slot, + payload.clone(), + )) + .await; + let observation_id = match observation_result { + Ok(observation_id) => observation_id, + Err(error) => return Err(error), + }; + let signal_result = persistence + .record_signal(&crate::DetectionSignalInput::new( + "signal.dex.trade_aggregation.recorded".to_string(), + crate::AnalysisSignalSeverity::Low, + transaction.signature.clone(), + Some(observation_id), + None, + payload, + )) + .await; + if let Err(error) = signal_result { + return Err(error); + } + return Ok(()); +} diff --git a/kb_lib/src/trade_metric_update.rs b/kb_lib/src/trade_metric_update.rs new file mode 100644 index 0000000..2359d0d --- /dev/null +++ b/kb_lib/src/trade_metric_update.rs @@ -0,0 +1,279 @@ +// file: kb_lib/src/trade_metric_update.rs + +//! Trade metric update and basic trade-pricing helpers. +//! +//! This module contains pure helpers used by trade aggregation: +//! pricing validation, raw amount accumulation and pair metric updates. + +/// Returns true when a decoded trade has enough positive values to be persisted. +pub(crate) fn is_priced_trade_event( + base_amount_raw: std::option::Option<&str>, + quote_amount_raw: std::option::Option<&str>, + price_quote_per_base: std::option::Option, +) -> bool { + let base_amount_raw = match base_amount_raw { + Some(base_amount_raw) => base_amount_raw.trim(), + None => return false, + }; + if base_amount_raw.is_empty() { + return false; + } + let base_amount_result = base_amount_raw.parse::(); + let base_amount = match base_amount_result { + Ok(base_amount) => base_amount, + Err(_) => return false, + }; + if base_amount <= 0 { + return false; + } + let quote_amount_raw = match quote_amount_raw { + Some(quote_amount_raw) => quote_amount_raw.trim(), + None => return false, + }; + if quote_amount_raw.is_empty() { + return false; + } + let quote_amount_result = quote_amount_raw.parse::(); + let quote_amount = match quote_amount_result { + Ok(quote_amount) => quote_amount, + Err(_) => return false, + }; + if quote_amount <= 0 { + return false; + } + let price = match price_quote_per_base { + Some(price) => price, + None => return false, + }; + if !price.is_finite() { + return false; + } + return price > 0.0; +} + +/// Converts an optional Solana slot to an optional signed database slot. +pub(crate) fn convert_slot_to_i64(slot: std::option::Option) -> std::option::Option { + match slot { + Some(slot) => match i64::try_from(slot) { + Ok(slot) => return Some(slot), + Err(_) => return None, + }, + None => return None, + } +} + +/// Applies one newly-created trade event to a pair metric. +pub(crate) fn apply_trade_to_pair_metric( + metric: &mut crate::PairMetricDto, + slot: std::option::Option, + signature: std::string::String, + trade_side: crate::SwapTradeSide, + base_amount_raw: std::option::Option, + quote_amount_raw: std::option::Option, + price_quote_per_base: std::option::Option, +) { + metric.trade_count += 1; + if trade_side == crate::SwapTradeSide::BuyBase { + metric.buy_count += 1; + } + if trade_side == crate::SwapTradeSide::SellBase { + metric.sell_count += 1; + } + if metric.first_slot.is_none() { + metric.first_slot = slot; + } + if metric.first_signature.is_none() { + metric.first_signature = Some(signature.clone()); + } + metric.last_slot = slot; + metric.last_signature = Some(signature); + metric.cumulative_base_amount_raw = crate::trade_metric_update::add_raw_amounts( + metric.cumulative_base_amount_raw.clone(), + base_amount_raw, + ); + metric.cumulative_quote_amount_raw = crate::trade_metric_update::add_raw_amounts( + metric.cumulative_quote_amount_raw.clone(), + quote_amount_raw, + ); + if price_quote_per_base.is_some() { + metric.last_price_quote_per_base = price_quote_per_base; + } + metric.updated_at = chrono::Utc::now(); +} + +/// Adds two optional raw integer amount strings. +pub(crate) fn add_raw_amounts( + left: std::option::Option, + right: std::option::Option, +) -> std::option::Option { + match (left, right) { + (None, None) => return None, + (Some(left), None) => return Some(left), + (None, Some(right)) => return Some(right), + (Some(left), Some(right)) => { + let left_value_result = left.parse::(); + let left_value = match left_value_result { + Ok(left_value) => left_value, + Err(_) => return Some(left), + }; + let right_value_result = right.parse::(); + let right_value = match right_value_result { + Ok(right_value) => right_value, + Err(_) => return Some(left), + }; + return Some((left_value + right_value).to_string()); + }, + } +} + +/// Computes quote/base price from raw amounts and token decimals. +pub(crate) fn compute_price_quote_per_base_from_raw_amounts_with_decimals( + base_amount_raw: std::option::Option<&str>, + quote_amount_raw: std::option::Option<&str>, + base_decimals: std::option::Option, + quote_decimals: std::option::Option, +) -> std::option::Option { + let base_decimals = match base_decimals { + Some(base_decimals) => base_decimals, + None => return None, + }; + let quote_decimals = match quote_decimals { + Some(quote_decimals) => quote_decimals, + None => return None, + }; + let base_amount_raw = match base_amount_raw { + Some(base_amount_raw) => base_amount_raw.trim(), + None => return None, + }; + let quote_amount_raw = match quote_amount_raw { + Some(quote_amount_raw) => quote_amount_raw.trim(), + None => return None, + }; + if base_amount_raw.is_empty() || quote_amount_raw.is_empty() { + return None; + } + let base_amount_result = base_amount_raw.parse::(); + let base_amount = match base_amount_result { + Ok(base_amount) => base_amount, + Err(_) => return None, + }; + let quote_amount_result = quote_amount_raw.parse::(); + let quote_amount = match quote_amount_result { + Ok(quote_amount) => quote_amount, + Err(_) => return None, + }; + if base_amount <= 0.0 || quote_amount <= 0.0 { + return None; + } + let base_scale = 10_f64.powi(i32::from(base_decimals)); + let quote_scale = 10_f64.powi(i32::from(quote_decimals)); + if base_scale <= 0.0 || quote_scale <= 0.0 { + return None; + } + let base_ui_amount = base_amount / base_scale; + let quote_ui_amount = quote_amount / quote_scale; + if base_ui_amount <= 0.0 || quote_ui_amount <= 0.0 { + return None; + } + return Some(quote_ui_amount / base_ui_amount); +} + +/// Computes quote/base price from raw amount strings without decimals. +pub(crate) fn compute_price_quote_per_base_from_raw_amounts( + base_amount_raw: std::option::Option<&str>, + quote_amount_raw: std::option::Option<&str>, +) -> std::option::Option { + let base_amount_raw = match base_amount_raw { + Some(base_amount_raw) => base_amount_raw.trim(), + None => return None, + }; + let quote_amount_raw = match quote_amount_raw { + Some(quote_amount_raw) => quote_amount_raw.trim(), + None => return None, + }; + if base_amount_raw.is_empty() || quote_amount_raw.is_empty() { + return None; + } + let base_amount_result = base_amount_raw.parse::(); + let base_amount = match base_amount_result { + Ok(base_amount) => base_amount, + Err(_) => return None, + }; + let quote_amount_result = quote_amount_raw.parse::(); + let quote_amount = match quote_amount_result { + Ok(quote_amount) => quote_amount, + Err(_) => return None, + }; + if base_amount <= 0.0 { + return None; + } + return Some(quote_amount / base_amount); +} + +#[cfg(test)] +mod tests { + #[test] + fn priced_trade_event_rejects_unpriced_values() { + let result = super::is_priced_trade_event(None, Some("2500"), Some(2.5)); + assert!(!result); + let result = super::is_priced_trade_event(Some("1000"), None, Some(2.5)); + assert!(!result); + let result = super::is_priced_trade_event(Some("1000"), Some("2500"), None); + assert!(!result); + let result = super::is_priced_trade_event(Some("0"), Some("2500"), Some(2.5)); + assert!(!result); + let result = super::is_priced_trade_event(Some("1000"), Some("0"), Some(2.5)); + assert!(!result); + let result = super::is_priced_trade_event(Some("-1"), Some("2500"), Some(2.5)); + assert!(!result); + let result = super::is_priced_trade_event(Some("1000"), Some("-1"), Some(2.5)); + assert!(!result); + let result = super::is_priced_trade_event(Some("abc"), Some("2500"), Some(2.5)); + assert!(!result); + let result = super::is_priced_trade_event(Some("1000"), Some("abc"), Some(2.5)); + assert!(!result); + let result = super::is_priced_trade_event(Some("1000"), Some("2500"), Some(0.0)); + assert!(!result); + + let result = super::is_priced_trade_event(Some("1000"), Some("2500"), Some(f64::NAN)); + assert!(!result); + let result = super::is_priced_trade_event(Some("1000"), Some("2500"), Some(2.5)); + assert!(result); + } + + #[test] + fn raw_amounts_are_added_when_both_are_valid() { + let result = super::add_raw_amounts(Some("1000".to_string()), Some("2500".to_string())); + assert_eq!(result, Some("3500".to_string())); + } + + #[test] + fn raw_amount_addition_keeps_left_when_right_is_invalid() { + let result = super::add_raw_amounts(Some("1000".to_string()), Some("abc".to_string())); + assert_eq!(result, Some("1000".to_string())); + } + + #[test] + fn price_with_decimals_is_computed() { + let price = super::compute_price_quote_per_base_from_raw_amounts_with_decimals( + Some("1000000"), + Some("2500000000"), + Some(6), + Some(9), + ); + assert_eq!(price, Some(2.5)); + } + + #[test] + fn price_without_decimals_is_computed() { + let price = + super::compute_price_quote_per_base_from_raw_amounts(Some("1000"), Some("2500")); + assert_eq!(price, Some(2.5)); + } + + #[test] + fn overflowing_slot_is_ignored() { + let slot = super::convert_slot_to_i64(Some(u64::MAX)); + assert_eq!(slot, None); + } +} diff --git a/kb_lib/src/trade_pump_swap_amounts.rs b/kb_lib/src/trade_pump_swap_amounts.rs new file mode 100644 index 0000000..fc745f8 --- /dev/null +++ b/kb_lib/src/trade_pump_swap_amounts.rs @@ -0,0 +1,1233 @@ +// file: kb_lib/src/trade_pump_swap_amounts.rs + +//! PumpSwap trade amount recovery helpers. +//! +//! PumpSwap decoded events can miss direct base/quote raw amounts. +//! This module contains fallback extraction strategies based on token +//! balance deltas and pool-owner balance changes. + +/// PumpSwap trade amounts recovered from pool-owner token balance deltas. +#[derive(Debug, Clone)] +pub(crate) struct PumpSwapPoolBalanceDeltaAmounts { + /// Absolute base amount in raw token units. + pub(crate) base_amount_raw: std::string::String, + /// Absolute quote amount in raw token units. + pub(crate) quote_amount_raw: std::string::String, + /// Quote/base price computed with token decimals. + pub(crate) price_quote_per_base: f64, +} + +/// Resolution state for pool-owner token balance delta recovery. +#[derive(Debug, Clone)] +pub(crate) enum PumpSwapPoolBalanceDeltaResolution { + /// Amounts were successfully recovered and direction matched the decoded event. + Matched(PumpSwapPoolBalanceDeltaAmounts), + /// Deltas exist but their direction contradicts the decoded event side. + DirectionMismatch, + /// Required data was missing or unusable. + MissingData, +} + +impl PumpSwapPoolBalanceDeltaResolution { + /// Returns a stable tracing label for this resolution. + pub(crate) fn as_label(&self) -> &'static str { + match self { + Self::Matched(_) => return "matched", + Self::DirectionMismatch => return "direction_mismatch", + Self::MissingData => return "missing_data", + } + } +} + +/// PumpSwap fallback amount extracted from transaction token balance deltas. +#[derive(Clone, Debug, PartialEq)] +pub(crate) struct PumpSwapBalanceDeltaTradeAmounts { + /// Base amount in UI token units. + pub(crate) base_amount: f64, + /// Quote amount in UI token units. + pub(crate) quote_amount: f64, + /// Quote/base price computed from UI amounts. + pub(crate) price_quote_per_base: f64, +} + +#[derive(Debug, Clone)] +struct TokenBalanceRawAmount { + raw_amount: i128, + decimals: std::option::Option, +} + +#[derive(Clone, Debug, Eq, Ord, PartialEq, PartialOrd)] +struct TokenBalanceDeltaKey { + account_index: i64, + mint: std::string::String, + owner: std::string::String, +} + +#[derive(Clone, Debug)] +struct TokenBalanceDeltaAccumulator { + pre_amount: f64, + post_amount: f64, +} + +/// Builds a transaction JSON value that contains `meta`. +pub(crate) fn build_transaction_value_with_meta_json( + transaction_json: &str, + meta_json: std::option::Option<&str>, +) -> Result { + let transaction_value_result = serde_json::from_str::(transaction_json); + let mut transaction_value = match transaction_value_result { + Ok(transaction_value) => transaction_value, + Err(error) => { + return Err(crate::Error::Json(format!( + "cannot parse transaction_json for pump_swap token-balance fallback: {}", + error + ))); + }, + }; + + let meta_json = match meta_json { + Some(meta_json) => meta_json, + None => return Ok(transaction_value), + }; + + let meta_value_result = serde_json::from_str::(meta_json); + let meta_value = match meta_value_result { + Ok(meta_value) => meta_value, + Err(error) => { + return Err(crate::Error::Json(format!( + "cannot parse meta_json for pump_swap token-balance fallback: {}", + error + ))); + }, + }; + + match transaction_value.as_object_mut() { + Some(object) => { + object.insert("meta".to_string(), meta_value); + return Ok(transaction_value); + }, + None => {}, + } + + return Ok(serde_json::json!({ + "transaction": transaction_value, + "meta": meta_value + })); +} + +/// Converts a positive UI amount to a raw amount string using decimals. +pub(crate) fn convert_ui_amount_to_raw_string( + amount: f64, + decimals: std::option::Option, +) -> std::option::Option { + if !amount.is_finite() { + return None; + } + if amount <= 0.0_f64 { + return None; + } + + let decimals = match decimals { + Some(decimals) => decimals, + None => return None, + }; + + let scale = 10_f64.powi(i32::from(decimals)); + if !scale.is_finite() { + return None; + } + if scale <= 0.0_f64 { + return None; + } + + let raw_amount = amount * scale; + if !raw_amount.is_finite() { + return None; + } + if raw_amount <= 0.0_f64 { + return None; + } + + let rounded = raw_amount.round(); + if rounded <= 0.0_f64 { + return None; + } + + return Some(format!("{:.0}", rounded)); +} + +/// Recovers PumpSwap amounts from full-transaction token balance deltas. +pub(crate) fn try_build_pump_swap_trade_amounts_from_token_balance_deltas( + transaction_json: &serde_json::Value, + base_mint: &str, + quote_mint: &str, +) -> std::option::Option { + if base_mint.is_empty() { + return std::option::Option::None; + } + if quote_mint.is_empty() { + return std::option::Option::None; + } + if base_mint == quote_mint { + return std::option::Option::None; + } + + let meta_json = match find_transaction_meta_json(transaction_json) { + std::option::Option::Some(value) => value, + std::option::Option::None => { + return std::option::Option::None; + }, + }; + + let pre_token_balances = match json_array_field(meta_json, "preTokenBalances") { + std::option::Option::Some(value) => value, + std::option::Option::None => { + return std::option::Option::None; + }, + }; + + let post_token_balances = match json_array_field(meta_json, "postTokenBalances") { + std::option::Option::Some(value) => value, + std::option::Option::None => { + return std::option::Option::None; + }, + }; + + let mut deltas: std::collections::BTreeMap = + std::collections::BTreeMap::new(); + + apply_token_balance_side_to_delta_map( + &mut deltas, + pre_token_balances, + base_mint, + quote_mint, + true, + ); + + apply_token_balance_side_to_delta_map( + &mut deltas, + post_token_balances, + base_mint, + quote_mint, + false, + ); + + let mut base_amount = 0.0_f64; + let mut quote_amount = 0.0_f64; + + for (key, accumulator) in &deltas { + let delta_abs = (accumulator.post_amount - accumulator.pre_amount).abs(); + if !delta_abs.is_finite() { + continue; + } + if delta_abs <= 0.0_f64 { + continue; + } + + if key.mint == base_mint { + if delta_abs > base_amount { + base_amount = delta_abs; + } + } else if key.mint == quote_mint && delta_abs > quote_amount { + quote_amount = delta_abs; + } + } + + if !base_amount.is_finite() { + return std::option::Option::None; + } + if !quote_amount.is_finite() { + return std::option::Option::None; + } + if base_amount <= 0.0_f64 { + return std::option::Option::None; + } + if quote_amount <= 0.0_f64 { + return std::option::Option::None; + } + + let price_quote_per_base = quote_amount / base_amount; + if !price_quote_per_base.is_finite() { + return std::option::Option::None; + } + if price_quote_per_base <= 0.0_f64 { + return std::option::Option::None; + } + + return std::option::Option::Some(PumpSwapBalanceDeltaTradeAmounts { + base_amount, + quote_amount, + price_quote_per_base, + }); +} + +/// Recovers PumpSwap amounts from pool-owner pre/post token balances. +pub(crate) fn resolve_pump_swap_trade_amounts_from_pool_balance_deltas( + meta_json: std::option::Option<&str>, + pool_owner: &str, + base_mint: &str, + quote_mint: &str, + event_kind: &str, + base_token_decimals: std::option::Option, + quote_token_decimals: std::option::Option, +) -> Result { + let meta_json = match meta_json { + Some(meta_json) => meta_json, + None => return Ok(PumpSwapPoolBalanceDeltaResolution::MissingData), + }; + + let pool_owner = pool_owner.trim(); + let base_mint = base_mint.trim(); + let quote_mint = quote_mint.trim(); + + if pool_owner.is_empty() || base_mint.is_empty() || quote_mint.is_empty() { + return Ok(PumpSwapPoolBalanceDeltaResolution::MissingData); + } + if base_mint == quote_mint { + return Ok(PumpSwapPoolBalanceDeltaResolution::MissingData); + } + + let meta_value_result = serde_json::from_str::(meta_json); + let meta_value = match meta_value_result { + Ok(meta_value) => meta_value, + Err(error) => { + return Err(crate::Error::Json(format!( + "cannot parse meta_json for pump_swap pool-owner balance delta extraction: {}", + error + ))); + }, + }; + + let base_pre = sum_token_balance_raw_amount_by_owner_and_mint( + &meta_value, + "preTokenBalances", + pool_owner, + base_mint, + ); + let base_post = sum_token_balance_raw_amount_by_owner_and_mint( + &meta_value, + "postTokenBalances", + pool_owner, + base_mint, + ); + let quote_pre = sum_token_balance_raw_amount_by_owner_and_mint( + &meta_value, + "preTokenBalances", + pool_owner, + quote_mint, + ); + let quote_post = sum_token_balance_raw_amount_by_owner_and_mint( + &meta_value, + "postTokenBalances", + pool_owner, + quote_mint, + ); + + if base_pre.is_none() && base_post.is_none() { + return Ok(PumpSwapPoolBalanceDeltaResolution::MissingData); + } + if quote_pre.is_none() && quote_post.is_none() { + return Ok(PumpSwapPoolBalanceDeltaResolution::MissingData); + } + + let base_pre_raw = token_balance_raw_amount_value(base_pre.as_ref()); + let base_post_raw = token_balance_raw_amount_value(base_post.as_ref()); + let quote_pre_raw = token_balance_raw_amount_value(quote_pre.as_ref()); + let quote_post_raw = token_balance_raw_amount_value(quote_post.as_ref()); + + let base_delta = base_post_raw - base_pre_raw; + let quote_delta = quote_post_raw - quote_pre_raw; + + if base_delta == 0 || quote_delta == 0 { + return Ok(PumpSwapPoolBalanceDeltaResolution::MissingData); + } + + let pool_delta_side = if base_delta < 0 && quote_delta > 0 { + crate::SwapTradeSide::BuyBase + } else if base_delta > 0 && quote_delta < 0 { + crate::SwapTradeSide::SellBase + } else { + return Ok(PumpSwapPoolBalanceDeltaResolution::MissingData); + }; + + let event_side = trade_side_from_event_kind(event_kind); + if event_side != pool_delta_side { + return Ok(PumpSwapPoolBalanceDeltaResolution::DirectionMismatch); + } + + let base_amount_abs = i128_abs(base_delta); + let quote_amount_abs = i128_abs(quote_delta); + + if base_amount_abs <= 0 || quote_amount_abs <= 0 { + return Ok(PumpSwapPoolBalanceDeltaResolution::MissingData); + } + + let base_decimals = first_some_u8( + base_token_decimals, + first_token_balance_decimals(base_pre.as_ref(), base_post.as_ref()), + ); + let quote_decimals = first_some_u8( + quote_token_decimals, + first_token_balance_decimals(quote_pre.as_ref(), quote_post.as_ref()), + ); + + let price_quote_per_base = compute_price_quote_per_base_from_i128_raw_amounts_with_decimals( + base_amount_abs, + quote_amount_abs, + base_decimals, + quote_decimals, + ); + + let price_quote_per_base = match price_quote_per_base { + Some(price_quote_per_base) => price_quote_per_base, + None => return Ok(PumpSwapPoolBalanceDeltaResolution::MissingData), + }; + + return Ok(PumpSwapPoolBalanceDeltaResolution::Matched(PumpSwapPoolBalanceDeltaAmounts { + base_amount_raw: base_amount_abs.to_string(), + quote_amount_raw: quote_amount_abs.to_string(), + price_quote_per_base, + })); +} + +fn apply_token_balance_side_to_delta_map( + deltas: &mut std::collections::BTreeMap, + token_balances: &[serde_json::Value], + base_mint: &str, + quote_mint: &str, + is_pre: bool, +) { + for token_balance in token_balances { + let mint = match json_string_field(token_balance, "mint") { + std::option::Option::Some(value) => value, + std::option::Option::None => { + continue; + }, + }; + + if mint != base_mint && mint != quote_mint { + continue; + } + + let account_index = match json_i64_field(token_balance, "accountIndex") { + std::option::Option::Some(value) => value, + std::option::Option::None => { + continue; + }, + }; + + let owner = match json_string_field(token_balance, "owner") { + std::option::Option::Some(value) => value, + std::option::Option::None => std::string::String::new(), + }; + + let ui_amount = match token_balance_ui_amount(token_balance) { + std::option::Option::Some(value) => value, + std::option::Option::None => { + continue; + }, + }; + + if !ui_amount.is_finite() { + continue; + } + + let key = TokenBalanceDeltaKey { account_index, mint, owner }; + let entry = deltas.entry(key).or_insert(TokenBalanceDeltaAccumulator { + pre_amount: 0.0_f64, + post_amount: 0.0_f64, + }); + + if is_pre { + entry.pre_amount = ui_amount; + } else { + entry.post_amount = ui_amount; + } + } +} + +fn token_balance_ui_amount(token_balance: &serde_json::Value) -> std::option::Option { + let ui_token_amount = match json_object_field(token_balance, "uiTokenAmount") { + std::option::Option::Some(value) => value, + std::option::Option::None => { + return std::option::Option::None; + }, + }; + + match json_f64_field(ui_token_amount, "uiAmount") { + std::option::Option::Some(value) => { + return std::option::Option::Some(value); + }, + std::option::Option::None => {}, + } + + match json_string_field(ui_token_amount, "uiAmountString") { + std::option::Option::Some(value) => match value.parse::() { + std::result::Result::Ok(parsed) => { + return std::option::Option::Some(parsed); + }, + std::result::Result::Err(_) => {}, + }, + std::option::Option::None => {}, + } + + let raw_amount = match json_string_field(ui_token_amount, "amount") { + std::option::Option::Some(value) => value, + std::option::Option::None => { + return std::option::Option::None; + }, + }; + + let decimals = match json_i64_field(ui_token_amount, "decimals") { + std::option::Option::Some(value) => value, + std::option::Option::None => { + return std::option::Option::None; + }, + }; + + if decimals < 0_i64 { + return std::option::Option::None; + } + if decimals > 18_i64 { + return std::option::Option::None; + } + + let raw_amount_f64 = match raw_amount.parse::() { + std::result::Result::Ok(value) => value, + std::result::Result::Err(_) => { + return std::option::Option::None; + }, + }; + + let divisor = 10_f64.powi(decimals as i32); + if divisor <= 0.0_f64 { + return std::option::Option::None; + } + + return std::option::Option::Some(raw_amount_f64 / divisor); +} + +fn find_transaction_meta_json( + transaction_json: &serde_json::Value, +) -> std::option::Option<&serde_json::Value> { + match json_object_field(transaction_json, "meta") { + std::option::Option::Some(value) => { + return std::option::Option::Some(value); + }, + std::option::Option::None => {}, + } + + match json_object_field(transaction_json, "transaction") { + std::option::Option::Some(transaction_value) => { + match json_object_field(transaction_value, "meta") { + std::option::Option::Some(value) => { + return std::option::Option::Some(value); + }, + std::option::Option::None => {}, + } + }, + std::option::Option::None => {}, + } + + match json_object_field(transaction_json, "result") { + std::option::Option::Some(result_value) => { + match json_object_field(result_value, "meta") { + std::option::Option::Some(value) => { + return std::option::Option::Some(value); + }, + std::option::Option::None => {}, + } + + match json_object_field(result_value, "transaction") { + std::option::Option::Some(transaction_value) => { + match json_object_field(transaction_value, "meta") { + std::option::Option::Some(value) => { + return std::option::Option::Some(value); + }, + std::option::Option::None => {}, + } + }, + std::option::Option::None => {}, + } + }, + std::option::Option::None => {}, + } + + return std::option::Option::None; +} + +fn sum_token_balance_raw_amount_by_owner_and_mint( + meta_value: &serde_json::Value, + field_name: &str, + owner: &str, + mint: &str, +) -> std::option::Option { + let balances_value = match json_array_field(meta_value, field_name) { + std::option::Option::Some(balances_value) => balances_value, + std::option::Option::None => return std::option::Option::None, + }; + + let mut total = 0_i128; + let mut found = false; + let mut decimals = std::option::Option::None; + + for balance in balances_value { + let balance_mint = match json_string_field(balance, "mint") { + std::option::Option::Some(balance_mint) => balance_mint, + std::option::Option::None => continue, + }; + + if balance_mint != mint { + continue; + } + + let balance_owner = match json_string_field(balance, "owner") { + std::option::Option::Some(balance_owner) => balance_owner, + std::option::Option::None => continue, + }; + + if balance_owner != owner { + continue; + } + + let ui_token_amount = match json_object_field(balance, "uiTokenAmount") { + std::option::Option::Some(ui_token_amount) => ui_token_amount, + std::option::Option::None => continue, + }; + + let amount_text = match json_string_field(ui_token_amount, "amount") { + std::option::Option::Some(amount_text) => amount_text, + std::option::Option::None => continue, + }; + + let amount_result = amount_text.parse::(); + let amount = match amount_result { + std::result::Result::Ok(amount) => amount, + std::result::Result::Err(_) => continue, + }; + + total += amount; + found = true; + + if decimals.is_none() { + let decimals_i64 = match json_i64_field(ui_token_amount, "decimals") { + std::option::Option::Some(decimals_i64) => decimals_i64, + std::option::Option::None => continue, + }; + + let decimals_u8 = match u8::try_from(decimals_i64) { + std::result::Result::Ok(decimals_u8) => decimals_u8, + std::result::Result::Err(_) => continue, + }; + + decimals = std::option::Option::Some(decimals_u8); + } + } + + if !found { + return std::option::Option::None; + } + + return std::option::Option::Some(TokenBalanceRawAmount { raw_amount: total, decimals }); +} + +fn token_balance_raw_amount_value(value: std::option::Option<&TokenBalanceRawAmount>) -> i128 { + match value { + std::option::Option::Some(value) => return value.raw_amount, + std::option::Option::None => return 0_i128, + } +} + +fn first_token_balance_decimals( + left: std::option::Option<&TokenBalanceRawAmount>, + right: std::option::Option<&TokenBalanceRawAmount>, +) -> std::option::Option { + if let std::option::Option::Some(left) = left { + if left.decimals.is_some() { + return left.decimals; + } + } + + if let std::option::Option::Some(right) = right { + return right.decimals; + } + + return std::option::Option::None; +} + +fn first_some_u8( + left: std::option::Option, + right: std::option::Option, +) -> std::option::Option { + match left { + std::option::Option::Some(left) => return std::option::Option::Some(left), + std::option::Option::None => return right, + } +} + +fn trade_side_from_event_kind(event_kind: &str) -> crate::SwapTradeSide { + if event_kind.ends_with(".buy") { + return crate::SwapTradeSide::BuyBase; + } + if event_kind.ends_with(".sell") { + return crate::SwapTradeSide::SellBase; + } + return crate::SwapTradeSide::Unknown; +} + +fn i128_abs(i128_abs: i128) -> i128 { + if i128_abs < 0_i128 { + return -i128_abs; + } + return i128_abs; +} + +fn compute_price_quote_per_base_from_i128_raw_amounts_with_decimals( + base_amount_raw: i128, + quote_amount_raw: i128, + base_decimals: std::option::Option, + quote_decimals: std::option::Option, +) -> std::option::Option { + if base_amount_raw <= 0_i128 || quote_amount_raw <= 0_i128 { + return std::option::Option::None; + } + + let base_decimals = match base_decimals { + std::option::Option::Some(base_decimals) => base_decimals, + std::option::Option::None => return std::option::Option::None, + }; + + let quote_decimals = match quote_decimals { + std::option::Option::Some(quote_decimals) => quote_decimals, + std::option::Option::None => return std::option::Option::None, + }; + + let base_scale = 10_f64.powi(i32::from(base_decimals)); + let quote_scale = 10_f64.powi(i32::from(quote_decimals)); + + if !base_scale.is_finite() || !quote_scale.is_finite() { + return std::option::Option::None; + } + if base_scale <= 0.0_f64 || quote_scale <= 0.0_f64 { + return std::option::Option::None; + } + + let base_amount = base_amount_raw as f64 / base_scale; + let quote_amount = quote_amount_raw as f64 / quote_scale; + + if !base_amount.is_finite() || !quote_amount.is_finite() { + return std::option::Option::None; + } + if base_amount <= 0.0_f64 || quote_amount <= 0.0_f64 { + return std::option::Option::None; + } + + let price_quote_per_base = quote_amount / base_amount; + if !price_quote_per_base.is_finite() { + return std::option::Option::None; + } + if price_quote_per_base <= 0.0_f64 { + return std::option::Option::None; + } + + return std::option::Option::Some(price_quote_per_base); +} + +fn json_object_field<'a>( + value: &'a serde_json::Value, + field_name: &str, +) -> std::option::Option<&'a serde_json::Value> { + match value { + serde_json::Value::Object(object) => match object.get(field_name) { + std::option::Option::Some(field_value) => { + return std::option::Option::Some(field_value); + }, + std::option::Option::None => return std::option::Option::None, + }, + _ => return std::option::Option::None, + } +} + +fn json_array_field<'a>( + value: &'a serde_json::Value, + field_name: &str, +) -> std::option::Option<&'a [serde_json::Value]> { + let field_value = match json_object_field(value, field_name) { + std::option::Option::Some(found) => found, + std::option::Option::None => { + return std::option::Option::None; + }, + }; + + match field_value { + serde_json::Value::Array(values) => return std::option::Option::Some(values.as_slice()), + _ => return std::option::Option::None, + } +} + +fn json_string_field( + value: &serde_json::Value, + field_name: &str, +) -> std::option::Option { + let field_value = match json_object_field(value, field_name) { + std::option::Option::Some(found) => found, + std::option::Option::None => { + return std::option::Option::None; + }, + }; + + match field_value { + serde_json::Value::String(text) => return std::option::Option::Some(text.clone()), + serde_json::Value::Number(number) => return std::option::Option::Some(number.to_string()), + _ => return std::option::Option::None, + } +} + +fn json_i64_field(value: &serde_json::Value, field_name: &str) -> std::option::Option { + let field_value = match json_object_field(value, field_name) { + std::option::Option::Some(found) => found, + std::option::Option::None => { + return std::option::Option::None; + }, + }; + + match field_value { + serde_json::Value::Number(number) => return number.as_i64(), + serde_json::Value::String(text) => match text.parse::() { + std::result::Result::Ok(parsed) => return std::option::Option::Some(parsed), + std::result::Result::Err(_) => return std::option::Option::None, + }, + _ => return std::option::Option::None, + } +} + +fn json_f64_field(value: &serde_json::Value, field_name: &str) -> std::option::Option { + let field_value = match json_object_field(value, field_name) { + std::option::Option::Some(found) => found, + std::option::Option::None => { + return std::option::Option::None; + }, + }; + + match field_value { + serde_json::Value::Number(number) => return number.as_f64(), + serde_json::Value::String(text) => match text.parse::() { + std::result::Result::Ok(parsed) => return std::option::Option::Some(parsed), + std::result::Result::Err(_) => return std::option::Option::None, + }, + _ => return std::option::Option::None, + } +} + +#[cfg(test)] +mod tests { + #[test] + fn pump_swap_balance_delta_amounts_are_recovered_from_meta() { + let transaction_json = serde_json::json!({ + "meta": { + "preTokenBalances": [ + { + "accountIndex": 1, + "mint": "BASE111", + "owner": "owner_a", + "uiTokenAmount": { + "uiAmount": 10.0, + "uiAmountString": "10", + "decimals": 6, + "amount": "10000000" + } + }, + { + "accountIndex": 2, + "mint": "QUOTE111", + "owner": "owner_b", + "uiTokenAmount": { + "uiAmount": 2.0, + "uiAmountString": "2", + "decimals": 9, + "amount": "2000000000" + } + } + ], + "postTokenBalances": [ + { + "accountIndex": 1, + "mint": "BASE111", + "owner": "owner_a", + "uiTokenAmount": { + "uiAmount": 15.0, + "uiAmountString": "15", + "decimals": 6, + "amount": "15000000" + } + }, + { + "accountIndex": 2, + "mint": "QUOTE111", + "owner": "owner_b", + "uiTokenAmount": { + "uiAmount": 1.5, + "uiAmountString": "1.5", + "decimals": 9, + "amount": "1500000000" + } + } + ] + } + }); + + let amounts = match super::try_build_pump_swap_trade_amounts_from_token_balance_deltas( + &transaction_json, + "BASE111", + "QUOTE111", + ) { + std::option::Option::Some(value) => value, + std::option::Option::None => { + panic!("expected pump_swap balance delta amounts"); + }, + }; + + assert_eq!(amounts.base_amount, 5.0_f64); + assert_eq!(amounts.quote_amount, 0.5_f64); + assert_eq!(amounts.price_quote_per_base, 0.1_f64); + } + + #[test] + fn pump_swap_balance_delta_amounts_use_raw_amount_when_ui_amount_is_null() { + let transaction_json = serde_json::json!({ + "meta": { + "preTokenBalances": [ + { + "accountIndex": 1, + "mint": "BASE111", + "owner": "owner_a", + "uiTokenAmount": { + "uiAmount": null, + "decimals": 6, + "amount": "10000000" + } + }, + { + "accountIndex": 2, + "mint": "QUOTE111", + "owner": "owner_b", + "uiTokenAmount": { + "uiAmount": null, + "decimals": 9, + "amount": "2000000000" + } + } + ], + "postTokenBalances": [ + { + "accountIndex": 1, + "mint": "BASE111", + "owner": "owner_a", + "uiTokenAmount": { + "uiAmount": null, + "decimals": 6, + "amount": "15000000" + } + }, + { + "accountIndex": 2, + "mint": "QUOTE111", + "owner": "owner_b", + "uiTokenAmount": { + "uiAmount": null, + "decimals": 9, + "amount": "1500000000" + } + } + ] + } + }); + + let amounts = match super::try_build_pump_swap_trade_amounts_from_token_balance_deltas( + &transaction_json, + "BASE111", + "QUOTE111", + ) { + std::option::Option::Some(value) => value, + std::option::Option::None => { + panic!("expected pump_swap balance delta amounts"); + }, + }; + + assert_eq!(amounts.base_amount, 5.0_f64); + assert_eq!(amounts.quote_amount, 0.5_f64); + assert_eq!(amounts.price_quote_per_base, 0.1_f64); + } + + #[test] + fn pump_swap_balance_delta_amounts_return_none_when_base_delta_is_zero() { + let transaction_json = serde_json::json!({ + "meta": { + "preTokenBalances": [ + { + "accountIndex": 1, + "mint": "BASE111", + "owner": "owner_a", + "uiTokenAmount": { + "uiAmount": 10.0, + "uiAmountString": "10", + "decimals": 6, + "amount": "10000000" + } + }, + { + "accountIndex": 2, + "mint": "QUOTE111", + "owner": "owner_b", + "uiTokenAmount": { + "uiAmount": 2.0, + "uiAmountString": "2", + "decimals": 9, + "amount": "2000000000" + } + } + ], + "postTokenBalances": [ + { + "accountIndex": 1, + "mint": "BASE111", + "owner": "owner_a", + "uiTokenAmount": { + "uiAmount": 10.0, + "uiAmountString": "10", + "decimals": 6, + "amount": "10000000" + } + }, + { + "accountIndex": 2, + "mint": "QUOTE111", + "owner": "owner_b", + "uiTokenAmount": { + "uiAmount": 1.5, + "uiAmountString": "1.5", + "decimals": 9, + "amount": "1500000000" + } + } + ] + } + }); + + let amounts = super::try_build_pump_swap_trade_amounts_from_token_balance_deltas( + &transaction_json, + "BASE111", + "QUOTE111", + ); + + assert_eq!(amounts, std::option::Option::None); + } + + #[test] + fn pump_swap_pool_owner_delta_accepts_buy_direction() { + let meta_json = serde_json::json!({ + "preTokenBalances": [ + { + "accountIndex": 1, + "mint": "BASE111", + "owner": "Pool111", + "uiTokenAmount": { + "amount": "7967771862", + "decimals": 9 + } + }, + { + "accountIndex": 2, + "mint": "QUOTE111", + "owner": "Pool111", + "uiTokenAmount": { + "amount": "8970607", + "decimals": 9 + } + } + ], + "postTokenBalances": [ + { + "accountIndex": 1, + "mint": "BASE111", + "owner": "Pool111", + "uiTokenAmount": { + "amount": "7573340642", + "decimals": 9 + } + }, + { + "accountIndex": 2, + "mint": "QUOTE111", + "owner": "Pool111", + "uiTokenAmount": { + "amount": "9438980", + "decimals": 9 + } + } + ] + }); + + let meta_json_text = meta_json.to_string(); + + let result = super::resolve_pump_swap_trade_amounts_from_pool_balance_deltas( + Some(meta_json_text.as_str()), + "Pool111", + "BASE111", + "QUOTE111", + "pump_swap.buy", + Some(9), + Some(9), + ); + + let resolution = match result { + Ok(resolution) => resolution, + Err(error) => panic!("pump_swap delta resolution must succeed: {}", error), + }; + + match resolution { + super::PumpSwapPoolBalanceDeltaResolution::Matched(amounts) => { + assert_eq!(amounts.base_amount_raw, "394431220"); + assert_eq!(amounts.quote_amount_raw, "468373"); + assert!(amounts.price_quote_per_base > 0.00118); + assert!(amounts.price_quote_per_base < 0.00119); + }, + _ => panic!("expected matched pump_swap buy delta"), + } + } + + #[test] + fn pump_swap_pool_owner_delta_rejects_wrong_direction() { + let meta_json = serde_json::json!({ + "preTokenBalances": [ + { + "accountIndex": 1, + "mint": "BASE111", + "owner": "Pool111", + "uiTokenAmount": { + "amount": "7967771862", + "decimals": 9 + } + }, + { + "accountIndex": 2, + "mint": "QUOTE111", + "owner": "Pool111", + "uiTokenAmount": { + "amount": "8970607", + "decimals": 9 + } + } + ], + "postTokenBalances": [ + { + "accountIndex": 1, + "mint": "BASE111", + "owner": "Pool111", + "uiTokenAmount": { + "amount": "7573340642", + "decimals": 9 + } + }, + { + "accountIndex": 2, + "mint": "QUOTE111", + "owner": "Pool111", + "uiTokenAmount": { + "amount": "9438980", + "decimals": 9 + } + } + ] + }); + + let meta_json_text = meta_json.to_string(); + + let result = super::resolve_pump_swap_trade_amounts_from_pool_balance_deltas( + Some(meta_json_text.as_str()), + "Pool111", + "BASE111", + "QUOTE111", + "pump_swap.sell", + Some(9), + Some(9), + ); + + let resolution = match result { + Ok(resolution) => resolution, + Err(error) => panic!("pump_swap delta resolution must succeed: {}", error), + }; + + match resolution { + super::PumpSwapPoolBalanceDeltaResolution::DirectionMismatch => {}, + _ => panic!("expected pump_swap direction mismatch"), + } + } + + #[test] + fn pump_swap_pool_owner_delta_ignores_non_pool_owner() { + let meta_json = serde_json::json!({ + "preTokenBalances": [ + { + "accountIndex": 1, + "mint": "BASE111", + "owner": "OtherOwner111", + "uiTokenAmount": { + "amount": "1000", + "decimals": 6 + } + }, + { + "accountIndex": 2, + "mint": "QUOTE111", + "owner": "OtherOwner111", + "uiTokenAmount": { + "amount": "2000", + "decimals": 6 + } + } + ], + "postTokenBalances": [ + { + "accountIndex": 1, + "mint": "BASE111", + "owner": "OtherOwner111", + "uiTokenAmount": { + "amount": "900", + "decimals": 6 + } + }, + { + "accountIndex": 2, + "mint": "QUOTE111", + "owner": "OtherOwner111", + "uiTokenAmount": { + "amount": "2100", + "decimals": 6 + } + } + ] + }); + + let meta_json_text = meta_json.to_string(); + + let result = super::resolve_pump_swap_trade_amounts_from_pool_balance_deltas( + Some(meta_json_text.as_str()), + "Pool111", + "BASE111", + "QUOTE111", + "pump_swap.buy", + Some(6), + Some(6), + ); + + let resolution = match result { + Ok(resolution) => resolution, + Err(error) => panic!("pump_swap delta resolution must succeed: {}", error), + }; + match resolution { + super::PumpSwapPoolBalanceDeltaResolution::MissingData => {}, + _ => panic!("expected missing data for non-pool owner"), + } + } +} diff --git a/kb_lib/src/trade_side_resolution.rs b/kb_lib/src/trade_side_resolution.rs new file mode 100644 index 0000000..7f383e2 --- /dev/null +++ b/kb_lib/src/trade_side_resolution.rs @@ -0,0 +1,118 @@ +// file: kb_lib/src/trade_side_resolution.rs + +//! Trade-side resolution helpers. +//! +//! This module resolves a normalized `SwapTradeSide` from a decoded event kind +//! and optional decoded payload metadata. + +/// Resolves the normalized trade side from payload metadata and event kind. +pub(crate) fn extract_trade_side( + event_kind: &str, + payload: &serde_json::Value, +) -> crate::SwapTradeSide { + let trade_side_option = crate::trade_side_resolution::extract_string_by_candidate_keys( + payload, + &["tradeSide", "trade_side"], + ); + match trade_side_option.as_deref() { + Some("BuyBase") => return crate::SwapTradeSide::BuyBase, + Some("buy") => return crate::SwapTradeSide::BuyBase, + Some("BUY") => return crate::SwapTradeSide::BuyBase, + Some("SellBase") => return crate::SwapTradeSide::SellBase, + Some("sell") => return crate::SwapTradeSide::SellBase, + Some("SELL") => return crate::SwapTradeSide::SellBase, + _ => {}, + } + if event_kind.ends_with(".buy") { + return crate::SwapTradeSide::BuyBase; + } + if event_kind.ends_with(".sell") { + return crate::SwapTradeSide::SellBase; + } + return crate::SwapTradeSide::Unknown; +} + +fn extract_string_by_candidate_keys( + value: &serde_json::Value, + candidate_keys: &[&str], +) -> std::option::Option { + if let Some(object) = value.as_object() { + for candidate_key in candidate_keys { + let direct_option = object.get(*candidate_key); + if let Some(direct) = direct_option { + let direct_text_option = direct.as_str(); + if let Some(direct_text) = direct_text_option { + return Some(direct_text.to_string()); + } + } + } + for nested_value in object.values() { + let nested_result = crate::trade_side_resolution::extract_string_by_candidate_keys( + nested_value, + candidate_keys, + ); + if nested_result.is_some() { + return nested_result; + } + } + return None; + } + if let Some(array) = value.as_array() { + for nested_value in array { + let nested_result = crate::trade_side_resolution::extract_string_by_candidate_keys( + nested_value, + candidate_keys, + ); + if nested_result.is_some() { + return nested_result; + } + } + } + return None; +} + +#[cfg(test)] +mod tests { + #[test] + fn payload_trade_side_wins_over_event_kind() { + let payload = serde_json::json!({ + "tradeSide": "SellBase" + }); + let side = super::extract_trade_side("pump_swap.buy", &payload); + assert_eq!(side, crate::SwapTradeSide::SellBase); + } + + #[test] + fn nested_payload_trade_side_is_resolved() { + let payload = serde_json::json!({ + "decoded": { + "meta": { + "trade_side": "buy" + } + } + }); + let side = super::extract_trade_side("raydium_cpmm.swap_base_input", &payload); + assert_eq!(side, crate::SwapTradeSide::BuyBase); + } + + #[test] + fn buy_suffix_is_resolved_when_payload_has_no_side() { + let payload = serde_json::json!({}); + let side = super::extract_trade_side("pump_fun.buy", &payload); + assert_eq!(side, crate::SwapTradeSide::BuyBase); + } + + #[test] + fn sell_suffix_is_resolved_when_payload_has_no_side() { + let payload = serde_json::json!({}); + let side = super::extract_trade_side("pump_fun.sell", &payload); + assert_eq!(side, crate::SwapTradeSide::SellBase); + } + + #[test] + fn unknown_side_is_returned_when_no_hint_exists() { + let payload = serde_json::json!({}); + let side = super::extract_trade_side("raydium_cpmm.swap_base_input", &payload); + assert_eq!(side, crate::SwapTradeSide::Unknown); + } +} diff --git a/kb_lib/src/trade_solana_amounts.rs b/kb_lib/src/trade_solana_amounts.rs new file mode 100644 index 0000000..657fdb6 --- /dev/null +++ b/kb_lib/src/trade_solana_amounts.rs @@ -0,0 +1,913 @@ +// file: kb_lib/src/trade_solana_amounts.rs + +//! Solana transaction/meta trade amount extraction helpers. +//! +//! This module contains generic fallback logic based on transaction JSON, +//! transaction meta, account keys, token balances, native balances and +//! instruction-scoped SPL token transfers. + +/// Extracted base/quote amounts and optional quote/base price. +pub(crate) type ExtractedTradeAmounts = ( + std::option::Option, + std::option::Option, + std::option::Option, +); + +/// Extracts base/quote amounts from token-account vault balance deltas. +pub(crate) fn extract_trade_amounts_from_vault_balance_deltas( + transaction_json: &str, + meta_json: std::option::Option<&str>, + base_vault_address: std::option::Option<&str>, + quote_vault_address: std::option::Option<&str>, +) -> Result { + let meta_json = match meta_json { + Some(meta_json) => meta_json, + None => return Ok((None, None, None)), + }; + let transaction_value_result = serde_json::from_str::(transaction_json); + let transaction_value = match transaction_value_result { + Ok(transaction_value) => transaction_value, + Err(error) => { + return Err(crate::Error::Json(format!( + "cannot parse transaction_json for vault balance amount extraction: {}", + error + ))); + }, + }; + let meta_value_result = serde_json::from_str::(meta_json); + let meta_value = match meta_value_result { + Ok(meta_value) => meta_value, + Err(error) => { + return Err(crate::Error::Json(format!( + "cannot parse meta_json for vault balance amount extraction: {}", + error + ))); + }, + }; + let account_keys_result = + crate::trade_solana_amounts::extract_transaction_account_keys(&transaction_value); + let account_keys = match account_keys_result { + Ok(account_keys) => account_keys, + Err(error) => return Err(error), + }; + let pre_balances_result = crate::trade_solana_amounts::extract_token_balance_map( + &meta_value, + &account_keys, + "preTokenBalances", + ); + let pre_balances = match pre_balances_result { + Ok(pre_balances) => pre_balances, + Err(error) => return Err(error), + }; + let post_balances_result = crate::trade_solana_amounts::extract_token_balance_map( + &meta_value, + &account_keys, + "postTokenBalances", + ); + let post_balances = match post_balances_result { + Ok(post_balances) => post_balances, + Err(error) => return Err(error), + }; + let mut base_amount_raw = None; + let mut quote_amount_raw = None; + let mut price_quote_per_base = None; + if let Some(base_vault_address) = base_vault_address { + let base_pre = pre_balances.get(base_vault_address); + let base_post = post_balances.get(base_vault_address); + let base_pre_raw = match base_pre { + Some(value) => Some(value.0.clone()), + None => None, + }; + let base_post_raw = match base_post { + Some(value) => Some(value.0.clone()), + None => None, + }; + base_amount_raw = + crate::trade_solana_amounts::compute_amount_delta_abs(base_pre_raw, base_post_raw); + let base_pre_ui = match base_pre { + Some(value) => value.1, + None => None, + }; + let base_post_ui = match base_post { + Some(value) => value.1, + None => None, + }; + let base_delta_ui = + crate::trade_solana_amounts::compute_ui_delta_abs(base_pre_ui, base_post_ui); + if let Some(quote_vault_address) = quote_vault_address { + let quote_pre = pre_balances.get(quote_vault_address); + let quote_post = post_balances.get(quote_vault_address); + let quote_pre_raw = match quote_pre { + Some(value) => Some(value.0.clone()), + None => None, + }; + let quote_post_raw = match quote_post { + Some(value) => Some(value.0.clone()), + None => None, + }; + quote_amount_raw = crate::trade_solana_amounts::compute_amount_delta_abs( + quote_pre_raw, + quote_post_raw, + ); + let quote_pre_ui = match quote_pre { + Some(value) => value.1, + None => None, + }; + let quote_post_ui = match quote_post { + Some(value) => value.1, + None => None, + }; + let quote_delta_ui = + crate::trade_solana_amounts::compute_ui_delta_abs(quote_pre_ui, quote_post_ui); + if let (Some(base_delta_ui), Some(quote_delta_ui)) = (base_delta_ui, quote_delta_ui) { + if base_delta_ui > 0.0 { + price_quote_per_base = Some(quote_delta_ui / base_delta_ui); + } + } + } + } + return Ok((base_amount_raw, quote_amount_raw, price_quote_per_base)); +} + +/// Extracts base/quote amounts from instruction-scoped SPL token transfers. +pub(crate) fn extract_trade_amounts_from_instruction_token_transfers( + meta_json: std::option::Option<&str>, + instruction_index: std::option::Option, + input_vault_address: std::option::Option<&str>, + output_vault_address: std::option::Option<&str>, + input_token_account: std::option::Option<&str>, + output_token_account: std::option::Option<&str>, + base_vault_address: std::option::Option<&str>, + quote_vault_address: std::option::Option<&str>, +) -> Result { + let meta_json = match meta_json { + Some(meta_json) => meta_json, + None => return Ok((None, None, None)), + }; + let instruction_index = match instruction_index { + Some(instruction_index) => u64::from(instruction_index), + None => return Ok((None, None, None)), + }; + let input_vault_address = match input_vault_address { + Some(input_vault_address) => input_vault_address.trim(), + None => return Ok((None, None, None)), + }; + let output_vault_address = match output_vault_address { + Some(output_vault_address) => output_vault_address.trim(), + None => return Ok((None, None, None)), + }; + let input_token_account = match input_token_account { + Some(input_token_account) => input_token_account.trim(), + None => return Ok((None, None, None)), + }; + let output_token_account = match output_token_account { + Some(output_token_account) => output_token_account.trim(), + None => return Ok((None, None, None)), + }; + let base_vault_address = match base_vault_address { + Some(base_vault_address) => base_vault_address.trim(), + None => return Ok((None, None, None)), + }; + let quote_vault_address = match quote_vault_address { + Some(quote_vault_address) => quote_vault_address.trim(), + None => return Ok((None, None, None)), + }; + if input_vault_address.is_empty() + || output_vault_address.is_empty() + || input_token_account.is_empty() + || output_token_account.is_empty() + || base_vault_address.is_empty() + || quote_vault_address.is_empty() + { + return Ok((None, None, None)); + } + let meta_value_result = serde_json::from_str::(meta_json); + let meta_value = match meta_value_result { + Ok(meta_value) => meta_value, + Err(error) => { + return Err(crate::Error::Json(format!( + "cannot parse meta_json for instruction-scoped token transfer amount extraction: {}", + error + ))); + }, + }; + let inner_groups_option = + meta_value.get("innerInstructions").and_then(|value| return value.as_array()); + let inner_groups = match inner_groups_option { + Some(inner_groups) => inner_groups, + None => return Ok((None, None, None)), + }; + let mut input_amount_raw = None; + let mut output_amount_raw = None; + for inner_group in inner_groups { + let group_index_option = inner_group.get("index").and_then(|value| return value.as_u64()); + let group_index = match group_index_option { + Some(group_index) => group_index, + None => continue, + }; + if group_index != instruction_index { + continue; + } + let instructions_option = + inner_group.get("instructions").and_then(|value| return value.as_array()); + let instructions = match instructions_option { + Some(instructions) => instructions, + None => continue, + }; + for instruction in instructions { + if !crate::trade_solana_amounts::is_spl_token_transfer_instruction(instruction) { + continue; + } + let parsed = match instruction.get("parsed") { + Some(parsed) => parsed, + None => continue, + }; + let info = match parsed.get("info") { + Some(info) => info, + None => continue, + }; + let source_option = + crate::trade_solana_amounts::extract_string_by_candidate_keys(info, &["source"]); + let source = match source_option { + Some(source) => source, + None => continue, + }; + let destination_option = crate::trade_solana_amounts::extract_string_by_candidate_keys( + info, + &["destination"], + ); + let destination = match destination_option { + Some(destination) => destination, + None => continue, + }; + let amount_option = + crate::trade_solana_amounts::extract_scalar_as_string_by_candidate_keys( + info, + &["amount"], + ); + let amount = match amount_option { + Some(amount) => amount, + None => continue, + }; + if input_amount_raw.is_none() + && crate::trade_solana_amounts::account_equals(source.as_str(), input_token_account) + && crate::trade_solana_amounts::account_equals( + destination.as_str(), + input_vault_address, + ) + { + input_amount_raw = Some(amount.clone()); + continue; + } + if output_amount_raw.is_none() + && crate::trade_solana_amounts::account_equals( + source.as_str(), + output_vault_address, + ) + && crate::trade_solana_amounts::account_equals( + destination.as_str(), + output_token_account, + ) + { + output_amount_raw = Some(amount); + continue; + } + } + } + if input_amount_raw.is_none() && output_amount_raw.is_none() { + return Ok((None, None, None)); + } + if crate::trade_solana_amounts::account_equals(input_vault_address, base_vault_address) + && crate::trade_solana_amounts::account_equals(output_vault_address, quote_vault_address) + { + return Ok((input_amount_raw, output_amount_raw, None)); + } + if crate::trade_solana_amounts::account_equals(input_vault_address, quote_vault_address) + && crate::trade_solana_amounts::account_equals(output_vault_address, base_vault_address) + { + return Ok((output_amount_raw, input_amount_raw, None)); + } + return Ok((None, None, None)); +} + +/// Extracts Pump.fun amounts from transaction token/native balance deltas. +pub(crate) fn extract_pump_fun_amounts_from_transaction( + transaction_json: &str, + meta_json: std::option::Option<&str>, + base_vault_address: std::option::Option<&str>, + quote_native_address: std::option::Option<&str>, +) -> Result { + let meta_json = match meta_json { + Some(meta_json) => meta_json, + None => return Ok((None, None, None)), + }; + let transaction_value_result = serde_json::from_str::(transaction_json); + let transaction_value = match transaction_value_result { + Ok(transaction_value) => transaction_value, + Err(error) => { + return Err(crate::Error::Json(format!( + "cannot parse transaction_json for pump_fun amount extraction: {}", + error + ))); + }, + }; + let meta_value_result = serde_json::from_str::(meta_json); + let meta_value = match meta_value_result { + Ok(meta_value) => meta_value, + Err(error) => { + return Err(crate::Error::Json(format!( + "cannot parse meta_json for pump_fun amount extraction: {}", + error + ))); + }, + }; + let account_keys_result = + crate::trade_solana_amounts::extract_transaction_account_keys(&transaction_value); + let account_keys = match account_keys_result { + Ok(account_keys) => account_keys, + Err(error) => return Err(error), + }; + let pre_balances_result = crate::trade_solana_amounts::extract_token_balance_map( + &meta_value, + &account_keys, + "preTokenBalances", + ); + let pre_balances = match pre_balances_result { + Ok(pre_balances) => pre_balances, + Err(error) => return Err(error), + }; + let post_balances_result = crate::trade_solana_amounts::extract_token_balance_map( + &meta_value, + &account_keys, + "postTokenBalances", + ); + let post_balances = match post_balances_result { + Ok(post_balances) => post_balances, + Err(error) => return Err(error), + }; + let mut base_amount_raw = None; + let mut quote_amount_raw = None; + let mut price_quote_per_base = None; + let mut base_delta_ui = None; + if let Some(base_vault_address) = base_vault_address { + let base_pre = pre_balances.get(base_vault_address); + let base_post = post_balances.get(base_vault_address); + let base_pre_raw = match base_pre { + Some(value) => Some(value.0.clone()), + None => None, + }; + let base_post_raw = match base_post { + Some(value) => Some(value.0.clone()), + None => None, + }; + base_amount_raw = + crate::trade_solana_amounts::compute_amount_delta_abs(base_pre_raw, base_post_raw); + let base_pre_ui = match base_pre { + Some(value) => value.1, + None => None, + }; + let base_post_ui = match base_post { + Some(value) => value.1, + None => None, + }; + base_delta_ui = + crate::trade_solana_amounts::compute_ui_delta_abs(base_pre_ui, base_post_ui); + } + if let Some(quote_native_address) = quote_native_address { + let quote_delta_result = + crate::trade_solana_amounts::extract_native_balance_delta_by_address( + &meta_value, + &account_keys, + quote_native_address, + ); + let quote_delta = match quote_delta_result { + Ok(quote_delta) => quote_delta, + Err(error) => return Err(error), + }; + if let Some(quote_delta_lamports) = quote_delta { + quote_amount_raw = Some(quote_delta_lamports.to_string()); + let quote_delta_ui = quote_delta_lamports as f64 / 1_000_000_000.0; + if let Some(base_delta_ui) = base_delta_ui { + if base_delta_ui > 0.0 { + price_quote_per_base = Some(quote_delta_ui / base_delta_ui); + } + } + } + } + return Ok((base_amount_raw, quote_amount_raw, price_quote_per_base)); +} + +/// Computes quote/base price from vault balance deltas. +pub(crate) fn compute_price_quote_per_base_with_decimals( + meta_json: std::option::Option<&str>, + transaction_json: &str, + base_vault_address: std::option::Option<&str>, + quote_vault_address: std::option::Option<&str>, +) -> std::option::Option { + let inferred_result = + crate::trade_solana_amounts::extract_trade_amounts_from_vault_balance_deltas( + transaction_json, + meta_json, + base_vault_address, + quote_vault_address, + ); + let inferred = match inferred_result { + Ok(inferred) => inferred, + Err(_) => return None, + }; + return inferred.2; +} + +fn is_spl_token_transfer_instruction(instruction: &serde_json::Value) -> bool { + let program_id_option = instruction.get("programId").and_then(|value| return value.as_str()); + if let Some(program_id) = program_id_option { + let spl_token_program_id = crate::SPL_TOKEN_PROGRAM_ID.to_string(); + let spl_token_2022_program_id = crate::SPL_TOKEN_2022_PROGRAM_ID.to_string(); + if program_id != spl_token_program_id.as_str() + && program_id != spl_token_2022_program_id.as_str() + { + return false; + } + } + let parsed_type_option = instruction + .get("parsed") + .and_then(|parsed| return parsed.get("type")) + .and_then(|value| return value.as_str()); + match parsed_type_option { + Some("transfer") => return true, + Some("transferChecked") => return true, + _ => return false, + } +} + +fn account_equals(left: &str, right: &str) -> bool { + let left = left.trim(); + let right = right.trim(); + if left.is_empty() || right.is_empty() { + return false; + } + return left == right; +} + +fn extract_native_balance_delta_by_address( + meta_value: &serde_json::Value, + account_keys: &[std::string::String], + address: &str, +) -> Result, crate::Error> { + let mut account_index = None; + for (index, account_key) in account_keys.iter().enumerate() { + if account_key.as_str() == address { + account_index = Some(index); + break; + } + } + let account_index = match account_index { + Some(account_index) => account_index, + None => return Ok(None), + }; + let pre_balances_option = + meta_value.get("preBalances").and_then(|value| return value.as_array()); + let post_balances_option = + meta_value.get("postBalances").and_then(|value| return value.as_array()); + let pre_balances = match pre_balances_option { + Some(pre_balances) => pre_balances, + None => return Ok(None), + }; + let post_balances = match post_balances_option { + Some(post_balances) => post_balances, + None => return Ok(None), + }; + if account_index >= pre_balances.len() || account_index >= post_balances.len() { + return Ok(None); + } + let pre_balance = match pre_balances[account_index].as_u64() { + Some(pre_balance) => pre_balance, + None => return Ok(None), + }; + let post_balance = match post_balances[account_index].as_u64() { + Some(post_balance) => post_balance, + None => return Ok(None), + }; + if post_balance >= pre_balance { + return Ok(Some(post_balance - pre_balance)); + } + return Ok(Some(pre_balance - post_balance)); +} + +fn extract_transaction_account_keys( + transaction_value: &serde_json::Value, +) -> Result, crate::Error> { + let candidate_arrays = [ + transaction_value + .get("message") + .and_then(|value| return value.get("accountKeys")), + transaction_value + .get("transaction") + .and_then(|value| return value.get("message")) + .and_then(|value| return value.get("accountKeys")), + transaction_value + .get("transaction") + .and_then(|value| return value.get("transaction")) + .and_then(|value| return value.get("message")) + .and_then(|value| return value.get("accountKeys")), + transaction_value.get("accountKeys"), + ]; + for candidate_array_option in candidate_arrays { + let candidate_array = match candidate_array_option { + Some(candidate_array) => candidate_array, + None => continue, + }; + let array = match candidate_array.as_array() { + Some(array) => array, + None => continue, + }; + let mut account_keys = std::vec::Vec::new(); + for item in array { + if let Some(value) = item.as_str() { + account_keys.push(value.to_string()); + continue; + } + let pubkey_option = item.get("pubkey").and_then(|value| return value.as_str()); + if let Some(pubkey) = pubkey_option { + account_keys.push(pubkey.to_string()); + continue; + } + } + if !account_keys.is_empty() { + return Ok(account_keys); + } + } + return Err(crate::Error::Json( + "cannot extract accountKeys from transaction_json".to_string(), + )); +} + +fn extract_token_balance_map( + meta_value: &serde_json::Value, + account_keys: &[std::string::String], + field_name: &str, +) -> Result< + std::collections::BTreeMap< + std::string::String, + (std::string::String, std::option::Option), + >, + crate::Error, +> { + let mut result = std::collections::BTreeMap::< + std::string::String, + (std::string::String, std::option::Option), + >::new(); + let balances_option = meta_value.get(field_name).and_then(|value| return value.as_array()); + let balances = match balances_option { + Some(balances) => balances, + None => return Ok(result), + }; + for balance in balances { + let account_index_option = + balance.get("accountIndex").and_then(|value| return value.as_u64()); + let account_index = match account_index_option { + Some(account_index) => account_index as usize, + None => continue, + }; + if account_index >= account_keys.len() { + continue; + } + let account_address = account_keys[account_index].clone(); + let ui_token_amount = match balance.get("uiTokenAmount") { + Some(ui_token_amount) => ui_token_amount, + None => continue, + }; + let raw_amount_option = + ui_token_amount.get("amount").and_then(|value| return value.as_str()); + let raw_amount = match raw_amount_option { + Some(raw_amount) => raw_amount.to_string(), + None => continue, + }; + let ui_amount_string_option = + ui_token_amount.get("uiAmountString").and_then(|value| return value.as_str()); + let ui_amount = match ui_amount_string_option { + Some(ui_amount_string) => { + let parse_result = ui_amount_string.parse::(); + match parse_result { + Ok(ui_amount) => Some(ui_amount), + Err(_) => None, + } + }, + None => None, + }; + result.insert(account_address, (raw_amount, ui_amount)); + } + return Ok(result); +} + +fn compute_amount_delta_abs( + pre_amount: std::option::Option, + post_amount: std::option::Option, +) -> std::option::Option { + let pre_amount = match pre_amount { + Some(pre_amount) => pre_amount, + None => "0".to_string(), + }; + let post_amount = match post_amount { + Some(post_amount) => post_amount, + None => "0".to_string(), + }; + let pre_value_result = pre_amount.parse::(); + let pre_value = match pre_value_result { + Ok(pre_value) => pre_value, + Err(_) => return None, + }; + let post_value_result = post_amount.parse::(); + let post_value = match post_value_result { + Ok(post_value) => post_value, + Err(_) => return None, + }; + let delta = if post_value >= pre_value { + post_value - pre_value + } else { + pre_value - post_value + }; + return Some(delta.to_string()); +} + +fn compute_ui_delta_abs( + pre_amount: std::option::Option, + post_amount: std::option::Option, +) -> std::option::Option { + let pre_amount = match pre_amount { + Some(pre_amount) => pre_amount, + None => 0.0, + }; + let post_amount = match post_amount { + Some(post_amount) => post_amount, + None => 0.0, + }; + let delta = if post_amount >= pre_amount { + post_amount - pre_amount + } else { + pre_amount - post_amount + }; + return Some(delta); +} + +fn extract_string_by_candidate_keys( + value: &serde_json::Value, + candidate_keys: &[&str], +) -> std::option::Option { + if let Some(object) = value.as_object() { + for candidate_key in candidate_keys { + let direct_option = object.get(*candidate_key); + if let Some(direct) = direct_option { + let direct_text_option = direct.as_str(); + if let Some(direct_text) = direct_text_option { + return Some(direct_text.to_string()); + } + } + } + for nested_value in object.values() { + let nested_result = crate::trade_solana_amounts::extract_string_by_candidate_keys( + nested_value, + candidate_keys, + ); + if nested_result.is_some() { + return nested_result; + } + } + return None; + } + + if let Some(array) = value.as_array() { + for nested_value in array { + let nested_result = crate::trade_solana_amounts::extract_string_by_candidate_keys( + nested_value, + candidate_keys, + ); + if nested_result.is_some() { + return nested_result; + } + } + } + return None; +} + +fn extract_scalar_as_string_by_candidate_keys( + value: &serde_json::Value, + candidate_keys: &[&str], +) -> std::option::Option { + if let Some(object) = value.as_object() { + for candidate_key in candidate_keys { + let direct_option = object.get(*candidate_key); + if let Some(direct) = direct_option { + if let Some(direct_text) = direct.as_str() { + return Some(direct_text.to_string()); + } + if let Some(direct_u64) = direct.as_u64() { + return Some(direct_u64.to_string()); + } + if let Some(direct_i64) = direct.as_i64() { + return Some(direct_i64.to_string()); + } + } + } + for nested_value in object.values() { + let nested_result = + crate::trade_solana_amounts::extract_scalar_as_string_by_candidate_keys( + nested_value, + candidate_keys, + ); + if nested_result.is_some() { + return nested_result; + } + } + return None; + } + if let Some(array) = value.as_array() { + for nested_value in array { + let nested_result = + crate::trade_solana_amounts::extract_scalar_as_string_by_candidate_keys( + nested_value, + candidate_keys, + ); + if nested_result.is_some() { + return nested_result; + } + } + } + return None; +} + +#[cfg(test)] +mod tests { + #[test] + fn vault_balance_deltas_extract_raw_amounts_and_price() { + let transaction_json = serde_json::json!({ + "transaction": { + "message": { + "accountKeys": [ + "BaseVault111", + "QuoteVault111" + ] + } + } + }); + let meta_json = serde_json::json!({ + "preTokenBalances": [ + { + "accountIndex": 0, + "uiTokenAmount": { + "amount": "1000000", + "uiAmountString": "1.0" + } + }, + { + "accountIndex": 1, + "uiTokenAmount": { + "amount": "2000000000", + "uiAmountString": "2.0" + } + } + ], + "postTokenBalances": [ + { + "accountIndex": 0, + "uiTokenAmount": { + "amount": "1500000", + "uiAmountString": "1.5" + } + }, + { + "accountIndex": 1, + "uiTokenAmount": { + "amount": "1000000000", + "uiAmountString": "1.0" + } + } + ] + }); + let transaction_json_text = transaction_json.to_string(); + let meta_json_text = meta_json.to_string(); + let result = super::extract_trade_amounts_from_vault_balance_deltas( + transaction_json_text.as_str(), + Some(meta_json_text.as_str()), + Some("BaseVault111"), + Some("QuoteVault111"), + ); + let amounts = match result { + Ok(amounts) => amounts, + Err(error) => panic!("vault delta extraction should succeed: {}", error), + }; + assert_eq!(amounts.0, Some("500000".to_string())); + assert_eq!(amounts.1, Some("1000000000".to_string())); + assert_eq!(amounts.2, Some(2.0)); + } + + #[test] + fn instruction_transfer_amounts_follow_base_quote_vault_mapping() { + let meta_json = serde_json::json!({ + "innerInstructions": [ + { + "index": 3, + "instructions": [ + { + "programId": crate::SPL_TOKEN_PROGRAM_ID, + "parsed": { + "type": "transfer", + "info": { + "source": "UserQuote111", + "destination": "QuoteVault111", + "amount": "2000000000" + } + } + }, + { + "programId": crate::SPL_TOKEN_PROGRAM_ID, + "parsed": { + "type": "transfer", + "info": { + "source": "BaseVault111", + "destination": "UserBase111", + "amount": "500000" + } + } + } + ] + } + ] + }); + let meta_json_text = meta_json.to_string(); + let result = super::extract_trade_amounts_from_instruction_token_transfers( + Some(meta_json_text.as_str()), + Some(3), + Some("QuoteVault111"), + Some("BaseVault111"), + Some("UserQuote111"), + Some("UserBase111"), + Some("BaseVault111"), + Some("QuoteVault111"), + ); + let amounts = match result { + Ok(amounts) => amounts, + Err(error) => panic!("instruction transfer extraction should succeed: {}", error), + }; + assert_eq!(amounts.0, Some("500000".to_string())); + assert_eq!(amounts.1, Some("2000000000".to_string())); + assert_eq!(amounts.2, None); + } + + #[test] + fn pump_fun_amounts_extract_token_delta_and_native_delta() { + let transaction_json = serde_json::json!({ + "transaction": { + "message": { + "accountKeys": [ + "BaseVault111", + "NativeQuote111" + ] + } + } + }); + let meta_json = serde_json::json!({ + "preTokenBalances": [ + { + "accountIndex": 0, + "uiTokenAmount": { + "amount": "1000000", + "uiAmountString": "1.0" + } + } + ], + "postTokenBalances": [ + { + "accountIndex": 0, + "uiTokenAmount": { + "amount": "1500000", + "uiAmountString": "1.5" + } + } + ], + "preBalances": [ + 0, + 3000000000u64 + ], + "postBalances": [ + 0, + 2000000000u64 + ] + }); + let transaction_json_text = transaction_json.to_string(); + let meta_json_text = meta_json.to_string(); + let result = super::extract_pump_fun_amounts_from_transaction( + transaction_json_text.as_str(), + Some(meta_json_text.as_str()), + Some("BaseVault111"), + Some("NativeQuote111"), + ); + let amounts = match result { + Ok(amounts) => amounts, + Err(error) => panic!("pump_fun extraction should succeed: {}", error), + }; + assert_eq!(amounts.0, Some("500000".to_string())); + assert_eq!(amounts.1, Some("1000000000".to_string())); + assert_eq!(amounts.2, Some(2.0)); + } +} diff --git a/kb_lib/src/transaction_classification.rs b/kb_lib/src/transaction_classification.rs new file mode 100644 index 0000000..b12a9a2 --- /dev/null +++ b/kb_lib/src/transaction_classification.rs @@ -0,0 +1,466 @@ +// file: kb_lib/src/transaction_classification.rs + +//! Transaction classification service. +//! +//! This service classifies projected Solana transactions after transaction +//! projection and optional DEX decoding. +//! +//! The first version is intentionally deterministic and conservative: +//! decoded DEX events win over program-id hints, and unknown transactions are +//! preserved as explicit `unknown_or_unclassified` rows. + +/// Service used to classify projected Solana transactions. +#[derive(Debug, Clone)] +pub struct TransactionClassificationService { + database: std::sync::Arc, +} + +impl TransactionClassificationService { + /// Creates a transaction classification service. + pub fn new(database: std::sync::Arc) -> Self { + return Self { database }; + } + + /// Classifies one transaction by signature and persists the classification. + pub async fn classify_transaction_by_signature( + &self, + signature: &str, + ) -> Result { + let context_result = + load_transaction_classification_context(self.database.as_ref(), signature).await; + let context = match context_result { + Ok(context) => context, + Err(error) => return Err(error), + }; + let classification = classify_transaction_context(&context); + let dto = crate::TransactionClassificationDto::new( + context.transaction_id, + context.transaction.signature.clone(), + context.transaction.slot, + classification.kind.to_string(), + classification.primary_protocol, + classification.primary_program_id, + classification.confidence_level, + classification.reason, + classification.evidence_json, + ); + let upsert_result = + crate::query_transaction_classifications_upsert(self.database.as_ref(), &dto).await; + if let Err(error) = upsert_result { + return Err(error); + } + let persisted_result = crate::query_transaction_classifications_get_by_transaction_id( + self.database.as_ref(), + context.transaction_id, + ) + .await; + let persisted_option = match persisted_result { + Ok(persisted_option) => persisted_option, + Err(error) => return Err(error), + }; + let persisted = match persisted_option { + Some(persisted) => persisted, + None => { + return Err(crate::Error::InvalidState(format!( + "transaction classification for '{}' disappeared after upsert", + signature + ))); + }, + }; + let candidate_recording_result = + crate::protocol_candidate_recording::record_protocol_candidates_for_classification( + crate::protocol_candidate_recording::ProtocolCandidateRecordingInput { + database: self.database.as_ref(), + transaction: &context.transaction, + transaction_id: context.transaction_id, + instructions: &context.instructions, + classification_kind: persisted.classification_kind.as_str(), + }, + ) + .await; + match candidate_recording_result { + Ok(candidate_count) => { + tracing::trace!( + signature = %context.transaction.signature, + classification_kind = %persisted.classification_kind, + protocol_candidate_count = candidate_count, + "transaction protocol candidates recorded" + ); + }, + Err(error) => return Err(error), + } + return Ok(persisted); + } +} + +struct TransactionClassificationContext { + transaction: crate::ChainTransactionDto, + transaction_id: i64, + instructions: std::vec::Vec, + decoded_events: std::vec::Vec, +} + +struct TransactionClassificationDecision { + kind: &'static str, + primary_protocol: std::option::Option, + primary_program_id: std::option::Option, + confidence_level: i16, + reason: std::string::String, + evidence_json: std::string::String, +} + +#[derive(Debug, Clone)] +struct KnownDexProgramMatch { + protocol_name: &'static str, + program_id: std::string::String, + instruction_id: std::option::Option, + instruction_index: u32, +} + +async fn load_transaction_classification_context( + database: &crate::Database, + signature: &str, +) -> Result { + let transaction_result = + crate::query_chain_transactions_get_by_signature(database, signature).await; + let transaction_option = match transaction_result { + Ok(transaction_option) => transaction_option, + Err(error) => return Err(error), + }; + let transaction = match transaction_option { + Some(transaction) => transaction, + None => { + return Err(crate::Error::InvalidState(format!( + "cannot classify unknown chain transaction '{}'", + signature + ))); + }, + }; + let transaction_id = match transaction.id { + Some(transaction_id) => transaction_id, + None => { + return Err(crate::Error::InvalidState(format!( + "chain transaction '{}' has no internal id", + signature + ))); + }, + }; + let instructions_result = + crate::query_chain_instructions_list_by_transaction_id(database, transaction_id).await; + let instructions = match instructions_result { + Ok(instructions) => instructions, + Err(error) => return Err(error), + }; + let decoded_events_result = + crate::query_dex_decoded_events_list_by_transaction_id(database, transaction_id).await; + let decoded_events = match decoded_events_result { + Ok(decoded_events) => decoded_events, + Err(error) => return Err(error), + }; + return Ok(TransactionClassificationContext { + transaction, + transaction_id, + instructions, + decoded_events, + }); +} + +fn classify_transaction_context( + context: &TransactionClassificationContext, +) -> TransactionClassificationDecision { + if !context.decoded_events.is_empty() { + return classify_from_decoded_events(context); + } + let known_program_matches = find_known_dex_program_matches(&context.instructions); + if !known_program_matches.is_empty() { + return classify_from_known_program_matches(context, &known_program_matches); + } + return build_decision( + "unknown_or_unclassified", + None, + None, + 25, + "transaction has no decoded DEX event and no known DEX program id".to_string(), + serde_json::json!({ + "transactionId": context.transaction_id, + "signature": context.transaction.signature, + "slot": context.transaction.slot, + "instructionCount": context.instructions.len(), + "decodedEventCount": context.decoded_events.len() + }), + ); +} + +fn classify_from_decoded_events( + context: &TransactionClassificationContext, +) -> TransactionClassificationDecision { + let mut first_protocol = None; + let mut first_program_id = None; + let mut trade_event_count = 0_i64; + let mut non_trade_event_count = 0_i64; + let mut decoded_event_evidence = std::vec::Vec::new(); + for decoded_event in &context.decoded_events { + if first_protocol.is_none() { + first_protocol = Some(decoded_event.protocol_name.clone()); + } + if first_program_id.is_none() { + first_program_id = Some(decoded_event.program_id.clone()); + } + let payload_value_result = + serde_json::from_str::(decoded_event.payload_json.as_str()); + let payload_value = match payload_value_result { + Ok(payload_value) => payload_value, + Err(_) => serde_json::Value::Null, + }; + let is_trade = crate::is_decoded_event_trade_candidate( + decoded_event.event_kind.as_str(), + &payload_value, + ); + if is_trade { + trade_event_count += 1; + } else { + non_trade_event_count += 1; + } + decoded_event_evidence.push(serde_json::json!({ + "id": decoded_event.id, + "protocolName": decoded_event.protocol_name, + "programId": decoded_event.program_id, + "eventKind": decoded_event.event_kind, + "poolAccount": decoded_event.pool_account, + "tradeCandidate": is_trade + })); + } + if trade_event_count > 0_i64 { + return build_decision( + "dex_trade", + first_protocol, + first_program_id, + 100, + "transaction has at least one decoded DEX trade event".to_string(), + serde_json::json!({ + "transactionId": context.transaction_id, + "signature": context.transaction.signature, + "slot": context.transaction.slot, + "decodedEventCount": context.decoded_events.len(), + "tradeEventCount": trade_event_count, + "nonTradeEventCount": non_trade_event_count, + "decodedEvents": decoded_event_evidence + }), + ); + } + return build_decision( + "dex_non_trade", + first_protocol, + first_program_id, + 95, + "transaction has decoded DEX events but no trade candidate".to_string(), + serde_json::json!({ + "transactionId": context.transaction_id, + "signature": context.transaction.signature, + "slot": context.transaction.slot, + "decodedEventCount": context.decoded_events.len(), + "tradeEventCount": trade_event_count, + "nonTradeEventCount": non_trade_event_count, + "decodedEvents": decoded_event_evidence + }), + ); +} + +fn classify_from_known_program_matches( + context: &TransactionClassificationContext, + known_program_matches: &[KnownDexProgramMatch], +) -> TransactionClassificationDecision { + let first_match = &known_program_matches[0]; + let mut evidence_items = std::vec::Vec::new(); + for known_program_match in known_program_matches { + evidence_items.push(serde_json::json!({ + "protocolName": known_program_match.protocol_name, + "programId": known_program_match.program_id, + "instructionId": known_program_match.instruction_id, + "instructionIndex": known_program_match.instruction_index + })); + } + return build_decision( + "known_dex_program_unclassified", + Some(first_match.protocol_name.to_string()), + Some(first_match.program_id.to_string()), + 75, + "transaction has known DEX program instructions but no decoded DEX event".to_string(), + serde_json::json!({ + "transactionId": context.transaction_id, + "signature": context.transaction.signature, + "slot": context.transaction.slot, + "instructionCount": context.instructions.len(), + "decodedEventCount": context.decoded_events.len(), + "knownDexProgramMatches": evidence_items + }), + ); +} + +fn build_decision( + kind: &'static str, + primary_protocol: std::option::Option, + primary_program_id: std::option::Option, + confidence_level: i16, + reason: std::string::String, + evidence_value: serde_json::Value, +) -> TransactionClassificationDecision { + let evidence_json_result = serde_json::to_string(&evidence_value); + let evidence_json = match evidence_json_result { + Ok(evidence_json) => evidence_json, + Err(error) => { + return TransactionClassificationDecision { + kind: "unknown_or_unclassified", + primary_protocol: None, + primary_program_id: None, + confidence_level: 0, + reason: format!("cannot serialize classification evidence: {}", error), + evidence_json: "{}".to_string(), + }; + }, + }; + return TransactionClassificationDecision { + kind, + primary_protocol, + primary_program_id, + confidence_level, + reason, + evidence_json, + }; +} + +fn find_known_dex_program_matches( + instructions: &[crate::ChainInstructionDto], +) -> std::vec::Vec { + let mut matches = std::vec::Vec::new(); + for instruction in instructions { + let program_match = known_dex_program_match(instruction); + let program_match = match program_match { + Some(program_match) => program_match, + None => continue, + }; + matches.push(program_match); + } + + return matches; +} + +fn known_dex_program_match( + instruction: &crate::ChainInstructionDto, +) -> std::option::Option { + let program_id = match instruction.program_id.as_deref() { + Some(program_id) => program_id, + None => return None, + }; + let protocol_name = if program_id == crate::RAYDIUM_AMM_V4_PROGRAM_ID { + "raydium_amm_v4" + } else if program_id == crate::RAYDIUM_CPMM_PROGRAM_ID { + "raydium_cpmm" + } else if program_id == crate::RAYDIUM_CLMM_PROGRAM_ID { + "raydium_clmm" + } else if program_id == crate::RAYDIUM_LAUNCHLAB_PROGRAM_ID { + "raydium_launchlab" + } else if program_id == crate::RAYDIUM_AMM_ROUTING_PROGRAM_ID { + "raydium_router" + } else if program_id == crate::RAYDIUM_STABLE_SWAP_AMM_PROGRAM_ID { + "raydium_stable_swap" + } else if program_id == crate::PUMP_FUN_PROGRAM_ID { + "pump_fun" + } else if program_id == crate::PUMP_SWAP_PROGRAM_ID { + "pump_swap" + } else if program_id == crate::METEORA_DBC_PROGRAM_ID { + "meteora_dbc" + } else if program_id == crate::METEORA_DLMM_PROGRAM_ID { + "meteora_dlmm" + } else if program_id == crate::METEORA_DAMM_V1_PROGRAM_ID { + "meteora_damm_v1" + } else if program_id == crate::METEORA_DAMM_V2_PROGRAM_ID { + "meteora_damm_v2" + } else if program_id == crate::ORCA_WHIRLPOOLS_PROGRAM_ID { + "orca_whirlpools" + } else if program_id == crate::FLUXBEAM_PROGRAM_ID { + "fluxbeam" + } else if program_id == crate::DEXLAB_PROGRAM_ID { + "dexlab" + } else { + return None; + }; + return Some(KnownDexProgramMatch { + protocol_name, + program_id: program_id.to_string(), + instruction_id: instruction.id, + instruction_index: instruction.instruction_index, + }); +} + +#[cfg(test)] +mod tests { + fn test_instruction( + program_id: std::option::Option, + ) -> crate::ChainInstructionDto { + return crate::ChainInstructionDto::new( + 1, + None, + 0, + None, + program_id, + None, + None, + "[]".to_string(), + None, + None, + Some(serde_json::json!({}).to_string()), + ); + } + + fn test_transaction() -> crate::ChainTransactionDto { + let mut transaction = crate::ChainTransactionDto::new( + "signature_1".to_string(), + Some(123), + None, + Some("test".to_string()), + None, + None, + None, + serde_json::json!({}).to_string(), + ); + transaction.id = Some(1); + return transaction; + } + + #[test] + fn known_dex_program_ids_are_matched() { + let instruction = test_instruction(Some(crate::RAYDIUM_CPMM_PROGRAM_ID.to_string())); + let program_match = match super::known_dex_program_match(&instruction) { + Some(program_match) => program_match, + None => { + panic!("expected raydium_cpmm program match"); + }, + }; + assert_eq!(program_match.protocol_name, "raydium_cpmm"); + assert_eq!(program_match.program_id, crate::RAYDIUM_CPMM_PROGRAM_ID); + assert_eq!(program_match.instruction_index, 0); + } + + #[test] + fn unknown_program_id_is_not_matched() { + let instruction = + test_instruction(Some("UnknownProgram111111111111111111111111111111111".to_string())); + let program_match = super::known_dex_program_match(&instruction); + assert!(program_match.is_none()); + } + + #[test] + fn unknown_context_is_classified_as_unknown_or_unclassified() { + let transaction = test_transaction(); + let context = super::TransactionClassificationContext { + transaction, + transaction_id: 1, + instructions: std::vec::Vec::new(), + decoded_events: std::vec::Vec::new(), + }; + let decision = super::classify_transaction_context(&context); + assert_eq!(decision.kind, "unknown_or_unclassified"); + assert_eq!(decision.confidence_level, 25); + } +} diff --git a/kb_lib/src/tx_resolution.rs b/kb_lib/src/tx_resolution.rs index 8548f2d..76b7ddc 100644 --- a/kb_lib/src/tx_resolution.rs +++ b/kb_lib/src/tx_resolution.rs @@ -109,6 +109,7 @@ pub struct TransactionResolutionService { wallet_holding_observation_service: crate::WalletHoldingObservationService, pair_candle_aggregation_service: crate::PairCandleAggregationService, pair_analytic_signal_service: crate::PairAnalyticSignalService, + transaction_classification_service: crate::TransactionClassificationService, resolved_signatures: std::sync::Arc>>, } @@ -133,6 +134,8 @@ impl TransactionResolutionService { let pair_candle_aggregation_service = crate::PairCandleAggregationService::new(database.clone()); let pair_analytic_signal_service = crate::PairAnalyticSignalService::new(database.clone()); + let transaction_classification_service = + crate::TransactionClassificationService::new(database.clone()); return Self { http_pool, persistence, @@ -147,6 +150,7 @@ impl TransactionResolutionService { wallet_holding_observation_service, pair_candle_aggregation_service, pair_analytic_signal_service, + transaction_classification_service, resolved_signatures: std::sync::Arc::new(tokio::sync::Mutex::new( std::collections::HashSet::new(), )), @@ -400,6 +404,17 @@ impl TransactionResolutionService { Err(error) => return Err(error), }; let pair_analytic_signal_count = pair_analytic_signals.len(); + let transaction_classification_result = self + .transaction_classification_service + .classify_transaction_by_signature(request.signature.as_str()) + .await; + let transaction_classification = match transaction_classification_result { + Ok(transaction_classification) => transaction_classification, + Err(error) => return Err(error), + }; + let transaction_classification_id = transaction_classification.id; + let transaction_classification_kind = + transaction_classification.classification_kind.clone(); let payload = serde_json::json!({ "status": "resolved", "signature": request.signature.clone(), @@ -417,6 +432,8 @@ impl TransactionResolutionService { "tradeEventCount": trade_event_count, "pairCandleCount": pair_candle_count, "pairAnalyticSignalCount": pair_analytic_signal_count, + "transactionClassificationId": transaction_classification_id, + "transactionClassificationKind": transaction_classification_kind, "transaction": transaction_value }); let observation_id_result = self