diff --git a/CHANGELOG.md b/CHANGELOG.md index 710e00f..d0ea17e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -60,3 +60,4 @@ 0.7.27 - Validation multi-DEX et non-régression du pipeline sur Pump.fun, PumpSwap, Raydium CPMM et Raydium CLMM, avec corpus de tests, diagnostics de référence et garanties sur les événements non pricés 0.7.28 - Refactor DEX commun et verrouillage des invariants de normalisation : séparation des événements décodés, actionnables, trade candidates et candle candidates ; conservation des transactions failed comme traçables mais non actionnables ; ajout de la règle bloquante empêchant tout trade/candle candidate sans payload de montants exploitable, notamment pour le cas partiel `meteora_damm_v1.swap` sans base/quote amount. 0.7.29 - Ajout d’une matrice DEX commune (`dex_support_matrix`) utilisée par le catalogue DEX, la classification transactionnelle et l’enregistrement des protocol candidates ; ajout du profil de validation `0.7.29_multi_dex_matrix_baseline` exposant la matrice dans le rapport de validation ; préparation explicite des surfaces planifiées sans inventer de program ids non vérifiés. +0.7.30 - Ajout d’une taxonomie DEX plus fine pour les événements décodés : `eventLifecycleKind`, `eventActionability`, `nonTradeUseful`, compteurs diagnostics des événements non-trade utiles, trades non actionnables et classifications inconnues ; ajout du profil `0.7.30_non_trade_event_classification` sans modification volontaire de la matérialisation trade/candle. diff --git a/Cargo.toml b/Cargo.toml index 927c4af..8e9a43f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,7 +8,7 @@ members = [ ] [workspace.package] -version = "0.7.29" +version = "0.7.30" edition = "2024" license = "MIT" repository = "https://git.sasedev.com/Sasedev/khadhroony-bobobot" diff --git a/README.md b/README.md index fd2ddbb..3466443 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ `khadhroony-bobobot` est un workspace Rust destiné à la détection, au décodage, à l’analyse et, à terme, au trading semi-automatisé de tokens Solana. -Le README précédent décrivait surtout l’état `0.3.1`. Ce fichier reflète l’état de reprise autour de `0.7.29` : le socle transport HTTP/WS, la résolution transactionnelle, le modèle SQLite, plusieurs connecteurs DEX, les candles, les signaux analytiques, la validation locale et une matrice DEX commune existent déjà. +Le README précédent décrivait surtout l’état `0.3.1`. Ce fichier reflète l’état de reprise autour de `0.7.30` : le socle transport HTTP/WS, la résolution transactionnelle, le modèle SQLite, plusieurs connecteurs DEX, les candles, les signaux analytiques, la validation locale et une matrice DEX commune existent déjà. ## 1. Objectif @@ -31,7 +31,7 @@ Le workspace contient deux crates principales. La logique métier doit rester dans `kb_lib`. `kb_demo_app` doit rester une façade UI/Tauri et ne doit pas récupérer de logique Solana ou DEX profonde. -## 3. État actuel autour de `0.7.29` +## 3. État actuel autour de `0.7.30` ### 3.1. Socle stabilisé à ne pas refactorer maintenant @@ -97,6 +97,8 @@ La distinction importante est la suivante : Depuis `0.7.29`, la matrice de support DEX est portée par `kb_lib/src/dex_support_matrix.rs`. Elle centralise le code interne, la famille, la version, le type de surface, les program ids vérifiés localement, le statut de support, les capacités actuelles et les raisons de skip. +Depuis `0.7.30`, les événements décodés reçoivent aussi une classification plus fine : `eventLifecycleKind`, `eventActionability` et `nonTradeUseful`. Cette classification sert aux diagnostics et prépare la matérialisation future des événements non-trade sans alimenter directement les trades/candles. + | Code cible | Type | Statut `0.7.29` | Prochaine action | |---|---:|---|---| | `pump_fun` | Launch + bonding curve | partiel | verrouiller le rattachement mint initial -> pools migrés | diff --git a/ROADMAP.md b/ROADMAP.md index 1b8f378..9b59866 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -825,7 +825,20 @@ Matrice cible initiale : | `heaven` | launch + AMM candidat | planifié | corpus et séparation launch/swap | | `zora` | à vérifier | à vérifier | hors phasage actif avant preuve Solana | -### 6.062. Version `0.7.30` — Transactions inconnues et protocol candidates +### 6.062. Version `0.7.30` — Classification fine des événements DEX décodés +Objectif : préparer les événements non-trade utiles sans modifier la matérialisation trade/candle validée. + +Fait / à valider : + +- ajouter `DexEventLifecycleKind` pour distinguer `trade_swap`, `pool_creation`, `pair_creation`, `liquidity_add`, `liquidity_remove`, `position_open`, `position_close`, `migration`, `launch`, `mint`, `burn`, `fee_collection`, `reward`, `admin_config` et `unknown`, +- ajouter `DexEventActionability` pour distinguer `trade_candidate`, `non_actionable_trade`, `non_trade_useful`, `failed_transaction`, `informational` et `unknown`, +- enrichir les payloads décodés avec `eventLifecycleKind`, `eventActionability` et `nonTradeUseful`, +- exposer les compteurs diagnostics `decodedNonTradeUsefulEventCount`, `decodedNonActionableTradeEventCount` et `decodedUnknownEventCount`, +- ajouter un résumé diagnostic par catégorie / lifecycle / actionability, +- ajouter le profil `0.7.30_non_trade_event_classification`, +- ne pas matérialiser encore les événements non-trade dans leurs tables dédiées. + +### 6.063. Version `0.7.31` — Transactions inconnues et protocol candidates Objectif : ne plus perdre les transactions utiles qui ne correspondent pas encore à un DEX connu. À faire : @@ -838,7 +851,7 @@ Objectif : ne plus perdre les transactions utiles qui ne correspondent pas encor - permettre de promouvoir plus tard un protocol candidate vers un vrai DEX/surface sans perdre l’historique, - garantir que ces tables n’alimentent jamais directement les trades/candles. -### 6.063. Version `0.7.31` — Événements non-trade v1 : liquidité et cycle de vie pool +### 6.064. Version `0.7.32` — Événements non-trade v1 : liquidité et cycle de vie pool Objectif : exploiter les événements utiles à l’analyse et au trading semi-automatique sans les mélanger avec les swaps/candles. À faire : @@ -851,7 +864,7 @@ Objectif : exploiter les événements utiles à l’analyse et au trading semi-a - alimenter les diagnostics locaux avec les compteurs liquidité/lifecycle, - garantir qu’un événement de liquidité ou de cycle de vie ne produit jamais de candle directement. -### 6.064. Version `0.7.32` — Événements non-trade v2 : fees, rewards et administration +### 6.065. Version `0.7.33` — Événements non-trade v2 : fees, rewards et administration Objectif : conserver les événements utiles au risque, au scoring, à l’économie du pool et à la traçabilité opérationnelle. À faire : @@ -865,7 +878,7 @@ Objectif : conserver les événements utiles au risque, au scoring, à l’écon - rattacher ces événements aux transactions, decoded events, pools, paires et wallets observés lorsque les comptes le permettent, - documenter clairement que ces événements ne sont ni des trades ni des candles. -### 6.065. Version `0.7.33` — Meteora : DBC / DAMM v1 / DAMM v2 / DLMM +### 6.066. Version `0.7.34` — Meteora : DBC / DAMM v1 / DAMM v2 / DLMM Objectif : consolider Meteora comme famille multi-programmes au lieu de traiter chaque variante comme un cas isolé incomplet. À faire : @@ -1172,7 +1185,9 @@ Réalisé / à maintenir : - exposition de la matrice dans le rapport de validation local ; - aucune modification volontaire du comportement trade/candle validé en `0.7.28`. -À poursuivre en `0.7.30` : matérialisation contrôlée des événements non-trade utiles et des transactions inconnues/partielles, sans alimenter les trades/candles actionnables. +Validé en `0.7.30` : classification fine des événements décodés via `eventLifecycleKind`, `eventActionability` et `nonTradeUseful`, avec diagnostics associés et sans changement volontaire sur les trades/candles. + +À poursuivre en `0.7.31` : transactions inconnues et protocol candidates ; puis en `0.7.32` : matérialisation contrôlée des événements non-trade utiles. ## 11. Documentation et livrables de référence Le projet doit maintenir au minimum : diff --git a/kb_demo_app/frontend/demo_pipeline2.html b/kb_demo_app/frontend/demo_pipeline2.html index 9aedb0c..45273f0 100644 --- a/kb_demo_app/frontend/demo_pipeline2.html +++ b/kb_demo_app/frontend/demo_pipeline2.html @@ -166,7 +166,8 @@
diff --git a/kb_demo_app/frontend/ts/bindings/DemoPipeline2LocalDecodedEventDiagnosticSummary.ts b/kb_demo_app/frontend/ts/bindings/DemoPipeline2LocalDecodedEventDiagnosticSummary.ts index e9dc632..efb0958 100644 --- a/kb_demo_app/frontend/ts/bindings/DemoPipeline2LocalDecodedEventDiagnosticSummary.ts +++ b/kb_demo_app/frontend/ts/bindings/DemoPipeline2LocalDecodedEventDiagnosticSummary.ts @@ -16,6 +16,18 @@ eventKind: string, * Event category. */ eventCategory: string | null, +/** + * Event lifecycle kind. + */ +eventLifecycleKind: string | null, +/** + * Event actionability class. + */ +eventActionability: string | null, +/** + * Whether the event is useful but not trade/candle materialized. + */ +nonTradeUseful: boolean | null, /** * Trade candidate flag. */ diff --git a/kb_demo_app/frontend/ts/bindings/DemoPipeline2LocalEventClassificationDiagnosticSummary.ts b/kb_demo_app/frontend/ts/bindings/DemoPipeline2LocalEventClassificationDiagnosticSummary.ts new file mode 100644 index 0000000..62eebdf --- /dev/null +++ b/kb_demo_app/frontend/ts/bindings/DemoPipeline2LocalEventClassificationDiagnosticSummary.ts @@ -0,0 +1,38 @@ +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. + +/** + * Local decoded-event classification summary for the UI. + */ +export type DemoPipeline2LocalEventClassificationDiagnosticSummary = { +/** + * Event category. + */ +eventCategory: string, +/** + * Event lifecycle kind. + */ +eventLifecycleKind: string, +/** + * Event actionability class. + */ +eventActionability: string, +/** + * Whether the event is useful but not trade/candle materialized. + */ +nonTradeUseful: boolean, +/** + * Event count. + */ +eventCount: number, +/** + * Decoded trade candidate count. + */ +decodedTradeCandidateCount: number, +/** + * Decoded candle candidate count. + */ +decodedCandleCandidateCount: number, +/** + * Linked trade-event count. + */ +tradeEventCount: number, }; diff --git a/kb_demo_app/frontend/ts/bindings/DemoPipeline2LocalPipelineDiagnosticSummary.ts b/kb_demo_app/frontend/ts/bindings/DemoPipeline2LocalPipelineDiagnosticSummary.ts index c26c72e..7c399bc 100644 --- a/kb_demo_app/frontend/ts/bindings/DemoPipeline2LocalPipelineDiagnosticSummary.ts +++ b/kb_demo_app/frontend/ts/bindings/DemoPipeline2LocalPipelineDiagnosticSummary.ts @@ -2,6 +2,7 @@ import type { DemoPipeline2LocalDecodedEventDiagnosticSummary } from "./DemoPipeline2LocalDecodedEventDiagnosticSummary"; import type { DemoPipeline2LocalDexDiagnosticSummary } from "./DemoPipeline2LocalDexDiagnosticSummary"; import type { DemoPipeline2LocalDuplicateDecodedEventTradeDiagnosticSample } from "./DemoPipeline2LocalDuplicateDecodedEventTradeDiagnosticSample"; +import type { DemoPipeline2LocalEventClassificationDiagnosticSummary } from "./DemoPipeline2LocalEventClassificationDiagnosticSummary"; import type { DemoPipeline2LocalMissingTradeEventDiagnosticSample } from "./DemoPipeline2LocalMissingTradeEventDiagnosticSample"; import type { DemoPipeline2LocalMissingTradeEventReasonSummary } from "./DemoPipeline2LocalMissingTradeEventReasonSummary"; import type { DemoPipeline2LocalMultiTradeSignaturePairDiagnosticSample } from "./DemoPipeline2LocalMultiTradeSignaturePairDiagnosticSample"; @@ -37,6 +38,18 @@ decodedTradeCandidateCount: number, * Total decoded DEX candle candidates. */ decodedCandleCandidateCount: number, +/** + * Total decoded useful non-trade events. + */ +decodedNonTradeUsefulEventCount: number, +/** + * Total decoded swap-like events that are intentionally non-actionable. + */ +decodedNonActionableTradeEventCount: number, +/** + * Total decoded events with unknown classification. + */ +decodedUnknownEventCount: number, /** * Whether the local persisted pipeline has no blocking diagnostic issue. */ @@ -130,6 +143,10 @@ pairSummaries: Array, * Diagnostics grouped by decoded event kind. */ decodedEventSummaries: Array, +/** + * Diagnostics grouped by decoded event classification. + */ +eventClassificationSummaries: Array, /** * Missing trade events grouped by diagnostic reason. */ diff --git a/kb_demo_app/frontend/ts/bindings/DemoPipeline2LocalPipelineValidationReport.ts b/kb_demo_app/frontend/ts/bindings/DemoPipeline2LocalPipelineValidationReport.ts index 65b5bc4..cc74a4f 100644 --- a/kb_demo_app/frontend/ts/bindings/DemoPipeline2LocalPipelineValidationReport.ts +++ b/kb_demo_app/frontend/ts/bindings/DemoPipeline2LocalPipelineValidationReport.ts @@ -30,6 +30,18 @@ expectedDexCodes: Array, * Observed DEX codes found in diagnostics. */ observedDexCodes: Array, +/** + * Total decoded useful non-trade events. + */ +decodedNonTradeUsefulEventCount: number, +/** + * Total decoded swap-like events that are intentionally non-actionable. + */ +decodedNonActionableTradeEventCount: number, +/** + * Total decoded events with unknown classification. + */ +decodedUnknownEventCount: number, /** * Number of entries currently exposed by the DEX support matrix. */ diff --git a/kb_demo_app/frontend/ts/demo_pipeline2.ts b/kb_demo_app/frontend/ts/demo_pipeline2.ts index d27c7a9..6502f10 100644 --- a/kb_demo_app/frontend/ts/demo_pipeline2.ts +++ b/kb_demo_app/frontend/ts/demo_pipeline2.ts @@ -640,7 +640,7 @@ document.addEventListener("DOMContentLoaded", async () => { appendLogLine( logTextarea, - `[ui] local pipeline diagnostics completed: ${payload.summary.decodedEventCount.toString()} decoded, ${payload.summary.tradeEventCount.toString()} trades, ${payload.summary.pairCandleCount.toString()} candles, actionableMissing='${payload.summary.actionableMissingTradeEventCount.toString()}', nonActionablePairs='${payload.summary.nonActionablePairCount.toString()}', blocking='${payload.summary.blockingIssueCount.toString()}'`, + `[ui] local pipeline diagnostics completed: ${payload.summary.decodedEventCount.toString()} decoded, ${payload.summary.tradeEventCount.toString()} trades, ${payload.summary.pairCandleCount.toString()} candles, actionableMissing='${payload.summary.actionableMissingTradeEventCount.toString()}', nonActionablePairs='${payload.summary.nonActionablePairCount.toString()}', blocking='${payload.summary.blockingIssueCount.toString()}', nonTradeUseful='${payload.summary.decodedNonTradeUsefulEventCount.toString()}', nonActionableTrade='${payload.summary.decodedNonActionableTradeEventCount.toString()}', unknownEvents='${payload.summary.decodedUnknownEventCount.toString()}'`, ); } catch (error) { appendLogLine(logTextarea, `[ui] local pipeline diagnostics error: ${String(error)}`); @@ -667,7 +667,7 @@ document.addEventListener("DOMContentLoaded", async () => { appendLogLine( logTextarea, - `[ui] local pipeline validation completed: profile='${payload.run.validationProfileCode}' passed='${payload.run.validationPassed ? "yes" : "no"}' blocking='${payload.run.blockingIssueCount.toString()}' warnings='${payload.run.warningCount.toString()}' dexMatrix='${payload.run.report.dexSupportMatrixEntryCount.toString()}'`, + `[ui] local pipeline validation completed: profile='${payload.run.validationProfileCode}' passed='${payload.run.validationPassed ? "yes" : "no"}' blocking='${payload.run.blockingIssueCount.toString()}' warnings='${payload.run.warningCount.toString()}' dexMatrix='${payload.run.report.dexSupportMatrixEntryCount.toString()}' nonTradeUseful='${payload.run.report.decodedNonTradeUsefulEventCount.toString()}' nonActionableTrade='${payload.run.report.decodedNonActionableTradeEventCount.toString()}' unknownEvents='${payload.run.report.decodedUnknownEventCount.toString()}'`, ); } catch (error) { appendLogLine(logTextarea, `[ui] local pipeline validation error: ${String(error)}`); diff --git a/kb_demo_app/src/demo_pipeline2.rs b/kb_demo_app/src/demo_pipeline2.rs index a98cb0e..a116f5c 100644 --- a/kb_demo_app/src/demo_pipeline2.rs +++ b/kb_demo_app/src/demo_pipeline2.rs @@ -152,6 +152,15 @@ pub(crate) struct DemoPipeline2LocalPipelineValidationReport { pub expected_dex_codes: std::vec::Vec, /// Observed DEX codes found in diagnostics. pub observed_dex_codes: std::vec::Vec, + /// Total decoded useful non-trade events. + #[ts(type = "number")] + pub decoded_non_trade_useful_event_count: i64, + /// Total decoded swap-like events that are intentionally non-actionable. + #[ts(type = "number")] + pub decoded_non_actionable_trade_event_count: i64, + /// Total decoded events with unknown classification. + #[ts(type = "number")] + pub decoded_unknown_event_count: i64, /// Number of entries currently exposed by the DEX support matrix. #[ts(type = "number")] pub dex_support_matrix_entry_count: i64, @@ -253,6 +262,15 @@ pub(crate) struct DemoPipeline2LocalPipelineDiagnosticSummary { /// Total decoded DEX candle candidates. #[ts(type = "number")] pub decoded_candle_candidate_count: i64, + /// Total decoded useful non-trade events. + #[ts(type = "number")] + pub decoded_non_trade_useful_event_count: i64, + /// Total decoded swap-like events that are intentionally non-actionable. + #[ts(type = "number")] + pub decoded_non_actionable_trade_event_count: i64, + /// Total decoded events with unknown classification. + #[ts(type = "number")] + pub decoded_unknown_event_count: i64, /// Whether the local persisted pipeline has no blocking diagnostic issue. pub diagnostics_clean: bool, /// Number of blocking diagnostic issues. @@ -321,6 +339,9 @@ pub(crate) struct DemoPipeline2LocalPipelineDiagnosticSummary { pub pair_summaries: std::vec::Vec, /// Diagnostics grouped by decoded event kind. pub decoded_event_summaries: std::vec::Vec, + /// Diagnostics grouped by decoded event classification. + pub event_classification_summaries: + std::vec::Vec, /// Missing trade events grouped by diagnostic reason. pub missing_trade_event_reason_summaries: std::vec::Vec, @@ -440,6 +461,12 @@ pub(crate) struct DemoPipeline2LocalDecodedEventDiagnosticSummary { pub event_kind: std::string::String, /// Event category. pub event_category: std::option::Option, + /// Event lifecycle kind. + pub event_lifecycle_kind: std::option::Option, + /// Event actionability class. + pub event_actionability: std::option::Option, + /// Whether the event is useful but not trade/candle materialized. + pub non_trade_useful: std::option::Option, /// Trade candidate flag. pub trade_candidate: std::option::Option, /// Candle candidate flag. @@ -452,6 +479,36 @@ pub(crate) struct DemoPipeline2LocalDecodedEventDiagnosticSummary { pub trade_event_count: i64, } +/// Local decoded-event classification summary for the UI. +#[derive(Clone, Debug, serde::Serialize, TS)] +#[ts( + export, + export_to = "../frontend/ts/bindings/DemoPipeline2LocalEventClassificationDiagnosticSummary.ts" +)] +#[serde(rename_all = "camelCase")] +pub(crate) struct DemoPipeline2LocalEventClassificationDiagnosticSummary { + /// Event category. + pub event_category: std::string::String, + /// Event lifecycle kind. + pub event_lifecycle_kind: std::string::String, + /// Event actionability class. + pub event_actionability: std::string::String, + /// Whether the event is useful but not trade/candle materialized. + pub non_trade_useful: bool, + /// Event count. + #[ts(type = "number")] + pub event_count: i64, + /// Decoded trade candidate count. + #[ts(type = "number")] + pub decoded_trade_candidate_count: i64, + /// Decoded candle candidate count. + #[ts(type = "number")] + pub decoded_candle_candidate_count: i64, + /// Linked trade-event count. + #[ts(type = "number")] + pub trade_event_count: i64, +} + /// Local missing-trade-event reason summary for the UI. #[derive(Clone, Debug, serde::Serialize, TS)] #[ts( @@ -914,7 +971,7 @@ pub(crate) async fn demo_pipeline2_validate_local_pipeline( let service = kb_lib::LocalPipelineValidationService::new(database.clone()); let profile_code = match request { Some(request) => request.profile_code, - None => "0.7.29_multi_dex_matrix_baseline".to_string(), + None => "0.7.30_non_trade_event_classification".to_string(), }; let run_result = match profile_code.as_str() { "0.7.27" | "0.7.27_dexes_non_regression" => { @@ -926,6 +983,9 @@ pub(crate) async fn demo_pipeline2_validate_local_pipeline( "0.7.29" | "0.7.29_multi_dex_matrix_baseline" => { service.validate_v0_7_29_current_database().await }, + "0.7.30" | "0.7.30_non_trade_event_classification" => { + service.validate_v0_7_30_current_database().await + }, other => Err(kb_lib::Error::InvalidState(format!( "unsupported local pipeline validation profile: {other}" ))), @@ -1351,6 +1411,9 @@ fn demo_pipeline2_map_local_validation_report( warning_count: report.warning_count, expected_dex_codes: report.expected_dex_codes, observed_dex_codes: report.observed_dex_codes, + decoded_non_trade_useful_event_count: report.decoded_non_trade_useful_event_count, + decoded_non_actionable_trade_event_count: report.decoded_non_actionable_trade_event_count, + decoded_unknown_event_count: report.decoded_unknown_event_count, dex_support_matrix_entry_count: report.dex_support_matrix_entry_count, dex_support_matrix, issues, @@ -1410,6 +1473,14 @@ fn demo_pipeline2_map_local_diagnostics_summary( decoded_event_summaries .push(demo_pipeline2_map_local_decoded_event_diagnostic_summary(decoded_event_summary)); } + let mut event_classification_summaries = std::vec::Vec::new(); + for classification_summary in summary.event_classification_summaries { + event_classification_summaries.push( + demo_pipeline2_map_local_event_classification_diagnostic_summary( + classification_summary, + ), + ); + } let mut missing_trade_event_reason_summaries = std::vec::Vec::new(); for reason_summary in summary.missing_trade_event_reason_summaries { missing_trade_event_reason_summaries @@ -1450,6 +1521,9 @@ fn demo_pipeline2_map_local_diagnostics_summary( decoded_event_count: summary.decoded_event_count, decoded_trade_candidate_count: summary.decoded_trade_candidate_count, decoded_candle_candidate_count: summary.decoded_candle_candidate_count, + decoded_non_trade_useful_event_count: summary.decoded_non_trade_useful_event_count, + decoded_non_actionable_trade_event_count: summary.decoded_non_actionable_trade_event_count, + decoded_unknown_event_count: summary.decoded_unknown_event_count, diagnostics_clean: summary.diagnostics_clean, blocking_issue_count: summary.blocking_issue_count, missing_trade_event_count: summary.missing_trade_event_count, @@ -1479,6 +1553,7 @@ fn demo_pipeline2_map_local_diagnostics_summary( dex_summaries, pair_summaries, decoded_event_summaries, + event_classification_summaries, missing_trade_event_reason_summaries, non_actionable_pair_count: summary.non_actionable_pair_count, non_actionable_pair_summaries, @@ -1534,6 +1609,9 @@ fn demo_pipeline2_map_local_decoded_event_diagnostic_summary( protocol_name: summary.protocol_name, event_kind: summary.event_kind, event_category: summary.event_category, + event_lifecycle_kind: summary.event_lifecycle_kind, + event_actionability: summary.event_actionability, + non_trade_useful: summary.non_trade_useful, trade_candidate: summary.trade_candidate, candle_candidate: summary.candle_candidate, event_count: summary.event_count, @@ -1541,6 +1619,21 @@ fn demo_pipeline2_map_local_decoded_event_diagnostic_summary( } } +fn demo_pipeline2_map_local_event_classification_diagnostic_summary( + summary: kb_lib::LocalEventClassificationDiagnosticSummaryDto, +) -> DemoPipeline2LocalEventClassificationDiagnosticSummary { + DemoPipeline2LocalEventClassificationDiagnosticSummary { + event_category: summary.event_category, + event_lifecycle_kind: summary.event_lifecycle_kind, + event_actionability: summary.event_actionability, + non_trade_useful: summary.non_trade_useful, + event_count: summary.event_count, + decoded_trade_candidate_count: summary.decoded_trade_candidate_count, + decoded_candle_candidate_count: summary.decoded_candle_candidate_count, + trade_event_count: summary.trade_event_count, + } +} + fn demo_pipeline2_map_missing_trade_event_reason_summary( summary: kb_lib::LocalMissingTradeEventReasonSummaryDto, ) -> DemoPipeline2LocalMissingTradeEventReasonSummary { diff --git a/kb_lib/src/db.rs b/kb_lib/src/db.rs index 7a43bce..128bead 100644 --- a/kb_lib/src/db.rs +++ b/kb_lib/src/db.rs @@ -32,6 +32,7 @@ pub use dtos::LiquidityEventDto; pub use dtos::LocalDecodedEventDiagnosticSummaryDto; pub use dtos::LocalDexDiagnosticSummaryDto; pub use dtos::LocalDuplicateDecodedEventTradeDiagnosticSampleDto; +pub use dtos::LocalEventClassificationDiagnosticSummaryDto; pub use dtos::LocalMissingTradeEventDiagnosticSampleDto; pub use dtos::LocalMissingTradeEventReasonSummaryDto; pub use dtos::LocalMultiTradeSignaturePairDiagnosticSampleDto; @@ -144,6 +145,7 @@ pub use queries::query_liquidity_events_list_recent; pub use queries::query_liquidity_events_upsert; pub use queries::query_local_decoded_event_diagnostic_list_summaries; pub use queries::query_local_duplicate_decoded_event_trade_diagnostic_list_samples; +pub use queries::query_local_event_classification_diagnostic_list_summaries; pub use queries::query_local_missing_trade_event_diagnostic_list_samples; pub use queries::query_local_missing_trade_event_reason_list_summaries; pub use queries::query_local_multi_trade_signature_pair_diagnostic_list_samples; diff --git a/kb_lib/src/db/dtos.rs b/kb_lib/src/db/dtos.rs index 6b67fff..d9b0c3c 100644 --- a/kb_lib/src/db/dtos.rs +++ b/kb_lib/src/db/dtos.rs @@ -42,6 +42,7 @@ mod program_instruction_discriminator_summary; mod wallet_participation; pub(crate) use local_pipeline_diagnostics::LocalDecodedEventDiagnosticSummaryRow; +pub(crate) use local_pipeline_diagnostics::LocalEventClassificationDiagnosticSummaryRow; pub(crate) use local_pipeline_diagnostics::LocalDexDiagnosticSummaryRow; pub(crate) use local_pipeline_diagnostics::LocalDuplicateDecodedEventTradeDiagnosticSampleRow; pub(crate) use local_pipeline_diagnostics::LocalMissingTradeEventDiagnosticSampleRow; @@ -68,6 +69,7 @@ pub use launch_surface::LaunchSurfaceDto; pub use launch_surface_key::LaunchSurfaceKeyDto; pub use liquidity_event::LiquidityEventDto; pub use local_pipeline_diagnostics::LocalDecodedEventDiagnosticSummaryDto; +pub use local_pipeline_diagnostics::LocalEventClassificationDiagnosticSummaryDto; pub use local_pipeline_diagnostics::LocalDexDiagnosticSummaryDto; pub use local_pipeline_diagnostics::LocalDuplicateDecodedEventTradeDiagnosticSampleDto; pub use local_pipeline_diagnostics::LocalMissingTradeEventDiagnosticSampleDto; diff --git a/kb_lib/src/db/dtos/local_pipeline_diagnostics.rs b/kb_lib/src/db/dtos/local_pipeline_diagnostics.rs index b34bf97..7525c7f 100644 --- a/kb_lib/src/db/dtos/local_pipeline_diagnostics.rs +++ b/kb_lib/src/db/dtos/local_pipeline_diagnostics.rs @@ -17,6 +17,12 @@ pub struct LocalPipelineDiagnosticSummaryDto { pub decoded_trade_candidate_count: i64, /// Total decoded DEX candle candidates. pub decoded_candle_candidate_count: i64, + /// Total decoded useful non-trade events. + pub decoded_non_trade_useful_event_count: i64, + /// Total decoded swap-like events that are intentionally non-actionable. + pub decoded_non_actionable_trade_event_count: i64, + /// Total decoded events with unknown classification. + pub decoded_unknown_event_count: i64, /// Whether the local persisted pipeline has no blocking diagnostic issue. pub diagnostics_clean: bool, /// Number of blocking diagnostic issues. @@ -69,6 +75,9 @@ pub struct LocalPipelineDiagnosticSummaryDto { pub pair_summaries: std::vec::Vec, /// Diagnostics grouped by decoded event kind. pub decoded_event_summaries: std::vec::Vec, + /// Diagnostics grouped by decoded event category, lifecycle kind and actionability. + pub event_classification_summaries: + std::vec::Vec, /// Missing trade events grouped by diagnostic reason. pub missing_trade_event_reason_summaries: std::vec::Vec, @@ -157,6 +166,12 @@ pub struct LocalDecodedEventDiagnosticSummaryDto { pub event_kind: std::string::String, /// Event category. pub event_category: std::option::Option, + /// Event lifecycle kind. + pub event_lifecycle_kind: std::option::Option, + /// Event actionability class. + pub event_actionability: std::option::Option, + /// Whether payload says this event is a useful non-trade event. + pub non_trade_useful: std::option::Option, /// Whether payload says this event is a trade candidate. pub trade_candidate: std::option::Option, /// Whether payload says this event is a candle candidate. @@ -167,6 +182,27 @@ pub struct LocalDecodedEventDiagnosticSummaryDto { pub trade_event_count: i64, } +/// Local decoded-event classification summary. +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +pub struct LocalEventClassificationDiagnosticSummaryDto { + /// Event category. + pub event_category: std::string::String, + /// Event lifecycle kind. + pub event_lifecycle_kind: std::string::String, + /// Event actionability class. + pub event_actionability: std::string::String, + /// Whether payload says this event is a useful non-trade event. + pub non_trade_useful: bool, + /// Total decoded events in this classification group. + pub event_count: i64, + /// Total decoded trade candidates in this classification group. + pub decoded_trade_candidate_count: i64, + /// Total decoded candle candidates in this classification group. + pub decoded_candle_candidate_count: i64, + /// Total linked trade events in this classification group. + pub trade_event_count: i64, +} + /// Missing trade event diagnostics grouped by reason. #[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] pub struct LocalMissingTradeEventReasonSummaryDto { @@ -238,6 +274,12 @@ pub struct LocalPipelineDiagnosticCountersDto { pub decoded_trade_candidate_count: i64, /// Total decoded DEX candle candidates. pub decoded_candle_candidate_count: i64, + /// Total decoded useful non-trade events. + pub decoded_non_trade_useful_event_count: i64, + /// Total decoded swap-like events that are intentionally non-actionable. + pub decoded_non_actionable_trade_event_count: i64, + /// Total decoded events with unknown classification. + pub decoded_unknown_event_count: i64, /// Total decoded trade candidates without trade event, including ignored failed transactions. pub missing_trade_event_count: i64, /// Explicit alias for decoded trade candidates without linked trade event. @@ -289,6 +331,9 @@ pub(crate) struct LocalPipelineDiagnosticCountersRow { pub(crate) decoded_event_count: i64, pub(crate) decoded_trade_candidate_count: i64, pub(crate) decoded_candle_candidate_count: i64, + pub(crate) decoded_non_trade_useful_event_count: i64, + pub(crate) decoded_non_actionable_trade_event_count: i64, + pub(crate) decoded_unknown_event_count: i64, pub(crate) missing_trade_event_count: i64, pub(crate) decoded_trade_candidate_without_trade_event_count: i64, pub(crate) decoded_trade_candidate_without_trade_event_on_ok_transaction_count: i64, @@ -350,12 +395,28 @@ pub(crate) struct LocalDecodedEventDiagnosticSummaryRow { pub(crate) protocol_name: std::string::String, pub(crate) event_kind: std::string::String, pub(crate) event_category: std::option::Option, + pub(crate) event_lifecycle_kind: std::option::Option, + pub(crate) event_actionability: std::option::Option, + pub(crate) non_trade_useful: std::option::Option, pub(crate) trade_candidate: std::option::Option, pub(crate) candle_candidate: std::option::Option, pub(crate) event_count: i64, pub(crate) trade_event_count: i64, } +/// SQL row for local decoded-event classification diagnostics. +#[derive(Debug, Clone, sqlx::FromRow)] +pub(crate) struct LocalEventClassificationDiagnosticSummaryRow { + pub(crate) event_category: std::string::String, + pub(crate) event_lifecycle_kind: std::string::String, + pub(crate) event_actionability: std::string::String, + pub(crate) non_trade_useful: i64, + pub(crate) event_count: i64, + pub(crate) decoded_trade_candidate_count: i64, + pub(crate) decoded_candle_candidate_count: i64, + pub(crate) trade_event_count: i64, +} + /// Sample of a decoded trade candidate without linked trade event. #[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] pub struct LocalMissingTradeEventDiagnosticSampleDto { diff --git a/kb_lib/src/db/queries.rs b/kb_lib/src/db/queries.rs index 6c6e1de..ebe5492 100644 --- a/kb_lib/src/db/queries.rs +++ b/kb_lib/src/db/queries.rs @@ -83,6 +83,7 @@ pub use liquidity_event::query_liquidity_events_list_recent; pub use liquidity_event::query_liquidity_events_upsert; pub use local_pipeline_diagnostics::query_local_decoded_event_diagnostic_list_summaries; pub use local_pipeline_diagnostics::query_local_duplicate_decoded_event_trade_diagnostic_list_samples; +pub use local_pipeline_diagnostics::query_local_event_classification_diagnostic_list_summaries; pub use local_pipeline_diagnostics::query_local_missing_trade_event_diagnostic_list_samples; pub use local_pipeline_diagnostics::query_local_missing_trade_event_reason_list_summaries; pub use local_pipeline_diagnostics::query_local_multi_trade_signature_pair_diagnostic_list_samples; diff --git a/kb_lib/src/db/queries/local_pipeline_diagnostics.rs b/kb_lib/src/db/queries/local_pipeline_diagnostics.rs index f2997f4..f9055bb 100644 --- a/kb_lib/src/db/queries/local_pipeline_diagnostics.rs +++ b/kb_lib/src/db/queries/local_pipeline_diagnostics.rs @@ -26,6 +26,28 @@ SELECT FROM k_sol_dex_decoded_events WHERE json_extract(payload_json, '$.candleCandidate') = 1 ) AS decoded_candle_candidate_count, + ( + SELECT COUNT(*) + FROM k_sol_dex_decoded_events + WHERE COALESCE(json_extract(payload_json, '$.nonTradeUseful'), 0) = 1 + OR COALESCE(json_extract(payload_json, '$.eventActionability'), '') = 'non_trade_useful' + ) AS decoded_non_trade_useful_event_count, + ( + SELECT COUNT(*) + FROM k_sol_dex_decoded_events + WHERE COALESCE(json_extract(payload_json, '$.eventActionability'), '') = 'non_actionable_trade' + OR ( + COALESCE(json_extract(payload_json, '$.eventActionability'), '') = '' + AND COALESCE(json_extract(payload_json, '$.eventCategory'), '') = 'trade' + AND COALESCE(json_extract(payload_json, '$.tradeCandidate'), 0) = 0 + AND COALESCE(json_extract(payload_json, '$.transactionFailed'), 0) = 0 + ) + ) AS decoded_non_actionable_trade_event_count, + ( + SELECT COUNT(*) + FROM k_sol_dex_decoded_events + WHERE COALESCE(json_extract(payload_json, '$.eventCategory'), 'unknown') = 'unknown' + ) AS decoded_unknown_event_count, ( SELECT COUNT(*) FROM k_sol_dex_decoded_events dde @@ -250,6 +272,10 @@ SELECT decoded_event_count: row.decoded_event_count, decoded_trade_candidate_count: row.decoded_trade_candidate_count, decoded_candle_candidate_count: row.decoded_candle_candidate_count, + decoded_non_trade_useful_event_count: row.decoded_non_trade_useful_event_count, + decoded_non_actionable_trade_event_count: row + .decoded_non_actionable_trade_event_count, + decoded_unknown_event_count: row.decoded_unknown_event_count, missing_trade_event_count: row.missing_trade_event_count, decoded_trade_candidate_without_trade_event_count: row .decoded_trade_candidate_without_trade_event_count, @@ -512,6 +538,9 @@ SELECT dde.protocol_name AS protocol_name, dde.event_kind AS event_kind, json_extract(dde.payload_json, '$.eventCategory') AS event_category, + json_extract(dde.payload_json, '$.eventLifecycleKind') AS event_lifecycle_kind, + json_extract(dde.payload_json, '$.eventActionability') AS event_actionability, + json_extract(dde.payload_json, '$.nonTradeUseful') AS non_trade_useful, json_extract(dde.payload_json, '$.tradeCandidate') AS trade_candidate, json_extract(dde.payload_json, '$.candleCandidate') AS candle_candidate, COUNT(dde.id) AS event_count, @@ -522,6 +551,9 @@ GROUP BY dde.protocol_name, dde.event_kind, event_category, + event_lifecycle_kind, + event_actionability, + non_trade_useful, trade_candidate, candle_candidate ORDER BY @@ -546,6 +578,9 @@ ORDER BY protocol_name: row.protocol_name, event_kind: row.event_kind, event_category: row.event_category, + event_lifecycle_kind: row.event_lifecycle_kind, + event_actionability: row.event_actionability, + non_trade_useful: sqlite_bool_to_option(row.non_trade_useful), trade_candidate: sqlite_bool_to_option(row.trade_candidate), candle_candidate: sqlite_bool_to_option(row.candle_candidate), event_count: row.event_count, @@ -557,6 +592,68 @@ ORDER BY } } +/// Lists local decoded-event classification diagnostic summaries. +pub async fn query_local_event_classification_diagnostic_list_summaries( + database: &crate::Database, +) -> Result, crate::Error> { + match database.connection() { + crate::DatabaseConnection::Sqlite(pool) => { + let rows_result = sqlx::query_as::< + sqlx::Sqlite, + crate::db::dtos::LocalEventClassificationDiagnosticSummaryRow, + >( + r#" +SELECT + COALESCE(json_extract(dde.payload_json, '$.eventCategory'), 'unknown') AS event_category, + COALESCE(json_extract(dde.payload_json, '$.eventLifecycleKind'), 'unknown') AS event_lifecycle_kind, + COALESCE(json_extract(dde.payload_json, '$.eventActionability'), 'unknown') AS event_actionability, + CASE WHEN COALESCE(json_extract(dde.payload_json, '$.nonTradeUseful'), 0) = 1 THEN 1 ELSE 0 END AS non_trade_useful, + COUNT(dde.id) AS event_count, + COUNT(CASE WHEN COALESCE(json_extract(dde.payload_json, '$.tradeCandidate'), 0) = 1 THEN dde.id END) AS decoded_trade_candidate_count, + COUNT(CASE WHEN COALESCE(json_extract(dde.payload_json, '$.candleCandidate'), 0) = 1 THEN dde.id END) AS decoded_candle_candidate_count, + COUNT(te.id) AS trade_event_count +FROM k_sol_dex_decoded_events dde +LEFT JOIN k_sol_trade_events te ON te.decoded_event_id = dde.id +GROUP BY + event_category, + event_lifecycle_kind, + event_actionability, + non_trade_useful +ORDER BY + event_category, + event_lifecycle_kind, + event_actionability + "#, + ) + .fetch_all(pool) + .await; + let rows = match rows_result { + Ok(rows) => rows, + Err(error) => { + return Err(crate::Error::Db(format!( + "cannot list local decoded event classification summaries on sqlite: {}", + error + ))); + }, + }; + let mut summaries = std::vec::Vec::new(); + for row in rows { + summaries.push(crate::LocalEventClassificationDiagnosticSummaryDto { + event_category: row.event_category, + event_lifecycle_kind: row.event_lifecycle_kind, + event_actionability: row.event_actionability, + non_trade_useful: row.non_trade_useful != 0, + event_count: row.event_count, + decoded_trade_candidate_count: row.decoded_trade_candidate_count, + decoded_candle_candidate_count: row.decoded_candle_candidate_count, + trade_event_count: row.trade_event_count, + }); + } + return Ok(summaries); + }, + } +} + /// Lists missing trade events grouped by diagnostic reason. pub async fn query_local_missing_trade_event_reason_list_summaries( database: &crate::Database, diff --git a/kb_lib/src/dex_decoded_event_materialization.rs b/kb_lib/src/dex_decoded_event_materialization.rs index 726f419..2715f1f 100644 --- a/kb_lib/src/dex_decoded_event_materialization.rs +++ b/kb_lib/src/dex_decoded_event_materialization.rs @@ -160,6 +160,11 @@ fn prepare_payload_for_transaction_status( }, }; object.insert("transactionFailed".to_string(), serde_json::Value::Bool(true)); + object.insert( + "eventActionability".to_string(), + serde_json::Value::String("failed_transaction".to_string()), + ); + object.insert("nonTradeUseful".to_string(), serde_json::Value::Bool(false)); object.insert("tradeCandidate".to_string(), serde_json::Value::Bool(false)); object.insert("candleCandidate".to_string(), serde_json::Value::Bool(false)); object.insert( diff --git a/kb_lib/src/dex_event_classification.rs b/kb_lib/src/dex_event_classification.rs index 6614915..20b1927 100644 --- a/kb_lib/src/dex_event_classification.rs +++ b/kb_lib/src/dex_event_classification.rs @@ -42,6 +42,95 @@ impl DexEventCategory { } } +/// Fine-grained lifecycle kind assigned to one decoded DEX event kind. +#[derive(Debug, Copy, Clone, Eq, PartialEq, serde::Serialize, serde::Deserialize)] +pub enum DexEventLifecycleKind { + /// Swap-like trade event. + TradeSwap, + /// Pool creation or initialization event. + PoolCreation, + /// Pair creation event when it can be distinguished from pool creation. + PairCreation, + /// Liquidity deposit or add-liquidity event. + LiquidityAdd, + /// Liquidity withdraw or remove-liquidity event. + LiquidityRemove, + /// Concentrated-liquidity position open event. + PositionOpen, + /// Concentrated-liquidity position close event. + PositionClose, + /// Migration event, for example launch surface to AMM/CLMM/DLMM. + Migration, + /// Launch or bonding-curve initialization event. + Launch, + /// Token mint event detected through a DEX or launch surface decoder. + Mint, + /// Token burn event detected through a DEX or launch surface decoder. + Burn, + /// Fee collection event. + FeeCollection, + /// Reward or emission event. + Reward, + /// Administration, configuration or permission update event. + AdminConfig, + /// Event kind that is not classified yet. + Unknown, +} + +impl DexEventLifecycleKind { + /// Returns the stable string code persisted inside decoded payload metadata. + pub fn as_str(self) -> &'static str { + match self { + Self::TradeSwap => return "trade_swap", + Self::PoolCreation => return "pool_creation", + Self::PairCreation => return "pair_creation", + Self::LiquidityAdd => return "liquidity_add", + Self::LiquidityRemove => return "liquidity_remove", + Self::PositionOpen => return "position_open", + Self::PositionClose => return "position_close", + Self::Migration => return "migration", + Self::Launch => return "launch", + Self::Mint => return "mint", + Self::Burn => return "burn", + Self::FeeCollection => return "fee_collection", + Self::Reward => return "reward", + Self::AdminConfig => return "admin_config", + Self::Unknown => return "unknown", + } + } +} + +/// Stable actionability class assigned to one decoded DEX event. +#[derive(Debug, Copy, Clone, Eq, PartialEq, serde::Serialize, serde::Deserialize)] +pub enum DexEventActionability { + /// Direct swap-like event that can feed trade/candle materialization. + TradeCandidate, + /// Swap-like event detected but not materializable as trade/candle yet. + NonActionableTrade, + /// Useful non-trade event that should remain visible for future materialization. + NonTradeUseful, + /// Failed transaction event retained for diagnostics but never actionable. + FailedTransaction, + /// Classified event that is informational only for the current pipeline. + Informational, + /// Event that is not classified yet. + Unknown, +} + +impl DexEventActionability { + /// Returns the stable string code persisted inside decoded payload metadata. + pub fn as_str(self) -> &'static str { + match self { + Self::TradeCandidate => return "trade_candidate", + Self::NonActionableTrade => return "non_actionable_trade", + Self::NonTradeUseful => return "non_trade_useful", + Self::FailedTransaction => return "failed_transaction", + Self::Informational => return "informational", + Self::Unknown => return "unknown", + } + } +} + /// Classifies a DEX event kind into a stable business category. pub fn classify_dex_event_category(event_kind: &str) -> DexEventCategory { if is_dex_reward_event_kind(event_kind) { @@ -70,6 +159,95 @@ pub fn classify_dex_event_category_code(event_kind: &str) -> &'static str { return classify_dex_event_category(event_kind).as_str(); } +/// Classifies a DEX event kind into a fine-grained lifecycle kind. +pub fn classify_dex_event_lifecycle_kind(event_kind: &str) -> DexEventLifecycleKind { + if is_dex_token_burn_event_kind(event_kind) { + return DexEventLifecycleKind::Burn; + } + if is_dex_token_mint_event_kind(event_kind) { + return DexEventLifecycleKind::Mint; + } + if is_dex_migration_event_kind(event_kind) { + return DexEventLifecycleKind::Migration; + } + if is_dex_launch_event_kind(event_kind) { + return DexEventLifecycleKind::Launch; + } + if is_dex_pair_creation_event_kind(event_kind) { + return DexEventLifecycleKind::PairCreation; + } + if is_dex_pool_creation_event_kind(event_kind) { + return DexEventLifecycleKind::PoolCreation; + } + if is_dex_liquidity_add_event_kind(event_kind) { + return DexEventLifecycleKind::LiquidityAdd; + } + if is_dex_liquidity_remove_event_kind(event_kind) { + return DexEventLifecycleKind::LiquidityRemove; + } + if is_dex_position_open_event_kind(event_kind) { + return DexEventLifecycleKind::PositionOpen; + } + if is_dex_position_close_event_kind(event_kind) { + return DexEventLifecycleKind::PositionClose; + } + if is_dex_fee_event_kind(event_kind) { + return DexEventLifecycleKind::FeeCollection; + } + if is_dex_reward_event_kind(event_kind) { + return DexEventLifecycleKind::Reward; + } + if is_dex_admin_event_kind(event_kind) { + return DexEventLifecycleKind::AdminConfig; + } + if is_dex_trade_event_kind(event_kind) { + return DexEventLifecycleKind::TradeSwap; + } + return DexEventLifecycleKind::Unknown; +} + +/// Classifies a DEX event kind and returns the persisted lifecycle kind code. +pub fn classify_dex_event_lifecycle_kind_code(event_kind: &str) -> &'static str { + return classify_dex_event_lifecycle_kind(event_kind).as_str(); +} + +/// Classifies one decoded DEX event actionability from its kind and candidate flags. +pub fn classify_dex_event_actionability( + event_kind: &str, + trade_candidate: bool, + transaction_failed: bool, +) -> DexEventActionability { + if transaction_failed { + return DexEventActionability::FailedTransaction; + } + if trade_candidate { + return DexEventActionability::TradeCandidate; + } + if is_dex_trade_event_kind(event_kind) { + return DexEventActionability::NonActionableTrade; + } + let category = classify_dex_event_category(event_kind); + match category { + DexEventCategory::Liquidity => return DexEventActionability::NonTradeUseful, + DexEventCategory::Fee => return DexEventActionability::NonTradeUseful, + DexEventCategory::Reward => return DexEventActionability::NonTradeUseful, + DexEventCategory::PoolLifecycle => return DexEventActionability::NonTradeUseful, + DexEventCategory::Admin => return DexEventActionability::NonTradeUseful, + DexEventCategory::Trade => return DexEventActionability::NonActionableTrade, + DexEventCategory::Unknown => return DexEventActionability::Unknown, + } +} + +/// Classifies one decoded DEX event actionability and returns its persisted code. +pub fn classify_dex_event_actionability_code( + event_kind: &str, + trade_candidate: bool, + transaction_failed: bool, +) -> &'static str { + return classify_dex_event_actionability(event_kind, trade_candidate, transaction_failed) + .as_str(); +} + /// Returns true when the event kind represents a swap-like event. pub fn is_dex_trade_event_kind(event_kind: &str) -> bool { if event_kind.ends_with(".buy") { @@ -127,6 +305,50 @@ pub fn is_dex_liquidity_event_kind(event_kind: &str) -> bool { return false; } +/// Returns true for liquidity add-like DEX events. +pub fn is_dex_liquidity_add_event_kind(event_kind: &str) -> bool { + if event_kind.contains(".deposit") { + return true; + } + if event_kind.contains(".add_liquidity") { + return true; + } + if event_kind.contains(".increase_liquidity") { + return true; + } + return false; +} + +/// Returns true for liquidity remove-like DEX events. +pub fn is_dex_liquidity_remove_event_kind(event_kind: &str) -> bool { + if event_kind.contains(".withdraw") { + return true; + } + if event_kind.contains(".remove_liquidity") { + return true; + } + if event_kind.contains(".decrease_liquidity") { + return true; + } + return false; +} + +/// Returns true for concentrated-liquidity position open events. +pub fn is_dex_position_open_event_kind(event_kind: &str) -> bool { + if event_kind.contains(".open_position") { + return true; + } + return false; +} + +/// Returns true for concentrated-liquidity position close events. +pub fn is_dex_position_close_event_kind(event_kind: &str) -> bool { + if event_kind.contains(".close_position") { + return true; + } + return false; +} + /// Returns true for fee collection events. pub fn is_dex_fee_event_kind(event_kind: &str) -> bool { if event_kind.contains("collect_creator_fee") { @@ -155,8 +377,81 @@ pub fn is_dex_reward_event_kind(event_kind: &str) -> bool { return false; } -/// Returns true for pool creation, initialization or migration events. +/// Returns true for pool, pair, launch, mint, burn or migration lifecycle events. pub fn is_dex_pool_lifecycle_event_kind(event_kind: &str) -> bool { + if is_dex_pool_creation_event_kind(event_kind) { + return true; + } + if is_dex_pair_creation_event_kind(event_kind) { + return true; + } + if is_dex_launch_event_kind(event_kind) { + return true; + } + if is_dex_token_mint_event_kind(event_kind) { + return true; + } + if is_dex_token_burn_event_kind(event_kind) { + return true; + } + if is_dex_migration_event_kind(event_kind) { + return true; + } + return false; +} + +/// Returns true for launch or bonding-curve creation events. +pub fn is_dex_launch_event_kind(event_kind: &str) -> bool { + if event_kind.contains("pump_fun.create") { + return true; + } + if event_kind.contains(".launch") { + return true; + } + if event_kind.contains(".create_v2_token") { + return true; + } + if event_kind.contains(".create_bonding_curve") { + return true; + } + return false; +} + +/// Returns true for token mint events detected by DEX or launch-surface decoders. +pub fn is_dex_token_mint_event_kind(event_kind: &str) -> bool { + if event_kind.contains(".mint") { + return true; + } + if event_kind.contains(".token_mint") { + return true; + } + return false; +} + +/// Returns true for token burn events detected by DEX or launch-surface decoders. +pub fn is_dex_token_burn_event_kind(event_kind: &str) -> bool { + if event_kind.contains(".burn") { + return true; + } + if event_kind.contains(".token_burn") { + return true; + } + return false; +} + +/// Returns true for launch-surface or pool migration events. +pub fn is_dex_migration_event_kind(event_kind: &str) -> bool { + if event_kind.contains(".migrate") { + return true; + } + if event_kind.contains(".migration") { + return true; + } + return false; +} + +/// Returns true for pool creation or initialization events. +pub fn is_dex_pool_creation_event_kind(event_kind: &str) -> bool { if event_kind.contains(".initialize") { return true; } @@ -166,10 +461,18 @@ pub fn is_dex_pool_lifecycle_event_kind(event_kind: &str) -> bool { if event_kind.contains(".create_pool") { return true; } - if event_kind.contains(".create_v2_token") { + if event_kind.contains(".create_amm") { return true; } - if event_kind.contains(".migrate") { + return false; +} + +/// Returns true for pair creation events when they are distinguishable from pool creation. +pub fn is_dex_pair_creation_event_kind(event_kind: &str) -> bool { + if event_kind.contains(".create_pair") { + return true; + } + if event_kind.contains(".pair_create") { return true; } return false; @@ -264,8 +567,7 @@ pub fn enrich_dex_decoded_payload( payload_json: serde_json::Value, ) -> serde_json::Value { let event_category = classify_dex_event_category_code(event_kind); - let trade_candidate = is_dex_trade_event_kind(event_kind); - let candle_candidate = is_dex_candle_candidate_event_kind(event_kind); + let event_lifecycle_kind = classify_dex_event_lifecycle_kind_code(event_kind); let mut object = match payload_json { serde_json::Value::Object(object) => object, other => { @@ -274,14 +576,54 @@ pub fn enrich_dex_decoded_payload( object }, }; + let payload_snapshot = serde_json::Value::Object(object.clone()); + let explicit_trade_candidate = extract_top_level_bool_by_candidate_keys( + &payload_snapshot, + &["tradeCandidate", "trade_candidate"], + ); + let trade_candidate = match explicit_trade_candidate { + Some(trade_candidate) => trade_candidate, + None => is_dex_trade_event_kind(event_kind), + }; + let explicit_candle_candidate = extract_top_level_bool_by_candidate_keys( + &payload_snapshot, + &["candleCandidate", "candle_candidate"], + ); + let candle_candidate = match explicit_candle_candidate { + Some(candle_candidate) => candle_candidate, + None => { + if !trade_candidate { + false + } else { + is_dex_candle_candidate_event_kind(event_kind) + } + }, + }; + let transaction_failed = match extract_top_level_bool_by_candidate_keys( + &payload_snapshot, + &["transactionFailed", "transaction_failed"], + ) { + Some(transaction_failed) => transaction_failed, + None => false, + }; + let event_actionability = + classify_dex_event_actionability_code(event_kind, trade_candidate, transaction_failed); + let non_trade_useful = event_actionability == DexEventActionability::NonTradeUseful.as_str(); json_insert_string_if_missing(&mut object, "protocolName", protocol_name); json_insert_string_if_missing(&mut object, "eventKind", event_kind); json_insert_string_if_missing(&mut object, "eventCategory", event_category); + json_insert_string_if_missing(&mut object, "eventLifecycleKind", event_lifecycle_kind); + json_insert_string_if_missing(&mut object, "eventActionability", event_actionability); + json_insert_bool_if_missing(&mut object, "nonTradeUseful", non_trade_useful); json_insert_bool_if_missing(&mut object, "tradeCandidate", trade_candidate); json_insert_bool_if_missing(&mut object, "candleCandidate", candle_candidate); - json_insert_i64_if_missing(&mut object, "eventClassificationVersion", 1); + json_insert_i64_if_missing(&mut object, "eventClassificationVersion", 2); if !trade_candidate { - json_insert_string_if_missing(&mut object, "skipTradeReason", "non_trade_event"); + if is_dex_trade_event_kind(event_kind) { + json_insert_string_if_missing(&mut object, "skipTradeReason", "non_actionable_trade"); + } else { + json_insert_string_if_missing(&mut object, "skipTradeReason", "non_trade_event"); + } } else if !candle_candidate { json_insert_string_if_missing( &mut object, @@ -525,6 +867,35 @@ mod tests { ); } + #[test] + fn classifies_fine_grained_non_trade_lifecycle_kinds() { + assert_eq!( + super::classify_dex_event_lifecycle_kind_code("raydium_cpmm.initialize"), + "pool_creation" + ); + assert_eq!(super::classify_dex_event_lifecycle_kind_code("pump_fun.create"), "launch"); + assert_eq!( + super::classify_dex_event_lifecycle_kind_code("meteora_dbc.migrate"), + "migration" + ); + assert_eq!( + super::classify_dex_event_lifecycle_kind_code("raydium_clmm.increase_liquidity_v2"), + "liquidity_add" + ); + assert_eq!( + super::classify_dex_event_lifecycle_kind_code("raydium_clmm.decrease_liquidity_v2"), + "liquidity_remove" + ); + assert_eq!( + super::classify_dex_event_actionability_code( + "raydium_clmm.increase_liquidity_v2", + false, + false, + ), + "non_trade_useful" + ); + } + #[test] fn enriched_payload_keeps_existing_fields() { let payload_json = serde_json::json!({ @@ -558,6 +929,45 @@ mod tests { ); assert_eq!(object.get("tradeCandidate"), Some(&serde_json::Value::Bool(true))); assert_eq!(object.get("candleCandidate"), Some(&serde_json::Value::Bool(true))); + assert_eq!( + object.get("eventLifecycleKind"), + Some(&serde_json::Value::String("trade_swap".to_owned())) + ); + assert_eq!( + object.get("eventActionability"), + Some(&serde_json::Value::String("trade_candidate".to_owned())) + ); + } + + #[test] + fn enriched_non_trade_payload_is_visible_but_not_trade_candidate() { + let enriched_payload = super::enrich_dex_decoded_payload( + "raydium_clmm", + "raydium_clmm.increase_liquidity_v2", + serde_json::json!({}), + ); + let object_option = enriched_payload.as_object(); + let object = match object_option { + Some(object) => object, + None => { + panic!("expected enriched payload object"); + }, + }; + assert_eq!( + object.get("eventCategory"), + Some(&serde_json::Value::String("liquidity".to_owned())) + ); + assert_eq!( + object.get("eventLifecycleKind"), + Some(&serde_json::Value::String("liquidity_add".to_owned())) + ); + assert_eq!( + object.get("eventActionability"), + Some(&serde_json::Value::String("non_trade_useful".to_owned())) + ); + assert_eq!(object.get("nonTradeUseful"), Some(&serde_json::Value::Bool(true))); + assert_eq!(object.get("tradeCandidate"), Some(&serde_json::Value::Bool(false))); + assert_eq!(object.get("candleCandidate"), Some(&serde_json::Value::Bool(false))); } #[test] diff --git a/kb_lib/src/lib.rs b/kb_lib/src/lib.rs index 899dab2..ac3e10d 100644 --- a/kb_lib/src/lib.rs +++ b/kb_lib/src/lib.rs @@ -347,6 +347,8 @@ pub use db::LocalDecodedEventDiagnosticSummaryDto; pub use db::LocalDexDiagnosticSummaryDto; /// Sample of duplicated trade rows grouped by decoded event id. pub use db::LocalDuplicateDecodedEventTradeDiagnosticSampleDto; +/// Local decoded-event classification diagnostics summary. +pub use db::LocalEventClassificationDiagnosticSummaryDto; /// Sample of a decoded trade candidate without linked trade event. pub use db::LocalMissingTradeEventDiagnosticSampleDto; /// Missing trade event diagnostics grouped by reason. @@ -559,6 +561,8 @@ pub use db::query_liquidity_events_upsert; pub use db::query_local_decoded_event_diagnostic_list_summaries; /// Lists samples of duplicated trade rows by decoded event id. pub use db::query_local_duplicate_decoded_event_trade_diagnostic_list_samples; +/// Lists local decoded-event classification diagnostic summaries. +pub use db::query_local_event_classification_diagnostic_list_summaries; /// Lists samples of decoded trade candidates without linked trade event. pub use db::query_local_missing_trade_event_diagnostic_list_samples; /// Lists missing trade events grouped by diagnostic reason. @@ -831,12 +835,24 @@ pub use dex_decode::DexDecodeService; pub use dex_detect::DexDetectService; /// Result of one business-level DEX pool detection. pub use dex_detect::DexPoolDetectionResult; +/// Stable DEX event actionability class. +pub use dex_event_classification::DexEventActionability; /// Stable DEX event business category. pub use dex_event_classification::DexEventCategory; +/// Fine-grained DEX event lifecycle kind. +pub use dex_event_classification::DexEventLifecycleKind; +/// Classifies a DEX event into an actionability class. +pub use dex_event_classification::classify_dex_event_actionability; +/// Classifies a DEX event into an actionability class and returns its persisted code. +pub use dex_event_classification::classify_dex_event_actionability_code; /// Classifies a DEX event kind into a stable category. pub use dex_event_classification::classify_dex_event_category; /// Classifies a DEX event kind and returns its persisted category code. pub use dex_event_classification::classify_dex_event_category_code; +/// Classifies a DEX event kind into a fine-grained lifecycle kind. +pub use dex_event_classification::classify_dex_event_lifecycle_kind; +/// Classifies a DEX event kind into a fine-grained lifecycle kind code. +pub use dex_event_classification::classify_dex_event_lifecycle_kind_code; /// Enriches and serializes a decoded DEX payload. pub use dex_event_classification::enrich_and_serialize_dex_decoded_payload; /// Enriches a decoded DEX payload with classification metadata. @@ -853,12 +869,32 @@ pub use dex_event_classification::is_dex_admin_event_kind; pub use dex_event_classification::is_dex_candle_candidate_event_kind; /// Returns true for fee collection DEX events. pub use dex_event_classification::is_dex_fee_event_kind; +/// Returns true for launch or bonding-curve creation DEX events. +pub use dex_event_classification::is_dex_launch_event_kind; +/// Returns true for liquidity add-like DEX events. +pub use dex_event_classification::is_dex_liquidity_add_event_kind; /// Returns true for liquidity lifecycle DEX events. pub use dex_event_classification::is_dex_liquidity_event_kind; +/// Returns true for liquidity remove-like DEX events. +pub use dex_event_classification::is_dex_liquidity_remove_event_kind; +/// Returns true for migration DEX events. +pub use dex_event_classification::is_dex_migration_event_kind; +/// Returns true for pair creation DEX events. +pub use dex_event_classification::is_dex_pair_creation_event_kind; +/// Returns true for pool creation DEX events. +pub use dex_event_classification::is_dex_pool_creation_event_kind; /// Returns true for pool lifecycle DEX events. pub use dex_event_classification::is_dex_pool_lifecycle_event_kind; +/// Returns true for position close DEX events. +pub use dex_event_classification::is_dex_position_close_event_kind; +/// Returns true for position open DEX events. +pub use dex_event_classification::is_dex_position_open_event_kind; /// Returns true for reward or emission DEX events. pub use dex_event_classification::is_dex_reward_event_kind; +/// Returns true for token burn DEX events. +pub use dex_event_classification::is_dex_token_burn_event_kind; +/// Returns true for token mint DEX events. +pub use dex_event_classification::is_dex_token_mint_event_kind; /// Returns true for swap-like DEX events. pub use dex_event_classification::is_dex_trade_event_kind; /// Static DEX support matrix entry. diff --git a/kb_lib/src/local_pipeline_diagnostics.rs b/kb_lib/src/local_pipeline_diagnostics.rs index 5e1d8d2..24033e9 100644 --- a/kb_lib/src/local_pipeline_diagnostics.rs +++ b/kb_lib/src/local_pipeline_diagnostics.rs @@ -42,6 +42,15 @@ impl LocalPipelineDiagnosticsService { Ok(decoded_event_summaries) => decoded_event_summaries, Err(error) => return Err(error), }; + let event_classification_summaries_result = + crate::query_local_event_classification_diagnostic_list_summaries( + self.database.as_ref(), + ) + .await; + let event_classification_summaries = match event_classification_summaries_result { + Ok(summaries) => summaries, + Err(error) => return Err(error), + }; let missing_trade_event_reason_summaries_result = crate::query_local_missing_trade_event_reason_list_summaries(self.database.as_ref()) .await; @@ -123,6 +132,10 @@ impl LocalPipelineDiagnosticsService { decoded_event_count: counters.decoded_event_count, decoded_trade_candidate_count: counters.decoded_trade_candidate_count, decoded_candle_candidate_count: counters.decoded_candle_candidate_count, + decoded_non_trade_useful_event_count: counters.decoded_non_trade_useful_event_count, + decoded_non_actionable_trade_event_count: counters + .decoded_non_actionable_trade_event_count, + decoded_unknown_event_count: counters.decoded_unknown_event_count, diagnostics_clean, blocking_issue_count, missing_trade_event_count: counters.missing_trade_event_count, @@ -152,6 +165,7 @@ impl LocalPipelineDiagnosticsService { dex_summaries, pair_summaries, decoded_event_summaries, + event_classification_summaries, missing_trade_event_reason_summaries, non_actionable_pair_count: counters.non_actionable_pair_count, non_actionable_pair_summaries, diff --git a/kb_lib/src/local_pipeline_validation.rs b/kb_lib/src/local_pipeline_validation.rs index dcd1899..793665e 100644 --- a/kb_lib/src/local_pipeline_validation.rs +++ b/kb_lib/src/local_pipeline_validation.rs @@ -64,7 +64,7 @@ impl LocalPipelineValidationConfig { /// Builds the strict validation config for `0.7.27` non-regression runs. pub fn v0_7_27_multi_dex_non_regression() -> Self { return Self { - profile_code: "0.7.27_multi_dex_non_regression (obsolete)".to_string(), + profile_code: "0.7.27_multi_dex_non_regression".to_string(), expected_dex_codes: vec![ "pump_fun".to_string(), "pump_swap".to_string(), @@ -146,6 +146,17 @@ impl LocalPipelineValidationConfig { require_candles_per_dex: false, }; } + + /// Builds the `0.7.30` non-trade event classification validation config. + /// + /// This profile keeps the `0.7.29` trade/candle checks and exposes the new + /// decoded-event classification counters. Non-trade events are intentionally + /// observable but not blocking until their dedicated materializers are added. + pub fn v0_7_30_non_trade_event_classification() -> Self { + let mut config = Self::v0_7_29_multi_dex_matrix_baseline(); + config.profile_code = "0.7.30_non_trade_event_classification".to_string(); + return config; + } } /// A single local pipeline validation issue. @@ -178,6 +189,12 @@ pub struct LocalPipelineValidationReportDto { pub expected_dex_codes: std::vec::Vec, /// Observed DEX codes found in diagnostics. pub observed_dex_codes: std::vec::Vec, + /// Total decoded useful non-trade events. + pub decoded_non_trade_useful_event_count: i64, + /// Total decoded swap-like events that are intentionally non-actionable. + pub decoded_non_actionable_trade_event_count: i64, + /// Total decoded events with unknown classification. + pub decoded_unknown_event_count: i64, /// Number of entries currently exposed by the DEX support matrix. pub dex_support_matrix_entry_count: i64, /// DEX support matrix snapshot exposed with the validation report. @@ -268,6 +285,14 @@ impl LocalPipelineValidationService { let config = crate::LocalPipelineValidationConfig::v0_7_29_multi_dex_matrix_baseline(); return self.validate_current_database(&config).await; } + + /// Diagnoses the current database with the `0.7.30` non-trade classification profile. + pub async fn validate_v0_7_30_current_database( + &self, + ) -> Result { + let config = crate::LocalPipelineValidationConfig::v0_7_30_non_trade_event_classification(); + return self.validate_current_database(&config).await; + } } /// Validates a diagnostics summary without performing database access. @@ -427,6 +452,10 @@ pub fn validate_local_pipeline_diagnostics_summary( warning_count, expected_dex_codes, observed_dex_codes, + decoded_non_trade_useful_event_count: summary.decoded_non_trade_useful_event_count, + decoded_non_actionable_trade_event_count: summary + .decoded_non_actionable_trade_event_count, + decoded_unknown_event_count: summary.decoded_unknown_event_count, dex_support_matrix_entry_count: crate::dex_support_matrix_entries().len() as i64, dex_support_matrix: crate::dex_support_matrix_entry_dtos(), issues, @@ -470,6 +499,9 @@ mod tests { decoded_event_count: 216, decoded_trade_candidate_count: 216, decoded_candle_candidate_count: 216, + decoded_non_trade_useful_event_count: 0, + decoded_non_actionable_trade_event_count: 0, + decoded_unknown_event_count: 0, diagnostics_clean: true, blocking_issue_count: 0, missing_trade_event_count: 6, @@ -536,6 +568,7 @@ mod tests { ], pair_summaries: vec![], decoded_event_summaries: vec![], + event_classification_summaries: vec![], missing_trade_event_reason_summaries: vec![], non_actionable_pair_summaries: vec![], missing_trade_event_samples: vec![], @@ -610,6 +643,33 @@ mod tests { assert!(report.expected_dex_codes.contains(&"meteora_damm_v1".to_string())); } + #[test] + fn validation_accepts_0_7_30_non_trade_classification_summary() { + let mut summary = make_0_7_28_summary_with_meteora(); + summary.decoded_non_trade_useful_event_count = 3; + summary.decoded_non_actionable_trade_event_count = 1; + summary.decoded_unknown_event_count = 0; + summary.event_classification_summaries.push( + crate::LocalEventClassificationDiagnosticSummaryDto { + event_category: "pool_lifecycle".to_string(), + event_lifecycle_kind: "pool_creation".to_string(), + event_actionability: "non_trade_useful".to_string(), + non_trade_useful: true, + event_count: 3, + decoded_trade_candidate_count: 0, + decoded_candle_candidate_count: 0, + trade_event_count: 0, + }, + ); + let config = crate::LocalPipelineValidationConfig::v0_7_30_non_trade_event_classification(); + let report = crate::validate_local_pipeline_diagnostics_summary(&summary, &config); + assert!(report.validation_passed); + assert_eq!(report.validation_profile_code, "0.7.30_non_trade_event_classification"); + assert_eq!(report.decoded_non_trade_useful_event_count, 3); + assert_eq!(report.decoded_non_actionable_trade_event_count, 1); + assert_eq!(report.decoded_unknown_event_count, 0); + } + #[test] fn validation_report_exposes_dex_support_matrix() { let summary = make_0_7_28_summary_with_meteora();