0.7.24-pre.0
This commit is contained in:
@@ -134,6 +134,7 @@ pub use queries::list_recent_onchain_observations;
|
||||
pub use queries::list_recent_swaps;
|
||||
pub use queries::list_recent_token_burn_events;
|
||||
pub use queries::list_recent_token_mint_events;
|
||||
pub use queries::list_tokens;
|
||||
pub use queries::list_trade_events_by_pair_id;
|
||||
pub use queries::list_trade_events_by_transaction_id;
|
||||
pub use queries::list_wallet_holdings_by_wallet_id;
|
||||
|
||||
@@ -109,6 +109,7 @@ pub use pool_token::upsert_pool_token;
|
||||
pub use swap::list_recent_swaps;
|
||||
pub use swap::upsert_swap;
|
||||
pub use token::get_token_by_mint;
|
||||
pub use token::list_tokens;
|
||||
pub use token::upsert_token;
|
||||
pub use token_burn_event::list_recent_token_burn_events;
|
||||
pub use token_burn_event::upsert_token_burn_event;
|
||||
|
||||
@@ -119,3 +119,51 @@ LIMIT 1
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/// Lists all normalized token rows ordered by mint.
|
||||
pub async fn list_tokens(
|
||||
database: &crate::KbDatabase,
|
||||
) -> Result<std::vec::Vec<crate::KbTokenDto>, crate::KbError> {
|
||||
match database.connection() {
|
||||
crate::KbDatabaseConnection::Sqlite(pool) => {
|
||||
let query_result = sqlx::query_as::<sqlx::Sqlite, crate::KbTokenEntity>(
|
||||
r#"
|
||||
SELECT
|
||||
id,
|
||||
mint,
|
||||
symbol,
|
||||
name,
|
||||
decimals,
|
||||
token_program,
|
||||
is_quote_token,
|
||||
first_seen_at,
|
||||
updated_at
|
||||
FROM kb_tokens
|
||||
ORDER BY mint ASC, id ASC
|
||||
"#,
|
||||
)
|
||||
.fetch_all(pool)
|
||||
.await;
|
||||
let entities = match query_result {
|
||||
Ok(entities) => entities,
|
||||
Err(error) => {
|
||||
return Err(crate::KbError::Db(format!(
|
||||
"cannot list kb_tokens on sqlite: {}",
|
||||
error
|
||||
)));
|
||||
}
|
||||
};
|
||||
let mut dtos = std::vec::Vec::new();
|
||||
for entity in entities {
|
||||
let dto_result = crate::KbTokenDto::try_from(entity);
|
||||
let dto = match dto_result {
|
||||
Ok(dto) => dto,
|
||||
Err(error) => return Err(error),
|
||||
};
|
||||
dtos.push(dto);
|
||||
}
|
||||
Ok(dtos)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -33,6 +33,10 @@ INSERT INTO kb_trade_events (
|
||||
)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT(decoded_event_id) DO UPDATE SET
|
||||
base_amount_raw = COALESCE(excluded.base_amount_raw, kb_trade_events.base_amount_raw),
|
||||
quote_amount_raw = COALESCE(excluded.quote_amount_raw, kb_trade_events.quote_amount_raw),
|
||||
price_quote_per_base = COALESCE(excluded.price_quote_per_base, kb_trade_events.price_quote_per_base),
|
||||
payload_json = excluded.payload_json,
|
||||
updated_at = excluded.updated_at
|
||||
"#,
|
||||
)
|
||||
|
||||
@@ -839,6 +839,25 @@ impl KbDexDetectService {
|
||||
Ok(base_token_id) => base_token_id,
|
||||
Err(error) => return Err(error),
|
||||
};
|
||||
let payload_value_result = kb_parse_payload_json(decoded_event.payload_json.as_str());
|
||||
let payload_value = match payload_value_result {
|
||||
Ok(payload_value) => payload_value,
|
||||
Err(error) => return Err(error),
|
||||
};
|
||||
let vault_addresses = kb_extract_pump_swap_vault_addresses(&payload_value);
|
||||
let token_a_vault_address = vault_addresses.0;
|
||||
let token_b_vault_address = vault_addresses.1;
|
||||
|
||||
let base_vault_address = if base_is_token_a {
|
||||
token_a_vault_address.clone()
|
||||
} else {
|
||||
token_b_vault_address.clone()
|
||||
};
|
||||
let quote_vault_address = if base_is_token_a {
|
||||
token_b_vault_address.clone()
|
||||
} else {
|
||||
token_a_vault_address.clone()
|
||||
};
|
||||
let quote_token_id_result = self.ensure_token(quote_mint.as_str()).await;
|
||||
let quote_token_id = match quote_token_id_result {
|
||||
Ok(quote_token_id) => quote_token_id,
|
||||
@@ -920,7 +939,7 @@ impl KbDexDetectService {
|
||||
pool_id,
|
||||
base_token_id,
|
||||
crate::KbPoolTokenRole::Base,
|
||||
None,
|
||||
base_vault_address,
|
||||
Some(0),
|
||||
),
|
||||
)
|
||||
@@ -934,7 +953,7 @@ impl KbDexDetectService {
|
||||
pool_id,
|
||||
quote_token_id,
|
||||
crate::KbPoolTokenRole::Quote,
|
||||
None,
|
||||
quote_vault_address,
|
||||
Some(1),
|
||||
),
|
||||
)
|
||||
@@ -961,11 +980,6 @@ impl KbDexDetectService {
|
||||
}
|
||||
}
|
||||
};
|
||||
let payload_value_result = kb_parse_payload_json(decoded_event.payload_json.as_str());
|
||||
let payload_value = match payload_value_result {
|
||||
Ok(payload_value) => payload_value,
|
||||
Err(error) => return Err(error),
|
||||
};
|
||||
if created_pool {
|
||||
let signal_result = self
|
||||
.record_detection_signal(
|
||||
@@ -2845,6 +2859,46 @@ fn kb_parse_payload_json(payload_json: &str) -> Result<serde_json::Value, crate:
|
||||
}
|
||||
}
|
||||
|
||||
fn kb_extract_string_from_array_index(
|
||||
values: &[serde_json::Value],
|
||||
index: usize,
|
||||
) -> std::option::Option<std::string::String> {
|
||||
if index >= values.len() {
|
||||
return None;
|
||||
}
|
||||
let value = &values[index];
|
||||
let text_option = value.as_str();
|
||||
let text = match text_option {
|
||||
Some(text) => text.trim(),
|
||||
None => return None,
|
||||
};
|
||||
if text.is_empty() {
|
||||
return None;
|
||||
}
|
||||
Some(text.to_string())
|
||||
}
|
||||
|
||||
fn kb_extract_pump_swap_vault_addresses(
|
||||
payload_value: &serde_json::Value,
|
||||
) -> (
|
||||
std::option::Option<std::string::String>,
|
||||
std::option::Option<std::string::String>,
|
||||
) {
|
||||
let accounts_option = payload_value.get("accounts");
|
||||
let accounts = match accounts_option {
|
||||
Some(accounts) => accounts,
|
||||
None => return (None, None),
|
||||
};
|
||||
let accounts_array_option = accounts.as_array();
|
||||
let accounts_array = match accounts_array_option {
|
||||
Some(accounts_array) => accounts_array,
|
||||
None => return (None, None),
|
||||
};
|
||||
let token_a_vault_address = kb_extract_string_from_array_index(accounts_array, 7);
|
||||
let token_b_vault_address = kb_extract_string_from_array_index(accounts_array, 8);
|
||||
(token_a_vault_address, token_b_vault_address)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
async fn make_database() -> std::sync::Arc<crate::KbDatabase> {
|
||||
|
||||
@@ -180,6 +180,7 @@ pub use db::list_recent_onchain_observations;
|
||||
pub use db::list_recent_swaps;
|
||||
pub use db::list_recent_token_burn_events;
|
||||
pub use db::list_recent_token_mint_events;
|
||||
pub use db::list_tokens;
|
||||
pub use db::list_trade_events_by_pair_id;
|
||||
pub use db::list_trade_events_by_transaction_id;
|
||||
pub use db::list_wallet_holdings_by_wallet_id;
|
||||
@@ -306,6 +307,7 @@ pub use pool_origin::KbPoolOriginService;
|
||||
pub use solana_pubsub_ws::KbSolanaWsTypedNotification;
|
||||
pub use solana_pubsub_ws::parse_kb_solana_ws_typed_notification;
|
||||
pub use solana_pubsub_ws::parse_kb_solana_ws_typed_notification_from_event;
|
||||
pub use token_backfill::KbPoolBackfillResult;
|
||||
pub use token_backfill::KbTokenBackfillResult;
|
||||
pub use token_backfill::KbTokenBackfillService;
|
||||
pub use tracing::KbTracingGuard;
|
||||
|
||||
@@ -33,6 +33,33 @@ pub struct KbTokenBackfillResult {
|
||||
pub trade_event_count: usize,
|
||||
}
|
||||
|
||||
/// One pool-backfill result summary.
|
||||
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
|
||||
pub struct KbPoolBackfillResult {
|
||||
/// Input pool address.
|
||||
pub pool_address: std::string::String,
|
||||
/// Number of signatures returned directly for the pool address.
|
||||
pub pool_signature_count: usize,
|
||||
/// Number of unique signatures processed during this run.
|
||||
pub unique_signature_count: usize,
|
||||
/// Number of transactions resolved through HTTP during this run.
|
||||
pub resolved_transaction_count: usize,
|
||||
/// Number of signatures whose `getTransaction` lookup returned `null`.
|
||||
pub missing_transaction_count: usize,
|
||||
/// Total number of decoded DEX events replayed during this run.
|
||||
pub decoded_event_count: usize,
|
||||
/// Total number of DEX detection results produced during this run.
|
||||
pub detection_count: usize,
|
||||
/// Total number of launch-attribution results produced during this run.
|
||||
pub launch_attribution_count: usize,
|
||||
/// Total number of pool-origin results produced during this run.
|
||||
pub pool_origin_count: usize,
|
||||
/// Total number of wallet-participation observations produced during this run.
|
||||
pub wallet_participation_count: usize,
|
||||
/// Total number of trade-aggregation results produced during this run.
|
||||
pub trade_event_count: usize,
|
||||
}
|
||||
|
||||
/// Historical token backfill service.
|
||||
///
|
||||
/// This service reuses the existing transaction projection and downstream
|
||||
@@ -310,11 +337,9 @@ impl KbTokenBackfillService {
|
||||
trade_event_count: 0,
|
||||
});
|
||||
}
|
||||
let existing_transaction_result = crate::get_chain_transaction_by_signature(
|
||||
self.database.as_ref(),
|
||||
signature.as_str(),
|
||||
)
|
||||
.await;
|
||||
let existing_transaction_result =
|
||||
crate::get_chain_transaction_by_signature(self.database.as_ref(), signature.as_str())
|
||||
.await;
|
||||
let existing_transaction_option = match existing_transaction_result {
|
||||
Ok(existing_transaction_option) => existing_transaction_option,
|
||||
Err(error) => return Err(error),
|
||||
@@ -391,6 +416,160 @@ impl KbTokenBackfillService {
|
||||
trade_event_count: trade_aggregations.len(),
|
||||
})
|
||||
}
|
||||
|
||||
/// Replays the historical activity of one pool address through the existing pipeline.
|
||||
pub async fn backfill_pool_by_address(
|
||||
&self,
|
||||
pool_address: &str,
|
||||
pool_signature_limit: usize,
|
||||
) -> Result<crate::KbPoolBackfillResult, crate::KbError> {
|
||||
let effective_limit = if pool_signature_limit > 1000 {
|
||||
1000
|
||||
} else {
|
||||
pool_signature_limit
|
||||
};
|
||||
let mut result = crate::KbPoolBackfillResult {
|
||||
pool_address: pool_address.to_string(),
|
||||
pool_signature_count: 0,
|
||||
unique_signature_count: 0,
|
||||
resolved_transaction_count: 0,
|
||||
missing_transaction_count: 0,
|
||||
decoded_event_count: 0,
|
||||
detection_count: 0,
|
||||
launch_attribution_count: 0,
|
||||
pool_origin_count: 0,
|
||||
wallet_participation_count: 0,
|
||||
trade_event_count: 0,
|
||||
};
|
||||
let mut seen_addresses = std::collections::BTreeSet::<std::string::String>::new();
|
||||
let mut addresses_to_scan = std::vec::Vec::<std::string::String>::new();
|
||||
let trimmed_pool_address = pool_address.trim().to_string();
|
||||
if trimmed_pool_address.is_empty() {
|
||||
return Err(crate::KbError::Config(
|
||||
"pool_address must not be empty".to_string(),
|
||||
));
|
||||
}
|
||||
seen_addresses.insert(trimmed_pool_address.clone());
|
||||
addresses_to_scan.push(trimmed_pool_address.clone());
|
||||
let pool_result =
|
||||
crate::get_pool_by_address(self.database.as_ref(), trimmed_pool_address.as_str()).await;
|
||||
let pool_option = match pool_result {
|
||||
Ok(pool_option) => pool_option,
|
||||
Err(error) => return Err(error),
|
||||
};
|
||||
if let Some(pool) = pool_option {
|
||||
let pool_id = match pool.id {
|
||||
Some(pool_id) => pool_id,
|
||||
None => {
|
||||
return Err(crate::KbError::InvalidState(format!(
|
||||
"pool '{}' has no internal id",
|
||||
pool.address
|
||||
)));
|
||||
}
|
||||
};
|
||||
let pool_tokens_result =
|
||||
crate::list_pool_tokens_by_pool_id(self.database.as_ref(), pool_id).await;
|
||||
let pool_tokens = match pool_tokens_result {
|
||||
Ok(pool_tokens) => pool_tokens,
|
||||
Err(error) => return Err(error),
|
||||
};
|
||||
for pool_token in pool_tokens {
|
||||
let vault_address_option = pool_token.vault_address.clone();
|
||||
let vault_address = match vault_address_option {
|
||||
Some(vault_address) => vault_address.trim().to_string(),
|
||||
None => continue,
|
||||
};
|
||||
if vault_address.is_empty() {
|
||||
continue;
|
||||
}
|
||||
if seen_addresses.contains(vault_address.as_str()) {
|
||||
continue;
|
||||
}
|
||||
seen_addresses.insert(vault_address.clone());
|
||||
addresses_to_scan.push(vault_address);
|
||||
}
|
||||
}
|
||||
let mut seen_signatures = std::collections::HashSet::<std::string::String>::new();
|
||||
for address in &addresses_to_scan {
|
||||
let signatures_result = self
|
||||
.fetch_signatures_for_address(address.clone(), effective_limit)
|
||||
.await;
|
||||
let mut signatures = match signatures_result {
|
||||
Ok(signatures) => signatures,
|
||||
Err(error) => return Err(error),
|
||||
};
|
||||
if address == &trimmed_pool_address {
|
||||
result.pool_signature_count = signatures.len();
|
||||
}
|
||||
signatures.reverse();
|
||||
for signature_status in signatures {
|
||||
let signature = signature_status.signature.clone();
|
||||
if seen_signatures.contains(signature.as_str()) {
|
||||
continue;
|
||||
}
|
||||
seen_signatures.insert(signature.clone());
|
||||
result.unique_signature_count += 1;
|
||||
let replay_result = self.replay_signature(signature).await;
|
||||
let replay_result = match replay_result {
|
||||
Ok(replay_result) => replay_result,
|
||||
Err(error) => return Err(error),
|
||||
};
|
||||
result.resolved_transaction_count += replay_result.resolved_transaction_count;
|
||||
result.missing_transaction_count += replay_result.missing_transaction_count;
|
||||
result.decoded_event_count += replay_result.decoded_event_count;
|
||||
result.detection_count += replay_result.detection_count;
|
||||
result.launch_attribution_count += replay_result.launch_attribution_count;
|
||||
result.pool_origin_count += replay_result.pool_origin_count;
|
||||
result.wallet_participation_count += replay_result.wallet_participation_count;
|
||||
result.trade_event_count += replay_result.trade_event_count;
|
||||
}
|
||||
}
|
||||
let summary_payload = serde_json::json!({
|
||||
"poolAddress": result.pool_address,
|
||||
"poolSignatureCount": result.pool_signature_count,
|
||||
"uniqueSignatureCount": result.unique_signature_count,
|
||||
"resolvedTransactionCount": result.resolved_transaction_count,
|
||||
"missingTransactionCount": result.missing_transaction_count,
|
||||
"decodedEventCount": result.decoded_event_count,
|
||||
"detectionCount": result.detection_count,
|
||||
"launchAttributionCount": result.launch_attribution_count,
|
||||
"poolOriginCount": result.pool_origin_count,
|
||||
"walletParticipationCount": result.wallet_participation_count,
|
||||
"tradeEventCount": result.trade_event_count,
|
||||
"scannedAddressCount": addresses_to_scan.len(),
|
||||
"effectiveSignatureLimit": effective_limit
|
||||
});
|
||||
let observation_result = self
|
||||
.persistence
|
||||
.record_observation(&crate::KbDetectionObservationInput::new(
|
||||
"pool.backfill.completed".to_string(),
|
||||
crate::KbObservationSourceKind::HttpRpc,
|
||||
Some(format!("backfill:{}", self.http_role)),
|
||||
pool_address.to_string(),
|
||||
None,
|
||||
summary_payload.clone(),
|
||||
))
|
||||
.await;
|
||||
let observation_id = match observation_result {
|
||||
Ok(observation_id) => observation_id,
|
||||
Err(error) => return Err(error),
|
||||
};
|
||||
let signal_result = self
|
||||
.persistence
|
||||
.record_signal(&crate::KbDetectionSignalInput::new(
|
||||
"signal.pool.backfill.completed".to_string(),
|
||||
crate::KbAnalysisSignalSeverity::Low,
|
||||
pool_address.to_string(),
|
||||
Some(observation_id),
|
||||
None,
|
||||
summary_payload,
|
||||
))
|
||||
.await;
|
||||
if let Err(error) = signal_result {
|
||||
return Err(error);
|
||||
}
|
||||
Ok(result)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Default)]
|
||||
|
||||
@@ -74,7 +74,7 @@ impl KbTradeAggregationService {
|
||||
};
|
||||
let mut results = std::vec::Vec::new();
|
||||
for decoded_event in &decoded_events {
|
||||
if !decoded_event.event_kind.ends_with(".swap") {
|
||||
if !kb_is_trade_event_kind(decoded_event.event_kind.as_str()) {
|
||||
continue;
|
||||
}
|
||||
let decoded_event_id = match decoded_event.id {
|
||||
@@ -135,6 +135,16 @@ impl KbTradeAggregationService {
|
||||
)));
|
||||
}
|
||||
};
|
||||
let pool_tokens_result =
|
||||
crate::list_pool_tokens_by_pool_id(self.database.as_ref(), pool_id).await;
|
||||
let pool_tokens = match pool_tokens_result {
|
||||
Ok(pool_tokens) => pool_tokens,
|
||||
Err(error) => return Err(error),
|
||||
};
|
||||
let base_vault_address =
|
||||
kb_find_pool_token_vault_address_by_token_id(&pool_tokens, pair.base_token_id);
|
||||
let quote_vault_address =
|
||||
kb_find_pool_token_vault_address_by_token_id(&pool_tokens, pair.quote_token_id);
|
||||
let payload_result =
|
||||
serde_json::from_str::<serde_json::Value>(decoded_event.payload_json.as_str());
|
||||
let payload = match payload_result {
|
||||
@@ -146,8 +156,8 @@ impl KbTradeAggregationService {
|
||||
)));
|
||||
}
|
||||
};
|
||||
let trade_side = kb_extract_trade_side(&payload);
|
||||
let base_amount_raw = kb_extract_amount_string(
|
||||
let trade_side = kb_extract_trade_side(decoded_event.event_kind.as_str(), &payload);
|
||||
let mut base_amount_raw = kb_extract_amount_string(
|
||||
&payload,
|
||||
&[
|
||||
"baseAmountRaw",
|
||||
@@ -157,7 +167,7 @@ impl KbTradeAggregationService {
|
||||
"amountInBase",
|
||||
],
|
||||
);
|
||||
let quote_amount_raw = kb_extract_amount_string(
|
||||
let mut quote_amount_raw = kb_extract_amount_string(
|
||||
&payload,
|
||||
&[
|
||||
"quoteAmountRaw",
|
||||
@@ -167,45 +177,85 @@ impl KbTradeAggregationService {
|
||||
"amountOutQuote",
|
||||
],
|
||||
);
|
||||
let price_quote_per_base =
|
||||
kb_compute_price_quote_per_base(base_amount_raw.clone(), quote_amount_raw.clone());
|
||||
let slot_i64 = kb_convert_slot_to_i64(transaction.slot);
|
||||
let created_trade_event = existing_trade_option.is_none();
|
||||
let trade_event_id = if let Some(existing_trade) = existing_trade_option {
|
||||
match existing_trade.id {
|
||||
Some(trade_event_id) => trade_event_id,
|
||||
None => {
|
||||
return Err(crate::KbError::InvalidState(
|
||||
"trade event has no internal id".to_string(),
|
||||
));
|
||||
}
|
||||
}
|
||||
} else {
|
||||
let trade_event_dto = crate::KbTradeEventDto::new(
|
||||
pool.dex_id,
|
||||
pool_id,
|
||||
pair_id,
|
||||
transaction_id,
|
||||
decoded_event_id,
|
||||
transaction.signature.clone(),
|
||||
slot_i64,
|
||||
trade_side,
|
||||
pair.base_token_id,
|
||||
pair.quote_token_id,
|
||||
base_amount_raw.clone(),
|
||||
quote_amount_raw.clone(),
|
||||
price_quote_per_base,
|
||||
crate::KbObservationSourceKind::Dex,
|
||||
transaction.source_endpoint_name.clone(),
|
||||
decoded_event.payload_json.clone(),
|
||||
let mut price_quote_per_base = None;
|
||||
if decoded_event.event_kind.starts_with("pump_swap.")
|
||||
&& (base_amount_raw.is_none()
|
||||
|| quote_amount_raw.is_none()
|
||||
|| price_quote_per_base.is_none())
|
||||
{
|
||||
let inferred_result = kb_extract_pump_swap_amounts_from_transaction(
|
||||
transaction.transaction_json.as_str(),
|
||||
transaction.meta_json.as_deref(),
|
||||
base_vault_address.as_deref(),
|
||||
quote_vault_address.as_deref(),
|
||||
);
|
||||
let upsert_result =
|
||||
crate::upsert_trade_event(self.database.as_ref(), &trade_event_dto).await;
|
||||
match upsert_result {
|
||||
Ok(trade_event_id) => trade_event_id,
|
||||
let inferred = match inferred_result {
|
||||
Ok(inferred) => inferred,
|
||||
Err(error) => return Err(error),
|
||||
};
|
||||
if base_amount_raw.is_none() {
|
||||
base_amount_raw = inferred.0;
|
||||
}
|
||||
if quote_amount_raw.is_none() {
|
||||
quote_amount_raw = inferred.1;
|
||||
}
|
||||
if price_quote_per_base.is_none() {
|
||||
price_quote_per_base = inferred.2;
|
||||
}
|
||||
}
|
||||
if price_quote_per_base.is_none() {
|
||||
price_quote_per_base = kb_compute_price_quote_per_base_with_decimals(
|
||||
transaction.meta_json.as_deref(),
|
||||
transaction.transaction_json.as_str(),
|
||||
base_vault_address.as_deref(),
|
||||
quote_vault_address.as_deref(),
|
||||
);
|
||||
}
|
||||
let slot_i64 = kb_convert_slot_to_i64(transaction.slot);
|
||||
let existing_trade_was_empty = match &existing_trade_option {
|
||||
Some(existing_trade) => {
|
||||
existing_trade.base_amount_raw.is_none()
|
||||
&& existing_trade.quote_amount_raw.is_none()
|
||||
&& existing_trade.price_quote_per_base.is_none()
|
||||
}
|
||||
None => false,
|
||||
};
|
||||
let trade_event_dto = crate::KbTradeEventDto::new(
|
||||
pool.dex_id,
|
||||
pool_id,
|
||||
pair_id,
|
||||
transaction_id,
|
||||
decoded_event_id,
|
||||
transaction.signature.clone(),
|
||||
slot_i64,
|
||||
trade_side,
|
||||
pair.base_token_id,
|
||||
pair.quote_token_id,
|
||||
base_amount_raw.clone(),
|
||||
quote_amount_raw.clone(),
|
||||
price_quote_per_base,
|
||||
crate::KbObservationSourceKind::Dex,
|
||||
transaction.source_endpoint_name.clone(),
|
||||
decoded_event.payload_json.clone(),
|
||||
);
|
||||
tracing::debug!(
|
||||
event_kind = %decoded_event.event_kind,
|
||||
pool_account = ?decoded_event.pool_account,
|
||||
decoded_event_id = ?decoded_event.id,
|
||||
"trade aggregation candidate"
|
||||
);
|
||||
let upsert_result =
|
||||
crate::upsert_trade_event(self.database.as_ref(), &trade_event_dto).await;
|
||||
let trade_event_id = match upsert_result {
|
||||
Ok(trade_event_id) => trade_event_id,
|
||||
Err(error) => return Err(error),
|
||||
};
|
||||
let created_trade_event = existing_trade_option.is_none();
|
||||
let repaired_trade_event = !created_trade_event
|
||||
&& existing_trade_was_empty
|
||||
&& (base_amount_raw.is_some()
|
||||
|| quote_amount_raw.is_some()
|
||||
|| price_quote_per_base.is_some());
|
||||
let pair_metric_result =
|
||||
crate::get_pair_metric_by_pair_id(self.database.as_ref(), pair_id).await;
|
||||
let pair_metric_option = match pair_metric_result {
|
||||
@@ -221,7 +271,7 @@ impl KbTradeAggregationService {
|
||||
));
|
||||
}
|
||||
};
|
||||
if created_trade_event {
|
||||
if created_trade_event || repaired_trade_event {
|
||||
let mut updated_metric = existing_metric.clone();
|
||||
kb_apply_trade_to_pair_metric(
|
||||
&mut updated_metric,
|
||||
@@ -310,6 +360,19 @@ impl KbTradeAggregationService {
|
||||
}
|
||||
}
|
||||
|
||||
fn kb_is_trade_event_kind(event_kind: &str) -> bool {
|
||||
if event_kind.ends_with(".swap") {
|
||||
return true;
|
||||
}
|
||||
if event_kind.ends_with(".buy") {
|
||||
return true;
|
||||
}
|
||||
if event_kind.ends_with(".sell") {
|
||||
return true;
|
||||
}
|
||||
false
|
||||
}
|
||||
|
||||
fn kb_convert_slot_to_i64(slot: std::option::Option<u64>) -> std::option::Option<i64> {
|
||||
match slot {
|
||||
Some(slot) => match i64::try_from(slot) {
|
||||
@@ -320,13 +383,20 @@ fn kb_convert_slot_to_i64(slot: std::option::Option<u64>) -> std::option::Option
|
||||
}
|
||||
}
|
||||
|
||||
fn kb_extract_trade_side(payload: &serde_json::Value) -> crate::KbSwapTradeSide {
|
||||
fn kb_extract_trade_side(event_kind: &str, payload: &serde_json::Value) -> crate::KbSwapTradeSide {
|
||||
let trade_side_option = kb_extract_string_by_candidate_keys(payload, &["tradeSide"]);
|
||||
match trade_side_option.as_deref() {
|
||||
Some("BuyBase") => crate::KbSwapTradeSide::BuyBase,
|
||||
Some("SellBase") => crate::KbSwapTradeSide::SellBase,
|
||||
_ => crate::KbSwapTradeSide::Unknown,
|
||||
Some("BuyBase") => return crate::KbSwapTradeSide::BuyBase,
|
||||
Some("SellBase") => return crate::KbSwapTradeSide::SellBase,
|
||||
_ => {}
|
||||
}
|
||||
if event_kind.ends_with(".buy") {
|
||||
return crate::KbSwapTradeSide::BuyBase;
|
||||
}
|
||||
if event_kind.ends_with(".sell") {
|
||||
return crate::KbSwapTradeSide::SellBase;
|
||||
}
|
||||
crate::KbSwapTradeSide::Unknown
|
||||
}
|
||||
|
||||
fn kb_extract_amount_string(
|
||||
@@ -336,34 +406,6 @@ fn kb_extract_amount_string(
|
||||
kb_extract_scalar_as_string_by_candidate_keys(payload, candidate_keys)
|
||||
}
|
||||
|
||||
fn kb_compute_price_quote_per_base(
|
||||
base_amount_raw: std::option::Option<std::string::String>,
|
||||
quote_amount_raw: std::option::Option<std::string::String>,
|
||||
) -> std::option::Option<f64> {
|
||||
let base_amount_text = match base_amount_raw {
|
||||
Some(base_amount_text) => base_amount_text,
|
||||
None => return None,
|
||||
};
|
||||
let quote_amount_text = match quote_amount_raw {
|
||||
Some(quote_amount_text) => quote_amount_text,
|
||||
None => return None,
|
||||
};
|
||||
let base_amount_result = base_amount_text.parse::<f64>();
|
||||
let base_amount = match base_amount_result {
|
||||
Ok(base_amount) => base_amount,
|
||||
Err(_) => return None,
|
||||
};
|
||||
if base_amount <= 0.0 {
|
||||
return None;
|
||||
}
|
||||
let quote_amount_result = quote_amount_text.parse::<f64>();
|
||||
let quote_amount = match quote_amount_result {
|
||||
Ok(quote_amount) => quote_amount,
|
||||
Err(_) => return None,
|
||||
};
|
||||
Some(quote_amount / base_amount)
|
||||
}
|
||||
|
||||
fn kb_apply_trade_to_pair_metric(
|
||||
metric: &mut crate::KbPairMetricDto,
|
||||
slot: std::option::Option<i64>,
|
||||
@@ -495,10 +537,292 @@ fn kb_extract_scalar_as_string_by_candidate_keys(
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
None
|
||||
}
|
||||
|
||||
fn kb_find_pool_token_vault_address_by_token_id(
|
||||
pool_tokens: &[crate::KbPoolTokenDto],
|
||||
token_id: i64,
|
||||
) -> std::option::Option<std::string::String> {
|
||||
for pool_token in pool_tokens {
|
||||
if pool_token.token_id != token_id {
|
||||
continue;
|
||||
}
|
||||
let vault_address_option = pool_token.vault_address.clone();
|
||||
let vault_address = match vault_address_option {
|
||||
Some(vault_address) => vault_address.trim().to_string(),
|
||||
None => continue,
|
||||
};
|
||||
if vault_address.is_empty() {
|
||||
continue;
|
||||
}
|
||||
return Some(vault_address);
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
fn kb_extract_pump_swap_amounts_from_transaction(
|
||||
transaction_json: &str,
|
||||
meta_json: std::option::Option<&str>,
|
||||
base_vault_address: std::option::Option<&str>,
|
||||
quote_vault_address: std::option::Option<&str>,
|
||||
) -> Result<
|
||||
(
|
||||
std::option::Option<std::string::String>,
|
||||
std::option::Option<std::string::String>,
|
||||
std::option::Option<f64>,
|
||||
),
|
||||
crate::KbError,
|
||||
> {
|
||||
let meta_json = match meta_json {
|
||||
Some(meta_json) => meta_json,
|
||||
None => return Ok((None, None, None)),
|
||||
};
|
||||
let transaction_value_result = serde_json::from_str::<serde_json::Value>(transaction_json);
|
||||
let transaction_value = match transaction_value_result {
|
||||
Ok(transaction_value) => transaction_value,
|
||||
Err(error) => {
|
||||
return Err(crate::KbError::Json(format!(
|
||||
"cannot parse transaction_json for pump_swap amount extraction: {}",
|
||||
error
|
||||
)));
|
||||
}
|
||||
};
|
||||
let meta_value_result = serde_json::from_str::<serde_json::Value>(meta_json);
|
||||
let meta_value = match meta_value_result {
|
||||
Ok(meta_value) => meta_value,
|
||||
Err(error) => {
|
||||
return Err(crate::KbError::Json(format!(
|
||||
"cannot parse meta_json for pump_swap amount extraction: {}",
|
||||
error
|
||||
)));
|
||||
}
|
||||
};
|
||||
let account_keys_result = kb_extract_transaction_account_keys(&transaction_value);
|
||||
let account_keys = match account_keys_result {
|
||||
Ok(account_keys) => account_keys,
|
||||
Err(error) => return Err(error),
|
||||
};
|
||||
let pre_balances_result =
|
||||
kb_extract_token_balance_map(&meta_value, &account_keys, "preTokenBalances");
|
||||
let pre_balances = match pre_balances_result {
|
||||
Ok(pre_balances) => pre_balances,
|
||||
Err(error) => return Err(error),
|
||||
};
|
||||
let post_balances_result =
|
||||
kb_extract_token_balance_map(&meta_value, &account_keys, "postTokenBalances");
|
||||
let post_balances = match post_balances_result {
|
||||
Ok(post_balances) => post_balances,
|
||||
Err(error) => return Err(error),
|
||||
};
|
||||
let mut base_amount_raw = None;
|
||||
let mut quote_amount_raw = None;
|
||||
let mut price_quote_per_base = None;
|
||||
if let Some(base_vault_address) = base_vault_address {
|
||||
let base_pre = pre_balances.get(base_vault_address);
|
||||
let base_post = post_balances.get(base_vault_address);
|
||||
let base_pre_raw = base_pre.map(|value| value.0.clone());
|
||||
let base_post_raw = base_post.map(|value| value.0.clone());
|
||||
base_amount_raw = kb_compute_amount_delta_abs(base_pre_raw, base_post_raw);
|
||||
let base_pre_ui = base_pre.and_then(|value| value.1);
|
||||
let base_post_ui = base_post.and_then(|value| value.1);
|
||||
let base_delta_ui = kb_compute_ui_delta_abs(base_pre_ui, base_post_ui);
|
||||
if let Some(quote_vault_address) = quote_vault_address {
|
||||
let quote_pre = pre_balances.get(quote_vault_address);
|
||||
let quote_post = post_balances.get(quote_vault_address);
|
||||
let quote_pre_raw = quote_pre.map(|value| value.0.clone());
|
||||
let quote_post_raw = quote_post.map(|value| value.0.clone());
|
||||
quote_amount_raw = kb_compute_amount_delta_abs(quote_pre_raw, quote_post_raw);
|
||||
let quote_pre_ui = quote_pre.and_then(|value| value.1);
|
||||
let quote_post_ui = quote_post.and_then(|value| value.1);
|
||||
let quote_delta_ui = kb_compute_ui_delta_abs(quote_pre_ui, quote_post_ui);
|
||||
match (base_delta_ui, quote_delta_ui) {
|
||||
(Some(base_delta_ui), Some(quote_delta_ui)) => {
|
||||
if base_delta_ui > 0.0 {
|
||||
price_quote_per_base = Some(quote_delta_ui / base_delta_ui);
|
||||
}
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok((base_amount_raw, quote_amount_raw, price_quote_per_base))
|
||||
}
|
||||
|
||||
fn kb_extract_transaction_account_keys(
|
||||
transaction_value: &serde_json::Value,
|
||||
) -> Result<std::vec::Vec<std::string::String>, crate::KbError> {
|
||||
let candidate_arrays = [
|
||||
transaction_value
|
||||
.get("message")
|
||||
.and_then(|value| value.get("accountKeys")),
|
||||
transaction_value
|
||||
.get("transaction")
|
||||
.and_then(|value| value.get("message"))
|
||||
.and_then(|value| value.get("accountKeys")),
|
||||
transaction_value.get("accountKeys"),
|
||||
];
|
||||
for candidate_array_option in candidate_arrays {
|
||||
let candidate_array = match candidate_array_option {
|
||||
Some(candidate_array) => candidate_array,
|
||||
None => continue,
|
||||
};
|
||||
let array = match candidate_array.as_array() {
|
||||
Some(array) => array,
|
||||
None => continue,
|
||||
};
|
||||
let mut account_keys = std::vec::Vec::new();
|
||||
for item in array {
|
||||
if let Some(value) = item.as_str() {
|
||||
account_keys.push(value.to_string());
|
||||
continue;
|
||||
}
|
||||
let pubkey_option = item.get("pubkey").and_then(|value| value.as_str());
|
||||
if let Some(pubkey) = pubkey_option {
|
||||
account_keys.push(pubkey.to_string());
|
||||
continue;
|
||||
}
|
||||
}
|
||||
if !account_keys.is_empty() {
|
||||
return Ok(account_keys);
|
||||
}
|
||||
}
|
||||
Err(crate::KbError::Json(
|
||||
"cannot extract accountKeys from transaction_json".to_string(),
|
||||
))
|
||||
}
|
||||
|
||||
fn kb_extract_token_balance_map(
|
||||
meta_value: &serde_json::Value,
|
||||
account_keys: &[std::string::String],
|
||||
field_name: &str,
|
||||
) -> Result<
|
||||
std::collections::BTreeMap<
|
||||
std::string::String,
|
||||
(std::string::String, std::option::Option<f64>),
|
||||
>,
|
||||
crate::KbError,
|
||||
> {
|
||||
let mut result = std::collections::BTreeMap::<
|
||||
std::string::String,
|
||||
(std::string::String, std::option::Option<f64>),
|
||||
>::new();
|
||||
let balances_option = meta_value
|
||||
.get(field_name)
|
||||
.and_then(|value| value.as_array());
|
||||
let balances = match balances_option {
|
||||
Some(balances) => balances,
|
||||
None => return Ok(result),
|
||||
};
|
||||
for balance in balances {
|
||||
let account_index_option = balance.get("accountIndex").and_then(|value| value.as_u64());
|
||||
let account_index = match account_index_option {
|
||||
Some(account_index) => account_index as usize,
|
||||
None => continue,
|
||||
};
|
||||
if account_index >= account_keys.len() {
|
||||
continue;
|
||||
}
|
||||
let account_address = account_keys[account_index].clone();
|
||||
let ui_token_amount_option = balance.get("uiTokenAmount");
|
||||
let ui_token_amount = match ui_token_amount_option {
|
||||
Some(ui_token_amount) => ui_token_amount,
|
||||
None => continue,
|
||||
};
|
||||
let raw_amount_option = ui_token_amount
|
||||
.get("amount")
|
||||
.and_then(|value| value.as_str());
|
||||
let raw_amount = match raw_amount_option {
|
||||
Some(raw_amount) => raw_amount.to_string(),
|
||||
None => continue,
|
||||
};
|
||||
let ui_amount_string_option = ui_token_amount
|
||||
.get("uiAmountString")
|
||||
.and_then(|value| value.as_str());
|
||||
let ui_amount = match ui_amount_string_option {
|
||||
Some(ui_amount_string) => {
|
||||
let parse_result = ui_amount_string.parse::<f64>();
|
||||
match parse_result {
|
||||
Ok(ui_amount) => Some(ui_amount),
|
||||
Err(_) => None,
|
||||
}
|
||||
}
|
||||
None => None,
|
||||
};
|
||||
result.insert(account_address, (raw_amount, ui_amount));
|
||||
}
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
fn kb_compute_amount_delta_abs(
|
||||
pre_amount: std::option::Option<std::string::String>,
|
||||
post_amount: std::option::Option<std::string::String>,
|
||||
) -> std::option::Option<std::string::String> {
|
||||
let pre_amount = match pre_amount {
|
||||
Some(pre_amount) => pre_amount,
|
||||
None => "0".to_string(),
|
||||
};
|
||||
let post_amount = match post_amount {
|
||||
Some(post_amount) => post_amount,
|
||||
None => "0".to_string(),
|
||||
};
|
||||
let pre_value_result = pre_amount.parse::<i128>();
|
||||
let pre_value = match pre_value_result {
|
||||
Ok(pre_value) => pre_value,
|
||||
Err(_) => return None,
|
||||
};
|
||||
let post_value_result = post_amount.parse::<i128>();
|
||||
let post_value = match post_value_result {
|
||||
Ok(post_value) => post_value,
|
||||
Err(_) => return None,
|
||||
};
|
||||
let delta = if post_value >= pre_value {
|
||||
post_value - pre_value
|
||||
} else {
|
||||
pre_value - post_value
|
||||
};
|
||||
Some(delta.to_string())
|
||||
}
|
||||
|
||||
fn kb_compute_ui_delta_abs(
|
||||
pre_amount: std::option::Option<f64>,
|
||||
post_amount: std::option::Option<f64>,
|
||||
) -> std::option::Option<f64> {
|
||||
let pre_amount = match pre_amount {
|
||||
Some(pre_amount) => pre_amount,
|
||||
None => 0.0,
|
||||
};
|
||||
let post_amount = match post_amount {
|
||||
Some(post_amount) => post_amount,
|
||||
None => 0.0,
|
||||
};
|
||||
let delta = if post_amount >= pre_amount {
|
||||
post_amount - pre_amount
|
||||
} else {
|
||||
pre_amount - post_amount
|
||||
};
|
||||
Some(delta)
|
||||
}
|
||||
|
||||
fn kb_compute_price_quote_per_base_with_decimals(
|
||||
meta_json: std::option::Option<&str>,
|
||||
transaction_json: &str,
|
||||
base_vault_address: std::option::Option<&str>,
|
||||
quote_vault_address: std::option::Option<&str>,
|
||||
) -> std::option::Option<f64> {
|
||||
let inferred_result = kb_extract_pump_swap_amounts_from_transaction(
|
||||
transaction_json,
|
||||
meta_json,
|
||||
base_vault_address,
|
||||
quote_vault_address,
|
||||
);
|
||||
let inferred = match inferred_result {
|
||||
Ok(inferred) => inferred,
|
||||
Err(_) => return None,
|
||||
};
|
||||
inferred.2
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
async fn make_database() -> std::sync::Arc<crate::KbDatabase> {
|
||||
|
||||
Reference in New Issue
Block a user