Sasedev/khadhroony-bobobot
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

0.7.25

Browse Source
This commit is contained in:
SinuS von SifriduS
2026-05-05 05:03:11 +02:00
parent 3e994995d7
commit f2c227e08f
132 changed files with 5767 additions and 4461 deletions
Download Patch File Download Diff File Expand all files Collapse all files

95
kb_lib/src/token_backfill.rs
View File

@@ -82,6 +82,7 @@ pub struct KbTokenBackfillService {
wallet_observation_service: crate::KbWalletObservationService,
trade_aggregation_service: crate::KbTradeAggregationService,
pair_candle_aggregation_service: crate::KbPairCandleAggregationService,
token_metadata_service: crate::KbTokenMetadataBackfillService,
}
impl KbTokenBackfillService {
@@ -101,7 +102,12 @@ impl KbTokenBackfillService {
let trade_aggregation_service = crate::KbTradeAggregationService::new(database.clone());
let pair_candle_aggregation_service =
crate::KbPairCandleAggregationService::new(database.clone());
Self {
let token_metadata_service = crate::KbTokenMetadataBackfillService::new(
http_pool.clone(),
database.clone(),
http_role.clone(),
);
return Self {
http_pool,
database,
persistence,
@@ -114,7 +120,8 @@ impl KbTokenBackfillService {
wallet_observation_service,
trade_aggregation_service,
pair_candle_aggregation_service,
}
token_metadata_service,
};
}
/// Replays the historical activity of one token mint through the existing pipeline.
@@ -195,6 +202,7 @@ impl KbTokenBackfillService {
kb_merge_token_backfill_signature_result(&mut result, replay_result);
}
}
self.backfill_missing_token_metadata_best_effort(100).await;
let summary_payload = serde_json::json!({
"tokenMint": result.token_mint,
"mintSignatureCount": result.mint_signature_count,
@@ -240,7 +248,7 @@ impl KbTokenBackfillService {
if let Err(error) = signal_result {
return Err(error);
}
Ok(result)
return Ok(result);
}
async fn fetch_signatures_for_address(
@@ -258,9 +266,10 @@ impl KbTokenBackfillService {
commitment: None,
min_context_slot: None,
};
self.http_pool
return self
.http_pool
.get_signatures_for_address_for_role(self.http_role.as_str(), address, Some(config))
.await
.await;
}
async fn collect_pool_addresses_for_token_mint(
@@ -283,7 +292,7 @@ impl KbTokenBackfillService {
"token '{}' has no internal id",
token.mint
)));
}
},
};
let pools_result = crate::list_pools(self.database.as_ref()).await;
let pools = match pools_result {
@@ -316,7 +325,7 @@ impl KbTokenBackfillService {
}
}
addresses.sort();
Ok(addresses)
return Ok(addresses);
}
async fn replay_signature(
@@ -424,7 +433,7 @@ impl KbTokenBackfillService {
Ok(pair_candle_aggregations) => pair_candle_aggregations,
Err(error) => return Err(error),
};
Ok(KbTokenBackfillSignatureResult {
return Ok(KbTokenBackfillSignatureResult {
resolved_transaction_count: 1,
missing_transaction_count: 0,
decoded_event_count: decoded.len(),
@@ -434,7 +443,7 @@ impl KbTokenBackfillService {
wallet_participation_count: wallet_observations.len(),
trade_event_count: trade_aggregations.len(),
pair_candle_count: pair_candle_aggregations.len(),
})
});
}
/// Replays the historical activity of one pool address through the existing pipeline.
@@ -443,11 +452,7 @@ impl KbTokenBackfillService {
pool_address: &str,
pool_signature_limit: usize,
) -> Result<crate::KbPoolBackfillResult, crate::KbError> {
let effective_limit = if pool_signature_limit > 1000 {
1000
} else {
pool_signature_limit
};
let effective_limit = if pool_signature_limit > 1000 { 1000 } else { pool_signature_limit };
let mut result = crate::KbPoolBackfillResult {
pool_address: pool_address.to_string(),
pool_signature_count: 0,
@@ -466,9 +471,7 @@ impl KbTokenBackfillService {
let mut addresses_to_scan = std::vec::Vec::<std::string::String>::new();
let trimmed_pool_address = pool_address.trim().to_string();
if trimmed_pool_address.is_empty() {
return Err(crate::KbError::Config(
"pool_address must not be empty".to_string(),
));
return Err(crate::KbError::Config("pool_address must not be empty".to_string()));
}
seen_addresses.insert(trimmed_pool_address.clone());
addresses_to_scan.push(trimmed_pool_address.clone());
@@ -486,7 +489,7 @@ impl KbTokenBackfillService {
"pool '{}' has no internal id",
pool.address
)));
}
},
};
let pool_tokens_result =
crate::list_pool_tokens_by_pool_id(self.database.as_ref(), pool_id).await;
@@ -512,9 +515,8 @@ impl KbTokenBackfillService {
}
let mut seen_signatures = std::collections::HashSet::<std::string::String>::new();
for address in &addresses_to_scan {
let signatures_result = self
.fetch_signatures_for_address(address.clone(), effective_limit)
.await;
let signatures_result =
self.fetch_signatures_for_address(address.clone(), effective_limit).await;
let mut signatures = match signatures_result {
Ok(signatures) => signatures,
Err(error) => return Err(error),
@@ -546,6 +548,7 @@ impl KbTokenBackfillService {
result.pair_candle_count += replay_result.pair_candle_count;
}
}
self.backfill_missing_token_metadata_best_effort(100).await;
let summary_payload = serde_json::json!({
"poolAddress": result.pool_address,
"poolSignatureCount": result.pool_signature_count,
@@ -591,7 +594,33 @@ impl KbTokenBackfillService {
if let Err(error) = signal_result {
return Err(error);
}
Ok(result)
return Ok(result);
}
async fn backfill_missing_token_metadata_best_effort(&self, limit: i64) {
let metadata_result =
self.token_metadata_service.backfill_missing_token_metadata(Some(limit)).await;
match metadata_result {
Ok(metadata_result) => {
tracing::debug!(
total_token_count = metadata_result.total_token_count,
attempted_token_count = metadata_result.attempted_token_count,
local_metadata_count = metadata_result.local_metadata_count,
mint_account_metadata_count = metadata_result.mint_account_metadata_count,
metaplex_metadata_count = metadata_result.metaplex_metadata_count,
updated_token_count = metadata_result.updated_token_count,
skipped_token_count = metadata_result.skipped_token_count,
error_count = metadata_result.error_count,
"token metadata backfill completed after historical replay"
);
},
Err(error) => {
tracing::warn!(
error = %error,
"token metadata backfill failed after historical replay"
);
},
}
}
}
@@ -880,10 +909,10 @@ mod tests {
}
}
});
Self {
return Self {
url: format!("http://{}", local_addr),
shutdown_tx: Some(shutdown_tx),
}
};
}
async fn shutdown(mut self) {
let shutdown_tx_option = self.shutdown_tx.take();
@@ -917,11 +946,11 @@ mod tests {
Ok(database) => database,
Err(error) => panic!("database init must succeed: {}", error),
};
std::sync::Arc::new(database)
return std::sync::Arc::new(database);
}
fn make_http_endpoint_config(url: std::string::String) -> crate::KbHttpEndpointConfig {
crate::KbHttpEndpointConfig {
return crate::KbHttpEndpointConfig {
name: "backfill_http".to_string(),
enabled: true,
provider: "test".to_string(),
@@ -939,7 +968,7 @@ mod tests {
max_idle_connections_per_host: 8,
pause_after_http_429_ms: None,
max_concurrent_requests_per_endpoint: 16,
}
};
}
#[tokio::test]
@@ -959,9 +988,7 @@ mod tests {
database.clone(),
"history_backfill".to_string(),
);
let backfill_result = service
.backfill_token_by_mint("BackfillToken111", 20, 20)
.await;
let backfill_result = service.backfill_token_by_mint("BackfillToken111", 20, 20).await;
let backfill = match backfill_result {
Ok(backfill) => backfill,
Err(error) => panic!("backfill must succeed: {}", error),
@@ -1059,16 +1086,12 @@ mod tests {
database.clone(),
"history_backfill".to_string(),
);
let first_result = service
.backfill_token_by_mint("BackfillToken111", 20, 20)
.await;
let first_result = service.backfill_token_by_mint("BackfillToken111", 20, 20).await;
if let Err(error) = first_result {
panic!("first backfill must succeed: {}", error);
}
let second_result = service
.backfill_token_by_mint("BackfillToken111", 20, 20)
.await;
let second_result = service.backfill_token_by_mint("BackfillToken111", 20, 20).await;
if let Err(error) = second_result {
panic!("second backfill must succeed: {}", error);
}