Sasedev/khadhroony-bobobot
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

0.7.42

Browse Source
This commit is contained in:
SinuS von SifriduS
2026-05-24 17:17:26 +02:00
parent 6176c5d4cd
commit 69c8f6c957
25 changed files with 2354 additions and 133 deletions
Download Patch File Download Diff File Expand all files Collapse all files

129
kb_lib/src/token_backfill.rs
View File

@@ -19,6 +19,10 @@ pub struct TokenBackfillResult {
pub resolved_transaction_count: usize,
/// Number of signatures whose `getTransaction` lookup returned `null`.
pub missing_transaction_count: usize,
/// Number of signatures whose `getTransaction` lookup failed after retries.
pub transaction_fetch_error_count: usize,
/// Last transaction fetch error observed during this run, if any.
pub last_transaction_fetch_error: std::option::Option<std::string::String>,
/// Total number of decoded DEX events replayed during this run.
pub decoded_event_count: usize,
/// Total number of DEX detection results produced during this run.
@@ -58,6 +62,10 @@ pub struct PoolBackfillResult {
pub resolved_transaction_count: usize,
/// Number of signatures whose `getTransaction` lookup returned `null`.
pub missing_transaction_count: usize,
/// Number of signatures whose `getTransaction` lookup failed after retries.
pub transaction_fetch_error_count: usize,
/// Last transaction fetch error observed during this run, if any.
pub last_transaction_fetch_error: std::option::Option<std::string::String>,
/// Total number of decoded DEX events replayed during this run.
pub decoded_event_count: usize,
/// Total number of DEX detection results produced during this run.
@@ -93,6 +101,10 @@ pub struct SignatureBackfillResult {
pub resolved_transaction_count: usize,
/// Number of signatures whose `getTransaction` lookup returned `null`.
pub missing_transaction_count: usize,
/// Number of signatures whose `getTransaction` lookup failed after retries.
pub transaction_fetch_error_count: usize,
/// Last transaction fetch error observed during this run, if any.
pub last_transaction_fetch_error: std::option::Option<std::string::String>,
/// Total number of decoded DEX events replayed during this run.
pub decoded_event_count: usize,
/// Total number of DEX detection results produced during this run.
@@ -142,6 +154,9 @@ pub struct TokenBackfillService {
token_metadata_service: crate::TokenMetadataBackfillService,
}
const TOKEN_BACKFILL_GET_TRANSACTION_MAX_ATTEMPTS: usize = 4;
const TOKEN_BACKFILL_GET_TRANSACTION_RETRY_BASE_DELAY_MS: u64 = 500;
impl TokenBackfillService {
/// Creates a new token-backfill service.
pub fn new(
@@ -202,6 +217,8 @@ impl TokenBackfillService {
unique_signature_count: 0,
resolved_transaction_count: 0,
missing_transaction_count: 0,
transaction_fetch_error_count: 0,
last_transaction_fetch_error: None,
decoded_event_count: 0,
detection_count: 0,
launch_attribution_count: 0,
@@ -279,6 +296,8 @@ impl TokenBackfillService {
"uniqueSignatureCount": result.unique_signature_count,
"resolvedTransactionCount": result.resolved_transaction_count,
"missingTransactionCount": result.missing_transaction_count,
"transactionFetchErrorCount": result.transaction_fetch_error_count,
"lastTransactionFetchError": result.last_transaction_fetch_error,
"decodedEventCount": result.decoded_event_count,
"detectionCount": result.detection_count,
"launchAttributionCount": result.launch_attribution_count,
@@ -410,18 +429,42 @@ impl TokenBackfillService {
"encoding": "jsonParsed",
"maxSupportedTransactionVersion": 0
}));
let transaction_value_result = self
.http_pool
.get_transaction_raw_for_role(self.http_role.as_str(), signature.clone(), config)
.await;
let transaction_value_result =
self.fetch_transaction_value_with_retry(signature.as_str(), config).await;
let transaction_value = match transaction_value_result {
Ok(transaction_value) => transaction_value,
Err(error) => return Err(error),
Err(error) => {
tracing::warn!(
signature = %signature,
error = %error,
"skipping signature after getTransaction retries failed during backfill"
);
return Ok(TokenBackfillSignatureResult {
resolved_transaction_count: 0,
missing_transaction_count: 0,
transaction_fetch_error_count: 1,
last_transaction_fetch_error: Some(error.to_string()),
decoded_event_count: 0,
detection_count: 0,
launch_attribution_count: 0,
pool_origin_count: 0,
wallet_participation_count: 0,
trade_event_count: 0,
liquidity_event_count: 0,
pool_lifecycle_event_count: 0,
fee_event_count: 0,
reward_event_count: 0,
pool_admin_event_count: 0,
pair_candle_count: 0,
});
},
};
if transaction_value.is_null() {
return Ok(TokenBackfillSignatureResult {
resolved_transaction_count: 0,
missing_transaction_count: 1,
transaction_fetch_error_count: 0,
last_transaction_fetch_error: None,
decoded_event_count: 0,
detection_count: 0,
launch_attribution_count: 0,
@@ -532,6 +575,8 @@ impl TokenBackfillService {
return Ok(TokenBackfillSignatureResult {
resolved_transaction_count: 1,
missing_transaction_count: 0,
transaction_fetch_error_count: 0,
last_transaction_fetch_error: None,
decoded_event_count: decoded.len(),
detection_count: detections.len(),
launch_attribution_count: launch_attributions.len(),
@@ -560,6 +605,8 @@ impl TokenBackfillService {
unique_signature_count: 0,
resolved_transaction_count: 0,
missing_transaction_count: 0,
transaction_fetch_error_count: 0,
last_transaction_fetch_error: None,
decoded_event_count: 0,
detection_count: 0,
launch_attribution_count: 0,
@@ -648,6 +695,11 @@ impl TokenBackfillService {
};
result.resolved_transaction_count += replay_result.resolved_transaction_count;
result.missing_transaction_count += replay_result.missing_transaction_count;
result.transaction_fetch_error_count += replay_result.transaction_fetch_error_count;
if replay_result.last_transaction_fetch_error.is_some() {
result.last_transaction_fetch_error =
replay_result.last_transaction_fetch_error.clone();
}
result.decoded_event_count += replay_result.decoded_event_count;
result.detection_count += replay_result.detection_count;
result.launch_attribution_count += replay_result.launch_attribution_count;
@@ -669,6 +721,8 @@ impl TokenBackfillService {
"uniqueSignatureCount": result.unique_signature_count,
"resolvedTransactionCount": result.resolved_transaction_count,
"missingTransactionCount": result.missing_transaction_count,
"transactionFetchErrorCount": result.transaction_fetch_error_count,
"lastTransactionFetchError": result.last_transaction_fetch_error,
"decodedEventCount": result.decoded_event_count,
"detectionCount": result.detection_count,
"launchAttributionCount": result.launch_attribution_count,
@@ -735,6 +789,8 @@ impl TokenBackfillService {
signature: trimmed_signature.clone(),
resolved_transaction_count: replay.resolved_transaction_count,
missing_transaction_count: replay.missing_transaction_count,
transaction_fetch_error_count: replay.transaction_fetch_error_count,
last_transaction_fetch_error: replay.last_transaction_fetch_error.clone(),
decoded_event_count: replay.decoded_event_count,
detection_count: replay.detection_count,
launch_attribution_count: replay.launch_attribution_count,
@@ -752,6 +808,8 @@ impl TokenBackfillService {
"signature": result.signature.clone(),
"resolvedTransactionCount": result.resolved_transaction_count,
"missingTransactionCount": result.missing_transaction_count,
"transactionFetchErrorCount": result.transaction_fetch_error_count,
"lastTransactionFetchError": result.last_transaction_fetch_error,
"decodedEventCount": result.decoded_event_count,
"detectionCount": result.detection_count,
"launchAttributionCount": result.launch_attribution_count,
@@ -797,6 +855,44 @@ impl TokenBackfillService {
return Ok(result);
}
async fn fetch_transaction_value_with_retry(
&self,
signature: &str,
config: std::option::Option<serde_json::Value>,
) -> Result<serde_json::Value, crate::Error> {
let mut attempt_index = 1usize;
loop {
let transaction_value_result = self
.http_pool
.get_transaction_raw_for_role(
self.http_role.as_str(),
signature.to_string(),
config.clone(),
)
.await;
match transaction_value_result {
Ok(transaction_value) => return Ok(transaction_value),
Err(error) => {
if !token_backfill_should_retry_http_error(&error)
|| attempt_index >= TOKEN_BACKFILL_GET_TRANSACTION_MAX_ATTEMPTS
{
return Err(error);
}
let delay_ms = token_backfill_retry_delay_ms(attempt_index);
tracing::warn!(
signature = %signature,
attempt = attempt_index,
delay_ms = delay_ms,
error = %error,
"getTransaction failed during backfill; retrying"
);
tokio::time::sleep(std::time::Duration::from_millis(delay_ms)).await;
attempt_index += 1;
},
}
}
}
async fn backfill_missing_token_metadata_best_effort(&self, limit: i64) {
let metadata_result =
self.token_metadata_service.backfill_missing_token_metadata(Some(limit)).await;
@@ -824,10 +920,29 @@ impl TokenBackfillService {
}
}
fn token_backfill_should_retry_http_error(error: &crate::Error) -> bool {
match error {
crate::Error::Http(_) => return true,
_ => return false,
}
}
fn token_backfill_retry_delay_ms(attempt_index: usize) -> u64 {
let multiplier = match attempt_index {
0 => 1,
1 => 1,
2 => 3,
_ => 6,
};
return TOKEN_BACKFILL_GET_TRANSACTION_RETRY_BASE_DELAY_MS * multiplier;
}
#[derive(Debug, Clone, Default)]
struct TokenBackfillSignatureResult {
resolved_transaction_count: usize,
missing_transaction_count: usize,
transaction_fetch_error_count: usize,
last_transaction_fetch_error: std::option::Option<std::string::String>,
decoded_event_count: usize,
detection_count: usize,
launch_attribution_count: usize,
@@ -848,6 +963,10 @@ fn merge_token_backfill_signature_result(
) {
aggregate.resolved_transaction_count += value.resolved_transaction_count;
aggregate.missing_transaction_count += value.missing_transaction_count;
aggregate.transaction_fetch_error_count += value.transaction_fetch_error_count;
if value.last_transaction_fetch_error.is_some() {
aggregate.last_transaction_fetch_error = value.last_transaction_fetch_error.clone();
}
aggregate.decoded_event_count += value.decoded_event_count;
aggregate.detection_count += value.detection_count;
aggregate.launch_attribution_count += value.launch_attribution_count;