Sasedev/khadhroony-bobobot
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

0.7.49

Browse Source
This commit is contained in:
SinuS von SifriduS
2026-06-05 14:53:16 +02:00
parent 27e25d5bf4
commit f81e0f3bea
66 changed files with 7655 additions and 214 deletions
Download Patch File Download Diff File Expand all files Collapse all files

34
kb_demo_app/frontend/demo_pipeline2.html
View File

@@ -122,6 +122,29 @@
Backfill signature
</button>
</div>
<hr class="my-4" />
<div class="mb-3">
<label for="demoPipeline2SignatureBatchTextarea" class="form-label">Signatures batch</label>
<textarea id="demoPipeline2SignatureBatchTextarea" class="form-control font-monospace" rows="8" spellcheck="false" placeholder="Une signature par ligne"></textarea>
<div class="form-text">
Backfill ciblé de plusieurs signatures. Les lignes vides et les doublons sont ignorés.
</div>
</div>
<div class="form-check mb-3">
<input id="demoPipeline2SignatureBatchContinueOnErrorCheckbox" class="form-check-input" type="checkbox" checked />
<label for="demoPipeline2SignatureBatchContinueOnErrorCheckbox" class="form-check-label">
Continuer après une erreur de signature
</label>
</div>
<div class="d-flex gap-2">
<button id="demoPipeline2BackfillSignatureBatchButton" type="button" class="btn btn-outline-primary">
Backfill signatures batch
</button>
</div>
</div>
</div>
</div>
@@ -162,15 +185,22 @@
</label>
</div>
<div class="form-check mb-3">
<div class="form-check mb-2">
<input class="form-check-input" type="checkbox" id="demoPipeline2ReplayForceDexDecodeCheckbox" />
<label class="form-check-label" for="demoPipeline2ReplayForceDexDecodeCheckbox">
Force DEX decode replay
</label>
</div>
<div class="form-check mb-3">
<input class="form-check-input" type="checkbox" id="demoPipeline2ReplayDeferInstructionObservationCheckbox" checked />
<label class="form-check-label" for="demoPipeline2ReplayDeferInstructionObservationCheckbox">
Refresh instruction observations once after replay
</label>
</div>
<p class="small text-body-secondary mb-3">
Le skip ne concerne que l’étape de décodage DEX certifiée par le ledger. Le reste du replay continue pour reconstruire les tables dérivées.
Le skip ne concerne que l’étape de décodage DEX certifiée par le ledger. Le reste du replay continue pour reconstruire les tables dérivées. Le refresh différé des observations accélère le replay mais les compteurs SQL d’instructions ne sont finalisés qu’à la fin.
</p>
<div class="d-flex gap-2">

18
kb_demo_app/frontend/ts/bindings/DemoPipeline2BackfillSignaturesBatchRequest.ts Normal file
View File

@@ -0,0 +1,18 @@
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
/**
* Request payload for batch signature backfill.
*/
export type DemoPipeline2BackfillSignaturesBatchRequest = {
/**
* Transaction signatures to resolve and replay.
*/
signatures: Array<string>,
/**
* Optional HTTP role.
*/
httpRole: string | null,
/**
* Whether the batch should continue after a hard per-signature error.
*/
continueOnError: boolean, };

84
kb_demo_app/frontend/ts/demo_pipeline2.ts
View File

@@ -11,6 +11,7 @@ import type { DemoPipeline2CatalogPayload } from "./bindings/DemoPipeline2Catalo
import type { DemoPipeline2BackfillTokenRequest } from "./bindings/DemoPipeline2BackfillTokenRequest.ts";
import type { DemoPipeline2BackfillPoolRequest } from "./bindings/DemoPipeline2BackfillPoolRequest.ts";
import type { DemoPipeline2BackfillSignatureRequest } from "./bindings/DemoPipeline2BackfillSignatureRequest.ts";
import type { DemoPipeline2BackfillSignaturesBatchRequest } from "./bindings/DemoPipeline2BackfillSignaturesBatchRequest.ts";
import type { DemoPipeline2BackfillPayload } from "./bindings/DemoPipeline2BackfillPayload.ts";
import type { DemoPipeline2PairCandlesRequest } from "./bindings/DemoPipeline2PairCandlesRequest.ts";
import type { DemoPipeline2PairCandlesPayload } from "./bindings/DemoPipeline2PairCandlesPayload.ts";
@@ -75,6 +76,8 @@ interface LocalPipelineReplayResult {
tokenMetadataUpdatedCount: number;
pairSymbolUpdatedCount: number;
resetMarketMaterializationDeletedCount: number;
instructionObservationScannedCount: number;
instructionObservationUpsertedCount: number;
globalErrorCount: number;
}
function appendLogLine(textarea: HTMLTextAreaElement, line: string): void {
@@ -152,6 +155,25 @@ function readOptionalPositiveIntegerInput(
return parsed;
}
function parseSignatureBatchText(raw: string): string[] {
const signatures: string[] = [];
const seen = new Set<string>();
for (const line of raw.split(/\r?\n/)) {
const signature = line.trim();
if (signature === "") {
continue;
}
if (seen.has(signature)) {
continue;
}
seen.add(signature);
signatures.push(signature);
}
return signatures;
}
function refreshPairSelect(
catalog: DemoPipeline2CatalogPayload,
select: HTMLSelectElement,
@@ -410,12 +432,16 @@ document.addEventListener("DOMContentLoaded", async () => {
const signatureInput = document.querySelector<HTMLInputElement>("#demoPipeline2SignatureInput");
const backfillSignatureButton = document.querySelector<HTMLButtonElement>("#demoPipeline2BackfillSignatureButton");
const signatureBatchTextarea = document.querySelector<HTMLTextAreaElement>("#demoPipeline2SignatureBatchTextarea");
const signatureBatchContinueOnErrorCheckbox = document.querySelector<HTMLInputElement>("#demoPipeline2SignatureBatchContinueOnErrorCheckbox");
const backfillSignatureBatchButton = document.querySelector<HTMLButtonElement>("#demoPipeline2BackfillSignatureBatchButton");
const replayLimitInput = document.querySelector<HTMLInputElement>("#demoPipeline2ReplayLimitInput");
const replayMetadataCheckbox = document.querySelector<HTMLInputElement>("#demoPipeline2ReplayMetadataCheckbox");
const replayMetadataLimitInput = document.querySelector<HTMLInputElement>("#demoPipeline2ReplayMetadataLimitInput");
const replaySkipCertifiedDexDecodeCheckbox = document.querySelector<HTMLInputElement>("#demoPipeline2ReplaySkipCertifiedDexDecodeCheckbox");
const replayForceDexDecodeCheckbox = document.querySelector<HTMLInputElement>("#demoPipeline2ReplayForceDexDecodeCheckbox");
const replayDeferInstructionObservationCheckbox = document.querySelector<HTMLInputElement>("#demoPipeline2ReplayDeferInstructionObservationCheckbox");
const replayLocalPipelineButton = document.querySelector<HTMLButtonElement>("#demoPipeline2ReplayLocalPipelineButton");
const diagnoseLocalPipelineButton = document.querySelector<HTMLButtonElement>("#demoPipeline2DiagnoseLocalPipelineButton");
const validateLocalPipelineButton = document.querySelector<HTMLButtonElement>("#demoPipeline2ValidateLocalPipelineButton");
@@ -462,11 +488,15 @@ document.addEventListener("DOMContentLoaded", async () => {
!backfillPoolButton ||
!signatureInput ||
!backfillSignatureButton ||
!signatureBatchTextarea ||
!signatureBatchContinueOnErrorCheckbox ||
!backfillSignatureBatchButton ||
!replayLimitInput ||
!replayMetadataCheckbox ||
!replayMetadataLimitInput ||
!replaySkipCertifiedDexDecodeCheckbox ||
!replayForceDexDecodeCheckbox ||
!replayDeferInstructionObservationCheckbox ||
!replayLocalPipelineButton ||
!diagnoseLocalPipelineButton ||
!validateLocalPipelineButton ||
@@ -685,6 +715,55 @@ document.addEventListener("DOMContentLoaded", async () => {
}
});
backfillSignatureBatchButton.addEventListener("click", async () => {
const signatures = parseSignatureBatchText(signatureBatchTextarea.value);
if (signatures.length === 0) {
appendLogLine(logTextarea, "[ui] signature batch is empty");
return;
}
const httpRoleText = httpRoleInput.value.trim();
const httpRole = httpRoleText === "" ? null : httpRoleText;
const continueOnError = signatureBatchContinueOnErrorCheckbox.checked;
appendLogLine(
logTextarea,
`[ui] launching signature batch backfill count='${signatures.length.toString()}' role='${httpRole ?? "history_backfill"}' continueOnError='${continueOnError ? "yes" : "no"}'`,
);
const request: DemoPipeline2BackfillSignaturesBatchRequest = {
signatures,
httpRole,
continueOnError,
};
backfillSignatureBatchButton.disabled = true;
backfillSignatureButton.disabled = true;
backfillMintButton.disabled = true;
backfillPoolButton.disabled = true;
try {
const payload = await invoke<DemoPipeline2BackfillPayload>(
"demo_pipeline2_backfill_signatures_batch",
{ request },
);
backfillSummaryTextarea.value = payload.summaryJson;
currentCatalog = payload.catalog;
renderCatalogTextareas(payload.catalog, tokensTextarea, poolsTextarea, pairsTextarea);
refreshPairSelect(payload.catalog, pairSelect);
appendLogLine(logTextarea, `[ui] signature batch backfill completed for '${payload.objectKey}'`);
} catch (error) {
appendLogLine(logTextarea, `[ui] signature batch backfill error: ${String(error)}`);
} finally {
backfillSignatureBatchButton.disabled = false;
backfillSignatureButton.disabled = false;
backfillMintButton.disabled = false;
backfillPoolButton.disabled = false;
}
});
replayLocalPipelineButton.addEventListener("click", async () => {
const replayLimit = readOptionalPositiveIntegerInput(
replayLimitInput,
@@ -706,7 +785,7 @@ document.addEventListener("DOMContentLoaded", async () => {
appendLogLine(
logTextarea,
`[ui] launching local pipeline replay limit='${replayLimit ?? "none"}' metadata='${replayMetadataCheckbox.checked ? "yes" : "no"}' skipDexDecode='${replaySkipCertifiedDexDecodeCheckbox.checked ? "yes" : "no"}' forceDexDecode='${replayForceDexDecodeCheckbox.checked ? "yes" : "no"}'`,
`[ui] launching local pipeline replay limit='${replayLimit ?? "none"}' metadata='${replayMetadataCheckbox.checked ? "yes" : "no"}' skipDexDecode='${replaySkipCertifiedDexDecodeCheckbox.checked ? "yes" : "no"}' forceDexDecode='${replayForceDexDecodeCheckbox.checked ? "yes" : "no"}' deferInstructionObservations='${replayDeferInstructionObservationCheckbox.checked ? "yes" : "no"}'`,
);
try {
@@ -718,6 +797,7 @@ document.addEventListener("DOMContentLoaded", async () => {
tokenMetadataLimit,
skipCertifiedDexDecode: replaySkipCertifiedDexDecodeCheckbox.checked,
forceDecodeReplay: replayForceDexDecodeCheckbox.checked,
deferInstructionObservationIndexRefresh: replayDeferInstructionObservationCheckbox.checked,
},
);
@@ -725,7 +805,7 @@ document.addEventListener("DOMContentLoaded", async () => {
appendLogLine(
logTextarea,
`[ui] local pipeline replay completed: ${result.replayedTransactionCount.toString()} replayed, ${result.decodeSkippedCount.toString()} decode skipped, ${result.decodeLedgerUpsertCount.toString()} ledger upserts, ${result.decodeLedgerUnsafeCount.toString()} unsafe ledger rows, ${result.tradeEventCount.toString()} trades, ${result.liquidityEventCount.toString()} liquidity, ${result.poolLifecycleEventCount.toString()} lifecycle, ${result.pairCandleUpsertCount.toString()} candle upserts, resetDeleted='${result.resetMarketMaterializationDeletedCount.toString()}'`,
`[ui] local pipeline replay completed: ${result.replayedTransactionCount.toString()} replayed, ${result.decodeSkippedCount.toString()} decode skipped, ${result.decodeLedgerUpsertCount.toString()} ledger upserts, ${result.decodeLedgerUnsafeCount.toString()} unsafe ledger rows, ${result.tradeEventCount.toString()} trades, ${result.liquidityEventCount.toString()} liquidity, ${result.poolLifecycleEventCount.toString()} lifecycle, ${result.pairCandleUpsertCount.toString()} candle upserts, instructionObservations='${result.instructionObservationUpsertedCount.toString()}', resetDeleted='${result.resetMarketMaterializationDeletedCount.toString()}'`,
);
await refreshCatalog();

2
kb_demo_app/package.json
View File

@@ -1,7 +1,7 @@
{
"name": "kb-demo-app",
"private": true,
"version": "0.7.48",
"version": "0.7.49",
"type": "module",
"scripts": {
"dev": "vite",

75
kb_demo_app/src/demo_pipeline2.rs
View File

@@ -1153,6 +1153,22 @@ pub(crate) struct DemoPipeline2BackfillSignatureRequest {
pub http_role: std::option::Option<std::string::String>,
}
/// Request payload for batch signature backfill.
#[derive(Clone, Debug, serde::Deserialize, TS)]
#[ts(
export,
export_to = "../frontend/ts/bindings/DemoPipeline2BackfillSignaturesBatchRequest.ts"
)]
#[serde(rename_all = "camelCase")]
pub(crate) struct DemoPipeline2BackfillSignaturesBatchRequest {
/// Transaction signatures to resolve and replay.
pub signatures: std::vec::Vec<std::string::String>,
/// Optional HTTP role.
pub http_role: std::option::Option<std::string::String>,
/// Whether the batch should continue after a hard per-signature error.
pub continue_on_error: bool,
}
/// Shared backfill response payload.
#[derive(Clone, Debug, serde::Serialize, TS)]
#[ts(export, export_to = "../frontend/ts/bindings/DemoPipeline2BackfillPayload.ts")]
@@ -1604,6 +1620,63 @@ pub(crate) async fn demo_pipeline2_backfill_signature(
})
}
/// Runs a targeted batch signature backfill then returns the refreshed catalog.
#[tauri::command]
pub(crate) async fn demo_pipeline2_backfill_signatures_batch(
state: tauri::State<'_, crate::AppState>,
request: DemoPipeline2BackfillSignaturesBatchRequest,
) -> Result<DemoPipeline2BackfillPayload, std::string::String> {
let mut signatures = std::vec::Vec::<std::string::String>::new();
for signature in request.signatures {
let trimmed_signature = signature.trim().to_string();
if trimmed_signature.is_empty() {
continue;
}
signatures.push(trimmed_signature);
}
if signatures.is_empty() {
return Err("signature batch must contain at least one signature".to_string());
}
let http_role = demo_pipeline2_normalize_http_role(request.http_role);
let database = state.database.clone();
let http_pool = std::sync::Arc::new(state.http_pool.clone());
let service = kb_lib::TokenBackfillService::new(http_pool, database.clone(), http_role.clone());
let result = service
.backfill_signatures(signatures.as_slice(), request.continue_on_error)
.await;
let backfill = match result {
Ok(backfill) => backfill,
Err(error) => {
return Err(format!(
"cannot backfill signature batch with role '{}': {}",
http_role, error
));
},
};
let summary_json_result = serde_json::to_string_pretty(&backfill);
let summary_json = match summary_json_result {
Ok(summary_json) => summary_json,
Err(error) => {
return Err(format!(
"cannot serialize signature batch backfill result: {}",
error
));
},
};
let catalog_result = demo_pipeline2_build_catalog(database).await;
let catalog = match catalog_result {
Ok(catalog) => catalog,
Err(error) => return Err(error),
};
Ok(DemoPipeline2BackfillPayload {
object_key: format!("{} signatures", backfill.unique_signature_count),
mode: "signatureBatch".to_string(),
http_role,
summary_json,
catalog,
})
}
/// Loads candles for one pair and one timeframe.
#[tauri::command]
pub(crate) async fn demo_pipeline2_get_pair_candles(
@@ -1783,6 +1856,7 @@ pub(crate) async fn demo_pipeline2_replay_local_pipeline(
token_metadata_limit: std::option::Option<i64>,
skip_certified_dex_decode: bool,
force_decode_replay: bool,
defer_instruction_observation_index_refresh: bool,
) -> Result<kb_lib::LocalPipelineReplayResult, std::string::String> {
let config = kb_lib::LocalPipelineReplayConfig {
limit,
@@ -1791,6 +1865,7 @@ pub(crate) async fn demo_pipeline2_replay_local_pipeline(
reset_market_materialization_before_replay: true,
skip_certified_dex_decode,
force_decode_replay,
defer_instruction_observation_index_refresh,
};
let database = state.database.clone();
let service = if refresh_missing_token_metadata {

1
kb_demo_app/src/lib.rs
View File

@@ -158,6 +158,7 @@ pub async fn run() -> Result<(), kb_lib::Error> {
crate::demo_pipeline2::demo_pipeline2_backfill_token_mint,
crate::demo_pipeline2::demo_pipeline2_backfill_pool_address,
crate::demo_pipeline2::demo_pipeline2_backfill_signature,
crate::demo_pipeline2::demo_pipeline2_backfill_signatures_batch,
crate::demo_pipeline2::demo_pipeline2_get_pair_candles,
crate::demo_pipeline2::demo_pipeline2_replay_local_pipeline,
crate::demo_pipeline2::demo_pipeline2_diagnose_local_pipeline,

2
kb_demo_app/tauri.conf.json
View File

@@ -1,7 +1,7 @@
{
"$schema": "https://schema.tauri.app/config/2",
"productName": "kb-demo-app",
"version": "0.7.48",
"version": "0.7.49",
"identifier": "com.sasedev.kb-demo-app",
"build": {
"beforeDevCommand": "npm run dev",