0.7.27 +Refactor

This commit is contained in:
2026-05-10 00:33:01 +02:00
parent cb2e8e7096
commit 1f0137b9de
261 changed files with 12308 additions and 8928 deletions

View File

@@ -0,0 +1,712 @@
// file: kb_demo_app/frontend/ts/demo_pipeline2.ts
import * as bootstrap from "bootstrap";
import "simplebar";
import ResizeObserver from "resize-observer-polyfill";
import * as echarts from "echarts";
import { invoke } from "@tauri-apps/api/core";
import { debug, takeoverConsole } from "@fltsci/tauri-plugin-tracing";
import type { DemoPipeline2CatalogPayload } from "./bindings/DemoPipeline2CatalogPayload.ts";
import type { DemoPipeline2BackfillTokenRequest } from "./bindings/DemoPipeline2BackfillTokenRequest.ts";
import type { DemoPipeline2BackfillPoolRequest } from "./bindings/DemoPipeline2BackfillPoolRequest.ts";
import type { DemoPipeline2BackfillPayload } from "./bindings/DemoPipeline2BackfillPayload.ts";
import type { DemoPipeline2PairCandlesRequest } from "./bindings/DemoPipeline2PairCandlesRequest.ts";
import type { DemoPipeline2PairCandlesPayload } from "./bindings/DemoPipeline2PairCandlesPayload.ts";
import type { DemoPipeline2LocalDiagnosticsPayload } from "./bindings/DemoPipeline2LocalDiagnosticsPayload.ts";
import type { DemoPipeline2LocalValidationPayload } from "./bindings/DemoPipeline2LocalValidationPayload.ts";
(window as Window & typeof globalThis & { bootstrap?: typeof bootstrap }).bootstrap = bootstrap;
(window as Window & typeof globalThis & { ResizeObserver?: typeof ResizeObserver }).ResizeObserver = ResizeObserver;
interface PairCandle {
id: number | null;
pair_id: number;
timeframe_seconds: number;
bucket_start_unix: number;
bucket_end_unix: number;
open_price_quote_per_base: number;
high_price_quote_per_base: number;
low_price_quote_per_base: number;
close_price_quote_per_base: number;
trade_count: number;
buy_count: number;
sell_count: number;
base_volume_raw: string | null;
quote_volume_raw: string | null;
first_trade_signature: string | null;
last_trade_signature: string | null;
created_at: string;
updated_at: string;
}
interface LocalPipelineReplayResult {
selectedTransactionCount: number;
replayedTransactionCount: number;
decodeErrorCount: number;
detectErrorCount: number;
tradeAggregationErrorCount: number;
pairCandleErrorCount: number;
analyticSignalErrorCount: number;
decodedEventCount: number;
detectionCount: number;
tradeEventCount: number;
pairCandleUpsertCount: number;
analyticSignalUpsertCount: number;
tokenMetadataUpdatedCount: number;
pairSymbolUpdatedCount: number;
globalErrorCount: number;
}
function appendLogLine(textarea: HTMLTextAreaElement, line: string): void {
const now = new Date();
const timestamp = now.toLocaleTimeString("fr-CH", { hour12: false });
const lines = textarea.value === "" ? [] : textarea.value.split("\n");
lines.push(`[${timestamp}] ${line}`);
textarea.value = lines.slice(-300).join("\n");
textarea.scrollTop = textarea.scrollHeight;
}
function setEmptyChart(
chart: echarts.ECharts,
chartMeta: HTMLElement,
message: string,
): void {
chartMeta.textContent = message;
chart.setOption({
animation: false,
title: {
text: message,
left: "center",
top: "middle",
textStyle: {
fontSize: 14,
fontWeight: "normal",
},
},
tooltip: {},
xAxis: { show: false, type: "category", data: [] },
yAxis: { show: false, type: "value" },
series: [],
}, true);
}
function readPositiveIntegerInput(
input: HTMLInputElement,
logTextarea: HTMLTextAreaElement,
label: string,
): number | undefined {
const text = input.value.trim();
if (text === "") {
appendLogLine(logTextarea, `[ui] ${label} is required`);
return undefined;
}
const parsed = Number.parseInt(text, 10);
if (Number.isNaN(parsed) || parsed <= 0) {
appendLogLine(logTextarea, `[ui] invalid ${label} '${text}'`);
return undefined;
}
return parsed;
}
function readOptionalPositiveIntegerInput(
input: HTMLInputElement,
logTextarea: HTMLTextAreaElement,
label: string,
): number | null | undefined {
const text = input.value.trim();
if (text === "") {
return null;
}
const parsed = Number.parseInt(text, 10);
if (Number.isNaN(parsed) || parsed <= 0) {
appendLogLine(logTextarea, `[ui] invalid ${label} '${text}'`);
return undefined;
}
return parsed;
}
function refreshPairSelect(
catalog: DemoPipeline2CatalogPayload,
select: HTMLSelectElement,
): void {
const previousValue = select.value;
select.innerHTML = "";
const emptyOption = document.createElement("option");
emptyOption.value = "";
emptyOption.textContent = "Aucune";
select.appendChild(emptyOption);
for (const pair of catalog.pairs) {
const option = document.createElement("option");
option.value = pair.pairId.toString();
option.textContent = `#${pair.pairId.toString()} ${pair.symbol ?? ""} ${pair.poolAddress}`.trim();
if (option.value === previousValue) {
option.selected = true;
}
select.appendChild(option);
}
}
function renderCatalogTextareas(
catalog: DemoPipeline2CatalogPayload,
tokensTextarea: HTMLTextAreaElement,
poolsTextarea: HTMLTextAreaElement,
pairsTextarea: HTMLTextAreaElement,
): void {
tokensTextarea.value = JSON.stringify(catalog.tokens, null, 2);
poolsTextarea.value = JSON.stringify(catalog.pools, null, 2);
pairsTextarea.value = JSON.stringify(catalog.pairs, null, 2);
}
function parseCandlesJson(raw: string): PairCandle[] {
if (raw.trim() === "") {
return [];
}
try {
return JSON.parse(raw) as PairCandle[];
} catch {
return [];
}
}
function parseVolume(text: string | null, fallback: number): number {
if (text === null || text.trim() === "") {
return Number(fallback);
}
const parsed = Number.parseFloat(text);
if (Number.isNaN(parsed)) {
return Number(fallback);
}
return parsed;
}
function renderCandlesChart(
chart: echarts.ECharts,
chartMeta: HTMLElement,
pairId: number,
timeframeSeconds: number,
candles: PairCandle[],
): void {
if (candles.length === 0) {
setEmptyChart(chart, chartMeta, "Aucune candle disponible.");
return;
}
const sorted = [...candles].sort(
(left, right) => left.bucket_start_unix - right.bucket_start_unix,
);
const categoryData = sorted.map((candle) =>
new Date(Number(candle.bucket_start_unix) * 1000).toLocaleString("fr-CH", {
hour12: false,
year: "2-digit",
month: "2-digit",
day: "2-digit",
hour: "2-digit",
minute: "2-digit",
}),
);
const ohlcData = sorted.map((candle) => [
candle.open_price_quote_per_base,
candle.close_price_quote_per_base,
candle.low_price_quote_per_base,
candle.high_price_quote_per_base,
]);
const volumeData = sorted.map((candle) =>
parseVolume(candle.quote_volume_raw, candle.trade_count),
);
chartMeta.textContent =
`Pair #${pairId.toString()} • timeframe ${timeframeSeconds.toString()}s • ${sorted.length} candles`;
chart.setOption({
animation: false,
legend: {
data: ["OHLC", "Volume"],
top: 0,
},
tooltip: {
trigger: "axis",
axisPointer: {
type: "cross",
},
},
axisPointer: {
link: [{ xAxisIndex: "all" }],
},
grid: [
{ left: 60, right: 24, top: 40, height: "58%" },
{ left: 60, right: 24, top: "74%", height: "16%" },
],
xAxis: [
{
type: "category",
data: categoryData,
boundaryGap: true,
axisLine: { onZero: false },
splitLine: { show: false },
min: "dataMin",
max: "dataMax",
},
{
type: "category",
gridIndex: 1,
data: categoryData,
boundaryGap: true,
axisLine: { onZero: false },
axisTick: { show: false },
splitLine: { show: false },
axisLabel: { show: false },
min: "dataMin",
max: "dataMax",
},
],
yAxis: [
{
scale: true,
splitArea: { show: false },
},
{
gridIndex: 1,
scale: true,
splitNumber: 2,
},
],
dataZoom: [
{
type: "inside",
xAxisIndex: [0, 1],
start: 0,
end: 100,
},
{
show: true,
type: "slider",
xAxisIndex: [0, 1],
bottom: 6,
start: 0,
end: 100,
},
],
series: [
{
name: "OHLC",
type: "candlestick",
data: ohlcData,
itemStyle: {
color: "#16a34a",
color0: "#dc2626",
borderColor: "#15803d",
borderColor0: "#b91c1c",
},
},
{
name: "Volume",
type: "bar",
xAxisIndex: 1,
yAxisIndex: 1,
data: volumeData,
},
],
}, true);
}
document.addEventListener("DOMContentLoaded", async () => {
void takeoverConsole();
debug("demo_pipeline2 window loaded");
const tooltipTriggerList = document.querySelectorAll('[data-bs-toggle="tooltip"]');
Array.from(tooltipTriggerList).map((tooltipTriggerEl) => new bootstrap.Tooltip(tooltipTriggerEl));
const refreshCatalogButton = document.querySelector<HTMLButtonElement>("#demoPipeline2RefreshCatalogButton");
const tokensTextarea = document.querySelector<HTMLTextAreaElement>("#demoPipeline2TokensTextarea");
const poolsTextarea = document.querySelector<HTMLTextAreaElement>("#demoPipeline2PoolsTextarea");
const pairsTextarea = document.querySelector<HTMLTextAreaElement>("#demoPipeline2PairsTextarea");
const httpRoleInput = document.querySelector<HTMLInputElement>("#demoPipeline2HttpRoleInput");
const mintInput = document.querySelector<HTMLInputElement>("#demoPipeline2MintInput");
const mintSignatureLimitInput = document.querySelector<HTMLInputElement>("#demoPipeline2MintSignatureLimitInput");
const mintPoolLimitInput = document.querySelector<HTMLInputElement>("#demoPipeline2MintPoolLimitInput");
const backfillMintButton = document.querySelector<HTMLButtonElement>("#demoPipeline2BackfillMintButton");
const poolInput = document.querySelector<HTMLInputElement>("#demoPipeline2PoolInput");
const poolSignatureLimitInput = document.querySelector<HTMLInputElement>("#demoPipeline2PoolSignatureLimitInput");
const backfillPoolButton = document.querySelector<HTMLButtonElement>("#demoPipeline2BackfillPoolButton");
const replayLimitInput = document.querySelector<HTMLInputElement>("#demoPipeline2ReplayLimitInput");
const replayMetadataCheckbox = document.querySelector<HTMLInputElement>("#demoPipeline2ReplayMetadataCheckbox");
const replayMetadataLimitInput = document.querySelector<HTMLInputElement>("#demoPipeline2ReplayMetadataLimitInput");
const replayLocalPipelineButton = document.querySelector<HTMLButtonElement>("#demoPipeline2ReplayLocalPipelineButton");
const diagnoseLocalPipelineButton = document.querySelector<HTMLButtonElement>("#demoPipeline2DiagnoseLocalPipelineButton");
const validateLocalPipelineButton = document.querySelector<HTMLButtonElement>("#demoPipeline2ValidateLocalPipelineButton");
const pairSelect = document.querySelector<HTMLSelectElement>("#demoPipeline2PairSelect");
const timeframeSelect = document.querySelector<HTMLSelectElement>("#demoPipeline2TimeframeSelect");
const customTimeframeInput = document.querySelector<HTMLInputElement>("#demoPipeline2CustomTimeframeInput");
const preferMaterializedInput = document.querySelector<HTMLInputElement>("#demoPipeline2PreferMaterializedInput");
const loadCandlesButton = document.querySelector<HTMLButtonElement>("#demoPipeline2LoadCandlesButton");
const backfillSummaryTextarea = document.querySelector<HTMLTextAreaElement>("#demoPipeline2BackfillSummaryTextarea");
const chartElement = document.querySelector<HTMLDivElement>("#demoPipeline2Chart");
const chartMeta = document.querySelector<HTMLDivElement>("#demoPipeline2ChartMeta");
const localDiagnosticsTextarea = document.querySelector<HTMLTextAreaElement>("#demoPipeline2LocalDiagnosticsTextarea");
const localValidationTextarea = document.querySelector<HTMLTextAreaElement>("#demoPipeline2LocalValidationTextarea");
const clearLogButton = document.querySelector<HTMLButtonElement>("#demoPipeline2ClearLogButton");
const logTextarea = document.querySelector<HTMLTextAreaElement>("#demoPipeline2LogTextarea");
if (
!refreshCatalogButton ||
!tokensTextarea ||
!poolsTextarea ||
!pairsTextarea ||
!httpRoleInput ||
!mintInput ||
!mintSignatureLimitInput ||
!mintPoolLimitInput ||
!backfillMintButton ||
!poolInput ||
!poolSignatureLimitInput ||
!backfillPoolButton ||
!replayLimitInput ||
!replayMetadataCheckbox ||
!replayMetadataLimitInput ||
!replayLocalPipelineButton ||
!diagnoseLocalPipelineButton ||
!validateLocalPipelineButton ||
!pairSelect ||
!timeframeSelect ||
!customTimeframeInput ||
!preferMaterializedInput ||
!loadCandlesButton ||
!backfillSummaryTextarea ||
!localDiagnosticsTextarea ||
!localValidationTextarea ||
!chartElement ||
!chartMeta ||
!clearLogButton ||
!logTextarea
) {
console.error("demo_pipeline2 DOM is incomplete");
return;
}
const safeTokensTextarea = tokensTextarea;
const safePoolsTextarea = poolsTextarea;
const safePairsTextarea = pairsTextarea;
const safePairSelect = pairSelect;
const safeChartElement = chartElement;
const safeChartMeta = chartMeta;
const safeLogTextarea = logTextarea;
const safeLocalDiagnosticsTextarea = localDiagnosticsTextarea;
const chart = echarts.init(safeChartElement);
setEmptyChart(chart, safeChartMeta, "Aucune candle disponible.");
window.addEventListener("resize", () => chart.resize());
clearLogButton.addEventListener("click", () => {
logTextarea.value = "";
});
let currentCatalog: DemoPipeline2CatalogPayload | null = null;
async function refreshCatalog(): Promise<void> {
appendLogLine(safeLogTextarea, "[ui] refreshing local catalog");
try {
const catalog = await invoke<DemoPipeline2CatalogPayload>("demo_pipeline2_get_catalog");
currentCatalog = catalog;
renderCatalogTextareas(catalog, safeTokensTextarea, safePoolsTextarea, safePairsTextarea);
refreshPairSelect(catalog, safePairSelect);
appendLogLine(
safeLogTextarea,
`[ui] catalog refreshed: ${catalog.tokens.length} tokens, ${catalog.pools.length} pools, ${catalog.pairs.length} pairs`,
);
} catch (error) {
appendLogLine(safeLogTextarea, `[ui] catalog refresh error: ${String(error)}`);
}
}
refreshCatalogButton.addEventListener("click", () => {
void refreshCatalog();
});
backfillMintButton.addEventListener("click", async () => {
const tokenMint = mintInput.value.trim();
if (tokenMint === "") {
appendLogLine(logTextarea, "[ui] token mint is required");
return;
}
const mintSignatureLimit = readPositiveIntegerInput(
mintSignatureLimitInput,
logTextarea,
"mintSignatureLimit",
);
if (mintSignatureLimit === undefined) {
return;
}
const poolSignatureLimit = readPositiveIntegerInput(
mintPoolLimitInput,
logTextarea,
"poolSignatureLimit",
);
if (poolSignatureLimit === undefined) {
return;
}
const httpRoleText = httpRoleInput.value.trim();
const httpRole = httpRoleText === "" ? null : httpRoleText;
appendLogLine(
logTextarea,
`[ui] launching token backfill for '${tokenMint}' with role '${httpRole ?? "history_backfill"}'`,
);
const request: DemoPipeline2BackfillTokenRequest = {
tokenMint,
httpRole,
mintSignatureLimit,
poolSignatureLimit,
};
try {
const payload = await invoke<DemoPipeline2BackfillPayload>(
"demo_pipeline2_backfill_token_mint",
{ request },
);
backfillSummaryTextarea.value = payload.summaryJson;
currentCatalog = payload.catalog;
renderCatalogTextareas(payload.catalog, tokensTextarea, poolsTextarea, pairsTextarea);
refreshPairSelect(payload.catalog, pairSelect);
appendLogLine(logTextarea, `[ui] token backfill completed for '${payload.objectKey}'`);
} catch (error) {
appendLogLine(logTextarea, `[ui] token backfill error: ${String(error)}`);
}
});
backfillPoolButton.addEventListener("click", async () => {
const poolAddress = poolInput.value.trim();
if (poolAddress === "") {
appendLogLine(logTextarea, "[ui] pool address is required");
return;
}
const poolSignatureLimit = readPositiveIntegerInput(
poolSignatureLimitInput,
logTextarea,
"poolSignatureLimit",
);
if (poolSignatureLimit === undefined) {
return;
}
const httpRoleText = httpRoleInput.value.trim();
const httpRole = httpRoleText === "" ? null : httpRoleText;
appendLogLine(
logTextarea,
`[ui] launching pool backfill for '${poolAddress}' with role '${httpRole ?? "history_backfill"}'`,
);
const request: DemoPipeline2BackfillPoolRequest = {
poolAddress,
httpRole,
poolSignatureLimit,
};
try {
const payload = await invoke<DemoPipeline2BackfillPayload>(
"demo_pipeline2_backfill_pool_address",
{ request },
);
backfillSummaryTextarea.value = payload.summaryJson;
currentCatalog = payload.catalog;
renderCatalogTextareas(payload.catalog, tokensTextarea, poolsTextarea, pairsTextarea);
refreshPairSelect(payload.catalog, pairSelect);
appendLogLine(logTextarea, `[ui] pool backfill completed for '${payload.objectKey}'`);
} catch (error) {
appendLogLine(logTextarea, `[ui] pool backfill error: ${String(error)}`);
}
});
replayLocalPipelineButton.addEventListener("click", async () => {
const replayLimit = readOptionalPositiveIntegerInput(
replayLimitInput,
logTextarea,
"replayLimit",
);
if (replayLimit === undefined) {
return;
}
const tokenMetadataLimit = readOptionalPositiveIntegerInput(
replayMetadataLimitInput,
logTextarea,
"tokenMetadataLimit",
);
if (tokenMetadataLimit === undefined) {
return;
}
appendLogLine(
logTextarea,
`[ui] launching local pipeline replay limit='${replayLimit ?? "none"}' metadata='${replayMetadataCheckbox.checked ? "yes" : "no"}'`,
);
try {
const result = await invoke<LocalPipelineReplayResult>(
"demo_pipeline2_replay_local_pipeline",
{
limit: replayLimit,
refreshMissingTokenMetadata: replayMetadataCheckbox.checked,
tokenMetadataLimit,
},
);
backfillSummaryTextarea.value = JSON.stringify(result, null, 2);
appendLogLine(
logTextarea,
`[ui] local pipeline replay completed: ${result.replayedTransactionCount.toString()} replayed, ${result.tradeEventCount.toString()} trades, ${result.pairCandleUpsertCount.toString()} candle upserts`,
);
await refreshCatalog();
} catch (error) {
appendLogLine(logTextarea, `[ui] local pipeline replay error: ${String(error)}`);
}
});
diagnoseLocalPipelineButton.addEventListener("click", async () => {
appendLogLine(logTextarea, "[ui] diagnosing local pipeline");
diagnoseLocalPipelineButton.disabled = true;
try {
const payload = await invoke<DemoPipeline2LocalDiagnosticsPayload>(
"demo_pipeline2_diagnose_local_pipeline",
);
safeLocalDiagnosticsTextarea.value = payload.summaryJson;
appendLogLine(
logTextarea,
`[ui] local pipeline diagnostics completed: ${payload.summary.decodedEventCount.toString()} decoded, ${payload.summary.tradeEventCount.toString()} trades, ${payload.summary.pairCandleCount.toString()} candles, actionableMissing='${payload.summary.actionableMissingTradeEventCount.toString()}', nonActionablePairs='${payload.summary.nonActionablePairCount.toString()}', blocking='${payload.summary.blockingIssueCount.toString()}'`,
);
} catch (error) {
appendLogLine(logTextarea, `[ui] local pipeline diagnostics error: ${String(error)}`);
} finally {
diagnoseLocalPipelineButton.disabled = false;
}
});
validateLocalPipelineButton.addEventListener("click", async () => {
appendLogLine(logTextarea, "[ui] validating local pipeline with 0.7.27 profile");
try {
const payload = await invoke<DemoPipeline2LocalValidationPayload>(
"demo_pipeline2_validate_local_pipeline",
);
localValidationTextarea.value = payload.validationJson;
localDiagnosticsTextarea.value = payload.summaryJson;
appendLogLine(
logTextarea,
`[ui] local pipeline validation completed: profile='${payload.run.validationProfileCode}' passed='${payload.run.validationPassed ? "yes" : "no"}' blocking='${payload.run.blockingIssueCount.toString()}' warnings='${payload.run.warningCount.toString()}'`,
);
} catch (error) {
appendLogLine(logTextarea, `[ui] local pipeline validation error: ${String(error)}`);
}
});
loadCandlesButton.addEventListener("click", async () => {
const pairIdText = pairSelect.value.trim();
if (pairIdText === "") {
appendLogLine(logTextarea, "[ui] pair selection is required");
return;
}
const parsedPairId = Number.parseInt(pairIdText, 10);
if (Number.isNaN(parsedPairId) || parsedPairId <= 0) {
appendLogLine(logTextarea, `[ui] invalid pair id '${pairIdText}'`);
return;
}
let timeframeSeconds = Number.parseInt(timeframeSelect.value.trim(), 10);
const customTimeframeText = customTimeframeInput.value.trim();
if (customTimeframeText !== "") {
const parsedCustom = Number.parseInt(customTimeframeText, 10);
if (Number.isNaN(parsedCustom) || parsedCustom <= 0) {
appendLogLine(logTextarea, `[ui] invalid custom timeframe '${customTimeframeText}'`);
return;
}
timeframeSeconds = parsedCustom;
}
appendLogLine(
logTextarea,
`[ui] loading candles for pair '${parsedPairId}' timeframe '${timeframeSeconds}s'`,
);
const request: DemoPipeline2PairCandlesRequest = {
pairId: parsedPairId,
timeframeSeconds,
preferMaterialized: preferMaterializedInput.checked,
};
try {
const payload = await invoke<DemoPipeline2PairCandlesPayload>(
"demo_pipeline2_get_pair_candles",
{ request },
);
const candles = parseCandlesJson(payload.candlesJson);
renderCandlesChart(
chart,
chartMeta,
payload.pairId,
payload.timeframeSeconds,
candles,
);
appendLogLine(
logTextarea,
`[ui] loaded ${candles.length} candles for pair '${payload.pairId.toString()}'`,
);
} catch (error) {
appendLogLine(logTextarea, `[ui] load candles error: ${String(error)}`);
setEmptyChart(chart, chartMeta, "Erreur lors du chargement des candles.");
}
});
await refreshCatalog();
if (currentCatalog !== null && currentCatalog.pairs.length > 0) {
pairSelect.value = currentCatalog.pairs[0].pairId.toString();
}
});