From 9712c7e1f77675dbc6c3f49a319493c38d55df4b Mon Sep 17 00:00:00 2001 From: SinuS Von SifriduS Date: Sun, 30 Aug 2026 08:41:26 +0200 Subject: [PATCH] v0.3.3-pre.002 --- Cargo.toml | 4 +- .../ksp-store-postgres-lib/src/migration.rs | 328 ++++++++++++------ crates/ksp-store-postgres-lib/src/runtime.rs | 5 +- .../tests/dependency_boundary.rs | 18 +- .../unit_tests/migration.rs | 83 +++-- deltas/0.3.3/pre.002.md | 286 +++++++++++++++ ...3_3_STORE_POSTGRES_RAW_TRANSACTION_PLAN.md | 24 +- ...0-V0_3_3_STORE_POSTGRES_RAW_TRANSACTION.md | 44 ++- 8 files changed, 640 insertions(+), 152 deletions(-) create mode 100644 deltas/0.3.3/pre.002.md diff --git a/Cargo.toml b/Cargo.toml index 9ff4e14..4eb1d4e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,12 +1,12 @@ # file: Cargo.toml -# version: 348 +# version: 349 [workspace] resolver = "3" members = ["crates/ksp-app-config-desk", "crates/ksp-app-solprices-desk", "crates/ksp-app-wallet-desk", "crates/ksp-config-lib", "crates/ksp-core-lib", "crates/ksp-interface-lib", "crates/ksp-logging-lib", "crates/ksp-offchain-transport-lib", "crates/ksp-onchain-transport-lib", "crates/ksp-program-api", "crates/ksp-store-api", "crates/ksp-store-lib", "crates/ksp-store-postgres-lib", "crates/ksp-wallet-lib"] [workspace.package] -version = "0.3.3-pre.1" +version = "0.3.3-pre.2" edition = "2024" license = "MIT" repository = "https://git.sasedev.com/Sasedev/khadhroony-solana-project" diff --git a/crates/ksp-store-postgres-lib/src/migration.rs b/crates/ksp-store-postgres-lib/src/migration.rs index 308cad4..f3cdfe9 100644 --- a/crates/ksp-store-postgres-lib/src/migration.rs +++ b/crates/ksp-store-postgres-lib/src/migration.rs @@ -1,12 +1,15 @@ // file: crates/ksp-store-postgres-lib/src/migration.rs -// version: 2 +// version: 3 use sha2::Digest; // rust-rules: trait-import const ADVISORY_LOCK_KEY: i64 = 0x4b53_5053_544f_5245; -const BOOTSTRAP_MIGRATION_NAME: &str = "bootstrap"; -const BOOTSTRAP_MIGRATION_SQL: &str = include_str!("../migrations/V000__bootstrap.sql"); -const BOOTSTRAP_MIGRATION_VERSION: i64 = 0; +const EMBEDDED_MIGRATIONS: &[EmbeddedMigration] = &[EmbeddedMigration { + hook: MigrationHook::None, + name: "bootstrap", + sql: include_str!("../migrations/V000__bootstrap.sql"), + version: 0, +}]; const HEX_LOWER: &[u8; 16] = b"0123456789abcdef"; const HISTORY_INSERT_SQL: &str = "INSERT INTO ksp_store_schema_migrations (version, name, checksum, applied_at) VALUES ($1, $2, $3, CURRENT_TIMESTAMP)"; const HISTORY_LOAD_SQL: &str = "SELECT version, name, checksum FROM ksp_store_schema_migrations ORDER BY version"; @@ -40,20 +43,44 @@ struct AppliedMigration { version: i64, } +#[derive(Clone, Copy)] +struct EmbeddedMigration { + hook: MigrationHook, + name: &'static str, + sql: &'static str, + version: i64, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum MigrationHook { + None, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum MigrationHookContext { + AppliedNow, + Existing, +} + /// Returns the latest migration version embedded by this backend runtime. #[must_use] pub(crate) const fn current_migration_version() -> i64 { - return BOOTSTRAP_MIGRATION_VERSION; + return EMBEDDED_MIGRATIONS[EMBEDDED_MIGRATIONS.len() - 1].version; } /// Runs the private bounded PostgreSQL schema bootstrap on one dedicated pooled client. pub(crate) async fn bootstrap( client: &mut deadpool_postgres::Client, + network: &ksp_store_api::RawNetworkId, auto_migrate: bool, migration_timeout: std::time::Duration, migration_lock_timeout: std::time::Duration, ) -> std::result::Result<(), crate::PostgresBackendError> { - let bounded = tokio::time::timeout(migration_timeout, bootstrap_inner(client, auto_migrate, migration_timeout, migration_lock_timeout)).await; + let registry_result = validate_embedded_registry(EMBEDDED_MIGRATIONS); + if let std::result::Result::Err(error) = registry_result { + return std::result::Result::Err(error); + } + let bounded = tokio::time::timeout(migration_timeout, bootstrap_inner(client, network, auto_migrate, migration_timeout, migration_lock_timeout)).await; return match bounded { std::result::Result::Ok(result) => result, std::result::Result::Err(_) => { @@ -64,6 +91,7 @@ pub(crate) async fn bootstrap( async fn bootstrap_inner( client: &mut deadpool_postgres::Client, + network: &ksp_store_api::RawNetworkId, auto_migrate: bool, migration_timeout: std::time::Duration, migration_lock_timeout: std::time::Duration, @@ -88,27 +116,7 @@ async fn bootstrap_inner( std::result::Result::Ok(value) => value, std::result::Result::Err(error) => return std::result::Result::Err(error), }; - let checksum = bootstrap_checksum(); - if !metadata_exists { - if !auto_migrate { - return std::result::Result::Err(crate::PostgresBackendError::new(crate::PostgresBackendErrorKind::MigrationFailed, "migration_pending")); - } - let create_result = transaction.batch_execute(BOOTSTRAP_MIGRATION_SQL).await; - if create_result.is_err() { - return std::result::Result::Err(crate::PostgresBackendError::new(crate::PostgresBackendErrorKind::MigrationFailed, "metadata_create")); - } - let shape_result = verify_metadata_shape(&transaction).await; - if let std::result::Result::Err(error) = shape_result { - return std::result::Result::Err(error); - } - let insert_result = transaction.execute(HISTORY_INSERT_SQL, &[&BOOTSTRAP_MIGRATION_VERSION, &BOOTSTRAP_MIGRATION_NAME, &checksum]).await; - match insert_result { - std::result::Result::Ok(1) => {}, - std::result::Result::Ok(_) | std::result::Result::Err(_) => { - return std::result::Result::Err(crate::PostgresBackendError::new(crate::PostgresBackendErrorKind::MigrationFailed, "history_insert")); - }, - } - } else { + let next_index = if metadata_exists { let shape_result = verify_metadata_shape(&transaction).await; if let std::result::Result::Err(error) = shape_result { return std::result::Result::Err(error); @@ -118,10 +126,24 @@ async fn bootstrap_inner( std::result::Result::Ok(value) => value, std::result::Result::Err(error) => return std::result::Result::Err(error), }; - let validation_result = validate_history(history.as_slice(), checksum.as_str()); - if let std::result::Result::Err(error) = validation_result { - return std::result::Result::Err(error); + let validation_result = validate_history(history.as_slice(), EMBEDDED_MIGRATIONS); + match validation_result { + std::result::Result::Ok(value) => value, + std::result::Result::Err(error) => return std::result::Result::Err(error), } + } else { + 0 + }; + let existing_hook_result = run_applied_migration_hooks(&transaction, network, next_index).await; + if let std::result::Result::Err(error) = existing_hook_result { + return std::result::Result::Err(error); + } + if next_index < EMBEDDED_MIGRATIONS.len() && !auto_migrate { + return std::result::Result::Err(crate::PostgresBackendError::new(crate::PostgresBackendErrorKind::MigrationFailed, "migration_pending")); + } + let apply_result = apply_pending_migrations(&transaction, network, next_index).await; + if let std::result::Result::Err(error) = apply_result { + return std::result::Result::Err(error); } let commit_result = transaction.commit().await; return match commit_result { @@ -171,20 +193,75 @@ async fn acquire_advisory_lock( } } -async fn set_statement_timeout( +async fn apply_migration( transaction: &deadpool_postgres::Transaction<'_>, - timeout: std::time::Duration, + network: &ksp_store_api::RawNetworkId, + migration: &EmbeddedMigration, ) -> std::result::Result<(), crate::PostgresBackendError> { - let timeout_value = format!("{}ms", timeout.as_millis()); - let result = transaction.query_one(SET_STATEMENT_TIMEOUT_SQL, &[&timeout_value]).await; - return match result { - std::result::Result::Ok(_) => std::result::Result::Ok(()), - std::result::Result::Err(_) => { - std::result::Result::Err(crate::PostgresBackendError::new(crate::PostgresBackendErrorKind::MigrationFailed, "statement_timeout")) + let execute_result = transaction.batch_execute(migration.sql).await; + if execute_result.is_err() { + return std::result::Result::Err(crate::PostgresBackendError::new(crate::PostgresBackendErrorKind::MigrationFailed, "migration_apply")); + } + let shape_result = verify_metadata_shape(transaction).await; + if let std::result::Result::Err(error) = shape_result { + return std::result::Result::Err(error); + } + let hook_result = run_migration_hook(transaction, network, migration.hook, MigrationHookContext::AppliedNow).await; + if let std::result::Result::Err(error) = hook_result { + return std::result::Result::Err(error); + } + let checksum = migration_checksum(migration.sql); + let insert_result = transaction.execute(HISTORY_INSERT_SQL, &[&migration.version, &migration.name, &checksum]).await; + return match insert_result { + std::result::Result::Ok(1) => std::result::Result::Ok(()), + std::result::Result::Ok(_) | std::result::Result::Err(_) => { + std::result::Result::Err(crate::PostgresBackendError::new(crate::PostgresBackendErrorKind::MigrationFailed, "history_insert")) }, }; } +async fn apply_pending_migrations( + transaction: &deadpool_postgres::Transaction<'_>, + network: &ksp_store_api::RawNetworkId, + next_index: usize, +) -> std::result::Result<(), crate::PostgresBackendError> { + let mut index = next_index; + while index < EMBEDDED_MIGRATIONS.len() { + let migration = &EMBEDDED_MIGRATIONS[index]; + let result = apply_migration(transaction, network, migration).await; + if let std::result::Result::Err(error) = result { + return std::result::Result::Err(error); + } + index += 1; + } + return std::result::Result::Ok(()); +} + +async fn load_history(transaction: &deadpool_postgres::Transaction<'_>) -> std::result::Result, crate::PostgresBackendError> { + let result = transaction.query(HISTORY_LOAD_SQL, &[]).await; + let rows = match result { + std::result::Result::Ok(value) => value, + std::result::Result::Err(_) => { + return std::result::Result::Err(crate::PostgresBackendError::new(crate::PostgresBackendErrorKind::MigrationFailed, "history_load")); + }, + }; + let mut history = std::vec::Vec::with_capacity(rows.len()); + for row in rows { + let version_result = row.try_get::(0); + let name_result = row.try_get::(1); + let checksum_result = row.try_get::(2); + match (version_result, name_result, checksum_result) { + (std::result::Result::Ok(version), std::result::Result::Ok(name), std::result::Result::Ok(checksum)) => { + history.push(AppliedMigration { checksum, name, version }); + }, + _ => { + return std::result::Result::Err(crate::PostgresBackendError::new(crate::PostgresBackendErrorKind::MigrationFailed, "history_decode")); + }, + } + } + return std::result::Result::Ok(history); +} + async fn metadata_exists(transaction: &deadpool_postgres::Transaction<'_>) -> std::result::Result { let result = transaction.query_one(METADATA_EXISTS_SQL, &[]).await; let row = match result { @@ -201,6 +278,104 @@ async fn metadata_exists(transaction: &deadpool_postgres::Transaction<'_>) -> st }; } +fn migration_checksum(sql: &str) -> std::string::String { + let mut hasher = sha2::Sha256::new(); + hasher.update(sql.as_bytes()); + let digest = hasher.finalize(); + let bytes = digest.as_slice(); + let mut encoded = std::string::String::with_capacity(bytes.len() * 2); + for byte in bytes { + let value = *byte; + encoded.push(char::from(HEX_LOWER[(value >> 4) as usize])); + encoded.push(char::from(HEX_LOWER[(value & 0x0f) as usize])); + } + return encoded; +} + +async fn run_applied_migration_hooks( + transaction: &deadpool_postgres::Transaction<'_>, + network: &ksp_store_api::RawNetworkId, + applied_count: usize, +) -> std::result::Result<(), crate::PostgresBackendError> { + let mut index = 0_usize; + while index < applied_count { + let migration = &EMBEDDED_MIGRATIONS[index]; + let result = run_migration_hook(transaction, network, migration.hook, MigrationHookContext::Existing).await; + if let std::result::Result::Err(error) = result { + return std::result::Result::Err(error); + } + index += 1; + } + return std::result::Result::Ok(()); +} + +async fn run_migration_hook( + _transaction: &deadpool_postgres::Transaction<'_>, + _network: &ksp_store_api::RawNetworkId, + hook: MigrationHook, + _context: MigrationHookContext, +) -> std::result::Result<(), crate::PostgresBackendError> { + return match hook { + MigrationHook::None => std::result::Result::Ok(()), + }; +} + +async fn set_statement_timeout( + transaction: &deadpool_postgres::Transaction<'_>, + timeout: std::time::Duration, +) -> std::result::Result<(), crate::PostgresBackendError> { + let timeout_value = format!("{}ms", timeout.as_millis()); + let result = transaction.query_one(SET_STATEMENT_TIMEOUT_SQL, &[&timeout_value]).await; + return match result { + std::result::Result::Ok(_) => std::result::Result::Ok(()), + std::result::Result::Err(_) => { + std::result::Result::Err(crate::PostgresBackendError::new(crate::PostgresBackendErrorKind::MigrationFailed, "statement_timeout")) + }, + }; +} + +fn validate_embedded_registry(migrations: &[EmbeddedMigration]) -> std::result::Result<(), crate::PostgresBackendError> { + if migrations.is_empty() { + return std::result::Result::Err(crate::PostgresBackendError::new(crate::PostgresBackendErrorKind::MigrationMismatch, "registry_empty")); + } + let mut expected_version = 0_i64; + for migration in migrations { + if migration.version != expected_version || migration.name.is_empty() || migration.sql.is_empty() { + return std::result::Result::Err(crate::PostgresBackendError::new(crate::PostgresBackendErrorKind::MigrationMismatch, "registry_invalid")); + } + expected_version = match expected_version.checked_add(1) { + std::option::Option::Some(value) => value, + std::option::Option::None => { + return std::result::Result::Err(crate::PostgresBackendError::new(crate::PostgresBackendErrorKind::MigrationMismatch, "registry_invalid")); + }, + }; + } + return std::result::Result::Ok(()); +} + +fn validate_history(history: &[AppliedMigration], migrations: &[EmbeddedMigration]) -> std::result::Result { + if history.is_empty() { + return std::result::Result::Err(crate::PostgresBackendError::new(crate::PostgresBackendErrorKind::MigrationMismatch, "history_missing")); + } + let latest_version = migrations[migrations.len() - 1].version; + let mut index = 0_usize; + for applied in history { + if applied.version > latest_version { + return std::result::Result::Err(crate::PostgresBackendError::new(crate::PostgresBackendErrorKind::SchemaNewer, "history_newer")); + } + if index >= migrations.len() { + return std::result::Result::Err(crate::PostgresBackendError::new(crate::PostgresBackendErrorKind::SchemaNewer, "history_newer")); + } + let expected = &migrations[index]; + let expected_checksum = migration_checksum(expected.sql); + if applied.version != expected.version || applied.name != expected.name || applied.checksum != expected_checksum { + return std::result::Result::Err(crate::PostgresBackendError::new(crate::PostgresBackendErrorKind::MigrationMismatch, "history_diverged")); + } + index += 1; + } + return std::result::Result::Ok(index); +} + async fn verify_metadata_shape(transaction: &deadpool_postgres::Transaction<'_>) -> std::result::Result<(), crate::PostgresBackendError> { let result = transaction.query(METADATA_SHAPE_SQL, &[]).await; let rows = match result { @@ -209,23 +384,21 @@ async fn verify_metadata_shape(transaction: &deadpool_postgres::Transaction<'_>) return std::result::Result::Err(crate::PostgresBackendError::new(crate::PostgresBackendErrorKind::MigrationFailed, "metadata_shape")); }, }; - let expected = [("version", "bigint", "NO"), ("name", "text", "NO"), ("checksum", "text", "NO"), ("applied_at", "timestamp with time zone", "NO")]; - let mut found = [false; 4]; + const REQUIRED: [(&str, &str, &str); 4] = + [("version", "bigint", "NO"), ("name", "text", "NO"), ("checksum", "text", "NO"), ("applied_at", "timestamp with time zone", "NO")]; + let mut found = [false; REQUIRED.len()]; for row in rows { - let column_result = row.try_get::(0); + let name_result = row.try_get::(0); let data_type_result = row.try_get::(1); let nullable_result = row.try_get::(2); - let (column, data_type, nullable) = match (column_result, data_type_result, nullable_result) { - (std::result::Result::Ok(column), std::result::Result::Ok(data_type), std::result::Result::Ok(nullable)) => (column, data_type, nullable), + let (name, data_type, nullable) = match (name_result, data_type_result, nullable_result) { + (std::result::Result::Ok(name), std::result::Result::Ok(data_type), std::result::Result::Ok(nullable)) => (name, data_type, nullable), _ => { return std::result::Result::Err(crate::PostgresBackendError::new(crate::PostgresBackendErrorKind::MigrationFailed, "metadata_shape_decode")); }, }; - for (index, expected_row) in expected.iter().enumerate() { - if column == expected_row.0 { - if found[index] || data_type != expected_row.1 || nullable != expected_row.2 { - return std::result::Result::Err(crate::PostgresBackendError::new(crate::PostgresBackendErrorKind::MigrationMismatch, "metadata_shape")); - } + for (index, required) in REQUIRED.iter().enumerate() { + if name == required.0 && data_type == required.1 && nullable == required.2 { found[index] = true; break; } @@ -254,65 +427,6 @@ async fn verify_metadata_shape(transaction: &deadpool_postgres::Transaction<'_>) }; } -async fn load_history(transaction: &deadpool_postgres::Transaction<'_>) -> std::result::Result, crate::PostgresBackendError> { - let result = transaction.query(HISTORY_LOAD_SQL, &[]).await; - let rows = match result { - std::result::Result::Ok(value) => value, - std::result::Result::Err(_) => { - return std::result::Result::Err(crate::PostgresBackendError::new(crate::PostgresBackendErrorKind::MigrationFailed, "history_load")); - }, - }; - let mut history = std::vec::Vec::with_capacity(rows.len()); - for row in rows { - let version_result = row.try_get::(0); - let name_result = row.try_get::(1); - let checksum_result = row.try_get::(2); - match (version_result, name_result, checksum_result) { - (std::result::Result::Ok(version), std::result::Result::Ok(name), std::result::Result::Ok(checksum)) => { - history.push(AppliedMigration { checksum, name, version }); - }, - _ => { - return std::result::Result::Err(crate::PostgresBackendError::new(crate::PostgresBackendErrorKind::MigrationFailed, "history_decode")); - }, - } - } - return std::result::Result::Ok(history); -} - -fn validate_history(history: &[AppliedMigration], expected_checksum: &str) -> std::result::Result<(), crate::PostgresBackendError> { - let mut sentinel_found = false; - for applied in history { - if applied.version > BOOTSTRAP_MIGRATION_VERSION { - return std::result::Result::Err(crate::PostgresBackendError::new(crate::PostgresBackendErrorKind::SchemaNewer, "history_newer")); - } - if applied.version < BOOTSTRAP_MIGRATION_VERSION { - return std::result::Result::Err(crate::PostgresBackendError::new(crate::PostgresBackendErrorKind::MigrationMismatch, "history_unknown")); - } - if applied.name != BOOTSTRAP_MIGRATION_NAME || applied.checksum != expected_checksum { - return std::result::Result::Err(crate::PostgresBackendError::new(crate::PostgresBackendErrorKind::MigrationMismatch, "history_diverged")); - } - sentinel_found = true; - } - if !sentinel_found { - return std::result::Result::Err(crate::PostgresBackendError::new(crate::PostgresBackendErrorKind::MigrationMismatch, "history_missing")); - } - return std::result::Result::Ok(()); -} - -fn bootstrap_checksum() -> std::string::String { - let mut hasher = sha2::Sha256::new(); - hasher.update(BOOTSTRAP_MIGRATION_SQL.as_bytes()); - let digest = hasher.finalize(); - let bytes = digest.as_slice(); - let mut encoded = std::string::String::with_capacity(bytes.len() * 2); - for byte in bytes { - let value = *byte; - encoded.push(char::from(HEX_LOWER[(value >> 4) as usize])); - encoded.push(char::from(HEX_LOWER[(value & 0x0f) as usize])); - } - return encoded; -} - #[cfg(test)] #[path = "../unit_tests/migration.rs"] mod tests; diff --git a/crates/ksp-store-postgres-lib/src/runtime.rs b/crates/ksp-store-postgres-lib/src/runtime.rs index 64b6c3d..5af073c 100644 --- a/crates/ksp-store-postgres-lib/src/runtime.rs +++ b/crates/ksp-store-postgres-lib/src/runtime.rs @@ -1,5 +1,5 @@ // file: crates/ksp-store-postgres-lib/src/runtime.rs -// version: 3 +// version: 4 const APPLICATION_NAME: &str = "ksp-store"; const MAX_CONNECTION_URI_BYTES: usize = 4_096; @@ -263,7 +263,8 @@ impl PostgresBackend { tls_mode = settings.tls_mode().code(), "PostgreSQL Store backend established initial physical connection" ); - let bootstrap_result = crate::bootstrap(&mut client, settings.auto_migrate, settings.migration_timeout, settings.migration_lock_timeout).await; + let bootstrap_result = + crate::bootstrap(&mut client, settings.network(), settings.auto_migrate, settings.migration_timeout, settings.migration_lock_timeout).await; if let std::result::Result::Err(error) = bootstrap_result { return std::result::Result::Err(error); } diff --git a/crates/ksp-store-postgres-lib/tests/dependency_boundary.rs b/crates/ksp-store-postgres-lib/tests/dependency_boundary.rs index fbdcdc3..a232915 100644 --- a/crates/ksp-store-postgres-lib/tests/dependency_boundary.rs +++ b/crates/ksp-store-postgres-lib/tests/dependency_boundary.rs @@ -1,5 +1,5 @@ // file: crates/ksp-store-postgres-lib/tests/dependency_boundary.rs -// version: 5 +// version: 6 #![warn(missing_docs)] #![deny(unreachable_pub)] @@ -82,3 +82,19 @@ fn pre_007_health_probe_remains_foundation_only_and_private_sql() { } return; } + +#[test] +fn pre_002_migration_engine_is_registry_driven_and_network_hook_ready_without_v001_schema() { + let migration = include_str!("../src/migration.rs"); + assert!(migration.contains("const EMBEDDED_MIGRATIONS: &[EmbeddedMigration]")); + assert!(migration.contains("MigrationHook::None")); + assert!(migration.contains("run_migration_hook(transaction, network, migration.hook, MigrationHookContext::AppliedNow).await")); + assert!(migration.contains("network: &ksp_store_api::RawNetworkId")); + assert!(migration.contains("MigrationHookContext::Existing")); + assert!(migration.contains("run_applied_migration_hooks(&transaction, network, next_index).await")); + assert!(migration.contains("validate_history(history.as_slice(), EMBEDDED_MIGRATIONS)")); + assert!(migration.contains("apply_pending_migrations(&transaction, network, next_index).await")); + assert!(!migration.contains("V001__raw_transaction.sql")); + assert!(!migration.contains("ksp_store_identity")); + return; +} diff --git a/crates/ksp-store-postgres-lib/unit_tests/migration.rs b/crates/ksp-store-postgres-lib/unit_tests/migration.rs index abacc06..a8f9cc2 100644 --- a/crates/ksp-store-postgres-lib/unit_tests/migration.rs +++ b/crates/ksp-store-postgres-lib/unit_tests/migration.rs @@ -1,50 +1,89 @@ // file: crates/ksp-store-postgres-lib/unit_tests/migration.rs -// version: 1 +// version: 2 fn applied(version: i64, name: &str, checksum: &str) -> super::AppliedMigration { return super::AppliedMigration { checksum: checksum.to_owned(), name: name.to_owned(), version }; } +fn embedded(version: i64, name: &'static str, sql: &'static str) -> super::EmbeddedMigration { + return super::EmbeddedMigration { hook: super::MigrationHook::None, name, sql, version }; +} + #[test] -fn bootstrap_migration_is_static_metadata_only_and_checksum_is_stable_sha256() { - assert_eq!(super::BOOTSTRAP_MIGRATION_VERSION, 0); - assert_eq!(super::BOOTSTRAP_MIGRATION_NAME, "bootstrap"); - assert!(super::BOOTSTRAP_MIGRATION_SQL.contains("CREATE TABLE ksp_store_schema_migrations")); +fn pre_002_embedded_registry_keeps_v000_immutable_and_current_version_registry_driven() { + assert_eq!(super::EMBEDDED_MIGRATIONS.len(), 1); + let migration = &super::EMBEDDED_MIGRATIONS[0]; + assert_eq!(migration.version, 0); + assert_eq!(migration.name, "bootstrap"); + assert_eq!(migration.hook, super::MigrationHook::None); + assert!(migration.sql.contains("CREATE TABLE ksp_store_schema_migrations")); for forbidden in ["RawTransaction", "RawAccountState", "raw_transaction", "raw_account", "CORE", "DECODE", "SPECIALIZED"] { - assert!(!super::BOOTSTRAP_MIGRATION_SQL.contains(forbidden), "business schema leaked into bootstrap SQL: {forbidden}"); + assert!(!migration.sql.contains(forbidden), "business schema leaked into bootstrap SQL: {forbidden}"); } - let checksum = super::bootstrap_checksum(); + assert!(super::validate_embedded_registry(super::EMBEDDED_MIGRATIONS).is_ok()); + assert_eq!(crate::current_migration_version(), 0); + let checksum = super::migration_checksum(migration.sql); assert_eq!(checksum.len(), 64); assert_eq!(checksum, "d29068b8c13b9dc0cc9ef6aaadd0fa12d41e0fe4c56541a1118c4bfc846a1450"); return; } #[test] -fn matching_sentinel_history_is_accepted() { - let checksum = super::bootstrap_checksum(); - let history = [applied(0, "bootstrap", checksum.as_str())]; - assert!(super::validate_history(&history, checksum.as_str()).is_ok()); +fn pre_002_ordered_registry_accepts_exact_history_prefix_and_full_history() { + let v000 = super::EMBEDDED_MIGRATIONS[0]; + let v001 = embedded(1, "synthetic", "SELECT 1;"); + let registry = [v000, v001]; + assert!(super::validate_embedded_registry(®istry).is_ok()); + let v000_checksum = super::migration_checksum(v000.sql); + let prefix = [applied(0, v000.name, v000_checksum.as_str())]; + assert_eq!(super::validate_history(&prefix, ®istry).ok(), std::option::Option::Some(1)); + let v001_checksum = super::migration_checksum(v001.sql); + let full = [applied(0, v000.name, v000_checksum.as_str()), applied(1, v001.name, v001_checksum.as_str())]; + assert_eq!(super::validate_history(&full, ®istry).ok(), std::option::Option::Some(2)); return; } #[test] -fn divergent_or_missing_sentinel_is_terminal_mismatch() { - let checksum = super::bootstrap_checksum(); - let wrong_name = [applied(0, "changed", checksum.as_str())]; - let wrong_checksum = [applied(0, "bootstrap", "00")]; - let missing: [super::AppliedMigration; 0] = []; - for history in [&wrong_name[..], &wrong_checksum[..], &missing[..]] { - let result = super::validate_history(history, checksum.as_str()); +fn pre_002_registry_rejects_empty_nonzero_gap_and_empty_metadata_entries() { + let empty: [super::EmbeddedMigration; 0] = []; + let starts_at_one = [embedded(1, "future", "SELECT 1;")]; + let gap = [embedded(0, "bootstrap", "SELECT 0;"), embedded(2, "future", "SELECT 2;")]; + let empty_name = [embedded(0, "", "SELECT 0;")]; + let empty_sql = [embedded(0, "bootstrap", "")]; + for registry in [&empty[..], &starts_at_one[..], &gap[..], &empty_name[..], &empty_sql[..]] { + let result = super::validate_embedded_registry(registry); assert_eq!(result.err().map(|value| return value.kind()), std::option::Option::Some(crate::PostgresBackendErrorKind::MigrationMismatch)); } return; } #[test] -fn newer_history_is_rejected_without_down_migration() { - let checksum = super::bootstrap_checksum(); - let history = [applied(0, "bootstrap", checksum.as_str()), applied(1, "future", "future-checksum")]; - let result = super::validate_history(&history, checksum.as_str()); +fn pre_002_divergent_missing_or_gapped_history_is_terminal_mismatch() { + let v000 = super::EMBEDDED_MIGRATIONS[0]; + let v001 = embedded(1, "synthetic", "SELECT 1;"); + let registry = [v000, v001]; + let v000_checksum = super::migration_checksum(v000.sql); + let v001_checksum = super::migration_checksum(v001.sql); + let wrong_name = [applied(0, "changed", v000_checksum.as_str())]; + let wrong_checksum = [applied(0, v000.name, "00")]; + let missing: [super::AppliedMigration; 0] = []; + let missing_v000 = [applied(1, v001.name, v001_checksum.as_str())]; + for history in [&wrong_name[..], &wrong_checksum[..], &missing[..], &missing_v000[..]] { + let result = super::validate_history(history, ®istry); + assert_eq!(result.err().map(|value| return value.kind()), std::option::Option::Some(crate::PostgresBackendErrorKind::MigrationMismatch)); + } + return; +} + +#[test] +fn pre_002_newer_history_is_rejected_without_down_migration() { + let v000 = super::EMBEDDED_MIGRATIONS[0]; + let v001 = embedded(1, "synthetic", "SELECT 1;"); + let registry = [v000, v001]; + let v000_checksum = super::migration_checksum(v000.sql); + let v001_checksum = super::migration_checksum(v001.sql); + let history = [applied(0, v000.name, v000_checksum.as_str()), applied(1, v001.name, v001_checksum.as_str()), applied(2, "future", "future-checksum")]; + let result = super::validate_history(&history, ®istry); assert_eq!(result.err().map(|value| return value.kind()), std::option::Option::Some(crate::PostgresBackendErrorKind::SchemaNewer)); return; } diff --git a/deltas/0.3.3/pre.002.md b/deltas/0.3.3/pre.002.md new file mode 100644 index 0000000..d859302 --- /dev/null +++ b/deltas/0.3.3/pre.002.md @@ -0,0 +1,286 @@ + + + +# Delta `0.3.3-pre.002` — moteur de migrations PostgreSQL multi-version + +## 1. Base requise + +```text +0.3.3-pre.001 +``` + +Le gate opérateur fourni pour `pre.001` est entièrement vert : + +```text +cargo fmt --all PASS +audit Rust général / exports / workspace PASS +audit Markdown PASS — 214 tables / 131 files +cargo check --workspace PASS +cargo clippy --workspace --all-targets PASS +cargo test -p ksp-store-api PASS +cargo test -p ksp-store-lib PASS +cargo test -p ksp-store-postgres-lib PASS +cargo test -p ksp-config-lib PASS +cargo check -p ksp-store-lib --no-default-features PASS +``` + +Les tests live `#[ignore]` restent volontairement hors de ce gate ; `pre.001` était documentaire et `pre.002` ne crée encore aucun SQL métier V001. + +## 2. Objectif + +Généraliser le bootstrap PostgreSQL V000 acquis en `0.3.2` vers un moteur embedded multi-version avant d'introduire la première migration métier. + +La tranche doit préserver : + +```text +checksum immuable +mismatch terminal +schema newer terminal +aucun down automatique +advisory transaction lock +migration timeout +statement timeout +transaction unique +metadata shape validation +erreurs externes non rendues +``` + +Elle prépare également la frontière transactionnelle du futur binding `RawNetworkId`, sans créer V001 ni `ksp_store_identity`. + +## 3. Version + +Le workspace passe à : + +```text +0.3.3-pre.2 +``` + +## 4. Registre embedded + +Le moteur ne possède plus une série de constantes spécialisées V000. Il utilise désormais : + +```text +EMBEDDED_MIGRATIONS: &[EmbeddedMigration] +``` + +Le registre réel de cette tranche contient exactement : + +```text +V000 bootstrap +``` + +Aucune V001 n'est anticipée dans les migrations physiques. + +Chaque entrée possède : + +```text +version +name +sql embedded +migration hook +``` + +Le registre est validé avant I/O : + +```text +non vide +commence à 0 +versions contiguës strictement ordonnées +name non vide +SQL non vide +``` + +La version courante est dérivée du dernier élément du registre. + +## 5. Validation d'historique multi-version + +L'historique PostgreSQL est maintenant validé comme **préfixe exact** du registre embedded. + +Cas : + +```text +historique exact complet -> prêt +préfixe exact -> migrations pending +nom divergent -> MigrationMismatch +checksum divergent -> MigrationMismatch +version/trou inattendu connu -> MigrationMismatch +version > runtime courant -> SchemaNewer +metadata existante sans V000 -> MigrationMismatch +``` + +Un préfixe exact n'est donc plus confondu avec une divergence : il représente précisément le cas normal d'upgrade V000 -> V001 attendu en `pre.003`. + +## 6. Application des migrations pending + +Sous la transaction PostgreSQL et l'advisory lock déjà acquis : + +```text +load/validate history +-> déterminer next_index +-> refuser si pending && auto_migrate=false +-> appliquer chaque migration dans l'ordre +-> vérifier metadata shape +-> exécuter migration hook +-> insérer checksum/name/version dans history +-> migration suivante +-> commit unique +``` + +Une erreur à n'importe quelle étape provoque le rollback transactionnel normal ; aucune migration partiellement enregistrée n'est considérée appliquée. + +## 7. Hook atomique de binding réseau + +`bootstrap` reçoit désormais le `RawNetworkId` du backend et le transmet jusqu'à chaque migration. + +Le moteur distingue deux contextes privés : + +```text +Existing migration déjà présente dans history à la réouverture +AppliedNow migration exécutée pendant le bootstrap courant +``` + +Pour les migrations déjà appliquées, le hook est rejoué sous la transaction/advisory lock avant que le backend puisse être déclaré prêt. Pour une migration nouvellement appliquée, l'ordre est : + +```text +batch_execute(migration.sql) +-> metadata shape check +-> migration hook AppliedNow +-> INSERT history +-> commit final +``` + +V000 utilise : + +```text +MigrationHook::None +``` + +`pre.003` pourra donc ajouter le hook V001 `ksp_store_identity` avec deux comportements sûrs : création/validation en `AppliedNow`, validation stricte sans recréation en `Existing`. La frontière reste transactionnelle et ne crée aucune seconde phase de bootstrap race-prone. + +Aucune identité réseau n'est écrite dans cette tranche. + +## 8. Compatibilité V000 + +Le fichier reste inchangé : + +```text +crates/ksp-store-postgres-lib/migrations/V000__bootstrap.sql +``` + +Checksum conservé : + +```text +d29068b8c13b9dc0cc9ef6aaadd0fa12d41e0fe4c56541a1118c4bfc846a1450 +``` + +La metadata historique reste : + +```text +ksp_store_schema_migrations +version BIGINT PRIMARY KEY +name TEXT NOT NULL +checksum TEXT NOT NULL +applied_at TIMESTAMPTZ NOT NULL +``` + +## 9. Tests du moteur + +Les unit tests `migration.rs` couvrent maintenant : + +```text +registre réel V000 immutable +checksum V000 stable +current version dérivée du registre +registre synthétique V000/V001 ordonné +historique V000 comme préfixe pending valide +historique synthétique V000/V001 complet +registre vide/non-zero/gap/name vide/SQL vide rejeté +nom/checksum/history missing/gap rejetés +version future -> SchemaNewer +``` + +Un canari d'intégration source vérifie aussi : + +```text +registre embedded présent +hook de migration + contextes `AppliedNow`/`Existing` présents +RawNetworkId transmis au hook +validation préfixe utilisée +application pending utilisée +aucun V001__raw_transaction.sql +aucun ksp_store_identity +``` + +## 10. Fichiers modifiés + +```text +Cargo.toml +crates/ksp-store-postgres-lib/src/migration.rs +crates/ksp-store-postgres-lib/src/runtime.rs +crates/ksp-store-postgres-lib/unit_tests/migration.rs +crates/ksp-store-postgres-lib/tests/dependency_boundary.rs +docs/plans/024-V0_3_3_STORE_POSTGRES_RAW_TRANSACTION_PLAN.md +docs/validation/020-V0_3_3_STORE_POSTGRES_RAW_TRANSACTION.md +``` + +## 11. Fichier ajouté + +```text +deltas/0.3.3/pre.002.md +``` + +## 12. Fichiers supprimés + +Aucun. + +## 13. Hors scope confirmé + +Aucun changement n'est apporté à : + +```text +V001 physique +ksp_store_identity +tables/indexes RawTransaction +mapping Row/SQL métier +six capabilities RawTransaction +ksp-store-lib dispatch +RawAccountState +Config std.store +workers/jobs/apps +codecs wire +``` + +## 14. Validations exécutées dans l'environnement de génération + +```text +python3 scripts/audit_rust_workspace_rules.py + General Rust rule audit: clean + Rust export completeness audit: 0 candidate(s) + KSP workspace Rust rule audit: clean + +python3 scripts/audit_markdown_tables.py README.md RULES.md ROADMAP.md CHANGELOG.md docs prompts crates deltas/0.3.3 + Markdown table audit: clean +``` + +`cargo`, `rustc` et `rustfmt` ne sont pas disponibles dans l'environnement d'assemblage. Les validations Cargo ci-dessous restent donc opérateur et ne sont jamais déclarées PASS ici. + +## 15. Gate opérateur demandé + +```bash +cargo fmt --all +python3 scripts/audit_rust_workspace_rules.py +python3 scripts/audit_markdown_tables.py README.md RULES.md ROADMAP.md CHANGELOG.md docs prompts crates deltas/0.3.3 +cargo check --workspace +cargo clippy --workspace --all-targets +cargo test -p ksp-store-api +cargo test -p ksp-store-lib +cargo test -p ksp-store-postgres-lib +cargo test -p ksp-config-lib +cargo check -p ksp-store-lib --no-default-features +``` + +Aucun test PostgreSQL live n'est requis : le SQL physique reste strictement V000 et aucun schéma métier n'est introduit. + +## 16. Suite + +Après gate vert, `0.3.3-pre.003` créera la vraie migration V001, les tables/constraints/indexes décidés en `pre.001` et le binding atomique `ksp_store_identity` via le hook préparé ici. diff --git a/docs/plans/024-V0_3_3_STORE_POSTGRES_RAW_TRANSACTION_PLAN.md b/docs/plans/024-V0_3_3_STORE_POSTGRES_RAW_TRANSACTION_PLAN.md index b9ebd69..f578a20 100644 --- a/docs/plans/024-V0_3_3_STORE_POSTGRES_RAW_TRANSACTION_PLAN.md +++ b/docs/plans/024-V0_3_3_STORE_POSTGRES_RAW_TRANSACTION_PLAN.md @@ -1,5 +1,5 @@ - + # Plan `0.3.3` — Store/PostgreSQL RawTransaction vertical slice @@ -17,13 +17,19 @@ Base canonique auditée : v0.3.2 ``` -Première tranche : +Première tranche validée : ```text 0.3.3-pre.001 — audit, threat model, physical design, sizing et planning ``` -`pre.001` est volontairement une tranche de conception. Elle ne crée ni migration métier `V001`, ni repository PostgreSQL métier, ni dispatch RAW dans `ksp-store-lib`. +Tranche technique courante : + +```text +0.3.3-pre.002 — moteur de migrations multi-version +``` + +`pre.001` est restée volontairement une tranche de conception. Son gate opérateur est vert. `pre.002` généralise uniquement le moteur de migrations PostgreSQL ; elle ne crée toujours ni migration métier `V001`, ni repository PostgreSQL métier, ni dispatch RAW dans `ksp-store-lib`. ## 2. Sources et autorité @@ -1007,6 +1013,18 @@ L'audit montre que le moteur de migration `0.3.2` doit être généralisé avant - préparer le hook atomique de binding réseau ; - ajouter les tests du moteur sans créer encore de repository métier. +Statut matérialisé en `pre.002` : + +- registre embedded privé ordonné, actuellement limité à V000 ; +- version courante dérivée du dernier élément du registre plutôt que d'une constante V000 dédiée ; +- historique validé comme préfixe exact et immuable du registre ; +- migrations pending appliquées séquentiellement dans la transaction déjà protégée par advisory lock ; +- `auto_migrate = false` refuse tout préfixe incomplet par `migration_pending` ; +- hook de migration exécuté pour les versions déjà appliquées à chaque ouverture et, pour une nouvelle version, après son SQL mais avant insertion de l'historique et commit ; +- `RawNetworkId` est déjà transmis jusqu'au hook afin que `pre.003` puisse binder V001 atomiquement sans déplacer la frontière transactionnelle ; +- V000 conserve strictement son SQL et son checksum historique ; +- aucun fichier V001 ni objet `ksp_store_identity` n'est encore créé. + ### `0.3.3-pre.003` — V001 physique + binding réseau - créer V001 ; diff --git a/docs/validation/020-V0_3_3_STORE_POSTGRES_RAW_TRANSACTION.md b/docs/validation/020-V0_3_3_STORE_POSTGRES_RAW_TRANSACTION.md index 1481178..92f59e2 100644 --- a/docs/validation/020-V0_3_3_STORE_POSTGRES_RAW_TRANSACTION.md +++ b/docs/validation/020-V0_3_3_STORE_POSTGRES_RAW_TRANSACTION.md @@ -1,5 +1,5 @@ - + # Validation `0.3.3` — Store/PostgreSQL RawTransaction vertical slice @@ -11,7 +11,7 @@ Cette validation accompagne : 0.3.3 — Store/PostgreSQL RawTransaction vertical slice ``` -Elle démarre en `0.3.3-pre.001` comme matrice de preuve. Les lignes non encore implémentées restent explicitement `À FAIRE`; elles ne sont pas présentées comme acquises. +Elle démarre en `0.3.3-pre.001` comme matrice de preuve. Le gate opérateur de `pre.001` est vert. Les lignes non encore implémentées restent explicitement `À FAIRE`; elles ne sont pas présentées comme acquises. ## 2. Baseline stable @@ -434,21 +434,33 @@ SQLSTATE et texte PostgreSQL ne font pas partie du contrat public. ### 17.1 Multi-version -Preuves nécessaires avant V001 : +Preuves acquises en `pre.002` avant création réelle de V001 : -- V000 reste checksum-identique ; -- registre embedded ordonné `[V000, V001]` ; -- historique vide -> bootstrap correct ; +- V000 reste byte-identique et conserve le checksum `d29068b8c13b9dc0cc9ef6aaadd0fa12d41e0fe4c56541a1118c4bfc846a1450` ; +- le registre embedded privé est ordonné et commence à V000 ; +- le moteur valide un historique comme préfixe exact du registre ; +- un registre synthétique `[V000, V001]` prouve qu'un historique V000 seul retourne l'index pending V001 sans être classé mismatch ; +- nom/checksum divergent, historique vide avec metadata existante ou trou de préfixe -> mismatch ; +- version supérieure au registre connu -> schema newer, sans down migration ; +- `auto_migrate = false` refuse un registre incomplet par `migration_pending` ; +- les migrations pending sont appliquées dans l'ordre sous la transaction et l'advisory lock déjà acquis ; +- chaque migration vérifie encore la forme de la metadata avant d'insérer sa ligne d'historique. + +Preuves différées à `pre.003` après création de la vraie V001 : + +- registre réel `[V000, V001]` ; - V000 seule + auto migrate -> V001 appliquée ; -- V000 seule + auto migrate disabled -> pending/non-ready ; - V001 checksum divergent -> mismatch ; -- version > V001 -> schema newer ; -- ligne de migration manquante/divergente -> mismatch ; -- deux open concurrents -> advisory lock, historique unique. +- binding réseau réel dans le hook ; +- inventaire/checksum exact de V001. + +La concurrence PostgreSQL réelle du moteur reste couverte par la preuve foundation acquise en `0.3.2`; la vertical slice métier recevra sa propre preuve live en `pre.009`. ### 17.2 Binding réseau atomique -L'identité réseau doit être créée/validée avant commit du bootstrap V001. Un crash ne doit pas pouvoir laisser V001 « appliquée » avec une base prête mais sans identité exploitable. +`pre.002` prépare la frontière sans créer l'identité : `RawNetworkId` est transmis au moteur et chaque migration possède un hook privé exécuté sous la même transaction/advisory lock. Deux contextes sont distingués : `AppliedNow` pour une migration qui vient d'être exécutée et `Existing` pour une migration déjà présente dans l'historique lors d'une réouverture. V000 déclare explicitement le hook neutre. + +`pre.003` ajoutera le hook V001 réel : en contexte `AppliedNow`, il créera puis validera `ksp_store_identity` avant insertion de l'historique V001 ; en contexte `Existing`, il validera strictement l'identité existante sans la recréer si elle a disparu. Un échec du binding fera donc rollback du SQL V001 et de son historique lors d'une première application, et une réouverture d'une base V001 incohérente échouera avant exposition du backend prêt. ## 18. Boundaries @@ -500,9 +512,11 @@ cap 500/1000 dans Store pagination ### `pre.002` -- migration registry multi-version ; -- tests V000 conservés ; -- préparation binding. +- migration registry multi-version : PASS statique ; +- V000/checksum conservés : PASS ; +- validation préfixe/mismatch/newer : PASS unit design + canaris source ; +- préparation hook binding réseau transactionnel : PASS ; +- aucune V001 métier créée : PASS. ### `pre.003` @@ -560,7 +574,7 @@ cap 500/1000 dans Store pagination - publication stable. -## 22. Gate courant `pre.001` +## 22. Gate courant `pre.002` ```bash cargo fmt --all