diff --git a/Cargo.toml b/Cargo.toml index 99459b2..c0a641b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,7 +6,7 @@ resolver = "3" members = ["crates/ksp-app-config-desk", "crates/ksp-app-solprices-desk", "crates/ksp-app-wallet-desk", "crates/ksp-config-lib", "crates/ksp-core-lib", "crates/ksp-interface-lib", "crates/ksp-logging-lib", "crates/ksp-offchain-transport-lib", "crates/ksp-onchain-transport-lib", "crates/ksp-program-api", "crates/ksp-store-api", "crates/ksp-store-lib", "crates/ksp-store-postgres-lib", "crates/ksp-wallet-lib"] [workspace.package] -version = "0.3.3-pre.3" +version = "0.3.3-pre.3.fix.1" edition = "2024" license = "MIT" repository = "https://git.sasedev.com/Sasedev/khadhroony-solana-project" diff --git a/config/examples/std.store.example.json b/config/examples/std.store.example.json index 1c62849..904a758 100644 --- a/config/examples/std.store.example.json +++ b/config/examples/std.store.example.json @@ -1,5 +1,5 @@ { - "format_version": 1, + "format_version": 2, "default_profile": "devnet", "profiles": [ { @@ -19,7 +19,8 @@ "mode": "verify_full" }, "bootstrap": { - "auto_migrate": true, + "schema_autocreate": true, + "schema_autoupdate": true, "migration_timeout_ms": 30000, "migration_lock_timeout_ms": 10000 }, @@ -43,7 +44,8 @@ "mode": "verify_full" }, "bootstrap": { - "auto_migrate": true, + "schema_autocreate": true, + "schema_autoupdate": true, "migration_timeout_ms": 30000, "migration_lock_timeout_ms": 10000 }, @@ -67,7 +69,8 @@ "mode": "verify_full" }, "bootstrap": { - "auto_migrate": true, + "schema_autocreate": true, + "schema_autoupdate": true, "migration_timeout_ms": 30000, "migration_lock_timeout_ms": 10000 }, diff --git a/config/schemas/std.store.schema.json b/config/schemas/std.store.schema.json index 427d7ab..de9d7ad 100644 --- a/config/schemas/std.store.schema.json +++ b/config/schemas/std.store.schema.json @@ -1,6 +1,6 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "urn:ksp:schema:std.store:v1", + "$id": "urn:ksp:schema:std.store:v2", "title": "KSP standard Store configuration", "type": "object", "additionalProperties": false, @@ -11,19 +11,45 @@ ], "properties": { "format_version": { - "const": 1 + "enum": [ + 1, + 2 + ] }, "default_profile": { "$ref": "#/$defs/profileId" }, "profiles": { "type": "array", - "minItems": 1, - "items": { - "$ref": "#/$defs/profile" - } + "minItems": 1 } }, + "oneOf": [ + { + "properties": { + "format_version": { + "const": 1 + }, + "profiles": { + "items": { + "$ref": "#/$defs/profileV1" + } + } + } + }, + { + "properties": { + "format_version": { + "const": 2 + }, + "profiles": { + "items": { + "$ref": "#/$defs/profileV2" + } + } + } + } + ], "$defs": { "profileId": { "type": "string", @@ -40,7 +66,7 @@ "minimum": 100, "maximum": 60000 }, - "profile": { + "profileV1": { "type": "object", "additionalProperties": false, "required": [ @@ -60,11 +86,35 @@ "const": "postgres" }, "postgres": { - "$ref": "#/$defs/postgres" + "$ref": "#/$defs/postgresV1" } } }, - "postgres": { + "profileV2": { + "type": "object", + "additionalProperties": false, + "required": [ + "profile_id", + "network", + "backend", + "postgres" + ], + "properties": { + "profile_id": { + "$ref": "#/$defs/profileId" + }, + "network": { + "$ref": "#/$defs/networkId" + }, + "backend": { + "const": "postgres" + }, + "postgres": { + "$ref": "#/$defs/postgresV2" + } + } + }, + "postgresV1": { "type": "object", "additionalProperties": false, "required": [ @@ -86,7 +136,38 @@ "$ref": "#/$defs/tls" }, "bootstrap": { - "$ref": "#/$defs/bootstrap" + "$ref": "#/$defs/bootstrapV1" + }, + "shutdown_timeout_ms": { + "type": "integer", + "minimum": 100, + "maximum": 30000 + } + } + }, + "postgresV2": { + "type": "object", + "additionalProperties": false, + "required": [ + "connection_uri", + "pool", + "tls", + "bootstrap", + "shutdown_timeout_ms" + ], + "properties": { + "connection_uri": { + "type": "string", + "minLength": 1 + }, + "pool": { + "$ref": "#/$defs/pool" + }, + "tls": { + "$ref": "#/$defs/tls" + }, + "bootstrap": { + "$ref": "#/$defs/bootstrapV2" }, "shutdown_timeout_ms": { "type": "integer", @@ -140,7 +221,7 @@ } } }, - "bootstrap": { + "bootstrapV1": { "type": "object", "additionalProperties": false, "required": [ @@ -163,6 +244,34 @@ "maximum": 120000 } } + }, + "bootstrapV2": { + "type": "object", + "additionalProperties": false, + "required": [ + "schema_autocreate", + "schema_autoupdate", + "migration_timeout_ms", + "migration_lock_timeout_ms" + ], + "properties": { + "schema_autocreate": { + "type": "boolean" + }, + "schema_autoupdate": { + "type": "boolean" + }, + "migration_timeout_ms": { + "type": "integer", + "minimum": 1000, + "maximum": 300000 + }, + "migration_lock_timeout_ms": { + "type": "integer", + "minimum": 100, + "maximum": 120000 + } + } } } } diff --git a/config/std.store.json b/config/std.store.json index 1c62849..904a758 100644 --- a/config/std.store.json +++ b/config/std.store.json @@ -1,5 +1,5 @@ { - "format_version": 1, + "format_version": 2, "default_profile": "devnet", "profiles": [ { @@ -19,7 +19,8 @@ "mode": "verify_full" }, "bootstrap": { - "auto_migrate": true, + "schema_autocreate": true, + "schema_autoupdate": true, "migration_timeout_ms": 30000, "migration_lock_timeout_ms": 10000 }, @@ -43,7 +44,8 @@ "mode": "verify_full" }, "bootstrap": { - "auto_migrate": true, + "schema_autocreate": true, + "schema_autoupdate": true, "migration_timeout_ms": 30000, "migration_lock_timeout_ms": 10000 }, @@ -67,7 +69,8 @@ "mode": "verify_full" }, "bootstrap": { - "auto_migrate": true, + "schema_autocreate": true, + "schema_autoupdate": true, "migration_timeout_ms": 30000, "migration_lock_timeout_ms": 10000 }, diff --git a/crates/ksp-config-lib/src/store.rs b/crates/ksp-config-lib/src/store.rs index 7905438..a9cb6c9 100644 --- a/crates/ksp-config-lib/src/store.rs +++ b/crates/ksp-config-lib/src/store.rs @@ -1,5 +1,5 @@ // file: crates/ksp-config-lib/src/store.rs -// version: 2 +// version: 3 /// Effective standard Store configuration mapped to `ksp_store_lib::StoreSettings`. pub struct ResolvedStoreConfig { @@ -153,9 +153,11 @@ struct EffectivePostgresTlsSource { #[derive(serde::Deserialize)] #[serde(deny_unknown_fields)] struct EffectivePostgresBootstrapSource { - auto_migrate: bool, + auto_migrate: std::option::Option, migration_lock_timeout_ms: u64, migration_timeout_ms: u64, + schema_autocreate: std::option::Option, + schema_autoupdate: std::option::Option, } fn resolve_store_profile(profile: &crate::ResolvedConfigProfile, environment: &crate::ConfigEnvironment) -> ksp_core_lib::Result { @@ -178,7 +180,7 @@ fn resolve_store_profile(profile: &crate::ResolvedConfigProfile, environment: &c ); }, }; - if source.format_version != 1 { + if source.format_version != 1 && source.format_version != 2 { return std::result::Result::Err(effective_error(profile, "effective Store format_version is unsupported")); } if source.profile_id != profile.profile_id() { @@ -199,11 +201,34 @@ fn resolve_store_profile(profile: &crate::ResolvedConfigProfile, environment: &c std::time::Duration::from_millis(source.postgres.pool.create_timeout_ms), std::time::Duration::from_millis(source.postgres.pool.recycle_timeout_ms), ); - let bootstrap = ksp_store_lib::PostgresBootstrapSettings::new( - source.postgres.bootstrap.auto_migrate, - std::time::Duration::from_millis(source.postgres.bootstrap.migration_timeout_ms), - std::time::Duration::from_millis(source.postgres.bootstrap.migration_lock_timeout_ms), - ); + let bootstrap = match source.format_version { + 1 => { + let auto_migrate = + match (source.postgres.bootstrap.auto_migrate, source.postgres.bootstrap.schema_autocreate, source.postgres.bootstrap.schema_autoupdate) { + (std::option::Option::Some(value), std::option::Option::None, std::option::Option::None) => value, + _ => return std::result::Result::Err(effective_error(profile, "effective Store V1 bootstrap policy is invalid")), + }; + ksp_store_lib::PostgresBootstrapSettings::new( + auto_migrate, + std::time::Duration::from_millis(source.postgres.bootstrap.migration_timeout_ms), + std::time::Duration::from_millis(source.postgres.bootstrap.migration_lock_timeout_ms), + ) + }, + 2 => { + let (schema_autocreate, schema_autoupdate) = + match (source.postgres.bootstrap.auto_migrate, source.postgres.bootstrap.schema_autocreate, source.postgres.bootstrap.schema_autoupdate) { + (std::option::Option::None, std::option::Option::Some(autocreate), std::option::Option::Some(autoupdate)) => (autocreate, autoupdate), + _ => return std::result::Result::Err(effective_error(profile, "effective Store V2 bootstrap policy is invalid")), + }; + ksp_store_lib::PostgresBootstrapSettings::with_schema_policy( + schema_autocreate, + schema_autoupdate, + std::time::Duration::from_millis(source.postgres.bootstrap.migration_timeout_ms), + std::time::Duration::from_millis(source.postgres.bootstrap.migration_lock_timeout_ms), + ) + }, + _ => return std::result::Result::Err(effective_error(profile, "effective Store format_version is unsupported")), + }; let network = ksp_store_lib::RawNetworkId::new(source.network); let network = match network { std::result::Result::Ok(value) => value, diff --git a/crates/ksp-config-lib/unit_tests/fixtures/std.store.json b/crates/ksp-config-lib/unit_tests/fixtures/std.store.json index 1c62849..904a758 100644 --- a/crates/ksp-config-lib/unit_tests/fixtures/std.store.json +++ b/crates/ksp-config-lib/unit_tests/fixtures/std.store.json @@ -1,5 +1,5 @@ { - "format_version": 1, + "format_version": 2, "default_profile": "devnet", "profiles": [ { @@ -19,7 +19,8 @@ "mode": "verify_full" }, "bootstrap": { - "auto_migrate": true, + "schema_autocreate": true, + "schema_autoupdate": true, "migration_timeout_ms": 30000, "migration_lock_timeout_ms": 10000 }, @@ -43,7 +44,8 @@ "mode": "verify_full" }, "bootstrap": { - "auto_migrate": true, + "schema_autocreate": true, + "schema_autoupdate": true, "migration_timeout_ms": 30000, "migration_lock_timeout_ms": 10000 }, @@ -67,7 +69,8 @@ "mode": "verify_full" }, "bootstrap": { - "auto_migrate": true, + "schema_autocreate": true, + "schema_autoupdate": true, "migration_timeout_ms": 30000, "migration_lock_timeout_ms": 10000 }, diff --git a/crates/ksp-config-lib/unit_tests/store.rs b/crates/ksp-config-lib/unit_tests/store.rs index 86ece9f..7ff7610 100644 --- a/crates/ksp-config-lib/unit_tests/store.rs +++ b/crates/ksp-config-lib/unit_tests/store.rs @@ -1,5 +1,5 @@ // file: crates/ksp-config-lib/unit_tests/store.rs -// version: 2 +// version: 3 #[test] fn committed_store_profile_maps_exact_runtime_settings_and_secret_fallback() { @@ -32,6 +32,8 @@ fn committed_store_profile_maps_exact_runtime_settings_and_secret_fallback() { assert_eq!(postgres.pool().recycle_timeout(), std::time::Duration::from_millis(5_000)); assert_eq!(postgres.tls_mode(), ksp_store_lib::PostgresTlsMode::VerifyFull); assert!(postgres.bootstrap().auto_migrate()); + assert!(postgres.bootstrap().schema_autocreate()); + assert!(postgres.bootstrap().schema_autoupdate()); assert_eq!(postgres.bootstrap().migration_timeout(), std::time::Duration::from_millis(30_000)); assert_eq!(postgres.bootstrap().migration_lock_timeout(), std::time::Duration::from_millis(10_000)); } @@ -140,6 +142,90 @@ fn named_store_targets_select_one_network_and_database_without_runtime_multiplex return; } +#[test] +fn pre_003_fix_001_v1_auto_migrate_remains_backward_readable_and_maps_both_schema_policies() { + for auto_migrate in [false, true] { + let fixture = tempfile::tempdir(); + assert!(fixture.is_ok()); + let fixture = match fixture { + std::result::Result::Ok(value) => value, + std::result::Result::Err(_) => return, + }; + let source = committed_document_value(); + let mut source = match source { + std::result::Result::Ok(value) => value, + std::result::Result::Err(_) => return, + }; + source["format_version"] = serde_json::Value::from(1); + let profiles = source.get_mut("profiles").and_then(serde_json::Value::as_array_mut); + if let std::option::Option::Some(profiles) = profiles { + for profile in profiles { + let bootstrap = profile.pointer_mut("/postgres/bootstrap").and_then(serde_json::Value::as_object_mut); + if let std::option::Option::Some(bootstrap) = bootstrap { + bootstrap.remove("schema_autocreate"); + bootstrap.remove("schema_autoupdate"); + bootstrap.insert("auto_migrate".to_owned(), serde_json::Value::Bool(auto_migrate)); + } + } + } + let engine = fixture_engine_with_document(fixture.path(), &source); + assert!(engine.is_ok()); + let engine = match engine { + std::result::Result::Ok(value) => value, + std::result::Result::Err(_) => return, + }; + let environment = crate::ConfigEnvironment::from_maps(std::collections::BTreeMap::new(), std::collections::BTreeMap::new()); + let resolved = engine.load_resolved_store_config(std::option::Option::None, &environment); + assert!(resolved.is_ok(), "Store Config V1 compatibility mapping failed: {resolved:?}"); + if let std::result::Result::Ok(resolved) = resolved + && let ksp_store_lib::StoreBackendSettings::Postgres(postgres) = resolved.settings().backend() + { + assert_eq!(postgres.bootstrap().schema_autocreate(), auto_migrate); + assert_eq!(postgres.bootstrap().schema_autoupdate(), auto_migrate); + assert_eq!(postgres.bootstrap().auto_migrate(), auto_migrate); + } + } + return; +} + +#[test] +fn pre_003_fix_001_v2_schema_creation_and_update_policies_map_independently() { + let fixture = tempfile::tempdir(); + assert!(fixture.is_ok()); + let fixture = match fixture { + std::result::Result::Ok(value) => value, + std::result::Result::Err(_) => return, + }; + let source = committed_document_value(); + let mut source = match source { + std::result::Result::Ok(value) => value, + std::result::Result::Err(_) => return, + }; + let profiles = source.get_mut("profiles").and_then(serde_json::Value::as_array_mut); + if let std::option::Option::Some(profiles) = profiles + && let std::option::Option::Some(profile) = profiles.first_mut() + { + profile["postgres"]["bootstrap"]["schema_autocreate"] = serde_json::Value::Bool(false); + profile["postgres"]["bootstrap"]["schema_autoupdate"] = serde_json::Value::Bool(true); + } + let engine = fixture_engine_with_document(fixture.path(), &source); + assert!(engine.is_ok()); + let engine = match engine { + std::result::Result::Ok(value) => value, + std::result::Result::Err(_) => return, + }; + let environment = crate::ConfigEnvironment::from_maps(std::collections::BTreeMap::new(), std::collections::BTreeMap::new()); + let resolved = engine.load_resolved_store_config(std::option::Option::None, &environment); + assert!(resolved.is_ok(), "Store Config V2 split schema policy mapping failed: {resolved:?}"); + if let std::result::Result::Ok(resolved) = resolved + && let ksp_store_lib::StoreBackendSettings::Postgres(postgres) = resolved.settings().backend() + { + assert!(!postgres.bootstrap().schema_autocreate()); + assert!(postgres.bootstrap().schema_autoupdate()); + } + return; +} + fn committed_engine() -> ksp_core_lib::Result { let workspace = workspace_root(); let bootstrap = crate::ConfigBootstrapOptions::from_paths(workspace.join("config"), workspace.join("config/schemas")); diff --git a/crates/ksp-store-lib/src/settings.rs b/crates/ksp-store-lib/src/settings.rs index 948e022..2f69ee2 100644 --- a/crates/ksp-store-lib/src/settings.rs +++ b/crates/ksp-store-lib/src/settings.rs @@ -1,5 +1,5 @@ // file: crates/ksp-store-lib/src/settings.rs -// version: 3 +// version: 4 const DEFAULT_CONNECT_TIMEOUT_MS: u64 = 10_000; const DEFAULT_MAX_CONNECTIONS: u32 = 8; @@ -153,22 +153,49 @@ impl std::default::Default for PostgresPoolSettings { /// Bounded PostgreSQL bootstrap settings owned by the Store facade. #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub struct PostgresBootstrapSettings { - auto_migrate: bool, migration_lock_timeout: std::time::Duration, migration_timeout: std::time::Duration, + schema_autocreate: bool, + schema_autoupdate: bool, } impl PostgresBootstrapSettings { - /// Creates explicit bootstrap behavior and migration deadlines. + /// Creates bootstrap settings using the legacy single migration switch for source compatibility. + /// + /// The supplied value is mapped to both schema auto-creation and schema auto-update. New code should prefer + /// [`Self::with_schema_policy`] when these policies need to differ. #[must_use] pub const fn new(auto_migrate: bool, migration_timeout: std::time::Duration, migration_lock_timeout: std::time::Duration) -> Self { - return Self { auto_migrate, migration_lock_timeout, migration_timeout }; + return Self::with_schema_policy(auto_migrate, auto_migrate, migration_timeout, migration_lock_timeout); } - /// Returns whether pending KSP-owned migrations may be applied during Store opening. + /// Creates explicit schema creation/update policy and migration deadlines. + #[must_use] + pub const fn with_schema_policy( + schema_autocreate: bool, + schema_autoupdate: bool, + migration_timeout: std::time::Duration, + migration_lock_timeout: std::time::Duration, + ) -> Self { + return Self { migration_lock_timeout, migration_timeout, schema_autocreate, schema_autoupdate }; + } + + /// Returns the legacy pending-migration switch, mapped to the schema auto-update policy. #[must_use] pub const fn auto_migrate(&self) -> bool { - return self.auto_migrate; + return self.schema_autoupdate; + } + + /// Returns whether an absent KSP-managed schema may be created or adopted during Store opening. + #[must_use] + pub const fn schema_autocreate(&self) -> bool { + return self.schema_autocreate; + } + + /// Returns whether pending migrations and safe additive schema repairs may be applied during Store opening. + #[must_use] + pub const fn schema_autoupdate(&self) -> bool { + return self.schema_autoupdate; } /// Returns the bounded wait allowed for the private PostgreSQL migration lock. @@ -205,7 +232,8 @@ impl PostgresBootstrapSettings { impl std::default::Default for PostgresBootstrapSettings { fn default() -> Self { - return Self::new( + return Self::with_schema_policy( + true, true, std::time::Duration::from_millis(DEFAULT_MIGRATION_TIMEOUT_MS), std::time::Duration::from_millis(DEFAULT_MIGRATION_LOCK_TIMEOUT_MS), diff --git a/crates/ksp-store-lib/src/store.rs b/crates/ksp-store-lib/src/store.rs index 17327f4..5e18a7f 100644 --- a/crates/ksp-store-lib/src/store.rs +++ b/crates/ksp-store-lib/src/store.rs @@ -1,5 +1,5 @@ // file: crates/ksp-store-lib/src/store.rs -// version: 4 +// version: 5 /// Opaque common Store runtime facade. /// @@ -134,7 +134,7 @@ async fn open_postgres( crate::PostgresTlsMode::Disabled => ksp_store_postgres_lib::PostgresBackendTlsMode::Disabled, crate::PostgresTlsMode::VerifyFull => ksp_store_postgres_lib::PostgresBackendTlsMode::VerifyFull, }; - let backend_settings = ksp_store_postgres_lib::PostgresBackendSettings::new( + let backend_settings = ksp_store_postgres_lib::PostgresBackendSettings::with_schema_policy( network.clone(), settings.connection_uri(), pool.max_connections(), @@ -143,7 +143,8 @@ async fn open_postgres( pool.create_timeout(), pool.recycle_timeout(), tls_mode, - bootstrap.auto_migrate(), + bootstrap.schema_autocreate(), + bootstrap.schema_autoupdate(), bootstrap.migration_timeout(), bootstrap.migration_lock_timeout(), ); diff --git a/crates/ksp-store-lib/unit_tests/settings.rs b/crates/ksp-store-lib/unit_tests/settings.rs index 70a4b3d..086b693 100644 --- a/crates/ksp-store-lib/unit_tests/settings.rs +++ b/crates/ksp-store-lib/unit_tests/settings.rs @@ -1,5 +1,5 @@ // file: crates/ksp-store-lib/unit_tests/settings.rs -// version: 3 +// version: 4 fn valid_network() -> crate::RawNetworkId { return match crate::RawNetworkId::new("devnet") { @@ -27,6 +27,12 @@ fn defaults_match_the_pre_001_runtime_bounds() { assert_eq!(pool.recycle_timeout(), std::time::Duration::from_millis(5_000)); let bootstrap = crate::PostgresBootstrapSettings::default(); assert!(bootstrap.auto_migrate()); + assert!(bootstrap.schema_autocreate()); + assert!(bootstrap.schema_autoupdate()); + let split_policy = + crate::PostgresBootstrapSettings::with_schema_policy(false, true, std::time::Duration::from_millis(30_000), std::time::Duration::from_millis(10_000)); + assert!(!split_policy.schema_autocreate()); + assert!(split_policy.schema_autoupdate()); assert_eq!(bootstrap.migration_timeout(), std::time::Duration::from_millis(30_000)); assert_eq!(bootstrap.migration_lock_timeout(), std::time::Duration::from_millis(10_000)); let store = crate::StoreSettings::with_default_shutdown(valid_network(), crate::StoreBackendSettings::Postgres(valid_postgres_settings())); diff --git a/crates/ksp-store-postgres-lib/migrations/V001__raw_transaction.sql b/crates/ksp-store-postgres-lib/migrations/V001__raw_transaction.sql deleted file mode 100644 index 6d35a08..0000000 --- a/crates/ksp-store-postgres-lib/migrations/V001__raw_transaction.sql +++ /dev/null @@ -1,133 +0,0 @@ -CREATE TABLE ksp_store_identity ( - singleton SMALLINT PRIMARY KEY, - network TEXT NOT NULL, - CONSTRAINT ck_ksp_store_identity_singleton CHECK (singleton = 1), - CONSTRAINT ck_ksp_store_identity_network CHECK ( - octet_length(network) BETWEEN 1 AND 128 - AND network ~ '^[A-Za-z0-9_.:-]+$' - ) -); - -CREATE TABLE ksp_raw_transactions ( - signature BYTEA PRIMARY KEY, - slot NUMERIC(20, 0) NOT NULL, - block_time_unix_millis BIGINT NULL, - format_id TEXT NOT NULL, - format_version BIGINT NOT NULL, - content_hash BYTEA NOT NULL, - payload BYTEA NULL, - retention_state TEXT NOT NULL, - CONSTRAINT ck_ksp_raw_transactions_signature CHECK (octet_length(signature) = 64), - CONSTRAINT ck_ksp_raw_transactions_slot CHECK (slot BETWEEN 0 AND 18446744073709551615), - CONSTRAINT ck_ksp_raw_transactions_block_time CHECK ( - block_time_unix_millis IS NULL - OR block_time_unix_millis BETWEEN 0 AND 253402300799999 - ), - CONSTRAINT ck_ksp_raw_transactions_format_id CHECK ( - octet_length(format_id) BETWEEN 1 AND 128 - AND format_id ~ '^[A-Za-z0-9_.:-]+$' - ), - CONSTRAINT ck_ksp_raw_transactions_format_version CHECK (format_version BETWEEN 1 AND 4294967295), - CONSTRAINT ck_ksp_raw_transactions_content_hash CHECK (octet_length(content_hash) = 32), - CONSTRAINT ck_ksp_raw_transactions_payload CHECK ( - payload IS NULL - OR octet_length(payload) BETWEEN 1 AND 16777216 - ), - CONSTRAINT ck_ksp_raw_transactions_retention_state CHECK (retention_state IN ('full', 'archived', 'purged')), - CONSTRAINT ck_ksp_raw_transactions_payload_state CHECK ( - (retention_state = 'full' AND payload IS NOT NULL) - OR (retention_state IN ('archived', 'purged') AND payload IS NULL) - ), - CONSTRAINT ck_ksp_raw_transactions_purged_block_time CHECK ( - retention_state <> 'purged' - OR block_time_unix_millis IS NULL - ) -); - -CREATE TABLE ksp_raw_transaction_observations ( - observation_key BYTEA PRIMARY KEY, - transaction_signature BYTEA NOT NULL REFERENCES ksp_raw_transactions(signature) ON DELETE RESTRICT, - provider TEXT NOT NULL, - protocol TEXT NOT NULL, - acquisition_method TEXT NOT NULL, - origin TEXT NOT NULL, - received_at_unix_millis BIGINT NOT NULL, - capture_session_id TEXT NULL, - commitment TEXT NULL, - endpoint_id TEXT NULL, - filter_id TEXT NULL, - observed_at_unix_millis BIGINT NULL, - source_payload_hash BYTEA NULL, - source_payload_size_bytes BIGINT NULL, - CONSTRAINT ck_ksp_raw_transaction_observations_key CHECK (octet_length(observation_key) = 32), - CONSTRAINT ck_ksp_raw_transaction_observations_signature CHECK (octet_length(transaction_signature) = 64), - CONSTRAINT ck_ksp_raw_transaction_observations_provider CHECK ( - octet_length(provider) BETWEEN 1 AND 128 - AND provider ~ '^[A-Za-z0-9_.:-]+$' - ), - CONSTRAINT ck_ksp_raw_transaction_observations_protocol CHECK ( - octet_length(protocol) BETWEEN 1 AND 128 - AND protocol ~ '^[A-Za-z0-9_.:-]+$' - ), - CONSTRAINT ck_ksp_raw_transaction_observations_method CHECK ( - octet_length(acquisition_method) BETWEEN 1 AND 128 - AND acquisition_method ~ '^[A-Za-z0-9_.:-]+$' - ), - CONSTRAINT ck_ksp_raw_transaction_observations_origin CHECK (origin IN ('backfill', 'import', 'live', 'repair', 'replay')), - CONSTRAINT ck_ksp_raw_transaction_observations_received_at CHECK (received_at_unix_millis BETWEEN 0 AND 253402300799999), - CONSTRAINT ck_ksp_raw_transaction_observations_capture_session CHECK ( - capture_session_id IS NULL - OR ( - octet_length(capture_session_id) BETWEEN 1 AND 128 - AND capture_session_id ~ '^[A-Za-z0-9_.:-]+$' - ) - ), - CONSTRAINT ck_ksp_raw_transaction_observations_commitment CHECK ( - commitment IS NULL - OR ( - octet_length(commitment) BETWEEN 1 AND 128 - AND commitment ~ '^[A-Za-z0-9_.:-]+$' - ) - ), - CONSTRAINT ck_ksp_raw_transaction_observations_endpoint CHECK ( - endpoint_id IS NULL - OR ( - octet_length(endpoint_id) BETWEEN 1 AND 128 - AND endpoint_id ~ '^[A-Za-z0-9_.:-]+$' - ) - ), - CONSTRAINT ck_ksp_raw_transaction_observations_filter CHECK ( - filter_id IS NULL - OR ( - octet_length(filter_id) BETWEEN 1 AND 128 - AND filter_id ~ '^[A-Za-z0-9_.:-]+$' - ) - ), - CONSTRAINT ck_ksp_raw_transaction_observations_observed_at CHECK ( - observed_at_unix_millis IS NULL - OR observed_at_unix_millis BETWEEN 0 AND 253402300799999 - ), - CONSTRAINT ck_ksp_raw_transaction_observations_time_order CHECK ( - observed_at_unix_millis IS NULL - OR observed_at_unix_millis <= received_at_unix_millis - ), - CONSTRAINT ck_ksp_raw_transaction_observations_source_hash CHECK ( - source_payload_hash IS NULL - OR octet_length(source_payload_hash) = 32 - ), - CONSTRAINT ck_ksp_raw_transaction_observations_source_size CHECK ( - source_payload_size_bytes IS NULL - OR source_payload_size_bytes BETWEEN 0 AND 67108864 - ) -); - -CREATE TABLE ksp_raw_transaction_archive_payloads ( - signature BYTEA PRIMARY KEY REFERENCES ksp_raw_transactions(signature) ON DELETE RESTRICT, - payload BYTEA NOT NULL, - CONSTRAINT ck_ksp_raw_transaction_archive_payloads_signature CHECK (octet_length(signature) = 64), - CONSTRAINT ck_ksp_raw_transaction_archive_payloads_payload CHECK (octet_length(payload) BETWEEN 1 AND 16777216) -); - -CREATE INDEX ix_ksp_raw_transactions_slot_signature - ON ksp_raw_transactions (slot, signature) - WHERE retention_state <> 'purged'; diff --git a/crates/ksp-store-postgres-lib/migrations/V000__bootstrap.sql b/crates/ksp-store-postgres-lib/migrations/v000_bootstrap/tables/001_ksp_store_schema_migrations.sql similarity index 100% rename from crates/ksp-store-postgres-lib/migrations/V000__bootstrap.sql rename to crates/ksp-store-postgres-lib/migrations/v000_bootstrap/tables/001_ksp_store_schema_migrations.sql diff --git a/crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/001_pk_ksp_store_identity.sql b/crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/001_pk_ksp_store_identity.sql new file mode 100644 index 0000000..39a184d --- /dev/null +++ b/crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/001_pk_ksp_store_identity.sql @@ -0,0 +1,12 @@ +DO $ksp$ +BEGIN + IF NOT EXISTS ( + SELECT 1 + FROM pg_constraint + WHERE conname = 'pk_ksp_store_identity' + AND conrelid = to_regclass('ksp_store_identity') + ) THEN + ALTER TABLE ksp_store_identity ADD CONSTRAINT pk_ksp_store_identity PRIMARY KEY (singleton); + END IF; +END +$ksp$; diff --git a/crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/002_ck_ksp_store_identity_singleton.sql b/crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/002_ck_ksp_store_identity_singleton.sql new file mode 100644 index 0000000..d5ee8ea --- /dev/null +++ b/crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/002_ck_ksp_store_identity_singleton.sql @@ -0,0 +1,12 @@ +DO $ksp$ +BEGIN + IF NOT EXISTS ( + SELECT 1 + FROM pg_constraint + WHERE conname = 'ck_ksp_store_identity_singleton' + AND conrelid = to_regclass('ksp_store_identity') + ) THEN + ALTER TABLE ksp_store_identity ADD CONSTRAINT ck_ksp_store_identity_singleton CHECK (singleton = 1); + END IF; +END +$ksp$; diff --git a/crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/003_ck_ksp_store_identity_network.sql b/crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/003_ck_ksp_store_identity_network.sql new file mode 100644 index 0000000..8f2df39 --- /dev/null +++ b/crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/003_ck_ksp_store_identity_network.sql @@ -0,0 +1,15 @@ +DO $ksp$ +BEGIN + IF NOT EXISTS ( + SELECT 1 + FROM pg_constraint + WHERE conname = 'ck_ksp_store_identity_network' + AND conrelid = to_regclass('ksp_store_identity') + ) THEN + ALTER TABLE ksp_store_identity ADD CONSTRAINT ck_ksp_store_identity_network CHECK ( + octet_length(network) >= 1 AND octet_length(network) <= 128 + AND network ~ '^[A-Za-z0-9_.:-]+$' +); + END IF; +END +$ksp$; diff --git a/crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/004_pk_ksp_raw_transactions.sql b/crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/004_pk_ksp_raw_transactions.sql new file mode 100644 index 0000000..f787681 --- /dev/null +++ b/crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/004_pk_ksp_raw_transactions.sql @@ -0,0 +1,12 @@ +DO $ksp$ +BEGIN + IF NOT EXISTS ( + SELECT 1 + FROM pg_constraint + WHERE conname = 'pk_ksp_raw_transactions' + AND conrelid = to_regclass('ksp_raw_transactions') + ) THEN + ALTER TABLE ksp_raw_transactions ADD CONSTRAINT pk_ksp_raw_transactions PRIMARY KEY (signature); + END IF; +END +$ksp$; diff --git a/crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/005_ck_ksp_raw_transactions_signature.sql b/crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/005_ck_ksp_raw_transactions_signature.sql new file mode 100644 index 0000000..5e16165 --- /dev/null +++ b/crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/005_ck_ksp_raw_transactions_signature.sql @@ -0,0 +1,12 @@ +DO $ksp$ +BEGIN + IF NOT EXISTS ( + SELECT 1 + FROM pg_constraint + WHERE conname = 'ck_ksp_raw_transactions_signature' + AND conrelid = to_regclass('ksp_raw_transactions') + ) THEN + ALTER TABLE ksp_raw_transactions ADD CONSTRAINT ck_ksp_raw_transactions_signature CHECK (octet_length(signature) = 64); + END IF; +END +$ksp$; diff --git a/crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/006_ck_ksp_raw_transactions_slot.sql b/crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/006_ck_ksp_raw_transactions_slot.sql new file mode 100644 index 0000000..820a802 --- /dev/null +++ b/crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/006_ck_ksp_raw_transactions_slot.sql @@ -0,0 +1,12 @@ +DO $ksp$ +BEGIN + IF NOT EXISTS ( + SELECT 1 + FROM pg_constraint + WHERE conname = 'ck_ksp_raw_transactions_slot' + AND conrelid = to_regclass('ksp_raw_transactions') + ) THEN + ALTER TABLE ksp_raw_transactions ADD CONSTRAINT ck_ksp_raw_transactions_slot CHECK (slot >= 0 AND slot <= 18446744073709551615); + END IF; +END +$ksp$; diff --git a/crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/007_ck_ksp_raw_transactions_block_time.sql b/crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/007_ck_ksp_raw_transactions_block_time.sql new file mode 100644 index 0000000..bcf297b --- /dev/null +++ b/crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/007_ck_ksp_raw_transactions_block_time.sql @@ -0,0 +1,15 @@ +DO $ksp$ +BEGIN + IF NOT EXISTS ( + SELECT 1 + FROM pg_constraint + WHERE conname = 'ck_ksp_raw_transactions_block_time' + AND conrelid = to_regclass('ksp_raw_transactions') + ) THEN + ALTER TABLE ksp_raw_transactions ADD CONSTRAINT ck_ksp_raw_transactions_block_time CHECK ( + block_time_unix_millis IS NULL + OR block_time_unix_millis >= 0 AND block_time_unix_millis <= 253402300799999 +); + END IF; +END +$ksp$; diff --git a/crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/008_ck_ksp_raw_transactions_format_id.sql b/crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/008_ck_ksp_raw_transactions_format_id.sql new file mode 100644 index 0000000..1a0d579 --- /dev/null +++ b/crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/008_ck_ksp_raw_transactions_format_id.sql @@ -0,0 +1,15 @@ +DO $ksp$ +BEGIN + IF NOT EXISTS ( + SELECT 1 + FROM pg_constraint + WHERE conname = 'ck_ksp_raw_transactions_format_id' + AND conrelid = to_regclass('ksp_raw_transactions') + ) THEN + ALTER TABLE ksp_raw_transactions ADD CONSTRAINT ck_ksp_raw_transactions_format_id CHECK ( + octet_length(format_id) >= 1 AND octet_length(format_id) <= 128 + AND format_id ~ '^[A-Za-z0-9_.:-]+$' +); + END IF; +END +$ksp$; diff --git a/crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/009_ck_ksp_raw_transactions_format_version.sql b/crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/009_ck_ksp_raw_transactions_format_version.sql new file mode 100644 index 0000000..1339214 --- /dev/null +++ b/crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/009_ck_ksp_raw_transactions_format_version.sql @@ -0,0 +1,12 @@ +DO $ksp$ +BEGIN + IF NOT EXISTS ( + SELECT 1 + FROM pg_constraint + WHERE conname = 'ck_ksp_raw_transactions_format_version' + AND conrelid = to_regclass('ksp_raw_transactions') + ) THEN + ALTER TABLE ksp_raw_transactions ADD CONSTRAINT ck_ksp_raw_transactions_format_version CHECK (format_version >= 1 AND format_version <= 4294967295); + END IF; +END +$ksp$; diff --git a/crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/010_ck_ksp_raw_transactions_content_hash.sql b/crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/010_ck_ksp_raw_transactions_content_hash.sql new file mode 100644 index 0000000..e29dd76 --- /dev/null +++ b/crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/010_ck_ksp_raw_transactions_content_hash.sql @@ -0,0 +1,12 @@ +DO $ksp$ +BEGIN + IF NOT EXISTS ( + SELECT 1 + FROM pg_constraint + WHERE conname = 'ck_ksp_raw_transactions_content_hash' + AND conrelid = to_regclass('ksp_raw_transactions') + ) THEN + ALTER TABLE ksp_raw_transactions ADD CONSTRAINT ck_ksp_raw_transactions_content_hash CHECK (octet_length(content_hash) = 32); + END IF; +END +$ksp$; diff --git a/crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/011_ck_ksp_raw_transactions_payload.sql b/crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/011_ck_ksp_raw_transactions_payload.sql new file mode 100644 index 0000000..374e8bf --- /dev/null +++ b/crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/011_ck_ksp_raw_transactions_payload.sql @@ -0,0 +1,15 @@ +DO $ksp$ +BEGIN + IF NOT EXISTS ( + SELECT 1 + FROM pg_constraint + WHERE conname = 'ck_ksp_raw_transactions_payload' + AND conrelid = to_regclass('ksp_raw_transactions') + ) THEN + ALTER TABLE ksp_raw_transactions ADD CONSTRAINT ck_ksp_raw_transactions_payload CHECK ( + payload IS NULL + OR octet_length(payload) >= 1 AND octet_length(payload) <= 16777216 +); + END IF; +END +$ksp$; diff --git a/crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/012_ck_ksp_raw_transactions_retention_state.sql b/crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/012_ck_ksp_raw_transactions_retention_state.sql new file mode 100644 index 0000000..ece119d --- /dev/null +++ b/crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/012_ck_ksp_raw_transactions_retention_state.sql @@ -0,0 +1,12 @@ +DO $ksp$ +BEGIN + IF NOT EXISTS ( + SELECT 1 + FROM pg_constraint + WHERE conname = 'ck_ksp_raw_transactions_retention_state' + AND conrelid = to_regclass('ksp_raw_transactions') + ) THEN + ALTER TABLE ksp_raw_transactions ADD CONSTRAINT ck_ksp_raw_transactions_retention_state CHECK ((retention_state = 'full' OR retention_state = 'archived' OR retention_state = 'purged')); + END IF; +END +$ksp$; diff --git a/crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/013_ck_ksp_raw_transactions_payload_state.sql b/crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/013_ck_ksp_raw_transactions_payload_state.sql new file mode 100644 index 0000000..7e0c2c2 --- /dev/null +++ b/crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/013_ck_ksp_raw_transactions_payload_state.sql @@ -0,0 +1,15 @@ +DO $ksp$ +BEGIN + IF NOT EXISTS ( + SELECT 1 + FROM pg_constraint + WHERE conname = 'ck_ksp_raw_transactions_payload_state' + AND conrelid = to_regclass('ksp_raw_transactions') + ) THEN + ALTER TABLE ksp_raw_transactions ADD CONSTRAINT ck_ksp_raw_transactions_payload_state CHECK ( + (retention_state = 'full' AND payload IS NOT NULL) + OR ((retention_state = 'archived' OR retention_state = 'purged') AND payload IS NULL) +); + END IF; +END +$ksp$; diff --git a/crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/014_ck_ksp_raw_transactions_purged_block_time.sql b/crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/014_ck_ksp_raw_transactions_purged_block_time.sql new file mode 100644 index 0000000..0a2841e --- /dev/null +++ b/crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/014_ck_ksp_raw_transactions_purged_block_time.sql @@ -0,0 +1,15 @@ +DO $ksp$ +BEGIN + IF NOT EXISTS ( + SELECT 1 + FROM pg_constraint + WHERE conname = 'ck_ksp_raw_transactions_purged_block_time' + AND conrelid = to_regclass('ksp_raw_transactions') + ) THEN + ALTER TABLE ksp_raw_transactions ADD CONSTRAINT ck_ksp_raw_transactions_purged_block_time CHECK ( + retention_state <> 'purged' + OR block_time_unix_millis IS NULL +); + END IF; +END +$ksp$; diff --git a/crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/015_pk_ksp_raw_transaction_observations.sql b/crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/015_pk_ksp_raw_transaction_observations.sql new file mode 100644 index 0000000..7ecb66e --- /dev/null +++ b/crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/015_pk_ksp_raw_transaction_observations.sql @@ -0,0 +1,12 @@ +DO $ksp$ +BEGIN + IF NOT EXISTS ( + SELECT 1 + FROM pg_constraint + WHERE conname = 'pk_ksp_raw_transaction_observations' + AND conrelid = to_regclass('ksp_raw_transaction_observations') + ) THEN + ALTER TABLE ksp_raw_transaction_observations ADD CONSTRAINT pk_ksp_raw_transaction_observations PRIMARY KEY (observation_key); + END IF; +END +$ksp$; diff --git a/crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/016_fk_ksp_raw_transaction_observations_transaction.sql b/crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/016_fk_ksp_raw_transaction_observations_transaction.sql new file mode 100644 index 0000000..e6aca1b --- /dev/null +++ b/crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/016_fk_ksp_raw_transaction_observations_transaction.sql @@ -0,0 +1,12 @@ +DO $ksp$ +BEGIN + IF NOT EXISTS ( + SELECT 1 + FROM pg_constraint + WHERE conname = 'fk_ksp_raw_transaction_observations_transaction' + AND conrelid = to_regclass('ksp_raw_transaction_observations') + ) THEN + ALTER TABLE ksp_raw_transaction_observations ADD CONSTRAINT fk_ksp_raw_transaction_observations_transaction FOREIGN KEY (transaction_signature) REFERENCES ksp_raw_transactions(signature) ON DELETE RESTRICT; + END IF; +END +$ksp$; diff --git a/crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/017_ck_ksp_raw_transaction_observations_key.sql b/crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/017_ck_ksp_raw_transaction_observations_key.sql new file mode 100644 index 0000000..b7ca655 --- /dev/null +++ b/crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/017_ck_ksp_raw_transaction_observations_key.sql @@ -0,0 +1,12 @@ +DO $ksp$ +BEGIN + IF NOT EXISTS ( + SELECT 1 + FROM pg_constraint + WHERE conname = 'ck_ksp_raw_transaction_observations_key' + AND conrelid = to_regclass('ksp_raw_transaction_observations') + ) THEN + ALTER TABLE ksp_raw_transaction_observations ADD CONSTRAINT ck_ksp_raw_transaction_observations_key CHECK (octet_length(observation_key) = 32); + END IF; +END +$ksp$; diff --git a/crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/018_ck_ksp_raw_transaction_observations_signature.sql b/crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/018_ck_ksp_raw_transaction_observations_signature.sql new file mode 100644 index 0000000..bc154e3 --- /dev/null +++ b/crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/018_ck_ksp_raw_transaction_observations_signature.sql @@ -0,0 +1,12 @@ +DO $ksp$ +BEGIN + IF NOT EXISTS ( + SELECT 1 + FROM pg_constraint + WHERE conname = 'ck_ksp_raw_transaction_observations_signature' + AND conrelid = to_regclass('ksp_raw_transaction_observations') + ) THEN + ALTER TABLE ksp_raw_transaction_observations ADD CONSTRAINT ck_ksp_raw_transaction_observations_signature CHECK (octet_length(transaction_signature) = 64); + END IF; +END +$ksp$; diff --git a/crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/019_ck_ksp_raw_transaction_observations_provider.sql b/crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/019_ck_ksp_raw_transaction_observations_provider.sql new file mode 100644 index 0000000..e2638e0 --- /dev/null +++ b/crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/019_ck_ksp_raw_transaction_observations_provider.sql @@ -0,0 +1,15 @@ +DO $ksp$ +BEGIN + IF NOT EXISTS ( + SELECT 1 + FROM pg_constraint + WHERE conname = 'ck_ksp_raw_transaction_observations_provider' + AND conrelid = to_regclass('ksp_raw_transaction_observations') + ) THEN + ALTER TABLE ksp_raw_transaction_observations ADD CONSTRAINT ck_ksp_raw_transaction_observations_provider CHECK ( + octet_length(provider) >= 1 AND octet_length(provider) <= 128 + AND provider ~ '^[A-Za-z0-9_.:-]+$' +); + END IF; +END +$ksp$; diff --git a/crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/020_ck_ksp_raw_transaction_observations_protocol.sql b/crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/020_ck_ksp_raw_transaction_observations_protocol.sql new file mode 100644 index 0000000..4600bbc --- /dev/null +++ b/crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/020_ck_ksp_raw_transaction_observations_protocol.sql @@ -0,0 +1,15 @@ +DO $ksp$ +BEGIN + IF NOT EXISTS ( + SELECT 1 + FROM pg_constraint + WHERE conname = 'ck_ksp_raw_transaction_observations_protocol' + AND conrelid = to_regclass('ksp_raw_transaction_observations') + ) THEN + ALTER TABLE ksp_raw_transaction_observations ADD CONSTRAINT ck_ksp_raw_transaction_observations_protocol CHECK ( + octet_length(protocol) >= 1 AND octet_length(protocol) <= 128 + AND protocol ~ '^[A-Za-z0-9_.:-]+$' +); + END IF; +END +$ksp$; diff --git a/crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/021_ck_ksp_raw_transaction_observations_method.sql b/crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/021_ck_ksp_raw_transaction_observations_method.sql new file mode 100644 index 0000000..b802e93 --- /dev/null +++ b/crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/021_ck_ksp_raw_transaction_observations_method.sql @@ -0,0 +1,15 @@ +DO $ksp$ +BEGIN + IF NOT EXISTS ( + SELECT 1 + FROM pg_constraint + WHERE conname = 'ck_ksp_raw_transaction_observations_method' + AND conrelid = to_regclass('ksp_raw_transaction_observations') + ) THEN + ALTER TABLE ksp_raw_transaction_observations ADD CONSTRAINT ck_ksp_raw_transaction_observations_method CHECK ( + octet_length(acquisition_method) >= 1 AND octet_length(acquisition_method) <= 128 + AND acquisition_method ~ '^[A-Za-z0-9_.:-]+$' +); + END IF; +END +$ksp$; diff --git a/crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/022_ck_ksp_raw_transaction_observations_origin.sql b/crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/022_ck_ksp_raw_transaction_observations_origin.sql new file mode 100644 index 0000000..639c3bc --- /dev/null +++ b/crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/022_ck_ksp_raw_transaction_observations_origin.sql @@ -0,0 +1,12 @@ +DO $ksp$ +BEGIN + IF NOT EXISTS ( + SELECT 1 + FROM pg_constraint + WHERE conname = 'ck_ksp_raw_transaction_observations_origin' + AND conrelid = to_regclass('ksp_raw_transaction_observations') + ) THEN + ALTER TABLE ksp_raw_transaction_observations ADD CONSTRAINT ck_ksp_raw_transaction_observations_origin CHECK ((origin = 'backfill' OR origin = 'import' OR origin = 'live' OR origin = 'repair' OR origin = 'replay')); + END IF; +END +$ksp$; diff --git a/crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/023_ck_ksp_raw_transaction_observations_received_at.sql b/crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/023_ck_ksp_raw_transaction_observations_received_at.sql new file mode 100644 index 0000000..13525de --- /dev/null +++ b/crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/023_ck_ksp_raw_transaction_observations_received_at.sql @@ -0,0 +1,12 @@ +DO $ksp$ +BEGIN + IF NOT EXISTS ( + SELECT 1 + FROM pg_constraint + WHERE conname = 'ck_ksp_raw_transaction_observations_received_at' + AND conrelid = to_regclass('ksp_raw_transaction_observations') + ) THEN + ALTER TABLE ksp_raw_transaction_observations ADD CONSTRAINT ck_ksp_raw_transaction_observations_received_at CHECK (received_at_unix_millis >= 0 AND received_at_unix_millis <= 253402300799999); + END IF; +END +$ksp$; diff --git a/crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/024_ck_ksp_raw_transaction_observations_capture_session.sql b/crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/024_ck_ksp_raw_transaction_observations_capture_session.sql new file mode 100644 index 0000000..5123689 --- /dev/null +++ b/crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/024_ck_ksp_raw_transaction_observations_capture_session.sql @@ -0,0 +1,18 @@ +DO $ksp$ +BEGIN + IF NOT EXISTS ( + SELECT 1 + FROM pg_constraint + WHERE conname = 'ck_ksp_raw_transaction_observations_capture_session' + AND conrelid = to_regclass('ksp_raw_transaction_observations') + ) THEN + ALTER TABLE ksp_raw_transaction_observations ADD CONSTRAINT ck_ksp_raw_transaction_observations_capture_session CHECK ( + capture_session_id IS NULL + OR ( + octet_length(capture_session_id) >= 1 AND octet_length(capture_session_id) <= 128 + AND capture_session_id ~ '^[A-Za-z0-9_.:-]+$' + ) +); + END IF; +END +$ksp$; diff --git a/crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/025_ck_ksp_raw_transaction_observations_commitment.sql b/crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/025_ck_ksp_raw_transaction_observations_commitment.sql new file mode 100644 index 0000000..d896447 --- /dev/null +++ b/crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/025_ck_ksp_raw_transaction_observations_commitment.sql @@ -0,0 +1,18 @@ +DO $ksp$ +BEGIN + IF NOT EXISTS ( + SELECT 1 + FROM pg_constraint + WHERE conname = 'ck_ksp_raw_transaction_observations_commitment' + AND conrelid = to_regclass('ksp_raw_transaction_observations') + ) THEN + ALTER TABLE ksp_raw_transaction_observations ADD CONSTRAINT ck_ksp_raw_transaction_observations_commitment CHECK ( + commitment IS NULL + OR ( + octet_length(commitment) >= 1 AND octet_length(commitment) <= 128 + AND commitment ~ '^[A-Za-z0-9_.:-]+$' + ) +); + END IF; +END +$ksp$; diff --git a/crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/026_ck_ksp_raw_transaction_observations_endpoint.sql b/crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/026_ck_ksp_raw_transaction_observations_endpoint.sql new file mode 100644 index 0000000..c2768b4 --- /dev/null +++ b/crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/026_ck_ksp_raw_transaction_observations_endpoint.sql @@ -0,0 +1,18 @@ +DO $ksp$ +BEGIN + IF NOT EXISTS ( + SELECT 1 + FROM pg_constraint + WHERE conname = 'ck_ksp_raw_transaction_observations_endpoint' + AND conrelid = to_regclass('ksp_raw_transaction_observations') + ) THEN + ALTER TABLE ksp_raw_transaction_observations ADD CONSTRAINT ck_ksp_raw_transaction_observations_endpoint CHECK ( + endpoint_id IS NULL + OR ( + octet_length(endpoint_id) >= 1 AND octet_length(endpoint_id) <= 128 + AND endpoint_id ~ '^[A-Za-z0-9_.:-]+$' + ) +); + END IF; +END +$ksp$; diff --git a/crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/027_ck_ksp_raw_transaction_observations_filter.sql b/crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/027_ck_ksp_raw_transaction_observations_filter.sql new file mode 100644 index 0000000..f9b5ae5 --- /dev/null +++ b/crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/027_ck_ksp_raw_transaction_observations_filter.sql @@ -0,0 +1,18 @@ +DO $ksp$ +BEGIN + IF NOT EXISTS ( + SELECT 1 + FROM pg_constraint + WHERE conname = 'ck_ksp_raw_transaction_observations_filter' + AND conrelid = to_regclass('ksp_raw_transaction_observations') + ) THEN + ALTER TABLE ksp_raw_transaction_observations ADD CONSTRAINT ck_ksp_raw_transaction_observations_filter CHECK ( + filter_id IS NULL + OR ( + octet_length(filter_id) >= 1 AND octet_length(filter_id) <= 128 + AND filter_id ~ '^[A-Za-z0-9_.:-]+$' + ) +); + END IF; +END +$ksp$; diff --git a/crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/028_ck_ksp_raw_transaction_observations_observed_at.sql b/crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/028_ck_ksp_raw_transaction_observations_observed_at.sql new file mode 100644 index 0000000..b97449d --- /dev/null +++ b/crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/028_ck_ksp_raw_transaction_observations_observed_at.sql @@ -0,0 +1,15 @@ +DO $ksp$ +BEGIN + IF NOT EXISTS ( + SELECT 1 + FROM pg_constraint + WHERE conname = 'ck_ksp_raw_transaction_observations_observed_at' + AND conrelid = to_regclass('ksp_raw_transaction_observations') + ) THEN + ALTER TABLE ksp_raw_transaction_observations ADD CONSTRAINT ck_ksp_raw_transaction_observations_observed_at CHECK ( + observed_at_unix_millis IS NULL + OR observed_at_unix_millis >= 0 AND observed_at_unix_millis <= 253402300799999 +); + END IF; +END +$ksp$; diff --git a/crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/029_ck_ksp_raw_transaction_observations_time_order.sql b/crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/029_ck_ksp_raw_transaction_observations_time_order.sql new file mode 100644 index 0000000..05c3280 --- /dev/null +++ b/crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/029_ck_ksp_raw_transaction_observations_time_order.sql @@ -0,0 +1,15 @@ +DO $ksp$ +BEGIN + IF NOT EXISTS ( + SELECT 1 + FROM pg_constraint + WHERE conname = 'ck_ksp_raw_transaction_observations_time_order' + AND conrelid = to_regclass('ksp_raw_transaction_observations') + ) THEN + ALTER TABLE ksp_raw_transaction_observations ADD CONSTRAINT ck_ksp_raw_transaction_observations_time_order CHECK ( + observed_at_unix_millis IS NULL + OR observed_at_unix_millis <= received_at_unix_millis +); + END IF; +END +$ksp$; diff --git a/crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/030_ck_ksp_raw_transaction_observations_source_hash.sql b/crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/030_ck_ksp_raw_transaction_observations_source_hash.sql new file mode 100644 index 0000000..fec826f --- /dev/null +++ b/crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/030_ck_ksp_raw_transaction_observations_source_hash.sql @@ -0,0 +1,15 @@ +DO $ksp$ +BEGIN + IF NOT EXISTS ( + SELECT 1 + FROM pg_constraint + WHERE conname = 'ck_ksp_raw_transaction_observations_source_hash' + AND conrelid = to_regclass('ksp_raw_transaction_observations') + ) THEN + ALTER TABLE ksp_raw_transaction_observations ADD CONSTRAINT ck_ksp_raw_transaction_observations_source_hash CHECK ( + source_payload_hash IS NULL + OR octet_length(source_payload_hash) = 32 +); + END IF; +END +$ksp$; diff --git a/crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/031_ck_ksp_raw_transaction_observations_source_size.sql b/crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/031_ck_ksp_raw_transaction_observations_source_size.sql new file mode 100644 index 0000000..c225f70 --- /dev/null +++ b/crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/031_ck_ksp_raw_transaction_observations_source_size.sql @@ -0,0 +1,15 @@ +DO $ksp$ +BEGIN + IF NOT EXISTS ( + SELECT 1 + FROM pg_constraint + WHERE conname = 'ck_ksp_raw_transaction_observations_source_size' + AND conrelid = to_regclass('ksp_raw_transaction_observations') + ) THEN + ALTER TABLE ksp_raw_transaction_observations ADD CONSTRAINT ck_ksp_raw_transaction_observations_source_size CHECK ( + source_payload_size_bytes IS NULL + OR source_payload_size_bytes >= 0 AND source_payload_size_bytes <= 67108864 +); + END IF; +END +$ksp$; diff --git a/crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/032_pk_ksp_raw_transaction_archive_payloads.sql b/crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/032_pk_ksp_raw_transaction_archive_payloads.sql new file mode 100644 index 0000000..eb6fd7b --- /dev/null +++ b/crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/032_pk_ksp_raw_transaction_archive_payloads.sql @@ -0,0 +1,12 @@ +DO $ksp$ +BEGIN + IF NOT EXISTS ( + SELECT 1 + FROM pg_constraint + WHERE conname = 'pk_ksp_raw_transaction_archive_payloads' + AND conrelid = to_regclass('ksp_raw_transaction_archive_payloads') + ) THEN + ALTER TABLE ksp_raw_transaction_archive_payloads ADD CONSTRAINT pk_ksp_raw_transaction_archive_payloads PRIMARY KEY (signature); + END IF; +END +$ksp$; diff --git a/crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/033_fk_ksp_raw_transaction_archive_payloads_transaction.sql b/crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/033_fk_ksp_raw_transaction_archive_payloads_transaction.sql new file mode 100644 index 0000000..3f510d6 --- /dev/null +++ b/crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/033_fk_ksp_raw_transaction_archive_payloads_transaction.sql @@ -0,0 +1,12 @@ +DO $ksp$ +BEGIN + IF NOT EXISTS ( + SELECT 1 + FROM pg_constraint + WHERE conname = 'fk_ksp_raw_transaction_archive_payloads_transaction' + AND conrelid = to_regclass('ksp_raw_transaction_archive_payloads') + ) THEN + ALTER TABLE ksp_raw_transaction_archive_payloads ADD CONSTRAINT fk_ksp_raw_transaction_archive_payloads_transaction FOREIGN KEY (signature) REFERENCES ksp_raw_transactions(signature) ON DELETE RESTRICT; + END IF; +END +$ksp$; diff --git a/crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/034_ck_ksp_raw_transaction_archive_payloads_signature.sql b/crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/034_ck_ksp_raw_transaction_archive_payloads_signature.sql new file mode 100644 index 0000000..15a86a5 --- /dev/null +++ b/crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/034_ck_ksp_raw_transaction_archive_payloads_signature.sql @@ -0,0 +1,12 @@ +DO $ksp$ +BEGIN + IF NOT EXISTS ( + SELECT 1 + FROM pg_constraint + WHERE conname = 'ck_ksp_raw_transaction_archive_payloads_signature' + AND conrelid = to_regclass('ksp_raw_transaction_archive_payloads') + ) THEN + ALTER TABLE ksp_raw_transaction_archive_payloads ADD CONSTRAINT ck_ksp_raw_transaction_archive_payloads_signature CHECK (octet_length(signature) = 64); + END IF; +END +$ksp$; diff --git a/crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/035_ck_ksp_raw_transaction_archive_payloads_payload.sql b/crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/035_ck_ksp_raw_transaction_archive_payloads_payload.sql new file mode 100644 index 0000000..4fcea6b --- /dev/null +++ b/crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/035_ck_ksp_raw_transaction_archive_payloads_payload.sql @@ -0,0 +1,12 @@ +DO $ksp$ +BEGIN + IF NOT EXISTS ( + SELECT 1 + FROM pg_constraint + WHERE conname = 'ck_ksp_raw_transaction_archive_payloads_payload' + AND conrelid = to_regclass('ksp_raw_transaction_archive_payloads') + ) THEN + ALTER TABLE ksp_raw_transaction_archive_payloads ADD CONSTRAINT ck_ksp_raw_transaction_archive_payloads_payload CHECK (octet_length(payload) >= 1 AND octet_length(payload) <= 16777216); + END IF; +END +$ksp$; diff --git a/crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/indexes/001_ix_ksp_raw_transactions_slot_signature.sql b/crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/indexes/001_ix_ksp_raw_transactions_slot_signature.sql new file mode 100644 index 0000000..9b8d02c --- /dev/null +++ b/crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/indexes/001_ix_ksp_raw_transactions_slot_signature.sql @@ -0,0 +1,3 @@ +CREATE INDEX IF NOT EXISTS ix_ksp_raw_transactions_slot_signature + ON ksp_raw_transactions (slot, signature) + WHERE retention_state <> 'purged'; diff --git a/crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/tables/001_ksp_store_identity.sql b/crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/tables/001_ksp_store_identity.sql new file mode 100644 index 0000000..2cd8e3a --- /dev/null +++ b/crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/tables/001_ksp_store_identity.sql @@ -0,0 +1,7 @@ +CREATE TABLE IF NOT EXISTS ksp_store_identity ( + singleton SMALLINT NOT NULL, + network TEXT NOT NULL +); + +ALTER TABLE ksp_store_identity ADD COLUMN IF NOT EXISTS singleton SMALLINT NOT NULL; +ALTER TABLE ksp_store_identity ADD COLUMN IF NOT EXISTS network TEXT NOT NULL; diff --git a/crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/tables/002_ksp_raw_transactions.sql b/crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/tables/002_ksp_raw_transactions.sql new file mode 100644 index 0000000..c3e05b6 --- /dev/null +++ b/crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/tables/002_ksp_raw_transactions.sql @@ -0,0 +1,19 @@ +CREATE TABLE IF NOT EXISTS ksp_raw_transactions ( + signature BYTEA NOT NULL, + slot NUMERIC(20, 0) NOT NULL, + block_time_unix_millis BIGINT NULL, + format_id TEXT NOT NULL, + format_version BIGINT NOT NULL, + content_hash BYTEA NOT NULL, + payload BYTEA NULL, + retention_state TEXT NOT NULL +); + +ALTER TABLE ksp_raw_transactions ADD COLUMN IF NOT EXISTS signature BYTEA NOT NULL; +ALTER TABLE ksp_raw_transactions ADD COLUMN IF NOT EXISTS slot NUMERIC(20, 0) NOT NULL; +ALTER TABLE ksp_raw_transactions ADD COLUMN IF NOT EXISTS block_time_unix_millis BIGINT NULL; +ALTER TABLE ksp_raw_transactions ADD COLUMN IF NOT EXISTS format_id TEXT NOT NULL; +ALTER TABLE ksp_raw_transactions ADD COLUMN IF NOT EXISTS format_version BIGINT NOT NULL; +ALTER TABLE ksp_raw_transactions ADD COLUMN IF NOT EXISTS content_hash BYTEA NOT NULL; +ALTER TABLE ksp_raw_transactions ADD COLUMN IF NOT EXISTS payload BYTEA NULL; +ALTER TABLE ksp_raw_transactions ADD COLUMN IF NOT EXISTS retention_state TEXT NOT NULL; diff --git a/crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/tables/003_ksp_raw_transaction_observations.sql b/crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/tables/003_ksp_raw_transaction_observations.sql new file mode 100644 index 0000000..1915947 --- /dev/null +++ b/crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/tables/003_ksp_raw_transaction_observations.sql @@ -0,0 +1,31 @@ +CREATE TABLE IF NOT EXISTS ksp_raw_transaction_observations ( + observation_key BYTEA NOT NULL, + transaction_signature BYTEA NOT NULL, + provider TEXT NOT NULL, + protocol TEXT NOT NULL, + acquisition_method TEXT NOT NULL, + origin TEXT NOT NULL, + received_at_unix_millis BIGINT NOT NULL, + capture_session_id TEXT NULL, + commitment TEXT NULL, + endpoint_id TEXT NULL, + filter_id TEXT NULL, + observed_at_unix_millis BIGINT NULL, + source_payload_hash BYTEA NULL, + source_payload_size_bytes BIGINT NULL +); + +ALTER TABLE ksp_raw_transaction_observations ADD COLUMN IF NOT EXISTS observation_key BYTEA NOT NULL; +ALTER TABLE ksp_raw_transaction_observations ADD COLUMN IF NOT EXISTS transaction_signature BYTEA NOT NULL; +ALTER TABLE ksp_raw_transaction_observations ADD COLUMN IF NOT EXISTS provider TEXT NOT NULL; +ALTER TABLE ksp_raw_transaction_observations ADD COLUMN IF NOT EXISTS protocol TEXT NOT NULL; +ALTER TABLE ksp_raw_transaction_observations ADD COLUMN IF NOT EXISTS acquisition_method TEXT NOT NULL; +ALTER TABLE ksp_raw_transaction_observations ADD COLUMN IF NOT EXISTS origin TEXT NOT NULL; +ALTER TABLE ksp_raw_transaction_observations ADD COLUMN IF NOT EXISTS received_at_unix_millis BIGINT NOT NULL; +ALTER TABLE ksp_raw_transaction_observations ADD COLUMN IF NOT EXISTS capture_session_id TEXT NULL; +ALTER TABLE ksp_raw_transaction_observations ADD COLUMN IF NOT EXISTS commitment TEXT NULL; +ALTER TABLE ksp_raw_transaction_observations ADD COLUMN IF NOT EXISTS endpoint_id TEXT NULL; +ALTER TABLE ksp_raw_transaction_observations ADD COLUMN IF NOT EXISTS filter_id TEXT NULL; +ALTER TABLE ksp_raw_transaction_observations ADD COLUMN IF NOT EXISTS observed_at_unix_millis BIGINT NULL; +ALTER TABLE ksp_raw_transaction_observations ADD COLUMN IF NOT EXISTS source_payload_hash BYTEA NULL; +ALTER TABLE ksp_raw_transaction_observations ADD COLUMN IF NOT EXISTS source_payload_size_bytes BIGINT NULL; diff --git a/crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/tables/004_ksp_raw_transaction_archive_payloads.sql b/crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/tables/004_ksp_raw_transaction_archive_payloads.sql new file mode 100644 index 0000000..50642c4 --- /dev/null +++ b/crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/tables/004_ksp_raw_transaction_archive_payloads.sql @@ -0,0 +1,7 @@ +CREATE TABLE IF NOT EXISTS ksp_raw_transaction_archive_payloads ( + signature BYTEA NOT NULL, + payload BYTEA NOT NULL +); + +ALTER TABLE ksp_raw_transaction_archive_payloads ADD COLUMN IF NOT EXISTS signature BYTEA NOT NULL; +ALTER TABLE ksp_raw_transaction_archive_payloads ADD COLUMN IF NOT EXISTS payload BYTEA NOT NULL; diff --git a/crates/ksp-store-postgres-lib/src/lib.rs b/crates/ksp-store-postgres-lib/src/lib.rs index c7a4dda..0b2d71b 100644 --- a/crates/ksp-store-postgres-lib/src/lib.rs +++ b/crates/ksp-store-postgres-lib/src/lib.rs @@ -1,5 +1,5 @@ // file: crates/ksp-store-postgres-lib/src/lib.rs -// version: 6 +// version: 8 #![warn(missing_docs)] #![deny(unreachable_pub)] @@ -9,9 +9,10 @@ //! //! The backend owns the physical `tokio-postgres` connection, bounded Deadpool //! pool, explicit Rustls TLS policy, private KSP migration/bootstrap engine and -//! safe lightweight health/readiness probe. `0.3.3-pre.003` adds the immutable -//! V001 RawTransaction physical schema and mono-network database binding; the -//! business capability implementations remain deferred to later prereleases. +//! safe lightweight health/readiness probe. `0.3.3-pre.003-fix.001` splits +//! migrations into versioned physical resources and verifies the effective +//! PostgreSQL schema contract before readiness; business capability +//! implementations remain deferred to later prereleases. //! //! This crate depends on `ksp-store-api` and never on `ksp-store-lib`. The //! common facade consumes only this crate's narrow backend bridge and never @@ -22,6 +23,7 @@ mod error; mod health; mod migration; mod runtime; +mod schema; /// Stable KSP error code for unsupported PostgreSQL retention compaction. pub use self::error::ERROR_CODE_POSTGRES_RETENTION_COMPACTION_UNSUPPORTED; @@ -52,5 +54,19 @@ pub(crate) use self::migration::current_migration_version; pub(crate) use self::runtime::map_pool_error; /// Private Deadpool status projector shared with the health probe. pub(crate) use self::runtime::runtime_snapshot_from_status; +/// Private physical schema resource descriptor consumed by the migration engine. +pub(crate) use self::schema::SchemaResource; +/// Private physical schema resource compatibility state consumed by the migration engine. +pub(crate) use self::schema::SchemaResourceState; +/// Private V000 schema resource inventory consumed by the migration engine. +pub(crate) use self::schema::V000_RESOURCES; +/// Private V001 schema resource inventory consumed by the migration engine. +pub(crate) use self::schema::V001_RESOURCES; +/// Private physical schema resource inspector consumed by the migration engine. +pub(crate) use self::schema::inspect_resource; +/// Private V001 adoption probe consumed by the migration engine. +pub(crate) use self::schema::managed_v001_objects_exist; +/// Private external-schema compatibility gate consumed by the migration engine. +pub(crate) use self::schema::verify_v001_external_compatibility; const _: &str = crate::TRACING_TARGET; diff --git a/crates/ksp-store-postgres-lib/src/migration.rs b/crates/ksp-store-postgres-lib/src/migration.rs index 79a3dea..4bc6a86 100644 --- a/crates/ksp-store-postgres-lib/src/migration.rs +++ b/crates/ksp-store-postgres-lib/src/migration.rs @@ -1,15 +1,22 @@ // file: crates/ksp-store-postgres-lib/src/migration.rs -// version: 4 +// version: 5 use sha2::Digest; // rust-rules: trait-import const ADVISORY_LOCK_KEY: i64 = 0x4b53_5053_544f_5245; const EMBEDDED_MIGRATIONS: &[EmbeddedMigration] = &[ - EmbeddedMigration { hook: MigrationHook::None, name: "bootstrap", sql: include_str!("../migrations/V000__bootstrap.sql"), version: 0 }, EmbeddedMigration { + checksum: MigrationChecksum::LegacySql(include_str!("../migrations/v000_bootstrap/tables/001_ksp_store_schema_migrations.sql")), + hook: MigrationHook::None, + name: "bootstrap", + resources: crate::V000_RESOURCES, + version: 0, + }, + EmbeddedMigration { + checksum: MigrationChecksum::Resources, hook: MigrationHook::StoreIdentity, name: "raw_transaction", - sql: include_str!("../migrations/V001__raw_transaction.sql"), + resources: crate::V001_RESOURCES, version: 1, }, ]; @@ -25,21 +32,6 @@ const METADATA_EXISTS_SQL: &str = r#"SELECT EXISTS ( AND table_name = 'ksp_store_schema_migrations' AND table_type = 'BASE TABLE' )"#; -const METADATA_PRIMARY_KEY_SQL: &str = r#"SELECT COUNT(*)::BIGINT, - COUNT(*) FILTER (WHERE kcu.column_name = 'version')::BIGINT -FROM information_schema.table_constraints tc -JOIN information_schema.key_column_usage kcu - ON tc.constraint_catalog = kcu.constraint_catalog - AND tc.constraint_schema = kcu.constraint_schema - AND tc.constraint_name = kcu.constraint_name -WHERE tc.table_schema = current_schema() - AND tc.table_name = 'ksp_store_schema_migrations' - AND tc.constraint_type = 'PRIMARY KEY'"#; -const METADATA_SHAPE_SQL: &str = r#"SELECT column_name, data_type, is_nullable -FROM information_schema.columns -WHERE table_schema = current_schema() - AND table_name = 'ksp_store_schema_migrations' -ORDER BY ordinal_position"#; const SET_STATEMENT_TIMEOUT_SQL: &str = "SELECT set_config('statement_timeout', $1, true)"; struct AppliedMigration { @@ -48,11 +40,18 @@ struct AppliedMigration { version: i64, } +#[derive(Clone, Copy)] +enum MigrationChecksum { + LegacySql(&'static str), + Resources, +} + #[derive(Clone, Copy)] struct EmbeddedMigration { + checksum: MigrationChecksum, hook: MigrationHook, name: &'static str, - sql: &'static str, + resources: &'static [crate::SchemaResource], version: i64, } @@ -68,6 +67,12 @@ enum MigrationHookContext { Existing, } +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum SchemaMutationMode { + Create, + Update, +} + /// Returns the latest migration version embedded by this backend runtime. #[must_use] pub(crate) const fn current_migration_version() -> i64 { @@ -78,7 +83,8 @@ pub(crate) const fn current_migration_version() -> i64 { pub(crate) async fn bootstrap( client: &mut deadpool_postgres::Client, network: &ksp_store_api::RawNetworkId, - auto_migrate: bool, + schema_autocreate: bool, + schema_autoupdate: bool, migration_timeout: std::time::Duration, migration_lock_timeout: std::time::Duration, ) -> std::result::Result<(), crate::PostgresBackendError> { @@ -88,7 +94,7 @@ pub(crate) async fn bootstrap( } let bounded = tokio::time::timeout( migration_timeout, - bootstrap_inner(client, network, auto_migrate, migration_timeout, migration_lock_timeout), + bootstrap_inner(client, network, schema_autocreate, schema_autoupdate, migration_timeout, migration_lock_timeout), ) .await; return match bounded { @@ -102,7 +108,8 @@ pub(crate) async fn bootstrap( async fn bootstrap_inner( client: &mut deadpool_postgres::Client, network: &ksp_store_api::RawNetworkId, - auto_migrate: bool, + schema_autocreate: bool, + schema_autoupdate: bool, migration_timeout: std::time::Duration, migration_lock_timeout: std::time::Duration, ) -> std::result::Result<(), crate::PostgresBackendError> { @@ -126,10 +133,15 @@ async fn bootstrap_inner( std::result::Result::Ok(value) => value, std::result::Result::Err(error) => return std::result::Result::Err(error), }; - let next_index = if metadata_exists { - let shape_result = verify_metadata_shape(&transaction).await; - if let std::result::Result::Err(error) = shape_result { - return std::result::Result::Err(error); + let (next_index, mutation_mode) = if metadata_exists { + let metadata_result = crate::inspect_resource(&transaction, &crate::V000_RESOURCES[0]).await; + match metadata_result { + std::result::Result::Ok(crate::SchemaResourceState::Compatible) => {}, + std::result::Result::Ok(crate::SchemaResourceState::Missing | crate::SchemaResourceState::Incompatible) => { + log_schema_block("metadata_incompatible"); + return std::result::Result::Err(crate::PostgresBackendError::new(crate::PostgresBackendErrorKind::MigrationMismatch, "metadata_incompatible")); + }, + std::result::Result::Err(error) => return std::result::Result::Err(error), } let history_result = load_history(&transaction).await; let history = match history_result { @@ -137,21 +149,40 @@ async fn bootstrap_inner( std::result::Result::Err(error) => return std::result::Result::Err(error), }; let validation_result = validate_history(history.as_slice(), EMBEDDED_MIGRATIONS); - match validation_result { + let index = match validation_result { std::result::Result::Ok(value) => value, std::result::Result::Err(error) => return std::result::Result::Err(error), - } + }; + (index, SchemaMutationMode::Update) } else { - 0 + if !schema_autocreate { + log_schema_block("schema_autocreate_disabled"); + return std::result::Result::Err(crate::PostgresBackendError::new(crate::PostgresBackendErrorKind::MigrationFailed, "schema_autocreate_disabled")); + } + let managed_result = crate::managed_v001_objects_exist(&transaction).await; + let managed_objects_exist = match managed_result { + std::result::Result::Ok(value) => value, + std::result::Result::Err(error) => return std::result::Result::Err(error), + }; + if managed_objects_exist && !schema_autoupdate { + log_schema_block("schema_adoption_disabled"); + return std::result::Result::Err(crate::PostgresBackendError::new(crate::PostgresBackendErrorKind::MigrationFailed, "schema_adoption_disabled")); + } + (0, SchemaMutationMode::Create) }; + let existing_schema_result = verify_or_repair_applied_migrations(&transaction, next_index, schema_autoupdate).await; + if let std::result::Result::Err(error) = existing_schema_result { + return std::result::Result::Err(error); + } let existing_hook_result = run_applied_migration_hooks(&transaction, network, next_index).await; if let std::result::Result::Err(error) = existing_hook_result { return std::result::Result::Err(error); } - if next_index < EMBEDDED_MIGRATIONS.len() && !auto_migrate { + if next_index < EMBEDDED_MIGRATIONS.len() && mutation_mode == SchemaMutationMode::Update && !schema_autoupdate { + log_schema_block("migration_pending"); return std::result::Result::Err(crate::PostgresBackendError::new(crate::PostgresBackendErrorKind::MigrationFailed, "migration_pending")); } - let apply_result = apply_pending_migrations(&transaction, network, next_index).await; + let apply_result = apply_pending_migrations(&transaction, network, next_index, mutation_mode).await; if let std::result::Result::Err(error) = apply_result { return std::result::Result::Err(error); } @@ -207,20 +238,23 @@ async fn apply_migration( transaction: &deadpool_postgres::Transaction<'_>, network: &ksp_store_api::RawNetworkId, migration: &EmbeddedMigration, + mutation_mode: SchemaMutationMode, ) -> std::result::Result<(), crate::PostgresBackendError> { - let execute_result = transaction.batch_execute(migration.sql).await; - if execute_result.is_err() { - return std::result::Result::Err(crate::PostgresBackendError::new(crate::PostgresBackendErrorKind::MigrationFailed, "migration_apply")); + for resource in migration.resources { + let result = ensure_resource(transaction, resource, mutation_mode, false).await; + if let std::result::Result::Err(error) = result { + return std::result::Result::Err(error); + } } - let shape_result = verify_metadata_shape(transaction).await; - if let std::result::Result::Err(error) = shape_result { + let contract_result = verify_migration_contract(transaction, migration.version).await; + if let std::result::Result::Err(error) = contract_result { return std::result::Result::Err(error); } let hook_result = run_migration_hook(transaction, network, migration.hook, MigrationHookContext::AppliedNow).await; if let std::result::Result::Err(error) = hook_result { return std::result::Result::Err(error); } - let checksum = migration_checksum(migration.sql); + let checksum = migration_checksum(migration); let insert_result = transaction.execute(HISTORY_INSERT_SQL, &[&migration.version, &migration.name, &checksum]).await; return match insert_result { std::result::Result::Ok(1) => std::result::Result::Ok(()), @@ -234,11 +268,12 @@ async fn apply_pending_migrations( transaction: &deadpool_postgres::Transaction<'_>, network: &ksp_store_api::RawNetworkId, next_index: usize, + mutation_mode: SchemaMutationMode, ) -> std::result::Result<(), crate::PostgresBackendError> { let mut index = next_index; while index < EMBEDDED_MIGRATIONS.len() { let migration = &EMBEDDED_MIGRATIONS[index]; - let result = apply_migration(transaction, network, migration).await; + let result = apply_migration(transaction, network, migration, mutation_mode).await; if let std::result::Result::Err(error) = result { return std::result::Result::Err(error); } @@ -247,6 +282,59 @@ async fn apply_pending_migrations( return std::result::Result::Ok(()); } +async fn ensure_resource( + transaction: &deadpool_postgres::Transaction<'_>, + resource: &crate::SchemaResource, + mutation_mode: SchemaMutationMode, + applied_history: bool, +) -> std::result::Result<(), crate::PostgresBackendError> { + let state_result = crate::inspect_resource(transaction, resource).await; + let state = match state_result { + std::result::Result::Ok(value) => value, + std::result::Result::Err(error) => return std::result::Result::Err(error), + }; + match state { + crate::SchemaResourceState::Compatible => return std::result::Result::Ok(()), + crate::SchemaResourceState::Incompatible => { + log_schema_resource_block(resource.id, "incompatible"); + return std::result::Result::Err(crate::PostgresBackendError::new( + crate::PostgresBackendErrorKind::MigrationMismatch, + "schema_resource_incompatible", + )); + }, + crate::SchemaResourceState::Missing => {}, + } + if applied_history && !resource.repair_existing { + log_schema_resource_block(resource.id, "repair_forbidden"); + return std::result::Result::Err(crate::PostgresBackendError::new(crate::PostgresBackendErrorKind::MigrationMismatch, "schema_resource_missing")); + } + if applied_history && mutation_mode != SchemaMutationMode::Update { + log_schema_resource_block(resource.id, "repair_mode_invalid"); + return std::result::Result::Err(crate::PostgresBackendError::new(crate::PostgresBackendErrorKind::MigrationMismatch, "schema_resource_missing")); + } + if applied_history { + ksp_logging_lib::warn!( + target: crate::TRACING_TARGET, + resource_id = resource.id, + "repairing missing PostgreSQL Store schema resource under schema_autoupdate policy" + ); + } + let execute_result = transaction.batch_execute(resource.sql).await; + if execute_result.is_err() { + log_schema_resource_block(resource.id, "apply_failed"); + return std::result::Result::Err(crate::PostgresBackendError::new(crate::PostgresBackendErrorKind::MigrationFailed, "schema_resource_apply")); + } + let verified_result = crate::inspect_resource(transaction, resource).await; + return match verified_result { + std::result::Result::Ok(crate::SchemaResourceState::Compatible) => std::result::Result::Ok(()), + std::result::Result::Ok(crate::SchemaResourceState::Missing | crate::SchemaResourceState::Incompatible) => { + log_schema_resource_block(resource.id, "post_apply_incompatible"); + std::result::Result::Err(crate::PostgresBackendError::new(crate::PostgresBackendErrorKind::MigrationMismatch, "schema_resource_post_apply")) + }, + std::result::Result::Err(error) => std::result::Result::Err(error), + }; +} + async fn load_history(transaction: &deadpool_postgres::Transaction<'_>) -> std::result::Result, crate::PostgresBackendError> { let result = transaction.query(HISTORY_LOAD_SQL, &[]).await; let rows = match result { @@ -288,11 +376,30 @@ async fn metadata_exists(transaction: &deadpool_postgres::Transaction<'_>) -> st }; } -fn migration_checksum(sql: &str) -> std::string::String { +fn migration_checksum(migration: &EmbeddedMigration) -> std::string::String { + return match migration.checksum { + MigrationChecksum::LegacySql(sql) => checksum_bytes(sql.as_bytes()), + MigrationChecksum::Resources => { + let mut hasher = sha2::Sha256::new(); + hasher.update(b"ksp-migration-resources-v1\0"); + for resource in migration.resources { + hasher.update(resource.id.as_bytes()); + hasher.update([0]); + hasher.update(resource.sql.as_bytes()); + hasher.update([0]); + } + return encode_digest(hasher.finalize().as_slice()); + }, + }; +} + +fn checksum_bytes(bytes: &[u8]) -> std::string::String { let mut hasher = sha2::Sha256::new(); - hasher.update(sql.as_bytes()); - let digest = hasher.finalize(); - let bytes = digest.as_slice(); + hasher.update(bytes); + return encode_digest(hasher.finalize().as_slice()); +} + +fn encode_digest(bytes: &[u8]) -> std::string::String { let mut encoded = std::string::String::with_capacity(bytes.len() * 2); for byte in bytes { let value = *byte; @@ -336,33 +443,44 @@ async fn bind_store_identity( network: &ksp_store_api::RawNetworkId, context: MigrationHookContext, ) -> std::result::Result<(), crate::PostgresBackendError> { - if context == MigrationHookContext::AppliedNow { + let first_read = load_store_identity(transaction).await; + let first = match first_read { + std::result::Result::Ok(value) => value, + std::result::Result::Err(error) => return std::result::Result::Err(error), + }; + if first.is_empty() && context == MigrationHookContext::AppliedNow { let insert_result = transaction.execute(IDENTITY_INSERT_SQL, &[&network.as_str()]).await; match insert_result { std::result::Result::Ok(1) => {}, std::result::Result::Ok(_) | std::result::Result::Err(_) => { - return std::result::Result::Err(crate::PostgresBackendError::new( - crate::PostgresBackendErrorKind::MigrationFailed, - "store_identity_insert", - )); + return std::result::Result::Err(crate::PostgresBackendError::new(crate::PostgresBackendErrorKind::MigrationFailed, "store_identity_insert")); }, } + let second_read = load_store_identity(transaction).await; + let second = match second_read { + std::result::Result::Ok(value) => value, + std::result::Result::Err(error) => return std::result::Result::Err(error), + }; + return validate_store_identity(second.as_slice(), network); } + return validate_store_identity(first.as_slice(), network); +} + +async fn load_store_identity( + transaction: &deadpool_postgres::Transaction<'_>, +) -> std::result::Result, crate::PostgresBackendError> { let rows_result = transaction.query(IDENTITY_LOAD_SQL, &[]).await; - let rows = match rows_result { - std::result::Result::Ok(value) => value, + return match rows_result { + std::result::Result::Ok(value) => std::result::Result::Ok(value), std::result::Result::Err(_) => { - return std::result::Result::Err(crate::PostgresBackendError::new( - crate::PostgresBackendErrorKind::MigrationMismatch, - "store_identity_read", - )); + std::result::Result::Err(crate::PostgresBackendError::new(crate::PostgresBackendErrorKind::MigrationMismatch, "store_identity_read")) }, }; +} + +fn validate_store_identity(rows: &[tokio_postgres::Row], network: &ksp_store_api::RawNetworkId) -> std::result::Result<(), crate::PostgresBackendError> { if rows.len() != 1 { - return std::result::Result::Err(crate::PostgresBackendError::new( - crate::PostgresBackendErrorKind::MigrationMismatch, - "store_identity_count", - )); + return std::result::Result::Err(crate::PostgresBackendError::new(crate::PostgresBackendErrorKind::MigrationMismatch, "store_identity_count")); } let row = &rows[0]; let singleton_result = row.try_get::(0); @@ -370,32 +488,20 @@ async fn bind_store_identity( let (singleton, stored_network) = match (singleton_result, network_result) { (std::result::Result::Ok(singleton), std::result::Result::Ok(stored_network)) => (singleton, stored_network), _ => { - return std::result::Result::Err(crate::PostgresBackendError::new( - crate::PostgresBackendErrorKind::MigrationMismatch, - "store_identity_decode", - )); + return std::result::Result::Err(crate::PostgresBackendError::new(crate::PostgresBackendErrorKind::MigrationMismatch, "store_identity_decode")); }, }; if singleton != 1 { - return std::result::Result::Err(crate::PostgresBackendError::new( - crate::PostgresBackendErrorKind::MigrationMismatch, - "store_identity_singleton", - )); + return std::result::Result::Err(crate::PostgresBackendError::new(crate::PostgresBackendErrorKind::MigrationMismatch, "store_identity_singleton")); } let stored_network = match ksp_store_api::RawNetworkId::new(stored_network) { std::result::Result::Ok(value) => value, std::result::Result::Err(_) => { - return std::result::Result::Err(crate::PostgresBackendError::new( - crate::PostgresBackendErrorKind::MigrationMismatch, - "store_identity_network", - )); + return std::result::Result::Err(crate::PostgresBackendError::new(crate::PostgresBackendErrorKind::MigrationMismatch, "store_identity_network")); }, }; if stored_network.as_str() != network.as_str() { - return std::result::Result::Err(crate::PostgresBackendError::new( - crate::PostgresBackendErrorKind::MigrationMismatch, - "store_identity_network", - )); + return std::result::Result::Err(crate::PostgresBackendError::new(crate::PostgresBackendErrorKind::MigrationMismatch, "store_identity_network")); } return std::result::Result::Ok(()); } @@ -414,15 +520,83 @@ async fn set_statement_timeout( }; } +async fn verify_migration_contract(transaction: &deadpool_postgres::Transaction<'_>, version: i64) -> std::result::Result<(), crate::PostgresBackendError> { + if version == 1 { + let result = crate::verify_v001_external_compatibility(transaction).await; + if let std::result::Result::Err(error) = result { + log_schema_block(error.phase()); + return std::result::Result::Err(error); + } + } + return std::result::Result::Ok(()); +} + +async fn verify_or_repair_applied_migrations( + transaction: &deadpool_postgres::Transaction<'_>, + applied_count: usize, + schema_autoupdate: bool, +) -> std::result::Result<(), crate::PostgresBackendError> { + let mut index = 0_usize; + while index < applied_count { + let migration = &EMBEDDED_MIGRATIONS[index]; + for resource in migration.resources { + let state_result = crate::inspect_resource(transaction, resource).await; + let state = match state_result { + std::result::Result::Ok(value) => value, + std::result::Result::Err(error) => return std::result::Result::Err(error), + }; + match state { + crate::SchemaResourceState::Compatible => {}, + crate::SchemaResourceState::Incompatible => { + log_schema_resource_block(resource.id, "incompatible"); + return std::result::Result::Err(crate::PostgresBackendError::new( + crate::PostgresBackendErrorKind::MigrationMismatch, + "schema_resource_incompatible", + )); + }, + crate::SchemaResourceState::Missing => { + if !schema_autoupdate { + log_schema_resource_block(resource.id, "schema_autoupdate_disabled"); + return std::result::Result::Err(crate::PostgresBackendError::new( + crate::PostgresBackendErrorKind::MigrationFailed, + "schema_autoupdate_disabled", + )); + } + let repair_result = ensure_resource(transaction, resource, SchemaMutationMode::Update, true).await; + if let std::result::Result::Err(error) = repair_result { + return std::result::Result::Err(error); + } + }, + } + } + let contract_result = verify_migration_contract(transaction, migration.version).await; + if let std::result::Result::Err(error) = contract_result { + return std::result::Result::Err(error); + } + index += 1; + } + return std::result::Result::Ok(()); +} + fn validate_embedded_registry(migrations: &[EmbeddedMigration]) -> std::result::Result<(), crate::PostgresBackendError> { if migrations.is_empty() { return std::result::Result::Err(crate::PostgresBackendError::new(crate::PostgresBackendErrorKind::MigrationMismatch, "registry_empty")); } let mut expected_version = 0_i64; for migration in migrations { - if migration.version != expected_version || migration.name.is_empty() || migration.sql.is_empty() { + if migration.version != expected_version || migration.name.is_empty() || migration.resources.is_empty() { return std::result::Result::Err(crate::PostgresBackendError::new(crate::PostgresBackendErrorKind::MigrationMismatch, "registry_invalid")); } + for (resource_index, resource) in migration.resources.iter().enumerate() { + if resource.id.is_empty() || resource.sql.is_empty() { + return std::result::Result::Err(crate::PostgresBackendError::new(crate::PostgresBackendErrorKind::MigrationMismatch, "registry_invalid")); + } + for previous in &migration.resources[..resource_index] { + if previous.id == resource.id { + return std::result::Result::Err(crate::PostgresBackendError::new(crate::PostgresBackendErrorKind::MigrationMismatch, "registry_invalid")); + } + } + } expected_version = match expected_version.checked_add(1) { std::option::Option::Some(value) => value, std::option::Option::None => { @@ -447,7 +621,7 @@ fn validate_history(history: &[AppliedMigration], migrations: &[EmbeddedMigratio return std::result::Result::Err(crate::PostgresBackendError::new(crate::PostgresBackendErrorKind::SchemaNewer, "history_newer")); } let expected = &migrations[index]; - let expected_checksum = migration_checksum(expected.sql); + let expected_checksum = migration_checksum(expected); if applied.version != expected.version || applied.name != expected.name || applied.checksum != expected_checksum { return std::result::Result::Err(crate::PostgresBackendError::new(crate::PostgresBackendErrorKind::MigrationMismatch, "history_diverged")); } @@ -456,59 +630,17 @@ fn validate_history(history: &[AppliedMigration], migrations: &[EmbeddedMigratio return std::result::Result::Ok(index); } -async fn verify_metadata_shape(transaction: &deadpool_postgres::Transaction<'_>) -> std::result::Result<(), crate::PostgresBackendError> { - let result = transaction.query(METADATA_SHAPE_SQL, &[]).await; - let rows = match result { - std::result::Result::Ok(value) => value, - std::result::Result::Err(_) => { - return std::result::Result::Err(crate::PostgresBackendError::new(crate::PostgresBackendErrorKind::MigrationFailed, "metadata_shape")); - }, - }; - const REQUIRED: [(&str, &str, &str); 4] = [ - ("version", "bigint", "NO"), - ("name", "text", "NO"), - ("checksum", "text", "NO"), - ("applied_at", "timestamp with time zone", "NO"), - ]; - let mut found = [false; REQUIRED.len()]; - for row in rows { - let name_result = row.try_get::(0); - let data_type_result = row.try_get::(1); - let nullable_result = row.try_get::(2); - let (name, data_type, nullable) = match (name_result, data_type_result, nullable_result) { - (std::result::Result::Ok(name), std::result::Result::Ok(data_type), std::result::Result::Ok(nullable)) => (name, data_type, nullable), - _ => { - return std::result::Result::Err(crate::PostgresBackendError::new(crate::PostgresBackendErrorKind::MigrationFailed, "metadata_shape_decode")); - }, - }; - for (index, required) in REQUIRED.iter().enumerate() { - if name == required.0 && data_type == required.1 && nullable == required.2 { - found[index] = true; - break; - } - } - } - for required in found { - if !required { - return std::result::Result::Err(crate::PostgresBackendError::new(crate::PostgresBackendErrorKind::MigrationMismatch, "metadata_shape")); - } - } - let key_result = transaction.query_one(METADATA_PRIMARY_KEY_SQL, &[]).await; - let key_row = match key_result { - std::result::Result::Ok(value) => value, - std::result::Result::Err(_) => { - return std::result::Result::Err(crate::PostgresBackendError::new(crate::PostgresBackendErrorKind::MigrationFailed, "metadata_primary_key")); - }, - }; - let key_count = key_row.try_get::(0); - let version_count = key_row.try_get::(1); - return match (key_count, version_count) { - (std::result::Result::Ok(1), std::result::Result::Ok(1)) => std::result::Result::Ok(()), - (std::result::Result::Ok(_), std::result::Result::Ok(_)) => { - std::result::Result::Err(crate::PostgresBackendError::new(crate::PostgresBackendErrorKind::MigrationMismatch, "metadata_primary_key")) - }, - _ => std::result::Result::Err(crate::PostgresBackendError::new(crate::PostgresBackendErrorKind::MigrationFailed, "metadata_primary_key_decode")), - }; +fn log_schema_block(phase: &'static str) { + ksp_logging_lib::warn!(target: crate::TRACING_TARGET, phase, "PostgreSQL Store schema compatibility gate blocked automatic opening"); +} + +fn log_schema_resource_block(resource_id: &'static str, reason: &'static str) { + ksp_logging_lib::warn!( + target: crate::TRACING_TARGET, + resource_id, + reason, + "PostgreSQL Store schema resource requires manual reconciliation" + ); } #[cfg(test)] diff --git a/crates/ksp-store-postgres-lib/src/runtime.rs b/crates/ksp-store-postgres-lib/src/runtime.rs index 5af073c..4048a13 100644 --- a/crates/ksp-store-postgres-lib/src/runtime.rs +++ b/crates/ksp-store-postgres-lib/src/runtime.rs @@ -1,5 +1,5 @@ // file: crates/ksp-store-postgres-lib/src/runtime.rs -// version: 4 +// version: 5 const APPLICATION_NAME: &str = "ksp-store"; const MAX_CONNECTION_URI_BYTES: usize = 4_096; @@ -143,18 +143,19 @@ pub struct PostgresBackendSettings { connect_timeout: std::time::Duration, connection_uri: std::string::String, create_timeout: std::time::Duration, - auto_migrate: bool, max_connections: u32, migration_lock_timeout: std::time::Duration, migration_timeout: std::time::Duration, network: ksp_store_api::RawNetworkId, recycle_timeout: std::time::Duration, + schema_autocreate: bool, + schema_autoupdate: bool, tls_mode: PostgresBackendTlsMode, wait_timeout: std::time::Duration, } impl PostgresBackendSettings { - /// Creates the physical PostgreSQL settings bridge from already validated facade-owned values. + /// Creates the physical PostgreSQL settings bridge using the legacy single migration switch. #[must_use] pub fn new( network: ksp_store_api::RawNetworkId, @@ -169,8 +170,39 @@ impl PostgresBackendSettings { migration_timeout: std::time::Duration, migration_lock_timeout: std::time::Duration, ) -> Self { - return Self { + return Self::with_schema_policy( + network, + connection_uri, + max_connections, + connect_timeout, + wait_timeout, + create_timeout, + recycle_timeout, + tls_mode, auto_migrate, + auto_migrate, + migration_timeout, + migration_lock_timeout, + ); + } + + /// Creates the physical PostgreSQL settings bridge with independent schema creation and update policies. + #[must_use] + pub fn with_schema_policy( + network: ksp_store_api::RawNetworkId, + connection_uri: impl std::convert::Into, + max_connections: u32, + connect_timeout: std::time::Duration, + wait_timeout: std::time::Duration, + create_timeout: std::time::Duration, + recycle_timeout: std::time::Duration, + tls_mode: PostgresBackendTlsMode, + schema_autocreate: bool, + schema_autoupdate: bool, + migration_timeout: std::time::Duration, + migration_lock_timeout: std::time::Duration, + ) -> Self { + return Self { connect_timeout, connection_uri: connection_uri.into(), create_timeout, @@ -179,6 +211,8 @@ impl PostgresBackendSettings { migration_timeout, network, recycle_timeout, + schema_autocreate, + schema_autoupdate, tls_mode, wait_timeout, }; @@ -203,7 +237,8 @@ impl std::fmt::Debug for PostgresBackendSettings { .debug_struct("PostgresBackendSettings") .field("network", &self.network) .field("connection_uri", &"") - .field("auto_migrate", &self.auto_migrate) + .field("schema_autocreate", &self.schema_autocreate) + .field("schema_autoupdate", &self.schema_autoupdate) .field("max_connections", &self.max_connections) .field("migration_timeout", &self.migration_timeout) .field("migration_lock_timeout", &self.migration_lock_timeout) @@ -263,8 +298,15 @@ impl PostgresBackend { tls_mode = settings.tls_mode().code(), "PostgreSQL Store backend established initial physical connection" ); - let bootstrap_result = - crate::bootstrap(&mut client, settings.network(), settings.auto_migrate, settings.migration_timeout, settings.migration_lock_timeout).await; + let bootstrap_result = crate::bootstrap( + &mut client, + settings.network(), + settings.schema_autocreate, + settings.schema_autoupdate, + settings.migration_timeout, + settings.migration_lock_timeout, + ) + .await; if let std::result::Result::Err(error) = bootstrap_result { return std::result::Result::Err(error); } @@ -272,7 +314,8 @@ impl PostgresBackend { ksp_logging_lib::debug!( target: crate::TRACING_TARGET, network = settings.network().as_str(), - auto_migrate = settings.auto_migrate, + schema_autocreate = settings.schema_autocreate, + schema_autoupdate = settings.schema_autoupdate, "PostgreSQL Store migration/bootstrap foundation verified" ); return std::result::Result::Ok(Self { network: settings.network, pool }); diff --git a/crates/ksp-store-postgres-lib/src/schema.rs b/crates/ksp-store-postgres-lib/src/schema.rs new file mode 100644 index 0000000..0c627d2 --- /dev/null +++ b/crates/ksp-store-postgres-lib/src/schema.rs @@ -0,0 +1,1146 @@ +// file: crates/ksp-store-postgres-lib/src/schema.rs +// version: 2 + +/// Immutable V000 physical schema resource inventory. +pub(crate) const V000_RESOURCES: &[SchemaResource] = &[SchemaResource { + id: "tables/001_ksp_store_schema_migrations.sql", + object: SchemaObjectContract::Table(TableContract { + columns: KSP_STORE_SCHEMA_MIGRATIONS_COLUMNS, + name: "ksp_store_schema_migrations", + primary_key_columns: std::option::Option::Some("version"), + }), + repair_existing: false, + sql: include_str!("../migrations/v000_bootstrap/tables/001_ksp_store_schema_migrations.sql"), +}]; +/// Immutable V001 physical schema resource inventory. +pub(crate) const V001_RESOURCES: &[SchemaResource] = &[ + SchemaResource { + id: "tables/001_ksp_store_identity.sql", + object: SchemaObjectContract::Table(TableContract { + columns: KSP_STORE_IDENTITY_COLUMNS, + name: "ksp_store_identity", + primary_key_columns: std::option::Option::None, + }), + repair_existing: false, + sql: include_str!("../migrations/v001_raw_transaction/tables/001_ksp_store_identity.sql"), + }, + SchemaResource { + id: "tables/002_ksp_raw_transactions.sql", + object: SchemaObjectContract::Table(TableContract { + columns: KSP_RAW_TRANSACTIONS_COLUMNS, + name: "ksp_raw_transactions", + primary_key_columns: std::option::Option::None, + }), + repair_existing: true, + sql: include_str!("../migrations/v001_raw_transaction/tables/002_ksp_raw_transactions.sql"), + }, + SchemaResource { + id: "tables/003_ksp_raw_transaction_observations.sql", + object: SchemaObjectContract::Table(TableContract { + columns: KSP_RAW_TRANSACTION_OBSERVATIONS_COLUMNS, + name: "ksp_raw_transaction_observations", + primary_key_columns: std::option::Option::None, + }), + repair_existing: true, + sql: include_str!("../migrations/v001_raw_transaction/tables/003_ksp_raw_transaction_observations.sql"), + }, + SchemaResource { + id: "tables/004_ksp_raw_transaction_archive_payloads.sql", + object: SchemaObjectContract::Table(TableContract { + columns: KSP_RAW_TRANSACTION_ARCHIVE_PAYLOADS_COLUMNS, + name: "ksp_raw_transaction_archive_payloads", + primary_key_columns: std::option::Option::None, + }), + repair_existing: true, + sql: include_str!("../migrations/v001_raw_transaction/tables/004_ksp_raw_transaction_archive_payloads.sql"), + }, + SchemaResource { + id: "constraints/001_pk_ksp_store_identity.sql", + object: SchemaObjectContract::Constraint(ConstraintContract { kind: "p", name: "pk_ksp_store_identity", table: "ksp_store_identity" }), + repair_existing: true, + sql: include_str!("../migrations/v001_raw_transaction/constraints/001_pk_ksp_store_identity.sql"), + }, + SchemaResource { + id: "constraints/002_ck_ksp_store_identity_singleton.sql", + object: SchemaObjectContract::Constraint(ConstraintContract { kind: "c", name: "ck_ksp_store_identity_singleton", table: "ksp_store_identity" }), + repair_existing: true, + sql: include_str!("../migrations/v001_raw_transaction/constraints/002_ck_ksp_store_identity_singleton.sql"), + }, + SchemaResource { + id: "constraints/003_ck_ksp_store_identity_network.sql", + object: SchemaObjectContract::Constraint(ConstraintContract { kind: "c", name: "ck_ksp_store_identity_network", table: "ksp_store_identity" }), + repair_existing: true, + sql: include_str!("../migrations/v001_raw_transaction/constraints/003_ck_ksp_store_identity_network.sql"), + }, + SchemaResource { + id: "constraints/004_pk_ksp_raw_transactions.sql", + object: SchemaObjectContract::Constraint(ConstraintContract { kind: "p", name: "pk_ksp_raw_transactions", table: "ksp_raw_transactions" }), + repair_existing: true, + sql: include_str!("../migrations/v001_raw_transaction/constraints/004_pk_ksp_raw_transactions.sql"), + }, + SchemaResource { + id: "constraints/005_ck_ksp_raw_transactions_signature.sql", + object: SchemaObjectContract::Constraint(ConstraintContract { kind: "c", name: "ck_ksp_raw_transactions_signature", table: "ksp_raw_transactions" }), + repair_existing: true, + sql: include_str!("../migrations/v001_raw_transaction/constraints/005_ck_ksp_raw_transactions_signature.sql"), + }, + SchemaResource { + id: "constraints/006_ck_ksp_raw_transactions_slot.sql", + object: SchemaObjectContract::Constraint(ConstraintContract { kind: "c", name: "ck_ksp_raw_transactions_slot", table: "ksp_raw_transactions" }), + repair_existing: true, + sql: include_str!("../migrations/v001_raw_transaction/constraints/006_ck_ksp_raw_transactions_slot.sql"), + }, + SchemaResource { + id: "constraints/007_ck_ksp_raw_transactions_block_time.sql", + object: SchemaObjectContract::Constraint(ConstraintContract { kind: "c", name: "ck_ksp_raw_transactions_block_time", table: "ksp_raw_transactions" }), + repair_existing: true, + sql: include_str!("../migrations/v001_raw_transaction/constraints/007_ck_ksp_raw_transactions_block_time.sql"), + }, + SchemaResource { + id: "constraints/008_ck_ksp_raw_transactions_format_id.sql", + object: SchemaObjectContract::Constraint(ConstraintContract { kind: "c", name: "ck_ksp_raw_transactions_format_id", table: "ksp_raw_transactions" }), + repair_existing: true, + sql: include_str!("../migrations/v001_raw_transaction/constraints/008_ck_ksp_raw_transactions_format_id.sql"), + }, + SchemaResource { + id: "constraints/009_ck_ksp_raw_transactions_format_version.sql", + object: SchemaObjectContract::Constraint(ConstraintContract { + kind: "c", + name: "ck_ksp_raw_transactions_format_version", + table: "ksp_raw_transactions", + }), + repair_existing: true, + sql: include_str!("../migrations/v001_raw_transaction/constraints/009_ck_ksp_raw_transactions_format_version.sql"), + }, + SchemaResource { + id: "constraints/010_ck_ksp_raw_transactions_content_hash.sql", + object: SchemaObjectContract::Constraint(ConstraintContract { kind: "c", name: "ck_ksp_raw_transactions_content_hash", table: "ksp_raw_transactions" }), + repair_existing: true, + sql: include_str!("../migrations/v001_raw_transaction/constraints/010_ck_ksp_raw_transactions_content_hash.sql"), + }, + SchemaResource { + id: "constraints/011_ck_ksp_raw_transactions_payload.sql", + object: SchemaObjectContract::Constraint(ConstraintContract { kind: "c", name: "ck_ksp_raw_transactions_payload", table: "ksp_raw_transactions" }), + repair_existing: true, + sql: include_str!("../migrations/v001_raw_transaction/constraints/011_ck_ksp_raw_transactions_payload.sql"), + }, + SchemaResource { + id: "constraints/012_ck_ksp_raw_transactions_retention_state.sql", + object: SchemaObjectContract::Constraint(ConstraintContract { + kind: "c", + name: "ck_ksp_raw_transactions_retention_state", + table: "ksp_raw_transactions", + }), + repair_existing: true, + sql: include_str!("../migrations/v001_raw_transaction/constraints/012_ck_ksp_raw_transactions_retention_state.sql"), + }, + SchemaResource { + id: "constraints/013_ck_ksp_raw_transactions_payload_state.sql", + object: SchemaObjectContract::Constraint(ConstraintContract { + kind: "c", + name: "ck_ksp_raw_transactions_payload_state", + table: "ksp_raw_transactions", + }), + repair_existing: true, + sql: include_str!("../migrations/v001_raw_transaction/constraints/013_ck_ksp_raw_transactions_payload_state.sql"), + }, + SchemaResource { + id: "constraints/014_ck_ksp_raw_transactions_purged_block_time.sql", + object: SchemaObjectContract::Constraint(ConstraintContract { + kind: "c", + name: "ck_ksp_raw_transactions_purged_block_time", + table: "ksp_raw_transactions", + }), + repair_existing: true, + sql: include_str!("../migrations/v001_raw_transaction/constraints/014_ck_ksp_raw_transactions_purged_block_time.sql"), + }, + SchemaResource { + id: "constraints/015_pk_ksp_raw_transaction_observations.sql", + object: SchemaObjectContract::Constraint(ConstraintContract { + kind: "p", + name: "pk_ksp_raw_transaction_observations", + table: "ksp_raw_transaction_observations", + }), + repair_existing: true, + sql: include_str!("../migrations/v001_raw_transaction/constraints/015_pk_ksp_raw_transaction_observations.sql"), + }, + SchemaResource { + id: "constraints/016_fk_ksp_raw_transaction_observations_transaction.sql", + object: SchemaObjectContract::Constraint(ConstraintContract { + kind: "f", + name: "fk_ksp_raw_transaction_observations_transaction", + table: "ksp_raw_transaction_observations", + }), + repair_existing: true, + sql: include_str!("../migrations/v001_raw_transaction/constraints/016_fk_ksp_raw_transaction_observations_transaction.sql"), + }, + SchemaResource { + id: "constraints/017_ck_ksp_raw_transaction_observations_key.sql", + object: SchemaObjectContract::Constraint(ConstraintContract { + kind: "c", + name: "ck_ksp_raw_transaction_observations_key", + table: "ksp_raw_transaction_observations", + }), + repair_existing: true, + sql: include_str!("../migrations/v001_raw_transaction/constraints/017_ck_ksp_raw_transaction_observations_key.sql"), + }, + SchemaResource { + id: "constraints/018_ck_ksp_raw_transaction_observations_signature.sql", + object: SchemaObjectContract::Constraint(ConstraintContract { + kind: "c", + name: "ck_ksp_raw_transaction_observations_signature", + table: "ksp_raw_transaction_observations", + }), + repair_existing: true, + sql: include_str!("../migrations/v001_raw_transaction/constraints/018_ck_ksp_raw_transaction_observations_signature.sql"), + }, + SchemaResource { + id: "constraints/019_ck_ksp_raw_transaction_observations_provider.sql", + object: SchemaObjectContract::Constraint(ConstraintContract { + kind: "c", + name: "ck_ksp_raw_transaction_observations_provider", + table: "ksp_raw_transaction_observations", + }), + repair_existing: true, + sql: include_str!("../migrations/v001_raw_transaction/constraints/019_ck_ksp_raw_transaction_observations_provider.sql"), + }, + SchemaResource { + id: "constraints/020_ck_ksp_raw_transaction_observations_protocol.sql", + object: SchemaObjectContract::Constraint(ConstraintContract { + kind: "c", + name: "ck_ksp_raw_transaction_observations_protocol", + table: "ksp_raw_transaction_observations", + }), + repair_existing: true, + sql: include_str!("../migrations/v001_raw_transaction/constraints/020_ck_ksp_raw_transaction_observations_protocol.sql"), + }, + SchemaResource { + id: "constraints/021_ck_ksp_raw_transaction_observations_method.sql", + object: SchemaObjectContract::Constraint(ConstraintContract { + kind: "c", + name: "ck_ksp_raw_transaction_observations_method", + table: "ksp_raw_transaction_observations", + }), + repair_existing: true, + sql: include_str!("../migrations/v001_raw_transaction/constraints/021_ck_ksp_raw_transaction_observations_method.sql"), + }, + SchemaResource { + id: "constraints/022_ck_ksp_raw_transaction_observations_origin.sql", + object: SchemaObjectContract::Constraint(ConstraintContract { + kind: "c", + name: "ck_ksp_raw_transaction_observations_origin", + table: "ksp_raw_transaction_observations", + }), + repair_existing: true, + sql: include_str!("../migrations/v001_raw_transaction/constraints/022_ck_ksp_raw_transaction_observations_origin.sql"), + }, + SchemaResource { + id: "constraints/023_ck_ksp_raw_transaction_observations_received_at.sql", + object: SchemaObjectContract::Constraint(ConstraintContract { + kind: "c", + name: "ck_ksp_raw_transaction_observations_received_at", + table: "ksp_raw_transaction_observations", + }), + repair_existing: true, + sql: include_str!("../migrations/v001_raw_transaction/constraints/023_ck_ksp_raw_transaction_observations_received_at.sql"), + }, + SchemaResource { + id: "constraints/024_ck_ksp_raw_transaction_observations_capture_session.sql", + object: SchemaObjectContract::Constraint(ConstraintContract { + kind: "c", + name: "ck_ksp_raw_transaction_observations_capture_session", + table: "ksp_raw_transaction_observations", + }), + repair_existing: true, + sql: include_str!("../migrations/v001_raw_transaction/constraints/024_ck_ksp_raw_transaction_observations_capture_session.sql"), + }, + SchemaResource { + id: "constraints/025_ck_ksp_raw_transaction_observations_commitment.sql", + object: SchemaObjectContract::Constraint(ConstraintContract { + kind: "c", + name: "ck_ksp_raw_transaction_observations_commitment", + table: "ksp_raw_transaction_observations", + }), + repair_existing: true, + sql: include_str!("../migrations/v001_raw_transaction/constraints/025_ck_ksp_raw_transaction_observations_commitment.sql"), + }, + SchemaResource { + id: "constraints/026_ck_ksp_raw_transaction_observations_endpoint.sql", + object: SchemaObjectContract::Constraint(ConstraintContract { + kind: "c", + name: "ck_ksp_raw_transaction_observations_endpoint", + table: "ksp_raw_transaction_observations", + }), + repair_existing: true, + sql: include_str!("../migrations/v001_raw_transaction/constraints/026_ck_ksp_raw_transaction_observations_endpoint.sql"), + }, + SchemaResource { + id: "constraints/027_ck_ksp_raw_transaction_observations_filter.sql", + object: SchemaObjectContract::Constraint(ConstraintContract { + kind: "c", + name: "ck_ksp_raw_transaction_observations_filter", + table: "ksp_raw_transaction_observations", + }), + repair_existing: true, + sql: include_str!("../migrations/v001_raw_transaction/constraints/027_ck_ksp_raw_transaction_observations_filter.sql"), + }, + SchemaResource { + id: "constraints/028_ck_ksp_raw_transaction_observations_observed_at.sql", + object: SchemaObjectContract::Constraint(ConstraintContract { + kind: "c", + name: "ck_ksp_raw_transaction_observations_observed_at", + table: "ksp_raw_transaction_observations", + }), + repair_existing: true, + sql: include_str!("../migrations/v001_raw_transaction/constraints/028_ck_ksp_raw_transaction_observations_observed_at.sql"), + }, + SchemaResource { + id: "constraints/029_ck_ksp_raw_transaction_observations_time_order.sql", + object: SchemaObjectContract::Constraint(ConstraintContract { + kind: "c", + name: "ck_ksp_raw_transaction_observations_time_order", + table: "ksp_raw_transaction_observations", + }), + repair_existing: true, + sql: include_str!("../migrations/v001_raw_transaction/constraints/029_ck_ksp_raw_transaction_observations_time_order.sql"), + }, + SchemaResource { + id: "constraints/030_ck_ksp_raw_transaction_observations_source_hash.sql", + object: SchemaObjectContract::Constraint(ConstraintContract { + kind: "c", + name: "ck_ksp_raw_transaction_observations_source_hash", + table: "ksp_raw_transaction_observations", + }), + repair_existing: true, + sql: include_str!("../migrations/v001_raw_transaction/constraints/030_ck_ksp_raw_transaction_observations_source_hash.sql"), + }, + SchemaResource { + id: "constraints/031_ck_ksp_raw_transaction_observations_source_size.sql", + object: SchemaObjectContract::Constraint(ConstraintContract { + kind: "c", + name: "ck_ksp_raw_transaction_observations_source_size", + table: "ksp_raw_transaction_observations", + }), + repair_existing: true, + sql: include_str!("../migrations/v001_raw_transaction/constraints/031_ck_ksp_raw_transaction_observations_source_size.sql"), + }, + SchemaResource { + id: "constraints/032_pk_ksp_raw_transaction_archive_payloads.sql", + object: SchemaObjectContract::Constraint(ConstraintContract { + kind: "p", + name: "pk_ksp_raw_transaction_archive_payloads", + table: "ksp_raw_transaction_archive_payloads", + }), + repair_existing: true, + sql: include_str!("../migrations/v001_raw_transaction/constraints/032_pk_ksp_raw_transaction_archive_payloads.sql"), + }, + SchemaResource { + id: "constraints/033_fk_ksp_raw_transaction_archive_payloads_transaction.sql", + object: SchemaObjectContract::Constraint(ConstraintContract { + kind: "f", + name: "fk_ksp_raw_transaction_archive_payloads_transaction", + table: "ksp_raw_transaction_archive_payloads", + }), + repair_existing: true, + sql: include_str!("../migrations/v001_raw_transaction/constraints/033_fk_ksp_raw_transaction_archive_payloads_transaction.sql"), + }, + SchemaResource { + id: "constraints/034_ck_ksp_raw_transaction_archive_payloads_signature.sql", + object: SchemaObjectContract::Constraint(ConstraintContract { + kind: "c", + name: "ck_ksp_raw_transaction_archive_payloads_signature", + table: "ksp_raw_transaction_archive_payloads", + }), + repair_existing: true, + sql: include_str!("../migrations/v001_raw_transaction/constraints/034_ck_ksp_raw_transaction_archive_payloads_signature.sql"), + }, + SchemaResource { + id: "constraints/035_ck_ksp_raw_transaction_archive_payloads_payload.sql", + object: SchemaObjectContract::Constraint(ConstraintContract { + kind: "c", + name: "ck_ksp_raw_transaction_archive_payloads_payload", + table: "ksp_raw_transaction_archive_payloads", + }), + repair_existing: true, + sql: include_str!("../migrations/v001_raw_transaction/constraints/035_ck_ksp_raw_transaction_archive_payloads_payload.sql"), + }, + SchemaResource { + id: "indexes/001_ix_ksp_raw_transactions_slot_signature.sql", + object: SchemaObjectContract::Index(IndexContract { + access_method: "btree", + key_fragment: "slot,signature", + name: "ix_ksp_raw_transactions_slot_signature", + predicate_fragment: "retention_state<>'purged'", + table: "ksp_raw_transactions", + unique: false, + }), + repair_existing: true, + sql: include_str!("../migrations/v001_raw_transaction/indexes/001_ix_ksp_raw_transactions_slot_signature.sql"), + }, +]; + +const COLUMN_LOAD_SQL: &str = r#"SELECT column_name::TEXT, udt_name::TEXT, (is_nullable = 'YES') AS nullable, numeric_precision::INTEGER, numeric_scale::INTEGER, column_default::TEXT, is_identity::TEXT, is_generated::TEXT +FROM information_schema.columns +WHERE table_schema = current_schema() AND table_name = $1 +ORDER BY ordinal_position"#; +const CONSTRAINT_LOAD_SQL: &str = r#"SELECT con.contype::TEXT, con.convalidated, con.condeferrable, con.condeferred, pg_get_constraintdef(con.oid, true)::TEXT +FROM pg_constraint con +JOIN pg_class rel ON rel.oid = con.conrelid +JOIN pg_namespace ns ON ns.oid = rel.relnamespace +WHERE ns.nspname = current_schema() AND rel.relname = $1 AND con.conname = $2"#; +const CONSTRAINT_FALLBACK_LOAD_SQL: &str = r#"SELECT con.contype::TEXT, con.convalidated, con.condeferrable, con.condeferred, pg_get_constraintdef(con.oid, true)::TEXT +FROM pg_constraint con +JOIN pg_class rel ON rel.oid = con.conrelid +JOIN pg_namespace ns ON ns.oid = rel.relnamespace +WHERE ns.nspname = current_schema() AND rel.relname = $1 AND con.contype::TEXT = $2 +ORDER BY con.conname"#; +const INDEX_LOAD_SQL: &str = r#"SELECT idx.indisunique, am.amname::TEXT, pg_get_indexdef(index_rel.oid)::TEXT, pg_get_expr(idx.indpred, idx.indrelid, true)::TEXT +FROM pg_index idx +JOIN pg_class index_rel ON index_rel.oid = idx.indexrelid +JOIN pg_class table_rel ON table_rel.oid = idx.indrelid +JOIN pg_namespace ns ON ns.oid = table_rel.relnamespace +JOIN pg_am am ON am.oid = index_rel.relam +WHERE ns.nspname = current_schema() AND table_rel.relname = $1 AND index_rel.relname = $2"#; +const MANAGED_OBJECT_EXISTS_SQL: &str = r#"SELECT EXISTS ( + SELECT 1 FROM information_schema.tables + WHERE table_schema = current_schema() + AND table_name IN ('ksp_store_identity', 'ksp_raw_transactions', 'ksp_raw_transaction_observations', 'ksp_raw_transaction_archive_payloads') + AND table_type = 'BASE TABLE' +)"#; +const PRIMARY_KEY_LOAD_SQL: &str = r#"SELECT string_agg(att.attname::TEXT, ',' ORDER BY key_part.ord)::TEXT +FROM pg_constraint con +JOIN pg_class rel ON rel.oid = con.conrelid +JOIN pg_namespace ns ON ns.oid = rel.relnamespace +JOIN LATERAL unnest(con.conkey) WITH ORDINALITY AS key_part(attnum, ord) ON true +JOIN pg_attribute att ON att.attrelid = con.conrelid AND att.attnum = key_part.attnum +WHERE ns.nspname = current_schema() AND rel.relname = $1 AND con.contype = 'p' +GROUP BY con.oid"#; +const RULE_COUNT_SQL: &str = r#"SELECT COUNT(*)::BIGINT +FROM pg_rewrite rw +JOIN pg_class rel ON rel.oid = rw.ev_class +JOIN pg_namespace ns ON ns.oid = rel.relnamespace +WHERE ns.nspname = current_schema() AND rel.relname = $1 AND rw.rulename <> '_RETURN'"#; +const TABLE_LOAD_SQL: &str = r#"SELECT rel.relkind::TEXT, rel.relrowsecurity, rel.relforcerowsecurity +FROM pg_class rel +JOIN pg_namespace ns ON ns.oid = rel.relnamespace +WHERE ns.nspname = current_schema() AND rel.relname = $1"#; +const TRIGGER_COUNT_SQL: &str = r#"SELECT COUNT(*)::BIGINT +FROM pg_trigger trg +JOIN pg_class rel ON rel.oid = trg.tgrelid +JOIN pg_namespace ns ON ns.oid = rel.relnamespace +WHERE ns.nspname = current_schema() AND rel.relname = $1 AND NOT trg.tgisinternal AND trg.tgenabled <> 'D'"#; +const UNEXPECTED_CONSTRAINTS_SQL: &str = r#"SELECT con.conname::TEXT, con.contype::TEXT, con.convalidated, con.condeferrable, con.condeferred, pg_get_constraintdef(con.oid, true)::TEXT +FROM pg_constraint con +JOIN pg_class rel ON rel.oid = con.conrelid +JOIN pg_namespace ns ON ns.oid = rel.relnamespace +WHERE ns.nspname = current_schema() AND rel.relname = $1 +ORDER BY con.conname"#; +const UNIQUE_INDEXES_SQL: &str = r#"SELECT index_rel.relname::TEXT, EXISTS (SELECT 1 FROM pg_constraint con WHERE con.conindid = idx.indexrelid) AS constraint_backed +FROM pg_index idx +JOIN pg_class index_rel ON index_rel.oid = idx.indexrelid +JOIN pg_class table_rel ON table_rel.oid = idx.indrelid +JOIN pg_namespace ns ON ns.oid = table_rel.relnamespace +WHERE ns.nspname = current_schema() AND table_rel.relname = $1 AND idx.indisunique +ORDER BY index_rel.relname"#; + +#[derive(Clone, Copy)] +struct ColumnContract { + name: &'static str, + udt_name: &'static str, + nullable: bool, + numeric_precision: std::option::Option, + numeric_scale: std::option::Option, +} + +#[derive(Clone, Copy)] +struct ConstraintContract { + kind: &'static str, + name: &'static str, + table: &'static str, +} + +#[derive(Clone, Copy)] +struct IndexContract { + access_method: &'static str, + key_fragment: &'static str, + name: &'static str, + predicate_fragment: &'static str, + table: &'static str, + unique: bool, +} + +#[derive(Clone, Copy)] +struct TableContract { + columns: &'static [ColumnContract], + name: &'static str, + primary_key_columns: std::option::Option<&'static str>, +} + +#[derive(Clone, Copy)] +enum SchemaObjectContract { + Constraint(ConstraintContract), + Index(IndexContract), + Table(TableContract), +} + +#[derive(Clone, Copy)] +/// Embedded physical schema resource owned by one logical migration. +pub(crate) struct SchemaResource { + /// Stable resource identifier relative to the logical migration directory. + pub(crate) id: &'static str, + object: SchemaObjectContract, + /// Whether an applied migration may safely recreate this resource when missing. + pub(crate) repair_existing: bool, + /// Embedded idempotent SQL used to create or additively repair the resource. + pub(crate) sql: &'static str, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +/// Compatibility state of one managed physical schema resource. +pub(crate) enum SchemaResourceState { + /// The effective PostgreSQL object satisfies the KSP contract. + Compatible, + /// An object exists but its effective structure can block or alter KSP semantics. + Incompatible, + /// The required managed object or one of its required additive parts is absent. + Missing, +} + +const KSP_STORE_SCHEMA_MIGRATIONS_COLUMNS: &[ColumnContract] = &[ + ColumnContract { + name: "version", + nullable: false, + numeric_precision: std::option::Option::None, + numeric_scale: std::option::Option::None, + udt_name: "int8", + }, + ColumnContract { + name: "name", + nullable: false, + numeric_precision: std::option::Option::None, + numeric_scale: std::option::Option::None, + udt_name: "text", + }, + ColumnContract { + name: "checksum", + nullable: false, + numeric_precision: std::option::Option::None, + numeric_scale: std::option::Option::None, + udt_name: "text", + }, + ColumnContract { + name: "applied_at", + nullable: false, + numeric_precision: std::option::Option::None, + numeric_scale: std::option::Option::None, + udt_name: "timestamptz", + }, +]; +const KSP_STORE_IDENTITY_COLUMNS: &[ColumnContract] = &[ + ColumnContract { + name: "singleton", + nullable: false, + numeric_precision: std::option::Option::None, + numeric_scale: std::option::Option::None, + udt_name: "int2", + }, + ColumnContract { + name: "network", + nullable: false, + numeric_precision: std::option::Option::None, + numeric_scale: std::option::Option::None, + udt_name: "text", + }, +]; +const KSP_RAW_TRANSACTIONS_COLUMNS: &[ColumnContract] = &[ + ColumnContract { + name: "signature", + nullable: false, + numeric_precision: std::option::Option::None, + numeric_scale: std::option::Option::None, + udt_name: "bytea", + }, + ColumnContract { + name: "slot", + nullable: false, + numeric_precision: std::option::Option::Some(20), + numeric_scale: std::option::Option::Some(0), + udt_name: "numeric", + }, + ColumnContract { + name: "block_time_unix_millis", + nullable: true, + numeric_precision: std::option::Option::None, + numeric_scale: std::option::Option::None, + udt_name: "int8", + }, + ColumnContract { + name: "format_id", + nullable: false, + numeric_precision: std::option::Option::None, + numeric_scale: std::option::Option::None, + udt_name: "text", + }, + ColumnContract { + name: "format_version", + nullable: false, + numeric_precision: std::option::Option::None, + numeric_scale: std::option::Option::None, + udt_name: "int8", + }, + ColumnContract { + name: "content_hash", + nullable: false, + numeric_precision: std::option::Option::None, + numeric_scale: std::option::Option::None, + udt_name: "bytea", + }, + ColumnContract { + name: "payload", + nullable: true, + numeric_precision: std::option::Option::None, + numeric_scale: std::option::Option::None, + udt_name: "bytea", + }, + ColumnContract { + name: "retention_state", + nullable: false, + numeric_precision: std::option::Option::None, + numeric_scale: std::option::Option::None, + udt_name: "text", + }, +]; +const KSP_RAW_TRANSACTION_OBSERVATIONS_COLUMNS: &[ColumnContract] = &[ + ColumnContract { + name: "observation_key", + nullable: false, + numeric_precision: std::option::Option::None, + numeric_scale: std::option::Option::None, + udt_name: "bytea", + }, + ColumnContract { + name: "transaction_signature", + nullable: false, + numeric_precision: std::option::Option::None, + numeric_scale: std::option::Option::None, + udt_name: "bytea", + }, + ColumnContract { + name: "provider", + nullable: false, + numeric_precision: std::option::Option::None, + numeric_scale: std::option::Option::None, + udt_name: "text", + }, + ColumnContract { + name: "protocol", + nullable: false, + numeric_precision: std::option::Option::None, + numeric_scale: std::option::Option::None, + udt_name: "text", + }, + ColumnContract { + name: "acquisition_method", + nullable: false, + numeric_precision: std::option::Option::None, + numeric_scale: std::option::Option::None, + udt_name: "text", + }, + ColumnContract { + name: "origin", + nullable: false, + numeric_precision: std::option::Option::None, + numeric_scale: std::option::Option::None, + udt_name: "text", + }, + ColumnContract { + name: "received_at_unix_millis", + nullable: false, + numeric_precision: std::option::Option::None, + numeric_scale: std::option::Option::None, + udt_name: "int8", + }, + ColumnContract { + name: "capture_session_id", + nullable: true, + numeric_precision: std::option::Option::None, + numeric_scale: std::option::Option::None, + udt_name: "text", + }, + ColumnContract { + name: "commitment", + nullable: true, + numeric_precision: std::option::Option::None, + numeric_scale: std::option::Option::None, + udt_name: "text", + }, + ColumnContract { + name: "endpoint_id", + nullable: true, + numeric_precision: std::option::Option::None, + numeric_scale: std::option::Option::None, + udt_name: "text", + }, + ColumnContract { + name: "filter_id", + nullable: true, + numeric_precision: std::option::Option::None, + numeric_scale: std::option::Option::None, + udt_name: "text", + }, + ColumnContract { + name: "observed_at_unix_millis", + nullable: true, + numeric_precision: std::option::Option::None, + numeric_scale: std::option::Option::None, + udt_name: "int8", + }, + ColumnContract { + name: "source_payload_hash", + nullable: true, + numeric_precision: std::option::Option::None, + numeric_scale: std::option::Option::None, + udt_name: "bytea", + }, + ColumnContract { + name: "source_payload_size_bytes", + nullable: true, + numeric_precision: std::option::Option::None, + numeric_scale: std::option::Option::None, + udt_name: "int8", + }, +]; +const KSP_RAW_TRANSACTION_ARCHIVE_PAYLOADS_COLUMNS: &[ColumnContract] = &[ + ColumnContract { + name: "signature", + nullable: false, + numeric_precision: std::option::Option::None, + numeric_scale: std::option::Option::None, + udt_name: "bytea", + }, + ColumnContract { + name: "payload", + nullable: false, + numeric_precision: std::option::Option::None, + numeric_scale: std::option::Option::None, + udt_name: "bytea", + }, +]; +const V001_TABLE_NAMES: &[&str] = &["ksp_store_identity", "ksp_raw_transactions", "ksp_raw_transaction_observations", "ksp_raw_transaction_archive_payloads"]; + +/// Inspects one embedded schema resource against the effective PostgreSQL catalog. +pub(crate) async fn inspect_resource( + transaction: &deadpool_postgres::Transaction<'_>, + resource: &SchemaResource, +) -> std::result::Result { + return match resource.object { + SchemaObjectContract::Constraint(contract) => inspect_constraint(transaction, contract, resource.sql).await, + SchemaObjectContract::Index(contract) => inspect_index(transaction, contract).await, + SchemaObjectContract::Table(contract) => inspect_table(transaction, contract).await, + }; +} + +/// Returns whether any V001-managed base table already exists in the active schema. +pub(crate) async fn managed_v001_objects_exist(transaction: &deadpool_postgres::Transaction<'_>) -> std::result::Result { + let result = transaction.query_one(MANAGED_OBJECT_EXISTS_SQL, &[]).await; + let row = match result { + std::result::Result::Ok(value) => value, + std::result::Result::Err(_) => return schema_query_error("schema_adoption_probe"), + }; + return match row.try_get::(0) { + std::result::Result::Ok(value) => std::result::Result::Ok(value), + std::result::Result::Err(_) => schema_query_error("schema_adoption_probe_decode"), + }; +} + +/// Rejects external V001 schema extensions that can constrain or mutate KSP writes. +pub(crate) async fn verify_v001_external_compatibility( + transaction: &deadpool_postgres::Transaction<'_>, +) -> std::result::Result<(), crate::PostgresBackendError> { + for table in V001_TABLE_NAMES { + let table = *table; + let constraint_rows = transaction.query(UNEXPECTED_CONSTRAINTS_SQL, &[table]).await; + let constraint_rows = match constraint_rows { + std::result::Result::Ok(value) => value, + std::result::Result::Err(_) => return schema_query_error("schema_constraint_inventory"), + }; + for row in constraint_rows { + let name = row.try_get::(0); + let kind = row.try_get::(1); + let validated = row.try_get::(2); + let deferrable = row.try_get::(3); + let deferred = row.try_get::(4); + let definition = row.try_get::(5); + let (name, kind, validated, deferrable, deferred, definition) = match (name, kind, validated, deferrable, deferred, definition) { + ( + std::result::Result::Ok(name), + std::result::Result::Ok(kind), + std::result::Result::Ok(validated), + std::result::Result::Ok(deferrable), + std::result::Result::Ok(deferred), + std::result::Result::Ok(definition), + ) => (name, kind, validated, deferrable, deferred, definition), + _ => return schema_query_error("schema_constraint_inventory_decode"), + }; + if is_expected_constraint(table, name.as_str()) { + continue; + } + let definition = normalize_catalog_sql(definition.as_str()); + if !validated || deferrable || deferred || !matches_expected_constraint_definition(table, kind.as_str(), definition.as_str()) { + return schema_incompatible("schema_external_constraint"); + } + } + let unique_rows = transaction.query(UNIQUE_INDEXES_SQL, &[table]).await; + let unique_rows = match unique_rows { + std::result::Result::Ok(value) => value, + std::result::Result::Err(_) => return schema_query_error("schema_unique_index_inventory"), + }; + for row in unique_rows { + let name = row.try_get::(0); + let constraint_backed = row.try_get::(1); + let (name, constraint_backed) = match (name, constraint_backed) { + (std::result::Result::Ok(name), std::result::Result::Ok(constraint_backed)) => (name, constraint_backed), + _ => return schema_query_error("schema_unique_index_inventory_decode"), + }; + if !constraint_backed { + return schema_incompatible("schema_external_unique_index"); + } + } + let trigger_count = load_count(transaction, TRIGGER_COUNT_SQL, table, "schema_trigger_inventory").await; + let trigger_count = match trigger_count { + std::result::Result::Ok(value) => value, + std::result::Result::Err(error) => return std::result::Result::Err(error), + }; + if trigger_count != 0 { + return schema_incompatible("schema_external_trigger"); + } + let rule_count = load_count(transaction, RULE_COUNT_SQL, table, "schema_rule_inventory").await; + let rule_count = match rule_count { + std::result::Result::Ok(value) => value, + std::result::Result::Err(error) => return std::result::Result::Err(error), + }; + if rule_count != 0 { + return schema_incompatible("schema_external_rule"); + } + } + return std::result::Result::Ok(()); +} + +async fn inspect_constraint( + transaction: &deadpool_postgres::Transaction<'_>, + contract: ConstraintContract, + resource_sql: &str, +) -> std::result::Result { + let expected_definition = match expected_constraint_definition(resource_sql, contract.name) { + std::option::Option::Some(value) => value, + std::option::Option::None => return schema_query_error("schema_constraint_contract"), + }; + let result = transaction.query_opt(CONSTRAINT_LOAD_SQL, &[&contract.table, &contract.name]).await; + let row = match result { + std::result::Result::Ok(std::option::Option::Some(value)) => value, + std::result::Result::Ok(std::option::Option::None) => { + return inspect_equivalent_constraint(transaction, contract, expected_definition.as_str()).await; + }, + std::result::Result::Err(_) => return schema_query_error("schema_constraint_probe"), + }; + let kind = row.try_get::(0); + let validated = row.try_get::(1); + let deferrable = row.try_get::(2); + let deferred = row.try_get::(3); + let definition = row.try_get::(4); + let (kind, validated, deferrable, deferred, definition) = match (kind, validated, deferrable, deferred, definition) { + ( + std::result::Result::Ok(kind), + std::result::Result::Ok(validated), + std::result::Result::Ok(deferrable), + std::result::Result::Ok(deferred), + std::result::Result::Ok(definition), + ) => (kind, validated, deferrable, deferred, definition), + _ => return schema_query_error("schema_constraint_probe_decode"), + }; + let definition = normalize_catalog_sql(definition.as_str()); + if kind != contract.kind || !validated || deferrable || deferred || definition != expected_definition { + return std::result::Result::Ok(SchemaResourceState::Incompatible); + } + return std::result::Result::Ok(SchemaResourceState::Compatible); +} + +async fn inspect_index( + transaction: &deadpool_postgres::Transaction<'_>, + contract: IndexContract, +) -> std::result::Result { + let result = transaction.query_opt(INDEX_LOAD_SQL, &[&contract.table, &contract.name]).await; + let row = match result { + std::result::Result::Ok(std::option::Option::Some(value)) => value, + std::result::Result::Ok(std::option::Option::None) => return std::result::Result::Ok(SchemaResourceState::Missing), + std::result::Result::Err(_) => return schema_query_error("schema_index_probe"), + }; + let unique = row.try_get::(0); + let access_method = row.try_get::(1); + let definition = row.try_get::(2); + let predicate = row.try_get::>(3); + let (unique, access_method, definition, predicate) = match (unique, access_method, definition, predicate) { + (std::result::Result::Ok(unique), std::result::Result::Ok(access_method), std::result::Result::Ok(definition), std::result::Result::Ok(predicate)) => { + (unique, access_method, definition, predicate) + }, + _ => return schema_query_error("schema_index_probe_decode"), + }; + let definition = normalize_catalog_sql(definition.as_str()); + let predicate = predicate.map(|value| return normalize_catalog_sql(value.as_str())); + let predicate_matches = match predicate.as_deref() { + std::option::Option::Some(value) => value.contains(contract.predicate_fragment), + std::option::Option::None => false, + }; + if unique != contract.unique || access_method != contract.access_method || !definition.contains(contract.key_fragment) || !predicate_matches { + return std::result::Result::Ok(SchemaResourceState::Incompatible); + } + return std::result::Result::Ok(SchemaResourceState::Compatible); +} + +async fn inspect_table( + transaction: &deadpool_postgres::Transaction<'_>, + contract: TableContract, +) -> std::result::Result { + let table_result = transaction.query_opt(TABLE_LOAD_SQL, &[&contract.name]).await; + let table_row = match table_result { + std::result::Result::Ok(std::option::Option::Some(value)) => value, + std::result::Result::Ok(std::option::Option::None) => return std::result::Result::Ok(SchemaResourceState::Missing), + std::result::Result::Err(_) => return schema_query_error("schema_table_probe"), + }; + let relkind = table_row.try_get::(0); + let row_security = table_row.try_get::(1); + let force_row_security = table_row.try_get::(2); + let (relkind, row_security, force_row_security) = match (relkind, row_security, force_row_security) { + (std::result::Result::Ok(relkind), std::result::Result::Ok(row_security), std::result::Result::Ok(force_row_security)) => { + (relkind, row_security, force_row_security) + }, + _ => return schema_query_error("schema_table_probe_decode"), + }; + if relkind != "r" || row_security || force_row_security { + return std::result::Result::Ok(SchemaResourceState::Incompatible); + } + let rows_result = transaction.query(COLUMN_LOAD_SQL, &[&contract.name]).await; + let rows = match rows_result { + std::result::Result::Ok(value) => value, + std::result::Result::Err(_) => return schema_query_error("schema_column_probe"), + }; + let mut actual = std::vec::Vec::::with_capacity(rows.len()); + for row in rows { + let decoded = ActualColumn::from_row(&row); + let decoded = match decoded { + std::result::Result::Ok(value) => value, + std::result::Result::Err(error) => return std::result::Result::Err(error), + }; + actual.push(decoded); + } + let mut missing = false; + for expected in contract.columns { + let found = actual.iter().find(|candidate| return candidate.name == expected.name); + let found = match found { + std::option::Option::Some(value) => value, + std::option::Option::None => { + missing = true; + continue; + }, + }; + if !found.matches(expected) { + return std::result::Result::Ok(SchemaResourceState::Incompatible); + } + } + for column in &actual { + if contract.columns.iter().any(|expected| return expected.name == column.name) { + continue; + } + if !column.is_non_blocking_extra() { + return std::result::Result::Ok(SchemaResourceState::Incompatible); + } + } + if let std::option::Option::Some(expected_key) = contract.primary_key_columns { + let key_result = transaction.query_opt(PRIMARY_KEY_LOAD_SQL, &[&contract.name]).await; + let key_row = match key_result { + std::result::Result::Ok(std::option::Option::Some(value)) => value, + std::result::Result::Ok(std::option::Option::None) => return std::result::Result::Ok(SchemaResourceState::Incompatible), + std::result::Result::Err(_) => return schema_query_error("schema_primary_key_probe"), + }; + let key_columns = key_row.try_get::(0); + let key_columns = match key_columns { + std::result::Result::Ok(value) => value, + std::result::Result::Err(_) => return schema_query_error("schema_primary_key_probe_decode"), + }; + if key_columns != expected_key { + return std::result::Result::Ok(SchemaResourceState::Incompatible); + } + } + if missing { + return std::result::Result::Ok(SchemaResourceState::Missing); + } + return std::result::Result::Ok(SchemaResourceState::Compatible); +} + +struct ActualColumn { + default: std::option::Option, + generated: std::string::String, + identity: std::string::String, + name: std::string::String, + nullable: bool, + numeric_precision: std::option::Option, + numeric_scale: std::option::Option, + udt_name: std::string::String, +} + +impl ActualColumn { + fn from_row(row: &tokio_postgres::Row) -> std::result::Result { + let name = row.try_get::(0); + let udt_name = row.try_get::(1); + let nullable = row.try_get::(2); + let numeric_precision = row.try_get::>(3); + let numeric_scale = row.try_get::>(4); + let default = row.try_get::>(5); + let identity = row.try_get::(6); + let generated = row.try_get::(7); + return match (name, udt_name, nullable, numeric_precision, numeric_scale, default, identity, generated) { + ( + std::result::Result::Ok(name), + std::result::Result::Ok(udt_name), + std::result::Result::Ok(nullable), + std::result::Result::Ok(numeric_precision), + std::result::Result::Ok(numeric_scale), + std::result::Result::Ok(default), + std::result::Result::Ok(identity), + std::result::Result::Ok(generated), + ) => std::result::Result::Ok(Self { default, generated, identity, name, nullable, numeric_precision, numeric_scale, udt_name }), + _ => schema_query_error("schema_column_probe_decode"), + }; + } + + fn is_non_blocking_extra(&self) -> bool { + return self.nullable && self.default.is_none() && self.identity == "NO" && self.generated == "NEVER"; + } + + fn matches(&self, expected: &ColumnContract) -> bool { + let numeric_matches = if expected.udt_name == "numeric" { + self.numeric_precision == expected.numeric_precision && self.numeric_scale == expected.numeric_scale + } else { + true + }; + return self.udt_name == expected.udt_name + && self.nullable == expected.nullable + && numeric_matches + && self.identity == "NO" + && self.generated == "NEVER"; + } +} + +async fn load_count( + transaction: &deadpool_postgres::Transaction<'_>, + sql: &str, + table: &str, + phase: &'static str, +) -> std::result::Result { + let result = transaction.query_one(sql, &[&table]).await; + let row = match result { + std::result::Result::Ok(value) => value, + std::result::Result::Err(_) => return schema_query_error(phase), + }; + return match row.try_get::(0) { + std::result::Result::Ok(value) => std::result::Result::Ok(value), + std::result::Result::Err(_) => schema_query_error("schema_inventory_decode"), + }; +} + +async fn inspect_equivalent_constraint( + transaction: &deadpool_postgres::Transaction<'_>, + contract: ConstraintContract, + expected_definition: &str, +) -> std::result::Result { + let rows = transaction.query(CONSTRAINT_FALLBACK_LOAD_SQL, &[&contract.table, &contract.kind]).await; + let rows = match rows { + std::result::Result::Ok(value) => value, + std::result::Result::Err(_) => return schema_query_error("schema_constraint_fallback_probe"), + }; + for row in rows { + let validated = row.try_get::(1); + let deferrable = row.try_get::(2); + let deferred = row.try_get::(3); + let definition = row.try_get::(4); + let (validated, deferrable, deferred, definition) = match (validated, deferrable, deferred, definition) { + ( + std::result::Result::Ok(validated), + std::result::Result::Ok(deferrable), + std::result::Result::Ok(deferred), + std::result::Result::Ok(definition), + ) => (validated, deferrable, deferred, definition), + _ => return schema_query_error("schema_constraint_fallback_probe_decode"), + }; + let definition = normalize_catalog_sql(definition.as_str()); + if validated && !deferrable && !deferred && definition == expected_definition { + return std::result::Result::Ok(SchemaResourceState::Compatible); + } + } + return std::result::Result::Ok(SchemaResourceState::Missing); +} + +fn expected_constraint_definition(resource_sql: &str, name: &str) -> std::option::Option { + let marker = std::format!("ADD CONSTRAINT {name} "); + let start = resource_sql.find(marker.as_str())?.checked_add(marker.len())?; + let tail = resource_sql.get(start..)?; + let end = tail.find(';')?; + let definition = tail.get(..end)?; + return std::option::Option::Some(normalize_catalog_sql(definition)); +} + +fn is_expected_constraint(table: &str, name: &str) -> bool { + for resource in V001_RESOURCES { + let contract = match resource.object { + SchemaObjectContract::Constraint(value) => value, + SchemaObjectContract::Index(_) | SchemaObjectContract::Table(_) => continue, + }; + if contract.table == table && contract.name == name { + return true; + } + } + return false; +} + +fn matches_expected_constraint_definition(table: &str, kind: &str, definition: &str) -> bool { + for resource in V001_RESOURCES { + let contract = match resource.object { + SchemaObjectContract::Constraint(value) => value, + SchemaObjectContract::Index(_) | SchemaObjectContract::Table(_) => continue, + }; + if contract.table != table || contract.kind != kind { + continue; + } + let expected = expected_constraint_definition(resource.sql, contract.name); + if let std::option::Option::Some(expected) = expected + && expected == definition + { + return true; + } + } + return false; +} + +fn normalize_catalog_sql(value: &str) -> std::string::String { + return value + .chars() + .filter(|character| return !character.is_whitespace() && *character != '"' && *character != '(' && *character != ')') + .collect::() + .replace("::text", "") + .replace("::numeric", "") + .replace("::bigint", "") + .replace("::smallint", "") + .to_ascii_lowercase(); +} + +fn schema_incompatible(phase: &'static str) -> std::result::Result { + return std::result::Result::Err(crate::PostgresBackendError::new(crate::PostgresBackendErrorKind::MigrationMismatch, phase)); +} + +fn schema_query_error(phase: &'static str) -> std::result::Result { + return std::result::Result::Err(crate::PostgresBackendError::new(crate::PostgresBackendErrorKind::MigrationFailed, phase)); +} + +#[cfg(test)] +#[path = "../unit_tests/schema.rs"] +mod tests; diff --git a/crates/ksp-store-postgres-lib/tests/dependency_boundary.rs b/crates/ksp-store-postgres-lib/tests/dependency_boundary.rs index dad8665..b60afc3 100644 --- a/crates/ksp-store-postgres-lib/tests/dependency_boundary.rs +++ b/crates/ksp-store-postgres-lib/tests/dependency_boundary.rs @@ -1,5 +1,5 @@ // file: crates/ksp-store-postgres-lib/tests/dependency_boundary.rs -// version: 7 +// version: 8 #![warn(missing_docs)] #![deny(unreachable_pub)] @@ -19,9 +19,12 @@ fn pre_005_backend_owns_exact_physical_runtime_dependencies_without_reverse_faca assert!(!manifest.contains(forbidden), "forbidden PostgreSQL backend dependency detected: {forbidden}"); } let migration = include_str!("../src/migration.rs"); - let bootstrap_sql = include_str!("../migrations/V000__bootstrap.sql"); - assert!(migration.contains("include_str!(\"../migrations/V000__bootstrap.sql\")")); - assert!(migration.contains("include_str!(\"../migrations/V001__raw_transaction.sql\")")); + let schema = include_str!("../src/schema.rs"); + let bootstrap_sql = include_str!("../migrations/v000_bootstrap/tables/001_ksp_store_schema_migrations.sql"); + assert!(migration.contains("crate::V000_RESOURCES")); + assert!(migration.contains("crate::V001_RESOURCES")); + assert!(schema.contains("../migrations/v000_bootstrap/tables/001_ksp_store_schema_migrations.sql")); + assert!(schema.contains("../migrations/v001_raw_transaction/tables/001_ksp_store_identity.sql")); assert!(bootstrap_sql.contains("ksp_store_schema_migrations")); for forbidden in ["RawTransaction", "RawAccountState", "raw_transaction", "raw_account", "CORE", "DECODE", "SPECIALIZED"] { assert!(!bootstrap_sql.contains(forbidden), "business schema leaked into immutable V000 SQL: {forbidden}"); @@ -36,6 +39,7 @@ fn pre_005_backend_keeps_environment_sql_migrations_and_physical_types_private() assert!(crate_root.contains("mod health;")); assert!(crate_root.contains("mod migration;")); assert!(crate_root.contains("mod runtime;")); + assert!(crate_root.contains("mod schema;")); assert!(crate_root.contains("const _: &str = crate::TRACING_TARGET;")); for forbidden in [ "pub mod ", @@ -84,28 +88,37 @@ fn pre_007_health_probe_remains_foundation_only_and_private_sql() { } #[test] -fn pre_003_migration_engine_embeds_v001_and_binds_network_without_repository_scope() { +fn pre_003_fix_001_migration_engine_uses_split_schema_contract_and_binds_network_without_repository_scope() { let migration = include_str!("../src/migration.rs"); - let v001 = include_str!("../migrations/V001__raw_transaction.sql"); + let schema = include_str!("../src/schema.rs"); + let identity_table = include_str!("../migrations/v001_raw_transaction/tables/001_ksp_store_identity.sql"); + let raw_table = include_str!("../migrations/v001_raw_transaction/tables/002_ksp_raw_transactions.sql"); + let observation_table = include_str!("../migrations/v001_raw_transaction/tables/003_ksp_raw_transaction_observations.sql"); + let archive_table = include_str!("../migrations/v001_raw_transaction/tables/004_ksp_raw_transaction_archive_payloads.sql"); + let index = include_str!("../migrations/v001_raw_transaction/indexes/001_ix_ksp_raw_transactions_slot_signature.sql"); assert!(migration.contains("const EMBEDDED_MIGRATIONS: &[EmbeddedMigration]")); assert!(migration.contains("MigrationHook::StoreIdentity")); assert!(migration.contains("MigrationHookContext::AppliedNow")); assert!(migration.contains("MigrationHookContext::Existing")); + assert!(migration.contains("schema_autocreate")); + assert!(migration.contains("schema_autoupdate")); assert!(migration.contains("INSERT INTO ksp_store_identity (singleton, network) VALUES (1, $1)")); assert!(migration.contains("SELECT singleton, network FROM ksp_store_identity ORDER BY singleton LIMIT 2")); assert!(migration.contains("ksp_store_api::RawNetworkId::new(stored_network)")); - for required in [ - "CREATE TABLE ksp_store_identity", - "CREATE TABLE ksp_raw_transactions", - "CREATE TABLE ksp_raw_transaction_observations", - "CREATE TABLE ksp_raw_transaction_archive_payloads", - "CREATE INDEX ix_ksp_raw_transactions_slot_signature", - ] { - assert!(v001.contains(required), "missing V001 physical object: {required}"); - } + assert!(schema.contains("SchemaResourceState")); + assert!(schema.contains("verify_v001_external_compatibility")); + assert!(identity_table.contains("CREATE TABLE IF NOT EXISTS ksp_store_identity")); + assert!(raw_table.contains("CREATE TABLE IF NOT EXISTS ksp_raw_transactions")); + assert!(observation_table.contains("CREATE TABLE IF NOT EXISTS ksp_raw_transaction_observations")); + assert!(archive_table.contains("CREATE TABLE IF NOT EXISTS ksp_raw_transaction_archive_payloads")); + assert!(index.contains("CREATE INDEX IF NOT EXISTS ix_ksp_raw_transactions_slot_signature")); for forbidden in ["impl ksp_store_api::RawTransaction", "repository", "sqlx", "RawAccountState"] { assert!(!migration.contains(forbidden), "repository/cross-scope implementation leaked into migration engine: {forbidden}"); - assert!(!v001.contains(forbidden), "forbidden V001 scope content detected: {forbidden}"); + assert!(!schema.contains(forbidden), "repository/cross-scope implementation leaked into schema contract: {forbidden}"); + } + for removed in ["migrations/V000__bootstrap.sql", "migrations/V001__raw_transaction.sql"] { + let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join(removed); + assert!(!path.exists(), "obsolete monolithic migration must be deleted by pre.003-fix.001: {removed}"); } return; } diff --git a/crates/ksp-store-postgres-lib/tests/hardening_completeness.rs b/crates/ksp-store-postgres-lib/tests/hardening_completeness.rs index 5ccbe80..9b5f6a9 100644 --- a/crates/ksp-store-postgres-lib/tests/hardening_completeness.rs +++ b/crates/ksp-store-postgres-lib/tests/hardening_completeness.rs @@ -1,5 +1,5 @@ // file: crates/ksp-store-postgres-lib/tests/hardening_completeness.rs -// version: 2 +// version: 3 #![warn(missing_docs)] #![deny(unreachable_pub)] @@ -109,7 +109,7 @@ fn assert_pre_io_rejection(connection_uri: &str, tls_mode: ksp_store_postgres_li #[test] fn pre_009_backend_modules_exports_and_manifest_dependencies_are_exact() { let crate_root = include_str!("../src/lib.rs"); - for required in ["mod constants;", "mod error;", "mod health;", "mod migration;", "mod runtime;"] { + for required in ["mod constants;", "mod error;", "mod health;", "mod migration;", "mod runtime;", "mod schema;"] { assert!(crate_root.contains(required), "missing PostgreSQL backend module: {required}"); } assert!(!crate_root.contains("pub mod ")); @@ -183,12 +183,14 @@ fn pre_009_backend_has_no_env_bypass_or_business_persistence_capability() { {} {} {} +{} {}", include_str!("../src/error.rs"), include_str!("../src/health.rs"), include_str!("../src/lib.rs"), include_str!("../src/migration.rs"), - include_str!("../src/runtime.rs") + include_str!("../src/runtime.rs"), + include_str!("../src/schema.rs") ); for forbidden in [ "std::env", @@ -212,7 +214,7 @@ fn pre_009_backend_has_no_env_bypass_or_business_persistence_capability() { ] { assert!(!production.contains(forbidden), "forbidden backend ownership/capability material detected: {forbidden}"); } - let bootstrap_sql = include_str!("../migrations/V000__bootstrap.sql"); + let bootstrap_sql = include_str!("../migrations/v000_bootstrap/tables/001_ksp_store_schema_migrations.sql"); assert!(bootstrap_sql.contains("ksp_store_schema_migrations")); for forbidden in ["RawTransaction", "RawAccountState", "raw_transaction", "raw_account", "CORE", "DECODE", "SPECIALIZED"] { assert!(!bootstrap_sql.contains(forbidden), "business schema leaked into foundation migration: {forbidden}"); diff --git a/crates/ksp-store-postgres-lib/tests/public_api.rs b/crates/ksp-store-postgres-lib/tests/public_api.rs index ac1236c..5652030 100644 --- a/crates/ksp-store-postgres-lib/tests/public_api.rs +++ b/crates/ksp-store-postgres-lib/tests/public_api.rs @@ -62,9 +62,6 @@ fn pre_007_backend_health_bridge_exposes_only_safe_snapshot_types() { #[test] fn pre_003_retention_compaction_error_code_matches_store_contract_value() { assert_eq!(ksp_store_postgres_lib::ERROR_CODE_POSTGRES_RETENTION_COMPACTION_UNSUPPORTED.domain(), "store"); - assert_eq!( - ksp_store_postgres_lib::ERROR_CODE_POSTGRES_RETENTION_COMPACTION_UNSUPPORTED.code(), - "postgres_retention_compaction_unsupported", - ); + assert_eq!(ksp_store_postgres_lib::ERROR_CODE_POSTGRES_RETENTION_COMPACTION_UNSUPPORTED.code(), "postgres_retention_compaction_unsupported",); return; } diff --git a/crates/ksp-store-postgres-lib/unit_tests/migration.rs b/crates/ksp-store-postgres-lib/unit_tests/migration.rs index fc9d770..813352b 100644 --- a/crates/ksp-store-postgres-lib/unit_tests/migration.rs +++ b/crates/ksp-store-postgres-lib/unit_tests/migration.rs @@ -1,89 +1,103 @@ // file: crates/ksp-store-postgres-lib/unit_tests/migration.rs -// version: 3 +// version: 4 fn applied(version: i64, name: &str, checksum: &str) -> super::AppliedMigration { return super::AppliedMigration { checksum: checksum.to_owned(), name: name.to_owned(), version }; } -fn embedded(version: i64, name: &'static str, sql: &'static str) -> super::EmbeddedMigration { - return super::EmbeddedMigration { hook: super::MigrationHook::None, name, sql, version }; +fn embedded(version: i64, name: &'static str, resources: &'static [crate::SchemaResource]) -> super::EmbeddedMigration { + return super::EmbeddedMigration { + checksum: super::MigrationChecksum::Resources, + hook: super::MigrationHook::None, + name, + resources, + version, + }; } #[test] -fn pre_003_embedded_registry_keeps_v000_immutable_and_adds_exact_v001() { +fn pre_003_fix_001_embedded_registry_keeps_v000_checksum_and_uses_resource_owned_v001() { assert_eq!(super::EMBEDDED_MIGRATIONS.len(), 2); let v000 = &super::EMBEDDED_MIGRATIONS[0]; assert_eq!(v000.version, 0); assert_eq!(v000.name, "bootstrap"); assert_eq!(v000.hook, super::MigrationHook::None); - assert!(v000.sql.contains("CREATE TABLE ksp_store_schema_migrations")); - for forbidden in ["RawTransaction", "RawAccountState", "raw_transaction", "raw_account", "CORE", "DECODE", "SPECIALIZED"] { - assert!(!v000.sql.contains(forbidden), "business schema leaked into immutable V000 SQL: {forbidden}"); - } - assert_eq!(super::migration_checksum(v000.sql), "d29068b8c13b9dc0cc9ef6aaadd0fa12d41e0fe4c56541a1118c4bfc846a1450"); + assert_eq!(v000.resources.len(), 1); + assert!(v000.resources[0].sql.contains("CREATE TABLE ksp_store_schema_migrations")); + assert_eq!(super::migration_checksum(v000), "d29068b8c13b9dc0cc9ef6aaadd0fa12d41e0fe4c56541a1118c4bfc846a1450"); let v001 = &super::EMBEDDED_MIGRATIONS[1]; assert_eq!(v001.version, 1); assert_eq!(v001.name, "raw_transaction"); assert_eq!(v001.hook, super::MigrationHook::StoreIdentity); - assert_eq!(super::migration_checksum(v001.sql), "6fe57ed0313d2ed295280dd6e49f6d86695d4e4effee2724a25e36db1ea17761"); + assert_eq!(v001.resources.len(), 40); + assert_eq!(super::migration_checksum(v001), "31488cda2f08f3f46c4cdbdbb6c18c243662fada02eac4487040c8735d72cc51"); assert!(super::validate_embedded_registry(super::EMBEDDED_MIGRATIONS).is_ok()); assert_eq!(crate::current_migration_version(), 1); return; } #[test] -fn pre_003_v001_inventory_indexes_and_api_bounds_are_exact() { - let sql = super::EMBEDDED_MIGRATIONS[1].sql; +fn pre_003_fix_001_v001_resource_order_and_api_bounds_are_exact() { + let v001 = &super::EMBEDDED_MIGRATIONS[1]; + let ids = v001.resources.iter().map(|resource| return resource.id).collect::>(); + assert_eq!(ids.iter().filter(|id| return id.starts_with("tables/")).count(), 4); + assert_eq!(ids.iter().filter(|id| return id.starts_with("constraints/")).count(), 35); + assert_eq!(ids.iter().filter(|id| return id.starts_with("indexes/")).count(), 1); + assert!(ids[..4].iter().all(|id| return id.starts_with("tables/"))); + assert!(ids[4..39].iter().all(|id| return id.starts_with("constraints/"))); + assert!(ids[39..].iter().all(|id| return id.starts_with("indexes/"))); + let mut unique = std::collections::BTreeSet::<&str>::new(); + for id in &ids { + assert!(unique.insert(id), "duplicate embedded migration resource: {id}"); + } + let sql = v001.resources.iter().map(|resource| return resource.sql).collect::>().concat(); for required in [ - "CREATE TABLE ksp_store_identity", - "CREATE TABLE ksp_raw_transactions", - "CREATE TABLE ksp_raw_transaction_observations", - "CREATE TABLE ksp_raw_transaction_archive_payloads", - "CREATE INDEX ix_ksp_raw_transactions_slot_signature", + "CREATE TABLE IF NOT EXISTS ksp_store_identity", + "CREATE TABLE IF NOT EXISTS ksp_raw_transactions", + "CREATE TABLE IF NOT EXISTS ksp_raw_transaction_observations", + "CREATE TABLE IF NOT EXISTS ksp_raw_transaction_archive_payloads", + "CREATE INDEX IF NOT EXISTS ix_ksp_raw_transactions_slot_signature", "WHERE retention_state <> 'purged'", "octet_length(signature) = 64", - "slot BETWEEN 0 AND 18446744073709551615", - "block_time_unix_millis BETWEEN 0 AND 253402300799999", + "slot >= 0 AND slot <= 18446744073709551615", + "block_time_unix_millis >= 0 AND block_time_unix_millis <= 253402300799999", "octet_length(content_hash) = 32", - "octet_length(payload) BETWEEN 1 AND 16777216", - "format_version BETWEEN 1 AND 4294967295", - "received_at_unix_millis BETWEEN 0 AND 253402300799999", - "source_payload_size_bytes BETWEEN 0 AND 67108864", - "origin IN ('backfill', 'import', 'live', 'repair', 'replay')", - "retention_state IN ('full', 'archived', 'purged')", + "octet_length(payload) >= 1 AND octet_length(payload) <= 16777216", + "format_version >= 1 AND format_version <= 4294967295", + "received_at_unix_millis >= 0 AND received_at_unix_millis <= 253402300799999", + "source_payload_size_bytes >= 0 AND source_payload_size_bytes <= 67108864", + "origin = 'backfill' OR origin = 'import' OR origin = 'live' OR origin = 'repair' OR origin = 'replay'", + "retention_state = 'full' OR retention_state = 'archived' OR retention_state = 'purged'", ] { assert!(sql.contains(required), "V001 physical contract is missing: {required}"); } - assert_eq!(sql.matches("PRIMARY KEY").count(), 4); - assert_eq!(sql.matches("REFERENCES ksp_raw_transactions(signature) ON DELETE RESTRICT").count(), 2); assert!(!sql.contains("compacted")); assert!(!sql.contains("BIGSERIAL")); assert!(!sql.contains("slot BIGINT")); - assert!(sql.contains("network TEXT NOT NULL")); return; } #[test] -fn pre_003_ordered_registry_accepts_v000_prefix_and_full_v001_history() { +fn pre_003_fix_001_ordered_registry_accepts_v000_prefix_and_full_v001_history() { let v000 = super::EMBEDDED_MIGRATIONS[0]; let v001 = super::EMBEDDED_MIGRATIONS[1]; - let v000_checksum = super::migration_checksum(v000.sql); + let v000_checksum = super::migration_checksum(&v000); let prefix = [applied(0, v000.name, v000_checksum.as_str())]; assert_eq!(super::validate_history(&prefix, super::EMBEDDED_MIGRATIONS).ok(), std::option::Option::Some(1)); - let v001_checksum = super::migration_checksum(v001.sql); + let v001_checksum = super::migration_checksum(&v001); let full = [applied(0, v000.name, v000_checksum.as_str()), applied(1, v001.name, v001_checksum.as_str())]; assert_eq!(super::validate_history(&full, super::EMBEDDED_MIGRATIONS).ok(), std::option::Option::Some(2)); return; } #[test] -fn pre_002_registry_rejects_empty_nonzero_gap_and_empty_metadata_entries() { +fn pre_003_fix_001_registry_rejects_empty_nonzero_gap_empty_metadata_and_empty_resources() { let empty: [super::EmbeddedMigration; 0] = []; - let starts_at_one = [embedded(1, "future", "SELECT 1;")]; - let gap = [embedded(0, "bootstrap", "SELECT 0;"), embedded(2, "future", "SELECT 2;")]; - let empty_name = [embedded(0, "", "SELECT 0;")]; - let empty_sql = [embedded(0, "bootstrap", "")]; - for registry in [&empty[..], &starts_at_one[..], &gap[..], &empty_name[..], &empty_sql[..]] { + let starts_at_one = [embedded(1, "future", crate::V000_RESOURCES)]; + let gap = [embedded(0, "bootstrap", crate::V000_RESOURCES), embedded(2, "future", crate::V000_RESOURCES)]; + let empty_name = [embedded(0, "", crate::V000_RESOURCES)]; + let empty_resources = [embedded(0, "bootstrap", &[])]; + for registry in [&empty[..], &starts_at_one[..], &gap[..], &empty_name[..], &empty_resources[..]] { let result = super::validate_embedded_registry(registry); assert_eq!(result.err().map(|value| return value.kind()), std::option::Option::Some(crate::PostgresBackendErrorKind::MigrationMismatch)); } @@ -91,11 +105,11 @@ fn pre_002_registry_rejects_empty_nonzero_gap_and_empty_metadata_entries() { } #[test] -fn pre_003_divergent_missing_or_gapped_history_is_terminal_mismatch() { +fn pre_003_fix_001_divergent_missing_or_gapped_history_is_terminal_mismatch() { let v000 = super::EMBEDDED_MIGRATIONS[0]; let v001 = super::EMBEDDED_MIGRATIONS[1]; - let v000_checksum = super::migration_checksum(v000.sql); - let v001_checksum = super::migration_checksum(v001.sql); + let v000_checksum = super::migration_checksum(&v000); + let v001_checksum = super::migration_checksum(&v001); let wrong_name = [applied(0, "changed", v000_checksum.as_str())]; let wrong_checksum = [applied(0, v000.name, "00")]; let missing: [super::AppliedMigration; 0] = []; @@ -108,16 +122,12 @@ fn pre_003_divergent_missing_or_gapped_history_is_terminal_mismatch() { } #[test] -fn pre_003_newer_history_is_rejected_without_down_migration() { +fn pre_003_fix_001_newer_history_is_rejected_without_down_migration() { let v000 = super::EMBEDDED_MIGRATIONS[0]; let v001 = super::EMBEDDED_MIGRATIONS[1]; - let v000_checksum = super::migration_checksum(v000.sql); - let v001_checksum = super::migration_checksum(v001.sql); - let history = [ - applied(0, v000.name, v000_checksum.as_str()), - applied(1, v001.name, v001_checksum.as_str()), - applied(2, "future", "future-checksum"), - ]; + let v000_checksum = super::migration_checksum(&v000); + let v001_checksum = super::migration_checksum(&v001); + let history = [applied(0, v000.name, v000_checksum.as_str()), applied(1, v001.name, v001_checksum.as_str()), applied(2, "future", "future-checksum")]; let result = super::validate_history(&history, super::EMBEDDED_MIGRATIONS); assert_eq!(result.err().map(|value| return value.kind()), std::option::Option::Some(crate::PostgresBackendErrorKind::SchemaNewer)); return; diff --git a/crates/ksp-store-postgres-lib/unit_tests/schema.rs b/crates/ksp-store-postgres-lib/unit_tests/schema.rs new file mode 100644 index 0000000..6eed2ce --- /dev/null +++ b/crates/ksp-store-postgres-lib/unit_tests/schema.rs @@ -0,0 +1,114 @@ +// file: crates/ksp-store-postgres-lib/unit_tests/schema.rs +// version: 1 + +fn actual_column(name: &str, udt_name: &str, nullable: bool) -> super::ActualColumn { + return super::ActualColumn { + default: std::option::Option::None, + generated: "NEVER".to_owned(), + identity: "NO".to_owned(), + name: name.to_owned(), + nullable, + numeric_precision: std::option::Option::None, + numeric_scale: std::option::Option::None, + udt_name: udt_name.to_owned(), + }; +} + +#[test] +fn pre_003_fix_001_v000_resource_is_relocated_without_changing_legacy_sql() { + assert_eq!(crate::V000_RESOURCES.len(), 1); + let resource = crate::V000_RESOURCES[0]; + assert_eq!(resource.id, "tables/001_ksp_store_schema_migrations.sql"); + assert_eq!( + resource.sql, + "CREATE TABLE ksp_store_schema_migrations (\n version BIGINT PRIMARY KEY,\n name TEXT NOT NULL,\n checksum TEXT NOT NULL,\n applied_at TIMESTAMPTZ NOT NULL\n);\n", + ); + return; +} + +#[test] +fn pre_003_fix_001_v001_resources_are_split_and_idempotent_by_object_family() { + assert_eq!(crate::V001_RESOURCES.len(), 40); + let table_resources = crate::V001_RESOURCES.iter().filter(|resource| return resource.id.starts_with("tables/")).collect::>(); + let constraint_resources = crate::V001_RESOURCES.iter().filter(|resource| return resource.id.starts_with("constraints/")).collect::>(); + let index_resources = crate::V001_RESOURCES.iter().filter(|resource| return resource.id.starts_with("indexes/")).collect::>(); + assert_eq!(table_resources.len(), 4); + assert_eq!(constraint_resources.len(), 35); + assert_eq!(index_resources.len(), 1); + for resource in table_resources { + assert!(resource.sql.contains("CREATE TABLE IF NOT EXISTS")); + assert!(resource.sql.contains("ADD COLUMN IF NOT EXISTS")); + } + for resource in constraint_resources { + assert!(resource.sql.contains("IF NOT EXISTS")); + } + for resource in index_resources { + assert!(resource.sql.contains("CREATE INDEX IF NOT EXISTS")); + } + return; +} + +#[test] +fn pre_003_fix_001_external_extra_columns_are_accepted_only_when_they_cannot_block_ksp_inserts() { + let nullable = actual_column("external_nullable", "text", true); + assert!(nullable.is_non_blocking_extra()); + let mut with_default = actual_column("external_default", "text", true); + with_default.default = std::option::Option::Some("'x'::text".to_owned()); + assert!(!with_default.is_non_blocking_extra()); + let mut identity = actual_column("external_identity", "int8", true); + identity.identity = "YES".to_owned(); + assert!(!identity.is_non_blocking_extra()); + let mut generated = actual_column("external_generated", "text", true); + generated.generated = "ALWAYS".to_owned(); + assert!(!generated.is_non_blocking_extra()); + let blocking = actual_column("external_required", "text", false); + assert!(!blocking.is_non_blocking_extra()); + return; +} + +#[test] +fn pre_003_fix_001_required_column_matching_ignores_integer_precision_but_requires_numeric_20_0() { + let expected_int = super::ColumnContract { + name: "version", + nullable: false, + numeric_precision: std::option::Option::None, + numeric_scale: std::option::Option::None, + udt_name: "int8", + }; + let mut actual_int = actual_column("version", "int8", false); + actual_int.numeric_precision = std::option::Option::Some(64); + actual_int.numeric_scale = std::option::Option::Some(0); + assert!(actual_int.matches(&expected_int)); + let expected_numeric = super::ColumnContract { + name: "slot", + nullable: false, + numeric_precision: std::option::Option::Some(20), + numeric_scale: std::option::Option::Some(0), + udt_name: "numeric", + }; + let mut actual_numeric = actual_column("slot", "numeric", false); + actual_numeric.numeric_precision = std::option::Option::Some(20); + actual_numeric.numeric_scale = std::option::Option::Some(0); + assert!(actual_numeric.matches(&expected_numeric)); + actual_numeric.numeric_precision = std::option::Option::Some(19); + assert!(!actual_numeric.matches(&expected_numeric)); + return; +} + +#[test] +fn pre_003_fix_001_catalog_normalization_and_resource_owned_constraint_definition_are_deterministic() { + let normalized = super::normalize_catalog_sql("CHECK ((slot >= (0)::numeric) AND (slot <= (18446744073709551615)::numeric))"); + assert_eq!(normalized, "checkslot>=0andslot<=18446744073709551615"); + let resource = crate::V001_RESOURCES.iter().find(|resource| return resource.id == "constraints/006_ck_ksp_raw_transactions_slot.sql"); + let resource = match resource { + std::option::Option::Some(value) => value, + std::option::Option::None => { + assert!(false, "V001 slot constraint resource must remain embedded"); + return; + }, + }; + let expected = super::expected_constraint_definition(resource.sql, "ck_ksp_raw_transactions_slot"); + assert_eq!(expected.as_deref(), std::option::Option::Some(normalized.as_str())); + assert_ne!(expected.as_deref(), std::option::Option::Some("checkslot>=0andslot<=10")); + return; +} diff --git a/deltas/0.3.3/pre.003-fix.001.md b/deltas/0.3.3/pre.003-fix.001.md new file mode 100644 index 0000000..0ff1898 --- /dev/null +++ b/deltas/0.3.3/pre.003-fix.001.md @@ -0,0 +1,273 @@ + + + +# Delta `0.3.3-pre.003-fix.001` — Store/PostgreSQL schema compatibility hardening + +## 1. Base et objectif + +Ce correctif s'applique **par-dessus le commit `0.3.3-pre.003` déjà réalisé**. Il ne réécrit ni ne remplace ce commit dans l'historique. + +Le gate opérateur de `pre.002` est vert. Le gate opérateur de `pre.003` n'a pas été fourni avant l'ouverture de ce fix ; aucune commande Cargo du fix n'est donc déclarée PASS dans ce document. + +Objectif du fix : conserver les bonnes garanties fonctionnelles historiques d'inspection/réconciliation du schéma PostgreSQL sans reprendre l'architecture kbot3, et corriger V001 avant l'ouverture du repository `RawTransaction` de `pre.004`. + +Version workspace après application : + +```text +0.3.3-pre.3.fix.1 +``` + +Identifiant de livraison : + +```text +0.3.3-pre.003-fix.001 +``` + +## 2. Décisions corrigées + +### 2.1 Migration logique vs ressources physiques + +V000 et V001 restent des **migrations logiques** versionnées. Les objets physiques ne deviennent pas chacun une version de migration. + +Arborescence retenue : + +```text +crates/ksp-store-postgres-lib/migrations/ + v000_bootstrap/ + tables/ + 001_ksp_store_schema_migrations.sql + v001_raw_transaction/ + tables/ + 001_ksp_store_identity.sql + 002_ksp_raw_transactions.sql + 003_ksp_raw_transaction_observations.sql + 004_ksp_raw_transaction_archive_payloads.sql + constraints/ + 001_...sql -> 035_...sql + indexes/ + 001_ix_ksp_raw_transactions_slot_signature.sql +``` + +V001 contient exactement : + +```text +4 tables +35 contraintes +1 index +40 ressources embedded +``` + +V000 est seulement déplacée : ses bytes restent strictement identiques. + +### 2.2 Checksums + +V000 historique : + +```text +d29068b8c13b9dc0cc9ef6aaadd0fa12d41e0fe4c56541a1118c4bfc846a1450 +``` + +V001 multi-ressources : + +```text +31488cda2f08f3f46c4cdbdbb6c18c243662fada02eac4487040c8735d72cc51 +``` + +Le checksum V001 est calculé sur le préfixe de format du registre puis, dans l'ordre embedded, sur l'identifiant relatif et les bytes exacts de chaque ressource. Le checksum monolithique `pre.003` n'est pas réutilisé. + +Une base ayant réellement enregistré l'ancien checksum V001 du commit `pre.003` n'est jamais réécrite silencieusement : elle doit être réinitialisée si elle est jetable ou réconciliée manuellement. + +### 2.3 `IF NOT EXISTS` n'est pas une preuve de compatibilité + +Les ressources utilisent `CREATE ... IF NOT EXISTS`, `ADD COLUMN IF NOT EXISTS` ou des guards catalogue lorsque cela permet une création additive idempotente. + +Après et avant mutation, le backend introspecte néanmoins le catalogue PostgreSQL. Chaque ressource est classée : + +```text +Compatible +Missing +Incompatible +``` + +Un objet préexistant du bon nom mais de mauvaise définition est donc refusé. + +### 2.4 Contrat physique vérifié + +Le module backend-private `schema` contrôle notamment : + +- tables gérées et nature `BASE TABLE` ; +- colonnes obligatoires, type, nullabilité et absence d'identity/generated sur le contrat KSP ; +- précision/scale `NUMERIC(20,0)` du slot ; +- PK, FK et CHECK ; +- définition canonique des contraintes à partir de leur propre ressource SQL ; +- index partiel `(slot, signature)` hors `purged` ; +- RLS ; +- triggers utilisateur actifs ; +- rewrite rules ; +- contraintes/indexes uniques externes susceptibles de modifier la sémantique de write. + +Les marqueurs COMMENT ne servent pas de preuve de compatibilité et ne sont pas requis. + +### 2.5 Extensions externes compatibles + +KSP ne requiert pas une égalité byte-for-byte du schéma complet avec son propre DDL. Les extensions externes qui sont prouvées non bloquantes restent admises. + +Le correctif tolère notamment : + +- une colonne externe nullable, sans default, identity ni generated expression ; +- un index externe non unique ; +- une contrainte attendue physiquement équivalente même si son nom diffère. + +Le correctif bloque conservativement les extensions susceptibles de modifier/contraindre les writes KSP lorsqu'il ne peut pas en prouver l'innocuité : mauvais type/nullabilité sur une colonne KSP, colonne externe write-blocking, contrainte non équivalente, unique index autonome, trigger actif, rule ou RLS. + +### 2.6 Politiques `schema_autocreate` / `schema_autoupdate` + +`std.store` passe en format V2 et sépare : + +```text +schema_autocreate +schema_autoupdate +``` + +`schema_autocreate` autorise l'initialisation ou l'adoption contrôlée d'une base sans metadata KSP. + +`schema_autoupdate` autorise les migrations pending et les réparations additives sûres de ressources manquantes sur un schéma KSP existant. + +Une incompatibilité exigeant une mutation destructive, ambiguë ou non prouvée sûre reste bloquante même avec `schema_autoupdate=true`; le backend émet uniquement une phase/resource sûre pour permettre l'intervention manuelle. + +Le format V1 reste lisible : `auto_migrate` est mappé vers les deux politiques. Les constructeurs Rust historiques restent également disponibles et mappent leur booléen vers les deux politiques. + +### 2.7 Identité réseau + +Le binding V001 reste sous la même transaction et le même advisory lock. + +Une `ksp_store_identity` absente alors que V001 est déjà enregistrée n'est jamais reconstruite automatiquement, même avec `schema_autoupdate=true`, afin d'interdire tout rebind réseau silencieux. + +## 3. Suppressions explicites après application de l'overlay + +L'extraction d'un zip n'efface pas les anciens fichiers. Après extraction du delta à la racine du dépôt, supprimer explicitement : + +```bash +rm crates/ksp-store-postgres-lib/migrations/V000__bootstrap.sql +rm crates/ksp-store-postgres-lib/migrations/V001__raw_transaction.sql +``` + +Fichiers supprimés : + +- `crates/ksp-store-postgres-lib/migrations/V000__bootstrap.sql` +- `crates/ksp-store-postgres-lib/migrations/V001__raw_transaction.sql` + +## 4. Fichiers modifiés + +- `Cargo.toml` +- `config/examples/std.store.example.json` +- `config/schemas/std.store.schema.json` +- `config/std.store.json` +- `crates/ksp-config-lib/src/store.rs` +- `crates/ksp-config-lib/unit_tests/fixtures/std.store.json` +- `crates/ksp-config-lib/unit_tests/store.rs` +- `crates/ksp-store-lib/src/settings.rs` +- `crates/ksp-store-lib/src/store.rs` +- `crates/ksp-store-lib/unit_tests/settings.rs` +- `crates/ksp-store-postgres-lib/src/lib.rs` +- `crates/ksp-store-postgres-lib/src/migration.rs` +- `crates/ksp-store-postgres-lib/src/runtime.rs` +- `crates/ksp-store-postgres-lib/tests/dependency_boundary.rs` +- `crates/ksp-store-postgres-lib/tests/hardening_completeness.rs` +- `crates/ksp-store-postgres-lib/unit_tests/migration.rs` +- `docs/plans/024-V0_3_3_STORE_POSTGRES_RAW_TRANSACTION_PLAN.md` +- `docs/validation/020-V0_3_3_STORE_POSTGRES_RAW_TRANSACTION.md` + +## 5. Fichiers ajoutés + +- `crates/ksp-store-postgres-lib/migrations/v000_bootstrap/tables/001_ksp_store_schema_migrations.sql` +- `crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/001_pk_ksp_store_identity.sql` +- `crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/002_ck_ksp_store_identity_singleton.sql` +- `crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/003_ck_ksp_store_identity_network.sql` +- `crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/004_pk_ksp_raw_transactions.sql` +- `crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/005_ck_ksp_raw_transactions_signature.sql` +- `crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/006_ck_ksp_raw_transactions_slot.sql` +- `crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/007_ck_ksp_raw_transactions_block_time.sql` +- `crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/008_ck_ksp_raw_transactions_format_id.sql` +- `crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/009_ck_ksp_raw_transactions_format_version.sql` +- `crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/010_ck_ksp_raw_transactions_content_hash.sql` +- `crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/011_ck_ksp_raw_transactions_payload.sql` +- `crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/012_ck_ksp_raw_transactions_retention_state.sql` +- `crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/013_ck_ksp_raw_transactions_payload_state.sql` +- `crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/014_ck_ksp_raw_transactions_purged_block_time.sql` +- `crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/015_pk_ksp_raw_transaction_observations.sql` +- `crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/016_fk_ksp_raw_transaction_observations_transaction.sql` +- `crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/017_ck_ksp_raw_transaction_observations_key.sql` +- `crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/018_ck_ksp_raw_transaction_observations_signature.sql` +- `crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/019_ck_ksp_raw_transaction_observations_provider.sql` +- `crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/020_ck_ksp_raw_transaction_observations_protocol.sql` +- `crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/021_ck_ksp_raw_transaction_observations_method.sql` +- `crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/022_ck_ksp_raw_transaction_observations_origin.sql` +- `crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/023_ck_ksp_raw_transaction_observations_received_at.sql` +- `crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/024_ck_ksp_raw_transaction_observations_capture_session.sql` +- `crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/025_ck_ksp_raw_transaction_observations_commitment.sql` +- `crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/026_ck_ksp_raw_transaction_observations_endpoint.sql` +- `crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/027_ck_ksp_raw_transaction_observations_filter.sql` +- `crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/028_ck_ksp_raw_transaction_observations_observed_at.sql` +- `crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/029_ck_ksp_raw_transaction_observations_time_order.sql` +- `crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/030_ck_ksp_raw_transaction_observations_source_hash.sql` +- `crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/031_ck_ksp_raw_transaction_observations_source_size.sql` +- `crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/032_pk_ksp_raw_transaction_archive_payloads.sql` +- `crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/033_fk_ksp_raw_transaction_archive_payloads_transaction.sql` +- `crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/034_ck_ksp_raw_transaction_archive_payloads_signature.sql` +- `crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/035_ck_ksp_raw_transaction_archive_payloads_payload.sql` +- `crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/indexes/001_ix_ksp_raw_transactions_slot_signature.sql` +- `crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/tables/001_ksp_store_identity.sql` +- `crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/tables/002_ksp_raw_transactions.sql` +- `crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/tables/003_ksp_raw_transaction_observations.sql` +- `crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/tables/004_ksp_raw_transaction_archive_payloads.sql` +- `crates/ksp-store-postgres-lib/src/schema.rs` +- `crates/ksp-store-postgres-lib/unit_tests/schema.rs` +- `deltas/0.3.3/pre.003-fix.001.md` + +## 6. Validation de génération + +Les validations réalisables dans l'environnement de génération doivent être propres avant publication de l'archive : + +```text +python3 scripts/audit_rust_workspace_rules.py +python3 scripts/audit_markdown_tables.py README.md RULES.md ROADMAP.md CHANGELOG.md docs prompts crates deltas/0.3.3 +validation JSON/JSON Schema std.store V1+V2 +inventaire/checksum migration V000/V001 +``` + +L'environnement de génération ne possède pas `cargo`/`rustfmt`. Les gates Cargo ci-dessous restent donc **NON EXÉCUTÉS** jusqu'au passage opérateur. + +## 7. Gate opérateur demandé + +Après extraction de l'overlay, suppression explicite des deux fichiers monolithiques puis application du formatage : + +```bash +cargo fmt --all +python3 scripts/audit_rust_workspace_rules.py +python3 scripts/audit_markdown_tables.py README.md RULES.md ROADMAP.md CHANGELOG.md docs prompts crates deltas/0.3.3 +cargo check --workspace +cargo clippy --workspace --all-targets +cargo test -p ksp-store-api +cargo test -p ksp-store-lib +cargo test -p ksp-store-postgres-lib +cargo test -p ksp-config-lib +cargo check -p ksp-store-lib --no-default-features +``` + +Canaris de suppression : + +```bash +test ! -e crates/ksp-store-postgres-lib/migrations/V000__bootstrap.sql +test ! -e crates/ksp-store-postgres-lib/migrations/V001__raw_transaction.sql +``` + +Le vrai PostgreSQL live du contrat V001, des dérives et des réparations reste volontairement réservé au gate live prévu plus tard dans `0.3.3`; ce fix ne le présente pas comme PASS. + +## 8. Suite + +Après gate opérateur vert de `pre.003-fix.001`, reprendre le séquencement avec : + +```text +0.3.3-pre.004 — mapping SQL privé + lectures RawTransaction/observation/rétention/tombstone +``` diff --git a/docs/plans/024-V0_3_3_STORE_POSTGRES_RAW_TRANSACTION_PLAN.md b/docs/plans/024-V0_3_3_STORE_POSTGRES_RAW_TRANSACTION_PLAN.md index 778b083..fb07d9c 100644 --- a/docs/plans/024-V0_3_3_STORE_POSTGRES_RAW_TRANSACTION_PLAN.md +++ b/docs/plans/024-V0_3_3_STORE_POSTGRES_RAW_TRANSACTION_PLAN.md @@ -1,5 +1,5 @@ - + # Plan `0.3.3` — Store/PostgreSQL RawTransaction vertical slice @@ -159,15 +159,17 @@ SHA-256 = d29068b8c13b9dc0cc9ef6aaadd0fa12d41e0fe4c56541a1118c4bfc846a1450 Le SQL crée uniquement `ksp_store_schema_migrations`. Le moteur stable considère toute migration appliquée `> 0` comme `SchemaNewer`; il doit donc être généralisé avant V001. -Convention figée pour la nouvelle ressource : +Convention initialement retenue en `pre.001`, puis corrigée par `pre.003-fix.001` avant toute tranche repository : V001 reste une migration logique unique (`version = 1`, `name = raw_transaction`) mais ses ressources physiques sont séparées et ordonnées sous : ```text -crates/ksp-store-postgres-lib/migrations/V001__raw_transaction.sql -version = 1 -name = raw_transaction -checksum SHA-256 calculé sur les bytes embedded exacts +crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/ + tables/ + constraints/ + indexes/ ``` +Le checksum logique V001 est calculé de manière déterministe sur l'identifiant relatif et les bytes exacts de chaque ressource embedded dans l'ordre du registre. Cette correction ne transforme pas chaque table/contrainte/index en version de migration autonome. + ### 5.3 Config `std.store` final `config/std.store.json` est `format_version = 1`, profil par défaut `devnet` et contient exactement : @@ -1040,17 +1042,36 @@ Statut matérialisé en `pre.002` : - tests de checksum/inventory/bounds ; - matérialiser `store.postgres_retention_compaction_unsupported`, sans modifier les contrats stables sauf preuve nouvelle d'un blocage réel. -Statut matérialisé en `pre.003` : +Statut du commit `pre.003` avant correctif : - registre réel `[V000, V001]`, version courante `1` ; -- V000 reste byte-identique avec son checksum historique ; -- V001 est embarquée sous `migrations/V001__raw_transaction.sql` avec checksum SHA-256 `6fe57ed0313d2ed295280dd6e49f6d86695d4e4effee2724a25e36db1ea17761` ; -- V001 crée exactement `ksp_store_identity`, `ksp_raw_transactions`, `ksp_raw_transaction_observations`, `ksp_raw_transaction_archive_payloads` et l'index partiel `(slot, signature)` hors `purged` ; -- `slot` reste `NUMERIC(20,0)` et toutes les bornes physiques suivent les invariants `ksp-store-api` sans narrowing ; -- le hook V001 crée puis valide l'identité en `AppliedNow`, et valide seulement en `Existing` ; absence, forme invalide ou réseau différent deviennent un `MigrationMismatch` sûr sans rendre la valeur persistée ; -- `store.postgres_retention_compaction_unsupported` est réservé par valeur dans le backend et la façade, sans reverse dependency ; +- V000 byte-identique avec son checksum historique ; +- V001 monolithique possédait le checksum `6fe57ed0313d2ed295280dd6e49f6d86695d4e4effee2724a25e36db1ea17761` ; +- les quatre tables, contraintes et l'index partiel étaient créés, mais l'historique de migration ne prouvait pas à lui seul la compatibilité physique des objets réellement présents ; +- le hook V001 créait/validait déjà l'identité réseau sans rebind silencieuse ; +- aucune capability/repository `RawTransaction*` n'était encore implémentée. + +### `0.3.3-pre.003-fix.001` — correction du contrat physique + +Le correctif s'applique au commit `pre.003` et remplace la matérialisation SQL sans changer le numéro logique V001 : + +- V000 est déplacée sous `migrations/v000_bootstrap/tables/001_ksp_store_schema_migrations.sql` **byte-identique** ; son checksum historique reste `d29068b8c13b9dc0cc9ef6aaadd0fa12d41e0fe4c56541a1118c4bfc846a1450` ; +- V001 est éclatée en 40 ressources embedded : 4 tables, 35 contraintes, 1 index ; +- le checksum logique V001 devient `31488cda2f08f3f46c4cdbdbb6c18c243662fada02eac4487040c8735d72cc51`, calculé sur les identifiants relatifs et bytes ordonnés des ressources ; +- chaque ressource sait comment être créée de façon additive, mais `IF NOT EXISTS` n'est jamais considéré comme preuve de compatibilité ; +- le backend introspecte les catalogues PostgreSQL et classe chaque ressource `Compatible`, `Missing` ou `Incompatible` ; +- tables, colonnes obligatoires, nullabilité/types, précision `NUMERIC(20,0)`, PK/FK/CHECK, index partiel, RLS, triggers, rules et contraintes externes susceptibles de modifier les writes sont vérifiés ; +- les CHECK sont comparés à la définition canonique dérivée de leur propre ressource SQL, et non à une liste permissive de fragments ; +- une colonne externe est acceptée uniquement lorsque sa présence est prouvée non bloquante pour les writes KSP ; les indexes externes non uniques sont tolérés ; les extensions ambiguës ou write-constraining bloquent `Ready` ; +- `schema_autocreate` contrôle l'initialisation/adoption d'une base sans metadata KSP ; `schema_autoupdate` contrôle les migrations pending et les réparations additives sûres d'un schéma KSP existant ; +- aucune réparation destructive ou ambiguë n'est exécutée automatiquement : le backend bloque avec une erreur/log sûrs pour intervention manuelle ; +- une identité V001 absente sur une migration déjà appliquée n'est jamais recréée automatiquement ; +- Config `std.store` passe au format V2 pour séparer les deux politiques, tout en conservant la lecture V1 où `auto_migrate` mappe les deux valeurs ; +- les fichiers monolithiques `migrations/V000__bootstrap.sql` et `migrations/V001__raw_transaction.sql` sont supprimés explicitement par le delta ; - aucune capability/repository `RawTransaction*` n'est encore implémentée. +Une base PostgreSQL sur laquelle le commit `pre.003` aurait déjà enregistré l'ancien checksum V001 n'est **pas** réécrite silencieusement : elle doit être réinitialisée si elle est jetable, ou réconciliée manuellement. Le correctif préserve ainsi la règle d'immutabilité de l'historique au lieu de masquer une divergence de prerelease. + ### `0.3.3-pre.004` — mapping et lectures - codecs SQL privés ; diff --git a/docs/validation/020-V0_3_3_STORE_POSTGRES_RAW_TRANSACTION.md b/docs/validation/020-V0_3_3_STORE_POSTGRES_RAW_TRANSACTION.md index 1074e5b..6bbeb6e 100644 --- a/docs/validation/020-V0_3_3_STORE_POSTGRES_RAW_TRANSACTION.md +++ b/docs/validation/020-V0_3_3_STORE_POSTGRES_RAW_TRANSACTION.md @@ -1,5 +1,5 @@ - + # Validation `0.3.3` — Store/PostgreSQL RawTransaction vertical slice @@ -11,7 +11,7 @@ Cette validation accompagne : 0.3.3 — Store/PostgreSQL RawTransaction vertical slice ``` -Elle démarre en `0.3.3-pre.001` comme matrice de preuve. Les gates opérateur de `pre.001` et `pre.002` sont verts. Les lignes non encore implémentées restent explicitement `À FAIRE`; elles ne sont pas présentées comme acquises. +Elle démarre en `0.3.3-pre.001` comme matrice de preuve. Les gates opérateur de `pre.001` et `pre.002` sont verts. `pre.003` a été commité avant son gate opérateur ; `pre.003-fix.001` corrige donc sa matérialisation physique par-dessus ce commit. Les lignes non encore implémentées restent explicitement `À FAIRE`; elles ne sont pas présentées comme acquises. ## 2. Baseline stable @@ -206,14 +206,14 @@ Critère : aucune branche ne doit utiliser `as i64`, `as i32`, clamp ou saturati ### 7.1 Bootstrap -| Cas | Résultat attendu | -|-------------------------------------------|------------------------------------------------------------------------| -| V001 appliquée pour la première fois | identity créée avec le réseau runtime dans la transaction de bootstrap | -| réouverture même réseau | succès idempotent | -| réouverture autre réseau | erreur terminale sûre | -| V001 présente mais identity absente | mismatch/corruption, jamais rebind silencieuse | -| identity malformée | erreur terminale sûre | -| migration pending + auto_migrate disabled | backend non prêt, aucune persistence métier | +| Cas | Résultat attendu | +|-----------------------------------------------|------------------------------------------------------------------------| +| V001 appliquée pour la première fois | identity créée avec le réseau runtime dans la transaction de bootstrap | +| réouverture même réseau | succès idempotent | +| réouverture autre réseau | erreur terminale sûre | +| V001 présente mais identity absente | mismatch/corruption, jamais rebind silencieuse | +| identity malformée | erreur terminale sûre | +| migration pending + `schema_autoupdate=false` | backend non prêt, aucune persistence métier | ### 7.2 Pré-I/O capability @@ -446,17 +446,25 @@ Preuves acquises en `pre.002` avant création réelle de V001 : - les migrations pending sont appliquées dans l'ordre sous la transaction et l'advisory lock déjà acquis ; - chaque migration vérifie encore la forme de la metadata avant d'insérer sa ligne d'historique. -Preuves matérialisées en `pre.003` avec la vraie V001 : +Le commit `pre.003` a matérialisé la vraie V001 et son binding réseau, mais son fichier SQL monolithique et le seul contrôle de l'historique ne suffisaient pas à prouver la compatibilité structurelle de la base. Son checksum V001 historique de prerelease était `6fe57ed0313d2ed295280dd6e49f6d86695d4e4effee2724a25e36db1ea17761`. -- registre réel `[V000, V001]` et version courante dérivée `1` ; -- historique V000 seul reconnu comme préfixe exact avec V001 pending ; -- historique V000+V001 exact reconnu comme complet ; -- checksum V001 figé à `6fe57ed0313d2ed295280dd6e49f6d86695d4e4effee2724a25e36db1ea17761` ; -- binding réseau réel branché dans le hook `StoreIdentity` ; -- inventaire physique exact : identity, canonical, observations, archive, index partiel ; -- bornes SQL statiquement alignées sur `u64`, `u32`, `RawTimestamp`, payload 16 MiB et source payload 64 MiB. +Preuves statiques ajoutées par `pre.003-fix.001` : -L'application V000 -> V001, le rollback réel du hook, la réouverture même/autre réseau et la divergence de checksum sur PostgreSQL réel restent à prouver dans le gate live dédié `pre.009`; ils ne sont pas déclarés PASS dans cette tranche. +- V000 déplacée byte-identique, checksum toujours `d29068b8c13b9dc0cc9ef6aaadd0fa12d41e0fe4c56541a1118c4bfc846a1450` ; +- V001 logique unique composée de 40 ressources ordonnées : 4 tables, 35 contraintes et 1 index ; +- checksum V001 multi-ressources `31488cda2f08f3f46c4cdbdbb6c18c243662fada02eac4487040c8735d72cc51` ; +- anciens fichiers monolithiques absents après application du delta ; +- `CREATE ... IF NOT EXISTS`/guards servent uniquement à la création additive, jamais à déclarer un objet compatible ; +- inspection catalogue indépendante de l'historique avec états `Compatible`, `Missing`, `Incompatible` ; +- contraintes attendues comparées exactement à une définition canonique issue de la ressource SQL correspondante ; +- colonne externe nullable sans default/identity/generated acceptée comme extension non bloquante ; +- index externe non unique accepté ; +- mauvais type/nullabilité sur colonne obligatoire, extra colonne write-blocking, contrainte externe non équivalente, unique index autonome, trigger actif, rule ou RLS bloquent `Ready` ; +- ressource obligatoire manquante réparable seulement lorsque la politique le permet et que la réparation est additive/sûre ; +- identité V001 manquante sur un historique déjà appliqué reste terminale et n'est jamais auto-recréée ; +- `std.store` V2 sépare `schema_autocreate` et `schema_autoupdate`, avec lecture V1 conservée et mapping de `auto_migrate` vers les deux valeurs. + +L'application V000 -> V001, les réparations réelles, le rollback du hook, la réouverture même/autre réseau et la matrice de drift sur PostgreSQL réel restent à prouver dans le gate live dédié `pre.009`; elles ne sont pas déclarées PASS par le correctif statique. Une base ayant déjà enregistré l'ancien checksum V001 de `pre.003` doit être réinitialisée ou réconciliée manuellement ; le moteur ne réécrit pas son historique silencieusement. ### 17.2 Binding réseau atomique @@ -522,11 +530,21 @@ cap 500/1000 dans Store pagination ### `pre.003` -- V001 + tables/constraints/indexes : PASS statique ; +- V001 + tables/constraints/indexes : matérialisé puis corrigé par `pre.003-fix.001` ; - binding réseau hook `AppliedNow`/`Existing` : PASS statique ; -- checksum/inventory/bounds tests : PASS statique ; - code `postgres_retention_compaction_unsupported` backend + façade : PASS ; -- preuve PostgreSQL réelle V001 : différée à `pre.009`. +- gate opérateur `pre.003` : NON FOURNI avant le correctif. + +### `pre.003-fix.001` + +- arborescence versionnée et 40 ressources V001 : PASS statique ; +- V000 byte-identique/checksum historique : PASS statique ; +- checksum/inventory/order V001 multi-ressources : PASS statique ; +- contrat catalogue `Compatible/Missing/Incompatible` : PASS statique ; +- politiques `schema_autocreate` / `schema_autoupdate` + compatibilité Config V1 : PASS statique ; +- extensions externes non bloquantes tolérées, extensions write-blocking refusées : PASS statique ; +- preuve PostgreSQL réelle V001/drift/réparation : différée à `pre.009` ; +- gate Cargo opérateur : À EXÉCUTER. ### `pre.004`