Sasedev/khadhroony-solana-project
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

v0.3.3-pre.003-fix.001

Browse Source
This commit is contained in:
SinuS von SifriduS
2026-08-30 10:21:48 +02:00
parent 61bf7ba468
commit c17e78c6a8
64 changed files with 2890 additions and 424 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
Cargo.toml
View File

@@ -6,7 +6,7 @@ resolver = "3"
members = ["crates/ksp-app-config-desk", "crates/ksp-app-solprices-desk", "crates/ksp-app-wallet-desk", "crates/ksp-config-lib", "crates/ksp-core-lib", "crates/ksp-interface-lib", "crates/ksp-logging-lib", "crates/ksp-offchain-transport-lib", "crates/ksp-onchain-transport-lib", "crates/ksp-program-api", "crates/ksp-store-api", "crates/ksp-store-lib", "crates/ksp-store-postgres-lib", "crates/ksp-wallet-lib"]
[workspace.package]
version = "0.3.3-pre.3"
version = "0.3.3-pre.3.fix.1"
edition = "2024"
license = "MIT"
repository = "https://git.sasedev.com/Sasedev/khadhroony-solana-project"

11
config/examples/std.store.example.json
View File

@@ -1,5 +1,5 @@
{
"format_version": 1,
"format_version": 2,
"default_profile": "devnet",
"profiles": [
{
@@ -19,7 +19,8 @@
"mode": "verify_full"
},
"bootstrap": {
"auto_migrate": true,
"schema_autocreate": true,
"schema_autoupdate": true,
"migration_timeout_ms": 30000,
"migration_lock_timeout_ms": 10000
},
@@ -43,7 +44,8 @@
"mode": "verify_full"
},
"bootstrap": {
"auto_migrate": true,
"schema_autocreate": true,
"schema_autoupdate": true,
"migration_timeout_ms": 30000,
"migration_lock_timeout_ms": 10000
},
@@ -67,7 +69,8 @@
"mode": "verify_full"
},
"bootstrap": {
"auto_migrate": true,
"schema_autocreate": true,
"schema_autoupdate": true,
"migration_timeout_ms": 30000,
"migration_lock_timeout_ms": 10000
},

131
config/schemas/std.store.schema.json
View File

@@ -1,6 +1,6 @@
{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"$id": "urn:ksp:schema:std.store:v1",
"$id": "urn:ksp:schema:std.store:v2",
"title": "KSP standard Store configuration",
"type": "object",
"additionalProperties": false,
@@ -11,19 +11,45 @@
],
"properties": {
"format_version": {
"const": 1
"enum": [
1,
2
]
},
"default_profile": {
"$ref": "#/$defs/profileId"
},
"profiles": {
"type": "array",
"minItems": 1,
"items": {
"$ref": "#/$defs/profile"
}
"minItems": 1
}
},
"oneOf": [
{
"properties": {
"format_version": {
"const": 1
},
"profiles": {
"items": {
"$ref": "#/$defs/profileV1"
}
}
}
},
{
"properties": {
"format_version": {
"const": 2
},
"profiles": {
"items": {
"$ref": "#/$defs/profileV2"
}
}
}
}
],
"$defs": {
"profileId": {
"type": "string",
@@ -40,7 +66,7 @@
"minimum": 100,
"maximum": 60000
},
"profile": {
"profileV1": {
"type": "object",
"additionalProperties": false,
"required": [
@@ -60,11 +86,35 @@
"const": "postgres"
},
"postgres": {
"$ref": "#/$defs/postgres"
"$ref": "#/$defs/postgresV1"
}
}
},
"postgres": {
"profileV2": {
"type": "object",
"additionalProperties": false,
"required": [
"profile_id",
"network",
"backend",
"postgres"
],
"properties": {
"profile_id": {
"$ref": "#/$defs/profileId"
},
"network": {
"$ref": "#/$defs/networkId"
},
"backend": {
"const": "postgres"
},
"postgres": {
"$ref": "#/$defs/postgresV2"
}
}
},
"postgresV1": {
"type": "object",
"additionalProperties": false,
"required": [
@@ -86,7 +136,38 @@
"$ref": "#/$defs/tls"
},
"bootstrap": {
"$ref": "#/$defs/bootstrap"
"$ref": "#/$defs/bootstrapV1"
},
"shutdown_timeout_ms": {
"type": "integer",
"minimum": 100,
"maximum": 30000
}
}
},
"postgresV2": {
"type": "object",
"additionalProperties": false,
"required": [
"connection_uri",
"pool",
"tls",
"bootstrap",
"shutdown_timeout_ms"
],
"properties": {
"connection_uri": {
"type": "string",
"minLength": 1
},
"pool": {
"$ref": "#/$defs/pool"
},
"tls": {
"$ref": "#/$defs/tls"
},
"bootstrap": {
"$ref": "#/$defs/bootstrapV2"
},
"shutdown_timeout_ms": {
"type": "integer",
@@ -140,7 +221,7 @@
}
}
},
"bootstrap": {
"bootstrapV1": {
"type": "object",
"additionalProperties": false,
"required": [
@@ -163,6 +244,34 @@
"maximum": 120000
}
}
},
"bootstrapV2": {
"type": "object",
"additionalProperties": false,
"required": [
"schema_autocreate",
"schema_autoupdate",
"migration_timeout_ms",
"migration_lock_timeout_ms"
],
"properties": {
"schema_autocreate": {
"type": "boolean"
},
"schema_autoupdate": {
"type": "boolean"
},
"migration_timeout_ms": {
"type": "integer",
"minimum": 1000,
"maximum": 300000
},
"migration_lock_timeout_ms": {
"type": "integer",
"minimum": 100,
"maximum": 120000
}
}
}
}
}

11
config/std.store.json
View File

@@ -1,5 +1,5 @@
{
"format_version": 1,
"format_version": 2,
"default_profile": "devnet",
"profiles": [
{
@@ -19,7 +19,8 @@
"mode": "verify_full"
},
"bootstrap": {
"auto_migrate": true,
"schema_autocreate": true,
"schema_autoupdate": true,
"migration_timeout_ms": 30000,
"migration_lock_timeout_ms": 10000
},
@@ -43,7 +44,8 @@
"mode": "verify_full"
},
"bootstrap": {
"auto_migrate": true,
"schema_autocreate": true,
"schema_autoupdate": true,
"migration_timeout_ms": 30000,
"migration_lock_timeout_ms": 10000
},
@@ -67,7 +69,8 @@
"mode": "verify_full"
},
"bootstrap": {
"auto_migrate": true,
"schema_autocreate": true,
"schema_autoupdate": true,
"migration_timeout_ms": 30000,
"migration_lock_timeout_ms": 10000
},

41
crates/ksp-config-lib/src/store.rs
View File

@@ -1,5 +1,5 @@
// file: crates/ksp-config-lib/src/store.rs
// version: 2
// version: 3
/// Effective standard Store configuration mapped to `ksp_store_lib::StoreSettings`.
pub struct ResolvedStoreConfig {
@@ -153,9 +153,11 @@ struct EffectivePostgresTlsSource {
#[derive(serde::Deserialize)]
#[serde(deny_unknown_fields)]
struct EffectivePostgresBootstrapSource {
auto_migrate: bool,
auto_migrate: std::option::Option<bool>,
migration_lock_timeout_ms: u64,
migration_timeout_ms: u64,
schema_autocreate: std::option::Option<bool>,
schema_autoupdate: std::option::Option<bool>,
}
fn resolve_store_profile(profile: &crate::ResolvedConfigProfile, environment: &crate::ConfigEnvironment) -> ksp_core_lib::Result<ResolvedStoreConfig> {
@@ -178,7 +180,7 @@ fn resolve_store_profile(profile: &crate::ResolvedConfigProfile, environment: &c
);
},
};
if source.format_version != 1 {
if source.format_version != 1 && source.format_version != 2 {
return std::result::Result::Err(effective_error(profile, "effective Store format_version is unsupported"));
}
if source.profile_id != profile.profile_id() {
@@ -199,11 +201,34 @@ fn resolve_store_profile(profile: &crate::ResolvedConfigProfile, environment: &c
std::time::Duration::from_millis(source.postgres.pool.create_timeout_ms),
std::time::Duration::from_millis(source.postgres.pool.recycle_timeout_ms),
);
let bootstrap = ksp_store_lib::PostgresBootstrapSettings::new(
source.postgres.bootstrap.auto_migrate,
std::time::Duration::from_millis(source.postgres.bootstrap.migration_timeout_ms),
std::time::Duration::from_millis(source.postgres.bootstrap.migration_lock_timeout_ms),
);
let bootstrap = match source.format_version {
1 => {
let auto_migrate =
match (source.postgres.bootstrap.auto_migrate, source.postgres.bootstrap.schema_autocreate, source.postgres.bootstrap.schema_autoupdate) {
(std::option::Option::Some(value), std::option::Option::None, std::option::Option::None) => value,
_ => return std::result::Result::Err(effective_error(profile, "effective Store V1 bootstrap policy is invalid")),
};
ksp_store_lib::PostgresBootstrapSettings::new(
auto_migrate,
std::time::Duration::from_millis(source.postgres.bootstrap.migration_timeout_ms),
std::time::Duration::from_millis(source.postgres.bootstrap.migration_lock_timeout_ms),
)
},
2 => {
let (schema_autocreate, schema_autoupdate) =
match (source.postgres.bootstrap.auto_migrate, source.postgres.bootstrap.schema_autocreate, source.postgres.bootstrap.schema_autoupdate) {
(std::option::Option::None, std::option::Option::Some(autocreate), std::option::Option::Some(autoupdate)) => (autocreate, autoupdate),
_ => return std::result::Result::Err(effective_error(profile, "effective Store V2 bootstrap policy is invalid")),
};
ksp_store_lib::PostgresBootstrapSettings::with_schema_policy(
schema_autocreate,
schema_autoupdate,
std::time::Duration::from_millis(source.postgres.bootstrap.migration_timeout_ms),
std::time::Duration::from_millis(source.postgres.bootstrap.migration_lock_timeout_ms),
)
},
_ => return std::result::Result::Err(effective_error(profile, "effective Store format_version is unsupported")),
};
let network = ksp_store_lib::RawNetworkId::new(source.network);
let network = match network {
std::result::Result::Ok(value) => value,

11
crates/ksp-config-lib/unit_tests/fixtures/std.store.json
View File

@@ -1,5 +1,5 @@
{
"format_version": 1,
"format_version": 2,
"default_profile": "devnet",
"profiles": [
{
@@ -19,7 +19,8 @@
"mode": "verify_full"
},
"bootstrap": {
"auto_migrate": true,
"schema_autocreate": true,
"schema_autoupdate": true,
"migration_timeout_ms": 30000,
"migration_lock_timeout_ms": 10000
},
@@ -43,7 +44,8 @@
"mode": "verify_full"
},
"bootstrap": {
"auto_migrate": true,
"schema_autocreate": true,
"schema_autoupdate": true,
"migration_timeout_ms": 30000,
"migration_lock_timeout_ms": 10000
},
@@ -67,7 +69,8 @@
"mode": "verify_full"
},
"bootstrap": {
"auto_migrate": true,
"schema_autocreate": true,
"schema_autoupdate": true,
"migration_timeout_ms": 30000,
"migration_lock_timeout_ms": 10000
},

88
crates/ksp-config-lib/unit_tests/store.rs
View File

@@ -1,5 +1,5 @@
// file: crates/ksp-config-lib/unit_tests/store.rs
// version: 2
// version: 3
#[test]
fn committed_store_profile_maps_exact_runtime_settings_and_secret_fallback() {
@@ -32,6 +32,8 @@ fn committed_store_profile_maps_exact_runtime_settings_and_secret_fallback() {
assert_eq!(postgres.pool().recycle_timeout(), std::time::Duration::from_millis(5_000));
assert_eq!(postgres.tls_mode(), ksp_store_lib::PostgresTlsMode::VerifyFull);
assert!(postgres.bootstrap().auto_migrate());
assert!(postgres.bootstrap().schema_autocreate());
assert!(postgres.bootstrap().schema_autoupdate());
assert_eq!(postgres.bootstrap().migration_timeout(), std::time::Duration::from_millis(30_000));
assert_eq!(postgres.bootstrap().migration_lock_timeout(), std::time::Duration::from_millis(10_000));
}
@@ -140,6 +142,90 @@ fn named_store_targets_select_one_network_and_database_without_runtime_multiplex
return;
}
#[test]
fn pre_003_fix_001_v1_auto_migrate_remains_backward_readable_and_maps_both_schema_policies() {
for auto_migrate in [false, true] {
let fixture = tempfile::tempdir();
assert!(fixture.is_ok());
let fixture = match fixture {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return,
};
let source = committed_document_value();
let mut source = match source {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return,
};
source["format_version"] = serde_json::Value::from(1);
let profiles = source.get_mut("profiles").and_then(serde_json::Value::as_array_mut);
if let std::option::Option::Some(profiles) = profiles {
for profile in profiles {
let bootstrap = profile.pointer_mut("/postgres/bootstrap").and_then(serde_json::Value::as_object_mut);
if let std::option::Option::Some(bootstrap) = bootstrap {
bootstrap.remove("schema_autocreate");
bootstrap.remove("schema_autoupdate");
bootstrap.insert("auto_migrate".to_owned(), serde_json::Value::Bool(auto_migrate));
}
}
}
let engine = fixture_engine_with_document(fixture.path(), &source);
assert!(engine.is_ok());
let engine = match engine {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return,
};
let environment = crate::ConfigEnvironment::from_maps(std::collections::BTreeMap::new(), std::collections::BTreeMap::new());
let resolved = engine.load_resolved_store_config(std::option::Option::None, &environment);
assert!(resolved.is_ok(), "Store Config V1 compatibility mapping failed: {resolved:?}");
if let std::result::Result::Ok(resolved) = resolved
&& let ksp_store_lib::StoreBackendSettings::Postgres(postgres) = resolved.settings().backend()
{
assert_eq!(postgres.bootstrap().schema_autocreate(), auto_migrate);
assert_eq!(postgres.bootstrap().schema_autoupdate(), auto_migrate);
assert_eq!(postgres.bootstrap().auto_migrate(), auto_migrate);
}
}
return;
}
#[test]
fn pre_003_fix_001_v2_schema_creation_and_update_policies_map_independently() {
let fixture = tempfile::tempdir();
assert!(fixture.is_ok());
let fixture = match fixture {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return,
};
let source = committed_document_value();
let mut source = match source {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return,
};
let profiles = source.get_mut("profiles").and_then(serde_json::Value::as_array_mut);
if let std::option::Option::Some(profiles) = profiles
&& let std::option::Option::Some(profile) = profiles.first_mut()
{
profile["postgres"]["bootstrap"]["schema_autocreate"] = serde_json::Value::Bool(false);
profile["postgres"]["bootstrap"]["schema_autoupdate"] = serde_json::Value::Bool(true);
}
let engine = fixture_engine_with_document(fixture.path(), &source);
assert!(engine.is_ok());
let engine = match engine {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => return,
};
let environment = crate::ConfigEnvironment::from_maps(std::collections::BTreeMap::new(), std::collections::BTreeMap::new());
let resolved = engine.load_resolved_store_config(std::option::Option::None, &environment);
assert!(resolved.is_ok(), "Store Config V2 split schema policy mapping failed: {resolved:?}");
if let std::result::Result::Ok(resolved) = resolved
&& let ksp_store_lib::StoreBackendSettings::Postgres(postgres) = resolved.settings().backend()
{
assert!(!postgres.bootstrap().schema_autocreate());
assert!(postgres.bootstrap().schema_autoupdate());
}
return;
}
fn committed_engine() -> ksp_core_lib::Result<crate::ConfigDocumentEngine> {
let workspace = workspace_root();
let bootstrap = crate::ConfigBootstrapOptions::from_paths(workspace.join("config"), workspace.join("config/schemas"));

42
crates/ksp-store-lib/src/settings.rs
View File

@@ -1,5 +1,5 @@
// file: crates/ksp-store-lib/src/settings.rs
// version: 3
// version: 4
const DEFAULT_CONNECT_TIMEOUT_MS: u64 = 10_000;
const DEFAULT_MAX_CONNECTIONS: u32 = 8;
@@ -153,22 +153,49 @@ impl std::default::Default for PostgresPoolSettings {
/// Bounded PostgreSQL bootstrap settings owned by the Store facade.
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub struct PostgresBootstrapSettings {
auto_migrate: bool,
migration_lock_timeout: std::time::Duration,
migration_timeout: std::time::Duration,
schema_autocreate: bool,
schema_autoupdate: bool,
}
impl PostgresBootstrapSettings {
/// Creates explicit bootstrap behavior and migration deadlines.
/// Creates bootstrap settings using the legacy single migration switch for source compatibility.
///
/// The supplied value is mapped to both schema auto-creation and schema auto-update. New code should prefer
/// [`Self::with_schema_policy`] when these policies need to differ.
#[must_use]
pub const fn new(auto_migrate: bool, migration_timeout: std::time::Duration, migration_lock_timeout: std::time::Duration) -> Self {
return Self { auto_migrate, migration_lock_timeout, migration_timeout };
return Self::with_schema_policy(auto_migrate, auto_migrate, migration_timeout, migration_lock_timeout);
}
/// Returns whether pending KSP-owned migrations may be applied during Store opening.
/// Creates explicit schema creation/update policy and migration deadlines.
#[must_use]
pub const fn with_schema_policy(
schema_autocreate: bool,
schema_autoupdate: bool,
migration_timeout: std::time::Duration,
migration_lock_timeout: std::time::Duration,
) -> Self {
return Self { migration_lock_timeout, migration_timeout, schema_autocreate, schema_autoupdate };
}
/// Returns the legacy pending-migration switch, mapped to the schema auto-update policy.
#[must_use]
pub const fn auto_migrate(&self) -> bool {
return self.auto_migrate;
return self.schema_autoupdate;
}
/// Returns whether an absent KSP-managed schema may be created or adopted during Store opening.
#[must_use]
pub const fn schema_autocreate(&self) -> bool {
return self.schema_autocreate;
}
/// Returns whether pending migrations and safe additive schema repairs may be applied during Store opening.
#[must_use]
pub const fn schema_autoupdate(&self) -> bool {
return self.schema_autoupdate;
}
/// Returns the bounded wait allowed for the private PostgreSQL migration lock.
@@ -205,7 +232,8 @@ impl PostgresBootstrapSettings {
impl std::default::Default for PostgresBootstrapSettings {
fn default() -> Self {
return Self::new(
return Self::with_schema_policy(
true,
true,
std::time::Duration::from_millis(DEFAULT_MIGRATION_TIMEOUT_MS),
std::time::Duration::from_millis(DEFAULT_MIGRATION_LOCK_TIMEOUT_MS),

7
crates/ksp-store-lib/src/store.rs
View File

@@ -1,5 +1,5 @@
// file: crates/ksp-store-lib/src/store.rs
// version: 4
// version: 5
/// Opaque common Store runtime facade.
///
@@ -134,7 +134,7 @@ async fn open_postgres(
crate::PostgresTlsMode::Disabled => ksp_store_postgres_lib::PostgresBackendTlsMode::Disabled,
crate::PostgresTlsMode::VerifyFull => ksp_store_postgres_lib::PostgresBackendTlsMode::VerifyFull,
};
let backend_settings = ksp_store_postgres_lib::PostgresBackendSettings::new(
let backend_settings = ksp_store_postgres_lib::PostgresBackendSettings::with_schema_policy(
network.clone(),
settings.connection_uri(),
pool.max_connections(),
@@ -143,7 +143,8 @@ async fn open_postgres(
pool.create_timeout(),
pool.recycle_timeout(),
tls_mode,
bootstrap.auto_migrate(),
bootstrap.schema_autocreate(),
bootstrap.schema_autoupdate(),
bootstrap.migration_timeout(),
bootstrap.migration_lock_timeout(),
);

8
crates/ksp-store-lib/unit_tests/settings.rs
View File

@@ -1,5 +1,5 @@
// file: crates/ksp-store-lib/unit_tests/settings.rs
// version: 3
// version: 4
fn valid_network() -> crate::RawNetworkId {
return match crate::RawNetworkId::new("devnet") {
@@ -27,6 +27,12 @@ fn defaults_match_the_pre_001_runtime_bounds() {
assert_eq!(pool.recycle_timeout(), std::time::Duration::from_millis(5_000));
let bootstrap = crate::PostgresBootstrapSettings::default();
assert!(bootstrap.auto_migrate());
assert!(bootstrap.schema_autocreate());
assert!(bootstrap.schema_autoupdate());
let split_policy =
crate::PostgresBootstrapSettings::with_schema_policy(false, true, std::time::Duration::from_millis(30_000), std::time::Duration::from_millis(10_000));
assert!(!split_policy.schema_autocreate());
assert!(split_policy.schema_autoupdate());
assert_eq!(bootstrap.migration_timeout(), std::time::Duration::from_millis(30_000));
assert_eq!(bootstrap.migration_lock_timeout(), std::time::Duration::from_millis(10_000));
let store = crate::StoreSettings::with_default_shutdown(valid_network(), crate::StoreBackendSettings::Postgres(valid_postgres_settings()));

133
crates/ksp-store-postgres-lib/migrations/V001__raw_transaction.sql
View File

@@ -1,133 +0,0 @@
CREATE TABLE ksp_store_identity (
singleton SMALLINT PRIMARY KEY,
network TEXT NOT NULL,
CONSTRAINT ck_ksp_store_identity_singleton CHECK (singleton = 1),
CONSTRAINT ck_ksp_store_identity_network CHECK (
octet_length(network) BETWEEN 1 AND 128
AND network ~ '^[A-Za-z0-9_.:-]+$'
)
);
CREATE TABLE ksp_raw_transactions (
signature BYTEA PRIMARY KEY,
slot NUMERIC(20, 0) NOT NULL,
block_time_unix_millis BIGINT NULL,
format_id TEXT NOT NULL,
format_version BIGINT NOT NULL,
content_hash BYTEA NOT NULL,
payload BYTEA NULL,
retention_state TEXT NOT NULL,
CONSTRAINT ck_ksp_raw_transactions_signature CHECK (octet_length(signature) = 64),
CONSTRAINT ck_ksp_raw_transactions_slot CHECK (slot BETWEEN 0 AND 18446744073709551615),
CONSTRAINT ck_ksp_raw_transactions_block_time CHECK (
block_time_unix_millis IS NULL
OR block_time_unix_millis BETWEEN 0 AND 253402300799999
),
CONSTRAINT ck_ksp_raw_transactions_format_id CHECK (
octet_length(format_id) BETWEEN 1 AND 128
AND format_id ~ '^[A-Za-z0-9_.:-]+$'
),
CONSTRAINT ck_ksp_raw_transactions_format_version CHECK (format_version BETWEEN 1 AND 4294967295),
CONSTRAINT ck_ksp_raw_transactions_content_hash CHECK (octet_length(content_hash) = 32),
CONSTRAINT ck_ksp_raw_transactions_payload CHECK (
payload IS NULL
OR octet_length(payload) BETWEEN 1 AND 16777216
),
CONSTRAINT ck_ksp_raw_transactions_retention_state CHECK (retention_state IN ('full', 'archived', 'purged')),
CONSTRAINT ck_ksp_raw_transactions_payload_state CHECK (
(retention_state = 'full' AND payload IS NOT NULL)
OR (retention_state IN ('archived', 'purged') AND payload IS NULL)
),
CONSTRAINT ck_ksp_raw_transactions_purged_block_time CHECK (
retention_state <> 'purged'
OR block_time_unix_millis IS NULL
)
);
CREATE TABLE ksp_raw_transaction_observations (
observation_key BYTEA PRIMARY KEY,
transaction_signature BYTEA NOT NULL REFERENCES ksp_raw_transactions(signature) ON DELETE RESTRICT,
provider TEXT NOT NULL,
protocol TEXT NOT NULL,
acquisition_method TEXT NOT NULL,
origin TEXT NOT NULL,
received_at_unix_millis BIGINT NOT NULL,
capture_session_id TEXT NULL,
commitment TEXT NULL,
endpoint_id TEXT NULL,
filter_id TEXT NULL,
observed_at_unix_millis BIGINT NULL,
source_payload_hash BYTEA NULL,
source_payload_size_bytes BIGINT NULL,
CONSTRAINT ck_ksp_raw_transaction_observations_key CHECK (octet_length(observation_key) = 32),
CONSTRAINT ck_ksp_raw_transaction_observations_signature CHECK (octet_length(transaction_signature) = 64),
CONSTRAINT ck_ksp_raw_transaction_observations_provider CHECK (
octet_length(provider) BETWEEN 1 AND 128
AND provider ~ '^[A-Za-z0-9_.:-]+$'
),
CONSTRAINT ck_ksp_raw_transaction_observations_protocol CHECK (
octet_length(protocol) BETWEEN 1 AND 128
AND protocol ~ '^[A-Za-z0-9_.:-]+$'
),
CONSTRAINT ck_ksp_raw_transaction_observations_method CHECK (
octet_length(acquisition_method) BETWEEN 1 AND 128
AND acquisition_method ~ '^[A-Za-z0-9_.:-]+$'
),
CONSTRAINT ck_ksp_raw_transaction_observations_origin CHECK (origin IN ('backfill', 'import', 'live', 'repair', 'replay')),
CONSTRAINT ck_ksp_raw_transaction_observations_received_at CHECK (received_at_unix_millis BETWEEN 0 AND 253402300799999),
CONSTRAINT ck_ksp_raw_transaction_observations_capture_session CHECK (
capture_session_id IS NULL
OR (
octet_length(capture_session_id) BETWEEN 1 AND 128
AND capture_session_id ~ '^[A-Za-z0-9_.:-]+$'
)
),
CONSTRAINT ck_ksp_raw_transaction_observations_commitment CHECK (
commitment IS NULL
OR (
octet_length(commitment) BETWEEN 1 AND 128
AND commitment ~ '^[A-Za-z0-9_.:-]+$'
)
),
CONSTRAINT ck_ksp_raw_transaction_observations_endpoint CHECK (
endpoint_id IS NULL
OR (
octet_length(endpoint_id) BETWEEN 1 AND 128
AND endpoint_id ~ '^[A-Za-z0-9_.:-]+$'
)
),
CONSTRAINT ck_ksp_raw_transaction_observations_filter CHECK (
filter_id IS NULL
OR (
octet_length(filter_id) BETWEEN 1 AND 128
AND filter_id ~ '^[A-Za-z0-9_.:-]+$'
)
),
CONSTRAINT ck_ksp_raw_transaction_observations_observed_at CHECK (
observed_at_unix_millis IS NULL
OR observed_at_unix_millis BETWEEN 0 AND 253402300799999
),
CONSTRAINT ck_ksp_raw_transaction_observations_time_order CHECK (
observed_at_unix_millis IS NULL
OR observed_at_unix_millis <= received_at_unix_millis
),
CONSTRAINT ck_ksp_raw_transaction_observations_source_hash CHECK (
source_payload_hash IS NULL
OR octet_length(source_payload_hash) = 32
),
CONSTRAINT ck_ksp_raw_transaction_observations_source_size CHECK (
source_payload_size_bytes IS NULL
OR source_payload_size_bytes BETWEEN 0 AND 67108864
)
);
CREATE TABLE ksp_raw_transaction_archive_payloads (
signature BYTEA PRIMARY KEY REFERENCES ksp_raw_transactions(signature) ON DELETE RESTRICT,
payload BYTEA NOT NULL,
CONSTRAINT ck_ksp_raw_transaction_archive_payloads_signature CHECK (octet_length(signature) = 64),
CONSTRAINT ck_ksp_raw_transaction_archive_payloads_payload CHECK (octet_length(payload) BETWEEN 1 AND 16777216)
);
CREATE INDEX ix_ksp_raw_transactions_slot_signature
ON ksp_raw_transactions (slot, signature)
WHERE retention_state <> 'purged';

0
crates/ksp-store-postgres-lib/migrations/V000__bootstrap.sql → crates/ksp-store-postgres-lib/migrations/v000_bootstrap/tables/001_ksp_store_schema_migrations.sql
View File

12
crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/001_pk_ksp_store_identity.sql Normal file
View File

@@ -0,0 +1,12 @@
DO $ksp$
BEGIN
IF NOT EXISTS (
SELECT 1
FROM pg_constraint
WHERE conname = 'pk_ksp_store_identity'
AND conrelid = to_regclass('ksp_store_identity')
) THEN
ALTER TABLE ksp_store_identity ADD CONSTRAINT pk_ksp_store_identity PRIMARY KEY (singleton);
END IF;
END
$ksp$;

12
crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/002_ck_ksp_store_identity_singleton.sql Normal file
View File

@@ -0,0 +1,12 @@
DO $ksp$
BEGIN
IF NOT EXISTS (
SELECT 1
FROM pg_constraint
WHERE conname = 'ck_ksp_store_identity_singleton'
AND conrelid = to_regclass('ksp_store_identity')
) THEN
ALTER TABLE ksp_store_identity ADD CONSTRAINT ck_ksp_store_identity_singleton CHECK (singleton = 1);
END IF;
END
$ksp$;

15
crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/003_ck_ksp_store_identity_network.sql Normal file
View File

@@ -0,0 +1,15 @@
DO $ksp$
BEGIN
IF NOT EXISTS (
SELECT 1
FROM pg_constraint
WHERE conname = 'ck_ksp_store_identity_network'
AND conrelid = to_regclass('ksp_store_identity')
) THEN
ALTER TABLE ksp_store_identity ADD CONSTRAINT ck_ksp_store_identity_network CHECK (
octet_length(network) >= 1 AND octet_length(network) <= 128
AND network ~ '^[A-Za-z0-9_.:-]+$'
);
END IF;
END
$ksp$;

12
crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/004_pk_ksp_raw_transactions.sql Normal file
View File

@@ -0,0 +1,12 @@
DO $ksp$
BEGIN
IF NOT EXISTS (
SELECT 1
FROM pg_constraint
WHERE conname = 'pk_ksp_raw_transactions'
AND conrelid = to_regclass('ksp_raw_transactions')
) THEN
ALTER TABLE ksp_raw_transactions ADD CONSTRAINT pk_ksp_raw_transactions PRIMARY KEY (signature);
END IF;
END
$ksp$;

12
crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/005_ck_ksp_raw_transactions_signature.sql Normal file
View File

@@ -0,0 +1,12 @@
DO $ksp$
BEGIN
IF NOT EXISTS (
SELECT 1
FROM pg_constraint
WHERE conname = 'ck_ksp_raw_transactions_signature'
AND conrelid = to_regclass('ksp_raw_transactions')
) THEN
ALTER TABLE ksp_raw_transactions ADD CONSTRAINT ck_ksp_raw_transactions_signature CHECK (octet_length(signature) = 64);
END IF;
END
$ksp$;

12
crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/006_ck_ksp_raw_transactions_slot.sql Normal file
View File

@@ -0,0 +1,12 @@
DO $ksp$
BEGIN
IF NOT EXISTS (
SELECT 1
FROM pg_constraint
WHERE conname = 'ck_ksp_raw_transactions_slot'
AND conrelid = to_regclass('ksp_raw_transactions')
) THEN
ALTER TABLE ksp_raw_transactions ADD CONSTRAINT ck_ksp_raw_transactions_slot CHECK (slot >= 0 AND slot <= 18446744073709551615);
END IF;
END
$ksp$;

15
crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/007_ck_ksp_raw_transactions_block_time.sql Normal file
View File

@@ -0,0 +1,15 @@
DO $ksp$
BEGIN
IF NOT EXISTS (
SELECT 1
FROM pg_constraint
WHERE conname = 'ck_ksp_raw_transactions_block_time'
AND conrelid = to_regclass('ksp_raw_transactions')
) THEN
ALTER TABLE ksp_raw_transactions ADD CONSTRAINT ck_ksp_raw_transactions_block_time CHECK (
block_time_unix_millis IS NULL
OR block_time_unix_millis >= 0 AND block_time_unix_millis <= 253402300799999
);
END IF;
END
$ksp$;

15
crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/008_ck_ksp_raw_transactions_format_id.sql Normal file
View File

@@ -0,0 +1,15 @@
DO $ksp$
BEGIN
IF NOT EXISTS (
SELECT 1
FROM pg_constraint
WHERE conname = 'ck_ksp_raw_transactions_format_id'
AND conrelid = to_regclass('ksp_raw_transactions')
) THEN
ALTER TABLE ksp_raw_transactions ADD CONSTRAINT ck_ksp_raw_transactions_format_id CHECK (
octet_length(format_id) >= 1 AND octet_length(format_id) <= 128
AND format_id ~ '^[A-Za-z0-9_.:-]+$'
);
END IF;
END
$ksp$;

12
crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/009_ck_ksp_raw_transactions_format_version.sql Normal file
View File

@@ -0,0 +1,12 @@
DO $ksp$
BEGIN
IF NOT EXISTS (
SELECT 1
FROM pg_constraint
WHERE conname = 'ck_ksp_raw_transactions_format_version'
AND conrelid = to_regclass('ksp_raw_transactions')
) THEN
ALTER TABLE ksp_raw_transactions ADD CONSTRAINT ck_ksp_raw_transactions_format_version CHECK (format_version >= 1 AND format_version <= 4294967295);
END IF;
END
$ksp$;

12
crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/010_ck_ksp_raw_transactions_content_hash.sql Normal file
View File

@@ -0,0 +1,12 @@
DO $ksp$
BEGIN
IF NOT EXISTS (
SELECT 1
FROM pg_constraint
WHERE conname = 'ck_ksp_raw_transactions_content_hash'
AND conrelid = to_regclass('ksp_raw_transactions')
) THEN
ALTER TABLE ksp_raw_transactions ADD CONSTRAINT ck_ksp_raw_transactions_content_hash CHECK (octet_length(content_hash) = 32);
END IF;
END
$ksp$;

15
crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/011_ck_ksp_raw_transactions_payload.sql Normal file
View File

@@ -0,0 +1,15 @@
DO $ksp$
BEGIN
IF NOT EXISTS (
SELECT 1
FROM pg_constraint
WHERE conname = 'ck_ksp_raw_transactions_payload'
AND conrelid = to_regclass('ksp_raw_transactions')
) THEN
ALTER TABLE ksp_raw_transactions ADD CONSTRAINT ck_ksp_raw_transactions_payload CHECK (
payload IS NULL
OR octet_length(payload) >= 1 AND octet_length(payload) <= 16777216
);
END IF;
END
$ksp$;

12
crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/012_ck_ksp_raw_transactions_retention_state.sql Normal file
View File

@@ -0,0 +1,12 @@
DO $ksp$
BEGIN
IF NOT EXISTS (
SELECT 1
FROM pg_constraint
WHERE conname = 'ck_ksp_raw_transactions_retention_state'
AND conrelid = to_regclass('ksp_raw_transactions')
) THEN
ALTER TABLE ksp_raw_transactions ADD CONSTRAINT ck_ksp_raw_transactions_retention_state CHECK ((retention_state = 'full' OR retention_state = 'archived' OR retention_state = 'purged'));
END IF;
END
$ksp$;

15
crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/013_ck_ksp_raw_transactions_payload_state.sql Normal file
View File

@@ -0,0 +1,15 @@
DO $ksp$
BEGIN
IF NOT EXISTS (
SELECT 1
FROM pg_constraint
WHERE conname = 'ck_ksp_raw_transactions_payload_state'
AND conrelid = to_regclass('ksp_raw_transactions')
) THEN
ALTER TABLE ksp_raw_transactions ADD CONSTRAINT ck_ksp_raw_transactions_payload_state CHECK (
(retention_state = 'full' AND payload IS NOT NULL)
OR ((retention_state = 'archived' OR retention_state = 'purged') AND payload IS NULL)
);
END IF;
END
$ksp$;

15
crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/014_ck_ksp_raw_transactions_purged_block_time.sql Normal file
View File

@@ -0,0 +1,15 @@
DO $ksp$
BEGIN
IF NOT EXISTS (
SELECT 1
FROM pg_constraint
WHERE conname = 'ck_ksp_raw_transactions_purged_block_time'
AND conrelid = to_regclass('ksp_raw_transactions')
) THEN
ALTER TABLE ksp_raw_transactions ADD CONSTRAINT ck_ksp_raw_transactions_purged_block_time CHECK (
retention_state <> 'purged'
OR block_time_unix_millis IS NULL
);
END IF;
END
$ksp$;

12
crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/015_pk_ksp_raw_transaction_observations.sql Normal file
View File

@@ -0,0 +1,12 @@
DO $ksp$
BEGIN
IF NOT EXISTS (
SELECT 1
FROM pg_constraint
WHERE conname = 'pk_ksp_raw_transaction_observations'
AND conrelid = to_regclass('ksp_raw_transaction_observations')
) THEN
ALTER TABLE ksp_raw_transaction_observations ADD CONSTRAINT pk_ksp_raw_transaction_observations PRIMARY KEY (observation_key);
END IF;
END
$ksp$;

12
crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/016_fk_ksp_raw_transaction_observations_transaction.sql Normal file
View File

@@ -0,0 +1,12 @@
DO $ksp$
BEGIN
IF NOT EXISTS (
SELECT 1
FROM pg_constraint
WHERE conname = 'fk_ksp_raw_transaction_observations_transaction'
AND conrelid = to_regclass('ksp_raw_transaction_observations')
) THEN
ALTER TABLE ksp_raw_transaction_observations ADD CONSTRAINT fk_ksp_raw_transaction_observations_transaction FOREIGN KEY (transaction_signature) REFERENCES ksp_raw_transactions(signature) ON DELETE RESTRICT;
END IF;
END
$ksp$;

12
crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/017_ck_ksp_raw_transaction_observations_key.sql Normal file
View File

@@ -0,0 +1,12 @@
DO $ksp$
BEGIN
IF NOT EXISTS (
SELECT 1
FROM pg_constraint
WHERE conname = 'ck_ksp_raw_transaction_observations_key'
AND conrelid = to_regclass('ksp_raw_transaction_observations')
) THEN
ALTER TABLE ksp_raw_transaction_observations ADD CONSTRAINT ck_ksp_raw_transaction_observations_key CHECK (octet_length(observation_key) = 32);
END IF;
END
$ksp$;

12
crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/018_ck_ksp_raw_transaction_observations_signature.sql Normal file
View File

@@ -0,0 +1,12 @@
DO $ksp$
BEGIN
IF NOT EXISTS (
SELECT 1
FROM pg_constraint
WHERE conname = 'ck_ksp_raw_transaction_observations_signature'
AND conrelid = to_regclass('ksp_raw_transaction_observations')
) THEN
ALTER TABLE ksp_raw_transaction_observations ADD CONSTRAINT ck_ksp_raw_transaction_observations_signature CHECK (octet_length(transaction_signature) = 64);
END IF;
END
$ksp$;

15
crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/019_ck_ksp_raw_transaction_observations_provider.sql Normal file
View File

@@ -0,0 +1,15 @@
DO $ksp$
BEGIN
IF NOT EXISTS (
SELECT 1
FROM pg_constraint
WHERE conname = 'ck_ksp_raw_transaction_observations_provider'
AND conrelid = to_regclass('ksp_raw_transaction_observations')
) THEN
ALTER TABLE ksp_raw_transaction_observations ADD CONSTRAINT ck_ksp_raw_transaction_observations_provider CHECK (
octet_length(provider) >= 1 AND octet_length(provider) <= 128
AND provider ~ '^[A-Za-z0-9_.:-]+$'
);
END IF;
END
$ksp$;

15
crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/020_ck_ksp_raw_transaction_observations_protocol.sql Normal file
View File

@@ -0,0 +1,15 @@
DO $ksp$
BEGIN
IF NOT EXISTS (
SELECT 1
FROM pg_constraint
WHERE conname = 'ck_ksp_raw_transaction_observations_protocol'
AND conrelid = to_regclass('ksp_raw_transaction_observations')
) THEN
ALTER TABLE ksp_raw_transaction_observations ADD CONSTRAINT ck_ksp_raw_transaction_observations_protocol CHECK (
octet_length(protocol) >= 1 AND octet_length(protocol) <= 128
AND protocol ~ '^[A-Za-z0-9_.:-]+$'
);
END IF;
END
$ksp$;

15
crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/021_ck_ksp_raw_transaction_observations_method.sql Normal file
View File

@@ -0,0 +1,15 @@
DO $ksp$
BEGIN
IF NOT EXISTS (
SELECT 1
FROM pg_constraint
WHERE conname = 'ck_ksp_raw_transaction_observations_method'
AND conrelid = to_regclass('ksp_raw_transaction_observations')
) THEN
ALTER TABLE ksp_raw_transaction_observations ADD CONSTRAINT ck_ksp_raw_transaction_observations_method CHECK (
octet_length(acquisition_method) >= 1 AND octet_length(acquisition_method) <= 128
AND acquisition_method ~ '^[A-Za-z0-9_.:-]+$'
);
END IF;
END
$ksp$;

12
crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/022_ck_ksp_raw_transaction_observations_origin.sql Normal file
View File

@@ -0,0 +1,12 @@
DO $ksp$
BEGIN
IF NOT EXISTS (
SELECT 1
FROM pg_constraint
WHERE conname = 'ck_ksp_raw_transaction_observations_origin'
AND conrelid = to_regclass('ksp_raw_transaction_observations')
) THEN
ALTER TABLE ksp_raw_transaction_observations ADD CONSTRAINT ck_ksp_raw_transaction_observations_origin CHECK ((origin = 'backfill' OR origin = 'import' OR origin = 'live' OR origin = 'repair' OR origin = 'replay'));
END IF;
END
$ksp$;

12
crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/023_ck_ksp_raw_transaction_observations_received_at.sql Normal file
View File

@@ -0,0 +1,12 @@
DO $ksp$
BEGIN
IF NOT EXISTS (
SELECT 1
FROM pg_constraint
WHERE conname = 'ck_ksp_raw_transaction_observations_received_at'
AND conrelid = to_regclass('ksp_raw_transaction_observations')
) THEN
ALTER TABLE ksp_raw_transaction_observations ADD CONSTRAINT ck_ksp_raw_transaction_observations_received_at CHECK (received_at_unix_millis >= 0 AND received_at_unix_millis <= 253402300799999);
END IF;
END
$ksp$;

18
crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/024_ck_ksp_raw_transaction_observations_capture_session.sql Normal file
View File

@@ -0,0 +1,18 @@
DO $ksp$
BEGIN
IF NOT EXISTS (
SELECT 1
FROM pg_constraint
WHERE conname = 'ck_ksp_raw_transaction_observations_capture_session'
AND conrelid = to_regclass('ksp_raw_transaction_observations')
) THEN
ALTER TABLE ksp_raw_transaction_observations ADD CONSTRAINT ck_ksp_raw_transaction_observations_capture_session CHECK (
capture_session_id IS NULL
OR (
octet_length(capture_session_id) >= 1 AND octet_length(capture_session_id) <= 128
AND capture_session_id ~ '^[A-Za-z0-9_.:-]+$'
)
);
END IF;
END
$ksp$;

18
crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/025_ck_ksp_raw_transaction_observations_commitment.sql Normal file
View File

@@ -0,0 +1,18 @@
DO $ksp$
BEGIN
IF NOT EXISTS (
SELECT 1
FROM pg_constraint
WHERE conname = 'ck_ksp_raw_transaction_observations_commitment'
AND conrelid = to_regclass('ksp_raw_transaction_observations')
) THEN
ALTER TABLE ksp_raw_transaction_observations ADD CONSTRAINT ck_ksp_raw_transaction_observations_commitment CHECK (
commitment IS NULL
OR (
octet_length(commitment) >= 1 AND octet_length(commitment) <= 128
AND commitment ~ '^[A-Za-z0-9_.:-]+$'
)
);
END IF;
END
$ksp$;

18
crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/026_ck_ksp_raw_transaction_observations_endpoint.sql Normal file
View File

@@ -0,0 +1,18 @@
DO $ksp$
BEGIN
IF NOT EXISTS (
SELECT 1
FROM pg_constraint
WHERE conname = 'ck_ksp_raw_transaction_observations_endpoint'
AND conrelid = to_regclass('ksp_raw_transaction_observations')
) THEN
ALTER TABLE ksp_raw_transaction_observations ADD CONSTRAINT ck_ksp_raw_transaction_observations_endpoint CHECK (
endpoint_id IS NULL
OR (
octet_length(endpoint_id) >= 1 AND octet_length(endpoint_id) <= 128
AND endpoint_id ~ '^[A-Za-z0-9_.:-]+$'
)
);
END IF;
END
$ksp$;

18
crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/027_ck_ksp_raw_transaction_observations_filter.sql Normal file
View File

@@ -0,0 +1,18 @@
DO $ksp$
BEGIN
IF NOT EXISTS (
SELECT 1
FROM pg_constraint
WHERE conname = 'ck_ksp_raw_transaction_observations_filter'
AND conrelid = to_regclass('ksp_raw_transaction_observations')
) THEN
ALTER TABLE ksp_raw_transaction_observations ADD CONSTRAINT ck_ksp_raw_transaction_observations_filter CHECK (
filter_id IS NULL
OR (
octet_length(filter_id) >= 1 AND octet_length(filter_id) <= 128
AND filter_id ~ '^[A-Za-z0-9_.:-]+$'
)
);
END IF;
END
$ksp$;

15
crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/028_ck_ksp_raw_transaction_observations_observed_at.sql Normal file
View File

@@ -0,0 +1,15 @@
DO $ksp$
BEGIN
IF NOT EXISTS (
SELECT 1
FROM pg_constraint
WHERE conname = 'ck_ksp_raw_transaction_observations_observed_at'
AND conrelid = to_regclass('ksp_raw_transaction_observations')
) THEN
ALTER TABLE ksp_raw_transaction_observations ADD CONSTRAINT ck_ksp_raw_transaction_observations_observed_at CHECK (
observed_at_unix_millis IS NULL
OR observed_at_unix_millis >= 0 AND observed_at_unix_millis <= 253402300799999
);
END IF;
END
$ksp$;

15
crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/029_ck_ksp_raw_transaction_observations_time_order.sql Normal file
View File

@@ -0,0 +1,15 @@
DO $ksp$
BEGIN
IF NOT EXISTS (
SELECT 1
FROM pg_constraint
WHERE conname = 'ck_ksp_raw_transaction_observations_time_order'
AND conrelid = to_regclass('ksp_raw_transaction_observations')
) THEN
ALTER TABLE ksp_raw_transaction_observations ADD CONSTRAINT ck_ksp_raw_transaction_observations_time_order CHECK (
observed_at_unix_millis IS NULL
OR observed_at_unix_millis <= received_at_unix_millis
);
END IF;
END
$ksp$;

15
crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/030_ck_ksp_raw_transaction_observations_source_hash.sql Normal file
View File

@@ -0,0 +1,15 @@
DO $ksp$
BEGIN
IF NOT EXISTS (
SELECT 1
FROM pg_constraint
WHERE conname = 'ck_ksp_raw_transaction_observations_source_hash'
AND conrelid = to_regclass('ksp_raw_transaction_observations')
) THEN
ALTER TABLE ksp_raw_transaction_observations ADD CONSTRAINT ck_ksp_raw_transaction_observations_source_hash CHECK (
source_payload_hash IS NULL
OR octet_length(source_payload_hash) = 32
);
END IF;
END
$ksp$;

15
crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/031_ck_ksp_raw_transaction_observations_source_size.sql Normal file
View File

@@ -0,0 +1,15 @@
DO $ksp$
BEGIN
IF NOT EXISTS (
SELECT 1
FROM pg_constraint
WHERE conname = 'ck_ksp_raw_transaction_observations_source_size'
AND conrelid = to_regclass('ksp_raw_transaction_observations')
) THEN
ALTER TABLE ksp_raw_transaction_observations ADD CONSTRAINT ck_ksp_raw_transaction_observations_source_size CHECK (
source_payload_size_bytes IS NULL
OR source_payload_size_bytes >= 0 AND source_payload_size_bytes <= 67108864
);
END IF;
END
$ksp$;

12
crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/032_pk_ksp_raw_transaction_archive_payloads.sql Normal file
View File

@@ -0,0 +1,12 @@
DO $ksp$
BEGIN
IF NOT EXISTS (
SELECT 1
FROM pg_constraint
WHERE conname = 'pk_ksp_raw_transaction_archive_payloads'
AND conrelid = to_regclass('ksp_raw_transaction_archive_payloads')
) THEN
ALTER TABLE ksp_raw_transaction_archive_payloads ADD CONSTRAINT pk_ksp_raw_transaction_archive_payloads PRIMARY KEY (signature);
END IF;
END
$ksp$;

12
crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/033_fk_ksp_raw_transaction_archive_payloads_transaction.sql Normal file
View File

@@ -0,0 +1,12 @@
DO $ksp$
BEGIN
IF NOT EXISTS (
SELECT 1
FROM pg_constraint
WHERE conname = 'fk_ksp_raw_transaction_archive_payloads_transaction'
AND conrelid = to_regclass('ksp_raw_transaction_archive_payloads')
) THEN
ALTER TABLE ksp_raw_transaction_archive_payloads ADD CONSTRAINT fk_ksp_raw_transaction_archive_payloads_transaction FOREIGN KEY (signature) REFERENCES ksp_raw_transactions(signature) ON DELETE RESTRICT;
END IF;
END
$ksp$;

12
crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/034_ck_ksp_raw_transaction_archive_payloads_signature.sql Normal file
View File

@@ -0,0 +1,12 @@
DO $ksp$
BEGIN
IF NOT EXISTS (
SELECT 1
FROM pg_constraint
WHERE conname = 'ck_ksp_raw_transaction_archive_payloads_signature'
AND conrelid = to_regclass('ksp_raw_transaction_archive_payloads')
) THEN
ALTER TABLE ksp_raw_transaction_archive_payloads ADD CONSTRAINT ck_ksp_raw_transaction_archive_payloads_signature CHECK (octet_length(signature) = 64);
END IF;
END
$ksp$;

12
crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/035_ck_ksp_raw_transaction_archive_payloads_payload.sql Normal file
View File

@@ -0,0 +1,12 @@
DO $ksp$
BEGIN
IF NOT EXISTS (
SELECT 1
FROM pg_constraint
WHERE conname = 'ck_ksp_raw_transaction_archive_payloads_payload'
AND conrelid = to_regclass('ksp_raw_transaction_archive_payloads')
) THEN
ALTER TABLE ksp_raw_transaction_archive_payloads ADD CONSTRAINT ck_ksp_raw_transaction_archive_payloads_payload CHECK (octet_length(payload) >= 1 AND octet_length(payload) <= 16777216);
END IF;
END
$ksp$;

3
crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/indexes/001_ix_ksp_raw_transactions_slot_signature.sql Normal file
View File

@@ -0,0 +1,3 @@
CREATE INDEX IF NOT EXISTS ix_ksp_raw_transactions_slot_signature
ON ksp_raw_transactions (slot, signature)
WHERE retention_state <> 'purged';

7
crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/tables/001_ksp_store_identity.sql Normal file
View File

@@ -0,0 +1,7 @@
CREATE TABLE IF NOT EXISTS ksp_store_identity (
singleton SMALLINT NOT NULL,
network TEXT NOT NULL
);
ALTER TABLE ksp_store_identity ADD COLUMN IF NOT EXISTS singleton SMALLINT NOT NULL;
ALTER TABLE ksp_store_identity ADD COLUMN IF NOT EXISTS network TEXT NOT NULL;

19
crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/tables/002_ksp_raw_transactions.sql Normal file
View File

@@ -0,0 +1,19 @@
CREATE TABLE IF NOT EXISTS ksp_raw_transactions (
signature BYTEA NOT NULL,
slot NUMERIC(20, 0) NOT NULL,
block_time_unix_millis BIGINT NULL,
format_id TEXT NOT NULL,
format_version BIGINT NOT NULL,
content_hash BYTEA NOT NULL,
payload BYTEA NULL,
retention_state TEXT NOT NULL
);
ALTER TABLE ksp_raw_transactions ADD COLUMN IF NOT EXISTS signature BYTEA NOT NULL;
ALTER TABLE ksp_raw_transactions ADD COLUMN IF NOT EXISTS slot NUMERIC(20, 0) NOT NULL;
ALTER TABLE ksp_raw_transactions ADD COLUMN IF NOT EXISTS block_time_unix_millis BIGINT NULL;
ALTER TABLE ksp_raw_transactions ADD COLUMN IF NOT EXISTS format_id TEXT NOT NULL;
ALTER TABLE ksp_raw_transactions ADD COLUMN IF NOT EXISTS format_version BIGINT NOT NULL;
ALTER TABLE ksp_raw_transactions ADD COLUMN IF NOT EXISTS content_hash BYTEA NOT NULL;
ALTER TABLE ksp_raw_transactions ADD COLUMN IF NOT EXISTS payload BYTEA NULL;
ALTER TABLE ksp_raw_transactions ADD COLUMN IF NOT EXISTS retention_state TEXT NOT NULL;

31
crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/tables/003_ksp_raw_transaction_observations.sql Normal file
View File

@@ -0,0 +1,31 @@
CREATE TABLE IF NOT EXISTS ksp_raw_transaction_observations (
observation_key BYTEA NOT NULL,
transaction_signature BYTEA NOT NULL,
provider TEXT NOT NULL,
protocol TEXT NOT NULL,
acquisition_method TEXT NOT NULL,
origin TEXT NOT NULL,
received_at_unix_millis BIGINT NOT NULL,
capture_session_id TEXT NULL,
commitment TEXT NULL,
endpoint_id TEXT NULL,
filter_id TEXT NULL,
observed_at_unix_millis BIGINT NULL,
source_payload_hash BYTEA NULL,
source_payload_size_bytes BIGINT NULL
);
ALTER TABLE ksp_raw_transaction_observations ADD COLUMN IF NOT EXISTS observation_key BYTEA NOT NULL;
ALTER TABLE ksp_raw_transaction_observations ADD COLUMN IF NOT EXISTS transaction_signature BYTEA NOT NULL;
ALTER TABLE ksp_raw_transaction_observations ADD COLUMN IF NOT EXISTS provider TEXT NOT NULL;
ALTER TABLE ksp_raw_transaction_observations ADD COLUMN IF NOT EXISTS protocol TEXT NOT NULL;
ALTER TABLE ksp_raw_transaction_observations ADD COLUMN IF NOT EXISTS acquisition_method TEXT NOT NULL;
ALTER TABLE ksp_raw_transaction_observations ADD COLUMN IF NOT EXISTS origin TEXT NOT NULL;
ALTER TABLE ksp_raw_transaction_observations ADD COLUMN IF NOT EXISTS received_at_unix_millis BIGINT NOT NULL;
ALTER TABLE ksp_raw_transaction_observations ADD COLUMN IF NOT EXISTS capture_session_id TEXT NULL;
ALTER TABLE ksp_raw_transaction_observations ADD COLUMN IF NOT EXISTS commitment TEXT NULL;
ALTER TABLE ksp_raw_transaction_observations ADD COLUMN IF NOT EXISTS endpoint_id TEXT NULL;
ALTER TABLE ksp_raw_transaction_observations ADD COLUMN IF NOT EXISTS filter_id TEXT NULL;
ALTER TABLE ksp_raw_transaction_observations ADD COLUMN IF NOT EXISTS observed_at_unix_millis BIGINT NULL;
ALTER TABLE ksp_raw_transaction_observations ADD COLUMN IF NOT EXISTS source_payload_hash BYTEA NULL;
ALTER TABLE ksp_raw_transaction_observations ADD COLUMN IF NOT EXISTS source_payload_size_bytes BIGINT NULL;

7
crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/tables/004_ksp_raw_transaction_archive_payloads.sql Normal file
View File

@@ -0,0 +1,7 @@
CREATE TABLE IF NOT EXISTS ksp_raw_transaction_archive_payloads (
signature BYTEA NOT NULL,
payload BYTEA NOT NULL
);
ALTER TABLE ksp_raw_transaction_archive_payloads ADD COLUMN IF NOT EXISTS signature BYTEA NOT NULL;
ALTER TABLE ksp_raw_transaction_archive_payloads ADD COLUMN IF NOT EXISTS payload BYTEA NOT NULL;

24
crates/ksp-store-postgres-lib/src/lib.rs
View File

@@ -1,5 +1,5 @@
// file: crates/ksp-store-postgres-lib/src/lib.rs
// version: 6
// version: 8
#![warn(missing_docs)]
#![deny(unreachable_pub)]
@@ -9,9 +9,10 @@
//!
//! The backend owns the physical `tokio-postgres` connection, bounded Deadpool
//! pool, explicit Rustls TLS policy, private KSP migration/bootstrap engine and
//! safe lightweight health/readiness probe. `0.3.3-pre.003` adds the immutable
//! V001 RawTransaction physical schema and mono-network database binding; the
//! business capability implementations remain deferred to later prereleases.
//! safe lightweight health/readiness probe. `0.3.3-pre.003-fix.001` splits
//! migrations into versioned physical resources and verifies the effective
//! PostgreSQL schema contract before readiness; business capability
//! implementations remain deferred to later prereleases.
//!
//! This crate depends on `ksp-store-api` and never on `ksp-store-lib`. The
//! common facade consumes only this crate's narrow backend bridge and never
@@ -22,6 +23,7 @@ mod error;
mod health;
mod migration;
mod runtime;
mod schema;
/// Stable KSP error code for unsupported PostgreSQL retention compaction.
pub use self::error::ERROR_CODE_POSTGRES_RETENTION_COMPACTION_UNSUPPORTED;
@@ -52,5 +54,19 @@ pub(crate) use self::migration::current_migration_version;
pub(crate) use self::runtime::map_pool_error;
/// Private Deadpool status projector shared with the health probe.
pub(crate) use self::runtime::runtime_snapshot_from_status;
/// Private physical schema resource descriptor consumed by the migration engine.
pub(crate) use self::schema::SchemaResource;
/// Private physical schema resource compatibility state consumed by the migration engine.
pub(crate) use self::schema::SchemaResourceState;
/// Private V000 schema resource inventory consumed by the migration engine.
pub(crate) use self::schema::V000_RESOURCES;
/// Private V001 schema resource inventory consumed by the migration engine.
pub(crate) use self::schema::V001_RESOURCES;
/// Private physical schema resource inspector consumed by the migration engine.
pub(crate) use self::schema::inspect_resource;
/// Private V001 adoption probe consumed by the migration engine.
pub(crate) use self::schema::managed_v001_objects_exist;
/// Private external-schema compatibility gate consumed by the migration engine.
pub(crate) use self::schema::verify_v001_external_compatibility;
const _: &str = crate::TRACING_TARGET;

388
crates/ksp-store-postgres-lib/src/migration.rs
View File

@@ -1,15 +1,22 @@
// file: crates/ksp-store-postgres-lib/src/migration.rs
// version: 4
// version: 5
use sha2::Digest; // rust-rules: trait-import
const ADVISORY_LOCK_KEY: i64 = 0x4b53_5053_544f_5245;
const EMBEDDED_MIGRATIONS: &[EmbeddedMigration] = &[
EmbeddedMigration { hook: MigrationHook::None, name: "bootstrap", sql: include_str!("../migrations/V000__bootstrap.sql"), version: 0 },
EmbeddedMigration {
checksum: MigrationChecksum::LegacySql(include_str!("../migrations/v000_bootstrap/tables/001_ksp_store_schema_migrations.sql")),
hook: MigrationHook::None,
name: "bootstrap",
resources: crate::V000_RESOURCES,
version: 0,
},
EmbeddedMigration {
checksum: MigrationChecksum::Resources,
hook: MigrationHook::StoreIdentity,
name: "raw_transaction",
sql: include_str!("../migrations/V001__raw_transaction.sql"),
resources: crate::V001_RESOURCES,
version: 1,
},
];
@@ -25,21 +32,6 @@ const METADATA_EXISTS_SQL: &str = r#"SELECT EXISTS (
AND table_name = 'ksp_store_schema_migrations'
AND table_type = 'BASE TABLE'
)"#;
const METADATA_PRIMARY_KEY_SQL: &str = r#"SELECT COUNT(*)::BIGINT,
COUNT(*) FILTER (WHERE kcu.column_name = 'version')::BIGINT
FROM information_schema.table_constraints tc
JOIN information_schema.key_column_usage kcu
ON tc.constraint_catalog = kcu.constraint_catalog
AND tc.constraint_schema = kcu.constraint_schema
AND tc.constraint_name = kcu.constraint_name
WHERE tc.table_schema = current_schema()
AND tc.table_name = 'ksp_store_schema_migrations'
AND tc.constraint_type = 'PRIMARY KEY'"#;
const METADATA_SHAPE_SQL: &str = r#"SELECT column_name, data_type, is_nullable
FROM information_schema.columns
WHERE table_schema = current_schema()
AND table_name = 'ksp_store_schema_migrations'
ORDER BY ordinal_position"#;
const SET_STATEMENT_TIMEOUT_SQL: &str = "SELECT set_config('statement_timeout', $1, true)";
struct AppliedMigration {
@@ -48,11 +40,18 @@ struct AppliedMigration {
version: i64,
}
#[derive(Clone, Copy)]
enum MigrationChecksum {
LegacySql(&'static str),
Resources,
}
#[derive(Clone, Copy)]
struct EmbeddedMigration {
checksum: MigrationChecksum,
hook: MigrationHook,
name: &'static str,
sql: &'static str,
resources: &'static [crate::SchemaResource],
version: i64,
}
@@ -68,6 +67,12 @@ enum MigrationHookContext {
Existing,
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
enum SchemaMutationMode {
Create,
Update,
}
/// Returns the latest migration version embedded by this backend runtime.
#[must_use]
pub(crate) const fn current_migration_version() -> i64 {
@@ -78,7 +83,8 @@ pub(crate) const fn current_migration_version() -> i64 {
pub(crate) async fn bootstrap(
client: &mut deadpool_postgres::Client,
network: &ksp_store_api::RawNetworkId,
auto_migrate: bool,
schema_autocreate: bool,
schema_autoupdate: bool,
migration_timeout: std::time::Duration,
migration_lock_timeout: std::time::Duration,
) -> std::result::Result<(), crate::PostgresBackendError> {
@@ -88,7 +94,7 @@ pub(crate) async fn bootstrap(
}
let bounded = tokio::time::timeout(
migration_timeout,
bootstrap_inner(client, network, auto_migrate, migration_timeout, migration_lock_timeout),
bootstrap_inner(client, network, schema_autocreate, schema_autoupdate, migration_timeout, migration_lock_timeout),
)
.await;
return match bounded {
@@ -102,7 +108,8 @@ pub(crate) async fn bootstrap(
async fn bootstrap_inner(
client: &mut deadpool_postgres::Client,
network: &ksp_store_api::RawNetworkId,
auto_migrate: bool,
schema_autocreate: bool,
schema_autoupdate: bool,
migration_timeout: std::time::Duration,
migration_lock_timeout: std::time::Duration,
) -> std::result::Result<(), crate::PostgresBackendError> {
@@ -126,10 +133,15 @@ async fn bootstrap_inner(
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let next_index = if metadata_exists {
let shape_result = verify_metadata_shape(&transaction).await;
if let std::result::Result::Err(error) = shape_result {
return std::result::Result::Err(error);
let (next_index, mutation_mode) = if metadata_exists {
let metadata_result = crate::inspect_resource(&transaction, &crate::V000_RESOURCES[0]).await;
match metadata_result {
std::result::Result::Ok(crate::SchemaResourceState::Compatible) => {},
std::result::Result::Ok(crate::SchemaResourceState::Missing | crate::SchemaResourceState::Incompatible) => {
log_schema_block("metadata_incompatible");
return std::result::Result::Err(crate::PostgresBackendError::new(crate::PostgresBackendErrorKind::MigrationMismatch, "metadata_incompatible"));
},
std::result::Result::Err(error) => return std::result::Result::Err(error),
}
let history_result = load_history(&transaction).await;
let history = match history_result {
@@ -137,21 +149,40 @@ async fn bootstrap_inner(
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
let validation_result = validate_history(history.as_slice(), EMBEDDED_MIGRATIONS);
match validation_result {
let index = match validation_result {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
}
};
(index, SchemaMutationMode::Update)
} else {
0
if !schema_autocreate {
log_schema_block("schema_autocreate_disabled");
return std::result::Result::Err(crate::PostgresBackendError::new(crate::PostgresBackendErrorKind::MigrationFailed, "schema_autocreate_disabled"));
}
let managed_result = crate::managed_v001_objects_exist(&transaction).await;
let managed_objects_exist = match managed_result {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
if managed_objects_exist && !schema_autoupdate {
log_schema_block("schema_adoption_disabled");
return std::result::Result::Err(crate::PostgresBackendError::new(crate::PostgresBackendErrorKind::MigrationFailed, "schema_adoption_disabled"));
}
(0, SchemaMutationMode::Create)
};
let existing_schema_result = verify_or_repair_applied_migrations(&transaction, next_index, schema_autoupdate).await;
if let std::result::Result::Err(error) = existing_schema_result {
return std::result::Result::Err(error);
}
let existing_hook_result = run_applied_migration_hooks(&transaction, network, next_index).await;
if let std::result::Result::Err(error) = existing_hook_result {
return std::result::Result::Err(error);
}
if next_index < EMBEDDED_MIGRATIONS.len() && !auto_migrate {
if next_index < EMBEDDED_MIGRATIONS.len() && mutation_mode == SchemaMutationMode::Update && !schema_autoupdate {
log_schema_block("migration_pending");
return std::result::Result::Err(crate::PostgresBackendError::new(crate::PostgresBackendErrorKind::MigrationFailed, "migration_pending"));
}
let apply_result = apply_pending_migrations(&transaction, network, next_index).await;
let apply_result = apply_pending_migrations(&transaction, network, next_index, mutation_mode).await;
if let std::result::Result::Err(error) = apply_result {
return std::result::Result::Err(error);
}
@@ -207,20 +238,23 @@ async fn apply_migration(
transaction: &deadpool_postgres::Transaction<'_>,
network: &ksp_store_api::RawNetworkId,
migration: &EmbeddedMigration,
mutation_mode: SchemaMutationMode,
) -> std::result::Result<(), crate::PostgresBackendError> {
let execute_result = transaction.batch_execute(migration.sql).await;
if execute_result.is_err() {
return std::result::Result::Err(crate::PostgresBackendError::new(crate::PostgresBackendErrorKind::MigrationFailed, "migration_apply"));
for resource in migration.resources {
let result = ensure_resource(transaction, resource, mutation_mode, false).await;
if let std::result::Result::Err(error) = result {
return std::result::Result::Err(error);
}
}
let shape_result = verify_metadata_shape(transaction).await;
if let std::result::Result::Err(error) = shape_result {
let contract_result = verify_migration_contract(transaction, migration.version).await;
if let std::result::Result::Err(error) = contract_result {
return std::result::Result::Err(error);
}
let hook_result = run_migration_hook(transaction, network, migration.hook, MigrationHookContext::AppliedNow).await;
if let std::result::Result::Err(error) = hook_result {
return std::result::Result::Err(error);
}
let checksum = migration_checksum(migration.sql);
let checksum = migration_checksum(migration);
let insert_result = transaction.execute(HISTORY_INSERT_SQL, &[&migration.version, &migration.name, &checksum]).await;
return match insert_result {
std::result::Result::Ok(1) => std::result::Result::Ok(()),
@@ -234,11 +268,12 @@ async fn apply_pending_migrations(
transaction: &deadpool_postgres::Transaction<'_>,
network: &ksp_store_api::RawNetworkId,
next_index: usize,
mutation_mode: SchemaMutationMode,
) -> std::result::Result<(), crate::PostgresBackendError> {
let mut index = next_index;
while index < EMBEDDED_MIGRATIONS.len() {
let migration = &EMBEDDED_MIGRATIONS[index];
let result = apply_migration(transaction, network, migration).await;
let result = apply_migration(transaction, network, migration, mutation_mode).await;
if let std::result::Result::Err(error) = result {
return std::result::Result::Err(error);
}
@@ -247,6 +282,59 @@ async fn apply_pending_migrations(
return std::result::Result::Ok(());
}
async fn ensure_resource(
transaction: &deadpool_postgres::Transaction<'_>,
resource: &crate::SchemaResource,
mutation_mode: SchemaMutationMode,
applied_history: bool,
) -> std::result::Result<(), crate::PostgresBackendError> {
let state_result = crate::inspect_resource(transaction, resource).await;
let state = match state_result {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
match state {
crate::SchemaResourceState::Compatible => return std::result::Result::Ok(()),
crate::SchemaResourceState::Incompatible => {
log_schema_resource_block(resource.id, "incompatible");
return std::result::Result::Err(crate::PostgresBackendError::new(
crate::PostgresBackendErrorKind::MigrationMismatch,
"schema_resource_incompatible",
));
},
crate::SchemaResourceState::Missing => {},
}
if applied_history && !resource.repair_existing {
log_schema_resource_block(resource.id, "repair_forbidden");
return std::result::Result::Err(crate::PostgresBackendError::new(crate::PostgresBackendErrorKind::MigrationMismatch, "schema_resource_missing"));
}
if applied_history && mutation_mode != SchemaMutationMode::Update {
log_schema_resource_block(resource.id, "repair_mode_invalid");
return std::result::Result::Err(crate::PostgresBackendError::new(crate::PostgresBackendErrorKind::MigrationMismatch, "schema_resource_missing"));
}
if applied_history {
ksp_logging_lib::warn!(
target: crate::TRACING_TARGET,
resource_id = resource.id,
"repairing missing PostgreSQL Store schema resource under schema_autoupdate policy"
);
}
let execute_result = transaction.batch_execute(resource.sql).await;
if execute_result.is_err() {
log_schema_resource_block(resource.id, "apply_failed");
return std::result::Result::Err(crate::PostgresBackendError::new(crate::PostgresBackendErrorKind::MigrationFailed, "schema_resource_apply"));
}
let verified_result = crate::inspect_resource(transaction, resource).await;
return match verified_result {
std::result::Result::Ok(crate::SchemaResourceState::Compatible) => std::result::Result::Ok(()),
std::result::Result::Ok(crate::SchemaResourceState::Missing | crate::SchemaResourceState::Incompatible) => {
log_schema_resource_block(resource.id, "post_apply_incompatible");
std::result::Result::Err(crate::PostgresBackendError::new(crate::PostgresBackendErrorKind::MigrationMismatch, "schema_resource_post_apply"))
},
std::result::Result::Err(error) => std::result::Result::Err(error),
};
}
async fn load_history(transaction: &deadpool_postgres::Transaction<'_>) -> std::result::Result<std::vec::Vec<AppliedMigration>, crate::PostgresBackendError> {
let result = transaction.query(HISTORY_LOAD_SQL, &[]).await;
let rows = match result {
@@ -288,11 +376,30 @@ async fn metadata_exists(transaction: &deadpool_postgres::Transaction<'_>) -> st
};
}
fn migration_checksum(sql: &str) -> std::string::String {
fn migration_checksum(migration: &EmbeddedMigration) -> std::string::String {
return match migration.checksum {
MigrationChecksum::LegacySql(sql) => checksum_bytes(sql.as_bytes()),
MigrationChecksum::Resources => {
let mut hasher = sha2::Sha256::new();
hasher.update(b"ksp-migration-resources-v1\0");
for resource in migration.resources {
hasher.update(resource.id.as_bytes());
hasher.update([0]);
hasher.update(resource.sql.as_bytes());
hasher.update([0]);
}
return encode_digest(hasher.finalize().as_slice());
},
};
}
fn checksum_bytes(bytes: &[u8]) -> std::string::String {
let mut hasher = sha2::Sha256::new();
hasher.update(sql.as_bytes());
let digest = hasher.finalize();
let bytes = digest.as_slice();
hasher.update(bytes);
return encode_digest(hasher.finalize().as_slice());
}
fn encode_digest(bytes: &[u8]) -> std::string::String {
let mut encoded = std::string::String::with_capacity(bytes.len() * 2);
for byte in bytes {
let value = *byte;
@@ -336,33 +443,44 @@ async fn bind_store_identity(
network: &ksp_store_api::RawNetworkId,
context: MigrationHookContext,
) -> std::result::Result<(), crate::PostgresBackendError> {
if context == MigrationHookContext::AppliedNow {
let first_read = load_store_identity(transaction).await;
let first = match first_read {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
if first.is_empty() && context == MigrationHookContext::AppliedNow {
let insert_result = transaction.execute(IDENTITY_INSERT_SQL, &[&network.as_str()]).await;
match insert_result {
std::result::Result::Ok(1) => {},
std::result::Result::Ok(_) | std::result::Result::Err(_) => {
return std::result::Result::Err(crate::PostgresBackendError::new(
crate::PostgresBackendErrorKind::MigrationFailed,
"store_identity_insert",
));
return std::result::Result::Err(crate::PostgresBackendError::new(crate::PostgresBackendErrorKind::MigrationFailed, "store_identity_insert"));
},
}
let second_read = load_store_identity(transaction).await;
let second = match second_read {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
return validate_store_identity(second.as_slice(), network);
}
return validate_store_identity(first.as_slice(), network);
}
async fn load_store_identity(
transaction: &deadpool_postgres::Transaction<'_>,
) -> std::result::Result<std::vec::Vec<tokio_postgres::Row>, crate::PostgresBackendError> {
let rows_result = transaction.query(IDENTITY_LOAD_SQL, &[]).await;
let rows = match rows_result {
std::result::Result::Ok(value) => value,
return match rows_result {
std::result::Result::Ok(value) => std::result::Result::Ok(value),
std::result::Result::Err(_) => {
return std::result::Result::Err(crate::PostgresBackendError::new(
crate::PostgresBackendErrorKind::MigrationMismatch,
"store_identity_read",
));
std::result::Result::Err(crate::PostgresBackendError::new(crate::PostgresBackendErrorKind::MigrationMismatch, "store_identity_read"))
},
};
}
fn validate_store_identity(rows: &[tokio_postgres::Row], network: &ksp_store_api::RawNetworkId) -> std::result::Result<(), crate::PostgresBackendError> {
if rows.len() != 1 {
return std::result::Result::Err(crate::PostgresBackendError::new(
crate::PostgresBackendErrorKind::MigrationMismatch,
"store_identity_count",
));
return std::result::Result::Err(crate::PostgresBackendError::new(crate::PostgresBackendErrorKind::MigrationMismatch, "store_identity_count"));
}
let row = &rows[0];
let singleton_result = row.try_get::<usize, i16>(0);
@@ -370,32 +488,20 @@ async fn bind_store_identity(
let (singleton, stored_network) = match (singleton_result, network_result) {
(std::result::Result::Ok(singleton), std::result::Result::Ok(stored_network)) => (singleton, stored_network),
_ => {
return std::result::Result::Err(crate::PostgresBackendError::new(
crate::PostgresBackendErrorKind::MigrationMismatch,
"store_identity_decode",
));
return std::result::Result::Err(crate::PostgresBackendError::new(crate::PostgresBackendErrorKind::MigrationMismatch, "store_identity_decode"));
},
};
if singleton != 1 {
return std::result::Result::Err(crate::PostgresBackendError::new(
crate::PostgresBackendErrorKind::MigrationMismatch,
"store_identity_singleton",
));
return std::result::Result::Err(crate::PostgresBackendError::new(crate::PostgresBackendErrorKind::MigrationMismatch, "store_identity_singleton"));
}
let stored_network = match ksp_store_api::RawNetworkId::new(stored_network) {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => {
return std::result::Result::Err(crate::PostgresBackendError::new(
crate::PostgresBackendErrorKind::MigrationMismatch,
"store_identity_network",
));
return std::result::Result::Err(crate::PostgresBackendError::new(crate::PostgresBackendErrorKind::MigrationMismatch, "store_identity_network"));
},
};
if stored_network.as_str() != network.as_str() {
return std::result::Result::Err(crate::PostgresBackendError::new(
crate::PostgresBackendErrorKind::MigrationMismatch,
"store_identity_network",
));
return std::result::Result::Err(crate::PostgresBackendError::new(crate::PostgresBackendErrorKind::MigrationMismatch, "store_identity_network"));
}
return std::result::Result::Ok(());
}
@@ -414,15 +520,83 @@ async fn set_statement_timeout(
};
}
async fn verify_migration_contract(transaction: &deadpool_postgres::Transaction<'_>, version: i64) -> std::result::Result<(), crate::PostgresBackendError> {
if version == 1 {
let result = crate::verify_v001_external_compatibility(transaction).await;
if let std::result::Result::Err(error) = result {
log_schema_block(error.phase());
return std::result::Result::Err(error);
}
}
return std::result::Result::Ok(());
}
async fn verify_or_repair_applied_migrations(
transaction: &deadpool_postgres::Transaction<'_>,
applied_count: usize,
schema_autoupdate: bool,
) -> std::result::Result<(), crate::PostgresBackendError> {
let mut index = 0_usize;
while index < applied_count {
let migration = &EMBEDDED_MIGRATIONS[index];
for resource in migration.resources {
let state_result = crate::inspect_resource(transaction, resource).await;
let state = match state_result {
std::result::Result::Ok(value) => value,
std::result::Result::Err(error) => return std::result::Result::Err(error),
};
match state {
crate::SchemaResourceState::Compatible => {},
crate::SchemaResourceState::Incompatible => {
log_schema_resource_block(resource.id, "incompatible");
return std::result::Result::Err(crate::PostgresBackendError::new(
crate::PostgresBackendErrorKind::MigrationMismatch,
"schema_resource_incompatible",
));
},
crate::SchemaResourceState::Missing => {
if !schema_autoupdate {
log_schema_resource_block(resource.id, "schema_autoupdate_disabled");
return std::result::Result::Err(crate::PostgresBackendError::new(
crate::PostgresBackendErrorKind::MigrationFailed,
"schema_autoupdate_disabled",
));
}
let repair_result = ensure_resource(transaction, resource, SchemaMutationMode::Update, true).await;
if let std::result::Result::Err(error) = repair_result {
return std::result::Result::Err(error);
}
},
}
}
let contract_result = verify_migration_contract(transaction, migration.version).await;
if let std::result::Result::Err(error) = contract_result {
return std::result::Result::Err(error);
}
index += 1;
}
return std::result::Result::Ok(());
}
fn validate_embedded_registry(migrations: &[EmbeddedMigration]) -> std::result::Result<(), crate::PostgresBackendError> {
if migrations.is_empty() {
return std::result::Result::Err(crate::PostgresBackendError::new(crate::PostgresBackendErrorKind::MigrationMismatch, "registry_empty"));
}
let mut expected_version = 0_i64;
for migration in migrations {
if migration.version != expected_version || migration.name.is_empty() || migration.sql.is_empty() {
if migration.version != expected_version || migration.name.is_empty() || migration.resources.is_empty() {
return std::result::Result::Err(crate::PostgresBackendError::new(crate::PostgresBackendErrorKind::MigrationMismatch, "registry_invalid"));
}
for (resource_index, resource) in migration.resources.iter().enumerate() {
if resource.id.is_empty() || resource.sql.is_empty() {
return std::result::Result::Err(crate::PostgresBackendError::new(crate::PostgresBackendErrorKind::MigrationMismatch, "registry_invalid"));
}
for previous in &migration.resources[..resource_index] {
if previous.id == resource.id {
return std::result::Result::Err(crate::PostgresBackendError::new(crate::PostgresBackendErrorKind::MigrationMismatch, "registry_invalid"));
}
}
}
expected_version = match expected_version.checked_add(1) {
std::option::Option::Some(value) => value,
std::option::Option::None => {
@@ -447,7 +621,7 @@ fn validate_history(history: &[AppliedMigration], migrations: &[EmbeddedMigratio
return std::result::Result::Err(crate::PostgresBackendError::new(crate::PostgresBackendErrorKind::SchemaNewer, "history_newer"));
}
let expected = &migrations[index];
let expected_checksum = migration_checksum(expected.sql);
let expected_checksum = migration_checksum(expected);
if applied.version != expected.version || applied.name != expected.name || applied.checksum != expected_checksum {
return std::result::Result::Err(crate::PostgresBackendError::new(crate::PostgresBackendErrorKind::MigrationMismatch, "history_diverged"));
}
@@ -456,59 +630,17 @@ fn validate_history(history: &[AppliedMigration], migrations: &[EmbeddedMigratio
return std::result::Result::Ok(index);
}
async fn verify_metadata_shape(transaction: &deadpool_postgres::Transaction<'_>) -> std::result::Result<(), crate::PostgresBackendError> {
let result = transaction.query(METADATA_SHAPE_SQL, &[]).await;
let rows = match result {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => {
return std::result::Result::Err(crate::PostgresBackendError::new(crate::PostgresBackendErrorKind::MigrationFailed, "metadata_shape"));
},
};
const REQUIRED: [(&str, &str, &str); 4] = [
("version", "bigint", "NO"),
("name", "text", "NO"),
("checksum", "text", "NO"),
("applied_at", "timestamp with time zone", "NO"),
];
let mut found = [false; REQUIRED.len()];
for row in rows {
let name_result = row.try_get::<usize, std::string::String>(0);
let data_type_result = row.try_get::<usize, std::string::String>(1);
let nullable_result = row.try_get::<usize, std::string::String>(2);
let (name, data_type, nullable) = match (name_result, data_type_result, nullable_result) {
(std::result::Result::Ok(name), std::result::Result::Ok(data_type), std::result::Result::Ok(nullable)) => (name, data_type, nullable),
_ => {
return std::result::Result::Err(crate::PostgresBackendError::new(crate::PostgresBackendErrorKind::MigrationFailed, "metadata_shape_decode"));
},
};
for (index, required) in REQUIRED.iter().enumerate() {
if name == required.0 && data_type == required.1 && nullable == required.2 {
found[index] = true;
break;
}
}
}
for required in found {
if !required {
return std::result::Result::Err(crate::PostgresBackendError::new(crate::PostgresBackendErrorKind::MigrationMismatch, "metadata_shape"));
}
}
let key_result = transaction.query_one(METADATA_PRIMARY_KEY_SQL, &[]).await;
let key_row = match key_result {
std::result::Result::Ok(value) => value,
std::result::Result::Err(_) => {
return std::result::Result::Err(crate::PostgresBackendError::new(crate::PostgresBackendErrorKind::MigrationFailed, "metadata_primary_key"));
},
};
let key_count = key_row.try_get::<usize, i64>(0);
let version_count = key_row.try_get::<usize, i64>(1);
return match (key_count, version_count) {
(std::result::Result::Ok(1), std::result::Result::Ok(1)) => std::result::Result::Ok(()),
(std::result::Result::Ok(_), std::result::Result::Ok(_)) => {
std::result::Result::Err(crate::PostgresBackendError::new(crate::PostgresBackendErrorKind::MigrationMismatch, "metadata_primary_key"))
},
_ => std::result::Result::Err(crate::PostgresBackendError::new(crate::PostgresBackendErrorKind::MigrationFailed, "metadata_primary_key_decode")),
};
fn log_schema_block(phase: &'static str) {
ksp_logging_lib::warn!(target: crate::TRACING_TARGET, phase, "PostgreSQL Store schema compatibility gate blocked automatic opening");
}
fn log_schema_resource_block(resource_id: &'static str, reason: &'static str) {
ksp_logging_lib::warn!(
target: crate::TRACING_TARGET,
resource_id,
reason,
"PostgreSQL Store schema resource requires manual reconciliation"
);
}
#[cfg(test)]

59
crates/ksp-store-postgres-lib/src/runtime.rs
View File

@@ -1,5 +1,5 @@
// file: crates/ksp-store-postgres-lib/src/runtime.rs
// version: 4
// version: 5
const APPLICATION_NAME: &str = "ksp-store";
const MAX_CONNECTION_URI_BYTES: usize = 4_096;
@@ -143,18 +143,19 @@ pub struct PostgresBackendSettings {
connect_timeout: std::time::Duration,
connection_uri: std::string::String,
create_timeout: std::time::Duration,
auto_migrate: bool,
max_connections: u32,
migration_lock_timeout: std::time::Duration,
migration_timeout: std::time::Duration,
network: ksp_store_api::RawNetworkId,
recycle_timeout: std::time::Duration,
schema_autocreate: bool,
schema_autoupdate: bool,
tls_mode: PostgresBackendTlsMode,
wait_timeout: std::time::Duration,
}
impl PostgresBackendSettings {
/// Creates the physical PostgreSQL settings bridge from already validated facade-owned values.
/// Creates the physical PostgreSQL settings bridge using the legacy single migration switch.
#[must_use]
pub fn new(
network: ksp_store_api::RawNetworkId,
@@ -169,8 +170,39 @@ impl PostgresBackendSettings {
migration_timeout: std::time::Duration,
migration_lock_timeout: std::time::Duration,
) -> Self {
return Self {
return Self::with_schema_policy(
network,
connection_uri,
max_connections,
connect_timeout,
wait_timeout,
create_timeout,
recycle_timeout,
tls_mode,
auto_migrate,
auto_migrate,
migration_timeout,
migration_lock_timeout,
);
}
/// Creates the physical PostgreSQL settings bridge with independent schema creation and update policies.
#[must_use]
pub fn with_schema_policy(
network: ksp_store_api::RawNetworkId,
connection_uri: impl std::convert::Into<std::string::String>,
max_connections: u32,
connect_timeout: std::time::Duration,
wait_timeout: std::time::Duration,
create_timeout: std::time::Duration,
recycle_timeout: std::time::Duration,
tls_mode: PostgresBackendTlsMode,
schema_autocreate: bool,
schema_autoupdate: bool,
migration_timeout: std::time::Duration,
migration_lock_timeout: std::time::Duration,
) -> Self {
return Self {
connect_timeout,
connection_uri: connection_uri.into(),
create_timeout,
@@ -179,6 +211,8 @@ impl PostgresBackendSettings {
migration_timeout,
network,
recycle_timeout,
schema_autocreate,
schema_autoupdate,
tls_mode,
wait_timeout,
};
@@ -203,7 +237,8 @@ impl std::fmt::Debug for PostgresBackendSettings {
.debug_struct("PostgresBackendSettings")
.field("network", &self.network)
.field("connection_uri", &"<redacted>")
.field("auto_migrate", &self.auto_migrate)
.field("schema_autocreate", &self.schema_autocreate)
.field("schema_autoupdate", &self.schema_autoupdate)
.field("max_connections", &self.max_connections)
.field("migration_timeout", &self.migration_timeout)
.field("migration_lock_timeout", &self.migration_lock_timeout)
@@ -263,8 +298,15 @@ impl PostgresBackend {
tls_mode = settings.tls_mode().code(),
"PostgreSQL Store backend established initial physical connection"
);
let bootstrap_result =
crate::bootstrap(&mut client, settings.network(), settings.auto_migrate, settings.migration_timeout, settings.migration_lock_timeout).await;
let bootstrap_result = crate::bootstrap(
&mut client,
settings.network(),
settings.schema_autocreate,
settings.schema_autoupdate,
settings.migration_timeout,
settings.migration_lock_timeout,
)
.await;
if let std::result::Result::Err(error) = bootstrap_result {
return std::result::Result::Err(error);
}
@@ -272,7 +314,8 @@ impl PostgresBackend {
ksp_logging_lib::debug!(
target: crate::TRACING_TARGET,
network = settings.network().as_str(),
auto_migrate = settings.auto_migrate,
schema_autocreate = settings.schema_autocreate,
schema_autoupdate = settings.schema_autoupdate,
"PostgreSQL Store migration/bootstrap foundation verified"
);
return std::result::Result::Ok(Self { network: settings.network, pool });

1146
crates/ksp-store-postgres-lib/src/schema.rs Normal file
View File

File diff suppressed because it is too large Load Diff

45
crates/ksp-store-postgres-lib/tests/dependency_boundary.rs
View File

@@ -1,5 +1,5 @@
// file: crates/ksp-store-postgres-lib/tests/dependency_boundary.rs
// version: 7
// version: 8
#![warn(missing_docs)]
#![deny(unreachable_pub)]
@@ -19,9 +19,12 @@ fn pre_005_backend_owns_exact_physical_runtime_dependencies_without_reverse_faca
assert!(!manifest.contains(forbidden), "forbidden PostgreSQL backend dependency detected: {forbidden}");
}
let migration = include_str!("../src/migration.rs");
let bootstrap_sql = include_str!("../migrations/V000__bootstrap.sql");
assert!(migration.contains("include_str!(\"../migrations/V000__bootstrap.sql\")"));
assert!(migration.contains("include_str!(\"../migrations/V001__raw_transaction.sql\")"));
let schema = include_str!("../src/schema.rs");
let bootstrap_sql = include_str!("../migrations/v000_bootstrap/tables/001_ksp_store_schema_migrations.sql");
assert!(migration.contains("crate::V000_RESOURCES"));
assert!(migration.contains("crate::V001_RESOURCES"));
assert!(schema.contains("../migrations/v000_bootstrap/tables/001_ksp_store_schema_migrations.sql"));
assert!(schema.contains("../migrations/v001_raw_transaction/tables/001_ksp_store_identity.sql"));
assert!(bootstrap_sql.contains("ksp_store_schema_migrations"));
for forbidden in ["RawTransaction", "RawAccountState", "raw_transaction", "raw_account", "CORE", "DECODE", "SPECIALIZED"] {
assert!(!bootstrap_sql.contains(forbidden), "business schema leaked into immutable V000 SQL: {forbidden}");
@@ -36,6 +39,7 @@ fn pre_005_backend_keeps_environment_sql_migrations_and_physical_types_private()
assert!(crate_root.contains("mod health;"));
assert!(crate_root.contains("mod migration;"));
assert!(crate_root.contains("mod runtime;"));
assert!(crate_root.contains("mod schema;"));
assert!(crate_root.contains("const _: &str = crate::TRACING_TARGET;"));
for forbidden in [
"pub mod ",
@@ -84,28 +88,37 @@ fn pre_007_health_probe_remains_foundation_only_and_private_sql() {
}
#[test]
fn pre_003_migration_engine_embeds_v001_and_binds_network_without_repository_scope() {
fn pre_003_fix_001_migration_engine_uses_split_schema_contract_and_binds_network_without_repository_scope() {
let migration = include_str!("../src/migration.rs");
let v001 = include_str!("../migrations/V001__raw_transaction.sql");
let schema = include_str!("../src/schema.rs");
let identity_table = include_str!("../migrations/v001_raw_transaction/tables/001_ksp_store_identity.sql");
let raw_table = include_str!("../migrations/v001_raw_transaction/tables/002_ksp_raw_transactions.sql");
let observation_table = include_str!("../migrations/v001_raw_transaction/tables/003_ksp_raw_transaction_observations.sql");
let archive_table = include_str!("../migrations/v001_raw_transaction/tables/004_ksp_raw_transaction_archive_payloads.sql");
let index = include_str!("../migrations/v001_raw_transaction/indexes/001_ix_ksp_raw_transactions_slot_signature.sql");
assert!(migration.contains("const EMBEDDED_MIGRATIONS: &[EmbeddedMigration]"));
assert!(migration.contains("MigrationHook::StoreIdentity"));
assert!(migration.contains("MigrationHookContext::AppliedNow"));
assert!(migration.contains("MigrationHookContext::Existing"));
assert!(migration.contains("schema_autocreate"));
assert!(migration.contains("schema_autoupdate"));
assert!(migration.contains("INSERT INTO ksp_store_identity (singleton, network) VALUES (1, $1)"));
assert!(migration.contains("SELECT singleton, network FROM ksp_store_identity ORDER BY singleton LIMIT 2"));
assert!(migration.contains("ksp_store_api::RawNetworkId::new(stored_network)"));
for required in [
"CREATE TABLE ksp_store_identity",
"CREATE TABLE ksp_raw_transactions",
"CREATE TABLE ksp_raw_transaction_observations",
"CREATE TABLE ksp_raw_transaction_archive_payloads",
"CREATE INDEX ix_ksp_raw_transactions_slot_signature",
] {
assert!(v001.contains(required), "missing V001 physical object: {required}");
}
assert!(schema.contains("SchemaResourceState"));
assert!(schema.contains("verify_v001_external_compatibility"));
assert!(identity_table.contains("CREATE TABLE IF NOT EXISTS ksp_store_identity"));
assert!(raw_table.contains("CREATE TABLE IF NOT EXISTS ksp_raw_transactions"));
assert!(observation_table.contains("CREATE TABLE IF NOT EXISTS ksp_raw_transaction_observations"));
assert!(archive_table.contains("CREATE TABLE IF NOT EXISTS ksp_raw_transaction_archive_payloads"));
assert!(index.contains("CREATE INDEX IF NOT EXISTS ix_ksp_raw_transactions_slot_signature"));
for forbidden in ["impl ksp_store_api::RawTransaction", "repository", "sqlx", "RawAccountState"] {
assert!(!migration.contains(forbidden), "repository/cross-scope implementation leaked into migration engine: {forbidden}");
assert!(!v001.contains(forbidden), "forbidden V001 scope content detected: {forbidden}");
assert!(!schema.contains(forbidden), "repository/cross-scope implementation leaked into schema contract: {forbidden}");
}
for removed in ["migrations/V000__bootstrap.sql", "migrations/V001__raw_transaction.sql"] {
let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join(removed);
assert!(!path.exists(), "obsolete monolithic migration must be deleted by pre.003-fix.001: {removed}");
}
return;
}

10
crates/ksp-store-postgres-lib/tests/hardening_completeness.rs
View File

@@ -1,5 +1,5 @@
// file: crates/ksp-store-postgres-lib/tests/hardening_completeness.rs
// version: 2
// version: 3
#![warn(missing_docs)]
#![deny(unreachable_pub)]
@@ -109,7 +109,7 @@ fn assert_pre_io_rejection(connection_uri: &str, tls_mode: ksp_store_postgres_li
#[test]
fn pre_009_backend_modules_exports_and_manifest_dependencies_are_exact() {
let crate_root = include_str!("../src/lib.rs");
for required in ["mod constants;", "mod error;", "mod health;", "mod migration;", "mod runtime;"] {
for required in ["mod constants;", "mod error;", "mod health;", "mod migration;", "mod runtime;", "mod schema;"] {
assert!(crate_root.contains(required), "missing PostgreSQL backend module: {required}");
}
assert!(!crate_root.contains("pub mod "));
@@ -183,12 +183,14 @@ fn pre_009_backend_has_no_env_bypass_or_business_persistence_capability() {
{}
{}
{}
{}
{}",
include_str!("../src/error.rs"),
include_str!("../src/health.rs"),
include_str!("../src/lib.rs"),
include_str!("../src/migration.rs"),
include_str!("../src/runtime.rs")
include_str!("../src/runtime.rs"),
include_str!("../src/schema.rs")
);
for forbidden in [
"std::env",
@@ -212,7 +214,7 @@ fn pre_009_backend_has_no_env_bypass_or_business_persistence_capability() {
] {
assert!(!production.contains(forbidden), "forbidden backend ownership/capability material detected: {forbidden}");
}
let bootstrap_sql = include_str!("../migrations/V000__bootstrap.sql");
let bootstrap_sql = include_str!("../migrations/v000_bootstrap/tables/001_ksp_store_schema_migrations.sql");
assert!(bootstrap_sql.contains("ksp_store_schema_migrations"));
for forbidden in ["RawTransaction", "RawAccountState", "raw_transaction", "raw_account", "CORE", "DECODE", "SPECIALIZED"] {
assert!(!bootstrap_sql.contains(forbidden), "business schema leaked into foundation migration: {forbidden}");

5
crates/ksp-store-postgres-lib/tests/public_api.rs
View File

@@ -62,9 +62,6 @@ fn pre_007_backend_health_bridge_exposes_only_safe_snapshot_types() {
#[test]
fn pre_003_retention_compaction_error_code_matches_store_contract_value() {
assert_eq!(ksp_store_postgres_lib::ERROR_CODE_POSTGRES_RETENTION_COMPACTION_UNSUPPORTED.domain(), "store");
assert_eq!(
ksp_store_postgres_lib::ERROR_CODE_POSTGRES_RETENTION_COMPACTION_UNSUPPORTED.code(),
"postgres_retention_compaction_unsupported",
);
assert_eq!(ksp_store_postgres_lib::ERROR_CODE_POSTGRES_RETENTION_COMPACTION_UNSUPPORTED.code(), "postgres_retention_compaction_unsupported",);
return;
}

106
crates/ksp-store-postgres-lib/unit_tests/migration.rs
View File

@@ -1,89 +1,103 @@
// file: crates/ksp-store-postgres-lib/unit_tests/migration.rs
// version: 3
// version: 4
fn applied(version: i64, name: &str, checksum: &str) -> super::AppliedMigration {
return super::AppliedMigration { checksum: checksum.to_owned(), name: name.to_owned(), version };
}
fn embedded(version: i64, name: &'static str, sql: &'static str) -> super::EmbeddedMigration {
return super::EmbeddedMigration { hook: super::MigrationHook::None, name, sql, version };
fn embedded(version: i64, name: &'static str, resources: &'static [crate::SchemaResource]) -> super::EmbeddedMigration {
return super::EmbeddedMigration {
checksum: super::MigrationChecksum::Resources,
hook: super::MigrationHook::None,
name,
resources,
version,
};
}
#[test]
fn pre_003_embedded_registry_keeps_v000_immutable_and_adds_exact_v001() {
fn pre_003_fix_001_embedded_registry_keeps_v000_checksum_and_uses_resource_owned_v001() {
assert_eq!(super::EMBEDDED_MIGRATIONS.len(), 2);
let v000 = &super::EMBEDDED_MIGRATIONS[0];
assert_eq!(v000.version, 0);
assert_eq!(v000.name, "bootstrap");
assert_eq!(v000.hook, super::MigrationHook::None);
assert!(v000.sql.contains("CREATE TABLE ksp_store_schema_migrations"));
for forbidden in ["RawTransaction", "RawAccountState", "raw_transaction", "raw_account", "CORE", "DECODE", "SPECIALIZED"] {
assert!(!v000.sql.contains(forbidden), "business schema leaked into immutable V000 SQL: {forbidden}");
}
assert_eq!(super::migration_checksum(v000.sql), "d29068b8c13b9dc0cc9ef6aaadd0fa12d41e0fe4c56541a1118c4bfc846a1450");
assert_eq!(v000.resources.len(), 1);
assert!(v000.resources[0].sql.contains("CREATE TABLE ksp_store_schema_migrations"));
assert_eq!(super::migration_checksum(v000), "d29068b8c13b9dc0cc9ef6aaadd0fa12d41e0fe4c56541a1118c4bfc846a1450");
let v001 = &super::EMBEDDED_MIGRATIONS[1];
assert_eq!(v001.version, 1);
assert_eq!(v001.name, "raw_transaction");
assert_eq!(v001.hook, super::MigrationHook::StoreIdentity);
assert_eq!(super::migration_checksum(v001.sql), "6fe57ed0313d2ed295280dd6e49f6d86695d4e4effee2724a25e36db1ea17761");
assert_eq!(v001.resources.len(), 40);
assert_eq!(super::migration_checksum(v001), "31488cda2f08f3f46c4cdbdbb6c18c243662fada02eac4487040c8735d72cc51");
assert!(super::validate_embedded_registry(super::EMBEDDED_MIGRATIONS).is_ok());
assert_eq!(crate::current_migration_version(), 1);
return;
}
#[test]
fn pre_003_v001_inventory_indexes_and_api_bounds_are_exact() {
let sql = super::EMBEDDED_MIGRATIONS[1].sql;
fn pre_003_fix_001_v001_resource_order_and_api_bounds_are_exact() {
let v001 = &super::EMBEDDED_MIGRATIONS[1];
let ids = v001.resources.iter().map(|resource| return resource.id).collect::<std::vec::Vec<_>>();
assert_eq!(ids.iter().filter(|id| return id.starts_with("tables/")).count(), 4);
assert_eq!(ids.iter().filter(|id| return id.starts_with("constraints/")).count(), 35);
assert_eq!(ids.iter().filter(|id| return id.starts_with("indexes/")).count(), 1);
assert!(ids[..4].iter().all(|id| return id.starts_with("tables/")));
assert!(ids[4..39].iter().all(|id| return id.starts_with("constraints/")));
assert!(ids[39..].iter().all(|id| return id.starts_with("indexes/")));
let mut unique = std::collections::BTreeSet::<&str>::new();
for id in &ids {
assert!(unique.insert(id), "duplicate embedded migration resource: {id}");
}
let sql = v001.resources.iter().map(|resource| return resource.sql).collect::<std::vec::Vec<_>>().concat();
for required in [
"CREATE TABLE ksp_store_identity",
"CREATE TABLE ksp_raw_transactions",
"CREATE TABLE ksp_raw_transaction_observations",
"CREATE TABLE ksp_raw_transaction_archive_payloads",
"CREATE INDEX ix_ksp_raw_transactions_slot_signature",
"CREATE TABLE IF NOT EXISTS ksp_store_identity",
"CREATE TABLE IF NOT EXISTS ksp_raw_transactions",
"CREATE TABLE IF NOT EXISTS ksp_raw_transaction_observations",
"CREATE TABLE IF NOT EXISTS ksp_raw_transaction_archive_payloads",
"CREATE INDEX IF NOT EXISTS ix_ksp_raw_transactions_slot_signature",
"WHERE retention_state <> 'purged'",
"octet_length(signature) = 64",
"slot BETWEEN 0 AND 18446744073709551615",
"block_time_unix_millis BETWEEN 0 AND 253402300799999",
"slot >= 0 AND slot <= 18446744073709551615",
"block_time_unix_millis >= 0 AND block_time_unix_millis <= 253402300799999",
"octet_length(content_hash) = 32",
"octet_length(payload) BETWEEN 1 AND 16777216",
"format_version BETWEEN 1 AND 4294967295",
"received_at_unix_millis BETWEEN 0 AND 253402300799999",
"source_payload_size_bytes BETWEEN 0 AND 67108864",
"origin IN ('backfill', 'import', 'live', 'repair', 'replay')",
"retention_state IN ('full', 'archived', 'purged')",
"octet_length(payload) >= 1 AND octet_length(payload) <= 16777216",
"format_version >= 1 AND format_version <= 4294967295",
"received_at_unix_millis >= 0 AND received_at_unix_millis <= 253402300799999",
"source_payload_size_bytes >= 0 AND source_payload_size_bytes <= 67108864",
"origin = 'backfill' OR origin = 'import' OR origin = 'live' OR origin = 'repair' OR origin = 'replay'",
"retention_state = 'full' OR retention_state = 'archived' OR retention_state = 'purged'",
] {
assert!(sql.contains(required), "V001 physical contract is missing: {required}");
}
assert_eq!(sql.matches("PRIMARY KEY").count(), 4);
assert_eq!(sql.matches("REFERENCES ksp_raw_transactions(signature) ON DELETE RESTRICT").count(), 2);
assert!(!sql.contains("compacted"));
assert!(!sql.contains("BIGSERIAL"));
assert!(!sql.contains("slot BIGINT"));
assert!(sql.contains("network TEXT NOT NULL"));
return;
}
#[test]
fn pre_003_ordered_registry_accepts_v000_prefix_and_full_v001_history() {
fn pre_003_fix_001_ordered_registry_accepts_v000_prefix_and_full_v001_history() {
let v000 = super::EMBEDDED_MIGRATIONS[0];
let v001 = super::EMBEDDED_MIGRATIONS[1];
let v000_checksum = super::migration_checksum(v000.sql);
let v000_checksum = super::migration_checksum(&v000);
let prefix = [applied(0, v000.name, v000_checksum.as_str())];
assert_eq!(super::validate_history(&prefix, super::EMBEDDED_MIGRATIONS).ok(), std::option::Option::Some(1));
let v001_checksum = super::migration_checksum(v001.sql);
let v001_checksum = super::migration_checksum(&v001);
let full = [applied(0, v000.name, v000_checksum.as_str()), applied(1, v001.name, v001_checksum.as_str())];
assert_eq!(super::validate_history(&full, super::EMBEDDED_MIGRATIONS).ok(), std::option::Option::Some(2));
return;
}
#[test]
fn pre_002_registry_rejects_empty_nonzero_gap_and_empty_metadata_entries() {
fn pre_003_fix_001_registry_rejects_empty_nonzero_gap_empty_metadata_and_empty_resources() {
let empty: [super::EmbeddedMigration; 0] = [];
let starts_at_one = [embedded(1, "future", "SELECT 1;")];
let gap = [embedded(0, "bootstrap", "SELECT 0;"), embedded(2, "future", "SELECT 2;")];
let empty_name = [embedded(0, "", "SELECT 0;")];
let empty_sql = [embedded(0, "bootstrap", "")];
for registry in [&empty[..], &starts_at_one[..], &gap[..], &empty_name[..], &empty_sql[..]] {
let starts_at_one = [embedded(1, "future", crate::V000_RESOURCES)];
let gap = [embedded(0, "bootstrap", crate::V000_RESOURCES), embedded(2, "future", crate::V000_RESOURCES)];
let empty_name = [embedded(0, "", crate::V000_RESOURCES)];
let empty_resources = [embedded(0, "bootstrap", &[])];
for registry in [&empty[..], &starts_at_one[..], &gap[..], &empty_name[..], &empty_resources[..]] {
let result = super::validate_embedded_registry(registry);
assert_eq!(result.err().map(|value| return value.kind()), std::option::Option::Some(crate::PostgresBackendErrorKind::MigrationMismatch));
}
@@ -91,11 +105,11 @@ fn pre_002_registry_rejects_empty_nonzero_gap_and_empty_metadata_entries() {
}
#[test]
fn pre_003_divergent_missing_or_gapped_history_is_terminal_mismatch() {
fn pre_003_fix_001_divergent_missing_or_gapped_history_is_terminal_mismatch() {
let v000 = super::EMBEDDED_MIGRATIONS[0];
let v001 = super::EMBEDDED_MIGRATIONS[1];
let v000_checksum = super::migration_checksum(v000.sql);
let v001_checksum = super::migration_checksum(v001.sql);
let v000_checksum = super::migration_checksum(&v000);
let v001_checksum = super::migration_checksum(&v001);
let wrong_name = [applied(0, "changed", v000_checksum.as_str())];
let wrong_checksum = [applied(0, v000.name, "00")];
let missing: [super::AppliedMigration; 0] = [];
@@ -108,16 +122,12 @@ fn pre_003_divergent_missing_or_gapped_history_is_terminal_mismatch() {
}
#[test]
fn pre_003_newer_history_is_rejected_without_down_migration() {
fn pre_003_fix_001_newer_history_is_rejected_without_down_migration() {
let v000 = super::EMBEDDED_MIGRATIONS[0];
let v001 = super::EMBEDDED_MIGRATIONS[1];
let v000_checksum = super::migration_checksum(v000.sql);
let v001_checksum = super::migration_checksum(v001.sql);
let history = [
applied(0, v000.name, v000_checksum.as_str()),
applied(1, v001.name, v001_checksum.as_str()),
applied(2, "future", "future-checksum"),
];
let v000_checksum = super::migration_checksum(&v000);
let v001_checksum = super::migration_checksum(&v001);
let history = [applied(0, v000.name, v000_checksum.as_str()), applied(1, v001.name, v001_checksum.as_str()), applied(2, "future", "future-checksum")];
let result = super::validate_history(&history, super::EMBEDDED_MIGRATIONS);
assert_eq!(result.err().map(|value| return value.kind()), std::option::Option::Some(crate::PostgresBackendErrorKind::SchemaNewer));
return;

114
crates/ksp-store-postgres-lib/unit_tests/schema.rs Normal file
View File

@@ -0,0 +1,114 @@
// file: crates/ksp-store-postgres-lib/unit_tests/schema.rs
// version: 1
fn actual_column(name: &str, udt_name: &str, nullable: bool) -> super::ActualColumn {
return super::ActualColumn {
default: std::option::Option::None,
generated: "NEVER".to_owned(),
identity: "NO".to_owned(),
name: name.to_owned(),
nullable,
numeric_precision: std::option::Option::None,
numeric_scale: std::option::Option::None,
udt_name: udt_name.to_owned(),
};
}
#[test]
fn pre_003_fix_001_v000_resource_is_relocated_without_changing_legacy_sql() {
assert_eq!(crate::V000_RESOURCES.len(), 1);
let resource = crate::V000_RESOURCES[0];
assert_eq!(resource.id, "tables/001_ksp_store_schema_migrations.sql");
assert_eq!(
resource.sql,
"CREATE TABLE ksp_store_schema_migrations (\n version BIGINT PRIMARY KEY,\n name TEXT NOT NULL,\n checksum TEXT NOT NULL,\n applied_at TIMESTAMPTZ NOT NULL\n);\n",
);
return;
}
#[test]
fn pre_003_fix_001_v001_resources_are_split_and_idempotent_by_object_family() {
assert_eq!(crate::V001_RESOURCES.len(), 40);
let table_resources = crate::V001_RESOURCES.iter().filter(|resource| return resource.id.starts_with("tables/")).collect::<std::vec::Vec<_>>();
let constraint_resources = crate::V001_RESOURCES.iter().filter(|resource| return resource.id.starts_with("constraints/")).collect::<std::vec::Vec<_>>();
let index_resources = crate::V001_RESOURCES.iter().filter(|resource| return resource.id.starts_with("indexes/")).collect::<std::vec::Vec<_>>();
assert_eq!(table_resources.len(), 4);
assert_eq!(constraint_resources.len(), 35);
assert_eq!(index_resources.len(), 1);
for resource in table_resources {
assert!(resource.sql.contains("CREATE TABLE IF NOT EXISTS"));
assert!(resource.sql.contains("ADD COLUMN IF NOT EXISTS"));
}
for resource in constraint_resources {
assert!(resource.sql.contains("IF NOT EXISTS"));
}
for resource in index_resources {
assert!(resource.sql.contains("CREATE INDEX IF NOT EXISTS"));
}
return;
}
#[test]
fn pre_003_fix_001_external_extra_columns_are_accepted_only_when_they_cannot_block_ksp_inserts() {
let nullable = actual_column("external_nullable", "text", true);
assert!(nullable.is_non_blocking_extra());
let mut with_default = actual_column("external_default", "text", true);
with_default.default = std::option::Option::Some("'x'::text".to_owned());
assert!(!with_default.is_non_blocking_extra());
let mut identity = actual_column("external_identity", "int8", true);
identity.identity = "YES".to_owned();
assert!(!identity.is_non_blocking_extra());
let mut generated = actual_column("external_generated", "text", true);
generated.generated = "ALWAYS".to_owned();
assert!(!generated.is_non_blocking_extra());
let blocking = actual_column("external_required", "text", false);
assert!(!blocking.is_non_blocking_extra());
return;
}
#[test]
fn pre_003_fix_001_required_column_matching_ignores_integer_precision_but_requires_numeric_20_0() {
let expected_int = super::ColumnContract {
name: "version",
nullable: false,
numeric_precision: std::option::Option::None,
numeric_scale: std::option::Option::None,
udt_name: "int8",
};
let mut actual_int = actual_column("version", "int8", false);
actual_int.numeric_precision = std::option::Option::Some(64);
actual_int.numeric_scale = std::option::Option::Some(0);
assert!(actual_int.matches(&expected_int));
let expected_numeric = super::ColumnContract {
name: "slot",
nullable: false,
numeric_precision: std::option::Option::Some(20),
numeric_scale: std::option::Option::Some(0),
udt_name: "numeric",
};
let mut actual_numeric = actual_column("slot", "numeric", false);
actual_numeric.numeric_precision = std::option::Option::Some(20);
actual_numeric.numeric_scale = std::option::Option::Some(0);
assert!(actual_numeric.matches(&expected_numeric));
actual_numeric.numeric_precision = std::option::Option::Some(19);
assert!(!actual_numeric.matches(&expected_numeric));
return;
}
#[test]
fn pre_003_fix_001_catalog_normalization_and_resource_owned_constraint_definition_are_deterministic() {
let normalized = super::normalize_catalog_sql("CHECK ((slot >= (0)::numeric) AND (slot <= (18446744073709551615)::numeric))");
assert_eq!(normalized, "checkslot>=0andslot<=18446744073709551615");
let resource = crate::V001_RESOURCES.iter().find(|resource| return resource.id == "constraints/006_ck_ksp_raw_transactions_slot.sql");
let resource = match resource {
std::option::Option::Some(value) => value,
std::option::Option::None => {
assert!(false, "V001 slot constraint resource must remain embedded");
return;
},
};
let expected = super::expected_constraint_definition(resource.sql, "ck_ksp_raw_transactions_slot");
assert_eq!(expected.as_deref(), std::option::Option::Some(normalized.as_str()));
assert_ne!(expected.as_deref(), std::option::Option::Some("checkslot>=0andslot<=10"));
return;
}

273
deltas/0.3.3/pre.003-fix.001.md Normal file
View File

@@ -0,0 +1,273 @@
<!-- file: deltas/0.3.3/pre.003-fix.001.md -->
<!-- version: 1 -->
# Delta `0.3.3-pre.003-fix.001` — Store/PostgreSQL schema compatibility hardening
## 1. Base et objectif
Ce correctif s'applique **par-dessus le commit `0.3.3-pre.003` déjà réalisé**. Il ne réécrit ni ne remplace ce commit dans l'historique.
Le gate opérateur de `pre.002` est vert. Le gate opérateur de `pre.003` n'a pas été fourni avant l'ouverture de ce fix ; aucune commande Cargo du fix n'est donc déclarée PASS dans ce document.
Objectif du fix : conserver les bonnes garanties fonctionnelles historiques d'inspection/réconciliation du schéma PostgreSQL sans reprendre l'architecture kbot3, et corriger V001 avant l'ouverture du repository `RawTransaction` de `pre.004`.
Version workspace après application :
```text
0.3.3-pre.3.fix.1
```
Identifiant de livraison :
```text
0.3.3-pre.003-fix.001
```
## 2. Décisions corrigées
### 2.1 Migration logique vs ressources physiques
V000 et V001 restent des **migrations logiques** versionnées. Les objets physiques ne deviennent pas chacun une version de migration.
Arborescence retenue :
```text
crates/ksp-store-postgres-lib/migrations/
v000_bootstrap/
tables/
001_ksp_store_schema_migrations.sql
v001_raw_transaction/
tables/
001_ksp_store_identity.sql
002_ksp_raw_transactions.sql
003_ksp_raw_transaction_observations.sql
004_ksp_raw_transaction_archive_payloads.sql
constraints/
001_...sql -> 035_...sql
indexes/
001_ix_ksp_raw_transactions_slot_signature.sql
```
V001 contient exactement :
```text
4 tables
35 contraintes
1 index
40 ressources embedded
```
V000 est seulement déplacée : ses bytes restent strictement identiques.
### 2.2 Checksums
V000 historique :
```text
d29068b8c13b9dc0cc9ef6aaadd0fa12d41e0fe4c56541a1118c4bfc846a1450
```
V001 multi-ressources :
```text
31488cda2f08f3f46c4cdbdbb6c18c243662fada02eac4487040c8735d72cc51
```
Le checksum V001 est calculé sur le préfixe de format du registre puis, dans l'ordre embedded, sur l'identifiant relatif et les bytes exacts de chaque ressource. Le checksum monolithique `pre.003` n'est pas réutilisé.
Une base ayant réellement enregistré l'ancien checksum V001 du commit `pre.003` n'est jamais réécrite silencieusement : elle doit être réinitialisée si elle est jetable ou réconciliée manuellement.
### 2.3 `IF NOT EXISTS` n'est pas une preuve de compatibilité
Les ressources utilisent `CREATE ... IF NOT EXISTS`, `ADD COLUMN IF NOT EXISTS` ou des guards catalogue lorsque cela permet une création additive idempotente.
Après et avant mutation, le backend introspecte néanmoins le catalogue PostgreSQL. Chaque ressource est classée :
```text
Compatible
Missing
Incompatible
```
Un objet préexistant du bon nom mais de mauvaise définition est donc refusé.
### 2.4 Contrat physique vérifié
Le module backend-private `schema` contrôle notamment :
- tables gérées et nature `BASE TABLE` ;
- colonnes obligatoires, type, nullabilité et absence d'identity/generated sur le contrat KSP ;
- précision/scale `NUMERIC(20,0)` du slot ;
- PK, FK et CHECK ;
- définition canonique des contraintes à partir de leur propre ressource SQL ;
- index partiel `(slot, signature)` hors `purged` ;
- RLS ;
- triggers utilisateur actifs ;
- rewrite rules ;
- contraintes/indexes uniques externes susceptibles de modifier la sémantique de write.
Les marqueurs COMMENT ne servent pas de preuve de compatibilité et ne sont pas requis.
### 2.5 Extensions externes compatibles
KSP ne requiert pas une égalité byte-for-byte du schéma complet avec son propre DDL. Les extensions externes qui sont prouvées non bloquantes restent admises.
Le correctif tolère notamment :
- une colonne externe nullable, sans default, identity ni generated expression ;
- un index externe non unique ;
- une contrainte attendue physiquement équivalente même si son nom diffère.
Le correctif bloque conservativement les extensions susceptibles de modifier/contraindre les writes KSP lorsqu'il ne peut pas en prouver l'innocuité : mauvais type/nullabilité sur une colonne KSP, colonne externe write-blocking, contrainte non équivalente, unique index autonome, trigger actif, rule ou RLS.
### 2.6 Politiques `schema_autocreate` / `schema_autoupdate`
`std.store` passe en format V2 et sépare :
```text
schema_autocreate
schema_autoupdate
```
`schema_autocreate` autorise l'initialisation ou l'adoption contrôlée d'une base sans metadata KSP.
`schema_autoupdate` autorise les migrations pending et les réparations additives sûres de ressources manquantes sur un schéma KSP existant.
Une incompatibilité exigeant une mutation destructive, ambiguë ou non prouvée sûre reste bloquante même avec `schema_autoupdate=true`; le backend émet uniquement une phase/resource sûre pour permettre l'intervention manuelle.
Le format V1 reste lisible : `auto_migrate` est mappé vers les deux politiques. Les constructeurs Rust historiques restent également disponibles et mappent leur booléen vers les deux politiques.
### 2.7 Identité réseau
Le binding V001 reste sous la même transaction et le même advisory lock.
Une `ksp_store_identity` absente alors que V001 est déjà enregistrée n'est jamais reconstruite automatiquement, même avec `schema_autoupdate=true`, afin d'interdire tout rebind réseau silencieux.
## 3. Suppressions explicites après application de l'overlay
L'extraction d'un zip n'efface pas les anciens fichiers. Après extraction du delta à la racine du dépôt, supprimer explicitement :
```bash
rm crates/ksp-store-postgres-lib/migrations/V000__bootstrap.sql
rm crates/ksp-store-postgres-lib/migrations/V001__raw_transaction.sql
```
Fichiers supprimés :
- `crates/ksp-store-postgres-lib/migrations/V000__bootstrap.sql`
- `crates/ksp-store-postgres-lib/migrations/V001__raw_transaction.sql`
## 4. Fichiers modifiés
- `Cargo.toml`
- `config/examples/std.store.example.json`
- `config/schemas/std.store.schema.json`
- `config/std.store.json`
- `crates/ksp-config-lib/src/store.rs`
- `crates/ksp-config-lib/unit_tests/fixtures/std.store.json`
- `crates/ksp-config-lib/unit_tests/store.rs`
- `crates/ksp-store-lib/src/settings.rs`
- `crates/ksp-store-lib/src/store.rs`
- `crates/ksp-store-lib/unit_tests/settings.rs`
- `crates/ksp-store-postgres-lib/src/lib.rs`
- `crates/ksp-store-postgres-lib/src/migration.rs`
- `crates/ksp-store-postgres-lib/src/runtime.rs`
- `crates/ksp-store-postgres-lib/tests/dependency_boundary.rs`
- `crates/ksp-store-postgres-lib/tests/hardening_completeness.rs`
- `crates/ksp-store-postgres-lib/unit_tests/migration.rs`
- `docs/plans/024-V0_3_3_STORE_POSTGRES_RAW_TRANSACTION_PLAN.md`
- `docs/validation/020-V0_3_3_STORE_POSTGRES_RAW_TRANSACTION.md`
## 5. Fichiers ajoutés
- `crates/ksp-store-postgres-lib/migrations/v000_bootstrap/tables/001_ksp_store_schema_migrations.sql`
- `crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/001_pk_ksp_store_identity.sql`
- `crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/002_ck_ksp_store_identity_singleton.sql`
- `crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/003_ck_ksp_store_identity_network.sql`
- `crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/004_pk_ksp_raw_transactions.sql`
- `crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/005_ck_ksp_raw_transactions_signature.sql`
- `crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/006_ck_ksp_raw_transactions_slot.sql`
- `crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/007_ck_ksp_raw_transactions_block_time.sql`
- `crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/008_ck_ksp_raw_transactions_format_id.sql`
- `crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/009_ck_ksp_raw_transactions_format_version.sql`
- `crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/010_ck_ksp_raw_transactions_content_hash.sql`
- `crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/011_ck_ksp_raw_transactions_payload.sql`
- `crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/012_ck_ksp_raw_transactions_retention_state.sql`
- `crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/013_ck_ksp_raw_transactions_payload_state.sql`
- `crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/014_ck_ksp_raw_transactions_purged_block_time.sql`
- `crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/015_pk_ksp_raw_transaction_observations.sql`
- `crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/016_fk_ksp_raw_transaction_observations_transaction.sql`
- `crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/017_ck_ksp_raw_transaction_observations_key.sql`
- `crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/018_ck_ksp_raw_transaction_observations_signature.sql`
- `crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/019_ck_ksp_raw_transaction_observations_provider.sql`
- `crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/020_ck_ksp_raw_transaction_observations_protocol.sql`
- `crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/021_ck_ksp_raw_transaction_observations_method.sql`
- `crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/022_ck_ksp_raw_transaction_observations_origin.sql`
- `crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/023_ck_ksp_raw_transaction_observations_received_at.sql`
- `crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/024_ck_ksp_raw_transaction_observations_capture_session.sql`
- `crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/025_ck_ksp_raw_transaction_observations_commitment.sql`
- `crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/026_ck_ksp_raw_transaction_observations_endpoint.sql`
- `crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/027_ck_ksp_raw_transaction_observations_filter.sql`
- `crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/028_ck_ksp_raw_transaction_observations_observed_at.sql`
- `crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/029_ck_ksp_raw_transaction_observations_time_order.sql`
- `crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/030_ck_ksp_raw_transaction_observations_source_hash.sql`
- `crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/031_ck_ksp_raw_transaction_observations_source_size.sql`
- `crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/032_pk_ksp_raw_transaction_archive_payloads.sql`
- `crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/033_fk_ksp_raw_transaction_archive_payloads_transaction.sql`
- `crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/034_ck_ksp_raw_transaction_archive_payloads_signature.sql`
- `crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/constraints/035_ck_ksp_raw_transaction_archive_payloads_payload.sql`
- `crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/indexes/001_ix_ksp_raw_transactions_slot_signature.sql`
- `crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/tables/001_ksp_store_identity.sql`
- `crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/tables/002_ksp_raw_transactions.sql`
- `crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/tables/003_ksp_raw_transaction_observations.sql`
- `crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/tables/004_ksp_raw_transaction_archive_payloads.sql`
- `crates/ksp-store-postgres-lib/src/schema.rs`
- `crates/ksp-store-postgres-lib/unit_tests/schema.rs`
- `deltas/0.3.3/pre.003-fix.001.md`
## 6. Validation de génération
Les validations réalisables dans l'environnement de génération doivent être propres avant publication de l'archive :
```text
python3 scripts/audit_rust_workspace_rules.py
python3 scripts/audit_markdown_tables.py README.md RULES.md ROADMAP.md CHANGELOG.md docs prompts crates deltas/0.3.3
validation JSON/JSON Schema std.store V1+V2
inventaire/checksum migration V000/V001
```
L'environnement de génération ne possède pas `cargo`/`rustfmt`. Les gates Cargo ci-dessous restent donc **NON EXÉCUTÉS** jusqu'au passage opérateur.
## 7. Gate opérateur demandé
Après extraction de l'overlay, suppression explicite des deux fichiers monolithiques puis application du formatage :
```bash
cargo fmt --all
python3 scripts/audit_rust_workspace_rules.py
python3 scripts/audit_markdown_tables.py README.md RULES.md ROADMAP.md CHANGELOG.md docs prompts crates deltas/0.3.3
cargo check --workspace
cargo clippy --workspace --all-targets
cargo test -p ksp-store-api
cargo test -p ksp-store-lib
cargo test -p ksp-store-postgres-lib
cargo test -p ksp-config-lib
cargo check -p ksp-store-lib --no-default-features
```
Canaris de suppression :
```bash
test ! -e crates/ksp-store-postgres-lib/migrations/V000__bootstrap.sql
test ! -e crates/ksp-store-postgres-lib/migrations/V001__raw_transaction.sql
```
Le vrai PostgreSQL live du contrat V001, des dérives et des réparations reste volontairement réservé au gate live prévu plus tard dans `0.3.3`; ce fix ne le présente pas comme PASS.
## 8. Suite
Après gate opérateur vert de `pre.003-fix.001`, reprendre le séquencement avec :
```text
0.3.3-pre.004 — mapping SQL privé + lectures RawTransaction/observation/rétention/tombstone
```

47
docs/plans/024-V0_3_3_STORE_POSTGRES_RAW_TRANSACTION_PLAN.md
View File

@@ -1,5 +1,5 @@
<!-- file: docs/plans/024-V0_3_3_STORE_POSTGRES_RAW_TRANSACTION_PLAN.md -->
<!-- version: 3 -->
<!-- version: 4 -->
# Plan `0.3.3` — Store/PostgreSQL RawTransaction vertical slice
@@ -159,15 +159,17 @@ SHA-256 = d29068b8c13b9dc0cc9ef6aaadd0fa12d41e0fe4c56541a1118c4bfc846a1450
Le SQL crée uniquement `ksp_store_schema_migrations`. Le moteur stable considère toute migration appliquée `> 0` comme `SchemaNewer`; il doit donc être généralisé avant V001.
Convention figée pour la nouvelle ressource :
Convention initialement retenue en `pre.001`, puis corrigée par `pre.003-fix.001` avant toute tranche repository : V001 reste une migration logique unique (`version = 1`, `name = raw_transaction`) mais ses ressources physiques sont séparées et ordonnées sous :
```text
crates/ksp-store-postgres-lib/migrations/V001__raw_transaction.sql
version = 1
name = raw_transaction
checksum SHA-256 calculé sur les bytes embedded exacts
crates/ksp-store-postgres-lib/migrations/v001_raw_transaction/
tables/
constraints/
indexes/
```
Le checksum logique V001 est calculé de manière déterministe sur l'identifiant relatif et les bytes exacts de chaque ressource embedded dans l'ordre du registre. Cette correction ne transforme pas chaque table/contrainte/index en version de migration autonome.
### 5.3 Config `std.store` final
`config/std.store.json` est `format_version = 1`, profil par défaut `devnet` et contient exactement :
@@ -1040,17 +1042,36 @@ Statut matérialisé en `pre.002` :
- tests de checksum/inventory/bounds ;
- matérialiser `store.postgres_retention_compaction_unsupported`, sans modifier les contrats stables sauf preuve nouvelle d'un blocage réel.
Statut matérialisé en `pre.003` :
Statut du commit `pre.003` avant correctif :
- registre réel `[V000, V001]`, version courante `1` ;
- V000 reste byte-identique avec son checksum historique ;
- V001 est embarquée sous `migrations/V001__raw_transaction.sql` avec checksum SHA-256 `6fe57ed0313d2ed295280dd6e49f6d86695d4e4effee2724a25e36db1ea17761` ;
- V001 crée exactement `ksp_store_identity`, `ksp_raw_transactions`, `ksp_raw_transaction_observations`, `ksp_raw_transaction_archive_payloads` et l'index partiel `(slot, signature)` hors `purged` ;
- `slot` reste `NUMERIC(20,0)` et toutes les bornes physiques suivent les invariants `ksp-store-api` sans narrowing ;
- le hook V001 crée puis valide l'identité en `AppliedNow`, et valide seulement en `Existing` ; absence, forme invalide ou réseau différent deviennent un `MigrationMismatch` sûr sans rendre la valeur persistée ;
- `store.postgres_retention_compaction_unsupported` est réservé par valeur dans le backend et la façade, sans reverse dependency ;
- V000 byte-identique avec son checksum historique ;
- V001 monolithique possédait le checksum `6fe57ed0313d2ed295280dd6e49f6d86695d4e4effee2724a25e36db1ea17761` ;
- les quatre tables, contraintes et l'index partiel étaient créés, mais l'historique de migration ne prouvait pas à lui seul la compatibilité physique des objets réellement présents ;
- le hook V001 créait/validait déjà l'identité réseau sans rebind silencieuse ;
- aucune capability/repository `RawTransaction*` n'était encore implémentée.
### `0.3.3-pre.003-fix.001` — correction du contrat physique
Le correctif s'applique au commit `pre.003` et remplace la matérialisation SQL sans changer le numéro logique V001 :
- V000 est déplacée sous `migrations/v000_bootstrap/tables/001_ksp_store_schema_migrations.sql` **byte-identique** ; son checksum historique reste `d29068b8c13b9dc0cc9ef6aaadd0fa12d41e0fe4c56541a1118c4bfc846a1450` ;
- V001 est éclatée en 40 ressources embedded : 4 tables, 35 contraintes, 1 index ;
- le checksum logique V001 devient `31488cda2f08f3f46c4cdbdbb6c18c243662fada02eac4487040c8735d72cc51`, calculé sur les identifiants relatifs et bytes ordonnés des ressources ;
- chaque ressource sait comment être créée de façon additive, mais `IF NOT EXISTS` n'est jamais considéré comme preuve de compatibilité ;
- le backend introspecte les catalogues PostgreSQL et classe chaque ressource `Compatible`, `Missing` ou `Incompatible` ;
- tables, colonnes obligatoires, nullabilité/types, précision `NUMERIC(20,0)`, PK/FK/CHECK, index partiel, RLS, triggers, rules et contraintes externes susceptibles de modifier les writes sont vérifiés ;
- les CHECK sont comparés à la définition canonique dérivée de leur propre ressource SQL, et non à une liste permissive de fragments ;
- une colonne externe est acceptée uniquement lorsque sa présence est prouvée non bloquante pour les writes KSP ; les indexes externes non uniques sont tolérés ; les extensions ambiguës ou write-constraining bloquent `Ready` ;
- `schema_autocreate` contrôle l'initialisation/adoption d'une base sans metadata KSP ; `schema_autoupdate` contrôle les migrations pending et les réparations additives sûres d'un schéma KSP existant ;
- aucune réparation destructive ou ambiguë n'est exécutée automatiquement : le backend bloque avec une erreur/log sûrs pour intervention manuelle ;
- une identité V001 absente sur une migration déjà appliquée n'est jamais recréée automatiquement ;
- Config `std.store` passe au format V2 pour séparer les deux politiques, tout en conservant la lecture V1 où `auto_migrate` mappe les deux valeurs ;
- les fichiers monolithiques `migrations/V000__bootstrap.sql` et `migrations/V001__raw_transaction.sql` sont supprimés explicitement par le delta ;
- aucune capability/repository `RawTransaction*` n'est encore implémentée.
Une base PostgreSQL sur laquelle le commit `pre.003` aurait déjà enregistré l'ancien checksum V001 n'est **pas** réécrite silencieusement : elle doit être réinitialisée si elle est jetable, ou réconciliée manuellement. Le correctif préserve ainsi la règle d'immutabilité de l'historique au lieu de masquer une divergence de prerelease.
### `0.3.3-pre.004` — mapping et lectures
- codecs SQL privés ;

62
docs/validation/020-V0_3_3_STORE_POSTGRES_RAW_TRANSACTION.md
View File

@@ -1,5 +1,5 @@
<!-- file: docs/validation/020-V0_3_3_STORE_POSTGRES_RAW_TRANSACTION.md -->
<!-- version: 3 -->
<!-- version: 4 -->
# Validation `0.3.3` — Store/PostgreSQL RawTransaction vertical slice
@@ -11,7 +11,7 @@ Cette validation accompagne :
0.3.3 — Store/PostgreSQL RawTransaction vertical slice
```
Elle démarre en `0.3.3-pre.001` comme matrice de preuve. Les gates opérateur de `pre.001` et `pre.002` sont verts. Les lignes non encore implémentées restent explicitement `À FAIRE`; elles ne sont pas présentées comme acquises.
Elle démarre en `0.3.3-pre.001` comme matrice de preuve. Les gates opérateur de `pre.001` et `pre.002` sont verts. `pre.003` a été commité avant son gate opérateur ; `pre.003-fix.001` corrige donc sa matérialisation physique par-dessus ce commit. Les lignes non encore implémentées restent explicitement `À FAIRE`; elles ne sont pas présentées comme acquises.
## 2. Baseline stable
@@ -206,14 +206,14 @@ Critère : aucune branche ne doit utiliser `as i64`, `as i32`, clamp ou saturati
### 7.1 Bootstrap
| Cas | Résultat attendu |
|-------------------------------------------|------------------------------------------------------------------------|
| V001 appliquée pour la première fois | identity créée avec le réseau runtime dans la transaction de bootstrap |
| réouverture même réseau | succès idempotent |
| réouverture autre réseau | erreur terminale sûre |
| V001 présente mais identity absente | mismatch/corruption, jamais rebind silencieuse |
| identity malformée | erreur terminale sûre |
| migration pending + auto_migrate disabled | backend non prêt, aucune persistence métier |
| Cas | Résultat attendu |
|-----------------------------------------------|------------------------------------------------------------------------|
| V001 appliquée pour la première fois | identity créée avec le réseau runtime dans la transaction de bootstrap |
| réouverture même réseau | succès idempotent |
| réouverture autre réseau | erreur terminale sûre |
| V001 présente mais identity absente | mismatch/corruption, jamais rebind silencieuse |
| identity malformée | erreur terminale sûre |
| migration pending + `schema_autoupdate=false` | backend non prêt, aucune persistence métier |
### 7.2 Pré-I/O capability
@@ -446,17 +446,25 @@ Preuves acquises en `pre.002` avant création réelle de V001 :
- les migrations pending sont appliquées dans l'ordre sous la transaction et l'advisory lock déjà acquis ;
- chaque migration vérifie encore la forme de la metadata avant d'insérer sa ligne d'historique.
Preuves matérialisées en `pre.003` avec la vraie V001 :
Le commit `pre.003` a matérialisé la vraie V001 et son binding réseau, mais son fichier SQL monolithique et le seul contrôle de l'historique ne suffisaient pas à prouver la compatibilité structurelle de la base. Son checksum V001 historique de prerelease était `6fe57ed0313d2ed295280dd6e49f6d86695d4e4effee2724a25e36db1ea17761`.
- registre réel `[V000, V001]` et version courante dérivée `1` ;
- historique V000 seul reconnu comme préfixe exact avec V001 pending ;
- historique V000+V001 exact reconnu comme complet ;
- checksum V001 figé à `6fe57ed0313d2ed295280dd6e49f6d86695d4e4effee2724a25e36db1ea17761` ;
- binding réseau réel branché dans le hook `StoreIdentity` ;
- inventaire physique exact : identity, canonical, observations, archive, index partiel ;
- bornes SQL statiquement alignées sur `u64`, `u32`, `RawTimestamp`, payload 16 MiB et source payload 64 MiB.
Preuves statiques ajoutées par `pre.003-fix.001` :
L'application V000 -> V001, le rollback réel du hook, la réouverture même/autre réseau et la divergence de checksum sur PostgreSQL réel restent à prouver dans le gate live dédié `pre.009`; ils ne sont pas déclarés PASS dans cette tranche.
- V000 déplacée byte-identique, checksum toujours `d29068b8c13b9dc0cc9ef6aaadd0fa12d41e0fe4c56541a1118c4bfc846a1450` ;
- V001 logique unique composée de 40 ressources ordonnées : 4 tables, 35 contraintes et 1 index ;
- checksum V001 multi-ressources `31488cda2f08f3f46c4cdbdbb6c18c243662fada02eac4487040c8735d72cc51` ;
- anciens fichiers monolithiques absents après application du delta ;
- `CREATE ... IF NOT EXISTS`/guards servent uniquement à la création additive, jamais à déclarer un objet compatible ;
- inspection catalogue indépendante de l'historique avec états `Compatible`, `Missing`, `Incompatible` ;
- contraintes attendues comparées exactement à une définition canonique issue de la ressource SQL correspondante ;
- colonne externe nullable sans default/identity/generated acceptée comme extension non bloquante ;
- index externe non unique accepté ;
- mauvais type/nullabilité sur colonne obligatoire, extra colonne write-blocking, contrainte externe non équivalente, unique index autonome, trigger actif, rule ou RLS bloquent `Ready` ;
- ressource obligatoire manquante réparable seulement lorsque la politique le permet et que la réparation est additive/sûre ;
- identité V001 manquante sur un historique déjà appliqué reste terminale et n'est jamais auto-recréée ;
- `std.store` V2 sépare `schema_autocreate` et `schema_autoupdate`, avec lecture V1 conservée et mapping de `auto_migrate` vers les deux valeurs.
L'application V000 -> V001, les réparations réelles, le rollback du hook, la réouverture même/autre réseau et la matrice de drift sur PostgreSQL réel restent à prouver dans le gate live dédié `pre.009`; elles ne sont pas déclarées PASS par le correctif statique. Une base ayant déjà enregistré l'ancien checksum V001 de `pre.003` doit être réinitialisée ou réconciliée manuellement ; le moteur ne réécrit pas son historique silencieusement.
### 17.2 Binding réseau atomique
@@ -522,11 +530,21 @@ cap 500/1000 dans Store pagination
### `pre.003`
- V001 + tables/constraints/indexes : PASS statique ;
- V001 + tables/constraints/indexes : matérialisé puis corrigé par `pre.003-fix.001` ;
- binding réseau hook `AppliedNow`/`Existing` : PASS statique ;
- checksum/inventory/bounds tests : PASS statique ;
- code `postgres_retention_compaction_unsupported` backend + façade : PASS ;
- preuve PostgreSQL réelle V001 : différée à `pre.009`.
- gate opérateur `pre.003` : NON FOURNI avant le correctif.
### `pre.003-fix.001`
- arborescence versionnée et 40 ressources V001 : PASS statique ;
- V000 byte-identique/checksum historique : PASS statique ;
- checksum/inventory/order V001 multi-ressources : PASS statique ;
- contrat catalogue `Compatible/Missing/Incompatible` : PASS statique ;
- politiques `schema_autocreate` / `schema_autoupdate` + compatibilité Config V1 : PASS statique ;
- extensions externes non bloquantes tolérées, extensions write-blocking refusées : PASS statique ;
- preuve PostgreSQL réelle V001/drift/réparation : différée à `pre.009` ;
- gate Cargo opérateur : À EXÉCUTER.
### `pre.004`