v0.3.15-pre.009

crates/ksp-app-raw-transaction-ingest-desk/src/route_runtime.rs

@@ -1,29 +1,41 @@
 // file: crates/ksp-app-raw-transaction-ingest-desk/src/route_runtime.rs
-// version: 4
+// version: 5
 
-//! Mono-route Store and Worker lifecycle owned by Raw Transaction Ingest Desk.
+//! Multi-route Store and independent Worker lifecycle owned by Raw Transaction Ingest Desk.
 
 const ROUTE_STOP_CLEANUP_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(2);
 const STORE_RECLAIM_POLL_INTERVAL: std::time::Duration = std::time::Duration::from_millis(5);
 const STORE_RECLAIM_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(1);
 
 struct RouteRuntimeInner {
-    last_terminal: std::option::Option<crate::RawIngestRouteRuntimeDto>,
+    last_terminals: std::vec::Vec<crate::RawIngestRouteRuntimeDto>,
+    network: std::option::Option<String>,
     next_sequence: u64,
-    slot: RouteRuntimeSlot,
+    routes: std::vec::Vec<RouteRuntimeSlot>,
+    store: std::option::Option<std::sync::Arc<ksp_store_lib::Store>>,
+    store_closing_token: std::option::Option<u64>,
+    store_opening_token: std::option::Option<u64>,
 }
 
-/// Shared mono-route runtime state. `pre.008` admits at most one active or starting Worker.
+/// Shared multi-route runtime state. Each route owns an independent Worker while same-network routes share one Store.
 pub(crate) struct RouteRuntimeState {
     inner: std::sync::Mutex<RouteRuntimeInner>,
 }
 
 impl crate::RouteRuntimeState {
-    /// Creates an idle mono-route runtime state.
+    /// Creates an idle multi-route runtime state without Store or Worker ownership.
     #[must_use]
     pub(crate) fn new() -> Self {
         return Self {
-            inner: std::sync::Mutex::new(RouteRuntimeInner { last_terminal: std::option::Option::None, next_sequence: 0, slot: RouteRuntimeSlot::Idle }),
+            inner: std::sync::Mutex::new(RouteRuntimeInner {
+                last_terminals: std::vec::Vec::new(),
+                network: std::option::Option::None,
+                next_sequence: 0,
+                routes: std::vec::Vec::new(),
+                store: std::option::Option::None,
+                store_closing_token: std::option::Option::None,
+                store_opening_token: std::option::Option::None,
+            }),
         };
     }
 
@@ -33,14 +45,34 @@ impl crate::RouteRuntimeState {
             std::result::Result::Ok(value) => value,
             std::result::Result::Err(_) => return std::result::Result::Err(runtime_lock_error()),
         };
-        if !matches!(inner.slot, RouteRuntimeSlot::Idle) {
+        if inner.store_closing_token.is_some() {
             return std::result::Result::Err(ksp_core_lib::Error::new(
                 crate::ERROR_CODE_ROUTE_RUNTIME_ACTIVE,
-                "Raw Transaction Ingest Desk already owns one mono-route Worker runtime",
+                "Raw Transaction Ingest Desk Store is still closing after the previous route set",
             ));
         }
-        let sequence = inner.next_sequence.checked_add(1);
-        let sequence = match sequence {
+        if inner.routes.iter().any(|slot| return slot.matches_identity(prepared.profile_id.as_str(), prepared.route_id)) {
+            return std::result::Result::Err(ksp_core_lib::Error::new(
+                crate::ERROR_CODE_ROUTE_RUNTIME_ACTIVE,
+                "Raw Transaction Ingest Desk already owns one Worker for the selected logical route",
+            ));
+        }
+        if let std::option::Option::Some(network) = inner.network.as_deref()
+            && network != prepared.network.as_str()
+        {
+            return std::result::Result::Err(ksp_core_lib::Error::new(
+                crate::ERROR_CODE_ROUTE_RUNTIME_NETWORK_MISMATCH,
+                "Raw Transaction Ingest Desk cannot share one Store across different logical networks",
+            ));
+        }
+        let needs_store_open = inner.store.is_none();
+        if needs_store_open && inner.store_opening_token.is_some() {
+            return std::result::Result::Err(ksp_core_lib::Error::new(
+                crate::ERROR_CODE_ROUTE_RUNTIME_ACTIVE,
+                "Raw Transaction Ingest Desk same-network Store is still opening",
+            ));
+        }
+        let sequence = match inner.next_sequence.checked_add(1) {
             std::option::Option::Some(value) => value,
             std::option::Option::None => {
                 return std::result::Result::Err(ksp_core_lib::Error::new(
@@ -66,10 +98,55 @@ impl crate::RouteRuntimeState {
             profile_id: prepared.profile_id.clone(),
             route_id: prepared.route_id,
         };
-        inner.last_terminal = std::option::Option::None;
+        if inner.network.is_none() {
+            inner.last_terminals.clear();
+            inner.network = std::option::Option::Some(prepared.network.clone());
+        }
+        inner.last_terminals.retain(|terminal| return !identity.matches_dto(terminal));
         inner.next_sequence = sequence;
-        inner.slot = RouteRuntimeSlot::Starting { token: sequence };
-        return std::result::Result::Ok(RouteRuntimeReservation { identity, token: sequence, worker_id });
+        if needs_store_open {
+            inner.store_opening_token = std::option::Option::Some(sequence);
+        }
+        inner.routes.push(RouteRuntimeSlot::Starting { identity: identity.clone(), token: sequence });
+        return std::result::Result::Ok(RouteRuntimeReservation { identity, needs_store_open, token: sequence, worker_id });
+    }
+
+    fn install_store(&self, token: u64, store: std::sync::Arc<ksp_store_lib::Store>) -> ksp_core_lib::Result<()> {
+        let inner = self.inner.lock();
+        let mut inner = match inner {
+            std::result::Result::Ok(value) => value,
+            std::result::Result::Err(_) => return std::result::Result::Err(runtime_lock_error()),
+        };
+        if inner.store_opening_token != std::option::Option::Some(token) || inner.store.is_some() {
+            return std::result::Result::Err(ksp_core_lib::Error::new(
+                crate::ERROR_CODE_ROUTE_RUNTIME_STATE_INVALID,
+                "Raw Transaction Ingest Desk shared Store reservation changed before installation",
+            ));
+        }
+        inner.store = std::option::Option::Some(store);
+        inner.store_opening_token = std::option::Option::None;
+        return std::result::Result::Ok(());
+    }
+
+    fn shared_store(&self, expected_network: &str) -> ksp_core_lib::Result<std::sync::Arc<ksp_store_lib::Store>> {
+        let inner = self.inner.lock();
+        let inner = match inner {
+            std::result::Result::Ok(value) => value,
+            std::result::Result::Err(_) => return std::result::Result::Err(runtime_lock_error()),
+        };
+        if inner.network.as_deref() != std::option::Option::Some(expected_network) {
+            return std::result::Result::Err(ksp_core_lib::Error::new(
+                crate::ERROR_CODE_ROUTE_RUNTIME_NETWORK_MISMATCH,
+                "Raw Transaction Ingest Desk shared Store network no longer matches the selected route",
+            ));
+        }
+        return match &inner.store {
+            std::option::Option::Some(store) => std::result::Result::Ok(std::sync::Arc::clone(store)),
+            std::option::Option::None => std::result::Result::Err(ksp_core_lib::Error::new(
+                crate::ERROR_CODE_ROUTE_RUNTIME_STATE_INVALID,
+                "Raw Transaction Ingest Desk shared Store is not installed",
+            )),
+        };
     }
 
     fn activate(
@@ -82,117 +159,226 @@ impl crate::RouteRuntimeState {
             std::result::Result::Ok(value) => value,
             std::result::Result::Err(_) => return std::result::Result::Err(runtime_lock_error()),
         };
-        match &inner.slot {
-            RouteRuntimeSlot::Starting { token } if *token == reservation.token => {},
-            RouteRuntimeSlot::Idle | RouteRuntimeSlot::Starting { .. } | RouteRuntimeSlot::Active { .. } => {
+        let position = inner.routes.iter().position(|slot| return slot.starting_token() == std::option::Option::Some(reservation.token));
+        let position = match position {
+            std::option::Option::Some(value) => value,
+            std::option::Option::None => {
                 return std::result::Result::Err(ksp_core_lib::Error::new(
                     crate::ERROR_CODE_ROUTE_RUNTIME_STATE_INVALID,
-                    "Raw Transaction Ingest Desk mono-route runtime reservation changed before Worker activation",
+                    "Raw Transaction Ingest Desk route reservation changed before Worker activation",
                 ));
             },
-        }
+        };
         let state = project_worker_state(handle.snapshot_source().current().worker_snapshot().state());
-        inner.slot = RouteRuntimeSlot::Active { handle, identity: reservation.identity.clone(), token: reservation.token };
+        inner.routes[position] = RouteRuntimeSlot::Active { handle, identity: reservation.identity.clone(), token: reservation.token };
         return std::result::Result::Ok(reservation.identity.dto(state));
     }
 
-    fn rollback(&self, token: u64) {
+    fn rollback(&self, token: u64) -> ksp_core_lib::Result<std::option::Option<std::sync::Arc<ksp_store_lib::Store>>> {
         let inner = self.inner.lock();
         let mut inner = match inner {
             std::result::Result::Ok(value) => value,
-            std::result::Result::Err(_) => return,
+            std::result::Result::Err(_) => return std::result::Result::Err(runtime_lock_error()),
         };
-        if matches!(&inner.slot, RouteRuntimeSlot::Starting { token: current, .. } if *current == token) {
-            inner.slot = RouteRuntimeSlot::Idle;
+        inner.routes.retain(|slot| return slot.token() != token);
+        if inner.store_opening_token == std::option::Option::Some(token) {
+            inner.store_opening_token = std::option::Option::None;
         }
-    }
-
-    fn finish(&self, token: u64, terminal: crate::RawIngestRouteRuntimeDto) {
-        let inner = self.inner.lock();
-        let mut inner = match inner {
-            std::result::Result::Ok(value) => value,
-            std::result::Result::Err(_) => return,
-        };
-        if matches!(&inner.slot, RouteRuntimeSlot::Active { token: current, .. } if *current == token) {
-            inner.last_terminal = std::option::Option::Some(terminal);
-            inner.slot = RouteRuntimeSlot::Idle;
-        }
-    }
-
-    /// Requests cooperative Stop, waits for terminal Worker state and returns after the terminal monitor has released the mono-route slot.
-    pub(crate) async fn stop_and_wait(&self) -> ksp_core_lib::Result<crate::RawIngestRouteRuntimeDto> {
-        let active = {
-            let inner = self.inner.lock();
-            let inner = match inner {
-                std::result::Result::Ok(value) => value,
-                std::result::Result::Err(_) => return std::result::Result::Err(runtime_lock_error()),
-            };
-            match &inner.slot {
-                RouteRuntimeSlot::Active { handle, identity, .. } => std::result::Result::Ok((handle.clone(), identity.clone())),
-                RouteRuntimeSlot::Idle => match &inner.last_terminal {
-                    std::option::Option::Some(terminal) => return std::result::Result::Ok(terminal.clone()),
-                    std::option::Option::None => std::result::Result::Err(ksp_core_lib::Error::new(
-                        crate::ERROR_CODE_ROUTE_RUNTIME_NOT_ACTIVE,
-                        "Raw Transaction Ingest Desk has no active mono-route Worker to stop",
-                    )),
-                },
-                RouteRuntimeSlot::Starting { .. } => std::result::Result::Err(ksp_core_lib::Error::new(
-                    crate::ERROR_CODE_ROUTE_RUNTIME_NOT_ACTIVE,
-                    "Raw Transaction Ingest Desk mono-route Worker is still starting and cannot be stopped yet",
-                )),
+        if inner.routes.is_empty() {
+            let store = inner.store.take();
+            if store.is_some() {
+                inner.store_closing_token = std::option::Option::Some(token);
+            } else {
+                inner.network = std::option::Option::None;
             }
+            return std::result::Result::Ok(store);
+        }
+        return std::result::Result::Ok(std::option::Option::None);
+    }
+
+    fn finish(&self, token: u64, terminal: crate::RawIngestRouteRuntimeDto) -> ksp_core_lib::Result<std::option::Option<std::sync::Arc<ksp_store_lib::Store>>> {
+        let inner = self.inner.lock();
+        let mut inner = match inner {
+            std::result::Result::Ok(value) => value,
+            std::result::Result::Err(_) => return std::result::Result::Err(runtime_lock_error()),
         };
-        let (handle, identity) = match active {
+        let before = inner.routes.len();
+        inner.routes.retain(|slot| return slot.token() != token);
+        if inner.routes.len() == before {
+            return std::result::Result::Ok(std::option::Option::None);
+        }
+        inner.last_terminals.retain(|previous| return previous.profile_id != terminal.profile_id || previous.route_id != terminal.route_id);
+        inner.last_terminals.push(terminal);
+        if inner.routes.is_empty() {
+            let store = inner.store.take();
+            if store.is_some() {
+                inner.store_closing_token = std::option::Option::Some(token);
+            } else {
+                inner.network = std::option::Option::None;
+            }
+            return std::result::Result::Ok(store);
+        }
+        return std::result::Result::Ok(std::option::Option::None);
+    }
+
+    fn complete_store_close(&self, token: u64) {
+        let inner = self.inner.lock();
+        let mut inner = match inner {
+            std::result::Result::Ok(value) => value,
+            std::result::Result::Err(_) => return,
+        };
+        if inner.store_closing_token == std::option::Option::Some(token) {
+            inner.store_closing_token = std::option::Option::None;
+            if inner.routes.is_empty() && inner.store.is_none() && inner.store_opening_token.is_none() {
+                inner.network = std::option::Option::None;
+            }
+        }
+    }
+
+    /// Requests cooperative Stop for one exact logical route and waits for its Worker cleanup.
+    pub(crate) async fn stop_and_wait(&self, request: &crate::RawIngestRouteStopRequestDto) -> ksp_core_lib::Result<crate::RawIngestRouteRuntimeDto> {
+        let target = self.stop_target(request);
+        let target = match target {
             std::result::Result::Ok(value) => value,
             std::result::Result::Err(error) => return std::result::Result::Err(error),
         };
+        let (handle, identity, token) = match target {
+            RouteStopTarget::Active { handle, identity, token } => (handle, identity, token),
+            RouteStopTarget::Terminal(value) => return std::result::Result::Ok(value),
+            RouteStopTarget::TerminalClosing => return self.wait_terminal_store_cleanup(request).await,
+        };
         let _first_stop_request = handle.request_stop();
-        let terminal = handle.wait_terminal().await;
-        let terminal = match terminal {
+        if let std::result::Result::Err(error) = handle.wait_terminal().await {
+            return std::result::Result::Err(
+                ksp_core_lib::Error::new(crate::ERROR_CODE_ROUTE_RUNTIME_STATE_INVALID, "Cannot observe terminal Raw Transaction Ingest Desk Worker state")
+                    .with_source(error),
+            );
+        }
+        return self.wait_route_cleanup(token, &identity).await;
+    }
+
+    fn stop_target(&self, request: &crate::RawIngestRouteStopRequestDto) -> ksp_core_lib::Result<RouteStopTarget> {
+        let inner = self.inner.lock();
+        let inner = match inner {
             std::result::Result::Ok(value) => value,
-            std::result::Result::Err(error) => {
-                return std::result::Result::Err(
-                    ksp_core_lib::Error::new(crate::ERROR_CODE_ROUTE_RUNTIME_STATE_INVALID, "Cannot observe terminal Raw Transaction Ingest Desk Worker state")
-                        .with_source(error),
-                );
+            std::result::Result::Err(_) => return std::result::Result::Err(runtime_lock_error()),
+        };
+        let slot = inner.routes.iter().find(|slot| return slot.matches_identity(request.profile_id.as_str(), request.route_id));
+        return match slot {
+            std::option::Option::Some(RouteRuntimeSlot::Active { handle, identity, token }) => {
+                std::result::Result::Ok(RouteStopTarget::Active { handle: handle.clone(), identity: identity.clone(), token: *token })
+            },
+            std::option::Option::Some(RouteRuntimeSlot::Starting { .. }) => std::result::Result::Err(ksp_core_lib::Error::new(
+                crate::ERROR_CODE_ROUTE_RUNTIME_NOT_ACTIVE,
+                "Raw Transaction Ingest Desk route Worker is still starting and cannot be stopped yet",
+            )),
+            std::option::Option::None => {
+                let terminal =
+                    inner.last_terminals.iter().find(|terminal| return terminal.profile_id == request.profile_id && terminal.route_id == request.route_id);
+                match terminal {
+                    std::option::Option::Some(value) if inner.store_closing_token.is_none() => {
+                        std::result::Result::Ok(RouteStopTarget::Terminal(value.clone()))
+                    },
+                    std::option::Option::Some(_) => std::result::Result::Ok(RouteStopTarget::TerminalClosing),
+                    std::option::Option::None => std::result::Result::Err(ksp_core_lib::Error::new(
+                        crate::ERROR_CODE_ROUTE_RUNTIME_NOT_ACTIVE,
+                        "Raw Transaction Ingest Desk has no active Worker for the selected logical route",
+                    )),
+                }
             },
         };
+    }
+
+    async fn wait_route_cleanup(&self, token: u64, identity: &RouteRuntimeIdentity) -> ksp_core_lib::Result<crate::RawIngestRouteRuntimeDto> {
         let started = std::time::Instant::now();
         loop {
-            let idle = self.is_idle();
-            let idle = match idle {
+            let cleanup = self.route_cleanup_complete(token, identity);
+            let cleanup = match cleanup {
                 std::result::Result::Ok(value) => value,
                 std::result::Result::Err(error) => return std::result::Result::Err(error),
             };
-            if idle {
-                return std::result::Result::Ok(identity.dto(project_worker_state(terminal)));
+            if let std::option::Option::Some(value) = cleanup {
+                return std::result::Result::Ok(value);
             }
             if started.elapsed() >= ROUTE_STOP_CLEANUP_TIMEOUT {
                 return std::result::Result::Err(ksp_core_lib::Error::new(
                     crate::ERROR_CODE_ROUTE_RUNTIME_STORE_SHUTDOWN_FAILED,
-                    "Raw Transaction Ingest Desk terminal cleanup did not release the mono-route slot",
+                    "Raw Transaction Ingest Desk terminal cleanup did not release the selected route",
                 ));
             }
             tokio::time::sleep(STORE_RECLAIM_POLL_INTERVAL).await;
         }
     }
 
-    fn is_idle(&self) -> ksp_core_lib::Result<bool> {
+    async fn wait_terminal_store_cleanup(&self, request: &crate::RawIngestRouteStopRequestDto) -> ksp_core_lib::Result<crate::RawIngestRouteRuntimeDto> {
+        let started = std::time::Instant::now();
+        loop {
+            let terminal = self.late_terminal_after_store_cleanup(request);
+            let terminal = match terminal {
+                std::result::Result::Ok(value) => value,
+                std::result::Result::Err(error) => return std::result::Result::Err(error),
+            };
+            if let std::option::Option::Some(value) = terminal {
+                return std::result::Result::Ok(value);
+            }
+            if started.elapsed() >= ROUTE_STOP_CLEANUP_TIMEOUT {
+                return std::result::Result::Err(ksp_core_lib::Error::new(
+                    crate::ERROR_CODE_ROUTE_RUNTIME_STORE_SHUTDOWN_FAILED,
+                    "Raw Transaction Ingest Desk final shared Store cleanup did not complete for the selected terminal route",
+                ));
+            }
+            tokio::time::sleep(STORE_RECLAIM_POLL_INTERVAL).await;
+        }
+    }
+
+    fn late_terminal_after_store_cleanup(
+        &self,
+        request: &crate::RawIngestRouteStopRequestDto,
+    ) -> ksp_core_lib::Result<std::option::Option<crate::RawIngestRouteRuntimeDto>> {
         let inner = self.inner.lock();
         let inner = match inner {
             std::result::Result::Ok(value) => value,
             std::result::Result::Err(_) => return std::result::Result::Err(runtime_lock_error()),
         };
-        return std::result::Result::Ok(matches!(inner.slot, RouteRuntimeSlot::Idle));
+        let terminal = inner.last_terminals.iter().find(|terminal| return terminal.profile_id == request.profile_id && terminal.route_id == request.route_id);
+        let terminal = match terminal {
+            std::option::Option::Some(value) => value,
+            std::option::Option::None => {
+                return std::result::Result::Err(ksp_core_lib::Error::new(
+                    crate::ERROR_CODE_ROUTE_RUNTIME_NOT_ACTIVE,
+                    "Raw Transaction Ingest Desk terminal route is no longer retained",
+                ));
+            },
+        };
+        if inner.store_closing_token.is_some() {
+            return std::result::Result::Ok(std::option::Option::None);
+        }
+        return std::result::Result::Ok(std::option::Option::Some(terminal.clone()));
+    }
+
+    fn route_cleanup_complete(
+        &self,
+        token: u64,
+        identity: &RouteRuntimeIdentity,
+    ) -> ksp_core_lib::Result<std::option::Option<crate::RawIngestRouteRuntimeDto>> {
+        let inner = self.inner.lock();
+        let inner = match inner {
+            std::result::Result::Ok(value) => value,
+            std::result::Result::Err(_) => return std::result::Result::Err(runtime_lock_error()),
+        };
+        if inner.routes.iter().any(|slot| return slot.token() == token) || inner.store_closing_token == std::option::Option::Some(token) {
+            return std::result::Result::Ok(std::option::Option::None);
+        }
+        let terminal = inner.last_terminals.iter().find(|value| return identity.matches_dto(value));
+        return std::result::Result::Ok(terminal.cloned());
     }
 }
 
-/// Launch ownership returned only after Store readiness, Worker creation and handle installation have succeeded.
+/// Launch ownership returned only after shared Store readiness, Worker creation and route-handle installation have succeeded.
 pub(crate) struct RouteRuntimeLaunch {
     acknowledgement: crate::RawIngestRouteRuntimeDto,
     handle: ksp_worker_raw_transaction_ingest_lib::RawTransactionIngestHandle,
     runtime_state: std::sync::Arc<crate::RouteRuntimeState>,
-    store: std::sync::Arc<ksp_store_lib::Store>,
     token: u64,
 }
 
@@ -203,7 +389,7 @@ impl crate::RouteRuntimeLaunch {
         return self.acknowledgement.clone();
     }
 
-    /// Waits for terminal Worker state, explicitly closes Store after the Worker releases its `Arc`, then frees the mono-route slot.
+    /// Waits for this Worker terminal state, removes only its route and closes Store only after the final same-network route terminates.
     pub(crate) async fn monitor(self) {
         let terminal = self.handle.wait_terminal().await;
         let terminal_state = match terminal {
@@ -212,7 +398,7 @@ impl crate::RouteRuntimeLaunch {
                     target: crate::TRACING_TARGET,
                     domain = crate::TRACING_DOMAIN_ROUTE_RUNTIME,
                     worker_state = state.code(),
-                    "Raw Transaction Ingest Desk mono-route Worker reached terminal state"
+                    "Raw Transaction Ingest Desk route Worker reached terminal state"
                 );
                 project_worker_state(state)
             },
@@ -227,64 +413,99 @@ impl crate::RouteRuntimeLaunch {
                 crate::RawIngestRouteState::Faulted
             },
         };
-        let close = close_store_arc(self.store).await;
-        if let std::result::Result::Err(error) = close {
-            ksp_logging_lib::warn!(
-                target: crate::TRACING_TARGET,
-                domain = crate::TRACING_DOMAIN_ROUTE_RUNTIME,
-                error_domain = error.code().domain(),
-                error_code = error.code().code(),
-                "Raw Transaction Ingest Desk Store shutdown failed after terminal Worker"
-            );
-        }
         let mut terminal = self.acknowledgement;
         terminal.state = terminal_state;
-        self.runtime_state.finish(self.token, terminal);
+        let store = self.runtime_state.finish(self.token, terminal);
+        let store = match store {
+            std::result::Result::Ok(value) => value,
+            std::result::Result::Err(error) => {
+                ksp_logging_lib::warn!(
+                    target: crate::TRACING_TARGET,
+                    domain = crate::TRACING_DOMAIN_ROUTE_RUNTIME,
+                    error_domain = error.code().domain(),
+                    error_code = error.code().code(),
+                    "Raw Transaction Ingest Desk route cleanup state update failed"
+                );
+                return;
+            },
+        };
+        if let std::option::Option::Some(store) = store {
+            let close = close_store_arc(store).await;
+            if let std::result::Result::Err(error) = close {
+                ksp_logging_lib::warn!(
+                    target: crate::TRACING_TARGET,
+                    domain = crate::TRACING_DOMAIN_ROUTE_RUNTIME,
+                    error_domain = error.code().domain(),
+                    error_code = error.code().code(),
+                    "Raw Transaction Ingest Desk shared Store shutdown failed after final Worker"
+                );
+            }
+            self.runtime_state.complete_store_close(self.token);
+        }
     }
 }
 
-/// Opens one Store, starts one exact Worker route and atomically installs its handle in the mono-route application slot.
+/// Starts one exact route Worker and reuses the same ready Store for any already-active same-network route.
 pub(crate) async fn start_route_runtime(
     runtime_state: std::sync::Arc<crate::RouteRuntimeState>,
     prepared: crate::PreparedRouteStart,
 ) -> ksp_core_lib::Result<crate::RouteRuntimeLaunch> {
-    let reservation = runtime_state.reserve(&prepared);
-    let reservation = match reservation {
+    let reservation = match runtime_state.reserve(&prepared) {
         std::result::Result::Ok(value) => value,
         std::result::Result::Err(error) => return std::result::Result::Err(error),
     };
-    let store = ksp_store_lib::Store::open(prepared.store_settings).await;
-    let store = match store {
-        std::result::Result::Ok(value) => value,
-        std::result::Result::Err(error) => {
-            runtime_state.rollback(reservation.token);
-            return std::result::Result::Err(
-                ksp_core_lib::Error::new(crate::ERROR_CODE_ROUTE_RUNTIME_START_FAILED, "Cannot open Raw Transaction Ingest Desk Store runtime")
+    let store = if reservation.needs_store_open {
+        let store = ksp_store_lib::Store::open(prepared.store_settings).await;
+        let store = match store {
+            std::result::Result::Ok(value) => value,
+            std::result::Result::Err(error) => {
+                let _rollback = runtime_state.rollback(reservation.token);
+                return std::result::Result::Err(
+                    ksp_core_lib::Error::new(crate::ERROR_CODE_ROUTE_RUNTIME_START_FAILED, "Cannot open Raw Transaction Ingest Desk shared Store runtime")
+                        .with_source(error),
+                );
+            },
+        };
+        let health = store.health().await;
+        if health.state() != ksp_store_lib::StoreHealthState::Ready {
+            let close = store.close().await;
+            let _rollback = runtime_state.rollback(reservation.token);
+            if let std::result::Result::Err(error) = close {
+                return std::result::Result::Err(
+                    ksp_core_lib::Error::new(
+                        crate::ERROR_CODE_ROUTE_RUNTIME_STORE_SHUTDOWN_FAILED,
+                        "Cannot close non-ready Raw Transaction Ingest Desk shared Store runtime",
+                    )
                     .with_source(error),
-            );
-        },
-    };
-    let health = store.health().await;
-    if health.state() != ksp_store_lib::StoreHealthState::Ready {
-        let close = store.close().await;
-        runtime_state.rollback(reservation.token);
-        if let std::result::Result::Err(error) = close {
-            return std::result::Result::Err(
-                ksp_core_lib::Error::new(
-                    crate::ERROR_CODE_ROUTE_RUNTIME_STORE_SHUTDOWN_FAILED,
-                    "Cannot close non-ready Raw Transaction Ingest Desk Store runtime",
-                )
-                .with_source(error),
-            );
+                );
+            }
+            return std::result::Result::Err(ksp_core_lib::Error::new(
+                crate::ERROR_CODE_ROUTE_RUNTIME_STORE_NOT_READY,
+                "Raw Transaction Ingest Desk shared Store readiness probe did not prove Ready",
+            ));
         }
-        return std::result::Result::Err(ksp_core_lib::Error::new(
-            crate::ERROR_CODE_ROUTE_RUNTIME_STORE_NOT_READY,
-            "Raw Transaction Ingest Desk Store readiness probe did not prove Ready",
-        ));
-    }
+        let store = std::sync::Arc::new(store);
+        if let std::result::Result::Err(error) = runtime_state.install_store(reservation.token, std::sync::Arc::clone(&store)) {
+            let close = close_store_arc(store).await;
+            let _rollback = runtime_state.rollback(reservation.token);
+            if let std::result::Result::Err(close_error) = close {
+                return std::result::Result::Err(close_error);
+            }
+            return std::result::Result::Err(error);
+        }
+        store
+    } else {
+        std::mem::drop(prepared.store_settings);
+        match runtime_state.shared_store(prepared.network.as_str()) {
+            std::result::Result::Ok(value) => value,
+            std::result::Result::Err(error) => {
+                let _rollback = runtime_state.rollback(reservation.token);
+                return std::result::Result::Err(error);
+            },
+        }
+    };
     let network = store.runtime_snapshot().network().clone();
     let settings = ksp_worker_raw_transaction_ingest_lib::RawTransactionIngestSettings::with_defaults(network, reservation.worker_id.clone());
-    let store = std::sync::Arc::new(store);
     let handle = ksp_worker_raw_transaction_ingest_lib::RawTransactionIngestWorker::start_with_runtime_resources(
         settings,
         std::sync::Arc::clone(&store),
@@ -293,13 +514,21 @@ pub(crate) async fn start_route_runtime(
     let handle = match handle {
         std::result::Result::Ok(value) => value,
         std::result::Result::Err(error) => {
-            let close = close_store_arc(store).await;
-            runtime_state.rollback(reservation.token);
-            if let std::result::Result::Err(close_error) = close {
-                return std::result::Result::Err(close_error);
+            let cleanup_store = runtime_state.rollback(reservation.token);
+            let cleanup_store = match cleanup_store {
+                std::result::Result::Ok(value) => value,
+                std::result::Result::Err(rollback_error) => return std::result::Result::Err(rollback_error),
+            };
+            std::mem::drop(store);
+            if let std::option::Option::Some(value) = cleanup_store {
+                let close = close_store_arc(value).await;
+                runtime_state.complete_store_close(reservation.token);
+                if let std::result::Result::Err(close_error) = close {
+                    return std::result::Result::Err(close_error);
+                }
             }
             return std::result::Result::Err(
-                ksp_core_lib::Error::new(crate::ERROR_CODE_ROUTE_RUNTIME_START_FAILED, "Cannot start Raw Transaction Ingest Desk Worker runtime")
+                ksp_core_lib::Error::new(crate::ERROR_CODE_ROUTE_RUNTIME_START_FAILED, "Cannot start Raw Transaction Ingest Desk route Worker runtime")
                     .with_source(error),
             );
         },
@@ -310,14 +539,23 @@ pub(crate) async fn start_route_runtime(
         std::result::Result::Err(error) => {
             let _stop_requested = handle.request_stop();
             let _terminal = handle.wait_terminal().await;
-            let close = close_store_arc(store).await;
-            runtime_state.rollback(reservation.token);
-            if let std::result::Result::Err(close_error) = close {
-                return std::result::Result::Err(close_error);
+            let cleanup_store = runtime_state.rollback(reservation.token);
+            let cleanup_store = match cleanup_store {
+                std::result::Result::Ok(value) => value,
+                std::result::Result::Err(rollback_error) => return std::result::Result::Err(rollback_error),
+            };
+            std::mem::drop(store);
+            if let std::option::Option::Some(value) = cleanup_store {
+                let close = close_store_arc(value).await;
+                runtime_state.complete_store_close(reservation.token);
+                if let std::result::Result::Err(close_error) = close {
+                    return std::result::Result::Err(close_error);
+                }
             }
             return std::result::Result::Err(error);
         },
     };
+    std::mem::drop(store);
     ksp_logging_lib::info!(
         target: crate::TRACING_TARGET,
         domain = crate::TRACING_DOMAIN_ROUTE_RUNTIME,
@@ -325,9 +563,9 @@ pub(crate) async fn start_route_runtime(
         network = reservation.identity.network.as_str(),
         route_id = reservation.identity.route_id.as_str(),
         commitment = reservation.identity.commitment.as_str(),
-        "Raw Transaction Ingest Desk mono-route Worker handle installed"
+        "Raw Transaction Ingest Desk independent route Worker handle installed on shared Store"
     );
-    return std::result::Result::Ok(crate::RouteRuntimeLaunch { acknowledgement, handle, runtime_state, store, token: reservation.token });
+    return std::result::Result::Ok(crate::RouteRuntimeLaunch { acknowledgement, handle, runtime_state, token: reservation.token });
 }
 
 #[derive(Clone)]
@@ -350,17 +588,32 @@ impl RouteRuntimeIdentity {
             state,
         };
     }
+
+    fn matches_dto(&self, value: &crate::RawIngestRouteRuntimeDto) -> bool {
+        return self.profile_id == value.profile_id && self.route_id == value.route_id;
+    }
+}
+
+enum RouteStopTarget {
+    Active {
+        handle: ksp_worker_raw_transaction_ingest_lib::RawTransactionIngestHandle,
+        identity: RouteRuntimeIdentity,
+        token: u64,
+    },
+    Terminal(crate::RawIngestRouteRuntimeDto),
+    TerminalClosing,
 }
 
 struct RouteRuntimeReservation {
     identity: RouteRuntimeIdentity,
+    needs_store_open: bool,
     token: u64,
     worker_id: ksp_worker_api::WorkerId,
 }
 
 enum RouteRuntimeSlot {
-    Idle,
     Starting {
+        identity: RouteRuntimeIdentity,
         token: u64,
     },
     Active {
@@ -370,6 +623,28 @@ enum RouteRuntimeSlot {
     },
 }
 
+impl RouteRuntimeSlot {
+    fn matches_identity(&self, profile_id: &str, route_id: crate::RawIngestRouteId) -> bool {
+        let identity = match self {
+            Self::Starting { identity, .. } | Self::Active { identity, .. } => identity,
+        };
+        return identity.profile_id == profile_id && identity.route_id == route_id;
+    }
+
+    fn starting_token(&self) -> std::option::Option<u64> {
+        return match self {
+            Self::Starting { token, .. } => std::option::Option::Some(*token),
+            Self::Active { .. } => std::option::Option::None,
+        };
+    }
+
+    fn token(&self) -> u64 {
+        return match self {
+            Self::Starting { token, .. } | Self::Active { token, .. } => *token,
+        };
+    }
+}
+
 async fn close_store_arc(store: std::sync::Arc<ksp_store_lib::Store>) -> ksp_core_lib::Result<()> {
     let started = std::time::Instant::now();
     let mut store = store;
@@ -382,20 +657,18 @@ async fn close_store_arc(store: std::sync::Arc<ksp_store_lib::Store>) -> ksp_cor
                     std::result::Result::Err(error) => std::result::Result::Err(
                         ksp_core_lib::Error::new(
                             crate::ERROR_CODE_ROUTE_RUNTIME_STORE_SHUTDOWN_FAILED,
-                            "Cannot close Raw Transaction Ingest Desk Store runtime",
+                            "Cannot close Raw Transaction Ingest Desk shared Store runtime",
                         )
                         .with_source(error),
                     ),
                 };
             },
-            std::result::Result::Err(shared) => {
-                store = shared;
-            },
+            std::result::Result::Err(shared) => store = shared,
         }
         if started.elapsed() >= STORE_RECLAIM_TIMEOUT {
             return std::result::Result::Err(ksp_core_lib::Error::new(
                 crate::ERROR_CODE_ROUTE_RUNTIME_STORE_SHUTDOWN_FAILED,
-                "Raw Transaction Ingest Desk Store remained shared after terminal Worker",
+                "Raw Transaction Ingest Desk shared Store remained referenced after final Worker",
             ));
         }
         tokio::time::sleep(STORE_RECLAIM_POLL_INTERVAL).await;
@@ -414,7 +687,7 @@ fn project_worker_state(state: ksp_worker_api::WorkerState) -> crate::RawIngestR
 }
 
 fn runtime_lock_error() -> ksp_core_lib::Error {
-    return ksp_core_lib::Error::new(crate::ERROR_CODE_APP_STATE_LOCK_FAILED, "Raw Transaction Ingest Desk mono-route runtime state lock is poisoned");
+    return ksp_core_lib::Error::new(crate::ERROR_CODE_APP_STATE_LOCK_FAILED, "Raw Transaction Ingest Desk multi-route runtime state lock is poisoned");
 }
 
 #[cfg(test)]