From 65d8d8d3f6760c5d6cc7900bf990825ebd90533b Mon Sep 17 00:00:00 2001 From: SinuS Von SifriduS Date: Mon, 30 Mar 2026 23:11:57 +0200 Subject: [PATCH] v0.2.0 working --- Cargo.toml | 3 +- frontend/index.html | 31 +++- frontend/main.ts | 221 +++++++++++++++++++----- frontend/styles.css | 1 + frontend/ts/bindings/AvStartResponse.ts | 3 + frontend/ts/bindings/AvStopResponse.ts | 3 + package.json | 2 +- src/app_state.rs | 14 ++ src/commands.rs | 23 +++ src/error.rs | 2 +- src/lib.rs | 3 + src/media_audio.rs | 47 +++-- src/media_av.rs | 120 +++++++++++++ src/media_video.rs | 5 - tauri.conf.json | 2 +- 15 files changed, 403 insertions(+), 77 deletions(-) create mode 100644 frontend/ts/bindings/AvStartResponse.ts create mode 100644 frontend/ts/bindings/AvStopResponse.ts create mode 100644 src/media_av.rs diff --git a/Cargo.toml b/Cargo.toml index 392aeb3..b7d8a0d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "tauri-video02" -version = "0.1.1" +version = "0.2.0" description = "A Tauri Video App" authors = ["sinus@sasedev.net"] edition = "2024" @@ -29,3 +29,4 @@ tauri = { version = "^2.10", features = ["default"] } tauri-plugin-opener = { version = "^2.5", features = [] } tracing = { version = "^0.1", features = ["async-await", "log"] } tracing-subscriber = { version = "^0.3", features = ["ansi", "env-filter", "chrono", "serde", "json"] } +ts-rs = { version = "^12.0", features = [] } diff --git a/frontend/index.html b/frontend/index.html index e740851..a6474d1 100644 --- a/frontend/index.html +++ b/frontend/index.html @@ -14,15 +14,22 @@

POC 2 - Recording + Preview

-

Audio

+

Video Preview

+ +
+ Video preview +
+
+ +
+

Audio + Video

- - + +
- 
Ready.
+ 
Ready.
@@ -34,11 +41,19 @@ video -
- Video preview + 
Ready.
+
+ +
+

Audio

+ +
+ +
- 
Ready.
+ 
Ready.
diff --git a/frontend/main.ts b/frontend/main.ts index ee56837..1db63f6 100644 --- a/frontend/main.ts +++ b/frontend/main.ts @@ -1,4 +1,9 @@ + import { invoke } from "@tauri-apps/api/core"; +import { AvStartResponse } from './ts/bindings/AvStartResponse'; +import { AvStopResponse } from './ts/bindings/AvStopResponse'; + +type Mode = "idle" | "audio" | "video" | "av"; const startAudioBtn = document.querySelector("#start-audio-btn"); const stopAudioBtn = document.querySelector("#stop-audio-btn"); @@ -7,7 +12,12 @@ const audioStatus = document.querySelector("#audio-status"); const startVideoBtn = document.querySelector("#start-video-btn"); const stopVideoBtn = document.querySelector("#stop-video-btn"); const videoStatus = document.querySelector("#video-status"); -const videoPreview = document.querySelector("#video-preview"); + +const startAvBtn = document.querySelector("#start-av-btn"); +const stopAvBtn = document.querySelector("#stop-av-btn"); +const avStatus = document.querySelector("#av-status"); + +const previewElement = document.querySelector("#video-preview"); if ( startAudioBtn === null || @@ -16,19 +26,34 @@ if ( startVideoBtn === null || stopVideoBtn === null || videoStatus === null || - videoPreview === null + startAvBtn === null || + stopAvBtn === null || + avStatus === null || + previewElement === null ) { throw new Error("missing UI elements"); } +let currentMode: Mode = "idle"; let previewTimer: number | null = null; let previewRequestInFlight = false; +let previewObjectUrl: string | null = null; function setAudioStatus(message: string): void { if (audioStatus) audioStatus.textContent = message; } +function setVideoStatus(message: string): void { + if (videoStatus) + videoStatus.textContent = message; +} + +function setAvStatus(message: string): void { + if (avStatus) + avStatus.textContent = message; +} + function setAudioButtons(isRecording: boolean): void { if (startAudioBtn) startAudioBtn.disabled = isRecording; @@ -36,11 +61,6 @@ function setAudioButtons(isRecording: boolean): void { stopAudioBtn.disabled = !isRecording; } -function setVideoStatus(message: string): void { - if (videoStatus) - videoStatus.textContent = message; -} - function setVideoButtons(isRecording: boolean): void { if (startVideoBtn) startVideoBtn.disabled = isRecording; @@ -48,11 +68,92 @@ function setVideoButtons(isRecording: boolean): void { stopVideoBtn.disabled = !isRecording; } +function setAvButtons(isRecording: boolean): void { + if (startAvBtn) + startAvBtn.disabled = isRecording; + if (stopAvBtn) + stopAvBtn.disabled = !isRecording; +} + +function setAllButtonsForMode(mode: Mode): void { + if (mode === "idle") { + setAudioButtons(false); + setVideoButtons(false); + setAvButtons(false); + return; + } + + setAudioButtons(mode === "audio"); + setVideoButtons(mode === "video"); + setAvButtons(mode === "av"); + + if (mode !== "audio") { + if (stopAudioBtn) + stopAudioBtn.disabled = true; + if (startAudioBtn) + startAudioBtn.disabled = true; + } + + if (mode !== "video") { + if (stopVideoBtn) + stopVideoBtn.disabled = true; + if (startVideoBtn) + startVideoBtn.disabled = true; + } + + if (mode !== "av") { + if (stopAvBtn) + stopAvBtn.disabled = true; + if (startAvBtn) + startAvBtn.disabled = true; + } +} + +function setCurrentMode(mode: Mode): void { + currentMode = mode; + setAllButtonsForMode(mode); +} + +function base64ToUint8Array(base64: string): Uint8Array { + const binary = window.atob(base64); + const bytes = new Uint8Array(binary.length); + + let index = 0; + while (index < binary.length) { + bytes[index] = binary.charCodeAt(index); + index += 1; + } + + return bytes; +} + +function updatePreviewImageFromBase64(encoded: string): void { + const bytes = base64ToUint8Array(encoded); + const copied = new Uint8Array(bytes.byteLength); + + copied.set(bytes); + + const blob = new Blob([copied.buffer], { type: "image/jpeg" }); + const objectUrl = URL.createObjectURL(blob); + + if (previewObjectUrl !== null) { + URL.revokeObjectURL(previewObjectUrl); + } + + previewObjectUrl = objectUrl; + if (previewElement) + previewElement.src = objectUrl; +} + async function refreshPreviewFrame(): Promise { if (previewRequestInFlight) { return; } + if (currentMode !== "video" && currentMode !== "av") { + return; + } + previewRequestInFlight = true; try { @@ -73,8 +174,6 @@ function startPreviewPolling(): void { return; } - console.log("starting preview polling"); - window.setTimeout(() => { void refreshPreviewFrame(); }, 120); @@ -91,97 +190,133 @@ function stopPreviewPolling(): void { } previewRequestInFlight = false; - if (videoPreview) - videoPreview.removeAttribute("src"); if (previewObjectUrl !== null) { URL.revokeObjectURL(previewObjectUrl); previewObjectUrl = null; } + + if (previewElement) + previewElement.removeAttribute("src"); } startAudioBtn.addEventListener("click", async () => { - startAudioBtn.disabled = true; + if (currentMode !== "idle") { + setAudioStatus("Another mode is already running."); + return; + } + + setCurrentMode("audio"); try { const path = await invoke("start_audio_recording"); setAudioStatus(`Audio recording started.\nOutput: ${path}`); - setAudioButtons(true); } catch (error) { setAudioStatus(`Start audio failed.\n${String(error)}`); - setAudioButtons(false); + setCurrentMode("idle"); } }); stopAudioBtn.addEventListener("click", async () => { + if (currentMode !== "audio") { + setAudioStatus("Audio mode is not running."); + return; + } + stopAudioBtn.disabled = true; try { const path = await invoke("stop_audio_recording"); setAudioStatus(`Audio recording stopped.\nSaved file: ${path}`); - setAudioButtons(false); + setCurrentMode("idle"); } catch (error) { setAudioStatus(`Stop audio failed.\n${String(error)}`); - setAudioButtons(true); + setCurrentMode("audio"); } }); startVideoBtn.addEventListener("click", async () => { - startVideoBtn.disabled = true; + if (currentMode !== "idle") { + setVideoStatus("Another mode is already running."); + return; + } + + setCurrentMode("video"); try { const path = await invoke("start_video_recording"); setVideoStatus(`Video recording started.\nOutput: ${path}`); - setVideoButtons(true); startPreviewPolling(); } catch (error) { setVideoStatus(`Start video failed.\n${String(error)}`); - setVideoButtons(false); stopPreviewPolling(); + setCurrentMode("idle"); } }); stopVideoBtn.addEventListener("click", async () => { + if (currentMode !== "video") { + setVideoStatus("Video mode is not running."); + return; + } + stopVideoBtn.disabled = true; try { const path = await invoke("stop_video_recording"); setVideoStatus(`Video recording stopped.\nSaved file: ${path}`); - setVideoButtons(false); stopPreviewPolling(); + setCurrentMode("idle"); } catch (error) { setVideoStatus(`Stop video failed.\n${String(error)}`); - setVideoButtons(true); + setCurrentMode("video"); } }); -let previewObjectUrl: string | null = null; - -function base64ToUint8Array(base64: string): Uint8Array { - const binary = window.atob(base64); - const bytes = new Uint8Array(binary.length); - - for (let index = 0;index < binary.length;index += 1) { - bytes[index] = binary.charCodeAt(index); +startAvBtn.addEventListener("click", async () => { + if (currentMode !== "idle") { + setAvStatus("Another mode is already running."); + return; } - return bytes; -} + setCurrentMode("av"); -function updatePreviewImageFromBase64(encoded: string): void { - const bytes = base64ToUint8Array(encoded); - const copied = new Uint8Array(bytes.byteLength); + try { + const result = await invoke("start_av_recording"); - copied.set(bytes); + setAvStatus( + `AV recording started.\nAudio: ${result.audio_path}\nVideo: ${result.video_path}` + ); - const blob = new Blob([copied.buffer], { type: "image/jpeg" }); - const objectUrl = URL.createObjectURL(blob); + startPreviewPolling(); + } catch (error) { + setAvStatus(`Start AV failed.\n${String(error)}`); + stopPreviewPolling(); + setCurrentMode("idle"); + } +}); - if (previewObjectUrl !== null) { - URL.revokeObjectURL(previewObjectUrl); +stopAvBtn.addEventListener("click", async () => { + if (currentMode !== "av") { + setAvStatus("AV mode is not running."); + return; } - previewObjectUrl = objectUrl; - if (videoPreview) - videoPreview.src = objectUrl; -} + stopAvBtn.disabled = true; + + try { + const result = await invoke("stop_av_recording"); + + setAvStatus( + `AV recording stopped.\nAudio: ${result.audio_path}\nVideo: ${result.video_path}` + ); + + stopPreviewPolling(); + setCurrentMode("idle"); + } catch (error) { + setAvStatus(`Stop AV failed.\n${String(error)}`); + setCurrentMode("av"); + } +}); + +setCurrentMode("idle"); diff --git a/frontend/styles.css b/frontend/styles.css index 235ed51..f0bbafc 100644 --- a/frontend/styles.css +++ b/frontend/styles.css @@ -27,6 +27,7 @@ body { display: flex; gap: 12px; margin-bottom: 16px; + flex-wrap: wrap; } button { diff --git a/frontend/ts/bindings/AvStartResponse.ts b/frontend/ts/bindings/AvStartResponse.ts new file mode 100644 index 0000000..7ad6fd4 --- /dev/null +++ b/frontend/ts/bindings/AvStartResponse.ts @@ -0,0 +1,3 @@ +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. + +export type AvStartResponse = { audio_path: string, video_path: string, }; diff --git a/frontend/ts/bindings/AvStopResponse.ts b/frontend/ts/bindings/AvStopResponse.ts new file mode 100644 index 0000000..00f6533 --- /dev/null +++ b/frontend/ts/bindings/AvStopResponse.ts @@ -0,0 +1,3 @@ +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. + +export type AvStopResponse = { audio_path: string, video_path: string, }; diff --git a/package.json b/package.json index f8a9ec1..4d9716a 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "tauri-video02", "private": true, - "version": "0.1.1", + "version": "0.2.0", "type": "module", "scripts": { "dev": "vite", diff --git a/src/app_state.rs b/src/app_state.rs index 8e37cc7..279a6f4 100644 --- a/src/app_state.rs +++ b/src/app_state.rs @@ -44,10 +44,23 @@ impl PreviewState { } } +pub struct AvRecorderState { + pub is_recording: bool, +} + +impl AvRecorderState { + pub fn new() -> Self { + Self { + is_recording: false, + } + } +} + pub struct AppState { pub audio: std::sync::Mutex, pub video: std::sync::Mutex, pub preview: PreviewState, + pub av: std::sync::Mutex, } impl AppState { @@ -56,6 +69,7 @@ impl AppState { audio: std::sync::Mutex::new(AudioRecorderState::new()), video: std::sync::Mutex::new(VideoRecorderState::new()), preview: PreviewState::new(), + av: std::sync::Mutex::new(AvRecorderState::new()), } } } diff --git a/src/commands.rs b/src/commands.rs index 8047688..2fdbc06 100644 --- a/src/commands.rs +++ b/src/commands.rs @@ -2,6 +2,7 @@ use crate::app_state::AppState; use crate::media_audio; +use crate::media_av; use crate::media_video; #[tauri::command] @@ -50,3 +51,25 @@ pub async fn get_video_preview_frame_base64( Err(error) => Err(error.to_user_message()), } } + +#[tauri::command] +pub async fn start_av_recording( + state: tauri::State<'_, AppState>, +) -> Result { + let result = media_av::start_av_recording(&state); + match result { + Ok(value) => Ok(value), + Err(error) => Err(error.to_user_message()), + } +} + +#[tauri::command] +pub async fn stop_av_recording( + state: tauri::State<'_, AppState>, +) -> Result { + let result = media_av::stop_av_recording(&state); + match result { + Ok(value) => Ok(value), + Err(error) => Err(error.to_user_message()), + } +} diff --git a/src/error.rs b/src/error.rs index 1cbb21d..5548e5f 100644 --- a/src/error.rs +++ b/src/error.rs @@ -15,4 +15,4 @@ impl AppError { AppError::State(message) => format!("state error: {message}"), } } -} \ No newline at end of file +} diff --git a/src/lib.rs b/src/lib.rs index 5f30229..f8b252e 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -4,6 +4,7 @@ mod app_state; mod commands; mod error; mod media_audio; +mod media_av; mod media_video; fn init_tracing() { @@ -40,6 +41,8 @@ pub fn run() { commands::start_video_recording, commands::stop_video_recording, commands::get_video_preview_frame_base64, + commands::start_av_recording, + commands::stop_av_recording, ]); let run_result = builder.run(tauri::generate_context!()); diff --git a/src/media_audio.rs b/src/media_audio.rs index 2ffc90e..28734cf 100644 --- a/src/media_audio.rs +++ b/src/media_audio.rs @@ -2,14 +2,16 @@ use crate::app_state::AppState; use crate::error::AppError; -use gstreamer as gst; use gst::prelude::*; +use gstreamer as gst; fn build_output_dir() -> Result { let app_data_dir = dirs::data_local_dir(); let Some(mut base_dir) = app_data_dir else { - return Err(AppError::Io("unable to resolve local data directory".to_string())); + return Err(AppError::Io( + "unable to resolve local data directory".to_string(), + )); }; base_dir.push("tauri-video02"); @@ -27,16 +29,26 @@ fn build_output_dir() -> Result { } fn build_output_path() -> Result { - let mut dir = build_output_dir()?; + let build_dir_result = build_output_dir(); + let mut dir = match build_dir_result { + Ok(value) => value, + Err(error) => return Err(error), + }; + let timestamp = chrono::Local::now().format("%Y%m%d-%H%M%S").to_string(); dir.push(format!("audio-{timestamp}.wav")); Ok(dir) } fn build_pipeline(output_path: &std::path::Path) -> Result { - let location = output_path.to_string_lossy().replace('\\', "\\\\").replace('"', "\\\""); + let location = output_path + .to_string_lossy() + .replace('\\', "\\\\") + .replace('"', "\\\""); + let pipeline_str = format!( - "autoaudiosrc ! audioconvert ! audioresample ! wavenc ! filesink location=\"{location}\"" + "autoaudiosrc ! audioconvert ! audioresample ! wavenc ! filesink location=\"{}\"", + location ); tracing::info!(pipeline = %pipeline_str, "building audio pipeline"); @@ -45,9 +57,7 @@ fn build_pipeline(output_path: &std::path::Path) -> Result value, Err(error) => { - return Err(AppError::Gst(format!( - "unable to parse pipeline: {error}" - ))); + return Err(AppError::Gst(format!("unable to parse pipeline: {error}"))); } }; @@ -64,9 +74,7 @@ pub fn start_audio_recording(state: &tauri::State<'_, AppState>) -> Result value, Err(_) => { - return Err(AppError::State( - "audio state lock poisoned".to_string(), - )); + return Err(AppError::State("audio state lock poisoned".to_string())); } }; @@ -76,8 +84,15 @@ pub fn start_audio_recording(state: &tauri::State<'_, AppState>) -> Result value, + Err(error) => return Err(error), + }; + + let pipeline = match build_pipeline(&output_path) { + Ok(value) => value, + Err(error) => return Err(error), + }; let set_state_result = pipeline.set_state(gst::State::Playing); match set_state_result { @@ -102,9 +117,7 @@ pub fn stop_audio_recording(state: &tauri::State<'_, AppState>) -> Result value, Err(_) => { - return Err(AppError::State( - "audio state lock poisoned".to_string(), - )); + return Err(AppError::State("audio state lock poisoned".to_string())); } }; @@ -134,7 +147,7 @@ pub fn stop_audio_recording(state: &tauri::State<'_, AppState>) -> Result) -> Result { + let mut av_guard = match state.av.lock() { + Ok(value) => value, + Err(_) => { + return Err(AppError::State("av state lock poisoned".to_string())); + } + }; + + if av_guard.is_recording { + return Err(AppError::State( + "av recording is already running".to_string(), + )); + } + + let audio_guard_result = state.audio.lock(); + match audio_guard_result { + Ok(audio_guard) => { + if audio_guard.is_recording { + return Err(AppError::State("audio already running".to_string())); + } + } + Err(_) => { + return Err(AppError::State("audio state lock poisoned".to_string())); + } + } + + let video_guard_result = state.video.lock(); + match video_guard_result { + Ok(video_guard) => { + if video_guard.is_recording { + return Err(AppError::State("video already running".to_string())); + } + } + Err(_) => { + return Err(AppError::State("video state lock poisoned".to_string())); + } + } + + let audio_start_result = media_audio::start_audio_recording(state); + let audio_path = match audio_start_result { + Ok(value) => value, + Err(error) => return Err(error), + }; + + let video_start_result = media_video::start_video_recording(state); + let video_path = match video_start_result { + Ok(value) => value, + Err(error) => { + let rollback_result = media_audio::stop_audio_recording(state); + if let Err(rollback_error) = rollback_result { + tracing::warn!( + error = %rollback_error.to_user_message(), + "failed to rollback audio after video start failure" + ); + } + + return Err(error); + } + }; + + av_guard.is_recording = true; + + Ok(AvStartResponse { + audio_path, + video_path, + }) +} + +pub fn stop_av_recording(state: &tauri::State<'_, AppState>) -> Result { + let mut av_guard = match state.av.lock() { + Ok(value) => value, + Err(_) => { + return Err(AppError::State("av state lock poisoned".to_string())); + } + }; + + if !av_guard.is_recording { + return Err(AppError::State("av recording is not running".to_string())); + } + + let audio_stop_result = media_audio::stop_audio_recording(state); + let audio_path = match audio_stop_result { + Ok(value) => value, + Err(error) => return Err(error), + }; + + let video_stop_result = media_video::stop_video_recording(state); + let video_path = match video_stop_result { + Ok(value) => value, + Err(error) => return Err(error), + }; + + av_guard.is_recording = false; + + Ok(AvStopResponse { + audio_path, + video_path, + }) +} diff --git a/src/media_video.rs b/src/media_video.rs index 7c74417..4b650f8 100644 --- a/src/media_video.rs +++ b/src/media_video.rs @@ -123,8 +123,6 @@ fn attach_preview_callbacks( let bytes = map.as_slice().to_vec(); - tracing::info!(size = bytes.len(), "preview sample received"); - let lock_result = preview_store.lock(); match lock_result { Ok(mut guard) => { @@ -286,12 +284,9 @@ pub fn get_video_preview_frame_base64( }; let Some(bytes) = guard.as_ref() else { - tracing::info!("preview frame requested but none is available yet"); return Ok(None); }; - tracing::info!(size = bytes.len(), "preview frame requested and returned"); - let encoded = base64::engine::general_purpose::STANDARD.encode(bytes); Ok(Some(encoded)) } diff --git a/tauri.conf.json b/tauri.conf.json index 0753bbb..9392fde 100644 --- a/tauri.conf.json +++ b/tauri.conf.json @@ -1,7 +1,7 @@ { "$schema": "https://schema.tauri.app/config/2", "productName": "tauri-video02", - "version": "0.1.1", + "version": "0.2.0", "identifier": "com.sinus.tauri-video02", "build": { "beforeDevCommand": "npm run dev",