diff --git a/docs/bailian-asr-models.md b/docs/bailian-asr-models.md new file mode 100644 index 000000000..1655d14ab --- /dev/null +++ b/docs/bailian-asr-models.md @@ -0,0 +1,32 @@ +# Alibaba Bailian ASR models + +OpenLess uses one Alibaba Bailian provider and selects the DashScope protocol from the model name. Configure the API key, region or workspace endpoint, and model in Settings > Providers. + +| Mode | Model examples | Behavior | +| --- | --- | --- | +| Realtime WebSocket | `fun-asr-realtime`, `fun-asr-flash-8k-realtime`, `paraformer-realtime-v2`, `sensevoice-realtime-v1` | Streams text while recording. OpenLess downsamples 16 kHz recorder audio for 8 kHz models. | +| Qwen realtime | `qwen3-asr-flash-realtime`, versioned snapshots | Streams text through the Qwen Realtime WebSocket API. | +| Synchronous recording | `fun-asr-flash-*`, `qwen3-asr-flash` | Sends the recording after capture and waits for one synchronous response. These models are intended for short recordings. | +| Asynchronous file transcription | `fun-asr`, `fun-asr-mtl`, versioned snapshots, `paraformer-v2`, `qwen3-asr-flash-filetrans` | Uploads the recording, starts an asynchronous task, polls it, then downloads the transcript. | + +Future dated snapshots that retain one of these model prefixes are routed to the same protocol, so they can be entered manually before they are added to the model picker. + +## Endpoints + +The default endpoint is `dashscope.aliyuncs.com`. A workspace or regional host may also be entered, for example `https://{WorkspaceId}.cn-beijing.maas.aliyuncs.com`. OpenLess preserves the host and derives the protocol-specific path and scheme automatically. + +API keys are region-specific. The endpoint and API key must belong to the same region. + +## Temporary uploads + +Asynchronous models require a URL. OpenLess obtains a temporary upload policy from DashScope, uploads the local WAV to the returned OSS host, and submits the resulting `oss://` URL with `X-DashScope-OssResourceResolve: enable`. + +Alibaba documents this temporary storage as a development and low-concurrency facility. It is rate limited, and uploaded URLs expire after 48 hours. OpenLess uploads one file per completed recording; it does not retain or reuse the temporary URL. + +For deployment or high-concurrency use, use stable Alibaba Cloud OSS storage and the official DashScope API directly. + +## References + +- [Fun-ASR non-real-time HTTP API](https://help.aliyun.com/en/model-studio/fun-asr-recorded-speech-recognition-http-api) +- [Qwen-ASR API reference](https://help.aliyun.com/en/model-studio/qwen-asr-api-reference) +- [DashScope temporary file upload](https://help.aliyun.com/en/model-studio/get-temporary-file-url) diff --git a/openless-all/app/src-tauri/src/asr/bailian.rs b/openless-all/app/src-tauri/src/asr/bailian.rs index 65230d95e..7b0f9b9f9 100644 --- a/openless-all/app/src-tauri/src/asr/bailian.rs +++ b/openless-all/app/src-tauri/src/asr/bailian.rs @@ -29,6 +29,7 @@ pub const DEFAULT_MODEL: &str = "fun-asr-realtime"; /// 100 ms of 16 kHz / 16-bit / mono PCM. pub const TARGET_AUDIO_CHUNK_BYTES: usize = 3_200; +const TARGET_AUDIO_CHUNK_BYTES_8K: usize = 1_600; const BYTES_PER_MS: u64 = 32; const FINAL_RESULT_TIMEOUT: Duration = Duration::from_secs(12); @@ -98,6 +99,7 @@ struct SyncState { task_id: String, pending_audio: Vec, audio_scratch: Vec, + downsample_remainder: Vec, bytes_received: u64, task_started: bool, task_finished: bool, @@ -246,6 +248,10 @@ impl BailianRealtimeASR { } let (send_tx, tail_chunks) = { let mut st = self.state.lock(); + if model_is_8k(&self.credentials.normalized_model()) { + let mut remainder = std::mem::take(&mut st.downsample_remainder); + flush_downsample_tail(&mut remainder, &mut st.audio_scratch); + } let send_tx = st.send_tx.clone(); if !st.pending_audio.is_empty() { let pending = std::mem::take(&mut st.pending_audio); @@ -288,6 +294,7 @@ impl BailianRealtimeASR { let mut st = self.state.lock(); st.pending_audio.clear(); st.audio_scratch.clear(); + st.downsample_remainder.clear(); st.send_tx.take(); st.final_tx.take(); st.task_finished = true; @@ -357,7 +364,10 @@ impl BailianRealtimeASR { st.audio_scratch.extend_from_slice(&pending); } let send_tx = st.send_tx.clone(); - let chunks = drain_audio_chunks(&mut st.audio_scratch); + let chunks = drain_audio_chunks_for_model( + &mut st.audio_scratch, + &self.credentials.normalized_model(), + ); (send_tx, chunks) }; if let Some(tx) = send_tx { @@ -505,15 +515,25 @@ impl AudioConsumer for BailianRealtimeASR { if pcm.is_empty() { return; } + let model = self.credentials.normalized_model(); let (send_tx, chunks) = { let mut st = self.state.lock(); st.bytes_received = st.bytes_received.saturating_add(pcm.len() as u64); + let audio = if model_is_8k(&model) { + st.downsample_remainder.extend_from_slice(pcm); + let complete_len = st.downsample_remainder.len() / 4 * 4; + let audio = downsample_pcm_16k_to_8k(&st.downsample_remainder[..complete_len]); + st.downsample_remainder.drain(..complete_len); + audio + } else { + pcm.to_vec() + }; if !st.task_started { - st.pending_audio.extend_from_slice(pcm); + st.pending_audio.extend_from_slice(&audio); return; } - st.audio_scratch.extend_from_slice(pcm); - let chunks = drain_audio_chunks(&mut st.audio_scratch); + st.audio_scratch.extend_from_slice(&audio); + let chunks = drain_audio_chunks_for_model(&mut st.audio_scratch, &model); (st.send_tx.clone(), chunks) }; if let Some(tx) = send_tx { @@ -525,13 +545,45 @@ impl AudioConsumer for BailianRealtimeASR { } fn drain_audio_chunks(buffer: &mut Vec) -> Vec> { + drain_audio_chunks_with_target(buffer, TARGET_AUDIO_CHUNK_BYTES) +} + +fn drain_audio_chunks_for_model(buffer: &mut Vec, model: &str) -> Vec> { + let target = if model_is_8k(model) { + TARGET_AUDIO_CHUNK_BYTES_8K + } else { + TARGET_AUDIO_CHUNK_BYTES + }; + drain_audio_chunks_with_target(buffer, target) +} + +fn drain_audio_chunks_with_target(buffer: &mut Vec, target: usize) -> Vec> { let mut chunks = Vec::new(); - while buffer.len() >= TARGET_AUDIO_CHUNK_BYTES { - chunks.push(buffer.drain(..TARGET_AUDIO_CHUNK_BYTES).collect()); + while buffer.len() >= target { + chunks.push(buffer.drain(..target).collect()); } chunks } +fn model_is_8k(model: &str) -> bool { + model.contains("-8k-") || model.starts_with("paraformer-8k-") +} + +fn downsample_pcm_16k_to_8k(pcm: &[u8]) -> Vec { + pcm.chunks_exact(4) + .flat_map(|pair| [pair[0], pair[1]]) + .collect() +} + +fn flush_downsample_tail(remainder: &mut Vec, output: &mut Vec) { + // Decimation keeps source indices 0, 2, 4, ... . A final complete i16 in + // `remainder` is therefore the next kept sample, not half of an output pair. + if remainder.len() >= 2 { + output.extend_from_slice(&remainder[..2]); + } + remainder.clear(); +} + /// 带重叠检测的文本段拼接:如果后一段的开头与前一段的末尾存在重叠, /// 只追加不重叠的尾部,避免因 API 重放或重复事件导致的累积文本重复。 /// @@ -562,7 +614,7 @@ fn merge_segments(segments: &[String]) -> String { fn run_task_message(task_id: &str, model: &str, vocabulary_id: Option<&str>) -> String { let mut parameters = json!({ - "sample_rate": 16000, + "sample_rate": if model_is_8k(model) { 8000 } else { 16000 }, "format": "pcm" }); if let Some(vocabulary_id) = vocabulary_id.map(str::trim).filter(|id| !id.is_empty()) { @@ -931,6 +983,78 @@ mod tests { assert!(value["payload"]["parameters"]["vocabulary_id"].is_null()); } + #[test] + fn run_task_message_uses_pcm_8k_for_8k_model() { + let value: Value = + serde_json::from_str(&run_task_message("abc", "fun-asr-flash-8k-realtime", None)) + .unwrap(); + assert_eq!(value["payload"]["parameters"]["sample_rate"], 8000); + } + + #[test] + fn downsample_16k_pcm_to_8k_keeps_every_other_sample() { + let pcm = [ + 1_i16.to_le_bytes(), + 2_i16.to_le_bytes(), + 3_i16.to_le_bytes(), + 4_i16.to_le_bytes(), + ] + .concat(); + let downsampled = downsample_pcm_16k_to_8k(&pcm); + let samples = downsampled + .chunks_exact(2) + .map(|chunk| i16::from_le_bytes([chunk[0], chunk[1]])) + .collect::>(); + assert_eq!(samples, vec![1, 3]); + } + + #[test] + fn downsample_flush_keeps_final_even_index_sample() { + let mut remainder = [ + 1_i16.to_le_bytes(), + 2_i16.to_le_bytes(), + 3_i16.to_le_bytes(), + ] + .concat(); + let complete_len = remainder.len() / 4 * 4; + let mut downsampled = downsample_pcm_16k_to_8k(&remainder[..complete_len]); + remainder.drain(..complete_len); + flush_downsample_tail(&mut remainder, &mut downsampled); + + let samples = downsampled + .chunks_exact(2) + .map(|chunk| i16::from_le_bytes([chunk[0], chunk[1]])) + .collect::>(); + assert_eq!(samples, vec![1, 3]); + assert!(remainder.is_empty()); + } + + #[test] + fn downsample_keeps_phase_across_recorder_chunks() { + let asr = BailianRealtimeASR::new(BailianCredentials { + api_key: "sk-test".to_string(), + endpoint: String::new(), + model: "fun-asr-flash-8k-realtime".to_string(), + vocabulary_id: None, + }); + let first = [ + 1_i16.to_le_bytes(), + 2_i16.to_le_bytes(), + 3_i16.to_le_bytes(), + ] + .concat(); + asr.consume_pcm_chunk(&first); + asr.consume_pcm_chunk(&4_i16.to_le_bytes()); + + let state = asr.state.lock(); + let samples = state + .pending_audio + .chunks_exact(2) + .map(|chunk| i16::from_le_bytes([chunk[0], chunk[1]])) + .collect::>(); + assert_eq!(samples, vec![1, 3]); + } + #[test] fn run_task_message_includes_optional_vocabulary_id() { let value: Value = diff --git a/openless-all/app/src-tauri/src/asr/dashscope_multimodal.rs b/openless-all/app/src-tauri/src/asr/dashscope_multimodal.rs index b2923f203..755d09cd6 100644 --- a/openless-all/app/src-tauri/src/asr/dashscope_multimodal.rs +++ b/openless-all/app/src-tauri/src/asr/dashscope_multimodal.rs @@ -14,6 +14,7 @@ use anyhow::{Context, Result}; use base64::Engine; use parking_lot::Mutex; use serde_json::Value; +use std::time::{Duration, Instant}; use crate::asr::mimo::{join_transcript_chunks, split_pcm_by_duration}; use crate::asr::wav::encode_wav_16k_mono; @@ -23,12 +24,55 @@ use crate::asr::RawTranscript; // 体积。沿用 mimo 验证过的 180s 预算(16k/16-bit/mono WAV base64 后约 7.7MB), // 稳稳落在时长和常见网关体积上限之内。超长录音按此切分后逐段识别再拼接。 const DASHSCOPE_MAX_CHUNK_DURATION_MS: u64 = 180_000; +const ASYNC_TASK_POLL_TIMEOUT_SECS: u64 = 600; +const ASYNC_WORKFLOW_OVERHEAD_SECS: u64 = 60; +const ASYNC_UPLOAD_BYTES_PER_SEC: u64 = 64 * 1024; pub const PROVIDER_ID: &str = "bailian-fun-asr-flash"; pub const DEFAULT_ENDPOINT: &str = "https://dashscope.aliyuncs.com/api/v1/services/aigc/multimodal-generation/generation"; +pub const ASYNC_DEFAULT_ENDPOINT: &str = + "https://dashscope.aliyuncs.com/api/v1/services/audio/asr/transcription"; pub const DEFAULT_MODEL: &str = "fun-asr-flash-2026-06-15"; +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum DashScopeBatchProtocol { + Multimodal, + AsyncTranscription, +} + +fn is_realtime_model(model: &str) -> bool { + model.contains("realtime") +} + +fn is_qwen_filetrans_model(model: &str) -> bool { + model.starts_with("qwen3-asr-flash-filetrans") +} + +fn is_qwen_sync_model(model: &str) -> bool { + model.starts_with("qwen3-asr-flash") + && !is_qwen_filetrans_model(model) + && !is_realtime_model(model) +} + +pub fn protocol_for_model(model: &str) -> Option { + let model = model.trim(); + if model.is_empty() || is_realtime_model(model) { + return None; + } + if model.starts_with("fun-asr-flash") || is_qwen_sync_model(model) { + return Some(DashScopeBatchProtocol::Multimodal); + } + if model == "fun-asr" + || model.starts_with("fun-asr-") + || model.starts_with("paraformer") + || is_qwen_filetrans_model(model) + { + return Some(DashScopeBatchProtocol::AsyncTranscription); + } + None +} + pub struct DashScopeMultimodalASR { api_key: String, base_url: String, @@ -50,6 +94,20 @@ impl DashScopeMultimodalASR { crate::asr::pcm::pcm_duration_ms(&self.buffer.lock()) } + pub fn transcribe_timeout(&self, audio_secs: f64) -> Duration { + if protocol_for_model(&self.model) == Some(DashScopeBatchProtocol::AsyncTranscription) { + let pcm_bytes = (audio_secs.max(0.0) * 32_000.0).ceil() as u64; + return async_upload_timeout(pcm_bytes.saturating_add(44)) + + Duration::from_secs( + ASYNC_TASK_POLL_TIMEOUT_SECS + ASYNC_WORKFLOW_OVERHEAD_SECS, + ); + } + let secs = ((audio_secs * 0.5).ceil() as u64) + .saturating_add(20) + .max(30); + Duration::from_secs(secs) + } + pub async fn transcribe(&self) -> Result { let pcm = self.buffer.lock().clone(); if pcm.is_empty() { @@ -72,6 +130,15 @@ impl DashScopeMultimodalASR { } let duration_ms = crate::asr::pcm::pcm_duration_ms(pcm); + if protocol_for_model(&self.model) == Some(DashScopeBatchProtocol::AsyncTranscription) { + let samples: Vec = pcm + .chunks_exact(2) + .map(|chunk| i16::from_le_bytes([chunk[0], chunk[1]])) + .collect(); + let wav = encode_wav_16k_mono(&samples); + let text = self.transcribe_async(&wav).await?; + return Ok(RawTranscript { text, duration_ms }); + } let chunks = split_pcm_by_duration(pcm, DASHSCOPE_MAX_CHUNK_DURATION_MS); let mut texts = Vec::with_capacity(chunks.len()); for chunk in chunks { @@ -92,14 +159,15 @@ impl DashScopeMultimodalASR { let wav = encode_wav_16k_mono(&samples); let body = dashscope_multimodal_body(&self.model, &wav); let url = generation_url(&self.base_url)?; - let client = reqwest::Client::new(); - let resp = client + let request_timeout = self.transcribe_timeout(crate::asr::pcm::pcm_duration_ms(pcm) as f64 / 1000.0); + let resp = crate::net::credential_http() .post(&url) .header("Authorization", format!("Bearer {}", self.api_key.trim())) .header("Content-Type", "application/json") // multimodal-generation 默认可 SSE 流式;显式关掉走一次性 JSON 响应。 .header("X-DashScope-SSE", "disable") .json(&body) + .timeout(request_timeout) .send() .await .context("DashScope ASR HTTP request failed")?; @@ -114,11 +182,204 @@ impl DashScopeMultimodalASR { Ok(extract_dashscope_text(&json).trim().to_string()) } + async fn transcribe_async(&self, wav: &[u8]) -> Result { + let file_url = self.upload_temporary_wav(wav).await?; + self.transcribe_async_url(&file_url).await + } + + async fn upload_temporary_wav(&self, wav: &[u8]) -> Result { + let mut policy_url = api_url(&self.base_url, "/api/v1/uploads")?; + policy_url + .query_pairs_mut() + .append_pair("action", "getPolicy") + .append_pair("model", self.model.trim()); + let response = crate::net::credential_http() + .get(policy_url) + .header("Authorization", format!("Bearer {}", self.api_key.trim())) + .header("Content-Type", "application/json") + .timeout(Duration::from_secs(30)) + .send() + .await + .context("request DashScope temporary upload policy")?; + let policy_json = response_json(response, "DashScope upload policy").await?; + let data = policy_json + .get("data") + .context("DashScope upload policy missing data")?; + let field = |name: &str| -> Result { + data.get(name) + .and_then(Value::as_str) + .map(str::trim) + .filter(|value| !value.is_empty()) + .map(ToOwned::to_owned) + .with_context(|| format!("DashScope upload policy missing {name}")) + }; + let upload_dir = field("upload_dir")?; + let object_key = format!("{}/audio.wav", upload_dir.trim_end_matches('/')); + let form = reqwest::multipart::Form::new() + .text("OSSAccessKeyId", field("oss_access_key_id")?) + .text("policy", field("policy")?) + .text("Signature", field("signature")?) + .text("key", object_key.clone()) + .text("x-oss-object-acl", field("x_oss_object_acl")?) + .text("x-oss-forbid-overwrite", field("x_oss_forbid_overwrite")?) + .text("success_action_status", "200") + .part( + "file", + reqwest::multipart::Part::bytes(wav.to_vec()) + .file_name("audio.wav") + .mime_str("audio/wav")?, + ); + let upload_url = dashscope_transfer_url(&field("upload_host")?)?; + let upload = crate::net::anonymous_no_redirect_http() + .post(upload_url) + .multipart(form) + .timeout(async_upload_timeout(wav.len() as u64)) + .send() + .await + .context("upload audio to DashScope temporary storage")?; + ensure_success(upload, "DashScope temporary upload").await?; + Ok(format!("oss://{object_key}")) + } + + pub async fn transcribe_async_url(&self, file_url: &str) -> Result { + let submit_url = async_transcription_url(&self.base_url)?; + let response = crate::net::credential_http() + .post(submit_url) + .header("Authorization", format!("Bearer {}", self.api_key.trim())) + .header("Content-Type", "application/json") + .header("X-DashScope-Async", "enable") + .header("X-DashScope-OssResourceResolve", "enable") + .json(&async_transcription_body(&self.model, file_url)) + .timeout(Duration::from_secs(30)) + .send() + .await + .context("submit DashScope async ASR task")?; + let submitted = response_json(response, "DashScope async ASR submission").await?; + let task_id = submitted + .pointer("/output/task_id") + .and_then(Value::as_str) + .map(str::trim) + .filter(|id| !id.is_empty()) + .context("DashScope async ASR response missing task_id")?; + let task_url = api_url(&self.base_url, &format!("/api/v1/tasks/{task_id}"))?; + let deadline = Instant::now() + Duration::from_secs(ASYNC_TASK_POLL_TIMEOUT_SECS); + let completed = loop { + let response = crate::net::credential_http() + .get(task_url.clone()) + .header("Authorization", format!("Bearer {}", self.api_key.trim())) + .timeout(Duration::from_secs(30)) + .send() + .await + .context("poll DashScope async ASR task")?; + let task = response_json(response, "DashScope async ASR task").await?; + match task + .pointer("/output/task_status") + .and_then(Value::as_str) + .unwrap_or_default() + { + "SUCCEEDED" => break task, + "FAILED" | "CANCELED" | "UNKNOWN" => { + let message = task + .get("message") + .or_else(|| task.pointer("/output/message")) + .and_then(Value::as_str) + .unwrap_or("task failed"); + anyhow::bail!("DashScope async ASR task failed: {message}"); + } + _ if Instant::now() >= deadline => { + anyhow::bail!("DashScope async ASR task timed out"); + } + _ => tokio::time::sleep(Duration::from_secs(1)).await, + } + }; + let result = download_async_result(&extract_async_result_url(&self.model, &completed)?).await?; + extract_async_transcript_text(&result) + } + pub fn cancel(&self) { self.buffer.lock().clear(); } } +fn api_url(base_url: &str, path: &str) -> Result { + let mut url = reqwest::Url::parse(base_url.trim()).context("parse DashScope base URL")?; + url.set_path(path); + url.set_query(None); + url.set_fragment(None); + Ok(url) +} + +fn dashscope_transfer_url(raw: &str) -> Result { + let mut url = reqwest::Url::parse(raw.trim()).context("parse DashScope transfer URL")?; + if !url.username().is_empty() || url.password().is_some() { + anyhow::bail!("DashScope transfer URL must not contain credentials"); + } + let host = url + .host_str() + .context("DashScope transfer URL missing host")? + .to_ascii_lowercase(); + + #[cfg(test)] + if host == "localhost" + || host + .parse::() + .is_ok_and(|ip| ip.is_loopback()) + { + return Ok(url); + } + + if host != "aliyuncs.com" && !host.ends_with(".aliyuncs.com") { + anyhow::bail!("DashScope transfer URL must use an Alibaba Cloud OSS host"); + } + match url.scheme() { + "https" => {} + "http" => url + .set_scheme("https") + .map_err(|_| anyhow::anyhow!("upgrade DashScope transfer URL to HTTPS"))?, + _ => anyhow::bail!("DashScope transfer URL must use HTTPS"), + } + Ok(url) +} + +fn async_upload_timeout(bytes: u64) -> Duration { + let transfer_secs = bytes + .saturating_add(ASYNC_UPLOAD_BYTES_PER_SEC - 1) + / ASYNC_UPLOAD_BYTES_PER_SEC; + Duration::from_secs(transfer_secs.saturating_add(30).max(60)) +} + +async fn download_async_result(raw_url: &str) -> Result { + let result_url = dashscope_transfer_url(raw_url)?; + let response = crate::net::anonymous_no_redirect_http() + .get(result_url) + .timeout(Duration::from_secs(30)) + .send() + .await + .context("download DashScope async ASR result")?; + response_json(response, "DashScope async ASR result").await +} + +async fn ensure_success(response: reqwest::Response, operation: &str) -> Result<()> { + if response.status().is_success() { + return Ok(()); + } + let status = response.status(); + let body = response.text().await.unwrap_or_default(); + anyhow::bail!("{operation} error {status}: {body}") +} + +async fn response_json(response: reqwest::Response, operation: &str) -> Result { + if !response.status().is_success() { + let status = response.status(); + let body = response.text().await.unwrap_or_default(); + anyhow::bail!("{operation} error {status}: {body}"); + } + response + .json() + .await + .with_context(|| format!("parse {operation} response")) +} + impl crate::recorder::AudioConsumer for DashScopeMultimodalASR { fn consume_pcm_chunk(&self, pcm: &[u8]) { self.buffer.lock().extend_from_slice(pcm); @@ -149,11 +410,30 @@ pub fn generation_url(base_url: &str) -> Result { Ok(url.to_string()) } +pub fn async_transcription_url(base_url: &str) -> Result { + Ok(api_url(base_url, "/api/v1/services/audio/asr/transcription")?.to_string()) +} + pub fn dashscope_multimodal_body(model: &str, wav: &[u8]) -> Value { let audio_data = format!( "data:audio/wav;base64,{}", base64::engine::general_purpose::STANDARD.encode(wav) ); + dashscope_multimodal_body_from_uri(model, &audio_data) +} + +pub fn dashscope_multimodal_body_from_uri(model: &str, audio_uri: &str) -> Value { + if is_qwen_sync_model(model) { + return serde_json::json!({ + "model": model, + "input": { + "messages": [{ + "role": "user", + "content": [{ "audio": audio_uri }], + }], + }, + }); + } serde_json::json!({ "model": model, "input": { @@ -161,7 +441,7 @@ pub fn dashscope_multimodal_body(model: &str, wav: &[u8]) -> Value { "role": "user", "content": [{ "type": "input_audio", - "input_audio": { "data": audio_data }, + "input_audio": { "data": audio_uri }, }], }], }, @@ -172,6 +452,73 @@ pub fn dashscope_multimodal_body(model: &str, wav: &[u8]) -> Value { }) } +pub fn async_transcription_body(model: &str, file_url: &str) -> Value { + if is_qwen_filetrans_model(model) { + serde_json::json!({ + "model": model, + "input": { "file_url": file_url }, + "parameters": {}, + }) + } else { + serde_json::json!({ + "model": model, + "input": { "file_urls": [file_url] }, + "parameters": {}, + }) + } +} + +pub fn extract_async_result_url(model: &str, json: &Value) -> Result { + let url = if is_qwen_filetrans_model(model) { + json.pointer("/output/result/transcription_url") + .and_then(Value::as_str) + } else { + json.pointer("/output/results/0/transcription_url") + .and_then(Value::as_str) + }; + url.map(str::trim) + .filter(|url| !url.is_empty()) + .map(ToOwned::to_owned) + .context("DashScope async ASR response missing transcription_url") +} + +pub fn extract_async_transcript_text(json: &Value) -> Result { + let transcripts = json + .get("transcripts") + .context("DashScope async ASR result missing transcripts")? + .as_array() + .context("DashScope async ASR transcripts must be an array")?; + let mut texts = Vec::new(); + for transcript in transcripts { + if let Some(value) = transcript.get("text") { + let text = value + .as_str() + .context("DashScope async ASR transcript text must be a string")? + .trim(); + if !text.is_empty() { + texts.push(text.to_string()); + } + continue; + } + let sentences = transcript + .get("sentences") + .context("DashScope async ASR transcript missing text or sentences")? + .as_array() + .context("DashScope async ASR sentences must be an array")?; + for sentence in sentences { + let text = sentence + .get("text") + .and_then(Value::as_str) + .context("DashScope async ASR sentence missing text")? + .trim(); + if !text.is_empty() { + texts.push(text.to_string()); + } + } + } + Ok(texts.join("")) +} + /// fun-asr-flash 的响应信封与标准多模态接口不同,且不同模型版本字段路径略有 /// 差异(`output.text` / `output.output.sentence.text` / 标准 `choices`)。 /// 这里按已知路径逐一兜底提取,取到第一个非空文本即返回,避免因单一路径假设 @@ -240,6 +587,10 @@ mod tests { use crate::recorder::AudioConsumer; use std::io::{Read, Write}; use std::net::TcpListener; + use std::sync::{ + atomic::{AtomicBool, Ordering}, + Arc, + }; use std::thread; use std::time::{Duration, Instant}; @@ -275,6 +626,126 @@ mod tests { assert!(body["parameters"].get("vocabulary_id").is_none()); } + #[test] + fn qwen_flash_body_uses_documented_audio_shape() { + let body = dashscope_multimodal_body("qwen3-asr-flash", b"wav"); + assert_eq!(body["model"], "qwen3-asr-flash"); + let audio = &body["input"]["messages"][0]["content"][0]; + assert!(audio["audio"] + .as_str() + .unwrap() + .starts_with("data:audio/wav;base64,")); + assert!(audio.get("input_audio").is_none()); + } + + #[test] + fn classifies_supported_batch_model_protocols() { + assert_eq!( + protocol_for_model("fun-asr-flash-2026-06-15"), + Some(DashScopeBatchProtocol::Multimodal) + ); + assert_eq!( + protocol_for_model("qwen3-asr-flash-2026-02-10"), + Some(DashScopeBatchProtocol::Multimodal) + ); + for model in [ + "fun-asr", + "fun-asr-mtl-2025-08-25", + "paraformer-v2", + "qwen3-asr-flash-filetrans-2025-11-17", + ] { + assert_eq!( + protocol_for_model(model), + Some(DashScopeBatchProtocol::AsyncTranscription), + "unexpected protocol for {model}" + ); + } + assert_eq!(protocol_for_model("unknown-asr"), None); + } + + #[test] + fn async_models_get_a_task_polling_timeout() { + let asr = DashScopeMultimodalASR::new( + "sk-test".to_string(), + ASYNC_DEFAULT_ENDPOINT.to_string(), + "fun-asr".to_string(), + ); + assert!(asr.transcribe_timeout(1.0) >= Duration::from_secs(660)); + assert!(asr.transcribe_timeout(1_800.0) >= Duration::from_secs(1_500)); + assert!(asr.transcribe_timeout(1_800.0) > asr.transcribe_timeout(1.0)); + assert!(async_upload_timeout(58_000_000) >= Duration::from_secs(900)); + } + + #[test] + fn async_body_uses_model_specific_input_shape() { + let funasr = async_transcription_body("fun-asr", "oss://bucket/test.wav"); + assert_eq!(funasr["input"]["file_urls"][0], "oss://bucket/test.wav"); + assert!(funasr["input"].get("file_url").is_none()); + assert_eq!(funasr["parameters"], serde_json::json!({})); + + let qwen = async_transcription_body("qwen3-asr-flash-filetrans", "oss://bucket/test.wav"); + assert_eq!(qwen["input"]["file_url"], "oss://bucket/test.wav"); + assert!(qwen["input"].get("file_urls").is_none()); + assert_eq!(qwen["parameters"], serde_json::json!({})); + } + + #[test] + fn extracts_model_specific_async_result_url() { + let funasr = serde_json::json!({ + "output": {"results": [{ + "subtask_status": "SUCCEEDED", + "transcription_url": "https://result.example/funasr.json" + }]} + }); + assert_eq!( + extract_async_result_url("fun-asr", &funasr).unwrap(), + "https://result.example/funasr.json" + ); + + let qwen = serde_json::json!({ + "output": {"result": { + "transcription_url": "https://result.example/qwen.json" + }} + }); + assert_eq!( + extract_async_result_url("qwen3-asr-flash-filetrans", &qwen).unwrap(), + "https://result.example/qwen.json" + ); + } + + #[test] + fn extracts_text_from_async_result_documents() { + let funasr = serde_json::json!({ + "transcripts": [{"sentences": [{"text": "第一句"}, {"text": "第二句"}]}] + }); + assert_eq!(extract_async_transcript_text(&funasr).unwrap(), "第一句第二句"); + + let qwen = serde_json::json!({ + "transcripts": [{"text": "Qwen 转写结果"}] + }); + assert_eq!(extract_async_transcript_text(&qwen).unwrap(), "Qwen 转写结果"); + } + + #[test] + fn rejects_malformed_async_result_documents() { + assert!(extract_async_transcript_text(&serde_json::json!({})).is_err()); + assert!(extract_async_transcript_text(&serde_json::json!({ + "transcripts": [{"unexpected": "shape"}] + })) + .is_err()); + } + + #[test] + fn validates_dashscope_transfer_urls() { + let upgraded = dashscope_transfer_url( + "http://dashscope-file.oss-cn-beijing.aliyuncs.com/result.json", + ) + .unwrap(); + assert_eq!(upgraded.scheme(), "https"); + assert!(dashscope_transfer_url("http://169.254.169.254/latest/meta-data").is_err()); + assert!(dashscope_transfer_url("https://aliyuncs.com.evil.example/result.json").is_err()); + } + #[test] fn extract_text_prefers_output_text() { let json = serde_json::json!({ "output": { "text": " 你好世界 " } }); @@ -366,6 +837,198 @@ mod tests { server.join().unwrap(); } + #[tokio::test] + async fn uploads_and_polls_async_transcription() { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let addr = listener.local_addr().unwrap(); + let server = thread::spawn(move || { + for step in 0..5 { + let (mut stream, _) = listener.accept().unwrap(); + stream + .set_read_timeout(Some(Duration::from_secs(5))) + .unwrap(); + let request = read_http_request(&mut stream); + let request_text = String::from_utf8_lossy(&request); + let lower = request_text.to_ascii_lowercase(); + if matches!(step, 0 | 2 | 3) { + assert!(lower.contains("authorization: bearer sk-test")); + } else { + assert!(!lower.contains("authorization:")); + } + match step { + 0 => { + assert!(request_text.starts_with( + "GET /api/v1/uploads?action=getPolicy&model=fun-asr HTTP/1.1" + )); + write_json_response( + &mut stream, + &format!( + r#"{{"data":{{"policy":"policy","signature":"signature","upload_dir":"dashscope-instant/test","upload_host":"http://{addr}","oss_access_key_id":"key-id","x_oss_object_acl":"private","x_oss_forbid_overwrite":"true"}}}}"# + ), + ); + } + 1 => { + assert!(request_text.starts_with("POST / HTTP/1.1")); + assert!(lower.contains("content-type: multipart/form-data")); + assert!(request_text.contains("OSSAccessKeyId")); + assert!(request_text.contains("success_action_status")); + assert!(request_text.contains("audio.wav")); + write_json_response(&mut stream, "{}"); + } + 2 => { + assert!(request_text + .starts_with("POST /api/v1/services/audio/asr/transcription HTTP/1.1")); + assert!(lower.contains("x-dashscope-async: enable")); + assert!(lower.contains("x-dashscope-ossresourceresolve: enable")); + assert!(request_text + .contains(r#""file_urls":["oss://dashscope-instant/test/audio.wav"]"#)); + write_json_response(&mut stream, r#"{"output":{"task_id":"task-1"}}"#); + } + 3 => { + assert!(request_text.starts_with("GET /api/v1/tasks/task-1 HTTP/1.1")); + write_json_response( + &mut stream, + &format!( + r#"{{"output":{{"task_status":"SUCCEEDED","results":[{{"subtask_status":"SUCCEEDED","transcription_url":"http://{addr}/result.json"}}]}}}}"# + ), + ); + } + 4 => { + assert!(request_text.starts_with("GET /result.json HTTP/1.1")); + write_json_response( + &mut stream, + r#"{"transcripts":[{"sentences":[{"text":"异步"},{"text":"转写"}]}]}"#, + ); + } + _ => unreachable!(), + } + } + }); + + let asr = DashScopeMultimodalASR::new( + "sk-test".to_string(), + format!("http://{addr}/api/v1/services/audio/asr/transcription"), + "fun-asr".to_string(), + ); + asr.consume_pcm_chunk(&vec![0u8; 32_000]); + let transcript = asr.transcribe().await.unwrap(); + assert_eq!(transcript.text, "异步转写"); + assert_eq!(transcript.duration_ms, 1_000); + server.join().unwrap(); + } + + #[tokio::test] + async fn async_policy_request_does_not_follow_redirects_with_credentials() { + let redirect_listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let redirect_addr = redirect_listener.local_addr().unwrap(); + let target_listener = TcpListener::bind("127.0.0.1:0").unwrap(); + target_listener.set_nonblocking(true).unwrap(); + let target_addr = target_listener.local_addr().unwrap(); + let followed = Arc::new(AtomicBool::new(false)); + let target_followed = Arc::clone(&followed); + + let redirect_server = thread::spawn(move || { + let (mut stream, _) = redirect_listener.accept().unwrap(); + let request = read_http_request(&mut stream); + assert!(String::from_utf8_lossy(&request) + .to_ascii_lowercase() + .contains("authorization: bearer sk-test")); + let response = format!( + "HTTP/1.1 302 Found\r\nlocation: http://{target_addr}/api/v1/uploads\r\ncontent-length: 0\r\nconnection: close\r\n\r\n" + ); + stream.write_all(response.as_bytes()).unwrap(); + stream.flush().unwrap(); + }); + let target_server = thread::spawn(move || { + let deadline = Instant::now() + Duration::from_secs(2); + loop { + match target_listener.accept() { + Ok((mut stream, _)) => { + target_followed.store(true, Ordering::SeqCst); + let _request = read_http_request(&mut stream); + let body = r#"{"message":"redirect followed"}"#; + let response = format!( + "HTTP/1.1 400 Bad Request\r\ncontent-type: application/json\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{}", + body.len(), + body + ); + stream.write_all(response.as_bytes()).unwrap(); + stream.flush().unwrap(); + break; + } + Err(err) if err.kind() == std::io::ErrorKind::WouldBlock => { + if Instant::now() >= deadline { + break; + } + thread::sleep(Duration::from_millis(10)); + } + Err(err) => panic!("accept redirect target request failed: {err}"), + } + } + }); + + let asr = DashScopeMultimodalASR::new( + "sk-test".to_string(), + format!("http://{redirect_addr}/api/v1/services/audio/asr/transcription"), + "fun-asr".to_string(), + ); + asr.consume_pcm_chunk(&vec![0u8; 32_000]); + let error = asr.transcribe().await.unwrap_err().to_string(); + + assert!(error.contains("302 Found"), "unexpected error: {error}"); + redirect_server.join().unwrap(); + target_server.join().unwrap(); + assert!(!followed.load(Ordering::SeqCst)); + } + + #[tokio::test] + async fn async_result_download_does_not_follow_redirects() { + let redirect_listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let redirect_addr = redirect_listener.local_addr().unwrap(); + let target_listener = TcpListener::bind("127.0.0.1:0").unwrap(); + target_listener.set_nonblocking(true).unwrap(); + let target_addr = target_listener.local_addr().unwrap(); + let followed = Arc::new(AtomicBool::new(false)); + let target_followed = Arc::clone(&followed); + let redirect_server = thread::spawn(move || { + let (mut stream, _) = redirect_listener.accept().unwrap(); + let request = read_http_request(&mut stream); + assert!(!String::from_utf8_lossy(&request) + .to_ascii_lowercase() + .contains("authorization:")); + let response = format!( + "HTTP/1.1 302 Found\r\nlocation: http://{target_addr}/result.json\r\ncontent-length: 0\r\nconnection: close\r\n\r\n" + ); + stream.write_all(response.as_bytes()).unwrap(); + stream.flush().unwrap(); + }); + let target_server = thread::spawn(move || { + let deadline = Instant::now() + Duration::from_secs(2); + while Instant::now() < deadline { + match target_listener.accept() { + Ok((_stream, _)) => { + target_followed.store(true, Ordering::SeqCst); + break; + } + Err(err) if err.kind() == std::io::ErrorKind::WouldBlock => { + thread::sleep(Duration::from_millis(10)); + } + Err(err) => panic!("accept result redirect target failed: {err}"), + } + } + }); + + let error = download_async_result(&format!("http://{redirect_addr}/result.json")) + .await + .unwrap_err() + .to_string(); + + assert!(error.contains("302 Found"), "unexpected error: {error}"); + redirect_server.join().unwrap(); + target_server.join().unwrap(); + assert!(!followed.load(Ordering::SeqCst)); + } + fn read_http_request(stream: &mut std::net::TcpStream) -> Vec { let mut request = Vec::new(); let mut buf = [0u8; 4096]; @@ -389,14 +1052,17 @@ mod tests { fn parse_expected_request_len(request: &[u8]) -> Option { let header_end = request.windows(4).position(|w| w == b"\r\n\r\n")? + 4; let headers = String::from_utf8_lossy(&request[..header_end]); - let content_len = headers.lines().find_map(|line| { - let (name, value) = line.split_once(':')?; - if name.eq_ignore_ascii_case("content-length") { - value.trim().parse::().ok() - } else { - None - } - })?; + let content_len = headers + .lines() + .find_map(|line| { + let (name, value) = line.split_once(':')?; + if name.eq_ignore_ascii_case("content-length") { + value.trim().parse::().ok() + } else { + None + } + }) + .unwrap_or(0); Some(header_end + content_len) } diff --git a/openless-all/app/src-tauri/src/commands/providers.rs b/openless-all/app/src-tauri/src/commands/providers.rs index b13148738..c5cebbd56 100644 --- a/openless-all/app/src-tauri/src/commands/providers.rs +++ b/openless-all/app/src-tauri/src/commands/providers.rs @@ -27,10 +27,7 @@ pub async fn validate_provider_credentials(kind: String) -> Result Result { - if kind == "asr" - && CredentialsVault::get_active_asr() == crate::asr::bailian::PROVIDER_ID - && !cfg!(mobile) - { + if kind == "asr" && CredentialsVault::get_active_asr() == crate::asr::bailian::PROVIDER_ID { // 统一「阿里云百炼」入口:三条协议(实时 fun-asr-realtime / 实时 qwen3 / // 录音文件 fun-asr-flash)收成一个 provider。百炼各网关都没有模型列表 HTTP // 接口,列表是静态的;但先跑一次与「验证」相同的、按当前所选模型对应协议的 @@ -42,22 +39,22 @@ pub async fn list_provider_models(kind: String) -> Result Result<(), String> { // 统一百炼复用配置中的区域/工作空间主机,并推导 multimodal 的 https 路径。 // 隐藏别名仍按原有完整 endpoint 读取。 + let model = CredentialsVault::get(CredentialAccount::AsrModel) + .map_err(|e| e.to_string())? + .filter(|s| !s.trim().is_empty()) + .unwrap_or_else(|| crate::asr::dashscope_multimodal::DEFAULT_MODEL.to_string()); + crate::coordinator::validate_dashscope_multimodal_model(&model)?; + let protocol = crate::asr::dashscope_multimodal::protocol_for_model(&model) + .unwrap_or(crate::asr::dashscope_multimodal::DashScopeBatchProtocol::Multimodal); let (api_key, base_url) = if crate::coordinator::unified_bailian_is_active() { let api_key = CredentialsVault::get(CredentialAccount::AsrApiKey) .map_err(|e| e.to_string())? @@ -406,39 +410,53 @@ async fn validate_dashscope_multimodal_asr_provider() -> Result<(), String> { let endpoint = CredentialsVault::get(CredentialAccount::AsrEndpoint) .map_err(|e| e.to_string())? .unwrap_or_default(); - let endpoint = crate::coordinator::derive_bailian_endpoint( - &endpoint, - crate::coordinator::BailianEndpointProtocol::Multimodal, - )?; + let endpoint_protocol = match protocol { + crate::asr::dashscope_multimodal::DashScopeBatchProtocol::Multimodal => { + crate::coordinator::BailianEndpointProtocol::Multimodal + } + crate::asr::dashscope_multimodal::DashScopeBatchProtocol::AsyncTranscription => { + crate::coordinator::BailianEndpointProtocol::AsyncTranscription + } + }; + let endpoint = crate::coordinator::derive_bailian_endpoint(&endpoint, endpoint_protocol)?; (api_key, endpoint) } else { let config = read_openai_provider_config("asr")?; (config.api_key, config.base_url) }; - let model = CredentialsVault::get(CredentialAccount::AsrModel) - .map_err(|e| e.to_string())? - .filter(|s| !s.trim().is_empty()) - .unwrap_or_else(|| crate::asr::dashscope_multimodal::DEFAULT_MODEL.to_string()); - crate::coordinator::validate_dashscope_multimodal_model(&model)?; + if protocol == crate::asr::dashscope_multimodal::DashScopeBatchProtocol::AsyncTranscription { + let asr = crate::asr::DashScopeMultimodalASR::new(api_key, base_url, model); + return match tokio::time::timeout( + std::time::Duration::from_secs(660), + asr.transcribe_async_url(DASHSCOPE_ASR_VALIDATE_SAMPLE_URL), + ) + .await + { + Ok(result) => result.map(|_| ()).map_err(|error| error.to_string()), + Err(_) => Err("providerRequestTimeout".to_string()), + }; + } let url = crate::asr::dashscope_multimodal::generation_url(&base_url) .map_err(|_| "endpointInvalid".to_string())?; - let body = serde_json::json!({ - "model": model, - "input": { "messages": [{ "role": "user", "content": [{ - "type": "input_audio", - "input_audio": { "data": DASHSCOPE_ASR_VALIDATE_SAMPLE_URL }, - }]}]}, - "parameters": { "format": "wav", "sample_rate": "16000" }, - }); - let client = http_client_builder(&url, 20) - .build() - .map_err(|_| "providerClientInitFailed".to_string())?; - let response = client - .post(&url) + let body = crate::asr::dashscope_multimodal::dashscope_multimodal_body_from_uri( + &model, + DASHSCOPE_ASR_VALIDATE_SAMPLE_URL, + ); + send_dashscope_multimodal_validation(&api_key, url.as_str(), &body).await +} + +async fn send_dashscope_multimodal_validation( + api_key: &str, + url: &str, + body: &Value, +) -> Result<(), String> { + let response = crate::net::credential_http() + .post(url) .header("Authorization", format!("Bearer {}", api_key)) .header("Content-Type", "application/json") .header("X-DashScope-SSE", "disable") .json(&body) + .timeout(std::time::Duration::from_secs(20)) .send() .await .map_err(|e| { @@ -914,7 +932,7 @@ mod tests { use super::{ asr_error_is_no_speech_rejection, fetch_provider_models, models_url, provider_llm_error_message, provider_log_context, provider_request_error_message, - sanitized_provider_destination, ProviderConfig, + sanitized_provider_destination, send_dashscope_multimodal_validation, ProviderConfig, }; use crate::endpoint_security::validate_http_endpoint; @@ -1107,6 +1125,45 @@ mod tests { ); } + #[tokio::test] + async fn dashscope_validation_does_not_follow_redirects_with_credentials() { + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + use tokio::net::TcpListener; + + let redirect_listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let redirect_addr = redirect_listener.local_addr().unwrap(); + let target_listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let target_addr = target_listener.local_addr().unwrap(); + let redirect_server = tokio::spawn(async move { + let (mut stream, _) = redirect_listener.accept().await.unwrap(); + let mut request = [0_u8; 2048]; + let count = stream.read(&mut request).await.unwrap(); + let request = String::from_utf8_lossy(&request[..count]).to_ascii_lowercase(); + assert!(request.contains("authorization: bearer sk-test")); + let response = format!( + "HTTP/1.1 302 Found\r\nLocation: http://{target_addr}/stolen\r\nContent-Length: 0\r\nConnection: close\r\n\r\n" + ); + stream.write_all(response.as_bytes()).await.unwrap(); + }); + let target_server = tokio::spawn(async move { + tokio::time::timeout(std::time::Duration::from_millis(500), target_listener.accept()) + .await + .is_ok() + }); + + let error = send_dashscope_multimodal_validation( + "sk-test", + &format!("http://{redirect_addr}/validate"), + &serde_json::json!({"model": "fun-asr-flash"}), + ) + .await + .unwrap_err(); + + redirect_server.await.unwrap(); + assert_eq!(error, "providerHttpStatus:302"); + assert!(!target_server.await.unwrap(), "validation followed redirect"); + } + #[test] fn asr_endpoint_rejects_metadata_cgnat_and_non_https_public() { // 元数据 / CGNAT / 非 https 外网:拒绝,避免带 API Key 的 ASR 请求被指向高价值目标 / 明文外泄。 diff --git a/openless-all/app/src-tauri/src/coordinator.rs b/openless-all/app/src-tauri/src/coordinator.rs index 868f8c97c..783af15e1 100644 --- a/openless-all/app/src-tauri/src/coordinator.rs +++ b/openless-all/app/src-tauri/src/coordinator.rs @@ -338,28 +338,23 @@ pub(crate) fn resolve_effective_asr_provider( return Ok(active_asr.to_string()); } - // Android/iOS 继续使用原来的 Bailian WebSocket 配置。统一入口的模型路由 - // 只在桌面端启用,避免共享设置页意外改变移动端行为。 - if cfg!(mobile) { - return Ok(crate::asr::bailian::PROVIDER_ID.to_string()); - } - let model = model.trim(); if model.is_empty() || is_classic_bailian_realtime_model(model) { Ok(crate::asr::bailian::PROVIDER_ID.to_string()) } else if model.starts_with("qwen3-asr-flash-realtime") { Ok(crate::asr::qwen_realtime::PROVIDER_ID.to_string()) - } else if model == crate::asr::dashscope_multimodal::DEFAULT_MODEL { + } else if crate::asr::dashscope_multimodal::protocol_for_model(model).is_some() { Ok(crate::asr::dashscope_multimodal::PROVIDER_ID.to_string()) } else { Err(format!( - "不支持的百炼 ASR 模型:{model}。支持 fun-asr-realtime、paraformer-realtime、sensevoice-realtime、qwen3-asr-flash-realtime 和 fun-asr-flash-2026-06-15" + "不支持的百炼 ASR 模型:{model}。支持 Fun-ASR、Paraformer、SenseVoice 和 Qwen3-ASR 的实时、同步及录音文件模型" )) } } fn is_classic_bailian_realtime_model(model: &str) -> bool { model.starts_with("fun-asr-realtime") + || model.starts_with("fun-asr-flash-8k-realtime") || model.starts_with("paraformer-realtime") || model.starts_with("sensevoice-realtime") } @@ -372,12 +367,10 @@ pub(crate) fn stepfun_model_is_stream(model: &str) -> bool { pub(crate) fn validate_dashscope_multimodal_model(model: &str) -> Result<(), String> { let model = model.trim(); - if model.is_empty() || model == crate::asr::dashscope_multimodal::DEFAULT_MODEL { + if model.is_empty() || crate::asr::dashscope_multimodal::protocol_for_model(model).is_some() { return Ok(()); } - Err(format!( - "不支持的 DashScope 录音文件 ASR 模型:{model}。该协议仅支持 fun-asr-flash-2026-06-15;fun-asr-flash-8k-realtime 系列属于 8 kHz 实时 WebSocket 模型,当前尚未支持" - )) + Err(format!("不支持的 DashScope 录音文件 ASR 模型:{model}")) } #[derive(Clone, Copy)] @@ -385,6 +378,7 @@ pub(crate) enum BailianEndpointProtocol { ClassicRealtime, QwenRealtime, Multimodal, + AsyncTranscription, } /// 统一百炼配置只需要表达区域/工作空间主机;具体协议的 scheme 与 path 由模型路由决定。 @@ -397,6 +391,9 @@ pub(crate) fn derive_bailian_endpoint( BailianEndpointProtocol::ClassicRealtime => crate::asr::bailian::DEFAULT_ENDPOINT, BailianEndpointProtocol::QwenRealtime => crate::asr::qwen_realtime::DEFAULT_ENDPOINT, BailianEndpointProtocol::Multimodal => crate::asr::dashscope_multimodal::DEFAULT_ENDPOINT, + BailianEndpointProtocol::AsyncTranscription => { + crate::asr::dashscope_multimodal::ASYNC_DEFAULT_ENDPOINT + } }; let source = if endpoint.trim().is_empty() { default_endpoint @@ -414,6 +411,9 @@ pub(crate) fn derive_bailian_endpoint( "https", "/api/v1/services/aigc/multimodal-generation/generation", ), + BailianEndpointProtocol::AsyncTranscription => { + ("https", "/api/v1/services/audio/asr/transcription") + } }; url.set_scheme(scheme) .map_err(|_| "endpointInvalid".to_string())?; @@ -1876,9 +1876,7 @@ impl Coordinator { let consumer = start.recorder_consumer(); consumer.consume_pcm_chunk(&pcm); let timeout = std::time::Duration::from_secs(COORDINATOR_GLOBAL_TIMEOUT_SECS); - let dashscope_timeout = whisper_transcribe_timeout( - crate::asr::pcm::pcm_duration_ms(&pcm) as f64 / 1000.0, - ); + let audio_secs = crate::asr::pcm::pcm_duration_ms(&pcm) as f64 / 1000.0; let elevenlabs_timeout = crate::asr::elevenlabs::transcribe_timeout( crate::asr::pcm::pcm_duration_ms(&pcm) as f64 / 1000.0, ); @@ -1920,7 +1918,7 @@ impl Coordinator { .map_err(|_| "重新转录超时".to_string())? .map_err(|e| e.to_string())?, ActiveAsr::DashScopeMultimodal(m) => { - tokio::time::timeout(dashscope_timeout, m.transcribe()) + tokio::time::timeout(m.transcribe_timeout(audio_secs), m.transcribe()) .await .map_err(|_| "重新转录超时".to_string())? .map_err(|e| e.to_string())? @@ -2358,9 +2356,20 @@ fn read_dashscope_multimodal_credentials() -> (String, String, String) { .ok() .flatten() .unwrap_or_default(); + let model = CredentialsVault::get(CredentialAccount::AsrModel) + .ok() + .flatten() + .filter(|s| !s.trim().is_empty()) + .unwrap_or_else(|| crate::asr::dashscope_multimodal::DEFAULT_MODEL.to_string()); let base_url = if unified_bailian_is_active() { let endpoint = read_asr_endpoint(crate::asr::bailian::DEFAULT_ENDPOINT); - derive_bailian_endpoint(&endpoint, BailianEndpointProtocol::Multimodal).unwrap_or(endpoint) + let protocol = match crate::asr::dashscope_multimodal::protocol_for_model(&model) { + Some(crate::asr::dashscope_multimodal::DashScopeBatchProtocol::AsyncTranscription) => { + BailianEndpointProtocol::AsyncTranscription + } + _ => BailianEndpointProtocol::Multimodal, + }; + derive_bailian_endpoint(&endpoint, protocol).unwrap_or(endpoint) } else { CredentialsVault::get(CredentialAccount::AsrEndpoint) .ok() @@ -2368,11 +2377,6 @@ fn read_dashscope_multimodal_credentials() -> (String, String, String) { .filter(|s| !s.trim().is_empty()) .unwrap_or_else(|| crate::asr::dashscope_multimodal::DEFAULT_ENDPOINT.to_string()) }; - let model = CredentialsVault::get(CredentialAccount::AsrModel) - .ok() - .flatten() - .filter(|s| !s.trim().is_empty()) - .unwrap_or_else(|| crate::asr::dashscope_multimodal::DEFAULT_MODEL.to_string()); (api_key, base_url, model) } @@ -3047,6 +3051,25 @@ mod tests { resolve_effective_asr_provider(bailian, "fun-asr-flash-2026-06-15").unwrap(), crate::asr::dashscope_multimodal::PROVIDER_ID ); + for model in [ + "fun-asr-flash-2026-09-01", + "qwen3-asr-flash", + "qwen3-asr-flash-2026-02-10", + "fun-asr", + "fun-asr-mtl-2025-08-25", + "qwen3-asr-flash-filetrans", + "paraformer-v2", + ] { + assert_eq!( + resolve_effective_asr_provider(bailian, model).unwrap(), + crate::asr::dashscope_multimodal::PROVIDER_ID, + "unexpected route for {model}" + ); + } + assert_eq!( + resolve_effective_asr_provider(bailian, "fun-asr-flash-8k-realtime").unwrap(), + crate::asr::bailian::PROVIDER_ID + ); assert_eq!( resolve_effective_asr_provider(bailian, "paraformer-realtime-v2").unwrap(), crate::asr::bailian::PROVIDER_ID @@ -3070,16 +3093,9 @@ mod tests { #[test] fn resolve_effective_asr_provider_rejects_unsupported_bailian_model() { - let error = resolve_effective_asr_provider(crate::asr::bailian::PROVIDER_ID, "paraformer-v2") + let error = resolve_effective_asr_provider(crate::asr::bailian::PROVIDER_ID, "unknown-asr") .unwrap_err(); assert!(error.contains("不支持的百炼 ASR 模型")); - - let error = resolve_effective_asr_provider( - crate::asr::bailian::PROVIDER_ID, - "fun-asr-flash-8k-realtime", - ) - .unwrap_err(); - assert!(error.contains("不支持的百炼 ASR 模型")); } #[test] @@ -3097,6 +3113,11 @@ mod tests { derive_bailian_endpoint(endpoint, BailianEndpointProtocol::Multimodal).unwrap(), "https://workspace.ap-southeast-1.maas.aliyuncs.com/api/v1/services/aigc/multimodal-generation/generation" ); + assert_eq!( + derive_bailian_endpoint(endpoint, BailianEndpointProtocol::AsyncTranscription) + .unwrap(), + "https://workspace.ap-southeast-1.maas.aliyuncs.com/api/v1/services/audio/asr/transcription" + ); } #[test] diff --git a/openless-all/app/src-tauri/src/coordinator/dictation.rs b/openless-all/app/src-tauri/src/coordinator/dictation.rs index f396df076..8be6f25d1 100644 --- a/openless-all/app/src-tauri/src/coordinator/dictation.rs +++ b/openless-all/app/src-tauri/src/coordinator/dictation.rs @@ -2722,7 +2722,7 @@ pub(super) async fn end_session(inner: &Arc) -> Result<(), String> { ActiveAsr::DashScopeMultimodal(m) => { debug_assert!(uses_global_timeout); let audio_secs = m.buffer_duration_ms() as f64 / 1000.0; - let timeout_duration = whisper_transcribe_timeout(audio_secs); + let timeout_duration = m.transcribe_timeout(audio_secs); log::info!( "[coord] DashScope Fun-ASR-Flash dynamic timeout: {}s (audio {:.2}s)", timeout_duration.as_secs(), diff --git a/openless-all/app/src-tauri/src/coordinator/qa_session.rs b/openless-all/app/src-tauri/src/coordinator/qa_session.rs index c720bc679..ac07a317a 100644 --- a/openless-all/app/src-tauri/src/coordinator/qa_session.rs +++ b/openless-all/app/src-tauri/src/coordinator/qa_session.rs @@ -480,11 +480,17 @@ pub(super) async fn transcribe_overlay_dictation_asr( ActiveAsr::DashScopeMultimodal(asr) => { debug_assert!(uses_global_timeout); let audio_secs = asr.buffer_duration_ms() as f64 / 1000.0; - let timeout_duration = whisper_transcribe_timeout(audio_secs); - match tokio::time::timeout(timeout_duration, asr.transcribe()).await { - Ok(Ok(raw)) => Ok(raw), - Ok(Err(error)) => Err(error.to_string()), - Err(_) => Err("dashscope multimodal global timeout".to_string()), + let timeout_duration = asr.transcribe_timeout(audio_secs); + tokio::select! { + result = tokio::time::timeout(timeout_duration, asr.transcribe()) => match result { + Ok(Ok(raw)) => Ok(raw), + Ok(Err(error)) => Err(error.to_string()), + Err(_) => Err("dashscope multimodal global timeout".to_string()), + }, + _ = wait_for_overlay_dictation_cancel(_inner, _current_session_id) => { + asr.cancel(); + return OverlayDictationTranscribeOutcome::Cancelled; + } } } ActiveAsr::ElevenLabs(asr) => { @@ -1145,22 +1151,29 @@ pub(super) async fn end_qa_session(inner: &Arc) -> Result<(), String> { ActiveAsr::DashScopeMultimodal(m) => { debug_assert!(uses_global_timeout); let audio_secs = m.buffer_duration_ms() as f64 / 1000.0; - let timeout_duration = whisper_transcribe_timeout(audio_secs); - match tokio::time::timeout(timeout_duration, m.transcribe()).await { - Ok(Ok(r)) => r, - Ok(Err(e)) => { - log::error!("[coord] QA: DashScope Fun-ASR-Flash transcribe failed: {e}"); - finish_qa_with_error_if_current(inner, session_id, format!("识别失败: {e}")); - return Err(e.to_string()); - } - Err(_) => { - log::error!( - "[coord] QA: DashScope Fun-ASR-Flash dynamic timeout {}s (audio {:.2}s)", - timeout_duration.as_secs(), - audio_secs - ); - finish_qa_with_error_if_current(inner, session_id, "识别超时".to_string()); - return Err("dashscope multimodal global timeout".to_string()); + let timeout_duration = m.transcribe_timeout(audio_secs); + tokio::select! { + result = tokio::time::timeout(timeout_duration, m.transcribe()) => match result { + Ok(Ok(r)) => r, + Ok(Err(e)) => { + log::error!("[coord] QA: DashScope Fun-ASR-Flash transcribe failed: {e}"); + finish_qa_with_error_if_current(inner, session_id, format!("识别失败: {e}")); + return Err(e.to_string()); + } + Err(_) => { + log::error!( + "[coord] QA: DashScope Fun-ASR-Flash dynamic timeout {}s (audio {:.2}s)", + timeout_duration.as_secs(), + audio_secs + ); + finish_qa_with_error_if_current(inner, session_id, "识别超时".to_string()); + return Err("dashscope multimodal global timeout".to_string()); + } + }, + _ = wait_for_qa_processing_cancel(inner, session_id) => { + m.cancel(); + finish_qa_idle_silently_if_current(inner, session_id); + return Ok(()); } } } diff --git a/openless-all/app/src/i18n/en.ts b/openless-all/app/src/i18n/en.ts index f3a9caa65..4e16cac32 100644 --- a/openless-all/app/src/i18n/en.ts +++ b/openless-all/app/src/i18n/en.ts @@ -837,7 +837,8 @@ export const en: typeof zhCN = { bailianVocabularyIdLabel: 'Hotword Vocabulary ID (optional)', bailianVocabularyIdNote: 'If you have created a DashScope hotword vocabulary, enter its vocab-... ID. Leave blank to skip hotwords.', bailianModelRealtimeHint: 'Realtime model · transcribes as you speak.', - bailianModelRecordedFileHint: 'Recorded-file model · transcribes after you finish (single clip ≤ 5 min).', + bailianModelSyncFileHint: 'Synchronous recording model · transcribes after you finish (single clip ≤ 5 min).', + bailianModelAsyncFileHint: 'Asynchronous file model · uploads the recording and waits for the transcription task.', appIdLabel: 'App ID', accessKeyLabel: 'Access Key', resourceIdLabel: 'Resource ID', diff --git a/openless-all/app/src/i18n/ja.ts b/openless-all/app/src/i18n/ja.ts index 17988bb57..ce01dafa0 100644 --- a/openless-all/app/src/i18n/ja.ts +++ b/openless-all/app/src/i18n/ja.ts @@ -839,7 +839,8 @@ export const ja: typeof zhCN = { bailianVocabularyIdLabel: 'ホットワード Vocabulary ID(任意)', bailianVocabularyIdNote: 'DashScope でホットワード辞書を作成済みの場合は vocab-... ID を入力します。空欄なら送信しません。', bailianModelRealtimeHint: 'リアルタイムモデル · 話しながら文字起こし。', - bailianModelRecordedFileHint: '録音ファイルモデル · 話し終えてから一括で文字起こし(1 本 ≤ 5 分)。', + bailianModelSyncFileHint: '同期録音モデル · 話し終えてから一括で文字起こし(1 本 ≤ 5 分)。', + bailianModelAsyncFileHint: '非同期ファイルモデル · 録音をアップロードし、文字起こしタスクの完了を待ちます。', appIdLabel: 'App ID(アプリケーション ID)', accessKeyLabel: 'Access Key', resourceIdLabel: 'Resource ID', diff --git a/openless-all/app/src/i18n/ko.ts b/openless-all/app/src/i18n/ko.ts index 81e86b390..865c224f1 100644 --- a/openless-all/app/src/i18n/ko.ts +++ b/openless-all/app/src/i18n/ko.ts @@ -839,7 +839,8 @@ export const ko: typeof zhCN = { bailianVocabularyIdLabel: '핫워드 Vocabulary ID(선택)', bailianVocabularyIdNote: 'DashScope에서 핫워드 사전을 만들었다면 vocab-... ID를 입력하세요. 비워 두면 핫워드를 전송하지 않습니다.', bailianModelRealtimeHint: '실시간 모델 · 말하는 동안 바로 전사.', - bailianModelRecordedFileHint: '녹음 파일 모델 · 말을 마친 뒤 전체 전사(한 클립 ≤ 5분).', + bailianModelSyncFileHint: '동기 녹음 모델 · 말을 마친 뒤 전체 전사(한 클립 ≤ 5분).', + bailianModelAsyncFileHint: '비동기 파일 모델 · 녹음을 업로드한 뒤 전사 작업 완료를 기다립니다.', appIdLabel: 'App ID(애플리케이션 ID)', accessKeyLabel: 'Access Key', resourceIdLabel: 'Resource ID', diff --git a/openless-all/app/src/i18n/zh-CN.ts b/openless-all/app/src/i18n/zh-CN.ts index a55d89fd0..707c91995 100644 --- a/openless-all/app/src/i18n/zh-CN.ts +++ b/openless-all/app/src/i18n/zh-CN.ts @@ -835,7 +835,8 @@ export const zhCN = { bailianVocabularyIdLabel: '热词 Vocabulary ID(可选)', bailianVocabularyIdNote: '如已在百炼创建热词表,可填写 vocab-...;留空则不下发热词。', bailianModelRealtimeHint: '实时模型 · 边说边出字。', - bailianModelRecordedFileHint: '录音文件模型 · 说完后整段转写(单条 ≤ 5 分钟)。', + bailianModelSyncFileHint: '同步录音模型 · 说完后整段转写(单条 ≤ 5 分钟)。', + bailianModelAsyncFileHint: '异步文件模型 · 录音上传后等待转写任务完成。', appIdLabel: 'App ID(应用 ID)', accessKeyLabel: 'Access Key', resourceIdLabel: '资源 ID', diff --git a/openless-all/app/src/i18n/zh-TW.ts b/openless-all/app/src/i18n/zh-TW.ts index 0a58f9e73..2045911b3 100644 --- a/openless-all/app/src/i18n/zh-TW.ts +++ b/openless-all/app/src/i18n/zh-TW.ts @@ -837,7 +837,8 @@ export const zhTW: typeof zhCN = { bailianVocabularyIdLabel: '熱詞 Vocabulary ID(可選)', bailianVocabularyIdNote: '如已在百煉建立熱詞表,可填寫 vocab-...;留空則不下發熱詞。', bailianModelRealtimeHint: '即時模型 · 邊說邊出字。', - bailianModelRecordedFileHint: '錄音檔模型 · 說完後整段轉寫(單條 ≤ 5 分鐘)。', + bailianModelSyncFileHint: '同步錄音模型 · 說完後整段轉寫(單條 ≤ 5 分鐘)。', + bailianModelAsyncFileHint: '非同步檔案模型 · 錄音上傳後等待轉寫工作完成。', appIdLabel: 'App ID(應用 ID)', accessKeyLabel: 'Access Key', resourceIdLabel: '資源 ID', diff --git a/openless-all/app/src/pages/settings/ProvidersSection.tsx b/openless-all/app/src/pages/settings/ProvidersSection.tsx index f0e2520bd..bee429069 100644 --- a/openless-all/app/src/pages/settings/ProvidersSection.tsx +++ b/openless-all/app/src/pages/settings/ProvidersSection.tsx @@ -194,7 +194,7 @@ export function ProvidersSection({ kind = 'all' }: ProvidersSectionProps = {}) { const [llmModelRevision, setLlmModelRevision] = useState(0); const [asrModelRevision, setAsrModelRevision] = useState(0); const os = detectOS(); - const unifiedBailian = committedAsrProvider === 'bailian' && os !== 'android'; + const unifiedBailian = committedAsrProvider === 'bailian'; const [bailianModel, setBailianModel] = useState(''); useEffect(() => { @@ -555,9 +555,14 @@ export function ProvidersSection({ kind = 'all' }: ProvidersSectionProps = {}) { // coordinator::resolve_effective_asr_provider 保持一致):qwen3-asr-flash-realtime* 与 // fun-asr-realtime* 与 fun-asr-flash-8k-realtime* 都是实时模型;当前支持的 // fun-asr-flash-2026-06-15 是「录音文件·说完转写」。 -function bailianModelIsRecordedFile(model: string): boolean { +function bailianModelProtocol(model: string): 'realtime' | 'sync' | 'async' { const m = model.trim(); - return m === 'fun-asr-flash-2026-06-15'; + if (!m || m.includes('realtime')) return 'realtime'; + if (m.startsWith('qwen3-asr-flash-filetrans') + || m === 'fun-asr' + || m.startsWith('fun-asr-') && !m.startsWith('fun-asr-flash') + || m.startsWith('paraformer')) return 'async'; + return 'sync'; } function bailianModelSupportsVocabulary(model: string): boolean { @@ -586,9 +591,12 @@ function BailianProtocolHint({ currentModel }: { currentModel: string }) { setModel(currentModel || 'fun-asr-realtime'); }, [currentModel]); - const hint = bailianModelIsRecordedFile(model) - ? t('settings.providers.bailianModelRecordedFileHint') - : t('settings.providers.bailianModelRealtimeHint'); + const protocol = bailianModelProtocol(model); + const hint = protocol === 'realtime' + ? t('settings.providers.bailianModelRealtimeHint') + : protocol === 'async' + ? t('settings.providers.bailianModelAsyncFileHint') + : t('settings.providers.bailianModelSyncFileHint'); return (