# coding=utf-8 """Realtime Web Audio app for MOSS-TTS Local Transformer v1.5.""" from __future__ import annotations import argparse import json import logging import mimetypes import os import queue import re import sys import threading import time import uuid from contextlib import asynccontextmanager from pathlib import Path from typing import Any from urllib.parse import unquote import orjson import torch import uvicorn from fastapi import FastAPI, File, Form, HTTPException, UploadFile from fastapi.responses import FileResponse, HTMLResponse, JSONResponse, StreamingResponse REPO_ROOT = Path(__file__).resolve().parent.parent STREAMING_MODULE_DIR = REPO_ROOT / "moss_tts_local_v1.5" if str(STREAMING_MODULE_DIR) not in sys.path: sys.path.insert(0, str(STREAMING_MODULE_DIR)) from streaming import ( DEFAULT_CODEC_DIR, DEFAULT_MODEL_DIR, DEFAULT_OUTPUT_DIR, StreamingRequest, StreamingRuntime, load_runtime, synthesize_stream, ) torch.backends.cuda.enable_cudnn_sdp(False) torch.backends.cuda.enable_flash_sdp(True) torch.backends.cuda.enable_mem_efficient_sdp(True) torch.backends.cuda.enable_math_sdp(True) DEFAULT_UPLOAD_DIR = Path("outputs/moss_tts_local_v1_5_uploads") DEFAULT_MAX_NEW_TOKENS = 7500 MODE_CLONE = "Clone" MODE_CONTINUE = "Continuation" MODE_CONTINUE_CLONE = "Continuation + Clone" CONTINUATION_NOTICE = ( "Continuation mode is active. Fill Reference Audio Transcript with the transcript of the reference audio." ) ZH_TOKENS_PER_CHAR = 3.098411951313033 EN_TOKENS_PER_CHAR = 0.8673376262755219 REFERENCE_AUDIO_DIR = REPO_ROOT / "assets" / "audio" EXAMPLE_TEXTS_JSONL_PATH = REPO_ROOT / "assets" / "text" / "moss_tts_example_texts.jsonl" LANGUAGE_TAG_AUTO = "Auto (omit)" LANGUAGE_TAG_CHOICES = [ LANGUAGE_TAG_AUTO, "Chinese", "Cantonese", "English", "Arabic", "Czech", "Danish", "Dutch", "Finnish", "French", "German", "Greek", "Hebrew", "Hindi", "Hungarian", "Italian", "Japanese", "Korean", "Macedonian", "Malay", "Persian (Farsi)", "Polish", "Portuguese", "Romanian", "Russian", "Spanish", "Swahili", "Swedish", "Tagalog", "Thai", "Turkish", "Vietnamese", ] def _parse_example_id(example_id: str) -> tuple[str, int] | None: matched = re.fullmatch(r"(zh|en)/(\d+)", (example_id or "").strip()) if matched is None: return None return matched.group(1), int(matched.group(2)) def _resolve_reference_audio_path(language: str, index: int) -> Path | None: stem = f"reference_{language}_{index}" for ext in (".wav", ".mp3", ".m4a"): audio_path = REFERENCE_AUDIO_DIR / f"{stem}{ext}" if audio_path.exists(): return audio_path return None def build_example_rows() -> list[dict[str, str]]: rows: list[dict[str, str]] = [] if not EXAMPLE_TEXTS_JSONL_PATH.exists(): return rows with open(EXAMPLE_TEXTS_JSONL_PATH, "rb") as f: for line in f: if not line.strip(): continue sample = orjson.loads(line) parsed = _parse_example_id(str(sample.get("id", ""))) if parsed is None: continue language, index = parsed audio_path = _resolve_reference_audio_path(language, index) if audio_path is None: continue rows.append( { "role": str(sample.get("role", "")).strip(), "audio_path": str(audio_path), "text": str(sample.get("text", "")).strip(), "language": "Chinese" if language == "zh" else "English", } ) return rows EXAMPLE_ROWS = build_example_rows() def _normalize_language(language_tag: str | None) -> str: value = (language_tag or "").strip() return "" if value == LANGUAGE_TAG_AUTO else value def _safe_int(value: Any, *, default: int, minimum: int, maximum: int | None = None) -> int: try: parsed = int(float(value)) except (TypeError, ValueError): parsed = int(default) parsed = max(int(minimum), parsed) if maximum is not None: parsed = min(int(maximum), parsed) return parsed def _safe_float(value: Any, *, default: float, minimum: float, maximum: float | None = None) -> float: try: parsed = float(value) except (TypeError, ValueError): parsed = float(default) parsed = max(float(minimum), parsed) if maximum is not None: parsed = min(float(maximum), parsed) return parsed def _decode_reference_path(path: str) -> str: decoded = str(path or "") for _ in range(2): next_decoded = unquote(decoded) if next_decoded == decoded: break decoded = next_decoded return decoded def _pcm16le_bytes(waveform: torch.Tensor) -> bytes: if waveform.ndim == 1: waveform = waveform.unsqueeze(0) if waveform.shape[0] == 1: waveform = waveform.repeat(2, 1) elif waveform.shape[0] > 2: waveform = waveform[:2] pcm = waveform.detach().cpu().to(torch.float32).clamp(-1.0, 1.0) pcm = (pcm * 32767.0).round().to(torch.int16) return pcm.transpose(0, 1).contiguous().numpy().tobytes() class RuntimeManager: def __init__( self, *, model_dir: str, codec_dir: str, device: str, tts_device: str, codec_device: str, dtype: str, attn_implementation: str, codec_weight_dtype: str, codec_compute_dtype: str, warmup: bool, ) -> None: self.model_dir = str(model_dir) self.codec_dir = str(codec_dir) self.device = device self.tts_device = tts_device self.codec_device = codec_device self.dtype = dtype self.attn_implementation = attn_implementation self.codec_weight_dtype = codec_weight_dtype self.codec_compute_dtype = codec_compute_dtype self.warmup = bool(warmup) self._lock = threading.Lock() self._status_lock = threading.Lock() self._runtime: StreamingRuntime | None = None self._loader_thread: threading.Thread | None = None self._state = "not_loaded" self._error: str | None = None self._load_started_at: float | None = None self._ready_at: float | None = None def _set_status(self, *, state: str, error: str | None = None) -> None: with self._status_lock: self._state = state self._error = error if state == "loading": self._load_started_at = time.time() self._ready_at = None elif state == "ready": self._ready_at = time.time() def status(self) -> dict[str, Any]: with self._status_lock: state = self._state error = self._error load_started_at = self._load_started_at ready_at = self._ready_at elapsed = None if load_started_at is not None: elapsed = max(0.0, (ready_at or time.time()) - load_started_at) return { "state": state, "error": error, "load_started_at": load_started_at, "ready_at": ready_at, "load_elapsed_seconds": elapsed, "model_dir": self.model_dir, "codec_dir": self.codec_dir, "device": self.device, "tts_device": self.tts_device, "codec_device": self.codec_device, "dtype": self.dtype, "requested_attn_implementation": self.attn_implementation, "attn_implementation": ( self.attn_implementation if self._runtime is None else self._runtime.attn_implementation ), "codec_weight_dtype": ( self.codec_weight_dtype if self._runtime is None else self._runtime.codec_weight_dtype ), "codec_compute_dtype": self.codec_compute_dtype, "n_vq": None if self._runtime is None else int(self._runtime.n_vq), "sample_rate": None if self._runtime is None else int(self._runtime.sample_rate), } def preload_async(self) -> None: with self._status_lock: if self._runtime is not None or self._state == "loading": return if self._loader_thread is not None and self._loader_thread.is_alive(): return def _load() -> None: try: self.get() except Exception: logging.exception("failed to preload MOSS-TTS Local v1.5 streaming runtime") self._loader_thread = threading.Thread(target=_load, name="moss-tts-local-v1.5-runtime-loader", daemon=True) self._loader_thread.start() def get(self) -> StreamingRuntime: with self._lock: if self._runtime is None: self._set_status(state="loading") try: self._runtime = load_runtime( model_dir=self.model_dir, codec_dir=self.codec_dir, device=self.device, tts_device=self.tts_device, codec_device=self.codec_device, dtype=self.dtype, attn_implementation=self.attn_implementation, codec_weight_dtype=self.codec_weight_dtype, codec_compute_dtype=self.codec_compute_dtype, warmup=self.warmup, ) except Exception as exc: self._set_status(state="error", error=str(exc)) raise self._set_status(state="ready") return self._runtime class StreamingJob: def __init__(self, job_id: str) -> None: self.job_id = job_id self.audio_queue: queue.Queue[bytes | None] = queue.Queue(maxsize=64) self.status_lock = threading.Lock() self.status: dict[str, Any] = { "job_id": job_id, "state": "queued", "created_at": time.time(), "started_at": None, "first_audio_at": None, "sample_rate": 48000, "channels": 2, "generated_frames": 0, "max_new_tokens": DEFAULT_MAX_NEW_TOKENS, "generated_audio_seconds": 0.0, "emitted_audio_seconds": 0.0, "lead_seconds": 0.0, "error": None, "closed": False, } self.result: dict[str, Any] | None = None self.thread: threading.Thread | None = None self.is_closed = False def update(self, **kwargs: Any) -> None: with self.status_lock: self.status.update(kwargs) def snapshot(self) -> dict[str, Any]: with self.status_lock: return dict(self.status) class StreamingJobManager: def __init__(self) -> None: self._jobs: dict[str, StreamingJob] = {} self._lock = threading.Lock() def create(self) -> StreamingJob: job = StreamingJob(uuid.uuid4().hex) with self._lock: self._jobs[job.job_id] = job return job def get(self, job_id: str) -> StreamingJob: with self._lock: job = self._jobs.get(job_id) if job is None: raise HTTPException(status_code=404, detail=f"stream job not found: {job_id}") return job def close(self, job_id: str) -> StreamingJob: job = self.get(job_id) with job.status_lock: job.is_closed = True job.status["closed"] = True if job.status.get("state") not in {"finished", "error"}: job.status["state"] = "closed" try: job.audio_queue.put_nowait(None) except queue.Full: pass return job def create_app( *, model_dir: str, codec_dir: str, output_dir: str | Path = DEFAULT_OUTPUT_DIR, upload_dir: str | Path = DEFAULT_UPLOAD_DIR, device: str = "cuda", tts_device: str = "cuda:0", codec_device: str = "cuda:0", dtype: str = "bfloat16", attn_implementation: str = "flash_attention_2", codec_weight_dtype: str = "fp32", codec_compute_dtype: str = "bf16", warmup: bool = True, preload: bool = True, ) -> FastAPI: runtime_manager = RuntimeManager( model_dir=str(model_dir), codec_dir=str(codec_dir), device=device, tts_device=tts_device, codec_device=codec_device, dtype=dtype, attn_implementation=attn_implementation, codec_weight_dtype=codec_weight_dtype, codec_compute_dtype=codec_compute_dtype, warmup=warmup, ) jobs = StreamingJobManager() output_dir = Path(output_dir) upload_dir = Path(upload_dir) output_dir.mkdir(parents=True, exist_ok=True) upload_dir.mkdir(parents=True, exist_ok=True) @asynccontextmanager async def lifespan(_: FastAPI): if preload: runtime_manager.get() yield app = FastAPI(title="MOSS-TTS Local v1.5 Realtime Streaming", lifespan=lifespan) @app.get("/", response_class=HTMLResponse) async def index() -> HTMLResponse: defaults = { "text": "欢迎关注模思智能、上海创智学院与复旦大学自然语言处理实验室。", "max_new_tokens": DEFAULT_MAX_NEW_TOKENS, "seed": 1234, } return HTMLResponse( _html( defaults=defaults, examples=EXAMPLE_ROWS, languages=LANGUAGE_TAG_CHOICES, runtime=runtime_manager.status(), ) ) def _put_stream_audio(job: StreamingJob, pcm_bytes: bytes) -> None: while True: with job.status_lock: if job.is_closed: return try: job.audio_queue.put(pcm_bytes, timeout=0.1) return except queue.Full: continue def _run_job(job: StreamingJob, request: StreamingRequest, mode_name: str, streaming_generation: bool) -> None: try: job.update( state="loading_runtime", started_at=time.time(), max_new_tokens=int(request.max_new_frames), mode=mode_name, streaming_generation=streaming_generation, ) runtime = runtime_manager.get() job.update(state="running", sample_rate=runtime.sample_rate, channels=2, n_vq=runtime.n_vq) for event in synthesize_stream(runtime, request, output_dir=output_dir): with job.status_lock: if job.is_closed: break if event.type == "metadata": job.update(**event.data) elif event.type == "progress": job.update(**event.data) elif event.type == "audio": waveform = event.data["waveform"] channels = 1 if waveform.ndim == 1 else int(min(2, waveform.shape[0])) with job.status_lock: if job.status.get("first_audio_at") is None: job.status["first_audio_at"] = time.time() if streaming_generation: _put_stream_audio(job, _pcm16le_bytes(waveform)) job.update( generated_frames=event.data.get("generated_frames", job.snapshot().get("generated_frames", 0)), emitted_audio_seconds=event.data.get("emitted_audio_seconds", 0.0), generated_audio_seconds=event.data.get("generated_audio_seconds", 0.0), sample_rate=event.data.get("sample_rate", runtime.sample_rate), channels=channels, lead_seconds=event.data.get("lead_seconds", 0.0), generation_lead_seconds=event.data.get("generation_lead_seconds", 0.0), playback_lead_seconds=event.data.get("playback_lead_seconds"), generation_realtime_factor=event.data.get("generation_realtime_factor", 0.0), post_first_generation_realtime_factor=event.data.get( "post_first_generation_realtime_factor" ), first_audio_latency_seconds=event.data.get("first_audio_latency_seconds"), decode_chunks_submitted=event.data.get("decode_chunks_submitted", 0), decode_queue_depth=event.data.get("decode_queue_depth", 0), pending_decode_frames=event.data.get("pending_decode_frames", 0), chunk_frames=event.data.get("chunk_frames", 0), ) elif event.type == "result": metadata = dict(event.data["metadata"]) job.result = { "audio_path": event.data["audio_path"], "tokens_path": event.data["tokens_path"], "metadata_path": event.data["metadata_path"], "metadata": metadata, } job.update( state="finished", generated_frames=metadata.get("generated_frames", 0), emitted_audio_seconds=metadata.get("duration_seconds", 0.0), audio_path=event.data["audio_path"], ) try: job.audio_queue.put_nowait(None) except queue.Full: pass except Exception as exc: # noqa: BLE001 job.update(state="error", error=str(exc)) try: job.audio_queue.put_nowait(None) except queue.Full: pass @app.post("/api/generate-stream/start") async def generate_stream_start( mode: str = Form("voice_clone"), language: str = Form(""), text: str = Form(...), prompt_text: str = Form(""), max_new_tokens: int = Form(DEFAULT_MAX_NEW_TOKENS), codec_chunk_frames: int = Form(8), seed: int = Form(1234), tokens_control: int = Form(0), tokens: int = Form(0), temperature: float = Form(1.7), top_p: float = Form(0.8), top_k: int = Form(25), repetition_penalty: float = Form(1.0), streaming_generation: int = Form(1), example_audio_path: str = Form(""), prompt_audio: UploadFile | None = File(None), ) -> JSONResponse: text = (text or "").strip() if not text: raise HTTPException(status_code=400, detail="text must not be empty") mode = (mode or "").strip().lower() if mode not in {"voice_clone", "continuation", "continuation_clone"}: mode = "voice_clone" mode_name = { "voice_clone": MODE_CLONE, "continuation": MODE_CONTINUE, "continuation_clone": MODE_CONTINUE_CLONE, }[mode] prompt_audio_path = "" if prompt_audio is not None and prompt_audio.filename: suffix = Path(prompt_audio.filename).suffix or ".wav" prompt_path = upload_dir / f"{uuid.uuid4().hex}{suffix}" prompt_path.write_bytes(await prompt_audio.read()) prompt_audio_path = str(prompt_path) elif example_audio_path: candidate = Path(_decode_reference_path(example_audio_path)) if candidate.exists() and REFERENCE_AUDIO_DIR in candidate.resolve().parents: prompt_audio_path = str(candidate) if not prompt_audio_path: mode_name = "Direct Generation" if mode in {"continuation", "continuation_clone"} and prompt_audio_path: if not text: raise HTTPException(status_code=400, detail="continuation mode requires text") if not (prompt_text or "").strip(): raise HTTPException( status_code=400, detail="continuation mode requires reference audio transcript", ) max_new_tokens = _safe_int( max_new_tokens, default=DEFAULT_MAX_NEW_TOKENS, minimum=1, maximum=DEFAULT_MAX_NEW_TOKENS, ) codec_chunk_frames = _safe_int(codec_chunk_frames, default=8, minimum=0, maximum=32) streaming_generation_enabled = bool(_safe_int(streaming_generation, default=1, minimum=0, maximum=1)) request = StreamingRequest( text=text, mode="continuation" if not prompt_audio_path or mode in {"continuation", "continuation_clone"} else "voice_clone", prompt_text=prompt_text or "", prompt_audio_path=prompt_audio_path or None, language=_normalize_language(language), tokens_control=bool(int(tokens_control)), tokens=_safe_int(tokens, default=0, minimum=0), max_new_frames=max_new_tokens, do_sample=True, temperature=_safe_float(temperature, default=1.7, minimum=0.1, maximum=3.0), top_p=_safe_float(top_p, default=0.8, minimum=0.1, maximum=1.0), top_k=_safe_int(top_k, default=25, minimum=1, maximum=200), repetition_penalty=_safe_float(repetition_penalty, default=1.0, minimum=0.8, maximum=2.0), seed=None if int(seed) < 0 else int(seed), codec_chunk_frames=codec_chunk_frames, ) job = jobs.create() thread = threading.Thread(target=_run_job, args=(job, request, mode_name, streaming_generation_enabled), daemon=True) job.thread = thread thread.start() return JSONResponse( { "job_id": job.job_id, "audio_url": f"/api/generate-stream/{job.job_id}/audio", "status_url": f"/api/generate-stream/{job.job_id}/status", "result_url": f"/api/generate-stream/{job.job_id}/result", "sample_rate": runtime_manager.status().get("sample_rate") or 48000, "channels": 2, } ) @app.get("/api/reference-audio") async def reference_audio(path: str) -> FileResponse: try: reference_root = REFERENCE_AUDIO_DIR.resolve(strict=True) candidate = Path(_decode_reference_path(path)).resolve(strict=True) except FileNotFoundError as exc: raise HTTPException(status_code=404, detail="reference audio not found") from exc if candidate != reference_root and reference_root not in candidate.parents: raise HTTPException(status_code=403, detail="reference audio path is not allowed") if not candidate.is_file(): raise HTTPException(status_code=404, detail="reference audio not found") media_type = mimetypes.guess_type(str(candidate))[0] or "application/octet-stream" return FileResponse(str(candidate), media_type=media_type, filename=candidate.name) @app.get("/api/generate-stream/{job_id}/audio") async def generate_stream_audio(job_id: str) -> StreamingResponse: job = jobs.get(job_id) def iterator(): while True: item = job.audio_queue.get() if item is None: break yield item snapshot = job.snapshot() return StreamingResponse( iterator(), media_type="application/octet-stream", headers={ "X-Audio-Codec": "pcm_s16le", "X-Audio-Sample-Rate": str(snapshot.get("sample_rate", 48000)), "X-Audio-Channels": str(snapshot.get("channels", 2)), "X-Stream-Id": job_id, }, ) @app.get("/api/generate-stream/{job_id}/status") async def generate_stream_status(job_id: str) -> JSONResponse: return JSONResponse(jobs.get(job_id).snapshot()) @app.get("/api/generate-stream/{job_id}/result") async def generate_stream_result(job_id: str) -> JSONResponse: job = jobs.get(job_id) if job.result is None: raise HTTPException(status_code=404, detail="result is not ready") return JSONResponse(job.result) @app.get("/api/generate-stream/{job_id}/result-audio") async def generate_stream_result_audio(job_id: str) -> FileResponse: job = jobs.get(job_id) if job.result is None: raise HTTPException(status_code=404, detail="result is not ready") return FileResponse(job.result["audio_path"], media_type="audio/wav", filename="generated.wav") @app.post("/api/generate-stream/{job_id}/close") async def generate_stream_close(job_id: str) -> JSONResponse: jobs.close(job_id) return JSONResponse({"ok": True}) @app.get("/api/runtime") async def runtime_info() -> JSONResponse: return JSONResponse( { "model_dir": str(model_dir), "codec_dir": str(codec_dir), "output_dir": str(output_dir), "upload_dir": str(upload_dir), "device": device, "tts_device": tts_device, "codec_device": codec_device, "dtype": dtype, "attn_implementation": attn_implementation, "codec_weight_dtype": codec_weight_dtype, "codec_compute_dtype": codec_compute_dtype, "runtime": runtime_manager.status(), } ) @app.get("/api/health") async def health() -> JSONResponse: return JSONResponse(runtime_manager.status()) return app def _html(*, defaults: dict[str, Any], examples: list[dict[str, str]], languages: list[str], runtime: dict[str, Any]) -> str: replacements = { "__DEFAULT_TEXT__": json.dumps(defaults["text"], ensure_ascii=False), "__DEFAULT_MAX_NEW_TOKENS__": str(defaults["max_new_tokens"]), "__DEFAULT_SEED__": str(defaults["seed"]), "__EXAMPLES_JSON__": json.dumps(examples, ensure_ascii=False), "__LANGUAGES_JSON__": json.dumps(languages, ensure_ascii=False), "__RUNTIME_JSON__": json.dumps(runtime, ensure_ascii=False), } html = INDEX_HTML for key, value in replacements.items(): html = html.replace(key, value) return html INDEX_HTML = r"""