Complete DDS training workflow and delivery package
This commit is contained in:
andreysk0304
parent
68dd83c7c2
commit
4c4b91064f
229 changed files with 11969 additions and 1024 deletions
|
|
@ -6,14 +6,18 @@ onnxruntime и torch отпускают GIL на время вычислений
|
|||
"""
|
||||
|
||||
import asyncio
|
||||
import io
|
||||
import logging
|
||||
import wave
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from functools import lru_cache
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
import httpx
|
||||
|
||||
from app.config import get_settings
|
||||
from app.dialog.llm import is_loopback_url
|
||||
from app.voice.text import normalize
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
|
@ -51,6 +55,47 @@ class Recognizer:
|
|||
self.transcribe((rng.normal(0, 0.05, 16_000)).astype(np.float32))
|
||||
|
||||
|
||||
class WhisperRecognizer:
|
||||
"""Локальный whisper.cpp: GGML Whisper small, без облака и Python-зависимости STT.
|
||||
|
||||
VAD уже отделил реплику. Сервер держит модель в памяти, а Python отправляет
|
||||
ему только 16-кГц WAV через loopback; адрес из конфигурации проверяется.
|
||||
"""
|
||||
|
||||
def __init__(self, server_url: str, client: httpx.Client | None = None) -> None:
|
||||
if not is_loopback_url(server_url):
|
||||
raise ValueError("Whisper server должен слушать только loopback")
|
||||
self._url = server_url.rstrip("/")
|
||||
self._client = client or httpx.Client(trust_env=False, timeout=30.0)
|
||||
|
||||
def transcribe(self, audio: np.ndarray) -> str:
|
||||
pcm = (np.clip(audio, -1, 1) * 32767).astype("<i2")
|
||||
with io.BytesIO() as buffer:
|
||||
with wave.open(buffer, "wb") as wav:
|
||||
wav.setnchannels(1)
|
||||
wav.setsampwidth(2)
|
||||
wav.setframerate(16_000)
|
||||
wav.writeframes(pcm.tobytes())
|
||||
payload = buffer.getvalue()
|
||||
response = self._client.post(
|
||||
self._url + "/inference",
|
||||
files={"file": ("utterance.wav", payload, "audio/wav")},
|
||||
data={"language": "ru", "response_format": "json", "temperature": "0",
|
||||
"temperature_inc": "0", "no_context": "true"},
|
||||
)
|
||||
response.raise_for_status()
|
||||
result = response.json()
|
||||
if not isinstance(result, dict) or not isinstance(result.get("text"), str):
|
||||
raise ValueError("некорректный ответ локального Whisper server")
|
||||
return result["text"].strip()
|
||||
|
||||
def warmup(self) -> None:
|
||||
# GET / отвечает локальная страница server.cpp. Не запускаем тяжёлый
|
||||
# инференс на тишине: сервер загрузил модель до готовности порта.
|
||||
response = self._client.get(self._url + "/", timeout=2.0)
|
||||
response.raise_for_status()
|
||||
|
||||
|
||||
class Synthesizer:
|
||||
"""Silero v5: около десятой доли длительности фразы на синтез (docs/LATENCY.md)."""
|
||||
|
||||
|
|
@ -80,7 +125,8 @@ class Synthesizer:
|
|||
|
||||
|
||||
class VoiceModels:
|
||||
def __init__(self, recognizer: Recognizer, synthesizer: Synthesizer, vad_path: Path) -> None:
|
||||
def __init__(self, recognizer: Recognizer | WhisperRecognizer, synthesizer: Synthesizer,
|
||||
vad_path: Path) -> None:
|
||||
self.recognizer = recognizer
|
||||
self.synthesizer = synthesizer
|
||||
self.vad_path = vad_path
|
||||
|
|
@ -100,8 +146,14 @@ def get_voice_models() -> VoiceModels | None:
|
|||
if not settings.voice_enabled:
|
||||
return None
|
||||
models = ROOT / settings.models_dir
|
||||
stt_path = (ROOT / settings.whisper_model_path / "ggml-small-q5_1.bin"
|
||||
if settings.stt_model == "whisper-small"
|
||||
else models / "gigaam-v3-onnx" / "v3_rnnt_encoder.int8.onnx")
|
||||
if settings.stt_model not in {"whisper-small", "gigaam-v3-rnnt"}:
|
||||
log.error("неизвестный STT_MODEL: %s", settings.stt_model)
|
||||
return None
|
||||
required = [
|
||||
models / "gigaam-v3-onnx" / "v3_rnnt_encoder.int8.onnx",
|
||||
stt_path,
|
||||
models / "silero-vad" / "silero_vad.onnx",
|
||||
models / "silero-tts" / "v5_ru.pt",
|
||||
]
|
||||
|
|
@ -110,12 +162,13 @@ def get_voice_models() -> VoiceModels | None:
|
|||
log.warning("голосовой контур выключен, нет моделей: %s — make models", ", ".join(missing))
|
||||
return None
|
||||
try:
|
||||
recognizer = Recognizer(models / "gigaam-v3-onnx")
|
||||
recognizer = (WhisperRecognizer(settings.whisper_server_url) if settings.stt_model == "whisper-small"
|
||||
else Recognizer(models / "gigaam-v3-onnx"))
|
||||
synthesizer = Synthesizer(models / "silero-tts" / "v5_ru.pt")
|
||||
# Прогрев здесь, на старте стенда, а не на первой реплике курсанта.
|
||||
recognizer.warmup()
|
||||
synthesizer.warmup()
|
||||
return VoiceModels(recognizer, synthesizer, models / "silero-vad" / "silero_vad.onnx")
|
||||
except ImportError as exc:
|
||||
log.warning("голосовой контур выключен: не установлены зависимости (%s) — uv sync --extra voice", exc)
|
||||
except (ImportError, httpx.HTTPError, ValueError) as exc:
|
||||
log.warning("голосовой контур выключен: модели/сервер не готовы (%s)", exc)
|
||||
return None
|
||||
|
|
|
|||
81
backend/app/voice/recording.py
Normal file
81
backend/app/voice/recording.py
Normal file
|
|
@ -0,0 +1,81 @@
|
|||
"""Локальная WAV-запись обеих сторон учебного голосового вызова."""
|
||||
|
||||
import os
|
||||
import time
|
||||
import wave
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from uuid import UUID
|
||||
|
||||
import numpy as np
|
||||
|
||||
from app.config import get_settings
|
||||
|
||||
TARGET_RATE = 16_000
|
||||
|
||||
|
||||
def recording_path(session_id: UUID) -> Path:
|
||||
return Path(get_settings().recordings_dir).resolve() / f"{session_id}.wav"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class _Segment:
|
||||
offset: int
|
||||
samples: np.ndarray
|
||||
|
||||
|
||||
class CallRecorder:
|
||||
"""Смешивает PCM16 разных частот на монотаймлайн 16 кГц.
|
||||
|
||||
Вход курсанта приходит по 16 кГц, TTS звонящего — по 24 кГц. Метка
|
||||
monotonic сохраняет паузы и взаимное расположение реплик; системные часы
|
||||
и изменение времени на хосте на запись не влияют.
|
||||
"""
|
||||
|
||||
def __init__(self, path: Path, *, clock=time.monotonic) -> None:
|
||||
self.path = path
|
||||
self._clock = clock
|
||||
self._started = clock()
|
||||
self._segments: list[_Segment] = []
|
||||
self._finalized = False
|
||||
|
||||
def add_pcm(self, pcm: bytes, *, sample_rate: int) -> None:
|
||||
if self._finalized or not pcm or sample_rate <= 0 or len(pcm) % 2:
|
||||
return
|
||||
source = np.frombuffer(pcm, dtype="<i2").astype(np.int32)
|
||||
if source.size == 0:
|
||||
return
|
||||
if sample_rate != TARGET_RATE:
|
||||
length = max(1, round(source.size * TARGET_RATE / sample_rate))
|
||||
points = np.linspace(0, source.size - 1, length)
|
||||
source = np.rint(np.interp(points, np.arange(source.size), source)).astype(np.int32)
|
||||
offset = max(0, round((self._clock() - self._started) * TARGET_RATE))
|
||||
self._segments.append(_Segment(offset=offset, samples=source))
|
||||
|
||||
def finalize(self) -> Path | None:
|
||||
if self._finalized:
|
||||
return self.path if self.path.is_file() else None
|
||||
self._finalized = True
|
||||
if not self._segments:
|
||||
return None
|
||||
total = max(item.offset + item.samples.size for item in self._segments)
|
||||
mixed = np.zeros(total, dtype=np.int32)
|
||||
for item in self._segments:
|
||||
mixed[item.offset:item.offset + item.samples.size] += item.samples
|
||||
pcm = np.clip(mixed, -32768, 32767).astype("<i2").tobytes()
|
||||
|
||||
self.path.parent.mkdir(parents=True, exist_ok=True)
|
||||
temporary = self.path.with_suffix(".wav.tmp")
|
||||
with wave.open(str(temporary), "wb") as target:
|
||||
target.setnchannels(1)
|
||||
target.setsampwidth(2)
|
||||
target.setframerate(TARGET_RATE)
|
||||
target.writeframes(pcm)
|
||||
os.replace(temporary, self.path)
|
||||
return self.path
|
||||
|
||||
|
||||
def start_recording(session_id: UUID) -> CallRecorder | None:
|
||||
if not get_settings().record_calls:
|
||||
return None
|
||||
return CallRecorder(recording_path(session_id))
|
||||
Loading…
Reference in a new issue