"""
Audio8: a compact realism-tuned cloner, offline, from a reference recording.
Module summary
--------------
Edge0's ``Audio8-TTS-Realistic`` (0.6B parameters, Apache-2.0, 23 languages
including French; arXiv 2610.14776) was trained SFT → GRPO on WER and speaker
similarity → GRPO on a *realism critic* that compares each output with a human
saying the same sentence in the same voice. It is the 2026-11 candidate for the
owner's offline case, where the brief is a voice that sounds alive, not a fast
one.
It renders by **cloning**: a reference recording *plus its exact transcript*
conditions every call. Without a reference the model picks a speaker of its
own, and measured on 2026-11-04 that speaker is male (median F0 139 Hz in
French, 163 Hz in English). So this backend always passes a reference, and by
default the bundled female one (``assets/ref-fr-female.wav`` + ``.txt``,
CC BY 4.0): with it the French output measured F0 274 Hz and a TitaNet
similarity of 0.72 to the reference.
One runtime, said plainly
-------------------------
Only the **MLX** path is wired: ``mlx-audio`` (≥ 0.6.4 — 0.6.1 fails to
dequantise the tied 4-bit embedding) running the community 4-bit conversion
``nativ-community/Audio8-TTS-Realistic-0.6b-4bit`` (~1.7 GB, cached once,
offline after). Upstream's PyTorch path needs ``transformers<5`` with
``trust_remote_code`` and has not been exercised here, so off a Mac this
backend reports itself unavailable instead of shipping untested code.
The codec decodes at 44.1 kHz; every other backend lays audio out at 24 kHz, so
the output is resampled — otherwise a narration mixing engines would carry
wrong cue times after the first segment.
There is **no phoneme input** and no streaming: pronunciation-lexicon entries
reach it through the respelling tier applied by
:class:`~speaker_helper.speaker.Speaker` (see :mod:`speaker_helper.lexicon`),
and each call returns one complete waveform.
Usage example
-------------
>>> from speaker_helper import Speaker, Settings
>>> # spk = Speaker(Settings.from_mapping({"backend": "audio8", "language": "fr"}))
>>> # await spk.say("Le rendez-vous est fixé au 14 novembre.")
Author
------
Warith HARCHAOUI — https://linkedin.com/in/warith-harchaoui
"""
from __future__ import annotations
import asyncio
import importlib.metadata
import importlib.util
import time
from pathlib import Path
from typing import Any
import os_helper as osh
from speaker_helper.config import Settings
from speaker_helper.types import AudioResult, Voice, VoiceList, VoiceSample
#: The rate this wrapper lays audio out at — the same as kokoro, chatterbox and
#: kyutai, so a narration can mix engines.
SAMPLE_RATE_HZ: int = 24_000
#: The codec's native rate; :func:`resample` bridges the two.
NATIVE_RATE_HZ: int = 44_100
#: The default model repository: the community MLX 4-bit conversion.
DEFAULT_REPO: str = "nativ-community/Audio8-TTS-Realistic-0.6b-4bit"
#: The oldest ``mlx-audio`` that loads the 4-bit checkpoint (0.6.1 raised a
#: matmul shape error on the tied embedding, measured 2026-11-04).
MIN_MLX_AUDIO: tuple[int, int, int] = (0, 6, 4)
#: Generation budget in codec frames. The model shares 2,048 positions between
#: text, reference and audio, and the runtime clamps to what is left; upstream's
#: 512 default stops at ~24 s (21.5 frames/s), shorter than one chunk can be.
DEFAULT_MAX_TOKENS: int = 2048
#: Fixed sampling seed, so a synthesis is reproducible run to run — the
#: evaluation layer compares takes, and a fresh draw each time is noise there.
DEFAULT_SEED: int = 0
def _version_tuple(raw: str) -> tuple[int, ...]:
"""Parse the leading numeric part of a version string.
Examples
--------
>>> _version_tuple("0.6.4")
(0, 6, 4)
>>> _version_tuple("0.7.0rc1")
(0, 7, 0)
"""
parts: list[int] = []
for piece in raw.split(".")[:3]:
digits = "".join(ch for ch in piece if ch.isdigit()) or "0"
parts.append(int(digits))
return tuple(parts)
[docs]
def install_problem() -> str:
"""Why this engine cannot run here, or ``""`` when it can.
Returns
-------
str
A one-line reason (not a Mac, ``mlx-audio`` missing or too old), empty
when the MLX runtime is usable.
"""
if not osh.macos():
return "only the MLX (Apple Silicon) runtime is wired for this backend"
if importlib.util.find_spec("mlx_audio") is None:
return "mlx-audio is not installed"
try:
found = importlib.metadata.version("mlx-audio")
except importlib.metadata.PackageNotFoundError:
return "mlx-audio is not installed"
if _version_tuple(found) < MIN_MLX_AUDIO:
wanted = ".".join(str(n) for n in MIN_MLX_AUDIO)
return f"mlx-audio {found} is too old to load Audio8 (needs >= {wanted})"
return ""
[docs]
def check_install() -> int:
"""Return 0 when the MLX runtime for Audio8 is usable."""
return 0 if not install_problem() else 1
[docs]
def resample(samples: Any, source_hz: int, target_hz: int) -> Any:
"""Band-limited resampling of a whole mono signal through the FFT.
Offline synthesis hands over the complete waveform, so the exact spectral
method costs nothing a streaming filter would save, and it keeps this
backend free of a SciPy dependency.
Parameters
----------
samples : numpy.ndarray
1-D float samples at ``source_hz``.
source_hz, target_hz : int
The two rates.
Returns
-------
numpy.ndarray
1-D float32 samples at ``target_hz``, the same duration.
Examples
--------
>>> import numpy as np
>>> resample(np.zeros(44_100), 44_100, 24_000).shape
(24000,)
"""
import numpy as np
data = np.asarray(samples, dtype=np.float64).reshape(-1)
if source_hz == target_hz or data.size == 0:
return data.astype(np.float32)
n_out = int(round(data.size * target_hz / source_hz))
spectrum = np.fft.rfft(data)
# Keep the bins both rates share (truncating = an ideal low-pass when
# downsampling); irfft pads with zeros when upsampling.
keep = min(spectrum.size, n_out // 2 + 1)
out = np.fft.irfft(spectrum[:keep], n=n_out) * (n_out / data.size)
return out.astype(np.float32)
[docs]
def reference_transcript(audio: Path, *, language: str) -> str:
"""The transcript that must accompany a reference recording.
Audio8 aligns the reference audio to its text, so a wrong or missing
transcript degrades the clone instead of failing. Lookup order: the bundled
reference's own transcript, a ``<stem>.txt`` beside the file, then
:func:`~speaker_helper.transcription.ensure_transcript` (vocal-helper,
cached as ``<file>.txt``).
Parameters
----------
audio : Path
The reference recording.
language : str
ISO-639-1 hint for the ASR fallback.
Returns
-------
str
The transcript.
Raises
------
RuntimeError
When no transcript is on disk and vocal-helper is not installed.
"""
from speaker_helper.cloning import DEFAULT_CLONE_AUDIO, default_reference_text
if audio == DEFAULT_CLONE_AUDIO:
text = default_reference_text()
if text:
return text
sibling = audio.with_suffix(".txt")
if sibling.is_file():
text = sibling.read_text(encoding="utf-8").strip()
if text:
return text
from speaker_helper.transcription import ensure_transcript
try:
return ensure_transcript(audio, language=language)
except ImportError as exc:
raise RuntimeError(
f"audio8 needs the exact transcript of {audio}: write it to "
f"{sibling.name} beside the recording, or install vocal-helper "
"(pip install 'speaker-helper[stt]') to derive it"
) from exc
class _MlxAudio8:
"""The ``mlx-audio`` implementation — the Apple-Silicon (Metal) path.
Parameters
----------
repo : str
Hugging Face repository of an MLX conversion.
"""
def __init__(self, repo: str) -> None:
from mlx_audio.tts.utils import load
self._model = load(repo)
@property
def sample_rate(self) -> int:
"""The codec's decoding rate in Hz."""
return int(self._model.sample_rate)
def generate(self, text: str, ref_audio: str, ref_text: str, **sampling: Any) -> Any:
"""Synthesise ``text`` in the voice of ``ref_audio``.
Returns
-------
tuple of (numpy.ndarray, int)
1-D float32 samples at :attr:`sample_rate`, and the number of codec
frames generated (to detect a budget-truncated take).
"""
import inspect
import mlx.core as mx
import numpy as np
# Not every mlx-audio model takes a `seed`; seeding MLX's global RNG
# makes the take reproducible either way.
seed = sampling.pop("seed", None)
if seed is not None:
mx.random.seed(seed)
if "seed" in inspect.signature(self._model.generate).parameters:
sampling["seed"] = seed
# A model may split long text into several results; one take is all of them.
parts = list(
self._model.generate(text=text, ref_audio=ref_audio, ref_text=ref_text, **sampling)
)
samples = np.concatenate([np.asarray(r.audio, dtype=np.float32).reshape(-1) for r in parts])
return samples, max(int(getattr(r, "token_count", 0) or 0) for r in parts)
[docs]
class Audio8Engine:
"""The Audio8 backend: text in, a cloned (female by default) voice out.
Parameters
----------
settings : Settings
``voice_sample`` (or a per-call ``voice``) names the reference
recording; empty picks the bundled female reference. The ``audio8``
mapping supplies the knobs:
* ``repo`` — MLX model repository (default :data:`DEFAULT_REPO`).
* ``temperature`` / ``top_p`` / ``top_k`` — sampling (upstream
defaults ``0.7`` / ``0.9`` / ``50``).
* ``seed`` — sampling seed (default ``0``; ``null`` draws fresh).
* ``max_tokens`` — codec-frame budget (default ``2048``, clamped by
the runtime to what the context leaves).
"""
#: Backend name, settings section and pip extra.
ENGINE: str = "audio8"
#: Default model repository and its rough download size (for the log line).
DEFAULT_REPO_ID: str = DEFAULT_REPO
DOWNLOAD_SIZE: str = "~1.7 GB"
def __init__(self, settings: Settings) -> None:
self._settings = settings
self._impl: Any = None
self._lock = asyncio.Lock()
knobs = getattr(settings, self.ENGINE, {}) or {}
self._repo: str = str(knobs.get("repo") or self.DEFAULT_REPO_ID)
self._sampling: dict[str, Any] = self._sampling_from(knobs)
# Always a reference: the model's own speaker measured male, and the
# project's contract is a female voice in every language.
from speaker_helper.cloning import DEFAULT_CLONE_AUDIO
raw = getattr(settings, "voice_sample", "") or ""
self._reference: Path = Path(raw) if raw else DEFAULT_CLONE_AUDIO
# Transcripts cached per recording: the ASR fallback is not free.
self._transcripts: dict[Path, str] = {}
def _sampling_from(self, knobs: dict[str, Any]) -> dict[str, Any]:
"""Sampling keywords for ``generate``, upstream defaults spelled out."""
raw_seed = knobs.get("seed", DEFAULT_SEED)
return {
"temperature": float(knobs.get("temperature", 0.7)),
"top_p": float(knobs.get("top_p", 0.9)),
"top_k": int(knobs.get("top_k", 50)),
"max_tokens": int(knobs.get("max_tokens", DEFAULT_MAX_TOKENS)),
"seed": None if raw_seed in (None, "", "none") else int(raw_seed),
}
def _language_kwargs(self, language: str) -> dict[str, Any]:
"""Per-call keywords naming the language; Audio8 reads it from the text."""
return {}
# -- lifecycle ----------------------------------------------------------
async def _loaded(self) -> Any:
"""The implementation, loaded once on first use (see KokoroEngine)."""
if self._impl is not None:
return self._impl
async with self._lock:
if self._impl is None:
self._impl = await asyncio.to_thread(self._load)
return self._impl
def _load(self) -> Any:
"""Build the MLX implementation. Blocking; called in a thread."""
problem = install_problem()
if problem:
raise RuntimeError(
f"{self.ENGINE} cannot run here: {problem}. On a Mac, install with:\n"
f" pip install 'speaker-helper[{self.ENGINE}]'"
)
osh.info(
"%s: loading %s — first time downloads %s", self.ENGINE, self._repo, self.DOWNLOAD_SIZE
)
return _MlxAudio8(self._repo)
[docs]
async def aclose(self) -> None:
"""Drop the implementation so MLX can free the weights."""
self._impl = None
def _transcript_for(self, audio: Path, language: str) -> str:
"""The (cached) transcript of ``audio``; see :func:`reference_transcript`."""
if audio not in self._transcripts:
self._transcripts[audio] = reference_transcript(audio, language=language)
return self._transcripts[audio]
# -- the TTSEngine protocol ---------------------------------------------
[docs]
async def synthesize(
self,
text: str,
*,
language: str | None = None,
voice: str | None = None,
) -> AudioResult:
"""Synthesise ``text`` into one :class:`AudioResult`.
No ``speed`` parameter on purpose: the model has no rate control, and
resampling would shift pitch with pace.
:meth:`~speaker_helper.speaker.Speaker.narrate` detects the missing
keyword and withdraws the knob instead of failing.
Parameters
----------
text : str
What to say (must be non-empty).
language : str or None
Recorded on the result and used as the ASR hint for a reference
transcript; the model itself reads the language from the text.
voice : str or None
Path to a reference recording for this call; empty keeps the
configured (or bundled female) reference.
Raises
------
ValueError
If ``text`` is empty or whitespace-only.
"""
import numpy as np
if not text or not text.strip():
raise ValueError("cannot synthesise empty text")
lang = ((language or self._settings.language or "fr").strip().lower()).split("-")[0]
reference = Path(voice) if voice else self._reference
if not reference.is_file():
raise FileNotFoundError(f"{self.ENGINE}: reference recording not found: {reference}")
ref_text = await asyncio.to_thread(self._transcript_for, reference, lang)
impl = await self._loaded()
started = time.perf_counter()
samples, frames = await asyncio.to_thread(
impl.generate,
text,
str(reference),
ref_text,
**self._sampling,
**self._language_kwargs(lang),
)
compute_s = time.perf_counter() - started
budget = self._sampling.get("max_tokens")
if budget and frames >= budget:
# The take ran into its budget instead of ending on the model's own
# stop token: the tail of the text is missing. Said, not hidden.
osh.warning(
"%s: generation hit its %d-frame budget; the end of %r may be cut",
self.ENGINE,
budget,
text[:60],
)
samples = resample(samples, impl.sample_rate, SAMPLE_RATE_HZ)
pcm = (np.clip(samples, -1.0, 1.0) * 32_767).astype(np.int16)
from speaker_helper.backends.kokoro import pcm_to_wav_bytes
return AudioResult(
wav_bytes=pcm_to_wav_bytes(pcm, SAMPLE_RATE_HZ),
sample_rate=SAMPLE_RATE_HZ,
duration_s=len(pcm) / float(SAMPLE_RATE_HZ),
compute_s=compute_s,
text=text,
voice_id=reference.stem,
language=lang,
)
[docs]
async def list_voices(self, engine: str | None = None) -> VoiceList:
"""One entry: the reference this engine will actually clone.
Same contract as the chatterbox backend: the bundled reference is named
with its provenance and its gender; somebody else's recording is named
by file, and its gender left to them.
"""
from speaker_helper.cloning import DEFAULT_CLONE_AUDIO, DEFAULT_CLONE_NAME
language = self._settings.language
if self._reference == DEFAULT_CLONE_AUDIO:
voice = Voice(
voice_id=DEFAULT_CLONE_NAME,
name="Bundled female reference — CML-TTS (CC BY 4.0)",
language=language,
gender="female",
engine=self.ENGINE,
)
else:
voice = Voice(
voice_id=self._reference.stem,
name=f"Cloned from {self._reference.name}",
language=language,
gender="",
engine=self.ENGINE,
)
return VoiceList(engine=self.ENGINE, voices=[voice])
[docs]
async def health(self) -> dict:
"""Whether this engine could synthesise, without loading the model."""
problem = install_problem()
if problem:
return {
"status": "unavailable",
"engine": self.ENGINE,
"reason": problem,
"fix": f"pip install 'speaker-helper[{self.ENGINE}]' (macOS, Apple Silicon)",
}
return {
"status": "ok",
"engine": self.ENGINE,
"loaded": self._impl is not None,
"repo": self._repo,
"implementation": "mlx",
"reference": str(self._reference),
"sample_rate": SAMPLE_RATE_HZ,
}
[docs]
async def clone_voice(
self,
name: str,
samples: list[VoiceSample],
*,
language: str | None = None,
) -> str:
"""Adopt the first sample, with its transcript, as the reference.
As with chatterbox there is no enrolment: the model conditions on the
recording at generation time, so the honest return value is the path.
Audio8 also needs what the recording says; an empty ``reference_text``
falls back to :func:`reference_transcript`.
"""
if not samples:
raise ValueError("cloning needs at least one reference recording")
first = samples[0]
if isinstance(first.audio, str):
reference = Path(first.audio)
else:
# The model takes a path, not bytes; kept on disk (not a temporary
# a later generate() would find deleted), as in the chatterbox one.
import tempfile
handle = tempfile.NamedTemporaryFile(suffix=".wav", delete=False)
handle.write(first.read_bytes())
handle.close()
reference = Path(handle.name)
if first.reference_text.strip():
self._transcripts[reference] = first.reference_text.strip()
else:
lang = (language or self._settings.language or "fr").split("-")[0]
self._transcripts[reference] = await asyncio.to_thread(
reference_transcript, reference, language=lang
)
self._reference = reference
return str(reference)
#: Qwen3-TTS names languages in words; anything unlisted is left to "auto".
QWEN3_LANGUAGE_NAMES: dict[str, str] = {
"fr": "french",
"en": "english",
"de": "german",
"es": "spanish",
"it": "italian",
"pt": "portuguese",
"ru": "russian",
"zh": "chinese",
"ja": "japanese",
"ko": "korean",
}
[docs]
class Qwen3TTSEngine(Audio8Engine):
"""Qwen3-TTS 0.6B (Apache-2.0) through the same MLX cloning contract.
Measured on 2026-11-04 (M2 Max, 8-bit, the bundled female reference, a
two-sentence French paragraph): the closest clone of any engine here
(TitaNet similarity 0.82, F0 233 Hz, transcript exact), sustained
streaming RTF 0.77–0.90 and first audio after ~0.7–0.9 s. That sits on the
router's 0.8 real-time ceiling rather than under it, and its public Elo is
below kokoro's, so it is a candidate for the owner's ear, not a route.
Same knobs as :class:`Audio8Engine` under ``settings.qwen3tts``, except
that sampling defaults are the model's own (only what you set is sent).
The codec decodes at 24 kHz, so nothing is resampled.
Caveat met while measuring: on an exFAT/external Hugging Face cache, macOS
AppleDouble files (``._model.safetensors``) make mlx-audio's loader fail
with "Invalid json header length"; ``dot_clean`` on the cache fixes it.
"""
ENGINE = "qwen3tts"
DEFAULT_REPO_ID = "mlx-community/Qwen3-TTS-12Hz-0.6B-Base-8bit"
DOWNLOAD_SIZE = "~2 GB"
def _sampling_from(self, knobs: dict[str, Any]) -> dict[str, Any]:
"""Only the knobs the caller set; the model's config holds its defaults."""
sampling: dict[str, Any] = {}
for key, cast in (("temperature", float), ("top_p", float), ("top_k", int)):
if knobs.get(key) is not None:
sampling[key] = cast(knobs[key])
if knobs.get("max_tokens") is not None:
sampling["max_tokens"] = int(knobs["max_tokens"])
raw_seed = knobs.get("seed", DEFAULT_SEED)
sampling["seed"] = None if raw_seed in (None, "", "none") else int(raw_seed)
return sampling
def _language_kwargs(self, language: str) -> dict[str, Any]:
"""Qwen3-TTS wants the language by name (``"french"``)."""
return {"lang_code": QWEN3_LANGUAGE_NAMES.get(language, "auto")}