"""
Quality priors and Pareto selection for choosing an engine.
Module summary
--------------
Two ideas make engine choice operational:
1. **A quality axis even without a reference.** When no transcriber is wired in,
fidelity is *estimated* from per-engine priors — order-of-magnitude scores in
``[0, 1]``, recalibrated from real runs. This keeps every evaluation
reporting *both* speed and quality.
2. **Pareto selection.** To pick an engine/voice you want the non-dominated set
on the quality↔RTF plane — high quality *and* fast. :func:`pareto_front`
extracts it.
Together these turn a batch of :class:`~speaker_helper.eval.runner.EvalReport`
(one per engine) into an actionable "which engine should I ship?" answer.
Usage example
-------------
>>> from speaker_helper.eval.priors import engine_quality_prior
>>> engine_quality_prior("kokoro")
0.75
Author
------
Warith HARCHAOUI — https://linkedin.com/in/warith-harchaoui
"""
from __future__ import annotations
from collections.abc import Sequence
from typing import Protocol
# TTS quality priors in [0, 1]: order-of-magnitude estimates keyed on the
# engine — recalibrate from measured round-trips (WER/chrF) as real data
# accumulates. Unknown engines get 0.80.
TTS_QUALITY_PRIORS: dict[str, float] = {
"kokoro": 0.75,
"chatterbox_turbo": 0.82,
"chatterbox": 0.90,
"kyutai": 0.88,
"audio8": 0.86,
"qwen3_tts": 0.88,
"qwen_customvoice": 0.90,
"f5": 0.86,
"xtts": 0.70,
"mock": 1.00, # deterministic reference signal — nominal "perfect".
}
_DEFAULT_PRIOR = 0.80
# Per-language refinements of the priors above, where the 2026-10 survey
# (STUDY.md, `research` branch) found language-specific evidence — a single
# number per engine hides exactly the asymmetry that matters here:
#
# * **kyutai** speaks native French (a French lab, FR WER 3.29% on its card)
# and good-but-not-leading English → fr above its base, en below.
# * **chatterbox** is the most expressive English open model but renders
# French at 6–8% CER through an English-first pipeline → en keeps the base,
# fr drops well under kyutai's.
#
# These are *priors* in the router's vocabulary (quality_source="prior") but
# they are not guesses: the owner RATIFIED this configuration on 2026-10-26 —
# fr → kyutai, en → chatterbox (Turbo), other languages → chatterbox
# (Multilingual) — after récusing automatic MOS predictors as the offline
# criterion (STUDY.md §8 bis). The French half was re-decided on 2026-10-28
# (STUDY.md §8 ter): the owner's choice now lives in
# `router.OFFLINE_RATIFIED_ENGINE`, deliberately apart from these numbers,
# which stay as they are. Human
# pairwise-listening scores (quality_source="measured_human") are the only
# measurement meant to supersede these values.
#
# * **audio8** (added 2026-11-04) is a candidate awaiting the owner's ear, not
# a pick: its own paper reports CV3 WER fr 7.94% / en 2.91%, and on our
# bench its French clone of the bundled reference measured a TitaNet
# similarity of 0.72 (chatterbox: 0.66). Its prior sits deliberately below
# chatterbox's in every language, so adding the backend changes no routing
# decision until a listening says otherwise.
#
# The ``"*"`` key is a per-engine floor for every *other* language: kyutai
# only speaks French and English, and would read Spanish with a
# French/English tongue — so outside fr/en it must lose to chatterbox's
# multilingual model (23 languages), which keeps its base prior there.
TTS_QUALITY_PRIORS_BY_LANG: dict[str, dict[str, float]] = {
"kyutai": {"fr": 0.92, "en": 0.85, "*": 0.30},
"chatterbox": {"fr": 0.78, "en": 0.90},
"audio8": {"fr": 0.77, "en": 0.86},
}
# Public arena Elo — the quality criterion the owner ratified for the
# STREAMING (online) condition: human pairwise votes, aggregated as Elo, from
# the community TTS Arena V2 (https://huggingface.co/spaces/TTS-AGI/TTS-Arena-V2,
# its /api/leaderboard JSON, pulled 2026-11-04; previous pull 2026-10-26 gave
# 1469 / 1466 — the order has not moved). Three honesty notes travel with the
# numbers:
#
# * the arena is English-centric — no public arena ranks open engines in
# French (rechecked 2026-11-04: neither TTS Arena V2 nor Artificial
# Analysis has a French board), so for French this axis is an
# English-quality proxy;
# * an engine absent from every public arena has **no evidence under this
# criterion** (``None``) — a documented data gap, not a zero and not a
# licence to fall back on a flattering prior. The router ranks unranked
# engines below every ranked one in the online condition;
# * a rating the arena itself has struck is not evidence either: Audio8 shows
# 1663 ±51 on 222 votes, ``suspended: true``, reason "vote manipulation"
# (suspended 2026-10-30). Its launch post's "#1 on the Hugging Face TTS
# Arena" is that struck rating.
#
# Two readings of the JSON to know about: the arena flags Chatterbox
# ``open: false`` (its weights are MIT on GitHub and Hugging Face all the same),
# and Kokoro ``active: false`` — out of rotation, so its rating is now frozen.
PUBLIC_ELO: dict[str, float | None] = {
"chatterbox": 1468.0, # ±21, 1,810 votes
"kokoro": 1465.0, # ±24, 1,033 votes, statistically tied with chatterbox
"kyutai": None, # not listed on any public arena (checked 2026-11-04)
"openvoice": None,
"audio8": None, # struck by the arena for vote manipulation (see above)
}
#: As-of date of the PUBLIC_ELO pull — Elo moves with every vote, so the date
#: is part of the claim (same rule as LANDSCAPE.md's licence table).
PUBLIC_ELO_AS_OF: str = "2026-11-04"
[docs]
def elo_to_unit(elo: float) -> float:
"""Map a public arena Elo onto the router's ``[0, 1]`` quality axis.
A transparent, order-preserving linear map — ``(elo - 1000) / 1000``,
clamped. The absolute value means little (Elo scales are arena-specific);
only the ORDER matters to the router's argmax, and a linear map keeps
the printed justification traceable back to the public number.
Parameters
----------
elo : float
An Elo rating from :data:`PUBLIC_ELO`'s arena.
Returns
-------
float
The rating rescaled to ``[0, 1]``.
Examples
--------
>>> elo_to_unit(1465.0)
0.465
>>> elo_to_unit(2200.0)
1.0
"""
return max(0.0, min(1.0, (elo - 1000.0) / 1000.0))
[docs]
def engine_quality_prior(engine: str, language: str | None = None) -> float:
"""Return the prior quality score in ``[0, 1]`` for a TTS ``engine``.
Parameters
----------
engine : str
Engine id (e.g. ``"kokoro"``).
language : str or None
ISO-639 code (regional variants accepted, e.g. ``"fr-fr"``). When a
per-language refinement exists for the engine it wins over the base
prior; ``None`` keeps the language-blind base.
Returns
-------
float
The most specific prior available, or ``0.80`` for an unknown engine.
Examples
--------
>>> engine_quality_prior("kyutai", "fr")
0.92
>>> engine_quality_prior("kyutai")
0.88
"""
# The language key is the bare ISO base: "fr-FR" and "fr" are one prior.
if language:
base = language.strip().lower().split("-")[0]
by_lang = TTS_QUALITY_PRIORS_BY_LANG.get(engine, {})
if base in by_lang:
return by_lang[base]
# The engine's own floor for languages it was never refined on —
# how kyutai is kept away from Spanish (see the table's comment).
if "*" in by_lang:
return by_lang["*"]
return TTS_QUALITY_PRIORS.get(engine, _DEFAULT_PRIOR)
class _ParetoPoint(Protocol):
"""A candidate carrying a quality (maximise) and an RTF (minimise)."""
@property
def quality(self) -> float:
"""Quality score to maximise (higher is better)."""
...
@property
def mean_rtf(self) -> float:
"""Mean real-time factor to minimise (lower is faster)."""
...
[docs]
def pareto_front(points: Sequence[_ParetoPoint]) -> list[_ParetoPoint]:
"""Return the non-dominated points: maximise quality, minimise RTF.
A point A dominates B if A is at least as good on both axes (higher or equal
``quality``, lower or equal ``mean_rtf``) and strictly better on one.
Parameters
----------
points : sequence
Candidates, each exposing ``quality`` and ``mean_rtf`` (e.g.
:class:`~speaker_helper.eval.runner.EvalReport`).
Returns
-------
list
The Pareto-optimal subset, in input order.
"""
front: list[_ParetoPoint] = []
for a in points:
# ``a`` stays on the front unless some other point dominates it.
dominated = False
for b in points:
# A point never dominates itself.
if b is a:
continue
# b dominates a when it is at least as good on BOTH axes (higher or
# equal quality, lower or equal RTF) and strictly better on one —
# i.e. b is a free improvement over a, so a is not Pareto-optimal.
if (
b.quality >= a.quality
and b.mean_rtf <= a.mean_rtf
and (b.quality > a.quality or b.mean_rtf < a.mean_rtf)
):
dominated = True
break
# Undominated points form the quality↔RTF frontier.
if not dominated:
front.append(a)
return front