Source code for speaker_helper.text

"""
Text segmentation for low-latency streaming synthesis.

Module summary
--------------
In streaming mode the time to first audio (TTFA) is dominated by how much text
the engine must synthesise before it can emit anything. Splitting a paragraph
into sentence-sized units lets speaker-helper synthesise and emit the first
unit quickly while later units are still being produced. This module holds the
deterministic, dependency-free splitter used for that.

Usage example
-------------
>>> from speaker_helper.text import split_sentences
>>> split_sentences("Bonjour. Comment ça va ? Bien !")
['Bonjour.', 'Comment ça va ?', 'Bien !']

Author
------
Warith HARCHAOUI — https://linkedin.com/in/warith-harchaoui
"""

from __future__ import annotations

import re

# Sentence terminators for the languages speaker-helper targets. We keep the
# terminator attached to the sentence so the engine still hears the prosodic
# cue. French guillemets and typographic punctuation are intentionally NOT
# treated as boundaries — only end-of-sentence marks are.
_SENTENCE_RE = re.compile(r"[^.!?…。!?]+[.!?…。!?]*", re.UNICODE)


[docs] def split_sentences(text: str) -> list[str]: """Split text into sentence-sized, non-empty, stripped units. Parameters ---------- text : str Arbitrary text, possibly multi-sentence and multi-line. Returns ------- list[str] Sentences in order, each stripped of surrounding whitespace, with their terminating punctuation preserved. Text without any terminator returns a single-element list (the whole stripped input); empty or whitespace-only input returns an empty list. Examples -------- >>> split_sentences("Un. Deux. Trois.") ['Un.', 'Deux.', 'Trois.'] >>> split_sentences("no terminator here") ['no terminator here'] >>> split_sentences(" ") [] """ # Whitespace-only input has no sentences to speak. if not text or not text.strip(): return [] # Each regex match is a run of text plus its trailing terminator(s); strip # surrounding whitespace so chunks glue back together cleanly. pieces = [m.group(0).strip() for m in _SENTENCE_RE.finditer(text)] # Drop empties left by back-to-back terminators (e.g. "?!"). sentences = [p for p in pieces if p] # A body with no terminator at all yields nothing above; fall back to the # whole stripped text so the caller always gets something to synthesise. return sentences or [text.strip()]
#: Chunks shorter than this many characters are folded into a neighbour by #: :func:`merge_short_fragments`. #: #: Why a character count, in a module that knows nothing about engines: the #: chatterbox multilingual tokenizer is a *grapheme* tokenizer, so its token #: count tracks characters almost exactly — "Un." is 3 of each, "Oui." is 4 of #: each. That is the boundary measured on 2026-10-28 (chatterbox-tts 0.1.7): #: below 4 tokens, upstream's `AlignmentStreamAnalyzer` has no alignment window #: and dies in `step()` with "max(): Expected reduction dim 1 to have non-zero #: size". "Un." and "A." fail there; "Oui." and "Non." succeed. #: #: The value sits exactly on the measured boundary rather than above it, so the #: merge changes grouping only where synthesis would otherwise break. If a #: fragment ever slips through, the chatterbox backend still refuses it with a #: diagnosis, so a miss is loud rather than silent — which is why no safety #: margin is baked in here. MIN_CHUNK_CHARS = 4
[docs] def merge_short_fragments(chunks: list[str], min_chars: int = MIN_CHUNK_CHARS) -> list[str]: """Fold chunks below ``min_chars`` into a neighbour, leaving words untouched. A three-character chunk is a poor synthesis unit for any engine — there is not enough context for sensible prosody — and for chatterbox's multilingual model it is an outright crash (see :data:`MIN_CHUNK_CHARS`). Merging is the conservative repair: the words spoken, and their order, are **identical**; only the grouping changes, so a short sentence is said in the breath of its neighbour instead of alone. A fragment merges into the chunk *before* it when there is one, so the first chunk stays small and time-to-first-audio is unaffected. A short *first* chunk has no predecessor, so it merges forward into the second. Parameters ---------- chunks : list of str Chunks in reading order. min_chars : int, optional Length below which a chunk is folded into a neighbour. Returns ------- list of str Chunks in the same order, with short fragments absorbed. A lone chunk is always returned as-is: with no neighbour there is nothing to merge into, and inventing words to pad it would change what is spoken. Examples -------- >>> merge_short_fragments(["Bonjour.", "Un.", "Et voilà."]) ['Bonjour. Un.', 'Et voilà.'] >>> merge_short_fragments(["Un.", "Bonjour tout le monde."]) ['Un. Bonjour tout le monde.'] >>> merge_short_fragments(["A.", "B.", "C."]) ['A. B. C.'] >>> merge_short_fragments(["Un."]) ['Un.'] """ if not chunks: return [] merged: list[str] = [] for chunk in chunks: # Too short and something precedes it: fold it into that predecessor. if merged and len(chunk) < min_chars: merged[-1] = f"{merged[-1]} {chunk}" else: merged.append(chunk) # The first chunk has no predecessor to fold into, so it is handled last, # forward into its successor — and only when a successor exists. if len(merged) > 1 and len(merged[0]) < min_chars: merged[1] = f"{merged[0]} {merged[1]}" merged.pop(0) return merged
[docs] def chunk_for_streaming(text: str, first_chunk_sentences: int = 1) -> list[str]: """Group sentences into synthesis chunks, keeping the first one small. Parameters ---------- text : str The text to synthesise. first_chunk_sentences : int, optional How many sentences to place in the first chunk. Keeping this at ``1`` (the default) minimises TTFA; larger values reduce per-chunk overhead at the cost of a slower first emission. Returns ------- list[str] Chunks of text to synthesise in order. The first chunk holds ``first_chunk_sentences`` sentences; every remaining sentence is its own chunk so downstream audio keeps flowing steadily — except that fragments too short to synthesise are folded into a neighbour by :func:`merge_short_fragments`, which changes grouping only, never the words or their order. Examples -------- >>> chunk_for_streaming("Bonjour. Ça va ? Très bien.", first_chunk_sentences=1) ['Bonjour.', 'Ça va ?', 'Très bien.'] >>> chunk_for_streaming("Bonjour. Ça va ? Très bien.", first_chunk_sentences=2) ['Bonjour. Ça va ?', 'Très bien.'] >>> chunk_for_streaming("Bonjour. Un. Et voilà.") # too short to stand alone ['Bonjour. Un.', 'Et voilà.'] """ sentences = split_sentences(text) if not sentences: return [] # Never emit a zero-sentence first chunk, whatever the caller passed. n = max(1, first_chunk_sentences) # The first chunk bundles ``n`` sentences (its size sets TTFA); every later # sentence is its own chunk so audio keeps flowing at a steady cadence. first = " ".join(sentences[:n]) # Sentence-per-chunk is what makes fragments possible in the first place # ("Un." is its own sentence), so the merge belongs right here. return merge_short_fragments([first, *sentences[n:]])