"""Embedders: one model on the build machine, its twin in the browser.
The index is embedded here, with sentence-transformers; the visitor's question
is embedded in the page, with transformers.js. Both must be the same model,
or cosines compare vectors from two different spaces. ``MODELS`` pairs each
supported Python name with its browser (ONNX) twin.
Long passages are cut into WINDOWS that the model reads entirely (it ignores
everything past its token limit), and a passage scores the best cosine of its
windows. The end offset of each window is kept: it tells the page WHERE in a
passage the answer is, and it bounds the excerpt the server sends back.
"""
from __future__ import annotations
import hashlib
from typing import Protocol
import numpy as np
from .text import strip_math, tokenize
#: Python model -> transformers.js model of the same weights.
MODELS = {
"paraphrase-multilingual-MiniLM-L12-v2": "Xenova/paraphrase-multilingual-MiniLM-L12-v2",
"sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2": "Xenova/paraphrase-multilingual-MiniLM-L12-v2",
"all-MiniLM-L6-v2": "Xenova/all-MiniLM-L6-v2",
"sentence-transformers/all-MiniLM-L6-v2": "Xenova/all-MiniLM-L6-v2",
}
DEFAULT_MODEL = "paraphrase-multilingual-MiniLM-L12-v2"
[docs]
class Embedder(Protocol):
name: str # Python-side model name
browser: str # transformers.js model id
dim: int
[docs]
def windows(self, text: str) -> list[int]:
"""End offsets (in characters) of the windows covering ``text``."""
[docs]
def encode(self, texts: list[str]) -> np.ndarray:
"""L2-normalised float32 vectors, one row per text."""
[docs]
class HashEmbedder:
"""Deterministic bag-of-words vectors, for tests and offline dry runs.
Not a semantic model: two texts are close when they share words. It lets
the whole pipeline, PHP included, be tested without downloading weights.
"""
browser = "hash"
def __init__(self, dim: int = 64, window_words: int = 40):
self.dim = dim
self.window_words = window_words
@property
def name(self) -> str:
# The parameters travel in the name, which the index records: an index
# built at 32 dimensions must be queried at 32, not at the default.
return f"hash:{self.dim}:{self.window_words}"
[docs]
def windows(self, text: str) -> list[int]:
ends, count = [], 0
for k, c in enumerate(text):
if c.isspace() and k and not text[k - 1].isspace():
count += 1
if count % self.window_words == 0:
ends.append(k)
return ends + [len(text)]
[docs]
def encode(self, texts: list[str]) -> np.ndarray:
out = np.zeros((len(texts), self.dim), dtype=np.float64)
for r, t in enumerate(texts):
for w in tokenize(t):
h = int.from_bytes(hashlib.blake2b(w.encode(), digest_size=8).digest(), "little")
out[r, h % self.dim] += 1.0 if (h >> 32) & 1 else -1.0
n = np.linalg.norm(out, axis=1, keepdims=True)
out = np.divide(out, n, out=np.zeros_like(out), where=n > 0)
return out.astype(np.float32)
[docs]
def window_texts(text: str, ends: list[int]) -> list[str]:
"""The text of each window, formulas reduced to their symbols."""
starts = [0] + ends[:-1]
return [strip_math(text[a:b]) for a, b in zip(starts, ends)]
[docs]
def make_embedder(name: str, browser: str | None = None):
"""The embedder an index records as ``meta["model"]``, rebuilt.
>>> make_embedder("hash:32:12").dim
32
"""
if name == "hash" or name.startswith("hash:"):
parts = name.split(":")
return HashEmbedder(*(int(x) for x in parts[1:3])) if len(parts) == 3 else HashEmbedder()
return SentenceTransformerEmbedder(name, browser)