"""The index file: everything the PHP engine needs, in one file it never serves.
The file is a PHP script that, requested by URL, answers 404 and exits. The
data follows ``__halt_compiler();``, which PHP never parses. The corpus is
therefore served in clear at no address, even when the web server ignores the
``.htaccess`` that also denies the folder (an FTP client that skips dotfiles
is enough for that).
Layout after the header (little-endian):
========== ====================================================================
``WSH1`` magic
u32 + JSON metadata: config, thresholds, passages, section of each passage
u32 × 4 number of vectors, sections, words, bytes of postings
f32 window vectors, ``n_vec × dim``, passages in order
f32 section centroids, ``n_sec × dim``
u32 + str the lexicon, words joined by ``\\n``, in order of first appearance
u32 ``n_words + 1`` offsets into the postings
(u32,u16) postings: (passage, term frequency), passages in increasing order
u32 length of each passage in tokens
========== ====================================================================
Everything that can be computed once is computed here, so that a request only
reads: the lexical index, the section centroids, the window offsets.
>>> from web_search_helper.corpus import Passage
>>> from web_search_helper.embed import HashEmbedder
>>> idx = build([Passage("Un éléphant dans une pièce.", "Doc", "S")], HashEmbedder(), "fr")
>>> data = to_bytes(idx)
>>> data.startswith(HEADER), from_bytes(data).passages[0]["t"]
(True, 'Un éléphant dans une pièce.')
"""
from __future__ import annotations
import json
import struct
from dataclasses import dataclass, field
from itertools import pairwise
from pathlib import Path
import numpy as np
import os_helper as osh
from .corpus import Passage
from .embed import Embedder, window_texts
from .text import stopwords as default_stopwords
from .text import tokenize
#: Requested by URL, the file answers 404 and stops. ``search.php`` checks the
#: same bytes before reading.
HEADER = b"<?php http_response_code(404); exit; __halt_compiler();"
MAGIC = b"WSH1"
FORMAT = 1
#: Thresholds of the "I cannot answer" rule. Defaults are those measured on the
#: corpus this engine was first built for; `calibrate` measures yours.
DEFAULT_THRESHOLDS = {
"sem_min": 0.55, # cosine of the best window
"sec_min": 0.561, # cosine of the best section centroid
"lex_min": 5.0, # BM25 of the best passage, when every word was found
"df_min": 9, # one-word query: passages that must contain the word
}
DEFAULT_LIMITS = {"per_hour": 120, "per_day": 400, "q_max": 300}
DEFAULT_RANKING = {"display_k": 6, "pool_k": 25, "answer_pool": 15, "rrf_k": 60}
[docs]
@dataclass
class Index:
meta: dict
passages: list[dict]
section_of: list[int]
vectors: np.ndarray # (n_vec, dim) float32
centroids: np.ndarray # (n_sec, dim) float32
words: list[str]
postings: dict[str, list[tuple[int, int]]]
lengths: list[int]
first_vec: list[int] = field(default_factory=list)
def __post_init__(self):
if not self.first_vec:
cur = 0
for p in self.passages:
self.first_vec.append(cur)
cur += len(p["o"])
if cur != len(self.vectors):
raise ValueError(f"{cur} windows announced, {len(self.vectors)} vectors")
@property
def lang(self) -> str:
return self.meta["lang"]
@property
def stop(self) -> frozenset[str]:
return frozenset(self.meta["stopwords"])
[docs]
def build(
passages: list[Passage],
embedder: Embedder,
lang: str,
stop: frozenset[str] | None = None,
thresholds: dict | None = None,
limits: dict | None = None,
ranking: dict | None = None,
batch: int = 256,
) -> Index:
"""Cut, embed and index ``passages``."""
if not passages:
raise ValueError("empty corpus")
stop = default_stopwords(lang) if stop is None else stop
rows = [p.to_dict() for p in passages]
texts: list[str] = []
for r in rows:
ends = embedder.windows(r["t"])
r["o"] = ends
texts.extend(window_texts(r["t"], ends))
vecs = np.concatenate(
[embedder.encode(texts[k : k + batch]) for k in range(0, len(texts), batch)]
).astype(np.float32)
return from_vectors(
rows,
vecs,
lang,
model={"python": embedder.name, "browser": embedder.browser},
stop=stop,
thresholds=thresholds,
limits=limits,
ranking=ranking,
)
[docs]
def from_vectors(
rows: list[dict],
vectors: np.ndarray,
lang: str,
model: dict,
stop: frozenset[str] | None = None,
thresholds: dict | None = None,
limits: dict | None = None,
ranking: dict | None = None,
) -> Index:
"""Index passages whose windows are already embedded, by another pipeline.
For a site that computes its own embeddings and wants the engine without
re-embedding (or wants the SAME vectors, to compare engines). ``rows`` are
dicts with ``t`` (text), ``c`` (document), ``s`` (section), ``o`` (end
offset of each window in ``t``) and optionally ``u`` (url) and ``m`` (meta
passed to the browser); ``vectors`` holds one L2-normalised row per window,
passages in order. ``model`` names the Python model and its transformers.js
twin, which the page must use for the question.
>>> rows = [{"t": "Un éléphant.", "c": "D", "s": "S", "o": [12]}]
>>> idx = from_vectors(rows, np.eye(1, 4, dtype=np.float32), "fr",
... {"python": "m", "browser": "Xenova/m"})
>>> idx.meta["dim"], idx.words
(4, ['elephant'])
"""
if not rows:
raise ValueError("empty corpus")
vecs = np.asarray(vectors, dtype=np.float32)
announced = 0
for k, r in enumerate(rows):
o = r.get("o")
# Non-decreasing, not strictly increasing: a pipeline may leave an empty
# window (a trailing " ." cut on its own, seen in the wild), whose
# vector exists and must stay aligned with the rest.
if not o or o[-1] != len(r["t"]) or any(a > b for a, b in pairwise(o)):
raise ValueError(f"passage {k}: 'o' must be non-decreasing window ends, the last at len(t)")
announced += len(o)
if announced != len(vecs):
raise ValueError(f"{announced} windows announced, {len(vecs)} vectors")
stop = default_stopwords(lang) if stop is None else stop
rows = [dict(r) for r in rows]
# Sections in order of first appearance; a centroid is the normalised sum
# of the section's windows. It judges whether a whole SECTION is about the
# question, which a single window can seem to be by accident of vocabulary.
keys: dict[tuple[str, str], int] = {}
section_of = [keys.setdefault((r["c"], r["s"]), len(keys)) for r in rows]
cents = np.zeros((len(keys), vecs.shape[1]), dtype=np.float64)
cur = 0
for r, s in zip(rows, section_of):
n = len(r["o"])
cents[s] += vecs[cur : cur + n].sum(axis=0)
cur += n
norms = np.linalg.norm(cents, axis=1, keepdims=True)
cents = np.divide(cents, norms, out=np.zeros_like(cents), where=norms > 0).astype(np.float32)
words: list[str] = []
postings: dict[str, list[tuple[int, int]]] = {}
lengths: list[int] = []
for i, r in enumerate(rows):
toks = tokenize(r["t"], stop)
lengths.append(len(toks) or 1)
tf: dict[str, int] = {}
for w in toks:
tf[w] = tf.get(w, 0) + 1
for w, c in tf.items():
if w not in postings:
postings[w] = []
words.append(w)
postings[w].append((i, c))
meta = {
"format": FORMAT,
"lang": lang,
"dim": int(vecs.shape[1]),
"model": dict(model),
"stopwords": sorted(stop),
"thresholds": {**DEFAULT_THRESHOLDS, **(thresholds or {})},
"limits": {**DEFAULT_LIMITS, **(limits or {})},
"ranking": {**DEFAULT_RANKING, **(ranking or {})},
"avg_len": sum(lengths) / len(lengths),
}
return Index(meta, rows, section_of, vecs, cents, words, postings, lengths)
[docs]
def vectors_from_int8(data: bytes, dim: int) -> np.ndarray:
"""int8 vectors (component × 127), read back as a browser would: each row
divided by its own norm, in float32.
>>> vectors_from_int8(bytes([127, 0, 0, 0]), 4).tolist()
[[1.0, 0.0, 0.0, 0.0]]
"""
b = np.frombuffer(data, dtype=np.int8).astype(np.float64).reshape(-1, dim)
n = np.sqrt((b * b).sum(axis=1))
inv = np.divide(1.0, n, out=np.zeros_like(n), where=n > 0)
return (b * inv[:, None]).astype(np.float32)
[docs]
def to_bytes(idx: Index) -> bytes:
meta = json.dumps(
{**idx.meta, "passages": idx.passages, "section": idx.section_of},
ensure_ascii=False,
separators=(",", ":"),
).encode("utf-8")
lex = "\n".join(idx.words).encode("utf-8")
offsets, blob = [], bytearray()
for w in idx.words:
offsets.append(len(blob))
for i, c in idx.postings[w]:
blob += struct.pack("<IH", i, min(c, 65535))
offsets.append(len(blob))
return b"".join(
[
HEADER,
MAGIC,
struct.pack("<I", len(meta)),
meta,
struct.pack("<IIII", len(idx.vectors), len(idx.centroids), len(idx.words), len(blob)),
idx.vectors.astype("<f4").tobytes(),
idx.centroids.astype("<f4").tobytes(),
struct.pack("<I", len(lex)),
lex,
struct.pack(f"<{len(offsets)}I", *offsets),
bytes(blob),
struct.pack(f"<{len(idx.lengths)}I", *idx.lengths),
]
)
[docs]
def from_bytes(data: bytes) -> Index:
if not data.startswith(HEADER) or data[len(HEADER) : len(HEADER) + 4] != MAGIC:
raise ValueError("not a web-search-helper index")
o = len(HEADER) + 4
(lm,) = struct.unpack_from("<I", data, o)
o += 4
meta = json.loads(data[o : o + lm])
o += lm
n_vec, n_sec, n_words, n_blob = struct.unpack_from("<IIII", data, o)
o += 16
dim = meta["dim"]
vecs = np.frombuffer(data, "<f4", n_vec * dim, o).reshape(n_vec, dim)
o += n_vec * dim * 4
cents = np.frombuffer(data, "<f4", n_sec * dim, o).reshape(n_sec, dim)
o += n_sec * dim * 4
(ll,) = struct.unpack_from("<I", data, o)
o += 4
words = data[o : o + ll].decode("utf-8").split("\n") if ll else []
o += ll
offsets = struct.unpack_from(f"<{n_words + 1}I", data, o)
o += 4 * (n_words + 1)
blob = data[o : o + n_blob]
o += n_blob
postings = {}
for k, w in enumerate(words):
postings[w] = [struct.unpack_from("<IH", blob, j) for j in range(offsets[k], offsets[k + 1], 6)]
passages = meta.pop("passages")
section_of = meta.pop("section")
lengths = list(struct.unpack_from(f"<{len(passages)}I", data, o))
return Index(meta, passages, section_of, np.array(vecs), np.array(cents), words, postings, lengths)
[docs]
def save(idx: Index, path: str | Path) -> Path:
path = Path(path)
osh.make_directory(str(path.parent))
# Written beside the target, then renamed over it: a reader (search.php,
# the API) never sees half an index.
tmp = path.with_suffix(path.suffix + ".tmp")
tmp.write_bytes(to_bytes(idx))
tmp.replace(path)
osh.info(f"index written: {path} ({len(idx.passages)} passages, {len(idx.vectors)} windows)")
return path
[docs]
def load(path: str | Path) -> Index:
return from_bytes(Path(path).read_bytes())