Source code for web_search_helper.text

"""Text normalisation shared by the index builder and the PHP engine.

The PHP endpoint tokenises the visitor's question; this module tokenises the
corpus at build time. The two must cut words the same way, or a word present
in the corpus will never be found. Every rule here therefore has a twin in
``assets/search.php``, and ``tests/test_php_parity.py`` runs both on the same
strings.

>>> fold("Éléphant")
'elephant'
>>> tokenize("La parabole de l'éléphant", stopwords("fr"))
['parabole', 'elephant']
"""

from __future__ import annotations

import re
import unicodedata

#: Default stopword lists. Short on purpose: BM25's idf already discounts
#: frequent words; these only remove words too frequent to mean anything in a
#: question. Folded spellings are what count, since tokens are folded first.
STOPWORDS = {
    "fr": (
        "le la les un une des de du et est sont que qui pour dans sur par ce cet "
        "cette ces son sa ses avec sans plus moins pas ne ou mais donc car ni si "
        "comme quand tout tous toute toutes il elle ils elles nous vous etre avoir "
        "fait faire quel quelle quels quelles quoi dont cela ceci leur leurs "
        "pourquoi comment"
    ),
    "en": (
        "the a an of and is are that which for in on by this these those his her "
        "its with without more less not or but so as when what how why there here "
        "it he she they we you be have do does did"
    ),
}

_MATH = re.compile(r"\$\$([\s\S]+?)\$\$|\\\(([\s\S]+?)\\\)")
_MACRO = re.compile(r"\\[a-zA-Z]+\s*")
_BRACES = re.compile(r"[{}\\]")


[docs] def stopwords(lang: str) -> frozenset[str]: """Folded stopwords for ``lang``; empty for a language without a list. >>> "the" in stopwords("en"), len(stopwords("xx")) (True, 0) """ return frozenset(fold(w) for w in STOPWORDS.get(lang, "").split())
[docs] def fold(s: str) -> str: """Lower-case, then strip accents (NFD, combining marks U+0300–U+036F). >>> fold("Œuvre à Noël") 'œuvre a noel' """ s = unicodedata.normalize("NFD", s.lower()) return "".join(c for c in s if not 0x300 <= ord(c) <= 0x36F)
[docs] def strip_math(s: str) -> str: r"""Keep only the symbols of ``$$…$$`` and ``\(…\)`` formulas. >>> strip_math(r"a \(\mathbf{x}_i\) b") 'a x_i b' """ def keep(m: re.Match) -> str: inner = m.group(1) or m.group(2) or "" return _BRACES.sub("", _MACRO.sub("", inner)) return _MATH.sub(keep, s)
def _is_word_char(c: str) -> bool: # Same class as PCRE's [\p{L}\p{N}]. return unicodedata.category(c)[0] in "LN"
[docs] def tokenize(s: str, stop: frozenset[str] = frozenset()) -> list[str]: """Words of more than two characters, folded, stopwords removed. >>> tokenize("What is the data wall?", stopwords("en")) ['data', 'wall'] """ out, cur = [], [] for c in fold(strip_math(s)) + " ": if _is_word_char(c): cur.append(c) elif cur: w = "".join(cur) cur = [] if len(w) > 2 and w not in stop: out.append(w) return out