Source code for web_search_helper.corpus

"""Read a corpus into passages.

A passage is the unit the engine ranks and returns an excerpt of. It carries
the text, the document and section it comes from (shown as the source), an
optional ``url`` the page can link to, and a free ``meta`` dict passed through
untouched to the browser.

Two inputs:

- **JSONL**, one passage per line: ``{"text": …, "doc": …, "section": …,
  "url": …, "meta": {…}}``. Use it when the passages are already cut, by
  your own pipeline.
- **A folder of Markdown or text files**: each ``#`` heading opens a section,
  the first heading (or the file name) titles the document, and paragraphs
  are grouped into passages of about ``target_words`` words.

>>> ps = passages_from_markdown("# Doc\\n\\n## Part\\n\\nOne two three.", "d.md")
>>> [(p.doc, p.section, p.text) for p in ps]
[('Doc', 'Part', 'One two three.')]
"""

from __future__ import annotations

import json
import re
from dataclasses import dataclass, field
from pathlib import Path

import os_helper as osh


[docs] @dataclass class Passage: text: str doc: str = "" section: str = "" url: str = "" meta: dict = field(default_factory=dict)
[docs] def to_dict(self) -> dict: d = {"t": self.text, "c": self.doc, "s": self.section} if self.url: d["u"] = self.url if self.meta: d["m"] = self.meta return d
[docs] def read_jsonl(path: str | Path) -> list[Passage]: out = [] for n, line in enumerate(Path(path).read_text(encoding="utf-8").splitlines(), 1): if not line.strip(): continue r = json.loads(line) if not isinstance(r.get("text"), str) or not r["text"].strip(): raise ValueError(f"{path}:{n}: 'text' missing or empty") out.append( Passage( r["text"].strip(), str(r.get("doc", "")), str(r.get("section", "")), str(r.get("url", "")), dict(r.get("meta") or {}), ) ) return out
_HEADING = re.compile(r"^(#{1,6})\s+(.*?)\s*#*\s*$")
[docs] def passages_from_markdown(source: str, name: str, target_words: int = 220) -> list[Passage]: """Cut one Markdown (or plain text) document into passages. Paragraphs are never split: a passage closes at the first paragraph boundary past ``target_words``, and never spans two sections. """ doc = Path(name).stem section = "" out: list[Passage] = [] buf: list[str] = [] def flush(): if buf: out.append(Passage("\n\n".join(buf), doc, section or doc)) buf.clear() first_heading = True for block in re.split(r"\n\s*\n", source.replace("\r\n", "\n")): block = block.strip() if not block: continue m = _HEADING.match(block.splitlines()[0]) if m: flush() title = m.group(2) if first_heading and len(m.group(1)) == 1: doc = title section = title first_heading = False rest = "\n".join(block.splitlines()[1:]).strip() if not rest: continue block = rest buf.append(" ".join(block.split())) if sum(len(b.split()) for b in buf) >= target_words: flush() flush() return out
[docs] def read_folder(folder: str | Path, target_words: int = 220) -> list[Passage]: """Every ``.md``, ``.markdown`` and ``.txt`` file under ``folder``, sorted by path.""" folder = Path(folder) out: list[Passage] = [] for f in sorted(folder.rglob("*")): if f.suffix.lower() in {".md", ".markdown", ".txt"} and osh.file_exists(str(f)): out.extend(passages_from_markdown(f.read_text(encoding="utf-8"), f.name, target_words)) return out
[docs] def read_corpus(path: str | Path, target_words: int = 220) -> list[Passage]: """A ``.jsonl`` file, a single Markdown/text file, or a folder of them.""" path = Path(path) # A folder, a JSONL file or a single Markdown/text file, in that order. if osh.dir_exists(str(path)): passages = read_folder(path, target_words) elif path.suffix.lower() == ".jsonl": passages = read_jsonl(path) else: passages = passages_from_markdown(path.read_text(encoding="utf-8"), path.name, target_words) osh.info(f"{path}: {len(passages)} passages read") return passages