"""
browser — choose a language model for a machine best-engine never sees: the visitor's browser.
Everywhere else best-engine probes the machine, then picks ONE model. In a web
page it cannot: the machine is the visitor's, and it reveals itself only at the
first question, through what the browser chooses to expose. So the browser
target works in two halves.
1. **In advance, here** — :func:`plan` reads a usage profile (``rag-answer``…)
and the browser catalog (``data/browser_models.yaml``), and writes an ordered
ladder of candidates, every one with its requirements: download size per
dtype, ``shader-f16``, a device-memory floor, the exact files to fetch.
2. **At run time, in the page** — the page walks the ladder down and takes the
first candidate the machine and the line can bear. :func:`choose` is that
walk, in Python: the reference the JavaScript twin is tested against, and a
way to simulate a device (``best-engine-ai-helper browser choose``).
The criteria are new to best-engine, because a page pays for things a server
never does:
- **the download**, judged against the bandwidth OBSERVED in the page (a 3 GB
model is fine on fibre and absurd on a train), and free once cached;
- **shader-f16**, which decides between q4f16 and q4 weights;
- **deviceMemory**, a rounded figure, as a proxy for a GPU nobody can query;
- **mobile** and **data saver**, which rule a multi-gigabyte download out;
- **the storage quota** left for the browser's cache.
A signal the browser does not expose (Safari has no deviceMemory) is never held
against the machine; an unknown bandwidth is replaced by a deliberately modest
assumption, so that silence never buys a 5 GB download.
Author
------
Warith Harchaoui <warithmetics@deraison.ai>
"""
from __future__ import annotations
import json
import urllib.request
from collections.abc import Callable
from datetime import datetime, timezone
from pathlib import Path
from typing import Any
import yaml
_SEED_PATH = Path(__file__).resolve().parent / "data" / "browser_models.yaml"
_CACHE_PATH = Path.home() / ".best-engine-ai-helper" / "browser_cache.yaml"
HUB = "https://huggingface.co"
#: Runtime the plan is written for. The page must import this exact build: the
#: catalog's ONNX exports and chat templates are checked against it.
TRANSFORMERS_JS = "@huggingface/transformers@4.3.2"
#: Policy written into every plan, read by the page.
DEFAULT_POLICY = {
# Longest first download offered without second thoughts, in seconds.
"max_download_s": 180,
# Assumed bandwidth when the page could not measure one: 20 Mbit/s.
"unknown_bandwidth_Bps": 2_500_000,
# Bytes fetched PER WEIGHT SHARD to measure the bandwidth: one Range
# request per shard, all at once, timed from the first byte, the way the
# model will load (a CDN caps each connection).
"probe_bytes": 4_000_000,
# Free storage needed, as a multiple of the download.
"storage_margin": 1.1,
# effectiveType values that rule a model download out.
"slow_networks": ["slow-2g", "2g", "3g"],
}
#: A model measured below this on the usage stays in the catalog, with its
#: score, but leaves the ladder: downloading it would mostly buy answers the
#: page then sets aside. Unmeasured models are kept, on their prior.
DEFAULT_MIN_SCORE = 0.5
[docs]
def load_browser_catalog(
seed: Path | None = None, cache: Path | None = None
) -> list[dict[str, Any]]:
"""
The browser catalog: the bundled seed, overlaid by the user cache by ``id``.
Examples
--------
>>> cat = load_browser_catalog(cache=Path("/nonexistent"))
>>> cat[0]["id"]
'onnx-community/gemma-4-E4B-it-ONNX'
>>> sorted(cat[1]["variants"])
['q4', 'q4f16']
"""
entries = yaml.safe_load((seed or _SEED_PATH).read_text(encoding="utf-8")) or []
cache = cache if cache is not None else _CACHE_PATH
if cache.exists():
overlay = {e["id"]: e for e in (yaml.safe_load(cache.read_text(encoding="utf-8")) or [])}
entries = [{**e, **overlay.get(e["id"], {})} for e in entries]
return entries
def _hub_tree(repo: str) -> list[dict[str, Any]]:
url = f"{HUB}/api/models/{repo}/tree/main/onnx"
with urllib.request.urlopen(url, timeout=30) as resp:
tree: list[dict[str, Any]] = json.load(resp)
return tree
[docs]
def refresh(
catalog: list[dict[str, Any]] | None = None,
*,
fetch: Callable[[str], list[dict[str, Any]]] = _hub_tree,
cache: Path | None = None,
) -> list[dict[str, Any]]:
"""
Re-read each variant's files and download size from the Hugging Face Hub.
A variant's files are the ONNX graphs and weight shards whose names end in
its dtype (``model_q4f16.onnx``, ``embed_tokens_q4f16.onnx_data``…); vision
and audio encoders are left out, the browser target being text-only. The
result is written to the user cache unless ``cache`` is ``False``-y.
Examples
--------
>>> tree = [{"path": "onnx/model_q4.onnx", "size": 10},
... {"path": "onnx/model_q4.onnx_data", "lfs": {"size": 90}},
... {"path": "onnx/vision_encoder_q4.onnx", "size": 5}]
>>> cat = [{"id": "x/y", "variants": {"q4": {"needs_f16": False}}}]
>>> out = refresh(cat, fetch=lambda repo: tree, cache=None)
>>> out[0]["variants"]["q4"]["bytes"], out[0]["variants"]["q4"]["files"]
(100, ['onnx/model_q4.onnx', 'onnx/model_q4.onnx_data'])
"""
catalog = catalog if catalog is not None else load_browser_catalog()
out = []
for entry in catalog:
tree = fetch(entry["id"])
variants = {}
for dtype, v in entry["variants"].items():
files, size = [], 0
for f in tree:
name = f["path"].split("/")[-1]
if "vision" in name or "audio" in name:
continue
if name.split(".onnx")[0].endswith("_" + dtype):
files.append(f["path"])
size += (f.get("lfs") or {}).get("size") or f.get("size") or 0
variants[dtype] = {**v, "bytes": size, "files": sorted(files)} if files else v
out.append(
{
**entry,
"variants": variants,
"fetched_at": datetime.now(timezone.utc).date().isoformat(),
}
)
if cache:
cache.parent.mkdir(parents=True, exist_ok=True)
cache.write_text(yaml.safe_dump(out, sort_keys=False, allow_unicode=True), encoding="utf-8")
return out
[docs]
def plan(
usage: str = "rag-answer",
*,
languages: list[str] | None = None,
catalog: list[dict[str, Any]] | None = None,
policy: dict[str, Any] | None = None,
min_score: float = DEFAULT_MIN_SCORE,
) -> dict[str, Any]:
"""
The ordered ladder of browser candidates for ``usage``, ready for a page.
One rung per (model, dtype), best first: models by their rank for the usage
(a measured score when the catalog carries one, the curator's prior
otherwise), and within a model the ``shader-f16`` variant first, since it
is both smaller and faster where it runs. A model that does not serve every
language in ``languages`` is left out, and so is a model MEASURED on the
usage below ``min_score``; the plan lists both under ``excluded``.
Examples
--------
>>> p = plan("rag-answer", languages=["fr", "en"])
>>> p["usage"], p["runtime"]
('rag-answer', 'transformers.js')
>>> [c["model"].split("/")[1] + " " + c["dtype"] for c in p["candidates"]]
['gemma-4-E4B-it-ONNX q4f16', 'gemma-4-E4B-it-ONNX q4', 'gemma-4-E2B-it-ONNX q4f16', \
'gemma-4-E2B-it-ONNX q4']
"""
from . import usages as _usages
profile = _usages.get_usage(usage) # KeyError on an unknown usage
catalog = catalog if catalog is not None else load_browser_catalog()
languages = languages or ["fr", "en"]
def rank(e: dict[str, Any]) -> tuple[int, float, int]:
measured = (e.get("measured") or {}).get(usage)
prior = (e.get("tasks") or {}).get(usage, 99)
# A measured score outranks any prior; higher score first, and the
# prior breaks ties: on a few dozen questions, equal scores are common
# and say nothing about which model is better.
return (0, -measured, prior) if measured is not None else (1, 0, prior)
pool, excluded = [], []
for e in catalog:
if usage not in (e.get("tasks") or {}):
continue
measured = (e.get("measured") or {}).get(usage)
if not set(languages) <= set(e.get("languages", [])):
excluded.append({"model": e["id"], "reason": "languages"})
elif measured is not None and measured < min_score:
excluded.append({"model": e["id"], "reason": f"measured {measured} < {min_score}"})
else:
pool.append(e)
candidates = []
for e in sorted(pool, key=rank):
for dtype, v in sorted(e["variants"].items(), key=lambda kv: not kv[1].get("needs_f16")):
candidates.append(
{
"model": e["id"],
"family": e.get("family"),
"dtype": dtype,
"bytes": int(v["bytes"]),
"needs_f16": bool(v.get("needs_f16")),
"min_device_memory_gb": e.get("min_device_memory_gb"),
"files": [f"{HUB}/{e['id']}/resolve/main/{f}" for f in v.get("files", [])],
"loader": e.get("loader", "tokenizer"),
"chat_kwargs": e.get("chat_kwargs") or {},
"params_b": e.get("params_b"),
"license": e.get("license"),
"measured": (e.get("measured") or {}).get(usage),
}
)
return {
"usage": usage,
"summary": profile.get("summary", ""),
"runtime": "transformers.js",
"transformers": TRANSFORMERS_JS,
"languages": languages,
"generated_at": datetime.now(timezone.utc).date().isoformat(),
"policy": {**DEFAULT_POLICY, **(policy or {})},
"min_score": min_score,
"candidates": candidates,
"excluded": excluded,
}
[docs]
def choose(the_plan: dict[str, Any], device: dict[str, Any]) -> dict[str, Any]:
"""
Walk the ladder for one device; the reference of the page's JavaScript.
``device`` holds what the page measured; every key is optional:
``webgpu``, ``f16``, ``mobile``, ``save_data``, ``effective_type``,
``device_memory_gb``, ``storage_free_bytes``, ``bandwidth_Bps`` and
``cached`` (the candidates, as ``"model|dtype"``, already in the cache).
Returns ``{"candidate", "reason", "seconds"}``; ``candidate`` is None when
nothing fits, and ``reason`` says why: ``webgpu``, ``mobile``, ``network``,
``device`` (no rung fits the GPU, memory or storage), ``download`` (every
fitting rung is too long to fetch on this line). A cached rung costs no
download, whatever its size.
Examples
--------
>>> p = plan("rag-answer")
>>> fast = {"webgpu": True, "f16": True, "bandwidth_Bps": 50e6}
>>> c = choose(p, fast)
>>> c["candidate"]["model"].split("/")[1], c["candidate"]["dtype"], c["reason"]
('gemma-4-E4B-it-ONNX', 'q4f16', 'ok')
>>> mid = {**fast, "bandwidth_Bps": 20e6} # 160 Mbit/s: E4B takes 4 min
>>> choose(p, mid)["candidate"]["model"].split("/")[1]
'gemma-4-E2B-it-ONNX'
>>> choose(p, {**fast, "bandwidth_Bps": 5e6})["reason"] # 40 Mbit/s: nothing in 3 min
'download'
>>> choose(p, {"webgpu": True, "mobile": True})["reason"]
'mobile'
"""
pol = the_plan["policy"]
if not device.get("webgpu"):
return {"candidate": None, "reason": "webgpu", "seconds": None}
if device.get("mobile"):
return {"candidate": None, "reason": "mobile", "seconds": None}
if device.get("save_data") or device.get("effective_type") in pol["slow_networks"]:
return {"candidate": None, "reason": "network", "seconds": None}
cached = set(device.get("cached") or [])
bandwidth = device.get("bandwidth_Bps") or pol["unknown_bandwidth_Bps"]
fits_device = False
for c in the_plan["candidates"]:
if c["needs_f16"] and not device.get("f16"):
continue
mem = device.get("device_memory_gb")
if mem is not None and c.get("min_device_memory_gb") and mem < c["min_device_memory_gb"]:
continue
if f"{c['model']}|{c['dtype']}" in cached:
return {"candidate": c, "reason": "cached", "seconds": 0.0}
free = device.get("storage_free_bytes")
if free is not None and free < c["bytes"] * pol["storage_margin"]:
continue
fits_device = True
seconds = c["bytes"] / bandwidth
if seconds <= pol["max_download_s"]:
return {"candidate": c, "reason": "ok", "seconds": round(seconds, 1)}
return {"candidate": None, "reason": "download" if fits_device else "device", "seconds": None}