""",
js_on_load="""
element.addEventListener('click', (e) => {
const btn = e.target.closest('.pill');
if (!btn) return;
props.value = btn.dataset.v;
trigger('change');
});""",
**kwargs,
)
def api_info(self):
return {"type": "string"}
class SelectBox(gr.HTML):
"""Type-to-filter combobox (replaces gr.Dropdown with filterable=True):
a text input that opens a filtered option list, with arrow/Enter/Escape
keyboard support. All listeners are delegated on the component root so
they survive the re-render that follows each value sync."""
def __init__(self, options, value, label, **kwargs):
super().__init__(
value=value,
options=list(options),
label_text=label,
html_template="""
${label_text}
▾
${options.map(o => `
${o.replace(/&/g,'&').replace(/`).join('')}
No match
""",
js_on_load="""
const list = () => element.querySelector('.cbx-list');
const input = () => element.querySelector('.cbx-input');
const items = () => [...element.querySelectorAll('.cbx-item')];
const visible = () => items().filter(it => !it.hidden);
function open(showAll) {
if (showAll) { items().forEach(it => { it.hidden = false; }); element.querySelector('.cbx-empty').hidden = true; }
list().hidden = false;
input().setAttribute('aria-expanded', 'true');
}
function close(restore) {
list().hidden = true;
input().setAttribute('aria-expanded', 'false');
items().forEach(it => it.classList.remove('active'));
if (restore) input().value = props.value;
}
function applyFilter() {
const q = input().value.toLowerCase();
let any = false;
items().forEach(it => {
it.hidden = !it.textContent.toLowerCase().includes(q);
it.classList.remove('active');
if (!it.hidden) any = true;
});
element.querySelector('.cbx-empty').hidden = any;
}
function move(dir) {
const vis = visible();
if (!vis.length) return;
const cur = vis.findIndex(it => it.classList.contains('active'));
const next = Math.min(Math.max(cur + dir, 0), vis.length - 1);
vis.forEach(it => it.classList.remove('active'));
vis[next].classList.add('active');
vis[next].scrollIntoView({ block: 'nearest' });
}
function pick(v) {
close(false);
props.value = v;
trigger('change');
}
element.addEventListener('focusin', (e) => {
if (!e.target.classList.contains('cbx-input')) return;
e.target.select();
open(true);
});
element.addEventListener('input', (e) => {
if (!e.target.classList.contains('cbx-input')) return;
open(false);
applyFilter();
});
element.addEventListener('keydown', (e) => {
if (!e.target.classList.contains('cbx-input')) return;
if (e.key === 'ArrowDown') { e.preventDefault(); if (list().hidden) open(true); move(1); }
else if (e.key === 'ArrowUp') { e.preventDefault(); move(-1); }
else if (e.key === 'Enter') {
e.preventDefault();
const target = visible().find(it => it.classList.contains('active')) || visible()[0];
if (target) pick(target.dataset.v);
}
else if (e.key === 'Escape') { close(true); e.target.blur(); }
});
element.addEventListener('pointerdown', (e) => {
const it = e.target.closest('.cbx-item');
if (!it) return;
e.preventDefault(); // beat the input's blur
pick(it.dataset.v);
});
element.addEventListener('focusout', (e) => {
if (!element.contains(e.relatedTarget)) close(true);
});""",
**kwargs,
)
def api_info(self):
return {"type": "string"}
class RangeSlider(gr.HTML):
"""Native range input with a value readout (replaces gr.Slider)."""
def __init__(self, minimum, maximum, value, step, label, **kwargs):
super().__init__(
value=value,
minimum=minimum,
maximum=maximum,
step=step,
label_text=label,
html_template="""
${label_text}
${value}
""",
js_on_load="""
element.addEventListener('input', (e) => {
if (!e.target.classList.contains('ctl-range')) return;
const out = element.querySelector('.ctl-range-val');
if (out) out.textContent = e.target.value;
});
element.addEventListener('change', (e) => {
if (!e.target.classList.contains('ctl-range')) return;
props.value = Number(e.target.value);
trigger('change');
});""",
**kwargs,
)
def api_info(self):
return {"type": "integer"}
class SearchBox(gr.HTML):
"""Debounced text search (replaces gr.Textbox). Focus is restored after the
value-sync re-render so typing isn't interrupted."""
def __init__(self, value, label, placeholder, **kwargs):
super().__init__(
value=value,
label_text=label,
placeholder=placeholder,
html_template="""
${label_text}
""",
js_on_load="""
let t = null;
element.addEventListener('input', (e) => {
if (!e.target.classList.contains('ctl-search')) return;
clearTimeout(t);
const v = e.target.value;
t = setTimeout(() => {
props.value = v;
trigger('change');
setTimeout(() => {
const inp = element.querySelector('.ctl-search');
if (inp && document.activeElement !== inp) {
inp.focus();
inp.setSelectionRange(inp.value.length, inp.value.length);
}
}, 60);
}, 250);
});""",
**kwargs,
)
def api_info(self):
return {"type": "string"}
# Client-side row filter for rendered tables: filters rows against the
# .tbl-filter input, no server round trip. Attached via js_on_load so the
# delegated listener survives value updates.
TABLE_FILTER_JS = """
element.addEventListener('input', (e) => {
if (!e.target.classList.contains('tbl-filter')) return;
const q = e.target.value.toLowerCase();
element.querySelectorAll('tbody tr').forEach(tr => {
tr.style.display = tr.textContent.toLowerCase().includes(q) ? '' : 'none';
});
});"""
# ---------------------------------------------------------------------------
# Small HTML helpers
# ---------------------------------------------------------------------------
def cat_chip(category: str) -> str:
return f'{esc(category)}'
def plain_chip(text: str) -> str:
return f'{esc(text)}'
def benchmark_meta_block(benchmark: str) -> str:
"""Description + categories/modality/language + paper/implementation links for
a benchmark, pulled from the dataset's `benchmarks` reference table. That table
only covers a subset of all referenced names, so this degrades gracefully."""
info = BENCH_META.get(benchmark)
if not info:
return '
No reference metadata catalogued for this benchmark yet.
'
chips = "".join(cat_chip(c) for c in info["categories"])
chips += "".join(plain_chip(m) for m in info["modality"])
chips += "".join(plain_chip(l) for l in info["language"])
links = " · ".join(
f'{label}'
for label, url in [("Paper", info["paper_url"]), ("Implementation", info["implementation_url"])]
if url
)
return (
f'
{esc(info["description"])}
'
f'
{chips}
'
+ (f'
{links}
' if links else "")
)
def openness_chip(is_open: bool) -> str:
color, text = ("var(--open)", "open") if is_open else ("var(--closed)", "closed")
return f'{text}'
def source_label(url: str) -> str:
"""The dataset no longer carries a source title/type, just a link — show its
host as a short, still-meaningful label (e.g. "arxiv.org", "huggingface.co")."""
return urlparse(url).netloc or url
def nice_ceil(v: float) -> float:
if v <= 0:
return 1
exp = 10 ** math.floor(math.log10(v))
frac = v / exp
for m in (1, 2, 2.5, 5, 10):
if frac <= m:
return m * exp
return v
def month_ticks(t0: pd.Timestamp, t1: pd.Timestamp) -> list[pd.Timestamp]:
span = max((t1 - t0).days, 1)
step = next((s for s in (1, 2, 3, 6, 12, 24) if span / (30.4 * s) <= 7), 24)
m0 = ((t0.month - 1) // step) * step + 1
d = pd.Timestamp(year=t0.year, month=m0, day=1)
ticks = []
while d <= t1:
if d >= t0:
ticks.append(d)
total = (d.month - 1) + step
d = pd.Timestamp(year=d.year + total // 12, month=total % 12 + 1, day=1)
return ticks
# ---------------------------------------------------------------------------
# Hero header
# ---------------------------------------------------------------------------
def render_hero() -> str:
d0, d1 = MODELS_DF["release_date"].min().date(), MODELS_DF["release_date"].max().date()
n_bench = USAGE_DF["benchmark"].nunique()
tiles = [
("Models", str(len(MODELS_DF)), ""),
("Labs", str(MODELS_DF["lab"].nunique()), ""),
("Distinct benchmarks", str(n_bench), ""),
("Catalogued", f"{len(BENCH_META)} / {n_bench}", "sm"),
("Coverage", f"{d0} → {d1}", "sm"),
]
tiles_html = "".join(
f'
{esc(lab)}
{esc(val)}
'
for lab, val, cls in tiles
)
return (
'
LLM Benchmark Usage Explorer
'
'
Exploring SaylorTwift/llm-benchmark-usage — which benchmarks labs report, '
"who uses them, and how the mix shifts over time. Categories/modality/language come "
'from a reference table that only '
"partially covers all referenced benchmarks — the rest show as "
"uncategorized.
"
f'
{tiles_html}
'
)
# ---------------------------------------------------------------------------
# Tab 0: Latest releases (landing view)
# ---------------------------------------------------------------------------
def rel_date(d: pd.Timestamp) -> str:
days = (pd.Timestamp.now().normalize() - d.normalize()).days
if days <= 0:
return "today"
if days == 1:
return "yesterday"
if days < 7:
return f"{days} days ago"
if days < 60:
return f"{days // 7} week{'s' if days >= 14 else ''} ago"
if days < 365:
return f"{days // 30} months ago"
return f"{days // 365} year{'s' if days >= 730 else ''} ago"
MAX_RELEASE_CHIPS = 16
def latest_releases_view(count) -> str:
recent = MODELS_DF.sort_values("release_date", ascending=False).head(int(count))
cat_rank = {c: i for i, c in enumerate(CATEGORY_ORDER)}
cards = []
for m in recent.itertuples():
sub = USAGE_DF[USAGE_DF["model_id"] == m.model_id]
if sub.empty:
body = '
No benchmark data recorded for this model.
'
else:
# A benchmark can span >1 category now, so collect them all per benchmark
# before picking a single chip color (lowest-ranked category wins).
per_bench = sub.groupby("benchmark")["category"].apply(
lambda s: sorted(set(s), key=lambda c: cat_rank.get(c, 99))
)
per_bench = per_bench.reset_index(name="categories")
per_bench["rank"] = per_bench["categories"].apply(lambda cs: cat_rank.get(cs[0], 99))
per_bench = per_bench.sort_values(["rank", "benchmark"])
shown = per_bench.head(MAX_RELEASE_CHIPS)
chips = "".join(
f''
f'{esc(r.benchmark)}'
for r in shown.itertuples()
)
if len(per_bench) > MAX_RELEASE_CHIPS:
chips += f'+{len(per_bench) - MAX_RELEASE_CHIPS} more'
src_url = sub.iloc[0]["source"]
body = (
f'
"
)
return "".join(cards)
# ---------------------------------------------------------------------------
# Tab 1: Benchmark popularity
# ---------------------------------------------------------------------------
def popularity_view(category_filter, openness_filter, min_models, search) -> str:
df = USAGE_DF
if category_filter and category_filter != "All":
df = df[df["category"] == category_filter]
if openness_filter == "Open only":
df = df[df["is_open"]]
elif openness_filter == "Closed only":
df = df[~df["is_open"]]
# A model can appear several times for one benchmark (multiple sources report
# it), so count on deduplicated (benchmark, model) pairs; categories keep all rows.
uniq = df.drop_duplicates(subset=["benchmark", "model_id"])
agg = (
uniq.groupby("benchmark")
.agg(
models=("model_id", "nunique"),
open_models=("is_open", "sum"),
first_seen=("release_date", "min"),
last_seen=("release_date", "max"),
labs=("lab", "nunique"),
)
.reset_index()
)
categories = df.groupby("benchmark")["category"].agg(lambda s: sorted(set(s))).rename("categories")
agg = agg.merge(categories, on="benchmark")
agg["closed_models"] = agg["models"] - agg["open_models"]
agg = agg[agg["models"] >= int(min_models)]
if search:
agg = agg[agg["benchmark"].str.contains(search, case=False, na=False, regex=False)]
agg = agg.sort_values(["models", "benchmark"], ascending=[False, True])
if agg.empty:
return '
No benchmarks match the current filters.
'
max_models = int(agg["models"].max())
rows = []
for i, r in enumerate(agg.itertuples(), start=1):
bar_pct = r.models / max_models * 100
tot = max(r.models, 1)
o_pct, c_pct = r.open_models / tot * 100, r.closed_models / tot * 100
chips = "".join(cat_chip(c) for c in r.categories)
rows.append(
f"
"
f'
{i}
'
f"
{esc(r.benchmark)}
"
f'
{r.models}
'
f'
{int(r.open_models)} / {int(r.closed_models)}'
f''
f'
'
f'
{r.labs}
'
f"
{chips}
"
f'
{r.first_seen.date()}
'
f'
{r.last_seen.date()}
'
f"
"
)
return (
f'
{len(agg)} benchmarks match · sorted by number of models
'
'
'
"
#
Benchmark
Models
Open / closed
Labs
"
"
Categories
First seen
Last seen
"
f'
{"".join(rows)}
'
)
# ---------------------------------------------------------------------------
# Tab 2: All benchmarks (catalog: usage count + HF dataset + implementation links)
# ---------------------------------------------------------------------------
MIN_CATALOG_MODELS = 3 # drop benchmarks used by 2 or fewer models
CATALOG_STALE_MONTHS = 6 # drop benchmarks with no model release in this window
def catalog_view(category_filter, sort_mode, search) -> str:
df = USAGE_DF
if category_filter and category_filter != "All":
df = df[df["category"] == category_filter]
uniq = df.drop_duplicates(subset=["benchmark", "model_id"])
agg = uniq.groupby("benchmark").agg(
models=("model_id", "nunique"), last_seen=("release_date", "max")
).reset_index()
categories = df.groupby("benchmark")["category"].agg(lambda s: sorted(set(s))).rename("categories")
agg = agg.merge(categories, on="benchmark")
# Drop rarely-used and stale benchmarks so the catalog stays focused on
# benchmarks that are both adopted and still current.
cutoff = pd.Timestamp.now() - pd.DateOffset(months=CATALOG_STALE_MONTHS)
n_before = len(agg)
agg = agg[(agg["models"] > MIN_CATALOG_MODELS - 1) & (agg["last_seen"] >= cutoff)]
n_hidden = n_before - len(agg)
if search:
agg = agg[agg["benchmark"].str.contains(search, case=False, na=False, regex=False)]
if sort_mode == "Category":
cat_rank = {c: i for i, c in enumerate(CATEGORY_ORDER)}
agg["_rank"] = agg["categories"].apply(lambda cs: cat_rank.get(cs[0], 99) if cs else 99)
agg = agg.sort_values(["_rank", "benchmark"])
elif sort_mode == "Name (A-Z)":
agg = agg.sort_values("benchmark")
else: # "Most used"
agg = agg.sort_values(["models", "benchmark"], ascending=[False, True])
hidden_note = (
f" · {n_hidden} niche/stale benchmark{'s' if n_hidden != 1 else ''} hidden "
f"(≤{MIN_CATALOG_MODELS - 1} models or none in the last {CATALOG_STALE_MONTHS} months)"
if n_hidden
else ""
)
if agg.empty:
return f'
No benchmarks match the current filters.{esc(hidden_note)}
'
max_models = int(agg["models"].max())
rows = []
for i, r in enumerate(agg.itertuples(), start=1):
info = BENCH_META.get(r.benchmark)
bar_pct = r.models / max_models * 100
chips = "".join(cat_chip(c) for c in r.categories)
hf_dataset = info.get("hf_dataset") if info else None
hf_url = f"https://huggingface.co/datasets/{hf_dataset}" if hf_dataset else None
hf_cell = f'🤗 {esc(hf_dataset)}' if hf_url else '—'
impl_url = info.get("implementation_url") if info else None
# Skip the implementation link when it's just the same HF dataset already shown.
impl_cell = (
f'{esc(source_label(impl_url))}'
if impl_url and impl_url != hf_url
else '—'
)
rows.append(
f"
"
f'
{i}
'
f"
{esc(r.benchmark)}
"
f'
{r.models}
'
f"
{chips}
"
f"
{hf_cell}
"
f"
{impl_cell}
"
f"
"
)
return (
f'
{len(agg)} benchmarks match · sorted by {esc(sort_mode.lower())}{esc(hidden_note)}
'
'
'
"
#
Benchmark
Models
Categories
HF dataset
Implementation
"
f'
{"".join(rows)}
'
)
# ---------------------------------------------------------------------------
# Tab 3: Benchmark -> Models (summary card, monthly-usage SVG bars, table)
# ---------------------------------------------------------------------------
def _rounded_top_rect(x, y, w, h, r) -> str:
r = min(r, h / 2, w / 2)
return (
f'M {x:.1f} {y + h:.1f} L {x:.1f} {y + r:.1f} Q {x:.1f} {y:.1f} {x + r:.1f} {y:.1f} '
f'L {x + w - r:.1f} {y:.1f} Q {x + w:.1f} {y:.1f} {x + w:.1f} {y + r:.1f} L {x + w:.1f} {y + h:.1f} Z'
)
def svg_benchmark_weekly_usage(df: pd.DataFrame) -> str:
"""Bars of how many model releases reported this benchmark each week, spanning
the whole dataset time range (empty weeks stay visible as gaps)."""
t0 = MODELS_DF["release_date"].min().to_period("W")
t1 = MODELS_DF["release_date"].max().to_period("W")
weeks = pd.period_range(t0, t1, freq="W")
counts = df["release_date"].dt.to_period("W").value_counts().reindex(weeks, fill_value=0)
W, H, ML, MR, MT, MB = 920, 300, 44, 10, 12, 36
plot_h = H - MT - MB
base = MT + plot_h
vmax = int(counts.max())
step = max(1, math.ceil(vmax / 4))
ymax = step * 4
scale = plot_h / ymax
band = (W - ML - MR) / len(weeks)
bw = max(band - 1, 1.0) # 1px surface gap between adjacent bars (weekly bands are narrow)
parts = [
f'")
return "".join(parts)
def benchmark_view(benchmark):
if not benchmark:
return "", "", '
{len(df)} models from '
f"{df['lab'].nunique()} labs report it — "
f"{n_open} open-weight, {n_closed} closed. First seen "
f"{df['release_date'].min().date()}, most recent "
f"{df['release_date'].max().date()}."
f"
{benchmark_meta_block(benchmark)}
"
)
chart = (
f'
How often is “{esc(benchmark)}” used? Model releases reporting it per week
'
+ svg_benchmark_weekly_usage(df)
)
rows = []
for r in df.itertuples():
rows.append(
"
'
)
sections = []
for category in CATEGORY_ORDER:
sub = df[df["category"] == category]
if sub.empty:
continue
chip_parts = []
for r in sub.itertuples():
info = BENCH_META.get(r.benchmark)
if info:
title = f"modality: {', '.join(info['modality'])} · language: {', '.join(info['language'])}"
else:
title = "no reference metadata catalogued yet"
chip_parts.append(
f''
f'{esc(r.benchmark)}'
)
sections.append(
f'
'
f'{esc(category)} ({len(sub)})
{"".join(chip_parts)}
'
)
return summary, "".join(sections)
# ---------------------------------------------------------------------------
# Tab 5: Category evolution (SVG stacked bars + table)
# ---------------------------------------------------------------------------
def svg_category_bars(counts: pd.DataFrame, normalize: bool) -> str:
periods = counts.index.tolist()
plot = counts.copy()
if normalize:
plot = plot.div(plot.sum(axis=1).replace(0, 1), axis=0) * 100
W, H, ML, MR, MT, MB, BW = 920, 430, 52, 10, 10, 36, 24
plot_h = H - MT - MB
base = MT + plot_h
ymax = 100.0 if normalize else float(nice_ceil(plot.sum(axis=1).max()))
scale = plot_h / ymax
band = (W - ML - MR) / max(len(periods), 1)
parts = [
f'")
return "".join(parts)
def category_view(openness_filter, mode):
df = USAGE_DF
if openness_filter == "Open only":
df = df[df["is_open"]]
elif openness_filter == "Closed only":
df = df[~df["is_open"]]
normalize = mode == "Share of period (%)"
df = df.drop_duplicates(subset=["model_id", "benchmark", "category"])
counts = df.groupby(["period", "category"]).size().unstack(fill_value=0)
counts = counts.reindex(columns=[c for c in CATEGORY_ORDER if c in counts.columns], fill_value=0)
counts = counts.reindex(PERIODS, fill_value=0)
legend = '
' + "".join(
f'{esc(c)}' for c in counts.columns
) + "
"
title = "Benchmark category mix over time" + (" (% of period total)" if normalize else " (raw benchmark-use count)")
chart = f'
{esc(title)}
' + legend + svg_category_bars(counts, normalize)
rows = []
for period, row in counts.iterrows():
cells = "".join(f'
{int(v)}
' for v in row)
rows.append(f"
{esc(period)}
{cells}
")
head = "
Period
" + "".join(f"
{esc(c)}
" for c in counts.columns)
table = (
'
'
f'{head}
{"".join(rows)}
'
)
return chart, table
# ---------------------------------------------------------------------------
# UI
# ---------------------------------------------------------------------------
DEFAULT_BENCH = "GPQA-Diamond" if "GPQA-Diamond" in ALL_BENCHMARKS else ALL_BENCHMARKS[0]
with gr.Blocks(title="LLM Benchmark Usage Explorer") as demo:
gr.HTML(render_hero())
with gr.Tab("🆕 Latest releases"):
gr.HTML('
The most recent model releases, the benchmarks they report, and where the numbers come from.