adding pages to display interactive benchmark results
This commit is contained in:
Donato Capitella
@@ -0,0 +1,584 @@
|
||||
<!doctype html>
|
||||
<html lang="en">
|
||||
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width,initial-scale=1" />
|
||||
<title>Strix Halo — Model ↔ Backend Comparator</title>
|
||||
<style>
|
||||
:root {
|
||||
--bg: #0b0c0f;
|
||||
--ink: #e6e7ea;
|
||||
--muted: #a3a8b3;
|
||||
--accent: #6ea8fe;
|
||||
--card: #11141a;
|
||||
--border: #202633;
|
||||
--sticky: #0f1218;
|
||||
--pill: #1b2230;
|
||||
--roc: #b19cff33;
|
||||
--roc-text: #e9e4ff;
|
||||
--warn: #ffcc66;
|
||||
--bad: #ff6b6b;
|
||||
--model-w: 300px;
|
||||
--winner-w: 220px;
|
||||
/* wider winner column */
|
||||
}
|
||||
|
||||
* {
|
||||
box-sizing: border-box
|
||||
}
|
||||
|
||||
body {
|
||||
margin: 0;
|
||||
background: var(--bg);
|
||||
color: var(--ink);
|
||||
font: 15px/1.5 system-ui, -apple-system, Segoe UI, Roboto, Inter
|
||||
}
|
||||
|
||||
a {
|
||||
color: var(--accent);
|
||||
text-decoration: none
|
||||
}
|
||||
|
||||
a:hover {
|
||||
text-decoration: underline
|
||||
}
|
||||
|
||||
header {
|
||||
padding: 14px 20px 0
|
||||
}
|
||||
|
||||
header h1 {
|
||||
margin: 0 0 6px;
|
||||
font-size: 22px
|
||||
}
|
||||
|
||||
header p {
|
||||
margin: 4px 0 12px;
|
||||
color: var(--muted);
|
||||
font-size: 13px
|
||||
}
|
||||
|
||||
.controls {
|
||||
display: grid;
|
||||
grid-template-columns: 1fr 160px 260px 180px;
|
||||
gap: 10px;
|
||||
align-items: end;
|
||||
padding: 10px 20px
|
||||
}
|
||||
|
||||
label {
|
||||
display: block;
|
||||
font-size: 12px;
|
||||
color: var(--muted);
|
||||
margin: 0 0 6px
|
||||
}
|
||||
|
||||
input[type="text"],
|
||||
select,
|
||||
input[type="number"] {
|
||||
width: 100%;
|
||||
background: #0d1117;
|
||||
color: var(--ink);
|
||||
border: 1px solid var(--border);
|
||||
border-radius: 10px;
|
||||
padding: 8px 10px;
|
||||
outline: none
|
||||
}
|
||||
|
||||
.slider {
|
||||
display: grid;
|
||||
grid-template-columns: 1fr 90px;
|
||||
gap: 8px;
|
||||
align-items: center
|
||||
}
|
||||
|
||||
input[type="range"] {
|
||||
width: 100%
|
||||
}
|
||||
|
||||
.ticks {
|
||||
display: flex;
|
||||
justify-content: space-between;
|
||||
color: var(--muted);
|
||||
font-size: 11px;
|
||||
margin-top: 2px
|
||||
}
|
||||
|
||||
.cols {
|
||||
display: flex;
|
||||
gap: 8px;
|
||||
flex-wrap: wrap;
|
||||
padding: 0 20px 8px
|
||||
}
|
||||
|
||||
.colbox {
|
||||
border: 1px solid var(--border);
|
||||
border-radius: 12px;
|
||||
padding: 8px 10px;
|
||||
min-width: 220px
|
||||
}
|
||||
|
||||
.colbox label {
|
||||
display: flex;
|
||||
gap: 8px;
|
||||
align-items: center;
|
||||
margin: 0
|
||||
}
|
||||
|
||||
.colbox input {
|
||||
transform: translateY(1px)
|
||||
}
|
||||
|
||||
.badge {
|
||||
display: inline-block;
|
||||
padding: 2px 8px;
|
||||
border-radius: 999px;
|
||||
background: var(--pill);
|
||||
font-size: 11px;
|
||||
margin-left: 6px
|
||||
}
|
||||
|
||||
.roc {
|
||||
background: var(--roc);
|
||||
color: var(--roc-text);
|
||||
border: 1px solid #b19cff55
|
||||
}
|
||||
|
||||
.meta {
|
||||
padding: 0 20px 14px;
|
||||
color: var(--muted);
|
||||
font-size: 12px
|
||||
}
|
||||
|
||||
.section {
|
||||
padding: 6px 20px 0
|
||||
}
|
||||
|
||||
.section h2 {
|
||||
margin: 6px 0 8px;
|
||||
font-size: 16px;
|
||||
color: #cfd3da;
|
||||
font-weight: 600
|
||||
}
|
||||
|
||||
.tablewrap {
|
||||
margin: 0 0 16px;
|
||||
border-top: 1px solid var(--border)
|
||||
}
|
||||
|
||||
.scroller {
|
||||
overflow-x: auto;
|
||||
overflow-y: hidden
|
||||
}
|
||||
|
||||
.scroller::-webkit-scrollbar {
|
||||
height: 10px
|
||||
}
|
||||
|
||||
.scroller::-webkit-scrollbar-thumb {
|
||||
background: #2a3140;
|
||||
border-radius: 8px
|
||||
}
|
||||
|
||||
table {
|
||||
width: max-content;
|
||||
min-width: 100vw;
|
||||
border-collapse: separate;
|
||||
border-spacing: 0
|
||||
}
|
||||
|
||||
thead th {
|
||||
position: sticky;
|
||||
top: 0;
|
||||
background: var(--card);
|
||||
z-index: 2
|
||||
}
|
||||
|
||||
th,
|
||||
td {
|
||||
padding: 10px 12px;
|
||||
border-bottom: 1px solid var(--border);
|
||||
vertical-align: top
|
||||
}
|
||||
|
||||
th {
|
||||
white-space: nowrap
|
||||
}
|
||||
|
||||
td {
|
||||
white-space: normal
|
||||
}
|
||||
|
||||
/* sticky cols */
|
||||
.col-model {
|
||||
position: sticky;
|
||||
left: 0;
|
||||
background: var(--sticky);
|
||||
z-index: 3;
|
||||
max-width: var(--model-w);
|
||||
width: var(--model-w)
|
||||
}
|
||||
|
||||
.col-winner {
|
||||
position: sticky;
|
||||
left: calc(var(--model-w));
|
||||
background: var(--sticky);
|
||||
z-index: 3;
|
||||
max-width: var(--winner-w);
|
||||
width: var(--winner-w)
|
||||
}
|
||||
|
||||
.model-title {
|
||||
font-weight: 600;
|
||||
word-wrap: break-word;
|
||||
overflow-wrap: anywhere
|
||||
}
|
||||
|
||||
.model-meta {
|
||||
color: var(--muted);
|
||||
font-size: 12px;
|
||||
margin-top: 2px
|
||||
}
|
||||
|
||||
/* centered value cells */
|
||||
.cell {
|
||||
min-width: 120px;
|
||||
text-align: center
|
||||
}
|
||||
|
||||
.tps {
|
||||
display: block;
|
||||
text-align: center
|
||||
}
|
||||
|
||||
.std {
|
||||
color: var(--muted);
|
||||
font-size: 12px;
|
||||
text-align: center
|
||||
}
|
||||
|
||||
.warn {
|
||||
color: var(--warn)
|
||||
}
|
||||
|
||||
.err {
|
||||
color: var(--bad)
|
||||
}
|
||||
|
||||
.winner-pill {
|
||||
display: inline-block;
|
||||
background: var(--pill);
|
||||
border-radius: 999px;
|
||||
padding: 2px 8px;
|
||||
margin: 4px 4px 4px 0;
|
||||
font-size: 12px
|
||||
}
|
||||
|
||||
/* a bit more vertical margin */
|
||||
.winner-wrap {
|
||||
display: flex;
|
||||
flex-wrap: wrap
|
||||
}
|
||||
|
||||
.env-sub {
|
||||
color: var(--muted);
|
||||
font-size: 12px
|
||||
}
|
||||
</style>
|
||||
</head>
|
||||
|
||||
<body>
|
||||
<header>
|
||||
<h1>Strix Halo — llama.cpp Backend Comparator</h1>
|
||||
<p class="muted">
|
||||
Compare model throughput across backends (pp512 & tg128).
|
||||
Repo: <a href="https://github.com/kyuz0/amd-strix-halo-toolboxes" target="_blank"
|
||||
rel="noreferrer">kyuz0/amd-strix-halo-toolboxes</a>
|
||||
</p>
|
||||
<p id="meta-line">Loading meta…</p>
|
||||
</header>
|
||||
|
||||
|
||||
<div class="controls">
|
||||
<div>
|
||||
<label for="search">Search model</label>
|
||||
<input id="search" type="text" placeholder="e.g. llama, qwen, 30B, Q8_K…">
|
||||
</div>
|
||||
<div>
|
||||
<label for="quant">Quant</label>
|
||||
<select id="quant">
|
||||
<option value="">Any</option>
|
||||
</select>
|
||||
</div>
|
||||
<div>
|
||||
<label>Model size (B from name)</label>
|
||||
<div class="slider">
|
||||
<input id="size" type="range" step="0.01" min="0" max="100" value="0" list="tickmarks">
|
||||
<input id="sizeOut" type="number" step="0.01">
|
||||
</div>
|
||||
<div class="ticks" id="tickLabels"></div>
|
||||
<datalist id="tickmarks"></datalist>
|
||||
</div>
|
||||
<div>
|
||||
<label for="faMode">Flash Attention</label>
|
||||
<select id="faMode">
|
||||
<option value="off">FA off</option>
|
||||
<option value="on">FA on</option> <!-- default ON -->
|
||||
<option value="both" selected>Both</option>
|
||||
</select>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="cols" id="columns"></div>
|
||||
<div class="meta">Winner = every selected backend within the best’s uncertainty range, combining ± errors from both results.</div>
|
||||
|
||||
<div class="section">
|
||||
<h2>Prompt Processing (pp512) — tokens/second</h2>
|
||||
</div>
|
||||
<section class="tablewrap">
|
||||
<div class="scroller">
|
||||
<table id="tbl-pp">
|
||||
<thead>
|
||||
<tr id="pp-h"></tr>
|
||||
</thead>
|
||||
<tbody></tbody>
|
||||
</table>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<div class="section">
|
||||
<h2>Text Generation (tg128) — tokens/second</h2>
|
||||
</div>
|
||||
<section class="tablewrap">
|
||||
<div class="scroller">
|
||||
<table id="tbl-tg">
|
||||
<thead>
|
||||
<tr id="tg-h"></tr>
|
||||
</thead>
|
||||
<tbody></tbody>
|
||||
</table>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<script>
|
||||
(async function () {
|
||||
// dynamic tie threshold: within K sigmas of best, with a small absolute floor
|
||||
const K_SIGMA = 1.0; // 1σ; set 1.96 for ~95% CI
|
||||
const MIN_TOL = 0.25; // tokens/s floor if stds are tiny/missing
|
||||
|
||||
const res = await fetch('results.json'); const data = await res.json();
|
||||
const allRuns = data.runs || [];
|
||||
const perfRuns = allRuns.filter(r => r.test === 'pp512' || r.test === 'tg128');
|
||||
|
||||
const metaLine = document.getElementById('meta-line');
|
||||
const search = document.getElementById('search');
|
||||
const quantSel = document.getElementById('quant');
|
||||
const size = document.getElementById('size');
|
||||
const sizeOut = document.getElementById('sizeOut');
|
||||
const ticks = document.getElementById('tickmarks');
|
||||
const tickLabels = document.getElementById('tickLabels');
|
||||
const colPicker = document.getElementById('columns');
|
||||
const faMode = document.getElementById('faMode');
|
||||
|
||||
metaLine.textContent = `${data.meta?.os_kernel || ''} — llama.cpp ${(data.meta?.llamacpp_builds || []).map(b => b.hash).join(', ') || 'unknown'} — generated ${data.meta?.generated_at || ''}`;
|
||||
|
||||
const models = [...new Set(perfRuns.map(r => r.model_clean || r.model))].sort((a, b) => a.localeCompare(b));
|
||||
const byModel = {};
|
||||
for (const name of models) { byModel[name] = { pp512: {}, tg128: {}, quant: null, sizeB: null }; }
|
||||
for (const r of perfRuns) {
|
||||
const name = r.model_clean || r.model;
|
||||
const cfg = `${r.env}|fa=${r.fa ? 1 : 0}`;
|
||||
byModel[name][r.test][cfg] = { mean: r.tps_mean, std: r.tps_std, err: r.error, et: r.error_type };
|
||||
if (!byModel[name].quant && r.quant) byModel[name].quant = r.quant;
|
||||
if (byModel[name].sizeB == null) byModel[name].sizeB = (r.name_params_b ?? r.params_b);
|
||||
}
|
||||
// include error-only logs so we can show ⚠️
|
||||
for (const r of allRuns) {
|
||||
if (r.test || !r.error) continue;
|
||||
const name = r.model_clean || r.model; if (!byModel[name]) continue;
|
||||
const cfg = `${r.env}|fa=${r.fa ? 1 : 0}`;
|
||||
for (const t of ['pp512', 'tg128']) {
|
||||
if (!byModel[name][t][cfg]) byModel[name][t][cfg] = { mean: null, std: null, err: true, et: r.error_type || 'error' };
|
||||
}
|
||||
}
|
||||
|
||||
const envs = [...new Set(allRuns.map(r => r.env))].sort();
|
||||
function envBox(env) {
|
||||
const roc = env.includes('rocwmma');
|
||||
const id = `env_${env.replace(/[^a-z0-9_-]/gi, '_')}`;
|
||||
return `
|
||||
<div class="colbox">
|
||||
<label for="${id}">
|
||||
<input id="${id}" type="checkbox" data-env="${env}" ${/vulkan_amdvlk|vulkan_radv|rocm6_4_2/.test(env) ? 'checked' : ''}>
|
||||
<span><strong>${env}</strong>${roc ? '<span class="badge roc">rocWMMA</span>' : ''}</span>
|
||||
</label>
|
||||
</div>`;
|
||||
}
|
||||
colPicker.innerHTML = envs.map(envBox).join('');
|
||||
const selectedEnvs = () => [...colPicker.querySelectorAll('input[type="checkbox"]')].filter(i => i.checked).map(i => i.dataset.env);
|
||||
|
||||
const quants = [...new Set(Object.values(byModel).map(m => (m.quant || 'Unknown').toUpperCase()))].sort();
|
||||
quants.forEach(q => { const o = document.createElement('option'); o.value = (q === 'UNKNOWN' ? '' : q); o.textContent = q; quantSel.appendChild(o); });
|
||||
|
||||
const sizes = Object.values(byModel).map(m => m.sizeB).filter(v => typeof v === 'number').sort((a, b) => a - b);
|
||||
const minB = sizes[0] ?? 0, maxB = sizes[sizes.length - 1] ?? 100;
|
||||
size.min = String(minB); size.max = String(maxB); size.step = "0.01"; size.value = String(minB); sizeOut.value = String(minB);
|
||||
const marks = 6; ticks.innerHTML = ''; tickLabels.innerHTML = '';
|
||||
for (let i = 0; i < marks; i++) {
|
||||
const v = (minB + (i * (maxB - minB) / (marks - 1)));
|
||||
const opt = document.createElement('option'); opt.value = v.toFixed(2); ticks.appendChild(opt);
|
||||
const lab = document.createElement('span'); lab.textContent = `${v.toFixed(0)}B`; tickLabels.appendChild(lab);
|
||||
}
|
||||
|
||||
const TOL = 1.00;
|
||||
|
||||
function matchesFilters(name) {
|
||||
const q = (search.value || '').toLowerCase();
|
||||
if (q && !name.toLowerCase().includes(q)) return false;
|
||||
const info = byModel[name];
|
||||
if (quantSel.value && (info.quant || '').toUpperCase() !== quantSel.value.toUpperCase()) return false;
|
||||
if (info.sizeB != null && info.sizeB < parseFloat(size.value) - 1e-9) return false;
|
||||
return true;
|
||||
}
|
||||
|
||||
// remove quantization tokens from the title (we still show quant below)
|
||||
function cleanTitle(name, quant) {
|
||||
const qTokens = ['BF16', 'F16', 'F32', 'FP16', 'MXFP2', 'MXFP4', 'MXFP8'];
|
||||
if (quant) qTokens.push(quant.toUpperCase());
|
||||
let t = name;
|
||||
// common GGUF quant forms like Q4_K_M, Q8_0, Q6_K, Q3_K_S, etc.
|
||||
t = t.replace(/[-_](Q\d+(?:_[A-Z0-9]+)*)/gi, '');
|
||||
// BF16/F16/F32/MXFPx anywhere
|
||||
t = t.replace(new RegExp(`[-_]?(${qTokens.join('|')})\\b`, 'gi'), '');
|
||||
// drop "-UD"
|
||||
t = t.replace(/-UD\b/gi, '');
|
||||
// collapse leftover double separators
|
||||
t = t.replace(/[-_]{2,}/g, '-')
|
||||
.replace(/\s{2,}/g, ' ')
|
||||
.replace(/[-_]+$/, '')
|
||||
.trim();
|
||||
return t;
|
||||
}
|
||||
|
||||
function buildColDefs(envSel, mode) {
|
||||
const cols = [];
|
||||
for (const e of envSel) {
|
||||
if (mode === 'both') {
|
||||
cols.push({ env: e, key: `${e}|fa=0`, label: `${e}<div class="env-sub">FA off</div>` });
|
||||
cols.push({ env: e, key: `${e}|fa=1`, label: `${e}<div class="env-sub">FA on</div>` });
|
||||
} else {
|
||||
const k = `${e}|fa=${mode === 'on' ? 1 : 0}`;
|
||||
cols.push({ env: e, key: k, label: `${e}<div class="env-sub">FA ${mode === 'on' ? 'on' : 'off'}</div>` });
|
||||
}
|
||||
}
|
||||
return cols;
|
||||
}
|
||||
|
||||
function headerHTML(test, colDefs) {
|
||||
return `<th class="col-model">Model</th><th class="col-winner">Winner (${test})</th>` +
|
||||
colDefs.map(c => `<th>${c.label}</th>`).join('');
|
||||
}
|
||||
|
||||
function cellHTML(name, test, key) {
|
||||
const v = byModel[name][test][key];
|
||||
if (!v) return '<div class="cell"><span class="muted">—</span></div>';
|
||||
if (v.err || v.mean == null) return `<div class="cell"><span class="warn">⚠ ${v.et || 'error'}</span></div>`;
|
||||
return `<div class="cell"><span class="tps">${v.mean.toFixed(2)}</span><span class="std">± ${v.std?.toFixed(2) ?? '—'}</span></div>`;
|
||||
}
|
||||
|
||||
function winnersHTML(name, test, colDefs) {
|
||||
// Collect (env, fa, mean, std) for visible columns
|
||||
const vals = [];
|
||||
for (const c of colDefs) {
|
||||
const v = byModel[name][test][c.key];
|
||||
if (v && !v.err && typeof v.mean === 'number') {
|
||||
vals.push({
|
||||
env: c.env,
|
||||
fa: c.key.endsWith('|fa=1') ? 'on' : 'off',
|
||||
mean: v.mean,
|
||||
std: (typeof v.std === 'number' ? v.std : 0)
|
||||
});
|
||||
}
|
||||
}
|
||||
if (!vals.length) return '<span class="muted">—</span>';
|
||||
|
||||
// Best backend by mean
|
||||
let best = vals[0];
|
||||
for (const v of vals) if (v.mean > best.mean) best = v;
|
||||
|
||||
// Tie rule: within max(MIN_TOL, K*sqrt(std_best^2 + std_i^2))
|
||||
const winners = [];
|
||||
for (const v of vals) {
|
||||
const pooled = Math.sqrt((best.std || 0) ** 2 + (v.std || 0) ** 2);
|
||||
const tol = Math.max(MIN_TOL, K_SIGMA * pooled);
|
||||
if ((best.mean - v.mean) <= tol) winners.push(v);
|
||||
}
|
||||
|
||||
// Render pills. If FA mode = both, show FA state on the pill.
|
||||
if (faMode.value !== 'both') {
|
||||
const envs = [...new Set(winners.map(w => w.env))];
|
||||
return `<div class="winner-wrap">${envs.map(env =>
|
||||
`<span class="winner-pill">🏆 ${env}</span>`).join('')}</div>`;
|
||||
} else {
|
||||
// group by env and list FA(s) that tied
|
||||
const byEnv = {};
|
||||
for (const w of winners) (byEnv[w.env] ||= new Set()).add(w.fa);
|
||||
return `<div class="winner-wrap">${Object.entries(byEnv).map(([env, fas]) => {
|
||||
const list = [...fas].sort().map(x => `FA ${x}`).join(' & ');
|
||||
return `<span class="winner-pill">🏆 ${env} (${list})</span>`;
|
||||
}).join('')
|
||||
}</div>`;
|
||||
}
|
||||
}
|
||||
|
||||
function renderTable(prefix, test) {
|
||||
const envSel = selectedEnvs(); const mode = faMode.value;
|
||||
const colDefs = buildColDefs(envSel, mode);
|
||||
|
||||
const h = document.getElementById(`${prefix}-h`);
|
||||
h.innerHTML = headerHTML(test, colDefs);
|
||||
|
||||
const tbody = document.querySelector(`#tbl-${prefix} tbody`);
|
||||
tbody.innerHTML = '';
|
||||
|
||||
for (const name of models.filter(matchesFilters)) {
|
||||
const tr = document.createElement('tr');
|
||||
|
||||
const tdM = document.createElement('td');
|
||||
tdM.className = 'col-model';
|
||||
const info = byModel[name];
|
||||
tdM.innerHTML = `<div class="model-title">${cleanTitle(name, info.quant)}</div><div class="model-meta">${info.quant || '—'} · ${(info.sizeB ?? '—')}B</div>`;
|
||||
tr.appendChild(tdM);
|
||||
|
||||
const tdW = document.createElement('td');
|
||||
tdW.className = 'col-winner';
|
||||
tdW.innerHTML = winnersHTML(name, test, colDefs);
|
||||
tr.appendChild(tdW);
|
||||
|
||||
for (const c of colDefs) {
|
||||
const td = document.createElement('td');
|
||||
td.innerHTML = cellHTML(name, test, c.key);
|
||||
tr.appendChild(td);
|
||||
}
|
||||
tbody.appendChild(tr);
|
||||
}
|
||||
}
|
||||
|
||||
function render() {
|
||||
renderTable('pp', 'pp512');
|
||||
renderTable('tg', 'tg128');
|
||||
}
|
||||
|
||||
// events
|
||||
[search, quantSel, faMode].forEach(el => el.addEventListener('input', render));
|
||||
colPicker.addEventListener('change', render);
|
||||
size.addEventListener('input', e => { sizeOut.value = Number(e.target.value).toFixed(2); render(); });
|
||||
sizeOut.addEventListener('input', e => { size.value = e.target.value; render(); });
|
||||
|
||||
render();
|
||||
})();
|
||||
</script>
|
||||
</body>
|
||||
|
||||
</html>
|
||||
Reference in New Issue
Block a user