Donato Capitella
584 lines
20 KiB
HTML
584 lines
20 KiB
HTML
<!doctype html>
|
||
<html lang="en">
|
||
|
||
<head>
|
||
<meta charset="utf-8" />
|
||
<meta name="viewport" content="width=device-width,initial-scale=1" />
|
||
<title>Strix Halo — Model ↔ Backend Comparator</title>
|
||
<style>
|
||
:root {
|
||
--bg: #0b0c0f;
|
||
--ink: #e6e7ea;
|
||
--muted: #a3a8b3;
|
||
--accent: #6ea8fe;
|
||
--card: #11141a;
|
||
--border: #202633;
|
||
--sticky: #0f1218;
|
||
--pill: #1b2230;
|
||
--roc: #b19cff33;
|
||
--roc-text: #e9e4ff;
|
||
--warn: #ffcc66;
|
||
--bad: #ff6b6b;
|
||
--model-w: 300px;
|
||
--winner-w: 220px;
|
||
/* wider winner column */
|
||
}
|
||
|
||
* {
|
||
box-sizing: border-box
|
||
}
|
||
|
||
body {
|
||
margin: 0;
|
||
background: var(--bg);
|
||
color: var(--ink);
|
||
font: 15px/1.5 system-ui, -apple-system, Segoe UI, Roboto, Inter
|
||
}
|
||
|
||
a {
|
||
color: var(--accent);
|
||
text-decoration: none
|
||
}
|
||
|
||
a:hover {
|
||
text-decoration: underline
|
||
}
|
||
|
||
header {
|
||
padding: 14px 20px 0
|
||
}
|
||
|
||
header h1 {
|
||
margin: 0 0 6px;
|
||
font-size: 22px
|
||
}
|
||
|
||
header p {
|
||
margin: 4px 0 12px;
|
||
color: var(--muted);
|
||
font-size: 13px
|
||
}
|
||
|
||
.controls {
|
||
display: grid;
|
||
grid-template-columns: 1fr 160px 260px 180px;
|
||
gap: 10px;
|
||
align-items: end;
|
||
padding: 10px 20px
|
||
}
|
||
|
||
label {
|
||
display: block;
|
||
font-size: 12px;
|
||
color: var(--muted);
|
||
margin: 0 0 6px
|
||
}
|
||
|
||
input[type="text"],
|
||
select,
|
||
input[type="number"] {
|
||
width: 100%;
|
||
background: #0d1117;
|
||
color: var(--ink);
|
||
border: 1px solid var(--border);
|
||
border-radius: 10px;
|
||
padding: 8px 10px;
|
||
outline: none
|
||
}
|
||
|
||
.slider {
|
||
display: grid;
|
||
grid-template-columns: 1fr 90px;
|
||
gap: 8px;
|
||
align-items: center
|
||
}
|
||
|
||
input[type="range"] {
|
||
width: 100%
|
||
}
|
||
|
||
.ticks {
|
||
display: flex;
|
||
justify-content: space-between;
|
||
color: var(--muted);
|
||
font-size: 11px;
|
||
margin-top: 2px
|
||
}
|
||
|
||
.cols {
|
||
display: flex;
|
||
gap: 8px;
|
||
flex-wrap: wrap;
|
||
padding: 0 20px 8px
|
||
}
|
||
|
||
.colbox {
|
||
border: 1px solid var(--border);
|
||
border-radius: 12px;
|
||
padding: 8px 10px;
|
||
min-width: 220px
|
||
}
|
||
|
||
.colbox label {
|
||
display: flex;
|
||
gap: 8px;
|
||
align-items: center;
|
||
margin: 0
|
||
}
|
||
|
||
.colbox input {
|
||
transform: translateY(1px)
|
||
}
|
||
|
||
.badge {
|
||
display: inline-block;
|
||
padding: 2px 8px;
|
||
border-radius: 999px;
|
||
background: var(--pill);
|
||
font-size: 11px;
|
||
margin-left: 6px
|
||
}
|
||
|
||
.roc {
|
||
background: var(--roc);
|
||
color: var(--roc-text);
|
||
border: 1px solid #b19cff55
|
||
}
|
||
|
||
.meta {
|
||
padding: 0 20px 14px;
|
||
color: var(--muted);
|
||
font-size: 12px
|
||
}
|
||
|
||
.section {
|
||
padding: 6px 20px 0
|
||
}
|
||
|
||
.section h2 {
|
||
margin: 6px 0 8px;
|
||
font-size: 16px;
|
||
color: #cfd3da;
|
||
font-weight: 600
|
||
}
|
||
|
||
.tablewrap {
|
||
margin: 0 0 16px;
|
||
border-top: 1px solid var(--border)
|
||
}
|
||
|
||
.scroller {
|
||
overflow-x: auto;
|
||
overflow-y: hidden
|
||
}
|
||
|
||
.scroller::-webkit-scrollbar {
|
||
height: 10px
|
||
}
|
||
|
||
.scroller::-webkit-scrollbar-thumb {
|
||
background: #2a3140;
|
||
border-radius: 8px
|
||
}
|
||
|
||
table {
|
||
width: max-content;
|
||
min-width: 100vw;
|
||
border-collapse: separate;
|
||
border-spacing: 0
|
||
}
|
||
|
||
thead th {
|
||
position: sticky;
|
||
top: 0;
|
||
background: var(--card);
|
||
z-index: 2
|
||
}
|
||
|
||
th,
|
||
td {
|
||
padding: 10px 12px;
|
||
border-bottom: 1px solid var(--border);
|
||
vertical-align: top
|
||
}
|
||
|
||
th {
|
||
white-space: nowrap
|
||
}
|
||
|
||
td {
|
||
white-space: normal
|
||
}
|
||
|
||
/* sticky cols */
|
||
.col-model {
|
||
position: sticky;
|
||
left: 0;
|
||
background: var(--sticky);
|
||
z-index: 3;
|
||
max-width: var(--model-w);
|
||
width: var(--model-w)
|
||
}
|
||
|
||
.col-winner {
|
||
position: sticky;
|
||
left: calc(var(--model-w));
|
||
background: var(--sticky);
|
||
z-index: 3;
|
||
max-width: var(--winner-w);
|
||
width: var(--winner-w)
|
||
}
|
||
|
||
.model-title {
|
||
font-weight: 600;
|
||
word-wrap: break-word;
|
||
overflow-wrap: anywhere
|
||
}
|
||
|
||
.model-meta {
|
||
color: var(--muted);
|
||
font-size: 12px;
|
||
margin-top: 2px
|
||
}
|
||
|
||
/* centered value cells */
|
||
.cell {
|
||
min-width: 120px;
|
||
text-align: center
|
||
}
|
||
|
||
.tps {
|
||
display: block;
|
||
text-align: center
|
||
}
|
||
|
||
.std {
|
||
color: var(--muted);
|
||
font-size: 12px;
|
||
text-align: center
|
||
}
|
||
|
||
.warn {
|
||
color: var(--warn)
|
||
}
|
||
|
||
.err {
|
||
color: var(--bad)
|
||
}
|
||
|
||
.winner-pill {
|
||
display: inline-block;
|
||
background: var(--pill);
|
||
border-radius: 999px;
|
||
padding: 2px 8px;
|
||
margin: 4px 4px 4px 0;
|
||
font-size: 12px
|
||
}
|
||
|
||
/* a bit more vertical margin */
|
||
.winner-wrap {
|
||
display: flex;
|
||
flex-wrap: wrap
|
||
}
|
||
|
||
.env-sub {
|
||
color: var(--muted);
|
||
font-size: 12px
|
||
}
|
||
</style>
|
||
</head>
|
||
|
||
<body>
|
||
<header>
|
||
<h1>Strix Halo — llama.cpp Backend Comparator</h1>
|
||
<p class="muted">
|
||
Compare model throughput across backends (pp512 & tg128).
|
||
Repo: <a href="https://github.com/kyuz0/amd-strix-halo-toolboxes" target="_blank"
|
||
rel="noreferrer">kyuz0/amd-strix-halo-toolboxes</a>
|
||
</p>
|
||
<p id="meta-line">Loading meta…</p>
|
||
</header>
|
||
|
||
|
||
<div class="controls">
|
||
<div>
|
||
<label for="search">Search model</label>
|
||
<input id="search" type="text" placeholder="e.g. llama, qwen, 30B, Q8_K…">
|
||
</div>
|
||
<div>
|
||
<label for="quant">Quant</label>
|
||
<select id="quant">
|
||
<option value="">Any</option>
|
||
</select>
|
||
</div>
|
||
<div>
|
||
<label>Model size (B from name)</label>
|
||
<div class="slider">
|
||
<input id="size" type="range" step="0.01" min="0" max="100" value="0" list="tickmarks">
|
||
<input id="sizeOut" type="number" step="0.01">
|
||
</div>
|
||
<div class="ticks" id="tickLabels"></div>
|
||
<datalist id="tickmarks"></datalist>
|
||
</div>
|
||
<div>
|
||
<label for="faMode">Flash Attention</label>
|
||
<select id="faMode">
|
||
<option value="off">FA off</option>
|
||
<option value="on">FA on</option> <!-- default ON -->
|
||
<option value="both" selected>Both</option>
|
||
</select>
|
||
</div>
|
||
</div>
|
||
|
||
<div class="cols" id="columns"></div>
|
||
<div class="meta">Winner = every selected backend within the best’s uncertainty range, combining ± errors from both results.</div>
|
||
|
||
<div class="section">
|
||
<h2>Prompt Processing (pp512) — tokens/second</h2>
|
||
</div>
|
||
<section class="tablewrap">
|
||
<div class="scroller">
|
||
<table id="tbl-pp">
|
||
<thead>
|
||
<tr id="pp-h"></tr>
|
||
</thead>
|
||
<tbody></tbody>
|
||
</table>
|
||
</div>
|
||
</section>
|
||
|
||
<div class="section">
|
||
<h2>Text Generation (tg128) — tokens/second</h2>
|
||
</div>
|
||
<section class="tablewrap">
|
||
<div class="scroller">
|
||
<table id="tbl-tg">
|
||
<thead>
|
||
<tr id="tg-h"></tr>
|
||
</thead>
|
||
<tbody></tbody>
|
||
</table>
|
||
</div>
|
||
</section>
|
||
|
||
<script>
|
||
(async function () {
|
||
// dynamic tie threshold: within K sigmas of best, with a small absolute floor
|
||
const K_SIGMA = 1.0; // 1σ; set 1.96 for ~95% CI
|
||
const MIN_TOL = 0.25; // tokens/s floor if stds are tiny/missing
|
||
|
||
const res = await fetch('results.json'); const data = await res.json();
|
||
const allRuns = data.runs || [];
|
||
const perfRuns = allRuns.filter(r => r.test === 'pp512' || r.test === 'tg128');
|
||
|
||
const metaLine = document.getElementById('meta-line');
|
||
const search = document.getElementById('search');
|
||
const quantSel = document.getElementById('quant');
|
||
const size = document.getElementById('size');
|
||
const sizeOut = document.getElementById('sizeOut');
|
||
const ticks = document.getElementById('tickmarks');
|
||
const tickLabels = document.getElementById('tickLabels');
|
||
const colPicker = document.getElementById('columns');
|
||
const faMode = document.getElementById('faMode');
|
||
|
||
metaLine.textContent = `${data.meta?.os_kernel || ''} — llama.cpp ${(data.meta?.llamacpp_builds || []).map(b => b.hash).join(', ') || 'unknown'} — generated ${data.meta?.generated_at || ''}`;
|
||
|
||
const models = [...new Set(perfRuns.map(r => r.model_clean || r.model))].sort((a, b) => a.localeCompare(b));
|
||
const byModel = {};
|
||
for (const name of models) { byModel[name] = { pp512: {}, tg128: {}, quant: null, sizeB: null }; }
|
||
for (const r of perfRuns) {
|
||
const name = r.model_clean || r.model;
|
||
const cfg = `${r.env}|fa=${r.fa ? 1 : 0}`;
|
||
byModel[name][r.test][cfg] = { mean: r.tps_mean, std: r.tps_std, err: r.error, et: r.error_type };
|
||
if (!byModel[name].quant && r.quant) byModel[name].quant = r.quant;
|
||
if (byModel[name].sizeB == null) byModel[name].sizeB = (r.name_params_b ?? r.params_b);
|
||
}
|
||
// include error-only logs so we can show ⚠️
|
||
for (const r of allRuns) {
|
||
if (r.test || !r.error) continue;
|
||
const name = r.model_clean || r.model; if (!byModel[name]) continue;
|
||
const cfg = `${r.env}|fa=${r.fa ? 1 : 0}`;
|
||
for (const t of ['pp512', 'tg128']) {
|
||
if (!byModel[name][t][cfg]) byModel[name][t][cfg] = { mean: null, std: null, err: true, et: r.error_type || 'error' };
|
||
}
|
||
}
|
||
|
||
const envs = [...new Set(allRuns.map(r => r.env))].sort();
|
||
function envBox(env) {
|
||
const roc = env.includes('rocwmma');
|
||
const id = `env_${env.replace(/[^a-z0-9_-]/gi, '_')}`;
|
||
return `
|
||
<div class="colbox">
|
||
<label for="${id}">
|
||
<input id="${id}" type="checkbox" data-env="${env}" ${/vulkan_amdvlk|vulkan_radv|rocm6_4_2/.test(env) ? 'checked' : ''}>
|
||
<span><strong>${env}</strong>${roc ? '<span class="badge roc">rocWMMA</span>' : ''}</span>
|
||
</label>
|
||
</div>`;
|
||
}
|
||
colPicker.innerHTML = envs.map(envBox).join('');
|
||
const selectedEnvs = () => [...colPicker.querySelectorAll('input[type="checkbox"]')].filter(i => i.checked).map(i => i.dataset.env);
|
||
|
||
const quants = [...new Set(Object.values(byModel).map(m => (m.quant || 'Unknown').toUpperCase()))].sort();
|
||
quants.forEach(q => { const o = document.createElement('option'); o.value = (q === 'UNKNOWN' ? '' : q); o.textContent = q; quantSel.appendChild(o); });
|
||
|
||
const sizes = Object.values(byModel).map(m => m.sizeB).filter(v => typeof v === 'number').sort((a, b) => a - b);
|
||
const minB = sizes[0] ?? 0, maxB = sizes[sizes.length - 1] ?? 100;
|
||
size.min = String(minB); size.max = String(maxB); size.step = "0.01"; size.value = String(minB); sizeOut.value = String(minB);
|
||
const marks = 6; ticks.innerHTML = ''; tickLabels.innerHTML = '';
|
||
for (let i = 0; i < marks; i++) {
|
||
const v = (minB + (i * (maxB - minB) / (marks - 1)));
|
||
const opt = document.createElement('option'); opt.value = v.toFixed(2); ticks.appendChild(opt);
|
||
const lab = document.createElement('span'); lab.textContent = `${v.toFixed(0)}B`; tickLabels.appendChild(lab);
|
||
}
|
||
|
||
const TOL = 1.00;
|
||
|
||
function matchesFilters(name) {
|
||
const q = (search.value || '').toLowerCase();
|
||
if (q && !name.toLowerCase().includes(q)) return false;
|
||
const info = byModel[name];
|
||
if (quantSel.value && (info.quant || '').toUpperCase() !== quantSel.value.toUpperCase()) return false;
|
||
if (info.sizeB != null && info.sizeB < parseFloat(size.value) - 1e-9) return false;
|
||
return true;
|
||
}
|
||
|
||
// remove quantization tokens from the title (we still show quant below)
|
||
function cleanTitle(name, quant) {
|
||
const qTokens = ['BF16', 'F16', 'F32', 'FP16', 'MXFP2', 'MXFP4', 'MXFP8'];
|
||
if (quant) qTokens.push(quant.toUpperCase());
|
||
let t = name;
|
||
// common GGUF quant forms like Q4_K_M, Q8_0, Q6_K, Q3_K_S, etc.
|
||
t = t.replace(/[-_](Q\d+(?:_[A-Z0-9]+)*)/gi, '');
|
||
// BF16/F16/F32/MXFPx anywhere
|
||
t = t.replace(new RegExp(`[-_]?(${qTokens.join('|')})\\b`, 'gi'), '');
|
||
// drop "-UD"
|
||
t = t.replace(/-UD\b/gi, '');
|
||
// collapse leftover double separators
|
||
t = t.replace(/[-_]{2,}/g, '-')
|
||
.replace(/\s{2,}/g, ' ')
|
||
.replace(/[-_]+$/, '')
|
||
.trim();
|
||
return t;
|
||
}
|
||
|
||
function buildColDefs(envSel, mode) {
|
||
const cols = [];
|
||
for (const e of envSel) {
|
||
if (mode === 'both') {
|
||
cols.push({ env: e, key: `${e}|fa=0`, label: `${e}<div class="env-sub">FA off</div>` });
|
||
cols.push({ env: e, key: `${e}|fa=1`, label: `${e}<div class="env-sub">FA on</div>` });
|
||
} else {
|
||
const k = `${e}|fa=${mode === 'on' ? 1 : 0}`;
|
||
cols.push({ env: e, key: k, label: `${e}<div class="env-sub">FA ${mode === 'on' ? 'on' : 'off'}</div>` });
|
||
}
|
||
}
|
||
return cols;
|
||
}
|
||
|
||
function headerHTML(test, colDefs) {
|
||
return `<th class="col-model">Model</th><th class="col-winner">Winner (${test})</th>` +
|
||
colDefs.map(c => `<th>${c.label}</th>`).join('');
|
||
}
|
||
|
||
function cellHTML(name, test, key) {
|
||
const v = byModel[name][test][key];
|
||
if (!v) return '<div class="cell"><span class="muted">—</span></div>';
|
||
if (v.err || v.mean == null) return `<div class="cell"><span class="warn">⚠ ${v.et || 'error'}</span></div>`;
|
||
return `<div class="cell"><span class="tps">${v.mean.toFixed(2)}</span><span class="std">± ${v.std?.toFixed(2) ?? '—'}</span></div>`;
|
||
}
|
||
|
||
function winnersHTML(name, test, colDefs) {
|
||
// Collect (env, fa, mean, std) for visible columns
|
||
const vals = [];
|
||
for (const c of colDefs) {
|
||
const v = byModel[name][test][c.key];
|
||
if (v && !v.err && typeof v.mean === 'number') {
|
||
vals.push({
|
||
env: c.env,
|
||
fa: c.key.endsWith('|fa=1') ? 'on' : 'off',
|
||
mean: v.mean,
|
||
std: (typeof v.std === 'number' ? v.std : 0)
|
||
});
|
||
}
|
||
}
|
||
if (!vals.length) return '<span class="muted">—</span>';
|
||
|
||
// Best backend by mean
|
||
let best = vals[0];
|
||
for (const v of vals) if (v.mean > best.mean) best = v;
|
||
|
||
// Tie rule: within max(MIN_TOL, K*sqrt(std_best^2 + std_i^2))
|
||
const winners = [];
|
||
for (const v of vals) {
|
||
const pooled = Math.sqrt((best.std || 0) ** 2 + (v.std || 0) ** 2);
|
||
const tol = Math.max(MIN_TOL, K_SIGMA * pooled);
|
||
if ((best.mean - v.mean) <= tol) winners.push(v);
|
||
}
|
||
|
||
// Render pills. If FA mode = both, show FA state on the pill.
|
||
if (faMode.value !== 'both') {
|
||
const envs = [...new Set(winners.map(w => w.env))];
|
||
return `<div class="winner-wrap">${envs.map(env =>
|
||
`<span class="winner-pill">🏆 ${env}</span>`).join('')}</div>`;
|
||
} else {
|
||
// group by env and list FA(s) that tied
|
||
const byEnv = {};
|
||
for (const w of winners) (byEnv[w.env] ||= new Set()).add(w.fa);
|
||
return `<div class="winner-wrap">${Object.entries(byEnv).map(([env, fas]) => {
|
||
const list = [...fas].sort().map(x => `FA ${x}`).join(' & ');
|
||
return `<span class="winner-pill">🏆 ${env} (${list})</span>`;
|
||
}).join('')
|
||
}</div>`;
|
||
}
|
||
}
|
||
|
||
function renderTable(prefix, test) {
|
||
const envSel = selectedEnvs(); const mode = faMode.value;
|
||
const colDefs = buildColDefs(envSel, mode);
|
||
|
||
const h = document.getElementById(`${prefix}-h`);
|
||
h.innerHTML = headerHTML(test, colDefs);
|
||
|
||
const tbody = document.querySelector(`#tbl-${prefix} tbody`);
|
||
tbody.innerHTML = '';
|
||
|
||
for (const name of models.filter(matchesFilters)) {
|
||
const tr = document.createElement('tr');
|
||
|
||
const tdM = document.createElement('td');
|
||
tdM.className = 'col-model';
|
||
const info = byModel[name];
|
||
tdM.innerHTML = `<div class="model-title">${cleanTitle(name, info.quant)}</div><div class="model-meta">${info.quant || '—'} · ${(info.sizeB ?? '—')}B</div>`;
|
||
tr.appendChild(tdM);
|
||
|
||
const tdW = document.createElement('td');
|
||
tdW.className = 'col-winner';
|
||
tdW.innerHTML = winnersHTML(name, test, colDefs);
|
||
tr.appendChild(tdW);
|
||
|
||
for (const c of colDefs) {
|
||
const td = document.createElement('td');
|
||
td.innerHTML = cellHTML(name, test, c.key);
|
||
tr.appendChild(td);
|
||
}
|
||
tbody.appendChild(tr);
|
||
}
|
||
}
|
||
|
||
function render() {
|
||
renderTable('pp', 'pp512');
|
||
renderTable('tg', 'tg128');
|
||
}
|
||
|
||
// events
|
||
[search, quantSel, faMode].forEach(el => el.addEventListener('input', render));
|
||
colPicker.addEventListener('change', render);
|
||
size.addEventListener('input', e => { sizeOut.value = Number(e.target.value).toFixed(2); render(); });
|
||
sizeOut.addEventListener('input', e => { size.value = e.target.value; render(); });
|
||
|
||
render();
|
||
})();
|
||
</script>
|
||
</body>
|
||
|
||
</html> |