Better summary results, uncluding flash attention settings.

benchmark/generate_readme_summary.py

@@ -36,7 +36,6 @@ ERROR_LABEL = {
     "runtime": "⚠️ Runtime Error",
 }
 
-# Display name → fuzzy key (case/UD/shard-insensitive)
 DEFAULT_MODELS = [
     ("Gemma3 12B Q8_0",            "gemma-3-12b"),
     ("Gemma3 27B BF16",            "gemma-3-27b"),
@@ -54,16 +53,14 @@ SHARD_RE = re.compile(r"-000\d+-of-000\d+", re.IGNORECASE)
 def norm_model(s: str) -> str:
     s = (s or "").lower().replace("_", "-")
     s = SHARD_RE.sub("", s)
-    s = s.replace("-ud", "")  # drop -UD tag for matching
+    s = s.replace("-ud", "")
     return s
 
-# Load JSON
 raw = json.loads(Path(RESULTS_FILE).read_text(encoding="utf-8"))
 runs = raw["runs"]
 
-# Bucket rows by (model_key, env, test, fa)
 buckets = defaultdict(list)
-error_only = defaultdict(list)  # (model_key, env) -> [error_type,...] for test=None rows
+error_only = defaultdict(list)
 all_models = set()
 
 for r in runs:
@@ -72,30 +69,24 @@ for r in runs:
         continue
     mkey = norm_model(r.get("model_clean") or r.get("model") or "")
     all_models.add(mkey)
-    test = r.get("test")  # "pp512", "tg128", or None for pure errors
+    test = r.get("test")
     if test in ("pp512", "tg128"):
         buckets[(mkey, env, test)].append(r)
     else:
-        # capture error-only rows so we can show ⚠️ instead of "—"
         if r.get("error"):
             error_only[(mkey, env)].append(r.get("error_type") or "runtime")
 
 def pick_best(rows):
-    """Choose the best non-error row by tps_mean; if all error, return an error row."""
-    best = None
-    best_val = -1
-    fallback = None
+    best, best_val, fallback = None, -1, None
     for r in rows:
         if r.get("error"):
             fallback = r
             continue
         v = r.get("tps_mean")
         if isinstance(v, (int, float)) and v > best_val:
-            best_val = v
-            best = r
+            best_val, best = v, r
     return best or fallback
 
-# Build chosen results per (model, env): {pp: row|None, tg: row|None, err_only: str|None}
 chosen = defaultdict(lambda: defaultdict(dict))
 for (mkey, env, test), rows in buckets.items():
     chosen_row = pick_best(rows)
@@ -103,7 +94,6 @@ for (mkey, env, test), rows in buckets.items():
 
 for (mkey, env), etypes in error_only.items():
     if etypes:
-        # prefer specific types in a stable order
         if "load" in etypes:
             chosen[mkey][env]["error_only"] = "load"
         elif "hang" in etypes:
@@ -111,42 +101,55 @@ for (mkey, env), etypes in error_only.items():
         else:
             chosen[mkey][env]["error_only"] = "runtime"
 
+def fa_tag(row):
+    if not row or row.get("error"):
+        return ""
+    fa = row.get("fa")
+    if fa is None:
+        return ""
+    return " (FA on)" if fa else " (FA off)"
+
 def format_cell(entry_dict):
     pp = entry_dict.get("pp512")
     tg = entry_dict.get("tg128")
-
-    # If either chosen row is an error, show that error (web UI behavior)
     for row in (pp, tg):
         if row and row.get("error"):
             return ERROR_LABEL.get(row.get("error_type") or "runtime", "⚠️ Error")
-
-    # If both pp/tg missing but we have an error-only marker, show it
     if not pp and not tg:
         et = entry_dict.get("error_only")
         if et:
             return ERROR_LABEL.get(et, "⚠️ Error")
-        return "—"  # truly absent
-
-    # Otherwise, print available values (partial allowed)
+        return "—"
     def fmt(v):
         return f"{int(round(v))}" if isinstance(v, (int, float)) else "—"
     ppv = pp.get("tps_mean") if pp else None
     tgv = tg.get("tps_mean") if tg else None
-    return f"{fmt(ppv)} pp / {tgv:.1f} tg" if isinstance(tgv, (int, float)) \
-           else f"{fmt(ppv)} pp / — tg"
+    pp_suffix = fa_tag(pp)
+    tg_suffix = fa_tag(tg)
+    if isinstance(tgv, (int, float)):
+        return f"{fmt(ppv)} pp{pp_suffix} / {tgv:.1f} tg{tg_suffix}"
+    else:
+        return f"{fmt(ppv)} pp{pp_suffix} / — tg"
 
 def best_env_for(mkey, test):
-    best_env, best_val = None, -1
+    best_env, best_val, best_row = None, -1, None
     for env in ENV_ORDER:
         row = chosen[mkey].get(env, {}).get(test)
         if not row or row.get("error"):
             continue
         v = row.get("tps_mean")
         if isinstance(v, (int, float)) and v > best_val:
-            best_env, best_val = env, v
-    return best_env
+            best_env, best_val, best_row = env, v, row
+    return best_env, (best_row.get("fa") if best_row else None)
+
+def win_label(env, fa):
+    if not env:
+        return "—"
+    base = WINNER_NAMES[env]
+    if fa is None:
+        return f"🏆 **{base}**"
+    return f"🏆 **{base}** ({'FA on' if fa else 'FA off'})"
 
-# Fuzzy match helper
 def find_model_key(fuzzy):
     needle = norm_model(fuzzy)
     for k in all_models:
@@ -154,21 +157,19 @@ def find_model_key(fuzzy):
             return k
     return None
 
-# Print table
-header = ["Model"] + [COL_NAMES[e] for e in ENV_ORDER] + ["🏆 Best PP", "🏆 Best TG"]
+# Header now has Best PP & Best TG right after Model
+header = ["Model", "🏆 Best PP", "🏆 Best TG"] + [COL_NAMES[e] for e in ENV_ORDER]
 print("| " + " | ".join(header) + " |")
 print("|" + "|".join(["---"] * len(header)) + "|")
 
 for disp, fuzzy in DEFAULT_MODELS:
     mkey = find_model_key(fuzzy)
     if not mkey:
-        print("| " + " | ".join([f"**{disp}**"] + ["—"]*len(ENV_ORDER) + ["—","—"]) + " |")
+        print("| " + " | ".join([f"**{disp}**", "—", "—"] + ["—"]*len(ENV_ORDER)) + " |")
         continue
-    row = [f"**{disp}**"]
+    bpp_env, bpp_fa = best_env_for(mkey, "pp512")
+    btg_env, btg_fa = best_env_for(mkey, "tg128")
+    row = [f"**{disp}**", win_label(bpp_env, bpp_fa), win_label(btg_env, btg_fa)]
     for env in ENV_ORDER:
         row.append(format_cell(chosen[mkey].get(env, {})))
-    bpp = best_env_for(mkey, "pp512")
-    btg = best_env_for(mkey, "tg128")
-    row.append(f"🏆 **{WINNER_NAMES[bpp]}**" if bpp else "—")
-    row.append(f"🏆 **{WINNER_NAMES[btg]}**" if btg else "—")
     print("| " + " | ".join(row) + " |")