Benchmarks

LLM leaderboards relevant to Human-Centered AI (HCAI): decision making under uncertainty, perspective taking, empathy, and creativity.

data = FileAttachment("benchmarks/benchmarks-data.json").json()

games = FileAttachment("benchmarks/repeated-games/data.json").json()

formatDate = value => value && Number.isFinite(Date.parse(value))
  ? new Date(value).toLocaleDateString()
  : "date not recorded"

countWord = n => ["Zero", "One", "Two", "Three", "Four", "Five", "Six", "Seven", "Eight", "Nine", "Ten", "Eleven", "Twelve", "Thirteen", "Fourteen", "Fifteen", "Sixteen", "Seventeen", "Eighteen", "Nineteen", "Twenty"][n] ?? String(n)

latestGamesRun = [games.results, ...(games.appended_runs || []).map(run => run.summary)]
  .filter(Boolean)
  .sort((a, b) => Date.parse(b.updated) - Date.parse(a.updated))[0]

refreshOverdue = Date.now() - Date.parse(data.checked_at || data.updated) > 8 * 24 * 60 * 60 * 1000

html`<div class="benchmark-list">
  <section aria-label="Repeated games">
    <h2><a href="benchmarks/repeated-games/index.html">Repeated games</a></h2>
    <p>${countWord(games.models.length)} models, each at the highest reasoning effort its provider offers, play ten rounds of the 144 two-player games from <a href="https://doi.org/10.1038/s41562-025-02172-y">Akata et al. (2025)</a>, plus Dictator and Ultimatum, against a scripted partner. Updated ${formatDate(latestGamesRun?.updated || games.generated_at)}.</p>
  </section>
  ${refreshOverdue ? html`<p class="alert alert-warning" role="status">The weekly check of the leaderboards below is overdue. Results may be out of date.</p>` : ``}
  ${data.benchmarks.map(b => html`
  <section aria-label="${b.name}">
    <h2><a href="${b.url}" target="_blank" rel="noopener">${b.name}${b.sub_leaderboard ? `: ${b.sub_leaderboard}` : ``}</a></h2>
    <p>${b.description} Checked ${formatDate(b.last_attempt || data.checked_at || data.updated)}.</p>
    ${b.status === "stale" ? html`<p class="alert alert-warning" role="status">The latest check failed. Previous results are shown below.</p>` : ``}
    ${b.status === "error" ? html`<p class="alert alert-warning" role="status">The latest check failed. No results are available.</p>` : ``}
    ${b.rows.length ? html`<div class="table-responsive"><table class="table table-striped">
      <thead><tr>${b.columns.map(c => html`<th scope="col">${c}</th>`)}</tr></thead>
      <tbody>${b.rows.map(r => html`<tr>${r.map(cell => html`<td>${cell}</td>`)}</tr>`)}</tbody>
    </table></div>` : ``}
  </section>
`)}</div>`