Skip to content

feat: overall author rankings in leaderboard #126

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
YiyanZhai wants to merge 1 commit into
base: main
Choose a base branch
from
Open

feat: overall author rankings in leaderboard #126

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
150 changes: 126 additions & 24 deletions web/apps/web/app/leaderboard/client.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ type DefinitionAuthorDetail = {
totalComparisons: number
totalWorkloads: number
coverage: Record<string, CoverageStats>
speedupSums: Record<string, number>
solutionNamesByAuthor: Record<string, string[]>
}

Expand All @@ -36,6 +37,14 @@ type DefinitionDetail = {
solutionNames: string[]
}

type OverallAuthorRanking = {
author: string
totalSpeedup: number
comparisons: number
coverage?: CoverageStats
correctness?: CorrectnessSummary
}

const DEFAULT_PIN = 0.95
const DEFAULT_VISIBLE = 5
const LIST_MAX_HEIGHT = 288 // 18rem
Expand Down Expand Up @@ -86,7 +95,7 @@ export function LeaderboardClient({
}: LeaderboardClientProps) {
const [pinnedP, setPinnedP] = useState<number | null>(initialPinnedP)
const [isListExpanded, setIsListExpanded] = useState(false)
const [activeTab, setActiveTab] = useState<"fast" | "correctness">("fast")
const [activeTab, setActiveTab] = useState<"fast" | "rankings">("fast")
const [selectedAuthor, setSelectedAuthor] = useState<string | null>(null)
const [isDrawerOpen, setIsDrawerOpen] = useState(false)
const [highlightedAuthor, setHighlightedAuthor] = useState<string | null>(null)
Expand Down Expand Up @@ -292,7 +301,43 @@ export function LeaderboardClient({
openAuthorDetail(highlightedAuthor)
}, [highlightedAuthor, openAuthorDetail])

const pinnedLabel = pinnedTarget.toFixed(2)
const correctnessByAuthor = useMemo(() => {
const map: Record<string, CorrectnessSummary> = {}
for (const entry of correctness.stats) {
map[entry.author] = {
total: entry.total,
passed: entry.passed,
incorrect: entry.incorrect,
runtime_error: entry.runtime_error,
other: entry.other,
}
}
return map
}, [correctness.stats])

const overallRanking = useMemo<OverallAuthorRanking[]>(() => {
const entries: OverallAuthorRanking[] = []
const speedupEntries = Object.entries(fast.speedupSums ?? {})
for (const [author, totalSpeedup] of speedupEntries) {
if (excludedSet.has(author)) continue
const comparisons = fast.comparisonCounts[author] ?? 0
if (comparisons <= 0) continue
entries.push({
author,
totalSpeedup,
comparisons,
coverage: fast.coverage?.[author],
correctness: correctnessByAuthor[author],
})
}
return entries.sort((a, b) => {
if (b.totalSpeedup !== a.totalSpeedup) return b.totalSpeedup - a.totalSpeedup
if (b.comparisons !== a.comparisons) return b.comparisons - a.comparisons
return a.author.localeCompare(b.author)
})
}, [correctnessByAuthor, excludedSet, fast.comparisonCounts, fast.coverage, fast.speedupSums])
Comment on lines +318 to +338
Copy link
Contributor

@gemini-code-assist gemini-code-assist bot Dec 5, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

medium

The overallRanking calculation can be rewritten using a more functional and declarative style with array methods like map, filter, and sort. This improves readability and avoids mutable state by removing the intermediate entries array.

  const overallRanking = useMemo<OverallAuthorRanking[]>(() => {
    return Object.entries(fast.speedupSums ?? {})
      .map(([author, totalSpeedup]) => ({
        author,
        totalSpeedup,
        comparisons: fast.comparisonCounts[author] ?? 0,
        coverage: fast.coverage?.[author],
        correctness: correctnessByAuthor[author],
      }))
      .filter((entry) => !excludedSet.has(entry.author) && entry.comparisons > 0)
      .sort((a, b) => {
        if (b.totalSpeedup !== a.totalSpeedup) return b.totalSpeedup - a.totalSpeedup;
        if (b.comparisons !== a.comparisons) return b.comparisons - a.comparisons;
        return a.author.localeCompare(b.author);
      });
  }, [correctnessByAuthor, excludedSet, fast.comparisonCounts, fast.coverage, fast.speedupSums])


const maxTotalSpeedup = overallRanking.length > 0 ? overallRanking[0].totalSpeedup : 0

const correctnessRanking = useMemo(() => {
return correctness.stats
Expand All @@ -313,20 +358,6 @@ export function LeaderboardClient({

const maxPassRate = correctnessRanking.length > 0 ? correctnessRanking[0].passRate : 0

const correctnessByAuthor = useMemo(() => {
const map: Record<string, CorrectnessSummary> = {}
for (const entry of correctness.stats) {
map[entry.author] = {
total: entry.total,
passed: entry.passed,
incorrect: entry.incorrect,
runtime_error: entry.runtime_error,
other: entry.other,
}
}
return map
}, [correctness.stats])

return (
<section>
<div className="container space-y-6 py-6 md:py-8">
Expand All @@ -337,14 +368,14 @@ export function LeaderboardClient({
</p>
</div>

<Tabs value={activeTab} onValueChange={(value) => setActiveTab(value as "fast" | "correctness")}
<Tabs value={activeTab} onValueChange={(value) => setActiveTab(value as "fast" | "rankings")}
className="space-y-6"
>
<TabsList className="w-fit">
<TabsTrigger value="fast">
<FastPLabel className="font-medium" />
</TabsTrigger>
<TabsTrigger value="correctness">Correctness</TabsTrigger>
<TabsTrigger value="rankings">Author rankings</TabsTrigger>
</TabsList>

<TabsContent value="fast" className="space-y-6">
Expand Down Expand Up @@ -497,10 +528,30 @@ export function LeaderboardClient({
</div>
)}
</div>

<OverallRankingCard
rankings={overallRanking}
maxTotalSpeedup={maxTotalSpeedup}
formatCoverage={formatCoverage}
title="Overall author rankings"
description="Area under the Fast-p curve across every workload; higher values mean more consistent speedups over the baseline."
/>
Comment on lines +532 to +538
Copy link
Contributor

@gemini-code-assist gemini-code-assist bot Dec 5, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

high

The OverallRankingCard component is rendered in both the "Fast-p" tab and the "Author rankings" tab. This seems redundant. Based on the PR description, this new ranking card is intended for the author rankings view. To avoid duplication and keep the UI clean, it should probably only be displayed in the "Author rankings" tab.

</TabsContent>

<TabsContent value="correctness" className="space-y-6">
<div className="rounded-lg border bg-card/50 p-4">
<TabsContent value="rankings" className="space-y-6">
<OverallRankingCard
rankings={overallRanking}
maxTotalSpeedup={maxTotalSpeedup}
formatCoverage={formatCoverage}
title="Overall author rankings"
description="Area under the Fast-p curve across every workload; higher values mean more consistent speedups over the baseline."
/>

<div className="rounded-lg border bg-card/50 p-4 space-y-4">
<div>
<h3 className="text-lg font-semibold">Correctness author rankings</h3>
<p className="text-xs text-muted-foreground">Pass rate across all workloads for each author.</p>
</div>
{correctnessRanking.length === 0 ? (
<p className="text-sm text-muted-foreground">No correctness data available.</p>
) : (
Expand All @@ -520,10 +571,7 @@ export function LeaderboardClient({
</div>
</div>
<div className="h-2 rounded bg-muted">
<div
className="h-full rounded bg-primary"
style={{ width }}
/>
<div className="h-full rounded bg-primary" style={{ width }} />
</div>
</div>
)
Expand Down Expand Up @@ -560,6 +608,60 @@ type AuthorDetailDrawerProps = {
onOpenChange: (open: boolean) => void
}

type OverallRankingCardProps = {
rankings: OverallAuthorRanking[]
maxTotalSpeedup: number
formatCoverage: (stats?: CoverageStats | null) => { percentText: string } | null
title: string
description: string
}

function OverallRankingCard({ rankings, maxTotalSpeedup, formatCoverage, title, description }: OverallRankingCardProps) {
const hasRankings = rankings.length > 0
return (
<div className="rounded-lg border bg-card/50 p-4 space-y-4">
<div>
<h3 className="text-lg font-semibold">{title}</h3>
<p className="text-xs text-muted-foreground">{description}</p>
</div>
{!hasRankings ? (
<p className="text-sm text-muted-foreground">No speedup data available.</p>
) : (
<div className="space-y-4">
{rankings.map((entry, index) => {
const widthPercent = maxTotalSpeedup > 0 ? Math.max(0, (entry.totalSpeedup / maxTotalSpeedup) * 100) : 0
const width = `${Math.min(widthPercent, 100)}%`
const totalSpeedupText = entry.totalSpeedup.toFixed(2)
const passRatePercent = entry.correctness && entry.correctness.total > 0
? ((entry.correctness.passed / entry.correctness.total) * 100).toFixed(1)
: null
const coverageInfo = formatCoverage(entry.coverage)
return (
<div key={entry.author} className="space-y-2">
<div className="flex items-center justify-between text-sm font-medium">
<div className="flex items-center gap-2">
<span className="text-xs font-semibold text-muted-foreground">{index + 1}.</span>
<span>{entry.author}</span>
</div>
<div className="text-xs text-muted-foreground">Area under Fast-p curve {totalSpeedupText}</div>
</div>
<div className="h-2 rounded bg-muted">
<div className="h-full rounded bg-primary" style={{ width }} />
</div>
<div className="flex flex-wrap items-center gap-x-4 gap-y-1 text-xs text-muted-foreground">
<span>{entry.comparisons} comparisons</span>
{coverageInfo && <span>{coverageInfo.percentText}% evaluated</span>}
{passRatePercent && <span>{passRatePercent}% pass</span>}
</div>
</div>
)
})}
</div>
)}
</div>
)
}

function AuthorDetailDrawer({ open, author, definitions, pinnedTarget, onOpenChange }: AuthorDetailDrawerProps) {
return (
<Dialog.Root open={open} onOpenChange={onOpenChange}>
Expand Down
4 changes: 3 additions & 1 deletion web/apps/web/app/leaderboard/section.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ type DefinitionAuthorDetail = {
totalComparisons: number
totalWorkloads: number
coverage: Record<string, CoverageStats>
speedupSums: Record<string, number>
solutionNamesByAuthor: Record<string, string[]>
}

Expand Down Expand Up @@ -63,7 +64,7 @@ export function LeaderboardSection({ entries, baselineLabel, initialPinnedP }: L
})

const definitionAuthorDetails: DefinitionAuthorDetail[] = filteredEntries.map((entry) => {
const { curves, comparisonCounts, totalComparisons, totalWorkloads, coverage } = computeFastPCurvesForAuthors({
const { curves, comparisonCounts, totalComparisons, totalWorkloads, coverage, speedupSums } = computeFastPCurvesForAuthors({
datasets: [
{
solutions: entry.solutions,
Expand All @@ -89,6 +90,7 @@ export function LeaderboardSection({ entries, baselineLabel, initialPinnedP }: L
totalComparisons,
totalWorkloads,
coverage,
speedupSums,
solutionNamesByAuthor,
}
})
Expand Down
15 changes: 14 additions & 1 deletion web/apps/web/lib/analytics.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ export type AuthorCurvesResponse = {
totalComparisons: number
totalWorkloads: number
coverage: Record<string, CoverageStats>
speedupSums: Record<string, number>
}

export type AuthorCorrectnessResponse = {
Expand Down Expand Up @@ -294,6 +295,7 @@ export function computeFastPCurvesForAuthors(params: {

const curves: Record<string, CurvePoint[]> = {}
const comparisonCounts: Record<string, number> = {}
const speedupSums: Record<string, number> = {}
let totalComparisons = 0

for (const [author, ratios] of authorRatios.entries()) {
Expand All @@ -315,6 +317,17 @@ export function computeFastPCurvesForAuthors(params: {
points.push({ p, percent })
}
curves[author] = points

let area = 0
for (let index = 1; index < points.length; index++) {
const prev = points[index - 1]
const current = points[index]
const width = current.p - prev.p
if (width <= 0) continue
const height = (prev.percent + current.percent) / 200
area += width * height
}
speedupSums[author] = area
}

const coverage: Record<string, CoverageStats> = {}
Expand All @@ -326,7 +339,7 @@ export function computeFastPCurvesForAuthors(params: {
coverage[author] = { attempted, total, percent }
}

return { curves, comparisonCounts, totalComparisons, totalWorkloads, coverage }
return { curves, comparisonCounts, totalComparisons, totalWorkloads, coverage, speedupSums }
}

export function computeAuthorCorrectnessSummary(params: {
Expand Down