Benchmark overhaul (#41408)

* Big refactor, still classes to move around and script to re-complexify * Move to streamer, isolate benches, propagate num tokens * Some refacto * Added compile mode to name * Re-order * Move to dt_tokens * Better format * Fix and disable use_cache by default * Fixed compile and SDPA backend default * Refactor results format * Added default compile mode * Always use cache * Fixed cache and added flex * Plan for missing modules * Experiments: no cg and shuffle * Disable compile for FA * Remove wall time, add sweep mode, get git commit * Review compliance, start * Apply suggestions from code review Co-authored-by: Luc Georges <McPatate@users.noreply.github.com> * Update benchmark_v2/framework/benchmark_runner.py Co-authored-by: Luc Georges <McPatate@users.noreply.github.com> * Disable workflow * Pretty print * Added some pretty names to have pretty logs * Review n2 compliance (end?) * Style and end of PR --------- Co-authored-by: Luc Georges <McPatate@users.noreply.github.com>

Author: remi-or

.github/workflows/benchmark.yml

@@ -1,10 +1,7 @@
 name: Self-hosted runner (benchmark)
 
 on:
- push:
- branches: [main]
- pull_request:
- types: [ opened, labeled, reopened, synchronize ]
+ workflow_dispatch:
 
 concurrency:
  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}

.github/workflows/benchmark_v2.yml

@@ -1,35 +1,7 @@
 name: Benchmark v2 Framework
 
 on:
- workflow_call:
- inputs:
- runner:
- description: 'GH Actions runner group to use'
- required: true
- type: string
- container_image:
- description: 'Docker image to use'
- required: true
- type: string
- container_options:
- description: 'Container options to use'
- required: true
- type: string
- commit_sha:
- description: 'Commit SHA to benchmark'
- required: false
- type: string
- default: ''
- run_id:
- description: 'Custom run ID for organizing results (auto-generated if not provided)'
- required: false
- type: string
- default: ''
- benchmark_repo_id:
- description: 'HuggingFace Dataset to upload results to (e.g., "org/benchmark-results")'
- required: false
- type: string
- default: ''
+ workflow_dispatch:
 
 env:
  HF_HOME: /mnt/cache
@@ -82,4 +54,4 @@ jobs:
  --token '${{ secrets.TRANSFORMERS_CI_RESULTS_UPLOAD_TOKEN }}' \
  --log-level INFO
  env:
- HF_TOKEN: ${{ secrets.HF_HUB_READ_TOKEN }}
+ HF_TOKEN: ${{ secrets.HF_HUB_READ_TOKEN }}

.github/workflows/benchmark_v2_a10_caller.yml

@@ -1,11 +1,7 @@
 name: Benchmark v2 Scheduled Runner - A10 Single-GPU
 
 on:
- schedule:
- # Run daily at 16:30 UTC
- - cron: "30 16 * * *"
- pull_request:
- types: [ opened, labeled, reopened, synchronize ]
+ workflow_dispatch:
 
 jobs:
  benchmark-v2-default:
@@ -18,4 +14,4 @@ jobs:
  commit_sha: ${{ github.sha }}
  run_id: ${{ github.run_id }}
  benchmark_repo_id: hf-internal-testing/transformers-daily-benchmarks
- secrets: inherit
+ secrets: inherit

.github/workflows/benchmark_v2_mi325_caller.yml

@@ -1,11 +1,7 @@
 name: Benchmark v2 Scheduled Runner - MI325 Single-GPU
 
 on:
- schedule:
- # Run daily at 16:30 UTC
- - cron: "30 16 * * *"
- pull_request:
- types: [ opened, labeled, reopened, synchronize ]
+ workflow_dispatch:
 
 jobs:
  benchmark-v2-default:
@@ -18,4 +14,4 @@ jobs:
  commit_sha: ${{ github.sha }}
  run_id: ${{ github.run_id }}
  benchmark_repo_id: hf-internal-testing/transformers-daily-benchmarks
- secrets: inherit
+ secrets: inherit

benchmark_v2/.gitignore

@@ -1 +1,2 @@
-benchmark_results/
+benchmark_results/
+benchmark_results_profiles/