Browser Throughput Benchmark #10
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Browser Throughput Benchmark | |
| on: | |
| push: | |
| branches: [master] | |
| paths: | |
| - 'benchmarks/browser/**' | |
| - 'benchmarks/src/util/**' | |
| - 'benchmarks/src/merge-results.ts' | |
| - 'package.json' | |
| schedule: | |
| - cron: '30 10 * * 5' # Weekly on Friday at 10:30am UTC (5:30am CDT, offset from main browser benchmark to avoid provider contention) | |
| workflow_dispatch: | |
| inputs: | |
| iterations: | |
| description: 'Iterations per provider (sessions)' | |
| required: false | |
| default: '100' | |
| dry_run: | |
| description: 'Run without committing results' | |
| required: false | |
| default: false | |
| type: boolean | |
| concurrency: | |
| group: browser-throughput-benchmarks | |
| cancel-in-progress: true | |
| permissions: | |
| contents: write | |
| pull-requests: write | |
| id-token: write | |
| jobs: | |
| setup: | |
| name: Resolve target pages | |
| runs-on: namespace-profile-default;permissions.additional_grant=vault/object:*:list;permissions.additional_grant=vault/object:*:describe | |
| # runs-on: self-hosted | |
| outputs: | |
| urls: ${{ steps.pick.outputs.urls }} | |
| steps: | |
| - name: Resolve one shared random article per iteration | |
| id: pick | |
| run: | | |
| # Resolve one concrete article per iteration up front so every | |
| # provider benchmarks against the same page in a given round — but a | |
| # different page each round, so we don't measure a warmed cache. | |
| count=${{ github.event_name == 'push' && '3' || github.event.inputs.iterations || '100' }} | |
| urls=() | |
| for i in $(seq 1 "$count"); do | |
| # Follow Special:Random's redirect once to a concrete article URL. | |
| # Retry up to 5 times until we find an article that has at least | |
| # one article-body link (href containing "/wiki/" without a colon, | |
| # which excludes namespace pages like Help:, File:, Special:). | |
| # Stub articles without body links cause the click action to time | |
| # out and skew throughput results. | |
| for attempt in 1 2 3 4 5; do | |
| url=$(curl -sL -A 'ComputeSDK-Benchmark' -o /dev/null -w '%{url_effective}' \ | |
| 'https://en.wikipedia.org/wiki/Special:Random') | |
| if [ -z "$url" ] || [[ "$url" == *"Special:Random"* ]]; then | |
| url='https://en.wikipedia.org/wiki/Special:Random' | |
| break | |
| fi | |
| # Fetch the article HTML and check for article-body links. | |
| # The benchmark selects #mw-content-text a[href*="/wiki/"] and | |
| # filters in JS for links without a namespace colon after /wiki/. | |
| # Body links appear as protocol-relative (//en.wikipedia.org/wiki/) | |
| # or absolute (https://en.wikipedia.org/wiki/) URLs. | |
| html=$(curl -sL -A 'ComputeSDK-Benchmark' "$url") | |
| if echo "$html" | grep -qP 'href="(//en\.wikipedia\.org)?/wiki/[^"]*" *' \ | |
| && echo "$html" | grep -oP 'href="[^"]*"' \ | |
| | grep -qP '/wiki/[^:]*$'; then | |
| break | |
| fi | |
| done | |
| urls+=("$url") | |
| done | |
| # Emit as a newline-delimited multiline output. Pure bash — avoids a | |
| # jq/python dependency that may be absent on the runner image. | |
| { | |
| echo "urls<<URLS_EOF" | |
| printf '%s\n' "${urls[@]}" | |
| echo "URLS_EOF" | |
| } >> "$GITHUB_OUTPUT" | |
| echo "Resolved ${#urls[@]} shared article URLs for all providers" | |
| bench: | |
| name: Bench ${{ matrix.provider }} | |
| runs-on: namespace-profile-default;permissions.additional_grant=vault/object:*:list;permissions.additional_grant=vault/object:*:describe | |
| # runs-on: self-hosted | |
| needs: setup | |
| timeout-minutes: 60 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| provider: | |
| - browserbase | |
| - browseruse | |
| - hyperbrowser | |
| - kernel | |
| - notte | |
| - steel | |
| - tilion | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: pnpm/action-setup@v4 | |
| - uses: actions/setup-node@v4 | |
| with: | |
| node-version: 24 | |
| cache: 'pnpm' | |
| - uses: namespacelabs/nscloud-setup@v0 | |
| - name: Install dependencies | |
| run: | | |
| if [ "${{ github.event_name }}" = "schedule" ]; then | |
| pnpm update | |
| else | |
| pnpm install --frozen-lockfile | |
| fi | |
| - name: Clear stale results from checkout | |
| run: rm -rf results/browser-throughput/ | |
| - name: Run browser throughput benchmark | |
| env: | |
| THROUGHPUT_URLS: ${{ needs.setup.outputs.urls }} | |
| run: | | |
| . benchmarks/scripts/load-vault-secrets.sh '^(BROWSERBASE_API_KEY|BROWSERBASE_PROJECT_ID|BROWSER_USE_API_KEY|HYPERBROWSER_API_KEY|KERNEL_API_KEY|NOTTE_API_KEY|STEEL_API_KEY|TILION_API_KEY|TILION_BASE_URL|COMPUTESDK_ADMIN_API_KEY|BENCHMARKS_PLATFORM_API_KEY)' | |
| # All matrix jobs for this workflow run share a single platform run. The | |
| # attempt number is included so re-runs do not rejoin completed runs. | |
| RUN_KEY="${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" | |
| npx tsx packages/benchsdk-runner/dist/bin.js run benchmarks/browser/browser-throughput.bench.ts \ | |
| --provider ${{ matrix.provider }} \ | |
| --iterations ${{ github.event_name == 'push' && '3' || github.event.inputs.iterations || '100' }} \ | |
| --run-key "$RUN_KEY" | |
| - name: Upload results | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: browser-throughput-results-${{ matrix.provider }} | |
| path: results/browser-throughput/ | |
| if-no-files-found: ignore | |
| retention-days: 7 | |
| collect: | |
| name: Collect Results | |
| runs-on: namespace-profile-default;permissions.additional_grant=vault/object:*:list;permissions.additional_grant=vault/object:*:describe | |
| # runs-on: self-hosted | |
| needs: bench | |
| if: always() | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| # Full history so the rebase-on-push retry below has a merge base. | |
| # A shallow (depth-1) clone can make `git rebase origin/<branch>` | |
| # fail when the remote has advanced past the shallow boundary. | |
| fetch-depth: 0 | |
| - uses: pnpm/action-setup@v4 | |
| - uses: actions/setup-node@v4 | |
| with: | |
| node-version: 24 | |
| cache: 'pnpm' | |
| - uses: namespacelabs/nscloud-setup@v0 | |
| - name: Install dependencies | |
| run: | | |
| if [ "${{ github.event_name }}" = "schedule" ]; then | |
| pnpm update | |
| else | |
| pnpm install --frozen-lockfile | |
| fi | |
| - name: Download all artifacts | |
| uses: actions/download-artifact@v4 | |
| with: | |
| path: artifacts/ | |
| pattern: browser-throughput-results-* | |
| - name: Merge results | |
| run: npx tsx benchmarks/src/merge-results.ts --input artifacts --mode browser-throughput | |
| - run: pnpm run generate-browser-throughput-svg | |
| - name: Upload SVG as artifact | |
| if: github.event_name == 'push' | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: browser-throughput-benchmark-svg | |
| path: browser-throughput.svg | |
| if-no-files-found: ignore | |
| retention-days: 7 | |
| - name: Post results to merged PR | |
| if: github.event_name == 'push' | |
| continue-on-error: true | |
| uses: actions/github-script@v7 | |
| with: | |
| script: | | |
| const fs = require('fs'); | |
| const path = require('path'); | |
| const runUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`; | |
| const latestPath = path.join('results', 'browser-throughput', 'latest.json'); | |
| let body = '## Browser Throughput Benchmark Results\n\n'; | |
| if (!fs.existsSync(latestPath)) { | |
| body += '> No browser-throughput benchmark results were generated.\n\n'; | |
| } else { | |
| const data = JSON.parse(fs.readFileSync(latestPath, 'utf-8')); | |
| const expectedActions = data.config?.actionsPerSession ?? 0; | |
| const results = data.results | |
| .filter(r => !r.skipped) | |
| .sort((a, b) => (b.compositeScore || 0) - (a.compositeScore || 0)); | |
| if (results.length === 0) { | |
| body += '> No browser-throughput benchmark results were generated.\n\n'; | |
| } else { | |
| body += '| # | Provider | Score | APS (med) | Task (med) | Task (p95) | Screenshot | Status |\n'; | |
| body += '|---|----------|-------|-----------|------------|------------|------------|--------|\n'; | |
| results.forEach((r, i) => { | |
| const name = r.provider.charAt(0).toUpperCase() + r.provider.slice(1); | |
| const score = r.compositeScore !== undefined ? r.compositeScore.toFixed(1) : '--'; | |
| const aps = r.summary.actionsPerSecond.median.toFixed(2) + '/s'; | |
| const taskMed = (r.summary.taskMs.median / 1000).toFixed(2) + 's'; | |
| const taskP95 = (r.summary.taskMs.p95 / 1000).toFixed(2) + 's'; | |
| const screenshotMed = Math.round(r.summary.perActionType.screenshot?.median || 0) + 'ms'; | |
| const ok = r.iterations.filter(it => !it.error && it.actionsCompleted === expectedActions).length; | |
| const count = r.iterations.length; | |
| body += `| ${i + 1} | ${name} | ${score} | ${aps} | ${taskMed} | ${taskP95} | ${screenshotMed} | ${ok}/${count} |\n`; | |
| }); | |
| body += '\n'; | |
| } | |
| } | |
| body += `---\n*[View full run](${runUrl}) · SVG available as [build artifact](${runUrl}#artifacts)*`; | |
| // This push is a merge to master — post results to the PR it closed. | |
| // A commit can be associated with several PRs (backports, merge | |
| // queue), so pick the one merged into the branch we pushed to. | |
| const target = context.ref.replace('refs/heads/', ''); | |
| const { data: prs } = await github.rest.repos.listPullRequestsAssociatedWithCommit({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| commit_sha: context.sha, | |
| }); | |
| const pr = prs.find(p => p.merged_at && p.base.ref === target); | |
| if (!pr) { | |
| // Direct push with no associated PR — fall back to a commit comment. | |
| await github.rest.repos.createCommitComment({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| commit_sha: context.sha, | |
| body, | |
| }); | |
| return; | |
| } | |
| const marker = '## Browser Throughput Benchmark Results'; | |
| const comments = await github.paginate(github.rest.issues.listComments, { | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| issue_number: pr.number, | |
| per_page: 100, | |
| }); | |
| const existing = comments.find(c => c.body.startsWith(marker)); | |
| if (existing) { | |
| await github.rest.issues.updateComment({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| comment_id: existing.id, | |
| body, | |
| }); | |
| } else { | |
| await github.rest.issues.createComment({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| issue_number: pr.number, | |
| body, | |
| }); | |
| } | |
| - name: Commit and push | |
| if: github.event_name != 'push' && github.event.inputs.dry_run != 'true' | |
| run: | | |
| git config user.name "github-actions[bot]" | |
| git config user.email "github-actions[bot]@users.noreply.github.com" | |
| git add package.json pnpm-lock.yaml browser-throughput.svg results/browser-throughput/ | |
| git diff --cached --quiet && echo "No changes to commit" && exit 0 | |
| git commit -m "chore: update browser throughput benchmark results [skip ci]" | |
| # Remote master can advance during the run (concurrent benchmark | |
| # workflows push to master too), so a plain push fails non-fast-forward. | |
| # Rebase onto the latest remote and retry a few times before giving up. | |
| branch="${GITHUB_REF#refs/heads/}" | |
| for attempt in 1 2 3 4 5; do | |
| # Rebase before each attempt (including the last) so every rebase | |
| # is followed by a push — otherwise the final iteration's rebase | |
| # would be wasted and a resolvable non-fast-forward could still fail. | |
| git fetch origin "${branch}" | |
| git rebase --autostash "origin/${branch}" || { git rebase --abort; exit 1; } | |
| if git push origin "HEAD:${branch}"; then | |
| echo "Pushed on attempt ${attempt}" | |
| exit 0 | |
| fi | |
| echo "Push rejected (attempt ${attempt}); will rebase and retry" | |
| done | |
| echo "Failed to push after multiple attempts" >&2 | |
| exit 1 |