benchmarks.yaml

name: Benchmarks
on:
  push:
    branches:
      - master
  workflow_dispatch:

permissions: {}
# Every run pushes to rmosolgo/graphql-ruby-benchmarks, so don't let runs race each other
concurrency:
  group: benchmarks
  cancel-in-progress: false
defaults:
  run:
    # Includes `-o pipefail`, so a failed benchmark isn't hidden by `tee`
    shell: bash

jobs:
  benchmarks:
    name: Run benchmarks and store results
    runs-on: ubuntu-latest
    permissions:
      contents: read
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          persist-credentials: false
      - uses: ruby/setup-ruby@de492cf36cbab201753d720ce4f4e336cd688c94 # v1.336.0
        with:
          # Allocations change between Ruby versions, so keep this pinned
          ruby-version: '3.4'
          bundler-cache: true
      - run: mkdir -p tmp
      - name: Run runtime benchmarks
        run: bundle exec rake bench:track_runtime | tee tmp/benchmark_runtime.txt
      - name: Run memory benchmarks
        run: bundle exec rake bench:track_memory | tee tmp/benchmark_memory.txt
      - name: Run schema memory benchmarks
        run: bundle exec rake bench:track_schema_memory | tee tmp/benchmark_schema_memory.txt
      - name: Measure Sorbet coverage
        run: bundle exec rake sorbet:coverage:track | tee tmp/sorbet_coverage.json
      # Results are charted at https://rmosolgo.github.io/graphql-ruby-benchmarks/dev/bench/
      - name: Store runtime results
        uses: rmosolgo/github-action-benchmark@2f17b61ba59e765ca322ad380b4f85f8722c9c38 # ruby-v1
        with:
          name: Runtime
          tool: rubyruntime
          output-file-path: tmp/benchmark_runtime.txt
          gh-repository: github.com/rmosolgo/graphql-ruby-benchmarks
          github-token: ${{ secrets.BENCHMARK_RESULTS_TOKEN }}
          auto-push: true
          # Shared runners are noisy, so only flag big slowdowns
          alert-threshold: '150%'
          fail-on-alert: false
          summary-always: true
      # The action clones gh-repository into ./benchmark-data-repository and leaves it there,
      # so the next store step's `git clone` fails unless it's removed first
      - name: Clean up benchmark data repository
        run: rm -rf benchmark-data-repository
      - name: Store memory results
        uses: rmosolgo/github-action-benchmark@2f17b61ba59e765ca322ad380b4f85f8722c9c38 # ruby-v1
        with:
          name: Memory
          tool: rubymemory
          output-file-path: tmp/benchmark_memory.txt
          gh-repository: github.com/rmosolgo/graphql-ruby-benchmarks
          github-token: ${{ secrets.BENCHMARK_RESULTS_TOKEN }}
          auto-push: true
          # Allocations are nearly deterministic, so flag small increases
          alert-threshold: '110%'
          fail-on-alert: false
          summary-always: true
      - name: Clean up benchmark data repository
        run: rm -rf benchmark-data-repository
      - name: Store schema memory results
        uses: rmosolgo/github-action-benchmark@2f17b61ba59e765ca322ad380b4f85f8722c9c38 # ruby-v1
        with:
          name: Schema memory
          tool: rubymemory
          output-file-path: tmp/benchmark_schema_memory.txt
          gh-repository: github.com/rmosolgo/graphql-ruby-benchmarks
          github-token: ${{ secrets.BENCHMARK_RESULTS_TOKEN }}
          auto-push: true
          # Retained memory is nearly deterministic, so flag small increases
          alert-threshold: '110%'
          fail-on-alert: false
          summary-always: true
      - name: Clean up benchmark data repository
        run: rm -rf benchmark-data-repository
      - name: Store Sorbet coverage results
        uses: rmosolgo/github-action-benchmark@2f17b61ba59e765ca322ad380b4f85f8722c9c38 # ruby-v1
        with:
          name: Sorbet coverage
          tool: customBiggerIsBetter
          output-file-path: tmp/sorbet_coverage.json
          gh-repository: github.com/rmosolgo/graphql-ruby-benchmarks
          github-token: ${{ secrets.BENCHMARK_RESULTS_TOKEN }}
          auto-push: true
          # Coverage is deterministic, so flag any noticeable drop (for example, new files without `typed: true`)
          alert-threshold: '102%'
          fail-on-alert: false
          summary-always: true