[CI/Build] Gate CPU and T-One RDMA performance regressions

.github/workflows/performance.yml

@@ -0,0 +1,151 @@
+name: Performance regression
+
+on:
+  pull_request:
+    branches: [main, 'release/**']
+    types: [opened, synchronize, reopened]
+  workflow_dispatch:
+    inputs:
+      baseline_ref:
+        description: Baseline commit or ref (use the candidate SHA for an A/A calibration)
+        required: true
+        default: main
+      candidate_ref:
+        description: Candidate commit or ref (empty uses the selected workflow ref)
+        required: false
+
+permissions:
+  contents: read
+
+concurrency:
+  group: performance-${{ github.event.pull_request.number || github.run_id }}
+  cancel-in-progress: true
+
+jobs:
+  tests:
+    name: Performance gate tests
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          persist-credentials: false
+      - run: python3 -m unittest discover -s scripts/ci/tests -p 'test_*perf*.py' -v
+
+  performance:
+    name: Performance Gate
+    needs: tests
+    if: ${{ always() }}
+    # Fork code always runs on a disposable hosted VM. A repository-owned PR
+    # can use the optional dedicated Linux runner label after A/A calibration.
+    runs-on: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork && 'ubuntu-22.04' || vars.PERF_RUNNER || 'ubuntu-22.04' }}
+    timeout-minutes: 75
+    env:
+      PERF_ROOT: ${{ github.workspace }}/performance-${{ github.run_id }}-${{ github.run_attempt }}
+    steps:
+      - name: Initialize run directory
+        run: mkdir -p "$PERF_ROOT/results"
+
+      - name: Require harness tests
+        env:
+          TEST_RESULT: ${{ needs.tests.result }}
+        run: test "$TEST_RESULT" = success
+
+      - name: Checkout candidate
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.candidate_ref || github.sha }}
+          path: candidate
+          fetch-depth: 2
+          submodules: true
+          persist-credentials: false
+
+      - name: Checkout baseline
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.baseline_ref || github.event.pull_request.base.sha }}
+          path: baseline
+          submodules: true
+          persist-credentials: false
+
+      - name: Freeze the benchmark and policy
+        env:
+          EVENT_NAME: ${{ github.event_name }}
+        run: |
+          set -euo pipefail
+          baseline_sha=$(git -C baseline rev-parse HEAD)
+          candidate_sha=$(git -C candidate rev-parse HEAD)
+          if [ "$EVENT_NAME" = pull_request ]; then
+            test "$(git -C candidate rev-parse HEAD^1)" = "$baseline_sha"
+          fi
+          harness_source=baseline
+          if [ ! -f baseline/scripts/ci/performance/perf_gate.py ]; then
+            harness_source=candidate
+            echo '::warning::Bootstrapping the performance harness from the candidate; review the new policy before merging.'
+          fi
+          mkdir -p "$PERF_ROOT/harness"
+          cp -R "$harness_source/scripts/ci/performance/." "$PERF_ROOT/harness/"
+          cp baseline/mooncake-store/benchmarks/offset_allocator_concurrency_bench.cpp \
+            candidate/mooncake-store/benchmarks/offset_allocator_concurrency_bench.cpp
+          python3 "$PERF_ROOT/harness/freeze_build.py" \
+            --baseline baseline --candidate candidate \
+            --manifest "$PERF_ROOT/results/build-config.json"
+          echo "BASELINE_SHA=$baseline_sha" >> "$GITHUB_ENV"
+          echo "CANDIDATE_SHA=$candidate_sha" >> "$GITHUB_ENV"
+          echo "PERF_HARNESS_SHA=$(git -C "$harness_source" rev-parse HEAD)" >> "$GITHUB_ENV"
+
+      - name: Install hosted runner dependencies
+        if: runner.environment == 'github-hosted'
+        timeout-minutes: 10
+        working-directory: baseline
+        run: sudo bash dependencies.sh -y
+
+      - name: Build both revisions with the same Release configuration
+        timeout-minutes: 35
+        run: |
+          set -euo pipefail
+          for revision in baseline candidate; do
+            cmake -S "$revision" -B "$revision/build-perf" -G Ninja \
+              -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_COMPILER=c++ \
+              -DBUILD_UNIT_TESTS=OFF -DBUILD_EXAMPLES=OFF -DBUILD_BENCHMARK=ON \
+              -DWITH_STORE_RUST=OFF -DUSE_CUDA=OFF -DUSE_ETCD=OFF \
+              -DSTORE_USE_ETCD=OFF -DENABLE_DEBUG_SYMBOLS=OFF \
+              2>&1 | tee "$PERF_ROOT/results/$revision-configure.log"
+            cmake --build "$revision/build-perf" --parallel 2 \
+              --target offset_allocator_concurrency_bench \
+              2>&1 | tee "$PERF_ROOT/results/$revision-build.log"
+          done
+
+      - name: Collect alternating baseline and candidate samples
+        timeout-minutes: 16
+        run: |
+          python3 "$PERF_ROOT/harness/perf_gate.py" collect \
+            --baseline baseline/build-perf/mooncake-store/benchmarks/offset_allocator_concurrency_bench \
+            --candidate candidate/build-perf/mooncake-store/benchmarks/offset_allocator_concurrency_bench \
+            --baseline-sha "$BASELINE_SHA" --candidate-sha "$CANDIDATE_SHA" \
+            --collection-timeout-seconds 900 \
+            --output "$PERF_ROOT/results"
+
+      - name: Publish comparison and enforce thresholds
+        if: ${{ always() }}
+        run: |
+          mkdir -p "$PERF_ROOT/results"
+          if [ ! -f "$PERF_ROOT/harness/perf_gate.py" ]; then
+            echo '## Performance Gate: ERROR — setup or harness tests failed' > "$PERF_ROOT/results/summary.md"
+            cat "$PERF_ROOT/results/summary.md" >> "$GITHUB_STEP_SUMMARY"
+            echo '::error::Performance gate setup failed; no valid measurement is available.'
+            exit 2
+          fi
+          cp "$PERF_ROOT/harness/policy.json" "$PERF_ROOT/results/policy.json"
+          python3 "$PERF_ROOT/harness/perf_gate.py" compare \
+            --baseline "$PERF_ROOT/results/baseline.json" --candidate "$PERF_ROOT/results/candidate.json" \
+            --baseline-sha "${BASELINE_SHA:-}" --candidate-sha "${CANDIDATE_SHA:-}" \
+            --output "$PERF_ROOT/results" --summary "$GITHUB_STEP_SUMMARY"
+
+      - name: Preserve raw samples and diagnostics
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: performance-results
+          path: ${{ env.PERF_ROOT }}/results/
+          if-no-files-found: error
+          retention-days: 14

.github/workflows/rdma-performance.yml

@@ -0,0 +1,89 @@
+name: RDMA performance regression
+
+on:
+  pull_request:
+    branches: [main, 'release/**']
+    types: [opened, synchronize, reopened]
+  workflow_dispatch:
+    inputs:
+      baseline_sha:
+        description: Full baseline commit SHA (same as candidate for A/A calibration)
+        required: true
+      candidate_sha:
+        description: Full candidate commit SHA
+        required: true
+  workflow_call:
+    inputs:
+      baseline_sha:
+        required: true
+        type: string
+      candidate_sha:
+        required: true
+        type: string
+    secrets:
+      TONE_USER_NAME:
+        required: true
+      TONE_USER_TOKEN:
+        required: true
+
+permissions:
+  contents: read
+
+# One fixed hardware pair; retain up to 100 waiting runs without interrupting it.
+concurrency:
+  group: rdma-performance-pair
+  cancel-in-progress: false
+  queue: max
+
+jobs:
+  rdma:
+    name: RDMA Performance Gate
+    if: ${{ github.event_name != 'pull_request' || vars.RDMA_PERF_ENABLED == 'true' }}
+    runs-on: ubuntu-22.04
+    timeout-minutes: 60
+    steps:
+      - name: Checkout the reviewed harness
+        uses: actions/checkout@v4
+        with:
+          # Never execute candidate code in the credential-bearing controller.
+          # Manual bootstrap uses the selected workflow revision, reviewed by its dispatcher.
+          ref: ${{ github.event.pull_request.base.sha || github.workflow_sha }}
+          persist-credentials: false
+
+      - name: Submit, collect, and enforce RDMA thresholds
+        env:
+          BASELINE_SHA: ${{ inputs.baseline_sha || github.event.pull_request.base.sha }}
+          CANDIDATE_SHA: ${{ inputs.candidate_sha || github.sha }}
+          TONE_USER_NAME: ${{ secrets.TONE_USER_NAME }}
+          TONE_USER_TOKEN: ${{ secrets.TONE_USER_TOKEN }}
+          TONE_WORKSPACE: ${{ vars.TONE_WORKSPACE }}
+          TONE_PROJECT: ${{ vars.TONE_PROJECT }}
+          TONE_RDMA_TEMPLATE: ${{ vars.TONE_RDMA_TEMPLATE }}
+          TONE_RESULT_ORIGIN: ${{ vars.TONE_RESULT_ORIGIN }}
+          RUN_ID: ${{ github.run_id }}-${{ github.run_attempt }}
+        run: |
+          python3 scripts/ci/performance/tone_rdma.py \
+            --baseline-sha "$BASELINE_SHA" --candidate-sha "$CANDIDATE_SHA" \
+            --harness-sha "$(git rev-parse HEAD)" --run-id "$RUN_ID" \
+            --output rdma-performance-results
+
+      - name: Publish setup errors
+        if: ${{ always() }}
+        run: |
+          if [ ! -f rdma-performance-results/summary.md ]; then
+            mkdir -p rdma-performance-results
+            echo '## RDMA Performance Gate: ERROR — no complete report was returned' \
+              > rdma-performance-results/summary.md
+            cat rdma-performance-results/summary.md >> "$GITHUB_STEP_SUMMARY"
+            echo '::error::RDMA setup failed or the run was interrupted.'
+            exit 2
+          fi
+
+      - name: Preserve results even on failure
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: rdma-performance-results
+          path: rdma-performance-results/
+          if-no-files-found: error
+          retention-days: 14

CONTRIBUTING.md

@@ -130,6 +130,10 @@ git commit -m "wip: skipping hooks" --no-verify
 But please avoid using `--no-verify` for routine commits to keep code quality high.
 
 #### CI Integration
+The [performance regression gate](scripts/ci/performance/README.md) compares
+baseline and candidate CPU benchmarks, publishes a result table in the Actions
+summary, and fails on regressions beyond the configured thresholds.
+
 GitHub pull-request and push checks validate only added or modified C/C++ line
 ranges relative to the selected base revision. This avoids failing a focused
 change solely because an otherwise untouched part of the same file has older

mooncake-transfer-engine/benchmark/tent_backend.cpp

@@ -163,6 +163,13 @@ static int resolveSegTypeParams(const std::string& seg_type,
     if (seg_type == "DRAM" || seg_type == "dram") {
         device_prefix = "cpu";
         num_buffers = numa_num_configured_nodes();
+        if (XferBenchConfig::dram_numa_node != -1) {
+            start_idx = XferBenchConfig::dram_numa_node;
+            LOG_ASSERT(start_idx >= 0 && start_idx <= numa_max_node() &&
+                       numa_bitmask_isbitset(numa_all_nodes_ptr, start_idx))
+                << "dram_numa_node is not an available NUMA node";
+            num_buffers = 1;
+        }
 #if defined(USE_CUDA) || defined(USE_SUNRISE)
     } else if (seg_type == "VRAM" || seg_type == "vram") {
         device_prefix = "cuda";

mooncake-transfer-engine/benchmark/utils.cpp

@@ -99,6 +99,8 @@ DEFINE_bool(deadline_bw_arbitration, false,
 DEFINE_int32(local_gpu_id, 0,
              "Local device ordinal for VRAM, -1 for all visible devices. "
              "Selects an MLU device with --backend=tent in USE_MLU builds.");
+DEFINE_int32(dram_numa_node, -1,
+             "TENT DRAM NUMA node, -1 allocates on all configured nodes");
 DEFINE_int32(target_gpu_id, 0,
              "Offset for round-robin assignment across target buffers; "
              "-1 uses 0.");
@@ -182,6 +184,7 @@ std::string XferBenchConfig::tent_transport_hint;
 std::string XferBenchConfig::tent_intent_type;
 
 int XferBenchConfig::local_gpu_id = 0;
+int XferBenchConfig::dram_numa_node = -1;
 int XferBenchConfig::target_gpu_id = 0;
 
 namespace {
@@ -274,6 +277,7 @@ void XferBenchConfig::loadFromFlags() {
     tent_intent_type = FLAGS_tent_intent_type;
 
     local_gpu_id = FLAGS_local_gpu_id;
+    dram_numa_node = FLAGS_dram_numa_node;
     target_gpu_id = FLAGS_target_gpu_id;
 }
 

mooncake-transfer-engine/benchmark/utils.h

@@ -104,6 +104,7 @@ struct XferBenchConfig {
     static std::string tent_intent_type;
 
     static int local_gpu_id;
+    static int dram_numa_node;
     static int target_gpu_id;
 };