test: Catch alert rules that break only inside vmalert

.github/workflows/test-alert-rules.yaml

@@ -0,0 +1,51 @@
+name: Test Alert Rules
+
+# Evaluates the alert component's rules with vmalert-tool, at the vmalert
+# version the platform runs, so a rule that only fails inside vmalert cannot
+# merge.
+
+on:
+  pull_request:
+    paths:
+      - '.github/workflows/test-alert-rules.yaml'
+      - 'hack/test-alert-rules'
+      - 'config/components/alerts/**'
+  push:
+    branches:
+      - main
+    paths:
+      - '.github/workflows/test-alert-rules.yaml'
+      - 'hack/test-alert-rules'
+      - 'config/components/alerts/**'
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.ref || github.sha }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+
+env:
+  # Keep in step with the vmalert tag in datum-cloud/infra
+  # infrastructure/monitoring/victoria-metrics/base/vmalert.yaml. vmalert only
+  # schedules the rules: production runs their queries in vmselect (v1.129.1),
+  # so MetricsQL changes between the two versions are not covered here.
+  VMUTILS_VERSION: v1.121.0
+  VMUTILS_SHA256: 09bb214fa89e3f4523815e839ad2beccedd601d515a6c93906007dc62d364e18
+
+jobs:
+  test-alert-rules:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install vmalert-tool
+        run: |
+          curl -fsSL --retry 3 -o "$RUNNER_TEMP/vmutils.tar.gz" "https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/${VMUTILS_VERSION}/vmutils-linux-amd64-${VMUTILS_VERSION}.tar.gz"
+          echo "${VMUTILS_SHA256}  $RUNNER_TEMP/vmutils.tar.gz" | sha256sum -c -
+          tar -xzf "$RUNNER_TEMP/vmutils.tar.gz" -C "$RUNNER_TEMP" vmalert-tool-prod
+
+      - name: Run rule unit tests
+        run: hack/test-alert-rules
+        env:
+          VMALERT_TOOL: ${{ runner.temp }}/vmalert-tool-prod
+
+      - name: Validate the alerts component builds
+        run: kubectl kustomize config/overlays/management-plane >/dev/null && kubectl kustomize config/overlays/single-cluster >/dev/null

config/components/alerts/tests/capacity.test.yaml

@@ -0,0 +1,100 @@
+rule_files:
+  - capacity.rules.yaml
+
+evaluation_interval: 20s
+
+tests:
+  # us-central-1 has its one host ready. us-east-1 has one of two ready, plus a
+  # cordoned host that counts toward neither. us-west-1 has none ready: one
+  # host's kraftlet virtual node is not Ready, and the other is down, its
+  # kubelet silent since the first minute but its Node object still present.
+  - interval: 20s
+    input_series:
+      - series: 'kubelet_node_name{cluster="us-central-1-alice",region="us-central-1",node="compute-a",compute_datumapis_com_runtime="kata"}'
+        values: "1x40"
+      - series: 'kubelet_node_name{cluster="us-east-1-alice",region="us-east-1",node="compute-b1",compute_datumapis_com_runtime="kata"}'
+        values: "1x40"
+      - series: 'kubelet_node_name{cluster="us-east-1-alice",region="us-east-1",node="compute-b2",compute_datumapis_com_runtime="kata"}'
+        values: "1x40"
+      - series: 'kubelet_node_name{cluster="us-east-1-alice",region="us-east-1",node="compute-b3",compute_datumapis_com_runtime="kata"}'
+        values: "1x40"
+      - series: 'kubelet_node_name{cluster="us-west-1-alice",region="us-west-1",node="compute-c1",compute_datumapis_com_runtime="kata"}'
+        values: "1x40"
+      - series: 'kubelet_node_name{cluster="us-west-1-alice",region="us-west-1",node="compute-c2",compute_datumapis_com_runtime="kata"}'
+        values: "1x2 _x38"
+
+      - series: 'kube_node_info{cluster="us-central-1-alice",region="us-central-1",node="compute-a"}'
+        values: "1x40"
+      - series: 'kube_node_info{cluster="us-central-1-alice",region="us-central-1",node="kraftlet-compute-a"}'
+        values: "1x40"
+      - series: 'kube_node_info{cluster="us-east-1-alice",region="us-east-1",node="compute-b1"}'
+        values: "1x40"
+      - series: 'kube_node_info{cluster="us-east-1-alice",region="us-east-1",node="kraftlet-compute-b1"}'
+        values: "1x40"
+      - series: 'kube_node_info{cluster="us-east-1-alice",region="us-east-1",node="compute-b2"}'
+        values: "1x40"
+      - series: 'kube_node_info{cluster="us-east-1-alice",region="us-east-1",node="kraftlet-compute-b2"}'
+        values: "1x40"
+      - series: 'kube_node_info{cluster="us-east-1-alice",region="us-east-1",node="compute-b3"}'
+        values: "1x40"
+      - series: 'kube_node_info{cluster="us-west-1-alice",region="us-west-1",node="compute-c1"}'
+        values: "1x40"
+      - series: 'kube_node_info{cluster="us-west-1-alice",region="us-west-1",node="kraftlet-compute-c1"}'
+        values: "1x40"
+      - series: 'kube_node_info{cluster="us-west-1-alice",region="us-west-1",node="compute-c2"}'
+        values: "1x40"
+
+      - series: 'kube_node_spec_unschedulable{cluster="us-central-1-alice",region="us-central-1",node="compute-a"}'
+        values: "0x40"
+      - series: 'kube_node_spec_unschedulable{cluster="us-east-1-alice",region="us-east-1",node="compute-b1"}'
+        values: "0x40"
+      - series: 'kube_node_spec_unschedulable{cluster="us-east-1-alice",region="us-east-1",node="compute-b2"}'
+        values: "0x40"
+      - series: 'kube_node_spec_unschedulable{cluster="us-east-1-alice",region="us-east-1",node="compute-b3"}'
+        values: "1x40"
+      - series: 'kube_node_spec_unschedulable{cluster="us-west-1-alice",region="us-west-1",node="compute-c1"}'
+        values: "0x40"
+      - series: 'kube_node_spec_unschedulable{cluster="us-west-1-alice",region="us-west-1",node="compute-c2"}'
+        values: "0x40"
+
+      - series: 'kube_node_status_condition{cluster="us-central-1-alice",region="us-central-1",node="compute-a",condition="Ready",status="true"}'
+        values: "1x40"
+      - series: 'kube_node_status_condition{cluster="us-central-1-alice",region="us-central-1",node="kraftlet-compute-a",condition="Ready",status="true"}'
+        values: "1x40"
+      - series: 'kube_node_status_condition{cluster="us-east-1-alice",region="us-east-1",node="compute-b1",condition="Ready",status="true"}'
+        values: "1x40"
+      - series: 'kube_node_status_condition{cluster="us-east-1-alice",region="us-east-1",node="kraftlet-compute-b1",condition="Ready",status="true"}'
+        values: "1x40"
+      - series: 'kube_node_status_condition{cluster="us-east-1-alice",region="us-east-1",node="compute-b2",condition="Ready",status="true"}'
+        values: "0x40"
+      - series: 'kube_node_status_condition{cluster="us-east-1-alice",region="us-east-1",node="kraftlet-compute-b2",condition="Ready",status="true"}'
+        values: "1x40"
+      - series: 'kube_node_status_condition{cluster="us-east-1-alice",region="us-east-1",node="compute-b3",condition="Ready",status="true"}'
+        values: "1x40"
+      - series: 'kube_node_status_condition{cluster="us-west-1-alice",region="us-west-1",node="compute-c1",condition="Ready",status="true"}'
+        values: "1x40"
+      - series: 'kube_node_status_condition{cluster="us-west-1-alice",region="us-west-1",node="kraftlet-compute-c1",condition="Ready",status="true"}'
+        values: "0x40"
+      - series: 'kube_node_status_condition{cluster="us-west-1-alice",region="us-west-1",node="compute-c2",condition="Ready",status="true"}'
+        values: "0x40"
+
+    metricsql_expr_test:
+      - expr: cell:compute_hosts:count
+        eval_time: 10m
+        exp_samples:
+          - labels: 'cell:compute_hosts:count{alerting_environment="test",cluster="us-central-1-alice",region="us-central-1"}'
+            value: 1
+          - labels: 'cell:compute_hosts:count{alerting_environment="test",cluster="us-east-1-alice",region="us-east-1"}'
+            value: 2
+          - labels: 'cell:compute_hosts:count{alerting_environment="test",cluster="us-west-1-alice",region="us-west-1"}'
+            value: 2
+
+      - expr: cell:compute_hosts_ready:count
+        eval_time: 10m
+        exp_samples:
+          - labels: 'cell:compute_hosts_ready:count{alerting_environment="test",cluster="us-central-1-alice",region="us-central-1"}'
+            value: 1
+          - labels: 'cell:compute_hosts_ready:count{alerting_environment="test",cluster="us-east-1-alice",region="us-east-1"}'
+            value: 1
+          - labels: 'cell:compute_hosts_ready:count{alerting_environment="test",cluster="us-west-1-alice",region="us-west-1"}'
+            value: 0

config/components/alerts/tests/reconcile-storm.test.yaml

@@ -0,0 +1,14 @@
+rule_files:
+  - reconcile-storm.rules.yaml
+
+evaluation_interval: 20s
+
+tests:
+  # Loads the rules so a broken expression fails here; no behaviour asserted.
+  - interval: 20s
+    input_series: []
+    metricsql_expr_test:
+      - expr: vector(1)
+        eval_time: 5m
+        exp_samples:
+          - value: 1

config/components/alerts/tests/shell-sessions.test.yaml

@@ -0,0 +1,14 @@
+rule_files:
+  - shell-sessions.rules.yaml
+
+evaluation_interval: 20s
+
+tests:
+  # Loads the rules so a broken expression fails here; no behaviour asserted.
+  - interval: 20s
+    input_series: []
+    metricsql_expr_test:
+      - expr: vector(1)
+        eval_time: 5m
+        exp_samples:
+          - value: 1

hack/test-alert-rules

@@ -0,0 +1,132 @@
+#!/usr/bin/env bash
+#
+# Runs the vmalert-tool unit tests in config/components/alerts/tests against the
+# VMRules beside them. vmalert adds its external labels to every series a rule
+# writes, and a rule that reads its group's own output back sees them, so the
+# tests run with one set the way the deployed vmalerts do.
+#
+# vmalert-tool loads only the rule files a test names, so every VMRule must be
+# named by at least one test or this script fails.
+#
+# Needs yq (v4) and vmalert-tool; set VMALERT_TOOL when it is not on PATH.
+
+set -euo pipefail
+
+REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+readonly REPO_ROOT
+cd "$REPO_ROOT"
+
+readonly RULES_DIR=config/components/alerts
+readonly TESTS_DIR="${RULES_DIR}/tests"
+readonly EXTERNAL_LABEL="alerting_environment=test"
+readonly VMALERT_TOOL="${VMALERT_TOOL:-vmalert-tool}"
+
+if ! command -v yq >/dev/null 2>&1; then
+  echo "FAIL: yq (v4) is not on PATH"
+  exit 1
+fi
+yq_version="$(yq --version 2>&1 || true)"
+if ! grep -qE 'mikefarah/yq.* version v?4\.' <<<"${yq_version}"; then
+  echo "FAIL: need mikefarah yq v4, found: ${yq_version}"
+  exit 1
+fi
+if ! command -v "${VMALERT_TOOL}" >/dev/null 2>&1; then
+  echo "FAIL: vmalert-tool not found at '${VMALERT_TOOL}'; install it or set VMALERT_TOOL"
+  exit 1
+fi
+
+work="$(mktemp -d)"
+readonly work
+trap 'rm -rf -- "${work}"' EXIT
+
+query() {
+  local file="$1" result
+  shift
+  if ! result="$(yq "$@" "${file}" 2>&1)"; then
+    echo "FAIL: cannot read ${file}: ${result}" >&2
+    return 1
+  fi
+  printf '%s\n' "${result}"
+}
+
+shopt -s nullglob
+
+rule_files=()
+manifests=()
+for manifest in "${RULES_DIR}"/*.yaml "${RULES_DIR}"/*.yml; do
+  documents="$(query "${manifest}" ea '[.] | length')"
+  if [[ "${documents}" -ne 1 ]]; then
+    echo "FAIL: ${manifest} holds ${documents} YAML documents; keep one manifest per file"
+    exit 1
+  fi
+  kind="$(query "${manifest}" '.kind')"
+  [[ "${kind}" == "VMRule" ]] || continue
+  name="${manifest##*/}"
+  name="${name%.yaml}"
+  name="${name%.yml}.rules.yaml"
+  if [[ -e "${work}/${name}" ]]; then
+    echo "FAIL: two VMRule files in ${RULES_DIR} map to ${name}"
+    exit 1
+  fi
+  query "${manifest}" '.spec' >"${work}/${name}"
+  groups="$(query "${work}/${name}" '.groups | length')"
+  if [[ "${groups}" -eq 0 ]]; then
+    echo "FAIL: ${manifest} has no rule groups"
+    exit 1
+  fi
+  rule_files+=("${name}")
+  manifests+=("${manifest}")
+done
+
+if [[ "${#rule_files[@]}" -eq 0 ]]; then
+  echo "FAIL: no VMRule found in ${RULES_DIR}"
+  exit 1
+fi
+
+files=()
+referenced=""
+for test_file in "${TESTS_DIR}"/*.test.yaml; do
+  [[ -e "${test_file}" ]] || continue
+  tests="$(query "${test_file}" '.tests | length')"
+  if [[ "${tests}" -eq 0 ]]; then
+    echo "FAIL: ${test_file} has no tests"
+    exit 1
+  fi
+  names="$(query "${test_file}" '.rule_files[]')"
+  referenced+="${names}"$'\n'
+  cp "${test_file}" "${work}/"
+  files+=("--files=$(basename "${test_file}")")
+done
+
+if [[ "${#files[@]}" -eq 0 ]]; then
+  echo "FAIL: no *.test.yaml found in ${TESTS_DIR}"
+  exit 1
+fi
+
+untested=0
+for i in "${!rule_files[@]}"; do
+  if ! grep -qxF "${rule_files[i]}" <<<"${referenced}"; then
+    echo "FAIL: no test in ${TESTS_DIR} lists ${rule_files[i]} in rule_files (from ${manifests[i]})"
+    untested=$((untested + 1))
+  fi
+done
+if [[ "${untested}" -gt 0 ]]; then
+  exit 1
+fi
+
+cd "${work}"
+rc=0
+output="$("${VMALERT_TOOL}" unittest --external.label="${EXTERNAL_LABEL}" "${files[@]}" 2>&1)" || rc=$?
+printf '%s\n' "${output}"
+
+if [[ "${rc}" -ne 0 ]] &&
+  grep -q 'nil pointer dereference' <<<"${output}" &&
+  grep -qE '\(\*(Recording|Alerting)Rule\)\.exec' <<<"${output}"; then
+  echo ""
+  echo "HINT: vmalert-tool panics when a rule fails to evaluate, instead of"
+  echo "printing the evaluation error. The usual cause is two series with one"
+  echo "labelset once vmalert adds its external label (datum-cloud/compute#513):"
+  echo "match an 'or' fallback 'on' the labels the computed side keeps."
+fi
+
+exit "${rc}"

observability/README.md

@@ -8,7 +8,12 @@ Never edit the rendered JSON by hand: CI re-renders it and fails on any
 difference.
 
 Alert rules are plain `VMRule` manifests under `config/components/alerts/`,
-with a runbook per alert under `docs/runbooks/`.
+with a runbook per alert under `docs/runbooks/`. Their unit tests live in
+`config/components/alerts/tests/` and run with `hack/test-alert-rules`, which
+needs `yq` and `vmalert-tool`. Every `VMRule` must be named in some test's
+`rule_files`. When a rule fails to evaluate, vmalert-tool panics with a nil
+pointer dereference instead of printing the error; the usual cause is two
+series with one labelset once vmalert adds its external label (#513).
 
 ## Layout