r9lab/compose.telemetry.gpu.yml at main · CMLPlatform/r9lab · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
# compose.telemetry.gpu.yml, VENDORED from the CML monitoring repository at a pinned tag.
#
# OPTIONAL OVERLAY: GPU metrics on a host with NVIDIA hardware. The Alloy agent
# discovers and scrapes the exporter automatically.
#
#   docker compose ... -f compose.telemetry.yml -f compose.telemetry.gpu.yml up -d
#
# Deploy tooling that builds its `-f` list from the host's `.env` should key on
# `GPU_METRICS=1`. Nothing in these templates reads it, but one name across every spoke
# keeps the runbooks identical, and the switch has to exist: including this overlay on a
# host without the NVIDIA container runtime makes `up` abort with "could not select
# device driver".
#
# nvidia_gpu_exporter, not dcgm-exporter: DCGM's profiling fields are datacentre-only,
# and on a consumer card it keeps only its SYS_ADMIN requirement.

services:
  nvidia-gpu-exporter:
    image: utkuozdemir/nvidia_gpu_exporter:1.15.1-nvml@sha256:92a73ec03e2f5cae4a7cfcc99c242b46797741c24de8f3768d394925f8e0a323
    restart: unless-stopped
    # The NVML backend reports XID error counters; the nvidia-smi backend cannot.
    environment:
      NVIDIA_DRIVER_CAPABILITIES: utility
    deploy:
      resources:
        reservations:
          devices:
            - driver: nvidia
              count: all
              capabilities: [gpu]
    mem_limit: 256m
    pids_limit: 256
    read_only: true
    cap_drop:
      - ALL
    security_opt:
      - no-new-privileges:true
    # No published ports: Alloy reaches it over the Compose network.
    networks:
      - egress
    logging:
      driver: json-file
      options:
        max-size: "10m"
        max-file: "3"