diff --git a/.github/workflows/bazel.yml b/.github/workflows/bazel.yml index abdc06131..85e481356 100644 --- a/.github/workflows/bazel.yml +++ b/.github/workflows/bazel.yml @@ -88,6 +88,7 @@ jobs: run: | bash tools/ci/test-bazel-cache-upload-mode bash tools/ci/test-bazel-remote-probe + bash tools/ci/test-image-push-manual - name: Compute changed subtrees id: detect diff --git a/.github/workflows/image-push-manual.yml b/.github/workflows/image-push-manual.yml index f0b710a47..95cc60d81 100644 --- a/.github/workflows/image-push-manual.yml +++ b/.github/workflows/image-push-manual.yml @@ -167,6 +167,8 @@ jobs: exit 1 fi echo "discovered: ${indexes[*]}" + declare -A seen_repos=() + repos=() for tgt in "${indexes[@]}"; do name="${tgt##*:}"; name="${name%_index}" # Two naming conventions exist in the tree and they mean different @@ -186,6 +188,20 @@ jobs: *-image) repo="${name%-image}" ;; *) sub="$(printf '%s' "$name" | tr '_' '-')"; repo="${svc}-${sub}" ;; esac + if [ -n "${seen_repos[$repo]:-}" ]; then + echo "ERROR: ${tgt} and ${seen_repos[$repo]} both map to image repository ${repo}" >&2 + echo "Give each oci_image_index target a distinct image name before publishing." >&2 + exit 1 + fi + seen_repos["$repo"]="$tgt" + repos+=("$repo") + done + # Validate every target-to-repository mapping before starting any + # push. A later collision must not leave earlier repositories with + # a new snapshot or latest-dispatch tag. + for i in "${!indexes[@]}"; do + tgt="${indexes[$i]}" + repo="${repos[$i]}" dest="${REGISTRY}/${repo}" echo "[push] ${tgt} -> ${dest}:${TAG} (+ latest-dispatch)" mkdir -p ci-ghcr diff --git a/deploy/helm/gateway-routes/README.md b/deploy/helm/gateway-routes/README.md index 306823874..e589464ba 100644 --- a/deploy/helm/gateway-routes/README.md +++ b/deploy/helm/gateway-routes/README.md @@ -4,9 +4,17 @@ This repository contains the Helm chart for deploying NVCF ingress routes via th ## Overview -The chart deploys `HTTPRoute`, `TCPRoute`, and `ReferenceGrant` resources that attach to an existing Gateway provisioned separately by the cluster operator (e.g. Envoy Gateway, Istio, Traefik, Kong). It also includes optional `PodMonitor` resources for scraping Envoy Gateway proxy metrics with Prometheus. - -The chart deploys routing configuration only. It does not include any container images. Backend services referenced by the routes (`api`, `nvct-api`, `api-keys`, `invocation`, `llm-api-gateway`, `vanity-gateway`, `reval`, `sis`, `grpc`, `nats`) must already be deployed separately. +The chart deploys `HTTPRoute`, `GRPCRoute`, `TCPRoute`, `UDPRoute`, and +`ReferenceGrant` resources that attach to an existing Gateway provisioned +separately by the cluster operator, such as Envoy Gateway, Istio, Traefik, or +Kong. It also includes optional `PodMonitor` resources for scraping Envoy +Gateway proxy metrics with Prometheus. + +The chart deploys routing configuration only. It does not include any +container images. Backend services referenced by the routes (`api`, +`nvct-api`, `api-keys`, `invocation`, `llm-api-gateway`, +`llm-request-router-backend-router`, `vanity-gateway`, `reval`, `sis`, `grpc`, +`nats`) must already be deployed separately. ## Prerequisites @@ -14,7 +22,8 @@ The chart deploys routing configuration only. It does not include any container - Helm 3.x - `kubectl` - A Gateway API compatible controller installed in the cluster -- An existing `Gateway` resource with an HTTP listener (and TCP listeners if the gRPC or NATS routes are enabled) +- Existing `Gateway` resources with the listeners required by each enabled route +- A Gateway controller with `UDPRoute` support when LLM worker routing is enabled - The backend services that the routes target, deployed in their respective namespaces ## Getting Started @@ -57,6 +66,8 @@ Important settings to review before deployment: - `nvcfGatewayRoutes.gateways.shared.*` for the HTTP Gateway name, namespace, and listener - `nvcfGatewayRoutes.gateways.grpc.*` for the TCP Gateway name, namespace, and listener - `nvcfGatewayRoutes.gateways.nats.*` for the NATS TCP Gateway name, namespace, and listener +- `nvcfGatewayRoutes.gateways.llmGrpc.*` for the LLM worker gRPC TCP listener +- `nvcfGatewayRoutes.gateways.llmQuic.*` for the LLM reverse-tunnel UDP listener - `nvcfGatewayRoutes.routes..enabled` to toggle individual routes - `nvcfGatewayRoutes.routes.nvcfApi.grpc.enabled` and `nvcfGatewayRoutes.routes.nvctApi.grpc.enabled` to expose API gRPC routes @@ -87,6 +98,7 @@ Enabled `HTTPRoute` entries must not share a resolved hostname because each `HTT | `grpc` | TCPRoute | Not rendered | `grpc.nvcf:10081` | | `grpcWorker` | TCPRoute (disabled by default) | Not rendered | `grpc.nvcf:10086` | | `nats` | TCPRoute (disabled by default) | Not rendered | `nats.nats-system:4222` | +| `llmWorker` | TCPRoute and UDPRoute (disabled by default) | Not rendered | `llm-request-router-backend-router.:50071/TCP,50072/UDP` | Cross-namespace routing is supported via `ReferenceGrant` resources rendered into each backend namespace. @@ -97,3 +109,9 @@ Cross-namespace routing is supported via `ReferenceGrant` resources rendered int - The `grpc` TCPRoute does not enforce HTTP hostname matching at the Gateway layer. Configure DNS or TCP load balancer routing outside this chart. - The `grpcWorker` TCPRoute is beta support for split or multi-cluster gRPC worker callbacks. It carries HTTP/1 CONNECT callback traffic only. Enable it only when the control-plane grpc-proxy runs one replica with HPA disabled. Multi-replica grpc-proxy requires pod-specific callback routing and is not supported by this shared TCPRoute. - Enabling the `nats` route requires a reachable TCP listener for NATS on the referenced Gateway. The HTTP Gateway address does not imply NATS reachability unless that same Gateway also has the NATS TCP listener configured. +- The `llmWorker` routes target Stargate's authority/SNI-aware backend router. + Set `nvcfGatewayRoutes.routes.llmWorker.backend.namespace` to the effective + namespace of the `llm-request-router` release. The gateway chart cannot + derive the namespace of a separate Helm release. + Keep the TCP and UDP Gateways separate when the infrastructure requires + separate load balancers for each protocol. diff --git a/deploy/helm/gateway-routes/chart/templates/_helpers.tpl b/deploy/helm/gateway-routes/chart/templates/_helpers.tpl index e22ae145d..124712752 100644 --- a/deploy/helm/gateway-routes/chart/templates/_helpers.tpl +++ b/deploy/helm/gateway-routes/chart/templates/_helpers.tpl @@ -53,6 +53,10 @@ app.kubernetes.io/name: {{ include "nvcf-gateway.name" . }} app.kubernetes.io/instance: {{ .Release.Name }} {{- end }} +{{- define "nvcf-gateway.llmWorkerBackendNamespace" -}} +{{- required "nvcfGatewayRoutes.routes.llmWorker.backend.namespace is required when llmWorker.enabled is true" .Values.nvcfGatewayRoutes.routes.llmWorker.backend.namespace -}} +{{- end }} + {{/* Validate that enabled HTTPRoutes do not compete for the same hostname and root PathPrefix match on the shared Gateway. All HTTPRoute templates in this diff --git a/deploy/helm/gateway-routes/chart/templates/referencegrant-llm-worker.yaml b/deploy/helm/gateway-routes/chart/templates/referencegrant-llm-worker.yaml new file mode 100644 index 000000000..fe7f592ad --- /dev/null +++ b/deploy/helm/gateway-routes/chart/templates/referencegrant-llm-worker.yaml @@ -0,0 +1,25 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +{{- if and .Values.nvcfGatewayRoutes.enabled .Values.nvcfGatewayRoutes.routes.llmWorker.enabled }} +{{- $backendNamespace := include "nvcf-gateway.llmWorkerBackendNamespace" . }} +apiVersion: gateway.networking.k8s.io/v1beta1 +kind: ReferenceGrant +metadata: + name: allow-llm-worker-routes + namespace: {{ $backendNamespace }} + labels: + {{- include "nvcf-gateway.labels" . | nindent 4 }} +spec: + from: + - group: gateway.networking.k8s.io + kind: TCPRoute + namespace: {{ .Values.nvcfGatewayRoutes.gateways.llmGrpc.namespace }} + - group: gateway.networking.k8s.io + kind: UDPRoute + namespace: {{ .Values.nvcfGatewayRoutes.gateways.llmQuic.namespace }} + to: + - group: "" + kind: Service + name: {{ .Values.nvcfGatewayRoutes.routes.llmWorker.backend.name }} +{{- end }} diff --git a/deploy/helm/gateway-routes/chart/templates/tcproute-llm-worker.yaml b/deploy/helm/gateway-routes/chart/templates/tcproute-llm-worker.yaml new file mode 100644 index 000000000..892ec696e --- /dev/null +++ b/deploy/helm/gateway-routes/chart/templates/tcproute-llm-worker.yaml @@ -0,0 +1,28 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +{{- if and .Values.nvcfGatewayRoutes.enabled .Values.nvcfGatewayRoutes.routes.llmWorker.enabled }} +{{- $backendNamespace := include "nvcf-gateway.llmWorkerBackendNamespace" . }} +apiVersion: gateway.networking.k8s.io/v1alpha2 +kind: TCPRoute +metadata: + name: {{ .Values.nvcfGatewayRoutes.routes.llmWorker.name }}-grpc + namespace: {{ .Values.nvcfGatewayRoutes.gateways.llmGrpc.namespace }} + labels: + {{- include "nvcf-gateway.labels" . | nindent 4 }} + app.kubernetes.io/component: llm-worker-grpc-route + {{- with .Values.nvcfGatewayRoutes.routes.llmWorker.routeAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + parentRefs: + - name: {{ .Values.nvcfGatewayRoutes.gateways.llmGrpc.name }} + namespace: {{ .Values.nvcfGatewayRoutes.gateways.llmGrpc.namespace }} + sectionName: {{ .Values.nvcfGatewayRoutes.gateways.llmGrpc.listenerName }} + rules: + - backendRefs: + - name: {{ .Values.nvcfGatewayRoutes.routes.llmWorker.backend.name }} + namespace: {{ $backendNamespace }} + port: {{ .Values.nvcfGatewayRoutes.routes.llmWorker.backend.grpcPort }} +{{- end }} diff --git a/deploy/helm/gateway-routes/chart/templates/udproute-llm-worker.yaml b/deploy/helm/gateway-routes/chart/templates/udproute-llm-worker.yaml new file mode 100644 index 000000000..cba18b322 --- /dev/null +++ b/deploy/helm/gateway-routes/chart/templates/udproute-llm-worker.yaml @@ -0,0 +1,28 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +{{- if and .Values.nvcfGatewayRoutes.enabled .Values.nvcfGatewayRoutes.routes.llmWorker.enabled }} +{{- $backendNamespace := include "nvcf-gateway.llmWorkerBackendNamespace" . }} +apiVersion: gateway.networking.k8s.io/v1alpha2 +kind: UDPRoute +metadata: + name: {{ .Values.nvcfGatewayRoutes.routes.llmWorker.name }}-quic + namespace: {{ .Values.nvcfGatewayRoutes.gateways.llmQuic.namespace }} + labels: + {{- include "nvcf-gateway.labels" . | nindent 4 }} + app.kubernetes.io/component: llm-worker-quic-route + {{- with .Values.nvcfGatewayRoutes.routes.llmWorker.routeAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + parentRefs: + - name: {{ .Values.nvcfGatewayRoutes.gateways.llmQuic.name }} + namespace: {{ .Values.nvcfGatewayRoutes.gateways.llmQuic.namespace }} + sectionName: {{ .Values.nvcfGatewayRoutes.gateways.llmQuic.listenerName }} + rules: + - backendRefs: + - name: {{ .Values.nvcfGatewayRoutes.routes.llmWorker.backend.name }} + namespace: {{ $backendNamespace }} + port: {{ .Values.nvcfGatewayRoutes.routes.llmWorker.backend.quicPort }} +{{- end }} diff --git a/deploy/helm/gateway-routes/chart/values.yaml b/deploy/helm/gateway-routes/chart/values.yaml index 462b53485..63c982fd1 100644 --- a/deploy/helm/gateway-routes/chart/values.yaml +++ b/deploy/helm/gateway-routes/chart/values.yaml @@ -51,6 +51,17 @@ nvcfGatewayRoutes: namespace: gateway # Listener (sectionName) on the Gateway to attach the route to listenerName: nats + # TCP Gateway for LLM worker registration and Stargate watches. + llmGrpc: + name: llm-grpc-gateway + namespace: gateway + listenerName: llm-grpc + # UDP Gateway for LLM reverse tunnels. This may be a separate load + # balancer from the TCP Gateway. + llmQuic: + name: llm-quic-gateway + namespace: gateway + listenerName: llm-quic # HTTPRoute configurations routes: @@ -239,6 +250,21 @@ nvcfGatewayRoutes: namespace: nats-system port: 4222 routeAnnotations: {} + + # Backend-facing LLM registration/watch and reverse-tunnel routes. + # The backend router selects the correct Stargate pod by gRPC authority + # and QUIC SNI, so this route supports multi-replica request routers. + llmWorker: + enabled: false + name: llm-worker + backend: + name: llm-request-router-backend-router + # Required when llmWorker is enabled. Set this to the effective + # llm-request-router namespace. + namespace: "" + grpcPort: 50071 + quicPort: 50072 + routeAnnotations: {} # NVCF Worker Container needs to be able to fetch secrets for function pods # In split/multi cluster planes, ess resides in control-plane and requires diff --git a/deploy/helm/gateway-routes/scripts/check-llm-worker-routes.sh b/deploy/helm/gateway-routes/scripts/check-llm-worker-routes.sh new file mode 100755 index 000000000..df606ce0d --- /dev/null +++ b/deploy/helm/gateway-routes/scripts/check-llm-worker-routes.sh @@ -0,0 +1,81 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +chart_dir="${script_dir}/../chart" +rendered="$(mktemp)" +disabled="$(mktemp)" +trap 'rm -f "$rendered" "$disabled"' EXIT + +helm template nvcf-gateway-routes "$chart_dir" \ + --namespace gateway \ + --set nvcfGatewayRoutes.routes.llmWorker.enabled=true \ + --set nvcfGatewayRoutes.gateways.llmGrpc.name=llm-grpc-gateway \ + --set nvcfGatewayRoutes.gateways.llmGrpc.namespace=gateway \ + --set nvcfGatewayRoutes.gateways.llmQuic.name=llm-quic-gateway \ + --set nvcfGatewayRoutes.gateways.llmQuic.namespace=gateway \ + --set nvcfGatewayRoutes.routes.llmWorker.backend.namespace=router-system \ + >"$rendered" + +assert_contains() { + local pattern="$1" + local message="$2" + if ! grep -Fq -- "$pattern" "$rendered"; then + echo "FAIL: ${message}" >&2 + exit 1 + fi +} + +assert_contains "kind: TCPRoute" \ + "LLM worker routing must expose gRPC registration over TCP" +assert_contains "kind: UDPRoute" \ + "LLM worker routing must expose reverse tunnels over UDP" +assert_contains "name: llm-request-router-backend-router" \ + "LLM worker routes must target the authority/SNI-aware backend router" +assert_contains "name: allow-llm-worker-routes" \ + "ReferenceGrant must permit cross-namespace LLM worker routes" +assert_contains "sectionName: llm-grpc" \ + "TCPRoute must attach to the configured LLM gRPC listener" +assert_contains "sectionName: llm-quic" \ + "UDPRoute must attach to the configured LLM QUIC listener" + +reference_grant_service_name="$(awk ' + $0 == "kind: ReferenceGrant" { in_grant = 1; target_grant = 0; in_to = 0 } + in_grant && !target_grant && $1 == "name:" && $2 == "allow-llm-worker-routes" { target_grant = 1 } + target_grant && $0 == " to:" { in_to = 1 } + target_grant && in_to && $1 == "name:" { print $2; exit } +' "$rendered")" +if [[ "$reference_grant_service_name" != "llm-request-router-backend-router" ]]; then + echo "FAIL: LLM worker ReferenceGrant must stay scoped to the configured backend Service" >&2 + exit 1 +fi + +backend_namespace_references="$(grep -Fc -- "namespace: router-system" "$rendered")" +if [[ "$backend_namespace_references" != "3" ]]; then + echo "FAIL: LLM worker routes and ReferenceGrant must use the configured backend namespace" >&2 + exit 1 +fi + +if helm template nvcf-gateway-routes "$chart_dir" \ + --namespace gateway \ + --set nvcfGatewayRoutes.routes.llmWorker.enabled=true \ + --set-string nvcfGatewayRoutes.routes.llmWorker.backend.namespace= \ + >/dev/null 2>&1; then + echo "FAIL: enabled LLM worker routing must require an explicit backend namespace" >&2 + exit 1 +fi + +helm template nvcf-gateway-routes "$chart_dir" \ + --namespace gateway \ + --set nvcfGatewayRoutes.routes.llmWorker.enabled=false \ + >"$disabled" + +if grep -Eq '^ name: (llm-worker-(grpc|quic)|allow-llm-worker-routes)$' "$disabled"; then + echo "FAIL: disabled LLM worker routing must not render route resources" >&2 + exit 1 +fi + +echo "PASS: LLM worker Gateway routes render correctly" diff --git a/deploy/helm/llm-request-router/README.md b/deploy/helm/llm-request-router/README.md index 3f8d6b15b..20d818cfc 100644 --- a/deploy/helm/llm-request-router/README.md +++ b/deploy/helm/llm-request-router/README.md @@ -4,7 +4,17 @@ This repository contains the Helm chart for deploying the NVCF LLM Request Route ## Overview -The chart packages the LLM Request Router StatefulSet with HTTP and gRPC services, a metrics endpoint, and a headless service for multi-instance DNS discovery. A Vault Agent sidecar is configured to fetch a service token from a Vault or OpenBao backend; the application reads `nvcfApiToken` from `/vault/secrets/secrets.json` and attaches it as a Bearer token to outgoing worker authentication gRPC calls. +The chart packages the LLM Request Router StatefulSet with HTTP and gRPC +services, a metrics endpoint, and a headless service for multi-instance DNS +discovery. It can also deploy the Stargate Kubernetes backend router for +worker gRPC registration and reverse QUIC tunnels through a shared Gateway or +load balancer. The backend router selects the correct Stargate pod from gRPC +authority and QUIC SNI. + +A Vault Agent sidecar is configured to fetch a service token from a Vault or +OpenBao backend. The application reads `nvcfApiToken` from +`/vault/secrets/secrets.json` and attaches it as a Bearer token to outgoing +worker authentication gRPC calls. The default chart values do not set the required image registry and repository. They must be supplied through an additional values file at install time, and access to those images must be arranged separately. @@ -70,6 +80,7 @@ Important settings to review before deployment: - `llmRequestRouter.imagePullSecrets` for private registry access - `llmRequestRouter.replicaCount`, resource requests, and limits for your environment - `llmRequestRouter.service.*` for HTTP, gRPC, metrics, and headless service ports +- `llmRequestRouter.backendRouter.*` for multi-replica worker gRPC and reverse-tunnel routing - `llmRequestRouter.metrics.enabled` to expose the metrics port on the Service (default: `false`) - `llmRequestRouter.metrics.serviceMonitor.enabled` to create a Prometheus `ServiceMonitor` (requires `metrics.enabled`) - `llmRequestRouter.certificate.*` to let cert-manager issue the Stargate QUIC server certificate @@ -80,6 +91,51 @@ Important settings to review before deployment: The default values include development-oriented placeholders. Override them before using the chart in any shared or production environment. +## Backend Worker Routing + +Enable `llmRequestRouter.backendRouter.enabled` when workers reach a +multi-replica request router through a shared endpoint. Set both pylon dial +addresses to the external endpoints that workers can resolve: + +```yaml +llmRequestRouter: + backendRouter: + enabled: true + image: + tag: + pylonGrpcDialAddress: llm-router.example.com:443 + pylonReverseTunnelDialAddress: llm-router.example.com:8080 +``` + +The chart uses the main Stargate image for both workloads. The image must +contain `/usr/local/bin/stargate-k8s-router`. Set +`llmRequestRouter.backendRouter.image.tag` to an image version that contains +that binary. The chart requires this explicit pin when backend routing is +enabled. + +The backend router watches EndpointSlices. The chart creates a dedicated +ServiceAccount by default and binds a namespaced Role to it when +`llmRequestRouter.rbac.create=true`. When +`llmRequestRouter.backendRouter.serviceAccount.create=false`, set +`llmRequestRouter.backendRouter.serviceAccount.name` to an existing account. +When `rbac.create=false`, grant `get`, `list`, and `watch` on +`discovery.k8s.io/endpointslices` to that account outside this chart. + +Route TCP port `50071` and UDP port `50072` to the +`llm-request-router-backend-router` Service. The NVCF gateway-routes chart can +create the matching `TCPRoute`, `UDPRoute`, and `ReferenceGrant` resources. +The Gateway implementation must support Gateway API `UDPRoute`. + +When QUIC verification is enabled, the mounted certificate must cover the +worker-facing reverse-tunnel hostname and the per-pod hostname template. The +default template is +`{pod_name}.llm-request-router-headless..svc.cluster.local`. + +Stargate and the backend router read the TLS certificate and key only during +process startup. After cert-manager or another issuer renews the Secret, +restart both workloads or configure a Secret reloader that triggers their +rollouts. + ## Load Balancer Configuration The chart can pass a Stargate load-balancer config in either of two ways: diff --git a/deploy/helm/llm-request-router/llm-request-router/templates/_helpers.tpl b/deploy/helm/llm-request-router/llm-request-router/templates/_helpers.tpl index 91c01a068..a637b7eee 100644 --- a/deploy/helm/llm-request-router/llm-request-router/templates/_helpers.tpl +++ b/deploy/helm/llm-request-router/llm-request-router/templates/_helpers.tpl @@ -49,6 +49,25 @@ app.kubernetes.io/instance: {{ .Release.Name }} {{- printf "app.kubernetes.io/name=%s,app.kubernetes.io/instance=%s" (include "llm-request-router.name" .) .Release.Name -}} {{- end }} +{{- define "llm-request-router.backendRouterName" -}} +{{- printf "%s-backend-router" (include "llm-request-router.fullname" .) | trunc 63 | trimSuffix "-" -}} +{{- end }} + +{{- define "llm-request-router.backendRouterSelectorLabels" -}} +app.kubernetes.io/name: {{ include "llm-request-router.backendRouterName" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{- define "llm-request-router.backendRouterLabels" -}} +helm.sh/chart: {{ include "llm-request-router.chart" . }} +{{ include "llm-request-router.backendRouterSelectorLabels" . }} +app.kubernetes.io/component: backend-router +{{- with .Values.llmRequestRouter.backendRouter.image.tag }} +app.kubernetes.io/version: {{ . | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + {{- define "llm-request-router.namespace" -}} {{- default .Release.Namespace .Values.llmRequestRouter.namespace -}} {{- end -}} @@ -61,6 +80,15 @@ app.kubernetes.io/instance: {{ .Release.Name }} {{- end }} {{- end }} +{{- define "llm-request-router.backendRouterServiceAccountName" -}} +{{- $serviceAccount := .Values.llmRequestRouter.backendRouter.serviceAccount | default dict -}} +{{- if $serviceAccount.create -}} +{{- default (include "llm-request-router.backendRouterName" .) $serviceAccount.name -}} +{{- else -}} +{{- required "llmRequestRouter.backendRouter.serviceAccount.name is required when backendRouter is enabled and backendRouter.serviceAccount.create is false" $serviceAccount.name -}} +{{- end -}} +{{- end }} + {{- define "llm-request-router.image" -}} {{- $registry := .Values.llmRequestRouter.image.registry -}} {{- $repository := .Values.llmRequestRouter.image.repository -}} @@ -72,6 +100,33 @@ app.kubernetes.io/instance: {{ .Release.Name }} {{- end -}} {{- end }} +{{- define "llm-request-router.advertisedHostnameTemplate" -}} +{{- $configured := .Values.llmRequestRouter.kubernetes.advertisedHostnameTemplate -}} +{{- $backendRouterEnabled := dig "backendRouter" "enabled" false .Values.llmRequestRouter -}} +{{- if and $backendRouterEnabled $configured (ne (len (splitList "{pod_name}" $configured)) 2) -}} +{{- fail "llmRequestRouter.kubernetes.advertisedHostnameTemplate must contain exactly one {pod_name} when backendRouter.enabled is true" -}} +{{- end -}} +{{- if $configured -}} +{{- $configured -}} +{{- else if or $backendRouterEnabled (gt (.Values.llmRequestRouter.replicaCount | int) 1) -}} +{{- printf "{pod_name}.%s.%s.svc.cluster.local" .Values.llmRequestRouter.service.headlessName (include "llm-request-router.namespace" .) -}} +{{- else -}} +{{- printf "%s.%s.svc.cluster.local" (include "llm-request-router.fullname" .) (include "llm-request-router.namespace" .) -}} +{{- end -}} +{{- end }} + +{{- define "llm-request-router.backendRouterImage" -}} +{{- $image := .Values.llmRequestRouter.backendRouter.image -}} +{{- $registry := default .Values.llmRequestRouter.image.registry $image.registry -}} +{{- $repository := default .Values.llmRequestRouter.image.repository $image.repository -}} +{{- $tag := required "llmRequestRouter.backendRouter.image.tag is required when backendRouter.enabled is true" $image.tag -}} +{{- if $registry -}} +{{- printf "%s/%s:%s" $registry $repository $tag -}} +{{- else -}} +{{- printf "%s:%s" $repository $tag -}} +{{- end -}} +{{- end }} + {{- define "llm-request-router.pkiMigrationsImage" -}} {{- $img := .Values.llmRequestRouter.pki.image -}} {{- $registry := $img.registry -}} @@ -107,6 +162,39 @@ app.kubernetes.io/instance: {{ .Release.Name }} {{- end -}} {{- end }} +{{- define "llm-request-router.validateTlsCertKeyDir" -}} +{{- $tls := .Values.llmRequestRouter.tls | default dict -}} +{{- if and $tls.certPath $tls.keyPath (ne (dir $tls.certPath) (dir $tls.keyPath)) -}} +{{- fail "llmRequestRouter.tls.certPath and llmRequestRouter.tls.keyPath must use the same directory" -}} +{{- end -}} +{{- end }} + +{{- define "llm-request-router.validateBackendRouterTls" -}} +{{- $tls := .Values.llmRequestRouter.tls | default dict -}} +{{- $secretName := include "llm-request-router.tlsSecretName" . -}} +{{- $hasSecret := not (empty $secretName) -}} +{{- $hasCert := not (empty $tls.certPath) -}} +{{- $hasKey := not (empty $tls.keyPath) -}} +{{- $hasAny := or $hasSecret $hasCert $hasKey -}} +{{- $hasAll := and $hasSecret $hasCert $hasKey -}} +{{- if and $hasAny (not $hasAll) -}} +{{- fail "llmRequestRouter backend routing requires tls.secretName (or certificate secret), tls.certPath, and tls.keyPath together" -}} +{{- end -}} +{{- if and (not $tls.quicInsecure) (not $hasAll) -}} +{{- fail "llmRequestRouter backend routing requires a TLS Secret and cert/key paths when tls.quicInsecure is false" -}} +{{- end -}} +{{- if $hasAll -}} +{{- include "llm-request-router.validateTlsCertKeyDir" . -}} +{{- end -}} +{{- if and $hasAll (ne (clean (include "llm-request-router.tlsMountPath" .)) (clean (dir $tls.certPath))) -}} +{{- fail "llmRequestRouter.tls.mountPath must match the directory containing tls.certPath and tls.keyPath" -}} +{{- end -}} +{{- end }} + +{{- define "llm-request-router.validateBackendRouterServiceAccount" -}} +{{- $_ := include "llm-request-router.backendRouterServiceAccountName" . -}} +{{- end }} + {{/* Vault Annotations */}} diff --git a/deploy/helm/llm-request-router/llm-request-router/templates/backend-router-rbac.yaml b/deploy/helm/llm-request-router/llm-request-router/templates/backend-router-rbac.yaml new file mode 100644 index 000000000..6e8dae34a --- /dev/null +++ b/deploy/helm/llm-request-router/llm-request-router/templates/backend-router-rbac.yaml @@ -0,0 +1,32 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +{{- if and .Values.llmRequestRouter.backendRouter.enabled .Values.llmRequestRouter.rbac.create }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ include "llm-request-router.backendRouterName" . }}-endpointslice-reader + namespace: {{ include "llm-request-router.namespace" . }} + labels: + {{- include "llm-request-router.backendRouterLabels" . | nindent 4 }} +rules: + - apiGroups: ["discovery.k8s.io"] + resources: ["endpointslices"] + verbs: ["get", "list", "watch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ include "llm-request-router.backendRouterName" . }}-endpointslice-reader + namespace: {{ include "llm-request-router.namespace" . }} + labels: + {{- include "llm-request-router.backendRouterLabels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ include "llm-request-router.backendRouterName" . }}-endpointslice-reader +subjects: + - kind: ServiceAccount + name: {{ include "llm-request-router.backendRouterServiceAccountName" . }} + namespace: {{ include "llm-request-router.namespace" . }} +{{- end }} diff --git a/deploy/helm/llm-request-router/llm-request-router/templates/backend-router-serviceaccount.yaml b/deploy/helm/llm-request-router/llm-request-router/templates/backend-router-serviceaccount.yaml new file mode 100644 index 000000000..e92bcdb85 --- /dev/null +++ b/deploy/helm/llm-request-router/llm-request-router/templates/backend-router-serviceaccount.yaml @@ -0,0 +1,16 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +{{- if and .Values.llmRequestRouter.backendRouter.enabled .Values.llmRequestRouter.backendRouter.serviceAccount.create }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "llm-request-router.backendRouterServiceAccountName" . }} + namespace: {{ include "llm-request-router.namespace" . }} + labels: + {{- include "llm-request-router.backendRouterLabels" . | nindent 4 }} + {{- with .Values.llmRequestRouter.backendRouter.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/deploy/helm/llm-request-router/llm-request-router/templates/backend-router-servicemonitor.yaml b/deploy/helm/llm-request-router/llm-request-router/templates/backend-router-servicemonitor.yaml new file mode 100644 index 000000000..a06c9e7eb --- /dev/null +++ b/deploy/helm/llm-request-router/llm-request-router/templates/backend-router-servicemonitor.yaml @@ -0,0 +1,24 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +{{- $metricsEnabled := dig "metrics" "enabled" false .Values.llmRequestRouter }} +{{- $serviceMonitorEnabled := dig "metrics" "serviceMonitor" "enabled" false .Values.llmRequestRouter }} +{{- if and .Values.llmRequestRouter.backendRouter.enabled $metricsEnabled $serviceMonitorEnabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ include "llm-request-router.backendRouterName" . }}-metrics + labels: + {{- include "llm-request-router.backendRouterLabels" . | nindent 4 }} +spec: + endpoints: + - interval: {{ dig "metrics" "serviceMonitor" "interval" "30s" .Values.llmRequestRouter | quote }} + path: /metrics + port: health + namespaceSelector: + matchNames: + - {{ include "llm-request-router.namespace" . }} + selector: + matchLabels: + {{- include "llm-request-router.backendRouterSelectorLabels" . | nindent 6 }} +{{- end }} diff --git a/deploy/helm/llm-request-router/llm-request-router/templates/backend-router.yaml b/deploy/helm/llm-request-router/llm-request-router/templates/backend-router.yaml new file mode 100644 index 000000000..30260b4ea --- /dev/null +++ b/deploy/helm/llm-request-router/llm-request-router/templates/backend-router.yaml @@ -0,0 +1,143 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +{{- if .Values.llmRequestRouter.backendRouter.enabled }} +{{- include "llm-request-router.validateBackendRouterTls" . }} +{{- include "llm-request-router.validateBackendRouterServiceAccount" . }} +{{- $advertisedHostnameTemplate := include "llm-request-router.advertisedHostnameTemplate" . }} +{{- $tlsSecretName := include "llm-request-router.tlsSecretName" . }} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "llm-request-router.backendRouterName" . }} + namespace: {{ include "llm-request-router.namespace" . }} + labels: + {{- include "llm-request-router.backendRouterLabels" . | nindent 4 }} +spec: + selector: + {{- include "llm-request-router.backendRouterSelectorLabels" . | nindent 4 }} + ports: + - name: grpc + port: {{ .Values.llmRequestRouter.backendRouter.service.grpcPort }} + targetPort: grpc + protocol: TCP + - name: quic + port: {{ .Values.llmRequestRouter.backendRouter.service.reverseTunnelPort }} + targetPort: quic + protocol: UDP + - name: health + port: {{ .Values.llmRequestRouter.backendRouter.service.healthPort }} + targetPort: health + protocol: TCP +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "llm-request-router.backendRouterName" . }} + namespace: {{ include "llm-request-router.namespace" . }} + labels: + {{- include "llm-request-router.backendRouterLabels" . | nindent 4 }} +spec: + replicas: {{ .Values.llmRequestRouter.backendRouter.replicaCount }} + selector: + matchLabels: + {{- include "llm-request-router.backendRouterSelectorLabels" . | nindent 6 }} + template: + metadata: + labels: + {{- include "llm-request-router.backendRouterSelectorLabels" . | nindent 8 }} + app.kubernetes.io/component: backend-router + {{- with .Values.llmRequestRouter.backendRouter.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.llmRequestRouter.backendRouter.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + serviceAccountName: {{ include "llm-request-router.backendRouterServiceAccountName" . }} + {{- with .Values.llmRequestRouter.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + securityContext: + {{- toYaml .Values.llmRequestRouter.podSecurityContext | nindent 8 }} + containers: + - name: backend-router + image: {{ include "llm-request-router.backendRouterImage" . }} + imagePullPolicy: {{ default .Values.llmRequestRouter.image.pullPolicy .Values.llmRequestRouter.backendRouter.image.pullPolicy }} + command: + - /usr/local/bin/stargate-k8s-router + args: + - --listen-addr=0.0.0.0:{{ .Values.llmRequestRouter.backendRouter.service.grpcPort }} + - --reverse-tunnel-listen-addr=0.0.0.0:{{ .Values.llmRequestRouter.backendRouter.service.reverseTunnelPort }} + - --health-listen-addr=0.0.0.0:{{ .Values.llmRequestRouter.backendRouter.service.healthPort }} + - --target-namespace={{ include "llm-request-router.namespace" . }} + - --target-service-name={{ include "llm-request-router.fullname" . }} + - --advertised-hostname-template={{ $advertisedHostnameTemplate }} + - --grpc-port-name=grpc + - --quic-port-name=quic + - --tunnel-protocol=raw-quic + {{- with .Values.llmRequestRouter.tls.certPath }} + - --tls-cert-path={{ . }} + {{- end }} + {{- with .Values.llmRequestRouter.tls.keyPath }} + - --tls-key-path={{ . }} + {{- end }} + {{- if .Values.llmRequestRouter.tls.quicInsecure }} + - --quic-insecure + {{- end }} + ports: + - name: grpc + containerPort: {{ .Values.llmRequestRouter.backendRouter.service.grpcPort }} + protocol: TCP + - name: quic + containerPort: {{ .Values.llmRequestRouter.backendRouter.service.reverseTunnelPort }} + protocol: UDP + - name: health + containerPort: {{ .Values.llmRequestRouter.backendRouter.service.healthPort }} + protocol: TCP + readinessProbe: + httpGet: + path: /readyz + port: health + periodSeconds: 2 + livenessProbe: + httpGet: + path: /livez + port: health + periodSeconds: 5 + resources: + {{- toYaml .Values.llmRequestRouter.backendRouter.resources | nindent 12 }} + securityContext: + {{- toYaml .Values.llmRequestRouter.securityContext | nindent 12 }} + {{- if and $tlsSecretName .Values.llmRequestRouter.tls.certPath .Values.llmRequestRouter.tls.keyPath }} + volumeMounts: + - name: stargate-tls + mountPath: {{ include "llm-request-router.tlsMountPath" . | quote }} + readOnly: true + {{- end }} + {{- if and $tlsSecretName .Values.llmRequestRouter.tls.certPath .Values.llmRequestRouter.tls.keyPath }} + volumes: + - name: stargate-tls + secret: + secretName: {{ $tlsSecretName | quote }} + items: + - key: tls.crt + path: {{ base .Values.llmRequestRouter.tls.certPath | quote }} + - key: tls.key + path: {{ base .Values.llmRequestRouter.tls.keyPath | quote }} + {{- end }} + {{- with .Values.llmRequestRouter.backendRouter.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.llmRequestRouter.backendRouter.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.llmRequestRouter.backendRouter.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} +{{- end }} diff --git a/deploy/helm/llm-request-router/llm-request-router/templates/certificate.yaml b/deploy/helm/llm-request-router/llm-request-router/templates/certificate.yaml index 4584b9f42..62b6413a6 100644 --- a/deploy/helm/llm-request-router/llm-request-router/templates/certificate.yaml +++ b/deploy/helm/llm-request-router/llm-request-router/templates/certificate.yaml @@ -16,6 +16,17 @@ {{ $certificate := .Values.llmRequestRouter.certificate | default dict -}} {{- if $certificate.enabled }} {{- $issuerRef := $certificate.issuerRef | default dict -}} +{{- $dnsNames := dig "dnsNames" (list) $certificate -}} +{{- if dig "backendRouter" "enabled" false .Values.llmRequestRouter }} +{{- $advertisedHostnameTemplate := include "llm-request-router.advertisedHostnameTemplate" . -}} +{{- $backendRouterWildcardDNSName := replace "{pod_name}" "*" $advertisedHostnameTemplate -}} +{{- if not (has $backendRouterWildcardDNSName $dnsNames) -}} +{{- $dnsNames = append $dnsNames $backendRouterWildcardDNSName -}} +{{- end -}} +{{- end }} +{{- if empty $dnsNames -}} +{{- fail "llmRequestRouter.certificate.dnsNames is required when certificate.enabled is true" -}} +{{- end }} apiVersion: cert-manager.io/v1 kind: Certificate metadata: @@ -31,7 +42,7 @@ spec: renewBefore: {{ . | quote }} {{- end }} dnsNames: - {{- required "llmRequestRouter.certificate.dnsNames is required when certificate.enabled is true" $certificate.dnsNames | toYaml | nindent 4 }} + {{- $dnsNames | toYaml | nindent 4 }} issuerRef: kind: {{ default "ClusterIssuer" $issuerRef.kind | quote }} name: {{ required "llmRequestRouter.certificate.issuerRef.name is required when certificate.enabled is true" $issuerRef.name | quote }} diff --git a/deploy/helm/llm-request-router/llm-request-router/templates/deployment.yaml b/deploy/helm/llm-request-router/llm-request-router/templates/deployment.yaml index ea06e5ab4..9fc8a23b6 100644 --- a/deploy/helm/llm-request-router/llm-request-router/templates/deployment.yaml +++ b/deploy/helm/llm-request-router/llm-request-router/templates/deployment.yaml @@ -31,13 +31,10 @@ tunnel target. {{- if and $disableDnsDiscovery (gt $replicaCount 1) }} {{- fail "llmRequestRouter.discovery.disableDnsDiscovery cannot be true when llmRequestRouter.replicaCount is greater than 1; multi-replica routers require DNS discovery" }} {{- end }} -{{- $advertisedHostnameTemplate := .Values.llmRequestRouter.kubernetes.advertisedHostnameTemplate }} -{{- if not $advertisedHostnameTemplate }} -{{- if eq $replicaCount 1 }} -{{- $advertisedHostnameTemplate = printf "%s.%s.svc.cluster.local" (include "llm-request-router.fullname" .) (include "llm-request-router.namespace" .) }} -{{- else }} -{{- $advertisedHostnameTemplate = printf "{pod_name}.%s.%s.svc.cluster.local" .Values.llmRequestRouter.service.headlessName (include "llm-request-router.namespace" .) }} -{{- end }} +{{- $advertisedHostnameTemplate := include "llm-request-router.advertisedHostnameTemplate" . }} +{{- $backendRouterEnabled := dig "backendRouter" "enabled" false .Values.llmRequestRouter }} +{{- if and $backendRouterEnabled (not .Values.llmRequestRouter.transport.reverseTunnelListenAddr) }} +{{- fail "llmRequestRouter.backendRouter.enabled requires llmRequestRouter.transport.reverseTunnelListenAddr" }} {{- end }} spec: serviceName: {{ .Values.llmRequestRouter.service.headlessName }} @@ -97,7 +94,10 @@ spec: - --backend-connectivity=reverse - --reverse-tunnel-listen-addr={{ .Values.llmRequestRouter.transport.reverseTunnelListenAddr }} {{- end }} - {{- if and .Values.llmRequestRouter.transport.reverseTunnelListenAddr (gt $replicaCount 1) }} + {{- if $backendRouterEnabled }} + - --grpc-pylon-dial-addr={{ required "llmRequestRouter.backendRouter.pylonGrpcDialAddress is required when backendRouter.enabled is true" .Values.llmRequestRouter.backendRouter.pylonGrpcDialAddress }} + - --reverse-tunnel-pylon-dial-addr={{ required "llmRequestRouter.backendRouter.pylonReverseTunnelDialAddress is required when backendRouter.enabled is true" .Values.llmRequestRouter.backendRouter.pylonReverseTunnelDialAddress }} + {{- else if and .Values.llmRequestRouter.transport.reverseTunnelListenAddr (gt $replicaCount 1) }} - --reverse-tunnel-pylon-dial-addr=$(POD_IP):{{ .Values.llmRequestRouter.service.reverseTunnelPort }} {{- end }} {{- if .Values.llmRequestRouter.transport.reverseTunnelConnectTimeoutMs }} @@ -180,9 +180,7 @@ spec: {{- end }} {{- $tlsSecretNameMount := include "llm-request-router.tlsSecretName" . }} {{- if and $tlsSecretNameMount .Values.llmRequestRouter.tls.certPath .Values.llmRequestRouter.tls.keyPath }} - {{- if ne (dir .Values.llmRequestRouter.tls.certPath) (dir .Values.llmRequestRouter.tls.keyPath) }} - {{- fail "llmRequestRouter.tls.certPath and llmRequestRouter.tls.keyPath must use the same directory" }} - {{- end }} + {{- include "llm-request-router.validateTlsCertKeyDir" . }} - name: stargate-tls mountPath: {{ include "llm-request-router.tlsMountPath" . | quote }} readOnly: true diff --git a/deploy/helm/llm-request-router/llm-request-router/values.yaml b/deploy/helm/llm-request-router/llm-request-router/values.yaml index 77a7f57d8..ee6368177 100644 --- a/deploy/helm/llm-request-router/llm-request-router/values.yaml +++ b/deploy/helm/llm-request-router/llm-request-router/values.yaml @@ -64,6 +64,43 @@ llmRequestRouter: reverseTunnelPort: 50072 headlessName: llm-request-router-headless + # Routes backend registration and reverse-tunnel traffic to the Stargate pod + # named by gRPC authority or QUIC SNI. Enable this when pylons connect through + # a shared Gateway or load balancer and request-router replicaCount is greater + # than one. + backendRouter: + enabled: false + # One replica is a single point of failure. Use two or more replicas when + # backend worker routing requires high availability. + replicaCount: 1 + image: + registry: "" + repository: "" + tag: "" + pullPolicy: "" + pylonGrpcDialAddress: "" + pylonReverseTunnelDialAddress: "" + serviceAccount: + create: true + annotations: {} + name: "" + service: + grpcPort: 50071 + reverseTunnelPort: 50072 + healthPort: 8080 + podAnnotations: {} + podLabels: {} + resources: + requests: + cpu: 50m + memory: 64Mi + limits: + cpu: 500m + memory: 256Mi + nodeSelector: {} + tolerations: [] + affinity: {} + resources: requests: cpu: 100m @@ -110,8 +147,8 @@ llmRequestRouter: # suffixes). REQUIRED when enabled. Typically: # ",cluster.local" # so the signing role accepts both the customer's external DNS and - # in-cluster service names. Missing value → script aborts non-zero → - # entrypoint accumulator → Job fails on backoff exhaustion. + # in-cluster service names. Missing value -> script aborts non-zero -> + # entrypoint accumulator -> Job fails on backoff exhaustion. allowedDomains: "" # nvcf-openbao-migrations image. The chart picks up the same image # that the k8s-openbao Helm hook uses; supply registry/repository/tag diff --git a/deploy/helm/llm-request-router/scripts/check-backend-router-render.sh b/deploy/helm/llm-request-router/scripts/check-backend-router-render.sh new file mode 100755 index 000000000..6b8cb1061 --- /dev/null +++ b/deploy/helm/llm-request-router/scripts/check-backend-router-render.sh @@ -0,0 +1,355 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +chart_dir="${script_dir}/../llm-request-router" +rendered="$(mktemp)" +disabled="$(mktemp)" +external_service_account="$(mktemp)" +wildcard_certificate="$(mktemp)" +trap 'rm -f "$rendered" "$disabled" "$external_service_account" "$wildcard_certificate"' EXIT + +helm template llm-request-router "$chart_dir" \ + --namespace nvcf \ + --set llmRequestRouter.image.registry=registry.example.invalid \ + --set llmRequestRouter.image.repository=nvcf/stargate \ + --set llmRequestRouter.backendRouter.enabled=true \ + --set llmRequestRouter.backendRouter.image.tag=next \ + --set llmRequestRouter.backendRouter.pylonGrpcDialAddress=llm-router.example.invalid:443 \ + --set llmRequestRouter.backendRouter.pylonReverseTunnelDialAddress=llm-router.example.invalid:8080 \ + --set llmRequestRouter.certificate.enabled=true \ + --set llmRequestRouter.certificate.issuerRef.name=test-issuer \ + --set 'llmRequestRouter.certificate.dnsNames[0]=llm-request-router.nvcf.svc.cluster.local' \ + --set llmRequestRouter.tls.secretName=stargate-quic-tls \ + --set llmRequestRouter.tls.certPath=/etc/stargate/tls/tls.crt \ + --set llmRequestRouter.tls.keyPath=/etc/stargate/tls/tls.key \ + --set llmRequestRouter.tls.quicInsecure=false \ + --set llmRequestRouter.metrics.enabled=true \ + --set llmRequestRouter.metrics.serviceMonitor.enabled=true \ + >"$rendered" + +assert_contains() { + local pattern="$1" + local message="$2" + if ! grep -Fq -- "$pattern" "$rendered"; then + echo "FAIL: ${message}" >&2 + exit 1 + fi +} + +assert_backend_router_replicas() { + local expected="$1" + local actual + actual="$(awk ' + $0 == "kind: Deployment" { in_deployment = 1; backend_router = 0 } + in_deployment && $1 == "name:" && $2 == "llm-request-router-backend-router" { backend_router = 1 } + backend_router && $1 == "replicas:" { print $2; exit } + ' "$rendered")" + if [[ "$actual" != "$expected" ]]; then + echo "FAIL: backend router must default to ${expected} replica; rendered ${actual:-none}" >&2 + exit 1 + fi +} + +assert_backend_router_role_binding_subject() { + local rendered_file="$1" + local expected="$2" + local actual + actual="$(awk ' + $0 == "kind: RoleBinding" { in_binding = 1; target_binding = 0; in_subjects = 0 } + in_binding && !target_binding && $1 == "name:" && $2 == "llm-request-router-backend-router-endpointslice-reader" { target_binding = 1 } + target_binding && $0 == "subjects:" { in_subjects = 1 } + target_binding && in_subjects && $1 == "name:" { print $2; exit } + ' "$rendered_file")" + if [[ "$actual" != "$expected" ]]; then + echo "FAIL: backend router RoleBinding must target ${expected}; rendered ${actual:-none}" >&2 + exit 1 + fi +} + +assert_service_account_exists() { + local rendered_file="$1" + local expected="$2" + if ! awk -v expected="$expected" ' + $0 == "---" { in_service_account = 0 } + $0 == "kind: ServiceAccount" { in_service_account = 1 } + in_service_account && $1 == "name:" && $2 == expected { found = 1 } + END { exit found ? 0 : 1 } + ' "$rendered_file"; then + echo "FAIL: chart must render the dedicated ${expected} ServiceAccount" >&2 + exit 1 + fi +} + +assert_contains "name: llm-request-router-backend-router" \ + "backend router workload and Service must use a stable name" +assert_contains "kind: Deployment" \ + "backend router must render as a Deployment" +assert_backend_router_replicas "1" +assert_contains "kind: Role" \ + "backend router must render namespaced RBAC" +assert_contains "resources: [\"endpointslices\"]" \ + "backend router must be allowed to watch EndpointSlices" +assert_contains "serviceAccountName: llm-request-router-backend-router" \ + "backend router must use its dedicated ServiceAccount" +assert_service_account_exists "$rendered" "llm-request-router-backend-router" +assert_backend_router_role_binding_subject "$rendered" "llm-request-router-backend-router" +assert_contains "command:" \ + "backend router must override the Stargate image entrypoint" +assert_contains "/usr/local/bin/stargate-k8s-router" \ + "Stargate image must include the Kubernetes router binary" +assert_contains "--target-service-name=llm-request-router" \ + "backend router must watch the readiness-respecting request-router Service" +assert_contains "--advertised-hostname-template={pod_name}.llm-request-router-headless.nvcf.svc.cluster.local" \ + "backend router authority and SNI template must match Stargate" +assert_contains "- '*.llm-request-router-headless.nvcf.svc.cluster.local'" \ + "request-router certificate must cover pod-specific backend routing hostnames" +assert_contains "image: registry.example.invalid/nvcf/stargate:next" \ + "backend router must use its explicitly pinned Stargate image" +assert_contains "app.kubernetes.io/version: \"next\"" \ + "backend router labels must identify the explicitly pinned image version" +assert_contains "--grpc-pylon-dial-addr=llm-router.example.invalid:443" \ + "Stargate must advertise the external gRPC endpoint to pylon" +assert_contains "--reverse-tunnel-pylon-dial-addr=llm-router.example.invalid:8080" \ + "Stargate must advertise the external reverse-tunnel endpoint to pylon" +assert_contains "--tls-cert-path=/etc/stargate/tls/tls.crt" \ + "backend router must use the Stargate TLS certificate" +assert_contains "secretName: \"stargate-quic-tls\"" \ + "backend router must mount the configured Stargate TLS Secret" +assert_contains "name: llm-request-router-backend-router-metrics" \ + "backend router metrics must be discoverable by the existing ServiceMonitor option" + +helm template llm-request-router "$chart_dir" \ + --namespace nvcf \ + --set llmRequestRouter.image.registry=registry.example.invalid \ + --set llmRequestRouter.image.repository=nvcf/stargate \ + --set llmRequestRouter.backendRouter.enabled=false \ + >"$disabled" + +if grep -Fq "llm-request-router-backend-router" "$disabled"; then + echo "FAIL: disabled backend router must not render router resources" >&2 + exit 1 +fi + +if helm template llm-request-router "$chart_dir" \ + --namespace nvcf \ + --set llmRequestRouter.image.registry=registry.example.invalid \ + --set llmRequestRouter.image.repository=nvcf/stargate \ + --set llmRequestRouter.backendRouter.enabled=true \ + --set llmRequestRouter.backendRouter.image.tag=next \ + >/dev/null 2>&1; then + echo "FAIL: enabled backend router must require pylon dial addresses" >&2 + exit 1 +fi + +if helm template llm-request-router "$chart_dir" \ + --namespace nvcf \ + --set llmRequestRouter.image.registry=registry.example.invalid \ + --set llmRequestRouter.image.repository=nvcf/stargate \ + --set-string 'llmRequestRouter.kubernetes.advertisedHostnameTemplate=\{pod_name\}\{pod_name\}' \ + --set llmRequestRouter.backendRouter.enabled=true \ + --set llmRequestRouter.backendRouter.image.tag=next \ + --set llmRequestRouter.backendRouter.pylonGrpcDialAddress=llm-router.example.invalid:443 \ + --set llmRequestRouter.backendRouter.pylonReverseTunnelDialAddress=llm-router.example.invalid:8080 \ + >/dev/null 2>&1; then + echo "FAIL: backend routing must reject multiple {pod_name} placeholders" >&2 + exit 1 +fi + +if helm template llm-request-router "$chart_dir" \ + --namespace nvcf \ + --set llmRequestRouter.image.registry=registry.example.invalid \ + --set llmRequestRouter.image.repository=nvcf/stargate \ + --set llmRequestRouter.backendRouter.enabled=true \ + --set llmRequestRouter.backendRouter.image.tag=next \ + --set llmRequestRouter.backendRouter.pylonGrpcDialAddress=llm-router.example.invalid:443 \ + --set llmRequestRouter.backendRouter.pylonReverseTunnelDialAddress=llm-router.example.invalid:8080 \ + --set llmRequestRouter.backendRouter.serviceAccount.create=false \ + >/dev/null 2>&1; then + echo "FAIL: chart-managed backend RBAC must not bind the namespace default ServiceAccount" >&2 + exit 1 +fi + +if helm template llm-request-router "$chart_dir" \ + --namespace nvcf \ + --set llmRequestRouter.image.registry=registry.example.invalid \ + --set llmRequestRouter.image.repository=nvcf/stargate \ + --set llmRequestRouter.backendRouter.enabled=true \ + --set llmRequestRouter.backendRouter.image.tag=next \ + --set llmRequestRouter.backendRouter.pylonGrpcDialAddress=llm-router.example.invalid:443 \ + --set llmRequestRouter.backendRouter.pylonReverseTunnelDialAddress=llm-router.example.invalid:8080 \ + --set llmRequestRouter.backendRouter.serviceAccount.create=false \ + --set llmRequestRouter.rbac.create=false \ + >/dev/null 2>&1; then + echo "FAIL: an external backend ServiceAccount must be named even when chart RBAC is disabled" >&2 + exit 1 +fi + +helm template llm-request-router "$chart_dir" \ + --namespace nvcf \ + --set llmRequestRouter.image.registry=registry.example.invalid \ + --set llmRequestRouter.image.repository=nvcf/stargate \ + --set llmRequestRouter.backendRouter.enabled=true \ + --set llmRequestRouter.backendRouter.image.tag=next \ + --set llmRequestRouter.backendRouter.pylonGrpcDialAddress=llm-router.example.invalid:443 \ + --set llmRequestRouter.backendRouter.pylonReverseTunnelDialAddress=llm-router.example.invalid:8080 \ + --set llmRequestRouter.backendRouter.serviceAccount.create=false \ + --set llmRequestRouter.backendRouter.serviceAccount.name=external-backend-router \ + >"$external_service_account" + +if ! grep -Fq -- "serviceAccountName: external-backend-router" "$external_service_account"; then + echo "FAIL: backend router must use the configured external ServiceAccount" >&2 + exit 1 +fi +assert_backend_router_role_binding_subject "$external_service_account" "external-backend-router" + +if helm template llm-request-router "$chart_dir" \ + --namespace nvcf \ + --set llmRequestRouter.image.registry=registry.example.invalid \ + --set llmRequestRouter.image.repository=nvcf/stargate \ + --set llmRequestRouter.backendRouter.enabled=true \ + --set llmRequestRouter.backendRouter.pylonGrpcDialAddress=llm-router.example.invalid:443 \ + --set llmRequestRouter.backendRouter.pylonReverseTunnelDialAddress=llm-router.example.invalid:8080 \ + >/dev/null 2>&1; then + echo "FAIL: enabled backend router must require an explicit image tag" >&2 + exit 1 +fi + +helm template llm-request-router "$chart_dir" \ + --namespace nvcf \ + --set llmRequestRouter.image.registry=registry.example.invalid \ + --set llmRequestRouter.image.repository=nvcf/stargate \ + --set llmRequestRouter.backendRouter.enabled=true \ + --set llmRequestRouter.backendRouter.image.tag=next \ + --set llmRequestRouter.backendRouter.pylonGrpcDialAddress=llm-router.example.invalid:443 \ + --set llmRequestRouter.backendRouter.pylonReverseTunnelDialAddress=llm-router.example.invalid:8080 \ + --set llmRequestRouter.certificate.enabled=true \ + --set llmRequestRouter.certificate.issuerRef.name=test-issuer \ + --set llmRequestRouter.tls.secretName=stargate-quic-tls \ + --set llmRequestRouter.tls.certPath=/etc/stargate/tls/tls.crt \ + --set llmRequestRouter.tls.keyPath=/etc/stargate/tls/tls.key \ + >"$wildcard_certificate" + +if ! grep -Fq -- "- '*.llm-request-router-headless.nvcf.svc.cluster.local'" "$wildcard_certificate"; then + echo "FAIL: backend routing must add its wildcard before certificate DNS-name validation" >&2 + exit 1 +fi + +if helm template llm-request-router "$chart_dir" \ + --namespace nvcf \ + --set llmRequestRouter.image.registry=registry.example.invalid \ + --set llmRequestRouter.image.repository=nvcf/stargate \ + --set llmRequestRouter.backendRouter.enabled=false \ + --set llmRequestRouter.certificate.enabled=true \ + --set llmRequestRouter.certificate.issuerRef.name=test-issuer \ + >/dev/null 2>&1; then + echo "FAIL: a certificate without an expanded DNS name must fail rendering" >&2 + exit 1 +fi + +if helm template llm-request-router "$chart_dir" \ + --namespace nvcf \ + --set llmRequestRouter.image.registry=registry.example.invalid \ + --set llmRequestRouter.image.repository=nvcf/stargate \ + --set llmRequestRouter.kubernetes.advertisedHostnameTemplate=llm-request-router.nvcf.svc.cluster.local \ + --set llmRequestRouter.backendRouter.enabled=true \ + --set llmRequestRouter.backendRouter.image.tag=next \ + --set llmRequestRouter.backendRouter.pylonGrpcDialAddress=llm-router.example.invalid:443 \ + --set llmRequestRouter.backendRouter.pylonReverseTunnelDialAddress=llm-router.example.invalid:8080 \ + >/dev/null 2>&1; then + echo "FAIL: backend routing must reject a hostname template without {pod_name}" >&2 + exit 1 +fi + +if helm template llm-request-router "$chart_dir" \ + --namespace nvcf \ + --set llmRequestRouter.image.registry=registry.example.invalid \ + --set llmRequestRouter.image.repository=nvcf/stargate \ + --set llmRequestRouter.backendRouter.enabled=true \ + --set llmRequestRouter.backendRouter.image.tag=next \ + --set llmRequestRouter.backendRouter.pylonGrpcDialAddress=llm-router.example.invalid:443 \ + --set llmRequestRouter.backendRouter.pylonReverseTunnelDialAddress=llm-router.example.invalid:8080 \ + --set llmRequestRouter.tls.quicInsecure=false \ + >/dev/null 2>&1; then + echo "FAIL: secure backend QUIC must require a TLS Secret and cert/key paths" >&2 + exit 1 +fi + +if helm template llm-request-router "$chart_dir" \ + --namespace nvcf \ + --set llmRequestRouter.image.registry=registry.example.invalid \ + --set llmRequestRouter.image.repository=nvcf/stargate \ + --set llmRequestRouter.backendRouter.enabled=true \ + --set llmRequestRouter.backendRouter.image.tag=next \ + --set llmRequestRouter.backendRouter.pylonGrpcDialAddress=llm-router.example.invalid:443 \ + --set llmRequestRouter.backendRouter.pylonReverseTunnelDialAddress=llm-router.example.invalid:8080 \ + --set llmRequestRouter.tls.certPath=/etc/stargate/tls/tls.crt \ + >/dev/null 2>&1; then + echo "FAIL: backend routing must reject a partial TLS configuration" >&2 + exit 1 +fi + +if helm template llm-request-router "$chart_dir" \ + --namespace nvcf \ + --set llmRequestRouter.image.registry=registry.example.invalid \ + --set llmRequestRouter.image.repository=nvcf/stargate \ + --set llmRequestRouter.backendRouter.enabled=true \ + --set llmRequestRouter.backendRouter.image.tag=next \ + --set llmRequestRouter.backendRouter.pylonGrpcDialAddress=llm-router.example.invalid:443 \ + --set llmRequestRouter.backendRouter.pylonReverseTunnelDialAddress=llm-router.example.invalid:8080 \ + --set llmRequestRouter.tls.secretName=stargate-quic-tls \ + --set llmRequestRouter.tls.certPath=/etc/stargate/tls/tls.crt \ + --set llmRequestRouter.tls.keyPath=/var/run/stargate/tls.key \ + >/dev/null 2>&1; then + echo "FAIL: backend TLS cert and key paths must use the same directory" >&2 + exit 1 +fi + +if helm template llm-request-router "$chart_dir" \ + --namespace nvcf \ + --set llmRequestRouter.image.registry=registry.example.invalid \ + --set llmRequestRouter.image.repository=nvcf/stargate \ + --set llmRequestRouter.backendRouter.enabled=false \ + --set llmRequestRouter.tls.secretName=stargate-quic-tls \ + --set llmRequestRouter.tls.certPath=/etc/stargate/tls/tls.crt \ + --set llmRequestRouter.tls.keyPath=/var/run/stargate/tls.key \ + >/dev/null 2>&1; then + echo "FAIL: Stargate TLS cert and key paths must use the same directory" >&2 + exit 1 +fi + +if helm template llm-request-router "$chart_dir" \ + --namespace nvcf \ + --set llmRequestRouter.image.registry=registry.example.invalid \ + --set llmRequestRouter.image.repository=nvcf/stargate \ + --set llmRequestRouter.backendRouter.enabled=true \ + --set llmRequestRouter.backendRouter.image.tag=next \ + --set llmRequestRouter.backendRouter.pylonGrpcDialAddress=llm-router.example.invalid:443 \ + --set llmRequestRouter.backendRouter.pylonReverseTunnelDialAddress=llm-router.example.invalid:8080 \ + --set llmRequestRouter.tls.secretName=stargate-quic-tls \ + --set llmRequestRouter.tls.mountPath=/var/run/stargate \ + --set llmRequestRouter.tls.certPath=/etc/stargate/tls/tls.crt \ + --set llmRequestRouter.tls.keyPath=/etc/stargate/tls/tls.key \ + >/dev/null 2>&1; then + echo "FAIL: backend TLS mount path must contain the configured cert and key paths" >&2 + exit 1 +fi + +single_replica="$(helm template llm-request-router "$chart_dir" \ + --namespace nvcf \ + --set llmRequestRouter.image.registry=registry.example.invalid \ + --set llmRequestRouter.image.repository=nvcf/stargate \ + --set llmRequestRouter.replicaCount=1 \ + --set llmRequestRouter.backendRouter.enabled=true \ + --set llmRequestRouter.backendRouter.image.tag=next \ + --set llmRequestRouter.backendRouter.pylonGrpcDialAddress=llm-router.example.invalid:443 \ + --set llmRequestRouter.backendRouter.pylonReverseTunnelDialAddress=llm-router.example.invalid:8080)" +if ! grep -Fq -- "--advertised-hostname-template={pod_name}.llm-request-router-headless.nvcf.svc.cluster.local" <<<"$single_replica"; then + echo "FAIL: backend routing must retain per-pod authority and SNI for one replica" >&2 + exit 1 +fi + +echo "PASS: LLM request-router backend routing renders correctly" diff --git a/src/compute-plane-services/nvca/pkg/nvca/cli.go b/src/compute-plane-services/nvca/pkg/nvca/cli.go index 6d6986a90..e0a5d2377 100644 --- a/src/compute-plane-services/nvca/pkg/nvca/cli.go +++ b/src/compute-plane-services/nvca/pkg/nvca/cli.go @@ -313,9 +313,8 @@ func setDefaults(cfg *nvcaconfig.Config) error { if err := k8sutil.SetConfigDefaultResources(cfg); err != nil { return err } - // NVCA no longer backfills a default LLM request-router address. The value - // rides down via worker EnvironmentB64 from nvcf-api as LLM_REQUEST_ROUTER_ADDRESS; - // STARGATE_ADDRESS is accepted as a legacy alias and is still injected downstream. + // NVCA does not invent an LLM request-router address. Translation prefers the + // worker environment and falls back to an operator-configured workload default. cmdutil.SetEmptyValue(&cfg.Authz.ClientID, os.Getenv(auth.ClientIDEnv)) cmdutil.SetEmptyValue(&cfg.Authz.ClientSecretKey, os.Getenv(auth.ClientSecretEnv)) return nil diff --git a/src/compute-plane-services/nvca/vendor/github.com/NVIDIA/nvcf/src/libraries/go/lib/pkg/icms-translate/translate/function/llm.go b/src/compute-plane-services/nvca/vendor/github.com/NVIDIA/nvcf/src/libraries/go/lib/pkg/icms-translate/translate/function/llm.go index 21ec72965..f485088ea 100644 --- a/src/compute-plane-services/nvca/vendor/github.com/NVIDIA/nvcf/src/libraries/go/lib/pkg/icms-translate/translate/function/llm.go +++ b/src/compute-plane-services/nvca/vendor/github.com/NVIDIA/nvcf/src/libraries/go/lib/pkg/icms-translate/translate/function/llm.go @@ -74,6 +74,9 @@ func newLLMRouterClientContainer( if llmRequestRouterAddress == "" { llmRequestRouterAddress = allEnvSet[legacyStargateAddressEnv] } + if llmRequestRouterAddress == "" { + llmRequestRouterAddress = tcfg.DefaultStargateAddress + } if llmRequestRouterAddress == "" { return corev1.Container{}, fmt.Errorf( "LLM request router address is not set (%s env or %s legacy env)", diff --git a/src/compute-plane-services/nvca/vendor/github.com/NVIDIA/nvcf/src/libraries/go/lib/pkg/icms-translate/translate/function/translate.go b/src/compute-plane-services/nvca/vendor/github.com/NVIDIA/nvcf/src/libraries/go/lib/pkg/icms-translate/translate/function/translate.go index 4042a38bf..e1455cdcf 100644 --- a/src/compute-plane-services/nvca/vendor/github.com/NVIDIA/nvcf/src/libraries/go/lib/pkg/icms-translate/translate/function/translate.go +++ b/src/compute-plane-services/nvca/vendor/github.com/NVIDIA/nvcf/src/libraries/go/lib/pkg/icms-translate/translate/function/translate.go @@ -41,7 +41,7 @@ type TranslateConfig struct { // to find only relevant secrets to translation. DisableHelmWorkloadSecretTranslation bool - // Deprecated: LLM request router address must be supplied by environment. + // Default LLM request router address used when the launch environment omits one. DefaultStargateAddress string `json:"defaultStargateAddress,omitempty"` // Whether to use QUIC insecure mode for stargate connections // If not set, "--quick-insecure" is omitted diff --git a/src/libraries/go/lib/pkg/icms-translate/translate/function/llm.go b/src/libraries/go/lib/pkg/icms-translate/translate/function/llm.go index 21ec72965..f485088ea 100644 --- a/src/libraries/go/lib/pkg/icms-translate/translate/function/llm.go +++ b/src/libraries/go/lib/pkg/icms-translate/translate/function/llm.go @@ -74,6 +74,9 @@ func newLLMRouterClientContainer( if llmRequestRouterAddress == "" { llmRequestRouterAddress = allEnvSet[legacyStargateAddressEnv] } + if llmRequestRouterAddress == "" { + llmRequestRouterAddress = tcfg.DefaultStargateAddress + } if llmRequestRouterAddress == "" { return corev1.Container{}, fmt.Errorf( "LLM request router address is not set (%s env or %s legacy env)", diff --git a/src/libraries/go/lib/pkg/icms-translate/translate/function/llm_test.go b/src/libraries/go/lib/pkg/icms-translate/translate/function/llm_test.go index 557e701e7..a856d58de 100644 --- a/src/libraries/go/lib/pkg/icms-translate/translate/function/llm_test.go +++ b/src/libraries/go/lib/pkg/icms-translate/translate/function/llm_test.go @@ -88,7 +88,7 @@ func TestNewLLMRouterClientContainer(t *testing.T) { }, }, { - name: "configured default stargate address without env returns error", + name: "configured default request router address is used when env is absent", ls: &LaunchSpecification{}, allEnvSet: map[string]string{ "INFERENCE_PORT": "9090", @@ -96,8 +96,14 @@ func TestNewLLMRouterClientContainer(t *testing.T) { tcfg: TranslateConfig{ DefaultStargateAddress: "default-stargate.example.com:443", }, - expError: "LLM request router address is not set " + - "(LLM_REQUEST_ROUTER_ADDRESS env or STARGATE_ADDRESS legacy env)", + instanceID: "inst-default", + isHelm: false, + validate: func(t *testing.T, c corev1.Container) { + assert.Contains(t, c.Args, "--stargate-address=default-stargate.example.com:443") + envMap := envSliceToMap(c.Env) + assert.Equal(t, "default-stargate.example.com:443", envMap["LLM_REQUEST_ROUTER_ADDRESS"]) + assert.Equal(t, "default-stargate.example.com:443", envMap["STARGATE_ADDRESS"]) + }, }, { name: "LLM request router env controls when both env names are present", diff --git a/src/libraries/go/lib/pkg/icms-translate/translate/function/translate.go b/src/libraries/go/lib/pkg/icms-translate/translate/function/translate.go index 4042a38bf..e1455cdcf 100644 --- a/src/libraries/go/lib/pkg/icms-translate/translate/function/translate.go +++ b/src/libraries/go/lib/pkg/icms-translate/translate/function/translate.go @@ -41,7 +41,7 @@ type TranslateConfig struct { // to find only relevant secrets to translation. DisableHelmWorkloadSecretTranslation bool - // Deprecated: LLM request router address must be supplied by environment. + // Default LLM request router address used when the launch environment omits one. DefaultStargateAddress string `json:"defaultStargateAddress,omitempty"` // Whether to use QUIC insecure mode for stargate connections // If not set, "--quick-insecure" is omitted diff --git a/src/libraries/rust/stargate/Dockerfile b/src/libraries/rust/stargate/Dockerfile index 0a301e513..217a98932 100644 --- a/src/libraries/rust/stargate/Dockerfile +++ b/src/libraries/rust/stargate/Dockerfile @@ -129,6 +129,7 @@ ARG TARGETARCH # Health probe changes less often than the binary, so copy it first for layer caching. COPY --from=health-probe-downloader /usr/local/bin/grpc_health_probe /usr/local/bin/grpc_health_probe COPY --from=binary-builder /out/stargate /usr/local/bin/stargate +COPY --from=binary-builder /out/stargate-k8s-router /usr/local/bin/stargate-k8s-router ENTRYPOINT ["stargate"] CMD [] diff --git a/src/libraries/rust/stargate/README.md b/src/libraries/rust/stargate/README.md index 8b5a90efb..a11a8ac4e 100644 --- a/src/libraries/rust/stargate/README.md +++ b/src/libraries/rust/stargate/README.md @@ -83,6 +83,10 @@ load-balancer topology for production backend traffic. - `crates/mock-dynamo`: local OpenAI-style backend - `crates/stargate-bench`: benchmark runner +The versioned Stargate runtime image also includes +`/usr/local/bin/stargate-k8s-router`. Kubernetes deployments can run the main +Stargate process and the backend router from the same immutable image tag. + ## Benchmarks ```bash diff --git a/src/libraries/rust/stargate/crates/pylon/BUILD.bazel b/src/libraries/rust/stargate/crates/pylon/BUILD.bazel index 04b37f9ff..321d6ecbf 100644 --- a/src/libraries/rust/stargate/crates/pylon/BUILD.bazel +++ b/src/libraries/rust/stargate/crates/pylon/BUILD.bazel @@ -30,7 +30,7 @@ rust_binary( # Multi-arch OCI image. distroless/cc base, binary at /usr/local/bin/pylon. rust_oci_image( - name = "image", + name = "pylon-image", base = "@distroless_cc", binary = ":pylon", binary_path = "/usr/local/bin/pylon", @@ -45,8 +45,8 @@ sh_test( name = "image_entrypoint_mode_test", srcs = ["//tools/ci:image_entrypoint_mode_test.sh"], args = [ - "$(location :image_layer)", + "$(location :pylon-image_layer)", "/usr/local/bin/pylon", ], - data = [":image_layer"], + data = [":pylon-image_layer"], ) diff --git a/src/libraries/rust/stargate/crates/stargate-k8s-router/BUILD.bazel b/src/libraries/rust/stargate/crates/stargate-k8s-router/BUILD.bazel index ac84acde5..2e0e711ea 100644 --- a/src/libraries/rust/stargate/crates/stargate-k8s-router/BUILD.bazel +++ b/src/libraries/rust/stargate/crates/stargate-k8s-router/BUILD.bazel @@ -48,7 +48,7 @@ rust_test( # Multi-arch OCI image. distroless/cc base, binary at # /usr/local/bin/stargate-k8s-router. rust_oci_image( - name = "image", + name = "stargate-k8s-router-image", base = "@distroless_cc", binary = ":stargate-k8s-router", binary_path = "/usr/local/bin/stargate-k8s-router", @@ -62,8 +62,8 @@ sh_test( name = "image_entrypoint_mode_test", srcs = ["//tools/ci:image_entrypoint_mode_test.sh"], args = [ - "$(location :image_layer)", + "$(location :stargate-k8s-router-image_layer)", "/usr/local/bin/stargate-k8s-router", ], - data = [":image_layer"], + data = [":stargate-k8s-router-image_layer"], ) diff --git a/src/libraries/rust/stargate/crates/stargate-k8s-router/src/metrics.rs b/src/libraries/rust/stargate/crates/stargate-k8s-router/src/metrics.rs index b5ee3019c..58cc2604b 100644 --- a/src/libraries/rust/stargate/crates/stargate-k8s-router/src/metrics.rs +++ b/src/libraries/rust/stargate/crates/stargate-k8s-router/src/metrics.rs @@ -16,6 +16,26 @@ use anyhow::Result; use prometheus::{Encoder, IntCounterVec, Opts, Registry, TextEncoder}; +const QUIC_CONNECTION_OUTCOMES: &[&str] = &[ + "accepted", + "completed", + "missing_sni", + "relay_error", + "target_unavailable", + "unknown_sni", +]; +const WEBTRANSPORT_SESSION_OUTCOMES: &[&str] = &[ + "accepted", + "completed", + "invalid_connect", + "missing_sni", + "relay_error", + "target_unavailable", + "unknown_sni", + "upstream_connect_error", + "upstream_rejected", +]; + #[derive(Clone)] pub struct RouterMetrics { registry: Registry, @@ -43,6 +63,13 @@ impl RouterMetrics { )?; registry.register(Box::new(webtransport_sessions_total.clone()))?; + for outcome in QUIC_CONNECTION_OUTCOMES { + let _ = quic_connections_total.with_label_values(&[outcome]); + } + for outcome in WEBTRANSPORT_SESSION_OUTCOMES { + let _ = webtransport_sessions_total.with_label_values(&[outcome]); + } + Ok(Self { registry, quic_connections_total, @@ -74,6 +101,30 @@ impl RouterMetrics { mod tests { use super::*; + #[test] + fn metrics_export_known_outcomes_before_traffic() { + let metrics = RouterMetrics::new().expect("metrics should initialize"); + + let body = metrics.gather().expect("metrics should encode"); + + for &outcome in QUIC_CONNECTION_OUTCOMES { + assert!( + body.contains(&format!( + r#"stargate_k8s_router_quic_connections_total{{outcome="{outcome}"}} 0"# + )), + "missing zero-valued QUIC series for {outcome}" + ); + } + for &outcome in WEBTRANSPORT_SESSION_OUTCOMES { + assert!( + body.contains(&format!( + r#"stargate_k8s_router_webtransport_sessions_total{{outcome="{outcome}"}} 0"# + )), + "missing zero-valued WebTransport series for {outcome}" + ); + } + } + #[test] fn metrics_exports_quic_connection_outcomes() { let metrics = RouterMetrics::new().expect("metrics should initialize"); diff --git a/src/libraries/rust/stargate/crates/stargate/BUILD.bazel b/src/libraries/rust/stargate/crates/stargate/BUILD.bazel index 96a5e8134..785692adf 100644 --- a/src/libraries/rust/stargate/crates/stargate/BUILD.bazel +++ b/src/libraries/rust/stargate/crates/stargate/BUILD.bazel @@ -115,6 +115,7 @@ rust_oci_image( base = "@distroless_cc", binary = ":stargate", binary_path = "/usr/local/bin/stargate", + extra_layers = ["//crates/stargate-k8s-router:stargate-k8s-router-image_layer"], tags = ["stargate"], visibility = ["//visibility:public"], ) @@ -131,3 +132,15 @@ sh_test( ], data = [":image_layer"], ) + +# Composite-image guard. Inspect the assembled OCI layout so removing the +# router's extra layer fails this test. +sh_test( + name = "image_router_binary_test", + srcs = ["//tools/ci:oci_image_contains_path_test.sh"], + args = [ + "$(location :image)", + "/usr/local/bin/stargate-k8s-router", + ], + data = [":image"], +) diff --git a/src/libraries/rust/stargate/tools/ci/BUILD.bazel b/src/libraries/rust/stargate/tools/ci/BUILD.bazel index 96e468f21..2f322349f 100644 --- a/src/libraries/rust/stargate/tools/ci/BUILD.bazel +++ b/src/libraries/rust/stargate/tools/ci/BUILD.bazel @@ -4,6 +4,7 @@ exports_files( [ "image_entrypoint_mode_test.sh", + "oci_image_contains_path_test.sh", ], visibility = ["//visibility:public"], ) diff --git a/src/libraries/rust/stargate/tools/ci/oci_image_contains_path_test.sh b/src/libraries/rust/stargate/tools/ci/oci_image_contains_path_test.sh new file mode 100755 index 000000000..c389a8607 --- /dev/null +++ b/src/libraries/rust/stargate/tools/ci/oci_image_contains_path_test.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +if [[ $# -ne 2 ]]; then + echo "usage: $0 " >&2 + exit 2 +fi + +image_layout="$1" +image_path="${2#/}" + +if [[ ! -f "${image_layout}/index.json" || ! -d "${image_layout}/blobs/sha256" ]]; then + echo "${image_layout} is not an OCI image layout" >&2 + exit 1 +fi + +while IFS= read -r -d '' blob; do + if entries="$(tar -tf "${blob}" 2>/dev/null)"; then + while IFS= read -r entry; do + entry="${entry#./}" + entry="${entry#/}" + if [[ "${entry}" == "${image_path}" ]]; then + exit 0 + fi + done <<< "${entries}" + fi +# rules_oci may symlink blob files to their source layers. Select those links +# directly without following symlinked directories outside the blob tree. +done < <(find "${image_layout}/blobs/sha256" \( -type f -o -type l \) -print0) + +echo "missing /${image_path} in ${image_layout}" >&2 +exit 1 diff --git a/tools/ci/test-image-push-manual b/tools/ci/test-image-push-manual new file mode 100755 index 000000000..1647c578b --- /dev/null +++ b/tools/ci/test-image-push-manual @@ -0,0 +1,97 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Behavioral regression test for the manual image-push workflow. A repository +# collision must fail before any image push begins, because every push also +# updates the shared latest-dispatch tag. +set -euo pipefail + +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +workflow="${repo_root}/.github/workflows/image-push-manual.yml" +work="$(mktemp -d)" +trap 'rm -rf "${work}"' EXIT + +awk ' + $0 == " - name: Build and push multi-arch image(s)" { + in_step = 1 + next + } + in_step && $0 == " run: |" { + in_script = 1 + next + } + in_script { + if ($0 ~ /^ - name:/) { + exit + } + sub(/^ /, "") + print + } +' "${workflow}" > "${work}/image-push-step" + +if [[ ! -s "${work}/image-push-step" ]]; then + echo "FAIL: could not extract the image-push shell block" >&2 + exit 1 +fi + +mkdir -p "${work}/bin" "${work}/home" "${work}/run" +cat > "${work}/bin/bazel" <<'STUB' +#!/usr/bin/env bash +set -euo pipefail + +printf '%s\n' "$*" >> "${BAZEL_CALL_LOG}" +case "${1:-}" in + query) + # Both targets map to example-service-worker under the workflow's two + # supported naming conventions. + printf '%s\n' \ + '//first:worker_image_index' \ + '//second:example-service-worker-image_index' + ;; + run) + touch "${PUSH_MARKER}" + ;; + *) + echo "unexpected bazel invocation: $*" >&2 + exit 1 + ;; +esac +STUB +chmod +x "${work}/bin/bazel" + +output="${work}/output" +calls="${work}/bazel-calls" +push_marker="${work}/push-started" + +if ( + cd "${work}/run" + HOME="${work}/home" \ + PATH="${work}/bin:${PATH}" \ + BAZEL_CALL_LOG="${calls}" \ + PUSH_MARKER="${push_marker}" \ + SVC_PATH="src/example-service" \ + SCOPE="//..." \ + TAG="gh.1-deadbeef" \ + REGISTRY="registry.example.com/nvcf" \ + bash "${work}/image-push-step" +) > "${output}" 2>&1; then + echo "FAIL: duplicate repository mapping unexpectedly succeeded" >&2 + exit 1 +fi + +if [[ -e "${push_marker}" ]] || grep -q '^run ' "${calls}"; then + echo "FAIL: bazel run executed before all repository mappings were validated" >&2 + sed 's/^/ /' "${calls}" >&2 + exit 1 +fi + +if ! grep -Fq \ + "//second:example-service-worker-image_index and //first:worker_image_index both map to image repository example-service-worker" \ + "${output}"; then + echo "FAIL: workflow did not report the conflicting target mappings" >&2 + sed 's/^/ /' "${output}" >&2 + exit 1 +fi + +echo "image-push-manual: duplicate mappings prevent every push"