test(tmachine): add K3s PostgreSQL and mTLS conformance installer

.github/workflows/branch-e2e.yml

@@ -256,6 +256,7 @@ jobs:
         [
           {"environment":"ubuntu-docker-rootful","installer":"deb","testsuite":"conformance"},
           {"environment":"ubuntu-k3s","installer":"k3s","testsuite":"conformance"},
+          {"environment":"ubuntu-k3s","installer":"k3s-ha-tls","testsuite":"conformance"},
           {"environment":"fedora-podman-rootful","installer":"rpm","testsuite":"conformance"},
           {"environment":"fedora-podman-rootless","installer":"rpm","testsuite":"conformance"}
         ]

.github/workflows/integration-runner.yml

@@ -26,6 +26,7 @@ on:
           [
             {"environment":"ubuntu-docker-rootful","installer":"deb","testsuite":"conformance"},
             {"environment":"ubuntu-k3s","installer":"k3s","testsuite":"conformance"},
+            {"environment":"ubuntu-k3s","installer":"k3s-ha-tls","testsuite":"conformance"},
             {"environment":"fedora-podman-rootful","installer":"rpm","testsuite":"conformance"},
             {"environment":"fedora-podman-rootless","installer":"rpm","testsuite":"conformance"}
           ]

.github/workflows/integration-test.yml

@@ -29,6 +29,7 @@ on:
           [
             {"environment":"ubuntu-docker-rootful","installer":"deb","testsuite":"conformance"},
             {"environment":"ubuntu-k3s","installer":"k3s","testsuite":"conformance"},
+            {"environment":"ubuntu-k3s","installer":"k3s-ha-tls","testsuite":"conformance"},
             {"environment":"fedora-podman-rootful","installer":"rpm","testsuite":"conformance"},
             {"environment":"fedora-podman-rootless","installer":"rpm","testsuite":"conformance"}
           ]

.github/workflows/release-dev.yml

@@ -127,6 +127,7 @@ jobs:
         [
           {"environment":"ubuntu-docker-rootful","installer":"deb","testsuite":"conformance"},
           {"environment":"ubuntu-k3s","installer":"k3s","testsuite":"conformance"},
+          {"environment":"ubuntu-k3s","installer":"k3s-ha-tls","testsuite":"conformance"},
           {"environment":"fedora-podman-rootful","installer":"rpm","testsuite":"conformance"},
           {"environment":"fedora-podman-rootless","installer":"rpm","testsuite":"conformance"}
         ]

.github/workflows/release-tag.yml

@@ -178,6 +178,7 @@ jobs:
         [
           {"environment":"ubuntu-docker-rootful","installer":"deb","testsuite":"conformance"},
           {"environment":"ubuntu-k3s","installer":"k3s","testsuite":"conformance"},
+          {"environment":"ubuntu-k3s","installer":"k3s-ha-tls","testsuite":"conformance"},
           {"environment":"fedora-podman-rootful","installer":"rpm","testsuite":"conformance"},
           {"environment":"fedora-podman-rootless","installer":"rpm","testsuite":"conformance"}
         ]

CI.md

@@ -149,8 +149,10 @@ compatibility baseline for the v1beta1 Sandbox API. It does not track the local
 K3s development default, currently v1.0.3. OpenShell also supports v0.4.6 through
 its v1alpha1 fallback, so v0.5.0 is not the overall minimum supported version.
 
-The `ubuntu-k3s` lane registers the gateway's ClusterIP Service directly, so
-recreating the Service requires reprovisioning the fixture. Before conformance,
+The `ubuntu-k3s` lanes register the gateway's ClusterIP Service directly, so
+recreating the Service requires reprovisioning the fixture. The `k3s-ha-tls`
+lane runs the same conformance suite against three gateway replicas sharing
+PostgreSQL, with the CLI connecting over mTLS. Before conformance,
 every installer waits up to five minutes for a connected gateway; interactive
 shells skip the wait. K3s failures upload diagnostics as `tmachine-diagnostics-*`
 artifacts.

tests/ansible/playbooks/openshell-k3s-ha-tls.yaml

@@ -0,0 +1,115 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+---
+- import_playbook: openshell-k3s.yaml
+
+- name: Add PostgreSQL and TLS to the K3s test installation
+  hosts: all
+  become: true
+  gather_facts: false
+  tasks:
+    - name: Copy PostgreSQL fixture
+      ansible.builtin.copy:
+        src: "{{ openshell_postgres_fixture }}"
+        dest: /var/lib/openshell/artifacts/postgres.yaml
+        mode: "0600"
+
+    - name: Apply PostgreSQL fixture
+      ansible.builtin.command:
+        argv: [/usr/local/bin/k3s, kubectl, --namespace, openshell, apply, --filename, /var/lib/openshell/artifacts/postgres.yaml]
+
+    - name: Wait for PostgreSQL
+      ansible.builtin.command:
+        argv: [/usr/local/bin/k3s, kubectl, --namespace, openshell, rollout, status, deployment/openshell-e2e-postgres, --timeout=300s]
+      changed_when: false
+
+    - name: Write PostgreSQL and TLS values overlay
+      ansible.builtin.copy:
+        dest: /var/lib/openshell/artifacts/values-ha-tls.yaml
+        mode: "0600"
+        content: |
+          replicaCount: 3
+          workload:
+            kind: deployment
+          server:
+            externalDbSecret: openshell-e2e-postgres-credentials
+            disableTls: false
+            auth:
+              # Kubernetes has no mTLS user-identity mode. Require the client
+              # certificate at the TLS layer and use the fixture's dev user.
+              allowUnauthenticatedUsers: true
+
+    - name: Apply PostgreSQL and TLS overlay to OpenShell
+      ansible.builtin.command:
+        argv:
+          - /usr/local/bin/helm
+          - upgrade
+          - openshell
+          - /var/lib/openshell/artifacts/helm-chart.tgz
+          - --namespace
+          - openshell
+          - --values
+          - /var/lib/openshell/artifacts/values.yaml
+          - --values
+          - /var/lib/openshell/artifacts/values-ha-tls.yaml
+          - --wait
+          - --timeout=5m
+      environment:
+        KUBECONFIG: /etc/rancher/k3s/k3s.yaml
+
+    # The chart's server certificate covers the Service DNS name but not its
+    # ClusterIP, and the node resolver cannot reach cluster DNS.
+    - name: Resolve the gateway Service name to its ClusterIP
+      ansible.builtin.lineinfile:
+        path: /etc/hosts
+        regexp: '\sopenshell\.openshell\.svc\.cluster\.local$'
+        line: "{{ k3s_gateway_address.stdout.split(':')[0] }} openshell.openshell.svc.cluster.local"
+
+    - name: Read chart client TLS Secret
+      ansible.builtin.command:
+        argv: [/usr/local/bin/k3s, kubectl, --namespace, openshell, get, secret, openshell-client-tls, --output, json]
+      register: k3s_client_tls
+      changed_when: false
+      no_log: true
+
+    - name: Create guest mTLS directory
+      ansible.builtin.file:
+        path: /home/tmachine/.config/openshell/gateways/tmachine/mtls
+        state: directory
+        owner: tmachine
+        group: tmachine
+        mode: "0700"
+
+    - name: Write guest client TLS bundle
+      ansible.builtin.copy:
+        content: "{{ (k3s_client_tls.stdout | from_json).data[item] | b64decode }}"
+        dest: "/home/tmachine/.config/openshell/gateways/tmachine/mtls/{{ item }}"
+        owner: tmachine
+        group: tmachine
+        mode: "0600"
+      loop: [ca.crt, tls.crt, tls.key]
+      no_log: true
+
+    - name: Remove baseline guest gateway registration
+      become: false
+      ansible.builtin.command:
+        argv: [/usr/local/bin/openshell, gateway, remove, tmachine]
+
+    - name: Register guest gateway
+      ansible.builtin.include_role:
+        name: openshell_client
+        apply:
+          become: false
+      vars:
+        openshell_client_gateway_endpoint: "https://openshell.openshell.svc.cluster.local:{{ k3s_gateway_address.stdout.split(':')[1] }}"
+
+    - name: Wait for authenticated gateway API
+      become: false
+      ansible.builtin.command:
+        argv: [timeout, 10s, /usr/local/bin/openshell, sandbox, list, --output, json]
+      register: k3s_gateway_api
+      until: k3s_gateway_api.rc == 0
+      retries: 24
+      delay: 5
+      changed_when: false

tests/ansible/roles/openshell_diagnostics/files/collect-k3s.sh

@@ -43,8 +43,13 @@ collect() {
   collect 'Endpoint readiness' /usr/local/bin/k3s kubectl --request-timeout=5s -n openshell get endpointslices \
     -o 'custom-columns=NAME:.metadata.name,PORTS:.ports,ADDRESSES:.endpoints[*].addresses,CONDITIONS:.endpoints[*].conditions'
   collect 'Gateway events' /usr/local/bin/k3s kubectl --request-timeout=5s -n openshell get events --sort-by=.lastTimestamp
-  collect 'Gateway current logs' /usr/local/bin/k3s kubectl --request-timeout=5s -n openshell logs openshell-0 -c openshell-gateway --tail=300 --timestamps
-  collect 'Gateway previous logs' /usr/local/bin/k3s kubectl --request-timeout=5s -n openshell logs openshell-0 -c openshell-gateway --previous --tail=300 --timestamps
+  # Gateway Pods are named by a StatefulSet or a Deployment, depending on the installer.
+  mapfile -t gateway_pods < <(timeout 10s /usr/local/bin/k3s kubectl --request-timeout=5s -n openshell get pods \
+    --selector app.kubernetes.io/name=openshell,app.kubernetes.io/instance=openshell --output name 2>/dev/null)
+  for pod in "${gateway_pods[@]}"; do
+    collect "Gateway current logs ($pod)" /usr/local/bin/k3s kubectl --request-timeout=5s -n openshell logs "$pod" -c openshell-gateway --tail=300 --timestamps
+    collect "Gateway previous logs ($pod)" /usr/local/bin/k3s kubectl --request-timeout=5s -n openshell logs "$pod" -c openshell-gateway --previous --tail=300 --timestamps
+  done
   collect 'Memory' free -m
   collect 'Disk' df -h / /var/lib/rancher/k3s
 } | sed -E \

tests/config.nix

@@ -94,6 +94,20 @@ let
           openshell_supervisor_image = "../artifacts/images/openshell-supervisor-tmachine.tar";
         };
       }
+      {
+        name = "k3s-ha-tls";
+        use_galaxy = false;
+        playbooks = [ "ansible/playbooks/openshell-k3s-ha-tls.yaml" ];
+        inputs = {
+          openshell_postgres_fixture = "../e2e/kubernetes/postgres-fixture.yaml";
+          agent_sandbox_version = "0.5.0";
+          openshell_cli_binary = "../artifacts/binaries/${muslTarget}/openshell";
+          openshell_gateway_image = "../artifacts/images/openshell-gateway-tmachine.tar";
+          openshell_helm_chart = "../artifacts/helm/helm-chart-0.0.0.tgz";
+          openshell_sandbox_image = "../artifacts/images/openshell-sandbox-tmachine.tar";
+          openshell_supervisor_image = "../artifacts/images/openshell-supervisor-tmachine.tar";
+        };
+      }
       {
         name = "none";
         use_galaxy = false;