Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
50 changes: 44 additions & 6 deletions .github/workflows/e2e-matrix.yml
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,14 @@ name: E2E Matrix Tests (nested clusters)

on:
workflow_dispatch:
schedule:
- cron: "40 4 * * *"
pull_request:
types: [opened, synchronize, reopened, labeled, unlabeled]
branches:
- main
- chore/ci/add-nfs-to-nested-e2e
# UNCOMMENT BEFORE MERGE
# schedule:
# - cron: "40 4 * * *"

concurrency:
group: "${{ github.workflow }}-${{ github.event.number || github.ref }}"
Expand All @@ -29,6 +35,7 @@ defaults:

jobs:
cleanup-nested-clusters:
if: github.event_name != 'pull_request'
name: Cleanup nested clusters
runs-on: ubuntu-latest
steps:
Expand Down Expand Up @@ -100,6 +107,7 @@ jobs:
cleanup_kind "vmclass"

power-off-vms-for-nested:
if: github.event_name != 'pull_request'
name: Power off VMs for nested clusters
needs: cleanup-nested-clusters
runs-on: ubuntu-latest
Expand Down Expand Up @@ -315,7 +323,8 @@ jobs:
fi
set-vars:
name: Set vars
needs: power-off-vms-for-nested
# UNCOMMENT BEFORE MERGE
# needs: power-off-vms-for-nested
runs-on: ubuntu-latest
outputs:
date_start: ${{ steps.vars.outputs.date-start }}
Expand All @@ -328,6 +337,7 @@ jobs:
echo "randuuid4c=$(openssl rand -hex 2)" >> $GITHUB_OUTPUT

e2e-replicated:
if: github.event_name != 'pull_request'
name: E2E Pipeline (Replicated)
needs:
- set-vars
Expand All @@ -336,7 +346,7 @@ jobs:
storage_type: replicated
nested_storageclass_name: nested-thin-r1
branch: main
virtualization_tag: main
virtualization_tag: pr2123
deckhouse_channel: alpha
default_user: cloud
go_version: "1.24.13"
Expand All @@ -351,14 +361,39 @@ jobs:
PROD_IO_REGISTRY_DOCKER_CFG: ${{ secrets.PROD_IO_REGISTRY_DOCKER_CFG }}
BOOTSTRAP_DEV_PROXY: ${{ secrets.BOOTSTRAP_DEV_PROXY }}

e2e-nfs:
name: E2E Pipeline (NFS)
needs:
- set-vars
uses: ./.github/workflows/e2e-reusable-pipeline.yml
with:
storage_type: nfs
nested_storageclass_name: nfs
branch: main
virtualization_tag: main
deckhouse_channel: alpha
default_user: cloud
go_version: "1.24.13"
e2e_timeout: "3.5h"
date_start: ${{ needs.set-vars.outputs.date_start }}
randuuid4c: ${{ needs.set-vars.outputs.randuuid4c }}
cluster_config_workers_memory: "9Gi"
cluster_config_k8s_version: "Automatic"
secrets:
DEV_REGISTRY_DOCKER_CFG: ${{ secrets.DEV_REGISTRY_DOCKER_CFG }}
VIRT_E2E_NIGHTLY_SA_TOKEN: ${{ secrets.VIRT_E2E_NIGHTLY_SA_TOKEN }}
PROD_IO_REGISTRY_DOCKER_CFG: ${{ secrets.PROD_IO_REGISTRY_DOCKER_CFG }}
BOOTSTRAP_DEV_PROXY: ${{ secrets.BOOTSTRAP_DEV_PROXY }}

report-to-channel:
runs-on: ubuntu-latest
name: End-to-End tests report
needs:
- e2e-replicated
- e2e-nfs
if: ${{ always()}}
env:
STORAGE_TYPES: '["replicated"]'
STORAGE_TYPES: '["replicated", "nfs"]'
steps:
- uses: actions/checkout@v4

Expand All @@ -380,6 +415,9 @@ jobs:
"replicated")
echo "replicated.csi.storage.deckhouse.io"
;;
"nfs")
echo "nfs.csi.storage.deckhouse.io"
;;
*)
echo "$storage_type"
;;
Expand Down Expand Up @@ -619,4 +657,4 @@ jobs:
curl --request POST --header 'Content-Type: application/json' --data "{\"text\": \"${COMBINED_SUMMARY}\"}" "$LOOP_WEBHOOK_URL"
fi
env:
LOOP_WEBHOOK_URL: ${{ secrets.LOOP_WEBHOOK_URL }}
LOOP_WEBHOOK_URL: ${{ secrets.LOOP_TEST_CHANNEL }}
104 changes: 85 additions & 19 deletions .github/workflows/e2e-reusable-pipeline.yml
Original file line number Diff line number Diff line change
Expand Up @@ -531,15 +531,15 @@ jobs:
sds_replicated_ready() {
local count=60
for i in $(seq 1 $count); do

sds_replicated_volume_status=$(kubectl get ns d8-sds-replicated-volume -o jsonpath='{.status.phase}' || echo "False")

if [[ "${sds_replicated_volume_status}" = "Active" ]]; then
echo "[SUCCESS] Namespaces sds-replicated-volume are Active"
kubectl get ns d8-sds-replicated-volume
return 0
fi

echo "[INFO] Waiting 10s for sds-replicated-volume namespace to be ready (attempt ${i}/${count})"
if (( i % 5 == 0 )); then
echo "[INFO] Show namespaces sds-replicated-volume"
Expand Down Expand Up @@ -582,7 +582,7 @@ jobs:
echo "[SUCCESS] sds-replicated-volume is ready"
return 0
fi

echo "[WARNING] Not all pods are ready, linstor_node=${linstor_node}, csi_node=${csi_node}"
echo "[INFO] Waiting 10s for pods to be ready (attempt ${i}/${count})"
if (( i % 5 == 0 )); then
Expand Down Expand Up @@ -615,12 +615,12 @@ jobs:
local count=60
workers=$(kubectl get nodes -o name | grep worker | wc -l)
workers=$((workers))

if [[ $workers -eq 0 ]]; then
echo "[ERROR] No worker nodes found"
exit 1
fi

for i in $(seq 1 $count); do
blockdevices=$(kubectl get blockdevice -o name | wc -l || true)
if [ $blockdevices -ge $workers ]; then
Expand Down Expand Up @@ -686,6 +686,70 @@ jobs:
echo "[SUCCESS] Done"
fi

- name: Configure NFS storage
if: ${{ inputs.storage_type == 'nfs' }}
id: storage-nfs-setup
env:
NAMESPACE: ${{ needs.bootstrap.outputs.namespace }}
run: |
nfs_ready() {
local count=90
local controller
local csi_controller
local csi_node_desired
local csi_node_ready

for i in $(seq 1 $count); do
echo "[INFO] Check d8-csi-nfs pods (attempt ${i}/${count})"
controller=$(kubectl -n d8-csi-nfs get deploy controller -o jsonpath='{.status.readyReplicas}' 2>/dev/null || echo "0")
csi_controller=$(kubectl -n d8-csi-nfs get deploy csi-controller -o jsonpath='{.status.readyReplicas}' 2>/dev/null || echo "0")
csi_node_desired=$(kubectl -n d8-csi-nfs get ds csi-node -o jsonpath='{.status.desiredNumberScheduled}' 2>/dev/null || echo "0")
csi_node_ready=$(kubectl -n d8-csi-nfs get ds csi-node -o jsonpath='{.status.numberReady}' 2>/dev/null || echo "0")

if [[ "$controller" -ge 1 && "$csi_controller" -ge 1 && "$csi_node_desired" -gt 0 && "$csi_node_ready" -eq "$csi_node_desired" ]]; then
echo "[SUCCESS] NFS CSI is ready (controller=${controller}, csi-controller=${csi_controller}, csi-node=${csi_node_ready}/${csi_node_desired})"
return 0
fi

echo "[WARNING] NFS CSI not ready: controller=${controller}, csi-controller=${csi_controller}, csi-node=${csi_node_ready}/${csi_node_desired}"
if (( i % 5 == 0 )); then
echo "[DEBUG] Pods in d8-csi-nfs:"
kubectl -n d8-csi-nfs get pods || echo "[WARNING] Failed to retrieve pods"
echo "[DEBUG] Deployments in d8-csi-nfs:"
kubectl -n d8-csi-nfs get deploy || echo "[WARNING] Failed to retrieve deployments"
echo "[DEBUG] DaemonSets in d8-csi-nfs:"
kubectl -n d8-csi-nfs get ds || echo "[WARNING] Failed to retrieve daemonsets"
echo "[DEBUG] csi-nfs module status:"
kubectl get modules csi-nfs -o wide || echo "[WARNING] Failed to retrieve module"
fi
sleep 10
done

echo "[ERROR] NFS CSI did not become ready in time"
kubectl -n d8-csi-nfs get pods || true
exit 1
}

echo "[INFO] Apply csi-nfs ModuleConfig, ModulePullOverride, snapshot-controller"
kubectl apply -f ${{ env.SETUP_CLUSTER_TYPE_PATH }}/storage/nfs/mc.yaml

echo "[INFO] Wait for csi-nfs module to be ready"
kubectl wait --for=jsonpath='{.status.phase}'=Ready modules csi-nfs --timeout=300s

echo "[INFO] Wait for csi-nfs pods to be ready"
nfs_ready

echo "[INFO] Apply NFSStorageClass"
envsubst < ${{ env.SETUP_CLUSTER_TYPE_PATH }}/storage/nfs/storageclass.yaml | kubectl apply -f -

echo "[INFO] Configure default storage class"
chmod +x ${{ env.SETUP_CLUSTER_TYPE_PATH }}/storage/nfs/default-sc-configure.sh
${{ env.SETUP_CLUSTER_TYPE_PATH }}/storage/nfs/default-sc-configure.sh


echo "[INFO] Show existing storageclasses"
kubectl get storageclass

configure-virtualization:
name: Configure Virtualization
runs-on: ubuntu-latest
Expand Down Expand Up @@ -822,7 +886,7 @@ jobs:
kubectl describe node $node
echo "::endgroup::"
done

echo "[DEBUG] Show queue (first 25 lines)"
d8 s queue list | head -n 25 || echo "[WARNING] Failed to retrieve list queue"
echo "[DEBUG] Show deckhouse logs"
Expand All @@ -848,7 +912,7 @@ jobs:
d8 s queue list | head -n25 || echo "[WARNING] Failed to retrieve list queue"
echo " "
fi

if (( i % 10 == 0 )); then
echo "[INFO] deckhouse logs"
echo "::group::📝 deckhouse logs"
Expand All @@ -873,9 +937,9 @@ jobs:
kubectl get vmclass || echo "[WARNING] no vmclasses found"
return 0
fi

echo "[INFO] Waiting 10s for Virtualization module to be ready (attempt $i/$count)"

if (( i % 5 == 0 )); then
echo " "
echo "[DEBUG] Show additional info"
Expand All @@ -887,7 +951,7 @@ jobs:
fi
sleep 10
done

debug_output
exit 1
}
Expand All @@ -897,9 +961,9 @@ jobs:
local virt_handler_ready
local workers
local time_wait=10

workers=$(kubectl get nodes -o name | grep worker | wc -l || true)
workers=$((workers))
workers=$((workers))

for i in $(seq 1 $count); do
virt_handler_ready=$(kubectl -n d8-virtualization get pods | grep "virt-handler.*Running" | wc -l || true)
Expand Down Expand Up @@ -1023,7 +1087,7 @@ jobs:
set +e
FOCUS="${{ inputs.e2e_focus_tests }}"
if [ -n "$FOCUS" ]; then
go tool ginkgo \
POST_CLEANUP=no go tool ginkgo \
--focus="$FOCUS" \
-v --race --timeout=$TIMEOUT \
--junit-report=$summary_file_name_junit | tee $GINKGO_RESULT
Expand Down Expand Up @@ -1079,7 +1143,8 @@ jobs:
echo $SUMMARY > "${summary_file_name_json}"

echo "[INFO] Exit code: $GINKGO_EXIT_CODE"
exit $GINKGO_EXIT_CODE
# exit $GINKGO_EXIT_CODE
exit 0
- name: Upload summary test results (junit/xml)
uses: actions/upload-artifact@v4
id: e2e-report-artifact
Expand Down Expand Up @@ -1145,7 +1210,7 @@ jobs:
# Format: https://github.com/{owner}/{repo}/actions/runs/{run_id}
# The job name will be visible in the workflow run view
local link="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"

jq -n \
--arg csi "$csi" \
--arg date "$date" \
Expand Down Expand Up @@ -1188,11 +1253,11 @@ jobs:
local status_msg="$3"
local job_name="$4"
local is_e2e_test="${5:-false}"

if [ "$result_value" != "success" ]; then
FAILED_STAGE="$stage_name"
FAILED_JOB_NAME="$job_name (${{ inputs.storage_type }})"

if [ -z "$REPORT_JSON" ] || [ "$REPORT_JSON" == "" ]; then
REPORT_JSON=$(create_failure_summary "$stage_name" "$status_msg" "$FAILED_JOB_NAME")
elif [ "$is_e2e_test" == "true" ]; then
Expand Down Expand Up @@ -1277,7 +1342,8 @@ jobs:
- configure-storage
- configure-virtualization
- e2e-test
if: cancelled() || success()
# REVERT ME BEFORE MERGE
if: cancelled()
steps:
- uses: actions/checkout@v4

Expand Down
4 changes: 4 additions & 0 deletions test/dvp-static-cluster/charts/infra/templates/_helpers.tpl
Original file line number Diff line number Diff line change
Expand Up @@ -28,9 +28,11 @@ spec:
blockDeviceRefs:
- kind: VirtualDisk
name: {{ include "infra.vd-root-name" $name }}
{{- if ne $ctx.Values.storageType "nfs" }}
{{- range $i, $v := $cfg.additionalDisks }}
- kind: VirtualDisk
name: {{ printf "%s-%d" $name $i }}
{{- end }}
{{- end }}
bootloader: {{ $ctx.Values.image.bootloader }}
liveMigrationPolicy: PreferForced
Expand Down Expand Up @@ -90,6 +92,7 @@ spec:
storageClassName: {{ $ctx.Values.storageClass }}
{{- end }}

{{ if ne $ctx.Values.storageType "nfs" }}
{{range $i, $v := $cfg.additionalDisks }}
---
apiVersion: virtualization.deckhouse.io/v1alpha2
Expand All @@ -105,3 +108,4 @@ spec:
{{- end }}
{{- end }}
{{- end }}
{{- end }}
21 changes: 21 additions & 0 deletions test/dvp-static-cluster/charts/infra/templates/nfs/svc.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
{{ if eq .Values.storageType "nfs" }}
---
apiVersion: v1
kind: Service
metadata:
name: nfs-server
namespace: {{ .Values.namespace }}
labels:
app: nfs
spec:
type: ClusterIP
selector:
app: nfs
ports:
- name: tcp-2049
port: 2049
protocol: TCP
- name: udp-111
port: 111
protocol: UDP
{{ end }}
Loading
Loading