Staging

src/orchestrator/input/omnia_config.yml

@@ -186,6 +186,10 @@ slurm_cluster:
 # File path for the values.yml file which will contain the Powerscale driver configuration parameters.
 # csi_powerscale_driver_values_file_path: User need to download values.yaml file and fill required data.
 # Provided the path of the values.yaml file here.
+# csi_powerscale_mount_options: Optional comma-separated NFS mount options applied to
+# PowerScale CSI volumes through the ps01 StorageClass mountOptions. Leave empty to use
+# the kernel defaults. For RHEL 10.x clients that report Input/output errors on PowerScale
+# volumes with the default 1 MiB transfer size, use "rsize=1044480,wsize=1044480".
 # mention configurable values
 
 # - k8s_crio_storage_size: Specifies the disk size allocated for CRI-O container storage.
@@ -224,5 +228,6 @@ service_k8s_cluster:
     enable_powerscale_csi: false
     csi_powerscale_driver_secret_file_path: ""
     csi_powerscale_driver_values_file_path: ""
+    csi_powerscale_mount_options: ""
     enable_k8s_ha: true
     virtual_ip_address: "172.16.107.1"

src/orchestrator/input/storage_config.yml

@@ -47,6 +47,17 @@
 #                   If specified, takes PRIORITY over mount_params profile
 #   - mount_on_oim: Mount this filesystem on the OIM node as well. Optional. Default: false
 #                   Ensure storage is network-accessible from OIM before enabling
+#   - node_nfs_mnt_opts: Mount options for the per-node NFS sub-mounts that Omnia creates
+#                   from the storage entry referenced by nfs_storage_name in omnia_config.yml.
+#                   Optional. Applies to Kubernetes (/var/lib/kubelet, /var/lib/etcd,
+#                   /etc/kubernetes, /var/log/pods, /var/lib/packages) and Slurm/login
+#                   (/etc/slurm, /etc/munge, /var/spool/slurmd, /var/log/slurm, /cert, ...).
+#                   For Kubernetes it is also passed to the nfs-client provisioner.
+#                   Default: "noatime,nolock" (Kubernetes) and "defaults,_netdev" (Slurm)
+#                   PowerScale with RHEL 10.x clients: if Input/output errors occur with the
+#                   default 1 MiB transfer size, set for example
+#                   "noatime,nolock,rsize=1044480,wsize=1044480" (Kubernetes) or
+#                   "defaults,_netdev,rsize=1044480,wsize=1044480" (Slurm)
 #
 # Node-specific bind mounts (node_key and node_mount_point are paired):
 #   - node_key:         Per-node subdirectory isolation variable. Optional

src/orchestrator/plugins/module_utils/orchestrator_validation/schema/omnia_config.json

@@ -276,6 +276,10 @@
         "csi_powerscale_driver_values_file_path": {
           "$ref": "#/definitions/optionalAbsolutePath"
         },
+        "csi_powerscale_mount_options": {
+          "type": "string",
+          "pattern": "^([A-Za-z0-9,=._@/-]+)?$"
+        },
         "enable_k8s_ha": {
           "type": "boolean"
         },

src/orchestrator/plugins/module_utils/orchestrator_validation/schema/storage_config.json

@@ -171,6 +171,9 @@
         "mnt_opts": {
           "$ref": "#/definitions/mountOptions"
         },
+        "node_nfs_mnt_opts": {
+          "$ref": "#/definitions/mountOptions"
+        },
         "dump_freq": {
           "$ref": "#/definitions/dumpFrequency"
         },

src/orchestrator/roles/k8s_config/tasks/create_k8s_config_nfs.yml

@@ -61,6 +61,10 @@
         csi_powerscale_driver_values_file_path: >-
           {{ deployed_service_k8s_cluster.get(
              'csi_powerscale_driver_values_file_path', '') }}
+        csi_powerscale_mount_options: >-
+          {{ (deployed_service_k8s_cluster.get('csi_powerscale_mount_options', '')
+              | default('', true) | string | trim).split(',')
+             | map('trim') | select | list }}
 
     - name: Validate PowerScale CSI input paths
       ansible.builtin.assert:

src/orchestrator/roles/k8s_config/templates/ps_storage_class.j2

@@ -6,9 +6,15 @@ provisioner: csi-isilon.dellemc.com
 reclaimPolicy: Retain
 allowVolumeExpansion: true
 volumeBindingMode: Immediate
+{% if csi_powerscale_mount_options | default([]) | length > 0 %}
+mountOptions:
+{% for opt in csi_powerscale_mount_options %}
+  - "{{ opt }}"
+{% endfor %}
+{% endif %}
 parameters:
   AccessZone: {{ ps_access_zone }}
   Isipath: {{ ps_isipath }}
   RootClientEnabled: "true"
   AzServiceIP: {{ ps_azserviceip }}
-  csi.storage.k8s.io/fstype: "nfs"
+  csi.storage.k8s.io/fstype: "nfs"

src/orchestrator/roles/provision_common/tasks/configure_metadata_svc.yml

@@ -91,6 +91,9 @@
     metadata_svc_nfs_path_openldap: "{{ metadata_slurm_nfs_storage_matches[0].source }}/openldap"
     metadata_svc_slurm_nfs_path: "{{ metadata_slurm_nfs_storage_matches[0].source }}"
     client_mount_path: "{{ metadata_slurm_nfs_storage_matches[0].mount_point }}"
+    slurm_nfs_mnt_opts: >-
+      {{ metadata_slurm_nfs_storage_matches[0].node_nfs_mnt_opts
+         | default(slurm_nfs_default_mnt_opts, true) }}
   when: target_category | default('') == 'slurm'
 
 - name: Set default effective directories for slurm paths
@@ -222,6 +225,11 @@
   ansible.builtin.set_fact:
     k8s_client_mount_path: "{{ metadata_k8s_storage_matches[0].mount_point }}"
     k8s_nfs_server_path: "{{ metadata_k8s_storage_matches[0].source }}"
+    k8s_nfs_mnt_opts: >-
+      {{ metadata_k8s_storage_matches[0].node_nfs_mnt_opts
+         | default(k8s_nfs_default_mnt_opts, true) }}
+    k8s_nfs_custom_mnt_opts: >-
+      {{ metadata_k8s_storage_matches[0].node_nfs_mnt_opts | default('', true) }}
   when: target_category | default('') == 'kubernetes'
 
 - name: Load K8s packages JSON for template variables

src/orchestrator/roles/provision_common/templates/metadata_svc/ms-group-login_compiler_node_aarch64.yaml.j2

@@ -237,7 +237,7 @@
         - /usr/local/bin/configure_vast_installation.sh
 {% endif %}
         - mkdir -p {{ slurm_slurmd_log_dir_effective }} {{ slurm_slurmd_pid_dir_effective }} {{ slurm_slurmd_spool_dir_effective }} {{ slurm_epilog_dirs_all | join(' ') }} /etc/munge /cert /var/log/track /var/lib/packages
-        - echo "{{ metadata_svc_nfs_path }}/cert  /cert   nfs defaults,_netdev 0 0" >> /etc/fstab
+        - echo "{{ metadata_svc_nfs_path }}/cert  /cert   nfs {{ slurm_nfs_mnt_opts | default(slurm_nfs_default_mnt_opts) }} 0 0" >> /etc/fstab
         - mount -av
         - rm -f /etc/slurm/slurm.conf /etc/slurm/slurm.conf.example
         - cp /cert/pulp_webserver.crt /etc/pki/ca-trust/source/anchors && update-ca-trust
@@ -274,13 +274,13 @@
         # Add the Slurm SSH child mount after the configured storage root is mounted
         - mkdir -p {{ client_mount_path }}/slurm/ssh
         - mkdir -p {{ slurm_slurmd_log_dir_effective }} {{ slurm_slurmd_pid_dir_effective }} {{ slurm_slurmd_spool_dir_effective }} {{ slurm_epilog_dirs_all | join(' ') }} /etc/munge /cert /var/log/track /var/lib/packages /hpc_tools
-        - echo "{{ metadata_svc_nfs_path }}/$(hostname -s)/var/log/slurm  {{ slurm_slurmd_log_dir_effective }}   nfs defaults,_netdev 0 0" >> /etc/fstab
-        - echo "{{ metadata_svc_nfs_path }}/$(hostname -s)/var/spool/slurmd      {{ slurm_slurmd_spool_dir_effective }}       nfs defaults,_netdev 0 0" >> /etc/fstab
-        - echo "{{ metadata_svc_nfs_path }}/$(hostname -s)/etc/slurm/epilog.d     /etc/slurm/epilog.d      nfs defaults,_netdev 0 0" >> /etc/fstab
-        - echo "{{ metadata_svc_nfs_path }}/$(hostname -s)/etc/munge      /etc/munge       nfs defaults,_netdev 0 0" >> /etc/fstab
-        - echo "{{ trackfile_nfs_path }}    /var/log/track       nfs defaults,_netdev 0 0" >> /etc/fstab
-        - echo "{{ metadata_svc_nfs_path }}/ssh {{ client_mount_path }}/slurm/ssh nfs defaults,_netdev 0 0" >> /etc/fstab
-        - echo "{{ metadata_svc_nfs_path }}/packages  /var/lib/packages   nfs defaults,_netdev 0 0" >> /etc/fstab
+        - echo "{{ metadata_svc_nfs_path }}/$(hostname -s)/var/log/slurm  {{ slurm_slurmd_log_dir_effective }}   nfs {{ slurm_nfs_mnt_opts | default(slurm_nfs_default_mnt_opts) }} 0 0" >> /etc/fstab
+        - echo "{{ metadata_svc_nfs_path }}/$(hostname -s)/var/spool/slurmd      {{ slurm_slurmd_spool_dir_effective }}       nfs {{ slurm_nfs_mnt_opts | default(slurm_nfs_default_mnt_opts) }} 0 0" >> /etc/fstab
+        - echo "{{ metadata_svc_nfs_path }}/$(hostname -s)/etc/slurm/epilog.d     /etc/slurm/epilog.d      nfs {{ slurm_nfs_mnt_opts | default(slurm_nfs_default_mnt_opts) }} 0 0" >> /etc/fstab
+        - echo "{{ metadata_svc_nfs_path }}/$(hostname -s)/etc/munge      /etc/munge       nfs {{ slurm_nfs_mnt_opts | default(slurm_nfs_default_mnt_opts) }} 0 0" >> /etc/fstab
+        - echo "{{ trackfile_nfs_path }}    /var/log/track       nfs {{ slurm_nfs_mnt_opts | default(slurm_nfs_default_mnt_opts) }} 0 0" >> /etc/fstab
+        - echo "{{ metadata_svc_nfs_path }}/ssh {{ client_mount_path }}/slurm/ssh nfs {{ slurm_nfs_mnt_opts | default(slurm_nfs_default_mnt_opts) }} 0 0" >> /etc/fstab
+        - echo "{{ metadata_svc_nfs_path }}/packages  /var/lib/packages   nfs {{ slurm_nfs_mnt_opts | default(slurm_nfs_default_mnt_opts) }} 0 0" >> /etc/fstab
         - chmod {{ file_mode }} /etc/fstab
         - mount -av
 
@@ -390,7 +390,7 @@
 {% if hostvars['localhost']['openldap_support'] %}
         - /usr/local/bin/update_ldap_conf.sh
         - mkdir /ldapcerts
-        - echo "{{ metadata_svc_nfs_path_openldap }}/certs                /ldapcerts       nfs defaults,_netdev 0 0" >> /etc/fstab
+        - echo "{{ metadata_svc_nfs_path_openldap }}/certs                /ldapcerts       nfs {{ slurm_nfs_mnt_opts | default(slurm_nfs_default_mnt_opts) }} 0 0" >> /etc/fstab
         - mkdir -pv {{ vast_mount_point }}/{{ openldap_dir_name }}/ldapuser /home
         - echo "{{ vast_mount_point }}/{{ openldap_dir_name }}/ldapuser /home  none  bind  0  0" >> /etc/fstab
         - chmod {{ file_mode }} /etc/fstab

src/orchestrator/roles/provision_common/templates/metadata_svc/ms-group-login_compiler_node_x86_64.yaml.j2

@@ -237,7 +237,7 @@
         - /usr/local/bin/configure_vast_installation.sh
 {% endif %}
         - mkdir -p {{ slurm_slurmd_log_dir_effective }} {{ slurm_slurmd_pid_dir_effective }} {{ slurm_slurmd_spool_dir_effective }} {{ slurm_epilog_dirs_all | join(' ') }} /etc/munge /cert /var/log/track /var/lib/packages
-        - echo "{{ metadata_svc_nfs_path }}/cert  /cert   nfs defaults,_netdev 0 0" >> /etc/fstab
+        - echo "{{ metadata_svc_nfs_path }}/cert  /cert   nfs {{ slurm_nfs_mnt_opts | default(slurm_nfs_default_mnt_opts) }} 0 0" >> /etc/fstab
         - mount -av
         - rm -f /etc/slurm/slurm.conf /etc/slurm/slurm.conf.example
         - cp /cert/pulp_webserver.crt /etc/pki/ca-trust/source/anchors && update-ca-trust
@@ -274,13 +274,13 @@
         # Add the Slurm SSH child mount after the configured storage root is mounted
         - mkdir -p {{ client_mount_path }}/slurm/ssh 
         - mkdir -p {{ slurm_slurmd_log_dir_effective }} {{ slurm_slurmd_pid_dir_effective }} {{ slurm_slurmd_spool_dir_effective }} {{ slurm_epilog_dirs_all | join(' ') }} /etc/munge /cert /var/log/track /var/lib/packages
-        - echo "{{ metadata_svc_nfs_path }}/$(hostname -s)/var/log/slurm  {{ slurm_slurmd_log_dir_effective }}   nfs defaults,_netdev 0 0" >> /etc/fstab
-        - echo "{{ metadata_svc_nfs_path }}/$(hostname -s)/var/spool/slurmd      {{ slurm_slurmd_spool_dir_effective }}       nfs defaults,_netdev 0 0" >> /etc/fstab
-        - echo "{{ metadata_svc_nfs_path }}/$(hostname -s)/etc/slurm/epilog.d     /etc/slurm/epilog.d      nfs defaults,_netdev 0 0" >> /etc/fstab
-        - echo "{{ metadata_svc_nfs_path }}/$(hostname -s)/etc/munge      /etc/munge       nfs defaults,_netdev 0 0" >> /etc/fstab
-        - echo "{{ trackfile_nfs_path }}    /var/log/track       nfs defaults,_netdev 0 0" >> /etc/fstab
-        - echo "{{ metadata_svc_nfs_path }}/ssh {{ client_mount_path }}/slurm/ssh nfs defaults,_netdev 0 0" >> /etc/fstab
-        - echo "{{ metadata_svc_nfs_path }}/packages  /var/lib/packages   nfs defaults,_netdev 0 0" >> /etc/fstab
+        - echo "{{ metadata_svc_nfs_path }}/$(hostname -s)/var/log/slurm  {{ slurm_slurmd_log_dir_effective }}   nfs {{ slurm_nfs_mnt_opts | default(slurm_nfs_default_mnt_opts) }} 0 0" >> /etc/fstab
+        - echo "{{ metadata_svc_nfs_path }}/$(hostname -s)/var/spool/slurmd      {{ slurm_slurmd_spool_dir_effective }}       nfs {{ slurm_nfs_mnt_opts | default(slurm_nfs_default_mnt_opts) }} 0 0" >> /etc/fstab
+        - echo "{{ metadata_svc_nfs_path }}/$(hostname -s)/etc/slurm/epilog.d     /etc/slurm/epilog.d      nfs {{ slurm_nfs_mnt_opts | default(slurm_nfs_default_mnt_opts) }} 0 0" >> /etc/fstab
+        - echo "{{ metadata_svc_nfs_path }}/$(hostname -s)/etc/munge      /etc/munge       nfs {{ slurm_nfs_mnt_opts | default(slurm_nfs_default_mnt_opts) }} 0 0" >> /etc/fstab
+        - echo "{{ trackfile_nfs_path }}    /var/log/track       nfs {{ slurm_nfs_mnt_opts | default(slurm_nfs_default_mnt_opts) }} 0 0" >> /etc/fstab
+        - echo "{{ metadata_svc_nfs_path }}/ssh {{ client_mount_path }}/slurm/ssh nfs {{ slurm_nfs_mnt_opts | default(slurm_nfs_default_mnt_opts) }} 0 0" >> /etc/fstab
+        - echo "{{ metadata_svc_nfs_path }}/packages  /var/lib/packages   nfs {{ slurm_nfs_mnt_opts | default(slurm_nfs_default_mnt_opts) }} 0 0" >> /etc/fstab
         - chmod {{ file_mode }} /etc/fstab
         - mount -a
 
@@ -392,7 +392,7 @@
 {% if hostvars['localhost']['openldap_support'] %}
         - /usr/local/bin/update_ldap_conf.sh
         - mkdir /ldapcerts
-        - echo "{{ metadata_svc_nfs_path_openldap }}/certs                /ldapcerts       nfs defaults,_netdev 0 0" >> /etc/fstab
+        - echo "{{ metadata_svc_nfs_path_openldap }}/certs                /ldapcerts       nfs {{ slurm_nfs_mnt_opts | default(slurm_nfs_default_mnt_opts) }} 0 0" >> /etc/fstab
         - mkdir -pv {{ vast_mount_point }}/{{ openldap_dir_name }}/ldapuser /home
         - echo "{{ vast_mount_point }}/{{ openldap_dir_name }}/ldapuser /home  none  bind  0  0" >> /etc/fstab
         - chmod {{ file_mode }} /etc/fstab

src/orchestrator/roles/provision_common/templates/metadata_svc/ms-group-login_node_aarch64.yaml.j2

@@ -193,7 +193,7 @@
         - /usr/local/bin/configure_vast_installation.sh
 {% endif %}
         - mkdir -p {{ slurm_slurmd_log_dir_effective }} {{ slurm_slurmd_pid_dir_effective }} {{ slurm_slurmd_spool_dir_effective }} {{ slurm_epilog_dirs_all | join(' ') }} /etc/munge /cert /var/log/track /var/lib/packages
-        - echo "{{ metadata_svc_nfs_path }}/cert  /cert   nfs defaults,_netdev 0 0" >> /etc/fstab
+        - echo "{{ metadata_svc_nfs_path }}/cert  /cert   nfs {{ slurm_nfs_mnt_opts | default(slurm_nfs_default_mnt_opts) }} 0 0" >> /etc/fstab
         - mount -av
         - rm -f /etc/slurm/slurm.conf /etc/slurm/slurm.conf.example
         - cp /cert/pulp_webserver.crt /etc/pki/ca-trust/source/anchors && update-ca-trust
@@ -230,13 +230,13 @@
         - mkdir -p {{ client_mount_path }}/slurm/ssh
         - mkdir -p {{ slurm_slurmd_log_dir_effective }} {{ slurm_slurmd_pid_dir_effective }} {{ slurm_slurmd_spool_dir_effective }} {{ slurm_epilog_dirs_all | join(' ') }} /etc/munge /cert /var/log/track /var/lib/packages
         # Additional Slurm login node mounts (excluding cert which was mounted earlier)
-        - echo "{{ metadata_svc_nfs_path }}/$(hostname -s)/var/log/slurm  {{ slurm_slurmd_log_dir_effective }}   nfs defaults,_netdev 0 0" >> /etc/fstab
-        - echo "{{ metadata_svc_nfs_path }}/$(hostname -s)/var/spool/slurmd      {{ slurm_slurmd_spool_dir_effective }}       nfs defaults,_netdev 0 0" >> /etc/fstab
-        - echo "{{ metadata_svc_nfs_path }}/$(hostname -s)/etc/slurm/epilog.d     /etc/slurm/epilog.d      nfs defaults,_netdev 0 0" >> /etc/fstab
-        - echo "{{ metadata_svc_nfs_path }}/$(hostname -s)/etc/munge      /etc/munge       nfs defaults,_netdev 0 0" >> /etc/fstab
-        - echo "{{ trackfile_nfs_path }}    /var/log/track       nfs defaults,_netdev 0 0" >> /etc/fstab
-        - echo "{{ metadata_svc_nfs_path }}/ssh {{ client_mount_path }}/slurm/ssh nfs defaults,_netdev 0 0" >> /etc/fstab
-        - echo "{{ metadata_svc_nfs_path }}/packages  /var/lib/packages   nfs defaults,_netdev 0 0" >> /etc/fstab
+        - echo "{{ metadata_svc_nfs_path }}/$(hostname -s)/var/log/slurm  {{ slurm_slurmd_log_dir_effective }}   nfs {{ slurm_nfs_mnt_opts | default(slurm_nfs_default_mnt_opts) }} 0 0" >> /etc/fstab
+        - echo "{{ metadata_svc_nfs_path }}/$(hostname -s)/var/spool/slurmd      {{ slurm_slurmd_spool_dir_effective }}       nfs {{ slurm_nfs_mnt_opts | default(slurm_nfs_default_mnt_opts) }} 0 0" >> /etc/fstab
+        - echo "{{ metadata_svc_nfs_path }}/$(hostname -s)/etc/slurm/epilog.d     /etc/slurm/epilog.d      nfs {{ slurm_nfs_mnt_opts | default(slurm_nfs_default_mnt_opts) }} 0 0" >> /etc/fstab
+        - echo "{{ metadata_svc_nfs_path }}/$(hostname -s)/etc/munge      /etc/munge       nfs {{ slurm_nfs_mnt_opts | default(slurm_nfs_default_mnt_opts) }} 0 0" >> /etc/fstab
+        - echo "{{ trackfile_nfs_path }}    /var/log/track       nfs {{ slurm_nfs_mnt_opts | default(slurm_nfs_default_mnt_opts) }} 0 0" >> /etc/fstab
+        - echo "{{ metadata_svc_nfs_path }}/ssh {{ client_mount_path }}/slurm/ssh nfs {{ slurm_nfs_mnt_opts | default(slurm_nfs_default_mnt_opts) }} 0 0" >> /etc/fstab
+        - echo "{{ metadata_svc_nfs_path }}/packages  /var/lib/packages   nfs {{ slurm_nfs_mnt_opts | default(slurm_nfs_default_mnt_opts) }} 0 0" >> /etc/fstab
         - chmod {{ file_mode }} /etc/fstab
         - mount -av
         - /usr/local/bin/check_slurm_controller_status.sh
@@ -326,7 +326,7 @@
 {% if hostvars['localhost']['openldap_support'] %}
         - /usr/local/bin/update_ldap_conf.sh
         - mkdir /ldapcerts
-        - echo "{{ metadata_svc_nfs_path_openldap }}/certs                /ldapcerts       nfs defaults,_netdev 0 0" >> /etc/fstab
+        - echo "{{ metadata_svc_nfs_path_openldap }}/certs                /ldapcerts       nfs {{ slurm_nfs_mnt_opts | default(slurm_nfs_default_mnt_opts) }} 0 0" >> /etc/fstab
         - mkdir -pv {{ vast_mount_point }}/{{ openldap_dir_name }}/ldapuser /home
         - echo "{{ vast_mount_point }}/{{ openldap_dir_name }}/ldapuser /home  none  bind  0  0" >> /etc/fstab
         - chmod {{ file_mode }} /etc/fstab