diff --git a/src/orchestrator/input/omnia_config.yml b/src/orchestrator/input/omnia_config.yml index 17ae4c810c..b39044ad22 100644 --- a/src/orchestrator/input/omnia_config.yml +++ b/src/orchestrator/input/omnia_config.yml @@ -186,6 +186,10 @@ slurm_cluster: # File path for the values.yml file which will contain the Powerscale driver configuration parameters. # csi_powerscale_driver_values_file_path: User need to download values.yaml file and fill required data. # Provided the path of the values.yaml file here. +# csi_powerscale_mount_options: Optional comma-separated NFS mount options applied to +# PowerScale CSI volumes through the ps01 StorageClass mountOptions. Leave empty to use +# the kernel defaults. For RHEL 10.x clients that report Input/output errors on PowerScale +# volumes with the default 1 MiB transfer size, use "rsize=1044480,wsize=1044480". # mention configurable values # - k8s_crio_storage_size: Specifies the disk size allocated for CRI-O container storage. @@ -224,5 +228,6 @@ service_k8s_cluster: enable_powerscale_csi: false csi_powerscale_driver_secret_file_path: "" csi_powerscale_driver_values_file_path: "" + csi_powerscale_mount_options: "" enable_k8s_ha: true virtual_ip_address: "172.16.107.1" diff --git a/src/orchestrator/input/storage_config.yml b/src/orchestrator/input/storage_config.yml index daf735a23c..a5c4e9714b 100644 --- a/src/orchestrator/input/storage_config.yml +++ b/src/orchestrator/input/storage_config.yml @@ -47,6 +47,17 @@ # If specified, takes PRIORITY over mount_params profile # - mount_on_oim: Mount this filesystem on the OIM node as well. Optional. Default: false # Ensure storage is network-accessible from OIM before enabling +# - node_nfs_mnt_opts: Mount options for the per-node NFS sub-mounts that Omnia creates +# from the storage entry referenced by nfs_storage_name in omnia_config.yml. +# Optional. Applies to Kubernetes (/var/lib/kubelet, /var/lib/etcd, +# /etc/kubernetes, /var/log/pods, /var/lib/packages) and Slurm/login +# (/etc/slurm, /etc/munge, /var/spool/slurmd, /var/log/slurm, /cert, ...). +# For Kubernetes it is also passed to the nfs-client provisioner. +# Default: "noatime,nolock" (Kubernetes) and "defaults,_netdev" (Slurm) +# PowerScale with RHEL 10.x clients: if Input/output errors occur with the +# default 1 MiB transfer size, set for example +# "noatime,nolock,rsize=1044480,wsize=1044480" (Kubernetes) or +# "defaults,_netdev,rsize=1044480,wsize=1044480" (Slurm) # # Node-specific bind mounts (node_key and node_mount_point are paired): # - node_key: Per-node subdirectory isolation variable. Optional diff --git a/src/orchestrator/plugins/module_utils/orchestrator_validation/schema/omnia_config.json b/src/orchestrator/plugins/module_utils/orchestrator_validation/schema/omnia_config.json index 5039a3708e..254efa9482 100644 --- a/src/orchestrator/plugins/module_utils/orchestrator_validation/schema/omnia_config.json +++ b/src/orchestrator/plugins/module_utils/orchestrator_validation/schema/omnia_config.json @@ -276,6 +276,10 @@ "csi_powerscale_driver_values_file_path": { "$ref": "#/definitions/optionalAbsolutePath" }, + "csi_powerscale_mount_options": { + "type": "string", + "pattern": "^([A-Za-z0-9,=._@/-]+)?$" + }, "enable_k8s_ha": { "type": "boolean" }, diff --git a/src/orchestrator/plugins/module_utils/orchestrator_validation/schema/storage_config.json b/src/orchestrator/plugins/module_utils/orchestrator_validation/schema/storage_config.json index 85dd430b2c..79ffa4ba65 100644 --- a/src/orchestrator/plugins/module_utils/orchestrator_validation/schema/storage_config.json +++ b/src/orchestrator/plugins/module_utils/orchestrator_validation/schema/storage_config.json @@ -171,6 +171,9 @@ "mnt_opts": { "$ref": "#/definitions/mountOptions" }, + "node_nfs_mnt_opts": { + "$ref": "#/definitions/mountOptions" + }, "dump_freq": { "$ref": "#/definitions/dumpFrequency" }, diff --git a/src/orchestrator/roles/k8s_config/tasks/create_k8s_config_nfs.yml b/src/orchestrator/roles/k8s_config/tasks/create_k8s_config_nfs.yml index a08bef336e..867a91d287 100644 --- a/src/orchestrator/roles/k8s_config/tasks/create_k8s_config_nfs.yml +++ b/src/orchestrator/roles/k8s_config/tasks/create_k8s_config_nfs.yml @@ -61,6 +61,10 @@ csi_powerscale_driver_values_file_path: >- {{ deployed_service_k8s_cluster.get( 'csi_powerscale_driver_values_file_path', '') }} + csi_powerscale_mount_options: >- + {{ (deployed_service_k8s_cluster.get('csi_powerscale_mount_options', '') + | default('', true) | string | trim).split(',') + | map('trim') | select | list }} - name: Validate PowerScale CSI input paths ansible.builtin.assert: diff --git a/src/orchestrator/roles/k8s_config/templates/ps_storage_class.j2 b/src/orchestrator/roles/k8s_config/templates/ps_storage_class.j2 index c407073b2e..405660c74d 100644 --- a/src/orchestrator/roles/k8s_config/templates/ps_storage_class.j2 +++ b/src/orchestrator/roles/k8s_config/templates/ps_storage_class.j2 @@ -6,9 +6,15 @@ provisioner: csi-isilon.dellemc.com reclaimPolicy: Retain allowVolumeExpansion: true volumeBindingMode: Immediate +{% if csi_powerscale_mount_options | default([]) | length > 0 %} +mountOptions: +{% for opt in csi_powerscale_mount_options %} + - "{{ opt }}" +{% endfor %} +{% endif %} parameters: AccessZone: {{ ps_access_zone }} Isipath: {{ ps_isipath }} RootClientEnabled: "true" AzServiceIP: {{ ps_azserviceip }} - csi.storage.k8s.io/fstype: "nfs" \ No newline at end of file + csi.storage.k8s.io/fstype: "nfs" diff --git a/src/orchestrator/roles/provision_common/tasks/configure_metadata_svc.yml b/src/orchestrator/roles/provision_common/tasks/configure_metadata_svc.yml index 1f527e56d3..c9e22efb20 100644 --- a/src/orchestrator/roles/provision_common/tasks/configure_metadata_svc.yml +++ b/src/orchestrator/roles/provision_common/tasks/configure_metadata_svc.yml @@ -91,6 +91,9 @@ metadata_svc_nfs_path_openldap: "{{ metadata_slurm_nfs_storage_matches[0].source }}/openldap" metadata_svc_slurm_nfs_path: "{{ metadata_slurm_nfs_storage_matches[0].source }}" client_mount_path: "{{ metadata_slurm_nfs_storage_matches[0].mount_point }}" + slurm_nfs_mnt_opts: >- + {{ metadata_slurm_nfs_storage_matches[0].node_nfs_mnt_opts + | default(slurm_nfs_default_mnt_opts, true) }} when: target_category | default('') == 'slurm' - name: Set default effective directories for slurm paths @@ -222,6 +225,11 @@ ansible.builtin.set_fact: k8s_client_mount_path: "{{ metadata_k8s_storage_matches[0].mount_point }}" k8s_nfs_server_path: "{{ metadata_k8s_storage_matches[0].source }}" + k8s_nfs_mnt_opts: >- + {{ metadata_k8s_storage_matches[0].node_nfs_mnt_opts + | default(k8s_nfs_default_mnt_opts, true) }} + k8s_nfs_custom_mnt_opts: >- + {{ metadata_k8s_storage_matches[0].node_nfs_mnt_opts | default('', true) }} when: target_category | default('') == 'kubernetes' - name: Load K8s packages JSON for template variables diff --git a/src/orchestrator/roles/provision_common/templates/metadata_svc/ms-group-login_compiler_node_aarch64.yaml.j2 b/src/orchestrator/roles/provision_common/templates/metadata_svc/ms-group-login_compiler_node_aarch64.yaml.j2 index 2a7f56de5b..4df9e48c69 100644 --- a/src/orchestrator/roles/provision_common/templates/metadata_svc/ms-group-login_compiler_node_aarch64.yaml.j2 +++ b/src/orchestrator/roles/provision_common/templates/metadata_svc/ms-group-login_compiler_node_aarch64.yaml.j2 @@ -237,7 +237,7 @@ - /usr/local/bin/configure_vast_installation.sh {% endif %} - mkdir -p {{ slurm_slurmd_log_dir_effective }} {{ slurm_slurmd_pid_dir_effective }} {{ slurm_slurmd_spool_dir_effective }} {{ slurm_epilog_dirs_all | join(' ') }} /etc/munge /cert /var/log/track /var/lib/packages - - echo "{{ metadata_svc_nfs_path }}/cert /cert nfs defaults,_netdev 0 0" >> /etc/fstab + - echo "{{ metadata_svc_nfs_path }}/cert /cert nfs {{ slurm_nfs_mnt_opts | default(slurm_nfs_default_mnt_opts) }} 0 0" >> /etc/fstab - mount -av - rm -f /etc/slurm/slurm.conf /etc/slurm/slurm.conf.example - cp /cert/pulp_webserver.crt /etc/pki/ca-trust/source/anchors && update-ca-trust @@ -274,13 +274,13 @@ # Add the Slurm SSH child mount after the configured storage root is mounted - mkdir -p {{ client_mount_path }}/slurm/ssh - mkdir -p {{ slurm_slurmd_log_dir_effective }} {{ slurm_slurmd_pid_dir_effective }} {{ slurm_slurmd_spool_dir_effective }} {{ slurm_epilog_dirs_all | join(' ') }} /etc/munge /cert /var/log/track /var/lib/packages /hpc_tools - - echo "{{ metadata_svc_nfs_path }}/$(hostname -s)/var/log/slurm {{ slurm_slurmd_log_dir_effective }} nfs defaults,_netdev 0 0" >> /etc/fstab - - echo "{{ metadata_svc_nfs_path }}/$(hostname -s)/var/spool/slurmd {{ slurm_slurmd_spool_dir_effective }} nfs defaults,_netdev 0 0" >> /etc/fstab - - echo "{{ metadata_svc_nfs_path }}/$(hostname -s)/etc/slurm/epilog.d /etc/slurm/epilog.d nfs defaults,_netdev 0 0" >> /etc/fstab - - echo "{{ metadata_svc_nfs_path }}/$(hostname -s)/etc/munge /etc/munge nfs defaults,_netdev 0 0" >> /etc/fstab - - echo "{{ trackfile_nfs_path }} /var/log/track nfs defaults,_netdev 0 0" >> /etc/fstab - - echo "{{ metadata_svc_nfs_path }}/ssh {{ client_mount_path }}/slurm/ssh nfs defaults,_netdev 0 0" >> /etc/fstab - - echo "{{ metadata_svc_nfs_path }}/packages /var/lib/packages nfs defaults,_netdev 0 0" >> /etc/fstab + - echo "{{ metadata_svc_nfs_path }}/$(hostname -s)/var/log/slurm {{ slurm_slurmd_log_dir_effective }} nfs {{ slurm_nfs_mnt_opts | default(slurm_nfs_default_mnt_opts) }} 0 0" >> /etc/fstab + - echo "{{ metadata_svc_nfs_path }}/$(hostname -s)/var/spool/slurmd {{ slurm_slurmd_spool_dir_effective }} nfs {{ slurm_nfs_mnt_opts | default(slurm_nfs_default_mnt_opts) }} 0 0" >> /etc/fstab + - echo "{{ metadata_svc_nfs_path }}/$(hostname -s)/etc/slurm/epilog.d /etc/slurm/epilog.d nfs {{ slurm_nfs_mnt_opts | default(slurm_nfs_default_mnt_opts) }} 0 0" >> /etc/fstab + - echo "{{ metadata_svc_nfs_path }}/$(hostname -s)/etc/munge /etc/munge nfs {{ slurm_nfs_mnt_opts | default(slurm_nfs_default_mnt_opts) }} 0 0" >> /etc/fstab + - echo "{{ trackfile_nfs_path }} /var/log/track nfs {{ slurm_nfs_mnt_opts | default(slurm_nfs_default_mnt_opts) }} 0 0" >> /etc/fstab + - echo "{{ metadata_svc_nfs_path }}/ssh {{ client_mount_path }}/slurm/ssh nfs {{ slurm_nfs_mnt_opts | default(slurm_nfs_default_mnt_opts) }} 0 0" >> /etc/fstab + - echo "{{ metadata_svc_nfs_path }}/packages /var/lib/packages nfs {{ slurm_nfs_mnt_opts | default(slurm_nfs_default_mnt_opts) }} 0 0" >> /etc/fstab - chmod {{ file_mode }} /etc/fstab - mount -av @@ -390,7 +390,7 @@ {% if hostvars['localhost']['openldap_support'] %} - /usr/local/bin/update_ldap_conf.sh - mkdir /ldapcerts - - echo "{{ metadata_svc_nfs_path_openldap }}/certs /ldapcerts nfs defaults,_netdev 0 0" >> /etc/fstab + - echo "{{ metadata_svc_nfs_path_openldap }}/certs /ldapcerts nfs {{ slurm_nfs_mnt_opts | default(slurm_nfs_default_mnt_opts) }} 0 0" >> /etc/fstab - mkdir -pv {{ vast_mount_point }}/{{ openldap_dir_name }}/ldapuser /home - echo "{{ vast_mount_point }}/{{ openldap_dir_name }}/ldapuser /home none bind 0 0" >> /etc/fstab - chmod {{ file_mode }} /etc/fstab diff --git a/src/orchestrator/roles/provision_common/templates/metadata_svc/ms-group-login_compiler_node_x86_64.yaml.j2 b/src/orchestrator/roles/provision_common/templates/metadata_svc/ms-group-login_compiler_node_x86_64.yaml.j2 index 70e0d6ee20..61ab5f2718 100644 --- a/src/orchestrator/roles/provision_common/templates/metadata_svc/ms-group-login_compiler_node_x86_64.yaml.j2 +++ b/src/orchestrator/roles/provision_common/templates/metadata_svc/ms-group-login_compiler_node_x86_64.yaml.j2 @@ -237,7 +237,7 @@ - /usr/local/bin/configure_vast_installation.sh {% endif %} - mkdir -p {{ slurm_slurmd_log_dir_effective }} {{ slurm_slurmd_pid_dir_effective }} {{ slurm_slurmd_spool_dir_effective }} {{ slurm_epilog_dirs_all | join(' ') }} /etc/munge /cert /var/log/track /var/lib/packages - - echo "{{ metadata_svc_nfs_path }}/cert /cert nfs defaults,_netdev 0 0" >> /etc/fstab + - echo "{{ metadata_svc_nfs_path }}/cert /cert nfs {{ slurm_nfs_mnt_opts | default(slurm_nfs_default_mnt_opts) }} 0 0" >> /etc/fstab - mount -av - rm -f /etc/slurm/slurm.conf /etc/slurm/slurm.conf.example - cp /cert/pulp_webserver.crt /etc/pki/ca-trust/source/anchors && update-ca-trust @@ -274,13 +274,13 @@ # Add the Slurm SSH child mount after the configured storage root is mounted - mkdir -p {{ client_mount_path }}/slurm/ssh - mkdir -p {{ slurm_slurmd_log_dir_effective }} {{ slurm_slurmd_pid_dir_effective }} {{ slurm_slurmd_spool_dir_effective }} {{ slurm_epilog_dirs_all | join(' ') }} /etc/munge /cert /var/log/track /var/lib/packages - - echo "{{ metadata_svc_nfs_path }}/$(hostname -s)/var/log/slurm {{ slurm_slurmd_log_dir_effective }} nfs defaults,_netdev 0 0" >> /etc/fstab - - echo "{{ metadata_svc_nfs_path }}/$(hostname -s)/var/spool/slurmd {{ slurm_slurmd_spool_dir_effective }} nfs defaults,_netdev 0 0" >> /etc/fstab - - echo "{{ metadata_svc_nfs_path }}/$(hostname -s)/etc/slurm/epilog.d /etc/slurm/epilog.d nfs defaults,_netdev 0 0" >> /etc/fstab - - echo "{{ metadata_svc_nfs_path }}/$(hostname -s)/etc/munge /etc/munge nfs defaults,_netdev 0 0" >> /etc/fstab - - echo "{{ trackfile_nfs_path }} /var/log/track nfs defaults,_netdev 0 0" >> /etc/fstab - - echo "{{ metadata_svc_nfs_path }}/ssh {{ client_mount_path }}/slurm/ssh nfs defaults,_netdev 0 0" >> /etc/fstab - - echo "{{ metadata_svc_nfs_path }}/packages /var/lib/packages nfs defaults,_netdev 0 0" >> /etc/fstab + - echo "{{ metadata_svc_nfs_path }}/$(hostname -s)/var/log/slurm {{ slurm_slurmd_log_dir_effective }} nfs {{ slurm_nfs_mnt_opts | default(slurm_nfs_default_mnt_opts) }} 0 0" >> /etc/fstab + - echo "{{ metadata_svc_nfs_path }}/$(hostname -s)/var/spool/slurmd {{ slurm_slurmd_spool_dir_effective }} nfs {{ slurm_nfs_mnt_opts | default(slurm_nfs_default_mnt_opts) }} 0 0" >> /etc/fstab + - echo "{{ metadata_svc_nfs_path }}/$(hostname -s)/etc/slurm/epilog.d /etc/slurm/epilog.d nfs {{ slurm_nfs_mnt_opts | default(slurm_nfs_default_mnt_opts) }} 0 0" >> /etc/fstab + - echo "{{ metadata_svc_nfs_path }}/$(hostname -s)/etc/munge /etc/munge nfs {{ slurm_nfs_mnt_opts | default(slurm_nfs_default_mnt_opts) }} 0 0" >> /etc/fstab + - echo "{{ trackfile_nfs_path }} /var/log/track nfs {{ slurm_nfs_mnt_opts | default(slurm_nfs_default_mnt_opts) }} 0 0" >> /etc/fstab + - echo "{{ metadata_svc_nfs_path }}/ssh {{ client_mount_path }}/slurm/ssh nfs {{ slurm_nfs_mnt_opts | default(slurm_nfs_default_mnt_opts) }} 0 0" >> /etc/fstab + - echo "{{ metadata_svc_nfs_path }}/packages /var/lib/packages nfs {{ slurm_nfs_mnt_opts | default(slurm_nfs_default_mnt_opts) }} 0 0" >> /etc/fstab - chmod {{ file_mode }} /etc/fstab - mount -a @@ -392,7 +392,7 @@ {% if hostvars['localhost']['openldap_support'] %} - /usr/local/bin/update_ldap_conf.sh - mkdir /ldapcerts - - echo "{{ metadata_svc_nfs_path_openldap }}/certs /ldapcerts nfs defaults,_netdev 0 0" >> /etc/fstab + - echo "{{ metadata_svc_nfs_path_openldap }}/certs /ldapcerts nfs {{ slurm_nfs_mnt_opts | default(slurm_nfs_default_mnt_opts) }} 0 0" >> /etc/fstab - mkdir -pv {{ vast_mount_point }}/{{ openldap_dir_name }}/ldapuser /home - echo "{{ vast_mount_point }}/{{ openldap_dir_name }}/ldapuser /home none bind 0 0" >> /etc/fstab - chmod {{ file_mode }} /etc/fstab diff --git a/src/orchestrator/roles/provision_common/templates/metadata_svc/ms-group-login_node_aarch64.yaml.j2 b/src/orchestrator/roles/provision_common/templates/metadata_svc/ms-group-login_node_aarch64.yaml.j2 index 3e65a32069..a601d80ad8 100644 --- a/src/orchestrator/roles/provision_common/templates/metadata_svc/ms-group-login_node_aarch64.yaml.j2 +++ b/src/orchestrator/roles/provision_common/templates/metadata_svc/ms-group-login_node_aarch64.yaml.j2 @@ -193,7 +193,7 @@ - /usr/local/bin/configure_vast_installation.sh {% endif %} - mkdir -p {{ slurm_slurmd_log_dir_effective }} {{ slurm_slurmd_pid_dir_effective }} {{ slurm_slurmd_spool_dir_effective }} {{ slurm_epilog_dirs_all | join(' ') }} /etc/munge /cert /var/log/track /var/lib/packages - - echo "{{ metadata_svc_nfs_path }}/cert /cert nfs defaults,_netdev 0 0" >> /etc/fstab + - echo "{{ metadata_svc_nfs_path }}/cert /cert nfs {{ slurm_nfs_mnt_opts | default(slurm_nfs_default_mnt_opts) }} 0 0" >> /etc/fstab - mount -av - rm -f /etc/slurm/slurm.conf /etc/slurm/slurm.conf.example - cp /cert/pulp_webserver.crt /etc/pki/ca-trust/source/anchors && update-ca-trust @@ -230,13 +230,13 @@ - mkdir -p {{ client_mount_path }}/slurm/ssh - mkdir -p {{ slurm_slurmd_log_dir_effective }} {{ slurm_slurmd_pid_dir_effective }} {{ slurm_slurmd_spool_dir_effective }} {{ slurm_epilog_dirs_all | join(' ') }} /etc/munge /cert /var/log/track /var/lib/packages # Additional Slurm login node mounts (excluding cert which was mounted earlier) - - echo "{{ metadata_svc_nfs_path }}/$(hostname -s)/var/log/slurm {{ slurm_slurmd_log_dir_effective }} nfs defaults,_netdev 0 0" >> /etc/fstab - - echo "{{ metadata_svc_nfs_path }}/$(hostname -s)/var/spool/slurmd {{ slurm_slurmd_spool_dir_effective }} nfs defaults,_netdev 0 0" >> /etc/fstab - - echo "{{ metadata_svc_nfs_path }}/$(hostname -s)/etc/slurm/epilog.d /etc/slurm/epilog.d nfs defaults,_netdev 0 0" >> /etc/fstab - - echo "{{ metadata_svc_nfs_path }}/$(hostname -s)/etc/munge /etc/munge nfs defaults,_netdev 0 0" >> /etc/fstab - - echo "{{ trackfile_nfs_path }} /var/log/track nfs defaults,_netdev 0 0" >> /etc/fstab - - echo "{{ metadata_svc_nfs_path }}/ssh {{ client_mount_path }}/slurm/ssh nfs defaults,_netdev 0 0" >> /etc/fstab - - echo "{{ metadata_svc_nfs_path }}/packages /var/lib/packages nfs defaults,_netdev 0 0" >> /etc/fstab + - echo "{{ metadata_svc_nfs_path }}/$(hostname -s)/var/log/slurm {{ slurm_slurmd_log_dir_effective }} nfs {{ slurm_nfs_mnt_opts | default(slurm_nfs_default_mnt_opts) }} 0 0" >> /etc/fstab + - echo "{{ metadata_svc_nfs_path }}/$(hostname -s)/var/spool/slurmd {{ slurm_slurmd_spool_dir_effective }} nfs {{ slurm_nfs_mnt_opts | default(slurm_nfs_default_mnt_opts) }} 0 0" >> /etc/fstab + - echo "{{ metadata_svc_nfs_path }}/$(hostname -s)/etc/slurm/epilog.d /etc/slurm/epilog.d nfs {{ slurm_nfs_mnt_opts | default(slurm_nfs_default_mnt_opts) }} 0 0" >> /etc/fstab + - echo "{{ metadata_svc_nfs_path }}/$(hostname -s)/etc/munge /etc/munge nfs {{ slurm_nfs_mnt_opts | default(slurm_nfs_default_mnt_opts) }} 0 0" >> /etc/fstab + - echo "{{ trackfile_nfs_path }} /var/log/track nfs {{ slurm_nfs_mnt_opts | default(slurm_nfs_default_mnt_opts) }} 0 0" >> /etc/fstab + - echo "{{ metadata_svc_nfs_path }}/ssh {{ client_mount_path }}/slurm/ssh nfs {{ slurm_nfs_mnt_opts | default(slurm_nfs_default_mnt_opts) }} 0 0" >> /etc/fstab + - echo "{{ metadata_svc_nfs_path }}/packages /var/lib/packages nfs {{ slurm_nfs_mnt_opts | default(slurm_nfs_default_mnt_opts) }} 0 0" >> /etc/fstab - chmod {{ file_mode }} /etc/fstab - mount -av - /usr/local/bin/check_slurm_controller_status.sh @@ -326,7 +326,7 @@ {% if hostvars['localhost']['openldap_support'] %} - /usr/local/bin/update_ldap_conf.sh - mkdir /ldapcerts - - echo "{{ metadata_svc_nfs_path_openldap }}/certs /ldapcerts nfs defaults,_netdev 0 0" >> /etc/fstab + - echo "{{ metadata_svc_nfs_path_openldap }}/certs /ldapcerts nfs {{ slurm_nfs_mnt_opts | default(slurm_nfs_default_mnt_opts) }} 0 0" >> /etc/fstab - mkdir -pv {{ vast_mount_point }}/{{ openldap_dir_name }}/ldapuser /home - echo "{{ vast_mount_point }}/{{ openldap_dir_name }}/ldapuser /home none bind 0 0" >> /etc/fstab - chmod {{ file_mode }} /etc/fstab diff --git a/src/orchestrator/roles/provision_common/templates/metadata_svc/ms-group-login_node_x86_64.yaml.j2 b/src/orchestrator/roles/provision_common/templates/metadata_svc/ms-group-login_node_x86_64.yaml.j2 index 0ffe8c2f2a..3b6412aeb1 100644 --- a/src/orchestrator/roles/provision_common/templates/metadata_svc/ms-group-login_node_x86_64.yaml.j2 +++ b/src/orchestrator/roles/provision_common/templates/metadata_svc/ms-group-login_node_x86_64.yaml.j2 @@ -192,7 +192,7 @@ - /usr/local/bin/configure_vast_installation.sh {% endif %} - mkdir -p {{ slurm_slurmd_log_dir_effective }} {{ slurm_slurmd_pid_dir_effective }} {{ slurm_slurmd_spool_dir_effective }} {{ slurm_epilog_dirs_all | join(' ') }} /etc/munge /cert /var/log/track /var/lib/packages - - echo "{{ metadata_svc_nfs_path }}/cert /cert nfs defaults,_netdev 0 0" >> /etc/fstab + - echo "{{ metadata_svc_nfs_path }}/cert /cert nfs {{ slurm_nfs_mnt_opts | default(slurm_nfs_default_mnt_opts) }} 0 0" >> /etc/fstab - mount -av - rm -f /etc/slurm/slurm.conf /etc/slurm/slurm.conf.example - cp /cert/pulp_webserver.crt /etc/pki/ca-trust/source/anchors && update-ca-trust @@ -226,14 +226,14 @@ - mount -av - chmod 1777 /scratch # Additional Slurm login node mounts (excluding cert which was mounted earlier) - - echo "{{ metadata_svc_nfs_path }}/$(hostname -s)/var/log/slurm {{ slurm_slurmd_log_dir_effective }} nfs defaults,_netdev 0 0" >> /etc/fstab - - echo "{{ metadata_svc_nfs_path }}/$(hostname -s)/var/spool/slurmd {{ slurm_slurmd_spool_dir_effective }} nfs defaults,_netdev 0 0" >> /etc/fstab - - echo "{{ metadata_svc_nfs_path }}/$(hostname -s)/etc/slurm/epilog.d /etc/slurm/epilog.d nfs defaults,_netdev 0 0" >> /etc/fstab - - echo "{{ metadata_svc_nfs_path }}/$(hostname -s)/etc/munge /etc/munge nfs defaults,_netdev 0 0" >> /etc/fstab - - echo "{{ trackfile_nfs_path }} /var/log/track nfs defaults,_netdev 0 0" >> /etc/fstab + - echo "{{ metadata_svc_nfs_path }}/$(hostname -s)/var/log/slurm {{ slurm_slurmd_log_dir_effective }} nfs {{ slurm_nfs_mnt_opts | default(slurm_nfs_default_mnt_opts) }} 0 0" >> /etc/fstab + - echo "{{ metadata_svc_nfs_path }}/$(hostname -s)/var/spool/slurmd {{ slurm_slurmd_spool_dir_effective }} nfs {{ slurm_nfs_mnt_opts | default(slurm_nfs_default_mnt_opts) }} 0 0" >> /etc/fstab + - echo "{{ metadata_svc_nfs_path }}/$(hostname -s)/etc/slurm/epilog.d /etc/slurm/epilog.d nfs {{ slurm_nfs_mnt_opts | default(slurm_nfs_default_mnt_opts) }} 0 0" >> /etc/fstab + - echo "{{ metadata_svc_nfs_path }}/$(hostname -s)/etc/munge /etc/munge nfs {{ slurm_nfs_mnt_opts | default(slurm_nfs_default_mnt_opts) }} 0 0" >> /etc/fstab + - echo "{{ trackfile_nfs_path }} /var/log/track nfs {{ slurm_nfs_mnt_opts | default(slurm_nfs_default_mnt_opts) }} 0 0" >> /etc/fstab - mkdir -p {{ client_mount_path }}/slurm/ssh - - echo "{{ metadata_svc_nfs_path }}/ssh {{ client_mount_path }}/slurm/ssh nfs defaults,_netdev 0 0" >> /etc/fstab - - echo "{{ metadata_svc_nfs_path }}/packages /var/lib/packages nfs defaults,_netdev 0 0" >> /etc/fstab + - echo "{{ metadata_svc_nfs_path }}/ssh {{ client_mount_path }}/slurm/ssh nfs {{ slurm_nfs_mnt_opts | default(slurm_nfs_default_mnt_opts) }} 0 0" >> /etc/fstab + - echo "{{ metadata_svc_nfs_path }}/packages /var/lib/packages nfs {{ slurm_nfs_mnt_opts | default(slurm_nfs_default_mnt_opts) }} 0 0" >> /etc/fstab - chmod {{ file_mode }} /etc/fstab - mount -av - /usr/local/bin/check_slurm_controller_status.sh @@ -324,7 +324,7 @@ {% if hostvars['localhost']['openldap_support'] %} - /usr/local/bin/update_ldap_conf.sh - mkdir /ldapcerts - - echo "{{ metadata_svc_nfs_path_openldap }}/certs /ldapcerts nfs defaults,_netdev 0 0" >> /etc/fstab + - echo "{{ metadata_svc_nfs_path_openldap }}/certs /ldapcerts nfs {{ slurm_nfs_mnt_opts | default(slurm_nfs_default_mnt_opts) }} 0 0" >> /etc/fstab - mkdir -pv {{ vast_mount_point }}/{{ openldap_dir_name }}/ldapuser /home - echo "{{ vast_mount_point }}/{{ openldap_dir_name }}/ldapuser /home none bind 0 0" >> /etc/fstab - chmod {{ file_mode }} /etc/fstab diff --git a/src/orchestrator/roles/provision_common/templates/metadata_svc/ms-group-service_kube_control_plane_first_x86_64.yaml.j2 b/src/orchestrator/roles/provision_common/templates/metadata_svc/ms-group-service_kube_control_plane_first_x86_64.yaml.j2 index e6e6ce745e..dcde98249d 100644 --- a/src/orchestrator/roles/provision_common/templates/metadata_svc/ms-group-service_kube_control_plane_first_x86_64.yaml.j2 +++ b/src/orchestrator/roles/provision_common/templates/metadata_svc/ms-group-service_kube_control_plane_first_x86_64.yaml.j2 @@ -406,12 +406,12 @@ {% if etcd_on_local_disk | default(false) %} - /usr/local/bin/etcd-fstab-update.sh {% else %} - - echo "{{ k8s_nfs_server_path }}/{{ "{{" }} ds.meta_data.instance_data.v1.local_ipv4 {{ "}}" }}/etcd /var/lib/etcd nfs noatime,nolock 0 0" >> /etc/fstab + - echo "{{ k8s_nfs_server_path }}/{{ "{{" }} ds.meta_data.instance_data.v1.local_ipv4 {{ "}}" }}/etcd /var/lib/etcd nfs {{ k8s_nfs_mnt_opts | default(k8s_nfs_default_mnt_opts) }} 0 0" >> /etc/fstab {% endif %} - - echo "{{ k8s_nfs_server_path }}/{{ "{{" }} ds.meta_data.instance_data.v1.local_ipv4 {{ "}}" }}/kubelet /var/lib/kubelet nfs noatime,nolock 0 0" >> /etc/fstab - - echo "{{ k8s_nfs_server_path }}/{{ "{{" }} ds.meta_data.instance_data.v1.local_ipv4 {{ "}}" }}/kubernetes /etc/kubernetes nfs noatime,nolock 0 0" >> /etc/fstab - - echo "{{ k8s_nfs_server_path }}/{{ "{{" }} ds.meta_data.instance_data.v1.local_ipv4 {{ "}}" }}/pod-logs /var/log/pods nfs noatime,nolock 0 0" >> /etc/fstab - - echo "{{ k8s_nfs_server_path }}/packages /var/lib/packages nfs noatime,nolock 0 0" >> /etc/fstab + - echo "{{ k8s_nfs_server_path }}/{{ "{{" }} ds.meta_data.instance_data.v1.local_ipv4 {{ "}}" }}/kubelet /var/lib/kubelet nfs {{ k8s_nfs_mnt_opts | default(k8s_nfs_default_mnt_opts) }} 0 0" >> /etc/fstab + - echo "{{ k8s_nfs_server_path }}/{{ "{{" }} ds.meta_data.instance_data.v1.local_ipv4 {{ "}}" }}/kubernetes /etc/kubernetes nfs {{ k8s_nfs_mnt_opts | default(k8s_nfs_default_mnt_opts) }} 0 0" >> /etc/fstab + - echo "{{ k8s_nfs_server_path }}/{{ "{{" }} ds.meta_data.instance_data.v1.local_ipv4 {{ "}}" }}/pod-logs /var/log/pods nfs {{ k8s_nfs_mnt_opts | default(k8s_nfs_default_mnt_opts) }} 0 0" >> /etc/fstab + - echo "{{ k8s_nfs_server_path }}/packages /var/lib/packages nfs {{ k8s_nfs_mnt_opts | default(k8s_nfs_default_mnt_opts) }} 0 0" >> /etc/fstab - echo "tmpfs /tmp/crio-storage tmpfs size={{ k8s_crio_storage_size }},noatime,nodev,nosuid 0 0" >> /etc/fstab - sudo swapoff -a - sudo sed -i '/ swap / s/^/#/' /etc/fstab @@ -896,6 +896,9 @@ --namespace default --create-namespace \ --set nfs.server={{ k8s_nfs_server_path.split(':')[0] }} \ --set nfs.path={{ k8s_nfs_server_path.split(':')[1] }} \ +{% if k8s_nfs_custom_mnt_opts | default('') | length > 0 %} + --set "nfs.mountOptions={{ '{' }}{{ k8s_nfs_custom_mnt_opts }}{{ '}' }}" \ +{% endif %} --set storageClass.defaultClass=true \ --set storageClass.reclaimPolicy=Retain echo "Waiting for nfs-subdir-external-provisioner pods to appear..." diff --git a/src/orchestrator/roles/provision_common/templates/metadata_svc/ms-group-service_kube_control_plane_x86_64.yaml.j2 b/src/orchestrator/roles/provision_common/templates/metadata_svc/ms-group-service_kube_control_plane_x86_64.yaml.j2 index 6e3c3eb13a..3dc9db97d7 100644 --- a/src/orchestrator/roles/provision_common/templates/metadata_svc/ms-group-service_kube_control_plane_x86_64.yaml.j2 +++ b/src/orchestrator/roles/provision_common/templates/metadata_svc/ms-group-service_kube_control_plane_x86_64.yaml.j2 @@ -302,12 +302,12 @@ {% if etcd_on_local_disk | default(false) %} - /usr/local/bin/etcd-fstab-update.sh {% else %} - - echo "{{ k8s_nfs_server_path }}/{{ "{{" }} ds.meta_data.instance_data.v1.local_ipv4 {{ "}}" }}/etcd /var/lib/etcd nfs noatime,nolock 0 0" >> /etc/fstab + - echo "{{ k8s_nfs_server_path }}/{{ "{{" }} ds.meta_data.instance_data.v1.local_ipv4 {{ "}}" }}/etcd /var/lib/etcd nfs {{ k8s_nfs_mnt_opts | default(k8s_nfs_default_mnt_opts) }} 0 0" >> /etc/fstab {% endif %} - - echo "{{ k8s_nfs_server_path }}/{{ "{{" }} ds.meta_data.instance_data.v1.local_ipv4 {{ "}}" }}/kubelet /var/lib/kubelet nfs noatime,nolock 0 0" >> /etc/fstab - - echo "{{ k8s_nfs_server_path }}/{{ "{{" }} ds.meta_data.instance_data.v1.local_ipv4 {{ "}}" }}/kubernetes /etc/kubernetes nfs noatime,nolock 0 0" >> /etc/fstab - - echo "{{ k8s_nfs_server_path }}/{{ "{{" }} ds.meta_data.instance_data.v1.local_ipv4 {{ "}}" }}/pod-logs /var/log/pods nfs noatime,nolock 0 0" >> /etc/fstab - - echo "{{ k8s_nfs_server_path }}/packages /var/lib/packages nfs noatime,nolock 0 0" >> /etc/fstab + - echo "{{ k8s_nfs_server_path }}/{{ "{{" }} ds.meta_data.instance_data.v1.local_ipv4 {{ "}}" }}/kubelet /var/lib/kubelet nfs {{ k8s_nfs_mnt_opts | default(k8s_nfs_default_mnt_opts) }} 0 0" >> /etc/fstab + - echo "{{ k8s_nfs_server_path }}/{{ "{{" }} ds.meta_data.instance_data.v1.local_ipv4 {{ "}}" }}/kubernetes /etc/kubernetes nfs {{ k8s_nfs_mnt_opts | default(k8s_nfs_default_mnt_opts) }} 0 0" >> /etc/fstab + - echo "{{ k8s_nfs_server_path }}/{{ "{{" }} ds.meta_data.instance_data.v1.local_ipv4 {{ "}}" }}/pod-logs /var/log/pods nfs {{ k8s_nfs_mnt_opts | default(k8s_nfs_default_mnt_opts) }} 0 0" >> /etc/fstab + - echo "{{ k8s_nfs_server_path }}/packages /var/lib/packages nfs {{ k8s_nfs_mnt_opts | default(k8s_nfs_default_mnt_opts) }} 0 0" >> /etc/fstab - echo "tmpfs /tmp/crio-storage tmpfs size={{ k8s_crio_storage_size }},noatime,nodev,nosuid 0 0" >> /etc/fstab - sudo swapoff -a - sudo sed -i '/ swap / s/^/#/' /etc/fstab diff --git a/src/orchestrator/roles/provision_common/templates/metadata_svc/ms-group-service_kube_node_x86_64.yaml.j2 b/src/orchestrator/roles/provision_common/templates/metadata_svc/ms-group-service_kube_node_x86_64.yaml.j2 index e90fa38dc5..4c0369e398 100644 --- a/src/orchestrator/roles/provision_common/templates/metadata_svc/ms-group-service_kube_node_x86_64.yaml.j2 +++ b/src/orchestrator/roles/provision_common/templates/metadata_svc/ms-group-service_kube_node_x86_64.yaml.j2 @@ -196,10 +196,10 @@ {% endfor %} # K8s NFS mount entries - mkdir -pv /tmp/crio-storage {{ k8s_client_mount_path }} /var/lib/kubelet /etc/kubernetes /var/log/pods /var/lib/packages - - echo "{{ k8s_nfs_server_path }}/{{ "{{" }} ds.meta_data.instance_data.v1.local_ipv4 {{ "}}" }}/kubelet /var/lib/kubelet nfs noatime,nolock 0 0" >> /etc/fstab - - echo "{{ k8s_nfs_server_path }}/{{ "{{" }} ds.meta_data.instance_data.v1.local_ipv4 {{ "}}" }}/kubernetes /etc/kubernetes nfs noatime,nolock 0 0" >> /etc/fstab - - echo "{{ k8s_nfs_server_path }}/{{ "{{" }} ds.meta_data.instance_data.v1.local_ipv4 {{ "}}" }}/pod-logs /var/log/pods nfs noatime,nolock 0 0" >> /etc/fstab - - echo "{{ k8s_nfs_server_path }}/packages /var/lib/packages nfs noatime,nolock 0 0" >> /etc/fstab + - echo "{{ k8s_nfs_server_path }}/{{ "{{" }} ds.meta_data.instance_data.v1.local_ipv4 {{ "}}" }}/kubelet /var/lib/kubelet nfs {{ k8s_nfs_mnt_opts | default(k8s_nfs_default_mnt_opts) }} 0 0" >> /etc/fstab + - echo "{{ k8s_nfs_server_path }}/{{ "{{" }} ds.meta_data.instance_data.v1.local_ipv4 {{ "}}" }}/kubernetes /etc/kubernetes nfs {{ k8s_nfs_mnt_opts | default(k8s_nfs_default_mnt_opts) }} 0 0" >> /etc/fstab + - echo "{{ k8s_nfs_server_path }}/{{ "{{" }} ds.meta_data.instance_data.v1.local_ipv4 {{ "}}" }}/pod-logs /var/log/pods nfs {{ k8s_nfs_mnt_opts | default(k8s_nfs_default_mnt_opts) }} 0 0" >> /etc/fstab + - echo "{{ k8s_nfs_server_path }}/packages /var/lib/packages nfs {{ k8s_nfs_mnt_opts | default(k8s_nfs_default_mnt_opts) }} 0 0" >> /etc/fstab - echo "tmpfs /tmp/crio-storage tmpfs size={{ k8s_crio_storage_size }},noatime,nodev,nosuid 0 0" >> /etc/fstab - sudo swapoff -a - sudo sed -i '/ swap / s/^/#/' /etc/fstab diff --git a/src/orchestrator/roles/provision_common/templates/metadata_svc/ms-group-slurm_control_node_x86_64.yaml.j2 b/src/orchestrator/roles/provision_common/templates/metadata_svc/ms-group-slurm_control_node_x86_64.yaml.j2 index 5513c0a3de..e66182315b 100644 --- a/src/orchestrator/roles/provision_common/templates/metadata_svc/ms-group-slurm_control_node_x86_64.yaml.j2 +++ b/src/orchestrator/roles/provision_common/templates/metadata_svc/ms-group-slurm_control_node_x86_64.yaml.j2 @@ -326,7 +326,7 @@ # DOCA prerequisites - moved early to ensure RDMA is ready before vendor_data mounts - mkdir -p {{ client_mount_path }}/slurm/ssh - mkdir -p {{ slurm_ctld_log_dir_effective }} {{ slurmdbd_log_dir_effective }} {{ slurm_ctld_pid_dir_effective }} {{ slurmdbd_pid_dir_effective }} {{ slurm_state_save_location_effective }} {% if slurm_sched_log_dir_effective %}{{ slurm_sched_log_dir_effective }} {% endif %}/etc/slurm {{ home_dir }} /etc/my.cnf.d /etc/munge /var/lib/mysql /var/log/mariadb /cert /var/log/track /var/lib/packages - - echo "{{ metadata_svc_nfs_path }}/cert /cert nfs defaults,_netdev 0 0" >> /etc/fstab + - echo "{{ metadata_svc_nfs_path }}/cert /cert nfs {{ slurm_nfs_mnt_opts | default(slurm_nfs_default_mnt_opts) }} 0 0" >> /etc/fstab - mount -av - cp /cert/pulp_webserver.crt /etc/pki/ca-trust/source/anchors && update-ca-trust - sed -i 's/^gpgcheck=1/gpgcheck=0/' /etc/dnf/dnf.conf @@ -351,19 +351,19 @@ - mkdir -pv {{ client_mount_path }}/slurm/ssh # Create directories for nfs and mount all - mkdir -p {{ slurm_ctld_log_dir_effective }} {{ slurmdbd_log_dir_effective }} {{ slurm_ctld_pid_dir_effective }} {{ slurmdbd_pid_dir_effective }} {{ slurm_state_save_location_effective }} {% if slurm_sched_log_dir_effective %}{{ slurm_sched_log_dir_effective }} {% endif %}/etc/slurm {{ home_dir }} /etc/my.cnf.d /etc/munge /var/lib/mysql /var/log/mariadb /cert /var/log/track /var/lib/packages - - echo "{{ metadata_svc_nfs_path }}/$(hostname -s)/etc/slurm /etc/slurm nfs defaults,_netdev 0 0" >> /etc/fstab - - echo "{{ metadata_svc_nfs_path }}/$(hostname -s)/etc/my.cnf.d /etc/my.cnf.d nfs defaults,_netdev 0 0" >> /etc/fstab - - echo "{{ metadata_svc_nfs_path }}/$(hostname -s)/var/log/mariadb /var/log/mariadb nfs defaults,_netdev 0 0" >> /etc/fstab - - echo "{{ metadata_svc_nfs_path }}/$(hostname -s)/var/log/slurm {{ slurm_ctld_log_dir_effective }} nfs defaults,_netdev 0 0" >> /etc/fstab + - echo "{{ metadata_svc_nfs_path }}/$(hostname -s)/etc/slurm /etc/slurm nfs {{ slurm_nfs_mnt_opts | default(slurm_nfs_default_mnt_opts) }} 0 0" >> /etc/fstab + - echo "{{ metadata_svc_nfs_path }}/$(hostname -s)/etc/my.cnf.d /etc/my.cnf.d nfs {{ slurm_nfs_mnt_opts | default(slurm_nfs_default_mnt_opts) }} 0 0" >> /etc/fstab + - echo "{{ metadata_svc_nfs_path }}/$(hostname -s)/var/log/mariadb /var/log/mariadb nfs {{ slurm_nfs_mnt_opts | default(slurm_nfs_default_mnt_opts) }} 0 0" >> /etc/fstab + - echo "{{ metadata_svc_nfs_path }}/$(hostname -s)/var/log/slurm {{ slurm_ctld_log_dir_effective }} nfs {{ slurm_nfs_mnt_opts | default(slurm_nfs_default_mnt_opts) }} 0 0" >> /etc/fstab {% if slurmdbd_log_dir_effective != slurm_ctld_log_dir_effective %} - - echo "{{ metadata_svc_nfs_path }}/$(hostname -s)/var/log/slurm {{ slurmdbd_log_dir_effective }} nfs defaults,_netdev 0 0" >> /etc/fstab + - echo "{{ metadata_svc_nfs_path }}/$(hostname -s)/var/log/slurm {{ slurmdbd_log_dir_effective }} nfs {{ slurm_nfs_mnt_opts | default(slurm_nfs_default_mnt_opts) }} 0 0" >> /etc/fstab {% endif %} - - if ! mountpoint -q /var/lib/mysql; then echo "{{ metadata_svc_nfs_path }}/$(hostname -s)/var/lib/mysql /var/lib/mysql nfs defaults,_netdev 0 0" >> /etc/fstab; fi - - if ! mountpoint -q {{ slurm_state_save_location_effective }}; then echo "{{ metadata_svc_nfs_path }}/$(hostname -s)/var/spool/slurmctld {{ slurm_state_save_location_effective }} nfs defaults,_netdev 0 0" >> /etc/fstab; fi - - echo "{{ metadata_svc_nfs_path }}/$(hostname -s)/etc/munge /etc/munge nfs defaults,_netdev 0 0" >> /etc/fstab - - echo "{{ trackfile_nfs_path }} /var/log/track nfs defaults,_netdev 0 0" >> /etc/fstab - - echo "{{ metadata_svc_nfs_path }}/packages /var/lib/packages nfs defaults,_netdev 0 0" >> /etc/fstab - - echo "{{ metadata_svc_nfs_path }}/ssh {{ client_mount_path }}/slurm/ssh nfs defaults,_netdev 0 0" >> /etc/fstab + - if ! mountpoint -q /var/lib/mysql; then echo "{{ metadata_svc_nfs_path }}/$(hostname -s)/var/lib/mysql /var/lib/mysql nfs {{ slurm_nfs_mnt_opts | default(slurm_nfs_default_mnt_opts) }} 0 0" >> /etc/fstab; fi + - if ! mountpoint -q {{ slurm_state_save_location_effective }}; then echo "{{ metadata_svc_nfs_path }}/$(hostname -s)/var/spool/slurmctld {{ slurm_state_save_location_effective }} nfs {{ slurm_nfs_mnt_opts | default(slurm_nfs_default_mnt_opts) }} 0 0" >> /etc/fstab; fi + - echo "{{ metadata_svc_nfs_path }}/$(hostname -s)/etc/munge /etc/munge nfs {{ slurm_nfs_mnt_opts | default(slurm_nfs_default_mnt_opts) }} 0 0" >> /etc/fstab + - echo "{{ trackfile_nfs_path }} /var/log/track nfs {{ slurm_nfs_mnt_opts | default(slurm_nfs_default_mnt_opts) }} 0 0" >> /etc/fstab + - echo "{{ metadata_svc_nfs_path }}/packages /var/lib/packages nfs {{ slurm_nfs_mnt_opts | default(slurm_nfs_default_mnt_opts) }} 0 0" >> /etc/fstab + - echo "{{ metadata_svc_nfs_path }}/ssh {{ client_mount_path }}/slurm/ssh nfs {{ slurm_nfs_mnt_opts | default(slurm_nfs_default_mnt_opts) }} 0 0" >> /etc/fstab - chmod {{ file_mode }} /etc/fstab - mount -av - chown -R {{ slurm_user }}:{{ slurm_user }} {{ home_dir }} @@ -392,9 +392,9 @@ {% if hostvars['localhost']['openldap_support'] %} - /usr/local/bin/update_ldap_conf.sh - mkdir /ldapcerts - - echo "{{ metadata_svc_nfs_path_openldap }}/certs /ldapcerts nfs defaults,_netdev 0 0" >> /etc/fstab + - echo "{{ metadata_svc_nfs_path_openldap }}/certs /ldapcerts nfs {{ slurm_nfs_mnt_opts | default(slurm_nfs_default_mnt_opts) }} 0 0" >> /etc/fstab # TODO: This mount may be redundant, remove this - - echo "{{ metadata_svc_nfs_path_openldap }}/ldapuser /home nfs defaults,_netdev 0 0" >> /etc/fstab + - echo "{{ metadata_svc_nfs_path_openldap }}/ldapuser /home nfs {{ slurm_nfs_mnt_opts | default(slurm_nfs_default_mnt_opts) }} 0 0" >> /etc/fstab - chmod {{ file_mode }} /etc/fstab - mount -a - yes | cp /ldapcerts/* /etc/openldap/certs diff --git a/src/orchestrator/roles/provision_common/templates/metadata_svc/ms-group-slurm_node_aarch64.yaml.j2 b/src/orchestrator/roles/provision_common/templates/metadata_svc/ms-group-slurm_node_aarch64.yaml.j2 index 690423e04f..df51db1624 100644 --- a/src/orchestrator/roles/provision_common/templates/metadata_svc/ms-group-slurm_node_aarch64.yaml.j2 +++ b/src/orchestrator/roles/provision_common/templates/metadata_svc/ms-group-slurm_node_aarch64.yaml.j2 @@ -166,13 +166,13 @@ mkdir -pv {{ slurm_slurmd_log_dir_effective }} {{ slurm_slurmd_pid_dir_effective }} {{ slurm_slurmd_spool_dir_effective }} {{ slurm_epilog_dirs_all | join(' ') }} /etc/munge /cert /var/log/track /var/lib/packages echo "[INFO] Updating /etc/fstab with NFS entries for Pulp cert, Slurm and Munge paths" - echo "{{ metadata_svc_nfs_path }}/$(hostname -s)/var/log/slurm {{ slurm_slurmd_log_dir_effective }} nfs defaults,_netdev 0 0" >> /etc/fstab - echo "{{ metadata_svc_nfs_path }}/$(hostname -s)/var/spool/slurmd {{ slurm_slurmd_spool_dir_effective }} nfs defaults,_netdev 0 0" >> /etc/fstab - echo "{{ metadata_svc_nfs_path }}/$(hostname -s)/etc/slurm/epilog.d /etc/slurm/epilog.d nfs defaults,_netdev 0 0" >> /etc/fstab - echo "{{ metadata_svc_nfs_path }}/$(hostname -s)/etc/munge /etc/munge nfs defaults,_netdev 0 0" >> /etc/fstab - echo "{{ trackfile_nfs_path }} /var/log/track nfs defaults,_netdev 0 0" >> /etc/fstab - echo "{{ metadata_svc_nfs_path }}/cert /cert nfs defaults,_netdev 0 0" >> /etc/fstab - echo "{{ metadata_svc_nfs_path }}/packages /var/lib/packages nfs defaults,_netdev 0 0" >> /etc/fstab + echo "{{ metadata_svc_nfs_path }}/$(hostname -s)/var/log/slurm {{ slurm_slurmd_log_dir_effective }} nfs {{ slurm_nfs_mnt_opts | default(slurm_nfs_default_mnt_opts) }} 0 0" >> /etc/fstab + echo "{{ metadata_svc_nfs_path }}/$(hostname -s)/var/spool/slurmd {{ slurm_slurmd_spool_dir_effective }} nfs {{ slurm_nfs_mnt_opts | default(slurm_nfs_default_mnt_opts) }} 0 0" >> /etc/fstab + echo "{{ metadata_svc_nfs_path }}/$(hostname -s)/etc/slurm/epilog.d /etc/slurm/epilog.d nfs {{ slurm_nfs_mnt_opts | default(slurm_nfs_default_mnt_opts) }} 0 0" >> /etc/fstab + echo "{{ metadata_svc_nfs_path }}/$(hostname -s)/etc/munge /etc/munge nfs {{ slurm_nfs_mnt_opts | default(slurm_nfs_default_mnt_opts) }} 0 0" >> /etc/fstab + echo "{{ trackfile_nfs_path }} /var/log/track nfs {{ slurm_nfs_mnt_opts | default(slurm_nfs_default_mnt_opts) }} 0 0" >> /etc/fstab + echo "{{ metadata_svc_nfs_path }}/cert /cert nfs {{ slurm_nfs_mnt_opts | default(slurm_nfs_default_mnt_opts) }} 0 0" >> /etc/fstab + echo "{{ metadata_svc_nfs_path }}/packages /var/lib/packages nfs {{ slurm_nfs_mnt_opts | default(slurm_nfs_default_mnt_opts) }} 0 0" >> /etc/fstab chmod {{ file_mode }} /etc/fstab @@ -268,7 +268,7 @@ echo "[INFO] Mounting controller slurm.conf from NFS: {{ metadata_svc_nfs_path }}/{{ ctld_list[0] }}/etc/slurm -> $CTLD_SLURM_DIR_MNT" mkdir -p "$CTLD_SLURM_DIR_MNT" - mount -t nfs "{{ metadata_svc_nfs_path }}/{{ ctld_list[0] }}/etc/slurm" "$CTLD_SLURM_DIR_MNT" || { + mount -t nfs -o "{{ slurm_nfs_mnt_opts | default(slurm_nfs_default_mnt_opts) }}" "{{ metadata_svc_nfs_path }}/{{ ctld_list[0] }}/etc/slurm" "$CTLD_SLURM_DIR_MNT" || { echo "[WARN] Failed to mount controller slurm.conf directory, falling back to defaults." SRUN_RANGE="$DEFAULT_SRUN_RANGE" SLURMD_PORT="$DEFAULT_SLURMD_PORT" @@ -480,7 +480,7 @@ {% endfor %} {% endif %} - mkdir -p {{ client_mount_path }}/slurm/ssh - - echo "{{ metadata_svc_nfs_path }}/ssh {{ client_mount_path }}/slurm/ssh nfs defaults,_netdev 0 0" >> /etc/fstab + - echo "{{ metadata_svc_nfs_path }}/ssh {{ client_mount_path }}/slurm/ssh nfs {{ slurm_nfs_mnt_opts | default(slurm_nfs_default_mnt_opts) }} 0 0" >> /etc/fstab {# mount_config host_mount_map block removed - incompatible with Pongo2 validation #} {# When mount_config bolt-on is needed, implement as a shell script in write_files #} {% set fg_swap = metadata_svc_groups_dict.get(functional_group_name, {}).get('swap', {}) %} @@ -518,7 +518,7 @@ {% if hostvars['localhost']['openldap_support'] %} - /usr/local/bin/update_ldap_conf.sh - mkdir /ldapcerts - - echo "{{ metadata_svc_nfs_path_openldap }}/certs /ldapcerts nfs defaults,_netdev 0 0" >> /etc/fstab + - echo "{{ metadata_svc_nfs_path_openldap }}/certs /ldapcerts nfs {{ slurm_nfs_mnt_opts | default(slurm_nfs_default_mnt_opts) }} 0 0" >> /etc/fstab - mkdir -pv {{ vast_mount_point }}/{{ openldap_dir_name }}/ldapuser /home - echo "{{ vast_mount_point }}/{{ openldap_dir_name }}/ldapuser /home none bind 0 0" >> /etc/fstab - chmod {{ file_mode }} /etc/fstab diff --git a/src/orchestrator/roles/provision_common/templates/metadata_svc/ms-group-slurm_node_x86_64.yaml.j2 b/src/orchestrator/roles/provision_common/templates/metadata_svc/ms-group-slurm_node_x86_64.yaml.j2 index 82b72ed928..d172947a75 100644 --- a/src/orchestrator/roles/provision_common/templates/metadata_svc/ms-group-slurm_node_x86_64.yaml.j2 +++ b/src/orchestrator/roles/provision_common/templates/metadata_svc/ms-group-slurm_node_x86_64.yaml.j2 @@ -199,13 +199,13 @@ mkdir -pv {{ slurm_slurmd_log_dir_effective }} {{ slurm_slurmd_pid_dir_effective }} {{ slurm_slurmd_spool_dir_effective }} {{ slurm_epilog_dirs_all | join(' ') }} /etc/munge /cert /var/log/track /var/lib/packages echo "[INFO] Updating /etc/fstab with NFS entries for Pulp cert, Slurm and Munge paths" - echo "{{ metadata_svc_nfs_path }}/cert /cert nfs defaults,_netdev 0 0" >> /etc/fstab - echo "{{ metadata_svc_nfs_path }}/$(hostname -s)/var/log/slurm {{ slurm_slurmd_log_dir_effective }} nfs defaults,_netdev 0 0" >> /etc/fstab - echo "{{ metadata_svc_nfs_path }}/$(hostname -s)/var/spool/slurmd {{ slurm_slurmd_spool_dir_effective }} nfs defaults,_netdev 0 0" >> /etc/fstab - echo "{{ metadata_svc_nfs_path }}/$(hostname -s)/etc/slurm/epilog.d /etc/slurm/epilog.d nfs defaults,_netdev 0 0" >> /etc/fstab - echo "{{ metadata_svc_nfs_path }}/$(hostname -s)/etc/munge /etc/munge nfs defaults,_netdev 0 0" >> /etc/fstab - echo "{{ trackfile_nfs_path }} /var/log/track nfs defaults,_netdev 0 0" >> /etc/fstab - echo "{{ metadata_svc_nfs_path }}/packages /var/lib/packages nfs defaults,_netdev 0 0" >> /etc/fstab + echo "{{ metadata_svc_nfs_path }}/cert /cert nfs {{ slurm_nfs_mnt_opts | default(slurm_nfs_default_mnt_opts) }} 0 0" >> /etc/fstab + echo "{{ metadata_svc_nfs_path }}/$(hostname -s)/var/log/slurm {{ slurm_slurmd_log_dir_effective }} nfs {{ slurm_nfs_mnt_opts | default(slurm_nfs_default_mnt_opts) }} 0 0" >> /etc/fstab + echo "{{ metadata_svc_nfs_path }}/$(hostname -s)/var/spool/slurmd {{ slurm_slurmd_spool_dir_effective }} nfs {{ slurm_nfs_mnt_opts | default(slurm_nfs_default_mnt_opts) }} 0 0" >> /etc/fstab + echo "{{ metadata_svc_nfs_path }}/$(hostname -s)/etc/slurm/epilog.d /etc/slurm/epilog.d nfs {{ slurm_nfs_mnt_opts | default(slurm_nfs_default_mnt_opts) }} 0 0" >> /etc/fstab + echo "{{ metadata_svc_nfs_path }}/$(hostname -s)/etc/munge /etc/munge nfs {{ slurm_nfs_mnt_opts | default(slurm_nfs_default_mnt_opts) }} 0 0" >> /etc/fstab + echo "{{ trackfile_nfs_path }} /var/log/track nfs {{ slurm_nfs_mnt_opts | default(slurm_nfs_default_mnt_opts) }} 0 0" >> /etc/fstab + echo "{{ metadata_svc_nfs_path }}/packages /var/lib/packages nfs {{ slurm_nfs_mnt_opts | default(slurm_nfs_default_mnt_opts) }} 0 0" >> /etc/fstab chmod {{ file_mode }} /etc/fstab echo "[INFO] Mounting all NFS entries from /etc/fstab" @@ -304,7 +304,7 @@ echo "[INFO] Mounting controller slurm.conf from NFS: {{ metadata_svc_nfs_path }}/{{ ctld_list[0] }}/etc/slurm -> $CTLD_SLURM_DIR_MNT" mkdir -p "$CTLD_SLURM_DIR_MNT" - mount -t nfs "{{ metadata_svc_nfs_path }}/{{ ctld_list[0] }}/etc/slurm" "$CTLD_SLURM_DIR_MNT" || { + mount -t nfs -o "{{ slurm_nfs_mnt_opts | default(slurm_nfs_default_mnt_opts) }}" "{{ metadata_svc_nfs_path }}/{{ ctld_list[0] }}/etc/slurm" "$CTLD_SLURM_DIR_MNT" || { echo "[WARN] Failed to mount controller slurm.conf directory, falling back to defaults." SRUN_RANGE="$DEFAULT_SRUN_RANGE" SLURMD_PORT="$DEFAULT_SLURMD_PORT" @@ -488,7 +488,7 @@ {% endfor %} {% endif %} - mkdir -p {{ client_mount_path }}/slurm/ssh - - echo "{{ metadata_svc_nfs_path }}/ssh {{ client_mount_path }}/slurm/ssh nfs defaults,_netdev 0 0" >> /etc/fstab + - echo "{{ metadata_svc_nfs_path }}/ssh {{ client_mount_path }}/slurm/ssh nfs {{ slurm_nfs_mnt_opts | default(slurm_nfs_default_mnt_opts) }} 0 0" >> /etc/fstab {# mount_config host_mount_map block removed - incompatible with Pongo2 validation #} {# When mount_config bolt-on is needed, implement as a shell script in write_files #} {% set fg_swap = metadata_svc_groups_dict.get(functional_group_name, {}).get('swap', {}) %} @@ -524,7 +524,7 @@ {% if hostvars['localhost']['openldap_support'] %} - /usr/local/bin/update_ldap_conf.sh - mkdir -p /ldapcerts || echo "LDAP cert directory already exists or creation failed (non-critical)" - - echo "{{ metadata_svc_nfs_path_openldap }}/certs /ldapcerts nfs defaults,_netdev 0 0" >> /etc/fstab + - echo "{{ metadata_svc_nfs_path_openldap }}/certs /ldapcerts nfs {{ slurm_nfs_mnt_opts | default(slurm_nfs_default_mnt_opts) }} 0 0" >> /etc/fstab - mkdir -pv {{ vast_mount_point }}/{{ openldap_dir_name }}/ldapuser /home - echo "{{ vast_mount_point }}/{{ openldap_dir_name }}/ldapuser /home none bind 0 0" >> /etc/fstab - chmod {{ file_mode }} /etc/fstab diff --git a/src/orchestrator/roles/provision_common/vars/main.yml b/src/orchestrator/roles/provision_common/vars/main.yml index 3b637a2112..05f064d4cf 100644 --- a/src/orchestrator/roles/provision_common/vars/main.yml +++ b/src/orchestrator/roles/provision_common/vars/main.yml @@ -36,6 +36,11 @@ openchami_hostname_vars_path: "{{ nodes_dir }}/hostname.yaml" ms_defaults_dest: "{{ metadata_svc_dir }}/ms-defaults.yaml" ms_group_common_dest: "{{ metadata_svc_dir }}/ms-group-common.yaml" +# Default options for Omnia-managed per-node NFS sub-mounts. +# Overridden per storage entry by node_nfs_mnt_opts in storage_config.yml. +k8s_nfs_default_mnt_opts: "noatime,nolock" +slurm_nfs_default_mnt_opts: "defaults,_netdev" + # SSH keys ssh_key_path: "{{ ansible_user_dir | default('/root') }}/.ssh/oim_rsa.pub" ssh_private_key_path: "{{ ansible_user_dir | default('/root') }}/.ssh/oim_rsa"