Exclude Filestore module from integration test

tools/cloud-build/daily-tests/ansible_playbooks/base-integration-test.yml

@@ -40,6 +40,12 @@
           awk -F'"' '{print $2}'
 
   ## Create cluster
+  - name: Run Pre Create Tasks
+    ansible.builtin.include_tasks: "{{ pre_create_task }}"
+    loop: "{{ pre_create_tasks | default([]) }}"
+    loop_control:
+      loop_var: pre_create_task
+
   - name: Create Deployment Directory
     ansible.builtin.include_tasks:
       file: tasks/create_deployment_directory.yml

tools/cloud-build/daily-tests/ansible_playbooks/pre-create-tasks/remove-modules.yml

@@ -0,0 +1,26 @@
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+---
+- name: Assert variables are defined
+  ansible.builtin.assert:
+    that:
+    - workspace is defined
+    - blueprint_yaml is defined
+    - custom_vars.modules_to_remove is defined
+
+- name: Remove modules
+  ansible.builtin.command: "{{ workspace }}/tools/cloud-build/daily-tests/remove_module.py --blueprint {{ blueprint_yaml }} --module {{ module_to_remove }}"
+  loop: "{{ custom_vars.modules_to_remove }}"
+  loop_control:
+    loop_var: module_to_remove

tools/cloud-build/daily-tests/ansible_playbooks/slurm-integration-test.yml

@@ -40,6 +40,12 @@
           awk -F'"' '{print $2}'
 
   ## Create cluster
+  - name: Run Pre Create Tasks
+    ansible.builtin.include_tasks: "{{ pre_create_task }}"
+    loop: "{{ pre_create_tasks | default([]) }}"
+    loop_control:
+      loop_var: pre_create_task
+
   - name: Create Deployment Directory
     ansible.builtin.include_tasks:
       file: tasks/create_deployment_directory.yml
@@ -53,7 +59,7 @@
       loop_control:
         loop_var: pre_deploy_task
 
-    - name: Create Cluster with GHPC
+    - name: Deploy Cluster with GHPC
       register: deployment
       changed_when: deployment.changed
       ansible.builtin.command: ./ghpc deploy {{ deployment_name }} --auto-approve

tools/cloud-build/daily-tests/remove_module.py

@@ -0,0 +1,52 @@
+#!/usr/bin/env python3
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import yaml # pip install pyyaml
+
+def delete_module(bp, mod_id: str):
+  for g in bp["deployment_groups"]:
+    for im, m in enumerate(g["modules"]):
+      if m["id"] == mod_id:
+        del g["modules"][im]
+        return
+  raise RuntimeError(f"Module {mod_id} not found")
+
+def update_bp(bp, mod_id: str):
+  delete_module(bp, mod_id)
+   # remove module from "use"
+  for g in bp["deployment_groups"]:
+    for m in g["modules"]:
+      if mod_id in m.get("use", []):
+        m["use"].remove(mod_id)
+  # TODO:
+  # - move all settings of removed module into some "/dev/null" to prevent
+  #   "variable not used" validation errors
+  # - handle references to removed module (e.g. raise error if referenced)
+
+def main():
+  import argparse
+  parser = argparse.ArgumentParser()
+  parser.add_argument("--blueprint", required=True, type=str)
+  parser.add_argument("--module", required=True, type=str, help="Module ID to remove")
+  args = parser.parse_args()
+
+  with open(args.blueprint) as yf:
+    bp = yaml.safe_load(yf)
+  update_bp(bp, mod_id=args.module)
+  with open(args.blueprint, "w") as yf:
+    yaml.dump(bp, yf)
+
+if __name__ == "__main__":
+  main()

tools/cloud-build/daily-tests/tests/hpc-slurm-chromedesktop.yml

@@ -21,21 +21,22 @@ deployment_name: "slm-crd-{{ build }}"
 slurm_cluster_name: "slmcrd{{ build[0:4] }}"
 zone: europe-west1-d
 cli_deployment_vars:
-  network_name: "{{ network }}"
   region: europe-west1
   zone: "{{ zone }}"
 workspace: /workspace
 blueprint_yaml: "{{ workspace }}/community/examples/hpc-slurm-chromedesktop-v5-legacy.yaml"
-network: "{{ test_name }}-net"
+network: "{{ deployment_name }}-net"
 # Note: Pattern matching in gcloud only supports 1 wildcard.
 login_node: "{{ slurm_cluster_name }}-login-*"
 controller_node: "{{ slurm_cluster_name }}-controller"
+pre_create_tasks:
+- pre-create-tasks/remove-modules.yml
 post_deploy_tests:
 - test-validation/test-mounts.yml
 - test-validation/test-crd.yml
 custom_vars:
-  mounts:
-  - /home
+  modules_to_remove:
+  - homefs
   partitions:
   - desktop
   - compute

tools/cloud-build/daily-tests/tests/slurm-v6-rocky8.yml

@@ -21,23 +21,24 @@ deployment_name: "rock-8-{{ build }}"
 slurm_cluster_name: "rock8{{ build[0:5] }}"
 
 cli_deployment_vars:
-  network_name: "{{ network }}"
   region: us-west4
   zone: us-west4-c
 
 zone: us-west4-c
 workspace: /workspace
 blueprint_yaml: "{{ workspace }}/examples/hpc-slurm.yaml"
-network: "{{ test_name }}-net"
+network: "{{ deployment_name }}-net"
 # Note: Pattern matching in gcloud only supports 1 wildcard, a*-login-* won't work.
 login_node: "{{ slurm_cluster_name }}-slurm-login-*"
 controller_node: "{{ slurm_cluster_name }}-controller"
+pre_create_tasks:
+- pre-create-tasks/remove-modules.yml
 post_deploy_tests:
 - test-validation/test-mounts.yml
 - test-validation/test-partitions.yml
 custom_vars:
+  modules_to_remove:
+  - homefs
   partitions:
   - compute
   - debug
-  mounts:
-  - /home