From 1ac624a596e0ce9116c75de0ff3e5e175375c26e Mon Sep 17 00:00:00 2001 From: Nagachandan-P Date: Thu, 12 Nov 2026 04:23:52 -0600 Subject: [PATCH 1/3] svt for catalog build theme Signed-off-by: Nagachandan-P --- test/system_test/README.md | 37 ++ test/system_test/_setup.py | 17 +- test/system_test/catalog_build_design.md | 123 ++++ .../system_test/catalog_build_test_config.yml | 128 +++++ .../profiles/catalog_build.yml | 48 ++ test/system_test/docs/CATALOG_BUILD.md | 537 ++++++++++++++++++ .../library/functions/catalog_func.py | 391 +++++++++++++ .../library/functions/catalog_settings.py | 165 ++++++ .../library/functions/config_func.py | 13 +- .../system_test/library/functions/fvt_func.py | 7 +- .../library/functions/readiness_func.py | 19 +- .../library/functions/setup_func.py | 8 +- .../library/functions/test_func.py | 27 +- test/system_test/library/messages/svt_msgs.py | 26 + test/system_test/library/vars/catalog_vars.py | 164 ++++++ .../library/vars/test_case_vars.py | 7 + test/system_test/library/vars/theme_vars.py | 36 ++ test/system_test/ut/test_catalog_build.py | 478 ++++++++++++++++ 18 files changed, 2216 insertions(+), 15 deletions(-) create mode 100644 test/system_test/catalog_build_design.md create mode 100644 test/system_test/catalog_build_test_config.yml create mode 100644 test/system_test/dataset_generator/profiles/catalog_build.yml create mode 100644 test/system_test/docs/CATALOG_BUILD.md create mode 100644 test/system_test/library/functions/catalog_func.py create mode 100644 test/system_test/library/functions/catalog_settings.py create mode 100644 test/system_test/library/vars/catalog_vars.py create mode 100644 test/system_test/ut/test_catalog_build.py diff --git a/test/system_test/README.md b/test/system_test/README.md index 2224e8a28e..2c2e3bcc48 100644 --- a/test/system_test/README.md +++ b/test/system_test/README.md @@ -83,12 +83,49 @@ tests of the stage are skipped (they need what it set up). The tests are data in [`library/vars/test_case_vars.py`](library/vars/test_case_vars.py); `svt/test_svt.py` generates one pytest test per entry. +## Theme catalog_build + +For every catalog, one at a time: `./omnia.sh --select-catalog `, then the +repo_manager and image_build_manager FVT steps (no orchestrator deploy, no PXE boot). +User guide: [`docs/CATALOG_BUILD.md`](docs/CATALOG_BUILD.md). Settings: +[`catalog_build_test_config.yml`](catalog_build_test_config.yml); design: +[`catalog_build_design.md`](catalog_build_design.md). + +```bash +./setup_env.sh --theme catalog_build # configure (freezes the catalog plan) -> install -> creds +./setup_env.sh --theme catalog_build --force # cleanup first (deletes OMNIA_DATA_PATH) +./run_validation.sh svt_catalog_build list # the catalog stages and their tests +./run_validation.sh svt_catalog_build all # every catalog stage, then summary +./run_validation.sh svt_catalog_build all --start-stage catalog_02 +./run_validation.sh svt_catalog_build all --retry +``` + +`--configure` resolves `catalog_build.catalogs` (`all` = the `./omnia.sh +--list-catalogs` order) into the plan of the run, `reports/catalog_build_plan.json`. +The theme's stages (`catalog_01`, `catalog_02` ...) and tests are generated from it +([`library/vars/catalog_vars.py`](library/vars/catalog_vars.py)) and run like the +build_stream tests. IDs are `SVT_CATALOG_BUILD_`. + +| Test of a catalog stage | What it does | Step | +|---|---|---| +| `NN00` | Put back the shared runtime inputs, apply `catalog_overrides///`, write `repo_config`, `caching_policy`, `repo_policies`, `force_rebuild` and `functional_groups_source: catalog` into `//input//`, then `./omnia.sh --select-catalog` | `svt select ` | +| `NN01`... | Every `steps` entry `[domain, tag]` of the theme block, as two tests | `fvt_ exec`, `fvt_ verify` | +| last | `cleanup_between` (`images`, `repos`, `full`); runs even after a failure | `fvt_ cleanup... test` | + +A catalog with an aarch64 layer is one skipped test (`NN00`, with the reason) unless +`include_aarch64: true` (needs `aarch64_inventory_host_ip` in user_inputs.yml). With +`continue_on_failure: true` every catalog stage is a soft stage: a failed catalog does +not stop the next one; the summary still fails. After a failed `exec` the orphaned +`ansible-playbook .yml` workers (and a running `dnf download`) are stopped, so +a playbook killed at the FVT timeout cannot overlap the next test. + ## Documentation | Doc | Contents | |---|---| | [docs/SETUP.md](docs/SETUP.md) | `./setup_env.sh` (configure, cleanup incl. NFS shares, install, credentials, status), `user_inputs.yml`, the dataset generator | | [docs/TEST_CONFIG.md](docs/TEST_CONFIG.md) | Every setting of `build_stream_test_config.yml`: catalogs, the cleanup flags per suite (telemetry, NFS shares ...), the report | +| [docs/CATALOG_BUILD.md](docs/CATALOG_BUILD.md) | **User guide of the `catalog_build` theme**: prerequisites, the two files to edit, step by step, every setting, reading the report, failures, troubleshooting | | [docs/RUNNER.md](docs/RUNNER.md) | `./run_validation.sh`, the run and its guard, retry, the SVT report, adding a theme, stage or test | | [docs/OPENBAO.md](docs/OPENBAO.md) | Passwords from OpenBao: one place for its settings, the pipeline and manual ways to read them | | [docs/PIPELINE.md](docs/PIPELINE.md) | Run the SVT from GitLab: how it works, where each secret goes, setup, **updating an existing pipeline**, troubleshooting | diff --git a/test/system_test/_setup.py b/test/system_test/_setup.py index 1c7d959860..0ee9de2439 100644 --- a/test/system_test/_setup.py +++ b/test/system_test/_setup.py @@ -49,6 +49,7 @@ from typing import Any, Callable, Dict, List, Optional from library.functions import ( + catalog_func, cleanup_func, readiness_func, run_func, @@ -172,9 +173,23 @@ def _mark_run(name: str, outcome: Dict[str, Any]) -> None: _emit(message) +def _configure(config: Dict[str, Any]) -> Dict[str, Any]: + """Configure; a catalog theme then freezes its catalog plan for the new run. + + The plan (catalog_func.write_plan) is what the theme's stages and tests are + generated from; when it cannot be built, configure fails and says why. + """ + outcome = setup_func.configure(config, _emit) + if outcome["status"] != "passed" or not THEMES[config["theme"]].get("catalog_plan"): + return outcome + rows, error = catalog_func.write_plan(config["catalog"], outcome["run_id"], _emit) + outcome["rows"] = [*(outcome.get("rows") or []), *rows] + return {**outcome, "status": "failed", "error": error} if error else outcome + + def _steps(args: argparse.Namespace, config: Dict[str, Any]) -> List[tuple]: """Return the (name, step) list of the requested action.""" - configure = ("configure", lambda: setup_func.configure(config, _emit)) + configure = ("configure", lambda: _configure(config)) cleanup = ("cleanup", lambda: cleanup_func.cleanup(config, _emit, args.yes)) install = ("install", lambda: setup_func.install(config, _emit, args.force, args.yes)) if args.action == "all": diff --git a/test/system_test/catalog_build_design.md b/test/system_test/catalog_build_design.md new file mode 100644 index 0000000000..a414fdb0ea --- /dev/null +++ b/test/system_test/catalog_build_design.md @@ -0,0 +1,123 @@ +# SVT Theme `catalog_build` — Design + +Status: **Implemented** on the merged SVT framework (PR #5711) +Runner category: `svt_catalog_build` + +## 1. Goal + +A manual run on one OIM (no GitLab needed). For **every catalog**, one at a time: + +``` +select catalog -> repo_manager FVTs (exec + verify) -> image_build_manager FVTs (exec + verify) +``` + +Every step is the domain's own FVT run (`test//_run.py fvt_ +exec|verify`); the theme adds no test logic. Scope ends after the image build: no +orchestrator deploy, no PXE boot. + +```bash +./setup_env.sh --theme catalog_build # configure (plan) -> install -> creds +./run_validation.sh svt_catalog_build list # catalog stages and their tests +./run_validation.sh svt_catalog_build all # every catalog stage, then summary +./run_validation.sh svt_catalog_build all --start-stage catalog_02 +./run_validation.sh svt_catalog_build all --retry +``` + +## 2. Aligned with the build_stream theme + +A theme of the merged framework is **data**: its stages and `stage_info` in +`theme_vars.THEMES`, its tests in `test_case_vars.TEST_CASES` (ID, title, order, +stage, FVT step), run by the generic `svt/test_svt.py` through +`stage_func.run_tests` (pytest `-m " and "`). catalog_build is the +same; only its data is generated: + +| Part | build_stream | catalog_build | +|---|---|---| +| Theme entry (`theme_vars.py`) | fixed stages | stages, `soft_stages`, `stage_info` from the catalog plan | +| Tests (`test_case_vars.py`) | written by hand | generated from the plan (`catalog_vars.catalog_test_cases`) | +| Test runner | `svt/test_svt.py` -> `run_svt_test` | the same | +| Report (JSON/HTML) | `svt_report_func` / `svt_report_html` | the same (stage titles name the catalog) | +| Failure handling | soft stages, skip the rest of a stage, `--retry`, `--start-stage` | the same (`continue_on_failure` -> soft stages) | +| Setup (`setup_env.sh`) | configure, cleanup, install, creds | the same; configure also writes the plan | +| Cleanup steps | build_stream, telemetry, orchestrator, image, repo, main | the same steps and flags | +| Credential helper | `test/build_stream/setup_env.sh` | the same (`creds_suite: build_stream`), repo_manager and image_build_manager only | +| Input profile | 9 input files | `omnia.env`, `repo_manager_config.yml`, `image_build_config.yml` | + +## 3. The catalog plan + +The catalog list lives in `catalog_build_test_config.yml`, but the stages and tests +must exist when the vars modules are imported (`domain_vars`, pytest markers, the +pipeline generator). So `./setup_env.sh --configure` (the `_configure` step of +`_setup.py`) resolves the list and freezes it: + +1. `catalogs: all` = every `./omnia.sh --list-catalogs` entry in that order; a list + keeps its own order (names or numbers, `setup_func.resolve_catalog`). +2. A catalog with an aarch64 functional layer is marked skipped unless + `include_aarch64: true`; `true` needs `aarch64_inventory_host_ip` in + user_inputs.yml, else configure fails. +3. The plan is written to `reports/catalog_build_plan.json` with the run id and the + normalized settings, and printed. + +`catalog_vars.load_plan()` reads it at import time. A changed +`catalog_build_test_config.yml` already needs a new `--configure` (the run guard), +so the plan always belongs to the current run; `initialization` checks it +(`Catalog plan of this run`). Without a plan the theme has no catalog stage. + +## 4. Tests of a catalog stage + +IDs `SVT_CATALOG_BUILD_`, order `2000 + NN*100 + step`: + +| Test | What it does | Step | +|---|---|---| +| `NN00 select` | SVT action (`catalog_func.run_action`): put back the shared runtime inputs, apply `catalog_overrides///`, write the global settings into `//input//`, then `./omnia.sh --select-catalog` | `svt select ` | +| `NN00 skip` | Instead of all the others, for a skipped catalog: one skipped test with the reason | `svt skip ` | +| `NN01`... | For every `steps` entry `[domain, tag]`: ` exec` (the playbook), then ` verify` | `fvt_ exec` / `verify` | +| last | `cleanup_between` steps (`always`: they run even after a failure) | `fvt_ cleanup... test` | + +Global settings written before every catalog: `repo_config`, `caching_policy`, +`repo_policies` (`"//": {policy, caching}`) into +`repo_manager_config.yml`; `build_image.force_rebuild` (default **true**) and +`functional_groups_source: catalog` into `image_build_config.yml`. + +The first test of a stage is the repo_manager precheck after select, so the existing +subscription-aware repository validation runs for every catalog; the theme adds no +repository check of its own. The FVT playbook timeout stays the omnia_auto default +(7200 s). + +## 5. Hooks in shared code (build_stream unchanged) + +| File | Change | +|---|---| +| `library/functions/test_func.py` | Optional test-case fields: `action` (an SVT action instead of an FVT step), `always` (runs after an earlier failure of the stage), `stop_leftovers` (after a failed exec, stop the orphaned `ansible-playbook .yml` workers and their children, which run in their own session and survive the FVT timeout). build_stream's tests carry none of them. | +| `library/functions/fvt_func.py` | `describe_step` labels an SVT action `svt ...` | +| `library/functions/config_func.py` | A theme with `catalog_plan` loads its block with `catalog_settings.py` | +| `library/functions/readiness_func.py` | `Catalog plan of this run` check for catalog themes | +| `library/functions/setup_func.py` | `bundled_catalogs` and `select_catalog` made public (reused by the plan and the select action) | +| `_setup.py` | `_configure`: configure, then the plan for catalog themes | +| `library/vars/theme_vars.py`, `test_case_vars.py` | The catalog_build entry and its generated tests | + +New files: `library/vars/catalog_vars.py`, `library/functions/catalog_settings.py`, +`library/functions/catalog_func.py`, `catalog_build_test_config.yml`, +`dataset_generator/profiles/catalog_build.yml`, `ut/test_catalog_build.py`. + +## 6. Settings (`catalog_build_test_config.yml`) + +| Key | Default | Effect | +|---|---|---| +| `catalogs` | `all` | `all` or an ordered list of `--list-catalogs` names / numbers | +| `include_aarch64` | `false` | `false` skips catalogs with an aarch64 layer | +| `repo_config` | `partial` | `always` / `partial` | +| `caching_policy` | `true` | repo_manager caching | +| `repo_policies` | `{}` | per-repository policy (`always` / `partial` / `never`) and caching | +| `force_rebuild` | `true` | `true` rebuilds every image; `false` reuses a matching image | +| `continue_on_failure` | `true` | every catalog stage is soft | +| `cleanup_between` | `none` | `none` / `images` / `repos` / `full` after every catalog | +| `steps` | 8 steps | `[domain, tag]` FVT steps of every catalog | + +`cleanup` and `report` blocks: as in `build_stream_test_config.yml`. + +## 7. Not covered + +* The GitLab pipeline (`pipeline/setup_svt_project.py`) generates jobs from the + theme's stages at generation time; for catalog_build that needs the plan of a + configured run on the generating host. The theme targets the manual run. diff --git a/test/system_test/catalog_build_test_config.yml b/test/system_test/catalog_build_test_config.yml new file mode 100644 index 0000000000..aa81a1c5fc --- /dev/null +++ b/test/system_test/catalog_build_test_config.yml @@ -0,0 +1,128 @@ +# Copyright 2026 Dell Inc. or its subsidiaries. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- +# ============================================================================= +# catalog_build_test_config.yml - settings of the SVT theme catalog_build +# ============================================================================= +# Read by ./setup_env.sh --theme catalog_build and by ./run_validation.sh +# svt_catalog_build. The cluster values (IPs, names, repository URLs) are in +# user_inputs.yml; credentials are never written here +# (./setup_env.sh --theme catalog_build --set-domain-creds). +# ./setup_env.sh --configure resolves the catalog list below into the plan of the +# run (reports/catalog_build_plan.json): one stage per catalog (catalog_01, +# catalog_02 ...). Each stage selects its catalog (./omnia.sh --select-catalog), +# then runs every step as the domain's FVT " exec" and " verify". A +# change here needs ./setup_env.sh --theme catalog_build --configure (a new run). +# ============================================================================= + +# ============================================================================= +# CATALOG_BUILD (the plan of the run; written into the runtime inputs before +# every catalog, not into the suites' test_config.yml) +# ============================================================================= +catalog_build: + # catalogs: all | a list of names or numbers of ./omnia.sh --list-catalogs. + # all = every bundled catalog in the --list-catalogs order; a list runs in + # the order given, e.g. + # catalogs: + # - rhel/10.2/slurm_x86_64_no_vast.json + # - rhel/10.2/slurm_service_k8s_x86_64_no_vast.json + catalogs: all + # include_aarch64: false (default) skips every catalog with an aarch64 layer + # (aarch64-only or mixed): its stage holds one skipped test with the reason. + # true runs them; it needs + # image_build_manager.image_build_config.aarch64_inventory_host_ip in + # user_inputs.yml. + include_aarch64: false + # ---- global for the whole run: written into the runtime inputs + # (//input//) before every catalog ---- + # repo_config: always | partial (repo_manager_config.yml repo_config) + repo_config: partial + # caching_policy: true | false (repo_manager_config.yml caching_policy) + caching_policy: true + # repo_policies: per repository "//": {policy, caching} + # policy: always | partial | never; caching: true | false, e.g. + # repo_policies: + # "10.2/x86_64/epel": {policy: always, caching: false} + repo_policies: {} + # force_rebuild: true (default) builds every image of every catalog from + # scratch; false reuses an image group already built (image build skips it). + force_rebuild: true + # continue_on_failure: true makes every catalog stage soft: a failed catalog + # does not stop the next one (the summary still fails). false stops the run + # at the first failed catalog. + continue_on_failure: true + # cleanup_between: none | images | repos | full, after every catalog (runs + # even when a step of the catalog failed): + # images = fvt_image_build_manager cleanup_images test + # repos = fvt_repo_manager cleanup test (the next catalog prepares again) + # full = images + image_build_manager cleanup + repos + cleanup_between: none + # steps: the FVT steps of every catalog, in order: [domain, tag] + # (domain repo_manager | image_build_manager; tag one of its FVT tags). + # Each becomes two SVT tests: " exec" (the playbook) and " verify". + steps: + - [repo_manager, precheck] + - [repo_manager, prepare] + - [repo_manager, execute] + - [repo_manager, status] + - [image_build_manager, precheck] + - [image_build_manager, validate] + - [image_build_manager, prepare] + - [image_build_manager, build] + +# ============================================================================= +# CLEANUP (./setup_env.sh --theme catalog_build --cleanup, or --force; +# destructive, asks 'yes' unless --yes) +# ============================================================================= +# The same steps as the build_stream theme, so an earlier BuildStream, telemetry +# or orchestrator deployment cannot block omnia.sh --cleanup --all: build_stream +# (when installed), telemetry (when deployed), orchestrator, image_build_manager, +# repo_manager, main. See build_stream_test_config.yml for every flag. +cleanup: + build_stream: + enabled: true + postgres_backup: false + cleanup_credentials: true + telemetry: + enabled: true + delete_sinks_volume: true + cleanup_credentials: true + cleanup_logs: true + orchestrator: + enabled: true + cleanup_credentials: true + cleanup_slurm: true + cleanup_k8s: true + cleanup_benchmark_tools: true + image_build_manager: + enabled: true + cleanup_credentials: true + repo_manager: + enabled: true + cleanup_credentials: true + main: + enabled: true + cleanup_all: true + +# ============================================================================= +# REPORT (one folder and one SVT report per run) +# ============================================================================= +report: + # report_path: parent of the run folders (default: "" = + # test/system_test/reports). Keep it outside OMNIA_DATA_PATH. + report_path: "" + # report_name: (default: "" = svt_catalog_build_report) + report_name: "" + # run_id: (default: "" = a new svt- at every ./setup_env.sh --configure) + run_id: "" diff --git a/test/system_test/dataset_generator/profiles/catalog_build.yml b/test/system_test/dataset_generator/profiles/catalog_build.yml new file mode 100644 index 0000000000..769ca3f71b --- /dev/null +++ b/test/system_test/dataset_generator/profiles/catalog_build.yml @@ -0,0 +1,48 @@ +# Copyright 2026 Dell Inc. or its subsidiaries. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- +# ============================================================================== +# Profile: catalog_build +# ============================================================================== +# The inputs the catalog_build theme needs: omnia.env, repo_manager and +# image_build_manager. The orchestrator and build_stream blocks of +# user_inputs.yml are not used. See build_stream.yml for the field meanings. +# ============================================================================== +name: catalog_build +description: >- + Inputs of the catalog_build system test: omnia.env, repo_manager and + image_build_manager. + +targets: + - section: main.omnia_env + file: src/main/omnia.env + format: env + out: main/omnia.env + - section: repo_manager.repo_manager_config + file: src/repo_manager/input/repo_manager_config.yml + format: yaml + out: repo_manager/input/repo_manager_config.yml + - section: image_build_manager.image_build_config + file: src/image_build_manager/input/image_build_config.yml + format: yaml + out: image_build_manager/input/image_build_config.yml + +input_dirs: + - src/repo_manager/input + - src/image_build_manager/input + +mandatory: + - main.omnia_env.SYSTEM_ADMIN_NIC_IPV4 + - main.omnia_env.SYSTEM_HOSTNAME + - main.omnia_env.SYSTEM_DOMAIN_NAME diff --git a/test/system_test/docs/CATALOG_BUILD.md b/test/system_test/docs/CATALOG_BUILD.md new file mode 100644 index 0000000000..cf7fdb3310 --- /dev/null +++ b/test/system_test/docs/CATALOG_BUILD.md @@ -0,0 +1,537 @@ +# User Guide: SVT Theme `catalog_build` + +Run the Repo Manager and Image Build Manager FVTs against many Omnia catalogs, one +catalog at a time, on one OIM, and get one SVT report for all of them. + +``` +for every catalog: + ./omnia.sh --select-catalog + repo_manager precheck -> prepare -> execute -> status (exec + verify each) + image_build_manager precheck -> validate -> prepare -> build (exec + verify each) +``` + +No orchestrator deploy and no PXE boot: a catalog is done when its images are built +and verified. + +**Contents** + +1. [Before you start](#1-before-you-start) +2. [Quick start](#2-quick-start) +3. [The files you edit](#3-the-files-you-edit) +4. [Step by step](#4-step-by-step) +5. [Settings reference](#5-settings-reference) +6. [Reading the results](#6-reading-the-results) +7. [When something fails](#7-when-something-fails) +8. [Common tasks](#8-common-tasks) +9. [Troubleshooting](#9-troubleshooting) +10. [Command reference](#10-command-reference) + +--- + +## 1. Before you start + +| You need | Why | +|---|---| +| A RHEL OIM, logged in as `root` | The SVT installs and runs Omnia on this host | +| An Omnia checkout on the OIM | The SVT runs from `/test/system_test` | +| Python 3.12+ | `setup_env.sh` builds the SVT's own `.venv` from it | +| A RHEL subscription on the OIM, or `baseos` / `appstream` / `codeready-builder` URLs | Repo Manager reads RHEL content from them | +| Mirrors of the non-public repositories: `slurm_custom`, `ldms` (and `vast` for catalogs without `_no_vast`) | Repo Manager precheck fails a catalog whose repositories have no URL | +| Repo Manager credentials (Pulp, Docker Hub) and Image Build Manager credentials (S3, registry) | Asked once by `setup_env.sh`, kept encrypted in the Omnia data path | +| Free disk space under `OMNIA_DATA_PATH` | Every catalog adds packages to Pulp and images to S3 / the registry | +| An aarch64 build host (optional) | Only for catalogs with aarch64 layers (`include_aarch64: true`) | + +> **Destructive option:** `--force` and `--cleanup` remove the previous Omnia installation +> of the OIM, **including everything under `OMNIA_DATA_PATH`** (inputs, credentials, +> Pulp content, images, logs). Back up what you need first. Without `--force` nothing +> is removed. + +All commands below run from: + +```bash +cd /test/system_test +``` + +--- + +## 2. Quick start + +```bash +# 1. Fill the OIM identity and the repository URLs (section 3.1) +vi user_inputs.yml + +# 2. Choose the catalogs and the settings (section 3.2) +vi catalog_build_test_config.yml + +# 3. Prepare the OIM: configure (writes the catalog plan) -> install -> credentials +./setup_env.sh --theme catalog_build # keep the current installation +# ./setup_env.sh --theme catalog_build --force # or: clean reinstall first (destructive) + +# 4. Check it is ready +./setup_env.sh --theme catalog_build --status # every line [ OK ], then READY + +# 5. See what will run +./run_validation.sh svt_catalog_build list + +# 6. Run every catalog, then the summary +./run_validation.sh svt_catalog_build all + +# 7. Open the report +ls -td reports/svt-*/ | head -1 # the run folder +# /svt_catalog_build_report.html +``` + +> **The rule to remember:** whenever you change `user_inputs.yml` or +> `catalog_build_test_config.yml`, run step 3 again +> (`./setup_env.sh --theme catalog_build`). It starts a new run, rewrites the inputs +> and the catalog plan, and installs them. Running only `--configure` after an edit +> leaves the installed inputs old and the next stage refuses to start. + +--- + +## 3. The files you edit + +You edit exactly two files. Everything else is generated. + +### 3.1 `user_inputs.yml` - the values of your cluster + +Shared with the `build_stream` theme. The `catalog_build` theme uses three blocks: + +| Block | What to fill | Written to | +|---|---|---| +| `main.omnia_env` | **Mandatory:** `SYSTEM_ADMIN_NIC_IPV4`, `SYSTEM_HOSTNAME`, `SYSTEM_DOMAIN_NAME` of the OIM | `src/main/omnia.env` | +| `repo_manager.repo_manager_config.repositories` | The URLs of every repository your catalogs use, per RHEL version and architecture | `src/repo_manager/input/repo_manager_config.yml` | +| `image_build_manager.image_build_config` | `aarch64_inventory_host_ip` (only with `include_aarch64: true`); optional keys of `image_build_config.yml`, e.g. `build_image.max_parallel` | `src/image_build_manager/input/image_build_config.yml` | + +The `orchestrator`, `build_stream` and `openbao` blocks are not used by this theme. + +Example (RHEL 10.2, x86_64): + +```yaml +main: + omnia_env: + SYSTEM_ADMIN_NIC_IPV4: "172.16.0.254" + SYSTEM_HOSTNAME: "demo" + SYSTEM_DOMAIN_NAME: "openchami.cluster" + +repo_manager: + repo_manager_config: + repositories: + "10.2": + x86_64: + baseos: {url: ""} # "" = the OIM's RHEL subscription + appstream: {url: ""} + codeready-builder: {url: ""} + user_repos: + slurm_custom: {url: "http:///omnia/rhel10.2/x86_64/slurm/25.05.2/with_gpu/"} + ldms: {url: "http:///omnia/rhel10.2/x86_64/ldms/4.5.2/"} + vast: {url: "http:///omnia/rhel10.2/x86_64/vast/"} + +image_build_manager: + image_build_config: + aarch64_inventory_host_ip: "" + build_image: + max_parallel: 2 +``` + +Rules of the file (checked when you configure): + +- A key must exist in the target input file; a typo is reported, not ignored. +- An empty value (`""`) keeps the shipped default. +- Credentials never go here; `setup_env.sh` asks for them. +- RHEL 10.0 catalogs and the `hybrid/` catalogs also need the `"10.0"` entries. + +### 3.2 `catalog_build_test_config.yml` - what the theme runs + +```yaml +catalog_build: + catalogs: all # "all" (shipped default), or a list in the order to run: + # catalogs: + # - rhel/10.2/slurm_x86_64_no_vast.json + # - rhel/10.2/slurm_service_k8s_x86_64_no_vast.json + include_aarch64: false + repo_config: partial + caching_policy: true + repo_policies: {} + force_rebuild: true + continue_on_failure: true + cleanup_between: none + steps: + - [repo_manager, precheck] + - [repo_manager, prepare] + - [repo_manager, execute] + - [repo_manager, status] + - [image_build_manager, precheck] + - [image_build_manager, validate] + - [image_build_manager, prepare] + - [image_build_manager, build] + +cleanup: # what --force / --cleanup removes (same flags as build_stream) + ... +report: # where the run folder and the report go + ... +``` + +Every key is explained in [section 5](#5-settings-reference). To see the catalog +names you can use: + +```bash +../../src/main/omnia.sh --list-catalogs # or: cd ../../src/main && ./omnia.sh --list-catalogs +``` + +A catalog can be given by its name (`rhel/10.2/slurm_x86_64.json`), by its file name +when that is unique (`slurm_x86_64.json`), or by its number in that list. + +--- + +## 4. Step by step + +### 4.1 Prepare the OIM: `./setup_env.sh --theme catalog_build` + +With no action, `setup_env.sh` runs every setup step in order: + +| Step | What it does | Changes | +|---|---|---| +| **configure** | Starts a **new run** (`svt-`), copies `user_inputs.yml` into the `src` inputs (dataset generator), writes the cleanup and report settings into each `test//test_config.yml`, then resolves the catalog list into the **catalog plan** and prints it | `src/.../input`, `src/main/omnia.env`, `test/*/test_config.yml`, `reports/catalog_build_plan.json` | +| **cleanup** (only with `--force`) | Removes the previous installation with each suite's own cleanup FVT: BuildStream (when installed), telemetry (when deployed), orchestrator, image_build_manager, repo_manager, then `omnia.sh --cleanup --all` | **Deletes** `OMNIA_DATA_PATH` and the Omnia services | +| **install** | `omnia.sh -s` (Omnia venv, domain inputs into `OMNIA_DATA_PATH`), then the FVT test requirements in the Omnia venv | Omnia installation | +| **creds** | Asks for the missing repo_manager and image_build_manager credential stores; skips the ones that exist | Encrypted credential files | + +The plan printed by configure looks like this (for the two-catalog list example): + +``` +catalog plan (2 catalog(s), 2 to run) -> .../reports/catalog_build_plan.json: + catalog_01 run rhel/10.2/slurm_x86_64_no_vast.json + catalog_02 run rhel/10.2/slurm_service_k8s_x86_64_no_vast.json +steps per catalog (exec, then verify): repo_manager precheck, ... image_build_manager build +settings: repo_config=partial, caching_policy=True, force_rebuild=True, ... +``` + +Configure fails (and tells you what to fix) when a catalog name is unknown, a key is +invalid, or `include_aarch64: true` has no `aarch64_inventory_host_ip`. + +Useful variants: + +```bash +./setup_env.sh --theme catalog_build --force # clean reinstall (destructive) +./setup_env.sh --theme catalog_build --force --yes # same, no 'yes' prompt +./setup_env.sh --theme catalog_build --cleanup --dry-run # show what cleanup would remove +./setup_env.sh --theme catalog_build --retry # resume setup at the step that failed +``` + +### 4.2 Check readiness: `--status` + +```bash +./setup_env.sh --theme catalog_build --status +``` + +``` +[ OK ] Inputs configured (user_inputs.yml, catalog_build_test_config.yml) +[ OK ] Omnia installed (omnia.sh -s: omnia.env, data path, venv, domain inputs) +[ OK ] Installed after the last configure +[ OK ] Catalog plan of this run (configure) +[ OK ] Test requirements in the Omnia venv (5 suites) +[ OK ] Credentials: repo_manager +[ OK ] Credentials: image_build_manager +READY: ./run_validation.sh svt_catalog_build all +``` + +Every `[FAIL]` line says how to fix it. `run_validation.sh` checks the same list in +its `initialization` stage. + +### 4.3 See the plan: `list` + +```bash +./run_validation.sh svt_catalog_build list +``` + +``` + initialization + catalog_01 + SVT_CATALOG_BUILD_0100 rhel/10.2/slurm_x86_64_no_vast.json: select (...) + SVT_CATALOG_BUILD_0101 rhel/10.2/slurm_x86_64_no_vast.json: repo_manager precheck exec + SVT_CATALOG_BUILD_0102 rhel/10.2/slurm_x86_64_no_vast.json: repo_manager precheck verify + ... + catalog_02 + ... + summary +``` + +### 4.4 Run: `all` + +```bash +./run_validation.sh svt_catalog_build all # output on the screen and in the logs +./run_validation.sh svt_catalog_build all --quiet # output in the logs only +``` + +The stages run in order: + +| Stage | What happens | +|---|---| +| `initialization` | The readiness checks of `--status`; the run stops here when one fails | +| `catalog_NN` | One per catalog (see below) | +| `summary` | Prints every test result, renders the report; fails when any stage failed | + +Inside a catalog stage: + +| Test | What it does | +|---|---| +| `NN00 select` | Puts back the shared runtime inputs, applies `catalog_overrides//`, writes the global settings (`repo_config`, `caching_policy`, `repo_policies`, `force_rebuild`, `functional_groups_source: catalog`) into `//input//`, then `./omnia.sh --select-catalog ` | +| `NN01`, `NN02` ... | For each `steps` entry: `fvt_ exec` (runs the playbook), then `fvt_ verify` (its verification tests) | +| last (with `cleanup_between`) | The cleanup FVT steps; they run even when the catalog failed | + +The first FVT step is the Repo Manager precheck: it validates the catalog against +`repo_manager_config.yml` (repository mapping, URLs, the RHEL subscription), so a +missing repository URL fails a catalog in about a minute, not hours later. + +You can also run one stage at a time: + +```bash +./run_validation.sh svt_catalog_build initialization +./run_validation.sh svt_catalog_build catalog_01 +./run_validation.sh svt_catalog_build summary +``` + +How long it takes depends on the catalogs, the mirrors and `force_rebuild`. For +reference, two RHEL 10.2 x86_64 catalogs on an OIM whose repositories were already +synced took about 35 and 42 minutes, most of it the image build +(`force_rebuild: true`). A first sync of a large catalog can take much longer. + +--- + +## 5. Settings reference + +All keys of the `catalog_build:` block of `catalog_build_test_config.yml`: + +| Key | Values | Default | Meaning | +|---|---|---|---| +| `catalogs` | `all` or a list | `all` | `all` = every catalog of `./omnia.sh --list-catalogs`, in that order. A list runs in the order you give | +| `include_aarch64` | `true` / `false` | `false` | `false`: a catalog with any aarch64 layer (aarch64-only, `combined`, `x86_64_aarch64`, the default catalog) is skipped as a whole and its stage says why. `true`: they run; set `aarch64_inventory_host_ip` in `user_inputs.yml` | +| `repo_config` | `always` / `partial` | `partial` | Written as `repo_config` of `repo_manager_config.yml` for every catalog | +| `caching_policy` | `true` / `false` | `true` | Written as `caching_policy` of `repo_manager_config.yml` | +| `repo_policies` | mapping | `{}` | Per repository: `"//": {policy: always\|partial\|never, caching: true\|false}`, e.g. `"10.2/x86_64/epel": {policy: always, caching: false}`. The repository must exist in `repo_manager_config.yml` | +| `force_rebuild` | `true` / `false` | `true` | Written as `build_image.force_rebuild`. `true` builds every image of every catalog from scratch. `false` reuses an image group that was already built (faster, but a reused image is not rebuilt) | +| `continue_on_failure` | `true` / `false` | `true` | `true`: a failed catalog does not stop the next one (the summary still fails). `false`: the run stops at the first failed catalog | +| `cleanup_between` | `none` / `images` / `repos` / `full` | `none` | After every catalog. `images` = `fvt_image_build_manager cleanup_images test`. `repos` = `fvt_repo_manager cleanup test` (removes Pulp; the next catalog prepares it again). `full` = both, plus `fvt_image_build_manager cleanup test` | +| `steps` | list of `[domain, tag]` | the 8 steps shown above | The FVT steps of every catalog, in order. `domain` is `repo_manager` or `image_build_manager`; `tag` is one of its FVT tags (checked at configure). Each step becomes an `exec` test and a `verify` test | + +What `repo_config` and `caching_policy` mean for Repo Manager: + +| `repo_config` | `caching_policy` | Pulp policy | Effect | +|---|---|---|---| +| `always` | `false` | `immediate` | Every package downloaded and stored | +| `always` | `true` | `on_demand` | Stored when first requested | +| `partial` | `false` | `streamed` | Packages only validated, not stored; the image build then needs the upstream repositories | +| `partial` | `true` | `on_demand` | Stored when first requested | + +The `cleanup` and `report` blocks are the same as in `build_stream_test_config.yml` +(see [TEST_CONFIG.md](TEST_CONFIG.md)). + +### 5.1 Per-catalog input files (optional) + +To give one catalog a different input file, place it under: + +``` +test/system_test/catalog_overrides/// +e.g. catalog_overrides/rhel/10.2/slurm_x86_64.json/image_build_manager/image_build_config.yml +``` + +The `select` test of that catalog copies it over the runtime input; the next catalog +gets the shared file back. The global settings of section 5 are written after the +override, so they always apply. Credential files are never overridden. + +--- + +## 6. Reading the results + +Everything of one run is in one folder: + +``` +reports// +├── svt_catalog_build_report.html the report (open in a browser) +├── svt_catalog_build_report.json the same data as JSON +├── initialization.log, catalog_01.log, catalog_02.log, summary.log full output per stage +├── .status passed | skipped | failed +├── .progress the current step of a running stage +├── run.json the run manifest (settings, setup, stage outcomes) +├── svt_results.json one line per SVT test (used by summary) +├── junit_catalog_build_.xml JUnit per stage +├── fvt/ the FVT suites' own reports +├── setup/ the setup_env.sh logs of this run +└── baseline_inputs/ shared inputs kept while an override was active +``` + +Find the latest run: + +```bash +ls -td reports/svt-*/ | head -1 +``` + +The HTML report shows, top to bottom: + +| Section | What you see | +|---|---| +| Banner | `OVERALL STATUS: PASSED`, `FAILED` or `RUNNING`, and how many stages ran | +| Cards | SVT steps passed / failed / skipped, pass rate, duration | +| Run Details | theme, run id, OIM, repository, branch, commit | +| Stage Overview | **one row per catalog** (`Catalog 1: rhel/10.2/...`): status, steps passed / failed / skipped, duration - the catalog matrix | +| Stage Details | per catalog, every test: its command, the FVT results (TC ID, test, status, details, error) and its screen output; failed tests open by default | + +Follow a running stage from another terminal: + +```bash +R=$(ls -td reports/svt-*/ | head -1) +cat "$R"/*.progress # the current step of each stage +tail -f "$R"/catalog_01.log # the full output +``` + +--- + +## 7. When something fails + +### What happens automatically + +| Situation | Behaviour | +|---|---| +| A test of a catalog fails | The later tests of that catalog are skipped (they need what it set up); `cleanup_between` still runs | +| A catalog stage fails, `continue_on_failure: true` | The next catalog runs; the summary fails | +| A catalog stage fails, `continue_on_failure: false` | The run stops; only the summary runs | +| A playbook hits the FVT timeout (7200 s) | The step fails; the theme stops the `ansible-playbook` workers and the `dnf download` it left running, so they cannot overlap the next test (noted in the report) | +| A catalog has aarch64 layers and `include_aarch64: false` | Its stage holds one skipped test with the reason | + +### What you do next + +1. Open the report, find the failed test, read its error and output (or the stage log). +2. Fix the cause (a repository URL, a credential, disk space, the mirror ...). +3. Choose: + +| You want to | Run | +|---|---| +| Repeat the failed stage in the **same run** (same report; the new attempt replaces the old one) | `./run_validation.sh svt_catalog_build all --retry` | +| Repeat only one stage in the same run | `./run_validation.sh svt_catalog_build catalog_02 --retry` | +| Start again from a catalog in a **new run** | `./setup_env.sh --theme catalog_build` then `./run_validation.sh svt_catalog_build all --start-stage catalog_02` | +| Change a setting or an input, then run again | Edit the file, `./setup_env.sh --theme catalog_build`, then `./run_validation.sh svt_catalog_build all` | + +A failed run takes no further step except `--retry` and the summary. That is on +purpose: one run is one report, and nothing is added to it after an unhandled +failure. `./setup_env.sh --theme catalog_build` starts a new run; the installation +and the credentials are kept. + +--- + +## 8. Common tasks + +**Run every bundled catalog** + +```yaml +catalog_build: + catalogs: all +``` + +With `include_aarch64: false` only the x86_64-only catalogs run; the others are +listed as skipped. Every catalog's repositories need URLs in `user_inputs.yml` +(RHEL 10.0 and 10.2, `vast` for the catalogs without `_no_vast`). + +**Run the aarch64 catalogs too** + +```yaml +# catalog_build_test_config.yml +catalog_build: + include_aarch64: true +# user_inputs.yml +image_build_manager: + image_build_config: + aarch64_inventory_host_ip: "" +``` + +Fill the `aarch64` repository entries in `user_inputs.yml` as well. + +**Faster re-runs:** `force_rebuild: false` reuses images that were already built; +Repo Manager syncs only what is new. + +**Keep the disk usage down on a long run:** `cleanup_between: images` (or `full`). + +**Test one Repo Manager policy:** `repo_config: always`, `caching_policy: false` +(every package stored), or a single repository through `repo_policies`. + +**Only Repo Manager:** keep only the `repo_manager` entries in `steps`. + +**Clean reinstall before the run** + +```bash +./setup_env.sh --theme catalog_build --force --yes +./run_validation.sh svt_catalog_build all +``` + +**Change a credential** + +```bash +./setup_env.sh --theme catalog_build --update-domain-creds --domain repo_manager +``` + +**Leave the checkout clean afterwards.** `--configure` writes into tracked files of +the checkout. Before committing anything: + +```bash +cd +git checkout -- src/main/omnia.env src/repo_manager/input/repo_manager_config.yml \ + src/image_build_manager/input/image_build_config.yml test/*/test_config.yml +``` + +Keep cluster-specific values (IPs, mirror URLs) out of `user_inputs.yml` in commits. + +--- + +## 9. Troubleshooting + +| Message | Cause | Fix | +|---|---|---| +| `main.omnia_env.SYSTEM_ADMIN_NIC_IPV4: MANDATORY, fill it` | `user_inputs.yml` misses the OIM identity | Fill the three `omnia_env` values, run `./setup_env.sh --theme catalog_build` | +| `Installed after the last configure: the inputs were configured again after the last install` | `--configure` ran after the last install, so the installed inputs are old | `./setup_env.sh --theme catalog_build` (configure + install) | +| `run ... has a failed step: ... A failed run takes no further step` | A step of the current run failed | `--retry` (same run) or `./setup_env.sh --theme catalog_build` (new run) | +| `no catalog plan for run ...` / `the catalog plan ... belongs to run ...` | The plan file is missing or from another run | `./setup_env.sh --theme catalog_build --configure`, then install (`./setup_env.sh --theme catalog_build`) | +| `catalog '...' is not in ./omnia.sh --list-catalogs` | A typo in `catalogs` | Use a name or number from `./omnia.sh --list-catalogs` | +| `include_aarch64 is true but ... aarch64_inventory_host_ip is empty` | aarch64 catalogs without a build host | Set the IP in `user_inputs.yml`, or `include_aarch64: false` | +| `catalog_build.: ...` | An invalid value in the theme block | The message names the key and the allowed values | +| `repo_policies: is not a repository of ...` | A `repo_policies` key that the config does not have | Use `"//"` of an existing repository | +| `repo_manager precheck exec` fails | A repository of the catalog has no URL, or no subscription for RHEL content | Add the URL to `user_inputs.yml`; check `subscription-manager status` | +| `Playbook execution timed out ... Timeout: 7200s` | One FVT playbook took more than two hours (often the first sync of a large catalog) | Run again: synced content is kept, so the next attempt is shorter; use faster mirrors | +| `another SVT step is running on this OIM` | Another `setup_env.sh` / `run_validation.sh` holds the SVT lock | Wait for it; a lock whose process is gone is taken over automatically | +| `Credentials: [FAIL]` | A credential store is missing (e.g. after `--force`) | `./setup_env.sh --theme catalog_build --set-domain-creds --domain ` | + +More on the shared parts: [SETUP.md](SETUP.md) (setup, credentials, OpenBao), +[RUNNER.md](RUNNER.md) (the run guard, retry, the report), +[TEST_CONFIG.md](TEST_CONFIG.md) (the cleanup and report blocks). + +--- + +## 10. Command reference + +**Setup** (`./setup_env.sh --theme catalog_build ...`) + +| Command | Purpose | +|---|---| +| *(no action)* | configure -> install -> missing credentials | +| `--force` | configure -> **cleanup** -> full install -> credentials | +| `--configure` | New run: inputs, suite settings, catalog plan | +| `--install` | `omnia.sh -s` and the test requirements | +| `--cleanup` | **Destructive** cleanup only (asks `yes`) | +| `--cleanup --dry-run` | Show what cleanup would remove | +| `--set-domain-creds [--domain ]` | Enter the credential stores | +| `--update-domain-creds --domain ` | Change a credential store | +| `--status` | Steps and readiness checks | +| `--retry` | Resume the setup at the failed step, same run | +| `--yes` | Answer every confirmation | + +**Run** (`./run_validation.sh svt_catalog_build ...`) + +| Command | Purpose | +|---|---| +| `list` | The stages and tests of the current plan | +| `all` | Every stage, then summary | +| `all --start-stage catalog_NN` | Start at a catalog (the earlier ones are not run) | +| `all --retry` | Resume the same run at the first stage that did not pass | +| `` | One stage: `initialization`, `catalog_NN`, `summary` | +| ` --retry` | Run a failed stage again in the same run | +| `--quiet` | Output to the stage logs only | +| `--help` | All themes and options | diff --git a/test/system_test/library/functions/catalog_func.py b/test/system_test/library/functions/catalog_func.py new file mode 100644 index 0000000000..7f6a4c717d --- /dev/null +++ b/test/system_test/library/functions/catalog_func.py @@ -0,0 +1,391 @@ +# Copyright 2026 Dell Inc. or its subsidiaries. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""catalog_build theme: the catalog plan and the SVT actions of a catalog stage. + +* ``write_plan`` (./setup_env.sh --configure): resolve the catalogs of the theme + block (``all`` = ./omnia.sh --list-catalogs order), mark the ones with an + aarch64 layer as skipped unless include_aarch64, and freeze the plan in + catalog_vars.CATALOG_PLAN_FILE. The stages and tests are generated from it + (catalog_vars.py) and run like every other theme: svt/test_svt.py. +* ``run_action`` (the first test of a catalog stage): put back the shared + runtime inputs, apply catalog_overrides///, write the + global settings (repo_config, caching_policy, repo_policies, force_rebuild, + functional_groups_source: catalog), then ./omnia.sh --select-catalog. +* ``stop_leftovers`` (after a failed exec): the ansible-playbook workers run in + their own session, so a playbook killed at the FVT timeout can leave them (and + a dnf download) running; they are stopped before the next test. + +Nothing is skipped for content an earlier catalog brought: repo_manager syncs +only what is new; image_build_manager rebuilds every image (force_rebuild true, +the default) or reuses a matching one (false). +""" + +import json +import os +import shutil +import signal +import time +from datetime import datetime, timezone +from typing import Any, Callable, Dict, List, Optional, Tuple + +import yaml +from ruamel.yaml.comments import CommentedMap + +from dataset_generator.dataset_edit import Change, dump_yaml, load_yaml, set_value, split_header + +from .catalog_settings import describe_settings +from .omnia_func import domain_input_dir, omnia_env +from .run_func import load_run +from .setup_func import bundled_catalogs, resolve_catalog, row, select_catalog +from ..messages.svt_msgs import CATALOG_MSGS +from ..vars.catalog_vars import ( + ACTION_SELECT, + ACTION_SKIP, + CATALOG_PLAN_FILE, + CATALOG_STAGE_PREFIX, + CATALOG_STEP_DOMAINS, + CATALOG_STEP_PLAYBOOKS, + load_plan, +) +from ..vars.common_vars import INPUTS_FILE, MODULE_ROOT, REPO_ROOT + +Emit = Callable[[str], None] + +SAMPLES_DIR = os.path.join(REPO_ROOT, "src", "main", "samples") +DEFAULT_CATALOG = "default" +DEFAULT_CATALOG_FILE = os.path.join(SAMPLES_DIR, "catalog_rhel.json") +OVERRIDES_DIR = os.path.join(MODULE_ROOT, "catalog_overrides") +BASELINE_DIR = "baseline_inputs" +REPO_INPUT = ("repo_manager", "repo_manager_config.yml") +IMAGE_INPUT = ("image_build_manager", "image_build_config.yml") +NESTED_REPO_SECTIONS = ("user_repos", "additional_repos") +AARCH64_HOST_KEY = ("image_build_manager", "image_build_config", "aarch64_inventory_host_ip") +PROC_DIR = "/proc" + + +# ------------------------------------------------------------------- plan +def catalog_file(name: str) -> str: + """Return the bundled catalog file of a --list-catalogs name.""" + if name == DEFAULT_CATALOG: + return DEFAULT_CATALOG_FILE + return os.path.join(SAMPLES_DIR, "catalogs", name) + + +def aarch64_layers(path: str) -> List[str]: + """Return the functional layers of a catalog built for aarch64.""" + with open(path, "r", encoding="utf-8") as stream: + catalog = json.load(stream).get("catalog", {}) + return [str(layer.get("name", "")) for layer in catalog.get("functionallayer", []) + if isinstance(layer, dict) and "aarch64" in str(layer.get("name", ""))] + + +def _aarch64_host() -> str: + try: + with open(INPUTS_FILE, "r", encoding="utf-8") as stream: + node: Any = yaml.safe_load(stream) + except (OSError, yaml.YAMLError): + return "" + for key in AARCH64_HOST_KEY: + node = node.get(key) if isinstance(node, dict) else None + return "" if node is None else str(node).strip() + + +def _selected_names(settings: Dict[str, Any]) -> List[str]: + """Return the --list-catalogs names to run, in run order.""" + listed, error = bundled_catalogs() + if error: + raise ValueError(error) + if settings["catalogs"] == "all": + return [listed[number] for number in sorted(listed, key=int)] + names = [] + for selection in settings["catalogs"]: + name, problem = resolve_catalog(selection) + if problem: + raise ValueError(CATALOG_MSGS["unknown_catalog"].format(selection=selection)) + names.append(name) + return names + + +def plan(settings: Dict[str, Any]) -> List[Dict[str, Any]]: + """Return one entry per catalog: stage, index, catalog, run (bool), reason. + + Raises: + ValueError: --list-catalogs fails, a catalog is unknown or missing, or + include_aarch64 is true without an aarch64 build host. + """ + if settings["include_aarch64"] and not _aarch64_host(): + raise ValueError(CATALOG_MSGS["aarch64_host_missing"]) + entries = [] + for index, name in enumerate(_selected_names(settings), 1): + path = catalog_file(name) + if not os.path.isfile(path): + raise ValueError(CATALOG_MSGS["catalog_file_missing"].format(path=path)) + layers = aarch64_layers(path) + skip = bool(layers) and not settings["include_aarch64"] + entries.append({"stage": f"{CATALOG_STAGE_PREFIX}{index:02d}", "index": index, + "catalog": name, "run": not skip, + "reason": CATALOG_MSGS["skip_aarch64"].format( + layers=", ".join(layers)) if skip else ""}) + return entries + + +def plan_lines(settings: Dict[str, Any], entries: List[Dict[str, Any]]) -> List[str]: + """Return the plan as printed by --configure.""" + lines = [CATALOG_MSGS["plan_header"].format( + count=len(entries), run=sum(1 for entry in entries if entry["run"]), + path=CATALOG_PLAN_FILE)] + lines.extend(CATALOG_MSGS["plan_line"].format( + stage=entry["stage"], mark="run" if entry["run"] else "skip", catalog=entry["catalog"], + reason=f" ({entry['reason']})" if entry["reason"] else "") for entry in entries) + lines.append(CATALOG_MSGS["steps_header"].format( + steps=", ".join(f"{domain} {tag}" for domain, tag in settings["steps"]))) + lines.append(CATALOG_MSGS["settings"].format(settings=describe_settings(settings))) + return lines + + +def write_plan(settings: Dict[str, Any], run_id: str, emit: Emit, + path: str = CATALOG_PLAN_FILE) -> Tuple[List[Dict[str, Any]], str]: + """Resolve and store the plan of run ``run_id``; return (report rows, error).""" + try: + entries = plan(settings) + except ValueError as exc: + return [row("catalog plan", "FAILED", error=str(exc))], str(exc) + lines = plan_lines(settings, entries) + for line in lines: + emit(line) + os.makedirs(os.path.dirname(path), mode=0o750, exist_ok=True) + temp = f"{path}.tmp" + with open(temp, "w", encoding="utf-8") as stream: + json.dump({"run_id": run_id, "created_at": datetime.now(timezone.utc).isoformat( + timespec="seconds"), "settings": {**settings, "steps": [list(step) for step in + settings["steps"]]}, + "entries": entries}, stream, indent=2) + os.replace(temp, path) + return [row("catalog plan", "PASSED", details="\n".join(lines))], "" + + +# ------------------------------------------------------------ runtime inputs +def _runtime_dir(domain: str) -> str: + return domain_input_dir(omnia_env(), domain) + + +def _is_credential(name: str) -> bool: + return name.startswith(".") or "credential" in name + + +def _override_files(catalog: str) -> List[tuple]: + """Return (domain, file name, source) of every override of ``catalog``.""" + files = [] + for domain in CATALOG_STEP_DOMAINS: + folder = os.path.join(OVERRIDES_DIR, catalog, domain) + if not os.path.isdir(folder): + continue + for name in sorted(os.listdir(folder)): + source = os.path.join(folder, name) + if os.path.isfile(source) and not os.path.islink(source) and not _is_credential(name): + files.append((domain, name, source)) + return files + + +def restore_and_override(run_folder: str, catalog: str, emit: Emit) -> List[str]: + """Put back every shared input an earlier override replaced, then apply ``catalog``'s. + + The shared copy of a file is kept in /baseline_inputs/ the + first time an override replaces it. Returns what was done. + """ + done = [] + baseline = os.path.join(run_folder, BASELINE_DIR) + for domain in CATALOG_STEP_DOMAINS: + saved = os.path.join(baseline, domain) + for name in sorted(os.listdir(saved)) if os.path.isdir(saved) else []: + shutil.copy2(os.path.join(saved, name), os.path.join(_runtime_dir(domain), name)) + for domain, name, source in _override_files(catalog): + target = os.path.join(_runtime_dir(domain), name) + keep = os.path.join(baseline, domain, name) + if not os.path.exists(keep) and os.path.isfile(target): + os.makedirs(os.path.dirname(keep), mode=0o750, exist_ok=True) + shutil.copy2(target, keep) + shutil.copy2(source, target) + emit(CATALOG_MSGS["override_applied"].format(source=source, target=target)) + done.append(f"override {domain}/{name}") + return done + + +def _edit_yaml(path: str, edit: Callable[[Any, List[Change]], None]) -> List[Change]: + """Apply ``edit`` to a YAML input file, keeping its comments; write only on change.""" + if not os.path.isfile(path): + raise ValueError(CATALOG_MSGS["no_runtime_input"].format(path=path)) + with open(path, "r", encoding="utf-8") as stream: + header, body = split_header(stream.read()) + document = load_yaml(body) + changes: List[Change] = [] + edit(document, changes) + if changes: + temp = f"{path}.svt-tmp" + with open(temp, "w", encoding="utf-8") as stream: + stream.write(header + dump_yaml(document)) + os.replace(temp, path) + return changes + + +def _key_of(mapping: Any, key: str) -> Optional[Any]: + """Return the key of ``mapping`` equal to ``key`` as text ("10.2" == 10.2).""" + return next((existing for existing in mapping if str(existing) == key), None) \ + if isinstance(mapping, dict) else None + + +def _repo_entry(document: Any, key: str, path: str) -> Any: + """Return the mapping of repository ``//`` (created when null).""" + version, arch, name = key.split("/") + repositories = document.get("repositories") if isinstance(document, dict) else None + arches = repositories.get(_key_of(repositories, version)) \ + if _key_of(repositories, version) is not None else None + repos = arches.get(arch) if isinstance(arches, dict) else None + holders = [repos] + [repos.get(section) for section in NESTED_REPO_SECTIONS] \ + if isinstance(repos, dict) else [] + for holder in holders: + if isinstance(holder, dict) and name in holder: + if not isinstance(holder[name], dict): + holder[name] = CommentedMap() + return holder[name] + raise ValueError(CATALOG_MSGS["repo_missing"].format(key=key, path=path)) + + +def write_settings(settings: Dict[str, Any]) -> List[Change]: + """Write the global settings into the runtime repo_manager and image inputs.""" + repo_path = os.path.join(_runtime_dir(REPO_INPUT[0]), REPO_INPUT[1]) + image_path = os.path.join(_runtime_dir(IMAGE_INPUT[0]), IMAGE_INPUT[1]) + + def repo(document: Any, changes: List[Change]) -> None: + set_value(document, "repo_config", settings["repo_config"], changes, repo_path, "top") + set_value(document, "caching_policy", settings["caching_policy"], changes, repo_path, + "top") + for key, entry in settings["repo_policies"].items(): + target = _repo_entry(document, key, repo_path) + for field, value in entry.items(): + set_value(target, field, value, changes, repo_path, key) + + def image(document: Any, changes: List[Change]) -> None: + build = document.get("build_image") + if not isinstance(build, dict): + build = document["build_image"] = CommentedMap() + set_value(build, "force_rebuild", settings["force_rebuild"], changes, image_path, + "build_image") + set_value(document, "functional_groups_source", "catalog", changes, image_path, "top") + + return _edit_yaml(repo_path, repo) + _edit_yaml(image_path, image) + + +# ------------------------------------------------------------------ actions +def _outcome(success: bool, started: float, command: str, output: str, step: tuple, + **extra: Any) -> Dict[str, Any]: + """Return an action result shaped like fvt_func.run_fvt_step's.""" + return {"success": success, "rc": 0 if success else 1, "duration": time.monotonic() - started, + "command": command, "output": output, "results": [], "step": " ".join(step[1]), + "fvt": {"suite": "test/system_test", "domain": "svt", "tag": step[1][0], + "action": "", "options": " ".join(step[1][1:])}, **extra} + + +def _select(case: Dict[str, Any], emit: Emit) -> Dict[str, Any]: + started = time.monotonic() + settings = load_plan().get("settings") or {} + run_folder = load_run().get("report_path", "") + rows: List[Dict[str, Any]] = [] + notes: List[str] = [] + error = "" + try: + notes.extend(restore_and_override(run_folder, case["catalog"], emit)) + changes = write_settings(settings) + notes.append(f"{len(changes)} setting(s) written: {describe_settings(settings)}") + emit(CATALOG_MSGS["inputs_written"].format(details=notes[-1])) + error = select_catalog(case["catalog"], emit, rows) + except (OSError, ValueError, KeyError) as exc: + error = f"{type(exc).__name__}: {exc}" + selected = rows[0] if rows else {} + output = "\n".join([*notes, str(selected.get("output") or ""), error]).strip() + return _outcome(not error, started, str(selected.get("command") or + f"./omnia.sh --select-catalog {case['catalog']}"), + output, case["step"], error=error) + + +def run_action(case: Dict[str, Any], emit: Emit = print) -> Dict[str, Any]: + """Run the SVT action of a catalog test case; return a run_fvt_step-shaped result. + + A skipped catalog returns ``skipped`` (the reason) instead of running anything. + """ + if case.get("action") == ACTION_SKIP: + return _outcome(True, time.monotonic(), "", case.get("reason", ""), case["step"], + skipped=case.get("reason", "")) + if case.get("action") == ACTION_SELECT: + return _select(case, emit) + raise ValueError(f"unknown SVT action {case.get('action')!r}") + + +# -------------------------------------------------------- leftover processes +def _processes(proc: str = PROC_DIR) -> Dict[int, Tuple[int, List[str]]]: + """Return {pid: (ppid, argv)} of every readable process under /proc.""" + found: Dict[int, Tuple[int, List[str]]] = {} + for name in os.listdir(proc): + if not name.isdigit(): + continue + try: + with open(os.path.join(proc, name, "stat"), "r", encoding="utf-8") as stream: + ppid = int(stream.read().rsplit(")", 1)[1].split()[1]) + with open(os.path.join(proc, name, "cmdline"), "rb") as stream: + argv = [part.decode("utf-8", "replace") + for part in stream.read().split(b"\0") if part] + except (OSError, IndexError, ValueError): + continue + found[int(name)] = (ppid, argv) + return found + + +def leftover_playbooks(domain: str, processes: Dict[int, Tuple[int, List[str]]]) -> List[int]: + """Return orphaned ``ansible-playbook `` processes and their descendants. + + Orphaned = reparented to init (ppid 1): its FVT step already ended. + """ + playbook = CATALOG_STEP_PLAYBOOKS.get(domain) + roots = [pid for pid, (ppid, argv) in processes.items() + if ppid == 1 and playbook in argv + and any(os.path.basename(arg) == "ansible-playbook" for arg in argv)] + tree = set(roots) + grown = True + while grown: + grown = False + for pid, (ppid, _) in processes.items(): + if ppid in tree and pid not in tree: + tree.add(pid) + grown = True + return sorted(tree) + + +def stop_leftovers(domain: str, emit: Emit, grace: float = 10.0) -> List[int]: + """Stop what a failed step of ``domain`` left running (SIGTERM, then SIGKILL).""" + pids = leftover_playbooks(domain, _processes()) + for sig in (signal.SIGTERM, signal.SIGKILL): + for pid in pids: + try: + os.kill(pid, sig) + except OSError: + pass + deadline = time.monotonic() + (grace if sig == signal.SIGTERM else 0) + while time.monotonic() < deadline and any(os.path.exists(f"{PROC_DIR}/{pid}") + for pid in pids): + time.sleep(0.5) + if pids: + emit(CATALOG_MSGS["leftovers_stopped"].format( + domain=domain, pids=", ".join(str(pid) for pid in pids))) + return pids diff --git a/test/system_test/library/functions/catalog_settings.py b/test/system_test/library/functions/catalog_settings.py new file mode 100644 index 0000000000..68798a8b9e --- /dev/null +++ b/test/system_test/library/functions/catalog_settings.py @@ -0,0 +1,165 @@ +# Copyright 2026 Dell Inc. or its subsidiaries. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""The catalog_build theme block of catalog_build_test_config.yml: check and normalize it. + +Pure validation (no command runs, nothing is written), so config_func can load +it with the rest of the theme's settings. The runtime is catalog_func.py. +""" + +import ast +import os +import re +from typing import Any, Dict, List, Tuple + +from ..messages.svt_msgs import CATALOG_MSGS +from ..vars.common_vars import TEST_ROOT +from ..vars.catalog_vars import ( + CATALOG_CLEANUP_BETWEEN, + CATALOG_DEFAULTS, + CATALOG_REPO_CONFIGS, + CATALOG_REPO_POLICIES, + CATALOG_STEP_DOMAINS, +) + +REPO_POLICY_KEY_RE = re.compile(r"^(\d+\.\d+)/(x86_64|aarch64)/([A-Za-z0-9_.-]+)$") +CATALOG_NAME_RE = re.compile(r"^[A-Za-z0-9_./-]+$") +BOOL_KEYS = ("include_aarch64", "caching_policy", "force_rebuild", "continue_on_failure") + + +class CatalogSettingsError(ValueError): + """The catalog_build block is invalid; the message says what to fix.""" + + +def domain_fvt_tags(domain: str) -> List[str]: + """Return FVT_TAGS of test//library/vars/domain_vars.py ([] when unreadable). + + Read with ast, so the suite's own imports are never executed here. + """ + path = os.path.join(TEST_ROOT, domain, "library", "vars", "domain_vars.py") + try: + with open(path, "r", encoding="utf-8") as stream: + tree = ast.parse(stream.read()) + except (OSError, SyntaxError): + return [] + for node in tree.body: + target = node.target if isinstance(node, ast.AnnAssign) else ( + node.targets[0] if isinstance(node, ast.Assign) and len(node.targets) == 1 else None) + if getattr(target, "id", None) == "FVT_TAGS" and node.value is not None: + try: + return [str(tag) for tag in ast.literal_eval(node.value)] + except ValueError: + return [] + return [] + + +def _error(path: str, key: str, problem: str) -> CatalogSettingsError: + return CatalogSettingsError(CATALOG_MSGS["invalid"].format(path=path, key=key, problem=problem)) + + +def _catalogs(value: Any, path: str) -> Any: + if value == "all": + return "all" + if not isinstance(value, list) or not value: + raise _error(path, "catalogs", 'use "all" or a non-empty list of catalog names/numbers') + catalogs = [] + for item in value: + text = str(item).strip() + if not text or not CATALOG_NAME_RE.fullmatch(text) or ".." in text.split("/"): + raise _error(path, "catalogs", f"invalid catalog {item!r}") + catalogs.append(text) + if len(set(catalogs)) != len(catalogs): + raise _error(path, "catalogs", "a catalog is listed twice") + return catalogs + + +def _repo_policies(value: Any, path: str) -> Dict[str, Dict[str, Any]]: + if value in (None, {}): + return {} + if not isinstance(value, dict): + raise _error(path, "repo_policies", "must be a mapping") + policies = {} + for key, entry in value.items(): + where = f"repo_policies.{key}" + if not REPO_POLICY_KEY_RE.fullmatch(str(key)): + raise _error(path, where, 'key must be "//", e.g. ' + '"10.2/x86_64/epel"') + if not isinstance(entry, dict) or not entry or set(entry) - {"policy", "caching"}: + raise _error(path, where, "give policy and/or caching only") + if "policy" in entry and entry["policy"] not in CATALOG_REPO_POLICIES: + raise _error(path, where, f"policy must be one of {', '.join(CATALOG_REPO_POLICIES)}") + if "caching" in entry and not isinstance(entry["caching"], bool): + raise _error(path, where, "caching must be true or false") + policies[str(key)] = dict(entry) + return policies + + +def _steps(value: Any, path: str) -> Tuple[Tuple[str, str], ...]: + if not isinstance(value, list) or not value: + raise _error(path, "steps", "must be a non-empty list of [domain, tag]") + steps = [] + tags = {domain: domain_fvt_tags(domain) for domain in CATALOG_STEP_DOMAINS} + for item in value: + if (not isinstance(item, (list, tuple)) or len(item) != 2 + or not all(isinstance(part, str) for part in item)): + raise _error(path, "steps", f"{item!r} must be [domain, tag]") + domain, tag = item + if domain not in CATALOG_STEP_DOMAINS: + raise _error(path, "steps", f"domain {domain!r} must be one of " + f"{', '.join(CATALOG_STEP_DOMAINS)}") + if tag not in tags[domain]: + raise _error(path, "steps", f"{domain} has no FVT tag {tag!r} " + f"(known: {', '.join(tags[domain]) or 'none found'})") + steps.append((domain, tag)) + return tuple(steps) + + +def load_catalog_settings(block: Any, path: str) -> Dict[str, Any]: + """Return the checked catalog_build block with every default filled in. + + Raises: + CatalogSettingsError: An unknown key or an invalid value. + """ + block = block or {} + if not isinstance(block, dict): + raise _error(path, "catalog_build", "must be a mapping") + unknown = sorted(set(block) - set(CATALOG_DEFAULTS)) + if unknown: + raise _error(path, ", ".join(unknown), + f"unknown key (known: {', '.join(CATALOG_DEFAULTS)})") + settings = {**CATALOG_DEFAULTS, **block} + for key in BOOL_KEYS: + if not isinstance(settings[key], bool): + raise _error(path, key, "must be true or false (without quotes)") + if settings["repo_config"] not in CATALOG_REPO_CONFIGS: + raise _error(path, "repo_config", f"must be one of {', '.join(CATALOG_REPO_CONFIGS)}") + if settings["cleanup_between"] not in CATALOG_CLEANUP_BETWEEN: + raise _error(path, "cleanup_between", + f"must be one of {', '.join(CATALOG_CLEANUP_BETWEEN)}") + settings["catalogs"] = _catalogs(settings["catalogs"], path) + settings["repo_policies"] = _repo_policies(settings["repo_policies"], path) + steps = settings["steps"] + settings["steps"] = steps if isinstance(steps, tuple) else _steps(steps, path) + return settings + + +def describe_settings(settings: Dict[str, Any]) -> str: + """Return the global sweep settings as one line (report and logs).""" + overrides = ", ".join(f"{key}={value}" for key, value in settings["repo_policies"].items()) + return (f"repo_config={settings['repo_config']}, caching_policy={settings['caching_policy']}" + f"{f', repo_policies: {overrides}' if overrides else ''}, " + f"force_rebuild={settings['force_rebuild']}, " + f"include_aarch64={settings['include_aarch64']}, " + f"continue_on_failure={settings['continue_on_failure']}, " + f"cleanup_between={settings['cleanup_between']}") diff --git a/test/system_test/library/functions/config_func.py b/test/system_test/library/functions/config_func.py index c923891162..84e14772bd 100644 --- a/test/system_test/library/functions/config_func.py +++ b/test/system_test/library/functions/config_func.py @@ -32,6 +32,7 @@ from dataset_generator.dataset_edit import Change, dump_yaml, load_yaml, set_value, split_header +from .catalog_settings import CatalogSettingsError, load_catalog_settings from .fvt_func import domain_dir from .svt_report_func import fvt_report_settings from ..messages.svt_msgs import CONFIG_MSGS @@ -134,7 +135,17 @@ def load_svt_config(theme: str) -> Dict[str, Any]: return {"theme": theme, "path": path, "cleanup": cleanup, "nfs": _load_nfs_block(theme, nfs_block, path), "report": _load_report_block(data.get("report") or {}, path), - "fvt_settings": _load_theme_block(theme, data.get(theme) or {}, path)} + **_load_theme_settings(theme, data.get(theme), path)} + + +def _load_theme_settings(theme: str, block: Any, path: str) -> Dict[str, Any]: + """Return the checked theme block: FVT settings, plus catalog settings for catalog themes.""" + if not THEMES[theme].get("catalog_plan"): + return {"fvt_settings": _load_theme_block(theme, block or {}, path)} + try: + return {"fvt_settings": {}, "catalog": load_catalog_settings(block, path)} + except CatalogSettingsError as exc: + raise ConfigError(str(exc)) from exc def _check_setting(kind: str, value: Any, where: str, path: str) -> Any: diff --git a/test/system_test/library/functions/fvt_func.py b/test/system_test/library/functions/fvt_func.py index 530357dc99..ccbe3162db 100644 --- a/test/system_test/library/functions/fvt_func.py +++ b/test/system_test/library/functions/fvt_func.py @@ -92,9 +92,12 @@ def fvt_python(domain: str, override: Optional[str] = None) -> str: def describe_step(step: FvtStep) -> str: - """Return the runner command of a step as text, e.g. ``fvt_build_stream execute exec``.""" + """Return the runner command of a step as text, e.g. ``fvt_build_stream execute exec``. + + An SVT action (domain ``svt``, catalog_build) is ``svt ``. + """ domain, arguments = step - return " ".join([f"fvt_{domain}", *arguments]) + return " ".join([domain if domain == "svt" else f"fvt_{domain}", *arguments]) def fvt_report_json(domain: str) -> str: diff --git a/test/system_test/library/functions/readiness_func.py b/test/system_test/library/functions/readiness_func.py index 44670624b1..4efd172598 100644 --- a/test/system_test/library/functions/readiness_func.py +++ b/test/system_test/library/functions/readiness_func.py @@ -27,7 +27,8 @@ from .config_func import theme_config_path from .fvt_func import domain_dir from .omnia_func import credential_path, missing_credentials, omnia_env, setup_problems -from ..messages.svt_msgs import SETUP_MSGS +from ..messages.svt_msgs import CATALOG_MSGS, SETUP_MSGS +from ..vars.catalog_vars import CATALOG_PLAN_FILE, load_plan from ..vars.common_vars import ( DEFAULT_THEME, FVT_REQUIREMENTS, @@ -36,6 +37,7 @@ OMNIA_AUTO_WHEEL, OMNIA_VENV_KEY, REQUIREMENTS_MARKER, + RUN_STATE, SETUP_RECORD, SETUP_REPORT_DIR, ) @@ -114,6 +116,18 @@ def _install_outdated(installed: Dict[str, Any], configured: Dict[str, Any]) -> return installed.get("inputs_sha256") != configured.get("inputs_sha256") +def catalog_plan_problem(path: str = CATALOG_PLAN_FILE, run_file: str = RUN_STATE) -> str: + """Return why the catalog plan does not belong to the current run ("" when it does).""" + run_id = read_json(run_file).get("run_id", "") + plan = load_plan(path) + if not plan: + return CATALOG_MSGS["plan_missing"].format(run_id=run_id, path=path) + if plan.get("run_id") != run_id: + return CATALOG_MSGS["plan_other_run"].format(path=path, plan_run=plan.get("run_id"), + run_id=run_id) + return "" + + def readiness_checks(theme: str) -> List[Check]: """Return (label, ok, how to fix) for everything ./run_validation.sh needs.""" record = load_setup_record() @@ -135,6 +149,9 @@ def readiness_checks(theme: str) -> List[Check]: else: detail = "" checks.append(("Installed after the last configure", not detail, detail)) + if THEMES[theme].get("catalog_plan"): + detail = catalog_plan_problem() + checks.append(("Catalog plan of this run (configure)", not detail, detail)) if setup: checks.append(("Test requirements in the Omnia venv", False, SETUP_MSGS["install_first"])) checks.append(("Domain credentials", False, SETUP_MSGS["creds_after_install"])) diff --git a/test/system_test/library/functions/setup_func.py b/test/system_test/library/functions/setup_func.py index 3b384b99f6..07086c4206 100644 --- a/test/system_test/library/functions/setup_func.py +++ b/test/system_test/library/functions/setup_func.py @@ -253,7 +253,7 @@ def _catalog_selection() -> str: return "" if node is None else str(node).strip() -def _bundled_catalogs() -> Tuple[Dict[str, str], str]: +def bundled_catalogs() -> Tuple[Dict[str, str], str]: """Return ({number: name} of ./omnia.sh --list-catalogs, error).""" try: listed = subprocess.run( # nosec B603 # fixed argument list, no shell @@ -273,7 +273,7 @@ def resolve_catalog(selection: str) -> Tuple[str, str]: The name is what --select-catalog gets, so the choice does not depend on the order of the list at install time. """ - catalogs, error = _bundled_catalogs() + catalogs, error = bundled_catalogs() if error: return "", error wanted = selection @@ -306,7 +306,7 @@ def _configure_catalog(emit: Emit, rows: List[Dict[str, Any]]) -> Tuple[str, str return name, "" -def _select_catalog(name: str, emit: Emit, rows: List[Dict[str, Any]]) -> str: +def select_catalog(name: str, emit: Emit, rows: List[Dict[str, Any]]) -> str: """Run ./omnia.sh --select-catalog ``name`` (replaces the active catalog); return an error.""" emit(SETUP_MSGS["catalog_select"].format(name=name)) result = tee_command(["bash", OMNIA_SH, OMNIA_SELECT_CATALOG_ARG, name], cwd=OMNIA_SH_DIR, @@ -435,7 +435,7 @@ def install(config: Dict[str, Any], emit: Emit, force: bool = False, return _failed(SETUP_MSGS["setup_incomplete"] + _problems(problems), rows=rows) catalog = record["configure"].get("catalog", "") if catalog: - error = _select_catalog(catalog, emit, rows) + error = select_catalog(catalog, emit, rows) if error: return _failed(error, rows=rows) venv = omnia_venv_path() diff --git a/test/system_test/library/functions/test_func.py b/test/system_test/library/functions/test_func.py index bb94b2045a..b5013c6fc9 100644 --- a/test/system_test/library/functions/test_func.py +++ b/test/system_test/library/functions/test_func.py @@ -27,10 +27,10 @@ import pytest from omnia_auto import TestLogger -from . import run_func, svt_report_func +from . import catalog_func, run_func, svt_report_func from .config_func import load_svt_config from .fvt_func import describe_step, run_fvt_step, step_info -from ..messages.svt_msgs import SETUP_MSGS, TEST_ASSERT_MSGS, TEST_LOG_MSGS +from ..messages.svt_msgs import CATALOG_MSGS, SETUP_MSGS, TEST_ASSERT_MSGS, TEST_LOG_MSGS from ..vars.common_vars import REPORT_SECTION_ENV from ..vars.test_case_vars import TEST_CASES @@ -56,19 +56,34 @@ def run_svt_test(theme: str, name: str) -> None: tc = TEST_CASES[f"{theme}_{name}"] label = describe_step(tc["step"]) blocked = _FAILED.get(tc["stage"]) - if blocked: + # "always": a test that runs even after an earlier failure of its stage (cleanup). + if blocked and not tc.get("always"): reason = TEST_LOG_MSGS["blocked"].format(test=blocked) _report(theme, svt_report_func.make_item({ "id": tc["id"], "name": tc["title"], "status": "Skipped", "details": reason, "command": label, "fvt": step_info(tc["step"])})) pytest.skip(reason) logger = TestLogger(tc["title"], tc["id"]) - result = run_fvt_step(tc["step"]) + # "action": an SVT action instead of an FVT step (catalog_build: select or skip a catalog). + result = catalog_func.run_action(tc) if tc.get("action") else run_fvt_step(tc["step"]) + if result.get("skipped"): + _report(theme, svt_report_func.make_item({ + "id": tc["id"], "name": tc["title"], "status": "Skipped", + "details": result["skipped"], "fvt": result["fvt"]})) + pytest.skip(result["skipped"]) + note = f"$ {result['command']}" + # "stop_leftovers": after a failed exec, stop the playbook workers it left running. + if not result["success"] and tc.get("stop_leftovers"): + stopped = catalog_func.stop_leftovers(tc["step"][0], print) + if stopped: + note += "\n" + CATALOG_MSGS["leftovers_stopped"].format( + domain=tc["step"][0], pids=", ".join(str(pid) for pid in stopped)) _report(theme, svt_report_func.make_item({ "id": tc["id"], "name": tc["title"], "status": "Passed" if result["success"] else "Failed", "duration": result["duration"], "command": label, "output": result["output"], - "results": result["results"], "fvt": result["fvt"], "details": f"$ {result['command']}", - "error": "" if result["success"] else f"{label} failed (rc={result['rc']})"})) + "results": result["results"], "fvt": result["fvt"], "details": note, + "error": "" if result["success"] else (result.get("error") + or f"{label} failed (rc={result['rc']})")})) fields = [("FVT step", label), ("rc", result["rc"]), ("FVT results", len(result["results"]))] if result["success"]: diff --git a/test/system_test/library/messages/svt_msgs.py b/test/system_test/library/messages/svt_msgs.py index 0e63da7b09..0d5cbe5bf9 100644 --- a/test/system_test/library/messages/svt_msgs.py +++ b/test/system_test/library/messages/svt_msgs.py @@ -226,6 +226,32 @@ "retry_done": "--retry: every stage of run {run_id} already passed; only summary runs", } +CATALOG_MSGS = { + "invalid": "{path}: catalog_build.{key}: {problem}", + "unknown_catalog": "catalog {selection!r} is not in ./omnia.sh --list-catalogs. " + "HOW TO FIX: use one of the names or numbers it lists", + "catalog_file_missing": "catalog file {path} not found", + "skip_aarch64": "skipped: contains aarch64 ({layers}); include_aarch64 is false", + "aarch64_host_missing": "include_aarch64 is true but " + "image_build_manager.image_build_config.aarch64_inventory_host_ip " + "is empty in user_inputs.yml. HOW TO FIX: set it, then " + "./setup_env.sh --theme catalog_build", + "plan_header": "catalog plan ({count} catalog(s), {run} to run) -> {path}:", + "plan_line": " {stage:<12} {mark:<5} {catalog}{reason}", + "steps_header": "steps per catalog (exec, then verify): {steps}", + "settings": "settings: {settings}", + "plan_missing": "no catalog plan for run {run_id} ({path}). HOW TO FIX: " + "./setup_env.sh --theme catalog_build --configure", + "plan_other_run": "the catalog plan {path} belongs to run {plan_run}, not {run_id}. " + "HOW TO FIX: ./setup_env.sh --theme catalog_build --configure", + "inputs_written": "runtime inputs: {details}", + "override_applied": "override: {source} -> {target}", + "repo_missing": "repo_policies: {key} is not a repository of {path}", + "no_runtime_input": "{path} is missing: run ./setup_env.sh --theme catalog_build first", + "leftovers_stopped": "stopped what the failed {domain} step left running " + "(orphaned ansible-playbook and children): pids {pids}", +} + LOCK_MSGS = { "held": "another SVT step is running on this OIM: {step} (pid {pid}, run {run_id}, " "heartbeat {heartbeat}). HOW TO FIX: wait for it, or stop it; a lock whose " diff --git a/test/system_test/library/vars/catalog_vars.py b/test/system_test/library/vars/catalog_vars.py new file mode 100644 index 0000000000..b95e76d1ce --- /dev/null +++ b/test/system_test/library/vars/catalog_vars.py @@ -0,0 +1,164 @@ +# Copyright 2026 Dell Inc. or its subsidiaries. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""catalog_build theme data: settings defaults and the stages and tests of its plan. + +``./setup_env.sh --theme catalog_build --configure`` resolves the catalog list +of catalog_build_test_config.yml (``./omnia.sh --list-catalogs``) and writes +the plan to CATALOG_PLAN_FILE. The theme is then data like any other theme: +one stage per catalog (catalog_01, catalog_02 ...) with its stage_info, and +one TEST_CASES entry per step, generated here from the plan. A changed +catalog_build_test_config.yml needs a new --configure (the run guard), so the +plan always belongs to the current run. + +Each catalog stage runs, in order: ``select`` (catalog_overrides/, the global +settings, ./omnia.sh --select-catalog: an SVT action, not an FVT step), then +`` exec`` and `` verify`` of every configured step, then the +cleanup_between steps (they run even after a failure). +""" + +import json +import os +from typing import Any, Dict, List, Tuple + +from .common_vars import REPORT_ROOT + +FvtStep = Tuple[str, Tuple[str, ...]] + +CATALOG_THEME = "catalog_build" +CATALOG_PLAN_FILE = os.path.join(REPORT_ROOT, "catalog_build_plan.json") +CATALOG_STAGE_PREFIX = "catalog_" +# Test IDs: SVT_CATALOG_BUILD_; order: base + catalog * 100 + step. +CATALOG_ID_FORMAT = "SVT_CATALOG_BUILD_{index:02d}{step:02d}" +CATALOG_ORDER_BASE = 2000 + +# Domains whose FVT steps a catalog runs, and the playbook each one runs (after a +# failed exec its orphaned ansible-playbook workers are stopped, catalog_func). +CATALOG_STEP_DOMAINS = ("repo_manager", "image_build_manager") +CATALOG_STEP_PLAYBOOKS: Dict[str, str] = { + "repo_manager": "repo_manager.yml", + "image_build_manager": "image_build_manager.yml", +} +CATALOG_DEFAULT_STEPS: Tuple[Tuple[str, str], ...] = ( + ("repo_manager", "precheck"), + ("repo_manager", "prepare"), + ("repo_manager", "execute"), + ("repo_manager", "status"), + ("image_build_manager", "precheck"), + ("image_build_manager", "validate"), + ("image_build_manager", "prepare"), + ("image_build_manager", "build"), +) +CATALOG_REPO_CONFIGS = ("always", "partial") +CATALOG_REPO_POLICIES = ("always", "partial", "never") +# cleanup_between -> FVT steps after every catalog. repo_manager has no whole-catalog +# repository cleanup (cleanup_repos takes one exact repository), so "repos" removes the +# repo_manager deployment; the next catalog prepares it again. +CATALOG_CLEANUP_BETWEEN: Dict[str, Tuple[FvtStep, ...]] = { + "none": (), + "images": (("image_build_manager", ("cleanup_images", "test")),), + "repos": (("repo_manager", ("cleanup", "test", "--marker", "destructive")),), + "full": (("image_build_manager", ("cleanup_images", "test")), + ("image_build_manager", ("cleanup", "test")), + ("repo_manager", ("cleanup", "test", "--marker", "destructive"))), +} +# Every key of the catalog_build block of catalog_build_test_config.yml, with its default. +CATALOG_DEFAULTS: Dict[str, Any] = { + "catalogs": "all", + "include_aarch64": False, + "repo_config": "partial", + "caching_policy": True, + "repo_policies": {}, + "force_rebuild": True, + "continue_on_failure": True, + "cleanup_between": "none", + "steps": CATALOG_DEFAULT_STEPS, +} +# SVT actions of a catalog stage (catalog_func.run_action). +ACTION_SELECT = "catalog_select" +ACTION_SKIP = "catalog_skip" + + +def load_plan(path: str = CATALOG_PLAN_FILE) -> Dict[str, Any]: + """Return the plan written by --configure ({} when none or unreadable).""" + try: + with open(path, "r", encoding="utf-8") as stream: + plan = json.load(stream) + except (OSError, json.JSONDecodeError): + return {} + return plan if isinstance(plan, dict) and isinstance(plan.get("entries"), list) else {} + + +def catalog_stages(plan: Dict[str, Any]) -> Tuple[str, ...]: + """Return the stage of every catalog of the plan, in run order.""" + return tuple(entry["stage"] for entry in plan.get("entries") or []) + + +def catalog_soft_stages(plan: Dict[str, Any]) -> Tuple[str, ...]: + """Return the catalog stages whose failure does not stop the next (continue_on_failure).""" + settings = plan.get("settings") or {} + return catalog_stages(plan) if settings.get("continue_on_failure", True) else () + + +def catalog_stage_info(plan: Dict[str, Any]) -> Dict[str, Tuple[str, str]]: + """Return (title, description) of every catalog stage (report and pipeline).""" + info = {} + for entry in plan.get("entries") or []: + title = f"Catalog {entry['index']}: {entry['catalog']}" + description = entry["reason"] or ( + "./omnia.sh --select-catalog, then the repo_manager and image_build_manager FVT " + "steps (exec, verify)") + info[entry["stage"]] = (title, description) + return info + + +def _case(entry: Dict[str, Any], number: int, name: str, title: str, + **fields: Any) -> Tuple[str, Dict[str, Any]]: + key = f"{CATALOG_THEME}_{entry['stage'][len(CATALOG_STAGE_PREFIX):]}_{name}" + return key, {"id": CATALOG_ID_FORMAT.format(index=entry["index"], step=number), + "order": CATALOG_ORDER_BASE + entry["index"] * 100 + number, + "stage": entry["stage"], "title": f"{entry['catalog']}: {title}", **fields} + + +def _catalog_cases(entry: Dict[str, Any], settings: Dict[str, Any]) -> List[Tuple[str, Dict]]: + if not entry["run"]: + return [_case(entry, 0, "skip", "skipped", action=ACTION_SKIP, reason=entry["reason"], + step=("svt", ("skip", entry["catalog"])))] + cases = [_case(entry, 0, "select", + "select (catalog_overrides, global settings, ./omnia.sh --select-catalog)", + action=ACTION_SELECT, catalog=entry["catalog"], + step=("svt", ("select", entry["catalog"])))] + number = 0 + for domain, tag in settings.get("steps") or CATALOG_DEFAULT_STEPS: + for action in ("exec", "verify"): + number += 1 + cases.append(_case(entry, number, f"{domain}_{tag}_{action}", + f"{domain} {tag} {action}", step=(domain, (tag, action)), + stop_leftovers=action == "exec")) + for domain, arguments in CATALOG_CLEANUP_BETWEEN.get(settings.get("cleanup_between"), ()): + number += 1 + cases.append(_case(entry, number, f"cleanup_{domain}_{arguments[0]}", + f"cleanup_between: {domain} {arguments[0]}", + step=(domain, tuple(arguments)), always=True)) + return cases + + +def catalog_test_cases(plan: Dict[str, Any]) -> Dict[str, Dict[str, Any]]: + """Return the TEST_CASES entries of every catalog of the plan.""" + settings = plan.get("settings") or {} + return dict(case for entry in plan.get("entries") or [] + for case in _catalog_cases(entry, settings)) + + +CATALOG_PLAN = load_plan() diff --git a/test/system_test/library/vars/test_case_vars.py b/test/system_test/library/vars/test_case_vars.py index ed62dfa38e..632a39768a 100644 --- a/test/system_test/library/vars/test_case_vars.py +++ b/test/system_test/library/vars/test_case_vars.py @@ -20,8 +20,13 @@ ``test//_run.py fvt_``: ``exec`` runs the playbook, ``verify`` runs its verification tests. Within a stage, a test is skipped once an earlier test of the stage failed. + +catalog_build (order 2000+): generated from the plan of the current run +(catalog_vars.catalog_test_cases), one stage per catalog. """ +from .catalog_vars import CATALOG_PLAN, catalog_test_cases + _BS = "build_stream" TEST_CASES = { @@ -102,3 +107,5 @@ "step": (_BS, ("execute", "verify", "--suite", "auto_cleanup")), }, } +# ------------------------------------------------------ theme catalog_build +TEST_CASES.update(catalog_test_cases(CATALOG_PLAN)) diff --git a/test/system_test/library/vars/theme_vars.py b/test/system_test/library/vars/theme_vars.py index 1fc58be73d..92e5830625 100644 --- a/test/system_test/library/vars/theme_vars.py +++ b/test/system_test/library/vars/theme_vars.py @@ -24,6 +24,8 @@ from typing import Dict, Tuple +from .catalog_vars import CATALOG_PLAN, catalog_soft_stages, catalog_stage_info, catalog_stages + FvtStep = Tuple[str, Tuple[str, ...]] # Previous installation, removed in this order before the setup: BuildStream, @@ -175,6 +177,40 @@ "debug_mode": ("build_stream", "", "debug_mode", "bool"), }, }, + "catalog_build": { + "title": "Catalog Build", + "description": "Catalog build: for every catalog, ./omnia.sh --select-catalog, then " + "the repo_manager and image_build_manager FVT steps", + "profile": "catalog_build", + # Same cleanup as build_stream: an earlier BuildStream, telemetry or orchestrator + # deployment must not block omnia.sh --cleanup --all. + "cleanup_steps": CLEANUP_STEPS, + "nfs_shares": {}, + # orchestrator, telemetry and build_stream: their requirements are installed only + # for their cleanup FVT; they get no stage and no credentials. + "fvt_domains": ("repo_manager", "image_build_manager", "orchestrator", + "telemetry", "build_stream"), + "credential_domains": ("repo_manager", "image_build_manager"), + "creds_suite": "build_stream", + "cleanup_requires": { + ("build_stream", ("cleanup", "test")): ( + "/etc/containers/systemd/omnia_build_stream.container", + "/etc/containers/systemd/omnia_postgres.container", + "{domain_data}/postgres", + "{domain_data}/build_stream_root", + ), + }, + "cleanup_when_deployed": ("telemetry",), + # One stage per catalog of the plan --configure wrote (catalog_vars.py); the + # stage titles name the catalog. With continue_on_failure every catalog stage is + # soft: a failed catalog does not stop the next one, the summary still fails. + "stages": catalog_stages(CATALOG_PLAN), + "soft_stages": catalog_soft_stages(CATALOG_PLAN), + "stage_info": catalog_stage_info(CATALOG_PLAN), + # The theme block is the catalog settings (catalog_settings.py), not FVT settings. + "catalog_plan": True, + "fvt_settings": {}, + }, } # Sections every theme has (title, description); the theme stages come from # THEMES[theme]["stage_info"]. In the report, cleanup and install (run by diff --git a/test/system_test/ut/test_catalog_build.py b/test/system_test/ut/test_catalog_build.py new file mode 100644 index 0000000000..3ff708e7de --- /dev/null +++ b/test/system_test/ut/test_catalog_build.py @@ -0,0 +1,478 @@ +# Copyright 2026 Dell Inc. or its subsidiaries. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Unit tests of the catalog_build theme (no command runs, nothing outside tmp is written). + +Run from test/system_test: .venv/bin/python -m pytest ut -q -p no:cacheprovider -o addopts="" +""" + +# pylint: disable=protected-access,import-error,wrong-import-position + +import json +import os +import sys +from unittest import mock + +import pytest +import yaml + +MODULE_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +sys.path.insert(0, MODULE_ROOT) + +from library.functions import catalog_func, readiness_func, svt_report_func, test_func # noqa: E402 +from library.functions.catalog_settings import ( # noqa: E402 + CatalogSettingsError, + domain_fvt_tags, + load_catalog_settings, +) +from library.functions.config_func import load_svt_config # noqa: E402 +from library.functions.fvt_func import describe_step # noqa: E402 +from library.vars import catalog_vars # noqa: E402 +from library.vars.catalog_vars import CATALOG_DEFAULT_STEPS # noqa: E402 +from library.vars.theme_vars import THEMES # noqa: E402 + +PATH = "catalog_build_test_config.yml" +LISTED = {"1": "default", "2": "rhel/10.2/slurm_x86_64_no_vast.json", + "3": "rhel/10.2/slurm_aarch64.json"} + + +def _settings(**block): + return load_catalog_settings(block, PATH) + + +# ---------------------------------------------------------------- settings +def test_defaults_are_filled(): + """An empty block gives every default (force_rebuild true).""" + settings = _settings() + assert settings["catalogs"] == "all" + assert settings["repo_config"] == "partial" + assert settings["caching_policy"] is True + assert settings["force_rebuild"] is True + assert settings["include_aarch64"] is False + assert settings["continue_on_failure"] is True + assert settings["cleanup_between"] == "none" + assert settings["steps"] == CATALOG_DEFAULT_STEPS + + +@pytest.mark.parametrize("block, fragment", [ + ({"unknown": 1}, "unknown key"), + ({"repo_config": "never"}, "repo_config"), + ({"force_rebuild": "true"}, "force_rebuild"), + ({"cleanup_between": "all"}, "cleanup_between"), + ({"catalogs": []}, "catalogs"), + ({"catalogs": ["a.json", "a.json"]}, "twice"), + ({"catalogs": ["../x.json"]}, "invalid catalog"), + ({"repo_policies": {"epel": {"policy": "always"}}}, "//"), + ({"repo_policies": {"10.2/x86_64/epel": {"policy": "sometimes"}}}, "policy"), + ({"repo_policies": {"10.2/x86_64/epel": {"caching": "no"}}}, "caching"), + ({"steps": [["orchestrator", "deploy"]]}, "domain"), + ({"steps": [["repo_manager", "no_such_tag"]]}, "no FVT tag"), +]) +def test_invalid_values_are_rejected(block, fragment): + """Every invalid value names the key to fix.""" + with pytest.raises(CatalogSettingsError, match=fragment): + load_catalog_settings(block, PATH) + + +def test_step_tags_come_from_the_suites(): + """The FVT tags are read from each suite's domain_vars.py.""" + assert {"precheck", "prepare", "execute", "status"} <= set(domain_fvt_tags("repo_manager")) + assert {"precheck", "validate", "prepare", "build"} <= set( + domain_fvt_tags("image_build_manager")) + assert domain_fvt_tags("no_such_suite") == [] + + +def test_shipped_theme_config_loads(): + """catalog_build_test_config.yml is valid and the theme mirrors build_stream's entry.""" + config = load_svt_config("catalog_build") + assert config["catalog"]["catalogs"] == "all" + assert config["catalog"]["force_rebuild"] is True + assert config["fvt_settings"] == {} + theme, base = THEMES["catalog_build"], THEMES["build_stream"] + assert theme["credential_domains"] == ("repo_manager", "image_build_manager") + for key in ("cleanup_steps", "fvt_domains", "creds_suite", "cleanup_requires", + "cleanup_when_deployed"): + assert theme[key] == base[key], key + + +def test_build_stream_theme_is_unchanged(): + """build_stream still loads its FVT settings and has no catalog block.""" + config = load_svt_config("build_stream") + assert "catalog" not in config + assert "catalog" in config["fvt_settings"] + assert not THEMES["build_stream"].get("catalog_plan") + + +# -------------------------------------------------------------------- plan +def test_aarch64_detection_on_bundled_catalogs(): + """aarch64 is read from the functional layers of the real catalog files.""" + assert catalog_func.aarch64_layers( + catalog_func.catalog_file("rhel/10.2/slurm_x86_64_no_vast.json")) == [] + assert catalog_func.aarch64_layers(catalog_func.catalog_file("rhel/10.2/slurm_aarch64.json")) + assert catalog_func.aarch64_layers(catalog_func.catalog_file("default")) + + +def test_plan_all_skips_aarch64_by_default(): + """'all' follows the --list-catalogs order; aarch64 catalogs are skipped with a reason.""" + with mock.patch.object(catalog_func, "bundled_catalogs", return_value=(LISTED, "")): + entries = catalog_func.plan(_settings()) + assert [entry["catalog"] for entry in entries] == list(LISTED.values()) + assert [entry["run"] for entry in entries] == [False, True, False] + assert [entry["stage"] for entry in entries] == ["catalog_01", "catalog_02", "catalog_03"] + assert "aarch64" in entries[0]["reason"] + + +def test_plan_aarch64_needs_the_build_host(): + """include_aarch64 without aarch64_inventory_host_ip stops the plan.""" + with mock.patch.object(catalog_func, "_aarch64_host", return_value=""): + with pytest.raises(ValueError, match="aarch64_inventory_host_ip"): + catalog_func.plan(_settings(include_aarch64=True)) + + +def test_plan_list_keeps_the_given_order(): + """An explicit list runs in its own order; an unknown catalog is reported.""" + names = {"b": "rhel/10.2/slurm_aarch64.json", "a": "rhel/10.2/slurm_x86_64_no_vast.json"} + with mock.patch.object(catalog_func, "resolve_catalog", + side_effect=lambda selection: (names.get(selection, ""), + "" if selection in names else "x")): + entries = catalog_func.plan(_settings(catalogs=["a", "b"])) + assert [entry["catalog"] for entry in entries] == [names["a"], names["b"]] + with pytest.raises(ValueError, match="not in ./omnia.sh --list-catalogs"): + catalog_func.plan(_settings(catalogs=["zzz"])) + + +@pytest.fixture(name="plan_file") +def _plan_file(tmp_path): + """A plan written by write_plan for run svt-1 (catalog 1 runs, catalog 2 is aarch64).""" + path = str(tmp_path / "catalog_build_plan.json") + with mock.patch.object(catalog_func, "bundled_catalogs", return_value=( + {"1": "rhel/10.2/slurm_x86_64_no_vast.json", "2": "rhel/10.2/slurm_aarch64.json"}, + "")): + rows, error = catalog_func.write_plan(_settings(cleanup_between="images"), "svt-1", + lambda _: None, path=path) + assert not error and rows[0]["status"] == "PASSED" + return path + + +def test_write_plan_stores_the_run(plan_file): + """The plan file holds the run id, the JSON-ready settings and the entries.""" + with open(plan_file, encoding="utf-8") as stream: + data = json.load(stream) + assert data["run_id"] == "svt-1" + assert data["settings"]["steps"][0] == ["repo_manager", "precheck"] + assert [entry["run"] for entry in data["entries"]] == [True, False] + + +def test_write_plan_failure_is_a_failed_row(): + """An unknown catalog fails the plan with a FAILED row and no file.""" + with mock.patch.object(catalog_func, "resolve_catalog", return_value=("", "unknown")): + rows, error = catalog_func.write_plan(_settings(catalogs=["zzz"]), "svt-1", + lambda _: None, path="/nonexistent/plan.json") + assert "zzz" in error and rows[0]["status"] == "FAILED" + + +def test_plan_becomes_theme_data(plan_file): + """Stages, soft stages, stage titles and TEST_CASES come from the plan.""" + plan = catalog_vars.load_plan(plan_file) + assert catalog_vars.catalog_stages(plan) == ("catalog_01", "catalog_02") + assert catalog_vars.catalog_soft_stages(plan) == ("catalog_01", "catalog_02") + info = catalog_vars.catalog_stage_info(plan) + assert info["catalog_01"][0] == "Catalog 1: rhel/10.2/slurm_x86_64_no_vast.json" + assert "aarch64" in info["catalog_02"][1] + cases = catalog_vars.catalog_test_cases(plan) + first = [key for key, case in cases.items() if case["stage"] == "catalog_01"] + assert first[0] == "catalog_build_01_select" + assert first[1:3] == ["catalog_build_01_repo_manager_precheck_exec", + "catalog_build_01_repo_manager_precheck_verify"] + assert len(first) == 1 + 2 * len(CATALOG_DEFAULT_STEPS) + 1 + assert cases["catalog_build_01_select"]["id"] == "SVT_CATALOG_BUILD_0100" + assert cases["catalog_build_01_repo_manager_precheck_exec"]["step"] == ( + "repo_manager", ("precheck", "exec")) + assert cases["catalog_build_01_repo_manager_precheck_exec"]["stop_leftovers"] is True + assert cases["catalog_build_01_repo_manager_precheck_verify"]["stop_leftovers"] is False + cleanup = cases["catalog_build_01_cleanup_image_build_manager_cleanup_images"] + assert cleanup["always"] is True + assert list(key for key, case in cases.items() if case["stage"] == "catalog_02") == [ + "catalog_build_02_skip"] + ids = [case["id"] for case in cases.values()] + orders = [case["order"] for case in cases.values()] + assert len(set(ids)) == len(ids) and orders == sorted(orders) + + +def test_continue_on_failure_false_has_no_soft_stage(plan_file): + """continue_on_failure false: every catalog stage is hard (stops the run).""" + plan = catalog_vars.load_plan(plan_file) + plan["settings"]["continue_on_failure"] = False + assert catalog_vars.catalog_soft_stages(plan) == () + + +def test_no_plan_is_no_stage(tmp_path): + """Without a plan the theme has no catalog stage and no test.""" + plan = catalog_vars.load_plan(str(tmp_path / "missing.json")) + assert plan == {} + assert not catalog_vars.catalog_stages(plan) and not catalog_vars.catalog_test_cases(plan) + + +def test_readiness_needs_the_plan_of_this_run(plan_file, tmp_path): + """The plan must exist and belong to the current run.""" + run_file = tmp_path / "current_run.json" + run_file.write_text(json.dumps({"run_id": "svt-1"})) + assert readiness_func.catalog_plan_problem(plan_file, str(run_file)) == "" + run_file.write_text(json.dumps({"run_id": "svt-2"})) + assert "belongs to run svt-1" in readiness_func.catalog_plan_problem(plan_file, str(run_file)) + assert "no catalog plan" in readiness_func.catalog_plan_problem( + str(tmp_path / "none.json"), str(run_file)) + + +# ------------------------------------------------------------ input writes +REPO_YAML = """--- +repo_config: "partial" # Options: always | partial +caching_policy: true +repositories: + "10.2": + x86_64: + baseos: {} + epel: + url: "https://mirror/epel" + user_repos: + slurm_custom: + url: "https://mirror/slurm" +""" +IMAGE_YAML = """--- +build_image: + max_parallel: 0 + force_rebuild: false +functional_groups_source: "config" +""" + + +@pytest.fixture(name="runtime") +def _runtime(tmp_path): + """Two runtime input folders and _runtime_dir pointed at them.""" + for domain, name, text in (("repo_manager", "repo_manager_config.yml", REPO_YAML), + ("image_build_manager", "image_build_config.yml", IMAGE_YAML)): + folder = tmp_path / "runtime" / domain + folder.mkdir(parents=True) + (folder / name).write_text(text, encoding="utf-8") + with mock.patch.object(catalog_func, "_runtime_dir", + side_effect=lambda domain: str(tmp_path / "runtime" / domain)): + yield tmp_path + + +def _read(runtime, domain, name): + return yaml.safe_load((runtime / "runtime" / domain / name).read_text(encoding="utf-8")) + + +def test_write_settings_updates_both_inputs(runtime): + """Policy, caching, per-repo policies, force_rebuild and catalog mode are written.""" + changes = catalog_func.write_settings(_settings( + repo_config="always", caching_policy=False, force_rebuild=True, + repo_policies={"10.2/x86_64/epel": {"policy": "never"}, + "10.2/x86_64/slurm_custom": {"caching": False}, + "10.2/x86_64/baseos": {"policy": "partial"}})) + assert changes + repo = _read(runtime, "repo_manager", "repo_manager_config.yml") + assert repo["repo_config"] == "always" and repo["caching_policy"] is False + x86 = repo["repositories"]["10.2"]["x86_64"] + assert x86["epel"]["policy"] == "never" + assert x86["user_repos"]["slurm_custom"]["caching"] is False + assert x86["baseos"]["policy"] == "partial" + image = _read(runtime, "image_build_manager", "image_build_config.yml") + assert image["build_image"]["force_rebuild"] is True + assert image["functional_groups_source"] == "catalog" + text = (runtime / "runtime" / "repo_manager" / "repo_manager_config.yml").read_text() + assert "# Options: always | partial" in text + + +@pytest.mark.usefixtures("runtime") +def test_write_settings_is_idempotent(): + """A second write with the same settings changes nothing.""" + settings = _settings(force_rebuild=True) + catalog_func.write_settings(settings) + assert not catalog_func.write_settings(settings) + + +@pytest.mark.usefixtures("runtime") +def test_unknown_repository_is_reported(): + """A repo_policies key that is not in repo_manager_config.yml fails clearly.""" + with pytest.raises(ValueError, match="not a repository"): + catalog_func.write_settings(_settings( + repo_policies={"10.2/x86_64/nope": {"policy": "always"}})) + + +def test_overrides_apply_and_restore(runtime, tmp_path): + """An override replaces one catalog's input; the next catalog gets the shared file back.""" + overrides = tmp_path / "overrides" + target = overrides / "rhel" / "10.2" / "a.json" / "repo_manager" + target.mkdir(parents=True) + (target / "repo_manager_config.yml").write_text("---\nrepo_config: always\n") + (target / "repo_manager_config_credentials.yml").write_text("secret") + run_folder = tmp_path / "run" + with mock.patch.object(catalog_func, "OVERRIDES_DIR", str(overrides)): + done = catalog_func.restore_and_override(str(run_folder), "rhel/10.2/a.json", print) + assert done == ["override repo_manager/repo_manager_config.yml"] + assert _read(runtime, "repo_manager", "repo_manager_config.yml")["repo_config"] == "always" + catalog_func.restore_and_override(str(run_folder), "rhel/10.2/b.json", print) + repo = _read(runtime, "repo_manager", "repo_manager_config.yml") + assert repo["repo_config"] == "partial" and "repositories" in repo + assert not (runtime / "runtime" / "repo_manager" + / "repo_manager_config_credentials.yml").exists() + + +# ----------------------------------------------------------------- actions +def test_skip_action_runs_nothing(): + """A skipped catalog returns its reason and runs no command.""" + result = catalog_func.run_action({"action": catalog_vars.ACTION_SKIP, "reason": "aarch64", + "step": ("svt", ("skip", "x.json"))}) + assert result["skipped"] == "aarch64" and result["success"] + + +def test_select_action_writes_and_selects(plan_file, tmp_path): + """select: overrides, settings, --select-catalog; a select error fails the action.""" + case = {"action": catalog_vars.ACTION_SELECT, "catalog": "x.json", + "step": ("svt", ("select", "x.json"))} + plan = catalog_vars.load_plan(plan_file) + + def selected(name, emit, rows): + del emit + rows.append({"command": f"./omnia.sh --select-catalog {name}", "output": "copied"}) + return "" + with mock.patch.object(catalog_func, "load_plan", return_value=plan), \ + mock.patch.object(catalog_func, "load_run", + return_value={"report_path": str(tmp_path)}), \ + mock.patch.object(catalog_func, "restore_and_override", return_value=[]), \ + mock.patch.object(catalog_func, "write_settings", return_value=[1, 2]), \ + mock.patch.object(catalog_func, "select_catalog", side_effect=selected): + result = catalog_func.run_action(case, lambda _: None) + assert result["success"] and "2 setting(s) written" in result["output"] + assert result["command"] == "./omnia.sh --select-catalog x.json" + with mock.patch.object(catalog_func, "select_catalog", return_value="select failed"): + assert not catalog_func.run_action(case, lambda _: None)["success"] + + +def test_svt_action_label(): + """An SVT action is labelled svt , an FVT step fvt_.""" + assert describe_step(("svt", ("select", "x.json"))) == "svt select x.json" + assert describe_step(("repo_manager", ("execute", "exec"))) == "fvt_repo_manager execute exec" + + +# ------------------------------------------------------- run_svt_test hooks +def _fvt(success): + return {"success": success, "rc": 0 if success else 1, "duration": 1.0, "command": "c", + "output": "o", "results": [], "fvt": {}} + + +@pytest.fixture(name="cases") +def _cases(): + """Fake TEST_CASES of one catalog stage; reports and the failure memory are isolated.""" + cases = { + "catalog_build_01_skip": {"id": "A", "stage": "s", "title": "t", "action": "catalog_skip", + "reason": "aarch64", "step": ("svt", ("skip", "x"))}, + "catalog_build_01_exec": {"id": "B", "stage": "s", "title": "t", "stop_leftovers": True, + "step": ("repo_manager", ("execute", "exec"))}, + "catalog_build_01_verify": {"id": "C", "stage": "s", "title": "t", + "step": ("repo_manager", ("execute", "verify"))}, + "catalog_build_01_cleanup": {"id": "D", "stage": "s", "title": "t", "always": True, + "step": ("image_build_manager", ("cleanup_images", "test"))}, + } + with mock.patch.dict(test_func.TEST_CASES, cases), \ + mock.patch.object(test_func, "_FAILED", {}), \ + mock.patch.object(test_func, "_report") as report: + yield report + + +def test_skip_action_skips_the_test(cases): + """The skip action reports Skipped and skips the pytest test.""" + with pytest.raises(pytest.skip.Exception, match="aarch64"): + test_func.run_svt_test("catalog_build", "01_skip") + assert cases.call_args.args[1]["status"] == "Skipped" + + +def test_failed_exec_stops_leftovers_and_blocks_the_stage(cases): + """A failed exec stops its leftovers; verify is skipped; the cleanup still runs.""" + with mock.patch.object(test_func, "run_fvt_step", return_value=_fvt(False)), \ + mock.patch.object(test_func.catalog_func, "stop_leftovers", + return_value=[7]) as stop: + with pytest.raises(AssertionError): + test_func.run_svt_test("catalog_build", "01_exec") + stop.assert_called_once() + assert "pids 7" in cases.call_args.args[1]["note"] + with pytest.raises(pytest.skip.Exception): + test_func.run_svt_test("catalog_build", "01_verify") + with mock.patch.object(test_func, "run_fvt_step", return_value=_fvt(True)) as step: + test_func.run_svt_test("catalog_build", "01_cleanup") + step.assert_called_once() + + +def test_passed_step_stops_nothing(cases): + """A passed exec never looks for leftovers.""" + del cases + with mock.patch.object(test_func, "run_fvt_step", return_value=_fvt(True)), \ + mock.patch.object(test_func.catalog_func, "stop_leftovers") as stop: + test_func.run_svt_test("catalog_build", "01_exec") + stop.assert_not_called() + + +# ------------------------------------------------------ leftover processes +PLAYBOOK = "/opt/omnia/venv/bin/ansible-playbook" +PROCS = { + 10: (1, [PLAYBOOK, "repo_manager.yml", "-v", "--tags", "execute"]), # orphan + 11: (10, ["python3.12"]), # its worker + 12: (11, ["/usr/bin/dnf", "download"]), # grandchild + 20: (500, [PLAYBOOK, "repo_manager.yml", "--tags", "precheck"]), # live step + 30: (1, [PLAYBOOK, "image_build_manager.yml", "--tags", "build"]), # other domain + 40: (1, ["/usr/bin/python3", "repo_manager.yml"]), # not a playbook +} + + +def test_leftovers_are_the_orphan_and_its_tree(): + """Only an orphaned ansible-playbook of the domain's playbook, with its descendants.""" + assert catalog_func.leftover_playbooks("repo_manager", PROCS) == [10, 11, 12] + assert catalog_func.leftover_playbooks("image_build_manager", PROCS) == [30] + assert not catalog_func.leftover_playbooks("repo_manager", {20: PROCS[20]}) + + +def test_processes_reads_proc(tmp_path): + """ppid and argv come from /proc//stat and cmdline (a comm with spaces too).""" + entry = tmp_path / "77" + entry.mkdir() + (entry / "stat").write_text("77 (ansible play) S 1 77 77 0") + (entry / "cmdline").write_bytes(b"ansible-playbook\0repo_manager.yml\0") + (tmp_path / "self").mkdir() + assert catalog_func._processes(str(tmp_path)) == { + 77: (1, ["ansible-playbook", "repo_manager.yml"])} + + +def test_stop_leftovers_signals_the_tree(): + """SIGTERM, then SIGKILL, to every leftover; nothing is signalled when there is none.""" + with mock.patch.object(catalog_func, "_processes", return_value=PROCS), \ + mock.patch.object(catalog_func.os, "kill") as kill, \ + mock.patch.object(catalog_func.os.path, "exists", return_value=False): + assert catalog_func.stop_leftovers("repo_manager", lambda _: None, grace=0) == [10, 11, 12] + assert [call.args for call in kill.call_args_list] == [ + (pid, sig) for sig in (catalog_func.signal.SIGTERM, catalog_func.signal.SIGKILL) + for pid in (10, 11, 12)] + with mock.patch.object(catalog_func, "_processes", return_value={}), \ + mock.patch.object(catalog_func.os, "kill") as kill: + assert not catalog_func.stop_leftovers("repo_manager", lambda _: None, grace=0) + kill.assert_not_called() + + +# ------------------------------------------------------------------ report +def test_report_item_of_a_catalog_step(): + """A catalog step becomes a standard report item (the build_stream format).""" + item = svt_report_func.make_item({"id": "SVT_CATALOG_BUILD_0101", "name": "x: precheck exec", + "status": "PASSED", "command": "fvt_repo_manager precheck " + "exec", "fvt": {"domain": "repo_manager"}}) + assert item["status"] == "Passed" and item["fvt"]["domain"] == "repo_manager" From 42b731ccc14a52d310f28b8f12fa49ee68d5e757 Mon Sep 17 00:00:00 2001 From: Nagachandan-P Date: Thu, 12 Nov 2026 05:08:26 -0600 Subject: [PATCH 2/3] for automation pipeline Signed-off-by: Nagachandan-P --- test/system_test/README.md | 7 +- test/system_test/catalog_build_design.md | 21 ++- test/system_test/docs/CATALOG_BUILD.md | 39 +++++ test/system_test/docs/PIPELINE.md | 12 +- test/system_test/docs/PIPELINE_CONFIG.md | 23 ++- .../system_test/pipeline/setup_svt_project.py | 67 ++++++-- test/system_test/pipeline/svt_ci.py | 36 ++++- .../pipeline/svt_pipeline_config.yml | 7 +- test/system_test/ut/test_catalog_pipeline.py | 149 ++++++++++++++++++ 9 files changed, 339 insertions(+), 22 deletions(-) create mode 100644 test/system_test/ut/test_catalog_pipeline.py diff --git a/test/system_test/README.md b/test/system_test/README.md index 2c2e3bcc48..175b319135 100644 --- a/test/system_test/README.md +++ b/test/system_test/README.md @@ -119,13 +119,18 @@ not stop the next one; the summary still fails. After a failed `exec` the orphan `ansible-playbook .yml` workers (and a running `dnf download`) are stopped, so a playbook killed at the FVT timeout cannot overlap the next test. +From GitLab (`svt.theme: "catalog_build"` in `pipeline/svt_pipeline_config.yml`) every +catalog is its own pipeline job, `catalog_01` ...; see +[docs/PIPELINE_CONFIG.md](docs/PIPELINE_CONFIG.md#theme-catalog_build-one-job-per-catalog). +How the theme compares with build_stream: [docs/CATALOG_BUILD.md](docs/CATALOG_BUILD.md). + ## Documentation | Doc | Contents | |---|---| | [docs/SETUP.md](docs/SETUP.md) | `./setup_env.sh` (configure, cleanup incl. NFS shares, install, credentials, status), `user_inputs.yml`, the dataset generator | | [docs/TEST_CONFIG.md](docs/TEST_CONFIG.md) | Every setting of `build_stream_test_config.yml`: catalogs, the cleanup flags per suite (telemetry, NFS shares ...), the report | -| [docs/CATALOG_BUILD.md](docs/CATALOG_BUILD.md) | **User guide of the `catalog_build` theme**: prerequisites, the two files to edit, step by step, every setting, reading the report, failures, troubleshooting | +| [docs/CATALOG_BUILD.md](docs/CATALOG_BUILD.md) | **User guide of the `catalog_build` theme**: prerequisites, the two files to edit, step by step, every setting, reading the report, failures, the GitLab pipeline (one job per catalog), how it compares with build_stream, troubleshooting | | [docs/RUNNER.md](docs/RUNNER.md) | `./run_validation.sh`, the run and its guard, retry, the SVT report, adding a theme, stage or test | | [docs/OPENBAO.md](docs/OPENBAO.md) | Passwords from OpenBao: one place for its settings, the pipeline and manual ways to read them | | [docs/PIPELINE.md](docs/PIPELINE.md) | Run the SVT from GitLab: how it works, where each secret goes, setup, **updating an existing pipeline**, troubleshooting | diff --git a/test/system_test/catalog_build_design.md b/test/system_test/catalog_build_design.md index a414fdb0ea..34050e1857 100644 --- a/test/system_test/catalog_build_design.md +++ b/test/system_test/catalog_build_design.md @@ -95,6 +95,7 @@ repository check of its own. The FVT playbook timeout stays the omnia_auto defau | `library/functions/setup_func.py` | `bundled_catalogs` and `select_catalog` made public (reused by the plan and the select action) | | `_setup.py` | `_configure`: configure, then the plan for catalog themes | | `library/vars/theme_vars.py`, `test_case_vars.py` | The catalog_build entry and its generated tests | +| `pipeline/setup_svt_project.py`, `pipeline/svt_ci.py` | One job per catalog, and the plan check at initialization (section 7) | New files: `library/vars/catalog_vars.py`, `library/functions/catalog_settings.py`, `library/functions/catalog_func.py`, `catalog_build_test_config.yml`, @@ -116,8 +117,20 @@ New files: `library/vars/catalog_vars.py`, `library/functions/catalog_settings.p `cleanup` and `report` blocks: as in `build_stream_test_config.yml`. -## 7. Not covered +## 7. GitLab pipeline -* The GitLab pipeline (`pipeline/setup_svt_project.py`) generates jobs from the - theme's stages at generation time; for catalog_build that needs the plan of a - configured run on the generating host. The theme targets the manual run. +The SVT pipeline (`pipeline/`) creates one job per theme stage when the CI files are +rendered, before any `--configure` on the OIM. For a catalog theme, +`setup_svt_project.catalog_plan()` therefore resolves the plan on the generating host +with the same `catalog_func.plan()` (this checkout's `catalog_build_test_config.yml`, +`user_inputs.yml` and `./omnia.sh --list-catalogs`), and renders: + +* one job per catalog (`catalog_01` ...), titled with the catalog, `allow_failure` + when `continue_on_failure` is true, timeout `timeouts_hours.catalog_stage`; +* `SVT_CATALOG_PLAN` = `=,...`. + +`svt_ci.py initialization` compares `SVT_CATALOG_PLAN` with the plan `--configure` +wrote on the OIM, before cleanup and install, and fails when they differ (a catalog list +edited in the project, or a tested branch with other bundled catalogs), because those +catalogs would have no job. The fix is `./setup_svt_project.py --update --update-inputs`. +build_stream's CI files are unchanged (no `SVT_CATALOG_PLAN`). diff --git a/test/system_test/docs/CATALOG_BUILD.md b/test/system_test/docs/CATALOG_BUILD.md index cf7fdb3310..17fd64be7c 100644 --- a/test/system_test/docs/CATALOG_BUILD.md +++ b/test/system_test/docs/CATALOG_BUILD.md @@ -13,6 +13,20 @@ for every catalog: No orchestrator deploy and no PXE boot: a catalog is done when its images are built and verified. +**Compared with the `build_stream` theme.** Both themes use the same SVT framework: +the same `setup_env.sh` / `run_validation.sh` commands and guards, test cases run by +`svt/test_svt.py`, the same report, the same cleanup and the same GitLab pipeline. What +differs comes from the catalog list: + +| | `build_stream` | `catalog_build` | +|---|---|---| +| Stages | Fixed: `prepare_build_stream`, `build_deploy`, `cadence`, `automatic_cleanup` | One per catalog: `catalog_01` ... `catalog_NN` | +| When the stages are known | Always | After `--configure` writes the catalog plan (on the OIM), or when the pipeline files are rendered | +| Theme block | `build_stream:` settings written into the build_stream suite | `catalog_build:` (catalogs, repo/image settings, steps), [section 5](#5-settings-reference) | +| Soft stages | `build_deploy`, `cadence`, `automatic_cleanup` | Every catalog stage when `continue_on_failure: true` | +| Extra checks | - | Initialization checks the catalog plan; the pipeline checks that the OIM plan matches its jobs | +| Pipeline timeouts | One key per stage | `catalog_stage` for every catalog job (a per-job key still wins) | + **Contents** 1. [Before you start](#1-before-you-start) @@ -480,6 +494,30 @@ git checkout -- src/main/omnia.env src/repo_manager/input/repo_manager_config.ym Keep cluster-specific values (IPs, mirror URLs) out of `user_inputs.yml` in commits. +### Run it from a GitLab pipeline (one job per catalog) + +The SVT pipeline ([PIPELINE.md](PIPELINE.md)) runs this theme too. Set +`svt.theme: "catalog_build"` in `pipeline/svt_pipeline_config.yml` and render the CI files +from a checkout whose `catalog_build_test_config.yml` and `user_inputs.yml` hold the +values to test: + +```bash +cd test/system_test/pipeline +./setup_svt_project.py --render # one job per catalog +./setup_svt_project.py --update --gitlab-url https:// --update-inputs +``` + +The pipeline is `preflight -> initialization -> catalog_01 -> catalog_02 -> ... -> summary`: +each catalog is its own GitLab job (log, status, Retry button), titled with the catalog +in `.gitlab-ci-svt.yml`. With `continue_on_failure: true` a failed catalog job does not +stop the next one; the summary still fails. `timeouts_hours.catalog_stage` sets the time +limit of every catalog job, and `skip_stages: "catalog_03"` leaves a catalog out of a +run. + +The jobs are fixed when the files are rendered. **After changing the catalog list, run +`--update --update-inputs` again.** If the plan written on the OIM differs from the jobs, +the initialization job stops right after configure (see [section 9](#9-troubleshooting)). + --- ## 9. Troubleshooting @@ -498,6 +536,7 @@ Keep cluster-specific values (IPs, mirror URLs) out of `user_inputs.yml` in comm | `Playbook execution timed out ... Timeout: 7200s` | One FVT playbook took more than two hours (often the first sync of a large catalog) | Run again: synced content is kept, so the next attempt is shorter; use faster mirrors | | `another SVT step is running on this OIM` | Another `setup_env.sh` / `run_validation.sh` holds the SVT lock | Wait for it; a lock whose process is gone is taken over automatically | | `Credentials: [FAIL]` | A credential store is missing (e.g. after `--force`) | `./setup_env.sh --theme catalog_build --set-domain-creds --domain ` | +| Pipeline: `the catalog plan of this run on the OIM differs from the pipeline jobs` | The catalog list was edited in the GitLab project, or the tested branch bundles other catalogs than the checkout the CI files were rendered from | Set the list in your checkout, then `./setup_svt_project.py --update --update-inputs` | More on the shared parts: [SETUP.md](SETUP.md) (setup, credentials, OpenBao), [RUNNER.md](RUNNER.md) (the run guard, retry, the report), diff --git a/test/system_test/docs/PIPELINE.md b/test/system_test/docs/PIPELINE.md index dc25a71b9b..cea1bd4f26 100644 --- a/test/system_test/docs/PIPELINE.md +++ b/test/system_test/docs/PIPELINE.md @@ -11,7 +11,7 @@ each). | `pipeline/setup_svt_project.py` | Creates or updates the project (GitLab client of `test/pipeline/setup_gitlab_project.py`) | | `pipeline/cleanup_svt_project.py` | Checks, empties (pipelines, logs, artifacts) or deletes the project, then verifies it: see [pipeline/README.md](../pipeline/README.md) | | `pipeline/svt_ci.py` | What every pipeline job runs on the GitLab runner (standard library only) | -| `build_stream_test_config.yml`, `user_inputs.yml` | What is tested and the cluster values; each cluster gets a copy in the project. Every key: [TEST_CONFIG.md](TEST_CONFIG.md), [SETUP.md](SETUP.md) | +| `build_stream_test_config.yml` (or `_test_config.yml`), `user_inputs.yml` | What is tested and the cluster values; each cluster gets a copy in the project. Every key: [TEST_CONFIG.md](TEST_CONFIG.md), [SETUP.md](SETUP.md); theme `catalog_build`: [CATALOG_BUILD.md](CATALOG_BUILD.md) | Contents: [how it works](#how-it-works-and-where-each-secret-goes) · [what you need](#1-what-you-need) · [OpenBao](#2-openbao-the-domain-credentials) · [fill the configs](#3-fill-the-configs) · @@ -72,7 +72,9 @@ use the system CA store. [PIPELINE_CONFIG.md](PIPELINE_CONFIG.md). 2. **`user_inputs.yml`**: the cluster values and the `openbao` block ([SETUP.md](SETUP.md)). 3. **`build_stream_test_config.yml`**: catalogs, what the cleanup removes (including the NFS - shares), the report ([TEST_CONFIG.md](TEST_CONFIG.md)). + shares), the report ([TEST_CONFIG.md](TEST_CONFIG.md)). With `svt.theme: "catalog_build"`: + **`catalog_build_test_config.yml`** instead ([CATALOG_BUILD.md](CATALOG_BUILD.md)); its + catalog list decides the pipeline jobs, so render the files after changing it. ## 4. Create the project @@ -153,7 +155,11 @@ Also: CI/CD > Schedules (nightly), or the API (`POST /projects/:id/pipeline` wit `automatic_cleanup`): `./run_validation.sh svt_ `. `prepare_build_stream` is a hard stage: when it fails, the stages after it do not run. `build_deploy`, `cadence` and `automatic_cleanup` are soft stages (`soft_stages` of the theme, `allow_failure: true` in the - generated job): when one fails, the next one still runs. + generated job): when one fails, the next one still runs. catalog_build: one job per catalog + (`catalog_01` ...), resolved from `catalog_build_test_config.yml` when the files are rendered, + all soft with `continue_on_failure: true`; the initialization job fails right after configure + when the plan on the OIM differs from these jobs + ([PIPELINE_CONFIG.md](PIPELINE_CONFIG.md#theme-catalog_build-one-job-per-catalog)). 4. `summary`: always, last. `./run_validation.sh svt_ summary`, the report collected, the email sent, then it **fails when any job or stage failed**. diff --git a/test/system_test/docs/PIPELINE_CONFIG.md b/test/system_test/docs/PIPELINE_CONFIG.md index 343fd0d53b..84804c6507 100644 --- a/test/system_test/docs/PIPELINE_CONFIG.md +++ b/test/system_test/docs/PIPELINE_CONFIG.md @@ -30,7 +30,7 @@ flags are in `build_stream_test_config.yml` ([TEST_CONFIG.md](TEST_CONFIG.md)). | Key | Default | Meaning | Becomes | |---|---|---|---| -| `theme` | `build_stream` | The SVT theme (`THEMES` in `library/vars/theme_vars.py`). Its stages become the pipeline jobs, so a new theme or stage needs no change here | read at generation time | +| `theme` | `build_stream` | The SVT theme (`THEMES` in `library/vars/theme_vars.py`). Its stages become the pipeline jobs, so a new theme or stage needs no change here. `catalog_build`: one job per catalog (see below) | read at generation time | | `poll_seconds` | `30` | Seconds between two reads of a running stage's log on the OIM (minimum 5) | `SVT_POLL_SECONDS` | | `log_limit_kb` | `2500` (empty) | How much of the OIM logs one job prints into the GitLab job log, in KB. GitLab Runner stops collecting a job log at 4096 KB (`output_limit` in the runner's `config.toml`) and timestamps every line, so the job hides base64-only lines, cuts lines over 600 characters and stops printing at this budget (shared by all logs of the job). The full logs stay on the OIM under `reports/ci//`, and the last 40 lines are printed when the job ends | `SVT_LOG_LIMIT_KB` | | `timeouts_hours` | see below | Time limit of each job, in hours | rendered into `.gitlab-ci-svt.yml` (GitLab does not expand variables in `timeout:`); the project's CI timeout is raised to the longest | @@ -41,6 +41,27 @@ theme's stages (`prepare_build_stream`, `build_deploy`, `cadence`, `automatic_cl shortest allowed limit is 10 minutes. The runner's own maximum timeout still applies, so raise that too for long stages. +### Theme `catalog_build`: one job per catalog + +With `theme: "catalog_build"` the pipeline has one job per catalog, in run order: +`preflight -> initialization -> catalog_01 -> catalog_02 -> ... -> summary`. The jobs are +resolved when the CI files are rendered (`--render`, `--create`, `--update`), from +`catalog_build_test_config.yml` and `./omnia.sh --list-catalogs` of this checkout, exactly as +`./setup_env.sh --configure` resolves them on the OIM: + +- Each job is titled with its catalog (`# Catalog 2: rhel/10.2/...` in `.gitlab-ci-svt.yml`). + A catalog skipped by `include_aarch64: false` still gets a short job that records the skip. +- With `continue_on_failure: true` every catalog job has `allow_failure`: a failed catalog does + not stop the next one, and the summary still fails. +- A catalog job without its own `timeouts_hours` entry uses `catalog_stage` (then + `default_hours`). +- `skip_stages` (or `SVT_SKIP_STAGES`) takes catalog stages too, e.g. `catalog_03,catalog_07`. +- The rendered catalog list is the CI/CD variable `SVT_CATALOG_PLAN`. After configure, the + initialization job compares it with the plan written on the OIM and fails, before cleanup + and install, when they differ (the catalog list was edited in the project, or the tested + branch bundles other catalogs). Change the list in this checkout, then + `./setup_svt_project.py --update --update-inputs`. + ## `openbao` | Key | Example | Meaning | Becomes | diff --git a/test/system_test/pipeline/setup_svt_project.py b/test/system_test/pipeline/setup_svt_project.py index 43a32aebbd..f5c0a775a2 100755 --- a/test/system_test/pipeline/setup_svt_project.py +++ b/test/system_test/pipeline/setup_svt_project.py @@ -70,6 +70,12 @@ _validate_token, _validate_url, ) +from library.vars.catalog_vars import ( # noqa: E402 + CATALOG_STAGE_PREFIX, + catalog_soft_stages, + catalog_stage_info, + catalog_stages, +) from library.vars.common_vars import INPUTS_FILE # noqa: E402 from library.vars.theme_vars import THEMES, soft_stages, stage_info, theme_title # noqa: E402 @@ -179,14 +185,54 @@ def ci_variables(config: Dict[str, Any]) -> Dict[str, str]: return variables -def jobs_of(theme: str) -> List[str]: +def catalog_plan(theme: str) -> Dict[str, Any]: + """Return the catalog plan of a catalog theme, resolved from this checkout ({} otherwise). + + The catalog stages are known only after ./setup_env.sh --configure, so the + pipeline resolves the same plan here (_test_config.yml and + ./omnia.sh --list-catalogs of this checkout): one job per catalog. The + initialization job fails when the plan --configure wrote on the OIM + differs (SVT_CATALOG_PLAN). + """ + if not THEMES[theme].get("catalog_plan"): + return {} + # pylint: disable=import-outside-toplevel # only catalog themes need these + from library.functions.catalog_func import plan + from library.functions.config_func import ConfigError, load_svt_config + try: + settings = load_svt_config(theme)["catalog"] + return {"settings": settings, "entries": plan(settings)} + except (ConfigError, ValueError) as exc: + raise SystemExit(f"catalog plan of {theme}: {exc}") from exc + + +def theme_stages(theme: str, plan: Dict[str, Any]) -> Tuple[Tuple[str, ...], Tuple[str, ...], + Dict[str, Tuple[str, str]]]: + """Return (stages, soft stages, {stage: (title, description)}) of the theme's jobs.""" + if plan: + return catalog_stages(plan), catalog_soft_stages(plan), catalog_stage_info(plan) + return (tuple(THEMES[theme]["stages"]), soft_stages(theme), + dict(THEMES[theme].get("stage_info") or {})) + + +def plan_variable(plan: Dict[str, Any]) -> str: + """Return SVT_CATALOG_PLAN: = of every catalog, comma separated.""" + return ",".join(f"{entry['stage']}={entry['catalog']}" for entry in plan.get("entries") or []) + + +def jobs_of(stages: Tuple[str, ...]) -> List[str]: """Return the pipeline jobs in order: preflight (checks and checkout), the SVT stages, summary.""" - return ["preflight", FIRST_STAGE, *THEMES[theme]["stages"], LAST_STAGE] + return ["preflight", FIRST_STAGE, *stages, LAST_STAGE] def _timeout(config: Dict[str, Any], job: str) -> Tuple[str, int]: + """Return (GitLab timeout, seconds) of a job: its own entry, else catalog_stage for a + catalog stage, else default_hours.""" hours = (config.get("svt") or {}).get("timeouts_hours") or {} - value = float(hours.get(job, hours.get("default_hours", 6))) + fallback = hours.get("default_hours", 6) + if job.startswith(CATALOG_STAGE_PREFIX): + fallback = hours.get("catalog_stage", fallback) + value = float(hours.get(job, fallback)) minutes = max(10, int(round(value * 60))) return f"{minutes}m", minutes * 60 @@ -247,7 +293,9 @@ def render_parent(config: Dict[str, Any]) -> str: def render_ci(config: Dict[str, Any]) -> Tuple[str, int]: """Return (.gitlab-ci-svt.yml text, longest job timeout in seconds).""" theme = config["svt"]["theme"] - jobs = jobs_of(theme) + plan = catalog_plan(theme) + stages, soft, info = theme_stages(theme, plan) + jobs = jobs_of(stages) bao = bool(openbao_variables()["BAO_SERVER_URL"]) longest = max(_timeout(config, job)[1] for job in jobs) lines = [ @@ -262,6 +310,7 @@ def render_ci(config: Dict[str, Any]) -> Tuple[str, int]: f' SVT_THEME: "{theme}"', f' SVT_JOB_ORDER: "{",".join(jobs)}"', f' SVT_CREDENTIAL_DOMAINS: "{",".join(THEMES[theme]["credential_domains"])}"', + *([f' SVT_CATALOG_PLAN: "{plan_variable(plan)}"'] if plan else []), ' GIT_DEPTH: "1"', "", "stages:", @@ -279,17 +328,17 @@ def render_ci(config: Dict[str, Any]) -> Tuple[str, int]: ] for job in jobs: timeout = _timeout(config, job)[0] - title, description = stage_info(theme, job) if job != "preflight" \ - else ("Preflight", "SSH to the OIM, OpenBao reachable, Omnia checkout on the OIM, " - "svt/ inputs copied") + title, description = info.get(job) or (stage_info(theme, job) if job != "preflight" else ( + "Preflight", "SSH to the OIM, OpenBao reachable, Omnia checkout on the OIM, " + "svt/ inputs copied")) lines += [f"# {title}: {description}", f"{job}:", f" stage: {job}", f" timeout: {timeout}"] - if job in soft_stages(theme): + if job in soft: lines += [" # A failure here does not stop the stages after it; the summary still fails.", " allow_failure: true"] if job == FIRST_STAGE and bao: lines += [" id_tokens:", " VAULT_ID_TOKEN:", " aud: $BAO_SERVER_URL"] - if job in THEMES[theme]["stages"]: + if job in stages: lines += [" rules:", f" - if: '$SVT_SKIP_STAGES =~ /(^|,) *{job} *(,|$)/'", " when: never", diff --git a/test/system_test/pipeline/svt_ci.py b/test/system_test/pipeline/svt_ci.py index 7ac63844c4..c4e0d45444 100755 --- a/test/system_test/pipeline/svt_ci.py +++ b/test/system_test/pipeline/svt_ci.py @@ -41,9 +41,10 @@ SVT_SSH_SKIP_HOST_KEY=true), SVT_OMNIA_REPO, SVT_OMNIA_BRANCH, SVT_OMNIA_INSTALL_PATH, SVT_CHECKOUT_MODE, SVT_THEME, SVT_CLEAN_REINSTALL, SVT_POLL_SECONDS, SVT_JOB_ORDER, -SVT_CREDENTIAL_DOMAINS, SVT_LABEL, BAO_SERVER_URL, BAO_AUTH_ROLE, -BAO_DATA_PATH, BAO_CACERT (optional file variable: verify OpenBao with this CA; -BAO_TLS_SKIP_VERIFY=false: verify with the system CA store), +SVT_CATALOG_PLAN (catalog themes), SVT_CREDENTIAL_DOMAINS, SVT_LABEL, +BAO_SERVER_URL, BAO_AUTH_ROLE, BAO_DATA_PATH, BAO_CACERT (optional file +variable: verify OpenBao with this CA; BAO_TLS_SKIP_VERIFY=false: verify with +the system CA store), BAO_TLS_SKIP_VERIFY, VAULT_ID_TOKEN, EMAIL_RECIPIENTS, EMAIL_SENDER, SMTP_SERVER, SMTP_PORT, SMTP_USER, SMTP_PASSWORD. No secret is printed, written to a file or passed as an argument. @@ -533,6 +534,34 @@ def _record_run() -> None: _say(f"SVT run {run['run_id']} (report {run.get('report_path')})") +def _check_catalog_plan() -> None: + """Catalog themes: the plan --configure wrote on the OIM must be the rendered jobs. + + setup_svt_project.py renders one job per catalog (SVT_CATALOG_PLAN, + =); a catalog list edited in the project, or a branch with + other bundled catalogs, gives another plan, whose catalogs would have no job. + """ + expected = _env("SVT_CATALOG_PLAN") + if not expected: + return + path = _svt_dir() + "/reports/catalog_build_plan.json" + rc, out = remote(f"cat {shlex.quote(path)}", capture=True) + entries: List[Dict[str, Any]] = [] + if rc == 0: + try: + entries = json.loads(out.decode(errors="replace")).get("entries") or [] + except (ValueError, AttributeError): + entries = [] + actual = ",".join(f"{entry.get('stage')}={entry.get('catalog')}" for entry in entries) + if actual != expected: + _fail("the catalog plan of this run on the OIM differs from the pipeline jobs:\n" + f" pipeline: {expected.replace(',', ', ')}\n" + f" OIM: {actual.replace(',', ', ') or '(no plan: ' + path + ')'}\n" + "Run ./setup_svt_project.py --update (with --update-inputs when the catalog " + "list changed in this checkout) so the jobs match the catalog list.") + _say(f"catalog plan matches the pipeline jobs ({len(entries)} catalog(s))") + + def initialization() -> int: """Clean reinstall, credentials, then the initialization stage of the theme. @@ -546,6 +575,7 @@ def initialization() -> int: rc = run_detached("configure", f"./setup_env.sh --theme {theme} --configure") _record_run() if rc == 0: + _check_catalog_plan() rc = run_detached("install", f"./setup_env.sh --theme {theme} --install --yes" + (" --force" if force else "")) if rc == 0 and payload: diff --git a/test/system_test/pipeline/svt_pipeline_config.yml b/test/system_test/pipeline/svt_pipeline_config.yml index 0ca045b247..412533aed0 100644 --- a/test/system_test/pipeline/svt_pipeline_config.yml +++ b/test/system_test/pipeline/svt_pipeline_config.yml @@ -57,7 +57,9 @@ gitlab: # ----------------------------------------------------------------------------- svt: # The SVT theme (a key of THEMES in ../library/vars/theme_vars.py). Its stages become - # the pipeline jobs, so a new theme or stage needs no change here. + # the pipeline jobs, so a new theme or stage needs no change here. catalog_build: one + # job per catalog of ../catalog_build_test_config.yml, resolved when the CI files are + # rendered; after changing the catalog list run --update --update-inputs. theme: "build_stream" # Seconds between two reads of a running stage's log on the OIM (minimum 5). A short # value shows the log sooner; a long one makes fewer SSH calls. @@ -83,6 +85,9 @@ svt: build_deploy: 8 cadence: 8 automatic_cleanup: 4 + # theme catalog_build: every catalog job (catalog_01, catalog_02 ...) without its + # own entry. + catalog_stage: 3 # Report and email. summary: 1 diff --git a/test/system_test/ut/test_catalog_pipeline.py b/test/system_test/ut/test_catalog_pipeline.py new file mode 100644 index 0000000000..95ad370aee --- /dev/null +++ b/test/system_test/ut/test_catalog_pipeline.py @@ -0,0 +1,149 @@ +# Copyright 2026 Dell Inc. or its subsidiaries. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Unit tests of the catalog_build GitLab pipeline: one job per catalog, plan check. + +No GitLab call and no SSH: the plan and the OIM are mocked. setup_svt_project.py +needs ``requests`` (the shared GitLab client), so its tests are skipped without it. +""" + +# pylint: disable=protected-access,import-error,wrong-import-position + +import json +import os +import sys +from unittest import mock + +import pytest +import yaml + +MODULE_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +sys.path.insert(0, MODULE_ROOT) +sys.path.insert(0, os.path.join(MODULE_ROOT, "pipeline")) + +import svt_ci # noqa: E402 + +ENTRIES = [ + {"stage": "catalog_01", "index": 1, "catalog": "rhel/10.2/slurm_x86_64_no_vast.json", + "run": True, "reason": ""}, + {"stage": "catalog_02", "index": 2, "catalog": "rhel/10.2/slurm_aarch64.json", + "run": False, "reason": "skipped: contains aarch64 (x); include_aarch64 is false"}, +] +PLAN_VALUE = ("catalog_01=rhel/10.2/slurm_x86_64_no_vast.json," + "catalog_02=rhel/10.2/slurm_aarch64.json") + + +@pytest.fixture(name="svt_project") +def _svt_project(): + pytest.importorskip("requests") + import setup_svt_project # pylint: disable=import-outside-toplevel + return setup_svt_project + + +def _config(theme, **hours): + return {"svt": {"theme": theme, "timeouts_hours": {"default_hours": 6, **hours}}, + "clusters": {"oim1": {}}} + + +def _plan(continue_on_failure=True): + return {"settings": {"continue_on_failure": continue_on_failure}, "entries": ENTRIES} + + +# ------------------------------------------------------------------ render +def test_catalog_theme_renders_one_job_per_catalog(svt_project): + """Every catalog is a job, titled with its catalog, soft, with the catalog timeout.""" + with mock.patch.object(svt_project, "catalog_plan", return_value=_plan()): + text, longest = svt_project.render_ci(_config("catalog_build", catalog_stage=3)) + data = yaml.safe_load(text) + assert data["variables"]["SVT_JOB_ORDER"] == ( + "preflight,initialization,catalog_01,catalog_02,summary") + assert data["variables"]["SVT_CATALOG_PLAN"] == PLAN_VALUE + assert data["stages"] == ["preflight", "initialization", "catalog_01", "catalog_02", + "summary"] + for stage in ("catalog_01", "catalog_02"): + assert data[stage]["script"] == [f"python3 svt_ci.py stage {stage}"] + assert data[stage]["allow_failure"] is True + assert data[stage]["timeout"] == "180m" + assert "# Catalog 1: rhel/10.2/slurm_x86_64_no_vast.json:" in text + assert "# Catalog 2: rhel/10.2/slurm_aarch64.json: skipped" in text + assert longest == 6 * 3600 + + +def test_catalog_jobs_are_hard_without_continue_on_failure(svt_project): + """continue_on_failure: false makes every catalog job a hard one.""" + with mock.patch.object(svt_project, "catalog_plan", return_value=_plan(False)): + data = yaml.safe_load(svt_project.render_ci(_config("catalog_build"))[0]) + assert "allow_failure" not in data["catalog_01"] + assert data["catalog_01"]["timeout"] == "360m" + + +def test_build_stream_render_is_unchanged(svt_project): + """build_stream: its fixed stages, no catalog plan, no plan variable.""" + data = yaml.safe_load(svt_project.render_ci(_config("build_stream"))[0]) + assert data["variables"]["SVT_JOB_ORDER"] == ( + "preflight,initialization,prepare_build_stream,build_deploy,cadence," + "automatic_cleanup,summary") + assert "SVT_CATALOG_PLAN" not in data["variables"] + assert data["build_deploy"]["allow_failure"] is True + assert "allow_failure" not in data["prepare_build_stream"] + assert svt_project.catalog_plan("build_stream") == {} + + +def test_catalog_plan_is_resolved_from_the_checkout(svt_project): + """The plan comes from the theme config and --list-catalogs, as --configure does.""" + from library.functions import catalog_func # pylint: disable=import-outside-toplevel + listed = {"1": "rhel/10.2/slurm_x86_64_no_vast.json"} + with mock.patch.object(catalog_func, "bundled_catalogs", return_value=(listed, "")): + plan = svt_project.catalog_plan("catalog_build") + assert [entry["stage"] for entry in plan["entries"]] == ["catalog_01"] + assert plan["settings"]["catalogs"] == "all" + with mock.patch.object(catalog_func, "bundled_catalogs", return_value=({}, "no list")): + with pytest.raises(SystemExit, match="catalog plan of catalog_build: no list"): + svt_project.catalog_plan("catalog_build") + + +# ------------------------------------------------------------- plan check +def _check(expected, remote_result): + with mock.patch.dict(os.environ, {"SVT_CATALOG_PLAN": expected, + "SVT_OMNIA_INSTALL_PATH": "/root/omnia"}), \ + mock.patch.object(svt_ci, "remote", return_value=remote_result) as remote: + svt_ci._check_catalog_plan() + return remote + + +def test_plan_check_passes_when_the_oim_plan_matches(): + """The OIM plan of this run is the rendered jobs: the check passes.""" + remote = _check(PLAN_VALUE, (0, json.dumps({"entries": ENTRIES}).encode())) + assert "/root/omnia/test/system_test/reports/catalog_build_plan.json" in \ + remote.call_args[0][0] + + +@pytest.mark.parametrize("remote_result", [ + (0, json.dumps({"entries": ENTRIES[:1]}).encode()), + (0, b"not json"), + (1, b""), +]) +def test_plan_check_fails_when_the_oim_plan_differs(tmp_path, monkeypatch, remote_result): + """Another, unreadable or missing OIM plan fails the job with the fix.""" + monkeypatch.chdir(tmp_path) + with pytest.raises(SystemExit): + _check(PLAN_VALUE, remote_result) + + +def test_plan_check_is_skipped_for_other_themes(): + """Without SVT_CATALOG_PLAN (build_stream) the OIM is not asked.""" + with mock.patch.dict(os.environ, {"SVT_CATALOG_PLAN": ""}), \ + mock.patch.object(svt_ci, "remote") as remote: + svt_ci._check_catalog_plan() + remote.assert_not_called() From afbb20360b91624172aced108c934ac8d3b4233f Mon Sep 17 00:00:00 2001 From: Nagachandan-P Date: Sun, 15 Nov 2026 03:40:22 -0600 Subject: [PATCH 3/3] pylint issue fixed Signed-off-by: Nagachandan-P --- test/system_test/library/functions/test_func.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/system_test/library/functions/test_func.py b/test/system_test/library/functions/test_func.py index cdb27f8321..2d376f1187 100644 --- a/test/system_test/library/functions/test_func.py +++ b/test/system_test/library/functions/test_func.py @@ -27,7 +27,7 @@ import pytest from omnia_auto import TestLogger -from . import catalog_func, run_func, svt_report_func +from . import catalog_func, run_func, svt_report_func, track_func from .config_func import load_svt_config from .fvt_func import describe_step, run_fvt_step, step_info from ..messages.svt_msgs import CATALOG_MSGS, SETUP_MSGS, TEST_ASSERT_MSGS, TEST_LOG_MSGS