diff --git a/.github/workflows/e2e-suites.yml b/.github/workflows/e2e-suites.yml index 791761befb..87be5cf119 100644 --- a/.github/workflows/e2e-suites.yml +++ b/.github/workflows/e2e-suites.yml @@ -73,6 +73,9 @@ jobs: {"environment":"fedora-podman-rootless","installer":"rpm","family":"conformance","package":"conformance","gateway":"openshell","runner":"ubuntu-24.04"}, {"environment":"ubuntu-k3s","installer":"k3s","family":"conformance","package":"conformance","gateway":"tmachine","runner":"ubuntu-24.04"}, {"environment":"ubuntu-k3s","installer":"k3s-ha","family":"conformance","package":"conformance","gateway":"tmachine","runner":"ubuntu-24.04"}, + {"environment":"ubuntu-docker-rootful-cdi","installer":"deb","family":"features","package":"cdi","gateway":"openshell","runner":"ubuntu-24.04"}, + {"environment":"fedora-podman-rootful-cdi","installer":"rpm","family":"features","package":"cdi","gateway":"openshell","runner":"ubuntu-24.04"}, + {"environment":"fedora-podman-rootless-cdi","installer":"rpm","family":"features","package":"cdi","gateway":"openshell","runner":"ubuntu-24.04"}, {"environment":"ubuntu-docker-rootful-keycloak","installer":"deb","family":"features","package":"keycloak-provider-refresh","gateway":"openshell","runner":"ubuntu-24.04"}, {"environment":"ubuntu-docker-rootful","installer":"deb","family":"features","package":"oci-image","gateway":"openshell","runner":"ubuntu-24.04"}, {"environment":"fedora-podman-rootful","installer":"rpm","family":"features","package":"oci-image","gateway":"openshell","runner":"ubuntu-24.04"}, diff --git a/TESTING.md b/TESTING.md index a3936ad649..ee27ae943a 100644 --- a/TESTING.md +++ b/TESTING.md @@ -317,6 +317,15 @@ nix run .#openshell-test-runner -- \ --family features --package keycloak-provider-refresh ``` +A suite that needs setup beyond a container engine runs in a feature +environment, named `-`, such as `ubuntu-docker-rootful-keycloak` +or `ubuntu-docker-rootful-cdi`. `features` in `tests/tmachine/config.nix` lists +each feature's setup playbooks, the inputs they read, and the bases it supports. +To add one, add an entry there and use the generated environment name in the +suite's row in `.github/workflows/e2e-suites.yml`. The CDI suite +(`tests/suites/features/cdi`) runs in `ubuntu-docker-rootful-cdi`, which creates +hardware-free CDI devices and the host files they reference. + ### Python E2E (`e2e/python/`) `mise run e2e:python` builds `openshell/e2e-python:dev` from diff --git a/tests/suites/features/Cargo.lock b/tests/suites/features/Cargo.lock index bb1a1427ee..15fe71fd4c 100644 --- a/tests/suites/features/Cargo.lock +++ b/tests/suites/features/Cargo.lock @@ -395,6 +395,14 @@ dependencies = [ "shlex", ] +[[package]] +name = "cdi" +version = "0.0.0" +dependencies = [ + "openshell-testkit", + "serde_json", +] + [[package]] name = "cfg-if" version = "1.0.4" diff --git a/tests/suites/features/Cargo.toml b/tests/suites/features/Cargo.toml index af907b4b15..a592b6f028 100644 --- a/tests/suites/features/Cargo.toml +++ b/tests/suites/features/Cargo.toml @@ -3,4 +3,4 @@ [workspace] resolver = "2" -members = ["host-gateway", "keycloak-provider-refresh", "kubernetes-updates", "oci-image"] +members = ["cdi", "host-gateway", "keycloak-provider-refresh", "kubernetes-updates", "oci-image"] diff --git a/tests/suites/features/cdi/Cargo.toml b/tests/suites/features/cdi/Cargo.toml new file mode 100644 index 0000000000..98c98cb7cc --- /dev/null +++ b/tests/suites/features/cdi/Cargo.toml @@ -0,0 +1,16 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +[package] +name = "cdi" +description = "Features suite: CDI device injection" +version = "0.0.0" +edition = "2024" +rust-version = "1.94" +license = "Apache-2.0" +repository = "https://github.com/NVIDIA/OpenShell" +publish = false + +[dependencies] +openshell-testkit = { path = "../../../testkit/testkit" } +serde_json = "1" diff --git a/tests/suites/features/cdi/src/lib.rs b/tests/suites/features/cdi/src/lib.rs new file mode 100644 index 0000000000..18f3c0eb1f --- /dev/null +++ b/tests/suites/features/cdi/src/lib.rs @@ -0,0 +1,702 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Scenario support for the CDI injection tests. +//! +//! The `cdi_fixtures` role in the tmachine Docker and Podman environments +//! creates one CDI +//! device per scenario, of class `tmachine.example/cdi-scenario`, together with +//! the host files it references, and publishes each scenario's expectations as +//! `scenarios.json` (see `vars/scenarios.yaml` in the role). [`scenario`] +//! creates a sandbox that selects one scenario device, runs a probe in it, and +//! compares what the workload observed with the published expectations. The +//! sandbox only reports what it sees. Every assertion is made here, so a +//! failure names the check that disagreed. +//! +//! The environment variable [`SCENARIOS_ENV`] overrides where the expectations +//! are read from. + +use std::collections::BTreeMap; +use std::fmt::Write as _; + +use serde_json::{Value, json}; +use testkit::{Context, Error, Result, ResultExt as _}; + +/// Where the `cdi_fixtures` role publishes the scenario expectations. +pub const SCENARIOS_PATH: &str = "/var/lib/openshell-test-inputs/cdi/scenarios.json"; +/// Overrides [`SCENARIOS_PATH`], for running the tests against another guest. +pub const SCENARIOS_ENV: &str = "OPENSHELL_CDI_SCENARIOS"; +/// The compute drivers that take a `cdi_devices` driver option. +const DRIVERS: [&str; 2] = ["docker", "podman"]; +/// The CDI device class the fixtures define. +pub const SCENARIO_CLASS: &str = "tmachine.example/cdi-scenario"; + +/// Whether an operation on a path is expected to work. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Access { + Ok, + Denied, +} + +impl Access { + fn result(self) -> &'static str { + match self { + Self::Ok => "ok", + Self::Denied => "denied", + } + } +} + +/// The kinds of file [`Check::FileType`] distinguishes. +const FILE_TYPES: [&str; 6] = [ + "file", + "directory", + "character-device", + "block-device", + "socket", + "fifo", +]; + +/// A fact the workload observes, together with its expected value. +#[derive(Clone, Debug, PartialEq, Eq)] +enum Check { + /// An environment variable, `None` when it must be unset. + Env { + name: String, + expect: Option, + }, + /// Whether `path` exists. + Exists { path: String, expect: bool }, + /// The contents of the file at `path`. + Contents { path: String, expect: String }, + /// The kind of file at `path`, one of [`FILE_TYPES`]. + FileType { path: String, expect: String }, + /// Opening `path` for reading works. + Read { path: String, expect: Access }, + /// Opening `path` for append works. Nothing is written, so a writable host + /// fixture is left as it was. + Write { path: String, expect: Access }, + /// Whether the workload runs as a non-root user. + NonRoot { expect: bool }, + /// Whether the workload's groups include the group that owns `path`. + InGroupOf { path: String, expect: bool }, + /// Connecting to the unix socket at `path` works and it echoes what is + /// sent. The probe needs `perl`, which the default workload image has. + EchoSocket { path: String, expect: Access }, +} + +impl Check { + /// Parse a check as written in `scenarios.yaml`. + fn from_json(value: &Value) -> Result { + let text = |key: &str| { + value[key] + .as_str() + .map(str::to_string) + .ok_or_else(|| Error::msg(format!("check {value} needs a string `{key}`"))) + }; + let optional = |key: &str| match &value[key] { + Value::Null => Ok(None), + Value::String(text) => Ok(Some(text.clone())), + _ => Err(Error::msg(format!( + "check {value} needs a string or null `{key}`" + ))), + }; + // True when omitted. + let flag = |key: &str| match &value[key] { + Value::Null => Ok(true), + Value::Bool(flag) => Ok(*flag), + _ => Err(Error::msg(format!("check {value} needs a boolean `{key}`"))), + }; + let access = || match text("expect")?.as_str() { + "ok" => Ok(Access::Ok), + "denied" => Ok(Access::Denied), + other => Err(Error::msg(format!( + "check {value}: expected ok or denied, got {other:?}" + ))), + }; + match value["type"].as_str() { + Some("env") => Ok(Self::Env { + name: text("name")?, + expect: optional("expect")?, + }), + Some("exists") => Ok(Self::Exists { + path: text("path")?, + expect: flag("expect")?, + }), + Some("contents") => Ok(Self::Contents { + path: text("path")?, + expect: text("expect")?, + }), + Some("filetype") => { + let expect = text("expect")?; + if !FILE_TYPES.contains(&expect.as_str()) { + return Err(Error::msg(format!( + "check {value}: file type must be one of {FILE_TYPES:?}" + ))); + } + Ok(Self::FileType { + path: text("path")?, + expect, + }) + } + Some("read") => Ok(Self::Read { + path: text("path")?, + expect: access()?, + }), + Some("write") => Ok(Self::Write { + path: text("path")?, + expect: access()?, + }), + Some("non-root") => Ok(Self::NonRoot { + expect: flag("expect")?, + }), + Some("in-group-of") => Ok(Self::InGroupOf { + path: text("path")?, + expect: flag("expect")?, + }), + Some("echo-socket") => Ok(Self::EchoSocket { + path: text("path")?, + expect: access()?, + }), + other => Err(Error::msg(format!( + "unknown check type {other:?} in {value}" + ))), + } + } + + /// Stable identifier used in the probe output and in failure reports. + fn id(&self) -> String { + match self { + Self::Env { name, .. } => format!("env:{name}"), + Self::Exists { path, .. } => format!("exists:{path}"), + Self::Contents { path, .. } => format!("contents:{path}"), + Self::FileType { path, .. } => format!("filetype:{path}"), + Self::Read { path, .. } => format!("read:{path}"), + Self::Write { path, .. } => format!("write:{path}"), + Self::NonRoot { .. } => "non-root".to_string(), + Self::InGroupOf { path, .. } => format!("in-group-of:{path}"), + Self::EchoSocket { path, .. } => format!("echo-socket:{path}"), + } + } + + /// Shell that prints the observed result for this check. + fn observe(&self) -> String { + match self { + Self::Env { name, .. } => format!( + r#"if [ "${{{name}+x}}" ]; then printf 'set:%s' "${name}"; else printf unset; fi"# + ), + Self::Exists { path, .. } => { + format!(r"if [ -e '{path}' ]; then printf present; else printf missing; fi") + } + Self::Contents { path, .. } => format!( + r#"if [ -e '{path}' ]; then printf 'content:%s' "$(cat '{path}' 2>&1)"; else printf missing; fi"# + ), + Self::FileType { path, .. } => format!( + "if [ -S '{path}' ]; then printf socket; \ + elif [ -c '{path}' ]; then printf character-device; \ + elif [ -b '{path}' ]; then printf block-device; \ + elif [ -p '{path}' ]; then printf fifo; \ + elif [ -d '{path}' ]; then printf directory; \ + elif [ -f '{path}' ]; then printf file; \ + else printf missing; fi" + ), + Self::Read { path, .. } => { + format!(r"if ( : < '{path}' ) 2>/dev/null; then printf ok; else printf denied; fi") + } + Self::Write { path, .. } => { + format!(r"if ( : >> '{path}' ) 2>/dev/null; then printf ok; else printf denied; fi") + } + Self::NonRoot { .. } => { + r#"if [ "$(id -u)" != 0 ]; then printf ok; else printf root; fi"#.to_string() + } + Self::InGroupOf { path, .. } => format!( + r#"set -- $(ls -ln '{path}' 2>/dev/null); case " $(id -G) " in *" $4 "*) printf ok ;; *) printf missing-group ;; esac"# + ), + // `no-client` and `no-echo` are not expected by any scenario, so a + // workload without perl or a socket that does not echo fails the + // check with a result that says why. + Self::EchoSocket { path, .. } => format!( + r#"if command -v perl >/dev/null 2>&1; then perl -MIO::Socket::UNIX -e '$s = IO::Socket::UNIX->new(Peer => $ARGV[0]) or print("denied") and exit; print $s "ping\n"; $s->shutdown(1); $r = <$s>; print((defined $r && $r eq "ping\n") ? "ok" : "no-echo")' '{path}'; else printf no-client; fi"# + ), + } + } + + /// The result [`Check::observe`] prints when the expectation holds. + fn expected(&self) -> String { + match self { + Self::Env { expect, .. } => expect + .as_ref() + .map_or_else(|| "unset".to_string(), |value| format!("set:{value}")), + Self::Exists { expect, .. } => if *expect { "present" } else { "missing" }.to_string(), + Self::Contents { expect, .. } => format!("content:{expect}"), + Self::FileType { expect, .. } => expect.clone(), + Self::Read { expect, .. } + | Self::Write { expect, .. } + | Self::EchoSocket { expect, .. } => expect.result().to_string(), + Self::NonRoot { expect } => if *expect { "ok" } else { "root" }.to_string(), + Self::InGroupOf { expect, .. } => { + if *expect { "ok" } else { "missing-group" }.to_string() + } + } + } +} + +/// What a scenario expects of sandbox creation and the workload. +#[derive(Debug)] +struct Scenario { + /// Facts the workload must observe once the sandbox is created. + checks: Vec, + /// Creation must fail with a diagnostic containing this text. Empty + /// matches any failure. + rejected: Option, +} + +/// Read a scenario's expectations, and the checks every scenario carries, from +/// the published document. +fn scenario_expectations(document: &Value, name: &str) -> Result { + let expect = document["scenarios"].get(name).ok_or_else(|| { + Error::msg(format!( + "scenario {name:?} is not defined in the published expectations" + )) + })?; + let checks_from = |value: &Value| -> Result> { + // An absent list means no checks; a list of any other type is a + // mistake that would silently drop coverage. + match value { + Value::Null => Ok(Vec::new()), + Value::Array(checks) => checks.iter().map(Check::from_json).collect(), + other => Err(Error::msg(format!( + "scenario {name:?}: checks must be a list, found {other}" + ))), + } + }; + let rejected = expect["rejected"].as_str().map(str::to_string); + // A rejected scenario never runs a workload, so its checks are not asserted. + let mut checks = Vec::new(); + if rejected.is_none() { + checks.extend(checks_from(&document["common_checks"])?); + checks.extend(checks_from(&expect["checks"])?); + } + Ok(Scenario { checks, rejected }) +} + +fn load(name: &str) -> Result { + let path = std::env::var(SCENARIOS_ENV).unwrap_or_else(|_| SCENARIOS_PATH.into()); + let text = std::fs::read_to_string(&path) + .map_err(Error::from_display) + .context(format!("read scenario expectations {path}"))?; + let document: Value = serde_json::from_str(&text) + .map_err(Error::from_display) + .context(format!("parse scenario expectations {path}"))?; + scenario_expectations(&document, name) +} + +/// Shell that prints one `check= result=` line per check. +fn probe_script(checks: &[Check]) -> String { + let mut script = String::new(); + for check in checks { + writeln!( + script, + "printf 'check=%s result=' '{}'; {}; printf '\\n'", + check.id(), + check.observe() + ) + .expect("writing to a String cannot fail"); + } + script +} + +fn parse_observations(stdout: &str) -> BTreeMap<&str, &str> { + stdout + .lines() + .filter_map(|line| line.strip_prefix("check=")) + .filter_map(|line| line.split_once(" result=")) + .collect() +} + +/// Describe every check whose observed result differs from its expectation. +fn mismatches(checks: &[Check], observed: &BTreeMap<&str, &str>) -> Vec { + checks + .iter() + .filter_map(|check| { + let expected = check.expected(); + let actual = observed.get(check.id().as_str()).copied(); + (actual != Some(expected.as_str())).then(|| { + format!( + "{}\n expected: {expected}\n observed: {}", + check.id(), + actual.unwrap_or("") + ) + }) + }) + .collect() +} + +/// The compute driver the gateway runs, when it is one of [`DRIVERS`]. +async fn compute_driver(cx: &Context) -> Result<&'static str> { + for driver in DRIVERS { + if cx.expect_compute_driver(driver).await.is_ok() { + return Ok(driver); + } + } + Err(Error::msg(format!( + "the gateway must run exactly one of the compute drivers {DRIVERS:?}" + ))) +} + +/// Create a sandbox that selects the named scenario device on the gateway's +/// compute driver and check it against the scenario's published expectations. +/// +/// A scenario with `checks` must create and observe each of them, including the +/// checks every scenario carries. A scenario with `rejected` must fail to +/// create, with the expected text in the diagnostic. +/// +/// # Errors +/// +/// Fails when the expectations cannot be read, the gateway does not run the +/// Docker or Podman driver, or the sandbox does not behave as the scenario expects. +pub async fn scenario(cx: &Context, name: &str) -> Result { + let Scenario { checks, rejected } = load(name)?; + let driver = compute_driver(cx).await?; + + let sandbox = cx.name("cdi"); + cx.track(&sandbox); + let device = format!("{SCENARIO_CLASS}={name}"); + let config = json!({driver: {"cdi_devices": [device]}}).to_string(); + // A rejected scenario must fail before the workload starts, so it needs no + // probe of its own. + let probe = if rejected.is_some() { + "exit 0".to_string() + } else { + probe_script(&checks) + }; + let output = cx + .cli([ + "sandbox", + "create", + "--name", + &sandbox, + "--no-tty", + "--gpu", + "1", + "--driver-config-json", + &config, + "--", + "sh", + "-c", + &probe, + ]) + .await?; + + if let Some(expected) = rejected { + let diagnostic = format!("{}\n{}", output.stdout, output.stderr); + testkit::ensure!( + output.exit_code != 0 && diagnostic.contains(&expected), + "creation of `{sandbox}` should fail with a diagnostic containing {expected:?}, \ + but `{}` exited with {}:\n{diagnostic}", + output.command, + output.exit_code + ); + return Ok(()); + } + + let output = output + .success() + .context(format!("sandbox with scenario device {name} starts"))?; + let failures = mismatches(&checks, &parse_observations(&output.stdout)); + testkit::ensure!( + failures.is_empty(), + "scenario {name}: checks that disagree:\n FAIL {}\nworkload output:\n{}", + failures.join("\n FAIL "), + output.stdout + ); + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::path::PathBuf; + use std::process::Command; + + fn document() -> Value { + json!({ + "common_checks": [{"type": "env", "name": "COMMON", "expect": "yes"}], + "scenarios": { + "plain": {"checks": [ + {"type": "contents", "path": "/a", "expect": "hello"}, + {"type": "write", "path": "/a", "expect": "denied"}, + {"type": "non-root"}, + ]}, + "refused": {"rejected": "no such device", "checks": [ + {"type": "read", "path": "/b", "expect": "ok"}, + ]}, + }, + }) + } + + fn checks(values: &[Value]) -> Vec { + values + .iter() + .map(|v| Check::from_json(v).unwrap()) + .collect() + } + + #[test] + fn scenario_includes_common_checks_first() { + let scenario = scenario_expectations(&document(), "plain").unwrap(); + assert_eq!(scenario.rejected, None); + let ids: Vec<_> = scenario.checks.iter().map(Check::id).collect(); + assert_eq!(ids, ["env:COMMON", "contents:/a", "write:/a", "non-root"]); + } + + #[test] + fn rejected_scenario_asserts_no_checks() { + let scenario = scenario_expectations(&document(), "refused").unwrap(); + assert_eq!(scenario.rejected.as_deref(), Some("no such device")); + assert!(scenario.checks.is_empty()); + } + + #[test] + fn unknown_scenario_and_check_type_are_errors() { + assert!(scenario_expectations(&document(), "missing").is_err()); + let bad = json!({"scenarios": {"x": {"checks": [{"type": "nope"}]}}}); + assert!(scenario_expectations(&bad, "x").is_err()); + } + + #[test] + fn non_list_checks_are_errors() { + for bad in [ + json!({"scenarios": {"x": {"checks": {"type": "non-root"}}}}), + json!({"scenarios": {"x": {"checks": "non-root"}}}), + json!({"common_checks": {"type": "non-root"}, "scenarios": {"x": {}}}), + ] { + assert!(scenario_expectations(&bad, "x").is_err(), "{bad}"); + } + let absent = json!({"scenarios": {"x": {}}}); + assert!( + scenario_expectations(&absent, "x") + .unwrap() + .checks + .is_empty() + ); + } + + #[test] + fn malformed_checks_are_errors() { + for bad in [ + // Contents need a string; absence is the `exists` check. + json!({"type": "contents", "path": "/a", "expect": null}), + json!({"type": "filetype", "path": "/a", "expect": "chardev"}), + json!({"type": "read", "path": "/a", "expect": "maybe"}), + json!({"type": "non-root", "expect": "yes"}), + // The retired names are not accepted. + json!({"type": "group-of", "path": "/d"}), + ] { + assert!(Check::from_json(&bad).is_err(), "{bad} should not parse"); + } + } + + #[test] + fn new_check_types_parse() { + let parsed = checks(&[ + json!({"type": "exists", "path": "/a", "expect": false}), + json!({"type": "exists", "path": "/a"}), + json!({"type": "filetype", "path": "/s", "expect": "socket"}), + json!({"type": "non-root", "expect": false}), + json!({"type": "in-group-of", "path": "/d"}), + json!({"type": "echo-socket", "path": "/s", "expect": "denied"}), + ]); + assert_eq!( + parsed[0], + Check::Exists { + path: "/a".into(), + expect: false + } + ); + // Existence is expected unless stated. + assert_eq!(parsed[1].expected(), "present"); + assert_eq!(parsed[2].expected(), "socket"); + assert_eq!(parsed[3].expected(), "root"); + assert_eq!(parsed[4].expected(), "ok"); + assert_eq!(parsed[5].expected(), "denied"); + assert_eq!(parsed[5].id(), "echo-socket:/s"); + } + + #[test] + fn observations_are_compared_to_expectations() { + let scenario = scenario_expectations(&document(), "plain").unwrap(); + let stdout = "check=env:COMMON result=set:yes\n\ + check=contents:/a result=content:hello\n\ + check=write:/a result=ok\n\ + check=non-root result=ok\n"; + let failures = mismatches(&scenario.checks, &parse_observations(stdout)); + assert_eq!(failures.len(), 1); + assert!(failures[0].starts_with("write:/a")); + assert!(failures[0].contains("expected: denied")); + assert!(failures[0].contains("observed: ok")); + } + + #[test] + fn missing_observation_is_reported() { + let scenario = scenario_expectations(&document(), "plain").unwrap(); + let failures = mismatches(&scenario.checks, &BTreeMap::new()); + assert_eq!(failures.len(), 4); + assert!(failures.iter().all(|f| f.contains(""))); + } + + #[test] + fn probe_script_prints_one_line_per_check() { + let scenario = scenario_expectations(&document(), "plain").unwrap(); + let script = probe_script(&scenario.checks); + assert_eq!( + script.lines().filter(|l| l.starts_with("printf")).count(), + 4 + ); + assert!(script.contains("check=%s result=")); + } + + /// A directory of fixtures for running the probe shell in this process's + /// own filesystem, so the generated shell is exercised and not only read. + struct Fixtures(PathBuf); + + impl Fixtures { + fn new(test: &str) -> Self { + let dir = std::env::temp_dir().join(format!("cdi-probe-{test}-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(dir.join("dir")).unwrap(); + std::fs::write(dir.join("file"), "marker\n").unwrap(); + let fifo = Command::new("mkfifo") + .arg(dir.join("fifo")) + .status() + .unwrap(); + assert!(fifo.success()); + Self(dir) + } + + fn path(&self, name: &str) -> String { + self.0.join(name).to_str().unwrap().to_string() + } + + /// What the probe observes for `check`, as the guest would print it. + fn observe(&self, check: &Check) -> String { + let output = Command::new("sh") + .arg("-c") + .arg(check.observe()) + .output() + .unwrap(); + String::from_utf8(output.stdout).unwrap() + } + } + + impl Drop for Fixtures { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.0); + } + } + + #[test] + fn probe_shell_observes_echo_socket() { + use std::io::{Read, Write}; + use std::os::unix::net::UnixListener; + + let fx = Fixtures::new("echo"); + let observe = |path: &str| { + fx.observe( + &checks(&[json!({"type": "echo-socket", "path": path, "expect": "ok"})]).remove(0), + ) + }; + let serve = |name: &str, echo: bool| { + let path = fx.path(name); + let listener = UnixListener::bind(&path).unwrap(); + std::thread::spawn(move || { + let (mut stream, _) = listener.accept().unwrap(); + let mut data = Vec::new(); + stream.read_to_end(&mut data).unwrap(); + if echo { + stream.write_all(&data).unwrap(); + } + }); + path + }; + + assert_eq!(observe(&serve("echo.sock", true)), "ok"); + assert_eq!(observe(&serve("mute.sock", false)), "no-echo"); + // A path that is not a listening socket cannot be connected to. + assert_eq!(observe(&fx.path("nope.sock")), "denied"); + assert_eq!(observe(&fx.path("file")), "denied"); + } + + #[test] + fn probe_shell_observes_files() { + let fx = Fixtures::new("files"); + let check = |value: Value| checks(&[value]).remove(0); + let observe = |value: Value| fx.observe(&check(value)); + let (file, dir, fifo) = (fx.path("file"), fx.path("dir"), fx.path("fifo")); + let missing = fx.path("nope"); + + assert_eq!(observe(json!({"type": "exists", "path": file})), "present"); + assert_eq!( + observe(json!({"type": "exists", "path": missing})), + "missing" + ); + assert_eq!( + observe(json!({"type": "contents", "path": file, "expect": "x"})), + "content:marker" + ); + assert_eq!( + observe(json!({"type": "contents", "path": missing, "expect": "x"})), + "missing" + ); + for (path, kind) in [ + (&file, "file"), + (&dir, "directory"), + (&fifo, "fifo"), + (&"/dev/null".to_string(), "character-device"), + (&missing, "missing"), + ] { + assert_eq!( + observe(json!({"type": "filetype", "path": path, "expect": "file"})), + kind, + "{path}" + ); + } + assert_eq!( + observe(json!({"type": "read", "path": file, "expect": "ok"})), + "ok" + ); + assert_eq!( + observe(json!({"type": "read", "path": missing, "expect": "ok"})), + "denied" + ); + assert_eq!( + observe(json!({"type": "write", "path": missing, "expect": "ok"})), + "ok" + ); + // Opening for append never creates a directory. + assert_eq!( + observe(json!({"type": "write", "path": dir, "expect": "ok"})), + "denied" + ); + } + + #[test] + fn probe_shell_observes_identity() { + let fx = Fixtures::new("identity"); + let uid = String::from_utf8(Command::new("id").arg("-u").output().unwrap().stdout).unwrap(); + let observe = |value: Value| fx.observe(&checks(&[value]).remove(0)); + + // The probe reports root only when this process is uid 0. + let expected = if uid.trim() == "0" { "root" } else { "ok" }; + assert_eq!(observe(json!({"type": "non-root"})), expected); + // The fixtures belong to the current user, so their group is one of its groups. + assert_eq!( + observe(json!({"type": "in-group-of", "path": fx.path("file")})), + "ok" + ); + assert_eq!( + observe(json!({"type": "in-group-of", "path": fx.path("nope")})), + "missing-group" + ); + } +} diff --git a/tests/suites/features/cdi/tests/injection.rs b/tests/suites/features/cdi/tests/injection.rs new file mode 100644 index 0000000000..40e6468567 --- /dev/null +++ b/tests/suites/features/cdi/tests/injection.rs @@ -0,0 +1,88 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! CDI device injection on the Docker and Podman compute drivers. +//! +//! Each test creates a sandbox that selects one scenario device and checks what +//! the workload sees against the scenario's published expectations. See the +//! `cdi` crate for how the scenarios and their checks work. No GPU hardware is +//! needed. + +use testkit::Context; + +/// A file mounted by the CDI spec under a directory the sandbox baseline +/// already makes read-only needs nothing derived from the spec: the workload +/// can read the mount and cannot write it. The spec-level environment variable +/// and mount every scenario carries show that the spec's edits and the +/// device's edits merge. +#[testkit::test] +async fn mount_under_read_only_baseline(cx: &mut Context) -> testkit::Result { + cdi::scenario(cx, "mount-under-read-only-baseline").await +} + +/// A read-only file mounted outside any directory the sandbox baseline +/// grants. The spec's expectations record whether the workload can read it. +#[testkit::test] +async fn read_only_file(cx: &mut Context) -> testkit::Result { + cdi::scenario(cx, "read-only-file").await +} + +/// A read-only directory mounted outside the baseline grants. The +/// expectations record whether its marker is readable and whether files can be +/// created in it. +#[testkit::test] +async fn read_only_directory(cx: &mut Context) -> testkit::Result { + cdi::scenario(cx, "read-only-directory").await +} + +/// A mount named like a shared library, `libfixture.so.1`, under a directory of +/// its own. The expectations record whether the library's parent directory is +/// granted for the workload. +#[testkit::test] +async fn library_parent(cx: &mut Context) -> testkit::Result { + cdi::scenario(cx, "library-parent").await +} + +/// A mount named like a shared library directly under `/`. Granting its parent +/// would grant the whole root, so the expectations record how that is handled. +#[testkit::test] +async fn broad_library_parent(cx: &mut Context) -> testkit::Result { + cdi::scenario(cx, "broad-library-parent").await +} + +/// A file the CDI spec mounts read-write. The expectations record whether the +/// workload can read or write it. +#[testkit::test] +async fn writable_file(cx: &mut Context) -> testkit::Result { + cdi::scenario(cx, "writable-file").await +} + +/// A directory the CDI spec mounts read-write. The expectations record whether +/// the workload can create files in it. +#[testkit::test] +async fn writable_directory(cx: &mut Context) -> testkit::Result { + cdi::scenario(cx, "writable-directory").await +} + +/// A bind mount whose options omit both `ro` and `rw`, so the access it grants +/// is implicit. The expectations record how the workload can use it. +#[testkit::test] +async fn omitted_access_file(cx: &mut Context) -> testkit::Result { + cdi::scenario(cx, "omitted-access-file").await +} + +/// A character device node injected from `/dev/null` with read-write +/// permissions. The expectations record whether a non-root workload can write +/// it. +#[testkit::test] +async fn device_node(cx: &mut Context) -> testkit::Result { + cdi::scenario(cx, "device-node").await +} + +/// A unix socket mounted on its own, without the directory it lives in. The +/// expectations record that a sibling file in the host directory stays +/// invisible. +#[testkit::test] +async fn unix_socket(cx: &mut Context) -> testkit::Result { + cdi::scenario(cx, "unix-socket").await +} diff --git a/tests/tmachine/ansible/playbooks/cdi.yaml b/tests/tmachine/ansible/playbooks/cdi.yaml new file mode 100644 index 0000000000..6ba8a22dc7 --- /dev/null +++ b/tests/tmachine/ansible/playbooks/cdi.yaml @@ -0,0 +1,12 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +--- +- name: Prepare hardware-free CDI fixtures in supported environments + hosts: all + gather_facts: false + pre_tasks: + - name: Wait for SSH + ansible.builtin.wait_for_connection: + roles: + - cdi_fixtures diff --git a/tests/tmachine/ansible/roles/cdi_fixtures/defaults/main.yaml b/tests/tmachine/ansible/roles/cdi_fixtures/defaults/main.yaml new file mode 100644 index 0000000000..ce6f92fa28 --- /dev/null +++ b/tests/tmachine/ansible/roles/cdi_fixtures/defaults/main.yaml @@ -0,0 +1,14 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +--- +# The published scenario expectations live under this root. +cdi_fixture_root: /var/lib/openshell-test-inputs/cdi + +# The files and the socket the specs mount into containers. Under /usr/local, +# SELinux hosts give them a type containers can read, which /var/lib does not. +cdi_fixture_files_root: /usr/local/share/openshell-test-inputs/cdi + +# The container engine reads static CDI specs from /etc/cdi, which persists +# across the setup, installation, and test disk layers. +cdi_spec_path: /etc/cdi/openshell-tmachine-scenarios.yaml diff --git a/tests/tmachine/ansible/roles/cdi_fixtures/tasks/main.yaml b/tests/tmachine/ansible/roles/cdi_fixtures/tasks/main.yaml new file mode 100644 index 0000000000..5cc60f858a --- /dev/null +++ b/tests/tmachine/ansible/roles/cdi_fixtures/tasks/main.yaml @@ -0,0 +1,53 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +--- +- name: Check the CDI spec path before creating the spec + become: true + ansible.builtin.stat: + path: "{{ cdi_spec_path }}" + follow: false + register: cdi_spec_stat + +- name: Require a guest without an existing CDI spec + ansible.builtin.assert: + that: [not cdi_spec_stat.stat.exists] + fail_msg: "Refusing to replace existing CDI spec {{ cdi_spec_path }}" + +- name: Create CDI fixture and spec directories + become: true + ansible.builtin.file: + path: "{{ item }}" + state: directory + owner: root + group: root + mode: "0755" + loop: + - "{{ cdi_fixture_root }}" + - "{{ cdi_fixture_files_root }}" + - "{{ cdi_spec_path | dirname }}" + +- name: Prepare named CDI scenarios + ansible.builtin.include_tasks: scenarios.yaml + +# The tests select the scenario device per sandbox through driver config, which +# the gateway rejects unless an administrator allows it. The installers add +# this to the active driver's settings. +- name: Create the test settings directory + become: true + ansible.builtin.file: + path: /etc/openshell-test + state: directory + owner: root + group: root + mode: "0755" + +- name: Allow caller driver config for the CDI tests + become: true + ansible.builtin.copy: + dest: /etc/openshell-test/driver.toml + content: | + allow_driver_config = true + owner: root + group: root + mode: "0644" diff --git a/tests/tmachine/ansible/roles/cdi_fixtures/tasks/scenarios.yaml b/tests/tmachine/ansible/roles/cdi_fixtures/tasks/scenarios.yaml new file mode 100644 index 0000000000..3758ec442d --- /dev/null +++ b/tests/tmachine/ansible/roles/cdi_fixtures/tasks/scenarios.yaml @@ -0,0 +1,214 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +--- +- name: Load named CDI scenarios + ansible.builtin.include_vars: + file: scenarios.yaml + +- name: Collect scenario fixtures + ansible.builtin.set_fact: + cdi_scenario_fixtures: >- + {{ cdi_scenario_common_fixtures + + (cdi_scenarios | map(attribute='fixtures', default=[]) | flatten) }} + +- name: Create the scenario fixture root + become: true + ansible.builtin.file: + path: "{{ cdi_scenario_dir }}" + state: directory + owner: root + group: root + mode: "0755" + +- name: Create scenario fixture directories + become: true + ansible.builtin.file: + path: "{{ cdi_scenario_dir }}/{{ item.path }}" + state: directory + owner: "{{ item.owner | default('root') }}" + group: "{{ item.group | default('root') }}" + mode: "{{ item.mode }}" + loop: "{{ cdi_scenario_fixtures | selectattr('type', 'eq', 'dir') | list }}" + loop_control: + label: "{{ item.path }}" + +- name: Write scenario fixture files + become: true + ansible.builtin.copy: + dest: "{{ cdi_scenario_dir }}/{{ item.path }}" + content: "{{ item.content }}" + owner: "{{ item.owner | default('root') }}" + group: "{{ item.group | default('root') }}" + mode: "{{ item.mode }}" + loop: "{{ cdi_scenario_fixtures | selectattr('type', 'eq', 'file') | list }}" + loop_control: + label: "{{ item.path }}" + +- name: Create scenario fixture named pipes + become: true + ansible.builtin.command: + argv: [mkfifo, "--mode={{ item.mode }}", "{{ cdi_scenario_dir }}/{{ item.path }}"] + creates: "{{ cdi_scenario_dir }}/{{ item.path }}" + loop: "{{ cdi_scenario_fixtures | selectattr('type', 'eq', 'fifo') | list }}" + loop_control: + label: "{{ item.path }}" + +- name: Set scenario fixture pipe ownership + become: true + ansible.builtin.file: + path: "{{ cdi_scenario_dir }}/{{ item.path }}" + owner: "{{ item.owner | default('root') }}" + group: "{{ item.group | default('root') }}" + mode: "{{ item.mode }}" + loop: "{{ cdi_scenario_fixtures | selectattr('type', 'eq', 'fifo') | list }}" + loop_control: + label: "{{ item.path }}" + +# A service that creates the socket, not a socket unit: on SELinux hosts systemd +# labels a socket it creates from the policy's default for the path, which +# denies /var/lib. +# A process creates a file with its directory's type, so the socket the service +# creates inherits this one, and does so again on every boot. On SELinux hosts +# (Fedora) a container cannot stat a socket of the default /usr/local type. The +# file module ignores the type where SELinux is not enabled. +- name: Label the socket fixture directory for containers + become: true + ansible.builtin.file: + path: "{{ cdi_scenario_dir }}/socket" + state: directory + setype: container_file_t + +- name: Check for systemd-socket-activate + ansible.builtin.stat: + path: /usr/bin/systemd-socket-activate + register: cdi_socket_activate + +- name: Require systemd-socket-activate for the socket fixture + ansible.builtin.assert: + that: [cdi_socket_activate.stat.exists] + fail_msg: systemd-socket-activate is not installed; the socket fixture needs it + +- name: Install the socket fixture unit + become: true + ansible.builtin.template: + src: openshell-cdi-echo.service.j2 + dest: /etc/systemd/system/openshell-cdi-echo.service + owner: root + group: root + mode: "0644" + +- name: Start the persistent socket fixture + block: + - name: Start the socket fixture service + become: true + ansible.builtin.systemd_service: + name: openshell-cdi-echo.service + daemon_reload: true + enabled: true + state: restarted + + - name: Wait for the socket fixture + become: true + ansible.builtin.wait_for: + path: "{{ cdi_scenario_dir }}/socket/echo.sock" + state: present + timeout: 15 + + # Connect as an unprivileged user, as a sandbox workload would. A fixture + # that exists but does not echo would otherwise pass the scenario, which + # only checks the socket's file type. + - name: Check that the socket fixture echoes + ansible.builtin.command: + argv: + - "{{ ansible_python_interpreter | default('/usr/bin/python3') }}" + - -c + - | + import socket, sys + payload = b"openshell-cdi-echo\n" + s = socket.socket(socket.AF_UNIX) + s.settimeout(5) + s.connect(sys.argv[1]) + s.sendall(payload) + s.shutdown(socket.SHUT_WR) + data = b"" + while True: + chunk = s.recv(1024) + if not chunk: + break + data += chunk + sys.exit(0 if data == payload else 1) + - "{{ cdi_scenario_dir }}/socket/echo.sock" + changed_when: false + + rescue: + - name: Collect socket fixture diagnostics + become: true + ansible.builtin.shell: | + systemctl status openshell-cdi-echo.service --no-pager -l + journalctl -u openshell-cdi-echo.service --no-pager -n 40 + ls -lZd {{ cdi_scenario_dir }}/socket + (ausearch -m avc,user_avc -ts recent 2>&1 || true) | tail -20 + changed_when: false + failed_when: false + register: cdi_socket_diagnostics + + - name: Report why the socket fixture did not start + ansible.builtin.fail: + msg: "openshell-cdi-echo.service did not start:\n{{ cdi_socket_diagnostics.stdout }}\n{{ cdi_socket_diagnostics.stderr }}" + +# The device-node scenario injects /dev/tmachine-cdi-null, a copy of /dev/null, +# from a host node at the same path. A rootless runtime cannot create nodes, so +# it bind-mounts the host node, and the container sees the host node's SELinux +# type: it needs the type of /dev/null, which containers may use. /dev is +# recreated on every boot, so a service creates and labels the node each time. +# The label is ignored where SELinux is not enabled. +- name: Install the host device node unit + become: true + ansible.builtin.copy: + dest: /etc/systemd/system/openshell-cdi-device.service + content: | + [Unit] + Description=OpenShell tmachine CDI host device node fixture + + [Service] + Type=oneshot + RemainAfterExit=yes + ExecStart=/bin/rm -f /dev/tmachine-cdi-null + ExecStart=/usr/bin/mknod --mode=0666 /dev/tmachine-cdi-null c 1 3 + ExecStart=-/usr/bin/chcon --type=null_device_t /dev/tmachine-cdi-null + + [Install] + WantedBy=multi-user.target + owner: root + group: root + mode: "0644" + +- name: Create the host device node + become: true + ansible.builtin.systemd_service: + name: openshell-cdi-device.service + daemon_reload: true + enabled: true + state: restarted + +- name: Generate named CDI scenario devices + become: true + ansible.builtin.template: + src: scenarios.yaml.j2 + dest: "{{ cdi_spec_path }}" + owner: root + group: root + mode: "0644" + +- name: Publish scenario expectations for the CDI feature tests + become: true + ansible.builtin.copy: + dest: "{{ cdi_fixture_root }}/scenarios.json" + content: >- + {{ {'common_checks': cdi_scenario_common_checks, + 'scenarios': cdi_scenarios | items2dict(key_name='name', value_name='expect')} + | to_nice_json }} + owner: root + group: root + mode: "0644" diff --git a/tests/tmachine/ansible/roles/cdi_fixtures/templates/openshell-cdi-echo.service.j2 b/tests/tmachine/ansible/roles/cdi_fixtures/templates/openshell-cdi-echo.service.j2 new file mode 100644 index 0000000000..f3be32766b --- /dev/null +++ b/tests/tmachine/ansible/roles/cdi_fixtures/templates/openshell-cdi-echo.service.j2 @@ -0,0 +1,25 @@ +{# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. #} +{# SPDX-License-Identifier: Apache-2.0 #} +[Unit] +Description=OpenShell tmachine CDI Unix socket fixture +After=local-fs.target + +[Service] +# The socket is created by this process, not by PID 1, and `cat` echoes each +# connection. `--inetd` puts the accepted connection on stdin and stdout, which +# is where `cat` reads and writes. +# On SELinux hosts a container may connect only to sockets of certain domains, +# which does not include the default one for a service. Serve the socket as a +# container, so the container that mounts it can connect. It is ignored where +# SELinux is off. +SELinuxContext=system_u:system_r:container_t:s0 +ExecStartPre=/bin/rm -f {{ cdi_scenario_dir }}/socket/echo.sock +ExecStart=/usr/bin/systemd-socket-activate --inetd --listen={{ cdi_scenario_dir }}/socket/echo.sock --accept /bin/cat +# The process creates the socket with mode 0644 whatever the unit's UMask, which +# only root can connect to. Open it to any user once it exists. A service +# start, including the one at every boot, creates the socket again. +ExecStartPost=/bin/sh -c 'until [ -S "$1" ]; do sleep 0.1; done; chmod 0666 "$1"' sh {{ cdi_scenario_dir }}/socket/echo.sock +Restart=always + +[Install] +WantedBy=multi-user.target diff --git a/tests/tmachine/ansible/roles/cdi_fixtures/templates/scenarios.yaml.j2 b/tests/tmachine/ansible/roles/cdi_fixtures/templates/scenarios.yaml.j2 new file mode 100644 index 0000000000..1d558f5a4b --- /dev/null +++ b/tests/tmachine/ansible/roles/cdi_fixtures/templates/scenarios.yaml.j2 @@ -0,0 +1,14 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# Generated from vars/scenarios.yaml. Scenarios are selected individually, so +# negative scenarios never combine with positive ones. +cdiVersion: "0.8.0" +kind: tmachine.example/cdi-scenario +containerEdits: +{{ cdi_scenario_spec_edits | to_nice_yaml(indent=2) | indent(2, true) }} +devices: +{% for case in cdi_scenarios %} + - name: {{ case.name }} + containerEdits: +{{ case.edits | to_nice_yaml(indent=2) | indent(6, true) }} +{% endfor %} diff --git a/tests/tmachine/ansible/roles/cdi_fixtures/vars/scenarios.yaml b/tests/tmachine/ansible/roles/cdi_fixtures/vars/scenarios.yaml new file mode 100644 index 0000000000..5e1fc311eb --- /dev/null +++ b/tests/tmachine/ansible/roles/cdi_fixtures/vars/scenarios.yaml @@ -0,0 +1,205 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +--- +# Each case is one device of the tmachine.example/cdi-scenario class, selected as +# `tmachine.example/cdi-scenario=`. A case declares the host fixtures it +# needs and the CDI edits that reference them; tasks/scenarios.yaml creates the +# fixtures and templates/scenarios.yaml.j2 renders the spec. +# +# Fixture `path` is relative to {{ cdi_scenario_dir }}. Types: +# dir directory (parents must be declared first) +# file regular file with `content` +# fifo named pipe +# All accept `mode`, `owner` and `group`. +# +# `expect` records the behavior the CDI feature tests assert for the scenario, +# and is rendered to {{ cdi_fixture_root }}/scenarios.json for them. It states +# what the sandbox does today; a change that alters that behavior updates the +# values here. Either: +# rejected: sandbox creation fails; must appear in the +# diagnostic (empty matches any failure). No workload +# runs, so no checks apply. +# checks: [...] sandbox creation succeeds and each check holds, where a +# check is one of +# {type: exists, path, expect} whether the path exists (true) +# {type: contents, path, expect} the file's contents +# {type: filetype, path, expect} `file`, `directory`, `character-device`, +# `block-device`, `socket` or `fifo` +# {type: read, path, expect} `ok` or `denied` +# {type: write, path, expect} `ok` or `denied` (nothing is written) +# {type: env, name, expect} value, null when unset +# {type: non-root, expect} whether the workload is not uid 0 (true) +# {type: in-group-of, path, expect} +# whether the workload's groups include +# the group that owns path (true) +# `non-root` and `in-group-of` describe the workload process, not a file's +# owner. +cdi_scenario_dir: "{{ cdi_fixture_files_root }}/scenarios" + +# Edits applied by every case, to check that spec and device edits merge. The +# marker is mounted under /usr/share, which the sandbox already allows reading, +# so these checks do not depend on grants derived from the CDI spec. +cdi_scenario_spec_edits: + env: + - TMACHINE_CDI_SCENARIO_SPEC=fixture + mounts: + - hostPath: "{{ cdi_scenario_dir }}/spec-marker" + containerPath: /usr/share/tmachine-cdi/common/spec-marker + options: [bind, ro] + +# Checked in every scenario that is created, to show that spec and device edits +# merge. +cdi_scenario_common_checks: + - {type: env, name: TMACHINE_CDI_SCENARIO_SPEC, expect: fixture} + - {type: contents, path: /usr/share/tmachine-cdi/common/spec-marker, expect: tmachine-cdi-spec-marker} + - {type: write, path: /usr/share/tmachine-cdi/common/spec-marker, expect: denied} + +cdi_scenario_common_fixtures: + - {path: spec-marker, type: file, content: "tmachine-cdi-spec-marker\n", mode: "0644"} + +cdi_scenarios: + # The mount point is under a directory the sandbox baseline already makes + # read-only, so nothing needs to be derived for the mount to be readable. + - name: mount-under-read-only-baseline + fixtures: + - {path: baseline-read-only-file, type: file, content: "tmachine-cdi-baseline-read-only-file\n", mode: "0644"} + edits: + mounts: + - hostPath: "{{ cdi_scenario_dir }}/baseline-read-only-file" + containerPath: /usr/share/tmachine-cdi/baseline-read-only-file + options: [bind, ro] + expect: + checks: + - {type: contents, path: /usr/share/tmachine-cdi/baseline-read-only-file, expect: tmachine-cdi-baseline-read-only-file} + - {type: write, path: /usr/share/tmachine-cdi/baseline-read-only-file, expect: denied} + + - name: read-only-file + fixtures: + - {path: read-only-file, type: file, content: "tmachine-cdi-read-only-file\n", mode: "0644"} + edits: + mounts: + - hostPath: "{{ cdi_scenario_dir }}/read-only-file" + containerPath: /tmachine-cdi/read-only/file + options: [bind, ro] + expect: + checks: + - {type: filetype, path: /tmachine-cdi/read-only/file, expect: file} + - {type: read, path: /tmachine-cdi/read-only/file, expect: denied} + - {type: write, path: /tmachine-cdi/read-only/file, expect: denied} + + - name: read-only-directory + fixtures: + - {path: read-only-directory, type: dir, mode: "0755"} + - {path: read-only-directory/marker, type: file, content: "tmachine-cdi-read-only-directory/marker\n", mode: "0644"} + edits: + mounts: + - hostPath: "{{ cdi_scenario_dir }}/read-only-directory" + containerPath: /tmachine-cdi/read-only/directory + options: [bind, ro] + expect: + checks: + - {type: filetype, path: /tmachine-cdi/read-only/directory, expect: directory} + - {type: read, path: /tmachine-cdi/read-only/directory/marker, expect: denied} + - {type: write, path: /tmachine-cdi/read-only/directory/marker, expect: denied} + - {type: write, path: /tmachine-cdi/read-only/directory/new-file, expect: denied} + + # Both library cases bind the same file; they differ in whether its parent + # directory is narrow or the container root. + + - name: library-parent + fixtures: + - {path: libfixture.so.1, type: file, content: "tmachine-cdi-libfixture.so.1\n", mode: "0644"} + edits: + mounts: + - hostPath: "{{ cdi_scenario_dir }}/libfixture.so.1" + containerPath: /tmachine-cdi/library/libfixture.so.1 + options: [bind, ro] + expect: + checks: + - {type: filetype, path: /tmachine-cdi/library/libfixture.so.1, expect: file} + - {type: read, path: /tmachine-cdi/library/libfixture.so.1, expect: denied} + - {type: write, path: /tmachine-cdi/library/libfixture.so.1, expect: denied} + + - name: broad-library-parent + edits: + mounts: + - hostPath: "{{ cdi_scenario_dir }}/libfixture.so.1" + containerPath: /libtmachinefixture.so.1 + options: [bind, ro] + expect: + checks: + - {type: filetype, path: /libtmachinefixture.so.1, expect: file} + - {type: read, path: /libtmachinefixture.so.1, expect: denied} + - {type: write, path: /libtmachinefixture.so.1, expect: denied} + + - name: writable-file + fixtures: + - {path: writable-file, type: file, content: "tmachine-cdi-original\n", mode: "0666"} + edits: + mounts: + - hostPath: "{{ cdi_scenario_dir }}/writable-file" + containerPath: /tmachine-cdi/writable/file + options: [bind, rw] + expect: + checks: + - {type: filetype, path: /tmachine-cdi/writable/file, expect: file} + - {type: read, path: /tmachine-cdi/writable/file, expect: denied} + - {type: write, path: /tmachine-cdi/writable/file, expect: denied} + + - name: writable-directory + fixtures: + - {path: writable-directory, type: dir, mode: "0777"} + edits: + mounts: + - hostPath: "{{ cdi_scenario_dir }}/writable-directory" + containerPath: /tmachine-cdi/writable/directory + options: [bind, rw] + expect: + checks: + - {type: filetype, path: /tmachine-cdi/writable/directory, expect: directory} + - {type: write, path: /tmachine-cdi/writable/directory/probe, expect: denied} + + - name: omitted-access-file + edits: + mounts: + - hostPath: "{{ cdi_scenario_dir }}/writable-file" + containerPath: /tmachine-cdi/writable/omitted-access + options: [bind] + expect: + checks: + - {type: filetype, path: /tmachine-cdi/writable/omitted-access, expect: file} + - {type: read, path: /tmachine-cdi/writable/omitted-access, expect: denied} + - {type: write, path: /tmachine-cdi/writable/omitted-access, expect: denied} + + - name: device-node + edits: + # The host node is at the same path, created by tasks/scenarios.yaml. + deviceNodes: + - path: /dev/tmachine-cdi-null + permissions: rw + expect: + checks: + - {type: filetype, path: /dev/tmachine-cdi-null, expect: character-device} + - {type: non-root} + - {type: write, path: /dev/tmachine-cdi-null, expect: denied} + + # The echo socket is served by the systemd service installed in scenarios.yaml. + # Its sibling file checks that only the socket path is exposed. + + - name: unix-socket + fixtures: + - {path: socket, type: dir, mode: "0755"} + - {path: socket/sibling, type: file, content: "tmachine-cdi-socket/sibling\n", mode: "0644"} + edits: + mounts: + - hostPath: "{{ cdi_scenario_dir }}/socket/echo.sock" + containerPath: /tmachine-cdi/socket/echo.sock + options: [bind] + expect: + checks: + - {type: filetype, path: /tmachine-cdi/socket/echo.sock, expect: socket} + # Only the socket is mounted, so its sibling in the host directory is + # not visible. + - {type: exists, path: /tmachine-cdi/socket/sibling, expect: false} + - {type: echo-socket, path: /tmachine-cdi/socket/echo.sock, expect: ok} diff --git a/tests/tmachine/ansible/roles/openshell_gateway/tasks/detect.yaml b/tests/tmachine/ansible/roles/openshell_gateway/tasks/detect.yaml index a25cf804ae..450e941d7f 100644 --- a/tests/tmachine/ansible/roles/openshell_gateway/tasks/detect.yaml +++ b/tests/tmachine/ansible/roles/openshell_gateway/tasks/detect.yaml @@ -20,3 +20,22 @@ and not tmachine_container_runtime_is_rootless else tmachine_container_runtime_tmachine_uid.stdout }} openshell_gateway_bind_address: "{{ '127.0.0.1:17670' if tmachine_container_runtime_name == 'docker' else '0.0.0.0:17670' }}" openshell_runtime_socket: "{{ tmachine_container_runtime_socket }}" + +# An environment can add settings to the active driver's gateway.toml table, +# such as `allow_driver_config`, by writing them to this file in the guest. +- name: Check for environment driver settings + ansible.builtin.stat: + path: /etc/openshell-test/driver.toml + register: openshell_gateway_driver_file + +- name: Read environment driver settings + ansible.builtin.slurp: + src: /etc/openshell-test/driver.toml + register: openshell_gateway_driver_slurp + when: openshell_gateway_driver_file.stat.exists + +- name: Select environment driver settings + ansible.builtin.set_fact: + openshell_gateway_driver_settings: >- + {{ openshell_gateway_driver_slurp.content | b64decode | trim + if openshell_gateway_driver_file.stat.exists else '' }} diff --git a/tests/tmachine/ansible/roles/openshell_gateway/templates/gateway-docker.toml.j2 b/tests/tmachine/ansible/roles/openshell_gateway/templates/gateway-docker.toml.j2 index ad55559dc2..72fc8670dc 100644 --- a/tests/tmachine/ansible/roles/openshell_gateway/templates/gateway-docker.toml.j2 +++ b/tests/tmachine/ansible/roles/openshell_gateway/templates/gateway-docker.toml.j2 @@ -7,4 +7,7 @@ socket_path = "{{ openshell_runtime_socket }}" image_pull_policy = "if_not_present" sandbox_label = "tmachine" grpc_endpoint = "http://127.0.0.1:17670" +{% if openshell_gateway_driver_settings %} +{{ openshell_gateway_driver_settings }} +{% endif %} {% endblock %} diff --git a/tests/tmachine/ansible/roles/openshell_gateway/templates/gateway-podman.toml.j2 b/tests/tmachine/ansible/roles/openshell_gateway/templates/gateway-podman.toml.j2 index 0ee292ea29..1ce257bfe1 100644 --- a/tests/tmachine/ansible/roles/openshell_gateway/templates/gateway-podman.toml.j2 +++ b/tests/tmachine/ansible/roles/openshell_gateway/templates/gateway-podman.toml.j2 @@ -11,4 +11,7 @@ stop_timeout_secs = 15 {% if openshell_gateway_podman_driver_settings %} {{ openshell_gateway_podman_driver_settings }} {% endif %} +{% if openshell_gateway_driver_settings %} +{{ openshell_gateway_driver_settings }} +{% endif %} {% endblock %} diff --git a/tests/tmachine/ansible/roles/openshell_packaged_gateway/tasks/main.yaml b/tests/tmachine/ansible/roles/openshell_packaged_gateway/tasks/main.yaml index 155eaa44c8..1172050042 100644 --- a/tests/tmachine/ansible/roles/openshell_packaged_gateway/tasks/main.yaml +++ b/tests/tmachine/ansible/roles/openshell_packaged_gateway/tasks/main.yaml @@ -41,6 +41,24 @@ group: "{{ openshell_gateway_user }}" mode: "0700" +# The package's service reads this operator-owned file when it exists, so write +# it only for an environment that adds driver settings (see detect.yaml). The +# images and driver still come from the environment file below. +- name: Configure environment driver settings + become: true + ansible.builtin.copy: + dest: "{{ openshell_gateway_home }}/.config/openshell/gateway.toml" + owner: "{{ openshell_gateway_user }}" + group: "{{ openshell_gateway_user }}" + mode: "0600" + content: | + [openshell] + version = 2 + + [openshell.drivers.{{ openshell_gateway_driver }}] + {{ openshell_gateway_driver_settings }} + when: openshell_gateway_driver_settings | length > 0 + # Exercise the package-owned service's environment-file mechanism without # generating operator-owned gateway TOML. Preflight and startup must resolve # the same candidate runtime images. diff --git a/tests/tmachine/config.nix b/tests/tmachine/config.nix index 24206097f9..07a2c6ef19 100644 --- a/tests/tmachine/config.nix +++ b/tests/tmachine/config.nix @@ -34,60 +34,95 @@ let openshell_sandbox_image = "../../artifacts/images/openshell-sandbox-tmachine.tar"; openshell_supervisor_image = "../../artifacts/images/openshell-supervisor-tmachine.tar"; }; - # A fedora-podman- environment ("rootful" or "rootless") with more - # setup playbooks after the base ones. - fedoraPodman = - { - name, - podman, - playbooks, - inputs, - }: + fedoraPodman = podman: { + machine = "fedora"; + use_galaxy = false; + playbooks = [ + "ansible/playbooks/nextest.yaml" + "ansible/playbooks/test-tools.yaml" + "ansible/playbooks/selinux.yaml" + "ansible/playbooks/podman-${podman}.yaml" + ]; + }; + # Base environments, keyed by name: a machine and the playbooks that set up + # its container engine (or k3s). + bases = { + ubuntu-docker-rootful = { + machine = "ubuntu"; + use_galaxy = true; + playbooks = [ + "ansible/playbooks/nextest.yaml" + "ansible/playbooks/test-tools.yaml" + "ansible/playbooks/docker.yaml" + ]; + }; + ubuntu-k3s = { + machine = "ubuntu"; + use_galaxy = false; + playbooks = [ + "ansible/playbooks/nextest.yaml" + "ansible/playbooks/test-tools.yaml" + "ansible/playbooks/k3s.yaml" + ]; + }; + fedora-podman-rootful = fedoraPodman "rootful"; + fedora-podman-rootless = fedoraPodman "rootless"; + }; + dockerBases = [ "ubuntu-docker-rootful" ]; + podmanBases = [ + "fedora-podman-rootful" + "fedora-podman-rootless" + ]; + # An environment named -: a base with more setup playbooks and + # the inputs they read, run after the base's own. + withSetup = + base: suffix: setup: + let + parent = bases.${base}; + in { - inherit name; - machine = "fedora"; + name = "${base}-${suffix}"; + inherit (parent) machine; setup = { - use_galaxy = false; - playbooks = [ - "ansible/playbooks/nextest.yaml" - "ansible/playbooks/test-tools.yaml" - "ansible/playbooks/selinux.yaml" - "ansible/playbooks/podman-${podman}.yaml" - ] - ++ playbooks; - inherit inputs; + inherit (parent) use_galaxy; + playbooks = parent.playbooks ++ setup.playbooks; + inputs = setup.inputs or { }; }; }; - # fedora-podman-{rootful,rootless}-keycloak run Keycloak for the - # keycloak-provider-refresh suite. - keycloakEnvironments = map ( - podman: - fedoraPodman { - name = "fedora-podman-${podman}-keycloak"; - inherit podman; - playbooks = [ "ansible/playbooks/keycloak.yaml" ]; - inputs.keycloak_realm_file = "../../scripts/keycloak-realm.json"; - } - ) podmanModes; - # fedora-podman-{rootful,rootless}-userns-{default,auto,keep-id,private} - # configure the Podman gateway's user-namespace mode, for the Podman driver - # suite. Only the binaries installer applies the mode. + # Feature environments, -, one per base the feature supports. + # A suite that needs a feature's setup runs in that environment, so the setup + # lands in a cached layer and other suites don't carry it. + features = { + # Keycloak, for the keycloak-provider-refresh suite. + keycloak = { + bases = dockerBases ++ podmanBases; + setup = { + playbooks = [ "ansible/playbooks/keycloak.yaml" ]; + inputs.keycloak_realm_file = "../../scripts/keycloak-realm.json"; + }; + }; + # Hardware-free CDI fixtures, for the cdi suite. + cdi = { + bases = dockerBases ++ podmanBases; + setup.playbooks = [ "ansible/playbooks/cdi.yaml" ]; + }; + }; + featureEnvironments = pkgs.lib.concatMap ( + feature: map (base: withSetup base feature features.${feature}.setup) features.${feature}.bases + ) (builtins.attrNames features); + # -userns-{default,auto,keep-id,private} configure the Podman + # gateway's user-namespace mode, for the Podman driver suite. Only the + # binaries installer applies the mode. podmanUsernsEnvironments = pkgs.lib.concatMap ( - podman: + base: map ( userns: - fedoraPodman { - name = "fedora-podman-${podman}-userns-${userns}"; - inherit podman; + withSetup base "userns-${userns}" { playbooks = [ "ansible/playbooks/drivers/podman/userns.yaml" ]; inputs.podman_userns_mode = userns; } ) podmanUsernsModes - ) podmanModes; - podmanModes = [ - "rootful" - "rootless" - ]; + ) podmanBases; podmanUsernsModes = [ "default" "auto" @@ -106,75 +141,16 @@ let } ]; - environments = [ - { - name = "ubuntu-docker-rootful"; - machine = "ubuntu"; + environments = + pkgs.lib.mapAttrsToList (name: base: { + inherit name; + inherit (base) machine; setup = { - use_galaxy = true; - playbooks = [ - "ansible/playbooks/nextest.yaml" - "ansible/playbooks/test-tools.yaml" - "ansible/playbooks/docker.yaml" - ]; + inherit (base) use_galaxy playbooks; }; - } - { - name = "ubuntu-docker-rootful-keycloak"; - machine = "ubuntu"; - setup = { - use_galaxy = true; - playbooks = [ - "ansible/playbooks/nextest.yaml" - "ansible/playbooks/docker.yaml" - "ansible/playbooks/keycloak.yaml" - ]; - inputs = { - keycloak_realm_file = "../../scripts/keycloak-realm.json"; - }; - }; - } - { - name = "ubuntu-k3s"; - machine = "ubuntu"; - setup = { - use_galaxy = false; - playbooks = [ - "ansible/playbooks/nextest.yaml" - "ansible/playbooks/test-tools.yaml" - "ansible/playbooks/k3s.yaml" - ]; - }; - } - { - name = "fedora-podman-rootful"; - machine = "fedora"; - setup = { - use_galaxy = false; - playbooks = [ - "ansible/playbooks/nextest.yaml" - "ansible/playbooks/test-tools.yaml" - "ansible/playbooks/selinux.yaml" - "ansible/playbooks/podman-rootful.yaml" - ]; - }; - } - { - name = "fedora-podman-rootless"; - machine = "fedora"; - setup = { - use_galaxy = false; - playbooks = [ - "ansible/playbooks/nextest.yaml" - "ansible/playbooks/test-tools.yaml" - "ansible/playbooks/selinux.yaml" - "ansible/playbooks/podman-rootless.yaml" - ]; - }; - } - ] - ++ keycloakEnvironments - ++ podmanUsernsEnvironments; + }) bases + ++ featureEnvironments + ++ podmanUsernsEnvironments; installers = [ {