Skip to content

Commit af4b200

Browse files
elezarelezar
authored
test(conformance): cover sandbox lifecycle in archives (#3375)
* test(conformance): add sandbox lifecycle coverage Signed-off-by: Evan Lezar <elezar@nvidia.com> * test(tmachine): rename smoke suite to conformance Signed-off-by: Evan Lezar <elezar@nvidia.com> * test(tmachine): configure client during scenario install Signed-off-by: Evan Lezar <elezar@nvidia.com> --------- Signed-off-by: Evan Lezar <elezar@nvidia.com>
1 parent 3dd5ce3 commit af4b200

8 files changed

Lines changed: 344 additions & 30 deletions

File tree

‎crates/openshell-conformance/src/lib.rs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ use tokio::time::sleep;
2323

2424
use self::executor::{CliExecutionError, CliExecutor, ProcessCli};
2525

26-
pub use scenarios::SMOKE_SCENARIO;
26+
pub use scenarios::{SANDBOX_LIFECYCLE_SCENARIO, SMOKE_SCENARIO};
2727

2828
/// An installed conformance scenario.
2929
#[derive(Debug)]
@@ -41,7 +41,7 @@ impl Scenario {
4141
}
4242
}
4343

44-
const SCENARIOS: &[Scenario] = &[SMOKE_SCENARIO];
44+
const SCENARIOS: &[Scenario] = &[SMOKE_SCENARIO, SANDBOX_LIFECYCLE_SCENARIO];
4545

4646
/// Returns every scenario compiled into this distribution.
4747
pub fn scenarios() -> &'static [Scenario] {

‎crates/openshell-conformance/src/scenarios/mod.rs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,8 @@
33

44
//! Registered, portable conformance scenarios.
55
6+
mod sandbox_lifecycle;
67
mod smoke;
78

9+
pub use sandbox_lifecycle::SANDBOX_LIFECYCLE_SCENARIO;
810
pub use smoke::SMOKE_SCENARIO;

‎crates/openshell-conformance/src/scenarios/sandbox_lifecycle.rs‎

Lines changed: 280 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,280 @@
1+
// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2+
// SPDX-License-Identifier: Apache-2.0
3+
4+
//! Portable sandbox lifecycle conformance scenarios.
5+
6+
use std::time::Duration;
7+
8+
use serde::Deserialize;
9+
10+
use crate::{OpenShellRunner, Poll, Scenario, ScenarioFuture};
11+
12+
const CREATE_TIMEOUT: Duration = Duration::from_mins(10);
13+
const COMMAND_TIMEOUT: Duration = Duration::from_mins(2);
14+
const TRANSITION_TIMEOUT: Duration = Duration::from_mins(4);
15+
const TRANSITION_INTERVAL: Duration = Duration::from_secs(2);
16+
17+
#[derive(Debug, Deserialize)]
18+
struct SandboxState {
19+
name: String,
20+
phase: String,
21+
}
22+
23+
/// Certify sandbox stop, start, and deletion lifecycle behavior.
24+
pub const SANDBOX_LIFECYCLE_SCENARIO: Scenario = Scenario {
25+
name: "sandbox-lifecycle",
26+
description: "Verify sandbox stop, start, and deletion lifecycle behavior.",
27+
run: run_sandbox_lifecycle,
28+
};
29+
30+
fn run_sandbox_lifecycle(runner: &mut OpenShellRunner) -> ScenarioFuture<'_> {
31+
Box::pin(async move {
32+
stop_start_preserves_workspace(runner).await?;
33+
stopped_can_be_deleted(runner).await
34+
})
35+
}
36+
37+
async fn stop_start_preserves_workspace(runner: &mut OpenShellRunner) -> Result<(), String> {
38+
let sandbox_name = format!("ct-{}-ss", runner.id());
39+
let sentinel = format!("openshell-stop-start-{}", runner.id());
40+
let sentinel_path = "/sandbox/.openshell-stop-start-sentinel";
41+
let run_count_path = "/sandbox/.openshell-main-run-count";
42+
let main = format!(
43+
"count=0; test ! -f '{run_count_path}' || count=$(cat '{run_count_path}'); \
44+
count=$((count + 1)); printf '%s\\n' \"$count\" > '{run_count_path}'; \
45+
exec sleep infinity"
46+
);
47+
48+
create_running_sandbox(runner, &sandbox_name, &main, "stop-start").await?;
49+
exec_expect_exact(
50+
runner,
51+
&sandbox_name,
52+
"write-sentinel",
53+
&[
54+
"sh",
55+
"-lc",
56+
&format!("printf '%s\\n' '{sentinel}' > '{sentinel_path}' && sync"),
57+
],
58+
"",
59+
)
60+
.await?;
61+
62+
run_lifecycle_command(runner, "stop", &sandbox_name, "stop-start/stop").await?;
63+
wait_for_phase(runner, &sandbox_name, "Stopped", "stop-start/stopped").await?;
64+
65+
let stopped_exec = runner
66+
.step("stop-start/exec-while-stopped")
67+
.description(format!(
68+
"sandbox '{sandbox_name}' rejects exec while stopped"
69+
))
70+
.with_timeout(COMMAND_TIMEOUT)
71+
.run(&[
72+
"sandbox",
73+
"exec",
74+
"--name",
75+
&sandbox_name,
76+
"--no-tty",
77+
"--",
78+
"cat",
79+
sentinel_path,
80+
])
81+
.await
82+
.map_err(|error| error.to_string())?;
83+
if stopped_exec.success() {
84+
return Err(
85+
stopped_exec.failure_diagnostic("sandbox exec fails while the sandbox is stopped")
86+
);
87+
}
88+
89+
run_lifecycle_command(runner, "start", &sandbox_name, "stop-start/start").await?;
90+
wait_for_phase(runner, &sandbox_name, "Ready", "stop-start/restarted").await?;
91+
92+
exec_expect_exact(
93+
runner,
94+
&sandbox_name,
95+
"read-sentinel",
96+
&["cat", sentinel_path],
97+
&format!("{sentinel}\n"),
98+
)
99+
.await?;
100+
exec_expect_exact(
101+
runner,
102+
&sandbox_name,
103+
"read-main-run-count",
104+
&["cat", run_count_path],
105+
"2\n",
106+
)
107+
.await
108+
}
109+
110+
async fn stopped_can_be_deleted(runner: &mut OpenShellRunner) -> Result<(), String> {
111+
let sandbox_name = format!("ct-{}-sd", runner.id());
112+
create_running_sandbox(
113+
runner,
114+
&sandbox_name,
115+
"exec sleep infinity",
116+
"stopped-delete",
117+
)
118+
.await?;
119+
120+
run_lifecycle_command(runner, "stop", &sandbox_name, "stopped-delete/stop").await?;
121+
wait_for_phase(runner, &sandbox_name, "Stopped", "stopped-delete/stopped").await?;
122+
run_lifecycle_command(runner, "delete", &sandbox_name, "stopped-delete/delete").await?;
123+
wait_for_absence(runner, &sandbox_name, "stopped-delete/deleted").await?;
124+
runner.forget_sandbox(&sandbox_name);
125+
Ok(())
126+
}
127+
128+
async fn create_running_sandbox(
129+
runner: &mut OpenShellRunner,
130+
sandbox_name: &str,
131+
main: &str,
132+
step: &str,
133+
) -> Result<(), String> {
134+
runner.track_sandbox(sandbox_name);
135+
let create = runner
136+
.step(format!("{step}/create"))
137+
.description(format!("sandbox '{sandbox_name}' is created"))
138+
.with_timeout(CREATE_TIMEOUT)
139+
.run(&[
140+
"sandbox",
141+
"create",
142+
"--name",
143+
sandbox_name,
144+
"--from",
145+
"base",
146+
"--detach",
147+
"--no-tty",
148+
"--",
149+
"sh",
150+
"-lc",
151+
main,
152+
])
153+
.await
154+
.map_err(|error| error.to_string())?;
155+
create.require_success()?;
156+
wait_for_phase(runner, sandbox_name, "Ready", &format!("{step}/ready")).await
157+
}
158+
159+
async fn run_lifecycle_command(
160+
runner: &OpenShellRunner,
161+
operation: &str,
162+
sandbox_name: &str,
163+
step: &str,
164+
) -> Result<(), String> {
165+
let result = runner
166+
.step(step)
167+
.description(format!("sandbox '{sandbox_name}' {operation} succeeds"))
168+
.with_timeout(COMMAND_TIMEOUT)
169+
.run(&["sandbox", operation, sandbox_name])
170+
.await
171+
.map_err(|error| error.to_string())?;
172+
result.require_success()
173+
}
174+
175+
async fn exec_expect_exact(
176+
runner: &OpenShellRunner,
177+
sandbox_name: &str,
178+
step: &str,
179+
command: &[&str],
180+
expected_stdout: &str,
181+
) -> Result<(), String> {
182+
let mut args = vec!["sandbox", "exec", "--name", sandbox_name, "--no-tty", "--"];
183+
args.extend_from_slice(command);
184+
let result = runner
185+
.step(format!("stop-start/{step}"))
186+
.description(format!("sandbox '{sandbox_name}' exec {step} succeeds"))
187+
.with_timeout(COMMAND_TIMEOUT)
188+
.run(&args)
189+
.await
190+
.map_err(|error| error.to_string())?;
191+
result.require_success()?;
192+
if result.stdout() == expected_stdout {
193+
Ok(())
194+
} else {
195+
Err(result.failure_diagnostic(&format!("stdout is exactly {expected_stdout:?}")))
196+
}
197+
}
198+
199+
async fn wait_for_phase(
200+
runner: &mut OpenShellRunner,
201+
sandbox_name: &str,
202+
expected_phase: &str,
203+
step: &str,
204+
) -> Result<(), String> {
205+
let sandbox_name = sandbox_name.to_string();
206+
let expected_phase = expected_phase.to_string();
207+
let step = step.to_string();
208+
let poll_step = step.clone();
209+
runner
210+
.poll_until(
211+
&poll_step,
212+
TRANSITION_TIMEOUT,
213+
TRANSITION_INTERVAL,
214+
async move |runner| {
215+
let result = runner
216+
.step(format!("{step}/get"))
217+
.description(format!(
218+
"sandbox '{sandbox_name}' reaches phase {expected_phase}"
219+
))
220+
.with_timeout(COMMAND_TIMEOUT)
221+
.run(&["sandbox", "get", &sandbox_name, "--output", "json"])
222+
.await;
223+
match result {
224+
Ok(result) if !result.success() => {
225+
Poll::Pending(result.failure_diagnostic(&format!(
226+
"sandbox '{sandbox_name}' can be retrieved"
227+
)))
228+
}
229+
Ok(result) => match result.json::<SandboxState>() {
230+
Ok(state) if state.name != sandbox_name => Poll::Failed(format!(
231+
"sandbox get returned {:?}; expected '{sandbox_name}'",
232+
state.name
233+
)),
234+
Ok(state) if state.phase == expected_phase => Poll::Ready(()),
235+
Ok(state) => Poll::Pending(format!(
236+
"sandbox '{sandbox_name}' phase is {:?}; expected {expected_phase:?}",
237+
state.phase
238+
)),
239+
Err(error) => Poll::Failed(error.to_string()),
240+
},
241+
Err(error) => Poll::Pending(error.to_string()),
242+
}
243+
},
244+
)
245+
.await
246+
.map_err(|error| error.to_string())
247+
}
248+
249+
async fn wait_for_absence(
250+
runner: &mut OpenShellRunner,
251+
sandbox_name: &str,
252+
step: &str,
253+
) -> Result<(), String> {
254+
let sandbox_name = sandbox_name.to_string();
255+
let step = step.to_string();
256+
let poll_step = step.clone();
257+
runner
258+
.poll_until(
259+
&poll_step,
260+
TRANSITION_TIMEOUT,
261+
TRANSITION_INTERVAL,
262+
async move |runner| {
263+
let result = runner
264+
.step(format!("{step}/get"))
265+
.description(format!("sandbox '{sandbox_name}' is no longer retrievable"))
266+
.with_timeout(COMMAND_TIMEOUT)
267+
.run(&["sandbox", "get", &sandbox_name, "--output", "json"])
268+
.await;
269+
match result {
270+
Ok(result) if !result.success() => Poll::Ready(()),
271+
Ok(_) => {
272+
Poll::Pending(format!("sandbox '{sandbox_name}' is still retrievable"))
273+
}
274+
Err(error) => Poll::Pending(error.to_string()),
275+
}
276+
},
277+
)
278+
.await
279+
.map_err(|error| error.to_string())
280+
}

tests/ansible/playbooks/smoke.yaml renamed to tests/ansible/playbooks/conformance/cli.yaml

Lines changed: 21 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
# SPDX-License-Identifier: Apache-2.0
33

44
---
5-
- name: Run OpenShell smoke tests
5+
- name: Run OpenShell conformance tests
66
hosts: all
77
gather_facts: false
88
tasks:
@@ -18,32 +18,28 @@
1818
group: tmachine
1919
mode: "0700"
2020

21-
- name: Install OpenShell conformance test bundle
21+
- name: Extract OpenShell conformance test bundle
2222
become: true
2323
ansible.builtin.unarchive:
2424
src: "{{ openshell_conformance_test_bundle }}"
2525
dest: /var/lib/openshell-conformance/tests
2626
owner: tmachine
2727
group: tmachine
2828

29-
- name: Wait for OpenShell gateway
30-
ansible.builtin.wait_for:
31-
host: 127.0.0.1
32-
port: 17670
33-
timeout: 60
29+
# Report a malformed outer bundle directly instead of failing later with
30+
# an opaque cargo-nextest missing-archive error.
31+
- name: Check OpenShell conformance nextest archive
32+
ansible.builtin.stat:
33+
path: /var/lib/openshell-conformance/tests/tests.tar.zst
34+
register: conformance_archive
3435

35-
- name: Register OpenShell gateway
36-
ansible.builtin.command:
37-
argv:
38-
- /usr/local/bin/openshell
39-
- gateway
40-
- add
41-
- http://127.0.0.1:17670
42-
- --local
43-
- --name
44-
- tmachine
36+
- name: Require OpenShell conformance nextest archive
37+
ansible.builtin.assert:
38+
that:
39+
- conformance_archive.stat.isreg | default(false)
40+
fail_msg: OpenShell conformance test bundle did not contain tests.tar.zst
4541

46-
- name: Run OpenShell smoke conformance archive
42+
- name: Run OpenShell conformance archive
4743
ansible.builtin.command:
4844
argv:
4945
- cargo-nextest
@@ -60,13 +56,12 @@
6056
changed_when: false
6157
failed_when: false
6258

63-
- name: Show OpenShell smoke conformance diagnostics
59+
# Preserve both streams for failures without adding passing-test output to
60+
# every tmachine run.
61+
- name: Show OpenShell conformance diagnostics
6462
ansible.builtin.debug:
65-
var: conformance_result.stderr_lines
66-
67-
- name: Show OpenShell smoke conformance result
68-
ansible.builtin.debug:
69-
var: conformance_result.stdout_lines
63+
var: conformance_result
64+
when: conformance_result.rc != 0
7065

7166
- name: Read OpenShell gateway logs
7267
become: true
@@ -88,8 +83,8 @@
8883
var: openshell_gateway_logs.stdout_lines
8984
when: conformance_result.rc != 0
9085

91-
- name: Require OpenShell smoke conformance success
86+
- name: Require OpenShell conformance success
9287
ansible.builtin.assert:
9388
that:
9489
- conformance_result.rc == 0
95-
fail_msg: OpenShell smoke conformance test failed
90+
fail_msg: OpenShell conformance test failed

‎tests/ansible/playbooks/gateway.yaml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,3 +7,4 @@
77
gather_facts: false
88
roles:
99
- openshell_gateway
10+
- openshell_client

0 commit comments

Comments
 (0)