Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
57 changes: 47 additions & 10 deletions chi/container.py
Original file line number Diff line number Diff line change
Expand Up @@ -113,11 +113,14 @@ def from_zun_container(cls, zun_container):
return container

@property
def status(self):
def zun_container(self):
if self.id:
container = zun().containers.get(self.id)
self._status = container.status
return self._status
self._zun_container = zun().containers.get(self.id)
return self._zun_container

@property
def status(self):
return getattr(self.zun_container, "status")

def submit(
self,
Expand Down Expand Up @@ -211,13 +214,47 @@ def wait(
if show == "widget" and context._is_ipynb():
pb.display()

state = {"status": None, "since": time.perf_counter(), "conditions": {}}

def _zun_attr(obj, name):
val = getattr(obj, name, None)
return val if val and val != "None" else None

def _callback():
# self.status is a property that refreshes itself
# NOTE: zun statuses are title case
if self.status.upper() == status.upper() or self.status == "Error":
print(f"Container has moved to status {self.status}")
return True
return False
zun_container = self.zun_container
now = time.perf_counter()
elapsed = int(now - state["since"])

current_status = getattr(zun_container, "status", None)
detail = _zun_attr(zun_container, "status_detail")
reason = _zun_attr(zun_container, "status_reason")

if current_status != state["status"]:
if state["status"]:
pb.log(f"[{elapsed}s] {state['status']} -> {current_status}")
state["status"] = current_status
state["since"] = now
state["conditions"] = {}

if detail:
entry = state["conditions"].setdefault(detail, {"count": 0})
entry["count"] += 1
entry["reason"] = reason

parts = [f"{current_status} ({elapsed}s)"]
for name, entry in state["conditions"].items():
count = entry["count"]

if count > 1:
line = f"{name} (x{count})"
else:
line = name
if entry.get("reason"):
line = f"{line} {entry['reason']}"
parts.append(line)
pb.update_status("\n".join(parts))

return current_status == "Error" or current_status.upper() == status.upper()

res = pb.wait(_callback, 2 * 60, timeout)
if not res:
Expand Down
41 changes: 40 additions & 1 deletion chi/util.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import base64
import logging
import os
import time
from datetime import datetime, timedelta
Expand All @@ -10,6 +11,8 @@

from chi.exception import ResourceError

LOG = logging.getLogger(__name__)


def random_base32(n_bytes):
rand_bytes = os.urandom(n_bytes)
Expand Down Expand Up @@ -60,9 +63,45 @@ def __init__(self):
orientation="horizontal",
)
self.label = widgets.Label()
self.status_output = widgets.HTML()
self._log_lines = []
self._current_status_text = ""
self._is_displayed = False

def display(self):
display(widgets.HBox([self.label, self.progress]))
display(
widgets.VBox(
[
widgets.HBox([self.label, self.progress]),
self.status_output,
]
)
)
self._is_displayed = True

def log(self, msg):
self._log_lines.append(msg)
if self._is_displayed:
self._render()
else:
LOG.info(msg)

def update_status(self, msg):
self._current_status_text = msg
if self._is_displayed:
self._render()

def _render(self):
from html import escape

lines = list(self._log_lines)
if self._current_status_text:
lines.append(self._current_status_text)
inner = "\n".join(escape(line) for line in lines)
self.status_output.value = (
'<pre style="max-height:200px;overflow:auto;margin:4px 0;font-family:inherit;">'
f"{inner}</pre>"
)

def wait(self, callback, expected_timeout, timeout, interval=5):
"""Wait and update the progress bar.
Expand Down
81 changes: 81 additions & 0 deletions tests/test_container.py
Original file line number Diff line number Diff line change
Expand Up @@ -192,3 +192,84 @@ def _get_side_effect(ref):
# disable optional behavor from wait_for_active and show
chi_container.submit(wait_for_active=False, show=None)
assert chi_container.id == "new-uuid"


# --- Container.wait() status display tests ---


def _make_zun_state(mocker, status, detail=None, reason=None):
return mocker.Mock(status=status, status_detail=detail, status_reason=reason)


def _setup_wait(mocker, states):
"""Common setup for wait tests: mock zun, ipynb check, and progress bar."""
container = Container(name="t", image_ref="img")
container.id = "fake"

zun_mock = mocker.patch("chi.container.zun")()
zun_mock.containers.get.side_effect = states
mocker.patch("chi.container.context._is_ipynb", return_value=False)

pb_cls = mocker.patch("chi.container.util.TimerProgressBar")
pb = pb_cls.return_value

def fake_wait(cb, expected, timeout, interval=5):
for _ in states:
if cb():
return True
return False

pb.wait.side_effect = fake_wait
return container, pb


def test_wait_logs_status_transitions(mocker):
"""Status changes produce log lines; same status does not."""
states = [
_make_zun_state(mocker, "Creating"),
_make_zun_state(mocker, "Creating"),
_make_zun_state(mocker, "Running"),
]
container, pb = _setup_wait(mocker, states)

container.wait(status="Running")

log_calls = [c.args[0] for c in pb.log.call_args_list]
assert any("Running" in log for log in log_calls)
assert len(log_calls) == 1 # one transition


def test_wait_accumulates_events_without_duplicates(mocker):
"""Different details accumulate; same detail is not repeated."""
states = [
_make_zun_state(mocker, "Creating", detail="Pulling image"),
_make_zun_state(mocker, "Creating", detail="Pulling image"),
_make_zun_state(mocker, "Creating", detail="Configuring net"),
_make_zun_state(mocker, "Running"),
]
container, pb = _setup_wait(mocker, states)

captured_updates = []
pb.update_status.side_effect = lambda msg: captured_updates.append(msg)

container.wait(status="Running")

creating_updates = [u for u in captured_updates if "Creating" in u]
last_creating = creating_updates[-1]
assert "Pulling image" in last_creating
assert "Configuring net" in last_creating
assert last_creating.count("Pulling image") == 1


def test_wait_stops_on_error(mocker):
"""Error status stops the wait loop and is logged."""
states = [
_make_zun_state(mocker, "Creating"),
_make_zun_state(mocker, "Error", detail="OOM", reason="Out of memory"),
]
container, pb = _setup_wait(mocker, states)

container.wait(status="Running")

log_calls = [c.args[0] for c in pb.log.call_args_list]
assert any("Error" in log for log in log_calls)