From 0418b5f98023a10668526c11b4ff3b47c6a361c6 Mon Sep 17 00:00:00 2001 From: Chris Geyer Date: Mon, 27 Jul 2026 14:27:26 +0000 Subject: [PATCH 1/3] feat(trackio): capture experiments into trackio + record the hosted URL on the DAG ([70]) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two pieces, both zero-metric-storage (TReqs stores no customer data — the DAG carries a URL to the externally-hosted experiment, like a W&B URL): 1. Shim injection (roar run --capture-trackio [--trackio-space OWNER/NAME]): the injected sitecustomize aliases the workload's wandb -> trackio and installs a minimal mlflow -> trackio shim, patching trackio.init to sync to the HF Space. So an UNMODIFIED repo's W&B/MLflow logging funnels into a hosted trackio experiment with no code edit. Gated by ROAR_CAPTURE_TRACKIO; best-effort (a missing trackio is a silent no-op). Extracted to trackio_shim.py for testing. (Empirically grounded end-to-end: the same shim captured real wandb+mlflow runs, and trackio.init(space_id=...) yields a live public HF-Space dashboard.) 2. ExperimentTrackerAnalyzer: new _extract_trackio_info reads trackio's per-project SQLite for run identity only and emits the hosted Space URL (https://huggingface.co/spaces/?project=) onto the DAG. URL-only — never the metrics. space_id from ROAR_TRACKIO_SPACE_ID or the synced run. Tests: 5 analyzer (incl. a no-metric-leak invariant) + 5 shim = 10; system_labels regression green; ruff + mypy clean. Should ride the 0.4.1 release. Co-Authored-By: Claude Opus 4.8 (1M context) --- roar/analyzers/experiment_trackers.py | 51 ++++++++++ roar/cli/commands/run.py | 25 +++++ .../execution/runtime/inject/sitecustomize.py | 8 ++ roar/execution/runtime/inject/trackio_shim.py | 96 +++++++++++++++++++ tests/execution/runtime/test_trackio_shim.py | 71 ++++++++++++++ tests/unit/test_experiment_trackers.py | 85 ++++++++++++++++ 6 files changed, 336 insertions(+) create mode 100644 roar/execution/runtime/inject/trackio_shim.py create mode 100644 tests/execution/runtime/test_trackio_shim.py create mode 100644 tests/unit/test_experiment_trackers.py diff --git a/roar/analyzers/experiment_trackers.py b/roar/analyzers/experiment_trackers.py index 541181ea..7c0b7a70 100644 --- a/roar/analyzers/experiment_trackers.py +++ b/roar/analyzers/experiment_trackers.py @@ -2,6 +2,7 @@ import json import re +import sqlite3 from pathlib import Path from typing import Any, ClassVar @@ -18,6 +19,7 @@ class ExperimentTrackerAnalyzer(Analyzer): TRACKER_PATTERNS: ClassVar[dict[str, list[str]]] = { "wandb": ["wandb/", ".wandb"], "mlflow": ["mlruns/", "mlartifacts/"], + "trackio": ["huggingface/trackio", "/trackio/"], "neptune": [".neptune/"], "tensorboard": ["/runs/", "events.out.tfevents"], } @@ -88,6 +90,8 @@ def analyze(self, context: dict) -> dict | None: results["ignore_patterns"].extend(["mlruns/*", "mlartifacts/*"]) if "neptune" in trackers_found: results["ignore_patterns"].append(".neptune/*") + if "trackio" in trackers_found: + results["ignore_patterns"].append("*/trackio/*") # Dedupe results["ignore_patterns"] = sorted(set(results["ignore_patterns"])) @@ -100,6 +104,8 @@ def _extract_run_info(self, tracker: str, written_files: list, env: dict) -> dic return self._extract_wandb_info(written_files, env) elif tracker == "mlflow": return self._extract_mlflow_info(written_files, env) + elif tracker == "trackio": + return self._extract_trackio_info(written_files, env) elif tracker == "neptune": return self._extract_neptune_info(written_files, env) return None @@ -198,6 +204,51 @@ def _extract_wandb_info(self, written_files: list, env: dict) -> dict | None: return info if len(info) > 1 else None + def _extract_trackio_info(self, written_files: list, env: dict) -> dict | None: + """Extract trackio run info + the hosted-dashboard URL. + + trackio stores one SQLite DB per project at ``.../huggingface/trackio/ + .db``. We read *only* the run identity + resolve the public HF + Space URL — never the metrics: TReqs stores no customer data, so the DAG + carries a link to the externally-hosted experiment (like a W&B URL), and + the metrics live on the dashboard, not in lineage. + + The Space is resolved from ``ROAR_TRACKIO_SPACE_ID`` / ``TRACKIO_SPACE_ID`` + (set when roar injects the trackio shim), falling back to the ``space_id`` + recorded on the synced run. URL form: + ``https://huggingface.co/spaces/?project=``. + """ + info: dict[str, Any] = {"tracker": "trackio"} + db_paths = sorted({p for p in written_files if "trackio/" in p and p.endswith(".db")}) + space_id = env.get("ROAR_TRACKIO_SPACE_ID") or env.get("TRACKIO_SPACE_ID") or "" + + for db_path in db_paths: + path = Path(db_path) + info["project"] = path.stem # .db -> + if not path.exists(): + continue + try: + con = sqlite3.connect(f"file:{db_path}?mode=ro", uri=True) + try: + row = con.execute( + "SELECT run_id, run_name, space_id FROM metrics ORDER BY id DESC LIMIT 1" + ).fetchone() + finally: + con.close() + except sqlite3.Error: + continue + if row: + info["run_id"] = row[0] + info["run_name"] = row[1] + if not space_id and row[2]: + space_id = row[2] + + if space_id and info.get("project"): + info["space_id"] = space_id + info["url"] = f"https://huggingface.co/spaces/{space_id}?project={info['project']}" + + return info if len(info) > 1 else None + def _extract_mlflow_info(self, written_files: list, env: dict) -> dict | None: """Extract MLflow run info from local files.""" info = {"tracker": "mlflow"} diff --git a/roar/cli/commands/run.py b/roar/cli/commands/run.py index f0b1b633..907329b9 100644 --- a/roar/cli/commands/run.py +++ b/roar/cli/commands/run.py @@ -5,6 +5,8 @@ roar run @N [--param=value ...] """ +import os + import click from ...application.run import RunRequest, run_command @@ -84,6 +86,19 @@ def _validate_add_tags( callback=_validate_add_tags, help="Stamp KIND=VALUE onto this run's output artifacts (repeatable).", ) +@click.option( + "--capture-trackio", + is_flag=True, + help="Route the workload's W&B / MLflow logging into trackio (zero code change), " + "so it syncs to a hosted HF Space whose URL is recorded on the DAG.", +) +@click.option( + "--trackio-space", + default=None, + metavar="OWNER/NAME", + help="HF Space id to sync trackio runs to (e.g. reproducible-ai/experiments); " + "used with --capture-trackio.", +) @click.pass_obj @require_init def run( @@ -97,6 +112,8 @@ def run( hash_algorithms: tuple[str, ...], block_tags: tuple[str, ...], add_tags: tuple[str, ...], + capture_trackio: bool, + trackio_space: str | None, ) -> None: """Run a command with provenance tracking. @@ -117,6 +134,14 @@ def run( roar run @2 # Re-run DAG node 2 roar run @2 --epochs=10 # Re-run with parameter override """ + # Trackio capture: set the env the injected sitecustomize reads. The traced + # child inherits os.environ (tracer builds env = dict(os.environ)), so the + # wandb/mlflow -> trackio shim fires and the analyzer records the Space URL. + if capture_trackio: + os.environ["ROAR_CAPTURE_TRACKIO"] = "1" + if trackio_space: + os.environ["ROAR_TRACKIO_SPACE_ID"] = trackio_space + args_list = list(args) # Check for help diff --git a/roar/execution/runtime/inject/sitecustomize.py b/roar/execution/runtime/inject/sitecustomize.py index 11846089..32ac9162 100644 --- a/roar/execution/runtime/inject/sitecustomize.py +++ b/roar/execution/runtime/inject/sitecustomize.py @@ -61,6 +61,14 @@ def _prepend_roar_runtime_pythonpath() -> None: _runtime_tracker.install() +try: + from roar.execution.runtime.inject.trackio_shim import install_trackio_shim + + install_trackio_shim(os.environ) +except Exception: + pass # best-effort: capture is opt-in and must never break a run + + def _repair_runtime_in_process(expected_soabi: str) -> bool: """Install + prepend an ABI-matched runtime tree for *this* interpreter. diff --git a/roar/execution/runtime/inject/trackio_shim.py b/roar/execution/runtime/inject/trackio_shim.py new file mode 100644 index 00000000..3475d8fc --- /dev/null +++ b/roar/execution/runtime/inject/trackio_shim.py @@ -0,0 +1,96 @@ +"""wandb / mlflow -> trackio shim, installed into a traced child ([70]). + +The injected ``sitecustomize`` calls :func:`install_trackio_shim` at interpreter +startup when ``ROAR_CAPTURE_TRACKIO`` is set (by ``roar run --capture-trackio``). +It aliases ``wandb`` to trackio and installs a minimal ``mlflow`` fluent-API shim, +so an *unmodified* workload's experiment logging funnels into trackio and syncs to +the configured HF Space — the experiment-tracking analyzer then records that Space +URL on the DAG (URL only; no metrics are stored). + +Extracted here (rather than inline in sitecustomize) so it can be unit-tested +without the sitecustomize's module-level runtime setup. Dependency-free and +best-effort: a missing trackio, or any error, is a silent no-op. +""" + +from __future__ import annotations + +import os +import sys +import types +from typing import Any + + +def _make_mlflow_shim(trackio: Any, space: str | None) -> types.ModuleType: + """A minimal ``mlflow`` module that funnels into trackio (metrics; params are + best-effort no-ops — trackio's config is set at init).""" + m = types.ModuleType("mlflow") + state: dict = {} + + def set_experiment(name: str | None = None, **_k: Any) -> None: + state["project"] = name + + def start_run(**_k: Any) -> Any: + trackio.init(project=state.get("project", "mlflow")) + + class _Run: + def __enter__(self) -> Any: + return self + + def __exit__(self, *_a: Any) -> None: + trackio.finish() + + return _Run() + + def noop(*_a: Any, **_k: Any) -> None: + return None + + m.__dict__.update( + { + "set_experiment": set_experiment, + "start_run": start_run, + "log_metric": lambda key, value, step=None, **k: trackio.log({key: value}, step=step), + "log_metrics": lambda d, step=None, **k: trackio.log(dict(d), step=step), + "log_param": noop, + "log_params": noop, + "set_tracking_uri": noop, + "autolog": noop, + "end_run": noop, + } + ) + return m + + +def install_trackio_shim(environ: Any = None) -> bool: + """Install the wandb/mlflow -> trackio shims if ``ROAR_CAPTURE_TRACKIO`` is set. + + Returns ``True`` iff the shims were installed. Best-effort: a missing trackio + or any error returns ``False`` and leaves the process unchanged. + """ + env = environ if environ is not None else os.environ + if env.get("ROAR_CAPTURE_TRACKIO", "") not in ("1", "true", "yes"): + return False + try: + import trackio + except Exception: + return False + space = env.get("ROAR_TRACKIO_SPACE_ID") or None + try: + # Every trackio run should sync to the campaign Space, even when the + # workload's wandb/mlflow calls don't pass a space_id. + if space: + _orig_init = trackio.init + + def _init(*args: Any, **kwargs: Any) -> Any: + kwargs.setdefault("space_id", space) + return _orig_init(*args, **kwargs) + + trackio.init = _init + + # W&B -> trackio (drop-in). setdefault: never clobber a real prior import. + sys.modules.setdefault("wandb", trackio) + # MLflow -> trackio. + if "mlflow" not in sys.modules: + sys.modules["mlflow"] = _make_mlflow_shim(trackio, space) + return True + except Exception: + return False diff --git a/tests/execution/runtime/test_trackio_shim.py b/tests/execution/runtime/test_trackio_shim.py new file mode 100644 index 00000000..1283289f --- /dev/null +++ b/tests/execution/runtime/test_trackio_shim.py @@ -0,0 +1,71 @@ +"""Unit tests for the wandb/mlflow -> trackio injection shim ([70]).""" + +import builtins +import sys +import types + +import pytest + +from roar.execution.runtime.inject.trackio_shim import install_trackio_shim + + +@pytest.fixture +def fake_trackio(monkeypatch): + tk = types.ModuleType("trackio") + tk.inits = [] + tk.logged = [] + tk.init = lambda *a, **k: tk.inits.append(k) + tk.log = lambda d, step=None: tk.logged.append((dict(d), step)) + tk.finish = lambda: None + monkeypatch.setitem(sys.modules, "trackio", tk) + monkeypatch.delitem(sys.modules, "wandb", raising=False) + monkeypatch.delitem(sys.modules, "mlflow", raising=False) + return tk + + +def test_disabled_by_default(fake_trackio): + assert install_trackio_shim({}) is False + assert "wandb" not in sys.modules + + +def test_installs_wandb_and_mlflow_with_space(fake_trackio): + ok = install_trackio_shim( + {"ROAR_CAPTURE_TRACKIO": "1", "ROAR_TRACKIO_SPACE_ID": "reproducible-ai/experiments"} + ) + assert ok is True + assert sys.modules["wandb"] is fake_trackio + assert "mlflow" in sys.modules + # wandb (== trackio) init injects the configured space_id + sys.modules["wandb"].init(project="m", config={"lr": 0.01}) + assert fake_trackio.inits[-1].get("space_id") == "reproducible-ai/experiments" + + +def test_mlflow_shim_funnels_metrics(fake_trackio): + install_trackio_shim({"ROAR_CAPTURE_TRACKIO": "1", "ROAR_TRACKIO_SPACE_ID": "org/space"}) + mlflow = sys.modules["mlflow"] + mlflow.set_experiment("exp1") + with mlflow.start_run(): + mlflow.log_metric("loss", 0.5, step=1) + mlflow.log_params({"lr": 0.01}) # no-op; must not raise + assert fake_trackio.inits[-1].get("project") == "exp1" + assert ({"loss": 0.5}, 1) in fake_trackio.logged + + +def test_does_not_clobber_a_real_wandb(fake_trackio): + sentinel = types.ModuleType("wandb") + sys.modules["wandb"] = sentinel + install_trackio_shim({"ROAR_CAPTURE_TRACKIO": "1"}) + assert sys.modules["wandb"] is sentinel # setdefault preserved the prior import + + +def test_missing_trackio_is_a_noop(monkeypatch): + monkeypatch.delitem(sys.modules, "trackio", raising=False) + real_import = builtins.__import__ + + def fake_import(name, *a, **k): + if name == "trackio": + raise ImportError("no trackio") + return real_import(name, *a, **k) + + monkeypatch.setattr(builtins, "__import__", fake_import) + assert install_trackio_shim({"ROAR_CAPTURE_TRACKIO": "1"}) is False diff --git a/tests/unit/test_experiment_trackers.py b/tests/unit/test_experiment_trackers.py new file mode 100644 index 00000000..ef196979 --- /dev/null +++ b/tests/unit/test_experiment_trackers.py @@ -0,0 +1,85 @@ +"""Unit tests for the trackio branch of ExperimentTrackerAnalyzer ([70]). + +trackio stores one SQLite DB per project; roar records only the run identity + the +hosted HF-Space URL (never the metrics — TReqs stores no customer data). +""" + +import sqlite3 +from pathlib import Path + +from roar.analyzers.experiment_trackers import ExperimentTrackerAnalyzer + + +def _make_trackio_db(dirpath: Path, project: str, space_id: str | None = None) -> str: + """Create a minimal trackio-shaped SQLite DB, return its path.""" + db = dirpath / "huggingface" / "trackio" / f"{project}.db" + db.parent.mkdir(parents=True, exist_ok=True) + con = sqlite3.connect(str(db)) + con.execute( + "CREATE TABLE metrics (id INTEGER PRIMARY KEY, run_id TEXT, timestamp TEXT, " + "run_name TEXT, step INTEGER, metrics TEXT, log_id TEXT, space_id TEXT)" + ) + for step in range(3): + con.execute( + "INSERT INTO metrics (run_id, timestamp, run_name, step, metrics, log_id, space_id) " + "VALUES (?,?,?,?,?,?,?)", + ( + "run123", + "2026-07-27T00:00:00Z", + "brave-run-0", + step, + f'{{"loss": {round(1.0 / (step + 1), 4)}}}', + f"log{step}", + space_id, + ), + ) + con.commit() + con.close() + return str(db) + + +class TestTrackioExtractor: + def test_url_from_env_space_id(self, tmp_path: Path): + db = _make_trackio_db(tmp_path, "minimind-o") + info = ExperimentTrackerAnalyzer()._extract_trackio_info( + [db], {"ROAR_TRACKIO_SPACE_ID": "reproducible-ai/experiments"} + ) + assert info is not None + assert info["tracker"] == "trackio" + assert info["project"] == "minimind-o" + assert info["run_id"] == "run123" + assert info["run_name"] == "brave-run-0" + assert ( + info["url"] + == "https://huggingface.co/spaces/reproducible-ai/experiments?project=minimind-o" + ) + + def test_space_id_from_db_column(self, tmp_path: Path): + db = _make_trackio_db(tmp_path, "yolo", space_id="reproducible-ai/experiments") + info = ExperimentTrackerAnalyzer()._extract_trackio_info([db], {}) # no env + assert info is not None + assert info["url"].endswith("/reproducible-ai/experiments?project=yolo") + + def test_no_space_id_records_identity_but_no_url(self, tmp_path: Path): + db = _make_trackio_db(tmp_path, "cosyvoice") # space_id None + no env + info = ExperimentTrackerAnalyzer()._extract_trackio_info([db], {}) + assert info is not None + assert info["project"] == "cosyvoice" + assert "url" not in info + + def test_detection_and_analyze_end_to_end(self, tmp_path: Path): + db = _make_trackio_db(tmp_path, "sam2", space_id="reproducible-ai/experiments") + analyzer = ExperimentTrackerAnalyzer() + ctx = {"tracer_data": {"written_files": [db]}, "env": {}} + assert analyzer.relevant(ctx) is True + res = analyzer.analyze(ctx) + assert res is not None + assert "trackio" in res["trackers_detected"] + run = next(r for r in res["runs"] if r["tracker"] == "trackio") + assert run["url"].endswith("?project=sam2") + + def test_url_only_no_metric_values_leak(self, tmp_path: Path): + """No-data-storage invariant: metric VALUES must never reach lineage.""" + db = _make_trackio_db(tmp_path, "m", space_id="reproducible-ai/experiments") + info = ExperimentTrackerAnalyzer()._extract_trackio_info([db], {}) + assert "loss" not in str(info) From 3d6762ebbdeb5068950833de790b8013b731e81f Mon Sep 17 00:00:00 2001 From: Chris Geyer Date: Mon, 27 Jul 2026 14:47:17 +0000 Subject: [PATCH 2/3] =?UTF-8?q?feat(analyzer):=20scrape=20trackio=20experi?= =?UTF-8?q?ment=20URL=20onto=20the=20DAG=20([70])=20=E2=80=94=20scrape-onl?= =?UTF-8?q?y,=20no=20flags?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Corrected per Chris: roar does NOTHING trackio-specific at run time (no shim, no --capture-trackio/--trackio-space flags, no env). It just SCRAPES the hosted- experiment link trackio leaves behind — exactly like it scrapes the W&B URL. _extract_trackio_info reads the run identity + the Space from trackio's own project_metadata (key='space_id', where trackio persists it after a run syncs) and emits https://huggingface.co/spaces/?project=. URL-only — never the metrics (TReqs stores no customer data). Grounded against a real synced run. The wandb->trackio aliasing (making an unmodified wandb repo emit trackio) is a separate, campaign-side concern for reproducibleailist.org — NOT roar, NOT a CLI arg — so the shim + flags added earlier are removed here. Tests: 4 analyzer (project_metadata scrape + no-space + end-to-end + no-metric-leak); ruff + mypy clean. Co-Authored-By: Claude Opus 4.8 (1M context) --- roar/analyzers/experiment_trackers.py | 49 +++++----- roar/cli/commands/run.py | 25 ----- .../execution/runtime/inject/sitecustomize.py | 8 -- roar/execution/runtime/inject/trackio_shim.py | 96 ------------------- tests/execution/runtime/test_trackio_shim.py | 71 -------------- tests/unit/test_experiment_trackers.py | 45 ++++----- 6 files changed, 45 insertions(+), 249 deletions(-) delete mode 100644 roar/execution/runtime/inject/trackio_shim.py delete mode 100644 tests/execution/runtime/test_trackio_shim.py diff --git a/roar/analyzers/experiment_trackers.py b/roar/analyzers/experiment_trackers.py index 7c0b7a70..8ec4235e 100644 --- a/roar/analyzers/experiment_trackers.py +++ b/roar/analyzers/experiment_trackers.py @@ -205,22 +205,22 @@ def _extract_wandb_info(self, written_files: list, env: dict) -> dict | None: return info if len(info) > 1 else None def _extract_trackio_info(self, written_files: list, env: dict) -> dict | None: - """Extract trackio run info + the hosted-dashboard URL. + """Extract trackio run info + the hosted-dashboard URL — scrape only. trackio stores one SQLite DB per project at ``.../huggingface/trackio/ - .db``. We read *only* the run identity + resolve the public HF - Space URL — never the metrics: TReqs stores no customer data, so the DAG - carries a link to the externally-hosted experiment (like a W&B URL), and - the metrics live on the dashboard, not in lineage. - - The Space is resolved from ``ROAR_TRACKIO_SPACE_ID`` / ``TRACKIO_SPACE_ID`` - (set when roar injects the trackio shim), falling back to the ``space_id`` - recorded on the synced run. URL form: - ``https://huggingface.co/spaces/?project=``. + .db`` and, when a run syncs to a HF Space, records the Space in + ``project_metadata`` (``key='space_id'``). We read that link + the run + identity, exactly as the W&B extractor reads wandb's on-disk run URL — no + roar-side config, no env, and **never the metrics**: TReqs stores no + customer data, so the DAG carries a link to the externally-hosted + experiment, and the metrics live on the dashboard. + + URL form: ``https://huggingface.co/spaces/?project=``. + (``env`` is unused — the space comes from what trackio itself persisted.) """ + del env info: dict[str, Any] = {"tracker": "trackio"} db_paths = sorted({p for p in written_files if "trackio/" in p and p.endswith(".db")}) - space_id = env.get("ROAR_TRACKIO_SPACE_ID") or env.get("TRACKIO_SPACE_ID") or "" for db_path in db_paths: path = Path(db_path) @@ -230,22 +230,27 @@ def _extract_trackio_info(self, written_files: list, env: dict) -> dict | None: try: con = sqlite3.connect(f"file:{db_path}?mode=ro", uri=True) try: - row = con.execute( - "SELECT run_id, run_name, space_id FROM metrics ORDER BY id DESC LIMIT 1" + run = con.execute( + "SELECT run_id, run_name FROM metrics ORDER BY id DESC LIMIT 1" + ).fetchone() + # trackio writes the hosted Space here once a run syncs to it. + space = con.execute( + "SELECT value FROM project_metadata WHERE key = 'space_id' LIMIT 1" ).fetchone() finally: con.close() except sqlite3.Error: continue - if row: - info["run_id"] = row[0] - info["run_name"] = row[1] - if not space_id and row[2]: - space_id = row[2] - - if space_id and info.get("project"): - info["space_id"] = space_id - info["url"] = f"https://huggingface.co/spaces/{space_id}?project={info['project']}" + if run: + info["run_id"] = run[0] + info["run_name"] = run[1] + if space and space[0]: + info["space_id"] = space[0] + + if info.get("space_id") and info.get("project"): + info["url"] = ( + f"https://huggingface.co/spaces/{info['space_id']}?project={info['project']}" + ) return info if len(info) > 1 else None diff --git a/roar/cli/commands/run.py b/roar/cli/commands/run.py index 907329b9..f0b1b633 100644 --- a/roar/cli/commands/run.py +++ b/roar/cli/commands/run.py @@ -5,8 +5,6 @@ roar run @N [--param=value ...] """ -import os - import click from ...application.run import RunRequest, run_command @@ -86,19 +84,6 @@ def _validate_add_tags( callback=_validate_add_tags, help="Stamp KIND=VALUE onto this run's output artifacts (repeatable).", ) -@click.option( - "--capture-trackio", - is_flag=True, - help="Route the workload's W&B / MLflow logging into trackio (zero code change), " - "so it syncs to a hosted HF Space whose URL is recorded on the DAG.", -) -@click.option( - "--trackio-space", - default=None, - metavar="OWNER/NAME", - help="HF Space id to sync trackio runs to (e.g. reproducible-ai/experiments); " - "used with --capture-trackio.", -) @click.pass_obj @require_init def run( @@ -112,8 +97,6 @@ def run( hash_algorithms: tuple[str, ...], block_tags: tuple[str, ...], add_tags: tuple[str, ...], - capture_trackio: bool, - trackio_space: str | None, ) -> None: """Run a command with provenance tracking. @@ -134,14 +117,6 @@ def run( roar run @2 # Re-run DAG node 2 roar run @2 --epochs=10 # Re-run with parameter override """ - # Trackio capture: set the env the injected sitecustomize reads. The traced - # child inherits os.environ (tracer builds env = dict(os.environ)), so the - # wandb/mlflow -> trackio shim fires and the analyzer records the Space URL. - if capture_trackio: - os.environ["ROAR_CAPTURE_TRACKIO"] = "1" - if trackio_space: - os.environ["ROAR_TRACKIO_SPACE_ID"] = trackio_space - args_list = list(args) # Check for help diff --git a/roar/execution/runtime/inject/sitecustomize.py b/roar/execution/runtime/inject/sitecustomize.py index 32ac9162..11846089 100644 --- a/roar/execution/runtime/inject/sitecustomize.py +++ b/roar/execution/runtime/inject/sitecustomize.py @@ -61,14 +61,6 @@ def _prepend_roar_runtime_pythonpath() -> None: _runtime_tracker.install() -try: - from roar.execution.runtime.inject.trackio_shim import install_trackio_shim - - install_trackio_shim(os.environ) -except Exception: - pass # best-effort: capture is opt-in and must never break a run - - def _repair_runtime_in_process(expected_soabi: str) -> bool: """Install + prepend an ABI-matched runtime tree for *this* interpreter. diff --git a/roar/execution/runtime/inject/trackio_shim.py b/roar/execution/runtime/inject/trackio_shim.py deleted file mode 100644 index 3475d8fc..00000000 --- a/roar/execution/runtime/inject/trackio_shim.py +++ /dev/null @@ -1,96 +0,0 @@ -"""wandb / mlflow -> trackio shim, installed into a traced child ([70]). - -The injected ``sitecustomize`` calls :func:`install_trackio_shim` at interpreter -startup when ``ROAR_CAPTURE_TRACKIO`` is set (by ``roar run --capture-trackio``). -It aliases ``wandb`` to trackio and installs a minimal ``mlflow`` fluent-API shim, -so an *unmodified* workload's experiment logging funnels into trackio and syncs to -the configured HF Space — the experiment-tracking analyzer then records that Space -URL on the DAG (URL only; no metrics are stored). - -Extracted here (rather than inline in sitecustomize) so it can be unit-tested -without the sitecustomize's module-level runtime setup. Dependency-free and -best-effort: a missing trackio, or any error, is a silent no-op. -""" - -from __future__ import annotations - -import os -import sys -import types -from typing import Any - - -def _make_mlflow_shim(trackio: Any, space: str | None) -> types.ModuleType: - """A minimal ``mlflow`` module that funnels into trackio (metrics; params are - best-effort no-ops — trackio's config is set at init).""" - m = types.ModuleType("mlflow") - state: dict = {} - - def set_experiment(name: str | None = None, **_k: Any) -> None: - state["project"] = name - - def start_run(**_k: Any) -> Any: - trackio.init(project=state.get("project", "mlflow")) - - class _Run: - def __enter__(self) -> Any: - return self - - def __exit__(self, *_a: Any) -> None: - trackio.finish() - - return _Run() - - def noop(*_a: Any, **_k: Any) -> None: - return None - - m.__dict__.update( - { - "set_experiment": set_experiment, - "start_run": start_run, - "log_metric": lambda key, value, step=None, **k: trackio.log({key: value}, step=step), - "log_metrics": lambda d, step=None, **k: trackio.log(dict(d), step=step), - "log_param": noop, - "log_params": noop, - "set_tracking_uri": noop, - "autolog": noop, - "end_run": noop, - } - ) - return m - - -def install_trackio_shim(environ: Any = None) -> bool: - """Install the wandb/mlflow -> trackio shims if ``ROAR_CAPTURE_TRACKIO`` is set. - - Returns ``True`` iff the shims were installed. Best-effort: a missing trackio - or any error returns ``False`` and leaves the process unchanged. - """ - env = environ if environ is not None else os.environ - if env.get("ROAR_CAPTURE_TRACKIO", "") not in ("1", "true", "yes"): - return False - try: - import trackio - except Exception: - return False - space = env.get("ROAR_TRACKIO_SPACE_ID") or None - try: - # Every trackio run should sync to the campaign Space, even when the - # workload's wandb/mlflow calls don't pass a space_id. - if space: - _orig_init = trackio.init - - def _init(*args: Any, **kwargs: Any) -> Any: - kwargs.setdefault("space_id", space) - return _orig_init(*args, **kwargs) - - trackio.init = _init - - # W&B -> trackio (drop-in). setdefault: never clobber a real prior import. - sys.modules.setdefault("wandb", trackio) - # MLflow -> trackio. - if "mlflow" not in sys.modules: - sys.modules["mlflow"] = _make_mlflow_shim(trackio, space) - return True - except Exception: - return False diff --git a/tests/execution/runtime/test_trackio_shim.py b/tests/execution/runtime/test_trackio_shim.py deleted file mode 100644 index 1283289f..00000000 --- a/tests/execution/runtime/test_trackio_shim.py +++ /dev/null @@ -1,71 +0,0 @@ -"""Unit tests for the wandb/mlflow -> trackio injection shim ([70]).""" - -import builtins -import sys -import types - -import pytest - -from roar.execution.runtime.inject.trackio_shim import install_trackio_shim - - -@pytest.fixture -def fake_trackio(monkeypatch): - tk = types.ModuleType("trackio") - tk.inits = [] - tk.logged = [] - tk.init = lambda *a, **k: tk.inits.append(k) - tk.log = lambda d, step=None: tk.logged.append((dict(d), step)) - tk.finish = lambda: None - monkeypatch.setitem(sys.modules, "trackio", tk) - monkeypatch.delitem(sys.modules, "wandb", raising=False) - monkeypatch.delitem(sys.modules, "mlflow", raising=False) - return tk - - -def test_disabled_by_default(fake_trackio): - assert install_trackio_shim({}) is False - assert "wandb" not in sys.modules - - -def test_installs_wandb_and_mlflow_with_space(fake_trackio): - ok = install_trackio_shim( - {"ROAR_CAPTURE_TRACKIO": "1", "ROAR_TRACKIO_SPACE_ID": "reproducible-ai/experiments"} - ) - assert ok is True - assert sys.modules["wandb"] is fake_trackio - assert "mlflow" in sys.modules - # wandb (== trackio) init injects the configured space_id - sys.modules["wandb"].init(project="m", config={"lr": 0.01}) - assert fake_trackio.inits[-1].get("space_id") == "reproducible-ai/experiments" - - -def test_mlflow_shim_funnels_metrics(fake_trackio): - install_trackio_shim({"ROAR_CAPTURE_TRACKIO": "1", "ROAR_TRACKIO_SPACE_ID": "org/space"}) - mlflow = sys.modules["mlflow"] - mlflow.set_experiment("exp1") - with mlflow.start_run(): - mlflow.log_metric("loss", 0.5, step=1) - mlflow.log_params({"lr": 0.01}) # no-op; must not raise - assert fake_trackio.inits[-1].get("project") == "exp1" - assert ({"loss": 0.5}, 1) in fake_trackio.logged - - -def test_does_not_clobber_a_real_wandb(fake_trackio): - sentinel = types.ModuleType("wandb") - sys.modules["wandb"] = sentinel - install_trackio_shim({"ROAR_CAPTURE_TRACKIO": "1"}) - assert sys.modules["wandb"] is sentinel # setdefault preserved the prior import - - -def test_missing_trackio_is_a_noop(monkeypatch): - monkeypatch.delitem(sys.modules, "trackio", raising=False) - real_import = builtins.__import__ - - def fake_import(name, *a, **k): - if name == "trackio": - raise ImportError("no trackio") - return real_import(name, *a, **k) - - monkeypatch.setattr(builtins, "__import__", fake_import) - assert install_trackio_shim({"ROAR_CAPTURE_TRACKIO": "1"}) is False diff --git a/tests/unit/test_experiment_trackers.py b/tests/unit/test_experiment_trackers.py index ef196979..f32b277c 100644 --- a/tests/unit/test_experiment_trackers.py +++ b/tests/unit/test_experiment_trackers.py @@ -1,7 +1,8 @@ """Unit tests for the trackio branch of ExperimentTrackerAnalyzer ([70]). -trackio stores one SQLite DB per project; roar records only the run identity + the -hosted HF-Space URL (never the metrics — TReqs stores no customer data). +roar *scrapes* trackio exactly like W&B: it reads the run identity + the hosted HF +Space URL that trackio itself persisted (``project_metadata.space_id``), and never +the metrics (TReqs stores no customer data). No roar-side config, env, or flags. """ import sqlite3 @@ -11,7 +12,11 @@ def _make_trackio_db(dirpath: Path, project: str, space_id: str | None = None) -> str: - """Create a minimal trackio-shaped SQLite DB, return its path.""" + """Create a minimal trackio-shaped SQLite DB, return its path. + + Mirrors the real schema: a ``metrics`` table and a ``project_metadata`` table + into which trackio writes ``space_id`` once a run syncs to a HF Space. + """ db = dirpath / "huggingface" / "trackio" / f"{project}.db" db.parent.mkdir(parents=True, exist_ok=True) con = sqlite3.connect(str(db)) @@ -19,31 +24,23 @@ def _make_trackio_db(dirpath: Path, project: str, space_id: str | None = None) - "CREATE TABLE metrics (id INTEGER PRIMARY KEY, run_id TEXT, timestamp TEXT, " "run_name TEXT, step INTEGER, metrics TEXT, log_id TEXT, space_id TEXT)" ) + con.execute("CREATE TABLE project_metadata (key TEXT, value TEXT)") for step in range(3): con.execute( - "INSERT INTO metrics (run_id, timestamp, run_name, step, metrics, log_id, space_id) " - "VALUES (?,?,?,?,?,?,?)", - ( - "run123", - "2026-07-27T00:00:00Z", - "brave-run-0", - step, - f'{{"loss": {round(1.0 / (step + 1), 4)}}}', - f"log{step}", - space_id, - ), + "INSERT INTO metrics (run_id, run_name, step, metrics) VALUES (?,?,?,?)", + ("run123", "brave-run-0", step, f'{{"loss": {round(1.0 / (step + 1), 4)}}}'), ) + if space_id: + con.execute("INSERT INTO project_metadata (key, value) VALUES ('space_id', ?)", (space_id,)) con.commit() con.close() return str(db) class TestTrackioExtractor: - def test_url_from_env_space_id(self, tmp_path: Path): - db = _make_trackio_db(tmp_path, "minimind-o") - info = ExperimentTrackerAnalyzer()._extract_trackio_info( - [db], {"ROAR_TRACKIO_SPACE_ID": "reproducible-ai/experiments"} - ) + def test_url_from_persisted_space_metadata(self, tmp_path: Path): + db = _make_trackio_db(tmp_path, "minimind-o", space_id="reproducible-ai/experiments") + info = ExperimentTrackerAnalyzer()._extract_trackio_info([db], {}) assert info is not None assert info["tracker"] == "trackio" assert info["project"] == "minimind-o" @@ -54,14 +51,8 @@ def test_url_from_env_space_id(self, tmp_path: Path): == "https://huggingface.co/spaces/reproducible-ai/experiments?project=minimind-o" ) - def test_space_id_from_db_column(self, tmp_path: Path): - db = _make_trackio_db(tmp_path, "yolo", space_id="reproducible-ai/experiments") - info = ExperimentTrackerAnalyzer()._extract_trackio_info([db], {}) # no env - assert info is not None - assert info["url"].endswith("/reproducible-ai/experiments?project=yolo") - - def test_no_space_id_records_identity_but_no_url(self, tmp_path: Path): - db = _make_trackio_db(tmp_path, "cosyvoice") # space_id None + no env + def test_no_space_metadata_records_identity_but_no_url(self, tmp_path: Path): + db = _make_trackio_db(tmp_path, "cosyvoice") # local-only run, never synced info = ExperimentTrackerAnalyzer()._extract_trackio_info([db], {}) assert info is not None assert info["project"] == "cosyvoice" From a511f40c35eec98eb6e97bb5ff7729e51ef20d78 Mon Sep 17 00:00:00 2001 From: Chris Geyer Date: Mon, 27 Jul 2026 18:01:54 +0000 Subject: [PATCH 3/3] fix(trackio): one record per project DB + schema-tolerant scrape ([70]) Fixes two QA-found bugs in the trackio experiment scraper: - F1 (cross-DB contamination): the extractor reused a single shared dict across all *.db files, so a run touching >1 trackio project could splice one project's Space onto another's name (.../alpha-space?project=zebra). Now emits one record per project DB and builds each URL per-DB. - F2 (schema brittleness): the run-identity and space-id lookups shared one try-block and hardcoded SELECT run_id, run_name, so an older/imported trackio schema (no run_id column, or no project_metadata table) discarded BOTH the identity and the URL. The two lookups are now independent and the run-identity query adapts to a run_id-less metrics table. Metric-leak invariant preserved (only run_id/run_name read, never metrics). Co-Authored-By: Claude Opus 4.8 (1M context) --- roar/analyzers/experiment_trackers.py | 96 ++++++++++++++++------ tests/unit/test_experiment_trackers.py | 107 ++++++++++++++++++++----- 2 files changed, 157 insertions(+), 46 deletions(-) diff --git a/roar/analyzers/experiment_trackers.py b/roar/analyzers/experiment_trackers.py index 8ec4235e..c6cdbe8e 100644 --- a/roar/analyzers/experiment_trackers.py +++ b/roar/analyzers/experiment_trackers.py @@ -74,7 +74,12 @@ def analyze(self, context: dict) -> dict | None: for tracker in trackers_found: run_info = self._extract_run_info(tracker, written, env) if run_info: - results["runs"].append(run_info) + # trackio reports one run per project DB (a run may touch + # several); the other trackers return a single run dict. + if isinstance(run_info, list): + results["runs"].extend(run_info) + else: + results["runs"].append(run_info) # Add ignore patterns for this tracker for pattern in self.IGNORE_PATTERNS: @@ -98,8 +103,12 @@ def analyze(self, context: dict) -> dict | None: return results if results["trackers_detected"] else None - def _extract_run_info(self, tracker: str, written_files: list, env: dict) -> dict | None: - """Extract run URL and metadata for a specific tracker.""" + def _extract_run_info(self, tracker: str, written_files: list, env: dict) -> dict | list | None: + """Extract run URL and metadata for a specific tracker. + + Most trackers return a single run dict; ``trackio`` returns a list of + per-project run dicts (or ``None`` when nothing usable was found). + """ if tracker == "wandb": return self._extract_wandb_info(written_files, env) elif tracker == "mlflow": @@ -204,7 +213,7 @@ def _extract_wandb_info(self, written_files: list, env: dict) -> dict | None: return info if len(info) > 1 else None - def _extract_trackio_info(self, written_files: list, env: dict) -> dict | None: + def _extract_trackio_info(self, written_files: list, env: dict) -> list | None: """Extract trackio run info + the hosted-dashboard URL — scrape only. trackio stores one SQLite DB per project at ``.../huggingface/trackio/ @@ -215,44 +224,79 @@ def _extract_trackio_info(self, written_files: list, env: dict) -> dict | None: customer data, so the DAG carries a link to the externally-hosted experiment, and the metrics live on the dashboard. - URL form: ``https://huggingface.co/spaces/?project=``. + Returns ONE record per project DB (never splicing one project's identity + onto another's Space), or ``None`` if no DB yielded anything usable. Each + record's URL is ``https://huggingface.co/spaces/?project=``. (``env`` is unused — the space comes from what trackio itself persisted.) """ del env - info: dict[str, Any] = {"tracker": "trackio"} db_paths = sorted({p for p in written_files if "trackio/" in p and p.endswith(".db")}) + runs: list[dict[str, Any]] = [] for db_path in db_paths: path = Path(db_path) - info["project"] = path.stem # .db -> if not path.exists(): continue + info: dict[str, Any] = {"tracker": "trackio", "project": path.stem} try: con = sqlite3.connect(f"file:{db_path}?mode=ro", uri=True) - try: - run = con.execute( - "SELECT run_id, run_name FROM metrics ORDER BY id DESC LIMIT 1" - ).fetchone() - # trackio writes the hosted Space here once a run syncs to it. - space = con.execute( - "SELECT value FROM project_metadata WHERE key = 'space_id' LIMIT 1" - ).fetchone() - finally: - con.close() except sqlite3.Error: continue + try: + # Two INDEPENDENT lookups: an older/imported trackio schema may + # lack a ``run_id`` column or the ``project_metadata`` table, so + # a failure of one must not discard the other. + run = self._trackio_run_identity(con) + space = self._trackio_space_id(con) + finally: + con.close() + if run: - info["run_id"] = run[0] - info["run_name"] = run[1] - if space and space[0]: - info["space_id"] = space[0] + info["run_id"], info["run_name"] = run + if space: + info["space_id"] = space + info["url"] = f"https://huggingface.co/spaces/{space}?project={info['project']}" - if info.get("space_id") and info.get("project"): - info["url"] = ( - f"https://huggingface.co/spaces/{info['space_id']}?project={info['project']}" - ) + # Keep only DBs that yielded a run identity and/or a hosted Space + # (more than the always-present tracker+project keys). + if len(info) > 2: + runs.append(info) - return info if len(info) > 1 else None + return runs or None + + @staticmethod + def _trackio_run_identity(con: "sqlite3.Connection") -> tuple | None: + """``(run_id, run_name)`` of the most recent metric row, tolerating + older schemas whose ``metrics`` table lacks a ``run_id`` column (trackio + produces these when importing TensorBoard/older runs).""" + try: + cols = {row[1] for row in con.execute("PRAGMA table_info(metrics)").fetchall()} + except sqlite3.Error: + return None + if not cols or not ({"run_id", "run_name"} & cols): + return None + run_id_expr = "run_id" if "run_id" in cols else "NULL" + run_name_expr = "run_name" if "run_name" in cols else "NULL" + try: + row = con.execute( + f"SELECT {run_id_expr}, {run_name_expr} FROM metrics ORDER BY rowid DESC LIMIT 1" + ).fetchone() + except sqlite3.Error: + return None + if not row or (row[0] is None and row[1] is None): + return None + return row[0], row[1] + + @staticmethod + def _trackio_space_id(con: "sqlite3.Connection") -> str | None: + """The hosted HF Space id trackio persisted once a run synced, or None.""" + try: + row = con.execute( + "SELECT value FROM project_metadata WHERE key = 'space_id' LIMIT 1" + ).fetchone() + except sqlite3.Error: + return None + return row[0] if row and row[0] else None def _extract_mlflow_info(self, written_files: list, env: dict) -> dict | None: """Extract MLflow run info from local files.""" diff --git a/tests/unit/test_experiment_trackers.py b/tests/unit/test_experiment_trackers.py index f32b277c..52da312d 100644 --- a/tests/unit/test_experiment_trackers.py +++ b/tests/unit/test_experiment_trackers.py @@ -3,6 +3,10 @@ roar *scrapes* trackio exactly like W&B: it reads the run identity + the hosted HF Space URL that trackio itself persisted (``project_metadata.space_id``), and never the metrics (TReqs stores no customer data). No roar-side config, env, or flags. + +The extractor returns ONE record per project DB (a run may touch several projects), +and tolerates older/imported trackio schemas that lack a ``run_id`` column or the +``project_metadata`` table. """ import sqlite3 @@ -11,27 +15,53 @@ from roar.analyzers.experiment_trackers import ExperimentTrackerAnalyzer -def _make_trackio_db(dirpath: Path, project: str, space_id: str | None = None) -> str: +def _make_trackio_db( + dirpath: Path, + project: str, + space_id: str | None = None, + *, + with_project_metadata: bool = True, + with_run_id: bool = True, + run_id: str = "run123", + run_name: str = "brave-run-0", +) -> str: """Create a minimal trackio-shaped SQLite DB, return its path. Mirrors the real schema: a ``metrics`` table and a ``project_metadata`` table - into which trackio writes ``space_id`` once a run syncs to a HF Space. + into which trackio writes ``space_id`` once a run syncs to a HF Space. The + flags let us reproduce older/imported schemas: ``with_run_id=False`` drops the + ``run_id`` column (trackio's TensorBoard import), ``with_project_metadata=False`` + drops the whole table (a never-synced/older DB). """ db = dirpath / "huggingface" / "trackio" / f"{project}.db" db.parent.mkdir(parents=True, exist_ok=True) con = sqlite3.connect(str(db)) - con.execute( - "CREATE TABLE metrics (id INTEGER PRIMARY KEY, run_id TEXT, timestamp TEXT, " - "run_name TEXT, step INTEGER, metrics TEXT, log_id TEXT, space_id TEXT)" - ) - con.execute("CREATE TABLE project_metadata (key TEXT, value TEXT)") - for step in range(3): + if with_run_id: + con.execute( + "CREATE TABLE metrics (id INTEGER PRIMARY KEY, run_id TEXT, timestamp TEXT, " + "run_name TEXT, step INTEGER, metrics TEXT, log_id TEXT, space_id TEXT)" + ) + for step in range(3): + con.execute( + "INSERT INTO metrics (run_id, run_name, step, metrics) VALUES (?,?,?,?)", + (run_id, run_name, step, f'{{"loss": {round(1.0 / (step + 1), 4)}}}'), + ) + else: con.execute( - "INSERT INTO metrics (run_id, run_name, step, metrics) VALUES (?,?,?,?)", - ("run123", "brave-run-0", step, f'{{"loss": {round(1.0 / (step + 1), 4)}}}'), + "CREATE TABLE metrics (id INTEGER PRIMARY KEY, timestamp TEXT, " + "run_name TEXT, step INTEGER, metrics TEXT)" ) - if space_id: - con.execute("INSERT INTO project_metadata (key, value) VALUES ('space_id', ?)", (space_id,)) + for step in range(3): + con.execute( + "INSERT INTO metrics (run_name, step, metrics) VALUES (?,?,?)", + (run_name, step, f'{{"loss": {round(1.0 / (step + 1), 4)}}}'), + ) + if with_project_metadata: + con.execute("CREATE TABLE project_metadata (key TEXT, value TEXT)") + if space_id: + con.execute( + "INSERT INTO project_metadata (key, value) VALUES ('space_id', ?)", (space_id,) + ) con.commit() con.close() return str(db) @@ -40,8 +70,9 @@ def _make_trackio_db(dirpath: Path, project: str, space_id: str | None = None) - class TestTrackioExtractor: def test_url_from_persisted_space_metadata(self, tmp_path: Path): db = _make_trackio_db(tmp_path, "minimind-o", space_id="reproducible-ai/experiments") - info = ExperimentTrackerAnalyzer()._extract_trackio_info([db], {}) - assert info is not None + infos = ExperimentTrackerAnalyzer()._extract_trackio_info([db], {}) + assert infos is not None and len(infos) == 1 + info = infos[0] assert info["tracker"] == "trackio" assert info["project"] == "minimind-o" assert info["run_id"] == "run123" @@ -53,10 +84,11 @@ def test_url_from_persisted_space_metadata(self, tmp_path: Path): def test_no_space_metadata_records_identity_but_no_url(self, tmp_path: Path): db = _make_trackio_db(tmp_path, "cosyvoice") # local-only run, never synced - info = ExperimentTrackerAnalyzer()._extract_trackio_info([db], {}) - assert info is not None - assert info["project"] == "cosyvoice" - assert "url" not in info + infos = ExperimentTrackerAnalyzer()._extract_trackio_info([db], {}) + assert infos is not None and len(infos) == 1 + assert infos[0]["project"] == "cosyvoice" + assert infos[0]["run_id"] == "run123" + assert "url" not in infos[0] def test_detection_and_analyze_end_to_end(self, tmp_path: Path): db = _make_trackio_db(tmp_path, "sam2", space_id="reproducible-ai/experiments") @@ -72,5 +104,40 @@ def test_detection_and_analyze_end_to_end(self, tmp_path: Path): def test_url_only_no_metric_values_leak(self, tmp_path: Path): """No-data-storage invariant: metric VALUES must never reach lineage.""" db = _make_trackio_db(tmp_path, "m", space_id="reproducible-ai/experiments") - info = ExperimentTrackerAnalyzer()._extract_trackio_info([db], {}) - assert "loss" not in str(info) + infos = ExperimentTrackerAnalyzer()._extract_trackio_info([db], {}) + assert "loss" not in str(infos) + + def test_multi_project_no_cross_contamination(self, tmp_path: Path): + """F1: a run touching >1 project must never splice one project's Space + onto another project's name.""" + alpha = _make_trackio_db(tmp_path, "alpha", space_id="team/alpha-space") + zebra = _make_trackio_db(tmp_path, "zebra") # local-only, no Space + infos = ExperimentTrackerAnalyzer()._extract_trackio_info([alpha, zebra], {}) + assert infos is not None and len(infos) == 2 + by_project = {i["project"]: i for i in infos} + assert ( + by_project["alpha"]["url"] + == "https://huggingface.co/spaces/team/alpha-space?project=alpha" + ) + # zebra was never synced → no URL, and alpha's Space never leaks onto it. + assert "url" not in by_project["zebra"] + assert not any("project=zebra" in i.get("url", "") for i in infos) + + def test_missing_project_metadata_keeps_run_identity(self, tmp_path: Path): + """F2: a DB without the project_metadata table must still record the run + identity (the two lookups are independent).""" + db = _make_trackio_db(tmp_path, "beta", with_project_metadata=False) + infos = ExperimentTrackerAnalyzer()._extract_trackio_info([db], {}) + assert infos is not None and len(infos) == 1 + assert infos[0]["run_name"] == "brave-run-0" + assert "url" not in infos[0] + + def test_runid_less_schema_still_records_url(self, tmp_path: Path): + """F2: an older/imported schema without a run_id column must still yield + the hosted-Space URL from project_metadata.""" + db = _make_trackio_db(tmp_path, "omega", space_id="team/omega-space", with_run_id=False) + infos = ExperimentTrackerAnalyzer()._extract_trackio_info([db], {}) + assert infos is not None and len(infos) == 1 + assert infos[0]["url"] == "https://huggingface.co/spaces/team/omega-space?project=omega" + assert infos[0]["run_name"] == "brave-run-0" + assert infos[0].get("run_id") is None