Evaluate the output file produced by otbench run

projects/otb/generate.py

@@ -16,6 +16,7 @@
 from datasets import load_from_disk
 from tqdm import tqdm
 from utils import call_llm
+from otbench.paths import output_path
 
 parser = argparse.ArgumentParser()
 parser.add_argument("--model", type=str, default="gpt-4o")
@@ -138,19 +139,9 @@ def process_row(row, model_name):
 df["tokens"] = all_tokens
 df["prompts_used"] = prompts_used
 
-if args.enable_nothink:
-    OUTPUT_FILE = OUTPUT.replace("{{model}}", MODEL_NAME.replace("/", "-")).replace(
-        ".jsonl", "-nothink.jsonl"
-    )
-else:
-    OUTPUT_FILE = OUTPUT.replace("{{model}}", MODEL_NAME.replace("/", "-"))
-
-if args.temperature != 0.6:
-    OUTPUT_FILE = OUTPUT_FILE.replace(".jsonl", f"-temp={args.temperature}.jsonl")
-
-if args.num_gens != 8:
-    OUTPUT_FILE = OUTPUT_FILE.replace(".jsonl", f"-num_gens={args.num_gens}.jsonl")
-
+OUTPUT_FILE = output_path(
+    OUTPUT, MODEL_NAME, args.enable_nothink, args.temperature, args.num_gens
+)
 
 print(f"Saving to {OUTPUT_FILE}")
 print(f"To postprocess, run `otbench eval {OUTPUT_FILE} --model {MODEL_NAME}`")

projects/otb/otbench/cli.py

@@ -10,6 +10,8 @@
 import os
 import sys
 
+from .paths import output_path
+
 ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 
 
@@ -103,7 +105,9 @@ def cmd_run_all(args: argparse.Namespace) -> int:
         if ret != 0:
             return ret
         output = args.output if args.output else "final_outputs/otbench/{{model}}.jsonl"
-        output = output.replace("{{model}}", args.model.replace("/", "-"))
+        output = output_path(
+            output, args.model, args.enable_nothink, args.temperature, args.num_gens
+        )
         return cmd_eval(argparse.Namespace(input=output, model=args.model))
 
     p_runall = sub.add_parser("run")

projects/otb/otbench/paths.py

@@ -0,0 +1,12 @@
+"""Output filenames shared by generation and the end-to-end CLI."""
+
+
+def output_path(output, model, enable_nothink=False, temperature=0.6, num_gens=8):
+    path = output.replace("{{model}}", model.replace("/", "-"))
+    if enable_nothink:
+        path = path.replace(".jsonl", "-nothink.jsonl")
+    if temperature != 0.6:
+        path = path.replace(".jsonl", f"-temp={temperature}.jsonl")
+    if num_gens != 8:
+        path = path.replace(".jsonl", f"-num_gens={num_gens}.jsonl")
+    return path

projects/otb/tests/test_run_output.py

@@ -0,0 +1,79 @@
+import contextlib
+import importlib
+import io
+from pathlib import Path
+import runpy
+import shlex
+import sys
+import tempfile
+import types
+import unittest
+from unittest.mock import patch
+
+ROOT = Path(__file__).resolve().parents[1]
+
+
+class RegressionTests(unittest.TestCase):
+    def test_run_evaluates_the_file_generation_writes(self):
+        class Frame:
+            def iterrows(self): return iter([])
+            def __setitem__(self, key, value): pass
+            def to_json(self, path, **kwargs): Path(path).write_text("")
+
+        class Pool:
+            def __init__(self, **kwargs): pass
+            def __enter__(self): return self
+            def __exit__(self, *args): pass
+            def map(self, fn, rows): return []
+
+        package = types.ModuleType("otbench")
+        package.__path__ = [str(ROOT / "otbench")]
+        modules = {
+            "otbench": package,
+            "datasets": types.SimpleNamespace(load_from_disk=lambda path: {
+                "train": types.SimpleNamespace(to_pandas=lambda: Frame())}),
+            "pandas": types.ModuleType("pandas"),
+            "requests": types.ModuleType("requests"),
+            "tqdm": types.SimpleNamespace(tqdm=lambda rows: rows),
+            "utils": types.SimpleNamespace(call_llm=lambda *args: self.fail("model call")),
+            "multiprocessing": types.SimpleNamespace(Pool=Pool),
+        }
+        with patch.dict(sys.modules, modules):
+            for name in ["otbench.cli", "otbench.paths"]:
+                sys.modules.pop(name, None)
+            cli = importlib.import_module("otbench.cli")
+            cases = [
+                (False, 0.6, 8, ""), (False, 0.6, 2, "-num_gens=2"),
+                (False, 0.2, 8, "-temp=0.2"), (False, 0.2, 2, "-temp=0.2-num_gens=2"),
+                (True, 0.6, 8, "-nothink"), (True, 0.6, 2, "-nothink-num_gens=2"),
+                (True, 0.2, 8, "-nothink-temp=0.2"),
+                (True, 0.2, 2, "-nothink-temp=0.2-num_gens=2"),
+            ]
+            for nothink, temperature, num_gens, suffix in cases:
+                with self.subTest(nothink=nothink, temperature=temperature, num_gens=num_gens):
+                    with tempfile.TemporaryDirectory() as tmp:
+                        output = str(Path(tmp) / "outputs" / "{{model}}.jsonl")
+                        argv = ["run", "--model", "org/model", "--output", output,
+                                "--temperature", str(temperature), "--num_gens", str(num_gens)]
+                        if nothink:
+                            argv.append("--enable_nothink")
+                        args = cli.build_parser().parse_args(argv)
+                        evaluated = []
+
+                        def run(command):
+                            parts = shlex.split(command)
+                            if parts[1].endswith("generate.py"):
+                                with patch.object(sys, "argv", parts[1:]), contextlib.redirect_stdout(io.StringIO()):
+                                    runpy.run_path(str(ROOT / "generate.py"), run_name="__main__")
+                            elif parts[1].endswith("eval.py"):
+                                evaluated.append(parts[2])
+                                self.assertTrue(Path(parts[2]).is_file(), parts[2])
+                            return 0
+
+                        with patch.object(cli, "_run", run):
+                            self.assertEqual(args.func(args), 0)
+                        self.assertEqual(evaluated, [str(Path(tmp) / "outputs" / ("org-model" + suffix + ".jsonl"))])
+
+
+if __name__ == "__main__":
+    unittest.main()