Regression tests: Handle no packages gracefully

Before, when running regression tests for a certain version/platform
combination that doesn't have any cases, mypy is run with invalid
arguments (no files), and the tests fail.

Now, a message is printed instead (unless the verbosity is `QUIET`) and
the tests succeed.

Author: srittau

lib/ts_utils/utils.py

@@ -83,6 +83,10 @@ def print_warning(message: str) -> None:
     print(colored(message, "yellow"))
 
 
+def print_skipped(message: str) -> None:
+    print(colored(message, "yellow"))
+
+
 def print_error(error: str, end: str = "\n", fix_path: tuple[str, str] = ("", "")) -> None:
     error_split = error.split("\n")
     old, new = fix_path

tests/regr_test.py

@@ -33,6 +33,7 @@
     get_all_testcase_directories,
     get_mypy_req,
     print_error,
+    print_skipped,
     venv_python,
 )
 
@@ -167,7 +168,7 @@ def setup_testcase_dir(package: DistributionTests, tempdir: Path, verbosity: Ver
 
 def run_testcases(
     package: DistributionTests, version: str, platform: str, *, tempdir: Path, verbosity: Verbosity
-) -> subprocess.CompletedProcess[str]:
+) -> subprocess.CompletedProcess[str] | None:
     env_vars = dict(os.environ)
     new_test_case_dir = tempdir / TEST_CASES_DIR
 
@@ -177,7 +178,6 @@ def run_testcases(
         configurations = mypy_configuration_from_distribution(package.name)
 
     with temporary_mypy_config_file(configurations) as temp_config:
-
         # "--enable-error-code ignore-without-code" is purposefully omitted.
         # See https://github.com/python/typeshed/pull/8083
         flags = [
@@ -216,18 +216,22 @@ def run_testcases(
 
         flags.extend(["--custom-typeshed-dir", str(custom_typeshed)])
 
-        # If the test-case filename ends with -py39,
-        # only run the test if --python-version was set to 3.9 or higher (for example)
+        # If the test-case filename ends with e.g. -py314,
+        # only run the test if --python-version was set to 3.14 or higher (for example)
+        files: list[str] = []
         for path in new_test_case_dir.rglob("*.py"):
-            if match := re.fullmatch(r".*-py3(\d{1,2})", path.stem):
+            if match := re.fullmatch(r".*-py3(\d\d)", path.stem):
                 minor_version_required = int(match[1])
                 assert f"3.{minor_version_required}" in SUPPORTED_VERSIONS
                 python_minor_version = int(version.split(".")[1])
                 if minor_version_required > python_minor_version:
                     continue
-            flags.append(str(path))
+            files.append(str(path))
+
+        if len(files) == 0:
+            return None
 
-        mypy_command = [python_exe, "-m", "mypy", *flags]
+        mypy_command = [python_exe, "-m", "mypy", *flags, *files]
         if verbosity is Verbosity.VERBOSE:
             description = f"{package.name}/{version}/{platform}"
             msg = f"{description}: {mypy_command=}\n"
@@ -243,13 +247,20 @@ def run_testcases(
 @dataclass(frozen=True)
 class Result:
     code: int
+
+    def print_description(self, verbosity: Verbosity) -> None:
+        raise NotImplementedError
+
+
+@dataclass(frozen=True)
+class RunResult(Result):
     command_run: str
     stderr: str
     stdout: str
     test_case_dir: Path
     tempdir: Path
 
-    def print_description(self) -> None:
+    def print_description(self, _: Verbosity) -> None:
         if self.code:
             print(f"{self.command_run}:", end=" ")
             print_error("FAILURE\n")
@@ -260,6 +271,17 @@ def print_description(self) -> None:
                 print_error(self.stdout, fix_path=replacements)
 
 
+@dataclass(frozen=True)
+class NoTestsResult(Result):
+    package: str
+    version: str
+    platform: str
+
+    def print_description(self, verbosity: Verbosity) -> None:
+        if verbosity != Verbosity.QUIET:
+            print_skipped(f"No test cases found for {self.package!r} on Python {self.version} for platform {self.platform!r}.")
+
+
 def test_testcase_directory(
     package: DistributionTests, version: str, platform: str, *, verbosity: Verbosity, tempdir: Path
 ) -> Result:
@@ -268,12 +290,15 @@ def test_testcase_directory(
     if verbosity > Verbosity.QUIET:
         _PRINT_QUEUE.put(f"Running {msg}...")
 
-    proc_info = run_testcases(package=package, version=version, platform=platform, tempdir=tempdir, verbosity=verbosity)
-    return Result(
-        code=proc_info.returncode,
+    result = run_testcases(package=package, version=version, platform=platform, tempdir=tempdir, verbosity=verbosity)
+    if result is None:
+        return NoTestsResult(0, package.name, version, platform)
+
+    return RunResult(
+        code=result.returncode,
         command_run=msg,
-        stderr=proc_info.stderr,
-        stdout=proc_info.stdout,
+        stderr=result.stderr,
+        stdout=result.stdout,
         test_case_dir=package.test_cases_path,
         tempdir=tempdir,
     )
@@ -399,7 +424,7 @@ def main() -> ReturnCode:
     print()
 
     for result in results:
-        result.print_description()
+        result.print_description(verbosity)
 
     code = max(result.code for result in results)