diff --git a/Project.toml b/Project.toml index 4e8f2ac..ad8c60f 100644 --- a/Project.toml +++ b/Project.toml @@ -1,12 +1,13 @@ name = "SciMLTesting" uuid = "09d9d899-5365-40a9-917a-5f67fddea283" authors = ["SciML"] -version = "1.8.0" +version = "2.0.0" [deps] Aqua = "4c88cf16-eb10-579e-8560-4a9242c79595" ExplicitImports = "7d51a73a-1435-4ff3-83d9-f097790105c7" Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" +REPL = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb" SafeTestsets = "1bc83da4-3b8d-516f-aca4-4fe02f6d838f" TOML = "fa267f1f-6049-4f14-aa54-33bafae1ed76" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" diff --git a/README.md b/README.md index b088009..68290b1 100644 --- a/README.md +++ b/README.md @@ -49,7 +49,9 @@ test = ["Test", "SciMLTesting", ...] | `current_group(; env = "GROUP", default = "All")` | Read the test-group env var, defaulting to `"All"` (empty string also normalizes to the default). | | `activate_group_env(group_dir; parent, develop, instantiate, develop_sources)` | `Pkg.activate` a per-group `Project.toml`, `develop` the parent package(s) by path, backport `[sources]`, `instantiate`. | | `develop_sources!(group_dir; parent)` | On Julia < 1.11, `Pkg.develop` the env's `[sources]` path graph (recursively); a no-op on 1.11+. | -| `run_qa(pkg; Aqua, JET, ExplicitImports, aqua, jet, explicit_imports, aqua_broken, jet_broken, ei_broken, ...)` | Run the standard Aqua/JET/ExplicitImports QA body. Aqua + ExplicitImports come from SciMLTesting's deps (always available; `aqua` on by default, `explicit_imports` opt-in); `using JET` registers JET via its weakdep extension and turns the JET check on. The `*_broken` kwargs mark known-broken findings as `@test_broken` (see [Known-broken findings](#known-broken-findings-aqua_broken-jet_broken-ei_broken)). | +| `run_qa(pkg; Aqua, JET, ExplicitImports, aqua, jet, explicit_imports, api_docs, aqua_broken, jet_broken, ei_broken, ...)` | Run the standard Aqua/JET/ExplicitImports QA body, plus the public-API documentation check. Aqua + ExplicitImports come from SciMLTesting's deps (always available; `aqua` on by default, `explicit_imports` opt-in); `using JET` registers JET via its weakdep extension and turns the JET check on; `api_docs` is **on by default** and runs `run_api_docs` (configure via `api_docs_kwargs`, or `api_docs = false` to skip). The `*_broken` kwargs mark known-broken findings as `@test_broken` (see [Known-broken findings](#known-broken-findings-aqua_broken-jet_broken-ei_broken)). | +| `run_api_docs(pkg; docstrings = true, rendered = false, docs_src, ignore, rendered_ignore, docstrings_broken, rendered_broken)` | Assert every exported/`public` name of `pkg` has a docstring (and, opt-in, is rendered in a `@docs` block under `docs/src`). The shared replacement for per-repo `test/QA/public_api_docs.jl` files. | +| `public_api_names(pkg)` | The sorted public API of `pkg` (exported names, plus `public` names on Julia ≥ 1.11), with the module's own name dropped. | | `detect_sublibrary_group(group, lib_dir; default_group = "Core")` | Map a `GROUP` value to a `(sublibrary, test_group)` pair for a monorepo. | All are documented with full docstrings; `?run_tests` etc. at the REPL. @@ -304,6 +306,45 @@ run_qa(MyPackage; explicit_imports = true, ei_kwargs = (; all_qualified_accesses_are_public = (; ignore = (:internal_dep_name,)))) ``` +### Public-API documentation check (`run_api_docs` / `api_docs = true`) + +Several SciML repos had grown a hand-copied `test/QA/public_api_docs.jl` asserting that +every exported name has a docstring (and is rendered in the manual). `run_api_docs` +replaces those per-repo files with one shared, maintained helper. It runs **by default +inside `run_qa`** (`api_docs = true`), so a plain `run_qa(MyPackage)` already enforces +the docstring check — configure it with `api_docs_kwargs`, or pass `api_docs = false` to +skip: + +```julia +using SciMLTesting, MyPackage + +# In the QA body — the docstring check runs by default: +run_qa(MyPackage; explicit_imports = true) + +# Also require each public name is rendered in a docs/src @docs block: +run_qa(MyPackage; explicit_imports = true, api_docs_kwargs = (; rendered = true)) + +# Standalone (outside run_qa), e.g. as its own QA file: +run_api_docs(MyPackage) # every exported/`public` name has a docstring +run_api_docs(MyPackage; rendered = true) # also require each is in a docs/src @docs block +``` + + * **`docstrings`** (default `true`) — every name in `public_api_names(pkg)` has a + docstring. A re-exported name documented in its defining package counts as + documented (the check follows the binding), so you are not forced to redocument + dependency re-exports. + * **`rendered`** (default `false`, opt-in) — every public name appears in a + ` ```@docs ` block under `docs_src` (defaults to `/docs/src`). A + ` ```@autodocs ` block satisfies it wholesale. Opt-in because not every repo has a + resolvable local manual (monorepos with shared docs, packages with no manual). + * **`ignore` / `rendered_ignore`** — names to exclude (e.g. an un-documentable + re-export), with a comment pointing at the tracking issue. + * **`docstrings_broken` / `rendered_broken`** — mark the check `@test_broken` for a + repo mid-migration; auto-flags an `Unexpected Pass` once the API is fully documented. + +On the Julia 1.10 LTS `public_api_names` returns only the exported names (the `public` +keyword is 1.11+), so no per-repo `if VERSION` guards are needed. + ### Known-broken findings (`aqua_broken`, `jet_broken`, `ei_broken`) When converting a hand-rolled `qa.jl` to `run_qa` would otherwise re-red a repo that diff --git a/src/SciMLTesting.jl b/src/SciMLTesting.jl index 4c47e6d..a301a2c 100644 --- a/src/SciMLTesting.jl +++ b/src/SciMLTesting.jl @@ -22,7 +22,13 @@ setup. The top-level [`run_tests`](@ref) dispatcher owns the whole `runtests.jl` control flow, so a repo replaces its hand-written `if GROUP == ...` ladder with a single declarative call. -It stays light: beyond the standard libraries `Pkg`, `TOML`, and `Test` and the +The same QA aggregator also owns the public-API *documentation* check: [`run_api_docs`](@ref) +asserts every exported/`public` name has a docstring (and, opt-in, is rendered in the +manual). It runs by default inside [`run_qa`](@ref) (`api_docs = true`), so repos drop +their hand-rolled `test/QA/public_api_docs.jl` and get the check for free from a plain +`run_qa(MyPkg)`. + +It stays light: beyond the standard libraries `Pkg`, `TOML`, `Test`, and `REPL` and the tiny `SafeTestsets` package (whose `@safetestset` macro runs each file-path group body in its own isolated module), it depends only on the lightweight, broad-compat QA tools `Aqua` and `ExplicitImports`, so [`run_qa`](@ref) runs them without the @@ -40,10 +46,15 @@ using Test: Test, @testset, @test, @test_broken using SafeTestsets: SafeTestsets, @safetestset using Aqua: Aqua using ExplicitImports: ExplicitImports +# REPL is imported purely for its load-time side effect: on Julia >= 1.11 the methods +# of `Base.Docs.doc(::Base.Docs.Binding)` (the docstring lookup `run_api_docs` uses) +# live in the REPL stdlib and are absent until it is loaded. Importing it here makes +# the docstring check work in a bare `Pkg.test` process that never starts a REPL. +import REPL export current_group, activate_group_env, run_qa, run_explicit_imports, detect_sublibrary_group, develop_sources!, run_tests, read_test_groups, - with_clean_persistent_tasks_sources + with_clean_persistent_tasks_sources, run_api_docs, public_api_names # Group names that are never sublibraries and never named functional groups: the # routing keywords `run_tests` and `detect_sublibrary_group` reserve. @@ -578,10 +589,10 @@ end """ run_qa(pkg; Aqua = Aqua, JET = ..., ExplicitImports = ExplicitImports, aqua = Aqua !== nothing, jet = JET !== nothing, - explicit_imports = false, + explicit_imports = false, api_docs = true, clean_sources = true, aqua_kwargs = (;), jet_kwargs = (; target_modules = (pkg,), mode = :typo), - ei_kwargs = (;), + ei_kwargs = (;), api_docs_kwargs = (;), aqua_broken = (), jet_broken = false, ei_broken = (), testset = "Quality Assurance") @@ -601,16 +612,22 @@ Each tool runs if it is both available and enabled: * `JET` + `jet` ⇒ `JET.test_package(pkg; jet_kwargs...)`, * `ExplicitImports` + `explicit_imports` ⇒ ExplicitImports' standard + public-API checks (see [`run_explicit_imports`](@ref)). + * `api_docs` ⇒ the public-API documentation check (see [`run_api_docs`](@ref)): every + exported/`public` name has a docstring (and, if `api_docs_kwargs` opts in with + `rendered = true`, is rendered in the manual). Enable-flag defaults: `aqua` defaults to `Aqua !== nothing` (on — Aqua is always available; pass `Aqua = nothing` or `aqua = false` to skip it), and `jet` defaults to -`JET !== nothing` (on exactly when `JET` has been loaded/registered). `explicit_imports` -defaults to **`false`** — a deliberate opt-in: because `ExplicitImports` is always -available, defaulting it on would silently turn the (per-repo-curated) ExplicitImports -checks on for every existing `run_qa` caller on a routine version bump, so a repo must -ask for them with `explicit_imports = true`. Setting an enable flag `true` while its -module is unavailable is a configuration error and throws an `ArgumentError`. The whole -thing runs inside a `@testset` named `testset`. +`JET !== nothing` (on exactly when `JET` has been loaded/registered). `api_docs` defaults +to **`true`**: the public-API docstring check needs no extra dependency and is a baseline +QA expectation across the fleet, so it runs for every `run_qa` caller by default — +document a repo's public API, or curate exceptions via `api_docs_kwargs` (`ignore`, +`docstrings_broken`), or pass `api_docs = false` to skip it. `api_docs_kwargs` is +forwarded to [`run_api_docs`](@ref) (e.g. `rendered`, `ignore`, `docstrings_broken`). +`explicit_imports` still defaults to **`false`** — its per-repo-curated ignore-lists make +it a deliberate opt-in (`explicit_imports = true`). Setting an enable flag `true` while +its module is unavailable is a configuration error and throws an `ArgumentError`. The +whole thing runs inside a `@testset` named `testset`. `clean_sources` (default `true`) wraps the `Aqua.test_all` call in [`with_clean_persistent_tasks_sources`](@ref), which strips *broken* path-`[sources]` @@ -707,10 +724,12 @@ function run_qa( aqua::Bool = Aqua !== nothing, jet::Bool = JET !== nothing, explicit_imports::Bool = false, + api_docs::Bool = true, clean_sources::Bool = true, aqua_kwargs = (;), jet_kwargs = (; target_modules = (pkg,), mode = :typo), ei_kwargs = (;), + api_docs_kwargs = (;), aqua_broken = (), jet_broken::Bool = false, ei_broken = (), @@ -753,6 +772,7 @@ function run_qa( end end explicit_imports && run_explicit_imports(pkg, ExplicitImports; ei_kwargs, ei_broken) + api_docs && run_api_docs(pkg; api_docs_kwargs...) end return nothing end @@ -821,6 +841,201 @@ function run_explicit_imports(pkg::Module, ExplicitImports; ei_kwargs = (;), ei_ return nothing end +""" + public_api_names(pkg::Module) -> Vector{Symbol} + +Return `pkg`'s public API: every exported name plus, on Julia >= 1.11, every name +declared with the `public` keyword — the exact set `names(pkg; all = false, +imported = false)` yields — with the module's own name dropped and the result sorted. + +This is the set [`run_api_docs`](@ref) requires documentation for. On the Julia 1.10 +LTS the `public` keyword does not exist, so a name made public only via a `public` +backport is *not* reported here (Base's `names` does not see it); on 1.10 the set is +therefore the exported names alone. Re-exported names (e.g. via `Reexport`) are +included when they are re-exported with `export`, since `names` reports them. + +# Examples + +```julia +julia> public_api_names(SciMLTesting) +11-element Vector{Symbol}: + :activate_group_env + ⋮ +``` +""" +function public_api_names(pkg::Module) + api = filter!(!=(nameof(pkg)), names(pkg; all = false, imported = false)) + return sort!(api) +end + +# Whether `pkg.name` resolves to a docstring. Uses `Base.Docs.doc` on the *binding* +# (not the object), so a re-exported name that is documented in the module that +# actually defines it counts as documented here too (the binding follows the import +# alias to that module's docs). `import REPL` at load time ensures the binding-lookup +# methods exist on Julia >= 1.11. A name with no binding in `pkg` (should not happen +# for a `names`-reported symbol) counts as undocumented. +function _has_docstring(pkg::Module, name::Symbol) + isdefined(pkg, name) || return false + doc = sprint(show, MIME"text/plain"(), Base.Docs.doc(Base.Docs.Binding(pkg, name))) + return !occursin("No documentation found", doc) +end + +# The bare name referenced by one line inside a ```@docs``` fenced block. A `@docs` +# entry is a name, optionally module-qualified and/or with a call signature, e.g. +# `foo`, `MyPkg.foo`, `foo(x::Int)`, `MyPkg.@mac`. Take the first whitespace token, +# drop a call-signature suffix, then drop a leading module qualifier (the text after +# the last dot), preserving a leading `@` for macros. +function _doc_entry_name(line::AbstractString) + tok = String(first(split(line))) + tok = String(first(split(tok, '('))) + dot = findlast('.', tok) + dot === nothing || (tok = tok[nextind(tok, dot):end]) + return Symbol(tok) +end + +# Scan every `*.md` file under `docs_src` and collect the set of names that appear +# inside a ```@docs``` fenced block, plus whether any ```@autodocs``` block is present. +# An `@autodocs` block renders whole modules' docstrings, so its presence means the +# per-name "rendered" check cannot meaningfully single out a missing name — the caller +# treats that as "everything documented is rendered". A missing `docs_src` yields an +# empty set and `autodocs = false`. +function _rendered_doc_names(docs_src::AbstractString) + rendered = Set{Symbol}() + autodocs = false + isdir(docs_src) || return (rendered, autodocs) + for (dir, _, files) in walkdir(docs_src) + for file in files + endswith(file, ".md") || continue + in_docs = false + for raw in eachline(joinpath(dir, file)) + line = strip(raw) + if startswith(line, "```@docs") + in_docs = true + continue + elseif startswith(line, "```@autodocs") + autodocs = true + in_docs = false + continue + elseif in_docs && startswith(line, "```") + in_docs = false + continue + end + in_docs || continue + (isempty(line) || startswith(line, "#")) && continue + push!(rendered, _doc_entry_name(line)) + end + end + end + return (rendered, autodocs) +end + +# Default docs source dir for a package: /docs/src, or "" when the package +# root cannot be located (e.g. `pkg` is `Main`). "" is a non-directory, so the +# rendered scan finds nothing and the check fails loudly if it was opted into without +# a resolvable docs tree. +_default_docs_src(pkg::Module) = + (root = pkgdir(pkg); root === nothing ? "" : joinpath(root, "docs", "src")) + +""" + run_api_docs(pkg::Module; docstrings = true, rendered = false, + docs_src = /docs/src, + ignore = (), rendered_ignore = (), + docstrings_broken = false, rendered_broken = false, + testset = "Public API documentation") + +Assert that `pkg`'s public API (see [`public_api_names`](@ref)) is documented. + +This is the shared, per-repo-free replacement for the hand-rolled +`test/QA/public_api_docs.jl` files that were copied into individual SciML repos. +[`run_qa`](@ref) runs it by default (`api_docs = true`), so a plain `run_qa(MyPkg)` +already covers the docstring check; call `run_api_docs` directly only to run it outside +`run_qa` or to opt into the `rendered` check. + +Two checks, each its own nested `@testset`: + + * **docstrings** (`docstrings = true`, on by default): every public API name has a + docstring. A re-exported name documented in its defining package counts as + documented (the check follows the binding, not a local docstring), so a repo is + not forced to redocument names it re-exports from a dependency. + * **rendered** (`rendered = false`, opt-in): every public API name appears inside a + ```` ```@docs ```` block somewhere under `docs_src`, so it is rendered in the + manual. This is opt-in because it needs a resolvable docs tree and does not fit + every repo (monorepos with shared docs, packages with no manual). If any + ```` ```@autodocs ```` block is present the check passes wholesale — `@autodocs` + renders whole modules, so anything with a docstring is already rendered. + +`docs_src` defaults to `/docs/src` located via `pkgdir(pkg)`, so a repo whose +docs live in the standard place needs no path plumbing. + +`ignore` (for the docstring check) and `rendered_ignore` (for the rendered check) are +collections of names (`Symbol`s) to exclude — use them for names that legitimately +have no local documentation obligation, e.g. a re-export you cannot document at the +source, with a comment pointing at the tracking issue. + +`docstrings_broken` / `rendered_broken` mark the respective check `@test_broken` (for a +repo mid-migration with many undocumented names): a green lane records `Broken` while +names remain undocumented, and the `@test_broken` flips to an `Unexpected Pass` (an +`Error`) once the API is fully documented, prompting the caller to drop the flag. + +On Julia 1.10, `public_api_names` returns only the exported names (the `public` keyword +is 1.11+), so the checks cover exactly the names Base reports as public on the running +version — no per-repo `if VERSION` guards needed. + +# Examples + +```julia +# In test/qa/qa.jl — the whole per-repo public-API-docs check: +using SciMLTesting, MyPackage +run_api_docs(MyPackage) # docstrings only +run_api_docs(MyPackage; rendered = true) # also require rendering in docs/src + +# It runs by default inside run_qa; use api_docs_kwargs to configure it, or +# api_docs = false to skip it: +run_qa(MyPackage; explicit_imports = true, + api_docs_kwargs = (; rendered = true, ignore = (:reexported_from_dep,))) +``` + +See also [`run_qa`](@ref), whose `api_docs`/`api_docs_kwargs` keywords call this. +""" +function run_api_docs( + pkg::Module; + docstrings::Bool = true, + rendered::Bool = false, + docs_src::AbstractString = _default_docs_src(pkg), + ignore = (), + rendered_ignore = (), + docstrings_broken::Bool = false, + rendered_broken::Bool = false, + testset::AbstractString = "Public API documentation", + ) + api = public_api_names(pkg) + @testset "$testset" begin + if docstrings + skip = Set{Symbol}(Symbol.(ignore)) + undocumented = sort!(filter(n -> !(n in skip) && !_has_docstring(pkg, n), api)) + @testset "public API has docstrings" begin + docstrings_broken ? (@test_broken isempty(undocumented)) : + (@test isempty(undocumented)) + isempty(undocumented) || + @info "run_api_docs: public API names missing a docstring" pkg undocumented + end + end + if rendered + (rendered_names, autodocs) = _rendered_doc_names(docs_src) + skip = Set{Symbol}(Symbol.(rendered_ignore)) + unrendered = autodocs ? Symbol[] : + sort!(filter(n -> !(n in skip) && !(n in rendered_names), api)) + @testset "public API is rendered in docs" begin + rendered_broken ? (@test_broken isempty(unrendered)) : + (@test isempty(unrendered)) + isempty(unrendered) || + @info "run_api_docs: public API names not rendered in a @docs block" pkg docs_src unrendered + end + end + end + return nothing +end + """ detect_sublibrary_group(group, lib_dir; default_group = "Core") -> (sublibrary, test_group) diff --git a/test/runtests.jl b/test/runtests.jl index 4c4f64e..3b5e241 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -77,6 +77,23 @@ module FakeExplicitImports end end +# Fixture exercising run_api_docs / public_api_names against a real module with a +# known mix of documented and undocumented public API. On Julia >= 1.11 it also +# declares `public` names (the eval(Expr(:public, ...)) form is a syntax error on the +# 1.10 LTS, so it is guarded out there — matching how public_api_names sees only +# exported names on 1.10). +module ApiFixture + export documented_fn, undocumented_fn, DocumentedType + "documented_fn doc" documented_fn(x) = x + undocumented_fn(x) = x + "DocumentedType doc" struct DocumentedType end + @static if VERSION >= v"1.11" + eval(Expr(:public, :documented_public, :undocumented_public)) + end + "documented_public doc" documented_public(x) = x + undocumented_public(x) = x +end + # A minimal AbstractTestSet that just collects every recorded result (including # nested testsets) and NEVER throws on finish. Wrapping a run_qa call in one lets a # test inspect the Broken/Pass/Fail/Error counts a broken-marker produced without @@ -520,17 +537,19 @@ end # ExplicitImports is always available (opt-in, so a routine bump never turns # the per-repo ExplicitImports checks on for existing callers). `aqua` is # forced off throughout so the real Aqua never runs against SciMLTesting itself. + # api_docs is forced off here too, so this testset stays focused on JET/aqua + # defaulting (the default-on api_docs check has its own testset). saved = copy(SciMLTesting._QA_MODULES) try delete!(SciMLTesting._QA_MODULES, :JET) - # JET unregistered + aqua off + EI default off => run_qa is a no-op (no error). - run_qa(SciMLTesting; aqua = false) + # JET unregistered + aqua off + EI/api_docs off => run_qa is a no-op (no error). + run_qa(SciMLTesting; aqua = false, api_docs = false) # Register a Fake JET: `jet` now defaults on and run_qa runs it. SciMLTesting._register_qa_tool!(:JET, FakeJET) @test SciMLTesting._qa_tool(:JET) === FakeJET - run_qa(SciMLTesting; aqua = false) # runs FakeJET via the registry default - run_qa(SciMLTesting; aqua = false, jet = false) # explicit off skips it (no error) + run_qa(SciMLTesting; aqua = false, api_docs = false) # runs FakeJET via the registry default + run_qa(SciMLTesting; aqua = false, jet = false, api_docs = false) # explicit off skips it (no error) finally empty!(SciMLTesting._QA_MODULES) merge!(SciMLTesting._QA_MODULES, saved) @@ -771,6 +790,266 @@ end end end + @testset "public_api_names" begin + api = public_api_names(ApiFixture) + @test issorted(api) + @test !(nameof(ApiFixture) in api) # the module's own name is dropped + # Exported names are always present; `public`-declared names only on >= 1.11. + @test :documented_fn in api + @test :undocumented_fn in api + @test :DocumentedType in api + if VERSION >= v"1.11" + @test :documented_public in api + @test :undocumented_public in api + else + @test !(:documented_public in api) # `public` keyword is 1.11+ + end + + # SciMLTesting's own exported API is exactly what `export` lists (it declares + # no `public` names), independent of Julia version. + st = public_api_names(SciMLTesting) + @test :run_api_docs in st && :run_qa in st && :run_tests in st + @test !(:SciMLTesting in st) + end + + @testset "_doc_entry_name" begin + @test SciMLTesting._doc_entry_name("foo") == :foo + @test SciMLTesting._doc_entry_name("MyPkg.foo") == :foo + @test SciMLTesting._doc_entry_name("foo(x::Int)") == :foo + @test SciMLTesting._doc_entry_name("MyPkg.foo(x::Int, y)") == :foo + @test SciMLTesting._doc_entry_name("Base.SubMod.bar") == :bar + @test SciMLTesting._doc_entry_name("@mac") == Symbol("@mac") + @test SciMLTesting._doc_entry_name("MyPkg.@mac") == Symbol("@mac") + end + + @testset "_rendered_doc_names" begin + # A docs/src tree with a @docs block (module-qualified + signature entries) in + # one file and a plain-prose file with no block in another. + droot = mktempdir() + src = joinpath(droot, "src"); mkpath(joinpath(src, "manual")) + write( + joinpath(src, "index.md"), + "# Home\n\nsome prose\n\n```@docs\nMyPkg.foo\nbar(x::Int)\n@mac\n```\n\nmore prose\n", + ) + write(joinpath(src, "manual", "extra.md"), "# Extra\n\n```@docs\nbaz\n```\n") + write(joinpath(src, "prose.md"), "# Just prose\n\nno docs block here\n") + (rendered, autodocs) = SciMLTesting._rendered_doc_names(src) + @test rendered == Set([:foo, :bar, Symbol("@mac"), :baz]) + @test autodocs == false + + # An @autodocs block flips the wholesale flag. + adroot = mktempdir() + asrc = joinpath(adroot, "src"); mkpath(asrc) + write( + joinpath(asrc, "api.md"), + "# API\n\n```@autodocs\nModules = [MyPkg]\n```\n", + ) + (_, autodocs2) = SciMLTesting._rendered_doc_names(asrc) + @test autodocs2 == true + + # A missing docs dir yields an empty set (never errors). + (empty_rendered, empty_auto) = SciMLTesting._rendered_doc_names(joinpath(droot, "nope")) + @test isempty(empty_rendered) && empty_auto == false + end + + @testset "run_api_docs docstrings check" begin + # Reuse the ProbeTestSet + count_results helpers from the broken-markers set to + # count Pass/Fail/Broken without failing the enclosing suite. + function count_results(ts) + counts = Dict(:pass => 0, :fail => 0, :error => 0, :broken => 0) + for r in ts.results + if r isa Test.Pass + counts[:pass] += 1 + elseif r isa Test.Fail + counts[:fail] += 1 + elseif r isa Test.Error + counts[:error] += 1 + elseif r isa Test.Broken + counts[:broken] += 1 + elseif r isa Test.AbstractTestSet + sub = count_results(r) + for k in keys(counts) + counts[k] += sub[k] + end + end + end + return counts + end + counts_of(body) = count_results( + @testset ProbeTestSet "probe" begin + body() + end + ) + + # All-documented public API passes with no failures. SciMLTesting itself is the + # real, non-mocked case: every exported name has a docstring, so this both + # proves the happy path AND guards the package's own API-doc coverage. + c = counts_of() do + run_api_docs(SciMLTesting) + end + @test c[:fail] == 0 && c[:error] == 0 + @test c[:pass] >= 1 + + # The fixture has undocumented public API -> one Fail (the docstrings @test). + c = counts_of() do + run_api_docs(ApiFixture) + end + @test c[:fail] == 1 + @test c[:broken] == 0 + + # Ignoring the undocumented names makes it pass. (:undocumented_public is not in + # the API on 1.10, so ignoring it there is a harmless no-op.) + c = counts_of() do + run_api_docs(ApiFixture; ignore = (:undocumented_fn, :undocumented_public)) + end + @test c[:fail] == 0 && c[:error] == 0 + @test c[:pass] == 1 + + # docstrings_broken records Broken while names remain undocumented (migration). + c = counts_of() do + run_api_docs(ApiFixture; docstrings_broken = true) + end + @test c[:broken] == 1 + @test c[:fail] == 0 + + # A fully-documented API under docstrings_broken is an Unexpected Pass (Error), + # auto-flagging the caller to drop the flag. + c = counts_of() do + run_api_docs(SciMLTesting; docstrings_broken = true) + end + @test c[:error] == 1 + @test c[:broken] == 0 + @test c[:fail] == 0 + end + + @testset "run_api_docs rendered check" begin + function count_results(ts) + counts = Dict(:pass => 0, :fail => 0, :error => 0, :broken => 0) + for r in ts.results + if r isa Test.Pass + counts[:pass] += 1 + elseif r isa Test.Fail + counts[:fail] += 1 + elseif r isa Test.Error + counts[:error] += 1 + elseif r isa Test.Broken + counts[:broken] += 1 + elseif r isa Test.AbstractTestSet + sub = count_results(r) + for k in keys(counts) + counts[k] += sub[k] + end + end + end + return counts + end + counts_of(body) = count_results( + @testset ProbeTestSet "probe" begin + body() + end + ) + + api = public_api_names(ApiFixture) + # A docs/src listing every public API name in a @docs block -> rendered passes. + droot = mktempdir() + src = joinpath(droot, "src"); mkpath(src) + write( + joinpath(src, "api.md"), + "# API\n\n```@docs\n" * join(("ApiFixture." * String(n) for n in api), "\n") * "\n```\n", + ) + c = counts_of() do + run_api_docs(ApiFixture; docstrings = false, rendered = true, docs_src = src) + end + @test c[:fail] == 0 && c[:error] == 0 + @test c[:pass] == 1 + + # Dropping one name from the block -> rendered fails (that name is unrendered). + write( + joinpath(src, "api.md"), + "# API\n\n```@docs\n" * + join(("ApiFixture." * String(n) for n in api if n != first(api)), "\n") * "\n```\n", + ) + c = counts_of() do + run_api_docs(ApiFixture; docstrings = false, rendered = true, docs_src = src) + end + @test c[:fail] == 1 + + # ... but rendered_ignore on the dropped name makes it pass again. + c = counts_of() do + run_api_docs( + ApiFixture; docstrings = false, rendered = true, docs_src = src, + rendered_ignore = (first(api),), + ) + end + @test c[:fail] == 0 && c[:pass] == 1 + + # An @autodocs block satisfies the rendered check wholesale (no per-name list). + adroot = mktempdir() + asrc = joinpath(adroot, "src"); mkpath(asrc) + write(joinpath(asrc, "api.md"), "```@autodocs\nModules = [ApiFixture]\n```\n") + c = counts_of() do + run_api_docs(ApiFixture; docstrings = false, rendered = true, docs_src = asrc) + end + @test c[:fail] == 0 && c[:pass] == 1 + end + + @testset "run_qa api_docs integration (default on)" begin + function count_results(ts) + counts = Dict(:pass => 0, :fail => 0, :error => 0, :broken => 0) + for r in ts.results + if r isa Test.Pass + counts[:pass] += 1 + elseif r isa Test.Fail + counts[:fail] += 1 + elseif r isa Test.Error + counts[:error] += 1 + elseif r isa Test.Broken + counts[:broken] += 1 + elseif r isa Test.AbstractTestSet + sub = count_results(r) + for k in keys(counts) + counts[k] += sub[k] + end + end + end + return counts + end + counts_of(body) = count_results( + @testset ProbeTestSet "probe" begin + body() + end + ) + + # api_docs defaults to `true`: a plain run_qa (all other tools off) still runs + # the public-API docstring check. Against SciMLTesting (fully documented) that is + # a clean pass — the default-on check fires (>=1 pass) with no failures. + c = counts_of() do + run_qa(SciMLTesting; Aqua = nothing, JET = nothing, ExplicitImports = nothing) + end + @test c[:pass] >= 1 + @test c[:fail] == 0 && c[:error] == 0 && c[:broken] == 0 + + # api_docs_kwargs is forwarded (docstrings_broken flips the pass to a Broken). + c = counts_of() do + run_qa( + SciMLTesting; Aqua = nothing, JET = nothing, ExplicitImports = nothing, + api_docs_kwargs = (; docstrings_broken = true), + ) + end + @test c[:error] == 1 # fully-documented under docstrings_broken => Unexpected Pass + @test c[:fail] == 0 + + # api_docs = false skips the check entirely: with every tool off, run_qa records + # nothing at all. + c = counts_of() do + run_qa( + SciMLTesting; Aqua = nothing, JET = nothing, ExplicitImports = nothing, + api_docs = false, + ) + end + @test c[:pass] == 0 && c[:fail] == 0 && c[:error] == 0 && c[:broken] == 0 + end + @testset "run_tests routing" begin # A scratch test/runtests-like layout with body files we can detect having # run via marker files (so we can assert routing without nested Pkg.test).