Reorganize docs, add info, reduce redundancies

docs/Project.toml

@@ -9,6 +9,9 @@ PowerSystemCaseBuilder = "f00506e0-b84f-492a-93c2-c0a9afc4364e"
 PowerSystems = "bcd98974-b02a-5e2f-9ee0-a103f5c450dd"
 PrettyTables = "08abe8d2-0d0c-5749-adfa-8a2ac140af0d"
 
+[sources]
+PowerNetworkMatrices = {path = ".."}
+
 [compat]
 Documenter = "1"
 julia = "^1.10"

docs/make.jl

@@ -4,40 +4,42 @@ using Literate
 using DocumenterInterLinks
 
 links = InterLinks(
+    "Julia" => "https://docs.julialang.org/en/v1/objects.inv",
     "PowerSystems" => "https://sienna-platform.github.io/PowerSystems.jl/stable/",
     "PowerSystemCaseBuilder" => "https://sienna-platform.github.io/PowerSystemCaseBuilder.jl/stable/",
 )
 
 include(joinpath(@__DIR__, "make_tutorials.jl"))
-make_tutorials()
+make_literate_folder("tutorials")
+make_literate_folder("how_to_guides")
 
 pages = OrderedDict(
     "Welcome Page" => "index.md",
     "Tutorials" => Any[
-        "Getting Started" => "tutorials/getting_started.md",
-        "Incidence, BA and ABA matrices" => "tutorials/tutorial_Incidence_BA_ABA_matrices.md",
-        "PTDF matrix" => "tutorials/tutorial_PTDF_matrix.md",
-        "VirtualPTDF matrix" => "tutorials/tutorial_VirtualPTDF_matrix.md",
-        "LODF matrix" => "tutorials/tutorial_LODF_matrix.md",
-        "VirtualLODF matrix" => "tutorials/tutorial_VirtualLODF_matrix.md",
-        "Industry DFAX values" => "tutorials/tutorial_DFAX.md",
-        "Radial Reduction" => "tutorials/tutorial_RadialReduction.md",
-        "Degree Two Reduction" => "tutorials/tutorial_DegreeTwoReduction.md",
+        "Introduction" => "tutorials/generated_introduction.md",
+        "Analysis at Scale" => "tutorials/generated_analysis_at_scale.md",
     ],
     "How-To Guides" => Any[
-        "Compute Network Matrices" => "how_to_guides/compute_network_matrices.md",
-        "Choose a Linear Solver" => "how_to_guides/choose_linear_solver.md",
+        "Build Multiple Matrices" => "how_to_guides/generated_build_multiple_matrices.md",
+        "Choose a Linear Solver" => "how_to_guides/generated_choose_linear_solver.md",
+        "Reproduce Industry DFAX Values" => "how_to_guides/generated_reproduce_dfax_values.md",
+        "Define and Apply Contingencies" => "how_to_guides/generated_contingencies.md",
+        "Diagnose Network Connectivity" => "how_to_guides/generated_diagnose_connectivity.md",
+    ],
+    "Reference" => Any[
+        "Matrix Overview and Indexing" => "reference/network_matrices_overview.md",
+        "Matrix Types" => "reference/matrix_types.md",
+        "Public API" => "reference/public.md",
+        "Internals" => "reference/internals.md",
     ],
     "Explanation" => Any[
-        "Computational Considertaions" => "explanation/computational_considerations.md",
         "DC Power Flow Approximation" => "explanation/dc_power_flow_approximation.md",
+        "Computational Considerations" => "explanation/computational_considerations.md",
         "Network Reduction Theory" => "explanation/network_reduction_theory.md",
         "Flowgate Methodology" => "explanation/flowgate_methodology.md",
-    ],
-    "Reference" => Any[
-        "Matrix Overview" => "reference/network_matrices_overview.md",
-        "Public API" => "reference/public.md",
-        "Internals" => "reference/internals.md",
+        "Concurrency and the KLU Lock" => "explanation/concurrency.md",
+        "Equivalent Representation of Reduced Branches" => "explanation/equivalent_branches.md",
+        "Slack Distribution and Reference-Bus Conventions" => "explanation/slack_conventions.md",
     ],
 )
 

docs/make_tutorials.jl

@@ -216,9 +216,9 @@ end
 # Download links:
 # - **Deployed / CI**: absolute URLs under `_DOCS_BASE_URL` when `_downloads_use_absolute_urls()` is true.
 # - **Local**: bare filenames (siblings of `generated_*.md` in `docs/src/tutorials/`).
-function add_download_links(content, jl_file, ipynb_file)
+function add_download_links(content, jl_file, ipynb_file, subdir = "tutorials")
     script_link, notebook_link = if _downloads_use_absolute_urls()
-        ("$_DOCS_BASE_URL/tutorials/$(jl_file)", "$_DOCS_BASE_URL/tutorials/$(ipynb_file)")
+        ("$_DOCS_BASE_URL/$subdir/$(jl_file)", "$_DOCS_BASE_URL/$subdir/$(ipynb_file)")
     else
         (jl_file, ipynb_file)
     end
@@ -345,8 +345,8 @@ end
 # - If a markdown cell contains one or more image fragments, append exactly one
 #   "view online" fallback note at the end of that cell.
 # - If the note already exists in the cell, no change is applied.
-function add_image_links(nb::Dict, outputfile_base::AbstractString)
-    tutorial_url = "$_DOCS_BASE_URL/tutorials/$(outputfile_base)/"
+function add_image_links(nb::Dict, outputfile_base::AbstractString, subdir = "tutorials")
+    tutorial_url = "$_DOCS_BASE_URL/$subdir/$(outputfile_base)/"
     msg = "_If image is not available when viewing in a Jupyter notebook, view the tutorial online [here]($tutorial_url)._"
     cells = get(nb, "cells", [])
     for (idx, cell) in enumerate(cells)
@@ -409,29 +409,30 @@ end
 # Process tutorials with Literate
 #########################################################
 
-# Generate tutorial markdown + notebook artifacts from literate .jl sources.
+# Generate markdown + notebook artifacts from the literate .jl sources in
+# docs/src/<subdir> (e.g. "tutorials", "how_to_guides").
 #
 # Pipeline:
-# 1) discover tutorial .jl files (excluding helper files starting with "_")
+# 1) discover .jl files in the folder (excluding helper files starting with "_")
 # 2) generate Documenter-flavored markdown with injected download links
 # 3) generate notebook with admonition conversion, setup preface, and image note
-function make_tutorials()
-    tutorials_dir = abspath(joinpath(@__DIR__, "src", "tutorials"))
+function make_literate_folder(subdir::String = "tutorials")
+    src_dir = abspath(joinpath(@__DIR__, "src", subdir))
     # Exclude helper scripts that start with "_"
-    if isdir(tutorials_dir)
-        tutorial_files =
+    if isdir(src_dir)
+        source_files =
             filter(
                 x -> endswith(x, ".jl") && !startswith(x, "_"),
-                readdir(tutorials_dir),
+                readdir(src_dir),
             )
-        if !isempty(tutorial_files)
-            # Clean up old generated tutorial files
-            tutorial_outputdir = tutorials_dir
-            clean_old_generated_files(tutorial_outputdir)
+        if !isempty(source_files)
+            # Clean up old generated files
+            outputdir = src_dir
+            clean_old_generated_files(outputdir)
 
-            for file in tutorial_files
+            for file in source_files
                 @show file
-                infile_path = joinpath(tutorials_dir, file)
+                infile_path = joinpath(src_dir, file)
                 execute =
                     if occursin("EXECUTE = TRUE", uppercase(readline(infile_path)))
                         true
@@ -443,7 +444,7 @@ function make_tutorials()
 
                 # Generate markdown
                 Literate.markdown(infile_path,
-                    tutorial_outputdir;
+                    outputdir;
                     name = outputfile,
                     credit = false,
                     flavor = Literate.DocumenterFlavor(),
@@ -453,6 +454,7 @@ function make_tutorials()
                             insert_md(content),
                             file,
                             string(outputfile, ".ipynb"),
+                            subdir,
                         )
                     ),
                     execute = execute)
@@ -461,13 +463,17 @@ function make_tutorials()
                 # preprocess_admonitions_for_notebook converts Documenter admonitions to blockquotes
                 # so they render in Jupyter; markdown output keeps !!! style for Documenter.
                 Literate.notebook(infile_path,
-                    tutorial_outputdir;
+                    outputdir;
                     name = outputfile,
                     credit = false,
                     execute = false,
                     preprocess = preprocess_admonitions_for_notebook,
                     postprocess = nb ->
-                        add_image_links(add_pkg_status_to_notebook(nb), outputfile))
+                        add_image_links(
+                            add_pkg_status_to_notebook(nb),
+                            outputfile,
+                            subdir,
+                        ))
             end
         end
     end

docs/src/explanation/computational_considerations.md

@@ -1,70 +1,60 @@
-## Computational Considerations
-
-### Matrix Construction
-
-All matrices in `PowerNetworkMatrices.jl` are derived from the `Ybus` matrix (i.e. building any matrix starts with building the `Ybus`). Additionally, all network reductions are applied to the `Ybus` matrix prior to computing the downstream matrices. This design choice is key for enabling high performance and code maintainability: Looping through the system objects is required only when building the `Ybus` (slow) and subsequent operations are built on fast matrix operations on (often sparse) matrices. In addition, network reductions are only defined for the `Ybus` but can be applied uniformly across all matrices.
-
-### Sparsity
-
-Power networks are sparse; most buses connect to only a few others. This sparsity is exploited for computational efficiency via sparse linear solvers:
-
-  - Incidence and admittance matrices are very sparse.
-  - Common sensitivity matrices (e.g. PTDF and LODF) are dense.
-
-### Automatic Sparsification Tolerance
-
-The PTDF and LODF are obtained by solving against the reduced susceptance matrix `ABA = Aᵀ B A`, which is sparse. In theory, however, **the inverse of a sparse matrix is dense**: `ABA` is a grounded graph Laplacian, and `ABA⁻¹` has essentially no zeros even though `ABA` does. So the sensitivity matrices come out dense, and for a large network a single PTDF column has one entry per bus — tens of thousands of numbers, almost all of them negligible (a branch is effectively insensitive to an injection electrically far away).
-
-To recover sparsity we apply a **tolerance** and drop entries below it (`droptol!`). The size of that cutoff is the question this package answers automatically. Every PTDF/LODF constructor takes a `tol` keyword:
-
-```julia
-tol::Union{Float64, AutoTolerance} = AutoTolerance()
-```
-
-  - A **`Float64`** is an explicit, *absolute* cutoff: any entry with `|x| ≤ tol` is dropped. Use it to pin an exact result (`tol = eps()`) or to sparsify by a fixed number. This is the backward-compatible path.
-
-  - An **[`AutoTolerance`](@ref)** (the default) chooses the cutoff from the data instead of a hand-tuned guess. It drops an entry of a row only when it is below the precision the input data can justify, *relative to that row's own peak*:
-
-    ```
-    drop entry j of row i   when   |row_i[j]| < α · max|row_i|,
-    α = clamp(safety · δ, 1e-6, 1e-2)
-    ```
-
-    where `δ` is the relative precision of the branch reactances (auto-discovered from their significant figures, or set explicitly via `data_precision`). Because the cutoff is *relative to each row's peak*, the achieved column sparsity is independent of the matrix scale and of how ill-conditioned `ABA` is. The 1-norm condition number of `ABA` is still estimated and logged as a diagnostic, but it never enters the cutoff.
-
-Sparsification only matters at scale, so `AutoTolerance` acts only where it pays off:
-
-  - **On-demand (virtual) matrices at or above `AUTO_TOLERANCE_BUS_LIMIT` buses** are sparsified per requested row/column — this is what lets a column of a large system come back sparse instead of dense.
-  - **Small systems and the dense `PTDF`/`LODF` constructors are returned exactly** (`AutoTolerance` is a no-op there), preserving their dense type and numerical values. Pass a `Float64` `tol` to sparsify those explicitly.
-
-For very large studies, prefer the [`VirtualPTDF`](@ref)/[`VirtualLODF`](@ref) variants: they compute rows on demand and, with the default `AutoTolerance`, store each one sparsely.
-
-#### Accuracy and limitations
-
-Sparsification trades exactness for memory, and the relative per-row rule has consequences worth understanding before you rely on a sparsified matrix for a sensitive calculation. Each dropped entry is below `α · max|row|` (`α ≤ 1e-2`, and typically `α ≈ δ ≈ 5e-4`), so the dominant sensitivities are never touched — but the following hold:
-
-  - **The error is one-signed, not zero-mean.** A dropped entry is set to exactly zero, never rounded, so each row's total mass strictly decreases. When you sum many small entries of a row (for example, aggregating a flow contribution across many buses), the truncation errors accumulate in the same direction instead of cancelling. The bias is bounded by `(number of dropped entries) · α · max|row|`.
-
-  - **The cutoff is per row, so global invariants are not preserved.** Each row is sparsified against *its own* peak, with no coupling between rows or columns, so quantities depending on cross-row or cross-column structure — Kirchhoff's current law, a column sum, or the *difference* of two entries — are not conserved. The sharp case is two buses `j, k` both far from branch `i`: `PTDF[i,j]` and `PTDF[i,k]` may be similar in magnitude yet fall on opposite sides of the cutoff. Each *absolute* error stays below the threshold, but the *relative* error on the (tiny) difference `PTDF[i,j] − PTDF[i,k]` can approach 100%.
-  - **Auto-discovered precision assumes a power-of-10 base.** With `data_precision = :auto`, `δ` counts the significant figures of the branch reactances. Decimal significant-figure counts are invariant under multiplication by a power of ten (the conventional 100 MVA base) but **not** under an arbitrary impedance base: data converted by a non-power-of-10 base (e.g. `Z_base = kV²/MVA = 190.44 Ω`) reads more figures than it carries, so `:auto` *over-estimates* precision. The direction is safe — a smaller `α`, hence *less* aggressive dropping — but on such data prefer an explicit `data_precision`.
-  - **Contingency (Woodbury) corrections do not amplify the error.** In [`VirtualMODF`](@ref) the cutoff is applied to the *final* post-contingency row, after the exact Woodbury solve; the correction is computed from exact factorization solves, never sparsified rows. The error stays bounded by the cutoff however near-critical (near-islanding) the contingency is, even when the Woodbury update is severely ill-conditioned. Verified directly in the test suite.
-
-When you need an exact result — to preserve KCL, to difference two small sensitivities, or to validate against a reference — pass a `Float64` `tol`: use `tol = eps()` for an unsparsified matrix, or a deliberate fixed cutoff for a reproducible absolute tolerance. `AutoTolerance` is the memory-versus-accuracy lever; the explicit `Float64` paths remain available for when exactness matters more than size.
-
-### Matrix Sizes
-
-A system with $N_b$ buses and $N_a$ arcs has matrix dimensions:
-
-  - Incidence: $N_a × N_b$ (sparse)
-  - Admittance: $N_b × N_b$ (sparse)
-  - PTDF: $N_a × N_b$ (dense)
-  - LODF: $N_a × N_a$ (dense)
-
-### Computational Complexity
-
-| Operation         | Complexity           | Notes                          |
-|:----------------- |:-------------------- |:------------------------------ |
-| Incidence Matrix  | O($N_a$)             | Simple topology scan           |
-| Admittance Matrix | O($N_a$)             | Includes electrical parameters |
-| PTDF              | O($N_b^3$)           | Requires matrix inversion      |
-| LODF              | O($N_a \cdot N_b^2$) | Derived from PTDF              |
+# Computational Considerations
+
+## Sparsity
+
+Power networks are sparse — most buses connect to only a few others — and this is
+exploited via sparse linear solvers, so the connectivity matrices ([`IncidenceMatrix`](@ref) and [`Ybus`](@ref))
+are very sparse. The sensitivity matrices ([`PTDF`](@ref), [`LODF`](@ref)), however, are obviously dense: e.g., one entry per bus in every
+[`PTDF`](@ref) column, most of them negligible because a branch is nearly insensitive
+to an injection electrically far away.
+
+## Sparsification and tolerance
+
+Those negligible entries can be dropped to recover sparsity. The **tolerance** is the
+cutoff below which an entry is set to zero. The default [`AutoTolerance`](@ref) picks
+it from the data as a *relative per-row* drop — an entry is dropped when
+`|x| < α · max|row|` — which keeps large matrices sparse while leaving small systems
+and the dense constructors exact. The exact rule (including the bus-count gate that
+makes it a no-op on small systems and the `Float64` `tol` alternative) is in the
+[`AutoTolerance`](@ref) docstring. For very large studies prefer the
+[`VirtualPTDF`](@ref)/[`VirtualLODF`](@ref) variants, which compute rows on demand
+and store each one sparsely.
+
+### Accuracy and limitations
+
+Sparsification trades exactness for memory. Because each dropped entry is below
+`α · max|row|` (`α ≤ 1e-2`, typically `α ≈ 5e-4`), the dominant sensitivities are
+never touched — but two properties are worth understanding before relying on a
+sparsified matrix for a sensitive calculation:
+
+  - **The error is one-signed, not zero-mean.** A dropped entry becomes exactly
+    zero, so a row's total mass strictly decreases. Summing many small entries of a
+    row — e.g. aggregating a transfer's flow contribution across a subsystem's buses —
+    accumulates that truncation in one direction instead of cancelling. The bias is
+    bounded by `(number of dropped entries) · α · max|row|`.
+  - **The cutoff is per row, so cross-row/column invariants are not preserved.** Each
+    row is sparsified against its own peak, so quantities that couple different rows or
+    columns — Kirchhoff's current law, a column sum, the *difference* of two entries —
+    are not conserved. Two buses `j, k` both far from branch `i` can have similar
+    `PTDF[i,j]`, `PTDF[i,k]` land on opposite sides of the cutoff: each absolute error
+    stays under threshold, yet the relative error on their tiny difference can approach
+    100%.
+
+Contingency corrections do not compound this: in [`VirtualMODF`](@ref) the cutoff is
+applied to the *final* post-contingency row, after the exact Woodbury solve, so the
+error stays bounded by the cutoff however near-critical the contingency.
+
+When you need an exact result — to preserve KCL, to difference two small
+sensitivities, or to validate against a reference — pass a `tol::Float64`
+(`tol = eps()` for an unsparsified matrix, or a deliberate fixed cutoff).
+
+## Matrix sizes and complexity
+
+A system with $N_b$ buses and $N_a$ arcs:
+
+| Operation         | Dimensions           | Complexity           | Notes                          |
+|:----------------- |:-------------------- |:-------------------- |:------------------------------ |
+| Incidence Matrix  | $N_a × N_b$ (sparse) | $O(N_a)$             | Simple topology scan           |
+| Admittance Matrix | $N_b × N_b$ (sparse) | $O(N_a)$             | Includes electrical parameters |
+| PTDF              | $N_a × N_b$ (dense)  | $O(N_b^3)$           | Requires matrix inversion      |
+| LODF              | $N_a × N_a$ (dense)  | $O(N_a \cdot N_b^2)$ | Derived from PTDF              |