Infra: Add custom lexers for PEP 823 + 824 (#5125)

Author: cdce8p

pep_sphinx_extensions/__init__.py

@@ -13,6 +13,7 @@
     create_rss_feed,
     get_from_doctree,
 )
+from pep_sphinx_extensions.lexers import get_custom_lexers
 from pep_sphinx_extensions.pep_processor.html import (
     pep_html_builder,
     pep_html_translator,
@@ -110,6 +111,10 @@ def setup(app: Sphinx) -> dict[str, bool]:
     app.add_directive("superseded", pep_banner_directive.SupersededBanner)
     app.add_directive("withdrawn", pep_banner_directive.WithdrawnBanner)
 
+    # Register custom lexers
+    for lexer in get_custom_lexers():
+        app.add_lexer(lexer.name, lexer)
+
     # Register event callbacks
     app.connect(
         "builder-inited", _update_config_for_builder

pep_sphinx_extensions/lexers/__init__.py

@@ -0,0 +1,21 @@
+# This file is placed in the public domain or under the
+# CC0-1.0-Universal license, whichever is more permissive.
+
+from __future__ import annotations
+
+import importlib
+import pkgutil
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from pygments.lexer import Lexer
+
+
+def get_custom_lexers() -> list[type[Lexer]]:
+    lexers: list[type[Lexer]] = []
+    for module_info in pkgutil.walk_packages(__path__, prefix=f"{__name__}."):
+        module = importlib.import_module(module_info.name)
+        if (register_func := getattr(module, "register", None)) is None:
+            continue
+        lexers.extend(register_func())
+    return lexers

pep_sphinx_extensions/lexers/pep823_lexer.py

@@ -0,0 +1,48 @@
+# This file is placed in the public domain or under the
+# CC0-1.0-Universal license, whichever is more permissive.
+
+"""Custom lexer for PEP 823."""
+
+# This lexer is most useful while PEP 823 is being discussed.
+# If it breaks in the future, it can be safely replaced by plain
+# Python or text lexers.
+
+from pygments.lexer import DelegatingLexer, inherit
+from pygments.lexers.python import (
+    PythonLexer,
+    PythonTracebackLexer,
+    _PythonConsoleLexerBase,
+)
+from pygments.token import Operator, Other
+
+
+class Py823Lexer(PythonLexer):
+    name = "py823"
+
+    tokens = {
+        "expr": [
+            (r"maybe\b", Operator.Word),
+            (r"\?", Operator),
+            inherit,
+        ],
+    }
+
+
+class Py823ConsoleLexer(DelegatingLexer):
+    name = "py823-console"
+
+    def __init__(self, **options):
+        pylexer = Py823Lexer
+        tblexer = PythonTracebackLexer
+
+        class _ReplaceInnerCode(DelegatingLexer):
+            def __init__(self, **options):
+                super().__init__(
+                    pylexer, _PythonConsoleLexerBase, Other.Code, **options
+                )
+
+        super().__init__(tblexer, _ReplaceInnerCode, Other.Traceback, **options)
+
+
+def register():
+    return [Py823Lexer, Py823ConsoleLexer]

pep_sphinx_extensions/lexers/pep824_lexer.py

@@ -0,0 +1,29 @@
+# This file is placed in the public domain or under the
+# CC0-1.0-Universal license, whichever is more permissive.
+
+"""Custom lexer for PEP 824."""
+
+# This lexer is most useful while PEP 824 is being discussed.
+# If it breaks in the future, it can be safely replaced by plain
+# Python or text lexers.
+
+from pygments.lexer import inherit
+from pygments.lexers.python import PythonLexer
+from pygments.token import Operator
+
+
+class Py824Lexer(PythonLexer):
+    name = "py824"
+
+    tokens = {
+        "expr": [
+            (r"\?\?(?=\s)", Operator.Word),
+            (r"otherwise\b", Operator.Word),
+            (r"\?", Operator),
+            inherit,
+        ],
+    }
+
+
+def register():
+    return [Py824Lexer]

peps/pep-0823.rst

@@ -149,7 +149,7 @@ kinds of expressions much simpler while being predictable and doing
 the correct things intuitively. Using these operators, the function
 could instead be written as:
 
-::
+.. code-block:: py823
 
     def get_customer_name(data: Data) -> str | None:
         return data.customer?.user?.name.lower()
@@ -198,7 +198,7 @@ least for dictionaries a useful helper method is ``dict.get(key)``.
 
 Writing it using ``?.`` and ``?[ ]`` would look like this:
 
-::
+.. code-block:: py823
 
     def get_customer_name(data: Data) -> str | None:
         return data.get("customer")?["user"]?["name"].lower()
@@ -221,14 +221,14 @@ hide in plain sight. Attribute and function names have been shortened.
     If code relied on this property, the expression cannot necessarily
     be replaced with ``?.`` or ``?[ ]``.
 
-::
+.. code-block:: py823
 
     # In assignments
 
     x = a.b if (a is not None) else None
     x = a?.b
 
-::
+.. code-block:: py823
 
     # In if statements often used as guard clause with early
     # return or raising of an exception
@@ -242,7 +242,7 @@ hide in plain sight. Attribute and function names have been shortened.
     if a is None or a.b is None: ...
     if a?.b is None: ...
 
-::
+.. code-block:: py823
 
     # Misc expressions
 
@@ -336,7 +336,7 @@ for trying to get a subscript of ``None`` are omitted. It is therefore
 not necessary to change subsequent ``.`` or ``[ ]`` on the right-hand
 side just because a ``?.`` or ``?[ ]`` is used prior.
 
-::
+.. code-block:: py823-console
 
     >>> a = None
     >>> print(a?.b.c[0].some_function())
@@ -348,7 +348,7 @@ their ``None``-aware counterparts, and call expressions). As a rule of
 thumb, short-circuiting is broken once an operator other than
 ``.``, ``[ ]``, ``?.``, ``?[ ]`` is reached.
 
-::
+.. code-block:: py823-console
 
     >>> a = None
     >>> print(a?.b.c)
@@ -370,7 +370,7 @@ be broken. For example function arguments or subscripts are evaluated
 on their own and would not short-circuit the remaining ``tail`` of the
 outer expression.
 
-::
+.. code-block:: py823
 
     # func(a?.b).c[d?.e]
 
@@ -389,7 +389,7 @@ if ``a is None``. This is conceptually identical to extracting the group
 contents and storing the result in a temporary variable before
 substituting it back into the original expression.
 
-::
+.. code-block:: py823
 
     # (a?.b).c
 
@@ -400,7 +400,7 @@ Common use cases for ``None``-aware access operators in groups are
 boolean or conditional expressions which can provide a fallback value
 in case the first part evaluates to ``None``.
 
-::
+.. code-block:: py823
 
     (a.b?.c or d).e?.func()
 
@@ -419,7 +419,7 @@ Assignments
 ``None``-aware expressions may only be used in a ``Load`` context.
 Assignments are not permitted and will raise a ``SyntaxError``.
 
-::
+.. code-block:: py823-console
 
     >>> a?.b = 1
     File "<python-input-1>", line 1
@@ -437,7 +437,7 @@ This does not apply if the ``None``-aware expressions is only part
 of a larger expression and evaluated on its own, for example as a
 function argument.
 
-::
+.. code-block:: py823-console
 
     >>> a = None
     >>> def f(a):
@@ -512,7 +512,7 @@ their needs, especially code formatters might prefer a style which
 conforms better to their existing preferences. An example of what
 is possible:
 
-::
+.. code-block:: py823
 
     def get_customer_name(data: Data) -> str | None:
         return (
@@ -675,7 +675,7 @@ because it might be too difficult to understand. Developers should
 instead change any subsequent attribute access or subscript to their
 ``None``-aware variants.
 
-::
+.. code-block:: py823
 
     # before
     a.b.optional?.c.d.e
@@ -706,7 +706,7 @@ instead of two new operators, it may also be **too general**, in a sense
 that it can be combine with any other operator. For example it is not
 clear what the following expressions would mean:
 
-::
+.. code-block:: py823-console
 
     >>> x? + 1
     >>> x? -= 1
@@ -717,7 +717,7 @@ clear what the following expressions would mean:
 Even if a default meaning of ``is not None else None`` is assumed, the
 expressions are likely to raise errors at some point.
 
-::
+.. code-block:: py823-console
 
     >>> x? + 1
     >>> (_t1 if ((_t1 := x) is not None) else None) + 1
@@ -896,7 +896,7 @@ the substitution principle. An expression ``(a?.b).c`` should behave
 the same whether or not ``a?.b`` is written inline inside a group or
 defined as a separate variable.
 
-::
+.. code-block:: py823
 
     (a?.b).c
 
@@ -1053,7 +1053,7 @@ for an ``optional`` value evaluates to ``None``, the result will be
 will be skipped. In the example below, if ``a.b`` is ``None``, so will
 be ``a.b?.c``:
 
-::
+.. code-block:: py823
 
     a.b?.c
     ^^^

peps/pep-0824.rst

@@ -111,7 +111,7 @@ Using the "``None``-coalescing" operator ``??`` instead, helps to keep
 the expression short and predictable while still clearly communicating
 the intent.
 
-::
+.. code-block:: py824
 
     def show_user_age(user: User):
         age = user.age ?? "unknown"
@@ -134,7 +134,7 @@ Using the "``None``-coalesce assignment" operator ``??=`` helps to
 avoid repeating the expression. Especially for more complex once,
 this will make it easier to read and write.
 
-::
+.. code-block:: py824
 
     def fix_user_name(user: User):
         user.name ??= "unknown"
@@ -157,7 +157,7 @@ and assign the fallback value inside the function itself.
 
 This could be rewritten as:
 
-::
+.. code-block:: py824
 
     def show_user_name(user: User | None):
         user ??= create_default_user()
@@ -188,7 +188,7 @@ conditional expressions. Parentheses can be added as necessary to
 modify the precedence of individual expressions. A few examples of
 how implicit parentheses would be placed:
 
-::
+.. code-block:: py824
 
     # x or y ?? 2
     (x or y) ?? 2
@@ -335,7 +335,7 @@ The following is therefore merely meant as a suggestion.
 +---------------------------+--------------------------+----------------------------+
 | Code                      | Pattern                  | Example                    |
 +===========================+==========================+============================+
-| ::                        | "... or ... if None"     | "user dot age ``or``       |
+| .. code-block:: py824     | "... or ... if None"     | "user dot age ``or``       |
 |                           |                          | unknown ``if None``"       |
 +     user.age ?? "unknown" +--------------------------+----------------------------+
 |                           | "... coalesce with ..."  | "user dot age              |
@@ -348,7 +348,7 @@ The following is therefore merely meant as a suggestion.
 +-----------------------------+------------------------------+------------------------------------+
 | Code                        | Pattern                      | Example                            |
 +=============================+==============================+====================================+
-| ::                          | "if ... is None, assign ..." | "``if`` user dot name ``is None``, |
+| .. code-block:: py824       | "if ... is None, assign ..." | "``if`` user dot name ``is None``, |
 |                             |                              | ``assign`` unknown"                |
 +     user.name ??= "unknown" +------------------------------+------------------------------------+
 |                             | "assign ... to ... if None"  | "``assign`` unknown ``to`` user    |
@@ -416,7 +416,7 @@ programming languages.
 Lastly, using a (soft-) keyword for the "``None``-coalescing assignment"
 operator poses additional questions and readability concerns.
 
-::
+.. code-block:: py824
 
     a = otherwise b