Source browser

scripts/highlight_docs.py

Read the complete file below or download its original bytes. Line numbers provide stable links to a declaration or processing step.

1#!/usr/bin/env python3
2"""Render static syntax highlighting with exact source text.
3
4Source generators call highlight_code or highlight_lines before adding line
5anchors. The command-line pass handles guide code excerpts and preserves logs,
6tables and numbered sources. Pygments renders token spans during HTML creation.
7Browsers display the resulting markup with the local stylesheet.
8"""
9import argparse  # Select HTML pages and an optional verification-only pass.
10from html import escape, unescape  # Preserve code text while making token markup safe.
11from html.parser import HTMLParser  # Locate the relevant code element boundaries.
12import os  # Calculate a local stylesheet link for nested documentation pages.
13from pathlib import Path  # Resolve source languages and static documentation files.
14import re  # Recognize explicit language markers and conservative unlabeled examples.
15
16
17ALIASES = {  # Accept the language names already used by the maintained guides.
18    "cpp": "cpp", "c++": "cpp", "cxx": "cpp", "cc": "cpp", "c": "cpp",  # Use the established C++ lexer for the shared header and host examples.
19    "glsl": "glsl", "vert": "glsl", "frag": "glsl", "comp": "glsl",  # Use native shader syntax for shader source bodies.
20    "py": "python", "python3": "python", "python": "python",  # Keep Python examples independent of interpreter version spelling.
21    "sh": "bash", "shell": "bash", "console": "bash", "bash": "bash",  # Read documented command lines as shell input.
22    "cmake": "cmake", "js": "javascript", "javascript": "javascript",  # Support maintained build integration and browser helpers.
23    "css": "css", "html": "html", "markdown": "markdown", "md": "markdown",  # Support the source browser's remaining text formats.
24    "text": "text", "plaintext": "text", "plain": "text", "none": "text",  # Leave deliberate plain-text examples uncolored.
25}  # Keep unknown language names on a safe plain-text fallback.
26
27
28def language_for_path(path):  # Choose the source lexer from its path and language label.
29    """Return a stable lexer name for a repository-relative source path."""
30    path = Path(path)  # Accept both strings and Path objects from documentation generators.
31    parts = path.as_posix().lower()  # Compare directory names consistently on all development hosts.
32    if "/examples/shaders/" in "/" + parts or path.suffix.lower() in (".comp", ".vert", ".frag", ".glsl"):  # Use GLSL syntax for native shader .cpp files.
33        return "glsl"  # Preserve native qualifiers, buffer declarations and builtin names.
34    if path.name == "CMakeLists.txt" or path.suffix.lower() == ".cmake":  # Recognize both top-level and reusable CMake files.
35        return "cmake"  # Use the established CMake command and variable rules.
36    extensions = {".h": "cpp", ".hpp": "cpp", ".cpp": "cpp", ".cc": "cpp", ".cxx": "cpp", ".py": "python", ".sh": "bash", ".bash": "bash", ".js": "javascript", ".css": "css", ".html": "html", ".md": "markdown"}  # Cover every format selected by the source browser.
37    return extensions.get(path.suffix.lower(), "text")  # Render other files as plain text.
38
39
40def _pygments():  # Load the optional maintainer dependency only when highlighting is requested.
41    try:  # Keep normal library builds and HTML reading independent of this package.
42        from pygments.lexers import get_lexer_by_name  # Use the language lexer supplied by Pygments.
43        from pygments.token import STANDARD_TYPES  # Reuse stable Pygments token category abbreviations.
44    except ImportError as failure:  # Report exactly which optional documentation tool is missing.
45        raise RuntimeError("Documentation highlighting requires Pygments. Install it with: python3 -m pip install Pygments. Reading the HTML and building shdr do not require it.") from failure  # Avoid a fallback that silently removes requested highlighting.
46    return get_lexer_by_name, STANDARD_TYPES  # Keep the public helper import itself free of optional dependencies.
47
48
49def _escape_text(text):  # Preserve carriage returns even when a browser normalizes HTML source newlines.
50    return escape(text, quote=False).replace("\r", "
")  # Preserve quote characters as source text.
51
52
53def highlight_code(text, language="cpp"):  # Highlight a complete source before any numbered line wrapping.
54    """Return escaped inline token spans with exact text.
55
56    Full-input lexing preserves multiline comment and string state. Each line
57    contains balanced spans and retains its original newline boundary. Callers
58    add code wrappers, line numbers and navigation links around this markup.
59    """
60    lexer_factory, standard = _pygments()  # Give every caller the same optional dependency check.
61    language = ALIASES.get(language.lower(), "text")  # Keep explicit plain text safe and predictable.
62    lexer = lexer_factory(language, stripnl=False, stripall=False, ensurenl=False, tabsize=0)  # Disable whitespace trimming, added newlines and tab expansion.
63    tokens = list(lexer.get_tokens_unprocessed(text))  # Avoid the convenience API's newline and byte-order-mark preprocessing.
64    if "".join(value for _, _, value in tokens) != text:  # Refuse any lexer result that would change the displayed source.
65        raise ValueError("Pygments changed source text while highlighting " + language)  # Keep exact source preservation an enforced contract.
66    pieces = []  # Accumulate safe inline markup in original token order.
67    for _, token, value in tokens:  # Reuse full-file lexer state across line boundaries.
68        parent = token  # Find the nearest token category supported by the local stylesheet.
69        while parent not in standard and parent.parent is not None:  # Handle lexer-specific token subcategories using established parent categories.
70            parent = parent.parent  # Keep classification stable across Pygments versions.
71        category = standard.get(parent, "")  # Append ordinary text directly.
72        fragments = value.split("\n")  # Close token spans before every source line boundary.
73        for index, fragment in enumerate(fragments):  # Reopen multiline token categories independently on later lines.
74            if index:  # Emit exactly the newline present in the original token.
75                pieces.append("\n")  # Keep line anchors aligned with the unchanged source text.
76            if fragment:  # Select nonempty text fragments for token spans.
77                encoded = _escape_text(fragment)  # Keep source markup and ampersands inert.
78                pieces.append('<span class="tok-' + category + '">' + encoded + '</span>' if category else encoded)  # Apply local token style classes.
79    return "".join(pieces)  # Return inline markup ready for existing source and signature wrappers.
80
81
82def highlight_lines(text, language="cpp"):  # Share exact full-file lexing with numbered source generators.
83    """Return balanced inline HTML lines, including a final empty line when present."""
84    return highlight_code(text, language).split("\n")  # Let existing generators retain their own line-count and anchor conventions.
85
86
87class _Stylesheets(HTMLParser):  # Read existing stylesheet links and their source positions.
88    def __init__(self):  # Start with an empty set of declared stylesheet URLs.
89        super().__init__(convert_charrefs=True)  # Decode HTML entities in existing href attributes.
90        self.hrefs = set()  # Retain only links relevant to idempotent insertion.
91
92    def handle_starttag(self, tag, attrs):  # Read the original link metadata.
93        attrs = dict(attrs)  # Look up attributes independently of their ordering.
94        if tag == "link" and "stylesheet" in attrs.get("rel", "").split():  # Ignore icons and unrelated resource links.
95            self.hrefs.add(attrs.get("href", ""))  # Keep the exact already declared stylesheet target.
96
97
98def add_stylesheet(document, href="highlight.css"):  # Add a local stylesheet while preserving page content and anchors.
99    """Insert the given stylesheet before head closes, unless it is already linked."""
100    existing = _Stylesheets()  # Preserve every unrelated byte in the original HTML.
101    existing.feed(document)  # Inspect only existing link elements.
102    if href in existing.hrefs:  # Reuse the existing generator-provided stylesheet.
103        return document  # Keep repeated generation byte-idempotent.
104    closing = re.search(r"</head\s*>", document, re.IGNORECASE)  # Find a complete document's stylesheet insertion point.
105    if not closing:  # Avoid inserting presentation markup into a fragment or compiler log.
106        raise ValueError("A complete HTML head is required to add highlighting CSS")  # Explain the caller's missing document shell.
107    link = '<link rel="stylesheet" href="' + escape(href, quote=True) + '">'  # Keep the generated URL local and safely quoted.
108    return document[:closing.start()] + link + document[closing.start():]  # Preserve all existing IDs, links and formatting.
109
110
111class _CodeBlocks(HTMLParser):  # Locate eligible guide code while preserving unrelated HTML byte-for-byte.
112    def __init__(self, document):  # Build offset mapping for the original document.
113        super().__init__(convert_charrefs=False)  # Keep original entity spellings outside selected code interiors.
114        self.document, self.blocks, self.pre, self.code = document, [], [], None  # Track only candidate pre/code sections.
115        self.offsets = [0]  # Map HTMLParser line positions back to original string offsets.
116        for line in document.splitlines(keepends=True):  # Preserve newline lengths while building the map.
117            self.offsets.append(self.offsets[-1] + len(line))  # Retain the start offset for every parser line.
118
119    def position(self):  # Convert the parser's current token position to an absolute offset.
120        line, column = self.getpos()  # Read the beginning of the current HTML tag.
121        return self.offsets[line - 1] + column  # Keep edits confined to the selected code interior.
122
123    def handle_starttag(self, tag, attrs):  # Find code blocks and record any nested markup that must be preserved.
124        attrs = dict(attrs)  # Read language and source-browser markers by name.
125        if tag == "pre":  # Remember whether an existing numbered source wrapper owns this content.
126            self.pre.append(attrs)  # Keep source blocks out of the guide postprocessor.
127        if self.code is not None:  # Preserve existing anchors, links and syntax spans.
128            self.code["nested"] = True  # Let the original generator own any structured code block.
129        if tag == "code" and self.pre and self.code is None:  # Select explicit pre/code source excerpts.
130            if "source-code" in self.pre[-1].get("class", "").split():  # Numbered files are highlighted before line wrapping by their generator.
131                return  # Preserve structured source lines and encoded downloads.
132            start = self.position() + len(self.get_starttag_text())  # Begin immediately after the unchanged code opening tag.
133            self.code = {"start": start, "attrs": attrs, "nested": False}  # Store only the information needed for a safe interior replacement.
134
135    def handle_endtag(self, tag):  # Close this candidate block at its original page boundary.
136        if tag == "code" and self.code is not None:  # Capture the original unmodified interior boundaries.
137            self.code["end"] = self.position()  # Stop before the original closing tag.
138            self.blocks.append(self.code)  # Defer replacement until all original offsets are known.
139            self.code = None  # Wait for the next explicit pre/code excerpt.
140        if tag == "pre" and self.pre:  # Leave the current code or log wrapper.
141            self.pre.pop()  # Preserve nesting information for later blocks.
142
143
144def _block_language(attrs, text):  # Prefer explicit guide markers over conservative content recognition.
145    for value in attrs.get("class", "").split():  # Accept the language-* and lang-* conventions used by static guides.
146        if value.startswith(("language-", "lang-")):  # Keep explicit plain text out of automatic coloring.
147            return ALIASES.get(value.split("-", 1)[1].lower(), "text")  # Keep the existing markup when its convention needs a custom parser.
148    if re.search(r"\blayout\s*\(|\bgl_(?:Global|Local)InvocationID\b", text) or re.search(r"^\s*#version\b", text, re.MULTILINE):  # Recognize unlabeled native shader examples.
149        return "glsl"  # Preserve shader qualifiers and builtins.
150    if re.search(r"^\s*(?:cmake|ctest|python3?|git|SHDR_\w+=|\./build/|pip\s)\b", text, re.MULTILINE):  # Recognize the documented shell command prefixes.
151        return "bash"  # Color the command text in its declared shell language.
152    if re.search(r"\b(?:cmake_minimum_required|add_executable|target_link_libraries|shdr_target_shaders)\s*\(", text):  # Recognize unlabeled CMake integration excerpts.
153        return "cmake"  # Use CMake variable and command rules.
154    if re.search(r"^\s*(?:from\s+\w+\s+import|import\s+\w+|def\s+\w+\s*\()",
155                 text, re.MULTILINE):  # Recognize Python source snippets.
156        return "python"  # Keep imports and function definitions in their actual language.
157    if re.search(r"\b(?:shdr::|std::|template\s*<|void\s+main\s*\(|#include|float\s+\w+\s*[=(])", text):  # Recognize the remaining documented C++ examples and signatures.
158        return "cpp"  # Use the same lexer as the API declaration generator.
159    return "text"  # Leave unclassified logs, diagrams and literal data unchanged.
160
161
162def highlight_document(document, stylesheet="highlight.css"):  # Process guide excerpts and preserve structured sources and logs.
163    """Return idempotently highlighted HTML with unchanged visible code and links."""
164    reader = _CodeBlocks(document)  # Keep original offsets while inspecting block structure.
165    reader.feed(document)  # Locate only explicit pre/code excerpts.
166    for block in reversed(reader.blocks):  # Preserve earlier offsets while replacing later code interiors.
167        if block["nested"]:  # Existing spans and anchors belong to a generator or earlier highlighting pass.
168            continue  # Avoid duplicate token spans and line-gutter corruption.
169        text = unescape(document[block["start"]:block["end"]])  # Recover the original visible code once before lexing.
170        language = _block_language(block["attrs"], text)  # Respect explicit language markers and conservative fallback rules.
171        if language == "text":  # Preserve raw logs and deliberate text examples byte-for-byte.
172            continue  # Keep tables and diagram-like content out of syntax processing.
173        encoded = highlight_code(text, language)  # Preserve full-block multiline state and all original code characters.
174        document = document[:block["start"]] + encoded + document[block["end"]:]  # Retain the original code tags, IDs and surrounding prose.
175    return add_stylesheet(document, stylesheet)  # Keep all highlighting local and independent of browser JavaScript.
176
177
178def main():  # Apply the reusable guide postprocessor to a static documentation tree.
179    root = Path(__file__).resolve().parent.parent  # Resolve default paths independently of the current shell directory.
180    parser = argparse.ArgumentParser(description=__doc__)  # Explain that this is optional maintainer tooling.
181    parser.add_argument("paths", nargs="*", type=Path)  # Accept individual HTML pages or a complete documentation tree.
182    parser.add_argument("--docs", "--site", dest="docs", type=Path, default=root / "docs")  # Locate the shared local highlighting stylesheet.
183    parser.add_argument("--check", action="store_true")  # Report stale highlighting in check mode.
184    args = parser.parse_args()  # Read explicit regeneration or verification choices.
185    files = sorted({file for path in (args.paths or [args.docs]) for file in (path.rglob("*.html") if path.is_dir() else [path]) if file.suffix.lower() == ".html"})  # Process only static HTML artifacts.
186    changed = []  # Track files whose excerpt markup or stylesheet needs updating.
187    try:  # Turn a missing optional package into one useful maintainer diagnostic.
188        for path in files:  # Keep scientific SVGs and all non-HTML data outside this pass.
189            before = path.read_text(encoding="utf-8")  # Read exact original markup for idempotence comparison.
190            href = Path(os.path.relpath(args.docs / "highlight.css", path.parent)).as_posix()  # Keep nested source and benchmark styles local.
191            after = highlight_document(before, href)  # Preserve logs, record values, IDs, anchors and downloads.
192            if before != after:  # Avoid rewriting pages whose markup is already current.
193                changed.append(path)  # Include the page in the maintainer summary.
194                if not args.check:  # Verification mode must remain read-only.
195                    path.write_text(after, encoding="utf-8")  # Write only the static presentation changes.
196    except (RuntimeError, ValueError) as failure:  # Report missing dependencies or a preservation contract violation clearly.
197        parser.exit(2, str(failure) + "\n")  # Avoid an opaque traceback for a routine documentation setup problem.
198    print(("Needs highlighting: " if args.check else "Updated highlighting: ") + str(len(changed)) + " of " + str(len(files)) + " HTML pages")  # Summarize the updated page count.
199    if args.check and changed:  # Let a maintainer or CI check detect stale presentation.
200        parser.exit(1)  # Keep the command's status consistent with its verification result.
201
202
203if __name__ == "__main__":  # Run the postprocessor only when invoked explicitly.
204    main()  # Render or check the documentation's static syntax highlighting.