Source browser
scripts/report_public_shaders.py
Read the complete file below or download its original bytes. Line numbers provide stable links to a declaration or processing step.
File fingerprint
1#!/usr/bin/env python3
2"""Render complete public-shader compiler records as an accessible standalone report.
3
4The renderer accepts the verifier's ordinary Python report dictionary. Every
5recorded leaf value remains visible and carries its original field path, so a
6reader can inspect every result directly in the report.
7"""
8from html import escape # Keep source names and diagnostics inert inside HTML.
9from pathlib import Path # Read the documented library version from the runtime header.
10import re # Turn recorded field names into readable headings.
11
12
13def documentation_version(): # Identify the library version and preserve recorded report fields.
14 header = Path(__file__).resolve().parent.parent / "shdr.h" # Locate the runtime documented by this checkout.
15 match = re.search(r'^\s*#define\s+SHDR_VERSION\s+"([^"]+)"', header.read_text(encoding="utf-8"), re.MULTILINE) # Read the header's public version string.
16 if not match: # Require an identifiable library version in the report title.
17 raise ValueError("SHDR_VERSION is missing from shdr.h") # Identify the missing version declaration.
18 return escape(match.group(1)) # Escape the library version for HTML.
19
20
21STYLE = """
22body{margin:0;background:#f5f7fb;color:#17243a;font:16px/1.6 system-ui,sans-serif}
23main{max-width:1160px;margin:auto;padding:2rem 1.4rem}a{color:#155a9c}
24h1,h2,h3{line-height:1.2}h1{font-size:clamp(2rem,5vw,3.5rem)}h2{margin-top:2.8rem}
25.lead{max-width:78ch;color:#46546a;font-size:1.12rem}.eyebrow{text-transform:uppercase;letter-spacing:.14em;font-weight:700;color:#52657d}
26.report-nav{display:flex;gap:1rem;flex-wrap:wrap}.card-grid{display:grid;grid-template-columns:repeat(auto-fit,minmax(180px,1fr));gap:1rem}
27.card,.callout{background:white;border:1px solid #d6deea;border-radius:12px;padding:1.1rem}.card strong{font-size:2rem;display:block}
28.callout{border-left:4px solid #1c7790}.table-wrap{overflow:auto}table{border-collapse:collapse;width:100%;margin:1rem 0}
29th,td{padding:.65rem .75rem;text-align:left;vertical-align:top;border-bottom:1px solid #d6deea}th{font-weight:700}
30caption{text-align:left;font-weight:700;margin-bottom:.5rem}code{font: .89em ui-monospace,monospace;overflow-wrap:anywhere}
31pre{white-space:pre-wrap;overflow-wrap:anywhere;background:#edf1f7;padding:1rem;border-radius:6px;font: .85rem/1.55 ui-monospace,monospace}
32details{background:white;border:1px solid #d6deea;border-radius:8px;padding:.8rem 1rem;margin:.7rem 0}
33summary{cursor:pointer;font-weight:650}summary:focus-visible,a:focus-visible,input:focus-visible,select:focus-visible,button:focus-visible{outline:3px solid #ce6a2e;outline-offset:3px}
34.record-path{overflow-wrap:anywhere}.status{display:inline-block;font-size:.78rem;border-radius:20px;background:#e4ebf5;padding:.15rem .55rem;margin-left:.5rem}
35.status.compiled{background:#d9f0e6;color:#165b43}.controls{display:flex;gap:.8rem;align-items:end;flex-wrap:wrap;margin:1rem 0}
36.controls label{display:flex;flex-direction:column;gap:.3rem}.controls input,.controls select,.controls button{font:inherit;padding:.55rem;border:1px solid #8c9bb0;border-radius:5px}
37.controls input{min-width:min(25rem,75vw)}.field-table th{width:15rem}section.record-section{margin:1.2rem 0}.record-section h4{margin-bottom:.3rem}
38.skip-link{position:absolute;left:-9999px}.skip-link:focus{left:1rem;top:.3rem;background:white;padding:.5rem;z-index:2}
39[hidden]{display:none!important}.footer{max-width:1160px;margin:2rem auto;padding:1rem 1.4rem;border-top:1px solid #d6deea;color:#52657d}
40@media print{body{background:white}.controls,.site-nav{display:none}details{break-inside:avoid}a{color:inherit}}
41""" # Supply all report styles when the output is opened outside the documentation site.
42
43
44SEARCH = """
45(() => {
46 const records = [...document.querySelectorAll('.corpus-record')];
47 const text = records.map(record => record.textContent.toLowerCase());
48 const query = document.getElementById('record-search');
49 const repository = document.getElementById('repository-filter');
50 const status = document.getElementById('status-filter');
51 const count = document.getElementById('record-count');
52 function filter() {
53 const words = query.value.toLowerCase().trim().split(/\\s+/).filter(Boolean);
54 let visible = 0;
55 records.forEach((record, index) => {
56 const show = (!repository.value || record.dataset.repository === repository.value)
57 && (!status.value || record.dataset.status === status.value)
58 && words.every(word => text[index].includes(word));
59 record.hidden = !show;
60 if (show) visible += 1;
61 });
62 count.textContent = `${visible} of ${records.length} source records shown`;
63 }
64 query.addEventListener('input', filter);
65 repository.addEventListener('change', filter);
66 status.addEventListener('change', filter);
67 document.getElementById('expand-records').addEventListener('click', () => {
68 records.filter(record => !record.hidden).forEach(record => { record.open = true; });
69 });
70 document.getElementById('collapse-records').addEventListener('click', () => {
71 records.forEach(record => { record.open = false; });
72 });
73 filter();
74})();
75""" # Search the rendered records directly in the browser.
76
77
78def label(value): # Make recorded field names readable while retaining their exact spellings below.
79 return re.sub(r"[_-]+", " ", str(value)).capitalize() # Separate words in report identifiers.
80
81
82def scalar(value, path): # Render one exact report value with a machine-checkable original path.
83 kind = "null" if value is None else "boolean" if isinstance(value, bool) else "number" if isinstance(value, (int, float)) else "string" # Preserve the original scalar category.
84 text = "null" if value is None else "true" if value is True else "false" if value is False else str(value) # Format booleans and numeric values according to their recorded types.
85 attrs = f'data-report-path="{escape(path, quote=True)}" data-report-type="{kind}"' # Identify this value independently of its presentation.
86 if text.startswith(("https://", "http://")) and "\n" not in text: # Keep recorded source and license URLs directly usable.
87 return f'<a {attrs} href="{escape(text, quote=True)}">{escape(text)}</a>' # Show the complete source URL and its provenance.
88 tag = "pre" if "\n" in text or path.endswith(".diagnostic") else "span" # Keep diagnostic line breaks available to readers and preservation checks.
89 return f'<{tag} {attrs}>{escape(text)}</{tag}>' # Escape compiler output before placing it in the document.
90
91
92def fields(value, path): # Render nested report dictionaries and lists with every field retained.
93 if isinstance(value, dict): # Expose every dictionary entry with its original key.
94 rows = [] # Build readable key and value rows in source order.
95 for key, child in value.items(): # Preserve optional configuration and compiler fields as well as the usual ones.
96 rendered = fields(child, path + "." + key) # Keep a complete path for every nested value.
97 rows.append(f'<tr><th scope="row"><code>{escape(key)}</code></th><td>{rendered}</td></tr>') # Show exact field names alongside their values.
98 return '<div class="table-wrap"><table class="field-table"><tbody>' + "".join(rows) + "</tbody></table></div>" # Keep wide hashes and diagnostics usable on small screens.
99 if isinstance(value, list): # Preserve source paths and any future list-valued metadata.
100 return "<ol>" + "".join(f"<li>{fields(child, path + '[' + str(index) + ']')}</li>" for index, child in enumerate(value)) + "</ol>" # Retain list order in the visible document.
101 return scalar(value, path) # Render the final scalar verbatim.
102
103
104def navigation(site_navigation): # Choose sibling-page navigation when the report is part of the guide.
105 if not site_navigation: # Keep arbitrary verifier output paths fully standalone.
106 return '<nav class="report-nav" aria-label="Report sections"><a href="#summary">Summary</a><a href="#workflow">Import a shader</a><a href="#provenance">Provenance</a><a href="#records">All source records</a></nav>' # Use only anchors present in this report.
107 return '<header class="site-header"><div class="header-inner"><a class="brand" href="index.html">shdr<span class="release-version">' + documentation_version() + '</span><span>CPU + GPU compute</span></a><nav class="site-nav" aria-label="Main navigation"><a href="index.html">Start</a><a href="examples.html">Examples</a><a href="results.html">Results</a><a href="corpus.html" aria-current="page">Public shaders</a><a href="api.html">API</a></nav></div></header>' # Match the shared static documentation shell.
108
109
110def import_guide(site_navigation): # Explain imports through the documented language subset.
111 subset = '<a href="portability.html">supported language subset</a>' if site_navigation else "supported language subset described by this library" # Avoid broken sibling links in standalone verifier output.
112 return f"""
113<section id="workflow"><h2>Import a public compute shader</h2>
114<p>Start with a compute shader that reads and writes independent buffer elements. Add its source to <code>shdr_target_shaders</code>, bind its declared resources by name, and run the resulting pipeline on the CPU or a compatible GPU. An existing <code>.comp</code> extension is accepted. The build embeds the portable GPU variant directly in the application.</p>
115<p>Use a unique identifier-shaped source basename in each linked target, such as <code>multiply.comp</code> or <code>filter.cpp</code>. The CMake helper derives its factory name from that basename. Each basename identifies one factory within the target.</p>
116<p>Backend compatibility depends on buffer layout, builtins, extensions and synchronization. Consult the {subset} before importing shared workgroup memory, atomics, samplers or custom records.</p>
117<p>A mutable global with an explicit initializer starts from that value for every invocation on both backends. Each invocation receives independent private global storage. Put shared or persistent data in a bound buffer or image, and assign uninitialized private variables before reading them.</p></section>
118<section id="scope"><h2>What these checks establish</h2>
119<p>The selected inputs cover complete compute programs, compiler conformance fixtures and support units. Reference acceptance means that the original input compiled and linked in OpenGL mode, or compiled to Vulkan SPIR-V when its declarations required Vulkan. CPU translation covers inputs accepted by at least one reference mode. Every record retains both reference outcomes.</p>
120<p>Checked-in reference transcripts identify upstream fixtures with expected compiler diagnostics. This metadata and the standalone compiler result describe their respective test configurations. Every source record preserves the original expectation flag.</p>
121<div class="callout"><strong>Compiler coverage.</strong> Successful exports establish compiler compatibility with the five language profiles. Applications supply the host resources, dimensions and valid input domains for runtime execution. Bundled runtime examples measure CPU/GPU agreement against independent references on the listed devices.</div>
122<p>Devices must also support the imported resource counts, workgroup sizes and input dimensions. The bundled runtime examples separately compare CPU and GPU values, including independent numerical references. Inspect each record and validate an imported kernel with its intended inputs before deployment.</p></section>
123<section id="configuration"><h2>Source assembly and portable imports</h2>
124<p>NCNN sources depend on host-supplied types and load/store macros. The verifier inserts the exact pinned <code>src/ncnn_glsl_ext.comp</code> prelude and records the FP32 buffer configuration. This configuration uses FP32 buffer storage. The records retain the shared-memory paths selected by <code>NCNN_shader_local_memory=1</code> and their compiler diagnostics. Upstream activation includes are resolved by glslang before CPU translation. The upstream body remains unchanged. Configuration, prelude, configured-source and preprocessed-source hashes are shown in the affected records.</p>
125<p>The adapter selects a version, assigns buffer bindings and layouts, maps descriptor set zero and specialization defaults, and exposes supported push-constant members as named uniforms. The portable GPU variant supplies numeric promotions required by GLSL ES, removes local <code>const</code> qualifiers from runtime initializers, and handles integer boolean-selector <code>mix</code> on older profiles. It preserves expression evaluation order and keeps output and input-output arguments as lvalues. Global and compile-time local constants retain their qualifiers. These imports use the OpenGL resource and dispatch API.</p>
126<p>The dual-backend subset covers independent invocations, single-array storage blocks, fixed local arrays, documented storage images and math operations. The detailed records identify requirements involving workgroup coordination, atomics, multi-field storage blocks, additional image formats, samplers, subgroups, cooperative matrices and reduced-precision types through their compiler diagnostics.</p></section>
127<section id="reproduce"><h2>Reproduce the compiler checks</h2>
128<p>Use Python 3.8 or newer, Git, a C++11 compiler and <code>glslangValidator</code>. The commands below fetch pinned inputs explicitly into a separate cache.</p>
129<pre><code># Fetch the pinned collection into a separate cache and write a standalone report.
130python3 scripts/verify_public_shaders.py --fetch \\
131 --cache build/public-shaders --output build/public-shader-results.html
132# Reuse the prepared local cache.
133python3 scripts/verify_public_shaders.py \\
134 --cache build/public-shaders --output build/public-shader-results.html</code></pre>
135<p>A build configured with glslang also provides <code>cmake --build build --target verify_public_shaders</code>. Populate its cache first. The <code>--jobs</code>, <code>--cxx</code> and <code>--glslang</code> options select parallelism and tools. Each report records fingerprints for the implementation held constant throughout verification.</p>
136<p>Fetch the upstream sources with the command above and follow their licenses when incorporating them. Every repository pin, source link and license link is retained below.</p></section>
137""" # Keep the complete import, scope, assembly and reproduction explanation with the report itself.
138
139
140def render_public_report(report, site_navigation=False): # Render the complete report with the Python standard library.
141 version = documentation_version() # Show the library version alongside the recorded compiler fingerprints.
142 counts = report["summary"]["counts"] # Use measured totals to describe compiler coverage.
143 records = report["results"] # Preserve every selected source, including reference rejections.
144 compiled = counts.get("cpu_and_all_profiles_compiled", 0) # Count only sources accepted by C++ and every generated profile.
145 accepted = counts.get("opengl_passed", 0) + counts.get("vulkan_only_passed", 0) # Keep OpenGL acceptance distinct from the Vulkan fallback.
146 extra_css = '<link rel="stylesheet" href="site.css">' if site_navigation else "" # Use shared site styling only where that asset is available.
147 pieces = [f'<!doctype html><html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1"><meta name="description" content="Complete pinned public compute-shader compiler records with source provenance and diagnostics."><title>Public shader compatibility · shdr {version}</title><style>{STYLE}</style>{extra_css}</head><body><a class="skip-link" href="#content">Skip to content</a>{navigation(site_navigation)}<div class="page-shell"><main class="page-main" id="content">'] # Make the output readable with JavaScript disabled or outside GitHub Pages.
148 pieces.append(f'<p class="eyebrow">Recorded compiler compatibility</p><h1>Every source, every outcome</h1><p class="lead">Explore {len(records)} public compute inputs from five pinned collections. Read the original links, exact configurations, compiler outcomes and diagnostics.</p><div class="card-grid"><div class="card"><strong>{len(records)}</strong>selected inputs</div><div class="card"><strong>{accepted}</strong>reference accepted</div><div class="card"><strong>{compiled}</strong>C++11 and all five profiles</div><div class="card"><strong>0</strong>public corpus shaders executed</div></div>') # State compiler coverage and its execution limit together.
149 pieces.append('<section id="summary"><h2>Recorded totals</h2>' + fields(report["summary"], "$.summary") + "</section>") # Retain all aggregate counts and rejection categories.
150 pieces.append(import_guide(site_navigation)) # Explain how to interpret and reproduce the outcomes.
151 metadata = {key: value for key, value in report.items() if key not in ("results", "summary", "repositories")} # Keep every top-level provenance field exactly once.
152 pieces.append('<section id="provenance"><h2>Compiler and implementation provenance</h2>' + fields(metadata, "$") + '<p>The recorded header, frontend and tool fingerprints identify the implementation used for these compiler checks. Source records retain the original compiler diagnostics.</p></section>') # Preserve tool versions, hashes, selection rules and excluded-input counts.
153 pieces.append('<section id="repositories"><h2>Pinned repositories and licenses</h2>') # Keep source selection and licensing together in the report.
154 for index, repository in enumerate(report["repositories"]): # Retain the original order of the five recorded collections.
155 pieces.append('<details open><summary>' + escape(repository["id"]) + "</summary>" + fields(repository, f"$.repositories[{index}]") + "</details>") # Include every pin, path, assembly link and optional configuration field.
156 pieces.append('</section><section id="records"><h2>All source records</h2><p>Search paths, hashes, compiler messages or configuration values. Repository and CPU-status filters combine with the search. Every record uses native expandable HTML. Search and filters use local JavaScript.</p><div class="controls"><label>Search records<input id="record-search" type="search" placeholder="Path, builtin, hash or diagnostic"></label><label>Repository<select id="repository-filter"><option value="">All repositories</option>') # Use native accessible controls over fully rendered content.
157 for repository in report["repositories"]: # Offer only repository choices represented by this report.
158 key = escape(repository["id"], quote=True) # Escape even verifier-controlled identifiers before embedding attributes.
159 pieces.append(f'<option value="{key}">{key}</option>') # Keep filter values identical to recorded repository IDs.
160 pieces.append('</select></label><label>CPU result<select id="status-filter"><option value="">All outcomes</option>') # Keep not-attempted records separate from compiler failures.
161 for status in sorted({record["cpu"]["status"] for record in records}): # Derive the available CPU outcomes from the recorded results.
162 pieces.append(f'<option value="{escape(status, quote=True)}">{escape(label(status))}</option>') # Offer readable status labels while preserving exact values.
163 pieces.append('</select></label><button id="expand-records" type="button">Expand shown</button><button id="collapse-records" type="button">Collapse all</button></div><p id="record-count" role="status" aria-live="polite">' + str(len(records)) + ' source records</p>') # Update the result count while keyboard focus stays in place.
164 for index, record in enumerate(records): # Show every detailed source record in its original order.
165 state = record["cpu"]["status"] # Identify the CPU outcome in the closed summary.
166 repository = record["repository"] # Group search results by the original collection identifier.
167 pieces.append(f'<details class="corpus-record" id="record-{index + 1:04d}" data-repository="{escape(repository, quote=True)}" data-status="{escape(state, quote=True)}"><summary><span class="record-path">{escape(repository)} / {escape(record["path"])}</span><span class="status {escape(state, quote=True)}">{escape(label(state))}</span></summary>{fields(record, f"$.results[{index}]")}</details>') # Preserve source hashes, configured hashes and all compiler-stage diagnostics.
168 pieces.append(f'</section></main></div><footer class="footer"><p>shdr {version} · Public collection compiler report. Resource limits and numerical behavior require application-specific validation.</p></footer><script>' + SEARCH + "</script></body></html>\n") # Inline search logic supports standalone reading.
169 return "\n".join(pieces) # Return the complete standalone HTML report.