Source browser
scripts/verify_public_shaders.py
Read the complete file below or download its original bytes. Line numbers provide stable links to a declaration or processing step.
File fingerprint
1#!/usr/bin/env python3
2"""Compile pinned public compute shaders and record their compiler outcomes.
3
4Upstream sources reside in a separate cache. Reference checks use standalone
5inputs or recorded upstream configurations. CPU checks compile generated C++11
6adapters, and accepted adapters export GPU programs for all five profiles.
7Numerical checks require application resources, valid inputs and reference values.
8"""
9
10import argparse # Read command-line options and show help.
11from collections import Counter # Count each compiler outcome by category.
12from concurrent.futures import ThreadPoolExecutor # Check independent sources with a bounded set of workers.
13import hashlib # Identify exact source and header snapshots.
14import json # Write compiler results as a machine-readable report.
15from pathlib import Path # Build paths and read or write files.
16import re # Find declarations, comments and diagnostic lines.
17import subprocess # Run compilers with direct argument lists.
18import tempfile # Keep intermediate source and binaries in temporary directories.
19import types # Load the frontend into an isolated module object.
20from report_public_shaders import render_public_report # Render complete compiler evidence as a standalone HTML document.
21
22
23ROOT = Path(__file__).resolve().parents[1] # Find the checkout containing the maintained source files.
24PROFILES = ("es31", "es32", "gl43", "gl45", "gl46") # List every supported shader-language profile.
25REPOSITORIES = ( # Pin upstream revisions and record their shader locations.
26 { # Describe this pinned upstream source collection.
27 "id": "glslang", # Name the repository and its cache folder.
28 "url": "https://github.com/KhronosGroup/glslang.git", # Record the upstream Git URL.
29 "revision": "2fbed5e1fb43a1c480fe020e3bb686a31010989a", # Pin the exact upstream commit.
30 "paths": ["Test"], # Select the tracked directories containing shader inputs.
31 "license_url": "https://github.com/KhronosGroup/glslang/blob/2fbed5e1fb43a1c480fe020e3bb686a31010989a/LICENSE.txt", # Link the upstream license for reuse decisions.
32 "description": "Khronos reference compiler compute fixtures, including negative tests", # Describe this source collection.
33 }, # Complete the pinned Khronos compiler fixture collection.
34 { # Describe this pinned upstream source collection.
35 "id": "ogl-samples", # Name the repository and its cache folder.
36 "url": "https://github.com/g-truc/ogl-samples.git", # Record the upstream Git URL.
37 "revision": "38cada7a9458864265e25415ae61586d500ff5fc", # Pin the exact upstream commit.
38 "paths": ["data"], # Select the tracked directories containing shader inputs.
39 "license_url": "https://github.com/g-truc/ogl-samples/blob/38cada7a9458864265e25415ae61586d500ff5fc/LICENSE.md", # Link the upstream license for reuse decisions.
40 "description": "MIT-licensed OpenGL sample compute shaders", # Describe this source collection.
41 }, # Complete the pinned desktop OpenGL sample collection.
42 { # Describe this pinned upstream source collection.
43 "id": "vulkan", # Name the repository and its cache folder.
44 "url": "https://github.com/SaschaWillems/Vulkan.git", # Record the upstream Git URL.
45 "revision": "4ee2501c8280cf71d1f9ce231cd8ebd47b58adde", # Pin the exact upstream commit.
46 "paths": ["shaders/glsl"], # Select the tracked directories containing shader inputs.
47 "license_url": "https://github.com/SaschaWillems/Vulkan/blob/4ee2501c8280cf71d1f9ce231cd8ebd47b58adde/LICENSE.md", # Link the upstream license for reuse decisions.
48 "description": "MIT-licensed Vulkan example GLSL compute shaders", # Describe this source collection.
49 }, # Complete the pinned Vulkan example collection.
50 { # Describe this pinned upstream source collection.
51 "id": "kompute", # Name the repository and its cache folder.
52 "url": "https://github.com/KomputeProject/kompute.git", # Record the upstream Git URL.
53 "revision": "267e019ec419ae57f634b341983b60972924b58f", # Pin the exact upstream commit.
54 "paths": ["examples", "test", "benchmark"], # Select the tracked directories containing shader inputs.
55 "license_url": "https://github.com/KomputeProject/kompute/blob/267e019ec419ae57f634b341983b60972924b58f/LICENSE", # Link the upstream license for reuse decisions.
56 "description": "Apache-2.0-licensed general-purpose compute examples", # Describe this source collection.
57 }, # Complete the pinned general compute collection.
58 { # Describe this pinned upstream source collection.
59 "id": "ncnn", # Name the repository and its cache folder.
60 "url": "https://github.com/Tencent/ncnn.git", # Record the upstream Git URL.
61 "revision": "fcde729c7b0b0fcecf273ed21b0e25dd2deb67be", # Pin the exact upstream commit.
62 "paths": ["src/layer/vulkan/shader"], # Select the tracked directories containing shader inputs.
63 "license_url": "https://github.com/Tencent/ncnn/blob/fcde729c7b0b0fcecf273ed21b0e25dd2deb67be/LICENSE.txt", # Link the upstream license for reuse decisions.
64 "description": "BSD-3-Clause-licensed neural-network compute kernels with the upstream FP32 buffer prelude", # Describe this source collection.
65 "configuration": "ncnn_fp32_buffer", # Record any required upstream build configuration.
66 "prelude_path": "src/ncnn_glsl_ext.comp", # Locate the upstream macro and type prelude.
67 "configuration_source_url": "https://github.com/Tencent/ncnn/blob/fcde729c7b0b0fcecf273ed21b0e25dd2deb67be/src/gpu.cpp#L6285", # Link the upstream source-assembly implementation.
68 }, # Complete the pinned configured neural-network collection.
69) # Keep all upstream metadata beside the selected revisions.
70
71
72def digest(data): # Identify exact source bytes with SHA-256.
73 return hashlib.sha256(data).hexdigest() # Return the hexadecimal source fingerprint.
74
75
76def run(command, cwd=None, timeout=45, input_text=None): # Capture a compiler exit status and its diagnostic output.
77 """Run compiler tools through direct argument lists."""
78 try: # Catch a compiler that exceeds its per-case timeout.
79 result = subprocess.run( # Capture this command's status and output.
80 command, # Pass each compiler argument directly.
81 cwd=cwd, # Run relative paths in the requested working directory.
82 stdout=subprocess.PIPE, # Capture standard output for the report.
83 stderr=subprocess.STDOUT, # Keep compiler errors in the same captured output.
84 text=True, # Decode process output as text.
85 timeout=timeout, # Stop a compiler that exceeds this case's time limit.
86 input=input_text, # Supply embedded GLSL text to the exporter through standard input.
87 ) # Capture the combined compiler output when the process finishes.
88 return result.returncode, result.stdout # Return the compiler status and captured diagnostic text.
89 except subprocess.TimeoutExpired: # Treat an expired compiler as a failed check with a distinct status.
90 return 124, "compiler timeout" # Return timeout status 124 with a short explanation.
91
92
93def checked(command, cwd=None): # Require a helper command to finish successfully.
94 code, output = run(command, cwd=cwd, timeout=300) # Allow cache and helper commands a longer timeout.
95 if code: # Handle a nonzero compiler status as a failed check.
96 raise RuntimeError(output.strip()) # Report helper failures before relying on their output.
97 return output.strip() # Return the successful command output.
98
99
100def prepare_cache(cache, fetch): # Check each pinned repository before reading its sources.
101 """Read pinned snapshots. Download them only when --fetch is supplied."""
102 cache.mkdir(parents=True, exist_ok=True) # Create the cache parent and preserve existing snapshots.
103 for repository in REPOSITORIES: # Visit every pinned public source collection.
104 folder = cache / repository["id"] # Locate the cached upstream repository.
105 if not (folder / ".git").exists(): # Create a Git snapshot for a fresh cache.
106 if not fetch: # Require explicit download permission through the fetch option.
107 raise RuntimeError("missing " + repository["id"] + " cache. Run with --fetch first") # Explain how to populate a missing pinned cache.
108 if folder.exists() and any(folder.iterdir()): # Avoid placing a repository into an unrelated nonempty directory.
109 raise RuntimeError("cache destination is not an empty repository: " + str(folder)) # Protect unrelated files in a nonempty cache destination.
110 folder.mkdir(parents=True, exist_ok=True) # Create the empty cache directory before Git uses it.
111 checked(["git", "init", "--quiet", str(folder)]) # Initialize an empty repository for this missing cache.
112 checked(["git", "remote", "add", "origin", repository["url"]], cwd=folder) # Record the upstream URL as this repository's origin.
113 checked( # Fetch the pinned commit with bounded download history.
114 [ # List the Git arguments for the pinned source download.
115 "git", # Use Git for the pinned source download.
116 "fetch", # Download the chosen revision before sparse checkout.
117 "--depth=1", # Avoid downloading unrelated commit history.
118 "--filter=blob:none", # Delay file downloads until the selected paths are checked out.
119 "origin", # Read from the recorded upstream repository.
120 repository["revision"], # Fetch the exact pinned commit.
121 ], # Complete the arguments for downloading only the pinned revision.
122 cwd=folder, # Run this Git operation inside its cache repository.
123 ) # Fetch inside this upstream cache repository.
124 checked(["git", "sparse-checkout", "init", "--cone"], cwd=folder) # Enable sparse checkout to limit cached source files.
125 checked(["git", "sparse-checkout", "set"] + repository["paths"], cwd=folder) # Select only the repository paths listed for this collection.
126 checked(["git", "checkout", "--quiet", "--detach", repository["revision"]], cwd=folder) # Check out the pinned commit in detached mode.
127 revision = checked(["git", "rev-parse", "HEAD"], cwd=folder) # Read the revision currently checked out in the cache.
128 if revision != repository["revision"]: # Require the cache to match its pinned commit.
129 raise RuntimeError( # Require the recorded snapshot revision.
130 repository["id"] + " cache is at a different revision. Use a fresh cache directory" # Identify the cache revision validation error.
131 ) # Name the mismatched cache in the failure message.
132 if checked(["git", "status", "--porcelain", "--untracked-files=no"], cwd=folder): # Reject modified tracked inputs before compiler checks begin.
133 raise RuntimeError(repository["id"] + " cache contains modified tracked files") # Reject tracked edits that would change the recorded upstream inputs.
134
135
136def cases_from_cache(cache): # Select tracked compute sources and record negative-test metadata.
137 cases = [] # Collect the selected source records.
138 excluded_hlsl = 0 # Count separately identified HLSL files.
139 for repository in REPOSITORIES: # Visit every pinned public source collection.
140 folder = cache / repository["id"] # Locate the cached upstream repository.
141 tracked = checked(["git", "ls-files", "-z", "--"] + repository["paths"], cwd=folder) # Read the repository's tracked source paths.
142 for relative in sorted(tracked.split("\0")): # Visit each tracked shader path in stable order.
143 if not relative.endswith(".comp"): # Select compute GLSL files and skip unrelated tracked files.
144 continue # Skip this item and inspect the next one.
145 source = folder / relative # Locate the tracked compute source inside this repository.
146 if "hlsl." in source.name: # Select filenames belonging to the GLSL collection.
147 excluded_hlsl += 1 # Count separately identified HLSL files.
148 continue # Move to the next path after counting an HLSL input.
149 golden = folder / "Test" / "baseResults" / (source.name + ".out") # Locate any upstream reference transcript for this fixture.
150 reference_has_errors = golden.exists() and "ERROR:" in golden.read_text( # Remember whether upstream marked this fixture as an error case.
151 errors="replace" # Read upstream transcripts even when they contain unusual bytes.
152 ) # Finish reading the optional upstream reference transcript.
153 cases.append((repository, relative, reference_has_errors)) # Keep this tracked source and its upstream test marker.
154 return cases, excluded_hlsl # Return selected GLSL cases and the separate HLSL count.
155
156
157def diagnostic(text, substitutions): # Keep a concise compiler explanation with generic paths.
158 """Keep concise compiler reasons with generic cache paths."""
159 for original, replacement in substitutions: # Remove each machine-specific path from the diagnostic.
160 text = text.replace(str(original), replacement) # Replace private paths while keeping the compiler explanation.
161 lines = [ # Collect only the most useful diagnostic lines.
162 line.strip() # Remove diagnostic indentation before storing it.
163 for line in text.splitlines() # Inspect each compiler output line for useful errors.
164 if "error" in line.lower() or "ERROR" in line or "unsupported" in line.lower() # Keep the compiler's substantive error messages.
165 ] # Keep the selected error lines in their original order.
166 return "\n".join(lines[:4])[:1200] if lines else text.strip()[:500] # Return a short portable explanation.
167
168
169def reason_category(reason): # Group rejected constructs by their reported reason.
170 for needle, category in ( # Classify reported resource and language requirements.
171 ("shared memory", "workgroup_synchronization_or_atomics"), # Group workgroup-state and synchronization failures.
172 ("uniform type", "uniform_or_sampler_interface"), # Group uniform and sampler interface diagnostics.
173 ("buffer blocks", "storage_buffer_layout"), # Group storage-block declaration diagnostics.
174 ("storage buffers", "storage_buffer_layout"), # Group incompatible storage element layouts.
175 ("image", "image_format_or_interface"), # Group image format and interface diagnostics.
176 ("layout", "layout_or_execution_qualifier"), # Group resource and execution qualifier diagnostics.
177 ("include", "external_shader_dependency"), # Group unresolved source dependencies.
178 ("main", "entrypoint_or_linked_unit"), # Group missing standalone entry points.
179 ("#version", "source_version_directive"), # Group source-version problems.
180 ): # Test these categories in order so the first match is stable.
181 if needle in reason: # Use the first category whose keyword appears in the rejection reason.
182 return category # Return the matching rejection category.
183 return "frontend_body_or_declaration" # Return the matching rejection category.
184
185
186def configured_source(repository, source, data, cache): # Assemble the documented NCNN FP32 buffer configuration.
187 """Assemble NCNN's documented FP32-buffer source and preserve its upstream body."""
188 if repository.get("configuration") != "ncnn_fp32_buffer": # Use the upstream configuration only when this source needs it.
189 return data.decode("utf-8"), None # Return the configured source and its provenance.
190 header = cache / repository["id"] / repository["prelude_path"] # Locate the upstream source prelude.
191 prelude = header.read_bytes() # Read the exact prelude bytes for configuration provenance.
192 defines = { # Select the documented FP32 buffer configuration.
193 "NCNN_fp16_storage": 0, # Set this upstream feature flag for FP32 buffer shaders.
194 "NCNN_fp16_packed": 0, # Set this upstream feature flag for FP32 buffer shaders.
195 "NCNN_fp16_arithmetic": 0, # Set this upstream feature flag for FP32 buffer shaders.
196 "NCNN_bf16_storage": 0, # Set this upstream feature flag for FP32 buffer shaders.
197 "NCNN_bf16_packed": 0, # Set this upstream feature flag for FP32 buffer shaders.
198 "NCNN_image_storage": 0, # Set this upstream feature flag for FP32 buffer shaders.
199 "NCNN_shader_local_memory": 1, # Set this upstream feature flag for FP32 buffer shaders.
200 "NCNN_moltenvk": 0, # Set this upstream feature flag for FP32 buffer shaders.
201 } # Finish the upstream configuration description.
202 options = "#extension GL_GOOGLE_include_directive : require\n" # Enable the include directive required by the upstream NCNN configuration.
203 options += ( # Add all selected feature definitions before the upstream prelude.
204 "\n".join("#define {} {}".format(name, value) for name, value in defines.items()) + "\n" # Emit the selected upstream feature definitions before its shader body.
205 ) # Finish the newline-terminated upstream configuration flags.
206 text = data.decode("utf-8") # Assemble the shader text used by this compiler check.
207 version = re.search(r"(?m)^[ \t]*#\s*version[^\n]*(?:\n|$)", text) # Locate the input version directive before inserting its prelude.
208 if version: # Insert configuration text after the existing version directive.
209 text = ( # Assemble the shader text used by this compiler check.
210 text[: version.end()] + options + prelude.decode("utf-8") + "\n" + text[version.end() :] # Keep the upstream body unchanged while inserting its configuration.
211 ) # Keep the existing version first, followed by configuration and shader body.
212 else: # Supply a desktop version for an unversioned upstream body.
213 text = "#version 450\n" + options + prelude.decode("utf-8") + "\n" + text # Assemble the shader text used by this compiler check.
214 metadata = { # Record how this configured source was assembled.
215 "name": repository["configuration"], # Name the configuration used for this source.
216 "defines": defines, # Record the selected upstream macro values.
217 "prelude_path": repository["prelude_path"], # Locate the upstream macro and type prelude.
218 "prelude_sha256": digest(prelude), # Identify the exact prelude bytes.
219 "include_directory": str(source.parent.relative_to(cache / repository["id"])), # Record the repository-relative include directory.
220 "source_assembly_url": repository["configuration_source_url"], # Link the source of the configuration recipe.
221 } # Finish the upstream configuration description.
222 return text, metadata # Return the configured source and its provenance.
223
224
225def verify_case(case, cache, options, frontend): # Check one source through the reference, CPU and GPU compilers.
226 repository, relative, reference_has_errors = case # Read the selected source and its upstream error marker.
227 source = cache / repository["id"] / relative # Locate the original upstream source file.
228 data = source.read_bytes() # Read the original upstream shader bytes.
229 record = { # Start the report entry for this exact source.
230 "repository": repository["id"], # Identify the upstream project for this source.
231 "path": relative, # Record the source path relative to its repository.
232 "source_url": repository["url"][:-4] + "/blob/" + repository["revision"] + "/" + relative, # Link the exact pinned upstream source.
233 "sha256": digest(data), # Identify the original source bytes.
234 "upstream_reference_has_errors": reference_has_errors, # Keep the upstream negative-test marker.
235 } # Finish recording the original source and its upstream error marker.
236 substitutions = [(cache, "<corpus>"), (ROOT, "<shdr>")] # Replace temporary and checkout paths in diagnostics.
237 with tempfile.TemporaryDirectory(prefix="shdr-public-check-") as directory: # Remove these intermediate compiler files when the check ends.
238 generated = Path(directory) # Locate this case's temporary compiler files.
239 substitutions.append((generated, "<generated>")) # Remove this temporary directory from compiler diagnostics.
240 text, configuration = configured_source(repository, source, data, cache) # Assemble the shader text used by this compiler check.
241 reference = source # Choose the standalone source supplied to the reference compiler.
242 include = [] # Pass upstream include paths only when its configuration needs them.
243 if configuration: # Use the upstream configuration only when this source needs it.
244 record["configuration"] = configuration # Record the upstream configuration used to assemble this source.
245 reference = generated / "configured.comp" # Choose the standalone source supplied to the reference compiler.
246 reference.write_text(text) # Write the configured source used by the reference compiler.
247 record["configured_sha256"] = digest(text.encode()) # Fingerprint the configured source before any frontend processing.
248 include = ["-I" + str(source.parent)] # Pass upstream include paths only when its configuration needs them.
249 opengl = [options.glslang, "-S", "comp", "-l"] + include + [str(reference)] # Build the reference OpenGL compile-and-link command.
250 code, output = run(opengl) # Compile and link the reference source in OpenGL mode.
251 record["opengl"] = {"status": "passed" if code == 0 else "rejected", "exit_code": code} # Keep this compiler status or diagnostic with the source record.
252 if code: # Handle a nonzero compiler status as a failed check.
253 record["opengl"]["diagnostic"] = diagnostic(output, substitutions) # Keep this compiler status or diagnostic with the source record.
254 vulkan = ( # Build the reference Vulkan SPIR-V fallback command.
255 [options.glslang, "-V", "-S", "comp", "-l", "-o", str(generated / "shader.spv")] # Select Vulkan compute compilation and a temporary SPIR-V output.
256 + include # Keep any required upstream include directory.
257 + [str(reference)] # Complete the Vulkan fallback command with the selected source path.
258 ) # Finish this compiler command or source-result record.
259 vk_code, vk_output = run(vulkan) # Try the Vulkan reference path when OpenGL mode rejects the input.
260 record["vulkan"] = { # Keep this compiler status or diagnostic with the source record.
261 "status": "passed" if vk_code == 0 else "rejected", # Record whether this compiler accepted the source.
262 "exit_code": vk_code, # Record the Vulkan fallback status alongside the OpenGL result.
263 } # Finish this compiler command or source-result record.
264 if vk_code: # Handle a nonzero compiler status as a failed check.
265 record["vulkan"]["diagnostic"] = diagnostic(vk_output, substitutions) # Keep this compiler status or diagnostic with the source record.
266 else: # Avoid a redundant Vulkan check after OpenGL compilation succeeds.
267 record["vulkan"] = {"status": "not_needed"} # Keep this compiler status or diagnostic with the source record.
268
269 if code and record["vulkan"]["status"] != "passed": # Stop adaptation when neither reference compiler accepted this source.
270 record["cpu"] = { # Keep this compiler status or diagnostic with the source record.
271 "status": "not_attempted", # Record whether this compiler accepted the source.
272 "reason": "reference_compiler_did_not_accept_standalone_source", # Explain the prerequisite for this compiler check.
273 } # Record the CPU check's reference-acceptance prerequisite.
274 return record # Return the completed compiler record for this source.
275 name = "public_" + digest(relative.encode())[:16] + "_kernel" # Make a unique valid C++ factory name from the source path.
276 if configuration: # Use the upstream configuration only when this source needs it.
277 preprocess = [options.glslang, "-E", "-S", "comp"] + include + [str(reference)] # Ask glslang to resolve the recorded upstream configuration.
278 preprocessed_code, text = run(preprocess) # Resolve upstream macros and includes before CPU translation.
279 if preprocessed_code: # Handle a nonzero compiler status as a failed check.
280 record["cpu"] = { # Keep this compiler status or diagnostic with the source record.
281 "status": "frontend_rejected", # Record whether this compiler accepted the source.
282 "category": "upstream_configuration_preprocessing", # Classify the rejected source construct.
283 "diagnostic": diagnostic(text, substitutions), # Record the upstream preprocessing failure before CPU translation.
284 } # Finish this compiler command or source-result record.
285 return record # Return the preprocessing failure as this source's result.
286 record["preprocessed_sha256"] = digest(text.encode()) # Fingerprint the upstream-preprocessed text supplied to the frontend.
287 try: # Catch translation errors while keeping this source's compiler record.
288 adapter = frontend.generate(text, name) # Translate supported GLSL into standard C++11.
289 except (frontend.ShaderError, UnicodeError) as error: # Record the frontend syntax diagnostic as a source rejection.
290 reason = str(error) # Keep the frontend's rejection explanation.
291 record["cpu"] = { # Keep this compiler status or diagnostic with the source record.
292 "status": "frontend_rejected", # Record whether this compiler accepted the source.
293 "category": reason_category(reason), # Classify the rejected source construct.
294 "diagnostic": reason, # Keep the short rejection explanation.
295 } # Finish recording the frontend rejection category and explanation.
296 return record # Return the completed compiler record for this source.
297 adapter_path = generated / "adapter.cpp" # Locate the generated CPU translation unit.
298 adapter_path.write_text(adapter) # Write the generated CPU translation unit.
299 command = [ # Build the strict C++11 compiler invocation.
300 options.cxx, # Use the requested C++ compiler.
301 "-std=c++11", # Require the library's minimum C++ language standard.
302 "-pedantic-errors", # Treat nonstandard C++ syntax as an error.
303 "-I", # Add the runtime header directory to the include search path.
304 str(ROOT), # Read the runtime header from this checkout.
305 "-fsyntax-only", # Compile the generated CPU adapter to an object file.
306 str(adapter_path), # Compile this source's generated CPU translation unit.
307 ] # Finish this compiler command or source-result record.
308 cpu_code, cpu_output = run(command) # Complete the strict syntax-only C++11 compiler command.
309 record["cpu"] = { # Keep this compiler status or diagnostic with the source record.
310 "status": "compiled" if cpu_code == 0 else "cpp_rejected", # Record whether this compiler accepted the source.
311 "exit_code": cpu_code, # Record the compiler's numeric exit status.
312 } # Finish this compiler command or source-result record.
313 if cpu_code: # Record the C++ compiler status for the generated adapter.
314 record["cpu"]["diagnostic"] = diagnostic(cpu_output, substitutions) # Keep this compiler status or diagnostic with the source record.
315 else: # Check GPU profiles only after the CPU adapter compiled.
316 _, gpu, _ = frontend.transform(text, name) # Read the GPU declarations and body from the same translated source.
317 record["generated_gpu"] = {} # Create the per-profile result map for this accepted adapter.
318 for profile in PROFILES: # Check every supported shader-language profile.
319 export_code, exported = run([str(options.exporter), profile], input_text=gpu) # Export the embedded source with this profile's runtime preamble.
320 if export_code: # Handle a nonzero compiler status as a failed check.
321 raise RuntimeError( # Classify exporter failures as infrastructure errors.
322 "embedded GLSL export failed: " + diagnostic(exported, substitutions) # Include the export failure with generic temporary paths.
323 ) # Report the exporter failure with portable diagnostic paths.
324 shader = generated / (profile + ".comp") # Locate the exported source for this GPU profile.
325 shader.write_text(exported) # Write the runtime-exported GLSL for this profile.
326 gpu_code, gpu_output = run([options.glslang, "-S", "comp", "-l", str(shader)]) # Compile and link the exported compute source for this profile.
327 result = { # Capture this command's status and output.
328 "status": "passed" if gpu_code == 0 else "rejected", # Record whether this compiler accepted the source.
329 "exit_code": gpu_code, # Record the compiler's numeric exit status.
330 } # Finish the shader-language result for this GPU profile.
331 if gpu_code: # Handle a nonzero compiler status as a failed check.
332 result["diagnostic"] = diagnostic(gpu_output, substitutions) # Store the profile-specific language error when compilation fails.
333 record["generated_gpu"][profile] = result # Attach this compiler result to its GPU profile name.
334 return record # Return the completed compiler record for this source.
335
336
337def summary(records): # Count accepted and rejected sources for the report.
338 counts = Counter() # Count outcomes across the complete collection.
339 repositories = {} # Keep a separate tally for each upstream project.
340 reasons = Counter() # Count frontend rejection categories.
341 for record in records: # Count each source's compiler outcomes.
342 count = repositories.setdefault(record["repository"], Counter()) # Select this repository's outcome counters.
343 for tally in (counts, count): # Update the collection total and this repository total.
344 tally["sources"] += 1 # Count this compiler outcome for the current total.
345 if record["upstream_reference_has_errors"]: # Count the upstream fixtures intentionally associated with errors.
346 tally["upstream_reference_has_errors"] += 1 # Count this compiler outcome for the current total.
347 if record["opengl"]["status"] == "passed": # Count original sources accepted in OpenGL mode.
348 tally["opengl_passed"] += 1 # Count this compiler outcome for the current total.
349 if not record["upstream_reference_has_errors"]: # Separate accepted fixtures from upstream negative-test cases.
350 tally["opengl_passed_without_reference_errors"] += 1 # Count this compiler outcome for the current total.
351 if record["cpu"]["status"] == "compiled": # Count generated adapters accepted by standard C++11.
352 tally["opengl_cpu_compiled"] += 1 # Count this compiler outcome for the current total.
353 elif record["vulkan"]["status"] == "passed": # Handle this source form.
354 tally["vulkan_only_passed"] += 1 # Count this compiler outcome for the current total.
355 else: # Count sources rejected by both reference compilation paths.
356 tally["reference_rejected"] += 1 # Count this compiler outcome for the current total.
357 tally["cpu_" + record["cpu"]["status"]] += 1 # Count this compiler outcome for the current total.
358 if record["cpu"]["status"] == "compiled": # Count generated adapters accepted by standard C++11.
359 for profile, result in record["generated_gpu"].items(): # Count successful GPU profile checks.
360 if result["status"] == "passed": # Count this generated GPU profile only when its compiler passed.
361 tally["cpu_and_" + profile + "_compiled"] += 1 # Count this compiler outcome for the current total.
362 if all(result["status"] == "passed" for result in record["generated_gpu"].values()): # Count universal language coverage only when all five profiles passed.
363 tally["cpu_and_all_profiles_compiled"] += 1 # Count this compiler outcome for the current total.
364 if record["cpu"]["status"] == "frontend_rejected": # Count the reason when the adapter frontend rejects a construct.
365 reasons[record["cpu"]["category"]] += 1 # Count this frontend rejection category.
366 return { # Return the collection, repository and rejection totals.
367 "counts": dict(counts), # Store totals across all selected sources.
368 "repositories": {name: dict(count) for name, count in repositories.items()}, # Record the pinned source collections.
369 "frontend_rejection_categories": dict(reasons), # Store the resource and language rejection categories.
370 } # Return all counters together with the original source records.
371
372
373def main(): # Read options and run this command-line tool.
374 parser = argparse.ArgumentParser(description=__doc__) # Describe the command and its supported options.
375 parser.add_argument("--cache", type=Path, default=Path("build/public-shaders")) # Select the cache containing the pinned upstream repositories.
376 parser.add_argument( # Offer an explicit option for downloading missing pinned snapshots.
377 "--fetch", action="store_true", help="download only the pinned public repository snapshots" # Describe the download option in the command help.
378 ) # Keep downloads disabled unless the fetch flag is supplied.
379 parser.add_argument("--glslang", default="glslangValidator") # Choose the reference GLSL compiler executable.
380 parser.add_argument("--cxx", default="c++") # Choose the compiler used for strict C++11 adapter checks.
381 parser.add_argument("--jobs", type=int, default=4) # Bound the number of concurrent source checks.
382 parser.add_argument("--output", type=Path, default=Path("build/public-shader-results.html")) # Choose where to write the complete browser-readable report.
383 options = parser.parse_args() # Read compiler, cache and output settings.
384 if options.jobs < 1: # Require a positive number of compiler workers.
385 parser.error("--jobs must be positive") # Stop with a helpful command-line usage error.
386 cache = options.cache.resolve() # Resolve the public source cache path.
387 try: # Capture failures from the following compiler or file operations.
388 prepare_cache(cache, options.fetch) # Verify cache revisions and download only when explicitly requested.
389 cases, excluded_hlsl = cases_from_cache(cache) # Collect the selected source records.
390 frontend_bytes = (ROOT / "scripts" / "compile_shader.py").read_bytes() # Capture the frontend snapshot before checks begin.
391 header_bytes = (ROOT / "shdr.h").read_bytes() # Capture the runtime-header snapshot before checks begin.
392 frontend = types.ModuleType("shdr_public_frontend") # Create an isolated namespace for the captured frontend.
393 exec(compile(frontend_bytes, "compile_shader.py", "exec"), frontend.__dict__) # Load only the captured local frontend code into this module.
394 with tempfile.TemporaryDirectory(prefix="shdr-public-export-") as export_directory: # Remove these intermediate compiler files when the check ends.
395 exporter_source = Path(export_directory) / "export.cpp" # Locate the temporary GLSL exporter source.
396 exporter_source.write_text( # Write the small helper that exports this runtime's GLSL preamble.
397 """#include "shdr.h" // Use the header's profile-specific GLSL exporter.
398#include <iostream> // Read shader text and print exported text or an error.
399#include <iterator> // Read standard input through character iterators.
400int main(int argc, char** argv) { // Export one profile named on the command line.
401 if (argc != 2) { // Require exactly one profile argument.
402 return 1; // Report incorrect command-line usage.
403 } // The remaining code has a valid argument count.
404 const std::string name(argv[1]); // Keep the requested profile name.
405 shdr::profile selected = shdr::profile::gl46; // Preserve the default for unrecognized names.
406 if (name == "es31") { // Select the minimum OpenGL ES compute profile.
407 selected = shdr::profile::es31; // Export GLSL ES 3.10.
408 } else if (name == "es32") { // Select the newer OpenGL ES profile.
409 selected = shdr::profile::es32; // Export GLSL ES 3.20.
410 } else if (name == "gl43") { // Select the minimum desktop compute profile.
411 selected = shdr::profile::gl43; // Export desktop GLSL 4.30.
412 } else if (name == "gl45") { // Select desktop OpenGL 4.5.
413 selected = shdr::profile::gl45; // Export desktop GLSL 4.50.
414 } // Other names keep the original OpenGL 4.6 fallback.
415 const std::string source( // Read the complete shader from standard input.
416 (std::istreambuf_iterator<char>(std::cin)), // Start at the first input character.
417 std::istreambuf_iterator<char>()); // Stop when standard input reaches its end.
418 try { // Convert export failures into a separate helper error status.
419 std::cout << shdr::glsl_source(source, selected); // Print standalone GLSL for this profile.
420 } catch (const std::exception& error) { // Capture a rejected source or export configuration.
421 std::cerr << error.what(); // Preserve the reason for the Python compiler report.
422 return 2; // Distinguish export errors from incorrect argument counts.
423 } // A successful export reaches the normal success return.
424} // Return zero implicitly after successful export.
425"""
426 ) # Write the complete temporary C++11 exporter source.
427 options.exporter = Path(export_directory) / "export" # Use the temporary exporter built for this header snapshot.
428 checked( # Build the temporary exporter before profile checks.
429 [ # List the strict compiler arguments for the exporter.
430 options.cxx, # Use the requested C++ compiler.
431 "-std=c++11", # Require the library's minimum C++ language standard.
432 "-pedantic-errors", # Treat nonstandard C++ syntax as an error.
433 "-pthread", # Link the runtime's thread support into the temporary exporter.
434 "-I", # Add the runtime header directory to the include search path.
435 str(ROOT), # Read the runtime header from this checkout.
436 str(exporter_source), # Compile the temporary runtime-export helper.
437 "-o", # Choose the temporary executable output path.
438 str(options.exporter), # Store the helper in this temporary directory.
439 ] # Complete the strict C++11 exporter build command.
440 ) # Require a working exporter before starting profile checks.
441 with ThreadPoolExecutor(max_workers=options.jobs) as pool: # Limit concurrent checks to the requested worker count.
442 records = list( # Collect one completed result per selected source.
443 pool.map(lambda case: verify_case(case, cache, options, frontend), cases) # Check each source independently while preserving selected-case order.
444 ) # Materialize ordered results before the worker pool closes.
445 if ( # Publish only results obtained from one unchanged frontend and header snapshot.
446 frontend_bytes != (ROOT / "scripts" / "compile_shader.py").read_bytes() # Reject results if the frontend changed while workers ran.
447 or header_bytes != (ROOT / "shdr.h").read_bytes() # Reject results if the runtime header changed during the checks.
448 ): # Detect any source snapshot change during the parallel checks.
449 raise RuntimeError( # Require every result to use the same source snapshot.
450 "frontend or header changed during verification. Rerun against a stable build" # Explain why a stable frontend and header are required.
451 ) # Stop before writing an inconsistent report.
452 report = { # Assemble provenance, counts and detailed compiler outcomes.
453 "schema_version": 1, # Identify the report layout version.
454 "repositories": REPOSITORIES, # Record the pinned source collections.
455 "tools": { # Record the compiler versions used for these results.
456 "glslang": checked([options.glslang, "--version"]).splitlines()[0], # Record the reference GLSL compiler version.
457 "cxx": checked([options.cxx, "--version"]).splitlines()[0], # Record the strict C++11 compiler version.
458 }, # Complete the compiler-version record.
459 "frontend_sha256": digest(frontend_bytes), # Identify the frontend used for CPU adaptation.
460 "header_sha256": digest(header_bytes), # Identify the runtime used for GPU source export.
461 "method": { # Describe how the compiler checks were performed.
462 "selection": "all tracked .comp files in the pinned paths, excluding filenames identifying HLSL", # Describe which tracked files were selected.
463 "reference": "original standalone GLSL files or explicitly configured unchanged NCNN bodies. OpenGL compile/link, then Vulkan SPIR-V fallback", # Describe the unmodified-source reference check.
464 "cpu": "current generated adapter compiled as standard C++11. No shader execution", # Describe the generated C++11 check.
465 "generated_gpu": "embedded source exported by the header and compiled/linked for each supported profile", # Describe the exported GLSL profile checks.
466 "excluded_hlsl": excluded_hlsl, # Count HLSL inputs separately from the GLSL collection.
467 }, # Complete the description of the reference and generated checks.
468 "summary": summary(records), # Store collection and repository totals.
469 "results": records, # Store all individual source records.
470 } # Complete the provenance and all individual source outcomes.
471 options.output.parent.mkdir(parents=True, exist_ok=True) # Create the report directory if needed.
472 options.output.write_text(render_public_report(report), encoding="utf-8") # Preserve every source record and fingerprint in a readable HTML report.
473 print(json.dumps(report["summary"], indent=2)) # Print the aggregate counters for a quick review.
474 print("Results: " + str(options.output)) # Print the path to the detailed machine-readable results.
475 except (OSError, RuntimeError) as error: # Report this failure through the command's normal error path.
476 parser.exit(1, str(error) + "\n") # Report the failure and exit with a nonzero status.
477
478
479if __name__ == "__main__": # Run only when this file is invoked as a command.
480 main() # Read options and run the requested checks.