Source browser
scripts/data_sources.py
Read the complete file below or download its original bytes. Line numbers provide stable links to a declaration or processing step.
File fingerprint
1"""Pinned public inputs and reproducible example-data metadata.
2
3SOURCE_RECORDS contains download URLs, source fingerprints and attribution.
4PREPARED_ARTIFACTS describes the ten data files used by the examples.
5TRAINING_PROVENANCE describes the current independently trained LeNet export.
6The archived training record retains the original metadata and fingerprint.
7Verification uses the Python standard library. Preparation and training update
8the current artifact and training assignments and retain the pinned sources.
9"""
10
11import ast # Locate only the two metadata assignments that preparation may update.
12import hashlib # Check the archived training record against its original fingerprint.
13import json # Reconstruct the archived serialization in memory.
14from pathlib import Path # Resolve the maintained metadata module for updates.
15import pprint # Write current Python data with readable, deterministic formatting.
16
17SCHEMA_VERSION = 1 # Version the record structure independently from prepared bytes.
18
19SOURCE_RECORDS = [{'name': 't10k-images-idx3-ubyte.gz', # Keep all eight pinned downloads and their original attribution.
20 'url': 'https://storage.googleapis.com/cvdf-datasets/mnist/t10k-images-idx3-ubyte.gz', # Download only this pinned public source.
21 'bytes': 1648877, # Require this exact file size before accepting its contents.
22 'sha256': '8d422c7b0a1c1c79245a5bcf07fe86e33eeafee792b84584aec276f5a2dbc4e6', # Identify every byte of this recorded artifact.
23 'dataset': 'MNIST', # Identify the public collection that supplied these values.
24 'source_page': 'https://github.com/cvdfoundation/mnist', # Retain the original publisher attribution and context.
25 'license': 'Not specified by the CVDF mirror', # Preserve the publisher's stated source notice.
26 'license_source': 'http://yann.lecun.com/exdb/mnist/', # Link to the recorded terms or original author page.
27 'credit': 'Yann LeCun, Corinna Cortes and Christopher J. C. Burges', # Retain attribution with redistributed sample data.
28 'additional_license_notice': {'url': 'https://huggingface.co/datasets/ylecun/mnist/raw/main/README.md', # Keep this separately hosted copy notice distinct from the download mirror.
29 'declared_license': 'MIT', # Record only the notice on that separately hosted copy.
30 'scope': 'Dataset-card metadata for that hosted copy. The CVDF ' # Explain which published copy carries this notice.
31 'mirror does not provide this notice.'}}, # Continue this recorded collection with its original values.
32 {'name': 't10k-labels-idx1-ubyte.gz', # Continue this recorded collection with its original values.
33 'url': 'https://storage.googleapis.com/cvdf-datasets/mnist/t10k-labels-idx1-ubyte.gz', # Download only this pinned public source.
34 'bytes': 4542, # Require this exact file size before accepting its contents.
35 'sha256': 'f7ae60f92e00ec6debd23a6088c31dbd2371eca3ffa0defaefb259924204aec6', # Identify every byte of this recorded artifact.
36 'dataset': 'MNIST', # Identify the public collection that supplied these values.
37 'source_page': 'https://github.com/cvdfoundation/mnist', # Retain the original publisher attribution and context.
38 'license': 'Not specified by the CVDF mirror', # Preserve the publisher's stated source notice.
39 'license_source': 'http://yann.lecun.com/exdb/mnist/', # Link to the recorded terms or original author page.
40 'credit': 'Yann LeCun, Corinna Cortes and Christopher J. C. Burges', # Retain attribution with redistributed sample data.
41 'additional_license_notice': {'url': 'https://huggingface.co/datasets/ylecun/mnist/raw/main/README.md', # Keep this separately hosted copy notice distinct from the download mirror.
42 'declared_license': 'MIT', # Record only the notice on that separately hosted copy.
43 'scope': 'Dataset-card metadata for that hosted copy. The CVDF ' # Explain which published copy carries this notice.
44 'mirror does not provide this notice.'}}, # Continue this recorded collection with its original values.
45 {'name': 'train-images-idx3-ubyte.gz', # Continue this recorded collection with its original values.
46 'url': 'https://storage.googleapis.com/cvdf-datasets/mnist/train-images-idx3-ubyte.gz', # Download only this pinned public source.
47 'bytes': 9912422, # Require this exact file size before accepting its contents.
48 'sha256': '440fcabf73cc546fa21475e81ea370265605f56be210a4024d2ca8f203523609', # Identify every byte of this recorded artifact.
49 'dataset': 'MNIST', # Identify the public collection that supplied these values.
50 'source_page': 'https://github.com/cvdfoundation/mnist', # Retain the original publisher attribution and context.
51 'license': 'Not specified by the CVDF mirror', # Preserve the publisher's stated source notice.
52 'license_source': 'http://yann.lecun.com/exdb/mnist/', # Link to the recorded terms or original author page.
53 'credit': 'Yann LeCun, Corinna Cortes and Christopher J. C. Burges', # Retain attribution with redistributed sample data.
54 'additional_license_notice': {'url': 'https://huggingface.co/datasets/ylecun/mnist/raw/main/README.md', # Keep this separately hosted copy notice distinct from the download mirror.
55 'declared_license': 'MIT', # Record only the notice on that separately hosted copy.
56 'scope': 'Dataset-card metadata for that hosted copy. The CVDF ' # Explain which published copy carries this notice.
57 'mirror does not provide this notice.'}}, # Continue this recorded collection with its original values.
58 {'name': 'train-labels-idx1-ubyte.gz', # Continue this recorded collection with its original values.
59 'url': 'https://storage.googleapis.com/cvdf-datasets/mnist/train-labels-idx1-ubyte.gz', # Download only this pinned public source.
60 'bytes': 28881, # Require this exact file size before accepting its contents.
61 'sha256': '3552534a0a558bbed6aed32b30c495cca23d567ec52cac8be1a0730e8010255c', # Identify every byte of this recorded artifact.
62 'dataset': 'MNIST', # Identify the public collection that supplied these values.
63 'source_page': 'https://github.com/cvdfoundation/mnist', # Retain the original publisher attribution and context.
64 'license': 'Not specified by the CVDF mirror', # Preserve the publisher's stated source notice.
65 'license_source': 'http://yann.lecun.com/exdb/mnist/', # Link to the recorded terms or original author page.
66 'credit': 'Yann LeCun, Corinna Cortes and Christopher J. C. Burges', # Retain attribution with redistributed sample data.
67 'additional_license_notice': {'url': 'https://huggingface.co/datasets/ylecun/mnist/raw/main/README.md', # Keep this separately hosted copy notice distinct from the download mirror.
68 'declared_license': 'MIT', # Record only the notice on that separately hosted copy.
69 'scope': 'Dataset-card metadata for that hosted copy. The CVDF ' # Explain which published copy carries this notice.
70 'mirror does not provide this notice.'}}, # Continue this recorded collection with its original values.
71 {'name': 'nasa-blue-marble.jpg', # Continue this recorded collection with its original values.
72 'url': 'https://assets.science.nasa.gov/dynamicimage/assets/science/psd/solar/2023/09/1/1-bluemarble_west-1.jpg?crop=faces%2Cfocalpoint&fit=clip&h=800&w=800', # Download only this pinned public source.
73 'bytes': 105654, # Require this exact file size before accepting its contents.
74 'sha256': '9824d30eca98594e73fd276388108ec1ee1ab494038e730fd378efc1aae4ffa6', # Identify every byte of this recorded artifact.
75 'dataset': 'Blue Marble 2002', # Identify the public collection that supplied these values.
76 'source_page': 'https://science.nasa.gov/resource/blue-marble-2002/', # Retain the original publisher attribution and context.
77 'license': 'NASA media usage guidelines', # Preserve the publisher's stated source notice.
78 'license_source': 'https://www.nasa.gov/nasa-brand-center/images-and-media/', # Link to the recorded terms or original author page.
79 'credit': "NASA's Earth Observatory"}, # Retain attribution with redistributed sample data.
80 {'name': 'mitdb-100.hea', # Continue this recorded collection with its original values.
81 'url': 'https://physionet.org/files/mitdb/1.0.0/100.hea', # Download only this pinned public source.
82 'bytes': 143, # Require this exact file size before accepting its contents.
83 'sha256': '60ebc904c7bf3e04d142638d3fd5c903e8dc9f10f1ea3264e07926aa089ee75e', # Identify every byte of this recorded artifact.
84 'dataset': 'MIT-BIH Arrhythmia Database record 100', # Identify the public collection that supplied these values.
85 'source_page': 'https://physionet.org/content/mitdb/1.0.0/', # Retain the original publisher attribution and context.
86 'license': 'ODC-By-1.0', # Preserve the publisher's stated source notice.
87 'license_source': 'https://opendatacommons.org/licenses/by/1-0/', # Link to the recorded terms or original author page.
88 'credit': 'George Moody and Roger Mark, MIT-BIH Arrhythmia Database version 1.0.0, PhysioNet, ' # Retain attribution with redistributed sample data.
89 'DOI 10.13026/C2F305'}, # Continue this recorded collection with its original values.
90 {'name': 'mitdb-100-prefix.dat', # Continue this recorded collection with its original values.
91 'url': 'https://physionet.org/files/mitdb/1.0.0/100.dat', # Download only this pinned public source.
92 'bytes': 6144, # Require this exact file size before accepting its contents.
93 'range': 'bytes=0-6143', # Request only this bounded original byte interval.
94 'source_bytes': 1950000, # Record the complete upstream file size alongside the selected byte range.
95 'sha256': 'bfd10bda039ed650bf1a29a57cbd08df3994736c45c9ba1bc16e62fe97ca271a', # Identify every byte of this recorded artifact.
96 'dataset': 'MIT-BIH Arrhythmia Database record 100', # Identify the public collection that supplied these values.
97 'source_page': 'https://physionet.org/content/mitdb/1.0.0/', # Retain the original publisher attribution and context.
98 'license': 'ODC-By-1.0', # Preserve the publisher's stated source notice.
99 'license_source': 'https://opendatacommons.org/licenses/by/1-0/', # Link to the recorded terms or original author page.
100 'credit': 'George Moody and Roger Mark, MIT-BIH Arrhythmia Database version 1.0.0, PhysioNet, ' # Retain attribution with redistributed sample data.
101 'DOI 10.13026/C2F305'}, # Continue this recorded collection with its original values.
102 {'name': 'laser-prefix.raw', # Continue this recorded collection with its original values.
103 'url': 'https://kdrive.infomaniak.com/2/app/975517/share/71d66a09-e3b6-480b-ba94-a1509e8ab2c8/files/12198/download', # Download only this pinned public source.
104 'bytes': 69632, # Require this exact file size before accepting its contents.
105 'range': 'bytes=0-69631', # Request only this bounded original byte interval.
106 'source_bytes': 341306299, # Record the complete upstream file size alongside the selected byte range.
107 'sha256': 'a398733173c1e684b68ea737a448aebb0f59dec6aa9a76ba587bfca824b4d6aa', # Identify every byte of this recorded artifact.
108 'dataset': 'Prophesee laser EVT3 sample', # Identify the public collection that supplied these values.
109 'source_page': 'https://docs.prophesee.ai/stable/datasets.html', # Retain the original publisher attribution and context.
110 'license': 'CC0-1.0', # Preserve the publisher's stated source notice.
111 'license_source': 'https://creativecommons.org/publicdomain/zero/1.0/', # Link to the recorded terms or original author page.
112 'credit': 'Prophesee sample recordings, laser, Gen4.1 sensor'}] # Retain attribution with redistributed sample data.
113
114PREPARED_ARTIFACTS = [{'name': 'earth_rgb.bin', # Keep the ten physical runtime input and model fingerprints.
115 'bytes': 12288, # Require this exact file size before accepting its contents.
116 'sha256': 'ed28281973e480d946477c1f9bd83b0749c81cdd3ec7a15c7c422250b3f4cf5d', # Identify every byte of this recorded artifact.
117 'format': 'RGB uint8, row-major, 64 by 64, three interleaved channels', # Describe the precise runtime byte layout.
118 'source': 'nasa-blue-marble.jpg', # Trace the prepared file to its checked input or preparation script.
119 'transform': 'JPEG decode to RGB, center-nearest downsample from 800 by 800 to 64 by 64', # Describe every conversion between the source and prepared values.
120 'pillow_version': '10.2.0'}, # Identify the JPEG decoder used for these exact RGB bytes.
121 {'name': 'signal_ecg.bin', # Continue this recorded collection with its original values.
122 'bytes': 16384, # Require this exact file size before accepting its contents.
123 'sha256': '9631961944e9e04e17722f44d522a216891b4c5763b348eb69d0160abfe3da44', # Identify every byte of this recorded artifact.
124 'format': 'float32 little-endian, two channel-major arrays of 2048 samples', # Describe the precise runtime byte layout.
125 'source': 'mitdb-100-prefix.dat', # Trace the prepared file to its checked input or preparation script.
126 'transform': 'WFDB 212 decode, first 2048 sample pairs, (ADC - 1024) / 200 millivolts', # Describe every conversion between the source and prepared values.
127 'sample_rate_hz': 360, # Interpret waveform positions at this original sampling rate.
128 'channels': ['MLII', 'V5']}, # Preserve the lead names in channel-major storage order.
129 {'name': 'evt3_words.bin', # Continue this recorded collection with its original values.
130 'bytes': 65536, # Require this exact file size before accepting its contents.
131 'sha256': 'da20cc642f8a3ba95bfcc3fe758cb7feed98334ffcfdf598c63fa2266775e014', # Identify every byte of this recorded artifact.
132 'format': 'uint16 little-endian, 32768 EVT3 words', # Describe the precise runtime byte layout.
133 'source': 'laser-prefix.raw', # Trace the prepared file to its checked input or preparation script.
134 'source_byte_offset': 166, # Locate the first encoded event word after the original RAW header.
135 'transform': 'Remove ASCII RAW header, retain first 32768 complete encoded words'}, # Describe every conversion between the source and prepared values.
136 {'name': 'evt3_header.txt', # Continue this recorded collection with its original values.
137 'bytes': 166, # Require this exact file size before accepting its contents.
138 'sha256': 'a767d8bb9bce0446440d582b635cf8ba126390edd043d9f56f59f78ff63d865f', # Identify every byte of this recorded artifact.
139 'format': 'ASCII RAW header', # Describe the precise runtime byte layout.
140 'source': 'laser-prefix.raw', # Trace the prepared file to its checked input or preparation script.
141 'transform': 'Copy the exact header preceding binary words'}, # Describe every conversion between the source and prepared values.
142 {'name': 'mnist_train.bin', # Continue this recorded collection with its original values.
143 'bytes': 100352, # Require this exact file size before accepting its contents.
144 'sha256': 'b4e671ad6e9b4d02b6ef48194346fc857be35d2d0b9711d45913464271c1522e', # Identify every byte of this recorded artifact.
145 'format': 'uint8, 128 row-major 28 by 28 images', # Describe the precise runtime byte layout.
146 'source': 'train-images-idx3-ubyte.gz', # Trace the prepared file to its checked input or preparation script.
147 'transform': 'First 64 instances each of digit 0 and 1 in original training order', # Describe every conversion between the source and prepared values.
148 'source_indices': [1, # Preserve the exact zero-based MNIST training selections.
149 3, # Retain this original zero-based training-image index.
150 6, # Retain this original zero-based training-image index.
151 8, # Retain this original zero-based training-image index.
152 14, # Retain this original zero-based training-image index.
153 21, # Retain this original zero-based training-image index.
154 23, # Retain this original zero-based training-image index.
155 24, # Retain this original zero-based training-image index.
156 34, # Retain this original zero-based training-image index.
157 37, # Retain this original zero-based training-image index.
158 40, # Retain this original zero-based training-image index.
159 51, # Retain this original zero-based training-image index.
160 56, # Retain this original zero-based training-image index.
161 59, # Retain this original zero-based training-image index.
162 63, # Retain this original zero-based training-image index.
163 67, # Retain this original zero-based training-image index.
164 68, # Retain this original zero-based training-image index.
165 69, # Retain this original zero-based training-image index.
166 70, # Retain this original zero-based training-image index.
167 72, # Retain this original zero-based training-image index.
168 75, # Retain this original zero-based training-image index.
169 77, # Retain this original zero-based training-image index.
170 78, # Retain this original zero-based training-image index.
171 81, # Retain this original zero-based training-image index.
172 88, # Retain this original zero-based training-image index.
173 95, # Retain this original zero-based training-image index.
174 99, # Retain this original zero-based training-image index.
175 102, # Retain this original zero-based training-image index.
176 104, # Retain this original zero-based training-image index.
177 105, # Retain this original zero-based training-image index.
178 108, # Retain this original zero-based training-image index.
179 112, # Retain this original zero-based training-image index.
180 113, # Retain this original zero-based training-image index.
181 114, # Retain this original zero-based training-image index.
182 118, # Retain this original zero-based training-image index.
183 119, # Retain this original zero-based training-image index.
184 121, # Retain this original zero-based training-image index.
185 124, # Retain this original zero-based training-image index.
186 128, # Retain this original zero-based training-image index.
187 134, # Retain this original zero-based training-image index.
188 152, # Retain this original zero-based training-image index.
189 156, # Retain this original zero-based training-image index.
190 169, # Retain this original zero-based training-image index.
191 174, # Retain this original zero-based training-image index.
192 177, # Retain this original zero-based training-image index.
193 184, # Retain this original zero-based training-image index.
194 192, # Retain this original zero-based training-image index.
195 200, # Retain this original zero-based training-image index.
196 201, # Retain this original zero-based training-image index.
197 205, # Retain this original zero-based training-image index.
198 206, # Retain this original zero-based training-image index.
199 208, # Retain this original zero-based training-image index.
200 209, # Retain this original zero-based training-image index.
201 210, # Retain this original zero-based training-image index.
202 211, # Retain this original zero-based training-image index.
203 216, # Retain this original zero-based training-image index.
204 224, # Retain this original zero-based training-image index.
205 229, # Retain this original zero-based training-image index.
206 231, # Retain this original zero-based training-image index.
207 232, # Retain this original zero-based training-image index.
208 234, # Retain this original zero-based training-image index.
209 246, # Retain this original zero-based training-image index.
210 248, # Retain this original zero-based training-image index.
211 249, # Retain this original zero-based training-image index.
212 251, # Retain this original zero-based training-image index.
213 260, # Retain this original zero-based training-image index.
214 269, # Retain this original zero-based training-image index.
215 270, # Retain this original zero-based training-image index.
216 276, # Retain this original zero-based training-image index.
217 283, # Retain this original zero-based training-image index.
218 290, # Retain this original zero-based training-image index.
219 293, # Retain this original zero-based training-image index.
220 296, # Retain this original zero-based training-image index.
221 303, # Retain this original zero-based training-image index.
222 309, # Retain this original zero-based training-image index.
223 310, # Retain this original zero-based training-image index.
224 315, # Retain this original zero-based training-image index.
225 320, # Retain this original zero-based training-image index.
226 326, # Retain this original zero-based training-image index.
227 345, # Retain this original zero-based training-image index.
228 351, # Retain this original zero-based training-image index.
229 355, # Retain this original zero-based training-image index.
230 357, # Retain this original zero-based training-image index.
231 358, # Retain this original zero-based training-image index.
232 359, # Retain this original zero-based training-image index.
233 366, # Retain this original zero-based training-image index.
234 382, # Retain this original zero-based training-image index.
235 394, # Retain this original zero-based training-image index.
236 397, # Retain this original zero-based training-image index.
237 398, # Retain this original zero-based training-image index.
238 399, # Retain this original zero-based training-image index.
239 406, # Retain this original zero-based training-image index.
240 408, # Retain this original zero-based training-image index.
241 416, # Retain this original zero-based training-image index.
242 427, # Retain this original zero-based training-image index.
243 429, # Retain this original zero-based training-image index.
244 435, # Retain this original zero-based training-image index.
245 440, # Retain this original zero-based training-image index.
246 443, # Retain this original zero-based training-image index.
247 447, # Retain this original zero-based training-image index.
248 450, # Retain this original zero-based training-image index.
249 451, # Retain this original zero-based training-image index.
250 453, # Retain this original zero-based training-image index.
251 454, # Retain this original zero-based training-image index.
252 455, # Retain this original zero-based training-image index.
253 458, # Retain this original zero-based training-image index.
254 462, # Retain this original zero-based training-image index.
255 464, # Retain this original zero-based training-image index.
256 466, # Retain this original zero-based training-image index.
257 470, # Retain this original zero-based training-image index.
258 473, # Retain this original zero-based training-image index.
259 475, # Retain this original zero-based training-image index.
260 484, # Retain this original zero-based training-image index.
261 489, # Retain this original zero-based training-image index.
262 519, # Retain this original zero-based training-image index.
263 524, # Retain this original zero-based training-image index.
264 526, # Retain this original zero-based training-image index.
265 527, # Retain this original zero-based training-image index.
266 542, # Retain this original zero-based training-image index.
267 577, # Retain this original zero-based training-image index.
268 582, # Retain this original zero-based training-image index.
269 596, # Retain this original zero-based training-image index.
270 603, # Retain this original zero-based training-image index.
271 612, # Retain this original zero-based training-image index.
272 633, # Retain this original zero-based training-image index.
273 639, # Retain this original zero-based training-image index.
274 656, # Retain this original zero-based training-image index.
275 662]}, # Continue this recorded collection with its original values.
276 {'name': 'mnist_train_labels.bin', # Continue this recorded collection with its original values.
277 'bytes': 128, # Require this exact file size before accepting its contents.
278 'sha256': '49994fd8dc3bf88c7dff6d9863bde8bbf0d96ff09b8e99b3aaa4fbfceab5d3bf', # Identify every byte of this recorded artifact.
279 'format': 'uint8, 128 digit labels', # Describe the precise runtime byte layout.
280 'source': 'train-labels-idx1-ubyte.gz', # Trace the prepared file to its checked input or preparation script.
281 'transform': 'Labels at the indices recorded in mnist_train.bin'}, # Describe every conversion between the source and prepared values.
282 {'name': 'mnist_test.bin', # Continue this recorded collection with its original values.
283 'bytes': 401408, # Require this exact file size before accepting its contents.
284 'sha256': '80d1589bb8ec86cbe5b156dc437990d35e226cf0a5c7dc8202fc5e58124eff1d', # Identify every byte of this recorded artifact.
285 'format': 'uint8, 512 row-major 28 by 28 images', # Describe the precise runtime byte layout.
286 'source': 't10k-images-idx3-ubyte.gz', # Trace the prepared file to its checked input or preparation script.
287 'transform': 'Remove IDX header, retain first 512 official test images'}, # Describe every conversion between the source and prepared values.
288 {'name': 'mnist_labels.bin', # Continue this recorded collection with its original values.
289 'bytes': 512, # Require this exact file size before accepting its contents.
290 'sha256': '828c90776b296fc6bbadc357389e4d66dd7819fbb1979982a700d983168da942', # Identify every byte of this recorded artifact.
291 'format': 'uint8, 512 digit labels', # Describe the precise runtime byte layout.
292 'source': 't10k-labels-idx1-ubyte.gz', # Trace the prepared file to its checked input or preparation script.
293 'transform': 'Remove IDX header, retain first 512 official test labels'}, # Describe every conversion between the source and prepared values.
294 {'name': 'lenet_weights.bin', # Continue this recorded collection with its original values.
295 'bytes': 246824, # Require this exact file size before accepting its contents.
296 'sha256': '327dd259df961e6ed9b8f9fc82d2cd56b1334ca7525f92403de9c6495d1fe8de', # Identify every byte of this recorded artifact.
297 'format': 'float32 little-endian, 61706 LeNet parameters', # Describe the precise runtime byte layout.
298 'source': 'scripts/train_lenet.py', # Trace the prepared file to its checked input or preparation script.
299 'transform': 'Offline full-MNIST training and reference export described in lenet_training.json'}, # Describe every conversion between the source and prepared values.
300 {'name': 'lenet_reference.bin', # Continue this recorded collection with its original values.
301 'bytes': 400000, # Require this exact file size before accepting its contents.
302 'sha256': 'e3623237df15d5671dbb5cfd04c072d58f3d14a31bc04fd6b80c1d29026195e5', # Identify every byte of this recorded artifact.
303 'format': 'float32 little-endian, 10000 by 10 official-test reference logits', # Describe the precise runtime byte layout.
304 'source': 'scripts/train_lenet.py', # Trace the prepared file to its checked input or preparation script.
305 'transform': 'Offline full-MNIST training and reference export described in lenet_training.json'}] # Describe every conversion between the source and prepared values.
306
307ARCHIVED_TRAINING_ARTIFACT = {'name': 'lenet_training.json', # Retain the original provenance-file record after moving its contents here.
308 'bytes': 1252, # Require this exact file size before accepting its contents.
309 'sha256': '02798d534978d33d7805ba04e6143eab77849853c1bc30d602e7de9726561dda', # Identify every byte of this recorded artifact.
310 'format': 'JSON training settings, dataset checksums and independent metrics', # Describe the precise runtime byte layout.
311 'source': 'scripts/train_lenet.py', # Trace the prepared file to its checked input or preparation script.
312 'transform': 'Offline full-MNIST training and reference export described in lenet_training.json'} # Describe every conversion between the source and prepared values.
313
314ARCHIVED_TRAINING_PROVENANCE = {'architecture': 'modern LeNet with full C3, ReLU, fixed average pooling and linear logits', # Preserve every value from the original independent training export.
315 'parameters': 61706, # Match the flat native parameter count.
316 'training_split': 60000, # Record the number of original training examples.
317 'test_split': 10000, # Keep the independent held-out evaluation population explicit.
318 'seed': 12345, # Reproduce parameter initialization and training permutations.
319 'epochs': 3, # Visit the complete training split this many times.
320 'batch_size': 128, # Bound each Adam update to this many training images.
321 'threads': 8, # Record the model-preparation thread count.
322 'optimizer': 'Adam', # Identify the independent reference training update rule.
323 'learning_rate': 0.001, # Record the Adam learning rate.
324 'input_scale': 'byte / 255 with a two-pixel zero border', # Match the native image normalization and convolution border.
325 'training_cross_entropy': [0.4789702771107356, 0.12624290428757667, 0.08441878777146339], # Preserve the measured mean loss after each complete training epoch.
326 'test_accuracy': 0.9778000116348267, # Keep the exact measured held-out accuracy.
327 'test_cross_entropy': 0.06259464472532272, # Preserve the independently measured held-out loss.
328 'training_seconds': 18.704168565000145, # Record the completed original preparation time.
329 'torch_version': '2.6.0+cpu', # Identify the optional reference training framework.
330 'numpy_version': '1.26.4', # Identify the optional float-array exporter.
331 'source_sha256': {'train-images-idx3-ubyte.gz': '440fcabf73cc546fa21475e81ea370265605f56be210a4024d2ca8f203523609', # Identify all four original archives used during training and evaluation.
332 'train-labels-idx1-ubyte.gz': '3552534a0a558bbed6aed32b30c495cca23d567ec52cac8be1a0730e8010255c', # Preserve this recorded source fingerprint.
333 't10k-images-idx3-ubyte.gz': '8d422c7b0a1c1c79245a5bcf07fe86e33eeafee792b84584aec276f5a2dbc4e6', # Preserve this recorded source fingerprint.
334 't10k-labels-idx1-ubyte.gz': 'f7ae60f92e00ec6debd23a6088c31dbd2371eca3ffa0defaefb259924204aec6'}, # Preserve this recorded source fingerprint.
335 'weights_sha256': '327dd259df961e6ed9b8f9fc82d2cd56b1334ca7525f92403de9c6495d1fe8de', # Identify the deployed flat model parameters.
336 'reference_sha256': 'e3623237df15d5671dbb5cfd04c072d58f3d14a31bc04fd6b80c1d29026195e5'} # Identify all independent held-out framework logits.
337
338TRAINING_PROVENANCE = ARCHIVED_TRAINING_PROVENANCE # Start with the archived model and replace this assignment when retraining.
339
340
341def archived_training_bytes(): # Reconstruct the former standalone provenance file for its historical integrity check.
342 """Return the original serialized provenance bytes."""
343 return (json.dumps(ARCHIVED_TRAINING_PROVENANCE, indent=2) + "\n").encode() # Preserve the original UTF-8 bytes and final newline.
344
345
346def verify_archived_training(): # Keep the migrated record as verifiable as the former standalone file.
347 """Check the original training metadata against its retained size and SHA-256."""
348 payload = archived_training_bytes() # Serialize the archived original training record.
349 record = ARCHIVED_TRAINING_ARTIFACT # Read the exact historical fingerprint copied during migration.
350 if len(payload) != record["bytes"] or hashlib.sha256(payload).hexdigest() != record["sha256"]: # Detect changed archived values or serialization.
351 raise ValueError("archived LeNet training provenance fingerprint differs") # Keep accidental provenance edits visible during offline verification.
352
353
354def _assignment(name, value): # Format the Python metadata assignment.
355 """Render ordinary Python values with brief field comments on each code line."""
356 lines = pprint.pformat(value, width=100, sort_dicts=False).splitlines() # Keep source and artifact ordering stable between updates.
357 result = [] # Collect one commented source line at a time.
358 for index, line in enumerate(lines): # Preserve the formatter's readable nested record layout.
359 stripped = line.lstrip() # Inspect a key and preserve its emitted indentation.
360 comment = "Continue this recorded collection." # Describe wrapped record values and closing delimiters.
361 if stripped.startswith("'") and "':" in stripped: # A dictionary key names the value on this line.
362 key = stripped.split("':", 1)[0][1:] # Read the Python string key through literal parsing.
363 comment = "Record " + key.replace("_", " ") + "." # Make each emitted metadata field easy to identify.
364 elif stripped.rstrip(",").isdigit(): # The selected training rows are stored as integer list entries.
365 comment = "Retain this original zero-based training-image index." # Explain why the individual numbers are preserved.
366 result.append((name + " = " if index == 0 else "") + line + " # " + comment) # Write a valid assignment with line comments.
367 return "\n".join(result) + "\n" # End the assignment with one normal source newline.
368
369
370def write_metadata(artifacts, training=None, path=None): # Update current outputs while retaining pinned sources and archived provenance.
371 """Write current prepared-file records and optional training results.
372
373 An alternate target starts from this module's source, allowing preparation
374 checks in a separate metadata module.
375 """
376 target = Path(path) if path is not None else Path(__file__).resolve() # Default to the single maintained metadata module.
377 source = target.read_text() if target.exists() else Path(__file__).resolve().read_text() # Copy the maintained module when checking another output directory.
378 tree = ast.parse(source) # Find assignment boundaries through the Python syntax tree.
379 replacements = {"PREPARED_ARTIFACTS": artifacts} # Always record the current ten prepared physical outputs.
380 if training is not None: # A training run also supplies its new measured model provenance.
381 replacements["TRAINING_PROVENANCE"] = training # Update current results and retain the archived original run.
382 spans = [] # Apply edits from the end so earlier line positions remain valid.
383 for statement in tree.body: # Examine only the module's top-level assignments.
384 if isinstance(statement, ast.Assign) and len(statement.targets) == 1 and isinstance(statement.targets[0], ast.Name): # Ignore helpers and complex Python statements.
385 name = statement.targets[0].id # Read the constant being assigned on this source span.
386 if name in replacements: # Restrict writes to the explicit current metadata fields.
387 spans.append((statement.lineno - 1, statement.end_lineno, _assignment(name, replacements.pop(name)))) # Retain complete assignment lines including comments.
388 if replacements: # Require every requested assignment in the metadata module.
389 raise ValueError("metadata assignments are missing: " + ", ".join(replacements)) # Identify the required metadata assignment.
390 lines = source.splitlines(keepends=True) # Preserve all source text outside the two selected assignments.
391 for start, end, replacement in reversed(spans): # Replace the later assignment before the earlier one.
392 lines[start:end] = [replacement] # Substitute only the current model or prepared-file record.
393 updated = "".join(lines) # Reassemble the metadata with its original surrounding documentation.
394 if updated != source or not target.exists(): # Preserve timestamps while still creating an explicitly selected new destination.
395 target.parent.mkdir(parents=True, exist_ok=True) # Permit an explicitly selected new metadata destination.
396 target.write_text(updated) # Store the updated records as readable importable Python.