Source browser
scripts/prepare_datasets.py
Read the complete file below or download its original bytes. Line numbers provide stable links to a declaration or processing step.
File fingerprint
1#!/usr/bin/env python3
2"""Prepare public-data fixtures from checked source bytes.
3
4Use --fetch to download missing recorded inputs. Use --verify to check local
5prepared files with the Python standard library. Use --full-mnist to unpack
6the complete official test split. JPEG preparation uses Pillow. Runtime
7examples read the prepared binary files.
8"""
9
10import argparse # Read preparation and verification options.
11import gzip # Unpack the original MNIST IDX files.
12import hashlib # Check source files and prepared fixture bytes.
13from pathlib import Path # Locate cache files and prepared outputs.
14import struct # Read IDX headers and write little-endian floating-point samples.
15import urllib.request # Download the recorded public input bytes.
16import data_sources # Share pinned downloads, prepared fingerprints and training provenance.
17
18ROOT = Path(__file__).resolve().parents[1] # Locate this source checkout.
19METADATA = Path(data_sources.__file__).resolve() # Keep maintained provenance in one importable Python module.
20
21
22def sha256(data): # Identify the exact bytes used by an example.
23 return hashlib.sha256(data).hexdigest() # Return the stable hexadecimal fingerprint.
24
25
26def require(condition, message): # Stop before accepting a malformed or changed fixture.
27 if not condition: # Check the format or checksum precondition.
28 raise RuntimeError(message) # Explain which source or transformation failed.
29
30
31def source_bytes(entry, cache, fetch): # Read one pinned input and download it only when requested.
32 path = cache / entry["name"] # Locate the cached source by its recorded filename.
33 if not path.exists(): # Keep ordinary preparation free of network requests.
34 require(fetch, "missing source " + entry["name"] + ". Run with --fetch") # Require explicit download mode.
35 headers = {"User-Agent": "shdr-public-data-preparation/1.0"} # Identify this small public-data client.
36 if "range" in entry: # Limit large ECG and event recordings to an exact prefix.
37 headers["Range"] = entry["range"] # Ask for only the recorded byte interval.
38 request = urllib.request.Request(entry["url"], headers=headers) # Use the public endpoint and bounded request.
39 with urllib.request.urlopen(request, timeout=60) as response: # Close the connection after reading this input.
40 if "range" in entry: # Reject servers that ignore the bounded prefix request.
41 require(response.status == 206, "server ignored byte range for " + entry["name"]) # Require a partial response.
42 data = response.read(entry["bytes"] + 1) # Read one extra byte to detect a changed source size.
43 require(len(data) == entry["bytes"], "source size changed: " + entry["name"]) # Check the recorded response length.
44 require(sha256(data) == entry["sha256"], "source checksum changed: " + entry["name"]) # Check bytes before caching them.
45 path.write_bytes(data) # Save only a complete verified source input.
46 data = path.read_bytes() # Read the source from the reusable cache.
47 require(len(data) == entry["bytes"], "cached source size differs: " + entry["name"]) # Reject incomplete cached files.
48 require(sha256(data) == entry["sha256"], "cached source checksum differs: " + entry["name"]) # Reject changed cache content.
49 return data # Return the verified source bytes to the conversion functions.
50
51
52def mnist_images(data, count): # Read the fixed MNIST image layout directly.
53 unpacked = gzip.decompress(data) # Expand the original compressed IDX file.
54 require(len(unpacked) >= 16, "MNIST image header is incomplete") # Require four big-endian header words.
55 magic, actual_count, rows, columns = struct.unpack(">4I", unpacked[:16]) # Read type, count and image dimensions.
56 require((magic, actual_count, rows, columns) == (2051, count, 28, 28), "unexpected MNIST image dimensions") # Check the official split.
57 require(len(unpacked) == 16 + count * 784, "MNIST image payload is incomplete") # Require one byte per image pixel.
58 return unpacked[16:] # Extract raw row-major image bytes after the IDX header.
59
60
61def mnist_labels(data, count): # Read target classes in the original image order.
62 unpacked = gzip.decompress(data) # Expand the original compressed label file.
63 require(len(unpacked) >= 8, "MNIST label header is incomplete") # Require both big-endian header words.
64 require(struct.unpack(">2I", unpacked[:8]) == (2049, count), "unexpected MNIST label count") # Check the official split.
65 labels = unpacked[8:] # Keep one target byte per image.
66 require(len(labels) == count and all(label < 10 for label in labels), "invalid MNIST label payload") # Require digit classes zero through nine.
67 return labels # Preserve the original target order.
68
69
70def prepare(output, full_output, inputs, full_mnist): # Convert recorded inputs into small runtime fixtures.
71 from PIL import Image, __version__ as pillow_version # Decode the one JPEG during preparation only.
72
73 artifacts = [] # Record every output file with its format and exact fingerprint.
74 output.mkdir(parents=True, exist_ok=True) # Create the destination for small committed fixtures.
75
76 def save(name, data, description): # Write one derived file and record how it was made.
77 (output / name).write_bytes(data) # Store the runtime-ready bytes.
78 artifacts.append({ # Retain size, checksum and transformation beside each output.
79 "name": name, # Identify the prepared fixture by its relative filename.
80 "bytes": len(data), # Record the exact byte count for runtime readers.
81 "sha256": sha256(data), # Identify the final input used for numerical checks.
82 **description, # Add the source, format and transformation description.
83 }) # Complete this fixture's provenance record.
84
85 with Image.open(inputs["nasa-blue-marble.jpg"]) as original: # Decode the checked NASA JPEG from its cache path.
86 image = original.convert("RGB") # Keep three unsigned color channels in RGB order.
87 width, height = image.size # Record the decoded source dimensions.
88 require((width, height) == (800, 800), "unexpected NASA image dimensions") # Check the pinned download's image size.
89 rgb = image.tobytes() # Read decoded pixels in row-major interleaved order.
90 reduced = bytearray() # Accumulate the sixty-four-by-sixty-four RGB fixture.
91 for y in range(64): # Visit every destination image row.
92 source_y = (2 * y + 1) * height // 128 # Sample the center of this destination pixel in source coordinates.
93 for x in range(64): # Visit every destination image column.
94 source_x = (2 * x + 1) * width // 128 # Use the same center-nearest rule horizontally.
95 offset = 3 * (source_y * width + source_x) # Locate the selected source RGB pixel.
96 reduced.extend(rgb[offset:offset + 3]) # Copy the three decoded RGB channel bytes.
97 save("earth_rgb.bin", bytes(reduced), { # Store the small satellite-image fixture.
98 "format": "RGB uint8, row-major, 64 by 64, three interleaved channels", # Describe the byte layout.
99 "source": "nasa-blue-marble.jpg", # Link this output to its checked source entry.
100 "transform": "JPEG decode to RGB, center-nearest downsample from 800 by 800 to 64 by 64", # State the exact conversion.
101 "pillow_version": pillow_version, # Record the image decoder used to create these bytes.
102 }) # Finish the image transformation record.
103
104 header = inputs["mitdb-100.hea"].read_text() # Read the ECG calibration accompanying the checked signal bytes.
105 require(header.splitlines()[0] == "100 2 360 650000", "unexpected ECG record metadata") # Check record, channels, rate and original sample count.
106 channel_lines = header.splitlines()[1:3] # Read the two channel-specific calibration lines.
107 require(all(line.split()[1:5] == ["212", "200", "11", "1024"] for line in channel_lines), "unexpected ECG calibration") # Check packed format, gain, resolution and zero level.
108 encoded = inputs["mitdb-100-prefix.dat"].read_bytes() # Read exactly 2048 pairs of packed samples.
109 channels = [[], []] # Collect both calibrated signals separately for channel-major storage.
110 for offset in range(0, len(encoded), 3): # Decode one pair of twelve-bit signed samples per three bytes.
111 first, middle, last = encoded[offset:offset + 3] # Read the shared-high-bits WFDB 212 encoding.
112 values = (first | ((middle & 15) << 8), last | ((middle >> 4) << 8)) # Restore both twelve-bit ADC values.
113 for channel, value in enumerate(values): # Calibrate each of the two lead samples independently.
114 if value >= 2048: # Sign-extend a twelve-bit negative sample.
115 value -= 4096 # Convert its unsigned bit pattern to the signed ADC value.
116 channels[channel].append((value - 1024) / 200.0) # Convert ADC counts into millivolts from the record header.
117 samples = channels[0] + channels[1] # Place all MLII samples before all V5 samples.
118 save("signal_ecg.bin", struct.pack("<4096f", *samples), { # Store 2048 samples per channel as little-endian floats.
119 "format": "float32 little-endian, two channel-major arrays of 2048 samples", # Describe the runtime signal layout.
120 "source": "mitdb-100-prefix.dat", # Link the waveform to its exact checked source interval.
121 "transform": "WFDB 212 decode, first 2048 sample pairs, (ADC - 1024) / 200 millivolts", # Record calibration and sample selection.
122 "sample_rate_hz": 360, # Preserve the official recording rate.
123 "channels": ["MLII", "V5"], # Preserve the lead names from the header.
124 }) # Finish the signal transformation record.
125
126 recording = inputs["laser-prefix.raw"].read_bytes() # Read the bounded official EVT3 recording prefix.
127 offset = 0 # Locate the first binary word after the ASCII metadata lines.
128 while recording[offset:offset + 1] == b"%": # Read the original RAW metadata lines before the event payload.
129 offset = recording.index(b"\n", offset) + 1 # Advance to the next header line or the binary payload.
130 require(b"% evt 3.0\n" in recording[:offset], "recording is not EVT3") # Check the format identified by the source header.
131 words = recording[offset:offset + 65536] # Keep the exact first 32768 sixteen-bit words.
132 require(len(words) == 65536, "EVT3 prefix does not contain enough complete words") # Avoid an incomplete runtime fixture.
133 save("evt3_words.bin", words, { # Store the original encoded words with their timestamps and coordinates.
134 "format": "uint16 little-endian, 32768 EVT3 words", # Describe the decoder input.
135 "source": "laser-prefix.raw", # Link the fixture to its official checked recording prefix.
136 "source_byte_offset": offset, # Record where its original binary payload begins.
137 "transform": "Remove ASCII RAW header, retain first 32768 complete encoded words", # State that event bytes remain unchanged.
138 }) # Finish the event transformation record.
139 save("evt3_header.txt", recording[:offset], { # Preserve the original recording metadata separately.
140 "format": "ASCII RAW header", # Describe the retained metadata encoding.
141 "source": "laser-prefix.raw", # Link this header to the same recording prefix.
142 "transform": "Copy the exact header preceding binary words", # Explain the separation of metadata and data.
143 }) # Finish the original-header record.
144
145 train_images = mnist_images(inputs["train-images-idx3-ubyte.gz"].read_bytes(), 60000) # Read the complete official training images.
146 train_labels = mnist_labels(inputs["train-labels-idx1-ubyte.gz"].read_bytes(), 60000) # Read their matching digit targets.
147 indices = [] # Keep the original indices selected for the small binary-classification fixture.
148 class_counts = [0, 0] # Select exactly sixty-four samples of each requested class.
149 for index, label in enumerate(train_labels): # Preserve original training order when selecting samples.
150 if label < 2 and class_counts[label] < 64: # Accept the first sixty-four instances of digit zero or one.
151 indices.append(index) # Record the source index before extracting its image.
152 class_counts[label] += 1 # Update the quota for this digit.
153 if class_counts == [64, 64]: # Stop once the balanced subset is complete.
154 break # Keep later training examples outside this small fixture.
155 save("mnist_train.bin", b"".join(train_images[index * 784:(index + 1) * 784] for index in indices), { # Extract only the selected raw images.
156 "format": "uint8, 128 row-major 28 by 28 images", # Describe the training fixture dimensions.
157 "source": "train-images-idx3-ubyte.gz", # Identify the official training split.
158 "transform": "First 64 instances each of digit 0 and 1 in original training order", # Explain the balanced selection.
159 "source_indices": indices, # Make every selected training sample traceable.
160 }) # Finish the binary training-image record.
161 save("mnist_train_labels.bin", bytes(train_labels[index] for index in indices), { # Extract labels in exactly the selected image order.
162 "format": "uint8, 128 digit labels", # Describe the corresponding target array.
163 "source": "train-labels-idx1-ubyte.gz", # Identify the original target split.
164 "transform": "Labels at the indices recorded in mnist_train.bin", # Preserve the selected image-to-target relationship.
165 }) # Finish the binary training-label record.
166
167 test_images = mnist_images(inputs["t10k-images-idx3-ubyte.gz"].read_bytes(), 10000) # Read the untouched official test split.
168 test_labels = mnist_labels(inputs["t10k-labels-idx1-ubyte.gz"].read_bytes(), 10000) # Read its matching targets.
169 save("mnist_test.bin", test_images[:512 * 784], { # Ship a small reproducible prefix for ordinary example checks.
170 "format": "uint8, 512 row-major 28 by 28 images", # Describe the ten-class test fixture.
171 "source": "t10k-images-idx3-ubyte.gz", # Keep training and testing sources distinct.
172 "transform": "Remove IDX header, retain first 512 official test images", # Record the fixed test selection.
173 }) # Finish the test-image record.
174 save("mnist_labels.bin", test_labels[:512], { # Retain the targets corresponding to the same test prefix.
175 "format": "uint8, 512 digit labels", # Describe the target array for the shipped test images.
176 "source": "t10k-labels-idx1-ubyte.gz", # Identify the official test-label file.
177 "transform": "Remove IDX header, retain first 512 official test labels", # Preserve original test ordering.
178 }) # Finish the test-label record.
179 if full_mnist: # Also prepare the larger official split when explicitly requested.
180 full_output.mkdir(parents=True, exist_ok=True) # Keep full test data outside the small committed fixtures.
181 (full_output / "mnist_full.bin").write_bytes(test_images) # Write all ten thousand original test images.
182 (full_output / "mnist_full_labels.bin").write_bytes(test_labels) # Write all ten thousand matching target bytes.
183 model_files = ("lenet_weights.bin", "lenet_reference.bin") # Identify the separately trained binary model bundle.
184 if any((output / name).exists() for name in model_files): # Include model integrity when the bundle accompanies these fixtures.
185 require(all((output / name).exists() for name in model_files), "LeNet model bundle is incomplete") # Require weights and reference logits together.
186 training = data_sources.TRAINING_PROVENANCE # Read the independent training export from the maintained metadata module.
187 weights = (output / "lenet_weights.bin").read_bytes() # Read the original trained flat parameter tensor.
188 logits = (output / "lenet_reference.bin").read_bytes() # Read the framework's held-out reference scores.
189 require(len(weights) == 61706 * 4 and sha256(weights) == training["weights_sha256"], "LeNet weights differ from training export") # Check the exact expected parameter bundle.
190 require(len(logits) == 10000 * 10 * 4 and sha256(logits) == training["reference_sha256"], "LeNet reference differs from training export") # Check every held-out reference logit.
191 formats = ( # Describe the two separately prepared binary model files.
192 "float32 little-endian, 61706 LeNet parameters", # Store convolution and dense tensors in documented flat order.
193 "float32 little-endian, 10000 by 10 official-test reference logits", # Preserve original test ordering for each class score.
194 ) # Keep formats in the same order as the model filenames.
195 for name, format_name in zip(model_files, formats): # Record the existing trained model outputs.
196 data = (output / name).read_bytes() # Fingerprint the existing exported file.
197 artifacts.append({ # Add this model artifact to the same offline integrity check.
198 "name": name, # Identify its relative filename in the fixture directory.
199 "bytes": len(data), # Record the exact exported file size.
200 "sha256": sha256(data), # Detect any change before native evaluation.
201 "format": format_name, # Describe how the C++ reader interprets its contents.
202 "source": "scripts/train_lenet.py", # Link to the independent reproducible training workflow.
203 "transform": "Offline full-MNIST training and reference export described in lenet_training.json", # Preserve the original record whose provenance now lives in data_sources.py.
204 }) # Complete this model's integrity and provenance entry.
205 return artifacts # Return the exact physical-file provenance for the maintained metadata module.
206
207
208def verify(output, artifacts=None): # Check local data files and archived provenance.
209 records = data_sources.PREPARED_ARTIFACTS if artifacts is None else artifacts # Use maintained pins or the files just prepared by this process.
210 for entry in records: # Check each small prepared input independently.
211 data = (output / entry["name"]).read_bytes() # Read the fixture expected by runtime examples.
212 require(len(data) == entry["bytes"], "fixture size differs: " + entry["name"]) # Detect truncation or replacement.
213 require(sha256(data) == entry["sha256"], "fixture checksum differs: " + entry["name"]) # Detect any byte change.
214 data_sources.verify_archived_training() # Retain the original eleventh fingerprint after migrating its record into Python.
215 print("Verified " + str(len(records)) + " data files + archived training-provenance fingerprint") # Distinguish physical runtime files from the retained historical record.
216
217
218def main(): # Select source preparation or offline fixture verification.
219 parser = argparse.ArgumentParser(description=__doc__) # Describe the optional dataset workflow.
220 parser.add_argument("--fetch", action="store_true") # Download missing sources only when this flag is supplied.
221 parser.add_argument("--verify", action="store_true") # Verify prepared inputs using the standard library.
222 parser.add_argument("--full-mnist", action="store_true") # Also unpack the complete ten-thousand-image test split.
223 parser.add_argument("--cache", type=Path, default=ROOT / "build" / "datasets" / "upstream") # Reuse downloaded source files.
224 parser.add_argument("--output", type=Path, default=ROOT / "examples" / "data") # Keep small fixtures with the example programs.
225 parser.add_argument("--full-output", type=Path, default=ROOT / "build" / "datasets") # Keep optional full test files in the build cache.
226 parser.add_argument("--metadata", type=Path, default=METADATA) # Update current fingerprints in this maintained Python metadata module.
227 options = parser.parse_args() # Read all paths and explicit preparation modes.
228 try: # Report preparation failures with a concise message.
229 if options.verify: # Use the lightweight offline integrity check when requested.
230 verify(options.output) # Compare every fixture to its recorded checksum.
231 return # Skip source preparation after successful verification.
232 records = data_sources.SOURCE_RECORDS # Read the pinned URLs, byte intervals and attribution from one maintained module.
233 options.cache.mkdir(parents=True, exist_ok=True) # Create the reusable source cache when needed.
234 inputs = {} # Collect paths only after source bytes have been verified.
235 for entry in records: # Check every input used by the deterministic preparation steps.
236 source_bytes(entry, options.cache, options.fetch) # Verify an existing source or download a missing one.
237 inputs[entry["name"]] = options.cache / entry["name"] # Retain its checked path for the conversion functions.
238 artifacts = prepare(options.output, options.full_output, inputs, options.full_mnist) # Produce small inputs and optional full test data.
239 data_sources.write_metadata(artifacts, path=options.metadata) # Refresh the current Python metadata records.
240 verify(options.output, artifacts) # Check the prepared files before reporting success.
241 except (OSError, RuntimeError, ValueError, ImportError) as error: # Handle missing inputs, formats and optional preparation dependencies.
242 parser.exit(1, str(error) + "\n") # Exit with a concise reason that identifies the failed preparation step.
243
244
245if __name__ == "__main__": # Run this workflow only when invoked as a command.
246 main() # Prepare or verify the requested public-data inputs.