Source browser
examples/neural_support.hpp
Read the complete file below or download its original bytes. Line numbers provide stable links to a declaration or processing step.
File fingerprint
1/** @file neural_support.hpp
2 * @brief Small fully trainable ANN/CNN pipelines built from ordinary shdr resources.
3 * Model parameters, batches, gradients and predictions are ordinary shdr buffers.
4 * Training computes independent sample gradients and reduces them in a separate SGD dispatch.
5 */
6#ifndef SHDR_EXAMPLES_NEURAL_SUPPORT_HPP // Open this include guard or optional feature branch.
7#define SHDR_EXAMPLES_NEURAL_SUPPORT_HPP // Define SHDR_EXAMPLES_NEURAL_SUPPORT_HPP for the declarations that follow.
8#include "shdr.h" // Include shdr.h declarations.
9#include <algorithm> // Include algorithm declarations.
10#include <cmath> // Include cmath declarations.
11#include <random> // Include random declarations.
12#include <vector> // Include vector declarations.
13
14namespace shdr_examples { // Group these helpers within their namespace.
15
16/** @brief Binary ANN or CNN with separate training, inference, and evaluation pipelines.
17 * ANN: two input features, eight ReLU neurons, sigmoid output, 33 parameters.
18 * CNN: 8x8 grayscale, two valid 3x3 ReLU filters, global pooling, sigmoid, 23 parameters.
19 * All parameters train. A new instance reproduces the same seeded data and initial weights.
20 */
21class binary_network { // Declare binary network.
22 bool cnn_; // Declare cnn for the following calculation.
23 unsigned batch_, parameter_count_; // Declare batch for the following calculation.
24 std::vector<float> initial_; // Declare initial for the following calculation.
25 std::unique_ptr<shdr::kernel> infer_, gradient_, sgd_, loss_shader_; // Declare infer for the following calculation.
26 shdr::pipeline train_, predict_, evaluate_, differentiate_; // Declare train for the following calculation.
27 static unsigned batch(unsigned n) { // Implement batch for the operations below.
28 if (!n || n > 4096u) // Check this condition before entering the following branch.
29 throw shdr::error("neural example batch size must be between 1 and 4096"); // Report neural example batch size must be between 1 and 4096.
30 return n; // Return n.
31 } // End this block.
32
33 public: // Set public access for the following class members.
34 shdr::buffer<float> inputs, targets, parameters, gradients, predictions, loss; // Declare inputs for the following calculation.
35
36 /** @brief Build a deterministic XOR-feature or image-orientation training problem. */
37 explicit binary_network(bool cnn = false, unsigned batch_count = 32) // Set cnn from this expression.
38 : cnn_(cnn), batch_(batch(batch_count)), parameter_count_(cnn ? 23u : 33u), // Declare cnn with the parameters continued below.
39 initial_(parameter_count_), // Apply initial to these arguments.
40 infer_(shdr::make_kernel(cnn ? "cnn_infer_kernel" : "ann_infer_kernel")), // Create the linked shader instance.
41 gradient_(shdr::make_kernel(cnn ? "cnn_gradient_kernel" : "ann_gradient_kernel")), // Create the linked shader instance.
42 sgd_(shdr::make_kernel("nn_sgd_kernel")), // Create the linked nn sgd kernel instance.
43 loss_shader_(shdr::make_kernel("nn_binary_loss_kernel")), // Create the linked nn binary loss kernel instance.
44 inputs(batch_ * (cnn ? 64u : 2u)), targets(batch_), parameters(parameter_count_), // Apply inputs to these arguments.
45 gradients(batch_ * parameter_count_), predictions(batch_), loss(1) { // Apply gradients to these arguments.
46 std::mt19937 random(12345u); // Construct random with these initial values.
47 std::uniform_real_distribution<float> noise(-0.12f, 0.12f), weights(-0.15f, 0.15f); // Construct noise with these initial values.
48 for (float& value : initial_) // Visit each value required by this loop.
49 value = weights(random); // Set value from this expression.
50 if (cnn_) { // Check this condition before entering the following branch.
51 for (unsigned filter = 0; filter < 2; ++filter) // Iterate over filter for this calculation.
52 for (unsigned y = 0; y < 3; ++y) // Iterate over y for this calculation.
53 for (unsigned x = 0; x < 3; ++x) // Iterate over x for this calculation.
54 initial_[9 * filter + 3 * y + x] = // Set initial [9 * filter + 3 * y + x] from this expression.
55 ((filter == 0 ? y : x) == 1 ? 0.4f : -0.2f) + weights(random) * 0.1f; // Apply weights to these arguments.
56 initial_[18] = initial_[19] = 0.04f; // Set initial [18] from this expression.
57 for (unsigned sample = 0; sample < batch_; ++sample) { // Iterate over sample for this calculation.
58 const bool horizontal = (sample & 1u) != 0; // Set horizontal from this expression.
59 const unsigned position = 2u + unsigned(random()) % 3u; // Set position from this expression.
60 targets.data()[sample] = horizontal ? 1.0f : 0.0f; // Apply data to these arguments.
61 for (unsigned y = 0; y < 8; ++y) // Iterate over y for this calculation.
62 for (unsigned x = 0; x < 8; ++x) { // Iterate over x for this calculation.
63 const unsigned coordinate = horizontal ? y : x; // Set coordinate from this expression.
64 const float signal = // Set signal from this expression.
65 (coordinate == position || coordinate == position + 1) ? 0.9f : 0.0f; // Continue the expression using coordinate, position, coordinate.
66 inputs.data()[64 * sample + 8 * y + x] = signal + noise(random) * 0.4f; // Apply data to these arguments.
67 } // End this block.
68 } // End this block.
69 } else { // Initialize the eight-neuron ANN and its two-feature training inputs.
70 const float directions[8] = {1, 1, 1, -1, -1, 1, -1, -1}; // Set directions[8] from this expression.
71 std::copy(directions, directions + 8, initial_.begin()); // Apply copy to these arguments.
72 for (unsigned neuron = 0; neuron < 8; ++neuron) // Iterate over neuron for this calculation.
73 initial_[16 + neuron] = -0.15f; // Set initial [16 + neuron] from this expression.
74 for (unsigned sample = 0; sample < batch_; ++sample) { // Iterate over sample for this calculation.
75 const bool positive_x = (sample & 1u) != 0, positive_y = (sample & 2u) != 0; // Set positive x from this expression.
76 inputs.data()[2 * sample] = (positive_x ? 1.0f : -1.0f) + noise(random); // Apply data to these arguments.
77 inputs.data()[2 * sample + 1] = (positive_y ? 1.0f : -1.0f) + noise(random); // Apply data to these arguments.
78 targets.data()[sample] = positive_x != positive_y ? 1.0f : 0.0f; // Apply data to these arguments.
79 } // End this block.
80 } // End this block.
81 reset_parameters(); // Apply reset parameters to these arguments.
82 infer_->bind("inputs", inputs) // Connect this shader port to its application resource.
83 .bind("parameters", parameters) // Connect this shader port to its application resource.
84 .bind("predictions", predictions) // Connect this shader port to its application resource.
85 .set("batch_count", batch_); // Apply set to these arguments.
86 gradient_->bind("inputs", inputs) // Connect this shader port to its application resource.
87 .bind("targets", targets) // Connect this shader port to its application resource.
88 .bind("parameters", parameters) // Connect this shader port to its application resource.
89 .bind("gradients", gradients) // Connect this shader port to its application resource.
90 .set("batch_count", batch_); // Apply set to these arguments.
91 sgd_->bind("parameters", parameters) // Connect this shader port to its application resource.
92 .bind("gradients", gradients) // Connect this shader port to its application resource.
93 .set("batch_count", batch_) // Apply set to these arguments.
94 .set("parameter_count", parameter_count_) // Apply set to these arguments.
95 .set("learning_rate", cnn ? 1.5f : 0.4f) // Apply set to these arguments.
96 .set("weight_decay", 0.0f); // Apply set to these arguments.
97 loss_shader_->bind("predictions", predictions) // Connect this shader port to its application resource.
98 .bind("targets", targets) // Connect this shader port to its application resource.
99 .bind("loss", loss) // Connect this shader port to its application resource.
100 .set("batch_count", batch_); // Apply set to these arguments.
101 differentiate_ = // Set differentiate from this expression.
102 shdr::dispatch(*gradient_, shdr::extent((batch_ * parameter_count_ + 63) / 64)); // Add a shader dispatch with the required workgroup counts.
103 train_ = differentiate_; // Set train from this expression.
104 train_ += shdr::dispatch(*sgd_, shdr::extent((parameter_count_ + 63) / 64)); // Add a shader dispatch with the required workgroup counts.
105 predict_ = shdr::dispatch(*infer_, shdr::extent((batch_ + 63) / 64)); // Add a shader dispatch with the required workgroup counts.
106 evaluate_ = predict_; // Set evaluate from this expression.
107 evaluate_ += shdr::dispatch(*loss_shader_, shdr::extent(1)); // Add a shader dispatch with the required workgroup counts.
108 } // End this block.
109 /** @brief Restore deterministic initial weights, outside a timed or queued processing loop. */
110 void reset_parameters() { // Implement reset parameters for the operations below.
111 std::copy(initial_.begin(), initial_.end(), parameters.data()); // Apply copy to these arguments.
112 } // End this block.
113 /** @brief Compute all independent per-sample analytic parameter gradients. */
114 const shdr::pipeline& gradient_pipeline() const { // Implement gradient pipeline for the operations below.
115 return differentiate_; // Return differentiate.
116 } // End this block.
117 /** @brief One complete mini-batch update: gradients followed by mean-gradient SGD. */
118 const shdr::pipeline& training_pipeline() const { // Implement training pipeline for the operations below.
119 return train_; // Return train.
120 } // End this block.
121 /** @brief Predict probabilities for the currently bound batch. */
122 const shdr::pipeline& prediction_pipeline() const { // Implement prediction pipeline for the operations below.
123 return predict_; // Return predict.
124 } // End this block.
125 /** @brief Predict, then compute a mean binary cross-entropy diagnostic. */
126 const shdr::pipeline& evaluation_pipeline() const { // Implement evaluation pipeline for the operations below.
127 return evaluate_; // Return evaluate.
128 } // End this block.
129 /** @brief Number of independent samples in the batch. */
130 unsigned batch_count() const { // Implement batch count for the operations below.
131 return batch_; // Return batch.
132 } // End this block.
133 /** @brief Number of trainable weights and biases. */
134 unsigned parameter_count() const { // Implement parameter count for the operations below.
135 return parameter_count_; // Return parameter count.
136 } // End this block.
137 /** @brief Whether this instance uses the convolutional image classifier. */
138 bool convolutional() const { // Implement convolutional for the operations below.
139 return cnn_; // Return cnn.
140 } // End this block.
141}; // Finish this declaration.
142
143/** @brief Independent double-precision prediction oracle for ANN/CNN fixtures. */
144inline double binary_reference_probability(const binary_network& network, // Declare binary reference probability with the parameters continued below.
145 const std::vector<double>& weights, // Declare weights for the following calculation.
146 unsigned sample) { // Declare sample for the following calculation.
147 const float* input = network.inputs.data(); // Set input from this expression.
148 double logit = weights.back(); // Use this floating-point value or resource in the following calculation.
149 if (network.convolutional()) { // Check this condition before entering the following branch.
150 for (unsigned filter = 0; filter < 2; ++filter) { // Iterate over filter for this calculation.
151 double pooled = 0; // Use this floating-point value or resource in the following calculation.
152 for (unsigned y = 0; y < 6; ++y) // Iterate over y for this calculation.
153 for (unsigned x = 0; x < 6; ++x) { // Iterate over x for this calculation.
154 double value = weights[18 + filter]; // Use this floating-point value or resource in the following calculation.
155 for (unsigned ky = 0; ky < 3; ++ky) // Iterate over ky for this calculation.
156 for (unsigned kx = 0; kx < 3; ++kx) // Iterate over kx for this calculation.
157 value += weights[9 * filter + 3 * ky + kx] * // Update value with this contribution.
158 input[64 * sample + 8 * (y + ky) + x + kx]; // Continue the expression using input, sample, y.
159 pooled += std::max(0.0, value) / 36.0; // Update pooled with this contribution.
160 } // End this block.
161 logit += weights[20 + filter] * pooled; // Update logit with this contribution.
162 } // End this block.
163 } else // Evaluate all eight ANN hidden neurons for the independent prediction.
164 for (unsigned neuron = 0; neuron < 8; ++neuron) { // Iterate over neuron for this calculation.
165 const double activation = std::max(0.0, // Set activation from this expression.
166 weights[2 * neuron] * input[2 * sample] + // Continue the expression using weights, neuron, input.
167 weights[2 * neuron + 1] * input[2 * sample + 1] + // Continue the expression using weights, neuron, input.
168 weights[16 + neuron]); // Continue the expression using weights, neuron.
169 logit += weights[24 + neuron] * activation; // Update logit with this contribution.
170 } // End this block.
171 if (logit >= 0) // Check this condition before entering the following branch.
172 return 1.0 / (1.0 + std::exp(-logit)); // Return the unit-valued numeric result.
173 const double e = std::exp(logit); // Set e from this expression.
174 return e / (1.0 + e); // Return e / (1.0 + e).
175} // End this block.
176
177/** @brief Check mean analytic gradients against central differences of a double oracle. */
178inline void check_binary_gradients(binary_network& network) { // Implement check binary gradients for the operations below.
179 network.gradient_pipeline().run(); // Execute the configured pipeline on the requested backend.
180 const shdr::buffer<float>& parameters = network.parameters; // Use a constant buffer view so GPU writes synchronize before host reads.
181 std::vector<double> weights(parameters.data(), parameters.data() + parameters.size()); // Construct weights with these initial values.
182 const float* gradient = static_cast<const shdr::buffer<float>&>(network.gradients).data(); // Use a constant buffer view so GPU writes synchronize before host reads.
183 const float* target = static_cast<const shdr::buffer<float>&>(network.targets).data(); // Use a constant buffer view so GPU writes synchronize before host reads.
184 const auto mean_loss = [&]() { // Set mean loss from this expression.
185 double sum = 0; // Use this floating-point value or resource in the following calculation.
186 for (unsigned sample = 0; sample < network.batch_count(); ++sample) { // Iterate over sample for this calculation.
187 const double p = binary_reference_probability(network, weights, sample); // Set p from this expression.
188 sum -= target[sample] * std::log(p) + (1.0 - target[sample]) * std::log(1.0 - p); // Update sum with this contribution.
189 } // End this block.
190 return sum / network.batch_count(); // Return sum / network.batch count().
191 }; // Finish this declaration.
192 for (unsigned parameter = 0; parameter < network.parameter_count(); ++parameter) { // Iterate over parameter for this calculation.
193 const double original = weights[parameter], epsilon = 0.0001; // Set original from this expression.
194 weights[parameter] = original + epsilon; // Set weights[parameter] from this expression.
195 const double plus = mean_loss(); // Set plus from this expression.
196 weights[parameter] = original - epsilon; // Set weights[parameter] from this expression.
197 const double minus = mean_loss(); // Set minus from this expression.
198 weights[parameter] = original; // Set weights[parameter] from this expression.
199 const double numerical = (plus - minus) / (2 * epsilon); // Set numerical from this expression.
200 double analytic = 0; // Use this floating-point value or resource in the following calculation.
201 for (unsigned sample = 0; sample < network.batch_count(); ++sample) // Iterate over sample for this calculation.
202 analytic += gradient[sample * network.parameter_count() + parameter] / // Update analytic with this contribution.
203 double(network.batch_count()); // Use this floating-point value or resource in the following calculation.
204 if (!std::isfinite(analytic) || // Check this condition before entering the following branch.
205 std::fabs(analytic - numerical) > 0.002 + 0.02 * std::fabs(numerical)) // Apply fabs to these arguments.
206 throw shdr::error("neural analytic gradient differs from finite-difference oracle"); // Report neural analytic gradient differs from finite-difference oracle.
207 } // End this block.
208} // End this block.
209} // namespace shdr_examples
210#endif // Finish this conditional compilation branch.