Added convolutional neural network

Virv12 · Jan 30, 2024 · 497e875 · 497e875
1 parent 4651055
commit 497e875
Show file tree

Hide file tree

Showing 5 changed files with 149 additions and 8 deletions.
diff --git a/Makefile b/Makefile
@@ -1,4 +1,4 @@
-# CC=g++ -std=c++17 -g -Og -Wall -Wextra -Iincludes -D_GLIBCXX_DEBUG -fsanitize=address
+# CC=g++ -std=c++17 -g -Ofast -Wall -Wextra -Iincludes -D_GLIBCXX_DEBUG -fsanitize=address
 CC=g++ -std=c++17 -Ofast -Wall -Wextra -Iincludes -DNDEBUG
 
 DATASET=$(addprefix dataset/,t10k-labels-idx1-ubyte train-images-idx3-ubyte train-labels-idx1-ubyte t10k-images-idx3-ubyte)

diff --git a/README.md b/README.md
@@ -5,16 +5,19 @@ Handwritten digit recognition implemented in c++ without libraries
 - [x] k-NN
 - [ ] k-NN with [k-d tree](https://en.wikipedia.org/wiki/K-d_tree)
 - [x] Deep Neural Network - Backpropagation
-- [ ] Convolutional Neural Network
+- [x] Convolutional Neural Network
 
 # Results
 ## k-NN
 Best result with 3 closest neighbours from a 60000 images dataset.  
 Error rate: 2.95%
 
-# dnn
+# Deep Neural Network
 Error rate: 3.87%
 
+# Convolutional Neural Network
+Error rate: 3.61%
+
 # Compilation
 ```bash
 make -j4

diff --git a/includes/nn.h b/includes/nn.h
@@ -1,3 +1,4 @@
+#include <array>
 #include <vector>
 
 struct Layer {
@@ -11,6 +12,7 @@ struct LayerLinear : Layer {
 	float *W, *A;
 
 	LayerLinear(size_t I, size_t O);
+	~LayerLinear();
 
 	virtual std::vector<float> operator() (std::vector<float>&) override;
 	virtual std::vector<float> backprop(std::vector<float>&, std::vector<float>&, const std::vector<float>&) override;
@@ -22,6 +24,28 @@ struct LayerSigmoid : Layer {
 	virtual std::vector<float> backprop(std::vector<float>& m, std::vector<float>& c, const std::vector<float>& p) override;
 };
 
+struct LayerAveragePooling : Layer {
+	std::array<size_t, 2> D, S;
+
+	LayerAveragePooling(std::array<size_t, 2> S, std::array<size_t, 2> D) : D(D), S(S) {}
+
+	virtual std::vector<float> operator() (std::vector<float>&) override;
+	virtual std::vector<float> backprop(std::vector<float>&, std::vector<float>&, const std::vector<float>&) override;
+};
+
+struct LayerConvolutional : Layer {
+	size_t I, O;
+	std::array<size_t, 2> S, K;
+	float *W, *A;
+
+	LayerConvolutional(size_t, size_t, std::array<size_t, 2>, std::array<size_t, 2>);
+	~LayerConvolutional();
+
+	virtual std::vector<float> operator() (std::vector<float>&) override;
+	virtual std::vector<float> backprop(std::vector<float>&, std::vector<float>&, const std::vector<float>&) override;
+	virtual void apply() override;
+};
+
 struct NN {
 	std::vector<Layer*> layers;
 

diff --git a/src/dnn.cpp b/src/dnn.cpp
@@ -12,14 +12,23 @@ using namespace std;
 
 int main() {
 	srand(time(0));
-
 	load_dataset();
+
+	// NN nn {
+	// 	new LayerLinear(28*28, 28),
+	// 	new LayerSigmoid,
+	// 	new LayerLinear(28   , 28),
+	// 	new LayerSigmoid,
+	// 	new LayerLinear(28   , 10),
+	// 	new LayerSigmoid,
+	// };
+
 	NN nn {
-		new LayerLinear(28*28, 28),
-		new LayerSigmoid,
-		new LayerLinear(28   , 28),
+		new LayerAveragePooling({14, 14}, {2, 2}),
+		new LayerConvolutional(1, 2, {10, 10}, {5, 5}),
 		new LayerSigmoid,
-		new LayerLinear(28   , 10),
+
+		new LayerLinear(2*10*10, 10),
 		new LayerSigmoid,
 	};
 
@@ -40,6 +49,11 @@ int main() {
 
 			if ((i + 1) % 10 == 0)
 				nn.apply();
+
+			if ((i + 1) % 500 == 0) {
+				printf("%lu / %lu\r", i+1, S.size());
+				fflush(stdout);
+			}
 		}
 
 		size_t C = 0;

diff --git a/src/nn.cpp b/src/nn.cpp
@@ -13,6 +13,11 @@ LayerLinear::LayerLinear(size_t I, size_t O) : I(I), O(O) {
 		W[i] = (float)rand() / RAND_MAX * 2 - 1;
 }
 
+LayerLinear::~LayerLinear() {
+	delete[] W;
+	delete[] A;
+}
+
 vector<float> LayerLinear::operator() (vector<float>& m) {
 	assert(m.size() == I);
 
@@ -63,6 +68,101 @@ vector<float> LayerSigmoid::backprop(vector<float>& m, vector<float>& c, const v
 	return m;
 }
 
+vector<float> LayerAveragePooling::operator() (vector<float>& m) {
+	assert(m.size() % (S[0] * D[0] * S[1] * D[1]) == 0);
+
+	size_t I = m.size() / (S[0] * D[0] * S[1] * D[1]);
+	vector<float> r(I * S[0] * S[1]);
+
+	for (size_t i = 0; i < I; i++)
+		for (size_t x = 0; x < S[0] * D[0]; x++)
+			for (size_t y = 0; y < S[1] * D[1]; y++)
+				r[(i * S[0] + x / D[0]) * S[1] + y / D[1]] += m[(i * S[0] * D[0] + x) * S[1] * D[1] + y] / (D[0] * D[1]);
+
+	return r;
+}
+
+vector<float> LayerAveragePooling::backprop(vector<float>& m, vector<float>&, const vector<float>&) {
+	assert(m.size() % (S[0] * S[1]) == 0);
+
+	size_t I = m.size() / (S[0] * S[1]);
+	vector<float> r(I * S[0] * D[0] * S[1] * D[1]);
+
+	for (size_t i = 0; i < I; i++)
+		for (size_t x = 0; x < S[0] * D[0]; x++)
+			for (size_t y = 0; y < S[1] * D[1]; y++)
+				r[(i * S[0] * D[0] + x) * S[1] * D[1] + y] = m[(i * S[0] + x / D[0]) * S[1] + y / D[1]] / (D[0] * D[1]);
+
+	return r;
+}
+
+LayerConvolutional::LayerConvolutional(size_t I, size_t O, array<size_t, 2> S, array<size_t, 2> K)
+	: I(I), O(O), S(S), K(K) {
+		W = new float[O * (I * K[0] * K[1] + 1)];
+		A = new float[O * (I * K[0] * K[1] + 1)];
+}
+
+LayerConvolutional::~LayerConvolutional() {
+	delete[] W;
+	delete[] A;
+}
+
+vector<float> LayerConvolutional::operator() (vector<float>& m) {
+	assert(m.size() == I * (S[0] + K[0] - 1) * (S[1] + K[1] - 1));
+
+	vector<float> r(O * S[0] * S[1]);
+
+	for (size_t o = 0; o < O; o++)
+		for (size_t x = 0; x < S[0]; x++)
+			for (size_t y = 0; y < S[1]; y++) {
+				r[(o * S[0] + x) * S[1] + y] = W[(o + 1) * (I * K[0] * K[1] + 1) - 1];
+
+				for (size_t i = 0; i < I; i++)
+					for (size_t dx = 0; dx < K[0]; dx++)
+						for (size_t dy = 0; dy < K[1]; dy++)
+							r[(o * S[0] + x) * S[1] + y] +=
+								m[(i * (S[0] + K[0] - 1) + x + dx) * (S[1] + K[1] - 1) + y + dy] *
+								W[o * (I * K[0] * K[1] + 1) + (i * K[0] + dx) * K[1] + dy];
+			}
+
+	return r;
+}
+
+vector<float> LayerConvolutional::backprop(vector<float>& m, vector<float>&, const vector<float>& p) {
+	assert(m.size() == O * S[0] * S[1]);
+	// assert(c.size() == O * S[0] * S[1]);
+	assert(p.size() == I * (S[0] + K[0] - 1) * (S[1] + K[1] - 1));
+
+	vector<float> r(I * (S[0] + K[0] - 1) * (S[1] + K[1] - 1));
+
+	for (size_t o = 0; o < O; o++)
+		for (size_t x = 0; x < S[0]; x++)
+			for (size_t y = 0; y < S[1]; y++) {
+				A[(o + 1) * (I * K[0] * K[1] + 1) - 1] += m[(o * S[0] + x) * S[1] + y];
+
+				for (size_t i = 0; i < I; i++)
+					for (size_t dx = 0; dx < K[0]; dx++)
+						for (size_t dy = 0; dy < K[1]; dy++) {
+							A[o * (I * K[0] * K[1] + 1) + (i * K[0] + dx) * K[1] + dy] +=
+								p[(i * (S[0] + K[0] - 1) + x + dx) * (S[1] + K[1] - 1) + y + dy] *
+								m[(o * S[0] + x) * S[1] + y];
+
+							r[(i * (S[0] + K[0] - 1) + x + dx) * (S[1] + K[1] - 1) + y + dy] +=
+								W[o * (I * K[0] * K[1] + 1) + (i * K[0] + dx) * K[1] + dy] *
+								m[(o * S[0] + x) * S[1] + y];
+						}
+			}
+
+	return r;
+}
+
+void LayerConvolutional::apply() {
+	for (size_t i = 0; i < O * I * K[0] * K[1]; i++) {
+		W[i] -= 0.03 * A[i];
+		A[i] = 0;
+	}
+}
+
 NN::NN(initializer_list<Layer*> il) : layers(il) {}
 
 vector<float> NN::operator() (vector<float> I) {