From 13ca1e16eff9ae0ae5fb047577449b81b2d4d848 Mon Sep 17 00:00:00 2001
From: chabisik <chabisik@etud.insa-toulouse.fr>
Date: Mon, 29 Nov 2021 00:32:25 +0100
Subject: [PATCH] Adding more activation functions

---
 activations.c | 34 ++++++++++++++++++++++++++++++++--
 activations.h |  6 ++++++
 main.c        |  7 +++----
 network.c     |  2 +-
 network.h     |  2 +-
 neurons.c     | 25 +++++++++++++++++++------
 training.c    | 12 +++++++-----
 training.h    |  4 +++-
 8 files changed, 72 insertions(+), 20 deletions(-)

diff --git a/activations.c b/activations.c
index 254ec63..ec52a07 100644
--- a/activations.c
+++ b/activations.c
@@ -5,10 +5,40 @@
 
 float sigmoid(float weighted_sum)
 {
-    return 1.0 / (1 + exp(-weighted_sum));
+    return 1.0 / (1.0 + exp(-weighted_sum));
 }
 
 float sigmoid_derivative(float output)
 {
-    return sigmoid(output) * (1 - sigmoid(output));
+    return sigmoid(output) * (1.0 - sigmoid(output));
+}
+
+float tan_hyp(float weighted_sum)
+{
+    return tanh(weighted_sum);
+}
+
+float tan_hyp_derivative(float output)
+{
+    return 1.0 - (tan_hyp(output) * tan_hyp(output));
+}
+
+float relu(float weighted_sum)
+{
+    return (weighted_sum > 0.0) ? weighted_sum : 0.0;
+}
+
+float relu_derivative(float output)
+{
+    return (output > 0.0) ? 1.0 : 0.0;
+}
+
+float linear(float weighted_sum)
+{
+    return weighted_sum;
+}
+
+float linear_derivative(float output)
+{
+    return 1.0;
 }
\ No newline at end of file
diff --git a/activations.h b/activations.h
index 2be3aed..2792d10 100644
--- a/activations.h
+++ b/activations.h
@@ -3,5 +3,11 @@
 
 float sigmoid(float weighted_sum);
 float sigmoid_derivative(float output);
+float tan_hyp(float weighted_sum);
+float tan_hyp_derivative(float output);
+float relu(float weighted_sum);
+float relu_derivative(float output);
+float linear(float weighted_sum);
+float linear_derivative(float output);
 
 #endif
\ No newline at end of file
diff --git a/main.c b/main.c
index b048de2..044ba9e 100644
--- a/main.c
+++ b/main.c
@@ -23,10 +23,9 @@ int main(int argc, char *argv[])
     float a=5.0;
     printf("%f\n", (float)exp((double)a));*/
     
-    //int n_neurons[] = {3,10,52,52,6};
-    int n_neurons[] = {15,120,220,120};
-    char *activations[] = {"sigmoid","sigmoid","sigmoid","sigmoid"};
-    Network *net = init_network(n_neurons, 4, activations);
+    int n_neurons[] = {15,120,220,120,200,25};
+    char *activations[] = {"relu","relu","relu","relu","relu","relu"};
+    Network *net = init_network(n_neurons, 6, activations);
     print_network(net);
     destroy_network(net);
     
diff --git a/network.c b/network.c
index 594c6a8..37c1fbe 100644
--- a/network.c
+++ b/network.c
@@ -53,7 +53,7 @@ Network *init_network(int n_neurons_per_layer[], int n_layers, char *activation_
    return network;
 }
 
-void print_network(Network *network)
+void print_network(const Network *network)
 {
     int i, n_params=0;
     printf("#>>==========================================<<#\n");
diff --git a/network.h b/network.h
index e1d9b33..71738f7 100644
--- a/network.h
+++ b/network.h
@@ -12,7 +12,7 @@ struct network
 
 Neuron *generate_layer(int n_neurons, int n_neurons_prev_layer, char *activation_function);
 Network *init_network(int n_neurons_per_layer[], int n_layers, char *activation_function_per_layer[]);
-void print_network(Network *network);
+void print_network(const Network *network);
 void destroy_network(Network *network);
 
 #endif
\ No newline at end of file
diff --git a/neurons.c b/neurons.c
index 6247db3..4cf226a 100644
--- a/neurons.c
+++ b/neurons.c
@@ -37,16 +37,29 @@ Neuron *init_neuron(int n_weights, char *activation_function)
     {
         neuron->weights = init_weight_list(n_weights);
     }
-    neuron->bias = random_float(0.0 , 1.0);
     neuron->output = 0.0;
-    if(strcmp(activation_function, "nothing") == 0)
-    {
-        //to be completed later with tanh, relu, etc : for now only sigmoid is supported and will be the default function
-    }
-    else
+    if(strcmp(activation_function, "sigmoid") == 0)
     {
         neuron->activation = sigmoid;
         neuron->activation_derivative = sigmoid_derivative;
+        neuron->bias = 1.0;
+    }
+    else if(strcmp(activation_function, "tanh") == 0)
+    {
+        neuron->activation = tan_hyp;
+        neuron->activation_derivative = tan_hyp_derivative;
+        neuron->bias = 1.0;
+    }else if(strcmp(activation_function, "linear") == 0)
+    {
+        neuron->activation = linear;
+        neuron->activation_derivative = linear_derivative;
+        neuron->bias = 1.0;
+    }
+    else //relu is assumed as default activation function
+    {
+        neuron->activation = relu;
+        neuron->activation_derivative = relu_derivative;
+        neuron->bias = 0.1; //as suggested by scientific articles (advice of setting bias to 0.1 when using ReLU and 1.0 with other functions)
     }
     neuron->delta_error = 0.0;
     neuron->same_layer_next_neuron = NULL;
diff --git a/training.c b/training.c
index af2862d..731b1a8 100644
--- a/training.c
+++ b/training.c
@@ -1,10 +1,12 @@
-/*#include <stdio.h>
+#include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
 #include "randomness.h"
-#include "network.h"
-#include "neurons.h"
 #include "activations.h"
+#include "neurons.h"
+#include "network.h"
+
+
 
 void forward(Network *network, float sample[])
 {
@@ -16,7 +18,7 @@ void errors_backpropagate(Network *network, float label[])
 
 }
 
-void apply_backpropagate(Network *network, float sample[])
+void apply_backpropagate(Network *network, float learning_rate)
 {
 
-}*/
\ No newline at end of file
+}
\ No newline at end of file
diff --git a/training.h b/training.h
index ddbb6e9..fdc0a10 100644
--- a/training.h
+++ b/training.h
@@ -1,6 +1,8 @@
 #ifndef TRAINING_H
 #define TRAINING_H
 
-
+void forward(Network *network, float sample[]);
+void errors_backpropagate(Network *network, float label[]);
+void apply_backpropagate(Network *network, float learning_rate);
 
 #endif
\ No newline at end of file