Add linear activation function (#119)

Co-authored-by: Richard Weed <[email protected]>

Author: milancurcic

src/nf/nf_activation_1d.f90

@@ -10,6 +10,7 @@ module nf_activation_1d
   public :: elu, elu_prime
   public :: exponential
   public :: gaussian, gaussian_prime
+  public :: linear, linear_prime
   public :: relu, relu_prime
   public :: sigmoid, sigmoid_prime
   public :: softmax, softmax_prime
@@ -72,6 +73,20 @@ pure function gaussian_prime(x) result(res)
     res = -2 * x * gaussian(x)
   end function gaussian_prime
 
+  pure function linear(x) result(res)
+    ! Linear activation function.
+    real, intent(in) :: x(:)
+    real :: res(size(x))
+    res = x
+  end function linear
+
+  pure function linear_prime(x) result(res)
+    ! First derivative of the Gaussian activation function.
+    real, intent(in) :: x(:)
+    real :: res(size(x))
+    res = 1
+  end function linear_prime
+
   pure function relu(x) result(res)
     !! Rectified Linear Unit (ReLU) activation function.
     real, intent(in) :: x(:)

src/nf/nf_activation_3d.f90

@@ -10,6 +10,7 @@ module nf_activation_3d
   public :: elu, elu_prime
   public :: exponential
   public :: gaussian, gaussian_prime
+  public :: linear, linear_prime
   public :: relu, relu_prime
   public :: sigmoid, sigmoid_prime
   public :: softmax, softmax_prime
@@ -72,6 +73,20 @@ pure function gaussian_prime(x) result(res)
     res = -2 * x * gaussian(x)
   end function gaussian_prime
 
+  pure function linear(x) result(res)
+    ! Linear activation function.
+    real, intent(in) :: x(:,:,:)
+    real :: res(size(x,1),size(x,2),size(x,3))
+    res = x
+  end function linear
+
+  pure function linear_prime(x) result(res)
+    ! First derivative of the linear activation function.
+    real, intent(in) :: x(:,:,:)
+    real :: res(size(x,1),size(x,2),size(x,3))
+    res = 1
+  end function linear_prime
+
   pure function relu(x) result(res)
     !! Rectified Linear Unit (ReLU) activation function.
     real, intent(in) :: x(:,:,:)

src/nf/nf_conv2d_layer_submodule.f90

@@ -4,6 +4,7 @@
                               elu, elu_prime, &
                               exponential, &
                               gaussian, gaussian_prime, &
+                              linear, linear_prime, &
                               relu, relu_prime, &
                               sigmoid, sigmoid_prime, &
                               softmax, softmax_prime, &
@@ -254,6 +255,11 @@ elemental module subroutine set_activation(self, activation)
         self % activation_prime => gaussian_prime
         self % activation_name = 'gaussian'
 
+      case('linear')
+        self % activation => linear
+        self % activation_prime => linear_prime
+        self % activation_name = 'linear'
+
       case('relu')
         self % activation => relu
         self % activation_prime => relu_prime
@@ -286,9 +292,8 @@ elemental module subroutine set_activation(self, activation)
 
       case default
         error stop 'Activation must be one of: ' // &
-          '"elu", "exponential", "gaussian", "relu", ' // &
-          '"sigmoid", "softmax", "softplus", "step", ' // &
-          'or "tanh".'
+          '"elu", "exponential", "gaussian", "linear", "relu", "sigmoid", ' // &
+          '"softmax", "softplus", "step", or "tanh".'
 
     end select
 

src/nf/nf_dense_layer_submodule.f90

@@ -4,6 +4,7 @@
                               elu, elu_prime, &
                               exponential, &
                               gaussian, gaussian_prime, &
+                              linear, linear_prime, &
                               relu, relu_prime, &
                               sigmoid, sigmoid_prime, &
                               softmax, softmax_prime, &
@@ -157,6 +158,11 @@ elemental module subroutine set_activation(self, activation)
         self % activation_prime => gaussian_prime
         self % activation_name = 'gaussian'
 
+      case('linear')
+        self % activation => linear
+        self % activation_prime => linear_prime
+        self % activation_name = 'linear'
+
       case('relu')
         self % activation => relu
         self % activation_prime => relu_prime
@@ -189,9 +195,8 @@ elemental module subroutine set_activation(self, activation)
 
       case default
         error stop 'Activation must be one of: ' // &
-          '"elu", "exponential", "gaussian", "relu", ' // &
-          '"sigmoid", "softmax", "softplus", "step", ' // &
-          'or "tanh".'
+          '"elu", "exponential", "gaussian", "linear", "relu", "sigmoid", ' // &
+          '"softmax", "softplus", "step", or "tanh".'
 
     end select