Added Mish activation function - Ready for merge

NeuralNetwork.NET/APIs/Enums/ActivationType.cs

@@ -1,4 +1,4 @@
-namespace NeuralNetworkNET.APIs.Enums
+namespace NeuralNetworkNET.APIs.Enums
 {
     /// <summary>
     /// Indicates an activation function to use in a neural network
@@ -59,6 +59,14 @@ public enum ActivationType : byte
         /// <summary>
         /// A linear activation function that just returns the input value
         /// </summary>
-        Identity
+        Identity,
+
+        /// <summary>
+        /// The Mish function, proposed by Diganta Misra (https://arxiv.org/abs/1908.08681)
+        /// Definition: x tanh(ln(1 + e^2)) 
+        /// Implimentation: x * Tanh(Softplus(x))
+        /// </summary>
+        Mish
+
     }
-}
+}

NeuralNetwork.NET/Networks/Activations/ActivationFunctionProvider.cs

@@ -1,4 +1,4 @@
-using System;
+using System;
 using JetBrains.Annotations;
 using NeuralNetworkNET.APIs.Enums;
 using NeuralNetworkNET.Networks.Activations.Delegates;
@@ -28,7 +28,8 @@ public static (ActivationFunction, ActivationFunction) GetActivations(Activation
                 case ActivationType.Softmax: return (ActivationFunctions.Softmax, null);
                 case ActivationType.Softplus: return (ActivationFunctions.Softplus, ActivationFunctions.Sigmoid);
                 case ActivationType.ELU: return (ActivationFunctions.ELU, ActivationFunctions.ELUPrime);
-                case ActivationType.Identity: return (ActivationFunctions.Identity, ActivationFunctions.Identityprime);
+                case ActivationType.Identity: return (ActivationFunctions.Identity, ActivationFunctions.IdentityPrime);
+                case ActivationType.Mish: return (ActivationFunctions.Mish, ActivationFunctions.MishPrime);
                 default:
                     throw new ArgumentOutOfRangeException(nameof(ActivationType), "Unsupported activation function");
             }

NeuralNetwork.NET/Networks/Activations/ActivationFunctions.cs

@@ -1,4 +1,4 @@
-using System;
+using System;
 using System.Runtime.CompilerServices;
 using JetBrains.Annotations;
 
@@ -217,6 +217,30 @@ public static float Softplus(float x)
         [PublicAPI]
         [Pure]
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        public static float Identityprime(float x) => 1;
+        public static float IdentityPrime(float x) => 1;
+
+        /// <summary>
+        /// Applies the Mish function
+        /// </summary>
+        /// <param name="x">The input to process</param>
+        [PublicAPI]
+        [Pure]
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public static float Mish(float x) => x * Tanh(Softplus(x));
+
+        /// <summary>
+        /// Applies the Mish Derivative function
+        /// </summary>
+        /// <param name="x">The input to process</param>
+        [PublicAPI]
+        [Pure]
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public static float MishPrime(float x)
+        {
+            float
+                s = 2 * (float)Math.Exp(x) + (float)Math.Exp(2 * x) + 2,
+                w = 4 * (x + 1) + (4 * ((float)Math.Exp(2 * x))) + (float)Math.Exp(3 * x) + (float)Math.Exp(x) * (4 * x + 6);
+            return (float)Math.Exp(x) * w / (s * s);
+        }
     }
 }