Add some comments to the neuralnets

Author: rushter

mla/neuralnet/activations.py

@@ -1,7 +1,10 @@
 import autograd.numpy as np
 
+"""
+References:
+https://en.wikipedia.org/wiki/Activation_function
+"""
 
-# https://en.wikipedia.org/wiki/Activation_function
 
 def sigmoid(z):
     return 1.0 / (1.0 + np.exp(-z))

mla/neuralnet/initializations.py

@@ -1,6 +1,10 @@
 import numpy as np
 
+"""
+References:
+http://jmlr.org/proceedings/papers/v9/glorot10a/glorot10a.pdf
 
+"""
 def normal(shape, scale=0.5):
     return np.random.normal(size=shape, scale=scale)
 
@@ -62,7 +66,7 @@ def he_uniform(shape, **kwargs):
 
 
 def get_initializer(name):
-    """Return initialization function by name"""
+    """Returns initialization function by the name."""
     try:
         return globals()[name]
     except:

mla/neuralnet/loss.py

@@ -4,7 +4,7 @@
 
 
 def get_loss(name):
-    """Return loss function by name"""
+    """Returns loss function by the name."""
 
     try:
         return globals()[name]

mla/neuralnet/nnet.py

@@ -42,7 +42,8 @@ def __init__(self, layers, optimizer, loss, max_epochs=10, batch_size=64, random
         self.training = False
         self._initialized = False
 
-    def _setup_layers(self, x_shape, ):
+    def _setup_layers(self, x_shape):
+        """Initialize model's layers."""
         x_shape = list(x_shape)
         x_shape[0] = self.batch_size
 
@@ -55,6 +56,8 @@ def _setup_layers(self, x_shape, ):
         logging.info('Total parameters: %s' % self.n_params)
 
     def _find_bprop_entry(self):
+        """Find entry layer for back propagation."""
+
         if len(self.layers) > 0 and not hasattr(self.layers[-1], 'parameters'):
             return -1
         return len(self.layers)
@@ -73,7 +76,10 @@ def fit(self, X, y=None):
         self.is_training = False
 
     def update(self, X, y):
+        # Forward pass
         y_pred = self.fprop(X)
+
+        # Backward pass
         grad = self.loss_grad(y, y_pred)
         for layer in reversed(self.layers[:self.bprop_entry]):
             grad = layer.backward_pass(grad)
@@ -100,14 +106,18 @@ def parametric_layers(self):
 
     @property
     def parameters(self):
+        """Returns a list of all parameters."""
         params = []
         for layer in self.parametric_layers:
             params.append(layer.parameters)
         return params
 
     def error(self, X=None, y=None):
+        """Calculate an error for given examples."""
         training_phase = self.is_training
         if training_phase:
+            # Temporally disable training.
+            # Some layers work differently while training (e.g. Dropout).
             self.is_training = False
         if X is None and y is None:
             y_pred = self._predict(self.X)
@@ -131,6 +141,7 @@ def is_training(self, train):
                 layer.is_training = train
 
     def shuffle_dataset(self):
+        """Shuffle rows in the dataset."""
         n_samples = self.X.shape[0]
         indices = np.arange(n_samples)
         np.random.shuffle(indices)
@@ -139,10 +150,12 @@ def shuffle_dataset(self):
 
     @property
     def n_layers(self):
+        """Returns the number of layers."""
         return self._n_layers
 
     @property
     def n_params(self):
+        """Return the number of trainable parameters."""
         return sum([layer.parameters.n_params for layer in self.parametric_layers])
 
     def reset(self):

mla/neuralnet/optimizers.py

@@ -19,29 +19,35 @@ def optimize(self, network):
         for i in range(network.max_epochs):
             if network.shuffle:
                 network.shuffle_dataset()
+
             start_time = time.time()
             loss = self.train_epoch(network)
             loss_history.append(loss)
             msg = "Epoch:%s, train loss: %s" % (i, loss)
+
             if network.log_metric:
                 msg += ', train %s: %s' % (network.metric_name, network.error())
             msg += ', elapsed: %s sec.' % (time.time() - start_time)
             logging.info(msg)
         return loss_history
 
     def update(self, network):
+        """Performs an update of parameters."""
         raise NotImplementedError
 
     def train_epoch(self, network):
         self._setup(network)
         losses = []
 
+        # Create batch iterator
         X_batch = batch_iterator(network.X, network.batch_size)
         y_batch = batch_iterator(network.y, network.batch_size)
+
         for X, y in tqdm(zip(X_batch, y_batch), 'Epoch progress'):
             loss = np.mean(network.update(X, y))
             self.update(network)
             losses.append(loss)
+
         epoch_loss = np.mean(losses)
         return epoch_loss
 

mla/neuralnet/parameters.py

@@ -16,9 +16,10 @@ def __init__(self, init='glorot_uniform', scale=0.5, bias=1.0, regularizers=None
             Initial values for bias.
         regularizers : dict
             Weight regularizers.
-            {'W' : L2()}
+            >>> {'W' : L2()}
         constraints : dict
-            Weight constraints. {'b' : MaxNorm()}
+            Weight constraints.
+            >>> {'b' : MaxNorm()}
         """
         if constraints is None:
             self.constraints = {}
@@ -60,6 +61,7 @@ def step(self, name, step):
             self._params[name] = self.constraints[name].clip(self._params[name])
 
     def update_grad(self, name, value):
+        """Update gradient values."""
         self._grads[name] = value
 
         if name in self.regularizers:

mla/neuralnet/regularizers.py

@@ -29,5 +29,6 @@ def _penalty(self, weights):
 
 
 class ElasticNet(Regularizer):
+    """Linear combination of L1 and L2 penalties."""
     def _penalty(self, weights):
         return 0.5 * self.C * weights ** 2 + (1.0 - self.C) * np.abs(weights)