[UPDATE] small fixes

scripts/README.md

@@ -0,0 +1,12 @@
+Examples
+============
+
+This folder contains examples to test and validate your code.
+
+## Jupyter Notebooks:
+
+- [eval1](eval1.ipynb): Dataset pre-processing, PCA and KNN;
+- [eval2](eval2.ipynb): Linear and Logistic Regression, grid search and cross-validation;
+- [eval3](eval3.ipynb): Decision Tree and bagging ensemble;
+- [eval4](eval4.ipynb): A simple XOR Neural Network;
+- [eval5](eval5.ipynb): CNN for MNIST dataset.

src/si/supervised/model.py

@@ -21,7 +21,7 @@ def __init__(self):
         self.is_fitted = False
 
     @abstractmethod
-    def fit(self, dataset):
+    def fit(self, dataset, **kwargs):
         raise NotImplementedError
 
     @abstractmethod

src/si/supervised/nn/__init__.py

@@ -1,4 +1,5 @@
-from .nn import NN, Dense, Flatten, Dropout
+from .nn import NN, Dense, Flatten, Dropout, BatchNormalization
 from .activation import *
 from .cnn import *
-from .optimizers import *
+from .optimizers import *
+from .rnn import *

src/si/supervised/nn/activation.py

@@ -292,17 +292,12 @@ def __str__(self):
         return "SoftMax"
 
     def fn(self, z):
-        assert len(z.shape) == 2
-        s = np.max(z, axis=1)
-        s = s[:, np.newaxis]
-        e_x = np.exp(z - s)
-        div = np.sum(e_x, axis=1)
-        div = div[:, np.newaxis]
-        return e_x / div
+        e_x = np.exp(z - np.max(z, axis=-1, keepdims=True))
+        return e_x / np.sum(e_x, axis=-1, keepdims=True)
 
     def prime(self, x):
-        s = x.reshape(-1, 1)
-        return np.diagflat(s) - np.dot(s, s.T)
+        p = self.fn(x)
+        return p * (1 - p)
 
 
 

src/si/supervised/nn/nn.py

@@ -79,8 +79,8 @@ def __init__(
         Neural Network model. The default loss function is the mean square error (MSE).
         A NN may be regarded as a sequence of layers, functions applied sequentialy one after the other.
 
-        :param int epochs: Number of epochs.
-        :param int batch_size: Minibach size
+        :param int epochs: Default number of epochs.
+        :param int batch_size: Default minibach size
         :param Optimizer optimizer: The optimizer.
         :param bool verbose: If all loss (and quality metric) are to be outputed. Default True.
         :param str loss: The loss function. Default `MSE`.
@@ -129,18 +129,23 @@ def set_loss(self, loss):
     def set_metric(self, metric):
         self.metric = metric
 
-    def fit(self, dataset):
-        X, y = dataset.getXy()
+    def fit(self, dataset, **kwargs):
+
+        epochs = kwargs.get('epochs', self.epochs)
+        batch_size = kwargs.get('batch_size', self.batch_size)
+
         self.dataset = dataset
+        X, y = dataset.getXy()
+
         self.history = dict()
-        for epoch in range(1, self.epochs + 1):
+        for epoch in range(1, epochs + 1):
             # lists to save the batch predicted and real values
             # to be later used to compute the epoch loss and
             # quality metrics
             x_ = []
             y_ = []
 
-            for batch in minibatch(X, y, self.batch_size):
+            for batch in minibatch(X, y, batch_size):
                 output_batch, y_batch = batch
                 # forward propagation
                 # propagates values across all layers from
@@ -159,24 +164,26 @@ def fit(self, dataset):
                 x_.append(output_batch)
                 y_.append(y_batch)
 
-            out = np.concatenate(x_)
+            # all the epoch outputs
+            out_all = np.concatenate(x_)
             y_all = np.concatenate(y_)
+
             # compute the loss
-            err = self.loss(y_all, out)
+            err = self.loss(y_all, out_all)
 
             # if a quality metric is defined
             if self.metric is not None:
-                score = self.metric(y_all, out)
+                score = self.metric(y_all, out_all)
                 score_s = f" {self.metric.__name__}={score}"
             else:
                 score = 0
                 score_s = ""
-
+            # save into the history
             self.history[epoch] = (err, score)
 
             # verbosity
             if epoch % self.step == 0:
-                s = f"epoch {epoch}/{self.epochs} loss={err}{score_s}"
+                s = f"epoch {epoch}/{epochs} loss={err}{score_s}"
                 if self.verbose:
                     print(s)
                 else:

src/si/supervised/nn/optimizers.py

@@ -12,6 +12,7 @@
 
 
 class Optimizer(ABC):
+    """Define how to update the learned parameters"""
     @abstractmethod
     def update(self, w, grad_wrt_w):
         raise NotImplementedError
@@ -33,7 +34,7 @@ def update(self, w, grad_wrt_w):
         return w - self.learning_rate * self.w_updt
 
 
-class Adam:
+class Adam(Optimizer):
     def __init__(self, learning_rate=0.001, b1=0.9, b2=0.999):
         self.learning_rate = learning_rate
         self.eps = 1e-8
@@ -80,7 +81,7 @@ def update(self, w, grad_func):
         return w - self.w_updt
 
 
-class Adagrad:
+class Adagrad(Optimizer):
     def __init__(self, learning_rate=0.01):
         self.learning_rate = learning_rate
         self.G = None  # Sum of squares of the gradients
@@ -96,7 +97,7 @@ def update(self, w, grad_wrt_w):
         return w - self.learning_rate * grad_wrt_w / np.sqrt(self.G + self.eps)
 
 
-class Adadelta:
+class Adadelta(Optimizer):
     def __init__(self, rho=0.95, eps=1e-6):
         self.E_w_updt = None  # Running average of squared parameter updates
         self.E_grad = None  # Running average of the squared gradient of w
@@ -131,7 +132,7 @@ def update(self, w, grad_wrt_w):
         return w - self.w_updt
 
 
-class RMSprop:
+class RMSprop(Optimizer):
     def __init__(self, learning_rate=0.01, rho=0.9):
         self.learning_rate = learning_rate
         self.Eg = None  # Running average of the square gradients at w

src/si/supervised/nn/rnn.py

@@ -16,25 +16,21 @@
 class RNN(Layer):
     """A Vanilla Fully-Connected Recurrent Neural Network layer.
 
-    Parameters:
-    -----------
-    n_units: int
-        The number of hidden states in the layer.
-    activation: string
-        The name of the activation function which will be applied to the output of each state.
-    bptt_trunc: int
-        Decides how many time steps the gradient should be propagated backwards through states
-        given the loss gradient for time step t.
-    input_shape: tuple
-        The expected input shape of the layer. For dense layers a single digit specifying
-        the number of features of the input. Must be specified if it is the first layer in
-        the network.
+    param int n_units: The number of hidden states in the layer.
+    param Activation activation: The activation function which will\
+        be applied to the output of each state.
+    param int bptt_trunc: Decides how many time steps the gradient\
+        should be propagated backwards through states given the loss gradient for time step t.
+    param tuple input_shape: The expected input shape of the layer. For dense layers a single
+        digit specifying the number of features of the input. Must be specified if it is the
+        first layer in the network.
     """
     def __init__(self, n_units, activation=None, bptt_trunc=5, input_shape=None):
         self.input_shape = input_shape
         self.n_units = n_units
         self.activation = Tanh() if activation is None else activation
         self.bptt_trunc = bptt_trunc
+
         self.W = None # Weight of the previous state
         self.V = None # Weight of the output
         self.U = None # Weight of the input
@@ -87,7 +83,7 @@ def backward(self, accum_grad):
             # Update gradient w.r.t V at time step t
             grad_V += accum_grad[:, t].T.dot(self.states[:, t])
             # Calculate the gradient w.r.t the state input
-            grad_wrt_state = accum_grad[:, t].dot(self.V) * self.activation.gradient(self.state_input[:, t])
+            grad_wrt_state = accum_grad[:, t].dot(self.V) * self.activation.prime(self.state_input[:, t])
             # Gradient w.r.t the layer input
             accum_grad_next[:, t] = grad_wrt_state.dot(self.U)
             # Update gradient w.r.t W and U by backprop. from time step t for at most
@@ -96,7 +92,7 @@ def backward(self, accum_grad):
                 grad_U += grad_wrt_state.T.dot(self.layer_input[:, t_])
                 grad_W += grad_wrt_state.T.dot(self.states[:, t_-1])
                 # Calculate gradient w.r.t previous state
-                grad_wrt_state = grad_wrt_state.dot(self.W) * self.activation.gradient(self.state_input[:, t_-1])
+                grad_wrt_state = grad_wrt_state.dot(self.W) * self.activation.prime(self.state_input[:, t_-1])
 
         # Update weights
         self.U = self.U_opt.update(self.U, grad_U)