[WIP] Add verbose to NCA and MLKR (#105)

* ENH: Add verbose to NCA

* ENH: add verbose to MLKR

* ENH: Add test for convergence warning, and fix datasets (use classification for classification and regression for regression

* STY: update code according to review #105 (review)

* FIX: return the real training time

* FIX: fix forgotten function call in test_no_verbose

* MAINT: Add L-BFGS-B to MLKR, and improve the tests with less features for regression

* FIX: remove MLKR previous arguments for conjugate gradient and put arguments for L-BFGS-B

* FIX: fix test string representation for mlkr

* FIX: convert y to numeric in MLKR (since it is a regression algorithm).

* FIX: fix MLKR test using the method mlkr._loss

Author: wdevazelhes

metric_learn/mlkr.py

@@ -7,12 +7,16 @@
 for dimensionality reduction and high dimensional data visualization.
 """
 from __future__ import division, print_function
+import time
+import sys
+import warnings
 import numpy as np
 from sklearn.utils.fixes import logsumexp
 from scipy.optimize import minimize
 from sklearn.decomposition import PCA
 from sklearn.metrics import pairwise_distances
 from sklearn.utils.validation import check_X_y
+from sklearn.exceptions import ConvergenceWarning
 
 from .base_metric import BaseMetricLearner
 
@@ -21,8 +25,8 @@
 
 class MLKR(BaseMetricLearner):
   """Metric Learning for Kernel Regression (MLKR)"""
-  def __init__(self, num_dims=None, A0=None, epsilon=0.01, alpha=0.0001,
-               max_iter=1000):
+  def __init__(self, num_dims=None, A0=None, tol=None, max_iter=1000,
+               verbose=False):
     """
     Initialize MLKR.
 
@@ -34,23 +38,23 @@ def __init__(self, num_dims=None, A0=None, epsilon=0.01, alpha=0.0001,
     A0: array-like, optional
         Initialization of transformation matrix. Defaults to PCA loadings.
 
-    epsilon: float, optional
-        Step size for congujate gradient descent.
-
-    alpha: float, optional
-        Stopping criterion for congujate gradient descent.
+    tol: float, optional (default=None)
+        Convergence tolerance for the optimization.
 
     max_iter: int, optional
         Cap on number of congugate gradient iterations.
+
+    verbose : bool, optional (default=False)
+        Whether to print progress messages or not.
     """
     self.num_dims = num_dims
     self.A0 = A0
-    self.epsilon = epsilon
-    self.alpha = alpha
+    self.tol = tol
     self.max_iter = max_iter
+    self.verbose = verbose
 
   def _process_inputs(self, X, y):
-      self.X_, y = check_X_y(X, y)
+      self.X_, y = check_X_y(X, y, y_numeric=True)
       n, d = self.X_.shape
       if y.shape[0] != n:
           raise ValueError('Data and label lengths mismatch: %d != %d'
@@ -80,30 +84,67 @@ def fit(self, X, y):
       """
       X, y, A = self._process_inputs(X, y)
 
-      res = minimize(_loss, A.ravel(), (X, y), method='CG', jac=True,
-                     tol=self.alpha,
-                     options=dict(maxiter=self.max_iter, eps=self.epsilon))
+      # Measure the total training time
+      train_time = time.time()
+
+      self.n_iter_ = 0
+      res = minimize(self._loss, A.ravel(), (X, y), method='L-BFGS-B',
+                     jac=True, tol=self.tol,
+                     options=dict(maxiter=self.max_iter))
       self.transformer_ = res.x.reshape(A.shape)
-      self.n_iter_ = res.nit
+
+      # Stop timer
+      train_time = time.time() - train_time
+      if self.verbose:
+          cls_name = self.__class__.__name__
+          # Warn the user if the algorithm did not converge
+          if not res.success:
+              warnings.warn('[{}] MLKR did not converge: {}'
+                            .format(cls_name, res.message), ConvergenceWarning)
+          print('[{}] Training took {:8.2f}s.'.format(cls_name, train_time))
+
       return self
 
   def transformer(self):
       return self.transformer_
 
-
-def _loss(flatA, X, y):
-  A = flatA.reshape((-1, X.shape[1]))
-  X_embedded = np.dot(X, A.T)
-  dist = pairwise_distances(X_embedded, squared=True)
-  np.fill_diagonal(dist, np.inf)
-  softmax = np.exp(- dist - logsumexp(- dist, axis=1)[:, np.newaxis])
-  yhat = softmax.dot(y)
-  ydiff = yhat - y
-  cost = (ydiff**2).sum()
-
-  # also compute the gradient
-  W = softmax * ydiff[:, np.newaxis] * (y - yhat[:, np.newaxis])
-  W_sym = W + W.T
-  np.fill_diagonal(W_sym, - W.sum(axis=0))
-  grad = 4 * (X_embedded.T.dot(W_sym)).dot(X)
-  return cost, grad.ravel()
+  def _loss(self, flatA, X, y):
+
+    if self.n_iter_ == 0 and self.verbose:
+      header_fields = ['Iteration', 'Objective Value', 'Time(s)']
+      header_fmt = '{:>10} {:>20} {:>10}'
+      header = header_fmt.format(*header_fields)
+      cls_name = self.__class__.__name__
+      print('[{cls}]'.format(cls=cls_name))
+      print('[{cls}] {header}\n[{cls}] {sep}'.format(cls=cls_name,
+                                                     header=header,
+                                                     sep='-' * len(header)))
+
+    start_time = time.time()
+
+    A = flatA.reshape((-1, X.shape[1]))
+    X_embedded = np.dot(X, A.T)
+    dist = pairwise_distances(X_embedded, squared=True)
+    np.fill_diagonal(dist, np.inf)
+    softmax = np.exp(- dist - logsumexp(- dist, axis=1)[:, np.newaxis])
+    yhat = softmax.dot(y)
+    ydiff = yhat - y
+    cost = (ydiff ** 2).sum()
+
+    # also compute the gradient
+    W = softmax * ydiff[:, np.newaxis] * (y - yhat[:, np.newaxis])
+    W_sym = W + W.T
+    np.fill_diagonal(W_sym, - W.sum(axis=0))
+    grad = 4 * (X_embedded.T.dot(W_sym)).dot(X)
+
+    if self.verbose:
+      start_time = time.time() - start_time
+      values_fmt = '[{cls}] {n_iter:>10} {loss:>20.6e} {start_time:>10.2f}'
+      print(values_fmt.format(cls=self.__class__.__name__,
+                              n_iter=self.n_iter_, loss=cost,
+                              start_time=start_time))
+      sys.stdout.flush()
+
+    self.n_iter_ += 1
+
+    return cost, grad.ravel()

metric_learn/nca.py

@@ -6,11 +6,13 @@
 from __future__ import absolute_import
 
 import warnings
+import time
+import sys
 import numpy as np
 from scipy.optimize import minimize
 from sklearn.metrics import pairwise_distances
 from sklearn.utils.validation import check_X_y
-from numpy.linalg import multi_dot
+from sklearn.exceptions import ConvergenceWarning
 
 try:  # scipy.misc.logsumexp is deprecated in scipy 1.0.0
     from scipy.special import logsumexp
@@ -24,7 +26,7 @@
 
 class NCA(BaseMetricLearner):
   def __init__(self, num_dims=None, max_iter=100, learning_rate='deprecated',
-               tol=None):
+               tol=None, verbose=False):
     """Neighborhood Components Analysis
 
     Parameters
@@ -45,11 +47,15 @@ def __init__(self, num_dims=None, max_iter=100, learning_rate='deprecated',
 
     tol : float, optional (default=None)
         Convergence tolerance for the optimization.
+
+    verbose : bool, optional (default=False)
+      Whether to print progress messages or not.
     """
     self.num_dims = num_dims
     self.max_iter = max_iter
     self.learning_rate = learning_rate  # TODO: remove in v.0.5.0
     self.tol = tol
+    self.verbose = verbose
 
   def transformer(self):
     return self.A_
@@ -70,6 +76,9 @@ def fit(self, X, y):
     if num_dims is None:
         num_dims = d
 
+    # Measure the total training time
+    train_time = time.time()
+
     # Initialize A to a scaling matrix
     A = np.zeros((num_dims, d))
     np.fill_diagonal(A, 1./(np.maximum(X.max(axis=0)-X.min(axis=0), EPS)))
@@ -86,15 +95,41 @@ def fit(self, X, y):
                         }
 
     # Call the optimizer
+    self.n_iter_ = 0
     opt_result = minimize(**optimizer_params)
 
     self.X_ = X
     self.A_ = opt_result.x.reshape(-1, X.shape[1])
     self.n_iter_ = opt_result.nit
+
+    # Stop timer
+    train_time = time.time() - train_time
+    if self.verbose:
+      cls_name = self.__class__.__name__
+
+      # Warn the user if the algorithm did not converge
+      if not opt_result.success:
+        warnings.warn('[{}] NCA did not converge: {}'.format(
+            cls_name, opt_result.message), ConvergenceWarning)
+
+      print('[{}] Training took {:8.2f}s.'.format(cls_name, train_time))
+
     return self
 
-  @staticmethod
-  def _loss_grad_lbfgs(A, X, mask, sign=1.0):
+  def _loss_grad_lbfgs(self, A, X, mask, sign=1.0):
+
+    if self.n_iter_ == 0 and self.verbose:
+      header_fields = ['Iteration', 'Objective Value', 'Time(s)']
+      header_fmt = '{:>10} {:>20} {:>10}'
+      header = header_fmt.format(*header_fields)
+      cls_name = self.__class__.__name__
+      print('[{cls}]'.format(cls=cls_name))
+      print('[{cls}] {header}\n[{cls}] {sep}'.format(cls=cls_name,
+                                                     header=header,
+                                                     sep='-' * len(header)))
+
+    start_time = time.time()
+
     A = A.reshape(-1, X.shape[1])
     X_embedded = np.dot(X, A.T)  # (n_samples, num_dims)
     # Compute softmax distances
@@ -113,4 +148,14 @@ def _loss_grad_lbfgs(A, X, mask, sign=1.0):
     weighted_p_ij_sym = weighted_p_ij + weighted_p_ij.T
     np.fill_diagonal(weighted_p_ij_sym, - weighted_p_ij.sum(axis=0))
     gradient = 2 * (X_embedded.T.dot(weighted_p_ij_sym)).dot(X)
+
+    if self.verbose:
+        start_time = time.time() - start_time
+        values_fmt = '[{cls}] {n_iter:>10} {loss:>20.6e} {start_time:>10.2f}'
+        print(values_fmt.format(cls=self.__class__.__name__,
+                                n_iter=self.n_iter_, loss=loss,
+                                start_time=start_time))
+        sys.stdout.flush()
+
+    self.n_iter_ += 1
     return sign * loss, sign * gradient.ravel()

test/metric_learn_test.py

@@ -1,11 +1,14 @@
+import re
 import unittest
+import pytest
 import numpy as np
 from scipy.optimize import check_grad
 from six.moves import xrange
 from sklearn.metrics import pairwise_distances
 from sklearn.datasets import load_iris, make_classification, make_regression
 from numpy.testing import assert_array_almost_equal, assert_array_equal
 from sklearn.utils.testing import assert_warns_message
+from sklearn.exceptions import ConvergenceWarning
 from sklearn.utils.validation import check_X_y
 
 from metric_learn import (LMNN, NCA, LFDA, Covariance, MLKR, MMC,
@@ -111,12 +114,14 @@ def test_finite_differences(self):
     X, y = make_classification()
     M = np.random.randn(np.random.randint(1, X.shape[1] + 1), X.shape[1])
     mask = y[:, np.newaxis] == y[np.newaxis, :]
+    nca = NCA()
+    nca.n_iter_ = 0
 
     def fun(M):
-      return NCA._loss_grad_lbfgs(M, X, mask)[0]
+      return nca._loss_grad_lbfgs(M, X, mask)[0]
 
     def grad(M):
-      return NCA._loss_grad_lbfgs(M, X, mask)[1].ravel()
+      return nca._loss_grad_lbfgs(M, X, mask)[1].ravel()
 
     # compute relative error
     rel_diff = check_grad(fun, grad, M.ravel()) / np.linalg.norm(grad(M))
@@ -257,13 +262,14 @@ def test_finite_differences(self):
     X, y = make_regression(n_features=4, random_state=1, n_samples=20)
     X, y = check_X_y(X, y)
     M = np.random.randn(2, X.shape[1])
-    from metric_learn.mlkr import _loss
+    mlkr = MLKR()
+    mlkr.n_iter_ = 0
 
     def fun(M):
-      return _loss(M, X, y)[0]
+      return mlkr._loss(M, X, y)[0]
 
     def grad_fn(M):
-      return _loss(M, X, y)[1].ravel()
+      return mlkr._loss(M, X, y)[1].ravel()
 
     # compute relative error
     rel_diff = check_grad(fun, grad_fn, M.ravel()) / np.linalg.norm(grad_fn(M))
@@ -307,5 +313,57 @@ def test_iris(self):
     self.assertLess(csep, 0.2)
 
 
+@pytest.mark.parametrize(('algo_class', 'dataset'),
+                         [(NCA, make_classification()),
+                          (MLKR, make_regression())])
+def test_verbose(algo_class, dataset, capsys):
+  # assert there is proper output when verbose = True
+  X, y = dataset
+  model = algo_class(verbose=True)
+  model.fit(X, y)
+  out, _ = capsys.readouterr()
+
+  # check output
+  lines = re.split('\n+', out)
+  header = '{:>10} {:>20} {:>10}'.format('Iteration', 'Objective Value',
+                                         'Time(s)')
+  assert lines[0] == '[{}]'.format(algo_class.__name__)
+  assert lines[1] == '[{}] {}'.format(algo_class.__name__, header)
+  assert lines[2] == '[{}] {}'.format(algo_class.__name__, '-' * len(header))
+  for line in lines[3:-2]:
+    # The following regex will match for instance:
+    # '[NCA]          0         6.988936e+01       0.01'
+    assert re.match("\[" + algo_class.__name__ + "\]\ *\d+\ *\d\.\d{6}e[+|-]"
+                    "\d+\ *\d+\.\d{2}", line)
+  assert re.match("\[" + algo_class.__name__ + "\] Training took\ *"
+                  "\d+\.\d{2}s\.", lines[-2])
+  assert lines[-1] == ''
+
+
+@pytest.mark.parametrize(('algo_class', 'dataset'),
+                         [(NCA, make_classification()),
+                          (MLKR, make_regression(n_features=10))])
+def test_no_verbose(dataset, algo_class, capsys):
+  # assert by default there is no output (verbose=False)
+  X, y = dataset
+  model = algo_class()
+  model.fit(X, y)
+  out, _ = capsys.readouterr()
+  # check output
+  assert (out == '')
+
+
+@pytest.mark.parametrize(('algo_class', 'dataset'),
+                         [(NCA, make_classification()),
+                          (MLKR, make_regression(n_features=10))])
+def test_convergence_warning(dataset, algo_class):
+    X, y = dataset
+    model = algo_class(max_iter=2, verbose=True)
+    cls_name = model.__class__.__name__
+    assert_warns_message(ConvergenceWarning,
+                         '[{}] {} did not converge'.format(cls_name, cls_name),
+                         model.fit, X, y)
+
+
 if __name__ == '__main__':
   unittest.main()

test/test_base_metric.py

@@ -17,7 +17,7 @@ def test_lmnn(self):
   def test_nca(self):
     self.assertEqual(str(metric_learn.NCA()),
                      ("NCA(learning_rate='deprecated', max_iter=100, "
-                      "num_dims=None, tol=None)"))
+                      "num_dims=None, tol=None,\n  verbose=False)"))
 
   def test_lfda(self):
     self.assertEqual(str(metric_learn.LFDA()),
@@ -61,8 +61,8 @@ def test_rca(self):
 
   def test_mlkr(self):
     self.assertEqual(str(metric_learn.MLKR()),
-                     "MLKR(A0=None, alpha=0.0001, epsilon=0.01, "
-                     "max_iter=1000, num_dims=None)")
+                     "MLKR(A0=None, max_iter=1000, num_dims=None, tol=None, "
+                     "verbose=False)")
 
   def test_mmc(self):
     self.assertEqual(str(metric_learn.MMC()), """