MAINT Unvendor joblib (scikit-learn#13531)

Author: rth

README.rst

@@ -50,6 +50,7 @@ scikit-learn requires:
 - Python (>= 3.5)
 - NumPy (>= 1.11.0)
 - SciPy (>= 0.17.0)
+- joblib (>= 0.11)
 
 **Scikit-learn 0.20 was the last version to support Python2.7.**
 Scikit-learn 0.21 and later require Python 3.5 or newer.

azure-pipelines.yml

@@ -11,7 +11,6 @@ jobs:
       py35_np_atlas:
         DISTRIB: 'ubuntu'
         PYTHON_VERSION: '3.5'
-        SKLEARN_SITE_JOBLIB: '1'
         JOBLIB_VERSION: '0.11'
         SKLEARN_NO_OPENMP: 'True'
       # Linux + Python 3.5 build with OpenBLAS and without SITE_JOBLIB
@@ -23,10 +22,11 @@ jobs:
         SCIPY_VERSION: '0.17.0'
         CYTHON_VERSION: '*'
         PILLOW_VERSION: '4.0.0'
+        # later version of joblib are not packaged in conda for Python 3.5
+        JOBLIB_VERSION: '0.12.3'
         COVERAGE: 'true'
       # Linux environment to test the latest available dependencies and MKL.
       # It runs tests requiring pandas and PyAMG.
-      # It also runs with the site joblib instead of the vendored copy of joblib.
       pylatest_conda:
         DISTRIB: 'conda'
         PYTHON_VERSION: '*'
@@ -41,7 +41,6 @@ jobs:
         COVERAGE: 'true'
         CHECK_PYTEST_SOFT_DEPENDENCY: 'true'
         TEST_DOCSTRINGS: 'true'
-        SKLEARN_SITE_JOBLIB: '1'
         CHECK_WARNINGS: 'true'
 
 - template: build_tools/azure/posix.yml

build_tools/azure/install.cmd

@@ -11,11 +11,11 @@ IF "%PYTHON_ARCH%"=="64" (
     call deactivate
     @rem Clean up any left-over from a previous build
     conda remove --all -q -y -n %VIRTUALENV%
-    conda create -n %VIRTUALENV% -q -y python=%PYTHON_VERSION% numpy scipy cython pytest wheel pillow
+    conda create -n %VIRTUALENV% -q -y python=%PYTHON_VERSION% numpy scipy cython pytest wheel pillow joblib
 
     call activate %VIRTUALENV%
 ) else (
-    pip install numpy scipy cython pytest wheel pillow
+    pip install numpy scipy cython pytest wheel pillow joblib
 )
 if "%COVERAGE%" == "true" (
     pip install coverage codecov pytest-cov

build_tools/azure/install.sh

@@ -27,7 +27,7 @@ make_conda() {
 if [[ "$DISTRIB" == "conda" ]]; then
     TO_INSTALL="python=$PYTHON_VERSION pip pytest pytest-cov \
                 numpy=$NUMPY_VERSION scipy=$SCIPY_VERSION \
-                cython=$CYTHON_VERSION"
+                cython=$CYTHON_VERSION joblib=$JOBLIB_VERSION"
 
     if [[ "$INSTALL_MKL" == "true" ]]; then
         TO_INSTALL="$TO_INSTALL mkl"
@@ -47,10 +47,6 @@ if [[ "$DISTRIB" == "conda" ]]; then
         TO_INSTALL="$TO_INSTALL pillow=$PILLOW_VERSION"
     fi
 
-    if [[ -n "$JOBLIB_VERSION" ]]; then
-        TO_INSTALL="$TO_INSTALL joblib=$JOBLIB_VERSION"
-    fi
-
 	make_conda $TO_INSTALL
 
 elif [[ "$DISTRIB" == "ubuntu" ]]; then

build_tools/travis/install.sh

@@ -113,7 +113,6 @@ elif [[ "$DISTRIB" == "scipy-dev" ]]; then
     pip install --pre --upgrade --timeout=60 -f $dev_url numpy scipy pandas cython
     echo "Installing joblib master"
     pip install https://github.com/joblib/joblib/archive/master.zip
-    export SKLEARN_SITE_JOBLIB=1
     echo "Installing pillow master"
     pip install https://github.com/python-pillow/Pillow/archive/master.zip
     pip install pytest pytest-cov

conftest.py

@@ -61,3 +61,13 @@ def pytest_collection_modifyitems(config, items):
         for item in items:
             if isinstance(item, DoctestItem):
                 item.add_marker(skip_marker)
+
+
+def pytest_configure(config):
+    import sys
+    sys._is_pytest_session = True
+
+
+def pytest_unconfigure(config):
+    import sys
+    del sys._is_pytest_session

doc/modules/computing.rst

@@ -553,6 +553,11 @@ These environment variables should be set before importing scikit-learn.
     is supported. In addition, dumps from joblib.Memory might be incompatible,
     and you might loose some caches and have to redownload some datasets.
 
+    .. deprecated:: 0.21
+
+       As of version 0.21 this parameter has no effect, vendored joblib was
+       removed and site joblib is always used.
+
 :SKLEARN_ASSUME_FINITE:
 
     Sets the default value for the `assume_finite` argument of

doc/whats_new/v0.21.rst

@@ -634,6 +634,16 @@ Multiple modules
   :issue:`13422` by :user:`Madhura Parikh <jdnc>` and
   :user:`Clément Doumouro <ClemDoum>`.
 
+
+Dependencies
+............
+
+- |Enhancement| Joblib is no longer vendored in scikit-learn, and becomes a
+  dependency. Minimal supported version is joblib 0.11, however using
+  version >= 0.13 is strongly recommended. 
+  :issue:`13531` by :user:`Roman Yurchak <rth>`.
+
+
 Changes to estimator checks
 ---------------------------
 

setup.py

@@ -57,6 +57,7 @@
     SCIPY_MIN_VERSION = '0.17.0'
     NUMPY_MIN_VERSION = '1.11.0'
 
+JOBLIB_MIN_VERSION = '0.11'
 
 # Optional setuptools features
 # We need to import setuptools early, if we want setuptools features,
@@ -226,7 +227,8 @@ def setup_package():
                     cmdclass=cmdclass,
                     install_requires=[
                         'numpy>={}'.format(NUMPY_MIN_VERSION),
-                        'scipy>={}'.format(SCIPY_MIN_VERSION)
+                        'scipy>={}'.format(SCIPY_MIN_VERSION),
+                        'joblib>={}'.format(JOBLIB_MIN_VERSION)
                     ],
                     **extra_setuptools_args)
 

sklearn/externals/README

@@ -1,9 +1,6 @@
 This directory contains bundled external dependencies that are updated
 every once in a while.
 
-Note to developers and advanced users: setting the SKLEARN_SITE_JOBLIB to
-a non null value will force scikit-learn to use the site joblib.
-
 Note for distribution packagers: if you want to remove the duplicated
 code and depend on a packaged version, we suggest that you simply do a
 symbolic link in this directory.