joblib 0.13.0 (scikit-learn#12531)

Author: ogrisel

sklearn/externals/joblib/__init__.py

@@ -14,11 +14,11 @@
  ==================== ===============================================
  **Documentation:** https://joblib.readthedocs.io
 
- **Download:** https://pypi.org/project/joblib/#files
+ **Download:** http://pypi.python.org/pypi/joblib#downloads
 
- **Source code:** https://github.com/joblib/joblib
+ **Source code:** http://github.com/joblib/joblib
 
- **Report issues:** https://github.com/joblib/joblib/issues
+ **Report issues:** http://github.com/joblib/joblib/issues
  ==================== ===============================================
 
 
@@ -106,7 +106,7 @@
 # Dev branch marker is: 'X.Y.dev' or 'X.Y.devN' where N is an integer.
 # 'X.Y.dev0' is the canonical version of 'X.Y.dev'
 #
-__version__ = '0.12.5'
+__version__ = '0.13.0'
 
 
 from .memory import Memory, MemorizedResult, register_store_backend
@@ -123,8 +123,11 @@
 from .parallel import parallel_backend
 from .parallel import effective_n_jobs
 
+from .externals.loky import wrap_non_picklable_objects
+
 
 __all__ = ['Memory', 'MemorizedResult', 'PrintTime', 'Logger', 'hash', 'dump',
  'load', 'Parallel', 'delayed', 'cpu_count', 'effective_n_jobs',
  'register_parallel_backend', 'parallel_backend',
- 'register_store_backend', 'register_compressor']
+ 'register_store_backend', 'register_compressor',
+ 'wrap_non_picklable_objects']

sklearn/externals/joblib/_dask.py

@@ -145,7 +145,7 @@ def __reduce__(self):
  return (DaskDistributedBackend, ())
 
  def get_nested_backend(self):
- return DaskDistributedBackend(client=self.client)
+ return DaskDistributedBackend(client=self.client), -1
 
  def configure(self, n_jobs=1, parallel=None, **backend_args):
  return self.effective_n_jobs(n_jobs)

sklearn/externals/joblib/_parallel_backends.py

@@ -126,9 +126,9 @@ def get_nested_backend(self):
  """
  nesting_level = getattr(self, 'nesting_level', 0) + 1
  if nesting_level > 1:
- return SequentialBackend(nesting_level=nesting_level)
+ return SequentialBackend(nesting_level=nesting_level), None
  else:
- return ThreadingBackend(nesting_level=nesting_level)
+ return ThreadingBackend(nesting_level=nesting_level), None
 
  @contextlib.contextmanager
  def retrieval_context(self):
@@ -185,8 +185,12 @@ def apply_async(self, func, callback=None):
  return result
 
  def get_nested_backend(self):
- nested_level = getattr(self, 'nesting_level', 0) + 1
- return SequentialBackend(nesting_level=nested_level)
+ # import is not top level to avoid cyclic import errors.
+ from .parallel import get_active_backend
+
+ # SequentialBackend should neither change the nesting level, the
+ # default backend or the number of jobs. Just return the current one.
+ return get_active_backend()
 
 
 class PoolManagerMixin(object):

sklearn/externals/joblib/compressor.py

@@ -30,7 +30,7 @@
  lz4 = None
 
 LZ4_NOT_INSTALLED_ERROR = ('LZ4 is not installed. Install it with pip: '
- 'https://python-lz4.readthedocs.io/')
+ 'http://python-lz4.readthedocs.io/')
 
 # Registered compressors
 _COMPRESSORS = {}

sklearn/externals/joblib/externals/cloudpickle/__init__.py

@@ -2,4 +2,4 @@
 
 from .cloudpickle import *
 
-__version__ = '0.5.6'
+__version__ = '0.6.1'

sklearn/externals/joblib/externals/cloudpickle/cloudpickle.py

@@ -42,20 +42,20 @@
 """
 from __future__ import print_function
 
-import dis
-from functools import partial
-import imp
 import io
-import itertools
-import logging
+import dis
+import sys
+import types
 import opcode
-import operator
 import pickle
 import struct
-import sys
-import traceback
-import types
+import logging
 import weakref
+import operator
+import importlib
+import itertools
+import traceback
+from functools import partial
 
 
 # cloudpickle is meant for inter process communication: we expect all
@@ -78,6 +78,22 @@
  PY3 = True
 
 
+# Container for the global namespace to ensure consistent unpickling of
+# functions defined in dynamic modules (modules not registed in sys.modules).
+_dynamic_modules_globals = weakref.WeakValueDictionary()
+
+
+class _DynamicModuleFuncGlobals(dict):
+ """Global variables referenced by a function defined in a dynamic module
+
+ To avoid leaking references we store such context in a WeakValueDictionary
+ instance. However instances of python builtin types such as dict cannot
+ be used directly as values in such a construct, hence the need for a
+ derived class.
+ """
+ pass
+
+
 def _make_cell_set_template_code():
  """Get the Python compiler to emit LOAD_FAST(arg); STORE_DEREF
 
@@ -288,20 +304,10 @@ def save_module(self, obj):
  """
  Save a module as an import
  """
- mod_name = obj.__name__
- # If module is successfully found then it is not a dynamically created module
- if hasattr(obj, '__file__'):
- is_dynamic = False
- else:
- try:
- _find_module(mod_name)
- is_dynamic = False
- except ImportError:
- is_dynamic = True
-
  self.modules.add(obj)
- if is_dynamic:
- self.save_reduce(dynamic_subimport, (obj.__name__, vars(obj)), obj=obj)
+ if _is_dynamic(obj):
+ self.save_reduce(dynamic_subimport, (obj.__name__, vars(obj)),
+ obj=obj)
  else:
  self.save_reduce(subimport, (obj.__name__,), obj=obj)
 
@@ -566,7 +572,7 @@ def save_function_tuple(self, func):
  'name': func.__name__,
  'doc': func.__doc__,
  }
- if hasattr(func, '__annotations__'):
+ if hasattr(func, '__annotations__') and sys.version_info >= (3, 7):
  state['annotations'] = func.__annotations__
  if hasattr(func, '__qualname__'):
  state['qualname'] = func.__qualname__
@@ -661,6 +667,13 @@ def save_global(self, obj, name=None, pack=struct.pack):
  The name of this method is somewhat misleading: all types get
  dispatched here.
  """
+ if obj is type(None):
+ return self.save_reduce(type, (None,), obj=obj)
+ elif obj is type(Ellipsis):
+ return self.save_reduce(type, (Ellipsis,), obj=obj)
+ elif obj is type(NotImplemented):
+ return self.save_reduce(type, (NotImplemented,), obj=obj)
+
  if obj.__module__ == "__main__":
  return self.save_dynamic_class(obj)
 
@@ -933,7 +946,7 @@ def subimport(name):
 
 
 def dynamic_subimport(name, vars):
- mod = imp.new_module(name)
+ mod = types.ModuleType(name)
  mod.__dict__.update(vars)
  return mod
 
@@ -1090,12 +1103,18 @@ def _make_skel_func(code, cell_count, base_globals=None):
  if base_globals is None:
  base_globals = {}
  elif isinstance(base_globals, str):
- if sys.modules.get(base_globals, None) is not None:
- # this checks if we can import the previous environment the object
- # lived in
- base_globals = vars(sys.modules[base_globals])
- else:
- base_globals = {}
+ base_globals_name = base_globals
+ try:
+ # First try to reuse the globals from the module containing the
+ # function. If it is not possible to retrieve it, fallback to an
+ # empty dictionary.
+ base_globals = vars(importlib.import_module(base_globals))
+ except ImportError:
+ base_globals = _dynamic_modules_globals.get(
+ base_globals_name, None)
+ if base_globals is None:
+ base_globals = _DynamicModuleFuncGlobals()
+ _dynamic_modules_globals[base_globals_name] = base_globals
 
  base_globals['__builtins__'] = __builtins__
 
@@ -1125,19 +1144,31 @@ def _rehydrate_skeleton_class(skeleton_class, class_dict):
  return skeleton_class
 
 
-def _find_module(mod_name):
+def _is_dynamic(module):
  """
- Iterate over each part instead of calling imp.find_module directly.
- This function is able to find submodules (e.g. scikit.tree)
+ Return True if the module is special module that cannot be imported by its
+ name.
  """
- path = None
- for part in mod_name.split('.'):
- if path is not None:
- path = [path]
- file, path, description = imp.find_module(part, path)
- if file is not None:
- file.close()
- return path, description
+ # Quick check: module that have __file__ attribute are not dynamic modules.
+ if hasattr(module, '__file__'):
+ return False
+
+ if hasattr(module, '__spec__'):
+ return module.__spec__ is None
+ else:
+ # Backward compat for Python 2
+ import imp
+ try:
+ path = None
+ for part in module.__name__.split('.'):
+ if path is not None:
+ path = [path]
+ f, path, description = imp.find_module(part, path)
+ if f is not None:
+ f.close()
+ except ImportError:
+ return True
+ return False
 
 
 """Constructors for 3rd party libraries

sklearn/externals/joblib/externals/loky/__init__.py

@@ -9,14 +9,17 @@
 from ._base import ALL_COMPLETED, FIRST_COMPLETED, FIRST_EXCEPTION
 
 from .backend.context import cpu_count
+from .backend.reduction import set_loky_pickler
 from .reusable_executor import get_reusable_executor
+from .cloudpickle_wrapper import wrap_non_picklable_objects
 from .process_executor import BrokenProcessPool, ProcessPoolExecutor
 
 
 __all__ = ["get_reusable_executor", "cpu_count", "wait", "as_completed",
  "Future", "Executor", "ProcessPoolExecutor",
  "BrokenProcessPool", "CancelledError", "TimeoutError",
- "FIRST_COMPLETED", "FIRST_EXCEPTION", "ALL_COMPLETED", ]
+ "FIRST_COMPLETED", "FIRST_EXCEPTION", "ALL_COMPLETED",
+ "wrap_non_picklable_objects", "set_loky_pickler"]
 
 
-__version__ = '2.3.1'
+__version__ = '2.4.2'

sklearn/externals/joblib/externals/loky/backend/__init__.py

@@ -3,8 +3,6 @@
 
 from .context import get_context
 
-LOKY_PICKLER = os.environ.get("LOKY_PICKLER")
-
 if sys.version_info > (3, 4):
 
  def _make_name():

sklearn/externals/joblib/externals/loky/backend/compat.py

@@ -1,4 +1,3 @@
-# flake8: noqa
 ###############################################################################
 # Compat file to import the correct modules for each platform and python
 # version.
@@ -7,20 +6,36 @@
 #
 import sys
 
-if sys.version_info[:2] >= (3, 3):
+PY3 = sys.version_info[:2] >= (3, 3)
+
+if PY3:
  import queue
 else:
  import Queue as queue
 
-from pickle import PicklingError
-
 if sys.version_info >= (3, 4):
  from multiprocessing.process import BaseProcess
 else:
  from multiprocessing.process import Process as BaseProcess
 
 # Platform specific compat
 if sys.platform == "win32":
- from .compat_win32 import *
+ from .compat_win32 import wait
 else:
- from .compat_posix import *
+ from .compat_posix import wait
+
+
+def set_cause(exc, cause):
+ exc.__cause__ = cause
+
+ if not PY3:
+ # Preformat message here.
+ if exc.__cause__ is not None:
+ exc.args = ("{}\n\nThis was caused directly by {}".format(
+ exc.args if len(exc.args) != 1 else exc.args[0],
+ str(exc.__cause__)),)
+
+ return exc
+
+
+__all__ = ["queue", "BaseProcess", "set_cause", "wait"]

sklearn/externals/joblib/externals/loky/backend/queues.py

@@ -22,7 +22,7 @@
 from multiprocessing.queues import _sentinel, Queue as mp_Queue
 from multiprocessing.queues import SimpleQueue as mp_SimpleQueue
 
-from .reduction import CustomizableLokyPickler
+from .reduction import loads, dumps
 from .context import assert_spawning, get_context
 
 
@@ -147,8 +147,7 @@ def _feed(buffer, notempty, send_bytes, writelock, close, reducers,
  return
 
  # serialize the data before acquiring the lock
- obj_ = CustomizableLokyPickler.dumps(
- obj, reducers=reducers)
+ obj_ = dumps(obj, reducers=reducers)
  if wacquire is None:
  send_bytes(obj_)
  else:
@@ -227,12 +226,12 @@ def get(self):
  with self._rlock:
  res = self._reader.recv_bytes()
  # unserialize the data after having released the lock
- return CustomizableLokyPickler.loads(res)
+ return loads(res)
 
  # Overload put to use our customizable reducer
  def put(self, obj):
  # serialize the data before acquiring the lock
- obj = CustomizableLokyPickler.dumps(obj, reducers=self._reducers)
+ obj = dumps(obj, reducers=self._reducers)
  if self._wlock is None:
  # writes to a message oriented win32 pipe are atomic
  self._writer.send_bytes(obj)