Hello community,
here is the log from the commit of package python-joblib for openSUSE:Factory
checked in at 2019-03-26 22:34:03
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Comparing /work/SRC/openSUSE:Factory/python-joblib (Old)
and /work/SRC/openSUSE:Factory/.python-joblib.new.25356 (New)
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Package is "python-joblib"
Tue Mar 26 22:34:03 2019 rev:8 rq:688766 version:0.13.2
Changes:
--------
--- /work/SRC/openSUSE:Factory/python-joblib/python-joblib.changes
2019-03-10 09:37:09.460150633 +0100
+++ /work/SRC/openSUSE:Factory/.python-joblib.new.25356/python-joblib.changes
2019-03-26 22:34:45.941671497 +0100
@@ -1,0 +2,7 @@
+Tue Mar 26 14:45:24 UTC 2019 - Tomáš Chvátal <tchva...@suse.com>
+
+- Update to 0.13.2:
+ * Upgrade to cloudpickle 0.8.0
+ * Add a non-regression test related to joblib issues #836 and #833,
reporting that cloudpickle versions between 0.5.4 and 0.7 introduced a bug
where global variables changes in a parent process between two calls to
joblib.Parallel would not be propagated into the workers
+
+-------------------------------------------------------------------
Old:
----
joblib-0.13.1.tar.gz
New:
----
joblib-0.13.2.tar.gz
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Other differences:
------------------
++++++ python-joblib.spec ++++++
--- /var/tmp/diff_new_pack.VFlGEs/_old 2019-03-26 22:34:46.805671125 +0100
+++ /var/tmp/diff_new_pack.VFlGEs/_new 2019-03-26 22:34:46.809671123 +0100
@@ -18,7 +18,7 @@
%{?!python_module:%define python_module() python-%{**} python3-%{**}}
Name: python-joblib
-Version: 0.13.1
+Version: 0.13.2
Release: 0
Summary: Module for using Python functions as pipeline jobs
License: BSD-3-Clause
++++++ joblib-0.13.1.tar.gz -> joblib-0.13.2.tar.gz ++++++
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/joblib-0.13.1/PKG-INFO new/joblib-0.13.2/PKG-INFO
--- old/joblib-0.13.1/PKG-INFO 2019-01-11 19:50:11.000000000 +0100
+++ new/joblib-0.13.2/PKG-INFO 2019-02-13 16:39:29.000000000 +0100
@@ -1,6 +1,6 @@
Metadata-Version: 1.1
Name: joblib
-Version: 0.13.1
+Version: 0.13.2
Summary: Lightweight pipelining: using Python functions as pipeline jobs.
Home-page: https://joblib.readthedocs.io
Author: Gael Varoquaux
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/joblib-0.13.1/joblib/__init__.py
new/joblib-0.13.2/joblib/__init__.py
--- old/joblib-0.13.1/joblib/__init__.py 2019-01-11 19:49:21.000000000
+0100
+++ new/joblib-0.13.2/joblib/__init__.py 2019-02-13 16:38:58.000000000
+0100
@@ -106,7 +106,7 @@
# Dev branch marker is: 'X.Y.dev' or 'X.Y.devN' where N is an integer.
# 'X.Y.dev0' is the canonical version of 'X.Y.dev'
#
-__version__ = '0.13.1'
+__version__ = '0.13.2'
from .memory import Memory, MemorizedResult, register_store_backend
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/joblib-0.13.1/joblib/externals/cloudpickle/__init__.py
new/joblib-0.13.2/joblib/externals/cloudpickle/__init__.py
--- old/joblib-0.13.1/joblib/externals/cloudpickle/__init__.py 2018-11-02
19:49:52.000000000 +0100
+++ new/joblib-0.13.2/joblib/externals/cloudpickle/__init__.py 2019-02-13
16:38:07.000000000 +0100
@@ -2,4 +2,4 @@
from .cloudpickle import *
-__version__ = '0.6.1'
+__version__ = '0.8.0'
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/joblib-0.13.1/joblib/externals/cloudpickle/cloudpickle.py
new/joblib-0.13.2/joblib/externals/cloudpickle/cloudpickle.py
--- old/joblib-0.13.1/joblib/externals/cloudpickle/cloudpickle.py
2018-11-02 19:49:52.000000000 +0100
+++ new/joblib-0.13.2/joblib/externals/cloudpickle/cloudpickle.py
2019-02-13 16:38:07.000000000 +0100
@@ -42,21 +42,20 @@
"""
from __future__ import print_function
-import io
import dis
-import sys
-import types
+from functools import partial
+import importlib
+import io
+import itertools
+import logging
import opcode
+import operator
import pickle
import struct
-import logging
-import weakref
-import operator
-import importlib
-import itertools
+import sys
import traceback
-from functools import partial
-
+import types
+import weakref
# cloudpickle is meant for inter process communication: we expect all
# communicating processes to run the same Python version hence we favor
@@ -64,36 +63,22 @@
DEFAULT_PROTOCOL = pickle.HIGHEST_PROTOCOL
-if sys.version < '3':
+if sys.version_info[0] < 3: # pragma: no branch
from pickle import Pickler
try:
from cStringIO import StringIO
except ImportError:
from StringIO import StringIO
+ string_types = (basestring,) # noqa
PY3 = False
else:
types.ClassType = type
from pickle import _Pickler as Pickler
from io import BytesIO as StringIO
+ string_types = (str,)
PY3 = True
-# Container for the global namespace to ensure consistent unpickling of
-# functions defined in dynamic modules (modules not registed in sys.modules).
-_dynamic_modules_globals = weakref.WeakValueDictionary()
-
-
-class _DynamicModuleFuncGlobals(dict):
- """Global variables referenced by a function defined in a dynamic module
-
- To avoid leaking references we store such context in a WeakValueDictionary
- instance. However instances of python builtin types such as dict cannot
- be used directly as values in such a construct, hence the need for a
- derived class.
- """
- pass
-
-
def _make_cell_set_template_code():
"""Get the Python compiler to emit LOAD_FAST(arg); STORE_DEREF
@@ -112,7 +97,7 @@
return _stub
- _cell_set_template_code = f()
+ _cell_set_template_code = f().__code__
This function is _only_ a LOAD_FAST(arg); STORE_DEREF, but that is
invalid syntax on Python 2. If we use this function we also don't need
@@ -127,7 +112,7 @@
# NOTE: we are marking the cell variable as a free variable intentionally
# so that we simulate an inner function instead of the outer function. This
# is what gives us the ``nonlocal`` behavior in a Python 2 compatible way.
- if not PY3:
+ if not PY3: # pragma: no branch
return types.CodeType(
co.co_argcount,
co.co_nlocals,
@@ -228,14 +213,14 @@
}
-if sys.version_info < (3, 4):
+if sys.version_info < (3, 4): # pragma: no branch
def _walk_global_ops(code):
"""
Yield (opcode, argument number) tuples for all
global-referencing instructions in *code*.
"""
code = getattr(code, 'co_code', b'')
- if not PY3:
+ if not PY3: # pragma: no branch
code = map(ord, code)
n = len(code)
@@ -273,8 +258,6 @@
if protocol is None:
protocol = DEFAULT_PROTOCOL
Pickler.__init__(self, file, protocol=protocol)
- # set of modules to unpickle
- self.modules = set()
# map ids to dictionary. used to ensure that functions can share
global env
self.globals_ref = {}
@@ -294,7 +277,7 @@
dispatch[memoryview] = save_memoryview
- if not PY3:
+ if not PY3: # pragma: no branch
def save_buffer(self, obj):
self.save(str(obj))
@@ -304,7 +287,6 @@
"""
Save a module as an import
"""
- self.modules.add(obj)
if _is_dynamic(obj):
self.save_reduce(dynamic_subimport, (obj.__name__, vars(obj)),
obj=obj)
@@ -317,7 +299,7 @@
"""
Save a code object
"""
- if PY3:
+ if PY3: # pragma: no branch
args = (
obj.co_argcount, obj.co_kwonlyargcount, obj.co_nlocals,
obj.co_stacksize,
obj.co_flags, obj.co_code, obj.co_consts, obj.co_names,
obj.co_varnames,
@@ -384,7 +366,6 @@
lookedup_by_name = None
if themodule:
- self.modules.add(themodule)
if lookedup_by_name is obj:
return self.save_global(obj, name)
@@ -396,7 +377,7 @@
# So we pickle them here using save_reduce; have to do it differently
# for different python versions.
if not hasattr(obj, '__code__'):
- if PY3:
+ if PY3: # pragma: no branch
rv = obj.__reduce_ex__(self.proto)
else:
if hasattr(obj, '__self__'):
@@ -434,8 +415,31 @@
def _save_subimports(self, code, top_level_dependencies):
"""
- Ensure de-pickler imports any package child-modules that
- are needed by the function
+ Save submodules used by a function but not listed in its globals.
+
+ In the example below:
+
+ ```
+ import concurrent.futures
+ import cloudpickle
+
+
+ def func():
+ x = concurrent.futures.ThreadPoolExecutor
+
+
+ if __name__ == '__main__':
+ cloudpickle.dumps(func)
+ ```
+
+ the globals extracted by cloudpickle in the function's state include
+ the concurrent module, but not its submodule (here,
+ concurrent.futures), which is the module used by func.
+
+ To ensure that calling the depickled function does not raise an
+ AttributeError, this function looks for any currently loaded submodule
+ that the function uses and whose parent is present in the function
+ globals, and saves it before saving the function.
"""
# check if any known dependency is an imported package
@@ -481,6 +485,17 @@
# doc can't participate in a cycle with the original class.
type_kwargs = {'__doc__': clsdict.pop('__doc__', None)}
+ if hasattr(obj, "__slots__"):
+ type_kwargs['__slots__'] = obj.__slots__
+ # pickle string length optimization: member descriptors of obj are
+ # created automatically from obj's __slots__ attribute, no need to
+ # save them in obj's state
+ if isinstance(obj.__slots__, string_types):
+ clsdict.pop(obj.__slots__)
+ else:
+ for k in obj.__slots__:
+ clsdict.pop(k, None)
+
# If type overrides __dict__ as a property, include it in the type
kwargs.
# In Python 2, we can't set this attribute after construction.
__dict__ = clsdict.pop('__dict__', None)
@@ -639,17 +654,17 @@
# save the dict
dct = func.__dict__
- base_globals = self.globals_ref.get(id(func.__globals__), None)
- if base_globals is None:
- # For functions defined in a well behaved module use
- # vars(func.__module__) for base_globals. This is necessary to
- # share the global variables across multiple pickled functions from
- # this module.
- if hasattr(func, '__module__') and func.__module__ is not None:
- base_globals = func.__module__
- else:
- base_globals = {}
- self.globals_ref[id(func.__globals__)] = base_globals
+ # base_globals represents the future global namespace of func at
+ # unpickling time. Looking it up and storing it in globals_ref allow
+ # functions sharing the same globals at pickling time to also
+ # share them once unpickled, at one condition: since globals_ref is
+ # an attribute of a Cloudpickler instance, and that a new CloudPickler
is
+ # created each time pickle.dump or pickle.dumps is called, functions
+ # also need to be saved within the same invokation of
+ # cloudpickle.dump/cloudpickle.dumps (for example:
cloudpickle.dumps([f1, f2])). There
+ # is no such limitation when using Cloudpickler.dump, as long as the
+ # multiple invokations are bound to the same Cloudpickler.
+ base_globals = self.globals_ref.setdefault(id(func.__globals__), {})
return (code, f_globals, defaults, closure, dct, base_globals)
@@ -699,7 +714,7 @@
if obj.__self__ is None:
self.save_reduce(getattr, (obj.im_class, obj.__name__))
else:
- if PY3:
+ if PY3: # pragma: no branch
self.save_reduce(types.MethodType, (obj.__func__,
obj.__self__), obj=obj)
else:
self.save_reduce(types.MethodType, (obj.__func__,
obj.__self__, obj.__self__.__class__),
@@ -752,7 +767,7 @@
save(stuff)
write(pickle.BUILD)
- if not PY3:
+ if not PY3: # pragma: no branch
dispatch[types.InstanceType] = save_inst
def save_property(self, obj):
@@ -852,7 +867,7 @@
try: # Python 2
dispatch[file] = save_file
- except NameError: # Python 3
+ except NameError: # Python 3 # pragma: no branch
dispatch[io.TextIOWrapper] = save_file
dispatch[type(Ellipsis)] = save_ellipsis
@@ -873,6 +888,12 @@
dispatch[logging.RootLogger] = save_root_logger
+ if hasattr(types, "MappingProxyType"): # pragma: no branch
+ def save_mappingproxy(self, obj):
+ self.save_reduce(types.MappingProxyType, (dict(obj),), obj=obj)
+
+ dispatch[types.MappingProxyType] = save_mappingproxy
+
"""Special functions for Add-on libraries"""
def inject_addons(self):
"""Plug in system. Register additional pickling functions if modules
already loaded"""
@@ -1059,10 +1080,16 @@
else:
raise ValueError('Unexpected _fill_value arguments: %r' % (args,))
- # Only set global variables that do not exist.
- for k, v in state['globals'].items():
- if k not in func.__globals__:
- func.__globals__[k] = v
+ # - At pickling time, any dynamic global variable used by func is
+ # serialized by value (in state['globals']).
+ # - At unpickling time, func's __globals__ attribute is initialized by
+ # first retrieving an empty isolated namespace that will be shared
+ # with other functions pickled from the same original module
+ # by the same CloudPickler instance and then updated with the
+ # content of state['globals'] to populate the shared isolated
+ # namespace with all the global variables that are specifically
+ # referenced for this function.
+ func.__globals__.update(state['globals'])
func.__defaults__ = state['defaults']
func.__dict__ = state['dict']
@@ -1100,21 +1127,11 @@
code and the correct number of cells in func_closure. All other
func attributes (e.g. func_globals) are empty.
"""
- if base_globals is None:
+ # This is backward-compatibility code: for cloudpickle versions between
+ # 0.5.4 and 0.7, base_globals could be a string or None. base_globals
+ # should now always be a dictionary.
+ if base_globals is None or isinstance(base_globals, str):
base_globals = {}
- elif isinstance(base_globals, str):
- base_globals_name = base_globals
- try:
- # First try to reuse the globals from the module containing the
- # function. If it is not possible to retrieve it, fallback to an
- # empty dictionary.
- base_globals = vars(importlib.import_module(base_globals))
- except ImportError:
- base_globals = _dynamic_modules_globals.get(
- base_globals_name, None)
- if base_globals is None:
- base_globals = _DynamicModuleFuncGlobals()
- _dynamic_modules_globals[base_globals_name] = base_globals
base_globals['__builtins__'] = __builtins__
@@ -1182,7 +1199,7 @@
""" Use copy_reg to extend global pickle definitions """
-if sys.version_info < (3, 4):
+if sys.version_info < (3, 4): # pragma: no branch
method_descriptor = type(str.upper)
def _reduce_method_descriptor(obj):
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/joblib-0.13.1/joblib/test/test_parallel.py
new/joblib-0.13.2/joblib/test/test_parallel.py
--- old/joblib-0.13.1/joblib/test/test_parallel.py 2019-01-11
19:48:46.000000000 +0100
+++ new/joblib-0.13.2/joblib/test/test_parallel.py 2019-02-13
16:38:07.000000000 +0100
@@ -1537,3 +1537,31 @@
with parallel_backend(ZeroWorkerBackend()):
with pytest.raises(RuntimeError, match=expected_msg):
Parallel(n_jobs=2)(delayed(id)(i) for i in range(2))
+
+
+def test_globals_update_at_each_parallel_call():
+ # This is a non-regression test related to joblib issues #836 and #833.
+ # Cloudpickle versions between 0.5.4 and 0.7 introduced a bug where global
+ # variables changes in a parent process between two calls to
+ # joblib.Parallel would not be propagated into the workers.
+ global MY_GLOBAL_VARIABLE
+ MY_GLOBAL_VARIABLE = "original value"
+
+ def check_globals():
+ global MY_GLOBAL_VARIABLE
+ return MY_GLOBAL_VARIABLE
+
+ assert check_globals() == "original value"
+
+ workers_global_variable = Parallel(n_jobs=2)(
+ delayed(check_globals)() for i in range(2))
+ assert set(workers_global_variable) == {"original value"}
+
+ # Change the value of MY_GLOBAL_VARIABLE, and make sure this change gets
+ # propagated into the workers environment
+ MY_GLOBAL_VARIABLE = "changed value"
+ assert check_globals() == "changed value"
+
+ workers_global_variable = Parallel(n_jobs=2)(
+ delayed(check_globals)() for i in range(2))
+ assert set(workers_global_variable) == {"changed value"}
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/joblib-0.13.1/joblib.egg-info/PKG-INFO
new/joblib-0.13.2/joblib.egg-info/PKG-INFO
--- old/joblib-0.13.1/joblib.egg-info/PKG-INFO 2019-01-11 19:50:11.000000000
+0100
+++ new/joblib-0.13.2/joblib.egg-info/PKG-INFO 2019-02-13 16:39:28.000000000
+0100
@@ -1,6 +1,6 @@
Metadata-Version: 1.1
Name: joblib
-Version: 0.13.1
+Version: 0.13.2
Summary: Lightweight pipelining: using Python functions as pipeline jobs.
Home-page: https://joblib.readthedocs.io
Author: Gael Varoquaux
