Github user BryanCutler commented on a diff in the pull request:
https://github.com/apache/spark/pull/20373#discussion_r164602283
--- Diff: python/pyspark/cloudpickle.py ---
@@ -608,37 +626,22 @@ def save_global(self, obj, name=None,
pack=struct.pack):
The name of this method is somewhat misleading: all types get
dispatched here.
"""
- if obj.__module__ == "__builtin__" or obj.__module__ == "builtins":
- if obj in _BUILTIN_TYPE_NAMES:
- return self.save_reduce(_builtin_type,
(_BUILTIN_TYPE_NAMES[obj],), obj=obj)
-
- if name is None:
- name = obj.__name__
-
- modname = getattr(obj, "__module__", None)
- if modname is None:
- try:
- # whichmodule() could fail, see
- #
https://bitbucket.org/gutworth/six/issues/63/importing-six-breaks-pickling
- modname = pickle.whichmodule(obj, name)
- except Exception:
- modname = '__main__'
+ if obj.__module__ == "__main__":
+ return self.save_dynamic_class(obj)
--- End diff --
[MRG] Regression on pickling classes from the main module
cloudpipe/cloudpickle#132
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
