[
https://issues.apache.org/jira/browse/SPARK-15679?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15308425#comment-15308425
]
nalin garg edited comment on SPARK-15679 at 5/31/16 7:35 PM:
-------------------------------------------------------------
Below is stack trace:
Traceback (most recent call last):
File "/Users/nalingarg/PycharmProjects/parallel_check/parallel_check.py",
line 90, in <module>
r.count()
File
"/Users/nalingarg/Documents/spark-1.6.1-bin-hadoop2.6/python/pyspark/rdd.py",
line 1004, in count
return self.mapPartitions(lambda i: [sum(1 for _ in i)]).sum()
File
"/Users/nalingarg/Documents/spark-1.6.1-bin-hadoop2.6/python/pyspark/rdd.py",
line 995, in sum
return self.mapPartitions(lambda x: [sum(x)]).fold(0, operator.add)
File
"/Users/nalingarg/Documents/spark-1.6.1-bin-hadoop2.6/python/pyspark/rdd.py",
line 869, in fold
vals = self.mapPartitions(func).collect()
File
"/Users/nalingarg/Documents/spark-1.6.1-bin-hadoop2.6/python/pyspark/rdd.py",
line 771, in collect
port = self.ctx._jvm.PythonRDD.collectAndServe(self._jrdd.rdd())
File
"/Users/nalingarg/Documents/spark-1.6.1-bin-hadoop2.6/python/pyspark/rdd.py",
line 2379, in _jrdd
pickled_cmd, bvars, env, includes = _prepare_for_python_RDD(self.ctx,
command, self)
File
"/Users/nalingarg/Documents/spark-1.6.1-bin-hadoop2.6/python/pyspark/rdd.py",
line 2299, in _prepare_for_python_RDD
pickled_command = ser.dumps(command)
File
"/Users/nalingarg/Documents/spark-1.6.1-bin-hadoop2.6/python/pyspark/serializers.py",
line 428, in dumps
return cloudpickle.dumps(obj, 2)
File
"/Users/nalingarg/Documents/spark-1.6.1-bin-hadoop2.6/python/pyspark/cloudpickle.py",
line 646, in dumps
cp.dump(obj)
File
"/Users/nalingarg/Documents/spark-1.6.1-bin-hadoop2.6/python/pyspark/cloudpickle.py",
line 107, in dump
return Pickler.dump(self, obj)
File
"/usr/local/Cellar/python/2.7.11/Frameworks/Python.framework/Versions/2.7/lib/python2.7/pickle.py",
line 224, in dump
self.save(obj)
File
"/usr/local/Cellar/python/2.7.11/Frameworks/Python.framework/Versions/2.7/lib/python2.7/pickle.py",
line 286, in save
f(self, obj) # Call unbound method with explicit self
File
"/usr/local/Cellar/python/2.7.11/Frameworks/Python.framework/Versions/2.7/lib/python2.7/pickle.py",
line 568, in save_tuple
save(element)
File
"/usr/local/Cellar/python/2.7.11/Frameworks/Python.framework/Versions/2.7/lib/python2.7/pickle.py",
line 286, in save
f(self, obj) # Call unbound method with explicit self
File
"/Users/nalingarg/Documents/spark-1.6.1-bin-hadoop2.6/python/pyspark/cloudpickle.py",
line 199, in save_function
self.save_function_tuple(obj)
File
"/Users/nalingarg/Documents/spark-1.6.1-bin-hadoop2.6/python/pyspark/cloudpickle.py",
line 236, in save_function_tuple
save((code, closure, base_globals))
File
"/usr/local/Cellar/python/2.7.11/Frameworks/Python.framework/Versions/2.7/lib/python2.7/pickle.py",
line 286, in save
f(self, obj) # Call unbound method with explicit self
File
"/usr/local/Cellar/python/2.7.11/Frameworks/Python.framework/Versions/2.7/lib/python2.7/pickle.py",
line 554, in save_tuple
save(element)
File
"/usr/local/Cellar/python/2.7.11/Frameworks/Python.framework/Versions/2.7/lib/python2.7/pickle.py",
line 286, in save
f(self, obj) # Call unbound method with explicit self
File
"/usr/local/Cellar/python/2.7.11/Frameworks/Python.framework/Versions/2.7/lib/python2.7/pickle.py",
line 606, in save_list
self._batch_appends(iter(obj))
File
"/usr/local/Cellar/python/2.7.11/Frameworks/Python.framework/Versions/2.7/lib/python2.7/pickle.py",
line 639, in _batch_appends
save(x)
File
"/usr/local/Cellar/python/2.7.11/Frameworks/Python.framework/Versions/2.7/lib/python2.7/pickle.py",
line 286, in save
f(self, obj) # Call unbound method with explicit self
File
"/Users/nalingarg/Documents/spark-1.6.1-bin-hadoop2.6/python/pyspark/cloudpickle.py",
line 199, in save_function
self.save_function_tuple(obj)
File
"/Users/nalingarg/Documents/spark-1.6.1-bin-hadoop2.6/python/pyspark/cloudpickle.py",
line 236, in save_function_tuple
save((code, closure, base_globals))
File
"/usr/local/Cellar/python/2.7.11/Frameworks/Python.framework/Versions/2.7/lib/python2.7/pickle.py",
line 286, in save
f(self, obj) # Call unbound method with explicit self
File
"/usr/local/Cellar/python/2.7.11/Frameworks/Python.framework/Versions/2.7/lib/python2.7/pickle.py",
line 554, in save_tuple
save(element)
File
"/usr/local/Cellar/python/2.7.11/Frameworks/Python.framework/Versions/2.7/lib/python2.7/pickle.py",
line 286, in save
f(self, obj) # Call unbound method with explicit self
File
"/usr/local/Cellar/python/2.7.11/Frameworks/Python.framework/Versions/2.7/lib/python2.7/pickle.py",
line 606, in save_list
self._batch_appends(iter(obj))
File
"/usr/local/Cellar/python/2.7.11/Frameworks/Python.framework/Versions/2.7/lib/python2.7/pickle.py",
line 639, in _batch_appends
save(x)
File
"/usr/local/Cellar/python/2.7.11/Frameworks/Python.framework/Versions/2.7/lib/python2.7/pickle.py",
line 286, in save
f(self, obj) # Call unbound method with explicit self
File
"/Users/nalingarg/Documents/spark-1.6.1-bin-hadoop2.6/python/pyspark/cloudpickle.py",
line 199, in save_function
self.save_function_tuple(obj)
File
"/Users/nalingarg/Documents/spark-1.6.1-bin-hadoop2.6/python/pyspark/cloudpickle.py",
line 236, in save_function_tuple
save((code, closure, base_globals))
File
"/usr/local/Cellar/python/2.7.11/Frameworks/Python.framework/Versions/2.7/lib/python2.7/pickle.py",
line 286, in save
f(self, obj) # Call unbound method with explicit self
File
"/usr/local/Cellar/python/2.7.11/Frameworks/Python.framework/Versions/2.7/lib/python2.7/pickle.py",
line 554, in save_tuple
save(element)
File
"/usr/local/Cellar/python/2.7.11/Frameworks/Python.framework/Versions/2.7/lib/python2.7/pickle.py",
line 286, in save
f(self, obj) # Call unbound method with explicit self
File
"/usr/local/Cellar/python/2.7.11/Frameworks/Python.framework/Versions/2.7/lib/python2.7/pickle.py",
line 606, in save_list
self._batch_appends(iter(obj))
File
"/usr/local/Cellar/python/2.7.11/Frameworks/Python.framework/Versions/2.7/lib/python2.7/pickle.py",
line 639, in _batch_appends
save(x)
File
"/usr/local/Cellar/python/2.7.11/Frameworks/Python.framework/Versions/2.7/lib/python2.7/pickle.py",
line 286, in save
f(self, obj) # Call unbound method with explicit self
File
"/Users/nalingarg/Documents/spark-1.6.1-bin-hadoop2.6/python/pyspark/cloudpickle.py",
line 199, in save_function
self.save_function_tuple(obj)
File
"/Users/nalingarg/Documents/spark-1.6.1-bin-hadoop2.6/python/pyspark/cloudpickle.py",
line 236, in save_function_tuple
save((code, closure, base_globals))
File
"/usr/local/Cellar/python/2.7.11/Frameworks/Python.framework/Versions/2.7/lib/python2.7/pickle.py",
line 286, in save
f(self, obj) # Call unbound method with explicit self
File
"/usr/local/Cellar/python/2.7.11/Frameworks/Python.framework/Versions/2.7/lib/python2.7/pickle.py",
line 554, in save_tuple
save(element)
File
"/usr/local/Cellar/python/2.7.11/Frameworks/Python.framework/Versions/2.7/lib/python2.7/pickle.py",
line 286, in save
f(self, obj) # Call unbound method with explicit self
File
"/usr/local/Cellar/python/2.7.11/Frameworks/Python.framework/Versions/2.7/lib/python2.7/pickle.py",
line 606, in save_list
self._batch_appends(iter(obj))
File
"/usr/local/Cellar/python/2.7.11/Frameworks/Python.framework/Versions/2.7/lib/python2.7/pickle.py",
line 642, in _batch_appends
save(tmp[0])
File
"/usr/local/Cellar/python/2.7.11/Frameworks/Python.framework/Versions/2.7/lib/python2.7/pickle.py",
line 286, in save
f(self, obj) # Call unbound method with explicit self
File
"/Users/nalingarg/Documents/spark-1.6.1-bin-hadoop2.6/python/pyspark/cloudpickle.py",
line 390, in save_instancemethod
obj=obj)
File
"/Users/nalingarg/Documents/spark-1.6.1-bin-hadoop2.6/python/pyspark/cloudpickle.py",
line 524, in save_reduce
save(args)
File
"/usr/local/Cellar/python/2.7.11/Frameworks/Python.framework/Versions/2.7/lib/python2.7/pickle.py",
line 286, in save
f(self, obj) # Call unbound method with explicit self
File
"/usr/local/Cellar/python/2.7.11/Frameworks/Python.framework/Versions/2.7/lib/python2.7/pickle.py",
line 554, in save_tuple
save(element)
File
"/usr/local/Cellar/python/2.7.11/Frameworks/Python.framework/Versions/2.7/lib/python2.7/pickle.py",
line 331, in save
self.save_reduce(obj=obj, *rv)
File
"/Users/nalingarg/Documents/spark-1.6.1-bin-hadoop2.6/python/pyspark/cloudpickle.py",
line 509, in save_reduce
save(cls)
File
"/usr/local/Cellar/python/2.7.11/Frameworks/Python.framework/Versions/2.7/lib/python2.7/pickle.py",
line 286, in save
f(self, obj) # Call unbound method with explicit self
File
"/Users/nalingarg/Documents/spark-1.6.1-bin-hadoop2.6/python/pyspark/cloudpickle.py",
line 374, in save_global
self.save(dd)
File
"/usr/local/Cellar/python/2.7.11/Frameworks/Python.framework/Versions/2.7/lib/python2.7/pickle.py",
line 286, in save
f(self, obj) # Call unbound method with explicit self
File
"/usr/local/Cellar/python/2.7.11/Frameworks/Python.framework/Versions/2.7/lib/python2.7/pickle.py",
line 655, in save_dict
self._batch_setitems(obj.iteritems())
File
"/usr/local/Cellar/python/2.7.11/Frameworks/Python.framework/Versions/2.7/lib/python2.7/pickle.py",
line 687, in _batch_setitems
save(v)
File
"/usr/local/Cellar/python/2.7.11/Frameworks/Python.framework/Versions/2.7/lib/python2.7/pickle.py",
line 286, in save
f(self, obj) # Call unbound method with explicit self
File
"/Users/nalingarg/Documents/spark-1.6.1-bin-hadoop2.6/python/pyspark/cloudpickle.py",
line 193, in save_function
self.save_function_tuple(obj)
File
"/Users/nalingarg/Documents/spark-1.6.1-bin-hadoop2.6/python/pyspark/cloudpickle.py",
line 241, in save_function_tuple
save(f_globals)
File
"/usr/local/Cellar/python/2.7.11/Frameworks/Python.framework/Versions/2.7/lib/python2.7/pickle.py",
line 286, in save
f(self, obj) # Call unbound method with explicit self
File
"/usr/local/Cellar/python/2.7.11/Frameworks/Python.framework/Versions/2.7/lib/python2.7/pickle.py",
line 655, in save_dict
self._batch_setitems(obj.iteritems())
File
"/usr/local/Cellar/python/2.7.11/Frameworks/Python.framework/Versions/2.7/lib/python2.7/pickle.py",
line 692, in _batch_setitems
save(v)
File
"/usr/local/Cellar/python/2.7.11/Frameworks/Python.framework/Versions/2.7/lib/python2.7/pickle.py",
line 331, in save
self.save_reduce(obj=obj, *rv)
File
"/Users/nalingarg/Documents/spark-1.6.1-bin-hadoop2.6/python/pyspark/cloudpickle.py",
line 542, in save_reduce
save(state)
File
"/usr/local/Cellar/python/2.7.11/Frameworks/Python.framework/Versions/2.7/lib/python2.7/pickle.py",
line 286, in save
f(self, obj) # Call unbound method with explicit self
File
"/usr/local/Cellar/python/2.7.11/Frameworks/Python.framework/Versions/2.7/lib/python2.7/pickle.py",
line 655, in save_dict
self._batch_setitems(obj.iteritems())
File
"/usr/local/Cellar/python/2.7.11/Frameworks/Python.framework/Versions/2.7/lib/python2.7/pickle.py",
line 687, in _batch_setitems
save(v)
File
"/usr/local/Cellar/python/2.7.11/Frameworks/Python.framework/Versions/2.7/lib/python2.7/pickle.py",
line 306, in save
rv = reduce(self.proto)
TypeError: 'JavaPackage' object is not callable
was (Author: nalingarg2):
[~sowen] below is stack trace:
Traceback (most recent call last):
File "/Users/nalingarg/PycharmProjects/parallel_check/parallel_check.py",
line 90, in <module>
r.count()
File
"/Users/nalingarg/Documents/spark-1.6.1-bin-hadoop2.6/python/pyspark/rdd.py",
line 1004, in count
return self.mapPartitions(lambda i: [sum(1 for _ in i)]).sum()
File
"/Users/nalingarg/Documents/spark-1.6.1-bin-hadoop2.6/python/pyspark/rdd.py",
line 995, in sum
return self.mapPartitions(lambda x: [sum(x)]).fold(0, operator.add)
File
"/Users/nalingarg/Documents/spark-1.6.1-bin-hadoop2.6/python/pyspark/rdd.py",
line 869, in fold
vals = self.mapPartitions(func).collect()
File
"/Users/nalingarg/Documents/spark-1.6.1-bin-hadoop2.6/python/pyspark/rdd.py",
line 771, in collect
port = self.ctx._jvm.PythonRDD.collectAndServe(self._jrdd.rdd())
File
"/Users/nalingarg/Documents/spark-1.6.1-bin-hadoop2.6/python/pyspark/rdd.py",
line 2379, in _jrdd
pickled_cmd, bvars, env, includes = _prepare_for_python_RDD(self.ctx,
command, self)
File
"/Users/nalingarg/Documents/spark-1.6.1-bin-hadoop2.6/python/pyspark/rdd.py",
line 2299, in _prepare_for_python_RDD
pickled_command = ser.dumps(command)
File
"/Users/nalingarg/Documents/spark-1.6.1-bin-hadoop2.6/python/pyspark/serializers.py",
line 428, in dumps
return cloudpickle.dumps(obj, 2)
File
"/Users/nalingarg/Documents/spark-1.6.1-bin-hadoop2.6/python/pyspark/cloudpickle.py",
line 646, in dumps
cp.dump(obj)
File
"/Users/nalingarg/Documents/spark-1.6.1-bin-hadoop2.6/python/pyspark/cloudpickle.py",
line 107, in dump
return Pickler.dump(self, obj)
File
"/usr/local/Cellar/python/2.7.11/Frameworks/Python.framework/Versions/2.7/lib/python2.7/pickle.py",
line 224, in dump
self.save(obj)
File
"/usr/local/Cellar/python/2.7.11/Frameworks/Python.framework/Versions/2.7/lib/python2.7/pickle.py",
line 286, in save
f(self, obj) # Call unbound method with explicit self
File
"/usr/local/Cellar/python/2.7.11/Frameworks/Python.framework/Versions/2.7/lib/python2.7/pickle.py",
line 568, in save_tuple
save(element)
File
"/usr/local/Cellar/python/2.7.11/Frameworks/Python.framework/Versions/2.7/lib/python2.7/pickle.py",
line 286, in save
f(self, obj) # Call unbound method with explicit self
File
"/Users/nalingarg/Documents/spark-1.6.1-bin-hadoop2.6/python/pyspark/cloudpickle.py",
line 199, in save_function
self.save_function_tuple(obj)
File
"/Users/nalingarg/Documents/spark-1.6.1-bin-hadoop2.6/python/pyspark/cloudpickle.py",
line 236, in save_function_tuple
save((code, closure, base_globals))
File
"/usr/local/Cellar/python/2.7.11/Frameworks/Python.framework/Versions/2.7/lib/python2.7/pickle.py",
line 286, in save
f(self, obj) # Call unbound method with explicit self
File
"/usr/local/Cellar/python/2.7.11/Frameworks/Python.framework/Versions/2.7/lib/python2.7/pickle.py",
line 554, in save_tuple
save(element)
File
"/usr/local/Cellar/python/2.7.11/Frameworks/Python.framework/Versions/2.7/lib/python2.7/pickle.py",
line 286, in save
f(self, obj) # Call unbound method with explicit self
File
"/usr/local/Cellar/python/2.7.11/Frameworks/Python.framework/Versions/2.7/lib/python2.7/pickle.py",
line 606, in save_list
self._batch_appends(iter(obj))
File
"/usr/local/Cellar/python/2.7.11/Frameworks/Python.framework/Versions/2.7/lib/python2.7/pickle.py",
line 639, in _batch_appends
save(x)
File
"/usr/local/Cellar/python/2.7.11/Frameworks/Python.framework/Versions/2.7/lib/python2.7/pickle.py",
line 286, in save
f(self, obj) # Call unbound method with explicit self
File
"/Users/nalingarg/Documents/spark-1.6.1-bin-hadoop2.6/python/pyspark/cloudpickle.py",
line 199, in save_function
self.save_function_tuple(obj)
File
"/Users/nalingarg/Documents/spark-1.6.1-bin-hadoop2.6/python/pyspark/cloudpickle.py",
line 236, in save_function_tuple
save((code, closure, base_globals))
File
"/usr/local/Cellar/python/2.7.11/Frameworks/Python.framework/Versions/2.7/lib/python2.7/pickle.py",
line 286, in save
f(self, obj) # Call unbound method with explicit self
File
"/usr/local/Cellar/python/2.7.11/Frameworks/Python.framework/Versions/2.7/lib/python2.7/pickle.py",
line 554, in save_tuple
save(element)
File
"/usr/local/Cellar/python/2.7.11/Frameworks/Python.framework/Versions/2.7/lib/python2.7/pickle.py",
line 286, in save
f(self, obj) # Call unbound method with explicit self
File
"/usr/local/Cellar/python/2.7.11/Frameworks/Python.framework/Versions/2.7/lib/python2.7/pickle.py",
line 606, in save_list
self._batch_appends(iter(obj))
File
"/usr/local/Cellar/python/2.7.11/Frameworks/Python.framework/Versions/2.7/lib/python2.7/pickle.py",
line 639, in _batch_appends
save(x)
File
"/usr/local/Cellar/python/2.7.11/Frameworks/Python.framework/Versions/2.7/lib/python2.7/pickle.py",
line 286, in save
f(self, obj) # Call unbound method with explicit self
File
"/Users/nalingarg/Documents/spark-1.6.1-bin-hadoop2.6/python/pyspark/cloudpickle.py",
line 199, in save_function
self.save_function_tuple(obj)
File
"/Users/nalingarg/Documents/spark-1.6.1-bin-hadoop2.6/python/pyspark/cloudpickle.py",
line 236, in save_function_tuple
save((code, closure, base_globals))
File
"/usr/local/Cellar/python/2.7.11/Frameworks/Python.framework/Versions/2.7/lib/python2.7/pickle.py",
line 286, in save
f(self, obj) # Call unbound method with explicit self
File
"/usr/local/Cellar/python/2.7.11/Frameworks/Python.framework/Versions/2.7/lib/python2.7/pickle.py",
line 554, in save_tuple
save(element)
File
"/usr/local/Cellar/python/2.7.11/Frameworks/Python.framework/Versions/2.7/lib/python2.7/pickle.py",
line 286, in save
f(self, obj) # Call unbound method with explicit self
File
"/usr/local/Cellar/python/2.7.11/Frameworks/Python.framework/Versions/2.7/lib/python2.7/pickle.py",
line 606, in save_list
self._batch_appends(iter(obj))
File
"/usr/local/Cellar/python/2.7.11/Frameworks/Python.framework/Versions/2.7/lib/python2.7/pickle.py",
line 639, in _batch_appends
save(x)
File
"/usr/local/Cellar/python/2.7.11/Frameworks/Python.framework/Versions/2.7/lib/python2.7/pickle.py",
line 286, in save
f(self, obj) # Call unbound method with explicit self
File
"/Users/nalingarg/Documents/spark-1.6.1-bin-hadoop2.6/python/pyspark/cloudpickle.py",
line 199, in save_function
self.save_function_tuple(obj)
File
"/Users/nalingarg/Documents/spark-1.6.1-bin-hadoop2.6/python/pyspark/cloudpickle.py",
line 236, in save_function_tuple
save((code, closure, base_globals))
File
"/usr/local/Cellar/python/2.7.11/Frameworks/Python.framework/Versions/2.7/lib/python2.7/pickle.py",
line 286, in save
f(self, obj) # Call unbound method with explicit self
File
"/usr/local/Cellar/python/2.7.11/Frameworks/Python.framework/Versions/2.7/lib/python2.7/pickle.py",
line 554, in save_tuple
save(element)
File
"/usr/local/Cellar/python/2.7.11/Frameworks/Python.framework/Versions/2.7/lib/python2.7/pickle.py",
line 286, in save
f(self, obj) # Call unbound method with explicit self
File
"/usr/local/Cellar/python/2.7.11/Frameworks/Python.framework/Versions/2.7/lib/python2.7/pickle.py",
line 606, in save_list
self._batch_appends(iter(obj))
File
"/usr/local/Cellar/python/2.7.11/Frameworks/Python.framework/Versions/2.7/lib/python2.7/pickle.py",
line 642, in _batch_appends
save(tmp[0])
File
"/usr/local/Cellar/python/2.7.11/Frameworks/Python.framework/Versions/2.7/lib/python2.7/pickle.py",
line 286, in save
f(self, obj) # Call unbound method with explicit self
File
"/Users/nalingarg/Documents/spark-1.6.1-bin-hadoop2.6/python/pyspark/cloudpickle.py",
line 390, in save_instancemethod
obj=obj)
File
"/Users/nalingarg/Documents/spark-1.6.1-bin-hadoop2.6/python/pyspark/cloudpickle.py",
line 524, in save_reduce
save(args)
File
"/usr/local/Cellar/python/2.7.11/Frameworks/Python.framework/Versions/2.7/lib/python2.7/pickle.py",
line 286, in save
f(self, obj) # Call unbound method with explicit self
File
"/usr/local/Cellar/python/2.7.11/Frameworks/Python.framework/Versions/2.7/lib/python2.7/pickle.py",
line 554, in save_tuple
save(element)
File
"/usr/local/Cellar/python/2.7.11/Frameworks/Python.framework/Versions/2.7/lib/python2.7/pickle.py",
line 331, in save
self.save_reduce(obj=obj, *rv)
File
"/Users/nalingarg/Documents/spark-1.6.1-bin-hadoop2.6/python/pyspark/cloudpickle.py",
line 509, in save_reduce
save(cls)
File
"/usr/local/Cellar/python/2.7.11/Frameworks/Python.framework/Versions/2.7/lib/python2.7/pickle.py",
line 286, in save
f(self, obj) # Call unbound method with explicit self
File
"/Users/nalingarg/Documents/spark-1.6.1-bin-hadoop2.6/python/pyspark/cloudpickle.py",
line 374, in save_global
self.save(dd)
File
"/usr/local/Cellar/python/2.7.11/Frameworks/Python.framework/Versions/2.7/lib/python2.7/pickle.py",
line 286, in save
f(self, obj) # Call unbound method with explicit self
File
"/usr/local/Cellar/python/2.7.11/Frameworks/Python.framework/Versions/2.7/lib/python2.7/pickle.py",
line 655, in save_dict
self._batch_setitems(obj.iteritems())
File
"/usr/local/Cellar/python/2.7.11/Frameworks/Python.framework/Versions/2.7/lib/python2.7/pickle.py",
line 687, in _batch_setitems
save(v)
File
"/usr/local/Cellar/python/2.7.11/Frameworks/Python.framework/Versions/2.7/lib/python2.7/pickle.py",
line 286, in save
f(self, obj) # Call unbound method with explicit self
File
"/Users/nalingarg/Documents/spark-1.6.1-bin-hadoop2.6/python/pyspark/cloudpickle.py",
line 193, in save_function
self.save_function_tuple(obj)
File
"/Users/nalingarg/Documents/spark-1.6.1-bin-hadoop2.6/python/pyspark/cloudpickle.py",
line 241, in save_function_tuple
save(f_globals)
File
"/usr/local/Cellar/python/2.7.11/Frameworks/Python.framework/Versions/2.7/lib/python2.7/pickle.py",
line 286, in save
f(self, obj) # Call unbound method with explicit self
File
"/usr/local/Cellar/python/2.7.11/Frameworks/Python.framework/Versions/2.7/lib/python2.7/pickle.py",
line 655, in save_dict
self._batch_setitems(obj.iteritems())
File
"/usr/local/Cellar/python/2.7.11/Frameworks/Python.framework/Versions/2.7/lib/python2.7/pickle.py",
line 692, in _batch_setitems
save(v)
File
"/usr/local/Cellar/python/2.7.11/Frameworks/Python.framework/Versions/2.7/lib/python2.7/pickle.py",
line 331, in save
self.save_reduce(obj=obj, *rv)
File
"/Users/nalingarg/Documents/spark-1.6.1-bin-hadoop2.6/python/pyspark/cloudpickle.py",
line 542, in save_reduce
save(state)
File
"/usr/local/Cellar/python/2.7.11/Frameworks/Python.framework/Versions/2.7/lib/python2.7/pickle.py",
line 286, in save
f(self, obj) # Call unbound method with explicit self
File
"/usr/local/Cellar/python/2.7.11/Frameworks/Python.framework/Versions/2.7/lib/python2.7/pickle.py",
line 655, in save_dict
self._batch_setitems(obj.iteritems())
File
"/usr/local/Cellar/python/2.7.11/Frameworks/Python.framework/Versions/2.7/lib/python2.7/pickle.py",
line 687, in _batch_setitems
save(v)
File
"/usr/local/Cellar/python/2.7.11/Frameworks/Python.framework/Versions/2.7/lib/python2.7/pickle.py",
line 306, in save
rv = reduce(self.proto)
TypeError: 'JavaPackage' object is not callable
> Passing functions do not work in classes
> ----------------------------------------
>
> Key: SPARK-15679
> URL: https://issues.apache.org/jira/browse/SPARK-15679
> Project: Spark
> Issue Type: Bug
> Reporter: nalin garg
>
> {code}
> from pyspark import SparkContext
> from pyspark.sql import SQLContext, HiveContext
> import csv, io, StringIO
> from pyspark.sql.functions import *
> from pyspark.sql import Row
> from pyspark.sql import *
> from pyspark.sql import functions as F
> from pyspark.sql.functions import asc, desc
> sc = SparkContext("local", "Summary Report")
> sqlContext = SQLContext(sc)
> class ParallelBuild(object):
> def myFunc(self, s):
> l = s.split(',')
> print l[0], l[1]
> return l[0]
> def list_to_csv_str(x):
> output = StringIO.StringIO("")
> csv.writer(output).writerow(x)
> return output.getvalue().strip()
> def run(self):
> data = [(1,'a'), (1,'b'), (1,'c'), (2,'d'), (3,'r'), (4,'a'),
> (2,'t'), (3,'y'), (1,'f')]
> df = sqlContext.createDataFrame(data, schema= ['uid', 'address_uid'])
> return df
> if __name__ == "__main__":
> df_rdd = ParallelBuild().run().map(lambda line: line).persist()
> r = df_rdd.map(ParallelBuild().myFunc)
> r.count()
> {code}
> Above code returns JavaPackage not callable exception.
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscr...@spark.apache.org
For additional commands, e-mail: issues-h...@spark.apache.org
