[ https://issues.apache.org/jira/browse/SPARK-36031?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]
Yikun Jiang updated SPARK-36031: -------------------------------- Description: There are many operations for series doen't follow the pandas, such as: ========================================================= *>>> pser = pd.Series([1, 2, np.nan], dtype=float)* *>>> psser = ps.from_pandas(pser)* *>>> pser.astype(int)* Traceback (most recent call last): File "/Users/jiangyikun/venv36/lib/python3.6/site-packages/IPython/core/interactiveshell.py", line 3343, in run_code exec(code_obj, self.user_global_ns, self.user_ns) File "<ipython-input-30-1ca2ff8756d2>", line 1, in <module> pser.astype(int) File "/Users/jiangyikun/venv36/lib/python3.6/site-packages/pandas/core/generic.py", line 5548, in astype new_data = self._mgr.astype(dtype=dtype, copy=copy, errors=errors,) File "/Users/jiangyikun/venv36/lib/python3.6/site-packages/pandas/core/internals/managers.py", line 604, in astype return self.apply("astype", dtype=dtype, copy=copy, errors=errors) File "/Users/jiangyikun/venv36/lib/python3.6/site-packages/pandas/core/internals/managers.py", line 409, in apply applied = getattr(b, f)(**kwargs) File "/Users/jiangyikun/venv36/lib/python3.6/site-packages/pandas/core/internals/blocks.py", line 595, in astype values = astype_nansafe(vals1d, dtype, copy=True) File "/Users/jiangyikun/venv36/lib/python3.6/site-packages/pandas/core/dtypes/cast.py", line 968, in astype_nansafe raise ValueError("Cannot convert non-finite values (NA or inf) to integer") ValueError: Cannot convert non-finite values (NA or inf) to integer *>>> psser.astype(int)* 0 1.0 1 2.0 2 NaN dtype: float64 ========================================================= *>>> pser = pd.Series([1, 2, np.nan], dtype=float)* *>>> psser = ps.from_pandas(pser)* *>>> pser ** False* 0 1.0 1 1.0 2 1.0 dtype: float64 *>>> psser ** False* 0 1.0 1 1.0 2 NaN dtype: float64 ========================================================= *>>> pser = pd.Series([decimal.Decimal(1.0), decimal.Decimal(2.0), decimal.Decimal(np.nan)])* >>> *psser = ps.from_pandas(pser)* *>>> pser + 1* 0 2 1 3 2 NaN *>>> psser + 1* *Driver stacktrace:* at org.apache.spark.scheduler.DAGScheduler.failJobAndIndependentStages(DAGScheduler.scala:2259) at org.apache.spark.scheduler.DAGScheduler.$anonfun$abortStage$2(DAGScheduler.scala:2208) at org.apache.spark.scheduler.DAGScheduler.$anonfun$abortStage$2$adapted(DAGScheduler.scala:2207) at scala.collection.mutable.ResizableArray.foreach(ResizableArray.scala:62) at scala.collection.mutable.ResizableArray.foreach$(ResizableArray.scala:55) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:49) at org.apache.spark.scheduler.DAGScheduler.abortStage(DAGScheduler.scala:2207) at org.apache.spark.scheduler.DAGScheduler.$anonfun$handleTaskSetFailed$1(DAGScheduler.scala:1084) at org.apache.spark.scheduler.DAGScheduler.$anonfun$handleTaskSetFailed$1$adapted(DAGScheduler.scala:1084) at scala.Option.foreach(Option.scala:407) at org.apache.spark.scheduler.DAGScheduler.handleTaskSetFailed(DAGScheduler.scala:1084) at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.doOnReceive(DAGScheduler.scala:2446) at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:2388) at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:2377) at org.apache.spark.util.EventLoop$$anon$1.run(EventLoop.scala:49) at org.apache.spark.scheduler.DAGScheduler.runJob(DAGScheduler.scala:873) at org.apache.spark.SparkContext.runJob(SparkContext.scala:2208) at org.apache.spark.SparkContext.runJob(SparkContext.scala:2303) at org.apache.spark.sql.Dataset.$anonfun$collectAsArrowToPython$5(Dataset.scala:3648) at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23) at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1437) at org.apache.spark.sql.Dataset.$anonfun$collectAsArrowToPython$2(Dataset.scala:3652) at org.apache.spark.sql.Dataset.$anonfun$collectAsArrowToPython$2$adapted(Dataset.scala:3629) at org.apache.spark.sql.Dataset.$anonfun$withAction$1(Dataset.scala:3706) at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$5(SQLExecution.scala:103) at org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(SQLExecution.scala:163) at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$1(SQLExecution.scala:90) at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:774) at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:64) at org.apache.spark.sql.Dataset.withAction(Dataset.scala:3704) at org.apache.spark.sql.Dataset.$anonfun$collectAsArrowToPython$1(Dataset.scala:3629) at org.apache.spark.sql.Dataset.$anonfun$collectAsArrowToPython$1$adapted(Dataset.scala:3628) at org.apache.spark.security.SocketAuthServer$.$anonfun$serveToStream$2(SocketAuthServer.scala:139) at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23) at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1437) at org.apache.spark.security.SocketAuthServer$.$anonfun$serveToStream$1(SocketAuthServer.scala:141) at org.apache.spark.security.SocketAuthServer$.$anonfun$serveToStream$1$adapted(SocketAuthServer.scala:136) at org.apache.spark.security.SocketFuncServer.handleConnection(SocketAuthServer.scala:113) at org.apache.spark.security.SocketFuncServer.handleConnection(SocketAuthServer.scala:107) at org.apache.spark.security.SocketAuthServer$$anon$1.$anonfun$run$4(SocketAuthServer.scala:68) at scala.util.Try$.apply(Try.scala:213) at org.apache.spark.security.SocketAuthServer$$anon$1.run(SocketAuthServer.scala:68) *Caused by: java.lang.NullPointerException* at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source) at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43) at org.apache.spark.sql.execution.WholeStageCodegenExec$$anon$1.hasNext(WholeStageCodegenExec.scala:759) at scala.collection.Iterator$SliceIterator.hasNext(Iterator.scala:266) at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:458) at org.apache.spark.shuffle.sort.BypassMergeSortShuffleWriter.write(BypassMergeSortShuffleWriter.java:131) at org.apache.spark.shuffle.ShuffleWriteProcessor.write(ShuffleWriteProcessor.scala:60) at org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:99) at org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:52) at org.apache.spark.scheduler.Task.run(Task.scala:131) at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$3(Executor.scala:498) at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1437) at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:501) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) was: There are many operations for series doen't follow the pandas, such as: ========================================================= *>>> pser = pd.Series([1, 2, np.nan], dtype=float)* *>>> psser = ps.from_pandas(pser)* *>>> pser.astype(int)* Traceback (most recent call last): File "/Users/jiangyikun/venv36/lib/python3.6/site-packages/IPython/core/interactiveshell.py", line 3343, in run_code exec(code_obj, self.user_global_ns, self.user_ns) File "<ipython-input-30-1ca2ff8756d2>", line 1, in <module> pser.astype(int) File "/Users/jiangyikun/venv36/lib/python3.6/site-packages/pandas/core/generic.py", line 5548, in astype new_data = self._mgr.astype(dtype=dtype, copy=copy, errors=errors,) File "/Users/jiangyikun/venv36/lib/python3.6/site-packages/pandas/core/internals/managers.py", line 604, in astype return self.apply("astype", dtype=dtype, copy=copy, errors=errors) File "/Users/jiangyikun/venv36/lib/python3.6/site-packages/pandas/core/internals/managers.py", line 409, in apply applied = getattr(b, f)(**kwargs) File "/Users/jiangyikun/venv36/lib/python3.6/site-packages/pandas/core/internals/blocks.py", line 595, in astype values = astype_nansafe(vals1d, dtype, copy=True) File "/Users/jiangyikun/venv36/lib/python3.6/site-packages/pandas/core/dtypes/cast.py", line 968, in astype_nansafe raise ValueError("Cannot convert non-finite values (NA or inf) to integer") ValueError: Cannot convert non-finite values (NA or inf) to integer *>>> psser.astype(int)* 0 1.0 1 2.0 2 NaN dtype: float64 ========================================================= *>>> pser = pd.Series([1, 2, np.nan], dtype=float)* *>>> psser = ps.from_pandas(pser)* *>>> pser ** False* pser ** False Out[6]: 0 1.0 1 1.0 2 1.0 dtype: float64 *>>> psser ** False* 0 1.0 1 1.0 2 NaN dtype: float64 > Keep same behavior with pandas for operations of series with nan > ----------------------------------------------------------------- > > Key: SPARK-36031 > URL: https://issues.apache.org/jira/browse/SPARK-36031 > Project: Spark > Issue Type: Sub-task > Components: PySpark > Affects Versions: 3.2.0, 3.3.0 > Reporter: Yikun Jiang > Priority: Major > > There are many operations for series doen't follow the pandas, such as: > ========================================================= > *>>> pser = pd.Series([1, 2, np.nan], dtype=float)* > *>>> psser = ps.from_pandas(pser)* > *>>> pser.astype(int)* > Traceback (most recent call last): > File > "/Users/jiangyikun/venv36/lib/python3.6/site-packages/IPython/core/interactiveshell.py", > line 3343, in run_code > exec(code_obj, self.user_global_ns, self.user_ns) > File "<ipython-input-30-1ca2ff8756d2>", line 1, in <module> > pser.astype(int) > File > "/Users/jiangyikun/venv36/lib/python3.6/site-packages/pandas/core/generic.py", > line 5548, in astype > new_data = self._mgr.astype(dtype=dtype, copy=copy, errors=errors,) > File > "/Users/jiangyikun/venv36/lib/python3.6/site-packages/pandas/core/internals/managers.py", > line 604, in astype > return self.apply("astype", dtype=dtype, copy=copy, errors=errors) > File > "/Users/jiangyikun/venv36/lib/python3.6/site-packages/pandas/core/internals/managers.py", > line 409, in apply > applied = getattr(b, f)(**kwargs) > File > "/Users/jiangyikun/venv36/lib/python3.6/site-packages/pandas/core/internals/blocks.py", > line 595, in astype > values = astype_nansafe(vals1d, dtype, copy=True) > File > "/Users/jiangyikun/venv36/lib/python3.6/site-packages/pandas/core/dtypes/cast.py", > line 968, in astype_nansafe > raise ValueError("Cannot convert non-finite values (NA or inf) to integer") > ValueError: Cannot convert non-finite values (NA or inf) to integer > *>>> psser.astype(int)* > 0 1.0 > 1 2.0 > 2 NaN > dtype: float64 > ========================================================= > *>>> pser = pd.Series([1, 2, np.nan], dtype=float)* > *>>> psser = ps.from_pandas(pser)* > *>>> pser ** False* > 0 1.0 > 1 1.0 > 2 1.0 > dtype: float64 > *>>> psser ** False* > 0 1.0 > 1 1.0 > 2 NaN > dtype: float64 > > ========================================================= > *>>> pser = pd.Series([decimal.Decimal(1.0), decimal.Decimal(2.0), > decimal.Decimal(np.nan)])* > >>> *psser = ps.from_pandas(pser)* > *>>> pser + 1* > 0 2 > 1 3 > 2 NaN > *>>> psser + 1* > *Driver stacktrace:* > at > org.apache.spark.scheduler.DAGScheduler.failJobAndIndependentStages(DAGScheduler.scala:2259) > at > org.apache.spark.scheduler.DAGScheduler.$anonfun$abortStage$2(DAGScheduler.scala:2208) > at > org.apache.spark.scheduler.DAGScheduler.$anonfun$abortStage$2$adapted(DAGScheduler.scala:2207) > at scala.collection.mutable.ResizableArray.foreach(ResizableArray.scala:62) > at scala.collection.mutable.ResizableArray.foreach$(ResizableArray.scala:55) > at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:49) > at > org.apache.spark.scheduler.DAGScheduler.abortStage(DAGScheduler.scala:2207) > at > org.apache.spark.scheduler.DAGScheduler.$anonfun$handleTaskSetFailed$1(DAGScheduler.scala:1084) > at > org.apache.spark.scheduler.DAGScheduler.$anonfun$handleTaskSetFailed$1$adapted(DAGScheduler.scala:1084) > at scala.Option.foreach(Option.scala:407) > at > org.apache.spark.scheduler.DAGScheduler.handleTaskSetFailed(DAGScheduler.scala:1084) > at > org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.doOnReceive(DAGScheduler.scala:2446) > at > org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:2388) > at > org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:2377) > at org.apache.spark.util.EventLoop$$anon$1.run(EventLoop.scala:49) > at org.apache.spark.scheduler.DAGScheduler.runJob(DAGScheduler.scala:873) > at org.apache.spark.SparkContext.runJob(SparkContext.scala:2208) > at org.apache.spark.SparkContext.runJob(SparkContext.scala:2303) > at > org.apache.spark.sql.Dataset.$anonfun$collectAsArrowToPython$5(Dataset.scala:3648) > at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23) > at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1437) > at > org.apache.spark.sql.Dataset.$anonfun$collectAsArrowToPython$2(Dataset.scala:3652) > at > org.apache.spark.sql.Dataset.$anonfun$collectAsArrowToPython$2$adapted(Dataset.scala:3629) > at org.apache.spark.sql.Dataset.$anonfun$withAction$1(Dataset.scala:3706) > at > org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$5(SQLExecution.scala:103) > at > org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(SQLExecution.scala:163) > at > org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$1(SQLExecution.scala:90) > at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:774) > at > org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:64) > at org.apache.spark.sql.Dataset.withAction(Dataset.scala:3704) > at > org.apache.spark.sql.Dataset.$anonfun$collectAsArrowToPython$1(Dataset.scala:3629) > at > org.apache.spark.sql.Dataset.$anonfun$collectAsArrowToPython$1$adapted(Dataset.scala:3628) > at > org.apache.spark.security.SocketAuthServer$.$anonfun$serveToStream$2(SocketAuthServer.scala:139) > at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23) > at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1437) > at > org.apache.spark.security.SocketAuthServer$.$anonfun$serveToStream$1(SocketAuthServer.scala:141) > at > org.apache.spark.security.SocketAuthServer$.$anonfun$serveToStream$1$adapted(SocketAuthServer.scala:136) > at > org.apache.spark.security.SocketFuncServer.handleConnection(SocketAuthServer.scala:113) > at > org.apache.spark.security.SocketFuncServer.handleConnection(SocketAuthServer.scala:107) > at > org.apache.spark.security.SocketAuthServer$$anon$1.$anonfun$run$4(SocketAuthServer.scala:68) > at scala.util.Try$.apply(Try.scala:213) > at > org.apache.spark.security.SocketAuthServer$$anon$1.run(SocketAuthServer.scala:68) > *Caused by: java.lang.NullPointerException* > at > org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown > Source) > at > org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43) > at > org.apache.spark.sql.execution.WholeStageCodegenExec$$anon$1.hasNext(WholeStageCodegenExec.scala:759) > at scala.collection.Iterator$SliceIterator.hasNext(Iterator.scala:266) > at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:458) > at > org.apache.spark.shuffle.sort.BypassMergeSortShuffleWriter.write(BypassMergeSortShuffleWriter.java:131) > at > org.apache.spark.shuffle.ShuffleWriteProcessor.write(ShuffleWriteProcessor.scala:60) > at org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:99) > at org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:52) > at org.apache.spark.scheduler.Task.run(Task.scala:131) > at > org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$3(Executor.scala:498) > at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1437) > at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:501) > at > java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) > at > java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) > at java.lang.Thread.run(Thread.java:748) > -- This message was sent by Atlassian Jira (v8.3.4#803005) --------------------------------------------------------------------- To unsubscribe, e-mail: issues-unsubscr...@spark.apache.org For additional commands, e-mail: issues-h...@spark.apache.org