pan3793 commented on code in PR #57332:
URL: https://github.com/apache/spark/pull/57332#discussion_r3627895611
##########
sql/core/src/main/scala/org/apache/spark/sql/execution/BaseScriptTransformationExec.scala:
##########
@@ -85,8 +85,10 @@ trait BaseScriptTransformationExec extends UnaryExecNode {
SparkFiles.getRootDirectory()
builder.environment().put("PATH", path)
// if OMP_NUM_THREADS is not explicitly set, override it with the value of
"spark.task.cpus"
+ // which may be fractional, so round up to an integer (at least 1) of
threads.
if (System.getenv("OMP_NUM_THREADS") == null) {
- builder.environment().put("OMP_NUM_THREADS",
conf.getConfString("spark.task.cpus", "1"))
+ val taskCpus = conf.getConfString("spark.task.cpus", "1.0").toDouble
Review Comment:
updated, reverted as it's a pre-existing issue, fix is non-trivial
##########
sql/core/src/main/scala/org/apache/spark/sql/execution/BaseScriptTransformationExec.scala:
##########
@@ -85,8 +85,10 @@ trait BaseScriptTransformationExec extends UnaryExecNode {
SparkFiles.getRootDirectory()
builder.environment().put("PATH", path)
// if OMP_NUM_THREADS is not explicitly set, override it with the value of
"spark.task.cpus"
+ // which may be fractional, so round up to an integer (at least 1) of
threads.
if (System.getenv("OMP_NUM_THREADS") == null) {
- builder.environment().put("OMP_NUM_THREADS",
conf.getConfString("spark.task.cpus", "1"))
+ val taskCpus = conf.getConfString("spark.task.cpus", "1.0").toDouble
Review Comment:
update: reverted as it's a pre-existing issue, fix is non-trivial
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
