(amoro) branch master updated: Fixed the memory display error in Spark optimizer resource allocation.

[klion26]

This is an automated email from the ASF dual-hosted git repository.

klion26 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/amoro.git


The following commit(s) were added to refs/heads/master by this push:
     new 4dda58a5e Fixed the memory display error in Spark optimizer resource 
allocation.
4dda58a5e is described below

commit 4dda58a5efae7728a78ce5e2c509c74ff05fcce5
Author: Darcy <linting...@lilith.com>
AuthorDate: Mon Jul 21 16:05:18 2025 +0800

    Fixed the memory display error in Spark optimizer resource allocation.
---
 .../main/java/org/apache/amoro/optimizer/spark/SparkOptimizer.java   | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git 
a/amoro-optimizer/amoro-optimizer-spark/src/main/java/org/apache/amoro/optimizer/spark/SparkOptimizer.java
 
b/amoro-optimizer/amoro-optimizer-spark/src/main/java/org/apache/amoro/optimizer/spark/SparkOptimizer.java
index d8c8c91a1..2eb38c028 100644
--- 
a/amoro-optimizer/amoro-optimizer-spark/src/main/java/org/apache/amoro/optimizer/spark/SparkOptimizer.java
+++ 
b/amoro-optimizer/amoro-optimizer-spark/src/main/java/org/apache/amoro/optimizer/spark/SparkOptimizer.java
@@ -56,7 +56,10 @@ public class SparkOptimizer extends Optimizer {
     // calculate optimizer memory allocation
     int driverMemory = 
Utils.memoryStringToMb(jsc.getConf().get("spark.driver.memory", "1g"));
     int executorMemory = 
Utils.memoryStringToMb(jsc.getConf().get("spark.executor.memory", "1g"));
-    config.setMemorySize(driverMemory + config.getExecutionParallel() * 
executorMemory);
+    int executorCores = jsc.getConf().getInt("spark.executor.cores", 1);
+    int executionParallel = config.getExecutionParallel();
+    int executorNum = (int) Math.ceil((double) executionParallel / 
executorCores);
+    config.setMemorySize(driverMemory + executorNum * executorMemory);
 
     SparkOptimizer optimizer = new SparkOptimizer(config, jsc);
     OptimizerToucher toucher = optimizer.getToucher();