This is an automated email from the ASF dual-hosted git repository.
hongze pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new cee1f3bd8 [GLUTEN-5414] [VL] Support Arrow native memory pool usage
track (#5550)
cee1f3bd8 is described below
commit cee1f3bd869340c156feb0cdaf0a867f32343d54
Author: Jin Chengcheng <[email protected]>
AuthorDate: Wed May 8 11:35:25 2024 +0800
[GLUTEN-5414] [VL] Support Arrow native memory pool usage track (#5550)
---
.../gluten/execution/RowToVeloxColumnarExec.scala | 2 +-
.../python/ColumnarArrowEvalPythonExec.scala | 2 +-
.../org/apache/gluten/utils/DatasourceUtil.scala | 2 +-
.../execution/ColumnarCachedBatchSerializer.scala | 2 +-
.../execution/VeloxColumnarWriteFilesExec.scala | 2 +-
.../velox/VeloxFormatWriterInjects.scala | 2 +-
.../VeloxCelebornColumnarBatchSerializer.scala | 2 +-
gluten-data/pom.xml | 28 ++++++++
.../gluten/columnarbatch/ColumnarBatches.java | 2 +-
.../alloc}/ArrowBufferAllocators.java | 2 +-
.../alloc}/ManagedAllocationListener.java | 2 +-
.../memory/arrow/pool/ArrowNativeMemoryPool.java | 75 ++++++++++++++++++++++
.../arrow/pool/ArrowReservationListener.java} | 33 +++++-----
.../vectorized/ArrowWritableColumnVector.java | 2 +-
.../gluten/vectorized/ColumnarBatchInIterator.java | 2 +-
.../org/apache/gluten/utils/ImplicitClass.scala | 2 +-
.../vectorized/ColumnarBatchSerializer.scala | 2 +-
.../sql/execution/ColumnarBuildSideRelation.scala | 2 +-
.../spark/sql/execution/utils/ExecUtil.scala | 2 +-
.../apache/spark/sql/utils/SparkVectorUtil.scala | 2 +-
20 files changed, 138 insertions(+), 32 deletions(-)
diff --git
a/backends-velox/src/main/scala/org/apache/gluten/execution/RowToVeloxColumnarExec.scala
b/backends-velox/src/main/scala/org/apache/gluten/execution/RowToVeloxColumnarExec.scala
index 22ca020d1..be1bc64e2 100644
---
a/backends-velox/src/main/scala/org/apache/gluten/execution/RowToVeloxColumnarExec.scala
+++
b/backends-velox/src/main/scala/org/apache/gluten/execution/RowToVeloxColumnarExec.scala
@@ -20,7 +20,7 @@ import org.apache.gluten.backendsapi.BackendsApiManager
import org.apache.gluten.columnarbatch.ColumnarBatches
import org.apache.gluten.exception.GlutenException
import org.apache.gluten.exec.Runtimes
-import org.apache.gluten.memory.arrowalloc.ArrowBufferAllocators
+import org.apache.gluten.memory.arrow.alloc.ArrowBufferAllocators
import org.apache.gluten.memory.nmm.NativeMemoryManagers
import org.apache.gluten.utils.{ArrowAbiUtil, Iterators}
import org.apache.gluten.vectorized._
diff --git
a/backends-velox/src/main/scala/org/apache/gluten/execution/python/ColumnarArrowEvalPythonExec.scala
b/backends-velox/src/main/scala/org/apache/gluten/execution/python/ColumnarArrowEvalPythonExec.scala
index f2beef6ca..77ef1c642 100644
---
a/backends-velox/src/main/scala/org/apache/gluten/execution/python/ColumnarArrowEvalPythonExec.scala
+++
b/backends-velox/src/main/scala/org/apache/gluten/execution/python/ColumnarArrowEvalPythonExec.scala
@@ -18,7 +18,7 @@ package org.apache.spark.api.python
import org.apache.gluten.columnarbatch.ColumnarBatches
import org.apache.gluten.extension.GlutenPlan
-import org.apache.gluten.memory.arrowalloc.ArrowBufferAllocators
+import org.apache.gluten.memory.arrow.alloc.ArrowBufferAllocators
import org.apache.gluten.utils.Iterators
import org.apache.gluten.vectorized.ArrowWritableColumnVector
diff --git
a/backends-velox/src/main/scala/org/apache/gluten/utils/DatasourceUtil.scala
b/backends-velox/src/main/scala/org/apache/gluten/utils/DatasourceUtil.scala
index f61cdb8e9..6150507b4 100644
--- a/backends-velox/src/main/scala/org/apache/gluten/utils/DatasourceUtil.scala
+++ b/backends-velox/src/main/scala/org/apache/gluten/utils/DatasourceUtil.scala
@@ -17,7 +17,7 @@
package org.apache.gluten.utils
import org.apache.gluten.datasource.DatasourceJniWrapper
-import org.apache.gluten.memory.arrowalloc.ArrowBufferAllocators
+import org.apache.gluten.memory.arrow.alloc.ArrowBufferAllocators
import org.apache.gluten.memory.nmm.NativeMemoryManagers
import org.apache.spark.sql.types.StructType
diff --git
a/backends-velox/src/main/scala/org/apache/spark/sql/execution/ColumnarCachedBatchSerializer.scala
b/backends-velox/src/main/scala/org/apache/spark/sql/execution/ColumnarCachedBatchSerializer.scala
index 88678cb5e..7385c53d6 100644
---
a/backends-velox/src/main/scala/org/apache/spark/sql/execution/ColumnarCachedBatchSerializer.scala
+++
b/backends-velox/src/main/scala/org/apache/spark/sql/execution/ColumnarCachedBatchSerializer.scala
@@ -21,7 +21,7 @@ import org.apache.gluten.backendsapi.BackendsApiManager
import org.apache.gluten.columnarbatch.ColumnarBatches
import org.apache.gluten.exec.Runtimes
import org.apache.gluten.execution.{RowToVeloxColumnarExec,
VeloxColumnarToRowExec}
-import org.apache.gluten.memory.arrowalloc.ArrowBufferAllocators
+import org.apache.gluten.memory.arrow.alloc.ArrowBufferAllocators
import org.apache.gluten.memory.nmm.NativeMemoryManagers
import org.apache.gluten.utils.{ArrowAbiUtil, Iterators}
import org.apache.gluten.vectorized.ColumnarBatchSerializerJniWrapper
diff --git
a/backends-velox/src/main/scala/org/apache/spark/sql/execution/VeloxColumnarWriteFilesExec.scala
b/backends-velox/src/main/scala/org/apache/spark/sql/execution/VeloxColumnarWriteFilesExec.scala
index 26d249f90..23dff990c 100644
---
a/backends-velox/src/main/scala/org/apache/spark/sql/execution/VeloxColumnarWriteFilesExec.scala
+++
b/backends-velox/src/main/scala/org/apache/spark/sql/execution/VeloxColumnarWriteFilesExec.scala
@@ -20,7 +20,7 @@ import org.apache.gluten.backendsapi.BackendsApiManager
import org.apache.gluten.columnarbatch.ColumnarBatches
import org.apache.gluten.exception.GlutenException
import org.apache.gluten.extension.GlutenPlan
-import org.apache.gluten.memory.arrowalloc.ArrowBufferAllocators
+import org.apache.gluten.memory.arrow.alloc.ArrowBufferAllocators
import org.apache.gluten.sql.shims.SparkShimLoader
import org.apache.spark.{Partition, SparkException, TaskContext,
TaskOutputFileAlreadyExistException}
diff --git
a/backends-velox/src/main/scala/org/apache/spark/sql/execution/datasources/velox/VeloxFormatWriterInjects.scala
b/backends-velox/src/main/scala/org/apache/spark/sql/execution/datasources/velox/VeloxFormatWriterInjects.scala
index 97ebb932a..c358d6372 100644
---
a/backends-velox/src/main/scala/org/apache/spark/sql/execution/datasources/velox/VeloxFormatWriterInjects.scala
+++
b/backends-velox/src/main/scala/org/apache/spark/sql/execution/datasources/velox/VeloxFormatWriterInjects.scala
@@ -20,7 +20,7 @@ import org.apache.gluten.columnarbatch.ColumnarBatches
import org.apache.gluten.datasource.DatasourceJniWrapper
import org.apache.gluten.exception.GlutenException
import org.apache.gluten.execution.datasource.GlutenRowSplitter
-import org.apache.gluten.memory.arrowalloc.ArrowBufferAllocators
+import org.apache.gluten.memory.arrow.alloc.ArrowBufferAllocators
import org.apache.gluten.memory.nmm.NativeMemoryManagers
import org.apache.gluten.utils.{ArrowAbiUtil, DatasourceUtil}
diff --git
a/gluten-celeborn/velox/src/main/scala/org/apache/spark/shuffle/VeloxCelebornColumnarBatchSerializer.scala
b/gluten-celeborn/velox/src/main/scala/org/apache/spark/shuffle/VeloxCelebornColumnarBatchSerializer.scala
index c5bd88748..d72977f59 100644
---
a/gluten-celeborn/velox/src/main/scala/org/apache/spark/shuffle/VeloxCelebornColumnarBatchSerializer.scala
+++
b/gluten-celeborn/velox/src/main/scala/org/apache/spark/shuffle/VeloxCelebornColumnarBatchSerializer.scala
@@ -18,7 +18,7 @@ package org.apache.spark.shuffle
import org.apache.gluten.GlutenConfig
import org.apache.gluten.exec.Runtimes
-import org.apache.gluten.memory.arrowalloc.ArrowBufferAllocators
+import org.apache.gluten.memory.arrow.alloc.ArrowBufferAllocators
import org.apache.gluten.memory.nmm.NativeMemoryManagers
import org.apache.gluten.utils.ArrowAbiUtil
import org.apache.gluten.vectorized._
diff --git a/gluten-data/pom.xml b/gluten-data/pom.xml
index 951f53ee8..db617112f 100644
--- a/gluten-data/pom.xml
+++ b/gluten-data/pom.xml
@@ -137,6 +137,34 @@
</exclusions>
<scope>compile</scope>
</dependency>
+ <dependency>
+ <groupId>org.apache.arrow</groupId>
+ <artifactId>arrow-dataset</artifactId>
+ <version>${arrow.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-common</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-buffer</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-core</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-annotations</artifactId>
+ </exclusion>
+ <exclusion>
+ <artifactId>protobuf-java</artifactId>
+ <groupId>com.google.protobuf</groupId>
+ </exclusion>
+ </exclusions>
+ <scope>compile</scope>
+ </dependency>
</dependencies>
</project>
diff --git
a/gluten-data/src/main/java/org/apache/gluten/columnarbatch/ColumnarBatches.java
b/gluten-data/src/main/java/org/apache/gluten/columnarbatch/ColumnarBatches.java
index 85ef875d6..624428dcb 100644
---
a/gluten-data/src/main/java/org/apache/gluten/columnarbatch/ColumnarBatches.java
+++
b/gluten-data/src/main/java/org/apache/gluten/columnarbatch/ColumnarBatches.java
@@ -19,7 +19,7 @@ package org.apache.gluten.columnarbatch;
import org.apache.gluten.exception.GlutenException;
import org.apache.gluten.exec.Runtime;
import org.apache.gluten.exec.Runtimes;
-import org.apache.gluten.memory.arrowalloc.ArrowBufferAllocators;
+import org.apache.gluten.memory.arrow.alloc.ArrowBufferAllocators;
import org.apache.gluten.memory.nmm.NativeMemoryManager;
import org.apache.gluten.utils.ArrowAbiUtil;
import org.apache.gluten.utils.ArrowUtil;
diff --git
a/gluten-data/src/main/java/org/apache/gluten/memory/arrowalloc/ArrowBufferAllocators.java
b/gluten-data/src/main/java/org/apache/gluten/memory/arrow/alloc/ArrowBufferAllocators.java
similarity index 98%
rename from
gluten-data/src/main/java/org/apache/gluten/memory/arrowalloc/ArrowBufferAllocators.java
rename to
gluten-data/src/main/java/org/apache/gluten/memory/arrow/alloc/ArrowBufferAllocators.java
index fcac26d6e..efee20e48 100644
---
a/gluten-data/src/main/java/org/apache/gluten/memory/arrowalloc/ArrowBufferAllocators.java
+++
b/gluten-data/src/main/java/org/apache/gluten/memory/arrow/alloc/ArrowBufferAllocators.java
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.gluten.memory.arrowalloc;
+package org.apache.gluten.memory.arrow.alloc;
import org.apache.gluten.memory.memtarget.MemoryTargets;
diff --git
a/gluten-data/src/main/java/org/apache/gluten/memory/arrowalloc/ManagedAllocationListener.java
b/gluten-data/src/main/java/org/apache/gluten/memory/arrow/alloc/ManagedAllocationListener.java
similarity index 98%
rename from
gluten-data/src/main/java/org/apache/gluten/memory/arrowalloc/ManagedAllocationListener.java
rename to
gluten-data/src/main/java/org/apache/gluten/memory/arrow/alloc/ManagedAllocationListener.java
index ded2af60b..a76c0aabe 100644
---
a/gluten-data/src/main/java/org/apache/gluten/memory/arrowalloc/ManagedAllocationListener.java
+++
b/gluten-data/src/main/java/org/apache/gluten/memory/arrow/alloc/ManagedAllocationListener.java
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.gluten.memory.arrowalloc;
+package org.apache.gluten.memory.arrow.alloc;
import org.apache.gluten.GlutenConfig;
import org.apache.gluten.memory.SimpleMemoryUsageRecorder;
diff --git
a/gluten-data/src/main/java/org/apache/gluten/memory/arrow/pool/ArrowNativeMemoryPool.java
b/gluten-data/src/main/java/org/apache/gluten/memory/arrow/pool/ArrowNativeMemoryPool.java
new file mode 100644
index 000000000..04a6e0002
--- /dev/null
+++
b/gluten-data/src/main/java/org/apache/gluten/memory/arrow/pool/ArrowNativeMemoryPool.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.gluten.memory.arrow.pool;
+
+import org.apache.arrow.dataset.jni.NativeMemoryPool;
+import org.apache.spark.util.TaskResource;
+import org.apache.spark.util.TaskResources;
+import org.apache.spark.util.Utils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class ArrowNativeMemoryPool implements TaskResource {
+ private static final Logger LOGGER =
LoggerFactory.getLogger(ArrowNativeMemoryPool.class);
+
+ private final NativeMemoryPool arrowPool;
+ private final ArrowReservationListener listener;
+
+ public ArrowNativeMemoryPool() {
+ listener = new ArrowReservationListener(TaskResources.getSharedUsage());
+ arrowPool = NativeMemoryPool.createListenable(listener);
+ }
+
+ public static NativeMemoryPool arrowPool(String name) {
+ if (!TaskResources.inSparkTask()) {
+ throw new IllegalStateException("This method must be called in a Spark
task.");
+ }
+ String id = "ArrowNativeMemoryPool:" + name;
+ return TaskResources.addResourceIfNotRegistered(id, () ->
createArrowNativeMemoryPool(name))
+ .getArrowPool();
+ }
+
+ private static ArrowNativeMemoryPool createArrowNativeMemoryPool(String
name) {
+ return new ArrowNativeMemoryPool();
+ }
+
+ @Override
+ public void release() throws Exception {
+ if (arrowPool.getBytesAllocated() != 0) {
+ LOGGER.warn(
+ String.format(
+ "Arrow pool still reserved non-zero bytes, "
+ + "which may cause memory leak, size: %s. ",
+ Utils.bytesToString(arrowPool.getBytesAllocated())));
+ }
+ arrowPool.close();
+ }
+
+ @Override
+ public int priority() {
+ return 0;
+ }
+
+ @Override
+ public String resourceName() {
+ return "arrow_mem";
+ }
+
+ public NativeMemoryPool getArrowPool() {
+ return arrowPool;
+ }
+}
diff --git
a/gluten-data/src/main/scala/org/apache/gluten/utils/ImplicitClass.scala
b/gluten-data/src/main/java/org/apache/gluten/memory/arrow/pool/ArrowReservationListener.java
similarity index 56%
copy from gluten-data/src/main/scala/org/apache/gluten/utils/ImplicitClass.scala
copy to
gluten-data/src/main/java/org/apache/gluten/memory/arrow/pool/ArrowReservationListener.java
index 1a5ae4277..2e1a25445 100644
--- a/gluten-data/src/main/scala/org/apache/gluten/utils/ImplicitClass.scala
+++
b/gluten-data/src/main/java/org/apache/gluten/memory/arrow/pool/ArrowReservationListener.java
@@ -14,25 +14,28 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.gluten.utils
+package org.apache.gluten.memory.arrow.pool;
-import org.apache.gluten.columnarbatch.ColumnarBatches
-import org.apache.gluten.memory.arrowalloc.ArrowBufferAllocators
-import org.apache.gluten.vectorized.ArrowWritableColumnVector
+import org.apache.gluten.memory.SimpleMemoryUsageRecorder;
-import org.apache.spark.sql.vectorized.ColumnarBatch
+public class ArrowReservationListener implements
org.apache.arrow.dataset.jni.ReservationListener {
+ private final SimpleMemoryUsageRecorder sharedUsage; // shared task metrics
-object ImplicitClass {
+ public ArrowReservationListener(SimpleMemoryUsageRecorder recorder) {
+ this.sharedUsage = recorder;
+ }
+
+ @Override
+ public void reserve(long size) {
+ synchronized (this) {
+ sharedUsage.inc(size);
+ }
+ }
- implicit class ArrowColumnarBatchRetainer(val cb: ColumnarBatch) {
- def retain(): Unit = {
- (0 until cb.numCols).toList.foreach(
- i =>
- ColumnarBatches
- .ensureLoaded(ArrowBufferAllocators.contextInstance(), cb)
- .column(i)
- .asInstanceOf[ArrowWritableColumnVector]
- .retain())
+ @Override
+ public void unreserve(long size) {
+ synchronized (this) {
+ sharedUsage.inc(-size);
}
}
}
diff --git
a/gluten-data/src/main/java/org/apache/gluten/vectorized/ArrowWritableColumnVector.java
b/gluten-data/src/main/java/org/apache/gluten/vectorized/ArrowWritableColumnVector.java
index 4603cbc41..dfd570deb 100644
---
a/gluten-data/src/main/java/org/apache/gluten/vectorized/ArrowWritableColumnVector.java
+++
b/gluten-data/src/main/java/org/apache/gluten/vectorized/ArrowWritableColumnVector.java
@@ -16,7 +16,7 @@
*/
package org.apache.gluten.vectorized;
-import org.apache.gluten.memory.arrowalloc.ArrowBufferAllocators;
+import org.apache.gluten.memory.arrow.alloc.ArrowBufferAllocators;
import org.apache.arrow.memory.BufferAllocator;
import org.apache.arrow.vector.BigIntVector;
diff --git
a/gluten-data/src/main/java/org/apache/gluten/vectorized/ColumnarBatchInIterator.java
b/gluten-data/src/main/java/org/apache/gluten/vectorized/ColumnarBatchInIterator.java
index 85b24166e..bd89f62a1 100644
---
a/gluten-data/src/main/java/org/apache/gluten/vectorized/ColumnarBatchInIterator.java
+++
b/gluten-data/src/main/java/org/apache/gluten/vectorized/ColumnarBatchInIterator.java
@@ -18,7 +18,7 @@ package org.apache.gluten.vectorized;
import org.apache.gluten.columnarbatch.ColumnarBatchJniWrapper;
import org.apache.gluten.columnarbatch.ColumnarBatches;
-import org.apache.gluten.memory.arrowalloc.ArrowBufferAllocators;
+import org.apache.gluten.memory.arrow.alloc.ArrowBufferAllocators;
import org.apache.spark.sql.vectorized.ColumnarBatch;
diff --git
a/gluten-data/src/main/scala/org/apache/gluten/utils/ImplicitClass.scala
b/gluten-data/src/main/scala/org/apache/gluten/utils/ImplicitClass.scala
index 1a5ae4277..4ffb3ab5c 100644
--- a/gluten-data/src/main/scala/org/apache/gluten/utils/ImplicitClass.scala
+++ b/gluten-data/src/main/scala/org/apache/gluten/utils/ImplicitClass.scala
@@ -17,7 +17,7 @@
package org.apache.gluten.utils
import org.apache.gluten.columnarbatch.ColumnarBatches
-import org.apache.gluten.memory.arrowalloc.ArrowBufferAllocators
+import org.apache.gluten.memory.arrow.alloc.ArrowBufferAllocators
import org.apache.gluten.vectorized.ArrowWritableColumnVector
import org.apache.spark.sql.vectorized.ColumnarBatch
diff --git
a/gluten-data/src/main/scala/org/apache/gluten/vectorized/ColumnarBatchSerializer.scala
b/gluten-data/src/main/scala/org/apache/gluten/vectorized/ColumnarBatchSerializer.scala
index 9fbdb3637..e632700e3 100644
---
a/gluten-data/src/main/scala/org/apache/gluten/vectorized/ColumnarBatchSerializer.scala
+++
b/gluten-data/src/main/scala/org/apache/gluten/vectorized/ColumnarBatchSerializer.scala
@@ -18,7 +18,7 @@ package org.apache.gluten.vectorized
import org.apache.gluten.GlutenConfig
import org.apache.gluten.exec.Runtimes
-import org.apache.gluten.memory.arrowalloc.ArrowBufferAllocators
+import org.apache.gluten.memory.arrow.alloc.ArrowBufferAllocators
import org.apache.gluten.memory.nmm.NativeMemoryManagers
import org.apache.gluten.utils.ArrowAbiUtil
diff --git
a/gluten-data/src/main/scala/org/apache/spark/sql/execution/ColumnarBuildSideRelation.scala
b/gluten-data/src/main/scala/org/apache/spark/sql/execution/ColumnarBuildSideRelation.scala
index ac94bfa89..9d9f5ab17 100644
---
a/gluten-data/src/main/scala/org/apache/spark/sql/execution/ColumnarBuildSideRelation.scala
+++
b/gluten-data/src/main/scala/org/apache/spark/sql/execution/ColumnarBuildSideRelation.scala
@@ -18,7 +18,7 @@ package org.apache.spark.sql.execution
import org.apache.gluten.columnarbatch.ColumnarBatches
import org.apache.gluten.exec.Runtimes
-import org.apache.gluten.memory.arrowalloc.ArrowBufferAllocators
+import org.apache.gluten.memory.arrow.alloc.ArrowBufferAllocators
import org.apache.gluten.memory.nmm.NativeMemoryManagers
import org.apache.gluten.sql.shims.SparkShimLoader
import org.apache.gluten.utils.{ArrowAbiUtil, Iterators}
diff --git
a/gluten-data/src/main/scala/org/apache/spark/sql/execution/utils/ExecUtil.scala
b/gluten-data/src/main/scala/org/apache/spark/sql/execution/utils/ExecUtil.scala
index fd0cf45c8..d3e7b4096 100644
---
a/gluten-data/src/main/scala/org/apache/spark/sql/execution/utils/ExecUtil.scala
+++
b/gluten-data/src/main/scala/org/apache/spark/sql/execution/utils/ExecUtil.scala
@@ -17,7 +17,7 @@
package org.apache.spark.sql.execution.utils
import org.apache.gluten.columnarbatch.ColumnarBatches
-import org.apache.gluten.memory.arrowalloc.ArrowBufferAllocators
+import org.apache.gluten.memory.arrow.alloc.ArrowBufferAllocators
import org.apache.gluten.memory.nmm.NativeMemoryManagers
import org.apache.gluten.utils.Iterators
import org.apache.gluten.vectorized.{ArrowWritableColumnVector,
NativeColumnarToRowInfo, NativeColumnarToRowJniWrapper, NativePartitioning}
diff --git
a/gluten-data/src/main/scala/org/apache/spark/sql/utils/SparkVectorUtil.scala
b/gluten-data/src/main/scala/org/apache/spark/sql/utils/SparkVectorUtil.scala
index 208afd294..3e86be79a 100644
---
a/gluten-data/src/main/scala/org/apache/spark/sql/utils/SparkVectorUtil.scala
+++
b/gluten-data/src/main/scala/org/apache/spark/sql/utils/SparkVectorUtil.scala
@@ -17,7 +17,7 @@
package org.apache.spark.sql.utils
import org.apache.gluten.columnarbatch.ColumnarBatches
-import org.apache.gluten.memory.arrowalloc.ArrowBufferAllocators
+import org.apache.gluten.memory.arrow.alloc.ArrowBufferAllocators
import org.apache.gluten.vectorized.ArrowWritableColumnVector
import org.apache.spark.sql.vectorized.ColumnarBatch
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
