[ https://issues.apache.org/jira/browse/ARROW-855?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16002278#comment-16002278 ]
xufusheng edited comment on ARROW-855 at 5/9/17 8:41 AM: --------------------------------------------------------- Hello,[~wesmckinn] [~julienledem],when elementSize = 12800,memory cannot be release. {color:red} environment: scala:scala-sdk-2.10.4 jdk:1.7.0_79 arrow:arrow-0.1.0 the following is the test code: {color} import java.util.concurrent.ConcurrentHashMap import org.apache.arrow.memory.{RootAllocator, BufferAllocator} import org.apache.arrow.vector.NullableVarBinaryVector import org.scalatest.{BeforeAndAfter, FunSuite} import scala.util.Random class ArrowLeakPureSuite extends FunSuite with BeforeAndAfter { val root: BufferAllocator = new RootAllocator(1024 * 1024 * 1024 * 8L) val elementCount = 1024 val elementSize = 12800 // when 25600, all is ok. val binaGenerator = new BinaryGenerator(elementSize) def pushVector(allocator: BufferAllocator, name: String, length: Int): NullableVarBinaryVector = { val vector = new NullableVarBinaryVector(name, allocator) vector.allocateNew() val mutator = vector.getMutator (0 to length -1).foreach(i =>{ val value = binaGenerator.next() mutator.setSafe(i, value, 0, value.length) }) vector } test("leak") { val vectorCount = 100 val entries = new ConcurrentHashMap[String, NullableVarBinaryVector] (0 to vectorCount).foreach(i => { val vector = pushVector(root,s"vector-$i", elementCount) entries.put(s"vector-$i", vector) }) // Push Finished. println("Push Finished.") Thread.sleep( 10 * 1000) (0 to vectorCount).foreach(i => { val vector = entries.get(s"vector-$i") vector.close() }) {color:red} // Close Finished, now the data in Memory should be released, However it didn't when elementSize = 12800. {color} println("Close Finished.") Thread.sleep( 20 * 1000) println("Test Finished.") } trait Generator[T] { def next(): T def nextStr(): String } abstract class GeneratorWithRandom[T] extends Generator[T] { val random = new Random() def nextStr() = next().toString } class BinaryGenerator(val length: Int) extends GeneratorWithRandom[Array[Byte]] { val binary = new Array[Byte](length) def next() = { random.nextBytes(binary) binary } override def nextStr() = { new String(binary) } } } was (Author: fushengxu): [~wesmckinn] [~julienledem] {color:red} environment: scala:scala-sdk-2.10.4 jdk:1.7.0_79 arrow:arrow-0.1.0 the following is the test code: {color} import java.util.concurrent.ConcurrentHashMap import org.apache.arrow.memory.{RootAllocator, BufferAllocator} import org.apache.arrow.vector.NullableVarBinaryVector import org.scalatest.{BeforeAndAfter, FunSuite} import scala.util.Random class ArrowLeakPureSuite extends FunSuite with BeforeAndAfter { val root: BufferAllocator = new RootAllocator(1024 * 1024 * 1024 * 8L) val elementCount = 1024 val elementSize = 12800 // when 25600, all is ok. val binaGenerator = new BinaryGenerator(elementSize) def pushVector(allocator: BufferAllocator, name: String, length: Int): NullableVarBinaryVector = { val vector = new NullableVarBinaryVector(name, allocator) vector.allocateNew() val mutator = vector.getMutator (0 to length -1).foreach(i =>{ val value = binaGenerator.next() mutator.setSafe(i, value, 0, value.length) }) vector } test("leak") { val vectorCount = 100 val entries = new ConcurrentHashMap[String, NullableVarBinaryVector] (0 to vectorCount).foreach(i => { val vector = pushVector(root,s"vector-$i", elementCount) entries.put(s"vector-$i", vector) }) // Push Finished. println("Push Finished.") Thread.sleep( 10 * 1000) (0 to vectorCount).foreach(i => { val vector = entries.get(s"vector-$i") vector.close() }) {color:red} // Close Finished, now the data in Memory should be released, However it didn't when elementSize = 12800. {color} println("Close Finished.") Thread.sleep( 20 * 1000) println("Test Finished.") } trait Generator[T] { def next(): T def nextStr(): String } abstract class GeneratorWithRandom[T] extends Generator[T] { val random = new Random() def nextStr() = next().toString } class BinaryGenerator(val length: Int) extends GeneratorWithRandom[Array[Byte]] { val binary = new Array[Byte](length) def next() = { random.nextBytes(binary) binary } override def nextStr() = { new String(binary) } } } > Arrow Memory Leak > ----------------- > > Key: ARROW-855 > URL: https://issues.apache.org/jira/browse/ARROW-855 > Project: Apache Arrow > Issue Type: Bug > Components: Java - Memory, Java - Vectors > Affects Versions: 0.1.0 > Environment: CentOS release 6.7+Indellij IDEA > Reporter: xufusheng > Priority: Critical > Labels: test > > we create a memory table by arrow and the source data come from HBase. > Create a memory table and then drop the table,there will be a memory leak. > Hundreds of times,There will be OutOfMemoryError. > anyone encounter similar problems? -- This message was sent by Atlassian JIRA (v6.3.15#6346)