[ https://issues.apache.org/jira/browse/HDFS-723?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]
Hairong Kuang updated HDFS-723: ------------------------------- Description: WhiIe was running some append-related tests, I hit this deadlock: {noformat} Found one Java-level deadlock: ============================= "Thread-3": waiting to lock monitor 0x000000012ee044f0 (object 0x0000000107a0ded0, a org.apache.hadoop.hdfs.DFSClient$DFSOutputStream), which is held by "main" "main": waiting to lock monitor 0x000000012eeb71a8 (object 0x00000001082b0748, a org.apache.hadoop.hdfs.DFSClient$LeaseChecker), which is held by "Thread-3" Java stack information for the threads listed above: =================================================== "Thread-3": at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.close(DFSClient.java:3582) - waiting to lock <0x0000000107a0ded0> (a org.apache.hadoop.hdfs.DFSClient$DFSOutputStream) at org.apache.hadoop.hdfs.DFSClient$LeaseChecker.close(DFSClient.java:1175) - locked <0x00000001082b0748> (a org.apache.hadoop.hdfs.DFSClient$LeaseChecker) at org.apache.hadoop.hdfs.DFSClient.close(DFSClient.java:306) - locked <0x000000010824d640> (a org.apache.hadoop.hdfs.DFSClient) - waiting to lock <0x0000000107a0ded0> (a org.apache.hadoop.hdfs.DFSClient$DFSOutputStream) at org.apache.hadoop.hdfs.DFSClient$LeaseChecker.close(DFSClient.java:1175) - locked <0x00000001082b0748> (a org.apache.hadoop.hdfs.DFSClient$LeaseChecker) at org.apache.hadoop.hdfs.DFSClient.close(DFSClient.java:306) - locked <0x000000010824d640> (a org.apache.hadoop.hdfs.DFSClient) at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:325) at org.apache.hadoop.fs.FileSystem$Cache.closeAll(FileSystem.java:1835) - locked <0x0000000107a77ec8> (a org.apache.hadoop.fs.FileSystem$Cache) at org.apache.hadoop.fs.FileSystem$Cache$ClientFinalizer.run(FileSystem.java:1851) - locked <0x00000001079daa00> (a org.apache.hadoop.fs.FileSystem$Cache$ClientFinalizer) "main": at org.apache.hadoop.hdfs.DFSClient$LeaseChecker.remove(DFSClient.java:1151) - waiting to lock <0x00000001082b0748> (a org.apache.hadoop.hdfs.DFSClient$LeaseChecker) at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.close(DFSClient.java:3609) - locked <0x0000000107a0ded0> (a org.apache.hadoop.hdfs.DFSClient$DFSOutputStream) at org.apache.hadoop.fs.FSDataOutputStream$PositionCache.close(FSDataOutputStream.java:61) at org.apache.hadoop.fs.FSDataOutputStream.close(FSDataOutputStream.java:86) at org.apache.hadoop.hdfs.TestFileAppend4.testAppend(TestFileAppend4.java:99) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at junit.framework.TestCase.runTest(TestCase.java:168) at junit.framework.TestCase.runBare(TestCase.java:134) at junit.framework.TestResult$1.protect(TestResult.java:110) at junit.framework.TestResult.runProtected(TestResult.java:128) at junit.framework.TestResult.run(TestResult.java:113) at junit.framework.TestCase.run(TestCase.java:124) at junit.framework.TestSuite.runTest(TestSuite.java:232) at junit.framework.TestSuite.run(TestSuite.java:227) at junit.extensions.TestDecorator.basicRun(TestDecorator.java:24) at junit.extensions.TestSetup$1.protect(TestSetup.java:23) at junit.framework.TestResult.runProtected(TestResult.java:128) at junit.extensions.TestSetup.run(TestSetup.java:27) at org.apache.tools.ant.taskdefs.optional.junit.JUnitTestRunner.run(JUnitTestRunner.java:420) at org.apache.tools.ant.taskdefs.optional.junit.JUnitTestRunner.launch(JUnitTestRunner.java:911) at org.apache.tools.ant.taskdefs.optional.junit.JUnitTestRunner.main(JUnitTestRunner.java:768) {noformat} was: WhiIe was running some append-related tests, I hit this deadlock: Found one Java-level deadlock: ============================= "Thread-3": waiting to lock monitor 0x000000012ee044f0 (object 0x0000000107a0ded0, a org.apache.hadoop.hdfs.DFSClient$DFSOutputStream), which is held by "main" "main": waiting to lock monitor 0x000000012eeb71a8 (object 0x00000001082b0748, a org.apache.hadoop.hdfs.DFSClient$LeaseChecker), which is held by "Thread-3" Java stack information for the threads listed above: =================================================== "Thread-3": at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.close(DFSClient.java:3582) - waiting to lock <0x0000000107a0ded0> (a org.apache.hadoop.hdfs.DFSClient$DFSOutputStream) at org.apache.hadoop.hdfs.DFSClient$LeaseChecker.close(DFSClient.java:1175) - locked <0x00000001082b0748> (a org.apache.hadoop.hdfs.DFSClient$LeaseChecker) at org.apache.hadoop.hdfs.DFSClient.close(DFSClient.java:306) - locked <0x000000010824d640> (a org.apache.hadoop.hdfs.DFSClient) - waiting to lock <0x0000000107a0ded0> (a org.apache.hadoop.hdfs.DFSClient$DFSOutputStream) at org.apache.hadoop.hdfs.DFSClient$LeaseChecker.close(DFSClient.java:1175) - locked <0x00000001082b0748> (a org.apache.hadoop.hdfs.DFSClient$LeaseChecker) at org.apache.hadoop.hdfs.DFSClient.close(DFSClient.java:306) - locked <0x000000010824d640> (a org.apache.hadoop.hdfs.DFSClient) at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:325) at org.apache.hadoop.fs.FileSystem$Cache.closeAll(FileSystem.java:1835) - locked <0x0000000107a77ec8> (a org.apache.hadoop.fs.FileSystem$Cache) at org.apache.hadoop.fs.FileSystem$Cache$ClientFinalizer.run(FileSystem.java:1851) - locked <0x00000001079daa00> (a org.apache.hadoop.fs.FileSystem$Cache$ClientFinalizer) "main": at org.apache.hadoop.hdfs.DFSClient$LeaseChecker.remove(DFSClient.java:1151) - waiting to lock <0x00000001082b0748> (a org.apache.hadoop.hdfs.DFSClient$LeaseChecker) at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.close(DFSClient.java:3609) - locked <0x0000000107a0ded0> (a org.apache.hadoop.hdfs.DFSClient$DFSOutputStream) at org.apache.hadoop.fs.FSDataOutputStream$PositionCache.close(FSDataOutputStream.java:61) at org.apache.hadoop.fs.FSDataOutputStream.close(FSDataOutputStream.java:86) at org.apache.hadoop.hdfs.TestFileAppend4.testAppend(TestFileAppend4.java:99) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at junit.framework.TestCase.runTest(TestCase.java:168) at junit.framework.TestCase.runBare(TestCase.java:134) at junit.framework.TestResult$1.protect(TestResult.java:110) at junit.framework.TestResult.runProtected(TestResult.java:128) at junit.framework.TestResult.run(TestResult.java:113) at junit.framework.TestCase.run(TestCase.java:124) at junit.framework.TestSuite.runTest(TestSuite.java:232) at junit.framework.TestSuite.run(TestSuite.java:227) at junit.extensions.TestDecorator.basicRun(TestDecorator.java:24) at junit.extensions.TestSetup$1.protect(TestSetup.java:23) at junit.framework.TestResult.runProtected(TestResult.java:128) at junit.extensions.TestSetup.run(TestSetup.java:27) at org.apache.tools.ant.taskdefs.optional.junit.JUnitTestRunner.run(JUnitTestRunner.java:420) at org.apache.tools.ant.taskdefs.optional.junit.JUnitTestRunner.launch(JUnitTestRunner.java:911) at org.apache.tools.ant.taskdefs.optional.junit.JUnitTestRunner.main(JUnitTestRunner.java:768) > Deadlock in DFSClient#DFSOutputStream > ------------------------------------- > > Key: HDFS-723 > URL: https://issues.apache.org/jira/browse/HDFS-723 > Project: Hadoop HDFS > Issue Type: Bug > Reporter: Hairong Kuang > Assignee: Hairong Kuang > Priority: Blocker > Fix For: 0.21.0 > > > WhiIe was running some append-related tests, I hit this deadlock: > {noformat} > Found one Java-level deadlock: > ============================= > "Thread-3": > waiting to lock monitor 0x000000012ee044f0 (object 0x0000000107a0ded0, a > org.apache.hadoop.hdfs.DFSClient$DFSOutputStream), > which is held by "main" > "main": > waiting to lock monitor 0x000000012eeb71a8 (object 0x00000001082b0748, a > org.apache.hadoop.hdfs.DFSClient$LeaseChecker), > which is held by "Thread-3" > Java stack information for the threads listed above: > =================================================== > "Thread-3": > at > org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.close(DFSClient.java:3582) > - waiting to lock <0x0000000107a0ded0> (a > org.apache.hadoop.hdfs.DFSClient$DFSOutputStream) > at > org.apache.hadoop.hdfs.DFSClient$LeaseChecker.close(DFSClient.java:1175) > - locked <0x00000001082b0748> (a > org.apache.hadoop.hdfs.DFSClient$LeaseChecker) > at org.apache.hadoop.hdfs.DFSClient.close(DFSClient.java:306) > - locked <0x000000010824d640> (a org.apache.hadoop.hdfs.DFSClient) > - waiting to lock <0x0000000107a0ded0> (a > org.apache.hadoop.hdfs.DFSClient$DFSOutputStream) > at > org.apache.hadoop.hdfs.DFSClient$LeaseChecker.close(DFSClient.java:1175) > - locked <0x00000001082b0748> (a > org.apache.hadoop.hdfs.DFSClient$LeaseChecker) > at org.apache.hadoop.hdfs.DFSClient.close(DFSClient.java:306) > - locked <0x000000010824d640> (a org.apache.hadoop.hdfs.DFSClient) > at > org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:325) > at > org.apache.hadoop.fs.FileSystem$Cache.closeAll(FileSystem.java:1835) > - locked <0x0000000107a77ec8> (a > org.apache.hadoop.fs.FileSystem$Cache) > at > org.apache.hadoop.fs.FileSystem$Cache$ClientFinalizer.run(FileSystem.java:1851) > - locked <0x00000001079daa00> (a > org.apache.hadoop.fs.FileSystem$Cache$ClientFinalizer) > "main": > at > org.apache.hadoop.hdfs.DFSClient$LeaseChecker.remove(DFSClient.java:1151) > - waiting to lock <0x00000001082b0748> (a > org.apache.hadoop.hdfs.DFSClient$LeaseChecker) > at > org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.close(DFSClient.java:3609) > - locked <0x0000000107a0ded0> (a > org.apache.hadoop.hdfs.DFSClient$DFSOutputStream) > at > org.apache.hadoop.fs.FSDataOutputStream$PositionCache.close(FSDataOutputStream.java:61) > at > org.apache.hadoop.fs.FSDataOutputStream.close(FSDataOutputStream.java:86) > at > org.apache.hadoop.hdfs.TestFileAppend4.testAppend(TestFileAppend4.java:99) > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) > at > sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39) > at > sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) > at java.lang.reflect.Method.invoke(Method.java:597) > at junit.framework.TestCase.runTest(TestCase.java:168) > at junit.framework.TestCase.runBare(TestCase.java:134) > at junit.framework.TestResult$1.protect(TestResult.java:110) > at junit.framework.TestResult.runProtected(TestResult.java:128) > at junit.framework.TestResult.run(TestResult.java:113) > at junit.framework.TestCase.run(TestCase.java:124) > at junit.framework.TestSuite.runTest(TestSuite.java:232) > at junit.framework.TestSuite.run(TestSuite.java:227) > at junit.extensions.TestDecorator.basicRun(TestDecorator.java:24) > at junit.extensions.TestSetup$1.protect(TestSetup.java:23) > at junit.framework.TestResult.runProtected(TestResult.java:128) > at junit.extensions.TestSetup.run(TestSetup.java:27) > at > org.apache.tools.ant.taskdefs.optional.junit.JUnitTestRunner.run(JUnitTestRunner.java:420) > at > org.apache.tools.ant.taskdefs.optional.junit.JUnitTestRunner.launch(JUnitTestRunner.java:911) > at > org.apache.tools.ant.taskdefs.optional.junit.JUnitTestRunner.main(JUnitTestRunner.java:768) > {noformat} -- This message is automatically generated by JIRA. - You can reply to this email to add a comment to the issue online.