[
https://issues.apache.org/jira/browse/HDDS-11193?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Duong updated HDDS-11193:
-------------------------
Description:
Testing KeyOutputStream with TestHSync#testConcurrentWriteHSync sometimes
fails. Bellow are the known error stacktraces.
Repeated test run CI: [https://github.com/apache/ozone/actions/runs/9961420355]
#1
{code:java}
java.io.IOException: java.lang.IndexOutOfBoundsException
at
org.apache.hadoop.ozone.client.io.KeyOutputStream.handleWrite(KeyOutputStream.java:243)
at
org.apache.hadoop.ozone.client.io.KeyOutputStream.write(KeyOutputStream.java:212)
at
org.apache.hadoop.ozone.client.io.OzoneOutputStream.write(OzoneOutputStream.java:94)
at
org.apache.hadoop.fs.ozone.OzoneFSOutputStream.lambda$write$1(OzoneFSOutputStream.java:58)
at
org.apache.hadoop.hdds.tracing.TracingUtil.executeInSpan(TracingUtil.java:184)
at
org.apache.hadoop.hdds.tracing.TracingUtil.executeInNewSpan(TracingUtil.java:149)
at
org.apache.hadoop.fs.ozone.OzoneFSOutputStream.write(OzoneFSOutputStream.java:54)
at
org.apache.hadoop.fs.FSDataOutputStream$PositionCache.write(FSDataOutputStream.java:62)
at java.base/java.io.DataOutputStream.write(DataOutputStream.java:112)
at java.base/java.io.FilterOutputStream.write(FilterOutputStream.java:108)
at
org.apache.hadoop.fs.ozone.TestHSync.lambda$runConcurrentWriteHSync$8(TestHSync.java:683)
at java.base/java.lang.Thread.run(Thread.java:840)
Caused by: java.lang.IndexOutOfBoundsException
at
org.apache.hadoop.hdds.scm.storage.BlockOutputStream.write(BlockOutputStream.java:328)
at
org.apache.hadoop.ozone.client.io.BlockOutputStreamEntry.write(BlockOutputStreamEntry.java:138)
at
org.apache.hadoop.ozone.client.io.KeyOutputStream.writeToOutputStream(KeyOutputStream.java:255)
at
org.apache.hadoop.ozone.client.io.KeyOutputStream.handleWrite(KeyOutputStream.java:233)
{code}
#2
{code:java}
java.io.IOException: java.lang.IllegalArgumentException
at
org.apache.hadoop.ozone.client.io.KeyOutputStream.handleWrite(KeyOutputStream.java:243)
at
org.apache.hadoop.ozone.client.io.KeyOutputStream.write(KeyOutputStream.java:212)
at
org.apache.hadoop.ozone.client.io.OzoneOutputStream.write(OzoneOutputStream.java:94)
at
org.apache.hadoop.fs.ozone.OzoneFSOutputStream.lambda$write$1(OzoneFSOutputStream.java:58)
at
org.apache.hadoop.hdds.tracing.TracingUtil.executeInSpan(TracingUtil.java:184)
at
org.apache.hadoop.hdds.tracing.TracingUtil.executeInNewSpan(TracingUtil.java:149)
at
org.apache.hadoop.fs.ozone.OzoneFSOutputStream.write(OzoneFSOutputStream.java:54)
at
org.apache.hadoop.fs.FSDataOutputStream$PositionCache.write(FSDataOutputStream.java:62)
at java.io.DataOutputStream.write(DataOutputStream.java:107)
at java.io.FilterOutputStream.write(FilterOutputStream.java:97)
at
org.apache.hadoop.fs.ozone.TestHSync.lambda$runConcurrentWriteHSync$8(TestHSync.java:683)
at java.lang.Thread.run(Thread.java:750)
Caused by: java.lang.IllegalArgumentException
at
com.google.common.base.Preconditions.checkArgument(Preconditions.java:129)
at
org.apache.hadoop.ozone.client.io.BlockOutputStreamEntryPool.allocateBlockIfNeeded(BlockOutputStreamEntryPool.java:398)
at
org.apache.hadoop.ozone.client.io.KeyOutputStream.handleWrite(KeyOutputStream.java:222)
{code}
#3
{code:java}
Error: org.apache.hadoop.fs.ozone.TestHSync.testConcurrentWriteHSync(int, int,
boolean)[3] Time elapsed: 12.476 s <<< ERROR!
18421java.lang.IllegalArgumentException: Expected offset: 6134992 expected len:
6143720
18422 at
com.google.common.base.Preconditions.checkArgument(Preconditions.java:143)
18423 at
org.apache.hadoop.ozone.client.io.BlockOutputStreamEntryPool.commitKey(BlockOutputStreamEntryPool.java:321)
18424 at
org.apache.hadoop.ozone.client.io.KeyOutputStream.close(KeyOutputStream.java:563)
18425 at
org.apache.hadoop.ozone.client.io.OzoneOutputStream.close(OzoneOutputStream.java:105)
18426 at
org.apache.hadoop.fs.ozone.OzoneFSOutputStream.close(OzoneFSOutputStream.java:70)
18427 at
org.apache.hadoop.fs.FSDataOutputStream$PositionCache.close(FSDataOutputStream.java:77)
18428 at
org.apache.hadoop.fs.FSDataOutputStream.close(FSDataOutputStream.java:106)
18429 at
org.apache.hadoop.fs.ozone.TestHSync.testConcurrentWriteHSync(TestHSync.java:757)
18430 at java.lang.reflect.Method.invoke(Method.java:498)
18431 {code}
was:
Testing KeyOutputStream with TestHSync#testConcurrentWriteHSync sometimes
fails. Bellow are the known error stacktraces.
#1
{code:java}
java.io.IOException: java.lang.IndexOutOfBoundsException
at
org.apache.hadoop.ozone.client.io.KeyOutputStream.handleWrite(KeyOutputStream.java:243)
at
org.apache.hadoop.ozone.client.io.KeyOutputStream.write(KeyOutputStream.java:212)
at
org.apache.hadoop.ozone.client.io.OzoneOutputStream.write(OzoneOutputStream.java:94)
at
org.apache.hadoop.fs.ozone.OzoneFSOutputStream.lambda$write$1(OzoneFSOutputStream.java:58)
at
org.apache.hadoop.hdds.tracing.TracingUtil.executeInSpan(TracingUtil.java:184)
at
org.apache.hadoop.hdds.tracing.TracingUtil.executeInNewSpan(TracingUtil.java:149)
at
org.apache.hadoop.fs.ozone.OzoneFSOutputStream.write(OzoneFSOutputStream.java:54)
at
org.apache.hadoop.fs.FSDataOutputStream$PositionCache.write(FSDataOutputStream.java:62)
at java.base/java.io.DataOutputStream.write(DataOutputStream.java:112)
at java.base/java.io.FilterOutputStream.write(FilterOutputStream.java:108)
at
org.apache.hadoop.fs.ozone.TestHSync.lambda$runConcurrentWriteHSync$8(TestHSync.java:683)
at java.base/java.lang.Thread.run(Thread.java:840)
Caused by: java.lang.IndexOutOfBoundsException
at
org.apache.hadoop.hdds.scm.storage.BlockOutputStream.write(BlockOutputStream.java:328)
at
org.apache.hadoop.ozone.client.io.BlockOutputStreamEntry.write(BlockOutputStreamEntry.java:138)
at
org.apache.hadoop.ozone.client.io.KeyOutputStream.writeToOutputStream(KeyOutputStream.java:255)
at
org.apache.hadoop.ozone.client.io.KeyOutputStream.handleWrite(KeyOutputStream.java:233)
{code}
#2
{code:java}
java.io.IOException: java.lang.IllegalArgumentException
at
org.apache.hadoop.ozone.client.io.KeyOutputStream.handleWrite(KeyOutputStream.java:243)
at
org.apache.hadoop.ozone.client.io.KeyOutputStream.write(KeyOutputStream.java:212)
at
org.apache.hadoop.ozone.client.io.OzoneOutputStream.write(OzoneOutputStream.java:94)
at
org.apache.hadoop.fs.ozone.OzoneFSOutputStream.lambda$write$1(OzoneFSOutputStream.java:58)
at
org.apache.hadoop.hdds.tracing.TracingUtil.executeInSpan(TracingUtil.java:184)
at
org.apache.hadoop.hdds.tracing.TracingUtil.executeInNewSpan(TracingUtil.java:149)
at
org.apache.hadoop.fs.ozone.OzoneFSOutputStream.write(OzoneFSOutputStream.java:54)
at
org.apache.hadoop.fs.FSDataOutputStream$PositionCache.write(FSDataOutputStream.java:62)
at java.io.DataOutputStream.write(DataOutputStream.java:107)
at java.io.FilterOutputStream.write(FilterOutputStream.java:97)
at
org.apache.hadoop.fs.ozone.TestHSync.lambda$runConcurrentWriteHSync$8(TestHSync.java:683)
at java.lang.Thread.run(Thread.java:750)
Caused by: java.lang.IllegalArgumentException
at
com.google.common.base.Preconditions.checkArgument(Preconditions.java:129)
at
org.apache.hadoop.ozone.client.io.BlockOutputStreamEntryPool.allocateBlockIfNeeded(BlockOutputStreamEntryPool.java:398)
at
org.apache.hadoop.ozone.client.io.KeyOutputStream.handleWrite(KeyOutputStream.java:222)
{code}
#3
{code:java}
Error: org.apache.hadoop.fs.ozone.TestHSync.testConcurrentWriteHSync(int, int,
boolean)[3] Time elapsed: 12.476 s <<< ERROR!
18421java.lang.IllegalArgumentException: Expected offset: 6134992 expected len:
6143720
18422 at
com.google.common.base.Preconditions.checkArgument(Preconditions.java:143)
18423 at
org.apache.hadoop.ozone.client.io.BlockOutputStreamEntryPool.commitKey(BlockOutputStreamEntryPool.java:321)
18424 at
org.apache.hadoop.ozone.client.io.KeyOutputStream.close(KeyOutputStream.java:563)
18425 at
org.apache.hadoop.ozone.client.io.OzoneOutputStream.close(OzoneOutputStream.java:105)
18426 at
org.apache.hadoop.fs.ozone.OzoneFSOutputStream.close(OzoneFSOutputStream.java:70)
18427 at
org.apache.hadoop.fs.FSDataOutputStream$PositionCache.close(FSDataOutputStream.java:77)
18428 at
org.apache.hadoop.fs.FSDataOutputStream.close(FSDataOutputStream.java:106)
18429 at
org.apache.hadoop.fs.ozone.TestHSync.testConcurrentWriteHSync(TestHSync.java:757)
18430 at java.lang.reflect.Method.invoke(Method.java:498)
18431 {code}
> KeyOutputStream flakiness when running write and hsync concurrently
> -------------------------------------------------------------------
>
> Key: HDDS-11193
> URL: https://issues.apache.org/jira/browse/HDDS-11193
> Project: Apache Ozone
> Issue Type: Sub-task
> Reporter: Duong
> Priority: Major
>
> Testing KeyOutputStream with TestHSync#testConcurrentWriteHSync sometimes
> fails. Bellow are the known error stacktraces.
> Repeated test run CI:
> [https://github.com/apache/ozone/actions/runs/9961420355]
> #1
> {code:java}
> java.io.IOException: java.lang.IndexOutOfBoundsException
> at
> org.apache.hadoop.ozone.client.io.KeyOutputStream.handleWrite(KeyOutputStream.java:243)
> at
> org.apache.hadoop.ozone.client.io.KeyOutputStream.write(KeyOutputStream.java:212)
> at
> org.apache.hadoop.ozone.client.io.OzoneOutputStream.write(OzoneOutputStream.java:94)
> at
> org.apache.hadoop.fs.ozone.OzoneFSOutputStream.lambda$write$1(OzoneFSOutputStream.java:58)
> at
> org.apache.hadoop.hdds.tracing.TracingUtil.executeInSpan(TracingUtil.java:184)
> at
> org.apache.hadoop.hdds.tracing.TracingUtil.executeInNewSpan(TracingUtil.java:149)
> at
> org.apache.hadoop.fs.ozone.OzoneFSOutputStream.write(OzoneFSOutputStream.java:54)
> at
> org.apache.hadoop.fs.FSDataOutputStream$PositionCache.write(FSDataOutputStream.java:62)
> at java.base/java.io.DataOutputStream.write(DataOutputStream.java:112)
> at java.base/java.io.FilterOutputStream.write(FilterOutputStream.java:108)
> at
> org.apache.hadoop.fs.ozone.TestHSync.lambda$runConcurrentWriteHSync$8(TestHSync.java:683)
> at java.base/java.lang.Thread.run(Thread.java:840)
> Caused by: java.lang.IndexOutOfBoundsException
> at
> org.apache.hadoop.hdds.scm.storage.BlockOutputStream.write(BlockOutputStream.java:328)
> at
> org.apache.hadoop.ozone.client.io.BlockOutputStreamEntry.write(BlockOutputStreamEntry.java:138)
> at
> org.apache.hadoop.ozone.client.io.KeyOutputStream.writeToOutputStream(KeyOutputStream.java:255)
> at
> org.apache.hadoop.ozone.client.io.KeyOutputStream.handleWrite(KeyOutputStream.java:233)
> {code}
> #2
> {code:java}
> java.io.IOException: java.lang.IllegalArgumentException
> at
> org.apache.hadoop.ozone.client.io.KeyOutputStream.handleWrite(KeyOutputStream.java:243)
> at
> org.apache.hadoop.ozone.client.io.KeyOutputStream.write(KeyOutputStream.java:212)
> at
> org.apache.hadoop.ozone.client.io.OzoneOutputStream.write(OzoneOutputStream.java:94)
> at
> org.apache.hadoop.fs.ozone.OzoneFSOutputStream.lambda$write$1(OzoneFSOutputStream.java:58)
> at
> org.apache.hadoop.hdds.tracing.TracingUtil.executeInSpan(TracingUtil.java:184)
> at
> org.apache.hadoop.hdds.tracing.TracingUtil.executeInNewSpan(TracingUtil.java:149)
> at
> org.apache.hadoop.fs.ozone.OzoneFSOutputStream.write(OzoneFSOutputStream.java:54)
> at
> org.apache.hadoop.fs.FSDataOutputStream$PositionCache.write(FSDataOutputStream.java:62)
> at java.io.DataOutputStream.write(DataOutputStream.java:107)
> at java.io.FilterOutputStream.write(FilterOutputStream.java:97)
> at
> org.apache.hadoop.fs.ozone.TestHSync.lambda$runConcurrentWriteHSync$8(TestHSync.java:683)
> at java.lang.Thread.run(Thread.java:750)
> Caused by: java.lang.IllegalArgumentException
> at
> com.google.common.base.Preconditions.checkArgument(Preconditions.java:129)
> at
> org.apache.hadoop.ozone.client.io.BlockOutputStreamEntryPool.allocateBlockIfNeeded(BlockOutputStreamEntryPool.java:398)
> at
> org.apache.hadoop.ozone.client.io.KeyOutputStream.handleWrite(KeyOutputStream.java:222)
> {code}
> #3
> {code:java}
> Error: org.apache.hadoop.fs.ozone.TestHSync.testConcurrentWriteHSync(int,
> int, boolean)[3] Time elapsed: 12.476 s <<< ERROR!
> 18421java.lang.IllegalArgumentException: Expected offset: 6134992 expected
> len: 6143720
> 18422 at
> com.google.common.base.Preconditions.checkArgument(Preconditions.java:143)
> 18423 at
> org.apache.hadoop.ozone.client.io.BlockOutputStreamEntryPool.commitKey(BlockOutputStreamEntryPool.java:321)
> 18424 at
> org.apache.hadoop.ozone.client.io.KeyOutputStream.close(KeyOutputStream.java:563)
> 18425 at
> org.apache.hadoop.ozone.client.io.OzoneOutputStream.close(OzoneOutputStream.java:105)
> 18426 at
> org.apache.hadoop.fs.ozone.OzoneFSOutputStream.close(OzoneFSOutputStream.java:70)
> 18427 at
> org.apache.hadoop.fs.FSDataOutputStream$PositionCache.close(FSDataOutputStream.java:77)
> 18428 at
> org.apache.hadoop.fs.FSDataOutputStream.close(FSDataOutputStream.java:106)
> 18429 at
> org.apache.hadoop.fs.ozone.TestHSync.testConcurrentWriteHSync(TestHSync.java:757)
> 18430 at java.lang.reflect.Method.invoke(Method.java:498)
> 18431 {code}
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscr...@ozone.apache.org
For additional commands, e-mail: issues-h...@ozone.apache.org
