[27/50] [abbrv] hbase git commit: HBASE-18587 Fix flaky TestFileIOEngine
HBASE-18587 Fix flaky TestFileIOEngine
This short circuits reads and writes with 0 length and also removes flakiness
in TestFileIOEngine
Signed-off-by: Michael Stack
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/5280c100
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/5280c100
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/5280c100
Branch: refs/heads/HBASE-18467
Commit: 5280c100ff93f65cd568ce830e088cc12a2f5585
Parents: 2b88edf
Author: Zach York
Authored: Thu Aug 10 16:55:28 2017 -0700
Committer: Michael Stack
Committed: Tue Aug 15 14:57:10 2017 -0700
--
.../hbase/io/hfile/bucket/FileIOEngine.java | 23 ++--
.../hbase/io/hfile/bucket/TestFileIOEngine.java | 123 +++
2 files changed, 88 insertions(+), 58 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/5280c100/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/FileIOEngine.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/FileIOEngine.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/FileIOEngine.java
index a847bfe..ab77696 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/FileIOEngine.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/FileIOEngine.java
@@ -33,6 +33,7 @@ import org.apache.hadoop.hbase.io.hfile.CacheableDeserializer;
import org.apache.hadoop.hbase.io.hfile.Cacheable.MemoryType;
import org.apache.hadoop.hbase.nio.ByteBuff;
import org.apache.hadoop.hbase.nio.SingleByteBuff;
+import org.apache.hadoop.hbase.shaded.com.google.common.base.Preconditions;
import org.apache.hadoop.util.StringUtils;
/**
@@ -122,15 +123,18 @@ public class FileIOEngine implements IOEngine {
@Override
public Cacheable read(long offset, int length,
CacheableDeserializer deserializer)
throws IOException {
+Preconditions.checkArgument(length >= 0, "Length of read can not be less
than 0.");
ByteBuffer dstBuffer = ByteBuffer.allocate(length);
-accessFile(readAccessor, dstBuffer, offset);
-// The buffer created out of the fileChannel is formed by copying the data
from the file
-// Hence in this case there is no shared memory that we point to. Even if
the BucketCache evicts
-// this buffer from the file the data is already copied and there is no
need to ensure that
-// the results are not corrupted before consuming them.
-if (dstBuffer.limit() != length) {
- throw new RuntimeException("Only " + dstBuffer.limit() + " bytes read, "
+ length
- + " expected");
+if (length != 0) {
+ accessFile(readAccessor, dstBuffer, offset);
+ // The buffer created out of the fileChannel is formed by copying the
data from the file
+ // Hence in this case there is no shared memory that we point to. Even
if the BucketCache evicts
+ // this buffer from the file the data is already copied and there is no
need to ensure that
+ // the results are not corrupted before consuming them.
+ if (dstBuffer.limit() != length) {
+throw new RuntimeException("Only " + dstBuffer.limit() + " bytes read,
" + length
++ " expected");
+ }
}
return deserializer.deserialize(new SingleByteBuff(dstBuffer), true,
MemoryType.EXCLUSIVE);
}
@@ -143,6 +147,9 @@ public class FileIOEngine implements IOEngine {
*/
@Override
public void write(ByteBuffer srcBuffer, long offset) throws IOException {
+if (!srcBuffer.hasRemaining()) {
+ return;
+}
accessFile(writeAccessor, srcBuffer, offset);
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/5280c100/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/bucket/TestFileIOEngine.java
--
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/bucket/TestFileIOEngine.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/bucket/TestFileIOEngine.java
index d13022d..4451c0c 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/bucket/TestFileIOEngine.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/bucket/TestFileIOEngine.java
@@ -18,6 +18,7 @@
*/
package org.apache.hadoop.hbase.io.hfile.bucket;
+import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertTrue;
import java.io.File;
@@ -30,6 +31,8 @@ import
org.apache.hadoop.hbase.io.hfile.bucket.TestByteBufferIOEngine.BufferGrab
import org.apache.hadoop.hbase.nio.ByteBuff;
import org.apache.hadoop.hbase.testclassification.IOTests;
import org.apache.hadoop.hbase.testclassification.SmallTests;
+import
[36/50] [abbrv] hbase git commit: HBASE-18587 Fix flaky TestFileIOEngine
HBASE-18587 Fix flaky TestFileIOEngine
This short circuits reads and writes with 0 length and also removes flakiness
in TestFileIOEngine
Signed-off-by: Michael Stack
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/5280c100
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/5280c100
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/5280c100
Branch: refs/heads/HBASE-14070.HLC
Commit: 5280c100ff93f65cd568ce830e088cc12a2f5585
Parents: 2b88edf
Author: Zach York
Authored: Thu Aug 10 16:55:28 2017 -0700
Committer: Michael Stack
Committed: Tue Aug 15 14:57:10 2017 -0700
--
.../hbase/io/hfile/bucket/FileIOEngine.java | 23 ++--
.../hbase/io/hfile/bucket/TestFileIOEngine.java | 123 +++
2 files changed, 88 insertions(+), 58 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/5280c100/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/FileIOEngine.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/FileIOEngine.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/FileIOEngine.java
index a847bfe..ab77696 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/FileIOEngine.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/FileIOEngine.java
@@ -33,6 +33,7 @@ import org.apache.hadoop.hbase.io.hfile.CacheableDeserializer;
import org.apache.hadoop.hbase.io.hfile.Cacheable.MemoryType;
import org.apache.hadoop.hbase.nio.ByteBuff;
import org.apache.hadoop.hbase.nio.SingleByteBuff;
+import org.apache.hadoop.hbase.shaded.com.google.common.base.Preconditions;
import org.apache.hadoop.util.StringUtils;
/**
@@ -122,15 +123,18 @@ public class FileIOEngine implements IOEngine {
@Override
public Cacheable read(long offset, int length,
CacheableDeserializer deserializer)
throws IOException {
+Preconditions.checkArgument(length >= 0, "Length of read can not be less
than 0.");
ByteBuffer dstBuffer = ByteBuffer.allocate(length);
-accessFile(readAccessor, dstBuffer, offset);
-// The buffer created out of the fileChannel is formed by copying the data
from the file
-// Hence in this case there is no shared memory that we point to. Even if
the BucketCache evicts
-// this buffer from the file the data is already copied and there is no
need to ensure that
-// the results are not corrupted before consuming them.
-if (dstBuffer.limit() != length) {
- throw new RuntimeException("Only " + dstBuffer.limit() + " bytes read, "
+ length
- + " expected");
+if (length != 0) {
+ accessFile(readAccessor, dstBuffer, offset);
+ // The buffer created out of the fileChannel is formed by copying the
data from the file
+ // Hence in this case there is no shared memory that we point to. Even
if the BucketCache evicts
+ // this buffer from the file the data is already copied and there is no
need to ensure that
+ // the results are not corrupted before consuming them.
+ if (dstBuffer.limit() != length) {
+throw new RuntimeException("Only " + dstBuffer.limit() + " bytes read,
" + length
++ " expected");
+ }
}
return deserializer.deserialize(new SingleByteBuff(dstBuffer), true,
MemoryType.EXCLUSIVE);
}
@@ -143,6 +147,9 @@ public class FileIOEngine implements IOEngine {
*/
@Override
public void write(ByteBuffer srcBuffer, long offset) throws IOException {
+if (!srcBuffer.hasRemaining()) {
+ return;
+}
accessFile(writeAccessor, srcBuffer, offset);
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/5280c100/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/bucket/TestFileIOEngine.java
--
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/bucket/TestFileIOEngine.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/bucket/TestFileIOEngine.java
index d13022d..4451c0c 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/bucket/TestFileIOEngine.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/bucket/TestFileIOEngine.java
@@ -18,6 +18,7 @@
*/
package org.apache.hadoop.hbase.io.hfile.bucket;
+import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertTrue;
import java.io.File;
@@ -30,6 +31,8 @@ import
org.apache.hadoop.hbase.io.hfile.bucket.TestByteBufferIOEngine.BufferGrab
import org.apache.hadoop.hbase.nio.ByteBuff;
import org.apache.hadoop.hbase.testclassification.IOTests;
import org.apache.hadoop.hbase.testclassification.SmallTests;
+imp
hbase git commit: HBASE-18587 Fix flaky TestFileIOEngine
Repository: hbase
Updated Branches:
refs/heads/branch-1.3 7800fa152 -> 2fd2b3f9b
HBASE-18587 Fix flaky TestFileIOEngine
This short circuits reads and writes with 0 length and also removes flakiness
in TestFileIOEngine
Signed-off-by: Michael Stack
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/2fd2b3f9
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/2fd2b3f9
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/2fd2b3f9
Branch: refs/heads/branch-1.3
Commit: 2fd2b3f9b86509c37a7ed22eb93bf1c493711739
Parents: 7800fa1
Author: Zach York
Authored: Thu Aug 10 16:55:28 2017 -0700
Committer: Michael Stack
Committed: Wed Aug 16 14:51:11 2017 -0700
--
.../hbase/io/hfile/bucket/FileIOEngine.java | 8 +-
.../hbase/io/hfile/bucket/TestFileIOEngine.java | 122 +++
2 files changed, 79 insertions(+), 51 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/2fd2b3f9/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/FileIOEngine.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/FileIOEngine.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/FileIOEngine.java
index a7d6956..3419587 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/FileIOEngine.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/FileIOEngine.java
@@ -102,7 +102,10 @@ public class FileIOEngine implements IOEngine {
*/
@Override
public int read(ByteBuffer dstBuffer, long offset) throws IOException {
-return accessFile(readAccessor, dstBuffer, offset);
+if (dstBuffer.remaining() != 0) {
+ return accessFile(readAccessor, dstBuffer, offset);
+}
+return 0;
}
/**
@@ -113,6 +116,9 @@ public class FileIOEngine implements IOEngine {
*/
@Override
public void write(ByteBuffer srcBuffer, long offset) throws IOException {
+if (!srcBuffer.hasRemaining()) {
+ return;
+}
accessFile(writeAccessor, srcBuffer, offset);
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/2fd2b3f9/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/bucket/TestFileIOEngine.java
--
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/bucket/TestFileIOEngine.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/bucket/TestFileIOEngine.java
index 8c71c09..a03818b 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/bucket/TestFileIOEngine.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/bucket/TestFileIOEngine.java
@@ -18,7 +18,7 @@
*/
package org.apache.hadoop.hbase.io.hfile.bucket;
-import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.assertArrayEquals;
import java.io.File;
import java.io.IOException;
@@ -27,6 +27,8 @@ import java.util.ArrayList;
import java.util.List;
import org.apache.hadoop.hbase.testclassification.SmallTests;
+import org.junit.After;
+import org.junit.Before;
import org.junit.Test;
import org.junit.experimental.categories.Category;
@@ -35,61 +37,81 @@ import org.junit.experimental.categories.Category;
*/
@Category(SmallTests.class)
public class TestFileIOEngine {
- @Test
- public void testFileIOEngine() throws IOException {
-long totalCapacity = 6 * 1024 * 1024; // 6 MB
-String[] filePaths = { "testFileIOEngine1", "testFileIOEngine2",
-"testFileIOEngine3" };
-long sizePerFile = totalCapacity / filePaths.length; // 2 MB per File
-List boundaryStartPositions = new ArrayList();
+
+ private static final long TOTAL_CAPACITY = 6 * 1024 * 1024; // 6 MB
+ private static final String[] FILE_PATHS = {"testFileIOEngine1",
"testFileIOEngine2",
+ "testFileIOEngine3"};
+ private static final long SIZE_PER_FILE = TOTAL_CAPACITY /
FILE_PATHS.length; // 2 MB per File
+ private final static List boundaryStartPositions = new
ArrayList();
+ private final static List boundaryStopPositions = new
ArrayList();
+
+ private FileIOEngine fileIOEngine;
+
+ static {
boundaryStartPositions.add(0L);
-for (int i = 1; i < filePaths.length; i++) {
- boundaryStartPositions.add(sizePerFile * i - 1);
- boundaryStartPositions.add(sizePerFile * i);
- boundaryStartPositions.add(sizePerFile * i + 1);
+for (int i = 1; i < FILE_PATHS.length; i++) {
+ boundaryStartPositions.add(SIZE_PER_FILE * i - 1);
+ boundaryStartPositions.add(SIZE_PER_FILE * i);
+ boundaryStartPositions.add(SIZE_PER_FILE * i + 1);
}
-List boundaryStopPositions = new ArrayList();
-for (int i = 1; i < filePaths.length; i++) {
hbase git commit: HBASE-18587 Fix flaky TestFileIOEngine
Repository: hbase
Updated Branches:
refs/heads/branch-1.4 ea5789a7f -> 255319a0e
HBASE-18587 Fix flaky TestFileIOEngine
This short circuits reads and writes with 0 length and also removes flakiness
in TestFileIOEngine
Signed-off-by: Michael Stack
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/255319a0
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/255319a0
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/255319a0
Branch: refs/heads/branch-1.4
Commit: 255319a0e680ce15c8179c54743e9ad4d343cadb
Parents: ea5789a
Author: Zach York
Authored: Thu Aug 10 16:55:28 2017 -0700
Committer: Michael Stack
Committed: Wed Aug 16 14:50:48 2017 -0700
--
.../hbase/io/hfile/bucket/FileIOEngine.java | 8 +-
.../hbase/io/hfile/bucket/TestFileIOEngine.java | 122 +++
2 files changed, 79 insertions(+), 51 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/255319a0/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/FileIOEngine.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/FileIOEngine.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/FileIOEngine.java
index a7d6956..3419587 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/FileIOEngine.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/FileIOEngine.java
@@ -102,7 +102,10 @@ public class FileIOEngine implements IOEngine {
*/
@Override
public int read(ByteBuffer dstBuffer, long offset) throws IOException {
-return accessFile(readAccessor, dstBuffer, offset);
+if (dstBuffer.remaining() != 0) {
+ return accessFile(readAccessor, dstBuffer, offset);
+}
+return 0;
}
/**
@@ -113,6 +116,9 @@ public class FileIOEngine implements IOEngine {
*/
@Override
public void write(ByteBuffer srcBuffer, long offset) throws IOException {
+if (!srcBuffer.hasRemaining()) {
+ return;
+}
accessFile(writeAccessor, srcBuffer, offset);
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/255319a0/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/bucket/TestFileIOEngine.java
--
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/bucket/TestFileIOEngine.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/bucket/TestFileIOEngine.java
index 8c71c09..a03818b 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/bucket/TestFileIOEngine.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/bucket/TestFileIOEngine.java
@@ -18,7 +18,7 @@
*/
package org.apache.hadoop.hbase.io.hfile.bucket;
-import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.assertArrayEquals;
import java.io.File;
import java.io.IOException;
@@ -27,6 +27,8 @@ import java.util.ArrayList;
import java.util.List;
import org.apache.hadoop.hbase.testclassification.SmallTests;
+import org.junit.After;
+import org.junit.Before;
import org.junit.Test;
import org.junit.experimental.categories.Category;
@@ -35,61 +37,81 @@ import org.junit.experimental.categories.Category;
*/
@Category(SmallTests.class)
public class TestFileIOEngine {
- @Test
- public void testFileIOEngine() throws IOException {
-long totalCapacity = 6 * 1024 * 1024; // 6 MB
-String[] filePaths = { "testFileIOEngine1", "testFileIOEngine2",
-"testFileIOEngine3" };
-long sizePerFile = totalCapacity / filePaths.length; // 2 MB per File
-List boundaryStartPositions = new ArrayList();
+
+ private static final long TOTAL_CAPACITY = 6 * 1024 * 1024; // 6 MB
+ private static final String[] FILE_PATHS = {"testFileIOEngine1",
"testFileIOEngine2",
+ "testFileIOEngine3"};
+ private static final long SIZE_PER_FILE = TOTAL_CAPACITY /
FILE_PATHS.length; // 2 MB per File
+ private final static List boundaryStartPositions = new
ArrayList();
+ private final static List boundaryStopPositions = new
ArrayList();
+
+ private FileIOEngine fileIOEngine;
+
+ static {
boundaryStartPositions.add(0L);
-for (int i = 1; i < filePaths.length; i++) {
- boundaryStartPositions.add(sizePerFile * i - 1);
- boundaryStartPositions.add(sizePerFile * i);
- boundaryStartPositions.add(sizePerFile * i + 1);
+for (int i = 1; i < FILE_PATHS.length; i++) {
+ boundaryStartPositions.add(SIZE_PER_FILE * i - 1);
+ boundaryStartPositions.add(SIZE_PER_FILE * i);
+ boundaryStartPositions.add(SIZE_PER_FILE * i + 1);
}
-List boundaryStopPositions = new ArrayList();
-for (int i = 1; i < filePaths.length; i++) {
hbase git commit: HBASE-18587 Fix flaky TestFileIOEngine
Repository: hbase
Updated Branches:
refs/heads/branch-1 3feb87b00 -> 1f1ab8c87
HBASE-18587 Fix flaky TestFileIOEngine
This short circuits reads and writes with 0 length and also removes flakiness
in TestFileIOEngine
Signed-off-by: Michael Stack
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/1f1ab8c8
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/1f1ab8c8
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/1f1ab8c8
Branch: refs/heads/branch-1
Commit: 1f1ab8c873b193675766969df83db91213137d72
Parents: 3feb87b
Author: Zach York
Authored: Thu Aug 10 16:55:28 2017 -0700
Committer: Michael Stack
Committed: Wed Aug 16 14:50:11 2017 -0700
--
.../hbase/io/hfile/bucket/FileIOEngine.java | 8 +-
.../hbase/io/hfile/bucket/TestFileIOEngine.java | 122 +++
2 files changed, 79 insertions(+), 51 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/1f1ab8c8/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/FileIOEngine.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/FileIOEngine.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/FileIOEngine.java
index a7d6956..3419587 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/FileIOEngine.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/FileIOEngine.java
@@ -102,7 +102,10 @@ public class FileIOEngine implements IOEngine {
*/
@Override
public int read(ByteBuffer dstBuffer, long offset) throws IOException {
-return accessFile(readAccessor, dstBuffer, offset);
+if (dstBuffer.remaining() != 0) {
+ return accessFile(readAccessor, dstBuffer, offset);
+}
+return 0;
}
/**
@@ -113,6 +116,9 @@ public class FileIOEngine implements IOEngine {
*/
@Override
public void write(ByteBuffer srcBuffer, long offset) throws IOException {
+if (!srcBuffer.hasRemaining()) {
+ return;
+}
accessFile(writeAccessor, srcBuffer, offset);
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/1f1ab8c8/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/bucket/TestFileIOEngine.java
--
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/bucket/TestFileIOEngine.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/bucket/TestFileIOEngine.java
index 8c71c09..a03818b 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/bucket/TestFileIOEngine.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/bucket/TestFileIOEngine.java
@@ -18,7 +18,7 @@
*/
package org.apache.hadoop.hbase.io.hfile.bucket;
-import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.assertArrayEquals;
import java.io.File;
import java.io.IOException;
@@ -27,6 +27,8 @@ import java.util.ArrayList;
import java.util.List;
import org.apache.hadoop.hbase.testclassification.SmallTests;
+import org.junit.After;
+import org.junit.Before;
import org.junit.Test;
import org.junit.experimental.categories.Category;
@@ -35,61 +37,81 @@ import org.junit.experimental.categories.Category;
*/
@Category(SmallTests.class)
public class TestFileIOEngine {
- @Test
- public void testFileIOEngine() throws IOException {
-long totalCapacity = 6 * 1024 * 1024; // 6 MB
-String[] filePaths = { "testFileIOEngine1", "testFileIOEngine2",
-"testFileIOEngine3" };
-long sizePerFile = totalCapacity / filePaths.length; // 2 MB per File
-List boundaryStartPositions = new ArrayList();
+
+ private static final long TOTAL_CAPACITY = 6 * 1024 * 1024; // 6 MB
+ private static final String[] FILE_PATHS = {"testFileIOEngine1",
"testFileIOEngine2",
+ "testFileIOEngine3"};
+ private static final long SIZE_PER_FILE = TOTAL_CAPACITY /
FILE_PATHS.length; // 2 MB per File
+ private final static List boundaryStartPositions = new
ArrayList();
+ private final static List boundaryStopPositions = new
ArrayList();
+
+ private FileIOEngine fileIOEngine;
+
+ static {
boundaryStartPositions.add(0L);
-for (int i = 1; i < filePaths.length; i++) {
- boundaryStartPositions.add(sizePerFile * i - 1);
- boundaryStartPositions.add(sizePerFile * i);
- boundaryStartPositions.add(sizePerFile * i + 1);
+for (int i = 1; i < FILE_PATHS.length; i++) {
+ boundaryStartPositions.add(SIZE_PER_FILE * i - 1);
+ boundaryStartPositions.add(SIZE_PER_FILE * i);
+ boundaryStartPositions.add(SIZE_PER_FILE * i + 1);
}
-List boundaryStopPositions = new ArrayList();
-for (int i = 1; i < filePaths.length; i++) {
-
hbase git commit: HBASE-18587 Fix flaky TestFileIOEngine
Repository: hbase
Updated Branches:
refs/heads/branch-2 c298ab65e -> 49ca224fc
HBASE-18587 Fix flaky TestFileIOEngine
This short circuits reads and writes with 0 length and also removes flakiness
in TestFileIOEngine
Signed-off-by: Michael Stack
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/49ca224f
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/49ca224f
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/49ca224f
Branch: refs/heads/branch-2
Commit: 49ca224fc37ba763229eae44e4df3f0e0bceb9b8
Parents: c298ab6
Author: Zach York
Authored: Thu Aug 10 16:55:28 2017 -0700
Committer: Michael Stack
Committed: Tue Aug 15 14:57:34 2017 -0700
--
.../hbase/io/hfile/bucket/FileIOEngine.java | 23 ++--
.../hbase/io/hfile/bucket/TestFileIOEngine.java | 123 +++
2 files changed, 88 insertions(+), 58 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/49ca224f/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/FileIOEngine.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/FileIOEngine.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/FileIOEngine.java
index a847bfe..ab77696 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/FileIOEngine.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/FileIOEngine.java
@@ -33,6 +33,7 @@ import org.apache.hadoop.hbase.io.hfile.CacheableDeserializer;
import org.apache.hadoop.hbase.io.hfile.Cacheable.MemoryType;
import org.apache.hadoop.hbase.nio.ByteBuff;
import org.apache.hadoop.hbase.nio.SingleByteBuff;
+import org.apache.hadoop.hbase.shaded.com.google.common.base.Preconditions;
import org.apache.hadoop.util.StringUtils;
/**
@@ -122,15 +123,18 @@ public class FileIOEngine implements IOEngine {
@Override
public Cacheable read(long offset, int length,
CacheableDeserializer deserializer)
throws IOException {
+Preconditions.checkArgument(length >= 0, "Length of read can not be less
than 0.");
ByteBuffer dstBuffer = ByteBuffer.allocate(length);
-accessFile(readAccessor, dstBuffer, offset);
-// The buffer created out of the fileChannel is formed by copying the data
from the file
-// Hence in this case there is no shared memory that we point to. Even if
the BucketCache evicts
-// this buffer from the file the data is already copied and there is no
need to ensure that
-// the results are not corrupted before consuming them.
-if (dstBuffer.limit() != length) {
- throw new RuntimeException("Only " + dstBuffer.limit() + " bytes read, "
+ length
- + " expected");
+if (length != 0) {
+ accessFile(readAccessor, dstBuffer, offset);
+ // The buffer created out of the fileChannel is formed by copying the
data from the file
+ // Hence in this case there is no shared memory that we point to. Even
if the BucketCache evicts
+ // this buffer from the file the data is already copied and there is no
need to ensure that
+ // the results are not corrupted before consuming them.
+ if (dstBuffer.limit() != length) {
+throw new RuntimeException("Only " + dstBuffer.limit() + " bytes read,
" + length
++ " expected");
+ }
}
return deserializer.deserialize(new SingleByteBuff(dstBuffer), true,
MemoryType.EXCLUSIVE);
}
@@ -143,6 +147,9 @@ public class FileIOEngine implements IOEngine {
*/
@Override
public void write(ByteBuffer srcBuffer, long offset) throws IOException {
+if (!srcBuffer.hasRemaining()) {
+ return;
+}
accessFile(writeAccessor, srcBuffer, offset);
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/49ca224f/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/bucket/TestFileIOEngine.java
--
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/bucket/TestFileIOEngine.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/bucket/TestFileIOEngine.java
index d13022d..4451c0c 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/bucket/TestFileIOEngine.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/bucket/TestFileIOEngine.java
@@ -18,6 +18,7 @@
*/
package org.apache.hadoop.hbase.io.hfile.bucket;
+import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertTrue;
import java.io.File;
@@ -30,6 +31,8 @@ import
org.apache.hadoop.hbase.io.hfile.bucket.TestByteBufferIOEngine.BufferGrab
import org.apache.hadoop.hbase.nio.ByteBuff;
import org.apache.hadoop.hbase.testclassification.
hbase git commit: HBASE-18587 Fix flaky TestFileIOEngine
Repository: hbase
Updated Branches:
refs/heads/master 2b88edfd8 -> 5280c100f
HBASE-18587 Fix flaky TestFileIOEngine
This short circuits reads and writes with 0 length and also removes flakiness
in TestFileIOEngine
Signed-off-by: Michael Stack
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/5280c100
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/5280c100
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/5280c100
Branch: refs/heads/master
Commit: 5280c100ff93f65cd568ce830e088cc12a2f5585
Parents: 2b88edf
Author: Zach York
Authored: Thu Aug 10 16:55:28 2017 -0700
Committer: Michael Stack
Committed: Tue Aug 15 14:57:10 2017 -0700
--
.../hbase/io/hfile/bucket/FileIOEngine.java | 23 ++--
.../hbase/io/hfile/bucket/TestFileIOEngine.java | 123 +++
2 files changed, 88 insertions(+), 58 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/5280c100/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/FileIOEngine.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/FileIOEngine.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/FileIOEngine.java
index a847bfe..ab77696 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/FileIOEngine.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/FileIOEngine.java
@@ -33,6 +33,7 @@ import org.apache.hadoop.hbase.io.hfile.CacheableDeserializer;
import org.apache.hadoop.hbase.io.hfile.Cacheable.MemoryType;
import org.apache.hadoop.hbase.nio.ByteBuff;
import org.apache.hadoop.hbase.nio.SingleByteBuff;
+import org.apache.hadoop.hbase.shaded.com.google.common.base.Preconditions;
import org.apache.hadoop.util.StringUtils;
/**
@@ -122,15 +123,18 @@ public class FileIOEngine implements IOEngine {
@Override
public Cacheable read(long offset, int length,
CacheableDeserializer deserializer)
throws IOException {
+Preconditions.checkArgument(length >= 0, "Length of read can not be less
than 0.");
ByteBuffer dstBuffer = ByteBuffer.allocate(length);
-accessFile(readAccessor, dstBuffer, offset);
-// The buffer created out of the fileChannel is formed by copying the data
from the file
-// Hence in this case there is no shared memory that we point to. Even if
the BucketCache evicts
-// this buffer from the file the data is already copied and there is no
need to ensure that
-// the results are not corrupted before consuming them.
-if (dstBuffer.limit() != length) {
- throw new RuntimeException("Only " + dstBuffer.limit() + " bytes read, "
+ length
- + " expected");
+if (length != 0) {
+ accessFile(readAccessor, dstBuffer, offset);
+ // The buffer created out of the fileChannel is formed by copying the
data from the file
+ // Hence in this case there is no shared memory that we point to. Even
if the BucketCache evicts
+ // this buffer from the file the data is already copied and there is no
need to ensure that
+ // the results are not corrupted before consuming them.
+ if (dstBuffer.limit() != length) {
+throw new RuntimeException("Only " + dstBuffer.limit() + " bytes read,
" + length
++ " expected");
+ }
}
return deserializer.deserialize(new SingleByteBuff(dstBuffer), true,
MemoryType.EXCLUSIVE);
}
@@ -143,6 +147,9 @@ public class FileIOEngine implements IOEngine {
*/
@Override
public void write(ByteBuffer srcBuffer, long offset) throws IOException {
+if (!srcBuffer.hasRemaining()) {
+ return;
+}
accessFile(writeAccessor, srcBuffer, offset);
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/5280c100/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/bucket/TestFileIOEngine.java
--
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/bucket/TestFileIOEngine.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/bucket/TestFileIOEngine.java
index d13022d..4451c0c 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/bucket/TestFileIOEngine.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/bucket/TestFileIOEngine.java
@@ -18,6 +18,7 @@
*/
package org.apache.hadoop.hbase.io.hfile.bucket;
+import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertTrue;
import java.io.File;
@@ -30,6 +31,8 @@ import
org.apache.hadoop.hbase.io.hfile.bucket.TestByteBufferIOEngine.BufferGrab
import org.apache.hadoop.hbase.nio.ByteBuff;
import org.apache.hadoop.hbase.testclassification.IOTe
