Michael Blow has submitted this change and it was merged. Change subject: ASTERIXDB-1455: Scale CharRecord Buffer Expansions ......................................................................
ASTERIXDB-1455: Scale CharRecord Buffer Expansions Reading a 30 MB record requires > 1500 buffer reallocations when increasing by a fixed amount. Updated to scale up the buffer increases to reduce the number of reallocations in case of large records. Change-Id: I4822b2a78e1f411de94d2c717523df6b2b4a381e Reviewed-on: https://asterix-gerrit.ics.uci.edu/861 Tested-by: Jenkins <[email protected]> Reviewed-by: abdullah alamoudi <[email protected]> --- M asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/CharArrayRecord.java M asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/AbstractCharRecordLookupReader.java M asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java 3 files changed, 4 insertions(+), 6 deletions(-) Approvals: abdullah alamoudi: Looks good to me, approved Jenkins: Verified diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/CharArrayRecord.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/CharArrayRecord.java index 6b8e10d..f174962 100644 --- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/CharArrayRecord.java +++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/CharArrayRecord.java @@ -69,10 +69,8 @@ throw new IOException( "Record is too large!. Maximum record size is " + ExternalDataConstants.MAX_RECORD_SIZE); } - int newSize = len + ExternalDataConstants.DEFAULT_BUFFER_INCREMENT; - if (newSize > ExternalDataConstants.MAX_RECORD_SIZE) { - newSize = ExternalDataConstants.MAX_RECORD_SIZE; - } + int newSize = Math.min((int)(len * ExternalDataConstants.DEFAULT_BUFFER_INCREMENT_FACTOR), + ExternalDataConstants.MAX_RECORD_SIZE); value = Arrays.copyOf(value, newSize); } } diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/AbstractCharRecordLookupReader.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/AbstractCharRecordLookupReader.java index 45d01e9..f9ec114 100644 --- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/AbstractCharRecordLookupReader.java +++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/AbstractCharRecordLookupReader.java @@ -64,7 +64,7 @@ reusableByteBuffer.clear(); if (reusableByteBuffer.remaining() < value.getLength()) { reusableByteBuffer = ByteBuffer - .allocateDirect(value.getLength() + ExternalDataConstants.DEFAULT_BUFFER_INCREMENT); + .allocateDirect((int)(value.getLength() * ExternalDataConstants.DEFAULT_BUFFER_INCREMENT_FACTOR)); } reusableByteBuffer.put(value.getBytes(), 0, value.getLength()); reusableByteBuffer.flip(); diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java index 0fd76cc..cf2b7f3 100644 --- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java +++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java @@ -206,7 +206,7 @@ * Size default values */ public static final int DEFAULT_BUFFER_SIZE = 4096; - public static final int DEFAULT_BUFFER_INCREMENT = 2048; + public static final float DEFAULT_BUFFER_INCREMENT_FACTOR = 1.5F; public static final int DEFAULT_QUEUE_SIZE = 64; public static final int MAX_RECORD_SIZE = 32000000; -- To view, visit https://asterix-gerrit.ics.uci.edu/861 To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-MessageType: merged Gerrit-Change-Id: I4822b2a78e1f411de94d2c717523df6b2b4a381e Gerrit-PatchSet: 3 Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-Owner: Michael Blow <[email protected]> Gerrit-Reviewer: Jenkins <[email protected]> Gerrit-Reviewer: Michael Blow <[email protected]> Gerrit-Reviewer: Till Westmann <[email protected]> Gerrit-Reviewer: abdullah alamoudi <[email protected]>
