Repository: phoenix Updated Branches: refs/heads/4.x-HBase-1.1 7f3e7b02e -> 3cdea9500
PHOENIX-3884 Correct MutationState size estimation. Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/3cdea950 Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/3cdea950 Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/3cdea950 Branch: refs/heads/4.x-HBase-1.1 Commit: 3cdea950090204e295b506ae06ee7cf34e3c35bf Parents: 7f3e7b0 Author: Lars Hofhansl <[email protected]> Authored: Fri May 26 21:50:18 2017 -0700 Committer: Lars Hofhansl <[email protected]> Committed: Fri May 26 21:50:18 2017 -0700 ---------------------------------------------------------------------- .../UngroupedAggregateRegionObserver.java | 24 ++++++++------------ .../apache/phoenix/execute/MutationState.java | 3 ++- .../org/apache/phoenix/util/KeyValueUtil.java | 17 ++++++++++++++ 3 files changed, 28 insertions(+), 16 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/phoenix/blob/3cdea950/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java index 70e74e9..16f19fb 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java @@ -58,7 +58,6 @@ import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.coprocessor.ObserverContext; import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment; -import org.apache.hadoop.hbase.io.HeapSize; import org.apache.hadoop.hbase.io.ImmutableBytesWritable; import org.apache.hadoop.hbase.regionserver.InternalScanner; import org.apache.hadoop.hbase.regionserver.Region; @@ -288,8 +287,8 @@ public class UngroupedAggregateRegionObserver extends BaseScannerRegionObserver return s; } - class MutationList extends ArrayList<Mutation> implements HeapSize { - private long heapSize = 0l; + class MutationList extends ArrayList<Mutation> { + private long byteSize = 0l; public MutationList() { super(); } @@ -302,23 +301,18 @@ public class UngroupedAggregateRegionObserver extends BaseScannerRegionObserver public boolean add(Mutation e) { boolean r = super.add(e); if (r) { - incrementHeapSize(e.heapSize()); + this.byteSize += KeyValueUtil.calculateMutationDiskSize(e); } return r; } - @Override - public long heapSize() { - return heapSize; - } - - private void incrementHeapSize(long heapSize) { - this.heapSize += heapSize; + public long byteSize() { + return byteSize; } @Override public void clear() { - heapSize = 0l; + byteSize = 0l; super.clear(); } } @@ -708,14 +702,14 @@ public class UngroupedAggregateRegionObserver extends BaseScannerRegionObserver } } } - if (readyToCommit(rowCount, mutations.heapSize(), maxBatchSize, maxBatchSizeBytes)) { + if (readyToCommit(rowCount, mutations.byteSize(), maxBatchSize, maxBatchSizeBytes)) { commit(region, mutations, indexUUID, blockingMemStoreSize, indexMaintainersPtr, txState, areMutationInSameRegion, targetHTable, useIndexProto); mutations.clear(); } // Commit in batches based on UPSERT_BATCH_SIZE_BYTES_ATTRIB in config - if (readyToCommit(rowCount, indexMutations.heapSize(), maxBatchSize, maxBatchSizeBytes)) { + if (readyToCommit(rowCount, indexMutations.byteSize(), maxBatchSize, maxBatchSizeBytes)) { commitBatch(region, indexMutations, null, blockingMemStoreSize, null, txState, useIndexProto); indexMutations.clear(); @@ -894,7 +888,7 @@ public class UngroupedAggregateRegionObserver extends BaseScannerRegionObserver del.addDeleteMarker(cell); } } - if (readyToCommit(rowCount, mutations.heapSize(), maxBatchSize, maxBatchSizeBytes)) { + if (readyToCommit(rowCount, mutations.byteSize(), maxBatchSize, maxBatchSizeBytes)) { region.batchMutate(mutations.toArray(new Mutation[mutations.size()]), HConstants.NO_NONCE, HConstants.NO_NONCE); uuidValue = ServerCacheClient.generateId(); http://git-wip-us.apache.org/repos/asf/phoenix/blob/3cdea950/phoenix-core/src/main/java/org/apache/phoenix/execute/MutationState.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/execute/MutationState.java b/phoenix-core/src/main/java/org/apache/phoenix/execute/MutationState.java index 37ab7a6..d5cad63 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/execute/MutationState.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/execute/MutationState.java @@ -87,6 +87,7 @@ import org.apache.phoenix.schema.types.PLong; import org.apache.phoenix.trace.util.Tracing; import org.apache.phoenix.util.ByteUtil; import org.apache.phoenix.util.IndexUtil; +import org.apache.phoenix.util.KeyValueUtil; import org.apache.phoenix.util.LogUtil; import org.apache.phoenix.util.PhoenixRuntime; import org.apache.phoenix.util.SQLCloseable; @@ -837,7 +838,7 @@ public class MutationState implements SQLCloseable { long byteSize = 0; if (GlobalClientMetrics.isMetricsEnabled()) { for (Mutation mutation : mutations) { - byteSize += mutation.heapSize(); + byteSize += KeyValueUtil.calculateMutationDiskSize(mutation); } } GLOBAL_MUTATION_BYTES.update(byteSize); http://git-wip-us.apache.org/repos/asf/phoenix/blob/3cdea950/phoenix-core/src/main/java/org/apache/phoenix/util/KeyValueUtil.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/KeyValueUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/util/KeyValueUtil.java index 071de66..d16521b 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/util/KeyValueUtil.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/util/KeyValueUtil.java @@ -21,11 +21,13 @@ import java.util.Arrays; import java.util.Collections; import java.util.Comparator; import java.util.List; +import java.util.Map.Entry; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellUtil; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.KeyValue.Type; +import org.apache.hadoop.hbase.client.Mutation; import org.apache.hadoop.hbase.io.ImmutableBytesWritable; import org.apache.phoenix.hbase.index.util.KeyValueBuilder; @@ -160,4 +162,19 @@ public class KeyValueUtil { return kvBuilder.compareQualifier(l, qualifier, 0, qualifier.length); } } + + /** + * Calculate the size a mutation will likely take when stored in HBase + * @param m The Mutation + * @return the disk size of the passed mutation + */ + public static long calculateMutationDiskSize(Mutation m) { + long size = 0; + for (Entry<byte [], List<Cell>> entry : m.getFamilyCellMap().entrySet()) { + for (Cell c : entry.getValue()) { + size += org.apache.hadoop.hbase.KeyValueUtil.length(c); + } + } + return size; + } }
