Repository: phoenix Updated Branches: refs/heads/calcite 69a2f2437 -> 9ac854ae5
PHOENIX-2649 - GC/OOM during BulkLoad Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/12f6a6f4 Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/12f6a6f4 Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/12f6a6f4 Branch: refs/heads/calcite Commit: 12f6a6f48c4a4739e09f1842d885d53e2e5550e2 Parents: 5c25a72 Author: ravimagham <ravi.mag...@bazaarvoice.com> Authored: Wed Feb 3 14:15:16 2016 -0800 Committer: ravimagham <ravi.mag...@bazaarvoice.com> Committed: Wed Feb 3 14:15:16 2016 -0800 ---------------------------------------------------------------------- .../mapreduce/bulkload/TableRowkeyPair.java | 41 +++--------- .../mapreduce/bulkload/TestTableRowkeyPair.java | 67 ++++++++++++++++++++ 2 files changed, 75 insertions(+), 33 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/phoenix/blob/12f6a6f4/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/bulkload/TableRowkeyPair.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/bulkload/TableRowkeyPair.java b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/bulkload/TableRowkeyPair.java index 412226f..e3032be 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/bulkload/TableRowkeyPair.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/bulkload/TableRowkeyPair.java @@ -84,6 +84,13 @@ public class TableRowkeyPair implements WritableComparable<TableRowkeyPair> { WritableUtils.writeString(output,tableName); rowkey.write(output); } + + @Override + public int hashCode() { + int result = this.tableName.hashCode(); + result = 31 * result + this.rowkey.hashCode(); + return result; + } @Override public int compareTo(TableRowkeyPair other) { @@ -95,40 +102,8 @@ public class TableRowkeyPair implements WritableComparable<TableRowkeyPair> { } } - /** Comparator optimized for <code>TableRowkeyPair</code>. */ - public static class Comparator extends WritableComparator { - private BytesWritable.Comparator comparator = new BytesWritable.Comparator(); - - public Comparator() { - super(TableRowkeyPair.class); - } - - @Override - public int compare(byte[] b1, int s1, int l1, byte[] b2, int s2, int l2) { - try { - int vintL1 = WritableUtils.decodeVIntSize(b1[s1]); - int vintL2 = WritableUtils.decodeVIntSize(b2[s2]); - int strL1 = readVInt(b1, s1); - int strL2 = readVInt(b2, s2); - int cmp = compareBytes(b1, s1 + vintL1, strL1, b2, s2 + vintL2, strL2); - if (cmp != 0) { - return cmp; - } - int vintL3 = WritableUtils.decodeVIntSize(b1[s1 + vintL1 + strL1]); - int vintL4 = WritableUtils.decodeVIntSize(b2[s2 + vintL2 + strL2]); - int strL3 = readVInt(b1, s1 + vintL1 + strL1); - int strL4 = readVInt(b2, s2 + vintL2 + strL2); - return comparator.compare(b1, s1 + vintL1 + strL1 + vintL3, strL3, b2, s2 - + vintL2 + strL2 + vintL4, strL4); - - } catch(Exception ex) { - throw new IllegalArgumentException(ex); - } - } - } - static { - WritableComparator.define(TableRowkeyPair.class, new Comparator()); + WritableComparator.define(TableRowkeyPair.class, new BytesWritable.Comparator()); } } http://git-wip-us.apache.org/repos/asf/phoenix/blob/12f6a6f4/phoenix-core/src/test/java/org/apache/phoenix/mapreduce/bulkload/TestTableRowkeyPair.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/test/java/org/apache/phoenix/mapreduce/bulkload/TestTableRowkeyPair.java b/phoenix-core/src/test/java/org/apache/phoenix/mapreduce/bulkload/TestTableRowkeyPair.java new file mode 100644 index 0000000..1fee4bb --- /dev/null +++ b/phoenix-core/src/test/java/org/apache/phoenix/mapreduce/bulkload/TestTableRowkeyPair.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.phoenix.mapreduce.bulkload; + +import java.io.ByteArrayOutputStream; +import java.io.DataOutputStream; +import java.io.IOException; + +import org.apache.hadoop.hbase.io.ImmutableBytesWritable; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.io.BytesWritable; +import org.junit.Assert; +import org.junit.Test; + +/** + * Tests for {@linkplain TableRowkeyPair} + */ +public class TestTableRowkeyPair { + + @Test + public void testRowkeyPair() throws IOException { + testsRowsKeys("first", "aa", "first", "aa", 0); + testsRowsKeys("first", "aa", "first", "ab", -1); + testsRowsKeys("second", "aa", "first", "aa", 1); + testsRowsKeys("first", "aa", "first", "aaa", -1); + } + + private void testsRowsKeys(String aTable, String akey, String bTable, String bkey, int expectedSignum) throws IOException { + + final ImmutableBytesWritable arowkey = new ImmutableBytesWritable(Bytes.toBytes(akey)); + TableRowkeyPair pair1 = new TableRowkeyPair(aTable, arowkey); + + ImmutableBytesWritable browkey = new ImmutableBytesWritable(Bytes.toBytes(bkey)); + TableRowkeyPair pair2 = new TableRowkeyPair(bTable, browkey); + + BytesWritable.Comparator comparator = new BytesWritable.Comparator(); + try( ByteArrayOutputStream baosA = new ByteArrayOutputStream(); + ByteArrayOutputStream baosB = new ByteArrayOutputStream()) { + + pair1.write(new DataOutputStream(baosA)); + pair2.write(new DataOutputStream(baosB)); + Assert.assertEquals(expectedSignum , signum(pair1.compareTo(pair2))); + Assert.assertEquals(expectedSignum , signum(comparator.compare(baosA.toByteArray(), 0, baosA.size(), baosB.toByteArray(), 0, baosB.size()))); + Assert.assertEquals(expectedSignum, -signum(comparator.compare(baosB.toByteArray(), 0, baosB.size(), baosA.toByteArray(), 0, baosA.size()))); + } + + } + + private int signum(int i) { + return i > 0 ? 1: (i == 0 ? 0: -1); + } +}