http://git-wip-us.apache.org/repos/asf/phoenix/blob/3c7ff99b/phoenix-core/src/test/java/org/apache/phoenix/query/ConnectionlessTest.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/test/java/org/apache/phoenix/query/ConnectionlessTest.java b/phoenix-core/src/test/java/org/apache/phoenix/query/ConnectionlessTest.java index 089c5f1..4571115 100644 --- a/phoenix-core/src/test/java/org/apache/phoenix/query/ConnectionlessTest.java +++ b/phoenix-core/src/test/java/org/apache/phoenix/query/ConnectionlessTest.java @@ -95,8 +95,8 @@ public class ConnectionlessTest { " entity_history_id char(12) not null,\n" + " created_by varchar,\n" + " created_date date\n" + - " CONSTRAINT pk PRIMARY KEY (organization_id, key_prefix, entity_history_id) ) " + - (saltBuckets == null ? "" : (PhoenixDatabaseMetaData.SALT_BUCKETS + "=" + saltBuckets)); + " CONSTRAINT pk PRIMARY KEY (organization_id, key_prefix, entity_history_id) ) COLUMN_ENCODED_BYTES=4 " + + (saltBuckets == null ? "" : " , " + (PhoenixDatabaseMetaData.SALT_BUCKETS + "=" + saltBuckets)); Properties props = new Properties(); Connection conn = DriverManager.getConnection(getUrl(), props); PreparedStatement statement = conn.prepareStatement(dmlStmt); @@ -141,31 +141,29 @@ public class ConnectionlessTest { assertTrue(iterator.hasNext()); kv = iterator.next(); assertArrayEquals(expectedRowKey1, kv.getRow()); - assertEquals(name1, PVarchar.INSTANCE.toObject(kv.getValue())); - assertTrue(iterator.hasNext()); + assertEquals(QueryConstants.EMPTY_COLUMN_VALUE, PVarchar.INSTANCE.toObject(kv.getValue())); kv = iterator.next(); assertArrayEquals(expectedRowKey1, kv.getRow()); - assertEquals(now, PDate.INSTANCE.toObject(kv.getValue())); + assertEquals(name1, PVarchar.INSTANCE.toObject(kv.getValue())); assertTrue(iterator.hasNext()); kv = iterator.next(); assertArrayEquals(expectedRowKey1, kv.getRow()); - assertEquals(QueryConstants.EMPTY_COLUMN_VALUE, PVarchar.INSTANCE.toObject(kv.getValue())); + assertEquals(now, PDate.INSTANCE.toObject(kv.getValue())); } private static void assertRow2(Iterator<KeyValue> iterator, byte[] expectedRowKey2) { KeyValue kv; - assertTrue(iterator.hasNext()); kv = iterator.next(); assertArrayEquals(expectedRowKey2, kv.getRow()); - assertEquals(name2, PVarchar.INSTANCE.toObject(kv.getValue())); + assertEquals(QueryConstants.EMPTY_COLUMN_VALUE, PVarchar.INSTANCE.toObject(kv.getValue())); assertTrue(iterator.hasNext()); kv = iterator.next(); assertArrayEquals(expectedRowKey2, kv.getRow()); - assertEquals(now, PDate.INSTANCE.toObject(kv.getValue())); + assertEquals(name2, PVarchar.INSTANCE.toObject(kv.getValue())); assertTrue(iterator.hasNext()); kv = iterator.next(); assertArrayEquals(expectedRowKey2, kv.getRow()); - assertEquals(QueryConstants.EMPTY_COLUMN_VALUE, PVarchar.INSTANCE.toObject(kv.getValue())); + assertEquals(now, PDate.INSTANCE.toObject(kv.getValue())); } @Test
http://git-wip-us.apache.org/repos/asf/phoenix/blob/3c7ff99b/phoenix-core/src/test/java/org/apache/phoenix/query/EncodedColumnQualifierCellsListTest.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/test/java/org/apache/phoenix/query/EncodedColumnQualifierCellsListTest.java b/phoenix-core/src/test/java/org/apache/phoenix/query/EncodedColumnQualifierCellsListTest.java new file mode 100644 index 0000000..1052184 --- /dev/null +++ b/phoenix-core/src/test/java/org/apache/phoenix/query/EncodedColumnQualifierCellsListTest.java @@ -0,0 +1,608 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.phoenix.query; + +import static org.apache.phoenix.schema.PTable.QualifierEncodingScheme.FOUR_BYTE_QUALIFIERS; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.ConcurrentModificationException; +import java.util.Iterator; +import java.util.List; +import java.util.ListIterator; +import java.util.NoSuchElementException; + +import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.phoenix.schema.tuple.EncodedColumnQualiferCellsList; +import org.junit.Test; + +public class EncodedColumnQualifierCellsListTest { + + private static final byte[] row = Bytes.toBytes("row"); + private static final byte[] cf = Bytes.toBytes("cf"); + + + @Test + public void testIterator() { + EncodedColumnQualiferCellsList list = new EncodedColumnQualiferCellsList(11, 16, FOUR_BYTE_QUALIFIERS); + Cell[] cells = new Cell[7]; + int i = 0; + populateListAndArray(list, cells); + Iterator itr = list.iterator(); + assertTrue(itr.hasNext()); + + // test itr.next() + i = 0; + while (itr.hasNext()) { + assertEquals(cells[i++], itr.next()); + } + + assertEquals(7, list.size()); + + // test itr.remove() + itr = list.iterator(); + i = 0; + int numRemoved = 0; + try { + itr.remove(); + fail("Remove not allowed till next() is called"); + } catch (IllegalStateException expected) {} + + while (itr.hasNext()) { + assertEquals(cells[i++], itr.next()); + itr.remove(); + numRemoved++; + } + assertEquals("Number of elements removed should have been the size of the list", 7, numRemoved); + } + + @Test + public void testSize() { + EncodedColumnQualiferCellsList list = new EncodedColumnQualiferCellsList(11, 16, FOUR_BYTE_QUALIFIERS); + assertEquals(0, list.size()); + + populateList(list); + + assertEquals(7, list.size()); + int originalSize = list.size(); + + Iterator itr = list.iterator(); + while (itr.hasNext()) { + itr.next(); + itr.remove(); + assertEquals(--originalSize, list.size()); + } + } + + @Test + public void testIsEmpty() throws Exception { + EncodedColumnQualiferCellsList list = new EncodedColumnQualiferCellsList(11, 16, FOUR_BYTE_QUALIFIERS); + assertTrue(list.isEmpty()); + populateList(list); + assertFalse(list.isEmpty()); + Iterator itr = list.iterator(); + while (itr.hasNext()) { + itr.next(); + itr.remove(); + if (itr.hasNext()) { + assertFalse(list.isEmpty()); + } + } + assertTrue(list.isEmpty()); + } + + @Test + public void testContains() throws Exception { + EncodedColumnQualiferCellsList list = new EncodedColumnQualiferCellsList(11, 16, FOUR_BYTE_QUALIFIERS); + Cell[] cells = new Cell[7]; + populateListAndArray(list, cells); + + for (Cell c : cells) { + assertTrue(list.contains(c)); + } + assertFalse(list.contains(KeyValue.createFirstOnRow(row, cf, FOUR_BYTE_QUALIFIERS.encode(13)))); + } + + @Test + public void testToArrayWithParam() throws Exception { + EncodedColumnQualiferCellsList list = new EncodedColumnQualiferCellsList(11, 16, FOUR_BYTE_QUALIFIERS); + Cell[] cells = new Cell[7]; + populateListAndArray(list, cells); + Cell[] array = list.toArray(new Cell[0]); + assertTrue(Arrays.equals(cells, array)); + } + + @Test + public void testToArrayWithoutParam() throws Exception { + EncodedColumnQualiferCellsList list = new EncodedColumnQualiferCellsList(11, 16, FOUR_BYTE_QUALIFIERS); + Cell[] cells = new Cell[7]; + populateListAndArray(list, cells); + Object[] array = list.toArray(); + assertTrue(Arrays.equals(cells, array)); + } + + @Test + public void testRemove() throws Exception { + EncodedColumnQualiferCellsList list = new EncodedColumnQualiferCellsList(11, 16, FOUR_BYTE_QUALIFIERS); + Cell[] cells = new Cell[7]; + populateListAndArray(list, cells); + assertTrue(list.remove(cells[0])); + assertEquals(6, list.size()); + assertTrue(list.remove(cells[6])); + assertEquals(5, list.size()); + assertTrue(list.remove(cells[3])); + assertEquals(4, list.size()); + assertFalse(list.remove(KeyValue.createFirstOnRow(row, cf, FOUR_BYTE_QUALIFIERS.encode(13)))); + assertEquals(4, list.size()); + } + + @Test + public void testContainsAll() throws Exception { + EncodedColumnQualiferCellsList list1 = new EncodedColumnQualiferCellsList(11, 16, FOUR_BYTE_QUALIFIERS); + populateList(list1); + EncodedColumnQualiferCellsList list2 = new EncodedColumnQualiferCellsList(11, 16, FOUR_BYTE_QUALIFIERS); + populateList(list2); + assertTrue(list1.containsAll(list2)); + list2.remove(KeyValue.createFirstOnRow(row, cf, FOUR_BYTE_QUALIFIERS.encode(11))); + assertTrue(list1.containsAll(list2)); + assertFalse(list2.containsAll(list1)); + list2.add(KeyValue.createFirstOnRow(row, cf, FOUR_BYTE_QUALIFIERS.encode(13))); + assertFalse(list1.containsAll(list2)); + assertFalse(list2.containsAll(list1)); + List<Cell> arrayList = new ArrayList<>(); + populateList(arrayList); + assertTrue(list1.containsAll(arrayList)); + } + + @Test + public void testAddAll() throws Exception { + EncodedColumnQualiferCellsList list1 = new EncodedColumnQualiferCellsList(11, 16, FOUR_BYTE_QUALIFIERS); + populateList(list1); + EncodedColumnQualiferCellsList list2 = new EncodedColumnQualiferCellsList(11, 16, FOUR_BYTE_QUALIFIERS); + populateList(list2); + /* + * Note that we don't care about equality of the element being added with the element already + * present at the index. + */ + assertTrue(list1.addAll(list2)); + } + + @Test + public void testAddAllAtIndexFails() throws Exception { + EncodedColumnQualiferCellsList list = new EncodedColumnQualiferCellsList(11, 16, FOUR_BYTE_QUALIFIERS); + populateList(list); + try { + list.addAll(0, new ArrayList<Cell>()); + } catch (UnsupportedOperationException expected) { + } + } + + @Test + public void testRemoveAll() throws Exception { + EncodedColumnQualiferCellsList list1 = new EncodedColumnQualiferCellsList(11, 16, FOUR_BYTE_QUALIFIERS); + populateList(list1); + ArrayList<Cell> list2 = new ArrayList<>(); + populateList(list2); + assertTrue(list1.removeAll(list2)); + assertTrue(list1.isEmpty()); + assertFalse(list2.isEmpty()); + } + + @Test + public void testRetainAll() throws Exception { + EncodedColumnQualiferCellsList list1 = new EncodedColumnQualiferCellsList(11, 16, FOUR_BYTE_QUALIFIERS); + populateList(list1); + EncodedColumnQualiferCellsList list2 = new EncodedColumnQualiferCellsList(11, 16, FOUR_BYTE_QUALIFIERS); + populateList(list2); + // retainAll won't be modifying the list1 since they both have the same elements equality wise + assertFalse(list1.retainAll(list2)); + list2.remove(KeyValue.createFirstOnRow(row, cf, FOUR_BYTE_QUALIFIERS.encode(12))); + assertTrue(list1.retainAll(list2)); + assertEquals(list1.size(), list2.size()); + for (Cell c : list1) { + assertTrue(list2.contains(c)); + } + } + + @Test + public void testClear() throws Exception { + EncodedColumnQualiferCellsList list = new EncodedColumnQualiferCellsList(11, 16, FOUR_BYTE_QUALIFIERS); + populateList(list); + list.clear(); + assertTrue(list.isEmpty()); + assertEquals(0, list.size()); + } + + @Test + public void testGetIndex() throws Exception { + EncodedColumnQualiferCellsList list = new EncodedColumnQualiferCellsList(11, 16, FOUR_BYTE_QUALIFIERS); + Cell[] cells = new Cell[7]; + populateListAndArray(list, cells); + for (int i = 0; i < cells.length; i++) { + assertEquals(cells[i], list.get(i)); + } + } + + @Test + public void testIndexOf() throws Exception { + EncodedColumnQualiferCellsList list = new EncodedColumnQualiferCellsList(11, 16, FOUR_BYTE_QUALIFIERS); + Cell[] cells = new Cell[7]; + populateListAndArray(list, cells); + for (int i = 0; i < cells.length; i++) { + assertEquals(i, list.indexOf(cells[i])); + } + } + + @Test + public void testLastIndexOf() throws Exception { + EncodedColumnQualiferCellsList list = new EncodedColumnQualiferCellsList(11, 16, FOUR_BYTE_QUALIFIERS); + Cell[] cells = new Cell[7]; + populateListAndArray(list, cells); + for (int i = 0; i < cells.length; i++) { + assertEquals(i, list.lastIndexOf(cells[i])); + } + } + + @Test + public void testListIterator() throws Exception { + EncodedColumnQualiferCellsList list = new EncodedColumnQualiferCellsList(11, 16, FOUR_BYTE_QUALIFIERS); + Cell[] cells = new Cell[7]; + int i = 0; + populateListAndArray(list, cells); + ListIterator<Cell> itr = list.listIterator(); + assertTrue(itr.hasNext()); + + // test itr.next() + i = 0; + while (itr.hasNext()) { + assertEquals(cells[i++], itr.next()); + } + + assertEquals(7, list.size()); + + // test itr.remove() + itr = list.listIterator(); + i = 0; + int numRemoved = 0; + try { + itr.remove(); + fail("Remove not allowed till next() is called"); + } catch (IllegalStateException expected) {} + + while (itr.hasNext()) { + assertEquals(cells[i++], itr.next()); + itr.remove(); + numRemoved++; + } + assertEquals("Number of elements removed should have been the size of the list", 7, numRemoved); + assertTrue(list.isEmpty()); + } + + @Test + public void testListIteratorSet() { + EncodedColumnQualiferCellsList list = new EncodedColumnQualiferCellsList(11, 16, FOUR_BYTE_QUALIFIERS); + Cell[] array = new Cell[7]; + populateListAndArray(list, array); + ListIterator<Cell> itr = list.listIterator(); + // This cell is KeyValue.createFirstOnRow(row, cf, getEncodedColumnQualifier(12)) + final Cell validCell = array[4]; + // This cell is KeyValue.createFirstOnRow(row, cf, getEncodedColumnQualifier(14)) + final Cell invalidCell = array[5]; + String validCellName = "Valid Cell"; + String invalidCellName = "Invalid Cell"; + Cell validReplacementCell = new DelegateCell(validCell, validCellName); + Cell invalidReplacementCell = new DelegateCell(invalidCell, invalidCellName); + int i = 0; + while (itr.hasNext()) { + Cell c = itr.next(); + if (i == 4) { + itr.set(validReplacementCell); + } + if (i == 6) { + try { + itr.set(invalidReplacementCell); + fail("This should have failed since " + invalidReplacementCell + " cannot be added where " + c + " is."); + } catch (IllegalArgumentException expected) { + } + } + i++; + } + itr = list.listIterator(); + i = 0; + // Assert that the valid cell was added and invalid cell wasn't. + while (itr.hasNext()) { + Cell c = itr.next(); + if (i == 4) { + assertEquals(validCellName, c.toString()); + } + if (i == 6) { + assertNotEquals(invalidCellName, c.toString()); + } + i++; + } + } + + @Test + public void testListIteratorNextAndPrevious() throws Exception { + EncodedColumnQualiferCellsList list = new EncodedColumnQualiferCellsList(11, 16, FOUR_BYTE_QUALIFIERS); + Cell[] array = new Cell[7]; + populateListAndArray(list, array); + ListIterator<Cell> itr = list.listIterator(); + try { + itr.previous(); + fail("Call to itr.previous() should have failed since the iterator hasn't been moved forward yet"); + } catch (NoSuchElementException expected) { + + } + Cell c = itr.next(); + Cell d = itr.previous(); + Cell e = itr.next(); + Cell f = itr.previous(); + assertTrue(c.equals(d) && c.equals(f) && c.equals(e)); + itr = list.listIterator(); + int i = 0; + assertEquals(array[i++], itr.next()); + assertEquals(array[i++], itr.next()); + assertEquals(array[i++], itr.next()); + assertEquals(array[--i], itr.previous()); + assertEquals(array[--i], itr.previous()); + assertEquals(array[i++], itr.next()); + + // move itr forward till next() is exhausted + while (itr.hasNext()) { + itr.next(); + } + i = 6; + while (itr.hasPrevious()) { + assertEquals(array[i--], itr.previous()); + } + assertEquals("Not all elements navigated using previous()", -1, i); + // now that previous is exhausted, move itr() forward till next() is exhausted + i = 0; + while (itr.hasNext()) { + assertEquals(array[i++], itr.next()); + } + assertEquals("Not all elements navigated using next()", 7, i); + } + + @Test + public void testSetNull() throws Exception { + EncodedColumnQualiferCellsList list = new EncodedColumnQualiferCellsList(11, 16, FOUR_BYTE_QUALIFIERS); + try { + list.add(null); + fail("Adding null elements to the list is not allowed"); + } catch (NullPointerException expected) { + + } + } + + @Test + public void testFailFastIterator() throws Exception { + EncodedColumnQualiferCellsList list = new EncodedColumnQualiferCellsList(11, 16, FOUR_BYTE_QUALIFIERS); + populateList(list); + int i = 0; + Iterator<Cell> itr = list.iterator(); + while (itr.hasNext()) { + i++; + try { + itr.next(); + list.add(KeyValue.createFirstOnRow(row, cf, FOUR_BYTE_QUALIFIERS.encode(0))); + if (i == 2) { + fail("ConcurrentModificationException should have been thrown as the list is being modified while being iterated through"); + } + } catch (ConcurrentModificationException expected) { + assertEquals("Exception should have been thrown when getting the second element", + 2, i); + break; + } + } + } + + @Test + public void testFailFastListIterator() throws Exception { + EncodedColumnQualiferCellsList list = new EncodedColumnQualiferCellsList(11, 16, FOUR_BYTE_QUALIFIERS); + populateList(list); + ListIterator<Cell> itr = list.listIterator(); + itr.next(); + list.add(KeyValue.createFirstOnRow(row, cf, FOUR_BYTE_QUALIFIERS.encode(0))); + try { + itr.next(); + fail("ConcurrentModificationException should have been thrown as the list was modified without using iterator"); + } catch (ConcurrentModificationException expected) { + + } + list = new EncodedColumnQualiferCellsList(11, 16, FOUR_BYTE_QUALIFIERS); + populateList(list); + itr = list.listIterator(); + itr.next(); + itr.next(); + itr.remove(); + itr.next(); + list.remove(KeyValue.createFirstOnRow(row, cf, FOUR_BYTE_QUALIFIERS.encode(0))); + try { + itr.next(); + fail("ConcurrentModificationException should have been thrown as the list was modified without using iterator"); + } catch (ConcurrentModificationException expected) { + + } + } + + private void populateListAndArray(List<Cell> list, Cell[] cells) { + // add elements in reserved range + list.add(cells[0] = KeyValue.createFirstOnRow(row, cf, FOUR_BYTE_QUALIFIERS.encode(0))); + list.add(cells[1] = KeyValue.createFirstOnRow(row, cf, FOUR_BYTE_QUALIFIERS.encode(5))); + list.add(cells[2] = KeyValue.createFirstOnRow(row, cf, FOUR_BYTE_QUALIFIERS.encode(10))); + + // add elements in qualifier range + list.add(cells[6] = KeyValue.createFirstOnRow(row, cf, FOUR_BYTE_QUALIFIERS.encode(16))); + list.add(cells[4] = KeyValue.createFirstOnRow(row, cf, FOUR_BYTE_QUALIFIERS.encode(12))); + list.add(cells[5] = KeyValue.createFirstOnRow(row, cf, FOUR_BYTE_QUALIFIERS.encode(14))); + list.add(cells[3] = KeyValue.createFirstOnRow(row, cf, FOUR_BYTE_QUALIFIERS.encode(11))); + } + + private void populateList(List<Cell> list) { + // add elements in reserved range + list.add(KeyValue.createFirstOnRow(row, cf, FOUR_BYTE_QUALIFIERS.encode(0))); + list.add(KeyValue.createFirstOnRow(row, cf, FOUR_BYTE_QUALIFIERS.encode(5))); + list.add(KeyValue.createFirstOnRow(row, cf, FOUR_BYTE_QUALIFIERS.encode(10))); + + // add elements in qualifier range + list.add(KeyValue.createFirstOnRow(row, cf, FOUR_BYTE_QUALIFIERS.encode(16))); + list.add(KeyValue.createFirstOnRow(row, cf, FOUR_BYTE_QUALIFIERS.encode(12))); + list.add(KeyValue.createFirstOnRow(row, cf, FOUR_BYTE_QUALIFIERS.encode(14))); + list.add(KeyValue.createFirstOnRow(row, cf, FOUR_BYTE_QUALIFIERS.encode(11))); + } + + private class DelegateCell implements Cell { + private final Cell delegate; + private final String name; + public DelegateCell(Cell delegate, String name) { + this.delegate = delegate; + this.name = name; + } + + @Override + public int getValueOffset() { + return delegate.getValueOffset(); + } + + @Override + public int getValueLength() { + return delegate.getValueLength(); + } + + @Override + public byte[] getValueArray() { + return delegate.getValueArray(); + } + + @Override + public byte[] getValue() { + return delegate.getValue(); + } + + @Override + public byte getTypeByte() { + return delegate.getTypeByte(); + } + + @Override + public long getTimestamp() { + return delegate.getTimestamp(); + } + + @Override + public int getTagsOffset() { + return delegate.getTagsOffset(); + } + + @Override + public byte[] getTagsArray() { + return delegate.getTagsArray(); + } + + @Override + public int getRowOffset() { + return delegate.getRowOffset(); + } + + @Override + public short getRowLength() { + return delegate.getRowLength(); + } + + @Override + public byte[] getRowArray() { + return delegate.getRowArray(); + } + + @Override + public byte[] getRow() { + return delegate.getRow(); + } + + @Override + public int getQualifierOffset() { + return delegate.getQualifierOffset(); + } + + @Override + public int getQualifierLength() { + return delegate.getQualifierLength(); + } + + @Override + public byte[] getQualifierArray() { + return delegate.getQualifierArray(); + } + + @Override + public byte[] getQualifier() { + return delegate.getQualifier(); + } + + @Override + public long getMvccVersion() { + return delegate.getMvccVersion(); + } + + @Override + public int getFamilyOffset() { + return delegate.getFamilyOffset(); + } + + @Override + public byte getFamilyLength() { + return delegate.getFamilyLength(); + } + + @Override + public byte[] getFamilyArray() { + return delegate.getFamilyArray(); + } + + @Override + public byte[] getFamily() { + return delegate.getFamily(); + } + + @Override + public String toString() { + return name; + } + + @Override + public long getSequenceId() { + return delegate.getSequenceId(); + } + + @Override + public int getTagsLength() { + return delegate.getTagsLength(); + } + + } + +} http://git-wip-us.apache.org/repos/asf/phoenix/blob/3c7ff99b/phoenix-core/src/test/java/org/apache/phoenix/schema/ImmutableStorageSchemeTest.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/test/java/org/apache/phoenix/schema/ImmutableStorageSchemeTest.java b/phoenix-core/src/test/java/org/apache/phoenix/schema/ImmutableStorageSchemeTest.java new file mode 100644 index 0000000..d8c5cdb --- /dev/null +++ b/phoenix-core/src/test/java/org/apache/phoenix/schema/ImmutableStorageSchemeTest.java @@ -0,0 +1,182 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.phoenix.schema; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import java.util.Arrays; +import java.util.List; + +import org.apache.hadoop.hbase.io.ImmutableBytesWritable; +import org.apache.phoenix.expression.DelegateExpression; +import org.apache.phoenix.expression.Expression; +import org.apache.phoenix.expression.LiteralExpression; +import org.apache.phoenix.expression.SingleCellConstructorExpression; +import org.apache.phoenix.hbase.index.util.ImmutableBytesPtr; +import org.apache.phoenix.query.QueryConstants; +import org.apache.phoenix.schema.PTable.ImmutableStorageScheme; +import org.apache.phoenix.schema.tuple.Tuple; +import org.apache.phoenix.schema.types.PVarbinary; +import org.apache.phoenix.util.ByteUtil; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.junit.runners.Parameterized.Parameters; + +import com.google.common.collect.Lists; + +@RunWith(Parameterized.class) +public class ImmutableStorageSchemeTest { + + protected static final LiteralExpression CONSTANT_EXPRESSION = LiteralExpression.newConstant(QueryConstants.EMPTY_COLUMN_VALUE_BYTES); + protected static final byte[] BYTE_ARRAY1 = new byte[]{1,2,3,4,5}; + protected static final byte[] BYTE_ARRAY2 = new byte[]{6,7,8}; + protected Expression FALSE_EVAL_EXPRESSION = new DelegateExpression(LiteralExpression.newConstant(null)) { + @Override + public boolean evaluate(Tuple tuple, ImmutableBytesWritable ptr) { + return false; + } + }; + private ImmutableStorageScheme immutableStorageScheme; + + @Parameters(name="ImmutableStorageSchemeTest_immutableStorageScheme={0}}") // name is used by failsafe as file name in reports + public static ImmutableStorageScheme[] data() { + ImmutableStorageScheme[] values = ImmutableStorageScheme.values(); + // skip ONE_CELL_PER_COLUMN + return Arrays.copyOfRange(values, 1, values.length); + } + + public ImmutableStorageSchemeTest(ImmutableStorageScheme immutableStorageScheme) { + this.immutableStorageScheme = immutableStorageScheme; + } + + @Test + public void testWithExpressionsThatEvaluatetoFalse() throws Exception { + List<Expression> children = Lists.newArrayListWithExpectedSize(4); + children.add(CONSTANT_EXPRESSION); + children.add(FALSE_EVAL_EXPRESSION); + children.add(LiteralExpression.newConstant(BYTE_ARRAY1, PVarbinary.INSTANCE)); + children.add(FALSE_EVAL_EXPRESSION); + children.add(LiteralExpression.newConstant(BYTE_ARRAY2, PVarbinary.INSTANCE)); + SingleCellConstructorExpression singleCellConstructorExpression = new SingleCellConstructorExpression(immutableStorageScheme, children); + ImmutableBytesPtr ptr = new ImmutableBytesPtr(); + singleCellConstructorExpression.evaluate(null, ptr); + + ImmutableBytesPtr ptrCopy = new ImmutableBytesPtr(ptr); + ColumnValueDecoder decoder = immutableStorageScheme.getDecoder(); + assertTrue(decoder.decode(ptrCopy, 0)); + assertArrayEquals(QueryConstants.EMPTY_COLUMN_VALUE_BYTES, ptrCopy.copyBytesIfNecessary()); + ptrCopy = new ImmutableBytesPtr(ptr); + assertFalse(decoder.decode(ptrCopy, 1)); + assertArrayEquals(ByteUtil.EMPTY_BYTE_ARRAY, ptrCopy.copyBytesIfNecessary()); + ptrCopy = new ImmutableBytesPtr(ptr); + assertTrue(decoder.decode(ptrCopy, 2)); + assertArrayEquals(BYTE_ARRAY1, ptrCopy.copyBytesIfNecessary()); + ptrCopy = new ImmutableBytesPtr(ptr); + assertFalse(decoder.decode(ptrCopy, 3)); + assertArrayEquals(ByteUtil.EMPTY_BYTE_ARRAY, ptrCopy.copyBytesIfNecessary()); + ptrCopy = new ImmutableBytesPtr(ptr); + assertTrue(decoder.decode(ptrCopy, 4)); + assertArrayEquals(BYTE_ARRAY2, ptrCopy.copyBytesIfNecessary()); + } + + @Test + public void testWithMaxOffsetLargerThanShortMax() throws Exception { + int numElements = Short.MAX_VALUE+2; + List<Expression> children = Lists.newArrayListWithExpectedSize(numElements); + for (int i=0; i<numElements; ++i) { + children.add(CONSTANT_EXPRESSION); + } + SingleCellConstructorExpression singleCellConstructorExpression = new SingleCellConstructorExpression(immutableStorageScheme, children); + ImmutableBytesPtr ptr = new ImmutableBytesPtr(); + singleCellConstructorExpression.evaluate(null, ptr); + + ImmutableBytesPtr ptrCopy = new ImmutableBytesPtr(ptr); + ColumnValueDecoder decoder = immutableStorageScheme.getDecoder(); + assertTrue(decoder.decode(ptrCopy, 0)); + assertArrayEquals(QueryConstants.EMPTY_COLUMN_VALUE_BYTES, ptrCopy.copyBytesIfNecessary()); + + ptrCopy = new ImmutableBytesPtr(ptr); + assertTrue(decoder.decode(ptrCopy, 14999)); + assertArrayEquals(QueryConstants.EMPTY_COLUMN_VALUE_BYTES, ptrCopy.copyBytesIfNecessary()); + + ptrCopy = new ImmutableBytesPtr(ptr); + assertTrue(decoder.decode(ptrCopy, numElements-1)); + assertArrayEquals(QueryConstants.EMPTY_COLUMN_VALUE_BYTES, ptrCopy.copyBytesIfNecessary()); + } + + @Test + public void testWithMaxOffsetSmallerThanShortMin() throws Exception { + int numElements = Short.MAX_VALUE+2; + List<Expression> children = Lists.newArrayListWithExpectedSize(numElements); + for (int i=0; i<=numElements; i+=2) { + children.add(CONSTANT_EXPRESSION); + children.add(FALSE_EVAL_EXPRESSION); + } + SingleCellConstructorExpression singleCellConstructorExpression = new SingleCellConstructorExpression(immutableStorageScheme, children); + ImmutableBytesPtr ptr = new ImmutableBytesPtr(); + singleCellConstructorExpression.evaluate(null, ptr); + + ImmutableBytesPtr ptrCopy = new ImmutableBytesPtr(ptr); + ColumnValueDecoder decoder = immutableStorageScheme.getDecoder(); + assertTrue(decoder.decode(ptrCopy, 0)); + assertArrayEquals(QueryConstants.EMPTY_COLUMN_VALUE_BYTES, ptrCopy.copyBytesIfNecessary()); + + ptrCopy = new ImmutableBytesPtr(ptr); + assertFalse(decoder.decode(ptrCopy, 1)); + assertArrayEquals(ByteUtil.EMPTY_BYTE_ARRAY, ptrCopy.copyBytesIfNecessary()); + + ptrCopy = new ImmutableBytesPtr(ptr); + assertTrue(decoder.decode(ptrCopy, numElements-1)); + assertArrayEquals(QueryConstants.EMPTY_COLUMN_VALUE_BYTES, ptrCopy.copyBytesIfNecessary()); + + ptrCopy = new ImmutableBytesPtr(ptr); + assertFalse(decoder.decode(ptrCopy, numElements)); + assertArrayEquals(ByteUtil.EMPTY_BYTE_ARRAY, ptrCopy.copyBytesIfNecessary()); + } + + @Test + public void testLeadingNulls() throws Exception { + List<Expression> children = Lists.newArrayListWithExpectedSize(4); + LiteralExpression nullExpression = LiteralExpression.newConstant(null); + children.add(nullExpression); + children.add(nullExpression); + children.add(LiteralExpression.newConstant(BYTE_ARRAY1, PVarbinary.INSTANCE)); + children.add(LiteralExpression.newConstant(BYTE_ARRAY2, PVarbinary.INSTANCE)); + SingleCellConstructorExpression singleCellConstructorExpression = new SingleCellConstructorExpression(immutableStorageScheme, children); + ImmutableBytesPtr ptr = new ImmutableBytesPtr(); + singleCellConstructorExpression.evaluate(null, ptr); + + ImmutableBytesPtr ptrCopy = new ImmutableBytesPtr(ptr); + ColumnValueDecoder decoder = immutableStorageScheme.getDecoder(); + assertTrue(decoder.decode(ptrCopy, 0)); + assertArrayEquals(ByteUtil.EMPTY_BYTE_ARRAY, ptrCopy.copyBytesIfNecessary()); + ptrCopy = new ImmutableBytesPtr(ptr); + assertTrue(decoder.decode(ptrCopy, 1)); + assertArrayEquals(ByteUtil.EMPTY_BYTE_ARRAY, ptrCopy.copyBytesIfNecessary()); + ptrCopy = new ImmutableBytesPtr(ptr); + assertTrue(decoder.decode(ptrCopy, 2)); + assertArrayEquals(BYTE_ARRAY1, ptrCopy.copyBytesIfNecessary()); + ptrCopy = new ImmutableBytesPtr(ptr); + assertTrue(decoder.decode(ptrCopy, 3)); + assertArrayEquals(BYTE_ARRAY2, ptrCopy.copyBytesIfNecessary()); + } + +} http://git-wip-us.apache.org/repos/asf/phoenix/blob/3c7ff99b/phoenix-core/src/test/java/org/apache/phoenix/schema/types/PDataTypeForArraysTest.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/test/java/org/apache/phoenix/schema/types/PDataTypeForArraysTest.java b/phoenix-core/src/test/java/org/apache/phoenix/schema/types/PDataTypeForArraysTest.java index 333fbf9..2aeeeb8 100644 --- a/phoenix-core/src/test/java/org/apache/phoenix/schema/types/PDataTypeForArraysTest.java +++ b/phoenix-core/src/test/java/org/apache/phoenix/schema/types/PDataTypeForArraysTest.java @@ -324,7 +324,7 @@ public class PDataTypeForArraysTest { PVarchar.INSTANCE, strArr); byte[] bytes = PVarcharArray.INSTANCE.toBytes(arr); ImmutableBytesWritable ptr = new ImmutableBytesWritable(bytes); - PArrayDataType.positionAtArrayElement(ptr, 4, PVarchar.INSTANCE, PVarchar.INSTANCE.getByteSize()); + PArrayDataTypeDecoder.positionAtArrayElement(ptr, 4, PVarchar.INSTANCE, PVarchar.INSTANCE.getByteSize()); int offset = ptr.getOffset(); int length = ptr.getLength(); byte[] bs = ptr.get(); @@ -342,7 +342,7 @@ public class PDataTypeForArraysTest { PVarchar.INSTANCE, strArr); byte[] bytes = PVarcharArray.INSTANCE.toBytes(arr); ImmutableBytesWritable ptr = new ImmutableBytesWritable(bytes); - PArrayDataType.positionAtArrayElement(ptr, 0, PVarchar.INSTANCE, PVarchar.INSTANCE.getByteSize()); + PArrayDataTypeDecoder.positionAtArrayElement(ptr, 0, PVarchar.INSTANCE, PVarchar.INSTANCE.getByteSize()); int offset = ptr.getOffset(); int length = ptr.getLength(); byte[] bs = ptr.get(); @@ -365,7 +365,7 @@ public class PDataTypeForArraysTest { PVarchar.INSTANCE, strArr); byte[] bytes = PVarcharArray.INSTANCE.toBytes(arr); ImmutableBytesWritable ptr = new ImmutableBytesWritable(bytes); - PArrayDataType.positionAtArrayElement(ptr, 3, PVarchar.INSTANCE, PVarchar.INSTANCE.getByteSize()); + PArrayDataTypeDecoder.positionAtArrayElement(ptr, 3, PVarchar.INSTANCE, PVarchar.INSTANCE.getByteSize()); int offset = ptr.getOffset(); int length = ptr.getLength(); byte[] bs = ptr.get(); @@ -402,7 +402,7 @@ public class PDataTypeForArraysTest { PVarchar.INSTANCE, strArr); byte[] bytes = PVarcharArray.INSTANCE.toBytes(arr); ImmutableBytesWritable ptr = new ImmutableBytesWritable(bytes); - PArrayDataType.positionAtArrayElement(ptr, 3, PVarchar.INSTANCE, PVarchar.INSTANCE.getByteSize()); + PArrayDataTypeDecoder.positionAtArrayElement(ptr, 3, PVarchar.INSTANCE, PVarchar.INSTANCE.getByteSize()); int offset = ptr.getOffset(); int length = ptr.getLength(); byte[] bs = ptr.get(); @@ -423,7 +423,7 @@ public class PDataTypeForArraysTest { PVarchar.INSTANCE, strArr); byte[] bytes = PVarcharArray.INSTANCE.toBytes(arr); ImmutableBytesWritable ptr = new ImmutableBytesWritable(bytes); - PArrayDataType.positionAtArrayElement(ptr, 2, PVarchar.INSTANCE, PVarchar.INSTANCE.getByteSize()); + PArrayDataTypeDecoder.positionAtArrayElement(ptr, 2, PVarchar.INSTANCE, PVarchar.INSTANCE.getByteSize()); int offset = ptr.getOffset(); int length = ptr.getLength(); byte[] bs = ptr.get(); @@ -444,7 +444,7 @@ public class PDataTypeForArraysTest { PVarchar.INSTANCE, strArr); byte[] bytes = PVarcharArray.INSTANCE.toBytes(arr); ImmutableBytesWritable ptr = new ImmutableBytesWritable(bytes); - PArrayDataType.positionAtArrayElement(ptr, 2, PVarchar.INSTANCE, PVarchar.INSTANCE.getByteSize()); + PArrayDataTypeDecoder.positionAtArrayElement(ptr, 2, PVarchar.INSTANCE, PVarchar.INSTANCE.getByteSize()); int offset = ptr.getOffset(); int length = ptr.getLength(); byte[] bs = ptr.get(); @@ -464,7 +464,7 @@ public class PDataTypeForArraysTest { PVarchar.INSTANCE, strArr); byte[] bytes = PVarcharArray.INSTANCE.toBytes(arr); ImmutableBytesWritable ptr = new ImmutableBytesWritable(bytes); - PArrayDataType.positionAtArrayElement(ptr, 4, PVarchar.INSTANCE, PVarchar.INSTANCE.getByteSize()); + PArrayDataTypeDecoder.positionAtArrayElement(ptr, 4, PVarchar.INSTANCE, PVarchar.INSTANCE.getByteSize()); int offset = ptr.getOffset(); int length = ptr.getLength(); byte[] bs = ptr.get(); @@ -485,7 +485,7 @@ public class PDataTypeForArraysTest { PVarchar.INSTANCE, strArr); byte[] bytes = PVarcharArray.INSTANCE.toBytes(arr); ImmutableBytesWritable ptr = new ImmutableBytesWritable(bytes); - PArrayDataType.positionAtArrayElement(ptr, 3, PVarchar.INSTANCE, PVarchar.INSTANCE.getByteSize()); + PArrayDataTypeDecoder.positionAtArrayElement(ptr, 3, PVarchar.INSTANCE, PVarchar.INSTANCE.getByteSize()); int offset = ptr.getOffset(); int length = ptr.getLength(); byte[] bs = ptr.get(); @@ -506,7 +506,7 @@ public class PDataTypeForArraysTest { PVarchar.INSTANCE, strArr); byte[] bytes = PVarcharArray.INSTANCE.toBytes(arr); ImmutableBytesWritable ptr = new ImmutableBytesWritable(bytes); - PArrayDataType.positionAtArrayElement(ptr, 4, PVarchar.INSTANCE, PVarchar.INSTANCE.getByteSize()); + PArrayDataTypeDecoder.positionAtArrayElement(ptr, 4, PVarchar.INSTANCE, PVarchar.INSTANCE.getByteSize()); int offset = ptr.getOffset(); int length = ptr.getLength(); byte[] bs = ptr.get(); @@ -528,7 +528,7 @@ public class PDataTypeForArraysTest { PVarchar.INSTANCE, strArr); byte[] bytes = PVarcharArray.INSTANCE.toBytes(arr); ImmutableBytesWritable ptr = new ImmutableBytesWritable(bytes); - PArrayDataType.positionAtArrayElement(ptr, 4, PVarchar.INSTANCE, PVarchar.INSTANCE.getByteSize()); + PArrayDataTypeDecoder.positionAtArrayElement(ptr, 4, PVarchar.INSTANCE, PVarchar.INSTANCE.getByteSize()); int offset = ptr.getOffset(); int length = ptr.getLength(); byte[] bs = ptr.get(); @@ -548,7 +548,7 @@ public class PDataTypeForArraysTest { PVarchar.INSTANCE, strArr); byte[] bytes = PVarcharArray.INSTANCE.toBytes(arr); ImmutableBytesWritable ptr = new ImmutableBytesWritable(bytes); - PArrayDataType.positionAtArrayElement(ptr, 3, PVarchar.INSTANCE, PVarchar.INSTANCE.getByteSize()); + PArrayDataTypeDecoder.positionAtArrayElement(ptr, 3, PVarchar.INSTANCE, PVarchar.INSTANCE.getByteSize()); int offset = ptr.getOffset(); int length = ptr.getLength(); byte[] bs = ptr.get(); @@ -569,7 +569,7 @@ public class PDataTypeForArraysTest { PVarchar.INSTANCE, strArr); byte[] bytes = PVarcharArray.INSTANCE.toBytes(arr); ImmutableBytesWritable ptr = new ImmutableBytesWritable(bytes); - PArrayDataType.positionAtArrayElement(ptr, 3, PVarchar.INSTANCE, PVarchar.INSTANCE.getByteSize()); + PArrayDataTypeDecoder.positionAtArrayElement(ptr, 3, PVarchar.INSTANCE, PVarchar.INSTANCE.getByteSize()); int offset = ptr.getOffset(); int length = ptr.getLength(); byte[] bs = ptr.get(); @@ -590,7 +590,7 @@ public class PDataTypeForArraysTest { PVarchar.INSTANCE, strArr); byte[] bytes = PVarcharArray.INSTANCE.toBytes(arr); ImmutableBytesWritable ptr = new ImmutableBytesWritable(bytes); - PArrayDataType.positionAtArrayElement(ptr, 0, PVarchar.INSTANCE, PVarchar.INSTANCE.getByteSize()); + PArrayDataTypeDecoder.positionAtArrayElement(ptr, 0, PVarchar.INSTANCE, PVarchar.INSTANCE.getByteSize()); int offset = ptr.getOffset(); int length = ptr.getLength(); byte[] bs = ptr.get(); @@ -611,7 +611,7 @@ public class PDataTypeForArraysTest { PVarchar.INSTANCE, strArr); byte[] bytes = PVarcharArray.INSTANCE.toBytes(arr); ImmutableBytesWritable ptr = new ImmutableBytesWritable(bytes); - PArrayDataType.positionAtArrayElement(ptr, 4, PVarchar.INSTANCE, PVarchar.INSTANCE.getByteSize()); + PArrayDataTypeDecoder.positionAtArrayElement(ptr, 4, PVarchar.INSTANCE, PVarchar.INSTANCE.getByteSize()); int offset = ptr.getOffset(); int length = ptr.getLength(); byte[] bs = ptr.get(); @@ -632,7 +632,7 @@ public class PDataTypeForArraysTest { PVarchar.INSTANCE, strArr); byte[] bytes = PVarcharArray.INSTANCE.toBytes(arr); ImmutableBytesWritable ptr = new ImmutableBytesWritable(bytes); - PArrayDataType.positionAtArrayElement(ptr, 4, PVarchar.INSTANCE, PVarchar.INSTANCE.getByteSize()); + PArrayDataTypeDecoder.positionAtArrayElement(ptr, 4, PVarchar.INSTANCE, PVarchar.INSTANCE.getByteSize()); int offset = ptr.getOffset(); int length = ptr.getLength(); byte[] bs = ptr.get(); @@ -649,7 +649,7 @@ public class PDataTypeForArraysTest { PVarchar.INSTANCE, strArr); byte[] bytes = PVarcharArray.INSTANCE.toBytes(arr); ImmutableBytesWritable ptr = new ImmutableBytesWritable(bytes); - PArrayDataType.positionAtArrayElement(ptr, 0, PVarchar.INSTANCE, PVarchar.INSTANCE.getByteSize()); + PArrayDataTypeDecoder.positionAtArrayElement(ptr, 0, PVarchar.INSTANCE, PVarchar.INSTANCE.getByteSize()); int offset = ptr.getOffset(); int length = ptr.getLength(); byte[] bs = ptr.get(); @@ -667,7 +667,7 @@ public class PDataTypeForArraysTest { PVarchar.INSTANCE, strArr); byte[] bytes = PVarcharArray.INSTANCE.toBytes(arr); ImmutableBytesWritable ptr = new ImmutableBytesWritable(bytes); - PArrayDataType.positionAtArrayElement(ptr, 1, PVarchar.INSTANCE, PVarchar.INSTANCE.getByteSize()); + PArrayDataTypeDecoder.positionAtArrayElement(ptr, 1, PVarchar.INSTANCE, PVarchar.INSTANCE.getByteSize()); int offset = ptr.getOffset(); int length = ptr.getLength(); byte[] bs = ptr.get(); @@ -688,7 +688,7 @@ public class PDataTypeForArraysTest { PLongArray.INSTANCE.toObject(arr, PLongArray.INSTANCE); byte[] bytes = PLongArray.INSTANCE.toBytes(arr); ImmutableBytesWritable ptr = new ImmutableBytesWritable(bytes); - PArrayDataType.positionAtArrayElement(ptr, 2, PLong.INSTANCE, PLong.INSTANCE.getByteSize()); + PArrayDataTypeDecoder.positionAtArrayElement(ptr, 2, PLong.INSTANCE, PLong.INSTANCE.getByteSize()); int offset = ptr.getOffset(); int length = ptr.getLength(); byte[] bs = ptr.get(); @@ -1196,7 +1196,7 @@ public class PDataTypeForArraysTest { PhoenixArray arr = new PhoenixArray(PVarchar.INSTANCE, objects); byte[] bytes = PVarcharArray.INSTANCE.toBytes(arr, PVarchar.INSTANCE, SortOrder.DESC); ImmutableBytesWritable ptr = new ImmutableBytesWritable(bytes); - PArrayDataType.positionAtArrayElement(ptr, 2, PVarchar.INSTANCE, null); + PArrayDataTypeDecoder.positionAtArrayElement(ptr, 2, PVarchar.INSTANCE, null); String value = (String)PVarchar.INSTANCE.toObject(ptr, SortOrder.DESC); assertEquals(null, value); } http://git-wip-us.apache.org/repos/asf/phoenix/blob/3c7ff99b/phoenix-core/src/test/java/org/apache/phoenix/util/PhoenixRuntimeTest.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/test/java/org/apache/phoenix/util/PhoenixRuntimeTest.java b/phoenix-core/src/test/java/org/apache/phoenix/util/PhoenixRuntimeTest.java index 430c20b..7b81c8d 100644 --- a/phoenix-core/src/test/java/org/apache/phoenix/util/PhoenixRuntimeTest.java +++ b/phoenix-core/src/test/java/org/apache/phoenix/util/PhoenixRuntimeTest.java @@ -26,6 +26,8 @@ import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; import java.sql.Connection; import java.sql.Date; import java.sql.DriverManager; @@ -37,6 +39,7 @@ import java.util.List; import java.util.Properties; import org.apache.commons.lang.exception.ExceptionUtils; +import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.Pair; import org.apache.phoenix.compile.QueryPlan; import org.apache.phoenix.expression.Expression; @@ -45,6 +48,7 @@ import org.apache.phoenix.jdbc.PhoenixStatement; import org.apache.phoenix.query.BaseConnectionlessQueryTest; import org.apache.phoenix.query.QueryConstants; import org.apache.phoenix.schema.PTable; +import org.apache.phoenix.schema.PTable.QualifierEncodingScheme; import org.apache.phoenix.schema.TableNotFoundException; import org.apache.phoenix.schema.types.PDataType; import org.junit.Test; @@ -342,5 +346,6 @@ public class PhoenixRuntimeTest extends BaseConnectionlessQueryTest { long skewedTs = ts + QueryConstants.MILLIS_IN_DAY; // skew of a day // Even with a day of skew, we won't consider the ts a nanos timestamp assertEquals(skewedTs, PhoenixRuntime.getWallClockTimeFromCellTimeStamp(skewedTs)); - } + } + } http://git-wip-us.apache.org/repos/asf/phoenix/blob/3c7ff99b/phoenix-core/src/test/java/org/apache/phoenix/util/QualifierEncodingSchemeTest.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/test/java/org/apache/phoenix/util/QualifierEncodingSchemeTest.java b/phoenix-core/src/test/java/org/apache/phoenix/util/QualifierEncodingSchemeTest.java new file mode 100644 index 0000000..2b08d7d --- /dev/null +++ b/phoenix-core/src/test/java/org/apache/phoenix/util/QualifierEncodingSchemeTest.java @@ -0,0 +1,119 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.phoenix.util; + +import static org.apache.phoenix.schema.PTable.QualifierEncodingScheme.FOUR_BYTE_QUALIFIERS; +import static org.apache.phoenix.schema.PTable.QualifierEncodingScheme.ONE_BYTE_QUALIFIERS; +import static org.apache.phoenix.schema.PTable.QualifierEncodingScheme.THREE_BYTE_QUALIFIERS; +import static org.apache.phoenix.schema.PTable.QualifierEncodingScheme.TWO_BYTE_QUALIFIERS; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +import org.apache.phoenix.schema.PTable.QualifierEncodingScheme.InvalidQualifierBytesException; +import org.junit.Test; + +public class QualifierEncodingSchemeTest { + + @Test + public void testOneByteQualifierEncodeDecode() { + assertEquals(1, ONE_BYTE_QUALIFIERS.decode(ONE_BYTE_QUALIFIERS.encode(1))); + assertEquals(127, ONE_BYTE_QUALIFIERS.decode(ONE_BYTE_QUALIFIERS.encode(127))); + assertEquals(63, ONE_BYTE_QUALIFIERS.decode(ONE_BYTE_QUALIFIERS.encode(63))); + assertEquals(130, ONE_BYTE_QUALIFIERS.decode(ONE_BYTE_QUALIFIERS.encode(130))); + assertEquals(255, ONE_BYTE_QUALIFIERS.decode(ONE_BYTE_QUALIFIERS.encode(255))); + byte[] arr1 = ONE_BYTE_QUALIFIERS.encode(255); + byte[] arr2 = new byte[] {-128, arr1[0]}; + assertEquals(255, ONE_BYTE_QUALIFIERS.decode(arr2, 1, 1)); + try { + ONE_BYTE_QUALIFIERS.decode(arr2); + fail(); + } catch (InvalidQualifierBytesException expected) {} + try { + ONE_BYTE_QUALIFIERS.decode(arr2, 0, 2); + fail(); + } catch (InvalidQualifierBytesException expected) {} + + } + + @Test + public void testTwoByteQualifierEncodeDecode() { + assertEquals(1, TWO_BYTE_QUALIFIERS.decode(TWO_BYTE_QUALIFIERS.encode(1))); + assertEquals(127, TWO_BYTE_QUALIFIERS.decode(TWO_BYTE_QUALIFIERS.encode(127))); + assertEquals(63, TWO_BYTE_QUALIFIERS.decode(TWO_BYTE_QUALIFIERS.encode(63))); + assertEquals(130, TWO_BYTE_QUALIFIERS.decode(TWO_BYTE_QUALIFIERS.encode(130))); + assertEquals(128, TWO_BYTE_QUALIFIERS.decode(TWO_BYTE_QUALIFIERS.encode(128))); + assertEquals(129, TWO_BYTE_QUALIFIERS.decode(TWO_BYTE_QUALIFIERS.encode(129))); + assertEquals(32767, TWO_BYTE_QUALIFIERS.decode(TWO_BYTE_QUALIFIERS.encode(32767))); + assertEquals(32768, TWO_BYTE_QUALIFIERS.decode(TWO_BYTE_QUALIFIERS.encode(32768))); + assertEquals(65535, TWO_BYTE_QUALIFIERS.decode(TWO_BYTE_QUALIFIERS.encode(65535))); + byte[] arr1 = TWO_BYTE_QUALIFIERS.encode(65535); + byte[] arr2 = new byte[] {-128, arr1[0], arr1[1]}; + assertEquals(65535, TWO_BYTE_QUALIFIERS.decode(arr2, 1, 2)); + try { + TWO_BYTE_QUALIFIERS.decode(arr2); + fail(); + } catch (InvalidQualifierBytesException expected) {} + } + + @Test + public void testThreeByteQualifierEncodeDecode() { + assertEquals(1, THREE_BYTE_QUALIFIERS.decode(THREE_BYTE_QUALIFIERS.encode(1))); + assertEquals(127, THREE_BYTE_QUALIFIERS.decode(THREE_BYTE_QUALIFIERS.encode(127))); + assertEquals(63, THREE_BYTE_QUALIFIERS.decode(THREE_BYTE_QUALIFIERS.encode(63))); + assertEquals(130, THREE_BYTE_QUALIFIERS.decode(THREE_BYTE_QUALIFIERS.encode(130))); + assertEquals(128, THREE_BYTE_QUALIFIERS.decode(THREE_BYTE_QUALIFIERS.encode(128))); + assertEquals(129, THREE_BYTE_QUALIFIERS.decode(THREE_BYTE_QUALIFIERS.encode(129))); + assertEquals(32767, THREE_BYTE_QUALIFIERS.decode(THREE_BYTE_QUALIFIERS.encode(32767))); + assertEquals(32768, THREE_BYTE_QUALIFIERS.decode(THREE_BYTE_QUALIFIERS.encode(32768))); + assertEquals(65535, THREE_BYTE_QUALIFIERS.decode(THREE_BYTE_QUALIFIERS.encode(65535))); + assertEquals(16777215, THREE_BYTE_QUALIFIERS.decode(THREE_BYTE_QUALIFIERS.encode(16777215))); + byte[] arr1 = THREE_BYTE_QUALIFIERS.encode(16777215); + byte[] arr2 = new byte[] {-128, arr1[0], arr1[1], arr1[2]}; + assertEquals(16777215, THREE_BYTE_QUALIFIERS.decode(arr2, 1, 3)); + try { + THREE_BYTE_QUALIFIERS.decode(arr2, 0, 2); + fail(); + } catch (InvalidQualifierBytesException expected) {} + } + + @Test + public void testFourByteQualifierEncodeDecode() { + assertEquals(1, FOUR_BYTE_QUALIFIERS.decode(FOUR_BYTE_QUALIFIERS.encode(1))); + assertEquals(127, FOUR_BYTE_QUALIFIERS.decode(FOUR_BYTE_QUALIFIERS.encode(127))); + assertEquals(63, FOUR_BYTE_QUALIFIERS.decode(FOUR_BYTE_QUALIFIERS.encode(63))); + assertEquals(130, FOUR_BYTE_QUALIFIERS.decode(FOUR_BYTE_QUALIFIERS.encode(130))); + assertEquals(128, FOUR_BYTE_QUALIFIERS.decode(FOUR_BYTE_QUALIFIERS.encode(128))); + assertEquals(129, FOUR_BYTE_QUALIFIERS.decode(FOUR_BYTE_QUALIFIERS.encode(129))); + assertEquals(32767, FOUR_BYTE_QUALIFIERS.decode(FOUR_BYTE_QUALIFIERS.encode(32767))); + assertEquals(32768, FOUR_BYTE_QUALIFIERS.decode(FOUR_BYTE_QUALIFIERS.encode(32768))); + assertEquals(65535, FOUR_BYTE_QUALIFIERS.decode(FOUR_BYTE_QUALIFIERS.encode(65535))); + assertEquals(Integer.MAX_VALUE, FOUR_BYTE_QUALIFIERS.decode(FOUR_BYTE_QUALIFIERS.encode(Integer.MAX_VALUE))); + byte[] arr1 = FOUR_BYTE_QUALIFIERS.encode(Integer.MAX_VALUE); + byte[] arr2 = new byte[] {-128, arr1[0], arr1[1], arr1[2], arr1[3]}; + assertEquals(Integer.MAX_VALUE, FOUR_BYTE_QUALIFIERS.decode(arr2, 1, 4)); + try { + FOUR_BYTE_QUALIFIERS.decode(arr2); + fail(); + } catch (InvalidQualifierBytesException expected) {} + try { + FOUR_BYTE_QUALIFIERS.decode(arr2, 0, 3); + fail(); + } catch (InvalidQualifierBytesException expected) {} + } + +} http://git-wip-us.apache.org/repos/asf/phoenix/blob/3c7ff99b/phoenix-core/src/test/java/org/apache/phoenix/util/TestUtil.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/test/java/org/apache/phoenix/util/TestUtil.java b/phoenix-core/src/test/java/org/apache/phoenix/util/TestUtil.java index 5feedb1..c427c8d 100644 --- a/phoenix-core/src/test/java/org/apache/phoenix/util/TestUtil.java +++ b/phoenix-core/src/test/java/org/apache/phoenix/util/TestUtil.java @@ -74,6 +74,7 @@ import org.apache.phoenix.compile.StatementContext; import org.apache.phoenix.coprocessor.generated.MetaDataProtos.ClearCacheRequest; import org.apache.phoenix.coprocessor.generated.MetaDataProtos.ClearCacheResponse; import org.apache.phoenix.coprocessor.generated.MetaDataProtos.MetaDataService; +import org.apache.phoenix.execute.MutationState; import org.apache.phoenix.expression.AndExpression; import org.apache.phoenix.expression.ByteBasedLikeExpression; import org.apache.phoenix.expression.ComparisonExpression; @@ -90,6 +91,7 @@ import org.apache.phoenix.expression.function.SingleAggregateFunction; import org.apache.phoenix.expression.function.SubstrFunction; import org.apache.phoenix.expression.function.SumAggregateFunction; import org.apache.phoenix.filter.MultiCQKeyValueComparisonFilter; +import org.apache.phoenix.filter.MultiEncodedCQKeyValueComparisonFilter; import org.apache.phoenix.filter.MultiKeyValueComparisonFilter; import org.apache.phoenix.filter.RowKeyComparisonFilter; import org.apache.phoenix.filter.SingleCQKeyValueComparisonFilter; @@ -108,6 +110,8 @@ import org.apache.phoenix.schema.PColumn; import org.apache.phoenix.schema.PLongColumn; import org.apache.phoenix.schema.PName; import org.apache.phoenix.schema.PTable; +import org.apache.phoenix.schema.PTable.QualifierEncodingScheme; +import org.apache.phoenix.schema.PTableKey; import org.apache.phoenix.schema.RowKeyValueAccessor; import org.apache.phoenix.schema.SortOrder; import org.apache.phoenix.schema.TableRef; @@ -359,6 +363,10 @@ public class TestUtil { public static MultiKeyValueComparisonFilter multiKVFilter(Expression e) { return new MultiCQKeyValueComparisonFilter(e); } + + public static MultiEncodedCQKeyValueComparisonFilter multiEncodedKVFilter(Expression e, QualifierEncodingScheme encodingScheme) { + return new MultiEncodedCQKeyValueComparisonFilter(e, encodingScheme); + } public static Expression and(Expression... expressions) { return new AndExpression(Arrays.asList(expressions)); @@ -610,7 +618,7 @@ public class TestUtil { } public static void analyzeTable(Connection conn, String tableName) throws IOException, SQLException { - analyzeTable(conn, tableName, false); + analyzeTable(conn, tableName, false); } public static void analyzeTable(Connection conn, String tableName, boolean transactional) throws IOException, SQLException { @@ -652,17 +660,17 @@ public class TestUtil { Date date = new Date(DateUtil.parseDate("2015-01-01 00:00:00").getTime() + (i - 1) * MILLIS_IN_DAY); stmt.setDate(6, date); } - + public static void validateRowKeyColumns(ResultSet rs, int i) throws SQLException { - assertTrue(rs.next()); - assertEquals(rs.getString(1), "varchar" + String.valueOf(i)); - assertEquals(rs.getString(2), "char" + String.valueOf(i)); - assertEquals(rs.getInt(3), i); - assertEquals(rs.getInt(4), i); - assertEquals(rs.getBigDecimal(5), new BigDecimal(i*0.5d)); - Date date = new Date(DateUtil.parseDate("2015-01-01 00:00:00").getTime() + (i - 1) * MILLIS_IN_DAY); - assertEquals(rs.getDate(6), date); - } + assertTrue(rs.next()); + assertEquals(rs.getString(1), "varchar" + String.valueOf(i)); + assertEquals(rs.getString(2), "char" + String.valueOf(i)); + assertEquals(rs.getInt(3), i); + assertEquals(rs.getInt(4), i); + assertEquals(rs.getBigDecimal(5), new BigDecimal(i*0.5d)); + Date date = new Date(DateUtil.parseDate("2015-01-01 00:00:00").getTime() + (i - 1) * MILLIS_IN_DAY); + assertEquals(rs.getDate(6), date); + } public static String getTableName(Boolean mutable, Boolean transactional) { StringBuilder tableNameBuilder = new StringBuilder(DEFAULT_DATA_TABLE_NAME); @@ -694,7 +702,7 @@ public class TestUtil { @Override public SortOrder getSortOrder() { - return SortOrder.getDefault(); + return SortOrder.getDefault(); } @Override @@ -720,10 +728,14 @@ public class TestUtil { public boolean isRowTimestamp() { return false; } - @Override - public boolean isDynamic() { - return false; - } + @Override + public boolean isDynamic() { + return false; + } + @Override + public byte[] getColumnQualifierBytes() { + return SINGLE_COLUMN_NAME.getBytes(); + } })), null); aggregationManager.setAggregators(new ClientAggregators(Collections.<SingleAggregateFunction>singletonList(func), 1)); ClientAggregators aggregators = aggregationManager.getAggregators(); @@ -765,15 +777,26 @@ public class TestUtil { // We simply write a marker row, request a major compaction, and then wait until the marker // row is gone + PhoenixConnection pconn = conn.unwrap(PhoenixConnection.class); + PTable table = pconn.getTable(new PTableKey(pconn.getTenantId(), tableName)); ConnectionQueryServices services = conn.unwrap(PhoenixConnection.class).getQueryServices(); - try (HTableInterface htable = services.getTable(Bytes.toBytes(tableName))) { + MutationState mutationState = pconn.getMutationState(); + if (table.isTransactional()) { + mutationState.startTransaction(); + } + try (HTableInterface htable = mutationState.getHTable(table)) { byte[] markerRowKey = Bytes.toBytes("TO_DELETE"); - + Put put = new Put(markerRowKey); - put.add(QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES, HConstants.EMPTY_BYTE_ARRAY, - HConstants.EMPTY_BYTE_ARRAY); + put.add(QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES, QueryConstants.EMPTY_COLUMN_VALUE_BYTES, QueryConstants.EMPTY_COLUMN_VALUE_BYTES); htable.put(put); - htable.delete(new Delete(markerRowKey)); + Delete delete = new Delete(markerRowKey); + delete.deleteColumn(QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES, QueryConstants.EMPTY_COLUMN_VALUE_BYTES); + htable.delete(delete); + htable.close(); + if (table.isTransactional()) { + mutationState.commit(); + } HBaseAdmin hbaseAdmin = services.getAdmin(); hbaseAdmin.flush(tableName); @@ -782,19 +805,28 @@ public class TestUtil { boolean compactionDone = false; while (!compactionDone) { - Thread.sleep(2000L); + Thread.sleep(6000L); Scan scan = new Scan(); scan.setStartRow(markerRowKey); scan.setStopRow(Bytes.add(markerRowKey, new byte[] { 0 })); scan.setRaw(true); - ResultScanner scanner = htable.getScanner(scan); - List<Result> results = Lists.newArrayList(scanner); - LOG.info("Results: " + results); - compactionDone = results.isEmpty(); - scanner.close(); - + try (HTableInterface htableForRawScan = services.getTable(Bytes.toBytes(tableName))) { + ResultScanner scanner = htableForRawScan.getScanner(scan); + List<Result> results = Lists.newArrayList(scanner); + LOG.info("Results: " + results); + compactionDone = results.isEmpty(); + scanner.close(); + } LOG.info("Compaction done: " + compactionDone); + + // need to run compaction after the next txn snapshot has been written so that compaction can remove deleted rows + if (!compactionDone && table.isTransactional()) { + hbaseAdmin = services.getAdmin(); + hbaseAdmin.flush(tableName); + hbaseAdmin.majorCompact(tableName); + hbaseAdmin.close(); + } } } } @@ -821,4 +853,3 @@ public class TestUtil { } } - http://git-wip-us.apache.org/repos/asf/phoenix/blob/3c7ff99b/phoenix-protocol/src/main/PTable.proto ---------------------------------------------------------------------- diff --git a/phoenix-protocol/src/main/PTable.proto b/phoenix-protocol/src/main/PTable.proto index a16263f..f2eb46c 100644 --- a/phoenix-protocol/src/main/PTable.proto +++ b/phoenix-protocol/src/main/PTable.proto @@ -47,6 +47,7 @@ message PColumn { optional string expression = 12; optional bool isRowTimestamp = 13; optional bool isDynamic = 14; + optional bytes columnQualifierBytes = 15; } message PTableStats { @@ -95,4 +96,12 @@ message PTable { optional string autoParititonSeqName = 31; optional bool isAppendOnlySchema = 32; optional bytes parentNameBytes = 33; + optional bytes storageScheme = 34; + optional bytes encodingScheme = 35; + repeated EncodedCQCounter encodedCQCounters = 36; +} + +message EncodedCQCounter { + required string colFamily = 1; + required int32 counter = 2; } http://git-wip-us.apache.org/repos/asf/phoenix/blob/3c7ff99b/phoenix-protocol/src/main/ServerCachingService.proto ---------------------------------------------------------------------- diff --git a/phoenix-protocol/src/main/ServerCachingService.proto b/phoenix-protocol/src/main/ServerCachingService.proto index a45b18f..044c111 100644 --- a/phoenix-protocol/src/main/ServerCachingService.proto +++ b/phoenix-protocol/src/main/ServerCachingService.proto @@ -30,12 +30,47 @@ message ImmutableBytesWritable { required int32 length = 3; } +message ColumnReference { + required bytes family = 1; + required bytes qualifier = 2; +} + +message ColumnInfo { + optional string familyName = 1; + required string columnName = 2; +} + +message IndexMaintainer { + required int32 saltBuckets = 1; + required bool isMultiTenant = 2; + optional bytes viewIndexId = 3; + repeated ColumnReference indexedColumns = 4; + repeated int32 indexedColumnTypeOrdinal = 5; + repeated ColumnReference dataTableColRefForCoveredColumns = 6; + repeated ColumnReference indexTableColRefForCoveredColumns = 7; + required bool isLocalIndex = 8; + required bytes indexTableName = 9; + required bool rowKeyOrderOptimizable = 10; + required bytes dataTableEmptyKeyValueColFamily = 11; + required ImmutableBytesWritable emptyKeyValueColFamily = 12; + optional bytes indexedExpressions = 13; + required bytes rowKeyMetadata = 14; + required int32 numDataTableColFamilies = 15; + required bool indexWalDisabled = 16; + required int32 indexRowKeyByteSize = 17; + required bool immutable = 18; + repeated ColumnInfo indexedColumnInfo = 19; + required int32 encodingScheme = 20; + required int32 immutableStorageScheme = 21; +} + message AddServerCacheRequest { optional bytes tenantId = 1; required bytes cacheId = 2; required ImmutableBytesWritable cachePtr = 3; required ServerCacheFactory cacheFactory = 4; optional bytes txState = 5; + optional bool hasProtoBufIndexMaintainer = 6; } message AddServerCacheResponse { http://git-wip-us.apache.org/repos/asf/phoenix/blob/3c7ff99b/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index c400312..d61813a 100644 --- a/pom.xml +++ b/pom.xml @@ -299,7 +299,7 @@ <forkCount>${numForkedIT}</forkCount> <runOrder>alphabetical</runOrder> <reuseForks>true</reuseForks> - <argLine>-enableassertions -Xmx2000m -XX:MaxPermSize=128m -Djava.security.egd=file:/dev/./urandom "-Djava.library.path=${hadoop.library.path}${path.separator}${java.library.path}" -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=./target/</argLine> + <argLine>-enableassertions -Xmx2500m -XX:MaxPermSize=128m -Djava.security.egd=file:/dev/./urandom "-Djava.library.path=${hadoop.library.path}${path.separator}${java.library.path}" -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=./target/</argLine> <redirectTestOutputToFile>${test.output.tofile}</redirectTestOutputToFile> <testSourceDirectory>${basedir}/src/it/java</testSourceDirectory> <groups>org.apache.phoenix.end2end.ClientManagedTimeTest</groups>