http://git-wip-us.apache.org/repos/asf/phoenix/blob/572d242d/phoenix-core/src/test/java/org/apache/phoenix/hbase/index/covered/NonTxIndexBuilderTest.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/test/java/org/apache/phoenix/hbase/index/covered/NonTxIndexBuilderTest.java b/phoenix-core/src/test/java/org/apache/phoenix/hbase/index/covered/NonTxIndexBuilderTest.java new file mode 100644 index 0000000..64e869b --- /dev/null +++ b/phoenix-core/src/test/java/org/apache/phoenix/hbase/index/covered/NonTxIndexBuilderTest.java @@ -0,0 +1,341 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.phoenix.hbase.index.covered; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertTrue; + +import java.io.IOException; +import java.sql.Connection; +import java.sql.DriverManager; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.NavigableMap; +import java.util.Properties; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.CellUtil; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.KeyValue.Type; +import org.apache.hadoop.hbase.client.Mutation; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment; +import org.apache.hadoop.hbase.io.ImmutableBytesWritable; +import org.apache.hadoop.hbase.io.TimeRange; +import org.apache.hadoop.hbase.regionserver.Region; +import org.apache.hadoop.hbase.regionserver.RegionScanner; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.Pair; +import org.apache.phoenix.coprocessor.BaseRegionScanner; +import org.apache.phoenix.hbase.index.MultiMutation; +import org.apache.phoenix.hbase.index.covered.data.LocalTable; +import org.apache.phoenix.hbase.index.covered.update.ColumnTracker; +import org.apache.phoenix.hbase.index.util.GenericKeyValueBuilder; +import org.apache.phoenix.hbase.index.util.ImmutableBytesPtr; +import org.apache.phoenix.hbase.index.util.IndexManagementUtil; +import org.apache.phoenix.index.IndexMaintainer; +import org.apache.phoenix.index.PhoenixIndexCodec; +import org.apache.phoenix.index.PhoenixIndexMetaData; +import org.apache.phoenix.jdbc.PhoenixConnection; +import org.apache.phoenix.query.BaseConnectionlessQueryTest; +import org.apache.phoenix.query.QueryConstants; +import org.apache.phoenix.query.QueryServices; +import org.apache.phoenix.schema.PTable; +import org.apache.phoenix.schema.PTableKey; +import org.apache.phoenix.util.PropertiesUtil; +import org.apache.phoenix.util.TestUtil; +import org.junit.Before; +import org.junit.Test; +import org.mockito.Mockito; +import org.mockito.invocation.InvocationOnMock; +import org.mockito.stubbing.Answer; + +import com.google.common.base.Optional; +import com.google.common.base.Predicate; +import com.google.common.collect.Iterables; +import com.google.common.collect.Lists; + +public class NonTxIndexBuilderTest extends BaseConnectionlessQueryTest { + private static final String TEST_TABLE_STRING = "TEST_TABLE"; + private static final String TEST_TABLE_DDL = "CREATE TABLE IF NOT EXISTS " + + TEST_TABLE_STRING + " (\n" + + " ORGANIZATION_ID CHAR(4) NOT NULL,\n" + + " ENTITY_ID CHAR(7) NOT NULL,\n" + + " SCORE INTEGER,\n" + + " LAST_UPDATE_TIME TIMESTAMP\n" + + " CONSTRAINT TEST_TABLE_PK PRIMARY KEY (\n" + + " ORGANIZATION_ID,\n" + + " ENTITY_ID\n" + + " )\n" + + ") VERSIONS=1, MULTI_TENANT=TRUE"; + private static final String TEST_TABLE_INDEX_STRING = "TEST_TABLE_SCORE"; + private static final String TEST_TABLE_INDEX_DDL = "CREATE INDEX IF NOT EXISTS " + + TEST_TABLE_INDEX_STRING + + " ON " + TEST_TABLE_STRING + " (SCORE DESC, ENTITY_ID DESC)"; + private static final byte[] ROW = Bytes.toBytes("org1entity1"); //length 4 + 7 (see ddl) + private static final String FAM_STRING = QueryConstants.DEFAULT_COLUMN_FAMILY; + private static final byte[] FAM = Bytes.toBytes(FAM_STRING); + private static final byte[] INDEXED_QUALIFIER = Bytes.toBytes("SCORE"); + private static final byte[] VALUE_1 = Bytes.toBytes(111); + private static final byte[] VALUE_2 = Bytes.toBytes(222); + private static final byte[] VALUE_3 = Bytes.toBytes(333); + private static final byte[] VALUE_4 = Bytes.toBytes(444); + private static final byte PUT_TYPE = KeyValue.Type.Put.getCode(); + + private NonTxIndexBuilder indexBuilder; + private PhoenixIndexMetaData mockIndexMetaData; + // Put your current row state in here - the index builder will read from this in LocalTable + // to determine whether the index has changed. + // Whatever we return here should match the table DDL (e.g. length of column value) + private List<Cell> currentRowCells; + + /** + * Test setup so that {@link NonTxIndexBuilder#getIndexUpdate(Mutation, IndexMetaData)} can be + * called, where any read requests to + * {@link LocalTable#getCurrentRowState(Mutation, Collection, boolean)} are read from our test + * field 'currentRowCells' + */ + @Before + public void setup() throws Exception { + RegionCoprocessorEnvironment env = Mockito.mock(RegionCoprocessorEnvironment.class); + Configuration conf = new Configuration(false); + conf.set(NonTxIndexBuilder.CODEC_CLASS_NAME_KEY, PhoenixIndexCodec.class.getName()); + Mockito.when(env.getConfiguration()).thenReturn(conf); + + // the following is used by LocalTable#getCurrentRowState() + Region mockRegion = Mockito.mock(Region.class); + Mockito.when(env.getRegion()).thenReturn(mockRegion); + + Mockito.when(mockRegion.getScanner(Mockito.any(Scan.class))) + .thenAnswer(new Answer<RegionScanner>() { + @Override + public RegionScanner answer(InvocationOnMock invocation) throws Throwable { + Scan sArg = (Scan) invocation.getArguments()[0]; + TimeRange timeRange = sArg.getTimeRange(); + return getMockTimeRangeRegionScanner(timeRange); + } + }); + + // the following is called by PhoenixIndexCodec#getIndexUpserts() , getIndexDeletes() + HRegionInfo mockRegionInfo = Mockito.mock(HRegionInfo.class); + Mockito.when(mockRegion.getRegionInfo()).thenReturn(mockRegionInfo); + Mockito.when(mockRegionInfo.getStartKey()).thenReturn(Bytes.toBytes("a")); + Mockito.when(mockRegionInfo.getEndKey()).thenReturn(Bytes.toBytes("z")); + + mockIndexMetaData = Mockito.mock(PhoenixIndexMetaData.class); + Mockito.when(mockIndexMetaData.isImmutableRows()).thenReturn(false); + Mockito.when(mockIndexMetaData.getIndexMaintainers()) + .thenReturn(Collections.singletonList(getTestIndexMaintainer())); + + indexBuilder = new NonTxIndexBuilder(); + indexBuilder.setup(env); + } + + // returns a RegionScanner which filters currentRowCells using the given TimeRange. + // This is called from LocalTable#getCurrentRowState() + // If testIndexMetaData.ignoreNewerMutations() is not set, default TimeRange is 0 to + // Long.MAX_VALUE + private RegionScanner getMockTimeRangeRegionScanner(final TimeRange timeRange) { + return new BaseRegionScanner(Mockito.mock(RegionScanner.class)) { + @Override + public boolean next(List<Cell> results) throws IOException { + for (Cell cell : currentRowCells) { + if (cell.getTimestamp() >= timeRange.getMin() + && cell.getTimestamp() < timeRange.getMax()) { + results.add(cell); + } + } + return false; // indicate no more results + } + }; + } + + private IndexMaintainer getTestIndexMaintainer() throws Exception { + Properties props = PropertiesUtil.deepCopy(TestUtil.TEST_PROPERTIES); + // disable column encoding, makes debugging easier + props.put(QueryServices.DEFAULT_COLUMN_ENCODED_BYTES_ATRRIB, "0"); + Connection conn = DriverManager.getConnection(getUrl(), props); + try { + conn.setAutoCommit(true); + conn.createStatement().execute(TEST_TABLE_DDL); + conn.createStatement().execute(TEST_TABLE_INDEX_DDL); + PhoenixConnection pconn = conn.unwrap(PhoenixConnection.class); + PTable table = pconn.getTable(new PTableKey(pconn.getTenantId(), TEST_TABLE_STRING)); + ImmutableBytesWritable ptr = new ImmutableBytesWritable(); + table.getIndexMaintainers(ptr, pconn); + List<IndexMaintainer> indexMaintainerList = + IndexMaintainer.deserialize(ptr, GenericKeyValueBuilder.INSTANCE, true); + assertEquals(1, indexMaintainerList.size()); + IndexMaintainer indexMaintainer = indexMaintainerList.get(0); + return indexMaintainer; + } finally { + conn.close(); + } + } + + /** + * Tests that updating an indexed column results in a DeleteFamily (prior index cell) and a Put + * (new index cell) + */ + @Test + public void testGetMutableIndexUpdate() throws IOException { + setCurrentRowState(FAM, INDEXED_QUALIFIER, 1, VALUE_1); + + // update ts and value + Put put = new Put(ROW); + put.addImmutable(FAM, INDEXED_QUALIFIER, 2, VALUE_2); + MultiMutation mutation = new MultiMutation(new ImmutableBytesPtr(ROW)); + mutation.addAll(put); + + Collection<Pair<Mutation, byte[]>> indexUpdates = + indexBuilder.getIndexUpdate(mutation, mockIndexMetaData); + assertEquals(2, indexUpdates.size()); + assertContains(indexUpdates, 2, ROW, KeyValue.Type.DeleteFamily, FAM, + new byte[0] /* qual not needed */, 2); + assertContains(indexUpdates, ColumnTracker.NO_NEWER_PRIMARY_TABLE_ENTRY_TIMESTAMP, ROW, + KeyValue.Type.Put, FAM, QueryConstants.EMPTY_COLUMN_BYTES, 2); + } + + /** + * Tests a partial rebuild of a row with multiple versions. 3 versions of the row in data table, + * and we rebuild the index starting from time t=2 + * + * There should be one index row version per data row version. + */ + @Test + public void testRebuildMultipleVersionRow() throws IOException { + // when doing a rebuild, we are replaying mutations so we want to ignore newer mutations + // see LocalTable#getCurrentRowState() + Mockito.when(mockIndexMetaData.ignoreNewerMutations()).thenReturn(true); + + // the current row state has 3 versions, but if we rebuild as of t=2, scanner in LocalTable + // should only return first + Cell currentCell1 = CellUtil.createCell(ROW, FAM, INDEXED_QUALIFIER, 1, PUT_TYPE, VALUE_1); + Cell currentCell2 = CellUtil.createCell(ROW, FAM, INDEXED_QUALIFIER, 2, PUT_TYPE, VALUE_2); + Cell currentCell3 = CellUtil.createCell(ROW, FAM, INDEXED_QUALIFIER, 3, PUT_TYPE, VALUE_3); + Cell currentCell4 = CellUtil.createCell(ROW, FAM, INDEXED_QUALIFIER, 4, PUT_TYPE, VALUE_4); + setCurrentRowState(Arrays.asList(currentCell4, currentCell3, currentCell2, currentCell1)); + + // rebuilder replays mutations starting from t=2 + MultiMutation mutation = new MultiMutation(new ImmutableBytesPtr(ROW)); + Put put = new Put(ROW); + put.addImmutable(FAM, INDEXED_QUALIFIER, 4, VALUE_4); + mutation.addAll(put); + put = new Put(ROW); + put.addImmutable(FAM, INDEXED_QUALIFIER, 3, VALUE_3); + mutation.addAll(put); + put = new Put(ROW); + put.addImmutable(FAM, INDEXED_QUALIFIER, 2, VALUE_2); + mutation.addAll(put); + + Collection<Pair<Mutation, byte[]>> indexUpdates = Lists.newArrayList(); + for (Mutation m : IndexManagementUtil.flattenMutationsByTimestamp(Collections.singletonList(mutation))) { + indexUpdates.addAll(indexBuilder.getIndexUpdate(m, mockIndexMetaData)); + } + // 3 puts and 3 deletes (one to hide existing index row for VALUE_1, and two to hide index + // rows for VALUE_2, VALUE_3) + assertEquals(6, indexUpdates.size()); + + assertContains(indexUpdates, 2, ROW, KeyValue.Type.DeleteFamily, FAM, + new byte[0] /* qual not needed */, 2); + assertContains(indexUpdates, ColumnTracker.NO_NEWER_PRIMARY_TABLE_ENTRY_TIMESTAMP, ROW, + KeyValue.Type.Put, FAM, QueryConstants.EMPTY_COLUMN_BYTES, 2); + assertContains(indexUpdates, 3, ROW, KeyValue.Type.DeleteFamily, FAM, + new byte[0] /* qual not needed */, 3); + assertContains(indexUpdates, ColumnTracker.NO_NEWER_PRIMARY_TABLE_ENTRY_TIMESTAMP, ROW, + KeyValue.Type.Put, FAM, QueryConstants.EMPTY_COLUMN_BYTES, 3); + assertContains(indexUpdates, 4, ROW, KeyValue.Type.DeleteFamily, FAM, + new byte[0] /* qual not needed */, 4); + assertContains(indexUpdates, ColumnTracker.NO_NEWER_PRIMARY_TABLE_ENTRY_TIMESTAMP, ROW, + KeyValue.Type.Put, FAM, QueryConstants.EMPTY_COLUMN_BYTES, 4); + } + + /** + * Tests getting an index update for a mutation with 200 versions Before, the issue PHOENIX-3807 + * was causing this test to take >90 seconds, so here we set a timeout of 5 seconds + */ + @Test(timeout = 10000) + public void testManyVersions() throws IOException { + // when doing a rebuild, we are replaying mutations so we want to ignore newer mutations + // see LocalTable#getCurrentRowState() + Mockito.when(mockIndexMetaData.ignoreNewerMutations()).thenReturn(true); + MultiMutation mutation = getMultipleVersionMutation(200); + currentRowCells = mutation.getFamilyCellMap().get(FAM); + + Collection<Pair<Mutation, byte[]>> indexUpdates = Lists.newArrayList(); + for (Mutation m : IndexManagementUtil.flattenMutationsByTimestamp(Collections.singletonList(mutation))) { + indexUpdates.addAll(indexBuilder.getIndexUpdate(m, mockIndexMetaData)); + } + assertNotEquals(0, indexUpdates.size()); + } + + // Assert that the given collection of indexUpdates contains the given cell + private void assertContains(Collection<Pair<Mutation, byte[]>> indexUpdates, + final long mutationTs, final byte[] row, final Type cellType, final byte[] fam, + final byte[] qual, final long cellTs) { + Predicate<Pair<Mutation, byte[]>> hasCellPredicate = + new Predicate<Pair<Mutation, byte[]>>() { + @Override + public boolean apply(Pair<Mutation, byte[]> input) { + assertEquals(TEST_TABLE_INDEX_STRING, Bytes.toString(input.getSecond())); + Mutation mutation = input.getFirst(); + if (mutationTs == mutation.getTimeStamp()) { + NavigableMap<byte[], List<Cell>> familyCellMap = + mutation.getFamilyCellMap(); + Cell updateCell = familyCellMap.get(fam).get(0); + if (cellType == KeyValue.Type.codeToType(updateCell.getTypeByte()) + && Bytes.compareTo(fam, CellUtil.cloneFamily(updateCell)) == 0 + && Bytes.compareTo(qual, + CellUtil.cloneQualifier(updateCell)) == 0 + && cellTs == updateCell.getTimestamp()) { + return true; + } + } + return false; + } + }; + Optional<Pair<Mutation, byte[]>> tryFind = + Iterables.tryFind(indexUpdates, hasCellPredicate); + assertTrue(tryFind.isPresent()); + } + + private void setCurrentRowState(byte[] fam2, byte[] indexedQualifier, int i, byte[] value1) { + Cell cell = CellUtil.createCell(ROW, FAM, INDEXED_QUALIFIER, 1, PUT_TYPE, VALUE_1); + currentRowCells = Collections.singletonList(cell); + } + + private void setCurrentRowState(List<Cell> cells) { + currentRowCells = cells; + } + + private MultiMutation getMultipleVersionMutation(int versions) { + MultiMutation mutation = new MultiMutation(new ImmutableBytesPtr(ROW)); + for (int i = versions - 1; i >= 0; i--) { + Put put = new Put(ROW); + put.addImmutable(FAM, INDEXED_QUALIFIER, i, Bytes.toBytes(i)); + mutation.addAll(put); + } + return mutation; + } +}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/572d242d/phoenix-core/src/test/java/org/apache/phoenix/hbase/index/covered/TestCoveredColumns.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/test/java/org/apache/phoenix/hbase/index/covered/TestCoveredColumns.java b/phoenix-core/src/test/java/org/apache/phoenix/hbase/index/covered/TestCoveredColumns.java deleted file mode 100644 index dd5223b..0000000 --- a/phoenix-core/src/test/java/org/apache/phoenix/hbase/index/covered/TestCoveredColumns.java +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.phoenix.hbase.index.covered; - -import static org.junit.Assert.assertEquals; - -import java.util.Arrays; - -import org.apache.hadoop.hbase.util.Bytes; -import org.junit.Test; - -import org.apache.phoenix.hbase.index.covered.CoveredColumns; -import org.apache.phoenix.hbase.index.covered.update.ColumnReference; - -public class TestCoveredColumns { - - private static final byte[] fam = Bytes.toBytes("fam"); - private static final byte[] qual = Bytes.toBytes("qual"); - - @Test - public void testCovering() { - ColumnReference ref = new ColumnReference(fam, qual); - CoveredColumns columns = new CoveredColumns(); - assertEquals("Should have only found a single column to cover", 1, columns - .findNonCoveredColumns(Arrays.asList(ref)).size()); - - columns.addColumn(ref); - assertEquals("Shouldn't have any columns to cover", 0, - columns.findNonCoveredColumns(Arrays.asList(ref)).size()); - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/phoenix/blob/572d242d/phoenix-core/src/test/java/org/apache/phoenix/hbase/index/covered/TestLocalTableState.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/test/java/org/apache/phoenix/hbase/index/covered/TestLocalTableState.java b/phoenix-core/src/test/java/org/apache/phoenix/hbase/index/covered/TestLocalTableState.java deleted file mode 100644 index dd027b0..0000000 --- a/phoenix-core/src/test/java/org/apache/phoenix/hbase/index/covered/TestLocalTableState.java +++ /dev/null @@ -1,293 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.phoenix.hbase.index.covered; - -import static org.junit.Assert.assertEquals; - -import java.util.Arrays; -import java.util.List; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.KeyValue; -import org.apache.hadoop.hbase.KeyValue.Type; -import org.apache.hadoop.hbase.KeyValueUtil; -import org.apache.hadoop.hbase.client.Put; -import org.apache.hadoop.hbase.client.Scan; -import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment; -import org.apache.hadoop.hbase.regionserver.Region; -import org.apache.hadoop.hbase.regionserver.RegionScanner; -import org.apache.hadoop.hbase.util.Bytes; -import org.apache.hadoop.hbase.util.Pair; -import org.apache.phoenix.hbase.index.covered.data.LocalHBaseState; -import org.apache.phoenix.hbase.index.covered.data.LocalTable; -import org.apache.phoenix.hbase.index.covered.update.ColumnReference; -import org.apache.phoenix.hbase.index.scanner.Scanner; -import org.apache.phoenix.hbase.index.scanner.ScannerBuilder.CoveredDeleteScanner; -import org.junit.Test; -import org.mockito.Mockito; -import org.mockito.invocation.InvocationOnMock; -import org.mockito.stubbing.Answer; - -/** - * - */ -public class TestLocalTableState { - - private static final byte[] row = Bytes.toBytes("row"); - private static final byte[] fam = Bytes.toBytes("fam"); - private static final byte[] qual = Bytes.toBytes("qual"); - private static final byte[] val = Bytes.toBytes("val"); - private static final long ts = 10; - private static final IndexMetaData indexMetaData = new IndexMetaData() { - - @Override - public boolean isImmutableRows() { - return false; - } - - @Override - public boolean ignoreNewerMutations() { - return false; - } - - }; - - @SuppressWarnings("unchecked") - @Test - public void testCorrectOrderingWithLazyLoadingColumns() throws Exception { - Put m = new Put(row); - m.add(fam, qual, ts, val); - // setup mocks - Configuration conf = new Configuration(false); - RegionCoprocessorEnvironment env = Mockito.mock(RegionCoprocessorEnvironment.class); - Mockito.when(env.getConfiguration()).thenReturn(conf); - - Region region = Mockito.mock(Region.class); - Mockito.when(env.getRegion()).thenReturn(region); - RegionScanner scanner = Mockito.mock(RegionScanner.class); - Mockito.when(region.getScanner(Mockito.any(Scan.class))).thenReturn(scanner); - final byte[] stored = Bytes.toBytes("stored-value"); - Mockito.when(scanner.next(Mockito.any(List.class))).thenAnswer(new Answer<Boolean>() { - @Override - public Boolean answer(InvocationOnMock invocation) throws Throwable { - List<KeyValue> list = (List<KeyValue>) invocation.getArguments()[0]; - KeyValue kv = new KeyValue(row, fam, qual, ts, Type.Put, stored); - kv.setSequenceId(0); - list.add(kv); - return false; - } - }); - - - LocalHBaseState state = new LocalTable(env); - LocalTableState table = new LocalTableState(env, state, m); - //add the kvs from the mutation - table.addPendingUpdates(KeyValueUtil.ensureKeyValues(m.get(fam, qual))); - - // setup the lookup - ColumnReference col = new ColumnReference(fam, qual); - table.setCurrentTimestamp(ts); - //check that our value still shows up first on scan, even though this is a lazy load - Pair<CoveredDeleteScanner, IndexUpdate> p = table.getIndexedColumnsTableState(Arrays.asList(col), false, false, indexMetaData); - Scanner s = p.getFirst(); - assertEquals("Didn't get the pending mutation's value first", m.get(fam, qual).get(0), s.next()); - } - - public static final class ScannerCreatedException extends RuntimeException { - ScannerCreatedException(String msg) { - super(msg); - } - } - - @Test(expected = ScannerCreatedException.class) - public void testScannerForMutableRows() throws Exception { - IndexMetaData indexMetaData = new IndexMetaData() { - - @Override - public boolean isImmutableRows() { - return false; - } - - @Override - public boolean ignoreNewerMutations() { - return false; - } - - }; - Put m = new Put(row); - m.add(fam, qual, ts, val); - // setup mocks - Configuration conf = new Configuration(false); - RegionCoprocessorEnvironment env = Mockito.mock(RegionCoprocessorEnvironment.class); - Mockito.when(env.getConfiguration()).thenReturn(conf); - - Region region = Mockito.mock(Region.class); - Mockito.when(env.getRegion()).thenReturn(region); - Mockito.when(region.getScanner(Mockito.any(Scan.class))).thenThrow(new ScannerCreatedException("Should not open scanner when data is immutable")); - - LocalHBaseState state = new LocalTable(env); - LocalTableState table = new LocalTableState(env, state, m); - //add the kvs from the mutation - table.addPendingUpdates(KeyValueUtil.ensureKeyValues(m.get(fam, qual))); - - // setup the lookup - ColumnReference col = new ColumnReference(fam, qual); - table.setCurrentTimestamp(ts); - table.getIndexedColumnsTableState(Arrays.asList(col), false, false, indexMetaData); - } - - @Test - public void testNoScannerForImmutableRows() throws Exception { - IndexMetaData indexMetaData = new IndexMetaData() { - - @Override - public boolean isImmutableRows() { - return true; - } - - @Override - public boolean ignoreNewerMutations() { - return false; - } - - }; - Put m = new Put(row); - m.add(fam, qual, ts, val); - // setup mocks - Configuration conf = new Configuration(false); - RegionCoprocessorEnvironment env = Mockito.mock(RegionCoprocessorEnvironment.class); - Mockito.when(env.getConfiguration()).thenReturn(conf); - - Region region = Mockito.mock(Region.class); - Mockito.when(env.getRegion()).thenReturn(region); - Mockito.when(region.getScanner(Mockito.any(Scan.class))).thenThrow(new ScannerCreatedException("Should not open scanner when data is immutable")); - - LocalHBaseState state = new LocalTable(env); - LocalTableState table = new LocalTableState(env, state, m); - //add the kvs from the mutation - table.addPendingUpdates(KeyValueUtil.ensureKeyValues(m.get(fam, qual))); - - // setup the lookup - ColumnReference col = new ColumnReference(fam, qual); - table.setCurrentTimestamp(ts); - //check that our value still shows up first on scan, even though this is a lazy load - Pair<CoveredDeleteScanner, IndexUpdate> p = table.getIndexedColumnsTableState(Arrays.asList(col), false, false, indexMetaData); - Scanner s = p.getFirst(); - assertEquals("Didn't get the pending mutation's value first", m.get(fam, qual).get(0), s.next()); - } - - /** - * Test that we correctly rollback the state of keyvalue - * @throws Exception - */ - @Test - @SuppressWarnings("unchecked") - public void testCorrectRollback() throws Exception { - Put m = new Put(row); - m.add(fam, qual, ts, val); - // setup mocks - RegionCoprocessorEnvironment env = Mockito.mock(RegionCoprocessorEnvironment.class); - - Region region = Mockito.mock(Region.class); - Mockito.when(env.getRegion()).thenReturn(region); - RegionScanner scanner = Mockito.mock(RegionScanner.class); - Mockito.when(region.getScanner(Mockito.any(Scan.class))).thenReturn(scanner); - final byte[] stored = Bytes.toBytes("stored-value"); - final KeyValue storedKv = new KeyValue(row, fam, qual, ts, Type.Put, stored); - storedKv.setSequenceId(2); - Mockito.when(scanner.next(Mockito.any(List.class))).thenAnswer(new Answer<Boolean>() { - @Override - public Boolean answer(InvocationOnMock invocation) throws Throwable { - List<KeyValue> list = (List<KeyValue>) invocation.getArguments()[0]; - - list.add(storedKv); - return false; - } - }); - LocalHBaseState state = new LocalTable(env); - LocalTableState table = new LocalTableState(env, state, m); - // add the kvs from the mutation - KeyValue kv = KeyValueUtil.ensureKeyValue(m.get(fam, qual).get(0)); - kv.setSequenceId(0); - table.addPendingUpdates(kv); - - // setup the lookup - ColumnReference col = new ColumnReference(fam, qual); - table.setCurrentTimestamp(ts); - // check that the value is there - Pair<CoveredDeleteScanner, IndexUpdate> p = table.getIndexedColumnsTableState(Arrays.asList(col), false, false, indexMetaData); - Scanner s = p.getFirst(); - assertEquals("Didn't get the pending mutation's value first", kv, s.next()); - - // rollback that value - table.rollback(Arrays.asList(kv)); - p = table.getIndexedColumnsTableState(Arrays.asList(col), false, false, indexMetaData); - s = p.getFirst(); - assertEquals("Didn't correctly rollback the row - still found it!", null, s.next()); - Mockito.verify(env, Mockito.times(1)).getRegion(); - Mockito.verify(region, Mockito.times(1)).getScanner(Mockito.any(Scan.class)); - } - - @SuppressWarnings("unchecked") - @Test - public void testOnlyLoadsRequestedColumns() throws Exception { - // setup mocks - RegionCoprocessorEnvironment env = Mockito.mock(RegionCoprocessorEnvironment.class); - - Region region = Mockito.mock(Region.class); - Mockito.when(env.getRegion()).thenReturn(region); - RegionScanner scanner = Mockito.mock(RegionScanner.class); - Mockito.when(region.getScanner(Mockito.any(Scan.class))).thenReturn(scanner); - final KeyValue storedKv = - new KeyValue(row, fam, qual, ts, Type.Put, Bytes.toBytes("stored-value")); - storedKv.setSequenceId(2); - Mockito.when(scanner.next(Mockito.any(List.class))).thenAnswer(new Answer<Boolean>() { - @Override - public Boolean answer(InvocationOnMock invocation) throws Throwable { - List<KeyValue> list = (List<KeyValue>) invocation.getArguments()[0]; - - list.add(storedKv); - return false; - } - }); - LocalHBaseState state = new LocalTable(env); - Put pendingUpdate = new Put(row); - pendingUpdate.add(fam, qual, ts, val); - LocalTableState table = new LocalTableState(env, state, pendingUpdate); - - // do the lookup for the given column - ColumnReference col = new ColumnReference(fam, qual); - table.setCurrentTimestamp(ts); - // check that the value is there - Pair<CoveredDeleteScanner, IndexUpdate> p = table.getIndexedColumnsTableState(Arrays.asList(col), false, false, indexMetaData); - Scanner s = p.getFirst(); - // make sure it read the table the one time - assertEquals("Didn't get the stored keyvalue!", storedKv, s.next()); - - // on the second lookup it shouldn't access the underlying table again - the cached columns - // should know they are done - p = table.getIndexedColumnsTableState(Arrays.asList(col), false, false, indexMetaData); - s = p.getFirst(); - assertEquals("Lost already loaded update!", storedKv, s.next()); - Mockito.verify(env, Mockito.times(1)).getRegion(); - Mockito.verify(region, Mockito.times(1)).getScanner(Mockito.any(Scan.class)); - } - - // TODO add test here for making sure multiple column references with the same column family don't - // cause an infinite loop -} http://git-wip-us.apache.org/repos/asf/phoenix/blob/572d242d/phoenix-core/src/test/java/org/apache/phoenix/hbase/index/covered/TestNonTxIndexBuilder.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/test/java/org/apache/phoenix/hbase/index/covered/TestNonTxIndexBuilder.java b/phoenix-core/src/test/java/org/apache/phoenix/hbase/index/covered/TestNonTxIndexBuilder.java deleted file mode 100644 index 763b7d5..0000000 --- a/phoenix-core/src/test/java/org/apache/phoenix/hbase/index/covered/TestNonTxIndexBuilder.java +++ /dev/null @@ -1,335 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.phoenix.hbase.index.covered; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotEquals; -import static org.junit.Assert.assertTrue; - -import java.io.IOException; -import java.sql.Connection; -import java.sql.DriverManager; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.List; -import java.util.NavigableMap; -import java.util.Properties; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.Cell; -import org.apache.hadoop.hbase.CellUtil; -import org.apache.hadoop.hbase.HRegionInfo; -import org.apache.hadoop.hbase.KeyValue; -import org.apache.hadoop.hbase.KeyValue.Type; -import org.apache.hadoop.hbase.client.Mutation; -import org.apache.hadoop.hbase.client.Put; -import org.apache.hadoop.hbase.client.Scan; -import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment; -import org.apache.hadoop.hbase.io.ImmutableBytesWritable; -import org.apache.hadoop.hbase.io.TimeRange; -import org.apache.hadoop.hbase.regionserver.Region; -import org.apache.hadoop.hbase.regionserver.RegionScanner; -import org.apache.hadoop.hbase.util.Bytes; -import org.apache.hadoop.hbase.util.Pair; -import org.apache.phoenix.coprocessor.BaseRegionScanner; -import org.apache.phoenix.hbase.index.MultiMutation; -import org.apache.phoenix.hbase.index.covered.data.LocalTable; -import org.apache.phoenix.hbase.index.covered.update.ColumnTracker; -import org.apache.phoenix.hbase.index.util.GenericKeyValueBuilder; -import org.apache.phoenix.hbase.index.util.ImmutableBytesPtr; -import org.apache.phoenix.index.IndexMaintainer; -import org.apache.phoenix.index.PhoenixIndexCodec; -import org.apache.phoenix.index.PhoenixIndexMetaData; -import org.apache.phoenix.jdbc.PhoenixConnection; -import org.apache.phoenix.query.BaseConnectionlessQueryTest; -import org.apache.phoenix.query.QueryConstants; -import org.apache.phoenix.query.QueryServices; -import org.apache.phoenix.schema.PTable; -import org.apache.phoenix.schema.PTableKey; -import org.apache.phoenix.util.PropertiesUtil; -import org.apache.phoenix.util.TestUtil; -import org.junit.Before; -import org.junit.Test; -import org.mockito.Mockito; -import org.mockito.invocation.InvocationOnMock; -import org.mockito.stubbing.Answer; - -import com.google.common.base.Optional; -import com.google.common.base.Predicate; -import com.google.common.collect.Iterables; - -public class TestNonTxIndexBuilder extends BaseConnectionlessQueryTest { - private static final String TEST_TABLE_STRING = "TEST_TABLE"; - private static final String TEST_TABLE_DDL = "CREATE TABLE IF NOT EXISTS " + - TEST_TABLE_STRING + " (\n" + - " ORGANIZATION_ID CHAR(4) NOT NULL,\n" + - " ENTITY_ID CHAR(7) NOT NULL,\n" + - " SCORE INTEGER,\n" + - " LAST_UPDATE_TIME TIMESTAMP\n" + - " CONSTRAINT TEST_TABLE_PK PRIMARY KEY (\n" + - " ORGANIZATION_ID,\n" + - " ENTITY_ID\n" + - " )\n" + - ") VERSIONS=1, MULTI_TENANT=TRUE"; - private static final String TEST_TABLE_INDEX_STRING = "TEST_TABLE_SCORE"; - private static final String TEST_TABLE_INDEX_DDL = "CREATE INDEX IF NOT EXISTS " + - TEST_TABLE_INDEX_STRING - + " ON " + TEST_TABLE_STRING + " (SCORE DESC, ENTITY_ID DESC)"; - private static final byte[] ROW = Bytes.toBytes("org1entity1"); //length 4 + 7 (see ddl) - private static final String FAM_STRING = QueryConstants.DEFAULT_COLUMN_FAMILY; - private static final byte[] FAM = Bytes.toBytes(FAM_STRING); - private static final byte[] INDEXED_QUALIFIER = Bytes.toBytes("SCORE"); - private static final byte[] VALUE_1 = Bytes.toBytes(111); - private static final byte[] VALUE_2 = Bytes.toBytes(222); - private static final byte[] VALUE_3 = Bytes.toBytes(333); - private static final byte[] VALUE_4 = Bytes.toBytes(444); - private static final byte PUT_TYPE = KeyValue.Type.Put.getCode(); - - private NonTxIndexBuilder indexBuilder; - private PhoenixIndexMetaData mockIndexMetaData; - // Put your current row state in here - the index builder will read from this in LocalTable - // to determine whether the index has changed. - // Whatever we return here should match the table DDL (e.g. length of column value) - private List<Cell> currentRowCells; - - /** - * Test setup so that {@link NonTxIndexBuilder#getIndexUpdate(Mutation, IndexMetaData)} can be - * called, where any read requests to - * {@link LocalTable#getCurrentRowState(Mutation, Collection, boolean)} are read from our test - * field 'currentRowCells' - */ - @Before - public void setup() throws Exception { - RegionCoprocessorEnvironment env = Mockito.mock(RegionCoprocessorEnvironment.class); - Configuration conf = new Configuration(false); - conf.set(NonTxIndexBuilder.CODEC_CLASS_NAME_KEY, PhoenixIndexCodec.class.getName()); - Mockito.when(env.getConfiguration()).thenReturn(conf); - - // the following is used by LocalTable#getCurrentRowState() - Region mockRegion = Mockito.mock(Region.class); - Mockito.when(env.getRegion()).thenReturn(mockRegion); - - Mockito.when(mockRegion.getScanner(Mockito.any(Scan.class))) - .thenAnswer(new Answer<RegionScanner>() { - @Override - public RegionScanner answer(InvocationOnMock invocation) throws Throwable { - Scan sArg = (Scan) invocation.getArguments()[0]; - TimeRange timeRange = sArg.getTimeRange(); - return getMockTimeRangeRegionScanner(timeRange); - } - }); - - // the following is called by PhoenixIndexCodec#getIndexUpserts() , getIndexDeletes() - HRegionInfo mockRegionInfo = Mockito.mock(HRegionInfo.class); - Mockito.when(mockRegion.getRegionInfo()).thenReturn(mockRegionInfo); - Mockito.when(mockRegionInfo.getStartKey()).thenReturn(Bytes.toBytes("a")); - Mockito.when(mockRegionInfo.getEndKey()).thenReturn(Bytes.toBytes("z")); - - mockIndexMetaData = Mockito.mock(PhoenixIndexMetaData.class); - Mockito.when(mockIndexMetaData.isImmutableRows()).thenReturn(false); - Mockito.when(mockIndexMetaData.getIndexMaintainers()) - .thenReturn(Collections.singletonList(getTestIndexMaintainer())); - - indexBuilder = new NonTxIndexBuilder(); - indexBuilder.setup(env); - } - - // returns a RegionScanner which filters currentRowCells using the given TimeRange. - // This is called from LocalTable#getCurrentRowState() - // If testIndexMetaData.ignoreNewerMutations() is not set, default TimeRange is 0 to - // Long.MAX_VALUE - private RegionScanner getMockTimeRangeRegionScanner(final TimeRange timeRange) { - return new BaseRegionScanner(Mockito.mock(RegionScanner.class)) { - @Override - public boolean next(List<Cell> results) throws IOException { - for (Cell cell : currentRowCells) { - if (cell.getTimestamp() >= timeRange.getMin() - && cell.getTimestamp() < timeRange.getMax()) { - results.add(cell); - } - } - return false; // indicate no more results - } - }; - } - - private IndexMaintainer getTestIndexMaintainer() throws Exception { - Properties props = PropertiesUtil.deepCopy(TestUtil.TEST_PROPERTIES); - // disable column encoding, makes debugging easier - props.put(QueryServices.DEFAULT_COLUMN_ENCODED_BYTES_ATRRIB, "0"); - Connection conn = DriverManager.getConnection(getUrl(), props); - try { - conn.setAutoCommit(true); - conn.createStatement().execute(TEST_TABLE_DDL); - conn.createStatement().execute(TEST_TABLE_INDEX_DDL); - PhoenixConnection pconn = conn.unwrap(PhoenixConnection.class); - PTable table = pconn.getTable(new PTableKey(pconn.getTenantId(), TEST_TABLE_STRING)); - ImmutableBytesWritable ptr = new ImmutableBytesWritable(); - table.getIndexMaintainers(ptr, pconn); - List<IndexMaintainer> indexMaintainerList = - IndexMaintainer.deserialize(ptr, GenericKeyValueBuilder.INSTANCE, true); - assertEquals(1, indexMaintainerList.size()); - IndexMaintainer indexMaintainer = indexMaintainerList.get(0); - return indexMaintainer; - } finally { - conn.close(); - } - } - - /** - * Tests that updating an indexed column results in a DeleteFamily (prior index cell) and a Put - * (new index cell) - */ - @Test - public void testGetMutableIndexUpdate() throws IOException { - setCurrentRowState(FAM, INDEXED_QUALIFIER, 1, VALUE_1); - - // update ts and value - Put put = new Put(ROW); - put.addImmutable(FAM, INDEXED_QUALIFIER, 2, VALUE_2); - MultiMutation mutation = new MultiMutation(new ImmutableBytesPtr(ROW)); - mutation.addAll(put); - - Collection<Pair<Mutation, byte[]>> indexUpdates = - indexBuilder.getIndexUpdate(mutation, mockIndexMetaData); - assertEquals(2, indexUpdates.size()); - assertContains(indexUpdates, 2, ROW, KeyValue.Type.DeleteFamily, FAM, - new byte[0] /* qual not needed */, 2); - assertContains(indexUpdates, ColumnTracker.NO_NEWER_PRIMARY_TABLE_ENTRY_TIMESTAMP, ROW, - KeyValue.Type.Put, FAM, QueryConstants.EMPTY_COLUMN_BYTES, 2); - } - - /** - * Tests a partial rebuild of a row with multiple versions. 3 versions of the row in data table, - * and we rebuild the index starting from time t=2 - * - * There should be one index row version per data row version. - */ - @Test - public void testRebuildMultipleVersionRow() throws IOException { - // when doing a rebuild, we are replaying mutations so we want to ignore newer mutations - // see LocalTable#getCurrentRowState() - Mockito.when(mockIndexMetaData.ignoreNewerMutations()).thenReturn(true); - - // the current row state has 3 versions, but if we rebuild as of t=2, scanner in LocalTable - // should only return first - Cell currentCell1 = CellUtil.createCell(ROW, FAM, INDEXED_QUALIFIER, 1, PUT_TYPE, VALUE_1); - Cell currentCell2 = CellUtil.createCell(ROW, FAM, INDEXED_QUALIFIER, 2, PUT_TYPE, VALUE_2); - Cell currentCell3 = CellUtil.createCell(ROW, FAM, INDEXED_QUALIFIER, 3, PUT_TYPE, VALUE_3); - Cell currentCell4 = CellUtil.createCell(ROW, FAM, INDEXED_QUALIFIER, 4, PUT_TYPE, VALUE_4); - setCurrentRowState(Arrays.asList(currentCell4, currentCell3, currentCell2, currentCell1)); - - // rebuilder replays mutations starting from t=2 - MultiMutation mutation = new MultiMutation(new ImmutableBytesPtr(ROW)); - Put put = new Put(ROW); - put.addImmutable(FAM, INDEXED_QUALIFIER, 4, VALUE_4); - mutation.addAll(put); - put = new Put(ROW); - put.addImmutable(FAM, INDEXED_QUALIFIER, 3, VALUE_3); - mutation.addAll(put); - put = new Put(ROW); - put.addImmutable(FAM, INDEXED_QUALIFIER, 2, VALUE_2); - mutation.addAll(put); - - Collection<Pair<Mutation, byte[]>> indexUpdates = - indexBuilder.getIndexUpdate(mutation, mockIndexMetaData); - // 3 puts and 3 deletes (one to hide existing index row for VALUE_1, and two to hide index - // rows for VALUE_2, VALUE_3) - assertEquals(6, indexUpdates.size()); - - assertContains(indexUpdates, 2, ROW, KeyValue.Type.DeleteFamily, FAM, - new byte[0] /* qual not needed */, 2); - assertContains(indexUpdates, ColumnTracker.NO_NEWER_PRIMARY_TABLE_ENTRY_TIMESTAMP, ROW, - KeyValue.Type.Put, FAM, QueryConstants.EMPTY_COLUMN_BYTES, 2); - assertContains(indexUpdates, 3, ROW, KeyValue.Type.DeleteFamily, FAM, - new byte[0] /* qual not needed */, 3); - assertContains(indexUpdates, ColumnTracker.NO_NEWER_PRIMARY_TABLE_ENTRY_TIMESTAMP, ROW, - KeyValue.Type.Put, FAM, QueryConstants.EMPTY_COLUMN_BYTES, 3); - assertContains(indexUpdates, 4, ROW, KeyValue.Type.DeleteFamily, FAM, - new byte[0] /* qual not needed */, 4); - assertContains(indexUpdates, ColumnTracker.NO_NEWER_PRIMARY_TABLE_ENTRY_TIMESTAMP, ROW, - KeyValue.Type.Put, FAM, QueryConstants.EMPTY_COLUMN_BYTES, 4); - } - - /** - * Tests getting an index update for a mutation with 200 versions Before, the issue PHOENIX-3807 - * was causing this test to take >90 seconds, so here we set a timeout of 5 seconds - */ - @Test(timeout = 10000) - public void testManyVersions() throws IOException { - // when doing a rebuild, we are replaying mutations so we want to ignore newer mutations - // see LocalTable#getCurrentRowState() - Mockito.when(mockIndexMetaData.ignoreNewerMutations()).thenReturn(true); - MultiMutation mutation = getMultipleVersionMutation(200); - currentRowCells = mutation.getFamilyCellMap().get(FAM); - - Collection<Pair<Mutation, byte[]>> indexUpdates = - indexBuilder.getIndexUpdate(mutation, mockIndexMetaData); - assertNotEquals(0, indexUpdates.size()); - } - - // Assert that the given collection of indexUpdates contains the given cell - private void assertContains(Collection<Pair<Mutation, byte[]>> indexUpdates, - final long mutationTs, final byte[] row, final Type cellType, final byte[] fam, - final byte[] qual, final long cellTs) { - Predicate<Pair<Mutation, byte[]>> hasCellPredicate = - new Predicate<Pair<Mutation, byte[]>>() { - @Override - public boolean apply(Pair<Mutation, byte[]> input) { - assertEquals(TEST_TABLE_INDEX_STRING, Bytes.toString(input.getSecond())); - Mutation mutation = input.getFirst(); - if (mutationTs == mutation.getTimeStamp()) { - NavigableMap<byte[], List<Cell>> familyCellMap = - mutation.getFamilyCellMap(); - Cell updateCell = familyCellMap.get(fam).get(0); - if (cellType == KeyValue.Type.codeToType(updateCell.getTypeByte()) - && Bytes.compareTo(fam, CellUtil.cloneFamily(updateCell)) == 0 - && Bytes.compareTo(qual, - CellUtil.cloneQualifier(updateCell)) == 0 - && cellTs == updateCell.getTimestamp()) { - return true; - } - } - return false; - } - }; - Optional<Pair<Mutation, byte[]>> tryFind = - Iterables.tryFind(indexUpdates, hasCellPredicate); - assertTrue(tryFind.isPresent()); - } - - private void setCurrentRowState(byte[] fam2, byte[] indexedQualifier, int i, byte[] value1) { - Cell cell = CellUtil.createCell(ROW, FAM, INDEXED_QUALIFIER, 1, PUT_TYPE, VALUE_1); - currentRowCells = Collections.singletonList(cell); - } - - private void setCurrentRowState(List<Cell> cells) { - currentRowCells = cells; - } - - private MultiMutation getMultipleVersionMutation(int versions) { - MultiMutation mutation = new MultiMutation(new ImmutableBytesPtr(ROW)); - for (int i = versions - 1; i >= 0; i--) { - Put put = new Put(ROW); - put.addImmutable(FAM, INDEXED_QUALIFIER, i, Bytes.toBytes(i)); - mutation.addAll(put); - } - return mutation; - } -} http://git-wip-us.apache.org/repos/asf/phoenix/blob/572d242d/phoenix-core/src/test/java/org/apache/phoenix/util/IndexScrutiny.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/test/java/org/apache/phoenix/util/IndexScrutiny.java b/phoenix-core/src/test/java/org/apache/phoenix/util/IndexScrutiny.java new file mode 100644 index 0000000..c78658d --- /dev/null +++ b/phoenix-core/src/test/java/org/apache/phoenix/util/IndexScrutiny.java @@ -0,0 +1,144 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.phoenix.util; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.ResultSetMetaData; +import java.sql.SQLException; + +import org.apache.phoenix.jdbc.PhoenixConnection; +import org.apache.phoenix.schema.PColumn; +import org.apache.phoenix.schema.PTable; +import org.apache.phoenix.schema.PTableKey; +import org.apache.phoenix.schema.types.PDataType; + +import com.google.common.base.Objects; + +public class IndexScrutiny { + + public static long scrutinizeIndex(Connection conn, String fullTableName, String fullIndexName) throws SQLException { + PhoenixConnection pconn = conn.unwrap(PhoenixConnection.class); + PTable ptable = pconn.getTable(new PTableKey(pconn.getTenantId(), fullTableName)); + PTable pindex = pconn.getTable(new PTableKey(pconn.getTenantId(), fullIndexName)); + StringBuilder indexQueryBuf = new StringBuilder("SELECT "); + for (PColumn dcol : ptable.getPKColumns()) { + indexQueryBuf.append("CAST(\"" + IndexUtil.getIndexColumnName(dcol) + "\" AS " + dcol.getDataType().getSqlTypeName() + ")"); + indexQueryBuf.append(","); + } + for (PColumn icol : pindex.getColumns()) { + PColumn dcol = IndexUtil.getDataColumn(ptable, icol.getName().getString()); + if (SchemaUtil.isPKColumn(icol) && !SchemaUtil.isPKColumn(dcol)) { + indexQueryBuf.append("CAST (\"" + icol.getName().getString() + "\" AS " + dcol.getDataType().getSqlTypeName() + ")"); + indexQueryBuf.append(","); + } + } + for (PColumn icol : pindex.getColumns()) { + if (!SchemaUtil.isPKColumn(icol)) { + PColumn dcol = IndexUtil.getDataColumn(ptable, icol.getName().getString()); + indexQueryBuf.append("CAST (\"" + icol.getName().getString() + "\" AS " + dcol.getDataType().getSqlTypeName() + ")"); + indexQueryBuf.append(","); + } + } + indexQueryBuf.setLength(indexQueryBuf.length()-1); + indexQueryBuf.append("\nFROM " + fullIndexName); + + StringBuilder tableQueryBuf = new StringBuilder("SELECT "); + for (PColumn dcol : ptable.getPKColumns()) { + tableQueryBuf.append("\"" + dcol.getName().getString() + "\""); + tableQueryBuf.append(","); + } + for (PColumn icol : pindex.getColumns()) { + PColumn dcol = IndexUtil.getDataColumn(ptable, icol.getName().getString()); + if (SchemaUtil.isPKColumn(icol) && !SchemaUtil.isPKColumn(dcol)) { + if (dcol.getFamilyName() != null) { + tableQueryBuf.append("\"" + dcol.getFamilyName().getString() + "\""); + tableQueryBuf.append("."); + } + tableQueryBuf.append("\"" + dcol.getName().getString() + "\""); + tableQueryBuf.append(","); + } + } + for (PColumn icol : pindex.getColumns()) { + if (!SchemaUtil.isPKColumn(icol)) { + PColumn dcol = IndexUtil.getDataColumn(ptable, icol.getName().getString()); + if (dcol.getFamilyName() != null) { + tableQueryBuf.append("\"" + dcol.getFamilyName().getString() + "\""); + tableQueryBuf.append("."); + } + tableQueryBuf.append("\"" + dcol.getName().getString() + "\""); + tableQueryBuf.append(","); + } + } + tableQueryBuf.setLength(tableQueryBuf.length()-1); + tableQueryBuf.append("\nFROM " + fullTableName + "\nWHERE ("); + for (PColumn dcol : ptable.getPKColumns()) { + tableQueryBuf.append("\"" + dcol.getName().getString() + "\""); + tableQueryBuf.append(","); + } + tableQueryBuf.setLength(tableQueryBuf.length()-1); + tableQueryBuf.append(") = (("); + for (int i = 0; i < ptable.getPKColumns().size(); i++) { + tableQueryBuf.append("?"); + tableQueryBuf.append(","); + } + tableQueryBuf.setLength(tableQueryBuf.length()-1); + tableQueryBuf.append("))"); + + String tableQuery = tableQueryBuf.toString(); + PreparedStatement istmt = conn.prepareStatement(tableQuery); + + String indexQuery = indexQueryBuf.toString(); + ResultSet irs = conn.createStatement().executeQuery(indexQuery); + ResultSetMetaData irsmd = irs.getMetaData(); + long icount = 0; + while (irs.next()) { + icount++; + StringBuilder pkBuf = new StringBuilder("("); + for (int i = 0; i < ptable.getPKColumns().size(); i++) { + PColumn dcol = ptable.getPKColumns().get(i); + Object pkVal = irs.getObject(i+1); + PDataType pkType = PDataType.fromTypeId(irsmd.getColumnType(i + 1)); + istmt.setObject(i+1, pkVal, dcol.getDataType().getSqlType()); + pkBuf.append(pkType.toStringLiteral(pkVal)); + pkBuf.append(","); + } + pkBuf.setLength(pkBuf.length()-1); + pkBuf.append(")"); + ResultSet drs = istmt.executeQuery(); + ResultSetMetaData drsmd = drs.getMetaData(); + assertTrue("Expected to find PK in data table: " + pkBuf, drs.next()); + for (int i = 0; i < irsmd.getColumnCount(); i++) { + Object iVal = irs.getObject(i + 1); + PDataType iType = PDataType.fromTypeId(irsmd.getColumnType(i + 1)); + Object dVal = drs.getObject(i + 1); + PDataType dType = PDataType.fromTypeId(drsmd.getColumnType(i + 1)); + assertTrue("Expected equality for " + drsmd.getColumnName(i + 1) + ", but " + iType.toStringLiteral(iVal) + "!=" + dType.toStringLiteral(dVal), Objects.equal(iVal, dVal)); + } + } + + long dcount = TestUtil.getRowCount(conn, fullTableName); + assertEquals("Expected data table row count to match", dcount, icount); + return dcount; + } + +} http://git-wip-us.apache.org/repos/asf/phoenix/blob/572d242d/phoenix-core/src/test/java/org/apache/phoenix/util/TestUtil.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/test/java/org/apache/phoenix/util/TestUtil.java b/phoenix-core/src/test/java/org/apache/phoenix/util/TestUtil.java index 0e4e7b8..5564e49 100644 --- a/phoenix-core/src/test/java/org/apache/phoenix/util/TestUtil.java +++ b/phoenix-core/src/test/java/org/apache/phoenix/util/TestUtil.java @@ -41,7 +41,6 @@ import java.sql.Date; import java.sql.DriverManager; import java.sql.PreparedStatement; import java.sql.ResultSet; -import java.sql.ResultSetMetaData; import java.sql.SQLException; import java.sql.Statement; import java.util.ArrayList; @@ -122,7 +121,6 @@ import org.apache.phoenix.schema.stats.GuidePostsKey; import org.apache.phoenix.schema.tuple.Tuple; import org.apache.phoenix.schema.types.PDataType; -import com.google.common.base.Objects; import com.google.common.collect.Lists; @@ -882,112 +880,7 @@ public class TestUtil { } } - public static long scrutinizeIndex(Connection conn, String fullTableName, String fullIndexName) throws SQLException { - PhoenixConnection pconn = conn.unwrap(PhoenixConnection.class); - PTable ptable = pconn.getTable(new PTableKey(pconn.getTenantId(), fullTableName)); - PTable pindex = pconn.getTable(new PTableKey(pconn.getTenantId(), fullIndexName)); - StringBuilder indexQueryBuf = new StringBuilder("SELECT "); - for (PColumn dcol : ptable.getPKColumns()) { - indexQueryBuf.append("CAST(\"" + IndexUtil.getIndexColumnName(dcol) + "\" AS " + dcol.getDataType().getSqlTypeName() + ")"); - indexQueryBuf.append(","); - } - for (PColumn icol : pindex.getColumns()) { - PColumn dcol = IndexUtil.getDataColumn(ptable, icol.getName().getString()); - if (SchemaUtil.isPKColumn(icol) && !SchemaUtil.isPKColumn(dcol)) { - indexQueryBuf.append("CAST (\"" + icol.getName().getString() + "\" AS " + dcol.getDataType().getSqlTypeName() + ")"); - indexQueryBuf.append(","); - } - } - for (PColumn icol : pindex.getColumns()) { - if (!SchemaUtil.isPKColumn(icol)) { - PColumn dcol = IndexUtil.getDataColumn(ptable, icol.getName().getString()); - indexQueryBuf.append("CAST (\"" + icol.getName().getString() + "\" AS " + dcol.getDataType().getSqlTypeName() + ")"); - indexQueryBuf.append(","); - } - } - indexQueryBuf.setLength(indexQueryBuf.length()-1); - indexQueryBuf.append("\nFROM " + fullIndexName); - - StringBuilder tableQueryBuf = new StringBuilder("SELECT "); - for (PColumn dcol : ptable.getPKColumns()) { - tableQueryBuf.append("\"" + dcol.getName().getString() + "\""); - tableQueryBuf.append(","); - } - for (PColumn icol : pindex.getColumns()) { - PColumn dcol = IndexUtil.getDataColumn(ptable, icol.getName().getString()); - if (SchemaUtil.isPKColumn(icol) && !SchemaUtil.isPKColumn(dcol)) { - if (dcol.getFamilyName() != null) { - tableQueryBuf.append("\"" + dcol.getFamilyName().getString() + "\""); - tableQueryBuf.append("."); - } - tableQueryBuf.append("\"" + dcol.getName().getString() + "\""); - tableQueryBuf.append(","); - } - } - for (PColumn icol : pindex.getColumns()) { - if (!SchemaUtil.isPKColumn(icol)) { - PColumn dcol = IndexUtil.getDataColumn(ptable, icol.getName().getString()); - if (dcol.getFamilyName() != null) { - tableQueryBuf.append("\"" + dcol.getFamilyName().getString() + "\""); - tableQueryBuf.append("."); - } - tableQueryBuf.append("\"" + dcol.getName().getString() + "\""); - tableQueryBuf.append(","); - } - } - tableQueryBuf.setLength(tableQueryBuf.length()-1); - tableQueryBuf.append("\nFROM " + fullTableName + "\nWHERE ("); - for (PColumn dcol : ptable.getPKColumns()) { - tableQueryBuf.append("\"" + dcol.getName().getString() + "\""); - tableQueryBuf.append(","); - } - tableQueryBuf.setLength(tableQueryBuf.length()-1); - tableQueryBuf.append(") = (("); - for (int i = 0; i < ptable.getPKColumns().size(); i++) { - tableQueryBuf.append("?"); - tableQueryBuf.append(","); - } - tableQueryBuf.setLength(tableQueryBuf.length()-1); - tableQueryBuf.append("))"); - - String tableQuery = tableQueryBuf.toString(); - PreparedStatement istmt = conn.prepareStatement(tableQuery); - - String indexQuery = indexQueryBuf.toString(); - ResultSet irs = conn.createStatement().executeQuery(indexQuery); - ResultSetMetaData irsmd = irs.getMetaData(); - long icount = 0; - while (irs.next()) { - icount++; - StringBuilder pkBuf = new StringBuilder("("); - for (int i = 0; i < ptable.getPKColumns().size(); i++) { - PColumn dcol = ptable.getPKColumns().get(i); - Object pkVal = irs.getObject(i+1); - PDataType pkType = PDataType.fromTypeId(irsmd.getColumnType(i + 1)); - istmt.setObject(i+1, pkVal, dcol.getDataType().getSqlType()); - pkBuf.append(pkType.toStringLiteral(pkVal)); - pkBuf.append(","); - } - pkBuf.setLength(pkBuf.length()-1); - pkBuf.append(")"); - ResultSet drs = istmt.executeQuery(); - ResultSetMetaData drsmd = drs.getMetaData(); - assertTrue("Expected to find PK in data table: " + pkBuf, drs.next()); - for (int i = 0; i < irsmd.getColumnCount(); i++) { - Object iVal = irs.getObject(i + 1); - PDataType iType = PDataType.fromTypeId(irsmd.getColumnType(i + 1)); - Object dVal = drs.getObject(i + 1); - PDataType dType = PDataType.fromTypeId(drsmd.getColumnType(i + 1)); - assertTrue("Expected equality for " + drsmd.getColumnName(i + 1) + ", but " + iType.toStringLiteral(iVal) + "!=" + dType.toStringLiteral(dVal), Objects.equal(iVal, dVal)); - } - } - - long dcount = getRowCount(conn, fullTableName); - assertEquals("Expected data table row count to match", dcount, icount); - return dcount; - } - - private static long getRowCount(Connection conn, String tableName) throws SQLException { + public static long getRowCount(Connection conn, String tableName) throws SQLException { ResultSet rs = conn.createStatement().executeQuery("SELECT /*+ NO_INDEX */ count(*) FROM " + tableName); assertTrue(rs.next()); return rs.getLong(1);
