Refactor code to store and use column qualifiers in SYSTEM.CATALOG
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/a41074a9 Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/a41074a9 Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/a41074a9 Branch: refs/heads/encodecolumns2 Commit: a41074a9601901c80ac4c37a69ef606bffe51171 Parents: be6861c Author: Samarth <samarth.j...@salesforce.com> Authored: Wed Dec 21 13:15:02 2016 -0800 Committer: Thomas D'Silva <tdsi...@salesforce.com> Committed: Thu Dec 22 13:03:52 2016 -0800 ---------------------------------------------------------------------- .../apache/phoenix/end2end/AlterTableIT.java | 7 +- .../apache/phoenix/end2end/StoreNullsIT.java | 10 +- .../phoenix/end2end/index/DropMetadataIT.java | 4 +- .../apache/phoenix/compile/FromCompiler.java | 19 ++- .../apache/phoenix/compile/JoinCompiler.java | 9 +- .../phoenix/compile/ListJarsQueryPlan.java | 6 +- .../apache/phoenix/compile/PostDDLCompiler.java | 2 +- .../phoenix/compile/ProjectionCompiler.java | 14 +- .../apache/phoenix/compile/TraceQueryPlan.java | 4 +- .../compile/TupleProjectionCompiler.java | 8 +- .../apache/phoenix/compile/UnionCompiler.java | 3 +- .../apache/phoenix/compile/WhereCompiler.java | 2 +- .../coprocessor/BaseScannerRegionObserver.java | 1 + .../GroupedAggregateRegionObserver.java | 6 +- .../coprocessor/MetaDataEndpointImpl.java | 26 +-- .../phoenix/coprocessor/ScanRegionObserver.java | 11 +- .../UngroupedAggregateRegionObserver.java | 3 +- .../coprocessor/generated/PTableProtos.java | 157 ++++++++++--------- .../apache/phoenix/execute/BaseQueryPlan.java | 2 +- .../expression/ArrayColumnExpression.java | 30 +++- .../expression/KeyValueColumnExpression.java | 30 ++-- .../apache/phoenix/index/IndexMaintainer.java | 29 ++-- .../phoenix/iterate/BaseResultIterators.java | 23 +-- .../iterate/RegionScannerResultIterator.java | 7 +- .../phoenix/jdbc/PhoenixDatabaseMetaData.java | 4 +- .../mapreduce/FormatToBytesWritableMapper.java | 2 +- .../mapreduce/FormatToKeyValueReducer.java | 2 +- .../apache/phoenix/query/QueryConstants.java | 18 ++- .../org/apache/phoenix/schema/ColumnRef.java | 6 +- .../apache/phoenix/schema/DelegateColumn.java | 4 +- .../apache/phoenix/schema/MetaDataClient.java | 34 ++-- .../java/org/apache/phoenix/schema/PColumn.java | 2 +- .../phoenix/schema/PColumnFamilyImpl.java | 41 +++-- .../org/apache/phoenix/schema/PColumnImpl.java | 28 ++-- .../apache/phoenix/schema/PMetaDataImpl.java | 2 +- .../java/org/apache/phoenix/schema/PTable.java | 135 +++++++--------- .../org/apache/phoenix/schema/PTableImpl.java | 118 ++++++++++---- .../apache/phoenix/schema/ProjectedColumn.java | 11 +- .../tuple/EncodedColumnQualiferCellsList.java | 19 ++- .../tuple/PositionBasedMultiKeyValueTuple.java | 3 +- .../schema/tuple/PositionBasedResultTuple.java | 3 +- .../apache/phoenix/util/EncodedColumnsUtil.java | 77 +++++---- .../java/org/apache/phoenix/util/IndexUtil.java | 9 +- .../phoenix/compile/WhereCompilerTest.java | 4 +- .../phoenix/execute/CorrelatePlanTest.java | 3 +- .../execute/LiteralResultIteratorPlanTest.java | 3 +- .../phoenix/execute/UnnestArrayPlanTest.java | 7 +- .../expression/ColumnExpressionTest.java | 35 +++-- .../EncodedColumnQualifierCellsListTest.java | 98 ++++++------ .../java/org/apache/phoenix/util/TestUtil.java | 10 +- phoenix-protocol/src/main/PTable.proto | 2 +- 51 files changed, 601 insertions(+), 492 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/phoenix/blob/a41074a9/phoenix-core/src/it/java/org/apache/phoenix/end2end/AlterTableIT.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/AlterTableIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/AlterTableIT.java index 3084a92..91e9964 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/AlterTableIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/AlterTableIT.java @@ -20,8 +20,8 @@ package org.apache.phoenix.end2end; import static org.apache.hadoop.hbase.HColumnDescriptor.DEFAULT_REPLICATION_SCOPE; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.COLUMN_FAMILY; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.COLUMN_NAME; +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.COLUMN_QUALIFIER; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.COLUMN_QUALIFIER_COUNTER; -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.ENCODED_COLUMN_QUALIFIER; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TABLE_NAME; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TABLE_SCHEM; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TABLE_SEQ_NUM; @@ -65,6 +65,7 @@ import org.apache.phoenix.query.QueryConstants; import org.apache.phoenix.query.QueryServices; import org.apache.phoenix.schema.PTable; import org.apache.phoenix.schema.PTable.EncodedCQCounter; +import org.apache.phoenix.schema.PTable.QualifierEncodingScheme; import org.apache.phoenix.schema.PTableKey; import org.apache.phoenix.schema.TableNotFoundException; import org.apache.phoenix.util.IndexUtil; @@ -2454,7 +2455,7 @@ public class AlterTableIT extends ParallelStatsDisabledIT { } private void assertEncodedCQValue(String columnFamily, String columnName, String schemaName, String tableName, int expectedValue) throws Exception { - String query = "SELECT " + ENCODED_COLUMN_QUALIFIER + " FROM SYSTEM.CATALOG WHERE " + TABLE_SCHEM + " = ? AND " + TABLE_NAME + String query = "SELECT " + COLUMN_QUALIFIER + " FROM SYSTEM.CATALOG WHERE " + TABLE_SCHEM + " = ? AND " + TABLE_NAME + " = ? " + " AND " + COLUMN_FAMILY + " = ?" + " AND " + COLUMN_NAME + " = ?"; try (Connection conn = DriverManager.getConnection(getUrl())) { PreparedStatement stmt = conn.prepareStatement(query); @@ -2464,7 +2465,7 @@ public class AlterTableIT extends ParallelStatsDisabledIT { stmt.setString(4, columnName); ResultSet rs = stmt.executeQuery(); assertTrue(rs.next()); - assertEquals(expectedValue, rs.getInt(1)); + assertTrue(Bytes.equals(QualifierEncodingScheme.FOUR_BYTE_QUALIFIERS.getEncodedBytes(expectedValue), rs.getBytes(1))); assertFalse(rs.next()); } } http://git-wip-us.apache.org/repos/asf/phoenix/blob/a41074a9/phoenix-core/src/it/java/org/apache/phoenix/end2end/StoreNullsIT.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/StoreNullsIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/StoreNullsIT.java index 07fc4e9..8e1db17 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/StoreNullsIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/StoreNullsIT.java @@ -17,14 +17,11 @@ */ package org.apache.phoenix.end2end; -import static org.apache.phoenix.query.QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES; -import static org.apache.phoenix.query.QueryConstants.ENCODED_EMPTY_COLUMN_BYTES; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; -import java.io.IOException; import java.sql.Connection; import java.sql.DriverManager; import java.sql.ResultSet; @@ -34,24 +31,21 @@ import java.util.Properties; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.client.HTable; import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.client.ResultScanner; import org.apache.hadoop.hbase.client.Scan; -import org.apache.hadoop.hbase.util.Bytes; import org.apache.phoenix.expression.ArrayColumnExpression; import org.apache.phoenix.hbase.index.util.ImmutableBytesPtr; import org.apache.phoenix.jdbc.PhoenixConnection; -import org.apache.phoenix.query.QueryConstants; import org.apache.phoenix.query.QueryServices; import org.apache.phoenix.schema.PColumn; +import org.apache.phoenix.schema.PTable.QualifierEncodingScheme; import org.apache.phoenix.schema.PTableKey; import org.apache.phoenix.schema.tuple.ResultTuple; import org.apache.phoenix.schema.types.PVarchar; import org.apache.phoenix.util.ByteUtil; import org.apache.phoenix.util.PhoenixRuntime; -import org.apache.phoenix.util.SchemaUtil; import org.apache.phoenix.util.TestUtil; import org.junit.After; import org.junit.Before; @@ -127,7 +121,7 @@ public class StoreNullsIT extends ParallelStatsDisabledIT { Result rs = scanner.next(); assertTrue(rs.size() == 2); // 2 because it also includes the empty key value column PColumn nameColumn = conn.unwrap(PhoenixConnection.class).getTable(new PTableKey(null, tableName)).getPColumnForColumnName("NAME"); - ArrayColumnExpression colExpression = new ArrayColumnExpression(nameColumn, "NAME", true); + ArrayColumnExpression colExpression = new ArrayColumnExpression(nameColumn, "NAME", QualifierEncodingScheme.FOUR_BYTE_QUALIFIERS); ImmutableBytesPtr ptr = new ImmutableBytesPtr(); colExpression.evaluate(new ResultTuple(rs), ptr); assertEquals(new ImmutableBytesPtr(PVarchar.INSTANCE.toBytes("v1")), ptr); http://git-wip-us.apache.org/repos/asf/phoenix/blob/a41074a9/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/DropMetadataIT.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/DropMetadataIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/DropMetadataIT.java index b892c4d..c769fdd 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/DropMetadataIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/DropMetadataIT.java @@ -187,7 +187,7 @@ public class DropMetadataIT extends ParallelStatsDisabledIT { Result result = results.next(); assertNotNull(result); assertNotNull("localIndexTableName2 row is missing", result.getValue(QueryConstants.DEFAULT_LOCAL_INDEX_COLUMN_FAMILY_BYTES, - EncodedColumnsUtil.getColumnQualifier(localIndex2.getPColumnForColumnName(IndexUtil.getIndexColumnName(QueryConstants.DEFAULT_COLUMN_FAMILY, "V1")), localIndex2))); + localIndex2.getPColumnForColumnName(IndexUtil.getIndexColumnName(QueryConstants.DEFAULT_COLUMN_FAMILY, "V1")).getColumnQualifierBytes())); assertNull(results.next()); } } @@ -300,7 +300,7 @@ public class DropMetadataIT extends ParallelStatsDisabledIT { assertNotNull(result); PTable viewIndexPTable = pconn.getTable(new PTableKey(pconn.getTenantId(), viewIndex2)); PColumn column = viewIndexPTable.getPColumnForColumnName(IndexUtil.getIndexColumnName(QueryConstants.DEFAULT_COLUMN_FAMILY, "V4")); - byte[] cq = EncodedColumnsUtil.getColumnQualifier(column, viewIndexPTable); + byte[] cq = column.getColumnQualifierBytes(); // there should be a single row belonging to VIEWINDEX2 assertNotNull(viewIndex2 + " row is missing", result.getValue(QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES, cq)); assertNull(results.next()); http://git-wip-us.apache.org/repos/asf/phoenix/blob/a41074a9/phoenix-core/src/main/java/org/apache/phoenix/compile/FromCompiler.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/compile/FromCompiler.java b/phoenix-core/src/main/java/org/apache/phoenix/compile/FromCompiler.java index 8d00996..1a14c7f 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/compile/FromCompiler.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/FromCompiler.java @@ -17,6 +17,8 @@ */ package org.apache.phoenix.compile; +import static org.apache.phoenix.schema.PTable.QualifierEncodingScheme.NON_ENCODED_QUALIFIERS; + import java.sql.SQLException; import java.sql.SQLFeatureNotSupportedException; import java.util.ArrayList; @@ -28,6 +30,7 @@ import java.util.Map; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.client.HTableInterface; +import org.apache.hadoop.hbase.util.Bytes; import org.apache.phoenix.coprocessor.MetaDataProtocol; import org.apache.phoenix.coprocessor.MetaDataProtocol.MetaDataMutationResult; import org.apache.phoenix.expression.Expression; @@ -189,7 +192,7 @@ public class FromCompiler { if (htable != null) Closeables.closeQuietly(htable); } tableNode = NamedTableNode.create(null, baseTable, statement.getColumnDefs()); - return new SingleTableColumnResolver(connection, tableNode, e.getTimeStamp(), new HashMap<String, UDFParseNode>(1), false, false); + return new SingleTableColumnResolver(connection, tableNode, e.getTimeStamp(), new HashMap<String, UDFParseNode>(1), false); } throw e; } @@ -261,7 +264,7 @@ public class FromCompiler { Expression sourceExpression = projector.getColumnProjector(column.getPosition()).getExpression(); PColumnImpl projectedColumn = new PColumnImpl(column.getName(), column.getFamilyName(), sourceExpression.getDataType(), sourceExpression.getMaxLength(), sourceExpression.getScale(), sourceExpression.isNullable(), - column.getPosition(), sourceExpression.getSortOrder(), column.getArraySize(), column.getViewConstant(), column.isViewReferenced(), column.getExpressionStr(), column.isRowTimestamp(), column.isDynamic(), column.getEncodedColumnQualifier()); + column.getPosition(), sourceExpression.getSortOrder(), column.getArraySize(), column.getViewConstant(), column.isViewReferenced(), column.getExpressionStr(), column.isRowTimestamp(), column.isDynamic(), column.getColumnQualifierBytes()); projectedColumns.add(projectedColumn); } PTable t = PTableImpl.makePTable(table, projectedColumns); @@ -336,12 +339,12 @@ public class FromCompiler { private final String alias; private final List<PSchema> schemas; - public SingleTableColumnResolver(PhoenixConnection connection, NamedTableNode table, long timeStamp, Map<String, UDFParseNode> udfParseNodes, boolean isNamespaceMapped, boolean useEncodedColumnNames) throws SQLException { + public SingleTableColumnResolver(PhoenixConnection connection, NamedTableNode table, long timeStamp, Map<String, UDFParseNode> udfParseNodes, boolean isNamespaceMapped) throws SQLException { super(connection, 0, false, udfParseNodes); List<PColumnFamily> families = Lists.newArrayListWithExpectedSize(table.getDynamicColumns().size()); for (ColumnDef def : table.getDynamicColumns()) { if (def.getColumnDefName().getFamilyName() != null) { - families.add(new PColumnFamilyImpl(PNameFactory.newName(def.getColumnDefName().getFamilyName()),Collections.<PColumn>emptyList(), useEncodedColumnNames)); + families.add(new PColumnFamilyImpl(PNameFactory.newName(def.getColumnDefName().getFamilyName()),Collections.<PColumn>emptyList()));//, NON_ENCODED_QUALIFIERS)); } } Long scn = connection.getSCN(); @@ -349,6 +352,8 @@ public class FromCompiler { if (connection.getSchema() != null) { schema = schema != null ? schema : connection.getSchema(); } + // Storage scheme and encoding scheme don't matter here since the PTable is being used only for the purposes of create table. + // The actual values of these two will be determined by the metadata client. PTable theTable = new PTableImpl(connection.getTenantId(), schema, table.getName().getTableName(), scn == null ? HConstants.LATEST_TIMESTAMP : scn, families, isNamespaceMapped); theTable = this.addDynamicColumns(table.getDynamicColumns(), theTable); @@ -675,9 +680,8 @@ public class FromCompiler { theTable.getColumnFamily(family); // Verifies that column family exists familyName = PNameFactory.newName(family); } - // Dynamic columns don't have an encoded column name allcolumns.add(new PColumnImpl(name, familyName, dynColumn.getDataType(), dynColumn.getMaxLength(), - dynColumn.getScale(), dynColumn.isNull(), position, dynColumn.getSortOrder(), dynColumn.getArraySize(), null, false, dynColumn.getExpression(), false, true, null)); + dynColumn.getScale(), dynColumn.isNull(), position, dynColumn.getSortOrder(), dynColumn.getArraySize(), null, false, dynColumn.getExpression(), false, true, Bytes.toBytes(dynColumn.getColumnDefName().getColumnName()))); position++; } theTable = PTableImpl.makePTable(theTable, allcolumns); @@ -779,9 +783,10 @@ public class FromCompiler { // referenced by an outer wild-card select. alias = String.valueOf(position); } + PName name = PNameFactory.newName(alias); PColumnImpl column = new PColumnImpl(PNameFactory.newName(alias), PNameFactory.newName(QueryConstants.DEFAULT_COLUMN_FAMILY), - null, 0, 0, true, position++, SortOrder.ASC, null, null, false, null, false, false, null); + null, 0, 0, true, position++, SortOrder.ASC, null, null, false, null, false, false, name.getBytes()); columns.add(column); } PTable t = PTableImpl.makePTable(null, PName.EMPTY_NAME, PName.EMPTY_NAME, PTableType.SUBQUERY, null, http://git-wip-us.apache.org/repos/asf/phoenix/blob/a41074a9/phoenix-core/src/main/java/org/apache/phoenix/compile/JoinCompiler.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/compile/JoinCompiler.java b/phoenix-core/src/main/java/org/apache/phoenix/compile/JoinCompiler.java index b72c550..1db28f5 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/compile/JoinCompiler.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/JoinCompiler.java @@ -17,6 +17,9 @@ */ package org.apache.phoenix.compile; +import static org.apache.phoenix.schema.PTable.QualifierEncodingScheme.NON_ENCODED_QUALIFIERS; +import static org.apache.phoenix.schema.PTable.StorageScheme.ONE_CELL_PER_KEYVALUE_COLUMN; + import java.sql.SQLException; import java.util.ArrayList; import java.util.Collections; @@ -1287,7 +1290,7 @@ public class JoinCompiler { if (type == JoinType.Full) { for (PColumn c : left.getColumns()) { merged.add(new ProjectedColumn(c.getName(), c.getFamilyName(), - c.getPosition(), true, ((ProjectedColumn) c).getSourceColumnRef())); + c.getPosition(), true, ((ProjectedColumn) c).getSourceColumnRef(), SchemaUtil.isPKColumn(c) ? null : c.getName().getBytes())); } } else { merged.addAll(left.getColumns()); @@ -1297,7 +1300,7 @@ public class JoinCompiler { if (!SchemaUtil.isPKColumn(c)) { PColumn column = new ProjectedColumn(c.getName(), c.getFamilyName(), position++, type == JoinType.Inner ? c.isNullable() : true, - ((ProjectedColumn) c).getSourceColumnRef()); + ((ProjectedColumn) c).getSourceColumnRef(), c.getName().getBytes()); merged.add(column); } } @@ -1313,7 +1316,7 @@ public class JoinCompiler { left.isMultiTenant(), left.getStoreNulls(), left.getViewType(), left.getViewIndexId(), left.getIndexType(), left.rowKeyOrderOptimizable(), left.isTransactional(), left.getUpdateCacheFrequency(), left.getIndexDisableTimestamp(), left.isNamespaceMapped(), - left.getAutoPartitionSeqName(), left.isAppendOnlySchema(), StorageScheme.ONE_CELL_PER_KEYVALUE_COLUMN, QualifierEncodingScheme.NON_ENCODED_QUALIFIERS, PTable.EncodedCQCounter.NULL_COUNTER); + left.getAutoPartitionSeqName(), left.isAppendOnlySchema(), ONE_CELL_PER_KEYVALUE_COLUMN, NON_ENCODED_QUALIFIERS, PTable.EncodedCQCounter.NULL_COUNTER); } } http://git-wip-us.apache.org/repos/asf/phoenix/blob/a41074a9/phoenix-core/src/main/java/org/apache/phoenix/compile/ListJarsQueryPlan.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/compile/ListJarsQueryPlan.java b/phoenix-core/src/main/java/org/apache/phoenix/compile/ListJarsQueryPlan.java index 436e69b..8265de8 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/compile/ListJarsQueryPlan.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/ListJarsQueryPlan.java @@ -57,6 +57,7 @@ import org.apache.phoenix.query.KeyRange; import org.apache.phoenix.query.QueryServices; import org.apache.phoenix.schema.PColumn; import org.apache.phoenix.schema.PColumnImpl; +import org.apache.phoenix.schema.PName; import org.apache.phoenix.schema.PNameFactory; import org.apache.phoenix.schema.RowKeyValueAccessor; import org.apache.phoenix.schema.SortOrder; @@ -77,10 +78,11 @@ public class ListJarsQueryPlan implements QueryPlan { static { List<ExpressionProjector> projectedColumns = new ArrayList<ExpressionProjector>(); + PName colName = PNameFactory.newName("jar_location"); PColumn column = - new PColumnImpl(PNameFactory.newName("jar_location"), null, + new PColumnImpl(colName, null, PVarchar.INSTANCE, null, null, false, 0, SortOrder.getDefault(), 0, null, - false, null, false, false, null); + false, null, false, false, colName.getBytes()); List<PColumn> columns = new ArrayList<PColumn>(); columns.add(column); Expression expression = http://git-wip-us.apache.org/repos/asf/phoenix/blob/a41074a9/phoenix-core/src/main/java/org/apache/phoenix/compile/PostDDLCompiler.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/compile/PostDDLCompiler.java b/phoenix-core/src/main/java/org/apache/phoenix/compile/PostDDLCompiler.java index 3820ee5..e4ef25f 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/compile/PostDDLCompiler.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/PostDDLCompiler.java @@ -238,7 +238,7 @@ public class PostDDLCompiler { // data empty column family to stay the same, while the index empty column family // changes. PColumn column = deleteList.get(0); - byte[] cq = EncodedColumnsUtil.getColumnQualifier(column, tableRef.getTable()); + byte[] cq = column.getColumnQualifierBytes(); if (emptyCF == null) { scan.addColumn(column.getFamilyName().getBytes(), cq); } http://git-wip-us.apache.org/repos/asf/phoenix/blob/a41074a9/phoenix-core/src/main/java/org/apache/phoenix/compile/ProjectionCompiler.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/compile/ProjectionCompiler.java b/phoenix-core/src/main/java/org/apache/phoenix/compile/ProjectionCompiler.java index 25aee35..e2e7573 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/compile/ProjectionCompiler.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/ProjectionCompiler.java @@ -485,11 +485,13 @@ public class ProjectionCompiler { } } else { for (byte[] cq : entry.getValue()) { - PColumn column = family.getPColumnForColumnQualifier(cq); - Integer maxLength = column.getMaxLength(); - int byteSize = column.getDataType().isFixedWidth() ? maxLength == null ? column.getDataType().getByteSize() : maxLength : RowKeySchema.ESTIMATED_VARIABLE_LENGTH_SIZE; - estimatedByteSize += SizedUtil.KEY_VALUE_SIZE + estimatedKeySize + byteSize; - } + //if (!Bytes.equals(cq, ByteUtil.EMPTY_BYTE_ARRAY) || cq.length > 0) { + PColumn column = family.getPColumnForColumnQualifier(cq); + Integer maxLength = column.getMaxLength(); + int byteSize = column.getDataType().isFixedWidth() ? maxLength == null ? column.getDataType().getByteSize() : maxLength : RowKeySchema.ESTIMATED_VARIABLE_LENGTH_SIZE; + estimatedByteSize += SizedUtil.KEY_VALUE_SIZE + estimatedKeySize + byteSize; + } + //} } } boolean isProjectEmptyKeyValue = false; @@ -665,7 +667,7 @@ public class ProjectionCompiler { if (expression.getDataType().isArrayType()) { indexProjectedColumns.add(expression); PColumn col = expression.getColumn(); - KeyValueColumnExpression keyValueColumnExpression = new KeyValueColumnExpression(col, EncodedColumnsUtil.hasEncodedColumnName(col)); + KeyValueColumnExpression keyValueColumnExpression = new KeyValueColumnExpression(col); indexKVs.add(keyValueColumnExpression); copyOfChildren.set(0, keyValueColumnExpression); Integer count = arrayExpressionCounts.get(expression); http://git-wip-us.apache.org/repos/asf/phoenix/blob/a41074a9/phoenix-core/src/main/java/org/apache/phoenix/compile/TraceQueryPlan.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/compile/TraceQueryPlan.java b/phoenix-core/src/main/java/org/apache/phoenix/compile/TraceQueryPlan.java index 10260cb..8fb435d 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/compile/TraceQueryPlan.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/TraceQueryPlan.java @@ -51,6 +51,7 @@ import org.apache.phoenix.parse.TraceStatement; import org.apache.phoenix.query.KeyRange; import org.apache.phoenix.schema.PColumn; import org.apache.phoenix.schema.PColumnImpl; +import org.apache.phoenix.schema.PName; import org.apache.phoenix.schema.PNameFactory; import org.apache.phoenix.schema.RowKeyValueAccessor; import org.apache.phoenix.schema.SortOrder; @@ -75,10 +76,11 @@ public class TraceQueryPlan implements QueryPlan { private static final RowProjector TRACE_PROJECTOR; static { List<ExpressionProjector> projectedColumns = new ArrayList<ExpressionProjector>(); + PName colName = PNameFactory.newName(MetricInfo.TRACE.columnName); PColumn column = new PColumnImpl(PNameFactory.newName(MetricInfo.TRACE.columnName), null, PLong.INSTANCE, null, null, false, 0, SortOrder.getDefault(), 0, null, - false, null, false, false, null); + false, null, false, false, colName.getBytes()); List<PColumn> columns = new ArrayList<PColumn>(); columns.add(column); Expression expression = http://git-wip-us.apache.org/repos/asf/phoenix/blob/a41074a9/phoenix-core/src/main/java/org/apache/phoenix/compile/TupleProjectionCompiler.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/compile/TupleProjectionCompiler.java b/phoenix-core/src/main/java/org/apache/phoenix/compile/TupleProjectionCompiler.java index 6663c93..b0e715c 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/compile/TupleProjectionCompiler.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/TupleProjectionCompiler.java @@ -123,7 +123,7 @@ public class TupleProjectionCompiler { PColumn sourceColumn = table.getPKColumns().get(i); ColumnRef sourceColumnRef = new ColumnRef(tableRef, sourceColumn.getPosition()); PColumn column = new ProjectedColumn(sourceColumn.getName(), sourceColumn.getFamilyName(), - position++, sourceColumn.isNullable(), sourceColumnRef); + position++, sourceColumn.isNullable(), sourceColumnRef, null); projectedColumns.add(column); } for (PColumn sourceColumn : table.getColumns()) { @@ -135,7 +135,7 @@ public class TupleProjectionCompiler { && !families.contains(sourceColumn.getFamilyName().getString())) continue; PColumn column = new ProjectedColumn(sourceColumn.getName(), sourceColumn.getFamilyName(), - position++, sourceColumn.isNullable(), sourceColumnRef); + position++, sourceColumn.isNullable(), sourceColumnRef, sourceColumn.getColumnQualifierBytes()); projectedColumns.add(column); // Wildcard or FamilyWildcard will be handled by ProjectionCompiler. if (!isWildcard && !families.contains(sourceColumn.getFamilyName())) { @@ -146,7 +146,7 @@ public class TupleProjectionCompiler { for (LocalIndexDataColumnRef sourceColumnRef : visitor.localIndexColumnRefSet) { PColumn column = new ProjectedColumn(sourceColumnRef.getColumn().getName(), sourceColumnRef.getColumn().getFamilyName(), position++, - sourceColumnRef.getColumn().isNullable(), sourceColumnRef); + sourceColumnRef.getColumn().isNullable(), sourceColumnRef, sourceColumnRef.getColumn().getColumnQualifierBytes()); projectedColumns.add(column); } @@ -176,7 +176,7 @@ public class TupleProjectionCompiler { PColumn column = new ProjectedColumn(PNameFactory.newName(aliasedName), retainPKColumns && SchemaUtil.isPKColumn(sourceColumn) ? null : PNameFactory.newName(VALUE_COLUMN_FAMILY), - position++, sourceColumn.isNullable(), sourceColumnRef); + position++, sourceColumn.isNullable(), sourceColumnRef, sourceColumn.getColumnQualifierBytes()); projectedColumns.add(column); } EncodedCQCounter cqCounter = EncodedCQCounter.NULL_COUNTER; http://git-wip-us.apache.org/repos/asf/phoenix/blob/a41074a9/phoenix-core/src/main/java/org/apache/phoenix/compile/UnionCompiler.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/compile/UnionCompiler.java b/phoenix-core/src/main/java/org/apache/phoenix/compile/UnionCompiler.java index 013169d..13b58d1 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/compile/UnionCompiler.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/UnionCompiler.java @@ -86,11 +86,12 @@ public class UnionCompiler { for (int i = 0; i < plan.getProjector().getColumnCount(); i++) { ColumnProjector colProj = plan.getProjector().getColumnProjector(i); String name = selectNodes == null ? colProj.getName() : selectNodes.get(i).getAlias(); + PName colName = PNameFactory.newName(name); PColumnImpl projectedColumn = new PColumnImpl(PNameFactory.newName(name), UNION_FAMILY_NAME, targetTypes.get(i).getType(), targetTypes.get(i).getMaxLength(), targetTypes.get(i).getScale(), colProj.getExpression().isNullable(), i, targetTypes.get(i).getSortOrder(), 500, null, false, - colProj.getExpression().toString(), false, false, null); + colProj.getExpression().toString(), false, false, colName.getBytes()); projectedColumns.add(projectedColumn); } Long scn = statement.getConnection().getSCN(); http://git-wip-us.apache.org/repos/asf/phoenix/blob/a41074a9/phoenix-core/src/main/java/org/apache/phoenix/compile/WhereCompiler.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/compile/WhereCompiler.java b/phoenix-core/src/main/java/org/apache/phoenix/compile/WhereCompiler.java index 598b433..a6ff1c5 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/compile/WhereCompiler.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/WhereCompiler.java @@ -175,7 +175,7 @@ public class WhereCompiler { Expression newColumnExpression = ref.newColumnExpression(node.isTableNameCaseSensitive(), node.isCaseSensitive()); if (tableRef.equals(context.getCurrentTable()) && !SchemaUtil.isPKColumn(ref.getColumn())) { byte[] cq = tableRef.getTable().getStorageScheme() == StorageScheme.ONE_CELL_PER_COLUMN_FAMILY - ? QueryConstants.SINGLE_KEYVALUE_COLUMN_QUALIFIER_BYTES : EncodedColumnsUtil.getColumnQualifier(ref.getColumn(), tableRef.getTable()); + ? QueryConstants.SINGLE_KEYVALUE_COLUMN_QUALIFIER_BYTES : ref.getColumn().getColumnQualifierBytes(); // track the where condition columns. Later we need to ensure the Scan in HRS scans these column CFs context.addWhereCoditionColumn(ref.getColumn().getFamilyName().getBytes(), cq); } http://git-wip-us.apache.org/repos/asf/phoenix/blob/a41074a9/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/BaseScannerRegionObserver.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/BaseScannerRegionObserver.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/BaseScannerRegionObserver.java index 9589c37..a64ed64 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/BaseScannerRegionObserver.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/BaseScannerRegionObserver.java @@ -107,6 +107,7 @@ abstract public class BaseScannerRegionObserver extends BaseRegionObserver { public static final String SCAN_STOP_ROW_SUFFIX = "_ScanStopRowSuffix"; public final static String MIN_QUALIFIER = "_MinQualifier"; public final static String MAX_QUALIFIER = "_MaxQualifier"; + public final static String QUALIFIER_ENCODING_SCHEME = "_QualifierEncodingScheme"; /** * Attribute name used to pass custom annotations in Scans and Mutations (later). Custom annotations http://git-wip-us.apache.org/repos/asf/phoenix/blob/a41074a9/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/GroupedAggregateRegionObserver.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/GroupedAggregateRegionObserver.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/GroupedAggregateRegionObserver.java index 793386d..ee79586 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/GroupedAggregateRegionObserver.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/GroupedAggregateRegionObserver.java @@ -24,6 +24,7 @@ import static org.apache.phoenix.query.QueryServices.GROUPBY_ESTIMATED_DISTINCT_ import static org.apache.phoenix.query.QueryServices.GROUPBY_SPILLABLE_ATTRIB; import static org.apache.phoenix.query.QueryServicesOptions.DEFAULT_GROUPBY_ESTIMATED_DISTINCT_VALUES; import static org.apache.phoenix.query.QueryServicesOptions.DEFAULT_GROUPBY_SPILLABLE; +import static org.apache.phoenix.util.EncodedColumnsUtil.getEncodingScheme; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; @@ -383,6 +384,7 @@ public class GroupedAggregateRegionObserver extends BaseScannerRegionObserver { estDistVals = Math.max(MIN_DISTINCT_VALUES, (int) (Bytes.toInt(estDistValsBytes) * 1.5f)); } + Pair<Integer, Integer> minMaxQualifiers = EncodedColumnsUtil.getMinMaxQualifiersFromScan(scan); boolean useQualifierAsIndex = EncodedColumnsUtil.useQualifierAsIndex(EncodedColumnsUtil.getMinMaxQualifiersFromScan(scan)); final boolean spillableEnabled = @@ -406,7 +408,7 @@ public class GroupedAggregateRegionObserver extends BaseScannerRegionObserver { acquiredLock = true; synchronized (scanner) { do { - List<Cell> results = useQualifierAsIndex ? new EncodedColumnQualiferCellsList(minMaxQualifiers.getFirst(), minMaxQualifiers.getSecond()) : new ArrayList<Cell>(); + List<Cell> results = useQualifierAsIndex ? new EncodedColumnQualiferCellsList(minMaxQualifiers.getFirst(), minMaxQualifiers.getSecond(), getEncodingScheme(scan)) : new ArrayList<Cell>(); // Results are potentially returned even when the return // value of s.next is false // since this is an indication of whether or not there are @@ -480,7 +482,7 @@ public class GroupedAggregateRegionObserver extends BaseScannerRegionObserver { acquiredLock = true; synchronized (scanner) { do { - List<Cell> kvs = useQualifierAsIndex ? new EncodedColumnQualiferCellsList(minMaxQualifiers.getFirst(), minMaxQualifiers.getSecond()) : new ArrayList<Cell>(); + List<Cell> kvs = useQualifierAsIndex ? new EncodedColumnQualiferCellsList(minMaxQualifiers.getFirst(), minMaxQualifiers.getSecond(), getEncodingScheme(scan)) : new ArrayList<Cell>(); // Results are potentially returned even when the return // value of s.next is false // since this is an indication of whether or not there http://git-wip-us.apache.org/repos/asf/phoenix/blob/a41074a9/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java index 55ee8f1..b46cb26 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java @@ -27,6 +27,7 @@ import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.CLASS_NAME_BYTES; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.COLUMN_COUNT_BYTES; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.COLUMN_DEF_BYTES; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.COLUMN_NAME_INDEX; +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.COLUMN_QUALIFIER_BYTES; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.COLUMN_QUALIFIER_COUNTER_BYTES; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.COLUMN_SIZE_BYTES; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.DATA_TABLE_NAME_BYTES; @@ -35,7 +36,6 @@ import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.DECIMAL_DIGITS_BYT import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.DEFAULT_COLUMN_FAMILY_NAME_BYTES; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.DEFAULT_VALUE_BYTES; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.DISABLE_WAL_BYTES; -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.ENCODED_COLUMN_QUALIFIER_BYTES; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.ENCODING_SCHEME_BYTES; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.FAMILY_NAME_INDEX; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.IMMUTABLE_ROWS_BYTES; @@ -78,7 +78,6 @@ import static org.apache.phoenix.query.QueryConstants.DIVERGED_VIEW_BASE_COLUMN_ import static org.apache.phoenix.query.QueryConstants.SEPARATOR_BYTE_ARRAY; import static org.apache.phoenix.schema.PTableType.INDEX; import static org.apache.phoenix.util.ByteUtil.EMPTY_BYTE_ARRAY; -import static org.apache.phoenix.util.EncodedColumnsUtil.getEncodedColumnQualifier; import static org.apache.phoenix.util.SchemaUtil.getVarCharLength; import static org.apache.phoenix.util.SchemaUtil.getVarChars; @@ -367,7 +366,7 @@ public class MetaDataEndpointImpl extends MetaDataProtocol implements Coprocesso private static final KeyValue IS_VIEW_REFERENCED_KV = KeyValue.createFirstOnRow(ByteUtil.EMPTY_BYTE_ARRAY, TABLE_FAMILY_BYTES, IS_VIEW_REFERENCED_BYTES); private static final KeyValue COLUMN_DEF_KV = KeyValue.createFirstOnRow(ByteUtil.EMPTY_BYTE_ARRAY, TABLE_FAMILY_BYTES, COLUMN_DEF_BYTES); private static final KeyValue IS_ROW_TIMESTAMP_KV = KeyValue.createFirstOnRow(ByteUtil.EMPTY_BYTE_ARRAY, TABLE_FAMILY_BYTES, IS_ROW_TIMESTAMP_BYTES); - private static final KeyValue ENCODED_COLUMN_QUALIFIER_KV = KeyValue.createFirstOnRow(ByteUtil.EMPTY_BYTE_ARRAY, TABLE_FAMILY_BYTES, ENCODED_COLUMN_QUALIFIER_BYTES); + private static final KeyValue COLUMN_QUALIFIER_KV = KeyValue.createFirstOnRow(ByteUtil.EMPTY_BYTE_ARRAY, TABLE_FAMILY_BYTES, COLUMN_QUALIFIER_BYTES); private static final List<KeyValue> COLUMN_KV_COLUMNS = Arrays.<KeyValue>asList( DECIMAL_DIGITS_KV, COLUMN_SIZE_KV, @@ -381,7 +380,7 @@ public class MetaDataEndpointImpl extends MetaDataProtocol implements Coprocesso IS_VIEW_REFERENCED_KV, COLUMN_DEF_KV, IS_ROW_TIMESTAMP_KV, - ENCODED_COLUMN_QUALIFIER_KV + COLUMN_QUALIFIER_KV ); static { Collections.sort(COLUMN_KV_COLUMNS, KeyValue.COMPARATOR); @@ -398,7 +397,7 @@ public class MetaDataEndpointImpl extends MetaDataProtocol implements Coprocesso private static final int IS_VIEW_REFERENCED_INDEX = COLUMN_KV_COLUMNS.indexOf(IS_VIEW_REFERENCED_KV); private static final int COLUMN_DEF_INDEX = COLUMN_KV_COLUMNS.indexOf(COLUMN_DEF_KV); private static final int IS_ROW_TIMESTAMP_INDEX = COLUMN_KV_COLUMNS.indexOf(IS_ROW_TIMESTAMP_KV); - private static final int ENCODED_COLUMN_QUALIFIER_INDEX = COLUMN_KV_COLUMNS.indexOf(ENCODED_COLUMN_QUALIFIER_KV); + private static final int COLUMN_QUALIFIER_INDEX = COLUMN_KV_COLUMNS.indexOf(COLUMN_QUALIFIER_KV); private static final int LINK_TYPE_INDEX = 0; @@ -737,12 +736,17 @@ public class MetaDataEndpointImpl extends MetaDataProtocol implements Coprocesso isRowTimestampKV == null ? false : Boolean.TRUE.equals(PBoolean.INSTANCE.toObject( isRowTimestampKV.getValueArray(), isRowTimestampKV.getValueOffset(), isRowTimestampKV.getValueLength())); - Cell columnQualifierKV = colKeyValues[ENCODED_COLUMN_QUALIFIER_INDEX]; - Integer columnQualifier = - columnQualifierKV == null ? null : getEncodedColumnQualifier( - columnQualifierKV.getValueArray(), columnQualifierKV.getValueOffset(), - columnQualifierKV.getValueLength()); - PColumn column = new PColumnImpl(colName, famName, dataType, maxLength, scale, isNullable, position-1, sortOrder, arraySize, viewConstant, isViewReferenced, expressionStr, isRowTimestamp, false, columnQualifier); + + //TODO: samarth verify which case is the right one to use. + boolean isPkColumn = famName == null || famName.getString() == null; + Cell columnQualifierKV = colKeyValues[COLUMN_QUALIFIER_INDEX]; + // Older tables won't have column qualifier metadata present. To make things simpler, just set the + // column qualifier bytes by using the column name. + byte[] columnQualifierBytes = columnQualifierKV != null ? + Arrays.copyOfRange(columnQualifierKV.getValueArray(), + columnQualifierKV.getValueOffset(), columnQualifierKV.getValueOffset() + + columnQualifierKV.getValueLength()) : (isPkColumn ? null : colName.getBytes()); + PColumn column = new PColumnImpl(colName, famName, dataType, maxLength, scale, isNullable, position-1, sortOrder, arraySize, viewConstant, isViewReferenced, expressionStr, isRowTimestamp, false, columnQualifierBytes); columns.add(column); } http://git-wip-us.apache.org/repos/asf/phoenix/blob/a41074a9/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/ScanRegionObserver.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/ScanRegionObserver.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/ScanRegionObserver.java index 3a61493..a35d53a 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/ScanRegionObserver.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/ScanRegionObserver.java @@ -17,6 +17,9 @@ */ package org.apache.phoenix.coprocessor; +import static org.apache.phoenix.util.EncodedColumnsUtil.getEncodingScheme; +import static org.apache.phoenix.util.EncodedColumnsUtil.getMinMaxQualifiersFromScan; + import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.DataInputStream; @@ -57,6 +60,7 @@ import org.apache.phoenix.memory.MemoryManager.MemoryChunk; import org.apache.phoenix.query.QueryConstants; import org.apache.phoenix.schema.KeyValueSchema; import org.apache.phoenix.schema.KeyValueSchema.KeyValueSchemaBuilder; +import org.apache.phoenix.schema.PTable.QualifierEncodingScheme; import org.apache.phoenix.schema.ValueBitSet; import org.apache.phoenix.schema.tuple.ResultTuple; import org.apache.phoenix.schema.tuple.Tuple; @@ -126,7 +130,8 @@ public class ScanRegionObserver extends BaseScannerRegionObserver { orderByExpression.readFields(input); orderByExpressions.add(orderByExpression); } - ResultIterator inner = new RegionScannerResultIterator(s, EncodedColumnsUtil.getMinMaxQualifiersFromScan(scan)); + QualifierEncodingScheme encodingScheme = getEncodingScheme(scan); + ResultIterator inner = new RegionScannerResultIterator(s, EncodedColumnsUtil.getMinMaxQualifiersFromScan(scan), encodingScheme); return new OrderedResultIterator(inner, orderByExpressions, thresholdBytes, limit >= 0 ? limit : null, null, estimatedRowSize); } catch (IOException e) { @@ -221,7 +226,7 @@ public class ScanRegionObserver extends BaseScannerRegionObserver { final HashJoinInfo j = HashJoinInfo.deserializeHashJoinFromScan(scan); //TODO: samarth make this a client side check by looking at order by and group by expressions. Then use that to set min max qualifiers. We can then make useQualifierListAsIndex // a member variable of BaseScannerRegionObserver. - boolean useQualifierAsIndex = EncodedColumnsUtil.useQualifierAsIndex(EncodedColumnsUtil.getMinMaxQualifiersFromScan(scan)) && scan.getAttribute(BaseScannerRegionObserver.TOPN) != null; + boolean useQualifierAsIndex = EncodedColumnsUtil.useQualifierAsIndex(getMinMaxQualifiersFromScan(scan)) && scan.getAttribute(BaseScannerRegionObserver.TOPN) != null; innerScanner = getWrappedScanner(c, innerScanner, arrayKVRefs, arrayFuncRefs, offset, scan, dataColumns, tupleProjector, dataRegion, indexMaintainer, tx, @@ -233,7 +238,7 @@ public class ScanRegionObserver extends BaseScannerRegionObserver { } if (scanOffset != null) { innerScanner = getOffsetScanner(c, innerScanner, - new OffsetResultIterator(new RegionScannerResultIterator(innerScanner, EncodedColumnsUtil.getMinMaxQualifiersFromScan(scan)), scanOffset), + new OffsetResultIterator(new RegionScannerResultIterator(innerScanner, getMinMaxQualifiersFromScan(scan), getEncodingScheme(scan)), scanOffset), scan.getAttribute(QueryConstants.LAST_SCAN) != null); } final OrderedResultIterator iterator = deserializeFromScan(scan, innerScanner); http://git-wip-us.apache.org/repos/asf/phoenix/blob/a41074a9/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java index 671b5f8..dadfaeb 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java @@ -24,6 +24,7 @@ import static org.apache.phoenix.query.QueryConstants.UNGROUPED_AGG_ROW_KEY; import static org.apache.phoenix.query.QueryServices.MUTATE_BATCH_SIZE_ATTRIB; import static org.apache.phoenix.schema.stats.StatisticsCollectionRunTracker.COMPACTION_UPDATE_STATS_ROW_COUNT; import static org.apache.phoenix.schema.stats.StatisticsCollectionRunTracker.CONCURRENT_UPDATE_STATS_ROW_COUNT; +import static org.apache.phoenix.util.EncodedColumnsUtil.getEncodingScheme; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; @@ -400,7 +401,7 @@ public class UngroupedAggregateRegionObserver extends BaseScannerRegionObserver acquiredLock = true; synchronized (innerScanner) { do { - List<Cell> results = useQualifierAsIndex ? new EncodedColumnQualiferCellsList(minMaxQualifiers.getFirst(), minMaxQualifiers.getSecond()) : new ArrayList<Cell>(); + List<Cell> results = useQualifierAsIndex ? new EncodedColumnQualiferCellsList(minMaxQualifiers.getFirst(), minMaxQualifiers.getSecond(), getEncodingScheme(scan)) : new ArrayList<Cell>(); // Results are potentially returned even when the return value of s.next is false // since this is an indication of whether or not there are more values after the // ones returned http://git-wip-us.apache.org/repos/asf/phoenix/blob/a41074a9/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/generated/PTableProtos.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/generated/PTableProtos.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/generated/PTableProtos.java index ead928f..b7ca46f 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/generated/PTableProtos.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/generated/PTableProtos.java @@ -270,15 +270,15 @@ public final class PTableProtos { */ boolean getIsDynamic(); - // optional int32 encodedColumnQualifier = 15; + // optional bytes columnQualifierBytes = 15; /** - * <code>optional int32 encodedColumnQualifier = 15;</code> + * <code>optional bytes columnQualifierBytes = 15;</code> */ - boolean hasEncodedColumnQualifier(); + boolean hasColumnQualifierBytes(); /** - * <code>optional int32 encodedColumnQualifier = 15;</code> + * <code>optional bytes columnQualifierBytes = 15;</code> */ - int getEncodedColumnQualifier(); + com.google.protobuf.ByteString getColumnQualifierBytes(); } /** * Protobuf type {@code PColumn} @@ -401,9 +401,9 @@ public final class PTableProtos { isDynamic_ = input.readBool(); break; } - case 120: { + case 122: { bitField0_ |= 0x00004000; - encodedColumnQualifier_ = input.readInt32(); + columnQualifierBytes_ = input.readBytes(); break; } } @@ -724,20 +724,20 @@ public final class PTableProtos { return isDynamic_; } - // optional int32 encodedColumnQualifier = 15; - public static final int ENCODEDCOLUMNQUALIFIER_FIELD_NUMBER = 15; - private int encodedColumnQualifier_; + // optional bytes columnQualifierBytes = 15; + public static final int COLUMNQUALIFIERBYTES_FIELD_NUMBER = 15; + private com.google.protobuf.ByteString columnQualifierBytes_; /** - * <code>optional int32 encodedColumnQualifier = 15;</code> + * <code>optional bytes columnQualifierBytes = 15;</code> */ - public boolean hasEncodedColumnQualifier() { + public boolean hasColumnQualifierBytes() { return ((bitField0_ & 0x00004000) == 0x00004000); } /** - * <code>optional int32 encodedColumnQualifier = 15;</code> + * <code>optional bytes columnQualifierBytes = 15;</code> */ - public int getEncodedColumnQualifier() { - return encodedColumnQualifier_; + public com.google.protobuf.ByteString getColumnQualifierBytes() { + return columnQualifierBytes_; } private void initFields() { @@ -755,7 +755,7 @@ public final class PTableProtos { expression_ = ""; isRowTimestamp_ = false; isDynamic_ = false; - encodedColumnQualifier_ = 0; + columnQualifierBytes_ = com.google.protobuf.ByteString.EMPTY; } private byte memoizedIsInitialized = -1; public final boolean isInitialized() { @@ -832,7 +832,7 @@ public final class PTableProtos { output.writeBool(14, isDynamic_); } if (((bitField0_ & 0x00004000) == 0x00004000)) { - output.writeInt32(15, encodedColumnQualifier_); + output.writeBytes(15, columnQualifierBytes_); } getUnknownFields().writeTo(output); } @@ -901,7 +901,7 @@ public final class PTableProtos { } if (((bitField0_ & 0x00004000) == 0x00004000)) { size += com.google.protobuf.CodedOutputStream - .computeInt32Size(15, encodedColumnQualifier_); + .computeBytesSize(15, columnQualifierBytes_); } size += getUnknownFields().getSerializedSize(); memoizedSerializedSize = size; @@ -996,10 +996,10 @@ public final class PTableProtos { result = result && (getIsDynamic() == other.getIsDynamic()); } - result = result && (hasEncodedColumnQualifier() == other.hasEncodedColumnQualifier()); - if (hasEncodedColumnQualifier()) { - result = result && (getEncodedColumnQualifier() - == other.getEncodedColumnQualifier()); + result = result && (hasColumnQualifierBytes() == other.hasColumnQualifierBytes()); + if (hasColumnQualifierBytes()) { + result = result && getColumnQualifierBytes() + .equals(other.getColumnQualifierBytes()); } result = result && getUnknownFields().equals(other.getUnknownFields()); @@ -1070,9 +1070,9 @@ public final class PTableProtos { hash = (37 * hash) + ISDYNAMIC_FIELD_NUMBER; hash = (53 * hash) + hashBoolean(getIsDynamic()); } - if (hasEncodedColumnQualifier()) { - hash = (37 * hash) + ENCODEDCOLUMNQUALIFIER_FIELD_NUMBER; - hash = (53 * hash) + getEncodedColumnQualifier(); + if (hasColumnQualifierBytes()) { + hash = (37 * hash) + COLUMNQUALIFIERBYTES_FIELD_NUMBER; + hash = (53 * hash) + getColumnQualifierBytes().hashCode(); } hash = (29 * hash) + getUnknownFields().hashCode(); memoizedHashCode = hash; @@ -1211,7 +1211,7 @@ public final class PTableProtos { bitField0_ = (bitField0_ & ~0x00001000); isDynamic_ = false; bitField0_ = (bitField0_ & ~0x00002000); - encodedColumnQualifier_ = 0; + columnQualifierBytes_ = com.google.protobuf.ByteString.EMPTY; bitField0_ = (bitField0_ & ~0x00004000); return this; } @@ -1300,7 +1300,7 @@ public final class PTableProtos { if (((from_bitField0_ & 0x00004000) == 0x00004000)) { to_bitField0_ |= 0x00004000; } - result.encodedColumnQualifier_ = encodedColumnQualifier_; + result.columnQualifierBytes_ = columnQualifierBytes_; result.bitField0_ = to_bitField0_; onBuilt(); return result; @@ -1363,8 +1363,8 @@ public final class PTableProtos { if (other.hasIsDynamic()) { setIsDynamic(other.getIsDynamic()); } - if (other.hasEncodedColumnQualifier()) { - setEncodedColumnQualifier(other.getEncodedColumnQualifier()); + if (other.hasColumnQualifierBytes()) { + setColumnQualifierBytes(other.getColumnQualifierBytes()); } this.mergeUnknownFields(other.getUnknownFields()); return this; @@ -1966,35 +1966,38 @@ public final class PTableProtos { return this; } - // optional int32 encodedColumnQualifier = 15; - private int encodedColumnQualifier_ ; + // optional bytes columnQualifierBytes = 15; + private com.google.protobuf.ByteString columnQualifierBytes_ = com.google.protobuf.ByteString.EMPTY; /** - * <code>optional int32 encodedColumnQualifier = 15;</code> + * <code>optional bytes columnQualifierBytes = 15;</code> */ - public boolean hasEncodedColumnQualifier() { + public boolean hasColumnQualifierBytes() { return ((bitField0_ & 0x00004000) == 0x00004000); } /** - * <code>optional int32 encodedColumnQualifier = 15;</code> + * <code>optional bytes columnQualifierBytes = 15;</code> */ - public int getEncodedColumnQualifier() { - return encodedColumnQualifier_; + public com.google.protobuf.ByteString getColumnQualifierBytes() { + return columnQualifierBytes_; } /** - * <code>optional int32 encodedColumnQualifier = 15;</code> + * <code>optional bytes columnQualifierBytes = 15;</code> */ - public Builder setEncodedColumnQualifier(int value) { - bitField0_ |= 0x00004000; - encodedColumnQualifier_ = value; + public Builder setColumnQualifierBytes(com.google.protobuf.ByteString value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00004000; + columnQualifierBytes_ = value; onChanged(); return this; } /** - * <code>optional int32 encodedColumnQualifier = 15;</code> + * <code>optional bytes columnQualifierBytes = 15;</code> */ - public Builder clearEncodedColumnQualifier() { + public Builder clearColumnQualifierBytes() { bitField0_ = (bitField0_ & ~0x00004000); - encodedColumnQualifier_ = 0; + columnQualifierBytes_ = getDefaultInstance().getColumnQualifierBytes(); onChanged(); return this; } @@ -8482,7 +8485,7 @@ public final class PTableProtos { descriptor; static { java.lang.String[] descriptorData = { - "\n\014PTable.proto\032\021PGuidePosts.proto\"\306\002\n\007PC" + + "\n\014PTable.proto\032\021PGuidePosts.proto\"\304\002\n\007PC" + "olumn\022\027\n\017columnNameBytes\030\001 \002(\014\022\027\n\017family" + "NameBytes\030\002 \001(\014\022\020\n\010dataType\030\003 \002(\t\022\021\n\tmax" + "Length\030\004 \001(\005\022\r\n\005scale\030\005 \001(\005\022\020\n\010nullable\030" + @@ -8490,38 +8493,38 @@ public final class PTableProtos { "(\005\022\021\n\tarraySize\030\t \001(\005\022\024\n\014viewConstant\030\n " + "\001(\014\022\026\n\016viewReferenced\030\013 \001(\010\022\022\n\nexpressio" + "n\030\014 \001(\t\022\026\n\016isRowTimestamp\030\r \001(\010\022\021\n\tisDyn" + - "amic\030\016 \001(\010\022\036\n\026encodedColumnQualifier\030\017 \001" + - "(\005\"\232\001\n\013PTableStats\022\013\n\003key\030\001 \002(\014\022\016\n\006value", - "s\030\002 \003(\014\022\033\n\023guidePostsByteCount\030\003 \001(\003\022\025\n\r" + - "keyBytesCount\030\004 \001(\003\022\027\n\017guidePostsCount\030\005" + - " \001(\005\022!\n\013pGuidePosts\030\006 \001(\0132\014.PGuidePosts\"" + - "\354\006\n\006PTable\022\027\n\017schemaNameBytes\030\001 \002(\014\022\026\n\016t" + - "ableNameBytes\030\002 \002(\014\022\036\n\ttableType\030\003 \002(\0162\013" + - ".PTableType\022\022\n\nindexState\030\004 \001(\t\022\026\n\016seque" + - "nceNumber\030\005 \002(\003\022\021\n\ttimeStamp\030\006 \002(\003\022\023\n\013pk" + - "NameBytes\030\007 \001(\014\022\021\n\tbucketNum\030\010 \002(\005\022\031\n\007co" + - "lumns\030\t \003(\0132\010.PColumn\022\030\n\007indexes\030\n \003(\0132\007" + - ".PTable\022\027\n\017isImmutableRows\030\013 \002(\010\022\032\n\022data", - "TableNameBytes\030\r \001(\014\022\031\n\021defaultFamilyNam" + - "e\030\016 \001(\014\022\022\n\ndisableWAL\030\017 \002(\010\022\023\n\013multiTena" + - "nt\030\020 \002(\010\022\020\n\010viewType\030\021 \001(\014\022\025\n\rviewStatem" + - "ent\030\022 \001(\014\022\025\n\rphysicalNames\030\023 \003(\014\022\020\n\010tena" + - "ntId\030\024 \001(\014\022\023\n\013viewIndexId\030\025 \001(\005\022\021\n\tindex" + - "Type\030\026 \001(\014\022\026\n\016statsTimeStamp\030\027 \001(\003\022\022\n\nst" + - "oreNulls\030\030 \001(\010\022\027\n\017baseColumnCount\030\031 \001(\005\022" + - "\036\n\026rowKeyOrderOptimizable\030\032 \001(\010\022\025\n\rtrans" + - "actional\030\033 \001(\010\022\034\n\024updateCacheFrequency\030\034" + - " \001(\003\022\035\n\025indexDisableTimestamp\030\035 \001(\003\022\031\n\021i", - "sNamespaceMapped\030\036 \001(\010\022\034\n\024autoParititonS" + - "eqName\030\037 \001(\t\022\032\n\022isAppendOnlySchema\030 \001(\010" + - "\022\027\n\017parentNameBytes\030! \001(\014\022\025\n\rstorageSche" + - "me\030\" \001(\014\022\026\n\016encodingScheme\030# \001(\014\022,\n\021enco" + - "dedCQCounters\030$ \003(\0132\021.EncodedCQCounter\"6" + - "\n\020EncodedCQCounter\022\021\n\tcolFamily\030\001 \002(\t\022\017\n" + - "\007counter\030\002 \002(\005*A\n\nPTableType\022\n\n\006SYSTEM\020\000" + - "\022\010\n\004USER\020\001\022\010\n\004VIEW\020\002\022\t\n\005INDEX\020\003\022\010\n\004JOIN\020" + - "\004B@\n(org.apache.phoenix.coprocessor.gene" + - "ratedB\014PTableProtosH\001\210\001\001\240\001\001" + "amic\030\016 \001(\010\022\034\n\024columnQualifierBytes\030\017 \001(\014" + + "\"\232\001\n\013PTableStats\022\013\n\003key\030\001 \002(\014\022\016\n\006values\030", + "\002 \003(\014\022\033\n\023guidePostsByteCount\030\003 \001(\003\022\025\n\rke" + + "yBytesCount\030\004 \001(\003\022\027\n\017guidePostsCount\030\005 \001" + + "(\005\022!\n\013pGuidePosts\030\006 \001(\0132\014.PGuidePosts\"\354\006" + + "\n\006PTable\022\027\n\017schemaNameBytes\030\001 \002(\014\022\026\n\016tab" + + "leNameBytes\030\002 \002(\014\022\036\n\ttableType\030\003 \002(\0162\013.P" + + "TableType\022\022\n\nindexState\030\004 \001(\t\022\026\n\016sequenc" + + "eNumber\030\005 \002(\003\022\021\n\ttimeStamp\030\006 \002(\003\022\023\n\013pkNa" + + "meBytes\030\007 \001(\014\022\021\n\tbucketNum\030\010 \002(\005\022\031\n\007colu" + + "mns\030\t \003(\0132\010.PColumn\022\030\n\007indexes\030\n \003(\0132\007.P" + + "Table\022\027\n\017isImmutableRows\030\013 \002(\010\022\032\n\022dataTa", + "bleNameBytes\030\r \001(\014\022\031\n\021defaultFamilyName\030" + + "\016 \001(\014\022\022\n\ndisableWAL\030\017 \002(\010\022\023\n\013multiTenant" + + "\030\020 \002(\010\022\020\n\010viewType\030\021 \001(\014\022\025\n\rviewStatemen" + + "t\030\022 \001(\014\022\025\n\rphysicalNames\030\023 \003(\014\022\020\n\010tenant" + + "Id\030\024 \001(\014\022\023\n\013viewIndexId\030\025 \001(\005\022\021\n\tindexTy" + + "pe\030\026 \001(\014\022\026\n\016statsTimeStamp\030\027 \001(\003\022\022\n\nstor" + + "eNulls\030\030 \001(\010\022\027\n\017baseColumnCount\030\031 \001(\005\022\036\n" + + "\026rowKeyOrderOptimizable\030\032 \001(\010\022\025\n\rtransac" + + "tional\030\033 \001(\010\022\034\n\024updateCacheFrequency\030\034 \001" + + "(\003\022\035\n\025indexDisableTimestamp\030\035 \001(\003\022\031\n\021isN", + "amespaceMapped\030\036 \001(\010\022\034\n\024autoParititonSeq" + + "Name\030\037 \001(\t\022\032\n\022isAppendOnlySchema\030 \001(\010\022\027" + + "\n\017parentNameBytes\030! \001(\014\022\025\n\rstorageScheme" + + "\030\" \001(\014\022\026\n\016encodingScheme\030# \001(\014\022,\n\021encode" + + "dCQCounters\030$ \003(\0132\021.EncodedCQCounter\"6\n\020" + + "EncodedCQCounter\022\021\n\tcolFamily\030\001 \002(\t\022\017\n\007c" + + "ounter\030\002 \002(\005*A\n\nPTableType\022\n\n\006SYSTEM\020\000\022\010" + + "\n\004USER\020\001\022\010\n\004VIEW\020\002\022\t\n\005INDEX\020\003\022\010\n\004JOIN\020\004B" + + "@\n(org.apache.phoenix.coprocessor.genera" + + "tedB\014PTableProtosH\001\210\001\001\240\001\001" }; com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner assigner = new com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner() { @@ -8533,7 +8536,7 @@ public final class PTableProtos { internal_static_PColumn_fieldAccessorTable = new com.google.protobuf.GeneratedMessage.FieldAccessorTable( internal_static_PColumn_descriptor, - new java.lang.String[] { "ColumnNameBytes", "FamilyNameBytes", "DataType", "MaxLength", "Scale", "Nullable", "Position", "SortOrder", "ArraySize", "ViewConstant", "ViewReferenced", "Expression", "IsRowTimestamp", "IsDynamic", "EncodedColumnQualifier", }); + new java.lang.String[] { "ColumnNameBytes", "FamilyNameBytes", "DataType", "MaxLength", "Scale", "Nullable", "Position", "SortOrder", "ArraySize", "ViewConstant", "ViewReferenced", "Expression", "IsRowTimestamp", "IsDynamic", "ColumnQualifierBytes", }); internal_static_PTableStats_descriptor = getDescriptor().getMessageTypes().get(1); internal_static_PTableStats_fieldAccessorTable = new http://git-wip-us.apache.org/repos/asf/phoenix/blob/a41074a9/phoenix-core/src/main/java/org/apache/phoenix/execute/BaseQueryPlan.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/execute/BaseQueryPlan.java b/phoenix-core/src/main/java/org/apache/phoenix/execute/BaseQueryPlan.java index cbd9bd3..6c7d05b 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/execute/BaseQueryPlan.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/execute/BaseQueryPlan.java @@ -439,7 +439,7 @@ public abstract class BaseQueryPlan implements QueryPlan { WritableUtils.writeVInt(output, dataColumns.size()); for (PColumn column : dataColumns) { byte[] cf = column.getFamilyName().getBytes(); - byte[] cq = EncodedColumnsUtil.getColumnQualifier(column, dataTable); + byte[] cq = column.getColumnQualifierBytes(); Bytes.writeByteArray(output, cf); Bytes.writeByteArray(output, cq); } http://git-wip-us.apache.org/repos/asf/phoenix/blob/a41074a9/phoenix-core/src/main/java/org/apache/phoenix/expression/ArrayColumnExpression.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/expression/ArrayColumnExpression.java b/phoenix-core/src/main/java/org/apache/phoenix/expression/ArrayColumnExpression.java index 747d7e6..b693ded 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/expression/ArrayColumnExpression.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/expression/ArrayColumnExpression.java @@ -17,6 +17,9 @@ */ package org.apache.phoenix.expression; +import static org.apache.phoenix.query.QueryConstants.SINGLE_KEYVALUE_COLUMN_QUALIFIER_BYTES; +import static org.apache.phoenix.schema.PTable.QualifierEncodingScheme.NON_ENCODED_QUALIFIERS; + import java.io.DataInput; import java.io.DataOutput; import java.io.IOException; @@ -25,9 +28,9 @@ import org.apache.hadoop.hbase.io.ImmutableBytesWritable; import org.apache.hadoop.io.WritableUtils; import org.apache.phoenix.compile.CreateTableCompiler.ViewWhereExpressionVisitor; import org.apache.phoenix.expression.visitor.ExpressionVisitor; -import org.apache.phoenix.query.QueryConstants; import org.apache.phoenix.schema.PColumn; import org.apache.phoenix.schema.PDatum; +import org.apache.phoenix.schema.PTable.QualifierEncodingScheme; import org.apache.phoenix.schema.SortOrder; import org.apache.phoenix.schema.tuple.Tuple; import org.apache.phoenix.schema.types.PArrayDataType; @@ -36,6 +39,8 @@ import org.apache.phoenix.schema.types.PVarbinary; import org.apache.phoenix.util.EncodedColumnsUtil; import org.apache.phoenix.util.SchemaUtil; +import com.google.common.base.Preconditions; + /** * * Class to access a column that is stored in a KeyValue that contains all @@ -47,20 +52,27 @@ public class ArrayColumnExpression extends KeyValueColumnExpression { private int positionInArray; private String arrayColDisplayName; private KeyValueColumnExpression keyValueColumnExpression; + private QualifierEncodingScheme encodingScheme; public ArrayColumnExpression() { } - public ArrayColumnExpression(PDatum column, byte[] cf, int encodedCQ) { - super(column, cf, QueryConstants.SINGLE_KEYVALUE_COLUMN_QUALIFIER_BYTES); - this.positionInArray = encodedCQ; + public ArrayColumnExpression(PDatum column, byte[] cf, byte[] cq, QualifierEncodingScheme encodingScheme) { + super(column, cf, SINGLE_KEYVALUE_COLUMN_QUALIFIER_BYTES); + Preconditions.checkNotNull(encodingScheme); + Preconditions.checkArgument(encodingScheme != NON_ENCODED_QUALIFIERS); + this.positionInArray = encodingScheme.getDecodedValue(cq); + this.encodingScheme = encodingScheme; setKeyValueExpression(); } - public ArrayColumnExpression(PColumn column, String displayName, boolean encodedColumnName) { - super(column, column.getFamilyName().getBytes(), QueryConstants.SINGLE_KEYVALUE_COLUMN_QUALIFIER_BYTES); + public ArrayColumnExpression(PColumn column, String displayName, QualifierEncodingScheme encodingScheme) { + super(column, column.getFamilyName().getBytes(), SINGLE_KEYVALUE_COLUMN_QUALIFIER_BYTES); + Preconditions.checkNotNull(encodingScheme); + Preconditions.checkArgument(encodingScheme != NON_ENCODED_QUALIFIERS); this.arrayColDisplayName = displayName; - this.positionInArray = column.getEncodedColumnQualifier(); + this.positionInArray = encodingScheme.getDecodedValue(column.getColumnQualifierBytes()); + this.encodingScheme = encodingScheme; setKeyValueExpression(); } @@ -81,6 +93,7 @@ public class ArrayColumnExpression extends KeyValueColumnExpression { public void readFields(DataInput input) throws IOException { super.readFields(input); this.positionInArray = WritableUtils.readVInt(input); + this.encodingScheme = WritableUtils.readEnum(input, QualifierEncodingScheme.class); setKeyValueExpression(); } @@ -88,6 +101,7 @@ public class ArrayColumnExpression extends KeyValueColumnExpression { public void write(DataOutput output) throws IOException { super.write(output); WritableUtils.writeVInt(output, positionInArray); + WritableUtils.writeEnum(output, encodingScheme); } public KeyValueColumnExpression getKeyValueExpression() { @@ -137,7 +151,7 @@ public class ArrayColumnExpression extends KeyValueColumnExpression { } public byte[] getPositionInArray() { - return EncodedColumnsUtil.getEncodedColumnQualifier(positionInArray); + return encodingScheme.getEncodedBytes(positionInArray); } @Override http://git-wip-us.apache.org/repos/asf/phoenix/blob/a41074a9/phoenix-core/src/main/java/org/apache/phoenix/expression/KeyValueColumnExpression.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/expression/KeyValueColumnExpression.java b/phoenix-core/src/main/java/org/apache/phoenix/expression/KeyValueColumnExpression.java index 5cd7433..5c0be57 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/expression/KeyValueColumnExpression.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/expression/KeyValueColumnExpression.java @@ -28,7 +28,6 @@ import org.apache.phoenix.expression.visitor.ExpressionVisitor; import org.apache.phoenix.schema.PColumn; import org.apache.phoenix.schema.PDatum; import org.apache.phoenix.schema.tuple.Tuple; -import org.apache.phoenix.util.EncodedColumnsUtil; import org.apache.phoenix.util.SchemaUtil; @@ -46,9 +45,21 @@ public class KeyValueColumnExpression extends ColumnExpression { public KeyValueColumnExpression() { } - - public KeyValueColumnExpression(PColumn column, boolean encodedColumnName) { - this(column, null, encodedColumnName); + + public KeyValueColumnExpression(PColumn column) { + super(column); + this.cf = column.getFamilyName().getBytes(); + // for backward compatibility since older tables won't have columnQualifierBytes in their metadata + this.cq = column.getColumnQualifierBytes() != null ? column.getColumnQualifierBytes() : column.getName().getBytes(); + this.displayName = column.getName().getString(); + } + + public KeyValueColumnExpression(PColumn column, String displayName) { + super(column); + this.cf = column.getFamilyName().getBytes(); + // for backward compatibility since older tables won't have columnQualifierBytes in their metadata + this.cq = column.getColumnQualifierBytes() != null ? column.getColumnQualifierBytes() : column.getName().getBytes(); + this.displayName = displayName; } public KeyValueColumnExpression(PDatum column, byte[] cf, byte[] cq) { @@ -57,13 +68,6 @@ public class KeyValueColumnExpression extends ColumnExpression { this.cq = cq; } - public KeyValueColumnExpression(PColumn column, String displayName, boolean encodedColumnName) { - super(column); - this.cf = column.getFamilyName().getBytes(); - this.cq = EncodedColumnsUtil.getColumnQualifier(column, encodedColumnName); - this.displayName = displayName; - } - public byte[] getColumnFamily() { return cf; } @@ -124,4 +128,8 @@ public class KeyValueColumnExpression extends ColumnExpression { public <T> T accept(ExpressionVisitor<T> visitor) { return visitor.visit(this); } + +// public void setDisplayName(String displayName) { +// this.displayName = displayName; +// } } http://git-wip-us.apache.org/repos/asf/phoenix/blob/a41074a9/phoenix-core/src/main/java/org/apache/phoenix/index/IndexMaintainer.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/index/IndexMaintainer.java b/phoenix-core/src/main/java/org/apache/phoenix/index/IndexMaintainer.java index 2ab3730..f19af6c 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/index/IndexMaintainer.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/index/IndexMaintainer.java @@ -17,8 +17,6 @@ */ package org.apache.phoenix.index; -import static org.apache.phoenix.util.EncodedColumnsUtil.getEncodedColumnQualifier; - import java.io.ByteArrayInputStream; import java.io.DataInput; import java.io.DataInputStream; @@ -83,6 +81,7 @@ import org.apache.phoenix.schema.PDatum; import org.apache.phoenix.schema.PIndexState; import org.apache.phoenix.schema.PTable; import org.apache.phoenix.schema.PTable.IndexType; +import org.apache.phoenix.schema.PTable.QualifierEncodingScheme; import org.apache.phoenix.schema.PTable.StorageScheme; import org.apache.phoenix.schema.PTableType; import org.apache.phoenix.schema.RowKeySchema; @@ -322,8 +321,8 @@ public class IndexMaintainer implements Writable, Iterable<ColumnReference> { private int maxTrailingNulls; private ColumnReference dataEmptyKeyValueRef; private boolean rowKeyOrderOptimizable; - private boolean usesEncodedColumnNames; private ImmutableBytesPtr emptyKeyValueQualifierPtr; + private QualifierEncodingScheme encodingScheme; private IndexMaintainer(RowKeySchema dataRowKeySchema, boolean isDataTableSalted) { this.dataRowKeySchema = dataRowKeySchema; @@ -337,7 +336,7 @@ public class IndexMaintainer implements Writable, Iterable<ColumnReference> { this.isMultiTenant = dataTable.isMultiTenant(); this.viewIndexId = index.getViewIndexId() == null ? null : MetaDataUtil.getViewIndexIdDataType().toBytes(index.getViewIndexId()); this.isLocalIndex = index.getIndexType() == IndexType.LOCAL; - this.usesEncodedColumnNames = EncodedColumnsUtil.usesEncodedColumnNames(index); + this.encodingScheme = index.getEncodingScheme(); byte[] indexTableName = index.getPhysicalName().getBytes(); // Use this for the nDataSaltBuckets as we need this for local indexes // TODO: persist nDataSaltBuckets separately, but maintain b/w compat. @@ -508,8 +507,8 @@ public class IndexMaintainer implements Writable, Iterable<ColumnReference> { PColumnFamily family = index.getColumnFamilies().get(i); for (PColumn indexColumn : family.getColumns()) { PColumn dataColumn = IndexUtil.getDataColumn(dataTable, indexColumn.getName().getString()); - byte[] dataColumnCq = EncodedColumnsUtil.getColumnQualifier(dataColumn, dataTable); - byte[] indexColumnCq = EncodedColumnsUtil.getColumnQualifier(indexColumn, index); + byte[] dataColumnCq = dataColumn.getColumnQualifierBytes(); + byte[] indexColumnCq = indexColumn.getColumnQualifierBytes(); this.coveredColumns.add(new ColumnReference(dataColumn.getFamilyName().getBytes(), dataColumnCq)); this.coveredColumnsMap.put(new ColumnReference(dataColumn.getFamilyName().getBytes(), dataColumnCq), new ColumnReference(indexColumn.getFamilyName().getBytes(), indexColumnCq)); @@ -953,7 +952,7 @@ public class IndexMaintainer implements Writable, Iterable<ColumnReference> { int maxIndex = Integer.MIN_VALUE; // find the max col qualifier for (Pair<ColumnReference, ColumnReference> colRefPair : colRefPairs) { - int qualifier = getEncodedColumnQualifier(colRefPair.getFirst().getQualifier()); + int qualifier = encodingScheme.getDecodedValue(colRefPair.getFirst().getQualifier()); maxIndex = Math.max(maxIndex, qualifier); } byte[][] colValues = new byte[maxIndex+1][]; @@ -961,7 +960,6 @@ public class IndexMaintainer implements Writable, Iterable<ColumnReference> { for (Pair<ColumnReference, ColumnReference> colRefPair : colRefPairs) { ColumnReference indexColRef = colRefPair.getFirst(); ColumnReference dataColRef = colRefPair.getSecond(); - int dataArrayPos = getEncodedColumnQualifier(dataColRef.getQualifier()); Expression expression = new ArrayColumnExpression(new PDatum() { @Override public boolean isNullable() { @@ -987,12 +985,12 @@ public class IndexMaintainer implements Writable, Iterable<ColumnReference> { public PDataType getDataType() { return null; } - }, dataColRef.getFamily(), dataArrayPos); + }, dataColRef.getFamily(), dataColRef.getQualifier(), encodingScheme); ImmutableBytesPtr ptr = new ImmutableBytesPtr(); expression.evaluate(new ValueGetterTuple(valueGetter), ptr); byte[] value = ptr.copyBytesIfNecessary(); if (value != null) { - int indexArrayPos = getEncodedColumnQualifier(indexColRef.getQualifier()); + int indexArrayPos = encodingScheme.getDecodedValue(indexColRef.getQualifier()); colValues[indexArrayPos] = value; } } @@ -1241,8 +1239,6 @@ public class IndexMaintainer implements Writable, Iterable<ColumnReference> { if (isNewClient) { int numIndexedExpressions = WritableUtils.readVInt(input); - usesEncodedColumnNames = numIndexedExpressions > 0; - numIndexedExpressions = Math.abs(numIndexedExpressions) - 1; indexedExpressions = Lists.newArrayListWithExpectedSize(numIndexedExpressions); for (int i = 0; i < numIndexedExpressions; i++) { Expression expression = ExpressionType.values()[WritableUtils.readVInt(input)].newInstance(); @@ -1311,6 +1307,7 @@ public class IndexMaintainer implements Writable, Iterable<ColumnReference> { byte[] dataTableCq = Bytes.readByteArray(input); coveredColumnsInfo.add(new Pair<>(Bytes.toString(dataTableCf), Bytes.toString(dataTableCq))); } + encodingScheme = WritableUtils.readEnum(input, QualifierEncodingScheme.class); storeColsInSingleCell = WritableUtils.readVInt(input) > 0; initCachedState(); } @@ -1351,10 +1348,7 @@ public class IndexMaintainer implements Writable, Iterable<ColumnReference> { // when indexedColumnTypes is removed, remove this WritableUtils.writeVInt(output,-emptyKeyValueCFPtr.getLength()); output.write(emptyKeyValueCFPtr.get(),emptyKeyValueCFPtr.getOffset(), emptyKeyValueCFPtr.getLength()); - - // Hack to encode usesEncodedColumnNames in indexedExpressions size. - int indexedExpressionsSize = (indexedExpressions.size() + 1) * (usesEncodedColumnNames ? 1 : -1); - WritableUtils.writeVInt(output, indexedExpressionsSize); + WritableUtils.writeVInt(output, indexedExpressions.size()); for (Expression expression : indexedExpressions) { WritableUtils.writeVInt(output, ExpressionType.valueOf(expression).ordinal()); expression.write(output); @@ -1375,6 +1369,7 @@ public class IndexMaintainer implements Writable, Iterable<ColumnReference> { Bytes.writeByteArray(output, colInfo.getFirst() == null ? null : colInfo.getFirst().getBytes()); Bytes.writeByteArray(output, colInfo.getSecond().getBytes()); } + WritableUtils.writeEnum(output, encodingScheme); WritableUtils.writeVInt(output, storeColsInSingleCell ? 1 : -1); } @@ -1422,7 +1417,7 @@ public class IndexMaintainer implements Writable, Iterable<ColumnReference> { * Init calculated state reading/creating */ private void initCachedState() { - byte[] emptyKvQualifier = EncodedColumnsUtil.getEmptyKeyValueInfo(usesEncodedColumnNames).getFirst(); + byte[] emptyKvQualifier = EncodedColumnsUtil.getEmptyKeyValueInfo(encodingScheme).getFirst(); dataEmptyKeyValueRef = new ColumnReference(emptyKeyValueCFPtr.copyBytesIfNecessary(), emptyKvQualifier); emptyKeyValueQualifierPtr = new ImmutableBytesPtr(emptyKvQualifier); this.allColumns = Sets.newLinkedHashSetWithExpectedSize(indexedExpressions.size() + coveredColumns.size());
