Repository: phoenix Updated Branches: refs/heads/encodecolumns d69e80a4a -> 3909c633c
Fix generation of next column qualifiers Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/3909c633 Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/3909c633 Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/3909c633 Branch: refs/heads/encodecolumns Commit: 3909c633c3b2cd1b4b3f91c449297e7399794e2c Parents: d69e80a Author: Samarth <samarth.j...@salesforce.com> Authored: Fri Mar 25 10:05:33 2016 -0700 Committer: Samarth <samarth.j...@salesforce.com> Committed: Fri Mar 25 10:05:33 2016 -0700 ---------------------------------------------------------------------- .../phoenix/end2end/index/IndexTestUtil.java | 3 +- .../phoenix/compile/ExpressionCompiler.java | 3 +- .../apache/phoenix/compile/FromCompiler.java | 4 +- .../apache/phoenix/compile/JoinCompiler.java | 3 +- .../apache/phoenix/compile/PostDDLCompiler.java | 6 +- .../phoenix/compile/ProjectionCompiler.java | 3 +- .../compile/TupleProjectionCompiler.java | 3 +- .../apache/phoenix/compile/WhereCompiler.java | 3 +- .../coprocessor/MetaDataEndpointImpl.java | 5 +- .../apache/phoenix/execute/BaseQueryPlan.java | 4 +- .../expression/KeyValueColumnExpression.java | 3 +- .../phoenix/filter/ColumnProjectionFilter.java | 6 +- .../apache/phoenix/index/IndexMaintainer.java | 9 ++- .../phoenix/iterate/BaseResultIterators.java | 5 +- .../mapreduce/FormatToBytesWritableMapper.java | 4 +- .../mapreduce/FormatToKeyValueReducer.java | 3 +- .../org/apache/phoenix/schema/ColumnRef.java | 3 +- .../apache/phoenix/schema/DelegateColumn.java | 4 +- .../apache/phoenix/schema/MetaDataClient.java | 13 ++-- .../java/org/apache/phoenix/schema/PColumn.java | 2 +- .../phoenix/schema/PColumnFamilyImpl.java | 21 +++--- .../org/apache/phoenix/schema/PColumnImpl.java | 8 +- .../apache/phoenix/schema/PMetaDataImpl.java | 2 +- .../org/apache/phoenix/schema/PTableImpl.java | 40 ++++------ .../apache/phoenix/util/EncodedColumnsUtil.java | 77 ++++++++++++++++++++ .../java/org/apache/phoenix/util/IndexUtil.java | 2 +- .../org/apache/phoenix/util/SchemaUtil.java | 65 ++++------------- .../iterate/AggregateResultScannerTest.java | 2 +- 28 files changed, 176 insertions(+), 130 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/phoenix/blob/3909c633/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexTestUtil.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexTestUtil.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexTestUtil.java index 42c0264..a2888cf 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexTestUtil.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexTestUtil.java @@ -47,6 +47,7 @@ import org.apache.phoenix.schema.PTable; import org.apache.phoenix.schema.RowKeySchema; import org.apache.phoenix.schema.SortOrder; import org.apache.phoenix.schema.types.PDataType; +import org.apache.phoenix.util.EncodedColumnsUtil; import org.apache.phoenix.util.IndexUtil; import org.apache.phoenix.util.MetaDataUtil; import org.apache.phoenix.util.SchemaUtil; @@ -134,7 +135,7 @@ public class IndexTestUtil { for (Cell kv : entry.getValue()) { @SuppressWarnings("deprecation") byte[] cq = kv.getQualifier(); - byte[] emptyKVQualifier = SchemaUtil.getEmptyKeyValueInfo(dataTable).getFirst(); + byte[] emptyKVQualifier = EncodedColumnsUtil.getEmptyKeyValueInfo(dataTable).getFirst(); if (Bytes.compareTo(emptyKVQualifier, cq) != 0) { try { PColumn dataColumn = family.getPColumnForColumnQualifier(cq); http://git-wip-us.apache.org/repos/asf/phoenix/blob/3909c633/phoenix-core/src/main/java/org/apache/phoenix/compile/ExpressionCompiler.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/compile/ExpressionCompiler.java b/phoenix-core/src/main/java/org/apache/phoenix/compile/ExpressionCompiler.java index 31a2f82..1623cab 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/compile/ExpressionCompiler.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/ExpressionCompiler.java @@ -133,6 +133,7 @@ import org.apache.phoenix.schema.types.PTimestamp; import org.apache.phoenix.schema.types.PUnsignedTimestamp; import org.apache.phoenix.schema.types.PVarbinary; import org.apache.phoenix.schema.types.PhoenixArray; +import org.apache.phoenix.util.EncodedColumnsUtil; import org.apache.phoenix.util.ExpressionUtil; import org.apache.phoenix.util.IndexUtil; import org.apache.phoenix.util.SchemaUtil; @@ -407,7 +408,7 @@ public class ExpressionCompiler extends UnsupportedAllParseNodeVisitor<Expressio return LiteralExpression.newConstant(column.getDataType().toObject(ptr), column.getDataType()); } if (tableRef.equals(context.getCurrentTable()) && !SchemaUtil.isPKColumn(column)) { // project only kv columns - context.getScan().addColumn(column.getFamilyName().getBytes(), SchemaUtil.getColumnQualifier(column, tableRef.getTable())); + context.getScan().addColumn(column.getFamilyName().getBytes(), EncodedColumnsUtil.getColumnQualifier(column, tableRef.getTable())); } Expression expression = ref.newColumnExpression(node.isTableNameCaseSensitive(), node.isCaseSensitive()); Expression wrappedExpression = wrapGroupByExpression(expression); http://git-wip-us.apache.org/repos/asf/phoenix/blob/3909c633/phoenix-core/src/main/java/org/apache/phoenix/compile/FromCompiler.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/compile/FromCompiler.java b/phoenix-core/src/main/java/org/apache/phoenix/compile/FromCompiler.java index 8462812..0ce8a29 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/compile/FromCompiler.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/FromCompiler.java @@ -223,7 +223,7 @@ public class FromCompiler { Expression sourceExpression = projector.getColumnProjector(column.getPosition()).getExpression(); PColumnImpl projectedColumn = new PColumnImpl(column.getName(), column.getFamilyName(), sourceExpression.getDataType(), sourceExpression.getMaxLength(), sourceExpression.getScale(), sourceExpression.isNullable(), - column.getPosition(), sourceExpression.getSortOrder(), column.getArraySize(), column.getViewConstant(), column.isViewReferenced(), column.getExpressionStr(), column.isRowTimestamp(), column.isDynamic(), column.getColumnQualifier()); + column.getPosition(), sourceExpression.getSortOrder(), column.getArraySize(), column.getViewConstant(), column.isViewReferenced(), column.getExpressionStr(), column.isRowTimestamp(), column.isDynamic(), column.getEncodedColumnQualifier()); projectedColumns.add(projectedColumn); } PTable t = PTableImpl.makePTable(table, projectedColumns); @@ -548,7 +548,7 @@ public class FromCompiler { theTable.getColumnFamily(family); // Verifies that column family exists familyName = PNameFactory.newName(family); } - // Dynamic columns don't have a corresponding column qualifier + // Dynamic columns don't have an encoded column name allcolumns.add(new PColumnImpl(name, familyName, dynColumn.getDataType(), dynColumn.getMaxLength(), dynColumn.getScale(), dynColumn.isNull(), position, dynColumn.getSortOrder(), dynColumn.getArraySize(), null, false, dynColumn.getExpression(), false, true, null)); position++; http://git-wip-us.apache.org/repos/asf/phoenix/blob/3909c633/phoenix-core/src/main/java/org/apache/phoenix/compile/JoinCompiler.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/compile/JoinCompiler.java b/phoenix-core/src/main/java/org/apache/phoenix/compile/JoinCompiler.java index 5f4adf6..8f3febc 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/compile/JoinCompiler.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/JoinCompiler.java @@ -91,6 +91,7 @@ import org.apache.phoenix.schema.types.PTimestamp; import org.apache.phoenix.schema.types.PTinyint; import org.apache.phoenix.schema.types.PVarbinary; import org.apache.phoenix.schema.types.PVarchar; +import org.apache.phoenix.util.EncodedColumnsUtil; import org.apache.phoenix.util.IndexUtil; import org.apache.phoenix.util.SchemaUtil; @@ -715,7 +716,7 @@ public class JoinCompiler { if (columnRef.getTableRef().equals(tableRef) && !SchemaUtil.isPKColumn(columnRef.getColumn()) && !(columnRef instanceof LocalIndexColumnRef)) { - scan.addColumn(columnRef.getColumn().getFamilyName().getBytes(), SchemaUtil.getColumnQualifier(columnRef.getColumn(), tableRef.getTable())); + scan.addColumn(columnRef.getColumn().getFamilyName().getBytes(), EncodedColumnsUtil.getColumnQualifier(columnRef.getColumn(), tableRef.getTable())); } } } http://git-wip-us.apache.org/repos/asf/phoenix/blob/3909c633/phoenix-core/src/main/java/org/apache/phoenix/compile/PostDDLCompiler.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/compile/PostDDLCompiler.java b/phoenix-core/src/main/java/org/apache/phoenix/compile/PostDDLCompiler.java index 10087af..35505ad 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/compile/PostDDLCompiler.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/PostDDLCompiler.java @@ -48,8 +48,8 @@ import org.apache.phoenix.schema.TableNotFoundException; import org.apache.phoenix.schema.TableRef; import org.apache.phoenix.schema.tuple.Tuple; import org.apache.phoenix.schema.types.PLong; +import org.apache.phoenix.util.EncodedColumnsUtil; import org.apache.phoenix.util.ScanUtil; -import org.apache.phoenix.util.SchemaUtil; import org.apache.phoenix.util.TransactionUtil; import com.google.common.collect.Lists; @@ -192,7 +192,7 @@ public class PostDDLCompiler { ScanUtil.setTimeRange(scan, ts); if (emptyCF != null) { scan.setAttribute(BaseScannerRegionObserver.EMPTY_CF, emptyCF); - scan.setAttribute(BaseScannerRegionObserver.EMPTY_COLUMN_QUALIFIER, SchemaUtil.getEmptyKeyValueInfo(tableRef.getTable()).getFirst()); + scan.setAttribute(BaseScannerRegionObserver.EMPTY_COLUMN_QUALIFIER, EncodedColumnsUtil.getEmptyKeyValueInfo(tableRef.getTable()).getFirst()); } ServerCache cache = null; try { @@ -216,7 +216,7 @@ public class PostDDLCompiler { // data empty column family to stay the same, while the index empty column family // changes. PColumn column = deleteList.get(0); - byte[] cq = SchemaUtil.getColumnQualifier(column, tableRef.getTable()); + byte[] cq = EncodedColumnsUtil.getColumnQualifier(column, tableRef.getTable()); if (emptyCF == null) { scan.addColumn(column.getFamilyName().getBytes(), cq); } http://git-wip-us.apache.org/repos/asf/phoenix/blob/3909c633/phoenix-core/src/main/java/org/apache/phoenix/compile/ProjectionCompiler.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/compile/ProjectionCompiler.java b/phoenix-core/src/main/java/org/apache/phoenix/compile/ProjectionCompiler.java index b9bfff3..2f0c377 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/compile/ProjectionCompiler.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/ProjectionCompiler.java @@ -88,6 +88,7 @@ import org.apache.phoenix.schema.ValueBitSet; import org.apache.phoenix.schema.tuple.Tuple; import org.apache.phoenix.schema.types.PDataType; import org.apache.phoenix.util.ByteUtil; +import org.apache.phoenix.util.EncodedColumnsUtil; import org.apache.phoenix.util.IndexUtil; import org.apache.phoenix.util.SchemaUtil; import org.apache.phoenix.util.SizedUtil; @@ -703,7 +704,7 @@ public class ProjectionCompiler { if (expression.getDataType().isArrayType()) { indexProjectedColumns.add(expression); PColumn col = expression.getColumn(); - KeyValueColumnExpression keyValueColumnExpression = new KeyValueColumnExpression(col, SchemaUtil.hasEncodedColumnName(col)); + KeyValueColumnExpression keyValueColumnExpression = new KeyValueColumnExpression(col, EncodedColumnsUtil.hasEncodedColumnName(col)); indexKVs.add(keyValueColumnExpression); copyOfChildren.set(0, keyValueColumnExpression); Integer count = arrayExpressionCounts.get(expression); http://git-wip-us.apache.org/repos/asf/phoenix/blob/3909c633/phoenix-core/src/main/java/org/apache/phoenix/compile/TupleProjectionCompiler.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/compile/TupleProjectionCompiler.java b/phoenix-core/src/main/java/org/apache/phoenix/compile/TupleProjectionCompiler.java index 0933e34..0b871e8 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/compile/TupleProjectionCompiler.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/TupleProjectionCompiler.java @@ -48,6 +48,7 @@ import org.apache.phoenix.schema.PTableImpl; import org.apache.phoenix.schema.PTableType; import org.apache.phoenix.schema.ProjectedColumn; import org.apache.phoenix.schema.TableRef; +import org.apache.phoenix.util.EncodedColumnsUtil; import org.apache.phoenix.util.IndexUtil; import org.apache.phoenix.util.SchemaUtil; @@ -136,7 +137,7 @@ public class TupleProjectionCompiler { projectedColumns.add(column); // Wildcard or FamilyWildcard will be handled by ProjectionCompiler. if (!isWildcard && !families.contains(sourceColumn.getFamilyName())) { - context.getScan().addColumn(sourceColumn.getFamilyName().getBytes(), SchemaUtil.getColumnQualifier(column, table)); + context.getScan().addColumn(sourceColumn.getFamilyName().getBytes(), EncodedColumnsUtil.getColumnQualifier(column, table)); } } // add LocalIndexDataColumnRef http://git-wip-us.apache.org/repos/asf/phoenix/blob/3909c633/phoenix-core/src/main/java/org/apache/phoenix/compile/WhereCompiler.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/compile/WhereCompiler.java b/phoenix-core/src/main/java/org/apache/phoenix/compile/WhereCompiler.java index c2e42f2..5e64209 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/compile/WhereCompiler.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/WhereCompiler.java @@ -56,6 +56,7 @@ import org.apache.phoenix.schema.TableRef; import org.apache.phoenix.schema.TypeMismatchException; import org.apache.phoenix.schema.types.PBoolean; import org.apache.phoenix.util.ByteUtil; +import org.apache.phoenix.util.EncodedColumnsUtil; import org.apache.phoenix.util.ScanUtil; import org.apache.phoenix.util.SchemaUtil; @@ -170,7 +171,7 @@ public class WhereCompiler { TableRef tableRef = ref.getTableRef(); if (tableRef.equals(context.getCurrentTable()) && !SchemaUtil.isPKColumn(ref.getColumn())) { // track the where condition columns. Later we need to ensure the Scan in HRS scans these column CFs - byte[] cq = SchemaUtil.getColumnQualifier(ref.getColumn(), tableRef.getTable()); + byte[] cq = EncodedColumnsUtil.getColumnQualifier(ref.getColumn(), tableRef.getTable()); context.addWhereCoditionColumn(ref.getColumn().getFamilyName().getBytes(), cq); } return ref.newColumnExpression(node.isTableNameCaseSensitive(), node.isCaseSensitive()); http://git-wip-us.apache.org/repos/asf/phoenix/blob/3909c633/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java index a2db5d6..67fff23 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java @@ -186,6 +186,7 @@ import org.apache.phoenix.schema.types.PVarbinary; import org.apache.phoenix.schema.types.PVarchar; import org.apache.phoenix.trace.util.Tracing; import org.apache.phoenix.util.ByteUtil; +import org.apache.phoenix.util.EncodedColumnsUtil; import org.apache.phoenix.util.EnvironmentEdgeManager; import org.apache.phoenix.util.IndexUtil; import org.apache.phoenix.util.KeyValueUtil; @@ -2659,7 +2660,7 @@ public class MetaDataEndpointImpl extends MetaDataProtocol implements Coprocesso byte[] indexKey = SchemaUtil.getTableKey(tenantId, index .getSchemaName().getBytes(), index.getTableName().getBytes()); - byte[] cq = SchemaUtil.getColumnQualifier(columnToDelete, index); + byte[] cq = EncodedColumnsUtil.getColumnQualifier(columnToDelete, index); // If index requires this column for its pk, then drop it if (indexColumns.contains(new ColumnReference(columnToDelete.getFamilyName().getBytes(), cq))) { // Since we're dropping the index, lock it to ensure @@ -2876,7 +2877,7 @@ public class MetaDataEndpointImpl extends MetaDataProtocol implements Coprocesso Put p = new Put(dataTableKey); // Decide on what column qualifier to use for empty key value. PTable currentTable = doGetTable(key, HConstants.LATEST_TIMESTAMP, rowLock); - Pair<byte[], byte[]> emptyKeyValuePair = SchemaUtil.getEmptyKeyValueInfo(currentTable); + Pair<byte[], byte[]> emptyKeyValuePair = EncodedColumnsUtil.getEmptyKeyValueInfo(currentTable); p.add(TABLE_FAMILY_BYTES, emptyKeyValuePair.getFirst(), timeStamp, emptyKeyValuePair.getSecond()); tableMetadata.add(p); } http://git-wip-us.apache.org/repos/asf/phoenix/blob/3909c633/phoenix-core/src/main/java/org/apache/phoenix/execute/BaseQueryPlan.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/execute/BaseQueryPlan.java b/phoenix-core/src/main/java/org/apache/phoenix/execute/BaseQueryPlan.java index 3d10545..9697e33 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/execute/BaseQueryPlan.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/execute/BaseQueryPlan.java @@ -69,12 +69,12 @@ import org.apache.phoenix.schema.TableRef; import org.apache.phoenix.trace.TracingIterator; import org.apache.phoenix.trace.util.Tracing; import org.apache.phoenix.util.ByteUtil; +import org.apache.phoenix.util.EncodedColumnsUtil; import org.apache.phoenix.util.IndexUtil; import org.apache.phoenix.util.LogUtil; import org.apache.phoenix.util.SQLCloseable; import org.apache.phoenix.util.SQLCloseables; import org.apache.phoenix.util.ScanUtil; -import org.apache.phoenix.util.SchemaUtil; import org.cloudera.htrace.TraceScope; import com.google.common.collect.ImmutableSet; @@ -406,7 +406,7 @@ public abstract class BaseQueryPlan implements QueryPlan { WritableUtils.writeVInt(output, dataColumns.size()); for (PColumn column : dataColumns) { Bytes.writeByteArray(output, column.getFamilyName().getBytes()); - Bytes.writeByteArray(output, SchemaUtil.getColumnQualifier(column, dataTable)); + Bytes.writeByteArray(output, EncodedColumnsUtil.getColumnQualifier(column, dataTable)); } scan.setAttribute(BaseScannerRegionObserver.DATA_TABLE_COLUMNS_TO_JOIN, stream.toByteArray()); } catch (IOException e) { http://git-wip-us.apache.org/repos/asf/phoenix/blob/3909c633/phoenix-core/src/main/java/org/apache/phoenix/expression/KeyValueColumnExpression.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/expression/KeyValueColumnExpression.java b/phoenix-core/src/main/java/org/apache/phoenix/expression/KeyValueColumnExpression.java index 3787d6e..35862c4 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/expression/KeyValueColumnExpression.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/expression/KeyValueColumnExpression.java @@ -28,6 +28,7 @@ import org.apache.phoenix.expression.visitor.ExpressionVisitor; import org.apache.phoenix.schema.PColumn; import org.apache.phoenix.schema.PDatum; import org.apache.phoenix.schema.tuple.Tuple; +import org.apache.phoenix.util.EncodedColumnsUtil; import org.apache.phoenix.util.SchemaUtil; @@ -59,7 +60,7 @@ public class KeyValueColumnExpression extends ColumnExpression { public KeyValueColumnExpression(PColumn column, String displayName, boolean encodedColumnName) { super(column); this.cf = column.getFamilyName().getBytes(); - this.cq = SchemaUtil.getColumnQualifier(column, encodedColumnName); + this.cq = EncodedColumnsUtil.getColumnQualifier(column, encodedColumnName); this.displayName = displayName; } http://git-wip-us.apache.org/repos/asf/phoenix/blob/3909c633/phoenix-core/src/main/java/org/apache/phoenix/filter/ColumnProjectionFilter.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/filter/ColumnProjectionFilter.java b/phoenix-core/src/main/java/org/apache/phoenix/filter/ColumnProjectionFilter.java index 4767090..d1f6211 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/filter/ColumnProjectionFilter.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/filter/ColumnProjectionFilter.java @@ -40,7 +40,7 @@ import org.apache.hadoop.hbase.util.Writables; import org.apache.hadoop.io.Writable; import org.apache.hadoop.io.WritableUtils; import org.apache.phoenix.hbase.index.util.ImmutableBytesPtr; -import org.apache.phoenix.util.SchemaUtil; +import org.apache.phoenix.util.EncodedColumnsUtil; /** * When selecting specific columns in a SELECT query, this filter passes only selected columns @@ -67,7 +67,7 @@ public class ColumnProjectionFilter extends FilterBase implements Writable { this.columnsTracker = columnsTracker; this.conditionOnlyCfs = conditionOnlyCfs; this.usesEncodedColumnNames = usesEncodedColumnNames; - this.emptyKVQualifier = SchemaUtil.getEmptyKeyValueInfo(usesEncodedColumnNames).getFirst(); + this.emptyKVQualifier = EncodedColumnsUtil.getEmptyKeyValueInfo(usesEncodedColumnNames).getFirst(); } @Override @@ -92,7 +92,7 @@ public class ColumnProjectionFilter extends FilterBase implements Writable { } int conditionOnlyCfsSize = WritableUtils.readVInt(input); usesEncodedColumnNames = conditionOnlyCfsSize > 0; - emptyKVQualifier = SchemaUtil.getEmptyKeyValueInfo(usesEncodedColumnNames).getFirst(); + emptyKVQualifier = EncodedColumnsUtil.getEmptyKeyValueInfo(usesEncodedColumnNames).getFirst(); conditionOnlyCfsSize = Math.abs(conditionOnlyCfsSize) - 1; // restore to the actual value. this.conditionOnlyCfs = new TreeSet<byte[]>(Bytes.BYTES_COMPARATOR); while (conditionOnlyCfsSize > 0) { http://git-wip-us.apache.org/repos/asf/phoenix/blob/3909c633/phoenix-core/src/main/java/org/apache/phoenix/index/IndexMaintainer.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/index/IndexMaintainer.java b/phoenix-core/src/main/java/org/apache/phoenix/index/IndexMaintainer.java index 8e8d493..170dccc 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/index/IndexMaintainer.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/index/IndexMaintainer.java @@ -87,6 +87,7 @@ import org.apache.phoenix.schema.tuple.ValueGetterTuple; import org.apache.phoenix.schema.types.PDataType; import org.apache.phoenix.util.BitSet; import org.apache.phoenix.util.ByteUtil; +import org.apache.phoenix.util.EncodedColumnsUtil; import org.apache.phoenix.util.IndexUtil; import org.apache.phoenix.util.MetaDataUtil; import org.apache.phoenix.util.SchemaUtil; @@ -323,7 +324,7 @@ public class IndexMaintainer implements Writable, Iterable<ColumnReference> { * that is serialized in it. Because of this we are forced to have the indexes inherit the * storage scheme of the parent data tables. */ - this.usesEncodedColumnNames = SchemaUtil.usesEncodedColumnNames(dataTable); + this.usesEncodedColumnNames = EncodedColumnsUtil.usesEncodedColumnNames(dataTable); byte[] indexTableName = index.getPhysicalName().getBytes(); // Use this for the nDataSaltBuckets as we need this for local indexes // TODO: persist nDataSaltBuckets separately, but maintain b/w compat. @@ -440,8 +441,8 @@ public class IndexMaintainer implements Writable, Iterable<ColumnReference> { PColumnFamily family = index.getColumnFamilies().get(i); for (PColumn indexColumn : family.getColumns()) { PColumn dataColumn = IndexUtil.getDataColumn(dataTable, indexColumn.getName().getString()); - byte[] dataColumnCq = SchemaUtil.getColumnQualifier(dataColumn, dataTable); - byte[] indexColumnCq = SchemaUtil.getColumnQualifier(indexColumn, index); + byte[] dataColumnCq = EncodedColumnsUtil.getColumnQualifier(dataColumn, dataTable); + byte[] indexColumnCq = EncodedColumnsUtil.getColumnQualifier(indexColumn, index); this.coveredColumns.add(new ColumnReference(dataColumn.getFamilyName().getBytes(), dataColumnCq)); this.coveredColumnsMap.put(new ColumnReference(dataColumn.getFamilyName().getBytes(), dataColumnCq), new ColumnReference(indexColumn.getFamilyName().getBytes(), indexColumnCq)); @@ -1236,7 +1237,7 @@ public class IndexMaintainer implements Writable, Iterable<ColumnReference> { * Init calculated state reading/creating */ private void initCachedState() { - byte[] emptyKvQualifier = SchemaUtil.getEmptyKeyValueInfo(usesEncodedColumnNames).getFirst(); + byte[] emptyKvQualifier = EncodedColumnsUtil.getEmptyKeyValueInfo(usesEncodedColumnNames).getFirst(); dataEmptyKeyValueRef = new ColumnReference(emptyKeyValueCFPtr.copyBytesIfNecessary(), emptyKvQualifier); emptyKeyValueQualifierPtr = new ImmutableBytesPtr(emptyKvQualifier); this.allColumns = Sets.newLinkedHashSetWithExpectedSize(indexedExpressions.size() + coveredColumns.size()); http://git-wip-us.apache.org/repos/asf/phoenix/blob/3909c633/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java b/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java index 7593718..55bee9e 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java @@ -83,6 +83,7 @@ import org.apache.phoenix.schema.stats.GuidePostsInfo; import org.apache.phoenix.schema.stats.PTableStats; import org.apache.phoenix.util.ByteUtil; import org.apache.phoenix.util.Closeables; +import org.apache.phoenix.util.EncodedColumnsUtil; import org.apache.phoenix.util.LogUtil; import org.apache.phoenix.util.PrefixByteCodec; import org.apache.phoenix.util.PrefixByteDecoder; @@ -204,7 +205,7 @@ public abstract class BaseResultIterators extends ExplainTable implements Result // Project empty key value unless the column family containing it has // been projected in its entirety. if (!familyMap.containsKey(ecf) || familyMap.get(ecf) != null) { - scan.addColumn(ecf, SchemaUtil.getEmptyKeyValueInfo(table).getFirst()); + scan.addColumn(ecf, EncodedColumnsUtil.getEmptyKeyValueInfo(table).getFirst()); } } } @@ -321,7 +322,7 @@ public abstract class BaseResultIterators extends ExplainTable implements Result // the ExplicitColumnTracker not to be used, though. if (!statement.isAggregate() && filteredColumnNotInProjection) { ScanUtil.andFilterAtEnd(scan, new ColumnProjectionFilter(SchemaUtil.getEmptyColumnFamily(table), - columnsTracker, conditionOnlyCfs, SchemaUtil.usesEncodedColumnNames(table))); + columnsTracker, conditionOnlyCfs, EncodedColumnsUtil.usesEncodedColumnNames(table))); } } } http://git-wip-us.apache.org/repos/asf/phoenix/blob/3909c633/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/FormatToBytesWritableMapper.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/FormatToBytesWritableMapper.java b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/FormatToBytesWritableMapper.java index b975a03..d9b5b7f 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/FormatToBytesWritableMapper.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/FormatToBytesWritableMapper.java @@ -48,9 +48,9 @@ import org.apache.phoenix.mapreduce.util.PhoenixConfigurationUtil; import org.apache.phoenix.schema.PColumn; import org.apache.phoenix.schema.PTable; import org.apache.phoenix.util.ColumnInfo; +import org.apache.phoenix.util.EncodedColumnsUtil; import org.apache.phoenix.util.PhoenixRuntime; import org.apache.phoenix.util.QueryUtil; -import org.apache.phoenix.util.SchemaUtil; import org.apache.phoenix.util.UpsertExecutor; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -135,7 +135,7 @@ public abstract class FormatToBytesWritableMapper<RECORD> extends Mapper<LongWri List<String> logicalTableNames = TargetTableRefFunctions.NAMES_FROM_JSON.apply(logicalNamesConf); for (String logicalTableName : logicalTableNames) { PTable table = PhoenixRuntime.getTable(conn, logicalTableName); - logicalTables.add(new Pair<>(logicalTableName, SchemaUtil.getEmptyKeyValueInfo(table).getFirst())); + logicalTables.add(new Pair<>(logicalTableName, EncodedColumnsUtil.getEmptyKeyValueInfo(table).getFirst())); } columnIndexes = initColumnIndexes(); } catch (SQLException | ClassNotFoundException e) { http://git-wip-us.apache.org/repos/asf/phoenix/blob/3909c633/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/FormatToKeyValueReducer.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/FormatToKeyValueReducer.java b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/FormatToKeyValueReducer.java index cc1559a..ddf6120 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/FormatToKeyValueReducer.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/FormatToKeyValueReducer.java @@ -44,6 +44,7 @@ import org.apache.phoenix.schema.PColumn; import org.apache.phoenix.schema.PTable; import org.apache.phoenix.util.ByteUtil; import org.apache.phoenix.util.Closeables; +import org.apache.phoenix.util.EncodedColumnsUtil; import org.apache.phoenix.util.PhoenixRuntime; import org.apache.phoenix.util.QueryUtil; import org.apache.phoenix.util.SchemaUtil; @@ -106,7 +107,7 @@ public class FormatToKeyValueReducer continue; } byte[] family = c.getFamilyName().getBytes(); - byte[] name = SchemaUtil.getColumnQualifier(c, table); + byte[] name = EncodedColumnsUtil.getColumnQualifier(c, table); list.add(new Pair<>(family, name)); } columnIndexes.add(list); http://git-wip-us.apache.org/repos/asf/phoenix/blob/3909c633/phoenix-core/src/main/java/org/apache/phoenix/schema/ColumnRef.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/ColumnRef.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/ColumnRef.java index d7c6456..544fb20 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/schema/ColumnRef.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/ColumnRef.java @@ -22,6 +22,7 @@ import org.apache.phoenix.expression.ColumnExpression; import org.apache.phoenix.expression.KeyValueColumnExpression; import org.apache.phoenix.expression.ProjectedColumnExpression; import org.apache.phoenix.expression.RowKeyColumnExpression; +import org.apache.phoenix.util.EncodedColumnsUtil; import org.apache.phoenix.util.SchemaUtil; @@ -109,7 +110,7 @@ public class ColumnRef { return new ProjectedColumnExpression(column, table, displayName); } - return new KeyValueColumnExpression(column, displayName, SchemaUtil.usesEncodedColumnNames(table)); + return new KeyValueColumnExpression(column, displayName, EncodedColumnsUtil.usesEncodedColumnNames(table)); } public ColumnRef cloneAtTimestamp(long timestamp) { http://git-wip-us.apache.org/repos/asf/phoenix/blob/3909c633/phoenix-core/src/main/java/org/apache/phoenix/schema/DelegateColumn.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/DelegateColumn.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/DelegateColumn.java index 65e362c..4ac8f46 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/schema/DelegateColumn.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/DelegateColumn.java @@ -92,7 +92,7 @@ public class DelegateColumn extends DelegateDatum implements PColumn { } @Override - public Integer getColumnQualifier() { - return getDelegate().getColumnQualifier(); + public Integer getEncodedColumnQualifier() { + return getDelegate().getEncodedColumnQualifier(); } } http://git-wip-us.apache.org/repos/asf/phoenix/blob/3909c633/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java index d7203ec..99ba25b 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java @@ -194,6 +194,7 @@ import org.apache.phoenix.schema.types.PUnsignedLong; import org.apache.phoenix.schema.types.PVarbinary; import org.apache.phoenix.schema.types.PVarchar; import org.apache.phoenix.util.ByteUtil; +import org.apache.phoenix.util.EncodedColumnsUtil; import org.apache.phoenix.util.IndexUtil; import org.apache.phoenix.util.LogUtil; import org.apache.phoenix.util.MetaDataUtil; @@ -753,10 +754,10 @@ public class MetaDataClient { } else { colUpsert.setString(18, column.getExpressionStr()); } - if (column.getColumnQualifier() == null) { + if (column.getEncodedColumnQualifier() == null) { colUpsert.setNull(19, Types.INTEGER); } else { - colUpsert.setInt(19, column.getColumnQualifier()); + colUpsert.setInt(19, column.getEncodedColumnQualifier()); } if (colUpsert.getParameterMetaData().getParameterCount() > 19) { colUpsert.setBoolean(20, column.isRowTimestamp()); @@ -1863,7 +1864,7 @@ public class MetaDataClient { int position = positionOffset; StorageScheme storageScheme = null; - Map<String, Integer> nextColumnQualifiers = null; // this would be null for tables created before phoenix 4.8. + Map<String, Integer> nextColumnQualifiers = null; // this would be null for tables created for columns with storage scheme != ENCODED_COLUMN_NAMES if (SchemaUtil.isSystemTable(Bytes.toBytes(SchemaUtil.getTableName(schemaName, tableName)))) { // System tables have hard-coded column qualifiers. So we can't use column encoding for them. storageScheme = StorageScheme.NON_ENCODED_COLUMN_NAMES; @@ -1877,7 +1878,7 @@ public class MetaDataClient { // HTable. storageScheme = parent.getStorageScheme(); if (storageScheme == StorageScheme.ENCODED_COLUMN_NAMES) { - nextColumnQualifiers = SchemaUtil.getNextColumnQualifiers(parent); + nextColumnQualifiers = SchemaUtil.getNextEncodedColumnQualifiers(parent); } } } else { @@ -2751,7 +2752,7 @@ public class MetaDataClient { List<PColumn> columns = Lists.newArrayListWithExpectedSize(columnDefs.size()); Set<String> colFamiliesForPColumnsToBeAdded = new LinkedHashSet<>(); Set<String> families = new LinkedHashSet<>(); - Map<String, Integer> nextColumnQualifiers = SchemaUtil.getNextColumnQualifiers(table); + Map<String, Integer> nextColumnQualifiers = SchemaUtil.getNextEncodedColumnQualifiers(table); if (columnDefs.size() > 0 ) { try (PreparedStatement colUpsert = connection.prepareStatement(INSERT_COLUMN_ALTER_TABLE)) { short nextKeySeq = SchemaUtil.getMaxKeySeq(table); @@ -3079,7 +3080,7 @@ public class MetaDataClient { Set<ColumnReference> coveredColumns = indexMaintainer.getCoverededColumns(); List<PColumn> indexColumnsToDrop = Lists.newArrayListWithExpectedSize(columnRefs.size()); for(PColumn columnToDrop : tableColumnsToDrop) { - ColumnReference columnToDropRef = new ColumnReference(columnToDrop.getFamilyName().getBytes(), SchemaUtil.getColumnQualifier(columnToDrop, index)); + ColumnReference columnToDropRef = new ColumnReference(columnToDrop.getFamilyName().getBytes(), EncodedColumnsUtil.getColumnQualifier(columnToDrop, index)); if (indexColumns.contains(columnToDropRef)) { indexesToDrop.add(new TableRef(index)); } http://git-wip-us.apache.org/repos/asf/phoenix/blob/3909c633/phoenix-core/src/main/java/org/apache/phoenix/schema/PColumn.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/PColumn.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/PColumn.java index 8f61d6d..a2fadac 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/schema/PColumn.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/PColumn.java @@ -66,5 +66,5 @@ public interface PColumn extends PDatum { * TODO: samarth I think we should should change this to return byte[] array. * Then we won't have to worry about calling SchemaUtil... everywhere */ - Integer getColumnQualifier(); + Integer getEncodedColumnQualifier(); } http://git-wip-us.apache.org/repos/asf/phoenix/blob/3909c633/phoenix-core/src/main/java/org/apache/phoenix/schema/PColumnFamilyImpl.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/PColumnFamilyImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/PColumnFamilyImpl.java index a3d855b..c175aa2 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/schema/PColumnFamilyImpl.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/PColumnFamilyImpl.java @@ -21,7 +21,7 @@ import java.util.List; import java.util.Map; import org.apache.hadoop.hbase.util.Bytes; -import org.apache.phoenix.util.SchemaUtil; +import org.apache.phoenix.util.EncodedColumnsUtil; import org.apache.phoenix.util.SizedUtil; import com.google.common.base.Preconditions; @@ -34,7 +34,7 @@ public class PColumnFamilyImpl implements PColumnFamily { private final List<PColumn> columns; private final Map<String, PColumn> columnNamesByStrings; private final Map<byte[], PColumn> columnNamesByBytes; - private final Map<byte[], PColumn> columnQualifiersByBytes; + private final Map<byte[], PColumn> encodedColumnQualifersByBytes; private final int estimatedSize; @Override @@ -51,19 +51,18 @@ public class PColumnFamilyImpl implements PColumnFamily { this.columns = ImmutableList.copyOf(columns); ImmutableMap.Builder<String, PColumn> columnNamesByStringBuilder = ImmutableMap.builder(); ImmutableSortedMap.Builder<byte[], PColumn> columnNamesByBytesBuilder = ImmutableSortedMap.orderedBy(Bytes.BYTES_COMPARATOR); - ImmutableSortedMap.Builder<byte[], PColumn> columnQualifiersByBytesBuilder = ImmutableSortedMap.orderedBy(Bytes.BYTES_COMPARATOR); + ImmutableSortedMap.Builder<byte[], PColumn> encodedColumnQualifiersByBytesBuilder = ImmutableSortedMap.orderedBy(Bytes.BYTES_COMPARATOR); for (PColumn column : columns) { estimatedSize += column.getEstimatedSize(); columnNamesByBytesBuilder.put(column.getName().getBytes(), column); columnNamesByStringBuilder.put(column.getName().getString(), column); - // TODO: samarth fix this. projected columns have column family for pk columns which messes up with checks. - if (!useEncodedColumnNames || (useEncodedColumnNames && !(column.getColumnQualifier() == null))) { - columnQualifiersByBytesBuilder.put(SchemaUtil.getColumnQualifier(column, useEncodedColumnNames), column); + if (useEncodedColumnNames && column.getEncodedColumnQualifier() != null) { + encodedColumnQualifiersByBytesBuilder.put(EncodedColumnsUtil.getEncodedColumnQualifier(column), column); } } this.columnNamesByBytes = columnNamesByBytesBuilder.build(); this.columnNamesByStrings = columnNamesByStringBuilder.build(); - this.columnQualifiersByBytes = useEncodedColumnNames ? columnQualifiersByBytesBuilder.build() : columnNamesByBytes; + this.encodedColumnQualifersByBytes = encodedColumnQualifiersByBytesBuilder.build(); this.estimatedSize = (int)estimatedSize; } @@ -97,9 +96,13 @@ public class PColumnFamilyImpl implements PColumnFamily { @Override public PColumn getPColumnForColumnQualifier(byte[] cq) throws ColumnNotFoundException { - PColumn column = columnQualifiersByBytes.get(cq); + Preconditions.checkNotNull(cq); + PColumn column = encodedColumnQualifersByBytes.get(cq); if (column == null) { - throw new ColumnNotFoundException(Bytes.toString(cq)); + // For tables with non-encoded column names, column qualifiers are + // column name bytes. Also dynamic columns don't have encoded column + // qualifiers. So they could be found in the column name by bytes map. + return getPColumnForColumnNameBytes(cq); } return column; } http://git-wip-us.apache.org/repos/asf/phoenix/blob/3909c633/phoenix-core/src/main/java/org/apache/phoenix/schema/PColumnImpl.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/PColumnImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/PColumnImpl.java index 5cf1465..d7d2e0c 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/schema/PColumnImpl.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/PColumnImpl.java @@ -58,7 +58,7 @@ public class PColumnImpl implements PColumn { public PColumnImpl(PColumn column, int position) { this(column.getName(), column.getFamilyName(), column.getDataType(), column.getMaxLength(), - column.getScale(), column.isNullable(), position, column.getSortOrder(), column.getArraySize(), column.getViewConstant(), column.isViewReferenced(), column.getExpressionStr(), column.isRowTimestamp(), column.isDynamic(), column.getColumnQualifier()); + column.getScale(), column.isNullable(), position, column.getSortOrder(), column.getArraySize(), column.getViewConstant(), column.isViewReferenced(), column.getExpressionStr(), column.isRowTimestamp(), column.isDynamic(), column.getEncodedColumnQualifier()); } private void init(PName name, @@ -209,7 +209,7 @@ public class PColumnImpl implements PColumn { } @Override - public Integer getColumnQualifier() { + public Integer getEncodedColumnQualifier() { return columnQualifier; } @@ -294,8 +294,8 @@ public class PColumnImpl implements PColumn { builder.setExpression(column.getExpressionStr()); } builder.setIsRowTimestamp(column.isRowTimestamp()); - if (column.getColumnQualifier() != null) { - builder.setColumnQualifier(column.getColumnQualifier()); + if (column.getEncodedColumnQualifier() != null) { + builder.setColumnQualifier(column.getEncodedColumnQualifier()); } return builder.build(); } http://git-wip-us.apache.org/repos/asf/phoenix/blob/3909c633/phoenix-core/src/main/java/org/apache/phoenix/schema/PMetaDataImpl.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/PMetaDataImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/PMetaDataImpl.java index 8c85ae5..99c4636 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/schema/PMetaDataImpl.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/PMetaDataImpl.java @@ -414,7 +414,7 @@ public class PMetaDataImpl implements PMetaData { // Update position of columns that follow removed column for (int i = position+1; i < oldColumns.size(); i++) { PColumn oldColumn = oldColumns.get(i); - PColumn newColumn = new PColumnImpl(oldColumn.getName(), oldColumn.getFamilyName(), oldColumn.getDataType(), oldColumn.getMaxLength(), oldColumn.getScale(), oldColumn.isNullable(), i-1+positionOffset, oldColumn.getSortOrder(), oldColumn.getArraySize(), oldColumn.getViewConstant(), oldColumn.isViewReferenced(), null, oldColumn.isRowTimestamp(), oldColumn.isDynamic(), oldColumn.getColumnQualifier()); + PColumn newColumn = new PColumnImpl(oldColumn.getName(), oldColumn.getFamilyName(), oldColumn.getDataType(), oldColumn.getMaxLength(), oldColumn.getScale(), oldColumn.isNullable(), i-1+positionOffset, oldColumn.getSortOrder(), oldColumn.getArraySize(), oldColumn.getViewConstant(), oldColumn.isViewReferenced(), null, oldColumn.isRowTimestamp(), oldColumn.isDynamic(), oldColumn.getEncodedColumnQualifier()); columns.add(newColumn); } http://git-wip-us.apache.org/repos/asf/phoenix/blob/3909c633/phoenix-core/src/main/java/org/apache/phoenix/schema/PTableImpl.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/PTableImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/PTableImpl.java index 0fdba22..e517fcf 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/schema/PTableImpl.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/PTableImpl.java @@ -17,7 +17,6 @@ */ package org.apache.phoenix.schema; -import static com.google.common.base.Preconditions.checkArgument; import static org.apache.phoenix.hbase.index.util.KeyValueBuilder.addQuietly; import static org.apache.phoenix.hbase.index.util.KeyValueBuilder.deleteQuietly; import static org.apache.phoenix.schema.SaltingUtil.SALTING_COLUMN; @@ -66,6 +65,7 @@ import org.apache.phoenix.schema.types.PFloat; import org.apache.phoenix.schema.types.PInteger; import org.apache.phoenix.schema.types.PVarchar; import org.apache.phoenix.util.ByteUtil; +import org.apache.phoenix.util.EncodedColumnsUtil; import org.apache.phoenix.util.SchemaUtil; import org.apache.phoenix.util.SizedUtil; import org.apache.phoenix.util.StringUtil; @@ -112,7 +112,7 @@ public class PTableImpl implements PTable { private Map<byte[], PColumnFamily> familyByBytes; private Map<String, PColumnFamily> familyByString; private ListMultimap<String, PColumn> columnsByName; - private ListMultimap<Integer, PColumn> kvColumnsByColumnQualifiers; + private ListMultimap<Integer, PColumn> kvColumnsByEncodedColumnNames; private PName pkName; private Integer bucketNum; private RowKeySchema rowKeySchema; @@ -392,7 +392,7 @@ public class PTableImpl implements PTable { PColumn[] allColumns; this.columnsByName = ArrayListMultimap.create(columns.size(), 1); - this.kvColumnsByColumnQualifiers = (storageScheme == StorageScheme.ENCODED_COLUMN_NAMES ? ArrayListMultimap.<Integer, PColumn>create(columns.size(), 1) : null); + this.kvColumnsByEncodedColumnNames = (storageScheme == StorageScheme.ENCODED_COLUMN_NAMES ? ArrayListMultimap.<Integer, PColumn>create(columns.size(), 1) : null); int numPKColumns = 0; if (bucketNum != null) { // Add salt column to allColumns and pkColumns, but don't add to @@ -424,12 +424,12 @@ public class PTableImpl implements PTable { } } } - Integer cq = column.getColumnQualifier(); + Integer cq = column.getEncodedColumnQualifier(); //TODO: samarth understand the implication of this. - if (kvColumnsByColumnQualifiers != null && cq != null) { - if (kvColumnsByColumnQualifiers.put(cq, column)) { + if (kvColumnsByEncodedColumnNames != null && cq != null) { + if (kvColumnsByEncodedColumnNames.put(cq, column)) { int count = 0; - for (PColumn dupColumn : kvColumnsByColumnQualifiers.get(cq)) { + for (PColumn dupColumn : kvColumnsByEncodedColumnNames.get(cq)) { if (Objects.equal(familyName, dupColumn.getFamilyName())) { count++; if (count > 1) { @@ -707,12 +707,13 @@ public class PTableImpl implements PTable { @Override public PColumn getPColumnForColumnQualifier(byte[] cq) throws ColumnNotFoundException, AmbiguousColumnException { - if (SchemaUtil.usesEncodedColumnNames(this)) { + Preconditions.checkNotNull(cq); + if (!EncodedColumnsUtil.usesEncodedColumnNames(this)) { String columnName = (String)PVarchar.INSTANCE.toObject(cq); return getPColumnForColumnName(columnName); } else { Integer qualifier = (Integer)PInteger.INSTANCE.toObject(cq); - List<PColumn> columns = kvColumnsByColumnQualifiers.get(qualifier); + List<PColumn> columns = kvColumnsByEncodedColumnNames.get(qualifier); int size = columns.size(); if (size == 0) { //TODO: samarth should we have a column qualifier not found exception? @@ -788,7 +789,7 @@ public class PTableImpl implements PTable { // Because we cannot enforce a not null constraint on a KV column (since we don't know if the row exists when // we upsert it), se instead add a KV that is always emtpy. This allows us to imitate SQL semantics given the // way HBase works. - Pair<byte[], byte[]> emptyKvInfo = SchemaUtil.getEmptyKeyValueInfo(PTableImpl.this); + Pair<byte[], byte[]> emptyKvInfo = EncodedColumnsUtil.getEmptyKeyValueInfo(PTableImpl.this); addQuietly(setValues, kvBuilder, kvBuilder.buildPut(keyPtr, SchemaUtil.getEmptyColumnFamilyPtr(PTableImpl.this), new ImmutableBytesPtr(emptyKvInfo.getFirst()), ts, @@ -827,7 +828,7 @@ public class PTableImpl implements PTable { deleteRow = null; byte[] family = column.getFamilyName().getBytes(); byte[] qualifier = getColumnQualifier(column); - ImmutableBytesPtr qualifierPtr = getColumnQualifierPtr(column); + ImmutableBytesPtr qualifierPtr = new ImmutableBytesPtr(qualifier); PDataType type = column.getDataType(); // Check null, since some types have no byte representation for null boolean isNull = type.isNull(byteValue); @@ -882,24 +883,9 @@ public class PTableImpl implements PTable { } private byte[] getColumnQualifier(PColumn column) { - //return column.getName().getBytes(); - checkArgument(!SchemaUtil.isPKColumn(column), "No column qualifiers for PK columns"); - boolean tableUsesEncodedColumnNames = SchemaUtil.usesEncodedColumnNames(PTableImpl.this); - if (!tableUsesEncodedColumnNames) { - return column.getName().getBytes(); - } - return PInteger.INSTANCE.toBytes(column.getColumnQualifier()); + return EncodedColumnsUtil.getColumnQualifier(column, PTableImpl.this); } - private ImmutableBytesPtr getColumnQualifierPtr(PColumn column) { -// return column.getName().getBytesPtr(); - checkArgument(!SchemaUtil.isPKColumn(column), "No column qualifiers for PK columns"); - boolean tableUsesEncodedColumnNames = SchemaUtil.usesEncodedColumnNames(PTableImpl.this); - if (!tableUsesEncodedColumnNames) { - return column.getName().getBytesPtr(); - } - return new ImmutableBytesPtr(PInteger.INSTANCE.toBytes(column.getColumnQualifier())); - } } @Override http://git-wip-us.apache.org/repos/asf/phoenix/blob/3909c633/phoenix-core/src/main/java/org/apache/phoenix/util/EncodedColumnsUtil.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/EncodedColumnsUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/util/EncodedColumnsUtil.java new file mode 100644 index 0000000..bfbfc54 --- /dev/null +++ b/phoenix-core/src/main/java/org/apache/phoenix/util/EncodedColumnsUtil.java @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.phoenix.util; + +import static com.google.common.base.Preconditions.checkArgument; + +import org.apache.hadoop.hbase.util.Pair; +import org.apache.phoenix.query.QueryConstants; +import org.apache.phoenix.schema.PColumn; +import org.apache.phoenix.schema.PTable; +import org.apache.phoenix.schema.PTable.StorageScheme; +import org.apache.phoenix.schema.types.PInteger; + +public class EncodedColumnsUtil { + + public static boolean usesEncodedColumnNames(PTable table) { + return table.getStorageScheme() != null && table.getStorageScheme() == StorageScheme.ENCODED_COLUMN_NAMES; + } + + public static byte[] getEncodedColumnQualifier(PColumn column) { + checkArgument(!SchemaUtil.isPKColumn(column), "No column qualifiers for PK columns"); + checkArgument(!column.isDynamic(), "No encoded column qualifiers for dynamic columns"); + return PInteger.INSTANCE.toBytes(column.getEncodedColumnQualifier()); + } + + public static byte[] getColumnQualifier(PColumn column, PTable table) { + return EncodedColumnsUtil.getColumnQualifier(column, usesEncodedColumnNames(table)); + } + + public static byte[] getColumnQualifier(PColumn column, boolean encodedColumnName) { + checkArgument(!SchemaUtil.isPKColumn(column), "No column qualifiers for PK columns"); + if (column.isDynamic()) { // Dynamic column names don't have encoded column names + return column.getName().getBytes(); + } + return encodedColumnName ? PInteger.INSTANCE.toBytes(column.getEncodedColumnQualifier()) : column.getName().getBytes(); + } + + /** + * @return pair of byte arrays. The first part of the pair is the empty key value's column qualifier, and the second + * part is the value to use for it. + */ + public static Pair<byte[], byte[]> getEmptyKeyValueInfo(PTable table) { + return usesEncodedColumnNames(table) ? new Pair<>(QueryConstants.ENCODED_EMPTY_COLUMN_BYTES, + QueryConstants.ENCODED_EMPTY_COLUMN_VALUE_BYTES) : new Pair<>(QueryConstants.EMPTY_COLUMN_BYTES, + QueryConstants.EMPTY_COLUMN_VALUE_BYTES); + } + + /** + * @return pair of byte arrays. The first part of the pair is the empty key value's column qualifier, and the second + * part is the value to use for it. + */ + public static Pair<byte[], byte[]> getEmptyKeyValueInfo(boolean usesEncodedColumnNames) { + return usesEncodedColumnNames ? new Pair<>(QueryConstants.ENCODED_EMPTY_COLUMN_BYTES, + QueryConstants.ENCODED_EMPTY_COLUMN_VALUE_BYTES) : new Pair<>(QueryConstants.EMPTY_COLUMN_BYTES, + QueryConstants.EMPTY_COLUMN_VALUE_BYTES); + } + + public static boolean hasEncodedColumnName(PColumn column){ + return !SchemaUtil.isPKColumn(column) && !column.isDynamic() && column.getEncodedColumnQualifier() != null; + } + +} http://git-wip-us.apache.org/repos/asf/phoenix/blob/3909c633/phoenix-core/src/main/java/org/apache/phoenix/util/IndexUtil.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/IndexUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/util/IndexUtil.java index e9fddab..9407a39 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/util/IndexUtil.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/util/IndexUtil.java @@ -198,7 +198,7 @@ public class IndexUtil { private static boolean isEmptyKeyValue(PTable table, ColumnReference ref) { byte[] emptyKeyValueCF = SchemaUtil.getEmptyColumnFamily(table); - byte[] emptyKeyValueQualifier = SchemaUtil.getEmptyKeyValueInfo(table).getFirst(); + byte[] emptyKeyValueQualifier = EncodedColumnsUtil.getEmptyKeyValueInfo(table).getFirst(); return (Bytes.compareTo(emptyKeyValueCF, 0, emptyKeyValueCF.length, ref.getFamilyWritable() .get(), ref.getFamilyWritable().getOffset(), ref.getFamilyWritable().getLength()) == 0 && Bytes .compareTo(emptyKeyValueQualifier, 0, http://git-wip-us.apache.org/repos/asf/phoenix/blob/3909c633/phoenix-core/src/main/java/org/apache/phoenix/util/SchemaUtil.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/SchemaUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/util/SchemaUtil.java index b76b50e..034e7ac 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/util/SchemaUtil.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/util/SchemaUtil.java @@ -44,7 +44,6 @@ import javax.annotation.Nullable; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding; import org.apache.hadoop.hbase.util.Bytes; -import org.apache.hadoop.hbase.util.Pair; import org.apache.phoenix.exception.SQLExceptionCode; import org.apache.phoenix.exception.SQLExceptionInfo; import org.apache.phoenix.expression.Expression; @@ -61,7 +60,6 @@ import org.apache.phoenix.schema.PColumnFamily; import org.apache.phoenix.schema.PDatum; import org.apache.phoenix.schema.PName; import org.apache.phoenix.schema.PTable; -import org.apache.phoenix.schema.PTable.StorageScheme; import org.apache.phoenix.schema.RowKeySchema; import org.apache.phoenix.schema.RowKeySchema.RowKeySchemaBuilder; import org.apache.phoenix.schema.SaltingUtil; @@ -69,7 +67,6 @@ import org.apache.phoenix.schema.SortOrder; import org.apache.phoenix.schema.TableProperty; import org.apache.phoenix.schema.ValueSchema.Field; import org.apache.phoenix.schema.types.PDataType; -import org.apache.phoenix.schema.types.PInteger; import org.apache.phoenix.schema.types.PVarbinary; import org.apache.phoenix.schema.types.PVarchar; @@ -146,7 +143,7 @@ public class SchemaUtil { rowSize += KeyValue.getKeyValueDataStructureSize(keyLength, column.getFamilyName().getBytes().length, column.getName().getBytes().length, valueLength); } } - byte[] emptyKeyValueKV = SchemaUtil.getEmptyKeyValueInfo(table).getFirst(); + byte[] emptyKeyValueKV = EncodedColumnsUtil.getEmptyKeyValueInfo(table).getFirst(); // Empty key value rowSize += KeyValue.getKeyValueDataStructureSize(keyLength, getEmptyColumnFamily(table).length, emptyKeyValueKV.length, 0); return rowSize; @@ -906,52 +903,22 @@ public class SchemaUtil { /** * Return a map of column family -> next column qualifier number to use. */ - public static Map<String, Integer> getNextColumnQualifiers(PTable table) { - Map<String, Integer> map = Maps.newHashMapWithExpectedSize(table.getColumns().size()); - for (PColumnFamily f : table.getColumnFamilies()) { - final int size = f.getColumns().size(); - // column qualifiers start with 1. - map.put(f.getName().getString(), size + 1); + public static Map<String, Integer> getNextEncodedColumnQualifiers(PTable table) { + if (EncodedColumnsUtil.usesEncodedColumnNames(table)) { + Map<String, Integer> map = Maps.newHashMapWithExpectedSize(table.getColumns().size()); + int max = 0; + for (PColumnFamily f : table.getColumnFamilies()) { + for (PColumn column : f.getColumns()) { + if (column.getEncodedColumnQualifier() > max) { + max = column.getEncodedColumnQualifier(); + } + } + // column qualifiers start with 1. + map.put(f.getName().getString(), max + 1); + } + return map; } - return map; - } - - public static boolean usesEncodedColumnNames(PTable table) { - return table.getStorageScheme() != null && table.getStorageScheme() == StorageScheme.ENCODED_COLUMN_NAMES; - } - - public static byte[] getColumnQualifier(PColumn column, PTable table) { - checkArgument(!SchemaUtil.isPKColumn(column), "No column qualifiers for PK columns"); - return usesEncodedColumnNames(table) ? PInteger.INSTANCE.toBytes(column.getColumnQualifier()) : column.getName().getBytes(); - } - - public static byte[] getColumnQualifier(PColumn column, boolean encodedColumnName) { - checkArgument(!SchemaUtil.isPKColumn(column), "No column qualifiers for PK columns"); - return encodedColumnName ? PInteger.INSTANCE.toBytes(column.getColumnQualifier()) : column.getName().getBytes(); - } - - /** - * @return pair of byte arrays. The first part of the pair is the empty key value's column qualifier, and the second - * part is the value to use for it. - */ - public static Pair<byte[], byte[]> getEmptyKeyValueInfo(PTable table) { - return usesEncodedColumnNames(table) ? new Pair<>(QueryConstants.ENCODED_EMPTY_COLUMN_BYTES, - QueryConstants.ENCODED_EMPTY_COLUMN_VALUE_BYTES) : new Pair<>(QueryConstants.EMPTY_COLUMN_BYTES, - QueryConstants.EMPTY_COLUMN_VALUE_BYTES); - } - - /** - * @return pair of byte arrays. The first part of the pair is the empty key value's column qualifier, and the second - * part is the value to use for it. - */ - public static Pair<byte[], byte[]> getEmptyKeyValueInfo(boolean usesEncodedColumnNames) { - return usesEncodedColumnNames ? new Pair<>(QueryConstants.ENCODED_EMPTY_COLUMN_BYTES, - QueryConstants.ENCODED_EMPTY_COLUMN_VALUE_BYTES) : new Pair<>(QueryConstants.EMPTY_COLUMN_BYTES, - QueryConstants.EMPTY_COLUMN_VALUE_BYTES); - } - - public static boolean hasEncodedColumnName(PColumn column){ - return !SchemaUtil.isPKColumn(column) && column.getColumnQualifier() != null; + return null; } } http://git-wip-us.apache.org/repos/asf/phoenix/blob/3909c633/phoenix-core/src/test/java/org/apache/phoenix/iterate/AggregateResultScannerTest.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/test/java/org/apache/phoenix/iterate/AggregateResultScannerTest.java b/phoenix-core/src/test/java/org/apache/phoenix/iterate/AggregateResultScannerTest.java index cb9c640..70156a1 100644 --- a/phoenix-core/src/test/java/org/apache/phoenix/iterate/AggregateResultScannerTest.java +++ b/phoenix-core/src/test/java/org/apache/phoenix/iterate/AggregateResultScannerTest.java @@ -138,7 +138,7 @@ public class AggregateResultScannerTest extends BaseConnectionlessQueryTest { return false; } @Override - public Integer getColumnQualifier() { + public Integer getEncodedColumnQualifier() { return null; } }, false)), null);