Repository: phoenix Updated Branches: refs/heads/4.x-HBase-1.1 16cde5018 -> 543a201bc
PHOENIX-3917 RowProjector#getEstimatedRowByteSize() returns incorrect value (Biju Nair) Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/543a201b Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/543a201b Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/543a201b Branch: refs/heads/4.x-HBase-1.1 Commit: 543a201bc265f549aa78ca555fc92ba4218783df Parents: 16cde50 Author: Samarth Jain <sama...@apache.org> Authored: Thu Jun 8 10:50:22 2017 -0700 Committer: Samarth Jain <sama...@apache.org> Committed: Thu Jun 8 10:50:22 2017 -0700 ---------------------------------------------------------------------- .../phoenix/compile/ProjectionCompiler.java | 140 +++++++++++++------ 1 file changed, 94 insertions(+), 46 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/phoenix/blob/543a201b/phoenix-core/src/main/java/org/apache/phoenix/compile/ProjectionCompiler.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/compile/ProjectionCompiler.java b/phoenix-core/src/main/java/org/apache/phoenix/compile/ProjectionCompiler.java index 4c4fb16..6362083 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/compile/ProjectionCompiler.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/ProjectionCompiler.java @@ -344,15 +344,20 @@ public class ProjectionCompiler { * @return projector used to access row values during scan * @throws SQLException */ - public static RowProjector compile(StatementContext context, SelectStatement statement, GroupBy groupBy, List<? extends PDatum> targetColumns, Expression where) throws SQLException { + public static RowProjector compile(StatementContext context, SelectStatement statement, + GroupBy groupBy, List<? extends PDatum> targetColumns, Expression where) + throws SQLException { List<KeyValueColumnExpression> arrayKVRefs = new ArrayList<KeyValueColumnExpression>(); - List<ProjectedColumnExpression> arrayProjectedColumnRefs = new ArrayList<ProjectedColumnExpression>(); + List<ProjectedColumnExpression> arrayProjectedColumnRefs = + new ArrayList<ProjectedColumnExpression>(); List<Expression> arrayKVFuncs = new ArrayList<Expression>(); List<Expression> arrayOldFuncs = new ArrayList<Expression>(); Map<Expression, Integer> arrayExpressionCounts = new HashMap<>(); List<AliasedNode> aliasedNodes = statement.getSelect(); // Setup projected columns in Scan - SelectClauseVisitor selectVisitor = new SelectClauseVisitor(context, groupBy, arrayKVRefs, arrayKVFuncs, arrayExpressionCounts, arrayProjectedColumnRefs, arrayOldFuncs, statement); + SelectClauseVisitor selectVisitor = + new SelectClauseVisitor(context, groupBy, arrayKVRefs, arrayKVFuncs, + arrayExpressionCounts, arrayProjectedColumnRefs, arrayOldFuncs, statement); List<ExpressionProjector> projectedColumns = new ArrayList<ExpressionProjector>(); ColumnResolver resolver = context.getResolver(); TableRef tableRef = context.getCurrentTable(); @@ -361,7 +366,8 @@ public class ProjectionCompiler { boolean isWildcard = false; Scan scan = context.getScan(); int index = 0; - List<Expression> projectedExpressions = Lists.newArrayListWithExpectedSize(aliasedNodes.size()); + List<Expression> projectedExpressions = + Lists.newArrayListWithExpectedSize(aliasedNodes.size()); List<byte[]> projectedFamilies = Lists.newArrayListWithExpectedSize(aliasedNodes.size()); for (AliasedNode aliasedNode : aliasedNodes) { ParseNode node = aliasedNode.getNode(); @@ -371,13 +377,18 @@ public class ProjectionCompiler { ExpressionCompiler.throwNonAggExpressionInAggException(node.toString()); } if (tableRef == TableRef.EMPTY_TABLE_REF) { - throw new SQLExceptionInfo.Builder(SQLExceptionCode.NO_TABLE_SPECIFIED_FOR_WILDCARD_SELECT).build().buildException(); + throw new SQLExceptionInfo.Builder( + SQLExceptionCode.NO_TABLE_SPECIFIED_FOR_WILDCARD_SELECT).build() + .buildException(); } isWildcard = true; - if (tableRef.getTable().getType() == PTableType.INDEX && ((WildcardParseNode)node).isRewrite()) { - projectAllIndexColumns(context, tableRef, resolveColumn, projectedExpressions, projectedColumns, targetColumns); + if (tableRef.getTable().getType() == PTableType.INDEX + && ((WildcardParseNode) node).isRewrite()) { + projectAllIndexColumns(context, tableRef, resolveColumn, projectedExpressions, + projectedColumns, targetColumns); } else { - projectAllTableColumns(context, tableRef, resolveColumn, projectedExpressions, projectedColumns, targetColumns); + projectAllTableColumns(context, tableRef, resolveColumn, projectedExpressions, + projectedColumns, targetColumns); } } else if (node instanceof TableWildcardParseNode) { TableName tName = ((TableWildcardParseNode) node).getTableName(); @@ -385,14 +396,19 @@ public class ProjectionCompiler { if (tRef.equals(tableRef)) { isWildcard = true; } - if (tRef.getTable().getType() == PTableType.INDEX && ((TableWildcardParseNode)node).isRewrite()) { - projectAllIndexColumns(context, tRef, true, projectedExpressions, projectedColumns, targetColumns); + if (tRef.getTable().getType() == PTableType.INDEX + && ((TableWildcardParseNode) node).isRewrite()) { + projectAllIndexColumns(context, tRef, true, projectedExpressions, + projectedColumns, targetColumns); } else { - projectAllTableColumns(context, tRef, true, projectedExpressions, projectedColumns, targetColumns); - } - } else if (node instanceof FamilyWildcardParseNode) { + projectAllTableColumns(context, tRef, true, projectedExpressions, + projectedColumns, targetColumns); + } + } else if (node instanceof FamilyWildcardParseNode) { if (tableRef == TableRef.EMPTY_TABLE_REF) { - throw new SQLExceptionInfo.Builder(SQLExceptionCode.NO_TABLE_SPECIFIED_FOR_WILDCARD_SELECT).build().buildException(); + throw new SQLExceptionInfo.Builder( + SQLExceptionCode.NO_TABLE_SPECIFIED_FOR_WILDCARD_SELECT).build() + .buildException(); } // Project everything for SELECT cf.* String cfName = ((FamilyWildcardParseNode) node).getName(); @@ -403,27 +419,42 @@ public class ProjectionCompiler { // around this, as this code depends on this function being the last place where // columns are projected (which is currently true, but could change). projectedFamilies.add(Bytes.toBytes(cfName)); - if (tableRef.getTable().getType() == PTableType.INDEX && ((FamilyWildcardParseNode)node).isRewrite()) { - projectIndexColumnFamily(context, cfName, tableRef, resolveColumn, projectedExpressions, projectedColumns); + if (tableRef.getTable().getType() == PTableType.INDEX + && ((FamilyWildcardParseNode) node).isRewrite()) { + projectIndexColumnFamily(context, cfName, tableRef, resolveColumn, + projectedExpressions, projectedColumns); } else { - projectTableColumnFamily(context, cfName, tableRef, resolveColumn, projectedExpressions, projectedColumns); + projectTableColumnFamily(context, cfName, tableRef, resolveColumn, + projectedExpressions, projectedColumns); } } else { Expression expression = node.accept(selectVisitor); projectedExpressions.add(expression); expression = coerceIfNecessary(index, targetColumns, expression); if (node instanceof BindParseNode) { - context.getBindManager().addParamMetaData((BindParseNode)node, expression); + context.getBindManager().addParamMetaData((BindParseNode) node, expression); } if (!node.isStateless()) { if (!selectVisitor.isAggregate() && statement.isAggregate()) { - ExpressionCompiler.throwNonAggExpressionInAggException(expression.toString()); + ExpressionCompiler + .throwNonAggExpressionInAggException(expression.toString()); } } - String columnAlias = aliasedNode.getAlias() != null ? aliasedNode.getAlias() : SchemaUtil.normalizeIdentifier(aliasedNode.getNode().getAlias()); - boolean isCaseSensitive = aliasedNode.getAlias() != null ? aliasedNode.isCaseSensitve() : (columnAlias != null ? SchemaUtil.isCaseSensitive(aliasedNode.getNode().getAlias()) : selectVisitor.isCaseSensitive); + String columnAlias = + aliasedNode.getAlias() != null ? aliasedNode.getAlias() + : SchemaUtil.normalizeIdentifier(aliasedNode.getNode().getAlias()); + boolean isCaseSensitive = + aliasedNode.getAlias() != null ? aliasedNode.isCaseSensitve() + : (columnAlias != null + ? SchemaUtil + .isCaseSensitive(aliasedNode.getNode().getAlias()) + : selectVisitor.isCaseSensitive); String name = columnAlias == null ? expression.toString() : columnAlias; - projectedColumns.add(new ExpressionProjector(name, tableRef.getTableAlias() == null ? (table.getName() == null ? "" : table.getName().getString()) : tableRef.getTableAlias(), expression, isCaseSensitive)); + projectedColumns.add(new ExpressionProjector(name, + tableRef.getTableAlias() == null + ? (table.getName() == null ? "" : table.getName().getString()) + : tableRef.getTableAlias(), + expression, isCaseSensitive)); } selectVisitor.reset(); @@ -455,50 +486,67 @@ public class ProjectionCompiler { KeyValueSchema arrayIndexesSchema = builder.build(); Map<Expression, Expression> replacementMap = new HashMap<>(); - for(int i = 0; i < arrayOldFuncs.size(); i++){ - Expression function =arrayKVFuncs.get(i); - replacementMap.put(arrayOldFuncs.get(i), new ArrayIndexExpression(i, function.getDataType(), arrayIndexesBitSet, arrayIndexesSchema)); + for (int i = 0; i < arrayOldFuncs.size(); i++) { + Expression function = arrayKVFuncs.get(i); + replacementMap.put(arrayOldFuncs.get(i), new ArrayIndexExpression(i, + function.getDataType(), arrayIndexesBitSet, arrayIndexesSchema)); } - ReplaceArrayFunctionExpressionVisitor visitor = new ReplaceArrayFunctionExpressionVisitor(replacementMap); + ReplaceArrayFunctionExpressionVisitor visitor = + new ReplaceArrayFunctionExpressionVisitor(replacementMap); for (int i = 0; i < projectedColumns.size(); i++) { ExpressionProjector projector = projectedColumns.get(i); - projectedColumns.set(i, new ExpressionProjector(projector.getName(), tableRef.getTableAlias() == null ? (table.getName() == null ? "" : table.getName().getString()) : tableRef.getTableAlias(), projector.getExpression().accept(visitor), projector.isCaseSensitive())); + projectedColumns.set(i, new ExpressionProjector(projector.getName(), + tableRef.getTableAlias() == null + ? (table.getName() == null ? "" : table.getName().getString()) + : tableRef.getTableAlias(), + projector.getExpression().accept(visitor), projector.isCaseSensitive())); + } + } + boolean isProjectEmptyKeyValue = false; + if (isWildcard) { + projectAllColumnFamilies(table, scan); + } else { + isProjectEmptyKeyValue = + where == null || LiteralExpression.isTrue(where) + || where.requiresFinalEvaluation(); + for (byte[] family : projectedFamilies) { + projectColumnFamily(table, scan, family); } } - // TODO make estimatedByteSize more accurate by counting the joined columns. int estimatedKeySize = table.getRowKeySchema().getEstimatedValueLength(); int estimatedByteSize = 0; - for (Map.Entry<byte[],NavigableSet<byte[]>> entry : scan.getFamilyMap().entrySet()) { + for (Map.Entry<byte[], NavigableSet<byte[]>> entry : scan.getFamilyMap().entrySet()) { PColumnFamily family = table.getColumnFamily(entry.getKey()); if (entry.getValue() == null) { for (PColumn column : family.getColumns()) { Integer maxLength = column.getMaxLength(); - int byteSize = column.getDataType().isFixedWidth() ? maxLength == null ? column.getDataType().getByteSize() : maxLength : RowKeySchema.ESTIMATED_VARIABLE_LENGTH_SIZE; + int byteSize = + column.getDataType().isFixedWidth() + ? maxLength == null ? column.getDataType().getByteSize() + : maxLength + : RowKeySchema.ESTIMATED_VARIABLE_LENGTH_SIZE; estimatedByteSize += SizedUtil.KEY_VALUE_SIZE + estimatedKeySize + byteSize; } } else { for (byte[] cq : entry.getValue()) { - //if (!Bytes.equals(cq, ByteUtil.EMPTY_BYTE_ARRAY) || cq.length > 0) { - PColumn column = family.getPColumnForColumnQualifier(cq); - Integer maxLength = column.getMaxLength(); - int byteSize = column.getDataType().isFixedWidth() ? maxLength == null ? column.getDataType().getByteSize() : maxLength : RowKeySchema.ESTIMATED_VARIABLE_LENGTH_SIZE; - estimatedByteSize += SizedUtil.KEY_VALUE_SIZE + estimatedKeySize + byteSize; - } - //} + PColumn column = family.getPColumnForColumnQualifier(cq); + Integer maxLength = column.getMaxLength(); + int byteSize = + column.getDataType().isFixedWidth() + ? maxLength == null ? column.getDataType().getByteSize() + : maxLength + : RowKeySchema.ESTIMATED_VARIABLE_LENGTH_SIZE; + estimatedByteSize += SizedUtil.KEY_VALUE_SIZE + estimatedKeySize + byteSize; + } } } - boolean isProjectEmptyKeyValue = false; - if (isWildcard) { - projectAllColumnFamilies(table, scan); - } else { - isProjectEmptyKeyValue = where == null || LiteralExpression.isTrue(where) || where.requiresFinalEvaluation(); - for (byte[] family : projectedFamilies) { - projectColumnFamily(table, scan, family); - } + if (estimatedByteSize == 0) { + estimatedByteSize = estimatedKeySize; } - return new RowProjector(projectedColumns, estimatedByteSize, isProjectEmptyKeyValue, resolver.hasUDFs(), isWildcard); + return new RowProjector(projectedColumns, estimatedByteSize, isProjectEmptyKeyValue, + resolver.hasUDFs(), isWildcard); } private static void projectAllColumnFamilies(PTable table, Scan scan) {
