Repository: phoenix Updated Branches: refs/heads/calcite 36ee23ab7 -> a78fdf5b6
1. Retrieve PhoenixTable stats the same way as PhoenixTableScan stats; 2. Handle no stats cases; 3. Revert temp changes for PHOENIX-2712 Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/a78fdf5b Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/a78fdf5b Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/a78fdf5b Branch: refs/heads/calcite Commit: a78fdf5b6db7b8cf1016811dc4872b5e70c4b748 Parents: 36ee23a Author: maryannxue <maryann....@gmail.com> Authored: Fri Feb 26 09:14:05 2016 -0500 Committer: maryannxue <maryann....@gmail.com> Committed: Fri Feb 26 09:14:05 2016 -0500 ---------------------------------------------------------------------- .../apache/phoenix/calcite/PhoenixTable.java | 51 ++++++++------------ .../phoenix/calcite/rel/PhoenixTableScan.java | 49 ++++++++++--------- .../coprocessor/MetaDataEndpointImpl.java | 18 ++++--- .../apache/phoenix/schema/MetaDataClient.java | 3 -- 4 files changed, 55 insertions(+), 66 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/phoenix/blob/a78fdf5b/phoenix-core/src/main/java/org/apache/phoenix/calcite/PhoenixTable.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/calcite/PhoenixTable.java b/phoenix-core/src/main/java/org/apache/phoenix/calcite/PhoenixTable.java index 101ee5d..47ac80a 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/calcite/PhoenixTable.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/calcite/PhoenixTable.java @@ -20,22 +20,27 @@ import org.apache.calcite.schema.TranslatableTable; import org.apache.calcite.schema.impl.AbstractTable; import org.apache.calcite.sql.type.SqlTypeName; import org.apache.calcite.util.ImmutableBitSet; +import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.util.Pair; import org.apache.phoenix.calcite.rel.PhoenixTableScan; +import org.apache.phoenix.compile.ColumnResolver; +import org.apache.phoenix.compile.FromCompiler; +import org.apache.phoenix.compile.SequenceManager; +import org.apache.phoenix.compile.StatementContext; +import org.apache.phoenix.iterate.BaseResultIterators; import org.apache.phoenix.jdbc.PhoenixConnection; +import org.apache.phoenix.jdbc.PhoenixStatement; import org.apache.phoenix.query.QueryServices; import org.apache.phoenix.query.QueryServicesOptions; import org.apache.phoenix.schema.PColumn; -import org.apache.phoenix.schema.PColumnFamily; import org.apache.phoenix.schema.PTable; -import org.apache.phoenix.schema.RowKeySchema; import org.apache.phoenix.schema.SortOrder; -import org.apache.phoenix.schema.stats.GuidePostsInfo; +import org.apache.phoenix.schema.TableRef; import org.apache.phoenix.schema.stats.StatisticsUtil; import org.apache.phoenix.schema.types.PDataType; import org.apache.phoenix.util.SchemaUtil; -import org.apache.phoenix.util.SizedUtil; - import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; @@ -89,12 +94,18 @@ public class PhoenixTable extends AbstractTable implements TranslatableTable { } this.pkBitSet = ImmutableBitSet.of(pkPositions); this.collation = RelCollationTraitDef.INSTANCE.canonize(RelCollations.of(fieldCollations)); - byte[] emptyCf = SchemaUtil.getEmptyColumnFamily(pTable); - GuidePostsInfo info = pTable.getTableStats().getGuidePosts().get(emptyCf); long rowCount = 0; long byteCount = 0; try { - if (info == null) { + PhoenixStatement stmt = new PhoenixStatement(pc); + TableRef tableRef = new TableRef(CalciteUtils.createTempAlias(), pTable, HConstants.LATEST_TIMESTAMP, false); + ColumnResolver resolver = FromCompiler.getResolver(tableRef); + StatementContext context = new StatementContext(stmt, resolver, new Scan(), new SequenceManager(stmt)); + Pair<Long, Long> estimatedCount = BaseResultIterators.getEstimatedCount(context, pTable); + if (estimatedCount != null) { + rowCount = estimatedCount.getFirst(); + byteCount = estimatedCount.getSecond(); + } else { // TODO The props might not be the same as server props. int guidepostPerRegion = pc.getQueryServices().getProps().getInt( QueryServices.STATS_GUIDEPOST_PER_REGION_ATTRIB, @@ -109,30 +120,8 @@ public class PhoenixTable extends AbstractTable implements TranslatableTable { } byteCount = StatisticsUtil.getGuidePostDepth( guidepostPerRegion, guidepostWidth, desc) / 2; - long keySize = pTable.getRowKeySchema().getEstimatedByteSize(); - long rowSize = 0; - for (PColumnFamily cf : pTable.getColumnFamilies()) { - for (PColumn column : cf.getColumns()) { - Integer maxLength = column.getMaxLength(); - int byteSize = column.getDataType().isFixedWidth() ? - maxLength == null ? - column.getDataType().getByteSize() - : maxLength - : RowKeySchema.ESTIMATED_VARIABLE_LENGTH_SIZE; - rowSize += SizedUtil.KEY_VALUE_SIZE + keySize + byteSize; - } - } - if (rowSize == 0) { - rowSize = keySize; - } + long rowSize = SchemaUtil.estimateRowSize(pTable); rowCount = byteCount / rowSize; - } else { - for (long b : info.getByteCounts()) { - byteCount += b; - } - for (long r : info.getRowCounts()) { - rowCount += r; - } } } catch (SQLException | IOException e) { throw new RuntimeException(e); http://git-wip-us.apache.org/repos/asf/phoenix/blob/a78fdf5b/phoenix-core/src/main/java/org/apache/phoenix/calcite/rel/PhoenixTableScan.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/calcite/rel/PhoenixTableScan.java b/phoenix-core/src/main/java/org/apache/phoenix/calcite/rel/PhoenixTableScan.java index dfdb507..691b5ac 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/calcite/rel/PhoenixTableScan.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/calcite/rel/PhoenixTableScan.java @@ -65,7 +65,7 @@ public class PhoenixTableScan extends TableScan implements PhoenixRel { public final ScanOrder scanOrder; public final ScanRanges scanRanges; - protected final long estimatedBytes; + protected final Long estimatedBytes; protected final float rowCountFactor; public static PhoenixTableScan create(RelOptCluster cluster, final RelOptTable table) { @@ -96,15 +96,16 @@ public class PhoenixTableScan extends TableScan implements PhoenixRel { this.scanOrder = scanOrder; final PhoenixTable phoenixTable = table.unwrap(PhoenixTable.class); this.rowCountFactor = phoenixTable.pc.getQueryServices() - .getProps().getFloat(PhoenixRel.ROW_COUNT_FACTOR, 1f); - - ScanRanges scanRanges = null; - long estimatedSize = 0; - if (filter != null) { - try { - // TODO simplify this code - PTable pTable = phoenixTable.getTable(); - TableRef tableRef = new TableRef(CalciteUtils.createTempAlias(), pTable, HConstants.LATEST_TIMESTAMP, false); + .getProps().getFloat(PhoenixRel.ROW_COUNT_FACTOR, 1f); + try { + // TODO simplify this code + PTable pTable = phoenixTable.getTable(); + TableRef tableRef = new TableRef(CalciteUtils.createTempAlias(), pTable, HConstants.LATEST_TIMESTAMP, false); + SelectStatement select = SelectStatement.SELECT_ONE; + PhoenixStatement stmt = new PhoenixStatement(phoenixTable.pc); + ColumnResolver resolver = FromCompiler.getResolver(tableRef); + StatementContext context = new StatementContext(stmt, resolver, new Scan(), new SequenceManager(stmt)); + if (filter != null) { // We use a implementor with a special implementation for field access // here, which translates RexFieldAccess into a LiteralExpression // with a sample value. This will achieve 3 goals at a time: @@ -124,21 +125,15 @@ public class PhoenixTableScan extends TableScan implements PhoenixRel { } }; tmpImplementor.setTableRef(tableRef); - SelectStatement select = SelectStatement.SELECT_ONE; - PhoenixStatement stmt = new PhoenixStatement(phoenixTable.pc); - ColumnResolver resolver = FromCompiler.getResolver(tableRef); - StatementContext context = new StatementContext(stmt, resolver, new Scan(), new SequenceManager(stmt)); Expression filterExpr = CalciteUtils.toExpression(filter, tmpImplementor); filterExpr = WhereOptimizer.pushKeyExpressionsToScan(context, select, filterExpr); WhereCompiler.setScanFilter(context, select, filterExpr, true, false); - scanRanges = context.getScanRanges(); - estimatedSize = BaseResultIterators.getEstimatedCount(context, tableRef.getTable()).getSecond(); - } catch (SQLException e) { - throw new RuntimeException(e); - } - } - this.scanRanges = scanRanges; - this.estimatedBytes = estimatedSize; + } + this.scanRanges = context.getScanRanges(); + this.estimatedBytes = BaseResultIterators.getEstimatedCount(context, pTable).getSecond(); + } catch (SQLException e) { + throw new RuntimeException(e); + } } private static ScanOrder getDefaultScanOrder(PhoenixTable table) { @@ -182,10 +177,16 @@ public class PhoenixTableScan extends TableScan implements PhoenixRel { public RelOptCost computeSelfCost(RelOptPlanner planner, RelMetadataQuery mq) { double byteCount; PhoenixTable phoenixTable = table.unwrap(PhoenixTable.class); - if (scanRanges != null) { + if (estimatedBytes != null) { byteCount = estimatedBytes; } else { - byteCount = phoenixTable.byteCount; + // If stats are not available, we estimate based on selectivity. + int pkCount = scanRanges.getBoundPkColumnCount(); + if (pkCount > 0) { + byteCount = phoenixTable.byteCount * Math.pow(mq.getSelectivity(this, filter), pkCount); + } else { + byteCount = phoenixTable.byteCount; + } } byteCount *= rowCountFactor; if (scanOrder != ScanOrder.NONE) { http://git-wip-us.apache.org/repos/asf/phoenix/blob/a78fdf5b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java index 7b19515..008e1d9 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java @@ -869,14 +869,16 @@ public class MetaDataEndpointImpl extends MetaDataProtocol implements Coprocesso schemaName.getString(), tableName.getString())) : physicalTables.get(0); PTableStats stats = PTableStats.EMPTY_STATS; HTableInterface statsHTable = null; - try { - statsHTable = ServerUtil.getHTableForCoprocessorScan(env, PhoenixDatabaseMetaData.SYSTEM_STATS_NAME_BYTES); - stats = StatisticsUtil.readStatistics(statsHTable, physicalTableName.getBytes(), clientTimeStamp); - timeStamp = Math.max(timeStamp, stats.getTimestamp()); - } catch (org.apache.hadoop.hbase.TableNotFoundException e) { - logger.warn(PhoenixDatabaseMetaData.SYSTEM_STATS_NAME + " not online yet?"); - } finally { - if (statsHTable != null) statsHTable.close(); + if (tenantId == null) { + try { + statsHTable = ServerUtil.getHTableForCoprocessorScan(env, PhoenixDatabaseMetaData.SYSTEM_STATS_NAME_BYTES); + stats = StatisticsUtil.readStatistics(statsHTable, physicalTableName.getBytes(), clientTimeStamp); + timeStamp = Math.max(timeStamp, stats.getTimestamp()); + } catch (org.apache.hadoop.hbase.TableNotFoundException e) { + logger.warn(PhoenixDatabaseMetaData.SYSTEM_STATS_NAME + " not online yet?"); + } finally { + if (statsHTable != null) statsHTable.close(); + } } return PTableImpl.makePTable(tenantId, schemaName, tableName, tableType, indexState, timeStamp, tableSeqNum, pkName, saltBucketNum, columns, tableType == INDEX ? schemaName : null, http://git-wip-us.apache.org/repos/asf/phoenix/blob/a78fdf5b/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java index 7e28933..7f3f850 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java @@ -963,9 +963,6 @@ public class MetaDataClient { * This supports scenarios in which a major compaction was manually initiated and the * client wants the modified stats to be reflected immediately. */ - connection.getQueryServices().clearTableFromCache(logicalTable.getTenantId() == null ? ByteUtil.EMPTY_BYTE_ARRAY : logicalTable.getTenantId().getBytes(), - Bytes.toBytes(SchemaUtil.getSchemaNameFromFullName(logicalTable.getName().toString())), - Bytes.toBytes(SchemaUtil.getTableNameFromFullName(logicalTable.getName().toString())), clientTimeStamp); connection.getQueryServices().clearTableFromCache(tenantIdBytes, Bytes.toBytes(SchemaUtil.getSchemaNameFromFullName(physicalName.getString())), Bytes.toBytes(SchemaUtil.getTableNameFromFullName(physicalName.getString())), clientTimeStamp);
