PHOENIX-3834 Provide a way to collect stats and not use it for query parallelization
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/3872648f Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/3872648f Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/3872648f Branch: refs/heads/master Commit: 3872648fe84426597936e67341b37dfc4a7ba44a Parents: a216899 Author: Samarth Jain <[email protected]> Authored: Tue May 30 02:16:27 2017 -0700 Committer: Samarth Jain <[email protected]> Committed: Tue May 30 02:16:27 2017 -0700 ---------------------------------------------------------------------- .../end2end/ExplainPlanWithStatsEnabledIT.java | 44 +++++++- .../phoenix/end2end/ParallelStatsEnabledIT.java | 1 + .../apache/phoenix/compile/FromCompiler.java | 2 +- .../apache/phoenix/compile/JoinCompiler.java | 2 +- .../compile/TupleProjectionCompiler.java | 4 +- .../apache/phoenix/compile/UnionCompiler.java | 2 +- .../coprocessor/MetaDataEndpointImpl.java | 10 +- .../phoenix/coprocessor/MetaDataProtocol.java | 2 +- .../coprocessor/generated/PTableProtos.java | 107 +++++++++++++++++-- .../phoenix/iterate/BaseResultIterators.java | 11 +- .../phoenix/jdbc/PhoenixDatabaseMetaData.java | 2 + .../query/ConnectionQueryServicesImpl.java | 10 +- .../apache/phoenix/query/QueryConstants.java | 2 + .../org/apache/phoenix/query/QueryServices.java | 5 + .../phoenix/query/QueryServicesOptions.java | 14 ++- .../apache/phoenix/schema/DelegateTable.java | 5 + .../apache/phoenix/schema/MetaDataClient.java | 48 +++++++-- .../java/org/apache/phoenix/schema/PTable.java | 1 + .../org/apache/phoenix/schema/PTableImpl.java | 57 ++++++---- .../apache/phoenix/schema/TableProperty.java | 18 ++++ .../stats/StatisticsCollectorFactory.java | 13 ++- .../org/apache/phoenix/util/UpgradeUtil.java | 2 +- .../phoenix/execute/CorrelatePlanTest.java | 2 +- .../execute/LiteralResultIteratorPlanTest.java | 2 +- .../phoenix/query/QueryServicesTestImpl.java | 1 - phoenix-protocol/src/main/PTable.proto | 1 + 26 files changed, 304 insertions(+), 64 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/phoenix/blob/3872648f/phoenix-core/src/it/java/org/apache/phoenix/end2end/ExplainPlanWithStatsEnabledIT.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/ExplainPlanWithStatsEnabledIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/ExplainPlanWithStatsEnabledIT.java index 2785f10..d5fcf60 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/ExplainPlanWithStatsEnabledIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/ExplainPlanWithStatsEnabledIT.java @@ -16,17 +16,22 @@ * limitations under the License. */ package org.apache.phoenix.end2end; + import static org.junit.Assert.assertEquals; import java.sql.Connection; import java.sql.DriverManager; import java.sql.PreparedStatement; import java.sql.ResultSet; +import java.sql.SQLException; import java.util.List; import java.util.Map; import org.apache.hadoop.hbase.util.Pair; +import org.apache.phoenix.jdbc.PhoenixConnection; import org.apache.phoenix.query.QueryServices; +import org.apache.phoenix.schema.PTableKey; +import org.apache.phoenix.schema.TableNotFoundException; import org.apache.phoenix.util.PhoenixRuntime; import org.apache.phoenix.util.ReadOnlyProps; import org.junit.BeforeClass; @@ -208,7 +213,7 @@ public class ExplainPlanWithStatsEnabledIT extends ParallelStatsEnabledIT { assertEquals((Long) 0l, info.getFirst()); } } - + @Test public void testBytesRowsForSelectExecutedSerially() throws Exception { String sql = "SELECT * FROM " + tableA + " LIMIT 2"; @@ -233,10 +238,43 @@ public class ExplainPlanWithStatsEnabledIT extends ParallelStatsEnabledIT { } ResultSet rs = statement.executeQuery(explainSql); rs.next(); - estimatedBytes = (Long) rs.getObject(PhoenixRuntime.EXPLAIN_PLAN_ESTIMATED_BYTES_READ_COLUMN); - estimatedRows = (Long) rs.getObject(PhoenixRuntime.EXPLAIN_PLAN_ESTIMATED_ROWS_READ_COLUMN); + estimatedBytes = + (Long) rs.getObject(PhoenixRuntime.EXPLAIN_PLAN_ESTIMATED_BYTES_READ_COLUMN); + estimatedRows = + (Long) rs.getObject(PhoenixRuntime.EXPLAIN_PLAN_ESTIMATED_ROWS_READ_COLUMN); } return new Pair<>(estimatedRows, estimatedBytes); } + @Test + public void testSettingUseStatsForQueryPlanProperty() throws Exception { + try (Connection conn = DriverManager.getConnection(getUrl())) { + String table = generateUniqueName(); + String ddl = + "CREATE TABLE " + table + + " (PK1 INTEGER NOT NULL PRIMARY KEY, KV1 VARCHAR) USE_STATS_FOR_QUERY_PLAN = false"; + conn.createStatement().execute(ddl); + assertUseStatsForQueryFlag(table, conn.unwrap(PhoenixConnection.class), false); + ddl = "ALTER TABLE " + table + " SET USE_STATS_FOR_QUERY_PLAN = true"; + conn.createStatement().execute(ddl); + assertUseStatsForQueryFlag(table, conn.unwrap(PhoenixConnection.class), true); + table = generateUniqueName(); + ddl = "CREATE TABLE " + table + " (PK1 INTEGER NOT NULL PRIMARY KEY, KV1 VARCHAR)"; + conn.createStatement().execute(ddl); + assertUseStatsForQueryFlag(table, conn.unwrap(PhoenixConnection.class), true); + } + } + + private static void assertUseStatsForQueryFlag(String tableName, PhoenixConnection conn, + boolean flag) throws TableNotFoundException, SQLException { + assertEquals(flag, conn.unwrap(PhoenixConnection.class).getMetaDataCache() + .getTableRef(new PTableKey(null, tableName)).getTable().useStatsForQueryPlan()); + String query = + "SELECT USE_STATS_FOR_QUERY_PLAN FROM SYSTEM.CATALOG WHERE TABLE_NAME = ? AND COLUMN_NAME IS NULL AND COLUMN_FAMILY IS NULL AND TENANT_ID IS NULL"; + PreparedStatement stmt = conn.prepareStatement(query); + stmt.setString(1, tableName); + ResultSet rs = stmt.executeQuery(); + rs.next(); + assertEquals(flag, rs.getBoolean(1)); + } } http://git-wip-us.apache.org/repos/asf/phoenix/blob/3872648f/phoenix-core/src/it/java/org/apache/phoenix/end2end/ParallelStatsEnabledIT.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/ParallelStatsEnabledIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/ParallelStatsEnabledIT.java index 312b49d..328a76f 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/ParallelStatsEnabledIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/ParallelStatsEnabledIT.java @@ -40,6 +40,7 @@ public abstract class ParallelStatsEnabledIT extends BaseTest { public static void doSetup() throws Exception { Map<String,String> props = Maps.newHashMapWithExpectedSize(1); props.put(QueryServices.STATS_GUIDEPOST_WIDTH_BYTES_ATTRIB, Long.toString(20)); + props.put(QueryServices.USE_STATS_FOR_PARALLELIZATION, Boolean.toString(true)); setUpTestDriver(new ReadOnlyProps(props.entrySet().iterator())); } } http://git-wip-us.apache.org/repos/asf/phoenix/blob/3872648f/phoenix-core/src/main/java/org/apache/phoenix/compile/FromCompiler.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/compile/FromCompiler.java b/phoenix-core/src/main/java/org/apache/phoenix/compile/FromCompiler.java index 6440769..7bd1aa7 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/compile/FromCompiler.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/FromCompiler.java @@ -807,7 +807,7 @@ public class FromCompiler { MetaDataProtocol.MIN_TABLE_TIMESTAMP, PTable.INITIAL_SEQ_NUM, null, null, columns, null, null, Collections.<PTable> emptyList(), false, Collections.<PName> emptyList(), null, null, false, false, false, null, null, null, false, false, 0, 0L, SchemaUtil - .isNamespaceMappingEnabled(PTableType.SUBQUERY, connection.getQueryServices().getProps()), null, false, ImmutableStorageScheme.ONE_CELL_PER_COLUMN, QualifierEncodingScheme.NON_ENCODED_QUALIFIERS, PTable.EncodedCQCounter.NULL_COUNTER); + .isNamespaceMappingEnabled(PTableType.SUBQUERY, connection.getQueryServices().getProps()), null, false, ImmutableStorageScheme.ONE_CELL_PER_COLUMN, QualifierEncodingScheme.NON_ENCODED_QUALIFIERS, PTable.EncodedCQCounter.NULL_COUNTER, true); String alias = subselectNode.getAlias(); TableRef tableRef = new TableRef(alias, t, MetaDataProtocol.MIN_TABLE_TIMESTAMP, false); http://git-wip-us.apache.org/repos/asf/phoenix/blob/3872648f/phoenix-core/src/main/java/org/apache/phoenix/compile/JoinCompiler.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/compile/JoinCompiler.java b/phoenix-core/src/main/java/org/apache/phoenix/compile/JoinCompiler.java index b1da739..c11ea98 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/compile/JoinCompiler.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/JoinCompiler.java @@ -1319,7 +1319,7 @@ public class JoinCompiler { left.isMultiTenant(), left.getStoreNulls(), left.getViewType(), left.getViewIndexId(), left.getIndexType(), left.rowKeyOrderOptimizable(), left.isTransactional(), left.getUpdateCacheFrequency(), left.getIndexDisableTimestamp(), left.isNamespaceMapped(), - left.getAutoPartitionSeqName(), left.isAppendOnlySchema(), ONE_CELL_PER_COLUMN, NON_ENCODED_QUALIFIERS, PTable.EncodedCQCounter.NULL_COUNTER); + left.getAutoPartitionSeqName(), left.isAppendOnlySchema(), ONE_CELL_PER_COLUMN, NON_ENCODED_QUALIFIERS, PTable.EncodedCQCounter.NULL_COUNTER, left.useStatsForQueryPlan()); } } http://git-wip-us.apache.org/repos/asf/phoenix/blob/3872648f/phoenix-core/src/main/java/org/apache/phoenix/compile/TupleProjectionCompiler.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/compile/TupleProjectionCompiler.java b/phoenix-core/src/main/java/org/apache/phoenix/compile/TupleProjectionCompiler.java index 32e9f68..d488a40 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/compile/TupleProjectionCompiler.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/TupleProjectionCompiler.java @@ -156,7 +156,7 @@ public class TupleProjectionCompiler { null, null, table.isWALDisabled(), table.isMultiTenant(), table.getStoreNulls(), table.getViewType(), table.getViewIndexId(), table.getIndexType(), table.rowKeyOrderOptimizable(), table.isTransactional(), table.getUpdateCacheFrequency(), - table.getIndexDisableTimestamp(), table.isNamespaceMapped(), table.getAutoPartitionSeqName(), table.isAppendOnlySchema(), table.getImmutableStorageScheme(), table.getEncodingScheme(), table.getEncodedCQCounter()); + table.getIndexDisableTimestamp(), table.isNamespaceMapped(), table.getAutoPartitionSeqName(), table.isAppendOnlySchema(), table.getImmutableStorageScheme(), table.getEncodingScheme(), table.getEncodedCQCounter(), table.useStatsForQueryPlan()); } public static PTable createProjectedTable(TableRef tableRef, List<ColumnRef> sourceColumnRefs, boolean retainPKColumns) throws SQLException { @@ -187,7 +187,7 @@ public class TupleProjectionCompiler { Collections.<PTable> emptyList(), table.isImmutableRows(), Collections.<PName> emptyList(), null, null, table.isWALDisabled(), table.isMultiTenant(), table.getStoreNulls(), table.getViewType(), table.getViewIndexId(), null, table.rowKeyOrderOptimizable(), table.isTransactional(), - table.getUpdateCacheFrequency(), table.getIndexDisableTimestamp(), table.isNamespaceMapped(), table.getAutoPartitionSeqName(), table.isAppendOnlySchema(), table.getImmutableStorageScheme(), table.getEncodingScheme(), cqCounter); + table.getUpdateCacheFrequency(), table.getIndexDisableTimestamp(), table.isNamespaceMapped(), table.getAutoPartitionSeqName(), table.isAppendOnlySchema(), table.getImmutableStorageScheme(), table.getEncodingScheme(), cqCounter, table.useStatsForQueryPlan()); } // For extracting column references from single select statement http://git-wip-us.apache.org/repos/asf/phoenix/blob/3872648f/phoenix-core/src/main/java/org/apache/phoenix/compile/UnionCompiler.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/compile/UnionCompiler.java b/phoenix-core/src/main/java/org/apache/phoenix/compile/UnionCompiler.java index c7f798c..d5bfef8 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/compile/UnionCompiler.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/UnionCompiler.java @@ -102,7 +102,7 @@ public class UnionCompiler { null, null, projectedColumns, null, null, null, true, null, null, null, true, true, true, null, null, null, false, false, 0, 0L, SchemaUtil.isNamespaceMappingEnabled(PTableType.SUBQUERY, - statement.getConnection().getQueryServices().getProps()), null, false, ImmutableStorageScheme.ONE_CELL_PER_COLUMN, QualifierEncodingScheme.NON_ENCODED_QUALIFIERS, PTable.EncodedCQCounter.NULL_COUNTER); + statement.getConnection().getQueryServices().getProps()), null, false, ImmutableStorageScheme.ONE_CELL_PER_COLUMN, QualifierEncodingScheme.NON_ENCODED_QUALIFIERS, PTable.EncodedCQCounter.NULL_COUNTER, true); TableRef tableRef = new TableRef(null, tempTable, 0, false); return tableRef; } http://git-wip-us.apache.org/repos/asf/phoenix/blob/3872648f/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java index d36bd7e..ef54bfc 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java @@ -71,6 +71,7 @@ import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TENANT_ID_INDEX; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TRANSACTIONAL_BYTES; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TYPE_BYTES; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.UPDATE_CACHE_FREQUENCY_BYTES; +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.USE_STATS_FOR_QUERY_PLAN_BYTES; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VIEW_CONSTANT_BYTES; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VIEW_INDEX_ID_BYTES; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VIEW_STATEMENT_BYTES; @@ -293,6 +294,7 @@ public class MetaDataEndpointImpl extends MetaDataProtocol implements Coprocesso private static final KeyValue APPEND_ONLY_SCHEMA_KV = createFirstOnRow(ByteUtil.EMPTY_BYTE_ARRAY, TABLE_FAMILY_BYTES, APPEND_ONLY_SCHEMA_BYTES); private static final KeyValue STORAGE_SCHEME_KV = createFirstOnRow(ByteUtil.EMPTY_BYTE_ARRAY, TABLE_FAMILY_BYTES, STORAGE_SCHEME_BYTES); private static final KeyValue ENCODING_SCHEME_KV = createFirstOnRow(ByteUtil.EMPTY_BYTE_ARRAY, TABLE_FAMILY_BYTES, ENCODING_SCHEME_BYTES); + private static final KeyValue USE_STATS_FOR_QUERY_PLAN_KV = createFirstOnRow(ByteUtil.EMPTY_BYTE_ARRAY, TABLE_FAMILY_BYTES, USE_STATS_FOR_QUERY_PLAN_BYTES); private static final List<KeyValue> TABLE_KV_COLUMNS = Arrays.<KeyValue>asList( EMPTY_KEYVALUE_KV, @@ -321,7 +323,8 @@ public class MetaDataEndpointImpl extends MetaDataProtocol implements Coprocesso AUTO_PARTITION_SEQ_KV, APPEND_ONLY_SCHEMA_KV, STORAGE_SCHEME_KV, - ENCODING_SCHEME_KV + ENCODING_SCHEME_KV, + USE_STATS_FOR_QUERY_PLAN_KV ); static { Collections.sort(TABLE_KV_COLUMNS, KeyValue.COMPARATOR); @@ -353,6 +356,7 @@ public class MetaDataEndpointImpl extends MetaDataProtocol implements Coprocesso private static final int APPEND_ONLY_SCHEMA_INDEX = TABLE_KV_COLUMNS.indexOf(APPEND_ONLY_SCHEMA_KV); private static final int STORAGE_SCHEME_INDEX = TABLE_KV_COLUMNS.indexOf(STORAGE_SCHEME_KV); private static final int QUALIFIER_ENCODING_SCHEME_INDEX = TABLE_KV_COLUMNS.indexOf(ENCODING_SCHEME_KV); + private static final int USE_STATS_FOR_QUERY_PLAN_INDEX = TABLE_KV_COLUMNS.indexOf(USE_STATS_FOR_QUERY_PLAN_KV); // KeyValues for Column private static final KeyValue DECIMAL_DIGITS_KV = createFirstOnRow(ByteUtil.EMPTY_BYTE_ARRAY, TABLE_FAMILY_BYTES, DECIMAL_DIGITS_BYTES); @@ -962,6 +966,8 @@ public class MetaDataEndpointImpl extends MetaDataProtocol implements Coprocesso QualifierEncodingScheme encodingScheme = encodingSchemeKv == null ? QualifierEncodingScheme.NON_ENCODED_QUALIFIERS : QualifierEncodingScheme .fromSerializedValue((byte)PTinyint.INSTANCE.toObject(encodingSchemeKv.getValueArray(), encodingSchemeKv.getValueOffset(), encodingSchemeKv.getValueLength())); + Cell useStatsForQueryPlanKv = tableKeyValues[USE_STATS_FOR_QUERY_PLAN_INDEX]; + boolean useStatsForQueryPlan = useStatsForQueryPlanKv == null ? true : Boolean.TRUE.equals(PBoolean.INSTANCE.toObject(useStatsForQueryPlanKv.getValueArray(), useStatsForQueryPlanKv.getValueOffset(), useStatsForQueryPlanKv.getValueLength())); List<PColumn> columns = Lists.newArrayListWithExpectedSize(columnCount); List<PTable> indexes = Lists.newArrayList(); @@ -1005,7 +1011,7 @@ public class MetaDataEndpointImpl extends MetaDataProtocol implements Coprocesso pkName, saltBucketNum, columns, parentSchemaName, parentTableName, indexes, isImmutableRows, physicalTables, defaultFamilyName, viewStatement, disableWAL, multiTenant, storeNulls, viewType, viewIndexId, indexType, rowKeyOrderOptimizable, transactional, updateCacheFrequency, baseColumnCount, - indexDisableTimestamp, isNamespaceMapped, autoPartitionSeq, isAppendOnlySchema, storageScheme, encodingScheme, cqCounter); + indexDisableTimestamp, isNamespaceMapped, autoPartitionSeq, isAppendOnlySchema, storageScheme, encodingScheme, cqCounter, useStatsForQueryPlan); } private boolean isQualifierCounterKV(Cell kv) { http://git-wip-us.apache.org/repos/asf/phoenix/blob/3872648f/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataProtocol.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataProtocol.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataProtocol.java index 93bba74..d01d987 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataProtocol.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataProtocol.java @@ -86,7 +86,7 @@ public abstract class MetaDataProtocol extends MetaDataService { public static final long MIN_SYSTEM_TABLE_TIMESTAMP_4_8_1 = MIN_TABLE_TIMESTAMP + 18; public static final long MIN_SYSTEM_TABLE_TIMESTAMP_4_9_0 = MIN_TABLE_TIMESTAMP + 20; public static final long MIN_SYSTEM_TABLE_TIMESTAMP_4_10_0 = MIN_TABLE_TIMESTAMP + 25; - public static final long MIN_SYSTEM_TABLE_TIMESTAMP_4_11_0 = MIN_TABLE_TIMESTAMP + 26; + public static final long MIN_SYSTEM_TABLE_TIMESTAMP_4_11_0 = MIN_TABLE_TIMESTAMP + 27; // MIN_SYSTEM_TABLE_TIMESTAMP needs to be set to the max of all the MIN_SYSTEM_TABLE_TIMESTAMP_* constants public static final long MIN_SYSTEM_TABLE_TIMESTAMP = MIN_SYSTEM_TABLE_TIMESTAMP_4_11_0; http://git-wip-us.apache.org/repos/asf/phoenix/blob/3872648f/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/generated/PTableProtos.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/generated/PTableProtos.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/generated/PTableProtos.java index b7ca46f..46d2586 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/generated/PTableProtos.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/generated/PTableProtos.java @@ -3510,6 +3510,16 @@ public final class PTableProtos { */ org.apache.phoenix.coprocessor.generated.PTableProtos.EncodedCQCounterOrBuilder getEncodedCQCountersOrBuilder( int index); + + // optional bool useStatsForQueryPlan = 37; + /** + * <code>optional bool useStatsForQueryPlan = 37;</code> + */ + boolean hasUseStatsForQueryPlan(); + /** + * <code>optional bool useStatsForQueryPlan = 37;</code> + */ + boolean getUseStatsForQueryPlan(); } /** * Protobuf type {@code PTable} @@ -3756,6 +3766,11 @@ public final class PTableProtos { encodedCQCounters_.add(input.readMessage(org.apache.phoenix.coprocessor.generated.PTableProtos.EncodedCQCounter.PARSER, extensionRegistry)); break; } + case 296: { + bitField0_ |= 0x80000000; + useStatsForQueryPlan_ = input.readBool(); + break; + } } } } catch (com.google.protobuf.InvalidProtocolBufferException e) { @@ -4503,6 +4518,22 @@ public final class PTableProtos { return encodedCQCounters_.get(index); } + // optional bool useStatsForQueryPlan = 37; + public static final int USESTATSFORQUERYPLAN_FIELD_NUMBER = 37; + private boolean useStatsForQueryPlan_; + /** + * <code>optional bool useStatsForQueryPlan = 37;</code> + */ + public boolean hasUseStatsForQueryPlan() { + return ((bitField0_ & 0x80000000) == 0x80000000); + } + /** + * <code>optional bool useStatsForQueryPlan = 37;</code> + */ + public boolean getUseStatsForQueryPlan() { + return useStatsForQueryPlan_; + } + private void initFields() { schemaNameBytes_ = com.google.protobuf.ByteString.EMPTY; tableNameBytes_ = com.google.protobuf.ByteString.EMPTY; @@ -4539,6 +4570,7 @@ public final class PTableProtos { storageScheme_ = com.google.protobuf.ByteString.EMPTY; encodingScheme_ = com.google.protobuf.ByteString.EMPTY; encodedCQCounters_ = java.util.Collections.emptyList(); + useStatsForQueryPlan_ = false; } private byte memoizedIsInitialized = -1; public final boolean isInitialized() { @@ -4711,6 +4743,9 @@ public final class PTableProtos { for (int i = 0; i < encodedCQCounters_.size(); i++) { output.writeMessage(36, encodedCQCounters_.get(i)); } + if (((bitField0_ & 0x80000000) == 0x80000000)) { + output.writeBool(37, useStatsForQueryPlan_); + } getUnknownFields().writeTo(output); } @@ -4865,6 +4900,10 @@ public final class PTableProtos { size += com.google.protobuf.CodedOutputStream .computeMessageSize(36, encodedCQCounters_.get(i)); } + if (((bitField0_ & 0x80000000) == 0x80000000)) { + size += com.google.protobuf.CodedOutputStream + .computeBoolSize(37, useStatsForQueryPlan_); + } size += getUnknownFields().getSerializedSize(); memoizedSerializedSize = size; return size; @@ -5051,6 +5090,11 @@ public final class PTableProtos { } result = result && getEncodedCQCountersList() .equals(other.getEncodedCQCountersList()); + result = result && (hasUseStatsForQueryPlan() == other.hasUseStatsForQueryPlan()); + if (hasUseStatsForQueryPlan()) { + result = result && (getUseStatsForQueryPlan() + == other.getUseStatsForQueryPlan()); + } result = result && getUnknownFields().equals(other.getUnknownFields()); return result; @@ -5204,6 +5248,10 @@ public final class PTableProtos { hash = (37 * hash) + ENCODEDCQCOUNTERS_FIELD_NUMBER; hash = (53 * hash) + getEncodedCQCountersList().hashCode(); } + if (hasUseStatsForQueryPlan()) { + hash = (37 * hash) + USESTATSFORQUERYPLAN_FIELD_NUMBER; + hash = (53 * hash) + hashBoolean(getUseStatsForQueryPlan()); + } hash = (29 * hash) + getUnknownFields().hashCode(); memoizedHashCode = hash; return hash; @@ -5398,6 +5446,8 @@ public final class PTableProtos { } else { encodedCQCountersBuilder_.clear(); } + useStatsForQueryPlan_ = false; + bitField1_ = (bitField1_ & ~0x00000008); return this; } @@ -5583,6 +5633,10 @@ public final class PTableProtos { } else { result.encodedCQCounters_ = encodedCQCountersBuilder_.build(); } + if (((from_bitField1_ & 0x00000008) == 0x00000008)) { + to_bitField0_ |= 0x80000000; + } + result.useStatsForQueryPlan_ = useStatsForQueryPlan_; result.bitField0_ = to_bitField0_; onBuilt(); return result; @@ -5784,6 +5838,9 @@ public final class PTableProtos { } } } + if (other.hasUseStatsForQueryPlan()) { + setUseStatsForQueryPlan(other.getUseStatsForQueryPlan()); + } this.mergeUnknownFields(other.getUnknownFields()); return this; } @@ -7830,6 +7887,39 @@ public final class PTableProtos { return encodedCQCountersBuilder_; } + // optional bool useStatsForQueryPlan = 37; + private boolean useStatsForQueryPlan_ ; + /** + * <code>optional bool useStatsForQueryPlan = 37;</code> + */ + public boolean hasUseStatsForQueryPlan() { + return ((bitField1_ & 0x00000008) == 0x00000008); + } + /** + * <code>optional bool useStatsForQueryPlan = 37;</code> + */ + public boolean getUseStatsForQueryPlan() { + return useStatsForQueryPlan_; + } + /** + * <code>optional bool useStatsForQueryPlan = 37;</code> + */ + public Builder setUseStatsForQueryPlan(boolean value) { + bitField1_ |= 0x00000008; + useStatsForQueryPlan_ = value; + onChanged(); + return this; + } + /** + * <code>optional bool useStatsForQueryPlan = 37;</code> + */ + public Builder clearUseStatsForQueryPlan() { + bitField1_ = (bitField1_ & ~0x00000008); + useStatsForQueryPlan_ = false; + onChanged(); + return this; + } + // @@protoc_insertion_point(builder_scope:PTable) } @@ -8497,7 +8587,7 @@ public final class PTableProtos { "\"\232\001\n\013PTableStats\022\013\n\003key\030\001 \002(\014\022\016\n\006values\030", "\002 \003(\014\022\033\n\023guidePostsByteCount\030\003 \001(\003\022\025\n\rke" + "yBytesCount\030\004 \001(\003\022\027\n\017guidePostsCount\030\005 \001" + - "(\005\022!\n\013pGuidePosts\030\006 \001(\0132\014.PGuidePosts\"\354\006" + + "(\005\022!\n\013pGuidePosts\030\006 \001(\0132\014.PGuidePosts\"\212\007" + "\n\006PTable\022\027\n\017schemaNameBytes\030\001 \002(\014\022\026\n\016tab" + "leNameBytes\030\002 \002(\014\022\036\n\ttableType\030\003 \002(\0162\013.P" + "TableType\022\022\n\nindexState\030\004 \001(\t\022\026\n\016sequenc" + @@ -8519,12 +8609,13 @@ public final class PTableProtos { "Name\030\037 \001(\t\022\032\n\022isAppendOnlySchema\030 \001(\010\022\027" + "\n\017parentNameBytes\030! \001(\014\022\025\n\rstorageScheme" + "\030\" \001(\014\022\026\n\016encodingScheme\030# \001(\014\022,\n\021encode" + - "dCQCounters\030$ \003(\0132\021.EncodedCQCounter\"6\n\020" + - "EncodedCQCounter\022\021\n\tcolFamily\030\001 \002(\t\022\017\n\007c" + - "ounter\030\002 \002(\005*A\n\nPTableType\022\n\n\006SYSTEM\020\000\022\010" + - "\n\004USER\020\001\022\010\n\004VIEW\020\002\022\t\n\005INDEX\020\003\022\010\n\004JOIN\020\004B" + - "@\n(org.apache.phoenix.coprocessor.genera" + - "tedB\014PTableProtosH\001\210\001\001\240\001\001" + "dCQCounters\030$ \003(\0132\021.EncodedCQCounter\022\034\n\024" + + "useStatsForQueryPlan\030% \001(\010\"6\n\020EncodedCQC" + + "ounter\022\021\n\tcolFamily\030\001 \002(\t\022\017\n\007counter\030\002 \002" + + "(\005*A\n\nPTableType\022\n\n\006SYSTEM\020\000\022\010\n\004USER\020\001\022\010" + + "\n\004VIEW\020\002\022\t\n\005INDEX\020\003\022\010\n\004JOIN\020\004B@\n(org.apa" + + "che.phoenix.coprocessor.generatedB\014PTabl", + "eProtosH\001\210\001\001\240\001\001" }; com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner assigner = new com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner() { @@ -8548,7 +8639,7 @@ public final class PTableProtos { internal_static_PTable_fieldAccessorTable = new com.google.protobuf.GeneratedMessage.FieldAccessorTable( internal_static_PTable_descriptor, - new java.lang.String[] { "SchemaNameBytes", "TableNameBytes", "TableType", "IndexState", "SequenceNumber", "TimeStamp", "PkNameBytes", "BucketNum", "Columns", "Indexes", "IsImmutableRows", "DataTableNameBytes", "DefaultFamilyName", "DisableWAL", "MultiTenant", "ViewType", "ViewStatement", "PhysicalNames", "TenantId", "ViewIndexId", "IndexType", "StatsTimeStamp", "StoreNulls", "BaseColumnCount", "RowKeyOrderOptimizable", "Transactional", "UpdateCacheFrequency", "IndexDisableTimestamp", "IsNamespaceMapped", "AutoParititonSeqName", "IsAppendOnlySchema", "ParentNameBytes", "StorageScheme", "EncodingScheme", "EncodedCQCounters", }); + new java.lang.String[] { "SchemaNameBytes", "TableNameBytes", "TableType", "IndexState", "SequenceNumber", "TimeStamp", "PkNameBytes", "BucketNum", "Columns", "Indexes", "IsImmutableRows", "DataTableNameBytes", "DefaultFamilyName", "DisableWAL", "MultiTenant", "ViewType", "ViewStatement", "PhysicalNames", "TenantId", "ViewIndexId", "IndexType", "StatsTimeStamp", "StoreNulls", "BaseColumnCount", "RowKeyOrderOptimizable", "Transactional", "UpdateCacheFrequency", "IndexDisableTimestamp", "IsNamespaceMapped", "AutoParititonSeqName", "IsAppendOnlySchema", "ParentNameBytes", "StorageScheme", "EncodingScheme", "EncodedCQCounters", "UseStatsForQueryPlan", }); internal_static_EncodedCQCounter_descriptor = getDescriptor().getMessageTypes().get(3); internal_static_EncodedCQCounter_fieldAccessorTable = new http://git-wip-us.apache.org/repos/asf/phoenix/blob/3872648f/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java b/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java index d563bc2..946a99d 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java @@ -23,6 +23,8 @@ import static org.apache.phoenix.coprocessor.BaseScannerRegionObserver.SCAN_STAR import static org.apache.phoenix.coprocessor.BaseScannerRegionObserver.SCAN_STOP_ROW_SUFFIX; import static org.apache.phoenix.monitoring.GlobalClientMetrics.GLOBAL_FAILED_QUERY_COUNTER; import static org.apache.phoenix.monitoring.GlobalClientMetrics.GLOBAL_QUERY_TIMEOUT_COUNTER; +import static org.apache.phoenix.query.QueryServices.USE_STATS_FOR_PARALLELIZATION; +import static org.apache.phoenix.query.QueryServicesOptions.DEFAULT_USE_STATS_FOR_QUERY_PLANNING; import static org.apache.phoenix.schema.PTable.IndexType.LOCAL; import static org.apache.phoenix.schema.PTableType.INDEX; import static org.apache.phoenix.util.ByteUtil.EMPTY_BYTE_ARRAY; @@ -142,6 +144,7 @@ public abstract class BaseResultIterators extends ExplainTable implements Result private Long estimatedSize; private boolean hasGuidePosts; private Scan scan; + private boolean useStatsForQueryPlanning; static final Function<HRegionLocation, KeyRange> TO_KEY_RANGE = new Function<HRegionLocation, KeyRange>() { @Override @@ -467,7 +470,9 @@ public abstract class BaseResultIterators extends ExplainTable implements Result scanId = new UUID(ThreadLocalRandom.current().nextLong(), ThreadLocalRandom.current().nextLong()).toString(); initializeScan(plan, perScanLimit, offset, scan); - + this.useStatsForQueryPlanning = + context.getConnection().getQueryServices().getConfiguration().getBoolean( + USE_STATS_FOR_PARALLELIZATION, DEFAULT_USE_STATS_FOR_QUERY_PLANNING); this.scans = getParallelScans(); List<KeyRange> splitRanges = Lists.newArrayListWithExpectedSize(scans.size() * ESTIMATED_GUIDEPOSTS_PER_REGION); for (List<Scan> scanList : scans) { @@ -727,7 +732,9 @@ public abstract class BaseResultIterators extends ExplainTable implements Result estimatedRows += gps.getRowCounts()[guideIndex]; estimatedSize += gps.getByteCounts()[guideIndex]; } - scans = addNewScan(parallelScans, scans, newScan, currentGuidePostBytes, false, regionLocation); + if (useStatsForQueryPlanning) { + scans = addNewScan(parallelScans, scans, newScan, currentGuidePostBytes, false, regionLocation); + } currentKeyBytes = currentGuidePostBytes; currentGuidePost = PrefixByteCodec.decode(decoder, input); currentGuidePostBytes = currentGuidePost.copyBytes(); http://git-wip-us.apache.org/repos/asf/phoenix/blob/3872648f/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java b/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java index e061406..762c4d3 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java @@ -333,6 +333,8 @@ public class PhoenixDatabaseMetaData implements DatabaseMetaData { public static final byte[] COLUMN_QUALIFIER_BYTES = Bytes.toBytes(COLUMN_QUALIFIER); public static final String COLUMN_QUALIFIER_COUNTER = "QUALIFIER_COUNTER"; public static final byte[] COLUMN_QUALIFIER_COUNTER_BYTES = Bytes.toBytes(COLUMN_QUALIFIER_COUNTER); + public static final String USE_STATS_FOR_QUERY_PLAN = "USE_STATS_FOR_QUERY_PLAN"; + public static final byte[] USE_STATS_FOR_QUERY_PLAN_BYTES = Bytes.toBytes(USE_STATS_FOR_QUERY_PLAN); PhoenixDatabaseMetaData(PhoenixConnection connection) throws SQLException { this.emptyResultSet = new PhoenixResultSet(ResultIterator.EMPTY_ITERATOR, RowProjector.EMPTY_PROJECTOR, new StatementContext(new PhoenixStatement(connection), false)); http://git-wip-us.apache.org/repos/asf/phoenix/blob/3872648f/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java index 3f05ba3..8070e07 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java @@ -58,7 +58,7 @@ import static org.apache.phoenix.query.QueryServicesOptions.DEFAULT_RENEW_LEASE_ import static org.apache.phoenix.query.QueryServicesOptions.DEFAULT_RENEW_LEASE_THRESHOLD_MILLISECONDS; import static org.apache.phoenix.query.QueryServicesOptions.DEFAULT_RUN_RENEW_LEASE_FREQUENCY_INTERVAL_MILLISECONDS; import static org.apache.phoenix.util.UpgradeUtil.getSysCatalogSnapshotName; -import static org.apache.phoenix.util.UpgradeUtil.upgradeTo4_11_0; +import static org.apache.phoenix.util.UpgradeUtil.addParentToChildLinks; import static org.apache.phoenix.util.UpgradeUtil.upgradeTo4_5_0; import java.io.IOException; @@ -2794,7 +2794,13 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement clearCache(); } if (currentServerSideTableTimeStamp < MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_11_0) { - upgradeTo4_11_0(metaConnection); + metaConnection = addColumnsIfNotExists( + metaConnection, + PhoenixDatabaseMetaData.SYSTEM_CATALOG, + MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_11_0, + PhoenixDatabaseMetaData.USE_STATS_FOR_QUERY_PLAN + " " + + PBoolean.INSTANCE.getSqlTypeName()); + addParentToChildLinks(metaConnection); } } http://git-wip-us.apache.org/repos/asf/phoenix/blob/3872648f/phoenix-core/src/main/java/org/apache/phoenix/query/QueryConstants.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryConstants.java b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryConstants.java index 6f105f1..d3143ec 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryConstants.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryConstants.java @@ -104,6 +104,7 @@ import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TYPE; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TYPE_NAME; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TYPE_SEQUENCE; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.UPDATE_CACHE_FREQUENCY; +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.USE_STATS_FOR_QUERY_PLAN; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VIEW_CONSTANT; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VIEW_INDEX_ID; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VIEW_STATEMENT; @@ -320,6 +321,7 @@ public interface QueryConstants { IMMUTABLE_STORAGE_SCHEME + " TINYINT, " + ENCODING_SCHEME + " TINYINT, " + COLUMN_QUALIFIER_COUNTER + " INTEGER, " + + USE_STATS_FOR_QUERY_PLAN + " BOOLEAN, " + "CONSTRAINT " + SYSTEM_TABLE_PK_NAME + " PRIMARY KEY (" + TENANT_ID + "," + TABLE_SCHEM + "," + TABLE_NAME + "," + COLUMN_NAME + "," + COLUMN_FAMILY + "))\n" + HConstants.VERSIONS + "=" + MetaDataProtocol.DEFAULT_MAX_META_DATA_VERSIONS + ",\n" + http://git-wip-us.apache.org/repos/asf/phoenix/blob/3872648f/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java index 6364ee3..6b856ac 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java @@ -167,7 +167,10 @@ public interface QueryServices extends SQLCloseable { public static final String STATS_GUIDEPOST_WIDTH_BYTES_ATTRIB = "phoenix.stats.guidepost.width"; public static final String STATS_GUIDEPOST_PER_REGION_ATTRIB = "phoenix.stats.guidepost.per.region"; public static final String STATS_USE_CURRENT_TIME_ATTRIB = "phoenix.stats.useCurrentTime"; + + @Deprecated // use STATS_COLLECTION_ENABLED config instead public static final String STATS_ENABLED_ATTRIB = "phoenix.stats.enabled"; + public static final String RUN_UPDATE_STATS_ASYNC = "phoenix.update.stats.command.async"; public static final String STATS_SERVER_POOL_SIZE = "phoenix.stats.pool.size"; public static final String COMMIT_STATS_ASYNC = "phoenix.stats.commit.async"; @@ -251,6 +254,8 @@ public interface QueryServices extends SQLCloseable { public static final String DEFAULT_COLUMN_ENCODED_BYTES_ATRRIB = "phoenix.default.column.encoded.bytes.attrib"; public static final String DEFAULT_IMMUTABLE_STORAGE_SCHEME_ATTRIB = "phoenix.default.immutable.storage.scheme"; public static final String DEFAULT_MULTITENANT_IMMUTABLE_STORAGE_SCHEME_ATTRIB = "phoenix.default.multitenant.immutable.storage.scheme"; + public static final String STATS_COLLECTION_ENABLED = "phoenix.stats.collection.enabled"; + public static final String USE_STATS_FOR_PARALLELIZATION = "phoenix.use.stats.parallelization"; // whether to enable server side RS -> RS calls for upsert select statements public static final String ENABLE_SERVER_UPSERT_SELECT ="phoenix.client.enable.server.upsert.select"; http://git-wip-us.apache.org/repos/asf/phoenix/blob/3872648f/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java index ae7d7aa..9af7795 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java @@ -73,6 +73,7 @@ import static org.apache.phoenix.query.QueryServices.SEQUENCE_CACHE_SIZE_ATTRIB; import static org.apache.phoenix.query.QueryServices.SEQUENCE_SALT_BUCKETS_ATTRIB; import static org.apache.phoenix.query.QueryServices.SPOOL_DIRECTORY; import static org.apache.phoenix.query.QueryServices.SPOOL_THRESHOLD_BYTES_ATTRIB; +import static org.apache.phoenix.query.QueryServices.STATS_COLLECTION_ENABLED; import static org.apache.phoenix.query.QueryServices.STATS_GUIDEPOST_WIDTH_BYTES_ATTRIB; import static org.apache.phoenix.query.QueryServices.STATS_UPDATE_FREQ_MS_ATTRIB; import static org.apache.phoenix.query.QueryServices.STATS_USE_CURRENT_TIME_ATTRIB; @@ -87,6 +88,7 @@ import static org.apache.phoenix.query.QueryServices.TRACING_STATS_TABLE_NAME_AT import static org.apache.phoenix.query.QueryServices.TRACING_BATCH_SIZE; import static org.apache.phoenix.query.QueryServices.TRACING_THREAD_POOL_SIZE; import static org.apache.phoenix.query.QueryServices.TRACING_TRACE_BUFFER_SIZE; +import static org.apache.phoenix.query.QueryServices.USE_STATS_FOR_PARALLELIZATION; import java.util.HashSet; import java.util.Map.Entry; @@ -284,6 +286,8 @@ public class QueryServicesOptions { //by default, max connections from one client to one cluster is unlimited public static final int DEFAULT_CLIENT_CONNECTION_MAX_ALLOWED_CONNECTIONS = 0; + public static final boolean DEFAULT_STATS_COLLECTION_ENABLED = true; + public static final boolean DEFAULT_USE_STATS_FOR_QUERY_PLANNING = true; @SuppressWarnings("serial") public static final Set<String> DEFAULT_QUERY_SERVER_SKIP_WORDS = new HashSet<String>() { @@ -372,7 +376,9 @@ public class QueryServicesOptions { .setIfUnset(UPLOAD_BINARY_DATA_TYPE_ENCODING, DEFAULT_UPLOAD_BINARY_DATA_TYPE_ENCODING) .setIfUnset(TRACING_ENABLED, DEFAULT_TRACING_ENABLED) .setIfUnset(TRACING_BATCH_SIZE, DEFAULT_TRACING_BATCH_SIZE) - .setIfUnset(TRACING_THREAD_POOL_SIZE, DEFAULT_TRACING_THREAD_POOL_SIZE); + .setIfUnset(TRACING_THREAD_POOL_SIZE, DEFAULT_TRACING_THREAD_POOL_SIZE) + .setIfUnset(STATS_COLLECTION_ENABLED, DEFAULT_STATS_COLLECTION_ENABLED) + .setIfUnset(USE_STATS_FOR_PARALLELIZATION, DEFAULT_USE_STATS_FOR_QUERY_PLANNING); // HBase sets this to 1, so we reset it to something more appropriate. // Hopefully HBase will change this, because we can't know if a user set // it to 1, so we'll change it. @@ -606,7 +612,6 @@ public class QueryServicesOptions { return config.getInt(SCAN_CACHE_SIZE_ATTRIB, DEFAULT_SCAN_CACHE_SIZE); } - public QueryServicesOptions setMaxServerCacheTTLMs(int ttl) { return set(MAX_SERVER_CACHE_TIME_TO_LIVE_MS_ATTRIB, ttl); } @@ -747,4 +752,9 @@ public class QueryServicesOptions { return this; } + public QueryServicesOptions setUseStatsForQueryPlanning(boolean flag) { + config.setBoolean(USE_STATS_FOR_PARALLELIZATION, flag); + return this; + } + } http://git-wip-us.apache.org/repos/asf/phoenix/blob/3872648f/phoenix-core/src/main/java/org/apache/phoenix/schema/DelegateTable.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/DelegateTable.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/DelegateTable.java index 12258e1..84c38c3 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/schema/DelegateTable.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/DelegateTable.java @@ -310,4 +310,9 @@ public class DelegateTable implements PTable { public QualifierEncodingScheme getEncodingScheme() { return delegate.getEncodingScheme(); } + + @Override + public boolean useStatsForQueryPlan() { + return delegate.useStatsForQueryPlan(); + } } http://git-wip-us.apache.org/repos/asf/phoenix/blob/3872648f/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java index 24f3848..2aab585 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java @@ -85,6 +85,7 @@ import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TENANT_ID; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TRANSACTIONAL; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TYPE; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.UPDATE_CACHE_FREQUENCY; +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.USE_STATS_FOR_QUERY_PLAN; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VIEW_CONSTANT; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VIEW_STATEMENT; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VIEW_TYPE; @@ -281,8 +282,9 @@ public class MetaDataClient { APPEND_ONLY_SCHEMA + "," + GUIDE_POSTS_WIDTH + "," + IMMUTABLE_STORAGE_SCHEME + "," + - ENCODING_SCHEME + - ") VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"; + ENCODING_SCHEME + "," + + USE_STATS_FOR_QUERY_PLAN + + ") VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"; private static final String CREATE_SCHEMA = "UPSERT INTO " + SYSTEM_CATALOG_SCHEMA + ".\"" + SYSTEM_CATALOG_TABLE + "\"( " + TABLE_SCHEM + "," + TABLE_NAME + ") VALUES (?,?)"; @@ -1989,6 +1991,16 @@ public class MetaDataClient { } } + boolean useStatsForQueryPlan = true; + Boolean useStatsForQueryPlanProp = (Boolean) TableProperty.USE_STATS_FOR_QUERY_PLAN.getValue(tableProps); + if (useStatsForQueryPlanProp != null) { + useStatsForQueryPlan = useStatsForQueryPlanProp; + } else { + useStatsForQueryPlan = connection.getQueryServices().getProps().getBoolean( + QueryServices.USE_STATS_FOR_PARALLELIZATION, + QueryServicesOptions.DEFAULT_USE_STATS_FOR_QUERY_PLANNING); + } + boolean sharedTable = statement.getTableType() == PTableType.VIEW || allocateIndexId; if (transactional) { // Tephra uses an empty value cell as its delete marker, so we need to turn on @@ -2425,7 +2437,7 @@ public class MetaDataClient { Collections.<PTable>emptyList(), isImmutableRows, Collections.<PName>emptyList(), defaultFamilyName == null ? null : PNameFactory.newName(defaultFamilyName), null, - Boolean.TRUE.equals(disableWAL), false, false, null, null, indexType, true, false, 0, 0L, isNamespaceMapped, autoPartitionSeq, isAppendOnlySchema, ONE_CELL_PER_COLUMN, NON_ENCODED_QUALIFIERS, PTable.EncodedCQCounter.NULL_COUNTER); + Boolean.TRUE.equals(disableWAL), false, false, null, null, indexType, true, false, 0, 0L, isNamespaceMapped, autoPartitionSeq, isAppendOnlySchema, ONE_CELL_PER_COLUMN, NON_ENCODED_QUALIFIERS, PTable.EncodedCQCounter.NULL_COUNTER, true); connection.addTable(table, MetaDataProtocol.MIN_TABLE_TIMESTAMP); } @@ -2581,6 +2593,7 @@ public class MetaDataClient { } tableUpsert.setByte(26, immutableStorageScheme.getSerializedMetadataValue()); tableUpsert.setByte(27, encodingScheme.getSerializedMetadataValue()); + tableUpsert.setBoolean(28, useStatsForQueryPlan); tableUpsert.execute(); if (asyncCreatedDate != null) { @@ -2685,7 +2698,7 @@ public class MetaDataClient { PTable.INITIAL_SEQ_NUM, pkName == null ? null : PNameFactory.newName(pkName), saltBucketNum, columns.values(), parent == null ? null : parent.getSchemaName(), parent == null ? null : parent.getTableName(), Collections.<PTable>emptyList(), isImmutableRows, physicalNames, defaultFamilyName == null ? null : PNameFactory.newName(defaultFamilyName), viewStatement, Boolean.TRUE.equals(disableWAL), multiTenant, storeNulls, viewType, - result.getViewIndexId(), indexType, rowKeyOrderOptimizable, transactional, updateCacheFrequency, 0L, isNamespaceMapped, autoPartitionSeq, isAppendOnlySchema, immutableStorageScheme, encodingScheme, cqCounterToBe); + result.getViewIndexId(), indexType, rowKeyOrderOptimizable, transactional, updateCacheFrequency, 0L, isNamespaceMapped, autoPartitionSeq, isAppendOnlySchema, immutableStorageScheme, encodingScheme, cqCounterToBe, useStatsForQueryPlan); result = new MetaDataMutationResult(code, result.getMutationTime(), table, true); addTableToCache(result); return table; @@ -2869,7 +2882,7 @@ public class MetaDataClient { PTable viewIndexTable = new PTableImpl(null, SchemaUtil.getSchemaNameFromFullName(viewIndexPhysicalName), SchemaUtil.getTableNameFromFullName(viewIndexPhysicalName), ts, - table.getColumnFamilies(),table.isNamespaceMapped(), table.getImmutableStorageScheme(), table.getEncodingScheme()); + table.getColumnFamilies(),table.isNamespaceMapped(), table.getImmutableStorageScheme(), table.getEncodingScheme(), table.useStatsForQueryPlan()); tableRefs.add(new TableRef(null, viewIndexTable, ts, false)); } } @@ -2990,12 +3003,13 @@ public class MetaDataClient { } private long incrementTableSeqNum(PTable table, PTableType expectedType, int columnCountDelta, Boolean isTransactional, Long updateCacheFrequency) throws SQLException { - return incrementTableSeqNum(table, expectedType, columnCountDelta, isTransactional, updateCacheFrequency, null, null, null, null, -1L, null, null); + return incrementTableSeqNum(table, expectedType, columnCountDelta, isTransactional, updateCacheFrequency, null, null, null, null, -1L, null, null, null); } private long incrementTableSeqNum(PTable table, PTableType expectedType, int columnCountDelta, Boolean isTransactional, Long updateCacheFrequency, Boolean isImmutableRows, Boolean disableWAL, - Boolean isMultiTenant, Boolean storeNulls, Long guidePostWidth, Boolean appendOnlySchema, ImmutableStorageScheme immutableStorageScheme) + Boolean isMultiTenant, Boolean storeNulls, Long guidePostWidth, Boolean appendOnlySchema, ImmutableStorageScheme immutableStorageScheme + , Boolean useStatsForQueryPlan) throws SQLException { String schemaName = table.getSchemaName().getString(); String tableName = table.getTableName().getString(); @@ -3042,7 +3056,9 @@ public class MetaDataClient { if (immutableStorageScheme !=null) { mutateStringProperty(tenantId, schemaName, tableName, IMMUTABLE_STORAGE_SCHEME, immutableStorageScheme.name()); } - + if (useStatsForQueryPlan != null) { + mutateBooleanProperty(tenantId, schemaName, tableName, USE_STATS_FOR_QUERY_PLAN, useStatsForQueryPlan); + } return seqNum; } @@ -3126,6 +3142,7 @@ public class MetaDataClient { Boolean appendOnlySchemaProp = null; Long guidePostWidth = -1L; ImmutableStorageScheme immutableStorageSchemeProp = null; + Boolean useStatsForQueryPlanProp = null; Map<String, List<Pair<String, Object>>> properties = new HashMap<>(stmtProperties.size()); List<ColumnDef> columnDefs = null; @@ -3190,6 +3207,8 @@ public class MetaDataClient { appendOnlySchemaProp = (Boolean) value; } else if (propName.equalsIgnoreCase(IMMUTABLE_STORAGE_SCHEME)) { immutableStorageSchemeProp = (ImmutableStorageScheme)value; + } else if (propName.equalsIgnoreCase(USE_STATS_FOR_QUERY_PLAN)) { + useStatsForQueryPlanProp = (Boolean)value; } } // if removeTableProps is true only add the property if it is not a HTable or Phoenix Table property @@ -3291,6 +3310,13 @@ public class MetaDataClient { changingPhoenixTableProperty = true; } } + Boolean useStatsForQueryPlan = null; + if (useStatsForQueryPlanProp != null) { + if (useStatsForQueryPlanProp.booleanValue() != table.useStatsForQueryPlan()) { + useStatsForQueryPlan = useStatsForQueryPlanProp; + changingPhoenixTableProperty = true; + } + } Boolean isTransactional = null; if (isTransactionalProp != null) { if (isTransactionalProp.booleanValue() != table.isTransactional()) { @@ -3465,7 +3491,7 @@ public class MetaDataClient { if (changingPhoenixTableProperty || columnDefs.size() > 0) { incrementTableSeqNum(table, tableType, columnDefs.size(), isTransactional, updateCacheFrequency, isImmutableRows, - disableWAL, multiTenant, storeNulls, guidePostWidth, appendOnlySchema, immutableStorageScheme); + disableWAL, multiTenant, storeNulls, guidePostWidth, appendOnlySchema, immutableStorageScheme, useStatsForQueryPlan); tableMetaData.addAll(connection.getMutationState().toMutations(timeStamp).next().getSecond()); connection.rollback(); } @@ -3562,7 +3588,7 @@ public class MetaDataClient { PTable viewIndexTable = new PTableImpl(null, SchemaUtil.getSchemaNameFromFullName(viewIndexPhysicalName), SchemaUtil.getTableNameFromFullName(viewIndexPhysicalName), ts, - table.getColumnFamilies(), table.isNamespaceMapped(), table.getImmutableStorageScheme(), table.getEncodingScheme()); + table.getColumnFamilies(), table.isNamespaceMapped(), table.getImmutableStorageScheme(), table.getEncodingScheme(), table.useStatsForQueryPlan()); List<TableRef> tableRefs = Collections.singletonList(new TableRef(null, viewIndexTable, ts, false)); MutationPlan plan = new PostDDLCompiler(connection).compile(tableRefs, null, null, Collections.<PColumn> emptyList(), ts); @@ -3827,7 +3853,7 @@ public class MetaDataClient { sharedTableState.getSchemaName(), sharedTableState.getTableName(), ts, table.getColumnFamilies(), sharedTableState.getColumns(), sharedTableState.getPhysicalNames(), sharedTableState.getViewIndexId(), - table.isMultiTenant(), table.isNamespaceMapped(), table.getImmutableStorageScheme(), table.getEncodingScheme(), table.getEncodedCQCounter()); + table.isMultiTenant(), table.isNamespaceMapped(), table.getImmutableStorageScheme(), table.getEncodingScheme(), table.getEncodedCQCounter(), table.useStatsForQueryPlan()); TableRef indexTableRef = new TableRef(viewIndexTable); PName indexTableTenantId = sharedTableState.getTenantId(); if (indexTableTenantId==null) { http://git-wip-us.apache.org/repos/asf/phoenix/blob/3872648f/phoenix-core/src/main/java/org/apache/phoenix/schema/PTable.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/PTable.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/PTable.java index 24a494f..a0461e5 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/schema/PTable.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/PTable.java @@ -705,6 +705,7 @@ public interface PTable extends PMetaDataEntity { ImmutableStorageScheme getImmutableStorageScheme(); QualifierEncodingScheme getEncodingScheme(); EncodedCQCounter getEncodedCQCounter(); + boolean useStatsForQueryPlan(); /** * Class to help track encoded column qualifier counters per column family. http://git-wip-us.apache.org/repos/asf/phoenix/blob/3872648f/phoenix-core/src/main/java/org/apache/phoenix/schema/PTableImpl.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/PTableImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/PTableImpl.java index 51f5b0b..fa9532e 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/schema/PTableImpl.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/PTableImpl.java @@ -151,6 +151,7 @@ public class PTableImpl implements PTable { private ImmutableStorageScheme immutableStorageScheme; private QualifierEncodingScheme qualifierEncodingScheme; private EncodedCQCounter encodedCQCounter; + private boolean useStatsForQueryPlan; public PTableImpl() { this.indexes = Collections.emptyList(); @@ -183,7 +184,7 @@ public class PTableImpl implements PTable { this.isNamespaceMapped = isNamespaceMapped; } - public PTableImpl(PName tenantId, String schemaName, String tableName, long timestamp, List<PColumnFamily> families, boolean isNamespaceMapped, ImmutableStorageScheme storageScheme, QualifierEncodingScheme encodingScheme) { // For base table of mapped VIEW + public PTableImpl(PName tenantId, String schemaName, String tableName, long timestamp, List<PColumnFamily> families, boolean isNamespaceMapped, ImmutableStorageScheme storageScheme, QualifierEncodingScheme encodingScheme, boolean useStatsForQueryPlan) { // For base table of mapped VIEW Preconditions.checkArgument(tenantId==null || tenantId.getBytes().length > 0); // tenantId should be null or not empty this.tenantId = tenantId; this.name = PNameFactory.newName(SchemaUtil.getTableName(schemaName, tableName)); @@ -207,12 +208,13 @@ public class PTableImpl implements PTable { this.isNamespaceMapped = isNamespaceMapped; this.immutableStorageScheme = storageScheme; this.qualifierEncodingScheme = encodingScheme; + this.useStatsForQueryPlan = useStatsForQueryPlan; } // For indexes stored in shared physical tables public PTableImpl(PName tenantId, PName schemaName, PName tableName, long timestamp, List<PColumnFamily> families, List<PColumn> columns, List<PName> physicalNames, Short viewIndexId, boolean multiTenant, boolean isNamespaceMpped, ImmutableStorageScheme storageScheme, QualifierEncodingScheme qualifierEncodingScheme, - EncodedCQCounter encodedCQCounter) throws SQLException { + EncodedCQCounter encodedCQCounter, boolean useStatsForQueryPlan) throws SQLException { this.pkColumns = this.allColumns = Collections.emptyList(); this.rowKeySchema = RowKeySchema.EMPTY_SCHEMA; this.indexes = Collections.emptyList(); @@ -226,7 +228,7 @@ public class PTableImpl implements PTable { init(tenantId, this.schemaName, this.tableName, PTableType.INDEX, state, timeStamp, sequenceNumber, pkName, bucketNum, columns, this.schemaName, parentTableName, indexes, isImmutableRows, physicalNames, defaultFamilyName, null, disableWAL, multiTenant, storeNulls, viewType, viewIndexId, indexType, baseColumnCount, rowKeyOrderOptimizable, - isTransactional, updateCacheFrequency, indexDisableTimestamp, isNamespaceMpped, null, false, storageScheme, qualifierEncodingScheme, encodedCQCounter); + isTransactional, updateCacheFrequency, indexDisableTimestamp, isNamespaceMpped, null, false, storageScheme, qualifierEncodingScheme, encodedCQCounter, useStatsForQueryPlan); } public PTableImpl(long timeStamp) { // For delete marker @@ -270,7 +272,7 @@ public class PTableImpl implements PTable { indexes, table.isImmutableRows(), physicalNames, table.getDefaultFamilyName(), viewStatement, table.isWALDisabled(), table.isMultiTenant(), table.getStoreNulls(), table.getViewType(), table.getViewIndexId(), table.getIndexType(), table.getBaseColumnCount(), table.rowKeyOrderOptimizable(), table.isTransactional(), updateCacheFrequency, - table.getIndexDisableTimestamp(), table.isNamespaceMapped(), table.getAutoPartitionSeqName(), table.isAppendOnlySchema(), table.getImmutableStorageScheme(), table.getEncodingScheme(), table.getEncodedCQCounter()); + table.getIndexDisableTimestamp(), table.isNamespaceMapped(), table.getAutoPartitionSeqName(), table.isAppendOnlySchema(), table.getImmutableStorageScheme(), table.getEncodingScheme(), table.getEncodedCQCounter(), table.useStatsForQueryPlan()); } public static PTableImpl makePTable(PTable table, long timeStamp, List<PTable> indexes, PName parentSchemaName, String viewStatement) throws SQLException { @@ -280,7 +282,7 @@ public class PTableImpl implements PTable { indexes, table.isImmutableRows(), table.getPhysicalNames(), table.getDefaultFamilyName(), viewStatement, table.isWALDisabled(), table.isMultiTenant(), table.getStoreNulls(), table.getViewType(), table.getViewIndexId(), table.getIndexType(), table.getBaseColumnCount(), table.rowKeyOrderOptimizable(), table.isTransactional(), table.getUpdateCacheFrequency(), - table.getIndexDisableTimestamp(), table.isNamespaceMapped(), table.getAutoPartitionSeqName(), table.isAppendOnlySchema(), table.getImmutableStorageScheme(), table.getEncodingScheme(), table.getEncodedCQCounter()); + table.getIndexDisableTimestamp(), table.isNamespaceMapped(), table.getAutoPartitionSeqName(), table.isAppendOnlySchema(), table.getImmutableStorageScheme(), table.getEncodingScheme(), table.getEncodedCQCounter(), table.useStatsForQueryPlan()); } public static PTableImpl makePTable(PTable table, Collection<PColumn> columns) throws SQLException { @@ -290,7 +292,7 @@ public class PTableImpl implements PTable { table.getIndexes(), table.isImmutableRows(), table.getPhysicalNames(), table.getDefaultFamilyName(), table.getViewStatement(), table.isWALDisabled(), table.isMultiTenant(), table.getStoreNulls(), table.getViewType(), table.getViewIndexId(), table.getIndexType(), table.getBaseColumnCount(), table.rowKeyOrderOptimizable(), table.isTransactional(), table.getUpdateCacheFrequency(), - table.getIndexDisableTimestamp(), table.isNamespaceMapped(), table.getAutoPartitionSeqName(), table.isAppendOnlySchema(), table.getImmutableStorageScheme(), table.getEncodingScheme(), table.getEncodedCQCounter()); + table.getIndexDisableTimestamp(), table.isNamespaceMapped(), table.getAutoPartitionSeqName(), table.isAppendOnlySchema(), table.getImmutableStorageScheme(), table.getEncodingScheme(), table.getEncodedCQCounter(), table.useStatsForQueryPlan()); } public static PTableImpl makePTable(PTable table, Collection<PColumn> columns, PName defaultFamily) throws SQLException { @@ -300,7 +302,7 @@ public class PTableImpl implements PTable { table.getIndexes(), table.isImmutableRows(), table.getPhysicalNames(), defaultFamily, table.getViewStatement(), table.isWALDisabled(), table.isMultiTenant(), table.getStoreNulls(), table.getViewType(), table.getViewIndexId(), table.getIndexType(), table.getBaseColumnCount(), table.rowKeyOrderOptimizable(), table.isTransactional(), table.getUpdateCacheFrequency(), - table.getIndexDisableTimestamp(), table.isNamespaceMapped(), table.getAutoPartitionSeqName(), table.isAppendOnlySchema(), table.getImmutableStorageScheme(), table.getEncodingScheme(), table.getEncodedCQCounter()); + table.getIndexDisableTimestamp(), table.isNamespaceMapped(), table.getAutoPartitionSeqName(), table.isAppendOnlySchema(), table.getImmutableStorageScheme(), table.getEncodingScheme(), table.getEncodedCQCounter(), table.useStatsForQueryPlan()); } public static PTableImpl makePTable(PTable table, long timeStamp, long sequenceNumber, Collection<PColumn> columns) throws SQLException { @@ -310,7 +312,7 @@ public class PTableImpl implements PTable { table.isImmutableRows(), table.getPhysicalNames(), table.getDefaultFamilyName(), table.getViewStatement(), table.isWALDisabled(), table.isMultiTenant(), table.getStoreNulls(), table.getViewType(), table.getViewIndexId(), table.getIndexType(), table.getBaseColumnCount(), table.rowKeyOrderOptimizable(), table.isTransactional(), table.getUpdateCacheFrequency(), table.getIndexDisableTimestamp(), - table.isNamespaceMapped(), table.getAutoPartitionSeqName(), table.isAppendOnlySchema(), table.getImmutableStorageScheme(), table.getEncodingScheme(), table.getEncodedCQCounter()); + table.isNamespaceMapped(), table.getAutoPartitionSeqName(), table.isAppendOnlySchema(), table.getImmutableStorageScheme(), table.getEncodingScheme(), table.getEncodedCQCounter(), table.useStatsForQueryPlan()); } public static PTableImpl makePTable(PTable table, long timeStamp, long sequenceNumber, Collection<PColumn> columns, boolean isImmutableRows) throws SQLException { @@ -320,7 +322,7 @@ public class PTableImpl implements PTable { table.getIndexes(), isImmutableRows, table.getPhysicalNames(), table.getDefaultFamilyName(), table.getViewStatement(), table.isWALDisabled(), table.isMultiTenant(), table.getStoreNulls(), table.getViewType(), table.getViewIndexId(), table.getIndexType(), table.getBaseColumnCount(), table.rowKeyOrderOptimizable(), table.isTransactional(), - table.getUpdateCacheFrequency(), table.getIndexDisableTimestamp(), table.isNamespaceMapped(), table.getAutoPartitionSeqName(), table.isAppendOnlySchema(), table.getImmutableStorageScheme(), table.getEncodingScheme(), table.getEncodedCQCounter()); + table.getUpdateCacheFrequency(), table.getIndexDisableTimestamp(), table.isNamespaceMapped(), table.getAutoPartitionSeqName(), table.isAppendOnlySchema(), table.getImmutableStorageScheme(), table.getEncodingScheme(), table.getEncodedCQCounter(), table.useStatsForQueryPlan()); } public static PTableImpl makePTable(PTable table, long timeStamp, long sequenceNumber, Collection<PColumn> columns, boolean isImmutableRows, boolean isWalDisabled, @@ -331,7 +333,7 @@ public class PTableImpl implements PTable { table.getIndexes(), isImmutableRows, table.getPhysicalNames(), table.getDefaultFamilyName(), table.getViewStatement(), isWalDisabled, isMultitenant, storeNulls, table.getViewType(), table.getViewIndexId(), table.getIndexType(), table.getBaseColumnCount(), table.rowKeyOrderOptimizable(), isTransactional, updateCacheFrequency, table.getIndexDisableTimestamp(), - isNamespaceMapped, table.getAutoPartitionSeqName(), table.isAppendOnlySchema(), table.getImmutableStorageScheme(), table.getEncodingScheme(), table.getEncodedCQCounter()); + isNamespaceMapped, table.getAutoPartitionSeqName(), table.isAppendOnlySchema(), table.getImmutableStorageScheme(), table.getEncodingScheme(), table.getEncodedCQCounter(), table.useStatsForQueryPlan()); } public static PTableImpl makePTable(PTable table, PIndexState state) throws SQLException { @@ -342,7 +344,7 @@ public class PTableImpl implements PTable { table.isImmutableRows(), table.getPhysicalNames(), table.getDefaultFamilyName(), table.getViewStatement(), table.isWALDisabled(), table.isMultiTenant(), table.getStoreNulls(), table.getViewType(), table.getViewIndexId(), table.getIndexType(), table.getBaseColumnCount(), table.rowKeyOrderOptimizable(), table.isTransactional(), table.getUpdateCacheFrequency(), - table.getIndexDisableTimestamp(), table.isNamespaceMapped(), table.getAutoPartitionSeqName(), table.isAppendOnlySchema(), table.getImmutableStorageScheme(), table.getEncodingScheme(), table.getEncodedCQCounter()); + table.getIndexDisableTimestamp(), table.isNamespaceMapped(), table.getAutoPartitionSeqName(), table.isAppendOnlySchema(), table.getImmutableStorageScheme(), table.getEncodingScheme(), table.getEncodedCQCounter(), table.useStatsForQueryPlan()); } public static PTableImpl makePTable(PTable table, boolean rowKeyOrderOptimizable) throws SQLException { @@ -353,7 +355,7 @@ public class PTableImpl implements PTable { table.isImmutableRows(), table.getPhysicalNames(), table.getDefaultFamilyName(), table.getViewStatement(), table.isWALDisabled(), table.isMultiTenant(), table.getStoreNulls(), table.getViewType(), table.getViewIndexId(), table.getIndexType(), table.getBaseColumnCount(), rowKeyOrderOptimizable, table.isTransactional(), table.getUpdateCacheFrequency(), table.getIndexDisableTimestamp(), table.isNamespaceMapped(), - table.getAutoPartitionSeqName(), table.isAppendOnlySchema(), table.getImmutableStorageScheme(), table.getEncodingScheme(), table.getEncodedCQCounter()); + table.getAutoPartitionSeqName(), table.isAppendOnlySchema(), table.getImmutableStorageScheme(), table.getEncodingScheme(), table.getEncodedCQCounter(), table.useStatsForQueryPlan()); } public static PTableImpl makePTable(PTable table) throws SQLException { @@ -364,7 +366,7 @@ public class PTableImpl implements PTable { table.isImmutableRows(), table.getPhysicalNames(), table.getDefaultFamilyName(), table.getViewStatement(), table.isWALDisabled(), table.isMultiTenant(), table.getStoreNulls(), table.getViewType(), table.getViewIndexId(), table.getIndexType(), table.getBaseColumnCount(), table.rowKeyOrderOptimizable(), table.isTransactional(), table.getUpdateCacheFrequency(), table.getIndexDisableTimestamp(), - table.isNamespaceMapped(), table.getAutoPartitionSeqName(), table.isAppendOnlySchema(), table.getImmutableStorageScheme(), table.getEncodingScheme(), table.getEncodedCQCounter()); + table.isNamespaceMapped(), table.getAutoPartitionSeqName(), table.isAppendOnlySchema(), table.getImmutableStorageScheme(), table.getEncodingScheme(), table.getEncodedCQCounter(), table.useStatsForQueryPlan()); } public static PTableImpl makePTable(PName tenantId, PName schemaName, PName tableName, PTableType type, @@ -373,12 +375,12 @@ public class PTableImpl implements PTable { boolean isImmutableRows, List<PName> physicalNames, PName defaultFamilyName, String viewExpression, boolean disableWAL, boolean multiTenant, boolean storeNulls, ViewType viewType, Short viewIndexId, IndexType indexType, boolean rowKeyOrderOptimizable, boolean isTransactional, long updateCacheFrequency, - long indexDisableTimestamp, boolean isNamespaceMapped, String autoPartitionSeqName, boolean isAppendOnlySchema, ImmutableStorageScheme storageScheme, QualifierEncodingScheme qualifierEncodingScheme, EncodedCQCounter encodedCQCounter) throws SQLException { + long indexDisableTimestamp, boolean isNamespaceMapped, String autoPartitionSeqName, boolean isAppendOnlySchema, ImmutableStorageScheme storageScheme, QualifierEncodingScheme qualifierEncodingScheme, EncodedCQCounter encodedCQCounter, boolean useStatsForQueryPlan) throws SQLException { return new PTableImpl(tenantId, schemaName, tableName, type, state, timeStamp, sequenceNumber, pkName, bucketNum, columns, dataSchemaName, dataTableName, indexes, isImmutableRows, physicalNames, defaultFamilyName, viewExpression, disableWAL, multiTenant, storeNulls, viewType, viewIndexId, indexType, QueryConstants.BASE_TABLE_BASE_COLUMN_COUNT, rowKeyOrderOptimizable, isTransactional, - updateCacheFrequency,indexDisableTimestamp, isNamespaceMapped, autoPartitionSeqName, isAppendOnlySchema, storageScheme, qualifierEncodingScheme, encodedCQCounter); + updateCacheFrequency,indexDisableTimestamp, isNamespaceMapped, autoPartitionSeqName, isAppendOnlySchema, storageScheme, qualifierEncodingScheme, encodedCQCounter, useStatsForQueryPlan); } public static PTableImpl makePTable(PName tenantId, PName schemaName, PName tableName, PTableType type, @@ -388,13 +390,14 @@ public class PTableImpl implements PTable { boolean disableWAL, boolean multiTenant, boolean storeNulls, ViewType viewType, Short viewIndexId, IndexType indexType, boolean rowKeyOrderOptimizable, boolean isTransactional, long updateCacheFrequency, int baseColumnCount, long indexDisableTimestamp, boolean isNamespaceMapped, - String autoPartitionSeqName, boolean isAppendOnlySchema, ImmutableStorageScheme storageScheme, QualifierEncodingScheme qualifierEncodingScheme, EncodedCQCounter encodedCQCounter) + String autoPartitionSeqName, boolean isAppendOnlySchema, ImmutableStorageScheme storageScheme, + QualifierEncodingScheme qualifierEncodingScheme, EncodedCQCounter encodedCQCounter, boolean useStatsForQueryPlan) throws SQLException { return new PTableImpl(tenantId, schemaName, tableName, type, state, timeStamp, sequenceNumber, pkName, bucketNum, columns, dataSchemaName, dataTableName, indexes, isImmutableRows, physicalNames, defaultFamilyName, viewExpression, disableWAL, multiTenant, storeNulls, viewType, viewIndexId, indexType, baseColumnCount, rowKeyOrderOptimizable, isTransactional, updateCacheFrequency, - indexDisableTimestamp, isNamespaceMapped, autoPartitionSeqName, isAppendOnlySchema, storageScheme, qualifierEncodingScheme, encodedCQCounter); + indexDisableTimestamp, isNamespaceMapped, autoPartitionSeqName, isAppendOnlySchema, storageScheme, qualifierEncodingScheme, encodedCQCounter, useStatsForQueryPlan); } private PTableImpl(PName tenantId, PName schemaName, PName tableName, PTableType type, PIndexState state, @@ -404,12 +407,12 @@ public class PTableImpl implements PTable { boolean storeNulls, ViewType viewType, Short viewIndexId, IndexType indexType, int baseColumnCount, boolean rowKeyOrderOptimizable, boolean isTransactional, long updateCacheFrequency, long indexDisableTimestamp, boolean isNamespaceMapped, String autoPartitionSeqName, boolean isAppendOnlySchema, ImmutableStorageScheme storageScheme, - QualifierEncodingScheme qualifierEncodingScheme, EncodedCQCounter encodedCQCounter) throws SQLException { + QualifierEncodingScheme qualifierEncodingScheme, EncodedCQCounter encodedCQCounter, boolean useStatsForQueryPlan) throws SQLException { init(tenantId, schemaName, tableName, type, state, timeStamp, sequenceNumber, pkName, bucketNum, columns, parentSchemaName, parentTableName, indexes, isImmutableRows, physicalNames, defaultFamilyName, viewExpression, disableWAL, multiTenant, storeNulls, viewType, viewIndexId, indexType, baseColumnCount, rowKeyOrderOptimizable, isTransactional, updateCacheFrequency, indexDisableTimestamp, isNamespaceMapped, autoPartitionSeqName, isAppendOnlySchema, storageScheme, - qualifierEncodingScheme, encodedCQCounter); + qualifierEncodingScheme, encodedCQCounter, useStatsForQueryPlan); } @Override @@ -444,7 +447,7 @@ public class PTableImpl implements PTable { boolean multiTenant, boolean storeNulls, ViewType viewType, Short viewIndexId, IndexType indexType , int baseColumnCount, boolean rowKeyOrderOptimizable, boolean isTransactional, long updateCacheFrequency, long indexDisableTimestamp, boolean isNamespaceMapped, String autoPartitionSeqName, boolean isAppendOnlySchema, ImmutableStorageScheme storageScheme, QualifierEncodingScheme qualifierEncodingScheme, - EncodedCQCounter encodedCQCounter) throws SQLException { + EncodedCQCounter encodedCQCounter, boolean useStatsForQueryPlan) throws SQLException { Preconditions.checkNotNull(schemaName); Preconditions.checkArgument(tenantId==null || tenantId.getBytes().length > 0); // tenantId should be null or not empty int estimatedSize = SizedUtil.OBJECT_SIZE * 2 + 23 * SizedUtil.POINTER_SIZE + 4 * SizedUtil.INT_SIZE + 2 * SizedUtil.LONG_SIZE + 2 * SizedUtil.INT_OBJECT_SIZE + @@ -616,6 +619,7 @@ public class PTableImpl implements PTable { this.estimatedSize = estimatedSize; this.baseColumnCount = baseColumnCount; this.encodedCQCounter = encodedCQCounter; + this.useStatsForQueryPlan = useStatsForQueryPlan; } @Override @@ -1323,7 +1327,10 @@ public class PTableImpl implements PTable { } } } - + boolean useStatsForQueryPlan = true; + if (table.hasUseStatsForQueryPlan()) { + useStatsForQueryPlan = table.getUseStatsForQueryPlan(); + } try { PTableImpl result = new PTableImpl(); result.init(tenantId, schemaName, tableName, tableType, indexState, timeStamp, sequenceNumber, pkName, @@ -1331,7 +1338,7 @@ public class PTableImpl implements PTable { isImmutableRows, physicalNames, defaultFamilyName, viewStatement, disableWAL, multiTenant, storeNulls, viewType, viewIndexId, indexType, baseColumnCount, rowKeyOrderOptimizable, isTransactional, updateCacheFrequency, indexDisableTimestamp, isNamespaceMapped, autoParititonSeqName, - isAppendOnlySchema, storageScheme, qualifierEncodingScheme, encodedColumnQualifierCounter); + isAppendOnlySchema, storageScheme, qualifierEncodingScheme, encodedColumnQualifierCounter, useStatsForQueryPlan); return result; } catch (SQLException e) { throw new RuntimeException(e); // Impossible @@ -1432,6 +1439,7 @@ public class PTableImpl implements PTable { if (table.getEncodingScheme() != null) { builder.setEncodingScheme(ByteStringer.wrap(new byte[]{table.getEncodingScheme().getSerializedMetadataValue()})); } + builder.setUseStatsForQueryPlan(table.useStatsForQueryPlan()); return builder.build(); } @@ -1517,6 +1525,11 @@ public class PTableImpl implements PTable { return qualifierEncodingScheme; } + @Override + public boolean useStatsForQueryPlan() { + return useStatsForQueryPlan; + } + private static final class KVColumnFamilyQualifier { @Nonnull private final String colFamilyName; http://git-wip-us.apache.org/repos/asf/phoenix/blob/3872648f/phoenix-core/src/main/java/org/apache/phoenix/schema/TableProperty.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/TableProperty.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/TableProperty.java index 3282cc1..71698d7 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/schema/TableProperty.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/TableProperty.java @@ -186,6 +186,24 @@ public enum TableProperty { return table.getImmutableStorageScheme(); } + }, + + USE_STATS_FOR_QUERY_PLAN(PhoenixDatabaseMetaData.USE_STATS_FOR_QUERY_PLAN, true, true, true) { + @Override + public Object getValue(Object value) { + if (value == null) { + return null; + } else if (value instanceof Boolean) { + return value; + } else { + throw new IllegalArgumentException("Use stats for query plan table property can only be either true or false"); + } + } + + @Override + public Object getPTableValue(PTable table) { + return table.useStatsForQueryPlan(); + } } ; http://git-wip-us.apache.org/repos/asf/phoenix/blob/3872648f/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsCollectorFactory.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsCollectorFactory.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsCollectorFactory.java index bbc7cd4..1fe329f 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsCollectorFactory.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsCollectorFactory.java @@ -17,10 +17,15 @@ */ package org.apache.phoenix.schema.stats; +import static org.apache.phoenix.query.QueryServices.STATS_COLLECTION_ENABLED; +import static org.apache.phoenix.query.QueryServices.STATS_ENABLED_ATTRIB; +import static org.apache.phoenix.query.QueryServicesOptions.DEFAULT_STATS_COLLECTION_ENABLED; + import java.io.IOException; import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment; import org.apache.phoenix.query.QueryServices; +import org.apache.phoenix.query.QueryServicesOptions; /** * Provides new {@link StatisticsCollector} instances based on configuration settings for a @@ -59,8 +64,12 @@ public class StatisticsCollectorFactory { * use case for that. */ private static boolean statisticsEnabled(RegionCoprocessorEnvironment env) { - return env.getConfiguration().getBoolean(QueryServices.STATS_ENABLED_ATTRIB, true) && - StatisticsUtil.isStatsEnabled(env.getRegionInfo().getTable()); + return (env.getConfiguration().getBoolean(STATS_COLLECTION_ENABLED, + DEFAULT_STATS_COLLECTION_ENABLED) + // old config left here for backward compatibility. TODO: remove in the next major release + || env.getConfiguration().getBoolean(STATS_ENABLED_ATTRIB, true) + ) + && StatisticsUtil.isStatsEnabled(env.getRegionInfo().getTable()); } } http://git-wip-us.apache.org/repos/asf/phoenix/blob/3872648f/phoenix-core/src/main/java/org/apache/phoenix/util/UpgradeUtil.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/UpgradeUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/util/UpgradeUtil.java index 5bd3ef1..e770f63 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/util/UpgradeUtil.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/util/UpgradeUtil.java @@ -1079,7 +1079,7 @@ public class UpgradeUtil { * @param oldMetaConnection caller should take care of closing the passed connection appropriately * @throws SQLException */ - public static void upgradeTo4_11_0(PhoenixConnection oldMetaConnection) throws SQLException { + public static void addParentToChildLinks(PhoenixConnection oldMetaConnection) throws SQLException { PhoenixConnection metaConnection = null; try { // Need to use own connection with max time stamp to be able to read all data from SYSTEM.CATALOG http://git-wip-us.apache.org/repos/asf/phoenix/blob/3872648f/phoenix-core/src/test/java/org/apache/phoenix/execute/CorrelatePlanTest.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/test/java/org/apache/phoenix/execute/CorrelatePlanTest.java b/phoenix-core/src/test/java/org/apache/phoenix/execute/CorrelatePlanTest.java index e04c787..76757b0 100644 --- a/phoenix-core/src/test/java/org/apache/phoenix/execute/CorrelatePlanTest.java +++ b/phoenix-core/src/test/java/org/apache/phoenix/execute/CorrelatePlanTest.java @@ -262,7 +262,7 @@ public class CorrelatePlanTest { PTableType.SUBQUERY, null, MetaDataProtocol.MIN_TABLE_TIMESTAMP, PTable.INITIAL_SEQ_NUM, null, null, columns, null, null, Collections.<PTable>emptyList(), false, Collections.<PName>emptyList(), null, null, false, false, false, null, - null, null, true, false, 0, 0L, Boolean.FALSE, null, false, ImmutableStorageScheme.ONE_CELL_PER_COLUMN, QualifierEncodingScheme.NON_ENCODED_QUALIFIERS, EncodedCQCounter.NULL_COUNTER); + null, null, true, false, 0, 0L, Boolean.FALSE, null, false, ImmutableStorageScheme.ONE_CELL_PER_COLUMN, QualifierEncodingScheme.NON_ENCODED_QUALIFIERS, EncodedCQCounter.NULL_COUNTER, true); TableRef sourceTable = new TableRef(pTable); List<ColumnRef> sourceColumnRefs = Lists.<ColumnRef> newArrayList(); for (PColumn column : sourceTable.getTable().getColumns()) {
