Repository: phoenix Updated Branches: refs/heads/4.x-HBase-1.2 e4b63d8c8 -> c6a10dc69
PHOENIX-4592 BaseResultIterators.getStatsForParallelizationProp() should use retry looking up the table without tenantId if cannot find the table using the tenantId Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/c6a10dc6 Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/c6a10dc6 Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/c6a10dc6 Branch: refs/heads/4.x-HBase-1.2 Commit: c6a10dc698133f339731ccade68231e9704cd1a0 Parents: e4b63d8 Author: Thomas D'Silva <tdsi...@apache.org> Authored: Tue Feb 13 14:31:42 2018 -0800 Committer: Thomas D'Silva <tdsi...@apache.org> Committed: Fri Feb 16 10:22:40 2018 -0800 ---------------------------------------------------------------------- .../end2end/ExplainPlanWithStatsEnabledIT.java | 32 +++++++++++++ .../coprocessor/MetaDataEndpointImpl.java | 9 ++-- .../phoenix/iterate/BaseResultIterators.java | 41 ++--------------- .../util/PhoenixConfigurationUtil.java | 48 ++++++++++++++++++++ 4 files changed, 89 insertions(+), 41 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/phoenix/blob/c6a10dc6/phoenix-core/src/it/java/org/apache/phoenix/end2end/ExplainPlanWithStatsEnabledIT.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/ExplainPlanWithStatsEnabledIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/ExplainPlanWithStatsEnabledIT.java index f13510b..f369be9 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/ExplainPlanWithStatsEnabledIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/ExplainPlanWithStatsEnabledIT.java @@ -17,6 +17,7 @@ */ package org.apache.phoenix.end2end; +import static org.apache.phoenix.query.QueryServicesOptions.DEFAULT_USE_STATS_FOR_PARALLELIZATION; import static org.apache.phoenix.util.PhoenixRuntime.TENANT_ID_ATTRIB; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -34,7 +35,9 @@ import org.apache.hadoop.hbase.client.HBaseAdmin; import org.apache.hadoop.hbase.util.Bytes; import org.apache.phoenix.jdbc.PhoenixConnection; import org.apache.phoenix.jdbc.PhoenixResultSet; +import org.apache.phoenix.mapreduce.util.PhoenixConfigurationUtil; import org.apache.phoenix.query.BaseTest; +import org.apache.phoenix.schema.PTable; import org.apache.phoenix.schema.PTableKey; import org.apache.phoenix.schema.TableNotFoundException; import org.apache.phoenix.util.EnvironmentEdge; @@ -1104,4 +1107,33 @@ public class ExplainPlanWithStatsEnabledIT extends ParallelStatsEnabledIT { assertEquals("B", rs.getString(1)); } + @Test + public void testUseStatsForParallelizationProperyOnViewIndex() throws SQLException { + String tableName = generateUniqueName(); + String viewName = generateUniqueName(); + String tenantViewName = generateUniqueName(); + String viewIndexName = generateUniqueName(); + boolean useStats = !DEFAULT_USE_STATS_FOR_PARALLELIZATION; + try (Connection conn = DriverManager.getConnection(getUrl())) { + conn.createStatement() + .execute("create table " + tableName + + "(tenantId CHAR(15) NOT NULL, pk1 integer NOT NULL, v varchar CONSTRAINT PK PRIMARY KEY " + + "(tenantId, pk1)) MULTI_TENANT=true"); + try (Connection tenantConn = getTenantConnection("tenant1")) { + conn.createStatement().execute("CREATE VIEW " + viewName + " AS SELECT * FROM " + tableName); + conn.createStatement().execute("CREATE INDEX " + viewIndexName + " on " + viewName + " (v) "); + tenantConn.createStatement().execute("CREATE VIEW " + tenantViewName + " AS SELECT * FROM " + viewName); + conn.createStatement() + .execute("ALTER TABLE " + tableName + " set USE_STATS_FOR_PARALLELIZATION=" + useStats); + // fetch the latest view ptable + PhoenixRuntime.getTableNoCache(tenantConn, viewName); + PhoenixConnection phxConn = conn.unwrap(PhoenixConnection.class); + PTable viewIndex = phxConn.getTable(new PTableKey(phxConn.getTenantId(), viewIndexName)); + assertEquals("USE_STATS_FOR_PARALLELIZATION property set incorrectly", useStats, + PhoenixConfigurationUtil + .getStatsForParallelizationProp(tenantConn.unwrap(PhoenixConnection.class), viewIndex)); + } + } + } + } http://git-wip-us.apache.org/repos/asf/phoenix/blob/c6a10dc6/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java index d08db27..11e5e45 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java @@ -2672,9 +2672,12 @@ public class MetaDataEndpointImpl extends MetaDataProtocol implements Coprocesso for (TableProperty tableProp : TableProperty.values()) { Cell tablePropertyCell = tablePropertyCellMap.get(tableProp); if ( tablePropertyCell != null) { - // set this table property on the view if it is not mutable on a view (which means the property is always the same as the base table) - // or if it is mutable on a view and the property value is the same as the base table property (which means it wasn't changed on the view) - if (!tableProp.isMutableOnView() || tableProp.getPTableValue(view).equals(tableProp.getPTableValue(basePhysicalTable))) { + // set this table property on the view : + // 1. if it is not mutable on a view (which means the property is always the same as the base table) + // 2. or if it is mutable on a view and if it doesn't exist on the view + // 3. or if it is mutable on a view and the property value is the same as the base table property (which means it wasn't changed on the view) + Object viewProp = tableProp.getPTableValue(view); + if (!tableProp.isMutableOnView() || viewProp==null || viewProp.equals(tableProp.getPTableValue(basePhysicalTable))) { viewHeaderRowPut.add(CellUtil.createCell(viewKey, CellUtil.cloneFamily(tablePropertyCell), CellUtil.cloneQualifier(tablePropertyCell), clientTimeStamp, tablePropertyCell.getTypeByte(), CellUtil.cloneValue(tablePropertyCell))); http://git-wip-us.apache.org/repos/asf/phoenix/blob/c6a10dc6/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java b/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java index 25722a9..0484748 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java @@ -23,8 +23,6 @@ import static org.apache.phoenix.coprocessor.BaseScannerRegionObserver.SCAN_STAR import static org.apache.phoenix.coprocessor.BaseScannerRegionObserver.SCAN_STOP_ROW_SUFFIX; import static org.apache.phoenix.monitoring.GlobalClientMetrics.GLOBAL_FAILED_QUERY_COUNTER; import static org.apache.phoenix.monitoring.GlobalClientMetrics.GLOBAL_QUERY_TIMEOUT_COUNTER; -import static org.apache.phoenix.query.QueryServices.USE_STATS_FOR_PARALLELIZATION; -import static org.apache.phoenix.query.QueryServicesOptions.DEFAULT_USE_STATS_FOR_PARALLELIZATION; import static org.apache.phoenix.schema.PTable.IndexType.LOCAL; import static org.apache.phoenix.schema.PTableType.INDEX; import static org.apache.phoenix.util.ByteUtil.EMPTY_BYTE_ARRAY; @@ -83,8 +81,8 @@ import org.apache.phoenix.filter.DistinctPrefixFilter; import org.apache.phoenix.filter.EncodedQualifiersColumnProjectionFilter; import org.apache.phoenix.hbase.index.util.ImmutableBytesPtr; import org.apache.phoenix.hbase.index.util.VersionUtil; -import org.apache.phoenix.jdbc.PhoenixConnection; import org.apache.phoenix.join.HashCacheClient; +import org.apache.phoenix.mapreduce.util.PhoenixConfigurationUtil; import org.apache.phoenix.parse.FilterableStatement; import org.apache.phoenix.parse.HintNode; import org.apache.phoenix.parse.HintNode.Hint; @@ -101,11 +99,9 @@ import org.apache.phoenix.schema.PTable.ImmutableStorageScheme; import org.apache.phoenix.schema.PTable.IndexType; import org.apache.phoenix.schema.PTable.QualifierEncodingScheme; import org.apache.phoenix.schema.PTable.ViewType; -import org.apache.phoenix.schema.PTableKey; import org.apache.phoenix.schema.PTableType; import org.apache.phoenix.schema.RowKeySchema; import org.apache.phoenix.schema.StaleRegionBoundaryCacheException; -import org.apache.phoenix.schema.TableNotFoundException; import org.apache.phoenix.schema.TableRef; import org.apache.phoenix.schema.ValueSchema.Field; import org.apache.phoenix.schema.stats.GuidePostsInfo; @@ -142,7 +138,7 @@ import com.google.common.collect.Lists; * @since 0.1 */ public abstract class BaseResultIterators extends ExplainTable implements ResultIterators { - private static final Logger logger = LoggerFactory.getLogger(BaseResultIterators.class); + public static final Logger logger = LoggerFactory.getLogger(BaseResultIterators.class); private static final int ESTIMATED_GUIDEPOSTS_PER_REGION = 20; private static final int MIN_SEEK_TO_COLUMN_VERSION = VersionUtil.encodeVersion("0", "98", "12"); private final List<List<Scan>> scans; @@ -501,7 +497,7 @@ public abstract class BaseResultIterators extends ExplainTable implements Result scanId = new UUID(ThreadLocalRandom.current().nextLong(), ThreadLocalRandom.current().nextLong()).toString(); initializeScan(plan, perScanLimit, offset, scan); - this.useStatsForParallelization = getStatsForParallelizationProp(context, table); + this.useStatsForParallelization = PhoenixConfigurationUtil.getStatsForParallelizationProp(context.getConnection(), table); this.scans = getParallelScans(); List<KeyRange> splitRanges = Lists.newArrayListWithExpectedSize(scans.size() * ESTIMATED_GUIDEPOSTS_PER_REGION); for (List<Scan> scanList : scans) { @@ -1439,35 +1435,4 @@ public abstract class BaseResultIterators extends ExplainTable implements Result return this.estimateInfoTimestamp; } - private boolean getStatsForParallelizationProp(StatementContext context, PTable table) { - Boolean useStats = table.useStatsForParallelization(); - if (useStats != null) { - return useStats; - } - /* - * For a view index, we use the property set on view. For indexes on base table, whether - * global or local, we use the property set on the base table. Null check needed when - * dropping local indexes. - */ - if (table.getType() == PTableType.INDEX && table.getParentName() != null) { - PhoenixConnection conn = context.getConnection(); - String parentTableName = table.getParentName().getString(); - try { - PTable parentTable = - conn.getTable(new PTableKey(conn.getTenantId(), parentTableName)); - useStats = parentTable.useStatsForParallelization(); - if (useStats != null) { - return useStats; - } - } catch (TableNotFoundException e) { - logger.warn("Unable to find parent table \"" + parentTableName + "\" of table \"" - + table.getName().getString() - + "\" to determine USE_STATS_FOR_PARALLELIZATION", - e); - } - } - return context.getConnection().getQueryServices().getConfiguration() - .getBoolean(USE_STATS_FOR_PARALLELIZATION, DEFAULT_USE_STATS_FOR_PARALLELIZATION); - } - } http://git-wip-us.apache.org/repos/asf/phoenix/blob/c6a10dc6/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/util/PhoenixConfigurationUtil.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/util/PhoenixConfigurationUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/util/PhoenixConfigurationUtil.java index db11f7d..3c27f65 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/util/PhoenixConfigurationUtil.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/util/PhoenixConfigurationUtil.java @@ -18,6 +18,8 @@ package org.apache.phoenix.mapreduce.util; import static org.apache.commons.lang.StringUtils.isNotEmpty; +import static org.apache.phoenix.query.QueryServices.USE_STATS_FOR_PARALLELIZATION; +import static org.apache.phoenix.query.QueryServicesOptions.DEFAULT_USE_STATS_FOR_PARALLELIZATION; import java.io.IOException; import java.sql.Connection; @@ -37,12 +39,18 @@ import org.apache.hadoop.mapreduce.Job; import org.apache.hadoop.mapreduce.lib.db.DBInputFormat.NullDBWritable; import org.apache.hadoop.mapreduce.lib.db.DBWritable; import org.apache.hadoop.util.ReflectionUtils; +import org.apache.phoenix.iterate.BaseResultIterators; import org.apache.phoenix.jdbc.PhoenixConnection; import org.apache.phoenix.mapreduce.FormatToBytesWritableMapper; import org.apache.phoenix.mapreduce.ImportPreUpsertKeyValueProcessor; import org.apache.phoenix.mapreduce.PhoenixInputFormat; import org.apache.phoenix.mapreduce.index.IndexScrutinyTool.OutputFormat; import org.apache.phoenix.mapreduce.index.IndexScrutinyTool.SourceTable; +import org.apache.phoenix.schema.PName; +import org.apache.phoenix.schema.PTable; +import org.apache.phoenix.schema.PTableKey; +import org.apache.phoenix.schema.PTableType; +import org.apache.phoenix.schema.TableNotFoundException; import org.apache.phoenix.util.ColumnInfo; import org.apache.phoenix.util.PhoenixRuntime; import org.apache.phoenix.util.QueryUtil; @@ -610,4 +618,44 @@ public final class PhoenixConfigurationUtil { boolean split = configuration.getBoolean(MAPREDUCE_SPLIT_BY_STATS, DEFAULT_SPLIT_BY_STATS); return split; } + + public static boolean getStatsForParallelizationProp(PhoenixConnection conn, PTable table) { + Boolean useStats = table.useStatsForParallelization(); + if (useStats != null) { + return useStats; + } + /* + * For a view index, we use the property set on view. For indexes on base table, whether + * global or local, we use the property set on the base table. Null check needed when + * dropping local indexes. + */ + PName tenantId = conn.getTenantId(); + int retryCount = 0; + while (retryCount++<2) { + if (table.getType() == PTableType.INDEX && table.getParentName() != null) { + String parentTableName = table.getParentName().getString(); + try { + PTable parentTable = + conn.getTable(new PTableKey(tenantId, parentTableName)); + useStats = parentTable.useStatsForParallelization(); + if (useStats != null) { + return useStats; + } + } catch (TableNotFoundException e) { + // try looking up the table without the tenant id (for + // global tables) + if (tenantId != null) { + tenantId = null; + } else { + BaseResultIterators.logger.warn( + "Unable to find parent table \"" + parentTableName + "\" of table \"" + + table.getName().getString() + "\" to determine USE_STATS_FOR_PARALLELIZATION", + e); + } + } + } + } + return conn.getQueryServices().getConfiguration() + .getBoolean(USE_STATS_FOR_PARALLELIZATION, DEFAULT_USE_STATS_FOR_PARALLELIZATION); + } }
