Repository: phoenix Updated Branches: refs/heads/4.x-HBase-1.3 d175c6465 -> f73f98c7a
http://git-wip-us.apache.org/repos/asf/phoenix/blob/f73f98c7/phoenix-core/src/main/java/org/apache/phoenix/monitoring/MutationMetricQueue.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/monitoring/MutationMetricQueue.java b/phoenix-core/src/main/java/org/apache/phoenix/monitoring/MutationMetricQueue.java index 3de2be1..1256f5c 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/monitoring/MutationMetricQueue.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/monitoring/MutationMetricQueue.java @@ -27,6 +27,8 @@ import java.util.HashMap; import java.util.Map; import java.util.Map.Entry; +import org.apache.phoenix.log.LogLevel; + /** * Queue that tracks various writes/mutations related phoenix request metrics. */ @@ -81,12 +83,16 @@ public class MutationMetricQueue { * Class that holds together the various metrics associated with mutations. */ public static class MutationMetric { - private final CombinableMetric numMutations = new CombinableMetricImpl(MUTATION_BATCH_SIZE); - private final CombinableMetric mutationsSizeBytes = new CombinableMetricImpl(MUTATION_BYTES); - private final CombinableMetric totalCommitTimeForMutations = new CombinableMetricImpl(MUTATION_COMMIT_TIME); - private final CombinableMetric numFailedMutations = new CombinableMetricImpl(MUTATION_BATCH_FAILED_SIZE); - - public MutationMetric(long numMutations, long mutationsSizeBytes, long commitTimeForMutations, long numFailedMutations) { + private final CombinableMetric numMutations;; + private final CombinableMetric mutationsSizeBytes; + private final CombinableMetric totalCommitTimeForMutations; + private final CombinableMetric numFailedMutations; + + public MutationMetric(LogLevel connectionLogLevel, long numMutations, long mutationsSizeBytes, long commitTimeForMutations, long numFailedMutations) { + this.numMutations = MetricUtil.getCombinableMetric(connectionLogLevel,MUTATION_BATCH_SIZE); + this.mutationsSizeBytes =MetricUtil.getCombinableMetric(connectionLogLevel,MUTATION_BYTES); + this.totalCommitTimeForMutations =MetricUtil.getCombinableMetric(connectionLogLevel,MUTATION_COMMIT_TIME); + this.numFailedMutations = MetricUtil.getCombinableMetric(connectionLogLevel,MUTATION_BATCH_FAILED_SIZE); this.numMutations.change(numMutations); this.mutationsSizeBytes.change(mutationsSizeBytes); this.totalCommitTimeForMutations.change(commitTimeForMutations); http://git-wip-us.apache.org/repos/asf/phoenix/blob/f73f98c7/phoenix-core/src/main/java/org/apache/phoenix/monitoring/OverAllQueryMetrics.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/monitoring/OverAllQueryMetrics.java b/phoenix-core/src/main/java/org/apache/phoenix/monitoring/OverAllQueryMetrics.java index b995267..3121ecd 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/monitoring/OverAllQueryMetrics.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/monitoring/OverAllQueryMetrics.java @@ -27,7 +27,7 @@ import static org.apache.phoenix.monitoring.MetricType.WALL_CLOCK_TIME_MS; import java.util.HashMap; import java.util.Map; -import org.apache.phoenix.monitoring.CombinableMetric.NoOpRequestMetric; +import org.apache.phoenix.log.LogLevel; /** * Class that represents the overall metrics associated with a query being executed by the phoenix. @@ -42,16 +42,15 @@ public class OverAllQueryMetrics { private final CombinableMetric queryFailed; private final CombinableMetric cacheRefreshedDueToSplits; - public OverAllQueryMetrics(boolean isMetricsEnabled) { - queryWatch = new MetricsStopWatch(isMetricsEnabled); - resultSetWatch = new MetricsStopWatch(isMetricsEnabled); - numParallelScans = isMetricsEnabled ? new CombinableMetricImpl(NUM_PARALLEL_SCANS) : NoOpRequestMetric.INSTANCE; - wallClockTimeMS = isMetricsEnabled ? new CombinableMetricImpl(WALL_CLOCK_TIME_MS) : NoOpRequestMetric.INSTANCE; - resultSetTimeMS = isMetricsEnabled ? new CombinableMetricImpl(RESULT_SET_TIME_MS) : NoOpRequestMetric.INSTANCE; - queryTimedOut = isMetricsEnabled ? new CombinableMetricImpl(QUERY_TIMEOUT_COUNTER) : NoOpRequestMetric.INSTANCE; - queryFailed = isMetricsEnabled ? new CombinableMetricImpl(QUERY_FAILED_COUNTER) : NoOpRequestMetric.INSTANCE; - cacheRefreshedDueToSplits = isMetricsEnabled ? new CombinableMetricImpl(CACHE_REFRESH_SPLITS_COUNTER) - : NoOpRequestMetric.INSTANCE; + public OverAllQueryMetrics(LogLevel connectionLogLevel) { + queryWatch = new MetricsStopWatch(WALL_CLOCK_TIME_MS.isLoggingEnabled(connectionLogLevel)); + resultSetWatch = new MetricsStopWatch(RESULT_SET_TIME_MS.isLoggingEnabled(connectionLogLevel)); + numParallelScans = MetricUtil.getCombinableMetric(connectionLogLevel, NUM_PARALLEL_SCANS); + wallClockTimeMS = MetricUtil.getCombinableMetric(connectionLogLevel, WALL_CLOCK_TIME_MS); + resultSetTimeMS = MetricUtil.getCombinableMetric(connectionLogLevel, RESULT_SET_TIME_MS); + queryTimedOut = MetricUtil.getCombinableMetric(connectionLogLevel, QUERY_TIMEOUT_COUNTER); + queryFailed = MetricUtil.getCombinableMetric(connectionLogLevel, QUERY_FAILED_COUNTER); + cacheRefreshedDueToSplits = MetricUtil.getCombinableMetric(connectionLogLevel, CACHE_REFRESH_SPLITS_COUNTER); } public void updateNumParallelScans(long numParallelScans) { http://git-wip-us.apache.org/repos/asf/phoenix/blob/f73f98c7/phoenix-core/src/main/java/org/apache/phoenix/monitoring/ReadMetricQueue.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/monitoring/ReadMetricQueue.java b/phoenix-core/src/main/java/org/apache/phoenix/monitoring/ReadMetricQueue.java index c008635..fa81e9c 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/monitoring/ReadMetricQueue.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/monitoring/ReadMetricQueue.java @@ -25,6 +25,7 @@ import java.util.concurrent.LinkedBlockingQueue; import javax.annotation.Nonnull; +import org.apache.phoenix.log.LogLevel; import org.apache.phoenix.monitoring.CombinableMetric.NoOpRequestMetric; import com.google.common.annotations.VisibleForTesting; @@ -40,19 +41,23 @@ public class ReadMetricQueue { private final List<ScanMetricsHolder> scanMetricsHolderList = new ArrayList<ScanMetricsHolder>(); - private final boolean isRequestMetricsEnabled; - public ReadMetricQueue(boolean isRequestMetricsEnabled) { - this.isRequestMetricsEnabled = isRequestMetricsEnabled; + private LogLevel connectionLogLevel; + + public ReadMetricQueue(LogLevel connectionLogLevel) { + this.connectionLogLevel = connectionLogLevel; } public CombinableMetric allotMetric(MetricType type, String tableName) { - if (!isRequestMetricsEnabled) { return NoOpRequestMetric.INSTANCE; } - MetricKey key = new MetricKey(type, tableName); - Queue<CombinableMetric> q = getMetricQueue(key); - CombinableMetric metric = getMetric(type); - q.offer(metric); - return metric; + if (type.isLoggingEnabled(connectionLogLevel)) { + MetricKey key = new MetricKey(type, tableName); + Queue<CombinableMetric> q = getMetricQueue(key); + CombinableMetric metric = getMetric(type); + q.offer(metric); + return metric; + } else { + return NoOpRequestMetric.INSTANCE; + } } @VisibleForTesting @@ -173,10 +178,6 @@ public class ReadMetricQueue { return q; } - public boolean isRequestMetricsEnabled() { - return isRequestMetricsEnabled; - } - public void addScanHolder(ScanMetricsHolder holder){ scanMetricsHolderList.add(holder); } http://git-wip-us.apache.org/repos/asf/phoenix/blob/f73f98c7/phoenix-core/src/main/java/org/apache/phoenix/monitoring/ScanMetricsHolder.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/monitoring/ScanMetricsHolder.java b/phoenix-core/src/main/java/org/apache/phoenix/monitoring/ScanMetricsHolder.java index 9125cd8..494b3e6 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/monitoring/ScanMetricsHolder.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/monitoring/ScanMetricsHolder.java @@ -34,6 +34,7 @@ import java.util.Map; import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.util.JsonMapper; +import org.apache.phoenix.log.LogLevel; public class ScanMetricsHolder { @@ -52,13 +53,11 @@ public class ScanMetricsHolder { private Object scan; private static final ScanMetricsHolder NO_OP_INSTANCE = - new ScanMetricsHolder(new ReadMetricQueue(false), "",null); + new ScanMetricsHolder(new ReadMetricQueue(LogLevel.OFF), "",null); public static ScanMetricsHolder getInstance(ReadMetricQueue readMetrics, String tableName, - Scan scan, boolean isRequestMetricsEnabled) { - if (!isRequestMetricsEnabled) { - return NO_OP_INSTANCE; - } + Scan scan, LogLevel connectionLogLevel) { + if (connectionLogLevel == LogLevel.OFF) { return NO_OP_INSTANCE; } scan.setScanMetricsEnabled(true); return new ScanMetricsHolder(readMetrics, tableName, scan); } http://git-wip-us.apache.org/repos/asf/phoenix/blob/f73f98c7/phoenix-core/src/main/java/org/apache/phoenix/monitoring/SpoolingMetricsHolder.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/monitoring/SpoolingMetricsHolder.java b/phoenix-core/src/main/java/org/apache/phoenix/monitoring/SpoolingMetricsHolder.java index 4373887..699982f 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/monitoring/SpoolingMetricsHolder.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/monitoring/SpoolingMetricsHolder.java @@ -17,6 +17,7 @@ */ package org.apache.phoenix.monitoring; +import org.apache.phoenix.log.LogLevel; /** * Class that encapsulates the various metrics associated with the spooling done by phoenix as part of servicing a @@ -26,7 +27,7 @@ public class SpoolingMetricsHolder { private final CombinableMetric spoolFileSizeMetric; private final CombinableMetric numSpoolFileMetric; - public static final SpoolingMetricsHolder NO_OP_INSTANCE = new SpoolingMetricsHolder(new ReadMetricQueue(false), ""); + public static final SpoolingMetricsHolder NO_OP_INSTANCE = new SpoolingMetricsHolder(new ReadMetricQueue(LogLevel.OFF), ""); public SpoolingMetricsHolder(ReadMetricQueue readMetrics, String tableName) { this.spoolFileSizeMetric = readMetrics.allotMetric(MetricType.SPOOL_FILE_SIZE, tableName); http://git-wip-us.apache.org/repos/asf/phoenix/blob/f73f98c7/phoenix-core/src/main/java/org/apache/phoenix/monitoring/TaskExecutionMetricsHolder.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/monitoring/TaskExecutionMetricsHolder.java b/phoenix-core/src/main/java/org/apache/phoenix/monitoring/TaskExecutionMetricsHolder.java index 98ff57c..6117b40 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/monitoring/TaskExecutionMetricsHolder.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/monitoring/TaskExecutionMetricsHolder.java @@ -23,6 +23,8 @@ import static org.apache.phoenix.monitoring.MetricType.TASK_EXECUTION_TIME; import static org.apache.phoenix.monitoring.MetricType.TASK_QUEUE_WAIT_TIME; import static org.apache.phoenix.monitoring.MetricType.TASK_REJECTED_COUNTER; +import org.apache.phoenix.log.LogLevel; + /** * Class to encapsulate the various metrics associated with submitting and executing a task to the phoenix client @@ -35,7 +37,7 @@ public class TaskExecutionMetricsHolder { private final CombinableMetric taskExecutionTime; private final CombinableMetric numTasks; private final CombinableMetric numRejectedTasks; - public static final TaskExecutionMetricsHolder NO_OP_INSTANCE = new TaskExecutionMetricsHolder(new ReadMetricQueue(false), ""); + public static final TaskExecutionMetricsHolder NO_OP_INSTANCE = new TaskExecutionMetricsHolder(new ReadMetricQueue(LogLevel.OFF), ""); public TaskExecutionMetricsHolder(ReadMetricQueue readMetrics, String tableName) { taskQueueWaitTime = readMetrics.allotMetric(TASK_QUEUE_WAIT_TIME, tableName); http://git-wip-us.apache.org/repos/asf/phoenix/blob/f73f98c7/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java index 1d3feed..8b7b708 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java @@ -2471,7 +2471,12 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement // Available for testing protected String getLogTableDDL() { - return QueryConstants.CREATE_LOG_METADATA; + return setSystemLogDDLProperties(QueryConstants.CREATE_LOG_METADATA); + } + + private String setSystemLogDDLProperties(String ddl) { + return String.format(ddl, props.getInt(LOG_SALT_BUCKETS_ATTRIB, QueryServicesOptions.DEFAULT_LOG_SALT_BUCKETS)); + } private String setSystemDDLProperties(String ddl) { http://git-wip-us.apache.org/repos/asf/phoenix/blob/f73f98c7/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionlessQueryServicesImpl.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionlessQueryServicesImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionlessQueryServicesImpl.java index 3531a87..a7fad8e 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionlessQueryServicesImpl.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionlessQueryServicesImpl.java @@ -164,7 +164,12 @@ public class ConnectionlessQueryServicesImpl extends DelegateQueryServices imple } protected String getLogTableDDL() { - return QueryConstants.CREATE_LOG_METADATA; + return setSystemLogDDLProperties(QueryConstants.CREATE_LOG_METADATA); + } + + private String setSystemLogDDLProperties(String ddl) { + return String.format(ddl, props.getInt(LOG_SALT_BUCKETS_ATTRIB, QueryServicesOptions.DEFAULT_LOG_SALT_BUCKETS)); + } private String setSystemDDLProperties(String ddl) { http://git-wip-us.apache.org/repos/asf/phoenix/blob/f73f98c7/phoenix-core/src/main/java/org/apache/phoenix/query/QueryConstants.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryConstants.java b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryConstants.java index 65806ae..52abed0 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryConstants.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryConstants.java @@ -18,112 +18,7 @@ package org.apache.phoenix.query; -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.APPEND_ONLY_SCHEMA; -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.ARG_POSITION; -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.ARRAY_SIZE; -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.AUTO_PARTITION_SEQ; -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.BASE_COLUMN_COUNT; -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.BIND_PARAMETERS; -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.BUFFER_LENGTH; -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.CACHE_SIZE; -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.CHAR_OCTET_LENGTH; -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.CLASS_NAME; -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.CLIENT_IP; -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.COLUMN_COUNT; -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.COLUMN_DEF; -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.COLUMN_FAMILY; -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.COLUMN_NAME; -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.COLUMN_QUALIFIER; -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.COLUMN_QUALIFIER_COUNTER; -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.COLUMN_SIZE; -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.CURRENT_VALUE; -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.CYCLE_FLAG; -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.DATA_TABLE_NAME; -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.DATA_TYPE; -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.DECIMAL_DIGITS; -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.DEFAULT_COLUMN_FAMILY_NAME; -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.DEFAULT_VALUE; -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.DISABLE_WAL; -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.ENCODING_SCHEME; -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.EXCEPTION_TRACE; -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.EXPLAIN_PLAN; -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.FUNCTION_NAME; -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.GLOBAL_SCAN_DETAILS; -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.GUIDE_POSTS_ROW_COUNT; -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.GUIDE_POSTS_WIDTH; -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.GUIDE_POST_KEY; -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.IMMUTABLE_ROWS; -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.IMMUTABLE_STORAGE_SCHEME; -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.INCREMENT_BY; -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.INDEX_DISABLE_TIMESTAMP; -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.INDEX_STATE; -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.INDEX_TYPE; -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.IS_ARRAY; -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.IS_AUTOINCREMENT; -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.IS_CONSTANT; -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.IS_NAMESPACE_MAPPED; -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.IS_NULLABLE; -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.IS_ROW_TIMESTAMP; -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.IS_VIEW_REFERENCED; -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.JAR_PATH; -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.KEY_SEQ; -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.LAST_STATS_UPDATE_TIME; -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.LIMIT_REACHED_FLAG; -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.LINK_TYPE; -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.MAX_VALUE; -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.MIN_VALUE; -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.MULTI_TENANT; -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.NO_OF_RESULTS_ITERATED; -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.NULLABLE; -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.NUM_ARGS; -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.NUM_PREC_RADIX; -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.ORDINAL_POSITION; -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.PHYSICAL_NAME; -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.PK_NAME; -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.QUERY; -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.QUERY_ID; -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.QUERY_STATUS; -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.REF_GENERATION; -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.REMARKS; -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.RETURN_TYPE; -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SALT_BUCKETS; -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SCAN_METRICS_JSON; -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SCOPE_CATALOG; -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SCOPE_SCHEMA; -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SCOPE_TABLE; -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SELF_REFERENCING_COL_NAME; -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SEQUENCE_NAME; -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SEQUENCE_SCHEMA; -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SORT_ORDER; -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SOURCE_DATA_TYPE; -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SQL_DATA_TYPE; -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SQL_DATETIME_SUB; -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.START_TIME; -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.START_WITH; -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.STORE_NULLS; -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SYSTEM_CATALOG_SCHEMA; -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SYSTEM_CATALOG_TABLE; -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SYSTEM_FUNCTION_TABLE; -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SYSTEM_LOG_TABLE; -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SYSTEM_STATS_TABLE; -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TABLE_NAME; -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TABLE_SCHEM; -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TABLE_SEQ_NUM; -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TABLE_TYPE; -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TENANT_ID; -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TOTAL_EXECUTION_TIME; -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TRANSACTIONAL; -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TRANSACTION_PROVIDER; -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TYPE; -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TYPE_NAME; -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TYPE_SEQUENCE; -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.UPDATE_CACHE_FREQUENCY; -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.USER; -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.USE_STATS_FOR_PARALLELIZATION; -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VIEW_CONSTANT; -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VIEW_INDEX_ID; -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VIEW_STATEMENT; -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VIEW_TYPE; +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.*; import java.math.BigDecimal; @@ -134,9 +29,11 @@ import org.apache.hadoop.hbase.util.Bytes; import org.apache.phoenix.coprocessor.MetaDataProtocol; import org.apache.phoenix.hbase.index.util.ImmutableBytesPtr; import org.apache.phoenix.jdbc.PhoenixDatabaseMetaData; +import org.apache.phoenix.monitoring.MetricType; import org.apache.phoenix.schema.MetaDataSplitPolicy; import org.apache.phoenix.schema.PName; import org.apache.phoenix.schema.PNameFactory; +import org.apache.phoenix.schema.PTable.ImmutableStorageScheme; import org.apache.phoenix.schema.PTable.QualifierEncodingScheme; import org.apache.phoenix.schema.SortOrder; import org.apache.phoenix.schema.SystemFunctionSplitPolicy; @@ -412,28 +309,31 @@ public interface QueryConstants { PhoenixDatabaseMetaData.TRANSACTIONAL + "=" + Boolean.FALSE; public static final String CREATE_LOG_METADATA = - "CREATE TABLE " + SYSTEM_CATALOG_SCHEMA + ".\"" + SYSTEM_LOG_TABLE + "\"(\n" + + "CREATE IMMUTABLE TABLE " + SYSTEM_CATALOG_SCHEMA + ".\"" + SYSTEM_LOG_TABLE + "\"(\n" + // Pk columns + START_TIME + " DECIMAL, \n" + + TABLE_NAME + " VARCHAR, \n" + + QUERY_ID + " VARCHAR NOT NULL,\n" + TENANT_ID + " VARCHAR ," + - QUERY_ID + " VARCHAR NOT NULL,\n" + USER + " VARCHAR , \n" + CLIENT_IP + " VARCHAR, \n" + // Function metadata (will be null for argument row) QUERY + " VARCHAR, \n" + EXPLAIN_PLAN + " VARCHAR, \n" + // Argument metadata (will be null for function row) - START_TIME + " TIMESTAMP, \n" + - TOTAL_EXECUTION_TIME + " BIGINT, \n" + NO_OF_RESULTS_ITERATED + " BIGINT, \n" + QUERY_STATUS + " VARCHAR, \n" + EXCEPTION_TRACE + " VARCHAR, \n" + GLOBAL_SCAN_DETAILS + " VARCHAR, \n" + BIND_PARAMETERS + " VARCHAR, \n" + SCAN_METRICS_JSON + " VARCHAR, \n" + - " CONSTRAINT " + SYSTEM_TABLE_PK_NAME + " PRIMARY KEY (QUERY_ID))\n" + + MetricType.getMetricColumnsDetails()+"\n"+ + " CONSTRAINT " + SYSTEM_TABLE_PK_NAME + " PRIMARY KEY (START_TIME, TABLE_NAME, QUERY_ID))\n" + + PhoenixDatabaseMetaData.SALT_BUCKETS + "=%s,\n"+ PhoenixDatabaseMetaData.TRANSACTIONAL + "=" + Boolean.FALSE+ ",\n" + HColumnDescriptor.TTL + "=" + MetaDataProtocol.DEFAULT_LOG_TTL+",\n"+ - PhoenixDatabaseMetaData.COLUMN_ENCODED_BYTES +" = 0"; + TableProperty.IMMUTABLE_STORAGE_SCHEME.toString() + " = " + ImmutableStorageScheme.SINGLE_CELL_ARRAY_WITH_OFFSETS.name() + ",\n" + + TableProperty.COLUMN_ENCODED_BYTES.toString()+" = 1"; public static final byte[] OFFSET_FAMILY = "f_offset".getBytes(); public static final byte[] OFFSET_COLUMN = "c_offset".getBytes(); http://git-wip-us.apache.org/repos/asf/phoenix/blob/f73f98c7/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java index db0b10b..559d165 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java @@ -197,6 +197,7 @@ public interface QueryServices extends SQLCloseable { public static final String COMMIT_STATS_ASYNC = "phoenix.stats.commit.async"; // Maximum size in bytes taken up by cached table stats in the client public static final String STATS_MAX_CACHE_SIZE = "phoenix.stats.cache.maxSize"; + public static final String LOG_SALT_BUCKETS_ATTRIB = "phoenix.log.saltBuckets"; public static final String SEQUENCE_SALT_BUCKETS_ATTRIB = "phoenix.sequence.saltBuckets"; public static final String COPROCESSOR_PRIORITY_ATTRIB = "phoenix.coprocessor.priority"; http://git-wip-us.apache.org/repos/asf/phoenix/blob/f73f98c7/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java index 0e6e89f..d708785 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java @@ -356,6 +356,7 @@ public class QueryServicesOptions { public static final boolean DEFAULT_COST_BASED_OPTIMIZER_ENABLED = false; public static final String DEFAULT_LOGGING_LEVEL = LogLevel.OFF.name(); public static final String DEFAULT_LOG_SAMPLE_RATE = "1.0"; + public static final int DEFAULT_LOG_SALT_BUCKETS = 32; private final Configuration config; http://git-wip-us.apache.org/repos/asf/phoenix/blob/f73f98c7/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java index 20ac732..8f14680 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java @@ -2286,7 +2286,7 @@ public class MetaDataClient { } } // System tables have hard-coded column qualifiers. So we can't use column encoding for them. - else if (!SchemaUtil.isSystemTable(Bytes.toBytes(SchemaUtil.getTableName(schemaName, tableName)))) { + else if (!SchemaUtil.isSystemTable(Bytes.toBytes(SchemaUtil.getTableName(schemaName, tableName)))|| SchemaUtil.isLogTable(schemaName, tableName)) { /* * Indexes inherit the storage scheme of the parent data tables. Otherwise, we always attempt to * create tables with encoded column names. http://git-wip-us.apache.org/repos/asf/phoenix/blob/f73f98c7/phoenix-core/src/main/java/org/apache/phoenix/util/QueryUtil.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/QueryUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/util/QueryUtil.java index 9d2e53c..94cbfea 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/util/QueryUtil.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/util/QueryUtil.java @@ -436,7 +436,7 @@ public final class QueryUtil { } return url; } - + private static int getInt(String key, int defaultValue, Properties props, Configuration conf) { if (conf == null) { Preconditions.checkNotNull(props); @@ -487,5 +487,10 @@ public final class QueryUtil { public static String getViewPartitionClause(String partitionColumnName, long autoPartitionNum) { return partitionColumnName + " " + toSQL(CompareOp.EQUAL) + " " + autoPartitionNum; } + + public static Connection getConnectionForQueryLog(Configuration config) throws ClassNotFoundException, SQLException { + //we don't need this connection to upgrade anything or start dispatcher + return getConnectionOnServer(config); + } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/phoenix/blob/f73f98c7/phoenix-core/src/main/java/org/apache/phoenix/util/SchemaUtil.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/SchemaUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/util/SchemaUtil.java index 92a2cde..dd00a69 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/util/SchemaUtil.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/util/SchemaUtil.java @@ -1170,6 +1170,10 @@ public class SchemaUtil { Cell isNamespaceMappedCell = currentResult.getColumnLatestCell(TABLE_FAMILY_BYTES, IS_NAMESPACE_MAPPED_BYTES); return isNamespaceMappedCell!=null && (boolean) PBoolean.INSTANCE.toObject(isNamespaceMappedCell.getValue()); } + + public static boolean isLogTable(String schemaName, String tableName) { + return PhoenixDatabaseMetaData.SYSTEM_CATALOG_SCHEMA.equals(schemaName) && PhoenixDatabaseMetaData.SYSTEM_LOG_TABLE.equals(tableName); + } } http://git-wip-us.apache.org/repos/asf/phoenix/blob/f73f98c7/phoenix-core/src/test/java/org/apache/phoenix/iterate/SpoolingResultIteratorTest.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/test/java/org/apache/phoenix/iterate/SpoolingResultIteratorTest.java b/phoenix-core/src/test/java/org/apache/phoenix/iterate/SpoolingResultIteratorTest.java index e0a731d..d858e72 100644 --- a/phoenix-core/src/test/java/org/apache/phoenix/iterate/SpoolingResultIteratorTest.java +++ b/phoenix-core/src/test/java/org/apache/phoenix/iterate/SpoolingResultIteratorTest.java @@ -24,10 +24,12 @@ import java.util.Arrays; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.util.Bytes; +import org.apache.phoenix.log.LogLevel; import org.apache.phoenix.memory.DelegatingMemoryManager; import org.apache.phoenix.memory.GlobalMemoryManager; import org.apache.phoenix.memory.MemoryManager; import org.apache.phoenix.monitoring.MemoryMetricsHolder; +import org.apache.phoenix.monitoring.ReadMetricQueue; import org.apache.phoenix.monitoring.SpoolingMetricsHolder; import org.apache.phoenix.query.QueryServicesOptions; import org.apache.phoenix.schema.tuple.SingleKeyValueTuple; @@ -54,7 +56,10 @@ public class SpoolingResultIteratorTest { }; MemoryManager memoryManager = new DelegatingMemoryManager(new GlobalMemoryManager(threshold)); - ResultIterator scanner = new SpoolingResultIterator(SpoolingMetricsHolder.NO_OP_INSTANCE, MemoryMetricsHolder.NO_OP_INSTANCE, iterator, memoryManager, threshold, maxSizeSpool,"/tmp"); + ResultIterator scanner = new SpoolingResultIterator( + SpoolingMetricsHolder.NO_OP_INSTANCE, + new MemoryMetricsHolder(new ReadMetricQueue(LogLevel.OFF), ""), iterator, memoryManager, threshold, + maxSizeSpool, "/tmp"); AssertResults.assertResults(scanner, expectedResults); } http://git-wip-us.apache.org/repos/asf/phoenix/blob/f73f98c7/phoenix-hive/src/main/java/org/apache/phoenix/hive/mapreduce/PhoenixRecordReader.java ---------------------------------------------------------------------- diff --git a/phoenix-hive/src/main/java/org/apache/phoenix/hive/mapreduce/PhoenixRecordReader.java b/phoenix-hive/src/main/java/org/apache/phoenix/hive/mapreduce/PhoenixRecordReader.java index c10c4d1..2ff8aca 100644 --- a/phoenix-hive/src/main/java/org/apache/phoenix/hive/mapreduce/PhoenixRecordReader.java +++ b/phoenix-hive/src/main/java/org/apache/phoenix/hive/mapreduce/PhoenixRecordReader.java @@ -112,11 +112,10 @@ public class PhoenixRecordReader<T extends DBWritable> implements String tableName = queryPlan.getTableRef().getTable().getPhysicalName().getString(); long renewScannerLeaseThreshold = queryPlan.getContext().getConnection() .getQueryServices().getRenewLeaseThresholdMilliSeconds(); - boolean isRequestMetricsEnabled = readMetrics.isRequestMetricsEnabled(); for (Scan scan : scans) { scan.setAttribute(BaseScannerRegionObserver.SKIP_REGION_BOUNDARY_CHECK, Bytes .toBytes(true)); - ScanMetricsHolder scanMetricsHolder = ScanMetricsHolder.getInstance(readMetrics, tableName, scan, isRequestMetricsEnabled); + ScanMetricsHolder scanMetricsHolder = ScanMetricsHolder.getInstance(readMetrics, tableName, scan, ctx.getConnection().getLogLevel()); final TableResultIterator tableResultIterator = new TableResultIterator( queryPlan.getContext().getConnection().getMutationState(), scan, scanMetricsHolder, renewScannerLeaseThreshold, queryPlan, MapReduceParallelScanGrouper.getInstance());
