This is an automated email from the ASF dual-hosted git repository. wyk pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/asterixdb.git
The following commit(s) were added to refs/heads/master by this push: new 782b89a84b [NO ISSUE][OTH] Reduce columnar log level 782b89a84b is described below commit 782b89a84b42b1b743a921d86b5b751af05716df Author: Wail Alkowaileet <wael....@gmail.com> AuthorDate: Thu Nov 9 14:33:21 2023 -0800 [NO ISSUE][OTH] Reduce columnar log level - user model changes: no - storage format changes: no - interface changes: no Details: Reduce the log level for columnar schemas to debug Change-Id: Ia595f67a830d7f2e9999a1549a7695a61e7527c3 Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17939 Reviewed-by: Ali Alsuliman <ali.al.solai...@gmail.com> Integration-Tests: Jenkins <jenk...@fulliautomatix.ics.uci.edu> Tested-by: Wail Alkowaileet <wael....@gmail.com> --- asterixdb/asterix-app/src/main/resources/cc.conf | 2 +- .../src/test/resources/log4j2-asterixdb-test.xml | 12 ++++++++---- .../api/cluster_state_1/cluster_state_1.1.regexadm | 2 +- .../operation/query/QueryColumnMetadata.java | 22 ++++++++++++---------- .../query/QueryColumnWithMetaMetadata.java | 8 +++++--- 5 files changed, 27 insertions(+), 19 deletions(-) diff --git a/asterixdb/asterix-app/src/main/resources/cc.conf b/asterixdb/asterix-app/src/main/resources/cc.conf index e0ca2b4f18..5d734f8d33 100644 --- a/asterixdb/asterix-app/src/main/resources/cc.conf +++ b/asterixdb/asterix-app/src/main/resources/cc.conf @@ -50,7 +50,7 @@ heartbeat.max.misses=25 [common] log.dir = logs/ -log.level = INFO +log.level = DEBUG compiler.framesize=32KB compiler.sortmemory=320KB compiler.groupmemory=160KB diff --git a/asterixdb/asterix-app/src/test/resources/log4j2-asterixdb-test.xml b/asterixdb/asterix-app/src/test/resources/log4j2-asterixdb-test.xml index b1e35fd3c4..8f81828d60 100644 --- a/asterixdb/asterix-app/src/test/resources/log4j2-asterixdb-test.xml +++ b/asterixdb/asterix-app/src/test/resources/log4j2-asterixdb-test.xml @@ -24,17 +24,21 @@ <File name="InfoLog" fileName="target/info.log"> <PatternLayout pattern="%d{HH:mm:ss.SSS} [%t] %-5level %logger{36} - %msg%n"/> </File> + <File name="DebugLog" fileName="target/debug.log"> + <PatternLayout pattern="%d{HH:mm:ss.SSS} [%t] %-5level %logger{36} - %msg%n"/> + </File> <Console name="ConsoleTest" target="SYSTEM_OUT"> <PatternLayout pattern="%d{HH:mm:ss.SSS} [%t] %c{1} - %msg%n"/> </Console> </Appenders> <Loggers> - <Root level="WARN"> - <AppenderRef ref="InfoLog"/> + <Root> + <AppenderRef ref="InfoLog" level="INFO"/> + <AppenderRef ref="DebugLog" level="DEBUG"/> </Root> <Logger name="org.apache.hyracks.control.nc.service" level="INFO"/> - <Logger name="org.apache.hyracks" level="INFO"/> - <Logger name="org.apache.asterix" level="INFO"/> + <Logger name="org.apache.hyracks" level="DEBUG"/> + <Logger name="org.apache.asterix" level="DEBUG"/> <Logger name="org.apache.hyracks.test" level="INFO"> <AppenderRef ref="ConsoleTest"/> </Logger> diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm index 1c3b061499..716e0b4b74 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm @@ -42,7 +42,7 @@ "default\.dir" : "target/io/dir/asterixdb", "library\.deploy\.timeout" : 1800, "log\.dir" : "logs/", - "log\.level" : "INFO", + "log\.level" : "DEBUG", "max\.wait\.active\.cluster" : 60, "max.web.request.size" : 209715200, "messaging\.frame\.count" : 512, diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/query/QueryColumnMetadata.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/query/QueryColumnMetadata.java index d9071b81cf..e32460978c 100644 --- a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/query/QueryColumnMetadata.java +++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/query/QueryColumnMetadata.java @@ -205,6 +205,7 @@ public class QueryColumnMetadata extends AbstractColumnImmutableReadMetadata { IColumnIterableFilterEvaluator columnFilterEvaluator = TrueColumnFilterEvaluator.INSTANCE; List<IColumnValuesReader> filterColumnReaders = Collections.emptyList(); List<IColumnRangeFilterValueAccessor> filterValueAccessors = Collections.emptyList(); + String jobId = null; if (context != null) { FilterAccessorProvider filterAccessorProvider = new FilterAccessorProvider(root, clipperVisitor, readerFactory, valueGetterFactory); @@ -220,13 +221,12 @@ public class QueryColumnMetadata extends AbstractColumnImmutableReadMetadata { filterAccessorProvider.reset(); columnFilterEvaluator = columnFilterEvaluatorFactory.create(filterAccessorProvider, evaluatorContext); filterColumnReaders = filterAccessorProvider.getFilterColumnReaders(); - + jobId = context.getJobletContext().getJobId().toString(); } - // log normalized filter - logFilter(normalizedFilterEvaluator, normalizedEvaluatorFactory.toString()); + logFilter(jobId, normalizedFilterEvaluator, normalizedEvaluatorFactory.toString()); // log requested schema - logSchema(clippedRoot, SchemaStringBuilderVisitor.RECORD_SCHEMA, fieldNamesDictionary); + logSchema(jobId, clippedRoot, SchemaStringBuilderVisitor.RECORD_SCHEMA, fieldNamesDictionary); // Primary key readers PrimitiveColumnValuesReader[] primaryKeyReaders = @@ -260,20 +260,22 @@ public class QueryColumnMetadata extends AbstractColumnImmutableReadMetadata { return primaryKeyReaders; } - protected static void logFilter(IColumnFilterEvaluator normalizedFilterEvaluator, String filterExpression) { - if (LOGGER.isInfoEnabled() && normalizedFilterEvaluator != TrueColumnFilterEvaluator.INSTANCE) { + protected static void logFilter(String jobId, IColumnFilterEvaluator normalizedFilterEvaluator, + String filterExpression) { + if (jobId != null && LOGGER.isDebugEnabled() + && normalizedFilterEvaluator != TrueColumnFilterEvaluator.INSTANCE) { String filterString = normalizedFilterEvaluator == FalseColumnFilterEvaluator.INSTANCE ? "SKIP_ALL" : LogRedactionUtil.userData(filterExpression); - LOGGER.info("Filter: {}", filterString); + LOGGER.debug("Filter [{}]: {}", jobId, filterString); } } - protected static void logSchema(ObjectSchemaNode root, String schemaSource, + protected static void logSchema(String jobId, ObjectSchemaNode root, String schemaSource, FieldNamesDictionary fieldNamesDictionary) throws HyracksDataException { - if (LOGGER.isInfoEnabled()) { + if (jobId != null && LOGGER.isDebugEnabled()) { SchemaStringBuilderVisitor schemaBuilder = new SchemaStringBuilderVisitor(fieldNamesDictionary); String schema = LogRedactionUtil.userData(schemaBuilder.build(root)); - LOGGER.info("Queried {} schema: \n {}", schemaSource, schema); + LOGGER.debug("Queried {} schema [{}]: \n {}", schemaSource, jobId, schema); } } } diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/query/QueryColumnWithMetaMetadata.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/query/QueryColumnWithMetaMetadata.java index c89a67b8c0..b4de820169 100644 --- a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/query/QueryColumnWithMetaMetadata.java +++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/query/QueryColumnWithMetaMetadata.java @@ -150,6 +150,7 @@ public final class QueryColumnWithMetaMetadata extends QueryColumnMetadata { IColumnIterableFilterEvaluator columnFilterEvaluator = TrueColumnFilterEvaluator.INSTANCE; List<IColumnValuesReader> filterColumnReaders = Collections.emptyList(); List<IColumnRangeFilterValueAccessor> filterValueAccessors = Collections.emptyList(); + String jobId = ""; if (context != null) { FilterAccessorProvider filterAccessorProvider = new FilterAccessorProvider(root, clipperVisitor, readerFactory, valueGetterFactory); @@ -165,14 +166,15 @@ public final class QueryColumnWithMetaMetadata extends QueryColumnMetadata { filterAccessorProvider.reset(); columnFilterEvaluator = columnFilterEvaluatorFactory.create(filterAccessorProvider, evaluatorContext); filterColumnReaders = filterAccessorProvider.getFilterColumnReaders(); + jobId = context.getJobletContext().getJobId().toString(); } // log normalized filter - logFilter(normalizedFilterEvaluator, normalizedEvaluatorFactory.toString()); + logFilter(jobId, normalizedFilterEvaluator, normalizedEvaluatorFactory.toString()); // log requested schema for record - logSchema(clippedRoot, SchemaStringBuilderVisitor.RECORD_SCHEMA, fieldNamesDictionary); + logSchema(jobId, clippedRoot, SchemaStringBuilderVisitor.RECORD_SCHEMA, fieldNamesDictionary); // log requested schema for meta-record - logSchema(metaClippedRoot, SchemaStringBuilderVisitor.META_RECORD_SCHEMA, fieldNamesDictionary); + logSchema(jobId, metaClippedRoot, SchemaStringBuilderVisitor.META_RECORD_SCHEMA, fieldNamesDictionary); // Primary key readers PrimitiveColumnValuesReader[] primaryKeyReaders =