LENS-669 : Add queryId in LogSegregationContext
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/dfeea698 Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/dfeea698 Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/dfeea698 Branch: refs/heads/LENS-581 Commit: dfeea69890cf75b1713d645117a4e1ec01761f72 Parents: 46dc5ad Author: Amareshwari Sriramadasu <[email protected]> Authored: Mon Jul 20 15:42:06 2015 +0530 Committer: Rajat Khandelwal <[email protected]> Committed: Mon Jul 20 15:42:06 2015 +0530 ---------------------------------------------------------------------- .../org/apache/lens/driver/jdbc/JDBCDriver.java | 2 +- .../server/api/query/AbstractQueryContext.java | 4 +- .../server/model/LogSegregationContext.java | 27 ++++++++- .../MappedDiagnosticLogSegregationContext.java | 23 +++++++- .../lens/server/api/query/MockQueryContext.java | 5 ++ ...stMappedDiagnosticLogSegregationContext.java | 60 ++++++++++++++++++++ .../server/LensRequestContextInitFilter.java | 2 +- .../java/org/apache/lens/server/LensServer.java | 2 +- .../org/apache/lens/server/LensServices.java | 2 +- .../lens/server/query/QueryEndNotifier.java | 2 +- .../server/query/QueryExecutionServiceImpl.java | 11 ++-- .../lens/server/query/QueryServiceResource.java | 2 +- .../lens/server/query/ResultFormatter.java | 2 +- .../store/log/StatisticsLogFileScannerTask.java | 2 +- 14 files changed, 125 insertions(+), 21 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/dfeea698/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java ---------------------------------------------------------------------- diff --git a/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java b/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java index a09cd1b..9445436 100644 --- a/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java +++ b/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java @@ -294,7 +294,7 @@ public class JDBCDriver implements LensDriver { @Override public QueryResult call() { - logSegregationContext.set(this.queryContext.getQueryHandleString()); + logSegregationContext.setLogSegragationAndQueryId(this.queryContext.getQueryHandleString()); Statement stmt = null; Connection conn = null; http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/dfeea698/lens-server-api/src/main/java/org/apache/lens/server/api/query/AbstractQueryContext.java ---------------------------------------------------------------------- diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/AbstractQueryContext.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/AbstractQueryContext.java index 1334d9a..71a37f0 100644 --- a/lens-server-api/src/main/java/org/apache/lens/server/api/query/AbstractQueryContext.java +++ b/lens-server-api/src/main/java/org/apache/lens/server/api/query/AbstractQueryContext.java @@ -455,9 +455,7 @@ public abstract class AbstractQueryContext implements Serializable { * Get handle of the query for logging purposes * @return */ - public String getLogHandle() { - return this.getUserQuery(); - } + public abstract String getLogHandle(); /** * Returns database set while launching query http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/dfeea698/lens-server-api/src/main/java/org/apache/lens/server/model/LogSegregationContext.java ---------------------------------------------------------------------- diff --git a/lens-server-api/src/main/java/org/apache/lens/server/model/LogSegregationContext.java b/lens-server-api/src/main/java/org/apache/lens/server/model/LogSegregationContext.java index 8691a05..f76e292 100644 --- a/lens-server-api/src/main/java/org/apache/lens/server/model/LogSegregationContext.java +++ b/lens-server-api/src/main/java/org/apache/lens/server/model/LogSegregationContext.java @@ -25,11 +25,34 @@ public interface LogSegregationContext { * * @param id the id to be added to every log line of current thread */ - void set(final String id); + void setLogSegregationId(final String id); /** * * @return the id being used by the current thread for log segregation */ - String get(); + String getLogSegragationId(); + + /** + * Sets query id to be used by current thread for log segregation for identifying current query. + * The same id is set as log segregation in every log line as well. + * + * @param id the query id + */ + void setLogSegragationAndQueryId(final String id); + + /** + * Sets query id to be used by current thread for log segregation for identifying current query. + * + * @param id the query id + */ + void setQueryId(final String id); + + /** + * Get current query id in log segregation + * + * @return the query id being used by the current thread for log segregation + */ + String getQueryId(); + } http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/dfeea698/lens-server-api/src/main/java/org/apache/lens/server/model/MappedDiagnosticLogSegregationContext.java ---------------------------------------------------------------------- diff --git a/lens-server-api/src/main/java/org/apache/lens/server/model/MappedDiagnosticLogSegregationContext.java b/lens-server-api/src/main/java/org/apache/lens/server/model/MappedDiagnosticLogSegregationContext.java index 989cb9d..fd5f012 100644 --- a/lens-server-api/src/main/java/org/apache/lens/server/model/MappedDiagnosticLogSegregationContext.java +++ b/lens-server-api/src/main/java/org/apache/lens/server/model/MappedDiagnosticLogSegregationContext.java @@ -22,15 +22,32 @@ import org.slf4j.MDC; public class MappedDiagnosticLogSegregationContext implements LogSegregationContext { - private static final String LOG_SEGREGATION_ID = "logSegregationId"; + public static final String LOG_SEGREGATION_ID = "logSegregationId"; + public static final String QUERY_LOG_ID = "queryLogId"; @Override - public void set(String id) { + public void setLogSegregationId(String id) { MDC.put(LOG_SEGREGATION_ID, id); } @Override - public String get() { + public String getLogSegragationId() { return MDC.get(LOG_SEGREGATION_ID); } + + @Override + public void setLogSegragationAndQueryId(String id) { + setLogSegregationId(id); + setQueryId(id); + } + + @Override + public String getQueryId() { + return MDC.get(QUERY_LOG_ID); + } + + @Override + public void setQueryId(String id) { + MDC.put(QUERY_LOG_ID, id); + } } http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/dfeea698/lens-server-api/src/test/java/org/apache/lens/server/api/query/MockQueryContext.java ---------------------------------------------------------------------- diff --git a/lens-server-api/src/test/java/org/apache/lens/server/api/query/MockQueryContext.java b/lens-server-api/src/test/java/org/apache/lens/server/api/query/MockQueryContext.java index 5a1ab66..3d38eab 100644 --- a/lens-server-api/src/test/java/org/apache/lens/server/api/query/MockQueryContext.java +++ b/lens-server-api/src/test/java/org/apache/lens/server/api/query/MockQueryContext.java @@ -33,4 +33,9 @@ public class MockQueryContext extends AbstractQueryContext { final Configuration conf, final Collection<LensDriver> drivers) { super(query, "testuser", qconf, conf, drivers, false); } + + @Override + public String getLogHandle() { + return super.getUserQuery(); + } } http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/dfeea698/lens-server-api/src/test/java/org/apache/lens/server/model/TestMappedDiagnosticLogSegregationContext.java ---------------------------------------------------------------------- diff --git a/lens-server-api/src/test/java/org/apache/lens/server/model/TestMappedDiagnosticLogSegregationContext.java b/lens-server-api/src/test/java/org/apache/lens/server/model/TestMappedDiagnosticLogSegregationContext.java new file mode 100644 index 0000000..227fa15 --- /dev/null +++ b/lens-server-api/src/test/java/org/apache/lens/server/model/TestMappedDiagnosticLogSegregationContext.java @@ -0,0 +1,60 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.lens.server.model; + +import static org.testng.Assert.*; + +import org.slf4j.MDC; +import org.testng.annotations.Test; + +/** + * Tests for MDC log segregation + */ +public class TestMappedDiagnosticLogSegregationContext { + + @Test + public void testIds() { + MappedDiagnosticLogSegregationContext testLogCtx = new MappedDiagnosticLogSegregationContext(); + String logId = "logid"; + testLogCtx.setLogSegregationId(logId); + assertEquals(testLogCtx.getLogSegragationId(), logId); + assertEquals(MDC.get(MappedDiagnosticLogSegregationContext.LOG_SEGREGATION_ID), logId); + + String queryId = "queryId"; + testLogCtx.setLogSegragationAndQueryId(queryId); + assertEquals(testLogCtx.getQueryId(), queryId); + assertEquals(testLogCtx.getLogSegragationId(), queryId); + assertEquals(MDC.get(MappedDiagnosticLogSegregationContext.LOG_SEGREGATION_ID), queryId); + assertEquals(MDC.get(MappedDiagnosticLogSegregationContext.QUERY_LOG_ID), queryId); + + String logId2 = "logid2"; + testLogCtx.setLogSegregationId(logId2); + assertEquals(testLogCtx.getLogSegragationId(), logId2); + assertEquals(MDC.get(MappedDiagnosticLogSegregationContext.LOG_SEGREGATION_ID), logId2); + assertEquals(testLogCtx.getQueryId(), queryId); + assertEquals(MDC.get(MappedDiagnosticLogSegregationContext.QUERY_LOG_ID), queryId); + + String queryId2 = "queryId2"; + testLogCtx.setQueryId(queryId2); + assertEquals(testLogCtx.getQueryId(), queryId2); + assertEquals(testLogCtx.getLogSegragationId(), logId2); + assertEquals(MDC.get(MappedDiagnosticLogSegregationContext.LOG_SEGREGATION_ID), logId2); + assertEquals(MDC.get(MappedDiagnosticLogSegregationContext.QUERY_LOG_ID), queryId2); + } +} http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/dfeea698/lens-server/src/main/java/org/apache/lens/server/LensRequestContextInitFilter.java ---------------------------------------------------------------------- diff --git a/lens-server/src/main/java/org/apache/lens/server/LensRequestContextInitFilter.java b/lens-server/src/main/java/org/apache/lens/server/LensRequestContextInitFilter.java index c9658fc..d2f2f91 100644 --- a/lens-server/src/main/java/org/apache/lens/server/LensRequestContextInitFilter.java +++ b/lens-server/src/main/java/org/apache/lens/server/LensRequestContextInitFilter.java @@ -49,7 +49,7 @@ public class LensRequestContextInitFilter implements ContainerRequestFilter { String uniqueRequesId = UUID.randomUUID().toString(); /* Add request id for appearing in every log line */ - new MappedDiagnosticLogSegregationContext().set(uniqueRequesId); + new MappedDiagnosticLogSegregationContext().setLogSegregationId(uniqueRequesId); /* Add request id to headers */ requestContext.getHeaders().add(REQUEST_ID, uniqueRequesId); http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/dfeea698/lens-server/src/main/java/org/apache/lens/server/LensServer.java ---------------------------------------------------------------------- diff --git a/lens-server/src/main/java/org/apache/lens/server/LensServer.java b/lens-server/src/main/java/org/apache/lens/server/LensServer.java index d3dfad4..d4c89c0 100644 --- a/lens-server/src/main/java/org/apache/lens/server/LensServer.java +++ b/lens-server/src/main/java/org/apache/lens/server/LensServer.java @@ -185,7 +185,7 @@ public class LensServer { public static void main(String[] args) throws Exception { final String runId = UUID.randomUUID().toString(); - new MappedDiagnosticLogSegregationContext().set(runId); + new MappedDiagnosticLogSegregationContext().setLogSegregationId(runId); printStartupMessage(); try { http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/dfeea698/lens-server/src/main/java/org/apache/lens/server/LensServices.java ---------------------------------------------------------------------- diff --git a/lens-server/src/main/java/org/apache/lens/server/LensServices.java b/lens-server/src/main/java/org/apache/lens/server/LensServices.java index 5fe3935..fe58e96 100644 --- a/lens-server/src/main/java/org/apache/lens/server/LensServices.java +++ b/lens-server/src/main/java/org/apache/lens/server/LensServices.java @@ -280,7 +280,7 @@ public class LensServices extends CompositeService implements ServiceProvider { public void run() { try { final String runId = UUID.randomUUID().toString(); - logSegregationContext.set(runId); + logSegregationContext.setLogSegregationId(runId); persistLensServiceState(); LOG.info("SnapShot of Lens Services created"); } catch (IOException e) { http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/dfeea698/lens-server/src/main/java/org/apache/lens/server/query/QueryEndNotifier.java ---------------------------------------------------------------------- diff --git a/lens-server/src/main/java/org/apache/lens/server/query/QueryEndNotifier.java b/lens-server/src/main/java/org/apache/lens/server/query/QueryEndNotifier.java index df36a21..757205a 100644 --- a/lens-server/src/main/java/org/apache/lens/server/query/QueryEndNotifier.java +++ b/lens-server/src/main/java/org/apache/lens/server/query/QueryEndNotifier.java @@ -117,7 +117,7 @@ public class QueryEndNotifier extends AsyncEventListener<QueryEnded> { + ". No email generated"); return; } - this.logSegregationContext.set(queryContext.getQueryHandleString()); + this.logSegregationContext.setLogSegragationAndQueryId(queryContext.getQueryHandleString()); boolean whetherMailNotify = Boolean.parseBoolean(queryContext.getConf().get(LensConfConstants.QUERY_MAIL_NOTIFY, LensConfConstants.WHETHER_MAIL_NOTIFY_DEFAULT)); http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/dfeea698/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java ---------------------------------------------------------------------- diff --git a/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java b/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java index 2d7a92a..47d27d0 100644 --- a/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java +++ b/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java @@ -503,7 +503,7 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu QueryContext ctx = queuedQueries.take(); /* Setting log segregation id */ - logSegregationContext.set(ctx.getQueryHandleString()); + logSegregationContext.setLogSegragationAndQueryId(ctx.getQueryHandleString()); synchronized (ctx) { if (ctx.getStatus().getStatus().equals(QUEUED)) { @@ -587,7 +587,7 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu return; } - logSegregationContext.set(ctx.getQueryHandleString()); + logSegregationContext.setLogSegragationAndQueryId(ctx.getQueryHandleString()); log.info("Polling status for " + ctx.getQueryHandle()); try { // session is not required to update status of the query @@ -808,7 +808,7 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu FinishedQuery finished = null; try { finished = finishedQueries.take(); - logSegregationContext.set(finished.getQueryHandleString()); + logSegregationContext.setLogSegragationAndQueryId(finished.getQueryHandleString()); } catch (InterruptedException e) { log.info("QueryPurger has been interrupted, exiting"); return; @@ -883,7 +883,7 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu while (!stopped && !prepareQueryPurger.isInterrupted()) { try { PreparedQueryContext prepared = preparedQueryQueue.take(); - logSegregationContext.set(prepared.getQueryHandleString()); + logSegregationContext.setLogSegragationAndQueryId(prepared.getQueryHandleString()); destroyPreparedQuery(prepared); log.info("Purged prepared query: " + prepared.getPrepareHandle()); } catch (LensException e) { @@ -1220,7 +1220,8 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu @Override public void run() { try { - logSegregationContext.set(ctx.getLogHandle()); + // With following set - explain estimate calls are setting queryLogId as requestid in logSegregationContext + logSegregationContext.setLogSegragationAndQueryId(ctx.getLogHandle()); acquire(ctx.getLensSessionIdentifier()); MethodMetricsContext rewriteGauge = MethodMetricsFactory.createMethodGauge(ctx.getDriverConf(driver), true, REWRITE_GAUGE); http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/dfeea698/lens-server/src/main/java/org/apache/lens/server/query/QueryServiceResource.java ---------------------------------------------------------------------- diff --git a/lens-server/src/main/java/org/apache/lens/server/query/QueryServiceResource.java b/lens-server/src/main/java/org/apache/lens/server/query/QueryServiceResource.java index 19ed94a..fce9cf5 100644 --- a/lens-server/src/main/java/org/apache/lens/server/query/QueryServiceResource.java +++ b/lens-server/src/main/java/org/apache/lens/server/query/QueryServiceResource.java @@ -210,7 +210,7 @@ public class QueryServiceResource { @FormDataParam("conf") LensConf conf, @DefaultValue("30000") @FormDataParam("timeoutmillis") Long timeoutmillis, @DefaultValue("") @FormDataParam("queryName") String queryName) throws LensException { - final String requestId = this.logSegregationContext.get(); + final String requestId = this.logSegregationContext.getLogSegragationId(); try { validateSessionId(sessionid); http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/dfeea698/lens-server/src/main/java/org/apache/lens/server/query/ResultFormatter.java ---------------------------------------------------------------------- diff --git a/lens-server/src/main/java/org/apache/lens/server/query/ResultFormatter.java b/lens-server/src/main/java/org/apache/lens/server/query/ResultFormatter.java index c526c02..877e296 100644 --- a/lens-server/src/main/java/org/apache/lens/server/query/ResultFormatter.java +++ b/lens-server/src/main/java/org/apache/lens/server/query/ResultFormatter.java @@ -79,7 +79,7 @@ public class ResultFormatter extends AsyncEventListener<QueryExecuted> { private void formatOutput(QueryExecuted event) { QueryHandle queryHandle = event.getQueryHandle(); QueryContext ctx = queryService.getQueryContext(queryHandle); - this.logSegregationContext.set(ctx.getQueryHandleString()); + this.logSegregationContext.setLogSegragationAndQueryId(ctx.getQueryHandleString()); try { if (!ctx.isPersistent()) { LOG.info("No result formatting required for query " + queryHandle); http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/dfeea698/lens-server/src/main/java/org/apache/lens/server/stats/store/log/StatisticsLogFileScannerTask.java ---------------------------------------------------------------------- diff --git a/lens-server/src/main/java/org/apache/lens/server/stats/store/log/StatisticsLogFileScannerTask.java b/lens-server/src/main/java/org/apache/lens/server/stats/store/log/StatisticsLogFileScannerTask.java index 7e9c74d..305c031 100644 --- a/lens-server/src/main/java/org/apache/lens/server/stats/store/log/StatisticsLogFileScannerTask.java +++ b/lens-server/src/main/java/org/apache/lens/server/stats/store/log/StatisticsLogFileScannerTask.java @@ -76,7 +76,7 @@ public class StatisticsLogFileScannerTask extends TimerTask { try { final String runId = UUID.randomUUID().toString(); - this.logSegregationContext.set(runId); + this.logSegregationContext.setLogSegregationId(runId); for (Map.Entry<String, String> entry : scanSet.entrySet()) { File f = new File(entry.getValue()).getAbsoluteFile();
