This is an automated email from the ASF dual-hosted git repository. jhung pushed a commit to branch branch-3.2 in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.2 by this push: new a1fa9a8 YARN-9762. Add submission context label to audit logs. Contributed by Manoj Kumar a1fa9a8 is described below commit a1fa9a8a7f79a1a711cd881b526724b502e03456 Author: Jonathan Hung <jh...@linkedin.com> AuthorDate: Mon Sep 23 11:42:41 2019 -0700 YARN-9762. Add submission context label to audit logs. Contributed by Manoj Kumar (cherry picked from commit 3d78b1223d3fdc29d500803cefd2931b54f44928) --- .../server/resourcemanager/ClientRMService.java | 6 ++- .../yarn/server/resourcemanager/RMAuditLogger.java | 53 ++++++++++++++++++++-- .../server/resourcemanager/TestRMAuditLogger.java | 21 +++++++-- 3 files changed, 69 insertions(+), 11 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java index 3d1f01d..571add2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java @@ -694,13 +694,15 @@ public class ClientRMService extends AbstractService implements " submitted by user " + user); RMAuditLogger.logSuccess(user, AuditConstants.SUBMIT_APP_REQUEST, "ClientRMService", applicationId, callerContext, - submissionContext.getQueue()); + submissionContext.getQueue(), + submissionContext.getNodeLabelExpression()); } catch (YarnException e) { LOG.info("Exception in submitting " + applicationId, e); RMAuditLogger.logFailure(user, AuditConstants.SUBMIT_APP_REQUEST, e.getMessage(), "ClientRMService", "Exception in submitting application", applicationId, callerContext, - submissionContext.getQueue()); + submissionContext.getQueue(), + submissionContext.getNodeLabelExpression()); throw e; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAuditLogger.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAuditLogger.java index 292aa8b..06ac64c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAuditLogger.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAuditLogger.java @@ -275,6 +275,16 @@ public class RMAuditLogger { } } + public static void logSuccess(String user, String operation, String target, + ApplicationId appId, CallerContext callerContext, String queueName, + String partition) { + if (LOG.isInfoEnabled()) { + LOG.info( + createSuccessLog(user, operation, target, appId, null, null, null, + callerContext, Server.getRemoteIp(), queueName, partition)); + } + } + /** * Create a readable and parseable audit log string for a successful event. * @@ -395,7 +405,8 @@ public class RMAuditLogger { static String createFailureLog(String user, String operation, String perm, String target, String description, ApplicationId appId, ApplicationAttemptId attemptId, ContainerId containerId, - Resource resource, CallerContext callerContext, String queueName) { + Resource resource, CallerContext callerContext, String queueName, + String partition) { StringBuilder b = createStringBuilderForFailureLog(user, operation, target, description, perm); if (appId != null) { @@ -414,6 +425,10 @@ public class RMAuditLogger { if (queueName != null) { add(Keys.QUEUENAME, queueName, b); } + if (partition != null) { + add(Keys.NODELABEL, partition, b); + } + return b.toString(); } @@ -424,7 +439,7 @@ public class RMAuditLogger { String target, String description, ApplicationId appId, ApplicationAttemptId attemptId, ContainerId containerId, Resource resource) { return createFailureLog(user, operation, perm, target, description, appId, - attemptId, containerId, resource, null, null); + attemptId, containerId, resource, null, null, null); } /** @@ -496,7 +511,7 @@ public class RMAuditLogger { CallerContext callerContext) { if (LOG.isWarnEnabled()) { LOG.warn(createFailureLog(user, operation, perm, target, description, - appId, null, null, null, callerContext, null)); + appId, null, null, null, callerContext, null, null)); } } @@ -505,7 +520,7 @@ public class RMAuditLogger { CallerContext callerContext, String queueName) { if (LOG.isWarnEnabled()) { LOG.warn(createFailureLog(user, operation, perm, target, description, - appId, null, null, null, callerContext, queueName)); + appId, null, null, null, callerContext, queueName, null)); } } @@ -537,7 +552,7 @@ public class RMAuditLogger { String queueName) { if (LOG.isWarnEnabled()) { LOG.warn(createFailureLog(user, operation, perm, target, description, - appId, null, null, null, null, queueName)); + appId, null, null, null, null, queueName, null)); } } @@ -586,6 +601,34 @@ public class RMAuditLogger { } /** + * Create a readable and parseable audit log string for a failed event. + * + * @param user User who made the service request. + * @param operation Operation requested by the user. + * @param perm Target permissions. + * @param target The target on which the operation is being performed. + * @param description Some additional information as to why the operation + * failed. + * @param appId ApplicationId in which operation was performed. + * @param callerContext Caller context + * @param queueName Name of queue. + * @param partition Name of labeled partition. + * + * <br><br> + * Note that the {@link RMAuditLogger} uses tabs ('\t') as a key-val delimiter + * and hence the value fields should not contains tabs ('\t'). + */ + public static void logFailure(String user, String operation, String perm, + String target, String description, ApplicationId appId, + CallerContext callerContext, String queueName, String partition) { + if (LOG.isWarnEnabled()) { + LOG.warn( + createFailureLog(user, operation, perm, target, description, appId, + null, null, null, callerContext, queueName, partition)); + } + } + + /** * A helper api to add remote IP address. */ static void addRemoteIP(StringBuilder b) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAuditLogger.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAuditLogger.java index 7d8eeab..84ea2d6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAuditLogger.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAuditLogger.java @@ -293,16 +293,17 @@ public class TestRMAuditLogger { ApplicationAttemptId attemptId, ContainerId containerId, CallerContext callerContext, Resource resource) { testFailureLogFormatHelper(checkIP, appId, attemptId, containerId, - callerContext, resource, null); + callerContext, resource, null, null, null); } private void testFailureLogFormatHelper(boolean checkIP, ApplicationId appId, ApplicationAttemptId attemptId, ContainerId containerId, CallerContext callerContext, Resource resource, - RMAuditLogger.ArgsBuilder args) { + String queueName, String partition, RMAuditLogger.ArgsBuilder args) { String fLog = args == null ? RMAuditLogger.createFailureLog(USER, OPERATION, PERM, TARGET, DESC, - appId, attemptId, containerId, resource, callerContext, null) : + appId, attemptId, containerId, resource, callerContext, + queueName, partition) : RMAuditLogger.createFailureLog(USER, OPERATION, PERM, TARGET, DESC, args); StringBuilder expLog = new StringBuilder(); @@ -334,6 +335,12 @@ public class TestRMAuditLogger { expLog.append("\tCALLERSIGNATURE=signature"); } } + if (queueName != null) { + expLog.append("\tQUEUENAME=" + QUEUE); + } + if (partition != null) { + expLog.append("\tNODELABEL=" + PARTITION); + } if (args != null) { expLog.append("\tQUEUENAME=root"); expLog.append("\tRECURSIVE=true"); @@ -364,10 +371,16 @@ public class TestRMAuditLogger { testFailureLogFormatHelper(checkIP, APPID, ATTEMPTID, CONTAINERID, new CallerContext.Builder(CALLER_CONTEXT).setSignature(CALLER_SIGNATURE) .build(), RESOURCE); + testFailureLogFormatHelper(checkIP, APPID, ATTEMPTID, CONTAINERID, + new CallerContext.Builder(CALLER_CONTEXT).setSignature(CALLER_SIGNATURE) + .build(), RESOURCE, QUEUE, null, null); + testFailureLogFormatHelper(checkIP, APPID, ATTEMPTID, CONTAINERID, + new CallerContext.Builder(CALLER_CONTEXT).setSignature(CALLER_SIGNATURE) + .build(), RESOURCE, QUEUE, PARTITION, null); RMAuditLogger.ArgsBuilder args = new RMAuditLogger.ArgsBuilder() .append(Keys.QUEUENAME, QUEUE).append(Keys.RECURSIVE, "true"); testFailureLogFormatHelper(checkIP, null, null, null, null, null, - args); + null, null, args); } /** --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org