KYLIN-2503 Spark cubing step should show YARN app link
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/f38d871a Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/f38d871a Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/f38d871a Branch: refs/heads/KYLIN-2360 Commit: f38d871a72e1e24331e092fd51664fcb8fd47e22 Parents: 5d012b9 Author: shaofengshi <shaofeng...@apache.org> Authored: Sat Mar 11 22:08:28 2017 +0800 Committer: shaofengshi <shaofeng...@apache.org> Committed: Sat Mar 11 22:08:28 2017 +0800 ---------------------------------------------------------------------- .../apache/kylin/job/common/ShellExecutable.java | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/f38d871a/core-job/src/main/java/org/apache/kylin/job/common/ShellExecutable.java ---------------------------------------------------------------------- diff --git a/core-job/src/main/java/org/apache/kylin/job/common/ShellExecutable.java b/core-job/src/main/java/org/apache/kylin/job/common/ShellExecutable.java index a68f242..5e0d0db 100644 --- a/core-job/src/main/java/org/apache/kylin/job/common/ShellExecutable.java +++ b/core-job/src/main/java/org/apache/kylin/job/common/ShellExecutable.java @@ -83,6 +83,10 @@ public class ShellExecutable extends AbstractExecutable { private static final Pattern PATTERN_HIVE_APP_ID_URL = Pattern.compile("Starting Job = (.*?), Tracking URL = (.*)"); private static final Pattern PATTERN_HIVE_BYTES_WRITTEN = Pattern.compile("(?:HD|MAPR)FS Read: (\\d+) HDFS Write: (\\d+) SUCCESS"); + // spark + private static final Pattern PATTERN_SPARK_APP_ID = Pattern.compile("Submitted application (.*?)"); + private static final Pattern PATTERN_SPARK_APP_URL = Pattern.compile("tracking URL: (.*)"); + @Override public void log(String message) { Matcher matcher = PATTERN_APP_ID.matcher(message); @@ -136,6 +140,19 @@ public class ShellExecutable extends AbstractExecutable { String hdfsWritten = matcher.group(2); info.put(ExecutableConstants.HDFS_BYTES_WRITTEN, hdfsWritten); } + + // spark + matcher = PATTERN_SPARK_APP_ID.matcher(message); + if (matcher.find()) { + String app_id = matcher.group(1); + info.put(ExecutableConstants.YARN_APP_ID, app_id); + } + + matcher = PATTERN_SPARK_APP_URL.matcher(message); + if (matcher.find()) { + String trackingUrl = matcher.group(1); + info.put(ExecutableConstants.YARN_APP_URL, trackingUrl); + } } Map<String, String> getInfo() {