[45/50] [abbrv] hadoop git commit: MAPREDUCE-6337. Added a mode to replay MR job history files and put them into the timeline service v2. Contributed by Sangjin Lee.
MAPREDUCE-6337. Added a mode to replay MR job history files and put them into
the timeline service v2. Contributed by Sangjin Lee.
(cherry picked from commit 463e070a8e7c882706a96eaa20ea49bfe9982875)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/bc11777a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/bc11777a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/bc11777a
Branch: refs/heads/YARN-2928
Commit: bc11777abe07242358c6e123109eb5b04a81
Parents: 940902a
Author: Zhijie Shen
Authored: Thu May 14 15:16:33 2015 -0700
Committer: Vinod Kumar Vavilapalli
Committed: Fri Aug 14 11:23:27 2015 -0700
--
hadoop-mapreduce-project/CHANGES.txt| 3 +
.../hadoop/mapred/JobHistoryFileParser.java | 53
.../mapred/JobHistoryFileReplayMapper.java | 301 +++
.../hadoop/mapred/SimpleEntityWriter.java | 139 +
.../hadoop/mapred/TimelineEntityConverter.java | 207 +
.../mapred/TimelineServicePerformanceV2.java| 191
.../collector/TimelineCollectorManager.java | 8 +-
.../storage/FileSystemTimelineWriterImpl.java | 23 +-
.../timelineservice/storage/package-info.java | 24 ++
9 files changed, 809 insertions(+), 140 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc11777a/hadoop-mapreduce-project/CHANGES.txt
--
diff --git a/hadoop-mapreduce-project/CHANGES.txt
b/hadoop-mapreduce-project/CHANGES.txt
index 78f0287..679fb52 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -12,6 +12,9 @@ Branch YARN-2928: Timeline Server Next Generation: Phase 1
MAPREDUCE-6335. Created MR job based performance test driver for the
timeline service v2. (Sangjin Lee via zjshen)
+MAPREDUCE-6337. Added a mode to replay MR job history files and put them
+into the timeline service v2. (Sangjin Lee via zjshen)
+
IMPROVEMENTS
OPTIMIZATIONS
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc11777a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/JobHistoryFileParser.java
--
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/JobHistoryFileParser.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/JobHistoryFileParser.java
new file mode 100644
index 000..9d051df
--- /dev/null
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/JobHistoryFileParser.java
@@ -0,0 +1,53 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.mapred;
+
+import java.io.IOException;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser;
+import org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser.JobInfo;
+
+class JobHistoryFileParser {
+ private static final Log LOG = LogFactory.getLog(JobHistoryFileParser.class);
+
+ private final FileSystem fs;
+
+ public JobHistoryFileParser(FileSystem fs) {
+LOG.info("JobHistoryFileParser created with " + fs);
+this.fs = fs;
+ }
+
+ public JobInfo parseHistoryFile(Path path) throws IOException {
+LOG.info("parsing job history file " + path);
+JobHistoryParser parser = new JobHistoryParser(fs, path);
+return parser.parse();
+ }
+
+ public Configuration parseConfiguration(Path path) throws IOException {
+LOG.info("parsing job configuration file " + path);
+Configuration conf = new Configuration(false);
+conf.addResource(fs.open(p
[45/50] [abbrv] hadoop git commit: MAPREDUCE-6337. Added a mode to replay MR job history files and put them into the timeline service v2. Contributed by Sangjin Lee.
MAPREDUCE-6337. Added a mode to replay MR job history files and put them into
the timeline service v2. Contributed by Sangjin Lee.
(cherry picked from commit 463e070a8e7c882706a96eaa20ea49bfe9982875)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/09a8b7b9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/09a8b7b9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/09a8b7b9
Branch: refs/heads/YARN-2928
Commit: 09a8b7b94b08572c3e279a9de3103529acbccc1f
Parents: 51029a7
Author: Zhijie Shen
Authored: Thu May 14 15:16:33 2015 -0700
Committer: Sangjin Lee
Committed: Tue Aug 25 10:47:16 2015 -0700
--
hadoop-mapreduce-project/CHANGES.txt| 3 +
.../hadoop/mapred/JobHistoryFileParser.java | 53
.../mapred/JobHistoryFileReplayMapper.java | 301 +++
.../hadoop/mapred/SimpleEntityWriter.java | 139 +
.../hadoop/mapred/TimelineEntityConverter.java | 207 +
.../mapred/TimelineServicePerformanceV2.java| 191
.../collector/TimelineCollectorManager.java | 8 +-
.../storage/FileSystemTimelineWriterImpl.java | 23 +-
.../timelineservice/storage/package-info.java | 24 ++
9 files changed, 809 insertions(+), 140 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/09a8b7b9/hadoop-mapreduce-project/CHANGES.txt
--
diff --git a/hadoop-mapreduce-project/CHANGES.txt
b/hadoop-mapreduce-project/CHANGES.txt
index 2805780..9c66a5e 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -12,6 +12,9 @@ Branch YARN-2928: Timeline Server Next Generation: Phase 1
MAPREDUCE-6335. Created MR job based performance test driver for the
timeline service v2. (Sangjin Lee via zjshen)
+MAPREDUCE-6337. Added a mode to replay MR job history files and put them
+into the timeline service v2. (Sangjin Lee via zjshen)
+
IMPROVEMENTS
OPTIMIZATIONS
http://git-wip-us.apache.org/repos/asf/hadoop/blob/09a8b7b9/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/JobHistoryFileParser.java
--
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/JobHistoryFileParser.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/JobHistoryFileParser.java
new file mode 100644
index 000..9d051df
--- /dev/null
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/JobHistoryFileParser.java
@@ -0,0 +1,53 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.mapred;
+
+import java.io.IOException;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser;
+import org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser.JobInfo;
+
+class JobHistoryFileParser {
+ private static final Log LOG = LogFactory.getLog(JobHistoryFileParser.class);
+
+ private final FileSystem fs;
+
+ public JobHistoryFileParser(FileSystem fs) {
+LOG.info("JobHistoryFileParser created with " + fs);
+this.fs = fs;
+ }
+
+ public JobInfo parseHistoryFile(Path path) throws IOException {
+LOG.info("parsing job history file " + path);
+JobHistoryParser parser = new JobHistoryParser(fs, path);
+return parser.parse();
+ }
+
+ public Configuration parseConfiguration(Path path) throws IOException {
+LOG.info("parsing job configuration file " + path);
+Configuration conf = new Configuration(false);
+conf.addResource(fs.open(path));
+
