Repository: incubator-eagle Updated Branches: refs/heads/master 6b87fb3a7 -> 15f39705d
[EAGLE-679] Fix missing CoprocessorService and MR Running feeder task number limi⦠Author: wujinhu <wujinhu...@126.com> Closes #558 from wujinhu/EAGLE-679. Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/15f39705 Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/15f39705 Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/15f39705 Branch: refs/heads/master Commit: 15f39705d0e4c86803d1c552810a3adf2f9b50d6 Parents: 6b87fb3 Author: wujinhu <wujinhu...@126.com> Authored: Tue Oct 25 12:07:48 2016 +0800 Committer: wujinhu <wujinhu...@126.com> Committed: Tue Oct 25 12:07:48 2016 +0800 ---------------------------------------------------------------------- .../apache/eagle/jpm/mr/running/parser/MRJobParser.java | 11 +++++++++++ eagle-topology-assembly/pom.xml | 4 ---- 2 files changed, 11 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/15f39705/eagle-jpm/eagle-jpm-mr-running/src/main/java/org/apache/eagle/jpm/mr/running/parser/MRJobParser.java ---------------------------------------------------------------------- diff --git a/eagle-jpm/eagle-jpm-mr-running/src/main/java/org/apache/eagle/jpm/mr/running/parser/MRJobParser.java b/eagle-jpm/eagle-jpm-mr-running/src/main/java/org/apache/eagle/jpm/mr/running/parser/MRJobParser.java index 0abf9a0..945e8ce 100644 --- a/eagle-jpm/eagle-jpm-mr-running/src/main/java/org/apache/eagle/jpm/mr/running/parser/MRJobParser.java +++ b/eagle-jpm/eagle-jpm-mr-running/src/main/java/org/apache/eagle/jpm/mr/running/parser/MRJobParser.java @@ -78,6 +78,7 @@ public class MRJobParser implements Runnable { private List<String> configKeys; private static final int TOP_BOTTOM_TASKS_BY_ELAPSED_TIME = 10; private static final int FLUSH_TASKS_EVERY_TIME = 5; + private static final int MAX_TASKS_PERMIT = 5000; static { OBJ_MAPPER.configure(JsonParser.Feature.ALLOW_NON_NUMERIC_NUMBERS, true); @@ -428,6 +429,16 @@ public class MRJobParser implements Runnable { } private Function<String, Boolean> fetchTasks = jobId -> { + try { + JobExecutionAPIEntity entity = this.mrJobEntityMap.get(jobId); + int taskNumber = entity.getNumTotalMaps() + entity.getNumTotalReduces(); + if (taskNumber > MAX_TASKS_PERMIT) { + LOG.info("too many tasks {}, ignore tasks", taskNumber); + return true; + } + } catch (Exception e) { + return true; + } String taskURL = app.getTrackingUrl() + Constants.MR_JOBS_URL + "/" + jobId + "/" + Constants.MR_TASKS_URL + "?" + Constants.ANONYMOUS_PARAMETER; InputStream is = null; List<MRTask> tasks = null; http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/15f39705/eagle-topology-assembly/pom.xml ---------------------------------------------------------------------- diff --git a/eagle-topology-assembly/pom.xml b/eagle-topology-assembly/pom.xml index a9dddb0..f5aee5f 100644 --- a/eagle-topology-assembly/pom.xml +++ b/eagle-topology-assembly/pom.xml @@ -42,10 +42,6 @@ <version>${project.version}</version> <exclusions> <exclusion> - <groupId>org.apache.hbase</groupId> - <artifactId>hbase-server</artifactId> - </exclusion> - <exclusion> <groupId>javax.servlet</groupId> <artifactId>servlet-api</artifactId> </exclusion>
