Repository: incubator-eagle Updated Branches: refs/heads/master 4f4a7a3b3 -> 89d136ae9
[EAGLE-820] add unit test for eagle-jpm-mr-history add unit test for eagle-jpm-mr-history https://issues.apache.org/jira/browse/EAGLE-820 Author: koone <[email protected]> Closes #710 from koone/EAGLE-775. Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/89d136ae Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/89d136ae Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/89d136ae Branch: refs/heads/master Commit: 89d136ae927bc1c32fc5d05175258bc44e921227 Parents: 4f4a7a3 Author: koone <[email protected]> Authored: Thu Dec 22 17:59:54 2016 +0800 Committer: wujinhu <[email protected]> Committed: Thu Dec 22 17:59:54 2016 +0800 ---------------------------------------------------------------------- eagle-jpm/eagle-jpm-mr-history/pom.xml | 11 + .../mr/history/storm/JobHistorySpoutTest.java | 216 +++++++++++++++++++ ...41898_508949-1481299030929-testhistory.jhist | 37 ++++ .../resources/job_1479206441898_508949_conf.xml | 18 ++ .../src/test/resources/mrconf_30784.xml | 2 +- 5 files changed, 283 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/89d136ae/eagle-jpm/eagle-jpm-mr-history/pom.xml ---------------------------------------------------------------------- diff --git a/eagle-jpm/eagle-jpm-mr-history/pom.xml b/eagle-jpm/eagle-jpm-mr-history/pom.xml index cbe8de1..feaf8dc 100644 --- a/eagle-jpm/eagle-jpm-mr-history/pom.xml +++ b/eagle-jpm/eagle-jpm-mr-history/pom.xml @@ -75,6 +75,17 @@ </exclusions> </dependency> <dependency> + <groupId>org.powermock</groupId> + <artifactId>powermock-module-junit4</artifactId> + <version>${powermock.version}</version> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.powermock</groupId> + <artifactId>powermock-api-mockito</artifactId> + <version>${powermock.version}</version> + </dependency> + <dependency> <groupId>dumbster</groupId> <artifactId>dumbster</artifactId> <version>1.6</version> http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/89d136ae/eagle-jpm/eagle-jpm-mr-history/src/test/java/org/apache/eagle/jpm/mr/history/storm/JobHistorySpoutTest.java ---------------------------------------------------------------------- diff --git a/eagle-jpm/eagle-jpm-mr-history/src/test/java/org/apache/eagle/jpm/mr/history/storm/JobHistorySpoutTest.java b/eagle-jpm/eagle-jpm-mr-history/src/test/java/org/apache/eagle/jpm/mr/history/storm/JobHistorySpoutTest.java new file mode 100644 index 0000000..a1b62e0 --- /dev/null +++ b/eagle-jpm/eagle-jpm-mr-history/src/test/java/org/apache/eagle/jpm/mr/history/storm/JobHistorySpoutTest.java @@ -0,0 +1,216 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * <p/> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p/> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.eagle.jpm.mr.history.storm; + +import backtype.storm.spout.ISpoutOutputCollector; +import backtype.storm.spout.SpoutOutputCollector; +import backtype.storm.task.TopologyContext; +import com.google.common.collect.Lists; +import com.sun.jersey.api.client.Client; +import com.typesafe.config.ConfigFactory; +import org.apache.commons.io.FileUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.curator.framework.CuratorFramework; +import org.apache.curator.framework.CuratorFrameworkFactory; +import org.apache.curator.framework.imps.CuratorFrameworkState; +import org.apache.curator.retry.ExponentialBackoffRetry; +import org.apache.curator.retry.RetryNTimes; +import org.apache.curator.test.TestingServer; +import org.apache.eagle.jpm.mr.history.MRHistoryJobConfig; +import org.apache.eagle.jpm.mr.history.crawler.JHFCrawlerDriver; +import org.apache.eagle.jpm.mr.history.crawler.JobHistoryContentFilter; +import org.apache.eagle.jpm.mr.history.crawler.JobHistoryContentFilterBuilder; +import org.apache.eagle.jpm.mr.history.metrics.JobCountMetricsGenerator; +import org.apache.eagle.jpm.util.Constants; +import org.apache.eagle.jpm.util.HDFSUtil; +import org.apache.eagle.service.client.impl.EagleServiceClientImpl; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.PathFilter; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.powermock.api.mockito.PowerMockito; +import org.powermock.core.classloader.annotations.PowerMockIgnore; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.InputStream; +import java.lang.reflect.Field; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; +import java.util.regex.Pattern; + +import static org.mockito.Mockito.*; + +/** + * Created by luokun on 12/1/16. + */ +@RunWith(PowerMockRunner.class) +@PrepareForTest( {CuratorFrameworkFactory.class, HDFSUtil.class, JobCountMetricsGenerator.class, JobHistorySpout.class}) +@PowerMockIgnore( {"javax.*", "com.sun.org.*", "org.apache.hadoop.conf.*"}) +public class JobHistorySpoutTest { + + private static final Logger LOG = LoggerFactory.getLogger(JobHistorySpoutTest.class); + + private TestingServer server; + private CuratorFramework zookeeper; + private FileSystem hdfs; + private MRHistoryJobConfig appConfig; + private EagleServiceClientImpl client; + + @Before + public void setUp() throws Exception { + this.appConfig = MRHistoryJobConfig.newInstance(ConfigFactory.load()); + createZk(); + mockHdfs(); + setupMock(); + } + + @After + public void tearDown() throws Exception { + try { + if (zookeeper != null) { + if (!zookeeper.getState().equals(CuratorFrameworkState.STOPPED)) { + zookeeper.close(); + } + } + } finally { + if (server != null) { + server.close(); + } + } + } + + @Test + public void testSpout() throws Exception { + JobHistorySpout spout = createJobHistorySpout(); + List<Object> tuples = new ArrayList<>(); + SpoutOutputCollector collector = new SpoutOutputCollector(new ISpoutOutputCollector() { + @Override + public List<Integer> emit(String s, List<Object> list, Object o) { + tuples.add(list); + return null; + } + + @Override + public void emitDirect(int i, String s, List<Object> list, Object o) { + + } + + @Override + public void reportError(Throwable throwable) { + + } + }); + PowerMockito.mockStatic(CuratorFrameworkFactory.class); + when(CuratorFrameworkFactory.newClient(anyString(), anyInt(), anyInt(), any(RetryNTimes.class))).thenReturn(zookeeper); + spout.open(null, createTopologyContext(), collector); + Field driverField = JobHistorySpout.class.getDeclaredField("driver"); + driverField.setAccessible(true); + JHFCrawlerDriver driver = (JHFCrawlerDriver) driverField.get(spout); + Assert.assertNotNull(driver); + spout.nextTuple(); + Assert.assertTrue(zookeeper.checkExists().forPath("/apps/mr/history/sandbox/partitions/0/timeStamps") != null); + Assert.assertTrue(StringUtils.isNotEmpty((new String(zookeeper.getData().forPath("/apps/mr/history/sandbox/partitions/0/timeStamps"), "UTF-8")))); + verify(client, times(2)).create(any()); + + } + + private void mockHdfs() throws Exception { + PowerMockito.mockStatic(HDFSUtil.class); + hdfs = mock(FileSystem.class); + when(HDFSUtil.getFileSystem(any(Configuration.class))).thenReturn(hdfs); + FileStatus fileDirStatus = new FileStatus(100l, true, 3, 1000l, new Date().getTime(), new Path("/user/history/done/2016/12/09/000508")); + when(hdfs.listStatus(any(Path.class))).thenReturn(new FileStatus[] {fileDirStatus}); + FileStatus filePartitionStatus = new FileStatus(100l, false, 3, 1000l, new Date().getTime(), new Path("/user/history/done/2016/12/09/000508/job_1479206441898_508949-1481299030929-testhistory.jhist")); + when(hdfs.listStatus(any(Path.class), any(PathFilter.class))).thenReturn(new FileStatus[] {filePartitionStatus}); + Path historyFilePath = mock(Path.class); + Path historyConfPath = mock(Path.class); + PowerMockito.whenNew(Path.class).withArguments("/mr-history/done/2016/12/12/000508/job_1479206441898_508949-1481299030929-testhistory.jhist").thenReturn(historyFilePath); + PowerMockito.whenNew(Path.class).withArguments("/mr-history/done/2016/12/12/000508/job_1479206441898_508949_conf.xml").thenReturn(historyConfPath); + + when((InputStream) hdfs.open(historyFilePath)).thenReturn(this.getClass().getResourceAsStream("job_1479206441898_508949-1481299030929-testhistory.jhist")); + when((InputStream) hdfs.open(historyConfPath)).thenReturn(this.getClass().getResourceAsStream("job_1479206441898_508949_conf.xml")); + } + + private TopologyContext createTopologyContext() { + TopologyContext topologyContext = mock(TopologyContext.class); + when(topologyContext.getThisTaskId()).thenReturn(1); + when(topologyContext.getComponentId(1)).thenReturn("test_component"); + List<Integer> globalTaskIds = Lists.newArrayList(1); + when(topologyContext.getComponentTasks("test_component")).thenReturn(globalTaskIds); + return topologyContext; + } + + private void setupMock() throws Exception { + client = mock(EagleServiceClientImpl.class); + MRHistoryJobConfig.EagleServiceConfig eagleServiceConfig = appConfig.getEagleServiceConfig(); + PowerMockito.whenNew(EagleServiceClientImpl.class).withArguments( + eagleServiceConfig.eagleServiceHost, + eagleServiceConfig.eagleServicePort, + eagleServiceConfig.username, + eagleServiceConfig.password).thenReturn(client); + PowerMockito.whenNew(EagleServiceClientImpl.class).withAnyArguments().thenReturn(client); + when(client.create(any())).thenReturn(null); + when(client.getJerseyClient()).thenReturn(new Client()); + } + + private JobHistorySpout createJobHistorySpout() { + //1. trigger init conf + com.typesafe.config.Config jhfAppConf = appConfig.getConfig(); + + //2. init JobHistoryContentFilter + final JobHistoryContentFilterBuilder builder = JobHistoryContentFilterBuilder.newBuilder().acceptJobFile().acceptJobConfFile(); + String[] confKeyPatternsSplit = jhfAppConf.getString("MRConfigureKeys.jobConfigKey").split(","); + List<String> confKeyPatterns = new ArrayList<>(confKeyPatternsSplit.length); + for (String confKeyPattern : confKeyPatternsSplit) { + confKeyPatterns.add(confKeyPattern.trim()); + } + confKeyPatterns.add(Constants.JobConfiguration.CASCADING_JOB); + confKeyPatterns.add(Constants.JobConfiguration.HIVE_JOB); + confKeyPatterns.add(Constants.JobConfiguration.PIG_JOB); + confKeyPatterns.add(Constants.JobConfiguration.SCOOBI_JOB); + + String jobNameKey = jhfAppConf.getString("MRConfigureKeys.jobNameKey"); + builder.setJobNameKey(jobNameKey); + + for (String key : confKeyPatterns) { + builder.includeJobKeyPatterns(Pattern.compile(key)); + } + JobHistoryContentFilter filter = builder.build(); + return new JobHistorySpout(filter, appConfig); + } + + private void createZk() throws Exception { + int port = 2111; + File logFile = new File(System.getProperty("java.io.tmpdir"), "zk/logs/zookeeper-test-" + port); + FileUtils.deleteQuietly(logFile); + server = new TestingServer(port, logFile); + ExponentialBackoffRetry retryPolicy = new ExponentialBackoffRetry(1000, 3); + zookeeper = CuratorFrameworkFactory.newClient(server.getConnectString(), retryPolicy); + } + +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/89d136ae/eagle-jpm/eagle-jpm-mr-history/src/test/resources/job_1479206441898_508949-1481299030929-testhistory.jhist ---------------------------------------------------------------------- diff --git a/eagle-jpm/eagle-jpm-mr-history/src/test/resources/job_1479206441898_508949-1481299030929-testhistory.jhist b/eagle-jpm/eagle-jpm-mr-history/src/test/resources/job_1479206441898_508949-1481299030929-testhistory.jhist new file mode 100644 index 0000000..8f462bb --- /dev/null +++ b/eagle-jpm/eagle-jpm-mr-history/src/test/resources/job_1479206441898_508949-1481299030929-testhistory.jhist @@ -0,0 +1,37 @@ +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +Avro-Json +{"type":"record","name":"Event","namespace":"org.apache.hadoop.mapreduce.jobhistory","fields":[{"name":"type","type":{"type":"enum","name":"EventType","symbols":["JOB_SUBMITTED","JOB_INITED","JOB_FINISHED","JOB_PRIORITY_CHANGED","JOB_STATUS_CHANGED","JOB_QUEUE_CHANGED","JOB_FAILED","JOB_KILLED","JOB_ERROR","JOB_INFO_CHANGED","TASK_STARTED","TASK_FINISHED","TASK_FAILED","TASK_UPDATED","NORMALIZED_RESOURCE","MAP_ATTEMPT_STARTED","MAP_ATTEMPT_FINISHED","MAP_ATTEMPT_FAILED","MAP_ATTEMPT_KILLED","REDUCE_ATTEMPT_STARTED","REDUCE_ATTEMPT_FINISHED","REDUCE_ATTEMPT_FAILED","REDUCE_ATTEMPT_KILLED","SETUP_ATTEMPT_STARTED","SETUP_ATTEMPT_FINISHED","SETUP_ATTEMPT_FAILED","SETUP_ATTEMPT_KILLED","CLEANUP_ATTEMPT_STARTED","CLEANUP_ATTEMPT_FINISHED","CLEANUP_ATTEMPT_FAILED","CLEANUP_ATTEMPT_KILLED","AM_STARTED"]}},{"name":"event","type":[{"type":"record","name":"JobFinished","fields":[{"name":"jobid","type":"string"},{"name":"finishTime","type":"long"},{"name":"finishedMaps","type":"int"},{"name":"f inishedReduces","type":"int"},{"name":"failedMaps","type":"int"},{"name":"failedReduces","type":"int"},{"name":"totalCounters","type":{"type":"record","name":"JhCounters","fields":[{"name":"name","type":"string"},{"name":"groups","type":{"type":"array","items":{"type":"record","name":"JhCounterGroup","fields":[{"name":"name","type":"string"},{"name":"displayName","type":"string"},{"name":"counts","type":{"type":"array","items":{"type":"record","name":"JhCounter","fields":[{"name":"name","type":"string"},{"name":"displayName","type":"string"},{"name":"value","type":"long"}]}}}]}}}]}},{"name":"mapCounters","type":"JhCounters"},{"name":"reduceCounters","type":"JhCounters"}]},{"type":"record","name":"JobInfoChange","fields":[{"name":"jobid","type":"string"},{"name":"submitTime","type":"long"},{"name":"launchTime","type":"long"}]},{"type":"record","name":"JobInited","fields":[{"name":"jobid","type":"string"},{"name":"launchTime","type":"long"},{"name":"totalMaps","type":"int"},{"name":"t otalReduces","type":"int"},{"name":"jobStatus","type":"string"},{"name":"uberized","type":"boolean"}]},{"type":"record","name":"AMStarted","fields":[{"name":"applicationAttemptId","type":"string"},{"name":"startTime","type":"long"},{"name":"containerId","type":"string"},{"name":"nodeManagerHost","type":"string"},{"name":"nodeManagerPort","type":"int"},{"name":"nodeManagerHttpPort","type":"int"}]},{"type":"record","name":"JobPriorityChange","fields":[{"name":"jobid","type":"string"},{"name":"priority","type":"string"}]},{"type":"record","name":"JobQueueChange","fields":[{"name":"jobid","type":"string"},{"name":"jobQueueName","type":"string"}]},{"type":"record","name":"JobStatusChanged","fields":[{"name":"jobid","type":"string"},{"name":"jobStatus","type":"string"}]},{"type":"record","name":"JobSubmitted","fields":[{"name":"jobid","type":"string"},{"name":"jobName","type":"string"},{"name":"userName","type":"string"},{"name":"submitTime","type":"long"},{"name":"jobConfPath","type":"st ring"},{"name":"acls","type":{"type":"map","values":"string"}},{"name":"jobQueueName","type":"string"},{"name":"workflowId","type":["null","string"],"default":null},{"name":"workflowName","type":["null","string"],"default":null},{"name":"workflowNodeName","type":["null","string"],"default":null},{"name":"workflowAdjacencies","type":["null","string"],"default":null},{"name":"workflowTags","type":["null","string"],"default":null}]},{"type":"record","name":"JobUnsuccessfulCompletion","fields":[{"name":"jobid","type":"string"},{"name":"finishTime","type":"long"},{"name":"finishedMaps","type":"int"},{"name":"finishedReduces","type":"int"},{"name":"jobStatus","type":"string"},{"name":"diagnostics","type":["null","string"],"default":null}]},{"type":"record","name":"MapAttemptFinished","fields":[{"name":"taskid","type":"string"},{"name":"attemptId","type":"string"},{"name":"taskType","type":"string"},{"name":"taskStatus","type":"string"},{"name":"mapFinishTime","type":"long"},{"name":"finis hTime","type":"long"},{"name":"hostname","type":"string"},{"name":"port","type":"int"},{"name":"rackname","type":"string"},{"name":"state","type":"string"},{"name":"counters","type":"JhCounters"},{"name":"clockSplits","type":{"type":"array","items":"int"}},{"name":"cpuUsages","type":{"type":"array","items":"int"}},{"name":"vMemKbytes","type":{"type":"array","items":"int"}},{"name":"physMemKbytes","type":{"type":"array","items":"int"}}]},{"type":"record","name":"ReduceAttemptFinished","fields":[{"name":"taskid","type":"string"},{"name":"attemptId","type":"string"},{"name":"taskType","type":"string"},{"name":"taskStatus","type":"string"},{"name":"shuffleFinishTime","type":"long"},{"name":"sortFinishTime","type":"long"},{"name":"finishTime","type":"long"},{"name":"hostname","type":"string"},{"name":"port","type":"int"},{"name":"rackname","type":"string"},{"name":"state","type":"string"},{"name":"counters","type":"JhCounters"},{"name":"clockSplits","type":{"type":"array","items":"int"}} ,{"name":"cpuUsages","type":{"type":"array","items":"int"}},{"name":"vMemKbytes","type":{"type":"array","items":"int"}},{"name":"physMemKbytes","type":{"type":"array","items":"int"}}]},{"type":"record","name":"TaskAttemptFinished","fields":[{"name":"taskid","type":"string"},{"name":"attemptId","type":"string"},{"name":"taskType","type":"string"},{"name":"taskStatus","type":"string"},{"name":"finishTime","type":"long"},{"name":"rackname","type":"string"},{"name":"hostname","type":"string"},{"name":"state","type":"string"},{"name":"counters","type":"JhCounters"}]},{"type":"record","name":"TaskAttemptStarted","fields":[{"name":"taskid","type":"string"},{"name":"taskType","type":"string"},{"name":"attemptId","type":"string"},{"name":"startTime","type":"long"},{"name":"trackerName","type":"string"},{"name":"httpPort","type":"int"},{"name":"shufflePort","type":"int"},{"name":"containerId","type":"string"},{"name":"locality","type":["null","string"],"default":null},{"name":"avataar","type" :["null","string"],"default":null}]},{"type":"record","name":"TaskAttemptUnsuccessfulCompletion","fields":[{"name":"taskid","type":"string"},{"name":"taskType","type":"string"},{"name":"attemptId","type":"string"},{"name":"finishTime","type":"long"},{"name":"hostname","type":"string"},{"name":"port","type":"int"},{"name":"rackname","type":"string"},{"name":"status","type":"string"},{"name":"error","type":"string"},{"name":"counters","type":["null","JhCounters"],"default":null},{"name":"clockSplits","type":{"type":"array","items":"int"}},{"name":"cpuUsages","type":{"type":"array","items":"int"}},{"name":"vMemKbytes","type":{"type":"array","items":"int"}},{"name":"physMemKbytes","type":{"type":"array","items":"int"}}]},{"type":"record","name":"TaskFailed","fields":[{"name":"taskid","type":"string"},{"name":"taskType","type":"string"},{"name":"finishTime","type":"long"},{"name":"error","type":"string"},{"name":"failedDueToAttempt","type":["null","string"]},{"name":"status","type":"stri ng"},{"name":"counters","type":["null","JhCounters"],"default":null}]},{"type":"record","name":"TaskFinished","fields":[{"name":"taskid","type":"string"},{"name":"taskType","type":"string"},{"name":"finishTime","type":"long"},{"name":"status","type":"string"},{"name":"counters","type":"JhCounters"},{"name":"successfulAttemptId","type":["null","string"],"default":null}]},{"type":"record","name":"TaskStarted","fields":[{"name":"taskid","type":"string"},{"name":"taskType","type":"string"},{"name":"startTime","type":"long"},{"name":"splitLocations","type":"string"}]},{"type":"record","name":"TaskUpdated","fields":[{"name":"taskid","type":"string"},{"name":"finishTime","type":"long"}]}]}]} +{"type":"AM_STARTED","event":{"org.apache.hadoop.mapreduce.jobhistory.AMStarted":{"applicationAttemptId":"appattempt_1479206441898_508949_000001","startTime":1481299032273,"containerId":"container_e11_1479206441898_508949_01_000001","nodeManagerHost":"eagle-hadoop-1","nodeManagerPort":8041,"nodeManagerHttpPort":8042}}} + +{"type":"JOB_SUBMITTED","event":{"org.apache.hadoop.mapreduce.jobhistory.JobSubmitted":{"jobid":"job_1479206441898_508949","jobName":"test","userName":"bi_etl","submitTime":1481299030929,"jobConfPath":"viewfs://cluster/tmp/job_1479206441898_508949/job.xml","acls":{},"jobQueueName":"bi_etl","workflowId":{"string":"hive_test"},"workflowName":{"string":"select * from 1"},"workflowNodeName":{"string":"Stage-2"},"workflowAdjacencies":{"string":"\"Stage-4\"=\"Stage-5\" \"Stage-5\"=\"Stage-0\" \"Stage-6\"=\"Stage-3,Stage-2,Stage-4\" \"Stage-1\"=\"Stage-6\" \"Stage-2\"=\"Stage-0\" \"Stage-3\"=\"Stage-0\" "},"workflowTags":{"string":""}}}} + +{"type":"JOB_QUEUE_CHANGED","event":{"org.apache.hadoop.mapreduce.jobhistory.JobQueueChange":{"jobid":"job_1479206441898_508949","jobQueueName":"bi_etl"}}} + +{"type":"JOB_INITED","event":{"org.apache.hadoop.mapreduce.jobhistory.JobInited":{"jobid":"job_1479206441898_508949","launchTime":1481299036224,"totalMaps":1,"totalReduces":0,"jobStatus":"INITED","uberized":false}}} + +{"type":"JOB_INFO_CHANGED","event":{"org.apache.hadoop.mapreduce.jobhistory.JobInfoChange":{"jobid":"job_1479206441898_508949","submitTime":1481299030929,"launchTime":1481299036224}}} + +{"type":"TASK_STARTED","event":{"org.apache.hadoop.mapreduce.jobhistory.TaskStarted":{"taskid":"task_1479206441898_508949_m_000000","taskType":"MAP","startTime":1481299036697,"splitLocations":"eagle-hadoop-1"}}} + +{"type":"MAP_ATTEMPT_STARTED","event":{"org.apache.hadoop.mapreduce.jobhistory.TaskAttemptStarted":{"taskid":"task_1479206441898_508949_m_000000","taskType":"MAP","attemptId":"attempt_1479206441898_508949_m_000000_0","startTime":1481299038480,"trackerName":"eagle-hadoop-1":8042,"shufflePort":13562,"containerId":"container_e11_1479206441898_508949_01_000002","locality":{"string":"OFF_SWITCH"},"avataar":{"string":"VIRGIN"}}}} + +{"type":"MAP_ATTEMPT_FINISHED","event":{"org.apache.hadoop.mapreduce.jobhistory.MapAttemptFinished":{"taskid":"task_1479206441898_508949_m_000000","attemptId":"attempt_1479206441898_508949_m_000000_0","taskType":"MAP","taskStatus":"SUCCEEDED","mapFinishTime":0,"finishTime":1481299042976,"hostname":"eagle-hadoop-1","port":8041,"rackname":"/rowb/rack12","state":"viewfs://cluster/tmp/000002_0:0+2263","counters":{"name":"COUNTERS","groups":[{"name":"org.apache.hadoop.mapreduce.FileSystemCounter","displayName":"File System Counters","counts":[{"name":"FILE_BYTES_READ","displayName":"FILE: Number of bytes read","value":0},{"name":"FILE_BYTES_WRITTEN","displayName":"FILE: Number of bytes written","value":248074},{"name":"FILE_READ_OPS","displayName":"FILE: Number of read operations","value":0},{"name":"FILE_LARGE_READ_OPS","displayName":"FILE: Number of large read operations","value":0},{"name":"FILE_WRITE_OPS","displayName":"FILE: Number of write operations","value":0},{"name":"HDFS_BYTES _READ","displayName":"HDFS: Number of bytes read","value":13558},{"name":"HDFS_BYTES_WRITTEN","displayName":"HDFS: Number of bytes written","value":11273},{"name":"HDFS_READ_OPS","displayName":"HDFS: Number of read operations","value":9},{"name":"HDFS_LARGE_READ_OPS","displayName":"HDFS: Number of large read operations","value":0},{"name":"HDFS_WRITE_OPS","displayName":"HDFS: Number of write operations","value":2},{"name":"VIEWFS_BYTES_READ","displayName":"VIEWFS: Number of bytes read","value":0},{"name":"VIEWFS_BYTES_WRITTEN","displayName":"VIEWFS: Number of bytes written","value":0},{"name":"VIEWFS_READ_OPS","displayName":"VIEWFS: Number of read operations","value":0},{"name":"VIEWFS_LARGE_READ_OPS","displayName":"VIEWFS: Number of large read operations","value":0},{"name":"VIEWFS_WRITE_OPS","displayName":"VIEWFS: Number of write operations","value":0}]},{"name":"org.apache.hadoop.mapreduce.TaskCounter","displayName":"Map-Reduce Framework","counts":[{"name":"MAP_INPUT_RECORDS","di splayName":"Map input records","value":150},{"name":"MAP_OUTPUT_RECORDS","displayName":"Map output records","value":0},{"name":"SPLIT_RAW_BYTES","displayName":"Input split bytes","value":877},{"name":"SPILLED_RECORDS","displayName":"Spilled Records","value":0},{"name":"FAILED_SHUFFLE","displayName":"Failed Shuffles","value":0},{"name":"MERGED_MAP_OUTPUTS","displayName":"Merged Map outputs","value":0},{"name":"GC_TIME_MILLIS","displayName":"GC time elapsed (ms)","value":28},{"name":"CPU_MILLISECONDS","displayName":"CPU time spent (ms)","value":2040},{"name":"PHYSICAL_MEMORY_BYTES","displayName":"Physical memory (bytes) snapshot","value":383455232},{"name":"VIRTUAL_MEMORY_BYTES","displayName":"Virtual memory (bytes) snapshot","value":2415357952},{"name":"COMMITTED_HEAP_BYTES","displayName":"Total committed heap usage (bytes)","value":1011351552}]},{"name":"HIVE","displayName":"HIVE","counts":[{"name":"CREATED_FILES","displayName":"CREATED_FILES","value":1},{"name":"DESERIALIZE_ERRORS" ,"displayName":"DESERIALIZE_ERRORS","value":0},{"name":"RECORDS_IN","displayName":"RECORDS_IN","value":150},{"name":"RECORDS_OUT_0","displayName":"RECORDS_OUT_0","value":150}]},{"name":"org.apache.hadoop.mapreduce.lib.input.FileInputFormatCounter","displayName":"File Input Format Counters ","counts":[{"name":"BYTES_READ","displayName":"Bytes Read","value":0}]},{"name":"org.apache.hadoop.mapreduce.lib.output.FileOutputFormatCounter","displayName":"File Output Format Counters ","counts":[{"name":"BYTES_WRITTEN","displayName":"Bytes Written","value":0}]}]},"clockSplits":[4333,13,14,13,13,14,13,13,14,13,13,14],"cpuUsages":[170,170,170,170,170,170,170,170,170,170,170,170],"vMemKbytes":[98281,294843,491405,687967,884530,1081092,1277655,1474216,1670779,1867342,2063903,2260466],"physMemKbytes":[15602,46808,78014,109219,140425,171630,202836,234041,265248,296453,327658,358865]}}} + +{"type":"TASK_FINISHED","event":{"org.apache.hadoop.mapreduce.jobhistory.TaskFinished":{"taskid":"task_1479206441898_508949_m_000000","taskType":"MAP","finishTime":1481299042976,"status":"SUCCEEDED","counters":{"name":"COUNTERS","groups":[{"name":"org.apache.hadoop.mapreduce.FileSystemCounter","displayName":"File System Counters","counts":[{"name":"FILE_BYTES_READ","displayName":"FILE: Number of bytes read","value":0},{"name":"FILE_BYTES_WRITTEN","displayName":"FILE: Number of bytes written","value":248074},{"name":"FILE_READ_OPS","displayName":"FILE: Number of read operations","value":0},{"name":"FILE_LARGE_READ_OPS","displayName":"FILE: Number of large read operations","value":0},{"name":"FILE_WRITE_OPS","displayName":"FILE: Number of write operations","value":0},{"name":"HDFS_BYTES_READ","displayName":"HDFS: Number of bytes read","value":13558},{"name":"HDFS_BYTES_WRITTEN","displayName":"HDFS: Number of bytes written","value":11273},{"name":"HDFS_READ_OPS","displayName":"HDFS: Nu mber of read operations","value":9},{"name":"HDFS_LARGE_READ_OPS","displayName":"HDFS: Number of large read operations","value":0},{"name":"HDFS_WRITE_OPS","displayName":"HDFS: Number of write operations","value":2},{"name":"VIEWFS_BYTES_READ","displayName":"VIEWFS: Number of bytes read","value":0},{"name":"VIEWFS_BYTES_WRITTEN","displayName":"VIEWFS: Number of bytes written","value":0},{"name":"VIEWFS_READ_OPS","displayName":"VIEWFS: Number of read operations","value":0},{"name":"VIEWFS_LARGE_READ_OPS","displayName":"VIEWFS: Number of large read operations","value":0},{"name":"VIEWFS_WRITE_OPS","displayName":"VIEWFS: Number of write operations","value":0}]},{"name":"org.apache.hadoop.mapreduce.TaskCounter","displayName":"Map-Reduce Framework","counts":[{"name":"MAP_INPUT_RECORDS","displayName":"Map input records","value":150},{"name":"MAP_OUTPUT_RECORDS","displayName":"Map output records","value":0},{"name":"SPLIT_RAW_BYTES","displayName":"Input split bytes","value":877},{"name":"S PILLED_RECORDS","displayName":"Spilled Records","value":0},{"name":"FAILED_SHUFFLE","displayName":"Failed Shuffles","value":0},{"name":"MERGED_MAP_OUTPUTS","displayName":"Merged Map outputs","value":0},{"name":"GC_TIME_MILLIS","displayName":"GC time elapsed (ms)","value":28},{"name":"CPU_MILLISECONDS","displayName":"CPU time spent (ms)","value":2040},{"name":"PHYSICAL_MEMORY_BYTES","displayName":"Physical memory (bytes) snapshot","value":383455232},{"name":"VIRTUAL_MEMORY_BYTES","displayName":"Virtual memory (bytes) snapshot","value":2415357952},{"name":"COMMITTED_HEAP_BYTES","displayName":"Total committed heap usage (bytes)","value":1011351552}]},{"name":"HIVE","displayName":"HIVE","counts":[{"name":"CREATED_FILES","displayName":"CREATED_FILES","value":1},{"name":"DESERIALIZE_ERRORS","displayName":"DESERIALIZE_ERRORS","value":0},{"name":"RECORDS_IN","displayName":"RECORDS_IN","value":150},{"name":"RECORDS_OUT_0","displayName":"RECORDS_OUT_0","value":150}]},{"name":"org.apache.hadoo p.mapreduce.lib.input.FileInputFormatCounter","displayName":"File Input Format Counters ","counts":[{"name":"BYTES_READ","displayName":"Bytes Read","value":0}]},{"name":"org.apache.hadoop.mapreduce.lib.output.FileOutputFormatCounter","displayName":"File Output Format Counters ","counts":[{"name":"BYTES_WRITTEN","displayName":"Bytes Written","value":0}]}]},"successfulAttemptId":{"string":"attempt_1479206441898_508949_m_000000_0"}}}} + +{"type":"JOB_FINISHED","event":{"org.apache.hadoop.mapreduce.jobhistory.JobFinished":{"jobid":"job_1479206441898_508949","finishTime":1481299043114,"finishedMaps":1,"finishedReduces":0,"failedMaps":0,"failedReduces":0,"totalCounters":{"name":"TOTAL_COUNTERS","groups":[{"name":"org.apache.hadoop.mapreduce.FileSystemCounter","displayName":"File System Counters","counts":[{"name":"FILE_BYTES_READ","displayName":"FILE: Number of bytes read","value":0},{"name":"FILE_BYTES_WRITTEN","displayName":"FILE: Number of bytes written","value":248074},{"name":"FILE_READ_OPS","displayName":"FILE: Number of read operations","value":0},{"name":"FILE_LARGE_READ_OPS","displayName":"FILE: Number of large read operations","value":0},{"name":"FILE_WRITE_OPS","displayName":"FILE: Number of write operations","value":0},{"name":"HDFS_BYTES_READ","displayName":"HDFS: Number of bytes read","value":13558},{"name":"HDFS_BYTES_WRITTEN","displayName":"HDFS: Number of bytes written","value":11273},{"name":"HDFS_REA D_OPS","displayName":"HDFS: Number of read operations","value":9},{"name":"HDFS_LARGE_READ_OPS","displayName":"HDFS: Number of large read operations","value":0},{"name":"HDFS_WRITE_OPS","displayName":"HDFS: Number of write operations","value":2},{"name":"VIEWFS_BYTES_READ","displayName":"VIEWFS: Number of bytes read","value":0},{"name":"VIEWFS_BYTES_WRITTEN","displayName":"VIEWFS: Number of bytes written","value":0},{"name":"VIEWFS_READ_OPS","displayName":"VIEWFS: Number of read operations","value":0},{"name":"VIEWFS_LARGE_READ_OPS","displayName":"VIEWFS: Number of large read operations","value":0},{"name":"VIEWFS_WRITE_OPS","displayName":"VIEWFS: Number of write operations","value":0}]},{"name":"org.apache.hadoop.mapreduce.JobCounter","displayName":"Job Counters ","counts":[{"name":"TOTAL_LAUNCHED_MAPS","displayName":"Launched map tasks","value":1},{"name":"OTHER_LOCAL_MAPS","displayName":"Other local map tasks","value":1},{"name":"SLOTS_MILLIS_MAPS","displayName":"Total time spent by all maps in occupied slots (ms)","value":8992},{"name":"MILLIS_MAPS","displayName":"Total time spent by all map tasks (ms)","value":4496},{"name":"VCORES_MILLIS_MAPS","displayName":"Total vcore-seconds taken by all map tasks","value":4496},{"name":"MB_MILLIS_MAPS","displayName":"Total megabyte-seconds taken by all map tasks","value":9207808}]},{"name":"org.apache.hadoop.mapreduce.TaskCounter","displayName":"Map-Reduce Framework","counts":[{"name":"MAP_INPUT_RECORDS","displayName":"Map input records","value":150},{"name":"MAP_OUTPUT_RECORDS","displayName":"Map output records","value":0},{"name":"SPLIT_RAW_BYTES","displayName":"Input split bytes","value":877},{"name":"SPILLED_RECORDS","displayName":"Spilled Records","value":0},{"name":"FAILED_SHUFFLE","displayName":"Failed Shuffles","value":0},{"name":"MERGED_MAP_OUTPUTS","displayName":"Merged Map outputs","value":0},{"name":"GC_TIME_MILLIS","displayName":"GC time elapsed (ms)","value":28},{"name":"CPU_MILLISECONDS","displayName": "CPU time spent (ms)","value":2040},{"name":"PHYSICAL_MEMORY_BYTES","displayName":"Physical memory (bytes) snapshot","value":383455232},{"name":"VIRTUAL_MEMORY_BYTES","displayName":"Virtual memory (bytes) snapshot","value":2415357952},{"name":"COMMITTED_HEAP_BYTES","displayName":"Total committed heap usage (bytes)","value":1011351552}]},{"name":"HIVE","displayName":"HIVE","counts":[{"name":"CREATED_FILES","displayName":"CREATED_FILES","value":1},{"name":"DESERIALIZE_ERRORS","displayName":"DESERIALIZE_ERRORS","value":0},{"name":"RECORDS_IN","displayName":"RECORDS_IN","value":150},{"name":"RECORDS_OUT_0","displayName":"RECORDS_OUT_0","value":150}]},{"name":"org.apache.hadoop.mapreduce.lib.input.FileInputFormatCounter","displayName":"File Input Format Counters ","counts":[{"name":"BYTES_READ","displayName":"Bytes Read","value":0}]},{"name":"org.apache.hadoop.mapreduce.lib.output.FileOutputFormatCounter","displayName":"File Output Format Counters ","counts":[{"name":"BYTES_WRITTEN","dis playName":"Bytes Written","value":0}]}]},"mapCounters":{"name":"MAP_COUNTERS","groups":[{"name":"org.apache.hadoop.mapreduce.FileSystemCounter","displayName":"File System Counters","counts":[{"name":"FILE_BYTES_READ","displayName":"FILE: Number of bytes read","value":0},{"name":"FILE_BYTES_WRITTEN","displayName":"FILE: Number of bytes written","value":248074},{"name":"FILE_READ_OPS","displayName":"FILE: Number of read operations","value":0},{"name":"FILE_LARGE_READ_OPS","displayName":"FILE: Number of large read operations","value":0},{"name":"FILE_WRITE_OPS","displayName":"FILE: Number of write operations","value":0},{"name":"HDFS_BYTES_READ","displayName":"HDFS: Number of bytes read","value":13558},{"name":"HDFS_BYTES_WRITTEN","displayName":"HDFS: Number of bytes written","value":11273},{"name":"HDFS_READ_OPS","displayName":"HDFS: Number of read operations","value":9},{"name":"HDFS_LARGE_READ_OPS","displayName":"HDFS: Number of large read operations","value":0},{"name":"HDFS_WRITE_ OPS","displayName":"HDFS: Number of write operations","value":2},{"name":"VIEWFS_BYTES_READ","displayName":"VIEWFS: Number of bytes read","value":0},{"name":"VIEWFS_BYTES_WRITTEN","displayName":"VIEWFS: Number of bytes written","value":0},{"name":"VIEWFS_READ_OPS","displayName":"VIEWFS: Number of read operations","value":0},{"name":"VIEWFS_LARGE_READ_OPS","displayName":"VIEWFS: Number of large read operations","value":0},{"name":"VIEWFS_WRITE_OPS","displayName":"VIEWFS: Number of write operations","value":0}]},{"name":"org.apache.hadoop.mapreduce.TaskCounter","displayName":"Map-Reduce Framework","counts":[{"name":"MAP_INPUT_RECORDS","displayName":"Map input records","value":150},{"name":"MAP_OUTPUT_RECORDS","displayName":"Map output records","value":0},{"name":"SPLIT_RAW_BYTES","displayName":"Input split bytes","value":877},{"name":"SPILLED_RECORDS","displayName":"Spilled Records","value":0},{"name":"FAILED_SHUFFLE","displayName":"Failed Shuffles","value":0},{"name":"MERGED_MAP_OUTP UTS","displayName":"Merged Map outputs","value":0},{"name":"GC_TIME_MILLIS","displayName":"GC time elapsed (ms)","value":28},{"name":"CPU_MILLISECONDS","displayName":"CPU time spent (ms)","value":2040},{"name":"PHYSICAL_MEMORY_BYTES","displayName":"Physical memory (bytes) snapshot","value":383455232},{"name":"VIRTUAL_MEMORY_BYTES","displayName":"Virtual memory (bytes) snapshot","value":2415357952},{"name":"COMMITTED_HEAP_BYTES","displayName":"Total committed heap usage (bytes)","value":1011351552}]},{"name":"HIVE","displayName":"HIVE","counts":[{"name":"CREATED_FILES","displayName":"CREATED_FILES","value":1},{"name":"DESERIALIZE_ERRORS","displayName":"DESERIALIZE_ERRORS","value":0},{"name":"RECORDS_IN","displayName":"RECORDS_IN","value":150},{"name":"RECORDS_OUT_0","displayName":"RECORDS_OUT_0","value":150}]},{"name":"org.apache.hadoop.mapreduce.lib.input.FileInputFormatCounter","displayName":"File Input Format Counters ","counts":[{"name":"BYTES_READ","displayName":"Bytes Read","va lue":0}]},{"name":"org.apache.hadoop.mapreduce.lib.output.FileOutputFormatCounter","displayName":"File Output Format Counters ","counts":[{"name":"BYTES_WRITTEN","displayName":"Bytes Written","value":0}]}]},"reduceCounters":{"name":"REDUCE_COUNTERS","groups":[]}}}} http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/89d136ae/eagle-jpm/eagle-jpm-mr-history/src/test/resources/job_1479206441898_508949_conf.xml ---------------------------------------------------------------------- diff --git a/eagle-jpm/eagle-jpm-mr-history/src/test/resources/job_1479206441898_508949_conf.xml b/eagle-jpm/eagle-jpm-mr-history/src/test/resources/job_1479206441898_508949_conf.xml new file mode 100644 index 0000000..6d22996 --- /dev/null +++ b/eagle-jpm/eagle-jpm-mr-history/src/test/resources/job_1479206441898_508949_conf.xml @@ -0,0 +1,18 @@ +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<?xml version="1.0" encoding="UTF-8" standalone="no"?><configuration> +</configuration> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/89d136ae/eagle-jpm/eagle-jpm-mr-running/src/test/resources/mrconf_30784.xml ---------------------------------------------------------------------- diff --git a/eagle-jpm/eagle-jpm-mr-running/src/test/resources/mrconf_30784.xml b/eagle-jpm/eagle-jpm-mr-running/src/test/resources/mrconf_30784.xml index 1b7b8ed..0324fb3 100644 --- a/eagle-jpm/eagle-jpm-mr-running/src/test/resources/mrconf_30784.xml +++ b/eagle-jpm/eagle-jpm-mr-running/src/test/resources/mrconf_30784.xml @@ -13,4 +13,4 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ---><conf><path>viewfs://xxxxx/user/xxx/.staging/job_1479206441898_30784/job.xml</path><property><name>eagle.job.name</name><value>eagletest</value><source>hdfs-default.xml</source><source>viewfs://xxxxx/user/xxx/.staging/job_1479206441898_30784/job.xml</source></property><property><name>hive.query.string</name><value>insert overwrite table xxxx</value><source>programatically</source><source>viewfs://xxx/user/xxx/.staging/job_1479206441898_124837/job.xml</source></property><property><name>hive.optimize.skewjoin.compiletime</name><value>false</value><source>programatically</source><source>org.apache.hadoop.hive.conf.LoopingByteArrayInputStream@70a6620d</source><source>viewfs://xxxxx/user/xxx/.staging/job_1479206441898_30784/job.xml</source></property><!--<property><name>hadoop.security.group.mapping.ldap.search.filter.user</name><value>(&(objectClass=user)(sAMAccountName={0}))</value><source>core-default.xml</source><source>viewfs://xxxxx/user/xxx/.staging/job_1479206441898_30784/job. xml</source></property>--><property><name>dfs.datanode.data.dir</name><value>file://${hadoop.tmp.dir}/dfs/data</value><source>hdfs-default.xml</source><source>viewfs://xxxxx/user/xxx/.staging/job_1479206441898_30784/job.xml</source></property></conf> \ No newline at end of file +--><conf><path>viewfs://xxxxx/user/xxx/.staging/job_1479206441898_30784/job.xml</path><property><name>eagle.job.name</name><value>eagletest</value><source>hdfs-default.xml</source><source>viewfs://xxxxx/user/xxx/.staging/job_1479206441898_30784/job.xml</source></property><property><name>hive.query.string</name><value>insert overwrite table xxxx</value><source>programatically</source><source>viewfs://xxx/user/xxx/.staging/job_1479206441898_124837/job.xml</source></property><property><name>hive.optimize.skewjoin.compiletime</name><value>false</value><source>programatically</source><source>org.apache.hadoop.hive.conf.LoopingByteArrayInputStream@70a6620d</source><source>viewfs://xxxxx/user/xxx/.staging/job_1479206441898_30784/job.xml</source></property><property><name>hadoop.security.group.mapping.ldap.search.filter.user</name><value>(&(objectClass=user)(sAMAccountName={0}))</value><source>core-default.xml</source><source>viewfs://xxxxx/user/xxx/.staging/job_1479206441898_30784/job. xml</source></property><property><name>dfs.datanode.data.dir</name><value>file://${hadoop.tmp.dir}/dfs/data</value><source>hdfs-default.xml</source><source>viewfs://xxxxx/user/xxx/.staging/job_1479206441898_30784/job.xml</source></property></conf> \ No newline at end of file
