Hi, I'm running Hadoop 2.2.0, and occasionally some my MR jobs failed due to below error.
The issue is the job was running on 2014-06-24, but the path was pointed to /2014/06/01, do you guys know what's going on here? 2014-06-24 08:04:28.170 -0700 [pool-1-thread-157] java.io.IOException: org.apache.hadoop.ipc.RemoteException(org.apache.hadoop.yarn.exceptions.YarnRuntimeException): java.io.FileNotFoundException: File /tmp/hadoop-yarn/staging/history/done/2014/06/01/000059 does not exist. at org.apache.hadoop.mapreduce.v2.hs.CachedHistoryStorage.getFullJob(CachedHistoryStorage.java:122) at org.apache.hadoop.mapreduce.v2.hs.JobHistory.getJob(JobHistory.java:207) at org.apache.hadoop.mapreduce.v2.hs.HistoryClientService$HSClientProtocolHandler$1.run(HistoryClientService.java:200) at org.apache.hadoop.mapreduce.v2.hs.HistoryClientService$HSClientProtocolHandler$1.run(HistoryClientService.java:196) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:396) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1491) at org.apache.hadoop.mapreduce.v2.hs.HistoryClientService$HSClientProtocolHandler.verifyAndGetJob(HistoryClientService.java:196) at org.apache.hadoop.mapreduce.v2.hs.HistoryClientService$HSClientProtocolHandler.getJobReport(HistoryClientService.java:228) at org.apache.hadoop.mapreduce.v2.api.impl.pb.service.MRClientProtocolPBServiceImpl.getJobReport(MRClientProtocolPBServiceImpl.java:122) at org.apache.hadoop.yarn.proto.MRClientProtocol$MRClientProtocolService$2.callBlockingMethod(MRClientProtocol.java:275) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:585) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:928) at org.apache.hadoop.ipc.Server$Handler$1.run(Server.java:2048) at org.apache.hadoop.ipc.Server$Handler$1.run(Server.java:2044) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:396) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1491) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2042) Caused by: java.io.FileNotFoundException: File /tmp/hadoop-yarn/staging/history/done/2014/06/01/000059 does not exist. at org.apache.hadoop.fs.Hdfs$DirListingIterator.<init>(Hdfs.java:205) at org.apache.hadoop.fs.Hdfs$DirListingIterator.<init>(Hdfs.java:189) at org.apache.hadoop.fs.Hdfs$2.<init>(Hdfs.java:171) at org.apache.hadoop.fs.Hdfs.listStatusIterator(Hdfs.java:171) at org.apache.hadoop.fs.FileContext$20.next(FileContext.java:1392) at org.apache.hadoop.fs.FileContext$20.next(FileContext.java:1387) at org.apache.hadoop.fs.FSLinkResolver.resolve(FSLinkResolver.java:90) at org.apache.hadoop.fs.FileContext.listStatus(FileContext.java:1387) at org.apache.hadoop.mapreduce.v2.hs.HistoryFileManager.scanDirectory(HistoryFileManager.java:655) at org.apache.hadoop.mapreduce.v2.hs.HistoryFileManager.scanDirectoryForHistoryFiles(HistoryFileManager.java:668) at org.apache.hadoop.mapreduce.v2.hs.HistoryFileManager.scanOldDirsForJob(HistoryFileManager.java:825) at org.apache.hadoop.mapreduce.v2.hs.HistoryFileManager.getFileInfo(HistoryFileManager.java:854) at org.apache.hadoop.mapreduce.v2.hs.CachedHistoryStorage.getFullJob(CachedHistoryStorage.java:107) ... 18 more at org.apache.hadoop.mapred.ClientServiceDelegate.invoke(ClientServiceDelegate.java:331) ~[thirdeye-action.jar:na] at org.apache.hadoop.mapred.ClientServiceDelegate.getJobStatus(ClientServiceDelegate.java:416) ~[thirdeye-action.jar:na] at org.apache.hadoop.mapred.TIEYarnRunner.getJobStatus(TIEYarnRunner.java:534) ~[thirdeye-action.jar:na] at org.apache.hadoop.mapreduce.Job$1.run(Job.java:314) ~[thirdeye-action.jar:na] at org.apache.hadoop.mapreduce.Job$1.run(Job.java:311) ~[thirdeye-action.jar:na] at java.security.AccessController.doPrivileged(Native Method) ~[na:1.6.0_23] at javax.security.auth.Subject.doAs(Subject.java:396) ~[na:1.6.0_23] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1491) ~[thirdeye-action.jar:na] at org.apache.hadoop.mapreduce.Job.updateStatus(Job.java:311) ~[thirdeye-action.jar:na] at org.apache.hadoop.mapreduce.Job.isComplete(Job.java:599) ~[thirdeye-action.jar:na] at org.apache.hadoop.mapreduce.Job.waitForCompletion(Job.java:1294) ~[thirdeye-action.jar:na] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:441) [na:1.6.0_23] at java.util.concurrent.FutureTask$Sync.innerRunAndReset(FutureTask.java:317) [na:1.6.0_23] at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:150) [na:1.6.0_23] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$101(ScheduledThreadPoolExecutor.java:98) [na:1.6.0_23] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.runPeriodic(ScheduledThreadPoolExecutor.java:180) [na:1.6.0_23] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:204) [na:1.6.0_23] at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886) [na:1.6.0_23] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908) [na:1.6.0_23] at java.lang.Thread.run(Thread.java:662) [na:1.6.0_23] Caused by: org.apache.hadoop.ipc.RemoteException: java.io.FileNotFoundException: File /tmp/hadoop-yarn/staging/history/done/2014/06/01/000059 does not exist. at org.apache.hadoop.mapreduce.v2.hs.CachedHistoryStorage.getFullJob(CachedHistoryStorage.java:122) at org.apache.hadoop.mapreduce.v2.hs.JobHistory.getJob(JobHistory.java:207) at org.apache.hadoop.mapreduce.v2.hs.HistoryClientService$HSClientProtocolHandler$1.run(HistoryClientService.java:200) at org.apache.hadoop.mapreduce.v2.hs.HistoryClientService$HSClientProtocolHandler$1.run(HistoryClientService.java:196) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:396) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1491) at org.apache.hadoop.mapreduce.v2.hs.HistoryClientService$HSClientProtocolHandler.verifyAndGetJob(HistoryClientService.java:196) at org.apache.hadoop.mapreduce.v2.hs.HistoryClientService$HSClientProtocolHandler.getJobReport(HistoryClientService.java:228) at org.apache.hadoop.mapreduce.v2.api.impl.pb.service.MRClientProtocolPBServiceImpl.getJobReport(MRClientProtocolPBServiceImpl.java:122) at org.apache.hadoop.yarn.proto.MRClientProtocol$MRClientProtocolService$2.callBlockingMethod(MRClientProtocol.java:275) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:585) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:928) at org.apache.hadoop.ipc.Server$Handler$1.run(Server.java:2048) at org.apache.hadoop.ipc.Server$Handler$1.run(Server.java:2044) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:396) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1491) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2042) Caused by: java.io.FileNotFoundException: File /tmp/hadoop-yarn/staging/history/done/2014/06/01/000059 does not exist. at org.apache.hadoop.fs.Hdfs$DirListingIterator.<init>(Hdfs.java:205) at org.apache.hadoop.fs.Hdfs$DirListingIterator.<init>(Hdfs.java:189) at org.apache.hadoop.fs.Hdfs$2.<init>(Hdfs.java:171) at org.apache.hadoop.fs.Hdfs.listStatusIterator(Hdfs.java:171) at org.apache.hadoop.fs.FileContext$20.next(FileContext.java:1392) at org.apache.hadoop.fs.FileContext$20.next(FileContext.java:1387) at org.apache.hadoop.fs.FSLinkResolver.resolve(FSLinkResolver.java:90) at org.apache.hadoop.fs.FileContext.listStatus(FileContext.java:1387) at org.apache.hadoop.mapreduce.v2.hs.HistoryFileManager.scanDirectory(HistoryFileManager.java:655) at org.apache.hadoop.mapreduce.v2.hs.HistoryFileManager.scanDirectoryForHistoryFiles(HistoryFileManager.java:668) at org.apache.hadoop.mapreduce.v2.hs.HistoryFileManager.scanOldDirsForJob(HistoryFileManager.java:825) at org.apache.hadoop.mapreduce.v2.hs.HistoryFileManager.getFileInfo(HistoryFileManager.java:854) at org.apache.hadoop.mapreduce.v2.hs.CachedHistoryStorage.getFullJob(CachedHistoryStorage.java:107) ... 18 more at org.apache.hadoop.ipc.Client.call(Client.java:1347) ~[thirdeye-action.jar:na] at org.apache.hadoop.ipc.Client.call(Client.java:1300) ~[thirdeye-action.jar:na] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:206) ~[thirdeye-action.jar:na] at $Proxy13.getJobReport(Unknown Source) ~[na:na] at org.apache.hadoop.mapreduce.v2.api.impl.pb.client.MRClientProtocolPBClientImpl.getJobReport(MRClientProtocolPBClientImpl.java:133) ~[thirdeye-action.jar:na] at sun.reflect.GeneratedMethodAccessor63.invoke(Unknown Source) ~[na:na] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) ~[na:1.6.0_23] at java.lang.reflect.Method.invoke(Method.java:597) ~[na:1.6.0_23] at org.apache.hadoop.mapred.ClientServiceDelegate.invoke(ClientServiceDelegate.java:317) ~[thirdeye-action.jar:na] ... 22 common frames omitted -- --Anfernee