michelle chiang created OOZIE-1211:
--------------------------------------
Summary: oozie does not support duplicated dataset
Key: OOZIE-1211
URL: https://issues.apache.org/jira/browse/OOZIE-1211
Project: Oozie
Issue Type: Bug
Components: coordinator
Affects Versions: trunk
Reporter: michelle chiang
Priority: Minor
in the example below, dataset B is defined in both coordinator.xml and include
dataset file. oozie fails to submit job with error:
Error: E0803 : E0803: IO error, null
coordinator.xml:
<datasets>
<include>${include_ds_files}</include>
<dataset name="B" frequency="${coord:minutes(5)}"
initial-instance="${ds_start}" timezone="${timezone}">
<uri-template>hcat://${hcat_server}/${hcat_out_db}/${hcat_out_table}/datestamp=${YEAR}${MONTH}${DAY}${HOUR}${MINUTE};region=${hcat_region}</uri-template>
</dataset>
</datasets>
include_ds_files:
<datasets>
<include>${include_ds_files}</include>
<dataset name="B" frequency="${coord:minutes(5)}"
initial-instance="${ds_start}" timezone="${timezone}">
<uri-template>hcat://${hcat_server}/${hcat_out_db}/${hcat_out_table}/datestamp=${YEAR}${MONTH}${DAY}${HOUR}${MINUTE};region=${hcat_region}</uri-template>
</dataset>
</datasets>
oozie log:
2013-02-07 01:02:01,308 WARN CoordSubmitXCommand:544 - USER[mchiang]
GROUP[users] TOKEN[-] APP[-] JOB[-] ACTION[-] ERROR:
java.util.ConcurrentModificationException
at
org.jdom.ContentList$FilterListIterator.checkConcurrentModification(ContentList.java:940)
at
org.jdom.ContentList$FilterListIterator.nextIndex(ContentList.java:829)
at org.jdom.ContentList$FilterListIterator.hasNext(ContentList.java:785)
at
org.apache.oozie.command.coord.CoordSubmitXCommand.removeDataSet(CoordSubmitXCommand.java:1017)
at
org.apache.oozie.command.coord.CoordSubmitXCommand.includeDataSets(CoordSubmitXCommand.java:952)
at
org.apache.oozie.command.coord.CoordSubmitXCommand.basicResolveAndIncludeDS(CoordSubmitXCommand.java:558)
at
org.apache.oozie.command.coord.CoordSubmitXCommand.submit(CoordSubmitXCommand.java:225)
at
org.apache.oozie.command.SubmitTransitionXCommand.execute(SubmitTransitionXCommand.java:81)
at
org.apache.oozie.command.SubmitTransitionXCommand.execute(SubmitTransitionXCommand.java:29)
at org.apache.oozie.command.XCommand.call(XCommand.java:277)
at
org.apache.oozie.CoordinatorEngine.submitJob(CoordinatorEngine.java:375)
at
org.apache.oozie.servlet.V1JobsServlet.submitCoordinatorJob(V1JobsServlet.java:219)
at
org.apache.oozie.servlet.V1JobsServlet.submitJob(V1JobsServlet.java:83)
at
org.apache.oozie.servlet.BaseJobsServlet.doPost(BaseJobsServlet.java:100)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:637)
at
org.apache.oozie.servlet.JsonRestServlet.service(JsonRestServlet.java:286)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:717)
at
org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:290)
at
org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:206)
at
com.yahoo.oozie.security.authentication.filter.YAuthFilter$2.doFilter(YAuthFilter.java:135)
at
org.apache.hadoop.security.authentication.server.AuthenticationFilter.doFilter(AuthenticationFilter.java:372)
at
com.yahoo.oozie.security.authentication.filter.YAuthFilter.doFilter(YAuthFilter.java:139)
at
org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:235)
at
org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:206)
at
yjava.servlet.FilterChainInvoker$ServletFilterChainInvoker.invoke(FilterChainInvoker.java:49)
at yjava.servlet.filter.YHdrsFilter.doFilter(YHdrsFilter.java:68)
at yjava.servlet.filter.YHdrsFilter.doFilter(YHdrsFilter.java:52)
at
org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:235)
at
org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:206)
at
yjava.servlet.FilterChainInvoker$ServletFilterChainInvoker.invoke(FilterChainInvoker.java:49)
at yjava.cookie.CookieDataFilter.doFilter(CookieDataFilter.java:447)
at yjava.cookie.CookieDataFilter.doFilter(CookieDataFilter.java:219)
at
org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:235)
at
org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:206)
at
yjava.servlet.FilterChainInvoker$ServletFilterChainInvoker.invoke(FilterChainInvoker.java:49)
at
yjava.servlet.filter.DoNotTrackFilter.doFilter(DoNotTrackFilter.java:104)
at
yjava.servlet.filter.DoNotTrackFilter.doFilter(DoNotTrackFilter.java:50)
at
org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:235)
at
org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:206)
at
yjava.servlet.FilterChainInvoker$ServletFilterChainInvoker.invoke(FilterChainInvoker.java:49)
at yjava.remote.ip.RemoteIPFilter.doFilter(RemoteIPFilter.java:104)
at yjava.remote.ip.RemoteIPFilter.doFilter(RemoteIPFilter.java:65)
at
org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:235)
at
org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:206)
at
yjava.servlet.FilterChainInvoker$ServletFilterChainInvoker.invoke(FilterChainInvoker.java:49)
at
yjava.security.yiv.servlet.InputValidationFilter.doFilter(InputValidationFilter.java:219)
at
yjava.security.yiv.servlet.InputValidationFilter.doFilter(InputValidationFilter.java:143)
at
org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:235)
at
org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:206)
at
org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:233)
at
org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:191)
at
org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:127)
at
org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:102)
at
org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:109)
at
yjava.tomcat.valves.YahooConnectionValve.invoke(YahooConnectionValve.java:191)
at
org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:293)
at
org.apache.coyote.http11.Http11Processor.process(Http11Processor.java:859)
at
org.apache.coyote.http11.Http11Protocol$Http11ConnectionHandler.process(Http11Protocol.java:602)
at
org.apache.tomcat.util.net.JIoEndpoint$Worker.run(JIoEndpoint.java:489)
at java.lang.Thread.run(Thread.java:662)
2013-02-07 01:02:01,309 ERROR CoordSubmitXCommand:538 - USER[mchiang]
GROUP[users] TOKEN[-] APP[-] JOB[-] ACTION[-] XException,
org.apache.oozie.command.CommandException: E0803: IO error, null
at
org.apache.oozie.command.coord.CoordSubmitXCommand.submit(CoordSubmitXCommand.java:297)
at
org.apache.oozie.command.SubmitTransitionXCommand.execute(SubmitTransitionXCommand.java:81)
at
org.apache.oozie.command.SubmitTransitionXCommand.execute(SubmitTransitionXCommand.java:29)
at org.apache.oozie.command.XCommand.call(XCommand.java:277)
at
org.apache.oozie.CoordinatorEngine.submitJob(CoordinatorEngine.java:375)
at
org.apache.oozie.servlet.V1JobsServlet.submitCoordinatorJob(V1JobsServlet.java:219)
at
org.apache.oozie.servlet.V1JobsServlet.submitJob(V1JobsServlet.java:83)
at
org.apache.oozie.servlet.BaseJobsServlet.doPost(BaseJobsServlet.java:100)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:637)
at
org.apache.oozie.servlet.JsonRestServlet.service(JsonRestServlet.java:286)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:717)
at
org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:290)
at
org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:206)
at
com.yahoo.oozie.security.authentication.filter.YAuthFilter$2.doFilter(YAuthFilter.java:135)
at
org.apache.hadoop.security.authentication.server.AuthenticationFilter.doFilter(AuthenticationFilter.java:372)
at
com.yahoo.oozie.security.authentication.filter.YAuthFilter.doFilter(YAuthFilter.java:139)
at
org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:235)
at
org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:206)
at
yjava.servlet.FilterChainInvoker$ServletFilterChainInvoker.invoke(FilterChainInvoker.java:49)
at yjava.servlet.filter.YHdrsFilter.doFilter(YHdrsFilter.java:68)
at yjava.servlet.filter.YHdrsFilter.doFilter(YHdrsFilter.java:52)
at
org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:235)
at
org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:206)
at
yjava.servlet.FilterChainInvoker$ServletFilterChainInvoker.invoke(FilterChainInvoker.java:49)
at yjava.cookie.CookieDataFilter.doFilter(CookieDataFilter.java:447)
at yjava.cookie.CookieDataFilter.doFilter(CookieDataFilter.java:219)
at
org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:235)
at
org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:206)
at
yjava.servlet.FilterChainInvoker$ServletFilterChainInvoker.invoke(FilterChainInvoker.java:49)
at
yjava.servlet.filter.DoNotTrackFilter.doFilter(DoNotTrackFilter.java:104)
at
yjava.servlet.filter.DoNotTrackFilter.doFilter(DoNotTrackFilter.java:50)
at
org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:235)
at
org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:206)
at
yjava.servlet.FilterChainInvoker$ServletFilterChainInvoker.invoke(FilterChainInvoker.java:49)
at yjava.remote.ip.RemoteIPFilter.doFilter(RemoteIPFilter.java:104)
at yjava.remote.ip.RemoteIPFilter.doFilter(RemoteIPFilter.java:65)
at
org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:235)
at
org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:206)
at
yjava.servlet.FilterChainInvoker$ServletFilterChainInvoker.invoke(FilterChainInvoker.java:49)
at
yjava.security.yiv.servlet.InputValidationFilter.doFilter(InputValidationFilter.java:219)
at
yjava.security.yiv.servlet.InputValidationFilter.doFilter(InputValidationFilter.java:143)
at
org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:235)
at
org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:206)
at
org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:233)
at
org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:191)
at
org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:127)
at
org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:102)
at
org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:109)
at
yjava.tomcat.valves.YahooConnectionValve.invoke(YahooConnectionValve.java:191)
at
org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:293)
at
org.apache.coyote.http11.Http11Processor.process(Http11Processor.java:859)
at
org.apache.coyote.http11.Http11Protocol$Http11ConnectionHandler.process(Http11Protocol.java:602)
at
org.apache.tomcat.util.net.JIoEndpoint$Worker.run(JIoEndpoint.java:489)
at java.lang.Thread.run(Thread.java:662)
Caused by: java.util.ConcurrentModificationException
at
org.jdom.ContentList$FilterListIterator.checkConcurrentModification(ContentList.java:940)
at
org.jdom.ContentList$FilterListIterator.nextIndex(ContentList.java:829)
at org.jdom.ContentList$FilterListIterator.hasNext(ContentList.java:785)
at
org.apache.oozie.command.coord.CoordSubmitXCommand.removeDataSet(CoordSubmitXCommand.java:1017)
at
org.apache.oozie.command.coord.CoordSubmitXCommand.includeDataSets(CoordSubmitXCommand.java:952)
at
org.apache.oozie.command.coord.CoordSubmitXCommand.basicResolveAndIncludeDS(CoordSubmitXCommand.java:558)
at
org.apache.oozie.command.coord.CoordSubmitXCommand.submit(CoordSubmitXCommand.java:225)
... 53 more
2013-02-07 01:02:01,310 WARN V1JobsServlet:544 - USER[mchiang] GROUP[users]
TOKEN[-] APP[-] JOB[-] ACTION[-] URL[POST
http://gsbl90547.blue.ygrid.yahoo.com:4080/oozie/v1/jobs?action=start]
error[E0803], E0803: IO error, null
org.apache.oozie.servlet.XServletException: E0803: IO error, null
at
org.apache.oozie.servlet.V1JobsServlet.submitCoordinatorJob(V1JobsServlet.java:224)
at
org.apache.oozie.servlet.V1JobsServlet.submitJob(V1JobsServlet.java:83)
at
org.apache.oozie.servlet.BaseJobsServlet.doPost(BaseJobsServlet.java:100)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:637)
at
org.apache.oozie.servlet.JsonRestServlet.service(JsonRestServlet.java:286)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:717)
at
org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:290)
at
org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:206)
at
com.yahoo.oozie.security.authentication.filter.YAuthFilter$2.doFilter(YAuthFilter.java:135)
at
org.apache.hadoop.security.authentication.server.AuthenticationFilter.doFilter(AuthenticationFilter.java:372)
at
com.yahoo.oozie.security.authentication.filter.YAuthFilter.doFilter(YAuthFilter.java:139)
at
org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:235)
at
org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:206)
at
yjava.servlet.FilterChainInvoker$ServletFilterChainInvoker.invoke(FilterChainInvoker.java:49)
at yjava.servlet.filter.YHdrsFilter.doFilter(YHdrsFilter.java:68)
at yjava.servlet.filter.YHdrsFilter.doFilter(YHdrsFilter.java:52)
at
org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:235)
at
org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:206)
at
yjava.servlet.FilterChainInvoker$ServletFilterChainInvoker.invoke(FilterChainInvoker.java:49)
at yjava.cookie.CookieDataFilter.doFilter(CookieDataFilter.java:447)
at yjava.cookie.CookieDataFilter.doFilter(CookieDataFilter.java:219)
at
org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:235)
at
org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:206)
at
yjava.servlet.FilterChainInvoker$ServletFilterChainInvoker.invoke(FilterChainInvoker.java:49)
at
yjava.servlet.filter.DoNotTrackFilter.doFilter(DoNotTrackFilter.java:104)
at
yjava.servlet.filter.DoNotTrackFilter.doFilter(DoNotTrackFilter.java:50)
at
org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:235)
at
org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:206)
at
yjava.servlet.FilterChainInvoker$ServletFilterChainInvoker.invoke(FilterChainInvoker.java:49)
at yjava.remote.ip.RemoteIPFilter.doFilter(RemoteIPFilter.java:104)
at yjava.remote.ip.RemoteIPFilter.doFilter(RemoteIPFilter.java:65)
at
org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:235)
at
org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:206)
at
yjava.servlet.FilterChainInvoker$ServletFilterChainInvoker.invoke(FilterChainInvoker.java:49)
at
yjava.security.yiv.servlet.InputValidationFilter.doFilter(InputValidationFilter.java:219)
at
yjava.security.yiv.servlet.InputValidationFilter.doFilter(InputValidationFilter.java:143)
at
org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:235)
at
org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:206)
at
org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:233)
at
org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:191)
at
org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:127)
at
org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:102)
at
org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:109)
at
yjava.tomcat.valves.YahooConnectionValve.invoke(YahooConnectionValve.java:191)
at
org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:293)
at
org.apache.coyote.http11.Http11Processor.process(Http11Processor.java:859)
at
org.apache.coyote.http11.Http11Protocol$Http11ConnectionHandler.process(Http11Protocol.java:602)
at
org.apache.tomcat.util.net.JIoEndpoint$Worker.run(JIoEndpoint.java:489)
at java.lang.Thread.run(Thread.java:662)
Caused by: org.apache.oozie.CoordinatorEngineException: E0803: IO error, null
at
org.apache.oozie.CoordinatorEngine.submitJob(CoordinatorEngine.java:378)
at
org.apache.oozie.servlet.V1JobsServlet.submitCoordinatorJob(V1JobsServlet.java:219)
... 48 more
Caused by: org.apache.oozie.command.CommandException: E0803: IO error, null
at
org.apache.oozie.command.coord.CoordSubmitXCommand.submit(CoordSubmitXCommand.java:297)
at
org.apache.oozie.command.SubmitTransitionXCommand.execute(SubmitTransitionXCommand.java:81)
at
org.apache.oozie.command.SubmitTransitionXCommand.execute(SubmitTransitionXCommand.java:29)
at org.apache.oozie.command.XCommand.call(XCommand.java:277)
at
org.apache.oozie.CoordinatorEngine.submitJob(CoordinatorEngine.java:375)
... 49 more
Caused by: java.util.ConcurrentModificationException
at
org.jdom.ContentList$FilterListIterator.checkConcurrentModification(ContentList.java:940)
at
org.jdom.ContentList$FilterListIterator.nextIndex(ContentList.java:829)
at org.jdom.ContentList$FilterListIterator.hasNext(ContentList.java:785)
at
org.apache.oozie.command.coord.CoordSubmitXCommand.removeDataSet(CoordSubmitXCommand.java:1017)
at
org.apache.oozie.command.coord.CoordSubmitXCommand.includeDataSets(CoordSubmitXCommand.java:952)
at
org.apache.oozie.command.coord.CoordSubmitXCommand.basicResolveAndIncludeDS(CoordSubmitXCommand.java:558)
at
org.apache.oozie.command.coord.CoordSubmitXCommand.submit(CoordSubmitXCommand.java:225)
... 53 more
--
This message is automatically generated by JIRA.
If you think it was sent incorrectly, please contact your JIRA administrators
For more information on JIRA, see: http://www.atlassian.com/software/jira
