[
https://issues.apache.org/jira/browse/FALCON-760?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Sowmya Ramesh updated FALCON-760:
---------------------------------
Description:
Couple of issues:
1> In FalconPostProcessing the sequence is to first send the user message and
then falcon message. For eviction instance paths deleted is deserialized from
csv file. During processing of user message, after instance paths are read from
csv file its being deleted. Hence while processing the Falcon message evicted
instance paths are NIL and hence "IGNORE" is being sent in feedInstancePaths
which is incorrect
2> MetadataMappingService expects instance path to construct Lineage. For HCat
retention in FeedEvictor only partitionInfo is sent instead of path which
breaks Lineage
{noformat}
2014-09-26 23:55:04,347 INFO - [ActiveMQ Session Task:] ~ Adding evicted feed
instance: 2014-09-26T23:54Z (MetadataMappingService:299)
2014-09-26 23:55:04,347 INFO - [ActiveMQ Session Task:] ~ Recording lineage
for evicted instances [2009] (InstanceRelationshipGraphBuilder:209)
2014-09-26 23:55:04,347 INFO - [ActiveMQ Session Task:] ~ Computing feed
instance for : name= raaw-logs16-7eaac361, path=[2009], in cluster:
corp-7dce063c (InstanceRelationshipGraphBuilder:216)
2014-09-26 23:55:04,348 ERROR - [ActiveMQ Session Task:] ~ Error in listener
org.apache.falcon.metadata.MetadataMappingService
(WorkflowJobEndNotificationService:104)
org.apache.falcon.FalconException: java.net.URISyntaxException: Illegal
character in path at index 0: [2009]
at
org.apache.falcon.metadata.InstanceRelationshipGraphBuilder.getFeedInstanceName(InstanceRelationshipGraphBuilder.java:284)
at
org.apache.falcon.metadata.InstanceRelationshipGraphBuilder.addEvictedInstance(InstanceRelationshipGraphBuilder.java:218)
at
org.apache.falcon.metadata.MetadataMappingService.onFeedInstanceEvicted(MetadataMappingService.java:300)
at
org.apache.falcon.metadata.MetadataMappingService.onSuccess(MetadataMappingService.java:274)
at
org.apache.falcon.workflow.WorkflowJobEndNotificationService.notifySuccess(WorkflowJobEndNotificationService.java:101)
at
org.apache.falcon.messaging.JMSMessageConsumer.onSuccess(JMSMessageConsumer.java:134)
at
org.apache.falcon.messaging.JMSMessageConsumer.onMessage(JMSMessageConsumer.java:106)
at
org.apache.activemq.ActiveMQMessageConsumer.dispatch(ActiveMQMessageConsumer.java:1229)
at
org.apache.activemq.ActiveMQSessionExecutor.dispatch(ActiveMQSessionExecutor.java:134)
at
org.apache.activemq.ActiveMQSessionExecutor.iterate(ActiveMQSessionExecutor.java:205)
at
org.apache.activemq.thread.PooledTaskRunner.runTask(PooledTaskRunner.java:122)
at
org.apache.activemq.thread.PooledTaskRunner$1.run(PooledTaskRunner.java:43)
at
java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
at java.lang.Thread.run(Thread.java:662)
Caused by: java.net.URISyntaxException: Illegal character in path at index 0:
[2009]
at java.net.URI$Parser.fail(URI.java:2810)
at java.net.URI$Parser.checkChars(URI.java:2983)
at java.net.URI$Parser.parseHierarchical(URI.java:3067)
at java.net.URI$Parser.parse(URI.java:3025)
at java.net.URI.<init>(URI.java:577)
at
org.apache.falcon.entity.CatalogStorage.<init>(CatalogStorage.java:155)
at
org.apache.falcon.entity.FeedHelper.createStorage(FeedHelper.java:154)
at
org.apache.falcon.metadata.InstanceRelationshipGraphBuilder.getTableFeedInstanceName(InstanceRelationshipGraphBuilder.java:290)
at
org.apache.falcon.metadata.InstanceRelationshipGraphBuilder.getFeedInstanceName(InstanceRelationshipGraphBuilder.java:279)
{noformat}
3> getFeedInstanceName in InstanceRelationshipGraphBuilder expects fully
qualified instance path
{noformat}
2014-09-30 05:44:51,422 INFO - [ActiveMQ Session Task:] ~ Adding evicted feed
instance: 2014-09-30T05:44Z (MetadataMappingService:299)
2014-09-30 05:44:51,422 INFO - [ActiveMQ Session Task:] ~ Recording lineage
for evicted instances /tmp/falcon-regression/RetentionTest/testFolders/1979
(InstanceRelationshipGraphBuilder:209)
2014-09-30 05:44:51,422 INFO - [ActiveMQ Session Task:] ~ Computing feed
instance for : name= inPath-b977b74a,
path=/tmp/falcon-regression/RetentionTest/testFolders/1979, in cluster:
corp-21498f71 (InstanceRelationshipGraphBuilder:216)
2014-09-30 05:44:51,430 ERROR - [ActiveMQ Session Task:] ~ Error in listener
org.apache.falcon.metadata.MetadataMappingService
(WorkflowJobEndNotificationService:104)
java.lang.StringIndexOutOfBoundsException: String index out of range: 53
at java.lang.String.substring(String.java:1934)
at
org.apache.falcon.entity.v0.SchemaHelper.formatDateUTCToISO8601(SchemaHelper.java:64)
at
org.apache.falcon.metadata.InstanceRelationshipGraphBuilder.getFileSystemFeedInstanceName(InstanceRelationshipGraphBuilder.java:307)
at
org.apache.falcon.metadata.InstanceRelationshipGraphBuilder.getFeedInstanceName(InstanceRelationshipGraphBuilder.java:279)
at
org.apache.falcon.metadata.InstanceRelationshipGraphBuilder.addEvictedInstance(InstanceRelationshipGraphBuilder.java:218)
at
org.apache.falcon.metadata.MetadataMappingService.onFeedInstanceEvicted(MetadataMappingService.java:300)
at
org.apache.falcon.metadata.MetadataMappingService.onSuccess(MetadataMappingService.java:274)
at
org.apache.falcon.workflow.WorkflowJobEndNotificationService.notifySuccess(WorkflowJobEndNotificationService.java:101)
at
org.apache.falcon.messaging.JMSMessageConsumer.onSuccess(JMSMessageConsumer.java:133)
at
org.apache.falcon.messaging.JMSMessageConsumer.onMessage(JMSMessageConsumer.java:105)
at
org.apache.activemq.ActiveMQMessageConsumer.dispatch(ActiveMQMessageConsumer.java:1229)
at
org.apache.activemq.ActiveMQSessionExecutor.dispatch(ActiveMQSessionExecutor.java:134)
at
org.apache.activemq.ActiveMQSessionExecutor.iterate(ActiveMQSessionExecutor.java:205)
at
org.apache.activemq.thread.PooledTaskRunner.runTask(PooledTaskRunner.java:122)
at
org.apache.activemq.thread.PooledTaskRunner$1.run(PooledTaskRunner.java:43)
at
java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
at java.lang.Thread.run(Thread.java:662)
{noformat}
was:
Couple of issues:
1> In FalconPostProcessing the sequence is to first send the user message and
then falcon message. For eviction instance paths deleted is deserialized from
csv file. During processing of user message, after instance paths are read from
csv file its being deleted. Hence while processing the Falcon message evicted
instance paths are NIL and hence "IGNORE" is being sent in feedInstancePaths
which is incorrect
2> MetadataMappingService expects instance path to construct Lineage. For HCat
retention in FeedEvictor only partitionInfo is sent instead of path which
breaks Lineage
{noformat}
2014-09-26 23:55:04,347 INFO - [ActiveMQ Session Task:] ~ Adding evicted feed
instance: 2014-09-26T23:54Z (MetadataMappingService:299)
2014-09-26 23:55:04,347 INFO - [ActiveMQ Session Task:] ~ Recording lineage
for evicted instances [2009] (InstanceRelationshipGraphBuilder:209)
2014-09-26 23:55:04,347 INFO - [ActiveMQ Session Task:] ~ Computing feed
instance for : name= raaw-logs16-7eaac361, path=[2009], in cluster:
corp-7dce063c (InstanceRelationshipGraphBuilder:216)
2014-09-26 23:55:04,348 ERROR - [ActiveMQ Session Task:] ~ Error in listener
org.apache.falcon.metadata.MetadataMappingService
(WorkflowJobEndNotificationService:104)
org.apache.falcon.FalconException: java.net.URISyntaxException: Illegal
character in path at index 0: [2009]
at
org.apache.falcon.metadata.InstanceRelationshipGraphBuilder.getFeedInstanceName(InstanceRelationshipGraphBuilder.java:284)
at
org.apache.falcon.metadata.InstanceRelationshipGraphBuilder.addEvictedInstance(InstanceRelationshipGraphBuilder.java:218)
at
org.apache.falcon.metadata.MetadataMappingService.onFeedInstanceEvicted(MetadataMappingService.java:300)
at
org.apache.falcon.metadata.MetadataMappingService.onSuccess(MetadataMappingService.java:274)
at
org.apache.falcon.workflow.WorkflowJobEndNotificationService.notifySuccess(WorkflowJobEndNotificationService.java:101)
at
org.apache.falcon.messaging.JMSMessageConsumer.onSuccess(JMSMessageConsumer.java:134)
at
org.apache.falcon.messaging.JMSMessageConsumer.onMessage(JMSMessageConsumer.java:106)
at
org.apache.activemq.ActiveMQMessageConsumer.dispatch(ActiveMQMessageConsumer.java:1229)
at
org.apache.activemq.ActiveMQSessionExecutor.dispatch(ActiveMQSessionExecutor.java:134)
at
org.apache.activemq.ActiveMQSessionExecutor.iterate(ActiveMQSessionExecutor.java:205)
at
org.apache.activemq.thread.PooledTaskRunner.runTask(PooledTaskRunner.java:122)
at
org.apache.activemq.thread.PooledTaskRunner$1.run(PooledTaskRunner.java:43)
at
java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
at java.lang.Thread.run(Thread.java:662)
Caused by: java.net.URISyntaxException: Illegal character in path at index 0:
[2009]
at java.net.URI$Parser.fail(URI.java:2810)
at java.net.URI$Parser.checkChars(URI.java:2983)
at java.net.URI$Parser.parseHierarchical(URI.java:3067)
at java.net.URI$Parser.parse(URI.java:3025)
at java.net.URI.<init>(URI.java:577)
at
org.apache.falcon.entity.CatalogStorage.<init>(CatalogStorage.java:155)
at
org.apache.falcon.entity.FeedHelper.createStorage(FeedHelper.java:154)
at
org.apache.falcon.metadata.InstanceRelationshipGraphBuilder.getTableFeedInstanceName(InstanceRelationshipGraphBuilder.java:290)
at
org.apache.falcon.metadata.InstanceRelationshipGraphBuilder.getFeedInstanceName(InstanceRelationshipGraphBuilder.java:279)
{noformat}
> Messaging is broken for FALCON.ENTITY.TOPIC in case of Eviction
> ---------------------------------------------------------------
>
> Key: FALCON-760
> URL: https://issues.apache.org/jira/browse/FALCON-760
> Project: Falcon
> Issue Type: Bug
> Affects Versions: 0.5, 0.6
> Reporter: Sowmya Ramesh
> Assignee: Sowmya Ramesh
> Labels: Lineage, messaging
> Fix For: 0.6
>
>
> Couple of issues:
> 1> In FalconPostProcessing the sequence is to first send the user message and
> then falcon message. For eviction instance paths deleted is deserialized from
> csv file. During processing of user message, after instance paths are read
> from csv file its being deleted. Hence while processing the Falcon message
> evicted instance paths are NIL and hence "IGNORE" is being sent in
> feedInstancePaths which is incorrect
> 2> MetadataMappingService expects instance path to construct Lineage. For
> HCat retention in FeedEvictor only partitionInfo is sent instead of path
> which breaks Lineage
> {noformat}
> 2014-09-26 23:55:04,347 INFO - [ActiveMQ Session Task:] ~ Adding evicted
> feed instance: 2014-09-26T23:54Z (MetadataMappingService:299)
> 2014-09-26 23:55:04,347 INFO - [ActiveMQ Session Task:] ~ Recording lineage
> for evicted instances [2009] (InstanceRelationshipGraphBuilder:209)
> 2014-09-26 23:55:04,347 INFO - [ActiveMQ Session Task:] ~ Computing feed
> instance for : name= raaw-logs16-7eaac361, path=[2009], in cluster:
> corp-7dce063c (InstanceRelationshipGraphBuilder:216)
> 2014-09-26 23:55:04,348 ERROR - [ActiveMQ Session Task:] ~ Error in listener
> org.apache.falcon.metadata.MetadataMappingService
> (WorkflowJobEndNotificationService:104)
> org.apache.falcon.FalconException: java.net.URISyntaxException: Illegal
> character in path at index 0: [2009]
> at
> org.apache.falcon.metadata.InstanceRelationshipGraphBuilder.getFeedInstanceName(InstanceRelationshipGraphBuilder.java:284)
> at
> org.apache.falcon.metadata.InstanceRelationshipGraphBuilder.addEvictedInstance(InstanceRelationshipGraphBuilder.java:218)
> at
> org.apache.falcon.metadata.MetadataMappingService.onFeedInstanceEvicted(MetadataMappingService.java:300)
> at
> org.apache.falcon.metadata.MetadataMappingService.onSuccess(MetadataMappingService.java:274)
> at
> org.apache.falcon.workflow.WorkflowJobEndNotificationService.notifySuccess(WorkflowJobEndNotificationService.java:101)
> at
> org.apache.falcon.messaging.JMSMessageConsumer.onSuccess(JMSMessageConsumer.java:134)
> at
> org.apache.falcon.messaging.JMSMessageConsumer.onMessage(JMSMessageConsumer.java:106)
> at
> org.apache.activemq.ActiveMQMessageConsumer.dispatch(ActiveMQMessageConsumer.java:1229)
> at
> org.apache.activemq.ActiveMQSessionExecutor.dispatch(ActiveMQSessionExecutor.java:134)
> at
> org.apache.activemq.ActiveMQSessionExecutor.iterate(ActiveMQSessionExecutor.java:205)
> at
> org.apache.activemq.thread.PooledTaskRunner.runTask(PooledTaskRunner.java:122)
> at
> org.apache.activemq.thread.PooledTaskRunner$1.run(PooledTaskRunner.java:43)
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
> at java.lang.Thread.run(Thread.java:662)
> Caused by: java.net.URISyntaxException: Illegal character in path at index 0:
> [2009]
> at java.net.URI$Parser.fail(URI.java:2810)
> at java.net.URI$Parser.checkChars(URI.java:2983)
> at java.net.URI$Parser.parseHierarchical(URI.java:3067)
> at java.net.URI$Parser.parse(URI.java:3025)
> at java.net.URI.<init>(URI.java:577)
> at
> org.apache.falcon.entity.CatalogStorage.<init>(CatalogStorage.java:155)
> at
> org.apache.falcon.entity.FeedHelper.createStorage(FeedHelper.java:154)
> at
> org.apache.falcon.metadata.InstanceRelationshipGraphBuilder.getTableFeedInstanceName(InstanceRelationshipGraphBuilder.java:290)
> at
> org.apache.falcon.metadata.InstanceRelationshipGraphBuilder.getFeedInstanceName(InstanceRelationshipGraphBuilder.java:279)
> {noformat}
> 3> getFeedInstanceName in InstanceRelationshipGraphBuilder expects fully
> qualified instance path
> {noformat}
> 2014-09-30 05:44:51,422 INFO - [ActiveMQ Session Task:] ~ Adding evicted
> feed instance: 2014-09-30T05:44Z (MetadataMappingService:299)
> 2014-09-30 05:44:51,422 INFO - [ActiveMQ Session Task:] ~ Recording lineage
> for evicted instances /tmp/falcon-regression/RetentionTest/testFolders/1979
> (InstanceRelationshipGraphBuilder:209)
> 2014-09-30 05:44:51,422 INFO - [ActiveMQ Session Task:] ~ Computing feed
> instance for : name= inPath-b977b74a,
> path=/tmp/falcon-regression/RetentionTest/testFolders/1979, in cluster:
> corp-21498f71 (InstanceRelationshipGraphBuilder:216)
> 2014-09-30 05:44:51,430 ERROR - [ActiveMQ Session Task:] ~ Error in listener
> org.apache.falcon.metadata.MetadataMappingService
> (WorkflowJobEndNotificationService:104)
> java.lang.StringIndexOutOfBoundsException: String index out of range: 53
> at java.lang.String.substring(String.java:1934)
> at
> org.apache.falcon.entity.v0.SchemaHelper.formatDateUTCToISO8601(SchemaHelper.java:64)
> at
> org.apache.falcon.metadata.InstanceRelationshipGraphBuilder.getFileSystemFeedInstanceName(InstanceRelationshipGraphBuilder.java:307)
> at
> org.apache.falcon.metadata.InstanceRelationshipGraphBuilder.getFeedInstanceName(InstanceRelationshipGraphBuilder.java:279)
> at
> org.apache.falcon.metadata.InstanceRelationshipGraphBuilder.addEvictedInstance(InstanceRelationshipGraphBuilder.java:218)
> at
> org.apache.falcon.metadata.MetadataMappingService.onFeedInstanceEvicted(MetadataMappingService.java:300)
> at
> org.apache.falcon.metadata.MetadataMappingService.onSuccess(MetadataMappingService.java:274)
> at
> org.apache.falcon.workflow.WorkflowJobEndNotificationService.notifySuccess(WorkflowJobEndNotificationService.java:101)
> at
> org.apache.falcon.messaging.JMSMessageConsumer.onSuccess(JMSMessageConsumer.java:133)
> at
> org.apache.falcon.messaging.JMSMessageConsumer.onMessage(JMSMessageConsumer.java:105)
> at
> org.apache.activemq.ActiveMQMessageConsumer.dispatch(ActiveMQMessageConsumer.java:1229)
> at
> org.apache.activemq.ActiveMQSessionExecutor.dispatch(ActiveMQSessionExecutor.java:134)
> at
> org.apache.activemq.ActiveMQSessionExecutor.iterate(ActiveMQSessionExecutor.java:205)
> at
> org.apache.activemq.thread.PooledTaskRunner.runTask(PooledTaskRunner.java:122)
> at
> org.apache.activemq.thread.PooledTaskRunner$1.run(PooledTaskRunner.java:43)
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
> at java.lang.Thread.run(Thread.java:662)
> {noformat}
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
