[ https://issues.apache.org/jira/browse/DRILL-7164?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]
Hanumath Rao Maduri updated DRILL-7164: --------------------------------------- Description: On my private build I am hitting kafka storage tests issue intermittently. Here is the issue which I came across. {code} at java.lang.Thread.run(Thread.java:745) ~[na:1.8.0_91] 15:01:39.852 [main] ERROR org.apache.drill.TestReporter - Test Failed (d: -292 B(75.4 KiB), h: -391.1 MiB(240.7 MiB), nh: 824.5 KiB(129.0 MiB)): testPushdownOffsetOneRecordReturnedWithBoundaryConditions(org.apache.drill.exec.store.kafka.KafkaFilterPushdownTest) java.lang.AssertionError: Unable to find expected string "kafkaScanSpec" : { "topicName" : "drill-pushdown-topic" }, .* .* "cost" in plan: { "head" : { "version" : 1, "generator" : { "type" : "ExplainHandler", "info" : "" }, "type" : "APACHE_DRILL_PHYSICAL", "options" : [ { "kind" : "STRING", "accessibleScopes" : "ALL", "name" : "store.kafka.record.reader", "string_val" : "org.apache.drill.exec.store.kafka.decoders.JsonMessageReader", "scope" : "SESSION" }, { "kind" : "BOOLEAN", "accessibleScopes" : "ALL", "name" : "exec.errors.verbose", "bool_val" : true, "scope" : "SESSION" }, { "kind" : "LONG", "accessibleScopes" : "ALL", "name" : "store.kafka.poll.timeout", "num_val" : 5000, "scope" : "SESSION" }, { "kind" : "LONG", "accessibleScopes" : "ALL", "name" : "planner.width.max_per_node", "num_val" : 2, "scope" : "SESSION" } ], "queue" : 0, "hasResourcePlan" : false, "resultMode" : "EXEC" }, "graph" : [ { "pop" : "kafka-scan", "@id" : 6, "userName" : "", "kafkaStoragePluginConfig" : { "type" : "kafka", "kafkaConsumerProps" : { "bootstrap.servers" : "127.0.0.1:56524", "group.id" : "drill-test-consumer" }, "enabled" : true }, "columns" : [ "`**`", "`kafkaMsgOffset`" ], "kafkaScanSpec" : { "topicName" : "drill-pushdown-topic" }, "initialAllocation" : 1000000, "maxAllocation" : 10000000000, "cost" : { "memoryCost" : 1.6777216E7, "outputRowCount" : 5.0 } }, { "pop" : "project", "@id" : 5, "exprs" : [ { "ref" : "`T23¦¦**`", "expr" : "`**`" }, { "ref" : "`kafkaMsgOffset`", "expr" : "`kafkaMsgOffset`" } ], "child" : 6, "outputProj" : false, "initialAllocation" : 1000000, "maxAllocation" : 10000000000, "cost" : { "memoryCost" : 1.6777216E7, "outputRowCount" : 5.0 } }, { "pop" : "filter", "@id" : 4, "child" : 5, "expr" : "equal(`kafkaMsgOffset`, 9) ", "initialAllocation" : 1000000, "maxAllocation" : 10000000000, "cost" : { "memoryCost" : 1.6777216E7, "outputRowCount" : 0.75 } }, { "pop" : "selection-vector-remover", "@id" : 3, "child" : 4, "initialAllocation" : 1000000, "maxAllocation" : 10000000000, "cost" : { "memoryCost" : 1.6777216E7, "outputRowCount" : 1.0 } }, { "pop" : "project", "@id" : 2, "exprs" : [ { "ref" : "`T23¦¦**`", "expr" : "`T23¦¦**`" } ], "child" : 3, "outputProj" : false, "initialAllocation" : 1000000, "maxAllocation" : 10000000000, "cost" : { "memoryCost" : 1.6777216E7, "outputRowCount" : 1.0 } }, { "pop" : "project", "@id" : 1, "exprs" : [ { "ref" : "`**`", "expr" : "`T23¦¦**`" } ], "child" : 2, "outputProj" : true, "initialAllocation" : 1000000, "maxAllocation" : 10000000000, "cost" : { "memoryCost" : 1.6777216E7, "outputRowCount" : 1.0 } }, { "pop" : "screen", "@id" : 0, "child" : 1, "initialAllocation" : 1000000, "maxAllocation" : 10000000000, "cost" : { "memoryCost" : 1.6777216E7, "outputRowCount" : 1.0 } } ] }! {code} In the earlier checkin (d22e68b83d1d0cc0539d79ae0cb3aa70ae3242ad ) there is a change in the way cost is being represented. It also has the changed the test which I think is not right. The pattern to compare in the plan should be made smart to fix this issue generically. was: On my private build I am hitting kafka storage tests issue intermittently. Here is the issue which I came across. {code} at java.lang.Thread.run(Thread.java:745) ~[na:1.8.0_91] 15:01:39.852 [main] ERROR org.apache.drill.TestReporter - Test Failed (d: -292 B(75.4 KiB), h: -391.1 MiB(240.7 MiB), nh: 824.5 KiB(129.0 MiB)): testPushdownOffsetOneRecordReturnedWithBoundaryConditions(org.apache.drill.exec.store.kafka.KafkaFilterPushdownTest) java.lang.AssertionError: Unable to find expected string "kafkaScanSpec" : { "topicName" : "drill-pushdown-topic" }, .* .* "cost" in plan: { "head" : { "version" : 1, "generator" : { "type" : "ExplainHandler", "info" : "" }, "type" : "APACHE_DRILL_PHYSICAL", "options" : [ { "kind" : "STRING", "accessibleScopes" : "ALL", "name" : "store.kafka.record.reader", "string_val" : "org.apache.drill.exec.store.kafka.decoders.JsonMessageReader", "scope" : "SESSION" }, { "kind" : "BOOLEAN", "accessibleScopes" : "ALL", "name" : "exec.errors.verbose", "bool_val" : true, "scope" : "SESSION" }, { "kind" : "LONG", "accessibleScopes" : "ALL", "name" : "store.kafka.poll.timeout", "num_val" : 5000, "scope" : "SESSION" }, { "kind" : "LONG", "accessibleScopes" : "ALL", "name" : "planner.width.max_per_node", "num_val" : 2, "scope" : "SESSION" } ], "queue" : 0, "hasResourcePlan" : false, "resultMode" : "EXEC" }, "graph" : [ { "pop" : "kafka-scan", "@id" : 6, "userName" : "", "kafkaStoragePluginConfig" : { "type" : "kafka", "kafkaConsumerProps" : { "bootstrap.servers" : "127.0.0.1:56524", "group.id" : "drill-test-consumer" }, "enabled" : true }, "columns" : [ "`**`", "`kafkaMsgOffset`" ], "kafkaScanSpec" : { "topicName" : "drill-pushdown-topic" }, "initialAllocation" : 1000000, "maxAllocation" : 10000000000, "cost" : { "memoryCost" : 1.6777216E7, "outputRowCount" : 5.0 } }, { "pop" : "project", "@id" : 5, "exprs" : [ { "ref" : "`T23¦¦**`", "expr" : "`**`" }, { "ref" : "`kafkaMsgOffset`", "expr" : "`kafkaMsgOffset`" } ], "child" : 6, "outputProj" : false, "initialAllocation" : 1000000, "maxAllocation" : 10000000000, "cost" : { "memoryCost" : 1.6777216E7, "outputRowCount" : 5.0 } }, { "pop" : "filter", "@id" : 4, "child" : 5, "expr" : "equal(`kafkaMsgOffset`, 9) ", "initialAllocation" : 1000000, "maxAllocation" : 10000000000, "cost" : { "memoryCost" : 1.6777216E7, "outputRowCount" : 0.75 } }, { "pop" : "selection-vector-remover", "@id" : 3, "child" : 4, "initialAllocation" : 1000000, "maxAllocation" : 10000000000, "cost" : { "memoryCost" : 1.6777216E7, "outputRowCount" : 1.0 } }, { "pop" : "project", "@id" : 2, "exprs" : [ { "ref" : "`T23¦¦**`", "expr" : "`T23¦¦**`" } ], "child" : 3, "outputProj" : false, "initialAllocation" : 1000000, "maxAllocation" : 10000000000, "cost" : { "memoryCost" : 1.6777216E7, "outputRowCount" : 1.0 } }, { "pop" : "project", "@id" : 1, "exprs" : [ { "ref" : "`**`", "expr" : "`T23¦¦**`" } ], "child" : 2, "outputProj" : true, "initialAllocation" : 1000000, "maxAllocation" : 10000000000, "cost" : { "memoryCost" : 1.6777216E7, "outputRowCount" : 1.0 } }, { "pop" : "screen", "@id" : 0, "child" : 1, "initialAllocation" : 1000000, "maxAllocation" : 10000000000, "cost" : { "memoryCost" : 1.6777216E7, "outputRowCount" : 1.0 } } ] }! {code} In the earlier checkin (d22e68b83d1d0cc0539d79ae0cb3aa70ae3242ad ) there is a change in the way cost is being represented. This has the changed the test which I think is not right. The pattern to compare in the plan should be made smart to fix this issue generically. > KafkaFilterPushdownTest is sometimes failing to pattern match correctly. > ------------------------------------------------------------------------ > > Key: DRILL-7164 > URL: https://issues.apache.org/jira/browse/DRILL-7164 > Project: Apache Drill > Issue Type: Bug > Components: Storage - Kafka > Affects Versions: 1.16.0 > Reporter: Hanumath Rao Maduri > Assignee: Abhishek Ravi > Priority: Major > Fix For: 1.17.0 > > > On my private build I am hitting kafka storage tests issue intermittently. > Here is the issue which I came across. > {code} > at java.lang.Thread.run(Thread.java:745) ~[na:1.8.0_91] > 15:01:39.852 [main] ERROR org.apache.drill.TestReporter - Test Failed (d: > -292 B(75.4 KiB), h: -391.1 MiB(240.7 MiB), nh: 824.5 KiB(129.0 MiB)): > testPushdownOffsetOneRecordReturnedWithBoundaryConditions(org.apache.drill.exec.store.kafka.KafkaFilterPushdownTest) > java.lang.AssertionError: Unable to find expected string "kafkaScanSpec" > : { > "topicName" : "drill-pushdown-topic" > }, > .* > .* > "cost" in plan: { > "head" : { > "version" : 1, > "generator" : { > "type" : "ExplainHandler", > "info" : "" > }, > "type" : "APACHE_DRILL_PHYSICAL", > "options" : [ { > "kind" : "STRING", > "accessibleScopes" : "ALL", > "name" : "store.kafka.record.reader", > "string_val" : > "org.apache.drill.exec.store.kafka.decoders.JsonMessageReader", > "scope" : "SESSION" > }, { > "kind" : "BOOLEAN", > "accessibleScopes" : "ALL", > "name" : "exec.errors.verbose", > "bool_val" : true, > "scope" : "SESSION" > }, { > "kind" : "LONG", > "accessibleScopes" : "ALL", > "name" : "store.kafka.poll.timeout", > "num_val" : 5000, > "scope" : "SESSION" > }, { > "kind" : "LONG", > "accessibleScopes" : "ALL", > "name" : "planner.width.max_per_node", > "num_val" : 2, > "scope" : "SESSION" > } ], > "queue" : 0, > "hasResourcePlan" : false, > "resultMode" : "EXEC" > }, > "graph" : [ { > "pop" : "kafka-scan", > "@id" : 6, > "userName" : "", > "kafkaStoragePluginConfig" : { > "type" : "kafka", > "kafkaConsumerProps" : { > "bootstrap.servers" : "127.0.0.1:56524", > "group.id" : "drill-test-consumer" > }, > "enabled" : true > }, > "columns" : [ "`**`", "`kafkaMsgOffset`" ], > "kafkaScanSpec" : { > "topicName" : "drill-pushdown-topic" > }, > "initialAllocation" : 1000000, > "maxAllocation" : 10000000000, > "cost" : { > "memoryCost" : 1.6777216E7, > "outputRowCount" : 5.0 > } > }, { > "pop" : "project", > "@id" : 5, > "exprs" : [ { > "ref" : "`T23¦¦**`", > "expr" : "`**`" > }, { > "ref" : "`kafkaMsgOffset`", > "expr" : "`kafkaMsgOffset`" > } ], > "child" : 6, > "outputProj" : false, > "initialAllocation" : 1000000, > "maxAllocation" : 10000000000, > "cost" : { > "memoryCost" : 1.6777216E7, > "outputRowCount" : 5.0 > } > }, { > "pop" : "filter", > "@id" : 4, > "child" : 5, > "expr" : "equal(`kafkaMsgOffset`, 9) ", > "initialAllocation" : 1000000, > "maxAllocation" : 10000000000, > "cost" : { > "memoryCost" : 1.6777216E7, > "outputRowCount" : 0.75 > } > }, { > "pop" : "selection-vector-remover", > "@id" : 3, > "child" : 4, > "initialAllocation" : 1000000, > "maxAllocation" : 10000000000, > "cost" : { > "memoryCost" : 1.6777216E7, > "outputRowCount" : 1.0 > } > }, { > "pop" : "project", > "@id" : 2, > "exprs" : [ { > "ref" : "`T23¦¦**`", > "expr" : "`T23¦¦**`" > } ], > "child" : 3, > "outputProj" : false, > "initialAllocation" : 1000000, > "maxAllocation" : 10000000000, > "cost" : { > "memoryCost" : 1.6777216E7, > "outputRowCount" : 1.0 > } > }, { > "pop" : "project", > "@id" : 1, > "exprs" : [ { > "ref" : "`**`", > "expr" : "`T23¦¦**`" > } ], > "child" : 2, > "outputProj" : true, > "initialAllocation" : 1000000, > "maxAllocation" : 10000000000, > "cost" : { > "memoryCost" : 1.6777216E7, > "outputRowCount" : 1.0 > } > }, { > "pop" : "screen", > "@id" : 0, > "child" : 1, > "initialAllocation" : 1000000, > "maxAllocation" : 10000000000, > "cost" : { > "memoryCost" : 1.6777216E7, > "outputRowCount" : 1.0 > } > } ] > }! > {code} > In the earlier checkin (d22e68b83d1d0cc0539d79ae0cb3aa70ae3242ad ) there is a > change in the way cost is being represented. It also has the changed the test > which I think is not right. The pattern to compare in the plan should be made > smart to fix this issue generically. -- This message was sent by Atlassian JIRA (v7.6.3#76005)