[ https://issues.apache.org/jira/browse/DRILL-5166?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]
Anton Gozhiy closed DRILL-5166. ------------------------------- Test data: Two files were generated, first (2 records): key_header, value_header key_1,value_1 second (500001 records): key_header, value_header key_1,value_1 ....... key_500000,value_500000 Query used: select * from table(dfs.tmp`*.csv`(type => 'text', fieldDelimiter => ',')) limit 10 Verified with Drill 1.13.0-SNAPSHOT (commit id: 7506cfbb5c8522d371c12dbdc2268d48a9449a48): 1. The query with the test data was completed successfully. 2 major fragments were created during the query execution. 2. Also verified querying files without the table function 3. Tried to execute previously generated physical plan, successfully. 4. Tried with different table function parameters (like "extractHeader => true" that is not set in the plugin config and is not default). 5. Tried with files with different field delimiters (like '\t') and extensions. Also verified with a file big enough (about 500 Mb) for creating 2 major fragments. > Select with options returns NPE > ------------------------------- > > Key: DRILL-5166 > URL: https://issues.apache.org/jira/browse/DRILL-5166 > Project: Apache Drill > Issue Type: Bug > Affects Versions: 1.9.0 > Reporter: Arina Ielchiieva > Assignee: Arina Ielchiieva > Priority: Major > Fix For: 1.12.0 > > > When querying two csv files: > First file (2 records): > {noformat} > key_header, value_header > key_1,value_1 > {noformat} > Second file (500000 records): > {noformat} > key_header, value_header > key_1,value_1 > ....... > key_499999,value_499999 > {noformat} > Select with options returns NPE: > {noformat} > select * from table(dfs.root.`/home/arina/files/ver/*.csv`(type => > 'text',extractHeader => true, fieldDelimiter => ',')) limit 10; > {noformat} > Querying without options works file: > {noformat} > select * from dfs.root.`/home/arina/files/ver/*.csv` limit 10; > {noformat} > Error: > {noformat} > Caused by: org.apache.drill.common.exceptions.UserRemoteException: SYSTEM > ERROR: NullPointerException > Fragment 1:0 > [Error Id: b789f5f8-f090-4097-b7ff-9f4efd3d01e8 on localhost:31013] > (com.fasterxml.jackson.databind.JsonMappingException) Instantiation of > [simple type, class org.apache.drill.exec.store.dfs.easy.EasySubScan] value > failed (java.lang.NullPointerException): null > at [Source: { > "pop" : "single-sender", > "@id" : 0, > "receiver-major-fragment" : 0, > "receiver-minor-fragment" : 0, > "child" : { > "pop" : "selection-vector-remover", > "@id" : 1, > "child" : { > "pop" : "limit", > "@id" : 2, > "child" : { > "pop" : "fs-sub-scan", > "@id" : 3, > "userName" : "arina", > "files" : [ { > "start" : 0, > "length" : 11777804, > "path" : "file:/home/arina/files/ver/key_value_500000.csv" > } ], > "storage" : { > "type" : "file", > "enabled" : true, > "connection" : "file:///", > "config" : null, > "workspaces" : { > "root" : { > "location" : "/", > "writable" : false, > "defaultInputFormat" : null > }, > "tmp" : { > "location" : "/tmp", > "writable" : false, > "defaultInputFormat" : null > } > }, > "formats" : { > "psv" : { > "type" : "text", > "extensions" : [ "tbl" ], > "delimiter" : "|" > }, > "csv" : { > "type" : "text", > "extensions" : [ "csv" ], > "delimiter" : "," > }, > "tsv" : { > "type" : "text", > "extensions" : [ "tsv" ], > "delimiter" : "\t" > }, > "httpd" : { > "type" : "httpd", > "logFormat" : "%h %t \"%r\" %>s %b \"%{Referer}i\"", > "timestampFormat" : null > }, > "parquet" : { > "type" : "parquet" > }, > "json" : { > "type" : "json", > "extensions" : [ "json" ] > }, > "avro" : { > "type" : "avro" > }, > "sequencefile" : { > "type" : "sequencefile", > "extensions" : [ "seq" ] > }, > "csvh" : { > "type" : "text", > "extensions" : [ "csvh" ], > "extractHeader" : true, > "delimiter" : "," > } > } > }, > "format" : { > "type" : "named", > "name" : "text" > }, > "columns" : [ "`*`" ], > "selectionRoot" : "file:/home/arina/files/ver", > "initialAllocation" : 1000000, > "maxAllocation" : 10000000000, > "cost" : 0.0 > }, > "first" : 0, > "last" : 10, > "initialAllocation" : 1000000, > "maxAllocation" : 10000000000, > "cost" : 10.0 > }, > "initialAllocation" : 1000000, > "maxAllocation" : 10000000000, > "cost" : 10.0 > }, > "destination" : "Cglsb2NhbGhvc3QQpfIBGKbyASCn8gEyDzEuMTAuMC1TTkFQU0hPVA==", > "initialAllocation" : 1000000, > "maxAllocation" : 10000000000, > "cost" : 10.0 > }; line: 90, column: 7] (through reference chain: > org.apache.drill.exec.physical.config.SingleSender["child"]->org.apache.drill.exec.physical.config.SelectionVectorRemover["child"]->org.apache.drill.exec.physical.config.Limit["child"]) > com.fasterxml.jackson.databind.JsonMappingException.from():223 > > com.fasterxml.jackson.databind.deser.std.StdValueInstantiator.wrapAsJsonMappingException():445 > > com.fasterxml.jackson.databind.deser.std.StdValueInstantiator.rewrapCtorProblem():464 > > com.fasterxml.jackson.databind.deser.std.StdValueInstantiator.createFromObjectWith():258 > com.fasterxml.jackson.databind.deser.impl.PropertyBasedCreator.build():135 > > com.fasterxml.jackson.databind.deser.BeanDeserializer._deserializeUsingPropertyBased():444 > > com.fasterxml.jackson.databind.deser.BeanDeserializerBase.deserializeFromObjectUsingNonDefault():1123 > > com.fasterxml.jackson.databind.deser.BeanDeserializer.deserializeFromObject():298 > > com.fasterxml.jackson.databind.deser.BeanDeserializerBase.deserializeWithObjectId():1094 > > com.fasterxml.jackson.databind.deser.BeanDeserializer._deserializeOther():166 > com.fasterxml.jackson.databind.deser.BeanDeserializer.deserialize():135 > > com.fasterxml.jackson.databind.jsontype.impl.AsPropertyTypeDeserializer._deserializeTypedForId():120 > > com.fasterxml.jackson.databind.jsontype.impl.AsPropertyTypeDeserializer.deserializeTypedFromObject():91 > > com.fasterxml.jackson.databind.deser.AbstractDeserializer.deserializeWithType():142 > > com.fasterxml.jackson.databind.deser.SettableBeanProperty.deserialize():488 > > com.fasterxml.jackson.databind.deser.BeanDeserializer._deserializeWithErrorWrapping():465 > > com.fasterxml.jackson.databind.deser.BeanDeserializer._deserializeUsingPropertyBased():380 > > com.fasterxml.jackson.databind.deser.BeanDeserializerBase.deserializeFromObjectUsingNonDefault():1123 > > com.fasterxml.jackson.databind.deser.BeanDeserializer.deserializeFromObject():298 > > com.fasterxml.jackson.databind.deser.BeanDeserializerBase.deserializeWithObjectId():1094 > > com.fasterxml.jackson.databind.deser.BeanDeserializer._deserializeOther():166 > com.fasterxml.jackson.databind.deser.BeanDeserializer.deserialize():135 > > com.fasterxml.jackson.databind.jsontype.impl.AsPropertyTypeDeserializer._deserializeTypedForId():120 > > com.fasterxml.jackson.databind.jsontype.impl.AsPropertyTypeDeserializer.deserializeTypedFromObject():91 > > com.fasterxml.jackson.databind.deser.AbstractDeserializer.deserializeWithType():142 > > com.fasterxml.jackson.databind.deser.SettableBeanProperty.deserialize():488 > > com.fasterxml.jackson.databind.deser.BeanDeserializer._deserializeWithErrorWrapping():465 > > com.fasterxml.jackson.databind.deser.BeanDeserializer._deserializeUsingPropertyBased():380 > > com.fasterxml.jackson.databind.deser.BeanDeserializerBase.deserializeFromObjectUsingNonDefault():1123 > > com.fasterxml.jackson.databind.deser.BeanDeserializer.deserializeFromObject():298 > > com.fasterxml.jackson.databind.deser.BeanDeserializerBase.deserializeWithObjectId():1094 > > com.fasterxml.jackson.databind.deser.BeanDeserializer._deserializeOther():166 > com.fasterxml.jackson.databind.deser.BeanDeserializer.deserialize():135 > > com.fasterxml.jackson.databind.jsontype.impl.AsPropertyTypeDeserializer._deserializeTypedForId():120 > > com.fasterxml.jackson.databind.jsontype.impl.AsPropertyTypeDeserializer.deserializeTypedFromObject():91 > > com.fasterxml.jackson.databind.deser.AbstractDeserializer.deserializeWithType():142 > > com.fasterxml.jackson.databind.deser.SettableBeanProperty.deserialize():488 > > com.fasterxml.jackson.databind.deser.BeanDeserializer._deserializeWithErrorWrapping():465 > > com.fasterxml.jackson.databind.deser.BeanDeserializer._deserializeUsingPropertyBased():380 > > com.fasterxml.jackson.databind.deser.BeanDeserializerBase.deserializeFromObjectUsingNonDefault():1123 > > com.fasterxml.jackson.databind.deser.BeanDeserializer.deserializeFromObject():298 > > com.fasterxml.jackson.databind.deser.BeanDeserializerBase.deserializeWithObjectId():1094 > > com.fasterxml.jackson.databind.deser.BeanDeserializer._deserializeOther():166 > com.fasterxml.jackson.databind.deser.BeanDeserializer.deserialize():135 > > com.fasterxml.jackson.databind.jsontype.impl.AsPropertyTypeDeserializer._deserializeTypedForId():120 > > com.fasterxml.jackson.databind.jsontype.impl.AsPropertyTypeDeserializer.deserializeTypedFromObject():91 > > com.fasterxml.jackson.databind.deser.AbstractDeserializer.deserializeWithType():142 > > com.fasterxml.jackson.databind.deser.impl.TypeWrappedDeserializer.deserialize():42 > com.fasterxml.jackson.databind.ObjectReader._bindAndClose():1575 > com.fasterxml.jackson.databind.ObjectReader.readValue():1200 > org.apache.drill.exec.planner.PhysicalPlanReader.readFragmentOperator():94 > org.apache.drill.exec.work.fragment.FragmentExecutor.run():203 > org.apache.drill.common.SelfCleaningRunnable.run():38 > java.util.concurrent.ThreadPoolExecutor.runWorker():1145 > java.util.concurrent.ThreadPoolExecutor$Worker.run():615 > java.lang.Thread.run():745 > Caused By (java.lang.NullPointerException) null > com.google.common.base.Preconditions.checkNotNull():210 > org.apache.drill.exec.store.dfs.easy.EasySubScan.<init>():60 > sun.reflect.NativeConstructorAccessorImpl.newInstance0():-2 > sun.reflect.NativeConstructorAccessorImpl.newInstance():57 > sun.reflect.DelegatingConstructorAccessorImpl.newInstance():45 > java.lang.reflect.Constructor.newInstance():526 > com.fasterxml.jackson.databind.introspect.AnnotatedConstructor.call():114 > > com.fasterxml.jackson.databind.deser.std.StdValueInstantiator.createFromObjectWith():256 > com.fasterxml.jackson.databind.deser.impl.PropertyBasedCreator.build():135 > > com.fasterxml.jackson.databind.deser.BeanDeserializer._deserializeUsingPropertyBased():444 > > com.fasterxml.jackson.databind.deser.BeanDeserializerBase.deserializeFromObjectUsingNonDefault():1123 > > com.fasterxml.jackson.databind.deser.BeanDeserializer.deserializeFromObject():298 > > com.fasterxml.jackson.databind.deser.BeanDeserializerBase.deserializeWithObjectId():1094 > > com.fasterxml.jackson.databind.deser.BeanDeserializer._deserializeOther():166 > com.fasterxml.jackson.databind.deser.BeanDeserializer.deserialize():135 > > com.fasterxml.jackson.databind.jsontype.impl.AsPropertyTypeDeserializer._deserializeTypedForId():120 > > com.fasterxml.jackson.databind.jsontype.impl.AsPropertyTypeDeserializer.deserializeTypedFromObject():91 > > com.fasterxml.jackson.databind.deser.AbstractDeserializer.deserializeWithType():142 > > com.fasterxml.jackson.databind.deser.SettableBeanProperty.deserialize():488 > > com.fasterxml.jackson.databind.deser.BeanDeserializer._deserializeWithErrorWrapping():465 > > com.fasterxml.jackson.databind.deser.BeanDeserializer._deserializeUsingPropertyBased():380 > > com.fasterxml.jackson.databind.deser.BeanDeserializerBase.deserializeFromObjectUsingNonDefault():1123 > > com.fasterxml.jackson.databind.deser.BeanDeserializer.deserializeFromObject():298 > > com.fasterxml.jackson.databind.deser.BeanDeserializerBase.deserializeWithObjectId():1094 > > com.fasterxml.jackson.databind.deser.BeanDeserializer._deserializeOther():166 > com.fasterxml.jackson.databind.deser.BeanDeserializer.deserialize():135 > > com.fasterxml.jackson.databind.jsontype.impl.AsPropertyTypeDeserializer._deserializeTypedForId():120 > > com.fasterxml.jackson.databind.jsontype.impl.AsPropertyTypeDeserializer.deserializeTypedFromObject():91 > > com.fasterxml.jackson.databind.deser.AbstractDeserializer.deserializeWithType():142 > > com.fasterxml.jackson.databind.deser.SettableBeanProperty.deserialize():488 > > com.fasterxml.jackson.databind.deser.BeanDeserializer._deserializeWithErrorWrapping():465 > > com.fasterxml.jackson.databind.deser.BeanDeserializer._deserializeUsingPropertyBased():380 > > com.fasterxml.jackson.databind.deser.BeanDeserializerBase.deserializeFromObjectUsingNonDefault():1123 > > com.fasterxml.jackson.databind.deser.BeanDeserializer.deserializeFromObject():298 > > com.fasterxml.jackson.databind.deser.BeanDeserializerBase.deserializeWithObjectId():1094 > > com.fasterxml.jackson.databind.deser.BeanDeserializer._deserializeOther():166 > com.fasterxml.jackson.databind.deser.BeanDeserializer.deserialize():135 > > com.fasterxml.jackson.databind.jsontype.impl.AsPropertyTypeDeserializer._deserializeTypedForId():120 > > com.fasterxml.jackson.databind.jsontype.impl.AsPropertyTypeDeserializer.deserializeTypedFromObject():91 > > com.fasterxml.jackson.databind.deser.AbstractDeserializer.deserializeWithType():142 > > com.fasterxml.jackson.databind.deser.SettableBeanProperty.deserialize():488 > > com.fasterxml.jackson.databind.deser.BeanDeserializer._deserializeWithErrorWrapping():465 > > com.fasterxml.jackson.databind.deser.BeanDeserializer._deserializeUsingPropertyBased():380 > > com.fasterxml.jackson.databind.deser.BeanDeserializerBase.deserializeFromObjectUsingNonDefault():1123 > > com.fasterxml.jackson.databind.deser.BeanDeserializer.deserializeFromObject():298 > > com.fasterxml.jackson.databind.deser.BeanDeserializerBase.deserializeWithObjectId():1094 > > com.fasterxml.jackson.databind.deser.BeanDeserializer._deserializeOther():166 > com.fasterxml.jackson.databind.deser.BeanDeserializer.deserialize():135 > > com.fasterxml.jackson.databind.jsontype.impl.AsPropertyTypeDeserializer._deserializeTypedForId():120 > > com.fasterxml.jackson.databind.jsontype.impl.AsPropertyTypeDeserializer.deserializeTypedFromObject():91 > > com.fasterxml.jackson.databind.deser.AbstractDeserializer.deserializeWithType():142 > > com.fasterxml.jackson.databind.deser.impl.TypeWrappedDeserializer.deserialize():42 > com.fasterxml.jackson.databind.ObjectReader._bindAndClose():1575 > com.fasterxml.jackson.databind.ObjectReader.readValue():1200 > org.apache.drill.exec.planner.PhysicalPlanReader.readFragmentOperator():94 > org.apache.drill.exec.work.fragment.FragmentExecutor.run():203 > org.apache.drill.common.SelfCleaningRunnable.run():38 > java.util.concurrent.ThreadPoolExecutor.runWorker():1145 > java.util.concurrent.ThreadPoolExecutor$Worker.run():615 > java.lang.Thread.run():745 > at > org.apache.drill.exec.rpc.user.QueryResultHandler.resultArrived(QueryResultHandler.java:123) > at > org.apache.drill.exec.rpc.user.UserClient.handleReponse(UserClient.java:145) > at > org.apache.drill.exec.rpc.BasicClientWithConnection.handle(BasicClientWithConnection.java:46) > at > org.apache.drill.exec.rpc.BasicClientWithConnection.handle(BasicClientWithConnection.java:31) > at org.apache.drill.exec.rpc.RpcBus.handle(RpcBus.java:65) > at org.apache.drill.exec.rpc.RpcBus$RequestEvent.run(RpcBus.java:363) > at > org.apache.drill.common.SerializedExecutor$RunnableProcessor.run(SerializedExecutor.java:89) > at > org.apache.drill.exec.rpc.RpcBus$SameExecutor.execute(RpcBus.java:240) > at > org.apache.drill.common.SerializedExecutor.execute(SerializedExecutor.java:123) > at > org.apache.drill.exec.rpc.RpcBus$InboundHandler.decode(RpcBus.java:274) > at > org.apache.drill.exec.rpc.RpcBus$InboundHandler.decode(RpcBus.java:245) > at > io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:89) > at > io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:339) > at > io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:324) > at > io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:254) > at > io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:339) > at > io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:324) > at > io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) > at > io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:339) > at > io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:324) > at > io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:242) > at > io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:339) > at > io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:324) > at > io.netty.channel.ChannelInboundHandlerAdapter.channelRead(ChannelInboundHandlerAdapter.java:86) > at > io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:339) > at > io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:324) > at > io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:847) > at > io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:131) > at > io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:511) > at > io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:468) > at > io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:382) > at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:354) > at > io.netty.util.concurrent.SingleThreadEventExecutor$2.run(SingleThreadEventExecutor.java:111) > at java.lang.Thread.run(Thread.java:745) > {noformat} -- This message was sent by Atlassian JIRA (v7.6.3#76005)