DRILL-1419: JSON reader regression from project pushdown patch. Patch amended to include a minimal subset of the test file needed to confirm the issue.
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/130e24a8 Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/130e24a8 Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/130e24a8 Branch: refs/heads/master Commit: 130e24a803d182145f8c657bd272d27bc7d6ad6f Parents: 2fa5936 Author: Jason Altekruse <[email protected]> Authored: Mon Sep 15 10:45:45 2014 -0700 Committer: Steven Phillips <[email protected]> Committed: Mon Sep 29 18:21:44 2014 -0700 ---------------------------------------------------------------------- .../drill/exec/store/easy/json/JSONRecordReader2.java | 11 +++-------- .../drill/exec/vector/complex/writer/TestJsonReader.java | 11 ++++++++++- exec/java-exec/src/test/resources/store/json/clicks.json | 5 +++++ 3 files changed, 18 insertions(+), 9 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/130e24a8/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/JSONRecordReader2.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/JSONRecordReader2.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/JSONRecordReader2.java index c2218b5..18cff5d 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/JSONRecordReader2.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/JSONRecordReader2.java @@ -126,16 +126,11 @@ public class JSONRecordReader2 extends AbstractRecordReader { for (SchemaPath sp :jsonReader.getNullColumns() ) { PathSegment root = sp.getRootSegment(); BaseWriter.MapWriter fieldWriter = writer.rootAsMap(); - if (root.getChild() != null && ! root.getChild().isArray()) { + while (root.getChild() != null && ! root.getChild().isArray()) { fieldWriter = fieldWriter.map(root.getNameSegment().getPath()); - while ( root.getChild().getChild() != null && ! root.getChild().isArray() ) { - fieldWriter = fieldWriter.map(root.getChild().getNameSegment().getPath()); - root = root.getChild(); - } - fieldWriter.integer(root.getChild().getNameSegment().getPath()); - } else { - fieldWriter.integer(root.getNameSegment().getPath()); + root = root.getChild(); } + fieldWriter.integer(root.getNameSegment().getPath()); } writer.setValueCount(recordCount); http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/130e24a8/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestJsonReader.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestJsonReader.java b/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestJsonReader.java index 9e13ae4..fd38bd3 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestJsonReader.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestJsonReader.java @@ -59,7 +59,7 @@ public class TestJsonReader extends BaseTestQuery { static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestJsonReader.class); private static BufferAllocator allocator; - private static final boolean VERBOSE_DEBUG = true; + private static final boolean VERBOSE_DEBUG = false; @BeforeClass public static void setupAllocator() { @@ -98,6 +98,15 @@ public class TestJsonReader extends BaseTestQuery { } @Test + public void testDrill_1419() throws Exception { + String[] queries = {"select t.trans_id, t.trans_info.prod_id[0],t.trans_info.prod_id[1] from cp.`/store/json/clicks.json` t limit 5"}; + long[] rowCounts = {5}; + String filename = "/store/json/clicks.json"; + runTestsOnFile(filename, UserBitShared.QueryType.SQL, queries, rowCounts); + } + + + @Test public void testSingleColumnRead_vector_fill_bug() throws Exception { String[] queries = {"select * from cp.`/store/json/single_column_long_file.json`"}; long[] rowCounts = {13512}; http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/130e24a8/exec/java-exec/src/test/resources/store/json/clicks.json ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/resources/store/json/clicks.json b/exec/java-exec/src/test/resources/store/json/clicks.json new file mode 100644 index 0000000..c79ea7e --- /dev/null +++ b/exec/java-exec/src/test/resources/store/json/clicks.json @@ -0,0 +1,5 @@ +{"trans_id":31920,"date":"2014-04-26","time":"12:17:12","user_info":{"cust_id":22526,"device":"IOS5","state":"il"},"trans_info":{"prod_id":[174,2],"purch_flag":"false"}} +{"trans_id":31026,"date":"2014-04-20","time":"13:50:29","user_info":{"cust_id":16368,"device":"AOS4.2","state":"nc"},"trans_info":{"prod_id":[],"purch_flag":"false"}} +{"trans_id":33848,"date":"2014-04-10","time":"04:44:42","user_info":{"cust_id":21449,"device":"IOS6","state":"oh"},"trans_info":{"prod_id":[582],"purch_flag":"false"}} +{"trans_id":32383,"date":"2014-04-18","time":"06:27:47","user_info":{"cust_id":20323,"device":"IOS5","state":"oh"},"trans_info":{"prod_id":[710,47],"purch_flag":"false"}} +{"trans_id":32359,"date":"2014-04-19","time":"23:13:25","user_info":{"cust_id":15360,"device":"IOS5","state":"ca"},"trans_info":{"prod_id":[0,8,170,173,1,124,46,764,30,711,0,3,25],"purch_flag":"true"}}
