DRILL-1295: Select returns null for non pivot fields with a nested select * Fix DrillTextRecordReader to handle the case when requested columns includes the entire group and a subset.
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/654c879f Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/654c879f Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/654c879f Branch: refs/heads/master Commit: 654c879f7caa13925edca911de1b59d04d8f1a8b Parents: bc06b4c Author: Mehant Baid <[email protected]> Authored: Sat Aug 16 17:15:33 2014 -0700 Committer: Jacques Nadeau <[email protected]> Committed: Mon Aug 18 09:23:50 2014 -0700 ---------------------------------------------------------------------- .../exec/store/text/DrillTextRecordReader.java | 12 +++++++++- .../org/apache/drill/TestExampleQueries.java | 8 +++++++ .../test/resources/store/text/data/nations.csv | 25 ++++++++++++++++++++ 3 files changed, 44 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/654c879f/exec/java-exec/src/main/java/org/apache/drill/exec/store/text/DrillTextRecordReader.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/text/DrillTextRecordReader.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/text/DrillTextRecordReader.java index 5bf571d..28babd1 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/text/DrillTextRecordReader.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/text/DrillTextRecordReader.java @@ -69,6 +69,8 @@ public class DrillTextRecordReader implements RecordReader { public DrillTextRecordReader(FileSplit split, FragmentContext context, char delimiter, List<SchemaPath> columns) { this.context = context; this.delimiter = (byte) delimiter; + boolean getEntireRow = false; + if(columns != null) { for (SchemaPath path : columns) { assert path.getRootSegment().isNamed(); @@ -78,12 +80,20 @@ public class DrillTextRecordReader implements RecordReader { Preconditions.checkArgument(path.getRootSegment().getChild().isArray(),"Selected column must be an array index"); int index = path.getRootSegment().getChild().getArraySegment().getIndex(); columnIds.add(index); + } else { + getEntireRow = true; } } Collections.sort(columnIds); } targetRecordCount = context.getConfig().getInt(ExecConstants.TEXT_LINE_READER_BATCH_SIZE); - numCols = columnIds.size(); + + /* If one of the columns requested is the entire row ('columns') then ignore the rest of the columns + * we are going copy all the values in the repeated varchar vector + */ + if (!getEntireRow) { + numCols = columnIds.size(); + } TextInputFormat inputFormat = new TextInputFormat(); JobConf job = new JobConf(); job.setInt("io.file.buffer.size", context.getConfig().getInt(ExecConstants.TEXT_LINE_READER_BUFFER_SIZE)); http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/654c879f/exec/java-exec/src/test/java/org/apache/drill/TestExampleQueries.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/TestExampleQueries.java b/exec/java-exec/src/test/java/org/apache/drill/TestExampleQueries.java index fcc10c0..0ac2c09 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/TestExampleQueries.java +++ b/exec/java-exec/src/test/java/org/apache/drill/TestExampleQueries.java @@ -300,4 +300,12 @@ public class TestExampleQueries extends BaseTestQuery{ test("select n_regionkey, count(*) from cp.`tpch/nation.parquet` group by n_regionkey limit 0"); } + @Test + public void testTextJoin() throws Exception { + String root = FileUtils.getResourceAsFile("/store/text/data/nations.csv").toURI().toString(); + String root1 = FileUtils.getResourceAsFile("/store/text/data/regions.csv").toURI().toString(); + String query = String.format("select t1.columns[1] from dfs_test.`%s` t1, dfs_test.`%s` t2 where t1.columns[0] = t2.columns[0]", root, root1); + test(query); + } + } http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/654c879f/exec/java-exec/src/test/resources/store/text/data/nations.csv ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/resources/store/text/data/nations.csv b/exec/java-exec/src/test/resources/store/text/data/nations.csv new file mode 100644 index 0000000..be2301b --- /dev/null +++ b/exec/java-exec/src/test/resources/store/text/data/nations.csv @@ -0,0 +1,25 @@ +0,ALGERIA,0,haggle.carefullyfinaldepositsdetectslylyagai, +1,ARGENTINA,1,alfoxespromiseslylyaccordingtotheregularaccounts.boldrequestsalon, +2,BRAZIL,1,yalongsideofthependingdeposits.carefullyspecialpackagesareabouttheironicforges.slylyspecial, +3,CANADA,1,eashangironic,silentpackages.slylyregularpackagesarefuriouslyoverthetithes.fluffilybold, +4,EGYPT,4,yabovethecarefullyunusualtheodolites.finaldugoutsarequicklyacrossthefuriouslyregulard, +5,ETHIOPIA,0,venpackageswakequickly.regu, +6,FRANCE,3,refullyfinalrequests.regular,ironi, +7,GERMANY,3,lplatelets.regularaccountsx-ray:unusual,regularacco, +8,INDIA,2,ssexcusescajoleslylyacrossthepackages.depositsprintaroun, +9,INDONESIA,2,slylyexpressasymptotes.regulardepositshaggleslyly.carefullyironichockeyplayerssleepblithely.carefull, +10,IRAN,4,efullyalongsideoftheslylyfinaldependencies., +11,IRAQ,4,nicdepositsboostatopthequicklyfinalrequests?quicklyregula, +12,JAPAN,2,ously.final,expressgiftscajolea, +13,JORDAN,4,icdepositsareblithelyaboutthecarefullyregularpa, +14,KENYA,0,pendingexcuseshagglefuriouslydeposits.pending,expresspintobeanswakefluffilypastt, +15,MOROCCO,0,rns.blithelyboldcourtsamongthecloselyregularpackagesusefuriouslyboldplatelets?, +16,MOZAMBIQUE,0,s.ironic,unusualasymptoteswakeblithelyr, +17,PERU,1,platelets.blithelypendingdependenciesusefluffilyacrosstheevenpintobeans.carefullysilentaccoun, +18,CHINA,2,cdependencies.furiouslyexpressnotornissleepslylyregularaccounts.ideassleep.depos, +19,ROMANIA,3,ularasymptotesareaboutthefuriousmultipliers.expressdependenciesnagabovetheironicallyironicaccount, +20,SAUDIARABIA,4,ts.silentrequestshaggle.closelyexpresspackagessleepacrosstheblithely, +21,VIETNAM,2,helyenticinglyexpressaccounts.even,final, +22,RUSSIA,3,requestsagainsttheplateletsuseneveraccordingtothequicklyregularpint, +23,UNITEDKINGDOM,3,eansboostcarefullyspecialrequests.accountsare.carefull, +24,UNITEDSTATES,1,yfinalpackages.slowfoxescajolequickly.quicklysilentplateletsbreachironicaccounts.unusualpintobe, \ No newline at end of file
