MD-703: Cannot select `_id` field from MapR-DB JSON Table + Included Smidth's fix for column shuffling. + Fix the private-drill-mapr-plugin build.
Project: http://git-wip-us.apache.org/repos/asf/drill/repo Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/5d36f801 Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/5d36f801 Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/5d36f801 Branch: refs/heads/master Commit: 5d36f8013829406f698610bc4a4f7185698365e6 Parents: 16d4ab8 Author: Aditya <adi...@mapr.com> Authored: Thu Feb 4 16:43:27 2016 -0800 Committer: Aditya Kishore <a...@apache.org> Committed: Fri Sep 9 10:08:35 2016 -0700 ---------------------------------------------------------------------- contrib/format-maprdb/pom.xml | 6 +++++ .../exec/store/maprdb/MapRDBGroupScan.java | 2 +- .../store/maprdb/MapRDBPushFilterIntoScan.java | 4 ++-- .../maprdb/json/MaprDBJsonRecordReader.java | 24 ++++++++++++-------- .../drill/maprdb/tests/json/TestSimpleJson.java | 4 ++-- 5 files changed, 26 insertions(+), 14 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/drill/blob/5d36f801/contrib/format-maprdb/pom.xml ---------------------------------------------------------------------- diff --git a/contrib/format-maprdb/pom.xml b/contrib/format-maprdb/pom.xml index bcb6c29..db0572f 100644 --- a/contrib/format-maprdb/pom.xml +++ b/contrib/format-maprdb/pom.xml @@ -35,6 +35,7 @@ <mapr.version>${project.parent.version}</mapr.version> <drill.version>${project.version}</drill.version> <hbase.version>0.98.12-mapr-1506</hbase.version> + <guava.version>18.0</guava.version> <maprdb.TestSuite>**/MaprDBTestsSuite.class</maprdb.TestSuite> </properties> @@ -118,6 +119,11 @@ </exclusions> </dependency> <dependency> + <groupId>com.google.guava</groupId> + <artifactId>guava</artifactId> + <version>${guava.version}</version> + </dependency> + <dependency> <groupId>org.apache.drill.exec</groupId> <artifactId>drill-java-exec</artifactId> <version>${drill.version}</version> http://git-wip-us.apache.org/repos/asf/drill/blob/5d36f801/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBGroupScan.java ---------------------------------------------------------------------- diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBGroupScan.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBGroupScan.java index cbfb18c..4d30399 100644 --- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBGroupScan.java +++ b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBGroupScan.java @@ -61,7 +61,7 @@ public abstract class MapRDBGroupScan extends AbstractGroupScan { private boolean filterPushedDown = false; - private Stopwatch watch = new Stopwatch(); + private Stopwatch watch = Stopwatch.createUnstarted(); private static final Comparator<List<MapRDBSubScanSpec>> LIST_SIZE_COMPARATOR = new Comparator<List<MapRDBSubScanSpec>>() { @Override http://git-wip-us.apache.org/repos/asf/drill/blob/5d36f801/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBPushFilterIntoScan.java ---------------------------------------------------------------------- diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBPushFilterIntoScan.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBPushFilterIntoScan.java index 714221f..8a0902e 100644 --- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBPushFilterIntoScan.java +++ b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBPushFilterIntoScan.java @@ -90,11 +90,11 @@ public abstract class MapRDBPushFilterIntoScan extends StoragePluginOptimizerRul if (scan.getGroupScan() instanceof BinaryTableGroupScan) { BinaryTableGroupScan groupScan = (BinaryTableGroupScan)scan.getGroupScan(); - doPushFilterIntoBinaryGroupScan(call, filter, null, scan, groupScan, condition); + doPushFilterIntoBinaryGroupScan(call, filter, project, scan, groupScan, condition); } else { assert(scan.getGroupScan() instanceof JsonTableGroupScan); JsonTableGroupScan groupScan = (JsonTableGroupScan)scan.getGroupScan(); - doPushFilterIntoJsonGroupScan(call, filter, null, scan, groupScan, condition); + doPushFilterIntoJsonGroupScan(call, filter, project, scan, groupScan, condition); } } http://git-wip-us.apache.org/repos/asf/drill/blob/5d36f801/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/MaprDBJsonRecordReader.java ---------------------------------------------------------------------- diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/MaprDBJsonRecordReader.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/MaprDBJsonRecordReader.java index 2e0e0c1..b69f4c4 100644 --- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/MaprDBJsonRecordReader.java +++ b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/MaprDBJsonRecordReader.java @@ -17,11 +17,9 @@ */ package org.apache.drill.exec.store.maprdb.json; -import static org.ojai.DocumentConstants.ID_FIELD; import static org.ojai.DocumentConstants.ID_KEY; import java.nio.ByteBuffer; -import java.util.ArrayList; import java.util.Collection; import java.util.Iterator; import java.util.List; @@ -51,7 +49,6 @@ import org.ojai.store.QueryCondition; import com.google.common.base.Preconditions; import com.google.common.base.Stopwatch; -import com.google.common.collect.Lists; import com.google.common.collect.Sets; import com.mapr.db.MapRDB; import com.mapr.db.Table; @@ -98,18 +95,27 @@ public class MaprDBJsonRecordReader extends AbstractRecordReader { protected Collection<SchemaPath> transformColumns(Collection<SchemaPath> columns) { Set<SchemaPath> transformed = Sets.newLinkedHashSet(); if (!isStarQuery()) { - ArrayList<Object> projectedFieldsList = Lists.newArrayList(); + Set<FieldPath> projectedFieldsSet = Sets.newTreeSet(); for (SchemaPath column : columns) { if (column.getRootSegment().getPath().equalsIgnoreCase(ID_KEY)) { + /* + * we do not include _id field in the set of projected fields + * because the DB currently can not return a document if only + * the _id field was projected. This should really be fixed in + * the DB client (Bug 21708) to avoid transferring the entire + * document when only _id is requested. + */ + // projectedFieldsList.add(ID_FIELD); transformed.add(ID_PATH); - projectedFieldsList.add(ID_FIELD); includeId = true; } else { transformed.add(SchemaPath.getSimplePath(column.getRootSegment().getPath())); - projectedFieldsList.add(FieldPath.parseFrom(column.getAsUnescapedPath())); + projectedFieldsSet.add(FieldPath.parseFrom(column.getAsUnescapedPath())); } } - projectedFields = projectedFieldsList.toArray(new FieldPath[projectedFieldsList.size()]); + if (projectedFieldsSet.size() > 0) { + projectedFields = projectedFieldsSet.toArray(new FieldPath[projectedFieldsSet.size()]); + } } else { transformed.add(ID_PATH); includeId = true; @@ -135,7 +141,7 @@ public class MaprDBJsonRecordReader extends AbstractRecordReader { @Override public int next() { - Stopwatch watch = new Stopwatch(); + Stopwatch watch = Stopwatch.createUnstarted(); watch.start(); writer.allocate(); @@ -230,7 +236,7 @@ public class MaprDBJsonRecordReader extends AbstractRecordReader { default: throw unsupportedError("Unsupported type: %s encountered during the query.", event); } - } catch (IllegalArgumentException e) { + } catch (IllegalStateException | IllegalArgumentException e) { logger.warn(String.format("Possible schema change at _id: '%s', field: '%s'", IdCodec.asString(reader.getId()), fieldName), e); } http://git-wip-us.apache.org/repos/asf/drill/blob/5d36f801/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/json/TestSimpleJson.java ---------------------------------------------------------------------- diff --git a/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/json/TestSimpleJson.java b/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/json/TestSimpleJson.java index f05b87a..414b823 100644 --- a/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/json/TestSimpleJson.java +++ b/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/json/TestSimpleJson.java @@ -48,9 +48,9 @@ public class TestSimpleJson extends BaseTestQuery { @Test public void testMe() throws Exception { - setColumnWidths(new int[] {25, 40, 40, 40}); + setColumnWidths(new int[] {23}); final String sql = "SELECT\n" - + " _id, name, categories, full_address\n" + + " _id\n" + "FROM\n" + " hbase.`business` business"; runSQLAndVerifyCount(sql, 10);
