[01/11] drill git commit: DRILL-5337: OpenTSDB storage plugin

2017-11-13 Thread arina 
Repository: drill
Updated Branches:
  refs/heads/master 29e054769 -> 3036d3700


http://git-wip-us.apache.org/repos/asf/drill/blob/496c97d1/contrib/storage-opentsdb/src/main/java/org/apache/drill/exec/store/openTSDB/schema/OpenTSDBSchemaFactory.java
--
diff --git 
a/contrib/storage-opentsdb/src/main/java/org/apache/drill/exec/store/openTSDB/schema/OpenTSDBSchemaFactory.java
 
b/contrib/storage-opentsdb/src/main/java/org/apache/drill/exec/store/openTSDB/schema/OpenTSDBSchemaFactory.java
new file mode 100644
index 000..cca39d8
--- /dev/null
+++ 
b/contrib/storage-opentsdb/src/main/java/org/apache/drill/exec/store/openTSDB/schema/OpenTSDBSchemaFactory.java
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.store.openTSDB.schema;
+
+import org.apache.calcite.schema.SchemaPlus;
+import org.apache.calcite.schema.Table;
+import org.apache.drill.exec.store.AbstractSchema;
+import org.apache.drill.exec.store.SchemaConfig;
+import org.apache.drill.exec.store.SchemaFactory;
+import org.apache.drill.exec.store.openTSDB.DrillOpenTSDBTable;
+import org.apache.drill.exec.store.openTSDB.OpenTSDBScanSpec;
+import org.apache.drill.exec.store.openTSDB.OpenTSDBStoragePlugin;
+import org.apache.drill.exec.store.openTSDB.OpenTSDBStoragePluginConfig;
+import org.apache.drill.exec.store.openTSDB.client.Schema;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.Set;
+
+public class OpenTSDBSchemaFactory implements SchemaFactory {
+
+  private static final Logger log = 
LoggerFactory.getLogger(OpenTSDBSchemaFactory.class);
+
+  private final String schemaName;
+  private final OpenTSDBStoragePlugin plugin;
+
+  public OpenTSDBSchemaFactory(OpenTSDBStoragePlugin plugin, String 
schemaName) {
+this.plugin = plugin;
+this.schemaName = schemaName;
+  }
+
+  @Override
+  public void registerSchemas(SchemaConfig schemaConfig, SchemaPlus parent) 
throws IOException {
+OpenTSDBSchema schema = new OpenTSDBSchema(schemaName);
+parent.add(schemaName, schema);
+  }
+
+  class OpenTSDBSchema extends AbstractSchema {
+
+OpenTSDBSchema(String name) {
+  super(Collections.emptyList(), name);
+}
+
+@Override
+public Table getTable(String name) {
+  OpenTSDBScanSpec scanSpec = new OpenTSDBScanSpec(name);
+return new DrillOpenTSDBTable(schemaName, plugin, new 
Schema(plugin.getClient(), name), scanSpec);
+}
+
+@Override
+public Set getTableNames() {
+  return plugin.getClient().getAllMetricNames();
+}
+
+@Override
+public String getTypeName() {
+  return OpenTSDBStoragePluginConfig.NAME;
+}
+  }
+}

http://git-wip-us.apache.org/repos/asf/drill/blob/496c97d1/contrib/storage-opentsdb/src/main/resources/bootstrap-storage-plugins.json
--
diff --git 
a/contrib/storage-opentsdb/src/main/resources/bootstrap-storage-plugins.json 
b/contrib/storage-opentsdb/src/main/resources/bootstrap-storage-plugins.json
new file mode 100644
index 000..d1055c1
--- /dev/null
+++ b/contrib/storage-opentsdb/src/main/resources/bootstrap-storage-plugins.json
@@ -0,0 +1,9 @@
+{
+  "storage": {
+openTSDB: {
+  type: "openTSDB",
+  connection: "http://localhost:1";,
+  enabled: false
+}
+  }
+}

http://git-wip-us.apache.org/repos/asf/drill/blob/496c97d1/contrib/storage-opentsdb/src/main/resources/drill-module.conf
--
diff --git a/contrib/storage-opentsdb/src/main/resources/drill-module.conf 
b/contrib/storage-opentsdb/src/main/resources/drill-module.conf
new file mode 100644
index 000..d5743da
--- /dev/null
+++ b/contrib/storage-opentsdb/src/main/resources/drill-module.conf
@@ -0,0 +1,21 @@
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements.  See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0

[08/11] drill git commit: DRILL-5909: Added new Counter metrics

2017-11-13 Thread arina 
DRILL-5909: Added new Counter metrics

closes #1019


Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/99e4504f
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/99e4504f
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/99e4504f

Branch: refs/heads/master
Commit: 99e4504f2aa9d11a6bc49a57f5edb70f8d621747
Parents: df95709
Author: Prasad Nagaraj Subramanya 
Authored: Wed Nov 1 13:49:43 2017 -0700
Committer: Arina Ielchiieva 
Committed: Mon Nov 13 11:06:53 2017 +0200

--
 .../org/apache/drill/exec/work/foreman/Foreman.java | 16 
 1 file changed, 16 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/drill/blob/99e4504f/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/Foreman.java
--
diff --git 
a/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/Foreman.java 
b/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/Foreman.java
index 10d8537..a1d150e 100644
--- 
a/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/Foreman.java
+++ 
b/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/Foreman.java
@@ -126,6 +126,9 @@ public class Foreman implements Runnable {
   private static final Counter enqueuedQueries = 
DrillMetrics.getRegistry().counter("drill.queries.enqueued");
   private static final Counter runningQueries = 
DrillMetrics.getRegistry().counter("drill.queries.running");
   private static final Counter completedQueries = 
DrillMetrics.getRegistry().counter("drill.queries.completed");
+  private static final Counter succeededQueries = 
DrillMetrics.getRegistry().counter("drill.queries.succeeded");
+  private static final Counter failedQueries = 
DrillMetrics.getRegistry().counter("drill.queries.failed");
+  private static final Counter canceledQueries = 
DrillMetrics.getRegistry().counter("drill.queries.canceled");
 
   private final QueryId queryId;
   private final String queryIdString;
@@ -837,6 +840,19 @@ public class Foreman implements Runnable {
 logger.warn("unable to close query manager", e);
   }
 
+  // Incrementing QueryState counters
+  switch (state) {
+case FAILED:
+  failedQueries.inc();
+  break;
+case CANCELED:
+  canceledQueries.inc();
+  break;
+case COMPLETED:
+  succeededQueries.inc();
+  break;
+  }
+
   runningQueries.dec();
   completedQueries.inc();
   try {

[10/11] drill git commit: DRILL-5923: Display name for query state

2017-11-13 Thread arina 
DRILL-5923: Display name for query state

closes #1021


Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/30da051b
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/30da051b
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/30da051b

Branch: refs/heads/master
Commit: 30da051b968926b7ead388b7df56c402dbdd5cb8
Parents: ed6c4bc
Author: Prasad Nagaraj Subramanya 
Authored: Thu Nov 9 15:00:15 2017 -0800
Committer: Arina Ielchiieva 
Committed: Mon Nov 13 11:45:20 2017 +0200

--
 .../server/rest/profile/ProfileResources.java   |  8 ++--
 .../exec/server/rest/profile/ProfileUtil.java   | 48 
 .../server/rest/profile/ProfileWrapper.java |  4 ++
 .../src/main/resources/rest/profile/profile.ftl |  3 +-
 4 files changed, 59 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/drill/blob/30da051b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/profile/ProfileResources.java
--
diff --git 
a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/profile/ProfileResources.java
 
b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/profile/ProfileResources.java
index 875c96e..14056b0 100644
--- 
a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/profile/ProfileResources.java
+++ 
b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/profile/ProfileResources.java
@@ -63,7 +63,7 @@ import com.google.common.collect.Lists;
 @Path("/")
 @RolesAllowed(DrillUserPrincipal.AUTHENTICATED_ROLE)
 public class ProfileResources {
-  static final org.slf4j.Logger logger = 
org.slf4j.LoggerFactory.getLogger(ProfileResources.class);
+  private static final org.slf4j.Logger logger = 
org.slf4j.LoggerFactory.getLogger(ProfileResources.class);
 
   @Inject UserAuthEnabled authEnabled;
   @Inject WorkManager work;
@@ -217,7 +217,8 @@ public class ProfileResources {
 runningQueries.add(
 new ProfileInfo(work.getContext().getConfig(),
 runningEntry.getKey(), profile.getStart(), 
System.currentTimeMillis(),
-profile.getForeman().getAddress(), profile.getQuery(), 
profile.getState().name(),
+profile.getForeman().getAddress(), profile.getQuery(),
+ProfileUtil.getQueryStateDisplayName(profile.getState()),
 profile.getUser(), profile.getTotalCost(), 
profile.getQueueName()));
   }
 } catch (Exception e) {
@@ -247,7 +248,8 @@ public class ProfileResources {
 finishedQueries.add(
 new ProfileInfo(work.getContext().getConfig(),
 profileEntry.getKey(), profile.getStart(), 
profile.getEnd(),
-profile.getForeman().getAddress(), profile.getQuery(), 
profile.getState().name(),
+profile.getForeman().getAddress(), profile.getQuery(),
+ProfileUtil.getQueryStateDisplayName(profile.getState()),
 profile.getUser(), profile.getTotalCost(), 
profile.getQueueName()));
   }
 } catch (Exception e) {

http://git-wip-us.apache.org/repos/asf/drill/blob/30da051b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/profile/ProfileUtil.java
--
diff --git 
a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/profile/ProfileUtil.java
 
b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/profile/ProfileUtil.java
new file mode 100644
index 000..cfc7977
--- /dev/null
+++ 
b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/profile/ProfileUtil.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.server.rest.profile;
+
+import org.apache.drill.exec.proto.UserBitShared.QueryResult.QueryState;
+
+public class ProfileUtil {
+  // Display names for QueryState enum in UserBitShared.proto
+  private sta

[09/11] drill git commit: DRILL-5921: Display counter metrics in table

2017-11-13 Thread arina 
DRILL-5921: Display counter metrics in table

closes #1020


Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/ed6c4bc8
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/ed6c4bc8
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/ed6c4bc8

Branch: refs/heads/master
Commit: ed6c4bc8e18b55e2aaac731f6cb4bc0f4480e490
Parents: 99e4504
Author: Prasad Nagaraj Subramanya 
Authored: Wed Nov 8 23:18:42 2017 -0800
Committer: Arina Ielchiieva 
Committed: Mon Nov 13 11:45:20 2017 +0200

--
 .../src/main/resources/rest/metrics/metrics.ftl | 34 +++-
 1 file changed, 18 insertions(+), 16 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/drill/blob/ed6c4bc8/exec/java-exec/src/main/resources/rest/metrics/metrics.ftl
--
diff --git a/exec/java-exec/src/main/resources/rest/metrics/metrics.ftl 
b/exec/java-exec/src/main/resources/rest/metrics/metrics.ftl
index 1600a2d..fbd2f4e 100644
--- a/exec/java-exec/src/main/resources/rest/metrics/metrics.ftl
+++ b/exec/java-exec/src/main/resources/rest/metrics/metrics.ftl
@@ -63,12 +63,6 @@
   No histograms.
 
   
-  Meters
-  
-
-  No meters.
-
-  
   Timers
   
 
@@ -98,6 +92,22 @@
   });
 };
 
+function createCountersTable(counters) {
+  $("#countersVal").html(function() {
+var table = "";
+table += "";
+$.each(counters, function(key, value) {
+  table += "";
+  table += "" + key + "";
+  table += "" + value.count + "";
+  table += "";
+});
+table += "";
+table += "";
+return table;
+  });
+};
+
 function updateBars(gauges) {
   $.each(["heap","non-heap","total"], function(i, key) {
 var used= gauges[key + ".used"].value;
@@ -138,21 +148,13 @@
   });
 };
 
-function updateOthers(metrics) {
-  $.each(["counters", "meters"], function(i, key) {
-if(! $.isEmptyObject(metrics[key])) {
-  $("#" + key + "Val").html(JSON.stringify(metrics[key], null, 2));
-}
-  });
-};
-
 var update = function() {
   $.get("/status/metrics", function(metrics) {
 updateGauges(metrics.gauges);
 updateBars(metrics.gauges);
 if(! $.isEmptyObject(metrics.timers)) createTable(metrics.timers, 
"timers");
 if(! $.isEmptyObject(metrics.histograms)) 
createTable(metrics.histograms, "histograms");
-updateOthers(metrics);
+if(! $.isEmptyObject(metrics.counters)) 
createCountersTable(metrics.counters);
   });
 };
 
@@ -161,4 +163,4 @@
   
 
 
-<@page_html/>
\ No newline at end of file
+<@page_html/>

[05/11] drill git commit: DRILL-5863: Sortable table incorrectly sorts fragments/time lexically

2017-11-13 Thread arina 
DRILL-5863: Sortable table incorrectly sorts fragments/time lexically

The DataTables jQuery library sorts data based on the value of the element in a 
column.
However, since Drill publishes sortable items like fragment IDs and time 
durations as non-numeric text, the sorting is incorrect.
This PR fixes the fragment and duration ordering based on their implicit 
numeric values (minor ID and millisecond representation, respectively).
Support memory chaining

closes #987


Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/59c74472
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/59c74472
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/59c74472

Branch: refs/heads/master
Commit: 59c7447262a22f7f1099f1e0f6d33d44acf8813f
Parents: c1118a3
Author: Kunal Khatua 
Authored: Wed Oct 11 21:35:47 2017 -0700
Committer: Arina Ielchiieva 
Committed: Mon Nov 13 11:05:51 2017 +0200

--
 .../server/rest/profile/FragmentWrapper.java|  7 ++-
 .../server/rest/profile/OperatorWrapper.java|  5 +-
 .../exec/server/rest/profile/TableBuilder.java  | 54 +++-
 3 files changed, 52 insertions(+), 14 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/drill/blob/59c74472/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/profile/FragmentWrapper.java
--
diff --git 
a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/profile/FragmentWrapper.java
 
b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/profile/FragmentWrapper.java
index 2233f2e..5496f83 100644
--- 
a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/profile/FragmentWrapper.java
+++ 
b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/profile/FragmentWrapper.java
@@ -19,7 +19,9 @@ package org.apache.drill.exec.server.rest.profile;
 
 import java.util.ArrayList;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 
 import org.apache.drill.exec.proto.UserBitShared.MajorFragmentProfile;
 import org.apache.drill.exec.proto.UserBitShared.MinorFragmentProfile;
@@ -228,6 +230,8 @@ public class FragmentWrapper {
 Collections2.filter(major.getMinorFragmentProfileList(), 
Filters.missingOperatorsOrTimes));
 
 Collections.sort(complete, Comparators.minorId);
+
+Map attributeMap = new HashMap(); 
//Reusing for different fragments
 for (final MinorFragmentProfile minor : complete) {
   final ArrayList ops = new 
ArrayList<>(minor.getOperatorProfileList());
 
@@ -244,7 +248,8 @@ public class FragmentWrapper {
 biggestBatches = Math.max(biggestBatches, batches);
   }
 
-  builder.appendCell(new 
OperatorPathBuilder().setMajor(major).setMinor(minor).build());
+  attributeMap.put("data-order", 
String.valueOf(minor.getMinorFragmentId())); //Overwrite values from previous 
fragments
+  builder.appendCell(new 
OperatorPathBuilder().setMajor(major).setMinor(minor).build(), attributeMap);
   builder.appendCell(minor.getEndpoint().getAddress());
   builder.appendMillis(minor.getStartTime() - start);
   builder.appendMillis(minor.getEndTime() - start);

http://git-wip-us.apache.org/repos/asf/drill/blob/59c74472/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/profile/OperatorWrapper.java
--
diff --git 
a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/profile/OperatorWrapper.java
 
b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/profile/OperatorWrapper.java
index cca9563..6322435 100644
--- 
a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/profile/OperatorWrapper.java
+++ 
b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/profile/OperatorWrapper.java
@@ -21,6 +21,7 @@ import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 import java.util.Set;
 import java.util.TreeSet;
 
@@ -76,12 +77,14 @@ public class OperatorWrapper {
   public String getContent() {
 TableBuilder builder = new TableBuilder(OPERATOR_COLUMNS, 
OPERATOR_COLUMNS_TOOLTIP, true);
 
+Map attributeMap = new HashMap(); 
//Reusing for different fragments
 for (ImmutablePair, String> ip : 
opsAndHosts) {
   int minor = ip.getLeft().getRight();
   OperatorProfile op = ip.getLeft().getLeft();
 
+  attributeMap.put("data-order", String.valueOf(minor)); //Overwrite 
values from previous fragments
   String path = new 
OperatorPathBuilder().setMajor(major).setMinor(minor).setOperator(op).build();
-  builder.appendCell(path);
+  builder.appendCell(path, attributeMap);
   builder.appendCell(ip.getRight(

[07/11] drill git commit: DRILL-5899: Simple pattern matchers can work with DrillBuf directly

2017-11-13 Thread arina 
DRILL-5899: Simple pattern matchers can work with DrillBuf directly

closes #1015


Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/df95709a
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/df95709a
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/df95709a

Branch: refs/heads/master
Commit: df95709a2dac47e46be59c47dc776efbbe615726
Parents: dfd43d0
Author: Padma Penumarthy 
Authored: Wed Oct 25 13:37:25 2017 -0700
Committer: Arina Ielchiieva 
Committed: Mon Nov 13 11:06:34 2017 +0200

--
 .../expr/fn/impl/AbstractSqlPatternMatcher.java |  66 ++
 .../expr/fn/impl/SqlPatternComplexMatcher.java  |  18 +-
 .../expr/fn/impl/SqlPatternConstantMatcher.java |  29 +-
 .../expr/fn/impl/SqlPatternContainsMatcher.java |  59 +-
 .../expr/fn/impl/SqlPatternEndsWithMatcher.java |  35 +-
 .../exec/expr/fn/impl/SqlPatternFactory.java|  15 +-
 .../exec/expr/fn/impl/SqlPatternMatcher.java|   4 +-
 .../fn/impl/SqlPatternStartsWithMatcher.java|  32 +-
 .../exec/expr/fn/impl/StringFunctions.java  |  16 +-
 .../exec/expr/fn/impl/TestSqlPatterns.java  | 729 +++
 .../exec/expr/fn/impl/TestStringFunctions.java  | 149 
 11 files changed, 574 insertions(+), 578 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/drill/blob/df95709a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/AbstractSqlPatternMatcher.java
--
diff --git 
a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/AbstractSqlPatternMatcher.java
 
b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/AbstractSqlPatternMatcher.java
new file mode 100644
index 000..245d585
--- /dev/null
+++ 
b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/AbstractSqlPatternMatcher.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.expr.fn.impl;
+
+import com.google.common.base.Charsets;
+import org.apache.drill.common.exceptions.UserException;
+import java.nio.ByteBuffer;
+import java.nio.CharBuffer;
+import java.nio.charset.CharacterCodingException;
+import java.nio.charset.CharsetEncoder;
+import static org.apache.drill.exec.expr.fn.impl.StringFunctionHelpers.logger;
+
+/**
+ * To get good performance for most commonly used pattern matches
+ * CONSTANT('ABC') {@link SqlPatternConstantMatcher}
+ * STARTSWITH('%ABC') {@link SqlPatternStartsWithMatcher}
+ * ENDSWITH('ABC%') {@link SqlPatternEndsWithMatcher }
+ * CONTAINS('%ABC%') {@link SqlPatternContainsMatcher}
+ * we have simple pattern matchers.
+ * Idea is to have our own implementation for simple pattern matchers so we can
+ * avoid heavy weight regex processing, skip UTF-8 decoding and char 
conversion.
+ * Instead, we encode the pattern string and do byte comparison against native 
memory.
+ * Overall, this approach
+ * gives us orders of magnitude performance improvement for simple pattern 
matches.
+ * Anything that is not simple is considered
+ * complex pattern and we use Java regex for complex pattern matches.
+ */
+
+public abstract class AbstractSqlPatternMatcher implements SqlPatternMatcher {
+  protected final String patternString;
+  protected final int patternLength;
+  protected final ByteBuffer patternByteBuffer;
+
+  public AbstractSqlPatternMatcher(String patternString) {
+this.patternString = patternString;
+
+final CharsetEncoder charsetEncoder = Charsets.UTF_8.newEncoder();
+final CharBuffer patternCharBuffer = CharBuffer.wrap(patternString);
+
+try {
+  patternByteBuffer = charsetEncoder.encode(patternCharBuffer);
+} catch (CharacterCodingException e) {
+  throw UserException.validationError(e)
+  .message("Failure to encode pattern %s using UTF-8", patternString)
+  .addContext("Message: ", e.getMessage())
+  .build(logger);
+}
+patternLength = patternByteBuffer.limit();
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/drill/blob/df95709a/exec/java-exec/src/main/

[03/11] drill git commit: DRILL-5717: Let some test cases be Local or TimeZone independent.

2017-11-13 Thread arina 
DRILL-5717: Let some test cases be Local or TimeZone independent.

closes #904


Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/17ca6181
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/17ca6181
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/17ca6181

Branch: refs/heads/master
Commit: 17ca6181ba924837d493ae789637b3d4514bd560
Parents: 496c97d
Author: weijie.tong 
Authored: Sun Aug 20 20:07:41 2017 +0800
Committer: Arina Ielchiieva 
Committed: Mon Nov 13 11:05:13 2017 +0200

--
 .../java/org/apache/drill/exec/ExecTest.java|  35 ++-
 .../drill/exec/fn/impl/TestCastFunctions.java   |  22 +-
 .../drill/exec/fn/impl/TestDateFunctions.java   | 226 ++-
 .../exec/fn/impl/TestNewDateFunctions.java  |   8 +-
 .../fn/impl/testing/TestDateConversions.java|  19 +-
 .../exec/fn/interp/TestConstantFolding.java |  17 +-
 .../complex/writer/TestExtendedTypes.java   |  19 +-
 .../expression/fn/JodaDateValidatorTest.java|   7 +-
 8 files changed, 200 insertions(+), 153 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/drill/blob/17ca6181/exec/java-exec/src/test/java/org/apache/drill/exec/ExecTest.java
--
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/ExecTest.java 
b/exec/java-exec/src/test/java/org/apache/drill/exec/ExecTest.java
index 4200f89..418e4a4 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/ExecTest.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/ExecTest.java
@@ -19,11 +19,13 @@ package org.apache.drill.exec;
 
 import com.codahale.metrics.MetricRegistry;
 import com.google.common.io.Files;
+import mockit.Mock;
+import mockit.MockUp;
 import mockit.NonStrictExpectations;
-import org.apache.commons.io.FileUtils;
 import org.antlr.runtime.ANTLRStringStream;
 import org.antlr.runtime.CommonTokenStream;
 import org.antlr.runtime.RecognitionException;
+import org.apache.commons.io.FileUtils;
 import org.apache.drill.common.config.DrillConfig;
 import org.apache.drill.common.expression.LogicalExpression;
 import org.apache.drill.common.expression.parser.ExprLexer;
@@ -35,18 +37,20 @@ import org.apache.drill.exec.metrics.DrillMetrics;
 import org.apache.drill.exec.physical.impl.OperatorCreatorRegistry;
 import org.apache.drill.exec.planner.PhysicalPlanReaderTestFactory;
 import org.apache.drill.exec.server.DrillbitContext;
-import org.apache.drill.exec.server.options.OptionValue;
 import org.apache.drill.exec.server.options.SystemOptionManager;
 import 
org.apache.drill.exec.store.sys.store.provider.LocalPersistentStoreProvider;
 import org.apache.drill.exec.util.GuavaPatcher;
 import org.apache.drill.test.DrillTest;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
+import org.joda.time.DateTimeUtils;
+import org.joda.time.DateTimeZone;
 import org.junit.After;
 import org.junit.BeforeClass;
-
 import java.io.File;
 import java.io.IOException;
+import java.text.DateFormatSymbols;
+import java.util.Locale;
 
 
 public class ExecTest extends DrillTest {
@@ -120,4 +124,29 @@ public class ExecTest extends DrillTest {
 return ret.e;
   }
 
+  /**
+   * This utility is to mock the method DateTimeUtils.getDateFormatSymbols()
+   * to mock the current local as US.
+   */
+  public static void mockUsDateFormatSymbols() {
+new MockUp() {
+  @Mock
+  public DateFormatSymbols getDateFormatSymbols(Locale locale) {
+return new DateFormatSymbols(Locale.US);
+  }
+};
+  }
+
+  /**
+   * This utility is to mock the method DateTimeZone.getDefault() to
+   * mock current timezone as UTC.
+   */
+  public static void mockUtcDateTimeZone() {
+new MockUp() {
+  @Mock
+  public DateTimeZone getDefault() {
+return DateTimeZone.UTC;
+  }
+};
+  }
 }

http://git-wip-us.apache.org/repos/asf/drill/blob/17ca6181/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestCastFunctions.java
--
diff --git 
a/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestCastFunctions.java
 
b/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestCastFunctions.java
index 42a048e..4aeb396 100644
--- 
a/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestCastFunctions.java
+++ 
b/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestCastFunctions.java
@@ -19,6 +19,7 @@ package org.apache.drill.exec.fn.impl;
 
 import com.google.common.collect.Lists;
 import com.google.common.collect.Maps;
+import mockit.integration.junit4.JMockit;
 import org.apache.drill.BaseTestQuery;
 import org.apache.drill.categories.SqlFunctionTest;
 import org.apache.drill.categories.UnlikelyTest;
@@ -26,11 +

[04/11] drill git commit: DRILL-5822: The query with "SELECT *" with "ORDER BY" clause and `planner.slice_target`=1 doesn't preserve column order

2017-11-13 Thread arina 
DRILL-5822: The query with "SELECT *" with "ORDER BY" clause and 
`planner.slice_target`=1 doesn't preserve column order

- The commit for DRILL-847 is oudated. There is no need to canonicalize the 
batch or container since RecordBatchLoader
  swallows the "schema change" for now if two batches have different column 
ordering.

  closes #1017


Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/c1118a3d
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/c1118a3d
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/c1118a3d

Branch: refs/heads/master
Commit: c1118a3d9a74cf24f28bc69efca2c21d2a6d5b1d
Parents: 17ca618
Author: Vitalii Diravka 
Authored: Thu Oct 26 18:07:33 2017 +
Committer: Arina Ielchiieva 
Committed: Mon Nov 13 11:05:29 2017 +0200

--
 .../exec/physical/impl/TopN/TopNBatch.java  | 11 +++--
 .../impl/mergereceiver/MergingRecordBatch.java  |  7 +-
 .../physical/impl/sort/RecordBatchData.java |  8 +--
 .../impl/sort/SortRecordBatchBuilder.java   |  6 -
 .../drill/exec/record/RecordBatchLoader.java| 18 --
 .../drill/exec/record/VectorContainer.java  | 25 
 .../java/org/apache/drill/TestStarQueries.java  | 17 +
 7 files changed, 22 insertions(+), 70 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/drill/blob/c1118a3d/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/TopN/TopNBatch.java
--
diff --git 
a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/TopN/TopNBatch.java
 
b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/TopN/TopNBatch.java
index 950e1fe..dcf67d4 100644
--- 
a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/TopN/TopNBatch.java
+++ 
b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/TopN/TopNBatch.java
@@ -238,7 +238,6 @@ public class TopNBatch extends AbstractRecordBatch {
   }
   boolean success = false;
   try {
-batch.canonicalize();
 if (priorityQueue == null) {
   assert !schemaChanged;
   priorityQueue = createNewPriorityQueue(context, 
config.getOrderings(), new ExpandableHyperContainer(batch.getContainer()), 
MAIN_MAPPING, LEFT_MAPPING, RIGHT_MAPPING);
@@ -323,7 +322,6 @@ public class TopNBatch extends AbstractRecordBatch {
   selectionVector4.clear();
   c.clear();
   VectorContainer newQueue = new VectorContainer();
-  builder.canonicalize();
   builder.build(context, newQueue);
   priorityQueue.resetQueue(newQueue, 
builder.getSv4().createNewWrapperCurrent());
   builder.getSv4().clear();
@@ -414,16 +412,13 @@ public class TopNBatch extends AbstractRecordBatch {
   selectionVector4.clear();
   c.clear();
   final VectorContainer oldSchemaContainer = new VectorContainer(oContext);
-  builder.canonicalize();
   builder.build(context, oldSchemaContainer);
   oldSchemaContainer.setRecordCount(builder.getSv4().getCount());
   final VectorContainer newSchemaContainer =  
SchemaUtil.coerceContainer(oldSchemaContainer, this.schema, oContext);
-  // Canonicalize new container since we canonicalize incoming batches 
before adding to queue.
-  final VectorContainer canonicalizedContainer = 
VectorContainer.canonicalize(newSchemaContainer);
-  canonicalizedContainer.buildSchema(SelectionVectorMode.FOUR_BYTE);
+  newSchemaContainer.buildSchema(SelectionVectorMode.FOUR_BYTE);
   priorityQueue.cleanup();
-  priorityQueue = createNewPriorityQueue(context, config.getOrderings(), 
canonicalizedContainer, MAIN_MAPPING, LEFT_MAPPING, RIGHT_MAPPING);
-  priorityQueue.resetQueue(canonicalizedContainer, 
builder.getSv4().createNewWrapperCurrent());
+  priorityQueue = createNewPriorityQueue(context, config.getOrderings(), 
newSchemaContainer, MAIN_MAPPING, LEFT_MAPPING, RIGHT_MAPPING);
+  priorityQueue.resetQueue(newSchemaContainer, 
builder.getSv4().createNewWrapperCurrent());
 } finally {
   builder.clear();
   builder.close();

http://git-wip-us.apache.org/repos/asf/drill/blob/c1118a3d/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/mergereceiver/MergingRecordBatch.java
--
diff --git 
a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/mergereceiver/MergingRecordBatch.java
 
b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/mergereceiver/MergingRecordBatch.java
index eff1ae9..ec945d6 100644
--- 
a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/mergereceiver/MergingRecordBatch.java
+++ 
b/exec/java-exec/src/main/java/org/apache/drill/exec/phy

[06/11] drill git commit: DRILL-5896: Handle HBase columns vector creation in the HBaseRecordReader

2017-11-13 Thread arina 
DRILL-5896: Handle HBase columns vector creation in the HBaseRecordReader

closes #1005


Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/dfd43d02
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/dfd43d02
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/dfd43d02

Branch: refs/heads/master
Commit: dfd43d020498c09dcb2c3fed4e8c6df23d755d55
Parents: 59c7447
Author: Prasad Nagaraj Subramanya 
Authored: Wed Oct 25 21:20:07 2017 -0700
Committer: Arina Ielchiieva 
Committed: Mon Nov 13 11:06:20 2017 +0200

--
 .../exec/store/hbase/HBaseRecordReader.java | 34 +---
 1 file changed, 23 insertions(+), 11 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/drill/blob/dfd43d02/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseRecordReader.java
--
diff --git 
a/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseRecordReader.java
 
b/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseRecordReader.java
index 4e822df..631c44d 100644
--- 
a/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseRecordReader.java
+++ 
b/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseRecordReader.java
@@ -75,6 +75,9 @@ public class HBaseRecordReader extends AbstractRecordReader 
implements DrillHBas
 
   private TableName hbaseTableName;
   private Scan hbaseScan;
+  // scan instance to capture columns for vector creation
+  private Scan hbaseScanColumnsOnly;
+  private Set completeFamilies;
   private OperatorContext operatorContext;
 
   private boolean rowKeyOnly;
@@ -87,6 +90,7 @@ public class HBaseRecordReader extends AbstractRecordReader 
implements DrillHBas
 hbaseTableName = TableName.valueOf(
 Preconditions.checkNotNull(subScanSpec, "HBase reader needs a sub-scan 
spec").getTableName());
 hbaseScan = new Scan(subScanSpec.getStartRow(), subScanSpec.getStopRow());
+hbaseScanColumnsOnly = new Scan();
 hbaseScan
 .setFilter(subScanSpec.getScanFilter())
 .setCaching(TARGET_RECORD_COUNT);
@@ -108,7 +112,8 @@ public class HBaseRecordReader extends AbstractRecordReader 
implements DrillHBas
   @Override
   protected Collection transformColumns(Collection 
columns) {
 Set transformed = Sets.newLinkedHashSet();
-Set completeFamilies = Sets.newHashSet();
+completeFamilies = Sets.newHashSet();
+
 rowKeyOnly = true;
 if (!isStarQuery()) {
   for (SchemaPath column : columns) {
@@ -121,16 +126,18 @@ public class HBaseRecordReader extends 
AbstractRecordReader implements DrillHBas
 byte[] family = root.getPath().getBytes();
 transformed.add(SchemaPath.getSimplePath(root.getPath()));
 PathSegment child = root.getChild();
-if (!completeFamilies.contains(new String(family, 
StandardCharsets.UTF_8).toLowerCase())) {
-  if (child != null && child.isNamed()) {
-byte[] qualifier = child.getNameSegment().getPath().getBytes();
+if (child != null && child.isNamed()) {
+  byte[] qualifier = child.getNameSegment().getPath().getBytes();
+  hbaseScanColumnsOnly.addColumn(family, qualifier);
+  if (!completeFamilies.contains(root.getPath())) {
 hbaseScan.addColumn(family, qualifier);
-  } else {
-hbaseScan.addFamily(family);
-completeFamilies.add(new String(family, 
StandardCharsets.UTF_8).toLowerCase());
   }
+} else {
+  hbaseScan.addFamily(family);
+  completeFamilies.add(root.getPath());
 }
   }
+
   /* if only the row key was requested, add a FirstKeyOnlyFilter to the 
scan
* to fetch only one KV from each row. If a filter is already part of 
this
* scan, add the FirstKeyOnlyFilter as the LAST filter of a MUST_PASS_ALL
@@ -168,11 +175,10 @@ public class HBaseRecordReader extends 
AbstractRecordReader implements DrillHBas
 }
   }
 
-  // Add map and child vectors for any HBase column families and/or HBase
-  // columns that are requested (in order to avoid later creation of dummy
-  // NullableIntVectors for them).
+  // Add map and child vectors for any HBase columns that are requested (in
+  // order to avoid later creation of dummy NullableIntVectors for them).
   final Set>> familiesEntries =
-  hbaseScan.getFamilyMap().entrySet();
+  hbaseScanColumnsOnly.getFamilyMap().entrySet();
   for (Map.Entry> familyEntry : 
familiesEntries) {
 final String familyName = new String(familyEntry.getKey(),
  StandardCharsets.UTF_8);
@@ -186,6 +192,12 @@ public class HBaseRecordReader ex

[02/11] drill git commit: DRILL-5337: OpenTSDB storage plugin

2017-11-13 Thread arina 
DRILL-5337: OpenTSDB storage plugin

closes #774


Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/496c97d1
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/496c97d1
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/496c97d1

Branch: refs/heads/master
Commit: 496c97d14eb428a5aff74e82d662a0da6930e94f
Parents: 29e0547
Author: Vlad Storona 
Authored: Fri Nov 25 20:28:02 2016 +0200
Committer: Arina Ielchiieva 
Committed: Mon Nov 13 11:04:54 2017 +0200

--
 contrib/pom.xml |   1 +
 contrib/storage-opentsdb/README.md  |  69 +
 contrib/storage-opentsdb/pom.xml|  80 ++
 .../drill/exec/store/openTSDB/Constants.java|  32 +++
 .../exec/store/openTSDB/DrillOpenTSDBTable.java |  81 ++
 .../store/openTSDB/OpenTSDBBatchCreator.java|  53 
 .../exec/store/openTSDB/OpenTSDBGroupScan.java  | 169 
 .../store/openTSDB/OpenTSDBRecordReader.java| 258 +++
 .../exec/store/openTSDB/OpenTSDBScanSpec.java   |  42 +++
 .../store/openTSDB/OpenTSDBStoragePlugin.java   |  77 ++
 .../openTSDB/OpenTSDBStoragePluginConfig.java   |  77 ++
 .../exec/store/openTSDB/OpenTSDBSubScan.java| 132 ++
 .../apache/drill/exec/store/openTSDB/Util.java  |  66 +
 .../exec/store/openTSDB/client/OpenTSDB.java|  50 
 .../store/openTSDB/client/OpenTSDBTypes.java|  28 ++
 .../exec/store/openTSDB/client/Schema.java  | 124 +
 .../exec/store/openTSDB/client/Service.java |  55 
 .../store/openTSDB/client/query/DBQuery.java| 148 +++
 .../exec/store/openTSDB/client/query/Query.java | 187 ++
 .../openTSDB/client/services/ServiceImpl.java   | 174 +
 .../exec/store/openTSDB/dto/ColumnDTO.java  |  63 +
 .../exec/store/openTSDB/dto/MetricDTO.java  |  77 ++
 .../openTSDB/schema/OpenTSDBSchemaFactory.java  |  77 ++
 .../resources/bootstrap-storage-plugins.json|   9 +
 .../src/main/resources/drill-module.conf|  21 ++
 .../drill/store/openTSDB/TestDataHolder.java| 247 ++
 .../store/openTSDB/TestOpenTSDBPlugin.java  | 189 ++
 distribution/pom.xml|   5 +
 28 files changed, 2591 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/drill/blob/496c97d1/contrib/pom.xml
--
diff --git a/contrib/pom.xml b/contrib/pom.xml
index 2014923..d4ad434 100644
--- a/contrib/pom.xml
+++ b/contrib/pom.xml
@@ -38,6 +38,7 @@
 storage-mongo
 storage-jdbc
 storage-kudu
+storage-opentsdb
 sqlline
 data
 gis

http://git-wip-us.apache.org/repos/asf/drill/blob/496c97d1/contrib/storage-opentsdb/README.md
--
diff --git a/contrib/storage-opentsdb/README.md 
b/contrib/storage-opentsdb/README.md
new file mode 100644
index 000..0c616b5
--- /dev/null
+++ b/contrib/storage-opentsdb/README.md
@@ -0,0 +1,69 @@
+# drill-storage-openTSDB
+
+Implementation of TSDB storage plugin. Plugin uses REST API to work with TSDB. 
+
+For more information about openTSDB follow this link 
+
+There is list of required params:
+
+* metric - The name of a metric stored in the db.
+
+* start  - The start time for the query. This can be a relative or 
absolute timestamp.
+
+* aggregator - The name of an aggregation function to use.
+
+optional param is: 
+
+* downsample - An optional downsampling function to reduce the amount of data 
returned.
+
+* end - An end time for the query. If not supplied, the TSD will assume the 
local system time on the server. 
+This may be a relative or absolute timestamp. This param is optional, and if 
it isn't specified we will send null
+to the db in this field, but in this case db will assume the local system time 
on the server.
+
+List of supported aggregators
+
+
+
+List of supported time 
+
+
+
+Params must be specified in FROM clause of the query separated by commas. For 
example
+
+`openTSDB.(metric=metric_name, start=4d-ago, aggregator=sum)`
+
+Supported queries for now are listed below:
+
+```
+USE openTSDB
+```
+
+```
+SHOW tables
+```
+Will print available metrics. Max number of the printed results is a 
Integer.MAX value
+
+```
+SELECT * FROM openTSDB. `(metric=warp.speed.test, start=47y-ago, 
aggregator=sum)` 
+```
+Return aggregated elements from `warp.speed.test` table since 47y-ago 
+
+```
+SELECT * FROM openTSDB.`(metric=warp.speed.test, aggregator=avg, 
start=47y-ago)`
+```
+Return aggregated elements from `warp.speed.test` table
+
+```
+

[11/11] drill git commit: DRILL-5795: Parquet Filter push down now work at rowgroup level

2017-11-13 Thread arina 
DRILL-5795: Parquet Filter push down now work at rowgroup level

Before this commit, the filter was pruning complete files. When a file
is composed of multiple rowgroups, it was not able to prune one
rowgroup from the file. Now, when the filter find that a rowgroup
doesn't match it will be remove from the scan.

closes #949


Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/3036d370
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/3036d370
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/3036d370

Branch: refs/heads/master
Commit: 3036d3700aa620bbbffc260e52f633cdaae1172c
Parents: 30da051
Author: Damien Profeta 
Authored: Fri Sep 15 11:01:58 2017 -0700
Committer: Arina Ielchiieva 
Committed: Mon Nov 13 11:45:21 2017 +0200

--
 .../exec/store/parquet/ParquetGroupScan.java| 171 ++-
 .../parquet/TestParquetFilterPushDown.java  |  10 ++
 .../resources/parquet/multirowgroup.parquet | Bin 0 -> 398 bytes
 3 files changed, 103 insertions(+), 78 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/drill/blob/3036d370/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetGroupScan.java
--
diff --git 
a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetGroupScan.java
 
b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetGroupScan.java
index 4e38ce9..972332c 100644
--- 
a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetGroupScan.java
+++ 
b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetGroupScan.java
@@ -738,7 +738,7 @@ public class ParquetGroupScan extends AbstractFileGroupScan 
{
 
 private EndpointByteMap byteMap;
 private int rowGroupIndex;
-private String root;
+private List columns;
 private long rowCount;  // rowCount = -1 indicates to include all rows.
 private long numRecordsToRead;
 
@@ -791,6 +791,14 @@ public class ParquetGroupScan extends 
AbstractFileGroupScan {
   return rowCount;
 }
 
+public List getColumns() {
+  return columns;
+}
+
+public void setColumns(List columns) {
+  this.columns = columns;
+}
+
   }
 
   /**
@@ -962,69 +970,70 @@ public class ParquetGroupScan extends 
AbstractFileGroupScan {
   }
 }
 rowGroupInfo.setEndpointByteMap(endpointByteMap);
+rowGroupInfo.setColumns(rg.getColumns());
 rgIndex++;
 rowGroupInfos.add(rowGroupInfo);
   }
 }
 
 this.endpointAffinities = AffinityCreator.getAffinityMap(rowGroupInfos);
+updatePartitionColTypeMap();
+  }
 
+  private void updatePartitionColTypeMap() {
 columnValueCounts = Maps.newHashMap();
 this.rowCount = 0;
 boolean first = true;
-for (ParquetFileMetadata file : parquetTableMetadata.getFiles()) {
-  for (RowGroupMetadata rowGroup : file.getRowGroups()) {
-long rowCount = rowGroup.getRowCount();
-for (ColumnMetadata column : rowGroup.getColumns()) {
-  SchemaPath schemaPath = SchemaPath.getCompoundPath(column.getName());
-  Long previousCount = columnValueCounts.get(schemaPath);
-  if (previousCount != null) {
-if (previousCount != GroupScan.NO_COLUMN_STATS) {
-  if (column.getNulls() != null) {
-Long newCount = rowCount - column.getNulls();
-columnValueCounts.put(schemaPath, 
columnValueCounts.get(schemaPath) + newCount);
-  }
-}
-  } else {
+for (RowGroupInfo rowGroup : this.rowGroupInfos) {
+  long rowCount = rowGroup.getRowCount();
+  for (ColumnMetadata column : rowGroup.getColumns()) {
+SchemaPath schemaPath = SchemaPath.getCompoundPath(column.getName());
+Long previousCount = columnValueCounts.get(schemaPath);
+if (previousCount != null) {
+  if (previousCount != GroupScan.NO_COLUMN_STATS) {
 if (column.getNulls() != null) {
   Long newCount = rowCount - column.getNulls();
-  columnValueCounts.put(schemaPath, newCount);
-} else {
-  columnValueCounts.put(schemaPath, GroupScan.NO_COLUMN_STATS);
+  columnValueCounts.put(schemaPath, 
columnValueCounts.get(schemaPath) + newCount);
 }
   }
-  boolean partitionColumn = checkForPartitionColumn(column, first, 
rowCount);
-  if (partitionColumn) {
-Map map = 
partitionValueMap.get(file.getPath());
-if (map == null) {
-  map = Maps.newHashMap();
-  partitionValueMap.put(file.getPath(), map);
+} else {
+  if (column.getNulls() != null) {
+Long newCount = rowCount - column.getNulls();

[1/2] drill git commit: DRILL-5832: Change OperatorFixture to use system option manager

2017-11-13 Thread arina 
Repository: drill
Updated Branches:
  refs/heads/master 3036d3700 -> 42fc11e53


http://git-wip-us.apache.org/repos/asf/drill/blob/42fc11e5/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/xsort/TestSortSpillWithException.java
--
diff --git 
a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/xsort/TestSortSpillWithException.java
 
b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/xsort/TestSortSpillWithException.java
index f6b3a8d..2694f64 100644
--- 
a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/xsort/TestSortSpillWithException.java
+++ 
b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/xsort/TestSortSpillWithException.java
@@ -32,7 +32,7 @@ import org.apache.drill.exec.testing.Controls;
 import org.apache.drill.exec.testing.ControlsInjectionUtil;
 import org.apache.drill.test.ClusterFixture;
 import org.apache.drill.test.ClusterTest;
-import org.apache.drill.test.FixtureBuilder;
+import org.apache.drill.test.ClusterFixtureBuilder;
 import org.junit.BeforeClass;
 import org.junit.Test;
 import org.junit.experimental.categories.Category;
@@ -51,7 +51,7 @@ public class TestSortSpillWithException extends ClusterTest {
 
   @BeforeClass
   public static void setup() throws Exception {
-FixtureBuilder builder = ClusterFixture.builder()
+ClusterFixtureBuilder builder = ClusterFixture.builder()
 .configProperty(ExecConstants.EXTERNAL_SORT_SPILL_THRESHOLD, 1) // 
Unmanaged
 .configProperty(ExecConstants.EXTERNAL_SORT_SPILL_GROUP_SIZE, 1) // 
Unmanaged
 .sessionOption(ExecConstants.MAX_QUERY_MEMORY_PER_NODE_KEY, 60 * 1024 
* 1024) // Spill early

http://git-wip-us.apache.org/repos/asf/drill/blob/42fc11e5/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/xsort/managed/SortTestUtilities.java
--
diff --git 
a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/xsort/managed/SortTestUtilities.java
 
b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/xsort/managed/SortTestUtilities.java
index f525b2a..4c90769 100644
--- 
a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/xsort/managed/SortTestUtilities.java
+++ 
b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/xsort/managed/SortTestUtilities.java
@@ -62,6 +62,7 @@ public class SortTestUtilities {
 return makeSchema(MinorType.INT, true);
   }
 
+  @SuppressWarnings("resource")
   public static PriorityQueueCopierWrapper makeCopier(OperatorFixture fixture, 
String sortOrder, String nullOrder) {
 FieldReference expr = FieldReference.getWithQuotedRef("key");
 Ordering ordering = new Ordering(sortOrder, expr, nullOrder);

http://git-wip-us.apache.org/repos/asf/drill/blob/42fc11e5/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/xsort/managed/TestShortArrays.java
--
diff --git 
a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/xsort/managed/TestShortArrays.java
 
b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/xsort/managed/TestShortArrays.java
index 1c9d49e..ba5dfce 100644
--- 
a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/xsort/managed/TestShortArrays.java
+++ 
b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/xsort/managed/TestShortArrays.java
@@ -83,7 +83,7 @@ public class TestShortArrays extends SubOperatorTest {
 assertEquals(bHint.elementCount, bCol.estElementCountPerArray, 0.001);
 
 // Create a new batch, and new vector, using the sizer and
-// intializer inferred from the previoius batch.
+// initializer inferred from the previous batch.
 
 SingleRowSet empty = fixture.rowSet(schema);
 vi.allocateBatch(empty.container(), 100);

http://git-wip-us.apache.org/repos/asf/drill/blob/42fc11e5/exec/java-exec/src/test/java/org/apache/drill/exec/server/TestDrillbitResilience.java
--
diff --git 
a/exec/java-exec/src/test/java/org/apache/drill/exec/server/TestDrillbitResilience.java
 
b/exec/java-exec/src/test/java/org/apache/drill/exec/server/TestDrillbitResilience.java
index bde6764..1548d7d 100644
--- 
a/exec/java-exec/src/test/java/org/apache/drill/exec/server/TestDrillbitResilience.java
+++ 
b/exec/java-exec/src/test/java/org/apache/drill/exec/server/TestDrillbitResilience.java
@@ -34,8 +34,8 @@ import org.apache.commons.math3.util.Pair;
 import org.apache.drill.BaseTestQuery;
 import org.apache.drill.QueryTestUtil;
 import org.apache.drill.SingleRowListener;
-import org.apache.drill.common.concurrent.ExtendedLatch;
 import org.apache.drill.common.DrillAutoCloseables;
+import org.apache.drill.common.concurrent.ExtendedLatch;
 import org.apache.drill.common.config.DrillConfig;
 import org.

[2/2] drill git commit: DRILL-5832: Change OperatorFixture to use system option manager

2017-11-13 Thread arina 
DRILL-5832: Change OperatorFixture to use system option manager

- Rename FixtureBuilder to ClusterFixtureBuilder
- Provide alternative way to reset system/session options
- Fix for DRILL-5833: random failure in TestParquetWriter
- Provide strict, but clear, errors for missing options

closes #970


Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/42fc11e5
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/42fc11e5
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/42fc11e5

Branch: refs/heads/master
Commit: 42fc11e53557477ac01c7dd31c3aa93e22fb4384
Parents: 3036d37
Author: Paul Rogers 
Authored: Mon Oct 2 15:45:21 2017 -0700
Committer: Arina Ielchiieva 
Committed: Mon Nov 13 14:59:19 2017 +0200

--
 .../drill/common/util/CoreDecimalUtility.java   |   9 +-
 .../apache/drill/exec/hive/TestHiveStorage.java |   7 +-
 .../drill/exec/ops/BufferManagerImpl.java   |   5 +-
 .../drill/exec/physical/impl/ImplCreator.java   |   3 +
 .../physical/impl/MergingReceiverCreator.java   |   5 +-
 .../drill/exec/physical/impl/ScanBatch.java |  24 +-
 .../drill/exec/physical/impl/ScreenCreator.java |   6 +-
 .../drill/exec/physical/impl/TraceInjector.java |   6 +-
 .../exec/physical/impl/WriterRecordBatch.java   |   2 +
 .../exec/server/options/BaseOptionManager.java  |  76 -
 .../exec/server/options/BaseOptionSet.java  |  57 
 .../drill/exec/server/options/OptionSet.java|  44 +++
 .../exec/server/options/OptionValidator.java|   1 -
 .../drill/exec/server/options/OptionValue.java  |   4 +
 .../server/options/PersistedOptionValue.java|   1 +
 .../server/options/SystemOptionManager.java |  26 +-
 .../drill/exec/store/dfs/FileSelection.java |  12 -
 .../drill/exec/store/dfs/FileSystemConfig.java  |   3 +-
 .../exec/store/dfs/FileSystemSchemaFactory.java |   3 +-
 .../drill/exec/store/dfs/ReadEntryFromHDFS.java |   4 +-
 .../store/parquet/ParquetScanBatchCreator.java  |   3 +-
 .../parquet/columnreaders/ParquetSchema.java|   2 +-
 .../parquet2/DrillParquetGroupConverter.java|  33 +-
 .../exec/store/parquet2/DrillParquetReader.java |  39 +--
 .../DrillParquetRecordMaterializer.java |   6 +-
 .../store/provider/InMemoryStoreProvider.java   |  11 +-
 .../java/org/apache/drill/BaseTestQuery.java|  35 +-
 .../physical/impl/agg/TestHashAggrSpill.java|   4 +-
 .../impl/limit/TestLimitWithExchanges.java  |   9 +-
 .../impl/validate/TestValidationOptions.java|   8 +-
 .../physical/impl/writer/TestParquetWriter.java | 113 ---
 .../writer/TestParquetWriterEmptyFiles.java |   3 +-
 .../impl/xsort/TestSimpleExternalSort.java  |  10 +-
 .../impl/xsort/TestSortSpillWithException.java  |   4 +-
 .../impl/xsort/managed/SortTestUtilities.java   |   1 +
 .../impl/xsort/managed/TestShortArrays.java |   2 +-
 .../exec/server/TestDrillbitResilience.java |  33 +-
 .../exec/server/rest/StatusResourcesTest.java   |   6 +-
 .../store/parquet/ParquetInternalsTest.java |   4 +-
 .../parquet/TestParquetFilterPushDown.java  |   8 +-
 .../exec/store/sys/TestPStoreProviders.java |   4 +-
 .../drill/exec/util/TestQueryMemoryAlloc.java   |  18 +-
 .../complex/writer/TestExtendedTypes.java   |  18 +-
 .../org/apache/drill/test/ClientFixture.java|  10 +-
 .../org/apache/drill/test/ClusterFixture.java   |  26 +-
 .../drill/test/ClusterFixtureBuilder.java   | 326 +++
 .../java/org/apache/drill/test/ClusterTest.java |   2 +-
 .../java/org/apache/drill/test/ExampleTest.java |   6 +-
 .../org/apache/drill/test/FixtureBuilder.java   | 326 ---
 .../org/apache/drill/test/OperatorFixture.java  | 114 ++-
 .../apache/drill/test/TestConfigLinkage.java|  34 +-
 .../org/apache/drill/test/package-info.java |   2 +-
 52 files changed, 761 insertions(+), 757 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/drill/blob/42fc11e5/common/src/main/java/org/apache/drill/common/util/CoreDecimalUtility.java
--
diff --git 
a/common/src/main/java/org/apache/drill/common/util/CoreDecimalUtility.java 
b/common/src/main/java/org/apache/drill/common/util/CoreDecimalUtility.java
index 9f61ccf..bf12cfb 100644
--- a/common/src/main/java/org/apache/drill/common/util/CoreDecimalUtility.java
+++ b/common/src/main/java/org/apache/drill/common/util/CoreDecimalUtility.java
@@ -82,10 +82,9 @@ public class CoreDecimalUtility {
   }
 
   public static boolean isDecimalType(TypeProtos.MinorType minorType) {
-if (minorType == TypeProtos.MinorType.DECIMAL9 || minorType == 
TypeProtos.MinorType.DECIMAL18 ||
-minorType == TypeProtos.MinorType.DECIMAL28SPARSE || minorType == 
TypeProtos.MinorType.DECIMAL38SPARSE) {
-  return true;
-}
-return false

[1/2] drill git commit: DRILL-5842: Refactor fragment, operator contexts

2017-11-13 Thread progers 
Repository: drill
Updated Branches:
  refs/heads/master 42fc11e53 -> c56de2f13


http://git-wip-us.apache.org/repos/asf/drill/blob/c56de2f1/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/xsort/managed/SorterWrapper.java
--
diff --git 
a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/xsort/managed/SorterWrapper.java
 
b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/xsort/managed/SorterWrapper.java
index 0f27884..e7a78ed 100644
--- 
a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/xsort/managed/SorterWrapper.java
+++ 
b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/xsort/managed/SorterWrapper.java
@@ -21,7 +21,7 @@ import org.apache.drill.common.exceptions.UserException;
 import org.apache.drill.exec.exception.SchemaChangeException;
 import org.apache.drill.exec.expr.ClassGenerator;
 import org.apache.drill.exec.expr.CodeGenerator;
-import org.apache.drill.exec.ops.OperExecContext;
+import org.apache.drill.exec.ops.OperatorContext;
 import org.apache.drill.exec.physical.config.ExternalSort;
 import org.apache.drill.exec.physical.impl.xsort.SingleBatchSorter;
 import org.apache.drill.exec.record.VectorAccessible;
@@ -47,7 +47,7 @@ public class SorterWrapper extends BaseSortWrapper {
 
   private SingleBatchSorter sorter;
 
-  public SorterWrapper(OperExecContext opContext) {
+  public SorterWrapper(OperatorContext opContext) {
 super(opContext);
   }
 
@@ -55,7 +55,7 @@ public class SorterWrapper extends BaseSortWrapper {
 
 SingleBatchSorter sorter = getSorter(convertedBatch);
 try {
-  sorter.setup(context, sv2, convertedBatch);
+  sorter.setup(context.getFragmentContext(), sv2, convertedBatch);
   sorter.sort(sv2);
 } catch (SchemaChangeException e) {
   convertedBatch.clear();
@@ -78,8 +78,8 @@ public class SorterWrapper extends BaseSortWrapper {
 
   private SingleBatchSorter newSorter(VectorAccessible batch) {
 CodeGenerator cg = CodeGenerator.get(
-SingleBatchSorter.TEMPLATE_DEFINITION, context.getFunctionRegistry(),
-context.getOptionSet());
+SingleBatchSorter.TEMPLATE_DEFINITION, 
context.getFragmentContext().getFunctionRegistry(),
+context.getFragmentContext().getOptionSet());
 ClassGenerator g = cg.getRoot();
 cg.plainJavaCapable(true);
 // Uncomment out this line to debug the generated code.

http://git-wip-us.apache.org/repos/asf/drill/blob/c56de2f1/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/xsort/managed/SpilledRuns.java
--
diff --git 
a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/xsort/managed/SpilledRuns.java
 
b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/xsort/managed/SpilledRuns.java
index b75ce77..3d7e63a 100644
--- 
a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/xsort/managed/SpilledRuns.java
+++ 
b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/xsort/managed/SpilledRuns.java
@@ -23,13 +23,13 @@ import java.util.List;
 
 import org.apache.drill.common.AutoCloseables;
 import org.apache.drill.common.exceptions.UserException;
-import org.apache.drill.exec.ops.OperExecContext;
+import org.apache.drill.exec.ops.OperatorContext;
 import org.apache.drill.exec.physical.impl.spill.SpillSet;
 import org.apache.drill.exec.physical.impl.xsort.managed.BatchGroup.SpilledRun;
 import org.apache.drill.exec.physical.impl.xsort.managed.SortImpl.SortResults;
 import org.apache.drill.exec.record.BatchSchema;
-import org.apache.drill.exec.record.VectorInitializer;
 import org.apache.drill.exec.record.VectorContainer;
+import org.apache.drill.exec.record.VectorInitializer;
 
 import com.google.common.collect.Lists;
 
@@ -56,9 +56,9 @@ public class SpilledRuns {
   private final PriorityQueueCopierWrapper copierHolder;
   private BatchSchema schema;
 
-  private final OperExecContext context;
+  private final OperatorContext context;
 
-  public SpilledRuns(OperExecContext opContext, SpillSet spillSet, 
PriorityQueueCopierWrapper copier) {
+  public SpilledRuns(OperatorContext opContext, SpillSet spillSet, 
PriorityQueueCopierWrapper copier) {
 this.context = opContext;
 this.spillSet = spillSet;
 //copierHolder = new PriorityQueueCopierWrapper(opContext);

http://git-wip-us.apache.org/repos/asf/drill/blob/c56de2f1/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/DrillFSDataInputStream.java
--
diff --git 
a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/DrillFSDataInputStream.java
 
b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/DrillFSDataInputStream.java
index 5e7be8d..489e03c 100644
--- 
a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/DrillFSDataInputStream.java
+++ 
b/e

[2/2] drill git commit: DRILL-5842: Refactor fragment, operator contexts

2017-11-13 Thread progers 
DRILL-5842: Refactor fragment, operator contexts

This closes #978


Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/c56de2f1
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/c56de2f1
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/c56de2f1

Branch: refs/heads/master
Commit: c56de2f13a36d673f0c5836d44817137e64b91e4
Parents: 42fc11e
Author: Paul Rogers 
Authored: Wed Oct 4 22:43:44 2017 -0700
Committer: Paul Rogers 
Committed: Mon Nov 13 14:55:39 2017 -0800

--
 .../exec/ops/AbstractOperatorExecContext.java   |  95 -
 .../drill/exec/ops/AccountingDataTunnel.java|   3 +-
 .../drill/exec/ops/BaseFragmentContext.java |  90 +
 .../drill/exec/ops/BaseOperatorContext.java | 196 +++
 .../apache/drill/exec/ops/FragmentContext.java  |  81 +++-
 .../exec/ops/FragmentContextInterface.java  | 149 ++
 .../drill/exec/ops/FragmentExecContext.java | 131 -
 .../apache/drill/exec/ops/OperExecContext.java  |  90 -
 .../drill/exec/ops/OperExecContextImpl.java | 146 --
 .../apache/drill/exec/ops/OperatorContext.java  |  97 -
 .../drill/exec/ops/OperatorContextImpl.java |  76 ++-
 .../drill/exec/ops/OperatorExecContext.java |  46 -
 .../drill/exec/ops/OperatorStatReceiver.java|   4 +
 .../apache/drill/exec/ops/OperatorStats.java|   2 +
 .../drill/exec/physical/impl/ScanBatch.java |   5 +-
 .../physical/impl/xsort/SingleBatchSorter.java  |   4 +-
 .../impl/xsort/SingleBatchSorterTemplate.java   |   6 +-
 .../impl/xsort/managed/BaseSortWrapper.java |   9 +-
 .../impl/xsort/managed/BaseWrapper.java |   8 +-
 .../impl/xsort/managed/BufferedBatches.java |   6 +-
 .../impl/xsort/managed/ExternalSortBatch.java   |  10 +-
 .../impl/xsort/managed/MSortTemplate.java   |   8 +-
 .../physical/impl/xsort/managed/MSorter.java|   4 +-
 .../impl/xsort/managed/MergeSortWrapper.java|  15 +-
 .../managed/PriorityQueueCopierWrapper.java |  10 +-
 .../physical/impl/xsort/managed/SortImpl.java   |  10 +-
 .../impl/xsort/managed/SortMetrics.java |   1 +
 .../impl/xsort/managed/SorterWrapper.java   |  10 +-
 .../impl/xsort/managed/SpilledRuns.java |   8 +-
 .../exec/store/dfs/DrillFSDataInputStream.java  |  11 +-
 .../drill/exec/store/dfs/DrillFileSystem.java   |   6 +-
 .../impl/xsort/managed/SortTestUtilities.java   |   8 +-
 .../impl/xsort/managed/TestSortImpl.java|   8 +-
 .../physical/impl/xsort/managed/TestSorter.java |   6 +-
 .../physical/unit/BasicPhysicalOpUnitTest.java  |   3 +
 .../physical/unit/MiniPlanUnitTestBase.java |  36 ++--
 .../physical/unit/PhysicalOpUnitTestBase.java   |  14 +-
 .../org/apache/drill/test/OperatorFixture.java  | 124 +++-
 38 files changed, 768 insertions(+), 768 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/drill/blob/c56de2f1/exec/java-exec/src/main/java/org/apache/drill/exec/ops/AbstractOperatorExecContext.java
--
diff --git 
a/exec/java-exec/src/main/java/org/apache/drill/exec/ops/AbstractOperatorExecContext.java
 
b/exec/java-exec/src/main/java/org/apache/drill/exec/ops/AbstractOperatorExecContext.java
deleted file mode 100644
index ebef55c..000
--- 
a/exec/java-exec/src/main/java/org/apache/drill/exec/ops/AbstractOperatorExecContext.java
+++ /dev/null
@@ -1,95 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.ops;
-
-import org.apache.drill.exec.memory.BufferAllocator;
-import org.apache.drill.exec.physical.base.PhysicalOperator;
-import org.apache.drill.exec.testing.ExecutionControls;
-
-import io.netty.buffer.DrillBuf;
-
-/**
- * Implementation of {@link OperatorExecContext} that provides services
- * needed by most run-time operators. Excludes services that need the
- * entire Drillbit. Allows easy testing of operator code that uses this
- * interface.
- */
-
-public class AbstractOperatorExecContext implem