[drill] 03/03: DRILL-7302: Bump Apache Avro to 1.9.0
This is an automated email from the ASF dual-hosted git repository. amansinha pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/drill.git commit 205e0289940e8b8342f3ae812640da331381ff16 Author: Fokko Driesprong AuthorDate: Thu Jun 20 15:41:57 2019 +0200 DRILL-7302: Bump Apache Avro to 1.9.0 Apache Avro 1.9.0 brings a lot of new features: Deprecate Joda-Time in favor of Java8 JSR310 and setting it as default Remove support for Hadoop 1.x Move from Jackson 1.x to 2.9 Add ZStandard Codec Lots of updates on the dependencies to fix CVE's Remove Jackson classes from public API Apache Avro is built by default with Java 8 Apache Avro is compiled and tested with Java 11 to guarantee compatibility Apache Avro MapReduce is compiled and tested with Hadoop 3 Apache Avro is now leaner, multiple dependencies were removed: guava, paranamer, commons-codec, and commons-logging and many, many more! close apache/drill#1812 --- .../org/apache/drill/exec/resourcemgr/config/selectors/AclSelector.java | 2 +- .../drill/exec/resourcemgr/config/selectors/NotEqualSelector.java | 2 +- .../drill/exec/physical/impl/join/TestLateralJoinCorrectness.java | 2 +- .../physical/impl/join/TestLateralJoinCorrectnessBatchProcessing.java | 2 +- .../java/org/apache/drill/exec/physical/impl/scan/ScanTestUtils.java| 2 +- pom.xml | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/resourcemgr/config/selectors/AclSelector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/resourcemgr/config/selectors/AclSelector.java index 50acb86..9c6d7c5 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/resourcemgr/config/selectors/AclSelector.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/resourcemgr/config/selectors/AclSelector.java @@ -17,7 +17,7 @@ */ package org.apache.drill.exec.resourcemgr.config.selectors; -import avro.shaded.com.google.common.annotations.VisibleForTesting; +import org.apache.drill.shaded.guava.com.google.common.annotations.VisibleForTesting; import com.typesafe.config.Config; import org.apache.drill.exec.ops.QueryContext; import org.apache.drill.exec.resourcemgr.config.exception.RMConfigException; diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/resourcemgr/config/selectors/NotEqualSelector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/resourcemgr/config/selectors/NotEqualSelector.java index 653c3b5..bb6da02 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/resourcemgr/config/selectors/NotEqualSelector.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/resourcemgr/config/selectors/NotEqualSelector.java @@ -17,7 +17,7 @@ */ package org.apache.drill.exec.resourcemgr.config.selectors; -import avro.shaded.com.google.common.annotations.VisibleForTesting; +import org.apache.drill.shaded.guava.com.google.common.annotations.VisibleForTesting; import com.typesafe.config.Config; import org.apache.drill.exec.ops.QueryContext; import org.apache.drill.exec.resourcemgr.config.exception.RMConfigException; diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/join/TestLateralJoinCorrectness.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/join/TestLateralJoinCorrectness.java index 4d4521d..ff1de9d 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/join/TestLateralJoinCorrectness.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/join/TestLateralJoinCorrectness.java @@ -17,7 +17,7 @@ */ package org.apache.drill.exec.physical.impl.join; -import avro.shaded.com.google.common.collect.Lists; +import org.apache.drill.shaded.guava.com.google.common.collect.Lists; import org.apache.calcite.rel.core.JoinRelType; import org.apache.drill.categories.OperatorTest; import org.apache.drill.common.exceptions.DrillRuntimeException; diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/join/TestLateralJoinCorrectnessBatchProcessing.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/join/TestLateralJoinCorrectnessBatchProcessing.java index 9eee038..ae9003c 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/join/TestLateralJoinCorrectnessBatchProcessing.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/join/TestLateralJoinCorrectnessBatchProcessing.java @@ -17,7 +17,7 @@ */ package org.apache.drill.exec.physical.impl.join; -import avro.shaded.com.google.common.collect.Lists; +import org.apache.drill.shaded.guava.com.google.common.collect.Lists; import org.apache.calcite.rel.core.JoinRelType; import org.apache.drill.common.types.TypeProtos; i
[drill] branch master updated (f3f7dbd -> 205e028)
This is an automated email from the ASF dual-hosted git repository.
amansinha pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git.
from f3f7dbd DRILL-7292: Remove V1 and V2 text readers
new ddc2c02 DRILL-7268: Read Hive array with parquet native reader
new d108677 DRILL-7297: Query hangs in planning stage when Error is thrown
new 205e028 DRILL-7302: Bump Apache Avro to 1.9.0
The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../drill/exec/store/hive/HiveUtilities.java |3 +-
.../exec/hive/complex_types/TestHiveArrays.java| 1439
.../resourcemgr/config/selectors/AclSelector.java |2 +-
.../config/selectors/NotEqualSelector.java |2 +-
.../store/parquet2/DrillParquetGroupConverter.java | 349 +++--
.../exec/store/parquet2/DrillParquetReader.java| 30 +-
.../parquet2/DrillParquetRecordMaterializer.java |2 +-
.../apache/drill/exec/work/foreman/Foreman.java|8 +-
.../java/org/apache/drill/TestFunctionsQuery.java |8 +
.../exec/fn/impl/testing/CustomErrorFunction.java} | 29 +-
.../impl/join/TestLateralJoinCorrectness.java |2 +-
.../TestLateralJoinCorrectnessBatchProcessing.java |2 +-
.../exec/physical/impl/scan/ScanTestUtils.java |2 +-
.../store/parquet2/TestDrillParquetReader.java | 855
.../test/resources/parquet2/hive_arrays_p.parquet | Bin 0 -> 14270 bytes
pom.xml|2 +-
16 files changed, 1697 insertions(+), 1038 deletions(-)
copy exec/java-exec/src/{main/java/org/apache/drill/exec/expr/fn/impl/Not.java
=> test/java/org/apache/drill/exec/fn/impl/testing/CustomErrorFunction.java}
(65%)
create mode 100755
exec/java-exec/src/test/resources/parquet2/hive_arrays_p.parquet
[drill] 02/03: DRILL-7297: Query hangs in planning stage when Error is thrown
This is an automated email from the ASF dual-hosted git repository.
amansinha pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git
commit d1086772ee4d9f027d3fd6f4673aae43158d2172
Author: Volodymyr Vysotskyi
AuthorDate: Thu Jun 20 15:33:00 2019 +0300
DRILL-7297: Query hangs in planning stage when Error is thrown
close apache/drill#1811
---
.../apache/drill/exec/work/foreman/Foreman.java| 8 ++---
.../java/org/apache/drill/TestFunctionsQuery.java | 8 +
.../exec/fn/impl/testing/CustomErrorFunction.java | 42 ++
3 files changed, 53 insertions(+), 5 deletions(-)
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/Foreman.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/Foreman.java
index ce03303..804254b 100644
---
a/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/Foreman.java
+++
b/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/Foreman.java
@@ -285,9 +285,9 @@ public class Foreman implements Runnable {
throw new IllegalStateException();
}
injector.injectChecked(queryContext.getExecutionControls(),
"run-try-end", ForemanException.class);
-} catch (final ForemanException e) {
+} catch (ForemanException | UserException e) {
queryStateProcessor.moveToState(QueryState.FAILED, e);
-} catch (final OutOfMemoryError | OutOfMemoryException e) {
+} catch (OutOfMemoryError | OutOfMemoryException e) {
if (FailureUtils.isDirectMemoryOOM(e)) {
queryStateProcessor.moveToState(QueryState.FAILED,
UserException.memoryError(e).build(logger));
} else {
@@ -298,9 +298,7 @@ public class Foreman implements Runnable {
*/
FailureUtils.unrecoverableFailure(e, "Unable to handle out of memory
condition in Foreman.", EXIT_CODE_HEAP_OOM);
}
-} catch (UserException e) {
- queryStateProcessor.moveToState(QueryState.FAILED, e);
-} catch (AssertionError | Exception ex) {
+} catch (Throwable ex) {
queryStateProcessor.moveToState(QueryState.FAILED,
new ForemanException("Unexpected exception during fragment
initialization: " + ex.getMessage(), ex));
} finally {
diff --git
a/exec/java-exec/src/test/java/org/apache/drill/TestFunctionsQuery.java
b/exec/java-exec/src/test/java/org/apache/drill/TestFunctionsQuery.java
index 16d9789..ae08d92 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/TestFunctionsQuery.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/TestFunctionsQuery.java
@@ -18,6 +18,7 @@
package org.apache.drill;
import static org.apache.drill.exec.expr.fn.impl.DateUtility.formatTimeStamp;
+import static org.hamcrest.CoreMatchers.containsString;
import java.math.BigDecimal;
import java.time.Instant;
@@ -1008,4 +1009,11 @@ public class TestFunctionsQuery extends BaseTestQuery {
.go();
}
}
+
+ @Test // DRILL-7297
+ public void testErrorInUdf() throws Exception {
+expectedException.expect(UserRemoteException.class);
+expectedException.expectMessage(containsString("Error from UDF"));
+test("select error_function()");
+ }
}
diff --git
a/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/testing/CustomErrorFunction.java
b/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/testing/CustomErrorFunction.java
new file mode 100644
index 000..c372e7b
--- /dev/null
+++
b/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/testing/CustomErrorFunction.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.fn.impl.testing;
+
+import org.apache.drill.exec.expr.DrillSimpleFunc;
+import org.apache.drill.exec.expr.annotations.FunctionTemplate;
+import org.apache.drill.exec.expr.annotations.Output;
+import org.apache.drill.exec.expr.holders.VarCharHolder;
+
+@FunctionTemplate(
+name="error_function",
+scope = FunctionTemplate.FunctionScope.SIMPLE,
+nulls = FunctionTemplate.NullHandling.NULL_IF_NULL
+)
+public class CustomErrorFunction implements DrillSimpleFunc {
+
+ @
[drill] branch master updated (0195d1f -> 05a1a3a)
This is an automated email from the ASF dual-hosted git repository. amansinha pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/drill.git. from 0195d1f DRILL-7237: Fix single_value aggregate function for variable length types add 05a1a3a DRILL-7242: Handle additional boundary cases and compute better estimates when popular values span multiple buckets. No new revisions were added by this update. Summary of changes: .../drill/exec/planner/common/Histogram.java | 2 +- .../planner/common/NumericEquiDepthHistogram.java | 125 ++--- .../exec/planner/cost/DrillRelMdSelectivity.java | 3 +- .../common/TestNumericEquiDepthHistogram.java | 113 +++ .../org/apache/drill/exec/sql/TestAnalyze.java | 12 ++ 5 files changed, 215 insertions(+), 40 deletions(-) create mode 100644 exec/java-exec/src/test/java/org/apache/drill/exec/planner/common/TestNumericEquiDepthHistogram.java
[drill] branch master updated (99707a3 -> ba9837e)
This is an automated email from the ASF dual-hosted git repository. amansinha pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/drill.git. from 99707a3 DRILL-7199: Optimize population of metadata for non-interesting columns closes #1771 add ba9837e DRILL-7187: Improve selectivity estimation of BETWEEN predicates and arbitrary combination of range predicates. No new revisions were added by this update. Summary of changes: .../drill/exec/planner/common/DrillStatsTable.java | 9 +- .../drill/exec/planner/common/Histogram.java | 3 +- .../planner/common/NumericEquiDepthHistogram.java | 300 - .../exec/planner/cost/DrillRelMdSelectivity.java | 84 +- .../org/apache/drill/exec/sql/TestAnalyze.java | 28 ++ .../drill/metastore/ColumnStatisticsKind.java | 17 ++ 6 files changed, 308 insertions(+), 133 deletions(-)
[drill] branch master updated: DRILL-7119: Compute range predicate selectivity using histograms.
This is an automated email from the ASF dual-hosted git repository.
amansinha pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git
The following commit(s) were added to refs/heads/master by this push:
new 849f896 DRILL-7119: Compute range predicate selectivity using
histograms.
849f896 is described below
commit 849f896c491118571303a1313f75bece37e80b7e
Author: Aman Sinha
AuthorDate: Mon Mar 25 06:55:57 2019 -0700
DRILL-7119: Compute range predicate selectivity using histograms.
Address code review comments. Add unit test for histogram usage.
close apache/drill#1733
---
.../drill/exec/planner/common/DrillStatsTable.java | 2 +-
.../drill/exec/planner/common/Histogram.java | 2 +-
.../planner/common/NumericEquiDepthHistogram.java | 188 -
.../exec/planner/cost/DrillRelMdSelectivity.java | 28 ++-
.../test/java/org/apache/drill/PlanTestBase.java | 4 +-
.../org/apache/drill/exec/sql/TestAnalyze.java | 15 ++
6 files changed, 225 insertions(+), 14 deletions(-)
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/common/DrillStatsTable.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/common/DrillStatsTable.java
index 65b0b64..d34a5e2 100644
---
a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/common/DrillStatsTable.java
+++
b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/common/DrillStatsTable.java
@@ -215,7 +215,7 @@ public class DrillStatsTable {
// get the histogram for this column
Histogram hist = cs.getHistogram();
-histogram.put(cs.getName(), hist);
+histogram.put(cs.getName().toUpperCase(), hist);
}
}
}
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/common/Histogram.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/common/Histogram.java
index 1980ee5..c6444a3 100644
---
a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/common/Histogram.java
+++
b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/common/Histogram.java
@@ -37,5 +37,5 @@ public interface Histogram {
* @param filter
* @return estimated selectivity or NULL if it could not be estimated for
any reason
*/
- Double estimatedSelectivity(RexNode filter);
+ Double estimatedSelectivity(final RexNode filter);
}
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/common/NumericEquiDepthHistogram.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/common/NumericEquiDepthHistogram.java
index 386141e..9d5bf6f 100644
---
a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/common/NumericEquiDepthHistogram.java
+++
b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/common/NumericEquiDepthHistogram.java
@@ -21,8 +21,15 @@ package org.apache.drill.exec.planner.common;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.annotation.JsonTypeName;
+import java.util.List;
+
+import org.apache.calcite.rex.RexCall;
import org.apache.calcite.rex.RexNode;
+import org.apache.calcite.rex.RexLiteral;
import com.clearspring.analytics.stream.quantile.TDigest;
+import org.apache.calcite.sql.SqlKind;
+import org.apache.calcite.sql.SqlOperator;
+import org.apache.drill.shaded.guava.com.google.common.base.Preconditions;
/**
* A column specific equi-depth histogram which is meant for numeric data types
@@ -30,6 +37,19 @@ import com.clearspring.analytics.stream.quantile.TDigest;
@JsonTypeName("numeric-equi-depth")
public class NumericEquiDepthHistogram implements Histogram {
+ /**
+ * Use a small non-zero selectivity rather than 0 to account for the fact
that
+ * histogram boundaries are approximate and even if some values lie outside
the
+ * range, we cannot be absolutely sure
+ */
+ static final double SMALL_SELECTIVITY = 0.0001;
+
+ /**
+ * Use a large selectivity of 1.0 whenever we are reasonably confident that
all rows
+ * qualify. Even if this is off by a small fraction, it is acceptable.
+ */
+ static final double LARGE_SELECTIVITY = 1.0;
+
// For equi-depth, all buckets will have same (approx) number of rows
@JsonProperty("numRowsPerBucket")
private long numRowsPerBucket;
@@ -69,27 +89,177 @@ public class NumericEquiDepthHistogram implements
Histogram {
}
@Override
- public Double estimatedSelectivity(RexNode filter) {
+ public Double estimatedSelectivity(final RexNode filter) {
if (numRowsPerBucket >= 0) {
- return 1.0;
-} else {
- return null;
+ // at a minimum, the histogram should have a start and end point of 1
bucket, so at least 2 entries
+ Preconditions.checkArgument(buckets.length >= 2, "Histogram has invalid
number of entries");
+ final int first = 0;
+ final int last = buckets.length - 1;
+
+ //
[drill] branch master updated: DRILL-7152: During histogram creation handle the case when all values of a column are NULLs.
This is an automated email from the ASF dual-hosted git repository.
amansinha pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git
The following commit(s) were added to refs/heads/master by this push:
new 54384a9 DRILL-7152: During histogram creation handle the case when
all values of a column are NULLs.
54384a9 is described below
commit 54384a992a0742aeab23afa82c7b7f4adcd388d3
Author: Aman Sinha
AuthorDate: Tue Apr 2 14:30:26 2019 -0700
DRILL-7152: During histogram creation handle the case when all values of a
column are NULLs.
close apache/drill#1730
---
.../drill/exec/expr/fn/impl/TDigestFunctions.java | 320 -
.../org/apache/drill/exec/sql/TestAnalyze.java | 36 +++
2 files changed, 228 insertions(+), 128 deletions(-)
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/TDigestFunctions.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/TDigestFunctions.java
index 3be70c2..041543b 100644
---
a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/TDigestFunctions.java
+++
b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/TDigestFunctions.java
@@ -83,14 +83,18 @@ public class TDigestFunctions {
if (work.obj != null) {
com.clearspring.analytics.stream.quantile.TDigest tdigest =
(com.clearspring.analytics.stream.quantile.TDigest) work.obj;
try {
- int size = tdigest.smallByteSize();
- java.nio.ByteBuffer byteBuf = java.nio.ByteBuffer.allocate(size);
- tdigest.asSmallBytes(byteBuf);
- out.buffer = buffer.reallocIfNeeded(size);
- out.start = 0;
- out.end = size;
- out.buffer.setBytes(0, byteBuf.array());
- out.isSet = 1;
+ if (tdigest.size() > 0) {
+int size = tdigest.smallByteSize();
+java.nio.ByteBuffer byteBuf = java.nio.ByteBuffer.allocate(size);
+tdigest.asSmallBytes(byteBuf);
+out.buffer = buffer.reallocIfNeeded(size);
+out.start = 0;
+out.end = size;
+out.buffer.setBytes(0, byteBuf.array());
+out.isSet = 1;
+ } else {
+out.isSet = 0;
+ }
} catch (Exception e) {
throw new
org.apache.drill.common.exceptions.DrillRuntimeException("Failed to get TDigest
output", e);
}
@@ -138,14 +142,18 @@ public class TDigestFunctions {
if (work.obj != null) {
com.clearspring.analytics.stream.quantile.TDigest tdigest =
(com.clearspring.analytics.stream.quantile.TDigest) work.obj;
try {
- int size = tdigest.smallByteSize();
- java.nio.ByteBuffer byteBuf = java.nio.ByteBuffer.allocate(size);
- tdigest.asSmallBytes(byteBuf);
- out.buffer = buffer.reallocIfNeeded(size);
- out.start = 0;
- out.end = size;
- out.buffer.setBytes(0, byteBuf.array());
- out.isSet = 1;
+ if (tdigest.size() > 0) {
+int size = tdigest.smallByteSize();
+java.nio.ByteBuffer byteBuf = java.nio.ByteBuffer.allocate(size);
+tdigest.asSmallBytes(byteBuf);
+out.buffer = buffer.reallocIfNeeded(size);
+out.start = 0;
+out.end = size;
+out.buffer.setBytes(0, byteBuf.array());
+out.isSet = 1;
+ } else {
+out.isSet = 0;
+ }
} catch (Exception e) {
throw new
org.apache.drill.common.exceptions.DrillRuntimeException("Failed to get TDigest
output", e);
}
@@ -189,14 +197,18 @@ public class TDigestFunctions {
if (work.obj != null) {
com.clearspring.analytics.stream.quantile.TDigest tdigest =
(com.clearspring.analytics.stream.quantile.TDigest) work.obj;
try {
- int size = tdigest.smallByteSize();
- java.nio.ByteBuffer byteBuf = java.nio.ByteBuffer.allocate(size);
- tdigest.asSmallBytes(byteBuf);
- out.buffer = buffer.reallocIfNeeded(size);
- out.start = 0;
- out.end = size;
- out.buffer.setBytes(0, byteBuf.array());
- out.isSet = 1;
+ if (tdigest.size() > 0) {
+int size = tdigest.smallByteSize();
+java.nio.ByteBuffer byteBuf = java.nio.ByteBuffer.allocate(size);
+tdigest.asSmallBytes(byteBuf);
+out.buffer = buffer.reallocIfNeeded(size);
+out.start = 0;
+out.end = size;
+out.buffer.setBytes(0, byteBuf.array());
+out.isSet = 1;
+ } else {
+out.isSet = 0;
+ }
} catch (Exception e) {
throw new
org.apache.drill.common.exceptions.DrillRuntimeException("Failed to get TDigest
output", e);
}
@@ -244,14 +256,18 @@ public class TDigestFunctions {
if (work.obj != null) {
com.clears
[drill] branch master updated: DRILL-7117: Support creation of equi-depth histogram for selected data types.
This is an automated email from the ASF dual-hosted git repository.
amansinha pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git
The following commit(s) were added to refs/heads/master by this push:
new fd331da DRILL-7117: Support creation of equi-depth histogram for
selected data types.
fd331da is described below
commit fd331da3872534b284ebd54fec0ff11f7bae47a5
Author: Aman Sinha
AuthorDate: Wed Feb 13 15:29:04 2019 -0800
DRILL-7117: Support creation of equi-depth histogram for selected data
types.
Support int/bigint/float4/float8, time/timestamp/date and boolean.
Build the histogram from the t-digest byte array and serialize as JSON
string.
More changes for serialization/deserialization.
Add code-gen stubs (empty) for VarChar/VarBinary types.
Address review comments (part 1). Add unit test.
Address review comments (part 2) for sampling.
close apache/drill#1715
---
.../java/org/apache/drill/exec/ExecConstants.java |8 +
.../drill/exec/expr/fn/impl/TDigestFunctions.java | 1082
.../impl/statistics/MergedStatisticFactory.java|1 +
.../exec/physical/impl/statistics/Statistic.java |2 +
.../impl/statistics/StatisticsMergeBatch.java |5 +-
.../impl/statistics/TDigestMergedStatistic.java| 130 +++
.../drill/exec/planner/common/DrillStatsTable.java | 47 +-
.../drill/exec/planner/common/Histogram.java | 41 +
.../drill/exec/planner/common/HistogramUtils.java | 53 +
.../planner/common/NumericEquiDepthHistogram.java | 114 +++
.../drill/exec/planner/physical/AnalyzePrule.java |7 +-
.../exec/server/options/SystemOptionManager.java |3 +-
.../easy/json/JsonStatisticsRecordWriter.java |6 +-
.../java-exec/src/main/resources/drill-module.conf |3 +-
.../org/apache/drill/exec/sql/TestAnalyze.java | 61 ++
exec/jdbc-all/pom.xml |1 +
16 files changed, 1557 insertions(+), 7 deletions(-)
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java
index 6d38045..3af8744 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java
@@ -979,4 +979,12 @@ public final class ExecConstants {
public static final String NDV_BLOOM_FILTER_FPOS_PROB =
"exec.statistics.ndv_extrapolation_bf_fpprobability";
public static final LongValidator NDV_BLOOM_FILTER_FPOS_PROB_VALIDATOR = new
PositiveLongValidator(NDV_BLOOM_FILTER_FPOS_PROB,
100, new OptionDescription("Controls trade-off between NDV statistic
computation memory cost and sampling extrapolation accuracy"));
+
+ /**
+ * Controls the 'compression' factor for the TDigest algorithm.
+ */
+ public static final String TDIGEST_COMPRESSION =
"exec.statistics.tdigest_compression";
+ public static final LongValidator TDIGEST_COMPRESSION_VALIDATOR = new
PositiveLongValidator(TDIGEST_COMPRESSION, 1,
+new OptionDescription("Controls trade-off between t-digest quantile
statistic storage cost and accuracy. " +
+ "Higher values use more groups (clusters) for the t-digest and improve
accuracy at the expense of extra storage. "));
}
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/TDigestFunctions.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/TDigestFunctions.java
new file mode 100644
index 000..3be70c2
--- /dev/null
+++
b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/TDigestFunctions.java
@@ -0,0 +1,1082 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.expr.fn.impl;
+
+import io.netty.buffer.DrillBuf;
+import org.apache.drill.exec.expr.DrillAggFunc;
+import org.apache.drill.exec.expr.annotations.FunctionTemplate;
+import org.apache.drill.exec.expr.annotations.Output;
+import org.apache.drill.exec.expr.annotations.Param;
+import org.apache.drill.exec.expr.annotat
[drill] branch master updated (7c2661d -> 189ebb4)
This is an automated email from the ASF dual-hosted git repository.
amansinha pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git.
from 7c2661d DRILL-7108: Improve selectivity estimates for (NOT)LIKE,
NOT_EQUALS, IS NOT NULL predicates
new 2d28be8 DRILL-7113: Fix creation of filter conditions for IS NULL and
IS NOT NULL for MapR-DB format plugin
new 189ebb4 DRILL-6852: Adapt current Parquet Metadata cache
implementation to use Drill Metastore API
The 3513 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../store/mapr/db/json/JsonConditionBuilder.java | 8 +-
.../drill/maprdb/tests/json/TestSimpleJson.java| 6 +-
.../store/hive/HiveDrillNativeParquetScan.java | 86 ++-
.../hive/HiveParquetTableMetadataProvider.java | 104 +++
...BooleanPredicate.java => BooleanPredicate.java} | 47 +-
...isonPredicate.java => ComparisonPredicate.java} | 158 +++--
.../drill/exec/expr/ExactStatisticsConstants.java | 14 +-
.../exec/expr/ExpressionTreeMaterializer.java | 31 +-
.../FilterBuilder.java}| 85 ++-
.../apache/drill/exec/expr/FilterPredicate.java| 10 +-
.../ParquetIsPredicate.java => IsPredicate.java} | 111 ++--
.../apache/drill/exec/expr/StatisticsProvider.java | 336 ++
.../exec/expr/stat/ParquetPredicatesHelper.java| 60 --
.../drill/exec/expr/stat/RangeExprEvaluator.java | 365 ---
...{ParquetFilterPredicate.java => RowsMatch.java} | 23 +-
.../exec/physical/base/AbstractGroupScan.java | 2 +-
.../base/AbstractGroupScanWithMetadata.java| 716 +
.../drill/exec/physical/base/DbGroupScan.java | 32 +-
.../drill/exec/physical/base/FileGroupScan.java| 7 +-
.../apache/drill/exec/physical/base/GroupScan.java | 2 +-
.../drill/exec/physical/base/IndexGroupScan.java | 18 +-
.../physical/base/ParquetMetadataProvider.java | 69 ++
...Scan.java => ParquetTableMetadataProvider.java} | 26 +-
.../exec/physical/base/TableMetadataProvider.java | 88 +++
.../exec/planner/ParquetPartitionDescriptor.java | 12 +-
.../logical/DrillPushProjectIntoScanRule.java | 6 +-
.../planner/logical/partition/PruneScanRule.java | 4 +-
.../exec/record/metadata/SchemaPathUtils.java | 85 +++
.../store/parquet/AbstractParquetGroupScan.java| 683 +++-
.../store/parquet/BaseParquetMetadataProvider.java | 347 ++
.../exec/store/parquet/FilterEvaluatorUtils.java | 139
.../drill/exec/store/parquet/ParquetGroupScan.java | 338 ++
.../store/parquet/ParquetGroupScanStatistics.java | 138 ++--
.../exec/store/parquet/ParquetPushDownFilter.java | 20 +-
.../store/parquet/ParquetRGFilterEvaluator.java| 141
.../parquet/ParquetTableMetadataProviderImpl.java | 338 ++
.../store/parquet/ParquetTableMetadataUtils.java | 670 +++
.../exec/store/parquet/metadata/Metadata.java | 23 +-
.../exec/store/parquet/metadata/Metadata_V2.java | 5 +-
.../exec/store/parquet/metadata/Metadata_V3.java | 3 +-
.../parquet/stat/ParquetFooterStatCollector.java | 196 --
.../parquet/stat/ParquetMetaStatCollector.java | 301 -
.../exec/store/parquet2/DrillParquetReader.java| 10 -
.../apache/drill/exec/util/ImpersonationUtil.java | 8 +-
.../org/apache/drill/metastore/BaseMetadata.java | 80 +++
.../CollectableColumnStatisticsKind.java} | 25 +-
.../CollectableTableStatisticsKind.java} | 25 +-
.../apache/drill/metastore/ColumnStatistics.java | 73 +++
.../drill/metastore/ColumnStatisticsImpl.java | 67 ++
.../drill/metastore/ColumnStatisticsKind.java | 120
.../org/apache/drill/metastore/FileMetadata.java | 102 +++
.../apache/drill/metastore/FileTableMetadata.java | 132
.../apache/drill/metastore/LocationProvider.java | 17 +-
.../apache/drill/metastore/PartitionMetadata.java | 123
.../apache/drill/metastore/RowGroupMetadata.java | 106 +++
.../StatisticsKind.java} | 38 +-
.../TableMetadata.java}| 19 +-
.../drill/metastore/TableStatisticsKind.java | 79 +++
.../store/parquet/TestParquetFilterPushDown.java | 132 ++--
.../store/parquet/TestParquetLimitPushDown.java| 8 +-
.../apache/drill/common/expression/SchemaPath.java | 3 +
.../org/apache/drill/common/graph/GraphValue.java | 6 +-
62 files changed, 4806 insertions(+), 2220 deletions(-)
create mode 100644
contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/HiveParquetTableMetadataProvider.java
rename
exec/java-exec/src/main/java/org/apache/drill/exec/expr
[drill] branch master updated (0a61b8f -> 3bec197)
This is an automated email from the ASF dual-hosted git repository. amansinha pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/drill.git. from 0a61b8f DRILL-6964: Implement CREATE / DROP SCHEMA commands new b5fe065 DRILL-7019: Add check for redundant imports new 35b42eb DRILL-7016: Wrong query result with RuntimeFilter enabled when order of join and filter condition is swapped new 3bec197 DRILL-6862: Update Calcite to 1.18.0 The 3458 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../initilializers/DefaultReadersInitializer.java | 1 - .../drill/yarn/client/TestCommandLineOptions.java | 2 - drill-yarn/src/test/resources/doy-test-logback.xml | 2 +- .../src/test/resources/second-test-config.conf | 4 +- drill-yarn/src/test/resources/test-doy-config.conf | 4 +- .../src/test/resources/test-doy-distrib.conf | 2 +- .../src/test/resources/third-test-config.conf | 4 +- exec/java-exec/src/main/codegen/data/Parser.tdd| 763 - .../apache/drill/exec/planner/DrillRelBuilder.java | 10 + .../apache/drill/exec/planner/RuleInstance.java| 3 +- .../planner/index/rules/DbScanSortRemovalRule.java | 1 - .../exec/planner/logical/DrillImplementor.java | 1 - .../planner/logical/DrillMergeProjectRule.java | 2 - .../drill/exec/planner/physical/TopNPrel.java | 2 - .../physical/visitor/RuntimeFilterVisitor.java | 8 +- .../drill/exec/planner/sql/SqlConverter.java | 15 +- .../drill/exec/planner/sql/TypeInferenceUtils.java | 6 +- .../test/java/org/apache/drill/TestCTASJson.java | 16 +- .../physical/impl/TestNestedDateTimeTimestamp.java | 12 +- .../impl/join/TestHashJoinJPPDCorrectness.java | 178 + .../impl/scan/project/TestNullColumnLoader.java| 3 - .../impl/scan/project/TestRowBatchMerger.java | 5 - .../impl/scan/project/TestScanLevelProjection.java | 2 - .../scan/project/TestSchemaLevelProjection.java| 29 +- .../impl/scan/project/TestSchemaSmoothing.java | 7 - .../store/parquet2/TestDrillParquetReader.java | 13 + .../java/org/apache/drill/test/TestBuilder.java| 11 +- .../java-exec/src/test/resources/drill-module.conf | 2 +- exec/java-exec/src/test/resources/zkacltest.conf | 22 +- exec/jdbc-all/pom.xml | 2 +- .../drill/jdbc/impl/AvaticaDrillSqlAccessor.java | 6 + .../apache/drill/jdbc/impl/DrillResultSetImpl.java | 12 +- .../apache/drill/jdbc/impl/WrappedAccessor.java| 447 .../io/netty/buffer/UnsafeDirectLittleEndian.java | 2 - .../exec/vector/complex/RepeatedValueVector.java | 3 +- pom.xml| 90 ++- src/main/resources/checkstyle-config.xml | 1 + src/main/resources/checkstyle-suppressions.xml | 2 +- 38 files changed, 1087 insertions(+), 608 deletions(-) create mode 100644 exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/join/TestHashJoinJPPDCorrectness.java delete mode 100644 exec/jdbc/src/main/java/org/apache/drill/jdbc/impl/WrappedAccessor.java
[drill] branch master updated (8fb85cd -> 0a61b8f)
This is an automated email from the ASF dual-hosted git repository.
amansinha pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git.
from 8fb85cd DRILL-7006: Add type conversion to row writers
new 982e980 DRILL-6997: Semijoin is changing the join ordering for some
tpcds queries.
new 0242241 DRILL-7002: whatever exec.hashjoin.num_partitions is set,
output right results
new 9c82e06 DRILL-7008: Drillbits: clear stale shutdown hooks
new b1594cb DRILL-7018: Fixed Parquet buffer overflow when reading
timestamp column
new 0a61b8f DRILL-6964: Implement CREATE / DROP SCHEMA commands
The 3455 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
exec/java-exec/pom.xml | 21 +
.../record/metadata/schema/parser/SchemaLexer.g4 | 90
.../record/metadata/schema/parser/SchemaParser.g4 | 72 +++
exec/java-exec/src/main/codegen/data/Parser.tdd| 9 +-
.../src/main/codegen/includes/parserImpls.ftl | 259 +--
.../exec/physical/impl/join/HashJoinBatch.java | 18 +-
.../apache/drill/exec/planner/PlannerPhase.java| 13 +-
.../apache/drill/exec/planner/RuleInstance.java| 1 -
.../exec/planner/logical/DrillSemiJoinRel.java | 56 ++-
.../exec/planner/logical/DrillSemiJoinRule.java| 183
.../drill/exec/planner/physical/JoinPrel.java | 58 +--
.../drill/exec/planner/sql/DrillSqlWorker.java | 2 +-
.../drill/exec/planner/sql/SchemaUtilites.java | 19 +-
.../planner/sql/handlers/DefaultSqlHandler.java| 10 +-
.../planner/sql/handlers/DropTableHandler.java | 24 +-
.../exec/planner/sql/handlers/SchemaHandler.java | 214 +
.../exec/planner/sql/handlers/ViewHandler.java | 4 +-
.../sql/parser/CompoundIdentifierConverter.java| 2 +
.../sql/parser/SqlCreateType.java} | 26 +-
.../exec/planner/sql/parser/SqlCreateView.java | 16 +-
.../drill/exec/planner/sql/parser/SqlSchema.java | 280
.../record/metadata/AbstractColumnMetadata.java| 30 ++
.../exec/record/metadata/MapColumnMetadata.java| 14 +
.../record/metadata/PrimitiveColumnMetadata.java | 46 ++
.../metadata/RepeatedListColumnMetadata.java | 6 +
.../drill/exec/record/metadata/TupleSchema.java| 7 +
.../metadata/schema/FsMetastoreSchemaProvider.java | 51 +++
.../metadata/schema/InlineSchemaProvider.java} | 46 +-
.../record/metadata/schema/PathSchemaProvider.java | 132 ++
.../record/metadata/schema/SchemaContainer.java| 145 ++
.../record/metadata/schema/SchemaProvider.java | 66 +++
.../metadata/schema/parser/SchemaExprParser.java | 96
.../schema/parser/SchemaParsingException.java} | 9 +-
.../metadata/schema/parser/SchemaVisitor.java | 269 +++
.../schema/parser/UpperCaseCharStream.java | 84
.../org/apache/drill/exec/server/Drillbit.java | 15 +-
.../NullableFixedByteAlignedReaders.java | 5 +-
.../java/org/apache/drill/TestSchemaCommands.java | 501 +
.../exec/physical/impl/join/TestSemiJoin.java | 25 +-
.../record/metadata/schema/TestSchemaProvider.java | 232 ++
.../schema/parser/TestParserErrorHandling.java | 157 +++
.../metadata/schema/parser/TestSchemaParser.java | 279
.../java/org/apache/drill/exec/sql/TestCTTAS.java | 10 +-
.../org/apache/drill/exec/sql/TestViewSupport.java | 2 +-
.../drill/exec/record/metadata/ColumnMetadata.java | 17 +
.../drill/exec/record/metadata/TupleMetadata.java | 8 +
46 files changed, 3426 insertions(+), 203 deletions(-)
create mode 100644
exec/java-exec/src/main/antlr4/org/apache/drill/exec/record/metadata/schema/parser/SchemaLexer.g4
create mode 100644
exec/java-exec/src/main/antlr4/org/apache/drill/exec/record/metadata/schema/parser/SchemaParser.g4
create mode 100644
exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillSemiJoinRule.java
create mode 100644
exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/SchemaHandler.java
copy
exec/java-exec/src/main/java/org/apache/drill/exec/{physical/impl/common/SpilledPartitionMetadata.java
=> planner/sql/parser/SqlCreateType.java} (59%)
create mode 100644
exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/SqlSchema.java
create mode 100644
exec/java-exec/src/main/java/org/apache/drill/exec/record/metadata/schema/FsMetastoreSchemaProvider.java
copy
exec/java-exec/src/main/java/org/apache/drill/exec/{physical/rowSet/model/ReaderIndex.java
=> record/metadata/schema/InlineSchemaProvider.java} (50%)
create mode 100644
exec/java-exec/src/main/java/org/apache/drill/e
[drill] 01/03: DRILL-6883: Force reload of options after resetting parameter
This is an automated email from the ASF dual-hosted git repository.
amansinha pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git
commit f6d54b480a123724f4cca7d146b0a5c3fb077652
Author: Kunal Khatua
AuthorDate: Wed Dec 5 23:34:26 2018 -0800
DRILL-6883: Force reload of options after resetting parameter
If an update is initiated, the webpage loaded changes. Attempting a reset
tries to reload this page, but it does not show the updated values from the
back end.
This patch forces the reload of updated values by redirecting to /options
page
Use AJAX call for Update and Reset
Eliminates the need for window.location=
is Number check
Added comments to explain extraction
close apache/drill#1563
---
exec/java-exec/src/main/resources/rest/options.ftl | 38 +-
1 file changed, 30 insertions(+), 8 deletions(-)
diff --git a/exec/java-exec/src/main/resources/rest/options.ftl
b/exec/java-exec/src/main/resources/rest/options.ftl
index 7ac8a2e..a8e03fa 100644
--- a/exec/java-exec/src/main/resources/rest/options.ftl
+++ b/exec/java-exec/src/main/resources/rest/options.ftl
@@ -22,9 +22,32 @@
-function resetToDefault(optionName, optionValue, optionKind) {
-$.post("/option/"+optionName, {kind: optionKind, name: optionName,
value: optionValue}, function (status) { location.reload(true); } );
+//Alter System Values
+function alterSysOption(optionName, optionValue, optionKind) {
+$.post("/option/"+optionName, {kind: optionKind, name: optionName,
value: optionValue}, function () {
+location.reload(true);
+});
+}
+
+//Read Values and apply
+function alterSysOptionUsingId(optionRawName) {
+//Escaping '.' for id search
+let optionName = optionRawName.replace(/\./gi, "\\.");
+//Extracting datatype from the form
+let optionKind = $("#"+optionName+" input[name='kind']").attr("value");
+//Extracting value from the form's INPUT element
+let optionValue = $("#"+optionName+" input[name='value']").val();
+if (optionKind == "BOOLEAN") {
+//Extracting boolean value from the form's SELECT element (since
this is a dropdown input)
+optionValue = $("#"+optionName+" select[name='value']").val();
+} else if (optionKind != "STRING") { //i.e. it is a number
(FLOAT/DOUBLE/LONG)
+if (isNaN(optionValue)) {
+alert(optionValue+" is not a valid number for option:
"+optionName);
+return;
+}
}
+alterSysOption(optionRawName, optionValue, optionKind);
+}
@@ -75,7 +98,7 @@ table.sortable thead .sorting_desc { background-image:
url("/static/img/black-de
${option.getName()}
-
+
@@ -89,10 +112,9 @@ table.sortable thead .sorting_desc { background-image:
url("/static/img/black-de
- Update
- disabled="true" style="pointer-events:none" <#else>
- title="Reset to
${option.getDefaultValue()}">Reset
+ Update
+ disabled="true" style="pointer-events:none" <#else>
+ title="Reset to
${option.getDefaultValue()}">Default
@@ -127,7 +149,7 @@ table.sortable thead .sorting_desc { background-image:
url("/static/img/black-de
//Inject Descriptions for table
let size = $('#optionsTbl tbody tr').length;
for (i = 1; i <= size; i++) {
- let currRow = $("#row-"+i);
+let currRow = $("#row-"+i);
let optionName = currRow.find("#optionName").text();
let setOptDescrip =
currRow.find("#description").text(getDescription(optionName));
}
[drill] 02/03: DRILL-6874: Close input stream after AsyncPageReaderTask is completed
This is an automated email from the ASF dual-hosted git repository.
amansinha pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git
commit 0543dbb356c669132c8ac75a99b318661bdcf664
Author: Bohdan Kazydub
AuthorDate: Thu Dec 6 15:03:25 2018 +0200
DRILL-6874: Close input stream after AsyncPageReaderTask is completed
close apache/drill#1565
---
.../drill/exec/store/parquet/columnreaders/AsyncPageReader.java| 7 +++
1 file changed, 7 insertions(+)
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/AsyncPageReader.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/AsyncPageReader.java
index e429fb6..8b5c926 100644
---
a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/AsyncPageReader.java
+++
b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/AsyncPageReader.java
@@ -417,6 +417,13 @@ class AsyncPageReader extends PageReader {
if (totalValuesRead >= totalValuesCount) {
try {
queue.put(ReadStatus.EMPTY);
+ // Some InputStreams (like S3ObjectInputStream) should be closed
+ // as soon as possible to make the connection reusable.
+ try {
+parent.inputStream.close();
+ } catch (IOException e) {
+logger.trace(String.format("[%s]: Failure while closing
InputStream", name), e);
+ }
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
// Do nothing.
[drill] 03/03: DRILL-6887: Fix FunctionInitializerTest.init() failure
This is an automated email from the ASF dual-hosted git repository.
amansinha pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git
commit e4782133563e365c858bcfb2b23845e4b0299dff
Author: Vitalii Diravka
AuthorDate: Fri Dec 7 15:55:48 2018 +0200
DRILL-6887: Fix FunctionInitializerTest.init() failure
close apache/drill#1566
---
exec/java-exec/src/test/resources/drill-udf/pom.xml | 3 +++
1 file changed, 3 insertions(+)
diff --git a/exec/java-exec/src/test/resources/drill-udf/pom.xml
b/exec/java-exec/src/test/resources/drill-udf/pom.xml
index 0e25cba..0a68198 100644
--- a/exec/java-exec/src/test/resources/drill-udf/pom.xml
+++ b/exec/java-exec/src/test/resources/drill-udf/pom.xml
@@ -28,6 +28,8 @@
1.0
+1.8
+1.8
${project.name}
${project.basedir}/target
1.13.0
@@ -58,6 +60,7 @@
org.apache.maven.plugins
maven-compiler-plugin
+3.8.0
${include.files}
[drill] branch master updated (637354e -> e478213)
This is an automated email from the ASF dual-hosted git repository. amansinha pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/drill.git. from 637354e DRILL-6751: Upgrade Apache parent POM to version 21 new f6d54b4 DRILL-6883: Force reload of options after resetting parameter new 0543dbb DRILL-6874: Close input stream after AsyncPageReaderTask is completed new e478213 DRILL-6887: Fix FunctionInitializerTest.init() failure The 3 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../parquet/columnreaders/AsyncPageReader.java | 7 exec/java-exec/src/main/resources/rest/options.ftl | 38 +- .../java-exec/src/test/resources/drill-udf/pom.xml | 3 ++ 3 files changed, 40 insertions(+), 8 deletions(-)
[drill] 03/04: DRILL-4546: Only generate one zip archive when using apache-release profile
This is an automated email from the ASF dual-hosted git repository.
amansinha pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git
commit 7d3763728fd82b3a033936b14ff35545e93d1c5c
Author: Laurent Goujon
AuthorDate: Mon Mar 28 13:31:53 2016 -0700
DRILL-4546: Only generate one zip archive when using apache-release profile
Drill root pom doesn't override completely Apache parent pom configuration
regarding assemblies, which caused a zip archive of the project to be
generated
twice, and deployed to a remote server twice too.
The fix uses the plugin properties to override the configuration. Also
remove
Drill source assembly descriptor as the Apache parent project provides the
same one.
---
pom.xml | 18 ++---
src/main/resources/assemblies/source-assembly.xml | 99 ---
2 files changed, 5 insertions(+), 112 deletions(-)
diff --git a/pom.xml b/pom.xml
index c9d2843..790dd11 100644
--- a/pom.xml
+++ b/pom.xml
@@ -38,6 +38,10 @@
http://drill.apache.org/
+1.8
+1.8
+
source-release-zip-tar
+
${project.basedir}/target/generated-sources
${project.basedir}/src/main/protobuf/
4.12
@@ -471,8 +475,6 @@
org.apache.maven.plugins
maven-compiler-plugin
- 1.8
- 1.8
2048m
false
true
@@ -1682,25 +1684,15 @@
false
-
+
maven-assembly-plugin
source-release-assembly
-package
-
- single
-
- true
false
-
-
src/main/resources/assemblies/source-assembly.xml
-
apache-drill-${project.version}-src
- gnu
diff --git a/src/main/resources/assemblies/source-assembly.xml
b/src/main/resources/assemblies/source-assembly.xml
deleted file mode 100644
index 4ed43f1..000
--- a/src/main/resources/assemblies/source-assembly.xml
+++ /dev/null
@@ -1,99 +0,0 @@
-
-
-
- source-release
-
-zip
-tar.gz
-
-
-
-
- .
- /
- true
-
-
-
%regex[(?!((?!${project.build.directory}/)[^/]+/)*src/).*${project.build.directory}.*]
-
-
-
-
-
-
%regex[(?!((?!${project.build.directory}/)[^/]+/)*src/)(.*/)?maven-eclipse\.xml]
-
-
%regex[(?!((?!${project.build.directory}/)[^/]+/)*src/)(.*/)?\.project]
-
-
%regex[(?!((?!${project.build.directory}/)[^/]+/)*src/)(.*/)?\.classpath]
-
-
%regex[(?!((?!${project.build.directory}/)[^/]+/)*src/)(.*/)?[^/]*\.iws]
-
-
%regex[(?!((?!${project.build.directory}/)[^/]+/)*src/)(.*/)?\.idea(/.*)?]
-
-
%regex[(?!((?!${project.build.directory}/)[^/]+/)*src/)(.*/)?out(/.*)?]
-
-
%regex[(?!((?!${project.build.directory}/)[^/]+/)*src/)(.*/)?[^/]*\.ipr]
-
-
%regex[(?!((?!${project.build.directory}/)[^/]+/)*src/)(.*/)?[^/]*\.iml]
-
-
%regex[(?!((?!${project.build.directory}/)[^/]+/)*src/)(.*/)?\.settings(/.*)?]
-
-
%regex[(?!((?!${project.build.directory}/)[^/]+/)*src/)(.*/)?\.externalToolBuilders(/.*)?]
-
-
%regex[(?!((?!${project.build.directory}/)[^/]+/)*src/)(.*/)?\.deployables(/.*)?]
-
-
%regex[(?!((?!${project.build.directory}/)[^/]+/)*src/)(.*/)?\.wtpmodules(/.*)?]
-
-
-
-
-
%regex[(?!((?!${project.build.directory}/)[^/]+/)*src/)(.*/)?\.gitignore(/.*)?]
-
-
-**/.buildpath
-**/sandbox/**
-
-
-
%regex[(?!((?!${project.build.directory}/)[^/]+/)*src/)(.*/)?cobertura\.ser]
-
-
-
-
%regex[(?!((?!${project.build.directory}/)[^/]+/)*src/)(.*/)?pom\.xml\.releaseBackup]
-
-
%regex[(?!((?!${project.build.directory}/)[^/]+/)*src/)(.*/)?release\.properties]
-
-
-
-
-
-
${project.build.directory}/maven-shared-archive-resources/META-INF
-
- /
-
-
-
[drill] 01/04: DRILL-6882: Handle the cases where RowKeyJoin's left pipeline being called multiple times.
This is an automated email from the ASF dual-hosted git repository.
amansinha pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git
commit f3eef38fd633be28f961139b371f524f5172914f
Author: Hanumath Rao Maduri
AuthorDate: Sat Sep 30 13:26:40 2017 -0700
DRILL-6882: Handle the cases where RowKeyJoin's left pipeline being called
multiple times.
close apache/drill#1562
---
.../exec/physical/config/IteratorValidator.java| 14 +++--
.../exec/physical/impl/join/RowKeyJoinBatch.java | 5
.../validate/IteratorValidatorBatchIterator.java | 20 +
.../impl/validate/IteratorValidatorCreator.java| 2 +-
.../impl/validate/IteratorValidatorInjector.java | 33 +++---
5 files changed, 62 insertions(+), 12 deletions(-)
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/IteratorValidator.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/IteratorValidator.java
index 9fbef97..4f73b00 100644
---
a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/IteratorValidator.java
+++
b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/IteratorValidator.java
@@ -23,10 +23,20 @@ import org.apache.drill.exec.physical.base.PhysicalVisitor;
public class IteratorValidator extends AbstractSingle{
static final org.slf4j.Logger logger =
org.slf4j.LoggerFactory.getLogger(IteratorValidator.class);
+ /* isRepeatable flag will be set to true if this validator is created by a
Repeatable pipeline.
+ * In a repeatable pipeline some state transitions are valid i.e downstream
operator
+ * can call the upstream operator even after receiving NONE.
+ */
+ public final boolean isRepeatable;
- public IteratorValidator(PhysicalOperator child) {
+ public IteratorValidator(PhysicalOperator child, boolean repeatable) {
super(child);
setCost(child.getCost());
+this.isRepeatable = repeatable;
+ }
+
+ public IteratorValidator(PhysicalOperator child) {
+this(child, false);
}
@Override
@@ -36,7 +46,7 @@ public class IteratorValidator extends AbstractSingle{
@Override
protected PhysicalOperator getNewWithChild(PhysicalOperator child) {
-return new IteratorValidator(child);
+return new IteratorValidator(child, isRepeatable);
}
@Override
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/RowKeyJoinBatch.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/RowKeyJoinBatch.java
index 941f321..2910da5 100644
---
a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/RowKeyJoinBatch.java
+++
b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/RowKeyJoinBatch.java
@@ -132,6 +132,11 @@ public class RowKeyJoinBatch extends
AbstractRecordBatch implemen
return IterOutcome.OK;
}
+ if (rightUpstream == IterOutcome.NONE) {
+rkJoinState = RowKeyJoinState.DONE;
+state = BatchState.DONE;
+return rightUpstream;
+ }
rightUpstream = next(right);
logger.debug("right input IterOutcome: {}", rightUpstream);
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/validate/IteratorValidatorBatchIterator.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/validate/IteratorValidatorBatchIterator.java
index 1ea3895..5c70f5d 100644
---
a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/validate/IteratorValidatorBatchIterator.java
+++
b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/validate/IteratorValidatorBatchIterator.java
@@ -47,6 +47,9 @@ public class IteratorValidatorBatchIterator implements
CloseableRecordBatch {
/** For logging/debuggability only. */
private static volatile int instanceCount;
+ /** @see org.apache.drill.exec.physical.config.IteratorValidator */
+ private final boolean isRepeatable;
+
/** For logging/debuggability only. */
private final int instNum;
{
@@ -102,12 +105,17 @@ public class IteratorValidatorBatchIterator implements
CloseableRecordBatch {
*/
private boolean validateBatches;
- public IteratorValidatorBatchIterator(RecordBatch incoming) {
+ public IteratorValidatorBatchIterator(RecordBatch incoming, boolean
isRepeatable) {
this.incoming = incoming;
batchTypeName = incoming.getClass().getSimpleName();
+this.isRepeatable = isRepeatable;
// (Log construction and close() at same level to bracket instance's
activity.)
-logger.trace( "[#{}; on {}]: Being constructed.", instNum, batchTypeName);
+logger.trace( "[#{}; on {}; repeatable: {}]: Being constructed.", instNum,
batchTypeName, isRepeatable);
+ }
+
+ public IteratorValidatorBatchIterator(RecordBatch incoming) {
+this(incoming, false);
}
@@ -217,7 +225,7 @@ public class I
[drill] 02/04: DRILL-6876: Enable CircleCI builds for JDK 9-11
This is an automated email from the ASF dual-hosted git repository. amansinha pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/drill.git commit abc51d80095cf5df2de64a3acbf75bdd88bd1db5 Author: Volodymyr Vysotskyi AuthorDate: Fri Nov 16 15:43:27 2018 +0200 DRILL-6876: Enable CircleCI builds for JDK 9-11 close apache/drill#1558 --- .circleci/config.yml | 135 ++- 1 file changed, 133 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 1b179df..e2b1141 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -16,7 +16,7 @@ version: 2 general: jobs: - build: + build_jdk8: machine: enabled: true image: circleci/classic:latest @@ -33,7 +33,7 @@ jobs: # TODO: Could be removed, once Machine Executor image is updated https://github.com/circleci/image-builder/issues/140 # and the possibility of specifying Maven version is added https://github.com/circleci/image-builder/issues/143 command: - curl -fsSL https://git.io/vpDIf | bash -s -- 3.5.4 + curl -fsSL https://git.io/vpDIf | bash -s -- 3.6.0 - run: name: Update packages list command: @@ -49,3 +49,134 @@ jobs: # TODO: 4. Resolving memory issues without "SlowTest" and "UnlikelyTest" excludedGroups in the build command: mvn install -Drat.skip=false -Dlicense.skip=false -DmemoryMb=2560 -DdirectMemoryMb=4608 -DexcludedGroups="org.apache.drill.categories.SlowTest,org.apache.drill.categories.UnlikelyTest" + + build_jdk9: +machine: + enabled: true + image: circleci/classic:latest +parallelism: 1 + +working_directory: ~/drill + +steps: +- checkout + +- run: +name: Update packages list +command: + sudo apt-get update + +- run: +name: Install java 9 +command: + sudo apt-get -y install openjdk-9-jdk + +- run: +name: Set default java 9 +command: + sudo update-java-alternatives --set java-1.9.0-openjdk-amd64 + +- run: +name: Update maven version +command: + curl -fsSL https://git.io/vpDIf | bash -s -- 3.6.0 + +- run: +name: Install libaio1.so library for MySQL integration tests +command: + sudo apt-get install libaio1 libaio-dev +- run: +name: Drill project build +command: + mvn install -Drat.skip=false -Dlicense.skip=false -DmemoryMb=2560 -DdirectMemoryMb=4608 -DexcludedGroups="org.apache.drill.categories.SlowTest,org.apache.drill.categories.UnlikelyTest" + + build_jdk10: +machine: + enabled: true + image: circleci/classic:latest +parallelism: 1 + +working_directory: ~/drill + +steps: +- checkout + +- run: +name: Update packages list +command: + sudo apt-get update + +- run: +name: Install java 10 +command: + sudo apt-get -y install openjdk-10-jdk + +- run: +name: Set default java 10 +command: + sudo update-java-alternatives --set java-1.10.0-openjdk-amd64 + +- run: +name: Update maven version +command: + curl -fsSL https://git.io/vpDIf | bash -s -- 3.6.0 + +- run: +name: Install libaio1.so library for MySQL integration tests +command: + sudo apt-get install libaio1 libaio-dev +- run: +name: Drill project build +command: + mvn install -Drat.skip=false -Dlicense.skip=false -DmemoryMb=2560 -DdirectMemoryMb=4608 -DexcludedGroups="org.apache.drill.categories.SlowTest,org.apache.drill.categories.UnlikelyTest" + + build_jdk11: +machine: + enabled: true + image: circleci/classic:latest +parallelism: 1 + +working_directory: ~/drill + +steps: +- checkout + +- run: +name: Update packages list +command: + sudo apt-get update + +- run: +name: Install java 11 +command: + sudo apt-get -y install openjdk-11-jdk + +- run: +name: Set default java 11 +command: + sudo update-java-alternatives --set java-1.11.0-openjdk-amd64 + +- run: +name: Update maven version +command: + curl -fsSL https://git.io/vpDIf | bash -s -- 3.6.0 + +- run: +name: Install libaio1.so library for MySQL integration tests +command: + sudo apt-get install libaio1 libaio-dev +- run: +name: Drill project build +# Set forkCount to 1 since tests use more memory and memory limitations for CircleCI is reached +# for default value of forkCount. +command: + mvn install -Drat.skip=false -Dlicense.skip=false -DmemoryMb=2560 -DdirectMemoryMb=4608 -DforkCount=1 -Dex
[drill] 04/04: DRILL-6751: Upgrade Apache parent POM to version 21
This is an automated email from the ASF dual-hosted git repository.
amansinha pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git
commit 637354e91b4b597a0f1930a385b7f0bff0093a2a
Author: Vitalii Diravka
AuthorDate: Thu Nov 15 17:40:21 2018 -0800
DRILL-6751: Upgrade Apache parent POM to version 21
- Update apache.pom file version to 21 (with updating some maven plugins
versions)
- Include Drill's sources jars on assembly stage in (not
)
for properincluding jars with last apache-21.pom
- Separate "distro-assembly" to the two execution stages to avoid:
[WARNING] Assembly file:
/distribution/target/apache-drill-1.15.0-SNAPSHOT is not a regular
file (it may be a directory). It cannot be attached to the project build
for installation or deployment.
- Remove unsused / in assebly descriptor to avoid:
[WARNING] The following patterns were never triggered in this artifact
inclusion filter
- Update "maven-assembly-plugin" version
- Update "slf4j" version
- Update "mockito-core" version
- Update "bcpkix-jdk15on" (Bouncy Castle Cryptography APIs) version
close apache/drill#1561
---
distribution/pom.xml | 49 +++--
distribution/src/assemble/bin-assembly-dir.xml | 33 +++
distribution/src/assemble/bin-assembly.xml | 33 +++
.../src/assemble/{bin.xml => component.xml}| 227 ++---
drill-shaded/pom.xml | 3 +-
.../java-exec/src/test/resources/drill-udf/pom.xml | 3 -
pom.xml| 23 +--
7 files changed, 218 insertions(+), 153 deletions(-)
diff --git a/distribution/pom.xml b/distribution/pom.xml
index 4952ca5..4fde4cd 100644
--- a/distribution/pom.xml
+++ b/distribution/pom.xml
@@ -156,26 +156,40 @@
-
+
maven-assembly-plugin
-
-distro-assembly
-package
-
-single
-
-
-
-src/assemble/bin.xml
-
-false
-apache-drill-${project.version}
-gnu
-
-
+
+distro-assembly
+package
+
+ single
+
+
+
+src/assemble/bin-assembly.xml
+
+ false
+ apache-drill-${project.version}
+
+
+
+distro-assembly-not-attached
+package
+
+ single
+
+
+ false
+
+src/assemble/bin-assembly-dir.xml
+
+ false
+ apache-drill-${project.version}
+
+
-
+
org.apache.rat
apache-rat-plugin
@@ -192,7 +206,6 @@
-
diff --git a/distribution/src/assemble/bin-assembly-dir.xml
b/distribution/src/assemble/bin-assembly-dir.xml
new file mode 100644
index 000..f134962
--- /dev/null
+++ b/distribution/src/assemble/bin-assembly-dir.xml
@@ -0,0 +1,33 @@
+
+
+http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2";
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";
+
xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2
http://maven.apache.org/xsd/assembly-1.1.2.xsd";>
+ binary-release-directory
+
+dir
+
+ true
+
+component.xml
+
+
diff --git a/distribution/src/assemble/bin-assembly.xml
b/distribution/src/assemble/bin-assembly.xml
new file mode 100644
index 000..5cd3636
--- /dev/null
+++ b/distribution/src/assemble/bin-assembly.xml
@@ -0,0 +1,33 @@
+
+
+http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2";
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";
+
xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2
http://maven.apache.org/xsd/assembly-1.1.2.xsd";>
+ binary-release
+
+tar.gz
+
+ true
+
+component.xml
+
+
diff --git a/distribution/src/assemble/bin.xml
b/distribution/src/assemble/component.xml
similarity index 72%
rename from distribution/src/assemble/bin.xml
rename to distribution/src/assemble/component.xml
index 85a4d5b..2b60299 100644
--- a/distribution/src/assemble/bin.xml
+++ b/distribution/src/assemble/component.xml
@@ -18,27 +18,115 @@
limitations under the License.
-->
-http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2";
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";
-
xsi:schemaLocation="http://maven.apache.org/p
[drill] branch master updated (2dbd609 -> 637354e)
This is an automated email from the ASF dual-hosted git repository.
amansinha pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git.
from 2dbd609 Add Vitalii's GPG key
new f3eef38 DRILL-6882: Handle the cases where RowKeyJoin's left pipeline
being called multiple times.
new abc51d8 DRILL-6876: Enable CircleCI builds for JDK 9-11
new 7d37637 DRILL-4546: Only generate one zip archive when using
apache-release profile
new 637354e DRILL-6751: Upgrade Apache parent POM to version 21
The 4 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.circleci/config.yml | 135 +++-
distribution/pom.xml | 49 +++--
.../src/assemble/bin-assembly-dir.xml | 23 ++-
.../src/assemble/bin-assembly.xml | 23 ++-
.../src/assemble/{bin.xml => component.xml}| 227 ++---
drill-shaded/pom.xml | 3 +-
.../exec/physical/config/IteratorValidator.java| 14 +-
.../exec/physical/impl/join/RowKeyJoinBatch.java | 5 +
.../validate/IteratorValidatorBatchIterator.java | 20 +-
.../impl/validate/IteratorValidatorCreator.java| 2 +-
.../impl/validate/IteratorValidatorInjector.java | 33 ++-
.../java-exec/src/test/resources/drill-udf/pom.xml | 3 -
pom.xml| 41 ++--
src/main/resources/assemblies/source-assembly.xml | 99 -
14 files changed, 380 insertions(+), 297 deletions(-)
copy src/main/resources/checkstyle-suppressions.xml =>
distribution/src/assemble/bin-assembly-dir.xml (58%)
copy src/main/resources/checkstyle-suppressions.xml =>
distribution/src/assemble/bin-assembly.xml (59%)
rename distribution/src/assemble/{bin.xml => component.xml} (72%)
delete mode 100644 src/main/resources/assemblies/source-assembly.xml
[drill] 07/08: DRILL-6381: Address review comments (part 2): fix formatting issues and add javadoc.
This is an automated email from the ASF dual-hosted git repository.
amansinha pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git
commit 387bc4fefc40b645685439fd1da43f7223e5933c
Author: Aman Sinha
AuthorDate: Wed Oct 10 16:36:48 2018 -0700
DRILL-6381: Address review comments (part 2): fix formatting issues and add
javadoc.
---
.../planner/index/MapRDBFunctionalIndexInfo.java | 22 ++--
.../exec/planner/index/MapRDBIndexDescriptor.java | 4 +-
.../exec/store/mapr/db/MapRDBScanBatchCreator.java | 4 +-
.../exec/store/mapr/db/MapRDBSubScanSpec.java | 2 +-
.../store/mapr/db/json/JsonTableGroupScan.java | 17 ++-
.../db/json/JsonTableRangePartitionFunction.java | 4 +-
.../store/mapr/db/json/OjaiFunctionsProcessor.java | 2 +-
.../mapr/db/json/RestrictedJsonTableGroupScan.java | 8 +-
.../drill/exec/udf/mapr/db/DecodeFieldPath.java| 4 +-
.../drill/exec/udf/mapr/db/MatchesPlaceholder.java | 7 +-
.../exec/udf/mapr/db/NotMatchesPlaceholder.java| 7 +-
.../exec/udf/mapr/db/NotTypeOfPlaceholder.java | 7 +-
.../drill/exec/udf/mapr/db/SizeOfPlaceholder.java | 7 +-
.../drill/exec/udf/mapr/db/TypeOfPlaceholder.java | 7 +-
.../maprdb/tests/index/IndexHintPlanTest.java | 19 ++-
.../drill/maprdb/tests/index/IndexPlanTest.java| 135 +
.../drill/maprdb/tests/index/LargeTableGen.java| 34 +++---
.../maprdb/tests/index/LargeTableGenBase.java | 25 ++--
.../drill/maprdb/tests/index/StatisticsTest.java | 21 ++--
.../drill/maprdb/tests/index/TableIndexCmd.java| 26 ++--
.../drill/exec/physical/base/IndexGroupScan.java | 1 -
.../exec/physical/impl/join/RowKeyJoinBatch.java | 18 +--
.../exec/planner/index/DrillIndexDefinition.java | 22 ++--
.../exec/planner/index/DrillIndexDescriptor.java | 24 +++-
.../exec/planner/index/IndexConditionInfo.java | 44 ---
.../drill/exec/planner/index/IndexPlanUtils.java | 74 +--
.../drill/exec/planner/index/IndexSelector.java| 2 +-
.../drill/exec/planner/index/SimpleRexRemap.java | 18 +--
.../planner/index/rules/AbstractMatchFunction.java | 8 +-
.../planner/logical/DrillMergeProjectRule.java | 2 +-
30 files changed, 271 insertions(+), 304 deletions(-)
diff --git
a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/planner/index/MapRDBFunctionalIndexInfo.java
b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/planner/index/MapRDBFunctionalIndexInfo.java
index ec38636..67938f3 100644
---
a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/planner/index/MapRDBFunctionalIndexInfo.java
+++
b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/planner/index/MapRDBFunctionalIndexInfo.java
@@ -32,15 +32,15 @@ public class MapRDBFunctionalIndexInfo implements
FunctionalIndexInfo {
private boolean hasFunctionalField = false;
- //when we scan schemaPath in groupscan's columns, we check if this
column(schemaPath) should be rewritten to '$N',
- //When there are more than two functions on the same column in index,
CAST(a.b as INT), CAST(a.b as VARCHAR),
+ // When we scan schemaPath in groupscan's columns, we check if this
column(schemaPath) should be rewritten to '$N',
+ // When there are more than two functions on the same column in index,
CAST(a.b as INT), CAST(a.b as VARCHAR),
// then we should map SchemaPath a.b to a set of SchemaPath, e.g. $1, $2
private Map> columnToConvert;
// map of functional index expression to destination SchemaPath e.g. $N
private Map exprToConvert;
- //map of SchemaPath involved in a functional field
+ // map of SchemaPath involved in a functional field
private Map> pathsInExpr;
private Set newPathsForIndexedFunction;
@@ -52,7 +52,7 @@ public class MapRDBFunctionalIndexInfo implements
FunctionalIndexInfo {
columnToConvert = Maps.newHashMap();
exprToConvert = Maps.newHashMap();
pathsInExpr = Maps.newHashMap();
-//keep the order of new paths, it may be related to the naming policy
+// keep the order of new paths, it may be related to the naming policy
newPathsForIndexedFunction = Sets.newLinkedHashSet();
allPathsInFunction = Sets.newHashSet();
init();
@@ -60,15 +60,15 @@ public class MapRDBFunctionalIndexInfo implements
FunctionalIndexInfo {
private void init() {
int count = 0;
-for(LogicalExpression indexedExpr : indexDesc.getIndexColumns()) {
- if( !(indexedExpr instanceof SchemaPath) ) {
+for (LogicalExpression indexedExpr : indexDesc.getIndexColumns()) {
+ if (!(indexedExpr instanceof SchemaPath)) {
hasFunctionalField = true;
SchemaPath functionalFieldPath = SchemaPath.getSimplePath("$"+count);
newPathsForIndexedFunction.add(functionalFieldPath);
-//now we handle only cast expression
-if(indexedExpr instanceof CastExpression)
[drill] 05/08: DRILL-6381: (Part 5) Update Javadoc for a few interfaces.
This is an automated email from the ASF dual-hosted git repository.
amansinha pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git
commit 3c2f9ab6c0ed290bed3bc440d72a670bbe512b5a
Author: Aman Sinha
AuthorDate: Thu Sep 13 03:55:47 2018 -0700
DRILL-6381: (Part 5) Update Javadoc for a few interfaces.
---
.../org/apache/drill/exec/physical/base/DbSubScan.java | 4 ++--
.../drill/exec/planner/index/IndexCollection.java | 4 +++-
.../drill/exec/planner/index/IndexDefinition.java | 18 --
.../drill/exec/planner/index/IndexDescriptor.java | 15 +++
4 files changed, 36 insertions(+), 5 deletions(-)
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/DbSubScan.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/DbSubScan.java
index 874468d..1977257 100644
---
a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/DbSubScan.java
+++
b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/DbSubScan.java
@@ -32,8 +32,8 @@ public interface DbSubScan extends SubScan {
boolean isRestrictedSubScan();
/**
- * For a restricted sub-scan, this method allows associating a (hash)join
instance. A subscan within a minor
- * fragment must have a corresponding (hash)join batch instance from which
it will retrieve its set of
+ * For a restricted sub-scan, this method allows associating a rowkey join
instance. A subscan within a minor
+ * fragment must have a corresponding rowkey join batch instance from which
it will retrieve its set of
* rowkeys to perform the restricted scan.
* @param batch
*/
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/index/IndexCollection.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/index/IndexCollection.java
index 9b4d170..f5ab173 100644
---
a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/index/IndexCollection.java
+++
b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/index/IndexCollection.java
@@ -21,7 +21,9 @@ import org.apache.calcite.rex.RexNode;
import org.apache.drill.common.expression.SchemaPath;
import org.apache.drill.exec.physical.base.IndexGroupScan;
-// Interface used to describe an index collection
+/**
+ * Interface used to describe an index collection
+ */
public interface IndexCollection extends Iterable {
/**
* Types of an index collections: NATIVE_SECONDARY_INDEX_COLLECTION,
EXTERNAL_SECONDARY_INDEX_COLLECTION
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/index/IndexDefinition.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/index/IndexDefinition.java
index 995d23c..bfbccc2 100644
---
a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/index/IndexDefinition.java
+++
b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/index/IndexDefinition.java
@@ -26,7 +26,9 @@ import java.util.Collection;
import java.util.List;
import java.util.Map;
-// Interface used to define an index,
+/**
+ * Top level interface used to define an index.
+ */
public interface IndexDefinition {
/**
* Types of an index: PRIMARY_KEY_INDEX, NATIVE_SECONDARY_INDEX,
EXTERNAL_SECONDARY_INDEX
@@ -87,13 +89,25 @@ public interface IndexDefinition {
*/
IndexType getIndexType();
-
+ /**
+ * Get the list of columns in the index key.
+ */
List getIndexColumns();
+ /**
+ * Get the list of columns that are in the 'included' or 'covered' fields.
+ */
List getNonIndexColumns();
+ /**
+ * Get the collation property (physical ordering) of the index.
+ */
RelCollation getCollation();
+ /**
+ * Get a mapping of the LogicalExpresion/SchemaPath to its field collation
+ * - e.g Ascending/Descending, Nulls First/Nulls Last
+ */
Map getCollationMap();
/**
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/index/IndexDescriptor.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/index/IndexDescriptor.java
index f355285..d43ba81 100644
---
a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/index/IndexDescriptor.java
+++
b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/index/IndexDescriptor.java
@@ -58,11 +58,26 @@ public interface IndexDescriptor extends IndexDefinition {
*/
boolean supportsFullTextSearch();
+ /**
+ * Get the functional index information associated with this index
(Functional indexes are
+ * indexes involving expressions e.g CAST(a as INT).
+ */
FunctionalIndexInfo getFunctionalInfo();
+ /**
+ * Get the total cost of index access (I/O, CPU) in the context of the
current query
+ * @param indexProps properties (metrics) of a single index in the context
of current query
+ * @param planner Planner instance
+ * @param numProjectedFields Number of projected fields
+ * @param primaryGroupScan Prima
[drill] 01/08: DRILL-6381: (Part 1) Secondary Index framework
This is an automated email from the ASF dual-hosted git repository.
amansinha pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git
commit 0abcbe3f36bf6c0a2b5fe07a778d201ead8dd2ce
Author: rebase
AuthorDate: Mon Feb 12 14:10:56 2018 -0800
DRILL-6381: (Part 1) Secondary Index framework
 1. Secondary Index planning interfaces and abstract classes like
DBGroupScan, DbSubScan, IndexDecriptor etc.
 2. Statistics and Cost model interfaces/classes: PluginCost, Statistics,
StatisticsPayload, AbstractIndexStatistics
 3. ScanBatch and RecordReader to support repeatable scan
 4. Secondary Index execution related interfaces: RangePartitionSender,
RowKeyJoin, PartitionFunction
5. MD-3979: Query using cast index plan fails with NPE
Co-authored-by: Aman Sinha
Co-authored-by: chunhui-shi
Co-authored-by: Gautam Parai
Co-authored-by: Padma Penumarthy
Co-authored-by: Hanumath Rao Maduri
Conflicts:
exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/ScanBatch.java
exec/java-exec/src/main/java/org/apache/drill/exec/planner/common/DrillRelOptUtil.java
exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillTable.java
protocol/src/main/java/org/apache/drill/exec/proto/UserBitShared.java
protocol/src/main/java/org/apache/drill/exec/proto/beans/CoreOperatorType.java
protocol/src/main/protobuf/UserBitShared.proto
---
.../java/org/apache/drill/exec/ExecConstants.java | 2 +
.../exec/physical/base/AbstractDbGroupScan.java| 95 +++
.../exec/physical/base/AbstractDbSubScan.java | 37 +++
.../physical/base/AbstractPhysicalVisitor.java | 12 +
.../drill/exec/physical/base/DbGroupScan.java | 129 +
.../apache/drill/exec/physical/base/DbSubScan.java | 43 +++
.../drill/exec/physical/base/IndexGroupScan.java | 76 ++
.../drill/exec/physical/base/PhysicalVisitor.java | 4 +
...{RangeSender.java => RangePartitionSender.java} | 52 ++--
.../apache/drill/exec/physical/impl/ScanBatch.java | 15 +-
.../drill/exec/physical/impl/join/RowKeyJoin.java | 79 ++
.../drill/exec/planner/common/DrillRelOptUtil.java | 17 ++
.../apache/drill/exec/planner/cost/PluginCost.java | 79 ++
.../planner/index/AbstractIndexCollection.java | 96 +++
.../planner/index/AbstractIndexDescriptor.java | 74 ++
.../planner/index/AbstractIndexStatistics.java | 51
.../drill/exec/planner/index/CollationContext.java | 37 +++
.../exec/planner/index/DrillIndexCollection.java | 75 ++
.../exec/planner/index/DrillIndexDefinition.java | 278
.../exec/planner/index/DrillIndexDescriptor.java | 110
.../exec/planner/index/FunctionalIndexInfo.java| 85 ++
.../drill/exec/planner/index/IndexCallContext.java | 76 ++
.../drill/exec/planner/index/IndexCollection.java | 99 +++
.../drill/exec/planner/index/IndexDefinition.java | 105
.../drill/exec/planner/index/IndexDescriptor.java | 68 +
.../drill/exec/planner/index/IndexDiscover.java| 23 ++
.../exec/planner/index/IndexDiscoverBase.java | 110
.../exec/planner/index/IndexDiscoverable.java | 37 +++
.../drill/exec/planner/index/IndexGroup.java | 63 +
.../drill/exec/planner/index/IndexProperties.java | 64 +
.../drill/exec/planner/index/IndexStatistics.java | 36 +++
.../exec/planner/index/IndexableExprMarker.java| 262 +++
.../index/InvalidIndexDefinitionException.java | 27 ++
.../drill/exec/planner/index/Statistics.java | 66 +
.../exec/planner/index/StatisticsPayload.java | 24 ++
.../drill/exec/planner/logical/DrillTable.java | 4 +
.../exec/planner/physical/PartitionFunction.java | 56
.../drill/exec/record/AbstractRecordBatch.java | 2 +-
.../drill/exec/store/AbstractRecordReader.java | 5 +
.../org/apache/drill/exec/store/RecordReader.java | 8 +
.../drill/exec/util/EncodedSchemaPathSet.java | 291 +
.../org/apache/drill/exec/proto/UserBitShared.java | 75 +++---
.../drill/exec/proto/beans/CoreOperatorType.java | 8 +-
protocol/src/main/protobuf/UserBitShared.proto | 3 +-
44 files changed, 2893 insertions(+), 65 deletions(-)
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java
index 21e16eb..cb0fc5c 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java
@@ -807,6 +807,8 @@ public final class ExecConstants {
*/
public static final String ENABLE_ITERATOR_VALIDATION =
"drill.exec.debug.validate_iterators";
+ public static final String QUERY_ROWKEYJOIN_BATCHSIZE_KEY =
"exec.query.ro
[drill] 08/08: DRILL-6381: Address code review comments (part 3).
This is an automated email from the ASF dual-hosted git repository.
amansinha pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git
commit 7571d52eab9b961687df7d4fb845d0a297b228bb
Author: Aman Sinha
AuthorDate: Sat Oct 13 23:38:17 2018 -0700
DRILL-6381: Address code review comments (part 3).
DRILL-6381: Add missing joinControl logic for INTERSECT_DISTINCT.
- Modified HashJoin's probe phase to process INTERSECT_DISTINCT.
- NOTE: For build phase, the functionality will be same as for SemiJoin
when it is added later.
DRILL-6381: Address code review comment for intersect_distinct.
DRILL-6381: Rebase on latest master and fix compilation issues.
DRILL-6381: Generate protobuf files for C++ native client.
DRILL-6381: Use shaded Guava classes. Add more comments and Javadoc.
---
.../planner/index/MapRDBFunctionalIndexInfo.java | 4 +-
.../exec/planner/index/MapRDBIndexDiscover.java| 12 ++---
.../drill/exec/store/mapr/PluginConstants.java | 3 ++
.../exec/store/mapr/db/MapRDBFormatPlugin.java | 6 ++-
.../store/mapr/db/MapRDBPushFilterIntoScan.java| 17 ---
.../store/mapr/db/MapRDBPushLimitIntoScan.java | 2 +-
.../store/mapr/db/MapRDBPushProjectIntoScan.java | 35 ++---
.../mapr/db/MapRDBRestrictedScanBatchCreator.java | 4 +-
.../db/json/JsonTableRangePartitionFunction.java | 4 +-
.../store/mapr/db/json/MaprDBJsonRecordReader.java | 2 +-
.../mapr/db/json/RestrictedJsonTableGroupScan.java | 6 +--
contrib/native/client/src/protobuf/BitData.pb.cc | 55 ++---
contrib/native/client/src/protobuf/BitData.pb.h| 36 +-
.../native/client/src/protobuf/UserBitShared.pb.cc | 57 +++---
.../native/client/src/protobuf/UserBitShared.pb.h | 7 +--
.../exec/store/hbase/HBasePushFilterIntoScan.java | 2 +-
.../store/kafka/KafkaPushDownFilterIntoScan.java | 4 +-
.../store/mongo/MongoPushDownFilterForScan.java| 2 +-
.../apache/drill/exec/physical/impl/ScanBatch.java | 17 +++
.../exec/physical/impl/common/HashPartition.java | 10 ++--
.../physical/impl/join/HashJoinProbeTemplate.java | 24 +++--
.../exec/physical/impl/join/RowKeyJoinBatch.java | 6 +++
.../physical/impl/partitionsender/Partitioner.java | 3 ++
.../RangePartitionRecordBatch.java | 7 +++
.../exec/planner/common/DrillScanRelBase.java | 2 +-
.../index/InvalidIndexDefinitionException.java | 4 ++
.../generators/IndexIntersectPlanGenerator.java| 18 +++
.../planner/logical/DrillMergeProjectRule.java | 8 +++
.../planner/physical/BroadcastExchangePrel.java| 5 +-
.../planner/physical/ConvertCountToDirectScan.java | 5 +-
.../drill/exec/planner/physical/ScanPrel.java | 9
.../drill/exec/planner/physical/ScanPrule.java | 2 +-
.../InfoSchemaPushFilterIntoRecordGenerator.java | 2 +-
.../exec/store/parquet/ParquetPushDownFilter.java | 2 +-
.../physical/impl/common/HashPartitionTest.java| 2 +-
35 files changed, 248 insertions(+), 136 deletions(-)
diff --git
a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/planner/index/MapRDBFunctionalIndexInfo.java
b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/planner/index/MapRDBFunctionalIndexInfo.java
index 67938f3..564a037 100644
---
a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/planner/index/MapRDBFunctionalIndexInfo.java
+++
b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/planner/index/MapRDBFunctionalIndexInfo.java
@@ -17,8 +17,8 @@
*/
package org.apache.drill.exec.planner.index;
-import com.google.common.collect.Maps;
-import com.google.common.collect.Sets;
+import org.apache.drill.shaded.guava.com.google.common.collect.Maps;
+import org.apache.drill.shaded.guava.com.google.common.collect.Sets;
import org.apache.drill.common.expression.CastExpression;
import org.apache.drill.common.expression.LogicalExpression;
import org.apache.drill.common.expression.SchemaPath;
diff --git
a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/planner/index/MapRDBIndexDiscover.java
b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/planner/index/MapRDBIndexDiscover.java
index c231e11..aed3e04 100644
---
a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/planner/index/MapRDBIndexDiscover.java
+++
b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/planner/index/MapRDBIndexDiscover.java
@@ -100,7 +100,7 @@ public class MapRDBIndexDiscover extends IndexDiscoverBase
implements IndexDisco
for (IndexDesc idx : indexes) {
DrillIndexDescriptor hbaseIdx = buildIndexDescriptor(tableName, idx);
if (hbaseIdx == null) {
- //not able to build a valid index based on the index info from MFS
+ // not able to build a valid index based on the index info from MFS
logger.error("Not able to build
[drill] branch master updated (61e8b46 -> 7571d52)
This is an automated email from the ASF dual-hosted git repository. amansinha pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/drill.git. from 61e8b46 DRILL-6715: Update descriptions for System Options table new 0abcbe3 DRILL-6381: (Part 1) Secondary Index framework new a4f62e9 DRILL-6381: (Part 2) MapRDB plugin update to 6.0.1 new 12f57f0 DRILL-6381: (Part 3) Planner and Execution implementation to support Secondary Indexes new 2a9e51f DRILL-6381: (Part 4) Enhance MapR-DB plugin to support querying secondary indexes new 3c2f9ab DRILL-6381: (Part 5) Update Javadoc for a few interfaces. new 5fa9c80 DRILL-6381: Address code review comments. new 387bc4f DRILL-6381: Address review comments (part 2): fix formatting issues and add javadoc. new 7571d52 DRILL-6381: Address code review comments (part 3). The 8 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: contrib/format-maprdb/pom.xml | 89 ++ .../planner/index/MapRDBFunctionalIndexInfo.java | 168 ++ .../exec/planner/index/MapRDBIndexDescriptor.java | 222 +++ .../exec/planner/index/MapRDBIndexDiscover.java| 376 + .../drill/exec/planner/index/MapRDBStatistics.java | 1006 .../planner/index/MapRDBStatisticsPayload.java | 44 +- .../drill/exec/store/mapr/PluginConstants.java | 93 ++ .../drill/exec/store/mapr/PluginErrorHandler.java | 50 + .../drill/exec/store/mapr/db/MapRDBCost.java | 104 ++ .../exec/store/mapr/db/MapRDBFormatMatcher.java| 46 +- .../exec/store/mapr/db/MapRDBFormatPlugin.java | 69 +- .../store/mapr/db/MapRDBFormatPluginConfig.java|4 + .../drill/exec/store/mapr/db/MapRDBGroupScan.java | 83 +- .../store/mapr/db/MapRDBPushFilterIntoScan.java| 20 +- .../store/mapr/db/MapRDBPushLimitIntoScan.java | 203 +++ .../store/mapr/db/MapRDBPushProjectIntoScan.java | 142 ++ java => MapRDBRestrictedScanBatchCreator.java} | 41 +- .../exec/store/mapr/db/MapRDBScanBatchCreator.java |6 +- .../drill/exec/store/mapr/db/MapRDBSubScan.java| 23 +- .../exec/store/mapr/db/MapRDBSubScanSpec.java | 54 +- .../drill/exec/store/mapr/db/MapRDBTableCache.java | 232 +++ .../store/mapr/db/RestrictedMapRDBSubScan.java | 80 + .../store/mapr/db/RestrictedMapRDBSubScanSpec.java | 191 +++ .../store/mapr/db/binary/BinaryTableGroupScan.java | 34 +- .../store/mapr/db/json/AllTextValueWriter.java | 80 + .../mapr/db/json/CompareFunctionsProcessor.java| 15 +- .../mapr/db/json/DocumentReaderVectorWriter.java | 42 + .../exec/store/mapr/db/json/FieldPathHelper.java | 75 + .../mapr/db/json/FieldTransferVectorWriter.java| 49 + .../store/mapr/db/json/IdOnlyVectorWriter.java | 65 + .../store/mapr/db/json/JsonConditionBuilder.java | 59 +- .../exec/store/mapr/db/json/JsonScanSpec.java | 93 +- .../exec/store/mapr/db/json/JsonSubScanSpec.java | 69 +- .../store/mapr/db/json/JsonTableGroupScan.java | 676 +++- .../db/json/JsonTableRangePartitionFunction.java | 237 +++ .../store/mapr/db/json/MaprDBJsonRecordReader.java | 538 +++ .../mapr/db/json/NumbersAsDoubleValueWriter.java | 51 + .../store/mapr/db/json/OjaiFunctionsProcessor.java | 214 +++ .../exec/store/mapr/db/json/OjaiValueWriter.java | 194 +++ .../db/json/ProjectionPassthroughVectorWriter.java | 83 + .../mapr/db/json/RestrictedJsonRecordReader.java | 248 +++ .../mapr/db/json/RestrictedJsonTableGroupScan.java | 184 +++ .../store/mapr/db/json/RowCountVectorWriter.java | 29 +- .../exec/udf/mapr/db/ConditionPlaceholder.java | 54 + .../drill/exec/udf/mapr/db/DecodeFieldPath.java| 65 + .../drill/exec/udf/mapr/db/MatchesPlaceholder.java | 53 + .../exec/udf/mapr/db/NotMatchesPlaceholder.java| 53 + .../exec/udf/mapr/db/NotTypeOfPlaceholder.java | 53 + .../drill/exec/udf/mapr/db/SizeOfPlaceholder.java | 54 + .../drill/exec/udf/mapr/db/TypeOfPlaceholder.java | 53 + .../src/main/resources/drill-module.conf | 20 +- .../mapr/drill/maprdb/tests/MaprDBTestsSuite.java | 63 +- .../maprdb/tests/index/IndexHintPlanTest.java | 166 ++ .../drill/maprdb/tests/index/IndexPlanTest.java| 1690 .../drill/maprdb/tests/index/LargeTableGen.java| 172 ++ .../maprdb/tests/index/LargeTableGenBase.java | 185 +++ .../drill/maprdb/tests/index/StatisticsTest.java | 108 ++ .../drill/maprdb/tests/index/TableIndexCmd.java| 123 ++ .../mapr/drill/maprdb/tests/json/BaseJsonTest.java | 13 + .../maprdb/tests/json/TestEncodedFieldPaths.java | 128 ++ .../maprdb/tests/json/TestFieldPathHelper.java | 52 + .../
[drill] 06/08: DRILL-6381: Address code review comments.
This is an automated email from the ASF dual-hosted git repository.
amansinha pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git
commit 5fa9c808daef8ea70a39ac2248daa99055955b72
Author: Aman Sinha
AuthorDate: Mon Oct 1 12:06:39 2018 -0700
DRILL-6381: Address code review comments.
---
.../planner/index/MapRDBFunctionalIndexInfo.java | 5
.../exec/planner/index/MapRDBIndexDiscover.java| 14 ++-
.../store/mapr/db/RestrictedMapRDBSubScanSpec.java | 28 --
.../db/json/JsonTableRangePartitionFunction.java | 2 +-
.../drill/exec/physical/config/HashJoinPOP.java| 12 --
.../planner/index/AbstractIndexDescriptor.java | 6 +
.../drill/exec/planner/index/IndexDescriptor.java | 11 +++--
.../drill/exec/planner/index/IndexGroup.java | 6 +
.../generators/IndexIntersectPlanGenerator.java| 22 +
.../generators/NonCoveringIndexPlanGenerator.java | 21
.../physical/DrillDistributionTraitDef.java| 8 ++-
.../exec/planner/physical/RowKeyJoinPrel.java | 7 --
12 files changed, 70 insertions(+), 72 deletions(-)
diff --git
a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/planner/index/MapRDBFunctionalIndexInfo.java
b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/planner/index/MapRDBFunctionalIndexInfo.java
index 01561a3..ec38636 100644
---
a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/planner/index/MapRDBFunctionalIndexInfo.java
+++
b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/planner/index/MapRDBFunctionalIndexInfo.java
@@ -138,6 +138,11 @@ public class MapRDBFunctionalIndexInfo implements
FunctionalIndexInfo {
}
/**
+ * Suppose the index key has functions (rather than plain columns): CAST(a
as int), CAST(b as varchar(10)),
+ * then we want to maintain a mapping of the logical expression of that
function to the schema path of the
+ * base column involved in the function. In this example map has 2 entries:
+ * CAST(a as int) --> 'a'
+ * CAST(b as varchar(10)) --> 'b'
* @return the map of indexed expression --> the involved schema paths in a
indexed expression
*/
public Map> getPathsInFunctionExpr() {
diff --git
a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/planner/index/MapRDBIndexDiscover.java
b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/planner/index/MapRDBIndexDiscover.java
index e1b8a61..c231e11 100644
---
a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/planner/index/MapRDBIndexDiscover.java
+++
b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/planner/index/MapRDBIndexDiscover.java
@@ -37,6 +37,7 @@ import org.apache.drill.common.expression.parser.ExprLexer;
import org.apache.drill.common.expression.parser.ExprParser;
import org.apache.drill.common.types.TypeProtos;
import org.apache.drill.common.types.Types;
+import org.apache.drill.common.util.DrillFileUtils;
import org.apache.drill.exec.physical.base.AbstractDbGroupScan;
import org.apache.drill.exec.physical.base.GroupScan;
import org.apache.drill.exec.planner.common.DrillScanRelBase;
@@ -67,6 +68,7 @@ import java.util.Set;
public class MapRDBIndexDiscover extends IndexDiscoverBase implements
IndexDiscover {
static final String DEFAULT_STRING_CAST_LEN_STR = "256";
+ static final String FIELD_DELIMITER = ":";
public MapRDBIndexDiscover(GroupScan inScan, DrillScanRelBase scanRel) {
super((AbstractDbGroupScan) inScan, scanRel);
@@ -78,14 +80,14 @@ public class MapRDBIndexDiscover extends IndexDiscoverBase
implements IndexDisco
@Override
public IndexCollection getTableIndex(String tableName) {
-//return getTableIndexFromCommandLine(tableName);
return getTableIndexFromMFS(tableName);
}
/**
- *
+ * For a given table name get the list of indexes defined on the table
according to the visibility of
+ * the indexes based on permissions.
* @param tableName
- * @return
+ * @return an IndexCollection representing the list of indexes for that table
*/
private IndexCollection getTableIndexFromMFS(String tableName) {
try {
@@ -120,8 +122,8 @@ public class MapRDBIndexDiscover extends IndexDiscoverBase
implements IndexDisco
FileSelection deriveFSSelection(DrillFileSystem fs, IndexDescriptor idxDesc)
throws IOException {
String tableName = idxDesc.getTableName();
-String[] tablePath = tableName.split("/");
-String tableParent = tableName.substring(0, tableName.lastIndexOf("/"));
+String[] tablePath = tableName.split(DrillFileUtils.SEPARATOR);
+String tableParent = tableName.substring(0,
tableName.lastIndexOf(DrillFileUtils.SEPARATOR));
return FileSelection.create(fs, tableParent, tablePath[tablePath.length -
1], false);
}
@@ -318,7 +320,7 @@ public class MapRDBI
[drill] 03/04: DRILL-6783: CAST string literal as INTERVAL MONTH/YEAR works inconsistently when selecting from a table with multiple rows
This is an automated email from the ASF dual-hosted git repository.
amansinha pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git
commit 2aede3a1cd1532fbcd0a622b49a3b9fce69979ef
Author: Bohdan Kazydub
AuthorDate: Mon Oct 8 16:50:22 2018 +0300
DRILL-6783: CAST string literal as INTERVAL MONTH/YEAR works inconsistently
when selecting from a table with multiple rows
close apache/drill#1496
---
.../drill/exec/fn/impl/TestCastFunctions.java | 38 +-
.../main/codegen/templates/FixedValueVectors.java | 3 +-
2 files changed, 39 insertions(+), 2 deletions(-)
diff --git
a/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestCastFunctions.java
b/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestCastFunctions.java
index cb0d022..7768909 100644
---
a/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestCastFunctions.java
+++
b/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestCastFunctions.java
@@ -19,14 +19,21 @@ package org.apache.drill.exec.fn.impl;
import java.math.BigDecimal;
import java.time.LocalDate;
+import java.util.HashSet;
import java.util.List;
import java.util.Map;
+import java.util.Set;
import org.apache.drill.categories.SqlFunctionTest;
import org.apache.drill.categories.UnlikelyTest;
import org.apache.drill.common.exceptions.UserRemoteException;
+import org.apache.drill.common.expression.SchemaPath;
import org.apache.drill.exec.planner.physical.PlannerSettings;
+import org.apache.drill.exec.record.RecordBatchLoader;
+import org.apache.drill.exec.rpc.user.QueryDataBatch;
+import org.apache.drill.exec.vector.IntervalYearVector;
import org.apache.drill.test.BaseTestQuery;
+import org.junit.Assert;
import org.junit.Rule;
import org.junit.Test;
import org.junit.experimental.categories.Category;
@@ -39,6 +46,7 @@ import
org.apache.drill.shaded.guava.com.google.common.collect.Maps;
import mockit.integration.junit4.JMockit;
import static org.hamcrest.CoreMatchers.containsString;
+import static org.hamcrest.CoreMatchers.hasItem;
@RunWith(JMockit.class)
@Category({UnlikelyTest.class, SqlFunctionTest.class})
@@ -664,4 +672,32 @@ public class TestCastFunctions extends BaseTestQuery {
test(query);
}
-}
\ No newline at end of file
+
+ @Test // DRILL-6783
+ public void testCastVarCharIntervalYear() throws Exception {
+String query = "select cast('P31M' as interval month) as i from
cp.`employee.json` limit 10";
+List result = testSqlWithResults(query);
+RecordBatchLoader loader = new
RecordBatchLoader(getDrillbitContext().getAllocator());
+
+QueryDataBatch b = result.get(0);
+loader.load(b.getHeader().getDef(), b.getData());
+
+IntervalYearVector vector = (IntervalYearVector)
loader.getValueAccessorById(
+ IntervalYearVector.class,
+
loader.getValueVectorId(SchemaPath.getCompoundPath("i")).getFieldIds())
+.getValueVector();
+
+Set resultSet = new HashSet<>();
+for (int i = 0; i < loader.getRecordCount(); i++) {
+ String displayValue =
vector.getAccessor().getAsStringBuilder(i).toString();
+ resultSet.add(displayValue);
+}
+
+Assert.assertEquals(
+"Casting literal string as INTERVAL should yield the same result for
each row", 1, resultSet.size());
+Assert.assertThat(resultSet, hasItem("2 years 7 months"));
+
+b.release();
+loader.clear();
+ }
+}
diff --git a/exec/vector/src/main/codegen/templates/FixedValueVectors.java
b/exec/vector/src/main/codegen/templates/FixedValueVectors.java
index a98aa66..508e484 100644
--- a/exec/vector/src/main/codegen/templates/FixedValueVectors.java
+++ b/exec/vector/src/main/codegen/templates/FixedValueVectors.java
@@ -532,7 +532,8 @@ public final class ${minor.class}Vector extends
BaseDataValueVector implements F
}
public StringBuilder getAsStringBuilder(int index) {
- return DateUtilities.intervalYearStringBuilder(data.getInt(index));
+ int value = get(index);
+ return DateUtilities.intervalYearStringBuilder(value);
}
<#elseif minor.class == "Time">
[drill] 02/04: DRILL-6777: Setup CircleCI configs for Drill
This is an automated email from the ASF dual-hosted git repository.
amansinha pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git
commit 484bcfb1e237b6f8f9a244d344f36dcb86dedb61
Author: Vitalii Diravka
AuthorDate: Tue Sep 11 14:39:13 2018 +0300
DRILL-6777: Setup CircleCI configs for Drill
- adding .circleci/config.yml to the project to launch CircleCI
- custom memory parameters
- usage of CircleCI machine
- excluding "SlowTest" and "UnlikelyTest" groups
- update maven version
- adding libaio.so library to solve MySQL integration tests
- update com.jcabi:jcabi-mysql-maven-plugin library version
- TODO descriptions for the future enhancements of CircleCI build for Drill
close apache/drill#1493
---
.circleci/config.yml | 51 ++
contrib/storage-jdbc/pom.xml | 2 +-
.../exec/store/jdbc/TestJdbcPluginWithMySQLIT.java | 1 +
3 files changed, 53 insertions(+), 1 deletion(-)
diff --git a/.circleci/config.yml b/.circleci/config.yml
new file mode 100644
index 000..1b179df
--- /dev/null
+++ b/.circleci/config.yml
@@ -0,0 +1,51 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+version: 2
+general:
+jobs:
+ build:
+machine:
+ enabled: true
+ image: circleci/classic:latest
+parallelism: 1 # TODO: 1. Configuring Parallel Jobs for Running Tests
https://circleci.com/docs/2.0/parallelism-faster-jobs/
+
+
+working_directory: ~/drill
+
+steps:
+- checkout
+
+- run:
+name: Update maven version
+# TODO: Could be removed, once Machine Executor image is updated
https://github.com/circleci/image-builder/issues/140
+# and the possibility of specifying Maven version is added
https://github.com/circleci/image-builder/issues/143
+command:
+ curl -fsSL https://git.io/vpDIf | bash -s -- 3.5.4
+- run:
+name: Update packages list
+command:
+ sudo apt-get update
+- run:
+name: Install libaio1.so library for MySQL integration tests
+command:
+ sudo apt-get install libaio1 libaio-dev
+- run:
+name: Drill project build
+# TODO: 2. Optimizing Maven Builds on CircleCI -
https://circleci.com/blog/optimizing-maven-builds-on-circleci/
+# TODO: 3. Caching Dependencies -
https://circleci.com/docs/2.0/caching/
+# TODO: 4. Resolving memory issues without "SlowTest" and
"UnlikelyTest" excludedGroups in the build
+command:
+ mvn install -Drat.skip=false -Dlicense.skip=false -DmemoryMb=2560
-DdirectMemoryMb=4608
-DexcludedGroups="org.apache.drill.categories.SlowTest,org.apache.drill.categories.UnlikelyTest"
diff --git a/contrib/storage-jdbc/pom.xml b/contrib/storage-jdbc/pom.xml
index a032e69..7de89f0 100755
--- a/contrib/storage-jdbc/pom.xml
+++ b/contrib/storage-jdbc/pom.xml
@@ -213,7 +213,7 @@
com.jcabi
jcabi-mysql-maven-plugin
-0.8.2
+0.9
mysql-test
diff --git
a/contrib/storage-jdbc/src/test/java/org/apache/drill/exec/store/jdbc/TestJdbcPluginWithMySQLIT.java
b/contrib/storage-jdbc/src/test/java/org/apache/drill/exec/store/jdbc/TestJdbcPluginWithMySQLIT.java
index 1640913..0f377e3 100644
---
a/contrib/storage-jdbc/src/test/java/org/apache/drill/exec/store/jdbc/TestJdbcPluginWithMySQLIT.java
+++
b/contrib/storage-jdbc/src/test/java/org/apache/drill/exec/store/jdbc/TestJdbcPluginWithMySQLIT.java
@@ -26,6 +26,7 @@ import org.junit.experimental.categories.Category;
/**
* JDBC storage plugin tests against MySQL.
+ * Note: it requires libaio.so library in the system
*/
@Category(JdbcStorageTest.class)
public class TestJdbcPluginWithMySQLIT extends PlanTestBase {
[drill] branch master updated (734c460 -> ec6acc7)
This is an automated email from the ASF dual-hosted git repository. amansinha pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/drill.git. from 734c460 DRILL-3988: Expose Drill built-in functions & UDFs in a system table (#1483) new 0a3cfde DRILL-6473: Update MapR Hive new 484bcfb DRILL-6777: Setup CircleCI configs for Drill new 2aede3a DRILL-6783: CAST string literal as INTERVAL MONTH/YEAR works inconsistently when selecting from a table with multiple rows new ec6acc7 DRILL-6775: The schema for empty output is not shown in Drill Web UI The 4 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .circleci/config.yml | 51 ++ .../store/hive/readers/HiveAbstractReader.java | 19 +++- contrib/storage-hive/hive-exec-shade/pom.xml | 3 ++ contrib/storage-jdbc/pom.xml | 2 +- .../exec/store/jdbc/TestJdbcPluginWithMySQLIT.java | 1 + .../drill/exec/server/rest/QueryWrapper.java | 5 --- .../drill/exec/server/rest/WebUserConnection.java | 8 ++-- .../drill/exec/fn/impl/TestCastFunctions.java | 38 +++- .../main/codegen/templates/FixedValueVectors.java | 3 +- pom.xml| 4 +- 10 files changed, 106 insertions(+), 28 deletions(-) create mode 100644 .circleci/config.yml
[drill] 01/04: DRILL-6473: Update MapR Hive
This is an automated email from the ASF dual-hosted git repository.
amansinha pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git
commit 0a3cfdebd25ea4186c2a2b75524c34323984ffca
Author: Bohdan Kazydub
AuthorDate: Mon May 21 19:11:23 2018 +0300
DRILL-6473: Update MapR Hive
close apache/drill#1307
---
.../exec/store/hive/readers/HiveAbstractReader.java | 19 +--
contrib/storage-hive/hive-exec-shade/pom.xml | 3 +++
pom.xml | 4 ++--
3 files changed, 10 insertions(+), 16 deletions(-)
diff --git
a/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/readers/HiveAbstractReader.java
b/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/readers/HiveAbstractReader.java
index ba1cd30..5ed6c3b 100644
---
a/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/readers/HiveAbstractReader.java
+++
b/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/readers/HiveAbstractReader.java
@@ -25,6 +25,7 @@ import java.util.List;
import java.util.Properties;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
+import java.util.stream.Collectors;
import
org.apache.drill.shaded.guava.com.google.common.util.concurrent.ListenableFuture;
import io.netty.buffer.DrillBuf;
@@ -203,20 +204,10 @@ public abstract class HiveAbstractReader extends
AbstractRecordReader {
}
}
}
- ColumnProjectionUtils.appendReadColumns(job, columnIds);
-
- // TODO: Use below overloaded method instead of above simpler version of
it, once Hive client dependencies
- // (from all profiles) will be updated to 2.3 version or above
-// ColumnProjectionUtils.appendReadColumns(job, columnIds,
selectedColumnNames,
-// Lists.newArrayList(Iterables.transform(getColumns(), new
Function()
-// {
-//@Nullable
-//@Override
-//public String apply(@Nullable SchemaPath path)
-//{
-// return path.getRootSegmentPath();
-//}
-// })));
+ List paths = getColumns().stream()
+ .map(SchemaPath::getRootSegmentPath)
+ .collect(Collectors.toList());
+ ColumnProjectionUtils.appendReadColumns(job, columnIds,
selectedColumnNames, paths);
for (String columnName : selectedColumnNames) {
StructField fieldRef = finalOI.getStructFieldRef(columnName);
diff --git a/contrib/storage-hive/hive-exec-shade/pom.xml
b/contrib/storage-hive/hive-exec-shade/pom.xml
index 3ef1839..b857758 100644
--- a/contrib/storage-hive/hive-exec-shade/pom.xml
+++ b/contrib/storage-hive/hive-exec-shade/pom.xml
@@ -139,6 +139,9 @@
org.apache.hive:hive-exec
+
org/apache/parquet/schema/*
diff --git a/pom.xml b/pom.xml
index 5d00ee6..3a2d21e 100644
--- a/pom.xml
+++ b/pom.xml
@@ -2439,9 +2439,9 @@
mapr
true
-2.1.1-mapr-1710
+2.3.3-mapr-1808
1.1.1-mapr-1602-m7-5.2.0
-2.7.0-mapr-1707
+2.7.0-mapr-1808
3.4.11-mapr-1808
[drill] 04/04: DRILL-6775: The schema for empty output is not shown in Drill Web UI
This is an automated email from the ASF dual-hosted git repository.
amansinha pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git
commit ec6acc7ab6ee1a6eb326a4783189978bf55c0bcf
Author: Anton Gozhiy
AuthorDate: Tue Oct 9 15:59:16 2018 +0300
DRILL-6775: The schema for empty output is not shown in Drill Web UI
Removed an excess check for the result emptiness that prevented retrieval
of the column names.
close apache/drill#1498
---
.../main/java/org/apache/drill/exec/server/rest/QueryWrapper.java | 5 -
.../java/org/apache/drill/exec/server/rest/WebUserConnection.java | 8
2 files changed, 4 insertions(+), 9 deletions(-)
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/QueryWrapper.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/QueryWrapper.java
index 42427aa..4e09858 100644
---
a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/QueryWrapper.java
+++
b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/QueryWrapper.java
@@ -19,7 +19,6 @@ package org.apache.drill.exec.server.rest;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;
-import org.apache.drill.shaded.guava.com.google.common.collect.Maps;
import org.apache.drill.common.exceptions.UserException;
import org.apache.drill.common.exceptions.UserRemoteException;
@@ -115,10 +114,6 @@ public class QueryWrapper {
throw new UserRemoteException(webUserConnection.getError());
}
-if (webUserConnection.results.isEmpty()) {
- webUserConnection.results.add(Maps.newHashMap());
-}
-
// Return the QueryResult.
return new QueryResult(queryId, webUserConnection.columns,
webUserConnection.results);
}
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/WebUserConnection.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/WebUserConnection.java
index df4c8a5..9d6e7e4 100644
---
a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/WebUserConnection.java
+++
b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/WebUserConnection.java
@@ -75,20 +75,20 @@ public class WebUserConnection extends
AbstractDisposableUserClientConnection im
@Override
public void sendData(RpcOutcomeListener listener, QueryWritableBatch
result) {
-// Check if there is any data or not. There can be overflow here but
DrillBuf doesn't support allocating with
+// There can be overflow here but DrillBuf doesn't support allocating with
// bytes in long. Hence we are just preserving the earlier behavior and
logging debug log for the case.
final int dataByteCount = (int) result.getByteCount();
-if (dataByteCount <= 0) {
+if (dataByteCount < 0) {
if (logger.isDebugEnabled()) {
-logger.debug("Either no data received in this batch or there is
BufferOverflow in dataByteCount: {}",
+logger.debug("There is BufferOverflow in dataByteCount: {}",
dataByteCount);
}
listener.success(Acks.OK, null);
return;
}
-// If here that means there is some data for sure. Create a ByteBuf with
all the data in it.
+// Create a ByteBuf with all the data in it.
final int rows = result.getHeader().getRowCount();
final BufferAllocator allocator = webSessionResources.getAllocator();
final DrillBuf bufferWithData = allocator.buffer(dataByteCount);
[drill] 02/02: DRILL-6272: Refactor dynamic UDFs and function initializer tests to generate needed binary and source jars at runtime
This is an automated email from the ASF dual-hosted git repository.
amansinha pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git
commit ca90229b6deea40282927f8ab5c07715a4e18620
Author: Arina Ielchiieva
AuthorDate: Sat May 5 15:32:07 2018 +0300
DRILL-6272: Refactor dynamic UDFs and function initializer tests to
generate needed binary and source jars at runtime
close apache/drill#1225
---
exec/java-exec/pom.xml | 48 ++
.../exec/expr/fn/FunctionInitializerTest.java | 54 +--
.../apache/drill/exec/udf/dynamic/JarBuilder.java | 96
.../udf/dynamic}/TestDynamicUDFSupport.java| 523 ++---
.../java-exec/src/test/resources/drill-udf/pom.xml | 92
.../drill/udf/dynamic/CustomAbsFunction.java | 63 +++
.../drill/udf/dynamic/CustomLogFunction.java | 58 +++
.../udf/dynamic/CustomLowerDummyFunction.java | 58 +++
.../drill/udf/dynamic/CustomLowerFunction.java | 64 +++
.../drill/udf/dynamic/CustomLowerFunctionV2.java | 64 +++
.../drill/udf/dynamic/CustomUpperFunction.java | 64 +++
.../apache/drill/udf/dynamic/LowerFunction.java| 64 +++
.../drill-udf/src/main/resources/drill-module.conf | 1 +
.../test/resources/jars/DrillUDF-1.0-sources.jar | Bin 1892 -> 0 bytes
.../src/test/resources/jars/DrillUDF-1.0.jar | Bin 3146 -> 0 bytes
.../test/resources/jars/DrillUDF-2.0-sources.jar | Bin 1891 -> 0 bytes
.../src/test/resources/jars/DrillUDF-2.0.jar | Bin 3142 -> 0 bytes
.../jars/DrillUDF-overloading-1.0-sources.jar | Bin 3473 -> 0 bytes
.../resources/jars/DrillUDF-overloading-1.0.jar| Bin 5779 -> 0 bytes
.../resources/jars/DrillUDF_Copy-1.0-sources.jar | Bin 1892 -> 0 bytes
.../src/test/resources/jars/DrillUDF_Copy-1.0.jar | Bin 3185 -> 0 bytes
.../jars/DrillUDF_DupFunc-1.0-sources.jar | Bin 1888 -> 0 bytes
.../test/resources/jars/DrillUDF_DupFunc-1.0.jar | Bin 3201 -> 0 bytes
.../resources/jars/DrillUDF_Empty-1.0-sources.jar | Bin 536 -> 0 bytes
.../src/test/resources/jars/DrillUDF_Empty-1.0.jar | Bin 1863 -> 0 bytes
.../jars/DrillUDF_NoMarkerFile-1.0-sources.jar | Bin 1715 -> 0 bytes
.../resources/jars/DrillUDF_NoMarkerFile-1.0.jar | Bin 3084 -> 0 bytes
.../resources/jars/v2/DrillUDF-1.0-sources.jar | Bin 1899 -> 0 bytes
.../src/test/resources/jars/v2/DrillUDF-1.0.jar| Bin 3215 -> 0 bytes
29 files changed, 948 insertions(+), 301 deletions(-)
diff --git a/exec/java-exec/pom.xml b/exec/java-exec/pom.xml
index cbc3a02..345e240 100644
--- a/exec/java-exec/pom.xml
+++ b/exec/java-exec/pom.xml
@@ -584,6 +584,54 @@
netty-tcnative
${netty.tcnative.classifier}
+
+ org.apache.maven
+ maven-embedder
+ 3.5.3
+ test
+
+
+ org.apache.maven
+ maven-compat
+ 3.5.3
+ test
+
+
+ org.eclipse.aether
+ aether-connector-basic
+ 1.1.0
+ test
+
+
+ org.eclipse.aether
+ aether-transport-wagon
+ 1.1.0
+ test
+
+
+ org.apache.maven.wagon
+ wagon-http
+ 3.0.0
+ test
+
+
+ commons-logging
+ commons-logging
+
+
+
+
+ org.apache.maven.wagon
+ wagon-provider-api
+ 3.0.0
+ test
+
+
+ commons-logging
+ commons-logging
+
+
+
diff --git
a/exec/java-exec/src/test/java/org/apache/drill/exec/expr/fn/FunctionInitializerTest.java
b/exec/java-exec/src/test/java/org/apache/drill/exec/expr/fn/FunctionInitializerTest.java
index 7c10bd3..2ecb8a0 100644
---
a/exec/java-exec/src/test/java/org/apache/drill/exec/expr/fn/FunctionInitializerTest.java
+++
b/exec/java-exec/src/test/java/org/apache/drill/exec/expr/fn/FunctionInitializerTest.java
@@ -19,20 +19,20 @@ package org.apache.drill.exec.expr.fn;
import com.google.common.collect.Lists;
import org.apache.drill.categories.SqlFunctionTest;
-import org.apache.drill.test.TestTools;
+import org.apache.drill.exec.udf.dynamic.JarBuilder;
import org.apache.drill.exec.util.JarUtil;
import org.codehaus.janino.Java.CompilationUnit;
import org.junit.BeforeClass;
+import org.junit.ClassRule;
import org.junit.Test;
import org.junit.experimental.categories.Category;
-import org.junit.runner.RunWith;
-import org.mockito.invocation.InvocationOnMock;
-import org.mockito.runners.MockitoJUnitRunner;
-import org.mockito.stubbing.Answer;
+import org.junit.rules.TemporaryFolder;
+import java.io.File;
+import java.io.IOException;
import java.net.URL;
import java.net.URLClassLoader;
-import java.nio.file.Path;
+import java.nio.file.Paths;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
@@ -45,25 +45,27 @@ import java.util.concurrent.atomic.AtomicInteger;
import static org.junit.Assert.assertEquals;
import st
[drill] branch master updated (c1f0adc -> ca90229)
This is an automated email from the ASF dual-hosted git repository.
amansinha pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git.
from c1f0adc DRILL-6242 Use java.time.Local{Date|Time|DateTime} for Drill
Date, Time, Timestamp types. (#3)
new 9a47d55 DRILL-6363: Upgrade jmockit and mockito libs
new ca90229 DRILL-6272: Refactor dynamic UDFs and function initializer
tests to generate needed binary and source jars at runtime
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../test/java/org/apache/drill/test/TestTools.java | 2 +-
exec/java-exec/pom.xml | 48 ++
.../drill/TestTpchDistributedConcurrent.java | 3 +-
.../drill/exec/coord/zk/TestZookeeperClient.java | 2 +-
.../exec/expr/fn/FunctionInitializerTest.java | 54 +--
.../rpc/user/TemporaryTablesAutomaticDropTest.java | 42 +-
.../java/org/apache/drill/exec/sql/TestCTTAS.java | 315 -
.../apache/drill/exec/udf/dynamic/JarBuilder.java | 96
.../udf/dynamic}/TestDynamicUDFSupport.java| 523 ++---
.../exec/vector/complex/TestEmptyPopulation.java | 3 -
.../java-exec/src/test/resources/drill-udf/pom.xml | 92
.../drill/udf/dynamic/CustomAbsFunction.java | 39 +-
.../drill/udf/dynamic/CustomLogFunction.java | 36 +-
.../udf/dynamic/CustomLowerDummyFunction.java} | 44 +-
.../drill/udf/dynamic/CustomLowerFunction.java | 42 +-
.../drill/udf/dynamic/CustomLowerFunctionV2.java | 42 +-
.../drill/udf/dynamic/CustomUpperFunction.java | 42 +-
.../apache/drill/udf/dynamic/LowerFunction.java| 42 +-
.../drill-udf/src/main/resources/drill-module.conf | 1 +
.../test/resources/jars/DrillUDF-1.0-sources.jar | Bin 1892 -> 0 bytes
.../src/test/resources/jars/DrillUDF-1.0.jar | Bin 3146 -> 0 bytes
.../test/resources/jars/DrillUDF-2.0-sources.jar | Bin 1891 -> 0 bytes
.../src/test/resources/jars/DrillUDF-2.0.jar | Bin 3142 -> 0 bytes
.../jars/DrillUDF-overloading-1.0-sources.jar | Bin 3473 -> 0 bytes
.../resources/jars/DrillUDF-overloading-1.0.jar| Bin 5779 -> 0 bytes
.../resources/jars/DrillUDF_Copy-1.0-sources.jar | Bin 1892 -> 0 bytes
.../src/test/resources/jars/DrillUDF_Copy-1.0.jar | Bin 3185 -> 0 bytes
.../jars/DrillUDF_DupFunc-1.0-sources.jar | Bin 1888 -> 0 bytes
.../test/resources/jars/DrillUDF_DupFunc-1.0.jar | Bin 3201 -> 0 bytes
.../resources/jars/DrillUDF_Empty-1.0-sources.jar | Bin 536 -> 0 bytes
.../src/test/resources/jars/DrillUDF_Empty-1.0.jar | Bin 1863 -> 0 bytes
.../jars/DrillUDF_NoMarkerFile-1.0-sources.jar | Bin 1715 -> 0 bytes
.../resources/jars/DrillUDF_NoMarkerFile-1.0.jar | Bin 3084 -> 0 bytes
.../resources/jars/v2/DrillUDF-1.0-sources.jar | Bin 1899 -> 0 bytes
.../src/test/resources/jars/v2/DrillUDF-1.0.jar| Bin 3215 -> 0 bytes
pom.xml| 7 +-
36 files changed, 812 insertions(+), 663 deletions(-)
create mode 100644
exec/java-exec/src/test/java/org/apache/drill/exec/udf/dynamic/JarBuilder.java
rename exec/java-exec/src/test/java/org/apache/drill/{ =>
exec/udf/dynamic}/TestDynamicUDFSupport.java (65%)
create mode 100644 exec/java-exec/src/test/resources/drill-udf/pom.xml
copy
contrib/gis/src/main/java/org/apache/drill/exec/expr/fn/impl/gis/STAsText.java
=>
exec/java-exec/src/test/resources/drill-udf/src/main/java/org/apache/drill/udf/dynamic/CustomAbsFunction.java
(61%)
copy
contrib/gis/src/main/java/org/apache/drill/exec/expr/fn/impl/gis/STAsText.java
=>
exec/java-exec/src/test/resources/drill-udf/src/main/java/org/apache/drill/udf/dynamic/CustomLogFunction.java
(63%)
copy
exec/java-exec/src/{main/java/org/apache/drill/exec/expr/fn/impl/conv/UTF8ConvertTo.java
=>
test/resources/drill-udf/src/main/java/org/apache/drill/udf/dynamic/CustomLowerDummyFunction.java}
(59%)
copy
contrib/gis/src/main/java/org/apache/drill/exec/expr/fn/impl/gis/STAsText.java
=>
exec/java-exec/src/test/resources/drill-udf/src/main/java/org/apache/drill/udf/dynamic/CustomLowerFunction.java
(61%)
copy
contrib/gis/src/main/java/org/apache/drill/exec/expr/fn/impl/gis/STAsText.java
=>
exec/java-exec/src/test/resources/drill-udf/src/main/java/org/apache/drill/udf/dynamic/CustomLowerFunctionV2.java
(61%)
copy
contrib/gis/src/main/java/org/apache/drill/exec/expr/fn/impl/gis/STAsText.java
=>
exec/java-exec/src/test/resources/drill-udf/src/main/java/org/apache/drill/udf/dynamic/CustomUpperFunction.java
(61%)
copy
contrib/gis/src/main/java/org/apache/drill/exec/expr/fn/impl/gis/STAsText.java
=>
exec/java-exec/src/test/resources/drill-
[drill] 01/02: DRILL-6363: Upgrade jmockit and mockito libs
This is an automated email from the ASF dual-hosted git repository.
amansinha pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git
commit 9a47d555b00052115016fbe35cad5a4147d42ad5
Author: Arina Ielchiieva
AuthorDate: Sat May 5 15:31:10 2018 +0300
DRILL-6363: Upgrade jmockit and mockito libs
---
.../test/java/org/apache/drill/test/TestTools.java | 2 +-
.../drill/TestTpchDistributedConcurrent.java | 3 +-
.../drill/exec/coord/zk/TestZookeeperClient.java | 2 +-
.../rpc/user/TemporaryTablesAutomaticDropTest.java | 42 ++-
.../java/org/apache/drill/exec/sql/TestCTTAS.java | 315 +++--
.../exec/vector/complex/TestEmptyPopulation.java | 3 -
pom.xml| 7 +-
7 files changed, 134 insertions(+), 240 deletions(-)
diff --git a/common/src/test/java/org/apache/drill/test/TestTools.java
b/common/src/test/java/org/apache/drill/test/TestTools.java
index 8cf7ca7..2735c54 100644
--- a/common/src/test/java/org/apache/drill/test/TestTools.java
+++ b/common/src/test/java/org/apache/drill/test/TestTools.java
@@ -47,7 +47,7 @@ public class TestTools {
.indexOf("-agentlib:jdwp") > 0;
public static TestRule getTimeoutRule(int timeout) {
-return IS_DEBUG ? new TestName() : new Timeout(timeout);
+return IS_DEBUG ? new TestName() : Timeout.millis(timeout);
}
/**
diff --git
a/exec/java-exec/src/test/java/org/apache/drill/TestTpchDistributedConcurrent.java
b/exec/java-exec/src/test/java/org/apache/drill/TestTpchDistributedConcurrent.java
index 8991b8b..74b9a5c 100644
---
a/exec/java-exec/src/test/java/org/apache/drill/TestTpchDistributedConcurrent.java
+++
b/exec/java-exec/src/test/java/org/apache/drill/TestTpchDistributedConcurrent.java
@@ -33,6 +33,7 @@ import org.apache.drill.exec.rpc.user.UserResultsListener;
import org.apache.drill.test.BaseTestQuery;
import org.apache.drill.test.QueryTestUtil;
import org.junit.Rule;
+import org.junit.Test;
import org.junit.experimental.categories.Category;
import org.junit.rules.TestRule;
@@ -176,7 +177,7 @@ public class TestTpchDistributedConcurrent extends
BaseTestQuery {
}
}
- //@Test
+ @Test
public void testConcurrentQueries() throws Exception {
QueryTestUtil.testRunAndPrint(client, UserBitShared.QueryType.SQL,
alterSession);
diff --git
a/exec/java-exec/src/test/java/org/apache/drill/exec/coord/zk/TestZookeeperClient.java
b/exec/java-exec/src/test/java/org/apache/drill/exec/coord/zk/TestZookeeperClient.java
index 87cf72d..e0e6c79 100644
---
a/exec/java-exec/src/test/java/org/apache/drill/exec/coord/zk/TestZookeeperClient.java
+++
b/exec/java-exec/src/test/java/org/apache/drill/exec/coord/zk/TestZookeeperClient.java
@@ -125,7 +125,7 @@ public class TestZookeeperClient {
Mockito
.when(client.getCache().getCurrentData(absPath))
-.thenThrow(Exception.class);
+.thenThrow(RuntimeException.class);
client.hasPath(path);
}
diff --git
a/exec/java-exec/src/test/java/org/apache/drill/exec/rpc/user/TemporaryTablesAutomaticDropTest.java
b/exec/java-exec/src/test/java/org/apache/drill/exec/rpc/user/TemporaryTablesAutomaticDropTest.java
index 5553519..5aef6f7 100644
---
a/exec/java-exec/src/test/java/org/apache/drill/exec/rpc/user/TemporaryTablesAutomaticDropTest.java
+++
b/exec/java-exec/src/test/java/org/apache/drill/exec/rpc/user/TemporaryTablesAutomaticDropTest.java
@@ -19,39 +19,40 @@ package org.apache.drill.exec.rpc.user;
import mockit.Mock;
import mockit.MockUp;
+import org.apache.drill.exec.store.StorageStrategy;
import org.apache.drill.test.BaseTestQuery;
import org.apache.drill.common.config.DrillConfig;
-import org.apache.drill.exec.ExecConstants;
import org.apache.drill.exec.store.StoragePluginRegistry;
import org.apache.drill.exec.util.StoragePluginTestUtils;
import org.apache.drill.test.DirTestWatcher;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.LocatedFileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.RemoteIterator;
import org.junit.Before;
import org.junit.Test;
import java.io.File;
-import java.util.Properties;
import java.util.UUID;
import static org.apache.drill.exec.util.StoragePluginTestUtils.DFS_TMP_SCHEMA;
+import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
public class TemporaryTablesAutomaticDropTest extends BaseTestQuery {
- private static final String session_id = "sessionId";
+ private static final UUID SESSION_UUID = UUID.randomUUID();
@Before
- public void setup() throws Exception {
+ public void setup() {
new MockUp() {
@Mock
public UUID randomUUID() {
-return UUID.nameUUIDFromBytes(session_id.getBytes());
+return SESSION_UUID;
}
};
-
-Properties testConfigurations = cloneDefa
[drill] branch master updated: DRILL-6242 Use java.time.Local{Date|Time|DateTime} for Drill Date, Time, Timestamp types. (#3)
This is an automated email from the ASF dual-hosted git repository.
amansinha pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git
The following commit(s) were added to refs/heads/master by this push:
new c1f0adc DRILL-6242 Use java.time.Local{Date|Time|DateTime} for Drill
Date, Time, Timestamp types. (#3)
c1f0adc is described below
commit c1f0adc9276ab8314ce9c5dff2d9c92066c71530
Author: jiang-wu
AuthorDate: Tue May 1 14:48:26 2018 -0700
DRILL-6242 Use java.time.Local{Date|Time|DateTime} for Drill Date, Time,
Timestamp types. (#3)
close apache/drill#1247
* DRILL-6242 - Use java.time.Local{Date|Time|DateTime} classes to hold
values from corresponding Drill date, time, and timestamp types.
Conflicts:
exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/fn/ExtendedJsonOutput.java
Fix merge conflicts and check style.
---
.../exec/TestHiveDrillNativeParquetReader.java | 20 +-
.../drill/exec/fn/hive/TestInbuiltHiveUDFs.java| 15 +-
.../apache/drill/exec/hive/TestHiveStorage.java| 37 ++--
.../exec/store/jdbc/TestJdbcPluginWithDerbyIT.java | 12 +-
.../exec/store/jdbc/TestJdbcPluginWithMySQLIT.java | 15 +-
.../main/codegen/templates/CastDateVarChar.java| 8 +-
.../main/codegen/templates/CastVarCharDate.java| 8 +-
.../drill/exec/expr/fn/impl/DateTypeFunctions.java | 8 +-
.../exec/planner/common/DrillValuesRelBase.java| 30 +--
.../drill/exec/store/bson/BsonRecordReader.java| 25 ++-
.../org/apache/drill/exec/util/VectorUtil.java | 10 +-
.../exec/vector/complex/fn/BasicJsonOutput.java| 36 ++--
.../exec/vector/complex/fn/ExtendedJsonOutput.java | 15 +-
.../drill/exec/vector/complex/fn/JsonOutput.java | 8 +-
.../drill/exec/vector/complex/fn/VectorOutput.java | 34 +--
.../java/org/apache/drill/TestFunctionsQuery.java | 22 +-
.../java/org/apache/drill/TestStarQueries.java | 2 +-
.../drill/exec/fn/impl/TestCastFunctions.java | 27 +--
.../drill/exec/fn/impl/TestDateAddFunctions.java | 29 +--
.../drill/exec/fn/impl/TestDateTruncFunctions.java | 139 ++--
.../drill/exec/fn/impl/TestNewDateFunctions.java | 10 +-
.../exec/fn/interp/ExpressionInterpreterTest.java | 8 +-
.../exec/physical/impl/TestConvertFunctions.java | 13 +-
.../physical/impl/TestNestedDateTimeTimestamp.java | 234 +
.../impl/limit/TestEarlyLimit0Optimization.java| 16 +-
.../writer/TestCorruptParquetDateCorrection.java | 66 +++---
.../physical/impl/writer/TestParquetWriter.java| 32 +--
.../planner/logical/TestCaseNullableTypes.java | 5 +-
.../drill/exec/store/avro/AvroFormatTest.java | 75 ---
.../exec/store/bson/TestBsonRecordReader.java | 5 +-
.../exec/store/parquet/TestParquetComplex.java | 52 ++---
.../store/parquet2/TestDrillParquetReader.java | 51 ++---
.../vector/complex/writer/TestExtendedTypes.java | 2 +-
.../java/org/apache/drill/test/TestBuilder.java| 19 +-
.../test/resources/baseline_nested_datetime.json | 13 ++
exec/java-exec/src/test/resources/datetime.parquet | Bin 0 -> 1217 bytes
.../src/main/codegen/data/ValueVectorTypes.tdd | 6 +-
.../src/main/codegen/includes/vv_imports.ftl | 8 +-
.../codegen/templates/AbstractFieldReader.java | 2 +-
.../main/codegen/templates/FixedValueVectors.java | 15 +-
.../main/codegen/templates/HolderReaderImpl.java | 12 ++
.../src/main/codegen/templates/NullReader.java | 2 +-
.../src/main/codegen/templates/UnionReader.java| 2 +-
.../drill/exec/expr/fn/impl/DateUtility.java | 107 --
.../drill/exec/util/JsonStringArrayList.java | 1 +
.../apache/drill/exec/util/JsonStringHashMap.java | 1 +
.../drill/exec/util/SerializationModule.java | 75 +++
47 files changed, 895 insertions(+), 437 deletions(-)
diff --git
a/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/TestHiveDrillNativeParquetReader.java
b/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/TestHiveDrillNativeParquetReader.java
index 23c67b5..fd9701c 100644
---
a/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/TestHiveDrillNativeParquetReader.java
+++
b/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/TestHiveDrillNativeParquetReader.java
@@ -17,14 +17,19 @@
*/
package org.apache.drill.exec;
+import static org.hamcrest.CoreMatchers.containsString;
+import static org.junit.Assert.assertEquals;
+
+import java.math.BigDecimal;
+
import org.apache.drill.PlanTestBase;
import org.apache.drill.categories.HiveStorageTest;
import org.apache.drill.categories.SlowTest;
import org.apache.drill.common.exceptions.UserRemoteException;
+import org.apache.drill.exec.expr.fn.impl.DateUtility;
import org.apache.drill.exec.hive.HiveTestBase;
import org.apache.drill.exec.planner.physical.PlannerSettings;
imp
[drill] 01/04: DRILL-6249: Adding more unit testing documentation.
This is an automated email from the ASF dual-hosted git repository.
amansinha pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git
commit dc8d0107cb6a4a31e1beaa9c98d6e51cf7486a67
Author: Timothy Farkas
AuthorDate: Thu Mar 15 14:27:55 2018 -0700
DRILL-6249: Adding more unit testing documentation.
close apache/drill#1251
---
README.md | 5 +-
.../java/org/apache/drill/test/DirTestWatcher.java | 14 +-
.../test/java/org/apache/drill/test/TestTools.java | 7 +-
docs/dev/BaseTestQuery.md | 4 +
docs/dev/ClusterTest.md| 4 +
docs/dev/DevDocs.md| 6 +-
docs/dev/GeneratedCode.md | 51 +++
docs/dev/InstantiatingComponents.md| 107 ++
docs/dev/LicenseHeaders.md | 34 +
docs/dev/PhysicalOpUnitTestBase.md | 31
docs/dev/RowSetFramework.md| 112 +++
docs/dev/TempDirectories.md| 17 +++
docs/dev/TestDataSets.md | 159 +
docs/dev/Testing.md| 134 +++--
.../drill/exec/ops/ExchangeFragmentContext.java| 3 +
.../drill/exec/ops/ExecutorFragmentContext.java| 4 +
.../org/apache/drill/exec/ops/FragmentContext.java | 10 +-
.../apache/drill/exec/ops/FragmentContextImpl.java | 28 +++-
.../apache/drill/exec/ops/RootFragmentContext.java | 3 +
.../org/apache/drill/exec/record/BatchSchema.java | 4 +
.../org/apache/drill/TestDynamicUDFSupport.java| 2 +-
.../exec/expr/fn/FunctionInitializerTest.java | 2 +-
.../physical/impl/window/GenerateTestData.java | 4 +-
.../store/dfs/TestSchemaNotFoundException.java | 8 +-
.../org/apache/drill/test/BaseDirTestWatcher.java | 16 ++-
.../java/org/apache/drill/test/BaseTestQuery.java | 4 +
.../java/org/apache/drill/test/ExampleTest.java| 24
.../apache/drill/test/rowSet/RowSetBuilder.java| 7 +
.../apache/drill/test/rowSet/RowSetUtilities.java | 6 +
.../drill/test/rowSet/file/JsonFileBuilder.java| 26
30 files changed, 794 insertions(+), 42 deletions(-)
diff --git a/README.md b/README.md
index a30cf07..f2b5b9d 100644
--- a/README.md
+++ b/README.md
@@ -20,7 +20,10 @@ Please see the [Apache Drill
Website](http://drill.apache.org/) or the [Apache D
## Join the community!
-Apache Drill is an Apache Foundation project and is seeking all types of
contributions. Please say hello on the Apache Drill mailing list or join our
Google Hangouts for more information. (More information can be found at the
Apache Drill website).
+Apache Drill is an Apache Foundation project and is seeking all types of
contributions.
+Please say hello on the [Apache Drill mailing
list](http://drill.apache.org/mailinglists/)
+or join our [Google Hangouts](http://drill.apache.org/community-resources/)
for more information.
+(More information can be found at the [Apache Drill
website](http://drill.apache.org/)).
## Export Control
This distribution includes cryptographic software. The country in which you
currently reside may have restrictions on the import, possession, use, and/or
re-export to another country, of encryption software. BEFORE using any
encryption software, please check your country's laws, regulations and policies
concerning the import, possession, or use, and re-export of encryption
software, to see if this is permitted. See <http://www.wassenaar.org/> for more
information.
diff --git a/common/src/test/java/org/apache/drill/test/DirTestWatcher.java
b/common/src/test/java/org/apache/drill/test/DirTestWatcher.java
index 271ac94..6bdc951 100644
--- a/common/src/test/java/org/apache/drill/test/DirTestWatcher.java
+++ b/common/src/test/java/org/apache/drill/test/DirTestWatcher.java
@@ -31,10 +31,10 @@ import java.nio.file.Paths;
*
*
*
- * A {@link DirTestWatcher} is added to a test by declaring it as a JUnit
{@link org.junit.Rule}. A {@link org.junit.Rule} is
- * a piece of code that is run before and after every JUnit test marked with
the {@link org.junit.Test} annotation. When the
+ * A {@link DirTestWatcher} is added to a test by declaring it as a JUnit
{@link org.junit.Rule}. A {@link org.junit.Rule Rule} is
+ * a piece of code that is run before and after every JUnit test marked with
the {@link org.junit.Test Test} annotation. When the
* {@link DirTestWatcher} is added to a test class the {@link DirTestWatcher}
will create a temp directory before each of your
- * {@link org.junit.Test}s and optionally delete the temp directory after each
of your {@link org.junit.Test}s. The temp directory
+ * {@link org.junit.Test Test}s and optionally delete the temp directory after
each of your {@link org.junit.Test Test}s. The base tem
[drill] 03/04: DRILL-6364: Handle Cluster Info in WebUI when existing/new bits restart
This is an automated email from the ASF dual-hosted git repository.
amansinha pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git
commit 11036c556f6445bcf1418b6512ba71b44a0a6031
Author: Kunal Khatua
AuthorDate: Fri Apr 27 16:27:45 2018 -0700
DRILL-6364: Handle Cluster Info in WebUI when existing/new bits restart
As a follow up to DRILL-6289, the following improvements have been done:
1. When loading the page for the first time, the WebUI enables the shutdown
button without actually checking the state of the Drillbits.
The ideal behaviour should be to disable the button till the state is
verified. [Done]
If a Drillbit is confirmed down (i.e. not in `/state` response), it is
marked as OFFLINE and button is disabled.
2. When shutting down the current Drillbit, the WebUI no more has access to
the cluster state.
The ideal behaviour here should be to fetch the state from any of the
other Drillbits to update the status. [Done]
With the current Drillbit down, the other bits are requested for cluster
state info and update accordingly.
3. When a new, previously unseen Drillbit comes up, the WebUI will never
render it because the table is statically generated during the first page load.
The idea behaviour should be to append to the table on discovery of a
new node. [Done]
The new Drillbit info is injected and a prompt appears to refresh the
page to re-populate any missing info. This also works with feature (2)
mentioned above.
The only Java code change was to have the state response carry the address
and http-port as a tuple, instead of the user-port (which is never used).
Updates based on review
1. Added descriptive comments to functions
2. Handled possible race condition from multiple httpRequests for cluster
state
3. Eliminated unused stateKey variable
4. Best practice of using local (let) instead of global (var) objects, and
substituting currentRow variable instead of the jQuery search.
Additional changes based on review
1. Random selection of Drillbits to query for state when primary Drillbit
is down (limited to 3; with a timeout of 3 sec)
2. Indicate when an Offline Drillbit is de-registered, versus just Offline
as per Zookeeper
3. Hide shutdown column when Authentication is enabled, but the user is NOT
an Admin.
When the column is visible, remote bits are disabled because
4. Metrics will be shown all the time (except HTTPS), because the
information is available and a non-admin user would anyway not have actionable
capabilities
5. Basic clean up on unused variables
Hide shutdown buttons for HTTPS scenarios
Fixed check for 'https'
Since, `location.protocol` returns a trailing ':' as well
Handle metrics lookup on local Drillbit for HTTPS
If the webpage is accessed via IP, the certificate is tied to the IP and
not the FQDN, which is what is used for fetching the metrics.
close apache/drill#1241
---
.../apache/drill/exec/server/rest/DrillRoot.java | 3 +-
exec/java-exec/src/main/resources/rest/index.ftl | 319 +++--
2 files changed, 239 insertions(+), 83 deletions(-)
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/DrillRoot.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/DrillRoot.java
index 50d99b0..0a48801 100644
---
a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/DrillRoot.java
+++
b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/DrillRoot.java
@@ -88,7 +88,7 @@ public class DrillRoot {
Collection drillbits = getClusterInfoJSON().getDrillbits();
Map drillStatusMap = new HashMap();
for (DrillbitInfo drillbit : drillbits) {
- drillStatusMap.put(drillbit.getAddress() + "-" + drillbit.getUserPort(),
drillbit.getState());
+ drillStatusMap.put(drillbit.getAddress() + "-" + drillbit.getHttpPort(),
drillbit.getState());
}
return setResponse(drillStatusMap);
}
@@ -239,6 +239,7 @@ public class DrillRoot {
private Response shutdown(String resp) throws Exception {
Map shutdownInfo = new HashMap();
new Thread(new Runnable() {
+@Override
public void run() {
try {
drillbit.close();
diff --git a/exec/java-exec/src/main/resources/rest/index.ftl
b/exec/java-exec/src/main/resources/rest/index.ftl
index 5259e44..11d1dd2 100644
--- a/exec/java-exec/src/main/resources/rest/index.ftl
+++ b/exec/java-exec/src/main/resources/rest/index.ftl
@@ -54,9 +54,11 @@
- Drillbits ${model.getDrillbits()?size}
+ Drillbits ${model.getDrillbits()?size}
+
+
-
+
#
@@ -71,9 +73,7 @@
Version
Status
[drill] branch master updated (8a1a7c5 -> 16659ab)
This is an automated email from the ASF dual-hosted git repository.
amansinha pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git.
from 8a1a7c5 DRILL-6386: Remove unused imports and star imports.
new dc8d010 DRILL-6249: Adding more unit testing documentation.
new 76728e5 DRILL-6321: Customize Drill's conformance. Allow support to
APPLY keywords
new 11036c5 DRILL-6364: Handle Cluster Info in WebUI when existing/new
bits restart
new 16659ab DRILL-6401: Precision for decimal data types may be lost for
the case when cast with literal is used
The 4 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
README.md | 5 +-
.../java/org/apache/drill/test/DirTestWatcher.java | 14 +-
.../test/java/org/apache/drill/test/TestTools.java | 7 +-
docs/dev/BaseTestQuery.md | 4 +
docs/dev/ClusterTest.md| 4 +
docs/dev/DevDocs.md| 6 +-
docs/dev/GeneratedCode.md | 51
docs/dev/InstantiatingComponents.md| 107 +++
docs/dev/LicenseHeaders.md | 34 +++
docs/dev/PhysicalOpUnitTestBase.md | 31 ++
docs/dev/RowSetFramework.md| 112
docs/dev/TempDirectories.md| 17 ++
docs/dev/TestDataSets.md | 159 ++
docs/dev/Testing.md| 134 -
.../drill/exec/ops/ExchangeFragmentContext.java| 3 +
.../drill/exec/ops/ExecutorFragmentContext.java| 4 +
.../org/apache/drill/exec/ops/FragmentContext.java | 10 +-
.../apache/drill/exec/ops/FragmentContextImpl.java | 28 +-
.../apache/drill/exec/ops/RootFragmentContext.java | 3 +
.../drill/exec/planner/logical/DrillOptiq.java | 4 +-
.../sql/{FixedRange.java => DrillConformance.java} | 37 ++-
.../drill/exec/planner/sql/DrillParserConfig.java | 4 +-
.../drill/exec/planner/sql/SqlConverter.java | 34 +--
.../org/apache/drill/exec/record/BatchSchema.java | 4 +
.../apache/drill/exec/server/rest/DrillRoot.java | 3 +-
.../exec/store/parquet/ParquetFilterBuilder.java | 4 +-
exec/java-exec/src/main/resources/rest/index.ftl | 319 +++--
.../org/apache/drill/TestDynamicUDFSupport.java| 2 +-
.../exec/expr/fn/FunctionInitializerTest.java | 2 +-
.../physical/impl/window/GenerateTestData.java | 4 +-
.../sql/TestConformance.java} | 35 +--
.../store/dfs/TestSchemaNotFoundException.java | 8 +-
.../org/apache/drill/test/BaseDirTestWatcher.java | 16 +-
.../java/org/apache/drill/test/BaseTestQuery.java | 4 +
.../java/org/apache/drill/test/ExampleTest.java| 24 ++
.../apache/drill/test/rowSet/RowSetBuilder.java| 7 +
.../apache/drill/test/rowSet/RowSetUtilities.java | 6 +
.../drill/test/rowSet/file/JsonFileBuilder.java| 26 ++
.../drill/common/expression/ValueExpressions.java | 16 +-
39 files changed, 1105 insertions(+), 187 deletions(-)
create mode 100644 docs/dev/BaseTestQuery.md
create mode 100644 docs/dev/ClusterTest.md
create mode 100644 docs/dev/GeneratedCode.md
create mode 100644 docs/dev/InstantiatingComponents.md
create mode 100644 docs/dev/LicenseHeaders.md
create mode 100644 docs/dev/PhysicalOpUnitTestBase.md
create mode 100644 docs/dev/RowSetFramework.md
create mode 100644 docs/dev/TempDirectories.md
create mode 100644 docs/dev/TestDataSets.md
copy
exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/{FixedRange.java
=> DrillConformance.java} (54%)
copy exec/java-exec/src/test/java/org/apache/drill/{TestImplicitCasting.java
=> exec/sql/TestConformance.java} (60%)
--
To stop receiving notification emails like this one, please contact
amansi...@apache.org.
[drill] 02/04: DRILL-6321: Customize Drill's conformance. Allow support to APPLY keywords
This is an automated email from the ASF dual-hosted git repository.
amansinha pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git
commit 76728e5ecd30418862f4bc1d32bc315042851eb1
Author: chunhui-shi
AuthorDate: Fri Feb 2 10:03:38 2018 -0800
DRILL-6321: Customize Drill's conformance. Allow support to APPLY keywords
close apache/drill#1224
Conflicts:
exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/SqlConverter.java
Fix checkstyle
---
.../drill/exec/planner/sql/DrillConformance.java | 43
.../drill/exec/planner/sql/DrillParserConfig.java | 4 +-
.../drill/exec/planner/sql/SqlConverter.java | 34
.../org/apache/drill/exec/sql/TestConformance.java | 46 ++
4 files changed, 109 insertions(+), 18 deletions(-)
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/DrillConformance.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/DrillConformance.java
new file mode 100644
index 000..e6efeb9
--- /dev/null
+++
b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/DrillConformance.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.planner.sql;
+
+import org.apache.calcite.sql.validate.SqlConformanceEnum;
+import org.apache.calcite.sql.validate.SqlDelegatingConformance;
+
+/**
+ * Drill's SQL conformance is SqlConformanceEnum.DEFAULT except for method
isApplyAllowed().
+ * Since Drill is going to allow OUTER APPLY and CROSS APPLY to allow each row
from left child of Join
+ * to join with output of right side (sub-query or table function that will be
invoked for each row).
+ * Refer to DRILL-5999 for more information.
+ */
+public class DrillConformance extends SqlDelegatingConformance {
+
+ public DrillConformance() {
+super(SqlConformanceEnum.DEFAULT);
+ }
+
+ public DrillConformance(SqlConformanceEnum flavor) {
+super(flavor);
+ }
+
+ @Override
+ public boolean isApplyAllowed() {
+return true;
+ }
+}
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/DrillParserConfig.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/DrillParserConfig.java
index 1f67c5e..575ad86 100644
---
a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/DrillParserConfig.java
+++
b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/DrillParserConfig.java
@@ -22,7 +22,6 @@ import org.apache.calcite.avatica.util.Quoting;
import org.apache.calcite.sql.parser.SqlParser;
import org.apache.calcite.sql.parser.SqlParserImplFactory;
import org.apache.calcite.sql.validate.SqlConformance;
-import org.apache.calcite.sql.validate.SqlConformanceEnum;
import org.apache.drill.exec.planner.physical.PlannerSettings;
import
org.apache.drill.exec.planner.sql.parser.impl.DrillParserWithCompoundIdConverter;
@@ -30,6 +29,7 @@ public class DrillParserConfig implements SqlParser.Config {
private final long identifierMaxLength;
private final Quoting quotingIdentifiers;
+ public final static SqlConformance DRILL_CONFORMANCE = new
DrillConformance();
public DrillParserConfig(PlannerSettings settings) {
identifierMaxLength = settings.getIdentifierMaxLength();
@@ -63,7 +63,7 @@ public class DrillParserConfig implements SqlParser.Config {
@Override
public SqlConformance conformance() {
-return SqlConformanceEnum.DEFAULT;
+return DRILL_CONFORMANCE;
}
@Override
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/SqlConverter.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/SqlConverter.java
index 7342256..c7e1b25 100644
---
a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/SqlConverter.java
+++
b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/SqlConverter.java
@@ -58,7 +58,7 @@ import org.apache.calcite.sql.parser.SqlParser;
import org.apache.calcite.sql.parser.SqlParserPos;
import org.apache.calcite.sql.type.SqlTypeName;
import org.apache.calcite.sql.util.Ch
[drill] 04/04: DRILL-6401: Precision for decimal data types may be lost for the case when cast with literal is used
This is an automated email from the ASF dual-hosted git repository.
amansinha pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git
commit 16659ab30fd9fca5af39edd22aaab36409a50930
Author: Volodymyr Vysotskyi
AuthorDate: Thu May 10 17:06:28 2018 +0300
DRILL-6401: Precision for decimal data types may be lost for the case when
cast with literal is used
close apache/drill#1254
---
.../apache/drill/exec/planner/logical/DrillOptiq.java| 4 +++-
.../drill/exec/store/parquet/ParquetFilterBuilder.java | 4 +++-
.../apache/drill/common/expression/ValueExpressions.java | 16 ++--
3 files changed, 16 insertions(+), 8 deletions(-)
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillOptiq.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillOptiq.java
index fd791dd..2cd6068 100644
---
a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillOptiq.java
+++
b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillOptiq.java
@@ -550,7 +550,9 @@ public class DrillOptiq {
literal.getType().getScale(),
literal.getType().getPrecision()));
}
- return ValueExpressions.getVarDecimal((BigDecimal)
literal.getValue());
+ return ValueExpressions.getVarDecimal((BigDecimal)
literal.getValue(),
+ literal.getType().getPrecision(),
+ literal.getType().getScale());
}
double dbl = ((BigDecimal) literal.getValue()).doubleValue();
logger.warn("Converting exact decimal into approximate decimal.\n" +
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetFilterBuilder.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetFilterBuilder.java
index 52c4723..e55425e 100644
---
a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetFilterBuilder.java
+++
b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetFilterBuilder.java
@@ -181,7 +181,9 @@ public class ParquetFilterBuilder extends
AbstractExprVisitor
[3/3] drill git commit: DRILL-6234: Improved documentation for VariableWidthVector mutators, and added simple unit tests demonstrating mutator behavior.
DRILL-6234: Improved documentation for VariableWidthVector mutators, and added
simple unit tests demonstrating mutator behavior.
close apache/drill#1164
Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/9a6cb59b
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/9a6cb59b
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/9a6cb59b
Branch: refs/heads/master
Commit: 9a6cb59b9b7a5b127e5f60309ce2f506ede9652a
Parents: f5b8223
Author: Timothy Farkas
Authored: Tue Mar 13 17:24:28 2018 -0700
Committer: Aman Sinha
Committed: Fri Mar 30 22:47:31 2018 -0700
--
exec/vector/pom.xml | 7 +-
.../templates/VariableLengthVectors.java| 61
.../apache/drill/exec/vector/ValueVector.java | 3 +-
.../exec/vector/VariableLengthVectorTest.java | 141 +++
4 files changed, 210 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/drill/blob/9a6cb59b/exec/vector/pom.xml
--
diff --git a/exec/vector/pom.xml b/exec/vector/pom.xml
index 0184305..21e138d 100644
--- a/exec/vector/pom.xml
+++ b/exec/vector/pom.xml
@@ -65,7 +65,12 @@
0.7.1
-
+
+ org.apache.drill
+ drill-common
+ ${project.version}
+ tests
+
http://git-wip-us.apache.org/repos/asf/drill/blob/9a6cb59b/exec/vector/src/main/codegen/templates/VariableLengthVectors.java
--
diff --git a/exec/vector/src/main/codegen/templates/VariableLengthVectors.java
b/exec/vector/src/main/codegen/templates/VariableLengthVectors.java
index 516eb52..ab995cd 100644
--- a/exec/vector/src/main/codegen/templates/VariableLengthVectors.java
+++ b/exec/vector/src/main/codegen/templates/VariableLengthVectors.java
@@ -512,6 +512,8 @@ public final class ${minor.class}Vector extends
BaseDataValueVector implements V
}
/**
+ * Overview
+ *
* Mutable${minor.class} implements a vector of variable width values.
Elements in the vector
* are accessed by position from the logical start of the vector. A fixed
width offsetVector
* is used to convert an element's position to it's offset from the start of
the (0-based)
@@ -520,6 +522,46 @@ public final class ${minor.class}Vector extends
BaseDataValueVector implements V
* The equivalent Java primitive is '${minor.javaType!type.javaType}'
*
* NB: this class is automatically generated from ValueVectorTypes.tdd using
FreeMarker.
+ *
+ * Contract
+ *
+ *
+ *
+ * Supported Writes: {@link VariableWidthVector}s do not
support random writes. In contrast {@link
org.apache.drill.exec.vector.FixedWidthVector}s do
+ * allow random writes but special care is needed.
+ *
+ *
+ * Writing Values: All set methods must be called with a
consecutive sequence of indices. With a few exceptions:
+ *
+ * You can update the last index you just set.
+ * You can reset a previous index (call it Idx), but you must
assume all the data after Idx is corrupt. Also
+ * note that the memory consumed by data that came after Idx is not
released.
+ *
+ *
+ *
+ * Setting Value Count: Vectors aren't explicitly aware of how
many values they contain. So you must keep track of the
+ * number of values you've written to the vector and once you are done
writing to the vector you must call {@link Mutator#setValueCount(int)}.
+ * It is possible to trim the vector by setting the value count to be
less than the number of values currently contained in the vector. Note the
extra memory consumed in
+ * the data buffer is not freed when this is done.
+ *
+ *
+ * Memory Allocation: When setting a value at an index you must
do one of the following to ensure you do not get an {@link
IndexOutOfBoundsException}.
+ *
+ *
+ * Allocate the exact amount of memory you need when using the
{@link Mutator#set(int, byte[])} methods. If you do not
+ * manually allocate sufficient memory an {@link
IndexOutOfBoundsException} can be thrown when the data buffer runs out of space.
+ *
+ *
+ * Or you can use the {@link Mutator#setSafe(int, byte[])}
methods, which will automatically grow your data buffer to
+ * fit your data.
+ *
+ *
+ *
+ *
+ * Immutability: Once a vector has been populated with data and
{@link #setValueCount(int)} has been called, it should be considered immutable.
+ *
+ *
+ *
*/
public final class Mutator extends BaseValueVector.BaseMutator implemen
[1/3] drill git commit: DRILL-6259: Support parquet filter push down for complex types
Repository: drill
Updated Branches:
refs/heads/master a264e7feb -> 9a6cb59b9
DRILL-6259: Support parquet filter push down for complex types
close apache/drill#1173
Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/4ee5625d
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/4ee5625d
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/4ee5625d
Branch: refs/heads/master
Commit: 4ee5625d57bd73d3d82b45f687a8574ea6660f8e
Parents: a264e7f
Author: Arina Ielchiieva
Authored: Tue Mar 13 19:54:25 2018 +0200
Committer: Aman Sinha
Committed: Fri Mar 30 22:45:33 2018 -0700
--
.../sig/ConstantExpressionIdentifier.java | 6 +
.../exec/expr/ExpressionTreeMaterializer.java | 3 +-
.../exec/expr/stat/ParquetIsPredicates.java | 23
.../exec/expr/stat/RangeExprEvaluator.java | 27 ++--
.../drill/exec/expr/stat/TypedFieldExpr.java| 63 --
.../exec/planner/common/DrillRelOptUtil.java| 14 +--
.../logical/DrillPushFilterPastProjectRule.java | 14 ++-
.../store/parquet/ParquetFilterBuilder.java | 80
.../store/parquet/ParquetPushDownFilter.java| 13 +-
.../parquet/stat/ParquetMetaStatCollector.java | 29 ++---
...estParquetFilterPushDownForComplexTypes.java | 124 +++
.../resources/parquet/users/users_1.parquet | Bin 0 -> 657 bytes
.../resources/parquet/users/users_2.parquet | Bin 0 -> 641 bytes
.../resources/parquet/users/users_3.parquet | Bin 0 -> 588 bytes
.../resources/parquet/users/users_4.parquet | Bin 0 -> 653 bytes
.../resources/parquet/users/users_5.parquet | Bin 0 -> 583 bytes
.../resources/parquet/users/users_6.parquet | Bin 0 -> 627 bytes
.../resources/parquet/users/users_7.parquet | Bin 0 -> 662 bytes
.../drill/common/expression/SchemaPath.java | 29 +++--
.../drill/common/expression/TypedFieldExpr.java | 60 +
.../visitors/AbstractExprVisitor.java | 6 +-
.../expression/visitors/AggregateChecker.java | 5 +
.../expression/visitors/ConstantChecker.java| 5 +
.../common/expression/visitors/ExprVisitor.java | 54
.../visitors/ExpressionValidator.java | 5 +
25 files changed, 365 insertions(+), 195 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/drill/blob/4ee5625d/exec/java-exec/src/main/java/org/apache/drill/exec/compile/sig/ConstantExpressionIdentifier.java
--
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/compile/sig/ConstantExpressionIdentifier.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/compile/sig/ConstantExpressionIdentifier.java
index 1e71773..1a0b7d5 100644
---
a/exec/java-exec/src/main/java/org/apache/drill/exec/compile/sig/ConstantExpressionIdentifier.java
+++
b/exec/java-exec/src/main/java/org/apache/drill/exec/compile/sig/ConstantExpressionIdentifier.java
@@ -31,6 +31,7 @@ import org.apache.drill.common.expression.IfExpression;
import org.apache.drill.common.expression.LogicalExpression;
import org.apache.drill.common.expression.NullExpression;
import org.apache.drill.common.expression.SchemaPath;
+import org.apache.drill.common.expression.TypedFieldExpr;
import org.apache.drill.common.expression.TypedNullConstant;
import org.apache.drill.common.expression.ValueExpressions;
import org.apache.drill.common.expression.ValueExpressions.BooleanExpression;
@@ -231,4 +232,9 @@ public class ConstantExpressionIdentifier implements
ExprVisitor value) throws RuntimeException {
return false;
}
+
+ @Override
+ public Boolean visitTypedFieldExpr(TypedFieldExpr e,
IdentityHashMap value) throws RuntimeException {
+return false;
+ }
}
http://git-wip-us.apache.org/repos/asf/drill/blob/4ee5625d/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ExpressionTreeMaterializer.java
--
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ExpressionTreeMaterializer.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ExpressionTreeMaterializer.java
index 23df262..f1b50c9 100644
---
a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ExpressionTreeMaterializer.java
+++
b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ExpressionTreeMaterializer.java
@@ -44,6 +44,7 @@ import
org.apache.drill.common.expression.IfExpression.IfCondition;
import org.apache.drill.common.expression.LogicalExpression;
import org.apache.drill.common.expression.NullExpression;
import org.apache.drill.common.expression.SchemaPath;
+import org.apache.drill.common.expression.TypedFieldExpr;
import org.apache.drill.common.expression.TypedNullConstant;
import org.apache.drill.common.expression.ValueExpressions
[2/3] drill git commit: DRILL-6103: lsb_release: command not found
DRILL-6103: lsb_release: command not found close apache/drill#1191 Thanks to Sanel Zukan for providing a small patch that checks for /etc/fedora-release path. This is more common, than lsb_release command on Linux distros. Project: http://git-wip-us.apache.org/repos/asf/drill/repo Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/f5b82239 Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/f5b82239 Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/f5b82239 Branch: refs/heads/master Commit: f5b82239ad823a972e32ea732237a2e8c4346db3 Parents: 4ee5625 Author: Kunal Khatua Authored: Mon Mar 26 23:30:31 2018 -0700 Committer: Aman Sinha Committed: Fri Mar 30 22:46:36 2018 -0700 -- distribution/src/resources/drill-config.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/drill/blob/f5b82239/distribution/src/resources/drill-config.sh -- diff --git a/distribution/src/resources/drill-config.sh b/distribution/src/resources/drill-config.sh index e3eaa64..c2f3d70 100644 --- a/distribution/src/resources/drill-config.sh +++ b/distribution/src/resources/drill-config.sh @@ -421,8 +421,7 @@ CP="$CP:$DRILL_HOME/jars/classb/*" if [[ "$OSTYPE" == "linux-gnu" ]]; then # Linux # check for Fedora. netty-tcnative has a Fedora variant - linuxvariant=$(lsb_release -i | cut -d: -f2 | sed s/'^\t'//) - if [[ "$linuxvariant" == "Fedora" ]]; then + if [[ -f "/etc/fedora-release" ]]; then CP="$CP:$DRILL_HOME/jars/3rdparty/fedora/*" else CP="$CP:$DRILL_HOME/jars/3rdparty/linux/*"
[7/7] drill git commit: DRILL-6125: Fix possible memory leak when query is cancelled or finished.
DRILL-6125: Fix possible memory leak when query is cancelled or finished.
close apache/drill#1105
Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/a264e7fe
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/a264e7fe
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/a264e7fe
Branch: refs/heads/master
Commit: a264e7feb1d02ffd5762bb1f652ea22d17aa5243
Parents: 03b245e
Author: Timothy Farkas
Authored: Tue Jan 30 15:55:41 2018 -0800
Committer: Aman Sinha
Committed: Thu Mar 29 23:24:09 2018 -0700
--
.../drill/exec/physical/impl/RootExec.java | 23 ++-
.../PartitionSenderRootExec.java| 32 ++--
.../exec/work/fragment/FragmentExecutor.java| 179 ++-
3 files changed, 160 insertions(+), 74 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/drill/blob/a264e7fe/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/RootExec.java
--
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/RootExec.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/RootExec.java
index 5e366fb..ddeb3e8 100644
---
a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/RootExec.java
+++
b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/RootExec.java
@@ -20,19 +20,28 @@ package org.apache.drill.exec.physical.impl;
import org.apache.drill.exec.proto.ExecProtos.FragmentHandle;
/**
- * A FragmentRoot is a node which is the last processing node in a query plan.
FragmentTerminals include Exchange
- * output nodes and storage nodes. They are there driving force behind the
completion of a query.
+ * Functionality
+ *
+ * A FragmentRoot is a node which is the last processing node in a query
plan. FragmentTerminals include Exchange
+ * output nodes and storage nodes. They are there driving force behind the
completion of a query.
+ *
+ * Assumptions
+ *
+ * All implementations of {@link RootExec} assume that all their methods are
called by the same thread.
+ *
*/
public interface RootExec extends AutoCloseable {
/**
* Do the next batch of work.
- * @return Whether or not additional batches of work are necessary. False
means that this fragment is done.
+ * @return Whether or not additional batches of work are necessary. False
means that this fragment is done.
*/
- public boolean next();
+ boolean next();
/**
- * Inform sender that receiving fragment is finished and doesn't need any
more data
- * @param handle
+ * Inform sender that receiving fragment is finished and doesn't need any
more data. This can be called multiple
+ * times (once for each downstream receiver). If all receivers are finished
then a subsequent call to {@link #next()}
+ * will return false.
+ * @param handle The handle pointing to the downstream receiver that does
not need anymore data.
*/
- public void receivingFragmentFinished(FragmentHandle handle);
+ void receivingFragmentFinished(FragmentHandle handle);
}
http://git-wip-us.apache.org/repos/asf/drill/blob/a264e7fe/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/partitionsender/PartitionSenderRootExec.java
--
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/partitionsender/PartitionSenderRootExec.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/partitionsender/PartitionSenderRootExec.java
index 25be50a..7e76238 100644
---
a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/partitionsender/PartitionSenderRootExec.java
+++
b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/partitionsender/PartitionSenderRootExec.java
@@ -65,14 +65,13 @@ public class PartitionSenderRootExec extends BaseRootExec {
private PartitionerDecorator partitioner;
private ExchangeFragmentContext context;
- private boolean ok = true;
private final int outGoingBatchCount;
private final HashPartitionSender popConfig;
private final double cost;
private final AtomicIntegerArray remainingReceivers;
private final AtomicInteger remaingReceiverCount;
- private volatile boolean done = false;
+ private boolean done = false;
private boolean first = true;
private boolean closeIncoming;
@@ -146,11 +145,8 @@ public class PartitionSenderRootExec extends BaseRootExec {
@Override
public boolean innerNext() {
-if (!ok) {
- return false;
-}
-
IterOutcome out;
+
if (!done) {
out = next(incoming);
} else {
@@ -252,13 +248,11 @@ public class PartitionSenderRootExec extends BaseRootExec
{
startIndex, endIndex);
}
- s
[1/7] drill git commit: DRILL-6300: Refresh protobuf C++ source files
Repository: drill
Updated Branches:
refs/heads/master f1cfaaf3a -> a264e7feb
DRILL-6300: Refresh protobuf C++ source files
close apache/drill#1194
Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/bfc86f17
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/bfc86f17
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/bfc86f17
Branch: refs/heads/master
Commit: bfc86f1719bc348a74886e8f29e49bed6fdfe8df
Parents: f1cfaaf
Author: Volodymyr Vysotskyi
Authored: Thu Mar 29 18:51:08 2018 +0300
Committer: Aman Sinha
Committed: Thu Mar 29 23:17:18 2018 -0700
--
contrib/native/client/readme.linux | 12 +-
.../client/src/protobuf/Coordination.pb.cc | 98 +++-
.../client/src/protobuf/Coordination.pb.h | 89 ++-
.../client/src/protobuf/UserBitShared.pb.cc | 239 ++-
.../client/src/protobuf/UserBitShared.pb.h | 24 +-
5 files changed, 326 insertions(+), 136 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/drill/blob/bfc86f17/contrib/native/client/readme.linux
--
diff --git a/contrib/native/client/readme.linux
b/contrib/native/client/readme.linux
index 34c791b..92fe519 100644
--- a/contrib/native/client/readme.linux
+++ b/contrib/native/client/readme.linux
@@ -47,7 +47,7 @@ Install Prerequisites
$> sudo yum install cppunit-devel
3.2) Download Zookeeper from :
-- http://apache.mirror.quintex.com/zookeeper/zookeeper-3.4.6/
+- https://archive.apache.org/dist/zookeeper/zookeeper-3.4.6/
- untar and then follow instructions in ZOOKEEPER_DIR/src/c/README to
build and install the client libs
3.3) run autoreconf
@@ -57,7 +57,7 @@ Install Prerequisites
$> ./configure --enable-debug --with-syncapi --enable-static
--enable-shared
$> make && sudo make install
-4) Install boost. The minumim version required is 1.53, which will probably
have to be built from source
+4) Install boost. The minimum version required is 1.53, which will probably
have to be built from source
# Remove any previous boost
$> sudo yum -y erase boost
@@ -69,7 +69,9 @@ Install Prerequisites
#install the binary rpms
#(Note: the "rpm" utility does not clean up old versions very well.)
$> sudo yum -y install ~/rpmbuild/RPMS/x86_64/*
-
+OR
+# Build boost for Drill using instruction from readme.boost.
+# Uncomment set(Boost_NAMESPACE drill_boost) line in client/CMakeLists.txt
OR
Download and build using boost build.
See this link for how to build:
http://www.boost.org/doc/libs/1_53_0/more/getting_started/unix-variants.html#prepare-to-use-a-boost-library-binary
@@ -117,12 +119,12 @@ Build drill client
Test
Run query submitter from the command line
-$> querySubmitter query='select * from
dfs.`/Users/pchandra/work/data/tpc-h/customer.parquet`' type=sql
connectStr=local=10.250.0.146:31010 api=async logLevel=trace user=yourUserName
password=yourPassWord
+$> querySubmitter query='SELECT * FROM cp.`employee.json` LIMIT 20'
type=sql connectStr=local=10.250.0.146:31010 api=async logLevel=trace
user=yourUserName password=yourPassWord
Valgrind
Examples to run Valgrind and see the log in Valkyrie
-$> valgrind --leak-check=yes --xml=yes --xml-file=qs-vg-log-a.xml
querySubmitter query='select LINEITEM from
dfs.`/Users/pchandra/work/data/tpc-h/customer.parquet`' type=sql
connectStr=local=10.250.0.146:31010 api=async logLevel=trace
+$> valgrind --leak-check=yes --xml=yes --xml-file=qs-vg-log-a.xml
querySubmitter query='SELECT * FROM cp.`employee.json` LIMIT 20' type=sql
connectStr=local=10.250.0.146:31010 api=async logLevel=trace
$> valkyrie -l qs-vg-log-a.xml
http://git-wip-us.apache.org/repos/asf/drill/blob/bfc86f17/contrib/native/client/src/protobuf/Coordination.pb.cc
--
diff --git a/contrib/native/client/src/protobuf/Coordination.pb.cc
b/contrib/native/client/src/protobuf/Coordination.pb.cc
index 923481b..14b3103 100644
--- a/contrib/native/client/src/protobuf/Coordination.pb.cc
+++ b/contrib/native/client/src/protobuf/Coordination.pb.cc
@@ -23,6 +23,7 @@ namespace {
const ::google::protobuf::Descriptor* DrillbitEndpoint_descriptor_ = NULL;
const ::google::protobuf::internal::GeneratedMessageReflection*
DrillbitEndpoint_reflection_ = NULL;
+const ::google::protobuf::EnumDescriptor* DrillbitEndpoint_State_descriptor_ =
NULL;
const ::google::protobuf::Descriptor* DrillServiceInstance_descriptor_ = NULL;
const ::google::protobuf::internal::GeneratedMessageReflection*
DrillServiceInstance_reflection_ = NULL;
@@ -40,13 +41,14 @@ void protobuf_AssignDesc_Coordination_2eproto() {
"Coordination.proto");
[6/7] drill git commit: DRILL-6256: Remove references to java 7 from readme and other files
DRILL-6256: Remove references to java 7 from readme and other files
close apache/drill#1172
Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/03b245ef
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/03b245ef
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/03b245ef
Branch: refs/heads/master
Commit: 03b245ef4d028458fc6cc3588682d5ae9cc3bb33
Parents: 7f64556
Author: vladimir tkach
Authored: Mon Mar 26 20:02:37 2018 +0300
Committer: Aman Sinha
Committed: Thu Mar 29 23:23:17 2018 -0700
--
INSTALL.md | 10 +-
distribution/src/resources/README.md | 2 +-
distribution/src/resources/drill-config.sh | 9 -
exec/java-exec/src/main/sh/drill-config.sh | 2 +-
exec/java-exec/src/main/sh/runbit | 4 ++--
pom.xml| 2 +-
6 files changed, 14 insertions(+), 15 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/drill/blob/03b245ef/INSTALL.md
--
diff --git a/INSTALL.md b/INSTALL.md
index 7c65611..72268e9 100644
--- a/INSTALL.md
+++ b/INSTALL.md
@@ -3,15 +3,15 @@
## Prerequisites
Currently, the Apache Drill build process is known to work on Linux, Windows
and OSX. To build, you need to have the following software installed on your
system to successfully complete a build.
- * Java 7
+ * Java 8
* Maven 3.x
## Confirm settings
# java -version
-java version "1.7.0_09"
-Java(TM) SE Runtime Environment (build 1.7.0_09-b05)
-Java HotSpot(TM) 64-Bit Server VM (build 23.5-b02, mixed mode)
-
+java version "1.8.0_161"
+Java(TM) SE Runtime Environment (build 1.8.0_161-b12)
+Java HotSpot(TM) 64-Bit Server VM (build 25.161-b12, mixed mode)
+
# mvn --version
Apache Maven 3.0.3 (r1075438; 2011-02-28 09:31:09-0800)
http://git-wip-us.apache.org/repos/asf/drill/blob/03b245ef/distribution/src/resources/README.md
--
diff --git a/distribution/src/resources/README.md
b/distribution/src/resources/README.md
index 40feb29..bb1cd07 100644
--- a/distribution/src/resources/README.md
+++ b/distribution/src/resources/README.md
@@ -3,7 +3,7 @@
## Prerequisites
* Linux, Windows or OSX
- * Oracle JDK 7 (JDK, not JRE)
+ * Oracle/OpenJDK 8 (JDK, not JRE)
Additional requirements when running in clustered mode:
* Hadoop 2.3+ distribution of Hadoop (such as Apache or MapR)
http://git-wip-us.apache.org/repos/asf/drill/blob/03b245ef/distribution/src/resources/drill-config.sh
--
diff --git a/distribution/src/resources/drill-config.sh
b/distribution/src/resources/drill-config.sh
index 55d032d..e3eaa64 100644
--- a/distribution/src/resources/drill-config.sh
+++ b/distribution/src/resources/drill-config.sh
@@ -123,10 +123,10 @@ if [ ! -e "$JAVA" ]; then
fatal_error "Java not found at JAVA_HOME=$JAVA_HOME."
fi
-# Ensure that Java version is at least 1.7
+# Ensure that Java version is at least 1.8
"$JAVA" -version 2>&1 | grep "version" | egrep -e "1\.4|1\.5|1\.6|1\.7" >
/dev/null
if [ $? -eq 0 ]; then
- fatal_error "Java 1.8 or later is required to run Apache Drill."
+ fatal_error "Java 1.8 is required to run Apache Drill."
fi
# Check if a file exists and has relevant lines for execution
@@ -214,9 +214,8 @@ fi
export SQLLINE_JAVA_OPTS=${SQLLINE_JAVA_OPTS:-""}
-# Class unloading is disabled by default in Java 7
-#
http://hg.openjdk.java.net/jdk7u/jdk7u60/hotspot/file/tip/src/share/vm/runtime/globals.hpp#l1622
-export SERVER_GC_OPTS="$SERVER_GC_OPTS -XX:+CMSClassUnloadingEnabled
-XX:+UseG1GC"
+
+export SERVER_GC_OPTS="$SERVER_GC_OPTS -XX:+UseG1GC"
# No GC options by default for SQLLine
export CLIENT_GC_OPTS=${CLIENT_GC_OPTS:-""}
http://git-wip-us.apache.org/repos/asf/drill/blob/03b245ef/exec/java-exec/src/main/sh/drill-config.sh
--
diff --git a/exec/java-exec/src/main/sh/drill-config.sh
b/exec/java-exec/src/main/sh/drill-config.sh
index 20102fc..fa62da8 100644
--- a/exec/java-exec/src/main/sh/drill-config.sh
+++ b/exec/java-exec/src/main/sh/drill-config.sh
@@ -88,7 +88,7 @@ if [ -z "$JAVA_HOME" ]; then
+==+
| Error: JAVA_HOME is not set and Java could not be found |
+--+
-| Drill requires Java 1.7 or later.|
+| Drill requires Java 1.8|
+==+
EOF
exit 1
http://git-wip-us.apache
[2/7] drill git commit: DRILL-6299: Fixed a filter pushed down issue when a column doesn't have stats
DRILL-6299: Fixed a filter pushed down issue when a column doesn't have stats
close apache/drill#1192
Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/ea643bfe
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/ea643bfe
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/ea643bfe
Branch: refs/heads/master
Commit: ea643bfebeff8991d4e43fa8762773076087d0df
Parents: bfc86f1
Author: Salim Achouche
Authored: Wed Mar 28 12:08:25 2018 -0700
Committer: Aman Sinha
Committed: Thu Mar 29 23:18:32 2018 -0700
--
.../drill/exec/expr/stat/ParquetIsPredicates.java | 17 ++---
.../exec/expr/stat/ParquetPredicatesHelper.java| 9 +
2 files changed, 15 insertions(+), 11 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/drill/blob/ea643bfe/exec/java-exec/src/main/java/org/apache/drill/exec/expr/stat/ParquetIsPredicates.java
--
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/stat/ParquetIsPredicates.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/stat/ParquetIsPredicates.java
index c6f9b2f..a58ce7c 100644
---
a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/stat/ParquetIsPredicates.java
+++
b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/stat/ParquetIsPredicates.java
@@ -62,7 +62,7 @@ public class ParquetIsPredicates {
public boolean canDrop(RangeExprEvaluator evaluator) {
Statistics exprStat = expr.accept(evaluator, null);
- if (exprStat == null) {
+ if (!ParquetPredicatesHelper.hasStats(exprStat)) {
return false;
}
@@ -87,8 +87,7 @@ public class ParquetIsPredicates {
public boolean canDrop(RangeExprEvaluator evaluator) {
Statistics exprStat = expr.accept(evaluator, null);
- if (exprStat == null ||
- exprStat.isEmpty()) {
+ if (!ParquetPredicatesHelper.hasStats(exprStat)) {
return false;
}
@@ -113,8 +112,7 @@ public class ParquetIsPredicates {
public boolean canDrop(RangeExprEvaluator evaluator) {
Statistics exprStat = expr.accept(evaluator, null);
- if (exprStat == null ||
- exprStat.isEmpty()) {
+ if (!ParquetPredicatesHelper.hasStats(exprStat)) {
return false;
}
@@ -140,8 +138,7 @@ public class ParquetIsPredicates {
public boolean canDrop(RangeExprEvaluator evaluator) {
Statistics exprStat = expr.accept(evaluator, null);
- if (exprStat == null ||
- exprStat.isEmpty()) {
+ if (!ParquetPredicatesHelper.hasStats(exprStat)) {
return false;
}
@@ -167,8 +164,7 @@ public class ParquetIsPredicates {
public boolean canDrop(RangeExprEvaluator evaluator) {
Statistics exprStat = expr.accept(evaluator, null);
- if (exprStat == null ||
- exprStat.isEmpty()) {
+ if (!ParquetPredicatesHelper.hasStats(exprStat)) {
return false;
}
@@ -193,8 +189,7 @@ public class ParquetIsPredicates {
public boolean canDrop(RangeExprEvaluator evaluator) {
Statistics exprStat = expr.accept(evaluator, null);
- if (exprStat == null ||
- exprStat.isEmpty()) {
+ if (!ParquetPredicatesHelper.hasStats(exprStat)) {
return false;
}
http://git-wip-us.apache.org/repos/asf/drill/blob/ea643bfe/exec/java-exec/src/main/java/org/apache/drill/exec/expr/stat/ParquetPredicatesHelper.java
--
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/stat/ParquetPredicatesHelper.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/stat/ParquetPredicatesHelper.java
index ac82d65..e43acd3 100644
---
a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/stat/ParquetPredicatesHelper.java
+++
b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/stat/ParquetPredicatesHelper.java
@@ -22,9 +22,18 @@ import org.apache.parquet.column.statistics.Statistics;
/**
* Parquet predicates class helper for filter pushdown.
*/
+@SuppressWarnings("rawtypes")
public class ParquetPredicatesHelper {
/**
+ * @param stat statistics object
+ * @return true if the input stat object has valid statistics; false
otherwise
+ */
+ public static boolean hasStats(Statistics stat) {
+return stat != null && !stat.isEmpty();
+ }
+
+ /**
* Checks that column chunk's statistics has only nulls
*
* @param stat parquet column statistics
[5/7] drill git commit: DRILL-6278: Removed temp codegen directory in testing framework.
DRILL-6278: Removed temp codegen directory in testing framework.
close apache/drill#1178
Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/7f645565
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/7f645565
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/7f645565
Branch: refs/heads/master
Commit: 7f645565cd68b1780e643bc20a43951610704008
Parents: 67710bb
Author: Timothy Farkas
Authored: Tue Mar 20 23:00:22 2018 -0700
Committer: Aman Sinha
Committed: Thu Mar 29 23:22:25 2018 -0700
--
.../drill/exec/physical/impl/TopN/TopNBatchTest.java | 6 +-
.../java/org/apache/drill/test/BaseDirTestWatcher.java| 10 --
.../test/java/org/apache/drill/test/BaseTestQuery.java| 1 -
.../test/java/org/apache/drill/test/ClusterFixture.java | 1 -
4 files changed, 1 insertion(+), 17 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/drill/blob/7f645565/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TopN/TopNBatchTest.java
--
diff --git
a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TopN/TopNBatchTest.java
b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TopN/TopNBatchTest.java
index 4860869..14f2ee8 100644
---
a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TopN/TopNBatchTest.java
+++
b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TopN/TopNBatchTest.java
@@ -63,8 +63,6 @@ public class TopNBatchTest extends PopUnitTestBase {
@Test
public void priorityQueueOrderingTest() throws Exception {
Properties properties = new Properties();
-properties.setProperty(ClassBuilder.CODE_DIR_OPTION,
dirTestWatcher.getDir().getAbsolutePath());
-
DrillConfig drillConfig = DrillConfig.create(properties);
FieldReference expr = FieldReference.getWithQuotedRef("colA");
@@ -158,9 +156,7 @@ public class TopNBatchTest extends PopUnitTestBase {
*/
@Test
public void sortOneKeyAscending() throws Throwable {
-ClusterFixtureBuilder builder = ClusterFixture.builder(dirTestWatcher)
- .configProperty(ClassBuilder.CODE_DIR_OPTION,
dirTestWatcher.getDir().getAbsolutePath())
- .configProperty(CodeCompiler.ENABLE_SAVE_CODE_FOR_DEBUG_TOPN, true);
+ClusterFixtureBuilder builder = ClusterFixture.builder(dirTestWatcher);
try (ClusterFixture cluster = builder.build();
ClientFixture client = cluster.clientFixture()) {
TestBuilder testBuilder = new TestBuilder(new
ClusterFixture.FixtureTestServices(client));
http://git-wip-us.apache.org/repos/asf/drill/blob/7f645565/exec/java-exec/src/test/java/org/apache/drill/test/BaseDirTestWatcher.java
--
diff --git
a/exec/java-exec/src/test/java/org/apache/drill/test/BaseDirTestWatcher.java
b/exec/java-exec/src/test/java/org/apache/drill/test/BaseDirTestWatcher.java
index d36423b..b595869 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/test/BaseDirTestWatcher.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/test/BaseDirTestWatcher.java
@@ -53,7 +53,6 @@ public class BaseDirTestWatcher extends DirTestWatcher {
TEST_TMP // Corresponds to the directory that should be mapped to dfs.tmp
}
- private File codegenDir;
private File tmpDir;
private File storeDir;
private File dfsTestTmpParentDir;
@@ -79,7 +78,6 @@ public class BaseDirTestWatcher extends DirTestWatcher {
protected void starting(Description description) {
super.starting(description);
-codegenDir = makeSubDir(Paths.get("codegen"));
rootDir = makeSubDir(Paths.get("root"));
tmpDir = makeSubDir(Paths.get("tmp"));
storeDir = makeSubDir(Paths.get("store"));
@@ -135,14 +133,6 @@ public class BaseDirTestWatcher extends DirTestWatcher {
}
/**
- * Gets the temp directory that should be used to save generated code files.
- * @return The temp directory that should be used to save generated code
files.
- */
- public File getCodegenDir() {
-return codegenDir;
- }
-
- /**
* This methods creates a new directory which can be mapped to
dfs.tmp.
*/
public void newDfsTestTmpDir() {
http://git-wip-us.apache.org/repos/asf/drill/blob/7f645565/exec/java-exec/src/test/java/org/apache/drill/test/BaseTestQuery.java
--
diff --git
a/exec/java-exec/src/test/java/org/apache/drill/test/BaseTestQuery.java
b/exec/java-exec/src/test/java/org/apache/drill/test/BaseTestQuery.java
index c3ecaf1..10cd94c 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/test/BaseTestQuery.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/test/BaseTestQuery.java
@@ -160,7 +160,6 @
[4/7] drill git commit: DRILL-6254: IllegalArgumentException: the requested size must be non-negative
DRILL-6254: IllegalArgumentException: the requested size must be non-negative
close apache/drill#1179
Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/67710bba
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/67710bba
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/67710bba
Branch: refs/heads/master
Commit: 67710bba7cdbc05428df7390bad8639b099769fc
Parents: 7088bfe
Author: Padma Penumarthy
Authored: Wed Mar 21 13:39:43 2018 -0700
Committer: Aman Sinha
Committed: Thu Mar 29 23:21:50 2018 -0700
--
.../drill/exec/physical/impl/flatten/FlattenRecordBatch.java| 5 -
1 file changed, 4 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/drill/blob/67710bba/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/flatten/FlattenRecordBatch.java
--
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/flatten/FlattenRecordBatch.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/flatten/FlattenRecordBatch.java
index 9dd1770..7509809 100644
---
a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/flatten/FlattenRecordBatch.java
+++
b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/flatten/FlattenRecordBatch.java
@@ -237,7 +237,10 @@ public class FlattenRecordBatch extends
AbstractSingleRecordBatch {
private void handleRemainder() {
int remainingRecordCount =
flattener.getFlattenField().getAccessor().getInnerValueCount() - remainderIndex;
-if (!doAlloc(remainingRecordCount)) {
+
+// remainingRecordCount can be much higher than number of rows we will
have in outgoing batch.
+// Do memory allocation only for number of rows we are going to have in
the batch.
+if (!doAlloc(Math.min(remainingRecordCount,
flattenMemoryManager.getOutputRowCount( {
outOfMemory = true;
return;
}
[3/7] drill git commit: DRILL-5937: drill-module.conf : Changed timeout to 30 seconds, ExecConstant.java : Changed comment
DRILL-5937: drill-module.conf : Changed timeout to 30 seconds,
ExecConstant.java : Changed comment
DRILL-5937: ExecConstant.java : removed comment
DRILL-5937: CheckStyle fix
close apache/drill#1190
Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/7088bfe4
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/7088bfe4
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/7088bfe4
Branch: refs/heads/master
Commit: 7088bfe4bd6e2379ba21b0c48f7280a77d2c3b83
Parents: ea643bf
Author: Pushpendra Jaiswal
Authored: Tue Mar 27 13:16:23 2018 +0530
Committer: Aman Sinha
Committed: Thu Mar 29 23:20:30 2018 -0700
--
.../src/main/java/org/apache/drill/exec/ExecConstants.java | 4
exec/java-exec/src/main/resources/drill-module.conf | 2 +-
2 files changed, 1 insertion(+), 5 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/drill/blob/7088bfe4/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java
--
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java
index 34aec1b..77fa211 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java
@@ -561,10 +561,6 @@ public final class ExecConstants {
public static final String CODE_GEN_EXP_IN_METHOD_SIZE =
"exec.java.compiler.exp_in_method_size";
public static final LongValidator CODE_GEN_EXP_IN_METHOD_SIZE_VALIDATOR =
new LongValidator(CODE_GEN_EXP_IN_METHOD_SIZE);
- /**
- * Timeout for create prepare statement request. If the request exceeds this
timeout, then request is timed out.
- * Default value is 10mins.
- */
public static final String CREATE_PREPARE_STATEMENT_TIMEOUT_MILLIS =
"prepare.statement.create_timeout_ms";
public static final OptionValidator
CREATE_PREPARE_STATEMENT_TIMEOUT_MILLIS_VALIDATOR =
new PositiveLongValidator(CREATE_PREPARE_STATEMENT_TIMEOUT_MILLIS,
Integer.MAX_VALUE);
http://git-wip-us.apache.org/repos/asf/drill/blob/7088bfe4/exec/java-exec/src/main/resources/drill-module.conf
--
diff --git a/exec/java-exec/src/main/resources/drill-module.conf
b/exec/java-exec/src/main/resources/drill-module.conf
index a227e0d..9af09bc 100644
--- a/exec/java-exec/src/main/resources/drill-module.conf
+++ b/exec/java-exec/src/main/resources/drill-module.conf
@@ -514,7 +514,7 @@ drill.exec.options: {
# it is dynamically computed based on cpu_load_average
planner.width.max_per_node: 0,
planner.width.max_per_query: 1000,
-prepare.statement.create_timeout_ms: 1,
+prepare.statement.create_timeout_ms: 3,
security.admin.user_groups: "%drill_process_user_groups%",
security.admin.users: "%drill_process_user%",
store.format: "parquet",
[2/2] drill git commit: DRILL-6118: Handle item star columns during project / filter push down and directory pruning
DRILL-6118: Handle item star columns during project / filter push down and
directory pruning
1. Added DrillFilterItemStarReWriterRule to re-write item star fields to
regular field references.
2. Refactored DrillPushProjectIntoScanRule to handle item star fields, factored
out helper classes and methods from PreUitl.class.
3. Fixed issue with dynamic star usage (after Calcite upgrade old usage of star
was still present, replaced WILDCARD -> DYNAMIC_STAR for clarity).
4. Added unit tests to check project / filter push down and directory pruning
with item star.
Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/9073aed6
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/9073aed6
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/9073aed6
Branch: refs/heads/master
Commit: 9073aed67d89e8b2188870d6c812706085c9c41b
Parents: 50efb80
Author: Arina Ielchiieva
Authored: Thu Dec 21 19:31:00 2017 +0200
Committer: Aman Sinha
Committed: Sat Feb 24 19:56:35 2018 -0800
--
.../org/apache/drill/exec/dotdrill/View.java| 5 +-
.../impl/project/ProjectRecordBatch.java| 12 +-
.../impl/values/ValuesBatchCreator.java | 5 +-
.../apache/drill/exec/planner/PlannerPhase.java | 7 +-
.../drill/exec/planner/StarColumnHelper.java| 13 +-
.../exec/planner/common/DrillScanRelBase.java | 10 +-
.../DrillFilterItemStarReWriterRule.java| 232 +
.../drill/exec/planner/logical/DrillOptiq.java | 3 +-
.../planner/logical/DrillPushProjIntoScan.java | 97 ---
.../logical/DrillPushProjectIntoScanRule.java | 256 +++
.../planner/logical/FieldsReWriterUtil.java | 138 ++
.../planner/logical/PreProcessLogicalRel.java | 6 +-
.../drill/exec/planner/physical/PrelUtil.java | 247 +-
.../visitor/SplitUpComplexExpressions.java | 6 +-
.../planner/sql/handlers/SqlHandlerUtil.java| 4 +-
.../exec/planner/types/RelDataTypeHolder.java | 13 +-
.../text/compliant/RepeatedVarCharOutput.java | 7 +-
.../drill/exec/store/mock/MockGroupScanPOP.java | 2 +-
.../parquet2/DrillParquetGroupConverter.java| 12 +-
.../exec/store/text/DrillTextRecordReader.java | 4 +-
.../org/apache/drill/exec/util/Utilities.java | 39 ++-
.../exec/vector/complex/fn/FieldSelection.java | 13 +-
.../apache/drill/exec/TestWindowFunctions.java | 4 +-
.../physical/unit/MiniPlanUnitTestBase.java | 3 +-
.../physical/unit/PhysicalOpUnitTestBase.java | 3 +-
.../TestPushDownAndPruningWithItemStar.java | 183 +
.../drill/common/expression/SchemaPath.java | 12 +-
27 files changed, 923 insertions(+), 413 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/drill/blob/9073aed6/exec/java-exec/src/main/java/org/apache/drill/exec/dotdrill/View.java
--
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/dotdrill/View.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/dotdrill/View.java
index 3524d73..615e3bc 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/dotdrill/View.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/dotdrill/View.java
@@ -23,6 +23,7 @@ import org.apache.calcite.avatica.util.TimeUnit;
import org.apache.calcite.sql.SqlIntervalQualifier;
import org.apache.calcite.sql.parser.SqlParserPos;
import org.apache.calcite.sql.type.SqlTypeFamily;
+import org.apache.drill.common.expression.SchemaPath;
import org.apache.drill.exec.planner.StarColumnHelper;
import org.apache.drill.exec.planner.types.RelDataTypeDrillImpl;
import org.apache.drill.exec.planner.types.RelDataTypeHolder;
@@ -72,9 +73,9 @@ public class View {
@JsonProperty("fractionalSecondPrecision") Integer
fractionalSecondPrecision,
@JsonProperty("isNullable") Boolean isNullable) {
// Fix for views which were created on Calcite 1.4.
- // After Calcite upgrade star "*" was changed on dynamic star "**"
+ // After Calcite upgrade star "*" was changed on dynamic star "**"
(SchemaPath.DYNAMIC_STAR)
// and type of star was changed to SqlTypeName.DYNAMIC_STAR
- this.name = "*".equals(name) ? "**" : name;
+ this.name = "*".equals(name) ? SchemaPath.DYNAMIC_STAR : name;
this.type = "*".equals(name) && type == SqlTypeName.ANY ?
SqlTypeName.DYNAMIC_STAR : type;
this.precision = precision;
this.scale = scale;
http://git-wip-us.apache.org/repos/asf/drill/blob/9073aed6/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectRecordBatch.java
--
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/Pr
[1/2] drill git commit: DRILL-6118: Handle item star columns during project / filter push down and directory pruning
Repository: drill
Updated Branches:
refs/heads/master 50efb806b -> 9073aed67
http://git-wip-us.apache.org/repos/asf/drill/blob/9073aed6/logical/src/main/java/org/apache/drill/common/expression/SchemaPath.java
--
diff --git
a/logical/src/main/java/org/apache/drill/common/expression/SchemaPath.java
b/logical/src/main/java/org/apache/drill/common/expression/SchemaPath.java
index 95f3dbb..583046a 100644
--- a/logical/src/main/java/org/apache/drill/common/expression/SchemaPath.java
+++ b/logical/src/main/java/org/apache/drill/common/expression/SchemaPath.java
@@ -42,8 +42,8 @@ import com.google.common.base.Preconditions;
public class SchemaPath extends LogicalExpressionBase {
- public static final String WILDCARD = "*";
- public static final SchemaPath STAR_COLUMN = getSimplePath(WILDCARD);
+ public static final String DYNAMIC_STAR = "**";
+ public static final SchemaPath STAR_COLUMN = getSimplePath(DYNAMIC_STAR);
private final NameSegment rootSegment;
@@ -205,14 +205,14 @@ public class SchemaPath extends LogicalExpressionBase {
}
/**
- * Return if this column is the special wildcard ("*") column which means to
+ * Return if this column is the special wildcard ("**") column which means to
* project all table columns.
*
- * @return true if the column is "*"
+ * @return true if the column is "**"
*/
- public boolean isWildcard() {
-return isLeaf() && nameEquals(WILDCARD);
+ public boolean isDynamicStar() {
+return isLeaf() && nameEquals(DYNAMIC_STAR);
}
/**
[3/3] drill git commit: DRILL-5741: Automatically manage memory allocations during startup
DRILL-5741: Automatically manage memory allocations during startup **Note:** This commit is rebased on (hence, requires) DRILL-6068 commit This commit provides a way for distributions and users to automatically define the Drillbit JVM's Heap, Direct and CodeCache allocations. To do this, it leverages the DRILL-6068 commit, which provides for configuration setup. The only new variable introduced is an environment variable - `DRILLBIT_MAX_PROC_MEM` that defines a Drillbit's max memory parameters. The logic defined by `auto-setup.sh` will check that if any (or all) of the JVM memory parameters have been declared; they, cumulatively, don't exceed the value specified by `DRILLBIT_MAX_PROC_MEM`. ``` NOTE: To disable this, simply unset (or don't define) the environment variable DRILLBIT_MAX_PROC_MEM ``` The variable can be defined in KB, MB, or GB; similar in syntax to how the JVM MaxHeap is specified. For e.g. ``` DRILLBIT_MAX_PROC_MEM=13G DRILLBIT_MAX_PROC_MEM=8192m DRILLBIT_MAX_PROC_MEM=4194304K ``` In addition, you can specify it as a percent of the total system memory prior to the Drillbit starting up: `DRILLBIT_MAX_PROC_MEM=40%` For a system with with 48GB free memory, when set to (say) 25% (with settings defined in drill-env.sh), and heap (8GB) and direct (10GB) are defined; the Drillbit fails startup with the following message: ``` 2018-01-03 14:27:57 [WARN] 25% of System Memory (47 GB) translates to 12 GB 2018-01-03 14:27:57 [ERROR]Unable to start Drillbit due to memory constraint violations Total Memory Requested : 19 GB Check the following settings to possibly modify (or increase the Max Memory Permitted): DRILLBIT_MAX_PROC_MEM=25% DRILL_HEAP=8G DRILL_MAX_DIRECT_MEMORY=10G DRILLBIT_CODE_CACHE_SIZE=1024m *NOTE: It is recommended not to specify DRILLBIT_CODE_CACHE_SIZE as this will be auto-computed based on the HeapSize and would not exceed 1GB ``` For all other combinations, the undefined parameters are adjusted to ensure that the total memory allocated is within the value specified by `DRILLBIT_MAX_PROC_MEM`, For a system with with 48GB free memory, when set to (say) 50% (with settings defined in drill-env.sh), and heap (8GB) and direct (10GB) are defined; the Drillbit startup with the following warning: ``` 2018-01-03 14:31:06 [WARN] 50% of System Memory (47 GB) translates to 24 GB 2018-01-03 14:31:06 [WARN] You have an allocation of 4 GB that is currently unused from a total of 24 GB. You can increase your existing memory configuration to use this extra memory DRILLBIT_MAX_PROC_MEM=50% DRILL_HEAP=8G DRILL_MAX_DIRECT_MEMORY=10G DRILLBIT_CODE_CACHE_SIZE=1024m *NOTE: It is recommended not to specify DRILLBIT_CODE_CACHE_SIZE as this will be auto-computed based on the HeapSize and would not exceed 1GB ``` In addition, if the available free memory is less than the allocation, an additional warning is provided under the assumption that the OS will reclaim more free memory when required: ``` 2018-01-03 14:31:06 [WARN] Total Memory Allocation for Drillbit (19GB) exceeds available free memory (11GB) 2018-01-03 14:31:06 [WARN] Drillbit will start up, but can potentially crash due to oversubscribing of system memory. ``` For more details, refer the attachments in https://issues.apache.org/jira/browse/DRILL-5741 Changes to auto configure messaging Publishing final values prior to startup Minor update for printing to console's err stream close apache/drill#1082 Project: http://git-wip-us.apache.org/repos/asf/drill/repo Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/50efb806 Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/50efb806 Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/50efb806 Branch: refs/heads/master Commit: 50efb806bb03494e1da4d6b48f90fbf58d699c18 Parents: 266250c Author: Kunal Khatua Authored: Thu Jan 11 17:35:13 2018 -0800 Committer: Aman Sinha Committed: Fri Feb 23 17:55:36 2018 -0800 -- distribution/src/resources/auto-setup.sh| 202 ++- distribution/src/resources/distrib-setup.sh | 2 +- distribution/src/resources/drill-env.sh | 10 ++ 3 files changed, 212 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/drill/blob/50efb806/distribution/src/resources/auto-setup.sh -- diff --git a/distribution/src/resources/auto-setup.sh b/distribution/src/resources/auto-setup.sh index 75bdda0..141648c 100644 --- a/distribution/src/resources/auto-setup.sh +++ b/distribution/src/resources/auto-setup.sh @@ -25,5 +25,205 @@ # if [ $status == "FAILED" ]; return 1; fi ###== -# FEATURES (Added here to describe supported add
[2/3] drill git commit: DRILL-6068: Support user/distrib-specific config checks during startup
DRILL-6068: Support user/distrib-specific config checks during startup
1. Allows for distrib/user specific checks to be done
2. Place-holder files for distribution and user specific checks
3. Moved JVM Version Check to head of script
Separation of setups into 3 levels
1. Apache Drill (Default)
2. Distribition Specific
3. User-Defined
1 & 2 are mutually exclusive.
3 is additional checks that the user can specify.
Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/266250cf
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/266250cf
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/266250cf
Branch: refs/heads/master
Commit: 266250cf6ca2269f8772796d6c4eb903aa795111
Parents: 27aa236
Author: Kunal Khatua
Authored: Thu Jan 11 15:45:04 2018 -0800
Committer: Aman Sinha
Committed: Fri Feb 23 17:55:06 2018 -0800
--
distribution/src/assemble/bin.xml | 15 ++
distribution/src/resources/auto-setup.sh| 29
distribution/src/resources/distrib-setup.sh | 29
distribution/src/resources/drill-config.sh | 178 +++
distribution/src/resources/drill-setup.sh | 26
5 files changed, 217 insertions(+), 60 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/drill/blob/266250cf/distribution/src/assemble/bin.xml
--
diff --git a/distribution/src/assemble/bin.xml
b/distribution/src/assemble/bin.xml
index b4290bb..bed34ed 100644
--- a/distribution/src/assemble/bin.xml
+++ b/distribution/src/assemble/bin.xml
@@ -347,6 +347,21 @@
conf
+ src/resources/auto-setup.sh
+ 0755
+ bin
+
+
+ src/resources/drill-setup.sh
+ 0755
+ conf
+
+
+ src/resources/distrib-setup.sh
+ 0755
+ conf
+
+
src/resources/drill-override-example.conf
conf
http://git-wip-us.apache.org/repos/asf/drill/blob/266250cf/distribution/src/resources/auto-setup.sh
--
diff --git a/distribution/src/resources/auto-setup.sh
b/distribution/src/resources/auto-setup.sh
new file mode 100644
index 000..75bdda0
--- /dev/null
+++ b/distribution/src/resources/auto-setup.sh
@@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This file is invoked by drill-config.sh during a Drillbit startup and
provides
+# default checks and autoconfiguration.
+# Distributions should not put anything in this file. Checks can be
+# specified in ${DRILL_HOME}/conf/distrib-setup.sh
+# Users should not put anything in this file. Additional checks can be defined
+# and put in ${DRILL_CONF_DIR}/drill-setup.sh instead.
+# To FAIL any check, return with a non-zero return code
+# e.g.
+# if [ $status == "FAILED" ]; return 1; fi
+
+###==
+# FEATURES (Added here to describe supported additions)
+###==
http://git-wip-us.apache.org/repos/asf/drill/blob/266250cf/distribution/src/resources/distrib-setup.sh
--
diff --git a/distribution/src/resources/distrib-setup.sh
b/distribution/src/resources/distrib-setup.sh
new file mode 100644
index 000..dbd2b90
--- /dev/null
+++ b/distribution/src/resources/distrib-setup.sh
@@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License
[1/3] drill git commit: DRILL-5978: Updating of Apache and MapR Hive libraries to 2.3.2 and 2.1.2-mapr-1710 versions respectively
Repository: drill
Updated Branches:
refs/heads/master 1c14d3c3c -> 50efb806b
DRILL-5978: Updating of Apache and MapR Hive libraries to 2.3.2 and
2.1.2-mapr-1710 versions respectively
* Improvements to allow of reading Hive bucketed transactional ORC tables;
* Updating hive properties for tests and resolving dependencies and API
conflicts:
- Fix for "hive.metastore.schema.verification", MetaException(message:
Version information
not found in metastore)
https://cwiki.apache.org/confluence/display/Hive/Hive+Schema+Tool
METASTORE_SCHEMA_VERIFICATION="false" property is added
- Added METASTORE_AUTO_CREATE_ALL="true", properties to tests, because some
additional
tables are necessary in Hive metastore
- Disabling calcite CBO for (Hive's CalcitePlanner) for tests, because it is
in conflict
with Drill's Calcite version for Drill unit tests. HIVE_CBO_ENABLED="false"
property
- jackson and parquet libraries are relocated in hive-exec-shade module
- org.apache.parquet:parquet-column Drill version is added to "hive-exec" to
allow of using Parquet empty group on MessageType level (PARQUET-278)
- Removing of commons-codec exclusion from hive core. This dependency is
necessary for hive-exec and hive-metastore.
- Setting Hive internal properties for transactional scan:
HiveConf.HIVE_TRANSACTIONAL_TABLE_SCAN and for schema evolution:
HiveConf.HIVE_SCHEMA_EVOLUTION,
IOConstants.SCHEMA_EVOLUTION_COLUMNS,
IOConstants.SCHEMA_EVOLUTION_COLUMNS_TYPES
- "io.dropwizard.metrics:metrics-core" with last 4.0.2 version is added to
dependencyManagement block in Drill root POM
- Exclusion of "hive-exec" in "hive-hbase-handler" is already in Drill root
dependencyManagement POM
- Hive Calcite libraries are excluded (Calcite CBO was disabled)
- "jackson-core" dependency is added to DependencyManagement block in Drill
root POM file
- For MapR Hive 2.1 client older
"com.fasterxml.jackson.core:jackson-databind" is included
- "log4j:log4j" dependency is excluded from "hive-exec", "hive-metastore",
"hive-hbase-handler".
close apache/drill#
Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/27aa2369
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/27aa2369
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/27aa2369
Branch: refs/heads/master
Commit: 27aa236975e6d998e5b5dc173de777c78bb44bdd
Parents: 1c14d3c
Author: Vitalii Diravka
Authored: Mon Nov 13 16:04:03 2017 +
Committer: Aman Sinha
Committed: Fri Feb 23 17:48:12 2018 -0800
--
contrib/storage-hive/core/pom.xml | 6 -
.../codegen/templates/HiveRecordReaders.java| 4 +-
.../exec/store/hive/HiveMetadataProvider.java | 1 +
.../drill/exec/store/hive/HiveUtilities.java| 65 +++-
.../store/hive/readers/HiveAbstractReader.java | 57 ---
.../apache/drill/exec/hive/TestHiveStorage.java | 10 ++
.../hive/BaseTestHiveImpersonation.java | 5 +-
.../hive/TestSqlStdBasedAuthorization.java | 8 +-
.../hive/TestStorageBasedHiveAuthorization.java | 6 +
.../exec/store/hive/HiveTestDataGenerator.java | 3 +
contrib/storage-hive/hive-exec-shade/pom.xml| 158 +++
exec/java-exec/pom.xml | 15 --
exec/jdbc-all/pom.xml | 20 ---
exec/jdbc/pom.xml | 1 -
pom.xml | 74 -
15 files changed, 292 insertions(+), 141 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/drill/blob/27aa2369/contrib/storage-hive/core/pom.xml
--
diff --git a/contrib/storage-hive/core/pom.xml
b/contrib/storage-hive/core/pom.xml
index cdf8fbe..184300f 100644
--- a/contrib/storage-hive/core/pom.xml
+++ b/contrib/storage-hive/core/pom.xml
@@ -63,12 +63,6 @@
org.apache.hive
hive-hbase-handler
-
-
- org.apache.hive
- hive-exec
-
-
org.apache.hbase
http://git-wip-us.apache.org/repos/asf/drill/blob/27aa2369/contrib/storage-hive/core/src/main/codegen/templates/HiveRecordReaders.java
--
diff --git
a/contrib/storage-hive/core/src/main/codegen/templates/HiveRecordReaders.java
b/contrib/storage-hive/core/src/main/codegen/templates/HiveRecordReaders.java
index 4a75ed3..a6e588b 100644
---
a/contrib/storage-hive/core/src/main/codegen/templates/HiveRecordReaders.java
+++
b/contrib/storage-hive/core/src/main/codegen/templates/HiveRecordReaders.java
@@ -130,7 +130,7 @@ public class Hive${entry.hiveReader}Reader extends
HiveAbstractReader {
while (!recordsInspector.isBatchFull() &&
hasNextValue(recordsInspector.getValueHold
[4/5] drill git commit: DRILL-6154: NaN, Infinity issues
DRILL-6154: NaN, Infinity issues
- changed comparison rules for NaN, Infinity values. For now NaN is the biggest
value, Infinity - second biggest value
- fixed min, max, trunc functions for NaN, Infinity values
- made drill use original sqrt function instead of power(x,0.5) substitution
close apache/drill#1123
Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/bcd358b3
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/bcd358b3
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/bcd358b3
Branch: refs/heads/master
Commit: bcd358b3c73f34381b543789dfc368bfaefa4ea9
Parents: b4d2a77
Author: vladimir tkach
Authored: Fri Feb 16 16:36:49 2018 +0200
Committer: Aman Sinha
Committed: Fri Feb 23 14:18:15 2018 -0800
--
.../codegen/templates/AggrTypeFunctions1.java | 23 ++--
.../codegen/templates/ComparisonFunctions.java | 13 -
.../main/codegen/templates/MathFunctions.java | 16 --
.../exec/planner/sql/DrillConvertletTable.java | 15 ++
.../org/apache/drill/TestExampleQueries.java| 1 -
.../fn/impl/TestMathFunctionsWithNanInf.java| 22 +---
.../vector/complex/writer/TestJsonNanInf.java | 57
7 files changed, 129 insertions(+), 18 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/drill/blob/bcd358b3/exec/java-exec/src/main/codegen/templates/AggrTypeFunctions1.java
--
diff --git a/exec/java-exec/src/main/codegen/templates/AggrTypeFunctions1.java
b/exec/java-exec/src/main/codegen/templates/AggrTypeFunctions1.java
index b363cd1..fa4e8f1 100644
--- a/exec/java-exec/src/main/codegen/templates/AggrTypeFunctions1.java
+++ b/exec/java-exec/src/main/codegen/templates/AggrTypeFunctions1.java
@@ -71,9 +71,9 @@ public static class ${type.inputType}${aggrtype.className}
implements DrillAggFu
<#elseif type.runningType?starts_with("BigInt")>
value.value = Long.MAX_VALUE;
<#elseif type.runningType?starts_with("Float4")>
- value.value = Float.MAX_VALUE;
+ value.value = Float.NaN;
<#elseif type.runningType?starts_with("Float8")>
- value.value = Double.MAX_VALUE;
+ value.value = Double.NaN;
<#elseif aggrtype.funcName == "max">
<#if type.runningType?starts_with("Bit")>
@@ -101,8 +101,21 @@ public static class ${type.inputType}${aggrtype.className}
implements DrillAggFu
}
nonNullCount.value = 1;
+ // For min/max functions: NaN is the biggest value,
+// Infinity is the second biggest value
+// -Infinity is the smallest value
<#if aggrtype.funcName == "min">
- value.value = Math.min(value.value, in.value);
+ <#if type.inputType?contains("Float4")>
+ if(!Float.isNaN(in.value)) {
+ value.value = Float.isNaN(value.value) ? in.value :
Math.min(value.value, in.value);
+ }
+ <#elseif type.inputType?contains("Float8")>
+ if(!Double.isNaN(in.value)) {
+ value.value = Double.isNaN(value.value) ? in.value :
Math.min(value.value, in.value);
+ }
+<#else>
+ value.value = Math.min(value.value, in.value);
+
<#elseif aggrtype.funcName == "max">
value.value = Math.max(value.value, in.value);
<#elseif aggrtype.funcName == "sum">
@@ -138,9 +151,9 @@ public static class ${type.inputType}${aggrtype.className}
implements DrillAggFu
<#elseif type.runningType?starts_with("BigInt")>
value.value = Long.MAX_VALUE;
<#elseif type.runningType?starts_with("Float4")>
- value.value = Float.MAX_VALUE;
+ value.value = Float.NaN;
<#elseif type.runningType?starts_with("Float8")>
- value.value = Double.MAX_VALUE;
+ value.value = Double.NaN;
<#elseif aggrtype.funcName == "max">
<#if type.runningType?starts_with("Int")>
http://git-wip-us.apache.org/repos/asf/drill/blob/bcd358b3/exec/java-exec/src/main/codegen/templates/ComparisonFunctions.java
--
diff --git a/exec/java-exec/src/main/codegen/templates/ComparisonFunctions.java
b/exec/java-exec/src/main/codegen/templates/ComparisonFunctions.java
index 633bb56..12a4ef2 100644
--- a/exec/java-exec/src/main/codegen/templates/ComparisonFunctions.java
+++ b/exec/java-exec/src/main/codegen/templates/ComparisonFunctions.java
@@ -118,9 +118,18 @@
{
<@compareNullsSubblock leftType=leftType rightType=rightType
output="out.value" breakTarget="outside" nullCompare=true
nullComparesHigh=nullComparesHigh />
-
+//
[3/5] drill git commit: DRILL-5902: Queries encounter random failure due to RPC connection timed out
DRILL-5902: Queries encounter random failure due to RPC connection timed out
close apache/drill#1113
Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/b4d2a770
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/b4d2a770
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/b4d2a770
Branch: refs/heads/master
Commit: b4d2a77038421b243970da71655311076c0a5a43
Parents: 4f203ea
Author: Vlad Rozov
Authored: Mon Feb 5 19:15:56 2018 -0800
Committer: Aman Sinha
Committed: Fri Feb 23 14:15:00 2018 -0800
--
.../server/rest/profile/ProfileResources.java | 5 +--
.../org/apache/drill/exec/work/WorkManager.java | 39
.../exec/work/batch/ControlMessageHandler.java | 4 +-
.../exec/work/foreman/QueryStateProcessor.java | 14 +++
.../apache/drill/exec/work/user/UserWorker.java | 5 +--
5 files changed, 48 insertions(+), 19 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/drill/blob/b4d2a770/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/profile/ProfileResources.java
--
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/profile/ProfileResources.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/profile/ProfileResources.java
index 8751ee6..ec06f0e 100644
---
a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/profile/ProfileResources.java
+++
b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/profile/ProfileResources.java
@@ -383,10 +383,7 @@ public class ProfileResources {
QueryId id = QueryIdHelper.getQueryIdFromString(queryId);
// first check local running
-Foreman f = work.getBee().getForemanForQueryId(id);
-if(f != null){
-
checkOrThrowQueryCancelAuthorization(f.getQueryContext().getQueryUserName(),
queryId);
- f.cancel();
+if (work.getBee().cancelForeman(id, principal)) {
return String.format("Cancelled query %s on locally running node.",
queryId);
}
http://git-wip-us.apache.org/repos/asf/drill/blob/b4d2a770/exec/java-exec/src/main/java/org/apache/drill/exec/work/WorkManager.java
--
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/work/WorkManager.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/work/WorkManager.java
index d75668c..7058c62 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/work/WorkManager.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/work/WorkManager.java
@@ -22,6 +22,7 @@ import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import org.apache.drill.common.SelfCleaningRunnable;
+import org.apache.drill.common.exceptions.UserException;
import org.apache.drill.exec.coord.ClusterCoordinator;
import org.apache.drill.exec.metrics.DrillMetrics;
import org.apache.drill.exec.proto.BitControl.FragmentStatus;
@@ -37,6 +38,7 @@ import org.apache.drill.exec.rpc.control.WorkEventBus;
import org.apache.drill.exec.rpc.data.DataConnectionCreator;
import org.apache.drill.exec.server.BootStrapContext;
import org.apache.drill.exec.server.DrillbitContext;
+import org.apache.drill.exec.server.rest.auth.DrillUserPrincipal;
import org.apache.drill.exec.store.sys.PersistentStoreProvider;
import org.apache.drill.exec.work.batch.ControlMessageHandler;
import org.apache.drill.exec.work.foreman.Foreman;
@@ -259,6 +261,43 @@ public class WorkManager implements AutoCloseable {
executor.execute(runnable);
}
+public boolean cancelForeman(final QueryId queryId, DrillUserPrincipal
principal) {
+ Preconditions.checkNotNull(queryId);
+
+ final Foreman foreman = queries.get(queryId);
+ if (foreman == null) {
+return false;
+ }
+
+ final String queryIdString = QueryIdHelper.getQueryId(queryId);
+
+ if (principal != null &&
!principal.canManageQueryOf(foreman.getQueryContext().getQueryUserName())) {
+throw UserException.permissionError()
+.message("Not authorized to cancel the query '%s'", queryIdString)
+.build(logger);
+ }
+
+ executor.execute(new Runnable()
+ {
+@Override
+public void run()
+{
+ final Thread currentThread = Thread.currentThread();
+ final String originalName = currentThread.getName();
+ try {
+currentThread.setName(queryIdString + ":foreman:cancel");
+logger.debug("Canceling foreman");
+foreman.cancel();
+ } catch (Throwable t) {
+logger.warn("Exception while canceling foreman", t);
+ } finally {
+currentThread.setName(original
[5/5] drill git commit: DRILL-6172: setValueCount of VariableLengthVectors throws IOB exception when called with 0 value after clearing vectors
DRILL-6172: setValueCount of VariableLengthVectors throws IOB exception when
called with 0 value after clearing vectors
close apache/drill#1124
Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/1c14d3c3
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/1c14d3c3
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/1c14d3c3
Branch: refs/heads/master
Commit: 1c14d3c3c836515b375ae7c47c4179503b1dcee0
Parents: bcd358b
Author: Sorabh Hamirwasia
Authored: Tue Feb 20 17:14:29 2018 -0800
Committer: Aman Sinha
Committed: Fri Feb 23 14:19:13 2018 -0800
--
.../exec/record/vector/TestValueVector.java | 42
.../templates/VariableLengthVectors.java| 6 ++-
2 files changed, 47 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/drill/blob/1c14d3c3/exec/java-exec/src/test/java/org/apache/drill/exec/record/vector/TestValueVector.java
--
diff --git
a/exec/java-exec/src/test/java/org/apache/drill/exec/record/vector/TestValueVector.java
b/exec/java-exec/src/test/java/org/apache/drill/exec/record/vector/TestValueVector.java
index 621d288..ba4c79c 100644
---
a/exec/java-exec/src/test/java/org/apache/drill/exec/record/vector/TestValueVector.java
+++
b/exec/java-exec/src/test/java/org/apache/drill/exec/record/vector/TestValueVector.java
@@ -17,6 +17,7 @@
*/
package org.apache.drill.exec.record.vector;
+import static junit.framework.TestCase.fail;
import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
@@ -48,6 +49,7 @@ import org.apache.drill.exec.memory.BufferAllocator;
import org.apache.drill.exec.memory.RootAllocatorFactory;
import org.apache.drill.exec.proto.UserBitShared;
import org.apache.drill.exec.record.MaterializedField;
+import org.apache.drill.exec.record.VectorAccessibleUtilities;
import org.apache.drill.exec.vector.BaseValueVector;
import org.apache.drill.exec.vector.BitVector;
import org.apache.drill.exec.vector.NullableFloat4Vector;
@@ -57,6 +59,7 @@ import org.apache.drill.exec.vector.RepeatedIntVector;
import org.apache.drill.exec.vector.UInt4Vector;
import org.apache.drill.exec.vector.ValueVector;
import org.apache.drill.exec.vector.VarCharVector;
+import org.apache.drill.exec.vector.VariableWidthVector;
import org.apache.drill.exec.vector.complex.ListVector;
import org.apache.drill.exec.vector.complex.MapVector;
import org.apache.drill.exec.vector.complex.RepeatedListVector;
@@ -828,4 +831,43 @@ the interface to load has changed
vector.clear();
}
+ /**
+ * For VariableLengthVectors when we clear of the vector and then explicitly
set the
+ * ValueCount of zero, then it should not fail with IndexOutOfBoundException.
+ * @throws Exception
+ */
+ @Test
+ public void testVarLengthVector_SetCountZeroAfterClear() throws Exception {
+try {
+ final MaterializedField field =
MaterializedField.create(EMPTY_SCHEMA_PATH, VarCharHolder.TYPE);
+ VariableWidthVector vector = new VarCharVector(field, allocator);
+ vector.allocateNew();
+ vector.clear();
+ assertTrue(vector.getAccessor().getValueCount() == 0);
+ vector.getMutator().setValueCount(0);
+ assertTrue(vector.getAccessor().getValueCount() == 0);
+} catch (Exception ex) {
+ fail();
+}
+ }
+
+ /** For VariableLengthVectors when we try to set value count greater than
value count for which memory is allocated,
+ * then it should fail with IndexOutOfBoundException.
+ * @throws Exception
+ */
+ @Test
+ public void testVarLengthVector_SetOOBCount() throws Exception {
+final MaterializedField field =
MaterializedField.create(EMPTY_SCHEMA_PATH, VarCharHolder.TYPE);
+VariableWidthVector vector = new VarCharVector(field, allocator);
+try {
+ vector.allocateNew(10, 1);
+ vector.getMutator().setValueCount(4);
+ fail();
+} catch (Exception ex) {
+ assertTrue(ex instanceof IndexOutOfBoundsException);
+} finally {
+ vector.clear();
+}
+ }
+
}
http://git-wip-us.apache.org/repos/asf/drill/blob/1c14d3c3/exec/vector/src/main/codegen/templates/VariableLengthVectors.java
--
diff --git a/exec/vector/src/main/codegen/templates/VariableLengthVectors.java
b/exec/vector/src/main/codegen/templates/VariableLengthVectors.java
index 87dbe95..3dec3f8 100644
--- a/exec/vector/src/main/codegen/templates/VariableLengthVectors.java
+++ b/exec/vector/src/main/codegen/templates/VariableLengthVectors.java
@@ -700,7 +700,11 @@ public final class ${minor.class}Vector extends
BaseDataValueVector implements V
@Override
public void setValueCount(int valu
[1/5] drill git commit: DRILL-6090: While connecting to drill-bits using JDBC Driver through Zookeeper, a lot of "Curator-Framework-0" threads are created if connection to drill-bit is not successful(
Repository: drill
Updated Branches:
refs/heads/master 894c0f58e -> 1c14d3c3c
DRILL-6090: While connecting to drill-bits using JDBC Driver through Zookeeper,
a lot of "Curator-Framework-0" threads are created if connection to drill-bit
is not successful(no drill-bits are up/reachable)
I am using Drill JDBC driver 1.12.0 to connect to MapR-DB. I am finding the
available drill-bits using Zookeepers. When drill-bits are not up or not
reachable, the connection is failed with exception: "Failure in connecting to
Drill: oadd.org.apache.drill.exec.rpc.RpcException: Failure setting up ZK for
client", which is expected, but number of threads created by
ZKClusterCoordinator just keeps on increasing.
Steps to reproduce the issue
Setup a connection with a drill-bit using Apache Drill JDBC driver 1.12.0
through Zookeeper hosts(port 5181)
Now stop the drill-bit services or block the drill-bit IPs using iptable rules
Truncate catalina logs
Try to connect to the drill-bit/hit a code path that requires connection to
drill-bits.
Take thread dump using kill -QUIT
grep -c "Curator-Framework-0" catalina.out
Observe that the curator framework thread just keep on accumulating
RCA:
ZKClusterCoordinator creates curator threads in the constructor
ZKClusterCoordinator is instantiated by DrillClient.connect
DrillClient.connect is called in DrillConnectionImpl constructor
Fix:
Call DrillConnectionImpl .cleanup() from all the catch blocks in the
DrillConnectionImpl constructor.
close apache/drill#1094
Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/8cb32343
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/8cb32343
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/8cb32343
Branch: refs/heads/master
Commit: 8cb32343b2816fe97f04efc055e6ffba77a9116a
Parents: 894c0f5
Author: milindt <35451050+mili...@users.noreply.github.com>
Authored: Wed Jan 17 14:35:33 2018 +0530
Committer: Aman Sinha
Committed: Fri Feb 23 14:08:30 2018 -0800
--
.../drill/jdbc/impl/DrillConnectionImpl.java| 130 ++-
1 file changed, 69 insertions(+), 61 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/drill/blob/8cb32343/exec/jdbc/src/main/java/org/apache/drill/jdbc/impl/DrillConnectionImpl.java
--
diff --git
a/exec/jdbc/src/main/java/org/apache/drill/jdbc/impl/DrillConnectionImpl.java
b/exec/jdbc/src/main/java/org/apache/drill/jdbc/impl/DrillConnectionImpl.java
index 689041c..4c1ac46 100644
---
a/exec/jdbc/src/main/java/org/apache/drill/jdbc/impl/DrillConnectionImpl.java
+++
b/exec/jdbc/src/main/java/org/apache/drill/jdbc/impl/DrillConnectionImpl.java
@@ -110,73 +110,81 @@ class DrillConnectionImpl extends AvaticaConnection
super.setReadOnly(false);
this.config = new DrillConnectionConfig(info);
+try{
+ try {
+String connect = null;
-try {
- String connect = null;
-
- if (config.isLocal()) {
-try {
- Class.forName("org.eclipse.jetty.server.Handler");
-} catch (final ClassNotFoundException e) {
- throw new SQLNonTransientConnectionException(
- "Running Drill in embedded mode using Drill's jdbc-all JDBC"
- + " driver Jar file alone is not supported.", e);
-}
-
-final DrillConfig dConfig = DrillConfig.create(info);
-this.allocator = RootAllocatorFactory.newRoot(dConfig);
-RemoteServiceSet set = GlobalServiceSetReference.SETS.get();
-if (set == null) {
- // We're embedded; start a local drill bit.
- serviceSet = RemoteServiceSet.getLocalServiceSet();
- set = serviceSet;
+if (config.isLocal()) {
try {
-bit = new Drillbit(dConfig, serviceSet);
-bit.run();
- } catch (final UserException e) {
-throw new SQLException(
-"Failure in starting embedded Drillbit: " + e.getMessage(),
-e);
- } catch (Exception e) {
-// (Include cause exception's text in wrapping exception's text so
-// it's more likely to get to user (e.g., via SQLLine), and use
-// toString() since getMessage() text doesn't always mention
error:)
-throw new SQLException("Failure in starting embedded Drillbit: " +
e, e);
+Class.forName("org.eclipse.jetty.server.Handler");
+ } catch (final ClassNotFoundException e) {
+throw new SQLNonTransientConnectionException(
+"Running Drill in embedded mode using Drill's jdbc-all JDBC"
++ " driver Jar file alone is not supported.", e);
+ }
+
+ final DrillConfig dConfig = DrillConfig.create(info);
+ this.allocator = RootAllocato
[2/5] drill git commit: DRILL-6115: SingleMergeExchange is not scaling up when many minor fragments are allocated for a query.
DRILL-6115: SingleMergeExchange is not scaling up when many minor fragments are
allocated for a query.
DRILL-6115: Refactoring the existing code.
close apache/drill#1110
Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/4f203ea9
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/4f203ea9
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/4f203ea9
Branch: refs/heads/master
Commit: 4f203ea99e2d6bd5596672a0ea67445720630228
Parents: 8cb3234
Author: Hanumath Rao Maduri
Authored: Mon Feb 12 18:19:07 2018 -0800
Committer: Aman Sinha
Committed: Fri Feb 23 14:13:40 2018 -0800
--
.../org/apache/drill/exec/ExecConstants.java| 3 +
.../physical/base/AbstractPhysicalVisitor.java | 18 --
.../exec/physical/base/PhysicalVisitor.java | 7 -
.../physical/config/AbstractMuxExchange.java| 15 ++
.../physical/config/OrderedMuxExchange.java | 52 +
.../physical/config/UnorderedMuxExchange.java | 9 +-
.../exec/planner/physical/ExchangePrel.java | 13 ++
.../exec/planner/physical/HashPrelUtil.java | 23 ++-
.../physical/HashToMergeExchangePrel.java | 4 -
.../physical/HashToRandomExchangePrel.java | 77
.../physical/OrderedMuxExchangePrel.java| 74 +++
.../exec/planner/physical/PlannerSettings.java | 1 +
.../physical/SingleMergeExchangePrel.java | 21 ++
.../visitor/InsertLocalExchangeVisitor.java | 115 +--
.../planner/physical/visitor/PrelVisitor.java | 1 -
.../server/options/SystemOptionManager.java | 1 +
.../src/main/resources/drill-module.conf| 1 +
.../physical/impl/TestOrderedMuxExchange.java | 193 +++
18 files changed, 479 insertions(+), 149 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/drill/blob/4f203ea9/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java
--
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java
index a1a94fa..fb2907d 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java
@@ -455,6 +455,9 @@ public final class ExecConstants {
public static final String AVERAGE_FIELD_WIDTH_KEY =
"planner.memory.average_field_width";
public static final OptionValidator AVERAGE_FIELD_WIDTH = new
PositiveLongValidator(AVERAGE_FIELD_WIDTH_KEY, Long.MAX_VALUE);
+ // Mux Exchange options.
+ public static final String ORDERED_MUX_EXCHANGE =
"planner.enable_ordered_mux_exchange";
+
// Resource management boot-time options.
public static final String MAX_MEMORY_PER_NODE =
"drill.exec.rm.memory_per_node";
http://git-wip-us.apache.org/repos/asf/drill/blob/4f203ea9/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/AbstractPhysicalVisitor.java
--
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/AbstractPhysicalVisitor.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/AbstractPhysicalVisitor.java
index a3b5f27..9933ddc 100644
---
a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/AbstractPhysicalVisitor.java
+++
b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/AbstractPhysicalVisitor.java
@@ -23,7 +23,6 @@ import org.apache.drill.exec.physical.config.FlattenPOP;
import org.apache.drill.exec.physical.config.HashAggregate;
import org.apache.drill.exec.physical.config.HashJoinPOP;
import org.apache.drill.exec.physical.config.HashPartitionSender;
-import org.apache.drill.exec.physical.config.HashToRandomExchange;
import org.apache.drill.exec.physical.config.IteratorValidator;
import org.apache.drill.exec.physical.config.Limit;
import org.apache.drill.exec.physical.config.MergeJoinPOP;
@@ -32,14 +31,12 @@ import
org.apache.drill.exec.physical.config.NestedLoopJoinPOP;
import org.apache.drill.exec.physical.config.OrderedPartitionSender;
import org.apache.drill.exec.physical.config.ProducerConsumer;
import org.apache.drill.exec.physical.config.Project;
-import org.apache.drill.exec.physical.config.RangeSender;
import org.apache.drill.exec.physical.config.Screen;
import org.apache.drill.exec.physical.config.SingleSender;
import org.apache.drill.exec.physical.config.Sort;
import org.apache.drill.exec.physical.config.StreamingAggregate;
import org.apache.drill.exec.physical.config.Trace;
import org.apache.drill.exec.physical.config.UnionAll;
-import org.apache.drill.exec.physical.config.UnionExchange;
import org.apache.drill.exec.physical.config.UnorderedReceiver;
import org.
drill git commit: DRILL-6100: Intermittent failure while reading Parquet file footer during planning phase
Repository: drill
Updated Branches:
refs/heads/master e791ed62b -> 9926eda21
DRILL-6100: Intermittent failure while reading Parquet file footer during
planning phase
close apache/drill#1097
Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/9926eda2
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/9926eda2
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/9926eda2
Branch: refs/heads/master
Commit: 9926eda21c748e96d67bce341a76dac3114002af
Parents: e791ed6
Author: Sorabh Hamirwasia
Authored: Sun Jan 21 16:30:25 2018 -0800
Committer: Aman Sinha
Committed: Wed Jan 24 13:12:31 2018 -0800
--
.../drill/exec/store/parquet/Metadata.java | 19 +--
1 file changed, 17 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/drill/blob/9926eda2/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/Metadata.java
--
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/Metadata.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/Metadata.java
index a611c6f..ab3bc19 100644
---
a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/Metadata.java
+++
b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/Metadata.java
@@ -18,6 +18,7 @@
package org.apache.drill.exec.store.parquet;
import java.io.IOException;
+import java.security.PrivilegedExceptionAction;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
@@ -41,6 +42,7 @@ import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.security.UserGroupInformation;
import org.apache.parquet.column.statistics.Statistics;
import org.apache.parquet.hadoop.ParquetFileReader;
import org.apache.parquet.hadoop.metadata.BlockMetaData;
@@ -416,8 +418,21 @@ public class Metadata {
* Get the metadata for a single file
*/
private ParquetFileMetadata_v3
getParquetFileMetadata_v3(ParquetTableMetadata_v3 parquetTableMetadata,
- FileStatus file) throws IOException {
-ParquetMetadata metadata = ParquetFileReader.readFooter(fs.getConf(),
file);
+ final FileStatus file) throws IOException, InterruptedException {
+final ParquetMetadata metadata;
+final UserGroupInformation processUserUgi =
ImpersonationUtil.getProcessUserUGI();
+try {
+ metadata = processUserUgi.doAs(new
PrivilegedExceptionAction() {
+public ParquetMetadata run() throws Exception {
+ return ParquetFileReader.readFooter(fs.getConf(), file);
+}
+ });
+} catch(Exception e) {
+ logger.error("Exception while reading footer of parquet file [Details -
path: {}, owner: {}] as process user {}",
+file.getPath(), file.getOwner(), processUserUgi.getShortUserName(), e);
+ throw e;
+}
+
MessageType schema = metadata.getFileMetaData().getSchema();
//Map originalTypeMap = Maps.newHashMap();
[08/11] drill git commit: DRILL-6049: Misc. hygiene and code cleanup changes
http://git-wip-us.apache.org/repos/asf/drill/blob/e791ed62/exec/vector/src/main/codegen/templates/FixedValueVectors.java
--
diff --git a/exec/vector/src/main/codegen/templates/FixedValueVectors.java
b/exec/vector/src/main/codegen/templates/FixedValueVectors.java
index 1f6a008..79beb52 100644
--- a/exec/vector/src/main/codegen/templates/FixedValueVectors.java
+++ b/exec/vector/src/main/codegen/templates/FixedValueVectors.java
@@ -106,7 +106,7 @@ public final class ${minor.class}Vector extends
BaseDataValueVector implements F
}
@Override
- public int getValueCapacity(){
+ public int getValueCapacity() {
return data.capacity() / VALUE_WIDTH;
}
@@ -129,7 +129,7 @@ public final class ${minor.class}Vector extends
BaseDataValueVector implements F
@Override
public void allocateNew() {
-if (!allocateNewSafe()){
+if (!allocateNewSafe()) {
throw new OutOfMemoryException("Failure while allocating buffer.");
}
}
@@ -264,12 +264,12 @@ public final class ${minor.class}Vector extends
BaseDataValueVector implements F
}
@Override
- public TransferPair getTransferPair(BufferAllocator allocator){
+ public TransferPair getTransferPair(BufferAllocator allocator) {
return new TransferImpl(getField(), allocator);
}
@Override
- public TransferPair getTransferPair(String ref, BufferAllocator allocator){
+ public TransferPair getTransferPair(String ref, BufferAllocator allocator) {
return new TransferImpl(getField().withPath(ref), allocator);
}
@@ -278,7 +278,7 @@ public final class ${minor.class}Vector extends
BaseDataValueVector implements F
return new TransferImpl((${minor.class}Vector) to);
}
- public void transferTo(${minor.class}Vector target){
+ public void transferTo(${minor.class}Vector target) {
target.clear();
target.data = data.transferOwnership(target.allocator).buffer;
target.data.writerIndex(data.writerIndex());
@@ -298,10 +298,10 @@ public final class ${minor.class}Vector extends
BaseDataValueVector implements F
return valueCount * ${type.width};
}
- private class TransferImpl implements TransferPair{
+ private class TransferImpl implements TransferPair {
private ${minor.class}Vector to;
-public TransferImpl(MaterializedField field, BufferAllocator allocator){
+public TransferImpl(MaterializedField field, BufferAllocator allocator) {
to = new ${minor.class}Vector(field, allocator);
}
@@ -310,12 +310,12 @@ public final class ${minor.class}Vector extends
BaseDataValueVector implements F
}
@Override
-public ${minor.class}Vector getTo(){
+public ${minor.class}Vector getTo() {
return to;
}
@Override
-public void transfer(){
+public void transfer() {
transferTo(to);
}
@@ -330,7 +330,7 @@ public final class ${minor.class}Vector extends
BaseDataValueVector implements F
}
}
- public void copyFrom(int fromIndex, int thisIndex, ${minor.class}Vector
from){
+ public void copyFrom(int fromIndex, int thisIndex, ${minor.class}Vector
from) {
<#if (type.width > 8)>
from.data.getBytes(fromIndex * VALUE_WIDTH, data, thisIndex * VALUE_WIDTH,
VALUE_WIDTH);
<#else> <#-- type.width <= 8 -->
@@ -340,7 +340,7 @@ public final class ${minor.class}Vector extends
BaseDataValueVector implements F
<#-- type.width -->
}
- public void copyFromSafe(int fromIndex, int thisIndex, ${minor.class}Vector
from){
+ public void copyFromSafe(int fromIndex, int thisIndex, ${minor.class}Vector
from) {
while(thisIndex >= getValueCapacity()) {
reAlloc();
}
@@ -376,24 +376,24 @@ public final class ${minor.class}Vector extends
BaseDataValueVector implements F
}
@Override
-public boolean isNull(int index){
+public boolean isNull(int index) {
return false;
}
-
<#if (type.width > 8)>
+
public ${minor.javaType!type.javaType} get(int index) {
return data.slice(index * VALUE_WIDTH, VALUE_WIDTH);
}
-
<#if (minor.class == "Interval")>
-public void get(int index, ${minor.class}Holder holder){
+
+public void get(int index, ${minor.class}Holder holder) {
final int offsetIndex = index * VALUE_WIDTH;
holder.months = data.getInt(offsetIndex);
holder.days = data.getInt(offsetIndex + ${minor.daysOffset});
holder.milliseconds = data.getInt(offsetIndex +
${minor.millisecondsOffset});
}
-public void get(int index, Nullable${minor.class}Holder holder){
+public void get(int index, Nullable${minor.class}Holder holder) {
final int offsetIndex = index * VALUE_WIDTH;
holder.isSet = 1;
holder.months = data.getInt(offsetIndex);
@@ -407,52 +407,25 @@ public final class ${minor.class}Vector extends
BaseDataValueVector implements F
final int months = data.getInt(offsetIndex);
final int days
[01/11] drill git commit: DRILL-5851: Empty table during a join operation with a non empty table produces cast exception.
Repository: drill
Updated Branches:
refs/heads/master a2a56af3e -> e791ed62b
DRILL-5851: Empty table during a join operation with a non empty table produces
cast exception.
close apache/drill#1059
Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/7d1e81a5
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/7d1e81a5
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/7d1e81a5
Branch: refs/heads/master
Commit: 7d1e81a5ac0d8ca0b6519dd2c90bf3804c3221b1
Parents: a2a56af
Author: Hanumath Maduri
Authored: Mon Oct 9 13:08:13 2017 -0700
Committer: Aman Sinha
Committed: Tue Jan 23 14:39:53 2018 -0800
--
.../exec/physical/impl/join/HashJoinBatch.java | 21 ++-
.../impl/join/HashJoinProbeTemplate.java| 6 +-
.../exec/physical/impl/join/MergeJoinBatch.java | 3 +-
.../impl/union/UnionAllRecordBatch.java | 1 +
.../IteratorValidatorBatchIterator.java | 1 +
.../exec/record/AbstractBinaryRecordBatch.java | 11 +++-
.../exec/physical/impl/join/JoinTestBase.java | 63 +++
.../impl/join/TestHashJoinAdvanced.java | 23 ++-
.../impl/join/TestMergeJoinAdvanced.java| 25 ++--
.../physical/impl/join/TestNestedLoopJoin.java | 64 ++--
10 files changed, 185 insertions(+), 33 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/drill/blob/7d1e81a5/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/HashJoinBatch.java
--
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/HashJoinBatch.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/HashJoinBatch.java
index 7e2859e..7b679c0 100644
---
a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/HashJoinBatch.java
+++
b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/HashJoinBatch.java
@@ -181,7 +181,9 @@ public class HashJoinBatch extends
AbstractBinaryRecordBatch {
hyperContainer = new ExpandableHyperContainer(vectors);
hjHelper.addNewBatch(0);
buildBatchIndex++;
- setupHashTable();
+ if (isFurtherProcessingRequired(rightUpstream) &&
this.right.getRecordCount() > 0) {
+setupHashTable();
+ }
hashJoinProbe = setupHashJoinProbe();
// Build the container schema and set the counts
for (final VectorWrapper w : container) {
@@ -212,7 +214,7 @@ public class HashJoinBatch extends
AbstractBinaryRecordBatch {
}
// Store the number of records projected
- if (!hashTable.isEmpty() || joinType != JoinRelType.INNER) {
+ if ((hashTable != null && !hashTable.isEmpty()) || joinType !=
JoinRelType.INNER) {
// Allocate the memory for the vectors in the output container
allocateVectors();
@@ -305,11 +307,15 @@ public class HashJoinBatch extends
AbstractBinaryRecordBatch {
//Setup the underlying hash table
// skip first batch if count is zero, as it may be an empty schema batch
-if (right.getRecordCount() == 0) {
+if (isFurtherProcessingRequired(rightUpstream) && right.getRecordCount()
== 0) {
for (final VectorWrapper w : right) {
w.clear();
}
rightUpstream = next(right);
+ if (isFurtherProcessingRequired(rightUpstream) &&
+ right.getRecordCount() > 0 && hashTable == null) {
+setupHashTable();
+ }
}
boolean moreData = true;
@@ -535,4 +541,13 @@ public class HashJoinBatch extends
AbstractBinaryRecordBatch {
}
super.close();
}
+
+ /**
+ * This method checks to see if join processing should be continued further.
+ * @param upStream up stream operator status.
+ * @@return true if up stream status is OK or OK_NEW_SCHEMA otherwise false.
+ */
+ private boolean isFurtherProcessingRequired(IterOutcome upStream) {
+return upStream == IterOutcome.OK || upStream == IterOutcome.OK_NEW_SCHEMA;
+ }
}
http://git-wip-us.apache.org/repos/asf/drill/blob/7d1e81a5/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/HashJoinProbeTemplate.java
--
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/HashJoinProbeTemplate.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/HashJoinProbeTemplate.java
index 3cdce2f..5c6371a 100644
---
a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/HashJoinProbeTemplate.java
+++
b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/HashJoinProbeTemplate.java
@@ -136,7 +136,9 @@ public abstract class HashJoinProbeTemplate implements
HashJoinProbe {
case OK_NEW_SCHEMA:
[10/11] drill git commit: DRILL-6049: Misc. hygiene and code cleanup changes
http://git-wip-us.apache.org/repos/asf/drill/blob/e791ed62/exec/java-exec/src/main/java/org/apache/drill/exec/ops/OperatorStats.java
--
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/ops/OperatorStats.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/ops/OperatorStats.java
index 1b96f28..a38c3c2 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/ops/OperatorStats.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/ops/OperatorStats.java
@@ -32,8 +32,9 @@ import com.carrotsearch.hppc.cursors.IntDoubleCursor;
import com.carrotsearch.hppc.cursors.IntLongCursor;
import com.carrotsearch.hppc.procedures.IntDoubleProcedure;
import com.carrotsearch.hppc.procedures.IntLongProcedure;
+import com.google.common.annotations.VisibleForTesting;
-public class OperatorStats implements OperatorStatReceiver {
+public class OperatorStats {
static final org.slf4j.Logger logger =
org.slf4j.LoggerFactory.getLogger(OperatorStats.class);
protected final int operatorId;
@@ -89,7 +90,8 @@ public class OperatorStats implements OperatorStatReceiver {
}
}
- private OperatorStats(int operatorId, int operatorType, int inputCount,
BufferAllocator allocator) {
+ @VisibleForTesting
+ public OperatorStats(int operatorId, int operatorType, int inputCount,
BufferAllocator allocator) {
super();
this.allocator = allocator;
this.operatorId = operatorId;
@@ -169,7 +171,6 @@ public class OperatorStats implements OperatorStatReceiver {
inProcessing = false;
}
- @Override
public synchronized void startWait() {
assert !inWait : assertionError("starting waiting");
stopProcessing();
@@ -177,7 +178,6 @@ public class OperatorStats implements OperatorStatReceiver {
waitMark = System.nanoTime();
}
- @Override
public synchronized void stopWait() {
assert inWait : assertionError("stopping waiting");
startProcessing();
@@ -203,7 +203,6 @@ public class OperatorStats implements OperatorStatReceiver {
.toString();
}
-
public OperatorProfile getProfile() {
final OperatorProfile.Builder b = OperatorProfile //
.newBuilder() //
@@ -213,14 +212,11 @@ public class OperatorStats implements
OperatorStatReceiver {
.setProcessNanos(processingNanos)
.setWaitNanos(waitNanos);
-if(allocator != null){
+if (allocator != null) {
b.setPeakLocalMemoryAllocated(allocator.getPeakMemoryAllocation());
}
-
-
addAllMetrics(b);
-
return b.build();
}
@@ -249,7 +245,6 @@ public class OperatorStats implements OperatorStatReceiver {
public void apply(int key, long value) {
builder.addMetric(MetricValue.newBuilder().setMetricId(key).setLongValue(value));
}
-
}
public void addLongMetrics(OperatorProfile.Builder builder) {
@@ -278,22 +273,62 @@ public class OperatorStats implements
OperatorStatReceiver {
}
}
- @Override
+ /**
+ * Set a stat to the specified long value. Creates the stat
+ * if the stat does not yet exist.
+ *
+ * @param metric the metric to update
+ * @param value the value to set
+ */
+
public void addLongStat(MetricDef metric, long value){
longMetrics.putOrAdd(metric.metricId(), value, value);
}
- @Override
+ @VisibleForTesting
+ public long getLongStat(MetricDef metric) {
+return longMetrics.get(metric.metricId());
+ }
+
+ /**
+ * Add a double value to the existing value. Creates the stat
+ * (with an initial value of zero) if the stat does not yet
+ * exist.
+ *
+ * @param metric the metric to update
+ * @param value the value to add to the existing value
+ */
+
public void addDoubleStat(MetricDef metric, double value){
doubleMetrics.putOrAdd(metric.metricId(), value, value);
}
- @Override
+ @VisibleForTesting
+ public double getDoubleStat(MetricDef metric) {
+return doubleMetrics.get(metric.metricId());
+ }
+
+ /**
+ * Add a long value to the existing value. Creates the stat
+ * (with an initial value of zero) if the stat does not yet
+ * exist.
+ *
+ * @param metric the metric to update
+ * @param value the value to add to the existing value
+ */
+
public void setLongStat(MetricDef metric, long value){
longMetrics.put(metric.metricId(), value);
}
- @Override
+ /**
+ * Set a stat to the specified double value. Creates the stat
+ * if the stat does not yet exist.
+ *
+ * @param metric the metric to update
+ * @param value the value to set
+ */
+
public void setDoubleStat(MetricDef metric, double value){
doubleMetrics.put(metric.metricId(), value);
}
@@ -313,5 +348,4 @@ public class OperatorStats implements OperatorStatReceiver {
public long getProcessingNanos() {
return processingNanos;
}
-
}
http://git-wip-us.apache.org/repos/asf/drill/blob/e791ed62/exec/java-exec/src/main/java/org/apache/drill/exec/physical/b
[03/11] drill git commit: DRILL-5868: Support SQL syntax highlighting of queries
DRILL-5868: Support SQL syntax highlighting of queries
Based on the commit for DRILL-5981 (PR #1043), this commit further leverages
the Ace JavaScript library with customizations specific to Drill.
This commit introduces the following to the Query Editor (including the Edit
Query tab within an existing profile to rerunning the query).
1. Syntax highlighting (This is supported for submitted query profiles
2. Autocomplete supported in editors
3. Specifying Drill specific keywords and functions in visible autocomplete
4. Key snippets (template SQLs) allowing for rapid writing of syntax:
i. Query System Tables
ii. CView, CTAS and CTempTAS
iii. Alter Session
iv. Explain and Select * queries
NOTE: The lists for #3 and #4 are not exhaustive. As more features are added to
Drill, these lists can be expanded.
Updates based on review comments
1. Disabled warning message
2. Extended reserved keyword list
3. Fixed bugs
List:
Update 1: Bug fix for when Impersonation is enabled
Update 2: Remove the duplicate editor that might get injected
Update 3: Removed trailing whitespaces in Javascripts
close apache/drill#1084
Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/4c8a55b8
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/4c8a55b8
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/4c8a55b8
Branch: refs/heads/master
Commit: 4c8a55b825b91ca26073f9ba059e18cdaa08670d
Parents: a2058fd
Author: Kunal Khatua
Authored: Mon Jan 22 12:01:32 2018 -0800
Committer: Aman Sinha
Committed: Tue Jan 23 14:42:23 2018 -0800
--
.../src/main/resources/rest/profile/profile.ftl | 83 +++-
.../src/main/resources/rest/query/query.ftl | 43 +++-
.../rest/static/js/ace-code-editor/ace.js | 3 +-
.../js/ace-code-editor/ext-language_tools.js| 4 +
.../static/js/ace-code-editor/mode-snippets.js | 199 +++
.../rest/static/js/ace-code-editor/mode-sql.js | 135 -
.../static/js/ace-code-editor/snippets/sql.js | 48 +
.../js/ace-code-editor/theme-sqlserver.js | 138 +
.../src/main/resources/rest/storage/update.ftl | 6 +-
9 files changed, 649 insertions(+), 10 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/drill/blob/4c8a55b8/exec/java-exec/src/main/resources/rest/profile/profile.ftl
--
diff --git a/exec/java-exec/src/main/resources/rest/profile/profile.ftl
b/exec/java-exec/src/main/resources/rest/profile/profile.ftl
index 1f38d2e..7f4474c 100644
--- a/exec/java-exec/src/main/resources/rest/profile/profile.ftl
+++ b/exec/java-exec/src/main/resources/rest/profile/profile.ftl
@@ -19,6 +19,13 @@
+
+
+
+
+
+
+
var globalconfig = {
@@ -65,7 +72,7 @@ table.sortable thead .sorting_desc { background-image:
url("/static/img/black-de
- ${model.getProfile().query}
+ ${model.getProfile().query}
${model.profile.plan}
@@ -84,9 +91,8 @@ table.sortable thead .sorting_desc { background-image:
url("/static/img/black-de