[drill] 03/03: DRILL-7302: Bump Apache Avro to 1.9.0
This is an automated email from the ASF dual-hosted git repository. amansinha pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/drill.git commit 205e0289940e8b8342f3ae812640da331381ff16 Author: Fokko Driesprong AuthorDate: Thu Jun 20 15:41:57 2019 +0200 DRILL-7302: Bump Apache Avro to 1.9.0 Apache Avro 1.9.0 brings a lot of new features: Deprecate Joda-Time in favor of Java8 JSR310 and setting it as default Remove support for Hadoop 1.x Move from Jackson 1.x to 2.9 Add ZStandard Codec Lots of updates on the dependencies to fix CVE's Remove Jackson classes from public API Apache Avro is built by default with Java 8 Apache Avro is compiled and tested with Java 11 to guarantee compatibility Apache Avro MapReduce is compiled and tested with Hadoop 3 Apache Avro is now leaner, multiple dependencies were removed: guava, paranamer, commons-codec, and commons-logging and many, many more! close apache/drill#1812 --- .../org/apache/drill/exec/resourcemgr/config/selectors/AclSelector.java | 2 +- .../drill/exec/resourcemgr/config/selectors/NotEqualSelector.java | 2 +- .../drill/exec/physical/impl/join/TestLateralJoinCorrectness.java | 2 +- .../physical/impl/join/TestLateralJoinCorrectnessBatchProcessing.java | 2 +- .../java/org/apache/drill/exec/physical/impl/scan/ScanTestUtils.java| 2 +- pom.xml | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/resourcemgr/config/selectors/AclSelector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/resourcemgr/config/selectors/AclSelector.java index 50acb86..9c6d7c5 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/resourcemgr/config/selectors/AclSelector.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/resourcemgr/config/selectors/AclSelector.java @@ -17,7 +17,7 @@ */ package org.apache.drill.exec.resourcemgr.config.selectors; -import avro.shaded.com.google.common.annotations.VisibleForTesting; +import org.apache.drill.shaded.guava.com.google.common.annotations.VisibleForTesting; import com.typesafe.config.Config; import org.apache.drill.exec.ops.QueryContext; import org.apache.drill.exec.resourcemgr.config.exception.RMConfigException; diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/resourcemgr/config/selectors/NotEqualSelector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/resourcemgr/config/selectors/NotEqualSelector.java index 653c3b5..bb6da02 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/resourcemgr/config/selectors/NotEqualSelector.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/resourcemgr/config/selectors/NotEqualSelector.java @@ -17,7 +17,7 @@ */ package org.apache.drill.exec.resourcemgr.config.selectors; -import avro.shaded.com.google.common.annotations.VisibleForTesting; +import org.apache.drill.shaded.guava.com.google.common.annotations.VisibleForTesting; import com.typesafe.config.Config; import org.apache.drill.exec.ops.QueryContext; import org.apache.drill.exec.resourcemgr.config.exception.RMConfigException; diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/join/TestLateralJoinCorrectness.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/join/TestLateralJoinCorrectness.java index 4d4521d..ff1de9d 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/join/TestLateralJoinCorrectness.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/join/TestLateralJoinCorrectness.java @@ -17,7 +17,7 @@ */ package org.apache.drill.exec.physical.impl.join; -import avro.shaded.com.google.common.collect.Lists; +import org.apache.drill.shaded.guava.com.google.common.collect.Lists; import org.apache.calcite.rel.core.JoinRelType; import org.apache.drill.categories.OperatorTest; import org.apache.drill.common.exceptions.DrillRuntimeException; diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/join/TestLateralJoinCorrectnessBatchProcessing.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/join/TestLateralJoinCorrectnessBatchProcessing.java index 9eee038..ae9003c 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/join/TestLateralJoinCorrectnessBatchProcessing.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/join/TestLateralJoinCorrectnessBatchProcessing.java @@ -17,7 +17,7 @@ */ package org.apache.drill.exec.physical.impl.join; -import avro.shaded.com.google.common.collect.Lists; +import org.apache.drill.shaded.guava.com.google.common.collect.Lists; import org.apache.calcite.rel.core.JoinRelType; import org.apache.drill.common.types.TypeProtos; i
[drill] branch master updated (f3f7dbd -> 205e028)
This is an automated email from the ASF dual-hosted git repository. amansinha pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/drill.git. from f3f7dbd DRILL-7292: Remove V1 and V2 text readers new ddc2c02 DRILL-7268: Read Hive array with parquet native reader new d108677 DRILL-7297: Query hangs in planning stage when Error is thrown new 205e028 DRILL-7302: Bump Apache Avro to 1.9.0 The 3 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../drill/exec/store/hive/HiveUtilities.java |3 +- .../exec/hive/complex_types/TestHiveArrays.java| 1439 .../resourcemgr/config/selectors/AclSelector.java |2 +- .../config/selectors/NotEqualSelector.java |2 +- .../store/parquet2/DrillParquetGroupConverter.java | 349 +++-- .../exec/store/parquet2/DrillParquetReader.java| 30 +- .../parquet2/DrillParquetRecordMaterializer.java |2 +- .../apache/drill/exec/work/foreman/Foreman.java|8 +- .../java/org/apache/drill/TestFunctionsQuery.java |8 + .../exec/fn/impl/testing/CustomErrorFunction.java} | 29 +- .../impl/join/TestLateralJoinCorrectness.java |2 +- .../TestLateralJoinCorrectnessBatchProcessing.java |2 +- .../exec/physical/impl/scan/ScanTestUtils.java |2 +- .../store/parquet2/TestDrillParquetReader.java | 855 .../test/resources/parquet2/hive_arrays_p.parquet | Bin 0 -> 14270 bytes pom.xml|2 +- 16 files changed, 1697 insertions(+), 1038 deletions(-) copy exec/java-exec/src/{main/java/org/apache/drill/exec/expr/fn/impl/Not.java => test/java/org/apache/drill/exec/fn/impl/testing/CustomErrorFunction.java} (65%) create mode 100755 exec/java-exec/src/test/resources/parquet2/hive_arrays_p.parquet
[drill] 02/03: DRILL-7297: Query hangs in planning stage when Error is thrown
This is an automated email from the ASF dual-hosted git repository. amansinha pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/drill.git commit d1086772ee4d9f027d3fd6f4673aae43158d2172 Author: Volodymyr Vysotskyi AuthorDate: Thu Jun 20 15:33:00 2019 +0300 DRILL-7297: Query hangs in planning stage when Error is thrown close apache/drill#1811 --- .../apache/drill/exec/work/foreman/Foreman.java| 8 ++--- .../java/org/apache/drill/TestFunctionsQuery.java | 8 + .../exec/fn/impl/testing/CustomErrorFunction.java | 42 ++ 3 files changed, 53 insertions(+), 5 deletions(-) diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/Foreman.java b/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/Foreman.java index ce03303..804254b 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/Foreman.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/Foreman.java @@ -285,9 +285,9 @@ public class Foreman implements Runnable { throw new IllegalStateException(); } injector.injectChecked(queryContext.getExecutionControls(), "run-try-end", ForemanException.class); -} catch (final ForemanException e) { +} catch (ForemanException | UserException e) { queryStateProcessor.moveToState(QueryState.FAILED, e); -} catch (final OutOfMemoryError | OutOfMemoryException e) { +} catch (OutOfMemoryError | OutOfMemoryException e) { if (FailureUtils.isDirectMemoryOOM(e)) { queryStateProcessor.moveToState(QueryState.FAILED, UserException.memoryError(e).build(logger)); } else { @@ -298,9 +298,7 @@ public class Foreman implements Runnable { */ FailureUtils.unrecoverableFailure(e, "Unable to handle out of memory condition in Foreman.", EXIT_CODE_HEAP_OOM); } -} catch (UserException e) { - queryStateProcessor.moveToState(QueryState.FAILED, e); -} catch (AssertionError | Exception ex) { +} catch (Throwable ex) { queryStateProcessor.moveToState(QueryState.FAILED, new ForemanException("Unexpected exception during fragment initialization: " + ex.getMessage(), ex)); } finally { diff --git a/exec/java-exec/src/test/java/org/apache/drill/TestFunctionsQuery.java b/exec/java-exec/src/test/java/org/apache/drill/TestFunctionsQuery.java index 16d9789..ae08d92 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/TestFunctionsQuery.java +++ b/exec/java-exec/src/test/java/org/apache/drill/TestFunctionsQuery.java @@ -18,6 +18,7 @@ package org.apache.drill; import static org.apache.drill.exec.expr.fn.impl.DateUtility.formatTimeStamp; +import static org.hamcrest.CoreMatchers.containsString; import java.math.BigDecimal; import java.time.Instant; @@ -1008,4 +1009,11 @@ public class TestFunctionsQuery extends BaseTestQuery { .go(); } } + + @Test // DRILL-7297 + public void testErrorInUdf() throws Exception { +expectedException.expect(UserRemoteException.class); +expectedException.expectMessage(containsString("Error from UDF")); +test("select error_function()"); + } } diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/testing/CustomErrorFunction.java b/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/testing/CustomErrorFunction.java new file mode 100644 index 000..c372e7b --- /dev/null +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/testing/CustomErrorFunction.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.fn.impl.testing; + +import org.apache.drill.exec.expr.DrillSimpleFunc; +import org.apache.drill.exec.expr.annotations.FunctionTemplate; +import org.apache.drill.exec.expr.annotations.Output; +import org.apache.drill.exec.expr.holders.VarCharHolder; + +@FunctionTemplate( +name="error_function", +scope = FunctionTemplate.FunctionScope.SIMPLE, +nulls = FunctionTemplate.NullHandling.NULL_IF_NULL +) +public class CustomErrorFunction implements DrillSimpleFunc { + + @
[drill] branch master updated (0195d1f -> 05a1a3a)
This is an automated email from the ASF dual-hosted git repository. amansinha pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/drill.git. from 0195d1f DRILL-7237: Fix single_value aggregate function for variable length types add 05a1a3a DRILL-7242: Handle additional boundary cases and compute better estimates when popular values span multiple buckets. No new revisions were added by this update. Summary of changes: .../drill/exec/planner/common/Histogram.java | 2 +- .../planner/common/NumericEquiDepthHistogram.java | 125 ++--- .../exec/planner/cost/DrillRelMdSelectivity.java | 3 +- .../common/TestNumericEquiDepthHistogram.java | 113 +++ .../org/apache/drill/exec/sql/TestAnalyze.java | 12 ++ 5 files changed, 215 insertions(+), 40 deletions(-) create mode 100644 exec/java-exec/src/test/java/org/apache/drill/exec/planner/common/TestNumericEquiDepthHistogram.java
[drill] branch master updated (99707a3 -> ba9837e)
This is an automated email from the ASF dual-hosted git repository. amansinha pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/drill.git. from 99707a3 DRILL-7199: Optimize population of metadata for non-interesting columns closes #1771 add ba9837e DRILL-7187: Improve selectivity estimation of BETWEEN predicates and arbitrary combination of range predicates. No new revisions were added by this update. Summary of changes: .../drill/exec/planner/common/DrillStatsTable.java | 9 +- .../drill/exec/planner/common/Histogram.java | 3 +- .../planner/common/NumericEquiDepthHistogram.java | 300 - .../exec/planner/cost/DrillRelMdSelectivity.java | 84 +- .../org/apache/drill/exec/sql/TestAnalyze.java | 28 ++ .../drill/metastore/ColumnStatisticsKind.java | 17 ++ 6 files changed, 308 insertions(+), 133 deletions(-)
[drill] branch master updated: DRILL-7119: Compute range predicate selectivity using histograms.
This is an automated email from the ASF dual-hosted git repository. amansinha pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/drill.git The following commit(s) were added to refs/heads/master by this push: new 849f896 DRILL-7119: Compute range predicate selectivity using histograms. 849f896 is described below commit 849f896c491118571303a1313f75bece37e80b7e Author: Aman Sinha AuthorDate: Mon Mar 25 06:55:57 2019 -0700 DRILL-7119: Compute range predicate selectivity using histograms. Address code review comments. Add unit test for histogram usage. close apache/drill#1733 --- .../drill/exec/planner/common/DrillStatsTable.java | 2 +- .../drill/exec/planner/common/Histogram.java | 2 +- .../planner/common/NumericEquiDepthHistogram.java | 188 - .../exec/planner/cost/DrillRelMdSelectivity.java | 28 ++- .../test/java/org/apache/drill/PlanTestBase.java | 4 +- .../org/apache/drill/exec/sql/TestAnalyze.java | 15 ++ 6 files changed, 225 insertions(+), 14 deletions(-) diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/common/DrillStatsTable.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/common/DrillStatsTable.java index 65b0b64..d34a5e2 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/common/DrillStatsTable.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/common/DrillStatsTable.java @@ -215,7 +215,7 @@ public class DrillStatsTable { // get the histogram for this column Histogram hist = cs.getHistogram(); -histogram.put(cs.getName(), hist); +histogram.put(cs.getName().toUpperCase(), hist); } } } diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/common/Histogram.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/common/Histogram.java index 1980ee5..c6444a3 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/common/Histogram.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/common/Histogram.java @@ -37,5 +37,5 @@ public interface Histogram { * @param filter * @return estimated selectivity or NULL if it could not be estimated for any reason */ - Double estimatedSelectivity(RexNode filter); + Double estimatedSelectivity(final RexNode filter); } diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/common/NumericEquiDepthHistogram.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/common/NumericEquiDepthHistogram.java index 386141e..9d5bf6f 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/common/NumericEquiDepthHistogram.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/common/NumericEquiDepthHistogram.java @@ -21,8 +21,15 @@ package org.apache.drill.exec.planner.common; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonTypeName; +import java.util.List; + +import org.apache.calcite.rex.RexCall; import org.apache.calcite.rex.RexNode; +import org.apache.calcite.rex.RexLiteral; import com.clearspring.analytics.stream.quantile.TDigest; +import org.apache.calcite.sql.SqlKind; +import org.apache.calcite.sql.SqlOperator; +import org.apache.drill.shaded.guava.com.google.common.base.Preconditions; /** * A column specific equi-depth histogram which is meant for numeric data types @@ -30,6 +37,19 @@ import com.clearspring.analytics.stream.quantile.TDigest; @JsonTypeName("numeric-equi-depth") public class NumericEquiDepthHistogram implements Histogram { + /** + * Use a small non-zero selectivity rather than 0 to account for the fact that + * histogram boundaries are approximate and even if some values lie outside the + * range, we cannot be absolutely sure + */ + static final double SMALL_SELECTIVITY = 0.0001; + + /** + * Use a large selectivity of 1.0 whenever we are reasonably confident that all rows + * qualify. Even if this is off by a small fraction, it is acceptable. + */ + static final double LARGE_SELECTIVITY = 1.0; + // For equi-depth, all buckets will have same (approx) number of rows @JsonProperty("numRowsPerBucket") private long numRowsPerBucket; @@ -69,27 +89,177 @@ public class NumericEquiDepthHistogram implements Histogram { } @Override - public Double estimatedSelectivity(RexNode filter) { + public Double estimatedSelectivity(final RexNode filter) { if (numRowsPerBucket >= 0) { - return 1.0; -} else { - return null; + // at a minimum, the histogram should have a start and end point of 1 bucket, so at least 2 entries + Preconditions.checkArgument(buckets.length >= 2, "Histogram has invalid number of entries"); + final int first = 0; + final int last = buckets.length - 1; + + //
[drill] branch master updated: DRILL-7152: During histogram creation handle the case when all values of a column are NULLs.
This is an automated email from the ASF dual-hosted git repository. amansinha pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/drill.git The following commit(s) were added to refs/heads/master by this push: new 54384a9 DRILL-7152: During histogram creation handle the case when all values of a column are NULLs. 54384a9 is described below commit 54384a992a0742aeab23afa82c7b7f4adcd388d3 Author: Aman Sinha AuthorDate: Tue Apr 2 14:30:26 2019 -0700 DRILL-7152: During histogram creation handle the case when all values of a column are NULLs. close apache/drill#1730 --- .../drill/exec/expr/fn/impl/TDigestFunctions.java | 320 - .../org/apache/drill/exec/sql/TestAnalyze.java | 36 +++ 2 files changed, 228 insertions(+), 128 deletions(-) diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/TDigestFunctions.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/TDigestFunctions.java index 3be70c2..041543b 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/TDigestFunctions.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/TDigestFunctions.java @@ -83,14 +83,18 @@ public class TDigestFunctions { if (work.obj != null) { com.clearspring.analytics.stream.quantile.TDigest tdigest = (com.clearspring.analytics.stream.quantile.TDigest) work.obj; try { - int size = tdigest.smallByteSize(); - java.nio.ByteBuffer byteBuf = java.nio.ByteBuffer.allocate(size); - tdigest.asSmallBytes(byteBuf); - out.buffer = buffer.reallocIfNeeded(size); - out.start = 0; - out.end = size; - out.buffer.setBytes(0, byteBuf.array()); - out.isSet = 1; + if (tdigest.size() > 0) { +int size = tdigest.smallByteSize(); +java.nio.ByteBuffer byteBuf = java.nio.ByteBuffer.allocate(size); +tdigest.asSmallBytes(byteBuf); +out.buffer = buffer.reallocIfNeeded(size); +out.start = 0; +out.end = size; +out.buffer.setBytes(0, byteBuf.array()); +out.isSet = 1; + } else { +out.isSet = 0; + } } catch (Exception e) { throw new org.apache.drill.common.exceptions.DrillRuntimeException("Failed to get TDigest output", e); } @@ -138,14 +142,18 @@ public class TDigestFunctions { if (work.obj != null) { com.clearspring.analytics.stream.quantile.TDigest tdigest = (com.clearspring.analytics.stream.quantile.TDigest) work.obj; try { - int size = tdigest.smallByteSize(); - java.nio.ByteBuffer byteBuf = java.nio.ByteBuffer.allocate(size); - tdigest.asSmallBytes(byteBuf); - out.buffer = buffer.reallocIfNeeded(size); - out.start = 0; - out.end = size; - out.buffer.setBytes(0, byteBuf.array()); - out.isSet = 1; + if (tdigest.size() > 0) { +int size = tdigest.smallByteSize(); +java.nio.ByteBuffer byteBuf = java.nio.ByteBuffer.allocate(size); +tdigest.asSmallBytes(byteBuf); +out.buffer = buffer.reallocIfNeeded(size); +out.start = 0; +out.end = size; +out.buffer.setBytes(0, byteBuf.array()); +out.isSet = 1; + } else { +out.isSet = 0; + } } catch (Exception e) { throw new org.apache.drill.common.exceptions.DrillRuntimeException("Failed to get TDigest output", e); } @@ -189,14 +197,18 @@ public class TDigestFunctions { if (work.obj != null) { com.clearspring.analytics.stream.quantile.TDigest tdigest = (com.clearspring.analytics.stream.quantile.TDigest) work.obj; try { - int size = tdigest.smallByteSize(); - java.nio.ByteBuffer byteBuf = java.nio.ByteBuffer.allocate(size); - tdigest.asSmallBytes(byteBuf); - out.buffer = buffer.reallocIfNeeded(size); - out.start = 0; - out.end = size; - out.buffer.setBytes(0, byteBuf.array()); - out.isSet = 1; + if (tdigest.size() > 0) { +int size = tdigest.smallByteSize(); +java.nio.ByteBuffer byteBuf = java.nio.ByteBuffer.allocate(size); +tdigest.asSmallBytes(byteBuf); +out.buffer = buffer.reallocIfNeeded(size); +out.start = 0; +out.end = size; +out.buffer.setBytes(0, byteBuf.array()); +out.isSet = 1; + } else { +out.isSet = 0; + } } catch (Exception e) { throw new org.apache.drill.common.exceptions.DrillRuntimeException("Failed to get TDigest output", e); } @@ -244,14 +256,18 @@ public class TDigestFunctions { if (work.obj != null) { com.clears
[drill] branch master updated: DRILL-7117: Support creation of equi-depth histogram for selected data types.
This is an automated email from the ASF dual-hosted git repository. amansinha pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/drill.git The following commit(s) were added to refs/heads/master by this push: new fd331da DRILL-7117: Support creation of equi-depth histogram for selected data types. fd331da is described below commit fd331da3872534b284ebd54fec0ff11f7bae47a5 Author: Aman Sinha AuthorDate: Wed Feb 13 15:29:04 2019 -0800 DRILL-7117: Support creation of equi-depth histogram for selected data types. Support int/bigint/float4/float8, time/timestamp/date and boolean. Build the histogram from the t-digest byte array and serialize as JSON string. More changes for serialization/deserialization. Add code-gen stubs (empty) for VarChar/VarBinary types. Address review comments (part 1). Add unit test. Address review comments (part 2) for sampling. close apache/drill#1715 --- .../java/org/apache/drill/exec/ExecConstants.java |8 + .../drill/exec/expr/fn/impl/TDigestFunctions.java | 1082 .../impl/statistics/MergedStatisticFactory.java|1 + .../exec/physical/impl/statistics/Statistic.java |2 + .../impl/statistics/StatisticsMergeBatch.java |5 +- .../impl/statistics/TDigestMergedStatistic.java| 130 +++ .../drill/exec/planner/common/DrillStatsTable.java | 47 +- .../drill/exec/planner/common/Histogram.java | 41 + .../drill/exec/planner/common/HistogramUtils.java | 53 + .../planner/common/NumericEquiDepthHistogram.java | 114 +++ .../drill/exec/planner/physical/AnalyzePrule.java |7 +- .../exec/server/options/SystemOptionManager.java |3 +- .../easy/json/JsonStatisticsRecordWriter.java |6 +- .../java-exec/src/main/resources/drill-module.conf |3 +- .../org/apache/drill/exec/sql/TestAnalyze.java | 61 ++ exec/jdbc-all/pom.xml |1 + 16 files changed, 1557 insertions(+), 7 deletions(-) diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java b/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java index 6d38045..3af8744 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java @@ -979,4 +979,12 @@ public final class ExecConstants { public static final String NDV_BLOOM_FILTER_FPOS_PROB = "exec.statistics.ndv_extrapolation_bf_fpprobability"; public static final LongValidator NDV_BLOOM_FILTER_FPOS_PROB_VALIDATOR = new PositiveLongValidator(NDV_BLOOM_FILTER_FPOS_PROB, 100, new OptionDescription("Controls trade-off between NDV statistic computation memory cost and sampling extrapolation accuracy")); + + /** + * Controls the 'compression' factor for the TDigest algorithm. + */ + public static final String TDIGEST_COMPRESSION = "exec.statistics.tdigest_compression"; + public static final LongValidator TDIGEST_COMPRESSION_VALIDATOR = new PositiveLongValidator(TDIGEST_COMPRESSION, 1, +new OptionDescription("Controls trade-off between t-digest quantile statistic storage cost and accuracy. " + + "Higher values use more groups (clusters) for the t-digest and improve accuracy at the expense of extra storage. ")); } diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/TDigestFunctions.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/TDigestFunctions.java new file mode 100644 index 000..3be70c2 --- /dev/null +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/TDigestFunctions.java @@ -0,0 +1,1082 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.expr.fn.impl; + +import io.netty.buffer.DrillBuf; +import org.apache.drill.exec.expr.DrillAggFunc; +import org.apache.drill.exec.expr.annotations.FunctionTemplate; +import org.apache.drill.exec.expr.annotations.Output; +import org.apache.drill.exec.expr.annotations.Param; +import org.apache.drill.exec.expr.annotat
[drill] branch master updated (7c2661d -> 189ebb4)
This is an automated email from the ASF dual-hosted git repository. amansinha pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/drill.git. from 7c2661d DRILL-7108: Improve selectivity estimates for (NOT)LIKE, NOT_EQUALS, IS NOT NULL predicates new 2d28be8 DRILL-7113: Fix creation of filter conditions for IS NULL and IS NOT NULL for MapR-DB format plugin new 189ebb4 DRILL-6852: Adapt current Parquet Metadata cache implementation to use Drill Metastore API The 3513 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../store/mapr/db/json/JsonConditionBuilder.java | 8 +- .../drill/maprdb/tests/json/TestSimpleJson.java| 6 +- .../store/hive/HiveDrillNativeParquetScan.java | 86 ++- .../hive/HiveParquetTableMetadataProvider.java | 104 +++ ...BooleanPredicate.java => BooleanPredicate.java} | 47 +- ...isonPredicate.java => ComparisonPredicate.java} | 158 +++-- .../drill/exec/expr/ExactStatisticsConstants.java | 14 +- .../exec/expr/ExpressionTreeMaterializer.java | 31 +- .../FilterBuilder.java}| 85 ++- .../apache/drill/exec/expr/FilterPredicate.java| 10 +- .../ParquetIsPredicate.java => IsPredicate.java} | 111 ++-- .../apache/drill/exec/expr/StatisticsProvider.java | 336 ++ .../exec/expr/stat/ParquetPredicatesHelper.java| 60 -- .../drill/exec/expr/stat/RangeExprEvaluator.java | 365 --- ...{ParquetFilterPredicate.java => RowsMatch.java} | 23 +- .../exec/physical/base/AbstractGroupScan.java | 2 +- .../base/AbstractGroupScanWithMetadata.java| 716 + .../drill/exec/physical/base/DbGroupScan.java | 32 +- .../drill/exec/physical/base/FileGroupScan.java| 7 +- .../apache/drill/exec/physical/base/GroupScan.java | 2 +- .../drill/exec/physical/base/IndexGroupScan.java | 18 +- .../physical/base/ParquetMetadataProvider.java | 69 ++ ...Scan.java => ParquetTableMetadataProvider.java} | 26 +- .../exec/physical/base/TableMetadataProvider.java | 88 +++ .../exec/planner/ParquetPartitionDescriptor.java | 12 +- .../logical/DrillPushProjectIntoScanRule.java | 6 +- .../planner/logical/partition/PruneScanRule.java | 4 +- .../exec/record/metadata/SchemaPathUtils.java | 85 +++ .../store/parquet/AbstractParquetGroupScan.java| 683 +++- .../store/parquet/BaseParquetMetadataProvider.java | 347 ++ .../exec/store/parquet/FilterEvaluatorUtils.java | 139 .../drill/exec/store/parquet/ParquetGroupScan.java | 338 ++ .../store/parquet/ParquetGroupScanStatistics.java | 138 ++-- .../exec/store/parquet/ParquetPushDownFilter.java | 20 +- .../store/parquet/ParquetRGFilterEvaluator.java| 141 .../parquet/ParquetTableMetadataProviderImpl.java | 338 ++ .../store/parquet/ParquetTableMetadataUtils.java | 670 +++ .../exec/store/parquet/metadata/Metadata.java | 23 +- .../exec/store/parquet/metadata/Metadata_V2.java | 5 +- .../exec/store/parquet/metadata/Metadata_V3.java | 3 +- .../parquet/stat/ParquetFooterStatCollector.java | 196 -- .../parquet/stat/ParquetMetaStatCollector.java | 301 - .../exec/store/parquet2/DrillParquetReader.java| 10 - .../apache/drill/exec/util/ImpersonationUtil.java | 8 +- .../org/apache/drill/metastore/BaseMetadata.java | 80 +++ .../CollectableColumnStatisticsKind.java} | 25 +- .../CollectableTableStatisticsKind.java} | 25 +- .../apache/drill/metastore/ColumnStatistics.java | 73 +++ .../drill/metastore/ColumnStatisticsImpl.java | 67 ++ .../drill/metastore/ColumnStatisticsKind.java | 120 .../org/apache/drill/metastore/FileMetadata.java | 102 +++ .../apache/drill/metastore/FileTableMetadata.java | 132 .../apache/drill/metastore/LocationProvider.java | 17 +- .../apache/drill/metastore/PartitionMetadata.java | 123 .../apache/drill/metastore/RowGroupMetadata.java | 106 +++ .../StatisticsKind.java} | 38 +- .../TableMetadata.java}| 19 +- .../drill/metastore/TableStatisticsKind.java | 79 +++ .../store/parquet/TestParquetFilterPushDown.java | 132 ++-- .../store/parquet/TestParquetLimitPushDown.java| 8 +- .../apache/drill/common/expression/SchemaPath.java | 3 + .../org/apache/drill/common/graph/GraphValue.java | 6 +- 62 files changed, 4806 insertions(+), 2220 deletions(-) create mode 100644 contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/HiveParquetTableMetadataProvider.java rename exec/java-exec/src/main/java/org/apache/drill/exec/expr
[drill] branch master updated (0a61b8f -> 3bec197)
This is an automated email from the ASF dual-hosted git repository. amansinha pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/drill.git. from 0a61b8f DRILL-6964: Implement CREATE / DROP SCHEMA commands new b5fe065 DRILL-7019: Add check for redundant imports new 35b42eb DRILL-7016: Wrong query result with RuntimeFilter enabled when order of join and filter condition is swapped new 3bec197 DRILL-6862: Update Calcite to 1.18.0 The 3458 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../initilializers/DefaultReadersInitializer.java | 1 - .../drill/yarn/client/TestCommandLineOptions.java | 2 - drill-yarn/src/test/resources/doy-test-logback.xml | 2 +- .../src/test/resources/second-test-config.conf | 4 +- drill-yarn/src/test/resources/test-doy-config.conf | 4 +- .../src/test/resources/test-doy-distrib.conf | 2 +- .../src/test/resources/third-test-config.conf | 4 +- exec/java-exec/src/main/codegen/data/Parser.tdd| 763 - .../apache/drill/exec/planner/DrillRelBuilder.java | 10 + .../apache/drill/exec/planner/RuleInstance.java| 3 +- .../planner/index/rules/DbScanSortRemovalRule.java | 1 - .../exec/planner/logical/DrillImplementor.java | 1 - .../planner/logical/DrillMergeProjectRule.java | 2 - .../drill/exec/planner/physical/TopNPrel.java | 2 - .../physical/visitor/RuntimeFilterVisitor.java | 8 +- .../drill/exec/planner/sql/SqlConverter.java | 15 +- .../drill/exec/planner/sql/TypeInferenceUtils.java | 6 +- .../test/java/org/apache/drill/TestCTASJson.java | 16 +- .../physical/impl/TestNestedDateTimeTimestamp.java | 12 +- .../impl/join/TestHashJoinJPPDCorrectness.java | 178 + .../impl/scan/project/TestNullColumnLoader.java| 3 - .../impl/scan/project/TestRowBatchMerger.java | 5 - .../impl/scan/project/TestScanLevelProjection.java | 2 - .../scan/project/TestSchemaLevelProjection.java| 29 +- .../impl/scan/project/TestSchemaSmoothing.java | 7 - .../store/parquet2/TestDrillParquetReader.java | 13 + .../java/org/apache/drill/test/TestBuilder.java| 11 +- .../java-exec/src/test/resources/drill-module.conf | 2 +- exec/java-exec/src/test/resources/zkacltest.conf | 22 +- exec/jdbc-all/pom.xml | 2 +- .../drill/jdbc/impl/AvaticaDrillSqlAccessor.java | 6 + .../apache/drill/jdbc/impl/DrillResultSetImpl.java | 12 +- .../apache/drill/jdbc/impl/WrappedAccessor.java| 447 .../io/netty/buffer/UnsafeDirectLittleEndian.java | 2 - .../exec/vector/complex/RepeatedValueVector.java | 3 +- pom.xml| 90 ++- src/main/resources/checkstyle-config.xml | 1 + src/main/resources/checkstyle-suppressions.xml | 2 +- 38 files changed, 1087 insertions(+), 608 deletions(-) create mode 100644 exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/join/TestHashJoinJPPDCorrectness.java delete mode 100644 exec/jdbc/src/main/java/org/apache/drill/jdbc/impl/WrappedAccessor.java
[drill] branch master updated (8fb85cd -> 0a61b8f)
This is an automated email from the ASF dual-hosted git repository. amansinha pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/drill.git. from 8fb85cd DRILL-7006: Add type conversion to row writers new 982e980 DRILL-6997: Semijoin is changing the join ordering for some tpcds queries. new 0242241 DRILL-7002: whatever exec.hashjoin.num_partitions is set, output right results new 9c82e06 DRILL-7008: Drillbits: clear stale shutdown hooks new b1594cb DRILL-7018: Fixed Parquet buffer overflow when reading timestamp column new 0a61b8f DRILL-6964: Implement CREATE / DROP SCHEMA commands The 3455 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: exec/java-exec/pom.xml | 21 + .../record/metadata/schema/parser/SchemaLexer.g4 | 90 .../record/metadata/schema/parser/SchemaParser.g4 | 72 +++ exec/java-exec/src/main/codegen/data/Parser.tdd| 9 +- .../src/main/codegen/includes/parserImpls.ftl | 259 +-- .../exec/physical/impl/join/HashJoinBatch.java | 18 +- .../apache/drill/exec/planner/PlannerPhase.java| 13 +- .../apache/drill/exec/planner/RuleInstance.java| 1 - .../exec/planner/logical/DrillSemiJoinRel.java | 56 ++- .../exec/planner/logical/DrillSemiJoinRule.java| 183 .../drill/exec/planner/physical/JoinPrel.java | 58 +-- .../drill/exec/planner/sql/DrillSqlWorker.java | 2 +- .../drill/exec/planner/sql/SchemaUtilites.java | 19 +- .../planner/sql/handlers/DefaultSqlHandler.java| 10 +- .../planner/sql/handlers/DropTableHandler.java | 24 +- .../exec/planner/sql/handlers/SchemaHandler.java | 214 + .../exec/planner/sql/handlers/ViewHandler.java | 4 +- .../sql/parser/CompoundIdentifierConverter.java| 2 + .../sql/parser/SqlCreateType.java} | 26 +- .../exec/planner/sql/parser/SqlCreateView.java | 16 +- .../drill/exec/planner/sql/parser/SqlSchema.java | 280 .../record/metadata/AbstractColumnMetadata.java| 30 ++ .../exec/record/metadata/MapColumnMetadata.java| 14 + .../record/metadata/PrimitiveColumnMetadata.java | 46 ++ .../metadata/RepeatedListColumnMetadata.java | 6 + .../drill/exec/record/metadata/TupleSchema.java| 7 + .../metadata/schema/FsMetastoreSchemaProvider.java | 51 +++ .../metadata/schema/InlineSchemaProvider.java} | 46 +- .../record/metadata/schema/PathSchemaProvider.java | 132 ++ .../record/metadata/schema/SchemaContainer.java| 145 ++ .../record/metadata/schema/SchemaProvider.java | 66 +++ .../metadata/schema/parser/SchemaExprParser.java | 96 .../schema/parser/SchemaParsingException.java} | 9 +- .../metadata/schema/parser/SchemaVisitor.java | 269 +++ .../schema/parser/UpperCaseCharStream.java | 84 .../org/apache/drill/exec/server/Drillbit.java | 15 +- .../NullableFixedByteAlignedReaders.java | 5 +- .../java/org/apache/drill/TestSchemaCommands.java | 501 + .../exec/physical/impl/join/TestSemiJoin.java | 25 +- .../record/metadata/schema/TestSchemaProvider.java | 232 ++ .../schema/parser/TestParserErrorHandling.java | 157 +++ .../metadata/schema/parser/TestSchemaParser.java | 279 .../java/org/apache/drill/exec/sql/TestCTTAS.java | 10 +- .../org/apache/drill/exec/sql/TestViewSupport.java | 2 +- .../drill/exec/record/metadata/ColumnMetadata.java | 17 + .../drill/exec/record/metadata/TupleMetadata.java | 8 + 46 files changed, 3426 insertions(+), 203 deletions(-) create mode 100644 exec/java-exec/src/main/antlr4/org/apache/drill/exec/record/metadata/schema/parser/SchemaLexer.g4 create mode 100644 exec/java-exec/src/main/antlr4/org/apache/drill/exec/record/metadata/schema/parser/SchemaParser.g4 create mode 100644 exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillSemiJoinRule.java create mode 100644 exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/SchemaHandler.java copy exec/java-exec/src/main/java/org/apache/drill/exec/{physical/impl/common/SpilledPartitionMetadata.java => planner/sql/parser/SqlCreateType.java} (59%) create mode 100644 exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/SqlSchema.java create mode 100644 exec/java-exec/src/main/java/org/apache/drill/exec/record/metadata/schema/FsMetastoreSchemaProvider.java copy exec/java-exec/src/main/java/org/apache/drill/exec/{physical/rowSet/model/ReaderIndex.java => record/metadata/schema/InlineSchemaProvider.java} (50%) create mode 100644 exec/java-exec/src/main/java/org/apache/drill/e
[drill] 01/03: DRILL-6883: Force reload of options after resetting parameter
This is an automated email from the ASF dual-hosted git repository. amansinha pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/drill.git commit f6d54b480a123724f4cca7d146b0a5c3fb077652 Author: Kunal Khatua AuthorDate: Wed Dec 5 23:34:26 2018 -0800 DRILL-6883: Force reload of options after resetting parameter If an update is initiated, the webpage loaded changes. Attempting a reset tries to reload this page, but it does not show the updated values from the back end. This patch forces the reload of updated values by redirecting to /options page Use AJAX call for Update and Reset Eliminates the need for window.location= is Number check Added comments to explain extraction close apache/drill#1563 --- exec/java-exec/src/main/resources/rest/options.ftl | 38 +- 1 file changed, 30 insertions(+), 8 deletions(-) diff --git a/exec/java-exec/src/main/resources/rest/options.ftl b/exec/java-exec/src/main/resources/rest/options.ftl index 7ac8a2e..a8e03fa 100644 --- a/exec/java-exec/src/main/resources/rest/options.ftl +++ b/exec/java-exec/src/main/resources/rest/options.ftl @@ -22,9 +22,32 @@ -function resetToDefault(optionName, optionValue, optionKind) { -$.post("/option/"+optionName, {kind: optionKind, name: optionName, value: optionValue}, function (status) { location.reload(true); } ); +//Alter System Values +function alterSysOption(optionName, optionValue, optionKind) { +$.post("/option/"+optionName, {kind: optionKind, name: optionName, value: optionValue}, function () { +location.reload(true); +}); +} + +//Read Values and apply +function alterSysOptionUsingId(optionRawName) { +//Escaping '.' for id search +let optionName = optionRawName.replace(/\./gi, "\\."); +//Extracting datatype from the form +let optionKind = $("#"+optionName+" input[name='kind']").attr("value"); +//Extracting value from the form's INPUT element +let optionValue = $("#"+optionName+" input[name='value']").val(); +if (optionKind == "BOOLEAN") { +//Extracting boolean value from the form's SELECT element (since this is a dropdown input) +optionValue = $("#"+optionName+" select[name='value']").val(); +} else if (optionKind != "STRING") { //i.e. it is a number (FLOAT/DOUBLE/LONG) +if (isNaN(optionValue)) { +alert(optionValue+" is not a valid number for option: "+optionName); +return; +} } +alterSysOption(optionRawName, optionValue, optionKind); +} @@ -75,7 +98,7 @@ table.sortable thead .sorting_desc { background-image: url("/static/img/black-de ${option.getName()} - + @@ -89,10 +112,9 @@ table.sortable thead .sorting_desc { background-image: url("/static/img/black-de - Update - disabled="true" style="pointer-events:none" <#else> - title="Reset to ${option.getDefaultValue()}">Reset + Update + disabled="true" style="pointer-events:none" <#else> + title="Reset to ${option.getDefaultValue()}">Default @@ -127,7 +149,7 @@ table.sortable thead .sorting_desc { background-image: url("/static/img/black-de //Inject Descriptions for table let size = $('#optionsTbl tbody tr').length; for (i = 1; i <= size; i++) { - let currRow = $("#row-"+i); +let currRow = $("#row-"+i); let optionName = currRow.find("#optionName").text(); let setOptDescrip = currRow.find("#description").text(getDescription(optionName)); }
[drill] 02/03: DRILL-6874: Close input stream after AsyncPageReaderTask is completed
This is an automated email from the ASF dual-hosted git repository. amansinha pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/drill.git commit 0543dbb356c669132c8ac75a99b318661bdcf664 Author: Bohdan Kazydub AuthorDate: Thu Dec 6 15:03:25 2018 +0200 DRILL-6874: Close input stream after AsyncPageReaderTask is completed close apache/drill#1565 --- .../drill/exec/store/parquet/columnreaders/AsyncPageReader.java| 7 +++ 1 file changed, 7 insertions(+) diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/AsyncPageReader.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/AsyncPageReader.java index e429fb6..8b5c926 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/AsyncPageReader.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/AsyncPageReader.java @@ -417,6 +417,13 @@ class AsyncPageReader extends PageReader { if (totalValuesRead >= totalValuesCount) { try { queue.put(ReadStatus.EMPTY); + // Some InputStreams (like S3ObjectInputStream) should be closed + // as soon as possible to make the connection reusable. + try { +parent.inputStream.close(); + } catch (IOException e) { +logger.trace(String.format("[%s]: Failure while closing InputStream", name), e); + } } catch (InterruptedException e) { Thread.currentThread().interrupt(); // Do nothing.
[drill] 03/03: DRILL-6887: Fix FunctionInitializerTest.init() failure
This is an automated email from the ASF dual-hosted git repository. amansinha pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/drill.git commit e4782133563e365c858bcfb2b23845e4b0299dff Author: Vitalii Diravka AuthorDate: Fri Dec 7 15:55:48 2018 +0200 DRILL-6887: Fix FunctionInitializerTest.init() failure close apache/drill#1566 --- exec/java-exec/src/test/resources/drill-udf/pom.xml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/exec/java-exec/src/test/resources/drill-udf/pom.xml b/exec/java-exec/src/test/resources/drill-udf/pom.xml index 0e25cba..0a68198 100644 --- a/exec/java-exec/src/test/resources/drill-udf/pom.xml +++ b/exec/java-exec/src/test/resources/drill-udf/pom.xml @@ -28,6 +28,8 @@ 1.0 +1.8 +1.8 ${project.name} ${project.basedir}/target 1.13.0 @@ -58,6 +60,7 @@ org.apache.maven.plugins maven-compiler-plugin +3.8.0 ${include.files}
[drill] branch master updated (637354e -> e478213)
This is an automated email from the ASF dual-hosted git repository. amansinha pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/drill.git. from 637354e DRILL-6751: Upgrade Apache parent POM to version 21 new f6d54b4 DRILL-6883: Force reload of options after resetting parameter new 0543dbb DRILL-6874: Close input stream after AsyncPageReaderTask is completed new e478213 DRILL-6887: Fix FunctionInitializerTest.init() failure The 3 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../parquet/columnreaders/AsyncPageReader.java | 7 exec/java-exec/src/main/resources/rest/options.ftl | 38 +- .../java-exec/src/test/resources/drill-udf/pom.xml | 3 ++ 3 files changed, 40 insertions(+), 8 deletions(-)
[drill] 03/04: DRILL-4546: Only generate one zip archive when using apache-release profile
This is an automated email from the ASF dual-hosted git repository. amansinha pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/drill.git commit 7d3763728fd82b3a033936b14ff35545e93d1c5c Author: Laurent Goujon AuthorDate: Mon Mar 28 13:31:53 2016 -0700 DRILL-4546: Only generate one zip archive when using apache-release profile Drill root pom doesn't override completely Apache parent pom configuration regarding assemblies, which caused a zip archive of the project to be generated twice, and deployed to a remote server twice too. The fix uses the plugin properties to override the configuration. Also remove Drill source assembly descriptor as the Apache parent project provides the same one. --- pom.xml | 18 ++--- src/main/resources/assemblies/source-assembly.xml | 99 --- 2 files changed, 5 insertions(+), 112 deletions(-) diff --git a/pom.xml b/pom.xml index c9d2843..790dd11 100644 --- a/pom.xml +++ b/pom.xml @@ -38,6 +38,10 @@ http://drill.apache.org/ +1.8 +1.8 + source-release-zip-tar + ${project.basedir}/target/generated-sources ${project.basedir}/src/main/protobuf/ 4.12 @@ -471,8 +475,6 @@ org.apache.maven.plugins maven-compiler-plugin - 1.8 - 1.8 2048m false true @@ -1682,25 +1684,15 @@ false - + maven-assembly-plugin source-release-assembly -package - - single - - true false - - src/main/resources/assemblies/source-assembly.xml - apache-drill-${project.version}-src - gnu diff --git a/src/main/resources/assemblies/source-assembly.xml b/src/main/resources/assemblies/source-assembly.xml deleted file mode 100644 index 4ed43f1..000 --- a/src/main/resources/assemblies/source-assembly.xml +++ /dev/null @@ -1,99 +0,0 @@ - - - - source-release - -zip -tar.gz - - - - - . - / - true - - - %regex[(?!((?!${project.build.directory}/)[^/]+/)*src/).*${project.build.directory}.*] - - - - - - %regex[(?!((?!${project.build.directory}/)[^/]+/)*src/)(.*/)?maven-eclipse\.xml] - - %regex[(?!((?!${project.build.directory}/)[^/]+/)*src/)(.*/)?\.project] - - %regex[(?!((?!${project.build.directory}/)[^/]+/)*src/)(.*/)?\.classpath] - - %regex[(?!((?!${project.build.directory}/)[^/]+/)*src/)(.*/)?[^/]*\.iws] - - %regex[(?!((?!${project.build.directory}/)[^/]+/)*src/)(.*/)?\.idea(/.*)?] - - %regex[(?!((?!${project.build.directory}/)[^/]+/)*src/)(.*/)?out(/.*)?] - - %regex[(?!((?!${project.build.directory}/)[^/]+/)*src/)(.*/)?[^/]*\.ipr] - - %regex[(?!((?!${project.build.directory}/)[^/]+/)*src/)(.*/)?[^/]*\.iml] - - %regex[(?!((?!${project.build.directory}/)[^/]+/)*src/)(.*/)?\.settings(/.*)?] - - %regex[(?!((?!${project.build.directory}/)[^/]+/)*src/)(.*/)?\.externalToolBuilders(/.*)?] - - %regex[(?!((?!${project.build.directory}/)[^/]+/)*src/)(.*/)?\.deployables(/.*)?] - - %regex[(?!((?!${project.build.directory}/)[^/]+/)*src/)(.*/)?\.wtpmodules(/.*)?] - - - - - %regex[(?!((?!${project.build.directory}/)[^/]+/)*src/)(.*/)?\.gitignore(/.*)?] - - -**/.buildpath -**/sandbox/** - - - %regex[(?!((?!${project.build.directory}/)[^/]+/)*src/)(.*/)?cobertura\.ser] - - - - %regex[(?!((?!${project.build.directory}/)[^/]+/)*src/)(.*/)?pom\.xml\.releaseBackup] - - %regex[(?!((?!${project.build.directory}/)[^/]+/)*src/)(.*/)?release\.properties] - - - - - - ${project.build.directory}/maven-shared-archive-resources/META-INF - - / - - -
[drill] 01/04: DRILL-6882: Handle the cases where RowKeyJoin's left pipeline being called multiple times.
This is an automated email from the ASF dual-hosted git repository. amansinha pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/drill.git commit f3eef38fd633be28f961139b371f524f5172914f Author: Hanumath Rao Maduri AuthorDate: Sat Sep 30 13:26:40 2017 -0700 DRILL-6882: Handle the cases where RowKeyJoin's left pipeline being called multiple times. close apache/drill#1562 --- .../exec/physical/config/IteratorValidator.java| 14 +++-- .../exec/physical/impl/join/RowKeyJoinBatch.java | 5 .../validate/IteratorValidatorBatchIterator.java | 20 + .../impl/validate/IteratorValidatorCreator.java| 2 +- .../impl/validate/IteratorValidatorInjector.java | 33 +++--- 5 files changed, 62 insertions(+), 12 deletions(-) diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/IteratorValidator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/IteratorValidator.java index 9fbef97..4f73b00 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/IteratorValidator.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/IteratorValidator.java @@ -23,10 +23,20 @@ import org.apache.drill.exec.physical.base.PhysicalVisitor; public class IteratorValidator extends AbstractSingle{ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(IteratorValidator.class); + /* isRepeatable flag will be set to true if this validator is created by a Repeatable pipeline. + * In a repeatable pipeline some state transitions are valid i.e downstream operator + * can call the upstream operator even after receiving NONE. + */ + public final boolean isRepeatable; - public IteratorValidator(PhysicalOperator child) { + public IteratorValidator(PhysicalOperator child, boolean repeatable) { super(child); setCost(child.getCost()); +this.isRepeatable = repeatable; + } + + public IteratorValidator(PhysicalOperator child) { +this(child, false); } @Override @@ -36,7 +46,7 @@ public class IteratorValidator extends AbstractSingle{ @Override protected PhysicalOperator getNewWithChild(PhysicalOperator child) { -return new IteratorValidator(child); +return new IteratorValidator(child, isRepeatable); } @Override diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/RowKeyJoinBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/RowKeyJoinBatch.java index 941f321..2910da5 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/RowKeyJoinBatch.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/RowKeyJoinBatch.java @@ -132,6 +132,11 @@ public class RowKeyJoinBatch extends AbstractRecordBatch implemen return IterOutcome.OK; } + if (rightUpstream == IterOutcome.NONE) { +rkJoinState = RowKeyJoinState.DONE; +state = BatchState.DONE; +return rightUpstream; + } rightUpstream = next(right); logger.debug("right input IterOutcome: {}", rightUpstream); diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/validate/IteratorValidatorBatchIterator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/validate/IteratorValidatorBatchIterator.java index 1ea3895..5c70f5d 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/validate/IteratorValidatorBatchIterator.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/validate/IteratorValidatorBatchIterator.java @@ -47,6 +47,9 @@ public class IteratorValidatorBatchIterator implements CloseableRecordBatch { /** For logging/debuggability only. */ private static volatile int instanceCount; + /** @see org.apache.drill.exec.physical.config.IteratorValidator */ + private final boolean isRepeatable; + /** For logging/debuggability only. */ private final int instNum; { @@ -102,12 +105,17 @@ public class IteratorValidatorBatchIterator implements CloseableRecordBatch { */ private boolean validateBatches; - public IteratorValidatorBatchIterator(RecordBatch incoming) { + public IteratorValidatorBatchIterator(RecordBatch incoming, boolean isRepeatable) { this.incoming = incoming; batchTypeName = incoming.getClass().getSimpleName(); +this.isRepeatable = isRepeatable; // (Log construction and close() at same level to bracket instance's activity.) -logger.trace( "[#{}; on {}]: Being constructed.", instNum, batchTypeName); +logger.trace( "[#{}; on {}; repeatable: {}]: Being constructed.", instNum, batchTypeName, isRepeatable); + } + + public IteratorValidatorBatchIterator(RecordBatch incoming) { +this(incoming, false); } @@ -217,7 +225,7 @@ public class I
[drill] 02/04: DRILL-6876: Enable CircleCI builds for JDK 9-11
This is an automated email from the ASF dual-hosted git repository. amansinha pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/drill.git commit abc51d80095cf5df2de64a3acbf75bdd88bd1db5 Author: Volodymyr Vysotskyi AuthorDate: Fri Nov 16 15:43:27 2018 +0200 DRILL-6876: Enable CircleCI builds for JDK 9-11 close apache/drill#1558 --- .circleci/config.yml | 135 ++- 1 file changed, 133 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 1b179df..e2b1141 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -16,7 +16,7 @@ version: 2 general: jobs: - build: + build_jdk8: machine: enabled: true image: circleci/classic:latest @@ -33,7 +33,7 @@ jobs: # TODO: Could be removed, once Machine Executor image is updated https://github.com/circleci/image-builder/issues/140 # and the possibility of specifying Maven version is added https://github.com/circleci/image-builder/issues/143 command: - curl -fsSL https://git.io/vpDIf | bash -s -- 3.5.4 + curl -fsSL https://git.io/vpDIf | bash -s -- 3.6.0 - run: name: Update packages list command: @@ -49,3 +49,134 @@ jobs: # TODO: 4. Resolving memory issues without "SlowTest" and "UnlikelyTest" excludedGroups in the build command: mvn install -Drat.skip=false -Dlicense.skip=false -DmemoryMb=2560 -DdirectMemoryMb=4608 -DexcludedGroups="org.apache.drill.categories.SlowTest,org.apache.drill.categories.UnlikelyTest" + + build_jdk9: +machine: + enabled: true + image: circleci/classic:latest +parallelism: 1 + +working_directory: ~/drill + +steps: +- checkout + +- run: +name: Update packages list +command: + sudo apt-get update + +- run: +name: Install java 9 +command: + sudo apt-get -y install openjdk-9-jdk + +- run: +name: Set default java 9 +command: + sudo update-java-alternatives --set java-1.9.0-openjdk-amd64 + +- run: +name: Update maven version +command: + curl -fsSL https://git.io/vpDIf | bash -s -- 3.6.0 + +- run: +name: Install libaio1.so library for MySQL integration tests +command: + sudo apt-get install libaio1 libaio-dev +- run: +name: Drill project build +command: + mvn install -Drat.skip=false -Dlicense.skip=false -DmemoryMb=2560 -DdirectMemoryMb=4608 -DexcludedGroups="org.apache.drill.categories.SlowTest,org.apache.drill.categories.UnlikelyTest" + + build_jdk10: +machine: + enabled: true + image: circleci/classic:latest +parallelism: 1 + +working_directory: ~/drill + +steps: +- checkout + +- run: +name: Update packages list +command: + sudo apt-get update + +- run: +name: Install java 10 +command: + sudo apt-get -y install openjdk-10-jdk + +- run: +name: Set default java 10 +command: + sudo update-java-alternatives --set java-1.10.0-openjdk-amd64 + +- run: +name: Update maven version +command: + curl -fsSL https://git.io/vpDIf | bash -s -- 3.6.0 + +- run: +name: Install libaio1.so library for MySQL integration tests +command: + sudo apt-get install libaio1 libaio-dev +- run: +name: Drill project build +command: + mvn install -Drat.skip=false -Dlicense.skip=false -DmemoryMb=2560 -DdirectMemoryMb=4608 -DexcludedGroups="org.apache.drill.categories.SlowTest,org.apache.drill.categories.UnlikelyTest" + + build_jdk11: +machine: + enabled: true + image: circleci/classic:latest +parallelism: 1 + +working_directory: ~/drill + +steps: +- checkout + +- run: +name: Update packages list +command: + sudo apt-get update + +- run: +name: Install java 11 +command: + sudo apt-get -y install openjdk-11-jdk + +- run: +name: Set default java 11 +command: + sudo update-java-alternatives --set java-1.11.0-openjdk-amd64 + +- run: +name: Update maven version +command: + curl -fsSL https://git.io/vpDIf | bash -s -- 3.6.0 + +- run: +name: Install libaio1.so library for MySQL integration tests +command: + sudo apt-get install libaio1 libaio-dev +- run: +name: Drill project build +# Set forkCount to 1 since tests use more memory and memory limitations for CircleCI is reached +# for default value of forkCount. +command: + mvn install -Drat.skip=false -Dlicense.skip=false -DmemoryMb=2560 -DdirectMemoryMb=4608 -DforkCount=1 -Dex
[drill] 04/04: DRILL-6751: Upgrade Apache parent POM to version 21
This is an automated email from the ASF dual-hosted git repository. amansinha pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/drill.git commit 637354e91b4b597a0f1930a385b7f0bff0093a2a Author: Vitalii Diravka AuthorDate: Thu Nov 15 17:40:21 2018 -0800 DRILL-6751: Upgrade Apache parent POM to version 21 - Update apache.pom file version to 21 (with updating some maven plugins versions) - Include Drill's sources jars on assembly stage in (not ) for properincluding jars with last apache-21.pom - Separate "distro-assembly" to the two execution stages to avoid: [WARNING] Assembly file: /distribution/target/apache-drill-1.15.0-SNAPSHOT is not a regular file (it may be a directory). It cannot be attached to the project build for installation or deployment. - Remove unsused / in assebly descriptor to avoid: [WARNING] The following patterns were never triggered in this artifact inclusion filter - Update "maven-assembly-plugin" version - Update "slf4j" version - Update "mockito-core" version - Update "bcpkix-jdk15on" (Bouncy Castle Cryptography APIs) version close apache/drill#1561 --- distribution/pom.xml | 49 +++-- distribution/src/assemble/bin-assembly-dir.xml | 33 +++ distribution/src/assemble/bin-assembly.xml | 33 +++ .../src/assemble/{bin.xml => component.xml}| 227 ++--- drill-shaded/pom.xml | 3 +- .../java-exec/src/test/resources/drill-udf/pom.xml | 3 - pom.xml| 23 +-- 7 files changed, 218 insertions(+), 153 deletions(-) diff --git a/distribution/pom.xml b/distribution/pom.xml index 4952ca5..4fde4cd 100644 --- a/distribution/pom.xml +++ b/distribution/pom.xml @@ -156,26 +156,40 @@ - + maven-assembly-plugin - -distro-assembly -package - -single - - - -src/assemble/bin.xml - -false -apache-drill-${project.version} -gnu - - + +distro-assembly +package + + single + + + +src/assemble/bin-assembly.xml + + false + apache-drill-${project.version} + + + +distro-assembly-not-attached +package + + single + + + false + +src/assemble/bin-assembly-dir.xml + + false + apache-drill-${project.version} + + - + org.apache.rat apache-rat-plugin @@ -192,7 +206,6 @@ - diff --git a/distribution/src/assemble/bin-assembly-dir.xml b/distribution/src/assemble/bin-assembly-dir.xml new file mode 100644 index 000..f134962 --- /dev/null +++ b/distribution/src/assemble/bin-assembly-dir.xml @@ -0,0 +1,33 @@ + + +http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2"; + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2 http://maven.apache.org/xsd/assembly-1.1.2.xsd";> + binary-release-directory + +dir + + true + +component.xml + + diff --git a/distribution/src/assemble/bin-assembly.xml b/distribution/src/assemble/bin-assembly.xml new file mode 100644 index 000..5cd3636 --- /dev/null +++ b/distribution/src/assemble/bin-assembly.xml @@ -0,0 +1,33 @@ + + +http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2"; + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2 http://maven.apache.org/xsd/assembly-1.1.2.xsd";> + binary-release + +tar.gz + + true + +component.xml + + diff --git a/distribution/src/assemble/bin.xml b/distribution/src/assemble/component.xml similarity index 72% rename from distribution/src/assemble/bin.xml rename to distribution/src/assemble/component.xml index 85a4d5b..2b60299 100644 --- a/distribution/src/assemble/bin.xml +++ b/distribution/src/assemble/component.xml @@ -18,27 +18,115 @@ limitations under the License. --> -http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2"; - xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; - xsi:schemaLocation="http://maven.apache.org/p
[drill] branch master updated (2dbd609 -> 637354e)
This is an automated email from the ASF dual-hosted git repository. amansinha pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/drill.git. from 2dbd609 Add Vitalii's GPG key new f3eef38 DRILL-6882: Handle the cases where RowKeyJoin's left pipeline being called multiple times. new abc51d8 DRILL-6876: Enable CircleCI builds for JDK 9-11 new 7d37637 DRILL-4546: Only generate one zip archive when using apache-release profile new 637354e DRILL-6751: Upgrade Apache parent POM to version 21 The 4 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .circleci/config.yml | 135 +++- distribution/pom.xml | 49 +++-- .../src/assemble/bin-assembly-dir.xml | 23 ++- .../src/assemble/bin-assembly.xml | 23 ++- .../src/assemble/{bin.xml => component.xml}| 227 ++--- drill-shaded/pom.xml | 3 +- .../exec/physical/config/IteratorValidator.java| 14 +- .../exec/physical/impl/join/RowKeyJoinBatch.java | 5 + .../validate/IteratorValidatorBatchIterator.java | 20 +- .../impl/validate/IteratorValidatorCreator.java| 2 +- .../impl/validate/IteratorValidatorInjector.java | 33 ++- .../java-exec/src/test/resources/drill-udf/pom.xml | 3 - pom.xml| 41 ++-- src/main/resources/assemblies/source-assembly.xml | 99 - 14 files changed, 380 insertions(+), 297 deletions(-) copy src/main/resources/checkstyle-suppressions.xml => distribution/src/assemble/bin-assembly-dir.xml (58%) copy src/main/resources/checkstyle-suppressions.xml => distribution/src/assemble/bin-assembly.xml (59%) rename distribution/src/assemble/{bin.xml => component.xml} (72%) delete mode 100644 src/main/resources/assemblies/source-assembly.xml
[drill] 07/08: DRILL-6381: Address review comments (part 2): fix formatting issues and add javadoc.
This is an automated email from the ASF dual-hosted git repository. amansinha pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/drill.git commit 387bc4fefc40b645685439fd1da43f7223e5933c Author: Aman Sinha AuthorDate: Wed Oct 10 16:36:48 2018 -0700 DRILL-6381: Address review comments (part 2): fix formatting issues and add javadoc. --- .../planner/index/MapRDBFunctionalIndexInfo.java | 22 ++-- .../exec/planner/index/MapRDBIndexDescriptor.java | 4 +- .../exec/store/mapr/db/MapRDBScanBatchCreator.java | 4 +- .../exec/store/mapr/db/MapRDBSubScanSpec.java | 2 +- .../store/mapr/db/json/JsonTableGroupScan.java | 17 ++- .../db/json/JsonTableRangePartitionFunction.java | 4 +- .../store/mapr/db/json/OjaiFunctionsProcessor.java | 2 +- .../mapr/db/json/RestrictedJsonTableGroupScan.java | 8 +- .../drill/exec/udf/mapr/db/DecodeFieldPath.java| 4 +- .../drill/exec/udf/mapr/db/MatchesPlaceholder.java | 7 +- .../exec/udf/mapr/db/NotMatchesPlaceholder.java| 7 +- .../exec/udf/mapr/db/NotTypeOfPlaceholder.java | 7 +- .../drill/exec/udf/mapr/db/SizeOfPlaceholder.java | 7 +- .../drill/exec/udf/mapr/db/TypeOfPlaceholder.java | 7 +- .../maprdb/tests/index/IndexHintPlanTest.java | 19 ++- .../drill/maprdb/tests/index/IndexPlanTest.java| 135 + .../drill/maprdb/tests/index/LargeTableGen.java| 34 +++--- .../maprdb/tests/index/LargeTableGenBase.java | 25 ++-- .../drill/maprdb/tests/index/StatisticsTest.java | 21 ++-- .../drill/maprdb/tests/index/TableIndexCmd.java| 26 ++-- .../drill/exec/physical/base/IndexGroupScan.java | 1 - .../exec/physical/impl/join/RowKeyJoinBatch.java | 18 +-- .../exec/planner/index/DrillIndexDefinition.java | 22 ++-- .../exec/planner/index/DrillIndexDescriptor.java | 24 +++- .../exec/planner/index/IndexConditionInfo.java | 44 --- .../drill/exec/planner/index/IndexPlanUtils.java | 74 +-- .../drill/exec/planner/index/IndexSelector.java| 2 +- .../drill/exec/planner/index/SimpleRexRemap.java | 18 +-- .../planner/index/rules/AbstractMatchFunction.java | 8 +- .../planner/logical/DrillMergeProjectRule.java | 2 +- 30 files changed, 271 insertions(+), 304 deletions(-) diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/planner/index/MapRDBFunctionalIndexInfo.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/planner/index/MapRDBFunctionalIndexInfo.java index ec38636..67938f3 100644 --- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/planner/index/MapRDBFunctionalIndexInfo.java +++ b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/planner/index/MapRDBFunctionalIndexInfo.java @@ -32,15 +32,15 @@ public class MapRDBFunctionalIndexInfo implements FunctionalIndexInfo { private boolean hasFunctionalField = false; - //when we scan schemaPath in groupscan's columns, we check if this column(schemaPath) should be rewritten to '$N', - //When there are more than two functions on the same column in index, CAST(a.b as INT), CAST(a.b as VARCHAR), + // When we scan schemaPath in groupscan's columns, we check if this column(schemaPath) should be rewritten to '$N', + // When there are more than two functions on the same column in index, CAST(a.b as INT), CAST(a.b as VARCHAR), // then we should map SchemaPath a.b to a set of SchemaPath, e.g. $1, $2 private Map> columnToConvert; // map of functional index expression to destination SchemaPath e.g. $N private Map exprToConvert; - //map of SchemaPath involved in a functional field + // map of SchemaPath involved in a functional field private Map> pathsInExpr; private Set newPathsForIndexedFunction; @@ -52,7 +52,7 @@ public class MapRDBFunctionalIndexInfo implements FunctionalIndexInfo { columnToConvert = Maps.newHashMap(); exprToConvert = Maps.newHashMap(); pathsInExpr = Maps.newHashMap(); -//keep the order of new paths, it may be related to the naming policy +// keep the order of new paths, it may be related to the naming policy newPathsForIndexedFunction = Sets.newLinkedHashSet(); allPathsInFunction = Sets.newHashSet(); init(); @@ -60,15 +60,15 @@ public class MapRDBFunctionalIndexInfo implements FunctionalIndexInfo { private void init() { int count = 0; -for(LogicalExpression indexedExpr : indexDesc.getIndexColumns()) { - if( !(indexedExpr instanceof SchemaPath) ) { +for (LogicalExpression indexedExpr : indexDesc.getIndexColumns()) { + if (!(indexedExpr instanceof SchemaPath)) { hasFunctionalField = true; SchemaPath functionalFieldPath = SchemaPath.getSimplePath("$"+count); newPathsForIndexedFunction.add(functionalFieldPath); -//now we handle only cast expression -if(indexedExpr instanceof CastExpression)
[drill] 05/08: DRILL-6381: (Part 5) Update Javadoc for a few interfaces.
This is an automated email from the ASF dual-hosted git repository. amansinha pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/drill.git commit 3c2f9ab6c0ed290bed3bc440d72a670bbe512b5a Author: Aman Sinha AuthorDate: Thu Sep 13 03:55:47 2018 -0700 DRILL-6381: (Part 5) Update Javadoc for a few interfaces. --- .../org/apache/drill/exec/physical/base/DbSubScan.java | 4 ++-- .../drill/exec/planner/index/IndexCollection.java | 4 +++- .../drill/exec/planner/index/IndexDefinition.java | 18 -- .../drill/exec/planner/index/IndexDescriptor.java | 15 +++ 4 files changed, 36 insertions(+), 5 deletions(-) diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/DbSubScan.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/DbSubScan.java index 874468d..1977257 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/DbSubScan.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/DbSubScan.java @@ -32,8 +32,8 @@ public interface DbSubScan extends SubScan { boolean isRestrictedSubScan(); /** - * For a restricted sub-scan, this method allows associating a (hash)join instance. A subscan within a minor - * fragment must have a corresponding (hash)join batch instance from which it will retrieve its set of + * For a restricted sub-scan, this method allows associating a rowkey join instance. A subscan within a minor + * fragment must have a corresponding rowkey join batch instance from which it will retrieve its set of * rowkeys to perform the restricted scan. * @param batch */ diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/index/IndexCollection.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/index/IndexCollection.java index 9b4d170..f5ab173 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/index/IndexCollection.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/index/IndexCollection.java @@ -21,7 +21,9 @@ import org.apache.calcite.rex.RexNode; import org.apache.drill.common.expression.SchemaPath; import org.apache.drill.exec.physical.base.IndexGroupScan; -// Interface used to describe an index collection +/** + * Interface used to describe an index collection + */ public interface IndexCollection extends Iterable { /** * Types of an index collections: NATIVE_SECONDARY_INDEX_COLLECTION, EXTERNAL_SECONDARY_INDEX_COLLECTION diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/index/IndexDefinition.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/index/IndexDefinition.java index 995d23c..bfbccc2 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/index/IndexDefinition.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/index/IndexDefinition.java @@ -26,7 +26,9 @@ import java.util.Collection; import java.util.List; import java.util.Map; -// Interface used to define an index, +/** + * Top level interface used to define an index. + */ public interface IndexDefinition { /** * Types of an index: PRIMARY_KEY_INDEX, NATIVE_SECONDARY_INDEX, EXTERNAL_SECONDARY_INDEX @@ -87,13 +89,25 @@ public interface IndexDefinition { */ IndexType getIndexType(); - + /** + * Get the list of columns in the index key. + */ List getIndexColumns(); + /** + * Get the list of columns that are in the 'included' or 'covered' fields. + */ List getNonIndexColumns(); + /** + * Get the collation property (physical ordering) of the index. + */ RelCollation getCollation(); + /** + * Get a mapping of the LogicalExpresion/SchemaPath to its field collation + * - e.g Ascending/Descending, Nulls First/Nulls Last + */ Map getCollationMap(); /** diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/index/IndexDescriptor.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/index/IndexDescriptor.java index f355285..d43ba81 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/index/IndexDescriptor.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/index/IndexDescriptor.java @@ -58,11 +58,26 @@ public interface IndexDescriptor extends IndexDefinition { */ boolean supportsFullTextSearch(); + /** + * Get the functional index information associated with this index (Functional indexes are + * indexes involving expressions e.g CAST(a as INT). + */ FunctionalIndexInfo getFunctionalInfo(); + /** + * Get the total cost of index access (I/O, CPU) in the context of the current query + * @param indexProps properties (metrics) of a single index in the context of current query + * @param planner Planner instance + * @param numProjectedFields Number of projected fields + * @param primaryGroupScan Prima
[drill] 01/08: DRILL-6381: (Part 1) Secondary Index framework
This is an automated email from the ASF dual-hosted git repository. amansinha pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/drill.git commit 0abcbe3f36bf6c0a2b5fe07a778d201ead8dd2ce Author: rebase AuthorDate: Mon Feb 12 14:10:56 2018 -0800 DRILL-6381: (Part 1) Secondary Index framework  1. Secondary Index planning interfaces and abstract classes like DBGroupScan, DbSubScan, IndexDecriptor etc.  2. Statistics and Cost model interfaces/classes: PluginCost, Statistics, StatisticsPayload, AbstractIndexStatistics  3. ScanBatch and RecordReader to support repeatable scan  4. Secondary Index execution related interfaces: RangePartitionSender, RowKeyJoin, PartitionFunction 5. MD-3979: Query using cast index plan fails with NPE Co-authored-by: Aman Sinha Co-authored-by: chunhui-shi Co-authored-by: Gautam Parai Co-authored-by: Padma Penumarthy Co-authored-by: Hanumath Rao Maduri Conflicts: exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/ScanBatch.java exec/java-exec/src/main/java/org/apache/drill/exec/planner/common/DrillRelOptUtil.java exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillTable.java protocol/src/main/java/org/apache/drill/exec/proto/UserBitShared.java protocol/src/main/java/org/apache/drill/exec/proto/beans/CoreOperatorType.java protocol/src/main/protobuf/UserBitShared.proto --- .../java/org/apache/drill/exec/ExecConstants.java | 2 + .../exec/physical/base/AbstractDbGroupScan.java| 95 +++ .../exec/physical/base/AbstractDbSubScan.java | 37 +++ .../physical/base/AbstractPhysicalVisitor.java | 12 + .../drill/exec/physical/base/DbGroupScan.java | 129 + .../apache/drill/exec/physical/base/DbSubScan.java | 43 +++ .../drill/exec/physical/base/IndexGroupScan.java | 76 ++ .../drill/exec/physical/base/PhysicalVisitor.java | 4 + ...{RangeSender.java => RangePartitionSender.java} | 52 ++-- .../apache/drill/exec/physical/impl/ScanBatch.java | 15 +- .../drill/exec/physical/impl/join/RowKeyJoin.java | 79 ++ .../drill/exec/planner/common/DrillRelOptUtil.java | 17 ++ .../apache/drill/exec/planner/cost/PluginCost.java | 79 ++ .../planner/index/AbstractIndexCollection.java | 96 +++ .../planner/index/AbstractIndexDescriptor.java | 74 ++ .../planner/index/AbstractIndexStatistics.java | 51 .../drill/exec/planner/index/CollationContext.java | 37 +++ .../exec/planner/index/DrillIndexCollection.java | 75 ++ .../exec/planner/index/DrillIndexDefinition.java | 278 .../exec/planner/index/DrillIndexDescriptor.java | 110 .../exec/planner/index/FunctionalIndexInfo.java| 85 ++ .../drill/exec/planner/index/IndexCallContext.java | 76 ++ .../drill/exec/planner/index/IndexCollection.java | 99 +++ .../drill/exec/planner/index/IndexDefinition.java | 105 .../drill/exec/planner/index/IndexDescriptor.java | 68 + .../drill/exec/planner/index/IndexDiscover.java| 23 ++ .../exec/planner/index/IndexDiscoverBase.java | 110 .../exec/planner/index/IndexDiscoverable.java | 37 +++ .../drill/exec/planner/index/IndexGroup.java | 63 + .../drill/exec/planner/index/IndexProperties.java | 64 + .../drill/exec/planner/index/IndexStatistics.java | 36 +++ .../exec/planner/index/IndexableExprMarker.java| 262 +++ .../index/InvalidIndexDefinitionException.java | 27 ++ .../drill/exec/planner/index/Statistics.java | 66 + .../exec/planner/index/StatisticsPayload.java | 24 ++ .../drill/exec/planner/logical/DrillTable.java | 4 + .../exec/planner/physical/PartitionFunction.java | 56 .../drill/exec/record/AbstractRecordBatch.java | 2 +- .../drill/exec/store/AbstractRecordReader.java | 5 + .../org/apache/drill/exec/store/RecordReader.java | 8 + .../drill/exec/util/EncodedSchemaPathSet.java | 291 + .../org/apache/drill/exec/proto/UserBitShared.java | 75 +++--- .../drill/exec/proto/beans/CoreOperatorType.java | 8 +- protocol/src/main/protobuf/UserBitShared.proto | 3 +- 44 files changed, 2893 insertions(+), 65 deletions(-) diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java b/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java index 21e16eb..cb0fc5c 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java @@ -807,6 +807,8 @@ public final class ExecConstants { */ public static final String ENABLE_ITERATOR_VALIDATION = "drill.exec.debug.validate_iterators"; + public static final String QUERY_ROWKEYJOIN_BATCHSIZE_KEY = "exec.query.ro
[drill] 08/08: DRILL-6381: Address code review comments (part 3).
This is an automated email from the ASF dual-hosted git repository. amansinha pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/drill.git commit 7571d52eab9b961687df7d4fb845d0a297b228bb Author: Aman Sinha AuthorDate: Sat Oct 13 23:38:17 2018 -0700 DRILL-6381: Address code review comments (part 3). DRILL-6381: Add missing joinControl logic for INTERSECT_DISTINCT. - Modified HashJoin's probe phase to process INTERSECT_DISTINCT. - NOTE: For build phase, the functionality will be same as for SemiJoin when it is added later. DRILL-6381: Address code review comment for intersect_distinct. DRILL-6381: Rebase on latest master and fix compilation issues. DRILL-6381: Generate protobuf files for C++ native client. DRILL-6381: Use shaded Guava classes. Add more comments and Javadoc. --- .../planner/index/MapRDBFunctionalIndexInfo.java | 4 +- .../exec/planner/index/MapRDBIndexDiscover.java| 12 ++--- .../drill/exec/store/mapr/PluginConstants.java | 3 ++ .../exec/store/mapr/db/MapRDBFormatPlugin.java | 6 ++- .../store/mapr/db/MapRDBPushFilterIntoScan.java| 17 --- .../store/mapr/db/MapRDBPushLimitIntoScan.java | 2 +- .../store/mapr/db/MapRDBPushProjectIntoScan.java | 35 ++--- .../mapr/db/MapRDBRestrictedScanBatchCreator.java | 4 +- .../db/json/JsonTableRangePartitionFunction.java | 4 +- .../store/mapr/db/json/MaprDBJsonRecordReader.java | 2 +- .../mapr/db/json/RestrictedJsonTableGroupScan.java | 6 +-- contrib/native/client/src/protobuf/BitData.pb.cc | 55 ++--- contrib/native/client/src/protobuf/BitData.pb.h| 36 +- .../native/client/src/protobuf/UserBitShared.pb.cc | 57 +++--- .../native/client/src/protobuf/UserBitShared.pb.h | 7 +-- .../exec/store/hbase/HBasePushFilterIntoScan.java | 2 +- .../store/kafka/KafkaPushDownFilterIntoScan.java | 4 +- .../store/mongo/MongoPushDownFilterForScan.java| 2 +- .../apache/drill/exec/physical/impl/ScanBatch.java | 17 +++ .../exec/physical/impl/common/HashPartition.java | 10 ++-- .../physical/impl/join/HashJoinProbeTemplate.java | 24 +++-- .../exec/physical/impl/join/RowKeyJoinBatch.java | 6 +++ .../physical/impl/partitionsender/Partitioner.java | 3 ++ .../RangePartitionRecordBatch.java | 7 +++ .../exec/planner/common/DrillScanRelBase.java | 2 +- .../index/InvalidIndexDefinitionException.java | 4 ++ .../generators/IndexIntersectPlanGenerator.java| 18 +++ .../planner/logical/DrillMergeProjectRule.java | 8 +++ .../planner/physical/BroadcastExchangePrel.java| 5 +- .../planner/physical/ConvertCountToDirectScan.java | 5 +- .../drill/exec/planner/physical/ScanPrel.java | 9 .../drill/exec/planner/physical/ScanPrule.java | 2 +- .../InfoSchemaPushFilterIntoRecordGenerator.java | 2 +- .../exec/store/parquet/ParquetPushDownFilter.java | 2 +- .../physical/impl/common/HashPartitionTest.java| 2 +- 35 files changed, 248 insertions(+), 136 deletions(-) diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/planner/index/MapRDBFunctionalIndexInfo.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/planner/index/MapRDBFunctionalIndexInfo.java index 67938f3..564a037 100644 --- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/planner/index/MapRDBFunctionalIndexInfo.java +++ b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/planner/index/MapRDBFunctionalIndexInfo.java @@ -17,8 +17,8 @@ */ package org.apache.drill.exec.planner.index; -import com.google.common.collect.Maps; -import com.google.common.collect.Sets; +import org.apache.drill.shaded.guava.com.google.common.collect.Maps; +import org.apache.drill.shaded.guava.com.google.common.collect.Sets; import org.apache.drill.common.expression.CastExpression; import org.apache.drill.common.expression.LogicalExpression; import org.apache.drill.common.expression.SchemaPath; diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/planner/index/MapRDBIndexDiscover.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/planner/index/MapRDBIndexDiscover.java index c231e11..aed3e04 100644 --- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/planner/index/MapRDBIndexDiscover.java +++ b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/planner/index/MapRDBIndexDiscover.java @@ -100,7 +100,7 @@ public class MapRDBIndexDiscover extends IndexDiscoverBase implements IndexDisco for (IndexDesc idx : indexes) { DrillIndexDescriptor hbaseIdx = buildIndexDescriptor(tableName, idx); if (hbaseIdx == null) { - //not able to build a valid index based on the index info from MFS + // not able to build a valid index based on the index info from MFS logger.error("Not able to build
[drill] branch master updated (61e8b46 -> 7571d52)
This is an automated email from the ASF dual-hosted git repository. amansinha pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/drill.git. from 61e8b46 DRILL-6715: Update descriptions for System Options table new 0abcbe3 DRILL-6381: (Part 1) Secondary Index framework new a4f62e9 DRILL-6381: (Part 2) MapRDB plugin update to 6.0.1 new 12f57f0 DRILL-6381: (Part 3) Planner and Execution implementation to support Secondary Indexes new 2a9e51f DRILL-6381: (Part 4) Enhance MapR-DB plugin to support querying secondary indexes new 3c2f9ab DRILL-6381: (Part 5) Update Javadoc for a few interfaces. new 5fa9c80 DRILL-6381: Address code review comments. new 387bc4f DRILL-6381: Address review comments (part 2): fix formatting issues and add javadoc. new 7571d52 DRILL-6381: Address code review comments (part 3). The 8 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: contrib/format-maprdb/pom.xml | 89 ++ .../planner/index/MapRDBFunctionalIndexInfo.java | 168 ++ .../exec/planner/index/MapRDBIndexDescriptor.java | 222 +++ .../exec/planner/index/MapRDBIndexDiscover.java| 376 + .../drill/exec/planner/index/MapRDBStatistics.java | 1006 .../planner/index/MapRDBStatisticsPayload.java | 44 +- .../drill/exec/store/mapr/PluginConstants.java | 93 ++ .../drill/exec/store/mapr/PluginErrorHandler.java | 50 + .../drill/exec/store/mapr/db/MapRDBCost.java | 104 ++ .../exec/store/mapr/db/MapRDBFormatMatcher.java| 46 +- .../exec/store/mapr/db/MapRDBFormatPlugin.java | 69 +- .../store/mapr/db/MapRDBFormatPluginConfig.java|4 + .../drill/exec/store/mapr/db/MapRDBGroupScan.java | 83 +- .../store/mapr/db/MapRDBPushFilterIntoScan.java| 20 +- .../store/mapr/db/MapRDBPushLimitIntoScan.java | 203 +++ .../store/mapr/db/MapRDBPushProjectIntoScan.java | 142 ++ java => MapRDBRestrictedScanBatchCreator.java} | 41 +- .../exec/store/mapr/db/MapRDBScanBatchCreator.java |6 +- .../drill/exec/store/mapr/db/MapRDBSubScan.java| 23 +- .../exec/store/mapr/db/MapRDBSubScanSpec.java | 54 +- .../drill/exec/store/mapr/db/MapRDBTableCache.java | 232 +++ .../store/mapr/db/RestrictedMapRDBSubScan.java | 80 + .../store/mapr/db/RestrictedMapRDBSubScanSpec.java | 191 +++ .../store/mapr/db/binary/BinaryTableGroupScan.java | 34 +- .../store/mapr/db/json/AllTextValueWriter.java | 80 + .../mapr/db/json/CompareFunctionsProcessor.java| 15 +- .../mapr/db/json/DocumentReaderVectorWriter.java | 42 + .../exec/store/mapr/db/json/FieldPathHelper.java | 75 + .../mapr/db/json/FieldTransferVectorWriter.java| 49 + .../store/mapr/db/json/IdOnlyVectorWriter.java | 65 + .../store/mapr/db/json/JsonConditionBuilder.java | 59 +- .../exec/store/mapr/db/json/JsonScanSpec.java | 93 +- .../exec/store/mapr/db/json/JsonSubScanSpec.java | 69 +- .../store/mapr/db/json/JsonTableGroupScan.java | 676 +++- .../db/json/JsonTableRangePartitionFunction.java | 237 +++ .../store/mapr/db/json/MaprDBJsonRecordReader.java | 538 +++ .../mapr/db/json/NumbersAsDoubleValueWriter.java | 51 + .../store/mapr/db/json/OjaiFunctionsProcessor.java | 214 +++ .../exec/store/mapr/db/json/OjaiValueWriter.java | 194 +++ .../db/json/ProjectionPassthroughVectorWriter.java | 83 + .../mapr/db/json/RestrictedJsonRecordReader.java | 248 +++ .../mapr/db/json/RestrictedJsonTableGroupScan.java | 184 +++ .../store/mapr/db/json/RowCountVectorWriter.java | 29 +- .../exec/udf/mapr/db/ConditionPlaceholder.java | 54 + .../drill/exec/udf/mapr/db/DecodeFieldPath.java| 65 + .../drill/exec/udf/mapr/db/MatchesPlaceholder.java | 53 + .../exec/udf/mapr/db/NotMatchesPlaceholder.java| 53 + .../exec/udf/mapr/db/NotTypeOfPlaceholder.java | 53 + .../drill/exec/udf/mapr/db/SizeOfPlaceholder.java | 54 + .../drill/exec/udf/mapr/db/TypeOfPlaceholder.java | 53 + .../src/main/resources/drill-module.conf | 20 +- .../mapr/drill/maprdb/tests/MaprDBTestsSuite.java | 63 +- .../maprdb/tests/index/IndexHintPlanTest.java | 166 ++ .../drill/maprdb/tests/index/IndexPlanTest.java| 1690 .../drill/maprdb/tests/index/LargeTableGen.java| 172 ++ .../maprdb/tests/index/LargeTableGenBase.java | 185 +++ .../drill/maprdb/tests/index/StatisticsTest.java | 108 ++ .../drill/maprdb/tests/index/TableIndexCmd.java| 123 ++ .../mapr/drill/maprdb/tests/json/BaseJsonTest.java | 13 + .../maprdb/tests/json/TestEncodedFieldPaths.java | 128 ++ .../maprdb/tests/json/TestFieldPathHelper.java | 52 + .../
[drill] 06/08: DRILL-6381: Address code review comments.
This is an automated email from the ASF dual-hosted git repository. amansinha pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/drill.git commit 5fa9c808daef8ea70a39ac2248daa99055955b72 Author: Aman Sinha AuthorDate: Mon Oct 1 12:06:39 2018 -0700 DRILL-6381: Address code review comments. --- .../planner/index/MapRDBFunctionalIndexInfo.java | 5 .../exec/planner/index/MapRDBIndexDiscover.java| 14 ++- .../store/mapr/db/RestrictedMapRDBSubScanSpec.java | 28 -- .../db/json/JsonTableRangePartitionFunction.java | 2 +- .../drill/exec/physical/config/HashJoinPOP.java| 12 -- .../planner/index/AbstractIndexDescriptor.java | 6 + .../drill/exec/planner/index/IndexDescriptor.java | 11 +++-- .../drill/exec/planner/index/IndexGroup.java | 6 + .../generators/IndexIntersectPlanGenerator.java| 22 + .../generators/NonCoveringIndexPlanGenerator.java | 21 .../physical/DrillDistributionTraitDef.java| 8 ++- .../exec/planner/physical/RowKeyJoinPrel.java | 7 -- 12 files changed, 70 insertions(+), 72 deletions(-) diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/planner/index/MapRDBFunctionalIndexInfo.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/planner/index/MapRDBFunctionalIndexInfo.java index 01561a3..ec38636 100644 --- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/planner/index/MapRDBFunctionalIndexInfo.java +++ b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/planner/index/MapRDBFunctionalIndexInfo.java @@ -138,6 +138,11 @@ public class MapRDBFunctionalIndexInfo implements FunctionalIndexInfo { } /** + * Suppose the index key has functions (rather than plain columns): CAST(a as int), CAST(b as varchar(10)), + * then we want to maintain a mapping of the logical expression of that function to the schema path of the + * base column involved in the function. In this example map has 2 entries: + * CAST(a as int) --> 'a' + * CAST(b as varchar(10)) --> 'b' * @return the map of indexed expression --> the involved schema paths in a indexed expression */ public Map> getPathsInFunctionExpr() { diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/planner/index/MapRDBIndexDiscover.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/planner/index/MapRDBIndexDiscover.java index e1b8a61..c231e11 100644 --- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/planner/index/MapRDBIndexDiscover.java +++ b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/planner/index/MapRDBIndexDiscover.java @@ -37,6 +37,7 @@ import org.apache.drill.common.expression.parser.ExprLexer; import org.apache.drill.common.expression.parser.ExprParser; import org.apache.drill.common.types.TypeProtos; import org.apache.drill.common.types.Types; +import org.apache.drill.common.util.DrillFileUtils; import org.apache.drill.exec.physical.base.AbstractDbGroupScan; import org.apache.drill.exec.physical.base.GroupScan; import org.apache.drill.exec.planner.common.DrillScanRelBase; @@ -67,6 +68,7 @@ import java.util.Set; public class MapRDBIndexDiscover extends IndexDiscoverBase implements IndexDiscover { static final String DEFAULT_STRING_CAST_LEN_STR = "256"; + static final String FIELD_DELIMITER = ":"; public MapRDBIndexDiscover(GroupScan inScan, DrillScanRelBase scanRel) { super((AbstractDbGroupScan) inScan, scanRel); @@ -78,14 +80,14 @@ public class MapRDBIndexDiscover extends IndexDiscoverBase implements IndexDisco @Override public IndexCollection getTableIndex(String tableName) { -//return getTableIndexFromCommandLine(tableName); return getTableIndexFromMFS(tableName); } /** - * + * For a given table name get the list of indexes defined on the table according to the visibility of + * the indexes based on permissions. * @param tableName - * @return + * @return an IndexCollection representing the list of indexes for that table */ private IndexCollection getTableIndexFromMFS(String tableName) { try { @@ -120,8 +122,8 @@ public class MapRDBIndexDiscover extends IndexDiscoverBase implements IndexDisco FileSelection deriveFSSelection(DrillFileSystem fs, IndexDescriptor idxDesc) throws IOException { String tableName = idxDesc.getTableName(); -String[] tablePath = tableName.split("/"); -String tableParent = tableName.substring(0, tableName.lastIndexOf("/")); +String[] tablePath = tableName.split(DrillFileUtils.SEPARATOR); +String tableParent = tableName.substring(0, tableName.lastIndexOf(DrillFileUtils.SEPARATOR)); return FileSelection.create(fs, tableParent, tablePath[tablePath.length - 1], false); } @@ -318,7 +320,7 @@ public class MapRDBI
[drill] 03/04: DRILL-6783: CAST string literal as INTERVAL MONTH/YEAR works inconsistently when selecting from a table with multiple rows
This is an automated email from the ASF dual-hosted git repository. amansinha pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/drill.git commit 2aede3a1cd1532fbcd0a622b49a3b9fce69979ef Author: Bohdan Kazydub AuthorDate: Mon Oct 8 16:50:22 2018 +0300 DRILL-6783: CAST string literal as INTERVAL MONTH/YEAR works inconsistently when selecting from a table with multiple rows close apache/drill#1496 --- .../drill/exec/fn/impl/TestCastFunctions.java | 38 +- .../main/codegen/templates/FixedValueVectors.java | 3 +- 2 files changed, 39 insertions(+), 2 deletions(-) diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestCastFunctions.java b/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestCastFunctions.java index cb0d022..7768909 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestCastFunctions.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestCastFunctions.java @@ -19,14 +19,21 @@ package org.apache.drill.exec.fn.impl; import java.math.BigDecimal; import java.time.LocalDate; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; import org.apache.drill.categories.SqlFunctionTest; import org.apache.drill.categories.UnlikelyTest; import org.apache.drill.common.exceptions.UserRemoteException; +import org.apache.drill.common.expression.SchemaPath; import org.apache.drill.exec.planner.physical.PlannerSettings; +import org.apache.drill.exec.record.RecordBatchLoader; +import org.apache.drill.exec.rpc.user.QueryDataBatch; +import org.apache.drill.exec.vector.IntervalYearVector; import org.apache.drill.test.BaseTestQuery; +import org.junit.Assert; import org.junit.Rule; import org.junit.Test; import org.junit.experimental.categories.Category; @@ -39,6 +46,7 @@ import org.apache.drill.shaded.guava.com.google.common.collect.Maps; import mockit.integration.junit4.JMockit; import static org.hamcrest.CoreMatchers.containsString; +import static org.hamcrest.CoreMatchers.hasItem; @RunWith(JMockit.class) @Category({UnlikelyTest.class, SqlFunctionTest.class}) @@ -664,4 +672,32 @@ public class TestCastFunctions extends BaseTestQuery { test(query); } -} \ No newline at end of file + + @Test // DRILL-6783 + public void testCastVarCharIntervalYear() throws Exception { +String query = "select cast('P31M' as interval month) as i from cp.`employee.json` limit 10"; +List result = testSqlWithResults(query); +RecordBatchLoader loader = new RecordBatchLoader(getDrillbitContext().getAllocator()); + +QueryDataBatch b = result.get(0); +loader.load(b.getHeader().getDef(), b.getData()); + +IntervalYearVector vector = (IntervalYearVector) loader.getValueAccessorById( + IntervalYearVector.class, + loader.getValueVectorId(SchemaPath.getCompoundPath("i")).getFieldIds()) +.getValueVector(); + +Set resultSet = new HashSet<>(); +for (int i = 0; i < loader.getRecordCount(); i++) { + String displayValue = vector.getAccessor().getAsStringBuilder(i).toString(); + resultSet.add(displayValue); +} + +Assert.assertEquals( +"Casting literal string as INTERVAL should yield the same result for each row", 1, resultSet.size()); +Assert.assertThat(resultSet, hasItem("2 years 7 months")); + +b.release(); +loader.clear(); + } +} diff --git a/exec/vector/src/main/codegen/templates/FixedValueVectors.java b/exec/vector/src/main/codegen/templates/FixedValueVectors.java index a98aa66..508e484 100644 --- a/exec/vector/src/main/codegen/templates/FixedValueVectors.java +++ b/exec/vector/src/main/codegen/templates/FixedValueVectors.java @@ -532,7 +532,8 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F } public StringBuilder getAsStringBuilder(int index) { - return DateUtilities.intervalYearStringBuilder(data.getInt(index)); + int value = get(index); + return DateUtilities.intervalYearStringBuilder(value); } <#elseif minor.class == "Time">
[drill] 02/04: DRILL-6777: Setup CircleCI configs for Drill
This is an automated email from the ASF dual-hosted git repository. amansinha pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/drill.git commit 484bcfb1e237b6f8f9a244d344f36dcb86dedb61 Author: Vitalii Diravka AuthorDate: Tue Sep 11 14:39:13 2018 +0300 DRILL-6777: Setup CircleCI configs for Drill - adding .circleci/config.yml to the project to launch CircleCI - custom memory parameters - usage of CircleCI machine - excluding "SlowTest" and "UnlikelyTest" groups - update maven version - adding libaio.so library to solve MySQL integration tests - update com.jcabi:jcabi-mysql-maven-plugin library version - TODO descriptions for the future enhancements of CircleCI build for Drill close apache/drill#1493 --- .circleci/config.yml | 51 ++ contrib/storage-jdbc/pom.xml | 2 +- .../exec/store/jdbc/TestJdbcPluginWithMySQLIT.java | 1 + 3 files changed, 53 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 000..1b179df --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,51 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +version: 2 +general: +jobs: + build: +machine: + enabled: true + image: circleci/classic:latest +parallelism: 1 # TODO: 1. Configuring Parallel Jobs for Running Tests https://circleci.com/docs/2.0/parallelism-faster-jobs/ + + +working_directory: ~/drill + +steps: +- checkout + +- run: +name: Update maven version +# TODO: Could be removed, once Machine Executor image is updated https://github.com/circleci/image-builder/issues/140 +# and the possibility of specifying Maven version is added https://github.com/circleci/image-builder/issues/143 +command: + curl -fsSL https://git.io/vpDIf | bash -s -- 3.5.4 +- run: +name: Update packages list +command: + sudo apt-get update +- run: +name: Install libaio1.so library for MySQL integration tests +command: + sudo apt-get install libaio1 libaio-dev +- run: +name: Drill project build +# TODO: 2. Optimizing Maven Builds on CircleCI - https://circleci.com/blog/optimizing-maven-builds-on-circleci/ +# TODO: 3. Caching Dependencies - https://circleci.com/docs/2.0/caching/ +# TODO: 4. Resolving memory issues without "SlowTest" and "UnlikelyTest" excludedGroups in the build +command: + mvn install -Drat.skip=false -Dlicense.skip=false -DmemoryMb=2560 -DdirectMemoryMb=4608 -DexcludedGroups="org.apache.drill.categories.SlowTest,org.apache.drill.categories.UnlikelyTest" diff --git a/contrib/storage-jdbc/pom.xml b/contrib/storage-jdbc/pom.xml index a032e69..7de89f0 100755 --- a/contrib/storage-jdbc/pom.xml +++ b/contrib/storage-jdbc/pom.xml @@ -213,7 +213,7 @@ com.jcabi jcabi-mysql-maven-plugin -0.8.2 +0.9 mysql-test diff --git a/contrib/storage-jdbc/src/test/java/org/apache/drill/exec/store/jdbc/TestJdbcPluginWithMySQLIT.java b/contrib/storage-jdbc/src/test/java/org/apache/drill/exec/store/jdbc/TestJdbcPluginWithMySQLIT.java index 1640913..0f377e3 100644 --- a/contrib/storage-jdbc/src/test/java/org/apache/drill/exec/store/jdbc/TestJdbcPluginWithMySQLIT.java +++ b/contrib/storage-jdbc/src/test/java/org/apache/drill/exec/store/jdbc/TestJdbcPluginWithMySQLIT.java @@ -26,6 +26,7 @@ import org.junit.experimental.categories.Category; /** * JDBC storage plugin tests against MySQL. + * Note: it requires libaio.so library in the system */ @Category(JdbcStorageTest.class) public class TestJdbcPluginWithMySQLIT extends PlanTestBase {
[drill] branch master updated (734c460 -> ec6acc7)
This is an automated email from the ASF dual-hosted git repository. amansinha pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/drill.git. from 734c460 DRILL-3988: Expose Drill built-in functions & UDFs in a system table (#1483) new 0a3cfde DRILL-6473: Update MapR Hive new 484bcfb DRILL-6777: Setup CircleCI configs for Drill new 2aede3a DRILL-6783: CAST string literal as INTERVAL MONTH/YEAR works inconsistently when selecting from a table with multiple rows new ec6acc7 DRILL-6775: The schema for empty output is not shown in Drill Web UI The 4 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .circleci/config.yml | 51 ++ .../store/hive/readers/HiveAbstractReader.java | 19 +++- contrib/storage-hive/hive-exec-shade/pom.xml | 3 ++ contrib/storage-jdbc/pom.xml | 2 +- .../exec/store/jdbc/TestJdbcPluginWithMySQLIT.java | 1 + .../drill/exec/server/rest/QueryWrapper.java | 5 --- .../drill/exec/server/rest/WebUserConnection.java | 8 ++-- .../drill/exec/fn/impl/TestCastFunctions.java | 38 +++- .../main/codegen/templates/FixedValueVectors.java | 3 +- pom.xml| 4 +- 10 files changed, 106 insertions(+), 28 deletions(-) create mode 100644 .circleci/config.yml
[drill] 01/04: DRILL-6473: Update MapR Hive
This is an automated email from the ASF dual-hosted git repository. amansinha pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/drill.git commit 0a3cfdebd25ea4186c2a2b75524c34323984ffca Author: Bohdan Kazydub AuthorDate: Mon May 21 19:11:23 2018 +0300 DRILL-6473: Update MapR Hive close apache/drill#1307 --- .../exec/store/hive/readers/HiveAbstractReader.java | 19 +-- contrib/storage-hive/hive-exec-shade/pom.xml | 3 +++ pom.xml | 4 ++-- 3 files changed, 10 insertions(+), 16 deletions(-) diff --git a/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/readers/HiveAbstractReader.java b/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/readers/HiveAbstractReader.java index ba1cd30..5ed6c3b 100644 --- a/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/readers/HiveAbstractReader.java +++ b/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/readers/HiveAbstractReader.java @@ -25,6 +25,7 @@ import java.util.List; import java.util.Properties; import java.util.concurrent.Callable; import java.util.concurrent.ExecutionException; +import java.util.stream.Collectors; import org.apache.drill.shaded.guava.com.google.common.util.concurrent.ListenableFuture; import io.netty.buffer.DrillBuf; @@ -203,20 +204,10 @@ public abstract class HiveAbstractReader extends AbstractRecordReader { } } } - ColumnProjectionUtils.appendReadColumns(job, columnIds); - - // TODO: Use below overloaded method instead of above simpler version of it, once Hive client dependencies - // (from all profiles) will be updated to 2.3 version or above -// ColumnProjectionUtils.appendReadColumns(job, columnIds, selectedColumnNames, -// Lists.newArrayList(Iterables.transform(getColumns(), new Function() -// { -//@Nullable -//@Override -//public String apply(@Nullable SchemaPath path) -//{ -// return path.getRootSegmentPath(); -//} -// }))); + List paths = getColumns().stream() + .map(SchemaPath::getRootSegmentPath) + .collect(Collectors.toList()); + ColumnProjectionUtils.appendReadColumns(job, columnIds, selectedColumnNames, paths); for (String columnName : selectedColumnNames) { StructField fieldRef = finalOI.getStructFieldRef(columnName); diff --git a/contrib/storage-hive/hive-exec-shade/pom.xml b/contrib/storage-hive/hive-exec-shade/pom.xml index 3ef1839..b857758 100644 --- a/contrib/storage-hive/hive-exec-shade/pom.xml +++ b/contrib/storage-hive/hive-exec-shade/pom.xml @@ -139,6 +139,9 @@ org.apache.hive:hive-exec + org/apache/parquet/schema/* diff --git a/pom.xml b/pom.xml index 5d00ee6..3a2d21e 100644 --- a/pom.xml +++ b/pom.xml @@ -2439,9 +2439,9 @@ mapr true -2.1.1-mapr-1710 +2.3.3-mapr-1808 1.1.1-mapr-1602-m7-5.2.0 -2.7.0-mapr-1707 +2.7.0-mapr-1808 3.4.11-mapr-1808
[drill] 04/04: DRILL-6775: The schema for empty output is not shown in Drill Web UI
This is an automated email from the ASF dual-hosted git repository. amansinha pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/drill.git commit ec6acc7ab6ee1a6eb326a4783189978bf55c0bcf Author: Anton Gozhiy AuthorDate: Tue Oct 9 15:59:16 2018 +0300 DRILL-6775: The schema for empty output is not shown in Drill Web UI Removed an excess check for the result emptiness that prevented retrieval of the column names. close apache/drill#1498 --- .../main/java/org/apache/drill/exec/server/rest/QueryWrapper.java | 5 - .../java/org/apache/drill/exec/server/rest/WebUserConnection.java | 8 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/QueryWrapper.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/QueryWrapper.java index 42427aa..4e09858 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/QueryWrapper.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/QueryWrapper.java @@ -19,7 +19,6 @@ package org.apache.drill.exec.server.rest; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; -import org.apache.drill.shaded.guava.com.google.common.collect.Maps; import org.apache.drill.common.exceptions.UserException; import org.apache.drill.common.exceptions.UserRemoteException; @@ -115,10 +114,6 @@ public class QueryWrapper { throw new UserRemoteException(webUserConnection.getError()); } -if (webUserConnection.results.isEmpty()) { - webUserConnection.results.add(Maps.newHashMap()); -} - // Return the QueryResult. return new QueryResult(queryId, webUserConnection.columns, webUserConnection.results); } diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/WebUserConnection.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/WebUserConnection.java index df4c8a5..9d6e7e4 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/WebUserConnection.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/WebUserConnection.java @@ -75,20 +75,20 @@ public class WebUserConnection extends AbstractDisposableUserClientConnection im @Override public void sendData(RpcOutcomeListener listener, QueryWritableBatch result) { -// Check if there is any data or not. There can be overflow here but DrillBuf doesn't support allocating with +// There can be overflow here but DrillBuf doesn't support allocating with // bytes in long. Hence we are just preserving the earlier behavior and logging debug log for the case. final int dataByteCount = (int) result.getByteCount(); -if (dataByteCount <= 0) { +if (dataByteCount < 0) { if (logger.isDebugEnabled()) { -logger.debug("Either no data received in this batch or there is BufferOverflow in dataByteCount: {}", +logger.debug("There is BufferOverflow in dataByteCount: {}", dataByteCount); } listener.success(Acks.OK, null); return; } -// If here that means there is some data for sure. Create a ByteBuf with all the data in it. +// Create a ByteBuf with all the data in it. final int rows = result.getHeader().getRowCount(); final BufferAllocator allocator = webSessionResources.getAllocator(); final DrillBuf bufferWithData = allocator.buffer(dataByteCount);
[drill] 02/02: DRILL-6272: Refactor dynamic UDFs and function initializer tests to generate needed binary and source jars at runtime
This is an automated email from the ASF dual-hosted git repository. amansinha pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/drill.git commit ca90229b6deea40282927f8ab5c07715a4e18620 Author: Arina Ielchiieva AuthorDate: Sat May 5 15:32:07 2018 +0300 DRILL-6272: Refactor dynamic UDFs and function initializer tests to generate needed binary and source jars at runtime close apache/drill#1225 --- exec/java-exec/pom.xml | 48 ++ .../exec/expr/fn/FunctionInitializerTest.java | 54 +-- .../apache/drill/exec/udf/dynamic/JarBuilder.java | 96 .../udf/dynamic}/TestDynamicUDFSupport.java| 523 ++--- .../java-exec/src/test/resources/drill-udf/pom.xml | 92 .../drill/udf/dynamic/CustomAbsFunction.java | 63 +++ .../drill/udf/dynamic/CustomLogFunction.java | 58 +++ .../udf/dynamic/CustomLowerDummyFunction.java | 58 +++ .../drill/udf/dynamic/CustomLowerFunction.java | 64 +++ .../drill/udf/dynamic/CustomLowerFunctionV2.java | 64 +++ .../drill/udf/dynamic/CustomUpperFunction.java | 64 +++ .../apache/drill/udf/dynamic/LowerFunction.java| 64 +++ .../drill-udf/src/main/resources/drill-module.conf | 1 + .../test/resources/jars/DrillUDF-1.0-sources.jar | Bin 1892 -> 0 bytes .../src/test/resources/jars/DrillUDF-1.0.jar | Bin 3146 -> 0 bytes .../test/resources/jars/DrillUDF-2.0-sources.jar | Bin 1891 -> 0 bytes .../src/test/resources/jars/DrillUDF-2.0.jar | Bin 3142 -> 0 bytes .../jars/DrillUDF-overloading-1.0-sources.jar | Bin 3473 -> 0 bytes .../resources/jars/DrillUDF-overloading-1.0.jar| Bin 5779 -> 0 bytes .../resources/jars/DrillUDF_Copy-1.0-sources.jar | Bin 1892 -> 0 bytes .../src/test/resources/jars/DrillUDF_Copy-1.0.jar | Bin 3185 -> 0 bytes .../jars/DrillUDF_DupFunc-1.0-sources.jar | Bin 1888 -> 0 bytes .../test/resources/jars/DrillUDF_DupFunc-1.0.jar | Bin 3201 -> 0 bytes .../resources/jars/DrillUDF_Empty-1.0-sources.jar | Bin 536 -> 0 bytes .../src/test/resources/jars/DrillUDF_Empty-1.0.jar | Bin 1863 -> 0 bytes .../jars/DrillUDF_NoMarkerFile-1.0-sources.jar | Bin 1715 -> 0 bytes .../resources/jars/DrillUDF_NoMarkerFile-1.0.jar | Bin 3084 -> 0 bytes .../resources/jars/v2/DrillUDF-1.0-sources.jar | Bin 1899 -> 0 bytes .../src/test/resources/jars/v2/DrillUDF-1.0.jar| Bin 3215 -> 0 bytes 29 files changed, 948 insertions(+), 301 deletions(-) diff --git a/exec/java-exec/pom.xml b/exec/java-exec/pom.xml index cbc3a02..345e240 100644 --- a/exec/java-exec/pom.xml +++ b/exec/java-exec/pom.xml @@ -584,6 +584,54 @@ netty-tcnative ${netty.tcnative.classifier} + + org.apache.maven + maven-embedder + 3.5.3 + test + + + org.apache.maven + maven-compat + 3.5.3 + test + + + org.eclipse.aether + aether-connector-basic + 1.1.0 + test + + + org.eclipse.aether + aether-transport-wagon + 1.1.0 + test + + + org.apache.maven.wagon + wagon-http + 3.0.0 + test + + + commons-logging + commons-logging + + + + + org.apache.maven.wagon + wagon-provider-api + 3.0.0 + test + + + commons-logging + commons-logging + + + diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/expr/fn/FunctionInitializerTest.java b/exec/java-exec/src/test/java/org/apache/drill/exec/expr/fn/FunctionInitializerTest.java index 7c10bd3..2ecb8a0 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/expr/fn/FunctionInitializerTest.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/expr/fn/FunctionInitializerTest.java @@ -19,20 +19,20 @@ package org.apache.drill.exec.expr.fn; import com.google.common.collect.Lists; import org.apache.drill.categories.SqlFunctionTest; -import org.apache.drill.test.TestTools; +import org.apache.drill.exec.udf.dynamic.JarBuilder; import org.apache.drill.exec.util.JarUtil; import org.codehaus.janino.Java.CompilationUnit; import org.junit.BeforeClass; +import org.junit.ClassRule; import org.junit.Test; import org.junit.experimental.categories.Category; -import org.junit.runner.RunWith; -import org.mockito.invocation.InvocationOnMock; -import org.mockito.runners.MockitoJUnitRunner; -import org.mockito.stubbing.Answer; +import org.junit.rules.TemporaryFolder; +import java.io.File; +import java.io.IOException; import java.net.URL; import java.net.URLClassLoader; -import java.nio.file.Path; +import java.nio.file.Paths; import java.util.Collections; import java.util.HashSet; import java.util.List; @@ -45,25 +45,27 @@ import java.util.concurrent.atomic.AtomicInteger; import static org.junit.Assert.assertEquals; import st
[drill] branch master updated (c1f0adc -> ca90229)
This is an automated email from the ASF dual-hosted git repository. amansinha pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/drill.git. from c1f0adc DRILL-6242 Use java.time.Local{Date|Time|DateTime} for Drill Date, Time, Timestamp types. (#3) new 9a47d55 DRILL-6363: Upgrade jmockit and mockito libs new ca90229 DRILL-6272: Refactor dynamic UDFs and function initializer tests to generate needed binary and source jars at runtime The 2 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../test/java/org/apache/drill/test/TestTools.java | 2 +- exec/java-exec/pom.xml | 48 ++ .../drill/TestTpchDistributedConcurrent.java | 3 +- .../drill/exec/coord/zk/TestZookeeperClient.java | 2 +- .../exec/expr/fn/FunctionInitializerTest.java | 54 +-- .../rpc/user/TemporaryTablesAutomaticDropTest.java | 42 +- .../java/org/apache/drill/exec/sql/TestCTTAS.java | 315 - .../apache/drill/exec/udf/dynamic/JarBuilder.java | 96 .../udf/dynamic}/TestDynamicUDFSupport.java| 523 ++--- .../exec/vector/complex/TestEmptyPopulation.java | 3 - .../java-exec/src/test/resources/drill-udf/pom.xml | 92 .../drill/udf/dynamic/CustomAbsFunction.java | 39 +- .../drill/udf/dynamic/CustomLogFunction.java | 36 +- .../udf/dynamic/CustomLowerDummyFunction.java} | 44 +- .../drill/udf/dynamic/CustomLowerFunction.java | 42 +- .../drill/udf/dynamic/CustomLowerFunctionV2.java | 42 +- .../drill/udf/dynamic/CustomUpperFunction.java | 42 +- .../apache/drill/udf/dynamic/LowerFunction.java| 42 +- .../drill-udf/src/main/resources/drill-module.conf | 1 + .../test/resources/jars/DrillUDF-1.0-sources.jar | Bin 1892 -> 0 bytes .../src/test/resources/jars/DrillUDF-1.0.jar | Bin 3146 -> 0 bytes .../test/resources/jars/DrillUDF-2.0-sources.jar | Bin 1891 -> 0 bytes .../src/test/resources/jars/DrillUDF-2.0.jar | Bin 3142 -> 0 bytes .../jars/DrillUDF-overloading-1.0-sources.jar | Bin 3473 -> 0 bytes .../resources/jars/DrillUDF-overloading-1.0.jar| Bin 5779 -> 0 bytes .../resources/jars/DrillUDF_Copy-1.0-sources.jar | Bin 1892 -> 0 bytes .../src/test/resources/jars/DrillUDF_Copy-1.0.jar | Bin 3185 -> 0 bytes .../jars/DrillUDF_DupFunc-1.0-sources.jar | Bin 1888 -> 0 bytes .../test/resources/jars/DrillUDF_DupFunc-1.0.jar | Bin 3201 -> 0 bytes .../resources/jars/DrillUDF_Empty-1.0-sources.jar | Bin 536 -> 0 bytes .../src/test/resources/jars/DrillUDF_Empty-1.0.jar | Bin 1863 -> 0 bytes .../jars/DrillUDF_NoMarkerFile-1.0-sources.jar | Bin 1715 -> 0 bytes .../resources/jars/DrillUDF_NoMarkerFile-1.0.jar | Bin 3084 -> 0 bytes .../resources/jars/v2/DrillUDF-1.0-sources.jar | Bin 1899 -> 0 bytes .../src/test/resources/jars/v2/DrillUDF-1.0.jar| Bin 3215 -> 0 bytes pom.xml| 7 +- 36 files changed, 812 insertions(+), 663 deletions(-) create mode 100644 exec/java-exec/src/test/java/org/apache/drill/exec/udf/dynamic/JarBuilder.java rename exec/java-exec/src/test/java/org/apache/drill/{ => exec/udf/dynamic}/TestDynamicUDFSupport.java (65%) create mode 100644 exec/java-exec/src/test/resources/drill-udf/pom.xml copy contrib/gis/src/main/java/org/apache/drill/exec/expr/fn/impl/gis/STAsText.java => exec/java-exec/src/test/resources/drill-udf/src/main/java/org/apache/drill/udf/dynamic/CustomAbsFunction.java (61%) copy contrib/gis/src/main/java/org/apache/drill/exec/expr/fn/impl/gis/STAsText.java => exec/java-exec/src/test/resources/drill-udf/src/main/java/org/apache/drill/udf/dynamic/CustomLogFunction.java (63%) copy exec/java-exec/src/{main/java/org/apache/drill/exec/expr/fn/impl/conv/UTF8ConvertTo.java => test/resources/drill-udf/src/main/java/org/apache/drill/udf/dynamic/CustomLowerDummyFunction.java} (59%) copy contrib/gis/src/main/java/org/apache/drill/exec/expr/fn/impl/gis/STAsText.java => exec/java-exec/src/test/resources/drill-udf/src/main/java/org/apache/drill/udf/dynamic/CustomLowerFunction.java (61%) copy contrib/gis/src/main/java/org/apache/drill/exec/expr/fn/impl/gis/STAsText.java => exec/java-exec/src/test/resources/drill-udf/src/main/java/org/apache/drill/udf/dynamic/CustomLowerFunctionV2.java (61%) copy contrib/gis/src/main/java/org/apache/drill/exec/expr/fn/impl/gis/STAsText.java => exec/java-exec/src/test/resources/drill-udf/src/main/java/org/apache/drill/udf/dynamic/CustomUpperFunction.java (61%) copy contrib/gis/src/main/java/org/apache/drill/exec/expr/fn/impl/gis/STAsText.java => exec/java-exec/src/test/resources/drill-
[drill] 01/02: DRILL-6363: Upgrade jmockit and mockito libs
This is an automated email from the ASF dual-hosted git repository. amansinha pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/drill.git commit 9a47d555b00052115016fbe35cad5a4147d42ad5 Author: Arina Ielchiieva AuthorDate: Sat May 5 15:31:10 2018 +0300 DRILL-6363: Upgrade jmockit and mockito libs --- .../test/java/org/apache/drill/test/TestTools.java | 2 +- .../drill/TestTpchDistributedConcurrent.java | 3 +- .../drill/exec/coord/zk/TestZookeeperClient.java | 2 +- .../rpc/user/TemporaryTablesAutomaticDropTest.java | 42 ++- .../java/org/apache/drill/exec/sql/TestCTTAS.java | 315 +++-- .../exec/vector/complex/TestEmptyPopulation.java | 3 - pom.xml| 7 +- 7 files changed, 134 insertions(+), 240 deletions(-) diff --git a/common/src/test/java/org/apache/drill/test/TestTools.java b/common/src/test/java/org/apache/drill/test/TestTools.java index 8cf7ca7..2735c54 100644 --- a/common/src/test/java/org/apache/drill/test/TestTools.java +++ b/common/src/test/java/org/apache/drill/test/TestTools.java @@ -47,7 +47,7 @@ public class TestTools { .indexOf("-agentlib:jdwp") > 0; public static TestRule getTimeoutRule(int timeout) { -return IS_DEBUG ? new TestName() : new Timeout(timeout); +return IS_DEBUG ? new TestName() : Timeout.millis(timeout); } /** diff --git a/exec/java-exec/src/test/java/org/apache/drill/TestTpchDistributedConcurrent.java b/exec/java-exec/src/test/java/org/apache/drill/TestTpchDistributedConcurrent.java index 8991b8b..74b9a5c 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/TestTpchDistributedConcurrent.java +++ b/exec/java-exec/src/test/java/org/apache/drill/TestTpchDistributedConcurrent.java @@ -33,6 +33,7 @@ import org.apache.drill.exec.rpc.user.UserResultsListener; import org.apache.drill.test.BaseTestQuery; import org.apache.drill.test.QueryTestUtil; import org.junit.Rule; +import org.junit.Test; import org.junit.experimental.categories.Category; import org.junit.rules.TestRule; @@ -176,7 +177,7 @@ public class TestTpchDistributedConcurrent extends BaseTestQuery { } } - //@Test + @Test public void testConcurrentQueries() throws Exception { QueryTestUtil.testRunAndPrint(client, UserBitShared.QueryType.SQL, alterSession); diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/coord/zk/TestZookeeperClient.java b/exec/java-exec/src/test/java/org/apache/drill/exec/coord/zk/TestZookeeperClient.java index 87cf72d..e0e6c79 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/coord/zk/TestZookeeperClient.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/coord/zk/TestZookeeperClient.java @@ -125,7 +125,7 @@ public class TestZookeeperClient { Mockito .when(client.getCache().getCurrentData(absPath)) -.thenThrow(Exception.class); +.thenThrow(RuntimeException.class); client.hasPath(path); } diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/rpc/user/TemporaryTablesAutomaticDropTest.java b/exec/java-exec/src/test/java/org/apache/drill/exec/rpc/user/TemporaryTablesAutomaticDropTest.java index 5553519..5aef6f7 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/rpc/user/TemporaryTablesAutomaticDropTest.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/rpc/user/TemporaryTablesAutomaticDropTest.java @@ -19,39 +19,40 @@ package org.apache.drill.exec.rpc.user; import mockit.Mock; import mockit.MockUp; +import org.apache.drill.exec.store.StorageStrategy; import org.apache.drill.test.BaseTestQuery; import org.apache.drill.common.config.DrillConfig; -import org.apache.drill.exec.ExecConstants; import org.apache.drill.exec.store.StoragePluginRegistry; import org.apache.drill.exec.util.StoragePluginTestUtils; import org.apache.drill.test.DirTestWatcher; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.LocatedFileStatus; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.RemoteIterator; import org.junit.Before; import org.junit.Test; import java.io.File; -import java.util.Properties; import java.util.UUID; import static org.apache.drill.exec.util.StoragePluginTestUtils.DFS_TMP_SCHEMA; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; public class TemporaryTablesAutomaticDropTest extends BaseTestQuery { - private static final String session_id = "sessionId"; + private static final UUID SESSION_UUID = UUID.randomUUID(); @Before - public void setup() throws Exception { + public void setup() { new MockUp() { @Mock public UUID randomUUID() { -return UUID.nameUUIDFromBytes(session_id.getBytes()); +return SESSION_UUID; } }; - -Properties testConfigurations = cloneDefa
[drill] branch master updated: DRILL-6242 Use java.time.Local{Date|Time|DateTime} for Drill Date, Time, Timestamp types. (#3)
This is an automated email from the ASF dual-hosted git repository. amansinha pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/drill.git The following commit(s) were added to refs/heads/master by this push: new c1f0adc DRILL-6242 Use java.time.Local{Date|Time|DateTime} for Drill Date, Time, Timestamp types. (#3) c1f0adc is described below commit c1f0adc9276ab8314ce9c5dff2d9c92066c71530 Author: jiang-wu AuthorDate: Tue May 1 14:48:26 2018 -0700 DRILL-6242 Use java.time.Local{Date|Time|DateTime} for Drill Date, Time, Timestamp types. (#3) close apache/drill#1247 * DRILL-6242 - Use java.time.Local{Date|Time|DateTime} classes to hold values from corresponding Drill date, time, and timestamp types. Conflicts: exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/fn/ExtendedJsonOutput.java Fix merge conflicts and check style. --- .../exec/TestHiveDrillNativeParquetReader.java | 20 +- .../drill/exec/fn/hive/TestInbuiltHiveUDFs.java| 15 +- .../apache/drill/exec/hive/TestHiveStorage.java| 37 ++-- .../exec/store/jdbc/TestJdbcPluginWithDerbyIT.java | 12 +- .../exec/store/jdbc/TestJdbcPluginWithMySQLIT.java | 15 +- .../main/codegen/templates/CastDateVarChar.java| 8 +- .../main/codegen/templates/CastVarCharDate.java| 8 +- .../drill/exec/expr/fn/impl/DateTypeFunctions.java | 8 +- .../exec/planner/common/DrillValuesRelBase.java| 30 +-- .../drill/exec/store/bson/BsonRecordReader.java| 25 ++- .../org/apache/drill/exec/util/VectorUtil.java | 10 +- .../exec/vector/complex/fn/BasicJsonOutput.java| 36 ++-- .../exec/vector/complex/fn/ExtendedJsonOutput.java | 15 +- .../drill/exec/vector/complex/fn/JsonOutput.java | 8 +- .../drill/exec/vector/complex/fn/VectorOutput.java | 34 +-- .../java/org/apache/drill/TestFunctionsQuery.java | 22 +- .../java/org/apache/drill/TestStarQueries.java | 2 +- .../drill/exec/fn/impl/TestCastFunctions.java | 27 +-- .../drill/exec/fn/impl/TestDateAddFunctions.java | 29 +-- .../drill/exec/fn/impl/TestDateTruncFunctions.java | 139 ++-- .../drill/exec/fn/impl/TestNewDateFunctions.java | 10 +- .../exec/fn/interp/ExpressionInterpreterTest.java | 8 +- .../exec/physical/impl/TestConvertFunctions.java | 13 +- .../physical/impl/TestNestedDateTimeTimestamp.java | 234 + .../impl/limit/TestEarlyLimit0Optimization.java| 16 +- .../writer/TestCorruptParquetDateCorrection.java | 66 +++--- .../physical/impl/writer/TestParquetWriter.java| 32 +-- .../planner/logical/TestCaseNullableTypes.java | 5 +- .../drill/exec/store/avro/AvroFormatTest.java | 75 --- .../exec/store/bson/TestBsonRecordReader.java | 5 +- .../exec/store/parquet/TestParquetComplex.java | 52 ++--- .../store/parquet2/TestDrillParquetReader.java | 51 ++--- .../vector/complex/writer/TestExtendedTypes.java | 2 +- .../java/org/apache/drill/test/TestBuilder.java| 19 +- .../test/resources/baseline_nested_datetime.json | 13 ++ exec/java-exec/src/test/resources/datetime.parquet | Bin 0 -> 1217 bytes .../src/main/codegen/data/ValueVectorTypes.tdd | 6 +- .../src/main/codegen/includes/vv_imports.ftl | 8 +- .../codegen/templates/AbstractFieldReader.java | 2 +- .../main/codegen/templates/FixedValueVectors.java | 15 +- .../main/codegen/templates/HolderReaderImpl.java | 12 ++ .../src/main/codegen/templates/NullReader.java | 2 +- .../src/main/codegen/templates/UnionReader.java| 2 +- .../drill/exec/expr/fn/impl/DateUtility.java | 107 -- .../drill/exec/util/JsonStringArrayList.java | 1 + .../apache/drill/exec/util/JsonStringHashMap.java | 1 + .../drill/exec/util/SerializationModule.java | 75 +++ 47 files changed, 895 insertions(+), 437 deletions(-) diff --git a/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/TestHiveDrillNativeParquetReader.java b/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/TestHiveDrillNativeParquetReader.java index 23c67b5..fd9701c 100644 --- a/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/TestHiveDrillNativeParquetReader.java +++ b/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/TestHiveDrillNativeParquetReader.java @@ -17,14 +17,19 @@ */ package org.apache.drill.exec; +import static org.hamcrest.CoreMatchers.containsString; +import static org.junit.Assert.assertEquals; + +import java.math.BigDecimal; + import org.apache.drill.PlanTestBase; import org.apache.drill.categories.HiveStorageTest; import org.apache.drill.categories.SlowTest; import org.apache.drill.common.exceptions.UserRemoteException; +import org.apache.drill.exec.expr.fn.impl.DateUtility; import org.apache.drill.exec.hive.HiveTestBase; import org.apache.drill.exec.planner.physical.PlannerSettings; imp
[drill] 01/04: DRILL-6249: Adding more unit testing documentation.
This is an automated email from the ASF dual-hosted git repository. amansinha pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/drill.git commit dc8d0107cb6a4a31e1beaa9c98d6e51cf7486a67 Author: Timothy Farkas AuthorDate: Thu Mar 15 14:27:55 2018 -0700 DRILL-6249: Adding more unit testing documentation. close apache/drill#1251 --- README.md | 5 +- .../java/org/apache/drill/test/DirTestWatcher.java | 14 +- .../test/java/org/apache/drill/test/TestTools.java | 7 +- docs/dev/BaseTestQuery.md | 4 + docs/dev/ClusterTest.md| 4 + docs/dev/DevDocs.md| 6 +- docs/dev/GeneratedCode.md | 51 +++ docs/dev/InstantiatingComponents.md| 107 ++ docs/dev/LicenseHeaders.md | 34 + docs/dev/PhysicalOpUnitTestBase.md | 31 docs/dev/RowSetFramework.md| 112 +++ docs/dev/TempDirectories.md| 17 +++ docs/dev/TestDataSets.md | 159 + docs/dev/Testing.md| 134 +++-- .../drill/exec/ops/ExchangeFragmentContext.java| 3 + .../drill/exec/ops/ExecutorFragmentContext.java| 4 + .../org/apache/drill/exec/ops/FragmentContext.java | 10 +- .../apache/drill/exec/ops/FragmentContextImpl.java | 28 +++- .../apache/drill/exec/ops/RootFragmentContext.java | 3 + .../org/apache/drill/exec/record/BatchSchema.java | 4 + .../org/apache/drill/TestDynamicUDFSupport.java| 2 +- .../exec/expr/fn/FunctionInitializerTest.java | 2 +- .../physical/impl/window/GenerateTestData.java | 4 +- .../store/dfs/TestSchemaNotFoundException.java | 8 +- .../org/apache/drill/test/BaseDirTestWatcher.java | 16 ++- .../java/org/apache/drill/test/BaseTestQuery.java | 4 + .../java/org/apache/drill/test/ExampleTest.java| 24 .../apache/drill/test/rowSet/RowSetBuilder.java| 7 + .../apache/drill/test/rowSet/RowSetUtilities.java | 6 + .../drill/test/rowSet/file/JsonFileBuilder.java| 26 30 files changed, 794 insertions(+), 42 deletions(-) diff --git a/README.md b/README.md index a30cf07..f2b5b9d 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,10 @@ Please see the [Apache Drill Website](http://drill.apache.org/) or the [Apache D ## Join the community! -Apache Drill is an Apache Foundation project and is seeking all types of contributions. Please say hello on the Apache Drill mailing list or join our Google Hangouts for more information. (More information can be found at the Apache Drill website). +Apache Drill is an Apache Foundation project and is seeking all types of contributions. +Please say hello on the [Apache Drill mailing list](http://drill.apache.org/mailinglists/) +or join our [Google Hangouts](http://drill.apache.org/community-resources/) for more information. +(More information can be found at the [Apache Drill website](http://drill.apache.org/)). ## Export Control This distribution includes cryptographic software. The country in which you currently reside may have restrictions on the import, possession, use, and/or re-export to another country, of encryption software. BEFORE using any encryption software, please check your country's laws, regulations and policies concerning the import, possession, or use, and re-export of encryption software, to see if this is permitted. See <http://www.wassenaar.org/> for more information. diff --git a/common/src/test/java/org/apache/drill/test/DirTestWatcher.java b/common/src/test/java/org/apache/drill/test/DirTestWatcher.java index 271ac94..6bdc951 100644 --- a/common/src/test/java/org/apache/drill/test/DirTestWatcher.java +++ b/common/src/test/java/org/apache/drill/test/DirTestWatcher.java @@ -31,10 +31,10 @@ import java.nio.file.Paths; * * * - * A {@link DirTestWatcher} is added to a test by declaring it as a JUnit {@link org.junit.Rule}. A {@link org.junit.Rule} is - * a piece of code that is run before and after every JUnit test marked with the {@link org.junit.Test} annotation. When the + * A {@link DirTestWatcher} is added to a test by declaring it as a JUnit {@link org.junit.Rule}. A {@link org.junit.Rule Rule} is + * a piece of code that is run before and after every JUnit test marked with the {@link org.junit.Test Test} annotation. When the * {@link DirTestWatcher} is added to a test class the {@link DirTestWatcher} will create a temp directory before each of your - * {@link org.junit.Test}s and optionally delete the temp directory after each of your {@link org.junit.Test}s. The temp directory + * {@link org.junit.Test Test}s and optionally delete the temp directory after each of your {@link org.junit.Test Test}s. The base tem
[drill] 03/04: DRILL-6364: Handle Cluster Info in WebUI when existing/new bits restart
This is an automated email from the ASF dual-hosted git repository. amansinha pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/drill.git commit 11036c556f6445bcf1418b6512ba71b44a0a6031 Author: Kunal Khatua AuthorDate: Fri Apr 27 16:27:45 2018 -0700 DRILL-6364: Handle Cluster Info in WebUI when existing/new bits restart As a follow up to DRILL-6289, the following improvements have been done: 1. When loading the page for the first time, the WebUI enables the shutdown button without actually checking the state of the Drillbits. The ideal behaviour should be to disable the button till the state is verified. [Done] If a Drillbit is confirmed down (i.e. not in `/state` response), it is marked as OFFLINE and button is disabled. 2. When shutting down the current Drillbit, the WebUI no more has access to the cluster state. The ideal behaviour here should be to fetch the state from any of the other Drillbits to update the status. [Done] With the current Drillbit down, the other bits are requested for cluster state info and update accordingly. 3. When a new, previously unseen Drillbit comes up, the WebUI will never render it because the table is statically generated during the first page load. The idea behaviour should be to append to the table on discovery of a new node. [Done] The new Drillbit info is injected and a prompt appears to refresh the page to re-populate any missing info. This also works with feature (2) mentioned above. The only Java code change was to have the state response carry the address and http-port as a tuple, instead of the user-port (which is never used). Updates based on review 1. Added descriptive comments to functions 2. Handled possible race condition from multiple httpRequests for cluster state 3. Eliminated unused stateKey variable 4. Best practice of using local (let) instead of global (var) objects, and substituting currentRow variable instead of the jQuery search. Additional changes based on review 1. Random selection of Drillbits to query for state when primary Drillbit is down (limited to 3; with a timeout of 3 sec) 2. Indicate when an Offline Drillbit is de-registered, versus just Offline as per Zookeeper 3. Hide shutdown column when Authentication is enabled, but the user is NOT an Admin. When the column is visible, remote bits are disabled because 4. Metrics will be shown all the time (except HTTPS), because the information is available and a non-admin user would anyway not have actionable capabilities 5. Basic clean up on unused variables Hide shutdown buttons for HTTPS scenarios Fixed check for 'https' Since, `location.protocol` returns a trailing ':' as well Handle metrics lookup on local Drillbit for HTTPS If the webpage is accessed via IP, the certificate is tied to the IP and not the FQDN, which is what is used for fetching the metrics. close apache/drill#1241 --- .../apache/drill/exec/server/rest/DrillRoot.java | 3 +- exec/java-exec/src/main/resources/rest/index.ftl | 319 +++-- 2 files changed, 239 insertions(+), 83 deletions(-) diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/DrillRoot.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/DrillRoot.java index 50d99b0..0a48801 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/DrillRoot.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/DrillRoot.java @@ -88,7 +88,7 @@ public class DrillRoot { Collection drillbits = getClusterInfoJSON().getDrillbits(); Map drillStatusMap = new HashMap(); for (DrillbitInfo drillbit : drillbits) { - drillStatusMap.put(drillbit.getAddress() + "-" + drillbit.getUserPort(), drillbit.getState()); + drillStatusMap.put(drillbit.getAddress() + "-" + drillbit.getHttpPort(), drillbit.getState()); } return setResponse(drillStatusMap); } @@ -239,6 +239,7 @@ public class DrillRoot { private Response shutdown(String resp) throws Exception { Map shutdownInfo = new HashMap(); new Thread(new Runnable() { +@Override public void run() { try { drillbit.close(); diff --git a/exec/java-exec/src/main/resources/rest/index.ftl b/exec/java-exec/src/main/resources/rest/index.ftl index 5259e44..11d1dd2 100644 --- a/exec/java-exec/src/main/resources/rest/index.ftl +++ b/exec/java-exec/src/main/resources/rest/index.ftl @@ -54,9 +54,11 @@ - Drillbits ${model.getDrillbits()?size} + Drillbits ${model.getDrillbits()?size} + + - + # @@ -71,9 +73,7 @@ Version Status
[drill] branch master updated (8a1a7c5 -> 16659ab)
This is an automated email from the ASF dual-hosted git repository. amansinha pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/drill.git. from 8a1a7c5 DRILL-6386: Remove unused imports and star imports. new dc8d010 DRILL-6249: Adding more unit testing documentation. new 76728e5 DRILL-6321: Customize Drill's conformance. Allow support to APPLY keywords new 11036c5 DRILL-6364: Handle Cluster Info in WebUI when existing/new bits restart new 16659ab DRILL-6401: Precision for decimal data types may be lost for the case when cast with literal is used The 4 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: README.md | 5 +- .../java/org/apache/drill/test/DirTestWatcher.java | 14 +- .../test/java/org/apache/drill/test/TestTools.java | 7 +- docs/dev/BaseTestQuery.md | 4 + docs/dev/ClusterTest.md| 4 + docs/dev/DevDocs.md| 6 +- docs/dev/GeneratedCode.md | 51 docs/dev/InstantiatingComponents.md| 107 +++ docs/dev/LicenseHeaders.md | 34 +++ docs/dev/PhysicalOpUnitTestBase.md | 31 ++ docs/dev/RowSetFramework.md| 112 docs/dev/TempDirectories.md| 17 ++ docs/dev/TestDataSets.md | 159 ++ docs/dev/Testing.md| 134 - .../drill/exec/ops/ExchangeFragmentContext.java| 3 + .../drill/exec/ops/ExecutorFragmentContext.java| 4 + .../org/apache/drill/exec/ops/FragmentContext.java | 10 +- .../apache/drill/exec/ops/FragmentContextImpl.java | 28 +- .../apache/drill/exec/ops/RootFragmentContext.java | 3 + .../drill/exec/planner/logical/DrillOptiq.java | 4 +- .../sql/{FixedRange.java => DrillConformance.java} | 37 ++- .../drill/exec/planner/sql/DrillParserConfig.java | 4 +- .../drill/exec/planner/sql/SqlConverter.java | 34 +-- .../org/apache/drill/exec/record/BatchSchema.java | 4 + .../apache/drill/exec/server/rest/DrillRoot.java | 3 +- .../exec/store/parquet/ParquetFilterBuilder.java | 4 +- exec/java-exec/src/main/resources/rest/index.ftl | 319 +++-- .../org/apache/drill/TestDynamicUDFSupport.java| 2 +- .../exec/expr/fn/FunctionInitializerTest.java | 2 +- .../physical/impl/window/GenerateTestData.java | 4 +- .../sql/TestConformance.java} | 35 +-- .../store/dfs/TestSchemaNotFoundException.java | 8 +- .../org/apache/drill/test/BaseDirTestWatcher.java | 16 +- .../java/org/apache/drill/test/BaseTestQuery.java | 4 + .../java/org/apache/drill/test/ExampleTest.java| 24 ++ .../apache/drill/test/rowSet/RowSetBuilder.java| 7 + .../apache/drill/test/rowSet/RowSetUtilities.java | 6 + .../drill/test/rowSet/file/JsonFileBuilder.java| 26 ++ .../drill/common/expression/ValueExpressions.java | 16 +- 39 files changed, 1105 insertions(+), 187 deletions(-) create mode 100644 docs/dev/BaseTestQuery.md create mode 100644 docs/dev/ClusterTest.md create mode 100644 docs/dev/GeneratedCode.md create mode 100644 docs/dev/InstantiatingComponents.md create mode 100644 docs/dev/LicenseHeaders.md create mode 100644 docs/dev/PhysicalOpUnitTestBase.md create mode 100644 docs/dev/RowSetFramework.md create mode 100644 docs/dev/TempDirectories.md create mode 100644 docs/dev/TestDataSets.md copy exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/{FixedRange.java => DrillConformance.java} (54%) copy exec/java-exec/src/test/java/org/apache/drill/{TestImplicitCasting.java => exec/sql/TestConformance.java} (60%) -- To stop receiving notification emails like this one, please contact amansi...@apache.org.
[drill] 02/04: DRILL-6321: Customize Drill's conformance. Allow support to APPLY keywords
This is an automated email from the ASF dual-hosted git repository. amansinha pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/drill.git commit 76728e5ecd30418862f4bc1d32bc315042851eb1 Author: chunhui-shi AuthorDate: Fri Feb 2 10:03:38 2018 -0800 DRILL-6321: Customize Drill's conformance. Allow support to APPLY keywords close apache/drill#1224 Conflicts: exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/SqlConverter.java Fix checkstyle --- .../drill/exec/planner/sql/DrillConformance.java | 43 .../drill/exec/planner/sql/DrillParserConfig.java | 4 +- .../drill/exec/planner/sql/SqlConverter.java | 34 .../org/apache/drill/exec/sql/TestConformance.java | 46 ++ 4 files changed, 109 insertions(+), 18 deletions(-) diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/DrillConformance.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/DrillConformance.java new file mode 100644 index 000..e6efeb9 --- /dev/null +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/DrillConformance.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.planner.sql; + +import org.apache.calcite.sql.validate.SqlConformanceEnum; +import org.apache.calcite.sql.validate.SqlDelegatingConformance; + +/** + * Drill's SQL conformance is SqlConformanceEnum.DEFAULT except for method isApplyAllowed(). + * Since Drill is going to allow OUTER APPLY and CROSS APPLY to allow each row from left child of Join + * to join with output of right side (sub-query or table function that will be invoked for each row). + * Refer to DRILL-5999 for more information. + */ +public class DrillConformance extends SqlDelegatingConformance { + + public DrillConformance() { +super(SqlConformanceEnum.DEFAULT); + } + + public DrillConformance(SqlConformanceEnum flavor) { +super(flavor); + } + + @Override + public boolean isApplyAllowed() { +return true; + } +} diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/DrillParserConfig.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/DrillParserConfig.java index 1f67c5e..575ad86 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/DrillParserConfig.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/DrillParserConfig.java @@ -22,7 +22,6 @@ import org.apache.calcite.avatica.util.Quoting; import org.apache.calcite.sql.parser.SqlParser; import org.apache.calcite.sql.parser.SqlParserImplFactory; import org.apache.calcite.sql.validate.SqlConformance; -import org.apache.calcite.sql.validate.SqlConformanceEnum; import org.apache.drill.exec.planner.physical.PlannerSettings; import org.apache.drill.exec.planner.sql.parser.impl.DrillParserWithCompoundIdConverter; @@ -30,6 +29,7 @@ public class DrillParserConfig implements SqlParser.Config { private final long identifierMaxLength; private final Quoting quotingIdentifiers; + public final static SqlConformance DRILL_CONFORMANCE = new DrillConformance(); public DrillParserConfig(PlannerSettings settings) { identifierMaxLength = settings.getIdentifierMaxLength(); @@ -63,7 +63,7 @@ public class DrillParserConfig implements SqlParser.Config { @Override public SqlConformance conformance() { -return SqlConformanceEnum.DEFAULT; +return DRILL_CONFORMANCE; } @Override diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/SqlConverter.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/SqlConverter.java index 7342256..c7e1b25 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/SqlConverter.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/SqlConverter.java @@ -58,7 +58,7 @@ import org.apache.calcite.sql.parser.SqlParser; import org.apache.calcite.sql.parser.SqlParserPos; import org.apache.calcite.sql.type.SqlTypeName; import org.apache.calcite.sql.util.Ch
[drill] 04/04: DRILL-6401: Precision for decimal data types may be lost for the case when cast with literal is used
This is an automated email from the ASF dual-hosted git repository. amansinha pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/drill.git commit 16659ab30fd9fca5af39edd22aaab36409a50930 Author: Volodymyr Vysotskyi AuthorDate: Thu May 10 17:06:28 2018 +0300 DRILL-6401: Precision for decimal data types may be lost for the case when cast with literal is used close apache/drill#1254 --- .../apache/drill/exec/planner/logical/DrillOptiq.java| 4 +++- .../drill/exec/store/parquet/ParquetFilterBuilder.java | 4 +++- .../apache/drill/common/expression/ValueExpressions.java | 16 ++-- 3 files changed, 16 insertions(+), 8 deletions(-) diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillOptiq.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillOptiq.java index fd791dd..2cd6068 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillOptiq.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillOptiq.java @@ -550,7 +550,9 @@ public class DrillOptiq { literal.getType().getScale(), literal.getType().getPrecision())); } - return ValueExpressions.getVarDecimal((BigDecimal) literal.getValue()); + return ValueExpressions.getVarDecimal((BigDecimal) literal.getValue(), + literal.getType().getPrecision(), + literal.getType().getScale()); } double dbl = ((BigDecimal) literal.getValue()).doubleValue(); logger.warn("Converting exact decimal into approximate decimal.\n" + diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetFilterBuilder.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetFilterBuilder.java index 52c4723..e55425e 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetFilterBuilder.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetFilterBuilder.java @@ -181,7 +181,9 @@ public class ParquetFilterBuilder extends AbstractExprVisitor
[3/3] drill git commit: DRILL-6234: Improved documentation for VariableWidthVector mutators, and added simple unit tests demonstrating mutator behavior.
DRILL-6234: Improved documentation for VariableWidthVector mutators, and added simple unit tests demonstrating mutator behavior. close apache/drill#1164 Project: http://git-wip-us.apache.org/repos/asf/drill/repo Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/9a6cb59b Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/9a6cb59b Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/9a6cb59b Branch: refs/heads/master Commit: 9a6cb59b9b7a5b127e5f60309ce2f506ede9652a Parents: f5b8223 Author: Timothy Farkas Authored: Tue Mar 13 17:24:28 2018 -0700 Committer: Aman Sinha Committed: Fri Mar 30 22:47:31 2018 -0700 -- exec/vector/pom.xml | 7 +- .../templates/VariableLengthVectors.java| 61 .../apache/drill/exec/vector/ValueVector.java | 3 +- .../exec/vector/VariableLengthVectorTest.java | 141 +++ 4 files changed, 210 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/drill/blob/9a6cb59b/exec/vector/pom.xml -- diff --git a/exec/vector/pom.xml b/exec/vector/pom.xml index 0184305..21e138d 100644 --- a/exec/vector/pom.xml +++ b/exec/vector/pom.xml @@ -65,7 +65,12 @@ 0.7.1 - + + org.apache.drill + drill-common + ${project.version} + tests + http://git-wip-us.apache.org/repos/asf/drill/blob/9a6cb59b/exec/vector/src/main/codegen/templates/VariableLengthVectors.java -- diff --git a/exec/vector/src/main/codegen/templates/VariableLengthVectors.java b/exec/vector/src/main/codegen/templates/VariableLengthVectors.java index 516eb52..ab995cd 100644 --- a/exec/vector/src/main/codegen/templates/VariableLengthVectors.java +++ b/exec/vector/src/main/codegen/templates/VariableLengthVectors.java @@ -512,6 +512,8 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements V } /** + * Overview + * * Mutable${minor.class} implements a vector of variable width values. Elements in the vector * are accessed by position from the logical start of the vector. A fixed width offsetVector * is used to convert an element's position to it's offset from the start of the (0-based) @@ -520,6 +522,46 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements V * The equivalent Java primitive is '${minor.javaType!type.javaType}' * * NB: this class is automatically generated from ValueVectorTypes.tdd using FreeMarker. + * + * Contract + * + * + * + * Supported Writes: {@link VariableWidthVector}s do not support random writes. In contrast {@link org.apache.drill.exec.vector.FixedWidthVector}s do + * allow random writes but special care is needed. + * + * + * Writing Values: All set methods must be called with a consecutive sequence of indices. With a few exceptions: + * + * You can update the last index you just set. + * You can reset a previous index (call it Idx), but you must assume all the data after Idx is corrupt. Also + * note that the memory consumed by data that came after Idx is not released. + * + * + * + * Setting Value Count: Vectors aren't explicitly aware of how many values they contain. So you must keep track of the + * number of values you've written to the vector and once you are done writing to the vector you must call {@link Mutator#setValueCount(int)}. + * It is possible to trim the vector by setting the value count to be less than the number of values currently contained in the vector. Note the extra memory consumed in + * the data buffer is not freed when this is done. + * + * + * Memory Allocation: When setting a value at an index you must do one of the following to ensure you do not get an {@link IndexOutOfBoundsException}. + * + * + * Allocate the exact amount of memory you need when using the {@link Mutator#set(int, byte[])} methods. If you do not + * manually allocate sufficient memory an {@link IndexOutOfBoundsException} can be thrown when the data buffer runs out of space. + * + * + * Or you can use the {@link Mutator#setSafe(int, byte[])} methods, which will automatically grow your data buffer to + * fit your data. + * + * + * + * + * Immutability: Once a vector has been populated with data and {@link #setValueCount(int)} has been called, it should be considered immutable. + * + * + * */ public final class Mutator extends BaseValueVector.BaseMutator implemen
[1/3] drill git commit: DRILL-6259: Support parquet filter push down for complex types
Repository: drill Updated Branches: refs/heads/master a264e7feb -> 9a6cb59b9 DRILL-6259: Support parquet filter push down for complex types close apache/drill#1173 Project: http://git-wip-us.apache.org/repos/asf/drill/repo Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/4ee5625d Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/4ee5625d Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/4ee5625d Branch: refs/heads/master Commit: 4ee5625d57bd73d3d82b45f687a8574ea6660f8e Parents: a264e7f Author: Arina Ielchiieva Authored: Tue Mar 13 19:54:25 2018 +0200 Committer: Aman Sinha Committed: Fri Mar 30 22:45:33 2018 -0700 -- .../sig/ConstantExpressionIdentifier.java | 6 + .../exec/expr/ExpressionTreeMaterializer.java | 3 +- .../exec/expr/stat/ParquetIsPredicates.java | 23 .../exec/expr/stat/RangeExprEvaluator.java | 27 ++-- .../drill/exec/expr/stat/TypedFieldExpr.java| 63 -- .../exec/planner/common/DrillRelOptUtil.java| 14 +-- .../logical/DrillPushFilterPastProjectRule.java | 14 ++- .../store/parquet/ParquetFilterBuilder.java | 80 .../store/parquet/ParquetPushDownFilter.java| 13 +- .../parquet/stat/ParquetMetaStatCollector.java | 29 ++--- ...estParquetFilterPushDownForComplexTypes.java | 124 +++ .../resources/parquet/users/users_1.parquet | Bin 0 -> 657 bytes .../resources/parquet/users/users_2.parquet | Bin 0 -> 641 bytes .../resources/parquet/users/users_3.parquet | Bin 0 -> 588 bytes .../resources/parquet/users/users_4.parquet | Bin 0 -> 653 bytes .../resources/parquet/users/users_5.parquet | Bin 0 -> 583 bytes .../resources/parquet/users/users_6.parquet | Bin 0 -> 627 bytes .../resources/parquet/users/users_7.parquet | Bin 0 -> 662 bytes .../drill/common/expression/SchemaPath.java | 29 +++-- .../drill/common/expression/TypedFieldExpr.java | 60 + .../visitors/AbstractExprVisitor.java | 6 +- .../expression/visitors/AggregateChecker.java | 5 + .../expression/visitors/ConstantChecker.java| 5 + .../common/expression/visitors/ExprVisitor.java | 54 .../visitors/ExpressionValidator.java | 5 + 25 files changed, 365 insertions(+), 195 deletions(-) -- http://git-wip-us.apache.org/repos/asf/drill/blob/4ee5625d/exec/java-exec/src/main/java/org/apache/drill/exec/compile/sig/ConstantExpressionIdentifier.java -- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/compile/sig/ConstantExpressionIdentifier.java b/exec/java-exec/src/main/java/org/apache/drill/exec/compile/sig/ConstantExpressionIdentifier.java index 1e71773..1a0b7d5 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/compile/sig/ConstantExpressionIdentifier.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/compile/sig/ConstantExpressionIdentifier.java @@ -31,6 +31,7 @@ import org.apache.drill.common.expression.IfExpression; import org.apache.drill.common.expression.LogicalExpression; import org.apache.drill.common.expression.NullExpression; import org.apache.drill.common.expression.SchemaPath; +import org.apache.drill.common.expression.TypedFieldExpr; import org.apache.drill.common.expression.TypedNullConstant; import org.apache.drill.common.expression.ValueExpressions; import org.apache.drill.common.expression.ValueExpressions.BooleanExpression; @@ -231,4 +232,9 @@ public class ConstantExpressionIdentifier implements ExprVisitor value) throws RuntimeException { return false; } + + @Override + public Boolean visitTypedFieldExpr(TypedFieldExpr e, IdentityHashMap value) throws RuntimeException { +return false; + } } http://git-wip-us.apache.org/repos/asf/drill/blob/4ee5625d/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ExpressionTreeMaterializer.java -- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ExpressionTreeMaterializer.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ExpressionTreeMaterializer.java index 23df262..f1b50c9 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ExpressionTreeMaterializer.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ExpressionTreeMaterializer.java @@ -44,6 +44,7 @@ import org.apache.drill.common.expression.IfExpression.IfCondition; import org.apache.drill.common.expression.LogicalExpression; import org.apache.drill.common.expression.NullExpression; import org.apache.drill.common.expression.SchemaPath; +import org.apache.drill.common.expression.TypedFieldExpr; import org.apache.drill.common.expression.TypedNullConstant; import org.apache.drill.common.expression.ValueExpressions
[2/3] drill git commit: DRILL-6103: lsb_release: command not found
DRILL-6103: lsb_release: command not found close apache/drill#1191 Thanks to Sanel Zukan for providing a small patch that checks for /etc/fedora-release path. This is more common, than lsb_release command on Linux distros. Project: http://git-wip-us.apache.org/repos/asf/drill/repo Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/f5b82239 Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/f5b82239 Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/f5b82239 Branch: refs/heads/master Commit: f5b82239ad823a972e32ea732237a2e8c4346db3 Parents: 4ee5625 Author: Kunal Khatua Authored: Mon Mar 26 23:30:31 2018 -0700 Committer: Aman Sinha Committed: Fri Mar 30 22:46:36 2018 -0700 -- distribution/src/resources/drill-config.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/drill/blob/f5b82239/distribution/src/resources/drill-config.sh -- diff --git a/distribution/src/resources/drill-config.sh b/distribution/src/resources/drill-config.sh index e3eaa64..c2f3d70 100644 --- a/distribution/src/resources/drill-config.sh +++ b/distribution/src/resources/drill-config.sh @@ -421,8 +421,7 @@ CP="$CP:$DRILL_HOME/jars/classb/*" if [[ "$OSTYPE" == "linux-gnu" ]]; then # Linux # check for Fedora. netty-tcnative has a Fedora variant - linuxvariant=$(lsb_release -i | cut -d: -f2 | sed s/'^\t'//) - if [[ "$linuxvariant" == "Fedora" ]]; then + if [[ -f "/etc/fedora-release" ]]; then CP="$CP:$DRILL_HOME/jars/3rdparty/fedora/*" else CP="$CP:$DRILL_HOME/jars/3rdparty/linux/*"
[7/7] drill git commit: DRILL-6125: Fix possible memory leak when query is cancelled or finished.
DRILL-6125: Fix possible memory leak when query is cancelled or finished. close apache/drill#1105 Project: http://git-wip-us.apache.org/repos/asf/drill/repo Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/a264e7fe Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/a264e7fe Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/a264e7fe Branch: refs/heads/master Commit: a264e7feb1d02ffd5762bb1f652ea22d17aa5243 Parents: 03b245e Author: Timothy Farkas Authored: Tue Jan 30 15:55:41 2018 -0800 Committer: Aman Sinha Committed: Thu Mar 29 23:24:09 2018 -0700 -- .../drill/exec/physical/impl/RootExec.java | 23 ++- .../PartitionSenderRootExec.java| 32 ++-- .../exec/work/fragment/FragmentExecutor.java| 179 ++- 3 files changed, 160 insertions(+), 74 deletions(-) -- http://git-wip-us.apache.org/repos/asf/drill/blob/a264e7fe/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/RootExec.java -- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/RootExec.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/RootExec.java index 5e366fb..ddeb3e8 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/RootExec.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/RootExec.java @@ -20,19 +20,28 @@ package org.apache.drill.exec.physical.impl; import org.apache.drill.exec.proto.ExecProtos.FragmentHandle; /** - * A FragmentRoot is a node which is the last processing node in a query plan. FragmentTerminals include Exchange - * output nodes and storage nodes. They are there driving force behind the completion of a query. + * Functionality + * + * A FragmentRoot is a node which is the last processing node in a query plan. FragmentTerminals include Exchange + * output nodes and storage nodes. They are there driving force behind the completion of a query. + * + * Assumptions + * + * All implementations of {@link RootExec} assume that all their methods are called by the same thread. + * */ public interface RootExec extends AutoCloseable { /** * Do the next batch of work. - * @return Whether or not additional batches of work are necessary. False means that this fragment is done. + * @return Whether or not additional batches of work are necessary. False means that this fragment is done. */ - public boolean next(); + boolean next(); /** - * Inform sender that receiving fragment is finished and doesn't need any more data - * @param handle + * Inform sender that receiving fragment is finished and doesn't need any more data. This can be called multiple + * times (once for each downstream receiver). If all receivers are finished then a subsequent call to {@link #next()} + * will return false. + * @param handle The handle pointing to the downstream receiver that does not need anymore data. */ - public void receivingFragmentFinished(FragmentHandle handle); + void receivingFragmentFinished(FragmentHandle handle); } http://git-wip-us.apache.org/repos/asf/drill/blob/a264e7fe/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/partitionsender/PartitionSenderRootExec.java -- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/partitionsender/PartitionSenderRootExec.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/partitionsender/PartitionSenderRootExec.java index 25be50a..7e76238 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/partitionsender/PartitionSenderRootExec.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/partitionsender/PartitionSenderRootExec.java @@ -65,14 +65,13 @@ public class PartitionSenderRootExec extends BaseRootExec { private PartitionerDecorator partitioner; private ExchangeFragmentContext context; - private boolean ok = true; private final int outGoingBatchCount; private final HashPartitionSender popConfig; private final double cost; private final AtomicIntegerArray remainingReceivers; private final AtomicInteger remaingReceiverCount; - private volatile boolean done = false; + private boolean done = false; private boolean first = true; private boolean closeIncoming; @@ -146,11 +145,8 @@ public class PartitionSenderRootExec extends BaseRootExec { @Override public boolean innerNext() { -if (!ok) { - return false; -} - IterOutcome out; + if (!done) { out = next(incoming); } else { @@ -252,13 +248,11 @@ public class PartitionSenderRootExec extends BaseRootExec { startIndex, endIndex); } - s
[1/7] drill git commit: DRILL-6300: Refresh protobuf C++ source files
Repository: drill Updated Branches: refs/heads/master f1cfaaf3a -> a264e7feb DRILL-6300: Refresh protobuf C++ source files close apache/drill#1194 Project: http://git-wip-us.apache.org/repos/asf/drill/repo Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/bfc86f17 Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/bfc86f17 Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/bfc86f17 Branch: refs/heads/master Commit: bfc86f1719bc348a74886e8f29e49bed6fdfe8df Parents: f1cfaaf Author: Volodymyr Vysotskyi Authored: Thu Mar 29 18:51:08 2018 +0300 Committer: Aman Sinha Committed: Thu Mar 29 23:17:18 2018 -0700 -- contrib/native/client/readme.linux | 12 +- .../client/src/protobuf/Coordination.pb.cc | 98 +++- .../client/src/protobuf/Coordination.pb.h | 89 ++- .../client/src/protobuf/UserBitShared.pb.cc | 239 ++- .../client/src/protobuf/UserBitShared.pb.h | 24 +- 5 files changed, 326 insertions(+), 136 deletions(-) -- http://git-wip-us.apache.org/repos/asf/drill/blob/bfc86f17/contrib/native/client/readme.linux -- diff --git a/contrib/native/client/readme.linux b/contrib/native/client/readme.linux index 34c791b..92fe519 100644 --- a/contrib/native/client/readme.linux +++ b/contrib/native/client/readme.linux @@ -47,7 +47,7 @@ Install Prerequisites $> sudo yum install cppunit-devel 3.2) Download Zookeeper from : -- http://apache.mirror.quintex.com/zookeeper/zookeeper-3.4.6/ +- https://archive.apache.org/dist/zookeeper/zookeeper-3.4.6/ - untar and then follow instructions in ZOOKEEPER_DIR/src/c/README to build and install the client libs 3.3) run autoreconf @@ -57,7 +57,7 @@ Install Prerequisites $> ./configure --enable-debug --with-syncapi --enable-static --enable-shared $> make && sudo make install -4) Install boost. The minumim version required is 1.53, which will probably have to be built from source +4) Install boost. The minimum version required is 1.53, which will probably have to be built from source # Remove any previous boost $> sudo yum -y erase boost @@ -69,7 +69,9 @@ Install Prerequisites #install the binary rpms #(Note: the "rpm" utility does not clean up old versions very well.) $> sudo yum -y install ~/rpmbuild/RPMS/x86_64/* - +OR +# Build boost for Drill using instruction from readme.boost. +# Uncomment set(Boost_NAMESPACE drill_boost) line in client/CMakeLists.txt OR Download and build using boost build. See this link for how to build: http://www.boost.org/doc/libs/1_53_0/more/getting_started/unix-variants.html#prepare-to-use-a-boost-library-binary @@ -117,12 +119,12 @@ Build drill client Test Run query submitter from the command line -$> querySubmitter query='select * from dfs.`/Users/pchandra/work/data/tpc-h/customer.parquet`' type=sql connectStr=local=10.250.0.146:31010 api=async logLevel=trace user=yourUserName password=yourPassWord +$> querySubmitter query='SELECT * FROM cp.`employee.json` LIMIT 20' type=sql connectStr=local=10.250.0.146:31010 api=async logLevel=trace user=yourUserName password=yourPassWord Valgrind Examples to run Valgrind and see the log in Valkyrie -$> valgrind --leak-check=yes --xml=yes --xml-file=qs-vg-log-a.xml querySubmitter query='select LINEITEM from dfs.`/Users/pchandra/work/data/tpc-h/customer.parquet`' type=sql connectStr=local=10.250.0.146:31010 api=async logLevel=trace +$> valgrind --leak-check=yes --xml=yes --xml-file=qs-vg-log-a.xml querySubmitter query='SELECT * FROM cp.`employee.json` LIMIT 20' type=sql connectStr=local=10.250.0.146:31010 api=async logLevel=trace $> valkyrie -l qs-vg-log-a.xml http://git-wip-us.apache.org/repos/asf/drill/blob/bfc86f17/contrib/native/client/src/protobuf/Coordination.pb.cc -- diff --git a/contrib/native/client/src/protobuf/Coordination.pb.cc b/contrib/native/client/src/protobuf/Coordination.pb.cc index 923481b..14b3103 100644 --- a/contrib/native/client/src/protobuf/Coordination.pb.cc +++ b/contrib/native/client/src/protobuf/Coordination.pb.cc @@ -23,6 +23,7 @@ namespace { const ::google::protobuf::Descriptor* DrillbitEndpoint_descriptor_ = NULL; const ::google::protobuf::internal::GeneratedMessageReflection* DrillbitEndpoint_reflection_ = NULL; +const ::google::protobuf::EnumDescriptor* DrillbitEndpoint_State_descriptor_ = NULL; const ::google::protobuf::Descriptor* DrillServiceInstance_descriptor_ = NULL; const ::google::protobuf::internal::GeneratedMessageReflection* DrillServiceInstance_reflection_ = NULL; @@ -40,13 +41,14 @@ void protobuf_AssignDesc_Coordination_2eproto() { "Coordination.proto");
[6/7] drill git commit: DRILL-6256: Remove references to java 7 from readme and other files
DRILL-6256: Remove references to java 7 from readme and other files close apache/drill#1172 Project: http://git-wip-us.apache.org/repos/asf/drill/repo Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/03b245ef Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/03b245ef Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/03b245ef Branch: refs/heads/master Commit: 03b245ef4d028458fc6cc3588682d5ae9cc3bb33 Parents: 7f64556 Author: vladimir tkach Authored: Mon Mar 26 20:02:37 2018 +0300 Committer: Aman Sinha Committed: Thu Mar 29 23:23:17 2018 -0700 -- INSTALL.md | 10 +- distribution/src/resources/README.md | 2 +- distribution/src/resources/drill-config.sh | 9 - exec/java-exec/src/main/sh/drill-config.sh | 2 +- exec/java-exec/src/main/sh/runbit | 4 ++-- pom.xml| 2 +- 6 files changed, 14 insertions(+), 15 deletions(-) -- http://git-wip-us.apache.org/repos/asf/drill/blob/03b245ef/INSTALL.md -- diff --git a/INSTALL.md b/INSTALL.md index 7c65611..72268e9 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -3,15 +3,15 @@ ## Prerequisites Currently, the Apache Drill build process is known to work on Linux, Windows and OSX. To build, you need to have the following software installed on your system to successfully complete a build. - * Java 7 + * Java 8 * Maven 3.x ## Confirm settings # java -version -java version "1.7.0_09" -Java(TM) SE Runtime Environment (build 1.7.0_09-b05) -Java HotSpot(TM) 64-Bit Server VM (build 23.5-b02, mixed mode) - +java version "1.8.0_161" +Java(TM) SE Runtime Environment (build 1.8.0_161-b12) +Java HotSpot(TM) 64-Bit Server VM (build 25.161-b12, mixed mode) + # mvn --version Apache Maven 3.0.3 (r1075438; 2011-02-28 09:31:09-0800) http://git-wip-us.apache.org/repos/asf/drill/blob/03b245ef/distribution/src/resources/README.md -- diff --git a/distribution/src/resources/README.md b/distribution/src/resources/README.md index 40feb29..bb1cd07 100644 --- a/distribution/src/resources/README.md +++ b/distribution/src/resources/README.md @@ -3,7 +3,7 @@ ## Prerequisites * Linux, Windows or OSX - * Oracle JDK 7 (JDK, not JRE) + * Oracle/OpenJDK 8 (JDK, not JRE) Additional requirements when running in clustered mode: * Hadoop 2.3+ distribution of Hadoop (such as Apache or MapR) http://git-wip-us.apache.org/repos/asf/drill/blob/03b245ef/distribution/src/resources/drill-config.sh -- diff --git a/distribution/src/resources/drill-config.sh b/distribution/src/resources/drill-config.sh index 55d032d..e3eaa64 100644 --- a/distribution/src/resources/drill-config.sh +++ b/distribution/src/resources/drill-config.sh @@ -123,10 +123,10 @@ if [ ! -e "$JAVA" ]; then fatal_error "Java not found at JAVA_HOME=$JAVA_HOME." fi -# Ensure that Java version is at least 1.7 +# Ensure that Java version is at least 1.8 "$JAVA" -version 2>&1 | grep "version" | egrep -e "1\.4|1\.5|1\.6|1\.7" > /dev/null if [ $? -eq 0 ]; then - fatal_error "Java 1.8 or later is required to run Apache Drill." + fatal_error "Java 1.8 is required to run Apache Drill." fi # Check if a file exists and has relevant lines for execution @@ -214,9 +214,8 @@ fi export SQLLINE_JAVA_OPTS=${SQLLINE_JAVA_OPTS:-""} -# Class unloading is disabled by default in Java 7 -# http://hg.openjdk.java.net/jdk7u/jdk7u60/hotspot/file/tip/src/share/vm/runtime/globals.hpp#l1622 -export SERVER_GC_OPTS="$SERVER_GC_OPTS -XX:+CMSClassUnloadingEnabled -XX:+UseG1GC" + +export SERVER_GC_OPTS="$SERVER_GC_OPTS -XX:+UseG1GC" # No GC options by default for SQLLine export CLIENT_GC_OPTS=${CLIENT_GC_OPTS:-""} http://git-wip-us.apache.org/repos/asf/drill/blob/03b245ef/exec/java-exec/src/main/sh/drill-config.sh -- diff --git a/exec/java-exec/src/main/sh/drill-config.sh b/exec/java-exec/src/main/sh/drill-config.sh index 20102fc..fa62da8 100644 --- a/exec/java-exec/src/main/sh/drill-config.sh +++ b/exec/java-exec/src/main/sh/drill-config.sh @@ -88,7 +88,7 @@ if [ -z "$JAVA_HOME" ]; then +==+ | Error: JAVA_HOME is not set and Java could not be found | +--+ -| Drill requires Java 1.7 or later.| +| Drill requires Java 1.8| +==+ EOF exit 1 http://git-wip-us.apache
[2/7] drill git commit: DRILL-6299: Fixed a filter pushed down issue when a column doesn't have stats
DRILL-6299: Fixed a filter pushed down issue when a column doesn't have stats close apache/drill#1192 Project: http://git-wip-us.apache.org/repos/asf/drill/repo Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/ea643bfe Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/ea643bfe Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/ea643bfe Branch: refs/heads/master Commit: ea643bfebeff8991d4e43fa8762773076087d0df Parents: bfc86f1 Author: Salim Achouche Authored: Wed Mar 28 12:08:25 2018 -0700 Committer: Aman Sinha Committed: Thu Mar 29 23:18:32 2018 -0700 -- .../drill/exec/expr/stat/ParquetIsPredicates.java | 17 ++--- .../exec/expr/stat/ParquetPredicatesHelper.java| 9 + 2 files changed, 15 insertions(+), 11 deletions(-) -- http://git-wip-us.apache.org/repos/asf/drill/blob/ea643bfe/exec/java-exec/src/main/java/org/apache/drill/exec/expr/stat/ParquetIsPredicates.java -- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/stat/ParquetIsPredicates.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/stat/ParquetIsPredicates.java index c6f9b2f..a58ce7c 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/stat/ParquetIsPredicates.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/stat/ParquetIsPredicates.java @@ -62,7 +62,7 @@ public class ParquetIsPredicates { public boolean canDrop(RangeExprEvaluator evaluator) { Statistics exprStat = expr.accept(evaluator, null); - if (exprStat == null) { + if (!ParquetPredicatesHelper.hasStats(exprStat)) { return false; } @@ -87,8 +87,7 @@ public class ParquetIsPredicates { public boolean canDrop(RangeExprEvaluator evaluator) { Statistics exprStat = expr.accept(evaluator, null); - if (exprStat == null || - exprStat.isEmpty()) { + if (!ParquetPredicatesHelper.hasStats(exprStat)) { return false; } @@ -113,8 +112,7 @@ public class ParquetIsPredicates { public boolean canDrop(RangeExprEvaluator evaluator) { Statistics exprStat = expr.accept(evaluator, null); - if (exprStat == null || - exprStat.isEmpty()) { + if (!ParquetPredicatesHelper.hasStats(exprStat)) { return false; } @@ -140,8 +138,7 @@ public class ParquetIsPredicates { public boolean canDrop(RangeExprEvaluator evaluator) { Statistics exprStat = expr.accept(evaluator, null); - if (exprStat == null || - exprStat.isEmpty()) { + if (!ParquetPredicatesHelper.hasStats(exprStat)) { return false; } @@ -167,8 +164,7 @@ public class ParquetIsPredicates { public boolean canDrop(RangeExprEvaluator evaluator) { Statistics exprStat = expr.accept(evaluator, null); - if (exprStat == null || - exprStat.isEmpty()) { + if (!ParquetPredicatesHelper.hasStats(exprStat)) { return false; } @@ -193,8 +189,7 @@ public class ParquetIsPredicates { public boolean canDrop(RangeExprEvaluator evaluator) { Statistics exprStat = expr.accept(evaluator, null); - if (exprStat == null || - exprStat.isEmpty()) { + if (!ParquetPredicatesHelper.hasStats(exprStat)) { return false; } http://git-wip-us.apache.org/repos/asf/drill/blob/ea643bfe/exec/java-exec/src/main/java/org/apache/drill/exec/expr/stat/ParquetPredicatesHelper.java -- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/stat/ParquetPredicatesHelper.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/stat/ParquetPredicatesHelper.java index ac82d65..e43acd3 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/stat/ParquetPredicatesHelper.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/stat/ParquetPredicatesHelper.java @@ -22,9 +22,18 @@ import org.apache.parquet.column.statistics.Statistics; /** * Parquet predicates class helper for filter pushdown. */ +@SuppressWarnings("rawtypes") public class ParquetPredicatesHelper { /** + * @param stat statistics object + * @return true if the input stat object has valid statistics; false otherwise + */ + public static boolean hasStats(Statistics stat) { +return stat != null && !stat.isEmpty(); + } + + /** * Checks that column chunk's statistics has only nulls * * @param stat parquet column statistics
[5/7] drill git commit: DRILL-6278: Removed temp codegen directory in testing framework.
DRILL-6278: Removed temp codegen directory in testing framework. close apache/drill#1178 Project: http://git-wip-us.apache.org/repos/asf/drill/repo Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/7f645565 Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/7f645565 Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/7f645565 Branch: refs/heads/master Commit: 7f645565cd68b1780e643bc20a43951610704008 Parents: 67710bb Author: Timothy Farkas Authored: Tue Mar 20 23:00:22 2018 -0700 Committer: Aman Sinha Committed: Thu Mar 29 23:22:25 2018 -0700 -- .../drill/exec/physical/impl/TopN/TopNBatchTest.java | 6 +- .../java/org/apache/drill/test/BaseDirTestWatcher.java| 10 -- .../test/java/org/apache/drill/test/BaseTestQuery.java| 1 - .../test/java/org/apache/drill/test/ClusterFixture.java | 1 - 4 files changed, 1 insertion(+), 17 deletions(-) -- http://git-wip-us.apache.org/repos/asf/drill/blob/7f645565/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TopN/TopNBatchTest.java -- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TopN/TopNBatchTest.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TopN/TopNBatchTest.java index 4860869..14f2ee8 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TopN/TopNBatchTest.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TopN/TopNBatchTest.java @@ -63,8 +63,6 @@ public class TopNBatchTest extends PopUnitTestBase { @Test public void priorityQueueOrderingTest() throws Exception { Properties properties = new Properties(); -properties.setProperty(ClassBuilder.CODE_DIR_OPTION, dirTestWatcher.getDir().getAbsolutePath()); - DrillConfig drillConfig = DrillConfig.create(properties); FieldReference expr = FieldReference.getWithQuotedRef("colA"); @@ -158,9 +156,7 @@ public class TopNBatchTest extends PopUnitTestBase { */ @Test public void sortOneKeyAscending() throws Throwable { -ClusterFixtureBuilder builder = ClusterFixture.builder(dirTestWatcher) - .configProperty(ClassBuilder.CODE_DIR_OPTION, dirTestWatcher.getDir().getAbsolutePath()) - .configProperty(CodeCompiler.ENABLE_SAVE_CODE_FOR_DEBUG_TOPN, true); +ClusterFixtureBuilder builder = ClusterFixture.builder(dirTestWatcher); try (ClusterFixture cluster = builder.build(); ClientFixture client = cluster.clientFixture()) { TestBuilder testBuilder = new TestBuilder(new ClusterFixture.FixtureTestServices(client)); http://git-wip-us.apache.org/repos/asf/drill/blob/7f645565/exec/java-exec/src/test/java/org/apache/drill/test/BaseDirTestWatcher.java -- diff --git a/exec/java-exec/src/test/java/org/apache/drill/test/BaseDirTestWatcher.java b/exec/java-exec/src/test/java/org/apache/drill/test/BaseDirTestWatcher.java index d36423b..b595869 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/test/BaseDirTestWatcher.java +++ b/exec/java-exec/src/test/java/org/apache/drill/test/BaseDirTestWatcher.java @@ -53,7 +53,6 @@ public class BaseDirTestWatcher extends DirTestWatcher { TEST_TMP // Corresponds to the directory that should be mapped to dfs.tmp } - private File codegenDir; private File tmpDir; private File storeDir; private File dfsTestTmpParentDir; @@ -79,7 +78,6 @@ public class BaseDirTestWatcher extends DirTestWatcher { protected void starting(Description description) { super.starting(description); -codegenDir = makeSubDir(Paths.get("codegen")); rootDir = makeSubDir(Paths.get("root")); tmpDir = makeSubDir(Paths.get("tmp")); storeDir = makeSubDir(Paths.get("store")); @@ -135,14 +133,6 @@ public class BaseDirTestWatcher extends DirTestWatcher { } /** - * Gets the temp directory that should be used to save generated code files. - * @return The temp directory that should be used to save generated code files. - */ - public File getCodegenDir() { -return codegenDir; - } - - /** * This methods creates a new directory which can be mapped to dfs.tmp. */ public void newDfsTestTmpDir() { http://git-wip-us.apache.org/repos/asf/drill/blob/7f645565/exec/java-exec/src/test/java/org/apache/drill/test/BaseTestQuery.java -- diff --git a/exec/java-exec/src/test/java/org/apache/drill/test/BaseTestQuery.java b/exec/java-exec/src/test/java/org/apache/drill/test/BaseTestQuery.java index c3ecaf1..10cd94c 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/test/BaseTestQuery.java +++ b/exec/java-exec/src/test/java/org/apache/drill/test/BaseTestQuery.java @@ -160,7 +160,6 @
[4/7] drill git commit: DRILL-6254: IllegalArgumentException: the requested size must be non-negative
DRILL-6254: IllegalArgumentException: the requested size must be non-negative close apache/drill#1179 Project: http://git-wip-us.apache.org/repos/asf/drill/repo Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/67710bba Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/67710bba Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/67710bba Branch: refs/heads/master Commit: 67710bba7cdbc05428df7390bad8639b099769fc Parents: 7088bfe Author: Padma Penumarthy Authored: Wed Mar 21 13:39:43 2018 -0700 Committer: Aman Sinha Committed: Thu Mar 29 23:21:50 2018 -0700 -- .../drill/exec/physical/impl/flatten/FlattenRecordBatch.java| 5 - 1 file changed, 4 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/drill/blob/67710bba/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/flatten/FlattenRecordBatch.java -- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/flatten/FlattenRecordBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/flatten/FlattenRecordBatch.java index 9dd1770..7509809 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/flatten/FlattenRecordBatch.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/flatten/FlattenRecordBatch.java @@ -237,7 +237,10 @@ public class FlattenRecordBatch extends AbstractSingleRecordBatch { private void handleRemainder() { int remainingRecordCount = flattener.getFlattenField().getAccessor().getInnerValueCount() - remainderIndex; -if (!doAlloc(remainingRecordCount)) { + +// remainingRecordCount can be much higher than number of rows we will have in outgoing batch. +// Do memory allocation only for number of rows we are going to have in the batch. +if (!doAlloc(Math.min(remainingRecordCount, flattenMemoryManager.getOutputRowCount( { outOfMemory = true; return; }
[3/7] drill git commit: DRILL-5937: drill-module.conf : Changed timeout to 30 seconds, ExecConstant.java : Changed comment
DRILL-5937: drill-module.conf : Changed timeout to 30 seconds, ExecConstant.java : Changed comment DRILL-5937: ExecConstant.java : removed comment DRILL-5937: CheckStyle fix close apache/drill#1190 Project: http://git-wip-us.apache.org/repos/asf/drill/repo Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/7088bfe4 Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/7088bfe4 Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/7088bfe4 Branch: refs/heads/master Commit: 7088bfe4bd6e2379ba21b0c48f7280a77d2c3b83 Parents: ea643bf Author: Pushpendra Jaiswal Authored: Tue Mar 27 13:16:23 2018 +0530 Committer: Aman Sinha Committed: Thu Mar 29 23:20:30 2018 -0700 -- .../src/main/java/org/apache/drill/exec/ExecConstants.java | 4 exec/java-exec/src/main/resources/drill-module.conf | 2 +- 2 files changed, 1 insertion(+), 5 deletions(-) -- http://git-wip-us.apache.org/repos/asf/drill/blob/7088bfe4/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java -- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java b/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java index 34aec1b..77fa211 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java @@ -561,10 +561,6 @@ public final class ExecConstants { public static final String CODE_GEN_EXP_IN_METHOD_SIZE = "exec.java.compiler.exp_in_method_size"; public static final LongValidator CODE_GEN_EXP_IN_METHOD_SIZE_VALIDATOR = new LongValidator(CODE_GEN_EXP_IN_METHOD_SIZE); - /** - * Timeout for create prepare statement request. If the request exceeds this timeout, then request is timed out. - * Default value is 10mins. - */ public static final String CREATE_PREPARE_STATEMENT_TIMEOUT_MILLIS = "prepare.statement.create_timeout_ms"; public static final OptionValidator CREATE_PREPARE_STATEMENT_TIMEOUT_MILLIS_VALIDATOR = new PositiveLongValidator(CREATE_PREPARE_STATEMENT_TIMEOUT_MILLIS, Integer.MAX_VALUE); http://git-wip-us.apache.org/repos/asf/drill/blob/7088bfe4/exec/java-exec/src/main/resources/drill-module.conf -- diff --git a/exec/java-exec/src/main/resources/drill-module.conf b/exec/java-exec/src/main/resources/drill-module.conf index a227e0d..9af09bc 100644 --- a/exec/java-exec/src/main/resources/drill-module.conf +++ b/exec/java-exec/src/main/resources/drill-module.conf @@ -514,7 +514,7 @@ drill.exec.options: { # it is dynamically computed based on cpu_load_average planner.width.max_per_node: 0, planner.width.max_per_query: 1000, -prepare.statement.create_timeout_ms: 1, +prepare.statement.create_timeout_ms: 3, security.admin.user_groups: "%drill_process_user_groups%", security.admin.users: "%drill_process_user%", store.format: "parquet",
[2/2] drill git commit: DRILL-6118: Handle item star columns during project / filter push down and directory pruning
DRILL-6118: Handle item star columns during project / filter push down and directory pruning 1. Added DrillFilterItemStarReWriterRule to re-write item star fields to regular field references. 2. Refactored DrillPushProjectIntoScanRule to handle item star fields, factored out helper classes and methods from PreUitl.class. 3. Fixed issue with dynamic star usage (after Calcite upgrade old usage of star was still present, replaced WILDCARD -> DYNAMIC_STAR for clarity). 4. Added unit tests to check project / filter push down and directory pruning with item star. Project: http://git-wip-us.apache.org/repos/asf/drill/repo Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/9073aed6 Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/9073aed6 Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/9073aed6 Branch: refs/heads/master Commit: 9073aed67d89e8b2188870d6c812706085c9c41b Parents: 50efb80 Author: Arina Ielchiieva Authored: Thu Dec 21 19:31:00 2017 +0200 Committer: Aman Sinha Committed: Sat Feb 24 19:56:35 2018 -0800 -- .../org/apache/drill/exec/dotdrill/View.java| 5 +- .../impl/project/ProjectRecordBatch.java| 12 +- .../impl/values/ValuesBatchCreator.java | 5 +- .../apache/drill/exec/planner/PlannerPhase.java | 7 +- .../drill/exec/planner/StarColumnHelper.java| 13 +- .../exec/planner/common/DrillScanRelBase.java | 10 +- .../DrillFilterItemStarReWriterRule.java| 232 + .../drill/exec/planner/logical/DrillOptiq.java | 3 +- .../planner/logical/DrillPushProjIntoScan.java | 97 --- .../logical/DrillPushProjectIntoScanRule.java | 256 +++ .../planner/logical/FieldsReWriterUtil.java | 138 ++ .../planner/logical/PreProcessLogicalRel.java | 6 +- .../drill/exec/planner/physical/PrelUtil.java | 247 +- .../visitor/SplitUpComplexExpressions.java | 6 +- .../planner/sql/handlers/SqlHandlerUtil.java| 4 +- .../exec/planner/types/RelDataTypeHolder.java | 13 +- .../text/compliant/RepeatedVarCharOutput.java | 7 +- .../drill/exec/store/mock/MockGroupScanPOP.java | 2 +- .../parquet2/DrillParquetGroupConverter.java| 12 +- .../exec/store/text/DrillTextRecordReader.java | 4 +- .../org/apache/drill/exec/util/Utilities.java | 39 ++- .../exec/vector/complex/fn/FieldSelection.java | 13 +- .../apache/drill/exec/TestWindowFunctions.java | 4 +- .../physical/unit/MiniPlanUnitTestBase.java | 3 +- .../physical/unit/PhysicalOpUnitTestBase.java | 3 +- .../TestPushDownAndPruningWithItemStar.java | 183 + .../drill/common/expression/SchemaPath.java | 12 +- 27 files changed, 923 insertions(+), 413 deletions(-) -- http://git-wip-us.apache.org/repos/asf/drill/blob/9073aed6/exec/java-exec/src/main/java/org/apache/drill/exec/dotdrill/View.java -- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/dotdrill/View.java b/exec/java-exec/src/main/java/org/apache/drill/exec/dotdrill/View.java index 3524d73..615e3bc 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/dotdrill/View.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/dotdrill/View.java @@ -23,6 +23,7 @@ import org.apache.calcite.avatica.util.TimeUnit; import org.apache.calcite.sql.SqlIntervalQualifier; import org.apache.calcite.sql.parser.SqlParserPos; import org.apache.calcite.sql.type.SqlTypeFamily; +import org.apache.drill.common.expression.SchemaPath; import org.apache.drill.exec.planner.StarColumnHelper; import org.apache.drill.exec.planner.types.RelDataTypeDrillImpl; import org.apache.drill.exec.planner.types.RelDataTypeHolder; @@ -72,9 +73,9 @@ public class View { @JsonProperty("fractionalSecondPrecision") Integer fractionalSecondPrecision, @JsonProperty("isNullable") Boolean isNullable) { // Fix for views which were created on Calcite 1.4. - // After Calcite upgrade star "*" was changed on dynamic star "**" + // After Calcite upgrade star "*" was changed on dynamic star "**" (SchemaPath.DYNAMIC_STAR) // and type of star was changed to SqlTypeName.DYNAMIC_STAR - this.name = "*".equals(name) ? "**" : name; + this.name = "*".equals(name) ? SchemaPath.DYNAMIC_STAR : name; this.type = "*".equals(name) && type == SqlTypeName.ANY ? SqlTypeName.DYNAMIC_STAR : type; this.precision = precision; this.scale = scale; http://git-wip-us.apache.org/repos/asf/drill/blob/9073aed6/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectRecordBatch.java -- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/Pr
[1/2] drill git commit: DRILL-6118: Handle item star columns during project / filter push down and directory pruning
Repository: drill Updated Branches: refs/heads/master 50efb806b -> 9073aed67 http://git-wip-us.apache.org/repos/asf/drill/blob/9073aed6/logical/src/main/java/org/apache/drill/common/expression/SchemaPath.java -- diff --git a/logical/src/main/java/org/apache/drill/common/expression/SchemaPath.java b/logical/src/main/java/org/apache/drill/common/expression/SchemaPath.java index 95f3dbb..583046a 100644 --- a/logical/src/main/java/org/apache/drill/common/expression/SchemaPath.java +++ b/logical/src/main/java/org/apache/drill/common/expression/SchemaPath.java @@ -42,8 +42,8 @@ import com.google.common.base.Preconditions; public class SchemaPath extends LogicalExpressionBase { - public static final String WILDCARD = "*"; - public static final SchemaPath STAR_COLUMN = getSimplePath(WILDCARD); + public static final String DYNAMIC_STAR = "**"; + public static final SchemaPath STAR_COLUMN = getSimplePath(DYNAMIC_STAR); private final NameSegment rootSegment; @@ -205,14 +205,14 @@ public class SchemaPath extends LogicalExpressionBase { } /** - * Return if this column is the special wildcard ("*") column which means to + * Return if this column is the special wildcard ("**") column which means to * project all table columns. * - * @return true if the column is "*" + * @return true if the column is "**" */ - public boolean isWildcard() { -return isLeaf() && nameEquals(WILDCARD); + public boolean isDynamicStar() { +return isLeaf() && nameEquals(DYNAMIC_STAR); } /**
[3/3] drill git commit: DRILL-5741: Automatically manage memory allocations during startup
DRILL-5741: Automatically manage memory allocations during startup **Note:** This commit is rebased on (hence, requires) DRILL-6068 commit This commit provides a way for distributions and users to automatically define the Drillbit JVM's Heap, Direct and CodeCache allocations. To do this, it leverages the DRILL-6068 commit, which provides for configuration setup. The only new variable introduced is an environment variable - `DRILLBIT_MAX_PROC_MEM` that defines a Drillbit's max memory parameters. The logic defined by `auto-setup.sh` will check that if any (or all) of the JVM memory parameters have been declared; they, cumulatively, don't exceed the value specified by `DRILLBIT_MAX_PROC_MEM`. ``` NOTE: To disable this, simply unset (or don't define) the environment variable DRILLBIT_MAX_PROC_MEM ``` The variable can be defined in KB, MB, or GB; similar in syntax to how the JVM MaxHeap is specified. For e.g. ``` DRILLBIT_MAX_PROC_MEM=13G DRILLBIT_MAX_PROC_MEM=8192m DRILLBIT_MAX_PROC_MEM=4194304K ``` In addition, you can specify it as a percent of the total system memory prior to the Drillbit starting up: `DRILLBIT_MAX_PROC_MEM=40%` For a system with with 48GB free memory, when set to (say) 25% (with settings defined in drill-env.sh), and heap (8GB) and direct (10GB) are defined; the Drillbit fails startup with the following message: ``` 2018-01-03 14:27:57 [WARN] 25% of System Memory (47 GB) translates to 12 GB 2018-01-03 14:27:57 [ERROR]Unable to start Drillbit due to memory constraint violations Total Memory Requested : 19 GB Check the following settings to possibly modify (or increase the Max Memory Permitted): DRILLBIT_MAX_PROC_MEM=25% DRILL_HEAP=8G DRILL_MAX_DIRECT_MEMORY=10G DRILLBIT_CODE_CACHE_SIZE=1024m *NOTE: It is recommended not to specify DRILLBIT_CODE_CACHE_SIZE as this will be auto-computed based on the HeapSize and would not exceed 1GB ``` For all other combinations, the undefined parameters are adjusted to ensure that the total memory allocated is within the value specified by `DRILLBIT_MAX_PROC_MEM`, For a system with with 48GB free memory, when set to (say) 50% (with settings defined in drill-env.sh), and heap (8GB) and direct (10GB) are defined; the Drillbit startup with the following warning: ``` 2018-01-03 14:31:06 [WARN] 50% of System Memory (47 GB) translates to 24 GB 2018-01-03 14:31:06 [WARN] You have an allocation of 4 GB that is currently unused from a total of 24 GB. You can increase your existing memory configuration to use this extra memory DRILLBIT_MAX_PROC_MEM=50% DRILL_HEAP=8G DRILL_MAX_DIRECT_MEMORY=10G DRILLBIT_CODE_CACHE_SIZE=1024m *NOTE: It is recommended not to specify DRILLBIT_CODE_CACHE_SIZE as this will be auto-computed based on the HeapSize and would not exceed 1GB ``` In addition, if the available free memory is less than the allocation, an additional warning is provided under the assumption that the OS will reclaim more free memory when required: ``` 2018-01-03 14:31:06 [WARN] Total Memory Allocation for Drillbit (19GB) exceeds available free memory (11GB) 2018-01-03 14:31:06 [WARN] Drillbit will start up, but can potentially crash due to oversubscribing of system memory. ``` For more details, refer the attachments in https://issues.apache.org/jira/browse/DRILL-5741 Changes to auto configure messaging Publishing final values prior to startup Minor update for printing to console's err stream close apache/drill#1082 Project: http://git-wip-us.apache.org/repos/asf/drill/repo Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/50efb806 Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/50efb806 Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/50efb806 Branch: refs/heads/master Commit: 50efb806bb03494e1da4d6b48f90fbf58d699c18 Parents: 266250c Author: Kunal Khatua Authored: Thu Jan 11 17:35:13 2018 -0800 Committer: Aman Sinha Committed: Fri Feb 23 17:55:36 2018 -0800 -- distribution/src/resources/auto-setup.sh| 202 ++- distribution/src/resources/distrib-setup.sh | 2 +- distribution/src/resources/drill-env.sh | 10 ++ 3 files changed, 212 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/drill/blob/50efb806/distribution/src/resources/auto-setup.sh -- diff --git a/distribution/src/resources/auto-setup.sh b/distribution/src/resources/auto-setup.sh index 75bdda0..141648c 100644 --- a/distribution/src/resources/auto-setup.sh +++ b/distribution/src/resources/auto-setup.sh @@ -25,5 +25,205 @@ # if [ $status == "FAILED" ]; return 1; fi ###== -# FEATURES (Added here to describe supported add
[2/3] drill git commit: DRILL-6068: Support user/distrib-specific config checks during startup
DRILL-6068: Support user/distrib-specific config checks during startup 1. Allows for distrib/user specific checks to be done 2. Place-holder files for distribution and user specific checks 3. Moved JVM Version Check to head of script Separation of setups into 3 levels 1. Apache Drill (Default) 2. Distribition Specific 3. User-Defined 1 & 2 are mutually exclusive. 3 is additional checks that the user can specify. Project: http://git-wip-us.apache.org/repos/asf/drill/repo Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/266250cf Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/266250cf Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/266250cf Branch: refs/heads/master Commit: 266250cf6ca2269f8772796d6c4eb903aa795111 Parents: 27aa236 Author: Kunal Khatua Authored: Thu Jan 11 15:45:04 2018 -0800 Committer: Aman Sinha Committed: Fri Feb 23 17:55:06 2018 -0800 -- distribution/src/assemble/bin.xml | 15 ++ distribution/src/resources/auto-setup.sh| 29 distribution/src/resources/distrib-setup.sh | 29 distribution/src/resources/drill-config.sh | 178 +++ distribution/src/resources/drill-setup.sh | 26 5 files changed, 217 insertions(+), 60 deletions(-) -- http://git-wip-us.apache.org/repos/asf/drill/blob/266250cf/distribution/src/assemble/bin.xml -- diff --git a/distribution/src/assemble/bin.xml b/distribution/src/assemble/bin.xml index b4290bb..bed34ed 100644 --- a/distribution/src/assemble/bin.xml +++ b/distribution/src/assemble/bin.xml @@ -347,6 +347,21 @@ conf + src/resources/auto-setup.sh + 0755 + bin + + + src/resources/drill-setup.sh + 0755 + conf + + + src/resources/distrib-setup.sh + 0755 + conf + + src/resources/drill-override-example.conf conf http://git-wip-us.apache.org/repos/asf/drill/blob/266250cf/distribution/src/resources/auto-setup.sh -- diff --git a/distribution/src/resources/auto-setup.sh b/distribution/src/resources/auto-setup.sh new file mode 100644 index 000..75bdda0 --- /dev/null +++ b/distribution/src/resources/auto-setup.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This file is invoked by drill-config.sh during a Drillbit startup and provides +# default checks and autoconfiguration. +# Distributions should not put anything in this file. Checks can be +# specified in ${DRILL_HOME}/conf/distrib-setup.sh +# Users should not put anything in this file. Additional checks can be defined +# and put in ${DRILL_CONF_DIR}/drill-setup.sh instead. +# To FAIL any check, return with a non-zero return code +# e.g. +# if [ $status == "FAILED" ]; return 1; fi + +###== +# FEATURES (Added here to describe supported additions) +###== http://git-wip-us.apache.org/repos/asf/drill/blob/266250cf/distribution/src/resources/distrib-setup.sh -- diff --git a/distribution/src/resources/distrib-setup.sh b/distribution/src/resources/distrib-setup.sh new file mode 100644 index 000..dbd2b90 --- /dev/null +++ b/distribution/src/resources/distrib-setup.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License
[1/3] drill git commit: DRILL-5978: Updating of Apache and MapR Hive libraries to 2.3.2 and 2.1.2-mapr-1710 versions respectively
Repository: drill Updated Branches: refs/heads/master 1c14d3c3c -> 50efb806b DRILL-5978: Updating of Apache and MapR Hive libraries to 2.3.2 and 2.1.2-mapr-1710 versions respectively * Improvements to allow of reading Hive bucketed transactional ORC tables; * Updating hive properties for tests and resolving dependencies and API conflicts: - Fix for "hive.metastore.schema.verification", MetaException(message: Version information not found in metastore) https://cwiki.apache.org/confluence/display/Hive/Hive+Schema+Tool METASTORE_SCHEMA_VERIFICATION="false" property is added - Added METASTORE_AUTO_CREATE_ALL="true", properties to tests, because some additional tables are necessary in Hive metastore - Disabling calcite CBO for (Hive's CalcitePlanner) for tests, because it is in conflict with Drill's Calcite version for Drill unit tests. HIVE_CBO_ENABLED="false" property - jackson and parquet libraries are relocated in hive-exec-shade module - org.apache.parquet:parquet-column Drill version is added to "hive-exec" to allow of using Parquet empty group on MessageType level (PARQUET-278) - Removing of commons-codec exclusion from hive core. This dependency is necessary for hive-exec and hive-metastore. - Setting Hive internal properties for transactional scan: HiveConf.HIVE_TRANSACTIONAL_TABLE_SCAN and for schema evolution: HiveConf.HIVE_SCHEMA_EVOLUTION, IOConstants.SCHEMA_EVOLUTION_COLUMNS, IOConstants.SCHEMA_EVOLUTION_COLUMNS_TYPES - "io.dropwizard.metrics:metrics-core" with last 4.0.2 version is added to dependencyManagement block in Drill root POM - Exclusion of "hive-exec" in "hive-hbase-handler" is already in Drill root dependencyManagement POM - Hive Calcite libraries are excluded (Calcite CBO was disabled) - "jackson-core" dependency is added to DependencyManagement block in Drill root POM file - For MapR Hive 2.1 client older "com.fasterxml.jackson.core:jackson-databind" is included - "log4j:log4j" dependency is excluded from "hive-exec", "hive-metastore", "hive-hbase-handler". close apache/drill# Project: http://git-wip-us.apache.org/repos/asf/drill/repo Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/27aa2369 Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/27aa2369 Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/27aa2369 Branch: refs/heads/master Commit: 27aa236975e6d998e5b5dc173de777c78bb44bdd Parents: 1c14d3c Author: Vitalii Diravka Authored: Mon Nov 13 16:04:03 2017 + Committer: Aman Sinha Committed: Fri Feb 23 17:48:12 2018 -0800 -- contrib/storage-hive/core/pom.xml | 6 - .../codegen/templates/HiveRecordReaders.java| 4 +- .../exec/store/hive/HiveMetadataProvider.java | 1 + .../drill/exec/store/hive/HiveUtilities.java| 65 +++- .../store/hive/readers/HiveAbstractReader.java | 57 --- .../apache/drill/exec/hive/TestHiveStorage.java | 10 ++ .../hive/BaseTestHiveImpersonation.java | 5 +- .../hive/TestSqlStdBasedAuthorization.java | 8 +- .../hive/TestStorageBasedHiveAuthorization.java | 6 + .../exec/store/hive/HiveTestDataGenerator.java | 3 + contrib/storage-hive/hive-exec-shade/pom.xml| 158 +++ exec/java-exec/pom.xml | 15 -- exec/jdbc-all/pom.xml | 20 --- exec/jdbc/pom.xml | 1 - pom.xml | 74 - 15 files changed, 292 insertions(+), 141 deletions(-) -- http://git-wip-us.apache.org/repos/asf/drill/blob/27aa2369/contrib/storage-hive/core/pom.xml -- diff --git a/contrib/storage-hive/core/pom.xml b/contrib/storage-hive/core/pom.xml index cdf8fbe..184300f 100644 --- a/contrib/storage-hive/core/pom.xml +++ b/contrib/storage-hive/core/pom.xml @@ -63,12 +63,6 @@ org.apache.hive hive-hbase-handler - - - org.apache.hive - hive-exec - - org.apache.hbase http://git-wip-us.apache.org/repos/asf/drill/blob/27aa2369/contrib/storage-hive/core/src/main/codegen/templates/HiveRecordReaders.java -- diff --git a/contrib/storage-hive/core/src/main/codegen/templates/HiveRecordReaders.java b/contrib/storage-hive/core/src/main/codegen/templates/HiveRecordReaders.java index 4a75ed3..a6e588b 100644 --- a/contrib/storage-hive/core/src/main/codegen/templates/HiveRecordReaders.java +++ b/contrib/storage-hive/core/src/main/codegen/templates/HiveRecordReaders.java @@ -130,7 +130,7 @@ public class Hive${entry.hiveReader}Reader extends HiveAbstractReader { while (!recordsInspector.isBatchFull() && hasNextValue(recordsInspector.getValueHold
[4/5] drill git commit: DRILL-6154: NaN, Infinity issues
DRILL-6154: NaN, Infinity issues - changed comparison rules for NaN, Infinity values. For now NaN is the biggest value, Infinity - second biggest value - fixed min, max, trunc functions for NaN, Infinity values - made drill use original sqrt function instead of power(x,0.5) substitution close apache/drill#1123 Project: http://git-wip-us.apache.org/repos/asf/drill/repo Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/bcd358b3 Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/bcd358b3 Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/bcd358b3 Branch: refs/heads/master Commit: bcd358b3c73f34381b543789dfc368bfaefa4ea9 Parents: b4d2a77 Author: vladimir tkach Authored: Fri Feb 16 16:36:49 2018 +0200 Committer: Aman Sinha Committed: Fri Feb 23 14:18:15 2018 -0800 -- .../codegen/templates/AggrTypeFunctions1.java | 23 ++-- .../codegen/templates/ComparisonFunctions.java | 13 - .../main/codegen/templates/MathFunctions.java | 16 -- .../exec/planner/sql/DrillConvertletTable.java | 15 ++ .../org/apache/drill/TestExampleQueries.java| 1 - .../fn/impl/TestMathFunctionsWithNanInf.java| 22 +--- .../vector/complex/writer/TestJsonNanInf.java | 57 7 files changed, 129 insertions(+), 18 deletions(-) -- http://git-wip-us.apache.org/repos/asf/drill/blob/bcd358b3/exec/java-exec/src/main/codegen/templates/AggrTypeFunctions1.java -- diff --git a/exec/java-exec/src/main/codegen/templates/AggrTypeFunctions1.java b/exec/java-exec/src/main/codegen/templates/AggrTypeFunctions1.java index b363cd1..fa4e8f1 100644 --- a/exec/java-exec/src/main/codegen/templates/AggrTypeFunctions1.java +++ b/exec/java-exec/src/main/codegen/templates/AggrTypeFunctions1.java @@ -71,9 +71,9 @@ public static class ${type.inputType}${aggrtype.className} implements DrillAggFu <#elseif type.runningType?starts_with("BigInt")> value.value = Long.MAX_VALUE; <#elseif type.runningType?starts_with("Float4")> - value.value = Float.MAX_VALUE; + value.value = Float.NaN; <#elseif type.runningType?starts_with("Float8")> - value.value = Double.MAX_VALUE; + value.value = Double.NaN; <#elseif aggrtype.funcName == "max"> <#if type.runningType?starts_with("Bit")> @@ -101,8 +101,21 @@ public static class ${type.inputType}${aggrtype.className} implements DrillAggFu } nonNullCount.value = 1; + // For min/max functions: NaN is the biggest value, +// Infinity is the second biggest value +// -Infinity is the smallest value <#if aggrtype.funcName == "min"> - value.value = Math.min(value.value, in.value); + <#if type.inputType?contains("Float4")> + if(!Float.isNaN(in.value)) { + value.value = Float.isNaN(value.value) ? in.value : Math.min(value.value, in.value); + } + <#elseif type.inputType?contains("Float8")> + if(!Double.isNaN(in.value)) { + value.value = Double.isNaN(value.value) ? in.value : Math.min(value.value, in.value); + } +<#else> + value.value = Math.min(value.value, in.value); + <#elseif aggrtype.funcName == "max"> value.value = Math.max(value.value, in.value); <#elseif aggrtype.funcName == "sum"> @@ -138,9 +151,9 @@ public static class ${type.inputType}${aggrtype.className} implements DrillAggFu <#elseif type.runningType?starts_with("BigInt")> value.value = Long.MAX_VALUE; <#elseif type.runningType?starts_with("Float4")> - value.value = Float.MAX_VALUE; + value.value = Float.NaN; <#elseif type.runningType?starts_with("Float8")> - value.value = Double.MAX_VALUE; + value.value = Double.NaN; <#elseif aggrtype.funcName == "max"> <#if type.runningType?starts_with("Int")> http://git-wip-us.apache.org/repos/asf/drill/blob/bcd358b3/exec/java-exec/src/main/codegen/templates/ComparisonFunctions.java -- diff --git a/exec/java-exec/src/main/codegen/templates/ComparisonFunctions.java b/exec/java-exec/src/main/codegen/templates/ComparisonFunctions.java index 633bb56..12a4ef2 100644 --- a/exec/java-exec/src/main/codegen/templates/ComparisonFunctions.java +++ b/exec/java-exec/src/main/codegen/templates/ComparisonFunctions.java @@ -118,9 +118,18 @@ { <@compareNullsSubblock leftType=leftType rightType=rightType output="out.value" breakTarget="outside" nullCompare=true nullComparesHigh=nullComparesHigh /> - +//
[3/5] drill git commit: DRILL-5902: Queries encounter random failure due to RPC connection timed out
DRILL-5902: Queries encounter random failure due to RPC connection timed out close apache/drill#1113 Project: http://git-wip-us.apache.org/repos/asf/drill/repo Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/b4d2a770 Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/b4d2a770 Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/b4d2a770 Branch: refs/heads/master Commit: b4d2a77038421b243970da71655311076c0a5a43 Parents: 4f203ea Author: Vlad Rozov Authored: Mon Feb 5 19:15:56 2018 -0800 Committer: Aman Sinha Committed: Fri Feb 23 14:15:00 2018 -0800 -- .../server/rest/profile/ProfileResources.java | 5 +-- .../org/apache/drill/exec/work/WorkManager.java | 39 .../exec/work/batch/ControlMessageHandler.java | 4 +- .../exec/work/foreman/QueryStateProcessor.java | 14 +++ .../apache/drill/exec/work/user/UserWorker.java | 5 +-- 5 files changed, 48 insertions(+), 19 deletions(-) -- http://git-wip-us.apache.org/repos/asf/drill/blob/b4d2a770/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/profile/ProfileResources.java -- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/profile/ProfileResources.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/profile/ProfileResources.java index 8751ee6..ec06f0e 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/profile/ProfileResources.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/profile/ProfileResources.java @@ -383,10 +383,7 @@ public class ProfileResources { QueryId id = QueryIdHelper.getQueryIdFromString(queryId); // first check local running -Foreman f = work.getBee().getForemanForQueryId(id); -if(f != null){ - checkOrThrowQueryCancelAuthorization(f.getQueryContext().getQueryUserName(), queryId); - f.cancel(); +if (work.getBee().cancelForeman(id, principal)) { return String.format("Cancelled query %s on locally running node.", queryId); } http://git-wip-us.apache.org/repos/asf/drill/blob/b4d2a770/exec/java-exec/src/main/java/org/apache/drill/exec/work/WorkManager.java -- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/work/WorkManager.java b/exec/java-exec/src/main/java/org/apache/drill/exec/work/WorkManager.java index d75668c..7058c62 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/work/WorkManager.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/work/WorkManager.java @@ -22,6 +22,7 @@ import com.google.common.base.Preconditions; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import org.apache.drill.common.SelfCleaningRunnable; +import org.apache.drill.common.exceptions.UserException; import org.apache.drill.exec.coord.ClusterCoordinator; import org.apache.drill.exec.metrics.DrillMetrics; import org.apache.drill.exec.proto.BitControl.FragmentStatus; @@ -37,6 +38,7 @@ import org.apache.drill.exec.rpc.control.WorkEventBus; import org.apache.drill.exec.rpc.data.DataConnectionCreator; import org.apache.drill.exec.server.BootStrapContext; import org.apache.drill.exec.server.DrillbitContext; +import org.apache.drill.exec.server.rest.auth.DrillUserPrincipal; import org.apache.drill.exec.store.sys.PersistentStoreProvider; import org.apache.drill.exec.work.batch.ControlMessageHandler; import org.apache.drill.exec.work.foreman.Foreman; @@ -259,6 +261,43 @@ public class WorkManager implements AutoCloseable { executor.execute(runnable); } +public boolean cancelForeman(final QueryId queryId, DrillUserPrincipal principal) { + Preconditions.checkNotNull(queryId); + + final Foreman foreman = queries.get(queryId); + if (foreman == null) { +return false; + } + + final String queryIdString = QueryIdHelper.getQueryId(queryId); + + if (principal != null && !principal.canManageQueryOf(foreman.getQueryContext().getQueryUserName())) { +throw UserException.permissionError() +.message("Not authorized to cancel the query '%s'", queryIdString) +.build(logger); + } + + executor.execute(new Runnable() + { +@Override +public void run() +{ + final Thread currentThread = Thread.currentThread(); + final String originalName = currentThread.getName(); + try { +currentThread.setName(queryIdString + ":foreman:cancel"); +logger.debug("Canceling foreman"); +foreman.cancel(); + } catch (Throwable t) { +logger.warn("Exception while canceling foreman", t); + } finally { +currentThread.setName(original
[5/5] drill git commit: DRILL-6172: setValueCount of VariableLengthVectors throws IOB exception when called with 0 value after clearing vectors
DRILL-6172: setValueCount of VariableLengthVectors throws IOB exception when called with 0 value after clearing vectors close apache/drill#1124 Project: http://git-wip-us.apache.org/repos/asf/drill/repo Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/1c14d3c3 Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/1c14d3c3 Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/1c14d3c3 Branch: refs/heads/master Commit: 1c14d3c3c836515b375ae7c47c4179503b1dcee0 Parents: bcd358b Author: Sorabh Hamirwasia Authored: Tue Feb 20 17:14:29 2018 -0800 Committer: Aman Sinha Committed: Fri Feb 23 14:19:13 2018 -0800 -- .../exec/record/vector/TestValueVector.java | 42 .../templates/VariableLengthVectors.java| 6 ++- 2 files changed, 47 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/drill/blob/1c14d3c3/exec/java-exec/src/test/java/org/apache/drill/exec/record/vector/TestValueVector.java -- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/record/vector/TestValueVector.java b/exec/java-exec/src/test/java/org/apache/drill/exec/record/vector/TestValueVector.java index 621d288..ba4c79c 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/record/vector/TestValueVector.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/record/vector/TestValueVector.java @@ -17,6 +17,7 @@ */ package org.apache.drill.exec.record.vector; +import static junit.framework.TestCase.fail; import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -48,6 +49,7 @@ import org.apache.drill.exec.memory.BufferAllocator; import org.apache.drill.exec.memory.RootAllocatorFactory; import org.apache.drill.exec.proto.UserBitShared; import org.apache.drill.exec.record.MaterializedField; +import org.apache.drill.exec.record.VectorAccessibleUtilities; import org.apache.drill.exec.vector.BaseValueVector; import org.apache.drill.exec.vector.BitVector; import org.apache.drill.exec.vector.NullableFloat4Vector; @@ -57,6 +59,7 @@ import org.apache.drill.exec.vector.RepeatedIntVector; import org.apache.drill.exec.vector.UInt4Vector; import org.apache.drill.exec.vector.ValueVector; import org.apache.drill.exec.vector.VarCharVector; +import org.apache.drill.exec.vector.VariableWidthVector; import org.apache.drill.exec.vector.complex.ListVector; import org.apache.drill.exec.vector.complex.MapVector; import org.apache.drill.exec.vector.complex.RepeatedListVector; @@ -828,4 +831,43 @@ the interface to load has changed vector.clear(); } + /** + * For VariableLengthVectors when we clear of the vector and then explicitly set the + * ValueCount of zero, then it should not fail with IndexOutOfBoundException. + * @throws Exception + */ + @Test + public void testVarLengthVector_SetCountZeroAfterClear() throws Exception { +try { + final MaterializedField field = MaterializedField.create(EMPTY_SCHEMA_PATH, VarCharHolder.TYPE); + VariableWidthVector vector = new VarCharVector(field, allocator); + vector.allocateNew(); + vector.clear(); + assertTrue(vector.getAccessor().getValueCount() == 0); + vector.getMutator().setValueCount(0); + assertTrue(vector.getAccessor().getValueCount() == 0); +} catch (Exception ex) { + fail(); +} + } + + /** For VariableLengthVectors when we try to set value count greater than value count for which memory is allocated, + * then it should fail with IndexOutOfBoundException. + * @throws Exception + */ + @Test + public void testVarLengthVector_SetOOBCount() throws Exception { +final MaterializedField field = MaterializedField.create(EMPTY_SCHEMA_PATH, VarCharHolder.TYPE); +VariableWidthVector vector = new VarCharVector(field, allocator); +try { + vector.allocateNew(10, 1); + vector.getMutator().setValueCount(4); + fail(); +} catch (Exception ex) { + assertTrue(ex instanceof IndexOutOfBoundsException); +} finally { + vector.clear(); +} + } + } http://git-wip-us.apache.org/repos/asf/drill/blob/1c14d3c3/exec/vector/src/main/codegen/templates/VariableLengthVectors.java -- diff --git a/exec/vector/src/main/codegen/templates/VariableLengthVectors.java b/exec/vector/src/main/codegen/templates/VariableLengthVectors.java index 87dbe95..3dec3f8 100644 --- a/exec/vector/src/main/codegen/templates/VariableLengthVectors.java +++ b/exec/vector/src/main/codegen/templates/VariableLengthVectors.java @@ -700,7 +700,11 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements V @Override public void setValueCount(int valu
[1/5] drill git commit: DRILL-6090: While connecting to drill-bits using JDBC Driver through Zookeeper, a lot of "Curator-Framework-0" threads are created if connection to drill-bit is not successful(
Repository: drill Updated Branches: refs/heads/master 894c0f58e -> 1c14d3c3c DRILL-6090: While connecting to drill-bits using JDBC Driver through Zookeeper, a lot of "Curator-Framework-0" threads are created if connection to drill-bit is not successful(no drill-bits are up/reachable) I am using Drill JDBC driver 1.12.0 to connect to MapR-DB. I am finding the available drill-bits using Zookeepers. When drill-bits are not up or not reachable, the connection is failed with exception: "Failure in connecting to Drill: oadd.org.apache.drill.exec.rpc.RpcException: Failure setting up ZK for client", which is expected, but number of threads created by ZKClusterCoordinator just keeps on increasing. Steps to reproduce the issue Setup a connection with a drill-bit using Apache Drill JDBC driver 1.12.0 through Zookeeper hosts(port 5181) Now stop the drill-bit services or block the drill-bit IPs using iptable rules Truncate catalina logs Try to connect to the drill-bit/hit a code path that requires connection to drill-bits. Take thread dump using kill -QUIT grep -c "Curator-Framework-0" catalina.out Observe that the curator framework thread just keep on accumulating RCA: ZKClusterCoordinator creates curator threads in the constructor ZKClusterCoordinator is instantiated by DrillClient.connect DrillClient.connect is called in DrillConnectionImpl constructor Fix: Call DrillConnectionImpl .cleanup() from all the catch blocks in the DrillConnectionImpl constructor. close apache/drill#1094 Project: http://git-wip-us.apache.org/repos/asf/drill/repo Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/8cb32343 Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/8cb32343 Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/8cb32343 Branch: refs/heads/master Commit: 8cb32343b2816fe97f04efc055e6ffba77a9116a Parents: 894c0f5 Author: milindt <35451050+mili...@users.noreply.github.com> Authored: Wed Jan 17 14:35:33 2018 +0530 Committer: Aman Sinha Committed: Fri Feb 23 14:08:30 2018 -0800 -- .../drill/jdbc/impl/DrillConnectionImpl.java| 130 ++- 1 file changed, 69 insertions(+), 61 deletions(-) -- http://git-wip-us.apache.org/repos/asf/drill/blob/8cb32343/exec/jdbc/src/main/java/org/apache/drill/jdbc/impl/DrillConnectionImpl.java -- diff --git a/exec/jdbc/src/main/java/org/apache/drill/jdbc/impl/DrillConnectionImpl.java b/exec/jdbc/src/main/java/org/apache/drill/jdbc/impl/DrillConnectionImpl.java index 689041c..4c1ac46 100644 --- a/exec/jdbc/src/main/java/org/apache/drill/jdbc/impl/DrillConnectionImpl.java +++ b/exec/jdbc/src/main/java/org/apache/drill/jdbc/impl/DrillConnectionImpl.java @@ -110,73 +110,81 @@ class DrillConnectionImpl extends AvaticaConnection super.setReadOnly(false); this.config = new DrillConnectionConfig(info); +try{ + try { +String connect = null; -try { - String connect = null; - - if (config.isLocal()) { -try { - Class.forName("org.eclipse.jetty.server.Handler"); -} catch (final ClassNotFoundException e) { - throw new SQLNonTransientConnectionException( - "Running Drill in embedded mode using Drill's jdbc-all JDBC" - + " driver Jar file alone is not supported.", e); -} - -final DrillConfig dConfig = DrillConfig.create(info); -this.allocator = RootAllocatorFactory.newRoot(dConfig); -RemoteServiceSet set = GlobalServiceSetReference.SETS.get(); -if (set == null) { - // We're embedded; start a local drill bit. - serviceSet = RemoteServiceSet.getLocalServiceSet(); - set = serviceSet; +if (config.isLocal()) { try { -bit = new Drillbit(dConfig, serviceSet); -bit.run(); - } catch (final UserException e) { -throw new SQLException( -"Failure in starting embedded Drillbit: " + e.getMessage(), -e); - } catch (Exception e) { -// (Include cause exception's text in wrapping exception's text so -// it's more likely to get to user (e.g., via SQLLine), and use -// toString() since getMessage() text doesn't always mention error:) -throw new SQLException("Failure in starting embedded Drillbit: " + e, e); +Class.forName("org.eclipse.jetty.server.Handler"); + } catch (final ClassNotFoundException e) { +throw new SQLNonTransientConnectionException( +"Running Drill in embedded mode using Drill's jdbc-all JDBC" ++ " driver Jar file alone is not supported.", e); + } + + final DrillConfig dConfig = DrillConfig.create(info); + this.allocator = RootAllocato
[2/5] drill git commit: DRILL-6115: SingleMergeExchange is not scaling up when many minor fragments are allocated for a query.
DRILL-6115: SingleMergeExchange is not scaling up when many minor fragments are allocated for a query. DRILL-6115: Refactoring the existing code. close apache/drill#1110 Project: http://git-wip-us.apache.org/repos/asf/drill/repo Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/4f203ea9 Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/4f203ea9 Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/4f203ea9 Branch: refs/heads/master Commit: 4f203ea99e2d6bd5596672a0ea67445720630228 Parents: 8cb3234 Author: Hanumath Rao Maduri Authored: Mon Feb 12 18:19:07 2018 -0800 Committer: Aman Sinha Committed: Fri Feb 23 14:13:40 2018 -0800 -- .../org/apache/drill/exec/ExecConstants.java| 3 + .../physical/base/AbstractPhysicalVisitor.java | 18 -- .../exec/physical/base/PhysicalVisitor.java | 7 - .../physical/config/AbstractMuxExchange.java| 15 ++ .../physical/config/OrderedMuxExchange.java | 52 + .../physical/config/UnorderedMuxExchange.java | 9 +- .../exec/planner/physical/ExchangePrel.java | 13 ++ .../exec/planner/physical/HashPrelUtil.java | 23 ++- .../physical/HashToMergeExchangePrel.java | 4 - .../physical/HashToRandomExchangePrel.java | 77 .../physical/OrderedMuxExchangePrel.java| 74 +++ .../exec/planner/physical/PlannerSettings.java | 1 + .../physical/SingleMergeExchangePrel.java | 21 ++ .../visitor/InsertLocalExchangeVisitor.java | 115 +-- .../planner/physical/visitor/PrelVisitor.java | 1 - .../server/options/SystemOptionManager.java | 1 + .../src/main/resources/drill-module.conf| 1 + .../physical/impl/TestOrderedMuxExchange.java | 193 +++ 18 files changed, 479 insertions(+), 149 deletions(-) -- http://git-wip-us.apache.org/repos/asf/drill/blob/4f203ea9/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java -- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java b/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java index a1a94fa..fb2907d 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java @@ -455,6 +455,9 @@ public final class ExecConstants { public static final String AVERAGE_FIELD_WIDTH_KEY = "planner.memory.average_field_width"; public static final OptionValidator AVERAGE_FIELD_WIDTH = new PositiveLongValidator(AVERAGE_FIELD_WIDTH_KEY, Long.MAX_VALUE); + // Mux Exchange options. + public static final String ORDERED_MUX_EXCHANGE = "planner.enable_ordered_mux_exchange"; + // Resource management boot-time options. public static final String MAX_MEMORY_PER_NODE = "drill.exec.rm.memory_per_node"; http://git-wip-us.apache.org/repos/asf/drill/blob/4f203ea9/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/AbstractPhysicalVisitor.java -- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/AbstractPhysicalVisitor.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/AbstractPhysicalVisitor.java index a3b5f27..9933ddc 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/AbstractPhysicalVisitor.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/AbstractPhysicalVisitor.java @@ -23,7 +23,6 @@ import org.apache.drill.exec.physical.config.FlattenPOP; import org.apache.drill.exec.physical.config.HashAggregate; import org.apache.drill.exec.physical.config.HashJoinPOP; import org.apache.drill.exec.physical.config.HashPartitionSender; -import org.apache.drill.exec.physical.config.HashToRandomExchange; import org.apache.drill.exec.physical.config.IteratorValidator; import org.apache.drill.exec.physical.config.Limit; import org.apache.drill.exec.physical.config.MergeJoinPOP; @@ -32,14 +31,12 @@ import org.apache.drill.exec.physical.config.NestedLoopJoinPOP; import org.apache.drill.exec.physical.config.OrderedPartitionSender; import org.apache.drill.exec.physical.config.ProducerConsumer; import org.apache.drill.exec.physical.config.Project; -import org.apache.drill.exec.physical.config.RangeSender; import org.apache.drill.exec.physical.config.Screen; import org.apache.drill.exec.physical.config.SingleSender; import org.apache.drill.exec.physical.config.Sort; import org.apache.drill.exec.physical.config.StreamingAggregate; import org.apache.drill.exec.physical.config.Trace; import org.apache.drill.exec.physical.config.UnionAll; -import org.apache.drill.exec.physical.config.UnionExchange; import org.apache.drill.exec.physical.config.UnorderedReceiver; import org.
drill git commit: DRILL-6100: Intermittent failure while reading Parquet file footer during planning phase
Repository: drill Updated Branches: refs/heads/master e791ed62b -> 9926eda21 DRILL-6100: Intermittent failure while reading Parquet file footer during planning phase close apache/drill#1097 Project: http://git-wip-us.apache.org/repos/asf/drill/repo Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/9926eda2 Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/9926eda2 Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/9926eda2 Branch: refs/heads/master Commit: 9926eda21c748e96d67bce341a76dac3114002af Parents: e791ed6 Author: Sorabh Hamirwasia Authored: Sun Jan 21 16:30:25 2018 -0800 Committer: Aman Sinha Committed: Wed Jan 24 13:12:31 2018 -0800 -- .../drill/exec/store/parquet/Metadata.java | 19 +-- 1 file changed, 17 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/drill/blob/9926eda2/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/Metadata.java -- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/Metadata.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/Metadata.java index a611c6f..ab3bc19 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/Metadata.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/Metadata.java @@ -18,6 +18,7 @@ package org.apache.drill.exec.store.parquet; import java.io.IOException; +import java.security.PrivilegedExceptionAction; import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -41,6 +42,7 @@ import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.security.UserGroupInformation; import org.apache.parquet.column.statistics.Statistics; import org.apache.parquet.hadoop.ParquetFileReader; import org.apache.parquet.hadoop.metadata.BlockMetaData; @@ -416,8 +418,21 @@ public class Metadata { * Get the metadata for a single file */ private ParquetFileMetadata_v3 getParquetFileMetadata_v3(ParquetTableMetadata_v3 parquetTableMetadata, - FileStatus file) throws IOException { -ParquetMetadata metadata = ParquetFileReader.readFooter(fs.getConf(), file); + final FileStatus file) throws IOException, InterruptedException { +final ParquetMetadata metadata; +final UserGroupInformation processUserUgi = ImpersonationUtil.getProcessUserUGI(); +try { + metadata = processUserUgi.doAs(new PrivilegedExceptionAction() { +public ParquetMetadata run() throws Exception { + return ParquetFileReader.readFooter(fs.getConf(), file); +} + }); +} catch(Exception e) { + logger.error("Exception while reading footer of parquet file [Details - path: {}, owner: {}] as process user {}", +file.getPath(), file.getOwner(), processUserUgi.getShortUserName(), e); + throw e; +} + MessageType schema = metadata.getFileMetaData().getSchema(); //Map originalTypeMap = Maps.newHashMap();
[08/11] drill git commit: DRILL-6049: Misc. hygiene and code cleanup changes
http://git-wip-us.apache.org/repos/asf/drill/blob/e791ed62/exec/vector/src/main/codegen/templates/FixedValueVectors.java -- diff --git a/exec/vector/src/main/codegen/templates/FixedValueVectors.java b/exec/vector/src/main/codegen/templates/FixedValueVectors.java index 1f6a008..79beb52 100644 --- a/exec/vector/src/main/codegen/templates/FixedValueVectors.java +++ b/exec/vector/src/main/codegen/templates/FixedValueVectors.java @@ -106,7 +106,7 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F } @Override - public int getValueCapacity(){ + public int getValueCapacity() { return data.capacity() / VALUE_WIDTH; } @@ -129,7 +129,7 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F @Override public void allocateNew() { -if (!allocateNewSafe()){ +if (!allocateNewSafe()) { throw new OutOfMemoryException("Failure while allocating buffer."); } } @@ -264,12 +264,12 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F } @Override - public TransferPair getTransferPair(BufferAllocator allocator){ + public TransferPair getTransferPair(BufferAllocator allocator) { return new TransferImpl(getField(), allocator); } @Override - public TransferPair getTransferPair(String ref, BufferAllocator allocator){ + public TransferPair getTransferPair(String ref, BufferAllocator allocator) { return new TransferImpl(getField().withPath(ref), allocator); } @@ -278,7 +278,7 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F return new TransferImpl((${minor.class}Vector) to); } - public void transferTo(${minor.class}Vector target){ + public void transferTo(${minor.class}Vector target) { target.clear(); target.data = data.transferOwnership(target.allocator).buffer; target.data.writerIndex(data.writerIndex()); @@ -298,10 +298,10 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F return valueCount * ${type.width}; } - private class TransferImpl implements TransferPair{ + private class TransferImpl implements TransferPair { private ${minor.class}Vector to; -public TransferImpl(MaterializedField field, BufferAllocator allocator){ +public TransferImpl(MaterializedField field, BufferAllocator allocator) { to = new ${minor.class}Vector(field, allocator); } @@ -310,12 +310,12 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F } @Override -public ${minor.class}Vector getTo(){ +public ${minor.class}Vector getTo() { return to; } @Override -public void transfer(){ +public void transfer() { transferTo(to); } @@ -330,7 +330,7 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F } } - public void copyFrom(int fromIndex, int thisIndex, ${minor.class}Vector from){ + public void copyFrom(int fromIndex, int thisIndex, ${minor.class}Vector from) { <#if (type.width > 8)> from.data.getBytes(fromIndex * VALUE_WIDTH, data, thisIndex * VALUE_WIDTH, VALUE_WIDTH); <#else> <#-- type.width <= 8 --> @@ -340,7 +340,7 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F <#-- type.width --> } - public void copyFromSafe(int fromIndex, int thisIndex, ${minor.class}Vector from){ + public void copyFromSafe(int fromIndex, int thisIndex, ${minor.class}Vector from) { while(thisIndex >= getValueCapacity()) { reAlloc(); } @@ -376,24 +376,24 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F } @Override -public boolean isNull(int index){ +public boolean isNull(int index) { return false; } - <#if (type.width > 8)> + public ${minor.javaType!type.javaType} get(int index) { return data.slice(index * VALUE_WIDTH, VALUE_WIDTH); } - <#if (minor.class == "Interval")> -public void get(int index, ${minor.class}Holder holder){ + +public void get(int index, ${minor.class}Holder holder) { final int offsetIndex = index * VALUE_WIDTH; holder.months = data.getInt(offsetIndex); holder.days = data.getInt(offsetIndex + ${minor.daysOffset}); holder.milliseconds = data.getInt(offsetIndex + ${minor.millisecondsOffset}); } -public void get(int index, Nullable${minor.class}Holder holder){ +public void get(int index, Nullable${minor.class}Holder holder) { final int offsetIndex = index * VALUE_WIDTH; holder.isSet = 1; holder.months = data.getInt(offsetIndex); @@ -407,52 +407,25 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F final int months = data.getInt(offsetIndex); final int days
[01/11] drill git commit: DRILL-5851: Empty table during a join operation with a non empty table produces cast exception.
Repository: drill Updated Branches: refs/heads/master a2a56af3e -> e791ed62b DRILL-5851: Empty table during a join operation with a non empty table produces cast exception. close apache/drill#1059 Project: http://git-wip-us.apache.org/repos/asf/drill/repo Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/7d1e81a5 Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/7d1e81a5 Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/7d1e81a5 Branch: refs/heads/master Commit: 7d1e81a5ac0d8ca0b6519dd2c90bf3804c3221b1 Parents: a2a56af Author: Hanumath Maduri Authored: Mon Oct 9 13:08:13 2017 -0700 Committer: Aman Sinha Committed: Tue Jan 23 14:39:53 2018 -0800 -- .../exec/physical/impl/join/HashJoinBatch.java | 21 ++- .../impl/join/HashJoinProbeTemplate.java| 6 +- .../exec/physical/impl/join/MergeJoinBatch.java | 3 +- .../impl/union/UnionAllRecordBatch.java | 1 + .../IteratorValidatorBatchIterator.java | 1 + .../exec/record/AbstractBinaryRecordBatch.java | 11 +++- .../exec/physical/impl/join/JoinTestBase.java | 63 +++ .../impl/join/TestHashJoinAdvanced.java | 23 ++- .../impl/join/TestMergeJoinAdvanced.java| 25 ++-- .../physical/impl/join/TestNestedLoopJoin.java | 64 ++-- 10 files changed, 185 insertions(+), 33 deletions(-) -- http://git-wip-us.apache.org/repos/asf/drill/blob/7d1e81a5/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/HashJoinBatch.java -- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/HashJoinBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/HashJoinBatch.java index 7e2859e..7b679c0 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/HashJoinBatch.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/HashJoinBatch.java @@ -181,7 +181,9 @@ public class HashJoinBatch extends AbstractBinaryRecordBatch { hyperContainer = new ExpandableHyperContainer(vectors); hjHelper.addNewBatch(0); buildBatchIndex++; - setupHashTable(); + if (isFurtherProcessingRequired(rightUpstream) && this.right.getRecordCount() > 0) { +setupHashTable(); + } hashJoinProbe = setupHashJoinProbe(); // Build the container schema and set the counts for (final VectorWrapper w : container) { @@ -212,7 +214,7 @@ public class HashJoinBatch extends AbstractBinaryRecordBatch { } // Store the number of records projected - if (!hashTable.isEmpty() || joinType != JoinRelType.INNER) { + if ((hashTable != null && !hashTable.isEmpty()) || joinType != JoinRelType.INNER) { // Allocate the memory for the vectors in the output container allocateVectors(); @@ -305,11 +307,15 @@ public class HashJoinBatch extends AbstractBinaryRecordBatch { //Setup the underlying hash table // skip first batch if count is zero, as it may be an empty schema batch -if (right.getRecordCount() == 0) { +if (isFurtherProcessingRequired(rightUpstream) && right.getRecordCount() == 0) { for (final VectorWrapper w : right) { w.clear(); } rightUpstream = next(right); + if (isFurtherProcessingRequired(rightUpstream) && + right.getRecordCount() > 0 && hashTable == null) { +setupHashTable(); + } } boolean moreData = true; @@ -535,4 +541,13 @@ public class HashJoinBatch extends AbstractBinaryRecordBatch { } super.close(); } + + /** + * This method checks to see if join processing should be continued further. + * @param upStream up stream operator status. + * @@return true if up stream status is OK or OK_NEW_SCHEMA otherwise false. + */ + private boolean isFurtherProcessingRequired(IterOutcome upStream) { +return upStream == IterOutcome.OK || upStream == IterOutcome.OK_NEW_SCHEMA; + } } http://git-wip-us.apache.org/repos/asf/drill/blob/7d1e81a5/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/HashJoinProbeTemplate.java -- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/HashJoinProbeTemplate.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/HashJoinProbeTemplate.java index 3cdce2f..5c6371a 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/HashJoinProbeTemplate.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/HashJoinProbeTemplate.java @@ -136,7 +136,9 @@ public abstract class HashJoinProbeTemplate implements HashJoinProbe { case OK_NEW_SCHEMA:
[10/11] drill git commit: DRILL-6049: Misc. hygiene and code cleanup changes
http://git-wip-us.apache.org/repos/asf/drill/blob/e791ed62/exec/java-exec/src/main/java/org/apache/drill/exec/ops/OperatorStats.java -- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/ops/OperatorStats.java b/exec/java-exec/src/main/java/org/apache/drill/exec/ops/OperatorStats.java index 1b96f28..a38c3c2 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/ops/OperatorStats.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/ops/OperatorStats.java @@ -32,8 +32,9 @@ import com.carrotsearch.hppc.cursors.IntDoubleCursor; import com.carrotsearch.hppc.cursors.IntLongCursor; import com.carrotsearch.hppc.procedures.IntDoubleProcedure; import com.carrotsearch.hppc.procedures.IntLongProcedure; +import com.google.common.annotations.VisibleForTesting; -public class OperatorStats implements OperatorStatReceiver { +public class OperatorStats { static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(OperatorStats.class); protected final int operatorId; @@ -89,7 +90,8 @@ public class OperatorStats implements OperatorStatReceiver { } } - private OperatorStats(int operatorId, int operatorType, int inputCount, BufferAllocator allocator) { + @VisibleForTesting + public OperatorStats(int operatorId, int operatorType, int inputCount, BufferAllocator allocator) { super(); this.allocator = allocator; this.operatorId = operatorId; @@ -169,7 +171,6 @@ public class OperatorStats implements OperatorStatReceiver { inProcessing = false; } - @Override public synchronized void startWait() { assert !inWait : assertionError("starting waiting"); stopProcessing(); @@ -177,7 +178,6 @@ public class OperatorStats implements OperatorStatReceiver { waitMark = System.nanoTime(); } - @Override public synchronized void stopWait() { assert inWait : assertionError("stopping waiting"); startProcessing(); @@ -203,7 +203,6 @@ public class OperatorStats implements OperatorStatReceiver { .toString(); } - public OperatorProfile getProfile() { final OperatorProfile.Builder b = OperatorProfile // .newBuilder() // @@ -213,14 +212,11 @@ public class OperatorStats implements OperatorStatReceiver { .setProcessNanos(processingNanos) .setWaitNanos(waitNanos); -if(allocator != null){ +if (allocator != null) { b.setPeakLocalMemoryAllocated(allocator.getPeakMemoryAllocation()); } - - addAllMetrics(b); - return b.build(); } @@ -249,7 +245,6 @@ public class OperatorStats implements OperatorStatReceiver { public void apply(int key, long value) { builder.addMetric(MetricValue.newBuilder().setMetricId(key).setLongValue(value)); } - } public void addLongMetrics(OperatorProfile.Builder builder) { @@ -278,22 +273,62 @@ public class OperatorStats implements OperatorStatReceiver { } } - @Override + /** + * Set a stat to the specified long value. Creates the stat + * if the stat does not yet exist. + * + * @param metric the metric to update + * @param value the value to set + */ + public void addLongStat(MetricDef metric, long value){ longMetrics.putOrAdd(metric.metricId(), value, value); } - @Override + @VisibleForTesting + public long getLongStat(MetricDef metric) { +return longMetrics.get(metric.metricId()); + } + + /** + * Add a double value to the existing value. Creates the stat + * (with an initial value of zero) if the stat does not yet + * exist. + * + * @param metric the metric to update + * @param value the value to add to the existing value + */ + public void addDoubleStat(MetricDef metric, double value){ doubleMetrics.putOrAdd(metric.metricId(), value, value); } - @Override + @VisibleForTesting + public double getDoubleStat(MetricDef metric) { +return doubleMetrics.get(metric.metricId()); + } + + /** + * Add a long value to the existing value. Creates the stat + * (with an initial value of zero) if the stat does not yet + * exist. + * + * @param metric the metric to update + * @param value the value to add to the existing value + */ + public void setLongStat(MetricDef metric, long value){ longMetrics.put(metric.metricId(), value); } - @Override + /** + * Set a stat to the specified double value. Creates the stat + * if the stat does not yet exist. + * + * @param metric the metric to update + * @param value the value to set + */ + public void setDoubleStat(MetricDef metric, double value){ doubleMetrics.put(metric.metricId(), value); } @@ -313,5 +348,4 @@ public class OperatorStats implements OperatorStatReceiver { public long getProcessingNanos() { return processingNanos; } - } http://git-wip-us.apache.org/repos/asf/drill/blob/e791ed62/exec/java-exec/src/main/java/org/apache/drill/exec/physical/b
[03/11] drill git commit: DRILL-5868: Support SQL syntax highlighting of queries
DRILL-5868: Support SQL syntax highlighting of queries Based on the commit for DRILL-5981 (PR #1043), this commit further leverages the Ace JavaScript library with customizations specific to Drill. This commit introduces the following to the Query Editor (including the Edit Query tab within an existing profile to rerunning the query). 1. Syntax highlighting (This is supported for submitted query profiles 2. Autocomplete supported in editors 3. Specifying Drill specific keywords and functions in visible autocomplete 4. Key snippets (template SQLs) allowing for rapid writing of syntax: i. Query System Tables ii. CView, CTAS and CTempTAS iii. Alter Session iv. Explain and Select * queries NOTE: The lists for #3 and #4 are not exhaustive. As more features are added to Drill, these lists can be expanded. Updates based on review comments 1. Disabled warning message 2. Extended reserved keyword list 3. Fixed bugs List: Update 1: Bug fix for when Impersonation is enabled Update 2: Remove the duplicate editor that might get injected Update 3: Removed trailing whitespaces in Javascripts close apache/drill#1084 Project: http://git-wip-us.apache.org/repos/asf/drill/repo Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/4c8a55b8 Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/4c8a55b8 Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/4c8a55b8 Branch: refs/heads/master Commit: 4c8a55b825b91ca26073f9ba059e18cdaa08670d Parents: a2058fd Author: Kunal Khatua Authored: Mon Jan 22 12:01:32 2018 -0800 Committer: Aman Sinha Committed: Tue Jan 23 14:42:23 2018 -0800 -- .../src/main/resources/rest/profile/profile.ftl | 83 +++- .../src/main/resources/rest/query/query.ftl | 43 +++- .../rest/static/js/ace-code-editor/ace.js | 3 +- .../js/ace-code-editor/ext-language_tools.js| 4 + .../static/js/ace-code-editor/mode-snippets.js | 199 +++ .../rest/static/js/ace-code-editor/mode-sql.js | 135 - .../static/js/ace-code-editor/snippets/sql.js | 48 + .../js/ace-code-editor/theme-sqlserver.js | 138 + .../src/main/resources/rest/storage/update.ftl | 6 +- 9 files changed, 649 insertions(+), 10 deletions(-) -- http://git-wip-us.apache.org/repos/asf/drill/blob/4c8a55b8/exec/java-exec/src/main/resources/rest/profile/profile.ftl -- diff --git a/exec/java-exec/src/main/resources/rest/profile/profile.ftl b/exec/java-exec/src/main/resources/rest/profile/profile.ftl index 1f38d2e..7f4474c 100644 --- a/exec/java-exec/src/main/resources/rest/profile/profile.ftl +++ b/exec/java-exec/src/main/resources/rest/profile/profile.ftl @@ -19,6 +19,13 @@ + + + + + + + var globalconfig = { @@ -65,7 +72,7 @@ table.sortable thead .sorting_desc { background-image: url("/static/img/black-de-${model.getProfile().query}+${model.getProfile().query}${model.profile.plan}@@ -84,9 +91,8 @@ table.sortable thead .sorting_desc { background-image: url("/static/img/black-de #if>