[drill] branch master updated: DRILL-7710: Fix TestMetastoreCommands#testDefaultSegment test
This is an automated email from the ASF dual-hosted git repository.
arina pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git
The following commit(s) were added to refs/heads/master by this push:
new 9426687 DRILL-7710: Fix TestMetastoreCommands#testDefaultSegment test
9426687 is described below
commit 94266872b4864f81b1a6486f2d1bac70186afb77
Author: Volodymyr Vysotskyi
AuthorDate: Tue Apr 21 21:17:19 2020 +0300
DRILL-7710: Fix TestMetastoreCommands#testDefaultSegment test
---
.../drill/exec/metastore/TestMetastoreWithEasyFormatPlugin.java | 5 ++---
.../org/apache/drill/exec/sql/TestInfoSchemaWithMetastore.java| 7 +--
.../java/org/apache/drill/exec/sql/TestMetastoreCommands.java | 8
3 files changed, 7 insertions(+), 13 deletions(-)
diff --git
a/exec/java-exec/src/test/java/org/apache/drill/exec/metastore/TestMetastoreWithEasyFormatPlugin.java
b/exec/java-exec/src/test/java/org/apache/drill/exec/metastore/TestMetastoreWithEasyFormatPlugin.java
index 67fb1de..a6924c0 100644
---
a/exec/java-exec/src/test/java/org/apache/drill/exec/metastore/TestMetastoreWithEasyFormatPlugin.java
+++
b/exec/java-exec/src/test/java/org/apache/drill/exec/metastore/TestMetastoreWithEasyFormatPlugin.java
@@ -74,7 +74,7 @@ import static org.junit.Assert.assertTrue;
@Category({SlowTest.class, MetastoreTest.class})
public class TestMetastoreWithEasyFormatPlugin extends ClusterTest {
- private static String SCHEMA_STRING = "'inline=(" +
+ private static final String SCHEMA_STRING = "'inline=(" +
"`o_orderkey` INT not null, " +
"`o_custkey` INT not null, " +
"`o_orderstatus` VARCHAR not null, " +
@@ -105,7 +105,7 @@ public class TestMetastoreWithEasyFormatPlugin extends
ClusterTest {
@BeforeClass
public static void setUp() throws Exception {
ClusterFixtureBuilder builder = ClusterFixture.builder(dirTestWatcher);
-builder.configProperty(ExecConstants.ZK_ROOT,
dirTestWatcher.getRootDir().getPath());
+builder.configProperty(ExecConstants.ZK_ROOT,
dirTestWatcher.getRootDir().getAbsolutePath());
startCluster(builder);
}
@@ -1087,7 +1087,6 @@ public class TestMetastoreWithEasyFormatPlugin extends
ClusterTest {
.sql("select * from dfs.tmp.`%s` limit 21", tableName)
.planMatcher()
.include("Limit", "numFiles=3")
- .exclude("bird is the word")
.match();
} finally {
run("analyze table dfs.tmp.`%s` drop metadata if exists", tableName);
diff --git
a/exec/java-exec/src/test/java/org/apache/drill/exec/sql/TestInfoSchemaWithMetastore.java
b/exec/java-exec/src/test/java/org/apache/drill/exec/sql/TestInfoSchemaWithMetastore.java
index 28d6a6a..9e07769 100644
---
a/exec/java-exec/src/test/java/org/apache/drill/exec/sql/TestInfoSchemaWithMetastore.java
+++
b/exec/java-exec/src/test/java/org/apache/drill/exec/sql/TestInfoSchemaWithMetastore.java
@@ -44,10 +44,8 @@ import org.apache.drill.test.ClusterFixtureBuilder;
import org.apache.drill.test.ClusterTest;
import org.apache.hadoop.fs.Path;
import org.junit.BeforeClass;
-import org.junit.ClassRule;
import org.junit.Test;
import org.junit.experimental.categories.Category;
-import org.junit.rules.TemporaryFolder;
import java.time.LocalDateTime;
import java.time.ZoneId;
@@ -71,15 +69,12 @@ public class TestInfoSchemaWithMetastore extends
ClusterTest {
InfoSchemaConstants.TBLS_COL_NUM_ROWS,
InfoSchemaConstants.TBLS_COL_LAST_MODIFIED_TIME);
- @ClassRule
- public static TemporaryFolder root = new TemporaryFolder();
-
private static Metastore metastore;
@BeforeClass
public static void setup() throws Exception {
ClusterFixtureBuilder builder = ClusterFixture.builder(dirTestWatcher);
-builder.configProperty(ExecConstants.ZK_ROOT, root.getRoot().toString());
+builder.configProperty(ExecConstants.ZK_ROOT,
dirTestWatcher.getRootDir().getAbsolutePath());
builder.sessionOption(ExecConstants.METASTORE_ENABLED, true);
startCluster(builder);
MetastoreRegistry metastoreRegistry =
client.cluster().drillbit().getContext().getMetastoreRegistry();
diff --git
a/exec/java-exec/src/test/java/org/apache/drill/exec/sql/TestMetastoreCommands.java
b/exec/java-exec/src/test/java/org/apache/drill/exec/sql/TestMetastoreCommands.java
index 09d9dbf..72989b3 100644
---
a/exec/java-exec/src/test/java/org/apache/drill/exec/sql/TestMetastoreCommands.java
+++
b/exec/java-exec/src/test/java/org/apache/drill/exec/sql/TestMetastoreCommands.java
@@ -188,7 +188,7 @@ public class TestMetastoreCommands extends ClusterTest {
@BeforeClass
public static void setUp() throws Exception {
ClusterFixtureBuilder builder = ClusterFixture.builder(dirTestWatcher);
-builder.configProperty(ExecConstants.ZK_ROOT,
dirTestWatcher.getRootDir().getPath
[drill] branch master updated: DRILL-7713: Upgrade misc libraries which outdated versions have reported vulnerabilities
This is an automated email from the ASF dual-hosted git repository.
arina pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git
The following commit(s) were added to refs/heads/master by this push:
new cd370d0 DRILL-7713: Upgrade misc libraries which outdated versions
have reported vulnerabilities
cd370d0 is described below
commit cd370d0a07058e4c0455543533437ef058396679
Author: Arina Ielchiieva
AuthorDate: Sat Apr 18 18:58:20 2020 +0300
DRILL-7713: Upgrade misc libraries which outdated versions have reported
vulnerabilities
1. Jackson
2. Retrofit
3. Commons-beanutils
4. Xalan
5. Xerdes
6. Commons-codec
7. Snakeyaml
8. Metadata-extractor
9. Protostuff
---
contrib/storage-hive/hive-exec-shade/pom.xml | 6 +
contrib/storage-opentsdb/pom.xml | 8 +-
exec/java-exec/pom.xml | 1 -
.../drill/exec/planner/PhysicalPlanReader.java | 2 +-
.../drill/exec/store/image/ImageRecordReader.java | 13 +-
.../exec/store/pojo/DynamicPojoRecordReader.java | 12 +-
.../src/test/resources/store/image/jpeg.json | 124 -
.../src/test/resources/store/image/mov.json| 46 +++
.../src/test/resources/store/image/mp4.json| 57
.../src/test/resources/store/image/png.json| 35 +++--
.../src/test/resources/store/image/psd.json| 146 ++---
.../src/test/resources/store/image/tiff.json | 119 -
exec/jdbc-all/pom.xml | 4 +-
pom.xml| 85 ++--
protocol/pom.xml | 2 -
15 files changed, 398 insertions(+), 262 deletions(-)
diff --git a/contrib/storage-hive/hive-exec-shade/pom.xml
b/contrib/storage-hive/hive-exec-shade/pom.xml
index 778ba68..ac56de4 100644
--- a/contrib/storage-hive/hive-exec-shade/pom.xml
+++ b/contrib/storage-hive/hive-exec-shade/pom.xml
@@ -208,6 +208,12 @@
shaded/parquet/org/**
+
+ com.fasterxml.jackson.core:*
+
+module-info.class
+
+
diff --git a/contrib/storage-opentsdb/pom.xml b/contrib/storage-opentsdb/pom.xml
index 5229802..23f193d 100644
--- a/contrib/storage-opentsdb/pom.xml
+++ b/contrib/storage-opentsdb/pom.xml
@@ -30,6 +30,10 @@
contrib/opentsdb-storage-plugin
+
+2.8.1
+
+
org.apache.drill.exec
@@ -64,12 +68,12 @@
com.squareup.retrofit2
retrofit
-2.1.0
+${retrofit.version}
com.squareup.retrofit2
converter-jackson
-2.1.0
+${retrofit.version}
com.madhukaraphatak
diff --git a/exec/java-exec/pom.xml b/exec/java-exec/pom.xml
index 49890c9..809539d 100644
--- a/exec/java-exec/pom.xml
+++ b/exec/java-exec/pom.xml
@@ -608,7 +608,6 @@
com.drewnoakes
metadata-extractor
- 2.11.0
fr.bmartel
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/PhysicalPlanReader.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/PhysicalPlanReader.java
index f693744..45975f2 100644
---
a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/PhysicalPlanReader.java
+++
b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/PhysicalPlanReader.java
@@ -67,7 +67,7 @@ public class PhysicalPlanReader {
.addSerializer(MajorType.class, new MajorTypeSerDe.Se())
.addDeserializer(MajorType.class, new MajorTypeSerDe.De())
.addDeserializer(DynamicPojoRecordReader.class,
-new StdDelegatingDeserializer<>(new
DynamicPojoRecordReader.Converter<>(lpMapper)))
+new StdDelegatingDeserializer<>(new
DynamicPojoRecordReader.Converter(lpMapper)))
.addSerializer(Path.class, new PathSerDe.Se());
lpMapper.registerModule(serDeModule);
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/store/image/ImageRecordReader.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/store/image/ImageRecordReader.java
index 08ed4fd..fa01743 100644
---
a/exec/java-exec/src/main/java/org/apache/drill/exec/store/image/ImageRecordReader.java
+++
b/exec/java-exec/src/main/java/org/apache/drill/exec/store/image/ImageRecordReader.java
@@ -15,7 +15,6 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
package org.apache.drill.exec.store.image;
import io.netty.buffer.DrillBuf;
@@ -23,14 +22,14 @@ import io.netty.buffer.DrillBuf;
import java.io.BufferedInputStream;
import java.util.Date;
import java.util.HashSet;
-import java.util.Iterator;
import java.util.List;
import
[drill] branch gh-pages updated: Update Maven version to 3.6.3
This is an automated email from the ASF dual-hosted git repository. arina pushed a commit to branch gh-pages in repository https://gitbox.apache.org/repos/asf/drill.git The following commit(s) were added to refs/heads/gh-pages by this push: new c5c040c Update Maven version to 3.6.3 c5c040c is described below commit c5c040cedb1b69e6bd8d1d3753a894d198dd87b7 Author: Arina Ielchiieva AuthorDate: Fri Apr 17 15:03:12 2020 +0300 Update Maven version to 3.6.3 --- .../develop-drill/010-compiling-drill-from-source.md| 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/_docs/developer-information/develop-drill/010-compiling-drill-from-source.md b/_docs/developer-information/develop-drill/010-compiling-drill-from-source.md index 053b248..b341660 100644 --- a/_docs/developer-information/develop-drill/010-compiling-drill-from-source.md +++ b/_docs/developer-information/develop-drill/010-compiling-drill-from-source.md @@ -13,7 +13,7 @@ Starting in Drill 1.15, you can build and deploy Apache Drill on the MapR platfo ## Prerequisites Apache Drill requires: -- Apache Maven 3.3.1 or later +- Apache Maven 3.6.3 or later - Oracle or OpenJDK 8 Apache Drill on the MapR platform also requires:
[drill] branch master updated: DRILL-7704: Update Maven to 3.6.3
This is an automated email from the ASF dual-hosted git repository.
arina pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git
The following commit(s) were added to refs/heads/master by this push:
new 20ad3c9 DRILL-7704: Update Maven to 3.6.3
20ad3c9 is described below
commit 20ad3c9837e9ada149c246fc7a4ac1fe02de6fe8
Author: Arina Ielchiieva
AuthorDate: Thu Apr 16 17:35:53 2020 +0300
DRILL-7704: Update Maven to 3.6.3
---
Dockerfile | 2 +-
docs/dev/Docker.md | 2 +-
docs/dev/Environment.md | 4 ++--
docs/dev/InstantiatingComponents.md | 4 ++--
exec/java-exec/pom.xml | 15 +--
pom.xml | 13 +
tools/fmpp/pom.xml | 4 ++--
7 files changed, 26 insertions(+), 18 deletions(-)
diff --git a/Dockerfile b/Dockerfile
index ab32b6c..cfe3a2d 100755
--- a/Dockerfile
+++ b/Dockerfile
@@ -20,7 +20,7 @@
# Drill and copies built binaries into the target image based on
openjdk:8u232-jdk image.
# Uses intermediate image for building Drill to reduce target image size
-FROM maven:3.6-jdk-8 as build
+FROM maven:3.6.3-jdk-8 as build
# Copy project sources into the container
COPY . /src
diff --git a/docs/dev/Docker.md b/docs/dev/Docker.md
index 8b1495d..3e25b56 100644
--- a/docs/dev/Docker.md
+++ b/docs/dev/Docker.md
@@ -4,7 +4,7 @@
To build an Apache Drill docker image, you need to have the following
software installed on your system to successfully complete a build.
* [Java
8](http://www.oracle.com/technetwork/java/javase/downloads/jdk8-downloads-2133151.html)
- * [Maven 3.3.3 or greater](https://maven.apache.org/download.cgi)
+ * [Maven 3.6.3 or greater](https://maven.apache.org/download.cgi)
* [Docker
CE](https://store.docker.com/search?type=edition=community)
If you are using an older Mac or PC, additionally configure
[docker-machine](https://docs.docker.com/machine/overview/#what-is-docker-machine)
on your system
diff --git a/docs/dev/Environment.md b/docs/dev/Environment.md
index 2110f97..47d37c1 100644
--- a/docs/dev/Environment.md
+++ b/docs/dev/Environment.md
@@ -4,7 +4,7 @@
Currently, the Apache Drill build process is known to work on Linux, Windows
and OSX. To build, you need to have the following software installed on your
system to successfully complete a build.
* Java 8
- * Maven 3.3.3 or greater
+ * Maven 3.6.3 or greater
## Confirm settings
# java -version
@@ -13,7 +13,7 @@ Currently, the Apache Drill build process is known to work on
Linux, Windows and
Java HotSpot(TM) 64-Bit Server VM (build 25.161-b12, mixed mode)
# mvn --version
-Apache Maven 3.3.3
+Apache Maven 3.6.3
## Checkout
diff --git a/docs/dev/InstantiatingComponents.md
b/docs/dev/InstantiatingComponents.md
index 80ba6b1..16f56b9 100644
--- a/docs/dev/InstantiatingComponents.md
+++ b/docs/dev/InstantiatingComponents.md
@@ -2,8 +2,8 @@
There are a few techniques for creating instances classes in unit tests:
-* Use a mocking library. **(Depracated)**
-* Provide a simple impementation of an interface
+* Use a mocking library. **(Deprecated)**
+* Provide a simple implementation of an interface
* Build a real instance of class using the class's builders / constructors
* Use the [ClusterFixture](ClusterFixture.md) or
[OperatorFixture](OperatorFixture.md) classes to create instances of objects.
diff --git a/exec/java-exec/pom.xml b/exec/java-exec/pom.xml
index 6c0aa18..49890c9 100644
--- a/exec/java-exec/pom.xml
+++ b/exec/java-exec/pom.xml
@@ -27,8 +27,11 @@
drill-java-exec
exec/Java Execution Engine
+
1.8-rev2
+1.1.0
+3.3.4
@@ -557,31 +560,31 @@
org.apache.maven
maven-embedder
- ${maven.embedder.version}
+ ${maven.version}
test
org.apache.maven
maven-compat
- 3.5.3
+ ${maven.version}
test
org.eclipse.aether
aether-connector-basic
- 1.1.0
+ ${aether.version}
test
org.eclipse.aether
aether-transport-wagon
- 1.1.0
+ ${aether.version}
test
org.apache.maven.wagon
wagon-http
- 3.0.0
+ ${wagon.version}
test
@@ -593,7 +596,7 @@
org.apache.maven.wagon
wagon-provider-api
- 3.0.0
+ ${wagon.version}
test
diff --git a/pom.xml b/pom.xml
index 6e4bdd1..515988a 100644
--- a/pom.xml
+++ b/pom.xml
@@ -71,7 +71,6 @@
2.4
1.3
-3.5.3
4.3.0
2.23.2
1.47
@@ -102,7 +101,7 @@
true
apache/drill
4.8-1
-3.3.3
+3.6.3
3.6
1.6
1.6
@@ -117,6 +116,7 @@
4.5.12
0.13.0
10.14.2.0
+1.4
@@ -256,7 +256,7 @@
org.apache.maven
maven-embedder
[drill] branch master updated (eb08d04 -> e7afccd)
This is an automated email from the ASF dual-hosted git repository.
arina pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git.
from eb08d04 DRILL-7429: Wrong column order when selecting complex data
using Hive storage plugin
new ae25994 DRILL-7675: Work around for partitions sender memory use
new 912698b DRILL-7683: Add "message parsing" to new JSON loader
new ffaaad6 DRILL-7668: Allow Time Bucket Function to Accept Floats and
Timestamps
new 84450d8 DRILL-7694: Register drill.queries.* counter metrics on
Drillbit startup
new e7afccd DRILL-7678: Update Yauaa dependency
The 5 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../apache/drill/exec/metrics/DrillCounters.java | 73 +
contrib/udfs/pom.xml | 2 +-
.../drill/exec/udfs/TimeBucketFunctions.java | 86 +-
.../exec/udfs/UserAgentAnalyzerProvider.java} | 22 +--
.../apache/drill/exec/udfs/UserAgentFunctions.java | 27 ++--
.../drill/exec/udfs/TestTimeBucketFunction.java| 66 ++--
.../drill/exec/udfs/TestUserAgentFunctions.java| 4 +-
.../java/org/apache/drill/exec/ExecConstants.java | 7 +-
.../apache/drill/exec/compile/ClassBuilder.java| 13 +-
.../exec/physical/impl/join/HashJoinBatch.java | 9 +-
.../physical/impl/partitionsender/Partitioner.java | 5 +-
.../impl/partitionsender/PartitionerTemplate.java | 68 +---
.../impl/scan/framework/SchemaNegotiator.java | 5 +
.../impl/scan/framework/SchemaNegotiatorImpl.java | 13 ++
.../org/apache/drill/exec/record/BatchSchema.java | 30 ++--
.../apache/drill/exec/record/VectorContainer.java | 14 ++
.../apache/drill/exec/server/DrillbitContext.java | 8 +
.../exec/server/options/BaseOptionManager.java | 5 +
.../drill/exec/server/options/OptionSet.java | 14 +-
.../exec/server/options/SystemOptionManager.java | 1 +
.../exec/store/easy/json/loader/JsonLoader.java| 8 +-
.../store/easy/json/loader/JsonLoaderImpl.java | 105 ++--
.../store/easy/json/loader/JsonLoaderOptions.java | 9 ++
.../exec/store/easy/json/parser/ErrorFactory.java | 8 +-
.../easy/json/parser/JsonStructureOptions.java | 14 ++
.../easy/json/parser/JsonStructureParser.java | 94 ++-
...stractElementParser.java => MessageParser.java} | 33 ++--
.../exec/store/easy/json/parser/RootParser.java| 34
.../easy/json/parser/SimpleMessageParser.java | 141
.../org/apache/drill/exec/vector/CopyUtil.java | 26 ++-
.../apache/drill/exec/work/foreman/Foreman.java| 31 ++--
.../drill/exec/work/foreman/QueryManager.java | 23 ++-
.../exec/work/foreman/QueryStateProcessor.java | 34 ++--
.../java-exec/src/main/resources/drill-module.conf | 1 +
.../store/easy/json/loader/BaseJsonLoaderTest.java | 13 +-
.../store/easy/json/parser/BaseTestJsonParser.java | 33 ++--
.../easy/json/parser/TestJsonParserMessage.java| 179 +
.../java/org/apache/drill/test/ClusterFixture.java | 35 ++--
.../apache/drill/test/ClusterFixtureBuilder.java | 2 -
.../drill/exec/record/MaterializedField.java | 75 ++---
40 files changed, 1099 insertions(+), 271 deletions(-)
create mode 100644
common/src/main/java/org/apache/drill/exec/metrics/DrillCounters.java
copy
contrib/{storage-kudu/src/main/java/org/apache/drill/exec/store/kudu/KuduScanSpec.java
=>
udfs/src/main/java/org/apache/drill/exec/udfs/UserAgentAnalyzerProvider.java}
(62%)
copy
exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/parser/{AbstractElementParser.java
=> MessageParser.java} (52%)
create mode 100644
exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/parser/SimpleMessageParser.java
create mode 100644
exec/java-exec/src/test/java/org/apache/drill/exec/store/easy/json/parser/TestJsonParserMessage.java
[drill] 05/05: DRILL-7678: Update Yauaa dependency
This is an automated email from the ASF dual-hosted git repository.
arina pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git
commit e7afccd376cdebbdee6192bbc7de0e504fe0ae6c
Author: Niels Basjes
AuthorDate: Tue Mar 31 11:44:30 2020 +0200
DRILL-7678: Update Yauaa dependency
closes #2044
---
contrib/udfs/pom.xml | 2 +-
.../drill/exec/udfs/UserAgentAnalyzerProvider.java | 36 ++
.../apache/drill/exec/udfs/UserAgentFunctions.java | 27 +---
.../drill/exec/udfs/TestUserAgentFunctions.java| 4 +--
4 files changed, 55 insertions(+), 14 deletions(-)
diff --git a/contrib/udfs/pom.xml b/contrib/udfs/pom.xml
index be24c96..6015f69 100644
--- a/contrib/udfs/pom.xml
+++ b/contrib/udfs/pom.xml
@@ -66,7 +66,7 @@
nl.basjes.parse.useragent
yauaa
- 5.13
+ 5.16
diff --git
a/contrib/udfs/src/main/java/org/apache/drill/exec/udfs/UserAgentAnalyzerProvider.java
b/contrib/udfs/src/main/java/org/apache/drill/exec/udfs/UserAgentAnalyzerProvider.java
new file mode 100644
index 000..5094527
--- /dev/null
+++
b/contrib/udfs/src/main/java/org/apache/drill/exec/udfs/UserAgentAnalyzerProvider.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.drill.exec.udfs;
+
+import nl.basjes.parse.useragent.UserAgentAnalyzer;
+
+public class UserAgentAnalyzerProvider {
+
+ public static UserAgentAnalyzer getInstance() {
+return UserAgentAnalyzerHolder.INSTANCE;
+ }
+
+ private static class UserAgentAnalyzerHolder {
+private static final UserAgentAnalyzer INSTANCE =
UserAgentAnalyzer.newBuilder()
+.dropTests()
+.hideMatcherLoadStats()
+.immediateInitialization()
+.build();
+ }
+}
diff --git
a/contrib/udfs/src/main/java/org/apache/drill/exec/udfs/UserAgentFunctions.java
b/contrib/udfs/src/main/java/org/apache/drill/exec/udfs/UserAgentFunctions.java
index f684a2d..40300d6 100644
---
a/contrib/udfs/src/main/java/org/apache/drill/exec/udfs/UserAgentFunctions.java
+++
b/contrib/udfs/src/main/java/org/apache/drill/exec/udfs/UserAgentFunctions.java
@@ -46,11 +46,14 @@ public class UserAgentFunctions {
DrillBuf outBuffer;
@Workspace
-nl.basjes.parse.useragent.UserAgentAnalyzerDirect uaa;
+nl.basjes.parse.useragent.UserAgentAnalyzer uaa;
+
+@Workspace
+java.util.List allFields;
public void setup() {
- uaa =
nl.basjes.parse.useragent.UserAgentAnalyzerDirect.newBuilder().dropTests().hideMatcherLoadStats().build();
- uaa.getAllPossibleFieldNamesSorted();
+ uaa = org.apache.drill.exec.udfs.UserAgentAnalyzerProvider.getInstance();
+ allFields = uaa.getAllPossibleFieldNamesSorted();
}
public void eval() {
@@ -60,7 +63,7 @@ public class UserAgentFunctions {
nl.basjes.parse.useragent.UserAgent agent = uaa.parse(userAgentString);
- for (String fieldName : agent.getAvailableFieldNamesSorted()) {
+ for (String fieldName: allFields) {
org.apache.drill.exec.expr.holders.VarCharHolder rowHolder = new
org.apache.drill.exec.expr.holders.VarCharHolder();
String field = agent.getValue(fieldName);
@@ -92,11 +95,14 @@ public class UserAgentFunctions {
DrillBuf outBuffer;
@Workspace
-nl.basjes.parse.useragent.UserAgentAnalyzerDirect uaa;
+nl.basjes.parse.useragent.UserAgentAnalyzer uaa;
+
+@Workspace
+java.util.List allFields;
public void setup() {
- uaa =
nl.basjes.parse.useragent.UserAgentAnalyzerDirect.newBuilder().dropTests().hideMatcherLoadStats().build();
- uaa.getAllPossibleFieldNamesSorted();
+ uaa = org.apache.drill.exec.udfs.UserAgentAnalyzerProvider.getInstance();
+ allFields = uaa.getAllPossibleFieldNamesSorted();
}
public void eval() {
@@ -111,7 +117,7 @@ public class UserAgentFunctions {
nl.basjes.parse.useragent.UserAgent agent = uaa.parse(userAgentString);
- for (String fieldName : agent.getAvailableFieldNamesSorted()) {
+ for (String fieldName: allFields) {
org.apache.drill.exec.expr.holders.Var
[drill] 02/05: DRILL-7683: Add "message parsing" to new JSON loader
This is an automated email from the ASF dual-hosted git repository.
arina pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git
commit 912698b58dca284efe525971707d53aebf9f9f3c
Author: Paul Rogers
AuthorDate: Tue Mar 31 13:23:05 2020 -0700
DRILL-7683: Add "message parsing" to new JSON loader
Adds the ability to parse "extra" JSON around the data payload,
as often needed for a REST API.
closes #2045
---
.../impl/scan/framework/SchemaNegotiator.java | 5 +
.../impl/scan/framework/SchemaNegotiatorImpl.java | 13 ++
.../exec/store/easy/json/loader/JsonLoader.java| 8 +-
.../store/easy/json/loader/JsonLoaderImpl.java | 105 ++--
.../store/easy/json/loader/JsonLoaderOptions.java | 9 ++
.../exec/store/easy/json/parser/ErrorFactory.java | 8 +-
.../easy/json/parser/JsonStructureOptions.java | 14 ++
.../easy/json/parser/JsonStructureParser.java | 94 ++-
.../exec/store/easy/json/parser/MessageParser.java | 43 +
.../exec/store/easy/json/parser/RootParser.java| 34
.../easy/json/parser/SimpleMessageParser.java | 141
.../store/easy/json/loader/BaseJsonLoaderTest.java | 13 +-
.../store/easy/json/parser/BaseTestJsonParser.java | 33 ++--
.../easy/json/parser/TestJsonParserMessage.java| 179 +
14 files changed, 654 insertions(+), 45 deletions(-)
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/scan/framework/SchemaNegotiator.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/scan/framework/SchemaNegotiator.java
index 7c9a753..b2a793d 100644
---
a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/scan/framework/SchemaNegotiator.java
+++
b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/scan/framework/SchemaNegotiator.java
@@ -22,6 +22,9 @@ import org.apache.drill.exec.ops.OperatorContext;
import org.apache.drill.exec.physical.resultSet.ResultSetLoader;
import org.apache.drill.exec.physical.resultSet.RowSetLoader;
import org.apache.drill.exec.record.metadata.TupleMetadata;
+import org.apache.drill.exec.server.options.OptionSet;
+
+import com.typesafe.config.Config;
/**
* Negotiates the table schema with the scanner framework and provides
@@ -100,6 +103,8 @@ import org.apache.drill.exec.record.metadata.TupleMetadata;
public interface SchemaNegotiator {
OperatorContext context();
+ Config drillConfig();
+ OptionSet queryOptions();
/**
* Specify an advanced error context which allows the reader to
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/scan/framework/SchemaNegotiatorImpl.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/scan/framework/SchemaNegotiatorImpl.java
index 8763dd4..64bac43 100644
---
a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/scan/framework/SchemaNegotiatorImpl.java
+++
b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/scan/framework/SchemaNegotiatorImpl.java
@@ -21,8 +21,11 @@ import org.apache.drill.common.exceptions.CustomErrorContext;
import org.apache.drill.exec.ops.OperatorContext;
import org.apache.drill.exec.physical.resultSet.ResultSetLoader;
import org.apache.drill.exec.record.metadata.TupleMetadata;
+import org.apache.drill.exec.server.options.OptionSet;
import org.apache.drill.exec.vector.ValueVector;
+import com.typesafe.config.Config;
+
/**
* Implementation of the schema negotiation between scan operator and
* batch reader. Anticipates that the select list (and/or the list of
@@ -94,6 +97,16 @@ public class SchemaNegotiatorImpl implements
SchemaNegotiator {
}
@Override
+ public Config drillConfig() {
+return context().getFragmentContext().getConfig();
+ }
+
+ @Override
+ public OptionSet queryOptions() {
+return context().getFragmentContext().getOptions();
+ }
+
+ @Override
public CustomErrorContext parentErrorContext() {
return framework.errorContext();
}
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/loader/JsonLoader.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/loader/JsonLoader.java
index 9d9afed..e4c5a0f 100644
---
a/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/loader/JsonLoader.java
+++
b/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/loader/JsonLoader.java
@@ -56,13 +56,7 @@ public interface JsonLoader {
* @throws RuntimeException for unexpected errors, most often due
* to code errors
*/
- boolean next();
-
- /**
- * Indicates that a batch is complete. Tells the loader to materialize
- * any deferred null fields. (See {@link TupleListener} for details.)
- */
- void endBatch();
+ boolean readBatch();
/**
* Releases resources held by this class including the input stream.
diff --git
a/exec/jav
[drill] 03/05: DRILL-7668: Allow Time Bucket Function to Accept Floats and Timestamps
This is an automated email from the ASF dual-hosted git repository.
arina pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git
commit ffaaad68475181c8aa9af171c9f823da3306f166
Author: Charles Givre
AuthorDate: Mon Apr 6 16:11:31 2020 -0400
DRILL-7668: Allow Time Bucket Function to Accept Floats and Timestamps
closes #2040
---
.../drill/exec/udfs/TimeBucketFunctions.java | 86 +-
.../drill/exec/udfs/TestTimeBucketFunction.java| 66 +
2 files changed, 132 insertions(+), 20 deletions(-)
diff --git
a/contrib/udfs/src/main/java/org/apache/drill/exec/udfs/TimeBucketFunctions.java
b/contrib/udfs/src/main/java/org/apache/drill/exec/udfs/TimeBucketFunctions.java
index b7f3fae..10efd5c 100644
---
a/contrib/udfs/src/main/java/org/apache/drill/exec/udfs/TimeBucketFunctions.java
+++
b/contrib/udfs/src/main/java/org/apache/drill/exec/udfs/TimeBucketFunctions.java
@@ -23,6 +23,8 @@ import
org.apache.drill.exec.expr.annotations.FunctionTemplate;
import org.apache.drill.exec.expr.annotations.Output;
import org.apache.drill.exec.expr.annotations.Param;
import org.apache.drill.exec.expr.holders.BigIntHolder;
+import org.apache.drill.exec.expr.holders.Float8Holder;
+import org.apache.drill.exec.expr.holders.TimeStampHolder;
public class TimeBucketFunctions {
@@ -59,9 +61,9 @@ public class TimeBucketFunctions {
long timestamp = inputDate.value;
// Get the interval in milliseconds and convert to nanoseconds
- long intervalToAdd = interval.value * 100;
+ long groupByInterval = interval.value * 100;
- out.value = timestamp - (timestamp % intervalToAdd);
+ out.value = timestamp - (timestamp % groupByInterval);
}
}
@@ -97,9 +99,85 @@ public class TimeBucketFunctions {
long timestamp = inputDate.value;
// Get the interval in milliseconds
- long intervalToAdd = interval.value;
+ long groupByInterval = interval.value;
- out.value = timestamp - (timestamp % intervalToAdd);
+ out.value = timestamp - (timestamp % groupByInterval);
+}
+ }
+
+ /**
+ * This function is used for facilitating time series analysis by creating
buckets of time intervals. See
+ *
https://blog.timescale.com/blog/simplified-time-series-analytics-using-the-time_bucket-function/
for usage. The function takes two arguments:
+ * 1. The timestamp (as a Drill timestamp)
+ * 2. The desired bucket interval IN milliseconds
+ *
+ * The function returns a BIGINT of the nearest time bucket.
+ */
+ @FunctionTemplate(name = "time_bucket",
+scope = FunctionTemplate.FunctionScope.SIMPLE,
+nulls = FunctionTemplate.NullHandling.NULL_IF_NULL)
+ public static class TimestampTimeBucketFunction implements DrillSimpleFunc {
+
+@Param
+TimeStampHolder inputDate;
+
+@Param
+BigIntHolder interval;
+
+@Output
+TimeStampHolder out;
+
+@Override
+public void setup() {
+}
+
+@Override
+public void eval() {
+ // Get the timestamp in milliseconds
+ long timestamp = inputDate.value;
+
+ // Get the interval in milliseconds
+ long groupByInterval = interval.value;
+
+ out.value = (timestamp - (timestamp % groupByInterval));
+}
+ }
+
+ /**
+ * This function is used for facilitating time series analysis by creating
buckets of time intervals. See
+ *
https://blog.timescale.com/blog/simplified-time-series-analytics-using-the-time_bucket-function/
for usage. The function takes two arguments:
+ * 1. The timestamp (as a Drill timestamp)
+ * 2. The desired bucket interval IN milliseconds
+ *
+ * The function returns a BIGINT of the nearest time bucket.
+ */
+ @FunctionTemplate(name = "time_bucket",
+scope = FunctionTemplate.FunctionScope.SIMPLE,
+nulls = FunctionTemplate.NullHandling.NULL_IF_NULL)
+ public static class DoubleTimeBucketFunction implements DrillSimpleFunc {
+
+@Param
+Float8Holder inputDate;
+
+@Param
+BigIntHolder interval;
+
+@Output
+BigIntHolder out;
+
+@Override
+public void setup() {
+}
+
+@Override
+public void eval() {
+ // Get the timestamp in milliseconds
+ long timestamp = java.lang.Math.round(inputDate.value);
+
+ // Get the interval in milliseconds
+ long groupByInterval = interval.value;
+
+ out.value = timestamp - (timestamp % groupByInterval);
}
}
}
diff --git
a/contrib/udfs/src/test/java/org/apache/drill/exec/udfs/TestTimeBucketFunction.java
b/contrib/udfs/src/test/java/org/apache/drill/exec/udfs/TestTimeBucketFunction.java
index a670e1e..127486f 100644
---
a/contrib/udfs/src/test/java/org/apache/drill/exec/udfs/TestTimeBucketFunction.java
+++
b/contrib/udfs/src/test/java/org/apache/drill/exec/udfs/TestTimeBucketFunction.java
@@ -27,10 +27,23 @@ import org.junit.BeforeClass;
import org.junit.Test;
import org.junit.experimental
[drill] 04/05: DRILL-7694: Register drill.queries.* counter metrics on Drillbit startup
This is an automated email from the ASF dual-hosted git repository.
arina pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git
commit 84450d8b81e690b003bb1bfac7a986f3db88dca5
Author: Bohdan Kazydub
AuthorDate: Thu Apr 9 16:46:18 2020 +0300
DRILL-7694: Register drill.queries.* counter metrics on Drillbit startup
closes #2050
---
.../apache/drill/exec/metrics/DrillCounters.java | 73 ++
.../apache/drill/exec/server/DrillbitContext.java | 8 +++
.../exec/work/foreman/QueryStateProcessor.java | 34 --
3 files changed, 94 insertions(+), 21 deletions(-)
diff --git
a/common/src/main/java/org/apache/drill/exec/metrics/DrillCounters.java
b/common/src/main/java/org/apache/drill/exec/metrics/DrillCounters.java
new file mode 100644
index 000..29685ae
--- /dev/null
+++ b/common/src/main/java/org/apache/drill/exec/metrics/DrillCounters.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.metrics;
+
+import com.codahale.metrics.Counter;
+
+/**
+ * Holder containing query state counter metrics.
+ */
+public class DrillCounters {
+
+ private static final DrillCounters INSTANCE = new DrillCounters();
+
+ private static final String QUERIES_METRICS_PREFIX = "drill.queries.";
+
+ private final Counter planningQueries =
DrillMetrics.getRegistry().counter(QUERIES_METRICS_PREFIX + "planning");
+ private final Counter enqueuedQueries =
DrillMetrics.getRegistry().counter(QUERIES_METRICS_PREFIX + "enqueued");
+ private final Counter runningQueries =
DrillMetrics.getRegistry().counter(QUERIES_METRICS_PREFIX + "running");
+ private final Counter completedQueries =
DrillMetrics.getRegistry().counter(QUERIES_METRICS_PREFIX + "completed");
+ private final Counter succeededQueries =
DrillMetrics.getRegistry().counter(QUERIES_METRICS_PREFIX + "succeeded");
+ private final Counter failedQueries =
DrillMetrics.getRegistry().counter(QUERIES_METRICS_PREFIX + "failed");
+ private final Counter canceledQueries =
DrillMetrics.getRegistry().counter(QUERIES_METRICS_PREFIX + "canceled");
+
+ private DrillCounters() {
+ }
+
+ public static DrillCounters getInstance() {
+return INSTANCE;
+ }
+
+ public Counter getPlanningQueries() {
+return planningQueries;
+ }
+
+ public Counter getEnqueuedQueries() {
+return enqueuedQueries;
+ }
+
+ public Counter getRunningQueries() {
+return runningQueries;
+ }
+
+ public Counter getCompletedQueries() {
+return completedQueries;
+ }
+
+ public Counter getSucceededQueries() {
+return succeededQueries;
+ }
+
+ public Counter getFailedQueries() {
+return failedQueries;
+ }
+
+ public Counter getCanceledQueries() {
+return canceledQueries;
+ }
+}
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/server/DrillbitContext.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/server/DrillbitContext.java
index 36c7b42..0afd984 100644
---
a/exec/java-exec/src/main/java/org/apache/drill/exec/server/DrillbitContext.java
+++
b/exec/java-exec/src/main/java/org/apache/drill/exec/server/DrillbitContext.java
@@ -28,6 +28,7 @@ import org.apache.drill.exec.coord.ClusterCoordinator;
import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry;
import org.apache.drill.exec.expr.fn.registry.RemoteFunctionRegistry;
import org.apache.drill.exec.memory.BufferAllocator;
+import org.apache.drill.exec.metrics.DrillCounters;
import org.apache.drill.exec.physical.impl.OperatorCreatorRegistry;
import org.apache.drill.exec.planner.PhysicalPlanReader;
import org.apache.drill.exec.planner.sql.DrillOperatorTable;
@@ -77,6 +78,7 @@ public class DrillbitContext implements AutoCloseable {
private final QueryProfileStoreContext profileStoreContext;
private ResourceManager resourceManager;
private final MetastoreRegistry metastoreRegistry;
+ private final DrillCounters counters;
public DrillbitContext(
DrillbitEndpoint endpoint,
@@ -125,6 +127,8 @@ public class DrillbitContext implements AutoCloseable {
[drill] 01/05: DRILL-7675: Work around for partitions sender memory use
This is an automated email from the ASF dual-hosted git repository.
arina pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git
commit ae25994ef40dfa3724a47fc804b83ca3c5128851
Author: Paul Rogers
AuthorDate: Thu Apr 2 11:45:12 2020 -0700
DRILL-7675: Work around for partitions sender memory use
Adds an ad-hoc system/session option to limit partition sender
memory use. See DRILL-7686 for the underlying issue.
Also includes code cleanup and diagnostic tools.
closes #2047
---
.../java/org/apache/drill/exec/ExecConstants.java | 7 +-
.../apache/drill/exec/compile/ClassBuilder.java| 13 +---
.../exec/physical/impl/join/HashJoinBatch.java | 9 +--
.../physical/impl/partitionsender/Partitioner.java | 5 +-
.../impl/partitionsender/PartitionerTemplate.java | 68 ++--
.../org/apache/drill/exec/record/BatchSchema.java | 30 +
.../apache/drill/exec/record/VectorContainer.java | 14
.../exec/server/options/BaseOptionManager.java | 5 ++
.../drill/exec/server/options/OptionSet.java | 14 ++--
.../exec/server/options/SystemOptionManager.java | 1 +
.../org/apache/drill/exec/vector/CopyUtil.java | 26
.../apache/drill/exec/work/foreman/Foreman.java| 31 +
.../drill/exec/work/foreman/QueryManager.java | 23 +++
.../java-exec/src/main/resources/drill-module.conf | 1 +
.../java/org/apache/drill/test/ClusterFixture.java | 35 --
.../apache/drill/test/ClusterFixtureBuilder.java | 2 -
.../drill/exec/record/MaterializedField.java | 75 +++---
17 files changed, 215 insertions(+), 144 deletions(-)
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java
index c421c73..3a16353 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java
@@ -162,7 +162,6 @@ public final class ExecConstants {
public static final PositiveLongValidator
HASHJOIN_RUNTIME_FILTER_MAX_WAITING_TIME = new
PositiveLongValidator(HASHJOIN_RUNTIME_FILTER_MAX_WAITING_TIME_KEY,
Character.MAX_VALUE, null);
-
// Hash Aggregate Options
public static final String HASHAGG_NUM_PARTITIONS_KEY =
"exec.hashagg.num_partitions";
public static final LongValidator HASHAGG_NUM_PARTITIONS_VALIDATOR = new
RangeLongValidator(HASHAGG_NUM_PARTITIONS_KEY, 1, 128,
@@ -189,6 +188,12 @@ public final class ExecConstants {
public static final BooleanValidator HASHAGG_FALLBACK_ENABLED_VALIDATOR =
new BooleanValidator(HASHAGG_FALLBACK_ENABLED_KEY,
new OptionDescription("Hash Aggregates ignore memory limits when enabled
(true). When disabled (false), Hash Aggregates fail when memory is set too
low."));
+ // Partitioner options
+ public static final String PARTITIONER_MEMORY_REDUCTION_THRESHOLD_KEY =
"exec.partition.mem_throttle";
+ public static final LongValidator
PARTITIONER_MEMORY_REDUCTION_THRESHOLD_VALIDATOR =
+ new RangeLongValidator(PARTITIONER_MEMORY_REDUCTION_THRESHOLD_KEY, 0,
Integer.MAX_VALUE,
+ new OptionDescription("Linearly reduces partition sender buffer row
count after this number of receivers. Default is 0 (disabled). (Since Drill
1.18)"));
+
public static final String SSL_PROVIDER = "drill.exec.ssl.provider"; //
valid values are "JDK", "OPENSSL" // default JDK
public static final String SSL_PROTOCOL = "drill.exec.ssl.protocol"; //
valid values are SSL, SSLV2, SSLV3, TLS, TLSV1, TLSv1.1, TLSv1.2(default)
public static final String SSL_KEYSTORE_TYPE = "drill.exec.ssl.keyStoreType";
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/compile/ClassBuilder.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/compile/ClassBuilder.java
index b4790c8..2c580da 100644
---
a/exec/java-exec/src/main/java/org/apache/drill/exec/compile/ClassBuilder.java
+++
b/exec/java-exec/src/main/java/org/apache/drill/exec/compile/ClassBuilder.java
@@ -29,6 +29,8 @@ import
org.apache.drill.exec.exception.ClassTransformationException;
import org.apache.drill.exec.expr.CodeGenerator;
import org.apache.drill.exec.server.options.OptionSet;
import org.codehaus.commons.compiler.CompileException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* Implements the "plain Java" method of code generation and
@@ -76,10 +78,9 @@ import org.codehaus.commons.compiler.CompileException;
* The setting to prefer plain Java is ignored for any remaining generated
* classes not marked as plain Java capable.
*/
-
public class ClassBuilder {
- private static final org.slf4j.Logger logger =
org.slf4j.LoggerFactory.getLogger(ClassBuilder.class);
+ private static final Logger logger =
LoggerFactory.getL
[drill] branch master updated: DRILL-7680: Place UDFs before plugins in contrib
This is an automated email from the ASF dual-hosted git repository. arina pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/drill.git The following commit(s) were added to refs/heads/master by this push: new 6d7e6b3 DRILL-7680: Place UDFs before plugins in contrib 6d7e6b3 is described below commit 6d7e6b3df4f8548169c7dc186e5e6ce49168a48e Author: Paul Rogers AuthorDate: Tue Mar 31 11:50:07 2020 -0700 DRILL-7680: Place UDFs before plugins in contrib --- contrib/pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/contrib/pom.xml b/contrib/pom.xml index 6c2bb89..8f81d48 100644 --- a/contrib/pom.xml +++ b/contrib/pom.xml @@ -39,6 +39,8 @@ +data +udfs storage-hbase format-maprdb format-syslog @@ -52,8 +54,6 @@ storage-kafka storage-kudu storage-opentsdb -data -udfs
[drill] branch master updated: DRILL-7665: Add UNION to schema parser
This is an automated email from the ASF dual-hosted git repository.
arina pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git
The following commit(s) were added to refs/heads/master by this push:
new f1ccdc2 DRILL-7665: Add UNION to schema parser
f1ccdc2 is described below
commit f1ccdc2091a26c7680d4e0999f2acbd607386c09
Author: Arina Ielchiieva
AuthorDate: Wed Mar 25 19:31:23 2020 +0200
DRILL-7665: Add UNION to schema parser
---
.../record/metadata/schema/parser/SchemaLexer.g4 | 1 +
.../record/metadata/schema/parser/SchemaParser.g4 | 8 -
.../metadata/schema/parser/SchemaVisitor.java | 20 +
.../metadata/schema/parser/TestSchemaParser.java | 34 --
4 files changed, 60 insertions(+), 3 deletions(-)
diff --git
a/exec/vector/src/main/antlr4/org/apache/drill/exec/record/metadata/schema/parser/SchemaLexer.g4
b/exec/vector/src/main/antlr4/org/apache/drill/exec/record/metadata/schema/parser/SchemaLexer.g4
index c531356..c5d1fe1 100644
---
a/exec/vector/src/main/antlr4/org/apache/drill/exec/record/metadata/schema/parser/SchemaLexer.g4
+++
b/exec/vector/src/main/antlr4/org/apache/drill/exec/record/metadata/schema/parser/SchemaLexer.g4
@@ -57,6 +57,7 @@ SECOND: 'SECOND';
MAP: 'MAP';
STRUCT: 'STRUCT';
ARRAY: 'ARRAY';
+UNION: 'UNION';
// additional data types, primary used for Parquet
UINT1: 'UINT1';
diff --git
a/exec/vector/src/main/antlr4/org/apache/drill/exec/record/metadata/schema/parser/SchemaParser.g4
b/exec/vector/src/main/antlr4/org/apache/drill/exec/record/metadata/schema/parser/SchemaParser.g4
index 1ff90ec..5196986 100644
---
a/exec/vector/src/main/antlr4/org/apache/drill/exec/record/metadata/schema/parser/SchemaParser.g4
+++
b/exec/vector/src/main/antlr4/org/apache/drill/exec/record/metadata/schema/parser/SchemaParser.g4
@@ -31,7 +31,7 @@ columns: column_def (COMMA column_def)*;
column_def: column property_values?;
-column: (primitive_column | struct_column | map_column | simple_array_column |
complex_array_column);
+column: (primitive_column | struct_column | map_column | simple_array_column |
complex_array_column | union_column);
primitive_column: column_id simple_type nullability? format_value?
default_value?;
@@ -43,6 +43,8 @@ map_column: column_id map_type nullability?;
complex_array_column: column_id complex_array_type nullability?;
+union_column: column_id union_type nullability?;
+
column_id
: ID # id
| QUOTED_ID # quoted_id
@@ -71,6 +73,8 @@ simple_type
| SMALLINT # smallint
;
+union_type: UNION;
+
array_type: (simple_array_type | complex_array_type);
simple_array_type: ARRAY LEFT_ANGLE_BRACKET simple_array_value_type
RIGHT_ANGLE_BRACKET;
@@ -79,6 +83,7 @@ simple_array_value_type
: simple_type # array_simple_type_def
| struct_type # array_struct_type_def
| map_type # array_map_type_def
+| union_type # array_union_type_def
;
complex_array_type: ARRAY LEFT_ANGLE_BRACKET array_type RIGHT_ANGLE_BRACKET;
@@ -100,6 +105,7 @@ map_value_type
| struct_type # map_value_struct_type_def
| map_type # map_value_map_type_def
| array_type # map_value_array_type_def
+| union_type # map_value_union_type_def
;
nullability: NOT NULL;
diff --git
a/exec/vector/src/main/java/org/apache/drill/exec/record/metadata/schema/parser/SchemaVisitor.java
b/exec/vector/src/main/java/org/apache/drill/exec/record/metadata/schema/parser/SchemaVisitor.java
index 18ed651..9db9dd0 100644
---
a/exec/vector/src/main/java/org/apache/drill/exec/record/metadata/schema/parser/SchemaVisitor.java
+++
b/exec/vector/src/main/java/org/apache/drill/exec/record/metadata/schema/parser/SchemaVisitor.java
@@ -28,6 +28,7 @@ import org.apache.drill.exec.record.metadata.MetadataUtils;
import org.apache.drill.exec.record.metadata.RepeatedListBuilder;
import org.apache.drill.exec.record.metadata.TupleMetadata;
import org.apache.drill.exec.record.metadata.TupleSchema;
+import org.apache.drill.exec.record.metadata.VariantColumnMetadata;
import org.apache.drill.exec.vector.complex.DictVector;
import org.apache.drill.shaded.guava.com.google.common.base.Preconditions;
@@ -127,6 +128,13 @@ public class SchemaVisitor extends
SchemaParserBaseVisitor {
builder.addColumn(child);
return builder.buildColumn();
}
+
+@Override
+public ColumnMetadata visitUnion_column(SchemaParser.Union_columnContext
ctx) {
+ String name = ctx.column_id().accept(new IdVisitor());
+ // nullability for UNION types are ignored, since they can hold any value
+ return VariantColumnMetadata.union(name);
+}
}
/**
@@ -387,6 +395,11 @@ public class SchemaVisitor extends
SchemaParserBaseVisitor {
public ColumnMetadata
visitMap_value_array_type_def(SchemaParser.Map_value_array_type_defContext ctx)
{
return ctx.array_type().accept(new
ArrayTypeVisitor(DictVector.FIELD_VALUE_NAME));
}
+
+@Override
+public ColumnMetadata
[drill] branch master updated: DRILL-7663: Code refactor to DefaultFunctionResolver
This is an automated email from the ASF dual-hosted git repository.
arina pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git
The following commit(s) were added to refs/heads/master by this push:
new 83bc8e6 DRILL-7663: Code refactor to DefaultFunctionResolver
83bc8e6 is described below
commit 83bc8e63586051962a6ab39731d3c0ae19b334d1
Author: weijie.tong
AuthorDate: Wed Mar 25 20:00:44 2020 +0800
DRILL-7663: Code refactor to DefaultFunctionResolver
---
.../apache/drill/exec/resolver/DefaultFunctionResolver.java | 11 ---
1 file changed, 4 insertions(+), 7 deletions(-)
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/resolver/DefaultFunctionResolver.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/resolver/DefaultFunctionResolver.java
index b768792..ae3c68b 100644
---
a/exec/java-exec/src/main/java/org/apache/drill/exec/resolver/DefaultFunctionResolver.java
+++
b/exec/java-exec/src/main/java/org/apache/drill/exec/resolver/DefaultFunctionResolver.java
@@ -19,8 +19,7 @@ package org.apache.drill.exec.resolver;
import java.util.LinkedList;
import java.util.List;
-
-import org.apache.drill.shaded.guava.com.google.common.collect.Lists;
+import java.util.stream.Collectors;
import org.apache.drill.common.expression.FunctionCall;
import org.apache.drill.common.expression.LogicalExpression;
import org.apache.drill.common.types.TypeProtos;
@@ -38,12 +37,10 @@ public class DefaultFunctionResolver implements
FunctionResolver {
int currcost = Integer.MAX_VALUE;
DrillFuncHolder bestmatch = null;
final List bestMatchAlternatives = new LinkedList<>();
-
+List argumentTypes = call.args().stream()
+.map(LogicalExpression::getMajorType)
+.collect(Collectors.toList());
for (DrillFuncHolder h : methods) {
- final List argumentTypes = Lists.newArrayList();
- for (LogicalExpression expression : call.args()) {
-argumentTypes.add(expression.getMajorType());
- }
currcost = TypeCastRules.getCost(argumentTypes, h);
// if cost is lower than 0, func implementation is not matched, either
w/ or w/o implicit casts
[drill] branch master updated (3b3c4af -> 84a50f0)
This is an automated email from the ASF dual-hosted git repository. arina pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/drill.git. from 3b3c4af DRILL-7648: Scrypt j_security_check works without security headers add 84a50f0 DRILL-7633: Fixes for union and repeated list accessors No new revisions were added by this update. Summary of changes: .../resultSet/impl/TestResultSetLoaderUnions.java | 210 - .../exec/physical/rowSet/TestSchemaBuilder.java| 2 - .../apache/drill/test/rowSet/RowSetComparison.java | 15 +- .../record/metadata/VariantColumnMetadata.java | 17 +- .../drill/exec/record/metadata/VariantSchema.java | 11 +- .../drill/exec/vector/accessor/VariantWriter.java | 11 +- .../vector/accessor/writer/UnionVectorShim.java| 11 +- 7 files changed, 198 insertions(+), 79 deletions(-)
[drill] branch master updated: DRILL-7648: Scrypt j_security_check works without security headers
This is an automated email from the ASF dual-hosted git repository.
arina pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git
The following commit(s) were added to refs/heads/master by this push:
new 3b3c4af DRILL-7648: Scrypt j_security_check works without security
headers
3b3c4af is described below
commit 3b3c4af39fdc26f255cc17d66c55eb7565552a7d
Author: Igor Guzenko
AuthorDate: Fri Mar 20 19:07:25 2020 +0200
DRILL-7648: Scrypt j_security_check works without security headers
1. Added callback for setting headers in DrillHttpSecurityHandlerProvider,
since ResponseHeadersSettingFilter doesn't covers this flow.
---
.../exec/server/rest/auth/DrillHttpSecurityHandlerProvider.java | 6 +-
1 file changed, 5 insertions(+), 1 deletion(-)
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/auth/DrillHttpSecurityHandlerProvider.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/auth/DrillHttpSecurityHandlerProvider.java
index 36a9863..fb10ac1 100644
---
a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/auth/DrillHttpSecurityHandlerProvider.java
+++
b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/auth/DrillHttpSecurityHandlerProvider.java
@@ -17,6 +17,7 @@
*/
package org.apache.drill.exec.server.rest.auth;
+import org.apache.drill.exec.server.rest.header.ResponseHeadersSettingFilter;
import org.apache.drill.shaded.guava.com.google.common.base.Preconditions;
import org.apache.drill.common.config.DrillConfig;
import org.apache.drill.common.exceptions.DrillException;
@@ -54,11 +55,14 @@ public class DrillHttpSecurityHandlerProvider extends
ConstraintSecurityHandler
private final Map
securityHandlers =
CaseInsensitiveMap.newHashMapWithExpectedSize(2);
+ private final Map responseHeaders;
+
@SuppressWarnings("unchecked")
public DrillHttpSecurityHandlerProvider(DrillConfig config, DrillbitContext
drillContext)
throws DrillbitStartupException {
Preconditions.checkState(config.getBoolean(ExecConstants.USER_AUTHENTICATION_ENABLED));
+this.responseHeaders =
ResponseHeadersSettingFilter.retrieveResponseHeaders(config);
final Set configuredMechanisms = getHttpAuthMechanisms(config);
final ScanResult scan = drillContext.getClasspathScan();
@@ -122,7 +126,7 @@ public class DrillHttpSecurityHandlerProvider extends
ConstraintSecurityHandler
throws IOException, ServletException {
Preconditions.checkState(securityHandlers.size() > 0);
-
+responseHeaders.forEach(response::setHeader);
HttpSession session = request.getSession(true);
SessionAuthentication authentication =
(SessionAuthentication)
session.getAttribute(SessionAuthentication.__J_AUTHENTICATED);
[drill] branch master updated: DRILL-7654: Add support for JDK 14
This is an automated email from the ASF dual-hosted git repository.
arina pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git
The following commit(s) were added to refs/heads/master by this push:
new 54399dc DRILL-7654: Add support for JDK 14
54399dc is described below
commit 54399dc268b46117174ea1af3e5f30aa72125674
Author: Volodymyr Vysotskyi
AuthorDate: Fri Mar 20 23:59:56 2020 +0200
DRILL-7654: Add support for JDK 14
---
.github/workflows/ci.yml | 2 +-
pom.xml | 6 +++---
2 files changed, 4 insertions(+), 4 deletions(-)
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 9becd45..67fe40e 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -33,7 +33,7 @@ jobs:
strategy:
matrix:
# Java versions to run unit tests
-java: [ '1.8', '11', '13' ]
+java: [ '1.8', '11', '14' ]
steps:
- name: Checkout
uses: actions/checkout@v2
diff --git a/pom.xml b/pom.xml
index a4e8ab1..bacd6bd 100644
--- a/pom.xml
+++ b/pom.xml
@@ -86,14 +86,14 @@
2.2.2
1.0
2.3.28
-3.26.0-GA
+3.27.0-GA
0.6.6
0.9.10
1.9.1
4.0.2
9.3.25.v20180904
2.25.1
-7.2
+7.3.1
4096
4096
@@ -585,7 +585,7 @@
[{$lowestMavenVersion},4)
- [1.8,14)
+ [1.8,15)
[drill] branch master updated: DRILL-7651: Increase timeout for TestLargeFileCompilation to avoid GitHub Action failures and fix concurrent issue in TestTpchDistributedConcurrent
This is an automated email from the ASF dual-hosted git repository.
arina pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git
The following commit(s) were added to refs/heads/master by this push:
new 0933f1b DRILL-7651: Increase timeout for TestLargeFileCompilation to
avoid GitHub Action failures and fix concurrent issue in
TestTpchDistributedConcurrent
0933f1b is described below
commit 0933f1bd6f496fe02f28e719c48839430add5977
Author: Volodymyr Vysotskyi
AuthorDate: Thu Mar 19 18:43:02 2020 +0200
DRILL-7651: Increase timeout for TestLargeFileCompilation to avoid GitHub
Action failures and fix concurrent issue in TestTpchDistributedConcurrent
---
.../drill/TestTpchDistributedConcurrent.java | 159 ++--
.../exec/compile/TestLargeFileCompilation.java | 163 -
2 files changed, 181 insertions(+), 141 deletions(-)
diff --git
a/exec/java-exec/src/test/java/org/apache/drill/TestTpchDistributedConcurrent.java
b/exec/java-exec/src/test/java/org/apache/drill/TestTpchDistributedConcurrent.java
index ba3bd44..c144c53 100644
---
a/exec/java-exec/src/test/java/org/apache/drill/TestTpchDistributedConcurrent.java
+++
b/exec/java-exec/src/test/java/org/apache/drill/TestTpchDistributedConcurrent.java
@@ -26,18 +26,25 @@ import java.util.concurrent.Semaphore;
import org.apache.drill.categories.SlowTest;
import org.apache.drill.common.exceptions.UserException;
+import org.apache.drill.exec.ExecConstants;
+import org.apache.drill.test.BaseTestQuery.SilentListener;
+import org.apache.drill.test.ClusterFixture;
+import org.apache.drill.test.ClusterFixtureBuilder;
+import org.apache.drill.test.ClusterTest;
import org.apache.drill.test.TestTools;
import org.apache.drill.exec.proto.UserBitShared;
import org.apache.drill.exec.proto.UserBitShared.QueryResult.QueryState;
import org.apache.drill.exec.rpc.user.UserResultsListener;
-import org.apache.drill.test.BaseTestQuery;
import org.apache.drill.test.QueryTestUtil;
+import org.junit.BeforeClass;
import org.junit.Rule;
import org.junit.Test;
import org.junit.experimental.categories.Category;
import org.junit.rules.TestRule;
import org.apache.drill.shaded.guava.com.google.common.collect.Sets;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
@@ -49,17 +56,18 @@ import static org.junit.Assert.assertNull;
* any particular order of execution. We ignore the results.
*/
@Category({SlowTest.class})
-public class TestTpchDistributedConcurrent extends BaseTestQuery {
- private static final org.slf4j.Logger logger =
org.slf4j.LoggerFactory.getLogger(TestTpchDistributedConcurrent.class);
+public class TestTpchDistributedConcurrent extends ClusterTest {
+ private static final Logger logger =
LoggerFactory.getLogger(TestTpchDistributedConcurrent.class);
- @Rule public final TestRule TIMEOUT = TestTools.getTimeoutRule(36); //
Longer timeout than usual.
+ @Rule
+ public final TestRule TIMEOUT = TestTools.getTimeoutRule(400_000); // 400
secs
/*
* Valid test names taken from TestTpchDistributed. Fuller path prefixes are
* used so that tests may also be taken from other locations -- more variety
* is better as far as this test goes.
*/
- private final static String queryFile[] = {
+ private static final String[] queryFile = {
"queries/tpch/01.sql",
"queries/tpch/03.sql",
"queries/tpch/04.sql",
@@ -80,45 +88,92 @@ public class TestTpchDistributedConcurrent extends
BaseTestQuery {
"queries/tpch/20.sql",
};
- private final static int TOTAL_QUERIES = 115;
- private final static int CONCURRENT_QUERIES = 15;
-
- private final static Random random = new Random(0xdeadbeef);
- private final static String alterSession = "alter session set
`planner.slice_target` = 10";
+ private static final int TOTAL_QUERIES = 115;
+ private static final int CONCURRENT_QUERIES = 15;
+ private static final Random random = new Random(0xdeadbeef);
private int remainingQueries = TOTAL_QUERIES - CONCURRENT_QUERIES;
private final Semaphore completionSemaphore = new Semaphore(0);
private final Semaphore submissionSemaphore = new Semaphore(0);
private final Set listeners = Sets.newIdentityHashSet();
+ private final List failedQueries = new LinkedList<>();
private Thread testThread = null; // used to interrupt semaphore wait in
case of error
- private static class FailedQuery {
-final String queryFile;
-final UserException userEx;
+ @BeforeClass
+ public static void setUp() throws Exception {
+ClusterFixtureBuilder builder = ClusterFixture.builder(dirTestWatcher)
+.configProperty(ExecConstants.USER_RPC_TIMEOUT, 5_000);
+startCluster(builder);
+ }
-public FailedQuery(final String queryFile, final UserException userEx)
[drill] branch master updated (e95f5a6 -> 129fa5b)
This is an automated email from the ASF dual-hosted git repository. arina pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/drill.git. from e95f5a6 DRILL-7650: Add option to enable Jetty's dump for troubleshooting add 129fa5b DRILL-7652: Add time_bucket() function for time series analysis No new revisions were added by this update. Summary of changes: contrib/udfs/README.md | 59 +- .../drill/exec/udfs/TimeBucketFunctions.java | 105 + .../drill/exec/udfs/TestTimeBucketFunction.java| 128 + 3 files changed, 289 insertions(+), 3 deletions(-) create mode 100644 contrib/udfs/src/main/java/org/apache/drill/exec/udfs/TimeBucketFunctions.java create mode 100644 contrib/udfs/src/test/java/org/apache/drill/exec/udfs/TestTimeBucketFunction.java
[drill] branch master updated: DRILL-7650: Add option to enable Jetty's dump for troubleshooting
This is an automated email from the ASF dual-hosted git repository.
arina pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git
The following commit(s) were added to refs/heads/master by this push:
new e95f5a6 DRILL-7650: Add option to enable Jetty's dump for
troubleshooting
e95f5a6 is described below
commit e95f5a6a403d76b58f33944927d9d707505cc7ea
Author: Igor Guzenko
AuthorDate: Thu Mar 19 13:13:56 2020 +0200
DRILL-7650: Add option to enable Jetty's dump for troubleshooting
1. Added option drill.exec.http.jetty.server.dumpAfterStart
2. Removed redundant setProtocol() call
---
distribution/src/main/resources/drill-override-example.conf| 2 ++
exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java | 1 +
.../src/main/java/org/apache/drill/exec/server/rest/WebServer.java | 1 +
.../drill/exec/server/rest/ssl/SslContextFactoryConfigurator.java | 1 -
exec/java-exec/src/main/resources/drill-module.conf| 3 ++-
5 files changed, 6 insertions(+), 2 deletions(-)
diff --git a/distribution/src/main/resources/drill-override-example.conf
b/distribution/src/main/resources/drill-override-example.conf
index 499e371..6fcca37 100644
--- a/distribution/src/main/resources/drill-override-example.conf
+++ b/distribution/src/main/resources/drill-override-example.conf
@@ -115,6 +115,8 @@ drill.exec: {
},
jetty: {
server: {
+# development option which allows to log Jetty server state after start
+dumpAfterStart: false,
# Optional params to set on Jetty's
org.eclipse.jetty.util.ssl.SslContextFactory when drill.exec.http.ssl_enabled
sslContextFactory: {
# allows to specify cert to use when multiple non-SNI certificates
are available.
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java
index 3db3bba..c421c73 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java
@@ -210,6 +210,7 @@ public final class ExecConstants {
public static final String HTTP_PROFILES_PER_PAGE =
"drill.exec.http.profiles_per_page";
public static final String HTTP_PORT = "drill.exec.http.port";
public static final String HTTP_PORT_HUNT = "drill.exec.http.porthunt";
+ public static final String HTTP_JETTY_SERVER_DUMP_AFTER_START =
"drill.exec.http.jetty.server.dumpAfterStart";
public static final String HTTP_JETTY_SERVER_ACCEPTORS =
"drill.exec.http.jetty.server.acceptors";
public static final String HTTP_JETTY_SERVER_SELECTORS =
"drill.exec.http.jetty.server.selectors";
public static final String HTTP_JETTY_SERVER_HANDLERS =
"drill.exec.http.jetty.server.handlers";
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/WebServer.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/WebServer.java
index 7a6c2aa..17d97d7 100644
---
a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/WebServer.java
+++
b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/WebServer.java
@@ -162,6 +162,7 @@ public class WebServer implements AutoCloseable {
threadPool.setMaxThreads(handlers + connector.getAcceptors() +
connector.getSelectorManager().getSelectorCount());
embeddedJetty.addConnector(connector);
+
embeddedJetty.setDumpAfterStart(config.getBoolean(ExecConstants.HTTP_JETTY_SERVER_DUMP_AFTER_START));
final boolean portHunt = config.getBoolean(ExecConstants.HTTP_PORT_HUNT);
for (int retry = 0; retry < PORT_HUNT_TRIES; retry++) {
connector.setPort(port);
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/ssl/SslContextFactoryConfigurator.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/ssl/SslContextFactoryConfigurator.java
index 62a740b..7da5465 100644
---
a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/ssl/SslContextFactoryConfigurator.java
+++
b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/ssl/SslContextFactoryConfigurator.java
@@ -93,7 +93,6 @@ public class SslContextFactoryConfigurator {
sslFactory.setTrustStorePassword(sslConf.getTrustStorePassword());
}
}
-sslFactory.setProtocol(sslConf.getProtocol());
sslFactory.setIncludeProtocols(sslConf.getProtocol());
logger.info("Web server configured to use TLS protocol '{}'",
sslConf.getProtocol());
if
(config.hasPath(ExecConstants.HTTP_JETTY_SSL_CONTEXT_FACTORY_OPTIONS_PREFIX)) {
diff --git a/exec/java-exec/src/main/resources/drill-module.conf
b/exec/java-exec/src/main/resources/drill-module.conf
index 03194c5..9e306e6 100644
--- a/exec/java-exec/src/main/resources/drill-module.conf
+++ b
[drill] branch master updated: DRILL-7644: Log SSL protocol version at Drill start up
This is an automated email from the ASF dual-hosted git repository.
arina pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git
The following commit(s) were added to refs/heads/master by this push:
new 370bd2a DRILL-7644: Log SSL protocol version at Drill start up
370bd2a is described below
commit 370bd2ad99a469056841c0fb35169a401827a9a9
Author: Igor Guzenko
AuthorDate: Tue Mar 17 17:58:06 2020 +0200
DRILL-7644: Log SSL protocol version at Drill start up
---
.../src/main/java/org/apache/drill/exec/rpc/user/UserServer.java | 1 +
.../apache/drill/exec/server/rest/ssl/SslContextFactoryConfigurator.java | 1 +
2 files changed, 2 insertions(+)
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/user/UserServer.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/user/UserServer.java
index 1c2e2e2..276758e 100644
---
a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/user/UserServer.java
+++
b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/user/UserServer.java
@@ -137,6 +137,7 @@ public class UserServer extends BasicServer {
.initializeSSLContext(true)
.validateKeyStore(true)
.build();
+ logger.info("Rpc server configured to use TLS protocol '{}'",
sslConfig.getProtocol());
} catch (DrillException e) {
throw new DrillbitStartupException(e.getMessage(), e.getCause());
}
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/ssl/SslContextFactoryConfigurator.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/ssl/SslContextFactoryConfigurator.java
index a5aa541..62a740b 100644
---
a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/ssl/SslContextFactoryConfigurator.java
+++
b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/ssl/SslContextFactoryConfigurator.java
@@ -95,6 +95,7 @@ public class SslContextFactoryConfigurator {
}
sslFactory.setProtocol(sslConf.getProtocol());
sslFactory.setIncludeProtocols(sslConf.getProtocol());
+logger.info("Web server configured to use TLS protocol '{}'",
sslConf.getProtocol());
if
(config.hasPath(ExecConstants.HTTP_JETTY_SSL_CONTEXT_FACTORY_OPTIONS_PREFIX)) {
setStringIfPresent(ExecConstants.HTTP_JETTY_SERVER_SSL_CONTEXT_FACTORY_CERT_ALIAS,
sslFactory::setCertAlias);
setStringIfPresent(ExecConstants.HTTP_JETTY_SERVER_SSL_CONTEXT_FACTORY_CRL_PATH,
sslFactory::setCrlPath);
[drill] branch master updated (63e64c2 -> 17a4660)
This is an automated email from the ASF dual-hosted git repository.
arina pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git.
from 63e64c2 DRILL-7620: Fix plugin mutability issues
add 17a4660 DRILL-7639: Replace DBCP2 with HikariCP in RDBMS (JDBC) plugin
No new revisions were added by this update.
Summary of changes:
contrib/storage-jdbc/pom.xml | 4 +-
.../drill/exec/store/jdbc/JdbcStoragePlugin.java | 65 +++---
...estBasicDataSource.java => TestDataSource.java} | 79 +-
.../exec/store/jdbc/TestJdbcPluginWithH2IT.java| 3 +-
pom.xml| 14 ++--
5 files changed, 79 insertions(+), 86 deletions(-)
rename
contrib/storage-jdbc/src/test/java/org/apache/drill/exec/store/jdbc/{TestBasicDataSource.java
=> TestDataSource.java} (52%)
[drill] branch master updated (7b1b83b -> a8c9a0f)
This is an automated email from the ASF dual-hosted git repository. arina pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/drill.git. from 7b1b83b DRILL-7635: Mongo tests fail for profiles with the non-file default file system new 88a84b9 DRILL-7601: Shift column conversion to reader from scan framework new a8c9a0f DRILL-7603: Allow default schema to be set for HTTP queries The 2 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../drill/exec/store/esri/ShpBatchReader.java | 6 +- .../drill/exec/store/esri/ShpFormatPlugin.java | 2 +- .../drill/exec/store/excel/ExcelFormatPlugin.java | 2 +- .../drill/exec/store/hdf5/HDF5BatchReader.java | 113 ++-- .../drill/exec/store/hdf5/HDF5FormatPlugin.java| 2 +- .../exec/store/hdf5/writers/HDF5DataWriter.java| 35 +- .../store/hdf5/writers/HDF5DoubleDataWriter.java | 34 +- .../store/hdf5/writers/HDF5EnumDataWriter.java | 18 +- .../store/hdf5/writers/HDF5FloatDataWriter.java| 32 +- .../exec/store/hdf5/writers/HDF5IntDataWriter.java | 35 +- .../store/hdf5/writers/HDF5LongDataWriter.java | 34 +- .../exec/store/hdf5/writers/HDF5MapDataWriter.java | 49 +- .../store/hdf5/writers/HDF5StringDataWriter.java | 28 +- .../hdf5/writers/HDF5TimestampDataWriter.java | 18 +- .../drill/exec/store/hdf5/writers/WriterSpec.java | 62 +++ .../drill/exec/ops/ExecutorFragmentContext.java| 4 +- .../drill/exec/physical/impl/ImplCreator.java | 2 +- .../physical/impl/OperatorCreatorRegistry.java | 5 +- .../impl/metadata/MetadataHandlerBatch.java| 10 +- .../impl/scan/columns/ColumnsArrayParser.java | 5 - .../scan}/convert/AbstractConvertFromString.java | 105 ++-- .../impl/scan/convert/ColumnConverter.java | 46 ++ .../impl/scan}/convert/ConvertBooleanToString.java | 4 +- .../impl/scan}/convert/ConvertDateToString.java| 4 +- .../impl/scan}/convert/ConvertDecimalToString.java | 4 +- .../impl/scan}/convert/ConvertDoubleToString.java | 4 +- .../impl/scan}/convert/ConvertIntToString.java | 4 +- .../scan}/convert/ConvertIntervalToString.java | 4 +- .../impl/scan}/convert/ConvertLongToString.java| 4 +- .../impl/scan}/convert/ConvertStringToBoolean.java | 2 +- .../impl/scan}/convert/ConvertStringToDate.java| 2 +- .../impl/scan}/convert/ConvertStringToDecimal.java | 2 +- .../impl/scan}/convert/ConvertStringToDouble.java | 2 +- .../impl/scan}/convert/ConvertStringToInt.java | 2 +- .../scan}/convert/ConvertStringToInterval.java | 2 +- .../impl/scan}/convert/ConvertStringToLong.java| 2 +- .../impl/scan}/convert/ConvertStringToTime.java| 2 +- .../scan}/convert/ConvertStringToTimeStamp.java| 2 +- .../scan}/convert/ConvertTimeStampToString.java| 4 +- .../impl/scan}/convert/ConvertTimeToString.java| 4 +- .../impl/scan/convert/DirectConverter.java}| 67 +-- .../impl/scan/convert/StandardConversions.java | 478 .../physical/impl/scan/convert/package-info.java | 78 +++ .../impl/scan/file/FileMetadataColumnsParser.java | 4 +- .../physical/impl/scan/file/FileScanFramework.java | 33 +- ...dataManager.java => ImplicitColumnManager.java} | 12 +- .../physical/impl/scan/file/PartitionColumn.java | 1 - .../impl/scan/framework/ManagedScanFramework.java | 8 +- .../impl/scan/framework/SchemaNegotiator.java | 126 +++-- .../impl/scan/framework/SchemaNegotiatorImpl.java | 35 +- .../impl/scan/framework/ShimBatchReader.java | 7 +- .../scan/project/AbstractUnresolvedColumn.java | 3 - .../impl/scan/project/ReaderLevelProjection.java | 2 - .../scan/project/ReaderSchemaOrchestrator.java | 50 +- .../physical/impl/scan/project/ResolvedColumn.java | 2 - .../impl/scan/project/ResolvedMapColumn.java | 1 - .../impl/scan/project/ResolvedNullColumn.java | 3 - .../impl/scan/project/ScanLevelProjection.java | 57 +- .../impl/scan/project/ScanSchemaOrchestrator.java | 62 +-- .../impl/scan/project/StaticColumnLoader.java | 6 +- .../project/projSet/AbstractProjectionSet.java | 80 --- .../scan/project/projSet/AbstractReadColProj.java | 46 -- .../scan/project/projSet/EmptyProjectionSet.java | 51 -- .../project/projSet/ExplicitProjectionSet.java | 118 .../scan/project/projSet/ProjectedDictColumn.java | 39 -- .../scan/project/projSet/ProjectedMapColumn.java | 39 -- .../scan/project/projSet/ProjectedReadColumn.java | 73 --- .../scan/project/projSet/ProjectionSetBuilder.java | 101 .../scan/project/projSet/ProjectionSetFactory.java | 79 --- .../impl/scan/project/projSet/TypeConverter.java |
[drill] 02/02: DRILL-7603: Allow default schema to be set for HTTP queries
This is an automated email from the ASF dual-hosted git repository.
arina pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git
commit a8c9a0f75477cb49284e337d5ebcda51318f4380
Author: Dobes Vandermeer
AuthorDate: Tue Mar 10 14:02:27 2020 -0700
DRILL-7603: Allow default schema to be set for HTTP queries
This allows REST API requests and Web UI requests to specify a
default
schema. Otherwise this is not possible for HTTP clients because the
"USE" command requires a session, which HTTP clients do not have.
closes #1996
---
.../org/apache/drill/exec/server/rest/QueryResources.java | 6 --
.../org/apache/drill/exec/server/rest/QueryWrapper.java | 15 +--
exec/java-exec/src/main/resources/rest/query/query.ftl| 2 ++
.../org/apache/drill/exec/server/rest/RestServerTest.java | 4 ++--
.../apache/drill/exec/server/rest/TestQueryWrapper.java | 10 +-
5 files changed, 30 insertions(+), 7 deletions(-)
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/QueryResources.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/QueryResources.java
index bf07fae..ec2a400 100644
---
a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/QueryResources.java
+++
b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/QueryResources.java
@@ -96,10 +96,12 @@ public class QueryResources {
public Viewable submitQuery(@FormParam("query") String query,
@FormParam("queryType") String queryType,
@FormParam("autoLimit") String autoLimit,
- @FormParam("userName") String userName) throws
Exception {
+ @FormParam("userName") String userName,
+ @FormParam("defaultSchema") String
defaultSchema) throws Exception {
try {
+ // Apply options from the form fields, if provided
final String trimmedQueryString =
CharMatcher.is(';').trimTrailingFrom(query.trim());
- final QueryResult result = submitQueryJSON(new
QueryWrapper(trimmedQueryString, queryType, autoLimit, userName));
+ final QueryResult result = submitQueryJSON(new
QueryWrapper(trimmedQueryString, queryType, autoLimit, userName,
defaultSchema));
List rowsPerPageValues =
work.getContext().getConfig().getIntList(ExecConstants.HTTP_WEB_CLIENT_RESULTSET_ROWS_PER_PAGE_VALUES);
Collections.sort(rowsPerPageValues);
final String rowsPerPageValuesAsStr =
Joiner.on(",").join(rowsPerPageValues);
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/QueryWrapper.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/QueryWrapper.java
index 3b99491..0a7bcd7 100644
---
a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/QueryWrapper.java
+++
b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/QueryWrapper.java
@@ -19,6 +19,7 @@ package org.apache.drill.exec.server.rest;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;
+import org.apache.calcite.schema.SchemaPlus;
import org.apache.drill.common.config.DrillConfig;
import org.apache.drill.common.exceptions.UserException;
import org.apache.drill.common.exceptions.UserRemoteException;
@@ -32,6 +33,7 @@ import org.apache.drill.exec.proto.UserProtos.RunQuery;
import org.apache.drill.exec.proto.helper.QueryIdHelper;
import org.apache.drill.exec.rpc.user.InboundImpersonationManager;
import org.apache.drill.exec.server.options.SessionOptionManager;
+import org.apache.drill.exec.store.SchemaTreeProvider;
import org.apache.drill.exec.util.ImpersonationUtil;
import org.apache.drill.exec.work.WorkManager;
import org.apache.parquet.Strings;
@@ -51,8 +53,8 @@ public class QueryWrapper {
private final String query;
private final String queryType;
private final int autoLimitRowCount;
-
private final String userName;
+ private final String defaultSchema;
private static MemoryMXBean memMXBean = ManagementFactory.getMemoryMXBean();
@@ -61,11 +63,13 @@ public class QueryWrapper {
@JsonProperty("query") String query,
@JsonProperty("queryType") String queryType,
@JsonProperty("autoLimit") String autoLimit,
-@JsonProperty("userName") String userName) {
+@JsonProperty("userName") String userName,
+@JsonProperty("defaultSchema") String defaultSchema) {
this.query = query;
this.queryType = queryType.toUpperCase();
this.autoLimitRowCount = autoLimit != null && autoLimit.matches("[0-9]+")
? Integer.valueOf(autoLimit) : 0;
this.userName = userName;
+this.defaultSchema = defaultSchema;
}
public String getQuery() {
@@ -90,6 +9
[drill] branch master updated (ab60b3d -> e9dc68b)
This is an automated email from the ASF dual-hosted git repository. arina pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/drill.git. from ab60b3d DRILL-7592: Add missing licenses and update plugins exclusion list and fix licenses add e9dc68b DRILL-7617: Disabled plugins not showing in Web UI No new revisions were added by this update. Summary of changes: .../drill/exec/server/rest/StorageResources.java | 3 +-- .../apache/drill/exec/store/StoragePluginRegistry.java | 8 .../drill/exec/store/StoragePluginRegistryImpl.java| 18 +- .../apache/drill/exec/store/TestPluginRegistry.java| 4 4 files changed, 18 insertions(+), 15 deletions(-)
[drill] branch master updated (cd0d5ec -> ab60b3d)
This is an automated email from the ASF dual-hosted git repository.
arina pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git.
from cd0d5ec DRILL-7596: drill-format-esri module Uses Hard Coded Version
Number
new 9c77284 DRILL-7614: Try to add some helpful tips to Testing.md
new 7abe97f DRILL-7477: Allow passing table function parameters into
ANALYZE statement
new 0c5e347 DRILL-7547: Support credentials store for mongo connections
new cf12325 DRILL-7587: Fix ValuesPrule distribution trait type
new 7509dc4 DRILL-7582: Moved Drillbits REST API communication to the
back end layer
new 7c1aaae DRILL-7590: Refactor plugin registry
new 3d76348 DRILL-7605: Preserve query form field values between reloads
/ navigation
new ab60b3d DRILL-7592: Add missing licenses and update plugins exclusion
list and fix licenses
The 8 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.github/workflows/ci.yml | 16 +-
.mvn/extensions.xml| 29 +-
.../org/apache/drill/common/PlanStringBuilder.java | 119 +++
.../drill/common/collections/ImmutableEntry.java | 17 +-
.../drill/common/config/ConfigConstants.java | 23 +-
.../apache/drill/common/config/DrillConfig.java|6 +-
.../common/exceptions/DrillRuntimeException.java |8 +-
.../drill/common/exceptions/UserException.java |4 +
.../apache/drill/common/scanner/BuildTimeScan.java |4 +-
.../drill/common/scanner/ClassPathScanner.java | 15 +-
.../apache/drill/common/scanner/RunTimeScan.java | 10 +-
.../common/scanner/persistence/ScanResult.java | 10 +-
.../org/apache/drill/exec/util/ActionOnFile.java | 14 +-
common/src/main/resources/drill-default.conf | 32 +-
common/src/main/resources/drill-module.conf| 38 +-
.../java/org/apache/drill/test/DirTestWatcher.java | 82 +-
common/src/test/resources/drill-module.conf| 32 +-
.../main/resources/bootstrap-format-plugins.json |8 +
.../src/main/resources/drill-module.conf | 39 +-
.../exec/store/esri/TestShapefileFormatPlugin.java |2 -
.../drill/exec/store/excel/ExcelFormatConfig.java | 30 +-
.../main/resources/bootstrap-format-plugins.json | 11 +
.../src/main/resources/drill-module.conf | 18 +
.../drill/exec/store/excel/TestExcelFormat.java|3 -
.../main/resources/bootstrap-format-plugins.json | 11 +
.../src/main/resources/drill-module.conf | 39 +-
.../drill/exec/store/hdf5/TestHDF5Format.java |2 -
.../main/resources/bootstrap-format-plugins.json | 11 +
.../src/main/resources/drill-module.conf | 39 +-
.../exec/store/ltsv/TestLTSVRecordReader.java | 23 +-
.../src/main/resources/drill-module.conf | 41 +-
.../mapr/drill/maprdb/tests/MaprDBTestsSuite.java |2 +-
.../exec/store/syslog/SyslogFormatConfig.java |2 +-
.../src/main/resources/drill-module.conf | 39 +-
.../drill/exec/store/syslog/TestSyslogFormat.java | 19 +-
.../apache/drill/exec/store/hbase/HBaseUtils.java |5 +-
.../src/main/resources/drill-module.conf | 39 +-
.../java/org/apache/drill/hbase/BaseHBaseTest.java |5 +-
.../storage-hive/core/src/main/codegen/config.fmpp |4 +-
.../core/src/main/codegen/data/HiveTypes.tdd |4 +-
.../drill/exec/store/hive/HiveStoragePlugin.java |1 +
.../core/src/main/resources/drill-module.conf | 24 +-
.../apache/drill/exec/hive/HiveTestFixture.java|6 +-
.../hive/BaseTestHiveImpersonation.java|2 +-
.../core/src/test/resources/drill-module.conf | 24 +-
.../src/main/resources/drill-module.conf | 24 +-
.../src/main/resources/drill-module.conf | 39 +-
.../exec/store/jdbc/TestJdbcPluginWithH2IT.java|4 +-
.../exec/store/jdbc/TestJdbcPluginWithMySQLIT.java |4 +-
.../src/main/resources/drill-module.conf | 39 +-
.../drill/exec/store/kafka/KafkaTestBase.java |2 +-
contrib/storage-kudu/src/main/codegen/config.fmpp |4 +-
.../src/main/resources/drill-module.conf | 39 +-
.../drill/exec/store/mongo/MongoStoragePlugin.java | 79 +-
.../src/main/resources/drill-module.conf | 39 +-
.../drill/exec/store/mongo/MongoTestBase.java | 11 +-
.../{MongoTestSuit.java => MongoTestSuite.java}| 52 +-
...TestMongoStoragePluginUsesCredentialsStore.java | 89 ++
.../storage-mongo/src/test/resources/core-site.xml | 37 +-
.../exec/store/openTSDB/OpenTSDBStoragePlugin.java |3 +-
.../src/main/resources/drill-module.conf | 39 +-
.../drill/store/openTSDB/
[drill] 02/08: DRILL-7477: Allow passing table function parameters into ANALYZE statement
This is an automated email from the ASF dual-hosted git repository.
arina pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git
commit 7abe97f4ed4b506dd9c86e6489f434c7f910d1c1
Author: Volodymyr Vysotskyi
AuthorDate: Thu Feb 20 18:39:33 2020 +0200
DRILL-7477: Allow passing table function parameters into ANALYZE statement
- Fix logical dir pruning when table function is used
closes #2005
---
.../src/main/codegen/includes/parserImpls.ftl | 24 ++-
.../drill/exec/planner/logical/DrillTable.java | 7 +-
.../drill/exec/planner/sql/SchemaUtilites.java | 13 ++
.../planner/sql/handlers/AnalyzeTableHandler.java | 17 +-
.../exec/planner/sql/handlers/DrillTableInfo.java | 171 +
.../sql/handlers/MetastoreAnalyzeTableHandler.java | 55 ++-
.../exec/planner/sql/parser/SqlAnalyzeTable.java | 30 +---
.../exec/planner/sql/parser/SqlCreateTable.java| 8 +-
.../exec/planner/sql/parser/SqlCreateView.java | 8 +-
.../exec/planner/sql/parser/SqlDropTable.java | 7 +-
.../planner/sql/parser/SqlDropTableMetadata.java | 8 +-
.../drill/exec/planner/sql/parser/SqlDropView.java | 7 +-
.../sql/parser/SqlMetastoreAnalyzeTable.java | 31 +---
.../planner/sql/parser/SqlRefreshMetadata.java | 10 +-
.../drill/exec/planner/sql/parser/SqlSchema.java | 7 +-
.../apache/drill/exec/store/AbstractSchema.java| 2 +
.../exec/store/dfs/WorkspaceSchemaFactory.java | 75 +
.../java/org/apache/drill/TestPartitionFilter.java | 113 ++
.../org/apache/drill/TestSelectWithOption.java | 40 +++--
.../org/apache/drill/exec/sql/TestAnalyze.java | 2 +-
.../drill/exec/sql/TestMetastoreCommands.java | 30
21 files changed, 445 insertions(+), 220 deletions(-)
diff --git a/exec/java-exec/src/main/codegen/includes/parserImpls.ftl
b/exec/java-exec/src/main/codegen/includes/parserImpls.ftl
index ae52f1f..67ecab0 100644
--- a/exec/java-exec/src/main/codegen/includes/parserImpls.ftl
+++ b/exec/java-exec/src/main/codegen/includes/parserImpls.ftl
@@ -700,15 +700,16 @@ Pair
ParenthesizedCompoundIdentifierList() :
/**
* Parses a analyze statements:
*
- * ANALYZE TABLE [table_name] [COLUMNS (col1, col2, ...)] REFRESH METADATA
[partition LEVEL] {COMPUTE | ESTIMATE} | STATISTICS [(column1, column2, ...)] [
SAMPLE numeric PERCENT ]
+ * ANALYZE TABLE [table_name | table({table function name}(parameters))]
[COLUMNS {(col1, col2, ...) | NONE}] REFRESH METADATA ['level' LEVEL] [{COMPUTE
| ESTIMATE} | STATISTICS [ SAMPLE number PERCENT ]]
* ANALYZE TABLE [table_name] DROP [METADATA|STATISTICS] [IF EXISTS]
- * ANALYZE TABLE [table_name] {COMPUTE | ESTIMATE} | STATISTICS [(column1,
column2, ...)] [ SAMPLE numeric PERCENT ]
+ * ANALYZE TABLE [table_name | table({table function name}(parameters))]
{COMPUTE | ESTIMATE} | STATISTICS [(column1, column2, ...)] [ SAMPLE numeric
PERCENT ]
*
*/
SqlNode SqlAnalyzeTable() :
{
SqlParserPos pos;
-SqlIdentifier tblName;
+SqlNode tableRef;
+Span s = null;
SqlNodeList fieldList = null;
SqlNode level = null;
SqlLiteral estimate = null;
@@ -719,7 +720,13 @@ SqlNode SqlAnalyzeTable() :
{
{ pos = getPos(); }
-tblName = CompoundIdentifier()
+(
+tableRef = CompoundIdentifier()
+|
+ { s = span(); }
+tableRef = TableFunctionCall(s.pos())
+
+)
[
(
(
@@ -749,7 +756,7 @@ SqlNode SqlAnalyzeTable() :
]
{
if (percent == null) { percent =
SqlLiteral.createExactNumeric("100.0", pos); }
-return new SqlAnalyzeTable(pos, tblName, estimate, fieldList,
percent);
+return new SqlAnalyzeTable(pos, tableRef, estimate, fieldList,
percent);
}
)
|
@@ -792,7 +799,7 @@ SqlNode SqlAnalyzeTable() :
}
]
{
-return new SqlMetastoreAnalyzeTable(pos, tblName, fieldList,
level, estimate, percent);
+return new SqlMetastoreAnalyzeTable(pos, tableRef, fieldList,
level, estimate, percent);
}
)
|
@@ -816,7 +823,10 @@ SqlNode SqlAnalyzeTable() :
if (checkMetadataExistence == null) {
checkMetadataExistence = SqlLiteral.createBoolean(true, pos);
}
-return new SqlDropTableMetadata(pos, tblName, dropMetadata,
checkMetadataExistence);
+if (s != null) {
+ throw new ParseException("Table functions shouldn't be used
in DROP METADATA statement.");
+}
+return new SqlDropTableMetadata(pos, (SqlIdentifier) tableRef,
dropMetadata, checkMetadataExistence);
}
)
]
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillTable.jav
[drill] 01/08: DRILL-7614: Try to add some helpful tips to Testing.md
This is an automated email from the ASF dual-hosted git repository.
arina pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git
commit 9c7728474440c007fb374c6696520b6b4356a80e
Author: Dobes Vandermeer
AuthorDate: Fri Feb 28 16:24:58 2020 -0800
DRILL-7614: Try to add some helpful tips to Testing.md
closes #2003
---
docs/dev/Testing.md | 55 -
1 file changed, 54 insertions(+), 1 deletion(-)
diff --git a/docs/dev/Testing.md b/docs/dev/Testing.md
index fcbdb14..337f932 100644
--- a/docs/dev/Testing.md
+++ b/docs/dev/Testing.md
@@ -172,6 +172,59 @@ class DrillBuf ...
}
```
+# Simulating GitHub Actions
+
+Sometimes a test failure that occurs in the GitHub Actions CI cannot be
reproduced locally. You can try to run the tests in a more similar environment
using https://github.com/nektos/act
+
+1. Install act according to its instructions
+2. Edit `.github/workflows/ci.yml` to remove extra "matrix" elements and
adjust the maven command line to set the tests you are interested in: ```run:
mvn install -Dsurefire.useFile=false -DtrimStackTrace=false
-Dtest=org.apache.drill.exec.server.rest.TestQueryWrapper
-DfailIfNoTests=false```
+3. Run `act -P ubuntu-latest=nektos/act-environments-ubuntu:18.04 --job build`
to run the build
+
# IntelliJ
-TBA
+IntelliJ IDEA has decent support for maven built in, IntelliJ IDEA users can
use these steps to get up and running:
+
+First, import maven pom.xml as projects.
+
+Change settings in File > Settings > Build, Execution, Deployment > Build
Tools > Maven:
+
+ 1. Enable "Work Offline" to avoid checking for dependency updates on every
run
+ 2. Increase the "Thread Count" to speed up builds - a reasonable default is
the number of CPU cores you have
+ 3. Go to the "Runner" subsection
+ 4. Enable "Delegate IDE build/run actions to Maven". The IDE didn't seem to
do things in a manner compatible with running the application
+ 5. Enable "skip tests" you will probably want to run tests manually rather
than every time you try to run something
+ 6. To skip some uninteresting build steps, you can also set the following
properties to `true`: `checkstyle.skip`, `findbugs.skip`, `license.skip`,
`maven.javadoc.skip`, `rat.skip`, `skipIfEmpty`
+
+At this point you should be able to use the built-in IDE features to run tests
+
+Running/debugging Drillbit is also possible:
+ 1. Create a new run configuration on the class
`org.apache.drill.exec.server.Drillbit`
+ 2. you have to create a run configuration and set the VM parameters:
+
+-Xms4G
+-Xmx4G
+-XX:MaxDirectMemorySize=8G
+-XX:ReservedCodeCacheSize=1G
+-Ddrill.exec.enable-epoll=false
+-XX:MaxPermSize=512M
+-XX:+CMSClassUnloadingEnabled
+-XX:+UseG1GC
+-Dlog.path=/opt/drill/log/drillbit.log
+-Dlog.query.path=/opt/drill/log/drillbit_queries.json
+-Xbootclasspath/a:/opt/drill/conf
+
+ 3. Set environment variables:
+
+DRILL_HOST_NAME=localhost:8047
+DRILL_LOG_DIR=/opt/drill/log
+
+ 4. Edit `/opt/drill/conf/drill-override.conf` to set some options:
+
+drill.exec: {
+ zk.connect: "localhost:2181",
+ allow_loopback_address_binding: true,
+}
+
+ 5. Install and run ZooKeeper on port 2181 (refer to ZooKeeper docs for this)
+ 6. Run the Drillbit. Note that which storage plugins are available depends
on your classpath.
+
[drill] 04/08: DRILL-7587: Fix ValuesPrule distribution trait type
This is an automated email from the ASF dual-hosted git repository.
arina pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git
commit cf1232590c37900b2c187b2981919eb635df662f
Author: Arina Ielchiieva
AuthorDate: Thu Feb 27 17:55:59 2020 +0200
DRILL-7587: Fix ValuesPrule distribution trait type
closes #2002
---
.../planner/physical/MetadataControllerPrule.java | 6 +-
.../drill/exec/planner/physical/ValuesPrule.java | 16 ++---
.../java/org/apache/drill/exec/sql/TestValues.java | 71 ++
3 files changed, 83 insertions(+), 10 deletions(-)
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/MetadataControllerPrule.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/MetadataControllerPrule.java
index 70f2f812..17fbd25 100644
---
a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/MetadataControllerPrule.java
+++
b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/MetadataControllerPrule.java
@@ -27,7 +27,7 @@ import org.apache.drill.exec.planner.logical.RelOptHelper;
public class MetadataControllerPrule extends Prule {
public static final MetadataControllerPrule INSTANCE = new
MetadataControllerPrule();
- public MetadataControllerPrule() {
+ private MetadataControllerPrule() {
super(RelOptHelper.some(MetadataControllerRel.class,
DrillRel.DRILL_LOGICAL,
RelOptHelper.any(RelNode.class)), "MetadataControllerPrule");
}
@@ -41,6 +41,8 @@ public class MetadataControllerPrule extends Prule {
RelNode convertedLeft = convert(left, traits);
RelNode convertedRight = convert(right, traits);
call.transformTo(new MetadataControllerPrel(relNode.getCluster(),
-relNode.getTraitSet().plus(Prel.DRILL_PHYSICAL), convertedLeft,
convertedRight, relNode.getContext()));
+ // force singleton execution since this is the final step for metadata
collection which collects all results into one
+
relNode.getTraitSet().plus(Prel.DRILL_PHYSICAL).plus(DrillDistributionTrait.SINGLETON),
+ convertedLeft, convertedRight, relNode.getContext()));
}
}
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/ValuesPrule.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/ValuesPrule.java
index 1251ddf..13bbd8e 100644
---
a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/ValuesPrule.java
+++
b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/ValuesPrule.java
@@ -17,25 +17,25 @@
*/
package org.apache.drill.exec.planner.physical;
-import org.apache.calcite.plan.RelOptRule;
-import org.apache.drill.exec.planner.logical.DrillRelFactories;
+import org.apache.calcite.plan.RelOptRuleCall;
+import org.apache.drill.exec.planner.logical.DrillRel;
import org.apache.drill.exec.planner.logical.DrillValuesRel;
import org.apache.drill.exec.planner.logical.RelOptHelper;
-import org.apache.calcite.plan.RelOptRuleCall;
-public class ValuesPrule extends RelOptRule {
+public class ValuesPrule extends Prule {
public static final ValuesPrule INSTANCE = new ValuesPrule();
private ValuesPrule() {
-super(RelOptHelper.any(DrillValuesRel.class),
DrillRelFactories.LOGICAL_BUILDER, "Prel.ValuesPrule");
+super(RelOptHelper.any(DrillValuesRel.class, DrillRel.DRILL_LOGICAL),
"Prel.ValuesPrule");
}
@Override
public void onMatch(final RelOptRuleCall call) {
-final DrillValuesRel rel = call.rel(0);
+DrillValuesRel rel = call.rel(0);
call.transformTo(new ValuesPrel(rel.getCluster(), rel.getRowType(),
rel.getTuples(),
-rel.getTraitSet().plus(Prel.DRILL_PHYSICAL), rel.getContent()));
+ // force singleton execution since values rel contains all results and
they cannot be split
+
rel.getTraitSet().plus(Prel.DRILL_PHYSICAL).plus(DrillDistributionTrait.SINGLETON),
+ rel.getContent()));
}
-
}
diff --git
a/exec/java-exec/src/test/java/org/apache/drill/exec/sql/TestValues.java
b/exec/java-exec/src/test/java/org/apache/drill/exec/sql/TestValues.java
new file mode 100644
index 000..0a301ef
--- /dev/null
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/sql/TestValues.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WA
[drill] 07/08: DRILL-7605: Preserve query form field values between reloads / navigation
This is an automated email from the ASF dual-hosted git repository. arina pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/drill.git commit 3d76348f36e1ef2285a0a9479ea90bf6e80e8a22 Author: Dobes Vandermeer AuthorDate: Tue Feb 25 13:58:21 2020 -0800 DRILL-7605: Preserve query form field values between reloads / navigation It is nice to continue where you left off when returning to the query form. closes #1995 --- .../src/main/resources/rest/query/query.ftl| 44 -- 1 file changed, 41 insertions(+), 3 deletions(-) diff --git a/exec/java-exec/src/main/resources/rest/query/query.ftl b/exec/java-exec/src/main/resources/rest/query/query.ftl index aea8bb3..a20cf3e 100644 --- a/exec/java-exec/src/main/resources/rest/query/query.ftl +++ b/exec/java-exec/src/main/resources/rest/query/query.ftl @@ -87,13 +87,51 @@
[drill] 05/08: DRILL-7582: Moved Drillbits REST API communication to the back end layer
This is an automated email from the ASF dual-hosted git repository.
arina pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git
commit 7509dc46d08d7c936aada557e6237fe5fae3624b
Author: Anton Gozhiy
AuthorDate: Mon Feb 17 17:24:45 2020 +0200
DRILL-7582: Moved Drillbits REST API communication to the back end layer
closes #1999
---
exec/java-exec/pom.xml | 11 +++
.../java/org/apache/drill/exec/ExecConstants.java | 1 +
.../apache/drill/exec/server/rest/DrillRoot.java | 15 +++
.../drill/exec/server/rest/StatusResources.java| 12 +++
.../apache/drill/exec/server/rest/WebUtils.java| 106 +
.../java-exec/src/main/resources/drill-module.conf | 3 +
exec/java-exec/src/main/resources/rest/index.ftl | 90 +++--
7 files changed, 181 insertions(+), 57 deletions(-)
diff --git a/exec/java-exec/pom.xml b/exec/java-exec/pom.xml
index 21fc9ff..96e0cec 100644
--- a/exec/java-exec/pom.xml
+++ b/exec/java-exec/pom.xml
@@ -37,6 +37,17 @@
hamcrest-core
+ org.apache.httpcomponents
+ httpasyncclient
+ 4.1.4
+
+
+ commons-logging
+ commons-logging
+
+
+
+
org.apache.kerby
kerb-client
${kerby.version}
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java
index 1276384..7532471 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java
@@ -212,6 +212,7 @@ public final class ExecConstants {
public static final String HTTP_JETTY_SERVER_SELECTORS =
"drill.exec.http.jetty.server.selectors";
public static final String HTTP_JETTY_SERVER_HANDLERS =
"drill.exec.http.jetty.server.handlers";
public static final String HTTP_ENABLE_SSL = "drill.exec.http.ssl_enabled";
+ public static final String HTTP_CLIENT_TIMEOUT =
"drill.exec.http.client.timeout";
public static final String HTTP_CORS_ENABLED =
"drill.exec.http.cors.enabled";
public static final String HTTP_CORS_ALLOWED_ORIGINS =
"drill.exec.http.cors.allowedOrigins";
public static final String HTTP_CORS_ALLOWED_METHODS =
"drill.exec.http.cors.allowedMethods";
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/DrillRoot.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/DrillRoot.java
index e3dad27..1910737 100644
---
a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/DrillRoot.java
+++
b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/DrillRoot.java
@@ -17,15 +17,18 @@
*/
package org.apache.drill.exec.server.rest;
+import java.net.URL;
import java.util.Collection;
import java.util.HashMap;
import java.util.Map;
import javax.annotation.security.PermitAll;
import javax.annotation.security.RolesAllowed;
import javax.inject.Inject;
+import javax.servlet.http.HttpServletRequest;
import javax.ws.rs.GET;
import javax.ws.rs.POST;
import javax.ws.rs.Path;
+import javax.ws.rs.PathParam;
import javax.ws.rs.Produces;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
@@ -53,6 +56,7 @@ import
org.apache.drill.exec.work.foreman.rm.DynamicResourceManager;
import org.apache.drill.exec.work.foreman.rm.QueryQueue;
import org.apache.drill.exec.work.foreman.rm.ResourceManager;
import org.apache.drill.exec.work.foreman.rm.ThrottledResourceManager;
+import org.apache.http.client.methods.HttpPost;
import org.glassfish.jersey.server.mvc.Viewable;
import com.fasterxml.jackson.annotation.JsonCreator;
@@ -72,6 +76,8 @@ public class DrillRoot {
SecurityContext sc;
@Inject
Drillbit drillbit;
+ @Inject
+ HttpServletRequest request;
@GET
@Produces(MediaType.TEXT_HTML)
@@ -133,6 +139,15 @@ public class DrillRoot {
}
@POST
+ @Path("/gracefulShutdown/{hostname}")
+ @Produces(MediaType.APPLICATION_JSON)
+ @RolesAllowed(ADMIN_ROLE)
+ public String shutdownDrillbitByName(@PathParam("hostname") String hostname)
throws Exception {
+URL shutdownURL = WebUtils.getDrillbitURL(work, request, hostname,
"/gracefulShutdown");
+return WebUtils.doHTTPRequest(new HttpPost(shutdownURL.toURI()),
work.getContext().getConfig());
+ }
+
+ @POST
@Path("/shutdown")
@Produces(MediaType.APPLICATION_JSON)
@RolesAllowed(ADMIN_ROLE)
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/StatusResources.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/StatusResources.java
index a27d9c1..00397ce 100644
---
a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/StatusResources.java
+++
b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/
[drill] 03/08: DRILL-7547: Support credentials store for mongo connections
This is an automated email from the ASF dual-hosted git repository.
arina pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git
commit 0c5e347341830e86a64491dd0cd6bf2f723e0072
Author: Dobes Vandermeer
AuthorDate: Thu Feb 27 17:41:07 2020 -0800
DRILL-7547: Support credentials store for mongo connections
This uses the hadoop `Configuration.getPassword` method to retrieve the
username and password for mongo connections. This allows the user to
supply credentials or credential store configuration in core-site.xml
instead of inlining the credentials in the storage plugin configuration
that is stored in ZooKeeper.
Refer to the CredentialProviderAPI document for more information about
how credential provider plugins work.
closes #2001
---
.../drill/exec/store/mongo/MongoStoragePlugin.java | 79 +--
.../drill/exec/store/mongo/MongoTestBase.java | 6 +-
.../{MongoTestSuit.java => MongoTestSuite.java}| 52 +-
...TestMongoStoragePluginUsesCredentialsStore.java | 89 +
.../storage-mongo/src/test/resources/core-site.xml | 37 ---
.../src/main/resources/core-site-example.xml | 108 ++---
6 files changed, 287 insertions(+), 84 deletions(-)
diff --git
a/contrib/storage-mongo/src/main/java/org/apache/drill/exec/store/mongo/MongoStoragePlugin.java
b/contrib/storage-mongo/src/main/java/org/apache/drill/exec/store/mongo/MongoStoragePlugin.java
index 93a80e6..2dfba8c 100644
---
a/contrib/storage-mongo/src/main/java/org/apache/drill/exec/store/mongo/MongoStoragePlugin.java
+++
b/contrib/storage-mongo/src/main/java/org/apache/drill/exec/store/mongo/MongoStoragePlugin.java
@@ -17,13 +17,12 @@
*/
package org.apache.drill.exec.store.mongo;
-import java.io.IOException;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.List;
-import java.util.Set;
-import java.util.concurrent.TimeUnit;
-
+import com.fasterxml.jackson.core.type.TypeReference;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.mongodb.MongoClient;
+import com.mongodb.MongoClientURI;
+import com.mongodb.MongoCredential;
+import com.mongodb.ServerAddress;
import org.apache.calcite.schema.SchemaPlus;
import org.apache.drill.common.JSONOptions;
import org.apache.drill.common.exceptions.ExecutionSetupException;
@@ -34,43 +33,68 @@ import org.apache.drill.exec.store.AbstractStoragePlugin;
import org.apache.drill.exec.store.SchemaConfig;
import org.apache.drill.exec.store.StoragePluginOptimizerRule;
import org.apache.drill.exec.store.mongo.schema.MongoSchemaFactory;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.fasterxml.jackson.core.type.TypeReference;
-import com.fasterxml.jackson.databind.ObjectMapper;
import org.apache.drill.shaded.guava.com.google.common.cache.Cache;
import org.apache.drill.shaded.guava.com.google.common.cache.CacheBuilder;
import org.apache.drill.shaded.guava.com.google.common.cache.RemovalListener;
import
org.apache.drill.shaded.guava.com.google.common.cache.RemovalNotification;
import org.apache.drill.shaded.guava.com.google.common.collect.ImmutableSet;
import org.apache.drill.shaded.guava.com.google.common.collect.Lists;
-import com.mongodb.MongoClient;
-import com.mongodb.MongoClientURI;
-import com.mongodb.MongoCredential;
-import com.mongodb.ServerAddress;
+import org.apache.hadoop.conf.Configuration;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.net.URLEncoder;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
public class MongoStoragePlugin extends AbstractStoragePlugin {
- static final Logger logger = LoggerFactory
- .getLogger(MongoStoragePlugin.class);
+ private static final Logger logger =
LoggerFactory.getLogger(MongoStoragePlugin.class);
private final MongoStoragePluginConfig mongoConfig;
private final MongoSchemaFactory schemaFactory;
private final Cache addressClientMap;
private final MongoClientURI clientURI;
- public MongoStoragePlugin(MongoStoragePluginConfig mongoConfig,
- DrillbitContext context, String name) throws IOException,
- ExecutionSetupException {
+ public MongoStoragePlugin(
+MongoStoragePluginConfig mongoConfig,
+DrillbitContext context,
+String name) throws IOException, ExecutionSetupException {
super(context, name);
this.mongoConfig = mongoConfig;
-this.clientURI = new MongoClientURI(this.mongoConfig.getConnection());
+String connection =
addCredentialsFromCredentialsProvider(this.mongoConfig.getConnection(), name);
+this.clientURI = new MongoClientURI(connection);
this.addressClientMap = CacheBuilder.newBuilder()
-.expireAfterAccess(24, TimeUnit.HOURS)
-.removalListener(new AddressCloser()).bu
[drill] branch master updated (fa3fbe0 -> cd0d5ec)
This is an automated email from the ASF dual-hosted git repository. arina pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/drill.git. from fa3fbe0 DRILL-7578: HDF5 Metadata Queries Fail with Large Files add cd0d5ec DRILL-7596: drill-format-esri module Uses Hard Coded Version Number No new revisions were added by this update. Summary of changes: contrib/format-esri/pom.xml | 91 +++-- 1 file changed, 46 insertions(+), 45 deletions(-)
[drill] branch master updated (70b3bcc -> fa3fbe0)
This is an automated email from the ASF dual-hosted git repository. arina pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/drill.git. from 70b3bcc DRILL-7584: Fix method name spelling add fa3fbe0 DRILL-7578: HDF5 Metadata Queries Fail with Large Files No new revisions were added by this update. Summary of changes: contrib/format-hdf5/README.md | 16 ++- .../drill/exec/store/hdf5/HDF5BatchReader.java | 143 +++-- .../drill/exec/store/hdf5/HDF5FormatConfig.java| 1 + .../drill/exec/store/hdf5/TestHDF5Format.java | 6 +- 4 files changed, 118 insertions(+), 48 deletions(-)
[drill] branch master updated (8a8e58b -> 70b3bcc)
This is an automated email from the ASF dual-hosted git repository. arina pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/drill.git. from 8a8e58b DRILL-7514: Update Apache POI to Latest Version add 70b3bcc DRILL-7584: Fix method name spelling No new revisions were added by this update. Summary of changes: .../org/apache/drill/exec/store/mongo/MongoStoragePluginConfig.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
[drill] branch master updated (1288e71 -> 1acbb3f)
This is an automated email from the ASF dual-hosted git repository. arina pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/drill.git. from 1288e71 DRILL-7543: Use GitHub Actions for CI add 1acbb3f DRILL-7570: Fix unstable statistics tests No new revisions were added by this update. Summary of changes: .github/workflows/ci.yml | 1 + .../org/apache/drill/exec/sql/TestAnalyze.java | 424 - .../drill/exec/sql/TestMetastoreCommands.java | 23 +- 3 files changed, 264 insertions(+), 184 deletions(-)
[drill] branch master updated: DRILL-7543: Use GitHub Actions for CI
This is an automated email from the ASF dual-hosted git repository.
arina pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git
The following commit(s) were added to refs/heads/master by this push:
new 1288e71 DRILL-7543: Use GitHub Actions for CI
1288e71 is described below
commit 1288e71a756ed409b5eaa6af48b27a320513e043
Author: Volodymyr Vysotskyi
AuthorDate: Tue Feb 4 13:39:21 2020 +0200
DRILL-7543: Use GitHub Actions for CI
---
.circleci/config.yml | 230 -
.github/workflows/ci.yml | 103 +
.travis.yml| 69 ---
README.md | 2 +-
contrib/pom.xml| 8 -
docs/dev/LicenseHeaders.md | 2 +-
docs/dev/Testing.md| 8 -
drill-yarn/pom.xml | 8 -
.../drill/exec/compile/DrillJavaFileManager.java | 15 +-
.../drill/exec/compile/JDKClassCompiler.java | 40 ++--
exec/jdbc-all/pom.xml | 6 -
exec/jdbc/pom.xml | 6 -
12 files changed, 134 insertions(+), 363 deletions(-)
diff --git a/.circleci/config.yml b/.circleci/config.yml
deleted file mode 100644
index 71d424f..000
--- a/.circleci/config.yml
+++ /dev/null
@@ -1,230 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-version: 2
-general:
-jobs:
- build_jdk8:
-machine:
- enabled: true
- image: circleci/classic:latest
-parallelism: 1 # TODO: 1. Configuring Parallel Jobs for Running Tests
https://circleci.com/docs/2.0/parallelism-faster-jobs/
-
-working_directory: ~/drill
-
-steps:
- - checkout
- - restore_cache:
- keys:
-- m2-{{ checksum "pom.xml" }}
-- m2- # used if checksum fails
- - run:
- name: Update maven version
- # TODO: Could be removed, once Machine Executor image is updated
https://github.com/circleci/image-builder/issues/140
- # and the possibility of specifying Maven version is added
https://github.com/circleci/image-builder/issues/143
- command:
-curl -fsSL https://git.io/vpDIf | bash -s -- 3.6.3
- - run:
- name: Update packages list
- command:
-sudo apt-get update
- - run:
- name: Install libaio1.so library for MySQL integration tests
- command:
-sudo apt-get install libaio1 libaio-dev
- - run:
- name: Drill project build
- # TODO: 2. Optimizing Maven Builds on CircleCI -
https://circleci.com/blog/optimizing-maven-builds-on-circleci/
- # TODO: 3. Resolving memory issues without "SlowTest" and
"UnlikelyTest" excludedGroups in the build
- command: >
-mvn install -Drat.skip=false -Dlicense.skip=false -DmemoryMb=2560
-DdirectMemoryMb=4608
-
-DexcludedGroups="org.apache.drill.categories.SlowTest,org.apache.drill.categories.UnlikelyTest"
--batch-mode
- - run:
- name: Save test results
- command: |
-mkdir -p ~/test-results/junit/
-find . -type f -regex ".*/target/surefire-reports/.*xml" -exec cp
{} ~/test-results/junit/ \;
- when: always
- - store_test_results:
- path: ~/test-results
- - save_cache:
- paths:
-- ~/.m2
- key: m2-{{ checksum "pom.xml" }}
-
- build_jdk11:
-machine:
- enabled: true
- image: circleci/classic:latest
-parallelism: 1
-
-working_directory: ~/drill
-
-steps:
- - checkout
- - restore_cache:
- keys:
-- m2-{{ checksum "pom.xml" }}
-- m2- # used if checksum fails
- - run:
- name: Update packages list
- command:
-sudo apt-get update
- - run:
- name: Install java 11
- command:
-sudo apt-get -y install openjdk-11-jdk
- - run:
- name: Set default java 11
-
[drill] branch master updated: DRILL-7544: Upgrade Iceberg version to support Parquet 1.11.0
This is an automated email from the ASF dual-hosted git repository.
arina pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git
The following commit(s) were added to refs/heads/master by this push:
new 9f7c58f DRILL-7544: Upgrade Iceberg version to support Parquet 1.11.0
9f7c58f is described below
commit 9f7c58fc5a47cfe67e5fb5f5f35971cf534133d5
Author: Arina Ielchiieva
AuthorDate: Wed Jan 22 15:55:09 2020 +0200
DRILL-7544: Upgrade Iceberg version to support Parquet 1.11.0
1. Upgraded Iceberg to the commit that supports Parquet 1.11.0.
2. Removed workaround in ExpirationHandler and used built-in logic from
Iceberg library.
3. Updated description about expiration logic in README.md.
---
metastore/iceberg-metastore/README.md | 10 +-
metastore/iceberg-metastore/pom.xml| 12 +-
.../drill/metastore/iceberg/IcebergMetastore.java | 6 +-
.../iceberg/components/tables/IcebergTables.java | 4 +-
.../iceberg/config/IcebergConfigConstants.java | 5 -
.../iceberg/operate/ExpirationHandler.java | 263 +++--
.../metastore/iceberg/operate/IcebergModify.java | 5 +-
.../src/main/resources/drill-metastore-module.conf | 14 +-
.../iceberg/operate/TestExpirationHandler.java | 149 +---
9 files changed, 119 insertions(+), 349 deletions(-)
diff --git a/metastore/iceberg-metastore/README.md
b/metastore/iceberg-metastore/README.md
index 3db08d1..094154b 100644
--- a/metastore/iceberg-metastore/README.md
+++ b/metastore/iceberg-metastore/README.md
@@ -183,8 +183,8 @@ expiration process is launched.
Iceberg table generates metadata for each modification operation:
snapshot, manifest file, table metadata file. Also when performing delete
operation,
previously stored data files are not deleted. These files with the time
-can occupy lots of space. `ExpirationHandler` allows to expire outdated
metadata and
-data files after configured time period
(`drill.metastore.iceberg.expiration.period`).
-If expiration period is not indicated, zero or negative, expiration won't be
performed.
-`ExpirationHandler` is called after each modification operation, it checks if
expiration period
-has elapsed and submits expiration process in a separate thread.
+can occupy lots of space. Two table properties
`write.metadata.delete-after-commit.enabled`
+and `write.metadata.previous-versions-max` control expiration process.
+Metadata files will be expired automatically if
`write.metadata.delete-after-commit.enabled`
+is enabled. Snapshots and data files will be expired using `ExpirationHandler`
+after each commit operation based on the same table properties values.
diff --git a/metastore/iceberg-metastore/pom.xml
b/metastore/iceberg-metastore/pom.xml
index 818ea08..aa68271 100644
--- a/metastore/iceberg-metastore/pom.xml
+++ b/metastore/iceberg-metastore/pom.xml
@@ -31,7 +31,7 @@
metastore/Drill Iceberg Metastore
-0.7.0-incubating
+2d75130
2.7.0
@@ -47,27 +47,27 @@
${project.version}
- org.apache.iceberg
+ com.github.apache.incubator-iceberg
iceberg-parquet
${iceberg.version}
- org.apache.iceberg
+ com.github.apache.incubator-iceberg
iceberg-data
${iceberg.version}
- org.apache.iceberg
+ com.github.apache.incubator-iceberg
iceberg-core
${iceberg.version}
- org.apache.iceberg
+ com.github.apache.incubator-iceberg
iceberg-common
${iceberg.version}
- org.apache.iceberg
+ com.github.apache.incubator-iceberg
iceberg-api
${iceberg.version}
diff --git
a/metastore/iceberg-metastore/src/main/java/org/apache/drill/metastore/iceberg/IcebergMetastore.java
b/metastore/iceberg-metastore/src/main/java/org/apache/drill/metastore/iceberg/IcebergMetastore.java
index e223c5a..94bc6bf 100644
---
a/metastore/iceberg-metastore/src/main/java/org/apache/drill/metastore/iceberg/IcebergMetastore.java
+++
b/metastore/iceberg-metastore/src/main/java/org/apache/drill/metastore/iceberg/IcebergMetastore.java
@@ -25,7 +25,6 @@ import org.apache.drill.metastore.components.views.Views;
import org.apache.drill.metastore.iceberg.components.tables.IcebergTables;
import org.apache.drill.metastore.iceberg.config.IcebergConfigConstants;
import org.apache.drill.metastore.iceberg.exceptions.IcebergMetastoreException;
-import org.apache.drill.metastore.iceberg.operate.ExpirationHandler;
import org.apache.drill.metastore.iceberg.schema.IcebergTableSchema;
import org.apache.drill.shaded.guava.com.google.common.collect.MapDifference;
import org.apache.drill.shaded.guava.com.google.common.collect.Maps;
@@ -59,7 +58,6 @@ public class IcebergMetastore implements Metastore {
private final org.apache.iceberg.Tables tables;
private final String baseLocation;
private final Map commonProperties
[drill] branch master updated (65fb7dd -> 9ad9b76)
This is an automated email from the ASF dual-hosted git repository. arina pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/drill.git. from 65fb7dd DRILL-7509: Incorrect TupleSchema is created for DICT column when querying Parquet files add 9ad9b76 DRILL-7540: Update Calcite.md doc No new revisions were added by this update. Summary of changes: docs/dev/Calcite.md | 30 +- 1 file changed, 13 insertions(+), 17 deletions(-)
[drill] 07/07: DRILL-7530: Fix class names in loggers
This is an automated email from the ASF dual-hosted git repository.
arina pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git
commit 1d2da6d35814b9fcad9379941590a6ec22d28da9
Author: Arina Ielchiieva
AuthorDate: Fri Jan 17 18:42:08 2020 +0200
DRILL-7530: Fix class names in loggers
1. Fix incorrect class names for loggers.
2. Minor code cleanup.
closes #1957
---
.../drill/exec/store/mapr/db/MapRDBTableCache.java | 5 ++-
.../apache/drill/exec/expr/HiveFuncHolderExpr.java | 4 +--
.../apache/drill/exec/expr/fn/HiveFuncHolder.java | 1 -
.../exec/store/hive/HiveMetadataProvider.java | 7 ++--
.../store/hive/client/TableEntryCacheLoader.java | 2 +-
.../drill/exec/store/kafka/TestKafkaSuit.java | 3 +-
.../exec/store/openTSDB/OpenTSDBRecordReader.java | 16 -
.../openTSDB/OpenTSDBStoragePluginConfig.java | 7 ++--
.../drill/exec/expr/fn/impl/MathFunctions.java | 5 +--
.../drill/exec/physical/config/UnionAll.java | 2 --
.../drill/exec/physical/impl/ScreenCreator.java| 8 ++---
.../exec/physical/impl/SingleSenderCreator.java| 8 ++---
.../physical/impl/aggregate/HashAggTemplate.java | 7 ++--
...ledRecordbatch.java => SpilledRecordBatch.java} | 16 -
.../impl/aggregate/StreamingAggTemplate.java | 2 +-
.../physical/impl/common/HashTableTemplate.java| 4 ++-
.../exec/physical/impl/join/HashJoinBatch.java | 10 +++---
.../impl/join/HashJoinMemoryCalculator.java| 4 +--
.../impl/join/HashJoinMemoryCalculatorImpl.java| 16 -
.../impl/limit/PartitionLimitRecordBatch.java | 3 +-
.../physical/impl/protocol/OperatorDriver.java | 13 +---
.../physical/impl/window/FrameSupportTemplate.java | 5 ++-
.../drill/exec/physical/impl/xsort/SortConfig.java | 5 ++-
.../drill/exec/physical/impl/xsort/SortImpl.java | 9 +++--
.../physical/impl/xsort/SortMemoryManager.java | 7 ++--
.../physical/resultSet/impl/SingleVectorState.java | 22 +++-
.../apache/drill/exec/planner/PlannerPhase.java| 1 -
.../drill/exec/planner/common/DrillRelOptUtil.java | 7 ++--
.../planner/index/AbstractIndexDescriptor.java | 1 -
.../exec/planner/index/DrillIndexDescriptor.java | 4 +++
.../generators/CoveringPlanNoFilterGenerator.java | 7 +++-
.../sql/handlers/AbstractSqlSetHandler.java| 5 ++-
.../drill/exec/record/AbstractRecordBatch.java | 4 +--
.../exec/server/options/PersistedOptionValue.java | 7 ++--
.../drill/exec/server/rest/ThreadsResources.java | 2 --
.../apache/drill/exec/ssl/SSLConfigBuilder.java| 8 -
.../easy/text/reader/RepeatedVarCharOutput.java| 2 +-
.../exec/store/httpd/HttpdLogFormatPlugin.java | 19 ++-
.../drill/exec/store/httpd/HttpdLogRecord.java | 39 +++---
.../apache/drill/exec/store/httpd/HttpdParser.java | 15 +
.../drill/exec/store/mock/MockSubScanPOP.java | 1 -
.../drill/exec/store/sys/store/InMemoryStore.java | 1 -
.../drill/exec/vector/complex/fn/VectorOutput.java | 13
.../hadoop/ParquetColumnChunkPageWriteStore.java | 6 ++--
.../apache/drill/TestDisabledFunctionality.java| 3 +-
.../physical/impl/common/HashPartitionTest.java| 4 +--
.../TestLateralJoinCorrectnessBatchProcessing.java | 1 -
.../exec/physical/impl/join/TestMergeJoin.java | 6 ++--
.../impl/protocol/TestOperatorRecordBatch.java | 5 ++-
.../drill/exec/testing/TestPauseInjection.java | 14
.../drill/exec/util/MiniZooKeeperCluster.java | 35 +--
.../complex/writer/TestComplexTypeWriter.java | 3 +-
.../org/apache/drill/test/DrillTestWrapper.java| 5 ++-
.../drill/jdbc/impl/DrillConnectionImpl.java | 8 ++---
.../org/apache/drill/jdbc/impl/DrillCursor.java| 5 ++-
55 files changed, 228 insertions(+), 194 deletions(-)
diff --git
a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/MapRDBTableCache.java
b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/MapRDBTableCache.java
index df8d59d..281fd6d 100644
---
a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/MapRDBTableCache.java
+++
b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/MapRDBTableCache.java
@@ -34,9 +34,12 @@ import
org.apache.drill.shaded.guava.com.google.common.cache.CacheLoader;
import org.apache.drill.shaded.guava.com.google.common.cache.LoadingCache;
import org.apache.drill.shaded.guava.com.google.common.cache.RemovalListener;
import
org.apache.drill.shaded.guava.com.google.common.cache.RemovalNotification;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public class MapRDBTableCache {
- static final org.slf4j.Logger logger =
org.slf4j.LoggerFactory.getLogger(MapRDBFormatPlugin.class);
+
+ private static final Logger logger =
LoggerFactory.getLogger(MapRDBTableCache.cl
[drill] branch master updated (a9caa32 -> 1d2da6d)
This is an automated email from the ASF dual-hosted git repository.
arina pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git.
from a9caa32 Updating dev environment setup doc to skip tests on install
new 91a9c56 DRILL-7506: Simplify code gen error handling
new 7453166 DRILL-7233: Format Plugin for HDF5
new 6fef082 DRILL-7527: DROP METADATA doesn't work with table name
starting with '/' inside workspace
new f1b9761 DRILL-7491: Incorrect count() returned for complex types in
parquet
new bf7277c DRILL-7454: Convert Avro to EVF
new 50cd931 DRILL-7467: Jdbc plugin enhancements and fixes
new 1d2da6d DRILL-7530: Fix class names in loggers
The 7 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../org/apache/drill/common/AutoCloseables.java| 20 +-
.../drill/common/exceptions/UserException.java | 20 +-
.../apache/drill/common/util/DrillStringUtils.java | 37 +-
.../test/java/org/apache/drill/test/BaseTest.java |2 +-
contrib/format-hdf5/README.md | 135 +++
contrib/{storage-jdbc => format-hdf5}/pom.xml | 71 +-
.../drill/exec/store/hdf5/HDF5Attribute.java | 34 +-
.../drill/exec/store/hdf5/HDF5BatchReader.java | 1120
.../drill/exec/store/hdf5/HDF5DrillMetadata.java | 47 +-
.../drill/exec/store/hdf5/HDF5FormatConfig.java| 46 +-
.../drill/exec/store/hdf5/HDF5FormatPlugin.java| 117 ++
.../apache/drill/exec/store/hdf5/HDF5Utils.java| 268 +
.../exec/store/hdf5/writers/HDF5DataWriter.java| 93 ++
.../store/hdf5/writers/HDF5DoubleDataWriter.java | 109 ++
.../store/hdf5/writers/HDF5EnumDataWriter.java | 57 +
.../store/hdf5/writers/HDF5FloatDataWriter.java| 109 ++
.../exec/store/hdf5/writers/HDF5IntDataWriter.java | 112 ++
.../store/hdf5/writers/HDF5LongDataWriter.java | 109 ++
.../exec/store/hdf5/writers/HDF5MapDataWriter.java | 139 +++
.../store/hdf5/writers/HDF5StringDataWriter.java | 71 ++
.../hdf5/writers/HDF5TimestampDataWriter.java | 62 ++
.../main/resources/bootstrap-format-plugins.json | 26 +
.../src/main/resources/drill-module.conf | 20 +
.../drill/exec/store/hdf5/TestHDF5Format.java | 938
.../drill/exec/store/hdf5/TestHDF5Utils.java | 58 +
.../src/test/resources/hdf5/browsing.h5| Bin 0 -> 13552 bytes
.../format-hdf5/src/test/resources/hdf5/dset.h5| Bin 0 -> 1496 bytes
.../src/test/resources/hdf5/non-scalar.h5 | Bin 0 -> 16464 bytes
.../format-hdf5/src/test/resources/hdf5/scalar.h5 | Bin 0 -> 14152 bytes
.../drill/exec/store/mapr/db/MapRDBTableCache.java |5 +-
.../native/client/src/protobuf/UserBitShared.pb.cc | 20 +-
.../native/client/src/protobuf/UserBitShared.pb.h |1 +
contrib/pom.xml|1 +
.../apache/drill/exec/expr/HiveFuncHolderExpr.java |4 +-
.../apache/drill/exec/expr/fn/HiveFuncHolder.java |1 -
.../exec/store/hive/HiveMetadataProvider.java |7 +-
.../store/hive/client/TableEntryCacheLoader.java |2 +-
contrib/storage-jdbc/pom.xml |4 +
.../drill/exec/store/jdbc/DrillJdbcRuleBase.java |7 +-
.../drill/exec/store/jdbc/JdbcBatchCreator.java|2 +-
.../org/apache/drill/exec/store/jdbc/JdbcDrel.java |1 -
.../drill/exec/store/jdbc/JdbcExpressionCheck.java | 11 +-
.../exec/store/jdbc/JdbcIntermediatePrel.java |6 +-
.../org/apache/drill/exec/store/jdbc/JdbcPrel.java |5 +-
.../drill/exec/store/jdbc/JdbcRecordReader.java| 117 +-
.../drill/exec/store/jdbc/JdbcStorageConfig.java | 41 +-
.../drill/exec/store/jdbc/JdbcStoragePlugin.java | 84 +-
.../main/resources/bootstrap-storage-plugins.json | 16 +
.../drill/exec/store/jdbc/TestBasicDataSource.java | 106 ++
.../exec/store/jdbc/TestJdbcPluginWithH2IT.java| 73 +-
.../exec/store/jdbc/TestJdbcPluginWithMySQLIT.java | 14 +-
.../drill/exec/store/kafka/TestKafkaSuit.java |3 +-
.../exec/store/openTSDB/OpenTSDBRecordReader.java | 16 +-
.../openTSDB/OpenTSDBStoragePluginConfig.java |7 +-
contrib/udfs/pom.xml |4 +-
distribution/pom.xml |5 +
distribution/src/assemble/component.xml|1 +
.../drill/exec/cache/CachedVectorContainer.java| 15 +-
.../exec/expr/ExpressionTreeMaterializer.java |9 +-
.../drill/exec/expr/fn/impl/MathFunctions.java |5 +-
.../apache/drill/exec/ops/BaseFragmentContext.java | 34 +-
.../org/apache/drill/exec/ops/FragmentContext.java | 14 +-
.../base/AbstractGroupScanWithMetadata.java
[drill] 06/07: DRILL-7467: Jdbc plugin enhancements and fixes
This is an automated email from the ASF dual-hosted git repository.
arina pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git
commit 50cd931da0364355535029db8e9d7a1445218803
Author: Arina Ielchiieva
AuthorDate: Tue Jan 14 23:48:06 2020 +0200
DRILL-7467: Jdbc plugin enhancements and fixes
1. Added logic to close data source when plugin is closed.
2. Added disabled jdbc plugin template to the bootstrap storage plugins.
3. Added new jdbc storage plugin configuration property sourceParameters
which would allow setting data source parameters described in BasicDataSource
Configuration Parameters.
4. Upgraded commons-dbcp2 version and added it to the dependency management
section in common pom.xml.
closes #1956
---
.../org/apache/drill/common/AutoCloseables.java| 20 ++--
contrib/storage-jdbc/pom.xml | 4 +
.../drill/exec/store/jdbc/DrillJdbcRuleBase.java | 7 +-
.../drill/exec/store/jdbc/JdbcBatchCreator.java| 2 +-
.../org/apache/drill/exec/store/jdbc/JdbcDrel.java | 1 -
.../drill/exec/store/jdbc/JdbcExpressionCheck.java | 11 +-
.../exec/store/jdbc/JdbcIntermediatePrel.java | 6 +-
.../org/apache/drill/exec/store/jdbc/JdbcPrel.java | 5 +-
.../drill/exec/store/jdbc/JdbcRecordReader.java| 117 +
.../drill/exec/store/jdbc/JdbcStorageConfig.java | 41
.../drill/exec/store/jdbc/JdbcStoragePlugin.java | 84 +++
.../main/resources/bootstrap-storage-plugins.json | 16 +++
.../drill/exec/store/jdbc/TestBasicDataSource.java | 106 +++
.../exec/store/jdbc/TestJdbcPluginWithH2IT.java| 73 +++--
.../exec/store/jdbc/TestJdbcPluginWithMySQLIT.java | 14 +--
contrib/udfs/pom.xml | 4 +-
.../drill/common/logical/StoragePluginConfig.java | 3 +-
pom.xml| 16 ++-
18 files changed, 348 insertions(+), 182 deletions(-)
diff --git a/common/src/main/java/org/apache/drill/common/AutoCloseables.java
b/common/src/main/java/org/apache/drill/common/AutoCloseables.java
index 7a5fb69..95c0b6f 100644
--- a/common/src/main/java/org/apache/drill/common/AutoCloseables.java
+++ b/common/src/main/java/org/apache/drill/common/AutoCloseables.java
@@ -30,7 +30,7 @@ import org.slf4j.LoggerFactory;
*/
public class AutoCloseables {
- private static final Logger LOGGER =
LoggerFactory.getLogger(AutoCloseables.class);
+ private static final Logger logger =
LoggerFactory.getLogger(AutoCloseables.class);
public interface Closeable extends AutoCloseable {
@Override
@@ -38,12 +38,7 @@ public class AutoCloseables {
}
public static AutoCloseable all(final Collection
autoCloseables) {
-return new AutoCloseable() {
- @Override
- public void close() throws Exception {
-AutoCloseables.close(autoCloseables);
- }
-};
+return () -> close(autoCloseables);
}
/**
@@ -80,7 +75,7 @@ public class AutoCloseables {
try {
close(Arrays.asList(autoCloseables));
} catch (Exception e) {
- throw UserException.dataReadError(e).build(LOGGER);
+ throw UserException.dataReadError(e).build(logger);
}
}
@@ -88,9 +83,9 @@ public class AutoCloseables {
* Closes all autoCloseables if not null and suppresses subsequent
exceptions if more than one
* @param autoCloseables the closeables to close
*/
- public static void close(Iterable ac) throws
Exception {
+ public static void close(Iterable autoCloseables)
throws Exception {
Exception topLevelException = null;
-for (AutoCloseable closeable : ac) {
+for (AutoCloseable closeable : autoCloseables) {
try {
if (closeable != null) {
closeable.close();
@@ -110,7 +105,7 @@ public class AutoCloseables {
/**
* Close all without caring about thrown exceptions
- * @param closeables - array containing auto closeables
+ * @param closeables array containing auto closeables
*/
public static void closeSilently(AutoCloseable... closeables) {
Arrays.stream(closeables).filter(Objects::nonNull)
@@ -118,9 +113,8 @@ public class AutoCloseables {
try {
target.close();
} catch (Exception e) {
-LOGGER.warn(String.format("Exception was thrown while closing auto
closeable: %s", target), e);
+logger.warn("Exception was thrown while closing auto closeable:
{}", target, e);
}
});
}
-
}
diff --git a/contrib/storage-jdbc/pom.xml b/contrib/storage-jdbc/pom.xml
index f702a2e..299d9db 100755
--- a/contrib/storage-jdbc/pom.xml
+++ b/contrib/storage-jdbc/pom.xml
@@ -40,6 +40,10 @@
drill-java-exec
${project.version}
+
+ org.apache.commons
+ commons-dbcp2
+
diff --git
a/contrib/storage-jdbc/src/main/java/org/apache/drill/exec/store/jdbc/Dri
[drill] 03/07: DRILL-7527: DROP METADATA doesn't work with table name starting with '/' inside workspace
This is an automated email from the ASF dual-hosted git repository.
arina pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git
commit 6fef0820d427aca92df42ca5c578e932d7a0b36b
Author: Volodymyr Vysotskyi
AuthorDate: Tue Jan 21 20:16:36 2020 +0200
DRILL-7527: DROP METADATA doesn't work with table name starting with '/'
inside workspace
closes #1958
---
.../apache/drill/common/util/DrillStringUtils.java | 37 +++---
.../sql/conversion/DrillCalciteCatalogReader.java | 4 +--
.../planner/sql/handlers/CreateTableHandler.java | 6 ++--
.../planner/sql/handlers/DropTableHandler.java | 4 +--
.../exec/planner/sql/handlers/ViewHandler.java | 6 ++--
.../drill/exec/planner/sql/parser/SqlSchema.java | 4 +--
.../apache/drill/exec/store/dfs/FileSelection.java | 12 ++-
.../exec/store/dfs/WorkspaceSchemaFactory.java | 3 +-
.../drill/exec/sql/TestMetastoreCommands.java | 30 ++
.../drill/exec/store/dfs/TestFileSelection.java| 5 +--
.../apache/drill/metastore/metadata/TableInfo.java | 4 ++-
11 files changed, 70 insertions(+), 45 deletions(-)
diff --git
a/common/src/main/java/org/apache/drill/common/util/DrillStringUtils.java
b/common/src/main/java/org/apache/drill/common/util/DrillStringUtils.java
index 0fb6f80..de65ff8 100644
--- a/common/src/main/java/org/apache/drill/common/util/DrillStringUtils.java
+++ b/common/src/main/java/org/apache/drill/common/util/DrillStringUtils.java
@@ -17,13 +17,13 @@
*/
package org.apache.drill.common.util;
-import org.apache.drill.shaded.guava.com.google.common.base.Joiner;
import io.netty.buffer.ByteBuf;
import org.apache.commons.lang3.StringEscapeUtils;
import org.apache.commons.lang3.StringUtils;
-import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.stream.Collectors;
public class DrillStringUtils {
@@ -51,7 +51,7 @@ public class DrillStringUtils {
* @param input the {@code String} to unescape, may be null
* @return a new unescaped {@code String}, {@code null} if null string input
*/
- public static final String unescapeJava(String input) {
+ public static String unescapeJava(String input) {
return StringEscapeUtils.unescapeJava(input);
}
@@ -72,11 +72,11 @@ public class DrillStringUtils {
* @param input String to escape values in, may be null
* @return String with escaped values, {@code null} if null string input
*/
- public static final String escapeJava(String input) {
+ public static String escapeJava(String input) {
return StringEscapeUtils.escapeJava(input);
}
- public static final String escapeNewLines(String input) {
+ public static String escapeNewLines(String input) {
if (input == null) {
return null;
}
@@ -209,21 +209,20 @@ public class DrillStringUtils {
* @return The sanitized CSV string
*/
public static String sanitizeCSV(String csv) {
-// tokenize
String[] tokens = csv.split(",");
-ArrayList sanitizedTokens = new ArrayList(tokens.length);
-// check for empties
-for (String s : tokens) {
- String trimmedToken = s.trim();
- if (trimmedToken.length() != 0) {
-sanitizedTokens.add(trimmedToken);
- }
-}
-String result = "";
-if (sanitizedTokens.size() != 0) {
- result = Joiner.on(",").join(sanitizedTokens);
-}
-return result;
+return Arrays.stream(tokens)
+.map(String::trim)
+.filter(StringUtils::isNotEmpty)
+.collect(Collectors.joining(","));
}
+ /**
+ * Removes all leading slash characters from specified string.
+ *
+ * @param path string to remove all leading slash characters
+ * @return string with removed leading slash characters
+ */
+ public static String removeLeadingSlash(String path) {
+return StringUtils.stripStart(path, "/");
+ }
}
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/conversion/DrillCalciteCatalogReader.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/conversion/DrillCalciteCatalogReader.java
index 4c8b254..b6fdcc5 100644
---
a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/conversion/DrillCalciteCatalogReader.java
+++
b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/conversion/DrillCalciteCatalogReader.java
@@ -35,11 +35,11 @@ import org.apache.calcite.sql.validate.SqlValidatorUtil;
import org.apache.calcite.util.Util;
import org.apache.drill.common.config.DrillConfig;
import org.apache.drill.common.exceptions.UserException;
+import org.apache.drill.common.util.DrillStringUtils;
import org.apache.drill.exec.metastore.MetadataProviderManager;
import org.apache.drill.exec.planner.logical.DrillTable;
import org.apache.drill.exec.planner.sql.SchemaUtilites;
import org.apache.drill.exec.rpc.user.UserSession;
-import org.
[drill] 04/07: DRILL-7491: Incorrect count() returned for complex types in parquet
This is an automated email from the ASF dual-hosted git repository.
arina pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git
commit f1b976126e592c67e5de2448573c6172e74ee763
Author: Igor Guzenko
AuthorDate: Mon Jan 13 20:35:54 2020 +0200
DRILL-7491: Incorrect count() returned for complex types in parquet
closes #1955
---
.../base/AbstractGroupScanWithMetadata.java| 41 +++--
.../logical/TestConvertCountToDirectScan.java | 18 +
.../exec/store/parquet/TestParquetComplex.java | 14 +++
.../parquet/hive_all/hive_alltypes.parquet | Bin 0 -> 2161 bytes
.../apache/drill/common/expression/SchemaPath.java | 34 +++--
5 files changed, 67 insertions(+), 40 deletions(-)
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/AbstractGroupScanWithMetadata.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/AbstractGroupScanWithMetadata.java
index 40ab594..9293a8d 100644
---
a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/AbstractGroupScanWithMetadata.java
+++
b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/AbstractGroupScanWithMetadata.java
@@ -167,29 +167,46 @@ public abstract class AbstractGroupScanWithMetadata columnStats =
getTableMetadata().getColumnStatistics(column);
-ColumnStatistics nonInterestingColStats = null;
-if (columnStats == null) {
- nonInterestingColStats =
getNonInterestingColumnsMetadata().getColumnStatistics(column);
-}
+ColumnStatistics nonInterestingColStats = columnStats == null
+? getNonInterestingColumnsMetadata().getColumnStatistics(column) :
null;
+long tableRowCount;
if (columnStats != null) {
tableRowCount =
TableStatisticsKind.ROW_COUNT.getValue(getTableMetadata());
} else if (nonInterestingColStats != null) {
tableRowCount =
TableStatisticsKind.ROW_COUNT.getValue(getNonInterestingColumnsMetadata());
+ columnStats = nonInterestingColStats;
+} else if (hasNestedStatsForColumn(column, getTableMetadata())
+|| hasNestedStatsForColumn(column,
getNonInterestingColumnsMetadata())) {
+ // When statistics for nested field exists, this is complex column which
is present in table.
+ // But its nested fields statistics can't be used to extract
tableRowCount for this column.
+ // So NO_COLUMN_STATS returned here to avoid problems described in
DRILL-7491.
+ return Statistic.NO_COLUMN_STATS;
} else {
return 0; // returns 0 if the column doesn't exist in the table.
}
-columnStats = columnStats != null ? columnStats : nonInterestingColStats;
-nulls = ColumnStatisticsKind.NULLS_COUNT.getFrom(columnStats);
-colNulls = nulls != null ? nulls : Statistic.NO_COLUMN_STATS;
+Long nulls = ColumnStatisticsKind.NULLS_COUNT.getFrom(columnStats);
+if (nulls == null || Statistic.NO_COLUMN_STATS == nulls ||
Statistic.NO_COLUMN_STATS == tableRowCount) {
+ return Statistic.NO_COLUMN_STATS;
+} else {
+ return tableRowCount - nulls;
+}
+ }
-return Statistic.NO_COLUMN_STATS == tableRowCount
-|| Statistic.NO_COLUMN_STATS == colNulls
-? Statistic.NO_COLUMN_STATS : tableRowCount - colNulls;
+ /**
+ * For complex columns, stats may be present only for nested fields. For
example, a column path is `a`,
+ * but stats present for `a`.`b`. So before making a decision that column is
absent, the case needs
+ * to be tested.
+ *
+ * @param column parent column path
+ * @param metadata metadata with column statistics
+ * @return whether stats exists for nested fields
+ */
+ private boolean hasNestedStatsForColumn(SchemaPath column, Metadata
metadata) {
+return metadata.getColumnsStatistics().keySet().stream()
+.anyMatch(path -> path.contains(column));
}
@Override
diff --git
a/exec/java-exec/src/test/java/org/apache/drill/exec/planner/logical/TestConvertCountToDirectScan.java
b/exec/java-exec/src/test/java/org/apache/drill/exec/planner/logical/TestConvertCountToDirectScan.java
index ae33f0a..8a5202b 100644
---
a/exec/java-exec/src/test/java/org/apache/drill/exec/planner/logical/TestConvertCountToDirectScan.java
+++
b/exec/java-exec/src/test/java/org/apache/drill/exec/planner/logical/TestConvertCountToDirectScan.java
@@ -287,6 +287,24 @@ public class TestConvertCountToDirectScan extends
ClusterTest {
}
@Test
+ public void textConvertAbsentColumn() throws Exception {
+String sql = "select count(abc) as cnt from cp.`tpch/nation.parquet`";
+
+queryBuilder()
+.sql(sql)
+.planMatcher()
+.include("DynamicPojoRecordReader")
+.match();
+
+testBuilder()
+.sqlQuery(sql)
+.unOrdered()
+.baselineColumns("cnt")
+.baselineValues(0L)
+.go();
+ }
+
+ @Test
public
[drill] branch master updated: Updating dev environment setup doc to skip tests on install
This is an automated email from the ASF dual-hosted git repository. arina pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/drill.git The following commit(s) were added to refs/heads/master by this push: new a9caa32 Updating dev environment setup doc to skip tests on install a9caa32 is described below commit a9caa32c844eb4ed365d9cb1da9e0341949d28c4 Author: Andrew Tate AuthorDate: Wed Jan 22 15:44:48 2020 -0700 Updating dev environment setup doc to skip tests on install --- docs/dev/Environment.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/dev/Environment.md b/docs/dev/Environment.md index 4567a08..2110f97 100644 --- a/docs/dev/Environment.md +++ b/docs/dev/Environment.md @@ -22,7 +22,7 @@ Currently, the Apache Drill build process is known to work on Linux, Windows and ## Build cd drill -mvn clean install +mvn clean install -DskipTests ## Explode tarball in installation directory
[drill] 01/04: DRILL-7388: Kafka improvements
This is an automated email from the ASF dual-hosted git repository. arina pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/drill.git commit ffd3c352586a5884747edcc5a93b0c625a47e100 Author: Arina Ielchiieva AuthorDate: Fri Nov 15 16:01:52 2019 +0200 DRILL-7388: Kafka improvements 1. Upgraded Kafka libraries to 2.3.1 (DRILL-6739). 2. Added new options to support the same features as native JSON reader: a. store.kafka.reader.skip_invalid_records, default: false (DRILL-6723); b. store.kafka.reader.allow_nan_inf, default: true; c. store.kafka.reader.allow_escape_any_char, default: false. 3. Fixed issue when Kafka topic contains only one message (DRILL-7388). 4. Replaced Gson parser with Jackson to parse JSON in the same manner as Drill native Json reader. 5. Performance improvements: Kafka consumers will be closed async, fixed issue with resource leak (DRILL-7290), moved to debug unnecessary info logging. 6. Updated bootstrap-storage-plugins.json to reflect actual Kafka connection properties. 7. Added unit tests. 8. Refactoring and code clean up. closes #1901 --- contrib/storage-kafka/README.md| 12 +- contrib/storage-kafka/pom.xml | 6 +- .../drill/exec/store/kafka/KafkaAsyncCloser.java | 105 + .../drill/exec/store/kafka/KafkaGroupScan.java | 52 --- .../drill/exec/store/kafka/KafkaNodeProcessor.java | 1 - .../exec/store/kafka/KafkaPartitionScanSpec.java | 10 +- .../store/kafka/KafkaPartitionScanSpecBuilder.java | 111 +++-- .../store/kafka/KafkaPushDownFilterIntoScan.java | 25 +-- .../drill/exec/store/kafka/KafkaRecordReader.java | 85 +- .../exec/store/kafka/KafkaScanBatchCreator.java| 14 +- .../drill/exec/store/kafka/KafkaScanSpec.java | 3 +- .../drill/exec/store/kafka/KafkaStoragePlugin.java | 21 +-- .../exec/store/kafka/KafkaStoragePluginConfig.java | 35 ++--- .../drill/exec/store/kafka/KafkaSubScan.java | 2 +- .../drill/exec/store/kafka/MessageIterator.java| 50 -- .../drill/exec/store/kafka/MetaDataField.java | 10 +- .../apache/drill/exec/store/kafka/ReadOptions.java | 95 +++ .../store/kafka/decoders/JsonMessageReader.java| 95 +++ .../exec/store/kafka/decoders/MessageReader.java | 10 +- .../store/kafka/decoders/MessageReaderFactory.java | 4 +- .../store/kafka/schema/KafkaMessageSchema.java | 14 +- .../store/kafka/schema/KafkaSchemaFactory.java | 5 +- .../main/resources/bootstrap-storage-plugins.json | 10 +- .../exec/store/kafka/KafkaFilterPushdownTest.java | 22 +-- .../exec/store/kafka/KafkaMessageGenerator.java| 99 ++-- .../drill/exec/store/kafka/KafkaQueriesTest.java | 173 ++--- .../drill/exec/store/kafka/KafkaTestBase.java | 12 +- .../exec/store/kafka/MessageIteratorTest.java | 5 +- .../drill/exec/store/kafka/TestKafkaSuit.java | 81 +- .../store/kafka/cluster/EmbeddedKafkaCluster.java | 57 --- .../kafka/decoders/MessageReaderFactoryTest.java | 6 +- .../java/org/apache/drill/exec/ExecConstants.java | 10 ++ .../exec/server/options/SystemOptionManager.java | 3 + .../java-exec/src/main/resources/drill-module.conf | 3 + 34 files changed, 807 insertions(+), 439 deletions(-) diff --git a/contrib/storage-kafka/README.md b/contrib/storage-kafka/README.md index a63731f..a26c6e1 100644 --- a/contrib/storage-kafka/README.md +++ b/contrib/storage-kafka/README.md @@ -211,18 +211,22 @@ Note: - store.kafka.record.reader system option can be used for setting record reader and default is org.apache.drill.exec.store.kafka.decoders.JsonMessageReader - Default store.kafka.poll.timeout is set to 200, user has to set this accordingly -- Custom record reader can be implemented by extending org.apache.drill.exec.store.kafka.decoders.MessageReader and setting store.kafka.record.reader accordinlgy +- Custom record reader can be implemented by extending org.apache.drill.exec.store.kafka.decoders.MessageReader and setting store.kafka.record.reader accordingly -In case of JSON message format, following system options can be used accordingly. More details can be found in [Drill Json Model](https://drill.apache.org/docs/json-data-model/) and in [Drill system options configurations](https://drill.apache.org/docs/configuration-options-introduction/) +In case of JSON message format, following system / session options can be used accordingly. More details can be found in [Drill Json Model](https://drill.apache.org/docs/json-data +-model/) and in [Drill system options configurations](https://drill.apache.org/docs/configuration-options-introduction/) - + ALTER SESSION SET `store.kafka.record.reader` = 'org.apache.drill.exec.store.kafka.decoders.JsonMessageReader'; ALTER SESSION SET `store.kafka.poll.timeout` = 200; ALTER
[drill] branch master updated (8e6cdbe -> 0230627)
This is an automated email from the ASF dual-hosted git repository. arina pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/drill.git. from 8e6cdbe DRILL-7446: Fix Eclipse compilation issue in AbstractParquetGroupScan new ffd3c35 DRILL-7388: Kafka improvements new 120e10b DRILL-7448: Fix warnings when running Drill memory tests new 5b9380e DRILL-7441: Fix issues with fillEmpties, offset vectors new 0230627 DRILL-7445: Create batch copier based on result set framework The 4 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: common/pom.xml | 8 - .../java/org/apache/drill/common/types/Types.java | 32 +- common/src/test/resources/logback-test.xml | 47 +- .../apache/drill/exec/hive/TestHiveStorage.java| 12 +- contrib/storage-kafka/README.md| 12 +- contrib/storage-kafka/pom.xml | 6 +- .../drill/exec/store/kafka/KafkaAsyncCloser.java | 105 .../drill/exec/store/kafka/KafkaGroupScan.java | 52 +- .../drill/exec/store/kafka/KafkaNodeProcessor.java | 1 - .../exec/store/kafka/KafkaPartitionScanSpec.java | 10 +- .../store/kafka/KafkaPartitionScanSpecBuilder.java | 111 ++-- .../store/kafka/KafkaPushDownFilterIntoScan.java | 25 +- .../drill/exec/store/kafka/KafkaRecordReader.java | 85 ++- .../exec/store/kafka/KafkaScanBatchCreator.java| 14 +- .../drill/exec/store/kafka/KafkaScanSpec.java | 3 +- .../drill/exec/store/kafka/KafkaStoragePlugin.java | 21 +- .../exec/store/kafka/KafkaStoragePluginConfig.java | 35 +- .../drill/exec/store/kafka/KafkaSubScan.java | 2 +- .../drill/exec/store/kafka/MessageIterator.java| 50 +- .../drill/exec/store/kafka/MetaDataField.java | 10 +- .../apache/drill/exec/store/kafka/ReadOptions.java | 95 +++ .../store/kafka/decoders/JsonMessageReader.java| 95 ++- .../exec/store/kafka/decoders/MessageReader.java | 10 +- .../store/kafka/decoders/MessageReaderFactory.java | 4 +- .../store/kafka/schema/KafkaMessageSchema.java | 14 +- .../store/kafka/schema/KafkaSchemaFactory.java | 5 +- .../main/resources/bootstrap-storage-plugins.json | 10 +- .../exec/store/kafka/KafkaFilterPushdownTest.java | 22 +- .../exec/store/kafka/KafkaMessageGenerator.java| 99 ++- .../drill/exec/store/kafka/KafkaQueriesTest.java | 173 +- .../drill/exec/store/kafka/KafkaTestBase.java | 12 +- .../exec/store/kafka/MessageIteratorTest.java | 5 +- .../drill/exec/store/kafka/TestKafkaSuit.java | 81 +-- .../store/kafka/cluster/EmbeddedKafkaCluster.java | 57 +- .../kafka/decoders/MessageReaderFactoryTest.java | 6 +- .../java/org/apache/drill/exec/ExecConstants.java | 10 + .../physical/impl/aggregate/BatchIterator.java | 2 - .../impl/protocol/IndirectContainerAccessor.java | 100 .../impl/protocol/VectorContainerAccessor.java | 41 +- .../physical/impl/validate/BatchValidator.java | 174 +++--- .../exec/physical/resultSet/ResultSetCopier.java | 189 ++ .../exec/physical/resultSet/ResultSetReader.java | 7 + .../resultSet/impl/ResultSetCopierImpl.java| 313 ++ .../resultSet/impl/ResultSetLoaderImpl.java| 1 + .../resultSet/impl/ResultSetReaderImpl.java| 3 + .../exec/physical/rowSet/RowSetFormatter.java | 13 +- .../exec/record/selection/SelectionVector2.java| 13 +- .../selection/SelectionVector2Builder.java}| 35 +- .../exec/server/options/SystemOptionManager.java | 3 + .../drill/exec/vector/complex/fn/JsonReader.java | 1 + .../drill/exec/work/batch/BaseRawBatchBuffer.java | 1 - .../java-exec/src/main/resources/drill-module.conf | 3 + .../test/java/org/apache/drill/TestBugFixes.java | 31 +- .../test/java/org/apache/drill/TestCTASJson.java | 20 +- .../java/org/apache/drill/TestFrameworkTest.java | 12 +- .../java/org/apache/drill/TestJoinNullable.java| 1 - .../exec/compile/TestLargeFileCompilation.java | 22 +- .../physical/impl/TestNestedDateTimeTimestamp.java | 10 +- .../physical/impl/join/TestMergeJoinAdvanced.java | 22 +- .../impl/protocol/TestOperatorRecordBatch.java | 9 +- .../physical/impl/validate/TestBatchValidator.java | 2 +- .../writer/TestCorruptParquetDateCorrection.java | 23 +- .../resultSet/impl/TestResultSetCopier.java| 663 + .../resultSet/impl/TestResultSetReader.java| 13 +- .../exec/physical/rowSet/TestMapAccessors.java | 16 +- .../physical/rowSet/TestOffsetVectorWriter.java| 14 +- .../exec/physical/rowSet/TestScalarAccessors.java | 217 ++- .../drill/exec/planner/sql/TestDrillSQLWorker.java |
[drill] 02/04: DRILL-7448: Fix warnings when running Drill memory tests
This is an automated email from the ASF dual-hosted git repository.
arina pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git
commit 120e10bb8d79d99117464404bb6c870a10169d92
Author: Bohdan Kazydub
AuthorDate: Mon Nov 18 19:51:55 2019 +0200
DRILL-7448: Fix warnings when running Drill memory tests
closes #1902
---
common/pom.xml | 8 -
common/src/test/resources/logback-test.xml | 47 ++
2 files changed, 22 insertions(+), 33 deletions(-)
diff --git a/common/pom.xml b/common/pom.xml
index 743374d..47b60fc 100644
--- a/common/pom.xml
+++ b/common/pom.xml
@@ -102,14 +102,6 @@
2.9
-
-
- org.codehaus.janino
- janino
- test
-
-
diff --git a/common/src/test/resources/logback-test.xml
b/common/src/test/resources/logback-test.xml
index 06004a6..f47af26 100644
--- a/common/src/test/resources/logback-test.xml
+++ b/common/src/test/resources/logback-test.xml
@@ -20,26 +20,26 @@
-->
-
-
-
-true
-1
-true
-${LILITH_HOSTNAME:-localhost}
-
-
-
-
-
-
-
-
-
-
-
-
-
+
+
+
[drill] 04/04: DRILL-7445: Create batch copier based on result set framework
This is an automated email from the ASF dual-hosted git repository.
arina pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git
commit 02306277bfacaece652ba76025a6f82def9570b6
Author: Paul Rogers
AuthorDate: Sun Nov 10 16:17:49 2019 -0800
DRILL-7445: Create batch copier based on result set framework
The result set framework now provides both a reader and writer.
This PR provides a copier that copies batches using this
framework. Such a copier can:
- Copy selected records
- Copy all records, such as for an SV2 or SV4
The copier uses the result set loader to create uniformly-sized
output batches from input batches of any size. It does this
by merging or splitting input batches as needed.
Since the result set reader handles both SV2 and SV4s, the
copier can filter or reorder rows based on the SV associated
with the input batch.
This version assumes single stream of input batches, and handles
any schema changes in that input by creating output batches
that track the input schema. This would be used in, say, the
selection vector remover. A different design is needed for merging
such as in the merging receiver.
Adds a "copy" method to the column writers. Copy is implemented
by doing a direct memory copy from source to destination vectors.
A unit test verifies functionality for various use cases
and data types.
closes #1899
---
.../java/org/apache/drill/common/types/Types.java | 32 +-
.../physical/impl/aggregate/BatchIterator.java | 2 -
.../impl/protocol/IndirectContainerAccessor.java | 100
.../impl/protocol/VectorContainerAccessor.java | 41 +-
.../exec/physical/resultSet/ResultSetCopier.java | 189 ++
.../exec/physical/resultSet/ResultSetReader.java | 7 +
.../resultSet/impl/ResultSetCopierImpl.java| 313 ++
.../resultSet/impl/ResultSetLoaderImpl.java| 1 +
.../resultSet/impl/ResultSetReaderImpl.java| 3 +
.../exec/physical/rowSet/RowSetFormatter.java | 13 +-
.../exec/record/selection/SelectionVector2.java| 13 +-
.../selection/SelectionVector2Builder.java}| 31 +-
.../impl/protocol/TestOperatorRecordBatch.java | 9 +-
.../resultSet/impl/TestResultSetCopier.java| 663 +
.../resultSet/impl/TestResultSetReader.java| 13 +-
.../org/apache/drill/test/BaseDirTestWatcher.java | 12 +-
.../java/org/apache/drill/test/ClusterFixture.java | 59 +-
.../main/codegen/templates/ColumnAccessors.java| 21 +-
.../drill/exec/record/MaterializedField.java | 1 +
.../drill/exec/vector/accessor/ColumnWriter.java | 9 +
.../drill/exec/vector/accessor/ObjectReader.java | 7 +
.../drill/exec/vector/accessor/ObjectWriter.java | 11 +-
.../accessor/convert/AbstractWriteConverter.java | 6 +
.../accessor/reader/AbstractObjectReader.java | 1 +
.../accessor/reader/AbstractTupleReader.java | 16 +-
.../vector/accessor/reader/ArrayReaderImpl.java| 1 -
.../vector/accessor/reader/BaseScalarReader.java | 12 +
.../vector/accessor/reader/OffsetVectorReader.java | 1 +
.../vector/accessor/reader/UnionReaderImpl.java| 12 +-
.../accessor/writer/AbstractArrayWriter.java | 14 +
.../accessor/writer/AbstractObjectWriter.java | 11 +-
.../accessor/writer/AbstractTupleWriter.java | 22 +-
.../vector/accessor/writer/BaseVarWidthWriter.java | 8 +-
.../vector/accessor/writer/BitColumnWriter.java| 8 +
.../exec/vector/accessor/writer/MapWriter.java | 9 +-
.../accessor/writer/NullableScalarWriter.java | 9 +
.../accessor/writer/OffsetVectorWriterImpl.java| 6 +
.../vector/accessor/writer/UnionWriterImpl.java| 13 +-
.../accessor/writer/dummy/DummyArrayWriter.java| 4 +
.../accessor/writer/dummy/DummyScalarWriter.java | 4 +
40 files changed, 1570 insertions(+), 137 deletions(-)
diff --git a/common/src/main/java/org/apache/drill/common/types/Types.java
b/common/src/main/java/org/apache/drill/common/types/Types.java
index 57a752e..5cdae56 100644
--- a/common/src/main/java/org/apache/drill/common/types/Types.java
+++ b/common/src/main/java/org/apache/drill/common/types/Types.java
@@ -806,16 +806,32 @@ public class Types {
return typeBuilder;
}
- public static boolean isEquivalent(MajorType type1, MajorType type2) {
+ /**
+ * Check if two "core" types are the same, ignoring subtypes and
+ * children. Primarily for non-complex types.
+ *
+ * @param type1 first type
+ * @param type2 second type
+ * @return true if the two types are are the same minor type, mode,
+ * precision and scale
+ */
+
+ public static boolean isSameType(MajorType type1, MajorType type2) {
+return type1.getMinorType() == type2.getMinorType() &&
+ type1.getMode() == type2.getMode() &&
+
[drill] branch master updated: DRILL-7439: Batch count fixes for six additional operators
This is an automated email from the ASF dual-hosted git repository.
arina pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git
The following commit(s) were added to refs/heads/master by this push:
new 9d8ac02 DRILL-7439: Batch count fixes for six additional operators
9d8ac02 is described below
commit 9d8ac02d05cf6f23ddc80065066722b121577656
Author: Paul Rogers
AuthorDate: Tue Nov 5 13:50:56 2019 -0800
DRILL-7439: Batch count fixes for six additional operators
Enables vector checks, and fixes batch count and vector issues for:
* StreamingAggBatch
* RuntimeFilterRecordBatch
* FlattenRecordBatch
* MergeJoinBatch
* NestedLoopJoinBatch
* LimitRecordBatch
Also fixes a zero-size batch validity issue for the CSV reader when
all files contain no data.
Includes code cleanup for files touched in this PR.
closes #1893
---
.../org/apache/drill/exec/util/AssertionUtil.java | 7 +-
.../physical/impl/aggregate/HashAggTemplate.java | 109 +--
.../physical/impl/aggregate/StreamingAggBatch.java | 124 +++--
.../impl/aggregate/StreamingAggTemplate.java | 43 ++---
.../exec/physical/impl/common/HashPartition.java | 73 +---
.../impl/filter/RuntimeFilterRecordBatch.java | 51 +++---
.../physical/impl/flatten/FlattenRecordBatch.java | 201 ++---
.../exec/physical/impl/join/MergeJoinBatch.java| 15 +-
.../physical/impl/join/NestedLoopJoinBatch.java| 20 +-
.../exec/physical/impl/limit/LimitRecordBatch.java | 23 ++-
.../impl/protocol/OperatorRecordBatch.java | 4 +-
.../drill/exec/physical/impl/scan/ReaderState.java | 4 +-
.../exec/physical/impl/scan/ScanOperatorExec.java | 11 +-
.../impl/scan/framework/ManagedScanFramework.java | 4 +-
.../physical/impl/scan/project/ResolvedTuple.java | 16 +-
.../physical/impl/validate/BatchValidator.java | 71 +---
.../resultSet/impl/ResultSetLoaderImpl.java| 4 +-
.../drill/exec/nested/TestFastComplexSchema.java | 4 +-
.../exec/physical/impl/flatten/TestFlatten.java| 11 +-
.../main/codegen/templates/FixedValueVectors.java | 64 +++
.../accessor/writer/OffsetVectorWriterImpl.java| 31 ++--
.../vector/complex/BaseRepeatedValueVector.java| 13 +-
22 files changed, 470 insertions(+), 433 deletions(-)
diff --git a/common/src/main/java/org/apache/drill/exec/util/AssertionUtil.java
b/common/src/main/java/org/apache/drill/exec/util/AssertionUtil.java
index f4fc9ab..446825a 100644
--- a/common/src/main/java/org/apache/drill/exec/util/AssertionUtil.java
+++ b/common/src/main/java/org/apache/drill/exec/util/AssertionUtil.java
@@ -18,11 +18,9 @@
package org.apache.drill.exec.util;
public class AssertionUtil {
- static final org.slf4j.Logger logger =
org.slf4j.LoggerFactory.getLogger(AssertionUtil.class);
-
public static final boolean ASSERT_ENABLED;
- static{
+ static {
boolean isAssertEnabled = false;
assert isAssertEnabled = true;
ASSERT_ENABLED = isAssertEnabled;
@@ -32,6 +30,5 @@ public class AssertionUtil {
return ASSERT_ENABLED;
}
- private AssertionUtil() {
- }
+ private AssertionUtil() { }
}
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/HashAggTemplate.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/HashAggTemplate.java
index 6cec50e..d166353 100644
---
a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/HashAggTemplate.java
+++
b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/HashAggTemplate.java
@@ -17,6 +17,9 @@
*/
package org.apache.drill.exec.physical.impl.aggregate;
+import static org.apache.drill.exec.physical.impl.common.HashTable.BATCH_MASK;
+import static org.apache.drill.exec.record.RecordBatch.MAX_BATCH_ROW_COUNT;
+
import java.io.IOException;
import java.util.ArrayList;
import java.util.Iterator;
@@ -31,7 +34,6 @@ import org.apache.drill.common.exceptions.UserException;
import org.apache.drill.common.expression.ExpressionPosition;
import org.apache.drill.common.expression.FieldReference;
import org.apache.drill.common.expression.LogicalExpression;
-
import org.apache.drill.exec.ExecConstants;
import org.apache.drill.exec.cache.VectorSerializer.Writer;
import org.apache.drill.exec.compile.sig.RuntimeOverridden;
@@ -40,7 +42,6 @@ import org.apache.drill.exec.exception.OutOfMemoryException;
import org.apache.drill.exec.exception.SchemaChangeException;
import org.apache.drill.exec.expr.ClassGenerator;
import org.apache.drill.exec.expr.TypeHelper;
-
import org.apache.drill.exec.memory.BaseAllocator;
import org.apache.drill.exec.memory.BufferAllocator;
import org.apache.drill.exec.ops.FragmentContext;
@@ -55,40 +56,30 @@ import org.apache.drill.exec.physical.impl.common.HashTable;
import
[drill] branch master updated (9195838 -> 916b399)
This is an automated email from the ASF dual-hosted git repository. arina pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/drill.git. from 9195838 DRILL-7391: Wrong result when doing left outer join on CSV table add 916b399 DRILL-7436: Fix record count, vector structure issues in several operators No new revisions were added by this update. Summary of changes: .../drill/exec/store/mongo/MongoTestConstants.java | 7 +- .../drill/exec/store/mongo/MongoTestSuit.java | 9 +- .../drill/exec/store/mongo/TestMongoQueries.java | 3 + .../drill/exec/store/mongo/TestTableGenerator.java | 17 +- .../exec/expr/ExpressionTreeMaterializer.java | 117 ++- .../exec/expr/annotations/FunctionTemplate.java| 34 ++-- .../apache/drill/exec/expr/fn/DrillFuncHolder.java | 33 ++-- .../drill/exec/expr/fn/DrillSimpleFuncHolder.java | 11 +- .../drill/exec/expr/fn/impl/UnionFunctions.java| 28 +-- .../expr/fn/output/DefaultReturnTypeInference.java | 22 ++- .../exec/physical/impl/aggregate/HashAggBatch.java | 22 ++- .../physical/impl/filter/FilterRecordBatch.java| 37 ++-- .../exec/physical/impl/filter/FilterTemplate2.java | 39 ++-- .../impl/limit/PartitionLimitRecordBatch.java | 101 ++ .../physical/impl/project/ProjectRecordBatch.java | 26 ++- .../physical/impl/project/ProjectorTemplate.java | 32 +-- .../physical/impl/svremover/AbstractCopier.java| 14 +- .../impl/svremover/RemovingRecordBatch.java| 12 +- .../physical/impl/union/UnionAllRecordBatch.java | 47 +++-- .../physical/impl/union/UnionAllerTemplate.java| 13 +- .../physical/impl/unnest/UnnestRecordBatch.java| 115 +-- .../physical/impl/validate/BatchValidator.java | 215 +++-- .../impl/xsort/managed/ExternalSortBatch.java | 48 +++-- .../drill/exec/record/AbstractRecordBatch.java | 29 ++- .../record/AbstractTableFunctionRecordBatch.java | 2 - .../exec/record/AbstractUnaryRecordBatch.java | 4 +- .../apache/drill/exec/record/VectorContainer.java | 16 ++ .../exec/store/easy/json/JSONRecordReader.java | 12 +- .../drill/exec/vector/complex/fn/JsonReader.java | 84 .../apache/drill/TestTpchDistributedStreaming.java | 1 - .../physical/impl/validate/TestBatchValidator.java | 2 +- .../exec/vector/complex/writer/TestJsonReader.java | 15 +- .../codegen/templates/NullableValueVectors.java| 55 +++--- .../src/main/codegen/templates/UnionVector.java| 82 +--- .../drill/exec/record/MaterializedField.java | 95 + .../drill/exec/vector/complex/MapVector.java | 35 ++-- .../drill/common/expression/IfExpression.java | 9 +- 37 files changed, 847 insertions(+), 596 deletions(-)
[drill] branch master updated: DRILL-7418: MetadataDirectGroupScan improvements
This is an automated email from the ASF dual-hosted git repository.
arina pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git
The following commit(s) were added to refs/heads/master by this push:
new 0928427 DRILL-7418: MetadataDirectGroupScan improvements
0928427 is described below
commit 09284270b85d35f7a1d521539c42876bb613a6e3
Author: Arina Ielchiieva
AuthorDate: Tue Oct 22 17:33:04 2019 +0300
DRILL-7418: MetadataDirectGroupScan improvements
1. Replaced files listing with selection root information to reduce query
plan size in MetadataDirectGroupScan.
2. Fixed MetadataDirectGroupScan ser / de issues.
3. Added PlanMatcher to QueryBuilder for more convenient plan matching.
4. Re-written TestConvertCountToDirectScan to use ClusterTest.
5. Refactoring and code clean up.
---
.../exec/TestHiveDrillNativeParquetReader.java | 19 +-
.../exec/physical/base/AbstractGroupScan.java | 5 +
.../apache/drill/exec/physical/base/GroupScan.java | 9 +-
.../apache/drill/exec/physical/base/ScanStats.java | 46 ++-
.../logical/ConvertCountToDirectScanRule.java | 27 +-
.../physical/ConvertCountToDirectScanPrule.java| 17 +-
.../drill/exec/store/direct/DirectGroupScan.java | 24 +-
.../exec/store/direct/MetadataDirectGroupScan.java | 49 ++-
.../parquet/AbstractParquetScanBatchCreator.java | 6 +-
.../drill/TestFunctionsWithTypeExpoQueries.java| 2 +-
.../logical/TestConvertCountToDirectScan.java | 444 -
.../java/org/apache/drill/test/ClientFixture.java | 76 ++--
.../java/org/apache/drill/test/QueryBuilder.java | 152 +--
13 files changed, 530 insertions(+), 346 deletions(-)
diff --git
a/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/TestHiveDrillNativeParquetReader.java
b/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/TestHiveDrillNativeParquetReader.java
index 5490640..6b9a7cd 100644
---
a/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/TestHiveDrillNativeParquetReader.java
+++
b/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/TestHiveDrillNativeParquetReader.java
@@ -64,8 +64,7 @@ public class TestHiveDrillNativeParquetReader extends
HiveTestBase {
int actualRowCount = testSql(query);
assertEquals("Expected and actual row count should match", 2,
actualRowCount);
-testPlanMatchingPatterns(query,
-new String[]{"HiveDrillNativeParquetScan", "numFiles=1"}, null);
+testPlanMatchingPatterns(query, "HiveDrillNativeParquetScan",
"numFiles=1");
}
@Test
@@ -75,8 +74,7 @@ public class TestHiveDrillNativeParquetReader extends
HiveTestBase {
int actualRowCount = testSql(query);
assertEquals("Expected and actual row count should match", 1,
actualRowCount);
-testPlanMatchingPatterns(query,
-new String[]{"HiveDrillNativeParquetScan", "numFiles=1"}, null);
+testPlanMatchingPatterns(query, "HiveDrillNativeParquetScan",
"numFiles=1");
}
@Test
@@ -114,15 +112,14 @@ public class TestHiveDrillNativeParquetReader extends
HiveTestBase {
int actualRowCount = testSql(query);
assertEquals("Expected and actual row count should match", 2,
actualRowCount);
-testPlanMatchingPatterns(query,
-new String[]{"HiveDrillNativeParquetScan", "numFiles=1"}, null);
+testPlanMatchingPatterns(query, "HiveDrillNativeParquetScan",
"numFiles=1");
}
@Test
public void testPartitionedExternalTable() throws Exception {
String query = "select * from hive.kv_native_ext";
-testPlanMatchingPatterns(query, new String[]{"HiveDrillNativeParquetScan",
"numFiles=2"}, null);
+testPlanMatchingPatterns(query, "HiveDrillNativeParquetScan",
"numFiles=2");
testBuilder()
.sqlQuery(query)
@@ -185,14 +182,16 @@ public class TestHiveDrillNativeParquetReader extends
HiveTestBase {
int actualRowCount = testSql(query);
assertEquals("Expected and actual row count should match", 2,
actualRowCount);
-testPlanMatchingPatterns(query, new String[]{"HiveDrillNativeParquetScan",
"numFiles=1"}, null);
+testPlanMatchingPatterns(query, "HiveDrillNativeParquetScan",
"numFiles=1");
}
@Test
public void testConvertCountToDirectScanOptimization() throws Exception {
String query = "select count(1) as cnt from hive.kv_native";
-testPlanMatchingPatterns(query, new String[]{"DynamicPojoRecordReader"},
null);
+testPlanMatchingPatterns(query, "DynamicPojoRecordReader");
+
+testPhysicalPlanExecutionBasedOnQuery(query);
testBuilder()
.sqlQuery(query)
@@ -224,7 +223,7 @@ public class TestHiveD
[drill] branch master updated: DRILL-7347: Upgrade Apache Iceberg to released version
This is an automated email from the ASF dual-hosted git repository.
arina pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git
The following commit(s) were added to refs/heads/master by this push:
new 5d6dbc8 DRILL-7347: Upgrade Apache Iceberg to released version
5d6dbc8 is described below
commit 5d6dbc8b3c7ec3fd5f3eefd1c9bf56e688e76ef0
Author: Arina Ielchiieva
AuthorDate: Mon Oct 28 13:51:21 2019 +0200
DRILL-7347: Upgrade Apache Iceberg to released version
---
metastore/iceberg-metastore/pom.xml | 12 ++--
1 file changed, 6 insertions(+), 6 deletions(-)
diff --git a/metastore/iceberg-metastore/pom.xml
b/metastore/iceberg-metastore/pom.xml
index 5a2746b..04fc137 100644
--- a/metastore/iceberg-metastore/pom.xml
+++ b/metastore/iceberg-metastore/pom.xml
@@ -33,7 +33,7 @@
metastore/Drill Iceberg Metastore
-4d8ae52
+0.7.0-incubating
2.7.0
@@ -49,27 +49,27 @@
${project.version}
- com.github.apache.incubator-iceberg
+ org.apache.iceberg
iceberg-parquet
${iceberg.version}
- com.github.apache.incubator-iceberg
+ org.apache.iceberg
iceberg-data
${iceberg.version}
- com.github.apache.incubator-iceberg
+ org.apache.iceberg
iceberg-core
${iceberg.version}
- com.github.apache.incubator-iceberg
+ org.apache.iceberg
iceberg-common
${iceberg.version}
- com.github.apache.incubator-iceberg
+ org.apache.iceberg
iceberg-api
${iceberg.version}
[drill] branch master updated (9388e1c -> f2654ee)
This is an automated email from the ASF dual-hosted git repository.
arina pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git.
from 9388e1c DRILL-7417: Add user logged in/out event in info level logs
add a18e180 DRILL-1709: Add desc alias for describe command
add f2654ee DRILL-7177: Format Plugin for Excel Files
No new revisions were added by this update.
Summary of changes:
contrib/format-excel/README.md | 59 +++
contrib/{format-syslog => format-excel}/pom.xml| 42 +-
.../drill/exec/store/excel/ExcelBatchReader.java | 527 +
.../drill/exec/store/excel/ExcelFormatConfig.java | 108 +
.../drill/exec/store/excel/ExcelFormatPlugin.java | 142 ++
.../main/resources/bootstrap-format-plugins.json | 12 +-
.../src/main/resources/drill-module.conf | 1 +
.../drill/exec/store/excel/TestExcelFormat.java| 402
.../src/test/resources/excel/test_data.xlsx| Bin 0 -> 15459 bytes
.../native/client/src/protobuf/UserBitShared.pb.cc | 15 +-
.../native/client/src/protobuf/UserBitShared.pb.h | 5 +-
contrib/pom.xml| 1 +
distribution/pom.xml | 5 +
distribution/src/assemble/component.xml| 1 +
.../src/main/codegen/includes/parserImpls.ftl | 6 +-
.../java/org/apache/drill/TestSchemaCommands.java | 6 +
.../org/apache/drill/exec/sql/TestInfoSchema.java | 7 +
.../store/parquet/TestParquetFilterPushDown.java | 10 +-
pom.xml| 2 +
.../org/apache/drill/exec/proto/UserBitShared.java | 21 +-
protocol/src/main/protobuf/UserBitShared.proto | 1 +
21 files changed, 1334 insertions(+), 39 deletions(-)
create mode 100644 contrib/format-excel/README.md
copy contrib/{format-syslog => format-excel}/pom.xml (70%)
create mode 100644
contrib/format-excel/src/main/java/org/apache/drill/exec/store/excel/ExcelBatchReader.java
create mode 100644
contrib/format-excel/src/main/java/org/apache/drill/exec/store/excel/ExcelFormatConfig.java
create mode 100644
contrib/format-excel/src/main/java/org/apache/drill/exec/store/excel/ExcelFormatPlugin.java
copy contrib/{format-ltsv =>
format-excel}/src/main/resources/bootstrap-format-plugins.json (66%)
create mode 100644 contrib/format-excel/src/main/resources/drill-module.conf
create mode 100644
contrib/format-excel/src/test/java/org/apache/drill/exec/store/excel/TestExcelFormat.java
create mode 100644 contrib/format-excel/src/test/resources/excel/test_data.xlsx
[drill] branch master updated: DRILL-7413: Test and fix scan operator vectors
This is an automated email from the ASF dual-hosted git repository.
arina pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git
The following commit(s) were added to refs/heads/master by this push:
new 8c37156 DRILL-7413: Test and fix scan operator vectors
8c37156 is described below
commit 8c37156a91ee531e4d1b672a41255cdeb11bdd57
Author: Paul Rogers
AuthorDate: Sun Oct 20 12:03:28 2019 -0700
DRILL-7413: Test and fix scan operator vectors
Enables vector validation tests for the ScanBatch and all
EasyFormat plugins. Fixes a bug in scan batch that failed to set
the record count in the output container.
Fixes a number of formatting and other issues found while adding
the tests.
---
.../apache/drill/exec/physical/impl/ScanBatch.java | 28 +-
.../physical/impl/validate/BatchValidator.java | 62 +++-
.../apache/drill/exec/record/VectorContainer.java | 5 +
.../exec/store/easy/json/JSONFormatPlugin.java | 24 +-
.../sequencefile/SequenceFileFormatPlugin.java | 9 +-
.../drill/exec/record/vector/TestDateTypes.java| 376 ++---
.../test/resources/parquet/alltypes_repeated.json | 4 +-
7 files changed, 278 insertions(+), 230 deletions(-)
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/ScanBatch.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/ScanBatch.java
index cdb36f4..79b40ee 100644
---
a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/ScanBatch.java
+++
b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/ScanBatch.java
@@ -17,9 +17,11 @@
*/
package org.apache.drill.exec.physical.impl;
-import
org.apache.drill.shaded.guava.com.google.common.annotations.VisibleForTesting;
-import org.apache.drill.shaded.guava.com.google.common.base.Preconditions;
-import io.netty.buffer.DrillBuf;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
import org.apache.drill.common.exceptions.ExecutionSetupException;
import org.apache.drill.common.exceptions.UserException;
import org.apache.drill.common.expression.SchemaPath;
@@ -54,10 +56,10 @@ import org.apache.drill.exec.vector.AllocationHelper;
import org.apache.drill.exec.vector.NullableVarCharVector;
import org.apache.drill.exec.vector.SchemaChangeCallBack;
import org.apache.drill.exec.vector.ValueVector;
-import java.util.Collections;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
+import
org.apache.drill.shaded.guava.com.google.common.annotations.VisibleForTesting;
+import org.apache.drill.shaded.guava.com.google.common.base.Preconditions;
+
+import io.netty.buffer.DrillBuf;
/**
* Record batch used for a particular scan. Operators against one or more
@@ -155,6 +157,8 @@ public class ScanBatch implements CloseableRecordBatch {
return context;
}
+ public OperatorContext getOperatorContext() { return oContext; }
+
@Override
public BatchSchema getSchema() {
return schema;
@@ -224,7 +228,8 @@ public class ScanBatch implements CloseableRecordBatch {
logger.trace("currentReader.next return recordCount={}", recordCount);
Preconditions.checkArgument(recordCount >= 0, "recordCount from
RecordReader.next() should not be negative");
boolean isNewSchema = mutator.isNewSchema();
- populateImplicitVectorsAndSetCount();
+ populateImplicitVectors();
+ mutator.container.setValueCount(recordCount);
oContext.getStats().batchReceived(0, recordCount, isNewSchema);
boolean toContinueIter = true;
@@ -339,12 +344,8 @@ public class ScanBatch implements CloseableRecordBatch {
}
}
- private void populateImplicitVectorsAndSetCount() {
+ private void populateImplicitVectors() {
mutator.populateImplicitVectors(implicitValues, recordCount);
-for (Map.Entry entry:
mutator.fieldVectorMap().entrySet()) {
- logger.debug("set record count {} for vv {}", recordCount,
entry.getKey());
- entry.getValue().getMutator().setValueCount(recordCount);
-}
}
@Override
@@ -602,6 +603,7 @@ public class ScanBatch implements CloseableRecordBatch {
public VectorContainer getContainer() {
return container;
}
+
/**
* Verify list of implicit column values is valid input:
* - Either implicit column list is empty;
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/validate/BatchValidator.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/validate/BatchValidator.java
index 2753f55..7b9ff80 100644
---
a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/validate/BatchValidator.java
+++
b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/validate/BatchValidator.java
@@ -20,12 +20,16 @@ package org.apache.drill.exec.physical.impl.validate;
import java.util.Ident
[drill] branch master updated: DRILL-5674: Support ZIP compression
This is an automated email from the ASF dual-hosted git repository.
arina pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git
The following commit(s) were added to refs/heads/master by this push:
new 7d33bf1 DRILL-5674: Support ZIP compression
7d33bf1 is described below
commit 7d33bf102e50089536b2c7c95d08ffb58aebf78e
Author: Arina Ielchiieva
AuthorDate: Fri Oct 18 18:22:15 2019 +0300
DRILL-5674: Support ZIP compression
1. Added ZipCodec implementation which can read / write single file.
2. Revisited Drill plugin formats to ensure 'openPossiblyCompressedStream'
method is used in those which support compression.
3. Added unit tests.
4. General refactoring.
---
.../drill/exec/store/ltsv/LTSVFormatPlugin.java| 15 +--
.../drill/exec/store/ltsv/LTSVRecordReader.java| 32 ++---
.../exec/store/syslog/SyslogFormatPlugin.java | 20 ++-
.../exec/store/syslog/SyslogRecordReader.java | 59 +++--
.../planner/logical/partition/PruneScanRule.java | 2 +-
.../drill/exec/store/dfs/BasicFormatMatcher.java | 65 +-
.../apache/drill/exec/store/dfs/FileSelection.java | 71 +--
.../drill/exec/store/dfs/FileSystemPlugin.java | 43 ++-
.../drill/exec/store/dfs/FormatSelection.java | 11 +-
.../org/apache/drill/exec/store/dfs/ZipCodec.java | 141 +
.../drill/exec/store/log/LogBatchReader.java | 2 +-
.../exec/store/parquet/ParquetFormatPlugin.java| 12 +-
.../drill/exec/store/pcap/PcapBatchReader.java | 27 +---
.../drill/exec/store/pcap/PcapFormatPlugin.java| 11 +-
.../exec/store/pcapng/PcapngFormatPlugin.java | 2 +-
.../exec/store/pcapng/PcapngRecordReader.java | 12 +-
.../drill/exec/store/pcapng/package-info.java | 2 +-
.../drill/exec/store/dfs/TestCompressedFiles.java | 111
18 files changed, 431 insertions(+), 207 deletions(-)
diff --git
a/contrib/format-ltsv/src/main/java/org/apache/drill/exec/store/ltsv/LTSVFormatPlugin.java
b/contrib/format-ltsv/src/main/java/org/apache/drill/exec/store/ltsv/LTSVFormatPlugin.java
index 8ff62ed..7284409 100644
---
a/contrib/format-ltsv/src/main/java/org/apache/drill/exec/store/ltsv/LTSVFormatPlugin.java
+++
b/contrib/format-ltsv/src/main/java/org/apache/drill/exec/store/ltsv/LTSVFormatPlugin.java
@@ -17,7 +17,6 @@
*/
package org.apache.drill.exec.store.ltsv;
-import org.apache.drill.common.exceptions.ExecutionSetupException;
import org.apache.drill.common.expression.SchemaPath;
import org.apache.drill.common.logical.StoragePluginConfig;
import org.apache.drill.exec.ops.FragmentContext;
@@ -34,17 +33,14 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import java.io.IOException;
import java.util.List;
public class LTSVFormatPlugin extends EasyFormatPlugin
{
- private static final boolean IS_COMPRESSIBLE = false;
+ private static final boolean IS_COMPRESSIBLE = true;
private static final String DEFAULT_NAME = "ltsv";
- private static final org.slf4j.Logger logger =
org.slf4j.LoggerFactory.getLogger(LTSVFormatPlugin.class);
-
public LTSVFormatPlugin(String name, DrillbitContext context, Configuration
fsConf, StoragePluginConfig storageConfig) {
this(name, context, fsConf, storageConfig, new LTSVFormatPluginConfig());
}
@@ -54,7 +50,7 @@ public class LTSVFormatPlugin extends
EasyFormatPlugin {
}
@Override
- public RecordReader getRecordReader(FragmentContext context, DrillFileSystem
dfs, FileWork fileWork, List columns, String userName) throws
ExecutionSetupException {
+ public RecordReader getRecordReader(FragmentContext context, DrillFileSystem
dfs, FileWork fileWork, List columns, String userName) {
return new LTSVRecordReader(context, fileWork.getPath(), dfs, columns);
}
@@ -75,7 +71,7 @@ public class LTSVFormatPlugin extends
EasyFormatPlugin {
}
@Override
- public RecordWriter getRecordWriter(FragmentContext context, EasyWriter
writer) throws IOException {
+ public RecordWriter getRecordWriter(FragmentContext context, EasyWriter
writer) {
throw new UnsupportedOperationException("Drill doesn't currently support
writing to LTSV files.");
}
@@ -85,13 +81,12 @@ public class LTSVFormatPlugin extends
EasyFormatPlugin {
}
@Override
- public TableStatistics readStatistics(FileSystem fs, Path statsTablePath)
throws IOException {
+ public TableStatistics readStatistics(FileSystem fs, Path statsTablePath) {
throw new UnsupportedOperationException("unimplemented");
}
@Override
- public void writeStatistics(TableStatistics statistics, FileSystem fs, Path
statsTablePath) throws IOException {
+ public void writeStatistics(TableStatistics statistics, FileSystem fs, Path
statsTablePath) {
throw new UnsupportedOperationException("unimplemented
[drill] 02/03: DRILL-7403: Validate batch checks, vector integretity in unit tests
This is an automated email from the ASF dual-hosted git repository.
arina pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git
commit ed3ad76cb7b4165911acdf5866b0b4e57cd2d680
Author: Paul Rogers
AuthorDate: Sun Oct 13 12:41:03 2019 -0700
DRILL-7403: Validate batch checks, vector integretity in unit tests
Enhances the existing record batch checks to check all the various
batch record counts, and to more fully validate all vector types.
This code revealed that virtually all record batches have
problems: they omit setting some record count or other, they
introduce some form of vector corruption.
Since we want things to work as we make fixes, this change enables
the checks for only one record batch: the "new" scan. Others are
to come as they are fixed.
closes #1871
---
.../physical/impl/validate/BatchValidator.java | 433 -
.../validate/IteratorValidatorBatchIterator.java | 38 +-
.../apache/drill/exec/vector/VectorValidator.java | 6 +-
.../physical/impl/validate/TestBatchValidator.java | 118 +++---
4 files changed, 416 insertions(+), 179 deletions(-)
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/validate/BatchValidator.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/validate/BatchValidator.java
index 83923c9..dde6583 100644
---
a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/validate/BatchValidator.java
+++
b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/validate/BatchValidator.java
@@ -17,23 +17,26 @@
*/
package org.apache.drill.exec.physical.impl.validate;
-import java.util.ArrayList;
-import java.util.List;
+import java.util.IdentityHashMap;
+import java.util.Map;
+import org.apache.drill.exec.record.RecordBatch;
import org.apache.drill.exec.record.SimpleVectorWrapper;
import org.apache.drill.exec.record.VectorAccessible;
+import org.apache.drill.exec.record.VectorContainer;
import org.apache.drill.exec.record.VectorWrapper;
-import org.apache.drill.exec.vector.BaseDataValueVector;
+import org.apache.drill.exec.vector.BitVector;
import org.apache.drill.exec.vector.FixedWidthVector;
-import org.apache.drill.exec.vector.NullableVarCharVector;
import org.apache.drill.exec.vector.NullableVector;
-import org.apache.drill.exec.vector.RepeatedVarCharVector;
+import org.apache.drill.exec.vector.RepeatedBitVector;
+import org.apache.drill.exec.vector.UInt1Vector;
import org.apache.drill.exec.vector.UInt4Vector;
import org.apache.drill.exec.vector.ValueVector;
import org.apache.drill.exec.vector.VarCharVector;
import org.apache.drill.exec.vector.VariableWidthVector;
import org.apache.drill.exec.vector.complex.BaseRepeatedValueVector;
-import org.apache.drill.exec.vector.complex.RepeatedFixedWidthVectorLike;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
@@ -45,103 +48,361 @@ import
org.apache.drill.exec.vector.complex.RepeatedFixedWidthVectorLike;
*/
public class BatchValidator {
- private static final org.slf4j.Logger logger =
- org.slf4j.LoggerFactory.getLogger(BatchValidator.class);
+ private static final Logger logger =
LoggerFactory.getLogger(BatchValidator.class);
+ public static final boolean LOG_TO_STDOUT = true;
public static final int MAX_ERRORS = 100;
- private final int rowCount;
- private final VectorAccessible batch;
- private final List errorList;
- private int errorCount;
+ public interface ErrorReporter {
+void error(String name, ValueVector vector, String msg);
+void warn(String name, ValueVector vector, String msg);
+void error(String msg);
+int errorCount();
+ }
+
+ public abstract static class BaseErrorReporter implements ErrorReporter {
+
+private final String opName;
+private int errorCount;
+
+public BaseErrorReporter(String opName) {
+ this.opName = opName;
+}
+
+protected boolean startError() {
+ if (errorCount == 0) {
+warn("Found one or more vector errors from " + opName);
+ }
+ errorCount++;
+ if (errorCount >= MAX_ERRORS) {
+return false;
+ }
+ return true;
+}
+
+@Override
+public void error(String name, ValueVector vector, String msg) {
+ error(String.format("%s - %s: %s",
+name, vector.getClass().getSimpleName(), msg));
+}
+
+@Override
+public void warn(String name, ValueVector vector, String msg) {
+ warn(String.format("%s - %s: %s",
+name, vector.getClass().getSimpleName(), msg));
+}
- public BatchValidator(VectorAccessible batch) {
-rowCount = batch.getRecordCount();
-this.batch = batch;
-errorList = null;
+public abstract void warn(String msg);
+
+@Override
+public int errorCount() { return errorCount; }
+ }
+
+ private static class StdOutReporter extends BaseE
[drill] 01/03: DRILL-6096: Provide mechanism to configure text writer configuration
This is an automated email from the ASF dual-hosted git repository.
arina pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git
commit a13608b8d1f8110d7bd944c10259b2252dfbf18b
Author: Arina Ielchiieva
AuthorDate: Thu Oct 10 15:43:33 2019 +0300
DRILL-6096: Provide mechanism to configure text writer configuration
1. Usage of format plugin configuration allows to specify line and field
delimiters, quotes and escape characters.
2. Usage of system / session options allows to specify if writer should add
headers, force quotes.
closes #1873
---
exec/java-exec/pom.xml | 2 +-
.../templates/StringOutputRecordWriter.java| 12 +-
.../java/org/apache/drill/exec/ExecConstants.java | 7 +
.../impl/scan/columns/ColumnsArrayParser.java | 13 +-
.../exec/server/options/SystemOptionManager.java | 2 +
.../exec/store/easy/text/TextFormatPlugin.java | 53 +++--
.../store/easy/text/reader/BaseFieldOutput.java| 2 +-
.../store/easy/text/reader/FieldVarCharOutput.java | 13 +-
.../exec/store/easy/text/reader/HeaderBuilder.java | 26 +-
.../easy/text/reader/RepeatedVarCharOutput.java| 21 +-
.../exec/store/easy/text/reader/TextInput.java | 13 +-
.../exec/store/easy/text/reader/TextOutput.java| 54 ++---
.../store/easy/text/reader/TextParsingContext.java | 119 ++
.../easy/text/reader/TextParsingSettings.java | 2 +-
.../exec/store/easy/text/reader/TextReader.java| 31 +--
.../store/easy/text/writer/TextRecordWriter.java | 121 ++
.../java-exec/src/main/resources/drill-module.conf | 2 +
.../exec/physical/impl/writer/TestTextWriter.java | 264 +
.../text/compliant/TestCsvTableProperties.java | 76 --
.../java/org/apache/drill/test/ClusterFixture.java | 94 +---
20 files changed, 667 insertions(+), 260 deletions(-)
diff --git a/exec/java-exec/pom.xml b/exec/java-exec/pom.xml
index a9ac685..9f1ace3 100644
--- a/exec/java-exec/pom.xml
+++ b/exec/java-exec/pom.xml
@@ -89,7 +89,7 @@
com.univocity
univocity-parsers
- 1.3.0
+ 2.8.3
org.apache.commons
diff --git
a/exec/java-exec/src/main/codegen/templates/StringOutputRecordWriter.java
b/exec/java-exec/src/main/codegen/templates/StringOutputRecordWriter.java
index 1bf1b09..017fda4 100644
--- a/exec/java-exec/src/main/codegen/templates/StringOutputRecordWriter.java
+++ b/exec/java-exec/src/main/codegen/templates/StringOutputRecordWriter.java
@@ -43,7 +43,7 @@ import java.util.Map;
/**
* Abstract implementation of RecordWriter interface which exposes interface:
- *{@link #writeHeader(List)}
+ *{@link #startNewSchema(BatchSchema)}
*{@link #addField(int,String)}
* to output the data in string format instead of implementing addField for
each type holder.
*
@@ -60,13 +60,7 @@ public abstract class StringOutputRecordWriter extends
AbstractRecordWriter {
@Override
public void updateSchema(VectorAccessible batch) throws IOException {
-BatchSchema schema = batch.getSchema();
-List columnNames = Lists.newArrayList();
-for (int i=0; i < schema.getFieldCount(); i++) {
- columnNames.add(schema.getColumn(i).getName());
-}
-
-startNewSchema(columnNames);
+startNewSchema(batch.getSchema());
}
@Override
@@ -160,6 +154,6 @@ public abstract class StringOutputRecordWriter extends
AbstractRecordWriter {
public void cleanup() throws IOException {
}
- public abstract void startNewSchema(List columnNames) throws
IOException;
+ public abstract void startNewSchema(BatchSchema schema) throws IOException;
public abstract void addField(int fieldId, String value) throws IOException;
}
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java
index 066d04d..20668b3 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java
@@ -431,6 +431,13 @@ public final class ExecConstants {
public static final DoubleValidator TEXT_ESTIMATED_ROW_SIZE = new
RangeDoubleValidator("store.text.estimated_row_size_bytes", 1, Long.MAX_VALUE,
new OptionDescription("Estimate of the row size in a delimited text
file, such as csv. The closer to actual, the better the query plan. Used for
all csv files in the system/session where the value is set. Impacts the
decision to plan a broadcast join or not."));
+ public static final String TEXT_WRITER_ADD_HEADER =
"store.text.writer.add_header";
+ public static final BooleanValidator TEXT_WRITER_ADD_HEADER_VALIDATOR = new
BooleanValidator(TEXT_WRITER_ADD_HEADER,
+new OptionDescription("Enables the TEXT writer to write header in newly
created file. Default is true. (Drill 1.17
[drill] 03/03: DRILL-7414: EVF incorrectly sets buffer writer index after rollover
This is an automated email from the ASF dual-hosted git repository.
arina pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git
commit ad75bbd9a300771e2b1d40f29876c449e9113b7c
Author: Paul Rogers
AuthorDate: Sun Oct 20 14:09:26 2019 -0700
DRILL-7414: EVF incorrectly sets buffer writer index after rollover
Enabling the vector validator on the "new" scan operator, in cases
in which overflow occurs, identified that the DrillBuf writer index
was not properly set for repeated vectors.
Enables such checking, adds unit tests, and fixes the writer index
issue.
closes #1878
---
.../physical/impl/validate/BatchValidator.java | 79 ++---
.../resultSet/impl/RepeatedVectorState.java| 10 +-
.../physical/resultSet/impl/SingleVectorState.java | 14 +-
.../drill/exec/physical/impl/MockRecordBatch.java | 84 +++---
.../impl/TestResultSetLoaderOverflow.java | 334 -
.../accessor/writer/OffsetVectorWriterImpl.java| 7 +-
6 files changed, 292 insertions(+), 236 deletions(-)
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/validate/BatchValidator.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/validate/BatchValidator.java
index dde6583..2753f55 100644
---
a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/validate/BatchValidator.java
+++
b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/validate/BatchValidator.java
@@ -20,6 +20,7 @@ package org.apache.drill.exec.physical.impl.validate;
import java.util.IdentityHashMap;
import java.util.Map;
+import org.apache.drill.exec.physical.impl.protocol.OperatorRecordBatch;
import org.apache.drill.exec.record.RecordBatch;
import org.apache.drill.exec.record.SimpleVectorWrapper;
import org.apache.drill.exec.record.VectorAccessible;
@@ -153,13 +154,13 @@ public class BatchValidator {
* unnecessary.
*/
private static Map, CheckMode> buildRules() {
-final Map, CheckMode> rules = new
IdentityHashMap<>();
-//rules.put(OperatorRecordBatch.class, CheckMode.ALL);
+Map, CheckMode> rules = new
IdentityHashMap<>();
+rules.put(OperatorRecordBatch.class, CheckMode.ALL);
return rules;
}
public static boolean validate(RecordBatch batch) {
-final CheckMode checkMode = checkRules.get(batch.getClass());
+CheckMode checkMode = checkRules.get(batch.getClass());
// If no rule, don't check this batch.
@@ -173,10 +174,10 @@ public class BatchValidator {
// All batches that do any checks will at least check counts.
-final ErrorReporter reporter = errorReporter(batch);
-final int rowCount = batch.getRecordCount();
+ErrorReporter reporter = errorReporter(batch);
+int rowCount = batch.getRecordCount();
int valueCount = rowCount;
-final VectorContainer container = batch.getContainer();
+VectorContainer container = batch.getContainer();
if (!container.hasRecordCount()) {
reporter.error(String.format(
"%s: Container record count not set",
@@ -185,11 +186,11 @@ public class BatchValidator {
// Row count will = container count for most operators.
// Row count <= container count for the filter operator.
- final int containerRowCount = container.getRecordCount();
+ int containerRowCount = container.getRecordCount();
valueCount = containerRowCount;
switch (batch.getSchema().getSelectionVectorMode()) {
case FOUR_BYTE:
-final int sv4Count = batch.getSelectionVector4().getCount();
+int sv4Count = batch.getSelectionVector4().getCount();
if (sv4Count != rowCount) {
reporter.error(String.format(
"Mismatch between %s record count = %d, SV4 record count = %d",
@@ -199,7 +200,7 @@ public class BatchValidator {
// TODO: Don't know how to check SV4 batches
return true;
case TWO_BYTE:
-final int sv2Count = batch.getSelectionVector2().getCount();
+int sv2Count = batch.getSelectionVector2().getCount();
if (sv2Count != rowCount) {
reporter.error(String.format(
"Mismatch between %s record count = %d, SV2 record count = %d",
@@ -212,7 +213,7 @@ public class BatchValidator {
batch.getClass().getSimpleName(),
containerRowCount, sv2Count));
}
-final int svTotalCount =
batch.getSelectionVector2().getBatchActualRecordCount();
+int svTotalCount =
batch.getSelectionVector2().getBatchActualRecordCount();
if (svTotalCount != containerRowCount) {
reporter.error(String.format(
"Mismatch between %s container count = %d, SV2 total count = %d",
@@ -237,13 +238,13 @@ public class BatchValidator {
}
public static boolean validate(VectorAccessible batch) {
-
[drill] branch master updated (849e2b6 -> ad75bbd)
This is an automated email from the ASF dual-hosted git repository. arina pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/drill.git. from 849e2b6 DRILL-7405: Avoiding download of TPC-H data new a13608b DRILL-6096: Provide mechanism to configure text writer configuration new ed3ad76 DRILL-7403: Validate batch checks, vector integretity in unit tests new ad75bbd DRILL-7414: EVF incorrectly sets buffer writer index after rollover The 3 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: exec/java-exec/pom.xml | 2 +- .../templates/StringOutputRecordWriter.java| 12 +- .../java/org/apache/drill/exec/ExecConstants.java | 7 + .../impl/scan/columns/ColumnsArrayParser.java | 13 +- .../physical/impl/validate/BatchValidator.java | 430 - .../validate/IteratorValidatorBatchIterator.java | 38 +- .../resultSet/impl/RepeatedVectorState.java| 10 +- .../physical/resultSet/impl/SingleVectorState.java | 14 +- .../exec/server/options/SystemOptionManager.java | 2 + .../exec/store/easy/text/TextFormatPlugin.java | 53 ++- .../store/easy/text/reader/BaseFieldOutput.java| 2 +- .../store/easy/text/reader/FieldVarCharOutput.java | 13 +- .../exec/store/easy/text/reader/HeaderBuilder.java | 26 +- .../easy/text/reader/RepeatedVarCharOutput.java| 21 +- .../exec/store/easy/text/reader/TextInput.java | 13 +- .../exec/store/easy/text/reader/TextOutput.java| 54 ++- .../store/easy/text/reader/TextParsingContext.java | 119 -- .../easy/text/reader/TextParsingSettings.java | 2 +- .../exec/store/easy/text/reader/TextReader.java| 31 +- .../store/easy/text/writer/TextRecordWriter.java | 121 +++--- .../apache/drill/exec/vector/VectorValidator.java | 6 +- .../java-exec/src/main/resources/drill-module.conf | 2 + .../drill/exec/physical/impl/MockRecordBatch.java | 84 ++-- .../physical/impl/validate/TestBatchValidator.java | 118 +++--- .../exec/physical/impl/writer/TestTextWriter.java | 264 + .../impl/TestResultSetLoaderOverflow.java | 334 +--- .../text/compliant/TestCsvTableProperties.java | 76 ++-- .../java/org/apache/drill/test/ClusterFixture.java | 94 +++-- .../accessor/writer/OffsetVectorWriterImpl.java| 7 +- 29 files changed, 1334 insertions(+), 634 deletions(-) create mode 100644 exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/writer/TestTextWriter.java
[drill] 01/04: DRILL-7401: Upgrade to SqlLine 1.9.0
This is an automated email from the ASF dual-hosted git repository.
arina pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git
commit 43d1ad31ad72ef9f307f7ef1e9c26be0a0b30f50
Author: Arina Ielchiieva
AuthorDate: Wed Oct 16 18:10:20 2019 +0300
DRILL-7401: Upgrade to SqlLine 1.9.0
closes #1875
---
distribution/src/resources/sqlline | 7 +--
distribution/src/resources/sqlline.bat | 13 ++---
exec/java-exec/src/main/resources/drill-sqlline.conf | 5 -
pom.xml | 2 +-
4 files changed, 12 insertions(+), 15 deletions(-)
diff --git a/distribution/src/resources/sqlline
b/distribution/src/resources/sqlline
index 8defce9..d8b36cf 100644
--- a/distribution/src/resources/sqlline
+++ b/distribution/src/resources/sqlline
@@ -78,12 +78,7 @@ if ! $is_cygwin; then
DRILL_SHELL_OPTS="$DRILL_SHELL_OPTS --color=true"
fi
-# To add nohup support for SQLline script
-if [[ ( ! $(ps -o stat= -p $$) =~ "+" ) && ! ( -p /dev/stdin ) ]]; then
-export SQLLINE_JAVA_OPTS="$SQLLINE_JAVA_OPTS
-Djline.terminal=jline.UnsupportedTerminal"
-fi
-
-SHELL_OPTS="$DRILL_SHELL_JAVA_OPTS $SQLLINE_JAVA_OPTS $DRILL_SHELL_LOG_OPTS
$CLIENT_GC_OPTS"
+SHELL_OPTS="$DRILL_SHELL_JAVA_OPTS $SQLLINE_JAVA_OPTS $DRILL_SHELL_LOG_OPTS
$CLIENT_GC_OPTS -Dorg.jline.terminal.dumb=true"
CMD="$JAVA $SHELL_OPTS -cp $CP sqlline.SqlLine -ac
org.apache.drill.exec.client.DrillSqlLineApplication"
# The wrapper is purely for unit testing.
diff --git a/distribution/src/resources/sqlline.bat
b/distribution/src/resources/sqlline.bat
index 6f79395..daf5afe 100755
--- a/distribution/src/resources/sqlline.bat
+++ b/distribution/src/resources/sqlline.bat
@@ -117,7 +117,6 @@ echo.
goto error
:initDrillEnv
-echo DRILL_ARGS - "%DRILL_ARGS%"
rem
rem Deal with Drill variables
@@ -131,7 +130,7 @@ if "test%DRILL_LOG_DIR%" == "test" (
set DRILL_LOG_DIR=%DRILL_HOME%\log
)
-@rem Drill temporary directory is used as base for temporary storage of
Dynamic UDF jars.
+rem Drill temporary directory is used as base for temporary storage of Dynamic
UDF jars
if "test%DRILL_TMP_DIR%" == "test" (
set DRILL_TMP_DIR=%TEMP%
)
@@ -141,11 +140,11 @@ rem Deal with Hadoop JARs, if HADOOP_HOME was specified
rem
if "test%HADOOP_HOME%" == "test" (
- echo HADOOP_HOME not detected...
+ rem HADOOP_HOME not detected...
set USE_HADOOP_CP=0
set HADOOP_HOME=%DRILL_HOME%\winutils
) else (
- echo Calculating HADOOP_CLASSPATH ...
+ rem Calculating HADOOP_CLASSPATH ...
for %%i in (%HADOOP_HOME%\lib\*.jar) do (
set IGNOREJAR=0
for /F "tokens=*" %%A in (%DRILL_BIN_DIR%\hadoop-excludes.txt) do (
@@ -163,10 +162,10 @@ rem Deal with HBase JARs, if HBASE_HOME was specified
rem
if "test%HBASE_HOME%" == "test" (
- echo HBASE_HOME not detected...
+ rem HBASE_HOME not detected...
set USE_HBASE_CP=0
) else (
- echo Calculating HBASE_CLASSPATH ...
+ rem Calculating HBASE_CLASSPATH ...
for %%i in (%HBASE_HOME%\lib\*.jar) do (
set IGNOREJAR=0
for /F "tokens=*" %%A in (%DRILL_BIN_DIR%\hadoop-excludes.txt) do (
@@ -179,7 +178,7 @@ if "test%HBASE_HOME%" == "test" (
set USE_HBASE_CP=1
)
-echo Calculating Drill classpath...
+rem Calculating Drill classpath...
set DRILL_CP=%DRILL_CONF_DIR%
if NOT "test%DRILL_CLASSPATH_PREFIX%"=="test" set
DRILL_CP=!DRILL_CP!;%DRILL_CLASSPATH_PREFIX%
diff --git a/exec/java-exec/src/main/resources/drill-sqlline.conf
b/exec/java-exec/src/main/resources/drill-sqlline.conf
index 58d0413..cde49cb 100644
--- a/exec/java-exec/src/main/resources/drill-sqlline.conf
+++ b/exec/java-exec/src/main/resources/drill-sqlline.conf
@@ -87,7 +87,10 @@ drill.sqlline {
maxColumnWidth: 80,
colorScheme: obsidian,
# display null values for varchar / char / boolean as null
-nullValue: "null"
+nullValue: "null",
+# will ask for the credentials if there is only one specified
+connectInteractionMode: useNPTogetherOrEmpty,
+showLineNumbers: true
},
# display prompt as `apache drill>` or when schema is set as `apache drill
(dfs.tmp)>`
prompt.with_schema: true
diff --git a/pom.xml b/pom.xml
index 36684cc..d551890 100644
--- a/pom.xml
+++ b/pom.xml
@@ -60,7 +60,7 @@
1.20.0-drill-r1
1.15.0
3.0.11
-1.8.0
+1.9.0
2.9.9
2.9.9
3.4.12
[drill] branch master updated (2840702 -> 849e2b6)
This is an automated email from the ASF dual-hosted git repository. arina pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/drill.git. from 2840702 DRILL-7385: Convert PCAP Format Plugin to EVF new 43d1ad3 DRILL-7401: Upgrade to SqlLine 1.9.0 new 698ec58 DRILL-7402: Suppress batch dumps for expected failures in tests new e8d9503 DRILL-7412: Minor unit test improvements new 849e2b6 DRILL-7405: Avoiding download of TPC-H data The 4 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../drill/common/exceptions/DrillException.java| 8 +- .../drill/common/exceptions/UserException.java | 10 +- common/src/test/resources/logback-test.xml | 2 +- contrib/data/tpch-sample-data/bin/pom.xml | 140 - .../data/tpch-sample-data/parquet/customer.parquet | Bin 0 -> 132891 bytes .../data/tpch-sample-data/parquet/lineitem.parquet | Bin 0 -> 2839887 bytes .../data/tpch-sample-data/parquet/nation.parquet | Bin 0 -> 1845 bytes .../data/tpch-sample-data/parquet/orders.parquet | Bin 0 -> 661050 bytes contrib/data/tpch-sample-data/parquet/part.parquet | Bin 0 -> 93889 bytes .../data/tpch-sample-data/parquet/partsupp.parquet | Bin 0 -> 449294 bytes .../data/tpch-sample-data/parquet/region.parquet | Bin .../data/tpch-sample-data/parquet/supplier.parquet | Bin 10467 -> 9699 bytes contrib/data/tpch-sample-data/pom.xml | 25 ++-- distribution/src/resources/sqlline | 7 +- distribution/src/resources/sqlline.bat | 13 +- .../drill/exec/physical/impl/BaseRootExec.java | 11 +- .../apache/drill/exec/work/foreman/Foreman.java| 2 - .../drill/exec/work/foreman/ForemanException.java | 4 +- .../exec/work/foreman/QueryStateProcessor.java | 9 +- .../java-exec/src/main/resources/drill-module.conf | 4 + .../src/main/resources/drill-sqlline.conf | 5 +- .../java/org/apache/drill/TestOperatorDump.java| 2 + .../drill/exec/coord/zk/TestEphemeralStore.java| 5 +- .../drill/exec/record/vector/TestValueVector.java | 7 +- .../java/org/apache/drill/test/BaseTestQuery.java | 19 +-- .../apache/drill/test/ClusterFixtureBuilder.java | 2 +- .../java/org/apache/drill/test/ConfigBuilder.java | 37 ++ .../org/apache/drill/test/QueryRowSetIterator.java | 5 +- .../java-exec/src/test/resources/drill-module.conf | 108 .../apache/drill/exec/memory/BoundsChecking.java | 29 +++-- .../main/codegen/templates/FixedValueVectors.java | 11 +- .../org/apache/drill/exec/vector/BitVector.java| 63 +- .../org/apache/drill/exec/vector/ValueVector.java | 10 +- .../apache/drill/exec/vector/VectorTrimmer.java| 6 - .../exec/vector/complex/RepeatedValueVector.java | 29 ++--- pom.xml| 2 +- 36 files changed, 221 insertions(+), 354 deletions(-) delete mode 100644 contrib/data/tpch-sample-data/bin/pom.xml create mode 100755 contrib/data/tpch-sample-data/parquet/customer.parquet create mode 100755 contrib/data/tpch-sample-data/parquet/lineitem.parquet create mode 100755 contrib/data/tpch-sample-data/parquet/nation.parquet create mode 100755 contrib/data/tpch-sample-data/parquet/orders.parquet create mode 100755 contrib/data/tpch-sample-data/parquet/part.parquet create mode 100755 contrib/data/tpch-sample-data/parquet/partsupp.parquet copy exec/java-exec/src/test/resources/directcount.parquet => contrib/data/tpch-sample-data/parquet/region.parquet (100%) copy exec/java-exec/src/test/resources/supplier_snappy.parquet => contrib/data/tpch-sample-data/parquet/supplier.parquet (50%) mode change 100644 => 100755
[drill] 02/04: DRILL-7402: Suppress batch dumps for expected failures in tests
This is an automated email from the ASF dual-hosted git repository.
arina pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git
commit 698ec5844d1769fa9d4a7c24380bfb7591c68805
Author: Paul Rogers
AuthorDate: Sun Oct 13 14:43:27 2019 -0700
DRILL-7402: Suppress batch dumps for expected failures in tests
Drill provides a way to dump the last few batches when an error
occurs. However, in tests, we often deliberately cause something
to fail. In this case, the batch dump is unnecessary.
This enhancement adds a config property, disabled in tests, that
controls the dump activity. The option is enabled in the one test
that needs it enabled.
closes #1872
---
.../drill/exec/physical/impl/BaseRootExec.java | 11 ++---
.../java-exec/src/main/resources/drill-module.conf | 4
.../java/org/apache/drill/TestOperatorDump.java| 2 ++
.../apache/drill/test/ClusterFixtureBuilder.java | 2 +-
.../java/org/apache/drill/test/ConfigBuilder.java | 27 +++---
5 files changed, 23 insertions(+), 23 deletions(-)
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/BaseRootExec.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/BaseRootExec.java
index 95a1235..04ea6b6 100644
---
a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/BaseRootExec.java
+++
b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/BaseRootExec.java
@@ -36,9 +36,10 @@ import org.apache.drill.exec.record.RecordBatch.IterOutcome;
public abstract class BaseRootExec implements RootExec {
private static final org.slf4j.Logger logger =
org.slf4j.LoggerFactory.getLogger(BaseRootExec.class);
- protected OperatorStats stats = null;
- protected OperatorContext oContext = null;
- protected RootFragmentContext fragmentContext = null;
+ public static final String ENABLE_BATCH_DUMP_CONFIG =
"drill.exec.debug.dump_batches";
+ protected OperatorStats stats;
+ protected OperatorContext oContext;
+ protected RootFragmentContext fragmentContext;
private List operators;
public BaseRootExec(final RootFragmentContext fragmentContext, final
PhysicalOperator config) throws OutOfMemoryException {
@@ -113,6 +114,7 @@ public abstract class BaseRootExec implements RootExec {
case OK:
stats.batchReceived(0, b.getRecordCount(), false);
break;
+ default:
}
return next;
}
@@ -129,6 +131,9 @@ public abstract class BaseRootExec implements RootExec {
if (operators == null) {
return;
}
+if (!fragmentContext.getConfig().getBoolean(ENABLE_BATCH_DUMP_CONFIG)) {
+ return;
+}
final int numberOfBatchesToDump = 2;
logger.error("Batch dump started: dumping last {} failed batches",
numberOfBatchesToDump);
diff --git a/exec/java-exec/src/main/resources/drill-module.conf
b/exec/java-exec/src/main/resources/drill-module.conf
index 62af353..7dfafd3 100644
--- a/exec/java-exec/src/main/resources/drill-module.conf
+++ b/exec/java-exec/src/main/resources/drill-module.conf
@@ -272,6 +272,10 @@ drill.exec: {
// the command line:
// java ... -ea -Ddrill.exec.debug.validate_vectors=true ...
validate_vectors: false
+// If true, dumps several record batches when an operator fails.
+// Generally disabled in testing, especially when we expect
+// a failure
+dump_batches: true
},
spill: {
// *** Options common to all the operators that may spill
diff --git
a/exec/java-exec/src/test/java/org/apache/drill/TestOperatorDump.java
b/exec/java-exec/src/test/java/org/apache/drill/TestOperatorDump.java
index 18ba61c..2c0fc99 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/TestOperatorDump.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/TestOperatorDump.java
@@ -21,6 +21,7 @@ import ch.qos.logback.classic.spi.ILoggingEvent;
import ch.qos.logback.core.ConsoleAppender;
import org.apache.drill.common.exceptions.UserRemoteException;
import org.apache.drill.exec.exception.OutOfMemoryException;
+import org.apache.drill.exec.physical.impl.BaseRootExec;
import org.apache.drill.exec.physical.impl.ScanBatch;
import org.apache.drill.exec.physical.impl.xsort.managed.ExternalSortBatch;
import org.apache.drill.exec.testing.Controls;
@@ -64,6 +65,7 @@ public class TestOperatorDump extends ClusterTest {
logFixture = LogFixture.builder()
.toConsole(appender, LogFixture.DEFAULT_CONSOLE_FORMAT)
.build();
+builder.configBuilder().put(BaseRootExec.ENABLE_BATCH_DUMP_CONFIG, "true");
startCluster(builder);
}
diff --git
a/exec/java-exec/src/test/java/org/apache/drill/test/ClusterFixtureBuilder.java
b/exec/java-exec/src/test/java/org/apache/drill/test/ClusterFixtureBuilder.java
index fb2813f..9657a5e 100644
---
a/exec/java-exec/src/test/java/org/apache/drill/test/ClusterFixtureBuilder.java
+++
b/ex
[drill] 03/04: DRILL-7412: Minor unit test improvements
This is an automated email from the ASF dual-hosted git repository.
arina pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git
commit e8d9503c6574ceef2876c902117fa2f35e3dd974
Author: Paul Rogers
AuthorDate: Sat Oct 19 23:48:39 2019 -0700
DRILL-7412: Minor unit test improvements
Many tests intentionally trigger errors. A debug-only log setting
sent those errors to stdout. The resulting stack dumps simply cluttered
the test output, so disabled error output to the console.
Drill can apply bounds checks to vectors. Tests run via Maven
enable bounds checking. Now, bounds checking is also enabled in
"debug mode" (when assertions are enabled, as in an IDE.)
Drill contains two test frameworks. The older BaseTestQuery was
marked as deprecated, but many tests still use it and are unlikely
to be changed soon. So, removed the deprecated marker to reduce the
number of spurious warnings.
Also includes a number of minor clean-ups.
closes #1876
---
.../drill/common/exceptions/DrillException.java| 8 +-
.../drill/common/exceptions/UserException.java | 10 +-
common/src/test/resources/logback-test.xml | 2 +-
.../apache/drill/exec/work/foreman/Foreman.java| 2 -
.../drill/exec/work/foreman/ForemanException.java | 4 +-
.../exec/work/foreman/QueryStateProcessor.java | 9 +-
.../drill/exec/coord/zk/TestEphemeralStore.java| 5 +-
.../drill/exec/record/vector/TestValueVector.java | 7 +-
.../java/org/apache/drill/test/BaseTestQuery.java | 19 ++--
.../java/org/apache/drill/test/ConfigBuilder.java | 20 ++--
.../org/apache/drill/test/QueryRowSetIterator.java | 5 +-
.../java-exec/src/test/resources/drill-module.conf | 108 ++---
.../apache/drill/exec/memory/BoundsChecking.java | 29 --
.../main/codegen/templates/FixedValueVectors.java | 11 +--
.../org/apache/drill/exec/vector/BitVector.java| 63 ++--
.../org/apache/drill/exec/vector/ValueVector.java | 10 +-
.../apache/drill/exec/vector/VectorTrimmer.java| 6 --
.../exec/vector/complex/RepeatedValueVector.java | 29 +++---
18 files changed, 177 insertions(+), 170 deletions(-)
diff --git
a/common/src/main/java/org/apache/drill/common/exceptions/DrillException.java
b/common/src/main/java/org/apache/drill/common/exceptions/DrillException.java
index 5acee3e..c3d1071 100644
---
a/common/src/main/java/org/apache/drill/common/exceptions/DrillException.java
+++
b/common/src/main/java/org/apache/drill/common/exceptions/DrillException.java
@@ -17,10 +17,10 @@
*/
package org.apache.drill.common.exceptions;
+@SuppressWarnings("serial")
public class DrillException extends Exception {
- public DrillException() {
-super();
- }
+
+ public DrillException() { }
public DrillException(String message, Throwable cause, boolean
enableSuppression, boolean writableStackTrace) {
super(message, cause, enableSuppression, writableStackTrace);
@@ -37,6 +37,4 @@ public class DrillException extends Exception {
public DrillException(Throwable cause) {
super(cause);
}
-
- static final org.slf4j.Logger logger =
org.slf4j.LoggerFactory.getLogger(DrillException.class);
}
diff --git
a/common/src/main/java/org/apache/drill/common/exceptions/UserException.java
b/common/src/main/java/org/apache/drill/common/exceptions/UserException.java
index eccdb8e..1ba8671 100644
--- a/common/src/main/java/org/apache/drill/common/exceptions/UserException.java
+++ b/common/src/main/java/org/apache/drill/common/exceptions/UserException.java
@@ -585,6 +585,7 @@ public class UserException extends DrillRuntimeException {
return this;
}
+private static final File SPIN_FILE = new File("/tmp/drill/spin");
/**
* builds a user exception or returns the wrapped one. If the error is a
system error, the error message is logged
* to the given {@link Logger}.
@@ -596,14 +597,13 @@ public class UserException extends DrillRuntimeException {
// To allow for debugging:
//
- // A spinner code to make the execution stop here while the file
'/tmp/drill/spin' exists
+ // A spinner code to make the execution stop here while the file
'/tmp/drill/spin' exists
// Can be used to attach a debugger, use jstack, etc
// (do "clush -a touch /tmp/drill/spin" to turn this on across all the
cluster nodes, and to
- // release the spinning threads do "clush -a rm /tmp/drill/spin")
+ // release the spinning threads do "clush -a rm /tmp/drill/spin")
// The processID of the spinning thread (along with the error message)
should then be found
// in a file like /tmp/drill/spin4148663301172491613.tmp
- final File spinFile = new File("/tmp/drill/spin");
- if ( spinFile.exists() ) {
+ if (SPIN_FILE.exists()) {
f
[drill] 04/04: DRILL-7405: Avoiding download of TPC-H data
This is an automated email from the ASF dual-hosted git repository.
arina pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git
commit 849e2b67d6f86090c70184051c596f0b1c8f15de
Author: Abhishek Girish
AuthorDate: Wed Oct 16 20:24:27 2019 -0700
DRILL-7405: Avoiding download of TPC-H data
closes #1874
---
contrib/data/tpch-sample-data/bin/pom.xml | 140 -
.../data/tpch-sample-data/parquet/customer.parquet | Bin 0 -> 132891 bytes
.../data/tpch-sample-data/parquet/lineitem.parquet | Bin 0 -> 2839887 bytes
.../data/tpch-sample-data/parquet/nation.parquet | Bin 0 -> 1845 bytes
.../data/tpch-sample-data/parquet/orders.parquet | Bin 0 -> 661050 bytes
contrib/data/tpch-sample-data/parquet/part.parquet | Bin 0 -> 93889 bytes
.../data/tpch-sample-data/parquet/partsupp.parquet | Bin 0 -> 449294 bytes
.../data/tpch-sample-data/parquet/region.parquet | Bin 0 -> 617 bytes
.../data/tpch-sample-data/parquet/supplier.parquet | Bin 0 -> 9699 bytes
contrib/data/tpch-sample-data/pom.xml | 25 ++--
10 files changed, 14 insertions(+), 151 deletions(-)
diff --git a/contrib/data/tpch-sample-data/bin/pom.xml
b/contrib/data/tpch-sample-data/bin/pom.xml
deleted file mode 100644
index 9afb8f1..000
--- a/contrib/data/tpch-sample-data/bin/pom.xml
+++ /dev/null
@@ -1,140 +0,0 @@
-
-
-http://maven.apache.org/POM/4.0.0;
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance;
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
http://maven.apache.org/xsd/maven-4.0.0.xsd;>
- 4.0.0
-
-drill-contrib-data-parent
-org.apache.drill.contrib.data
-0.8.0-SNAPSHOT
-
-
- tpch-sample-data
- contrib/data/tpch-sample-data
- jar
-
-
-
-
-
-
-com.googlecode.maven-download-plugin
-maven-download-plugin
-1.1.0
-
-
-install-tgz
-package
-
- wget
-
-
-
http://apache-drill.s3.amazonaws.com/files/sf-0.01_tpc-h_parquet.tgz
- tpch.tgz
- true
- ${project.build.directory}
-
-
-
-
-
-
-org.codehaus.mojo
-truezip-maven-plugin
-1.0
-
-
-copy-out-files
-
- copy
-
-package
-
-
-
-
-
-
-
-copy-out-fileset
-
- copy
-
-package
-
-
-${project.build.directory}/tpch.tgz
-
${project.build.directory}/data
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- sonatype-public-repository
- https://oss.sonatype.org/content/groups/public
-
-true
-
-
-true
-
-
-
-
-
diff --git a/contrib/data/tpch-sample-data/parquet/customer.parquet
b/contrib/data/tpch-sample-data/parquet/customer.parquet
new file mode 100755
index 000..25c6553
Binary files /dev/null and
b/contrib/data/tpch-sample-data/parquet/customer.parquet differ
diff --git a/contrib/data/tpch-sample-data/parquet/lineitem.parquet
b/contrib/data/tpch-sample-data/parquet/lineitem.parquet
new file mode 100755
index 000..7af0a72
Binary files /dev/null and
b/contrib/data/tpch-sample-data/parquet/lineitem.parquet differ
diff --git a/contrib/data/tpch-sample-data/parquet/nation.parquet
b/contrib/data/tpch-sample-data/parquet/nation.parquet
new file mode 100755
index 000..29f2c22
Binary files /dev/null and
b/contrib/data/tpch-sample-data/parquet/nation.parquet differ
diff --git a/contrib/data/tpch-sample-data/parquet/orders.parquet
b/contrib/data/tpch-sample-data/parquet/orders.parquet
new file mode 100755
index 000..3c728ac
Binary files /dev/null and
b/contrib/data/tpch-sample-data/parquet/orders.parquet differ
diff --git a/contrib/data/tpch-sample-data/parquet/part.parquet
b/contrib/data/tpch-sample-data/parquet/part.parquet
new file mode 100755
index 000..e86fe08
Binary files /dev/null and b/contrib/data/tpch-sample-data/parquet/part.parquet
differ
diff --git a/contrib/data/tpch-sample-data/parquet/partsupp.parquet
b/contrib/data/tpch-sample-data/parquet/partsupp.parquet
new file mode 100755
index 000..3321245
Binary files /dev/null and
b/contrib/data/tpch-sample-data/parquet/partsupp.parquet differ
diff --git a/contrib/data/tpch-sample-data/parquet/region.parquet
b/contrib/data/tpch-sample-data/parquet/region.parquet
new file mode 100755
index 000..2994a58
Binary files /dev/null and
b/contrib/data/tpch-samp
[drill] branch master updated (8087223 -> 2840702)
This is an automated email from the ASF dual-hosted git repository. arina pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/drill.git. from 8087223 DRILL-7377: Nested schemas for dynamic EVF columns add 2840702 DRILL-7385: Convert PCAP Format Plugin to EVF No new revisions were added by this update. Summary of changes: .../drill/exec/store/pcap/PcapBatchReader.java | 330 + .../drill/exec/store/pcap/PcapDrillTable.java | 76 - .../drill/exec/store/pcap/PcapFormatConfig.java| 28 +- .../drill/exec/store/pcap/PcapFormatPlugin.java| 132 +++-- .../drill/exec/store/pcap/PcapFormatUtils.java | 5 + .../drill/exec/store/pcap/decoder/Packet.java | 4 + .../drill/exec/store/pcap/schema/Schema.java | 23 +- .../drill/exec/store/pcap/TestPcapEVFReader.java | 103 +++ .../exec/store/pcap/TestPcapRecordReader.java | 13 +- 9 files changed, 543 insertions(+), 171 deletions(-) create mode 100644 exec/java-exec/src/main/java/org/apache/drill/exec/store/pcap/PcapBatchReader.java delete mode 100644 exec/java-exec/src/main/java/org/apache/drill/exec/store/pcap/PcapDrillTable.java create mode 100644 exec/java-exec/src/test/java/org/apache/drill/exec/store/pcap/TestPcapEVFReader.java
[drill] branch master updated: DRILL-7377: Nested schemas for dynamic EVF columns
This is an automated email from the ASF dual-hosted git repository.
arina pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git
The following commit(s) were added to refs/heads/master by this push:
new 8087223 DRILL-7377: Nested schemas for dynamic EVF columns
8087223 is described below
commit 8087223667c01f141f72e0383a0c4f42af99f5ac
Author: Paul Rogers
AuthorDate: Sun Oct 6 22:09:44 2019 -0700
DRILL-7377: Nested schemas for dynamic EVF columns
The Result Set Loader (part of EVF) allows adding columns up-front
before reading rows (so-called "early schema.") Such schemas allow
nested columns (maps with members, repeated lists with a type, etc.)
The Result Set Loader also allows adding columns dynamically
while loading data (so-called "late schema".) Previously, the code
assumed that columns would be added top-down: first the map, then
the map's contents, etc.
Charles found a need to allow adding a nested column (a repeated
list with a declared list type.)
This patch revises the code to use the same mechanism in both the
early- and late-schema cases, allowing adding nested columns at
any time.
Testing: Added a new unit test case for the repeated list late
schema with content case.
---
.../physical/resultSet/impl/BuildFromSchema.java | 94 +++---
.../physical/resultSet/impl/ColumnBuilder.java | 6 ++
.../resultSet/impl/ResultSetLoaderImpl.java| 2 +-
.../exec/physical/resultSet/impl/TupleState.java | 15 +---
.../impl/TestResultSetLoaderRepeatedList.java | 73 -
.../accessor/writer/AbstractTupleWriter.java | 26 --
6 files changed, 162 insertions(+), 54 deletions(-)
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/resultSet/impl/BuildFromSchema.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/resultSet/impl/BuildFromSchema.java
index e0d9e27..bf1256d 100644
---
a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/resultSet/impl/BuildFromSchema.java
+++
b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/resultSet/impl/BuildFromSchema.java
@@ -32,6 +32,21 @@ import
org.apache.drill.exec.vector.accessor.writer.RepeatedListWriter;
*
* Recursion is much easier if we can go bottom-up. But, writers
* require top-down construction.
+ *
+ * This particular class builds a column and all its contents.
+ * For example, given a map, which contains a repeated list which
+ * contains a repeated INT, this class first builds the map,
+ * then adds the repeated list, then adds the INT array. To do
+ * so, it will create a copy of the structured metadata.
+ *
+ * A drawback of this approach is that the metadata objects used
+ * in the "parent" writers will be copies of, not the same as, those
+ * in the schema from which we are building the writers. At present,
+ * this is not an issue, but it is something to be aware of as uses
+ * become more sophisticated.
+ *
+ * This class contrasts with the @{link ColumnBuilder} class which
+ * builds the structure within a single vector and writer.
*/
public class BuildFromSchema {
@@ -49,6 +64,11 @@ public class BuildFromSchema {
ObjectWriter add(ColumnMetadata colSchema);
}
+ /**
+ * Shim used for adding a column to a tuple directly.
+ * This method will recursively invoke this builder
+ * to expand any nested content.
+ */
private static class TupleShim implements ParentShim {
private final TupleWriter writer;
@@ -63,6 +83,24 @@ public class BuildFromSchema {
}
}
+ /**
+ * Shim used when implementing the add of a column to
+ * a tuple in the result set loader. Directly calls the
+ * internal method to add a column to the "tuple state."
+ */
+ private static class TupleStateShim implements ParentShim {
+private final TupleState state;
+
+public TupleStateShim(TupleState state) {
+ this.state = state;
+}
+
+@Override
+public ObjectWriter add(ColumnMetadata colSchema) {
+ return state.addColumn(colSchema).writer();
+}
+ }
+
private static class UnionShim implements ParentShim {
private final VariantWriter writer;
@@ -89,6 +127,12 @@ public class BuildFromSchema {
}
}
+ private static BuildFromSchema instance = new BuildFromSchema();
+
+ private BuildFromSchema() { }
+
+ public static BuildFromSchema instance() { return instance; }
+
/**
* When creating a schema up front, provide the schema of the desired tuple,
* then build vectors and writers to match. Allows up-front schema definition
@@ -105,17 +149,36 @@ public class BuildFromSchema {
}
}
- private void buildColumn(ParentShim parent, ColumnMetadata colSchema) {
+ /**
+ * Build a column recursively. Called internally when adding a column
+ * via the addColumn() method on the tuple
[drill] 01/02: DRILL-5983: Add missing nullable Parquet readers for INT and UINT logical types
This is an automated email from the ASF dual-hosted git repository.
arina pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git
commit 09d15f39e2cda10049affca472689a4e2cbca45d
Author: Arina Ielchiieva
AuthorDate: Fri Oct 4 14:36:01 2019 +0300
DRILL-5983: Add missing nullable Parquet readers for INT and UINT logical
types
closes #1866
---
.../parquet/columnreaders/ColumnReaderFactory.java | 36 ---
.../NullableFixedByteAlignedReaders.java | 63 +--
.../ParquetFixedWidthDictionaryReaders.java| 72 +++---
3 files changed, 120 insertions(+), 51 deletions(-)
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/ColumnReaderFactory.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/ColumnReaderFactory.java
index 03d5382..7f8c018 100644
---
a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/ColumnReaderFactory.java
+++
b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/ColumnReaderFactory.java
@@ -19,8 +19,6 @@ package org.apache.drill.exec.store.parquet.columnreaders;
import org.apache.drill.common.exceptions.ExecutionSetupException;
import org.apache.drill.exec.ExecConstants;
-import org.apache.drill.exec.vector.VarDecimalVector;
-import org.apache.drill.exec.vector.NullableVarDecimalVector;
import org.apache.drill.exec.vector.BigIntVector;
import org.apache.drill.exec.vector.BitVector;
import org.apache.drill.exec.vector.DateVector;
@@ -37,8 +35,11 @@ import org.apache.drill.exec.vector.NullableIntVector;
import org.apache.drill.exec.vector.NullableIntervalVector;
import org.apache.drill.exec.vector.NullableTimeStampVector;
import org.apache.drill.exec.vector.NullableTimeVector;
+import org.apache.drill.exec.vector.NullableUInt4Vector;
+import org.apache.drill.exec.vector.NullableUInt8Vector;
import org.apache.drill.exec.vector.NullableVarBinaryVector;
import org.apache.drill.exec.vector.NullableVarCharVector;
+import org.apache.drill.exec.vector.NullableVarDecimalVector;
import org.apache.drill.exec.vector.TimeStampVector;
import org.apache.drill.exec.vector.TimeVector;
import org.apache.drill.exec.vector.UInt4Vector;
@@ -46,6 +47,7 @@ import org.apache.drill.exec.vector.UInt8Vector;
import org.apache.drill.exec.vector.ValueVector;
import org.apache.drill.exec.vector.VarBinaryVector;
import org.apache.drill.exec.vector.VarCharVector;
+import org.apache.drill.exec.vector.VarDecimalVector;
import org.apache.drill.exec.vector.VariableWidthVector;
import org.apache.parquet.column.ColumnDescriptor;
import org.apache.parquet.column.Encoding;
@@ -57,21 +59,22 @@ import org.apache.parquet.schema.PrimitiveType;
public class ColumnReaderFactory {
/**
- * @param fixedLength
- * @param descriptor
- * @param columnChunkMetaData
+ * Creates fixed column reader for the given column based on its metadata.
+ *
+ * @param fixedLength if fixed length reader should be used
+ * @param descriptor column descriptor
+ * @param columnChunkMetaData column metadata
+ *
* @return ColumnReader object instance
- * @throws SchemaChangeException
*/
static ColumnReader createFixedColumnReader(ParquetRecordReader
recordReader, boolean fixedLength, ColumnDescriptor descriptor,
- ColumnChunkMetaData
columnChunkMetaData, ValueVector v,
- SchemaElement schemaElement)
- throws Exception {
+ ColumnChunkMetaData
columnChunkMetaData, ValueVector v,
+ SchemaElement schemaElement)
throws Exception {
ConvertedType convertedType = schemaElement.getConverted_type();
// if the column is required, or repeated (in which case we just want to
use this to generate our appropriate
// ColumnReader for actually transferring data into the data vector inside
of our repeated vector
if (descriptor.getMaxDefinitionLevel() == 0 ||
descriptor.getMaxRepetitionLevel() > 0) {
- if (columnChunkMetaData.getType() ==
PrimitiveType.PrimitiveTypeName.BOOLEAN){
+ if (columnChunkMetaData.getType() ==
PrimitiveType.PrimitiveTypeName.BOOLEAN) {
return new BitReader(recordReader, descriptor, columnChunkMetaData,
fixedLength, (BitVector) v, schemaElement);
} else if
(!columnChunkMetaData.getEncodings().contains(Encoding.PLAIN_DICTIONARY) && (
@@ -279,6 +282,16 @@ public class ColumnReaderFactory {
return new
NullableFixedByteAlignedReaders.NullableDictionaryIntReader(parentReader,
columnDescriptor, columnChunkMetaData, fixedLength, (NullableIntVector)
valueVec, schemaElement);
}
switch (convertedType) {
+case INT_8:
+case INT_16:
+
[drill] branch master updated (23db12c -> 9993fa3)
This is an automated email from the ASF dual-hosted git repository. arina pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/drill.git. from 23db12c DRILL-7387: Failed to get value by int key from map nested into struct new 09d15f3 DRILL-5983: Add missing nullable Parquet readers for INT and UINT logical types new 9993fa3 DRILL-7358: Fix COUNT(*) for empty text files The 2 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../exec/physical/impl/protocol/SchemaTracker.java | 2 +- .../impl/scan/framework/SchemaNegotiatorImpl.java | 2 +- .../scan/project/projSet/EmptyProjectionSet.java | 3 + .../project/projSet/ExplicitProjectionSet.java | 7 +- .../project/projSet/WildcardProjectionSet.java | 7 ++ .../exec/physical/resultSet/ProjectionSet.java | 1 + .../exec/physical/resultSet/impl/ColumnState.java | 3 +- .../resultSet/impl/ResultSetLoaderImpl.java| 11 +++ .../store/easy/text/reader/BaseFieldOutput.java| 2 +- .../parquet/columnreaders/ColumnReaderFactory.java | 36 ++--- .../NullableFixedByteAlignedReaders.java | 63 +-- .../ParquetFixedWidthDictionaryReaders.java| 72 - .../easy/text/compliant/TestCsvWithHeaders.java| 90 -- .../java/org/apache/drill/test/QueryBuilder.java | 10 ++- 14 files changed, 240 insertions(+), 69 deletions(-)
[drill] 02/02: DRILL-7358: Fix COUNT(*) for empty text files
This is an automated email from the ASF dual-hosted git repository.
arina pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git
commit 9993fa3547b029db5fe33a2210fa6f07e8ac1990
Author: Paul Rogers
AuthorDate: Sat Oct 5 18:57:14 2019 -0700
DRILL-7358: Fix COUNT(*) for empty text files
Fixes a subtle error when a text file has a header (and so has a
schema), but is in a COUNT(*) query, so that no columns are
projected. Ensures that, in this case, an empty schema is
treated as a valid result set.
Tests: updated CSV tests to include this case.
closes #1867
---
.../exec/physical/impl/protocol/SchemaTracker.java | 2 +-
.../impl/scan/framework/SchemaNegotiatorImpl.java | 2 +-
.../scan/project/projSet/EmptyProjectionSet.java | 3 +
.../project/projSet/ExplicitProjectionSet.java | 7 +-
.../project/projSet/WildcardProjectionSet.java | 7 ++
.../exec/physical/resultSet/ProjectionSet.java | 1 +
.../exec/physical/resultSet/impl/ColumnState.java | 3 +-
.../resultSet/impl/ResultSetLoaderImpl.java| 11 +++
.../store/easy/text/reader/BaseFieldOutput.java| 2 +-
.../easy/text/compliant/TestCsvWithHeaders.java| 90 --
.../java/org/apache/drill/test/QueryBuilder.java | 10 ++-
11 files changed, 120 insertions(+), 18 deletions(-)
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/protocol/SchemaTracker.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/protocol/SchemaTracker.java
index 0494498..4149850 100644
---
a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/protocol/SchemaTracker.java
+++
b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/protocol/SchemaTracker.java
@@ -65,7 +65,7 @@ public class SchemaTracker {
private List currentVectors = new ArrayList<>();
public void trackSchema(VectorContainer newBatch) {
-if (! isSameSchema(newBatch)) {
+if (schemaVersion == 0 || ! isSameSchema(newBatch)) {
schemaVersion++;
captureSchema(newBatch);
}
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/scan/framework/SchemaNegotiatorImpl.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/scan/framework/SchemaNegotiatorImpl.java
index 8c934b5..e1302218 100644
---
a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/scan/framework/SchemaNegotiatorImpl.java
+++
b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/scan/framework/SchemaNegotiatorImpl.java
@@ -92,7 +92,7 @@ public class SchemaNegotiatorImpl implements SchemaNegotiator
{
@Override
public void setTableSchema(TupleMetadata schema, boolean isComplete) {
tableSchema = schema;
-this.isSchemaComplete = schema != null && isComplete;
+isSchemaComplete = schema != null && isComplete;
}
@Override
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/scan/project/projSet/EmptyProjectionSet.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/scan/project/projSet/EmptyProjectionSet.java
index 1345006..016cc63 100644
---
a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/scan/project/projSet/EmptyProjectionSet.java
+++
b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/scan/project/projSet/EmptyProjectionSet.java
@@ -37,4 +37,7 @@ public class EmptyProjectionSet implements ProjectionSet {
@Override
public void setErrorContext(CustomErrorContext errorContext) { }
+
+ @Override
+ public boolean isEmpty() { return true; }
}
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/scan/project/projSet/ExplicitProjectionSet.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/scan/project/projSet/ExplicitProjectionSet.java
index 94114f5..cfa82dd 100644
---
a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/scan/project/projSet/ExplicitProjectionSet.java
+++
b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/scan/project/projSet/ExplicitProjectionSet.java
@@ -26,6 +26,8 @@ import
org.apache.drill.exec.physical.resultSet.project.RequestedTuple.Requested
import
org.apache.drill.exec.physical.resultSet.project.RequestedTuple.TupleProjectionType;
import org.apache.drill.exec.record.metadata.ColumnMetadata;
import org.apache.drill.exec.vector.accessor.convert.ColumnConversionFactory;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* Projection set based on an explicit set of columns provided
@@ -34,7 +36,7 @@ import
org.apache.drill.exec.vector.accessor.convert.ColumnConversionFactory;
*/
public class ExplicitProjectionSet extends AbstractProjectionSet {
- private static final org.slf4j.Logger logger =
org.slf4j.LoggerFactory.getLogger(ExplicitProjectionSet.class);
+ private stat
[drill] 03/03: DRILL-7374: Support for IPV6 address
This is an automated email from the ASF dual-hosted git repository.
arina pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git
commit 7535ed9b90165a4745fce066fb8bff4018a32461
Author: Arjun Gupta
AuthorDate: Wed Sep 18 11:13:44 2019 +0530
DRILL-7374: Support for IPV6 address
closes #1857
---
contrib/native/client/src/clientlib/channel.cpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/contrib/native/client/src/clientlib/channel.cpp
b/contrib/native/client/src/clientlib/channel.cpp
index 3af17d6..5e96388 100644
--- a/contrib/native/client/src/clientlib/channel.cpp
+++ b/contrib/native/client/src/clientlib/channel.cpp
@@ -308,7 +308,7 @@ connectionStatus_t Channel::connectInternal() {
const char *port = m_pEndpoint->getPort().c_str();
try {
tcp::resolver resolver(m_ioService);
-tcp::resolver::query query(tcp::v4(), host, port);
+tcp::resolver::query query(host, port);
tcp::resolver::iterator iter = resolver.resolve(query);
tcp::resolver::iterator end;
while (iter != end) {
[drill] branch master updated (d2645c7 -> 7535ed9)
This is an automated email from the ASF dual-hosted git repository. arina pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/drill.git. from d2645c7 DRILL-7170: Ignore uninitialized vector containers for OOM error messages new 636047d DRILL-7357: Expose Drill Metastore data through information_schema new eac7ccc DRILL-7174: Expose complex to Json control in the Drill C++ Client new 7535ed9 DRILL-7374: Support for IPV6 address The 3 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../java/org/apache/drill/common/types/Types.java | 38 ++ .../test/java/org/apache/drill/test/DrillTest.java | 7 +- contrib/format-maprdb/pom.xml | 78 ++-- contrib/native/client/example/querySubmitter.cpp | 14 +- contrib/native/client/src/clientlib/channel.cpp| 2 +- .../client/src/clientlib/drillClientImpl.cpp | 18 + .../client/src/clientlib/drillClientImpl.hpp | 1 + contrib/native/client/src/include/drill/common.hpp | 1 + contrib/native/client/src/test/CMakeLists.txt | 2 + contrib/native/client/src/test/DrillClientTest.cpp | 55 +++ .../exec/hive/TestInfoSchemaOnHiveStorage.java | 2 +- .../drill/exec/store/ischema/FilterEvaluator.java | 216 +++ .../exec/store/ischema/InfoSchemaBatchCreator.java | 10 +- .../drill/exec/store/ischema/InfoSchemaConfig.java | 1 - .../exec/store/ischema/InfoSchemaConstants.java| 138 +++ .../exec/store/ischema/InfoSchemaDrillTable.java | 2 - .../drill/exec/store/ischema/InfoSchemaFilter.java | 30 +- .../store/ischema/InfoSchemaFilterBuilder.java | 28 +- .../exec/store/ischema/InfoSchemaGroupScan.java| 29 +- .../InfoSchemaPushFilterIntoRecordGenerator.java | 25 +- .../store/ischema/InfoSchemaRecordGenerator.java | 386 --- .../store/ischema/InfoSchemaStoragePlugin.java | 23 +- .../exec/store/ischema/InfoSchemaSubScan.java | 9 +- .../drill/exec/store/ischema/InfoSchemaTable.java | 322 +--- .../exec/store/ischema/InfoSchemaTableType.java| 41 +- .../drill/exec/store/ischema/RecordCollector.java | 413 + .../apache/drill/exec/store/ischema/Records.java | 285 -- .../drill/exec/store/ischema/package-info.java | 21 -- .../drill/exec/work/metadata/MetadataProvider.java | 17 +- .../test/java/org/apache/drill/exec/ExecTest.java | 4 - .../org/apache/drill/exec/sql/TestInfoSchema.java | 24 +- .../exec/sql/TestInfoSchemaWithMetastore.java | 401 .../exec/sql/TestSchemaCaseInsensitivity.java | 5 +- .../org/apache/drill/exec/sql/TestViewSupport.java | 3 +- .../exec/work/metadata/TestMetadataProvider.java | 8 +- .../jdbc/test/TestInformationSchemaColumns.java| 2 +- .../org/apache/drill/jdbc/test/TestJdbcQuery.java | 2 + .../components/tables/BasicTablesRequests.java | 12 +- .../drill/metastore/metadata/MetadataInfo.java | 2 + .../components/tables/TestBasicTablesRequests.java | 5 +- pom.xml| 23 +- 41 files changed, 1970 insertions(+), 735 deletions(-) create mode 100644 contrib/native/client/src/test/DrillClientTest.cpp create mode 100644 exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/FilterEvaluator.java create mode 100644 exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/RecordCollector.java delete mode 100644 exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/package-info.java create mode 100644 exec/java-exec/src/test/java/org/apache/drill/exec/sql/TestInfoSchemaWithMetastore.java
[drill] 02/03: DRILL-7174: Expose complex to Json control in the Drill C++ Client
This is an automated email from the ASF dual-hosted git repository.
arina pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git
commit eac7ccc26b1090494bd9520ab3171eec984496cf
Author: Arjun Gupta
AuthorDate: Tue Jun 25 15:21:19 2019 +0530
DRILL-7174: Expose complex to Json control in the Drill C++ Client
closes #1814
---
contrib/native/client/example/querySubmitter.cpp | 14 --
.../client/src/clientlib/drillClientImpl.cpp | 18 +++
.../client/src/clientlib/drillClientImpl.hpp | 1 +
contrib/native/client/src/include/drill/common.hpp | 1 +
contrib/native/client/src/test/CMakeLists.txt | 2 +
contrib/native/client/src/test/DrillClientTest.cpp | 55 ++
6 files changed, 86 insertions(+), 5 deletions(-)
diff --git a/contrib/native/client/example/querySubmitter.cpp
b/contrib/native/client/example/querySubmitter.cpp
index a84d1db..1ca7668 100644
--- a/contrib/native/client/example/querySubmitter.cpp
+++ b/contrib/native/client/example/querySubmitter.cpp
@@ -23,13 +23,13 @@
#include
#include "drill/drillc.hpp"
-int nOptions=25;
+int nOptions=27;
struct Option{
char name[32];
char desc[128];
bool required;
-}qsOptions[]= {
+} qsOptions[] = {
{"plan", "Plan files separated by semicolons", false},
{"query", "Query strings, separated by semicolons", false},
{"type", "Query type [physical|logical|sql|server]", true},
@@ -54,9 +54,9 @@ struct Option{
{"certFilePath", "Path to SSL certificate file", false},
{"disableHostnameVerification", "disable host name verification", false},
{"disableCertVerification", "disable certificate verification", false},
-{"useSystemTrustStore", "[Windows only]. Use the system truststore.",
false },
-{"CustomSSLCtxOptions", "The custom SSL CTX Options", false}
-
+{"useSystemTrustStore", "[Windows only]. Use the system truststore.",
false},
+{"CustomSSLCtxOptions", "The custom SSL CTX Options", false},
+{"supportComplexTypes", "Toggle for supporting complex types", false}
};
std::map qsOptionValues;
@@ -317,6 +317,7 @@ int main(int argc, char* argv[]) {
std::string
disableCertVerification=qsOptionValues["disableCertVerification"];
std::string useSystemTrustStore =
qsOptionValues["useSystemTrustStore"];
std::string customSSLOptions = qsOptionValues["CustomSSLCtxOptions"];
+std::string supportComplexTypes =
qsOptionValues["supportComplexTypes"];
Drill::QueryType type;
@@ -422,6 +423,9 @@ int main(int argc, char* argv[]) {
props.setProperty(USERPROP_CUSTOM_SSLCTXOPTIONS,
customSSLOptions);
}
}
+if (supportComplexTypes.length() > 0){
+props.setProperty(USERPROP_SUPPORT_COMPLEX_TYPES,
supportComplexTypes);
+}
if(client.connect(connectStr.c_str(), )!=Drill::CONN_SUCCESS){
std::cerr<< "Failed to connect with error: "<< client.getError()
<< " (Using:"<isPropSet(USERPROP_SUPPORT_COMPLEX_TYPES) &&
+boost::iequals(userProperties->getProp(USERPROP_SUPPORT_COMPLEX_TYPES,
val), "true");
+return support_complex_types;
+}
+
connectionStatus_t DrillClientImpl::validateHandshake(DrillUserProperties*
properties){
DRILL_MT_LOG(DRILL_LOG(LOG_TRACE) << "validateHandShake\n";)
@@ -547,6 +564,7 @@ connectionStatus_t
DrillClientImpl::validateHandshake(DrillUserProperties* prope
u2b.set_support_listening(true);
u2b.set_support_timeout(DrillClientConfig::getHeartbeatFrequency() > 0);
u2b.set_sasl_support(exec::user::SASL_PRIVACY);
+u2b.set_support_complex_types(handleComplexTypes(properties));
// Adding version info
exec::user::RpcEndpointInfos* infos = u2b.mutable_client_infos();
diff --git a/contrib/native/client/src/clientlib/drillClientImpl.hpp
b/contrib/native/client/src/clientlib/drillClientImpl.hpp
index d6531b2..08f35d3 100644
--- a/contrib/native/client/src/clientlib/drillClientImpl.hpp
+++ b/contrib/native/client/src/clientlib/drillClientImpl.hpp
@@ -473,6 +473,7 @@ class DrillClientImpl : public DrillClientImplBase{
static bool clientNeedsAuthentication(const DrillUserProperties*
userProperties);
+bool handleComplexTypes(const DrillUserProperties* userProperties);
private:
friend class meta::DrillMetadata;
diff --git a/contrib/native/client/src/include/drill/common.hpp
b/contrib/native/client/src/include/drill/common.hpp
index 1324601..9f57446 100644
--- a/contrib/native/client/src/include/drill/common.hpp
+++ b/contrib/native/client/sr
[drill] branch master updated: DRILL-7368: Fix Iceberg Metastore failure when filter column contains nulls
This is an automated email from the ASF dual-hosted git repository.
arina pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git
The following commit(s) were added to refs/heads/master by this push:
new b827d7f DRILL-7368: Fix Iceberg Metastore failure when filter column
contains nulls
b827d7f is described below
commit b827d7f29590dd120fda1a6b653fa34c5701f664
Author: Arina Ielchiieva
AuthorDate: Tue Sep 10 15:30:10 2019 +0300
DRILL-7368: Fix Iceberg Metastore failure when filter column contains nulls
---
exec/java-exec/pom.xml | 6 ++
.../drill/exec/ops/ExecutorFragmentContext.java| 4 +-
.../org/apache/drill/exec/ops/FragmentContext.java | 37 +---
.../apache/drill/exec/ops/FragmentContextImpl.java | 43 +
.../org/apache/drill/exec/ops/QueryContext.java| 5 -
.../apache/drill/exec/server/DrillbitContext.java | 7 +-
.../org/apache/drill/test/OperatorFixture.java | 6 ++
.../apache/drill/test/PhysicalOpUnitTestBase.java | 2 +-
metastore/iceberg-metastore/pom.xml| 2 +-
.../drill/metastore/iceberg/operate/Overwrite.java | 2 +-
.../components/tables/TestBasicRequests.java | 105 +
11 files changed, 136 insertions(+), 83 deletions(-)
diff --git a/exec/java-exec/pom.xml b/exec/java-exec/pom.xml
index a80bc48..a9ac685 100644
--- a/exec/java-exec/pom.xml
+++ b/exec/java-exec/pom.xml
@@ -547,6 +547,12 @@
sqlline
sqlline
+
+ org.apache.drill.metastore
+ drill-iceberg-metastore
+ ${project.version}
+ test
+
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/ops/ExecutorFragmentContext.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/ops/ExecutorFragmentContext.java
index dbf440c..df782b1 100644
---
a/exec/java-exec/src/main/java/org/apache/drill/exec/ops/ExecutorFragmentContext.java
+++
b/exec/java-exec/src/main/java/org/apache/drill/exec/ops/ExecutorFragmentContext.java
@@ -36,8 +36,10 @@ import java.util.Set;
* {@link org.apache.drill.exec.work.fragment.FragmentExecutor}.
*/
public interface ExecutorFragmentContext extends RootFragmentContext {
+
/**
* Returns the root allocator for the Drillbit.
+ *
* @return The root allocator for the Drillbit.
*/
BufferAllocator getRootAllocator();
@@ -58,7 +60,7 @@ public interface ExecutorFragmentContext extends
RootFragmentContext {
QueryProfileStoreContext getProfileStoreContext();
- WorkEventBus getWorkEventbus();
+ WorkEventBus getWorkEventBus();
Set> getUserConnections();
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/ops/FragmentContext.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/ops/FragmentContext.java
index 46f94f0..414d583 100644
---
a/exec/java-exec/src/main/java/org/apache/drill/exec/ops/FragmentContext.java
+++
b/exec/java-exec/src/main/java/org/apache/drill/exec/ops/FragmentContext.java
@@ -21,6 +21,8 @@ import java.io.IOException;
import java.util.List;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.TimeUnit;
+
+import org.apache.drill.metastore.MetastoreRegistry;
import
org.apache.drill.shaded.guava.com.google.common.annotations.VisibleForTesting;
import org.apache.calcite.schema.SchemaPlus;
import org.apache.drill.common.config.DrillConfig;
@@ -96,9 +98,10 @@ public interface FragmentContext extends UdfUtilities,
AutoCloseable {
/**
* Returns the statement type (e.g. SELECT, CTAS, ANALYZE) from the query
context.
+ *
* @return query statement type {@link SqlStatementType}, if known.
*/
- public SqlStatementType getSQLStatementType();
+ SqlStatementType getSQLStatementType();
/**
* Get this node's identity.
@@ -145,12 +148,12 @@ public interface FragmentContext extends UdfUtilities,
AutoCloseable {
/**
* @return ID {@link java.util.UUID} of the current query
*/
- public QueryId getQueryId();
+ QueryId getQueryId();
/**
* @return The string representation of the ID {@link java.util.UUID} of the
current query
*/
- public String getQueryIdString();
+ String getQueryIdString();
OperatorContext newOperatorContext(PhysicalOperator popConfig);
@@ -171,22 +174,32 @@ public interface FragmentContext extends UdfUtilities,
AutoCloseable {
@Override
void close();
+
/**
- * add a RuntimeFilter when the RuntimeFilter receiver belongs to the same
MinorFragment
- * @param runtimeFilter
+ * Add a RuntimeFilter when the RuntimeFilter receiver belongs to the same
MinorFragment.
+ *
+ * @param runtimeFilter runtime filter
*/
- public void addRuntimeFilter(RuntimeFilterWritable runtimeFilter);
+ void addRuntimeFilter(RuntimeFilterWritable runtimeFilter);
- public RuntimeFilterWritable getRuntimeFilter(long rfIdentifier);
+ RuntimeFilterWritable getRuntimeFilter(long rfIdentifier);
/**
- * get the RuntimeFil
[drill] branch master updated: DRILL-7168: Implement ALTER SCHEMA ADD / REMOVE commands
This is an automated email from the ASF dual-hosted git repository.
arina pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git
The following commit(s) were added to refs/heads/master by this push:
new d4a2aef DRILL-7168: Implement ALTER SCHEMA ADD / REMOVE commands
d4a2aef is described below
commit d4a2aefd530dae6bdb0a3f115bd8ef8643dbc364
Author: Arina Ielchiieva
AuthorDate: Thu Aug 29 16:15:43 2019 +0300
DRILL-7168: Implement ALTER SCHEMA ADD / REMOVE commands
---
exec/java-exec/src/main/codegen/data/Parser.tdd| 9 +-
.../src/main/codegen/includes/parserImpls.ftl | 121 +-
.../exec/planner/sql/handlers/SchemaHandler.java | 182 -
.../exec/planner/sql/handlers/SqlHandlerUtil.java | 22 ++
.../sql/parser/CompoundIdentifierConverter.java| 2 +
.../drill/exec/planner/sql/parser/SqlSchema.java | 228 +--
.../metadata/schema/InlineSchemaProvider.java | 4 +-
.../record/metadata/schema/PathSchemaProvider.java | 7 +-
.../record/metadata/schema/SchemaProvider.java | 8 +-
.../record/metadata/schema/StorageProperties.java | 71
.../java/org/apache/drill/TestSchemaCommands.java | 415 -
.../exec/record/metadata/TestTupleSchema.java | 64
.../record/metadata/schema/TestSchemaProvider.java | 23 +-
.../record/metadata/AbstractColumnMetadata.java| 1 +
.../exec/record/metadata/AbstractPropertied.java | 5 +
.../drill/exec/record/metadata/Propertied.java | 8 +-
.../drill/exec/record/metadata/TupleSchema.java| 1 +
17 files changed, 1098 insertions(+), 73 deletions(-)
diff --git a/exec/java-exec/src/main/codegen/data/Parser.tdd
b/exec/java-exec/src/main/codegen/data/Parser.tdd
index 2da27c3..3480754 100644
--- a/exec/java-exec/src/main/codegen/data/Parser.tdd
+++ b/exec/java-exec/src/main/codegen/data/Parser.tdd
@@ -43,7 +43,8 @@
"ESTIMATE",
"STATISTICS",
"SAMPLE",
-"COLUMNS"
+"COLUMNS",
+"REMOVE"
]
# List of methods for parsing custom SQL statements.
@@ -61,7 +62,8 @@
"SqlDropFunction()",
"SqlAnalyzeTable()",
"DrillSqlSetOption(Span.of(), null)",
-"DrillSqlResetOption(Span.of(), null)"
+"DrillSqlResetOption(Span.of(), null)",
+"SqlAlterSchema()"
]
# List of methods for parsing custom literals.
@@ -859,7 +861,8 @@
"YEAR",
# "YEARS", # not a keyword in Calcite
"ZONE",
-"COLUMNS"
+"COLUMNS",
+"REMOVE"
]
# List of additional join types. Each is a method with no arguments.
diff --git a/exec/java-exec/src/main/codegen/includes/parserImpls.ftl
b/exec/java-exec/src/main/codegen/includes/parserImpls.ftl
index be30318..70e0c28 100644
--- a/exec/java-exec/src/main/codegen/includes/parserImpls.ftl
+++ b/exec/java-exec/src/main/codegen/includes/parserImpls.ftl
@@ -314,12 +314,12 @@ SqlNode SqlCreateSchema(SqlParserPos pos, String
createType) :
token_source.SwitchTo(SCH);
}
(
-
+
{
load = StringLiteral();
}
|
-
+
{
schema = SqlLiteral.createCharString(token.image, getPos());
}
@@ -373,13 +373,13 @@ void addProperty(SqlNodeList properties) :
}
TOKEN : {
-< LOAD: "LOAD" > { popState(); }
- | < NUM: (" " | "\t" | "\n" | "\r")* >
+< SCH_LOAD: "LOAD" > { popState(); }
+ | < SCH_NUM: (" " | "\t" | "\n" | "\r")* >
// once schema is found, switch back to initial lexical state
// must be enclosed in the parentheses
// inside may have left parenthesis only if number precedes (covers cases
with varchar(10)),
// if left parenthesis is present in column name, it must be escaped with
backslash
- | < PAREN_STRING: ((~[")"]) | ( ")") | ("\\)"))* > {
popState(); }
+ | < SCH_PAREN_STRING: ((~[")"]) | ( ")") | ("\\)"))*
> { popState(); }
}
/**
@@ -465,7 +465,7 @@ SqlNode SqlDropSchema(SqlParserPos pos) :
/**
* Parse refresh table metadata statement.
- * REFRESH TABLE METADATA [COLUMNS ((field1, field2,..) | NONE)] tblname
+ * REFRESH TABLE METADATA [COLUMNS ((field1, field2,..) | NONE)] table_name
*/
SqlNode SqlRefreshMetadata() :
{
@@ -535,6 +535,110 @@ SqlNode SqlDescribeSchema() :
}
/**
+* Parses ALTER SCHEMA statements:
+*
+* ALTER SCHEMA
+* (FOR TABLE dfs.tmp.nation | PATH '/tmp/schema.json')
+* ADD [OR REPLACE]
+* [COLUMNS (col1 int, col2 varchar)]
+* [PROPERTIES ('prop1'='val1', 'prop2'='val2')]
+*
+* ALTER SCHEMA
+* (FOR TABLE dfs.tmp.nation | PATH '/tmp/schema.json')
+* REMOVE
+
[drill] branch master updated (31a4199 -> 236276c)
This is an automated email from the ASF dual-hosted git repository. arina pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/drill.git. from 31a4199 DRILL-7353: Wrong driver class is written to the java.sql.Driver add 236276c DRILL-7360: Refactor WatchService in Drillbit class and fix concurrency issues No new revisions were added by this update. Summary of changes: .../org/apache/drill/exec/server/Drillbit.java | 138 ++--- 1 file changed, 93 insertions(+), 45 deletions(-)
[drill] 01/06: DRILL-7156: Support empty Parquet files creation
This is an automated email from the ASF dual-hosted git repository.
arina pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git
commit ef93515f88caf232f00fd6a4b37a6751592bea8a
Author: Oleg Zinoviev
AuthorDate: Sun Jun 16 21:21:46 2019 +0300
DRILL-7156: Support empty Parquet files creation
closes #1836
---
.../exec/store/parquet/ParquetRecordWriter.java| 95 +---
.../impl/writer/TestParquetWriterEmptyFiles.java | 123 +
2 files changed, 159 insertions(+), 59 deletions(-)
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetRecordWriter.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetRecordWriter.java
index 5a64f40..a9f7f14 100644
---
a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetRecordWriter.java
+++
b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetRecordWriter.java
@@ -122,6 +122,9 @@ public class ParquetRecordWriter extends
ParquetOutputRecordWriter {
private PrimitiveTypeName logicalTypeForDecimals;
private boolean usePrimitiveTypesForDecimals;
+ /** Is used to ensure that empty Parquet file will be written if no rows
were provided. */
+ private boolean empty = true;
+
public ParquetRecordWriter(FragmentContext context, ParquetWriter writer)
throws OutOfMemoryException {
this.oContext = context.newOperatorContext(writer);
this.codecFactory =
CodecFactory.createDirectCodecFactory(writer.getFormatPlugin().getFsConf(),
@@ -205,7 +208,7 @@ public class ParquetRecordWriter extends
ParquetOutputRecordWriter {
public void updateSchema(VectorAccessible batch) throws IOException {
if (this.batchSchema == null ||
!this.batchSchema.equals(batch.getSchema()) ||
containsComplexVectors(this.batchSchema)) {
if (this.batchSchema != null) {
-flush();
+flush(false);
}
this.batchSchema = batch.getSchema();
newSchema();
@@ -310,7 +313,7 @@ public class ParquetRecordWriter extends
ParquetOutputRecordWriter {
try {
boolean newPartition = newPartition(index);
if (newPartition) {
-flush();
+flush(false);
newSchema();
}
} catch (Exception e) {
@@ -318,19 +321,18 @@ public class ParquetRecordWriter extends
ParquetOutputRecordWriter {
}
}
- private void flush() throws IOException {
+ private void flush(boolean cleanUp) throws IOException {
try {
if (recordCount > 0) {
-parquetFileWriter.startBlock(recordCount);
-consumer.flush();
-store.flush();
-pageStore.flushToFileWriter(parquetFileWriter);
-recordCount = 0;
-parquetFileWriter.endBlock();
-
-// we are writing one single block per file
-parquetFileWriter.end(extraMetaData);
-parquetFileWriter = null;
+flushParquetFileWriter();
+ } else if (cleanUp && empty && schema != null && schema.getFieldCount()
> 0) {
+// Write empty parquet if:
+// 1) This is a cleanup - no any additional records can be written
+// 2) No file was written until this moment
+// 3) Schema is set
+// 4) Schema is not empty
+createParquetFileWriter();
+flushParquetFileWriter();
}
} finally {
store.close();
@@ -347,7 +349,7 @@ public class ParquetRecordWriter extends
ParquetOutputRecordWriter {
long memSize = store.getBufferedSize();
if (memSize > blockSize) {
logger.debug("Reached block size " + blockSize);
-flush();
+flush(false);
newSchema();
recordCountForNextMemCheck = min(max(MINIMUM_RECORD_COUNT_FOR_CHECK,
recordCount / 2), MAXIMUM_RECORD_COUNT_FOR_CHECK);
} else {
@@ -435,29 +437,10 @@ public class ParquetRecordWriter extends
ParquetOutputRecordWriter {
// we wait until there is at least one record before creating the parquet
file
if (parquetFileWriter == null) {
- Path path = new Path(location, prefix + "_" + index + ".parquet");
- // to ensure that our writer was the first to create output file, we
create empty file first and fail if file exists
- Path firstCreatedPath = storageStrategy.createFileAndApply(fs, path);
-
- // since parquet reader supports partitions, it means that several
output files may be created
- // if this writer was the one to create table folder, we store only
folder and delete it with its content in case of abort
- // if table location was created before, we store only files created by
this writer and delete them in case of abort
- addCleanUpLocation(fs, firstCreatedPath);
-
- // since ParquetFileWriter will overwrite empty output file (append is
not supported)
- // we need to re-apply file permission
- if (useSingleFSBlock) {
-// Passing blo
[drill] 04/06: DRILL-7339: Iceberg commit upgrade and Metastore tests categorization
This is an automated email from the ASF dual-hosted git repository.
arina pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git
commit 305f040ae5c4a3851bf139e9ed4b4efb70b09dd8
Author: Arina Ielchiieva
AuthorDate: Wed Aug 14 19:17:46 2019 +0300
DRILL-7339: Iceberg commit upgrade and Metastore tests categorization
1. Upgraded Iceberg commit to fix issue with deletes in transaction
2. Categorize Metastore tests
closes #1842
---
.../org/apache/drill/categories/MetastoreTest.java | 24 ++
metastore/iceberg-metastore/pom.xml| 2 +-
.../drill/metastore/iceberg/IcebergBaseTest.java | 3 +++
.../components/tables/TestBasicTablesRequests.java | 3 +++
.../tables/TestBasicTablesTransformer.java | 3 +++
.../components/tables/TestMetastoreTableInfo.java | 3 +++
.../tables/TestTableMetadataUnitConversion.java| 3 +++
.../metastore/metadata/MetadataSerDeTest.java | 3 +++
8 files changed, 43 insertions(+), 1 deletion(-)
diff --git
a/common/src/test/java/org/apache/drill/categories/MetastoreTest.java
b/common/src/test/java/org/apache/drill/categories/MetastoreTest.java
new file mode 100644
index 000..9da1e03
--- /dev/null
+++ b/common/src/test/java/org/apache/drill/categories/MetastoreTest.java
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.categories;
+
+/**
+ * This is a category used to mark unit tests that test the Drill Metastore
and its components.
+ */
+public interface MetastoreTest {
+}
diff --git a/metastore/iceberg-metastore/pom.xml
b/metastore/iceberg-metastore/pom.xml
index d935750..41d4690 100644
--- a/metastore/iceberg-metastore/pom.xml
+++ b/metastore/iceberg-metastore/pom.xml
@@ -33,7 +33,7 @@
metastore/Drill Iceberg Metastore
-08e0873
+1b0b9c2
2.7.0
diff --git
a/metastore/iceberg-metastore/src/test/java/org/apache/drill/metastore/iceberg/IcebergBaseTest.java
b/metastore/iceberg-metastore/src/test/java/org/apache/drill/metastore/iceberg/IcebergBaseTest.java
index 0250ce9..5ef7b80 100644
---
a/metastore/iceberg-metastore/src/test/java/org/apache/drill/metastore/iceberg/IcebergBaseTest.java
+++
b/metastore/iceberg-metastore/src/test/java/org/apache/drill/metastore/iceberg/IcebergBaseTest.java
@@ -19,6 +19,7 @@ package org.apache.drill.metastore.iceberg;
import com.typesafe.config.Config;
import com.typesafe.config.ConfigValueFactory;
+import org.apache.drill.categories.MetastoreTest;
import org.apache.drill.common.config.DrillConfig;
import org.apache.drill.common.util.GuavaPatcher;
import org.apache.drill.metastore.iceberg.config.IcebergConfigConstants;
@@ -28,11 +29,13 @@ import org.apache.hadoop.fs.Path;
import org.junit.BeforeClass;
import org.junit.ClassRule;
import org.junit.Rule;
+import org.junit.experimental.categories.Category;
import org.junit.rules.ExpectedException;
import org.junit.rules.TemporaryFolder;
import java.io.File;
+@Category(MetastoreTest.class)
public abstract class IcebergBaseTest {
@ClassRule
diff --git
a/metastore/metastore-api/src/test/java/org/apache/drill/metastore/components/tables/TestBasicTablesRequests.java
b/metastore/metastore-api/src/test/java/org/apache/drill/metastore/components/tables/TestBasicTablesRequests.java
index 4a8baf7..9c6c45c 100644
---
a/metastore/metastore-api/src/test/java/org/apache/drill/metastore/components/tables/TestBasicTablesRequests.java
+++
b/metastore/metastore-api/src/test/java/org/apache/drill/metastore/components/tables/TestBasicTablesRequests.java
@@ -17,8 +17,10 @@
*/
package org.apache.drill.metastore.components.tables;
+import org.apache.drill.categories.MetastoreTest;
import org.apache.drill.metastore.expressions.FilterExpression;
import org.junit.Test;
+import org.junit.experimental.categories.Category;
import java.util.Arrays;
import java.util.Collections;
@@ -28,6 +30,7 @@ import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
+@Category(MetastoreTest.class)
public class TestBasicTablesRequests {
@Test
diff --git
a
[drill] 03/06: DRILL-7326: Support repeated lists for CTAS parquet format
This is an automated email from the ASF dual-hosted git repository.
arina pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git
commit ffab527451e0a23eca96f38bce52c790553cc47e
Author: Igor Guzenko
AuthorDate: Mon Aug 19 20:02:51 2019 +0300
DRILL-7326: Support repeated lists for CTAS parquet format
closes #1844
---
.../codegen/templates/EventBasedRecordWriter.java | 13 ++
.../templates/ParquetOutputRecordWriter.java | 107 +++-
.../exec/store/parquet/ParquetRecordWriter.java| 136 -
.../physical/impl/writer/TestParquetWriter.java| 108
.../resources/jsoninput/repeated_list_of_maps.json | 2 +
5 files changed, 333 insertions(+), 33 deletions(-)
diff --git
a/exec/java-exec/src/main/codegen/templates/EventBasedRecordWriter.java
b/exec/java-exec/src/main/codegen/templates/EventBasedRecordWriter.java
index 7357243..d87eeb3 100644
--- a/exec/java-exec/src/main/codegen/templates/EventBasedRecordWriter.java
+++ b/exec/java-exec/src/main/codegen/templates/EventBasedRecordWriter.java
@@ -118,6 +118,19 @@ public class EventBasedRecordWriter {
}
public abstract void writeField() throws IOException;
+
+/**
+ * Used by repeated converters for writing Parquet logical lists.
+ *
+ * @throws IOException may be thrown by subsequent invocation of {{@link
#writeField()}}
+ * in overriden methods
+ * @see https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#lists;>Lists
+ */
+public void writeListField() throws IOException {
+ throw new UnsupportedOperationException(String.format(
+ "Converter '%s' doesn't support writing list fields.",
+ getClass().getSimpleName()));
+}
}
public static FieldConverter getConverter(RecordWriter recordWriter, int
fieldId, String fieldName, FieldReader reader) {
diff --git
a/exec/java-exec/src/main/codegen/templates/ParquetOutputRecordWriter.java
b/exec/java-exec/src/main/codegen/templates/ParquetOutputRecordWriter.java
index ff80701..1da206d 100644
--- a/exec/java-exec/src/main/codegen/templates/ParquetOutputRecordWriter.java
+++ b/exec/java-exec/src/main/codegen/templates/ParquetOutputRecordWriter.java
@@ -71,6 +71,10 @@ import java.util.Map;
*/
public abstract class ParquetOutputRecordWriter extends AbstractRecordWriter
implements RecordWriter {
+ protected static final String LIST = "list";
+ protected static final String ELEMENT = "element";
+ protected static final int ZERO_IDX = 0;
+
private RecordConsumer consumer;
private MessageType schema;
@@ -206,9 +210,9 @@ public abstract class ParquetOutputRecordWriter extends
AbstractRecordWriter imp
holder.buffer, holder.start,
${minor.class}Holder.nDecimalDigits,
holder.scale).unscaledValue().toByteArray();
byte[] output = new
byte[ParquetTypeHelper.getLengthForMinorType(MinorType.${minor.class?upper_case})];
if (holder.getSign(holder.start, holder.buffer)) {
-Arrays.fill(output, 0, output.length - bytes.length, (byte)0xFF);
+Arrays.fill(output, 0, output.length - bytes.length, (byte) -1);
} else {
-Arrays.fill(output, 0, output.length - bytes.length, (byte)0x0);
+Arrays.fill(output, 0, output.length - bytes.length, (byte) 0);
}
System.arraycopy(bytes, 0, output, output.length - bytes.length,
bytes.length);
consumer.addBinary(Binary.fromByteArray(output));
@@ -268,10 +272,109 @@ public abstract class ParquetOutputRecordWriter extends
AbstractRecordWriter imp
consumer.endField(fieldName, fieldId);
}
+
+<#if mode.prefix == "Repeated">
+ @Override
+ public void writeListField() {
+ if (reader.size() == 0) {
+return;
+ }
+ consumer.startField(LIST, ZERO_IDX);
+ for (int i = 0; i < reader.size(); i++) {
+consumer.startGroup();
+consumer.startField(ELEMENT, ZERO_IDX);
+
+ <#if minor.class == "TinyInt" ||
+ minor.class == "UInt1" ||
+ minor.class == "UInt2" ||
+ minor.class == "SmallInt" ||
+ minor.class == "Int" ||
+ minor.class == "Time" ||
+ minor.class == "Decimal9" ||
+ minor.class == "UInt4">
+reader.read(i, holder);
+consumer.addInteger(holder.value);
+ <#elseif minor.class == "Float4">
+reader.read(i, holder);
+consumer.addFloat(holder.value);
+ <#elseif minor.class == "BigInt" ||
+minor.class == "Decimal18" ||
+minor.class == "TimeStamp" ||
+minor.class == "UInt8">
+reader.read(i, holder);
+consumer.addLong(holder.value);
+ <#elseif minor.class == "Date">
+reader.read(i, holde
[drill] 02/06: DRILL-7356: Introduce session options for the Drill Metastore
This is an automated email from the ASF dual-hosted git repository.
arina pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git
commit b44b7127306f68c250beafb5a4e7980303fd5cf1
Author: Volodymyr Vysotskyi
AuthorDate: Thu Aug 22 19:01:15 2019 +0300
DRILL-7356: Introduce session options for the Drill Metastore
closes #1846
---
.../java/org/apache/drill/exec/ExecConstants.java | 63 ++
.../exec/server/options/SystemOptionManager.java | 9 +++-
.../java-exec/src/main/resources/drill-module.conf | 10 +++-
3 files changed, 80 insertions(+), 2 deletions(-)
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java
index 6f3f17d..549d374 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java
@@ -1044,4 +1044,67 @@ public final class ExecConstants {
public static final RangeLongValidator QUERY_MAX_ROWS_VALIDATOR = new
RangeLongValidator(QUERY_MAX_ROWS, 0, Integer.MAX_VALUE,
new OptionDescription("The maximum number of rows that the query will
return. This can be only set at a SYSTEM level by an admin. (Drill 1.16+)"));
+ /**
+ * Option that enables Drill Metastore usage.
+ */
+ public static final String METASTORE_ENABLED = "metastore.enabled";
+ public static final BooleanValidator METASTORE_ENABLED_VALIDATOR = new
BooleanValidator(METASTORE_ENABLED,
+ new OptionDescription("Enables Drill Metastore usage to be able to store
table metadata " +
+ "during ANALYZE TABLE commands execution and to be able to read
table metadata during regular " +
+ "queries execution or when querying some INFORMATION_SCHEMA tables.
" +
+ "This option is not active for now. Default is false. (Drill
1.17+)"));
+
+ /**
+ * Option for specifying maximum level depth for collecting metadata
+ * which will be stored in the Drill Metastore. For example, when {@code
FILE} level value
+ * is set, {@code ROW_GROUP} level metadata won't be collected and stored
into the Metastore.
+ */
+ public static final String METASTORE_METADATA_STORE_DEPTH_LEVEL =
"metastore.metadata.store.depth_level";
+ public static final EnumeratedStringValidator
METASTORE_METADATA_STORE_DEPTH_LEVEL_VALIDATOR = new
EnumeratedStringValidator(METASTORE_METADATA_STORE_DEPTH_LEVEL,
+ new OptionDescription("Specifies maximum level depth for collecting
metadata. " +
+ "This option is not active for now. Default is 'ROW_GROUP'. (Drill
1.17+)"),
+ "TABLE", "SEGMENT", "PARTITION", "FILE", "ROW_GROUP");
+
+ /**
+ * Option for enabling schema usage, stored to the Metastore.
+ */
+ public static final String METASTORE_USE_SCHEMA_METADATA =
"metastore.metadata.use_schema";
+ public static final BooleanValidator METASTORE_USE_SCHEMA_METADATA_VALIDATOR
= new BooleanValidator(METASTORE_USE_SCHEMA_METADATA,
+ new OptionDescription("Enables schema usage, stored to the Metastore. " +
+ "This option is not active for now. Default is false. (Drill
1.17+)"));
+
+ /**
+ * Option for enabling statistics usage, stored in the Metastore, at the
planning stage.
+ */
+ public static final String METASTORE_USE_STATISTICS_METADATA =
"metastore.metadata.use_statistics";
+ public static final BooleanValidator
METASTORE_USE_STATISTICS_METADATA_VALIDATOR = new
BooleanValidator(METASTORE_USE_STATISTICS_METADATA,
+ new OptionDescription("Enables statistics usage, stored in the
Metastore, at the planning stage. " +
+ "This option is not active for now. Default is false. (Drill
1.17+)"));
+
+ /**
+ * Option for collecting schema and / or column statistics for every table
after CTAS and CTTAS execution.
+ */
+ public static final String METASTORE_CTAS_AUTO_COLLECT_METADATA =
"metastore.metadata.ctas.auto-collect";
+ public static final EnumeratedStringValidator
METASTORE_CTAS_AUTO_COLLECT_METADATA_VALIDATOR = new
EnumeratedStringValidator(METASTORE_CTAS_AUTO_COLLECT_METADATA,
+ new OptionDescription("Specifies whether schema and / or column
statistics will be " +
+ "automatically collected for every table after CTAS and CTTAS. " +
+ "This option is not active for now. Default is 'NONE'. (Drill
1.17+)"),
+ "NONE", "ALL", "SCHEMA");
+
+ /**
+ * Option for allowing using file metadata cache if required metadata is
absent in the Metastore.
+ */
+ public static final String METASTORE_FALLBACK_TO_FILE_METADATA =
"metastore.metadata.fallback_to_file_metadata&q
[drill] 06/06: DRILL-7353: Wrong driver class is written to the java.sql.Driver
This is an automated email from the ASF dual-hosted git repository.
arina pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git
commit 31a41995c3f708894cc77bad3b27ce72203c423c
Author: Anton Gozhiy
AuthorDate: Mon Aug 19 20:33:14 2019 +0300
DRILL-7353: Wrong driver class is written to the java.sql.Driver
closes #1845
---
exec/jdbc-all/pom.xml | 16 +
.../org/apache/drill/jdbc/ITTestShadedJar.java | 28 ++
2 files changed, 39 insertions(+), 5 deletions(-)
diff --git a/exec/jdbc-all/pom.xml b/exec/jdbc-all/pom.xml
index 2dab193..13234fd 100644
--- a/exec/jdbc-all/pom.xml
+++ b/exec/jdbc-all/pom.xml
@@ -496,6 +496,14 @@
webapps/**
+
+
+ org.apache.calcite.avatica:*
+
+ META-INF/services/java.sql.Driver
+
+
@@ -799,6 +807,14 @@
webapps/**
+
+
+org.apache.calcite.avatica:*
+
+ META-INF/services/java.sql.Driver
+
+
diff --git
a/exec/jdbc-all/src/test/java/org/apache/drill/jdbc/ITTestShadedJar.java
b/exec/jdbc-all/src/test/java/org/apache/drill/jdbc/ITTestShadedJar.java
index 4fed146..c343037 100644
--- a/exec/jdbc-all/src/test/java/org/apache/drill/jdbc/ITTestShadedJar.java
+++ b/exec/jdbc-all/src/test/java/org/apache/drill/jdbc/ITTestShadedJar.java
@@ -17,9 +17,16 @@
*/
package org.apache.drill.jdbc;
-import static org.junit.Assert.assertEquals;
+import org.junit.ClassRule;
+import org.junit.Test;
+import org.junit.rules.TestWatcher;
+import org.junit.runner.Description;
+import java.io.BufferedReader;
import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
import java.lang.reflect.Field;
import java.lang.reflect.Method;
import java.net.MalformedURLException;
@@ -32,11 +39,10 @@ import java.sql.SQLException;
import java.sql.Statement;
import java.util.Vector;
import java.util.concurrent.Semaphore;
+import java.util.stream.Collectors;
-import org.junit.ClassRule;
-import org.junit.Test;
-import org.junit.rules.TestWatcher;
-import org.junit.runner.Description;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
public class ITTestShadedJar {
private static final org.slf4j.Logger logger =
org.slf4j.LoggerFactory.getLogger(ITTestShadedJar.class);
@@ -166,6 +172,18 @@ public class ITTestShadedJar {
}
+ @Test
+ public void serviceFileContainsCorrectDriver() throws IOException {
+URLClassLoader loader = URLClassLoader.newInstance(new
URL[]{getJdbcUrl()});
+try (InputStream resourceStream =
loader.getResourceAsStream("META-INF/services/java.sql.Driver")) {
+ assertNotNull("java.sql.Driver is not present in the jdbc jar",
resourceStream);
+ try (BufferedReader reader = new BufferedReader(new
InputStreamReader(resourceStream))) {
+String driverClass =
reader.lines().collect(Collectors.joining(System.lineSeparator()));
+assertEquals("org.apache.drill.jdbc.Driver", driverClass);
+ }
+}
+ }
+
private static void printQuery(Connection c, String query) throws
SQLException {
final StringBuilder sb = new StringBuilder();
[drill] 05/06: DRILL-7222: Visualize estimated and actual row counts for a query
This is an automated email from the ASF dual-hosted git repository.
arina pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git
commit b9a61b04b622ff4be0fa87091d3c879148d9b1ed
Author: Kunal Khatua
AuthorDate: Thu Aug 22 10:00:58 2019 -0700
DRILL-7222: Visualize estimated and actual row counts for a query
With statistics in place, it is useful to have the estimated rowcount along
side the actual rowcount query profile's operator overview. A toggle button
allows this with the estimated rows hidden by default
We can extract this from the Physical Plan section of the profile.
Added a toggle-ready table-column header
closes #1779
---
.../java/org/apache/drill/exec/ExecConstants.java | 2 +
.../exec/server/rest/profile/HtmlAttribute.java| 2 +
.../exec/server/rest/profile/OperatorWrapper.java | 8 ++-
.../exec/server/rest/profile/ProfileWrapper.java | 6 ++
.../java-exec/src/main/resources/drill-module.conf | 1 +
.../src/main/resources/rest/profile/profile.ftl| 64 +++---
6 files changed, 74 insertions(+), 9 deletions(-)
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java
index 549d374..463b0a0 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java
@@ -875,6 +875,8 @@ public final class ExecConstants {
public static final BooleanValidator DYNAMIC_UDF_SUPPORT_ENABLED_VALIDATOR =
new BooleanValidator(DYNAMIC_UDF_SUPPORT_ENABLED,
new OptionDescription("Enables users to dynamically upload UDFs. Users
must upload their UDF (source and binary) JAR files to a staging directory in
the distributed file system before issuing the CREATE FUNCTION USING JAR
command to register a UDF. Default is true. (Drill 1.9+)"));
+ //Display estimated rows in operator overview by default
+ public static final String PROFILE_STATISTICS_ESTIMATED_ROWS_SHOW =
"drill.exec.http.profile.statistics.estimated_rows.show";
//Trigger warning in UX if fragments appear to be doing no work (units are
in seconds).
public static final String PROFILE_WARNING_PROGRESS_THRESHOLD =
"drill.exec.http.profile.warning.progress.threshold";
//Trigger warning in UX if slowest fragment operator crosses min threshold
and exceeds ratio with average (units are in seconds).
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/profile/HtmlAttribute.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/profile/HtmlAttribute.java
index 75db298..95c0507 100644
---
a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/profile/HtmlAttribute.java
+++
b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/profile/HtmlAttribute.java
@@ -23,6 +23,7 @@ package org.apache.drill.exec.server.rest.profile;
public class HtmlAttribute {
//Attributes
public static final String CLASS = "class";
+ public static final String KEY = "key";
public static final String DATA_ORDER = "data-order";
public static final String TITLE = "title";
public static final String SPILLS = "spills";
@@ -33,5 +34,6 @@ public class HtmlAttribute {
public static final String CLASS_VALUE_NO_PROGRESS_TAG = "no-progress-tag";
public static final String CLASS_VALUE_TIME_SKEW_TAG = "time-skew-tag";
public static final String CLASS_VALUE_SCAN_WAIT_TAG = "scan-wait-tag";
+ public static final String CLASS_VALUE_EST_ROWS_ANCHOR = "estRowsAnchor";
public static final String STYLE_VALUE_CURSOR_HELP = "cursor:help;";
}
\ No newline at end of file
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/profile/OperatorWrapper.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/profile/OperatorWrapper.java
index 0f61170..2e593b6 100644
---
a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/profile/OperatorWrapper.java
+++
b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/profile/OperatorWrapper.java
@@ -143,7 +143,7 @@ public class OperatorWrapper {
OverviewTblTxt.AVG_SETUP_TIME, OverviewTblTxt.MAX_SETUP_TIME,
OverviewTblTxt.AVG_PROCESS_TIME, OverviewTblTxt.MAX_PROCESS_TIME,
OverviewTblTxt.MIN_WAIT_TIME, OverviewTblTxt.AVG_WAIT_TIME,
OverviewTblTxt.MAX_WAIT_TIME,
- OverviewTblTxt.PERCENT_FRAGMENT_TIME, OverviewTblTxt.PERCENT_QUERY_TIME,
OverviewTblTxt.ROWS,
+ OverviewTblTxt.PERCENT_FRAGMENT_TIME, OverviewTblTxt.PERCENT_QUERY_TIME,
OverviewTblTxt.ROWS.concat(OverviewTblTxt.ESTIMATED_ROWS),
OverviewTblTxt.AVG_PEAK_MEMORY, OverviewTblTxt.MAX_PEAK_MEMORY
};
@@ -269,7 +269,10 @@ public class OperatorWrapper {
tb.appendPercent(process
[drill] branch master updated (9c62bf1 -> 31a4199)
This is an automated email from the ASF dual-hosted git repository.
arina pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git.
from 9c62bf1 DRILL-7350: Move RowSet related classes from test folder
new ef93515 DRILL-7156: Support empty Parquet files creation
new b44b712 DRILL-7356: Introduce session options for the Drill Metastore
new ffab527 DRILL-7326: Support repeated lists for CTAS parquet format
new 305f040 DRILL-7339: Iceberg commit upgrade and Metastore tests
categorization
new b9a61b0 DRILL-7222: Visualize estimated and actual row counts for a
query
new 31a4199 DRILL-7353: Wrong driver class is written to the
java.sql.Driver
The 6 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../{JdbcTest.java => MetastoreTest.java} | 4 +-
.../codegen/templates/EventBasedRecordWriter.java | 13 ++
.../templates/ParquetOutputRecordWriter.java | 107 +-
.../java/org/apache/drill/exec/ExecConstants.java | 65 ++
.../exec/server/options/SystemOptionManager.java | 9 +-
.../exec/server/rest/profile/HtmlAttribute.java| 2 +
.../exec/server/rest/profile/OperatorWrapper.java | 8 +-
.../exec/server/rest/profile/ProfileWrapper.java | 6 +
.../exec/store/parquet/ParquetRecordWriter.java| 231 +
.../java-exec/src/main/resources/drill-module.conf | 11 +-
.../src/main/resources/rest/profile/profile.ftl| 64 +-
.../physical/impl/writer/TestParquetWriter.java| 108 +++---
.../impl/writer/TestParquetWriterEmptyFiles.java | 123 +--
.../resources/jsoninput/repeated_list_of_maps.json | 2 +
exec/jdbc-all/pom.xml | 16 ++
.../org/apache/drill/jdbc/ITTestShadedJar.java | 28 ++-
metastore/iceberg-metastore/pom.xml| 2 +-
.../drill/metastore/iceberg/IcebergBaseTest.java | 3 +
.../components/tables/TestBasicTablesRequests.java | 3 +
.../tables/TestBasicTablesTransformer.java | 3 +
.../components/tables/TestMetastoreTableInfo.java | 3 +
.../tables/TestTableMetadataUnitConversion.java| 3 +
.../metastore/metadata/MetadataSerDeTest.java | 3 +
23 files changed, 706 insertions(+), 111 deletions(-)
copy common/src/test/java/org/apache/drill/categories/{JdbcTest.java =>
MetastoreTest.java} (93%)
create mode 100644
exec/java-exec/src/test/resources/jsoninput/repeated_list_of_maps.json
[drill] branch master updated: DRILL-7307: casthigh for decimal type can lead to the issues with VarDecimalHolder
This is an automated email from the ASF dual-hosted git repository.
arina pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git
The following commit(s) were added to refs/heads/master by this push:
new ddcb578 DRILL-7307: casthigh for decimal type can lead to the issues
with VarDecimalHolder
ddcb578 is described below
commit ddcb57865f388f2588df35bcad3fd52e69d22db4
Author: Dmytriy Grinchenko
AuthorDate: Wed Jul 3 16:11:42 2019 +0300
DRILL-7307: casthigh for decimal type can lead to the issues with
VarDecimalHolder
- Fixed code-gen for VarDecimal type
- Fixed code-gen issue with nullable holders for simple cast functions
with passed constants as arguments.
- Code-gen now honnoring DataType.Optional type defined by UDF for
NULL-IF-NULL functions.
---
.../apache/drill/exec/expr/fn/HiveFuncHolder.java | 2 +-
.../src/main/codegen/templates/CastHigh.java | 10 +-
.../apache/drill/exec/expr/EvaluationVisitor.java | 276 +
.../drill/exec/expr/fn/AbstractFuncHolder.java | 4 +-
.../drill/exec/expr/fn/DrillAggFuncHolder.java | 3 +-
.../expr/fn/DrillComplexWriterAggFuncHolder.java | 3 +-
.../exec/expr/fn/DrillComplexWriterFuncHolder.java | 4 +-
.../drill/exec/expr/fn/DrillSimpleFuncHolder.java | 11 +-
.../expr/fn/output/DecimalReturnTypeInference.java | 9 +-
.../drill/exec/sql/TestSimpleCastFunctions.java| 34 +++
10 files changed, 123 insertions(+), 233 deletions(-)
diff --git
a/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/expr/fn/HiveFuncHolder.java
b/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/expr/fn/HiveFuncHolder.java
index 80f299e..7ac1460 100644
---
a/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/expr/fn/HiveFuncHolder.java
+++
b/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/expr/fn/HiveFuncHolder.java
@@ -145,7 +145,7 @@ public class HiveFuncHolder extends AbstractFuncHolder {
@Override
public HoldingContainer renderEnd(ClassGenerator classGenerator,
HoldingContainer[] inputVariables,
-JVar[] workspaceJVars, FieldReference
fieldReference) {
+JVar[] workspaceJVars,
FunctionHolderExpression holderExpr) {
generateSetup(classGenerator, workspaceJVars);
return generateEval(classGenerator, inputVariables, workspaceJVars);
}
diff --git a/exec/java-exec/src/main/codegen/templates/CastHigh.java
b/exec/java-exec/src/main/codegen/templates/CastHigh.java
index 1a876c8..08f0efe 100644
--- a/exec/java-exec/src/main/codegen/templates/CastHigh.java
+++ b/exec/java-exec/src/main/codegen/templates/CastHigh.java
@@ -60,11 +60,17 @@ public class CastHighFunctions {
public void setup() {}
public void eval() {
- <#if type.value>
+<#if type.from.contains("VarDecimal")>
+ out.buffer = (DrillBuf) in.buffer;
+ out.start = (int) in.start;
+ out.scale = (int) in.scale;
+ out.precision = (int) in.precision;
+ out.end = (int) in.end;
+<#elseif type.value>
out.value = (double) in.value;
<#else>
out = in;
-
+
}
}
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/EvaluationVisitor.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/EvaluationVisitor.java
index 54644bf..1f8a779 100644
---
a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/EvaluationVisitor.java
+++
b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/EvaluationVisitor.java
@@ -65,6 +65,7 @@ import
org.apache.drill.exec.compile.sig.ConstantExpressionIdentifier;
import org.apache.drill.exec.compile.sig.GeneratorMapping;
import org.apache.drill.exec.compile.sig.MappingSet;
import org.apache.drill.exec.expr.ClassGenerator.HoldingContainer;
+import org.apache.drill.exec.expr.annotations.FunctionTemplate;
import org.apache.drill.exec.expr.fn.AbstractFuncHolder;
import org.apache.drill.exec.expr.holders.ValueHolder;
import org.apache.drill.exec.physical.impl.filter.ReturnValueExpression;
@@ -105,6 +106,13 @@ public class EvaluationVisitor {
return e.accept(new CSEFilter(constantBoundaries), generator);
}
+ /**
+ * Callback function for the expression visitor
+ */
+ private interface VisitorCallback {
+HoldingContainer getHolder();
+ }
+
private class ExpressionHolder {
private LogicalExpression expression;
private GeneratorMapping mapping;
@@ -203,7 +211,7 @@ public class EvaluationVisitor {
generator.getMappingSet().exitChild();
}
- return holder.renderEnd(generator, args, workspaceVars,
holderExpr.getFieldReference());
+ return holder.renderEnd(generator, args, workspaceVars, holderExpr);
}
@Override
@@ -,6 +1119,8 @@ public class EvaluationVisitor {
}
}
+
+
private class ConstantFilter extend
[drill] branch master updated: DRILL-6951: Merge row set based mock data source
This is an automated email from the ASF dual-hosted git repository.
arina pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git
The following commit(s) were added to refs/heads/master by this push:
new 3599dfd DRILL-6951: Merge row set based mock data source
3599dfd is described below
commit 3599dfd2690023db6a7af66512b3e44759eb34a1
Author: Paul Rogers
AuthorDate: Tue Jun 18 19:26:16 2019 -0700
DRILL-6951: Merge row set based mock data source
The mock data source is used in several tests to generate a large volume
of sample data, such as when testing spilling. The mock data source also
lets us try new plugin featues in a very simple context. During the
development of the row set framework, the mock data source was converted
to use the new framework to verify functionality. This commit upgrades
the mock data source with that work.
The work changes non of the functionality. It does, however, improve
memory usage. Batchs are limited, by default, to 10 MB in size. The row
set framework minimizes internal fragmentation in the largest vector.
(Previously, internal fragmentation averaged 25% but could be as high as
50%.)
As it turns out, the hash aggregate tests depended on the internal
fragmentation: without it, the hash agg no longer spilled for the same
row count. Adjusted the generated row counts to recreate a data volume
that caused spilling.
One test in particular always failed due to assertions in the hash agg
code. These seem true bugs and are described in DRILL-7301. After
multiple failed attempts to get the test to work, it ws disabled until
DRILL-7301 is fixed.
Added a new unit test to sanity check the mock data source. (No test
already existed for this functionality except as verified via other unit
tests.)
---
.../impl/scan/framework/BasicScanFactory.java | 4 +-
.../{BooleanGen.java => AbstractFieldGen.java} | 23 +-
.../apache/drill/exec/store/mock/BooleanGen.java | 21 +-
.../apache/drill/exec/store/mock/ColumnDef.java| 20 +-
.../org/apache/drill/exec/store/mock/DateGen.java | 21 +-
.../apache/drill/exec/store/mock/DoubleGen.java| 21 +-
.../exec/store/mock/ExtendedMockBatchReader.java | 148 ++
.../exec/store/mock/ExtendedMockRecordReader.java | 158 ---
.../org/apache/drill/exec/store/mock/FieldGen.java | 6 +-
.../org/apache/drill/exec/store/mock/IntGen.java | 21 +-
.../exec/store/mock/MockScanBatchCreator.java | 72 -
.../apache/drill/exec/store/mock/MockStorePOP.java | 2 -
.../apache/drill/exec/store/mock/MockTableDef.java | 4 +-
.../org/apache/drill/exec/store/mock/MoneyGen.java | 22 +-
.../apache/drill/exec/store/mock/StringGen.java| 16 +-
.../drill/exec/store/mock/VaryingStringGen.java| 14 +-
.../exec/fn/interp/ExpressionInterpreterTest.java | 9 +-
.../exec/physical/impl/agg/TestHashAggrSpill.java | 129 +
.../impl/scan/BaseScanOperatorExecTest.java| 23 +-
.../drill/exec/store/mock/TestMockPlugin.java | 148 ++
.../drill/exec/store/mock/TestMockRowReader.java | 304 +
.../drill/exec/record/metadata/SchemaBuilder.java | 7 +
22 files changed, 821 insertions(+), 372 deletions(-)
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/scan/framework/BasicScanFactory.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/scan/framework/BasicScanFactory.java
index d9e8847..81e7dad 100644
---
a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/scan/framework/BasicScanFactory.java
+++
b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/scan/framework/BasicScanFactory.java
@@ -35,9 +35,9 @@ import
org.apache.drill.exec.physical.impl.scan.framework.ManagedScanFramework.R
public class BasicScanFactory implements ReaderFactory {
- private final Iterator> iterator;
+ private final Iterator> iterator;
- public BasicScanFactory(Iterator>
iterator) {
+ public BasicScanFactory(Iterator> iterator) {
this.iterator = iterator;
}
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/store/mock/BooleanGen.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/store/mock/AbstractFieldGen.java
similarity index 67%
copy from
exec/java-exec/src/main/java/org/apache/drill/exec/store/mock/BooleanGen.java
copy to
exec/java-exec/src/main/java/org/apache/drill/exec/store/mock/AbstractFieldGen.java
index dd84f4d..c4eb2ab 100644
---
a/exec/java-exec/src/main/java/org/apache/drill/exec/store/mock/BooleanGen.java
+++
b/exec/java-exec/src/main/java/org/apache/drill/exec/store/mock/AbstractFieldGen.java
@@ -19,24 +19,17 @@ package org.apache.drill.exec.store.mock;
import java.util.Random;
-import org.apache.drill.exec.vector.BitVector;
-import org.apache.drill.exec.vector.Value
[drill] branch master updated: DRILL-7293: Convert the regex ("log") plugin to use EVF
This is an automated email from the ASF dual-hosted git repository.
arina pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git
The following commit(s) were added to refs/heads/master by this push:
new 2224ee1 DRILL-7293: Convert the regex ("log") plugin to use EVF
2224ee1 is described below
commit 2224ee1014cfdcc2b76f1a8290171ca63db76a53
Author: Paul Rogers
AuthorDate: Wed Jun 12 18:28:16 2019 -0700
DRILL-7293: Convert the regex ("log") plugin to use EVF
Converts the log format plugin (which uses a regex for parsing) to work
with the Extended Vector Format.
User-visible behavior changes added to the README file.
* Use the plugin config object to pass config to the Easy framework.
* Use the EVF scan mechanism in place of the legacy "ScanBatch"
mechanism.
* Minor code and README cleanup.
* Replace ad-hoc type conversion with builtin conversions
The provided schema support in the enhanced vector framework (EVF)
provides automatic conversions from VARCHAR to most types. The log
format plugin was created before EVF was available and provided its own
conversion mechanism. This commit removes the ad-hoc conversion code and
instead uses the log plugin config schema information to create an
"output schema" just as if it was provided by the provided schema
framework.
Because we need the schema in the plugin (rather than the reader), moved
the schema-parsing code out of the reader into the plugin. The plugin
creates two schemas: an "output schema" with the desired output types,
and a "reader schema" that uses only VARCHAR. This causes the EVF to
perform conversions.
* Enable provided schema support
Allows the user to specify types using either the format config (as
previously) or a provided schema. If a schema is provided, it will match
columns using names specified in the format config.
The provided schema can specify both types and modes (nullable or not
null.)
If a schema is provided, then the types specified in the plugin config
are ignored. No attempt is made to merge schemas.
If a schema is provided, but a column is omitted from the schema, the
type defaults to VARCHAR.
* Added ability to specify regex in table properties
Allows the user to specify the regex, and the column schema,
using a CREATE SCHEMA statement. The README file provides the details.
Unit tests demonstrate and verify the functionality.
* Used the custom error context provided by EVF to enhance the log format
reader error messages.
* Added user name to default EVF error context
* Added support for table functions
Can set the regex and maxErrors fields, but not the schema.
Schema will default to "field_0", "field_1", etc. of type
VARCHAR.
* Added unit tests to verify the functionality.
* Added a check, and a test, for a regex with no groups.
* Added columns array support
When the log regex plugin is given no schema, it previously
created a list of columns "field_0", "field_1", etc. After
this change, the plugin instead follows the pattern set by
the text plugin: it will place all fields into the columns
array. (The two special fields are still separate.)
A few adjustments were necessary to the columns array
framework to allow use of the special columns along with
the `columns` column.
Modified unit tests and the README to reflect this change.
The change should be backward compatible because few users
are likely relying on the dummy field names.
Added unit tests to verify that schema-based table
functions work. A test shows that, due to the unforunate
config property name "schema", users of this plugin cannot
combine a config table function with the schema attribute
in the way promised in DRILL-6965.
---
.../impl/scan/columns/ColumnsArrayManager.java | 23 +-
.../impl/scan/columns/ColumnsArrayParser.java | 32 +-
.../impl/scan/columns/ColumnsScanFramework.java| 19 +-
.../impl/scan/project/projSet/TypeConverter.java | 2 +
.../exec/store/easy/text/TextFormatPlugin.java | 4 +-
.../easy/text/reader/CompliantTextBatchReader.java | 6 +-
.../drill/exec/store/log/LogBatchReader.java | 278
.../drill/exec/store/log/LogFormatConfig.java | 57 +-
.../drill/exec/store/log/LogFormatField.java | 61 +-
.../drill/exec/store/log/LogFormatPlugin.java | 402 +--
.../drill/exec/store/log/LogRecordReader.java | 769 -
.../java/org/apache/drill/exec/store/log/README.md | 251 ++-
.../apache/drill/TestSchemaWithTableFunction.java | 16 +-
.../exec/physical/impl/scan/TestCo
[drill] branch master updated: DRILL-7306: Disable schema-only batch for new scan framework
This is an automated email from the ASF dual-hosted git repository. arina pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/drill.git The following commit(s) were added to refs/heads/master by this push: new 66e8dd9 DRILL-7306: Disable schema-only batch for new scan framework 66e8dd9 is described below commit 66e8dd9f05ea068e27b272ed1b787004213c29e6 Author: Paul Rogers AuthorDate: Sun Jun 23 22:10:54 2019 -0700 DRILL-7306: Disable schema-only batch for new scan framework The EVF framework is set up to return a "fast schema" empty batch with only schema as its first batch because, when the code was written, it seemed that's how we wanted operators to work. However, DRILL-7305 notes that many operators cannot handle empty batches. Since the empty-batch bugs show that Drill does not, in fact, provide a "fast schema" batch, this ticket asks to disable the feature in the new scan framework. The feature is disabled with a config option; it can be re-enabled if ever it is needed. SQL differentiates between two subtle cases, and both are supported by this change. 1. Empty results: the query found a schema, but no rows are returned. If no reader returns any rows, but at least one reader provides a schema, then the scan returns an empty batch with the schema. 2. Null results: the query found no schema or rows. No schema is returned. If no reader returns rows or schema, then the scan returns no batch: it instead immediately returns a DONE status. For CSV, an empty file with headers returns the null result set (because we don't know the schema.) An empty CSV file without headers returns an empty result set because we do know the schema: it will always be the columns array. Old tests validate the original schema-batch mode, new tests added to validate the no-schema-batch mode. --- .../exec/physical/impl/filter/FilterTemplate2.java | 6 +- .../physical/impl/protocol/OperatorDriver.java | 57 +-- .../impl/protocol/OperatorRecordBatch.java | 5 +- .../exec/physical/impl/protocol/SchemaTracker.java | 6 +- .../impl/protocol/VectorContainerAccessor.java | 30 -- .../drill/exec/physical/impl/scan/ReaderState.java | 64 .../exec/physical/impl/scan/ScanOperatorExec.java | 109 +++-- .../impl/scan/columns/ColumnsScanFramework.java| 3 +- .../physical/impl/scan/file/FileScanFramework.java | 4 +- .../impl/scan/framework/ManagedScanFramework.java | 5 + .../impl/scan/project/ScanSchemaOrchestrator.java | 64 +++- .../physical/impl/scan/project/SchemaSmoother.java | 3 +- .../apache/drill/exec/record/VectorContainer.java | 36 --- .../exec/store/dfs/easy/EasyFormatPlugin.java | 76 +++--- .../exec/store/easy/text/TextFormatPlugin.java | 9 +- .../easy/text/reader/CompliantTextBatchReader.java | 2 +- .../apache/drill/TestSchemaWithTableFunction.java | 11 +-- .../org/apache/drill/exec/TestEmptyInputSql.java | 6 +- .../impl/protocol/TestOperatorRecordBatch.java | 24 ++--- .../impl/scan/BaseScanOperatorExecTest.java| 30 +++--- .../exec/physical/impl/scan/ScanTestUtils.java | 27 +++-- .../exec/physical/impl/scan/TestColumnsArray.java | 6 +- .../impl/scan/TestColumnsArrayFramework.java | 6 +- .../physical/impl/scan/TestFileScanFramework.java | 6 +- .../physical/impl/scan/TestScanOperExecBasics.java | 11 ++- .../impl/scan/TestScanOperExecEarlySchema.java | 52 +- .../impl/scan/TestScanOperExecLateSchema.java | 44 + .../impl/scan/TestScanOperExecSmoothing.java | 21 .../impl/scan/TestScanOrchestratorEarlySchema.java | 27 ++--- .../impl/scan/TestScanOrchestratorLateSchema.java | 5 +- .../impl/scan/TestScanOrchestratorMetadata.java| 11 ++- .../impl/scan/project/TestSchemaSmoothing.java | 3 +- .../store/easy/text/compliant/BaseCsvTest.java | 11 +++ .../store/easy/text/compliant/TestCsvHeader.java | 10 +- .../easy/text/compliant/TestCsvWithHeaders.java| 66 - .../easy/text/compliant/TestCsvWithSchema.java | 2 +- .../easy/text/compliant/TestCsvWithoutHeaders.java | 43 ++-- .../easy/text/compliant/TestPartitionRace.java | 35 --- .../apache/drill/exec/store/log/TestLogReader.java | 4 - .../java/org/apache/drill/test/QueryBuilder.java | 35 +-- .../apache/drill/test/rowSet/RowSetBuilder.java| 10 +- .../apache/drill/test/rowSet/RowSetUtilities.java | 2 +- .../drill/exec/record/MaterializedField.java | 4 +- .../drill/exec/record/metadata/ColumnBuilder.java | 17 +++- .../record/metadata/PrimitiveColumnMetadata.java | 18 +++- 45 files changed, 764 insertions(+), 262 deletions(-) diff --git a/exec/java-exec/src/main/java/org/apache
[drill] branch master updated: DRILL-7314: Use TupleMetadata instead of concrete implementation
This is an automated email from the ASF dual-hosted git repository.
arina pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git
The following commit(s) were added to refs/heads/master by this push:
new e843588 DRILL-7314: Use TupleMetadata instead of concrete
implementation
e843588 is described below
commit e843588178dd25408c447217aac25c0970924ebc
Author: Arina Ielchiieva
AuthorDate: Fri Jul 5 16:43:37 2019 +0300
DRILL-7314: Use TupleMetadata instead of concrete implementation
1. Add ser / de implementation for TupleMetadata interface based on types.
2. Replace TupleSchema usage where possible.
3. Move patcher classes into commons.
4. Upgrade some dependencies and general refactoring.
---
common/pom.xml | 2 +-
.../apache/drill/common}/util/GuavaPatcher.java| 14 +++--
.../apache/drill/common}/util/ProtobufPatcher.java | 8 ++-
.../drill/exec/store/mapr/db/MapRDBSubScan.java| 4 +-
.../store/mapr/db/RestrictedMapRDBSubScan.java | 19 +++---
.../store/mapr/db/binary/BinaryTableGroupScan.java | 10 ++-
.../store/mapr/db/json/JsonTableGroupScan.java | 27
.../mapr/db/json/RestrictedJsonTableGroupScan.java | 5 +-
.../drill/maprdb/tests/index/TableIndexCmd.java| 2 +-
.../org/apache/drill/hbase/HBaseTestsSuite.java| 4 +-
.../hive/HiveDrillNativeParquetRowGroupScan.java | 30 +
.../store/hive/HiveDrillNativeParquetScan.java | 3 +-
exec/java-exec/pom.xml | 2 +-
.../drill/exec/coord/ClusterCoordinator.java | 16 ++---
.../base/AbstractGroupScanWithMetadata.java| 12 ++--
.../exec/physical/rowSet/impl/TupleState.java | 2 +-
.../exec/physical/rowSet/model/BaseTupleModel.java | 6 +-
.../rowSet/model/hyper/HyperSchemaInference.java | 2 +-
.../record/metadata/schema/SchemaContainer.java| 3 +-
.../org/apache/drill/exec/server/Drillbit.java | 18 +++---
.../exec/store/StoragePluginRegistryImpl.java | 2 +-
.../drill/exec/store/dfs/easy/EasyGroupScan.java | 4 +-
.../drill/exec/store/dfs/easy/EasySubScan.java | 3 +-
.../store/parquet/AbstractParquetGroupScan.java| 1 -
.../store/parquet/AbstractParquetRowGroupScan.java | 20 +++---
.../parquet/AbstractParquetScanBatchCreator.java | 20 +++---
.../drill/exec/store/parquet/ParquetGroupScan.java | 10 +--
.../exec/store/parquet/ParquetRowGroupScan.java| 15 +++--
.../org/apache/drill/exec/work/WorkManager.java| 71 +-
.../test/java/org/apache/drill/exec/ExecTest.java | 2 +-
.../physical/impl/join/TestHashJoinOutcome.java| 20 +++---
.../exec/record/metadata/TestTupleSchema.java | 47 ++
.../record/metadata/schema/TestSchemaProvider.java | 1 +
.../drill/exec/store/pcapng/TestPcapngHeaders.java | 13 ++--
exec/jdbc-all/pom.xml | 2 +-
.../drill/exec/record/metadata/TupleMetadata.java | 51 +++-
.../drill/exec/record/metadata/TupleSchema.java| 2 +
logical/pom.xml| 2 +-
pom.xml| 30 -
39 files changed, 289 insertions(+), 216 deletions(-)
diff --git a/common/pom.xml b/common/pom.xml
index bc3a9aa..47b60fc 100644
--- a/common/pom.xml
+++ b/common/pom.xml
@@ -42,7 +42,7 @@
junit
junit
- ${dep.junit.version}
+ ${junit.version}
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/util/GuavaPatcher.java
b/common/src/main/java/org/apache/drill/common/util/GuavaPatcher.java
similarity index 89%
rename from
exec/java-exec/src/main/java/org/apache/drill/exec/util/GuavaPatcher.java
rename to common/src/main/java/org/apache/drill/common/util/GuavaPatcher.java
index 3269aa8..22a4cb5 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/util/GuavaPatcher.java
+++ b/common/src/main/java/org/apache/drill/common/util/GuavaPatcher.java
@@ -15,7 +15,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.drill.exec.util;
+package org.apache.drill.common.util;
import java.lang.reflect.Modifier;
@@ -24,9 +24,11 @@ import javassist.CtClass;
import javassist.CtConstructor;
import javassist.CtMethod;
import javassist.CtNewMethod;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public class GuavaPatcher {
- static final org.slf4j.Logger logger =
org.slf4j.LoggerFactory.getLogger(GuavaPatcher.class);
+ private static final Logger logger =
LoggerFactory.getLogger(GuavaPatcher.class);
private static boolean patched;
@@ -58,9 +60,9 @@ public class GuavaPatcher {
}
// Add back the Stopwatch.elapsedMillis() method for old consumers.
-CtMethod newmethod = CtNewMethod.make(
+CtMethod newMethod = CtNewMethod.make(
"public long elapsedMillis() { r
[drill] branch master updated (dc865e8 -> f3d6b69)
This is an automated email from the ASF dual-hosted git repository. arina pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/drill.git. from dc865e8 DRILL-7271: Refactor Metadata interfaces and classes to contain all needed information for the File based Metastore new e8d9b0a DRILL-6711: Use jitpack repository for Drill Calcite project artifacts instead of repository.mapr.com new f3d6b69 DRILL-7310: Move schema-related classes from exec module to be able to use them in metastore module The 2 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: common/pom.xml | 2 +- contrib/storage-hive/core/pom.xml | 14 +++- contrib/storage-hive/hive-exec-shade/pom.xml | 4 +- docs/dev/Calcite.md| 26 ++ exec/java-exec/pom.xml | 25 +- .../exec/physical/rowSet/impl/ColumnBuilder.java | 13 +-- .../drill/exec/record/BatchSchemaBuilder.java | 57 + .../org/apache/drill/exec/record/SchemaUtil.java | 11 +++ .../store/parquet/BaseParquetMetadataProvider.java | 5 +- .../store/parquet/ParquetTableMetadataUtils.java | 4 +- .../java/org/apache/drill/TestFunctionsQuery.java | 7 +- .../java/org/apache/drill/TestStarQueries.java | 19 +++-- .../test/java/org/apache/drill/TestUnionAll.java | 7 +- .../java/org/apache/drill/TestUnionDistinct.java | 7 +- .../java/org/apache/drill/TestUntypedNull.java | 19 +++-- .../org/apache/drill/exec/TestEmptyInputSql.java | 61 +- .../drill/exec/cache/TestBatchSerialization.java | 31 --- .../drill/exec/fn/impl/TestCastFunctions.java | 7 +- .../impl/agg/TestStreamingAggEmitOutcome.java | 2 - .../physical/impl/join/TestHashJoinAdvanced.java | 7 +- .../physical/impl/join/TestHashJoinOutcome.java| 4 +- .../impl/protocol/TestOperatorRecordBatch.java | 32 +--- .../impl/scan/BaseScanOperatorExecTest.java| 5 +- .../physical/impl/scan/TestFileScanFramework.java | 30 --- .../physical/impl/scan/TestScanBatchWriters.java | 6 +- .../impl/scan/TestScanOperExecSmoothing.java | 5 +- .../impl/scan/TestScanOrchestratorEarlySchema.java | 24 +++--- .../impl/scan/TestScanOrchestratorLateSchema.java | 5 +- .../impl/scan/TestScanOrchestratorMetadata.java| 17 ++-- .../scan/project/TestConstantColumnLoader.java | 10 +-- .../impl/scan/project/TestRowBatchMerger.java | 46 +-- .../impl/svremover/AbstractGenericCopierTest.java | 7 +- .../physical/impl/validate/TestBatchValidator.java | 42 +- .../exec/physical/impl/xsort/TestExternalSort.java | 26 +++--- .../physical/impl/xsort/managed/TestCopier.java| 7 +- .../impl/xsort/managed/TestShortArrays.java| 6 +- .../physical/impl/xsort/managed/TestSortImpl.java | 3 +- .../physical/impl/xsort/managed/TestSorter.java| 14 ++-- .../impl/TestResultSetLoaderOmittedValues.java | 5 +- .../rowSet/impl/TestResultSetLoaderProjection.java | 13 ++- .../rowSet/impl/TestResultSetSchemaChange.java | 20 +++-- .../drill/exec/physical/unit/TestMiniPlan.java | 19 +++-- .../exec/physical/unit/TestNullInputMiniPlan.java | 69 ++-- .../drill/exec/record/TestRecordBatchSizer.java| 95 +++--- .../drill/exec/record/TestVectorContainer.java | 13 +-- .../exec/record/metadata/TestTupleSchema.java | 6 +- .../apache/drill/exec/record/vector/TestLoad.java | 76 +++-- .../drill/exec/store/TestImplicitFileColumns.java | 19 +++-- .../drill/exec/store/httpd/TestHTTPDLogReader.java | 6 +- .../apache/drill/exec/store/log/TestLogReader.java | 54 ++-- .../store/parquet/TestParquetMetadataCache.java| 9 +- .../java/org/apache/drill/test/ClientFixture.java | 7 +- .../java/org/apache/drill/test/ExampleTest.java| 6 +- .../rowSet/test/TestRepeatedListAccessors.java | 21 +++-- .../test/rowSet/test/TestVariantAccessors.java | 5 +- exec/jdbc-all/pom.xml | 8 +- exec/vector/pom.xml| 21 + .../record/metadata/schema/parser/SchemaLexer.g4 | 0 .../record/metadata/schema/parser/SchemaParser.g4 | 0 .../record/metadata/AbstractColumnMetadata.java| 0 .../drill/exec/record/metadata/ColumnBuilder.java | 0 .../drill/exec/record/metadata/MapBuilder.java | 0 .../exec/record/metadata/MapColumnMetadata.java| 0 .../drill/exec/record/metadata/MetadataUtils.java | 45 -- .../record/metadata/PrimitiveColumnMetadata.java | 6 +- .../exec/record/metadata/RepeatedListBuilder.java | 0 .../metadata/RepeatedListColumnMetadata.java | 0 .../drill/
[drill] 01/02: DRILL-6711: Use jitpack repository for Drill Calcite project artifacts instead of repository.mapr.com
This is an automated email from the ASF dual-hosted git repository. arina pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/drill.git commit e8d9b0ae6253db10fa5c2e8b5aeb4eecb3a1feb9 Author: Volodymyr Vysotskyi AuthorDate: Mon Jun 24 14:52:59 2019 +0300 DRILL-6711: Use jitpack repository for Drill Calcite project artifacts instead of repository.mapr.com closes #1815 --- common/pom.xml | 2 +- contrib/storage-hive/core/pom.xml| 14 ++- contrib/storage-hive/hive-exec-shade/pom.xml | 4 ++-- docs/dev/Calcite.md | 26 + exec/java-exec/pom.xml | 4 ++-- exec/jdbc-all/pom.xml| 8 +++ logical/pom.xml | 2 +- pom.xml | 35 +++- 8 files changed, 57 insertions(+), 38 deletions(-) diff --git a/common/pom.xml b/common/pom.xml index 8c71404..8c9593c 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -46,7 +46,7 @@ - org.apache.calcite + com.github.vvysotskyi.drill-calcite calcite-core diff --git a/contrib/storage-hive/core/pom.xml b/contrib/storage-hive/core/pom.xml index 9912b83..88003ae 100644 --- a/contrib/storage-hive/core/pom.xml +++ b/contrib/storage-hive/core/pom.xml @@ -96,7 +96,7 @@ - org.apache.calcite + com.github.vvysotskyi.drill-calcite calcite-core @@ -150,6 +150,18 @@ test + org.apache.calcite + calcite-core + + + org.apache.calcite + calcite-linq4j + + + org.apache.calcite + calcite-druid + + commons-logging commons-logging diff --git a/contrib/storage-hive/hive-exec-shade/pom.xml b/contrib/storage-hive/hive-exec-shade/pom.xml index a767000..4e79444 100644 --- a/contrib/storage-hive/hive-exec-shade/pom.xml +++ b/contrib/storage-hive/hive-exec-shade/pom.xml @@ -65,8 +65,8 @@ calcite-core - org.apache.calcite - calcite-avatica + org.apache.calcite.avatica + avatica org.apache.calcite diff --git a/docs/dev/Calcite.md b/docs/dev/Calcite.md new file mode 100644 index 000..39d5fe4 --- /dev/null +++ b/docs/dev/Calcite.md @@ -0,0 +1,26 @@ +# Drill-specific commits in Apache Calcite + +Currently, Drill uses Apache Calcite with additional changes, required for Drill. All the commits were left after +update from Calcite `1.4.0` to Calcite `1.15.0` and weren't merged to the Calcite's master yet since there is no consensus on them in Calcite community. + +List of Jiras with Drill-specific commits: + +|Jira|Summary|The reason why it wasn't merged| +||---|---| +|[CALCITE-2018](https://issues.apache.org/jira/browse/CALCITE-2018)|Queries failed with AssertionError: rel has lower cost than best cost of subset|Pull request with the fix was created ([PR-552](https://github.com/apache/calcite/pull/552)), but [CALCITE-2166](https://issues.apache.org/jira/browse/CALCITE-2166) which blocks it was found and is not resolved yet.| +|[CALCITE-2087](https://issues.apache.org/jira/browse/CALCITE-2087)|Add new method to ViewExpander interface to allow passing SchemaPlus.|Pull request into Apache Calcite was created, but it was declined. See conversation in Jira.| +|[CALCITE-1178](https://issues.apache.org/jira/browse/CALCITE-1178)|Allow SqlBetweenOperator to compare DATE and TIMESTAMP|SQL spec does not allow to compare datetime types if they have different ``s. Therefore Calcite community won’t accept these changes. Similar issues were reported in [CALCITE-2829](https://issues.apache.org/jira/browse/CALCITE-2829) and in [CALCITE-2745](https://issues.apache.org/jira/browse/CALCITE-2745).| + +# Drill-Calcite repository + +Repository with source code is placed [here](https://github.com/vvysotskyi/drill-calcite). For backward +compatibility, a couple of previous Drill Calcite branches were pushed into this repo. + +Drill committers who need write permissions to the repository, should notify its owner. + +# Process of updating Calcite version + +- Push required changes to the existing branch, or create new branch. *Though this repository contains Drill specific commits, it is forbidden to add additional specific commits which were not merged to Apache Calcite master if it wasn't discussed in Drill community first.* +- The last commit must be a commit which updates the version number. +- Create and push tag for commit with the version update. Tag name should match the version, for example, `1.18.0-drill-r0`, `1.18.0-drill-r1`, `1.18.0-drill-r2` and so on. +- Bump-up Drill Calcite version in Drill pom.xml file. \ No newline at end of file diff --git
[drill] 05/05: DRILL-7236: SqlLine 1.8 upgrade
This is an automated email from the ASF dual-hosted git repository. arina pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/drill.git commit 2766e653cda8b1de817b234c66b0058e707750d0 Author: Arina Ielchiieva AuthorDate: Mon Jun 3 17:15:09 2019 +0300 DRILL-7236: SqlLine 1.8 upgrade closes #1804 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index a1ec48e..e141ea0 100644 --- a/pom.xml +++ b/pom.xml @@ -53,7 +53,7 @@ 1.18.0-drill-r2 1.13.0 3.0.11 -1.7.0 +1.8.0 2.9.5 2.9.5 3.4.12
[drill] branch master updated (2615d68 -> 2766e65)
This is an automated email from the ASF dual-hosted git repository.
arina pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git.
from 2615d68 DRILL-4782 / DRILL-7139: Fix DATE_ADD and TO_TIME functions
new 8a7007f DRILL-7258: Remove field width limit for text reader
new 1bf7f15 DRILL-7261: Simplify Easy framework config for new scan
new 20ae96a DRILL-7279: Enable provided schema for text files without
headers
new 0dbd501 DRILL-7278: Refactor result set loader projection mechanism
new 2766e65 DRILL-7236: SqlLine 1.8 upgrade
The 5 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../physical/impl/scan/file/FileScanFramework.java | 8 +-
.../impl/scan/framework/ManagedScanFramework.java | 7 +
.../impl/scan/framework/SchemaNegotiator.java | 7 +
.../impl/scan/framework/SchemaNegotiatorImpl.java | 5 +
.../scan/project/AbstractUnresolvedColumn.java | 38 +-
.../scan/project/ReaderSchemaOrchestrator.java | 18 +-
.../impl/scan/project/ResolvedNullColumn.java | 12 +
.../impl/scan/project/ScanLevelProjection.java | 65 ++-
.../impl/scan/project/ScanSchemaOrchestrator.java | 68 +--
.../scan/project/WildcardSchemaProjection.java | 35 +-
.../physical/impl/scan/project/package-info.java | 182 +-
.../project/projSet/AbstractProjectionSet.java | 80 +++
.../scan/project/projSet/AbstractReadColProj.java | 50 ++
.../scan/project/projSet/EmptyProjectionSet.java | 24 +-
.../project/projSet/ExplicitProjectionSet.java | 109
.../projSet/ProjectedMapColumn.java} | 21 +-
.../scan/project/projSet/ProjectedReadColumn.java | 78 +++
.../scan/project/projSet/ProjectionSetBuilder.java | 101
.../scan/project/projSet/ProjectionSetFactory.java | 79 +++
.../impl/scan/project/projSet/TypeConverter.java | 173 ++
.../project/projSet/UnprojectedReadColumn.java}| 23 +-
.../project/projSet/WildcardProjectionSet.java | 55 ++
.../impl/scan/project/projSet/package-info.java| 99
.../drill/exec/physical/rowSet/ProjectionSet.java | 105
.../exec/physical/rowSet/impl/ColumnBuilder.java | 218 ++-
.../exec/physical/rowSet/impl/ColumnState.java | 3 +-
.../exec/physical/rowSet/impl/ContainerState.java | 18 +-
.../rowSet/impl/DefaultSchemaTransformer.java | 77 ---
.../drill/exec/physical/rowSet/impl/ListState.java | 7 +-
.../exec/physical/rowSet/impl/LoaderInternals.java | 8 +
.../exec/physical/rowSet/impl/OptionBuilder.java | 47 +-
.../physical/rowSet/impl/RepeatedListState.java| 5 +-
.../physical/rowSet/impl/ResultSetLoaderImpl.java | 36 +-
.../physical/rowSet/impl/SchemaTransformer.java| 46 --
.../rowSet/impl/SchemaTransformerImpl.java | 194 ---
.../exec/physical/rowSet/impl/TupleState.java | 12 +-
.../exec/physical/rowSet/impl/UnionState.java | 5 +-
.../exec/physical/rowSet/impl/WriterIndexImpl.java | 3 +
.../rowSet/project/ImpliedTupleRequest.java| 8 +-
.../physical/rowSet/project/ProjectionType.java| 178 ++
.../rowSet/project/RequestedColumnImpl.java| 29 +-
.../physical/rowSet/project/RequestedTuple.java| 14 +-
.../rowSet/project/RequestedTupleImpl.java | 30 +-
.../record/metadata/AbstractColumnMetadata.java| 34 +-
.../exec/store/dfs/easy/EasyFormatPlugin.java | 412 +++---
.../exec/store/easy/text/TextFormatPlugin.java | 193 ---
.../easy/text/compliant/v3/BaseFieldOutput.java| 52 +-
.../compliant/v3/CompliantTextBatchReader.java | 17 +-
.../easy/text/compliant/v3/FieldVarCharOutput.java | 14 +-
.../text/compliant/v3/RepeatedVarCharOutput.java | 9 +-
.../store/easy/text/compliant/v3/TextInput.java| 11 +-
.../text/compliant/v3/TextParsingSettingsV3.java | 282 --
.../store/easy/text/compliant/v3/TextReader.java | 43 +-
.../physical/impl/scan/TestFileScanFramework.java | 1 -
.../impl/scan/TestScanOperExecOuputSchema.java | 75 ++-
.../impl/scan/TestScanOrchestratorEarlySchema.java | 8 +-
.../impl/scan/project/TestScanLevelProjection.java | 208 +--
.../scan/project/projSet/TestProjectionSet.java| 625 +
.../impl/TestResultSetLoaderEmptyProject.java | 13 +-
.../rowSet/impl/TestResultSetLoaderOverflow.java | 76 ++-
.../rowSet/impl/TestResultSetLoaderProjection.java | 92 +--
.../impl/TestResultSetLoaderTypeConversion.java| 22 +-
.../{impl => project}/TestProjectedTuple.java | 65 ++-
.../rowSet/project/TestProjectionType.java | 154 +
.../record/metadata/TestMetadataProperties.java| 20 -
.../store/easy/text/compliant/BaseCsvTest.java | 43 ++
[drill] 03/05: DRILL-7279: Enable provided schema for text files without headers
This is an automated email from the ASF dual-hosted git repository.
arina pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git
commit 20ae96a4d0de23d329de1b5534683fdaac8816a7
Author: Paul Rogers
AuthorDate: Sun May 26 23:32:40 2019 -0700
DRILL-7279: Enable provided schema for text files without headers
* Allows a provided schema for text files without headers. The
provided schema columns replace the `columns` column that is
normally used.
* Allows customizing text format properties using table properties.
The table properties "override" properties set in the plugin config.
* Added unit tests for the newly supported use cases.
* Fixed bug in quote escape handling.
closes #1798
---
.../impl/scan/project/ScanLevelProjection.java | 2 +-
.../impl/scan/project/ScanSchemaOrchestrator.java | 2 +-
.../exec/store/dfs/easy/EasyFormatPlugin.java | 140 ++-
.../exec/store/easy/text/TextFormatPlugin.java | 117 +-
.../compliant/v3/CompliantTextBatchReader.java | 17 +-
.../store/easy/text/compliant/v3/TextInput.java| 2 -
.../text/compliant/v3/TextParsingSettingsV3.java | 282 ++---
.../store/easy/text/compliant/v3/TextReader.java | 43 +-
.../store/easy/text/compliant/BaseCsvTest.java | 25 ++
.../easy/text/compliant/TestCsvIgnoreHeaders.java | 11 +-
.../text/compliant/TestCsvTableProperties.java | 451 +
.../easy/text/compliant/TestCsvWithSchema.java | 27 +-
.../easy/text/compliant/TestCsvWithoutHeaders.java | 10 +-
.../exec/record/metadata/AbstractPropertied.java | 13 +-
.../drill/exec/record/metadata/Propertied.java | 3 +-
15 files changed, 917 insertions(+), 228 deletions(-)
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/scan/project/ScanLevelProjection.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/scan/project/ScanLevelProjection.java
index 7718119..4a02b33 100644
---
a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/scan/project/ScanLevelProjection.java
+++
b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/scan/project/ScanLevelProjection.java
@@ -456,7 +456,7 @@ public class ScanLevelProjection {
if (hasOutputSchema()) {
projectionType =
- outputSchema.getBooleanProperty(TupleMetadata.IS_STRICT_SCHEMA_PROP)
+ outputSchema.booleanProperty(TupleMetadata.IS_STRICT_SCHEMA_PROP)
? ScanProjectionType.STRICT_SCHEMA_WILDCARD
: ScanProjectionType.SCHEMA_WILDCARD;
} else if (wildcardPosn != -1) {
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/scan/project/ScanSchemaOrchestrator.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/scan/project/ScanSchemaOrchestrator.java
index a315a3f..37f7c75 100644
---
a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/scan/project/ScanSchemaOrchestrator.java
+++
b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/scan/project/ScanSchemaOrchestrator.java
@@ -322,7 +322,7 @@ public class ScanSchemaOrchestrator {
// Use only implicit conversions
schemaTransformer = new SchemaTransformerImpl(
builder.outputSchema, builder.conversionProps);
-if
(builder.outputSchema.getBooleanProperty(TupleMetadata.IS_STRICT_SCHEMA_PROP)) {
+if
(builder.outputSchema.booleanProperty(TupleMetadata.IS_STRICT_SCHEMA_PROP)) {
allowRequiredNulls = true;
}
} else {
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/easy/EasyFormatPlugin.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/easy/EasyFormatPlugin.java
index a170122..3c79aea 100644
---
a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/easy/EasyFormatPlugin.java
+++
b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/easy/EasyFormatPlugin.java
@@ -147,6 +147,142 @@ public abstract class EasyFormatPlugin implements
.addContext(errorContext())
.build(logger);
}
+
+ return new ScanBatch(context, oContext, readers, implicitColumns);
+}
+
+/**
+ * Create a record reader given a file system, a file description and other
+ * information. For backward compatibility, calls the plugin method by
+ * default.
+ *
+ * @param plugin
+ * the plugin creating the scan
+ * @param context
+ * fragment context for the fragment running the scan
+ * @param dfs
+ * Drill's distributed file system facade
+ * @param fileWork
+ * description of the file to scan
+ * @param columns
+ * list of columns to project
+ * @param userName
+ * the name of the user performing the scan
+ * @return a scan operator
+ * @throws ExecutionSetu
[drill] 01/05: DRILL-7258: Remove field width limit for text reader
This is an automated email from the ASF dual-hosted git repository.
arina pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git
commit 8a7007f03397849b555a74297a6a637293958cc5
Author: Paul Rogers
AuthorDate: Thu May 30 18:43:09 2019 -0700
DRILL-7258: Remove field width limit for text reader
The V2 text reader enforced a limit of 64K characters when using
column headers, but not when using the columns[] array. The V3 reader
enforced the 64K limit in both cases.
This patch removes the limit in both cases. The limit now is the
16MB vector size limit. With headers, no one column can exceed 16MB.
With the columns[] array, no one row can exceed 16MB. (The 16MB
limit is set by the Netty memory allocator.)
Added an "appendBytes()" method to the scalar column writer which adds
additional bytes to those already written for a specific column or
array element value. The method is implemented for VarChar, Var16Char
and VarBinary vectors. It throws an exception for all other types.
When used with a type conversion shim, the appendBytes() method throws
an exception. This should be OK because, the previous setBytes() should
have failed because a huge value is not acceptable for numeric or date
types conversions.
Added unit tests of the append feature, and for the append feature in
the batch overflow case (when appending bytes causes the vector or
batch to overflow.) Also added tests to verify the lack of column width
limit with the text reader, both with and without headers.
closes #1802
---
.../exec/physical/rowSet/impl/WriterIndexImpl.java | 3 +
.../easy/text/compliant/v3/BaseFieldOutput.java| 52 ++---
.../easy/text/compliant/v3/FieldVarCharOutput.java | 12 +--
.../text/compliant/v3/RepeatedVarCharOutput.java | 7 +-
.../store/easy/text/compliant/v3/TextInput.java| 9 +--
.../rowSet/impl/TestResultSetLoaderOverflow.java | 76 +-
.../store/easy/text/compliant/BaseCsvTest.java | 20 +
.../easy/text/compliant/TestCsvWithHeaders.java| 25 ++
.../easy/text/compliant/TestCsvWithoutHeaders.java | 29 +++
.../apache/drill/test/rowSet/RowSetWriterImpl.java | 3 +
.../drill/test/rowSet/test/PerformanceTool.java| 5 +-
.../test/rowSet/test/TestFixedWidthWriter.java | 3 +
.../test/rowSet/test/TestScalarAccessors.java | 89 ++
.../main/codegen/templates/ColumnAccessors.java| 11 +++
.../exec/vector/accessor/ColumnWriterIndex.java| 13 +++-
.../drill/exec/vector/accessor/ScalarWriter.java | 1 +
.../accessor/convert/AbstractWriteConverter.java | 5 ++
.../accessor/writer/AbstractArrayWriter.java | 7 +-
.../vector/accessor/writer/BaseScalarWriter.java | 5 ++
.../vector/accessor/writer/BaseVarWidthWriter.java | 7 ++
.../exec/vector/accessor/writer/MapWriter.java | 1 +
.../accessor/writer/NullableScalarWriter.java | 8 ++
.../accessor/writer/OffsetVectorWriterImpl.java| 6 ++
.../vector/accessor/writer/ScalarArrayWriter.java | 3 +
.../accessor/writer/dummy/DummyScalarWriter.java | 3 +
25 files changed, 376 insertions(+), 27 deletions(-)
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/rowSet/impl/WriterIndexImpl.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/rowSet/impl/WriterIndexImpl.java
index 9fb3e4e..6119791 100644
---
a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/rowSet/impl/WriterIndexImpl.java
+++
b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/rowSet/impl/WriterIndexImpl.java
@@ -106,6 +106,9 @@ class WriterIndexImpl implements ColumnWriterIndex {
public void nextElement() { }
@Override
+ public void prevElement() { }
+
+ @Override
public ColumnWriterIndex outerIndex() { return null; }
@Override
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/text/compliant/v3/BaseFieldOutput.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/text/compliant/v3/BaseFieldOutput.java
index 6bf0bb6..5dd4284 100644
---
a/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/text/compliant/v3/BaseFieldOutput.java
+++
b/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/text/compliant/v3/BaseFieldOutput.java
@@ -17,13 +17,17 @@
*/
package org.apache.drill.exec.store.easy.text.compliant.v3;
-import org.apache.drill.common.exceptions.UserException;
import org.apache.drill.exec.physical.rowSet.RowSetLoader;
+import org.apache.drill.exec.vector.accessor.ScalarWriter;
public abstract class BaseFieldOutput extends TextOutput {
- private static final org.slf4j.Logger logger =
org.slf4j.LoggerFactory.getLogger(BaseFieldOutput.class);
- private static final int MAX_FIELD_LENGTH = 1024 * 64;
+ /**
+ * Width of the per-field data buffer. Fields can
[drill] 02/05: DRILL-7261: Simplify Easy framework config for new scan
This is an automated email from the ASF dual-hosted git repository.
arina pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git
commit 1bf7f1525374c9e414f89d5a17056e860a125510
Author: Paul Rogers
AuthorDate: Sat May 25 20:40:22 2019 -0700
DRILL-7261: Simplify Easy framework config for new scan
Most format plugins are created using the Easy format plugin. A recent
change added support for the "row set" scan framework. After converting
the text and log reader plugins, it became clear that the setup code
could be made simpler.
* Add the user name to the "file scan" framework.
* Pass the file system, split and user name to the batch reader via
the "schema negotiator" rather than via the constructor.
* Create the traditional "scan batch" scan or the new row-set scan via
functions instead of classes.
* Add Easy config option and method to choose the kind of scan
framework.
* Add Easy config options for some newer options such as whether the
plugin supports statistics.
Simplified reader creation
* The batch reader can be created just by overriding a method.
* A default error context is provided if the plugin does not provide
one.
Tested by running all unit tests for the CSV reader which is based on
the new framework, and by testing the converted log reader (that reader
is not part of this commit.)
closes #1796
---
.../physical/impl/scan/file/FileScanFramework.java | 8 +-
.../impl/scan/framework/ManagedScanFramework.java | 7 +
.../impl/scan/framework/SchemaNegotiator.java | 7 +
.../impl/scan/framework/SchemaNegotiatorImpl.java | 5 +
.../exec/store/dfs/easy/EasyFormatPlugin.java | 408 ++---
.../exec/store/easy/text/TextFormatPlugin.java | 142 +++
.../easy/text/compliant/TestCsvWithHeaders.java| 4 +-
7 files changed, 282 insertions(+), 299 deletions(-)
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/scan/file/FileScanFramework.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/scan/file/FileScanFramework.java
index f69aa40..761f68b 100644
---
a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/scan/file/FileScanFramework.java
+++
b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/scan/file/FileScanFramework.java
@@ -62,7 +62,8 @@ import org.apache.hadoop.mapred.FileSplit;
public class FileScanFramework extends ManagedScanFramework {
- private static final org.slf4j.Logger logger =
org.slf4j.LoggerFactory.getLogger(FileScanFramework.class);
+ private static final org.slf4j.Logger logger =
+ org.slf4j.LoggerFactory.getLogger(FileScanFramework.class);
/**
* The file schema negotiator adds no behavior at present, but is
@@ -80,7 +81,6 @@ public class FileScanFramework extends ManagedScanFramework {
/**
* Gives the Drill file system for this operator.
*/
-
DrillFileSystem fileSystem();
/**
@@ -186,6 +186,10 @@ public class FileScanFramework extends
ManagedScanFramework {
return newReader();
}
+public CustomErrorContext errorContext() {
+ return fileFramework == null ? null : fileFramework.errorContext();
+}
+
public abstract ManagedReader newReader();
}
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/scan/framework/ManagedScanFramework.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/scan/framework/ManagedScanFramework.java
index 52203da..a7a46fe 100644
---
a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/scan/framework/ManagedScanFramework.java
+++
b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/scan/framework/ManagedScanFramework.java
@@ -135,10 +135,17 @@ public class ManagedScanFramework implements
ScanOperatorEvents {
public static class ScanFrameworkBuilder extends ScanOrchestratorBuilder {
protected ReaderFactory readerFactory;
+protected String userName;
public void setReaderFactory(ReaderFactory readerFactory) {
this.readerFactory = readerFactory;
}
+
+public ReaderFactory readerFactory() { return readerFactory; }
+
+public void setUserName(String userName) {
+ this.userName = userName;
+}
}
// Inputs
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/scan/framework/SchemaNegotiator.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/scan/framework/SchemaNegotiator.java
index bc303ec..74ea512 100644
---
a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/scan/framework/SchemaNegotiator.java
+++
b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/scan/framework/SchemaNegotiator.java
@@ -64,8 +64,15 @@ public interface SchemaNegotiator {
*
[drill] branch master updated: DRILL-7181: Improve V3 text reader (row set) error messages
This is an automated email from the ASF dual-hosted git repository. arina pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/drill.git The following commit(s) were added to refs/heads/master by this push: new 9f67e95 DRILL-7181: Improve V3 text reader (row set) error messages 9f67e95 is described below commit 9f67e95a91824a617785764f1bfbf1faa768526d Author: Paul Rogers AuthorDate: Wed May 15 19:35:19 2019 -0700 DRILL-7181: Improve V3 text reader (row set) error messages Adds an error context to the User Error mechanism. The context allows information to be passed through an intermediate layer and applied when errors are raised in lower-level code; without the need for that low-level code to know the details of the error context information. Modifies the scan framework and V3 text plugin to use the framework to improve error messages. Refines how the `columns` column can be used with the text reader. If headers are used, then `columns` is just another column. An error is raised, however, if `columns[x]` is used when headers are enabled. Added another builder abstraction where a constructor argument list became too long. Added the drill file system and split to the file schema negotiator to simplify reader construction. Added additional unit tests to fully define the `columns` column behavior. --- .../drill/common/exceptions/ChildErrorContext.java | 41 .../common/exceptions/CustomErrorContext.java | 44 .../drill/common/exceptions/EmptyErrorContext.java | 26 + .../drill/common/exceptions/UserException.java | 7 ++ .../impl/scan/columns/ColumnsArrayParser.java | 40 ++-- .../physical/impl/scan/file/FileScanFramework.java | 69 +++-- .../impl/scan/framework/ManagedScanFramework.java | 3 + .../impl/scan/framework/SchemaNegotiator.java | 16 +++ .../impl/scan/framework/SchemaNegotiatorImpl.java | 16 +++ .../impl/scan/framework/ShimBatchReader.java | 3 +- .../scan/project/ExplicitSchemaProjection.java | 21 ++-- .../scan/project/ReaderSchemaOrchestrator.java | 8 ++ .../impl/scan/project/ScanLevelProjection.java | 106 --- .../impl/scan/project/ScanSchemaOrchestrator.java | 42 ++-- .../exec/physical/rowSet/ResultSetLoader.java | 8 ++ .../exec/physical/rowSet/impl/ColumnBuilder.java | 33 ++ .../exec/physical/rowSet/impl/OptionBuilder.java | 16 ++- .../physical/rowSet/impl/ResultSetLoaderImpl.java | 13 ++- .../rowSet/impl/SchemaTransformerImpl.java | 6 +- .../exec/store/easy/text/TextFormatPlugin.java | 25 - .../compliant/v3/CompliantTextBatchReader.java | 18 ++-- .../exec/physical/impl/scan/TestColumnsArray.java | 41 +++- .../impl/scan/TestColumnsArrayFramework.java | 9 +- .../physical/impl/scan/TestColumnsArrayParser.java | 40 .../impl/scan/TestFileMetadataColumnParser.java| 35 +++ .../impl/scan/TestFileMetadataProjection.java | 12 +-- .../physical/impl/scan/TestFileScanFramework.java | 5 +- .../scan/project/TestReaderLevelProjection.java| 28 ++--- .../impl/scan/project/TestScanLevelProjection.java | 22 ++-- .../impl/scan/project/TestSchemaSmoothing.java | 30 +++--- .../exec/physical/rowSet/impl/RowSetTestUtils.java | 2 +- .../impl/TestResultSetLoaderTypeConversion.java| 3 +- .../easy/text/compliant/TestCsvWithHeaders.java| 113 - .../easy/text/compliant/TestCsvWithoutHeaders.java | 60 +-- .../drill/exec/record/metadata/ProjectionType.java | 17 35 files changed, 783 insertions(+), 195 deletions(-) diff --git a/common/src/main/java/org/apache/drill/common/exceptions/ChildErrorContext.java b/common/src/main/java/org/apache/drill/common/exceptions/ChildErrorContext.java new file mode 100644 index 000..eb9bcc7 --- /dev/null +++ b/common/src/main/java/org/apache/drill/common/exceptions/ChildErrorContext.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.common.exceptions; +
[drill] branch master updated: DRILL-7257: Set nullable var-width vector lastSet value
This is an automated email from the ASF dual-hosted git repository.
arina pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git
The following commit(s) were added to refs/heads/master by this push:
new 2c1a8a0 DRILL-7257: Set nullable var-width vector lastSet value
2c1a8a0 is described below
commit 2c1a8a0afa3a1d8853e7651f1b3d47d52969f50a
Author: Paul Rogers
AuthorDate: Thu May 16 14:54:39 2019 -0700
DRILL-7257: Set nullable var-width vector lastSet value
Turns out this is due to a subtle issue with variable-width nullable
vectors. Such vectors have a lastSet attribute in the Mutator class.
When using "transfer pairs" to copy values, the code somehow decides
to zero-fill from the lastSet value to the record count. The row set
framework did not set this value, meaning that the RemovingRecordBatch
zero-filled the dir0 column when it chose to use transfer pairs rather
than copying values. The use of transfer pairs occurs when all rows in
a batch pass the filter prior to the removing record batch.
Modified the nullable vector writer to properly set the lastSet value at
the end of each batch. Added a unit test to verify the value is set
correctly.
Includes a bit of code clean-up.
---
.../physical/impl/svremover/AbstractSV2Copier.java | 19 +++-
.../physical/impl/svremover/GenericSV2Copier.java | 8 +++
.../apache/drill/exec/record/VectorAccessible.java | 2 +-
.../apache/drill/exec/record/VectorContainer.java | 5 ++---
.../exec/record/selection/SelectionVector2.java| 10 -
.../test/rowSet/test/TestScalarAccessors.java | 13 +++
.../codegen/templates/NullableValueVectors.java| 25 --
.../apache/drill/exec/vector/NullableVector.java | 1 +
.../accessor/writer/NullableScalarWriter.java | 3 +++
9 files changed, 46 insertions(+), 40 deletions(-)
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/svremover/AbstractSV2Copier.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/svremover/AbstractSV2Copier.java
index d273fd3..ed33703 100644
---
a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/svremover/AbstractSV2Copier.java
+++
b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/svremover/AbstractSV2Copier.java
@@ -17,6 +17,9 @@
*/
package org.apache.drill.exec.physical.impl.svremover;
+import java.util.ArrayList;
+import java.util.List;
+
import org.apache.drill.exec.record.TransferPair;
import org.apache.drill.exec.record.VectorAccessible;
import org.apache.drill.exec.record.VectorContainer;
@@ -24,9 +27,6 @@ import org.apache.drill.exec.record.VectorWrapper;
import org.apache.drill.exec.record.selection.SelectionVector2;
import org.apache.drill.exec.vector.ValueVector;
-import java.util.ArrayList;
-import java.util.List;
-
public abstract class AbstractSV2Copier extends AbstractCopier {
protected ValueVector[] vvIn;
private SelectionVector2 sv2;
@@ -35,21 +35,18 @@ public abstract class AbstractSV2Copier extends
AbstractCopier {
@Override
public void setup(VectorAccessible incoming, VectorContainer outgoing) {
super.setup(incoming, outgoing);
-this.sv2 = incoming.getSelectionVector2();
+sv2 = incoming.getSelectionVector2();
final int count = outgoing.getNumberOfColumns();
vvIn = new ValueVector[count];
-{
- int index = 0;
-
- for (VectorWrapper vectorWrapper: incoming) {
-vvIn[index] = vectorWrapper.getValueVector();
-index++;
- }
+int index = 0;
+for (VectorWrapper vectorWrapper: incoming) {
+ vvIn[index++] = vectorWrapper.getValueVector();
}
}
+ @Override
public void copyEntryIndirect(int inIndex, int outIndex) {
copyEntry(sv2.getIndex(inIndex), outIndex);
}
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/svremover/GenericSV2Copier.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/svremover/GenericSV2Copier.java
index f607e8c..d2f8dac 100644
---
a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/svremover/GenericSV2Copier.java
+++
b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/svremover/GenericSV2Copier.java
@@ -18,7 +18,6 @@
package org.apache.drill.exec.physical.impl.svremover;
import org.apache.drill.exec.record.RecordBatch;
-import org.apache.drill.exec.record.TransferPair;
import org.apache.drill.exec.record.VectorContainer;
import org.apache.drill.exec.record.VectorWrapper;
import org.apache.drill.exec.vector.SchemaChangeCallBack;
@@ -28,15 +27,14 @@ public class GenericSV2Copier extends AbstractSV2Copier {
public GenericSV2Copier(RecordBatch incomingBatch, VectorContainer
outputContainer,
SchemaChangeCallBack callBack) {
for(VectorWrapper vv : inc
[drill] 02/06: DRILL-7157: Wrap SchemaParsingException into UserException when creating schema
This is an automated email from the ASF dual-hosted git repository.
arina pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git
commit 9ffd6d933cdd16e10a5818991dd80ab3c36cf88f
Author: Arina Ielchiieva
AuthorDate: Mon Apr 8 10:32:11 2019 +0300
DRILL-7157: Wrap SchemaParsingException into UserException when creating
schema
closes #1740
---
.../org/apache/drill/exec/planner/sql/handlers/SchemaHandler.java | 7 ++-
1 file changed, 6 insertions(+), 1 deletion(-)
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/SchemaHandler.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/SchemaHandler.java
index f1f7353..4e7c8ae 100644
---
a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/SchemaHandler.java
+++
b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/SchemaHandler.java
@@ -35,6 +35,7 @@ import
org.apache.drill.exec.record.metadata.schema.FsMetastoreSchemaProvider;
import org.apache.drill.exec.record.metadata.schema.PathSchemaProvider;
import org.apache.drill.exec.record.metadata.schema.SchemaContainer;
import org.apache.drill.exec.record.metadata.schema.SchemaProvider;
+import
org.apache.drill.exec.record.metadata.schema.parser.SchemaParsingException;
import org.apache.drill.exec.store.AbstractSchema;
import org.apache.drill.exec.store.StorageStrategy;
import org.apache.drill.exec.store.dfs.WorkspaceSchemaFactory;
@@ -133,7 +134,11 @@ public abstract class SchemaHandler extends
DefaultSqlHandler {
ExecConstants.PERSISTENT_TABLE_UMASK).string_val, false);
schemaProvider.store(schemaString, sqlCall.getProperties(),
storageStrategy);
return DirectPlan.createDirectPlan(context, true,
String.format("Created schema for [%s]", schemaSource));
-
+ } catch (SchemaParsingException e) {
+throw UserException.parseError(e)
+ .message(e.getMessage())
+ .addContext("Schema: " + schemaString)
+ .build(logger);
} catch (IOException e) {
throw UserException.resourceError(e)
.message(e.getMessage())
[drill] 06/06: DRILL-7045: Updates to address review comments
This is an automated email from the ASF dual-hosted git repository.
arina pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git
commit 771fd270b684bbf388c0e2fa10b359eba3dfdb7c
Author: Sorabh Hamirwasia
AuthorDate: Tue Apr 2 13:24:59 2019 -0700
DRILL-7045: Updates to address review comments
closes #7134
---
.../drill/exec/vector/complex/writer/TestJsonNanInf.java | 11 +++
1 file changed, 3 insertions(+), 8 deletions(-)
diff --git
a/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestJsonNanInf.java
b/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestJsonNanInf.java
index 851ce0e..b7b3744 100644
---
a/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestJsonNanInf.java
+++
b/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestJsonNanInf.java
@@ -18,15 +18,15 @@
package org.apache.drill.exec.vector.complex.writer;
import org.apache.commons.io.FileUtils;
-import org.apache.drill.exec.physical.impl.join.JoinTestBase;
-import org.apache.drill.test.BaseTestQuery;
import org.apache.drill.common.exceptions.UserRemoteException;
import org.apache.drill.common.expression.SchemaPath;
import org.apache.drill.exec.ExecConstants;
+import org.apache.drill.exec.physical.impl.join.JoinTestBase;
import org.apache.drill.exec.record.RecordBatchLoader;
import org.apache.drill.exec.record.VectorWrapper;
import org.apache.drill.exec.rpc.user.QueryDataBatch;
import org.apache.drill.exec.vector.VarCharVector;
+import org.apache.drill.test.BaseTestQuery;
import org.junit.Ignore;
import org.junit.Test;
@@ -244,12 +244,7 @@ public class TestJsonNanInf extends BaseTestQuery {
builder.append(chunk);
}
String data = builder.toString();
-String query = String.format("select string_binary(binary_string('%s'))
from (values(1))", data);
-List results = testSqlWithResults(query);
-RecordBatchLoader batchLoader = new RecordBatchLoader(getAllocator());
-QueryDataBatch batch = results.get(0);
-batch.release();
-batchLoader.clear();
+test("select string_binary(binary_string('%s')) from (values(1))", data);
}
@Test
[drill] 05/06: DRILL-7045 UDF string_binary java.lang.IndexOutOfBoundsException
This is an automated email from the ASF dual-hosted git repository.
arina pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git
commit a1986a3fec1634812712e47be0be2565b303ea2d
Author: Jean-Claude
AuthorDate: Wed Feb 20 23:52:16 2019 -0500
DRILL-7045 UDF string_binary java.lang.IndexOutOfBoundsException
UDF string_binary was not reallocating the drillbuffer so it would fill
up and throw and out of bounds exception
---
.../apache/drill/exec/expr/fn/impl/StringFunctions.java | 6 +++---
.../drill/exec/vector/complex/writer/TestJsonNanInf.java | 16
2 files changed, 19 insertions(+), 3 deletions(-)
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/StringFunctions.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/StringFunctions.java
index 6353e55..70db585 100644
---
a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/StringFunctions.java
+++
b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/StringFunctions.java
@@ -1662,12 +1662,12 @@ public class StringFunctions{
@Override
public void eval() {
byte[] buf =
org.apache.drill.common.util.DrillStringUtils.toBinaryString(in.buffer,
in.start, in.end).getBytes(charset);
- buffer.setBytes(0, buf);
- buffer.setIndex(0, buf.length);
+ out.buffer = buffer.reallocIfNeeded(buf.length);
+ out.buffer.setBytes(0, buf);
+ out.buffer.setIndex(0, buf.length);
out.start = 0;
out.end = buf.length;
- out.buffer = buffer;
}
}
diff --git
a/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestJsonNanInf.java
b/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestJsonNanInf.java
index 138d9b2..851ce0e 100644
---
a/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestJsonNanInf.java
+++
b/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestJsonNanInf.java
@@ -237,6 +237,22 @@ public class TestJsonNanInf extends BaseTestQuery {
@Test
+ public void testLargeStringBinary() throws Exception {
+String chunk = "0123456789";
+StringBuilder builder = new StringBuilder();
+for (int i = 0; i < 1000; i++) {
+ builder.append(chunk);
+}
+String data = builder.toString();
+String query = String.format("select string_binary(binary_string('%s'))
from (values(1))", data);
+List results = testSqlWithResults(query);
+RecordBatchLoader batchLoader = new RecordBatchLoader(getAllocator());
+QueryDataBatch batch = results.get(0);
+batch.release();
+batchLoader.clear();
+ }
+
+ @Test
public void testConvertToJsonFunction() throws Exception {
String table = "nan_test.csv";
File file = new File(dirTestWatcher.getRootDir(), table);
[drill] branch master updated (ac11a6b -> 771fd27)
This is an automated email from the ASF dual-hosted git repository.
arina pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git.
from ac11a6b DRILL-7048: Implement JDBC Statement.setMaxRows() with System
Option
new cd44d1e DRILL-7143: Support default value for empty columns
new 9ffd6d9 DRILL-7157: Wrap SchemaParsingException into UserException
when creating schema
new 9844b61 DRILL-7049 return VARBINARY as a string with escaped non
printable bytes
new 9cbfaad DRILL-7049: REST API returns the toString of byte arrays
(VARBINARY types)
new a1986a3 DRILL-7045 UDF string_binary
java.lang.IndexOutOfBoundsException
new 771fd27 DRILL-7045: Updates to address review comments
The 6 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../apache/drill/common/types}/BooleanType.java| 49 +-
.../java/org/apache/drill/common/types/Types.java | 55 +-
.../exec/expr/fn/impl/SimpleCastFunctions.java | 4 +-
.../drill/exec/expr/fn/impl/StringFunctions.java | 6 +-
.../impl/scan/columns/ColumnsScanFramework.java| 5 -
.../impl/scan/project/NullColumnBuilder.java | 2 +-
.../impl/scan/project/NullColumnLoader.java| 58 +-
.../impl/scan/project/ResolvedNullColumn.java | 8 +-
.../impl/scan/project/ScanSchemaOrchestrator.java | 21 +-
.../exec/physical/rowSet/impl/ColumnBuilder.java | 3 +-
.../physical/rowSet/impl/ResultSetLoaderImpl.java | 23 +-
.../rowSet/impl/SchemaTransformerImpl.java | 90 ++-
.../exec/physical/rowSet/impl/TupleState.java | 4 +-
.../exec/planner/sql/handlers/SchemaHandler.java | 12 +-
.../record/metadata/AbstractColumnMetadata.java| 11 +-
.../drill/exec/record/metadata/MapBuilder.java | 12 +
.../drill/exec/record/metadata/MetadataUtils.java | 43 +-
.../record/metadata/PrimitiveColumnMetadata.java | 9 +-
.../drill/exec/record/metadata/SchemaBuilder.java | 10 +-
.../exec/store/dfs/easy/EasyFormatPlugin.java | 4 +
.../exec/store/easy/text/TextFormatPlugin.java | 7 +
.../exec/util/ValueVectorElementFormatter.java | 5 +
.../apache/drill/exec/work/foreman/Foreman.java| 9 +-
.../exec/fn/impl/TestByteComparisonFunctions.java | 1 -
.../impl/scan/project/TestNullColumnLoader.java| 29 +-
.../impl/TestResultSetLoaderOmittedValues.java | 43 ++
.../impl/TestResultSetLoaderTypeConversion.java| 61 +-
.../exec/record/metadata/TestTupleSchema.java | 3 +-
.../store/easy/text/compliant/BaseCsvTest.java | 8 +
.../easy/text/compliant/TestCsvWithSchema.java | 819 ++---
.../exec/util/TestValueVectorElementFormatter.java | 11 +
.../exec/vector/complex/writer/TestJsonNanInf.java | 15 +-
.../java/org/apache/drill/test/ClusterTest.java| 2 +-
.../apache/drill/test/rowSet/RowSetBuilder.java| 1 +
.../apache/drill/test/rowSet/RowSetUtilities.java | 20 +
.../test/rowSet/test/TestColumnConverter.java | 188 -
.../{DummyWriterTest.java => TestDummyWriter.java} | 2 +-
.../drill/test/rowSet/test/TestFillEmpties.java| 233 +-
.../test/rowSet/test/TestScalarAccessors.java | 111 ++-
.../drill/test/rowSet/test/TestSchemaBuilder.java | 108 ++-
.../org/apache/drill/exec/rpc/RequestIdMap.java| 25 +-
.../main/codegen/templates/ColumnAccessors.java| 256 ---
.../drill/exec/record/metadata/ColumnMetadata.java | 17 +-
.../drill/exec/vector/accessor/ScalarReader.java | 7 +
.../drill/exec/vector/accessor/ScalarWriter.java | 20 +
.../drill/exec/vector/accessor/ValueType.java | 6 +
.../convert/AbstractConvertFromString.java | 210 ++
.../accessor/convert/AbstractWriteConverter.java | 11 +
.../accessor/convert/ConvertBooleanToString.java | 6 +-
.../accessor/convert/ConvertStringToBoolean.java | 26 +-
.../accessor/convert/ConvertStringToDate.java | 25 +-
.../accessor/convert/ConvertStringToDecimal.java | 24 +-
.../accessor/convert/ConvertStringToDouble.java| 25 +-
.../accessor/convert/ConvertStringToInt.java | 25 +-
.../accessor/convert/ConvertStringToInterval.java | 24 +-
.../accessor/convert/ConvertStringToLong.java | 25 +-
.../accessor/convert/ConvertStringToTime.java | 25 +-
.../accessor/convert/ConvertStringToTimeStamp.java | 25 +-
.../accessor/convert/StandardConversions.java | 35 +-
.../exec/vector/accessor/impl/VectorPrinter.java | 7 +-
.../accessor/reader/AbstractScalarReader.java | 24 +
.../vector/accessor/reader/BitColumnReader.java} | 35 +-
.../accessor/writer/AbstractArrayWriter.java | 9 +-
.../accessor/writer/AbstractFixedWidthWriter.java | 75 +-
.../accessor/writer/Ab
